diff --git a/CHANGELOG.md b/CHANGELOG.md index aac5d416fb..015ab34043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,101 @@ # SQLCipher Change Log -All notable changes to this project will be documented in this file. +Notable changes to this project are documented in this file. + +## [4.13.0] - (January 2026 - [4.13.0 changes]) +- Updates baseline to SQLite 3.51.2 +- Corrects encoding for `sqlcipher_export()` function registration + +## [4.12.0] - (December 2025 - [4.12.0 changes]) +- Updates baseline to SQLite 3.51.1 +- Adds `PRAGMA cipher_status` so applications can verify a database handle is using encryption +- Improves guards against key/rekey/attach misuse +- Adds criteria for `PRAGMA cipher_migrate` tests +- Fixes check for `__has_feature` macro to separate it from use +- Fixes CHANGELOG.md markdown formatting, typos, and inline code snippets +- Fixes conditional in SQLCipher pragma handling +- Removes deprecated providers for LibTomCrypt and NSS +- Removes unnecessary shutdown and URI config changes in core tests +- Ensures all test suite database handles are closed before delete + +## [4.11.0] - (October 2025 - [4.11.0 changes]) +- Converts log output to UTF-16 when writing to stdout or stderr on Windows +- Fixes scope issues to allow `--disable-amalgamation` to work properly +- Replaces fortuna seeding mechanism for libtomcrypt with `rng_get_bytes()` +- Removes CocoaPods support (`SQLCipher.podspec.json`) +- Fixes includes and macros to support non-amalgamated builds +- Fixes check for `__has_feature` to resolve issue with compilers that don't support it +- Corrects return value from `sqlcipher_fprintf` +- Fixes use of provider `free_ctx` +- Fixes some compiler warnings + +## [4.10.0] - (August 2025 - [4.10.0 changes]) +- Updates baseline to SQLite 3.50.4 +- Allows compile time override of default log level via `SQLCIPHER_LOG_LEVEL_DEFAULT` macro +- Fixes issue building with `-fsanitize=address` on macOS +- Fixes detection of CommonCrypto version on macOS +- Improves CommonCrypto version detection on iOS + +## [4.9.0] - (May 2025 - [4.9.0 changes]) +- Updates baseline to upstream SQLite 3.49.2 +- Removes use of static mutex in `sqlcipher_extra_shutdown()` + +## [4.8.0] - (April 2025 - [4.8.0 changes]) +- Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database +- Adds selective locking in critical sections of the library for shared cache connections (Note: use of shared cache is still strongly discouraged) +- Standardizes initial private heap size to 48KB to ensure `mlock` under constrained limits +- Removes changes to windows working set sizes +- Improvements to logging of memory stats and other cleanup + +## [4.7.0] - (March 2025 - [4.7.0 changes]) +- Updates baseline to upstream SQLite 3.49.1, including complete upstream SQLite refactoring of build system to use autosetup +- Significantly refactors and optimizes library initialization and cleanup +- Allocates majority of requisite memory at startup to improve memory locking on constrained platforms (i.e. Android and Windows) and reduce fragmentation +- Expands `sqlcipher_provider` interface to include `init` and `shutdown` functions +- Adds support for `.recover` shell command on corrupt databases with a full plaintext first page +- Performs fast random overwrite of freed memory segments for improved security +- Adds basic obfuscation of context key material for improved security +- Generates keyspecs dynamically on demand instead of storing them +- Expands keyspec/raw key format to accept key, HMAC key, and salt +- Improves error handling in `sqlcipher_export()` and `PRAGMA cipher_migrate` +- Allows setting custom compile-time default cryptographic provider via the `SQLCIPHER_CRYPTO_CUSTOM` macro +- Removes support for end-of-life OpenSSL versions older than 3.0 +- __BREAKING CHANGE__: `SELECT` statements (now also including schema independent queries like `SELECT 1`) cannot be executed on encrypted databases prior to setting the database key (behavior inherited from upstream SQLite) +- __BREAKING CHANGE__: Renames `configure` flag `--enable-tempstore=yes` to `--with-tempstore=yes` for alignment with SQLite (change required for upstream SQLite autosetup) +- __BREAKING CHANGE__: Renames default executable and library build outputs from `sqlcipher` and `libsqlcipher` to `sqlite3` and `libsqlite3` (for alignment with SQLite) +- __BREAKING CHANGE__: Removes `configure` flag `--with-crypto-lib` (replace with appropriate `-DSQLCIPHER_CRYPTO_*` CFLAG) +- __BREAKING CHANGE__: Requires defining `SQLITE_EXTRA_INIT=sqlcipher_extra_init` and `SQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown` at compile time for optimized library initialization and cleanup +- __BREAKING CHANGE__: Enforces thread safe mode (i.e. `SQLITE_THREADSAFE` of 1 or 2) and temporary storage (i.e. `SQLITE_TEMP_STORE` of 2 or 3) settings at compile time + +## [4.6.1] - (August 2024 - [4.6.1 changes]) +- Updates baseline to upstream SQLite 3.46.1 +- Significant refactor to merge `crypto.h`, `crypto.c`, and `crypto_impl.c` into a single `sqlcipher.c` source file for simplicity. +- Updates minimum working set size on windows to increase lockable pages +- Adds new `PRAGMA cipher_log_source` for filtering log output on higher verbosity levels +- Improves log output by including the log level and source prior to message +- Improves error logging in `PRAGMA cipher_migrate` +- Fixes issue where log level and target would be overwritten if set prior to initialization +- Corrects Podspec license element to use specific BSD 3 Clause +- Fixes default log output to console for macOS + +## [4.6.0] - (May 2024 - [4.6.0 changes]) +- Sets default log level to WARN +- Sends default log output to: logcat for Android; Console for iOS and macOS; and stderr for all other platforms +- General improvements to log level assignments, output, and sanitization +- Fixes Apple Privacy Manifest by removing empty NSPrivacyCollectedDataType from PrivacyInfo.xcprivacy +- Moves Swift support defines for podspec user_target_xcconfig so they only apply to the consuming project + +## [4.5.7] - (April 2024 - [4.5.7 changes]) +- Updates baseline to upstream SQLite 3.45.3 +- Adds "device" logging and profile target using `os_log` for Apple (and logcat on Android) +- Fixes issues compiling with `SQLITE_OMIT_LOG` +- Fixes malformed man page caused by old merge conflict +- Updates podspec for current Xcode versions, improved Swift support, and Privacy Manifest + +## [4.5.6] - (January 2024 - [4.5.6 changes]) +- Updates baseline to upstream SQLite 3.44.2 +- Improves `PRAGMA cipher_integrity_check` to report expected page size if invalid +- Implements `PRAGMA page_size` compatibility with `PRAGMA cipher_page_size` so both will operate properly on encrypted databases +- Updates `LICENSE.md` with SQLCipher license to avoid ambiguity and remove redundancy ## [4.5.5] - (August 2023 - [4.5.5 changes]) - Updates baseline to upstream SQLite 3.42.0 @@ -12,7 +108,7 @@ All notable changes to this project will be documented in this file. ## [4.5.4] - (April 2023 - [4.5.4 changes]) - Updates baseline to upstream SQLite 3.41.2 - Updates minimum Apple SDK versions in podspec for new Xcode compatibility -- Return runtime OpenSSL version from PRAGMA cipher_provider_version (instead of hardcoded value) +- Return runtime OpenSSL version from `PRAGMA cipher_provider_version` (instead of hardcoded value) - Adds guard against zero block size and crash if cryptographic provider initialization fails - When an ATTACH occurs creating a new encrypted database as the first operation after keying the main database, the new database will have the same salt value. @@ -22,46 +118,46 @@ All notable changes to this project will be documented in this file. ## [4.5.2] - (August 2022 - [4.5.2 changes]) - Updates source code baseline to upstream SQLite 3.39.2 - Simplifies OpenSSL version conditional code -- Fixes issue where PRAGMA cipher_memory_security could report OFF when it was actually ON -- Fixes fix unfreed OpenSSL allocation when compiled against version 3 +- Fixes issue where `PRAGMA cipher_memory_security` could report OFF when it was actually ON +- Fixes unfreed OpenSSL allocation when compiled against version 3 - Fixes support for building against recent versions of BoringSSL ## [4.5.1] - (March 2022 - [4.5.1 changes]) - Updates source code baseline to upstream SQLite 3.37.2 -- Adds PRAGMA cipher_log and cipher_log_level features to allow logging of TRACE, DEBUG, INFO, WARN, and ERROR messages to stdout, stderr, file, or logcat -- Modifies PRAGMA cipher_profile to use sqlite3_trace_v2 and adds logcat target for Android -- Updates OpenSSL provider to use EVP_MAC API with version 3+ -- Adds new PRAGMA cipher_test_on, cipher_test_off, and cipher_test_rand (available when compiled with -DSQLCIPHER_TEST) to facilitate simulation of error conditions -- Fixes PRAGMA cipher_integrity_check to work properly with databases larger that 2GB -- Fixes missing munlock before free for context internal buffer (thanks to Fedor Indutny) +- Adds `PRAGMA cipher_log` and `PRAGMA cipher_log_level` features to allow logging of TRACE, DEBUG, INFO, WARN, and ERROR messages to stdout, stderr, file, or logcat +- Modifies `PRAGMA cipher_profile` to use `sqlite3_trace_v2` and adds logcat target for Android +- Updates OpenSSL provider to use `EVP_MAC` API with version 3+ +- Adds new `PRAGMA cipher_test_on`, `PRAGMA cipher_test_off`, and `PRAGMA cipher_test_rand` (available when compiled with `-DSQLCIPHER_TEST`) to facilitate simulation of error conditions +- Fixes `PRAGMA cipher_integrity_check` to work properly with databases larger that 2GB +- Fixes missing `munlock` before free for context internal buffer (thanks to Fedor Indutny) ## [4.5.0] - (October 2021 - [4.5.0 changes]) - Updates baseline to upstream SQLite 3.36.0 -- Changes the enhanced memory security feature to be DISABLED by default; once enabled by PRAGMA cipher_memory_security = ON, it can't be turned off for the lifetime of the process -- Changes PRAGMA cipher_migrate to permanently enter an error state if a migration fails +- Changes the enhanced memory security feature to be DISABLED by default; once enabled by `PRAGMA cipher_memory_security = ON`, it can't be turned off for the lifetime of the process +- Changes `PRAGMA cipher_migrate` to permanently enter an error state if a migration fails - Fixes memory locking/unlocking issue with realloc implementation on hardened runtimes when memory security is enabled -- Fixes cipher_migrate to cleanup the temporary database if a migration fails +- Fixes `PRAGMA cipher_migrate` to clean up the temporary database if a migration fails - Removes logging of non-string pointers when compiling with trace level logging ## [4.4.3] - (February 2021 - [4.4.3 changes]) -- Updates baseline to ustream SQLite 3.34.1 -- Fixes sqlcipher_export handling of NULL parameters +- Updates baseline to upstream SQLite 3.34.1 +- Fixes `sqlcipher_export` handling of NULL parameters - Removes randomization of rekey-delete tests to avoid false test failures -- Changes internal usage of sqlite_master to sqlite_schema -- Omits unusued profiling function under certain defines to avoid compiler warnings +- Changes internal usage of `sqlite_master` to `sqlite_schema` +- Omits unused profiling function under certain defines to avoid compiler warnings ## [4.4.2] - (November 2020 - [4.4.2 changes]) -- Improve error handling to resolve potential corruption if an encryption operation failed while operating in WAL mode +- Improves error handling to resolve potential corruption if an encryption operation failed while operating in WAL mode - Changes to OpenSSL library cryptographic provider to reduce initialization complexity -- Adjust cipher_integrity_check to skip locking page to avoid a spurious error report for very large databases +- Adjust `cipher_integrity_check` to skip locking page to avoid a spurious error report for very large databases - Miscellaneous code and comment cleanup ## [4.4.1] - (October 2020 - [4.4.1 changes]) - Updates baseline to upstream SQLite 3.33.0 -- Fixes double-free bug in cipher_default_plaintext_header_size +- Fixes double-free bug in `cipher_default_plaintext_header_size` - Changes SQLCipher tests to use suite runner -- Improvement to cipher_integrity_check tests to minimize false negatives -- Deprecates PRAGMA cipher_store_pass +- Improvement to `cipher_integrity_check` tests to minimize false negatives +- Deprecates `PRAGMA cipher_store_pass` ## [4.4.0] - (May 2020 - [4.4.0 changes]) - Updates baseline to upstream SQLite 3.31.0 @@ -71,40 +167,40 @@ All notable changes to this project will be documented in this file. ## [4.3.0] - (November 2019 - [4.3.0 changes]) - Updates baseline to upstream SQLite 3.30.1 -- PRAGMA key now returns text result value "ok" after execution +- `PRAGMA key` now returns text result value "ok" after execution - Adjusts backup API so that encrypted to encrypted backups are permitted - Adds NSS crypto provider implementation - Fixes OpenSSL provider compatibility with BoringSSL - Separates memory related traces to reduce verbosity of logging -- Fixes output of PRAGMA cipher_integrity_check on big endian platforms -- Cryptograpic provider interface cleanup +- Fixes output of `PRAGMA cipher_integrity_check` on big endian platforms +- Cryptographic provider interface cleanup - Rework of mutex allocation and management - Resolves miscellaneous build warnings - Force error state at database pager level if SQLCipher initialization fails ## [4.2.0] - (May 2019 - [4.2.0 changes]) -- Adds PRAGMA cipher_integrity_check to perform independent verification of page HMACs +- Adds `PRAGMA cipher_integrity_check` to perform independent verification of page HMACs - Updates baseline to upstream SQLite 3.28.0 -- Improves PRAGMA cipher_migrate to handle keys containing non-terminating zero bytes +- Improves `PRAGMA cipher_migrate` to handle keys containing non-terminating zero bytes ## [4.1.0] - (March 2019 - [4.1.0 changes]) - Defer reading salt from header until key derivation is triggered -- Clarify usage of sqlite3_rekey for plaintext databases in header +- Clarify usage of `sqlite3_rekey` for plaintext databases in header - Normalize attach behavior when key is not yet derived -- Adds PRAGMA cipher_settings to query current database codec settings -- Adds PRAGMA cipher_default_settings to query current default SQLCipher options -- PRAGMA cipher_hmac_pgno is now deprecated -- PRAGMA cipher_hmac_salt_mask is now deprecated -- PRAGMA fast_kdf_iter is now deprecated -- Improve sqlcipher_export routine and restore all database flags -- Clear codec data buffers if a crypographic provider operation fails +- Adds `PRAGMA cipher_settings` to query current database codec settings +- Adds `PRAGMA cipher_default_settings` to query current default SQLCipher options +- `PRAGMA cipher_hmac_pgno` is now deprecated +- `PRAGMA cipher_hmac_salt_mask` is now deprecated +- `PRAGMA fast_kdf_iter` is now deprecated +- Improve `sqlcipher_export` routine and restore all database flags +- Clear codec data buffers if a cryptographic provider operation fails - Disable backup API for encrypted databases (this was previously documented as not-working and non-supported, but will now explicitly error out on initialization) - Updates baseline to upstream SQLite 3.27.2 ## [4.0.1] - (December 2018 - [4.0.1 changes]) - Based on upstream SQLite 3.26.0 (addresses SQLite “Magellan” issue) -- Adds PRAGMA cipher_compatibility and cipher_default_compatibility which take automatcially configure appropriate compatibility settings for the specified SQLCipher major version number -- Filters attach statements with KEY parameters from readline history +- Adds `PRAGMA cipher_compatibility` and `PRAGMA cipher_default_compatibility` which automatically configure appropriate compatibility settings for the specified SQLCipher major version number +- Filters attach statements with `KEY` parameters from readline history - Fixes crash in command line shell with empty input (i.e. ^D) - Fixes warnings when compiled with strict-prototypes @@ -114,32 +210,32 @@ All notable changes to this project will be documented in this file. - Default PBKDF2 iterations increased to 256,000 (up from 64,000) * - Default KDF algorithm is now PBKDF2-HMAC-SHA512 (from PBKDF2-HMAC-SHA1) * - Default HMAC algorithm is now HMAC-SHA512 (from HMAC-SHA1) * -- PRAGMA cipher is now disabled and no longer supported (after multi-year deprecation) * -- PRAGMA rekey_cipher is now disabled and no longer supported * -- PRAGMA rekey_kdf_iter is now disabled and no longer supported * -- By default all memory allocated internally by SQLite before the memory is wiped before it is freed -- PRAGMA cipher_memory_security: allows full memory wiping to be disabled for performance when the feature is not required -- PRAGMA cipher_kdf_algorithm, cipher_default_kdf_algorithm to control KDF algorithm selection between PBKDF2-HMAC-SHA1, PBKDF2-HMAC-SHA256 and PBKDF2-HMAC-SHA512 -- PRAGMA cipher_hmac_algorithm, cipher_default_hmac_algorithm to control HMAC algorithm selection between HMAC-SHA1, HMAC-SHA256 and PBKDF2-HMAC-SHA512 +- `PRAGMA cipher` is now disabled and no longer supported (after multi-year deprecation) * +- `PRAGMA rekey_cipher` is now disabled and no longer supported * +- `PRAGMA rekey_kdf_iter` is now disabled and no longer supported * +- By default all memory allocated internally by SQLite is wiped before it is freed +- `PRAGMA cipher_memory_security`: allows full memory wiping to be disabled for performance when the feature is not required +- `PRAGMA cipher_kdf_algorithm`, `PRAGMA cipher_default_kdf_algorithm` to control KDF algorithm selection between PBKDF2-HMAC-SHA1, PBKDF2-HMAC-SHA256 and PBKDF2-HMAC-SHA512 +- `PRAGMA cipher_hmac_algorithm`, `PRAGMA cipher_default_hmac_algorithm` to control HMAC algorithm selection between HMAC-SHA1, HMAC-SHA256 and PBKDF2-HMAC-SHA512 - Based on upstream SQLite 3.25.2 -- When compiled with readline support, PRAGMA key and rekey lines will no longer be +- When compiled with readline support, `PRAGMA key` and `PRAGMA rekey` lines will no longer be saved to history -- Adds second optional parameter to sqlcipher_export to specify source database to +- Adds second optional parameter to `sqlcipher_export` to specify source database to support bidirectional exports - Fixes compatibility with LibreSSL 2.7.0+ - Fixes compatibility with OpenSSL 1.1.x -- Simplified and improved performance for PRAGMA cipher_migrate when migrating older database versions +- Simplified and improved performance for `PRAGMA cipher_migrate` when migrating older database versions - Refactoring of SQLCipher tests into separate files by test type -- PRAGMA cipher_plaintext_header_size and cipher_default_plaintext_header_size: allocates a portion of the database header which will not be encrypted to allow identification as a SQLite database -- PRAGMA cipher_salt: retrieve or set the salt value for the database +- `PRAGMA cipher_plaintext_header_size` and `PRAGMA cipher_default_plaintext_header_size`: allocates a portion of the database header which will not be encrypted to allow identification as a SQLite database +- `PRAGMA cipher_salt`: retrieve or set the salt value for the database - Adds Podspec for using tagged versions of SQLCipher -- Define SQLCIPHER_PROFILE_USE_FOPEN for WinXP support +- Define `SQLCIPHER_PROFILE_USE_FOPEN` for WinXP support - Improved error handling for cryptographic providers -- Improved memory handling for PRAGMA commands that return values +- Improved memory handling for `PRAGMA` commands that return values - Improved version reporting to assist with identification of distribution - Major rewrite and simplification of internal codec and pager extension -- Fixes compilation with --disable-amalgamation -- Removes sqlcipher.xcodeproj build support +- Fixes compilation with `--disable-amalgamation` +- Removes `sqlcipher.xcodeproj` build support ## [3.4.2] - (December 2017 - [3.4.2 changes]) ### Added @@ -151,7 +247,7 @@ All notable changes to this project will be documented in this file. - Remove static modifier for codec password functions - Page alignment for `mlock` - Fix segfault in `sqlcipher_cipher_ctx_cmp` during rekey operation -- Fix `sqlcipher_export` and `cipher_migrate` when tracing API in use +- Fix `sqlcipher_export` and `PRAGMA cipher_migrate` when tracing API in use - Validate codec page size when setting - Guard OpenSSL initialization and cleanup routines - Allow additional linker options to be passed via command line for Windows platforms @@ -194,7 +290,7 @@ All notable changes to this project will be documented in this file. ### Changed - Merged upstream SQLite 3.8.6 -- Renmed README to README.md +- Renamed `README` to `README.md` ## [3.1.0] - (April 2014 - [3.1.0 changes]) ### Added @@ -217,7 +313,7 @@ All notable changes to this project will be documented in this file. ### Changed - Merged upstream SQLite 3.8.0.2 -- Remove usage of VirtualLock/Unlock on WinRT and Windows Phone +- Remove usage of `VirtualLock/Unlock` on WinRT and Windows Phone - Ignore HMAC read during Btree file copy - Fix lib naming for pkg-config - Use _v2 version of `sqlite3_key` and `sqlite3_rekey` @@ -226,8 +322,30 @@ All notable changes to this project will be documented in this file. ### Security - Change KDF iteration length from 4,000 to 64,000 -[unreleased]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.5...prerelease -[4.5.5]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.4...v4.5.5 +[4.13.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.13.0 +[4.13.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.12.0...v4.13.0 +[4.12.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.12.0 +[4.12.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.11.0...v4.12.0 +[4.11.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.11.0 +[4.11.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.10.0...v4.11.0 +[4.10.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.10.0 +[4.10.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.9.0...v4.10.0 +[4.9.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.9.0 +[4.9.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.8.0...v4.9.0 +[4.8.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.8.0 +[4.8.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.0...v4.8.0 +[4.7.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.7.0 +[4.7.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.6.1...v4.7.0 +[4.6.1]: https://github.com/sqlcipher/sqlcipher/tree/v4.6.1 +[4.6.1 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.6.0...v4.6.1 +[4.6.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.6.0 +[4.6.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.7...v4.6.0 +[4.5.7]: https://github.com/sqlcipher/sqlcipher/tree/v4.5.7 +[4.5.7 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.6...v4.5.7 +[4.5.6]: https://github.com/sqlcipher/sqlcipher/tree/v4.5.6 +[4.5.6 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.5...v4.5.6 +[4.5.5]: https://github.com/sqlcipher/sqlcipher/tree/v4.5.5 +[4.5.5 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.4...v4.5.5 [4.5.4]: https://github.com/sqlcipher/sqlcipher/tree/v4.5.4 [4.5.4 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.5.3...v4.5.4 [4.5.3]: https://github.com/sqlcipher/sqlcipher/tree/v4.5.3 diff --git a/LICENSE.md b/LICENSE.md index f68a6c175f..3f71443161 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,24 @@ -The author disclaims copyright to this source code. In place of -a legal notice, here is a blessing: +Copyright (c) 2025, ZETETIC LLC +All rights reserved. - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the ZETETIC LLC nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENSE b/LICENSE.txt similarity index 97% rename from LICENSE rename to LICENSE.txt index 7167addca1..3f71443161 100644 --- a/LICENSE +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2008-2023, ZETETIC LLC +Copyright (c) 2025, ZETETIC LLC All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/Makefile.in b/Makefile.in index 097081a09e..a597a86a35 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1,130 +1,230 @@ -#!/usr/make +#!/usr/bin/make +# ^^^^ help out editors which guess this file's type. # # Makefile for SQLITE # -# This makefile is suppose to be configured automatically using the -# autoconf. But if that does not work for you, you can configure -# the makefile manually. Just set the parameters below to values that -# work well for your system. +# This makefile is intended to be configured automatically using the +# configure script. # -# If the configure script does not work out-of-the-box, you might -# be able to get it to work by giving it some hints. See the comment -# at the beginning of configure.in for additional information. +# The docs for many of its variables are in the primary static +# makefile, main.mk (which this one includes at runtime). # - -# The toplevel directory of the source tree. This is the directory -# that contains this "Makefile.in" and the "configure.in" script. +all: +######################################################################## +# Maintenance reminders: # -TOP = @abs_srcdir@ - - -# C Compiler and options for use in building executables that -# will run on the platform that is doing the build. +# - This makefile should remain as POSIX-make-compatible as possible: +# https://pubs.opengroup.org/onlinepubs/9799919799/utilities/make.html # -BCC = @BUILD_CC@ @BUILD_CFLAGS@ - -# TCC is the C Compile and options for use in building executables that -# will run on the target platform. (BCC and TCC are usually the -# same unless your are cross-compiling.) Separate CC and CFLAGS macros -# are provide so that these aspects of the build process can be changed -# on the "make" command-line. Ex: "make CC=clang CFLAGS=-fsanitize=undefined" +# - The naming convention of some vars, using periods instead of +# underscores, though unconventional, was selected for a couple of +# reasons: 1) Personal taste (for which there is no accounting). 2) +# It is thought to help defend against inadvertent injection of +# those vars via environment variables (because X.Y is not a legal +# environment variable name). "Feature or bug?" is debatable and +# this naming convention may be reverted if it causes any grief. # -CC = @CC@ -CFLAGS = @CPPFLAGS@ @CFLAGS@ -TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu -TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session -TCC += -I${TOP}/ext/userauth -# Define this for the autoconf-based build, so that the code knows it can -# include the generated sqlite_cfg.h # -TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite - -# Define -DNDEBUG to compile without debugging (i.e., for production usage) -# Omitting the define will cause extra debugging code to be inserted and -# includes extra comments when "EXPLAIN stmt" is used. +# The top-most directory of the source tree. This is the directory +# that contains this "Makefile.in" and the "configure" script. # -TCC += @TARGET_DEBUG@ +TOP = @abs_top_srcdir@ -# Compiler options needed for programs that use the TCL library. # -TCC += @TCL_INCLUDE_SPEC@ - -# The library that programs using TCL must link against. +# Autotools-conventional vars which are used by package installation +# rules in main.mk. To get sane handling when a user overrides only +# a subset of these, we perform some acrobatics with these vars +# in the configure script: see [proj-remap-autoconf-dir-vars] for +# full details. # -LIBTCL = @TCL_LIB_SPEC@ - -# Compiler options needed for programs that use the readline() library. +# For completeness's sake, the aforementioned conventional vars which +# are relevant to our installation rules are: # -READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@ -READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@ - -# The library that programs using readline() must link against. +# datadir = $(prefix)/share +# mandir = $(datadir)/man +# includedir = $(prefix)/include +# exec_prefix = $(prefix) +# bindir = $(exec_prefix)/bin +# libdir = $(exec_prefix)/lib # -LIBREADLINE = @TARGET_READLINE_LIBS@ - -# Should the database engine be compiled threadsafe +# Our builds do not require any of their relatives: # -TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@ +# sbindir = $(exec_prefix)/sbin +# sysconfdir = /etc +# sharedstatedir = $(prefix)/com +# localstatedir = /var +# runstatedir = /run +# infodir = $(datadir)/info +# libexecdir = $(exec_prefix)/libexec +# +prefix = @prefix@ +datadir = @datadir@ +mandir = @mandir@ +includedir = @includedir@ +exec_prefix = @exec_prefix@ +bindir = @bindir@ +libdir = @libdir@ -# Any target libraries which libsqlite must be linked against +INSTALL = @BIN_INSTALL@ +AR = @AR@ +AR.flags = cr +CC = @CC@ +B.cc = @CC_FOR_BUILD@ @BUILD_CFLAGS@ +T.cc = $(CC) +# +# $(CFLAGS) is problematic because it is frequently overridden when +# invoking make, which loses things like -fPIC. So... we avoid using +# it directly and instead add a level of indirection. We combine +# $(CFLAGS) and $(CPPFLAGS) here because that's the way the legacy +# build did it and many builds rely on that. See main.mk for more +# details. +# +# Historical note: the pre-3.48 build only honored CPPFLAGS at +# configure-time, and expanded them into the generated Makefile. There +# are, in that build, no uses of CPPFLAGS in the configure-expanded +# Makefile. Ergo: if a client configures with CPPFLAGS=... and then +# explicitly passes CFLAGS=... to make, the CPPFLAGS will be +# lost. That behavior is retained in 3.48+. # -TLIBS = @LIBS@ $(LIBS) +CFLAGS = @CFLAGS@ @CPPFLAGS@ +# +# $(LDFLAGS.configure) represents any LDFLAGS=... the client passes to +# configure. See main.mk. +# +LDFLAGS.configure = @LDFLAGS@ -# Flags controlling use of the in memory btree implementation # -# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to -# default to file, 2 to default to memory, and 3 to force temporary -# tables to always be in memory. +# CFLAGS.core is documented in main.mk. # -TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@ +CFLAGS.core = @SH_CFLAGS@ +LDFLAGS.shlib = @SH_LDFLAGS@ +LDFLAGS.zlib = @LDFLAGS_ZLIB@ +LDFLAGS.math = @LDFLAGS_MATH@ +LDFLAGS.rpath = @LDFLAGS_RPATH@ +LDFLAGS.pthread = @LDFLAGS_PTHREAD@ +LDFLAGS.dlopen = @LDFLAGS_DLOPEN@ +LDFLAGS.readline = @LDFLAGS_READLINE@ +CFLAGS.readline = @CFLAGS_READLINE@ +LDFLAGS.icu = @LDFLAGS_ICU@ +LDFLAGS.rt = @LDFLAGS_RT@ +CFLAGS.icu = @CFLAGS_ICU@ +LDFLAGS.libsqlite3.soname = @LDFLAGS_LIBSQLITE3_SONAME@ +# soname: see https://sqlite.org/src/forumpost/5a3b44f510df8ded +LDFLAGS.libsqlite3.os-specific = \ + @LDFLAGS_MAC_CVERSION@ @LDFLAGS_MAC_INSTALL_NAME@ @LDFLAGS_OUT_IMPLIB@ +# os-specific: see +# - https://sqlite.org/forum/forumpost/9dfd5b8fd525a5d7 +# - https://sqlite.org/forum/forumpost/0c7fc097b2 +# - https://sqlite.org/forum/forumpost/5651662b8875ec0a -# Enable/disable loadable extensions, and other optional features -# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). -# The same set of OMIT and ENABLE flags should be passed to the -# LEMON parser generator and the mkkeywordhash tool as well. -OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ +libsqlite3.DLL.basename = @SQLITE_DLL_BASENAME@ +# DLL.basename: see https://sqlite.org/forum/forumpost/828fdfe904 +libsqlite3.out.implib = @SQLITE_OUT_IMPLIB@ +# libsqlite3.out.implib => the output filename part of LDFLAGS_OUT_IMPLIB. +ENABLE_LIB_SHARED = @ENABLE_LIB_SHARED@ +ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ +HAVE_WASI_SDK = @HAVE_WASI_SDK@ +libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ -TCC += $(OPT_FEATURE_FLAGS) +# -fsanitize flags for the fuzzcheck-asap app +CFLAGS.fuzzcheck-asan.fsanitize = @CFLAGS_ASAN_FSANITIZE@ -# Add in any optional parameters specified on the make commane line -# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". -TCC += $(OPTS) +# +# Intended to either be empty or be set to -g -DSQLITE_DEBUG=1. +# +T.cc.TARGET_DEBUG = @TARGET_DEBUG@ -# Add in compile-time options for some libraries used by extensions -TCC += @HAVE_ZLIB@ +# +# $(JIMSH) and $(CFLAGS.jimsh) are documented in main.mk. $(JIMSH) +# must start with a path component so that it can be invoked as a +# shell command. +# +CFLAGS.jimsh = @CFLAGS_JIMSH@ +JIMSH = ./jimsh$(T.exe) + +# +# $(B.tclsh) is documented in main.mk. +# +B.tclsh = @BTCLSH@ +$(B.tclsh): + +# +# $(OPT_FEATURE_FLAGS) is documented in main.mk. +# +# The appending of $(OPTIONS) to $(OPT_FEATURE_FLAGS) is historical +# and somewhat confusing because there's another var, $(OPTS), which +# has a similar (but not identical) role. +# +OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ $(OPTIONS) -# Version numbers and release number for the SQLite being compiled. # -VERSION = @VERSION@ -VERSION_NUMBER = @VERSION_NUMBER@ -RELEASE = @RELEASE@ +# Version (X.Y.Z) number for the SQLite being compiled. +# +PACKAGE_VERSION = @PACKAGE_VERSION@ -# Filename extensions # -BEXE = @BUILD_EXEEXT@ -TEXE = @TARGET_EXEEXT@ +# Filename extensions for binaries and libraries +# +B.exe = @BUILD_EXEEXT@ +T.exe = @TARGET_EXEEXT@ +B.dll = @BUILD_DLLEXT@ +T.dll = @TARGET_DLLEXT@ +B.lib = @BUILD_LIBEXT@ +T.lib = @TARGET_LIBEXT@ -# The following variable is "1" if the configure script was able to locate -# the tclConfig.sh file. It is an empty string otherwise. When this -# variable is "1", the TCL extension library (libtclsqlite3.so) is built -# and installed. +# +# $(HAVE_TCL) is 1 if the configure script was able to locate the +# tclConfig.sh file, else it is 0. When this variable is 1, the TCL +# extension library (libtclsqlite3.so) and related testing apps are +# built. # HAVE_TCL = @HAVE_TCL@ -# This is the command to use for tclsh - normally just "tclsh", but we may -# know the specific version we want to use +# +# $(TCLSH_CMD) is the command to use for tclsh - normally just +# "tclsh", but we may know the specific version we want to use. This +# must point to a canonical TCL interpreter, not JimTCL. # TCLSH_CMD = @TCLSH_CMD@ +TCL_CONFIG_SH = @TCL_CONFIG_SH@ -# Where do we want to install the tcl plugin +# +# TCL config info from tclConfig.sh +# +# We have to inject this differently in main.mk to accommodate static +# makefiles, so we don't currently bother to export it here. This +# block is retained in case we decide that we do indeed need to export +# it at configure-time instead of calculate it at make-time. +# +#TCL_INCLUDE_SPEC = @TCL_INCLUDE_SPEC@ +#TCL_LIB_SPEC = @TCL_LIB_SPEC@ +#TCL_STUB_LIB_SPEC = @TCL_STUB_LIB_SPEC@ +#TCL_EXEC_PREFIX = @TCL_EXEC_PREFIX@ +#TCL_VERSION = @TCL_VERSION@ +TCL_MAJOR_VERSION = @TCL_MAJOR_VERSION@ +# ^^^ main.mk optionally uses this for determining the Tcl extension's +# DLL name. +TCL_EXT_DLL_BASENAME = @TCL_EXT_DLL_BASENAME@ +# ^^^ base name of the Tcl extension DLL. It varies by platform and +# Tcl version. + +# +# $(TCLLIBDIR) = where to install the tcl plugin. If this is empty, it +# is calculated at make-time by the targets which need it but we +# export it here so that it can be set at configure-time, so that +# clients are not required to pass it at make-time, or may set it in +# their environment to override it. # TCLLIBDIR = @TCLLIBDIR@ -# The suffix used on shared libraries. Ex: ".dll", ".so", ".dylib" # -SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ +# Additional options when running tests using testrunner.tcl +# This is usually either blank or --status. +# +TSTRNNR_OPTS = @TSTRNNR_OPTS@ +# # If gcov support was enabled by the configure script, add the appropriate # flags here. It's not always as easy as just having the user add the right # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which @@ -136,1453 +236,104 @@ SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ # # for more info. # -GCOV_CFLAGS1 = -DSQLITE_COVERAGE_TEST=1 -fprofile-arcs -ftest-coverage -GCOV_LDFLAGS1 = -lgcov +CFLAGS.gcov1 = -DSQLITE_COVERAGE_TEST=1 -fprofile-arcs -ftest-coverage +LDFLAGS.gcov1 = -lgcov USE_GCOV = @USE_GCOV@ -LTCOMPILE_EXTRAS += $(GCOV_CFLAGS$(USE_GCOV)) -LTLINK_EXTRAS += $(GCOV_LDFLAGS$(USE_GCOV)) - -# BEGIN CRYPTO -CRYPTOLIBOBJ = \ - crypto.lo \ - crypto_impl.lo \ - crypto_openssl.lo \ - crypto_libtomcrypt.lo \ - crypto_nss.lo \ - crypto_cc.lo - -CRYPTOSRC = \ - $(TOP)/src/crypto.h \ - $(TOP)/src/sqlcipher.h \ - $(TOP)/src/crypto.c \ - $(TOP)/src/crypto_impl.c \ - $(TOP)/src/crypto_libtomcrypt.c \ - $(TOP)/src/crypto_nss.c \ - $(TOP)/src/crypto_openssl.c \ - $(TOP)/src/crypto_cc.c - -# END CRYPTO +T.compile.gcov = $(CFLAGS.gcov$(USE_GCOV)) +T.link.gcov = $(LDFLAGS.gcov$(USE_GCOV)) -# The directory into which to store package information for - -# Some standard variables and programs # -prefix = @prefix@ -exec_prefix = @exec_prefix@ -libdir = @libdir@ -pkgconfigdir = $(libdir)/pkgconfig -bindir = @bindir@ -includedir = @includedir@/sqlcipher -INSTALL = @INSTALL@ -LIBTOOL = ./libtool -ALLOWRELEASE = @ALLOWRELEASE@ - -# libtool compile/link/install -LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS) -LTLINK = $(LIBTOOL) --mode=link $(TCC) $(LTCOMPILE_EXTRAS) @LDFLAGS@ $(LTLINK_EXTRAS) -LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL) - -# You should not have to change anything below this line -############################################################################### - -USE_AMALGAMATION = @USE_AMALGAMATION@ -AMALGAMATION_LINE_MACROS = @AMALGAMATION_LINE_MACROS@ - -# Object files for the SQLite library (non-amalgamation). -# -LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ - backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo \ - date.lo dbpage.lo dbstat.lo delete.lo \ - expr.lo fault.lo fkey.lo \ - fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ - fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ - fts3_tokenize_vtab.lo \ - fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ - fts5.lo \ - func.lo global.lo hash.lo \ - icu.lo insert.lo json.lo legacy.lo loadext.lo \ - main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ - memdb.lo memjournal.lo \ - mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ - notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \ - pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo \ - sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \ - table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ - update.lo userauth.lo upsert.lo util.lo vacuum.lo \ - vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ - vdbetrace.lo vdbevtab.lo \ - wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ - window.lo utf.lo vtab.lo $(CRYPTOLIBOBJ) - -# Object files for the amalgamation. +# Vars with the AS_ prefix are specifically related to AutoSetup. # -LIBOBJS1 = sqlite3.lo - -# Determine the real value of LIBOBJ based on the 'configure' script +# AS_AUTO_DEF is the main configure script. # -LIBOBJ = $(LIBOBJS$(USE_AMALGAMATION)) - - -# All of the source code files. -# -SRC = \ - $(CRYPTOSRC) \ - $(TOP)/src/alter.c \ - $(TOP)/src/analyze.c \ - $(TOP)/src/attach.c \ - $(TOP)/src/auth.c \ - $(TOP)/src/backup.c \ - $(TOP)/src/bitvec.c \ - $(TOP)/src/btmutex.c \ - $(TOP)/src/btree.c \ - $(TOP)/src/btree.h \ - $(TOP)/src/btreeInt.h \ - $(TOP)/src/build.c \ - $(TOP)/src/callback.c \ - $(TOP)/src/complete.c \ - $(TOP)/src/ctime.c \ - $(TOP)/src/date.c \ - $(TOP)/src/dbpage.c \ - $(TOP)/src/dbstat.c \ - $(TOP)/src/delete.c \ - $(TOP)/src/expr.c \ - $(TOP)/src/fault.c \ - $(TOP)/src/fkey.c \ - $(TOP)/src/func.c \ - $(TOP)/src/global.c \ - $(TOP)/src/hash.c \ - $(TOP)/src/hash.h \ - $(TOP)/src/hwtime.h \ - $(TOP)/src/insert.c \ - $(TOP)/src/json.c \ - $(TOP)/src/legacy.c \ - $(TOP)/src/loadext.c \ - $(TOP)/src/main.c \ - $(TOP)/src/malloc.c \ - $(TOP)/src/mem0.c \ - $(TOP)/src/mem1.c \ - $(TOP)/src/mem2.c \ - $(TOP)/src/mem3.c \ - $(TOP)/src/mem5.c \ - $(TOP)/src/memdb.c \ - $(TOP)/src/memjournal.c \ - $(TOP)/src/msvc.h \ - $(TOP)/src/mutex.c \ - $(TOP)/src/mutex.h \ - $(TOP)/src/mutex_noop.c \ - $(TOP)/src/mutex_unix.c \ - $(TOP)/src/mutex_w32.c \ - $(TOP)/src/notify.c \ - $(TOP)/src/os.c \ - $(TOP)/src/os.h \ - $(TOP)/src/os_common.h \ - $(TOP)/src/os_setup.h \ - $(TOP)/src/os_kv.c \ - $(TOP)/src/os_unix.c \ - $(TOP)/src/os_win.c \ - $(TOP)/src/os_win.h \ - $(TOP)/src/pager.c \ - $(TOP)/src/pager.h \ - $(TOP)/src/parse.y \ - $(TOP)/src/pcache.c \ - $(TOP)/src/pcache.h \ - $(TOP)/src/pcache1.c \ - $(TOP)/src/pragma.c \ - $(TOP)/src/pragma.h \ - $(TOP)/src/prepare.c \ - $(TOP)/src/printf.c \ - $(TOP)/src/random.c \ - $(TOP)/src/resolve.c \ - $(TOP)/src/rowset.c \ - $(TOP)/src/select.c \ - $(TOP)/src/status.c \ - $(TOP)/src/shell.c.in \ - $(TOP)/src/sqlite.h.in \ - $(TOP)/src/sqlite3ext.h \ - $(TOP)/src/sqliteInt.h \ - $(TOP)/src/sqliteLimit.h \ - $(TOP)/src/table.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/src/threads.c \ - $(TOP)/src/tokenize.c \ - $(TOP)/src/treeview.c \ - $(TOP)/src/trigger.c \ - $(TOP)/src/utf.c \ - $(TOP)/src/update.c \ - $(TOP)/src/upsert.c \ - $(TOP)/src/util.c \ - $(TOP)/src/vacuum.c \ - $(TOP)/src/vdbe.c \ - $(TOP)/src/vdbe.h \ - $(TOP)/src/vdbeapi.c \ - $(TOP)/src/vdbeaux.c \ - $(TOP)/src/vdbeblob.c \ - $(TOP)/src/vdbemem.c \ - $(TOP)/src/vdbesort.c \ - $(TOP)/src/vdbetrace.c \ - $(TOP)/src/vdbevtab.c \ - $(TOP)/src/vdbeInt.h \ - $(TOP)/src/vtab.c \ - $(TOP)/src/vxworks.h \ - $(TOP)/src/wal.c \ - $(TOP)/src/wal.h \ - $(TOP)/src/walker.c \ - $(TOP)/src/where.c \ - $(TOP)/src/wherecode.c \ - $(TOP)/src/whereexpr.c \ - $(TOP)/src/whereInt.h \ - $(TOP)/src/window.c - -# Source code for extensions -# -SRC += \ - $(TOP)/ext/fts3/fts3.c \ - $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3Int.h \ - $(TOP)/ext/fts3/fts3_aux.c \ - $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_hash.c \ - $(TOP)/ext/fts3/fts3_hash.h \ - $(TOP)/ext/fts3/fts3_icu.c \ - $(TOP)/ext/fts3/fts3_porter.c \ - $(TOP)/ext/fts3/fts3_snippet.c \ - $(TOP)/ext/fts3/fts3_tokenizer.h \ - $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_tokenizer1.c \ - $(TOP)/ext/fts3/fts3_tokenize_vtab.c \ - $(TOP)/ext/fts3/fts3_unicode.c \ - $(TOP)/ext/fts3/fts3_unicode2.c \ - $(TOP)/ext/fts3/fts3_write.c -SRC += \ - $(TOP)/ext/icu/sqliteicu.h \ - $(TOP)/ext/icu/icu.c -SRC += \ - $(TOP)/ext/rtree/rtree.h \ - $(TOP)/ext/rtree/rtree.c \ - $(TOP)/ext/rtree/geopoly.c -SRC += \ - $(TOP)/ext/session/sqlite3session.c \ - $(TOP)/ext/session/sqlite3session.h -SRC += \ - $(TOP)/ext/userauth/userauth.c \ - $(TOP)/ext/userauth/sqlite3userauth.h -SRC += \ - $(TOP)/ext/rbu/sqlite3rbu.h \ - $(TOP)/ext/rbu/sqlite3rbu.c -SRC += \ - $(TOP)/ext/misc/stmt.c - -# Generated source code files -# -SRC += \ - keywordhash.h \ - opcodes.c \ - opcodes.h \ - parse.c \ - parse.h \ - sqlite_cfg.h \ - shell.c \ - sqlite3.h - -# Source code to the test files. -# -TESTSRC = \ - $(TOP)/src/test1.c \ - $(TOP)/src/test2.c \ - $(TOP)/src/test3.c \ - $(TOP)/src/test4.c \ - $(TOP)/src/test5.c \ - $(TOP)/src/test6.c \ - $(TOP)/src/test8.c \ - $(TOP)/src/test9.c \ - $(TOP)/src/test_autoext.c \ - $(TOP)/src/test_async.c \ - $(TOP)/src/test_backup.c \ - $(TOP)/src/test_bestindex.c \ - $(TOP)/src/test_blob.c \ - $(TOP)/src/test_btree.c \ - $(TOP)/src/test_config.c \ - $(TOP)/src/test_delete.c \ - $(TOP)/src/test_demovfs.c \ - $(TOP)/src/test_devsym.c \ - $(TOP)/src/test_fs.c \ - $(TOP)/src/test_func.c \ - $(TOP)/src/test_hexio.c \ - $(TOP)/src/test_init.c \ - $(TOP)/src/test_intarray.c \ - $(TOP)/src/test_journal.c \ - $(TOP)/src/test_malloc.c \ - $(TOP)/src/test_md5.c \ - $(TOP)/src/test_multiplex.c \ - $(TOP)/src/test_mutex.c \ - $(TOP)/src/test_onefile.c \ - $(TOP)/src/test_osinst.c \ - $(TOP)/src/test_pcache.c \ - $(TOP)/src/test_quota.c \ - $(TOP)/src/test_rtree.c \ - $(TOP)/src/test_schema.c \ - $(TOP)/src/test_superlock.c \ - $(TOP)/src/test_syscall.c \ - $(TOP)/src/test_tclsh.c \ - $(TOP)/src/test_tclvar.c \ - $(TOP)/src/test_thread.c \ - $(TOP)/src/test_vdbecov.c \ - $(TOP)/src/test_vfs.c \ - $(TOP)/src/test_windirent.c \ - $(TOP)/src/test_window.c \ - $(TOP)/src/test_wsd.c \ - $(TOP)/ext/fts3/fts3_term.c \ - $(TOP)/ext/fts3/fts3_test.c \ - $(TOP)/ext/session/test_session.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/test_recover.c \ - $(TOP)/ext/rbu/test_rbu.c - -# Statically linked extensions -# -TESTSRC += \ - $(TOP)/ext/expert/sqlite3expert.c \ - $(TOP)/ext/expert/test_expert.c \ - $(TOP)/ext/misc/amatch.c \ - $(TOP)/ext/misc/appendvfs.c \ - $(TOP)/ext/misc/basexx.c \ - $(TOP)/ext/misc/carray.c \ - $(TOP)/ext/misc/cksumvfs.c \ - $(TOP)/ext/misc/closure.c \ - $(TOP)/ext/misc/csv.c \ - $(TOP)/ext/misc/decimal.c \ - $(TOP)/ext/misc/eval.c \ - $(TOP)/ext/misc/explain.c \ - $(TOP)/ext/misc/fileio.c \ - $(TOP)/ext/misc/fuzzer.c \ - $(TOP)/ext/fts5/fts5_tcl.c \ - $(TOP)/ext/fts5/fts5_test_mi.c \ - $(TOP)/ext/fts5/fts5_test_tok.c \ - $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/mmapwarm.c \ - $(TOP)/ext/misc/nextchar.c \ - $(TOP)/ext/misc/normalize.c \ - $(TOP)/ext/misc/percentile.c \ - $(TOP)/ext/misc/prefixes.c \ - $(TOP)/ext/misc/qpvtab.c \ - $(TOP)/ext/misc/regexp.c \ - $(TOP)/ext/misc/remember.c \ - $(TOP)/ext/misc/series.c \ - $(TOP)/ext/misc/spellfix.c \ - $(TOP)/ext/misc/totype.c \ - $(TOP)/ext/misc/unionvtab.c \ - $(TOP)/ext/misc/wholenumber.c \ - $(TOP)/ext/misc/zipfile.c \ - $(TOP)/ext/userauth/userauth.c \ - $(TOP)/ext/rtree/test_rtreedoc.c - -# Source code to the library files needed by the test fixture -# -TESTSRC2 = \ - $(TOP)/src/attach.c \ - $(TOP)/src/backup.c \ - $(TOP)/src/bitvec.c \ - $(TOP)/src/btree.c \ - $(TOP)/src/build.c \ - $(TOP)/src/ctime.c \ - $(TOP)/src/date.c \ - $(TOP)/src/dbpage.c \ - $(TOP)/src/dbstat.c \ - $(TOP)/src/expr.c \ - $(TOP)/src/func.c \ - $(TOP)/src/global.c \ - $(TOP)/src/insert.c \ - $(TOP)/src/wal.c \ - $(TOP)/src/main.c \ - $(TOP)/src/mem5.c \ - $(TOP)/src/os.c \ - $(TOP)/src/os_kv.c \ - $(TOP)/src/os_unix.c \ - $(TOP)/src/os_win.c \ - $(TOP)/src/pager.c \ - $(TOP)/src/pragma.c \ - $(TOP)/src/prepare.c \ - $(TOP)/src/printf.c \ - $(TOP)/src/random.c \ - $(TOP)/src/pcache.c \ - $(TOP)/src/pcache1.c \ - $(TOP)/src/select.c \ - $(TOP)/src/tokenize.c \ - $(TOP)/src/treeview.c \ - $(TOP)/src/utf.c \ - $(TOP)/src/util.c \ - $(TOP)/src/vdbeapi.c \ - $(TOP)/src/vdbeaux.c \ - $(TOP)/src/vdbe.c \ - $(TOP)/src/vdbemem.c \ - $(TOP)/src/vdbetrace.c \ - $(TOP)/src/vdbevtab.c \ - $(TOP)/src/where.c \ - $(TOP)/src/wherecode.c \ - $(TOP)/src/whereexpr.c \ - $(TOP)/src/window.c \ - parse.c \ - $(TOP)/ext/fts3/fts3.c \ - $(TOP)/ext/fts3/fts3_aux.c \ - $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_term.c \ - $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c \ - $(TOP)/ext/session/sqlite3session.c \ - $(TOP)/ext/misc/stmt.c \ - fts5.c - -# Header files used by all library source files. -# -HDR = \ - $(TOP)/src/btree.h \ - $(TOP)/src/btreeInt.h \ - $(TOP)/src/hash.h \ - $(TOP)/src/hwtime.h \ - keywordhash.h \ - $(TOP)/src/msvc.h \ - $(TOP)/src/mutex.h \ - opcodes.h \ - $(TOP)/src/os.h \ - $(TOP)/src/os_common.h \ - $(TOP)/src/os_setup.h \ - $(TOP)/src/os_win.h \ - $(TOP)/src/pager.h \ - $(TOP)/src/pcache.h \ - parse.h \ - $(TOP)/src/pragma.h \ - sqlite3.h \ - $(TOP)/src/sqlite3ext.h \ - $(TOP)/src/sqliteInt.h \ - $(TOP)/src/sqliteLimit.h \ - $(TOP)/src/vdbe.h \ - $(TOP)/src/vdbeInt.h \ - $(TOP)/src/vxworks.h \ - $(TOP)/src/whereInt.h \ - sqlite_cfg.h - -# Header files used by extensions -# -EXTHDR += \ - $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3Int.h \ - $(TOP)/ext/fts3/fts3_hash.h \ - $(TOP)/ext/fts3/fts3_tokenizer.h -EXTHDR += \ - $(TOP)/ext/rtree/rtree.h \ - $(TOP)/ext/rtree/geopoly.c -EXTHDR += \ - $(TOP)/ext/icu/sqliteicu.h -EXTHDR += \ - $(TOP)/ext/rtree/sqlite3rtree.h -EXTHDR += \ - $(TOP)/ext/userauth/sqlite3userauth.h - -# executables needed for testing -# -TESTPROGS = \ - testfixture$(TEXE) \ - sqlite3$(TEXE) \ - sqlite3_analyzer$(TEXE) \ - sqldiff$(TEXE) \ - dbhash$(TEXE) \ - sqltclsh$(TEXE) - -# Databases containing fuzzer test cases -# -FUZZDATA = \ - $(TOP)/test/fuzzdata1.db \ - $(TOP)/test/fuzzdata2.db \ - $(TOP)/test/fuzzdata3.db \ - $(TOP)/test/fuzzdata4.db \ - $(TOP)/test/fuzzdata5.db \ - $(TOP)/test/fuzzdata6.db \ - $(TOP)/test/fuzzdata7.db \ - $(TOP)/test/fuzzdata8.db - -# Standard options to testfixture -# -TESTOPTS = --verbose=file --output=test-out.txt - -# Extra compiler options for various shell tools -# -SHELL_OPT += -DSQLITE_DQS=0 -SHELL_OPT += -DSQLITE_ENABLE_FTS4 -#SHELL_OPT += -DSQLITE_ENABLE_FTS5 -SHELL_OPT += -DSQLITE_ENABLE_RTREE -SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS -SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB -SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -FUZZERSHELL_OPT = -FUZZCHECK_OPT += -I$(TOP)/test -FUZZCHECK_OPT += -I$(TOP)/ext/recover -FUZZCHECK_OPT += \ - -DSQLITE_OSS_FUZZ \ - -DSQLITE_ENABLE_BYTECODE_VTAB \ - -DSQLITE_ENABLE_DBPAGE_VTAB \ - -DSQLITE_ENABLE_DBSTAT_VTAB \ - -DSQLITE_ENABLE_BYTECODE_VTAB \ - -DSQLITE_ENABLE_DESERIALIZE \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ - -DSQLITE_ENABLE_FTS3_PARENTHESIS \ - -DSQLITE_ENABLE_FTS4 \ - -DSQLITE_ENABLE_FTS5 \ - -DSQLITE_ENABLE_GEOPOLY \ - -DSQLITE_ENABLE_MATH_FUNCTIONS \ - -DSQLITE_ENABLE_MEMSYS5 \ - -DSQLITE_ENABLE_NORMALIZE \ - -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ - -DSQLITE_ENABLE_PREUPDATE_HOOK \ - -DSQLITE_ENABLE_RTREE \ - -DSQLITE_ENABLE_SESSION \ - -DSQLITE_ENABLE_STMTVTAB \ - -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ - -DSQLITE_ENABLE_STAT4 \ - -DSQLITE_ENABLE_STMT_SCANSTATUS \ - -DSQLITE_MAX_MEMORY=50000000 \ - -DSQLITE_MAX_MMAP_SIZE=0 \ - -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ - -DSQLITE_PRIVATE="" - -FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c -FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c -FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c -FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c -FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c -FUZZCHECK_SRC += $(TOP)/test/vt02.c -DBFUZZ_OPT = -ST_OPT = -DSQLITE_OS_KV_OPTIONAL - - -# In wasi-sdk builds, disable the CLI shell build in the "all" target. -SQLITE3_SHELL_TARGET_ = sqlcipher$(TEXE) -SQLITE3_SHELL_TARGET_1 = -SQLITE3_SHELL_TARGET = $(SQLITE3_SHELL_TARGET_@HAVE_WASI_SDK@) - -# This is the default Makefile target. The objects listed here -# are what get build when you type just "make" with no arguments. +AS_AUTO_DEF = $(TOP)/auto.def # -all: sqlite3.h libsqlcipher.la $(SQLITE3_SHELL_TARGET) \ - $(HAVE_TCL:1=libtclsqlite3.la) - -Makefile: $(TOP)/Makefile.in - ./config.status - -sqlcipher.pc: $(TOP)/sqlcipher.pc.in - ./config.status - -libsqlcipher.la: $(LIBOBJ) - $(LTLINK) -no-undefined -o $@ $(LIBOBJ) $(TLIBS) \ - ${ALLOWRELEASE} -rpath "$(libdir)" -version-info "8:6:8" - -libtclsqlite3.la: tclsqlite.lo libsqlcipher.la - $(LTLINK) -no-undefined -o $@ tclsqlite.lo \ - libsqlcipher.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \ - -rpath "$(TCLLIBDIR)" \ - -version-info "8:6:8" \ - -avoid-version - -sqlcipher$(TEXE): shell.c sqlite3.c - $(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \ - shell.c sqlite3.c \ - $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" - -sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h - $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS) - -dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h - $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) - -scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo - $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ - $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) - -srcck1$(BEXE): $(TOP)/tool/srcck1.c - $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c - -sourcetest: srcck1$(BEXE) sqlite3.c - ./srcck1 sqlite3.c - -fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ - $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) - -fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) - -fuzzcheck-asan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) - -ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ - $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) - -sessionfuzz$(TEXE): $(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(TOP)/test/sessionfuzz.c $(TLIBS) - -dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS) - -DBFUZZ2_OPTS = \ - -DSQLITE_THREADSAFE=0 \ - -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_DEBUG \ - -DSQLITE_ENABLE_DBSTAT_VTAB \ - -DSQLITE_ENABLE_BYTECODE_VTAB \ - -DSQLITE_ENABLE_RTREE \ - -DSQLITE_ENABLE_FTS4 \ - -DSQLITE_ENABLE_FTS5 - -dbfuzz2$(TEXE): $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h - $(CC) $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \ - -DSTANDALONE -o dbfuzz2 \ - $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS) - mkdir -p dbfuzz2-dir - cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir - -dbfuzz2-asan: $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h - clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \ - -fsanitize=fuzzer,undefined,address -o dbfuzz2-asan \ - $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS) - mkdir -p dbfuzz2-dir - cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir - -dbfuzz2-msan: $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h - clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \ - -fsanitize=fuzzer,undefined,memory -o dbfuzz2-msan \ - $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS) - mkdir -p dbfuzz2-dir - cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir - -mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c - $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \ - $(TLIBS) -rpath "$(libdir)" - -MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 -MPTEST2=./mptester$(TEXE) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20 -mptest: mptester$(TEXE) - rm -f mptest.db - $(MPTEST1) --journalmode DELETE - $(MPTEST2) --journalmode WAL - $(MPTEST1) --journalmode WAL - $(MPTEST2) --journalmode PERSIST - $(MPTEST1) --journalmode PERSIST - $(MPTEST2) --journalmode TRUNCATE - $(MPTEST1) --journalmode TRUNCATE - $(MPTEST2) --journalmode DELETE - - -# This target creates a directory named "tsrc" and fills it with -# copies of all of the C source code and header files needed to -# build on the target system. Some of the C source code and header -# files are automatically generated. This target takes care of -# all that automatic generation. -# -.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c - rm -rf tsrc - mkdir tsrc - cp -f $(SRC) tsrc - rm tsrc/sqlite.h.in tsrc/parse.y - $(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl $(OPTS) vdbe.new - mv vdbe.new tsrc/vdbe.c - cp fts5.c fts5.h tsrc - touch .target_source - -sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl - $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) - cp tsrc/sqlite3ext.h . - cp $(TOP)/ext/session/sqlite3session.h . - -sqlite3r.h: sqlite3.h - $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h - -sqlite3r.c: sqlite3.c sqlite3r.h - cp $(TOP)/ext/recover/sqlite3recover.c tsrc/ - cp $(TOP)/ext/recover/sqlite3recover.h tsrc/ - cp $(TOP)/ext/recover/dbdata.c tsrc/ - $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) - -sqlite3ext.h: .target_source - cp tsrc/sqlite3ext.h . - -tclsqlite3.c: sqlite3.c - echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c - cat sqlite3.c >>tclsqlite3.c - echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c - cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c - -sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl - $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl - -# Rule to build the amalgamation -# -sqlite3.lo: sqlite3.c - $(LTCOMPILE) $(TEMP_STORE) -c sqlite3.c - -# Rules to build the LEMON compiler generator -# -lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c - $(BCC) -o $@ $(TOP)/tool/lemon.c - cp $(TOP)/tool/lempar.c . - -# Rules to build the program that generates the source-id -# -mksourceid$(BEXE): $(TOP)/tool/mksourceid.c - $(BCC) -o $@ $(TOP)/tool/mksourceid.c - -# Rules to build individual *.o files from generated *.c files. This -# applies to: -# -# parse.o -# opcodes.o -# -parse.lo: parse.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c parse.c - -opcodes.lo: opcodes.c - $(LTCOMPILE) $(TEMP_STORE) -c opcodes.c - -# BEGIN CRYPTO -crypto.lo: $(TOP)/src/crypto.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/crypto.c -crypto_impl.lo: $(TOP)/src/crypto_impl.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/crypto_impl.c -crypto_openssl.lo: $(TOP)/src/crypto_openssl.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/crypto_openssl.c -crypto_nss.lo: $(TOP)/src/crypto_nss.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/crypto_nss.c -crypto_libtomcrypt.lo: $(TOP)/src/crypto_libtomcrypt.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/crypto_libtomcrypt.c -crypto_cc.lo: $(TOP)/src/crypto_cc.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/crypto_cc.c -# END CRYPTO - -# Rules to build individual *.o files from files in the src directory. -# -alter.lo: $(TOP)/src/alter.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/alter.c - -analyze.lo: $(TOP)/src/analyze.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/analyze.c - -attach.lo: $(TOP)/src/attach.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/attach.c - -auth.lo: $(TOP)/src/auth.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/auth.c - -backup.lo: $(TOP)/src/backup.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/backup.c - -bitvec.lo: $(TOP)/src/bitvec.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/bitvec.c - -btmutex.lo: $(TOP)/src/btmutex.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/btmutex.c - -btree.lo: $(TOP)/src/btree.c $(HDR) $(TOP)/src/pager.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/btree.c - -build.lo: $(TOP)/src/build.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/build.c - -callback.lo: $(TOP)/src/callback.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/callback.c - -complete.lo: $(TOP)/src/complete.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/complete.c - -ctime.lo: $(TOP)/src/ctime.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/ctime.c - -date.lo: $(TOP)/src/date.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c - -dbpage.lo: $(TOP)/src/dbpage.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbpage.c - -dbstat.lo: $(TOP)/src/dbstat.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/dbstat.c - -delete.lo: $(TOP)/src/delete.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/delete.c - -expr.lo: $(TOP)/src/expr.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/expr.c - -fault.lo: $(TOP)/src/fault.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/fault.c - -fkey.lo: $(TOP)/src/fkey.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/fkey.c - -func.lo: $(TOP)/src/func.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/func.c - -global.lo: $(TOP)/src/global.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/global.c - -hash.lo: $(TOP)/src/hash.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/hash.c - -insert.lo: $(TOP)/src/insert.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c - -json.lo: $(TOP)/src/json.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/json.c - -legacy.lo: $(TOP)/src/legacy.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c - -loadext.lo: $(TOP)/src/loadext.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/loadext.c - -main.lo: $(TOP)/src/main.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/main.c - -malloc.lo: $(TOP)/src/malloc.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/malloc.c - -mem0.lo: $(TOP)/src/mem0.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem0.c - -mem1.lo: $(TOP)/src/mem1.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem1.c - -mem2.lo: $(TOP)/src/mem2.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem2.c - -mem3.lo: $(TOP)/src/mem3.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem3.c - -mem5.lo: $(TOP)/src/mem5.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mem5.c - -memdb.lo: $(TOP)/src/memdb.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memdb.c - -memjournal.lo: $(TOP)/src/memjournal.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/memjournal.c - -mutex.lo: $(TOP)/src/mutex.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex.c - -mutex_noop.lo: $(TOP)/src/mutex_noop.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex_noop.c - -mutex_unix.lo: $(TOP)/src/mutex_unix.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex_unix.c - -mutex_w32.lo: $(TOP)/src/mutex_w32.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/mutex_w32.c - -notify.lo: $(TOP)/src/notify.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/notify.c - -pager.lo: $(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pager.c - -pcache.lo: $(TOP)/src/pcache.c $(HDR) $(TOP)/src/pcache.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pcache.c - -pcache1.lo: $(TOP)/src/pcache1.c $(HDR) $(TOP)/src/pcache.h - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pcache1.c - -os.lo: $(TOP)/src/os.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os.c - -os_kv.lo: $(TOP)/src/os_kv.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_kv.c - -os_unix.lo: $(TOP)/src/os_unix.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_unix.c - -os_win.lo: $(TOP)/src/os_win.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_win.c - -pragma.lo: $(TOP)/src/pragma.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pragma.c - -prepare.lo: $(TOP)/src/prepare.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/prepare.c - -printf.lo: $(TOP)/src/printf.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/printf.c - -random.lo: $(TOP)/src/random.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/random.c - -resolve.lo: $(TOP)/src/resolve.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/resolve.c - -rowset.lo: $(TOP)/src/rowset.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/rowset.c - -select.lo: $(TOP)/src/select.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/select.c - -status.lo: $(TOP)/src/status.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/status.c - -table.lo: $(TOP)/src/table.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/table.c - -threads.lo: $(TOP)/src/threads.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/threads.c - -tokenize.lo: $(TOP)/src/tokenize.c keywordhash.h $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c - -treeview.lo: $(TOP)/src/treeview.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/treeview.c - -trigger.lo: $(TOP)/src/trigger.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c - -update.lo: $(TOP)/src/update.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/update.c - -upsert.lo: $(TOP)/src/upsert.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/upsert.c - -utf.lo: $(TOP)/src/utf.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/utf.c - -util.lo: $(TOP)/src/util.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/util.c - -vacuum.lo: $(TOP)/src/vacuum.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vacuum.c - -vdbe.lo: $(TOP)/src/vdbe.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbe.c - -vdbeapi.lo: $(TOP)/src/vdbeapi.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbeapi.c - -vdbeaux.lo: $(TOP)/src/vdbeaux.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbeaux.c - -vdbeblob.lo: $(TOP)/src/vdbeblob.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbeblob.c - -vdbemem.lo: $(TOP)/src/vdbemem.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbemem.c - -vdbesort.lo: $(TOP)/src/vdbesort.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbesort.c - -vdbetrace.lo: $(TOP)/src/vdbetrace.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbetrace.c - -vdbevtab.lo: $(TOP)/src/vdbevtab.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vdbevtab.c - -vtab.lo: $(TOP)/src/vtab.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vtab.c - -wal.lo: $(TOP)/src/wal.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/wal.c - -walker.lo: $(TOP)/src/walker.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/walker.c - -where.lo: $(TOP)/src/where.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/where.c - -wherecode.lo: $(TOP)/src/wherecode.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/wherecode.c - -whereexpr.lo: $(TOP)/src/whereexpr.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c - -window.lo: $(TOP)/src/window.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/window.c - -tclsqlite.lo: $(TOP)/src/tclsqlite.c $(HDR) - $(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c - -tclsqlite-shell.lo: $(TOP)/src/tclsqlite.c $(HDR) - $(LTCOMPILE) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c - -tclsqlite-stubs.lo: $(TOP)/src/tclsqlite.c $(HDR) - $(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c - -tclsqlcipher$(TEXE): tclsqlite-shell.lo libsqlcipher.la - $(LTLINK) -o $@ tclsqlite-shell.lo \ - libsqlcipher.la $(LIBTCL) - -# Rules to build opcodes.c and opcodes.h +# Shell commands to re-run $(TOP)/configure with the same args it was +# invoked with to produce this makefile. # -opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl - $(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c - -opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl - cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h +AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@ +.PHONY: config reconfigure +config reconfigure: + $(AS_AUTORECONFIG) +USE_AMALGAMATION ?= @USE_AMALGAMATION@ +LINK_TOOLS_DYNAMICALLY ?= @LINK_TOOLS_DYNAMICALLY@ +AMALGAMATION_GEN_FLAGS ?= --linemacros=@AMALGAMATION_LINE_MACROS@ +EXTRA_SRC ?= @AMALGAMATION_EXTRA_SRC@ +STATIC_TCLSQLITE3 = @STATIC_TCLSQLITE3@ +STATIC_CLI_SHELL = @STATIC_CLI_SHELL@ -# Rules to build parse.c and parse.h - the outputs of lemon. # -parse.h: parse.c - -parse.c: $(TOP)/src/parse.y lemon$(BEXE) - cp $(TOP)/src/parse.y . - ./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y - -sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION - $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h - -sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION - echo '#ifndef SQLITE_RESOURCE_VERSION' >$@ - echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@ - cat $(TOP)/VERSION | $(TCLSH_CMD) $(TOP)/tool/replace.tcl exact . , >>$@ - echo '#endif' >>sqlite3rc.h - -keywordhash.h: $(TOP)/tool/mkkeywordhash.c - $(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c - ./mkkeywordhash$(BEXE) >keywordhash.h - -# Source files that go into making shell.c -SHELL_SRC = \ - $(TOP)/src/shell.c.in \ - $(TOP)/ext/misc/appendvfs.c \ - $(TOP)/ext/misc/completion.c \ - $(TOP)/ext/misc/decimal.c \ - $(TOP)/ext/misc/basexx.c \ - $(TOP)/ext/misc/base64.c \ - $(TOP)/ext/misc/base85.c \ - $(TOP)/ext/misc/fileio.c \ - $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/regexp.c \ - $(TOP)/ext/misc/series.c \ - $(TOP)/ext/misc/shathree.c \ - $(TOP)/ext/misc/sqlar.c \ - $(TOP)/ext/misc/uint.c \ - $(TOP)/ext/expert/sqlite3expert.c \ - $(TOP)/ext/expert/sqlite3expert.h \ - $(TOP)/ext/misc/zipfile.c \ - $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/sqlite3recover.h \ - $(TOP)/src/test_windirent.c - -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl - $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c - - - - -# Rules to build the extension objects. +# CFLAGS for sqlite3$(T.exe) # -icu.lo: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c - -fts3.lo: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c - -fts3_aux.lo: $(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c - -fts3_expr.lo: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c - -fts3_hash.lo: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c - -fts3_icu.lo: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c - -fts3_porter.lo: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c - -fts3_snippet.lo: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c - -fts3_tokenizer.lo: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c - -fts3_tokenizer1.lo: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c - -fts3_tokenize_vtab.lo: $(TOP)/ext/fts3/fts3_tokenize_vtab.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenize_vtab.c - -fts3_unicode.lo: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c - -fts3_unicode2.lo: $(TOP)/ext/fts3/fts3_unicode2.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode2.c - -fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c - -rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c - -userauth.lo: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c +SHELL_OPT ?= @OPT_SHELL@ -sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c +Makefile: $(TOP)/Makefile.in $(AS_AUTO_DEF) + $(AS_AUTORECONFIG) + @touch $@ -stmt.lo: $(TOP)/ext/misc/stmt.c - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c +sqlite3.pc: $(TOP)/sqlite3.pc.in $(AS_AUTO_DEF) + $(AS_AUTORECONFIG) + @touch $@ +install: install-pc # defined in main.mk -# FTS5 things -# -FTS5_SRC = \ - $(TOP)/ext/fts5/fts5.h \ - $(TOP)/ext/fts5/fts5Int.h \ - $(TOP)/ext/fts5/fts5_aux.c \ - $(TOP)/ext/fts5/fts5_buffer.c \ - $(TOP)/ext/fts5/fts5_main.c \ - $(TOP)/ext/fts5/fts5_config.c \ - $(TOP)/ext/fts5/fts5_expr.c \ - $(TOP)/ext/fts5/fts5_hash.c \ - $(TOP)/ext/fts5/fts5_index.c \ - fts5parse.c fts5parse.h \ - $(TOP)/ext/fts5/fts5_storage.c \ - $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c \ - $(TOP)/ext/fts5/fts5_varint.c \ - $(TOP)/ext/fts5/fts5_vocab.c \ +sqlite_cfg.h: $(AS_AUTO_DEF) + $(AS_AUTORECONFIG) + @touch $@ -fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon$(BEXE) - cp $(TOP)/ext/fts5/fts5parse.y . - rm -f fts5parse.h - ./lemon$(BEXE) $(OPTS) -S fts5parse.y - -fts5parse.h: fts5parse.c - -fts5.c: $(FTS5_SRC) - $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl - cp $(TOP)/ext/fts5/fts5.h . - -fts5.lo: fts5.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c fts5.c - -sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c - - -# Rules to build the 'testfixture' application. -# -# If using the amalgamation, use sqlite3.c directly to build the test -# fixture. Otherwise link against libsqlcipher.la. (This distinction is -# necessary because the test fixture requires non-API symbols which are -# hidden when the library is built via the amalgamation). -# -TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 -TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit -TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE -TESTFIXTURE_FLAGS += -DBUILD_sqlite -TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 -TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 -TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB -TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB -TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB -TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC - -TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlcipher.la -TESTFIXTURE_SRC1 = sqlite3.c -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c -TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) - -testfixture$(TEXE): $(TESTFIXTURE_SRC) - $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ - -o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS) - -coretestprogs: $(TESTPROGS) - -testprogs: coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE) - -# A very detailed test running most or all test cases -fulltest: alltest fuzztest - -# Run most or all tcl test cases -alltest: $(TESTPROGS) - ./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS) - -# Really really long testing -soaktest: $(TESTPROGS) - ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 $(TESTOPTS) - -# Do extra testing but not everything. -fulltestonly: $(TESTPROGS) fuzztest - ./testfixture$(TEXE) $(TOP)/test/full.test - -# Fuzz testing -fuzztest: fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db - ./fuzzcheck$(TEXE) $(FUZZDATA) - ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db - -valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db - valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M $(FUZZDATA) - valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db - -# The veryquick.test TCL tests. # -tcltest: ./testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) - -# Runs all the same tests cases as the "tcltest" target but uses -# the testrunner.tcl script to run them in multiple cores -# concurrently. -testrunner: testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl - -# Runs both fuzztest and testrunner, consecutively. +# Fiddle app # -devtest: testfixture$(TEXE) fuzztest testrunner - -# Testing for a release +# EMCC_WRAPPER must refer to the genuine emcc binary, or a +# call-compatible wrapper, e.g. $(TOP)/tool/emcc.sh. If it's empty, +# build components requiring Emscripten will not build. # -releasetest: testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release - -# Minimal testing that runs in less than 3 minutes +# Achtung: though _this_ makefile is POSIX-make compatible, the fiddle +# build requires GNU make. # -quicktest: ./testfixture$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS) +EMCC_WRAPPER = @EMCC_WRAPPER@ +fiddle: sqlite3.c shell.c + @if [ x = "x$(EMCC_WRAPPER)" ]; then \ + echo "Emscripten SDK not found by configure. Cannot build fiddle." 1&>2; \ + exit 1; \ + fi + $(MAKE) -C ext/wasm fiddle emcc_opt=-Os -# This is the common case. Run many tests that do not take too long, -# including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: fuzztest sourcetest $(TESTPROGS) tcltest - -# Run a test using valgrind. This can take a really long time -# because valgrind is so much slower than a native machine. +# Spell-checking for source comments +# The sources checked are either C sources or C source templates. +# Their comments are extracted and processed through aspell using +# a custom dictionary that contains scads of odd identifiers that +# find their way into the comments. # -valgrindtest: $(TESTPROGS) valgrindfuzz - OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS) - -# A very fast test that checks basic sanity. The name comes from -# the 60s-era electronics testing: "Turn it on and see if smoke -# comes out." +# Currently, this target is setup to be "made" in-tree only. +# The output is ephemeral. Redirect it to guide spelling fixups, +# either to correct spelling or add words to tool/custom.txt. # -smoketest: $(TESTPROGS) fuzzcheck$(TEXE) - ./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS) - -shelltest: $(TESTPROGS) - ./testfixture$(TEXT) $(TOP)/test/permutations.test shell - -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in - $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c - -sqlite3_analyzer$(TEXE): sqlite3_analyzer.c - $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) - -sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in - $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c - -sqltclsh$(TEXE): sqltclsh.c - $(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS) - -sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c - $(LTLINK) $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(TLIBS) - -CHECKER_DEPS =\ - $(TOP)/tool/mkccode.tcl \ - sqlite3.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/ext/repair/sqlite3_checker.tcl \ - $(TOP)/ext/repair/checkindex.c \ - $(TOP)/ext/repair/checkfreelist.c \ - $(TOP)/ext/misc/btreeinfo.c \ - $(TOP)/ext/repair/sqlite3_checker.c.in - -sqlite3_checker.c: $(CHECKER_DEPS) - $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ - -sqlite3_checker$(TEXE): sqlite3_checker.c - $(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS) - -dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo - $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ - $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) - -dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c - $(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c - -showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) +./custom.rws: ./tool/custom.txt + @echo 'Updating custom dictionary from tool/custom.txt' + aspell --lang=en create master ./custom.rws < ./tool/custom.txt +misspell: ./custom.rws has_tclsh84 +# $(JIMSH) does not work with spellsift.tcl + $(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in -showstat4$(TEXE): $(TOP)/tool/showstat4.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS) - -showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS) - -showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) - -showshm$(TEXE): $(TOP)/tool/showshm.c - $(LTLINK) -o $@ $(TOP)/tool/showshm.c - -index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo - $(LTLINK) $(SHELL_OPT) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS) - -changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) - -changesetfuzz$(TEXE): $(TOP)/ext/session/changesetfuzz.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.lo $(TLIBS) - -rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) - -atrc$(TEXX): $(TOP)/test/atrc.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/test/atrc.c sqlite3.lo $(TLIBS) - -LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h - $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c - -wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) - -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c Makefile - $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) - -startup$(TEXE): $(TOP)/test/startup.c sqlite3.c - $(CC) -Os -g -DSQLITE_THREADSAFE=0 -o $@ $(TOP)/test/startup.c sqlite3.c $(TLIBS) - -KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ - -kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c - $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) - -rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo - $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) - -loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la - $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) - -# This target will fail if the SQLite amalgamation contains any exported -# symbols that do not begin with "sqlite3_". It is run as part of the -# releasetest.tcl script. -# -VALIDIDS=' sqlite3(changeset|changegroup|session)?_' -checksymbols: sqlite3.o - nm -g --defined-only sqlite3.o - nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0 - echo '0 errors out of 1 tests' - -# Build the amalgamation-autoconf package. The amalamgation-tarball target builds -# a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. -# The snapshot-tarball target builds a tarball named by the SHA1 hash # -amalgamation-tarball: sqlite3.c sqlite3rc.h - TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal - -snapshot-tarball: sqlite3.c sqlite3rc.h - TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot - -# The next two rules are used to support the "threadtest" target. Building -# threadtest runs a few thread-safety tests that are implemented in C. This -# target is invoked by the releasetest.tcl script. -# -THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ - $(TOP)/test/tt3_checkpoint.c \ - $(TOP)/test/tt3_index.c \ - $(TOP)/test/tt3_vacuum.c \ - $(TOP)/test/tt3_stress.c \ - $(TOP)/test/tt3_lookaside1.c - -threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC) - $(LTLINK) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.lo -o $@ $(TLIBS) - -threadtest: threadtest3$(TEXE) - ./threadtest3$(TEXE) - -threadtest5: sqlite3.c $(TOP)/test/threadtest5.c - $(LTLINK) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(TLIBS) - -# Standard install and cleanup targets +# clean/distclean are mostly defined in main.mk. In this makefile we +# perform cleanup known to be relevant to (only) the autosetup-driven +# build. # -lib_install: libsqlcipher.la - $(INSTALL) -d $(DESTDIR)$(libdir) - $(LTINSTALL) libsqlcipher.la $(DESTDIR)$(libdir) - -install: sqlcipher$(TEXE) lib_install sqlite3.h sqlcipher.pc ${HAVE_TCL:1=tcl_install} - $(INSTALL) -d $(DESTDIR)$(bindir) - $(LTINSTALL) sqlcipher$(TEXE) $(DESTDIR)$(bindir) - $(INSTALL) -d $(DESTDIR)$(includedir) - $(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir) - $(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir) - $(INSTALL) -d $(DESTDIR)$(pkgconfigdir) - $(INSTALL) -m 0644 sqlcipher.pc $(DESTDIR)$(pkgconfigdir) - -pkgIndex.tcl: - echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@ -tcl_install: lib_install libtclsqlite3.la pkgIndex.tcl - $(INSTALL) -d $(DESTDIR)$(TCLLIBDIR) - $(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR) - rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a - $(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR) - -clean: - rm -f *.lo *.la *.o sqlcipher$(TEXE) libsqlcipher.la - rm -f sqlite3.h opcodes.* - rm -rf .libs .deps - rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz - rm -f mkkeywordhash$(BEXE) keywordhash.h - rm -f mksourceid$(BEXE) - rm -f *.da *.bb *.bbg gmon.out - rm -rf tsrc .target_source - rm -f tclsqlcipher$(TEXE) - rm -f testfixture$(TEXE) test.db - rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) - rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) - rm -f wordcount$(TEXE) changeset$(TEXE) - rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def - rm -f sqlite3.c - rm -f sqlite3rc.h - rm -f shell.c sqlite3ext.h - rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c - rm -f sqlite-*-output.vsix - rm -f mptester mptester.exe - rm -f rbu rbu.exe - rm -f srcck1 srcck1.exe - rm -f fuzzershell fuzzershell.exe - rm -f fuzzcheck fuzzcheck.exe - rm -f sqldiff sqldiff.exe - rm -f dbhash dbhash.exe - rm -f fts5.* fts5parse.* - rm -f threadtest5 - -distclean: clean - rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlcipher.pc \ - $(TESTPROGS) +distclean-autosetup: + rm -f sqlite_cfg.h config.log config.status config.defines.* Makefile sqlite3.pc + rm -f $(TOP)/tool/emcc.sh + rm -f libsqlite3*$(T.dll) + rm -f jimsh0* +distclean: distclean-autosetup # -# Windows section +# tool/version-info: a utility for emitting sqlite3 version info +# in various forms. It's used by ext/wasm/. # -dll: sqlite3.dll - -REAL_LIBOBJ = $(LIBOBJ:%.lo=.libs/%.o) - -$(REAL_LIBOBJ): $(LIBOBJ) - -sqlite3.def: $(REAL_LIBOBJ) - echo 'EXPORTS' >sqlite3.def - nm $(REAL_LIBOBJ) | grep ' T ' | grep ' _sqlite3_' \ - | sed 's/^.* _//' >>sqlite3.def - -sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def - $(TCC) -shared -o $@ sqlite3.def \ - -Wl,"--strip-all" $(REAL_LIBOBJ) +version-info$(T.exe): $(TOP)/tool/version-info.c Makefile sqlite3.h + $(T.link) $(ST_OPT) -o $@ $(TOP)/tool/version-info.c -# -# Fiddle app -# -fiddle: sqlite3.c shell.c - make -C ext/wasm fiddle emcc_opt=-Os +include $(TOP)/main.mk diff --git a/Makefile.linux-gcc b/Makefile.linux-gcc deleted file mode 100644 index ad5d4dd093..0000000000 --- a/Makefile.linux-gcc +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/make -# -# Makefile for SQLITE -# -# This is a template makefile for SQLite. Most people prefer to -# use the autoconf generated "configure" script to generate the -# makefile automatically. But that does not work for everybody -# and in every situation. If you are having problems with the -# "configure" script, you might want to try this makefile as an -# alternative. Create a copy of this file, edit the parameters -# below and type "make". -# - -#### The toplevel directory of the source tree. This is the directory -# that contains this "Makefile.in" and the "configure.in" script. -# -TOP = ../sqlite - -#### C Compiler and options for use in building executables that -# will run on the platform that is doing the build. -# -BCC = gcc -g -O0 -#BCC = /opt/ancic/bin/c89 -0 - -#### If the target operating system supports the "usleep()" system -# call, then define the HAVE_USLEEP macro for all C modules. -# -#USLEEP = -USLEEP = -DHAVE_USLEEP=1 - -#### If you want the SQLite library to be safe for use within a -# multi-threaded program, then define the following macro -# appropriately: -# -#THREADSAFE = -DTHREADSAFE=1 -THREADSAFE = -DTHREADSAFE=0 - -#### Specify any extra linker options needed to make the library -# thread safe -# -THREADLIB = -lpthread -lm -ldl -#THREADLIB = - -#### Specify any extra libraries needed to access required functions. -# -#TLIBS = -lrt # fdatasync on Solaris 8 -TLIBS = - -#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1 -# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all -# malloc()s and free()s in order to track down memory leaks. -# -# SQLite uses some expensive assert() statements in the inner loop. -# You can make the library go almost twice as fast if you compile -# with -DNDEBUG=1 -# -OPTS += -DSQLITE_DEBUG=1 -OPTS += -DSQLITE_ENABLE_WHERETRACE -OPTS += -DSQLITE_ENABLE_SELECTTRACE - -#### The suffix to add to executable files. ".exe" for windows. -# Nothing for unix. -# -#EXE = .exe -EXE = - -#### C Compile and options for use in building executables that -# will run on the target platform. This is usually the same -# as BCC, unless you are cross-compiling. -# -TCC = gcc -O0 -#TCC = gcc -g -O0 -Wall -#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage -#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6 -#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive - -#### Tools used to build a static library. -# -AR = ar cr -#AR = /opt/mingw/bin/i386-mingw32-ar cr -RANLIB = ranlib -#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib - -MKSHLIB = gcc -shared -SO = so -SHPREFIX = lib -# SO = dll -# SHPREFIX = - -#### Extra compiler options needed for programs that use the TCL library. -# -TCL_FLAGS = -I/home/drh/tcl/include/tcl8.6 - -#### Linker options needed to link against the TCL library. -# -#LIBTCL = -ltcl -lm -ldl -LIBTCL = /home/drh/tcl/lib/libtcl8.6.a -lm -lpthread -ldl -lz - -#### Additional objects for SQLite library when TCL support is enabled. -#TCLOBJ = -TCLOBJ = tclsqlite.o - -#### Compiler options needed for programs that use the readline() library. -# -READLINE_FLAGS = -#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline - -#### Linker options needed by programs using readline() must link against. -# -LIBREADLINE = -#LIBREADLINE = -static -lreadline -ltermcap - -# You should not have to change anything below this line -############################################################################### -include $(TOP)/main.mk diff --git a/Makefile.linux-generic b/Makefile.linux-generic new file mode 100644 index 0000000000..c7441fa517 --- /dev/null +++ b/Makefile.linux-generic @@ -0,0 +1,64 @@ +#!/usr/make +all: +# +# Makefile for SQLITE +# +# This is a template makefile for SQLite. Most people prefer to +# use the autoconf generated "configure" script to generate the +# makefile automatically. But that does not work for everybody +# and in every situation. If you are having problems with the +# "configure" script, you might want to try this makefile as an +# alternative. Create a copy of this file, edit the parameters +# below and type "make". +# +# Maintenance note: because this is the template for Linux systems, it +# is assumed that the platform has GNU make and this file takes +# advantage of that. +# +#### +# +# $(TOP) = The toplevel directory of the source tree. This is the +# directory that contains "Makefile.in" and "auto.def". +# +TOP ?= $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) + +# +# $(CFLAGS) will be used when compiling the library and most +# utilities. It must normally contain -fPIC on Linux systems, +# but overriding CFLAGS is an easy way for users to inadvertently +# remove -fPIC from their builds, so we generally expect to see +# -fPIC in $(CFLAGS.core), which main.mk will integrate with +# the CFLAGS where needed. +# +CFLAGS = +CFLAGS.core = -fPIC + +# +# $(SHELL_OPT) contains CFLAGS for building the sqlite3 CLI shell. +# See main.mk for other potentially-relevant vars which may need +# tweaking, like $(LDFLAGS_READLINE). +# +SHELL_OPT += -DHAVE_READLINE=1 +SHELL_OPT += -DSQLITE_HAVE_ZLIB=1 +LDFLAGS.readline = -lreadline # may need -lcurses etc, depending on the system +CFLAGS.readline = # needs -I... if readline.h is in an unusual place. +LDFLAGS.zlib = -lz + +# +# Library's version number. +# +PACKAGE_VERSION ?= $(shell cat $(TOP)/VERSION 2>/dev/null) + +# sqlite_cfg.h is typically created by the configure script. It's +# commonly not needed but main.mk does not know that so we have to +# create a dummy if we don't already have one. +sqlite_cfg.h: + touch $@ +distclean-.: + rm -f sqlite_cfg.h + +# +# With the above in place, we can now import the rules make use of +# it... +# +include $(TOP)/main.mk diff --git a/Makefile.msc b/Makefile.msc index d5df8b0d58..d3fe57de68 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -18,6 +18,15 @@ USE_AMALGAMATION = 1 !ENDIF # <> +# Optionally set EXTRA_SRC to a list of C files to append to +# the generated sqlite3.c. Any sqlite3 extensions added this +# way may require manual editing, as described in +# https://sqlite.org/forum/forumpost/903f721f3e7c0d25 +# +!IFNDEF EXTRA_SRC +EXTRA_SRC = +!ENDIF + # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN @@ -52,6 +61,21 @@ MINIMAL_AMALGAMATION = 0 USE_STDCALL = 0 !ENDIF +# Use the USE_SEH=0 option on the nmake command line to omit structured +# exception handling (SEH) support. SEH is on by default. +# +!IFNDEF USE_SEH +USE_SEH = 1 +!ENDIF + +# Use STATICALLY_LINK_TCL=1 to statically link against TCL +# +!IFNDEF STATICALLY_LINK_TCL +STATICALLY_LINK_TCL = 0 +!ELSEIF $(STATICALLY_LINK_TCL)!=0 +CCOPTS = $(CCOPTS) -DSTATIC_BUILD +!ENDIF + # Set this non-0 to have the shell executable link against the core dynamic # link library. # @@ -218,6 +242,12 @@ WIN32HEAP = 0 OSTRACE = 0 !ENDIF +# enable address sanitizer using ASAN=1 on the command-line. +# +!IFNDEF ASAN +ASAN = 0 +!ENDIF + # Set this to one of the following values to enable various debugging # features. Each level includes the debugging options from the previous # levels. Currently, the recognized values for DEBUG are: @@ -264,6 +294,12 @@ SESSION = 0 RBU = 0 !ENDIF +# Set this to non-0 to enable support for blocking locks. +# +!IFNDEF SETLK_TIMEOUT +SETLK_TIMEOUT = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -359,18 +395,29 @@ SQLITE_TCL_DEP = # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +OPT_FEATURE_FLAGS = $(OPT_XTRA) !IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Additional feature-options above and beyond what are normally used can be +# be added using OPTIONS=.... on the command-line. These values are +# appended to the OPT_FEATURE_FLAGS variable. +# +!IFDEF OPTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS) +!ENDIF + # Should the session extension be enabled? If so, add compilation options # to enable it. # @@ -381,6 +428,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 # Always enable math functions on Windows OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE # Should the rbu extension be enabled? If so, add compilation options # to enable it. @@ -389,6 +437,14 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1 !ENDIF +# Should structured exception handling (SEH) be enabled for WAL mode in +# the core library? It is on by default. Only omit it if the +# USE_SEH=0 option is provided on the nmake command-line. +# +!IF $(USE_SEH)==0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1 +!ENDIF + # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # @@ -402,6 +458,10 @@ EXT_FEATURE_FLAGS = !ENDIF !ENDIF +!IF $(SETLK_TIMEOUT)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SETLK_TIMEOUT +!ENDIF + ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### @@ -745,17 +805,21 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd +ZLIBCFLAGS = -nologo -MDd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MD BCC = $(BCC) -MD +ZLIBCFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi !ENDIF !ELSE !IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd +ZLIBCFLAGS = -nologo -MTd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT +ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF !ENDIF @@ -826,7 +890,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_DEBUG=1 -DSQLITE_USE_W32_FOR_CONSOLE_IO RCC = $(RCC) -DSQLITE_DEBUG=1 !IF $(DYNAMIC_SHELL)==0 TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE @@ -877,6 +941,13 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF !ENDIF + +# Address sanitizer if ASAN=1 +# +!IF $(ASAN)>0 +TCC = $(TCC) /fsanitize=address +!ENDIF + # <> # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables @@ -884,16 +955,28 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 # prior to running nmake in order to match the actual installed location and # version on this machine. # -!IFNDEF TCLVERSION -TCLVERSION = 86 +!IF $(STATICALLY_LINK_TCL)!=0 +TCLSUFFIX = s !ENDIF - !IFNDEF TCLSUFFIX TCLSUFFIX = !ENDIF !IFNDEF TCLDIR -TCLDIR = $(TOP)\compat\tcl +TCLDIR = C:\Tcl +!ENDIF + +!IFNDEF TCLVERSION +!IF EXISTS("$(TCLDIR)\lib\tcl90$(TCLSUFFIX).lib") +TCLVERSION = 90 +!ELSEIF EXISTS("$(TCLDIR)\lib\tcl86$(TCLSUFFIX).lib") +TCLVERSION = 86 +!ELSEIF EXISTS("$(TCLDIR)\lib\tcl86t.lib") +TCLSUFFIX = t +TCLVERSION = 86 +!ELSE +TCLVERSION = 90 +!ENDIF !ENDIF !IFNDEF TCLINCDIR @@ -908,8 +991,24 @@ TCLLIBDIR = $(TCLDIR)\lib LIBTCL = tcl$(TCLVERSION)$(TCLSUFFIX).lib !ENDIF +!IFNDEF TCLLIBS +!IF $(STATICALLY_LINK_TCL)!=0 +TCLLIBS = /NODEFAULTLIB:libucrt.lib netapi32.lib user32.lib ucrt.lib +!ELSE +TCLLIBS = +!ENDIF +!ENDIF + !IFNDEF LIBTCLSTUB +!IF EXISTS("$(TCLLIBDIR)\tclstub$(TCLVERSION)$(TCLSUFFIX).lib") LIBTCLSTUB = tclstub$(TCLVERSION)$(TCLSUFFIX).lib +!ELSEIF EXISTS("$(TCLLIBDIR)\tclstub$(TCLSUFFIX).lib") +LIBTCLSTUB = tclstub$(TCLSUFFIX).lib +!ELSEIF EXISTS("$(TCLLIBDIR)\tclstub$(TCLVERSION).lib") +LIBTCLSTUB = tclstub$(TCLVERSION).lib +!ELSE +LIBTCLSTUB = tclstub.lib +!ENDIF !ENDIF !IFNDEF LIBTCLPATH @@ -968,12 +1067,32 @@ LIBICU = icuuc.lib icuin.lib # specific Tcl shell to use. # !IFNDEF TCLSH_CMD -!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe") -TCLSH_CMD = tclsh -!ELSE +!IF EXISTS("$(TCLDIR)\bin\tclsh$(TCLVERSION).exe") +TCLSH_CMD = $(TCLDIR)\bin\tclsh$(TCLVERSION).exe +!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh90.exe") +TCLSH_CMD = $(TCLDIR)\bin\tclsh90.exe +!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh86.exe") +TCLSH_CMD = $(TCLDIR)\bin\tclsh86.exe +!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh86t.exe") +TCLSH_CMD = $(TCLDIR)\bin\tclsh86t.exe +!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh.exe") TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe +!ELSE +TCLSH_CMD = tclsh !ENDIF !ENDIF + +# A light-weight TCLSH replacement that can be used for code generation +# but which is not adequate for testing. This is "jimsh0" by default, +# with source code in the repository. To force the whole build to use +# the full, official tclsh, add WITHOUT_JIMSH=1 to the nmake command line. +# +!IFDEF WITHOUT_JIMSH +JIM_TCLSH = $(TCLSH_CMD) +!ENDIF +!IFNDEF JIM_TCLSH +JIM_TCLSH = jimsh0.exe +!ENDIF # <> # Compiler options needed for programs that use the readline() library. @@ -1004,15 +1123,6 @@ RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 TLIBS = !ENDIF -# Flags controlling use of the in memory btree implementation -# -# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to -# default to file, 2 to default to memory, and 3 to force temporary -# tables to always be in memory. -# -TCC = $(TCC) -DSQLITE_TEMP_STORE=1 -RCC = $(RCC) -DSQLITE_TEMP_STORE=1 - # Enable/disable loadable extensions, and other optional features # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). # The same set of OMIT and ENABLE flags should be passed to the @@ -1239,7 +1349,7 @@ LTLIBS = $(LTLIBS) $(LIBICU) # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo \ + callback.lo carray.lo complete.lo ctime.lo \ date.lo dbpage.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ @@ -1291,13 +1401,9 @@ LIBRESOBJS = # Core source code files, part 1. # SRC00 = \ - $(TOP)\src\crypto.c \ + $(TOP)\src\sqlcipher.c \ $(TOP)\src\crypto_cc.c \ - $(TOP)\src\crypto_impl.c \ - $(TOP)\src\crypto_libtomcrypt.c \ - $(TOP)\src\crypto_nss.c \ $(TOP)\src\crypto_openssl.c \ - $(TOP)\src\crypto.h \ $(TOP)\src\sqlcipher.h \ $(TOP)\src\alter.c \ $(TOP)\src\analyze.c \ @@ -1309,8 +1415,9 @@ SRC00 = \ $(TOP)\src\btree.c \ $(TOP)\src\build.c \ $(TOP)\src\callback.c \ + $(TOP)\src\carray.c \ $(TOP)\src\complete.c \ - $(TOP)\src\ctime.c \ + ctime.c \ $(TOP)\src\date.c \ $(TOP)\src\dbpage.c \ $(TOP)\src\dbstat.c \ @@ -1409,7 +1516,7 @@ SRC04 = \ SRC05 = \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ - $(TOP)\src\pragma.h \ + pragma.h \ $(TOP)\src\sqlite.h.in \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ @@ -1496,7 +1603,6 @@ TESTSRC = \ $(TOP)\src\test8.c \ $(TOP)\src\test9.c \ $(TOP)\src\test_autoext.c \ - $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ $(TOP)\src\test_bestindex.c \ $(TOP)\src\test_blob.c \ @@ -1528,7 +1634,6 @@ TESTSRC = \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vdbecov.c \ $(TOP)\src\test_vfs.c \ - $(TOP)\src\test_windirent.c \ $(TOP)\src\test_window.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ @@ -1544,7 +1649,6 @@ TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\appendvfs.c \ $(TOP)\ext\misc\basexx.c \ - $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\cksumvfs.c \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\csv.c \ @@ -1560,21 +1664,23 @@ TESTEXT = \ $(TOP)\ext\misc\mmapwarm.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\normalize.c \ - $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\prefixes.c \ $(TOP)\ext\misc\qpvtab.c \ + $(TOP)\ext\misc\randomjson.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ + $(TOP)\ext\misc\stmtrand.c \ $(TOP)\ext\misc\totype.c \ $(TOP)\ext\misc\unionvtab.c \ $(TOP)\ext\misc\wholenumber.c \ $(TOP)\ext\rtree\test_rtreedoc.c \ $(TOP)\ext\recover\sqlite3recover.c \ $(TOP)\ext\recover\test_recover.c \ - $(TOP)\ext\recover\dbdata.c \ - fts5.c + $(TOP)\ext\intck\test_intck.c \ + $(TOP)\ext\intck\sqlite3intck.c \ + $(TOP)\ext\recover\dbdata.c # If use of zlib is enabled, add the "zipfile.c" source file. # @@ -1590,7 +1696,7 @@ TESTSRC2 = \ $(SRC01) \ $(SRC07) \ $(SRC10) \ - $(TOP)\ext\async\sqlite3async.c + fts5.c # Header files used by all library source files. # @@ -1610,7 +1716,7 @@ HDR = \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ parse.h \ - $(TOP)\src\pragma.h \ + pragma.h \ $(SQLITE3H) \ sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ @@ -1669,6 +1775,10 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF # <> @@ -1680,6 +1790,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_CARRAY FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB @@ -1693,6 +1804,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PERCENTILE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION @@ -1705,6 +1817,8 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STATIC_RANDOMJSON FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 @@ -1721,6 +1835,7 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION @@ -1761,6 +1876,11 @@ dll: $(SQLITE3DLL) # shell: $(SQLITE3EXE) +# jimsh0 - replacement for tclsh +# +jimsh0.exe: $(TOP)\autosetup\jimsh0.c + cl -DHAVE__FULLPATH=1 $(TOP)\autosetup\jimsh0.c + # <> libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) @@ -1776,22 +1896,38 @@ tclsqlite3.def: tclsqlite.lo pkgIndex.tcl: $(TOP)\VERSION for /F %%V in ('type "$(TOP)\VERSION"') do ( \ - echo package ifneeded sqlite3 @version@ [list load [file join $$dir $(SQLITE3TCLDLL)] sqlite3] \ + echo package ifneeded sqlite3 @version@ [list load [file join $$dir $(SQLITE3TCLDLL)] Sqlite3] \ | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact @version@ %%V > pkgIndex.tcl \ ) $(SQLITE3TCLDLL): libtclsqlite3.lib $(LIBRESOBJS) tclsqlite3.def pkgIndex.tcl $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:tclsqlite3.def /OUT:$@ libtclsqlite3.lib $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + +tclextension: $(SQLITE3TCLDLL) + +tclextension-install: $(SQLITE3TCLDLL) + $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --install-only + +tclextension-uninstall: + $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --uninstall + +tclextension-list: + @ $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --info + +tclextension-verify: sqlite3.h + @ $(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --version-check + + # <> $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # <> -sqlite3.def: libsqlite3.lib +sqlite3.def: libsqlite3.lib $(JIM_TCLSH) echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser|rbu)?_[^@]*)(?:@\d+)?$$" \1 \ + | $(JIM_TCLSH) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset|changegroup|rebaser|rbu)?_[^@]*)(?:@\d+)?$$" \1 \ | sort >> sqlite3.def # <> @@ -1800,12 +1936,25 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <> -sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +RSYNC_SRC = \ + $(TOP)\tool\sqlite3_rsync.c \ + $(SQLITE3C) + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3_rsync.exe: $(RSYNC_SRC) $(LIBRESOBJS) + $(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) + scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -1815,6 +1964,12 @@ srcck1.exe: $(TOP)\tool\srcck1.c sourcetest: srcck1.exe $(SQLITE3C) srcck1.exe $(SQLITE3C) +src-verify.exe: $(TOP)\tool\src-verify.c + $(LTLINK) $(NO_WARN) $(TOP)\tool\src-verify.c + +verify-source: src-verify.exe + src-verify.exe $(TOP) + fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -1856,33 +2011,22 @@ mptest: mptester.exe # files are automatically generated. This target takes care of # all that automatic generation. # -.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP) +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP) $(JIM_TCLSH) -rmdir /Q/S tsrc 2>NUL -mkdir tsrc - for %i in ($(SRC00)) do copy /Y %i tsrc - for %i in ($(SRC01)) do copy /Y %i tsrc - for %i in ($(SRC03)) do copy /Y %i tsrc - for %i in ($(SRC04)) do copy /Y %i tsrc - for %i in ($(SRC05)) do copy /Y %i tsrc - for %i in ($(SRC07)) do copy /Y %i tsrc - for %i in ($(SRC09)) do copy /Y %i tsrc - for %i in ($(SRC10)) do copy /Y %i tsrc - for %i in ($(SRC11)) do copy /Y %i tsrc - for %i in ($(SRC12)) do copy /Y %i tsrc - copy /Y fts5.c tsrc + $(JIM_TCLSH) $(TOP)\tool\cp.tcl $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC07) $(SRC09) $(SRC10) $(SRC11) $(SRC12) fts5.c fts5.h tsrc copy /B tsrc\fts5.c +,, - copy /Y fts5.h tsrc copy /B tsrc\fts5.h +,, del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL - $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new + $(JIM_TCLSH) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source -sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) - $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) +sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-verify.exe $(JIM_TCLSH) + $(JIM_TCLSH) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) $(EXTRA_SRC) -sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl - $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl +sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\split-sqlite3c.tcl # <> # Rule to build the amalgamation @@ -1924,11 +2068,11 @@ opcodes.lo: opcodes.c # !IF $(USE_RC)!=0 # <> -$(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(SQLITE3H) $(TOP)\VERSION +$(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(SQLITE3H) $(TOP)\VERSION $(JIM_TCLSH) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ echo #define SQLITE_RESOURCE_VERSION %%V \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \ + | $(JIM_TCLSH) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \ ) echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc @@ -1968,11 +2112,17 @@ build.lo: $(TOP)\src\build.c $(HDR) callback.lo: $(TOP)\src\callback.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\callback.c +carray.lo: $(TOP)\src\carray.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\carray.c + complete.lo: $(TOP)\src\complete.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\complete.c -ctime.lo: $(TOP)\src\ctime.c $(HDR) - $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\ctime.c +ctime.c: $(TOP)\tool\mkctimec.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkctimec.tcl + +ctime.lo: ctime.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c ctime.c date.lo: $(TOP)\src\date.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c @@ -2189,11 +2339,11 @@ tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) # Rules to build opcodes.c and opcodes.h # -opcodes.c: opcodes.h $(TOP)\tool\mkopcodec.tcl - $(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c +opcodes.c: opcodes.h $(TOP)\tool\mkopcodec.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c -opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl - type parse.h $(TOP)\src\vdbe.c | $(TCLSH_CMD) $(TOP)\tool\mkopcodeh.tcl > opcodes.h +opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl $(JIM_TCLSH) + type parse.h $(TOP)\src\vdbe.c | $(JIM_TCLSH) $(TOP)\tool\mkopcodeh.tcl > opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # @@ -2205,8 +2355,11 @@ parse.c: $(TOP)\src\parse.y lemon.exe copy /B parse.y +,, .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) -S parse.y -$(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS) +pragma.h: $(TOP)\tool\mkpragmatab.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkpragmatab.tcl + +$(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mksqlite3h.tcl "$(TOP:\=/)" -o $(SQLITE3H) $(MKSQLITE3H_ARGS) sqlite3ext.h: .target_source !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 @@ -2229,40 +2382,49 @@ mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h -# Source files that go into making shell.c -SHELL_SRC = \ - $(TOP)\src\shell.c.in \ - $(TOP)\ext\misc\appendvfs.c \ - $(TOP)\ext\misc\completion.c \ - $(TOP)\ext\misc\base64.c \ - $(TOP)\ext\misc\base85.c \ - $(TOP)\ext\misc\decimal.c \ - $(TOP)\ext\misc\fileio.c \ - $(TOP)\ext\misc\ieee754.c \ - $(TOP)\ext\misc\regexp.c \ - $(TOP)\ext\misc\series.c \ - $(TOP)\ext\misc\shathree.c \ - $(TOP)\ext\misc\uint.c \ - $(TOP)\ext\expert\sqlite3expert.c \ - $(TOP)\ext\expert\sqlite3expert.h \ - $(TOP)\ext\misc\memtrace.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/sqlite3recover.h \ - $(TOP)\src\test_windirent.c +# Source and header files that shell.c depends on +SHELL_DEP = \ + $(TOP)\src\shell.c.in \ + $(TOP)\ext\expert\sqlite3expert.c \ + $(TOP)\ext\expert\sqlite3expert.h \ + $(TOP)\ext\intck\sqlite3intck.c \ + $(TOP)\ext\intck\sqlite3intck.h \ + $(TOP)\ext\misc\appendvfs.c \ + $(TOP)\ext\misc\base64.c \ + $(TOP)\ext\misc\base85.c \ + $(TOP)\ext\misc\completion.c \ + $(TOP)\ext\misc\decimal.c \ + $(TOP)\ext\misc\fileio.c \ + $(TOP)\ext\misc\ieee754.c \ + $(TOP)\ext\misc\memtrace.c \ + $(TOP)\ext\misc\pcachetrace.c \ + $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\series.c \ + $(TOP)\ext\misc\sha1.c \ + $(TOP)\ext\misc\shathree.c \ + $(TOP)\ext\misc\sqlar.c \ + $(TOP)\ext\misc\sqlite3_stdio.c \ + $(TOP)\ext\misc\sqlite3_stdio.h \ + $(TOP)\ext\misc\uint.c \ + $(TOP)\ext\misc\vfstrace.c \ + $(TOP)\ext\misc\windirent.h \ + $(TOP)\ext\misc\zipfile.c \ + $(TOP)\ext\recover\dbdata.c \ + $(TOP)\ext\recover\sqlite3recover.c \ + $(TOP)\ext\recover\sqlite3recover.h # If use of zlib is enabled, add the "zipfile.c" source file. # !IF $(USE_ZLIB)!=0 -SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c -SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c +SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\sqlar.c +SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\zipfile.c !ENDIF -shell.c: $(SHELL_SRC) $(TOP)\tool\mkshellc.tcl - $(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c +shell.c: $(SHELL_DEP) $(TOP)\tool\mkshellc.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkshellc.tcl shell.c zlib: - pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd + pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) CFLAGS="$(ZLIBCFLAGS)" && popd # Rules to build the extension objects. # @@ -2336,24 +2498,6 @@ FTS5_SRC = \ $(TOP)\ext\fts5\fts5_varint.c \ $(TOP)\ext\fts5\fts5_vocab.c -LSM1_SRC = \ - $(TOP)\ext\lsm1\lsm.h \ - $(TOP)\ext\lsm1\lsmInt.h \ - $(TOP)\ext\lsm1\lsm_ckpt.c \ - $(TOP)\ext\lsm1\lsm_file.c \ - $(TOP)\ext\lsm1\lsm_log.c \ - $(TOP)\ext\lsm1\lsm_main.c \ - $(TOP)\ext\lsm1\lsm_mem.c \ - $(TOP)\ext\lsm1\lsm_mutex.c \ - $(TOP)\ext\lsm1\lsm_shared.c \ - $(TOP)\ext\lsm1\lsm_sorted.c \ - $(TOP)\ext\lsm1\lsm_str.c \ - $(TOP)\ext\lsm1\lsm_tree.c \ - $(TOP)\ext\lsm1\lsm_unix.c \ - $(TOP)\ext\lsm1\lsm_varint.c \ - $(TOP)\ext\lsm1\lsm_vtab.c \ - $(TOP)\ext\lsm1\lsm_win32.c - fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe copy /Y $(TOP)\ext\fts5\fts5parse.y . copy /B fts5parse.y +,, @@ -2362,16 +2506,11 @@ fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe fts5parse.h: fts5parse.c -fts5.c: $(FTS5_SRC) - $(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl +fts5.c: $(FTS5_SRC) $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\ext\fts5\tool\mkfts5c.tcl copy /Y $(TOP)\ext\fts5\fts5.h . copy /B fts5.h +,, -lsm1.c: $(LSM1_SRC) - $(TCLSH_CMD) $(TOP)\ext\lsm1\tool\mklsm1c.tcl - copy /Y $(TOP)\ext\lsm1\lsm.h . - copy /B lsm.h +,, - fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c @@ -2399,8 +2538,12 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) @@ -2432,17 +2575,28 @@ sqlite_tcl.h: testfixture.exe: $(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ - -DBUILD_sqlite -I$(TCLINCDIR) \ + -DBUILD_sqlite -I$(TCLINCDIR) -I$(TOP)\ext\misc \ $(TESTFIXTURE_SRC) \ /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) +# A small helper for manually running individual tests +tf.bat: testfixture.exe Makefile.msc + echo @set PATH=$(LIBTCLPATH);%PATH% > $@ + echo .\testfixture.exe %* >> $@ + extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS) -coretestprogs: $(TESTPROGS) +tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe sqlite3.dll $(TOP)\tool\mktoolzip.tcl + .\testfixture.exe $(TOP)\tool\mktoolzip.tcl + +snapshot-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe sqlite3.dll $(TOP)\tool\mktoolzip.tcl + .\testfixture.exe $(TOP)\tool\mktoolzip.tcl --snapshot -testprogs: coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe +coretestprogs: testfixture.exe sqlite3.exe + +testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe fulltest: alltest fuzztest @@ -2465,6 +2619,13 @@ queryplantest: testfixture.exe shell fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) +# Legacy testing target for third-party integrators. The SQLite +# developers seldom use this target themselves. Instead +# they use "nmake /f Makefile.msc devtest" which runs tests on +# a standard set of options +# +test: $(TESTPROGS) sourcetest fuzztest tcltest + # Minimal testing that runs in less than 3 minutes (on a fast machine) # quicktest: testfixture.exe sourcetest @@ -2474,7 +2635,6 @@ quicktest: testfixture.exe sourcetest # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: $(TESTPROGS) sourcetest fuzztest tcltest # The veryquick.test TCL tests. # @@ -2488,14 +2648,34 @@ tcltest: testfixture.exe testrunner: testfixture.exe .\testfixture.exe $(TOP)\test\testrunner.tcl -# Runs both fuzztest and testrunner, consecutively. +# This is the testing target preferred by the core SQLite developers. +# It runs tests under a standard configuration. The devs run +# "nmake /f Makefile.msc devtest" prior to each check-in, at a minimum. +# Probably other tests too, but at least this one. # -devtest: testfixture.exe fuzztest testrunner +devtest: srctree-check sourcetest + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest + +mdevtest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest + +# Validate that various generated files in the source tree +# are up-to-date. +# +srctree-check: $(TOP)\tool\srctree-check.tcl + $(TCLSH_CMD) $(TOP)\tool\srctree-check.tcl # Testing for a release # -releasetest: testfixture.exe fuzztest - testfixture.exe $(TOP)/test/testrunner.tcl release +releasetest: verify-source + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release + +# xdevtest is like releasetest, except that it skips the +# dependency on verify-source so that xdevtest can be run from +# a modified source tree. +# +xdevtest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release smoketest: $(TESTPROGS) @@ -2505,8 +2685,8 @@ smoketest: $(TESTPROGS) shelltest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\permutations.test shell -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP) - $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@ +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl -DINCLUDE_SQLITE3_C $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ @@ -2523,14 +2703,14 @@ sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\exp $(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS) CHECKER_DEPS =\ - $(TOP)/tool/mkccode.tcl \ + $(TOP)\tool\mkccode.tcl \ sqlite3.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/ext/repair/sqlite3_checker.tcl \ - $(TOP)/ext/repair/checkindex.c \ - $(TOP)/ext/repair/checkfreelist.c \ - $(TOP)/ext/misc/btreeinfo.c \ - $(TOP)/ext/repair/sqlite3_checker.c.in + $(TOP)\src\tclsqlite.c \ + $(TOP)\ext\repair\sqlite3_checker.tcl \ + $(TOP)\ext\repair\checkindex.c \ + $(TOP)\ext\repair\checkfreelist.c \ + $(TOP)\ext\misc\btreeinfo.c \ + $(TOP)\ext\repair\sqlite3_checker.c.in sqlite3_checker.c: $(CHECKER_DEPS) $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ @@ -2627,8 +2807,145 @@ THREADTEST3_SRC = \ threadtest3.exe: $(THREADTEST3_SRC) $(TOP)\src\test_multiplex.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\test\threadtest3.c $(TOP)\src\test_multiplex.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -LSMDIR=$(TOP)\ext\lsm1 -!INCLUDE $(LSMDIR)\Makefile.msc +# Display key variables that control which version of TCL is to be used. +# +tcl-env: + @echo TCLDIR = $(TCLDIR) + @echo TCLVERSION = $(TCLVERSION) + @echo TCLSUFFIX = $(TCLSUFFIX) + @echo LIBTCL = $(LIBTCL) + @echo LIBTCLSTUB = $(LIBTCLSTUB) + @echo TCLSH_CMD = $(TCLSH_CMD) + @echo JIM_TCLSH = $(JIM_TCLSH) + @echo VISUALSTUDIOVERSION = $(VISUALSTUDIOVERSION) + +env: + @echo ALL_TCL_TARGETS = $(ALL_TCL_TARGETS) + @echo API_ARMOR = $(API_ARMOR) + @echo ASAN = $(ASAN) + @echo BCC = $(BCC) + @echo BUILD_ZLIB = $(BUILD_ZLIB) + @echo CC = $(CC) + @echo CCOPTS = $(CCOPTS) + @echo CHECKER_DEPS = $(CHECKER_DEPS) + @echo CORE_CCONV_OPTS = $(CORE_CCONV_OPTS) + @echo CORE_COMPILE_OPTS = $(CORE_COMPILE_OPTS) + @echo CORE_LINK_DEP = $(CORE_LINK_DEP) + @echo CORE_LINK_OPTS = $(CORE_LINK_OPTS) + @echo CRTLIBPATH = $(CRTLIBPATH) + @echo CSC = $(CSC) + @echo DBFUZZ_COMPILE_OPTS = $(DBFUZZ_COMPILE_OPTS) + @echo DEBUG = $(DEBUG) + @echo DYNAMIC_SHELL = $(DYNAMIC_SHELL) + @echo EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) + @echo EXTHDR = $(EXTHDR) + @echo EXTRA_SRC = $(EXTRA_SRC) + @echo FOR_UWP = $(FOR_UWP) + @echo FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) + @echo FUZZCHECK_SRC = $(FUZZCHECK_SRC) + @echo FUZZDATA = $(FUZZDATA) + @echo FUZZERSHELL_COMPILE_OPTS = $(FUZZERSHELL_COMPILE_OPTS) + @echo HDR = $(HDR) + @echo ICUDIR = $(ICUDIR) + @echo ICUINCDIR = $(ICUINCDIR) + @echo ICULIBDIR = $(ICULIBDIR) + @echo JIM_TCLSH = $(JIM_TCLSH) + @echo KV_COMPILE_OPTS = $(KV_COMPILE_OPTS) + @echo LDFLAGS = $(LDFLAGS) + @echo LD = $(LD) + @echo LIBICU = $(LIBICU) + @echo LIBOBJ = $(LIBOBJ) + @echo LIBREADLINE = $(LIBREADLINE) + @echo LIBRESOBJS = $(LIBRESOBJS) + @echo LIBTCLPATH = $(LIBTCLPATH) + @echo LIBTCLSTUB = $(LIBTCLSTUB) + @echo LIBTCL = $(LIBTCL) + @echo LTCOMPILE = $(LTCOMPILE) + @echo LTLIB = $(LTLIB) + @echo LTLIBOPTS = $(LTLIBOPTS) + @echo LTLIBPATHS = $(LTLIBPATHS) + @echo LTLIBS = $(LTLIBS) + @echo LTLINK = $(LTLINK) + @echo LTLINKOPTS = $(LTLINKOPTS) + @echo LTRCOMPILE = $(LTRCOMPILE) + @echo MEMDEBUG = $(MEMDEBUG) + @echo MINIMAL_AMALGAMATION = $(MINIMAL_AMALGAMATION) + @echo MPTESTER_COMPILE_OPTS = $(MPTESTER_COMPILE_OPTS) + @echo NCC = $(NCC) + @echo NCRTLIBPATH = $(NCRTLIBPATH) + @echo NLTLIBPATHS = $(NLTLIBPATHS) + @echo NO_LINEMACROS = $(NO_LINEMACROS) + @echo NO_TCL = $(NO_TCL) + @echo NO_WARN = $(NO_WARN) + @echo NSDKLIBPATH = $(NSDKLIBPATH) + @echo NUCRTLIBPATH = $(NUCRTLIBPATH) + @echo OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) + @echo OPTIMIZATIONS = $(OPTIMIZATIONS) + @echo OSSSHELL_SRC = $(OSSSHELL_SRC) + @echo OSTRACE = $(OSTRACE) + @echo RBU = $(RBU) + @echo RCC = $(RCC) + @echo RC = $(RC) + @echo READLINE_FLAGS = $(READLINE_FLAGS) + @echo REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) + @echo RSYNC_OPT = $(RSYNC_OPT) + @echo RSYNC_SRC = $(RSYNC_SRC) + @echo SESSION = $(SESSION) + @echo SETLK_TIMEOUT = $(SETLK_TIMEOUT) + @echo SHELL_CCONV_OPTS = $(SHELL_CCONV_OPTS) + @echo SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) + @echo SHELL_CORE_DEP = $(SHELL_CORE_DEP) + @echo SHELL_CORE_LIB = $(SHELL_CORE_LIB) + @echo SHELL_CORE_SRC = $(SHELL_CORE_SRC) + @echo SHELL_DEP = $(SHELL_DEP) + @echo SHELL_LINK_OPTS = $(SHELL_LINK_OPTS) + @echo SPLIT_AMALGAMATION = $(SPLIT_AMALGAMATION) + @echo SQLITETCLDECLSH = $(SQLITETCLDECLSH) + @echo SQLITE_TCL_DEP = $(SQLITE_TCL_DEP) + @echo SQLITETCLH = $(SQLITETCLH) + @echo SRC = $(SRC) + @echo STATICALLY_LINK_TCL = $(STATICALLY_LINK_TCL) + @echo ST_COMPILE_OPTS = $(ST_COMPILE_OPTS) + @echo STORELIBPATH = $(STORELIBPATH) + @echo SYMBOLS = $(SYMBOLS) + @echo TCC = $(TCC) + @echo TCLDIR = $(TCLDIR) + @echo TCLINCDIR = $(TCLINCDIR) + @echo TCLLIBDIR = $(TCLLIBDIR) + @echo TCLLIBPATHS = $(TCLLIBPATHS) + @echo TCLLIBS = $(TCLLIBS) + @echo TCLSH_CMD = $(TCLSH_CMD) + @echo TCLSQLITEEX = $(TCLSQLITEEX) + @echo TCLSUFFIX = $(TCLSUFFIX) + @echo TCLVERSION = $(TCLVERSION) + @echo TEST_CCONV_OPTS = $(TEST_CCONV_OPTS) + @echo TESTEXT = $(TESTEXT) + @echo TESTFIXTURE_DEP = $(TESTFIXTURE_DEP) + @echo TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) + @echo TESTFIXTURE_SRC = $(TESTFIXTURE_SRC) + @echo TESTOPTS = $(TESTOPTS) + @echo TESTPROGS = $(TESTPROGS) + @echo TESTSRC = $(TESTSRC) + @echo TLIBS = $(TLIBS) + @echo TOP = $(TOP) + @echo UCRTLIBPATH = $(UCRTLIBPATH) + @echo USE_AMALGAMATION = $(USE_AMALGAMATION) + @echo USE_CRT_DLL = $(USE_CRT_DLL) + @echo USE_FATAL_WARN = $(USE_FATAL_WARN) + @echo USE_FULLWARN = $(USE_FULLWARN) + @echo USE_ICU = $(USE_ICU) + @echo USE_LISTINGS = $(USE_LISTINGS) + @echo USE_NATIVE_LIBPATHS = $(USE_NATIVE_LIBPATHS) + @echo USE_RC = $(USE_RC) + @echo USE_RUNTIME_CHECKS = $(USE_RUNTIME_CHECKS) + @echo USE_SEH = $(USE_SEH) + @echo USE_STDCALL = $(USE_STDCALL) + @echo USE_ZLIB = $(USE_ZLIB) + @echo XCOMPILE = $(XCOMPILE) + @echo ZLIBDIR = $(ZLIBDIR) + @echo ZLIBINCDIR = $(ZLIBINCDIR) + @echo ZLIBLIBDIR = $(ZLIBLIBDIR) + @echo ZLIBLIB = $(ZLIBLIB) moreclean: clean del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL @@ -2636,7 +2953,8 @@ moreclean: clean clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL - del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q sqlite3.def tclsqlite3.def ctime.c pragma.h 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL # <> del /Q $(SQLITE3TCLDLL) pkgIndex.tcl 2>NUL @@ -2652,7 +2970,9 @@ clean: del /Q lsm.dll lsmtest.exe 2>NUL del /Q atrc.exe changesetfuzz.exe dbtotxt.exe index_usage.exe 2>NUL del /Q testloadext.dll 2>NUL - del /Q testfixture.exe test.db 2>NUL + del /Q testfixture.exe test.db tf.bat 2>NUL + del /Q /S testdir 2>/NUL + -rmdir /Q /S testdir 2>NUL del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL @@ -2660,7 +2980,7 @@ clean: del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL del /Q sqlite3rc.h 2>NUL del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL - del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL + del /Q sqlite3_analyzer.exe sqlite3_analyzer.c sqlite3_rsync.exe 2>NUL del /Q sqlite-*-output.vsix 2>NUL del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL del /Q sqltclsh.* 2>NUL @@ -2668,5 +2988,6 @@ clean: del /Q kvtest.exe ossshell.exe scrub.exe 2>NUL del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL - del /Q lsm.h lsm1.c 2>NUL + del /q src-verify.exe 2>NUL + del /q jimsh.exe jimsh0.exe 2>NUL # <> diff --git a/README.md b/README.md index 127b09cca0..b18f572325 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,7 @@ SQLCipher is maintained by Zetetic, LLC, and additional information and document - 100% of data in the database file is encrypted - Good security practices (CBC mode, HMAC, key derivation) - Zero-configuration and application level cryptography -- Algorithms provided by the peer reviewed OpenSSL crypto library. -- Configurable crypto providers +- Support for multiple cryptographic providers ## Compatibility @@ -32,26 +31,18 @@ The SQLCipher team welcomes contributions to the core library. All contributions ## Compiling -Building SQLCipher is similar to compiling a regular version of SQLite from source, with a couple of small exceptions: +Building SQLCipher is similar to compiling a regular version of SQLite from source, with a few small exceptions. You must: - 1. You *must* define `SQLITE_HAS_CODEC` and either `SQLITE_TEMP_STORE=2` or `SQLITE_TEMP_STORE=3` - 2. You will need to link against a support cryptographic provider (OpenSSL, LibTomCrypt, CommonCrypto/Security.framework, or NSS) + 1. define `SQLITE_HAS_CODEC` + 2. define `SQLITE_TEMP_STORE=2` or `SQLITE_TEMP_STORE=3` (or use `configure`'s --with-tempstore=yes option) + 3. define `SQLITE_EXTRA_INIT=sqlcipher_extra_init` and `SQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown` + 4. define `SQLITE_THREADSAFE` to `1` or `2` (enabled automatically by `configure`) + 2. compile and link with a supported cryptographic provider (OpenSSL, LibTomCrypt, CommonCrypto/Security.framework, or NSS) -The following examples demonstrate linking against OpenSSL, which is a readily available provider on most Unix-like systems. +The following examples demonstrate use of OpenSSL, which is a readily available provider on most Unix-like systems. Note that, in this example, `--with-tempstore=yes` is setting `SQLITE_TEMP_STORE=2` for the build, and `SQLITE_THREADSAFE` has a default value of `1`. -Example 1. Static linking (replace /opt/local/lib with the path to libcrypto.a). Note in this -example, `--enable-tempstore=yes` is setting `SQLITE_TEMP_STORE=2` for the build. - -``` -$ ./configure --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC" \ - LDFLAGS="/opt/local/lib/libcrypto.a" -$ make ``` - -Example 2. Dynamic linking - -``` -$ ./configure --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC" \ +$ ./configure --with-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown" \ LDFLAGS="-lcrypto" $ make ``` @@ -65,7 +56,7 @@ As a result, the SQLCipher package includes it's own independent tests that exer To run SQLCipher specific tests, configure as described here and run the following to execute the tests and receive a report of the results: ``` -$ ./configure --enable-tempstore=yes --enable-fts5 CFLAGS="-DSQLITE_HAS_CODEC -DSQLCIPHER_TEST" \ +$ ./configure --with-tempstore=yes --enable-fts5 CFLAGS="-DSQLITE_HAS_CODEC -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown -DSQLCIPHER_TEST" \ LDFLAGS="-lcrypto" $ make testfixture $ ./testfixture test/sqlcipher.test @@ -133,7 +124,7 @@ support@zetetic.net! ## Community Edition Open Source License -Copyright (c) 2020, ZETETIC LLC +Copyright (c) 2025, ZETETIC LLC All rights reserved. Redistribution and use in source and binary forms, with or without @@ -163,14 +154,19 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

SQLite Source Repository

This repository contains the complete source code for the -[SQLite database engine](https://sqlite.org/). Some test scripts -are also included. However, many other test scripts -and most of the documentation are managed separately. +[SQLite database engine](https://sqlite.org/), including +many tests. Additional tests and most documentation +are managed separately. + +See the [on-line documentation](https://sqlite.org/) for more information +about what SQLite is and how it works from a user's perspective. This +README file is about the source code that goes into building SQLite, +not about how SQLite is used. ## Version Control SQLite sources are managed using -[Fossil](https://www.fossil-scm.org/), a distributed version control system +[Fossil](https://fossil-scm.org/), a distributed version control system that was specifically designed and written to support SQLite development. The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext. @@ -187,46 +183,62 @@ If you pulled your SQLite source code from a secondary source and want to verify its integrity, there are hints on how to do that in the [Verifying Code Authenticity](#vauth) section below. -## Obtaining The Code +## Contacting The SQLite Developers + +The preferred way to ask questions or make comments about SQLite or to +report bugs against SQLite is to visit the +[SQLite Forum](https://sqlite.org/forum) at . +Anonymous postings are permitted. + +If you think you have found a bug that has security implications and +you do not want to report it on the public forum, you can send a private +email to drh at sqlite dot org. + +## Public Domain + +The SQLite source code is in the public domain. See + for details. + +Because SQLite is in the public domain, we do not normally accept pull +requests, because if we did take a pull request, the changes in that +pull request might carry a copyright and the SQLite source code would +then no longer be fully in the public domain. + +## Obtaining The SQLite Source Code -If you do not want to use Fossil, you can download tarballs or ZIP -archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: +Source code tarballs or ZIP archives are available at: - * Latest trunk check-in as - [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz), - [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or - [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar). + * [Latest trunk check-in](https://sqlite.org/src/rchvdwnld/trunk). - * Latest release as - [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release), - [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or - [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release). + * [Latest release](https://sqlite.org/src/rchvdwnld/release) - * For other check-ins, substitute an appropriate branch name or - tag or hash prefix in place of "release" in the URLs of the previous - bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) - to locate the check-in desired, click on its information page link, - then click on the "Tarball" or "ZIP Archive" links on the information - page. + * For other check-ins, browse the + [project timeline](https://sqlite.org/src/timeline?y=ci) and + click on the check-in hash of the check-in you want to download. + On the resulting "info" page, click one of the options to the + right of the "**Downloads:**" label in the "**Overview**" section + near the top. -If you do want to use Fossil to check out the source tree, +To access sources directly using [Fossil](https://fossil-scm.org/home), first install Fossil version 2.0 or later. -(Source tarballs and precompiled binaries available -[here](https://www.fossil-scm.org/fossil/uv/download.html). Fossil is +Source tarballs and precompiled binaries for Fossil are available at +. Fossil is a stand-alone program. To install, simply download or build the single -executable file and put that file someplace on your $PATH.) +executable file and put that file someplace on your $PATH or %PATH%. Then run commands like this: - mkdir -p ~/sqlite ~/Fossils + mkdir -p ~/sqlite cd ~/sqlite - fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil - fossil open ~/Fossils/sqlite.fossil + fossil open https://sqlite.org/src -After setting up a repository using the steps above, you can always -update to the latest version using: +The initial "fossil open" command will take two or three minutes. Afterwards, +you can do fast, bandwidth-efficient updates to the whatever versions +of SQLite you like. Some examples: - fossil update trunk ;# latest trunk check-in - fossil update release ;# latest official release + fossil update trunk ;# latest trunk check-in + fossil update release ;# latest official release + fossil update trunk:2024-01-01 ;# First trunk check-in after 2024-01-01 + fossil update version-3.39.0 ;# Version 3.39.0 Or type "fossil ui" to get a web-based user interface. @@ -240,15 +252,42 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Run the makefile. - make sqlite3.c ;# Build the "amalgamation" source file - make test ;# Run some tests (requires Tcl) + apt install gcc make tcl-dev ;# Make sure you have all the necessary build tools + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make sqlite3 ;# Builds the "sqlite3" command-line tool + make sqlite3.c ;# Build the "amalgamation" source file + make sqldiff ;# Builds the "sqldiff" command-line tool + # Makefile targets below this point require tcl-dev + make tclextension-install ;# Build and install the SQLite TCL extension + make devtest ;# Run development tests + make releasetest ;# Run full release tests + make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool -See the makefile for additional targets. +See the makefile for additional targets. For debugging builds, the +core developers typically run "configure" with options like this: + + ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g' + +For release builds, the core developers usually do: + + ../sqlite/configure --enable-all + +Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or +later. The "tclextension-install" target and the test targets that follow +all require TCL development libraries too. ("apt install tcl-dev"). It is +helpful, but is not required, to install the SQLite TCL extension (the +"tclextension-install" target) prior to running tests. The "releasetest" +target has additional requirements, such as "valgrind". + +On "make" command-lines, one can add "OPTIONS=..." to specify additional +compile-time options over and above those set by ./configure. For example, +to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say: + + ./configure --enable-all + make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3 The configure script uses autoconf 2.61 and libtool. If the configure script does not work out for you, there is a generic makefile named @@ -256,60 +295,86 @@ script does not work out for you, there is a generic makefile named can copy and edit to suit your needs. Comments on the generic makefile show what changes are needed. -## Using MSVC for Windows systems - -On Windows, all applicable build products can be compiled with MSVC. -First open the command prompt window associated with the desired compiler -version (e.g. "Developer Command Prompt for VS2013"). Next, use NMAKE -with the provided "Makefile.msc" to build one of the supported targets. - -For example, from the parent directory of the source subtree named "sqlite": - - mkdir bld - cd bld - nmake /f ..\sqlite\Makefile.msc TOP=..\sqlite - nmake /f ..\sqlite\Makefile.msc sqlite3.c TOP=..\sqlite - nmake /f ..\sqlite\Makefile.msc sqlite3.dll TOP=..\sqlite - nmake /f ..\sqlite\Makefile.msc sqlite3.exe TOP=..\sqlite - nmake /f ..\sqlite\Makefile.msc test TOP=..\sqlite - -There are several build options that can be set via the NMAKE command -line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument -to the "sqlite3.dll" command line above. When debugging into the SQLite -code, adding the "DEBUG=1" argument to one of the above command lines is -recommended. - -SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation -is required by the makefiles (including those for MSVC). SQLite contains -a lot of generated code and Tcl is used to do much of that code generation. - -## Source Code Tour - -Most of the core source files are in the **src/** subdirectory. The -**src/** folder also contains files used to build the "testfixture" test -harness. The names of the source files used by "testfixture" all begin -with "test". -The **src/** also contains the "shell.c" file -which is the main program for the "sqlite3.exe" -[command-line shell](https://sqlite.org/cli.html) and -the "tclsqlite.c" file which implements the -[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite. -(Historical note: SQLite began as a Tcl -extension and only later escaped to the wild as an independent library.) - -Test scripts and programs are found in the **test/** subdirectory. -Additional test code is found in other source repositories. -See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for -additional information. - -The **ext/** subdirectory contains code for extensions. The -Full-text search engine is in **ext/fts3**. The R-Tree engine is in -**ext/rtree**. The **ext/misc** subdirectory contains a number of -smaller, single-file extensions, such as a REGEXP operator. - -The **tool/** subdirectory contains various scripts and programs used -for building generated source code files or for testing or for generating -accessory programs such as "sqlite3_analyzer(.exe)". +## Compiling for Windows Using MSVC + +On Windows, everything can be compiled with MSVC. +You will also need a working installation of TCL if you want to run tests. +TCL is not required if you just want to build SQLite itself. +See the [compile-for-windows.md](doc/compile-for-windows.md) document for +additional information about how to install MSVC and TCL and configure your +build environment. + +If you want to run tests, you need to let SQLite know the location of your +TCL library, using a command like this: + + set TCLDIR=c:\Tcl + +SQLite uses "tclsh.exe" as part of the build process, and so that +program will need to be somewhere on your %PATH%. SQLite itself +does not contain any TCL code, but it does use TCL to run tests. +You may need to install TCL development +libraries in order to successfully complete some makefile targets. +It is helpful, but is not required, to install the SQLite TCL extension +(the "tclextension-install" target) prior to running tests. + +Build using Makefile.msc. Example: + + nmake /f Makefile.msc sqlite3.exe + nmake /f Makefile.msc sqlite3.c + nmake /f Makefile.msc sqldiff.exe + # Makefile targets below this point require TCL development libraries + nmake /f Makefile.msc tclextension-install + nmake /f Makefile.msc devtest + nmake /f Makefile.msc releasetest + nmake /f Makefile.msc sqlite3_analyzer.exe + +There are many other makefile targets. See comments in Makefile.msc for +details. + +As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake +command-line to enable new compile-time options. For example: + + nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe + +## Source Tree Map + + * **src/** - This directory contains the primary source code for the + SQLite core. For historical reasons, C-code used for testing is + also found here. Source files intended for testing begin with "`test`". + The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface + for SQLite and are also not part of the core. + + * **test/** - This directory and its subdirectories contains code used + for testing. Files that end in "`.test`" are TCL scripts that run + tests using an augmented TCL interpreter named "testfixture". Use + a command like "`make testfixture`" (unix) or + "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that + augmented TCL interpreter, then run individual tests using commands like + "`testfixture test/main.test`". This test/ subdirectory also contains + additional C code modules and scripts for other kinds of testing. + + * **tool/** - This directory contains programs and scripts used to + build some of the machine-generated code that goes into the SQLite + core, as well as to build and run tests and perform diagnostics. + The source code to [the Lemon parser generator](./doc/lemon.html) is + found here. There are also TCL scripts used to build and/or transform + source code files. For example, the tool/mksqlite3h.tcl script reads + the src/sqlite.h.in file and uses it as a template to construct + the deliverable "sqlite3.h" file that defines the SQLite interface. + + * **ext/** - Various extensions to SQLite are found under this + directory. For example, the FTS5 subsystem is in "ext/fts5/". + Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built + into the SQLite amalgamation, but not all of them. The + "ext/misc/" subdirectory contains an assortment of one-file extensions, + many of which are omitted from the SQLite core, but which are included + in the [SQLite CLI](https://sqlite.org/cli.html). + + * **doc/** - Some documentation files about SQLite internals are found + here. Note, however, that the primary documentation designed for + application developers and users of SQLite is in a completely separate + repository. Note also that the primary API documentation is derived + from specially constructed comments in the src/sqlite.h.in file. ### Generated Source Code Files @@ -323,7 +388,7 @@ manually-edited files and automatically-generated files. The SQLite interface is defined by the **sqlite3.h** header file, which is generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The -[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. +[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. The manifest.uuid file contains the SHA3 hash of the particular check-in and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file contains the current SQLite version number. The sqlite3.h header is really @@ -390,33 +455,39 @@ individual source file exceeds 32K lines in length. ## How It All Fits Together SQLite is modular in design. -See the [architectural description](http://www.sqlite.org/arch.html) +See the [architectural description](https://sqlite.org/arch.html) for details. Other documents that are useful in -(helping to understand how SQLite works include the -[file format](http://www.sqlite.org/fileformat2.html) description, -the [virtual machine](http://www.sqlite.org/opcode.html) that runs +helping to understand how SQLite works include the +[file format](https://sqlite.org/fileformat2.html) description, +the [virtual machine](https://sqlite.org/opcode.html) that runs prepared statements, the description of -[how transactions work](http://www.sqlite.org/atomiccommit.html), and -the [overview of the query planner](http://www.sqlite.org/optoverview.html). +[how transactions work](https://sqlite.org/atomiccommit.html), and +the [overview of the query planner](https://sqlite.org/optoverview.html). -Years of effort have gone into optimizing SQLite, both +Decades of effort have gone into optimizing SQLite, both for small size and high performance. And optimizations tend to result in complex code. So there is a lot of complexity in the current SQLite implementation. It will not be the easiest library in the world to hack. -Key files: +### Key source code files * **sqlite.h.in** - This file defines the public interface to the SQLite library. Readers will need to be familiar with this interface before - trying to understand how the library works internally. + trying to understand how the library works internally. This file is + really a template that is transformed into the "sqlite3.h" deliverable + using a script invoked by the makefile. * **sqliteInt.h** - this header file defines many of the data objects used internally by SQLite. In addition to "sqliteInt.h", some - subsystems have their own header files. + subsystems inside of sQLite have their own header files. These internal + interfaces are not for use by applications. They can and do change + from one release of SQLite to the next. * **parse.y** - This file describes the LALR(1) grammar that SQLite uses to parse SQL statements, and the actions that are taken at each step - in the parsing process. + in the parsing process. The file is processed by the + [Lemon Parser Generator](./doc/lemon.html) to produce the actual C code + used for parsing. * **vdbe.c** - This file implements the virtual machine that runs prepared statements. There are various helper files whose names @@ -452,15 +523,17 @@ Key files: is not part of the core SQLite library. But as most of the tests in this repository are written in Tcl, the Tcl language bindings are important. - * **test*.c** - Files in the src/ folder that begin with "test" go into + * **test\*.c** - Files in the src/ folder that begin with "test" go into building the "testfixture.exe" program. The testfixture.exe program is an enhanced Tcl shell. The testfixture.exe program runs scripts in the test/ folder to validate the core SQLite code. The testfixture program (and some other test programs too) is built and run when you type "make test". - * **ext/misc/json1.c** - This file implements the various JSON functions - that are built into SQLite. + * **VERSION**, **manifest**, and **manifest.uuid** - These files define + the current SQLite version number. The "VERSION" file is human generated, + but the "manifest" and "manifest.uuid" files are automatically generated + by the [Fossil version control system](https://fossil-scm.org/). There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. @@ -469,8 +542,8 @@ describes its purpose and role within the larger system. ## Verifying Code Authenticity The `manifest` file at the root directory of the source tree -contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for -older files) for every source file in the repository. +contains either a SHA3-256 hash or a SHA1 hash +for every source file in the repository. The name of the version of the entire source tree is just the SHA3-256 hash of the `manifest` file itself, possibly with the last line of that file omitted if the last line begins with @@ -478,14 +551,29 @@ last line of that file omitted if the last line begins with The `manifest.uuid` file should contain the SHA3-256 hash of the `manifest` file. If all of the above hash comparisons are correct, then you can be confident that your source tree is authentic and unadulterated. +Details on the format for the `manifest` files are available +[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest). + +The process of checking source code authenticity is automated by the +makefile: + +> make verify-source -The format of the `manifest` file should be mostly self-explanatory, but -if you want details, they are available -[here](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest). +Or on windows: + +> nmake /f Makefile.msc verify-source + +Using the makefile to verify source integrity is good for detecting +accidental changes to the source tree, but malicious changes could be +hidden by also modifying the makefiles. ## Contacts -The main SQLite website is [http://www.sqlite.org/](http://www.sqlite.org/) +The main SQLite website is [https://sqlite.org/](https://sqlite.org/) with geographically distributed backups at -[http://www2.sqlite.org/](http://www2.sqlite.org) and -[http://www3.sqlite.org/](http://www3.sqlite.org). +[https://www2.sqlite.org/](https://www2.sqlite.org) and +[https://www3.sqlite.org/](https://www3.sqlite.org). + +Contact the SQLite developers through the +[SQLite Forum](https://sqlite.org/forum/). In an emergency, you +can send private email to the lead developer at drh at sqlite dot org. diff --git a/SQLCipher.podspec.json b/SQLCipher.podspec.json deleted file mode 100644 index d6a04af28e..0000000000 --- a/SQLCipher.podspec.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "authors": "Zetetic LLC", - "default_subspecs": "standard", - "description": "SQLCipher is an open source extension to SQLite that provides transparent 256-bit AES encryption of database files.", - "homepage": "https://www.zetetic.net/sqlcipher/", - "license": "BSD", - "name": "SQLCipher", - "platforms": { - "ios": "11.0", - "osx": "10.13", - "tvos": "11.0", - "watchos": "7.0" - }, - "prepare_command": "./configure --enable-tempstore=yes --with-crypto-lib=commoncrypto CFLAGS=\"-DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2 -DSQLITE_SOUNDEX -DSQLITE_THREADSAFE -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_STAT3 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_LOAD_EXTENSION -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_UNICODE61 -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 -DHAVE_USLEEP=1 -DSQLITE_MAX_VARIABLE_NUMBER=99999\"; make sqlite3.c", - "requires_arc": false, - "source": { - "git": "https://github.com/sqlcipher/sqlcipher.git", - "tag": "v4.5.5" - }, - "summary": "Full Database Encryption for SQLite.", - "version": "4.5.5", - "subspecs": [ - { - "compiler_flags": [ - "-DNDEBUG", - "-DSQLITE_HAS_CODEC", - "-DSQLITE_TEMP_STORE=2", - "-DSQLITE_SOUNDEX", - "-DSQLITE_THREADSAFE", - "-DSQLITE_ENABLE_RTREE", - "-DSQLITE_ENABLE_STAT3", - "-DSQLITE_ENABLE_STAT4", - "-DSQLITE_ENABLE_COLUMN_METADATA", - "-DSQLITE_ENABLE_MEMORY_MANAGEMENT", - "-DSQLITE_ENABLE_LOAD_EXTENSION", - "-DSQLITE_ENABLE_FTS4", - "-DSQLITE_ENABLE_FTS4_UNICODE61", - "-DSQLITE_ENABLE_FTS3_PARENTHESIS", - "-DSQLITE_ENABLE_UNLOCK_NOTIFY", - "-DSQLITE_ENABLE_JSON1", - "-DSQLITE_ENABLE_FTS5", - "-DSQLCIPHER_CRYPTO_CC", - "-DHAVE_USLEEP=1", - "-DSQLITE_MAX_VARIABLE_NUMBER=99999" - ], - "frameworks": [ - "Foundation", - "Security" - ], - "name": "common", - "source_files": "sqlite3.{h,c}", - "xcconfig": { - "HEADER_SEARCH_PATHS": "$(PODS_ROOT)/SQLCipher", - "GCC_PREPROCESSOR_DEFINITIONS": "$(inherited) SQLITE_HAS_CODEC=1", - "OTHER_CFLAGS": "$(inherited) -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2 -DSQLITE_SOUNDEX -DSQLITE_THREADSAFE -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_STAT3 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_LOAD_EXTENSION -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_UNICODE61 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 -DSQLCIPHER_CRYPTO_CC -DHAVE_USLEEP=1 -DSQLITE_MAX_VARIABLE_NUMBER=99999" - } - }, - { - "dependencies": { - "SQLCipher/common": [ - - ] - }, - "name": "standard" - }, - { - "compiler_flags": "", - "dependencies": { - "SQLCipher/common": [ - - ] - }, - "name": "fts", - "xcconfig": { - "OTHER_CFLAGS": "$(inherited)" - } - }, - { - "compiler_flags": "", - "dependencies": { - "SQLCipher/common": [ - - ] - }, - "name": "unlock_notify", - "xcconfig": { - "OTHER_CFLAGS": "$(inherited)" - } - } - ] -} diff --git a/SQLITE_LICENSE.md b/SQLITE_LICENSE.md new file mode 100644 index 0000000000..4029dc9e7f --- /dev/null +++ b/SQLITE_LICENSE.md @@ -0,0 +1,79 @@ +The SQLite source code, including all of the files in the directories +listed in the bullets below are +[Public Domain](https://sqlite.org/copyright.html). +The authors have submitted written affidavits releasing their work to +the public for any use. Every byte of the public-domain code can be +traced back to the original authors. The files of this repository +that are public domain include the following: + + * All of the primary SQLite source code files found in the + [src/ directory](https://sqlite.org/src/tree/src?type=tree&expand) + * All of the test cases and testing code in the + [test/ directory](https://sqlite.org/src/tree/test?type=tree&expand) + * All of the SQLite extension source code and test cases in the + [ext/ directory](https://sqlite.org/src/tree/ext?type=tree&expand) + * All code that ends up in the "sqlite3.c" and "sqlite3.h" build products + that actually implement the SQLite RDBMS. + * All of the code used to compile the + [command-line interface](https://sqlite.org/cli.html) + * All of the code used to build various utility programs such as + "sqldiff", "sqlite3_rsync", and "sqlite3_analyzer". + + +The public domain source files usually contain a header comment +similar to the following to make it clear that the software is +public domain. + +> The author disclaims copyright to this source code. In place of +> a legal notice, here is a blessing: +> +> * May you do good and not evil. +> * May you find forgiveness for yourself and forgive others. +> * May you share freely, never taking more than you give. + +Almost every file you find in this source repository will be +public domain. But there are a small number of exceptions: + +Non-Public-Domain Code Included With This Source Repository AS A Convenience +---------------------------------------------------------------------------- + +This repository contains a (relatively) small amount of non-public-domain +code used to help implement the configuration and build logic. In other +words, there are some non-public-domain files used to implement: + +> ./configure && make + +In all cases, the non-public-domain files included with this +repository have generous BSD-style licenses. So anyone is free to +use any of the code in this source repository for any purpose, though +attribution may be required to reuse or republish the configure and +build scripts. None of the non-public-domain code ever actually reaches +the build products, such as "sqlite3.c", however, so no attribution is +required to use SQLite itself. The non-public-domain code consists of +scripts used to help compile SQLite. The non-public-domain code is +technically not part of SQLite. The non-public-domain code is +included in this repository as a convenience to developers, so that those +who want to build SQLite do not need to go download a bunch of +third-party build scripts in order to compile SQLite. + +Non-public-domain code included in this respository includes: + + * The ["autosetup"](http://msteveb.github.io/autosetup/) configuration + system that is contained (mostly) in the autosetup/ directory, but also + includes the "./configure" script at the top-level of this archive. + Autosetup has a separate BSD-style license. See the + [autosetup/LICENSE](http://msteveb.github.io/autosetup/license/) + for details. + + * There are BSD-style licenses on some of the configuration + software found in the legacy autoconf/ directory and its + subdirectories. + +The following unix shell command can be run from the top-level +of this source repository in order to remove all non-public-domain +code: + +> rm -rf configure autosetup autoconf + +If you unpack this source repository and then run the command above, what +is left will be 100% public domain. diff --git a/VERSION b/VERSION index 476cebe063..0306a564de 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.42.0 +3.51.2 diff --git a/aclocal.m4 b/aclocal.m4 deleted file mode 100644 index 8ce4d37a7a..0000000000 --- a/aclocal.m4 +++ /dev/null @@ -1,9068 +0,0 @@ -# generated automatically by aclocal 1.16.1 -*- Autoconf -*- - -# Copyright (C) 1996-2018 Free Software Foundation, Inc. - -# This file is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY, to the extent permitted by law; without -# even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. - -m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) -# libtool.m4 - Configure libtool for the host system. -*-Autoconf-*- -# -# Copyright (C) 1996-2001, 2003-2015 Free Software Foundation, Inc. -# Written by Gordon Matzigkeit, 1996 -# -# This file is free software; the Free Software Foundation gives -# unlimited permission to copy and/or distribute it, with or without -# modifications, as long as this notice is preserved. - -m4_define([_LT_COPYING], [dnl -# Copyright (C) 2014 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# GNU Libtool is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of of the License, or -# (at your option) any later version. -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program or library that is built -# using GNU Libtool, you may include this file under the same -# distribution terms that you use for the rest of that program. -# -# GNU Libtool is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -]) - -# serial 58 LT_INIT - - -# LT_PREREQ(VERSION) -# ------------------ -# Complain and exit if this libtool version is less that VERSION. -m4_defun([LT_PREREQ], -[m4_if(m4_version_compare(m4_defn([LT_PACKAGE_VERSION]), [$1]), -1, - [m4_default([$3], - [m4_fatal([Libtool version $1 or higher is required], - 63)])], - [$2])]) - - -# _LT_CHECK_BUILDDIR -# ------------------ -# Complain if the absolute build directory name contains unusual characters -m4_defun([_LT_CHECK_BUILDDIR], -[case `pwd` in - *\ * | *\ *) - AC_MSG_WARN([Libtool does not cope well with whitespace in `pwd`]) ;; -esac -]) - - -# LT_INIT([OPTIONS]) -# ------------------ -AC_DEFUN([LT_INIT], -[AC_PREREQ([2.62])dnl We use AC_PATH_PROGS_FEATURE_CHECK -AC_REQUIRE([AC_CONFIG_AUX_DIR_DEFAULT])dnl -AC_BEFORE([$0], [LT_LANG])dnl -AC_BEFORE([$0], [LT_OUTPUT])dnl -AC_BEFORE([$0], [LTDL_INIT])dnl -m4_require([_LT_CHECK_BUILDDIR])dnl - -dnl Autoconf doesn't catch unexpanded LT_ macros by default: -m4_pattern_forbid([^_?LT_[A-Z_]+$])dnl -m4_pattern_allow([^(_LT_EOF|LT_DLGLOBAL|LT_DLLAZY_OR_NOW|LT_MULTI_MODULE)$])dnl -dnl aclocal doesn't pull ltoptions.m4, ltsugar.m4, or ltversion.m4 -dnl unless we require an AC_DEFUNed macro: -AC_REQUIRE([LTOPTIONS_VERSION])dnl -AC_REQUIRE([LTSUGAR_VERSION])dnl -AC_REQUIRE([LTVERSION_VERSION])dnl -AC_REQUIRE([LTOBSOLETE_VERSION])dnl -m4_require([_LT_PROG_LTMAIN])dnl - -_LT_SHELL_INIT([SHELL=${CONFIG_SHELL-/bin/sh}]) - -dnl Parse OPTIONS -_LT_SET_OPTIONS([$0], [$1]) - -# This can be used to rebuild libtool when needed -LIBTOOL_DEPS=$ltmain - -# Always use our own libtool. -LIBTOOL='$(SHELL) $(top_builddir)/libtool' -AC_SUBST(LIBTOOL)dnl - -_LT_SETUP - -# Only expand once: -m4_define([LT_INIT]) -])# LT_INIT - -# Old names: -AU_ALIAS([AC_PROG_LIBTOOL], [LT_INIT]) -AU_ALIAS([AM_PROG_LIBTOOL], [LT_INIT]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_PROG_LIBTOOL], []) -dnl AC_DEFUN([AM_PROG_LIBTOOL], []) - - -# _LT_PREPARE_CC_BASENAME -# ----------------------- -m4_defun([_LT_PREPARE_CC_BASENAME], [ -# Calculate cc_basename. Skip known compiler wrappers and cross-prefix. -func_cc_basename () -{ - for cc_temp in @S|@*""; do - case $cc_temp in - compile | *[[\\/]]compile | ccache | *[[\\/]]ccache ) ;; - distcc | *[[\\/]]distcc | purify | *[[\\/]]purify ) ;; - \-*) ;; - *) break;; - esac - done - func_cc_basename_result=`$ECHO "$cc_temp" | $SED "s%.*/%%; s%^$host_alias-%%"` -} -])# _LT_PREPARE_CC_BASENAME - - -# _LT_CC_BASENAME(CC) -# ------------------- -# It would be clearer to call AC_REQUIREs from _LT_PREPARE_CC_BASENAME, -# but that macro is also expanded into generated libtool script, which -# arranges for $SED and $ECHO to be set by different means. -m4_defun([_LT_CC_BASENAME], -[m4_require([_LT_PREPARE_CC_BASENAME])dnl -AC_REQUIRE([_LT_DECL_SED])dnl -AC_REQUIRE([_LT_PROG_ECHO_BACKSLASH])dnl -func_cc_basename $1 -cc_basename=$func_cc_basename_result -]) - - -# _LT_FILEUTILS_DEFAULTS -# ---------------------- -# It is okay to use these file commands and assume they have been set -# sensibly after 'm4_require([_LT_FILEUTILS_DEFAULTS])'. -m4_defun([_LT_FILEUTILS_DEFAULTS], -[: ${CP="cp -f"} -: ${MV="mv -f"} -: ${RM="rm -f"} -])# _LT_FILEUTILS_DEFAULTS - - -# _LT_SETUP -# --------- -m4_defun([_LT_SETUP], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -AC_REQUIRE([AC_CANONICAL_BUILD])dnl -AC_REQUIRE([_LT_PREPARE_SED_QUOTE_VARS])dnl -AC_REQUIRE([_LT_PROG_ECHO_BACKSLASH])dnl - -_LT_DECL([], [PATH_SEPARATOR], [1], [The PATH separator for the build system])dnl -dnl -_LT_DECL([], [host_alias], [0], [The host system])dnl -_LT_DECL([], [host], [0])dnl -_LT_DECL([], [host_os], [0])dnl -dnl -_LT_DECL([], [build_alias], [0], [The build system])dnl -_LT_DECL([], [build], [0])dnl -_LT_DECL([], [build_os], [0])dnl -dnl -AC_REQUIRE([AC_PROG_CC])dnl -AC_REQUIRE([LT_PATH_LD])dnl -AC_REQUIRE([LT_PATH_NM])dnl -dnl -AC_REQUIRE([AC_PROG_LN_S])dnl -test -z "$LN_S" && LN_S="ln -s" -_LT_DECL([], [LN_S], [1], [Whether we need soft or hard links])dnl -dnl -AC_REQUIRE([LT_CMD_MAX_LEN])dnl -_LT_DECL([objext], [ac_objext], [0], [Object file suffix (normally "o")])dnl -_LT_DECL([], [exeext], [0], [Executable file suffix (normally "")])dnl -dnl -m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_CHECK_SHELL_FEATURES])dnl -m4_require([_LT_PATH_CONVERSION_FUNCTIONS])dnl -m4_require([_LT_CMD_RELOAD])dnl -m4_require([_LT_CHECK_MAGIC_METHOD])dnl -m4_require([_LT_CHECK_SHAREDLIB_FROM_LINKLIB])dnl -m4_require([_LT_CMD_OLD_ARCHIVE])dnl -m4_require([_LT_CMD_GLOBAL_SYMBOLS])dnl -m4_require([_LT_WITH_SYSROOT])dnl -m4_require([_LT_CMD_TRUNCATE])dnl - -_LT_CONFIG_LIBTOOL_INIT([ -# See if we are running on zsh, and set the options that allow our -# commands through without removal of \ escapes INIT. -if test -n "\${ZSH_VERSION+set}"; then - setopt NO_GLOB_SUBST -fi -]) -if test -n "${ZSH_VERSION+set}"; then - setopt NO_GLOB_SUBST -fi - -_LT_CHECK_OBJDIR - -m4_require([_LT_TAG_COMPILER])dnl - -case $host_os in -aix3*) - # AIX sometimes has problems with the GCC collect2 program. For some - # reason, if we set the COLLECT_NAMES environment variable, the problems - # vanish in a puff of smoke. - if test set != "${COLLECT_NAMES+set}"; then - COLLECT_NAMES= - export COLLECT_NAMES - fi - ;; -esac - -# Global variables: -ofile=libtool -can_build_shared=yes - -# All known linkers require a '.a' archive for static linking (except MSVC, -# which needs '.lib'). -libext=a - -with_gnu_ld=$lt_cv_prog_gnu_ld - -old_CC=$CC -old_CFLAGS=$CFLAGS - -# Set sane defaults for various variables -test -z "$CC" && CC=cc -test -z "$LTCC" && LTCC=$CC -test -z "$LTCFLAGS" && LTCFLAGS=$CFLAGS -test -z "$LD" && LD=ld -test -z "$ac_objext" && ac_objext=o - -_LT_CC_BASENAME([$compiler]) - -# Only perform the check for file, if the check method requires it -test -z "$MAGIC_CMD" && MAGIC_CMD=file -case $deplibs_check_method in -file_magic*) - if test "$file_magic_cmd" = '$MAGIC_CMD'; then - _LT_PATH_MAGIC - fi - ;; -esac - -# Use C for the default configuration in the libtool script -LT_SUPPORTED_TAG([CC]) -_LT_LANG_C_CONFIG -_LT_LANG_DEFAULT_CONFIG -_LT_CONFIG_COMMANDS -])# _LT_SETUP - - -# _LT_PREPARE_SED_QUOTE_VARS -# -------------------------- -# Define a few sed substitution that help us do robust quoting. -m4_defun([_LT_PREPARE_SED_QUOTE_VARS], -[# Backslashify metacharacters that are still active within -# double-quoted strings. -sed_quote_subst='s/\([["`$\\]]\)/\\\1/g' - -# Same as above, but do not quote variable references. -double_quote_subst='s/\([["`\\]]\)/\\\1/g' - -# Sed substitution to delay expansion of an escaped shell variable in a -# double_quote_subst'ed string. -delay_variable_subst='s/\\\\\\\\\\\$/\\\\\\$/g' - -# Sed substitution to delay expansion of an escaped single quote. -delay_single_quote_subst='s/'\''/'\'\\\\\\\'\''/g' - -# Sed substitution to avoid accidental globbing in evaled expressions -no_glob_subst='s/\*/\\\*/g' -]) - -# _LT_PROG_LTMAIN -# --------------- -# Note that this code is called both from 'configure', and 'config.status' -# now that we use AC_CONFIG_COMMANDS to generate libtool. Notably, -# 'config.status' has no value for ac_aux_dir unless we are using Automake, -# so we pass a copy along to make sure it has a sensible value anyway. -m4_defun([_LT_PROG_LTMAIN], -[m4_ifdef([AC_REQUIRE_AUX_FILE], [AC_REQUIRE_AUX_FILE([ltmain.sh])])dnl -_LT_CONFIG_LIBTOOL_INIT([ac_aux_dir='$ac_aux_dir']) -ltmain=$ac_aux_dir/ltmain.sh -])# _LT_PROG_LTMAIN - - - -# So that we can recreate a full libtool script including additional -# tags, we accumulate the chunks of code to send to AC_CONFIG_COMMANDS -# in macros and then make a single call at the end using the 'libtool' -# label. - - -# _LT_CONFIG_LIBTOOL_INIT([INIT-COMMANDS]) -# ---------------------------------------- -# Register INIT-COMMANDS to be passed to AC_CONFIG_COMMANDS later. -m4_define([_LT_CONFIG_LIBTOOL_INIT], -[m4_ifval([$1], - [m4_append([_LT_OUTPUT_LIBTOOL_INIT], - [$1 -])])]) - -# Initialize. -m4_define([_LT_OUTPUT_LIBTOOL_INIT]) - - -# _LT_CONFIG_LIBTOOL([COMMANDS]) -# ------------------------------ -# Register COMMANDS to be passed to AC_CONFIG_COMMANDS later. -m4_define([_LT_CONFIG_LIBTOOL], -[m4_ifval([$1], - [m4_append([_LT_OUTPUT_LIBTOOL_COMMANDS], - [$1 -])])]) - -# Initialize. -m4_define([_LT_OUTPUT_LIBTOOL_COMMANDS]) - - -# _LT_CONFIG_SAVE_COMMANDS([COMMANDS], [INIT_COMMANDS]) -# ----------------------------------------------------- -m4_defun([_LT_CONFIG_SAVE_COMMANDS], -[_LT_CONFIG_LIBTOOL([$1]) -_LT_CONFIG_LIBTOOL_INIT([$2]) -]) - - -# _LT_FORMAT_COMMENT([COMMENT]) -# ----------------------------- -# Add leading comment marks to the start of each line, and a trailing -# full-stop to the whole comment if one is not present already. -m4_define([_LT_FORMAT_COMMENT], -[m4_ifval([$1], [ -m4_bpatsubst([m4_bpatsubst([$1], [^ *], [# ])], - [['`$\]], [\\\&])]m4_bmatch([$1], [[!?.]$], [], [.]) -)]) - - - - - -# _LT_DECL([CONFIGNAME], VARNAME, VALUE, [DESCRIPTION], [IS-TAGGED?]) -# ------------------------------------------------------------------- -# CONFIGNAME is the name given to the value in the libtool script. -# VARNAME is the (base) name used in the configure script. -# VALUE may be 0, 1 or 2 for a computed quote escaped value based on -# VARNAME. Any other value will be used directly. -m4_define([_LT_DECL], -[lt_if_append_uniq([lt_decl_varnames], [$2], [, ], - [lt_dict_add_subkey([lt_decl_dict], [$2], [libtool_name], - [m4_ifval([$1], [$1], [$2])]) - lt_dict_add_subkey([lt_decl_dict], [$2], [value], [$3]) - m4_ifval([$4], - [lt_dict_add_subkey([lt_decl_dict], [$2], [description], [$4])]) - lt_dict_add_subkey([lt_decl_dict], [$2], - [tagged?], [m4_ifval([$5], [yes], [no])])]) -]) - - -# _LT_TAGDECL([CONFIGNAME], VARNAME, VALUE, [DESCRIPTION]) -# -------------------------------------------------------- -m4_define([_LT_TAGDECL], [_LT_DECL([$1], [$2], [$3], [$4], [yes])]) - - -# lt_decl_tag_varnames([SEPARATOR], [VARNAME1...]) -# ------------------------------------------------ -m4_define([lt_decl_tag_varnames], -[_lt_decl_filter([tagged?], [yes], $@)]) - - -# _lt_decl_filter(SUBKEY, VALUE, [SEPARATOR], [VARNAME1..]) -# --------------------------------------------------------- -m4_define([_lt_decl_filter], -[m4_case([$#], - [0], [m4_fatal([$0: too few arguments: $#])], - [1], [m4_fatal([$0: too few arguments: $#: $1])], - [2], [lt_dict_filter([lt_decl_dict], [$1], [$2], [], lt_decl_varnames)], - [3], [lt_dict_filter([lt_decl_dict], [$1], [$2], [$3], lt_decl_varnames)], - [lt_dict_filter([lt_decl_dict], $@)])[]dnl -]) - - -# lt_decl_quote_varnames([SEPARATOR], [VARNAME1...]) -# -------------------------------------------------- -m4_define([lt_decl_quote_varnames], -[_lt_decl_filter([value], [1], $@)]) - - -# lt_decl_dquote_varnames([SEPARATOR], [VARNAME1...]) -# --------------------------------------------------- -m4_define([lt_decl_dquote_varnames], -[_lt_decl_filter([value], [2], $@)]) - - -# lt_decl_varnames_tagged([SEPARATOR], [VARNAME1...]) -# --------------------------------------------------- -m4_define([lt_decl_varnames_tagged], -[m4_assert([$# <= 2])dnl -_$0(m4_quote(m4_default([$1], [[, ]])), - m4_ifval([$2], [[$2]], [m4_dquote(lt_decl_tag_varnames)]), - m4_split(m4_normalize(m4_quote(_LT_TAGS)), [ ]))]) -m4_define([_lt_decl_varnames_tagged], -[m4_ifval([$3], [lt_combine([$1], [$2], [_], $3)])]) - - -# lt_decl_all_varnames([SEPARATOR], [VARNAME1...]) -# ------------------------------------------------ -m4_define([lt_decl_all_varnames], -[_$0(m4_quote(m4_default([$1], [[, ]])), - m4_if([$2], [], - m4_quote(lt_decl_varnames), - m4_quote(m4_shift($@))))[]dnl -]) -m4_define([_lt_decl_all_varnames], -[lt_join($@, lt_decl_varnames_tagged([$1], - lt_decl_tag_varnames([[, ]], m4_shift($@))))dnl -]) - - -# _LT_CONFIG_STATUS_DECLARE([VARNAME]) -# ------------------------------------ -# Quote a variable value, and forward it to 'config.status' so that its -# declaration there will have the same value as in 'configure'. VARNAME -# must have a single quote delimited value for this to work. -m4_define([_LT_CONFIG_STATUS_DECLARE], -[$1='`$ECHO "$][$1" | $SED "$delay_single_quote_subst"`']) - - -# _LT_CONFIG_STATUS_DECLARATIONS -# ------------------------------ -# We delimit libtool config variables with single quotes, so when -# we write them to config.status, we have to be sure to quote all -# embedded single quotes properly. In configure, this macro expands -# each variable declared with _LT_DECL (and _LT_TAGDECL) into: -# -# ='`$ECHO "$" | $SED "$delay_single_quote_subst"`' -m4_defun([_LT_CONFIG_STATUS_DECLARATIONS], -[m4_foreach([_lt_var], m4_quote(lt_decl_all_varnames), - [m4_n([_LT_CONFIG_STATUS_DECLARE(_lt_var)])])]) - - -# _LT_LIBTOOL_TAGS -# ---------------- -# Output comment and list of tags supported by the script -m4_defun([_LT_LIBTOOL_TAGS], -[_LT_FORMAT_COMMENT([The names of the tagged configurations supported by this script])dnl -available_tags='_LT_TAGS'dnl -]) - - -# _LT_LIBTOOL_DECLARE(VARNAME, [TAG]) -# ----------------------------------- -# Extract the dictionary values for VARNAME (optionally with TAG) and -# expand to a commented shell variable setting: -# -# # Some comment about what VAR is for. -# visible_name=$lt_internal_name -m4_define([_LT_LIBTOOL_DECLARE], -[_LT_FORMAT_COMMENT(m4_quote(lt_dict_fetch([lt_decl_dict], [$1], - [description])))[]dnl -m4_pushdef([_libtool_name], - m4_quote(lt_dict_fetch([lt_decl_dict], [$1], [libtool_name])))[]dnl -m4_case(m4_quote(lt_dict_fetch([lt_decl_dict], [$1], [value])), - [0], [_libtool_name=[$]$1], - [1], [_libtool_name=$lt_[]$1], - [2], [_libtool_name=$lt_[]$1], - [_libtool_name=lt_dict_fetch([lt_decl_dict], [$1], [value])])[]dnl -m4_ifval([$2], [_$2])[]m4_popdef([_libtool_name])[]dnl -]) - - -# _LT_LIBTOOL_CONFIG_VARS -# ----------------------- -# Produce commented declarations of non-tagged libtool config variables -# suitable for insertion in the LIBTOOL CONFIG section of the 'libtool' -# script. Tagged libtool config variables (even for the LIBTOOL CONFIG -# section) are produced by _LT_LIBTOOL_TAG_VARS. -m4_defun([_LT_LIBTOOL_CONFIG_VARS], -[m4_foreach([_lt_var], - m4_quote(_lt_decl_filter([tagged?], [no], [], lt_decl_varnames)), - [m4_n([_LT_LIBTOOL_DECLARE(_lt_var)])])]) - - -# _LT_LIBTOOL_TAG_VARS(TAG) -# ------------------------- -m4_define([_LT_LIBTOOL_TAG_VARS], -[m4_foreach([_lt_var], m4_quote(lt_decl_tag_varnames), - [m4_n([_LT_LIBTOOL_DECLARE(_lt_var, [$1])])])]) - - -# _LT_TAGVAR(VARNAME, [TAGNAME]) -# ------------------------------ -m4_define([_LT_TAGVAR], [m4_ifval([$2], [$1_$2], [$1])]) - - -# _LT_CONFIG_COMMANDS -# ------------------- -# Send accumulated output to $CONFIG_STATUS. Thanks to the lists of -# variables for single and double quote escaping we saved from calls -# to _LT_DECL, we can put quote escaped variables declarations -# into 'config.status', and then the shell code to quote escape them in -# for loops in 'config.status'. Finally, any additional code accumulated -# from calls to _LT_CONFIG_LIBTOOL_INIT is expanded. -m4_defun([_LT_CONFIG_COMMANDS], -[AC_PROVIDE_IFELSE([LT_OUTPUT], - dnl If the libtool generation code has been placed in $CONFIG_LT, - dnl instead of duplicating it all over again into config.status, - dnl then we will have config.status run $CONFIG_LT later, so it - dnl needs to know what name is stored there: - [AC_CONFIG_COMMANDS([libtool], - [$SHELL $CONFIG_LT || AS_EXIT(1)], [CONFIG_LT='$CONFIG_LT'])], - dnl If the libtool generation code is destined for config.status, - dnl expand the accumulated commands and init code now: - [AC_CONFIG_COMMANDS([libtool], - [_LT_OUTPUT_LIBTOOL_COMMANDS], [_LT_OUTPUT_LIBTOOL_COMMANDS_INIT])]) -])#_LT_CONFIG_COMMANDS - - -# Initialize. -m4_define([_LT_OUTPUT_LIBTOOL_COMMANDS_INIT], -[ - -# The HP-UX ksh and POSIX shell print the target directory to stdout -# if CDPATH is set. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH - -sed_quote_subst='$sed_quote_subst' -double_quote_subst='$double_quote_subst' -delay_variable_subst='$delay_variable_subst' -_LT_CONFIG_STATUS_DECLARATIONS -LTCC='$LTCC' -LTCFLAGS='$LTCFLAGS' -compiler='$compiler_DEFAULT' - -# A function that is used when there is no print builtin or printf. -func_fallback_echo () -{ - eval 'cat <<_LTECHO_EOF -\$[]1 -_LTECHO_EOF' -} - -# Quote evaled strings. -for var in lt_decl_all_varnames([[ \ -]], lt_decl_quote_varnames); do - case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in - *[[\\\\\\\`\\"\\\$]]*) - eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED \\"\\\$sed_quote_subst\\"\\\`\\\\\\"" ## exclude from sc_prohibit_nested_quotes - ;; - *) - eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" - ;; - esac -done - -# Double-quote double-evaled strings. -for var in lt_decl_all_varnames([[ \ -]], lt_decl_dquote_varnames); do - case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in - *[[\\\\\\\`\\"\\\$]]*) - eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\"" ## exclude from sc_prohibit_nested_quotes - ;; - *) - eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" - ;; - esac -done - -_LT_OUTPUT_LIBTOOL_INIT -]) - -# _LT_GENERATED_FILE_INIT(FILE, [COMMENT]) -# ------------------------------------ -# Generate a child script FILE with all initialization necessary to -# reuse the environment learned by the parent script, and make the -# file executable. If COMMENT is supplied, it is inserted after the -# '#!' sequence but before initialization text begins. After this -# macro, additional text can be appended to FILE to form the body of -# the child script. The macro ends with non-zero status if the -# file could not be fully written (such as if the disk is full). -m4_ifdef([AS_INIT_GENERATED], -[m4_defun([_LT_GENERATED_FILE_INIT],[AS_INIT_GENERATED($@)])], -[m4_defun([_LT_GENERATED_FILE_INIT], -[m4_require([AS_PREPARE])]dnl -[m4_pushdef([AS_MESSAGE_LOG_FD])]dnl -[lt_write_fail=0 -cat >$1 <<_ASEOF || lt_write_fail=1 -#! $SHELL -# Generated by $as_me. -$2 -SHELL=\${CONFIG_SHELL-$SHELL} -export SHELL -_ASEOF -cat >>$1 <<\_ASEOF || lt_write_fail=1 -AS_SHELL_SANITIZE -_AS_PREPARE -exec AS_MESSAGE_FD>&1 -_ASEOF -test 0 = "$lt_write_fail" && chmod +x $1[]dnl -m4_popdef([AS_MESSAGE_LOG_FD])])])# _LT_GENERATED_FILE_INIT - -# LT_OUTPUT -# --------- -# This macro allows early generation of the libtool script (before -# AC_OUTPUT is called), incase it is used in configure for compilation -# tests. -AC_DEFUN([LT_OUTPUT], -[: ${CONFIG_LT=./config.lt} -AC_MSG_NOTICE([creating $CONFIG_LT]) -_LT_GENERATED_FILE_INIT(["$CONFIG_LT"], -[# Run this file to recreate a libtool stub with the current configuration.]) - -cat >>"$CONFIG_LT" <<\_LTEOF -lt_cl_silent=false -exec AS_MESSAGE_LOG_FD>>config.log -{ - echo - AS_BOX([Running $as_me.]) -} >&AS_MESSAGE_LOG_FD - -lt_cl_help="\ -'$as_me' creates a local libtool stub from the current configuration, -for use in further configure time tests before the real libtool is -generated. - -Usage: $[0] [[OPTIONS]] - - -h, --help print this help, then exit - -V, --version print version number, then exit - -q, --quiet do not print progress messages - -d, --debug don't remove temporary files - -Report bugs to ." - -lt_cl_version="\ -m4_ifset([AC_PACKAGE_NAME], [AC_PACKAGE_NAME ])config.lt[]dnl -m4_ifset([AC_PACKAGE_VERSION], [ AC_PACKAGE_VERSION]) -configured by $[0], generated by m4_PACKAGE_STRING. - -Copyright (C) 2011 Free Software Foundation, Inc. -This config.lt script is free software; the Free Software Foundation -gives unlimited permision to copy, distribute and modify it." - -while test 0 != $[#] -do - case $[1] in - --version | --v* | -V ) - echo "$lt_cl_version"; exit 0 ;; - --help | --h* | -h ) - echo "$lt_cl_help"; exit 0 ;; - --debug | --d* | -d ) - debug=: ;; - --quiet | --q* | --silent | --s* | -q ) - lt_cl_silent=: ;; - - -*) AC_MSG_ERROR([unrecognized option: $[1] -Try '$[0] --help' for more information.]) ;; - - *) AC_MSG_ERROR([unrecognized argument: $[1] -Try '$[0] --help' for more information.]) ;; - esac - shift -done - -if $lt_cl_silent; then - exec AS_MESSAGE_FD>/dev/null -fi -_LTEOF - -cat >>"$CONFIG_LT" <<_LTEOF -_LT_OUTPUT_LIBTOOL_COMMANDS_INIT -_LTEOF - -cat >>"$CONFIG_LT" <<\_LTEOF -AC_MSG_NOTICE([creating $ofile]) -_LT_OUTPUT_LIBTOOL_COMMANDS -AS_EXIT(0) -_LTEOF -chmod +x "$CONFIG_LT" - -# configure is writing to config.log, but config.lt does its own redirection, -# appending to config.log, which fails on DOS, as config.log is still kept -# open by configure. Here we exec the FD to /dev/null, effectively closing -# config.log, so it can be properly (re)opened and appended to by config.lt. -lt_cl_success=: -test yes = "$silent" && - lt_config_lt_args="$lt_config_lt_args --quiet" -exec AS_MESSAGE_LOG_FD>/dev/null -$SHELL "$CONFIG_LT" $lt_config_lt_args || lt_cl_success=false -exec AS_MESSAGE_LOG_FD>>config.log -$lt_cl_success || AS_EXIT(1) -])# LT_OUTPUT - - -# _LT_CONFIG(TAG) -# --------------- -# If TAG is the built-in tag, create an initial libtool script with a -# default configuration from the untagged config vars. Otherwise add code -# to config.status for appending the configuration named by TAG from the -# matching tagged config vars. -m4_defun([_LT_CONFIG], -[m4_require([_LT_FILEUTILS_DEFAULTS])dnl -_LT_CONFIG_SAVE_COMMANDS([ - m4_define([_LT_TAG], m4_if([$1], [], [C], [$1]))dnl - m4_if(_LT_TAG, [C], [ - # See if we are running on zsh, and set the options that allow our - # commands through without removal of \ escapes. - if test -n "${ZSH_VERSION+set}"; then - setopt NO_GLOB_SUBST - fi - - cfgfile=${ofile}T - trap "$RM \"$cfgfile\"; exit 1" 1 2 15 - $RM "$cfgfile" - - cat <<_LT_EOF >> "$cfgfile" -#! $SHELL -# Generated automatically by $as_me ($PACKAGE) $VERSION -# NOTE: Changes made to this file will be lost: look at ltmain.sh. - -# Provide generalized library-building support services. -# Written by Gordon Matzigkeit, 1996 - -_LT_COPYING -_LT_LIBTOOL_TAGS - -# Configured defaults for sys_lib_dlsearch_path munging. -: \${LT_SYS_LIBRARY_PATH="$configure_time_lt_sys_library_path"} - -# ### BEGIN LIBTOOL CONFIG -_LT_LIBTOOL_CONFIG_VARS -_LT_LIBTOOL_TAG_VARS -# ### END LIBTOOL CONFIG - -_LT_EOF - - cat <<'_LT_EOF' >> "$cfgfile" - -# ### BEGIN FUNCTIONS SHARED WITH CONFIGURE - -_LT_PREPARE_MUNGE_PATH_LIST -_LT_PREPARE_CC_BASENAME - -# ### END FUNCTIONS SHARED WITH CONFIGURE - -_LT_EOF - - case $host_os in - aix3*) - cat <<\_LT_EOF >> "$cfgfile" -# AIX sometimes has problems with the GCC collect2 program. For some -# reason, if we set the COLLECT_NAMES environment variable, the problems -# vanish in a puff of smoke. -if test set != "${COLLECT_NAMES+set}"; then - COLLECT_NAMES= - export COLLECT_NAMES -fi -_LT_EOF - ;; - esac - - _LT_PROG_LTMAIN - - # We use sed instead of cat because bash on DJGPP gets confused if - # if finds mixed CR/LF and LF-only lines. Since sed operates in - # text mode, it properly converts lines to CR/LF. This bash problem - # is reportedly fixed, but why not run on old versions too? - sed '$q' "$ltmain" >> "$cfgfile" \ - || (rm -f "$cfgfile"; exit 1) - - mv -f "$cfgfile" "$ofile" || - (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile") - chmod +x "$ofile" -], -[cat <<_LT_EOF >> "$ofile" - -dnl Unfortunately we have to use $1 here, since _LT_TAG is not expanded -dnl in a comment (ie after a #). -# ### BEGIN LIBTOOL TAG CONFIG: $1 -_LT_LIBTOOL_TAG_VARS(_LT_TAG) -# ### END LIBTOOL TAG CONFIG: $1 -_LT_EOF -])dnl /m4_if -], -[m4_if([$1], [], [ - PACKAGE='$PACKAGE' - VERSION='$VERSION' - RM='$RM' - ofile='$ofile'], []) -])dnl /_LT_CONFIG_SAVE_COMMANDS -])# _LT_CONFIG - - -# LT_SUPPORTED_TAG(TAG) -# --------------------- -# Trace this macro to discover what tags are supported by the libtool -# --tag option, using: -# autoconf --trace 'LT_SUPPORTED_TAG:$1' -AC_DEFUN([LT_SUPPORTED_TAG], []) - - -# C support is built-in for now -m4_define([_LT_LANG_C_enabled], []) -m4_define([_LT_TAGS], []) - - -# LT_LANG(LANG) -# ------------- -# Enable libtool support for the given language if not already enabled. -AC_DEFUN([LT_LANG], -[AC_BEFORE([$0], [LT_OUTPUT])dnl -m4_case([$1], - [C], [_LT_LANG(C)], - [C++], [_LT_LANG(CXX)], - [Go], [_LT_LANG(GO)], - [Java], [_LT_LANG(GCJ)], - [Fortran 77], [_LT_LANG(F77)], - [Fortran], [_LT_LANG(FC)], - [Windows Resource], [_LT_LANG(RC)], - [m4_ifdef([_LT_LANG_]$1[_CONFIG], - [_LT_LANG($1)], - [m4_fatal([$0: unsupported language: "$1"])])])dnl -])# LT_LANG - - -# _LT_LANG(LANGNAME) -# ------------------ -m4_defun([_LT_LANG], -[m4_ifdef([_LT_LANG_]$1[_enabled], [], - [LT_SUPPORTED_TAG([$1])dnl - m4_append([_LT_TAGS], [$1 ])dnl - m4_define([_LT_LANG_]$1[_enabled], [])dnl - _LT_LANG_$1_CONFIG($1)])dnl -])# _LT_LANG - - -m4_ifndef([AC_PROG_GO], [ -# NOTE: This macro has been submitted for inclusion into # -# GNU Autoconf as AC_PROG_GO. When it is available in # -# a released version of Autoconf we should remove this # -# macro and use it instead. # -m4_defun([AC_PROG_GO], -[AC_LANG_PUSH(Go)dnl -AC_ARG_VAR([GOC], [Go compiler command])dnl -AC_ARG_VAR([GOFLAGS], [Go compiler flags])dnl -_AC_ARG_VAR_LDFLAGS()dnl -AC_CHECK_TOOL(GOC, gccgo) -if test -z "$GOC"; then - if test -n "$ac_tool_prefix"; then - AC_CHECK_PROG(GOC, [${ac_tool_prefix}gccgo], [${ac_tool_prefix}gccgo]) - fi -fi -if test -z "$GOC"; then - AC_CHECK_PROG(GOC, gccgo, gccgo, false) -fi -])#m4_defun -])#m4_ifndef - - -# _LT_LANG_DEFAULT_CONFIG -# ----------------------- -m4_defun([_LT_LANG_DEFAULT_CONFIG], -[AC_PROVIDE_IFELSE([AC_PROG_CXX], - [LT_LANG(CXX)], - [m4_define([AC_PROG_CXX], defn([AC_PROG_CXX])[LT_LANG(CXX)])]) - -AC_PROVIDE_IFELSE([AC_PROG_F77], - [LT_LANG(F77)], - [m4_define([AC_PROG_F77], defn([AC_PROG_F77])[LT_LANG(F77)])]) - -AC_PROVIDE_IFELSE([AC_PROG_FC], - [LT_LANG(FC)], - [m4_define([AC_PROG_FC], defn([AC_PROG_FC])[LT_LANG(FC)])]) - -dnl The call to [A][M_PROG_GCJ] is quoted like that to stop aclocal -dnl pulling things in needlessly. -AC_PROVIDE_IFELSE([AC_PROG_GCJ], - [LT_LANG(GCJ)], - [AC_PROVIDE_IFELSE([A][M_PROG_GCJ], - [LT_LANG(GCJ)], - [AC_PROVIDE_IFELSE([LT_PROG_GCJ], - [LT_LANG(GCJ)], - [m4_ifdef([AC_PROG_GCJ], - [m4_define([AC_PROG_GCJ], defn([AC_PROG_GCJ])[LT_LANG(GCJ)])]) - m4_ifdef([A][M_PROG_GCJ], - [m4_define([A][M_PROG_GCJ], defn([A][M_PROG_GCJ])[LT_LANG(GCJ)])]) - m4_ifdef([LT_PROG_GCJ], - [m4_define([LT_PROG_GCJ], defn([LT_PROG_GCJ])[LT_LANG(GCJ)])])])])]) - -AC_PROVIDE_IFELSE([AC_PROG_GO], - [LT_LANG(GO)], - [m4_define([AC_PROG_GO], defn([AC_PROG_GO])[LT_LANG(GO)])]) - -AC_PROVIDE_IFELSE([LT_PROG_RC], - [LT_LANG(RC)], - [m4_define([LT_PROG_RC], defn([LT_PROG_RC])[LT_LANG(RC)])]) -])# _LT_LANG_DEFAULT_CONFIG - -# Obsolete macros: -AU_DEFUN([AC_LIBTOOL_CXX], [LT_LANG(C++)]) -AU_DEFUN([AC_LIBTOOL_F77], [LT_LANG(Fortran 77)]) -AU_DEFUN([AC_LIBTOOL_FC], [LT_LANG(Fortran)]) -AU_DEFUN([AC_LIBTOOL_GCJ], [LT_LANG(Java)]) -AU_DEFUN([AC_LIBTOOL_RC], [LT_LANG(Windows Resource)]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_CXX], []) -dnl AC_DEFUN([AC_LIBTOOL_F77], []) -dnl AC_DEFUN([AC_LIBTOOL_FC], []) -dnl AC_DEFUN([AC_LIBTOOL_GCJ], []) -dnl AC_DEFUN([AC_LIBTOOL_RC], []) - - -# _LT_TAG_COMPILER -# ---------------- -m4_defun([_LT_TAG_COMPILER], -[AC_REQUIRE([AC_PROG_CC])dnl - -_LT_DECL([LTCC], [CC], [1], [A C compiler])dnl -_LT_DECL([LTCFLAGS], [CFLAGS], [1], [LTCC compiler flags])dnl -_LT_TAGDECL([CC], [compiler], [1], [A language specific compiler])dnl -_LT_TAGDECL([with_gcc], [GCC], [0], [Is the compiler the GNU compiler?])dnl - -# If no C compiler was specified, use CC. -LTCC=${LTCC-"$CC"} - -# If no C compiler flags were specified, use CFLAGS. -LTCFLAGS=${LTCFLAGS-"$CFLAGS"} - -# Allow CC to be a program name with arguments. -compiler=$CC -])# _LT_TAG_COMPILER - - -# _LT_COMPILER_BOILERPLATE -# ------------------------ -# Check for compiler boilerplate output or warnings with -# the simple compiler test code. -m4_defun([_LT_COMPILER_BOILERPLATE], -[m4_require([_LT_DECL_SED])dnl -ac_outfile=conftest.$ac_objext -echo "$lt_simple_compile_test_code" >conftest.$ac_ext -eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err -_lt_compiler_boilerplate=`cat conftest.err` -$RM conftest* -])# _LT_COMPILER_BOILERPLATE - - -# _LT_LINKER_BOILERPLATE -# ---------------------- -# Check for linker boilerplate output or warnings with -# the simple link test code. -m4_defun([_LT_LINKER_BOILERPLATE], -[m4_require([_LT_DECL_SED])dnl -ac_outfile=conftest.$ac_objext -echo "$lt_simple_link_test_code" >conftest.$ac_ext -eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err -_lt_linker_boilerplate=`cat conftest.err` -$RM -r conftest* -])# _LT_LINKER_BOILERPLATE - -# _LT_REQUIRED_DARWIN_CHECKS -# ------------------------- -m4_defun_once([_LT_REQUIRED_DARWIN_CHECKS],[ - case $host_os in - rhapsody* | darwin*) - AC_CHECK_TOOL([DSYMUTIL], [dsymutil], [:]) - AC_CHECK_TOOL([NMEDIT], [nmedit], [:]) - AC_CHECK_TOOL([LIPO], [lipo], [:]) - AC_CHECK_TOOL([OTOOL], [otool], [:]) - AC_CHECK_TOOL([OTOOL64], [otool64], [:]) - _LT_DECL([], [DSYMUTIL], [1], - [Tool to manipulate archived DWARF debug symbol files on Mac OS X]) - _LT_DECL([], [NMEDIT], [1], - [Tool to change global to local symbols on Mac OS X]) - _LT_DECL([], [LIPO], [1], - [Tool to manipulate fat objects and archives on Mac OS X]) - _LT_DECL([], [OTOOL], [1], - [ldd/readelf like tool for Mach-O binaries on Mac OS X]) - _LT_DECL([], [OTOOL64], [1], - [ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4]) - - AC_CACHE_CHECK([for -single_module linker flag],[lt_cv_apple_cc_single_mod], - [lt_cv_apple_cc_single_mod=no - if test -z "$LT_MULTI_MODULE"; then - # By default we will add the -single_module flag. You can override - # by either setting the environment variable LT_MULTI_MODULE - # non-empty at configure time, or by adding -multi_module to the - # link flags. - rm -rf libconftest.dylib* - echo "int foo(void){return 1;}" > conftest.c - echo "$LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ --dynamiclib -Wl,-single_module conftest.c" >&AS_MESSAGE_LOG_FD - $LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ - -dynamiclib -Wl,-single_module conftest.c 2>conftest.err - _lt_result=$? - # If there is a non-empty error log, and "single_module" - # appears in it, assume the flag caused a linker warning - if test -s conftest.err && $GREP single_module conftest.err; then - cat conftest.err >&AS_MESSAGE_LOG_FD - # Otherwise, if the output was created with a 0 exit code from - # the compiler, it worked. - elif test -f libconftest.dylib && test 0 = "$_lt_result"; then - lt_cv_apple_cc_single_mod=yes - else - cat conftest.err >&AS_MESSAGE_LOG_FD - fi - rm -rf libconftest.dylib* - rm -f conftest.* - fi]) - - AC_CACHE_CHECK([for -exported_symbols_list linker flag], - [lt_cv_ld_exported_symbols_list], - [lt_cv_ld_exported_symbols_list=no - save_LDFLAGS=$LDFLAGS - echo "_main" > conftest.sym - LDFLAGS="$LDFLAGS -Wl,-exported_symbols_list,conftest.sym" - AC_LINK_IFELSE([AC_LANG_PROGRAM([],[])], - [lt_cv_ld_exported_symbols_list=yes], - [lt_cv_ld_exported_symbols_list=no]) - LDFLAGS=$save_LDFLAGS - ]) - - AC_CACHE_CHECK([for -force_load linker flag],[lt_cv_ld_force_load], - [lt_cv_ld_force_load=no - cat > conftest.c << _LT_EOF -int forced_loaded() { return 2;} -_LT_EOF - echo "$LTCC $LTCFLAGS -c -o conftest.o conftest.c" >&AS_MESSAGE_LOG_FD - $LTCC $LTCFLAGS -c -o conftest.o conftest.c 2>&AS_MESSAGE_LOG_FD - echo "$AR cr libconftest.a conftest.o" >&AS_MESSAGE_LOG_FD - $AR cr libconftest.a conftest.o 2>&AS_MESSAGE_LOG_FD - echo "$RANLIB libconftest.a" >&AS_MESSAGE_LOG_FD - $RANLIB libconftest.a 2>&AS_MESSAGE_LOG_FD - cat > conftest.c << _LT_EOF -int main() { return 0;} -_LT_EOF - echo "$LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a" >&AS_MESSAGE_LOG_FD - $LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a 2>conftest.err - _lt_result=$? - if test -s conftest.err && $GREP force_load conftest.err; then - cat conftest.err >&AS_MESSAGE_LOG_FD - elif test -f conftest && test 0 = "$_lt_result" && $GREP forced_load conftest >/dev/null 2>&1; then - lt_cv_ld_force_load=yes - else - cat conftest.err >&AS_MESSAGE_LOG_FD - fi - rm -f conftest.err libconftest.a conftest conftest.c - rm -rf conftest.dSYM - ]) - case $host_os in - rhapsody* | darwin1.[[012]]) - _lt_dar_allow_undefined='$wl-undefined ${wl}suppress' ;; - darwin1.*) - _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - darwin*) # darwin 5.x on - # if running on 10.5 or later, the deployment target defaults - # to the OS version, if on x86, and 10.4, the deployment - # target defaults to 10.4. Don't you love it? - case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in - 10.0,*86*-darwin8*|10.0,*-darwin[[91]]*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; - 10.[[012]][[,.]]*) - _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - 10.*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; - esac - ;; - esac - if test yes = "$lt_cv_apple_cc_single_mod"; then - _lt_dar_single_mod='$single_module' - fi - if test yes = "$lt_cv_ld_exported_symbols_list"; then - _lt_dar_export_syms=' $wl-exported_symbols_list,$output_objdir/$libname-symbols.expsym' - else - _lt_dar_export_syms='~$NMEDIT -s $output_objdir/$libname-symbols.expsym $lib' - fi - if test : != "$DSYMUTIL" && test no = "$lt_cv_ld_force_load"; then - _lt_dsymutil='~$DSYMUTIL $lib || :' - else - _lt_dsymutil= - fi - ;; - esac -]) - - -# _LT_DARWIN_LINKER_FEATURES([TAG]) -# --------------------------------- -# Checks for linker and compiler features on darwin -m4_defun([_LT_DARWIN_LINKER_FEATURES], -[ - m4_require([_LT_REQUIRED_DARWIN_CHECKS]) - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_automatic, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported - if test yes = "$lt_cv_ld_force_load"; then - _LT_TAGVAR(whole_archive_flag_spec, $1)='`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience $wl-force_load,$conv\"; done; func_echo_all \"$new_convenience\"`' - m4_case([$1], [F77], [_LT_TAGVAR(compiler_needs_object, $1)=yes], - [FC], [_LT_TAGVAR(compiler_needs_object, $1)=yes]) - else - _LT_TAGVAR(whole_archive_flag_spec, $1)='' - fi - _LT_TAGVAR(link_all_deplibs, $1)=yes - _LT_TAGVAR(allow_undefined_flag, $1)=$_lt_dar_allow_undefined - case $cc_basename in - ifort*|nagfor*) _lt_dar_can_shared=yes ;; - *) _lt_dar_can_shared=$GCC ;; - esac - if test yes = "$_lt_dar_can_shared"; then - output_verbose_link_cmd=func_echo_all - _LT_TAGVAR(archive_cmds, $1)="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dsymutil" - _LT_TAGVAR(module_cmds, $1)="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dsymutil" - _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" - _LT_TAGVAR(module_expsym_cmds, $1)="sed -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" - m4_if([$1], [CXX], -[ if test yes != "$lt_cv_apple_cc_single_mod"; then - _LT_TAGVAR(archive_cmds, $1)="\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dsymutil" - _LT_TAGVAR(archive_expsym_cmds, $1)="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -r -keep_private_externs -nostdlib -o \$lib-master.o \$libobjs~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$lib-master.o \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring$_lt_dar_export_syms$_lt_dsymutil" - fi -],[]) - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi -]) - -# _LT_SYS_MODULE_PATH_AIX([TAGNAME]) -# ---------------------------------- -# Links a minimal program and checks the executable -# for the system default hardcoded library path. In most cases, -# this is /usr/lib:/lib, but when the MPI compilers are used -# the location of the communication and MPI libs are included too. -# If we don't find anything, use the default library path according -# to the aix ld manual. -# Store the results from the different compilers for each TAGNAME. -# Allow to override them for all tags through lt_cv_aix_libpath. -m4_defun([_LT_SYS_MODULE_PATH_AIX], -[m4_require([_LT_DECL_SED])dnl -if test set = "${lt_cv_aix_libpath+set}"; then - aix_libpath=$lt_cv_aix_libpath -else - AC_CACHE_VAL([_LT_TAGVAR([lt_cv_aix_libpath_], [$1])], - [AC_LINK_IFELSE([AC_LANG_PROGRAM],[ - lt_aix_libpath_sed='[ - /Import File Strings/,/^$/ { - /^0/ { - s/^0 *\([^ ]*\) *$/\1/ - p - } - }]' - _LT_TAGVAR([lt_cv_aix_libpath_], [$1])=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` - # Check for a 64-bit object if we didn't find anything. - if test -z "$_LT_TAGVAR([lt_cv_aix_libpath_], [$1])"; then - _LT_TAGVAR([lt_cv_aix_libpath_], [$1])=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` - fi],[]) - if test -z "$_LT_TAGVAR([lt_cv_aix_libpath_], [$1])"; then - _LT_TAGVAR([lt_cv_aix_libpath_], [$1])=/usr/lib:/lib - fi - ]) - aix_libpath=$_LT_TAGVAR([lt_cv_aix_libpath_], [$1]) -fi -])# _LT_SYS_MODULE_PATH_AIX - - -# _LT_SHELL_INIT(ARG) -# ------------------- -m4_define([_LT_SHELL_INIT], -[m4_divert_text([M4SH-INIT], [$1 -])])# _LT_SHELL_INIT - - - -# _LT_PROG_ECHO_BACKSLASH -# ----------------------- -# Find how we can fake an echo command that does not interpret backslash. -# In particular, with Autoconf 2.60 or later we add some code to the start -# of the generated configure script that will find a shell with a builtin -# printf (that we can use as an echo command). -m4_defun([_LT_PROG_ECHO_BACKSLASH], -[ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' -ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO -ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO$ECHO - -AC_MSG_CHECKING([how to print strings]) -# Test print first, because it will be a builtin if present. -if test "X`( print -r -- -n ) 2>/dev/null`" = X-n && \ - test "X`print -r -- $ECHO 2>/dev/null`" = "X$ECHO"; then - ECHO='print -r --' -elif test "X`printf %s $ECHO 2>/dev/null`" = "X$ECHO"; then - ECHO='printf %s\n' -else - # Use this function as a fallback that always works. - func_fallback_echo () - { - eval 'cat <<_LTECHO_EOF -$[]1 -_LTECHO_EOF' - } - ECHO='func_fallback_echo' -fi - -# func_echo_all arg... -# Invoke $ECHO with all args, space-separated. -func_echo_all () -{ - $ECHO "$*" -} - -case $ECHO in - printf*) AC_MSG_RESULT([printf]) ;; - print*) AC_MSG_RESULT([print -r]) ;; - *) AC_MSG_RESULT([cat]) ;; -esac - -m4_ifdef([_AS_DETECT_SUGGESTED], -[_AS_DETECT_SUGGESTED([ - test -n "${ZSH_VERSION+set}${BASH_VERSION+set}" || ( - ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' - ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO - ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO$ECHO - PATH=/empty FPATH=/empty; export PATH FPATH - test "X`printf %s $ECHO`" = "X$ECHO" \ - || test "X`print -r -- $ECHO`" = "X$ECHO" )])]) - -_LT_DECL([], [SHELL], [1], [Shell to use when invoking shell scripts]) -_LT_DECL([], [ECHO], [1], [An echo program that protects backslashes]) -])# _LT_PROG_ECHO_BACKSLASH - - -# _LT_WITH_SYSROOT -# ---------------- -AC_DEFUN([_LT_WITH_SYSROOT], -[AC_MSG_CHECKING([for sysroot]) -AC_ARG_WITH([sysroot], -[AS_HELP_STRING([--with-sysroot@<:@=DIR@:>@], - [Search for dependent libraries within DIR (or the compiler's sysroot - if not specified).])], -[], [with_sysroot=no]) - -dnl lt_sysroot will always be passed unquoted. We quote it here -dnl in case the user passed a directory name. -lt_sysroot= -case $with_sysroot in #( - yes) - if test yes = "$GCC"; then - lt_sysroot=`$CC --print-sysroot 2>/dev/null` - fi - ;; #( - /*) - lt_sysroot=`echo "$with_sysroot" | sed -e "$sed_quote_subst"` - ;; #( - no|'') - ;; #( - *) - AC_MSG_RESULT([$with_sysroot]) - AC_MSG_ERROR([The sysroot must be an absolute path.]) - ;; -esac - - AC_MSG_RESULT([${lt_sysroot:-no}]) -_LT_DECL([], [lt_sysroot], [0], [The root where to search for ]dnl -[dependent libraries, and where our libraries should be installed.])]) - -# _LT_ENABLE_LOCK -# --------------- -m4_defun([_LT_ENABLE_LOCK], -[AC_ARG_ENABLE([libtool-lock], - [AS_HELP_STRING([--disable-libtool-lock], - [avoid locking (might break parallel builds)])]) -test no = "$enable_libtool_lock" || enable_libtool_lock=yes - -# Some flags need to be propagated to the compiler or linker for good -# libtool support. -case $host in -ia64-*-hpux*) - # Find out what ABI is being produced by ac_compile, and set mode - # options accordingly. - echo 'int i;' > conftest.$ac_ext - if AC_TRY_EVAL(ac_compile); then - case `/usr/bin/file conftest.$ac_objext` in - *ELF-32*) - HPUX_IA64_MODE=32 - ;; - *ELF-64*) - HPUX_IA64_MODE=64 - ;; - esac - fi - rm -rf conftest* - ;; -*-*-irix6*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. - echo '[#]line '$LINENO' "configure"' > conftest.$ac_ext - if AC_TRY_EVAL(ac_compile); then - if test yes = "$lt_cv_prog_gnu_ld"; then - case `/usr/bin/file conftest.$ac_objext` in - *32-bit*) - LD="${LD-ld} -melf32bsmip" - ;; - *N32*) - LD="${LD-ld} -melf32bmipn32" - ;; - *64-bit*) - LD="${LD-ld} -melf64bmip" - ;; - esac - else - case `/usr/bin/file conftest.$ac_objext` in - *32-bit*) - LD="${LD-ld} -32" - ;; - *N32*) - LD="${LD-ld} -n32" - ;; - *64-bit*) - LD="${LD-ld} -64" - ;; - esac - fi - fi - rm -rf conftest* - ;; - -mips64*-*linux*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. - echo '[#]line '$LINENO' "configure"' > conftest.$ac_ext - if AC_TRY_EVAL(ac_compile); then - emul=elf - case `/usr/bin/file conftest.$ac_objext` in - *32-bit*) - emul="${emul}32" - ;; - *64-bit*) - emul="${emul}64" - ;; - esac - case `/usr/bin/file conftest.$ac_objext` in - *MSB*) - emul="${emul}btsmip" - ;; - *LSB*) - emul="${emul}ltsmip" - ;; - esac - case `/usr/bin/file conftest.$ac_objext` in - *N32*) - emul="${emul}n32" - ;; - esac - LD="${LD-ld} -m $emul" - fi - rm -rf conftest* - ;; - -x86_64-*kfreebsd*-gnu|x86_64-*linux*|powerpc*-*linux*| \ -s390*-*linux*|s390*-*tpf*|sparc*-*linux*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. Note that the listed cases only cover the - # situations where additional linker options are needed (such as when - # doing 32-bit compilation for a host where ld defaults to 64-bit, or - # vice versa); the common cases where no linker options are needed do - # not appear in the list. - echo 'int i;' > conftest.$ac_ext - if AC_TRY_EVAL(ac_compile); then - case `/usr/bin/file conftest.o` in - *32-bit*) - case $host in - x86_64-*kfreebsd*-gnu) - LD="${LD-ld} -m elf_i386_fbsd" - ;; - x86_64-*linux*) - case `/usr/bin/file conftest.o` in - *x86-64*) - LD="${LD-ld} -m elf32_x86_64" - ;; - *) - LD="${LD-ld} -m elf_i386" - ;; - esac - ;; - powerpc64le-*linux*) - LD="${LD-ld} -m elf32lppclinux" - ;; - powerpc64-*linux*) - LD="${LD-ld} -m elf32ppclinux" - ;; - s390x-*linux*) - LD="${LD-ld} -m elf_s390" - ;; - sparc64-*linux*) - LD="${LD-ld} -m elf32_sparc" - ;; - esac - ;; - *64-bit*) - case $host in - x86_64-*kfreebsd*-gnu) - LD="${LD-ld} -m elf_x86_64_fbsd" - ;; - x86_64-*linux*) - LD="${LD-ld} -m elf_x86_64" - ;; - powerpcle-*linux*) - LD="${LD-ld} -m elf64lppc" - ;; - powerpc-*linux*) - LD="${LD-ld} -m elf64ppc" - ;; - s390*-*linux*|s390*-*tpf*) - LD="${LD-ld} -m elf64_s390" - ;; - sparc*-*linux*) - LD="${LD-ld} -m elf64_sparc" - ;; - esac - ;; - esac - fi - rm -rf conftest* - ;; - -*-*-sco3.2v5*) - # On SCO OpenServer 5, we need -belf to get full-featured binaries. - SAVE_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -belf" - AC_CACHE_CHECK([whether the C compiler needs -belf], lt_cv_cc_needs_belf, - [AC_LANG_PUSH(C) - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]],[[]])],[lt_cv_cc_needs_belf=yes],[lt_cv_cc_needs_belf=no]) - AC_LANG_POP]) - if test yes != "$lt_cv_cc_needs_belf"; then - # this is probably gcc 2.8.0, egcs 1.0 or newer; no need for -belf - CFLAGS=$SAVE_CFLAGS - fi - ;; -*-*solaris*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. - echo 'int i;' > conftest.$ac_ext - if AC_TRY_EVAL(ac_compile); then - case `/usr/bin/file conftest.o` in - *64-bit*) - case $lt_cv_prog_gnu_ld in - yes*) - case $host in - i?86-*-solaris*|x86_64-*-solaris*) - LD="${LD-ld} -m elf_x86_64" - ;; - sparc*-*-solaris*) - LD="${LD-ld} -m elf64_sparc" - ;; - esac - # GNU ld 2.21 introduced _sol2 emulations. Use them if available. - if ${LD-ld} -V | grep _sol2 >/dev/null 2>&1; then - LD=${LD-ld}_sol2 - fi - ;; - *) - if ${LD-ld} -64 -r -o conftest2.o conftest.o >/dev/null 2>&1; then - LD="${LD-ld} -64" - fi - ;; - esac - ;; - esac - fi - rm -rf conftest* - ;; -esac - -need_locks=$enable_libtool_lock -])# _LT_ENABLE_LOCK - - -# _LT_PROG_AR -# ----------- -m4_defun([_LT_PROG_AR], -[AC_CHECK_TOOLS(AR, [ar], false) -: ${AR=ar} -: ${AR_FLAGS=cr} -_LT_DECL([], [AR], [1], [The archiver]) -_LT_DECL([], [AR_FLAGS], [1], [Flags to create an archive]) - -AC_CACHE_CHECK([for archiver @FILE support], [lt_cv_ar_at_file], - [lt_cv_ar_at_file=no - AC_COMPILE_IFELSE([AC_LANG_PROGRAM], - [echo conftest.$ac_objext > conftest.lst - lt_ar_try='$AR $AR_FLAGS libconftest.a @conftest.lst >&AS_MESSAGE_LOG_FD' - AC_TRY_EVAL([lt_ar_try]) - if test 0 -eq "$ac_status"; then - # Ensure the archiver fails upon bogus file names. - rm -f conftest.$ac_objext libconftest.a - AC_TRY_EVAL([lt_ar_try]) - if test 0 -ne "$ac_status"; then - lt_cv_ar_at_file=@ - fi - fi - rm -f conftest.* libconftest.a - ]) - ]) - -if test no = "$lt_cv_ar_at_file"; then - archiver_list_spec= -else - archiver_list_spec=$lt_cv_ar_at_file -fi -_LT_DECL([], [archiver_list_spec], [1], - [How to feed a file listing to the archiver]) -])# _LT_PROG_AR - - -# _LT_CMD_OLD_ARCHIVE -# ------------------- -m4_defun([_LT_CMD_OLD_ARCHIVE], -[_LT_PROG_AR - -AC_CHECK_TOOL(STRIP, strip, :) -test -z "$STRIP" && STRIP=: -_LT_DECL([], [STRIP], [1], [A symbol stripping program]) - -AC_CHECK_TOOL(RANLIB, ranlib, :) -test -z "$RANLIB" && RANLIB=: -_LT_DECL([], [RANLIB], [1], - [Commands used to install an old-style archive]) - -# Determine commands to create old-style static archives. -old_archive_cmds='$AR $AR_FLAGS $oldlib$oldobjs' -old_postinstall_cmds='chmod 644 $oldlib' -old_postuninstall_cmds= - -if test -n "$RANLIB"; then - case $host_os in - bitrig* | openbsd*) - old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB -t \$tool_oldlib" - ;; - *) - old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB \$tool_oldlib" - ;; - esac - old_archive_cmds="$old_archive_cmds~\$RANLIB \$tool_oldlib" -fi - -case $host_os in - darwin*) - lock_old_archive_extraction=yes ;; - *) - lock_old_archive_extraction=no ;; -esac -_LT_DECL([], [old_postinstall_cmds], [2]) -_LT_DECL([], [old_postuninstall_cmds], [2]) -_LT_TAGDECL([], [old_archive_cmds], [2], - [Commands used to build an old-style archive]) -_LT_DECL([], [lock_old_archive_extraction], [0], - [Whether to use a lock for old archive extraction]) -])# _LT_CMD_OLD_ARCHIVE - - -# _LT_COMPILER_OPTION(MESSAGE, VARIABLE-NAME, FLAGS, -# [OUTPUT-FILE], [ACTION-SUCCESS], [ACTION-FAILURE]) -# ---------------------------------------------------------------- -# Check whether the given compiler option works -AC_DEFUN([_LT_COMPILER_OPTION], -[m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_DECL_SED])dnl -AC_CACHE_CHECK([$1], [$2], - [$2=no - m4_if([$4], , [ac_outfile=conftest.$ac_objext], [ac_outfile=$4]) - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - lt_compiler_flag="$3" ## exclude from sc_useless_quotes_in_assignment - # Insert the option either (1) after the last *FLAGS variable, or - # (2) before a word containing "conftest.", or (3) at the end. - # Note that $ac_compile itself does not contain backslashes and begins - # with a dollar sign (not a hyphen), so the echo should work correctly. - # The option is referenced via a variable to avoid confusing sed. - lt_compile=`echo "$ac_compile" | $SED \ - -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ - -e 's: [[^ ]]*conftest\.: $lt_compiler_flag&:; t' \ - -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&AS_MESSAGE_LOG_FD) - (eval "$lt_compile" 2>conftest.err) - ac_status=$? - cat conftest.err >&AS_MESSAGE_LOG_FD - echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD - if (exit $ac_status) && test -s "$ac_outfile"; then - # The compiler can only warn and ignore the option if not recognized - # So say no if there are warnings other than the usual output. - $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp - $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 - if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then - $2=yes - fi - fi - $RM conftest* -]) - -if test yes = "[$]$2"; then - m4_if([$5], , :, [$5]) -else - m4_if([$6], , :, [$6]) -fi -])# _LT_COMPILER_OPTION - -# Old name: -AU_ALIAS([AC_LIBTOOL_COMPILER_OPTION], [_LT_COMPILER_OPTION]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_COMPILER_OPTION], []) - - -# _LT_LINKER_OPTION(MESSAGE, VARIABLE-NAME, FLAGS, -# [ACTION-SUCCESS], [ACTION-FAILURE]) -# ---------------------------------------------------- -# Check whether the given linker option works -AC_DEFUN([_LT_LINKER_OPTION], -[m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_DECL_SED])dnl -AC_CACHE_CHECK([$1], [$2], - [$2=no - save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS $3" - echo "$lt_simple_link_test_code" > conftest.$ac_ext - if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then - # The linker can only warn and ignore the option if not recognized - # So say no if there are warnings - if test -s conftest.err; then - # Append any errors to the config.log. - cat conftest.err 1>&AS_MESSAGE_LOG_FD - $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp - $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 - if diff conftest.exp conftest.er2 >/dev/null; then - $2=yes - fi - else - $2=yes - fi - fi - $RM -r conftest* - LDFLAGS=$save_LDFLAGS -]) - -if test yes = "[$]$2"; then - m4_if([$4], , :, [$4]) -else - m4_if([$5], , :, [$5]) -fi -])# _LT_LINKER_OPTION - -# Old name: -AU_ALIAS([AC_LIBTOOL_LINKER_OPTION], [_LT_LINKER_OPTION]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_LINKER_OPTION], []) - - -# LT_CMD_MAX_LEN -#--------------- -AC_DEFUN([LT_CMD_MAX_LEN], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -# find the maximum length of command line arguments -AC_MSG_CHECKING([the maximum length of command line arguments]) -AC_CACHE_VAL([lt_cv_sys_max_cmd_len], [dnl - i=0 - teststring=ABCD - - case $build_os in - msdosdjgpp*) - # On DJGPP, this test can blow up pretty badly due to problems in libc - # (any single argument exceeding 2000 bytes causes a buffer overrun - # during glob expansion). Even if it were fixed, the result of this - # check would be larger than it should be. - lt_cv_sys_max_cmd_len=12288; # 12K is about right - ;; - - gnu*) - # Under GNU Hurd, this test is not required because there is - # no limit to the length of command line arguments. - # Libtool will interpret -1 as no limit whatsoever - lt_cv_sys_max_cmd_len=-1; - ;; - - cygwin* | mingw* | cegcc*) - # On Win9x/ME, this test blows up -- it succeeds, but takes - # about 5 minutes as the teststring grows exponentially. - # Worse, since 9x/ME are not pre-emptively multitasking, - # you end up with a "frozen" computer, even though with patience - # the test eventually succeeds (with a max line length of 256k). - # Instead, let's just punt: use the minimum linelength reported by - # all of the supported platforms: 8192 (on NT/2K/XP). - lt_cv_sys_max_cmd_len=8192; - ;; - - mint*) - # On MiNT this can take a long time and run out of memory. - lt_cv_sys_max_cmd_len=8192; - ;; - - amigaos*) - # On AmigaOS with pdksh, this test takes hours, literally. - # So we just punt and use a minimum line length of 8192. - lt_cv_sys_max_cmd_len=8192; - ;; - - bitrig* | darwin* | dragonfly* | freebsd* | netbsd* | openbsd*) - # This has been around since 386BSD, at least. Likely further. - if test -x /sbin/sysctl; then - lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax` - elif test -x /usr/sbin/sysctl; then - lt_cv_sys_max_cmd_len=`/usr/sbin/sysctl -n kern.argmax` - else - lt_cv_sys_max_cmd_len=65536 # usable default for all BSDs - fi - # And add a safety zone - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` - ;; - - interix*) - # We know the value 262144 and hardcode it with a safety zone (like BSD) - lt_cv_sys_max_cmd_len=196608 - ;; - - os2*) - # The test takes a long time on OS/2. - lt_cv_sys_max_cmd_len=8192 - ;; - - osf*) - # Dr. Hans Ekkehard Plesser reports seeing a kernel panic running configure - # due to this test when exec_disable_arg_limit is 1 on Tru64. It is not - # nice to cause kernel panics so lets avoid the loop below. - # First set a reasonable default. - lt_cv_sys_max_cmd_len=16384 - # - if test -x /sbin/sysconfig; then - case `/sbin/sysconfig -q proc exec_disable_arg_limit` in - *1*) lt_cv_sys_max_cmd_len=-1 ;; - esac - fi - ;; - sco3.2v5*) - lt_cv_sys_max_cmd_len=102400 - ;; - sysv5* | sco5v6* | sysv4.2uw2*) - kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null` - if test -n "$kargmax"; then - lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[[ ]]//'` - else - lt_cv_sys_max_cmd_len=32768 - fi - ;; - *) - lt_cv_sys_max_cmd_len=`(getconf ARG_MAX) 2> /dev/null` - if test -n "$lt_cv_sys_max_cmd_len" && \ - test undefined != "$lt_cv_sys_max_cmd_len"; then - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` - else - # Make teststring a little bigger before we do anything with it. - # a 1K string should be a reasonable start. - for i in 1 2 3 4 5 6 7 8; do - teststring=$teststring$teststring - done - SHELL=${SHELL-${CONFIG_SHELL-/bin/sh}} - # If test is not a shell built-in, we'll probably end up computing a - # maximum length that is only half of the actual maximum length, but - # we can't tell. - while { test X`env echo "$teststring$teststring" 2>/dev/null` \ - = "X$teststring$teststring"; } >/dev/null 2>&1 && - test 17 != "$i" # 1/2 MB should be enough - do - i=`expr $i + 1` - teststring=$teststring$teststring - done - # Only check the string length outside the loop. - lt_cv_sys_max_cmd_len=`expr "X$teststring" : ".*" 2>&1` - teststring= - # Add a significant safety factor because C++ compilers can tack on - # massive amounts of additional arguments before passing them to the - # linker. It appears as though 1/2 is a usable value. - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 2` - fi - ;; - esac -]) -if test -n "$lt_cv_sys_max_cmd_len"; then - AC_MSG_RESULT($lt_cv_sys_max_cmd_len) -else - AC_MSG_RESULT(none) -fi -max_cmd_len=$lt_cv_sys_max_cmd_len -_LT_DECL([], [max_cmd_len], [0], - [What is the maximum length of a command?]) -])# LT_CMD_MAX_LEN - -# Old name: -AU_ALIAS([AC_LIBTOOL_SYS_MAX_CMD_LEN], [LT_CMD_MAX_LEN]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_SYS_MAX_CMD_LEN], []) - - -# _LT_HEADER_DLFCN -# ---------------- -m4_defun([_LT_HEADER_DLFCN], -[AC_CHECK_HEADERS([dlfcn.h], [], [], [AC_INCLUDES_DEFAULT])dnl -])# _LT_HEADER_DLFCN - - -# _LT_TRY_DLOPEN_SELF (ACTION-IF-TRUE, ACTION-IF-TRUE-W-USCORE, -# ACTION-IF-FALSE, ACTION-IF-CROSS-COMPILING) -# ---------------------------------------------------------------- -m4_defun([_LT_TRY_DLOPEN_SELF], -[m4_require([_LT_HEADER_DLFCN])dnl -if test yes = "$cross_compiling"; then : - [$4] -else - lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 - lt_status=$lt_dlunknown - cat > conftest.$ac_ext <<_LT_EOF -[#line $LINENO "configure" -#include "confdefs.h" - -#if HAVE_DLFCN_H -#include -#endif - -#include - -#ifdef RTLD_GLOBAL -# define LT_DLGLOBAL RTLD_GLOBAL -#else -# ifdef DL_GLOBAL -# define LT_DLGLOBAL DL_GLOBAL -# else -# define LT_DLGLOBAL 0 -# endif -#endif - -/* We may have to define LT_DLLAZY_OR_NOW in the command line if we - find out it does not work in some platform. */ -#ifndef LT_DLLAZY_OR_NOW -# ifdef RTLD_LAZY -# define LT_DLLAZY_OR_NOW RTLD_LAZY -# else -# ifdef DL_LAZY -# define LT_DLLAZY_OR_NOW DL_LAZY -# else -# ifdef RTLD_NOW -# define LT_DLLAZY_OR_NOW RTLD_NOW -# else -# ifdef DL_NOW -# define LT_DLLAZY_OR_NOW DL_NOW -# else -# define LT_DLLAZY_OR_NOW 0 -# endif -# endif -# endif -# endif -#endif - -/* When -fvisibility=hidden is used, assume the code has been annotated - correspondingly for the symbols needed. */ -#if defined __GNUC__ && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)) || (__GNUC__ > 3)) -int fnord () __attribute__((visibility("default"))); -#endif - -int fnord () { return 42; } -int main () -{ - void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW); - int status = $lt_dlunknown; - - if (self) - { - if (dlsym (self,"fnord")) status = $lt_dlno_uscore; - else - { - if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore; - else puts (dlerror ()); - } - /* dlclose (self); */ - } - else - puts (dlerror ()); - - return status; -}] -_LT_EOF - if AC_TRY_EVAL(ac_link) && test -s "conftest$ac_exeext" 2>/dev/null; then - (./conftest; exit; ) >&AS_MESSAGE_LOG_FD 2>/dev/null - lt_status=$? - case x$lt_status in - x$lt_dlno_uscore) $1 ;; - x$lt_dlneed_uscore) $2 ;; - x$lt_dlunknown|x*) $3 ;; - esac - else : - # compilation failed - $3 - fi -fi -rm -fr conftest* -])# _LT_TRY_DLOPEN_SELF - - -# LT_SYS_DLOPEN_SELF -# ------------------ -AC_DEFUN([LT_SYS_DLOPEN_SELF], -[m4_require([_LT_HEADER_DLFCN])dnl -if test yes != "$enable_dlopen"; then - enable_dlopen=unknown - enable_dlopen_self=unknown - enable_dlopen_self_static=unknown -else - lt_cv_dlopen=no - lt_cv_dlopen_libs= - - case $host_os in - beos*) - lt_cv_dlopen=load_add_on - lt_cv_dlopen_libs= - lt_cv_dlopen_self=yes - ;; - - mingw* | pw32* | cegcc*) - lt_cv_dlopen=LoadLibrary - lt_cv_dlopen_libs= - ;; - - cygwin*) - lt_cv_dlopen=dlopen - lt_cv_dlopen_libs= - ;; - - darwin*) - # if libdl is installed we need to link against it - AC_CHECK_LIB([dl], [dlopen], - [lt_cv_dlopen=dlopen lt_cv_dlopen_libs=-ldl],[ - lt_cv_dlopen=dyld - lt_cv_dlopen_libs= - lt_cv_dlopen_self=yes - ]) - ;; - - tpf*) - # Don't try to run any link tests for TPF. We know it's impossible - # because TPF is a cross-compiler, and we know how we open DSOs. - lt_cv_dlopen=dlopen - lt_cv_dlopen_libs= - lt_cv_dlopen_self=no - ;; - - *) - AC_CHECK_FUNC([shl_load], - [lt_cv_dlopen=shl_load], - [AC_CHECK_LIB([dld], [shl_load], - [lt_cv_dlopen=shl_load lt_cv_dlopen_libs=-ldld], - [AC_CHECK_FUNC([dlopen], - [lt_cv_dlopen=dlopen], - [AC_CHECK_LIB([dl], [dlopen], - [lt_cv_dlopen=dlopen lt_cv_dlopen_libs=-ldl], - [AC_CHECK_LIB([svld], [dlopen], - [lt_cv_dlopen=dlopen lt_cv_dlopen_libs=-lsvld], - [AC_CHECK_LIB([dld], [dld_link], - [lt_cv_dlopen=dld_link lt_cv_dlopen_libs=-ldld]) - ]) - ]) - ]) - ]) - ]) - ;; - esac - - if test no = "$lt_cv_dlopen"; then - enable_dlopen=no - else - enable_dlopen=yes - fi - - case $lt_cv_dlopen in - dlopen) - save_CPPFLAGS=$CPPFLAGS - test yes = "$ac_cv_header_dlfcn_h" && CPPFLAGS="$CPPFLAGS -DHAVE_DLFCN_H" - - save_LDFLAGS=$LDFLAGS - wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $export_dynamic_flag_spec\" - - save_LIBS=$LIBS - LIBS="$lt_cv_dlopen_libs $LIBS" - - AC_CACHE_CHECK([whether a program can dlopen itself], - lt_cv_dlopen_self, [dnl - _LT_TRY_DLOPEN_SELF( - lt_cv_dlopen_self=yes, lt_cv_dlopen_self=yes, - lt_cv_dlopen_self=no, lt_cv_dlopen_self=cross) - ]) - - if test yes = "$lt_cv_dlopen_self"; then - wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $lt_prog_compiler_static\" - AC_CACHE_CHECK([whether a statically linked program can dlopen itself], - lt_cv_dlopen_self_static, [dnl - _LT_TRY_DLOPEN_SELF( - lt_cv_dlopen_self_static=yes, lt_cv_dlopen_self_static=yes, - lt_cv_dlopen_self_static=no, lt_cv_dlopen_self_static=cross) - ]) - fi - - CPPFLAGS=$save_CPPFLAGS - LDFLAGS=$save_LDFLAGS - LIBS=$save_LIBS - ;; - esac - - case $lt_cv_dlopen_self in - yes|no) enable_dlopen_self=$lt_cv_dlopen_self ;; - *) enable_dlopen_self=unknown ;; - esac - - case $lt_cv_dlopen_self_static in - yes|no) enable_dlopen_self_static=$lt_cv_dlopen_self_static ;; - *) enable_dlopen_self_static=unknown ;; - esac -fi -_LT_DECL([dlopen_support], [enable_dlopen], [0], - [Whether dlopen is supported]) -_LT_DECL([dlopen_self], [enable_dlopen_self], [0], - [Whether dlopen of programs is supported]) -_LT_DECL([dlopen_self_static], [enable_dlopen_self_static], [0], - [Whether dlopen of statically linked programs is supported]) -])# LT_SYS_DLOPEN_SELF - -# Old name: -AU_ALIAS([AC_LIBTOOL_DLOPEN_SELF], [LT_SYS_DLOPEN_SELF]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_DLOPEN_SELF], []) - - -# _LT_COMPILER_C_O([TAGNAME]) -# --------------------------- -# Check to see if options -c and -o are simultaneously supported by compiler. -# This macro does not hard code the compiler like AC_PROG_CC_C_O. -m4_defun([_LT_COMPILER_C_O], -[m4_require([_LT_DECL_SED])dnl -m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_TAG_COMPILER])dnl -AC_CACHE_CHECK([if $compiler supports -c -o file.$ac_objext], - [_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)], - [_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=no - $RM -r conftest 2>/dev/null - mkdir conftest - cd conftest - mkdir out - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - - lt_compiler_flag="-o out/conftest2.$ac_objext" - # Insert the option either (1) after the last *FLAGS variable, or - # (2) before a word containing "conftest.", or (3) at the end. - # Note that $ac_compile itself does not contain backslashes and begins - # with a dollar sign (not a hyphen), so the echo should work correctly. - lt_compile=`echo "$ac_compile" | $SED \ - -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ - -e 's: [[^ ]]*conftest\.: $lt_compiler_flag&:; t' \ - -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&AS_MESSAGE_LOG_FD) - (eval "$lt_compile" 2>out/conftest.err) - ac_status=$? - cat out/conftest.err >&AS_MESSAGE_LOG_FD - echo "$as_me:$LINENO: \$? = $ac_status" >&AS_MESSAGE_LOG_FD - if (exit $ac_status) && test -s out/conftest2.$ac_objext - then - # The compiler can only warn and ignore the option if not recognized - # So say no if there are warnings - $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp - $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 - if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then - _LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=yes - fi - fi - chmod u+w . 2>&AS_MESSAGE_LOG_FD - $RM conftest* - # SGI C++ compiler will create directory out/ii_files/ for - # template instantiation - test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files - $RM out/* && rmdir out - cd .. - $RM -r conftest - $RM conftest* -]) -_LT_TAGDECL([compiler_c_o], [lt_cv_prog_compiler_c_o], [1], - [Does compiler simultaneously support -c and -o options?]) -])# _LT_COMPILER_C_O - - -# _LT_COMPILER_FILE_LOCKS([TAGNAME]) -# ---------------------------------- -# Check to see if we can do hard links to lock some files if needed -m4_defun([_LT_COMPILER_FILE_LOCKS], -[m4_require([_LT_ENABLE_LOCK])dnl -m4_require([_LT_FILEUTILS_DEFAULTS])dnl -_LT_COMPILER_C_O([$1]) - -hard_links=nottested -if test no = "$_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)" && test no != "$need_locks"; then - # do not overwrite the value of need_locks provided by the user - AC_MSG_CHECKING([if we can lock with hard links]) - hard_links=yes - $RM conftest* - ln conftest.a conftest.b 2>/dev/null && hard_links=no - touch conftest.a - ln conftest.a conftest.b 2>&5 || hard_links=no - ln conftest.a conftest.b 2>/dev/null && hard_links=no - AC_MSG_RESULT([$hard_links]) - if test no = "$hard_links"; then - AC_MSG_WARN(['$CC' does not support '-c -o', so 'make -j' may be unsafe]) - need_locks=warn - fi -else - need_locks=no -fi -_LT_DECL([], [need_locks], [1], [Must we lock files when doing compilation?]) -])# _LT_COMPILER_FILE_LOCKS - - -# _LT_CHECK_OBJDIR -# ---------------- -m4_defun([_LT_CHECK_OBJDIR], -[AC_CACHE_CHECK([for objdir], [lt_cv_objdir], -[rm -f .libs 2>/dev/null -mkdir .libs 2>/dev/null -if test -d .libs; then - lt_cv_objdir=.libs -else - # MS-DOS does not allow filenames that begin with a dot. - lt_cv_objdir=_libs -fi -rmdir .libs 2>/dev/null]) -objdir=$lt_cv_objdir -_LT_DECL([], [objdir], [0], - [The name of the directory that contains temporary libtool files])dnl -m4_pattern_allow([LT_OBJDIR])dnl -AC_DEFINE_UNQUOTED([LT_OBJDIR], "$lt_cv_objdir/", - [Define to the sub-directory where libtool stores uninstalled libraries.]) -])# _LT_CHECK_OBJDIR - - -# _LT_LINKER_HARDCODE_LIBPATH([TAGNAME]) -# -------------------------------------- -# Check hardcoding attributes. -m4_defun([_LT_LINKER_HARDCODE_LIBPATH], -[AC_MSG_CHECKING([how to hardcode library paths into programs]) -_LT_TAGVAR(hardcode_action, $1)= -if test -n "$_LT_TAGVAR(hardcode_libdir_flag_spec, $1)" || - test -n "$_LT_TAGVAR(runpath_var, $1)" || - test yes = "$_LT_TAGVAR(hardcode_automatic, $1)"; then - - # We can hardcode non-existent directories. - if test no != "$_LT_TAGVAR(hardcode_direct, $1)" && - # If the only mechanism to avoid hardcoding is shlibpath_var, we - # have to relink, otherwise we might link with an installed library - # when we should be linking with a yet-to-be-installed one - ## test no != "$_LT_TAGVAR(hardcode_shlibpath_var, $1)" && - test no != "$_LT_TAGVAR(hardcode_minus_L, $1)"; then - # Linking always hardcodes the temporary library directory. - _LT_TAGVAR(hardcode_action, $1)=relink - else - # We can link without hardcoding, and we can hardcode nonexisting dirs. - _LT_TAGVAR(hardcode_action, $1)=immediate - fi -else - # We cannot hardcode anything, or else we can only hardcode existing - # directories. - _LT_TAGVAR(hardcode_action, $1)=unsupported -fi -AC_MSG_RESULT([$_LT_TAGVAR(hardcode_action, $1)]) - -if test relink = "$_LT_TAGVAR(hardcode_action, $1)" || - test yes = "$_LT_TAGVAR(inherit_rpath, $1)"; then - # Fast installation is not supported - enable_fast_install=no -elif test yes = "$shlibpath_overrides_runpath" || - test no = "$enable_shared"; then - # Fast installation is not necessary - enable_fast_install=needless -fi -_LT_TAGDECL([], [hardcode_action], [0], - [How to hardcode a shared library path into an executable]) -])# _LT_LINKER_HARDCODE_LIBPATH - - -# _LT_CMD_STRIPLIB -# ---------------- -m4_defun([_LT_CMD_STRIPLIB], -[m4_require([_LT_DECL_EGREP]) -striplib= -old_striplib= -AC_MSG_CHECKING([whether stripping libraries is possible]) -if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then - test -z "$old_striplib" && old_striplib="$STRIP --strip-debug" - test -z "$striplib" && striplib="$STRIP --strip-unneeded" - AC_MSG_RESULT([yes]) -else -# FIXME - insert some real tests, host_os isn't really good enough - case $host_os in - darwin*) - if test -n "$STRIP"; then - striplib="$STRIP -x" - old_striplib="$STRIP -S" - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi - ;; - *) - AC_MSG_RESULT([no]) - ;; - esac -fi -_LT_DECL([], [old_striplib], [1], [Commands to strip libraries]) -_LT_DECL([], [striplib], [1]) -])# _LT_CMD_STRIPLIB - - -# _LT_PREPARE_MUNGE_PATH_LIST -# --------------------------- -# Make sure func_munge_path_list() is defined correctly. -m4_defun([_LT_PREPARE_MUNGE_PATH_LIST], -[[# func_munge_path_list VARIABLE PATH -# ----------------------------------- -# VARIABLE is name of variable containing _space_ separated list of -# directories to be munged by the contents of PATH, which is string -# having a format: -# "DIR[:DIR]:" -# string "DIR[ DIR]" will be prepended to VARIABLE -# ":DIR[:DIR]" -# string "DIR[ DIR]" will be appended to VARIABLE -# "DIRP[:DIRP]::[DIRA:]DIRA" -# string "DIRP[ DIRP]" will be prepended to VARIABLE and string -# "DIRA[ DIRA]" will be appended to VARIABLE -# "DIR[:DIR]" -# VARIABLE will be replaced by "DIR[ DIR]" -func_munge_path_list () -{ - case x@S|@2 in - x) - ;; - *:) - eval @S|@1=\"`$ECHO @S|@2 | $SED 's/:/ /g'` \@S|@@S|@1\" - ;; - x:*) - eval @S|@1=\"\@S|@@S|@1 `$ECHO @S|@2 | $SED 's/:/ /g'`\" - ;; - *::*) - eval @S|@1=\"\@S|@@S|@1\ `$ECHO @S|@2 | $SED -e 's/.*:://' -e 's/:/ /g'`\" - eval @S|@1=\"`$ECHO @S|@2 | $SED -e 's/::.*//' -e 's/:/ /g'`\ \@S|@@S|@1\" - ;; - *) - eval @S|@1=\"`$ECHO @S|@2 | $SED 's/:/ /g'`\" - ;; - esac -} -]])# _LT_PREPARE_PATH_LIST - - -# _LT_SYS_DYNAMIC_LINKER([TAG]) -# ----------------------------- -# PORTME Fill in your ld.so characteristics -m4_defun([_LT_SYS_DYNAMIC_LINKER], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -m4_require([_LT_DECL_EGREP])dnl -m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_DECL_OBJDUMP])dnl -m4_require([_LT_DECL_SED])dnl -m4_require([_LT_CHECK_SHELL_FEATURES])dnl -m4_require([_LT_PREPARE_MUNGE_PATH_LIST])dnl -AC_MSG_CHECKING([dynamic linker characteristics]) -m4_if([$1], - [], [ -if test yes = "$GCC"; then - case $host_os in - darwin*) lt_awk_arg='/^libraries:/,/LR/' ;; - *) lt_awk_arg='/^libraries:/' ;; - esac - case $host_os in - mingw* | cegcc*) lt_sed_strip_eq='s|=\([[A-Za-z]]:\)|\1|g' ;; - *) lt_sed_strip_eq='s|=/|/|g' ;; - esac - lt_search_path_spec=`$CC -print-search-dirs | awk $lt_awk_arg | $SED -e "s/^libraries://" -e $lt_sed_strip_eq` - case $lt_search_path_spec in - *\;*) - # if the path contains ";" then we assume it to be the separator - # otherwise default to the standard path separator (i.e. ":") - it is - # assumed that no part of a normal pathname contains ";" but that should - # okay in the real world where ";" in dirpaths is itself problematic. - lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED 's/;/ /g'` - ;; - *) - lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED "s/$PATH_SEPARATOR/ /g"` - ;; - esac - # Ok, now we have the path, separated by spaces, we can step through it - # and add multilib dir if necessary... - lt_tmp_lt_search_path_spec= - lt_multi_os_dir=/`$CC $CPPFLAGS $CFLAGS $LDFLAGS -print-multi-os-directory 2>/dev/null` - # ...but if some path component already ends with the multilib dir we assume - # that all is fine and trust -print-search-dirs as is (GCC 4.2? or newer). - case "$lt_multi_os_dir; $lt_search_path_spec " in - "/; "* | "/.; "* | "/./; "* | *"$lt_multi_os_dir "* | *"$lt_multi_os_dir/ "*) - lt_multi_os_dir= - ;; - esac - for lt_sys_path in $lt_search_path_spec; do - if test -d "$lt_sys_path$lt_multi_os_dir"; then - lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path$lt_multi_os_dir" - elif test -n "$lt_multi_os_dir"; then - test -d "$lt_sys_path" && \ - lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path" - fi - done - lt_search_path_spec=`$ECHO "$lt_tmp_lt_search_path_spec" | awk ' -BEGIN {RS = " "; FS = "/|\n";} { - lt_foo = ""; - lt_count = 0; - for (lt_i = NF; lt_i > 0; lt_i--) { - if ($lt_i != "" && $lt_i != ".") { - if ($lt_i == "..") { - lt_count++; - } else { - if (lt_count == 0) { - lt_foo = "/" $lt_i lt_foo; - } else { - lt_count--; - } - } - } - } - if (lt_foo != "") { lt_freq[[lt_foo]]++; } - if (lt_freq[[lt_foo]] == 1) { print lt_foo; } -}'` - # AWK program above erroneously prepends '/' to C:/dos/paths - # for these hosts. - case $host_os in - mingw* | cegcc*) lt_search_path_spec=`$ECHO "$lt_search_path_spec" |\ - $SED 's|/\([[A-Za-z]]:\)|\1|g'` ;; - esac - sys_lib_search_path_spec=`$ECHO "$lt_search_path_spec" | $lt_NL2SP` -else - sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib" -fi]) -library_names_spec= -libname_spec='lib$name' -soname_spec= -shrext_cmds=.so -postinstall_cmds= -postuninstall_cmds= -finish_cmds= -finish_eval= -shlibpath_var= -shlibpath_overrides_runpath=unknown -version_type=none -dynamic_linker="$host_os ld.so" -sys_lib_dlsearch_path_spec="/lib /usr/lib" -need_lib_prefix=unknown -hardcode_into_libs=no - -# when you set need_version to no, make sure it does not cause -set_version -# flags to be left without arguments -need_version=unknown - -AC_ARG_VAR([LT_SYS_LIBRARY_PATH], -[User-defined run-time library search path.]) - -case $host_os in -aix3*) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname.a' - shlibpath_var=LIBPATH - - # AIX 3 has no versioning support, so we append a major version to the name. - soname_spec='$libname$release$shared_ext$major' - ;; - -aix[[4-9]]*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - hardcode_into_libs=yes - if test ia64 = "$host_cpu"; then - # AIX 5 supports IA64 - library_names_spec='$libname$release$shared_ext$major $libname$release$shared_ext$versuffix $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - else - # With GCC up to 2.95.x, collect2 would create an import file - # for dependence libraries. The import file would start with - # the line '#! .'. This would cause the generated library to - # depend on '.', always an invalid library. This was fixed in - # development snapshots of GCC prior to 3.0. - case $host_os in - aix4 | aix4.[[01]] | aix4.[[01]].*) - if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)' - echo ' yes ' - echo '#endif'; } | $CC -E - | $GREP yes > /dev/null; then - : - else - can_build_shared=no - fi - ;; - esac - # Using Import Files as archive members, it is possible to support - # filename-based versioning of shared library archives on AIX. While - # this would work for both with and without runtime linking, it will - # prevent static linking of such archives. So we do filename-based - # shared library versioning with .so extension only, which is used - # when both runtime linking and shared linking is enabled. - # Unfortunately, runtime linking may impact performance, so we do - # not want this to be the default eventually. Also, we use the - # versioned .so libs for executables only if there is the -brtl - # linker flag in LDFLAGS as well, or --with-aix-soname=svr4 only. - # To allow for filename-based versioning support, we need to create - # libNAME.so.V as an archive file, containing: - # *) an Import File, referring to the versioned filename of the - # archive as well as the shared archive member, telling the - # bitwidth (32 or 64) of that shared object, and providing the - # list of exported symbols of that shared object, eventually - # decorated with the 'weak' keyword - # *) the shared object with the F_LOADONLY flag set, to really avoid - # it being seen by the linker. - # At run time we better use the real file rather than another symlink, - # but for link time we create the symlink libNAME.so -> libNAME.so.V - - case $with_aix_soname,$aix_use_runtimelinking in - # AIX (on Power*) has no versioning support, so currently we cannot hardcode correct - # soname into executable. Probably we can add versioning support to - # collect2, so additional links can be useful in future. - aix,yes) # traditional libtool - dynamic_linker='AIX unversionable lib.so' - # If using run time linking (on AIX 4.2 or later) use lib.so - # instead of lib.a to let people know that these are not - # typical AIX shared libraries. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - ;; - aix,no) # traditional AIX only - dynamic_linker='AIX lib.a[(]lib.so.V[)]' - # We preserve .a as extension for shared libraries through AIX4.2 - # and later when we are not doing run time linking. - library_names_spec='$libname$release.a $libname.a' - soname_spec='$libname$release$shared_ext$major' - ;; - svr4,*) # full svr4 only - dynamic_linker="AIX lib.so.V[(]$shared_archive_member_spec.o[)]" - library_names_spec='$libname$release$shared_ext$major $libname$shared_ext' - # We do not specify a path in Import Files, so LIBPATH fires. - shlibpath_overrides_runpath=yes - ;; - *,yes) # both, prefer svr4 - dynamic_linker="AIX lib.so.V[(]$shared_archive_member_spec.o[)], lib.a[(]lib.so.V[)]" - library_names_spec='$libname$release$shared_ext$major $libname$shared_ext' - # unpreferred sharedlib libNAME.a needs extra handling - postinstall_cmds='test -n "$linkname" || linkname="$realname"~func_stripname "" ".so" "$linkname"~$install_shared_prog "$dir/$func_stripname_result.$libext" "$destdir/$func_stripname_result.$libext"~test -z "$tstripme" || test -z "$striplib" || $striplib "$destdir/$func_stripname_result.$libext"' - postuninstall_cmds='for n in $library_names $old_library; do :; done~func_stripname "" ".so" "$n"~test "$func_stripname_result" = "$n" || func_append rmfiles " $odir/$func_stripname_result.$libext"' - # We do not specify a path in Import Files, so LIBPATH fires. - shlibpath_overrides_runpath=yes - ;; - *,no) # both, prefer aix - dynamic_linker="AIX lib.a[(]lib.so.V[)], lib.so.V[(]$shared_archive_member_spec.o[)]" - library_names_spec='$libname$release.a $libname.a' - soname_spec='$libname$release$shared_ext$major' - # unpreferred sharedlib libNAME.so.V and symlink libNAME.so need extra handling - postinstall_cmds='test -z "$dlname" || $install_shared_prog $dir/$dlname $destdir/$dlname~test -z "$tstripme" || test -z "$striplib" || $striplib $destdir/$dlname~test -n "$linkname" || linkname=$realname~func_stripname "" ".a" "$linkname"~(cd "$destdir" && $LN_S -f $dlname $func_stripname_result.so)' - postuninstall_cmds='test -z "$dlname" || func_append rmfiles " $odir/$dlname"~for n in $old_library $library_names; do :; done~func_stripname "" ".a" "$n"~func_append rmfiles " $odir/$func_stripname_result.so"' - ;; - esac - shlibpath_var=LIBPATH - fi - ;; - -amigaos*) - case $host_cpu in - powerpc) - # Since July 2007 AmigaOS4 officially supports .so libraries. - # When compiling the executable, add -use-dynld -Lsobjs: to the compileline. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - ;; - m68k) - library_names_spec='$libname.ixlibrary $libname.a' - # Create ${libname}_ixlibrary.a entries in /sys/libs. - finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`func_echo_all "$lib" | $SED '\''s%^.*/\([[^/]]*\)\.ixlibrary$%\1%'\''`; $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done' - ;; - esac - ;; - -beos*) - library_names_spec='$libname$shared_ext' - dynamic_linker="$host_os ld.so" - shlibpath_var=LIBRARY_PATH - ;; - -bsdi[[45]]*) - version_type=linux # correct to gnu/linux during the next big refactor - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir' - shlibpath_var=LD_LIBRARY_PATH - sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib" - sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib" - # the default ld.so.conf also contains /usr/contrib/lib and - # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow - # libtool to hard-code these into programs - ;; - -cygwin* | mingw* | pw32* | cegcc*) - version_type=windows - shrext_cmds=.dll - need_version=no - need_lib_prefix=no - - case $GCC,$cc_basename in - yes,*) - # gcc - library_names_spec='$libname.dll.a' - # DLL is installed to $(libdir)/../bin by postinstall_cmds - postinstall_cmds='base_file=`basename \$file`~ - dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\$base_file'\''i; echo \$dlname'\''`~ - dldir=$destdir/`dirname \$dlpath`~ - test -d \$dldir || mkdir -p \$dldir~ - $install_prog $dir/$dlname \$dldir/$dlname~ - chmod a+x \$dldir/$dlname~ - if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then - eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; - fi' - postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ - dlpath=$dir/\$dldll~ - $RM \$dlpath' - shlibpath_overrides_runpath=yes - - case $host_os in - cygwin*) - # Cygwin DLLs use 'cyg' prefix rather than 'lib' - soname_spec='`echo $libname | sed -e 's/^lib/cyg/'``echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' -m4_if([$1], [],[ - sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/lib/w32api"]) - ;; - mingw* | cegcc*) - # MinGW DLLs use traditional 'lib' prefix - soname_spec='$libname`echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' - ;; - pw32*) - # pw32 DLLs use 'pw' prefix rather than 'lib' - library_names_spec='`echo $libname | sed -e 's/^lib/pw/'``echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' - ;; - esac - dynamic_linker='Win32 ld.exe' - ;; - - *,cl*) - # Native MSVC - libname_spec='$name' - soname_spec='$libname`echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext' - library_names_spec='$libname.dll.lib' - - case $build_os in - mingw*) - sys_lib_search_path_spec= - lt_save_ifs=$IFS - IFS=';' - for lt_path in $LIB - do - IFS=$lt_save_ifs - # Let DOS variable expansion print the short 8.3 style file name. - lt_path=`cd "$lt_path" 2>/dev/null && cmd //C "for %i in (".") do @echo %~si"` - sys_lib_search_path_spec="$sys_lib_search_path_spec $lt_path" - done - IFS=$lt_save_ifs - # Convert to MSYS style. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([[a-zA-Z]]\\):| /\\1|g' -e 's|^ ||'` - ;; - cygwin*) - # Convert to unix form, then to dos form, then back to unix form - # but this time dos style (no spaces!) so that the unix form looks - # like /cygdrive/c/PROGRA~1:/cygdr... - sys_lib_search_path_spec=`cygpath --path --unix "$LIB"` - sys_lib_search_path_spec=`cygpath --path --dos "$sys_lib_search_path_spec" 2>/dev/null` - sys_lib_search_path_spec=`cygpath --path --unix "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` - ;; - *) - sys_lib_search_path_spec=$LIB - if $ECHO "$sys_lib_search_path_spec" | [$GREP ';[c-zC-Z]:/' >/dev/null]; then - # It is most probably a Windows format PATH. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'` - else - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` - fi - # FIXME: find the short name or the path components, as spaces are - # common. (e.g. "Program Files" -> "PROGRA~1") - ;; - esac - - # DLL is installed to $(libdir)/../bin by postinstall_cmds - postinstall_cmds='base_file=`basename \$file`~ - dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\$base_file'\''i; echo \$dlname'\''`~ - dldir=$destdir/`dirname \$dlpath`~ - test -d \$dldir || mkdir -p \$dldir~ - $install_prog $dir/$dlname \$dldir/$dlname' - postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ - dlpath=$dir/\$dldll~ - $RM \$dlpath' - shlibpath_overrides_runpath=yes - dynamic_linker='Win32 link.exe' - ;; - - *) - # Assume MSVC wrapper - library_names_spec='$libname`echo $release | $SED -e 's/[[.]]/-/g'`$versuffix$shared_ext $libname.lib' - dynamic_linker='Win32 ld.exe' - ;; - esac - # FIXME: first we should search . and the directory the executable is in - shlibpath_var=PATH - ;; - -darwin* | rhapsody*) - dynamic_linker="$host_os dyld" - version_type=darwin - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$major$shared_ext $libname$shared_ext' - soname_spec='$libname$release$major$shared_ext' - shlibpath_overrides_runpath=yes - shlibpath_var=DYLD_LIBRARY_PATH - shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`' -m4_if([$1], [],[ - sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/local/lib"]) - sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib' - ;; - -dgux*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - ;; - -freebsd* | dragonfly*) - # DragonFly does not have aout. When/if they implement a new - # versioning mechanism, adjust this. - if test -x /usr/bin/objformat; then - objformat=`/usr/bin/objformat` - else - case $host_os in - freebsd[[23]].*) objformat=aout ;; - *) objformat=elf ;; - esac - fi - version_type=freebsd-$objformat - case $version_type in - freebsd-elf*) - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - need_version=no - need_lib_prefix=no - ;; - freebsd-*) - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - need_version=yes - ;; - esac - shlibpath_var=LD_LIBRARY_PATH - case $host_os in - freebsd2.*) - shlibpath_overrides_runpath=yes - ;; - freebsd3.[[01]]* | freebsdelf3.[[01]]*) - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - ;; - freebsd3.[[2-9]]* | freebsdelf3.[[2-9]]* | \ - freebsd4.[[0-5]] | freebsdelf4.[[0-5]] | freebsd4.1.1 | freebsdelf4.1.1) - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - ;; - *) # from 4.6 on, and DragonFly - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - ;; - esac - ;; - -haiku*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - dynamic_linker="$host_os runtime_loader" - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LIBRARY_PATH - shlibpath_overrides_runpath=no - sys_lib_dlsearch_path_spec='/boot/home/config/lib /boot/common/lib /boot/system/lib' - hardcode_into_libs=yes - ;; - -hpux9* | hpux10* | hpux11*) - # Give a soname corresponding to the major version so that dld.sl refuses to - # link against other versions. - version_type=sunos - need_lib_prefix=no - need_version=no - case $host_cpu in - ia64*) - shrext_cmds='.so' - hardcode_into_libs=yes - dynamic_linker="$host_os dld.so" - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - if test 32 = "$HPUX_IA64_MODE"; then - sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib" - sys_lib_dlsearch_path_spec=/usr/lib/hpux32 - else - sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64" - sys_lib_dlsearch_path_spec=/usr/lib/hpux64 - fi - ;; - hppa*64*) - shrext_cmds='.sl' - hardcode_into_libs=yes - dynamic_linker="$host_os dld.sl" - shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH - shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64" - sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec - ;; - *) - shrext_cmds='.sl' - dynamic_linker="$host_os dld.sl" - shlibpath_var=SHLIB_PATH - shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - ;; - esac - # HP-UX runs *really* slowly unless shared libraries are mode 555, ... - postinstall_cmds='chmod 555 $lib' - # or fails outright, so override atomically: - install_override_mode=555 - ;; - -interix[[3-9]]*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - ;; - -irix5* | irix6* | nonstopux*) - case $host_os in - nonstopux*) version_type=nonstopux ;; - *) - if test yes = "$lt_cv_prog_gnu_ld"; then - version_type=linux # correct to gnu/linux during the next big refactor - else - version_type=irix - fi ;; - esac - need_lib_prefix=no - need_version=no - soname_spec='$libname$release$shared_ext$major' - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$release$shared_ext $libname$shared_ext' - case $host_os in - irix5* | nonstopux*) - libsuff= shlibsuff= - ;; - *) - case $LD in # libtool.m4 will add one of these switches to LD - *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ") - libsuff= shlibsuff= libmagic=32-bit;; - *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ") - libsuff=32 shlibsuff=N32 libmagic=N32;; - *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ") - libsuff=64 shlibsuff=64 libmagic=64-bit;; - *) libsuff= shlibsuff= libmagic=never-match;; - esac - ;; - esac - shlibpath_var=LD_LIBRARY${shlibsuff}_PATH - shlibpath_overrides_runpath=no - sys_lib_search_path_spec="/usr/lib$libsuff /lib$libsuff /usr/local/lib$libsuff" - sys_lib_dlsearch_path_spec="/usr/lib$libsuff /lib$libsuff" - hardcode_into_libs=yes - ;; - -# No shared lib support for Linux oldld, aout, or coff. -linux*oldld* | linux*aout* | linux*coff*) - dynamic_linker=no - ;; - -linux*android*) - version_type=none # Android doesn't support versioned libraries. - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext' - soname_spec='$libname$release$shared_ext' - finish_cmds= - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - - # This implies no fast_install, which is unacceptable. - # Some rework will be needed to allow for fast_install - # before this can be enabled. - hardcode_into_libs=yes - - dynamic_linker='Android linker' - # Don't embed -rpath directories since the linker doesn't support them. - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - ;; - -# This must be glibc/ELF. -linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - - # Some binutils ld are patched to set DT_RUNPATH - AC_CACHE_VAL([lt_cv_shlibpath_overrides_runpath], - [lt_cv_shlibpath_overrides_runpath=no - save_LDFLAGS=$LDFLAGS - save_libdir=$libdir - eval "libdir=/foo; wl=\"$_LT_TAGVAR(lt_prog_compiler_wl, $1)\"; \ - LDFLAGS=\"\$LDFLAGS $_LT_TAGVAR(hardcode_libdir_flag_spec, $1)\"" - AC_LINK_IFELSE([AC_LANG_PROGRAM([],[])], - [AS_IF([ ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null], - [lt_cv_shlibpath_overrides_runpath=yes])]) - LDFLAGS=$save_LDFLAGS - libdir=$save_libdir - ]) - shlibpath_overrides_runpath=$lt_cv_shlibpath_overrides_runpath - - # This implies no fast_install, which is unacceptable. - # Some rework will be needed to allow for fast_install - # before this can be enabled. - hardcode_into_libs=yes - - # Ideally, we could use ldconfig to report *all* directores which are - # searched for libraries, however this is still not possible. Aside from not - # being certain /sbin/ldconfig is available, command - # 'ldconfig -N -X -v | grep ^/' on 64bit Fedora does not report /usr/lib64, - # even though it is searched at run-time. Try to do the best guess by - # appending ld.so.conf contents (and includes) to the search path. - if test -f /etc/ld.so.conf; then - lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \[$]2)); skip = 1; } { if (!skip) print \[$]0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;s/"//g;/^$/d' | tr '\n' ' '` - sys_lib_dlsearch_path_spec="/lib /usr/lib $lt_ld_extra" - fi - - # We used to test for /lib/ld.so.1 and disable shared libraries on - # powerpc, because MkLinux only supported shared libraries with the - # GNU dynamic linker. Since this was broken with cross compilers, - # most powerpc-linux boxes support dynamic linking these days and - # people can always --disable-shared, the test was removed, and we - # assume the GNU/Linux dynamic linker is in use. - dynamic_linker='GNU/Linux ld.so' - ;; - -netbsdelf*-gnu) - version_type=linux - need_lib_prefix=no - need_version=no - library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' - soname_spec='${libname}${release}${shared_ext}$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - dynamic_linker='NetBSD ld.elf_so' - ;; - -netbsd*) - version_type=sunos - need_lib_prefix=no - need_version=no - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' - dynamic_linker='NetBSD (a.out) ld.so' - else - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - dynamic_linker='NetBSD ld.elf_so' - fi - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - ;; - -newsos6) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - ;; - -*nto* | *qnx*) - version_type=qnx - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - dynamic_linker='ldqnx.so' - ;; - -openbsd* | bitrig*) - version_type=sunos - sys_lib_dlsearch_path_spec=/usr/lib - need_lib_prefix=no - if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`"; then - need_version=no - else - need_version=yes - fi - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - ;; - -os2*) - libname_spec='$name' - version_type=windows - shrext_cmds=.dll - need_version=no - need_lib_prefix=no - # OS/2 can only load a DLL with a base name of 8 characters or less. - soname_spec='`test -n "$os2dllname" && libname="$os2dllname"; - v=$($ECHO $release$versuffix | tr -d .-); - n=$($ECHO $libname | cut -b -$((8 - ${#v})) | tr . _); - $ECHO $n$v`$shared_ext' - library_names_spec='${libname}_dll.$libext' - dynamic_linker='OS/2 ld.exe' - shlibpath_var=BEGINLIBPATH - sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib" - sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec - postinstall_cmds='base_file=`basename \$file`~ - dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\$base_file'\''i; $ECHO \$dlname'\''`~ - dldir=$destdir/`dirname \$dlpath`~ - test -d \$dldir || mkdir -p \$dldir~ - $install_prog $dir/$dlname \$dldir/$dlname~ - chmod a+x \$dldir/$dlname~ - if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then - eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; - fi' - postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; $ECHO \$dlname'\''`~ - dlpath=$dir/\$dldll~ - $RM \$dlpath' - ;; - -osf3* | osf4* | osf5*) - version_type=osf - need_lib_prefix=no - need_version=no - soname_spec='$libname$release$shared_ext$major' - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib" - sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec - ;; - -rdos*) - dynamic_linker=no - ;; - -solaris*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - # ldd complains unless libraries are executable - postinstall_cmds='chmod +x $lib' - ;; - -sunos4*) - version_type=sunos - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - if test yes = "$with_gnu_ld"; then - need_lib_prefix=no - fi - need_version=yes - ;; - -sysv4 | sysv4.3*) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - case $host_vendor in - sni) - shlibpath_overrides_runpath=no - need_lib_prefix=no - runpath_var=LD_RUN_PATH - ;; - siemens) - need_lib_prefix=no - ;; - motorola) - need_lib_prefix=no - need_version=no - shlibpath_overrides_runpath=no - sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib' - ;; - esac - ;; - -sysv4*MP*) - if test -d /usr/nec; then - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$shared_ext.$versuffix $libname$shared_ext.$major $libname$shared_ext' - soname_spec='$libname$shared_ext.$major' - shlibpath_var=LD_LIBRARY_PATH - fi - ;; - -sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) - version_type=sco - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - if test yes = "$with_gnu_ld"; then - sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib' - else - sys_lib_search_path_spec='/usr/ccs/lib /usr/lib' - case $host_os in - sco3.2v5*) - sys_lib_search_path_spec="$sys_lib_search_path_spec /lib" - ;; - esac - fi - sys_lib_dlsearch_path_spec='/usr/lib' - ;; - -tpf*) - # TPF is a cross-target only. Preferred cross-host = GNU/Linux. - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - ;; - -uts4*) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - ;; - -*) - dynamic_linker=no - ;; -esac -AC_MSG_RESULT([$dynamic_linker]) -test no = "$dynamic_linker" && can_build_shared=no - -variables_saved_for_relink="PATH $shlibpath_var $runpath_var" -if test yes = "$GCC"; then - variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH" -fi - -if test set = "${lt_cv_sys_lib_search_path_spec+set}"; then - sys_lib_search_path_spec=$lt_cv_sys_lib_search_path_spec -fi - -if test set = "${lt_cv_sys_lib_dlsearch_path_spec+set}"; then - sys_lib_dlsearch_path_spec=$lt_cv_sys_lib_dlsearch_path_spec -fi - -# remember unaugmented sys_lib_dlsearch_path content for libtool script decls... -configure_time_dlsearch_path=$sys_lib_dlsearch_path_spec - -# ... but it needs LT_SYS_LIBRARY_PATH munging for other configure-time code -func_munge_path_list sys_lib_dlsearch_path_spec "$LT_SYS_LIBRARY_PATH" - -# to be used as default LT_SYS_LIBRARY_PATH value in generated libtool -configure_time_lt_sys_library_path=$LT_SYS_LIBRARY_PATH - -_LT_DECL([], [variables_saved_for_relink], [1], - [Variables whose values should be saved in libtool wrapper scripts and - restored at link time]) -_LT_DECL([], [need_lib_prefix], [0], - [Do we need the "lib" prefix for modules?]) -_LT_DECL([], [need_version], [0], [Do we need a version for libraries?]) -_LT_DECL([], [version_type], [0], [Library versioning type]) -_LT_DECL([], [runpath_var], [0], [Shared library runtime path variable]) -_LT_DECL([], [shlibpath_var], [0],[Shared library path variable]) -_LT_DECL([], [shlibpath_overrides_runpath], [0], - [Is shlibpath searched before the hard-coded library search path?]) -_LT_DECL([], [libname_spec], [1], [Format of library name prefix]) -_LT_DECL([], [library_names_spec], [1], - [[List of archive names. First name is the real one, the rest are links. - The last name is the one that the linker finds with -lNAME]]) -_LT_DECL([], [soname_spec], [1], - [[The coded name of the library, if different from the real name]]) -_LT_DECL([], [install_override_mode], [1], - [Permission mode override for installation of shared libraries]) -_LT_DECL([], [postinstall_cmds], [2], - [Command to use after installation of a shared archive]) -_LT_DECL([], [postuninstall_cmds], [2], - [Command to use after uninstallation of a shared archive]) -_LT_DECL([], [finish_cmds], [2], - [Commands used to finish a libtool library installation in a directory]) -_LT_DECL([], [finish_eval], [1], - [[As "finish_cmds", except a single script fragment to be evaled but - not shown]]) -_LT_DECL([], [hardcode_into_libs], [0], - [Whether we should hardcode library paths into libraries]) -_LT_DECL([], [sys_lib_search_path_spec], [2], - [Compile-time system search path for libraries]) -_LT_DECL([sys_lib_dlsearch_path_spec], [configure_time_dlsearch_path], [2], - [Detected run-time system search path for libraries]) -_LT_DECL([], [configure_time_lt_sys_library_path], [2], - [Explicit LT_SYS_LIBRARY_PATH set during ./configure time]) -])# _LT_SYS_DYNAMIC_LINKER - - -# _LT_PATH_TOOL_PREFIX(TOOL) -# -------------------------- -# find a file program that can recognize shared library -AC_DEFUN([_LT_PATH_TOOL_PREFIX], -[m4_require([_LT_DECL_EGREP])dnl -AC_MSG_CHECKING([for $1]) -AC_CACHE_VAL(lt_cv_path_MAGIC_CMD, -[case $MAGIC_CMD in -[[\\/*] | ?:[\\/]*]) - lt_cv_path_MAGIC_CMD=$MAGIC_CMD # Let the user override the test with a path. - ;; -*) - lt_save_MAGIC_CMD=$MAGIC_CMD - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR -dnl $ac_dummy forces splitting on constant user-supplied paths. -dnl POSIX.2 word splitting is done only on the output of word expansions, -dnl not every word. This closes a longstanding sh security hole. - ac_dummy="m4_if([$2], , $PATH, [$2])" - for ac_dir in $ac_dummy; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - if test -f "$ac_dir/$1"; then - lt_cv_path_MAGIC_CMD=$ac_dir/"$1" - if test -n "$file_magic_test_file"; then - case $deplibs_check_method in - "file_magic "*) - file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"` - MAGIC_CMD=$lt_cv_path_MAGIC_CMD - if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null | - $EGREP "$file_magic_regex" > /dev/null; then - : - else - cat <<_LT_EOF 1>&2 - -*** Warning: the command libtool uses to detect shared libraries, -*** $file_magic_cmd, produces output that libtool cannot recognize. -*** The result is that libtool may fail to recognize shared libraries -*** as such. This will affect the creation of libtool libraries that -*** depend on shared libraries, but programs linked with such libtool -*** libraries will work regardless of this problem. Nevertheless, you -*** may want to report the problem to your system manager and/or to -*** bug-libtool@gnu.org - -_LT_EOF - fi ;; - esac - fi - break - fi - done - IFS=$lt_save_ifs - MAGIC_CMD=$lt_save_MAGIC_CMD - ;; -esac]) -MAGIC_CMD=$lt_cv_path_MAGIC_CMD -if test -n "$MAGIC_CMD"; then - AC_MSG_RESULT($MAGIC_CMD) -else - AC_MSG_RESULT(no) -fi -_LT_DECL([], [MAGIC_CMD], [0], - [Used to examine libraries when file_magic_cmd begins with "file"])dnl -])# _LT_PATH_TOOL_PREFIX - -# Old name: -AU_ALIAS([AC_PATH_TOOL_PREFIX], [_LT_PATH_TOOL_PREFIX]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_PATH_TOOL_PREFIX], []) - - -# _LT_PATH_MAGIC -# -------------- -# find a file program that can recognize a shared library -m4_defun([_LT_PATH_MAGIC], -[_LT_PATH_TOOL_PREFIX(${ac_tool_prefix}file, /usr/bin$PATH_SEPARATOR$PATH) -if test -z "$lt_cv_path_MAGIC_CMD"; then - if test -n "$ac_tool_prefix"; then - _LT_PATH_TOOL_PREFIX(file, /usr/bin$PATH_SEPARATOR$PATH) - else - MAGIC_CMD=: - fi -fi -])# _LT_PATH_MAGIC - - -# LT_PATH_LD -# ---------- -# find the pathname to the GNU or non-GNU linker -AC_DEFUN([LT_PATH_LD], -[AC_REQUIRE([AC_PROG_CC])dnl -AC_REQUIRE([AC_CANONICAL_HOST])dnl -AC_REQUIRE([AC_CANONICAL_BUILD])dnl -m4_require([_LT_DECL_SED])dnl -m4_require([_LT_DECL_EGREP])dnl -m4_require([_LT_PROG_ECHO_BACKSLASH])dnl - -AC_ARG_WITH([gnu-ld], - [AS_HELP_STRING([--with-gnu-ld], - [assume the C compiler uses GNU ld @<:@default=no@:>@])], - [test no = "$withval" || with_gnu_ld=yes], - [with_gnu_ld=no])dnl - -ac_prog=ld -if test yes = "$GCC"; then - # Check if gcc -print-prog-name=ld gives a path. - AC_MSG_CHECKING([for ld used by $CC]) - case $host in - *-*-mingw*) - # gcc leaves a trailing carriage return, which upsets mingw - ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;; - *) - ac_prog=`($CC -print-prog-name=ld) 2>&5` ;; - esac - case $ac_prog in - # Accept absolute paths. - [[\\/]]* | ?:[[\\/]]*) - re_direlt='/[[^/]][[^/]]*/\.\./' - # Canonicalize the pathname of ld - ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'` - while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do - ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"` - done - test -z "$LD" && LD=$ac_prog - ;; - "") - # If it fails, then pretend we aren't using GCC. - ac_prog=ld - ;; - *) - # If it is relative, then search for the first ld in PATH. - with_gnu_ld=unknown - ;; - esac -elif test yes = "$with_gnu_ld"; then - AC_MSG_CHECKING([for GNU ld]) -else - AC_MSG_CHECKING([for non-GNU ld]) -fi -AC_CACHE_VAL(lt_cv_path_LD, -[if test -z "$LD"; then - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR - for ac_dir in $PATH; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then - lt_cv_path_LD=$ac_dir/$ac_prog - # Check to see if the program is GNU ld. I'd rather use --version, - # but apparently some variants of GNU ld only accept -v. - # Break only if it was the GNU/non-GNU ld that we prefer. - case `"$lt_cv_path_LD" -v 2>&1 &1 conftest.i -cat conftest.i conftest.i >conftest2.i -: ${lt_DD:=$DD} -AC_PATH_PROGS_FEATURE_CHECK([lt_DD], [dd], -[if "$ac_path_lt_DD" bs=32 count=1 conftest.out 2>/dev/null; then - cmp -s conftest.i conftest.out \ - && ac_cv_path_lt_DD="$ac_path_lt_DD" ac_path_lt_DD_found=: -fi]) -rm -f conftest.i conftest2.i conftest.out]) -])# _LT_PATH_DD - - -# _LT_CMD_TRUNCATE -# ---------------- -# find command to truncate a binary pipe -m4_defun([_LT_CMD_TRUNCATE], -[m4_require([_LT_PATH_DD]) -AC_CACHE_CHECK([how to truncate binary pipes], [lt_cv_truncate_bin], -[printf 0123456789abcdef0123456789abcdef >conftest.i -cat conftest.i conftest.i >conftest2.i -lt_cv_truncate_bin= -if "$ac_cv_path_lt_DD" bs=32 count=1 conftest.out 2>/dev/null; then - cmp -s conftest.i conftest.out \ - && lt_cv_truncate_bin="$ac_cv_path_lt_DD bs=4096 count=1" -fi -rm -f conftest.i conftest2.i conftest.out -test -z "$lt_cv_truncate_bin" && lt_cv_truncate_bin="$SED -e 4q"]) -_LT_DECL([lt_truncate_bin], [lt_cv_truncate_bin], [1], - [Command to truncate a binary pipe]) -])# _LT_CMD_TRUNCATE - - -# _LT_CHECK_MAGIC_METHOD -# ---------------------- -# how to check for library dependencies -# -- PORTME fill in with the dynamic library characteristics -m4_defun([_LT_CHECK_MAGIC_METHOD], -[m4_require([_LT_DECL_EGREP]) -m4_require([_LT_DECL_OBJDUMP]) -AC_CACHE_CHECK([how to recognize dependent libraries], -lt_cv_deplibs_check_method, -[lt_cv_file_magic_cmd='$MAGIC_CMD' -lt_cv_file_magic_test_file= -lt_cv_deplibs_check_method='unknown' -# Need to set the preceding variable on all platforms that support -# interlibrary dependencies. -# 'none' -- dependencies not supported. -# 'unknown' -- same as none, but documents that we really don't know. -# 'pass_all' -- all dependencies passed with no checks. -# 'test_compile' -- check by making test program. -# 'file_magic [[regex]]' -- check by looking for files in library path -# that responds to the $file_magic_cmd with a given extended regex. -# If you have 'file' or equivalent on your system and you're not sure -# whether 'pass_all' will *always* work, you probably want this one. - -case $host_os in -aix[[4-9]]*) - lt_cv_deplibs_check_method=pass_all - ;; - -beos*) - lt_cv_deplibs_check_method=pass_all - ;; - -bsdi[[45]]*) - lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (shared object|dynamic lib)' - lt_cv_file_magic_cmd='/usr/bin/file -L' - lt_cv_file_magic_test_file=/shlib/libc.so - ;; - -cygwin*) - # func_win32_libid is a shell function defined in ltmain.sh - lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' - lt_cv_file_magic_cmd='func_win32_libid' - ;; - -mingw* | pw32*) - # Base MSYS/MinGW do not provide the 'file' command needed by - # func_win32_libid shell function, so use a weaker test based on 'objdump', - # unless we find 'file', for example because we are cross-compiling. - if ( file / ) >/dev/null 2>&1; then - lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' - lt_cv_file_magic_cmd='func_win32_libid' - else - # Keep this pattern in sync with the one in func_win32_libid. - lt_cv_deplibs_check_method='file_magic file format (pei*-i386(.*architecture: i386)?|pe-arm-wince|pe-x86-64)' - lt_cv_file_magic_cmd='$OBJDUMP -f' - fi - ;; - -cegcc*) - # use the weaker test based on 'objdump'. See mingw*. - lt_cv_deplibs_check_method='file_magic file format pe-arm-.*little(.*architecture: arm)?' - lt_cv_file_magic_cmd='$OBJDUMP -f' - ;; - -darwin* | rhapsody*) - lt_cv_deplibs_check_method=pass_all - ;; - -freebsd* | dragonfly*) - if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then - case $host_cpu in - i*86 ) - # Not sure whether the presence of OpenBSD here was a mistake. - # Let's accept both of them until this is cleared up. - lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[[3-9]]86 (compact )?demand paged shared library' - lt_cv_file_magic_cmd=/usr/bin/file - lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*` - ;; - esac - else - lt_cv_deplibs_check_method=pass_all - fi - ;; - -haiku*) - lt_cv_deplibs_check_method=pass_all - ;; - -hpux10.20* | hpux11*) - lt_cv_file_magic_cmd=/usr/bin/file - case $host_cpu in - ia64*) - lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|ELF-[[0-9]][[0-9]]) shared object file - IA64' - lt_cv_file_magic_test_file=/usr/lib/hpux32/libc.so - ;; - hppa*64*) - [lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF[ -][0-9][0-9])(-bit)?( [LM]SB)? shared object( file)?[, -]* PA-RISC [0-9]\.[0-9]'] - lt_cv_file_magic_test_file=/usr/lib/pa20_64/libc.sl - ;; - *) - lt_cv_deplibs_check_method='file_magic (s[[0-9]][[0-9]][[0-9]]|PA-RISC[[0-9]]\.[[0-9]]) shared library' - lt_cv_file_magic_test_file=/usr/lib/libc.sl - ;; - esac - ;; - -interix[[3-9]]*) - # PIC code is broken on Interix 3.x, that's why |\.a not |_pic\.a here - lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so|\.a)$' - ;; - -irix5* | irix6* | nonstopux*) - case $LD in - *-32|*"-32 ") libmagic=32-bit;; - *-n32|*"-n32 ") libmagic=N32;; - *-64|*"-64 ") libmagic=64-bit;; - *) libmagic=never-match;; - esac - lt_cv_deplibs_check_method=pass_all - ;; - -# This must be glibc/ELF. -linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - lt_cv_deplibs_check_method=pass_all - ;; - -netbsd* | netbsdelf*-gnu) - if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then - lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|_pic\.a)$' - else - lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so|_pic\.a)$' - fi - ;; - -newos6*) - lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (executable|dynamic lib)' - lt_cv_file_magic_cmd=/usr/bin/file - lt_cv_file_magic_test_file=/usr/lib/libnls.so - ;; - -*nto* | *qnx*) - lt_cv_deplibs_check_method=pass_all - ;; - -openbsd* | bitrig*) - if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`"; then - lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|\.so|_pic\.a)$' - else - lt_cv_deplibs_check_method='match_pattern /lib[[^/]]+(\.so\.[[0-9]]+\.[[0-9]]+|_pic\.a)$' - fi - ;; - -osf3* | osf4* | osf5*) - lt_cv_deplibs_check_method=pass_all - ;; - -rdos*) - lt_cv_deplibs_check_method=pass_all - ;; - -solaris*) - lt_cv_deplibs_check_method=pass_all - ;; - -sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) - lt_cv_deplibs_check_method=pass_all - ;; - -sysv4 | sysv4.3*) - case $host_vendor in - motorola) - lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[ML]]SB (shared object|dynamic lib) M[[0-9]][[0-9]]* Version [[0-9]]' - lt_cv_file_magic_test_file=`echo /usr/lib/libc.so*` - ;; - ncr) - lt_cv_deplibs_check_method=pass_all - ;; - sequent) - lt_cv_file_magic_cmd='/bin/file' - lt_cv_deplibs_check_method='file_magic ELF [[0-9]][[0-9]]*-bit [[LM]]SB (shared object|dynamic lib )' - ;; - sni) - lt_cv_file_magic_cmd='/bin/file' - lt_cv_deplibs_check_method="file_magic ELF [[0-9]][[0-9]]*-bit [[LM]]SB dynamic lib" - lt_cv_file_magic_test_file=/lib/libc.so - ;; - siemens) - lt_cv_deplibs_check_method=pass_all - ;; - pc) - lt_cv_deplibs_check_method=pass_all - ;; - esac - ;; - -tpf*) - lt_cv_deplibs_check_method=pass_all - ;; -os2*) - lt_cv_deplibs_check_method=pass_all - ;; -esac -]) - -file_magic_glob= -want_nocaseglob=no -if test "$build" = "$host"; then - case $host_os in - mingw* | pw32*) - if ( shopt | grep nocaseglob ) >/dev/null 2>&1; then - want_nocaseglob=yes - else - file_magic_glob=`echo aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ | $SED -e "s/\(..\)/s\/[[\1]]\/[[\1]]\/g;/g"` - fi - ;; - esac -fi - -file_magic_cmd=$lt_cv_file_magic_cmd -deplibs_check_method=$lt_cv_deplibs_check_method -test -z "$deplibs_check_method" && deplibs_check_method=unknown - -_LT_DECL([], [deplibs_check_method], [1], - [Method to check whether dependent libraries are shared objects]) -_LT_DECL([], [file_magic_cmd], [1], - [Command to use when deplibs_check_method = "file_magic"]) -_LT_DECL([], [file_magic_glob], [1], - [How to find potential files when deplibs_check_method = "file_magic"]) -_LT_DECL([], [want_nocaseglob], [1], - [Find potential files using nocaseglob when deplibs_check_method = "file_magic"]) -])# _LT_CHECK_MAGIC_METHOD - - -# LT_PATH_NM -# ---------- -# find the pathname to a BSD- or MS-compatible name lister -AC_DEFUN([LT_PATH_NM], -[AC_REQUIRE([AC_PROG_CC])dnl -AC_CACHE_CHECK([for BSD- or MS-compatible name lister (nm)], lt_cv_path_NM, -[if test -n "$NM"; then - # Let the user override the test. - lt_cv_path_NM=$NM -else - lt_nm_to_check=${ac_tool_prefix}nm - if test -n "$ac_tool_prefix" && test "$build" = "$host"; then - lt_nm_to_check="$lt_nm_to_check nm" - fi - for lt_tmp_nm in $lt_nm_to_check; do - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR - for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - tmp_nm=$ac_dir/$lt_tmp_nm - if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext"; then - # Check to see if the nm accepts a BSD-compat flag. - # Adding the 'sed 1q' prevents false positives on HP-UX, which says: - # nm: unknown option "B" ignored - # Tru64's nm complains that /dev/null is an invalid object file - # MSYS converts /dev/null to NUL, MinGW nm treats NUL as empty - case $build_os in - mingw*) lt_bad_file=conftest.nm/nofile ;; - *) lt_bad_file=/dev/null ;; - esac - case `"$tmp_nm" -B $lt_bad_file 2>&1 | sed '1q'` in - *$lt_bad_file* | *'Invalid file or object type'*) - lt_cv_path_NM="$tmp_nm -B" - break 2 - ;; - *) - case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in - */dev/null*) - lt_cv_path_NM="$tmp_nm -p" - break 2 - ;; - *) - lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but - continue # so that we can try to find one that supports BSD flags - ;; - esac - ;; - esac - fi - done - IFS=$lt_save_ifs - done - : ${lt_cv_path_NM=no} -fi]) -if test no != "$lt_cv_path_NM"; then - NM=$lt_cv_path_NM -else - # Didn't find any BSD compatible name lister, look for dumpbin. - if test -n "$DUMPBIN"; then : - # Let the user override the test. - else - AC_CHECK_TOOLS(DUMPBIN, [dumpbin "link -dump"], :) - case `$DUMPBIN -symbols -headers /dev/null 2>&1 | sed '1q'` in - *COFF*) - DUMPBIN="$DUMPBIN -symbols -headers" - ;; - *) - DUMPBIN=: - ;; - esac - fi - AC_SUBST([DUMPBIN]) - if test : != "$DUMPBIN"; then - NM=$DUMPBIN - fi -fi -test -z "$NM" && NM=nm -AC_SUBST([NM]) -_LT_DECL([], [NM], [1], [A BSD- or MS-compatible name lister])dnl - -AC_CACHE_CHECK([the name lister ($NM) interface], [lt_cv_nm_interface], - [lt_cv_nm_interface="BSD nm" - echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:$LINENO: $ac_compile\"" >&AS_MESSAGE_LOG_FD) - (eval "$ac_compile" 2>conftest.err) - cat conftest.err >&AS_MESSAGE_LOG_FD - (eval echo "\"\$as_me:$LINENO: $NM \\\"conftest.$ac_objext\\\"\"" >&AS_MESSAGE_LOG_FD) - (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) - cat conftest.err >&AS_MESSAGE_LOG_FD - (eval echo "\"\$as_me:$LINENO: output\"" >&AS_MESSAGE_LOG_FD) - cat conftest.out >&AS_MESSAGE_LOG_FD - if $GREP 'External.*some_variable' conftest.out > /dev/null; then - lt_cv_nm_interface="MS dumpbin" - fi - rm -f conftest*]) -])# LT_PATH_NM - -# Old names: -AU_ALIAS([AM_PROG_NM], [LT_PATH_NM]) -AU_ALIAS([AC_PROG_NM], [LT_PATH_NM]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AM_PROG_NM], []) -dnl AC_DEFUN([AC_PROG_NM], []) - -# _LT_CHECK_SHAREDLIB_FROM_LINKLIB -# -------------------------------- -# how to determine the name of the shared library -# associated with a specific link library. -# -- PORTME fill in with the dynamic library characteristics -m4_defun([_LT_CHECK_SHAREDLIB_FROM_LINKLIB], -[m4_require([_LT_DECL_EGREP]) -m4_require([_LT_DECL_OBJDUMP]) -m4_require([_LT_DECL_DLLTOOL]) -AC_CACHE_CHECK([how to associate runtime and link libraries], -lt_cv_sharedlib_from_linklib_cmd, -[lt_cv_sharedlib_from_linklib_cmd='unknown' - -case $host_os in -cygwin* | mingw* | pw32* | cegcc*) - # two different shell functions defined in ltmain.sh; - # decide which one to use based on capabilities of $DLLTOOL - case `$DLLTOOL --help 2>&1` in - *--identify-strict*) - lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib - ;; - *) - lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib_fallback - ;; - esac - ;; -*) - # fallback: assume linklib IS sharedlib - lt_cv_sharedlib_from_linklib_cmd=$ECHO - ;; -esac -]) -sharedlib_from_linklib_cmd=$lt_cv_sharedlib_from_linklib_cmd -test -z "$sharedlib_from_linklib_cmd" && sharedlib_from_linklib_cmd=$ECHO - -_LT_DECL([], [sharedlib_from_linklib_cmd], [1], - [Command to associate shared and link libraries]) -])# _LT_CHECK_SHAREDLIB_FROM_LINKLIB - - -# _LT_PATH_MANIFEST_TOOL -# ---------------------- -# locate the manifest tool -m4_defun([_LT_PATH_MANIFEST_TOOL], -[AC_CHECK_TOOL(MANIFEST_TOOL, mt, :) -test -z "$MANIFEST_TOOL" && MANIFEST_TOOL=mt -AC_CACHE_CHECK([if $MANIFEST_TOOL is a manifest tool], [lt_cv_path_mainfest_tool], - [lt_cv_path_mainfest_tool=no - echo "$as_me:$LINENO: $MANIFEST_TOOL '-?'" >&AS_MESSAGE_LOG_FD - $MANIFEST_TOOL '-?' 2>conftest.err > conftest.out - cat conftest.err >&AS_MESSAGE_LOG_FD - if $GREP 'Manifest Tool' conftest.out > /dev/null; then - lt_cv_path_mainfest_tool=yes - fi - rm -f conftest*]) -if test yes != "$lt_cv_path_mainfest_tool"; then - MANIFEST_TOOL=: -fi -_LT_DECL([], [MANIFEST_TOOL], [1], [Manifest tool])dnl -])# _LT_PATH_MANIFEST_TOOL - - -# _LT_DLL_DEF_P([FILE]) -# --------------------- -# True iff FILE is a Windows DLL '.def' file. -# Keep in sync with func_dll_def_p in the libtool script -AC_DEFUN([_LT_DLL_DEF_P], -[dnl - test DEF = "`$SED -n dnl - -e '\''s/^[[ ]]*//'\'' dnl Strip leading whitespace - -e '\''/^\(;.*\)*$/d'\'' dnl Delete empty lines and comments - -e '\''s/^\(EXPORTS\|LIBRARY\)\([[ ]].*\)*$/DEF/p'\'' dnl - -e q dnl Only consider the first "real" line - $1`" dnl -])# _LT_DLL_DEF_P - - -# LT_LIB_M -# -------- -# check for math library -AC_DEFUN([LT_LIB_M], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -LIBM= -case $host in -*-*-beos* | *-*-cegcc* | *-*-cygwin* | *-*-haiku* | *-*-pw32* | *-*-darwin*) - # These system don't have libm, or don't need it - ;; -*-ncr-sysv4.3*) - AC_CHECK_LIB(mw, _mwvalidcheckl, LIBM=-lmw) - AC_CHECK_LIB(m, cos, LIBM="$LIBM -lm") - ;; -*) - AC_CHECK_LIB(m, cos, LIBM=-lm) - ;; -esac -AC_SUBST([LIBM]) -])# LT_LIB_M - -# Old name: -AU_ALIAS([AC_CHECK_LIBM], [LT_LIB_M]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_CHECK_LIBM], []) - - -# _LT_COMPILER_NO_RTTI([TAGNAME]) -# ------------------------------- -m4_defun([_LT_COMPILER_NO_RTTI], -[m4_require([_LT_TAG_COMPILER])dnl - -_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)= - -if test yes = "$GCC"; then - case $cc_basename in - nvcc*) - _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -Xcompiler -fno-builtin' ;; - *) - _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -fno-builtin' ;; - esac - - _LT_COMPILER_OPTION([if $compiler supports -fno-rtti -fno-exceptions], - lt_cv_prog_compiler_rtti_exceptions, - [-fno-rtti -fno-exceptions], [], - [_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)="$_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1) -fno-rtti -fno-exceptions"]) -fi -_LT_TAGDECL([no_builtin_flag], [lt_prog_compiler_no_builtin_flag], [1], - [Compiler flag to turn off builtin functions]) -])# _LT_COMPILER_NO_RTTI - - -# _LT_CMD_GLOBAL_SYMBOLS -# ---------------------- -m4_defun([_LT_CMD_GLOBAL_SYMBOLS], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -AC_REQUIRE([AC_PROG_CC])dnl -AC_REQUIRE([AC_PROG_AWK])dnl -AC_REQUIRE([LT_PATH_NM])dnl -AC_REQUIRE([LT_PATH_LD])dnl -m4_require([_LT_DECL_SED])dnl -m4_require([_LT_DECL_EGREP])dnl -m4_require([_LT_TAG_COMPILER])dnl - -# Check for command to grab the raw symbol name followed by C symbol from nm. -AC_MSG_CHECKING([command to parse $NM output from $compiler object]) -AC_CACHE_VAL([lt_cv_sys_global_symbol_pipe], -[ -# These are sane defaults that work on at least a few old systems. -# [They come from Ultrix. What could be older than Ultrix?!! ;)] - -# Character class describing NM global symbol codes. -symcode='[[BCDEGRST]]' - -# Regexp to match symbols that can be accessed directly from C. -sympat='\([[_A-Za-z]][[_A-Za-z0-9]]*\)' - -# Define system-specific variables. -case $host_os in -aix*) - symcode='[[BCDT]]' - ;; -cygwin* | mingw* | pw32* | cegcc*) - symcode='[[ABCDGISTW]]' - ;; -hpux*) - if test ia64 = "$host_cpu"; then - symcode='[[ABCDEGRST]]' - fi - ;; -irix* | nonstopux*) - symcode='[[BCDEGRST]]' - ;; -osf*) - symcode='[[BCDEGQRST]]' - ;; -solaris*) - symcode='[[BDRT]]' - ;; -sco3.2v5*) - symcode='[[DT]]' - ;; -sysv4.2uw2*) - symcode='[[DT]]' - ;; -sysv5* | sco5v6* | unixware* | OpenUNIX*) - symcode='[[ABDT]]' - ;; -sysv4) - symcode='[[DFNSTU]]' - ;; -esac - -# If we're using GNU nm, then use its standard symbol codes. -case `$NM -V 2>&1` in -*GNU* | *'with BFD'*) - symcode='[[ABCDGIRSTW]]' ;; -esac - -if test "$lt_cv_nm_interface" = "MS dumpbin"; then - # Gets list of data symbols to import. - lt_cv_sys_global_symbol_to_import="sed -n -e 's/^I .* \(.*\)$/\1/p'" - # Adjust the below global symbol transforms to fixup imported variables. - lt_cdecl_hook=" -e 's/^I .* \(.*\)$/extern __declspec(dllimport) char \1;/p'" - lt_c_name_hook=" -e 's/^I .* \(.*\)$/ {\"\1\", (void *) 0},/p'" - lt_c_name_lib_hook="\ - -e 's/^I .* \(lib.*\)$/ {\"\1\", (void *) 0},/p'\ - -e 's/^I .* \(.*\)$/ {\"lib\1\", (void *) 0},/p'" -else - # Disable hooks by default. - lt_cv_sys_global_symbol_to_import= - lt_cdecl_hook= - lt_c_name_hook= - lt_c_name_lib_hook= -fi - -# Transform an extracted symbol line into a proper C declaration. -# Some systems (esp. on ia64) link data and code symbols differently, -# so use this general approach. -lt_cv_sys_global_symbol_to_cdecl="sed -n"\ -$lt_cdecl_hook\ -" -e 's/^T .* \(.*\)$/extern int \1();/p'"\ -" -e 's/^$symcode$symcode* .* \(.*\)$/extern char \1;/p'" - -# Transform an extracted symbol line into symbol name and symbol address -lt_cv_sys_global_symbol_to_c_name_address="sed -n"\ -$lt_c_name_hook\ -" -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ -" -e 's/^$symcode$symcode* .* \(.*\)$/ {\"\1\", (void *) \&\1},/p'" - -# Transform an extracted symbol line into symbol name with lib prefix and -# symbol address. -lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n"\ -$lt_c_name_lib_hook\ -" -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ -" -e 's/^$symcode$symcode* .* \(lib.*\)$/ {\"\1\", (void *) \&\1},/p'"\ -" -e 's/^$symcode$symcode* .* \(.*\)$/ {\"lib\1\", (void *) \&\1},/p'" - -# Handle CRLF in mingw tool chain -opt_cr= -case $build_os in -mingw*) - opt_cr=`$ECHO 'x\{0,1\}' | tr x '\015'` # option cr in regexp - ;; -esac - -# Try without a prefix underscore, then with it. -for ac_symprfx in "" "_"; do - - # Transform symcode, sympat, and symprfx into a raw symbol and a C symbol. - symxfrm="\\1 $ac_symprfx\\2 \\2" - - # Write the raw and C identifiers. - if test "$lt_cv_nm_interface" = "MS dumpbin"; then - # Fake it for dumpbin and say T for any non-static function, - # D for any global variable and I for any imported variable. - # Also find C++ and __fastcall symbols from MSVC++, - # which start with @ or ?. - lt_cv_sys_global_symbol_pipe="$AWK ['"\ -" {last_section=section; section=\$ 3};"\ -" /^COFF SYMBOL TABLE/{for(i in hide) delete hide[i]};"\ -" /Section length .*#relocs.*(pick any)/{hide[last_section]=1};"\ -" /^ *Symbol name *: /{split(\$ 0,sn,\":\"); si=substr(sn[2],2)};"\ -" /^ *Type *: code/{print \"T\",si,substr(si,length(prfx))};"\ -" /^ *Type *: data/{print \"I\",si,substr(si,length(prfx))};"\ -" \$ 0!~/External *\|/{next};"\ -" / 0+ UNDEF /{next}; / UNDEF \([^|]\)*()/{next};"\ -" {if(hide[section]) next};"\ -" {f=\"D\"}; \$ 0~/\(\).*\|/{f=\"T\"};"\ -" {split(\$ 0,a,/\||\r/); split(a[2],s)};"\ -" s[1]~/^[@?]/{print f,s[1],s[1]; next};"\ -" s[1]~prfx {split(s[1],t,\"@\"); print f,t[1],substr(t[1],length(prfx))}"\ -" ' prfx=^$ac_symprfx]" - else - lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[[ ]]\($symcode$symcode*\)[[ ]][[ ]]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" - fi - lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | sed '/ __gnu_lto/d'" - - # Check to see that the pipe works correctly. - pipe_works=no - - rm -f conftest* - cat > conftest.$ac_ext <<_LT_EOF -#ifdef __cplusplus -extern "C" { -#endif -char nm_test_var; -void nm_test_func(void); -void nm_test_func(void){} -#ifdef __cplusplus -} -#endif -int main(){nm_test_var='a';nm_test_func();return(0);} -_LT_EOF - - if AC_TRY_EVAL(ac_compile); then - # Now try to grab the symbols. - nlist=conftest.nm - $ECHO "$as_me:$LINENO: $NM conftest.$ac_objext | $lt_cv_sys_global_symbol_pipe > $nlist" >&AS_MESSAGE_LOG_FD - if eval "$NM" conftest.$ac_objext \| "$lt_cv_sys_global_symbol_pipe" \> $nlist 2>&AS_MESSAGE_LOG_FD && test -s "$nlist"; then - # Try sorting and uniquifying the output. - if sort "$nlist" | uniq > "$nlist"T; then - mv -f "$nlist"T "$nlist" - else - rm -f "$nlist"T - fi - - # Make sure that we snagged all the symbols we need. - if $GREP ' nm_test_var$' "$nlist" >/dev/null; then - if $GREP ' nm_test_func$' "$nlist" >/dev/null; then - cat <<_LT_EOF > conftest.$ac_ext -/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ -#if defined _WIN32 || defined __CYGWIN__ || defined _WIN32_WCE -/* DATA imports from DLLs on WIN32 can't be const, because runtime - relocations are performed -- see ld's documentation on pseudo-relocs. */ -# define LT@&t@_DLSYM_CONST -#elif defined __osf__ -/* This system does not cope well with relocations in const data. */ -# define LT@&t@_DLSYM_CONST -#else -# define LT@&t@_DLSYM_CONST const -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -_LT_EOF - # Now generate the symbol file. - eval "$lt_cv_sys_global_symbol_to_cdecl"' < "$nlist" | $GREP -v main >> conftest.$ac_ext' - - cat <<_LT_EOF >> conftest.$ac_ext - -/* The mapping between symbol names and symbols. */ -LT@&t@_DLSYM_CONST struct { - const char *name; - void *address; -} -lt__PROGRAM__LTX_preloaded_symbols[[]] = -{ - { "@PROGRAM@", (void *) 0 }, -_LT_EOF - $SED "s/^$symcode$symcode* .* \(.*\)$/ {\"\1\", (void *) \&\1},/" < "$nlist" | $GREP -v main >> conftest.$ac_ext - cat <<\_LT_EOF >> conftest.$ac_ext - {0, (void *) 0} -}; - -/* This works around a problem in FreeBSD linker */ -#ifdef FREEBSD_WORKAROUND -static const void *lt_preloaded_setup() { - return lt__PROGRAM__LTX_preloaded_symbols; -} -#endif - -#ifdef __cplusplus -} -#endif -_LT_EOF - # Now try linking the two files. - mv conftest.$ac_objext conftstm.$ac_objext - lt_globsym_save_LIBS=$LIBS - lt_globsym_save_CFLAGS=$CFLAGS - LIBS=conftstm.$ac_objext - CFLAGS="$CFLAGS$_LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)" - if AC_TRY_EVAL(ac_link) && test -s conftest$ac_exeext; then - pipe_works=yes - fi - LIBS=$lt_globsym_save_LIBS - CFLAGS=$lt_globsym_save_CFLAGS - else - echo "cannot find nm_test_func in $nlist" >&AS_MESSAGE_LOG_FD - fi - else - echo "cannot find nm_test_var in $nlist" >&AS_MESSAGE_LOG_FD - fi - else - echo "cannot run $lt_cv_sys_global_symbol_pipe" >&AS_MESSAGE_LOG_FD - fi - else - echo "$progname: failed program was:" >&AS_MESSAGE_LOG_FD - cat conftest.$ac_ext >&5 - fi - rm -rf conftest* conftst* - - # Do not use the global_symbol_pipe unless it works. - if test yes = "$pipe_works"; then - break - else - lt_cv_sys_global_symbol_pipe= - fi -done -]) -if test -z "$lt_cv_sys_global_symbol_pipe"; then - lt_cv_sys_global_symbol_to_cdecl= -fi -if test -z "$lt_cv_sys_global_symbol_pipe$lt_cv_sys_global_symbol_to_cdecl"; then - AC_MSG_RESULT(failed) -else - AC_MSG_RESULT(ok) -fi - -# Response file support. -if test "$lt_cv_nm_interface" = "MS dumpbin"; then - nm_file_list_spec='@' -elif $NM --help 2>/dev/null | grep '[[@]]FILE' >/dev/null; then - nm_file_list_spec='@' -fi - -_LT_DECL([global_symbol_pipe], [lt_cv_sys_global_symbol_pipe], [1], - [Take the output of nm and produce a listing of raw symbols and C names]) -_LT_DECL([global_symbol_to_cdecl], [lt_cv_sys_global_symbol_to_cdecl], [1], - [Transform the output of nm in a proper C declaration]) -_LT_DECL([global_symbol_to_import], [lt_cv_sys_global_symbol_to_import], [1], - [Transform the output of nm into a list of symbols to manually relocate]) -_LT_DECL([global_symbol_to_c_name_address], - [lt_cv_sys_global_symbol_to_c_name_address], [1], - [Transform the output of nm in a C name address pair]) -_LT_DECL([global_symbol_to_c_name_address_lib_prefix], - [lt_cv_sys_global_symbol_to_c_name_address_lib_prefix], [1], - [Transform the output of nm in a C name address pair when lib prefix is needed]) -_LT_DECL([nm_interface], [lt_cv_nm_interface], [1], - [The name lister interface]) -_LT_DECL([], [nm_file_list_spec], [1], - [Specify filename containing input files for $NM]) -]) # _LT_CMD_GLOBAL_SYMBOLS - - -# _LT_COMPILER_PIC([TAGNAME]) -# --------------------------- -m4_defun([_LT_COMPILER_PIC], -[m4_require([_LT_TAG_COMPILER])dnl -_LT_TAGVAR(lt_prog_compiler_wl, $1)= -_LT_TAGVAR(lt_prog_compiler_pic, $1)= -_LT_TAGVAR(lt_prog_compiler_static, $1)= - -m4_if([$1], [CXX], [ - # C++ specific cases for pic, static, wl, etc. - if test yes = "$GXX"; then - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - - case $host_os in - aix*) - # All AIX code is PIC. - if test ia64 = "$host_cpu"; then - # AIX 5 now supports IA64 processor - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - fi - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - m68k) - # FIXME: we need at least 68020 code to build shared libraries, but - # adding the '-m68020' flag to GCC prevents building anything better, - # like '-m68040'. - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-m68020 -resident32 -malways-restore-a4' - ;; - esac - ;; - - beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) - # PIC is the default for these OSes. - ;; - mingw* | cygwin* | os2* | pw32* | cegcc*) - # This hack is so that the source file can tell whether it is being - # built for inclusion in a dll (and should export symbols for example). - # Although the cygwin gcc ignores -fPIC, still need this for old-style - # (--disable-auto-import) libraries - m4_if([$1], [GCJ], [], - [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) - case $host_os in - os2*) - _LT_TAGVAR(lt_prog_compiler_static, $1)='$wl-static' - ;; - esac - ;; - darwin* | rhapsody*) - # PIC is the default on this platform - # Common symbols not allowed in MH_DYLIB files - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common' - ;; - *djgpp*) - # DJGPP does not support shared libraries at all - _LT_TAGVAR(lt_prog_compiler_pic, $1)= - ;; - haiku*) - # PIC is the default for Haiku. - # The "-static" flag exists, but is broken. - _LT_TAGVAR(lt_prog_compiler_static, $1)= - ;; - interix[[3-9]]*) - # Interix 3.x gcc -fpic/-fPIC options generate broken code. - # Instead, we relocate shared libraries at runtime. - ;; - sysv4*MP*) - if test -d /usr/nec; then - _LT_TAGVAR(lt_prog_compiler_pic, $1)=-Kconform_pic - fi - ;; - hpux*) - # PIC is the default for 64-bit PA HP-UX, but not for 32-bit - # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag - # sets the default TLS model and affects inlining. - case $host_cpu in - hppa*64*) - ;; - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - esac - ;; - *qnx* | *nto*) - # QNX uses GNU C++, but need to define -shared option too, otherwise - # it will coredump. - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' - ;; - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - esac - else - case $host_os in - aix[[4-9]]*) - # All AIX code is PIC. - if test ia64 = "$host_cpu"; then - # AIX 5 now supports IA64 processor - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - else - _LT_TAGVAR(lt_prog_compiler_static, $1)='-bnso -bI:/lib/syscalls.exp' - fi - ;; - chorus*) - case $cc_basename in - cxch68*) - # Green Hills C++ Compiler - # _LT_TAGVAR(lt_prog_compiler_static, $1)="--no_auto_instantiation -u __main -u __premain -u _abort -r $COOL_DIR/lib/libOrb.a $MVME_DIR/lib/CC/libC.a $MVME_DIR/lib/classix/libcx.s.a" - ;; - esac - ;; - mingw* | cygwin* | os2* | pw32* | cegcc*) - # This hack is so that the source file can tell whether it is being - # built for inclusion in a dll (and should export symbols for example). - m4_if([$1], [GCJ], [], - [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) - ;; - dgux*) - case $cc_basename in - ec++*) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - ;; - ghcx*) - # Green Hills C++ Compiler - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' - ;; - *) - ;; - esac - ;; - freebsd* | dragonfly*) - # FreeBSD uses GNU C++ - ;; - hpux9* | hpux10* | hpux11*) - case $cc_basename in - CC*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_static, $1)='$wl-a ${wl}archive' - if test ia64 != "$host_cpu"; then - _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z' - fi - ;; - aCC*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_static, $1)='$wl-a ${wl}archive' - case $host_cpu in - hppa*64*|ia64*) - # +Z the default - ;; - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z' - ;; - esac - ;; - *) - ;; - esac - ;; - interix*) - # This is c89, which is MS Visual C++ (no shared libs) - # Anyone wants to do a port? - ;; - irix5* | irix6* | nonstopux*) - case $cc_basename in - CC*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - # CC pic flag -KPIC is the default. - ;; - *) - ;; - esac - ;; - linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - case $cc_basename in - KCC*) - # KAI C++ Compiler - _LT_TAGVAR(lt_prog_compiler_wl, $1)='--backend -Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - ecpc* ) - # old Intel C++ for x86_64, which still supported -KPIC. - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - icpc* ) - # Intel C++, used to be incompatible with GCC. - # ICC 10 doesn't accept -KPIC any more. - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - pgCC* | pgcpp*) - # Portland Group C++ compiler - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - cxx*) - # Compaq C++ - # Make sure the PIC flag is empty. It appears that all Alpha - # Linux and Compaq Tru64 Unix objects are PIC. - _LT_TAGVAR(lt_prog_compiler_pic, $1)= - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - ;; - xlc* | xlC* | bgxl[[cC]]* | mpixl[[cC]]*) - # IBM XL 8.0, 9.0 on PPC and BlueGene - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-qpic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink' - ;; - *) - case `$CC -V 2>&1 | sed 5q` in - *Sun\ C*) - # Sun C++ 5.9 - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' - ;; - esac - ;; - esac - ;; - lynxos*) - ;; - m88k*) - ;; - mvs*) - case $cc_basename in - cxx*) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-W c,exportall' - ;; - *) - ;; - esac - ;; - netbsd* | netbsdelf*-gnu) - ;; - *qnx* | *nto*) - # QNX uses GNU C++, but need to define -shared option too, otherwise - # it will coredump. - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' - ;; - osf3* | osf4* | osf5*) - case $cc_basename in - KCC*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='--backend -Wl,' - ;; - RCC*) - # Rational C++ 2.4.1 - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' - ;; - cxx*) - # Digital/Compaq C++ - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - # Make sure the PIC flag is empty. It appears that all Alpha - # Linux and Compaq Tru64 Unix objects are PIC. - _LT_TAGVAR(lt_prog_compiler_pic, $1)= - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - ;; - *) - ;; - esac - ;; - psos*) - ;; - solaris*) - case $cc_basename in - CC* | sunCC*) - # Sun C++ 4.2, 5.x and Centerline C++ - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' - ;; - gcx*) - # Green Hills C++ Compiler - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' - ;; - *) - ;; - esac - ;; - sunos4*) - case $cc_basename in - CC*) - # Sun C++ 4.x - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - lcc*) - # Lucid - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' - ;; - *) - ;; - esac - ;; - sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) - case $cc_basename in - CC*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - esac - ;; - tandem*) - case $cc_basename in - NCC*) - # NonStop-UX NCC 3.20 - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - ;; - *) - ;; - esac - ;; - vxworks*) - ;; - *) - _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no - ;; - esac - fi -], -[ - if test yes = "$GCC"; then - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - - case $host_os in - aix*) - # All AIX code is PIC. - if test ia64 = "$host_cpu"; then - # AIX 5 now supports IA64 processor - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - fi - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - m68k) - # FIXME: we need at least 68020 code to build shared libraries, but - # adding the '-m68020' flag to GCC prevents building anything better, - # like '-m68040'. - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-m68020 -resident32 -malways-restore-a4' - ;; - esac - ;; - - beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) - # PIC is the default for these OSes. - ;; - - mingw* | cygwin* | pw32* | os2* | cegcc*) - # This hack is so that the source file can tell whether it is being - # built for inclusion in a dll (and should export symbols for example). - # Although the cygwin gcc ignores -fPIC, still need this for old-style - # (--disable-auto-import) libraries - m4_if([$1], [GCJ], [], - [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) - case $host_os in - os2*) - _LT_TAGVAR(lt_prog_compiler_static, $1)='$wl-static' - ;; - esac - ;; - - darwin* | rhapsody*) - # PIC is the default on this platform - # Common symbols not allowed in MH_DYLIB files - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common' - ;; - - haiku*) - # PIC is the default for Haiku. - # The "-static" flag exists, but is broken. - _LT_TAGVAR(lt_prog_compiler_static, $1)= - ;; - - hpux*) - # PIC is the default for 64-bit PA HP-UX, but not for 32-bit - # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag - # sets the default TLS model and affects inlining. - case $host_cpu in - hppa*64*) - # +Z the default - ;; - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - esac - ;; - - interix[[3-9]]*) - # Interix 3.x gcc -fpic/-fPIC options generate broken code. - # Instead, we relocate shared libraries at runtime. - ;; - - msdosdjgpp*) - # Just because we use GCC doesn't mean we suddenly get shared libraries - # on systems that don't support them. - _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no - enable_shared=no - ;; - - *nto* | *qnx*) - # QNX uses GNU C++, but need to define -shared option too, otherwise - # it will coredump. - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' - ;; - - sysv4*MP*) - if test -d /usr/nec; then - _LT_TAGVAR(lt_prog_compiler_pic, $1)=-Kconform_pic - fi - ;; - - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - ;; - esac - - case $cc_basename in - nvcc*) # Cuda Compiler Driver 2.2 - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Xlinker ' - if test -n "$_LT_TAGVAR(lt_prog_compiler_pic, $1)"; then - _LT_TAGVAR(lt_prog_compiler_pic, $1)="-Xcompiler $_LT_TAGVAR(lt_prog_compiler_pic, $1)" - fi - ;; - esac - else - # PORTME Check for flag to pass linker flags through the system compiler. - case $host_os in - aix*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - if test ia64 = "$host_cpu"; then - # AIX 5 now supports IA64 processor - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - else - _LT_TAGVAR(lt_prog_compiler_static, $1)='-bnso -bI:/lib/syscalls.exp' - fi - ;; - - darwin* | rhapsody*) - # PIC is the default on this platform - # Common symbols not allowed in MH_DYLIB files - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fno-common' - case $cc_basename in - nagfor*) - # NAG Fortran compiler - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,-Wl,,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - esac - ;; - - mingw* | cygwin* | pw32* | os2* | cegcc*) - # This hack is so that the source file can tell whether it is being - # built for inclusion in a dll (and should export symbols for example). - m4_if([$1], [GCJ], [], - [_LT_TAGVAR(lt_prog_compiler_pic, $1)='-DDLL_EXPORT']) - case $host_os in - os2*) - _LT_TAGVAR(lt_prog_compiler_static, $1)='$wl-static' - ;; - esac - ;; - - hpux9* | hpux10* | hpux11*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - # PIC is the default for IA64 HP-UX and 64-bit HP-UX, but - # not for PA HP-UX. - case $host_cpu in - hppa*64*|ia64*) - # +Z the default - ;; - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='+Z' - ;; - esac - # Is there a better lt_prog_compiler_static that works with the bundled CC? - _LT_TAGVAR(lt_prog_compiler_static, $1)='$wl-a ${wl}archive' - ;; - - irix5* | irix6* | nonstopux*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - # PIC (with -KPIC) is the default. - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - ;; - - linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - case $cc_basename in - # old Intel for x86_64, which still supported -KPIC. - ecc*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - # flang / f18. f95 an alias for gfortran or flang on Debian - flang* | f18* | f95*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - # icc used to be incompatible with GCC. - # ICC 10 doesn't accept -KPIC any more. - icc* | ifort*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - # Lahey Fortran 8.1. - lf95*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='--shared' - _LT_TAGVAR(lt_prog_compiler_static, $1)='--static' - ;; - nagfor*) - # NAG Fortran compiler - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,-Wl,,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - tcc*) - # Fabrice Bellard et al's Tiny C Compiler - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - pgcc* | pgf77* | pgf90* | pgf95* | pgfortran*) - # Portland Group compilers (*not* the Pentium gcc compiler, - # which looks to be a dead project) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - ccc*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - # All Alpha code is PIC. - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - ;; - xl* | bgxl* | bgf* | mpixl*) - # IBM XL C 8.0/Fortran 10.1, 11.1 on PPC and BlueGene - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-qpic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-qstaticlink' - ;; - *) - case `$CC -V 2>&1 | sed 5q` in - *Sun\ Ceres\ Fortran* | *Sun*Fortran*\ [[1-7]].* | *Sun*Fortran*\ 8.[[0-3]]*) - # Sun Fortran 8.3 passes all unrecognized flags to the linker - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - _LT_TAGVAR(lt_prog_compiler_wl, $1)='' - ;; - *Sun\ F* | *Sun*Fortran*) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' - ;; - *Sun\ C*) - # Sun C 5.9 - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - ;; - *Intel*\ [[CF]]*Compiler*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-static' - ;; - *Portland\ Group*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fpic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - esac - ;; - esac - ;; - - newsos6) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - - *nto* | *qnx*) - # QNX uses GNU C++, but need to define -shared option too, otherwise - # it will coredump. - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-fPIC -shared' - ;; - - osf3* | osf4* | osf5*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - # All OSF/1 code is PIC. - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - ;; - - rdos*) - _LT_TAGVAR(lt_prog_compiler_static, $1)='-non_shared' - ;; - - solaris*) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - case $cc_basename in - f77* | f90* | f95* | sunf77* | sunf90* | sunf95*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ';; - *) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,';; - esac - ;; - - sunos4*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Qoption ld ' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-PIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - - sysv4 | sysv4.2uw2* | sysv4.3*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - - sysv4*MP*) - if test -d /usr/nec; then - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-Kconform_pic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - fi - ;; - - sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-KPIC' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - - unicos*) - _LT_TAGVAR(lt_prog_compiler_wl, $1)='-Wl,' - _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no - ;; - - uts4*) - _LT_TAGVAR(lt_prog_compiler_pic, $1)='-pic' - _LT_TAGVAR(lt_prog_compiler_static, $1)='-Bstatic' - ;; - - *) - _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no - ;; - esac - fi -]) -case $host_os in - # For platforms that do not support PIC, -DPIC is meaningless: - *djgpp*) - _LT_TAGVAR(lt_prog_compiler_pic, $1)= - ;; - *) - _LT_TAGVAR(lt_prog_compiler_pic, $1)="$_LT_TAGVAR(lt_prog_compiler_pic, $1)@&t@m4_if([$1],[],[ -DPIC],[m4_if([$1],[CXX],[ -DPIC],[])])" - ;; -esac - -AC_CACHE_CHECK([for $compiler option to produce PIC], - [_LT_TAGVAR(lt_cv_prog_compiler_pic, $1)], - [_LT_TAGVAR(lt_cv_prog_compiler_pic, $1)=$_LT_TAGVAR(lt_prog_compiler_pic, $1)]) -_LT_TAGVAR(lt_prog_compiler_pic, $1)=$_LT_TAGVAR(lt_cv_prog_compiler_pic, $1) - -# -# Check to make sure the PIC flag actually works. -# -if test -n "$_LT_TAGVAR(lt_prog_compiler_pic, $1)"; then - _LT_COMPILER_OPTION([if $compiler PIC flag $_LT_TAGVAR(lt_prog_compiler_pic, $1) works], - [_LT_TAGVAR(lt_cv_prog_compiler_pic_works, $1)], - [$_LT_TAGVAR(lt_prog_compiler_pic, $1)@&t@m4_if([$1],[],[ -DPIC],[m4_if([$1],[CXX],[ -DPIC],[])])], [], - [case $_LT_TAGVAR(lt_prog_compiler_pic, $1) in - "" | " "*) ;; - *) _LT_TAGVAR(lt_prog_compiler_pic, $1)=" $_LT_TAGVAR(lt_prog_compiler_pic, $1)" ;; - esac], - [_LT_TAGVAR(lt_prog_compiler_pic, $1)= - _LT_TAGVAR(lt_prog_compiler_can_build_shared, $1)=no]) -fi -_LT_TAGDECL([pic_flag], [lt_prog_compiler_pic], [1], - [Additional compiler flags for building library objects]) - -_LT_TAGDECL([wl], [lt_prog_compiler_wl], [1], - [How to pass a linker flag through the compiler]) -# -# Check to make sure the static flag actually works. -# -wl=$_LT_TAGVAR(lt_prog_compiler_wl, $1) eval lt_tmp_static_flag=\"$_LT_TAGVAR(lt_prog_compiler_static, $1)\" -_LT_LINKER_OPTION([if $compiler static flag $lt_tmp_static_flag works], - _LT_TAGVAR(lt_cv_prog_compiler_static_works, $1), - $lt_tmp_static_flag, - [], - [_LT_TAGVAR(lt_prog_compiler_static, $1)=]) -_LT_TAGDECL([link_static_flag], [lt_prog_compiler_static], [1], - [Compiler flag to prevent dynamic linking]) -])# _LT_COMPILER_PIC - - -# _LT_LINKER_SHLIBS([TAGNAME]) -# ---------------------------- -# See if the linker supports building shared libraries. -m4_defun([_LT_LINKER_SHLIBS], -[AC_REQUIRE([LT_PATH_LD])dnl -AC_REQUIRE([LT_PATH_NM])dnl -m4_require([_LT_PATH_MANIFEST_TOOL])dnl -m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_DECL_EGREP])dnl -m4_require([_LT_DECL_SED])dnl -m4_require([_LT_CMD_GLOBAL_SYMBOLS])dnl -m4_require([_LT_TAG_COMPILER])dnl -AC_MSG_CHECKING([whether the $compiler linker ($LD) supports shared libraries]) -m4_if([$1], [CXX], [ - _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' - _LT_TAGVAR(exclude_expsyms, $1)=['_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*'] - case $host_os in - aix[[4-9]]*) - # If we're using GNU nm, then we don't want the "-C" option. - # -C means demangle to GNU nm, but means don't demangle to AIX nm. - # Without the "-l" option, or with the "-B" option, AIX nm treats - # weak defined symbols like other global defined symbols, whereas - # GNU nm marks them as "W". - # While the 'weak' keyword is ignored in the Export File, we need - # it in the Import File for the 'aix-soname' feature, so we have - # to replace the "-B" option with "-P" for AIX nm. - if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then - _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && ([substr](\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' - else - _LT_TAGVAR(export_symbols_cmds, $1)='`func_echo_all $NM | $SED -e '\''s/B\([[^B]]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && ([substr](\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' - fi - ;; - pw32*) - _LT_TAGVAR(export_symbols_cmds, $1)=$ltdll_cmds - ;; - cygwin* | mingw* | cegcc*) - case $cc_basename in - cl*) - _LT_TAGVAR(exclude_expsyms, $1)='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' - ;; - *) - _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1 DATA/;s/^.*[[ ]]__nm__\([[^ ]]*\)[[ ]][[^ ]]*/\1 DATA/;/^I[[ ]]/d;/^[[AITW]][[ ]]/s/.* //'\'' | sort | uniq > $export_symbols' - _LT_TAGVAR(exclude_expsyms, $1)=['[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname'] - ;; - esac - ;; - linux* | k*bsd*-gnu | gnu*) - _LT_TAGVAR(link_all_deplibs, $1)=no - ;; - *) - _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' - ;; - esac -], [ - runpath_var= - _LT_TAGVAR(allow_undefined_flag, $1)= - _LT_TAGVAR(always_export_symbols, $1)=no - _LT_TAGVAR(archive_cmds, $1)= - _LT_TAGVAR(archive_expsym_cmds, $1)= - _LT_TAGVAR(compiler_needs_object, $1)=no - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no - _LT_TAGVAR(export_dynamic_flag_spec, $1)= - _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' - _LT_TAGVAR(hardcode_automatic, $1)=no - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_direct_absolute, $1)=no - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)= - _LT_TAGVAR(hardcode_libdir_separator, $1)= - _LT_TAGVAR(hardcode_minus_L, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported - _LT_TAGVAR(inherit_rpath, $1)=no - _LT_TAGVAR(link_all_deplibs, $1)=unknown - _LT_TAGVAR(module_cmds, $1)= - _LT_TAGVAR(module_expsym_cmds, $1)= - _LT_TAGVAR(old_archive_from_new_cmds, $1)= - _LT_TAGVAR(old_archive_from_expsyms_cmds, $1)= - _LT_TAGVAR(thread_safe_flag_spec, $1)= - _LT_TAGVAR(whole_archive_flag_spec, $1)= - # include_expsyms should be a list of space-separated symbols to be *always* - # included in the symbol list - _LT_TAGVAR(include_expsyms, $1)= - # exclude_expsyms can be an extended regexp of symbols to exclude - # it will be wrapped by ' (' and ')$', so one must not match beginning or - # end of line. Example: 'a|bc|.*d.*' will exclude the symbols 'a' and 'bc', - # as well as any symbol that contains 'd'. - _LT_TAGVAR(exclude_expsyms, $1)=['_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*'] - # Although _GLOBAL_OFFSET_TABLE_ is a valid symbol C name, most a.out - # platforms (ab)use it in PIC code, but their linkers get confused if - # the symbol is explicitly referenced. Since portable code cannot - # rely on this symbol name, it's probably fine to never include it in - # preloaded symbol tables. - # Exclude shared library initialization/finalization symbols. -dnl Note also adjust exclude_expsyms for C++ above. - extract_expsyms_cmds= - - case $host_os in - cygwin* | mingw* | pw32* | cegcc*) - # FIXME: the MSVC++ port hasn't been tested in a loooong time - # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. - if test yes != "$GCC"; then - with_gnu_ld=no - fi - ;; - interix*) - # we just hope/assume this is gcc and not c89 (= MSVC++) - with_gnu_ld=yes - ;; - openbsd* | bitrig*) - with_gnu_ld=no - ;; - linux* | k*bsd*-gnu | gnu*) - _LT_TAGVAR(link_all_deplibs, $1)=no - ;; - esac - - _LT_TAGVAR(ld_shlibs, $1)=yes - - # On some targets, GNU ld is compatible enough with the native linker - # that we're better off using the native interface for both. - lt_use_gnu_ld_interface=no - if test yes = "$with_gnu_ld"; then - case $host_os in - aix*) - # The AIX port of GNU ld has always aspired to compatibility - # with the native linker. However, as the warning in the GNU ld - # block says, versions before 2.19.5* couldn't really create working - # shared libraries, regardless of the interface used. - case `$LD -v 2>&1` in - *\ \(GNU\ Binutils\)\ 2.19.5*) ;; - *\ \(GNU\ Binutils\)\ 2.[[2-9]]*) ;; - *\ \(GNU\ Binutils\)\ [[3-9]]*) ;; - *) - lt_use_gnu_ld_interface=yes - ;; - esac - ;; - *) - lt_use_gnu_ld_interface=yes - ;; - esac - fi - - if test yes = "$lt_use_gnu_ld_interface"; then - # If archive_cmds runs LD, not CC, wlarc should be empty - wlarc='$wl' - - # Set some defaults for GNU ld with shared library support. These - # are reset later if shared libraries are not supported. Putting them - # here allows them to be overridden if necessary. - runpath_var=LD_RUN_PATH - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-dynamic' - # ancient GNU ld didn't support --whole-archive et. al. - if $LD --help 2>&1 | $GREP 'no-whole-archive' > /dev/null; then - _LT_TAGVAR(whole_archive_flag_spec, $1)=$wlarc'--whole-archive$convenience '$wlarc'--no-whole-archive' - else - _LT_TAGVAR(whole_archive_flag_spec, $1)= - fi - supports_anon_versioning=no - case `$LD -v | $SED -e 's/([^)]\+)\s\+//' 2>&1` in - *GNU\ gold*) supports_anon_versioning=yes ;; - *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.10.*) ;; # catch versions < 2.11 - *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ... - *\ 2.11.92.0.12\ *) supports_anon_versioning=yes ;; # Mandrake 8.2 ... - *\ 2.11.*) ;; # other 2.11 versions - *) supports_anon_versioning=yes ;; - esac - - # See if GNU ld supports shared libraries. - case $host_os in - aix[[3-9]]*) - # On AIX/PPC, the GNU linker is very broken - if test ia64 != "$host_cpu"; then - _LT_TAGVAR(ld_shlibs, $1)=no - cat <<_LT_EOF 1>&2 - -*** Warning: the GNU linker, at least up to release 2.19, is reported -*** to be unable to reliably create shared libraries on AIX. -*** Therefore, libtool is disabling shared libraries support. If you -*** really care for shared libraries, you may want to install binutils -*** 2.20 or above, or modify your PATH so that a non-GNU linker is found. -*** You will then need to restart the configuration process. - -_LT_EOF - fi - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='' - ;; - m68k) - _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_minus_L, $1)=yes - ;; - esac - ;; - - beos*) - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - # Joseph Beckenbach says some releases of gcc - # support --undefined. This deserves some investigation. FIXME - _LT_TAGVAR(archive_cmds, $1)='$CC -nostart $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - cygwin* | mingw* | pw32* | cegcc*) - # _LT_TAGVAR(hardcode_libdir_flag_spec, $1) is actually meaningless, - # as there is no search path for DLLs. - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-all-symbols' - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - _LT_TAGVAR(always_export_symbols, $1)=no - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1 DATA/;s/^.*[[ ]]__nm__\([[^ ]]*\)[[ ]][[^ ]]*/\1 DATA/;/^I[[ ]]/d;/^[[AITW]][[ ]]/s/.* //'\'' | sort | uniq > $export_symbols' - _LT_TAGVAR(exclude_expsyms, $1)=['[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname'] - - if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -o $output_objdir/$soname $wl--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' - # If the export-symbols file already is a .def file, use it as - # is; otherwise, prepend EXPORTS... - _LT_TAGVAR(archive_expsym_cmds, $1)='if _LT_DLL_DEF_P([$export_symbols]); then - cp $export_symbols $output_objdir/$soname.def; - else - echo EXPORTS > $output_objdir/$soname.def; - cat $export_symbols >> $output_objdir/$soname.def; - fi~ - $CC -shared $output_objdir/$soname.def $libobjs $deplibs $compiler_flags -o $output_objdir/$soname $wl--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - haiku*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(link_all_deplibs, $1)=yes - ;; - - os2*) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - shrext_cmds=.dll - _LT_TAGVAR(archive_cmds, $1)='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - emxexp $libobjs | $SED /"_DLL_InitTerm"/d >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - _LT_TAGVAR(archive_expsym_cmds, $1)='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - prefix_cmds="$SED"~ - if test EXPORTS = "`$SED 1q $export_symbols`"; then - prefix_cmds="$prefix_cmds -e 1d"; - fi~ - prefix_cmds="$prefix_cmds -e \"s/^\(.*\)$/_\1/g\""~ - cat $export_symbols | $prefix_cmds >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - _LT_TAGVAR(old_archive_From_new_cmds, $1)='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - ;; - - interix[[3-9]]*) - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. - # Instead, shared libraries are loaded at an image base (0x10000000 by - # default) and relocated if they conflict, which is a slow very memory - # consuming and fragmenting process. To avoid this, we pick a random, - # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link - # time. Moving up from 0x10000000 also allows more sbrk(2) space. - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - ;; - - gnu* | linux* | tpf* | k*bsd*-gnu | kopensolaris*-gnu) - tmp_diet=no - if test linux-dietlibc = "$host_os"; then - case $cc_basename in - diet\ *) tmp_diet=yes;; # linux-dietlibc with static linking (!diet-dyn) - esac - fi - if $LD --help 2>&1 | $EGREP ': supported targets:.* elf' > /dev/null \ - && test no = "$tmp_diet" - then - tmp_addflag=' $pic_flag' - tmp_sharedflag='-shared' - case $cc_basename,$host_cpu in - pgcc*) # Portland Group C compiler - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - tmp_addflag=' $pic_flag' - ;; - pgf77* | pgf90* | pgf95* | pgfortran*) - # Portland Group f77 and f90 compilers - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - tmp_addflag=' $pic_flag -Mnomain' ;; - ecc*,ia64* | icc*,ia64*) # Intel C compiler on ia64 - tmp_addflag=' -i_dynamic' ;; - efc*,ia64* | ifort*,ia64*) # Intel Fortran compiler on ia64 - tmp_addflag=' -i_dynamic -nofor_main' ;; - ifc* | ifort*) # Intel Fortran compiler - tmp_addflag=' -nofor_main' ;; - lf95*) # Lahey Fortran 8.1 - _LT_TAGVAR(whole_archive_flag_spec, $1)= - tmp_sharedflag='--shared' ;; - nagfor*) # NAGFOR 5.3 - tmp_sharedflag='-Wl,-shared' ;; - xl[[cC]]* | bgxl[[cC]]* | mpixl[[cC]]*) # IBM XL C 8.0 on PPC (deal with xlf below) - tmp_sharedflag='-qmkshrobj' - tmp_addflag= ;; - nvcc*) # Cuda Compiler Driver 2.2 - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - _LT_TAGVAR(compiler_needs_object, $1)=yes - ;; - esac - case `$CC -V 2>&1 | sed 5q` in - *Sun\ C*) # Sun C 5.9 - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - _LT_TAGVAR(compiler_needs_object, $1)=yes - tmp_sharedflag='-G' ;; - *Sun\ F*) # Sun Fortran 8.3 - tmp_sharedflag='-G' ;; - esac - _LT_TAGVAR(archive_cmds, $1)='$CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - - if test yes = "$supports_anon_versioning"; then - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ - echo "local: *; };" >> $output_objdir/$libname.ver~ - $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' - fi - - case $cc_basename in - tcc*) - _LT_TAGVAR(export_dynamic_flag_spec, $1)='-rdynamic' - ;; - xlf* | bgf* | bgxlf* | mpixlf*) - # IBM XL Fortran 10.1 on PPC cannot create shared libs itself - _LT_TAGVAR(whole_archive_flag_spec, $1)='--whole-archive$convenience --no-whole-archive' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(archive_cmds, $1)='$LD -shared $libobjs $deplibs $linker_flags -soname $soname -o $lib' - if test yes = "$supports_anon_versioning"; then - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ - echo "local: *; };" >> $output_objdir/$libname.ver~ - $LD -shared $libobjs $deplibs $linker_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib' - fi - ;; - esac - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - netbsd* | netbsdelf*-gnu) - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable $libobjs $deplibs $linker_flags -o $lib' - wlarc= - else - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - fi - ;; - - solaris*) - if $LD -v 2>&1 | $GREP 'BFD 2\.8' > /dev/null; then - _LT_TAGVAR(ld_shlibs, $1)=no - cat <<_LT_EOF 1>&2 - -*** Warning: The releases 2.8.* of the GNU linker cannot reliably -*** create shared libraries on Solaris systems. Therefore, libtool -*** is disabling shared libraries support. We urge you to upgrade GNU -*** binutils to release 2.9.1 or newer. Another option is to modify -*** your PATH or compiler configuration so that the native linker is -*** used, and then restart. - -_LT_EOF - elif $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX*) - case `$LD -v 2>&1` in - *\ [[01]].* | *\ 2.[[0-9]].* | *\ 2.1[[0-5]].*) - _LT_TAGVAR(ld_shlibs, $1)=no - cat <<_LT_EOF 1>&2 - -*** Warning: Releases of the GNU linker prior to 2.16.91.0.3 cannot -*** reliably create shared libraries on SCO systems. Therefore, libtool -*** is disabling shared libraries support. We urge you to upgrade GNU -*** binutils to release 2.16.91.0.3 or newer. Another option is to modify -*** your PATH or compiler configuration so that the native linker is -*** used, and then restart. - -_LT_EOF - ;; - *) - # For security reasons, it is highly recommended that you always - # use absolute paths for naming shared libraries, and exclude the - # DT_RUNPATH tag from executables and libraries. But doing so - # requires that you compile everything twice, which is a pain. - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - esac - ;; - - sunos4*) - _LT_TAGVAR(archive_cmds, $1)='$LD -assert pure-text -Bshareable -o $lib $libobjs $deplibs $linker_flags' - wlarc= - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - *) - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - esac - - if test no = "$_LT_TAGVAR(ld_shlibs, $1)"; then - runpath_var= - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)= - _LT_TAGVAR(export_dynamic_flag_spec, $1)= - _LT_TAGVAR(whole_archive_flag_spec, $1)= - fi - else - # PORTME fill in a description of your system's linker (not GNU ld) - case $host_os in - aix3*) - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - _LT_TAGVAR(always_export_symbols, $1)=yes - _LT_TAGVAR(archive_expsym_cmds, $1)='$LD -o $output_objdir/$soname $libobjs $deplibs $linker_flags -bE:$export_symbols -T512 -H512 -bM:SRE~$AR $AR_FLAGS $lib $output_objdir/$soname' - # Note: this linker hardcodes the directories in LIBPATH if there - # are no directories specified by -L. - _LT_TAGVAR(hardcode_minus_L, $1)=yes - if test yes = "$GCC" && test -z "$lt_prog_compiler_static"; then - # Neither direct hardcoding nor static linking is supported with a - # broken collect2. - _LT_TAGVAR(hardcode_direct, $1)=unsupported - fi - ;; - - aix[[4-9]]*) - if test ia64 = "$host_cpu"; then - # On IA64, the linker does run time linking by default, so we don't - # have to do anything special. - aix_use_runtimelinking=no - exp_sym_flag='-Bexport' - no_entry_flag= - else - # If we're using GNU nm, then we don't want the "-C" option. - # -C means demangle to GNU nm, but means don't demangle to AIX nm. - # Without the "-l" option, or with the "-B" option, AIX nm treats - # weak defined symbols like other global defined symbols, whereas - # GNU nm marks them as "W". - # While the 'weak' keyword is ignored in the Export File, we need - # it in the Import File for the 'aix-soname' feature, so we have - # to replace the "-B" option with "-P" for AIX nm. - if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then - _LT_TAGVAR(export_symbols_cmds, $1)='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && ([substr](\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' - else - _LT_TAGVAR(export_symbols_cmds, $1)='`func_echo_all $NM | $SED -e '\''s/B\([[^B]]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && ([substr](\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' - fi - aix_use_runtimelinking=no - - # Test if we are trying to use run time linking or normal - # AIX style linking. If -brtl is somewhere in LDFLAGS, we - # have runtime linking enabled, and use it for executables. - # For shared libraries, we enable/disable runtime linking - # depending on the kind of the shared library created - - # when "with_aix_soname,aix_use_runtimelinking" is: - # "aix,no" lib.a(lib.so.V) shared, rtl:no, for executables - # "aix,yes" lib.so shared, rtl:yes, for executables - # lib.a static archive - # "both,no" lib.so.V(shr.o) shared, rtl:yes - # lib.a(lib.so.V) shared, rtl:no, for executables - # "both,yes" lib.so.V(shr.o) shared, rtl:yes, for executables - # lib.a(lib.so.V) shared, rtl:no - # "svr4,*" lib.so.V(shr.o) shared, rtl:yes, for executables - # lib.a static archive - case $host_os in aix4.[[23]]|aix4.[[23]].*|aix[[5-9]]*) - for ld_flag in $LDFLAGS; do - if (test x-brtl = "x$ld_flag" || test x-Wl,-brtl = "x$ld_flag"); then - aix_use_runtimelinking=yes - break - fi - done - if test svr4,no = "$with_aix_soname,$aix_use_runtimelinking"; then - # With aix-soname=svr4, we create the lib.so.V shared archives only, - # so we don't have lib.a shared libs to link our executables. - # We have to force runtime linking in this case. - aix_use_runtimelinking=yes - LDFLAGS="$LDFLAGS -Wl,-brtl" - fi - ;; - esac - - exp_sym_flag='-bexport' - no_entry_flag='-bnoentry' - fi - - # When large executables or shared objects are built, AIX ld can - # have problems creating the table of contents. If linking a library - # or program results in "error TOC overflow" add -mminimal-toc to - # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not - # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. - - _LT_TAGVAR(archive_cmds, $1)='' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - _LT_TAGVAR(hardcode_libdir_separator, $1)=':' - _LT_TAGVAR(link_all_deplibs, $1)=yes - _LT_TAGVAR(file_list_spec, $1)='$wl-f,' - case $with_aix_soname,$aix_use_runtimelinking in - aix,*) ;; # traditional, no import file - svr4,* | *,yes) # use import file - # The Import File defines what to hardcode. - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_direct_absolute, $1)=no - ;; - esac - - if test yes = "$GCC"; then - case $host_os in aix4.[[012]]|aix4.[[012]].*) - # We only want to do this on AIX 4.2 and lower, the check - # below for broken collect2 doesn't work under 4.3+ - collect2name=`$CC -print-prog-name=collect2` - if test -f "$collect2name" && - strings "$collect2name" | $GREP resolve_lib_name >/dev/null - then - # We have reworked collect2 - : - else - # We have old collect2 - _LT_TAGVAR(hardcode_direct, $1)=unsupported - # It fails to find uninstalled libraries when the uninstalled - # path is not listed in the libpath. Setting hardcode_minus_L - # to unsupported forces relinking - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)= - fi - ;; - esac - shared_flag='-shared' - if test yes = "$aix_use_runtimelinking"; then - shared_flag="$shared_flag "'$wl-G' - fi - # Need to ensure runtime linking is disabled for the traditional - # shared library, or the linker may eventually find shared libraries - # /with/ Import File - we do not want to mix them. - shared_flag_aix='-shared' - shared_flag_svr4='-shared $wl-G' - else - # not using gcc - if test ia64 = "$host_cpu"; then - # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release - # chokes on -Wl,-G. The following line is correct: - shared_flag='-G' - else - if test yes = "$aix_use_runtimelinking"; then - shared_flag='$wl-G' - else - shared_flag='$wl-bM:SRE' - fi - shared_flag_aix='$wl-bM:SRE' - shared_flag_svr4='$wl-G' - fi - fi - - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-bexpall' - # It seems that -bexpall does not export symbols beginning with - # underscore (_), so it is better to generate a list of symbols to export. - _LT_TAGVAR(always_export_symbols, $1)=yes - if test aix,yes = "$with_aix_soname,$aix_use_runtimelinking"; then - # Warning - without using the other runtime loading flags (-brtl), - # -berok will link without error, but may produce a broken library. - _LT_TAGVAR(allow_undefined_flag, $1)='-berok' - # Determine the default libpath from the value encoded in an - # empty executable. - _LT_SYS_MODULE_PATH_AIX([$1]) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-blibpath:$libdir:'"$aix_libpath" - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $deplibs $wl'$no_entry_flag' $compiler_flags `if test -n "$allow_undefined_flag"; then func_echo_all "$wl$allow_undefined_flag"; else :; fi` $wl'$exp_sym_flag:\$export_symbols' '$shared_flag - else - if test ia64 = "$host_cpu"; then - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-R $libdir:/usr/lib:/lib' - _LT_TAGVAR(allow_undefined_flag, $1)="-z nodefs" - _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\$wl$no_entry_flag"' $compiler_flags $wl$allow_undefined_flag '"\$wl$exp_sym_flag:\$export_symbols" - else - # Determine the default libpath from the value encoded in an - # empty executable. - _LT_SYS_MODULE_PATH_AIX([$1]) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-blibpath:$libdir:'"$aix_libpath" - # Warning - without using the other run time loading flags, - # -berok will link without error, but may produce a broken library. - _LT_TAGVAR(no_undefined_flag, $1)=' $wl-bernotok' - _LT_TAGVAR(allow_undefined_flag, $1)=' $wl-berok' - if test yes = "$with_gnu_ld"; then - # We only use this code for GNU lds that support --whole-archive. - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive$convenience $wl--no-whole-archive' - else - # Exported symbols can be pulled into shared objects from archives - _LT_TAGVAR(whole_archive_flag_spec, $1)='$convenience' - fi - _LT_TAGVAR(archive_cmds_need_lc, $1)=yes - _LT_TAGVAR(archive_expsym_cmds, $1)='$RM -r $output_objdir/$realname.d~$MKDIR $output_objdir/$realname.d' - # -brtl affects multiple linker settings, -berok does not and is overridden later - compiler_flags_filtered='`func_echo_all "$compiler_flags " | $SED -e "s%-brtl\\([[, ]]\\)%-berok\\1%g"`' - if test svr4 != "$with_aix_soname"; then - # This is similar to how AIX traditionally builds its shared libraries. - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_aix' -o $output_objdir/$realname.d/$soname $libobjs $deplibs $wl-bnoentry '$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$realname.d/$soname' - fi - if test aix != "$with_aix_soname"; then - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_svr4' -o $output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs $wl-bnoentry '$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e $output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! $soname($shared_archive_member_spec.o)"; if test shr_64 = "$shared_archive_member_spec"; then func_echo_all "# 64"; else func_echo_all "# 32"; fi; cat $export_symbols ) > $output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS $output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o $output_objdir/$realname.d/$shared_archive_member_spec.imp' - else - # used by -dlpreopen to get the symbols - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$MV $output_objdir/$realname.d/$soname $output_objdir' - fi - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$RM -r $output_objdir/$realname.d' - fi - fi - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='' - ;; - m68k) - _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_minus_L, $1)=yes - ;; - esac - ;; - - bsdi[[45]]*) - _LT_TAGVAR(export_dynamic_flag_spec, $1)=-rdynamic - ;; - - cygwin* | mingw* | pw32* | cegcc*) - # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. - # hardcode_libdir_flag_spec is actually meaningless, as there is - # no search path for DLLs. - case $cc_basename in - cl*) - # Native MSVC - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - _LT_TAGVAR(always_export_symbols, $1)=yes - _LT_TAGVAR(file_list_spec, $1)='@' - # Tell ltmain to make .lib files, not .a files. - libext=lib - # Tell ltmain to make .dll files, not .so files. - shrext_cmds=.dll - # FIXME: Setting linknames here is a bad hack. - _LT_TAGVAR(archive_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~linknames=' - _LT_TAGVAR(archive_expsym_cmds, $1)='if _LT_DLL_DEF_P([$export_symbols]); then - cp "$export_symbols" "$output_objdir/$soname.def"; - echo "$tool_output_objdir$soname.def" > "$output_objdir/$soname.exp"; - else - $SED -e '\''s/^/-link -EXPORT:/'\'' < $export_symbols > $output_objdir/$soname.exp; - fi~ - $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ - linknames=' - # The linker will not automatically build a static lib if we build a DLL. - # _LT_TAGVAR(old_archive_from_new_cmds, $1)='true' - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - _LT_TAGVAR(exclude_expsyms, $1)='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' - _LT_TAGVAR(export_symbols_cmds, $1)='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[[BCDGRS]][[ ]]/s/.*[[ ]]\([[^ ]]*\)/\1,DATA/'\'' | $SED -e '\''/^[[AITW]][[ ]]/s/.*[[ ]]//'\'' | sort | uniq > $export_symbols' - # Don't use ranlib - _LT_TAGVAR(old_postinstall_cmds, $1)='chmod 644 $oldlib' - _LT_TAGVAR(postlink_cmds, $1)='lt_outputfile="@OUTPUT@"~ - lt_tool_outputfile="@TOOL_OUTPUT@"~ - case $lt_outputfile in - *.exe|*.EXE) ;; - *) - lt_outputfile=$lt_outputfile.exe - lt_tool_outputfile=$lt_tool_outputfile.exe - ;; - esac~ - if test : != "$MANIFEST_TOOL" && test -f "$lt_outputfile.manifest"; then - $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; - $RM "$lt_outputfile.manifest"; - fi' - ;; - *) - # Assume MSVC wrapper - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - # Tell ltmain to make .lib files, not .a files. - libext=lib - # Tell ltmain to make .dll files, not .so files. - shrext_cmds=.dll - # FIXME: Setting linknames here is a bad hack. - _LT_TAGVAR(archive_cmds, $1)='$CC -o $lib $libobjs $compiler_flags `func_echo_all "$deplibs" | $SED '\''s/ -lc$//'\''` -link -dll~linknames=' - # The linker will automatically build a .lib file if we build a DLL. - _LT_TAGVAR(old_archive_from_new_cmds, $1)='true' - # FIXME: Should let the user specify the lib program. - _LT_TAGVAR(old_archive_cmds, $1)='lib -OUT:$oldlib$oldobjs$old_deplibs' - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - ;; - esac - ;; - - darwin* | rhapsody*) - _LT_DARWIN_LINKER_FEATURES($1) - ;; - - dgux*) - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - # FreeBSD 2.2.[012] allows us to include c++rt0.o to get C++ constructor - # support. Future versions do this automatically, but an explicit c++rt0.o - # does not break anything, and helps significantly (at the cost of a little - # extra space). - freebsd2.2*) - _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags /usr/lib/c++rt0.o' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - # Unfortunately, older versions of FreeBSD 2 do not have this feature. - freebsd2.*) - _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - # FreeBSD 3 and greater uses gcc -shared to do shared libraries. - freebsd* | dragonfly*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - hpux9*) - if test yes = "$GCC"; then - _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -shared $pic_flag $wl+b $wl$install_libdir -o $output_objdir/$soname $libobjs $deplibs $compiler_flags~test "x$output_objdir/$soname" = "x$lib" || mv $output_objdir/$soname $lib' - else - _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$LD -b +b $install_libdir -o $output_objdir/$soname $libobjs $deplibs $linker_flags~test "x$output_objdir/$soname" = "x$lib" || mv $output_objdir/$soname $lib' - fi - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl+b $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - _LT_TAGVAR(hardcode_direct, $1)=yes - - # hardcode_minus_L: Not really in the search PATH, - # but as the default location of the library. - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - ;; - - hpux10*) - if test yes,no = "$GCC,$with_gnu_ld"; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags' - else - _LT_TAGVAR(archive_cmds, $1)='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags' - fi - if test no = "$with_gnu_ld"; then - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl+b $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - # hardcode_minus_L: Not really in the search PATH, - # but as the default location of the library. - _LT_TAGVAR(hardcode_minus_L, $1)=yes - fi - ;; - - hpux11*) - if test yes,no = "$GCC,$with_gnu_ld"; then - case $host_cpu in - hppa*64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $wl+h $wl$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - ia64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $wl+h $wl$soname $wl+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' - ;; - *) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags' - ;; - esac - else - case $host_cpu in - hppa*64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - ia64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname $wl+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' - ;; - *) - m4_if($1, [], [ - # Older versions of the 11.00 compiler do not understand -b yet - # (HP92453-01 A.11.01.20 doesn't, HP92453-01 B.11.X.35175-35176.GP does) - _LT_LINKER_OPTION([if $CC understands -b], - _LT_TAGVAR(lt_cv_prog_compiler__b, $1), [-b], - [_LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags'], - [_LT_TAGVAR(archive_cmds, $1)='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags'])], - [_LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags']) - ;; - esac - fi - if test no = "$with_gnu_ld"; then - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl+b $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - - case $host_cpu in - hppa*64*|ia64*) - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - *) - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - - # hardcode_minus_L: Not really in the search PATH, - # but as the default location of the library. - _LT_TAGVAR(hardcode_minus_L, $1)=yes - ;; - esac - fi - ;; - - irix5* | irix6* | nonstopux*) - if test yes = "$GCC"; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - # Try to use the -exported_symbol ld option, if it does not - # work, assume that -exports_file does not work either and - # implicitly export all symbols. - # This should be the same for all languages, so no per-tag cache variable. - AC_CACHE_CHECK([whether the $host_os linker accepts -exported_symbol], - [lt_cv_irix_exported_symbol], - [save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS -shared $wl-exported_symbol ${wl}foo $wl-update_registry $wl/dev/null" - AC_LINK_IFELSE( - [AC_LANG_SOURCE( - [AC_LANG_CASE([C], [[int foo (void) { return 0; }]], - [C++], [[int foo (void) { return 0; }]], - [Fortran 77], [[ - subroutine foo - end]], - [Fortran], [[ - subroutine foo - end]])])], - [lt_cv_irix_exported_symbol=yes], - [lt_cv_irix_exported_symbol=no]) - LDFLAGS=$save_LDFLAGS]) - if test yes = "$lt_cv_irix_exported_symbol"; then - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations $wl-exports_file $wl$export_symbols -o $lib' - fi - _LT_TAGVAR(link_all_deplibs, $1)=no - else - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -exports_file $export_symbols -o $lib' - fi - _LT_TAGVAR(archive_cmds_need_lc, $1)='no' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - _LT_TAGVAR(inherit_rpath, $1)=yes - _LT_TAGVAR(link_all_deplibs, $1)=yes - ;; - - linux*) - case $cc_basename in - tcc*) - # Fabrice Bellard et al's Tiny C Compiler - _LT_TAGVAR(ld_shlibs, $1)=yes - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - ;; - esac - ;; - - netbsd* | netbsdelf*-gnu) - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' # a.out - else - _LT_TAGVAR(archive_cmds, $1)='$LD -shared -o $lib $libobjs $deplibs $linker_flags' # ELF - fi - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - newsos6) - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - *nto* | *qnx*) - ;; - - openbsd* | bitrig*) - if test -f /usr/libexec/ld.so; then - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`"; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags $wl-retain-symbols-file,$export_symbols' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - else - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - fi - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - os2*) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - shrext_cmds=.dll - _LT_TAGVAR(archive_cmds, $1)='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - emxexp $libobjs | $SED /"_DLL_InitTerm"/d >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - _LT_TAGVAR(archive_expsym_cmds, $1)='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - prefix_cmds="$SED"~ - if test EXPORTS = "`$SED 1q $export_symbols`"; then - prefix_cmds="$prefix_cmds -e 1d"; - fi~ - prefix_cmds="$prefix_cmds -e \"s/^\(.*\)$/_\1/g\""~ - cat $export_symbols | $prefix_cmds >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - _LT_TAGVAR(old_archive_From_new_cmds, $1)='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - ;; - - osf3*) - if test yes = "$GCC"; then - _LT_TAGVAR(allow_undefined_flag, $1)=' $wl-expect_unresolved $wl\*' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared$allow_undefined_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - else - _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared$allow_undefined_flag $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - fi - _LT_TAGVAR(archive_cmds_need_lc, $1)='no' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - ;; - - osf4* | osf5*) # as osf3* with the addition of -msym flag - if test yes = "$GCC"; then - _LT_TAGVAR(allow_undefined_flag, $1)=' $wl-expect_unresolved $wl\*' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared$allow_undefined_flag $pic_flag $libobjs $deplibs $compiler_flags $wl-msym $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - else - _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared$allow_undefined_flag $libobjs $deplibs $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done; printf "%s\\n" "-hidden">> $lib.exp~ - $CC -shared$allow_undefined_flag $wl-input $wl$lib.exp $compiler_flags $libobjs $deplibs -soname $soname `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib~$RM $lib.exp' - - # Both c and cxx compiler support -rpath directly - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir' - fi - _LT_TAGVAR(archive_cmds_need_lc, $1)='no' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - ;; - - solaris*) - _LT_TAGVAR(no_undefined_flag, $1)=' -z defs' - if test yes = "$GCC"; then - wlarc='$wl' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $wl-z ${wl}text $wl-h $wl$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -shared $pic_flag $wl-z ${wl}text $wl-M $wl$lib.exp $wl-h $wl$soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' - else - case `$CC -V 2>&1` in - *"Compilers 5.0"*) - wlarc='' - _LT_TAGVAR(archive_cmds, $1)='$LD -G$allow_undefined_flag -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $LD -G$allow_undefined_flag -M $lib.exp -h $soname -o $lib $libobjs $deplibs $linker_flags~$RM $lib.exp' - ;; - *) - wlarc='$wl' - _LT_TAGVAR(archive_cmds, $1)='$CC -G$allow_undefined_flag -h $soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -G$allow_undefined_flag -M $lib.exp -h $soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' - ;; - esac - fi - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - case $host_os in - solaris2.[[0-5]] | solaris2.[[0-5]].*) ;; - *) - # The compiler driver will combine and reorder linker options, - # but understands '-z linker_flag'. GCC discards it without '$wl', - # but is careful enough not to reorder. - # Supported since Solaris 2.6 (maybe 2.5.1?) - if test yes = "$GCC"; then - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl-z ${wl}allextract$convenience $wl-z ${wl}defaultextract' - else - _LT_TAGVAR(whole_archive_flag_spec, $1)='-z allextract$convenience -z defaultextract' - fi - ;; - esac - _LT_TAGVAR(link_all_deplibs, $1)=yes - ;; - - sunos4*) - if test sequent = "$host_vendor"; then - # Use $CC to link under sequent, because it throws in some extra .o - # files that make .init and .fini sections work. - _LT_TAGVAR(archive_cmds, $1)='$CC -G $wl-h $soname -o $lib $libobjs $deplibs $compiler_flags' - else - _LT_TAGVAR(archive_cmds, $1)='$LD -assert pure-text -Bstatic -o $lib $libobjs $deplibs $linker_flags' - fi - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - sysv4) - case $host_vendor in - sni) - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_direct, $1)=yes # is this really true??? - ;; - siemens) - ## LD is ld it makes a PLAMLIB - ## CC just makes a GrossModule. - _LT_TAGVAR(archive_cmds, $1)='$LD -G -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(reload_cmds, $1)='$CC -r -o $output$reload_objs' - _LT_TAGVAR(hardcode_direct, $1)=no - ;; - motorola) - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_direct, $1)=no #Motorola manual says yes, but my tests say they lie - ;; - esac - runpath_var='LD_RUN_PATH' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - sysv4.3*) - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(export_dynamic_flag_spec, $1)='-Bexport' - ;; - - sysv4*MP*) - if test -d /usr/nec; then - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - runpath_var=LD_RUN_PATH - hardcode_runpath_var=yes - _LT_TAGVAR(ld_shlibs, $1)=yes - fi - ;; - - sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[[01]].[[10]]* | unixware7* | sco3.2v5.0.[[024]]*) - _LT_TAGVAR(no_undefined_flag, $1)='$wl-z,text' - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - runpath_var='LD_RUN_PATH' - - if test yes = "$GCC"; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - else - _LT_TAGVAR(archive_cmds, $1)='$CC -G $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - fi - ;; - - sysv5* | sco3.2v5* | sco5v6*) - # Note: We CANNOT use -z defs as we might desire, because we do not - # link with -lc, and that would cause any symbols used from libc to - # always be unresolved, which means just about no library would - # ever link correctly. If we're not using GNU ld we use -z text - # though, which does catch some bad symbols but isn't as heavy-handed - # as -z defs. - _LT_TAGVAR(no_undefined_flag, $1)='$wl-z,text' - _LT_TAGVAR(allow_undefined_flag, $1)='$wl-z,nodefs' - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-R,$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=':' - _LT_TAGVAR(link_all_deplibs, $1)=yes - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-Bexport' - runpath_var='LD_RUN_PATH' - - if test yes = "$GCC"; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - else - _LT_TAGVAR(archive_cmds, $1)='$CC -G $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - fi - ;; - - uts4*) - _LT_TAGVAR(archive_cmds, $1)='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - - *) - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - - if test sni = "$host_vendor"; then - case $host in - sysv4 | sysv4.2uw2* | sysv4.3* | sysv5*) - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-Blargedynsym' - ;; - esac - fi - fi -]) -AC_MSG_RESULT([$_LT_TAGVAR(ld_shlibs, $1)]) -test no = "$_LT_TAGVAR(ld_shlibs, $1)" && can_build_shared=no - -_LT_TAGVAR(with_gnu_ld, $1)=$with_gnu_ld - -_LT_DECL([], [libext], [0], [Old archive suffix (normally "a")])dnl -_LT_DECL([], [shrext_cmds], [1], [Shared library suffix (normally ".so")])dnl -_LT_DECL([], [extract_expsyms_cmds], [2], - [The commands to extract the exported symbol list from a shared archive]) - -# -# Do we need to explicitly link libc? -# -case "x$_LT_TAGVAR(archive_cmds_need_lc, $1)" in -x|xyes) - # Assume -lc should be added - _LT_TAGVAR(archive_cmds_need_lc, $1)=yes - - if test yes,yes = "$GCC,$enable_shared"; then - case $_LT_TAGVAR(archive_cmds, $1) in - *'~'*) - # FIXME: we may have to deal with multi-command sequences. - ;; - '$CC '*) - # Test whether the compiler implicitly links with -lc since on some - # systems, -lgcc has to come before -lc. If gcc already passes -lc - # to ld, don't add -lc before -lgcc. - AC_CACHE_CHECK([whether -lc should be explicitly linked in], - [lt_cv_]_LT_TAGVAR(archive_cmds_need_lc, $1), - [$RM conftest* - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - - if AC_TRY_EVAL(ac_compile) 2>conftest.err; then - soname=conftest - lib=conftest - libobjs=conftest.$ac_objext - deplibs= - wl=$_LT_TAGVAR(lt_prog_compiler_wl, $1) - pic_flag=$_LT_TAGVAR(lt_prog_compiler_pic, $1) - compiler_flags=-v - linker_flags=-v - verstring= - output_objdir=. - libname=conftest - lt_save_allow_undefined_flag=$_LT_TAGVAR(allow_undefined_flag, $1) - _LT_TAGVAR(allow_undefined_flag, $1)= - if AC_TRY_EVAL(_LT_TAGVAR(archive_cmds, $1) 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1) - then - lt_cv_[]_LT_TAGVAR(archive_cmds_need_lc, $1)=no - else - lt_cv_[]_LT_TAGVAR(archive_cmds_need_lc, $1)=yes - fi - _LT_TAGVAR(allow_undefined_flag, $1)=$lt_save_allow_undefined_flag - else - cat conftest.err 1>&5 - fi - $RM conftest* - ]) - _LT_TAGVAR(archive_cmds_need_lc, $1)=$lt_cv_[]_LT_TAGVAR(archive_cmds_need_lc, $1) - ;; - esac - fi - ;; -esac - -_LT_TAGDECL([build_libtool_need_lc], [archive_cmds_need_lc], [0], - [Whether or not to add -lc for building shared libraries]) -_LT_TAGDECL([allow_libtool_libs_with_static_runtimes], - [enable_shared_with_static_runtimes], [0], - [Whether or not to disallow shared libs when runtime libs are static]) -_LT_TAGDECL([], [export_dynamic_flag_spec], [1], - [Compiler flag to allow reflexive dlopens]) -_LT_TAGDECL([], [whole_archive_flag_spec], [1], - [Compiler flag to generate shared objects directly from archives]) -_LT_TAGDECL([], [compiler_needs_object], [1], - [Whether the compiler copes with passing no objects directly]) -_LT_TAGDECL([], [old_archive_from_new_cmds], [2], - [Create an old-style archive from a shared archive]) -_LT_TAGDECL([], [old_archive_from_expsyms_cmds], [2], - [Create a temporary old-style archive to link instead of a shared archive]) -_LT_TAGDECL([], [archive_cmds], [2], [Commands used to build a shared archive]) -_LT_TAGDECL([], [archive_expsym_cmds], [2]) -_LT_TAGDECL([], [module_cmds], [2], - [Commands used to build a loadable module if different from building - a shared archive.]) -_LT_TAGDECL([], [module_expsym_cmds], [2]) -_LT_TAGDECL([], [with_gnu_ld], [1], - [Whether we are building with GNU ld or not]) -_LT_TAGDECL([], [allow_undefined_flag], [1], - [Flag that allows shared libraries with undefined symbols to be built]) -_LT_TAGDECL([], [no_undefined_flag], [1], - [Flag that enforces no undefined symbols]) -_LT_TAGDECL([], [hardcode_libdir_flag_spec], [1], - [Flag to hardcode $libdir into a binary during linking. - This must work even if $libdir does not exist]) -_LT_TAGDECL([], [hardcode_libdir_separator], [1], - [Whether we need a single "-rpath" flag with a separated argument]) -_LT_TAGDECL([], [hardcode_direct], [0], - [Set to "yes" if using DIR/libNAME$shared_ext during linking hardcodes - DIR into the resulting binary]) -_LT_TAGDECL([], [hardcode_direct_absolute], [0], - [Set to "yes" if using DIR/libNAME$shared_ext during linking hardcodes - DIR into the resulting binary and the resulting library dependency is - "absolute", i.e impossible to change by setting $shlibpath_var if the - library is relocated]) -_LT_TAGDECL([], [hardcode_minus_L], [0], - [Set to "yes" if using the -LDIR flag during linking hardcodes DIR - into the resulting binary]) -_LT_TAGDECL([], [hardcode_shlibpath_var], [0], - [Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR - into the resulting binary]) -_LT_TAGDECL([], [hardcode_automatic], [0], - [Set to "yes" if building a shared library automatically hardcodes DIR - into the library and all subsequent libraries and executables linked - against it]) -_LT_TAGDECL([], [inherit_rpath], [0], - [Set to yes if linker adds runtime paths of dependent libraries - to runtime path list]) -_LT_TAGDECL([], [link_all_deplibs], [0], - [Whether libtool must link a program against all its dependency libraries]) -_LT_TAGDECL([], [always_export_symbols], [0], - [Set to "yes" if exported symbols are required]) -_LT_TAGDECL([], [export_symbols_cmds], [2], - [The commands to list exported symbols]) -_LT_TAGDECL([], [exclude_expsyms], [1], - [Symbols that should not be listed in the preloaded symbols]) -_LT_TAGDECL([], [include_expsyms], [1], - [Symbols that must always be exported]) -_LT_TAGDECL([], [prelink_cmds], [2], - [Commands necessary for linking programs (against libraries) with templates]) -_LT_TAGDECL([], [postlink_cmds], [2], - [Commands necessary for finishing linking programs]) -_LT_TAGDECL([], [file_list_spec], [1], - [Specify filename containing input files]) -dnl FIXME: Not yet implemented -dnl _LT_TAGDECL([], [thread_safe_flag_spec], [1], -dnl [Compiler flag to generate thread safe objects]) -])# _LT_LINKER_SHLIBS - - -# _LT_LANG_C_CONFIG([TAG]) -# ------------------------ -# Ensure that the configuration variables for a C compiler are suitably -# defined. These variables are subsequently used by _LT_CONFIG to write -# the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_C_CONFIG], -[m4_require([_LT_DECL_EGREP])dnl -lt_save_CC=$CC -AC_LANG_PUSH(C) - -# Source file extension for C test sources. -ac_ext=c - -# Object file extension for compiled C test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# Code to be used in simple compile tests -lt_simple_compile_test_code="int some_variable = 0;" - -# Code to be used in simple link tests -lt_simple_link_test_code='int main(){return(0);}' - -_LT_TAG_COMPILER -# Save the default compiler, since it gets overwritten when the other -# tags are being tested, and _LT_TAGVAR(compiler, []) is a NOP. -compiler_DEFAULT=$CC - -# save warnings/boilerplate of simple test code -_LT_COMPILER_BOILERPLATE -_LT_LINKER_BOILERPLATE - -if test -n "$compiler"; then - _LT_COMPILER_NO_RTTI($1) - _LT_COMPILER_PIC($1) - _LT_COMPILER_C_O($1) - _LT_COMPILER_FILE_LOCKS($1) - _LT_LINKER_SHLIBS($1) - _LT_SYS_DYNAMIC_LINKER($1) - _LT_LINKER_HARDCODE_LIBPATH($1) - LT_SYS_DLOPEN_SELF - _LT_CMD_STRIPLIB - - # Report what library types will actually be built - AC_MSG_CHECKING([if libtool supports shared libraries]) - AC_MSG_RESULT([$can_build_shared]) - - AC_MSG_CHECKING([whether to build shared libraries]) - test no = "$can_build_shared" && enable_shared=no - - # On AIX, shared libraries and static libraries use the same namespace, and - # are all built from PIC. - case $host_os in - aix3*) - test yes = "$enable_shared" && enable_static=no - if test -n "$RANLIB"; then - archive_cmds="$archive_cmds~\$RANLIB \$lib" - postinstall_cmds='$RANLIB $lib' - fi - ;; - - aix[[4-9]]*) - if test ia64 != "$host_cpu"; then - case $enable_shared,$with_aix_soname,$aix_use_runtimelinking in - yes,aix,yes) ;; # shared object as lib.so file only - yes,svr4,*) ;; # shared object as lib.so archive member only - yes,*) enable_static=no ;; # shared object in lib.a archive as well - esac - fi - ;; - esac - AC_MSG_RESULT([$enable_shared]) - - AC_MSG_CHECKING([whether to build static libraries]) - # Make sure either enable_shared or enable_static is yes. - test yes = "$enable_shared" || enable_static=yes - AC_MSG_RESULT([$enable_static]) - - _LT_CONFIG($1) -fi -AC_LANG_POP -CC=$lt_save_CC -])# _LT_LANG_C_CONFIG - - -# _LT_LANG_CXX_CONFIG([TAG]) -# -------------------------- -# Ensure that the configuration variables for a C++ compiler are suitably -# defined. These variables are subsequently used by _LT_CONFIG to write -# the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_CXX_CONFIG], -[m4_require([_LT_FILEUTILS_DEFAULTS])dnl -m4_require([_LT_DECL_EGREP])dnl -m4_require([_LT_PATH_MANIFEST_TOOL])dnl -if test -n "$CXX" && ( test no != "$CXX" && - ( (test g++ = "$CXX" && `g++ -v >/dev/null 2>&1` ) || - (test g++ != "$CXX"))); then - AC_PROG_CXXCPP -else - _lt_caught_CXX_error=yes -fi - -AC_LANG_PUSH(C++) -_LT_TAGVAR(archive_cmds_need_lc, $1)=no -_LT_TAGVAR(allow_undefined_flag, $1)= -_LT_TAGVAR(always_export_symbols, $1)=no -_LT_TAGVAR(archive_expsym_cmds, $1)= -_LT_TAGVAR(compiler_needs_object, $1)=no -_LT_TAGVAR(export_dynamic_flag_spec, $1)= -_LT_TAGVAR(hardcode_direct, $1)=no -_LT_TAGVAR(hardcode_direct_absolute, $1)=no -_LT_TAGVAR(hardcode_libdir_flag_spec, $1)= -_LT_TAGVAR(hardcode_libdir_separator, $1)= -_LT_TAGVAR(hardcode_minus_L, $1)=no -_LT_TAGVAR(hardcode_shlibpath_var, $1)=unsupported -_LT_TAGVAR(hardcode_automatic, $1)=no -_LT_TAGVAR(inherit_rpath, $1)=no -_LT_TAGVAR(module_cmds, $1)= -_LT_TAGVAR(module_expsym_cmds, $1)= -_LT_TAGVAR(link_all_deplibs, $1)=unknown -_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds -_LT_TAGVAR(reload_flag, $1)=$reload_flag -_LT_TAGVAR(reload_cmds, $1)=$reload_cmds -_LT_TAGVAR(no_undefined_flag, $1)= -_LT_TAGVAR(whole_archive_flag_spec, $1)= -_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no - -# Source file extension for C++ test sources. -ac_ext=cpp - -# Object file extension for compiled C++ test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# No sense in running all these tests if we already determined that -# the CXX compiler isn't working. Some variables (like enable_shared) -# are currently assumed to apply to all compilers on this platform, -# and will be corrupted by setting them based on a non-working compiler. -if test yes != "$_lt_caught_CXX_error"; then - # Code to be used in simple compile tests - lt_simple_compile_test_code="int some_variable = 0;" - - # Code to be used in simple link tests - lt_simple_link_test_code='int main(int, char *[[]]) { return(0); }' - - # ltmain only uses $CC for tagged configurations so make sure $CC is set. - _LT_TAG_COMPILER - - # save warnings/boilerplate of simple test code - _LT_COMPILER_BOILERPLATE - _LT_LINKER_BOILERPLATE - - # Allow CC to be a program name with arguments. - lt_save_CC=$CC - lt_save_CFLAGS=$CFLAGS - lt_save_LD=$LD - lt_save_GCC=$GCC - GCC=$GXX - lt_save_with_gnu_ld=$with_gnu_ld - lt_save_path_LD=$lt_cv_path_LD - if test -n "${lt_cv_prog_gnu_ldcxx+set}"; then - lt_cv_prog_gnu_ld=$lt_cv_prog_gnu_ldcxx - else - $as_unset lt_cv_prog_gnu_ld - fi - if test -n "${lt_cv_path_LDCXX+set}"; then - lt_cv_path_LD=$lt_cv_path_LDCXX - else - $as_unset lt_cv_path_LD - fi - test -z "${LDCXX+set}" || LD=$LDCXX - CC=${CXX-"c++"} - CFLAGS=$CXXFLAGS - compiler=$CC - _LT_TAGVAR(compiler, $1)=$CC - _LT_CC_BASENAME([$compiler]) - - if test -n "$compiler"; then - # We don't want -fno-exception when compiling C++ code, so set the - # no_builtin_flag separately - if test yes = "$GXX"; then - _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)=' -fno-builtin' - else - _LT_TAGVAR(lt_prog_compiler_no_builtin_flag, $1)= - fi - - if test yes = "$GXX"; then - # Set up default GNU C++ configuration - - LT_PATH_LD - - # Check if GNU C++ uses GNU ld as the underlying linker, since the - # archiving commands below assume that GNU ld is being used. - if test yes = "$with_gnu_ld"; then - _LT_TAGVAR(archive_cmds, $1)='$CC $pic_flag -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC $pic_flag -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-dynamic' - - # If archive_cmds runs LD, not CC, wlarc should be empty - # XXX I think wlarc can be eliminated in ltcf-cxx, but I need to - # investigate it a little bit more. (MM) - wlarc='$wl' - - # ancient GNU ld didn't support --whole-archive et. al. - if eval "`$CC -print-prog-name=ld` --help 2>&1" | - $GREP 'no-whole-archive' > /dev/null; then - _LT_TAGVAR(whole_archive_flag_spec, $1)=$wlarc'--whole-archive$convenience '$wlarc'--no-whole-archive' - else - _LT_TAGVAR(whole_archive_flag_spec, $1)= - fi - else - with_gnu_ld=no - wlarc= - - # A generic and very simple default shared library creation - # command for GNU C++ for the case where it uses the native - # linker, instead of GNU ld. If possible, this setting should - # overridden to take advantage of the native linker features on - # the platform it is being used on. - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib' - fi - - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP " \-L"' - - else - GXX=no - with_gnu_ld=no - wlarc= - fi - - # PORTME: fill in a description of your system's C++ link characteristics - AC_MSG_CHECKING([whether the $compiler linker ($LD) supports shared libraries]) - _LT_TAGVAR(ld_shlibs, $1)=yes - case $host_os in - aix3*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - aix[[4-9]]*) - if test ia64 = "$host_cpu"; then - # On IA64, the linker does run time linking by default, so we don't - # have to do anything special. - aix_use_runtimelinking=no - exp_sym_flag='-Bexport' - no_entry_flag= - else - aix_use_runtimelinking=no - - # Test if we are trying to use run time linking or normal - # AIX style linking. If -brtl is somewhere in LDFLAGS, we - # have runtime linking enabled, and use it for executables. - # For shared libraries, we enable/disable runtime linking - # depending on the kind of the shared library created - - # when "with_aix_soname,aix_use_runtimelinking" is: - # "aix,no" lib.a(lib.so.V) shared, rtl:no, for executables - # "aix,yes" lib.so shared, rtl:yes, for executables - # lib.a static archive - # "both,no" lib.so.V(shr.o) shared, rtl:yes - # lib.a(lib.so.V) shared, rtl:no, for executables - # "both,yes" lib.so.V(shr.o) shared, rtl:yes, for executables - # lib.a(lib.so.V) shared, rtl:no - # "svr4,*" lib.so.V(shr.o) shared, rtl:yes, for executables - # lib.a static archive - case $host_os in aix4.[[23]]|aix4.[[23]].*|aix[[5-9]]*) - for ld_flag in $LDFLAGS; do - case $ld_flag in - *-brtl*) - aix_use_runtimelinking=yes - break - ;; - esac - done - if test svr4,no = "$with_aix_soname,$aix_use_runtimelinking"; then - # With aix-soname=svr4, we create the lib.so.V shared archives only, - # so we don't have lib.a shared libs to link our executables. - # We have to force runtime linking in this case. - aix_use_runtimelinking=yes - LDFLAGS="$LDFLAGS -Wl,-brtl" - fi - ;; - esac - - exp_sym_flag='-bexport' - no_entry_flag='-bnoentry' - fi - - # When large executables or shared objects are built, AIX ld can - # have problems creating the table of contents. If linking a library - # or program results in "error TOC overflow" add -mminimal-toc to - # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not - # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. - - _LT_TAGVAR(archive_cmds, $1)='' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - _LT_TAGVAR(hardcode_libdir_separator, $1)=':' - _LT_TAGVAR(link_all_deplibs, $1)=yes - _LT_TAGVAR(file_list_spec, $1)='$wl-f,' - case $with_aix_soname,$aix_use_runtimelinking in - aix,*) ;; # no import file - svr4,* | *,yes) # use import file - # The Import File defines what to hardcode. - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_direct_absolute, $1)=no - ;; - esac - - if test yes = "$GXX"; then - case $host_os in aix4.[[012]]|aix4.[[012]].*) - # We only want to do this on AIX 4.2 and lower, the check - # below for broken collect2 doesn't work under 4.3+ - collect2name=`$CC -print-prog-name=collect2` - if test -f "$collect2name" && - strings "$collect2name" | $GREP resolve_lib_name >/dev/null - then - # We have reworked collect2 - : - else - # We have old collect2 - _LT_TAGVAR(hardcode_direct, $1)=unsupported - # It fails to find uninstalled libraries when the uninstalled - # path is not listed in the libpath. Setting hardcode_minus_L - # to unsupported forces relinking - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)= - fi - esac - shared_flag='-shared' - if test yes = "$aix_use_runtimelinking"; then - shared_flag=$shared_flag' $wl-G' - fi - # Need to ensure runtime linking is disabled for the traditional - # shared library, or the linker may eventually find shared libraries - # /with/ Import File - we do not want to mix them. - shared_flag_aix='-shared' - shared_flag_svr4='-shared $wl-G' - else - # not using gcc - if test ia64 = "$host_cpu"; then - # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release - # chokes on -Wl,-G. The following line is correct: - shared_flag='-G' - else - if test yes = "$aix_use_runtimelinking"; then - shared_flag='$wl-G' - else - shared_flag='$wl-bM:SRE' - fi - shared_flag_aix='$wl-bM:SRE' - shared_flag_svr4='$wl-G' - fi - fi - - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-bexpall' - # It seems that -bexpall does not export symbols beginning with - # underscore (_), so it is better to generate a list of symbols to - # export. - _LT_TAGVAR(always_export_symbols, $1)=yes - if test aix,yes = "$with_aix_soname,$aix_use_runtimelinking"; then - # Warning - without using the other runtime loading flags (-brtl), - # -berok will link without error, but may produce a broken library. - # The "-G" linker flag allows undefined symbols. - _LT_TAGVAR(no_undefined_flag, $1)='-bernotok' - # Determine the default libpath from the value encoded in an empty - # executable. - _LT_SYS_MODULE_PATH_AIX([$1]) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-blibpath:$libdir:'"$aix_libpath" - - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $deplibs $wl'$no_entry_flag' $compiler_flags `if test -n "$allow_undefined_flag"; then func_echo_all "$wl$allow_undefined_flag"; else :; fi` $wl'$exp_sym_flag:\$export_symbols' '$shared_flag - else - if test ia64 = "$host_cpu"; then - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-R $libdir:/usr/lib:/lib' - _LT_TAGVAR(allow_undefined_flag, $1)="-z nodefs" - _LT_TAGVAR(archive_expsym_cmds, $1)="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\$wl$no_entry_flag"' $compiler_flags $wl$allow_undefined_flag '"\$wl$exp_sym_flag:\$export_symbols" - else - # Determine the default libpath from the value encoded in an - # empty executable. - _LT_SYS_MODULE_PATH_AIX([$1]) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-blibpath:$libdir:'"$aix_libpath" - # Warning - without using the other run time loading flags, - # -berok will link without error, but may produce a broken library. - _LT_TAGVAR(no_undefined_flag, $1)=' $wl-bernotok' - _LT_TAGVAR(allow_undefined_flag, $1)=' $wl-berok' - if test yes = "$with_gnu_ld"; then - # We only use this code for GNU lds that support --whole-archive. - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive$convenience $wl--no-whole-archive' - else - # Exported symbols can be pulled into shared objects from archives - _LT_TAGVAR(whole_archive_flag_spec, $1)='$convenience' - fi - _LT_TAGVAR(archive_cmds_need_lc, $1)=yes - _LT_TAGVAR(archive_expsym_cmds, $1)='$RM -r $output_objdir/$realname.d~$MKDIR $output_objdir/$realname.d' - # -brtl affects multiple linker settings, -berok does not and is overridden later - compiler_flags_filtered='`func_echo_all "$compiler_flags " | $SED -e "s%-brtl\\([[, ]]\\)%-berok\\1%g"`' - if test svr4 != "$with_aix_soname"; then - # This is similar to how AIX traditionally builds its shared - # libraries. Need -bnortl late, we may have -brtl in LDFLAGS. - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_aix' -o $output_objdir/$realname.d/$soname $libobjs $deplibs $wl-bnoentry '$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$realname.d/$soname' - fi - if test aix != "$with_aix_soname"; then - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$CC '$shared_flag_svr4' -o $output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs $wl-bnoentry '$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e $output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! $soname($shared_archive_member_spec.o)"; if test shr_64 = "$shared_archive_member_spec"; then func_echo_all "# 64"; else func_echo_all "# 32"; fi; cat $export_symbols ) > $output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS $output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o $output_objdir/$realname.d/$shared_archive_member_spec.imp' - else - # used by -dlpreopen to get the symbols - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$MV $output_objdir/$realname.d/$soname $output_objdir' - fi - _LT_TAGVAR(archive_expsym_cmds, $1)="$_LT_TAGVAR(archive_expsym_cmds, $1)"'~$RM -r $output_objdir/$realname.d' - fi - fi - ;; - - beos*) - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - # Joseph Beckenbach says some releases of gcc - # support --undefined. This deserves some investigation. FIXME - _LT_TAGVAR(archive_cmds, $1)='$CC -nostart $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - chorus*) - case $cc_basename in - *) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - ;; - - cygwin* | mingw* | pw32* | cegcc*) - case $GXX,$cc_basename in - ,cl* | no,cl*) - # Native MSVC - # hardcode_libdir_flag_spec is actually meaningless, as there is - # no search path for DLLs. - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)=' ' - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - _LT_TAGVAR(always_export_symbols, $1)=yes - _LT_TAGVAR(file_list_spec, $1)='@' - # Tell ltmain to make .lib files, not .a files. - libext=lib - # Tell ltmain to make .dll files, not .so files. - shrext_cmds=.dll - # FIXME: Setting linknames here is a bad hack. - _LT_TAGVAR(archive_cmds, $1)='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~linknames=' - _LT_TAGVAR(archive_expsym_cmds, $1)='if _LT_DLL_DEF_P([$export_symbols]); then - cp "$export_symbols" "$output_objdir/$soname.def"; - echo "$tool_output_objdir$soname.def" > "$output_objdir/$soname.exp"; - else - $SED -e '\''s/^/-link -EXPORT:/'\'' < $export_symbols > $output_objdir/$soname.exp; - fi~ - $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ - linknames=' - # The linker will not automatically build a static lib if we build a DLL. - # _LT_TAGVAR(old_archive_from_new_cmds, $1)='true' - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - # Don't use ranlib - _LT_TAGVAR(old_postinstall_cmds, $1)='chmod 644 $oldlib' - _LT_TAGVAR(postlink_cmds, $1)='lt_outputfile="@OUTPUT@"~ - lt_tool_outputfile="@TOOL_OUTPUT@"~ - case $lt_outputfile in - *.exe|*.EXE) ;; - *) - lt_outputfile=$lt_outputfile.exe - lt_tool_outputfile=$lt_tool_outputfile.exe - ;; - esac~ - func_to_tool_file "$lt_outputfile"~ - if test : != "$MANIFEST_TOOL" && test -f "$lt_outputfile.manifest"; then - $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; - $RM "$lt_outputfile.manifest"; - fi' - ;; - *) - # g++ - # _LT_TAGVAR(hardcode_libdir_flag_spec, $1) is actually meaningless, - # as there is no search path for DLLs. - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-all-symbols' - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - _LT_TAGVAR(always_export_symbols, $1)=no - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - - if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname $wl--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' - # If the export-symbols file already is a .def file, use it as - # is; otherwise, prepend EXPORTS... - _LT_TAGVAR(archive_expsym_cmds, $1)='if _LT_DLL_DEF_P([$export_symbols]); then - cp $export_symbols $output_objdir/$soname.def; - else - echo EXPORTS > $output_objdir/$soname.def; - cat $export_symbols >> $output_objdir/$soname.def; - fi~ - $CC -shared -nostdlib $output_objdir/$soname.def $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $output_objdir/$soname $wl--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - esac - ;; - darwin* | rhapsody*) - _LT_DARWIN_LINKER_FEATURES($1) - ;; - - os2*) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-L$libdir' - _LT_TAGVAR(hardcode_minus_L, $1)=yes - _LT_TAGVAR(allow_undefined_flag, $1)=unsupported - shrext_cmds=.dll - _LT_TAGVAR(archive_cmds, $1)='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - emxexp $libobjs | $SED /"_DLL_InitTerm"/d >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - _LT_TAGVAR(archive_expsym_cmds, $1)='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - prefix_cmds="$SED"~ - if test EXPORTS = "`$SED 1q $export_symbols`"; then - prefix_cmds="$prefix_cmds -e 1d"; - fi~ - prefix_cmds="$prefix_cmds -e \"s/^\(.*\)$/_\1/g\""~ - cat $export_symbols | $prefix_cmds >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - _LT_TAGVAR(old_archive_From_new_cmds, $1)='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' - _LT_TAGVAR(enable_shared_with_static_runtimes, $1)=yes - ;; - - dgux*) - case $cc_basename in - ec++*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - ghcx*) - # Green Hills C++ Compiler - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - *) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - ;; - - freebsd2.*) - # C++ shared libraries reported to be fairly broken before - # switch to ELF - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - - freebsd-elf*) - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - ;; - - freebsd* | dragonfly*) - # FreeBSD 3 and later use GNU C++ and GNU ld with standard ELF - # conventions - _LT_TAGVAR(ld_shlibs, $1)=yes - ;; - - haiku*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(link_all_deplibs, $1)=yes - ;; - - hpux9*) - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl+b $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_minus_L, $1)=yes # Not in the search PATH, - # but as the default - # location of the library. - - case $cc_basename in - CC*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - aCC*) - _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -b $wl+b $wl$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test "x$output_objdir/$soname" = "x$lib" || mv $output_objdir/$soname $lib' - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - # - # There doesn't appear to be a way to prevent this compiler from - # explicitly linking system object files so we need to strip them - # from the output so that they don't get included in the library - # dependencies. - output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $EGREP " \-L"`; list= ; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' - ;; - *) - if test yes = "$GXX"; then - _LT_TAGVAR(archive_cmds, $1)='$RM $output_objdir/$soname~$CC -shared -nostdlib $pic_flag $wl+b $wl$install_libdir -o $output_objdir/$soname $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~test "x$output_objdir/$soname" = "x$lib" || mv $output_objdir/$soname $lib' - else - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - esac - ;; - - hpux10*|hpux11*) - if test no = "$with_gnu_ld"; then - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl+b $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - - case $host_cpu in - hppa*64*|ia64*) - ;; - *) - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - ;; - esac - fi - case $host_cpu in - hppa*64*|ia64*) - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - ;; - *) - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - _LT_TAGVAR(hardcode_minus_L, $1)=yes # Not in the search PATH, - # but as the default - # location of the library. - ;; - esac - - case $cc_basename in - CC*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - aCC*) - case $host_cpu in - hppa*64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - ;; - ia64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname $wl+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - ;; - *) - _LT_TAGVAR(archive_cmds, $1)='$CC -b $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - ;; - esac - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - # - # There doesn't appear to be a way to prevent this compiler from - # explicitly linking system object files so we need to strip them - # from the output so that they don't get included in the library - # dependencies. - output_verbose_link_cmd='templist=`($CC -b $CFLAGS -v conftest.$objext 2>&1) | $GREP " \-L"`; list= ; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' - ;; - *) - if test yes = "$GXX"; then - if test no = "$with_gnu_ld"; then - case $host_cpu in - hppa*64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib -fPIC $wl+h $wl$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - ;; - ia64*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $pic_flag $wl+h $wl$soname $wl+nodefaultrpath -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - ;; - *) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $pic_flag $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - ;; - esac - fi - else - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - esac - ;; - - interix[[3-9]]*) - _LT_TAGVAR(hardcode_direct, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. - # Instead, shared libraries are loaded at an image base (0x10000000 by - # default) and relocated if they conflict, which is a slow very memory - # consuming and fragmenting process. To avoid this, we pick a random, - # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link - # time. Moving up from 0x10000000 also allows more sbrk(2) space. - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - ;; - irix5* | irix6*) - case $cc_basename in - CC*) - # SGI C++ - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -all -multigot $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - - # Archives containing C++ object files must be created using - # "CC -ar", where "CC" is the IRIX C++ compiler. This is - # necessary to make sure instantiated templates are included - # in the archive. - _LT_TAGVAR(old_archive_cmds, $1)='$CC -ar -WR,-u -o $oldlib $oldobjs' - ;; - *) - if test yes = "$GXX"; then - if test no = "$with_gnu_ld"; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - else - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` -o $lib' - fi - fi - _LT_TAGVAR(link_all_deplibs, $1)=yes - ;; - esac - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - _LT_TAGVAR(inherit_rpath, $1)=yes - ;; - - linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - case $cc_basename in - KCC*) - # Kuck and Associates, Inc. (KAI) C++ Compiler - - # KCC will only create a shared library if the output file - # ends with ".so" (or ".sl" for HP-UX), so rename the library - # to its proper name (with version) after linking. - _LT_TAGVAR(archive_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\$tempext\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo $lib | $SED -e "s/\$tempext\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib $wl-retain-symbols-file,$export_symbols; mv \$templib $lib' - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - # - # There doesn't appear to be a way to prevent this compiler from - # explicitly linking system object files so we need to strip them - # from the output so that they don't get included in the library - # dependencies. - output_verbose_link_cmd='templist=`$CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 | $GREP "ld"`; rm -f libconftest$shared_ext; list= ; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-dynamic' - - # Archives containing C++ object files must be created using - # "CC -Bstatic", where "CC" is the KAI C++ compiler. - _LT_TAGVAR(old_archive_cmds, $1)='$CC -Bstatic -o $oldlib $oldobjs' - ;; - icpc* | ecpc* ) - # Intel C++ - with_gnu_ld=yes - # version 8.0 and above of icpc choke on multiply defined symbols - # if we add $predep_objects and $postdep_objects, however 7.1 and - # earlier do not add the objects themselves. - case `$CC -V 2>&1` in - *"Version 7."*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - ;; - *) # Version 8.0 or newer - tmp_idyn= - case $host_cpu in - ia64*) tmp_idyn=' -i_dynamic';; - esac - _LT_TAGVAR(archive_cmds, $1)='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared'"$tmp_idyn"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - ;; - esac - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-dynamic' - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive$convenience $wl--no-whole-archive' - ;; - pgCC* | pgcpp*) - # Portland Group C++ compiler - case `$CC -V` in - *pgCC\ [[1-5]].* | *pgcpp\ [[1-5]].*) - _LT_TAGVAR(prelink_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $objs $libobjs $compile_deplibs~ - compile_command="$compile_command `find $tpldir -name \*.o | sort | $NL2SP`"' - _LT_TAGVAR(old_archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $oldobjs$old_deplibs~ - $AR $AR_FLAGS $oldlib$oldobjs$old_deplibs `find $tpldir -name \*.o | sort | $NL2SP`~ - $RANLIB $oldlib' - _LT_TAGVAR(archive_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ - $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='tpldir=Template.dir~ - rm -rf $tpldir~ - $CC --prelink_objects --instantiation_dir $tpldir $predep_objects $libobjs $deplibs $convenience $postdep_objects~ - $CC -shared $pic_flag $predep_objects $libobjs $deplibs `find $tpldir -name \*.o | sort | $NL2SP` $postdep_objects $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - ;; - *) # Version 6 and above use weak symbols - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - ;; - esac - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl--rpath $wl$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-dynamic' - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - ;; - cxx*) - # Compaq C++ - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname -o $lib $wl-retain-symbols-file $wl$export_symbols' - - runpath_var=LD_RUN_PATH - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - # - # There doesn't appear to be a way to prevent this compiler from - # explicitly linking system object files so we need to strip them - # from the output so that they don't get included in the library - # dependencies. - output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld"`; templist=`func_echo_all "$templist" | $SED "s/\(^.*ld.*\)\( .*ld .*$\)/\1/"`; list= ; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "X$list" | $Xsed' - ;; - xl* | mpixl* | bgxl*) - # IBM XL 8.0 on PPC, with GNU ld - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl--export-dynamic' - _LT_TAGVAR(archive_cmds, $1)='$CC -qmkshrobj $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - if test yes = "$supports_anon_versioning"; then - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ - echo "local: *; };" >> $output_objdir/$libname.ver~ - $CC -qmkshrobj $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' - fi - ;; - *) - case `$CC -V 2>&1 | sed 5q` in - *Sun\ C*) - # Sun C++ 5.9 - _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs' - _LT_TAGVAR(archive_cmds, $1)='$CC -G$allow_undefined_flag -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G$allow_undefined_flag -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-retain-symbols-file $wl$export_symbols' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - _LT_TAGVAR(compiler_needs_object, $1)=yes - - # Not sure whether something based on - # $CC $CFLAGS -v conftest.$objext -o libconftest$shared_ext 2>&1 - # would be better. - output_verbose_link_cmd='func_echo_all' - - # Archives containing C++ object files must be created using - # "CC -xar", where "CC" is the Sun C++ compiler. This is - # necessary to make sure instantiated templates are included - # in the archive. - _LT_TAGVAR(old_archive_cmds, $1)='$CC -xar -o $oldlib $oldobjs' - ;; - esac - ;; - esac - ;; - - lynxos*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - - m88k*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - - mvs*) - case $cc_basename in - cxx*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - *) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - ;; - - netbsd*) - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$LD -Bshareable -o $lib $predep_objects $libobjs $deplibs $postdep_objects $linker_flags' - wlarc= - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - fi - # Workaround some broken pre-1.5 toolchains - output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP conftest.$objext | $SED -e "s:-lgcc -lc -lgcc::"' - ;; - - *nto* | *qnx*) - _LT_TAGVAR(ld_shlibs, $1)=yes - ;; - - openbsd* | bitrig*) - if test -f /usr/libexec/ld.so; then - _LT_TAGVAR(hardcode_direct, $1)=yes - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(hardcode_direct_absolute, $1)=yes - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -o $lib' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - if test -z "`echo __ELF__ | $CC -E - | grep __ELF__`"; then - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $pic_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-retain-symbols-file,$export_symbols -o $lib' - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-E' - _LT_TAGVAR(whole_archive_flag_spec, $1)=$wlarc'--whole-archive$convenience '$wlarc'--no-whole-archive' - fi - output_verbose_link_cmd=func_echo_all - else - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - - osf3* | osf4* | osf5*) - case $cc_basename in - KCC*) - # Kuck and Associates, Inc. (KAI) C++ Compiler - - # KCC will only create a shared library if the output file - # ends with ".so" (or ".sl" for HP-UX), so rename the library - # to its proper name (with version) after linking. - _LT_TAGVAR(archive_cmds, $1)='tempext=`echo $shared_ext | $SED -e '\''s/\([[^()0-9A-Za-z{}]]\)/\\\\\1/g'\''`; templib=`echo "$lib" | $SED -e "s/\$tempext\..*/.so/"`; $CC $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags --soname $soname -o \$templib; mv \$templib $lib' - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath,$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - - # Archives containing C++ object files must be created using - # the KAI C++ compiler. - case $host in - osf3*) _LT_TAGVAR(old_archive_cmds, $1)='$CC -Bstatic -o $oldlib $oldobjs' ;; - *) _LT_TAGVAR(old_archive_cmds, $1)='$CC -o $oldlib $oldobjs' ;; - esac - ;; - RCC*) - # Rational C++ 2.4.1 - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - cxx*) - case $host in - osf3*) - _LT_TAGVAR(allow_undefined_flag, $1)=' $wl-expect_unresolved $wl\*' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared$allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $soname `test -n "$verstring" && func_echo_all "$wl-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - ;; - *) - _LT_TAGVAR(allow_undefined_flag, $1)=' -expect_unresolved \*' - _LT_TAGVAR(archive_cmds, $1)='$CC -shared$allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done~ - echo "-hidden">> $lib.exp~ - $CC -shared$allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags -msym -soname $soname $wl-input $wl$lib.exp `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib~ - $RM $lib.exp' - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-rpath $libdir' - ;; - esac - - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - # - # There doesn't appear to be a way to prevent this compiler from - # explicitly linking system object files so we need to strip them - # from the output so that they don't get included in the library - # dependencies. - output_verbose_link_cmd='templist=`$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP "ld" | $GREP -v "ld:"`; templist=`func_echo_all "$templist" | $SED "s/\(^.*ld.*\)\( .*ld.*$\)/\1/"`; list= ; for z in $templist; do case $z in conftest.$objext) list="$list $z";; *.$objext);; *) list="$list $z";;esac; done; func_echo_all "$list"' - ;; - *) - if test yes,no = "$GXX,$with_gnu_ld"; then - _LT_TAGVAR(allow_undefined_flag, $1)=' $wl-expect_unresolved $wl\*' - case $host in - osf3*) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared -nostdlib $allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - ;; - *) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $allow_undefined_flag $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-msym $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - ;; - esac - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-rpath $wl$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=: - - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP " \-L"' - - else - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - fi - ;; - esac - ;; - - psos*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - - sunos4*) - case $cc_basename in - CC*) - # Sun C++ 4.x - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - lcc*) - # Lucid - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - *) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - ;; - - solaris*) - case $cc_basename in - CC* | sunCC*) - # Sun C++ 4.2, 5.x and Centerline C++ - _LT_TAGVAR(archive_cmds_need_lc,$1)=yes - _LT_TAGVAR(no_undefined_flag, $1)=' -zdefs' - _LT_TAGVAR(archive_cmds, $1)='$CC -G$allow_undefined_flag -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -G$allow_undefined_flag $wl-M $wl$lib.exp -h$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='-R$libdir' - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - case $host_os in - solaris2.[[0-5]] | solaris2.[[0-5]].*) ;; - *) - # The compiler driver will combine and reorder linker options, - # but understands '-z linker_flag'. - # Supported since Solaris 2.6 (maybe 2.5.1?) - _LT_TAGVAR(whole_archive_flag_spec, $1)='-z allextract$convenience -z defaultextract' - ;; - esac - _LT_TAGVAR(link_all_deplibs, $1)=yes - - output_verbose_link_cmd='func_echo_all' - - # Archives containing C++ object files must be created using - # "CC -xar", where "CC" is the Sun C++ compiler. This is - # necessary to make sure instantiated templates are included - # in the archive. - _LT_TAGVAR(old_archive_cmds, $1)='$CC -xar -o $oldlib $oldobjs' - ;; - gcx*) - # Green Hills C++ Compiler - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-h $wl$soname -o $lib' - - # The C++ compiler must be used to create the archive. - _LT_TAGVAR(old_archive_cmds, $1)='$CC $LDFLAGS -archive -o $oldlib $oldobjs' - ;; - *) - # GNU C++ compiler with Solaris linker - if test yes,no = "$GXX,$with_gnu_ld"; then - _LT_TAGVAR(no_undefined_flag, $1)=' $wl-z ${wl}defs' - if $CC --version | $GREP -v '^2\.7' > /dev/null; then - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $pic_flag -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-h $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -shared $pic_flag -nostdlib $wl-M $wl$lib.exp $wl-h $wl$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' - - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - output_verbose_link_cmd='$CC -shared $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP " \-L"' - else - # g++ 2.7 appears to require '-G' NOT '-shared' on this - # platform. - _LT_TAGVAR(archive_cmds, $1)='$CC -G -nostdlib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags $wl-h $wl$soname -o $lib' - _LT_TAGVAR(archive_expsym_cmds, $1)='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -G -nostdlib $wl-M $wl$lib.exp $wl-h $wl$soname -o $lib $predep_objects $libobjs $deplibs $postdep_objects $compiler_flags~$RM $lib.exp' - - # Commands to make compiler produce verbose output that lists - # what "hidden" libraries, object files and flags are used when - # linking a shared library. - output_verbose_link_cmd='$CC -G $CFLAGS -v conftest.$objext 2>&1 | $GREP -v "^Configured with:" | $GREP " \-L"' - fi - - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-R $wl$libdir' - case $host_os in - solaris2.[[0-5]] | solaris2.[[0-5]].*) ;; - *) - _LT_TAGVAR(whole_archive_flag_spec, $1)='$wl-z ${wl}allextract$convenience $wl-z ${wl}defaultextract' - ;; - esac - fi - ;; - esac - ;; - - sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[[01]].[[10]]* | unixware7* | sco3.2v5.0.[[024]]*) - _LT_TAGVAR(no_undefined_flag, $1)='$wl-z,text' - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - runpath_var='LD_RUN_PATH' - - case $cc_basename in - CC*) - _LT_TAGVAR(archive_cmds, $1)='$CC -G $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - *) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - esac - ;; - - sysv5* | sco3.2v5* | sco5v6*) - # Note: We CANNOT use -z defs as we might desire, because we do not - # link with -lc, and that would cause any symbols used from libc to - # always be unresolved, which means just about no library would - # ever link correctly. If we're not using GNU ld we use -z text - # though, which does catch some bad symbols but isn't as heavy-handed - # as -z defs. - _LT_TAGVAR(no_undefined_flag, $1)='$wl-z,text' - _LT_TAGVAR(allow_undefined_flag, $1)='$wl-z,nodefs' - _LT_TAGVAR(archive_cmds_need_lc, $1)=no - _LT_TAGVAR(hardcode_shlibpath_var, $1)=no - _LT_TAGVAR(hardcode_libdir_flag_spec, $1)='$wl-R,$libdir' - _LT_TAGVAR(hardcode_libdir_separator, $1)=':' - _LT_TAGVAR(link_all_deplibs, $1)=yes - _LT_TAGVAR(export_dynamic_flag_spec, $1)='$wl-Bexport' - runpath_var='LD_RUN_PATH' - - case $cc_basename in - CC*) - _LT_TAGVAR(archive_cmds, $1)='$CC -G $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -G $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(old_archive_cmds, $1)='$CC -Tprelink_objects $oldobjs~ - '"$_LT_TAGVAR(old_archive_cmds, $1)" - _LT_TAGVAR(reload_cmds, $1)='$CC -Tprelink_objects $reload_objs~ - '"$_LT_TAGVAR(reload_cmds, $1)" - ;; - *) - _LT_TAGVAR(archive_cmds, $1)='$CC -shared $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - _LT_TAGVAR(archive_expsym_cmds, $1)='$CC -shared $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - esac - ;; - - tandem*) - case $cc_basename in - NCC*) - # NonStop-UX NCC 3.20 - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - *) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - ;; - - vxworks*) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - - *) - # FIXME: insert proper C++ library support - _LT_TAGVAR(ld_shlibs, $1)=no - ;; - esac - - AC_MSG_RESULT([$_LT_TAGVAR(ld_shlibs, $1)]) - test no = "$_LT_TAGVAR(ld_shlibs, $1)" && can_build_shared=no - - _LT_TAGVAR(GCC, $1)=$GXX - _LT_TAGVAR(LD, $1)=$LD - - ## CAVEAT EMPTOR: - ## There is no encapsulation within the following macros, do not change - ## the running order or otherwise move them around unless you know exactly - ## what you are doing... - _LT_SYS_HIDDEN_LIBDEPS($1) - _LT_COMPILER_PIC($1) - _LT_COMPILER_C_O($1) - _LT_COMPILER_FILE_LOCKS($1) - _LT_LINKER_SHLIBS($1) - _LT_SYS_DYNAMIC_LINKER($1) - _LT_LINKER_HARDCODE_LIBPATH($1) - - _LT_CONFIG($1) - fi # test -n "$compiler" - - CC=$lt_save_CC - CFLAGS=$lt_save_CFLAGS - LDCXX=$LD - LD=$lt_save_LD - GCC=$lt_save_GCC - with_gnu_ld=$lt_save_with_gnu_ld - lt_cv_path_LDCXX=$lt_cv_path_LD - lt_cv_path_LD=$lt_save_path_LD - lt_cv_prog_gnu_ldcxx=$lt_cv_prog_gnu_ld - lt_cv_prog_gnu_ld=$lt_save_with_gnu_ld -fi # test yes != "$_lt_caught_CXX_error" - -AC_LANG_POP -])# _LT_LANG_CXX_CONFIG - - -# _LT_FUNC_STRIPNAME_CNF -# ---------------------- -# func_stripname_cnf prefix suffix name -# strip PREFIX and SUFFIX off of NAME. -# PREFIX and SUFFIX must not contain globbing or regex special -# characters, hashes, percent signs, but SUFFIX may contain a leading -# dot (in which case that matches only a dot). -# -# This function is identical to the (non-XSI) version of func_stripname, -# except this one can be used by m4 code that may be executed by configure, -# rather than the libtool script. -m4_defun([_LT_FUNC_STRIPNAME_CNF],[dnl -AC_REQUIRE([_LT_DECL_SED]) -AC_REQUIRE([_LT_PROG_ECHO_BACKSLASH]) -func_stripname_cnf () -{ - case @S|@2 in - .*) func_stripname_result=`$ECHO "@S|@3" | $SED "s%^@S|@1%%; s%\\\\@S|@2\$%%"`;; - *) func_stripname_result=`$ECHO "@S|@3" | $SED "s%^@S|@1%%; s%@S|@2\$%%"`;; - esac -} # func_stripname_cnf -])# _LT_FUNC_STRIPNAME_CNF - - -# _LT_SYS_HIDDEN_LIBDEPS([TAGNAME]) -# --------------------------------- -# Figure out "hidden" library dependencies from verbose -# compiler output when linking a shared library. -# Parse the compiler output and extract the necessary -# objects, libraries and library flags. -m4_defun([_LT_SYS_HIDDEN_LIBDEPS], -[m4_require([_LT_FILEUTILS_DEFAULTS])dnl -AC_REQUIRE([_LT_FUNC_STRIPNAME_CNF])dnl -# Dependencies to place before and after the object being linked: -_LT_TAGVAR(predep_objects, $1)= -_LT_TAGVAR(postdep_objects, $1)= -_LT_TAGVAR(predeps, $1)= -_LT_TAGVAR(postdeps, $1)= -_LT_TAGVAR(compiler_lib_search_path, $1)= - -dnl we can't use the lt_simple_compile_test_code here, -dnl because it contains code intended for an executable, -dnl not a library. It's possible we should let each -dnl tag define a new lt_????_link_test_code variable, -dnl but it's only used here... -m4_if([$1], [], [cat > conftest.$ac_ext <<_LT_EOF -int a; -void foo (void) { a = 0; } -_LT_EOF -], [$1], [CXX], [cat > conftest.$ac_ext <<_LT_EOF -class Foo -{ -public: - Foo (void) { a = 0; } -private: - int a; -}; -_LT_EOF -], [$1], [F77], [cat > conftest.$ac_ext <<_LT_EOF - subroutine foo - implicit none - integer*4 a - a=0 - return - end -_LT_EOF -], [$1], [FC], [cat > conftest.$ac_ext <<_LT_EOF - subroutine foo - implicit none - integer a - a=0 - return - end -_LT_EOF -], [$1], [GCJ], [cat > conftest.$ac_ext <<_LT_EOF -public class foo { - private int a; - public void bar (void) { - a = 0; - } -}; -_LT_EOF -], [$1], [GO], [cat > conftest.$ac_ext <<_LT_EOF -package foo -func foo() { -} -_LT_EOF -]) - -_lt_libdeps_save_CFLAGS=$CFLAGS -case "$CC $CFLAGS " in #( -*\ -flto*\ *) CFLAGS="$CFLAGS -fno-lto" ;; -*\ -fwhopr*\ *) CFLAGS="$CFLAGS -fno-whopr" ;; -*\ -fuse-linker-plugin*\ *) CFLAGS="$CFLAGS -fno-use-linker-plugin" ;; -esac - -dnl Parse the compiler output and extract the necessary -dnl objects, libraries and library flags. -if AC_TRY_EVAL(ac_compile); then - # Parse the compiler output and extract the necessary - # objects, libraries and library flags. - - # Sentinel used to keep track of whether or not we are before - # the conftest object file. - pre_test_object_deps_done=no - - for p in `eval "$output_verbose_link_cmd"`; do - case $prev$p in - - -L* | -R* | -l*) - # Some compilers place space between "-{L,R}" and the path. - # Remove the space. - if test x-L = "$p" || - test x-R = "$p"; then - prev=$p - continue - fi - - # Expand the sysroot to ease extracting the directories later. - if test -z "$prev"; then - case $p in - -L*) func_stripname_cnf '-L' '' "$p"; prev=-L; p=$func_stripname_result ;; - -R*) func_stripname_cnf '-R' '' "$p"; prev=-R; p=$func_stripname_result ;; - -l*) func_stripname_cnf '-l' '' "$p"; prev=-l; p=$func_stripname_result ;; - esac - fi - case $p in - =*) func_stripname_cnf '=' '' "$p"; p=$lt_sysroot$func_stripname_result ;; - esac - if test no = "$pre_test_object_deps_done"; then - case $prev in - -L | -R) - # Internal compiler library paths should come after those - # provided the user. The postdeps already come after the - # user supplied libs so there is no need to process them. - if test -z "$_LT_TAGVAR(compiler_lib_search_path, $1)"; then - _LT_TAGVAR(compiler_lib_search_path, $1)=$prev$p - else - _LT_TAGVAR(compiler_lib_search_path, $1)="${_LT_TAGVAR(compiler_lib_search_path, $1)} $prev$p" - fi - ;; - # The "-l" case would never come before the object being - # linked, so don't bother handling this case. - esac - else - if test -z "$_LT_TAGVAR(postdeps, $1)"; then - _LT_TAGVAR(postdeps, $1)=$prev$p - else - _LT_TAGVAR(postdeps, $1)="${_LT_TAGVAR(postdeps, $1)} $prev$p" - fi - fi - prev= - ;; - - *.lto.$objext) ;; # Ignore GCC LTO objects - *.$objext) - # This assumes that the test object file only shows up - # once in the compiler output. - if test "$p" = "conftest.$objext"; then - pre_test_object_deps_done=yes - continue - fi - - if test no = "$pre_test_object_deps_done"; then - if test -z "$_LT_TAGVAR(predep_objects, $1)"; then - _LT_TAGVAR(predep_objects, $1)=$p - else - _LT_TAGVAR(predep_objects, $1)="$_LT_TAGVAR(predep_objects, $1) $p" - fi - else - if test -z "$_LT_TAGVAR(postdep_objects, $1)"; then - _LT_TAGVAR(postdep_objects, $1)=$p - else - _LT_TAGVAR(postdep_objects, $1)="$_LT_TAGVAR(postdep_objects, $1) $p" - fi - fi - ;; - - *) ;; # Ignore the rest. - - esac - done - - # Clean up. - rm -f a.out a.exe -else - echo "libtool.m4: error: problem compiling $1 test program" -fi - -$RM -f confest.$objext -CFLAGS=$_lt_libdeps_save_CFLAGS - -# PORTME: override above test on systems where it is broken -m4_if([$1], [CXX], -[case $host_os in -interix[[3-9]]*) - # Interix 3.5 installs completely hosed .la files for C++, so rather than - # hack all around it, let's just trust "g++" to DTRT. - _LT_TAGVAR(predep_objects,$1)= - _LT_TAGVAR(postdep_objects,$1)= - _LT_TAGVAR(postdeps,$1)= - ;; -esac -]) - -case " $_LT_TAGVAR(postdeps, $1) " in -*" -lc "*) _LT_TAGVAR(archive_cmds_need_lc, $1)=no ;; -esac - _LT_TAGVAR(compiler_lib_search_dirs, $1)= -if test -n "${_LT_TAGVAR(compiler_lib_search_path, $1)}"; then - _LT_TAGVAR(compiler_lib_search_dirs, $1)=`echo " ${_LT_TAGVAR(compiler_lib_search_path, $1)}" | $SED -e 's! -L! !g' -e 's!^ !!'` -fi -_LT_TAGDECL([], [compiler_lib_search_dirs], [1], - [The directories searched by this compiler when creating a shared library]) -_LT_TAGDECL([], [predep_objects], [1], - [Dependencies to place before and after the objects being linked to - create a shared library]) -_LT_TAGDECL([], [postdep_objects], [1]) -_LT_TAGDECL([], [predeps], [1]) -_LT_TAGDECL([], [postdeps], [1]) -_LT_TAGDECL([], [compiler_lib_search_path], [1], - [The library search path used internally by the compiler when linking - a shared library]) -])# _LT_SYS_HIDDEN_LIBDEPS - - -# _LT_LANG_F77_CONFIG([TAG]) -# -------------------------- -# Ensure that the configuration variables for a Fortran 77 compiler are -# suitably defined. These variables are subsequently used by _LT_CONFIG -# to write the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_F77_CONFIG], -[AC_LANG_PUSH(Fortran 77) -if test -z "$F77" || test no = "$F77"; then - _lt_disable_F77=yes -fi - -_LT_TAGVAR(archive_cmds_need_lc, $1)=no -_LT_TAGVAR(allow_undefined_flag, $1)= -_LT_TAGVAR(always_export_symbols, $1)=no -_LT_TAGVAR(archive_expsym_cmds, $1)= -_LT_TAGVAR(export_dynamic_flag_spec, $1)= -_LT_TAGVAR(hardcode_direct, $1)=no -_LT_TAGVAR(hardcode_direct_absolute, $1)=no -_LT_TAGVAR(hardcode_libdir_flag_spec, $1)= -_LT_TAGVAR(hardcode_libdir_separator, $1)= -_LT_TAGVAR(hardcode_minus_L, $1)=no -_LT_TAGVAR(hardcode_automatic, $1)=no -_LT_TAGVAR(inherit_rpath, $1)=no -_LT_TAGVAR(module_cmds, $1)= -_LT_TAGVAR(module_expsym_cmds, $1)= -_LT_TAGVAR(link_all_deplibs, $1)=unknown -_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds -_LT_TAGVAR(reload_flag, $1)=$reload_flag -_LT_TAGVAR(reload_cmds, $1)=$reload_cmds -_LT_TAGVAR(no_undefined_flag, $1)= -_LT_TAGVAR(whole_archive_flag_spec, $1)= -_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no - -# Source file extension for f77 test sources. -ac_ext=f - -# Object file extension for compiled f77 test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# No sense in running all these tests if we already determined that -# the F77 compiler isn't working. Some variables (like enable_shared) -# are currently assumed to apply to all compilers on this platform, -# and will be corrupted by setting them based on a non-working compiler. -if test yes != "$_lt_disable_F77"; then - # Code to be used in simple compile tests - lt_simple_compile_test_code="\ - subroutine t - return - end -" - - # Code to be used in simple link tests - lt_simple_link_test_code="\ - program t - end -" - - # ltmain only uses $CC for tagged configurations so make sure $CC is set. - _LT_TAG_COMPILER - - # save warnings/boilerplate of simple test code - _LT_COMPILER_BOILERPLATE - _LT_LINKER_BOILERPLATE - - # Allow CC to be a program name with arguments. - lt_save_CC=$CC - lt_save_GCC=$GCC - lt_save_CFLAGS=$CFLAGS - CC=${F77-"f77"} - CFLAGS=$FFLAGS - compiler=$CC - _LT_TAGVAR(compiler, $1)=$CC - _LT_CC_BASENAME([$compiler]) - GCC=$G77 - if test -n "$compiler"; then - AC_MSG_CHECKING([if libtool supports shared libraries]) - AC_MSG_RESULT([$can_build_shared]) - - AC_MSG_CHECKING([whether to build shared libraries]) - test no = "$can_build_shared" && enable_shared=no - - # On AIX, shared libraries and static libraries use the same namespace, and - # are all built from PIC. - case $host_os in - aix3*) - test yes = "$enable_shared" && enable_static=no - if test -n "$RANLIB"; then - archive_cmds="$archive_cmds~\$RANLIB \$lib" - postinstall_cmds='$RANLIB $lib' - fi - ;; - aix[[4-9]]*) - if test ia64 != "$host_cpu"; then - case $enable_shared,$with_aix_soname,$aix_use_runtimelinking in - yes,aix,yes) ;; # shared object as lib.so file only - yes,svr4,*) ;; # shared object as lib.so archive member only - yes,*) enable_static=no ;; # shared object in lib.a archive as well - esac - fi - ;; - esac - AC_MSG_RESULT([$enable_shared]) - - AC_MSG_CHECKING([whether to build static libraries]) - # Make sure either enable_shared or enable_static is yes. - test yes = "$enable_shared" || enable_static=yes - AC_MSG_RESULT([$enable_static]) - - _LT_TAGVAR(GCC, $1)=$G77 - _LT_TAGVAR(LD, $1)=$LD - - ## CAVEAT EMPTOR: - ## There is no encapsulation within the following macros, do not change - ## the running order or otherwise move them around unless you know exactly - ## what you are doing... - _LT_COMPILER_PIC($1) - _LT_COMPILER_C_O($1) - _LT_COMPILER_FILE_LOCKS($1) - _LT_LINKER_SHLIBS($1) - _LT_SYS_DYNAMIC_LINKER($1) - _LT_LINKER_HARDCODE_LIBPATH($1) - - _LT_CONFIG($1) - fi # test -n "$compiler" - - GCC=$lt_save_GCC - CC=$lt_save_CC - CFLAGS=$lt_save_CFLAGS -fi # test yes != "$_lt_disable_F77" - -AC_LANG_POP -])# _LT_LANG_F77_CONFIG - - -# _LT_LANG_FC_CONFIG([TAG]) -# ------------------------- -# Ensure that the configuration variables for a Fortran compiler are -# suitably defined. These variables are subsequently used by _LT_CONFIG -# to write the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_FC_CONFIG], -[AC_LANG_PUSH(Fortran) - -if test -z "$FC" || test no = "$FC"; then - _lt_disable_FC=yes -fi - -_LT_TAGVAR(archive_cmds_need_lc, $1)=no -_LT_TAGVAR(allow_undefined_flag, $1)= -_LT_TAGVAR(always_export_symbols, $1)=no -_LT_TAGVAR(archive_expsym_cmds, $1)= -_LT_TAGVAR(export_dynamic_flag_spec, $1)= -_LT_TAGVAR(hardcode_direct, $1)=no -_LT_TAGVAR(hardcode_direct_absolute, $1)=no -_LT_TAGVAR(hardcode_libdir_flag_spec, $1)= -_LT_TAGVAR(hardcode_libdir_separator, $1)= -_LT_TAGVAR(hardcode_minus_L, $1)=no -_LT_TAGVAR(hardcode_automatic, $1)=no -_LT_TAGVAR(inherit_rpath, $1)=no -_LT_TAGVAR(module_cmds, $1)= -_LT_TAGVAR(module_expsym_cmds, $1)= -_LT_TAGVAR(link_all_deplibs, $1)=unknown -_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds -_LT_TAGVAR(reload_flag, $1)=$reload_flag -_LT_TAGVAR(reload_cmds, $1)=$reload_cmds -_LT_TAGVAR(no_undefined_flag, $1)= -_LT_TAGVAR(whole_archive_flag_spec, $1)= -_LT_TAGVAR(enable_shared_with_static_runtimes, $1)=no - -# Source file extension for fc test sources. -ac_ext=${ac_fc_srcext-f} - -# Object file extension for compiled fc test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# No sense in running all these tests if we already determined that -# the FC compiler isn't working. Some variables (like enable_shared) -# are currently assumed to apply to all compilers on this platform, -# and will be corrupted by setting them based on a non-working compiler. -if test yes != "$_lt_disable_FC"; then - # Code to be used in simple compile tests - lt_simple_compile_test_code="\ - subroutine t - return - end -" - - # Code to be used in simple link tests - lt_simple_link_test_code="\ - program t - end -" - - # ltmain only uses $CC for tagged configurations so make sure $CC is set. - _LT_TAG_COMPILER - - # save warnings/boilerplate of simple test code - _LT_COMPILER_BOILERPLATE - _LT_LINKER_BOILERPLATE - - # Allow CC to be a program name with arguments. - lt_save_CC=$CC - lt_save_GCC=$GCC - lt_save_CFLAGS=$CFLAGS - CC=${FC-"f95"} - CFLAGS=$FCFLAGS - compiler=$CC - GCC=$ac_cv_fc_compiler_gnu - - _LT_TAGVAR(compiler, $1)=$CC - _LT_CC_BASENAME([$compiler]) - - if test -n "$compiler"; then - AC_MSG_CHECKING([if libtool supports shared libraries]) - AC_MSG_RESULT([$can_build_shared]) - - AC_MSG_CHECKING([whether to build shared libraries]) - test no = "$can_build_shared" && enable_shared=no - - # On AIX, shared libraries and static libraries use the same namespace, and - # are all built from PIC. - case $host_os in - aix3*) - test yes = "$enable_shared" && enable_static=no - if test -n "$RANLIB"; then - archive_cmds="$archive_cmds~\$RANLIB \$lib" - postinstall_cmds='$RANLIB $lib' - fi - ;; - aix[[4-9]]*) - if test ia64 != "$host_cpu"; then - case $enable_shared,$with_aix_soname,$aix_use_runtimelinking in - yes,aix,yes) ;; # shared object as lib.so file only - yes,svr4,*) ;; # shared object as lib.so archive member only - yes,*) enable_static=no ;; # shared object in lib.a archive as well - esac - fi - ;; - esac - AC_MSG_RESULT([$enable_shared]) - - AC_MSG_CHECKING([whether to build static libraries]) - # Make sure either enable_shared or enable_static is yes. - test yes = "$enable_shared" || enable_static=yes - AC_MSG_RESULT([$enable_static]) - - _LT_TAGVAR(GCC, $1)=$ac_cv_fc_compiler_gnu - _LT_TAGVAR(LD, $1)=$LD - - ## CAVEAT EMPTOR: - ## There is no encapsulation within the following macros, do not change - ## the running order or otherwise move them around unless you know exactly - ## what you are doing... - _LT_SYS_HIDDEN_LIBDEPS($1) - _LT_COMPILER_PIC($1) - _LT_COMPILER_C_O($1) - _LT_COMPILER_FILE_LOCKS($1) - _LT_LINKER_SHLIBS($1) - _LT_SYS_DYNAMIC_LINKER($1) - _LT_LINKER_HARDCODE_LIBPATH($1) - - _LT_CONFIG($1) - fi # test -n "$compiler" - - GCC=$lt_save_GCC - CC=$lt_save_CC - CFLAGS=$lt_save_CFLAGS -fi # test yes != "$_lt_disable_FC" - -AC_LANG_POP -])# _LT_LANG_FC_CONFIG - - -# _LT_LANG_GCJ_CONFIG([TAG]) -# -------------------------- -# Ensure that the configuration variables for the GNU Java Compiler compiler -# are suitably defined. These variables are subsequently used by _LT_CONFIG -# to write the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_GCJ_CONFIG], -[AC_REQUIRE([LT_PROG_GCJ])dnl -AC_LANG_SAVE - -# Source file extension for Java test sources. -ac_ext=java - -# Object file extension for compiled Java test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# Code to be used in simple compile tests -lt_simple_compile_test_code="class foo {}" - -# Code to be used in simple link tests -lt_simple_link_test_code='public class conftest { public static void main(String[[]] argv) {}; }' - -# ltmain only uses $CC for tagged configurations so make sure $CC is set. -_LT_TAG_COMPILER - -# save warnings/boilerplate of simple test code -_LT_COMPILER_BOILERPLATE -_LT_LINKER_BOILERPLATE - -# Allow CC to be a program name with arguments. -lt_save_CC=$CC -lt_save_CFLAGS=$CFLAGS -lt_save_GCC=$GCC -GCC=yes -CC=${GCJ-"gcj"} -CFLAGS=$GCJFLAGS -compiler=$CC -_LT_TAGVAR(compiler, $1)=$CC -_LT_TAGVAR(LD, $1)=$LD -_LT_CC_BASENAME([$compiler]) - -# GCJ did not exist at the time GCC didn't implicitly link libc in. -_LT_TAGVAR(archive_cmds_need_lc, $1)=no - -_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds -_LT_TAGVAR(reload_flag, $1)=$reload_flag -_LT_TAGVAR(reload_cmds, $1)=$reload_cmds - -if test -n "$compiler"; then - _LT_COMPILER_NO_RTTI($1) - _LT_COMPILER_PIC($1) - _LT_COMPILER_C_O($1) - _LT_COMPILER_FILE_LOCKS($1) - _LT_LINKER_SHLIBS($1) - _LT_LINKER_HARDCODE_LIBPATH($1) - - _LT_CONFIG($1) -fi - -AC_LANG_RESTORE - -GCC=$lt_save_GCC -CC=$lt_save_CC -CFLAGS=$lt_save_CFLAGS -])# _LT_LANG_GCJ_CONFIG - - -# _LT_LANG_GO_CONFIG([TAG]) -# -------------------------- -# Ensure that the configuration variables for the GNU Go compiler -# are suitably defined. These variables are subsequently used by _LT_CONFIG -# to write the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_GO_CONFIG], -[AC_REQUIRE([LT_PROG_GO])dnl -AC_LANG_SAVE - -# Source file extension for Go test sources. -ac_ext=go - -# Object file extension for compiled Go test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# Code to be used in simple compile tests -lt_simple_compile_test_code="package main; func main() { }" - -# Code to be used in simple link tests -lt_simple_link_test_code='package main; func main() { }' - -# ltmain only uses $CC for tagged configurations so make sure $CC is set. -_LT_TAG_COMPILER - -# save warnings/boilerplate of simple test code -_LT_COMPILER_BOILERPLATE -_LT_LINKER_BOILERPLATE - -# Allow CC to be a program name with arguments. -lt_save_CC=$CC -lt_save_CFLAGS=$CFLAGS -lt_save_GCC=$GCC -GCC=yes -CC=${GOC-"gccgo"} -CFLAGS=$GOFLAGS -compiler=$CC -_LT_TAGVAR(compiler, $1)=$CC -_LT_TAGVAR(LD, $1)=$LD -_LT_CC_BASENAME([$compiler]) - -# Go did not exist at the time GCC didn't implicitly link libc in. -_LT_TAGVAR(archive_cmds_need_lc, $1)=no - -_LT_TAGVAR(old_archive_cmds, $1)=$old_archive_cmds -_LT_TAGVAR(reload_flag, $1)=$reload_flag -_LT_TAGVAR(reload_cmds, $1)=$reload_cmds - -if test -n "$compiler"; then - _LT_COMPILER_NO_RTTI($1) - _LT_COMPILER_PIC($1) - _LT_COMPILER_C_O($1) - _LT_COMPILER_FILE_LOCKS($1) - _LT_LINKER_SHLIBS($1) - _LT_LINKER_HARDCODE_LIBPATH($1) - - _LT_CONFIG($1) -fi - -AC_LANG_RESTORE - -GCC=$lt_save_GCC -CC=$lt_save_CC -CFLAGS=$lt_save_CFLAGS -])# _LT_LANG_GO_CONFIG - - -# _LT_LANG_RC_CONFIG([TAG]) -# ------------------------- -# Ensure that the configuration variables for the Windows resource compiler -# are suitably defined. These variables are subsequently used by _LT_CONFIG -# to write the compiler configuration to 'libtool'. -m4_defun([_LT_LANG_RC_CONFIG], -[AC_REQUIRE([LT_PROG_RC])dnl -AC_LANG_SAVE - -# Source file extension for RC test sources. -ac_ext=rc - -# Object file extension for compiled RC test sources. -objext=o -_LT_TAGVAR(objext, $1)=$objext - -# Code to be used in simple compile tests -lt_simple_compile_test_code='sample MENU { MENUITEM "&Soup", 100, CHECKED }' - -# Code to be used in simple link tests -lt_simple_link_test_code=$lt_simple_compile_test_code - -# ltmain only uses $CC for tagged configurations so make sure $CC is set. -_LT_TAG_COMPILER - -# save warnings/boilerplate of simple test code -_LT_COMPILER_BOILERPLATE -_LT_LINKER_BOILERPLATE - -# Allow CC to be a program name with arguments. -lt_save_CC=$CC -lt_save_CFLAGS=$CFLAGS -lt_save_GCC=$GCC -GCC= -CC=${RC-"windres"} -CFLAGS= -compiler=$CC -_LT_TAGVAR(compiler, $1)=$CC -_LT_CC_BASENAME([$compiler]) -_LT_TAGVAR(lt_cv_prog_compiler_c_o, $1)=yes - -if test -n "$compiler"; then - : - _LT_CONFIG($1) -fi - -GCC=$lt_save_GCC -AC_LANG_RESTORE -CC=$lt_save_CC -CFLAGS=$lt_save_CFLAGS -])# _LT_LANG_RC_CONFIG - - -# LT_PROG_GCJ -# ----------- -AC_DEFUN([LT_PROG_GCJ], -[m4_ifdef([AC_PROG_GCJ], [AC_PROG_GCJ], - [m4_ifdef([A][M_PROG_GCJ], [A][M_PROG_GCJ], - [AC_CHECK_TOOL(GCJ, gcj,) - test set = "${GCJFLAGS+set}" || GCJFLAGS="-g -O2" - AC_SUBST(GCJFLAGS)])])[]dnl -]) - -# Old name: -AU_ALIAS([LT_AC_PROG_GCJ], [LT_PROG_GCJ]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([LT_AC_PROG_GCJ], []) - - -# LT_PROG_GO -# ---------- -AC_DEFUN([LT_PROG_GO], -[AC_CHECK_TOOL(GOC, gccgo,) -]) - - -# LT_PROG_RC -# ---------- -AC_DEFUN([LT_PROG_RC], -[AC_CHECK_TOOL(RC, windres,) -]) - -# Old name: -AU_ALIAS([LT_AC_PROG_RC], [LT_PROG_RC]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([LT_AC_PROG_RC], []) - - -# _LT_DECL_EGREP -# -------------- -# If we don't have a new enough Autoconf to choose the best grep -# available, choose the one first in the user's PATH. -m4_defun([_LT_DECL_EGREP], -[AC_REQUIRE([AC_PROG_EGREP])dnl -AC_REQUIRE([AC_PROG_FGREP])dnl -test -z "$GREP" && GREP=grep -_LT_DECL([], [GREP], [1], [A grep program that handles long lines]) -_LT_DECL([], [EGREP], [1], [An ERE matcher]) -_LT_DECL([], [FGREP], [1], [A literal string matcher]) -dnl Non-bleeding-edge autoconf doesn't subst GREP, so do it here too -AC_SUBST([GREP]) -]) - - -# _LT_DECL_OBJDUMP -# -------------- -# If we don't have a new enough Autoconf to choose the best objdump -# available, choose the one first in the user's PATH. -m4_defun([_LT_DECL_OBJDUMP], -[AC_CHECK_TOOL(OBJDUMP, objdump, false) -test -z "$OBJDUMP" && OBJDUMP=objdump -_LT_DECL([], [OBJDUMP], [1], [An object symbol dumper]) -AC_SUBST([OBJDUMP]) -]) - -# _LT_DECL_DLLTOOL -# ---------------- -# Ensure DLLTOOL variable is set. -m4_defun([_LT_DECL_DLLTOOL], -[AC_CHECK_TOOL(DLLTOOL, dlltool, false) -test -z "$DLLTOOL" && DLLTOOL=dlltool -_LT_DECL([], [DLLTOOL], [1], [DLL creation program]) -AC_SUBST([DLLTOOL]) -]) - -# _LT_DECL_SED -# ------------ -# Check for a fully-functional sed program, that truncates -# as few characters as possible. Prefer GNU sed if found. -m4_defun([_LT_DECL_SED], -[AC_PROG_SED -test -z "$SED" && SED=sed -Xsed="$SED -e 1s/^X//" -_LT_DECL([], [SED], [1], [A sed program that does not truncate output]) -_LT_DECL([], [Xsed], ["\$SED -e 1s/^X//"], - [Sed that helps us avoid accidentally triggering echo(1) options like -n]) -])# _LT_DECL_SED - -m4_ifndef([AC_PROG_SED], [ -# NOTE: This macro has been submitted for inclusion into # -# GNU Autoconf as AC_PROG_SED. When it is available in # -# a released version of Autoconf we should remove this # -# macro and use it instead. # - -m4_defun([AC_PROG_SED], -[AC_MSG_CHECKING([for a sed that does not truncate output]) -AC_CACHE_VAL(lt_cv_path_SED, -[# Loop through the user's path and test for sed and gsed. -# Then use that list of sed's as ones to test for truncation. -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for lt_ac_prog in sed gsed; do - for ac_exec_ext in '' $ac_executable_extensions; do - if $as_executable_p "$as_dir/$lt_ac_prog$ac_exec_ext"; then - lt_ac_sed_list="$lt_ac_sed_list $as_dir/$lt_ac_prog$ac_exec_ext" - fi - done - done -done -IFS=$as_save_IFS -lt_ac_max=0 -lt_ac_count=0 -# Add /usr/xpg4/bin/sed as it is typically found on Solaris -# along with /bin/sed that truncates output. -for lt_ac_sed in $lt_ac_sed_list /usr/xpg4/bin/sed; do - test ! -f "$lt_ac_sed" && continue - cat /dev/null > conftest.in - lt_ac_count=0 - echo $ECHO_N "0123456789$ECHO_C" >conftest.in - # Check for GNU sed and select it if it is found. - if "$lt_ac_sed" --version 2>&1 < /dev/null | grep 'GNU' > /dev/null; then - lt_cv_path_SED=$lt_ac_sed - break - fi - while true; do - cat conftest.in conftest.in >conftest.tmp - mv conftest.tmp conftest.in - cp conftest.in conftest.nl - echo >>conftest.nl - $lt_ac_sed -e 's/a$//' < conftest.nl >conftest.out || break - cmp -s conftest.out conftest.nl || break - # 10000 chars as input seems more than enough - test 10 -lt "$lt_ac_count" && break - lt_ac_count=`expr $lt_ac_count + 1` - if test "$lt_ac_count" -gt "$lt_ac_max"; then - lt_ac_max=$lt_ac_count - lt_cv_path_SED=$lt_ac_sed - fi - done -done -]) -SED=$lt_cv_path_SED -AC_SUBST([SED]) -AC_MSG_RESULT([$SED]) -])#AC_PROG_SED -])#m4_ifndef - -# Old name: -AU_ALIAS([LT_AC_PROG_SED], [AC_PROG_SED]) -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([LT_AC_PROG_SED], []) - - -# _LT_CHECK_SHELL_FEATURES -# ------------------------ -# Find out whether the shell is Bourne or XSI compatible, -# or has some other useful features. -m4_defun([_LT_CHECK_SHELL_FEATURES], -[if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then - lt_unset=unset -else - lt_unset=false -fi -_LT_DECL([], [lt_unset], [0], [whether the shell understands "unset"])dnl - -# test EBCDIC or ASCII -case `echo X|tr X '\101'` in - A) # ASCII based system - # \n is not interpreted correctly by Solaris 8 /usr/ucb/tr - lt_SP2NL='tr \040 \012' - lt_NL2SP='tr \015\012 \040\040' - ;; - *) # EBCDIC based system - lt_SP2NL='tr \100 \n' - lt_NL2SP='tr \r\n \100\100' - ;; -esac -_LT_DECL([SP2NL], [lt_SP2NL], [1], [turn spaces into newlines])dnl -_LT_DECL([NL2SP], [lt_NL2SP], [1], [turn newlines into spaces])dnl -])# _LT_CHECK_SHELL_FEATURES - - -# _LT_PATH_CONVERSION_FUNCTIONS -# ----------------------------- -# Determine what file name conversion functions should be used by -# func_to_host_file (and, implicitly, by func_to_host_path). These are needed -# for certain cross-compile configurations and native mingw. -m4_defun([_LT_PATH_CONVERSION_FUNCTIONS], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -AC_REQUIRE([AC_CANONICAL_BUILD])dnl -AC_MSG_CHECKING([how to convert $build file names to $host format]) -AC_CACHE_VAL(lt_cv_to_host_file_cmd, -[case $host in - *-*-mingw* ) - case $build in - *-*-mingw* ) # actually msys - lt_cv_to_host_file_cmd=func_convert_file_msys_to_w32 - ;; - *-*-cygwin* ) - lt_cv_to_host_file_cmd=func_convert_file_cygwin_to_w32 - ;; - * ) # otherwise, assume *nix - lt_cv_to_host_file_cmd=func_convert_file_nix_to_w32 - ;; - esac - ;; - *-*-cygwin* ) - case $build in - *-*-mingw* ) # actually msys - lt_cv_to_host_file_cmd=func_convert_file_msys_to_cygwin - ;; - *-*-cygwin* ) - lt_cv_to_host_file_cmd=func_convert_file_noop - ;; - * ) # otherwise, assume *nix - lt_cv_to_host_file_cmd=func_convert_file_nix_to_cygwin - ;; - esac - ;; - * ) # unhandled hosts (and "normal" native builds) - lt_cv_to_host_file_cmd=func_convert_file_noop - ;; -esac -]) -to_host_file_cmd=$lt_cv_to_host_file_cmd -AC_MSG_RESULT([$lt_cv_to_host_file_cmd]) -_LT_DECL([to_host_file_cmd], [lt_cv_to_host_file_cmd], - [0], [convert $build file names to $host format])dnl - -AC_MSG_CHECKING([how to convert $build file names to toolchain format]) -AC_CACHE_VAL(lt_cv_to_tool_file_cmd, -[#assume ordinary cross tools, or native build. -lt_cv_to_tool_file_cmd=func_convert_file_noop -case $host in - *-*-mingw* ) - case $build in - *-*-mingw* ) # actually msys - lt_cv_to_tool_file_cmd=func_convert_file_msys_to_w32 - ;; - esac - ;; -esac -]) -to_tool_file_cmd=$lt_cv_to_tool_file_cmd -AC_MSG_RESULT([$lt_cv_to_tool_file_cmd]) -_LT_DECL([to_tool_file_cmd], [lt_cv_to_tool_file_cmd], - [0], [convert $build files to toolchain format])dnl -])# _LT_PATH_CONVERSION_FUNCTIONS - -# Helper functions for option handling. -*- Autoconf -*- -# -# Copyright (C) 2004-2005, 2007-2009, 2011-2015 Free Software -# Foundation, Inc. -# Written by Gary V. Vaughan, 2004 -# -# This file is free software; the Free Software Foundation gives -# unlimited permission to copy and/or distribute it, with or without -# modifications, as long as this notice is preserved. - -# serial 8 ltoptions.m4 - -# This is to help aclocal find these macros, as it can't see m4_define. -AC_DEFUN([LTOPTIONS_VERSION], [m4_if([1])]) - - -# _LT_MANGLE_OPTION(MACRO-NAME, OPTION-NAME) -# ------------------------------------------ -m4_define([_LT_MANGLE_OPTION], -[[_LT_OPTION_]m4_bpatsubst($1__$2, [[^a-zA-Z0-9_]], [_])]) - - -# _LT_SET_OPTION(MACRO-NAME, OPTION-NAME) -# --------------------------------------- -# Set option OPTION-NAME for macro MACRO-NAME, and if there is a -# matching handler defined, dispatch to it. Other OPTION-NAMEs are -# saved as a flag. -m4_define([_LT_SET_OPTION], -[m4_define(_LT_MANGLE_OPTION([$1], [$2]))dnl -m4_ifdef(_LT_MANGLE_DEFUN([$1], [$2]), - _LT_MANGLE_DEFUN([$1], [$2]), - [m4_warning([Unknown $1 option '$2'])])[]dnl -]) - - -# _LT_IF_OPTION(MACRO-NAME, OPTION-NAME, IF-SET, [IF-NOT-SET]) -# ------------------------------------------------------------ -# Execute IF-SET if OPTION is set, IF-NOT-SET otherwise. -m4_define([_LT_IF_OPTION], -[m4_ifdef(_LT_MANGLE_OPTION([$1], [$2]), [$3], [$4])]) - - -# _LT_UNLESS_OPTIONS(MACRO-NAME, OPTION-LIST, IF-NOT-SET) -# ------------------------------------------------------- -# Execute IF-NOT-SET unless all options in OPTION-LIST for MACRO-NAME -# are set. -m4_define([_LT_UNLESS_OPTIONS], -[m4_foreach([_LT_Option], m4_split(m4_normalize([$2])), - [m4_ifdef(_LT_MANGLE_OPTION([$1], _LT_Option), - [m4_define([$0_found])])])[]dnl -m4_ifdef([$0_found], [m4_undefine([$0_found])], [$3 -])[]dnl -]) - - -# _LT_SET_OPTIONS(MACRO-NAME, OPTION-LIST) -# ---------------------------------------- -# OPTION-LIST is a space-separated list of Libtool options associated -# with MACRO-NAME. If any OPTION has a matching handler declared with -# LT_OPTION_DEFINE, dispatch to that macro; otherwise complain about -# the unknown option and exit. -m4_defun([_LT_SET_OPTIONS], -[# Set options -m4_foreach([_LT_Option], m4_split(m4_normalize([$2])), - [_LT_SET_OPTION([$1], _LT_Option)]) - -m4_if([$1],[LT_INIT],[ - dnl - dnl Simply set some default values (i.e off) if boolean options were not - dnl specified: - _LT_UNLESS_OPTIONS([LT_INIT], [dlopen], [enable_dlopen=no - ]) - _LT_UNLESS_OPTIONS([LT_INIT], [win32-dll], [enable_win32_dll=no - ]) - dnl - dnl If no reference was made to various pairs of opposing options, then - dnl we run the default mode handler for the pair. For example, if neither - dnl 'shared' nor 'disable-shared' was passed, we enable building of shared - dnl archives by default: - _LT_UNLESS_OPTIONS([LT_INIT], [shared disable-shared], [_LT_ENABLE_SHARED]) - _LT_UNLESS_OPTIONS([LT_INIT], [static disable-static], [_LT_ENABLE_STATIC]) - _LT_UNLESS_OPTIONS([LT_INIT], [pic-only no-pic], [_LT_WITH_PIC]) - _LT_UNLESS_OPTIONS([LT_INIT], [fast-install disable-fast-install], - [_LT_ENABLE_FAST_INSTALL]) - _LT_UNLESS_OPTIONS([LT_INIT], [aix-soname=aix aix-soname=both aix-soname=svr4], - [_LT_WITH_AIX_SONAME([aix])]) - ]) -])# _LT_SET_OPTIONS - - - -# _LT_MANGLE_DEFUN(MACRO-NAME, OPTION-NAME) -# ----------------------------------------- -m4_define([_LT_MANGLE_DEFUN], -[[_LT_OPTION_DEFUN_]m4_bpatsubst(m4_toupper([$1__$2]), [[^A-Z0-9_]], [_])]) - - -# LT_OPTION_DEFINE(MACRO-NAME, OPTION-NAME, CODE) -# ----------------------------------------------- -m4_define([LT_OPTION_DEFINE], -[m4_define(_LT_MANGLE_DEFUN([$1], [$2]), [$3])[]dnl -])# LT_OPTION_DEFINE - - -# dlopen -# ------ -LT_OPTION_DEFINE([LT_INIT], [dlopen], [enable_dlopen=yes -]) - -AU_DEFUN([AC_LIBTOOL_DLOPEN], -[_LT_SET_OPTION([LT_INIT], [dlopen]) -AC_DIAGNOSE([obsolete], -[$0: Remove this warning and the call to _LT_SET_OPTION when you -put the 'dlopen' option into LT_INIT's first parameter.]) -]) - -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_DLOPEN], []) - - -# win32-dll -# --------- -# Declare package support for building win32 dll's. -LT_OPTION_DEFINE([LT_INIT], [win32-dll], -[enable_win32_dll=yes - -case $host in -*-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-cegcc*) - AC_CHECK_TOOL(AS, as, false) - AC_CHECK_TOOL(DLLTOOL, dlltool, false) - AC_CHECK_TOOL(OBJDUMP, objdump, false) - ;; -esac - -test -z "$AS" && AS=as -_LT_DECL([], [AS], [1], [Assembler program])dnl - -test -z "$DLLTOOL" && DLLTOOL=dlltool -_LT_DECL([], [DLLTOOL], [1], [DLL creation program])dnl - -test -z "$OBJDUMP" && OBJDUMP=objdump -_LT_DECL([], [OBJDUMP], [1], [Object dumper program])dnl -])# win32-dll - -AU_DEFUN([AC_LIBTOOL_WIN32_DLL], -[AC_REQUIRE([AC_CANONICAL_HOST])dnl -_LT_SET_OPTION([LT_INIT], [win32-dll]) -AC_DIAGNOSE([obsolete], -[$0: Remove this warning and the call to _LT_SET_OPTION when you -put the 'win32-dll' option into LT_INIT's first parameter.]) -]) - -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_WIN32_DLL], []) - - -# _LT_ENABLE_SHARED([DEFAULT]) -# ---------------------------- -# implement the --enable-shared flag, and supports the 'shared' and -# 'disable-shared' LT_INIT options. -# DEFAULT is either 'yes' or 'no'. If omitted, it defaults to 'yes'. -m4_define([_LT_ENABLE_SHARED], -[m4_define([_LT_ENABLE_SHARED_DEFAULT], [m4_if($1, no, no, yes)])dnl -AC_ARG_ENABLE([shared], - [AS_HELP_STRING([--enable-shared@<:@=PKGS@:>@], - [build shared libraries @<:@default=]_LT_ENABLE_SHARED_DEFAULT[@:>@])], - [p=${PACKAGE-default} - case $enableval in - yes) enable_shared=yes ;; - no) enable_shared=no ;; - *) - enable_shared=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_shared=yes - fi - done - IFS=$lt_save_ifs - ;; - esac], - [enable_shared=]_LT_ENABLE_SHARED_DEFAULT) - - _LT_DECL([build_libtool_libs], [enable_shared], [0], - [Whether or not to build shared libraries]) -])# _LT_ENABLE_SHARED - -LT_OPTION_DEFINE([LT_INIT], [shared], [_LT_ENABLE_SHARED([yes])]) -LT_OPTION_DEFINE([LT_INIT], [disable-shared], [_LT_ENABLE_SHARED([no])]) - -# Old names: -AC_DEFUN([AC_ENABLE_SHARED], -[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[shared]) -]) - -AC_DEFUN([AC_DISABLE_SHARED], -[_LT_SET_OPTION([LT_INIT], [disable-shared]) -]) - -AU_DEFUN([AM_ENABLE_SHARED], [AC_ENABLE_SHARED($@)]) -AU_DEFUN([AM_DISABLE_SHARED], [AC_DISABLE_SHARED($@)]) - -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AM_ENABLE_SHARED], []) -dnl AC_DEFUN([AM_DISABLE_SHARED], []) - - - -# _LT_ENABLE_STATIC([DEFAULT]) -# ---------------------------- -# implement the --enable-static flag, and support the 'static' and -# 'disable-static' LT_INIT options. -# DEFAULT is either 'yes' or 'no'. If omitted, it defaults to 'yes'. -m4_define([_LT_ENABLE_STATIC], -[m4_define([_LT_ENABLE_STATIC_DEFAULT], [m4_if($1, no, no, yes)])dnl -AC_ARG_ENABLE([static], - [AS_HELP_STRING([--enable-static@<:@=PKGS@:>@], - [build static libraries @<:@default=]_LT_ENABLE_STATIC_DEFAULT[@:>@])], - [p=${PACKAGE-default} - case $enableval in - yes) enable_static=yes ;; - no) enable_static=no ;; - *) - enable_static=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_static=yes - fi - done - IFS=$lt_save_ifs - ;; - esac], - [enable_static=]_LT_ENABLE_STATIC_DEFAULT) - - _LT_DECL([build_old_libs], [enable_static], [0], - [Whether or not to build static libraries]) -])# _LT_ENABLE_STATIC - -LT_OPTION_DEFINE([LT_INIT], [static], [_LT_ENABLE_STATIC([yes])]) -LT_OPTION_DEFINE([LT_INIT], [disable-static], [_LT_ENABLE_STATIC([no])]) - -# Old names: -AC_DEFUN([AC_ENABLE_STATIC], -[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[static]) -]) - -AC_DEFUN([AC_DISABLE_STATIC], -[_LT_SET_OPTION([LT_INIT], [disable-static]) -]) - -AU_DEFUN([AM_ENABLE_STATIC], [AC_ENABLE_STATIC($@)]) -AU_DEFUN([AM_DISABLE_STATIC], [AC_DISABLE_STATIC($@)]) - -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AM_ENABLE_STATIC], []) -dnl AC_DEFUN([AM_DISABLE_STATIC], []) - - - -# _LT_ENABLE_FAST_INSTALL([DEFAULT]) -# ---------------------------------- -# implement the --enable-fast-install flag, and support the 'fast-install' -# and 'disable-fast-install' LT_INIT options. -# DEFAULT is either 'yes' or 'no'. If omitted, it defaults to 'yes'. -m4_define([_LT_ENABLE_FAST_INSTALL], -[m4_define([_LT_ENABLE_FAST_INSTALL_DEFAULT], [m4_if($1, no, no, yes)])dnl -AC_ARG_ENABLE([fast-install], - [AS_HELP_STRING([--enable-fast-install@<:@=PKGS@:>@], - [optimize for fast installation @<:@default=]_LT_ENABLE_FAST_INSTALL_DEFAULT[@:>@])], - [p=${PACKAGE-default} - case $enableval in - yes) enable_fast_install=yes ;; - no) enable_fast_install=no ;; - *) - enable_fast_install=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_fast_install=yes - fi - done - IFS=$lt_save_ifs - ;; - esac], - [enable_fast_install=]_LT_ENABLE_FAST_INSTALL_DEFAULT) - -_LT_DECL([fast_install], [enable_fast_install], [0], - [Whether or not to optimize for fast installation])dnl -])# _LT_ENABLE_FAST_INSTALL - -LT_OPTION_DEFINE([LT_INIT], [fast-install], [_LT_ENABLE_FAST_INSTALL([yes])]) -LT_OPTION_DEFINE([LT_INIT], [disable-fast-install], [_LT_ENABLE_FAST_INSTALL([no])]) - -# Old names: -AU_DEFUN([AC_ENABLE_FAST_INSTALL], -[_LT_SET_OPTION([LT_INIT], m4_if([$1], [no], [disable-])[fast-install]) -AC_DIAGNOSE([obsolete], -[$0: Remove this warning and the call to _LT_SET_OPTION when you put -the 'fast-install' option into LT_INIT's first parameter.]) -]) - -AU_DEFUN([AC_DISABLE_FAST_INSTALL], -[_LT_SET_OPTION([LT_INIT], [disable-fast-install]) -AC_DIAGNOSE([obsolete], -[$0: Remove this warning and the call to _LT_SET_OPTION when you put -the 'disable-fast-install' option into LT_INIT's first parameter.]) -]) - -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_ENABLE_FAST_INSTALL], []) -dnl AC_DEFUN([AM_DISABLE_FAST_INSTALL], []) - - -# _LT_WITH_AIX_SONAME([DEFAULT]) -# ---------------------------------- -# implement the --with-aix-soname flag, and support the `aix-soname=aix' -# and `aix-soname=both' and `aix-soname=svr4' LT_INIT options. DEFAULT -# is either `aix', `both' or `svr4'. If omitted, it defaults to `aix'. -m4_define([_LT_WITH_AIX_SONAME], -[m4_define([_LT_WITH_AIX_SONAME_DEFAULT], [m4_if($1, svr4, svr4, m4_if($1, both, both, aix))])dnl -shared_archive_member_spec= -case $host,$enable_shared in -power*-*-aix[[5-9]]*,yes) - AC_MSG_CHECKING([which variant of shared library versioning to provide]) - AC_ARG_WITH([aix-soname], - [AS_HELP_STRING([--with-aix-soname=aix|svr4|both], - [shared library versioning (aka "SONAME") variant to provide on AIX, @<:@default=]_LT_WITH_AIX_SONAME_DEFAULT[@:>@.])], - [case $withval in - aix|svr4|both) - ;; - *) - AC_MSG_ERROR([Unknown argument to --with-aix-soname]) - ;; - esac - lt_cv_with_aix_soname=$with_aix_soname], - [AC_CACHE_VAL([lt_cv_with_aix_soname], - [lt_cv_with_aix_soname=]_LT_WITH_AIX_SONAME_DEFAULT) - with_aix_soname=$lt_cv_with_aix_soname]) - AC_MSG_RESULT([$with_aix_soname]) - if test aix != "$with_aix_soname"; then - # For the AIX way of multilib, we name the shared archive member - # based on the bitwidth used, traditionally 'shr.o' or 'shr_64.o', - # and 'shr.imp' or 'shr_64.imp', respectively, for the Import File. - # Even when GNU compilers ignore OBJECT_MODE but need '-maix64' flag, - # the AIX toolchain works better with OBJECT_MODE set (default 32). - if test 64 = "${OBJECT_MODE-32}"; then - shared_archive_member_spec=shr_64 - else - shared_archive_member_spec=shr - fi - fi - ;; -*) - with_aix_soname=aix - ;; -esac - -_LT_DECL([], [shared_archive_member_spec], [0], - [Shared archive member basename, for filename based shared library versioning on AIX])dnl -])# _LT_WITH_AIX_SONAME - -LT_OPTION_DEFINE([LT_INIT], [aix-soname=aix], [_LT_WITH_AIX_SONAME([aix])]) -LT_OPTION_DEFINE([LT_INIT], [aix-soname=both], [_LT_WITH_AIX_SONAME([both])]) -LT_OPTION_DEFINE([LT_INIT], [aix-soname=svr4], [_LT_WITH_AIX_SONAME([svr4])]) - - -# _LT_WITH_PIC([MODE]) -# -------------------- -# implement the --with-pic flag, and support the 'pic-only' and 'no-pic' -# LT_INIT options. -# MODE is either 'yes' or 'no'. If omitted, it defaults to 'both'. -m4_define([_LT_WITH_PIC], -[AC_ARG_WITH([pic], - [AS_HELP_STRING([--with-pic@<:@=PKGS@:>@], - [try to use only PIC/non-PIC objects @<:@default=use both@:>@])], - [lt_p=${PACKAGE-default} - case $withval in - yes|no) pic_mode=$withval ;; - *) - pic_mode=default - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for lt_pkg in $withval; do - IFS=$lt_save_ifs - if test "X$lt_pkg" = "X$lt_p"; then - pic_mode=yes - fi - done - IFS=$lt_save_ifs - ;; - esac], - [pic_mode=m4_default([$1], [default])]) - -_LT_DECL([], [pic_mode], [0], [What type of objects to build])dnl -])# _LT_WITH_PIC - -LT_OPTION_DEFINE([LT_INIT], [pic-only], [_LT_WITH_PIC([yes])]) -LT_OPTION_DEFINE([LT_INIT], [no-pic], [_LT_WITH_PIC([no])]) - -# Old name: -AU_DEFUN([AC_LIBTOOL_PICMODE], -[_LT_SET_OPTION([LT_INIT], [pic-only]) -AC_DIAGNOSE([obsolete], -[$0: Remove this warning and the call to _LT_SET_OPTION when you -put the 'pic-only' option into LT_INIT's first parameter.]) -]) - -dnl aclocal-1.4 backwards compatibility: -dnl AC_DEFUN([AC_LIBTOOL_PICMODE], []) - - -m4_define([_LTDL_MODE], []) -LT_OPTION_DEFINE([LTDL_INIT], [nonrecursive], - [m4_define([_LTDL_MODE], [nonrecursive])]) -LT_OPTION_DEFINE([LTDL_INIT], [recursive], - [m4_define([_LTDL_MODE], [recursive])]) -LT_OPTION_DEFINE([LTDL_INIT], [subproject], - [m4_define([_LTDL_MODE], [subproject])]) - -m4_define([_LTDL_TYPE], []) -LT_OPTION_DEFINE([LTDL_INIT], [installable], - [m4_define([_LTDL_TYPE], [installable])]) -LT_OPTION_DEFINE([LTDL_INIT], [convenience], - [m4_define([_LTDL_TYPE], [convenience])]) - -# ltsugar.m4 -- libtool m4 base layer. -*-Autoconf-*- -# -# Copyright (C) 2004-2005, 2007-2008, 2011-2015 Free Software -# Foundation, Inc. -# Written by Gary V. Vaughan, 2004 -# -# This file is free software; the Free Software Foundation gives -# unlimited permission to copy and/or distribute it, with or without -# modifications, as long as this notice is preserved. - -# serial 6 ltsugar.m4 - -# This is to help aclocal find these macros, as it can't see m4_define. -AC_DEFUN([LTSUGAR_VERSION], [m4_if([0.1])]) - - -# lt_join(SEP, ARG1, [ARG2...]) -# ----------------------------- -# Produce ARG1SEPARG2...SEPARGn, omitting [] arguments and their -# associated separator. -# Needed until we can rely on m4_join from Autoconf 2.62, since all earlier -# versions in m4sugar had bugs. -m4_define([lt_join], -[m4_if([$#], [1], [], - [$#], [2], [[$2]], - [m4_if([$2], [], [], [[$2]_])$0([$1], m4_shift(m4_shift($@)))])]) -m4_define([_lt_join], -[m4_if([$#$2], [2], [], - [m4_if([$2], [], [], [[$1$2]])$0([$1], m4_shift(m4_shift($@)))])]) - - -# lt_car(LIST) -# lt_cdr(LIST) -# ------------ -# Manipulate m4 lists. -# These macros are necessary as long as will still need to support -# Autoconf-2.59, which quotes differently. -m4_define([lt_car], [[$1]]) -m4_define([lt_cdr], -[m4_if([$#], 0, [m4_fatal([$0: cannot be called without arguments])], - [$#], 1, [], - [m4_dquote(m4_shift($@))])]) -m4_define([lt_unquote], $1) - - -# lt_append(MACRO-NAME, STRING, [SEPARATOR]) -# ------------------------------------------ -# Redefine MACRO-NAME to hold its former content plus 'SEPARATOR''STRING'. -# Note that neither SEPARATOR nor STRING are expanded; they are appended -# to MACRO-NAME as is (leaving the expansion for when MACRO-NAME is invoked). -# No SEPARATOR is output if MACRO-NAME was previously undefined (different -# than defined and empty). -# -# This macro is needed until we can rely on Autoconf 2.62, since earlier -# versions of m4sugar mistakenly expanded SEPARATOR but not STRING. -m4_define([lt_append], -[m4_define([$1], - m4_ifdef([$1], [m4_defn([$1])[$3]])[$2])]) - - - -# lt_combine(SEP, PREFIX-LIST, INFIX, SUFFIX1, [SUFFIX2...]) -# ---------------------------------------------------------- -# Produce a SEP delimited list of all paired combinations of elements of -# PREFIX-LIST with SUFFIX1 through SUFFIXn. Each element of the list -# has the form PREFIXmINFIXSUFFIXn. -# Needed until we can rely on m4_combine added in Autoconf 2.62. -m4_define([lt_combine], -[m4_if(m4_eval([$# > 3]), [1], - [m4_pushdef([_Lt_sep], [m4_define([_Lt_sep], m4_defn([lt_car]))])]]dnl -[[m4_foreach([_Lt_prefix], [$2], - [m4_foreach([_Lt_suffix], - ]m4_dquote(m4_dquote(m4_shift(m4_shift(m4_shift($@)))))[, - [_Lt_sep([$1])[]m4_defn([_Lt_prefix])[$3]m4_defn([_Lt_suffix])])])])]) - - -# lt_if_append_uniq(MACRO-NAME, VARNAME, [SEPARATOR], [UNIQ], [NOT-UNIQ]) -# ----------------------------------------------------------------------- -# Iff MACRO-NAME does not yet contain VARNAME, then append it (delimited -# by SEPARATOR if supplied) and expand UNIQ, else NOT-UNIQ. -m4_define([lt_if_append_uniq], -[m4_ifdef([$1], - [m4_if(m4_index([$3]m4_defn([$1])[$3], [$3$2$3]), [-1], - [lt_append([$1], [$2], [$3])$4], - [$5])], - [lt_append([$1], [$2], [$3])$4])]) - - -# lt_dict_add(DICT, KEY, VALUE) -# ----------------------------- -m4_define([lt_dict_add], -[m4_define([$1($2)], [$3])]) - - -# lt_dict_add_subkey(DICT, KEY, SUBKEY, VALUE) -# -------------------------------------------- -m4_define([lt_dict_add_subkey], -[m4_define([$1($2:$3)], [$4])]) - - -# lt_dict_fetch(DICT, KEY, [SUBKEY]) -# ---------------------------------- -m4_define([lt_dict_fetch], -[m4_ifval([$3], - m4_ifdef([$1($2:$3)], [m4_defn([$1($2:$3)])]), - m4_ifdef([$1($2)], [m4_defn([$1($2)])]))]) - - -# lt_if_dict_fetch(DICT, KEY, [SUBKEY], VALUE, IF-TRUE, [IF-FALSE]) -# ----------------------------------------------------------------- -m4_define([lt_if_dict_fetch], -[m4_if(lt_dict_fetch([$1], [$2], [$3]), [$4], - [$5], - [$6])]) - - -# lt_dict_filter(DICT, [SUBKEY], VALUE, [SEPARATOR], KEY, [...]) -# -------------------------------------------------------------- -m4_define([lt_dict_filter], -[m4_if([$5], [], [], - [lt_join(m4_quote(m4_default([$4], [[, ]])), - lt_unquote(m4_split(m4_normalize(m4_foreach(_Lt_key, lt_car([m4_shiftn(4, $@)]), - [lt_if_dict_fetch([$1], _Lt_key, [$2], [$3], [_Lt_key ])])))))])[]dnl -]) - -# ltversion.m4 -- version numbers -*- Autoconf -*- -# -# Copyright (C) 2004, 2011-2015 Free Software Foundation, Inc. -# Written by Scott James Remnant, 2004 -# -# This file is free software; the Free Software Foundation gives -# unlimited permission to copy and/or distribute it, with or without -# modifications, as long as this notice is preserved. - -# @configure_input@ - -# serial 4179 ltversion.m4 -# This file is part of GNU Libtool - -m4_define([LT_PACKAGE_VERSION], [2.4.6]) -m4_define([LT_PACKAGE_REVISION], [2.4.6]) - -AC_DEFUN([LTVERSION_VERSION], -[macro_version='2.4.6' -macro_revision='2.4.6' -_LT_DECL(, macro_version, 0, [Which release of libtool.m4 was used?]) -_LT_DECL(, macro_revision, 0) -]) - -# lt~obsolete.m4 -- aclocal satisfying obsolete definitions. -*-Autoconf-*- -# -# Copyright (C) 2004-2005, 2007, 2009, 2011-2015 Free Software -# Foundation, Inc. -# Written by Scott James Remnant, 2004. -# -# This file is free software; the Free Software Foundation gives -# unlimited permission to copy and/or distribute it, with or without -# modifications, as long as this notice is preserved. - -# serial 5 lt~obsolete.m4 - -# These exist entirely to fool aclocal when bootstrapping libtool. -# -# In the past libtool.m4 has provided macros via AC_DEFUN (or AU_DEFUN), -# which have later been changed to m4_define as they aren't part of the -# exported API, or moved to Autoconf or Automake where they belong. -# -# The trouble is, aclocal is a bit thick. It'll see the old AC_DEFUN -# in /usr/share/aclocal/libtool.m4 and remember it, then when it sees us -# using a macro with the same name in our local m4/libtool.m4 it'll -# pull the old libtool.m4 in (it doesn't see our shiny new m4_define -# and doesn't know about Autoconf macros at all.) -# -# So we provide this file, which has a silly filename so it's always -# included after everything else. This provides aclocal with the -# AC_DEFUNs it wants, but when m4 processes it, it doesn't do anything -# because those macros already exist, or will be overwritten later. -# We use AC_DEFUN over AU_DEFUN for compatibility with aclocal-1.6. -# -# Anytime we withdraw an AC_DEFUN or AU_DEFUN, remember to add it here. -# Yes, that means every name once taken will need to remain here until -# we give up compatibility with versions before 1.7, at which point -# we need to keep only those names which we still refer to. - -# This is to help aclocal find these macros, as it can't see m4_define. -AC_DEFUN([LTOBSOLETE_VERSION], [m4_if([1])]) - -m4_ifndef([AC_LIBTOOL_LINKER_OPTION], [AC_DEFUN([AC_LIBTOOL_LINKER_OPTION])]) -m4_ifndef([AC_PROG_EGREP], [AC_DEFUN([AC_PROG_EGREP])]) -m4_ifndef([_LT_AC_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_AC_PROG_ECHO_BACKSLASH])]) -m4_ifndef([_LT_AC_SHELL_INIT], [AC_DEFUN([_LT_AC_SHELL_INIT])]) -m4_ifndef([_LT_AC_SYS_LIBPATH_AIX], [AC_DEFUN([_LT_AC_SYS_LIBPATH_AIX])]) -m4_ifndef([_LT_PROG_LTMAIN], [AC_DEFUN([_LT_PROG_LTMAIN])]) -m4_ifndef([_LT_AC_TAGVAR], [AC_DEFUN([_LT_AC_TAGVAR])]) -m4_ifndef([AC_LTDL_ENABLE_INSTALL], [AC_DEFUN([AC_LTDL_ENABLE_INSTALL])]) -m4_ifndef([AC_LTDL_PREOPEN], [AC_DEFUN([AC_LTDL_PREOPEN])]) -m4_ifndef([_LT_AC_SYS_COMPILER], [AC_DEFUN([_LT_AC_SYS_COMPILER])]) -m4_ifndef([_LT_AC_LOCK], [AC_DEFUN([_LT_AC_LOCK])]) -m4_ifndef([AC_LIBTOOL_SYS_OLD_ARCHIVE], [AC_DEFUN([AC_LIBTOOL_SYS_OLD_ARCHIVE])]) -m4_ifndef([_LT_AC_TRY_DLOPEN_SELF], [AC_DEFUN([_LT_AC_TRY_DLOPEN_SELF])]) -m4_ifndef([AC_LIBTOOL_PROG_CC_C_O], [AC_DEFUN([AC_LIBTOOL_PROG_CC_C_O])]) -m4_ifndef([AC_LIBTOOL_SYS_HARD_LINK_LOCKS], [AC_DEFUN([AC_LIBTOOL_SYS_HARD_LINK_LOCKS])]) -m4_ifndef([AC_LIBTOOL_OBJDIR], [AC_DEFUN([AC_LIBTOOL_OBJDIR])]) -m4_ifndef([AC_LTDL_OBJDIR], [AC_DEFUN([AC_LTDL_OBJDIR])]) -m4_ifndef([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH], [AC_DEFUN([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH])]) -m4_ifndef([AC_LIBTOOL_SYS_LIB_STRIP], [AC_DEFUN([AC_LIBTOOL_SYS_LIB_STRIP])]) -m4_ifndef([AC_PATH_MAGIC], [AC_DEFUN([AC_PATH_MAGIC])]) -m4_ifndef([AC_PROG_LD_GNU], [AC_DEFUN([AC_PROG_LD_GNU])]) -m4_ifndef([AC_PROG_LD_RELOAD_FLAG], [AC_DEFUN([AC_PROG_LD_RELOAD_FLAG])]) -m4_ifndef([AC_DEPLIBS_CHECK_METHOD], [AC_DEFUN([AC_DEPLIBS_CHECK_METHOD])]) -m4_ifndef([AC_LIBTOOL_PROG_COMPILER_NO_RTTI], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_NO_RTTI])]) -m4_ifndef([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE], [AC_DEFUN([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE])]) -m4_ifndef([AC_LIBTOOL_PROG_COMPILER_PIC], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_PIC])]) -m4_ifndef([AC_LIBTOOL_PROG_LD_SHLIBS], [AC_DEFUN([AC_LIBTOOL_PROG_LD_SHLIBS])]) -m4_ifndef([AC_LIBTOOL_POSTDEP_PREDEP], [AC_DEFUN([AC_LIBTOOL_POSTDEP_PREDEP])]) -m4_ifndef([LT_AC_PROG_EGREP], [AC_DEFUN([LT_AC_PROG_EGREP])]) -m4_ifndef([LT_AC_PROG_SED], [AC_DEFUN([LT_AC_PROG_SED])]) -m4_ifndef([_LT_CC_BASENAME], [AC_DEFUN([_LT_CC_BASENAME])]) -m4_ifndef([_LT_COMPILER_BOILERPLATE], [AC_DEFUN([_LT_COMPILER_BOILERPLATE])]) -m4_ifndef([_LT_LINKER_BOILERPLATE], [AC_DEFUN([_LT_LINKER_BOILERPLATE])]) -m4_ifndef([_AC_PROG_LIBTOOL], [AC_DEFUN([_AC_PROG_LIBTOOL])]) -m4_ifndef([AC_LIBTOOL_SETUP], [AC_DEFUN([AC_LIBTOOL_SETUP])]) -m4_ifndef([_LT_AC_CHECK_DLFCN], [AC_DEFUN([_LT_AC_CHECK_DLFCN])]) -m4_ifndef([AC_LIBTOOL_SYS_DYNAMIC_LINKER], [AC_DEFUN([AC_LIBTOOL_SYS_DYNAMIC_LINKER])]) -m4_ifndef([_LT_AC_TAGCONFIG], [AC_DEFUN([_LT_AC_TAGCONFIG])]) -m4_ifndef([AC_DISABLE_FAST_INSTALL], [AC_DEFUN([AC_DISABLE_FAST_INSTALL])]) -m4_ifndef([_LT_AC_LANG_CXX], [AC_DEFUN([_LT_AC_LANG_CXX])]) -m4_ifndef([_LT_AC_LANG_F77], [AC_DEFUN([_LT_AC_LANG_F77])]) -m4_ifndef([_LT_AC_LANG_GCJ], [AC_DEFUN([_LT_AC_LANG_GCJ])]) -m4_ifndef([AC_LIBTOOL_LANG_C_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_C_CONFIG])]) -m4_ifndef([_LT_AC_LANG_C_CONFIG], [AC_DEFUN([_LT_AC_LANG_C_CONFIG])]) -m4_ifndef([AC_LIBTOOL_LANG_CXX_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_CXX_CONFIG])]) -m4_ifndef([_LT_AC_LANG_CXX_CONFIG], [AC_DEFUN([_LT_AC_LANG_CXX_CONFIG])]) -m4_ifndef([AC_LIBTOOL_LANG_F77_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_F77_CONFIG])]) -m4_ifndef([_LT_AC_LANG_F77_CONFIG], [AC_DEFUN([_LT_AC_LANG_F77_CONFIG])]) -m4_ifndef([AC_LIBTOOL_LANG_GCJ_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_GCJ_CONFIG])]) -m4_ifndef([_LT_AC_LANG_GCJ_CONFIG], [AC_DEFUN([_LT_AC_LANG_GCJ_CONFIG])]) -m4_ifndef([AC_LIBTOOL_LANG_RC_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_RC_CONFIG])]) -m4_ifndef([_LT_AC_LANG_RC_CONFIG], [AC_DEFUN([_LT_AC_LANG_RC_CONFIG])]) -m4_ifndef([AC_LIBTOOL_CONFIG], [AC_DEFUN([AC_LIBTOOL_CONFIG])]) -m4_ifndef([_LT_AC_FILE_LTDLL_C], [AC_DEFUN([_LT_AC_FILE_LTDLL_C])]) -m4_ifndef([_LT_REQUIRED_DARWIN_CHECKS], [AC_DEFUN([_LT_REQUIRED_DARWIN_CHECKS])]) -m4_ifndef([_LT_AC_PROG_CXXCPP], [AC_DEFUN([_LT_AC_PROG_CXXCPP])]) -m4_ifndef([_LT_PREPARE_SED_QUOTE_VARS], [AC_DEFUN([_LT_PREPARE_SED_QUOTE_VARS])]) -m4_ifndef([_LT_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_PROG_ECHO_BACKSLASH])]) -m4_ifndef([_LT_PROG_F77], [AC_DEFUN([_LT_PROG_F77])]) -m4_ifndef([_LT_PROG_FC], [AC_DEFUN([_LT_PROG_FC])]) -m4_ifndef([_LT_PROG_CXX], [AC_DEFUN([_LT_PROG_CXX])]) - diff --git a/art/icon-243x273.gif b/art/icon-243x273.gif new file mode 100644 index 0000000000..e1cdfd0b51 Binary files /dev/null and b/art/icon-243x273.gif differ diff --git a/art/icon-80x90.gif b/art/icon-80x90.gif new file mode 100644 index 0000000000..ebb2390005 Binary files /dev/null and b/art/icon-80x90.gif differ diff --git a/art/sqlite370.svg b/art/sqlite370.svg new file mode 100644 index 0000000000..9a050b593d --- /dev/null +++ b/art/sqlite370.svg @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/auto.def b/auto.def new file mode 100644 index 0000000000..214ef22304 --- /dev/null +++ b/auto.def @@ -0,0 +1,69 @@ +#!/do/not/tclsh +# ^^^ help out editors which guess this file's content type. +# +# This is the main autosetup-compatible configure script for the +# SQLite project. +# +# This script and all of its dependencies must be kept compatible with +# JimTCL, a copy of which is included in this source tree as +# ./autosetup/jimsh0.c. The number of incompatibilities between +# canonical TCL and JimTCL is very low and alternative formulations of +# incompatible constructs have, so far, been easy to find. +# +# JimTCL: https://jim.tcl.tk +# +# Code-diver notes: APIs names starting with "sqlite-" are specific to +# this project and can be found in autosetup/sqlite-config.tcl. Names +# starting with "proj-" are project-agnostic and found in +# autosetup/proj.tcl. +# +use sqlite-config +sqlite-configure canonical { + proj-if-opt-truthy dev { + # --enable-dev needs to come early so that the downstream tests + # which check for the following flags use their updated state. + proj-opt-set all 1 + proj-opt-set debug 1 + proj-opt-set amalgamation 0 + define CFLAGS [get-env CFLAGS {-O0 -g}] + # -------------^^^^^^^ intentionally using [get-env] instead of + # [proj-get-env] here because [sqlite-setup-default-cflags] uses + # [proj-get-env] and we want this to supercede that. + sqlite-munge-cflags; # straighten out -DSQLITE_ENABLE/OMIT flags + } + sqlite-handle-debug ;# must come after --dev flag check + sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] + sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment + sqlite-check-common-system-deps + + proj-define-for-opt amalgamation USE_AMALGAMATION "Use amalgamation for builds?" + + proj-define-for-opt gcov USE_GCOV "Use gcov?" + + proj-define-for-opt test-status TSTRNNR_OPTS \ + "test-runner flags:" {--status} {} + + proj-define-for-opt linemacros AMALGAMATION_LINE_MACROS \ + "Use #line macros in the amalgamation:" + + define AMALGAMATION_EXTRA_SRC \ + [join [opt-val amalgamation-extra-src ""] " "] + + define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] + + if {[set fsan [join [opt-val asan-fsanitize] ","]] in {auto ""}} { + set fsan address,bounds-strict + } + define CFLAGS_ASAN_FSANITIZE [proj-check-fsanitize [split $fsan ", "]] + + sqlite-handle-tcl + sqlite-handle-emsdk + + proj-if-opt-truthy static-shells { + proj-opt-set static-tclsqlite3 1 + proj-opt-set static-cli-shell 1 + } + proj-define-for-opt static-tclsqlite3 STATIC_TCLSQLITE3 "Statically link tclsqlite3?" + proj-define-for-opt static-cli-shell STATIC_CLI_SHELL "Statically link CLI shell?" + +}; # sqlite-configure diff --git a/autoconf/INSTALL b/autoconf/INSTALL deleted file mode 100644 index a1e89e18ad..0000000000 --- a/autoconf/INSTALL +++ /dev/null @@ -1,370 +0,0 @@ -Installation Instructions -************************* - -Copyright (C) 1994-1996, 1999-2002, 2004-2011 Free Software Foundation, -Inc. - - Copying and distribution of this file, with or without modification, -are permitted in any medium without royalty provided the copyright -notice and this notice are preserved. This file is offered as-is, -without warranty of any kind. - -Basic Installation -================== - - Briefly, the shell commands `./configure; make; make install' should -configure, build, and install this package. The following -more-detailed instructions are generic; see the `README' file for -instructions specific to this package. Some packages provide this -`INSTALL' file but do not implement all of the features documented -below. The lack of an optional feature in a given package is not -necessarily a bug. More recommendations for GNU packages can be found -in *note Makefile Conventions: (standards)Makefile Conventions. - - The `configure' shell script attempts to guess correct values for -various system-dependent variables used during compilation. It uses -those values to create a `Makefile' in each directory of the package. -It may also create one or more `.h' files containing system-dependent -definitions. Finally, it creates a shell script `config.status' that -you can run in the future to recreate the current configuration, and a -file `config.log' containing compiler output (useful mainly for -debugging `configure'). - - It can also use an optional file (typically called `config.cache' -and enabled with `--cache-file=config.cache' or simply `-C') that saves -the results of its tests to speed up reconfiguring. Caching is -disabled by default to prevent problems with accidental use of stale -cache files. - - If you need to do unusual things to compile the package, please try -to figure out how `configure' could check whether to do them, and mail -diffs or instructions to the address given in the `README' so they can -be considered for the next release. If you are using the cache, and at -some point `config.cache' contains results you don't want to keep, you -may remove or edit it. - - The file `configure.ac' (or `configure.in') is used to create -`configure' by a program called `autoconf'. You need `configure.ac' if -you want to change it or regenerate `configure' using a newer version -of `autoconf'. - - The simplest way to compile this package is: - - 1. `cd' to the directory containing the package's source code and type - `./configure' to configure the package for your system. - - Running `configure' might take a while. While running, it prints - some messages telling which features it is checking for. - - 2. Type `make' to compile the package. - - 3. Optionally, type `make check' to run any self-tests that come with - the package, generally using the just-built uninstalled binaries. - - 4. Type `make install' to install the programs and any data files and - documentation. When installing into a prefix owned by root, it is - recommended that the package be configured and built as a regular - user, and only the `make install' phase executed with root - privileges. - - 5. Optionally, type `make installcheck' to repeat any self-tests, but - this time using the binaries in their final installed location. - This target does not install anything. Running this target as a - regular user, particularly if the prior `make install' required - root privileges, verifies that the installation completed - correctly. - - 6. You can remove the program binaries and object files from the - source code directory by typing `make clean'. To also remove the - files that `configure' created (so you can compile the package for - a different kind of computer), type `make distclean'. There is - also a `make maintainer-clean' target, but that is intended mainly - for the package's developers. If you use it, you may have to get - all sorts of other programs in order to regenerate files that came - with the distribution. - - 7. Often, you can also type `make uninstall' to remove the installed - files again. In practice, not all packages have tested that - uninstallation works correctly, even though it is required by the - GNU Coding Standards. - - 8. Some packages, particularly those that use Automake, provide `make - distcheck', which can by used by developers to test that all other - targets like `make install' and `make uninstall' work correctly. - This target is generally not run by end users. - -Compilers and Options -===================== - - Some systems require unusual options for compilation or linking that -the `configure' script does not know about. Run `./configure --help' -for details on some of the pertinent environment variables. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is an example: - - ./configure CC=c99 CFLAGS=-g LIBS=-lposix - - *Note Defining Variables::, for more details. - -Compiling For Multiple Architectures -==================================== - - You can compile the package for more than one kind of computer at the -same time, by placing the object files for each architecture in their -own directory. To do this, you can use GNU `make'. `cd' to the -directory where you want the object files and executables to go and run -the `configure' script. `configure' automatically checks for the -source code in the directory that `configure' is in and in `..'. This -is known as a "VPATH" build. - - With a non-GNU `make', it is safer to compile the package for one -architecture at a time in the source code directory. After you have -installed the package for one architecture, use `make distclean' before -reconfiguring for another architecture. - - On MacOS X 10.5 and later systems, you can create libraries and -executables that work on multiple system types--known as "fat" or -"universal" binaries--by specifying multiple `-arch' options to the -compiler but only a single `-arch' option to the preprocessor. Like -this: - - ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CPP="gcc -E" CXXCPP="g++ -E" - - This is not guaranteed to produce working output in all cases, you -may have to build one architecture at a time and combine the results -using the `lipo' tool if you have problems. - -Installation Names -================== - - By default, `make install' installs the package's commands under -`/usr/local/bin', include files under `/usr/local/include', etc. You -can specify an installation prefix other than `/usr/local' by giving -`configure' the option `--prefix=PREFIX', where PREFIX must be an -absolute file name. - - You can specify separate installation prefixes for -architecture-specific files and architecture-independent files. If you -pass the option `--exec-prefix=PREFIX' to `configure', the package uses -PREFIX as the prefix for installing programs and libraries. -Documentation and other data files still use the regular prefix. - - In addition, if you use an unusual directory layout you can give -options like `--bindir=DIR' to specify different values for particular -kinds of files. Run `configure --help' for a list of the directories -you can set and what kinds of files go in them. In general, the -default for these options is expressed in terms of `${prefix}', so that -specifying just `--prefix' will affect all of the other directory -specifications that were not explicitly provided. - - The most portable way to affect installation locations is to pass the -correct locations to `configure'; however, many packages provide one or -both of the following shortcuts of passing variable assignments to the -`make install' command line to change installation locations without -having to reconfigure or recompile. - - The first method involves providing an override variable for each -affected directory. For example, `make install -prefix=/alternate/directory' will choose an alternate location for all -directory configuration variables that were expressed in terms of -`${prefix}'. Any directories that were specified during `configure', -but not in terms of `${prefix}', must each be overridden at install -time for the entire installation to be relocated. The approach of -makefile variable overrides for each directory variable is required by -the GNU Coding Standards, and ideally causes no recompilation. -However, some platforms have known limitations with the semantics of -shared libraries that end up requiring recompilation when using this -method, particularly noticeable in packages that use GNU Libtool. - - The second method involves providing the `DESTDIR' variable. For -example, `make install DESTDIR=/alternate/directory' will prepend -`/alternate/directory' before all installation names. The approach of -`DESTDIR' overrides is not required by the GNU Coding Standards, and -does not work on platforms that have drive letters. On the other hand, -it does better at avoiding recompilation issues, and works well even -when some directory options were not specified in terms of `${prefix}' -at `configure' time. - -Optional Features -================= - - If the package supports it, you can cause programs to be installed -with an extra prefix or suffix on their names by giving `configure' the -option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. - - Some packages pay attention to `--enable-FEATURE' options to -`configure', where FEATURE indicates an optional part of the package. -They may also pay attention to `--with-PACKAGE' options, where PACKAGE -is something like `gnu-as' or `x' (for the X Window System). The -`README' should mention any `--enable-' and `--with-' options that the -package recognizes. - - For packages that use the X Window System, `configure' can usually -find the X include and library files automatically, but if it doesn't, -you can use the `configure' options `--x-includes=DIR' and -`--x-libraries=DIR' to specify their locations. - - Some packages offer the ability to configure how verbose the -execution of `make' will be. For these packages, running `./configure ---enable-silent-rules' sets the default to minimal output, which can be -overridden with `make V=1'; while running `./configure ---disable-silent-rules' sets the default to verbose, which can be -overridden with `make V=0'. - -Particular systems -================== - - On HP-UX, the default C compiler is not ANSI C compatible. If GNU -CC is not installed, it is recommended to use the following options in -order to use an ANSI C compiler: - - ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" - -and if that doesn't work, install pre-built binaries of GCC for HP-UX. - - HP-UX `make' updates targets which have the same time stamps as -their prerequisites, which makes it generally unusable when shipped -generated files such as `configure' are involved. Use GNU `make' -instead. - - On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot -parse its `' header file. The option `-nodtk' can be used as -a workaround. If GNU CC is not installed, it is therefore recommended -to try - - ./configure CC="cc" - -and if that doesn't work, try - - ./configure CC="cc -nodtk" - - On Solaris, don't put `/usr/ucb' early in your `PATH'. This -directory contains several dysfunctional programs; working variants of -these programs are available in `/usr/bin'. So, if you need `/usr/ucb' -in your `PATH', put it _after_ `/usr/bin'. - - On Haiku, software installed for all users goes in `/boot/common', -not `/usr/local'. It is recommended to use the following options: - - ./configure --prefix=/boot/common - -Specifying the System Type -========================== - - There may be some features `configure' cannot figure out -automatically, but needs to determine by the type of machine the package -will run on. Usually, assuming the package is built to be run on the -_same_ architectures, `configure' can figure that out, but if it prints -a message saying it cannot guess the machine type, give it the -`--build=TYPE' option. TYPE can either be a short name for the system -type, such as `sun4', or a canonical name which has the form: - - CPU-COMPANY-SYSTEM - -where SYSTEM can have one of these forms: - - OS - KERNEL-OS - - See the file `config.sub' for the possible values of each field. If -`config.sub' isn't included in this package, then this package doesn't -need to know the machine type. - - If you are _building_ compiler tools for cross-compiling, you should -use the option `--target=TYPE' to select the type of system they will -produce code for. - - If you want to _use_ a cross compiler, that generates code for a -platform different from the build platform, you should specify the -"host" platform (i.e., that on which the generated programs will -eventually be run) with `--host=TYPE'. - -Sharing Defaults -================ - - If you want to set default values for `configure' scripts to share, -you can create a site shell script called `config.site' that gives -default values for variables like `CC', `cache_file', and `prefix'. -`configure' looks for `PREFIX/share/config.site' if it exists, then -`PREFIX/etc/config.site' if it exists. Or, you can set the -`CONFIG_SITE' environment variable to the location of the site script. -A warning: not all `configure' scripts look for a site script. - -Defining Variables -================== - - Variables not defined in a site shell script can be set in the -environment passed to `configure'. However, some packages may run -configure again during the build, and the customized values of these -variables may be lost. In order to avoid this problem, you should set -them in the `configure' command line, using `VAR=value'. For example: - - ./configure CC=/usr/local2/bin/gcc - -causes the specified `gcc' to be used as the C compiler (unless it is -overridden in the site shell script). - -Unfortunately, this technique does not work for `CONFIG_SHELL' due to -an Autoconf bug. Until the bug is fixed you can use this workaround: - - CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash - -`configure' Invocation -====================== - - `configure' recognizes the following options to control how it -operates. - -`--help' -`-h' - Print a summary of all of the options to `configure', and exit. - -`--help=short' -`--help=recursive' - Print a summary of the options unique to this package's - `configure', and exit. The `short' variant lists options used - only in the top level, while the `recursive' variant lists options - also present in any nested packages. - -`--version' -`-V' - Print the version of Autoconf used to generate the `configure' - script, and exit. - -`--cache-file=FILE' - Enable the cache: use and save the results of the tests in FILE, - traditionally `config.cache'. FILE defaults to `/dev/null' to - disable caching. - -`--config-cache' -`-C' - Alias for `--cache-file=config.cache'. - -`--quiet' -`--silent' -`-q' - Do not print messages saying which checks are being made. To - suppress all normal output, redirect it to `/dev/null' (any error - messages will still be shown). - -`--srcdir=DIR' - Look for the package's source code in directory DIR. Usually - `configure' can determine that directory automatically. - -`--prefix=DIR' - Use DIR as the installation prefix. *note Installation Names:: - for more details, including other options available for fine-tuning - the installation locations. - -`--no-create' -`-n' - Run the configure checks, but stop before creating any output - files. - -`configure' also accepts some other, not widely useful, options. Run -`configure --help' for more details. - diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am deleted file mode 100644 index 694419b27d..0000000000 --- a/autoconf/Makefile.am +++ /dev/null @@ -1,20 +0,0 @@ - -AM_CFLAGS = @BUILD_CFLAGS@ -lib_LTLIBRARIES = libsqlite3.la -libsqlite3_la_SOURCES = sqlite3.c -libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 - -bin_PROGRAMS = sqlite3 -sqlite3_SOURCES = shell.c sqlite3.h -EXTRA_sqlite3_SOURCES = sqlite3.c -sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ -sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ -sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS) - -include_HEADERS = sqlite3.h sqlite3ext.h - -EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc sqlite3rc.h README.txt Replace.cs Makefile.fallback -pkgconfigdir = ${libdir}/pkgconfig -pkgconfig_DATA = sqlite3.pc - -man_MANS = sqlite3.1 diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in new file mode 100644 index 0000000000..c938ffe1bf --- /dev/null +++ b/autoconf/Makefile.in @@ -0,0 +1,314 @@ +######################################################################## +# This is a main makefile for the "autoconf" bundle of SQLite. This is +# a trimmed-down version of the canonical makefile, devoid of most +# documentation. For the full docs, see /main.mk in the canonical +# source tree. +# +# Maintenance reminders: +# +# - To keep this working with an out-of-tree build, be sure to prefix +# input file names with $(TOP)/ where appropriate (which is most +# places). +# +# - The original/canonical recipes can be found in /main.mk in the +# canonical source tree. +all: + +TOP = @abs_top_srcdir@ + +PACKAGE_VERSION = @PACKAGE_VERSION@ + +# +# Filename extensions for binaries and libraries +# +B.exe = @BUILD_EXEEXT@ +T.exe = @TARGET_EXEEXT@ +B.dll = @BUILD_DLLEXT@ +T.dll = @TARGET_DLLEXT@ +B.lib = @BUILD_LIBEXT@ +T.lib = @TARGET_LIBEXT@ + +# +# Autotools-compatibility dirs +# +prefix = @prefix@ +datadir = @datadir@ +mandir = @mandir@ +includedir = @includedir@ +exec_prefix = @exec_prefix@ +bindir = @bindir@ +libdir = @libdir@ + +# +# Required binaries +# +INSTALL = @BIN_INSTALL@ +AR = @AR@ +AR.flags = cr +CC = @CC@ + + +ENABLE_LIB_SHARED = @ENABLE_LIB_SHARED@ +ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ +HAVE_WASI_SDK = @HAVE_WASI_SDK@ + +CFLAGS = @CFLAGS@ @CPPFLAGS@ +# +# $(LDFLAGS.configure) represents any LDFLAGS=... the client passes to +# configure. See main.mk. +# +LDFLAGS.configure = @LDFLAGS@ + +CFLAGS.core = @SH_CFLAGS@ +LDFLAGS.shlib = @SH_LDFLAGS@ +LDFLAGS.zlib = @LDFLAGS_ZLIB@ +LDFLAGS.math = @LDFLAGS_MATH@ +LDFLAGS.rpath = @LDFLAGS_RPATH@ +LDFLAGS.pthread = @LDFLAGS_PTHREAD@ +LDFLAGS.dlopen = @LDFLAGS_DLOPEN@ +LDFLAGS.readline = @LDFLAGS_READLINE@ +CFLAGS.readline = @CFLAGS_READLINE@ +LDFLAGS.rt = @LDFLAGS_RT@ +LDFLAGS.icu = @LDFLAGS_ICU@ +CFLAGS.icu = @CFLAGS_ICU@ + +# INSTALL reminder: we specifically do not strip binaries, +# as discussed in https://sqlite.org/forum/forumpost/9a67df63eda9925c. +INSTALL.noexec = $(INSTALL) -m 0644 + +install-dir.bin = $(DESTDIR)$(bindir) +install-dir.lib = $(DESTDIR)$(libdir) +install-dir.include = $(DESTDIR)$(includedir) +install-dir.pkgconfig = $(DESTDIR)$(libdir)/pkgconfig +install-dir.man1 = $(DESTDIR)$(mandir)/man1 +install-dir.all = $(install-dir.bin) $(install-dir.include) \ + $(install-dir.lib) $(install-dir.man1) \ + $(install-dir.pkgconfig) +$(install-dir.all): + @if [ ! -d "$@" ]; then set -x; $(INSTALL) -d "$@"; fi +# ^^^^ on some platforms, install -d fails if the target already exists. + + +# +# Vars with the AS_ prefix are specifically related to AutoSetup. +# +# AS_AUTO_DEF is the main configure script. +# +AS_AUTO_DEF = $(TOP)/auto.def + +# +# Shell commands to re-run $(TOP)/configure with the same args it was +# invoked with to produce this makefile. +# +AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@ +Makefile: $(TOP)/Makefile.in $(AS_AUTO_DEF) + $(AS_AUTORECONFIG) + @touch $@ + +sqlite3.pc: $(TOP)/sqlite3.pc.in $(AS_AUTO_DEF) + $(AS_AUTORECONFIG) + @touch $@ + +sqlite_cfg.h: $(AS_AUTO_DEF) + $(AS_AUTORECONFIG) + @touch $@ + +# +# CFLAGS for sqlite3$(T.exe) +# +SHELL_OPT ?= @OPT_SHELL@ +SHELL_OPT += -DSQLITE_DQS=0 +SHELL_OPT += -DSQLITE_ENABLE_FTS4 +#SHELL_OPT += -DSQLITE_ENABLE_FTS5 +SHELL_OPT += -DSQLITE_ENABLE_RTREE +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB +SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 + +# +# Library-level feature flags +# +OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ + +LDFLAGS.libsqlite3.soname = @LDFLAGS_LIBSQLITE3_SONAME@ +# soname: see https://sqlite.org/src/forumpost/5a3b44f510df8ded +LDFLAGS.libsqlite3.os-specific = \ + @LDFLAGS_MAC_CVERSION@ @LDFLAGS_MAC_INSTALL_NAME@ @LDFLAGS_OUT_IMPLIB@ + +LDFLAGS.libsqlite3 = \ + $(LDFLAGS.rpath) $(LDFLAGS.pthread) \ + $(LDFLAGS.math) $(LDFLAGS.dlopen) \ + $(LDFLAGS.zlib) $(LDFLAGS.icu) \ + $(LDFLAGS.rt) $(LDFLAGS.configure) +CFLAGS.libsqlite3 = -I. $(CFLAGS.core) $(CFLAGS.icu) $(OPT_FEATURE_FLAGS) + +sqlite3.o: $(TOP)/sqlite3.h $(TOP)/sqlite3.c + $(CC) -c $(TOP)/sqlite3.c -o $@ $(CFLAGS) $(CFLAGS.libsqlite3) + +libsqlite3.LIB = libsqlite3$(T.lib) +libsqlite3.DLL.basename = @SQLITE_DLL_BASENAME@ +libsqlite3.out.implib = @SQLITE_OUT_IMPLIB@ +libsqlite3.DLL = $(libsqlite3.DLL.basename)$(T.dll) +libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ + +$(libsqlite3.DLL): sqlite3.o + $(CC) -o $@ sqlite3.o $(LDFLAGS.shlib) \ + $(LDFLAGS) $(LDFLAGS.libsqlite3) \ + $(LDFLAGS.libsqlite3.os-specific) $(LDFLAGS.libsqlite3.soname) +$(libsqlite3.DLL)-1: $(libsqlite3.DLL) +$(libsqlite3.DLL)-0: +all: $(libsqlite3.DLL)-$(ENABLE_LIB_SHARED) + +$(libsqlite3.LIB): sqlite3.o + $(AR) $(AR.flags) $@ sqlite3.o +$(libsqlite3.LIB)-1: $(libsqlite3.LIB) +$(libsqlite3.LIB)-0: +all: $(libsqlite3.LIB)-$(ENABLE_LIB_STATIC) + +# +# Maintenance reminder: the install-dll-... rules must be kept in sync +# with the main copies rom /main.mk. +# +install-dll-out-implib: $(install-dir.lib) $(libsqlite3.DLL) + if [ x != "x$(libsqlite3.out.implib)" ] && [ -f "$(libsqlite3.out.implib)" ]; then \ + $(INSTALL) $(libsqlite3.out.implib) "$(install-dir.lib)"; \ + fi + +install-dll-unix-generic: install-dll-out-implib + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f $(libsqlite3.DLL).0 $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + mv $(libsqlite3.DLL) $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0 || exit $$?; \ + ls -la $(libsqlite3.DLL) $(libsqlite3.DLL).[a03]*; \ + if [ -e $(libsqlite3.DLL).0.8.6 ]; then \ + echo "ACHTUNG: legacy libtool-compatible install found. Re-linking it..."; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ + elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \ + echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ + fi + +install-dll-msys: install-dll-out-implib $(install-dir.bin) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.bin)" +# ----------------------------------------------^^^ yes, bin +# Each of {msys,mingw,cygwin} uses a different name for the DLL, but +# that is already accounted for via $(libsqlite3.DLL). +install-dll-mingw: install-dll-msys +install-dll-cygwin: install-dll-msys + +install-dll-darwin: $(install-dir.lib) $(libsqlite3.DLL) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f libsqlite3.0$(T.dll) libsqlite3.$(PACKAGE_VERSION)$(T.dll) || exit $$?; \ + dllname=libsqlite3.$(PACKAGE_VERSION)$(T.dll); \ + mv $(libsqlite3.DLL) $$dllname || exit $$?; \ + ln -s $$dllname $(libsqlite3.DLL) || exit $$?; \ + ln -s $$dllname libsqlite3.0$(T.dll) || exit $$?; \ + ls -la $$dllname $(libsqlite3.DLL) libsqlite3.0$(T.dll) + +install-dll-1: install-dll-$(libsqlite3.DLL.install-rules) +install-dll-0 install-dll-: +install-dll: install-dll-$(ENABLE_LIB_SHARED) +install: install-dll + +install-lib-1: $(install-dir.lib) $(libsqlite3.LIB) + $(INSTALL.noexec) $(libsqlite3.LIB) "$(install-dir.lib)" +install-lib-0 install-lib-: +install-lib: install-lib-$(ENABLE_LIB_STATIC) +install: install-lib + +# +# Flags to link the shell app either directly against sqlite3.c +# (ENABLE_STATIC_SHELL==1) or libsqlite3.so (ENABLE_STATIC_SHELL==0). +# +# Maintenance reminder: placement of $(LDFLAGS) is more relevant for +# some platforms than others: +# https://sqlite.org/forum/forumpost/d80ecdaddd +ENABLE_STATIC_SHELL = @ENABLE_STATIC_SHELL@ +sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS) $(LDFLAGS.libsqlite3) +sqlite3-shell-link-flags.0 = $(LDFLAGS) -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) +sqlite3-shell-deps.1 = $(TOP)/sqlite3.c +sqlite3-shell-deps.0 = $(libsqlite3.DLL) +# +# STATIC_CLI_SHELL = 1 to statically link sqlite3$(T.exe), else +# 0. Requires static versions of all requisite libraries. Primarily +# intended for use with static-friendly environments like Alpine +# Linux. +# +STATIC_CLI_SHELL = @STATIC_CLI_SHELL@ +# +# sqlite3-shell-static.flags.N = N is $(STATIC_CLI_SHELL) +# +sqlite3-shell-static.flags.1 = -static +sqlite3-shell-static.flags.0 = +sqlite3$(T.exe): $(TOP)/shell.c $(sqlite3-shell-deps.$(ENABLE_STATIC_SHELL)) + $(CC) -o $@ \ + $(TOP)/shell.c $(sqlite3-shell-link-flags.$(ENABLE_STATIC_SHELL)) \ + $(sqlite3-shell-static.flags.$(STATIC_CLI_SHELL)) \ + -I. $(OPT_FEATURE_FLAGS) $(SHELL_OPT) \ + $(CFLAGS) $(CFLAGS.readline) $(CFLAGS.icu) \ + $(LDFLAGS.readline) + +sqlite3$(T.exe)-1: +sqlite3$(T.exe)-0: sqlite3$(T.exe) +all: sqlite3$(T.exe)-$(HAVE_WASI_SDK) + +install-shell-0: sqlite3$(T.exe) $(install-dir.bin) + $(INSTALL) sqlite3$(T.exe) "$(install-dir.bin)" +install-shell-1: +install: install-shell-$(HAVE_WASI_SDK) + +install-headers: $(TOP)/sqlite3.h $(install-dir.include) + $(INSTALL.noexec) $(TOP)/sqlite3.h $(TOP)/sqlite3ext.h "$(install-dir.include)" +install: install-headers + +install-pc: sqlite3.pc $(install-dir.pkgconfig) + $(INSTALL.noexec) sqlite3.pc "$(install-dir.pkgconfig)" +install: install-pc + +install-man1: $(TOP)/sqlite3.1 $(install-dir.man1) + $(INSTALL.noexec) $(TOP)/sqlite3.1 "$(install-dir.man1)" +install: install-man1 + +clean: + rm -f *.o sqlite3$(T.exe) + rm -f $(libsqlite3.LIB) $(libsqlite3.DLL) libsqlite3$(T.dll).a + +distclean: clean + rm -f jimsh0$(T.exe) config.* sqlite3.pc sqlite_cfg.h Makefile + +DIST_FILES := \ + README.txt VERSION \ + auto.def autosetup configure tea \ + sqlite3.h sqlite3.c shell.c sqlite3ext.h \ + Makefile.in Makefile.msc Makefile.fallback \ + sqlite3.rc sqlite3rc.h Replace.cs \ + sqlite3.pc.in sqlite3.1 + +# +# Maintenance note: dist_name must be sqlite-$(PACKAGE_VERSION) so +# that tool/mkautoconfamal.sh knows how to find it. +# +dist_name = sqlite-$(PACKAGE_VERSION) +dist_tarball = $(dist_name).tar.gz +dist: + rm -fr $(dist_name) + mkdir -p $(dist_name) + cp -rp $(DIST_FILES) $(dist_name)/. + tar czf $(dist_tarball) $(dist_name) + rm -fr $(dist_name) + ls -l $(dist_tarball) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 09daa867ec..365e1f0fb5 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -18,6 +18,15 @@ TOP = . +# Optionally set EXTRA_SRC to a list of C files to append to +# the generated sqlite3.c. Any sqlite3 extensions added this +# way may require manual editing, as described in +# https://sqlite.org/forum/forumpost/903f721f3e7c0d25 +# +!IFNDEF EXTRA_SRC +EXTRA_SRC = +!ENDIF + # Set this non-0 to enable full warnings (-W4, etc) when compiling. # !IFNDEF USE_FULLWARN @@ -52,6 +61,21 @@ MINIMAL_AMALGAMATION = 0 USE_STDCALL = 0 !ENDIF +# Use the USE_SEH=0 option on the nmake command line to omit structured +# exception handling (SEH) support. SEH is on by default. +# +!IFNDEF USE_SEH +USE_SEH = 1 +!ENDIF + +# Use STATICALLY_LINK_TCL=1 to statically link against TCL +# +!IFNDEF STATICALLY_LINK_TCL +STATICALLY_LINK_TCL = 0 +!ELSEIF $(STATICALLY_LINK_TCL)!=0 +CCOPTS = $(CCOPTS) -DSTATIC_BUILD +!ENDIF + # Set this non-0 to have the shell executable link against the core dynamic # link library. # @@ -180,6 +204,12 @@ WIN32HEAP = 0 OSTRACE = 0 !ENDIF +# enable address sanitizer using ASAN=1 on the command-line. +# +!IFNDEF ASAN +ASAN = 0 +!ENDIF + # Set this to one of the following values to enable various debugging # features. Each level includes the debugging options from the previous # levels. Currently, the recognized values for DEBUG are: @@ -217,6 +247,12 @@ SESSION = 0 RBU = 0 !ENDIF +# Set this to non-0 to enable support for blocking locks. +# +!IFNDEF SETLK_TIMEOUT +SETLK_TIMEOUT = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -281,18 +317,29 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +OPT_FEATURE_FLAGS = $(OPT_XTRA) !IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Additional feature-options above and beyond what are normally used can be +# be added using OPTIONS=.... on the command-line. These values are +# appended to the OPT_FEATURE_FLAGS variable. +# +!IFDEF OPTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS) +!ENDIF + # Should the session extension be enabled? If so, add compilation options # to enable it. # @@ -303,6 +350,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 # Always enable math functions on Windows OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE # Should the rbu extension be enabled? If so, add compilation options # to enable it. @@ -311,6 +359,14 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1 !ENDIF +# Should structured exception handling (SEH) be enabled for WAL mode in +# the core library? It is on by default. Only omit it if the +# USE_SEH=0 option is provided on the nmake command-line. +# +!IF $(USE_SEH)==0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1 +!ENDIF + # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # @@ -324,6 +380,10 @@ EXT_FEATURE_FLAGS = !ENDIF !ENDIF +!IF $(SETLK_TIMEOUT)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SETLK_TIMEOUT +!ENDIF + ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### @@ -635,17 +695,21 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd +ZLIBCFLAGS = -nologo -MDd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MD BCC = $(BCC) -MD +ZLIBCFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi !ENDIF !ELSE !IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd +ZLIBCFLAGS = -nologo -MTd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT +ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF !ENDIF @@ -666,7 +730,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_DEBUG=1 -DSQLITE_USE_W32_FOR_CONSOLE_IO RCC = $(RCC) -DSQLITE_DEBUG=1 !IF $(DYNAMIC_SHELL)==0 TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE @@ -718,6 +782,13 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF +# Address sanitizer if ASAN=1 +# +!IF $(ASAN)>0 +TCC = $(TCC) /fsanitize=address +!ENDIF + + # Compiler options needed for programs that use the readline() library. # !IFNDEF READLINE_FLAGS @@ -746,15 +817,6 @@ RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 TLIBS = !ENDIF -# Flags controlling use of the in memory btree implementation -# -# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to -# default to file, 2 to default to memory, and 3 to force temporary -# tables to always be in memory. -# -TCC = $(TCC) -DSQLITE_TEMP_STORE=1 -RCC = $(RCC) -DSQLITE_TEMP_STORE=1 - # Enable/disable loadable extensions, and other optional features # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). # The same set of OMIT and ENABLE flags should be passed to the @@ -959,6 +1021,10 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -983,6 +1049,11 @@ dll: $(SQLITE3DLL) # shell: $(SQLITE3EXE) +# jimsh0 - replacement for tclsh +# +jimsh0.exe: $(TOP)\autosetup\jimsh0.c + cl -DHAVE__FULLPATH=1 $(TOP)\autosetup\jimsh0.c + $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) @@ -1033,5 +1104,6 @@ $(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL - del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q sqlite3.def tclsqlite3.def ctime.c pragma.h 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL diff --git a/autoconf/README.first b/autoconf/README.first index 5c2ea0a70f..75c4a76d61 100644 --- a/autoconf/README.first +++ b/autoconf/README.first @@ -1,11 +1,12 @@ -This directory contains components use to build an autoconf-ready package -of the SQLite amalgamation: sqlite-autoconf-30XXXXXX.tar.gz +This directory contains components used to build an autoconf-like +package of the SQLite amalgamation: sqlite-autoconf-30XXXXXX.tar.gz -To build the autoconf amalgamation, run from the top-level: +To build the autoconf amalgamation, run from the top of the canonical +source tree: ./configure make amalgamation-tarball -The amalgamation-tarball target (also available in "main.mk") runs the -script tool/mkautoconfamal.sh which does the work. Refer to that script -for details. +The amalgamation-tarball target (available in "main.mk") runs the +script tool/mkautoconfamal.sh which does the work. Refer to that +script for details. diff --git a/autoconf/README.txt b/autoconf/README.txt index ccf5e235ac..ca0ed20fd4 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -4,13 +4,44 @@ This package contains: * the sqlite3.h and sqlite3ext.h header files that define the C-language interface to the sqlite3.c library file * the shell.c file used to build the sqlite3 command-line shell program - * autoconf/automake installation infrastucture for building on POSIX + * autoconf-like installation infrastucture for building on POSIX compliant systems * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft Visual C++ on Windows -SUMMARY OF HOW TO BUILD -======================= +WHY USE THIS PACKAGE? +===================== + +The canonical make system for SQLite requires TCL as part of the build +process. Various TCL scripts are used to generate parts of the code and +TCL is used to run tests. But some people would prefer to build SQLite +using only generic tools and without having to install TCL. The purpose +of this package is to provide that capability. + +This package contains a pre-build SQLite amalgamation file "sqlite3.c" +(and its associated header file "sqlite3.h"). Because the +amalgamation has been pre-built, no TCL is required for the code +generate (the configure script itself is written in TCL but it can use +the embedded copy of JimTCL). + +REASONS TO USE THE CANONICAL BUILD SYSTEM RATHER THAN THIS PACKAGE +================================================================== + + * the canonical build system allows you to run tests to verify that + the build worked + * the canonical build system supports more compile-time options + * the canonical build system works for any arbitrary check-in to + the SQLite source tree + +Step-by-step instructions on how to build using the canonical make +system for SQLite can be found at: + + https://sqlite.org/src/doc/trunk/doc/compile-for-unix.md + https://sqlite.org/src/doc/trunk/doc/compile-for-windows.md + + +SUMMARY OF HOW TO BUILD USING THIS PACKAGE +========================================== Unix: ./configure; make Windows: nmake /f Makefile.msc @@ -18,14 +49,12 @@ SUMMARY OF HOW TO BUILD BUILDING ON POSIX ================= -The generic installation instructions for autoconf/automake are found -in the INSTALL file. - -The following SQLite specific boolean options are supported: +The configure script follows common conventions, making it easy +to use for anyone who has configured a software tree before. +It supports a number of build-time flags, the full list of which +can be seen by running: - --enable-readline use readline in shell tool [default=yes] - --enable-threadsafe build a thread-safe library [default=yes] - --enable-dynamic-extensions support loadable extensions [default=yes] + ./configure --help The default value for the CFLAGS variable (options passed to the C compiler) includes debugging symbols in the build, resulting in larger @@ -36,10 +65,11 @@ line like this: to produce a smaller installation footprint. -Other SQLite compilation parameters can also be set using CFLAGS. For +Many SQLite compilation parameters can be defined by passing flags +to the configure script. Others may be passed on in the CFLAGS. For example: - $ CFLAGS="-Os -DSQLITE_THREADSAFE=0" ./configure + $ CFLAGS="-Os -DSQLITE_OMIT_DEPRECATED" ./configure BUILDING WITH MICROSOFT VISUAL C++ @@ -53,48 +83,6 @@ Using Microsoft Visual C++ 2005 (or later) is recommended. Several Windows platform variants may be built by adding additional macros to the NMAKE command line. -Building for WinRT 8.0 ----------------------- - - FOR_WINRT=1 - -Using Microsoft Visual C++ 2012 (or later) is required. When using the -above, something like the following macro will need to be added to the -NMAKE command line as well: - - "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86" - -Building for WinRT 8.1 ----------------------- - - FOR_WINRT=1 - -Using Microsoft Visual C++ 2013 (or later) is required. When using the -above, something like the following macro will need to be added to the -NMAKE command line as well: - - "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86" - -Building for UWP 10.0 ---------------------- - - FOR_WINRT=1 FOR_UWP=1 - -Using Microsoft Visual C++ 2015 (or later) is required. When using the -above, something like the following macros will need to be added to the -NMAKE command line as well: - - "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" - "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86" - "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86" - -Building for the Windows 10 SDK -------------------------------- - - FOR_WIN10=1 - -Using Microsoft Visual C++ 2015 (or later) is required. When using the -above, no other macros should be needed on the NMAKE command line. Other preprocessor defines -------------------------- @@ -102,7 +90,7 @@ Other preprocessor defines Additionally, preprocessor defines may be specified by using the OPTS macro on the NMAKE command line. However, not all possible preprocessor defines may be specified in this manner as some require the amalgamation to be built -with them enabled (see http://www.sqlite.org/compile.html). For example, the +with them enabled (see http://sqlite.org/compile.html). For example, the following will work: "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_JSON=1" diff --git a/autoconf/auto.def b/autoconf/auto.def new file mode 100644 index 0000000000..c61d81e506 --- /dev/null +++ b/autoconf/auto.def @@ -0,0 +1,25 @@ +#!/do/not/tclsh +# ^^^ help out editors which guess this file's content type. +# +# This is the main autosetup-compatible configure script for the +# "autoconf" bundle of the SQLite project. +use sqlite-config +sqlite-configure autoconf { + sqlite-handle-debug + sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] + sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment + sqlite-check-common-system-deps + proj-define-for-opt static-shell ENABLE_STATIC_SHELL \ + "Link library statically into the CLI shell?" + proj-define-for-opt static-cli-shell STATIC_CLI_SHELL "Statically link CLI shell?" + if {![opt-bool static-shell] && [opt-bool static-cli-shell]} { + proj-fatal "--disable-static-shell and --static-cli-shell are mutualy exclusive" + } + if {![opt-bool shared] && ![opt-bool static-shell]} { + proj-opt-set shared 1 + proj-indented-notice { + NOTICE: ignoring --disable-shared because --disable-static-shell + was specified. + } + } +} diff --git a/autoconf/configure.ac b/autoconf/configure.ac deleted file mode 100644 index 0c7a32db1f..0000000000 --- a/autoconf/configure.ac +++ /dev/null @@ -1,270 +0,0 @@ - -#----------------------------------------------------------------------- -# Supports the following non-standard switches. -# -# --enable-threadsafe -# --enable-readline -# --enable-editline -# --enable-static-shell -# --enable-dynamic-extensions -# - -AC_PREREQ(2.61) -AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org) -AC_CONFIG_SRCDIR([sqlite3.c]) -AC_CONFIG_AUX_DIR([.]) - -# Use automake. -AM_INIT_AUTOMAKE([foreign]) - -AC_SYS_LARGEFILE - -# Check for required programs. -AC_PROG_CC -AC_PROG_LIBTOOL -AC_PROG_MKDIR_P - -# Check for library functions that SQLite can optionally use. -AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r]) -AC_FUNC_STRERROR_R - -AC_CONFIG_FILES([Makefile sqlite3.pc]) -BUILD_CFLAGS= -AC_SUBST(BUILD_CFLAGS) - -#------------------------------------------------------------------------- -# Two options to enable readline compatible libraries: -# -# --enable-editline -# --enable-readline -# -# Both are enabled by default. If, after command line processing both are -# still enabled, the script searches for editline first and automatically -# disables readline if it is found. So, to use readline explicitly, the -# user must pass "--disable-editline". To disable command line editing -# support altogether, "--disable-editline --disable-readline". -# -# When searching for either library, check for headers before libraries -# as some distros supply packages that contain libraries but not header -# files, which come as a separate development package. -# -AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])]) -AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])]) - -AS_IF([ test x"$enable_editline" != xno ],[ - AC_CHECK_HEADERS([editline/readline.h],[ - sLIBS=$LIBS - LIBS="" - AC_SEARCH_LIBS([readline],[edit],[ - AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) - READLINE_LIBS="$LIBS -ltinfo" - enable_readline=no - ],[],[-ltinfo]) - AS_UNSET(ac_cv_search_readline) - LIBS=$sLIBS - ]) -]) - -AS_IF([ test x"$enable_readline" != xno ],[ - AC_CHECK_HEADERS([readline/readline.h],[ - sLIBS=$LIBS - LIBS="" - AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], []) - AC_SEARCH_LIBS(readline,[readline edit], [ - AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper) - READLINE_LIBS=$LIBS - ]) - LIBS=$sLIBS - ]) -]) - -AC_SUBST(READLINE_LIBS) -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-threadsafe -# -AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( - [--enable-threadsafe], [build a thread-safe library [default=yes]])], - [], [enable_threadsafe=yes]) -if test x"$enable_threadsafe" == "xno"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_THREADSAFE=0" -else - BUILD_CFLAGS="$BUILD_CFLAGS -D_REENTRANT=1 -DSQLITE_THREADSAFE=1" - AC_SEARCH_LIBS(pthread_create, pthread) - AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-dynamic-extensions -# -AC_ARG_ENABLE(dynamic-extensions, [AS_HELP_STRING( - [--enable-dynamic-extensions], [support loadable extensions [default=yes]])], - [], [enable_dynamic_extensions=yes]) -if test x"$enable_dynamic_extensions" != "xno"; then - AC_SEARCH_LIBS(dlopen, dl) -else - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_OMIT_LOAD_EXTENSION=1" -fi -AC_MSG_CHECKING([for whether to support dynamic extensions]) -AC_MSG_RESULT($enable_dynamic_extensions) -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-math -# -AC_ARG_ENABLE(math, [AS_HELP_STRING( - [--enable-math], [SQL math functions [default=yes]])], - [], [enable_math=yes]) -AC_MSG_CHECKING([SQL math functions]) -if test x"$enable_math" = "xyes"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_MATH_FUNCTIONS" - AC_MSG_RESULT([enabled]) - AC_SEARCH_LIBS(ceil, m) -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-fts4 -# -AC_ARG_ENABLE(fts4, [AS_HELP_STRING( - [--enable-fts4], [include fts4 support [default=yes]])], - [], [enable_fts4=yes]) -AC_MSG_CHECKING([FTS4 extension]) -if test x"$enable_fts4" = "xyes"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS4" - AC_MSG_RESULT([enabled]) -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-fts3 -# -AC_ARG_ENABLE(fts3, [AS_HELP_STRING( - [--enable-fts3], [include fts3 support [default=no]])], - [], []) -AC_MSG_CHECKING([FTS3 extension]) -if test x"$enable_fts3" = "xyes" -a x"$enable_fts4" = "xno"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS3" - AC_MSG_RESULT([enabled]) -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-fts5 -# -AC_ARG_ENABLE(fts5, [AS_HELP_STRING( - [--enable-fts5], [include fts5 support [default=yes]])], - [], [enable_fts5=yes]) -AC_MSG_CHECKING([FTS5 extension]) -if test x"$enable_fts5" = "xyes"; then - AC_MSG_RESULT([enabled]) - AC_SEARCH_LIBS(log, m) - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_FTS5" -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-rtree -# -AC_ARG_ENABLE(rtree, [AS_HELP_STRING( - [--enable-rtree], [include rtree support [default=yes]])], - [], [enable_rtree=yes]) -AC_MSG_CHECKING([RTREE extension]) -if test x"$enable_rtree" = "xyes"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_GEOPOLY" - AC_MSG_RESULT([enabled]) -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-session -# -AC_ARG_ENABLE(session, [AS_HELP_STRING( - [--enable-session], [enable the session extension [default=no]])], - [], []) -AC_MSG_CHECKING([Session extension]) -if test x"$enable_session" = "xyes"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK" - AC_MSG_RESULT([enabled]) -else - AC_MSG_RESULT([disabled]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-debug -# -AC_ARG_ENABLE(debug, [AS_HELP_STRING( - [--enable-debug], [build with debugging features enabled [default=no]])], - [], []) -AC_MSG_CHECKING([Build type]) -if test x"$enable_debug" = "xyes"; then - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_DEBUG -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE" - CFLAGS="-g -O0" - AC_MSG_RESULT([debug]) -else - AC_MSG_RESULT([release]) -fi -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# --enable-static-shell -# -AC_ARG_ENABLE(static-shell, [AS_HELP_STRING( - [--enable-static-shell], - [statically link libsqlite3 into shell tool [default=yes]])], - [], [enable_static_shell=yes]) -if test x"$enable_static_shell" = "xyes"; then - EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT -else - EXTRA_SHELL_OBJ=libsqlite3.la -fi -AC_SUBST(EXTRA_SHELL_OBJ) -#----------------------------------------------------------------------- - -AC_CHECK_FUNCS(posix_fallocate) -AC_CHECK_HEADERS(zlib.h,[ - AC_SEARCH_LIBS(deflate,z,[BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_HAVE_ZLIB"]) -]) - -AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"]) -AC_SUBST(SHELL_CFLAGS) - -#----------------------------------------------------------------------- -# UPDATE: Maybe it's better if users just set CFLAGS before invoking -# configure. This option doesn't really add much... -# -# --enable-tempstore -# -# AC_ARG_ENABLE(tempstore, [AS_HELP_STRING( -# [--enable-tempstore], -# [in-memory temporary tables (never, no, yes, always) [default=no]])], -# [], [enable_tempstore=no]) -# AC_MSG_CHECKING([for whether or not to store temp tables in-memory]) -# case "$enable_tempstore" in -# never ) TEMP_STORE=0 ;; -# no ) TEMP_STORE=1 ;; -# always ) TEMP_STORE=3 ;; -# yes ) TEMP_STORE=3 ;; -# * ) -# TEMP_STORE=1 -# enable_tempstore=yes -# ;; -# esac -# AC_MSG_RESULT($enable_tempstore) -# AC_SUBST(TEMP_STORE) -#----------------------------------------------------------------------- - -AC_OUTPUT diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 5264f89f3c..04c8f87f55 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -1,475 +1,559 @@ -# Makefile.in -- -# -# This file is a Makefile for Sample TEA Extension. If it has the name -# "Makefile.in" then it is a template for a Makefile; to generate the -# actual Makefile, run "./configure", which is a configuration script -# generated by the "autoconf" program (constructs like "@foo@" will get -# replaced in the actual Makefile. -# -# Copyright (c) 1999 Scriptics Corporation. -# Copyright (c) 2002-2005 ActiveState Corporation. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. - -#======================================================================== -# Add additional lines to handle any additional AC_SUBST cases that -# have been added in a customized configure script. -#======================================================================== - -#SAMPLE_NEW_VAR = @SAMPLE_NEW_VAR@ - -#======================================================================== -# Nothing of the variables below this line should need to be changed. -# Please check the TARGETS section below to make sure the make targets -# are correct. -#======================================================================== - -#======================================================================== -# The names of the source files is defined in the configure script. -# The object files are used for linking into the final library. -# This will be used when a dist target is added to the Makefile. -# It is not important to specify the directory, as long as it is the -# $(srcdir) or in the generic, win or unix subdirectory. -#======================================================================== - -PKG_SOURCES = @PKG_SOURCES@ -PKG_OBJECTS = @PKG_OBJECTS@ - -PKG_STUB_SOURCES = @PKG_STUB_SOURCES@ -PKG_STUB_OBJECTS = @PKG_STUB_OBJECTS@ - -#======================================================================== -# PKG_TCL_SOURCES identifies Tcl runtime files that are associated with -# this package that need to be installed, if any. -#======================================================================== - -PKG_TCL_SOURCES = @PKG_TCL_SOURCES@ - -#======================================================================== -# This is a list of public header files to be installed, if any. -#======================================================================== - -PKG_HEADERS = @PKG_HEADERS@ - -#======================================================================== -# "PKG_LIB_FILE" refers to the library (dynamic or static as per -# configuration options) composed of the named objects. -#======================================================================== - -PKG_LIB_FILE = @PKG_LIB_FILE@ -PKG_LIB_FILE8 = @PKG_LIB_FILE8@ -PKG_LIB_FILE9 = @PKG_LIB_FILE9@ -PKG_STUB_LIB_FILE = @PKG_STUB_LIB_FILE@ - -lib_BINARIES = $(PKG_LIB_FILE) -BINARIES = $(lib_BINARIES) - -SHELL = @SHELL@ - -srcdir = @srcdir@ -prefix = @prefix@ -exec_prefix = @exec_prefix@ - -bindir = @bindir@ -libdir = @libdir@ -includedir = @includedir@ -datarootdir = @datarootdir@ -runstatedir = @runstatedir@ -datadir = @datadir@ -mandir = @mandir@ - -DESTDIR = - -PKG_DIR = $(PACKAGE_NAME)$(PACKAGE_VERSION) -pkgdatadir = $(datadir)/$(PKG_DIR) -pkglibdir = $(libdir)/$(PKG_DIR) -pkgincludedir = $(includedir)/$(PKG_DIR) - -top_builddir = @abs_top_builddir@ - -INSTALL_OPTIONS = -INSTALL = @INSTALL@ $(INSTALL_OPTIONS) -INSTALL_DATA_DIR = @INSTALL_DATA_DIR@ -INSTALL_DATA = @INSTALL_DATA@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ -INSTALL_SCRIPT = @INSTALL_SCRIPT@ -INSTALL_LIBRARY = @INSTALL_LIBRARY@ - -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -CC = @CC@ -CCLD = @CCLD@ -CFLAGS_DEFAULT = @CFLAGS_DEFAULT@ -CFLAGS_WARNING = @CFLAGS_WARNING@ -EXEEXT = @EXEEXT@ -LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@ -MAKE_LIB = @MAKE_LIB@ -MAKE_STUB_LIB = @MAKE_STUB_LIB@ -OBJEXT = @OBJEXT@ -RANLIB = @RANLIB@ -RANLIB_STUB = @RANLIB_STUB@ -SHLIB_CFLAGS = @SHLIB_CFLAGS@ -SHLIB_LD = @SHLIB_LD@ -SHLIB_LD_LIBS = @SHLIB_LD_LIBS@ -STLIB_LD = @STLIB_LD@ -#TCL_DEFS = @TCL_DEFS@ -TCL_BIN_DIR = @TCL_BIN_DIR@ -TCL_SRC_DIR = @TCL_SRC_DIR@ -#TK_BIN_DIR = @TK_BIN_DIR@ -#TK_SRC_DIR = @TK_SRC_DIR@ - -# Not used, but retained for reference of what libs Tcl required -#TCL_LIBS = @TCL_LIBS@ - -#======================================================================== -# TCLLIBPATH seeds the auto_path in Tcl's init.tcl so we can test our -# package without installing. The other environment variables allow us -# to test against an uninstalled Tcl. Add special env vars that you -# require for testing here (like TCLX_LIBRARY). -#======================================================================== - -EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR) -#EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR):$(TK_BIN_DIR) -TCLLIBPATH = $(top_builddir) -TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` -PKG_ENV = @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \ - PATH="$(EXTRA_PATH):$(PATH)" \ - TCLLIBPATH="$(TCLLIBPATH)" - -TCLSH_PROG = @TCLSH_PROG@ -TCLSH = $(TCLSH_ENV) $(PKG_ENV) $(TCLSH_PROG) - -#WISH_ENV = TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library` -#WISH_PROG = @WISH_PROG@ -#WISH = $(TCLSH_ENV) $(WISH_ENV) $(PKG_ENV) $(WISH_PROG) - -SHARED_BUILD = @SHARED_BUILD@ - -INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I. -I$(srcdir)/.. -#INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ @TK_INCLUDES@ @TK_XINCLUDES@ - -PKG_CFLAGS = @PKG_CFLAGS@ - -# TCL_DEFS is not strictly need here, but if you remove it, then you -# must make sure that configure.ac checks for the necessary components -# that your library may use. TCL_DEFS can actually be a problem if -# you do not compile with a similar machine setup as the Tcl core was -# compiled with. -#DEFS = $(TCL_DEFS) @DEFS@ $(PKG_CFLAGS) -DEFS = @DEFS@ $(PKG_CFLAGS) - -# Move pkgIndex.tcl to 'BINARIES' var if it is generated in the Makefile -CONFIG_CLEAN_FILES = Makefile pkgIndex.tcl -CLEANFILES = @CLEANFILES@ - -CPPFLAGS = @CPPFLAGS@ -LIBS = @PKG_LIBS@ @LIBS@ -AR = @AR@ -CFLAGS = @CFLAGS@ -LDFLAGS = @LDFLAGS@ -LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@ -COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) \ - $(CFLAGS_DEFAULT) $(CFLAGS_WARNING) $(SHLIB_CFLAGS) $(CFLAGS) - -GDB = gdb -VALGRIND = valgrind -VALGRINDARGS = --tool=memcheck --num-callers=8 --leak-resolution=high \ - --leak-check=yes --show-reachable=yes -v - -.SUFFIXES: .c .$(OBJEXT) - -#======================================================================== -# Start of user-definable TARGETS section -#======================================================================== - -#======================================================================== -# TEA TARGETS. Please note that the "libraries:" target refers to platform -# independent files, and the "binaries:" target includes executable programs and -# platform-dependent libraries. Modify these targets so that they install -# the various pieces of your package. The make and install rules -# for the BINARIES that you specified above have already been done. -#======================================================================== - -all: binaries libraries doc - -#======================================================================== -# The binaries target builds executable programs, Windows .dll's, unix -# shared/static libraries, and any other platform-dependent files. -# The list of targets to build for "binaries:" is specified at the top -# of the Makefile, in the "BINARIES" variable. -#======================================================================== - -binaries: $(BINARIES) - -libraries: - -#======================================================================== -# Your doc target should differentiate from doc builds (by the developer) -# and doc installs (see install-doc), which just install the docs on the -# end user machine when building from source. -#======================================================================== - -doc: - @echo "If you have documentation to create, place the commands to" - @echo "build the docs in the 'doc:' target. For example:" - @echo " xml2nroff sample.xml > sample.n" - @echo " xml2html sample.xml > sample.html" - -install: all install-binaries install-libraries install-doc - -install-binaries: binaries install-lib-binaries install-bin-binaries - -#======================================================================== -# This rule installs platform-independent files, such as header files. -# The list=...; for p in $$list handles the empty list case x-platform. -#======================================================================== - -install-libraries: libraries - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(includedir)" - @echo "Installing header files in $(DESTDIR)$(includedir)" - @list='$(PKG_HEADERS)'; for i in $$list; do \ - echo "Installing $(srcdir)/$$i" ; \ - $(INSTALL_DATA) $(srcdir)/$$i "$(DESTDIR)$(includedir)" ; \ - done; - -#======================================================================== -# Install documentation. Unix manpages should go in the $(mandir) -# directory. -#======================================================================== - -install-doc: doc - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(mandir)/mann" - @echo "Installing documentation in $(DESTDIR)$(mandir)" - @list='$(srcdir)/doc/*.n'; for i in $$list; do \ - echo "Installing $$i"; \ - $(INSTALL_DATA) $$i "$(DESTDIR)$(mandir)/mann" ; \ - done - -test: binaries libraries - @echo "SQLite TEA distribution does not include tests" - -shell: binaries libraries - @$(TCLSH) $(SCRIPT) - -gdb: - $(TCLSH_ENV) $(PKG_ENV) $(GDB) $(TCLSH_PROG) $(SCRIPT) - -gdb-test: binaries libraries - $(TCLSH_ENV) $(PKG_ENV) $(GDB) \ - --args $(TCLSH_PROG) `@CYGPATH@ $(srcdir)/tests/all.tcl` \ - $(TESTFLAGS) -singleproc 1 \ - -load "package ifneeded $(PACKAGE_NAME) $(PACKAGE_VERSION) \ - [list load `@CYGPATH@ $(PKG_LIB_FILE)` [string totitle $(PACKAGE_NAME)]]" - -valgrind: binaries libraries - $(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) \ - `@CYGPATH@ $(srcdir)/tests/all.tcl` $(TESTFLAGS) - -valgrindshell: binaries libraries - $(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) $(SCRIPT) - -depend: - -#======================================================================== -# $(PKG_LIB_FILE) should be listed as part of the BINARIES variable -# mentioned above. That will ensure that this target is built when you -# run "make binaries". -# -# The $(PKG_OBJECTS) objects are created and linked into the final -# library. In most cases these object files will correspond to the -# source files above. -#======================================================================== - -$(PKG_LIB_FILE): $(PKG_OBJECTS) - -rm -f $(PKG_LIB_FILE) - ${MAKE_LIB} - $(RANLIB) $(PKG_LIB_FILE) - -$(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS) - -rm -f $(PKG_STUB_LIB_FILE) - ${MAKE_STUB_LIB} - $(RANLIB_STUB) $(PKG_STUB_LIB_FILE) - -#======================================================================== -# We need to enumerate the list of .c to .o lines here. -# -# In the following lines, $(srcdir) refers to the toplevel directory -# containing your extension. If your sources are in a subdirectory, -# you will have to modify the paths to reflect this: -# -# sample.$(OBJEXT): $(srcdir)/generic/sample.c -# $(COMPILE) -c `@CYGPATH@ $(srcdir)/generic/sample.c` -o $@ -# -# Setting the VPATH variable to a list of paths will cause the makefile -# to look into these paths when resolving .c to .obj dependencies. -# As necessary, add $(srcdir):$(srcdir)/compat:.... -#======================================================================== - -VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/macosx - -.c.@OBJEXT@: - $(COMPILE) -c `@CYGPATH@ $<` -o $@ - -tclsample.@OBJEXT@: sampleUuid.h - -$(srcdir)/manifest.uuid: - printf "git-" >$(srcdir)/manifest.uuid - (cd $(srcdir); git rev-parse HEAD >>$(srcdir)/manifest.uuid || \ - (printf "svn-r" >$(srcdir)/manifest.uuid ; \ - svn info --show-item last-changed-revision >>$(srcdir)/manifest.uuid) || \ - printf "unknown" >$(srcdir)/manifest.uuid) - -sampleUuid.h: $(srcdir)/manifest.uuid - echo "#define SAMPLE_VERSION_UUID \\" >$@ - cat $(srcdir)/manifest.uuid >>$@ - echo "" >>$@ - -#======================================================================== -# Distribution creation -# You may need to tweak this target to make it work correctly. -#======================================================================== - -#COMPRESS = tar cvf $(PKG_DIR).tar $(PKG_DIR); compress $(PKG_DIR).tar -COMPRESS = tar zcvf $(PKG_DIR).tar.gz $(PKG_DIR) -DIST_ROOT = /tmp/dist -DIST_DIR = $(DIST_ROOT)/$(PKG_DIR) - -DIST_INSTALL_DATA = CPPROG='cp -p' $(INSTALL) -m 644 -DIST_INSTALL_SCRIPT = CPPROG='cp -p' $(INSTALL) -m 755 - -dist-clean: - rm -rf $(DIST_DIR) $(DIST_ROOT)/$(PKG_DIR).tar.* - -dist: dist-clean $(srcdir)/manifest.uuid - $(INSTALL_DATA_DIR) $(DIST_DIR) - - # TEA files - $(DIST_INSTALL_DATA) $(srcdir)/Makefile.in \ - $(srcdir)/aclocal.m4 $(srcdir)/configure.ac \ - $(DIST_DIR)/ - $(DIST_INSTALL_SCRIPT) $(srcdir)/configure $(DIST_DIR)/ - - $(INSTALL_DATA_DIR) $(DIST_DIR)/tclconfig - $(DIST_INSTALL_DATA) $(srcdir)/tclconfig/README.txt \ - $(srcdir)/manifest.uuid \ - $(srcdir)/tclconfig/tcl.m4 $(srcdir)/tclconfig/install-sh \ - $(DIST_DIR)/tclconfig/ - - # Extension files - $(DIST_INSTALL_DATA) \ - $(srcdir)/ChangeLog \ - $(srcdir)/README.sha \ - $(srcdir)/license.terms \ - $(srcdir)/README \ - $(srcdir)/pkgIndex.tcl.in \ - $(DIST_DIR)/ - - list='demos doc generic library macosx tests unix win'; \ - for p in $$list; do \ - if test -d $(srcdir)/$$p ; then \ - $(INSTALL_DATA_DIR) $(DIST_DIR)/$$p; \ - $(DIST_INSTALL_DATA) $(srcdir)/$$p/* $(DIST_DIR)/$$p/; \ - fi; \ - done - - (cd $(DIST_ROOT); $(COMPRESS);) - -#======================================================================== -# End of user-definable section -#======================================================================== - -#======================================================================== -# Don't modify the file to clean here. Instead, set the "CLEANFILES" -# variable in configure.ac -#======================================================================== - -clean: - -test -z "$(BINARIES)" || rm -f $(BINARIES) - -rm -f *.$(OBJEXT) core *.core - -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) - -distclean: clean - -rm -f *.tab.c - -rm -f $(CONFIG_CLEAN_FILES) - -rm -f config.cache config.log config.status - -#======================================================================== -# Install binary object libraries. On Windows this includes both .dll and -# .lib files. Because the .lib files are not explicitly listed anywhere, -# we need to deduce their existence from the .dll file of the same name. -# Library files go into the lib directory. -# In addition, this will generate the pkgIndex.tcl -# file in the install location (assuming it can find a usable tclsh shell) -# -# You should not have to modify this target. -#======================================================================== - -install-lib-binaries: binaries - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(pkglibdir)" - @list='$(lib_BINARIES)'; for p in $$list; do \ - if test -f $$p; then \ - echo " $(INSTALL_LIBRARY) $$p $(DESTDIR)$(pkglibdir)/$$p"; \ - $(INSTALL_LIBRARY) $$p "$(DESTDIR)$(pkglibdir)/$$p"; \ - ext=`echo $$p|sed -e "s/.*\.//"`; \ - if test "x$$ext" = "xdll"; then \ - lib=`basename $$p|sed -e 's/.[^.]*$$//'`.lib; \ - if test -f $$lib; then \ - echo " $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib"; \ - $(INSTALL_DATA) $$lib "$(DESTDIR)$(pkglibdir)/$$lib"; \ - fi; \ - fi; \ - fi; \ - done - @list='$(PKG_TCL_SOURCES)'; for p in $$list; do \ - if test -f $(srcdir)/$$p; then \ - destp=`basename $$p`; \ - echo " Install $$destp $(DESTDIR)$(pkglibdir)/$$destp"; \ - $(INSTALL_DATA) $(srcdir)/$$p "$(DESTDIR)$(pkglibdir)/$$destp"; \ - fi; \ - done - @if test "x$(SHARED_BUILD)" = "x1"; then \ - echo " Install pkgIndex.tcl $(DESTDIR)$(pkglibdir)"; \ - $(INSTALL_DATA) pkgIndex.tcl "$(DESTDIR)$(pkglibdir)"; \ +all: +# +# Unless this file is named Makefile.in, you are probably looking +# at an automatically generated/filtered copy and should probably not +# edit it. +# +# This makefile is part of the teaish framework, a tool for building +# Tcl extensions, conceptually related to TEA/tclconfig but using the +# Autosetup configuration system instead of the GNU Autotools. +# +# A copy of this makefile gets processed for each extension separately +# and populated with info about how to build, test, and install the +# extension. +# +# Maintenance reminder: this file needs to stay portable with POSIX +# Make, not just GNU Make. Yes, that's unfortunate because it makes +# some things impossible (like skipping over swathes of rules when +# 'make distclean' is invoked). +# + +CC = @CC@ +INSTALL = @BIN_INSTALL@ +INSTALL.noexec = $(INSTALL) -m 0644 + +# +# Var name prefixes: +# +# teaish. => teaish core +# tx. => teaish extension +# +# Vars with a "tx." or "teaish." prefix are all "public" for purposes +# of the extension makefile, but the extension must not any "teaish." +# vars and must only modify "tx." vars where that allowance is +# specifically noted. +# +# Vars with a "teaish__" prefix are "private" and must not be used by +# the extension makefile. They may change semantics or be removed in +# any given teaish build. +# +tx.name = @TEAISH_NAME@ +tx.version = @TEAISH_VERSION@ +tx.name.pkg = @TEAISH_PKGNAME@ +tx.libdir = @TEAISH_LIBDIR_NAME@ +tx.loadPrefix = @TEAISH_LOAD_PREFIX@ +tx.tcl = @TEAISH_TCL@ +tx.makefile = @TEAISH_MAKEFILE@ +tx.makefile.in = @TEAISH_MAKEFILE_IN@ +tx.dll8.basename = @TEAISH_DLL8_BASENAME@ +tx.dll9.basename = @TEAISH_DLL9_BASENAME@ +tx.dll8 = @TEAISH_DLL8@ +tx.dll9 = @TEAISH_DLL9@ +tx.dll = $(tx.dll$(TCL_MAJOR_VERSION)) +tx.dir = @TEAISH_EXT_DIR@ +@if TEAISH_TM_TCL +# Input filename for tcl::tm-style module +tx.tm = @TEAISH_TM_TCL@ +# Target filename for tcl::tm-style installation +tx.tm.tgt = $(tx.name.pkg)-$(tx.version).tm +@endif + +@if TEAISH_DIST_NAME +tx.name.dist = @TEAISH_DIST_NAME@ +@else +tx.name.dist = $(teaish.name) +@endif + +teaish.dir = @abs_top_srcdir@ +#teaish.dir.autosetup = @TEAISH_AUTOSETUP_DIR@ +teaish.makefile = Makefile +teaish.makefile.in = $(teaish.dir)/Makefile.in +teaish__auto.def = $(teaish.dir)/auto.def + +# +# Autotools-conventional vars. We don't actually use these in this +# makefile but some may be referenced by vars imported via +# tclConfig.sh. They are part of the public API and may be reliably +# depended on from teaish.make.in. +# +bindir = @bindir@ +datadir = @datadir@ +exec_prefix = @exec_prefix@ +includedir = @includedir@ +infodir = @infodir@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +prefix = @prefix@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sysconfdir = @sysconfdir@ + + +# +# Vars derived (mostly) from tclConfig.sh. These may be reliably +# used from the extension makefile. +# +TCLSH = @TCLSH_CMD@ +TCL_CONFIG_SH = @TCL_CONFIG_SH@ +TCL_EXEC_PREFIX = @TCL_EXEC_PREFIX@ +TCL_INCLUDE_SPEC = @TCL_INCLUDE_SPEC@ +TCL_LIBS = @TCL_LIBS@ +TCL_LIB_SPEC = @TCL_LIB_SPEC@ +TCL_MAJOR_VERSION = @TCL_MAJOR_VERSION@ +TCL_MINOR_VERSION = @TCL_MINOR_VERSION@ +TCL_PATCH_LEVEL = @TCL_PATCH_LEVEL@ +TCL_PREFIX = @TCL_PREFIX@ +TCL_SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ +TCL_STUB_LIB_SPEC = @TCL_STUB_LIB_SPEC@ +TCL_VERSION = @TCL_VERSION@ +TCLLIBDIR = @TCLLIBDIR@ + +# +# CFLAGS.configure = CFLAGS as known at configure-time. +# +# This ordering is deliberate: flags populated via tcl's +# [teaish-cflags-add] should preceed CFLAGS and CPPFLAGS (which +# typically come from the ./configure command-line invocation). +# +CFLAGS.configure = @SH_CFLAGS@ @TEAISH_CFLAGS@ @CFLAGS@ @CPPFLAGS@ $(TCL_INCLUDE_SPEC) +CFLAGS.configure += -DUSE_TCL_STUBS=@TEAISH_USE_STUBS@ + +# +# LDFLAGS.configure = LDFLAGS as known at configure-time. +# +# This ordering is deliberate: flags populated via tcl's +# [teaish-ldflags-add] should precede LDFLAGS (which typically +# comes from the ./configure command-line invocation). +# +LDFLAGS.configure = @TEAISH_LDFLAGS@ @LDFLAGS@ + +# +# Linker flags for linkhing a shared library. +# +LDFLAGS.shlib = @SH_LDFLAGS@ + +# +# The following tx.XYZ vars may be populated/modified by teaish.tcl +# and/or teaish.make. +# + +# +# tx.src is the list of source or object files to include in the +# (single) compiler/linker invocation. This will initially contain any +# sources passed to [teaish-src-add], but may also be appended to by +# teaish.make. +# +tx.src = @TEAISH_EXT_SRC@ + +# +# tx.CFLAGS is typically set by teaish.make, whereas TEAISH_CFLAGS +# gets set up via the configure script. +# +tx.CFLAGS = +tx.CPPFLAGS = + +# +# tx.LDFLAGS is typically set by teaish.make, whereas TEAISH_LDFLAGS +# gets set up via the configure script. +# +tx.LDFLAGS = + +# +# The list of 'dist' files may be appended to from teaish.make.in. +# It can also be set up from teaish.tcl using [teaish-dist-add] +# and/or [teaish-src-add -dist ...]. +# +tx.dist.files = @TEAISH_DIST_FILES@ + +# +# The base name for a distribution tar/zip file. +# +tx.dist.basename = $(tx.name.dist)-$(tx.version) + +# List of deps which may trigger an auto-reconfigure. +# +teaish__autogen.deps = \ + $(tx.makefile.in) $(teaish.makefile.in) \ + $(tx.tcl) \ + @TEAISH_PKGINDEX_TCL_IN@ @TEAISH_TM_TCL_IN@ \ + @AUTODEPS@ + +@if TEAISH_MAKEFILE_IN +$(tx.makefile): $(tx.makefile.in) +@endif + +teaish.autoreconfig = \ + @TEAISH_AUTORECONFIG@ + +# +# Problem: when more than one target can invoke TEAISH_AUTORECONFIG, +# we can get parallel reconfigures running. Thus, targets which +# may require reconfigure should depend on... +# +config.log: $(teaish__autogen.deps) + $(teaish.autoreconfig) +# ^^^ We would love to skip this when running [dist]clean, but there's +# no POSIX Make-portable way to do that. GNU Make can. +.PHONY: reconfigure +reconfigure: + $(teaish.autoreconfig) + +$(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ + @AUTODEPS@ + +@if TEAISH_TESTER_TCL_IN +@TEAISH_TESTER_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TESTER_TCL_IN@ +@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ +@endif +@if TEAISH_TEST_TCL_IN +@TEAISH_TEST_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TEST_TCL_IN@ +@TEAISH_TEST_TCL@: @TEAISH_TEST_TCL_IN@ +@endif + +# +# CC variant for compiling Tcl-using sources. +# +CC.tcl = \ + $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) $(tx.CPPFLAGS) + +# +# CC variant for linking $(tx.src) into an extension DLL. Note that +# $(tx.src) must come before $(LDFLAGS...) for linking to third-party +# libs to work. +# +CC.dll = \ + $(CC.tcl) $(tx.src) $(LDFLAGS.shlib) \ + $(tx.LDFLAGS) $(LDFLAGS.configure) $(LDFLAGS) $(TCL_STUB_LIB_SPEC) + +@if TEAISH_ENABLE_DLL +# +# The rest of this makefile exists solely to support this brief +# target: the extension shared lib. +# +$(tx.dll): $(tx.src) config.log + @if [ "x" = "x$(tx.src)" ]; then \ + echo "Makefile var tx.src (source/object files) is empty" 1>&2; \ + exit 1; \ + fi + $(CC.dll) + +all: $(tx.dll) +@endif # TEAISH_ENABLE_DLL + +tclsh: $(teaish.makefile) config.log + @{ echo "#!/bin/sh"; echo 'exec $(TCLSH) "$$@"'; } > $@ + @chmod +x $@ + @echo "Created $@" + +# +# Run the generated test script. +# +.PHONY: test-pre test-prepre test-core test test-post test-extension +test-extension: # this name is reserved for use by teaish.make[.in] +@if TEAISH_ENABLE_DLL +test-prepre: $(tx.dll) +@endif +@if TEAISH_TESTER_TCL +teaish.tester.tcl = @TEAISH_TESTER_TCL@ +test-core.args = $(teaish.tester.tcl) +@if TEAISH_ENABLE_DLL +test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' +@else +test-core.args += '' '' +@endif +test-core.args += @TEAISH_TESTUTIL_TCL@ +# Clients may pass additional args via test.args=... +# and ::argv will be rewritten before the test script loads, to +# remove $(test-core.args) +test.args ?= +test-core: test-pre + $(TCLSH) $(test-core.args) $(test.args) +test-gdb: $(teaish.tester.tcl) + gdb --args $(TCLSH) $(test-core.args) $(test.args) +test-vg.flags ?= --leak-check=full -v --show-reachable=yes --track-origins=yes +test-vg: $(teaish.tester.tcl) + valgrind $(test-vg.flags) $(TCLSH) $(test-core.args) $(test.args) +@else # !TEAISH_TESTER_TCL +test-prepre: +@endif # TEAISH_TESTER_TCL +test-pre: test-prepre +test-core: test-pre +test-post: test-core +test: test-post + +# +# Cleanup rules... +# +#.PHONY: clean-pre clean-core clean-post clean-extension +# +clean-pre: +clean-core: clean-pre + rm -f $(tx.dll8) $(tx.dll9) tclsh +clean-post: clean-core +clean: clean-post + +.PHONY: distclean-pre distclean-core distclean-post clean-extension +distclean-pre: clean +distclean-core: distclean-pre + rm -f Makefile + rm -f config.log config.defines.txt +@if TEAISH_MAKEFILE_IN +@if TEAISH_MAKEFILE + rm -f @TEAISH_MAKEFILE@ +@endif +@endif +@if TEAISH_TESTER_TCL_IN + rm -f $(teaish.tester.tcl) +@endif +@if TEAISH_PKGINDEX_TCL_IN + rm -f @TEAISH_PKGINDEX_TCL@ +@endif +@if TEAISH_PKGINIT_TCL_IN + rm -f @TEAISH_PKGINIT_TCL@ +@endif +@if TEAISH_TEST_TCL_IN + rm -f @TEAISH_TEST_TCL@ +@endif +distclean-post: distclean-core +distclean: distclean-post +# +# The (dist)clean-extension targets are reserved for use by +# client-side teaish.make. +# +# Client code which wants to clean up extra stuff should do so by +# adding their cleanup target (e.g. clean-extension) as a dependency +# to the 'clean' target, like so: +# +# clean: distclean-extension +# distclean: distclean-extension +# +distclean-extension: +clean-extension: + +# +# Installation rules... +# +@if TEAISH_ENABLE_INSTALL +.PHONY: install-pre install-core install-post install-test install-prepre install-extension +install-extension: # this name is reserved for use by teaish.make + +@if TEAISH_ENABLE_DLL +install-prepre: $(tx.dll) +@else +install-prepre: +@endif + +@if TEAISH_TM_TCL +install-core.tmdir = $(DESTDIR)@TEAISH_TCL_TM_DIR@ +@endif + +install-pre: install-prepre +install-core: install-pre + @if [ ! -d "$(DESTDIR)$(TCLLIBDIR)" ]; then \ + set -x; $(INSTALL) -d "$(DESTDIR)$(TCLLIBDIR)"; \ + fi +# ^^^^ on some platforms, install -d fails if the target already exists. +@if TEAISH_ENABLE_DLL + $(INSTALL) $(tx.dll) "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_PKGINDEX_TCL + $(INSTALL.noexec) "@TEAISH_PKGINDEX_TCL@" "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_PKGINIT_TCL + $(INSTALL.noexec) "@TEAISH_PKGINIT_TCL@" "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_TM_TCL + @if [ ! -d "$(install-core.tmdir)" ]; then \ + set -x; $(INSTALL) -d "$(install-core.tmdir)"; \ + fi + $(INSTALL.noexec) "@TEAISH_TM_TCL@" "$(install-core.tmdir)/$(tx.tm.tgt)" +@endif +install-test: install-core + @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ + set xtra=""; \ + if [ x != "x$(DESTDIR)" ]; then \ + xtra='set ::auto_path [linsert $$::auto_path 0 [file normalize $(DESTDIR)$(TCLLIBDIR)/..]];'; \ + fi; \ + if echo \ + 'set c 0; ' $$xtra \ + '@TEAISH_POSTINST_PREREQUIRE@' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)} xc]} {incr c};' \ + 'if {$$c && "" ne $$xc} {puts $$xc; puts "auto_path=$$::auto_path"};' \ + 'exit $$c' \ + | $(TCLSH) ; then \ + echo "passed"; \ + else \ + echo "FAILED"; \ + exit 1; \ fi +install-post: install-test +install: install-post + +# +# Uninstall rules... +# +.PHONY: uninstall uninstall-pre uninstall-core uninstall-post uninstall-extension +uninstall-extension: # this name is reserved for use by teaish.make +uninstall-pre: +uninstall-core: uninstall-pre +@if TEAISH_ENABLE_DLL + rm -fr "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_TM_TCL + rm -f "$(DESTDIR)$(install-core.tmdir)/$(tx.tm.tgt)" +@endif + +uninstall-post: uninstall-core + @echo "Uninstalled Tcl extension $(tx.name) $(tx.version)" +uninstall: uninstall-post +@endif # TEAISH_ENABLE_INSTALL + +@if TEAISH_MAKEFILE_IN +Makefile: $(tx.makefile.in) +config.log: $(teaish.makefile.in) +@endif + +# +# Package archive generation ("dist") rules... +# +@if TEAISH_ENABLE_DIST +@if BIN_TAR +@if BIN_ZIP + +# When installing teaish as part of "make dist", we need to run +# configure with similar flags to what we last configured with but we +# must not pass on any extension-specific flags, as those won't be +# recognized when running in --teaish-install mode, causing +# the sub-configure to fail. +dist.flags = --with-tclsh=$(TCLSH) +dist.reconfig = $(teaish.dir)/configure $(tx.dist.reconfig-flags) $(dist.flags) + +# Temp dir for dist.zip. Must be different than dist.tgz or else +# parallel builds may hose the dist. +teaish__dist.tmp.zip = teaish__dist_zip +# +# Make a distribution zip file... +# +dist.zip = $(tx.dist.basename).zip +.PHONY: dist.zip dist.zip-core dist.zip-post +#dist.zip-pre: +# We apparently can't add a pre-hook here, else "make dist" rebuilds +# the archive each time it's run. +$(dist.zip): $(tx.dist.files) + @rm -fr $(teaish__dist.tmp.zip) + @mkdir -p $(teaish__dist.tmp.zip)/$(tx.dist.basename) + @tar cf $(teaish__dist.tmp.zip)/tmp.tar $(tx.dist.files) + @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(tx.dist.basename) +@if TEAISH_DIST_FULL + @$(dist.reconfig) \ + --teaish-install=$(teaish__dist.tmp.zip)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null +@endif + @rm -f $(tx.dist.basename)/tmp.tar $(dist.zip) + @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(tx.dist.basename) + @rm -fr $(teaish__dist.tmp.zip) + @ls -la $(dist.zip) +dist.zip-core: $(dist.zip) +dist.zip-post: dist.zip-core +dist.zip: dist.zip-post +dist: dist.zip +undist-zip: + rm -f $(dist.zip) +undist: undist-zip +@endif #BIN_ZIP + +# +# Make a distribution tarball... +# +teaish__dist.tmp.tgz = teaish__dist_tgz +dist.tgz = $(tx.dist.basename).tar.gz +.PHONY: dist.tgz dist.tgz-core dist.tgz-post +# dist.tgz-pre: +# see notes in dist.zip +$(dist.tgz): $(tx.dist.files) + @rm -fr $(teaish__dist.tmp.tgz) + @mkdir -p $(teaish__dist.tmp.tgz)/$(tx.dist.basename) + @tar cf $(teaish__dist.tmp.tgz)/tmp.tar $(tx.dist.files) + @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(tx.dist.basename) +@if TEAISH_DIST_FULL + @rm -f $(teaish__dist.tmp.tgz)/$(tx.dist.basename)/pkgIndex.tcl.in; # kludge + @$(dist.reconfig) \ + --teaish-install=$(teaish__dist.tmp.tgz)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null +@endif + @rm -f $(tx.dist.basename)/tmp.tar $(dist.tgz) + @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(tx.dist.basename) + @rm -fr $(teaish__dist.tmp.tgz) + @ls -la $(dist.tgz) +dist.tgz-core: $(dist.tgz) +dist.tgz-post: dist.tgz-core +dist.tgz: dist.tgz-post +dist: dist.tgz +undist-tgz: + rm -f $(dist.tgz) +undist: undist-tgz +@else #!BIN_TAR +dist: + @echo "The dist rules require tar, which configure did not find." 1>&2; exit 1 +@endif #BIN_TAR +@else #!TEAISH_ENABLE_DIST +undist: +dist: +@if TEAISH_OUT_OF_EXT_TREE + @echo "'dist' can only be used from an extension's home dir" 1>&2; \ + echo "In this case: @TEAISH_EXT_DIR@" 1>&2; exit 1 +@endif +@endif #TEAISH_ENABLE_DIST + +Makefile: @TEAISH_TCL@ + +@if TEAISH_MAKEFILE_CODE +# +# TEAISH_MAKEFILE_CODE may contain literal makefile code, which +# gets pasted verbatim here. Either [define TEAISH_MAKEFILE_CODE +# ...] or use [teaish-make-add] to incrementally build up this +# content. +# +# +@TEAISH_MAKEFILE_CODE@ +# +@endif -#======================================================================== -# Install binary executables (e.g. .exe files and dependent .dll files) -# This is for files that must go in the bin directory (located next to -# wish and tclsh), like dependent .dll files on Windows. -# -# You should not have to modify this target, except to define bin_BINARIES -# above if necessary. -#======================================================================== - -install-bin-binaries: binaries - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(bindir)" - @list='$(bin_BINARIES)'; for p in $$list; do \ - if test -f $$p; then \ - echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p"; \ - $(INSTALL_PROGRAM) $$p "$(DESTDIR)$(bindir)/$$p"; \ - fi; \ - done - -Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status - cd $(top_builddir) \ - && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status - -uninstall-binaries: - list='$(lib_BINARIES)'; for p in $$list; do \ - rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \ - done - list='$(PKG_TCL_SOURCES)'; for p in $$list; do \ - p=`basename $$p`; \ - rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \ - done - list='$(bin_BINARIES)'; for p in $$list; do \ - rm -f "$(DESTDIR)$(bindir)/$$p"; \ - done - -.PHONY: all binaries clean depend distclean doc install libraries test -.PHONY: gdb gdb-test valgrind valgrindshell - -# Tell versions [3.59,3.63) of GNU make to not export all variables. -# Otherwise a system limit (for SysV at least) may be exceeded. -.NOEXPORT: +@if TEAISH_MAKEFILE +# +# TEAISH_MAKEFILE[_IN] defines any extension-specific state this file +# needs. +# +# It must set the following vars if they're not already accounted for +# via teaish.tcl. +# +# - tx.src = list of the extension's source files, being sure to +# prefix each with $(tx.dir) (if it's in the same dir as the +# extension) so that out-of-tree builds can find them. Optionally, +# [define] TEAISH_EXT_SRC or pass them to [teaish-src-add]. +# +# It may optionally set the following vars: +# +# - tx.CFLAGS = CFLAGS/CPPFLAGS. Optionally, [define] TEAISH_CFLAGS +# or pass them to [teaish-cflags-add]. +# +# - tx.LDFLAGS = LDFLAGS. Optionally, [define] TEAISH_LDFLAGS or +# pass them to [teaish-ldflags-add]. +# +# It may optionally hook into various targets as documented in +# /doc/extensions.md in the canonical teaish source tree. +# +# Interestingly, we don't have to pre-filter teaish.makefile.in - we +# can just @include it here. That skips its teaish-specific validation +# though. Hmm. +# +# +Makefile: @TEAISH_MAKEFILE@ +@include @TEAISH_MAKEFILE@ +# +@endif diff --git a/autoconf/tea/README b/autoconf/tea/README deleted file mode 100644 index 99dc8b8f03..0000000000 --- a/autoconf/tea/README +++ /dev/null @@ -1,36 +0,0 @@ -This is the SQLite extension for Tcl using the Tcl Extension -Architecture (TEA). For additional information on SQLite see - - http://www.sqlite.org/ - - -UNIX BUILD -========== - -Building under most UNIX systems is easy, just run the configure script -and then run make. For more information about the build process, see -the tcl/unix/README file in the Tcl src dist. The following minimal -example will install the extension in the /opt/tcl directory. - - $ cd sqlite-*-tea - $ ./configure --prefix=/opt/tcl - $ make - $ make install - -WINDOWS BUILD -============= - -The recommended method to build extensions under windows is to use the -Msys + Mingw build process. This provides a Unix-style build while -generating native Windows binaries. Using the Msys + Mingw build tools -means that you can use the same configure script as per the Unix build -to create a Makefile. See the tcl/win/README file for the URL of -the Msys + Mingw download. - -If you have VC++ then you may wish to use the files in the win -subdirectory and build the extension using just VC++. These files have -been designed to be as generic as possible but will require some -additional maintenance by the project developer to synchronise with -the TEA configure.in and Makefile.in files. Instructions for using the -VC++ makefile are written in the first part of the Makefile.vc -file. diff --git a/autoconf/tea/README.txt b/autoconf/tea/README.txt new file mode 100644 index 0000000000..122b08d32d --- /dev/null +++ b/autoconf/tea/README.txt @@ -0,0 +1,94 @@ +This is the SQLite extension for Tcl using something akin to +the Tcl Extension Architecture (TEA). To build it: + + ./configure ...flags... + +e.g.: + + ./configure --with-tcl=/path/to/tcl/install/root + +or: + + ./configure --with-tclsh=/path/to/tcl/install/root + +Run ./configure --help for the full list of flags. + +The configuration process will fail if tclConfig.sh cannot be found. + +The makefile will only honor CFLAGS and CPPFLAGS passed to the +configure script, not those directly passed to the makefile. + +Then: + + make test install + +----------------------- THE PREFERRED WAY --------------------------- + +The preferred way to build the TCL extension for SQLite is to use the +canonical source code tarball. For Unix: + + ./configure --with-tclsh=$(TCLSH) + make tclextension-install + +For Windows: + + nmake /f Makefile.msc tclextension-install TCLSH_CMD=$(TCLSH) + +In both of the above, replace $(TCLSH) with the full pathname of +of the tclsh that you want the SQLite extension to work with. See +step-by-step instructions at the links below for more information: + + https://sqlite.org/src/doc/trunk/doc/compile-for-unix.md + https://sqlite.org/src/doc/trunk/doc/compile-for-windows.md + +And info about the extension's Tcl interface can be found at: + + https://sqlite.org/tclsqlite.html + +The whole point of the amalgamation-autoconf tarball (in which this +README.txt file is embedded) is to provide a means of compiling SQLite +that does not require first installing TCL and/or "tclsh". The +canonical Makefile in the SQLite source tree provides more +capabilities (such as the the ability to run test cases to ensure that +the build worked) and is better maintained. The only downside of the +canonical Makefile is that it requires a TCL installation. But if you +are wanting to build the TCL extension for SQLite, then presumably you +already have a TCL installation. So why not just use the more-capable +and better-maintained canoncal Makefile? + +As of version 3.50.0, this build process uses "teaish": + + https://fossil.wanderinghorse.net/r/teaish + +which is conceptually derived from the pre-3.50 toolchain, TEA: + + http://core.tcl-lang.org/tclconfig + http://core.tcl-lang.org/sampleextension + +It to works for us. It might also work for you. But we cannot +promise that. + +If you want to use this TEA builder and it works for you, that's fine. +But if you have trouble, the first thing you should do is go back +to using the canonical Makefile in the SQLite source tree. + +------------------------------------------------------------------ + + +UNIX BUILD +========== + +Building under most UNIX systems is easy, just run the configure +script and then run make. For example: + + $ cd sqlite-*-tea + $ ./configure --with-tcl=/path/to/tcl/install/root + $ make test + $ make install + +WINDOWS BUILD +============= + +On Windows this build is known to work on Cygwin and some Msys2 +environments. We do not currently support Microsoft makefiles for +native Windows builds. diff --git a/autoconf/tea/_teaish.tester.tcl.in b/autoconf/tea/_teaish.tester.tcl.in new file mode 100644 index 0000000000..e04d8e63e7 --- /dev/null +++ b/autoconf/tea/_teaish.tester.tcl.in @@ -0,0 +1,50 @@ +# -*- tcl -*- +# +# Unless this file is named _teaish.tester.tcl.in, you are probably +# looking at an automatically generated/filtered copy and should +# probably not edit it. +# +# This is the wrapper script invoked by teaish's "make test" recipe. +# It gets passed 3 args: +# +# $1 = the DLL name, or "" if the extension has no DLL +# +# $2 = the "load prefix" for Tcl's [load] or empty if $1 is empty +# +# $3 = the /path/to/teaish/tester.tcl (test utility code) +# +@if TEAISH_VSATISFIES_CODE +@TEAISH_VSATISFIES_CODE@ +@endif +if {[llength [lindex $::argv 0]] > 0} { + load [file normalize [lindex $::argv 0]] [lindex $::argv 1]; + # ----^^^^^^^ needed on Haiku when argv 0 is just a filename, else + # load cannot find the file. +} +set ::argv [lassign $argv - -] +source -encoding utf-8 [lindex $::argv 0]; # teaish/tester.tcl +@if TEAISH_PKGINIT_TCL +apply {{file} { + set dir [file dirname $::argv0] + source -encoding utf-8 $file +}} [join {@TEAISH_PKGINIT_TCL@}] +@endif +@if TEAISH_TM_TCL +apply {{file} { + set dir [file dirname $::argv0] + source -encoding utf-8 $file +}} [join {@TEAISH_TM_TCL@}] +@endif +@if TEAISH_TEST_TCL +apply {{file} { + # Populate state for [tester.tcl::teaish-build-flag*] + array set ::teaish__BuildFlags @TEAISH__DEFINES_MAP@ + set dir [file normalize [file dirname $file]] + #test-fail "Just testing" + source -encoding utf-8 $file +}} [join {@TEAISH_TEST_TCL@}] +@else # TEAISH_TEST_TCL +# No $TEAISH_TEST_TCL provided, so here's a default test which simply +# loads the extension. +puts {Extension @TEAISH_NAME@ @TEAISH_VERSION@ successfully loaded from @TEAISH_TESTER_TCL@} +@endif diff --git a/autoconf/tea/aclocal.m4 b/autoconf/tea/aclocal.m4 deleted file mode 100644 index 0b057391d2..0000000000 --- a/autoconf/tea/aclocal.m4 +++ /dev/null @@ -1,9 +0,0 @@ -# -# Include the TEA standard macro set -# - -builtin(include,tclconfig/tcl.m4) - -# -# Add here whatever m4 macros you want to define for your package -# diff --git a/autoconf/tea/auto.def b/autoconf/tea/auto.def new file mode 100644 index 0000000000..7170b3d1fe --- /dev/null +++ b/autoconf/tea/auto.def @@ -0,0 +1,8 @@ +#/do/not/tclsh +# ^^^ help out editors which guess this file's content type. +# +# Main configure script entry point for the TEA(ish) framework. All +# extension-specific customization goes in teaish.tcl.in or +# teaish.tcl. +use teaish/core +teaish-configure-core diff --git a/autoconf/tea/configure b/autoconf/tea/configure new file mode 100755 index 0000000000..01b3abcc2f --- /dev/null +++ b/autoconf/tea/configure @@ -0,0 +1,20 @@ +#!/bin/sh +# Look for and run autosetup... +dir0="`dirname "$0"`" +dirA="$dir0" +if [ -d $dirA/autosetup ]; then + # A local copy of autosetup + dirA=$dirA/autosetup +elif [ -d $dirA/../autosetup ]; then + # SQLite "autoconf" bundle + dirA=$dirA/../autosetup +elif [ -d $dirA/../../autosetup ]; then + # SQLite canonical source tree + dirA=$dirA/../../autosetup +else + echo "$0: Cannot find autosetup" 1>&2 + exit 1 +fi +WRAPPER="$0"; export WRAPPER; exec "`"$dirA/autosetup-find-tclsh"`" \ + "$dirA/autosetup" --teaish-extension-dir="$dir0" \ + "$@" diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac deleted file mode 100644 index e26780e2e9..0000000000 --- a/autoconf/tea/configure.ac +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/bash -norc -dnl This file is an input file used by the GNU "autoconf" program to -dnl generate the file "configure", which is run during Tcl installation -dnl to configure the system for the local environment. - -#----------------------------------------------------------------------- -# Sample configure.ac for Tcl Extensions. The only places you should -# need to modify this file are marked by the string __CHANGE__ -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# __CHANGE__ -# Set your package name and version numbers here. -# -# This initializes the environment with PACKAGE_NAME and PACKAGE_VERSION -# set as provided. These will also be added as -D defs in your Makefile -# so you can encode the package version directly into the source files. -# This will also define a special symbol for Windows (BUILD_ -# so that we create the export library with the dll. -#----------------------------------------------------------------------- - -AC_INIT([sqlite],[3.42.0]) - -#-------------------------------------------------------------------- -# Call TEA_INIT as the first TEA_ macro to set up initial vars. -# This will define a ${TEA_PLATFORM} variable == "unix" or "windows" -# as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE. -#-------------------------------------------------------------------- - -TEA_INIT() - -AC_CONFIG_AUX_DIR(tclconfig) - -#-------------------------------------------------------------------- -# Load the tclConfig.sh file -#-------------------------------------------------------------------- - -TEA_PATH_TCLCONFIG -TEA_LOAD_TCLCONFIG - -#-------------------------------------------------------------------- -# Load the tkConfig.sh file if necessary (Tk extension) -#-------------------------------------------------------------------- - -#TEA_PATH_TKCONFIG -#TEA_LOAD_TKCONFIG - -#----------------------------------------------------------------------- -# Handle the --prefix=... option by defaulting to what Tcl gave. -# Must be called after TEA_LOAD_TCLCONFIG and before TEA_SETUP_COMPILER. -#----------------------------------------------------------------------- - -TEA_PREFIX - -#----------------------------------------------------------------------- -# Standard compiler checks. -# This sets up CC by using the CC env var, or looks for gcc otherwise. -# This also calls AC_PROG_CC and a few others to create the basic setup -# necessary to compile executables. -#----------------------------------------------------------------------- - -TEA_SETUP_COMPILER - -#----------------------------------------------------------------------- -# __CHANGE__ -# Specify the C source files to compile in TEA_ADD_SOURCES, -# public headers that need to be installed in TEA_ADD_HEADERS, -# stub library C source files to compile in TEA_ADD_STUB_SOURCES, -# and runtime Tcl library files in TEA_ADD_TCL_SOURCES. -# This defines PKG(_STUB)_SOURCES, PKG(_STUB)_OBJECTS, PKG_HEADERS -# and PKG_TCL_SOURCES. -#----------------------------------------------------------------------- - -TEA_ADD_SOURCES([tclsqlite3.c]) -TEA_ADD_HEADERS([]) -TEA_ADD_INCLUDES([]) -TEA_ADD_LIBS([]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS4=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS5=1]) -TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_GEOPOLY=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_MATH_FUNCTIONS=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DESERIALIZE=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBPAGE_VTAB=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_BYTECODE_VTAB=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBSTAT_VTAB=1]) -TEA_ADD_STUB_SOURCES([]) -TEA_ADD_TCL_SOURCES([]) - -#-------------------------------------------------------------------- -# The --with-system-sqlite causes the TCL bindings to SQLite to use -# the system shared library for SQLite rather than statically linking -# against its own private copy. This is dangerous and leads to -# undersirable dependences and is not recommended. -# Patchs from rmax. -#-------------------------------------------------------------------- -AC_ARG_WITH([system-sqlite], - [AC_HELP_STRING([--with-system-sqlite], - [use a system-supplied libsqlite3 instead of the bundled one])], - [], [with_system_sqlite=no]) -if test x$with_system_sqlite != xno; then - AC_CHECK_HEADER([sqlite3.h], - [AC_CHECK_LIB([sqlite3],[sqlite3_initialize], - [AC_DEFINE(USE_SYSTEM_SQLITE) - LIBS="$LIBS -lsqlite3"])]) -fi - -#-------------------------------------------------------------------- -# __CHANGE__ -# -# You can add more files to clean if your extension creates any extra -# files by extending CLEANFILES. -# Add pkgIndex.tcl if it is generated in the Makefile instead of ./configure -# and change Makefile.in to move it from CONFIG_CLEAN_FILES to BINARIES var. -# -# A few miscellaneous platform-specific items: -# TEA_ADD_* any platform specific compiler/build info here. -#-------------------------------------------------------------------- - -#CLEANFILES="$CLEANFILES pkgIndex.tcl" -if test "${TEA_PLATFORM}" = "windows" ; then - # Ensure no empty if clauses - : - #TEA_ADD_SOURCES([win/winFile.c]) - #TEA_ADD_INCLUDES([-I\"$(${CYGPATH} ${srcdir}/win)\"]) -else - # Ensure no empty else clauses - : - #TEA_ADD_SOURCES([unix/unixFile.c]) - #TEA_ADD_LIBS([-lsuperfly]) -fi - -#-------------------------------------------------------------------- -# __CHANGE__ -# Choose which headers you need. Extension authors should try very -# hard to only rely on the Tcl public header files. Internal headers -# contain private data structures and are subject to change without -# notice. -# This MUST be called after TEA_LOAD_TCLCONFIG / TEA_LOAD_TKCONFIG -#-------------------------------------------------------------------- - -TEA_PUBLIC_TCL_HEADERS -#TEA_PRIVATE_TCL_HEADERS - -#TEA_PUBLIC_TK_HEADERS -#TEA_PRIVATE_TK_HEADERS -#TEA_PATH_X - -#-------------------------------------------------------------------- -# Check whether --enable-threads or --disable-threads was given. -# This auto-enables if Tcl was compiled threaded. -#-------------------------------------------------------------------- - -TEA_ENABLE_THREADS -if test "${TCL_THREADS}" = "1" ; then - AC_DEFINE(SQLITE_THREADSAFE, 1, [Trigger sqlite threadsafe build]) - # Not automatically added by Tcl because its assumed Tcl links to them, - # but it may not if it isn't really a threaded build. - TEA_ADD_LIBS([$THREADS_LIBS]) -else - AC_DEFINE(SQLITE_THREADSAFE, 0, [Trigger sqlite non-threadsafe build]) -fi - -#-------------------------------------------------------------------- -# The statement below defines a collection of symbols related to -# building as a shared library instead of a static library. -#-------------------------------------------------------------------- - -TEA_ENABLE_SHARED - -#-------------------------------------------------------------------- -# This macro figures out what flags to use with the compiler/linker -# when building shared/static debug/optimized objects. This information -# can be taken from the tclConfig.sh file, but this figures it all out. -#-------------------------------------------------------------------- - -TEA_CONFIG_CFLAGS - -#-------------------------------------------------------------------- -# Set the default compiler switches based on the --enable-symbols option. -#-------------------------------------------------------------------- - -TEA_ENABLE_SYMBOLS - -#-------------------------------------------------------------------- -# This macro generates a line to use when building a library. It -# depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS, -# and TEA_LOAD_TCLCONFIG macros above. -#-------------------------------------------------------------------- - -TEA_MAKE_LIB - -#-------------------------------------------------------------------- -# Determine the name of the tclsh and/or wish executables in the -# Tcl and Tk build directories or the location they were installed -# into. These paths are used to support running test cases only, -# the Makefile should not be making use of these paths to generate -# a pkgIndex.tcl file or anything else at extension build time. -#-------------------------------------------------------------------- - -TEA_PROG_TCLSH -#TEA_PROG_WISH - -#-------------------------------------------------------------------- -# Setup a *Config.sh.in configuration file. -#-------------------------------------------------------------------- - -#TEA_EXPORT_CONFIG([sample]) -#AC_SUBST(SAMPLE_VAR) - -#-------------------------------------------------------------------- -# Specify files to substitute AC variables in. You may alternatively -# have a special pkgIndex.tcl.in or other files which require -# substituting the AC variables in. Include these here. -#-------------------------------------------------------------------- - -AC_CONFIG_FILES([Makefile pkgIndex.tcl]) -#AC_CONFIG_FILES([sampleConfig.sh]) - -#-------------------------------------------------------------------- -# Finally, substitute all of the various values into the files -# specified with AC_CONFIG_FILES. -#-------------------------------------------------------------------- - -AC_OUTPUT diff --git a/autoconf/tea/doc/sqlite3.n b/autoconf/tea/doc/sqlite3.n deleted file mode 100644 index 13913e5583..0000000000 --- a/autoconf/tea/doc/sqlite3.n +++ /dev/null @@ -1,15 +0,0 @@ -.TH sqlite3 n 4.1 "Tcl-Extensions" -.HS sqlite3 tcl -.BS -.SH NAME -sqlite3 \- an interface to the SQLite3 database engine -.SH SYNOPSIS -\fBsqlite3\fI command_name ?filename?\fR -.br -.SH DESCRIPTION -SQLite3 is a self-contains, zero-configuration, transactional SQL database -engine. This extension provides an easy to use interface for accessing -SQLite database files from Tcl. -.PP -For full documentation see \fIhttp://www.sqlite.org/\fR and -in particular \fIhttp://www.sqlite.org/tclsqlite.html\fR. diff --git a/autoconf/tea/pkgIndex.tcl.in b/autoconf/tea/pkgIndex.tcl.in index f95f7d3893..c93fcc6854 100644 --- a/autoconf/tea/pkgIndex.tcl.in +++ b/autoconf/tea/pkgIndex.tcl.in @@ -1,10 +1,40 @@ # -*- tcl -*- -# Tcl package index file, version 1.1 +# Tcl package index file # +# Unless this file is named pkgIndex.tcl.in, you are probably looking +# at an automatically generated/filtered copy and should probably not +# edit it. +# +# Adapted from https://core.tcl-lang.org/tcltls +@if TEAISH_VSATISFIES_CODE +@TEAISH_VSATISFIES_CODE@ +@endif if {[package vsatisfies [package provide Tcl] 9.0-]} { - package ifneeded sqlite3 @PACKAGE_VERSION@ \ - [list load [file join $dir @PKG_LIB_FILE9@] sqlite3] + package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { +@if TEAISH_ENABLE_DLL + load [file join $dir {@TEAISH_DLL9@}] @TEAISH_LOAD_PREFIX@ +@endif +@if TEAISH_PKGINIT_TCL_TAIL + set initScript [file join $dir {@TEAISH_PKGINIT_TCL_TAIL@}] + if {[file exists $initScript]} { + source -encoding utf-8 $initScript + } +@endif + }} $dir] } else { - package ifneeded sqlite3 @PACKAGE_VERSION@ \ - [list load [file join $dir @PKG_LIB_FILE8@] sqlite3] + package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { +@if TEAISH_ENABLE_DLL + if {[string tolower [file extension {@TEAISH_DLL8@}]] in [list .dll .dylib .so]} { + load [file join $dir {@TEAISH_DLL8@}] @TEAISH_LOAD_PREFIX@ + } else { + load {} @TEAISH_LOAD_PREFIX@ + } +@endif +@if TEAISH_PKGINIT_TCL_TAIL + set initScript [file join $dir {@TEAISH_PKGINIT_TCL_TAIL@}] + if {[file exists $initScript]} { + source -encoding utf-8 $initScript + } +@endif + }} $dir] } diff --git a/autoconf/tea/tclconfig/install-sh b/autoconf/tea/tclconfig/install-sh deleted file mode 100644 index 7c34c3f926..0000000000 --- a/autoconf/tea/tclconfig/install-sh +++ /dev/null @@ -1,528 +0,0 @@ -#!/bin/sh -# install - install a program, script, or datafile - -scriptversion=2011-04-20.01; # UTC - -# This originates from X11R5 (mit/util/scripts/install.sh), which was -# later released in X11R6 (xc/config/util/install.sh) with the -# following copyright and license. -# -# Copyright (C) 1994 X Consortium -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN -# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- -# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# Except as contained in this notice, the name of the X Consortium shall not -# be used in advertising or otherwise to promote the sale, use or other deal- -# ings in this Software without prior written authorization from the X Consor- -# tium. -# -# -# FSF changes to this file are in the public domain. -# -# Calling this script install-sh is preferred over install.sh, to prevent -# `make' implicit rules from creating a file called install from it -# when there is no Makefile. -# -# This script is compatible with the BSD install script, but was written -# from scratch. - -nl=' -' -IFS=" "" $nl" - -# set DOITPROG to echo to test this script - -# Don't use :- since 4.3BSD and earlier shells don't like it. -doit=${DOITPROG-} -if test -z "$doit"; then - doit_exec=exec -else - doit_exec=$doit -fi - -# Put in absolute file names if you don't have them in your path; -# or use environment vars. - -chgrpprog=${CHGRPPROG-chgrp} -chmodprog=${CHMODPROG-chmod} -chownprog=${CHOWNPROG-chown} -cmpprog=${CMPPROG-cmp} -cpprog=${CPPROG-cp} -mkdirprog=${MKDIRPROG-mkdir} -mvprog=${MVPROG-mv} -rmprog=${RMPROG-rm} -stripprog=${STRIPPROG-strip} - -posix_glob='?' -initialize_posix_glob=' - test "$posix_glob" != "?" || { - if (set -f) 2>/dev/null; then - posix_glob= - else - posix_glob=: - fi - } -' - -posix_mkdir= - -# Desired mode of installed file. -mode=0755 - -chgrpcmd= -chmodcmd=$chmodprog -chowncmd= -mvcmd=$mvprog -rmcmd="$rmprog -f" -stripcmd= - -src= -dst= -dir_arg= -dst_arg= - -copy_on_change=false -no_target_directory= - -usage="\ -Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE - or: $0 [OPTION]... SRCFILES... DIRECTORY - or: $0 [OPTION]... -t DIRECTORY SRCFILES... - or: $0 [OPTION]... -d DIRECTORIES... - -In the 1st form, copy SRCFILE to DSTFILE. -In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. -In the 4th, create DIRECTORIES. - -Options: - --help display this help and exit. - --version display version info and exit. - - -c (ignored) - -C install only if different (preserve the last data modification time) - -d create directories instead of installing files. - -g GROUP $chgrpprog installed files to GROUP. - -m MODE $chmodprog installed files to MODE. - -o USER $chownprog installed files to USER. - -s $stripprog installed files. - -S $stripprog installed files. - -t DIRECTORY install into DIRECTORY. - -T report an error if DSTFILE is a directory. - -Environment variables override the default commands: - CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG - RMPROG STRIPPROG -" - -while test $# -ne 0; do - case $1 in - -c) ;; - - -C) copy_on_change=true;; - - -d) dir_arg=true;; - - -g) chgrpcmd="$chgrpprog $2" - shift;; - - --help) echo "$usage"; exit $?;; - - -m) mode=$2 - case $mode in - *' '* | *' '* | *' -'* | *'*'* | *'?'* | *'['*) - echo "$0: invalid mode: $mode" >&2 - exit 1;; - esac - shift;; - - -o) chowncmd="$chownprog $2" - shift;; - - -s) stripcmd=$stripprog;; - - -S) stripcmd="$stripprog $2" - shift;; - - -t) dst_arg=$2 - shift;; - - -T) no_target_directory=true;; - - --version) echo "$0 $scriptversion"; exit $?;; - - --) shift - break;; - - -*) echo "$0: invalid option: $1" >&2 - exit 1;; - - *) break;; - esac - shift -done - -if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then - # When -d is used, all remaining arguments are directories to create. - # When -t is used, the destination is already specified. - # Otherwise, the last argument is the destination. Remove it from $@. - for arg - do - if test -n "$dst_arg"; then - # $@ is not empty: it contains at least $arg. - set fnord "$@" "$dst_arg" - shift # fnord - fi - shift # arg - dst_arg=$arg - done -fi - -if test $# -eq 0; then - if test -z "$dir_arg"; then - echo "$0: no input file specified." >&2 - exit 1 - fi - # It's OK to call `install-sh -d' without argument. - # This can happen when creating conditional directories. - exit 0 -fi - -if test -z "$dir_arg"; then - do_exit='(exit $ret); exit $ret' - trap "ret=129; $do_exit" 1 - trap "ret=130; $do_exit" 2 - trap "ret=141; $do_exit" 13 - trap "ret=143; $do_exit" 15 - - # Set umask so as not to create temps with too-generous modes. - # However, 'strip' requires both read and write access to temps. - case $mode in - # Optimize common cases. - *644) cp_umask=133;; - *755) cp_umask=22;; - - *[0-7]) - if test -z "$stripcmd"; then - u_plus_rw= - else - u_plus_rw='% 200' - fi - cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; - *) - if test -z "$stripcmd"; then - u_plus_rw= - else - u_plus_rw=,u+rw - fi - cp_umask=$mode$u_plus_rw;; - esac -fi - -for src -do - # Protect names starting with `-'. - case $src in - -*) src=./$src;; - esac - - if test -n "$dir_arg"; then - dst=$src - dstdir=$dst - test -d "$dstdir" - dstdir_status=$? - else - - # Waiting for this to be detected by the "$cpprog $src $dsttmp" command - # might cause directories to be created, which would be especially bad - # if $src (and thus $dsttmp) contains '*'. - if test ! -f "$src" && test ! -d "$src"; then - echo "$0: $src does not exist." >&2 - exit 1 - fi - - if test -z "$dst_arg"; then - echo "$0: no destination specified." >&2 - exit 1 - fi - - dst=$dst_arg - # Protect names starting with `-'. - case $dst in - -*) dst=./$dst;; - esac - - # If destination is a directory, append the input filename; won't work - # if double slashes aren't ignored. - if test -d "$dst"; then - if test -n "$no_target_directory"; then - echo "$0: $dst_arg: Is a directory" >&2 - exit 1 - fi - dstdir=$dst - dst=$dstdir/`basename "$src"` - dstdir_status=0 - else - # Prefer dirname, but fall back on a substitute if dirname fails. - dstdir=` - (dirname "$dst") 2>/dev/null || - expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$dst" : 'X\(//\)[^/]' \| \ - X"$dst" : 'X\(//\)$' \| \ - X"$dst" : 'X\(/\)' \| . 2>/dev/null || - echo X"$dst" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ - s//\1/ - q - } - /^X\(\/\/\)[^/].*/{ - s//\1/ - q - } - /^X\(\/\/\)$/{ - s//\1/ - q - } - /^X\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q' - ` - - test -d "$dstdir" - dstdir_status=$? - fi - fi - - obsolete_mkdir_used=false - - if test $dstdir_status != 0; then - case $posix_mkdir in - '') - # Create intermediate dirs using mode 755 as modified by the umask. - # This is like FreeBSD 'install' as of 1997-10-28. - umask=`umask` - case $stripcmd.$umask in - # Optimize common cases. - *[2367][2367]) mkdir_umask=$umask;; - .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; - - *[0-7]) - mkdir_umask=`expr $umask + 22 \ - - $umask % 100 % 40 + $umask % 20 \ - - $umask % 10 % 4 + $umask % 2 - `;; - *) mkdir_umask=$umask,go-w;; - esac - - # With -d, create the new directory with the user-specified mode. - # Otherwise, rely on $mkdir_umask. - if test -n "$dir_arg"; then - mkdir_mode=-m$mode - else - mkdir_mode= - fi - - posix_mkdir=false - case $umask in - *[123567][0-7][0-7]) - # POSIX mkdir -p sets u+wx bits regardless of umask, which - # is incompatible with FreeBSD 'install' when (umask & 300) != 0. - ;; - *) - tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ - trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 - - if (umask $mkdir_umask && - exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 - then - if test -z "$dir_arg" || { - # Check for POSIX incompatibilities with -m. - # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or - # other-writeable bit of parent directory when it shouldn't. - # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. - ls_ld_tmpdir=`ls -ld "$tmpdir"` - case $ls_ld_tmpdir in - d????-?r-*) different_mode=700;; - d????-?--*) different_mode=755;; - *) false;; - esac && - $mkdirprog -m$different_mode -p -- "$tmpdir" && { - ls_ld_tmpdir_1=`ls -ld "$tmpdir"` - test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" - } - } - then posix_mkdir=: - fi - rmdir "$tmpdir/d" "$tmpdir" - else - # Remove any dirs left behind by ancient mkdir implementations. - rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null - fi - trap '' 0;; - esac;; - esac - - if - $posix_mkdir && ( - umask $mkdir_umask && - $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" - ) - then : - else - - # The umask is ridiculous, or mkdir does not conform to POSIX, - # or it failed possibly due to a race condition. Create the - # directory the slow way, step by step, checking for races as we go. - - case $dstdir in - /*) prefix='/';; - -*) prefix='./';; - *) prefix='';; - esac - - eval "$initialize_posix_glob" - - oIFS=$IFS - IFS=/ - $posix_glob set -f - set fnord $dstdir - shift - $posix_glob set +f - IFS=$oIFS - - prefixes= - - for d - do - test -z "$d" && continue - - prefix=$prefix$d - if test -d "$prefix"; then - prefixes= - else - if $posix_mkdir; then - (umask=$mkdir_umask && - $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break - # Don't fail if two instances are running concurrently. - test -d "$prefix" || exit 1 - else - case $prefix in - *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; - *) qprefix=$prefix;; - esac - prefixes="$prefixes '$qprefix'" - fi - fi - prefix=$prefix/ - done - - if test -n "$prefixes"; then - # Don't fail if two instances are running concurrently. - (umask $mkdir_umask && - eval "\$doit_exec \$mkdirprog $prefixes") || - test -d "$dstdir" || exit 1 - obsolete_mkdir_used=true - fi - fi - fi - - if test -n "$dir_arg"; then - { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && - { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && - { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || - test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 - else - - # Make a couple of temp file names in the proper directory. - dsttmp=$dstdir/_inst.$$_ - rmtmp=$dstdir/_rm.$$_ - - # Trap to clean up those temp files at exit. - trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 - - # Copy the file name to the temp name. - (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && - - # and set any options; do chmod last to preserve setuid bits. - # - # If any of these fail, we abort the whole thing. If we want to - # ignore errors from any of these, just make sure not to ignore - # errors from the above "$doit $cpprog $src $dsttmp" command. - # - { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && - { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && - { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && - { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && - - # If -C, don't bother to copy if it wouldn't change the file. - if $copy_on_change && - old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && - new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && - - eval "$initialize_posix_glob" && - $posix_glob set -f && - set X $old && old=:$2:$4:$5:$6 && - set X $new && new=:$2:$4:$5:$6 && - $posix_glob set +f && - - test "$old" = "$new" && - $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 - then - rm -f "$dsttmp" - else - # Rename the file to the real destination. - $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || - - # The rename failed, perhaps because mv can't rename something else - # to itself, or perhaps because mv is so ancient that it does not - # support -f. - { - # Now remove or move aside any old file at destination location. - # We try this two ways since rm can't unlink itself on some - # systems and the destination file might be busy for other - # reasons. In this case, the final cleanup might fail but the new - # file should still install successfully. - { - test ! -f "$dst" || - $doit $rmcmd -f "$dst" 2>/dev/null || - { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && - { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } - } || - { echo "$0: cannot unlink or rename $dst" >&2 - (exit 1); exit 1 - } - } && - - # Now rename the file to the real destination. - $doit $mvcmd "$dsttmp" "$dst" - } - fi || exit 1 - - trap '' 0 - fi -done - -# Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "scriptversion=" -# time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-time-zone: "UTC" -# time-stamp-end: "; # UTC" -# End: diff --git a/autoconf/tea/tclconfig/tcl.m4 b/autoconf/tea/tclconfig/tcl.m4 deleted file mode 100644 index c83d660e18..0000000000 --- a/autoconf/tea/tclconfig/tcl.m4 +++ /dev/null @@ -1,4067 +0,0 @@ -# tcl.m4 -- -# -# This file provides a set of autoconf macros to help TEA-enable -# a Tcl extension. -# -# Copyright (c) 1999-2000 Ajuba Solutions. -# Copyright (c) 2002-2005 ActiveState Corporation. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. - -AC_PREREQ([2.69]) - -# Possible values for key variables defined: -# -# TEA_WINDOWINGSYSTEM - win32 aqua x11 (mirrors 'tk windowingsystem') -# TEA_PLATFORM - windows unix -# TEA_TK_EXTENSION - True if this is a Tk extension -# - -#------------------------------------------------------------------------ -# TEA_PATH_TCLCONFIG -- -# -# Locate the tclConfig.sh file and perform a sanity check on -# the Tcl compile flags -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tcl=... -# -# Defines the following vars: -# TCL_BIN_DIR Full path to the directory containing -# the tclConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PATH_TCLCONFIG], [ - dnl TEA specific: Make sure we are initialized - AC_REQUIRE([TEA_INIT]) - # - # Ok, lets find the tcl configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tcl - # - - if test x"${no_tcl}" = x ; then - # we reset no_tcl in case something fails here - no_tcl=true - AC_ARG_WITH(tcl, - AS_HELP_STRING([--with-tcl], - [directory containing tcl configuration (tclConfig.sh)]), - [with_tclconfig="${withval}"]) - AC_MSG_CHECKING([for Tcl configuration]) - AC_CACHE_VAL(ac_cv_c_tclconfig,[ - - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - case "${with_tclconfig}" in - */tclConfig.sh ) - if test -f "${with_tclconfig}"; then - AC_MSG_WARN([--with-tcl argument should refer to directory containing tclConfig.sh, not to tclConfig.sh itself]) - with_tclconfig="`echo "${with_tclconfig}" | sed 's!/tclConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd "${with_tclconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh]) - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/Library/Frameworks/Tcl.framework 2>/dev/null` \ - `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/Network/Library/Frameworks/Tcl.framework 2>/dev/null` \ - `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/Tcl.framework 2>/dev/null` \ - ; do - if test -f "$i/Tcl.framework/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/Tcl.framework; pwd)`" - break - fi - done - fi - - # TEA specific: on Windows, check in common installation locations - if test "${TEA_PLATFORM}" = "windows" \ - -a x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d C:/Tcl/lib 2>/dev/null` \ - `ls -d C:/Progra~1/Tcl/lib 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/pkg/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - `ls -d /usr/lib/tcl8.6 2>/dev/null` \ - `ls -d /usr/lib/tcl8.5 2>/dev/null` \ - `ls -d /usr/local/lib/tcl8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tcl8.5 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tcl8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tcl8.5 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tclconfig}" = x ; then - TCL_BIN_DIR="# no Tcl configs found" - AC_MSG_ERROR([Can't find Tcl configuration definitions. Use --with-tcl to specify a directory containing tclConfig.sh]) - else - no_tcl= - TCL_BIN_DIR="${ac_cv_c_tclconfig}" - AC_MSG_RESULT([found ${TCL_BIN_DIR}/tclConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_PATH_TKCONFIG -- -# -# Locate the tkConfig.sh file -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tk=... -# -# Defines the following vars: -# TK_BIN_DIR Full path to the directory containing -# the tkConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PATH_TKCONFIG], [ - # - # Ok, lets find the tk configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tk - # - - if test x"${no_tk}" = x ; then - # we reset no_tk in case something fails here - no_tk=true - AC_ARG_WITH(tk, - AS_HELP_STRING([--with-tk], - [directory containing tk configuration (tkConfig.sh)]), - [with_tkconfig="${withval}"]) - AC_MSG_CHECKING([for Tk configuration]) - AC_CACHE_VAL(ac_cv_c_tkconfig,[ - - # First check to see if --with-tkconfig was specified. - if test x"${with_tkconfig}" != x ; then - case "${with_tkconfig}" in - */tkConfig.sh ) - if test -f "${with_tkconfig}"; then - AC_MSG_WARN([--with-tk argument should refer to directory containing tkConfig.sh, not to tkConfig.sh itself]) - with_tkconfig="`echo "${with_tkconfig}" | sed 's!/tkConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tkconfig}/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd "${with_tkconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tkconfig} directory doesn't contain tkConfig.sh]) - fi - fi - - # then check for a private Tk library - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ../tk \ - `ls -dr ../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tk \ - `ls -dr ../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tk \ - `ls -dr ../../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - ; do - if test -f "$i/Tk.framework/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/Tk.framework; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/pkg/lib 2>/dev/null` \ - `ls -d /usr/lib/tk8.6 2>/dev/null` \ - `ls -d /usr/lib/tk8.5 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - `ls -d /usr/local/lib/tk8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tk8.5 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tk8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tk8.5 2>/dev/null` \ - ; do - if test -f "$i/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # TEA specific: on Windows, check in common installation locations - if test "${TEA_PLATFORM}" = "windows" \ - -a x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d C:/Tcl/lib 2>/dev/null` \ - `ls -d C:/Progra~1/Tcl/lib 2>/dev/null` \ - ; do - if test -f "$i/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ${srcdir}/../tk \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tkconfig}" = x ; then - TK_BIN_DIR="# no Tk configs found" - AC_MSG_ERROR([Can't find Tk configuration definitions. Use --with-tk to specify a directory containing tkConfig.sh]) - else - no_tk= - TK_BIN_DIR="${ac_cv_c_tkconfig}" - AC_MSG_RESULT([found ${TK_BIN_DIR}/tkConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_TCLCONFIG -- -# -# Load the tclConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TCL_BIN_DIR -# -# Results: -# -# Substitutes the following vars: -# TCL_BIN_DIR -# TCL_SRC_DIR -# TCL_LIB_FILE -# TCL_ZIP_FILE -# TCL_ZIPFS_SUPPORT -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LOAD_TCLCONFIG], [ - AC_MSG_CHECKING([for existence of ${TCL_BIN_DIR}/tclConfig.sh]) - - if test -f "${TCL_BIN_DIR}/tclConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TCL_BIN_DIR}/tclConfig.sh" - else - AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh]) - fi - - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TCL_BIN_DIR}/Makefile" ; then - TCL_LIB_SPEC="${TCL_BUILD_LIB_SPEC}" - TCL_STUB_LIB_SPEC="${TCL_BUILD_STUB_LIB_SPEC}" - TCL_STUB_LIB_PATH="${TCL_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tcl.framework installed in an arbitrary location. - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - if test -f "${TCL_BIN_DIR}/${TCL_LIB_FILE}"; then - for i in "`cd "${TCL_BIN_DIR}"; pwd`" \ - "`cd "${TCL_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TCL_LIB_FILE}.framework"; then - TCL_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TCL_LIB_FILE}" - break - fi - done - fi - if test -f "${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}"; then - TCL_STUB_LIB_SPEC="-L`echo "${TCL_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TCL_STUB_LIB_FLAG}" - TCL_STUB_LIB_PATH="${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}" - fi - ;; - esac - fi - - AC_SUBST(TCL_VERSION) - AC_SUBST(TCL_PATCH_LEVEL) - AC_SUBST(TCL_BIN_DIR) - AC_SUBST(TCL_SRC_DIR) - - AC_SUBST(TCL_LIB_FILE) - AC_SUBST(TCL_LIB_FLAG) - AC_SUBST(TCL_LIB_SPEC) - - AC_SUBST(TCL_STUB_LIB_FILE) - AC_SUBST(TCL_STUB_LIB_FLAG) - AC_SUBST(TCL_STUB_LIB_SPEC) - - AC_MSG_CHECKING([platform]) - hold_cc=$CC; CC="$TCL_CC" - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ - #ifdef _WIN32 - #error win32 - #endif - ]])],[ - # first test we've already retrieved platform (cross-compile), fallback to unix otherwise: - TEA_PLATFORM="${TEA_PLATFORM-unix}" - CYGPATH=echo - ],[ - TEA_PLATFORM="windows" - AC_CHECK_PROG(CYGPATH, cygpath, cygpath -m, echo) - ]) - CC=$hold_cc - AC_MSG_RESULT($TEA_PLATFORM) - - # The BUILD_$pkg is to define the correct extern storage class - # handling when making this package - AC_DEFINE_UNQUOTED(BUILD_${PACKAGE_NAME}, [], - [Building extension source?]) - # Do this here as we have fully defined TEA_PLATFORM now - if test "${TEA_PLATFORM}" = "windows" ; then - EXEEXT=".exe" - CLEANFILES="$CLEANFILES *.lib *.dll *.pdb *.exp" - fi - - # TEA specific: - AC_SUBST(CLEANFILES) - AC_SUBST(TCL_LIBS) - AC_SUBST(TCL_DEFS) - AC_SUBST(TCL_EXTRA_CFLAGS) - AC_SUBST(TCL_LD_FLAGS) - AC_SUBST(TCL_SHLIB_LD_LIBS) -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_TKCONFIG -- -# -# Load the tkConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TK_BIN_DIR -# -# Results: -# -# Sets the following vars that should be in tkConfig.sh: -# TK_BIN_DIR -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LOAD_TKCONFIG], [ - AC_MSG_CHECKING([for existence of ${TK_BIN_DIR}/tkConfig.sh]) - - if test -f "${TK_BIN_DIR}/tkConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TK_BIN_DIR}/tkConfig.sh" - else - AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh]) - fi - - # If the TK_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TK_LIB_SPEC will be set to the value - # of TK_BUILD_LIB_SPEC. An extension should make use of TK_LIB_SPEC - # instead of TK_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TK_BIN_DIR}/Makefile" ; then - TK_LIB_SPEC="${TK_BUILD_LIB_SPEC}" - TK_STUB_LIB_SPEC="${TK_BUILD_STUB_LIB_SPEC}" - TK_STUB_LIB_PATH="${TK_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tk.framework installed in an arbitrary location. - case ${TK_DEFS} in - *TK_FRAMEWORK*) - if test -f "${TK_BIN_DIR}/${TK_LIB_FILE}"; then - for i in "`cd "${TK_BIN_DIR}"; pwd`" \ - "`cd "${TK_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TK_LIB_FILE}.framework"; then - TK_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TK_LIB_FILE}" - break - fi - done - fi - if test -f "${TK_BIN_DIR}/${TK_STUB_LIB_FILE}"; then - TK_STUB_LIB_SPEC="-L` echo "${TK_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TK_STUB_LIB_FLAG}" - TK_STUB_LIB_PATH="${TK_BIN_DIR}/${TK_STUB_LIB_FILE}" - fi - ;; - esac - fi - - # TEA specific: Ensure windowingsystem is defined - if test "${TEA_PLATFORM}" = "unix" ; then - case ${TK_DEFS} in - *MAC_OSX_TK*) - AC_DEFINE(MAC_OSX_TK, 1, [Are we building against Mac OS X TkAqua?]) - TEA_WINDOWINGSYSTEM="aqua" - ;; - *) - TEA_WINDOWINGSYSTEM="x11" - ;; - esac - elif test "${TEA_PLATFORM}" = "windows" ; then - TEA_WINDOWINGSYSTEM="win32" - fi - - AC_SUBST(TK_VERSION) - AC_SUBST(TK_BIN_DIR) - AC_SUBST(TK_SRC_DIR) - - AC_SUBST(TK_LIB_FILE) - AC_SUBST(TK_LIB_FLAG) - AC_SUBST(TK_LIB_SPEC) - - AC_SUBST(TK_STUB_LIB_FILE) - AC_SUBST(TK_STUB_LIB_FLAG) - AC_SUBST(TK_STUB_LIB_SPEC) - - # TEA specific: - AC_SUBST(TK_LIBS) - AC_SUBST(TK_XINCLUDES) -]) - -#------------------------------------------------------------------------ -# TEA_PROG_TCLSH -# Determine the fully qualified path name of the tclsh executable -# in the Tcl build directory or the tclsh installed in a bin -# directory. This macro will correctly determine the name -# of the tclsh executable even if tclsh has not yet been -# built in the build directory. The tclsh found is always -# associated with a tclConfig.sh file. This tclsh should be used -# only for running extension test cases. It should never be -# or generation of files (like pkgIndex.tcl) at build time. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# TCLSH_PROG -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PROG_TCLSH], [ - AC_MSG_CHECKING([for tclsh]) - if test -f "${TCL_BIN_DIR}/Makefile" ; then - # tclConfig.sh is in Tcl build directory - if test "${TEA_PLATFORM}" = "windows"; then - if test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" - elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" - elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" - elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" - fi - else - TCLSH_PROG="${TCL_BIN_DIR}/tclsh" - fi - else - # tclConfig.sh is in install location - if test "${TEA_PLATFORM}" = "windows"; then - TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" - else - TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}.${TCL_MINOR_VERSION}" - fi - list="`ls -d ${TCL_BIN_DIR}/../bin 2>/dev/null` \ - `ls -d ${TCL_BIN_DIR}/.. 2>/dev/null` \ - `ls -d ${TCL_PREFIX}/bin 2>/dev/null`" - for i in $list ; do - if test -f "$i/${TCLSH_PROG}" ; then - REAL_TCL_BIN_DIR="`cd "$i"; pwd`/" - break - fi - done - TCLSH_PROG="${REAL_TCL_BIN_DIR}${TCLSH_PROG}" - fi - AC_MSG_RESULT([${TCLSH_PROG}]) - AC_SUBST(TCLSH_PROG) -]) - -#------------------------------------------------------------------------ -# TEA_PROG_WISH -# Determine the fully qualified path name of the wish executable -# in the Tk build directory or the wish installed in a bin -# directory. This macro will correctly determine the name -# of the wish executable even if wish has not yet been -# built in the build directory. The wish found is always -# associated with a tkConfig.sh file. This wish should be used -# only for running extension test cases. It should never be -# or generation of files (like pkgIndex.tcl) at build time. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# WISH_PROG -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PROG_WISH], [ - AC_MSG_CHECKING([for wish]) - if test -f "${TK_BIN_DIR}/Makefile" ; then - # tkConfig.sh is in Tk build directory - if test "${TEA_PLATFORM}" = "windows"; then - if test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" - elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}s${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}$s{EXEEXT}" - elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" - elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" - fi - else - WISH_PROG="${TK_BIN_DIR}/wish" - fi - else - # tkConfig.sh is in install location - if test "${TEA_PLATFORM}" = "windows"; then - WISH_PROG="wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" - else - WISH_PROG="wish${TK_MAJOR_VERSION}.${TK_MINOR_VERSION}" - fi - list="`ls -d ${TK_BIN_DIR}/../bin 2>/dev/null` \ - `ls -d ${TK_BIN_DIR}/.. 2>/dev/null` \ - `ls -d ${TK_PREFIX}/bin 2>/dev/null`" - for i in $list ; do - if test -f "$i/${WISH_PROG}" ; then - REAL_TK_BIN_DIR="`cd "$i"; pwd`/" - break - fi - done - WISH_PROG="${REAL_TK_BIN_DIR}${WISH_PROG}" - fi - AC_MSG_RESULT([${WISH_PROG}]) - AC_SUBST(WISH_PROG) -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_SHARED -- -# -# Allows the building of shared libraries -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-shared=yes|no -# --enable-stubs=yes|no -# -# Defines the following vars: -# STATIC_BUILD Used for building import/export libraries -# on Windows. -# -# Sets the following vars: -# SHARED_BUILD Value of 1 or 0 -# STUBS_BUILD Value if 1 or 0 -# USE_TCL_STUBS Value true: if SHARED_BUILD or --enable-stubs -# USE_TCLOO_STUBS Value true: if SHARED_BUILD or --enable-stubs -# USE_TK_STUBS Value true: if SHARED_BUILD or --enable-stubs -# AND TEA_WINDOWING_SYSTEM != "" -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ENABLE_SHARED], [ - AC_MSG_CHECKING([how to build libraries]) - AC_ARG_ENABLE(shared, - AS_HELP_STRING([--enable-shared], - [build and link with shared libraries (default: on)]), - [shared_ok=$enableval], [shared_ok=yes]) - - if test "${enable_shared+set}" = set; then - enableval="$enable_shared" - shared_ok=$enableval - else - shared_ok=yes - fi - - AC_ARG_ENABLE(stubs, - AS_HELP_STRING([--enable-stubs], - [build and link with stub libraries. Always true for shared builds (default: on)]), - [stubs_ok=$enableval], [stubs_ok=yes]) - - if test "${enable_stubs+set}" = set; then - enableval="$enable_stubs" - stubs_ok=$enableval - else - stubs_ok=yes - fi - - # Stubs are always enabled for shared builds - if test "$shared_ok" = "yes" ; then - AC_MSG_RESULT([shared]) - SHARED_BUILD=1 - STUBS_BUILD=1 - else - AC_MSG_RESULT([static]) - SHARED_BUILD=0 - AC_DEFINE(STATIC_BUILD, 1, [This a static build]) - if test "$stubs_ok" = "yes" ; then - STUBS_BUILD=1 - else - STUBS_BUILD=0 - fi - fi - if test "${STUBS_BUILD}" = "1" ; then - AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs]) - AC_DEFINE(USE_TCLOO_STUBS, 1, [Use TclOO stubs]) - if test "${TEA_WINDOWINGSYSTEM}" != ""; then - AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs]) - fi - fi - - AC_SUBST(SHARED_BUILD) - AC_SUBST(STUBS_BUILD) -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_THREADS -- -# -# Specify if thread support should be enabled. If "yes" is specified -# as an arg (optional), threads are enabled by default, "no" means -# threads are disabled. "yes" is the default. -# -# TCL_THREADS is checked so that if you are compiling an extension -# against a threaded core, your extension must be compiled threaded -# as well. -# -# Note that it is legal to have a thread enabled extension run in a -# threaded or non-threaded Tcl core, but a non-threaded extension may -# only run in a non-threaded Tcl core. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-threads -# -# Sets the following vars: -# THREADS_LIBS Thread library(s) -# -# Defines the following vars: -# TCL_THREADS -# _REENTRANT -# _THREAD_SAFE -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ENABLE_THREADS], [ - AC_ARG_ENABLE(threads, - AS_HELP_STRING([--enable-threads], - [build with threads (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) - - if test "${enable_threads+set}" = set; then - enableval="$enable_threads" - tcl_ok=$enableval - else - tcl_ok=yes - fi - - if test "$tcl_ok" = "yes" -o "${TCL_THREADS}" = 1; then - TCL_THREADS=1 - - if test "${TEA_PLATFORM}" != "windows" ; then - # We are always OK on Windows, so check what this platform wants: - - # USE_THREAD_ALLOC tells us to try the special thread-based - # allocator that significantly reduces lock contention - AC_DEFINE(USE_THREAD_ALLOC, 1, - [Do we want to use the threaded memory allocator?]) - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - if test "`uname -s`" = "SunOS" ; then - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - fi - AC_DEFINE(_THREAD_SAFE, 1, [Do we want the thread-safe OS API?]) - AC_CHECK_LIB(pthread,pthread_mutex_init,tcl_ok=yes,tcl_ok=no) - if test "$tcl_ok" = "no"; then - # Check a little harder for __pthread_mutex_init in the same - # library, as some systems hide it there until pthread.h is - # defined. We could alternatively do an AC_TRY_COMPILE with - # pthread.h, but that will work with libpthread really doesn't - # exist, like AIX 4.2. [Bug: 4359] - AC_CHECK_LIB(pthread, __pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - fi - - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthread" - else - AC_CHECK_LIB(pthreads, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthreads" - else - AC_CHECK_LIB(c, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "no"; then - AC_CHECK_LIB(c_r, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -pthread" - else - TCL_THREADS=0 - AC_MSG_WARN([Do not know how to find pthread lib on your system - thread support disabled]) - fi - fi - fi - fi - fi - else - TCL_THREADS=0 - fi - # Do checking message here to not mess up interleaved configure output - AC_MSG_CHECKING([for building with threads]) - if test "${TCL_THREADS}" = 1; then - AC_DEFINE(TCL_THREADS, 1, [Are we building with threads enabled?]) - AC_MSG_RESULT([yes (default)]) - else - AC_MSG_RESULT([no]) - fi - # TCL_THREADS sanity checking. See if our request for building with - # threads is the same as the way Tcl was built. If not, warn the user. - case ${TCL_DEFS} in - *THREADS=1*) - if test "${TCL_THREADS}" = "0"; then - AC_MSG_WARN([ - Building ${PACKAGE_NAME} without threads enabled, but building against Tcl - that IS thread-enabled. It is recommended to use --enable-threads.]) - fi - ;; - esac - AC_SUBST(TCL_THREADS) -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_SYMBOLS -- -# -# Specify if debugging symbols should be used. -# Memory (TCL_MEM_DEBUG) debugging can also be enabled. -# -# Arguments: -# none -# -# TEA varies from core Tcl in that C|LDFLAGS_DEFAULT receives -# the value of C|LDFLAGS_OPTIMIZE|DEBUG already substituted. -# Requires the following vars to be set in the Makefile: -# CFLAGS_DEFAULT -# LDFLAGS_DEFAULT -# -# Results: -# -# Adds the following arguments to configure: -# --enable-symbols -# -# Defines the following vars: -# CFLAGS_DEFAULT Sets to $(CFLAGS_DEBUG) if true -# Sets to "$(CFLAGS_OPTIMIZE) -DNDEBUG" if false -# LDFLAGS_DEFAULT Sets to $(LDFLAGS_DEBUG) if true -# Sets to $(LDFLAGS_OPTIMIZE) if false -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ENABLE_SYMBOLS], [ - dnl TEA specific: Make sure we are initialized - AC_REQUIRE([TEA_CONFIG_CFLAGS]) - AC_MSG_CHECKING([for build with symbols]) - AC_ARG_ENABLE(symbols, - AS_HELP_STRING([--enable-symbols], - [build with debugging symbols (default: off)]), - [tcl_ok=$enableval], [tcl_ok=no]) - if test "$tcl_ok" = "no"; then - CFLAGS_DEFAULT="${CFLAGS_OPTIMIZE} -DNDEBUG" - LDFLAGS_DEFAULT="${LDFLAGS_OPTIMIZE}" - AC_MSG_RESULT([no]) - AC_DEFINE(TCL_CFG_OPTIMIZED, 1, [Is this an optimized build?]) - else - CFLAGS_DEFAULT="${CFLAGS_DEBUG}" - LDFLAGS_DEFAULT="${LDFLAGS_DEBUG}" - if test "$tcl_ok" = "yes"; then - AC_MSG_RESULT([yes (standard debugging)]) - fi - fi - AC_SUBST(CFLAGS_DEFAULT) - AC_SUBST(LDFLAGS_DEFAULT) - - if test "$tcl_ok" = "mem" -o "$tcl_ok" = "all"; then - AC_DEFINE(TCL_MEM_DEBUG, 1, [Is memory debugging enabled?]) - fi - - if test "$tcl_ok" != "yes" -a "$tcl_ok" != "no"; then - if test "$tcl_ok" = "all"; then - AC_MSG_RESULT([enabled symbols mem debugging]) - else - AC_MSG_RESULT([enabled $tcl_ok debugging]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_LANGINFO -- -# -# Allows use of modern nl_langinfo check for better l10n. -# This is only relevant for Unix. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-langinfo=yes|no (default is yes) -# -# Defines the following vars: -# HAVE_LANGINFO Triggers use of nl_langinfo if defined. -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ENABLE_LANGINFO], [ - AC_ARG_ENABLE(langinfo, - AS_HELP_STRING([--enable-langinfo], - [use nl_langinfo if possible to determine encoding at startup, otherwise use old heuristic (default: on)]), - [langinfo_ok=$enableval], [langinfo_ok=yes]) - - HAVE_LANGINFO=0 - if test "$langinfo_ok" = "yes"; then - AC_CHECK_HEADER(langinfo.h,[langinfo_ok=yes],[langinfo_ok=no]) - fi - AC_MSG_CHECKING([whether to use nl_langinfo]) - if test "$langinfo_ok" = "yes"; then - AC_CACHE_VAL(tcl_cv_langinfo_h, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[nl_langinfo(CODESET);]])], - [tcl_cv_langinfo_h=yes],[tcl_cv_langinfo_h=no])]) - AC_MSG_RESULT([$tcl_cv_langinfo_h]) - if test $tcl_cv_langinfo_h = yes; then - AC_DEFINE(HAVE_LANGINFO, 1, [Do we have nl_langinfo()?]) - fi - else - AC_MSG_RESULT([$langinfo_ok]) - fi -]) - -#-------------------------------------------------------------------- -# TEA_CONFIG_SYSTEM -# -# Determine what the system is (some things cannot be easily checked -# on a feature-driven basis, alas). This can usually be done via the -# "uname" command. -# -# Arguments: -# none -# -# Results: -# Defines the following var: -# -# system - System/platform/version identification code. -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_CONFIG_SYSTEM], [ - AC_CACHE_CHECK([system version], tcl_cv_sys_version, [ - # TEA specific: - if test "${TEA_PLATFORM}" = "windows" ; then - tcl_cv_sys_version=windows - else - tcl_cv_sys_version=`uname -s`-`uname -r` - if test "$?" -ne 0 ; then - AC_MSG_WARN([can't find uname command]) - tcl_cv_sys_version=unknown - else - if test "`uname -s`" = "AIX" ; then - tcl_cv_sys_version=AIX-`uname -v`.`uname -r` - fi - if test "`uname -s`" = "NetBSD" -a -f /etc/debian_version ; then - tcl_cv_sys_version=NetBSD-Debian - fi - fi - fi - ]) - system=$tcl_cv_sys_version -]) - -#-------------------------------------------------------------------- -# TEA_CONFIG_CFLAGS -# -# Try to determine the proper flags to pass to the compiler -# for building shared libraries and other such nonsense. -# -# Arguments: -# none -# -# Results: -# -# Defines and substitutes the following vars: -# -# DL_OBJS, DL_LIBS - removed for TEA, only needed by core. -# LDFLAGS - Flags to pass to the compiler when linking object -# files into an executable application binary such -# as tclsh. -# LD_SEARCH_FLAGS-Flags to pass to ld, such as "-R /usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. Could -# be the same as CC_SEARCH_FLAGS if ${CC} is used to link. -# CC_SEARCH_FLAGS-Flags to pass to ${CC}, such as "-Wl,-rpath,/usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. -# SHLIB_CFLAGS - Flags to pass to cc when compiling the components -# of a shared library (may request position-independent -# code, among other things). -# SHLIB_LD - Base command to use for combining object files -# into a shared library. -# SHLIB_LD_LIBS - Dependent libraries for the linker to scan when -# creating shared libraries. This symbol typically -# goes at the end of the "ld" commands that build -# shared libraries. The value of the symbol defaults to -# "${LIBS}" if all of the dependent libraries should -# be specified when creating a shared library. If -# dependent libraries should not be specified (as on -# SunOS 4.x, where they cause the link to fail, or in -# general if Tcl and Tk aren't themselves shared -# libraries), then this symbol has an empty string -# as its value. -# SHLIB_SUFFIX - Suffix to use for the names of dynamically loadable -# extensions. An empty string means we don't know how -# to use shared libraries on this platform. -# LIB_SUFFIX - Specifies everything that comes after the "libfoo" -# in a static or shared library name, using the $PACKAGE_VERSION variable -# to put the version in the right place. This is used -# by platforms that need non-standard library names. -# Examples: ${PACKAGE_VERSION}.so.1.1 on NetBSD, since it needs -# to have a version after the .so, and ${PACKAGE_VERSION}.a -# on AIX, since a shared library needs to have -# a .a extension whereas shared objects for loadable -# extensions have a .so extension. Defaults to -# ${PACKAGE_VERSION}${SHLIB_SUFFIX}. -# CFLAGS_DEBUG - -# Flags used when running the compiler in debug mode -# CFLAGS_OPTIMIZE - -# Flags used when running the compiler in optimize mode -# CFLAGS - Additional CFLAGS added as necessary (usually 64-bit) -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_CONFIG_CFLAGS], [ - dnl TEA specific: Make sure we are initialized - AC_REQUIRE([TEA_INIT]) - - # Step 0.a: Enable 64 bit support? - - AC_MSG_CHECKING([if 64bit support is requested]) - AC_ARG_ENABLE(64bit, - AS_HELP_STRING([--enable-64bit], - [enable 64bit support (default: off)]), - [do64bit=$enableval], [do64bit=no]) - AC_MSG_RESULT([$do64bit]) - - # Step 0.b: Enable Solaris 64 bit VIS support? - - AC_MSG_CHECKING([if 64bit Sparc VIS support is requested]) - AC_ARG_ENABLE(64bit-vis, - AS_HELP_STRING([--enable-64bit-vis], - [enable 64bit Sparc VIS support (default: off)]), - [do64bitVIS=$enableval], [do64bitVIS=no]) - AC_MSG_RESULT([$do64bitVIS]) - # Force 64bit on with VIS - AS_IF([test "$do64bitVIS" = "yes"], [do64bit=yes]) - - # Step 0.c: Check if visibility support is available. Do this here so - # that platform specific alternatives can be used below if this fails. - - AC_CACHE_CHECK([if compiler supports visibility "hidden"], - tcl_cv_cc_visibility_hidden, [ - hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -Werror" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - extern __attribute__((__visibility__("hidden"))) void f(void); - void f(void) {}]], [[f();]])],[tcl_cv_cc_visibility_hidden=yes], - [tcl_cv_cc_visibility_hidden=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_visibility_hidden = yes], [ - AC_DEFINE(MODULE_SCOPE, - [extern __attribute__((__visibility__("hidden")))], - [Compiler support for module scope symbols]) - AC_DEFINE(HAVE_HIDDEN, [1], [Compiler support for module scope symbols]) - ]) - - # Step 0.d: Disable -rpath support? - - AC_MSG_CHECKING([if rpath support is requested]) - AC_ARG_ENABLE(rpath, - AS_HELP_STRING([--disable-rpath], - [disable rpath support (default: on)]), - [doRpath=$enableval], [doRpath=yes]) - AC_MSG_RESULT([$doRpath]) - - # Set the variable "system" to hold the name and version number - # for the system. - - TEA_CONFIG_SYSTEM - - # Require ranlib early so we can override it in special cases below. - - AC_REQUIRE([AC_PROG_RANLIB]) - - # Set configuration options based on system name and version. - # This is similar to Tcl's unix/tcl.m4 except that we've added a - # "windows" case and removed some core-only vars. - - do64bit_ok=no - # default to '{$LIBS}' and set to "" on per-platform necessary basis - SHLIB_LD_LIBS='${LIBS}' - # When ld needs options to work in 64-bit mode, put them in - # LDFLAGS_ARCH so they eventually end up in LDFLAGS even if [load] - # is disabled by the user. [Bug 1016796] - LDFLAGS_ARCH="" - UNSHARED_LIB_SUFFIX="" - # TEA specific: use PACKAGE_VERSION instead of VERSION - TCL_TRIM_DOTS='`echo ${PACKAGE_VERSION} | tr -d .`' - ECHO_VERSION='`echo ${PACKAGE_VERSION}`' - TCL_LIB_VERSIONS_OK=ok - CFLAGS_DEBUG=-g - AS_IF([test "$GCC" = yes], [ - CFLAGS_OPTIMIZE=-O2 - CFLAGS_WARNING="-Wall" - ], [ - CFLAGS_OPTIMIZE=-O - CFLAGS_WARNING="" - ]) - AC_CHECK_TOOL(AR, ar) - STLIB_LD='${AR} cr' - LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH" - AS_IF([test "x$SHLIB_VERSION" = x],[SHLIB_VERSION=""],[SHLIB_VERSION=".$SHLIB_VERSION"]) - case $system in - # TEA specific: - windows) - MACHINE="X86" - if test "$do64bit" != "no" ; then - case "$do64bit" in - amd64|x64|yes) - MACHINE="AMD64" ; # default to AMD64 64-bit build - ;; - arm64|aarch64) - MACHINE="ARM64" - ;; - ia64) - MACHINE="IA64" - ;; - esac - fi - - if test "$GCC" != "yes" ; then - if test "${SHARED_BUILD}" = "0" ; then - runtime=-MT - else - runtime=-MD - fi - case "x`echo \${VisualStudioVersion}`" in - x1[[4-9]]*) - lflags="${lflags} -nodefaultlib:libucrt.lib" - TEA_ADD_LIBS([ucrt.lib]) - ;; - *) - ;; - esac - - if test "$do64bit" != "no" ; then - CC="cl.exe" - RC="rc.exe" - lflags="${lflags} -nologo -MACHINE:${MACHINE} " - LINKBIN="link.exe" - CFLAGS_DEBUG="-nologo -Zi -Od -W3 ${runtime}d" - CFLAGS_OPTIMIZE="-nologo -O2 -W2 ${runtime}" - # Avoid 'unresolved external symbol __security_cookie' - # errors, c.f. http://support.microsoft.com/?id=894573 - TEA_ADD_LIBS([bufferoverflowU.lib]) - else - RC="rc" - lflags="${lflags} -nologo" - LINKBIN="link" - CFLAGS_DEBUG="-nologo -Z7 -Od -W3 -WX ${runtime}d" - CFLAGS_OPTIMIZE="-nologo -O2 -W2 ${runtime}" - fi - fi - - if test "$GCC" = "yes"; then - # mingw gcc mode - AC_CHECK_TOOL(RC, windres) - CFLAGS_DEBUG="-g" - CFLAGS_OPTIMIZE="-O2 -fomit-frame-pointer" - SHLIB_LD='${CC} -shared' - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - LDFLAGS_CONSOLE="-wl,--subsystem,console ${lflags}" - LDFLAGS_WINDOW="-wl,--subsystem,windows ${lflags}" - - AC_CACHE_CHECK(for cross-compile version of gcc, - ac_cv_cross, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef _WIN32 - #error cross-compiler - #endif - ]], [[]])], - [ac_cv_cross=yes], - [ac_cv_cross=no]) - ) - if test "$ac_cv_cross" = "yes"; then - case "$do64bit" in - amd64|x64|yes) - CC="x86_64-w64-mingw32-${CC}" - LD="x86_64-w64-mingw32-ld" - AR="x86_64-w64-mingw32-ar" - RANLIB="x86_64-w64-mingw32-ranlib" - RC="x86_64-w64-mingw32-windres" - ;; - arm64|aarch64) - CC="aarch64-w64-mingw32-clang" - LD="aarch64-w64-mingw32-ld" - AR="aarch64-w64-mingw32-ar" - RANLIB="aarch64-w64-mingw32-ranlib" - RC="aarch64-w64-mingw32-windres" - ;; - *) - CC="i686-w64-mingw32-${CC}" - LD="i686-w64-mingw32-ld" - AR="i686-w64-mingw32-ar" - RANLIB="i686-w64-mingw32-ranlib" - RC="i686-w64-mingw32-windres" - ;; - esac - fi - - else - SHLIB_LD="${LINKBIN} -dll ${lflags}" - # link -lib only works when -lib is the first arg - STLIB_LD="${LINKBIN} -lib ${lflags}" - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.lib' - PATHTYPE=-w - # For information on what debugtype is most useful, see: - # http://msdn.microsoft.com/library/en-us/dnvc60/html/gendepdebug.asp - # and also - # http://msdn2.microsoft.com/en-us/library/y0zzbyt4%28VS.80%29.aspx - # This essentially turns it all on. - LDFLAGS_DEBUG="-debug -debugtype:cv" - LDFLAGS_OPTIMIZE="-release" - LDFLAGS_CONSOLE="-link -subsystem:console ${lflags}" - LDFLAGS_WINDOW="-link -subsystem:windows ${lflags}" - fi - - SHLIB_SUFFIX=".dll" - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.dll' - - TCL_LIB_VERSIONS_OK=nodots - ;; - AIX-*) - AS_IF([test "$GCC" != "yes"], [ - # AIX requires the _r compiler when gcc isn't being used - case "${CC}" in - *_r|*_r\ *) - # ok ... - ;; - *) - # Make sure only first arg gets _r - CC=`echo "$CC" | sed -e 's/^\([[^ ]]*\)/\1_r/'` - ;; - esac - AC_MSG_RESULT([Using $CC for compiling with threads]) - ]) - LIBS="$LIBS -lc" - SHLIB_CFLAGS="" - SHLIB_SUFFIX=".so" - - LD_LIBRARY_PATH_VAR="LIBPATH" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = yes], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -q64" - LDFLAGS_ARCH="-q64" - RANLIB="${RANLIB} -X64" - AR="${AR} -X64" - SHLIB_LD_FLAGS="-b64" - ]) - ]) - - AS_IF([test "`uname -m`" = ia64], [ - # AIX-5 uses ELF style dynamic libraries on IA-64, but not PPC - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - AS_IF([test "$GCC" = yes], [ - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - ], [ - CC_SEARCH_FLAGS='"-R${LIB_RUNTIME_DIR}"' - ]) - LD_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' - ], [ - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared -Wl,-bexpall' - ], [ - SHLIB_LD="/bin/ld -bhalt:4 -bM:SRE -bexpall -H512 -T512 -bnoentry" - LDFLAGS="$LDFLAGS -brtl" - ]) - SHLIB_LD="${SHLIB_LD} ${SHLIB_LD_FLAGS}" - CC_SEARCH_FLAGS='"-L${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - ;; - BeOS*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -nostart' - SHLIB_SUFFIX=".so" - - #----------------------------------------------------------- - # Check for inet_ntoa in -lbind, for BeOS (which also needs - # -lsocket, even if the network functions are in -lnet which - # is always linked to, for compatibility. - #----------------------------------------------------------- - AC_CHECK_LIB(bind, inet_ntoa, [LIBS="$LIBS -lbind -lsocket"]) - ;; - BSD/OS-2.1*|BSD/OS-3*) - SHLIB_CFLAGS="" - SHLIB_LD="shlicc -r" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - BSD/OS-4.*) - SHLIB_CFLAGS="-export-dynamic -fPIC" - SHLIB_LD='${CC} -shared' - SHLIB_SUFFIX=".so" - LDFLAGS="$LDFLAGS -export-dynamic" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - CYGWIN_*) - SHLIB_CFLAGS="" - SHLIB_LD='${CC} -shared' - SHLIB_SUFFIX=".dll" - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} -Wl,--out-implib,\$[@].a" - EXEEXT=".exe" - do64bit_ok=yes - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - dgux*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - Haiku*) - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - SHLIB_CFLAGS="-fPIC" - SHLIB_SUFFIX=".so" - SHLIB_LD='${CC} ${CFLAGS} ${LDFLAGS} -shared' - AC_CHECK_LIB(network, inet_ntoa, [LIBS="$LIBS -lnetwork"]) - ;; - HP-UX-*.11.*) - # Use updated header definitions where possible - AC_DEFINE(_XOPEN_SOURCE_EXTENDED, 1, [Do we want to use the XOPEN network library?]) - # TEA specific: Needed by Tcl, but not most extensions - #AC_DEFINE(_XOPEN_SOURCE, 1, [Do we want to use the XOPEN network library?]) - #LIBS="$LIBS -lxnet" # Use the XOPEN network library - - AS_IF([test "`uname -m`" = ia64], [ - SHLIB_SUFFIX=".so" - ], [ - SHLIB_SUFFIX=".sl" - ]) - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='"-Wl,+s,+b,${LIB_RUNTIME_DIR}:."' - LD_SEARCH_FLAGS='+s +b "${LIB_RUNTIME_DIR}:."' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ], [ - CFLAGS="$CFLAGS -z" - ]) - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = "yes"], [ - AS_IF([test "$GCC" = yes], [ - case `${CC} -dumpmachine` in - hppa64*) - # 64-bit gcc in use. Fix flags for GNU ld. - do64bit_ok=yes - SHLIB_LD='${CC} -shared' - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ;; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ;; - esac - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS +DD64" - LDFLAGS_ARCH="+DD64" - ]) - ]) ;; - HP-UX-*.08.*|HP-UX-*.09.*|HP-UX-*.10.*) - SHLIB_SUFFIX=".sl" - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - SHLIB_LD_LIBS="" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='"-Wl,+s,+b,${LIB_RUNTIME_DIR}:."' - LD_SEARCH_FLAGS='+s +b "${LIB_RUNTIME_DIR}:."' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) ;; - IRIX-5.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -shared -rdata_shared" - SHLIB_SUFFIX=".so" - AC_LIBOBJ(mkstemp) - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) - ;; - IRIX-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) - AS_IF([test "$GCC" = yes], [ - CFLAGS="$CFLAGS -mabi=n32" - LDFLAGS="$LDFLAGS -mabi=n32" - ], [ - case $system in - IRIX-6.3) - # Use to build 6.2 compatible binaries on 6.3. - CFLAGS="$CFLAGS -n32 -D_OLD_TERMIOS" - ;; - *) - CFLAGS="$CFLAGS -n32" - ;; - esac - LDFLAGS="$LDFLAGS -n32" - ]) - ;; - IRIX64-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) - - # Check to enable 64-bit flags for compiler/linker - - AS_IF([test "$do64bit" = yes], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported by gcc]) - ], [ - do64bit_ok=yes - SHLIB_LD="ld -64 -shared -rdata_shared" - CFLAGS="$CFLAGS -64" - LDFLAGS_ARCH="-64" - ]) - ]) - ;; - Linux*|GNU*|NetBSD-Debian|DragonFly-*|FreeBSD-*) - SHLIB_CFLAGS="-fPIC" - SHLIB_SUFFIX=".so" - - # TEA specific: - CFLAGS_OPTIMIZE="-O2 -fomit-frame-pointer" - - # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} ${CFLAGS} ${LDFLAGS_DEFAULT} -shared' - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - - case $system in - DragonFly-*|FreeBSD-*) - AS_IF([test "${TCL_THREADS}" = "1"], [ - # The -pthread needs to go in the LDFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LDFLAGS="$LDFLAGS $PTHREAD_LIBS"]) - ;; - esac - - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) - AS_IF([test $do64bit = yes], [ - AC_CACHE_CHECK([if compiler accepts -m64 flag], tcl_cv_cc_m64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -m64" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], - [tcl_cv_cc_m64=yes],[tcl_cv_cc_m64=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_m64 = yes], [ - CFLAGS="$CFLAGS -m64" - do64bit_ok=yes - ]) - ]) - - # The combo of gcc + glibc has a bug related to inlining of - # functions like strtod(). The -fno-builtin flag should address - # this problem but it does not work. The -fno-inline flag is kind - # of overkill but it works. Disable inlining only when one of the - # files in compat/*.c is being linked in. - - AS_IF([test x"${USE_COMPAT}" != x],[CFLAGS="$CFLAGS -fno-inline"]) - ;; - Lynx*) - SHLIB_CFLAGS="-fPIC" - SHLIB_SUFFIX=".so" - CFLAGS_OPTIMIZE=-02 - SHLIB_LD='${CC} -shared' - LD_FLAGS="-Wl,--export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - ;; - OpenBSD-*) - arch=`arch -s` - case "$arch" in - alpha|sparc64) - SHLIB_CFLAGS="-fPIC" - ;; - *) - SHLIB_CFLAGS="-fpic" - ;; - esac - SHLIB_LD='${CC} ${SHLIB_CFLAGS} -shared' - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so${SHLIB_VERSION}' - LDFLAGS="$LDFLAGS -Wl,-export-dynamic" - CFLAGS_OPTIMIZE="-O2" - # On OpenBSD: Compile with -pthread - # Don't link with -lpthread - LIBS=`echo $LIBS | sed s/-lpthread//` - CFLAGS="$CFLAGS -pthread" - # OpenBSD doesn't do version numbers with dots. - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - NetBSD-*) - # NetBSD has ELF and can use 'cc -shared' to build shared libs - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} ${SHLIB_CFLAGS} -shared' - SHLIB_SUFFIX=".so" - LDFLAGS="$LDFLAGS -export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - # The -pthread needs to go in the CFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ;; - Darwin-*) - CFLAGS_OPTIMIZE="-Os" - SHLIB_CFLAGS="-fno-common" - # To avoid discrepancies between what headers configure sees during - # preprocessing tests and compiling tests, move any -isysroot and - # -mmacosx-version-min flags from CFLAGS to CPPFLAGS: - CPPFLAGS="${CPPFLAGS} `echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if ([$]i~/^(isysroot|mmacosx-version-min)/) print "-"[$]i}'`" - CFLAGS="`echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if (!([$]i~/^(isysroot|mmacosx-version-min)/)) print "-"[$]i}'`" - AS_IF([test $do64bit = yes], [ - case `arch` in - ppc) - AC_CACHE_CHECK([if compiler accepts -arch ppc64 flag], - tcl_cv_cc_arch_ppc64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], - [tcl_cv_cc_arch_ppc64=yes],[tcl_cv_cc_arch_ppc64=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_ppc64 = yes], [ - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - do64bit_ok=yes - ]);; - i386) - AC_CACHE_CHECK([if compiler accepts -arch x86_64 flag], - tcl_cv_cc_arch_x86_64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch x86_64" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], - [tcl_cv_cc_arch_x86_64=yes],[tcl_cv_cc_arch_x86_64=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_x86_64 = yes], [ - CFLAGS="$CFLAGS -arch x86_64" - do64bit_ok=yes - ]);; - *) - AC_MSG_WARN([Don't know how enable 64-bit on architecture `arch`]);; - esac - ], [ - # Check for combined 32-bit and 64-bit fat build - AS_IF([echo "$CFLAGS " |grep -E -q -- '-arch (ppc64|x86_64) ' \ - && echo "$CFLAGS " |grep -E -q -- '-arch (ppc|i386) '], [ - fat_32_64=yes]) - ]) - # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} -dynamiclib ${CFLAGS} ${LDFLAGS_DEFAULT}' - AC_CACHE_CHECK([if ld accepts -single_module flag], tcl_cv_ld_single_module, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -dynamiclib -Wl,-single_module" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], - [tcl_cv_ld_single_module=yes],[tcl_cv_ld_single_module=no]) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_single_module = yes], [ - SHLIB_LD="${SHLIB_LD} -Wl,-single_module" - ]) - # TEA specific: link shlib with current and compatibility version flags - vers=`echo ${PACKAGE_VERSION} | sed -e 's/^\([[0-9]]\{1,5\}\)\(\(\.[[0-9]]\{1,3\}\)\{0,2\}\).*$/\1\2/p' -e d` - SHLIB_LD="${SHLIB_LD} -current_version ${vers:-0} -compatibility_version ${vers:-0}" - SHLIB_SUFFIX=".dylib" - LDFLAGS="$LDFLAGS -headerpad_max_install_names" - AC_CACHE_CHECK([if ld accepts -search_paths_first flag], - tcl_cv_ld_search_paths_first, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], - [tcl_cv_ld_search_paths_first=yes],[tcl_cv_ld_search_paths_first=no]) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_search_paths_first = yes], [ - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - ]) - AS_IF([test "$tcl_cv_cc_visibility_hidden" != yes], [ - AC_DEFINE(MODULE_SCOPE, [__private_extern__], - [Compiler support for module scope symbols]) - tcl_cv_cc_visibility_hidden=yes - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - LD_LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" - # TEA specific: for combined 32 & 64 bit fat builds of Tk - # extensions, verify that 64-bit build is possible. - AS_IF([test "$fat_32_64" = yes && test -n "${TK_BIN_DIR}"], [ - AS_IF([test "${TEA_WINDOWINGSYSTEM}" = x11], [ - AC_CACHE_CHECK([for 64-bit X11], tcl_cv_lib_x11_64, [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc / /g" -e "s/-arch i386 / /g"`"' - done - CPPFLAGS="$CPPFLAGS -I/usr/X11R6/include" - LDFLAGS="$LDFLAGS -L/usr/X11R6/lib -lX11" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[XrmInitialize();]])], - [tcl_cv_lib_x11_64=yes],[tcl_cv_lib_x11_64=no]) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - ]) - AS_IF([test "${TEA_WINDOWINGSYSTEM}" = aqua], [ - AC_CACHE_CHECK([for 64-bit Tk], tcl_cv_lib_tk_64, [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc / /g" -e "s/-arch i386 / /g"`"' - done - CPPFLAGS="$CPPFLAGS -DUSE_TCL_STUBS=1 -DUSE_TK_STUBS=1 ${TCL_INCLUDES} ${TK_INCLUDES}" - LDFLAGS="$LDFLAGS ${TCL_STUB_LIB_SPEC} ${TK_STUB_LIB_SPEC}" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[Tk_InitStubs(NULL, "", 0);]])], - [tcl_cv_lib_tk_64=yes],[tcl_cv_lib_tk_64=no]) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - ]) - # remove 64-bit arch flags from CFLAGS et al. if configuration - # does not support 64-bit. - AS_IF([test "$tcl_cv_lib_tk_64" = no -o "$tcl_cv_lib_x11_64" = no], [ - AC_MSG_NOTICE([Removing 64-bit architectures from compiler & linker flags]) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="`echo "$'$v' "|sed -e "s/-arch ppc64 / /g" -e "s/-arch x86_64 / /g"`"' - done]) - ]) - ;; - OS/390-*) - CFLAGS_OPTIMIZE="" # Optimizer is buggy - AC_DEFINE(_OE_SOCKETS, 1, # needed in sys/socket.h - [Should OS/390 do the right thing with sockets?]) - ;; - OSF1-V*) - # Digital OSF/1 - SHLIB_CFLAGS="" - AS_IF([test "$SHARED_BUILD" = 1], [ - SHLIB_LD='ld -shared -expect_unresolved "*"' - ], [ - SHLIB_LD='ld -non_shared -expect_unresolved "*"' - ]) - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AS_IF([test "$GCC" = yes], [CFLAGS="$CFLAGS -mieee"], [ - CFLAGS="$CFLAGS -DHAVE_TZSET -std1 -ieee"]) - # see pthread_intro(3) for pthread support on osf1, k.furukawa - CFLAGS="$CFLAGS -DHAVE_PTHREAD_ATTR_SETSTACKSIZE" - CFLAGS="$CFLAGS -DTCL_THREAD_STACK_MIN=PTHREAD_STACK_MIN*64" - LIBS=`echo $LIBS | sed s/-lpthreads//` - AS_IF([test "$GCC" = yes], [ - LIBS="$LIBS -lpthread -lmach -lexc" - ], [ - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - ;; - QNX-6*) - # QNX RTP - # This may work for all QNX, but it was only reported for v6. - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="ld -Bshareable -x" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SCO_SV-3.2*) - AS_IF([test "$GCC" = yes], [ - SHLIB_CFLAGS="-fPIC -melf" - LDFLAGS="$LDFLAGS -melf -Wl,-Bexport" - ], [ - SHLIB_CFLAGS="-Kpic -belf" - LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" - ]) - SHLIB_LD="ld -G" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SunOS-5.[[0-6]]) - # Careful to not let 5.10+ fall into this case - - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - SHLIB_SUFFIX=".so" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ], [ - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - CC_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - ;; - SunOS-5*) - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = yes], [ - arch=`isainfo` - AS_IF([test "$arch" = "sparcv9 sparc"], [ - AS_IF([test "$GCC" = yes], [ - AS_IF([test "`${CC} -dumpversion | awk -F. '{print [$]1}'`" -lt 3], [ - AC_MSG_WARN([64bit mode not supported with GCC < 3.2 on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -m64 -mcpu=v9" - LDFLAGS="$LDFLAGS -m64 -mcpu=v9" - SHLIB_CFLAGS="-fPIC" - ]) - ], [ - do64bit_ok=yes - AS_IF([test "$do64bitVIS" = yes], [ - CFLAGS="$CFLAGS -xarch=v9a" - LDFLAGS_ARCH="-xarch=v9a" - ], [ - CFLAGS="$CFLAGS -xarch=v9" - LDFLAGS_ARCH="-xarch=v9" - ]) - # Solaris 64 uses this as well - #LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH_64" - ]) - ], [AS_IF([test "$arch" = "amd64 i386"], [ - AS_IF([test "$GCC" = yes], [ - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - do64bit_ok=yes - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]);; - esac - ], [ - do64bit_ok=yes - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - CFLAGS="$CFLAGS -xarch=amd64" - LDFLAGS="$LDFLAGS -xarch=amd64";; - esac - ]) - ], [AC_MSG_WARN([64bit mode not supported for $arch])])]) - ]) - - SHLIB_SUFFIX=".so" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "$do64bit_ok" = yes], [ - AS_IF([test "$arch" = "sparcv9 sparc"], [ - # We need to specify -static-libgcc or we need to - # add the path to the sparv9 libgcc. - # JH: static-libgcc is necessary for core Tcl, but may - # not be necessary for extensions. - SHLIB_LD="$SHLIB_LD -m64 -mcpu=v9 -static-libgcc" - # for finding sparcv9 libgcc, get the regular libgcc - # path, remove so name and append 'sparcv9' - #v9gcclibdir="`gcc -print-file-name=libgcc_s.so` | ..." - #CC_SEARCH_FLAGS="${CC_SEARCH_FLAGS},-R,$v9gcclibdir" - ], [AS_IF([test "$arch" = "amd64 i386"], [ - # JH: static-libgcc is necessary for core Tcl, but may - # not be necessary for extensions. - SHLIB_LD="$SHLIB_LD -m64 -static-libgcc" - ])]) - ]) - ], [ - case $system in - SunOS-5.[[1-9]][[0-9]]*) - # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} -G -z text ${LDFLAGS_DEFAULT}';; - *) - SHLIB_LD='/usr/ccs/bin/ld -G -z text';; - esac - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' - ]) - ;; - UNIX_SV* | UnixWare-5*) - SHLIB_CFLAGS="-KPIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - # Some UNIX_SV* systems (unixware 1.1.2 for example) have linkers - # that don't grok the -Bexport option. Test that it does. - AC_CACHE_CHECK([for ld accepts -Bexport flag], tcl_cv_ld_Bexport, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-Bexport" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], - [tcl_cv_ld_Bexport=yes],[tcl_cv_ld_Bexport=no]) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_Bexport = yes], [ - LDFLAGS="$LDFLAGS -Wl,-Bexport" - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - esac - - AS_IF([test "$do64bit" = yes -a "$do64bit_ok" = no], [ - AC_MSG_WARN([64bit support being disabled -- don't know magic for this platform]) - ]) - -dnl # Add any CPPFLAGS set in the environment to our CFLAGS, but delay doing so -dnl # until the end of configure, as configure's compile and link tests use -dnl # both CPPFLAGS and CFLAGS (unlike our compile and link) but configure's -dnl # preprocessing tests use only CPPFLAGS. - AC_CONFIG_COMMANDS_PRE([CFLAGS="${CFLAGS} ${CPPFLAGS}"; CPPFLAGS=""]) - - # Add in the arch flags late to ensure it wasn't removed. - # Not necessary in TEA, but this is aligned with core - LDFLAGS="$LDFLAGS $LDFLAGS_ARCH" - - # If we're running gcc, then change the C flags for compiling shared - # libraries to the right flags for gcc, instead of those for the - # standard manufacturer compiler. - - AS_IF([test "$GCC" = yes], [ - case $system in - AIX-*) ;; - BSD/OS*) ;; - CYGWIN_*|MINGW32_*|MINGW64_*|MSYS_*) ;; - IRIX*) ;; - NetBSD-*|DragonFly-*|FreeBSD-*|OpenBSD-*) ;; - Darwin-*) ;; - SCO_SV-3.2*) ;; - windows) ;; - *) SHLIB_CFLAGS="-fPIC" ;; - esac]) - - AS_IF([test "$tcl_cv_cc_visibility_hidden" != yes], [ - AC_DEFINE(MODULE_SCOPE, [extern], - [No Compiler support for module scope symbols]) - ]) - - AS_IF([test "$SHARED_LIB_SUFFIX" = ""], [ - # TEA specific: use PACKAGE_VERSION instead of VERSION - SHARED_LIB_SUFFIX='${PACKAGE_VERSION}${SHLIB_SUFFIX}']) - AS_IF([test "$UNSHARED_LIB_SUFFIX" = ""], [ - # TEA specific: use PACKAGE_VERSION instead of VERSION - UNSHARED_LIB_SUFFIX='${PACKAGE_VERSION}.a']) - - if test "${GCC}" = "yes" -a ${SHLIB_SUFFIX} = ".dll"; then - AC_CACHE_CHECK(for SEH support in compiler, - tcl_cv_seh, - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN - - int main(int argc, char** argv) { - int a, b = 0; - __try { - a = 666 / b; - } - __except (EXCEPTION_EXECUTE_HANDLER) { - return 0; - } - return 1; - } - ]])], - [tcl_cv_seh=yes], - [tcl_cv_seh=no], - [tcl_cv_seh=no]) - ) - if test "$tcl_cv_seh" = "no" ; then - AC_DEFINE(HAVE_NO_SEH, 1, - [Defined when mingw does not support SEH]) - fi - - # - # Check to see if the excpt.h include file provided contains the - # definition for EXCEPTION_DISPOSITION; if not, which is the case - # with Cygwin's version as of 2002-04-10, define it to be int, - # sufficient for getting the current code to work. - # - AC_CACHE_CHECK(for EXCEPTION_DISPOSITION support in include files, - tcl_cv_eh_disposition, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -# define WIN32_LEAN_AND_MEAN -# include -# undef WIN32_LEAN_AND_MEAN - ]], [[ - EXCEPTION_DISPOSITION x; - ]])], - [tcl_cv_eh_disposition=yes], - [tcl_cv_eh_disposition=no]) - ) - if test "$tcl_cv_eh_disposition" = "no" ; then - AC_DEFINE(EXCEPTION_DISPOSITION, int, - [Defined when cygwin/mingw does not support EXCEPTION DISPOSITION]) - fi - - # Check to see if winnt.h defines CHAR, SHORT, and LONG - # even if VOID has already been #defined. The win32api - # used by mingw and cygwin is known to do this. - - AC_CACHE_CHECK(for winnt.h that ignores VOID define, - tcl_cv_winnt_ignore_void, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -#define VOID void -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN - ]], [[ - CHAR c; - SHORT s; - LONG l; - ]])], - [tcl_cv_winnt_ignore_void=yes], - [tcl_cv_winnt_ignore_void=no]) - ) - if test "$tcl_cv_winnt_ignore_void" = "yes" ; then - AC_DEFINE(HAVE_WINNT_IGNORE_VOID, 1, - [Defined when cygwin/mingw ignores VOID define in winnt.h]) - fi - fi - - # See if the compiler supports casting to a union type. - # This is used to stop gcc from printing a compiler - # warning when initializing a union member. - - AC_CACHE_CHECK(for cast to union support, - tcl_cv_cast_to_union, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ - union foo { int i; double d; }; - union foo f = (union foo) (int) 0; - ]])], - [tcl_cv_cast_to_union=yes], - [tcl_cv_cast_to_union=no]) - ) - if test "$tcl_cv_cast_to_union" = "yes"; then - AC_DEFINE(HAVE_CAST_TO_UNION, 1, - [Defined when compiler supports casting to union type.]) - fi - - AC_CHECK_HEADER(stdbool.h, [AC_DEFINE(HAVE_STDBOOL_H, 1, [Do we have ?])],) - - AC_SUBST(CFLAGS_DEBUG) - AC_SUBST(CFLAGS_OPTIMIZE) - AC_SUBST(CFLAGS_WARNING) - AC_SUBST(LDFLAGS_DEBUG) - AC_SUBST(LDFLAGS_OPTIMIZE) - - AC_SUBST(STLIB_LD) - AC_SUBST(SHLIB_LD) - - AC_SUBST(SHLIB_LD_LIBS) - AC_SUBST(SHLIB_CFLAGS) - - AC_SUBST(LD_LIBRARY_PATH_VAR) - - # These must be called after we do the basic CFLAGS checks and - # verify any possible 64-bit or similar switches are necessary - TEA_TCL_EARLY_FLAGS - TEA_TCL_64BIT_FLAGS -]) - -#-------------------------------------------------------------------- -# TEA_SERIAL_PORT -# -# Determine which interface to use to talk to the serial port. -# Note that #include lines must begin in leftmost column for -# some compilers to recognize them as preprocessor directives, -# and some build environments have stdin not pointing at a -# pseudo-terminal (usually /dev/null instead.) -# -# Arguments: -# none -# -# Results: -# -# Defines only one of the following vars: -# HAVE_SYS_MODEM_H -# USE_TERMIOS -# USE_TERMIO -# USE_SGTTY -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_SERIAL_PORT], [ - AC_CHECK_HEADERS(sys/modem.h) - AC_CACHE_CHECK([termios vs. termio vs. sgtty], tcl_cv_api_serial, [ - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=termios],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - if test $tcl_cv_api_serial = no ; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=termio],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no ; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=sgtty],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no ; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=termios],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; - }]])],[tcl_cv_api_serial=termio],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=sgtty],[tcl_cv_api_serial=none],[tcl_cv_api_serial=none]) - fi]) - case $tcl_cv_api_serial in - termios) AC_DEFINE(USE_TERMIOS, 1, [Use the termios API for serial lines]);; - termio) AC_DEFINE(USE_TERMIO, 1, [Use the termio API for serial lines]);; - sgtty) AC_DEFINE(USE_SGTTY, 1, [Use the sgtty API for serial lines]);; - esac -]) - -#-------------------------------------------------------------------- -# TEA_PATH_X -# -# Locate the X11 header files and the X11 library archive. Try -# the ac_path_x macro first, but if it doesn't find the X stuff -# (e.g. because there's no xmkmf program) then check through -# a list of possible directories. Under some conditions the -# autoconf macro will return an include directory that contains -# no include files, so double-check its result just to be safe. -# -# This should be called after TEA_CONFIG_CFLAGS as setting the -# LIBS line can confuse some configure macro magic. -# -# Arguments: -# none -# -# Results: -# -# Sets the following vars: -# XINCLUDES -# XLIBSW -# PKG_LIBS (appends to) -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_PATH_X], [ - if test "${TEA_WINDOWINGSYSTEM}" = "x11" ; then - TEA_PATH_UNIX_X - fi -]) - -AC_DEFUN([TEA_PATH_UNIX_X], [ - AC_PATH_X - not_really_there="" - if test "$no_x" = ""; then - if test "$x_includes" = ""; then - AC_PREPROC_IFELSE([AC_LANG_SOURCE([[#include ]])],[],[not_really_there="yes"]) - else - if test ! -r $x_includes/X11/Xlib.h; then - not_really_there="yes" - fi - fi - fi - if test "$no_x" = "yes" -o "$not_really_there" = "yes"; then - AC_MSG_CHECKING([for X11 header files]) - found_xincludes="no" - AC_PREPROC_IFELSE([AC_LANG_SOURCE([[#include ]])],[found_xincludes="yes"],[found_xincludes="no"]) - if test "$found_xincludes" = "no"; then - dirs="/usr/unsupported/include /usr/local/include /usr/X386/include /usr/X11R6/include /usr/X11R5/include /usr/include/X11R5 /usr/include/X11R4 /usr/openwin/include /usr/X11/include /usr/sww/include" - for i in $dirs ; do - if test -r $i/X11/Xlib.h; then - AC_MSG_RESULT([$i]) - XINCLUDES=" -I$i" - found_xincludes="yes" - break - fi - done - fi - else - if test "$x_includes" != ""; then - XINCLUDES="-I$x_includes" - found_xincludes="yes" - fi - fi - if test "$found_xincludes" = "no"; then - AC_MSG_RESULT([couldn't find any!]) - fi - - if test "$no_x" = yes; then - AC_MSG_CHECKING([for X11 libraries]) - XLIBSW=nope - dirs="/usr/unsupported/lib /usr/local/lib /usr/X386/lib /usr/X11R6/lib /usr/X11R5/lib /usr/lib/X11R5 /usr/lib/X11R4 /usr/openwin/lib /usr/X11/lib /usr/sww/X11/lib" - for i in $dirs ; do - if test -r $i/libX11.a -o -r $i/libX11.so -o -r $i/libX11.sl -o -r $i/libX11.dylib; then - AC_MSG_RESULT([$i]) - XLIBSW="-L$i -lX11" - x_libraries="$i" - break - fi - done - else - if test "$x_libraries" = ""; then - XLIBSW=-lX11 - else - XLIBSW="-L$x_libraries -lX11" - fi - fi - if test "$XLIBSW" = nope ; then - AC_CHECK_LIB(Xwindow, XCreateWindow, XLIBSW=-lXwindow) - fi - if test "$XLIBSW" = nope ; then - AC_MSG_RESULT([could not find any! Using -lX11.]) - XLIBSW=-lX11 - fi - # TEA specific: - if test x"${XLIBSW}" != x ; then - PKG_LIBS="${PKG_LIBS} ${XLIBSW}" - fi -]) - -#-------------------------------------------------------------------- -# TEA_BLOCKING_STYLE -# -# The statements below check for systems where POSIX-style -# non-blocking I/O (O_NONBLOCK) doesn't work or is unimplemented. -# On these systems (mostly older ones), use the old BSD-style -# FIONBIO approach instead. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# HAVE_SYS_IOCTL_H -# HAVE_SYS_FILIO_H -# USE_FIONBIO -# O_NONBLOCK -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_BLOCKING_STYLE], [ - AC_CHECK_HEADERS(sys/ioctl.h) - AC_CHECK_HEADERS(sys/filio.h) - TEA_CONFIG_SYSTEM - AC_MSG_CHECKING([FIONBIO vs. O_NONBLOCK for nonblocking I/O]) - case $system in - OSF*) - AC_DEFINE(USE_FIONBIO, 1, [Should we use FIONBIO?]) - AC_MSG_RESULT([FIONBIO]) - ;; - *) - AC_MSG_RESULT([O_NONBLOCK]) - ;; - esac -]) - -#-------------------------------------------------------------------- -# TEA_TIME_HANDLER -# -# Checks how the system deals with time.h, what time structures -# are used on the system, and what fields the structures have. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# USE_DELTA_FOR_TZ -# HAVE_TM_GMTOFF -# HAVE_TM_TZADJ -# HAVE_TIMEZONE_VAR -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TIME_HANDLER], [ - AC_CHECK_HEADERS(sys/time.h) - AC_HEADER_TIME - AC_STRUCT_TIMEZONE - - AC_CHECK_FUNCS(gmtime_r localtime_r mktime) - - AC_CACHE_CHECK([tm_tzadj in struct tm], tcl_cv_member_tm_tzadj, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct tm tm; (void)tm.tm_tzadj;]])], - [tcl_cv_member_tm_tzadj=yes], - [tcl_cv_member_tm_tzadj=no])]) - if test $tcl_cv_member_tm_tzadj = yes ; then - AC_DEFINE(HAVE_TM_TZADJ, 1, [Should we use the tm_tzadj field of struct tm?]) - fi - - AC_CACHE_CHECK([tm_gmtoff in struct tm], tcl_cv_member_tm_gmtoff, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct tm tm; (void)tm.tm_gmtoff;]])], - [tcl_cv_member_tm_gmtoff=yes], - [tcl_cv_member_tm_gmtoff=no])]) - if test $tcl_cv_member_tm_gmtoff = yes ; then - AC_DEFINE(HAVE_TM_GMTOFF, 1, [Should we use the tm_gmtoff field of struct tm?]) - fi - - # - # Its important to include time.h in this check, as some systems - # (like convex) have timezone functions, etc. - # - AC_CACHE_CHECK([long timezone variable], tcl_cv_timezone_long, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], - [[extern long timezone; - timezone += 1; - exit (0);]])], - [tcl_cv_timezone_long=yes], [tcl_cv_timezone_long=no])]) - if test $tcl_cv_timezone_long = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - else - # - # On some systems (eg IRIX 6.2), timezone is a time_t and not a long. - # - AC_CACHE_CHECK([time_t timezone variable], tcl_cv_timezone_time, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], - [[extern time_t timezone; - timezone += 1; - exit (0);]])], - [tcl_cv_timezone_time=yes], [tcl_cv_timezone_time=no])]) - if test $tcl_cv_timezone_time = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# TEA_BUGGY_STRTOD -# -# Under Solaris 2.4, strtod returns the wrong value for the -# terminating character under some conditions. Check for this -# and if the problem exists use a substitute procedure -# "fixstrtod" (provided by Tcl) that corrects the error. -# Also, on Compaq's Tru64 Unix 5.0, -# strtod(" ") returns 0.0 instead of a failure to convert. -# -# Arguments: -# none -# -# Results: -# -# Might defines some of the following vars: -# strtod (=fixstrtod) -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_BUGGY_STRTOD], [ - AC_CHECK_FUNC(strtod, tcl_strtod=1, tcl_strtod=0) - if test "$tcl_strtod" = 1; then - AC_CACHE_CHECK([for Solaris2.4/Tru64 strtod bugs], tcl_cv_strtod_buggy,[ - AC_RUN_IFELSE([AC_LANG_SOURCE([[ - #include - extern double strtod(); - int main() { - char *infString="Inf", *nanString="NaN", *spaceString=" "; - char *term; - double value; - value = strtod(infString, &term); - if ((term != infString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(nanString, &term); - if ((term != nanString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(spaceString, &term); - if (term == (spaceString+1)) { - exit(1); - } - exit(0); - }]])], [tcl_cv_strtod_buggy=ok], [tcl_cv_strtod_buggy=buggy], - [tcl_cv_strtod_buggy=buggy])]) - if test "$tcl_cv_strtod_buggy" = buggy; then - AC_LIBOBJ([fixstrtod]) - USE_COMPAT=1 - AC_DEFINE(strtod, fixstrtod, [Do we want to use the strtod() in compat?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# TEA_TCL_LINK_LIBS -# -# Search for the libraries needed to link the Tcl shell. -# Things like the math library (-lm), socket stuff (-lsocket vs. -# -lnsl), zlib (-lz) and libtommath (-ltommath) are dealt with here. -# -# Arguments: -# None. -# -# Results: -# -# Might append to the following vars: -# LIBS -# MATH_LIBS -# -# Might define the following vars: -# HAVE_NET_ERRNO_H -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TCL_LINK_LIBS], [ - #-------------------------------------------------------------------- - # On a few very rare systems, all of the libm.a stuff is - # already in libc.a. Set compiler flags accordingly. - #-------------------------------------------------------------------- - - AC_CHECK_FUNC(sin, MATH_LIBS="", MATH_LIBS="-lm") - - #-------------------------------------------------------------------- - # Interactive UNIX requires -linet instead of -lsocket, plus it - # needs net/errno.h to define the socket-related error codes. - #-------------------------------------------------------------------- - - AC_CHECK_LIB(inet, main, [LIBS="$LIBS -linet"]) - AC_CHECK_HEADER(net/errno.h, [ - AC_DEFINE(HAVE_NET_ERRNO_H, 1, [Do we have ?])]) - - #-------------------------------------------------------------------- - # Check for the existence of the -lsocket and -lnsl libraries. - # The order here is important, so that they end up in the right - # order in the command line generated by make. Here are some - # special considerations: - # 1. Use "connect" and "accept" to check for -lsocket, and - # "gethostbyname" to check for -lnsl. - # 2. Use each function name only once: can't redo a check because - # autoconf caches the results of the last check and won't redo it. - # 3. Use -lnsl and -lsocket only if they supply procedures that - # aren't already present in the normal libraries. This is because - # IRIX 5.2 has libraries, but they aren't needed and they're - # bogus: they goof up name resolution if used. - # 4. On some SVR4 systems, can't use -lsocket without -lnsl too. - # To get around this problem, check for both libraries together - # if -lsocket doesn't work by itself. - #-------------------------------------------------------------------- - - tcl_checkBoth=0 - AC_CHECK_FUNC(connect, tcl_checkSocket=0, tcl_checkSocket=1) - if test "$tcl_checkSocket" = 1; then - AC_CHECK_FUNC(setsockopt, , [AC_CHECK_LIB(socket, setsockopt, - LIBS="$LIBS -lsocket", tcl_checkBoth=1)]) - fi - if test "$tcl_checkBoth" = 1; then - tk_oldLibs=$LIBS - LIBS="$LIBS -lsocket -lnsl" - AC_CHECK_FUNC(accept, tcl_checkNsl=0, [LIBS=$tk_oldLibs]) - fi - AC_CHECK_FUNC(gethostbyname, , [AC_CHECK_LIB(nsl, gethostbyname, - [LIBS="$LIBS -lnsl"])]) - AC_CHECK_FUNC(mp_log_u32, , [AC_CHECK_LIB(tommath, mp_log_u32, - [LIBS="$LIBS -ltommath"])]) - AC_CHECK_FUNC(deflateSetHeader, , [AC_CHECK_LIB(z, deflateSetHeader, - [LIBS="$LIBS -lz"])]) -]) - -#-------------------------------------------------------------------- -# TEA_TCL_EARLY_FLAGS -# -# Check for what flags are needed to be passed so the correct OS -# features are available. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# _ISOC99_SOURCE -# _LARGEFILE64_SOURCE -# _LARGEFILE_SOURCE64 -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TCL_EARLY_FLAG],[ - AC_CACHE_VAL([tcl_cv_flag_]translit($1,[A-Z],[a-z]), - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$2]], [[$3]])], - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no,[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[[#define ]$1[ 1 -]$2]], [[$3]])], - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no)])) - if test ["x${tcl_cv_flag_]translit($1,[A-Z],[a-z])[}" = "xyes"] ; then - AC_DEFINE($1, 1, [Add the ]$1[ flag when building]) - tcl_flags="$tcl_flags $1" - fi -]) - -AC_DEFUN([TEA_TCL_EARLY_FLAGS],[ - AC_MSG_CHECKING([for required early compiler flags]) - tcl_flags="" - TEA_TCL_EARLY_FLAG(_ISOC99_SOURCE,[#include ], - [char *p = (char *)strtoll; char *q = (char *)strtoull;]) - TEA_TCL_EARLY_FLAG(_LARGEFILE64_SOURCE,[#include ], - [struct stat64 buf; int i = stat64("/", &buf);]) - TEA_TCL_EARLY_FLAG(_LARGEFILE_SOURCE64,[#include ], - [char *p = (char *)open64;]) - if test "x${tcl_flags}" = "x" ; then - AC_MSG_RESULT([none]) - else - AC_MSG_RESULT([${tcl_flags}]) - fi -]) - -#-------------------------------------------------------------------- -# TEA_TCL_64BIT_FLAGS -# -# Check for what is defined in the way of 64-bit features. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# TCL_WIDE_INT_IS_LONG -# TCL_WIDE_INT_TYPE -# HAVE_STRUCT_DIRENT64, HAVE_DIR64 -# HAVE_STRUCT_STAT64 -# HAVE_TYPE_OFF64_T -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ - AC_MSG_CHECKING([for 64-bit integer type]) - AC_CACHE_VAL(tcl_cv_type_64bit,[ - tcl_cv_type_64bit=none - # See if the compiler knows natively about __int64 - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[__int64 value = (__int64) 0;]])], - [tcl_type_64bit=__int64],[tcl_type_64bit="long long"]) - # See if we could use long anyway Note that we substitute in the - # type that is our current guess for a 64-bit type inside this check - # program, so it should be modified only carefully... - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[switch (0) { - case 1: case (sizeof(${tcl_type_64bit})==sizeof(long)): ; - }]])],[tcl_cv_type_64bit=${tcl_type_64bit}],[])]) - if test "${tcl_cv_type_64bit}" = none ; then - AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Do 'long' and 'long long' have the same size (64-bit)?]) - AC_MSG_RESULT([yes]) - elif test "${tcl_cv_type_64bit}" = "__int64" \ - -a "${TEA_PLATFORM}" = "windows" ; then - # TEA specific: We actually want to use the default tcl.h checks in - # this case to handle both TCL_WIDE_INT_TYPE and TCL_LL_MODIFIER* - AC_MSG_RESULT([using Tcl header defaults]) - else - AC_DEFINE_UNQUOTED(TCL_WIDE_INT_TYPE,${tcl_cv_type_64bit}, - [What type should be used to define wide integers?]) - AC_MSG_RESULT([${tcl_cv_type_64bit}]) - - # Now check for auxiliary declarations - AC_CACHE_CHECK([for struct dirent64], tcl_cv_struct_dirent64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], [[struct dirent64 p;]])], - [tcl_cv_struct_dirent64=yes],[tcl_cv_struct_dirent64=no])]) - if test "x${tcl_cv_struct_dirent64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_DIRENT64, 1, [Is 'struct dirent64' in ?]) - fi - - AC_CACHE_CHECK([for DIR64], tcl_cv_DIR64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], [[struct dirent64 *p; DIR64 d = opendir64("."); - p = readdir64(d); rewinddir64(d); closedir64(d);]])], - [tcl_cv_DIR64=yes], [tcl_cv_DIR64=no])]) - if test "x${tcl_cv_DIR64}" = "xyes" ; then - AC_DEFINE(HAVE_DIR64, 1, [Is 'DIR64' in ?]) - fi - - AC_CACHE_CHECK([for struct stat64], tcl_cv_struct_stat64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct stat64 p; -]])], - [tcl_cv_struct_stat64=yes], [tcl_cv_struct_stat64=no])]) - if test "x${tcl_cv_struct_stat64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_STAT64, 1, [Is 'struct stat64' in ?]) - fi - - AC_CHECK_FUNCS(open64 lseek64) - AC_MSG_CHECKING([for off64_t]) - AC_CACHE_VAL(tcl_cv_type_off64_t,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[off64_t offset; -]])], - [tcl_cv_type_off64_t=yes], [tcl_cv_type_off64_t=no])]) - dnl Define HAVE_TYPE_OFF64_T only when the off64_t type and the - dnl functions lseek64 and open64 are defined. - if test "x${tcl_cv_type_off64_t}" = "xyes" && \ - test "x${ac_cv_func_lseek64}" = "xyes" && \ - test "x${ac_cv_func_open64}" = "xyes" ; then - AC_DEFINE(HAVE_TYPE_OFF64_T, 1, [Is off64_t in ?]) - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi - fi -]) - -## -## Here ends the standard Tcl configuration bits and starts the -## TEA specific functions -## - -#------------------------------------------------------------------------ -# TEA_INIT -- -# -# Init various Tcl Extension Architecture (TEA) variables. -# This should be the first called TEA_* macro. -# -# Arguments: -# none -# -# Results: -# -# Defines and substs the following vars: -# CYGPATH -# EXEEXT -# Defines only: -# TEA_VERSION -# TEA_INITED -# TEA_PLATFORM (windows or unix) -# -# "cygpath" is used on windows to generate native path names for include -# files. These variables should only be used with the compiler and linker -# since they generate native path names. -# -# EXEEXT -# Select the executable extension based on the host type. This -# is a lightweight replacement for AC_EXEEXT that doesn't require -# a compiler. -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_INIT], [ - TEA_VERSION="3.13" - - AC_MSG_CHECKING([TEA configuration]) - if test x"${PACKAGE_NAME}" = x ; then - AC_MSG_ERROR([ -The PACKAGE_NAME variable must be defined by your TEA configure.ac]) - fi - AC_MSG_RESULT([ok (TEA ${TEA_VERSION})]) - - # If the user did not set CFLAGS, set it now to keep macros - # like AC_PROG_CC and AC_TRY_COMPILE from adding "-g -O2". - if test "${CFLAGS+set}" != "set" ; then - CFLAGS="" - fi - - case "`uname -s`" in - *win32*|*WIN32*|*MINGW32_*|*MINGW64_*|*MSYS_*) - AC_CHECK_PROG(CYGPATH, cygpath, cygpath -m, echo) - EXEEXT=".exe" - TEA_PLATFORM="windows" - ;; - *CYGWIN_*) - EXEEXT=".exe" - # CYGPATH and TEA_PLATFORM are determined later in LOAD_TCLCONFIG - ;; - *) - CYGPATH=echo - # Maybe we are cross-compiling.... - case ${host_alias} in - *mingw32*) - EXEEXT=".exe" - TEA_PLATFORM="windows" - ;; - *) - EXEEXT="" - TEA_PLATFORM="unix" - ;; - esac - ;; - esac - - # Check if exec_prefix is set. If not use fall back to prefix. - # Note when adjusted, so that TEA_PREFIX can correct for this. - # This is needed for recursive configures, since autoconf propagates - # $prefix, but not $exec_prefix (doh!). - if test x$exec_prefix = xNONE ; then - exec_prefix_default=yes - exec_prefix=$prefix - fi - - AC_MSG_NOTICE([configuring ${PACKAGE_NAME} ${PACKAGE_VERSION}]) - - AC_SUBST(EXEEXT) - AC_SUBST(CYGPATH) - - # This package name must be replaced statically for AC_SUBST to work - AC_SUBST(PKG_LIB_FILE) - AC_SUBST(PKG_LIB_FILE8) - AC_SUBST(PKG_LIB_FILE9) - # Substitute STUB_LIB_FILE in case package creates a stub library too. - AC_SUBST(PKG_STUB_LIB_FILE) - - # We AC_SUBST these here to ensure they are subst'ed, - # in case the user doesn't call TEA_ADD_... - AC_SUBST(PKG_STUB_SOURCES) - AC_SUBST(PKG_STUB_OBJECTS) - AC_SUBST(PKG_TCL_SOURCES) - AC_SUBST(PKG_HEADERS) - AC_SUBST(PKG_INCLUDES) - AC_SUBST(PKG_LIBS) - AC_SUBST(PKG_CFLAGS) - - # Configure the installer. - TEA_INSTALLER -]) - -#------------------------------------------------------------------------ -# TEA_ADD_SOURCES -- -# -# Specify one or more source files. Users should check for -# the right platform before adding to their list. -# It is not important to specify the directory, as long as it is -# in the generic, win or unix subdirectory of $(srcdir). -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_SOURCES -# PKG_OBJECTS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_SOURCES], [ - vars="$@" - for i in $vars; do - case $i in - [\$]*) - # allow $-var names - PKG_SOURCES="$PKG_SOURCES $i" - PKG_OBJECTS="$PKG_OBJECTS $i" - ;; - *) - # check for existence - allows for generic/win/unix VPATH - # To add more dirs here (like 'src'), you have to update VPATH - # in Makefile.in as well - if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \ - -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \ - -a ! -f "${srcdir}/macosx/$i" \ - ; then - AC_MSG_ERROR([could not find source file '$i']) - fi - PKG_SOURCES="$PKG_SOURCES $i" - # this assumes it is in a VPATH dir - i=`basename $i` - # handle user calling this before or after TEA_SETUP_COMPILER - if test x"${OBJEXT}" != x ; then - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.${OBJEXT}" - else - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.\${OBJEXT}" - fi - PKG_OBJECTS="$PKG_OBJECTS $j" - ;; - esac - done - AC_SUBST(PKG_SOURCES) - AC_SUBST(PKG_OBJECTS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_STUB_SOURCES -- -# -# Specify one or more source files. Users should check for -# the right platform before adding to their list. -# It is not important to specify the directory, as long as it is -# in the generic, win or unix subdirectory of $(srcdir). -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_STUB_SOURCES -# PKG_STUB_OBJECTS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_STUB_SOURCES], [ - vars="$@" - for i in $vars; do - # check for existence - allows for generic/win/unix VPATH - if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \ - -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \ - -a ! -f "${srcdir}/macosx/$i" \ - ; then - AC_MSG_ERROR([could not find stub source file '$i']) - fi - PKG_STUB_SOURCES="$PKG_STUB_SOURCES $i" - # this assumes it is in a VPATH dir - i=`basename $i` - # handle user calling this before or after TEA_SETUP_COMPILER - if test x"${OBJEXT}" != x ; then - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.${OBJEXT}" - else - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.\${OBJEXT}" - fi - PKG_STUB_OBJECTS="$PKG_STUB_OBJECTS $j" - done - AC_SUBST(PKG_STUB_SOURCES) - AC_SUBST(PKG_STUB_OBJECTS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_TCL_SOURCES -- -# -# Specify one or more Tcl source files. These should be platform -# independent runtime files. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_TCL_SOURCES -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_TCL_SOURCES], [ - vars="$@" - for i in $vars; do - # check for existence, be strict because it is installed - if test ! -f "${srcdir}/$i" ; then - AC_MSG_ERROR([could not find tcl source file '${srcdir}/$i']) - fi - PKG_TCL_SOURCES="$PKG_TCL_SOURCES $i" - done - AC_SUBST(PKG_TCL_SOURCES) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_HEADERS -- -# -# Specify one or more source headers. Users should check for -# the right platform before adding to their list. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_HEADERS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_HEADERS], [ - vars="$@" - for i in $vars; do - # check for existence, be strict because it is installed - if test ! -f "${srcdir}/$i" ; then - AC_MSG_ERROR([could not find header file '${srcdir}/$i']) - fi - PKG_HEADERS="$PKG_HEADERS $i" - done - AC_SUBST(PKG_HEADERS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_INCLUDES -- -# -# Specify one or more include dirs. Users should check for -# the right platform before adding to their list. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_INCLUDES -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_INCLUDES], [ - vars="$@" - for i in $vars; do - PKG_INCLUDES="$PKG_INCLUDES $i" - done - AC_SUBST(PKG_INCLUDES) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_LIBS -- -# -# Specify one or more libraries. Users should check for -# the right platform before adding to their list. For Windows, -# libraries provided in "foo.lib" format will be converted to -# "-lfoo" when using GCC (mingw). -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_LIBS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_LIBS], [ - vars="$@" - for i in $vars; do - if test "${TEA_PLATFORM}" = "windows" -a "$GCC" = "yes" ; then - # Convert foo.lib to -lfoo for GCC. No-op if not *.lib - i=`echo "$i" | sed -e 's/^\([[^-]].*\)\.[[lL]][[iI]][[bB]][$]/-l\1/'` - fi - PKG_LIBS="$PKG_LIBS $i" - done - AC_SUBST(PKG_LIBS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_CFLAGS -- -# -# Specify one or more CFLAGS. Users should check for -# the right platform before adding to their list. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_CFLAGS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_CFLAGS], [ - PKG_CFLAGS="$PKG_CFLAGS $@" - AC_SUBST(PKG_CFLAGS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_CLEANFILES -- -# -# Specify one or more CLEANFILES. -# -# Arguments: -# one or more file names to clean target -# -# Results: -# -# Appends to CLEANFILES, already defined for subst in LOAD_TCLCONFIG -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_CLEANFILES], [ - CLEANFILES="$CLEANFILES $@" -]) - -#------------------------------------------------------------------------ -# TEA_PREFIX -- -# -# Handle the --prefix=... option by defaulting to what Tcl gave -# -# Arguments: -# none -# -# Results: -# -# If --prefix or --exec-prefix was not specified, $prefix and -# $exec_prefix will be set to the values given to Tcl when it was -# configured. -#------------------------------------------------------------------------ -AC_DEFUN([TEA_PREFIX], [ - if test "${prefix}" = "NONE"; then - prefix_default=yes - if test x"${TCL_PREFIX}" != x; then - AC_MSG_NOTICE([--prefix defaulting to TCL_PREFIX ${TCL_PREFIX}]) - prefix=${TCL_PREFIX} - else - AC_MSG_NOTICE([--prefix defaulting to /usr/local]) - prefix=/usr/local - fi - fi - if test "${exec_prefix}" = "NONE" -a x"${prefix_default}" = x"yes" \ - -o x"${exec_prefix_default}" = x"yes" ; then - if test x"${TCL_EXEC_PREFIX}" != x; then - AC_MSG_NOTICE([--exec-prefix defaulting to TCL_EXEC_PREFIX ${TCL_EXEC_PREFIX}]) - exec_prefix=${TCL_EXEC_PREFIX} - else - AC_MSG_NOTICE([--exec-prefix defaulting to ${prefix}]) - exec_prefix=$prefix - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_SETUP_COMPILER_CC -- -# -# Do compiler checks the way we want. This is just a replacement -# for AC_PROG_CC in TEA configure.ac files to make them cleaner. -# -# Arguments: -# none -# -# Results: -# -# Sets up CC var and other standard bits we need to make executables. -#------------------------------------------------------------------------ -AC_DEFUN([TEA_SETUP_COMPILER_CC], [ - # Don't put any macros that use the compiler (e.g. AC_TRY_COMPILE) - # in this macro, they need to go into TEA_SETUP_COMPILER instead. - - AC_PROG_CC - AC_PROG_CPP - - #-------------------------------------------------------------------- - # Checks to see if the make program sets the $MAKE variable. - #-------------------------------------------------------------------- - - AC_PROG_MAKE_SET - - #-------------------------------------------------------------------- - # Find ranlib - #-------------------------------------------------------------------- - - AC_CHECK_TOOL(RANLIB, ranlib) - - #-------------------------------------------------------------------- - # Determines the correct binary file extension (.o, .obj, .exe etc.) - #-------------------------------------------------------------------- - - AC_OBJEXT - AC_EXEEXT -]) - -#------------------------------------------------------------------------ -# TEA_SETUP_COMPILER -- -# -# Do compiler checks that use the compiler. This must go after -# TEA_SETUP_COMPILER_CC, which does the actual compiler check. -# -# Arguments: -# none -# -# Results: -# -# Sets up CC var and other standard bits we need to make executables. -#------------------------------------------------------------------------ -AC_DEFUN([TEA_SETUP_COMPILER], [ - # Any macros that use the compiler (e.g. AC_TRY_COMPILE) have to go here. - AC_REQUIRE([TEA_SETUP_COMPILER_CC]) - - #------------------------------------------------------------------------ - # If we're using GCC, see if the compiler understands -pipe. If so, use it. - # It makes compiling go faster. (This is only a performance feature.) - #------------------------------------------------------------------------ - - if test -z "$no_pipe" -a -n "$GCC"; then - AC_CACHE_CHECK([if the compiler understands -pipe], - tcl_cv_cc_pipe, [ - hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -pipe" - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],[tcl_cv_cc_pipe=yes],[tcl_cv_cc_pipe=no]) - CFLAGS=$hold_cflags]) - if test $tcl_cv_cc_pipe = yes; then - CFLAGS="$CFLAGS -pipe" - fi - fi - - #-------------------------------------------------------------------- - # Common compiler flag setup - #-------------------------------------------------------------------- - - AC_C_BIGENDIAN -]) - -#------------------------------------------------------------------------ -# TEA_MAKE_LIB -- -# -# Generate a line that can be used to build a shared/unshared library -# in a platform independent manner. -# -# Arguments: -# none -# -# Requires: -# -# Results: -# -# Defines the following vars: -# CFLAGS - Done late here to note disturb other AC macros -# MAKE_LIB - Command to execute to build the Tcl library; -# differs depending on whether or not Tcl is being -# compiled as a shared library. -# MAKE_SHARED_LIB Makefile rule for building a shared library -# MAKE_STATIC_LIB Makefile rule for building a static library -# MAKE_STUB_LIB Makefile rule for building a stub library -# VC_MANIFEST_EMBED_DLL Makefile rule for embedded VC manifest in DLL -# VC_MANIFEST_EMBED_EXE Makefile rule for embedded VC manifest in EXE -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_MAKE_LIB], [ - if test "${TEA_PLATFORM}" = "windows" -a "$GCC" != "yes"; then - MAKE_STATIC_LIB="\${STLIB_LD} -out:\[$]@ \$(PKG_OBJECTS)" - MAKE_SHARED_LIB="\${SHLIB_LD} \${LDFLAGS} \${LDFLAGS_DEFAULT} -out:\[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" - AC_EGREP_CPP([manifest needed], [ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -print("manifest needed") -#endif - ], [ - # Could do a CHECK_PROG for mt, but should always be with MSVC8+ - VC_MANIFEST_EMBED_DLL="if test -f \[$]@.manifest ; then mt.exe -nologo -manifest \[$]@.manifest -outputresource:\[$]@\;2 ; fi" - VC_MANIFEST_EMBED_EXE="if test -f \[$]@.manifest ; then mt.exe -nologo -manifest \[$]@.manifest -outputresource:\[$]@\;1 ; fi" - MAKE_SHARED_LIB="${MAKE_SHARED_LIB} ; ${VC_MANIFEST_EMBED_DLL}" - TEA_ADD_CLEANFILES([*.manifest]) - ]) - MAKE_STUB_LIB="\${STLIB_LD} -nodefaultlib -out:\[$]@ \$(PKG_STUB_OBJECTS)" - else - MAKE_STATIC_LIB="\${STLIB_LD} \[$]@ \$(PKG_OBJECTS)" - MAKE_SHARED_LIB="\${SHLIB_LD} \${LDFLAGS} \${LDFLAGS_DEFAULT} -o \[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" - MAKE_STUB_LIB="\${STLIB_LD} \[$]@ \$(PKG_STUB_OBJECTS)" - fi - - if test "${SHARED_BUILD}" = "1" ; then - MAKE_LIB="${MAKE_SHARED_LIB} " - else - MAKE_LIB="${MAKE_STATIC_LIB} " - fi - - #-------------------------------------------------------------------- - # Shared libraries and static libraries have different names. - # Use the double eval to make sure any variables in the suffix is - # substituted. (@@@ Might not be necessary anymore) - #-------------------------------------------------------------------- - - PACKAGE_LIB_PREFIX8="${PACKAGE_LIB_PREFIX}" - PACKAGE_LIB_PREFIX9="${PACKAGE_LIB_PREFIX}tcl9" - if test "${TCL_MAJOR_VERSION}" -gt 8 ; then - PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX9}" - else - PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX8}" - fi - if test "${TEA_PLATFORM}" = "windows" ; then - if test "${SHARED_BUILD}" = "1" ; then - # We force the unresolved linking of symbols that are really in - # the private libraries of Tcl and Tk. - if test x"${TK_BIN_DIR}" != x ; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} \"`${CYGPATH} ${TK_BIN_DIR}/${TK_STUB_LIB_FILE}`\"" - fi - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} \"`${CYGPATH} ${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}`\"" - if test "$GCC" = "yes"; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} -static-libgcc" - fi - eval eval "PKG_LIB_FILE8=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - else - if test "$GCC" = "yes"; then - PACKAGE_LIB_PREFIX=lib${PACKAGE_LIB_PREFIX} - fi - eval eval "PKG_LIB_FILE8=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - fi - # Some packages build their own stubs libraries - eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" - if test "$GCC" = "yes"; then - PKG_STUB_LIB_FILE=lib${PKG_STUB_LIB_FILE} - fi - # These aren't needed on Windows (either MSVC or gcc) - RANLIB=: - RANLIB_STUB=: - else - RANLIB_STUB="${RANLIB}" - if test "${SHARED_BUILD}" = "1" ; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} ${TCL_STUB_LIB_SPEC}" - if test x"${TK_BIN_DIR}" != x ; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} ${TK_STUB_LIB_SPEC}" - fi - eval eval "PKG_LIB_FILE8=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - RANLIB=: - else - eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - fi - # Some packages build their own stubs libraries - eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" - fi - - # These are escaped so that only CFLAGS is picked up at configure time. - # The other values will be substituted at make time. - CFLAGS="${CFLAGS} \${CFLAGS_DEFAULT} \${CFLAGS_WARNING}" - if test "${SHARED_BUILD}" = "1" ; then - CFLAGS="${CFLAGS} \${SHLIB_CFLAGS}" - fi - - AC_SUBST(MAKE_LIB) - AC_SUBST(MAKE_SHARED_LIB) - AC_SUBST(MAKE_STATIC_LIB) - AC_SUBST(MAKE_STUB_LIB) - AC_SUBST(RANLIB_STUB) - AC_SUBST(VC_MANIFEST_EMBED_DLL) - AC_SUBST(VC_MANIFEST_EMBED_EXE) -]) - -#------------------------------------------------------------------------ -# TEA_LIB_SPEC -- -# -# Compute the name of an existing object library located in libdir -# from the given base name and produce the appropriate linker flags. -# -# Arguments: -# basename The base name of the library without version -# numbers, extensions, or "lib" prefixes. -# extra_dir Extra directory in which to search for the -# library. This location is used first, then -# $prefix/$exec-prefix, then some defaults. -# -# Requires: -# TEA_INIT and TEA_PREFIX must be called first. -# -# Results: -# -# Defines the following vars: -# ${basename}_LIB_NAME The computed library name. -# ${basename}_LIB_SPEC The computed linker flags. -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LIB_SPEC], [ - AC_MSG_CHECKING([for $1 library]) - - # Look in exec-prefix for the library (defined by TEA_PREFIX). - - tea_lib_name_dir="${exec_prefix}/lib" - - # Or in a user-specified location. - - if test x"$2" != x ; then - tea_extra_lib_dir=$2 - else - tea_extra_lib_dir=NONE - fi - - for i in \ - `ls -dr ${tea_extra_lib_dir}/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr ${tea_extra_lib_dir}/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr ${tea_lib_name_dir}/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr ${tea_lib_name_dir}/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr /usr/lib/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr /usr/lib/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr /usr/lib64/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr /usr/lib64/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr /usr/local/lib/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr /usr/local/lib/lib$1[[0-9]]* 2>/dev/null ` ; do - if test -f "$i" ; then - tea_lib_name_dir=`dirname $i` - $1_LIB_NAME=`basename $i` - $1_LIB_PATH_NAME=$i - break - fi - done - - if test "${TEA_PLATFORM}" = "windows"; then - $1_LIB_SPEC=\"`${CYGPATH} ${$1_LIB_PATH_NAME} 2>/dev/null`\" - else - # Strip off the leading "lib" and trailing ".a" or ".so" - - tea_lib_name_lib=`echo ${$1_LIB_NAME}|sed -e 's/^lib//' -e 's/\.[[^.]]*$//' -e 's/\.so.*//'` - $1_LIB_SPEC="-L${tea_lib_name_dir} -l${tea_lib_name_lib}" - fi - - if test "x${$1_LIB_NAME}" = x ; then - AC_MSG_ERROR([not found]) - else - AC_MSG_RESULT([${$1_LIB_SPEC}]) - fi -]) - -#------------------------------------------------------------------------ -# TEA_PRIVATE_TCL_HEADERS -- -# -# Locate the private Tcl include files -# -# Arguments: -# -# Requires: -# TCL_SRC_DIR Assumes that TEA_LOAD_TCLCONFIG has -# already been called. -# -# Results: -# -# Substitutes the following vars: -# TCL_TOP_DIR_NATIVE -# TCL_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PRIVATE_TCL_HEADERS], [ - # Allow for --with-tclinclude to take effect and define ${ac_cv_c_tclh} - AC_REQUIRE([TEA_PUBLIC_TCL_HEADERS]) - AC_MSG_CHECKING([for Tcl private include files]) - - TCL_SRC_DIR_NATIVE=`${CYGPATH} ${TCL_SRC_DIR}` - TCL_TOP_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}\" - - # Check to see if tclPort.h isn't already with the public headers - # Don't look for tclInt.h because that resides with tcl.h in the core - # sources, but the Port headers are in a different directory - if test "${TEA_PLATFORM}" = "windows" -a \ - -f "${ac_cv_c_tclh}/tclWinPort.h"; then - result="private headers found with public headers" - elif test "${TEA_PLATFORM}" = "unix" -a \ - -f "${ac_cv_c_tclh}/tclUnixPort.h"; then - result="private headers found with public headers" - else - TCL_GENERIC_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}/generic\" - if test "${TEA_PLATFORM}" = "windows"; then - TCL_PLATFORM_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}/win\" - else - TCL_PLATFORM_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}/unix\" - fi - # Overwrite the previous TCL_INCLUDES as this should capture both - # public and private headers in the same set. - # We want to ensure these are substituted so as not to require - # any *_NATIVE vars be defined in the Makefile - TCL_INCLUDES="-I${TCL_GENERIC_DIR_NATIVE} -I${TCL_PLATFORM_DIR_NATIVE}" - if test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use - # the framework's Headers and PrivateHeaders directories - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - if test -d "${TCL_BIN_DIR}/Headers" -a \ - -d "${TCL_BIN_DIR}/PrivateHeaders"; then - TCL_INCLUDES="-I\"${TCL_BIN_DIR}/Headers\" -I\"${TCL_BIN_DIR}/PrivateHeaders\" ${TCL_INCLUDES}" - else - TCL_INCLUDES="${TCL_INCLUDES} ${TCL_INCLUDE_SPEC} `echo "${TCL_INCLUDE_SPEC}" | sed -e 's/Headers/PrivateHeaders/'`" - fi - ;; - esac - result="Using ${TCL_INCLUDES}" - else - if test ! -f "${TCL_SRC_DIR}/generic/tclInt.h" ; then - AC_MSG_ERROR([Cannot find private header tclInt.h in ${TCL_SRC_DIR}]) - fi - result="Using srcdir found in tclConfig.sh: ${TCL_SRC_DIR}" - fi - fi - - AC_SUBST(TCL_TOP_DIR_NATIVE) - - AC_SUBST(TCL_INCLUDES) - AC_MSG_RESULT([${result}]) -]) - -#------------------------------------------------------------------------ -# TEA_PUBLIC_TCL_HEADERS -- -# -# Locate the installed public Tcl header files -# -# Arguments: -# None. -# -# Requires: -# CYGPATH must be set -# -# Results: -# -# Adds a --with-tclinclude switch to configure. -# Result is cached. -# -# Substitutes the following vars: -# TCL_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PUBLIC_TCL_HEADERS], [ - AC_MSG_CHECKING([for Tcl public headers]) - - AC_ARG_WITH(tclinclude, [ --with-tclinclude directory containing the public Tcl header files], with_tclinclude=${withval}) - - AC_CACHE_VAL(ac_cv_c_tclh, [ - # Use the value from --with-tclinclude, if it was given - - if test x"${with_tclinclude}" != x ; then - if test -f "${with_tclinclude}/tcl.h" ; then - ac_cv_c_tclh=${with_tclinclude} - else - AC_MSG_ERROR([${with_tclinclude} directory does not contain tcl.h]) - fi - else - list="" - if test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use - # the framework's Headers directory - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - list="`ls -d ${TCL_BIN_DIR}/Headers 2>/dev/null`" - ;; - esac - fi - - # Look in the source dir only if Tcl is not installed, - # and in that situation, look there before installed locations. - if test -f "${TCL_BIN_DIR}/Makefile" ; then - list="$list `ls -d ${TCL_SRC_DIR}/generic 2>/dev/null`" - fi - - # Check order: pkg --prefix location, Tcl's --prefix location, - # relative to directory of tclConfig.sh. - - eval "temp_includedir=${includedir}" - list="$list \ - `ls -d ${temp_includedir} 2>/dev/null` \ - `ls -d ${TCL_PREFIX}/include 2>/dev/null` \ - `ls -d ${TCL_BIN_DIR}/../include 2>/dev/null`" - if test "${TEA_PLATFORM}" != "windows" -o "$GCC" = "yes"; then - list="$list /usr/local/include /usr/include" - if test x"${TCL_INCLUDE_SPEC}" != x ; then - d=`echo "${TCL_INCLUDE_SPEC}" | sed -e 's/^-I//'` - list="$list `ls -d ${d} 2>/dev/null`" - fi - fi - for i in $list ; do - if test -f "$i/tcl.h" ; then - ac_cv_c_tclh=$i - break - fi - done - fi - ]) - - # Print a message based on how we determined the include path - - if test x"${ac_cv_c_tclh}" = x ; then - AC_MSG_ERROR([tcl.h not found. Please specify its location with --with-tclinclude]) - else - AC_MSG_RESULT([${ac_cv_c_tclh}]) - fi - - # Convert to a native path and substitute into the output files. - - INCLUDE_DIR_NATIVE=`${CYGPATH} ${ac_cv_c_tclh}` - - TCL_INCLUDES=-I\"${INCLUDE_DIR_NATIVE}\" - - AC_SUBST(TCL_INCLUDES) -]) - -#------------------------------------------------------------------------ -# TEA_PRIVATE_TK_HEADERS -- -# -# Locate the private Tk include files -# -# Arguments: -# -# Requires: -# TK_SRC_DIR Assumes that TEA_LOAD_TKCONFIG has -# already been called. -# -# Results: -# -# Substitutes the following vars: -# TK_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PRIVATE_TK_HEADERS], [ - # Allow for --with-tkinclude to take effect and define ${ac_cv_c_tkh} - AC_REQUIRE([TEA_PUBLIC_TK_HEADERS]) - AC_MSG_CHECKING([for Tk private include files]) - - TK_SRC_DIR_NATIVE=`${CYGPATH} ${TK_SRC_DIR}` - TK_TOP_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}\" - - # Check to see if tkPort.h isn't already with the public headers - # Don't look for tkInt.h because that resides with tk.h in the core - # sources, but the Port headers are in a different directory - if test "${TEA_PLATFORM}" = "windows" -a \ - -f "${ac_cv_c_tkh}/tkWinPort.h"; then - result="private headers found with public headers" - elif test "${TEA_PLATFORM}" = "unix" -a \ - -f "${ac_cv_c_tkh}/tkUnixPort.h"; then - result="private headers found with public headers" - else - TK_GENERIC_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/generic\" - TK_XLIB_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/xlib\" - if test "${TEA_PLATFORM}" = "windows"; then - TK_PLATFORM_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/win\" - else - TK_PLATFORM_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/unix\" - fi - # Overwrite the previous TK_INCLUDES as this should capture both - # public and private headers in the same set. - # We want to ensure these are substituted so as not to require - # any *_NATIVE vars be defined in the Makefile - TK_INCLUDES="-I${TK_GENERIC_DIR_NATIVE} -I${TK_PLATFORM_DIR_NATIVE}" - # Detect and add ttk subdir - if test -d "${TK_SRC_DIR}/generic/ttk"; then - TK_INCLUDES="${TK_INCLUDES} -I\"${TK_SRC_DIR_NATIVE}/generic/ttk\"" - fi - if test "${TEA_WINDOWINGSYSTEM}" != "x11"; then - TK_INCLUDES="${TK_INCLUDES} -I\"${TK_XLIB_DIR_NATIVE}\"" - fi - if test "${TEA_WINDOWINGSYSTEM}" = "aqua"; then - TK_INCLUDES="${TK_INCLUDES} -I\"${TK_SRC_DIR_NATIVE}/macosx\"" - fi - if test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use - # the framework's Headers and PrivateHeaders directories - case ${TK_DEFS} in - *TK_FRAMEWORK*) - if test -d "${TK_BIN_DIR}/Headers" -a \ - -d "${TK_BIN_DIR}/PrivateHeaders"; then - TK_INCLUDES="-I\"${TK_BIN_DIR}/Headers\" -I\"${TK_BIN_DIR}/PrivateHeaders\" ${TK_INCLUDES}" - else - TK_INCLUDES="${TK_INCLUDES} ${TK_INCLUDE_SPEC} `echo "${TK_INCLUDE_SPEC}" | sed -e 's/Headers/PrivateHeaders/'`" - fi - ;; - esac - result="Using ${TK_INCLUDES}" - else - if test ! -f "${TK_SRC_DIR}/generic/tkInt.h" ; then - AC_MSG_ERROR([Cannot find private header tkInt.h in ${TK_SRC_DIR}]) - fi - result="Using srcdir found in tkConfig.sh: ${TK_SRC_DIR}" - fi - fi - - AC_SUBST(TK_TOP_DIR_NATIVE) - AC_SUBST(TK_XLIB_DIR_NATIVE) - - AC_SUBST(TK_INCLUDES) - AC_MSG_RESULT([${result}]) -]) - -#------------------------------------------------------------------------ -# TEA_PUBLIC_TK_HEADERS -- -# -# Locate the installed public Tk header files -# -# Arguments: -# None. -# -# Requires: -# CYGPATH must be set -# -# Results: -# -# Adds a --with-tkinclude switch to configure. -# Result is cached. -# -# Substitutes the following vars: -# TK_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PUBLIC_TK_HEADERS], [ - AC_MSG_CHECKING([for Tk public headers]) - - AC_ARG_WITH(tkinclude, [ --with-tkinclude directory containing the public Tk header files], with_tkinclude=${withval}) - - AC_CACHE_VAL(ac_cv_c_tkh, [ - # Use the value from --with-tkinclude, if it was given - - if test x"${with_tkinclude}" != x ; then - if test -f "${with_tkinclude}/tk.h" ; then - ac_cv_c_tkh=${with_tkinclude} - else - AC_MSG_ERROR([${with_tkinclude} directory does not contain tk.h]) - fi - else - list="" - if test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use - # the framework's Headers directory. - case ${TK_DEFS} in - *TK_FRAMEWORK*) - list="`ls -d ${TK_BIN_DIR}/Headers 2>/dev/null`" - ;; - esac - fi - - # Look in the source dir only if Tk is not installed, - # and in that situation, look there before installed locations. - if test -f "${TK_BIN_DIR}/Makefile" ; then - list="$list `ls -d ${TK_SRC_DIR}/generic 2>/dev/null`" - fi - - # Check order: pkg --prefix location, Tk's --prefix location, - # relative to directory of tkConfig.sh, Tcl's --prefix location, - # relative to directory of tclConfig.sh. - - eval "temp_includedir=${includedir}" - list="$list \ - `ls -d ${temp_includedir} 2>/dev/null` \ - `ls -d ${TK_PREFIX}/include 2>/dev/null` \ - `ls -d ${TK_BIN_DIR}/../include 2>/dev/null` \ - `ls -d ${TCL_PREFIX}/include 2>/dev/null` \ - `ls -d ${TCL_BIN_DIR}/../include 2>/dev/null`" - if test "${TEA_PLATFORM}" != "windows" -o "$GCC" = "yes"; then - list="$list /usr/local/include /usr/include" - if test x"${TK_INCLUDE_SPEC}" != x ; then - d=`echo "${TK_INCLUDE_SPEC}" | sed -e 's/^-I//'` - list="$list `ls -d ${d} 2>/dev/null`" - fi - fi - for i in $list ; do - if test -f "$i/tk.h" ; then - ac_cv_c_tkh=$i - break - fi - done - fi - ]) - - # Print a message based on how we determined the include path - - if test x"${ac_cv_c_tkh}" = x ; then - AC_MSG_ERROR([tk.h not found. Please specify its location with --with-tkinclude]) - else - AC_MSG_RESULT([${ac_cv_c_tkh}]) - fi - - # Convert to a native path and substitute into the output files. - - INCLUDE_DIR_NATIVE=`${CYGPATH} ${ac_cv_c_tkh}` - - TK_INCLUDES=-I\"${INCLUDE_DIR_NATIVE}\" - - AC_SUBST(TK_INCLUDES) - - if test "${TEA_WINDOWINGSYSTEM}" != "x11"; then - # On Windows and Aqua, we need the X compat headers - AC_MSG_CHECKING([for X11 header files]) - if test ! -r "${INCLUDE_DIR_NATIVE}/X11/Xlib.h"; then - INCLUDE_DIR_NATIVE="`${CYGPATH} ${TK_SRC_DIR}/xlib`" - TK_XINCLUDES=-I\"${INCLUDE_DIR_NATIVE}\" - AC_SUBST(TK_XINCLUDES) - fi - AC_MSG_RESULT([${INCLUDE_DIR_NATIVE}]) - fi -]) - -#------------------------------------------------------------------------ -# TEA_PATH_CONFIG -- -# -# Locate the ${1}Config.sh file and perform a sanity check on -# the ${1} compile flags. These are used by packages like -# [incr Tk] that load *Config.sh files from more than Tcl and Tk. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-$1=... -# -# Defines the following vars: -# $1_BIN_DIR Full path to the directory containing -# the $1Config.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PATH_CONFIG], [ - # - # Ok, lets find the $1 configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-$1 - # - - if test x"${no_$1}" = x ; then - # we reset no_$1 in case something fails here - no_$1=true - AC_ARG_WITH($1, [ --with-$1 directory containing $1 configuration ($1Config.sh)], with_$1config=${withval}) - AC_MSG_CHECKING([for $1 configuration]) - AC_CACHE_VAL(ac_cv_c_$1config,[ - - # First check to see if --with-$1 was specified. - if test x"${with_$1config}" != x ; then - case ${with_$1config} in - */$1Config.sh ) - if test -f ${with_$1config}; then - AC_MSG_WARN([--with-$1 argument should refer to directory containing $1Config.sh, not to $1Config.sh itself]) - with_$1config=`echo ${with_$1config} | sed 's!/$1Config\.sh$!!'` - fi;; - esac - if test -f "${with_$1config}/$1Config.sh" ; then - ac_cv_c_$1config=`(cd ${with_$1config}; pwd)` - else - AC_MSG_ERROR([${with_$1config} directory doesn't contain $1Config.sh]) - fi - fi - - # then check for a private $1 installation - if test x"${ac_cv_c_$1config}" = x ; then - for i in \ - ../$1 \ - `ls -dr ../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ../../$1 \ - `ls -dr ../../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ../../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ../../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ../../../$1 \ - `ls -dr ../../../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ../../../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ${srcdir}/../$1 \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ; do - if test -f "$i/$1Config.sh" ; then - ac_cv_c_$1config=`(cd $i; pwd)` - break - fi - if test -f "$i/unix/$1Config.sh" ; then - ac_cv_c_$1config=`(cd $i/unix; pwd)` - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_$1config}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/pkg/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - ; do - if test -f "$i/$1Config.sh" ; then - ac_cv_c_$1config=`(cd $i; pwd)` - break - fi - done - fi - ]) - - if test x"${ac_cv_c_$1config}" = x ; then - $1_BIN_DIR="# no $1 configs found" - AC_MSG_WARN([Cannot find $1 configuration definitions]) - exit 0 - else - no_$1= - $1_BIN_DIR=${ac_cv_c_$1config} - AC_MSG_RESULT([found $$1_BIN_DIR/$1Config.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_CONFIG -- -# -# Load the $1Config.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# $1_BIN_DIR -# -# Results: -# -# Substitutes the following vars: -# $1_SRC_DIR -# $1_LIB_FILE -# $1_LIB_SPEC -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LOAD_CONFIG], [ - AC_MSG_CHECKING([for existence of ${$1_BIN_DIR}/$1Config.sh]) - - if test -f "${$1_BIN_DIR}/$1Config.sh" ; then - AC_MSG_RESULT([loading]) - . "${$1_BIN_DIR}/$1Config.sh" - else - AC_MSG_RESULT([file not found]) - fi - - # - # If the $1_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable $1_LIB_SPEC will be set to the value - # of $1_BUILD_LIB_SPEC. An extension should make use of $1_LIB_SPEC - # instead of $1_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - # - - if test -f "${$1_BIN_DIR}/Makefile" ; then - AC_MSG_WARN([Found Makefile - using build library specs for $1]) - $1_LIB_SPEC=${$1_BUILD_LIB_SPEC} - $1_STUB_LIB_SPEC=${$1_BUILD_STUB_LIB_SPEC} - $1_STUB_LIB_PATH=${$1_BUILD_STUB_LIB_PATH} - $1_INCLUDE_SPEC=${$1_BUILD_INCLUDE_SPEC} - $1_LIBRARY_PATH=${$1_LIBRARY_PATH} - fi - - AC_SUBST($1_VERSION) - AC_SUBST($1_BIN_DIR) - AC_SUBST($1_SRC_DIR) - - AC_SUBST($1_LIB_FILE) - AC_SUBST($1_LIB_SPEC) - - AC_SUBST($1_STUB_LIB_FILE) - AC_SUBST($1_STUB_LIB_SPEC) - AC_SUBST($1_STUB_LIB_PATH) - - # Allow the caller to prevent this auto-check by specifying any 2nd arg - AS_IF([test "x$2" = x], [ - # Check both upper and lower-case variants - # If a dev wanted non-stubs libs, this function could take an option - # to not use _STUB in the paths below - AS_IF([test "x${$1_STUB_LIB_SPEC}" = x], - [TEA_LOAD_CONFIG_LIB(translit($1,[a-z],[A-Z])_STUB)], - [TEA_LOAD_CONFIG_LIB($1_STUB)]) - ]) -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_CONFIG_LIB -- -# -# Helper function to load correct library from another extension's -# ${PACKAGE}Config.sh. -# -# Results: -# Adds to LIBS the appropriate extension library -#------------------------------------------------------------------------ -AC_DEFUN([TEA_LOAD_CONFIG_LIB], [ - AC_MSG_CHECKING([For $1 library for LIBS]) - # This simplifies the use of stub libraries by automatically adding - # the stub lib to your path. Normally this would add to SHLIB_LD_LIBS, - # but this is called before CONFIG_CFLAGS. More importantly, this adds - # to PKG_LIBS, which becomes LIBS, and that is only used by SHLIB_LD. - if test "x${$1_LIB_SPEC}" != "x" ; then - if test "${TEA_PLATFORM}" = "windows" -a "$GCC" != "yes" ; then - TEA_ADD_LIBS([\"`${CYGPATH} ${$1_LIB_PATH}`\"]) - AC_MSG_RESULT([using $1_LIB_PATH ${$1_LIB_PATH}]) - else - TEA_ADD_LIBS([${$1_LIB_SPEC}]) - AC_MSG_RESULT([using $1_LIB_SPEC ${$1_LIB_SPEC}]) - fi - else - AC_MSG_RESULT([file not found]) - fi -]) - -#------------------------------------------------------------------------ -# TEA_EXPORT_CONFIG -- -# -# Define the data to insert into the ${PACKAGE}Config.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# $1 -# -# Results: -# Substitutes the following vars: -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_EXPORT_CONFIG], [ - #-------------------------------------------------------------------- - # These are for $1Config.sh - #-------------------------------------------------------------------- - - # pkglibdir must be a fully qualified path and (not ${exec_prefix}/lib) - eval pkglibdir="[$]{libdir}/$1${PACKAGE_VERSION}" - if test "${TCL_LIB_VERSIONS_OK}" = "ok"; then - eval $1_LIB_FLAG="-l$1${PACKAGE_VERSION}" - eval $1_STUB_LIB_FLAG="-l$1stub${PACKAGE_VERSION}" - else - eval $1_LIB_FLAG="-l$1`echo ${PACKAGE_VERSION} | tr -d .`" - eval $1_STUB_LIB_FLAG="-l$1stub`echo ${PACKAGE_VERSION} | tr -d .`" - fi - $1_BUILD_LIB_SPEC="-L`$CYGPATH $(pwd)` ${$1_LIB_FLAG}" - $1_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` ${$1_LIB_FLAG}" - $1_BUILD_STUB_LIB_SPEC="-L`$CYGPATH $(pwd)` [$]{$1_STUB_LIB_FLAG}" - $1_STUB_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` [$]{$1_STUB_LIB_FLAG}" - $1_BUILD_STUB_LIB_PATH="`$CYGPATH $(pwd)`/[$]{PKG_STUB_LIB_FILE}" - $1_STUB_LIB_PATH="`$CYGPATH ${pkglibdir}`/[$]{PKG_STUB_LIB_FILE}" - - AC_SUBST($1_BUILD_LIB_SPEC) - AC_SUBST($1_LIB_SPEC) - AC_SUBST($1_BUILD_STUB_LIB_SPEC) - AC_SUBST($1_STUB_LIB_SPEC) - AC_SUBST($1_BUILD_STUB_LIB_PATH) - AC_SUBST($1_STUB_LIB_PATH) - - AC_SUBST(MAJOR_VERSION) - AC_SUBST(MINOR_VERSION) - AC_SUBST(PATCHLEVEL) -]) - - -#------------------------------------------------------------------------ -# TEA_INSTALLER -- -# -# Configure the installer. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# INSTALL -# INSTALL_DATA_DIR -# INSTALL_DATA -# INSTALL_PROGRAM -# INSTALL_SCRIPT -# INSTALL_LIBRARY -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_INSTALLER], [ - INSTALL='$(SHELL) $(srcdir)/tclconfig/install-sh -c' - INSTALL_DATA_DIR='${INSTALL} -d -m 755' - INSTALL_DATA='${INSTALL} -m 644' - INSTALL_PROGRAM='${INSTALL} -m 755' - INSTALL_SCRIPT='${INSTALL} -m 755' - - TEA_CONFIG_SYSTEM - case $system in - HP-UX-*) INSTALL_LIBRARY='${INSTALL} -m 755' ;; - *) INSTALL_LIBRARY='${INSTALL} -m 644' ;; - esac - - AC_SUBST(INSTALL) - AC_SUBST(INSTALL_DATA_DIR) - AC_SUBST(INSTALL_DATA) - AC_SUBST(INSTALL_PROGRAM) - AC_SUBST(INSTALL_SCRIPT) - AC_SUBST(INSTALL_LIBRARY) -]) - -### -# Tip 430 - ZipFS Modifications -### -#------------------------------------------------------------------------ -# TEA_ZIPFS_SUPPORT -# Locate a zip encoder installed on the system path, or none. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# MACHER_PROG -# ZIP_PROG -# ZIP_PROG_OPTIONS -# ZIP_PROG_VFSSEARCH -# ZIP_INSTALL_OBJS -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ZIPFS_SUPPORT], [ - MACHER_PROG="" - ZIP_PROG="" - ZIP_PROG_OPTIONS="" - ZIP_PROG_VFSSEARCH="" - ZIP_INSTALL_OBJS="" - - AC_MSG_CHECKING([for macher]) - AC_CACHE_VAL(ac_cv_path_macher, [ - search_path=`echo ${PATH} | sed -e 's/:/ /g'` - for dir in $search_path ; do - for j in `ls -r $dir/macher 2> /dev/null` \ - `ls -r $dir/macher 2> /dev/null` ; do - if test x"$ac_cv_path_macher" = x ; then - if test -f "$j" ; then - ac_cv_path_macher=$j - break - fi - fi - done - done - ]) - if test -f "$ac_cv_path_macher" ; then - MACHER_PROG="$ac_cv_path_macher" - AC_MSG_RESULT([$MACHER_PROG]) - AC_MSG_RESULT([Found macher in environment]) - fi - AC_MSG_CHECKING([for zip]) - AC_CACHE_VAL(ac_cv_path_zip, [ - search_path=`echo ${PATH} | sed -e 's/:/ /g'` - for dir in $search_path ; do - for j in `ls -r $dir/zip 2> /dev/null` \ - `ls -r $dir/zip 2> /dev/null` ; do - if test x"$ac_cv_path_zip" = x ; then - if test -f "$j" ; then - ac_cv_path_zip=$j - break - fi - fi - done - done - ]) - if test -f "$ac_cv_path_zip" ; then - ZIP_PROG="$ac_cv_path_zip" - AC_MSG_RESULT([$ZIP_PROG]) - ZIP_PROG_OPTIONS="-rq" - ZIP_PROG_VFSSEARCH="*" - AC_MSG_RESULT([Found INFO Zip in environment]) - # Use standard arguments for zip - else - # It is not an error if an installed version of Zip can't be located. - # We can use the locally distributed minizip instead - ZIP_PROG="./minizip${EXEEXT_FOR_BUILD}" - ZIP_PROG_OPTIONS="-o -r" - ZIP_PROG_VFSSEARCH="*" - ZIP_INSTALL_OBJS="minizip${EXEEXT_FOR_BUILD}" - AC_MSG_RESULT([No zip found on PATH. Building minizip]) - fi - AC_SUBST(MACHER_PROG) - AC_SUBST(ZIP_PROG) - AC_SUBST(ZIP_PROG_OPTIONS) - AC_SUBST(ZIP_PROG_VFSSEARCH) - AC_SUBST(ZIP_INSTALL_OBJS) -]) - -# Local Variables: -# mode: autoconf -# End: \ No newline at end of file diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl new file mode 100644 index 0000000000..47e0ea7013 --- /dev/null +++ b/autoconf/tea/teaish.tcl @@ -0,0 +1,569 @@ +# Teaish configure script for the SQLite Tcl extension + +# +# State for disparate config-time pieces. +# +array set sqlite__Config [proj-strip-hash-comments { + # + # The list of feature --flags which the --all flag implies. This + # requires special handling in a few places. + # + all-flag-enables {fts3 fts4 fts5 rtree geopoly} + + # >0 if building in the canonical tree. -1=undetermined + is-canonical -1 +}] + +# +# Set up the package info for teaish... +# +apply {{dir} { + # Figure out the version number... + set version "" + if {[file exists $dir/../VERSION]} { + # The canonical SQLite TEA(ish) build + set version [proj-file-content -trim $dir/../VERSION] + set ::sqlite__Config(is-canonical) 1 + set distname sqlite-tcl + } elseif {[file exists $dir/generic/tclsqlite3.c]} { + # The copy from the teaish tree, used as a dev/test bed before + # updating SQLite's tree. + set ::sqlite__Config(is-canonical) 0 + set fd [open $dir/generic/tclsqlite3.c rb] + while {[gets $fd line] >=0} { + if {[regexp {^#define[ ]+SQLITE_VERSION[ ]+"(3.+)"} \ + $line - version]} { + set distname sqlite-teaish + break + } + } + close $fd + } + + if {"" eq $version} { + proj-fatal "Cannot determine the SQLite version number" + } + + proj-assert {$::sqlite__Config(is-canonical) > -1} + proj-assert {[string match 3.*.* $version]} \ + "Unexpected SQLite version: $version" + + set pragmas {} + if {$::sqlite__Config(is-canonical)} { + # Disable "make dist" in the canonical tree. That tree is + # generated from several pieces and creating/testing working + # "dist" rules for that sub-build currently feels unnecessary. The + # copy in the teaish tree, though, should be able to "make dist". + lappend pragmas no-dist + } else { + lappend pragmas full-dist + } + + teaish-pkginfo-set -vars { + -name sqlite + -name.pkg sqlite3 + -version $version + -name.dist $distname + -libDir sqlite$version + -pragmas $pragmas + -src generic/tclsqlite3.c + } + # We should also have: + # -vsatisfies 8.6- + # But at least one platform is failing this vsatisfies check + # for no apparent reason: + # https://sqlite.org/forum/forumpost/fde857fb8101a4be +}} [teaish-get -dir] + + +# +# Must return either an empty string or a list in the form accepted by +# autosetup's [options] function. +# +proc teaish-options {} { + # These flags and defaults mostly derive from the historical TEA + # build. Some, like ICU, are taken from the canonical SQLite tree. + return [subst -nocommands -nobackslashes { + with-system-sqlite=0 + => {Use the system-level SQLite instead of the copy in this tree. + Also requires use of --override-sqlite-version so that the build + knows what version number to associate with the system-level SQLite.} + override-sqlite-version:VERSION + => {For use with --with-system-sqlite to set the version number.} + threadsafe=1 => {Disable mutexing} + with-tempstore:=no => {Use an in-RAM database for temporary tables: never,no,yes,always} + load-extension=0 => {Enable loading of external extensions} + math=1 => {Disable math functions} + json=1 => {Disable JSON functions} + fts3 => {Enable the FTS3 extension} + fts4 => {Enable the FTS4 extension} + fts5 => {Enable the FTS5 extension} + update-limit => {Enable the UPDATE/DELETE LIMIT clause} + geopoly => {Enable the GEOPOLY extension} + rtree => {Enable the RTREE extension} + session => {Enable the SESSION extension} + all=1 => {Disable $::sqlite__Config(all-flag-enables)} + with-icu-ldflags:LDFLAGS + => {Enable SQLITE_ENABLE_ICU and add the given linker flags for the + ICU libraries. e.g. on Ubuntu systems, try '-licui18n -licuuc -licudata'.} + with-icu-cflags:CFLAGS + => {Apply extra CFLAGS/CPPFLAGS necessary for building with ICU. + e.g. -I/usr/local/include} + with-icu-config:=auto + => {Enable SQLITE_ENABLE_ICU. Value must be one of: auto, pkg-config, + /path/to/icu-config} + icu-collations=0 + => {Enable SQLITE_ENABLE_ICU_COLLATIONS. Requires --with-icu-ldflags=... + or --with-icu-config} + }] +} + +# +# Gets called by tea-configure-core. Must perform any configuration +# work needed for this extension. +# +proc teaish-configure {} { + use teaish/feature + + if {[proj-opt-was-provided override-sqlite-version]} { + teaish-pkginfo-set -version [opt-val override-sqlite-version] + proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] + } elseif {[proj-opt-was-provided with-system-sqlite] + && [opt-val with-system-sqlite] ne "0"} { + proj-fatal "when using --with-system-sqlite also use" \ + "--override-sqlite-version to specify a library version number." + } + + define CFLAGS [proj-get-env CFLAGS {-O2}] + sqlite-munge-cflags + + # + # Add feature flags from legacy configure.ac which are not covered by + # --flags. + # + sqlite-add-feature-flag { + -DSQLITE_3_SUFFIX_ONLY=1 + -DSQLITE_ENABLE_DESERIALIZE=1 + -DSQLITE_ENABLE_DBPAGE_VTAB=1 + -DSQLITE_ENABLE_BYTECODE_VTAB=1 + -DSQLITE_ENABLE_DBSTAT_VTAB=1 + } + + if {[opt-bool with-system-sqlite]} { + msg-result "Using system-level sqlite3." + teaish-cflags-add -DUSE_SYSTEM_SQLITE + teaish-ldflags-add -lsqlite3 + } elseif {$::sqlite__Config(is-canonical)} { + teaish-cflags-add -I[teaish-get -dir]/.. + } + + teaish-check-librt + teaish-check-libz + sqlite-handle-threadsafe + sqlite-handle-tempstore + sqlite-handle-load-extension + sqlite-handle-math + sqlite-handle-icu + + sqlite-handle-common-feature-flags; # must be late in the process +}; # teaish-configure + +define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. +# +# Adds $args, if not empty, to OPT_FEATURE_FLAGS. This is intended only for holding +# -DSQLITE_ENABLE/OMIT/... flags, but that is not enforced here. +proc sqlite-add-feature-flag {args} { + if {"" ne $args} { + define-append OPT_FEATURE_FLAGS {*}$args + } +} + +# +# Check for log(3) in libm and die with an error if it is not +# found. $featureName should be the feature name which requires that +# function (it's used only in error messages). defines LDFLAGS_MATH to +# the required linker flags (which may be empty even if the math APIs +# are found, depending on the OS). +proc sqlite-affirm-have-math {featureName} { + if {"" eq [get-define LDFLAGS_MATH ""]} { + if {![msg-quiet proj-check-function-in-lib log m]} { + user-error "Missing math APIs for $featureName" + } + set lfl [get-define lib_log ""] + undefine lib_log + if {"" ne $lfl} { + user-notice "Forcing requirement of $lfl for $featureName" + } + define LDFLAGS_MATH $lfl + teaish-ldflags-prepend $lfl + } +} + +# +# Handle various SQLITE_ENABLE/OMIT_... feature flags. +proc sqlite-handle-common-feature-flags {} { + msg-result "Feature flags..." + if {![opt-bool all]} { + # Special handling for --disable-all + foreach flag $::sqlite__Config(all-flag-enables) { + if {![proj-opt-was-provided $flag]} { + proj-opt-set $flag 0 + } + } + } + foreach {boolFlag featureFlag ifSetEvalThis} [proj-strip-hash-comments { + all {} { + # The 'all' option must be first in this list. This impl makes + # an effort to only apply flags which the user did not already + # apply, so that combinations like (--all --disable-geopoly) + # will indeed disable geopoly. There are corner cases where + # flags which depend on each other will behave in non-intuitive + # ways: + # + # --all --disable-rtree + # + # Will NOT disable geopoly, though geopoly depends on rtree. + # The --geopoly flag, though, will automatically re-enable + # --rtree, so --disable-rtree won't actually disable anything in + # that case. + foreach k $::sqlite__Config(all-flag-enables) { + if {![proj-opt-was-provided $k]} { + proj-opt-set $k 1 + } + } + } + fts3 -DSQLITE_ENABLE_FTS3 {sqlite-affirm-have-math fts3} + fts4 -DSQLITE_ENABLE_FTS4 {sqlite-affirm-have-math fts4} + fts5 -DSQLITE_ENABLE_FTS5 {sqlite-affirm-have-math fts5} + geopoly -DSQLITE_ENABLE_GEOPOLY {proj-opt-set rtree} + rtree -DSQLITE_ENABLE_RTREE {} + session {-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK} {} + update-limit -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT {} + scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + }] { + if {$boolFlag ni $::autosetup(options)} { + # Skip flags which are in the canonical build but not + # the autoconf bundle. + continue + } + proj-if-opt-truthy $boolFlag { + sqlite-add-feature-flag $featureFlag + if {0 != [eval $ifSetEvalThis] && "all" ne $boolFlag} { + msg-result " + $boolFlag" + } + } { + if {"all" ne $boolFlag} { + msg-result " - $boolFlag" + } + } + } + # + # Invert the above loop's logic for some SQLITE_OMIT_... cases. If + # config option $boolFlag is false, [sqlite-add-feature-flag + # $featureFlag], where $featureFlag is intended to be + # -DSQLITE_OMIT_... + foreach {boolFlag featureFlag} { + json -DSQLITE_OMIT_JSON + } { + if {[proj-opt-truthy $boolFlag]} { + msg-result " + $boolFlag" + } else { + sqlite-add-feature-flag $featureFlag + msg-result " - $boolFlag" + } + } + + ## + # Remove duplicates from the final feature flag sets and show them + # to the user. + set oFF [get-define OPT_FEATURE_FLAGS] + if {"" ne $oFF} { + define OPT_FEATURE_FLAGS [lsort -unique $oFF] + msg-result "Library feature flags: [get-define OPT_FEATURE_FLAGS]" + } + if {[lsearch [get-define TARGET_DEBUG ""] -DSQLITE_DEBUG=1] > -1} { + msg-result "Note: this is a debug build, so performance will suffer." + } + teaish-cflags-add -define OPT_FEATURE_FLAGS +}; # sqlite-handle-common-feature-flags + +# +# If --enable-threadsafe is set, this adds -DSQLITE_THREADSAFE=1 to +# OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to the linker flags +# needed for linking pthread (possibly an empty string). If +# --enable-threadsafe is not set, adds -DSQLITE_THREADSAFE=0 to +# OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to an empty string. +# +# It prepends the flags to the global LDFLAGS. +proc sqlite-handle-threadsafe {} { + msg-checking "Support threadsafe operation? " + define LDFLAGS_PTHREAD "" + set enable 0 + if {[proj-opt-was-provided threadsafe]} { + proj-if-opt-truthy threadsafe { + if {[proj-check-function-in-lib pthread_create pthread] + && [proj-check-function-in-lib pthread_mutexattr_init pthread]} { + incr enable + set ldf [get-define lib_pthread_create] + define LDFLAGS_PTHREAD $ldf + teaish-ldflags-prepend $ldf + undefine lib_pthread_create + undefine lib_pthread_mutexattr_init + } else { + user-error "Missing required pthread libraries. Use --disable-threadsafe to disable this check." + } + # Recall that LDFLAGS_PTHREAD might be empty even if pthreads if + # found because it's in -lc on some platforms. + } { + msg-result "Disabled using --disable-threadsafe" + } + } else { + # + # If user does not specify --[disable-]threadsafe then select a + # default based on whether it looks like Tcl has threading + # support. + # + catch { + scan [exec echo {puts [tcl::pkgconfig get threaded]} | [get-define TCLSH_CMD]] \ + %d enable + } + if {$enable} { + set flagName "--threadsafe" + set lblAbled "enabled" + msg-result yes + } else { + set flagName "--disable-threadsafe" + set lblAbled "disabled" + msg-result no + } + msg-result "Defaulting to ${flagName} because Tcl has threading ${lblAbled}." + # ^^^ We (probably) don't need to link against -lpthread in the + # is-enabled case. We might in the case of static linking. Unsure. + } + sqlite-add-feature-flag -DSQLITE_THREADSAFE=${enable} + return $enable +} + +# +# Handles the --enable-load-extension flag. Returns 1 if the support +# is enabled, else 0. If support for that feature is not found, a +# fatal error is triggered if --enable-load-extension is explicitly +# provided, else a loud warning is instead emitted. If +# --disable-load-extension is used, no check is performed. +# +# Makes the following environment changes: +# +# - defines LDFLAGS_DLOPEN to any linker flags needed for this +# feature. It may legally be empty on some systems where dlopen() +# is in libc. +# +# - If the feature is not available, adds +# -DSQLITE_OMIT_LOAD_EXTENSION=1 to the feature flags list. +proc sqlite-handle-load-extension {} { + define LDFLAGS_DLOPEN "" + set found 0 + proj-if-opt-truthy load-extension { + set found [proj-check-function-in-lib dlopen dl] + if {$found} { + set ldf [get-define lib_dlopen] + define LDFLAGS_DLOPEN $ldf + teaish-ldflags-prepend $ldf + undefine lib_dlopen + } else { + if {[proj-opt-was-provided load-extension]} { + # Explicit --enable-load-extension: fail if not found + proj-indented-notice -error { + --enable-load-extension was provided but dlopen() + not found. Use --disable-load-extension to bypass this + check. + } + } else { + # It was implicitly enabled: warn if not found + proj-indented-notice { + WARNING: dlopen() not found, so loadable module support will + be disabled. Use --disable-load-extension to bypass this + check. + } + } + } + } + if {$found} { + msg-result "Loadable extension support enabled." + } else { + msg-result "Disabling loadable extension support. Use --enable-load-extension to enable them." + sqlite-add-feature-flag -DSQLITE_OMIT_LOAD_EXTENSION=1 + } + return $found +} + +# +# ICU - International Components for Unicode +# +# Handles these flags: +# +# --with-icu-ldflags=LDFLAGS +# --with-icu-cflags=CFLAGS +# --with-icu-config[=auto | pkg-config | /path/to/icu-config] +# --enable-icu-collations +# +# --with-icu-config values: +# +# - auto: use the first one of (pkg-config, icu-config) found on the +# system. +# - pkg-config: use only pkg-config to determine flags +# - /path/to/icu-config: use that to determine flags +# +# If --with-icu-config is used as neither pkg-config nor icu-config +# are found, fail fatally. +# +# If both --with-icu-ldflags and --with-icu-config are provided, they +# are cumulative. If neither are provided, icu-collations is not +# honored and a warning is emitted if it is provided. +# +# Design note: though we could automatically enable ICU if the +# icu-config binary or (pkg-config icu-io) are found, we specifically +# do not. ICU is always an opt-in feature. +proc sqlite-handle-icu {} { + define LDFLAGS_LIBICU [join [opt-val with-icu-ldflags ""]] + define CFLAGS_LIBICU [join [opt-val with-icu-cflags ""]] + if {[proj-opt-was-provided with-icu-config]} { + msg-result "Checking for ICU support..." + set icuConfigBin [opt-val with-icu-config] + set tryIcuConfigBin 1; # set to 0 if we end up using pkg-config + if {$icuConfigBin in {auto pkg-config}} { + uplevel 3 { use pkg-config } + if {[pkg-config-init 0] && [pkg-config icu-io]} { + # Maintenance reminder: historical docs say to use both of + # (icu-io, icu-uc). icu-uc lacks a required lib and icu-io has + # all of them on tested OSes. + set tryIcuConfigBin 0 + define LDFLAGS_LIBICU [get-define PKG_ICU_IO_LDFLAGS] + define-append LDFLAGS_LIBICU [get-define PKG_ICU_IO_LIBS] + define CFLAGS_LIBICU [get-define PKG_ICU_IO_CFLAGS] + } elseif {"pkg-config" eq $icuConfigBin} { + proj-fatal "pkg-config cannot find package icu-io" + } else { + proj-assert {"auto" eq $icuConfigBin} + } + } + if {$tryIcuConfigBin} { + if {"auto" eq $icuConfigBin} { + set icuConfigBin [proj-first-bin-of \ + /usr/local/bin/icu-config \ + /usr/bin/icu-config] + if {"" eq $icuConfigBin} { + proj-indented-notice -error { + --with-icu-config=auto cannot find (pkg-config icu-io) or icu-config binary. + On Ubuntu-like systems try: + --with-icu-ldflags='-licui18n -licuuc -licudata' + } + } + } + if {[file-isexec $icuConfigBin]} { + set x [exec $icuConfigBin --ldflags] + if {"" eq $x} { + proj-indented-notice -error \ + [subst { + $icuConfigBin --ldflags returned no data. + On Ubuntu-like systems try: + --with-icu-ldflags='-licui18n -licuuc -licudata' + }] + } + define-append LDFLAGS_LIBICU $x + set x [exec $icuConfigBin --cppflags] + define-append CFLAGS_LIBICU $x + } else { + proj-fatal "--with-icu-config=$icuConfigBin does not refer to an executable" + } + } + } + set ldflags [define LDFLAGS_LIBICU [string trim [get-define LDFLAGS_LIBICU]]] + set cflags [define CFLAGS_LIBICU [string trim [get-define CFLAGS_LIBICU]]] + if {"" ne $ldflags} { + sqlite-add-feature-flag -DSQLITE_ENABLE_ICU + msg-result "Enabling ICU support with flags: $ldflags $cflags" + if {[opt-bool icu-collations]} { + msg-result "Enabling ICU collations." + sqlite-add-feature-flag -DSQLITE_ENABLE_ICU_COLLATIONS + } + teaish-ldflags-prepend $ldflags + teaish-cflags-add $cflags + } elseif {[opt-bool icu-collations]} { + proj-warn "ignoring --enable-icu-collations because neither --with-icu-ldflags nor --with-icu-config provided any linker flags" + } else { + msg-result "ICU support is disabled." + } +}; # sqlite-handle-icu + + +# +# Handles the --with-tempstore flag. +# +# The test fixture likes to set SQLITE_TEMP_STORE on its own, so do +# not set that feature flag unless it was explicitly provided to the +# configure script. +proc sqlite-handle-tempstore {} { + if {[proj-opt-was-provided with-tempstore]} { + set ts [opt-val with-tempstore no] + set tsn 1 + msg-checking "Use an in-RAM database for temporary tables? " + switch -exact -- $ts { + never { set tsn 0 } + no { set tsn 1 } + yes { set tsn 2 } + always { set tsn 3 } + default { + user-error "Invalid --with-tempstore value '$ts'. Use one of: never, no, yes, always" + } + } + msg-result $ts + sqlite-add-feature-flag -DSQLITE_TEMP_STORE=$tsn + } +} + +# +# Handles the --enable-math flag. +proc sqlite-handle-math {} { + proj-if-opt-truthy math { + if {![proj-check-function-in-lib ceil m]} { + user-error "Cannot find libm functions. Use --disable-math to bypass this." + } + set lfl [get-define lib_ceil] + undefine lib_ceil + define LDFLAGS_MATH $lfl + teaish-ldflags-prepend $lfl + sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS + msg-result "Enabling math SQL functions" + } { + define LDFLAGS_MATH "" + msg-result "Disabling math SQL functions" + } +} + +# +# Move -DSQLITE_OMIT... and -DSQLITE_ENABLE... flags from CFLAGS and +# CPPFLAGS to OPT_FEATURE_FLAGS and remove them from BUILD_CFLAGS. +proc sqlite-munge-cflags {} { + # Move CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and + # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived + # from the pre-3.48 build. + # + # If any configure flags for features are in conflict with + # CFLAGS/CPPFLAGS-specified feature flags, all bets are off. There + # are no guarantees about which one will take precedence. + foreach flagDef {CFLAGS CPPFLAGS} { + set tmp "" + foreach cf [get-define $flagDef ""] { + switch -glob -- $cf { + -DSQLITE_OMIT* - + -DSQLITE_ENABLE* { + sqlite-add-feature-flag $cf + } + default { + lappend tmp $cf + } + } + } + define $flagDef $tmp + } +} diff --git a/autoconf/tea/teaish.test.tcl b/autoconf/tea/teaish.test.tcl new file mode 100644 index 0000000000..b63c9426e3 --- /dev/null +++ b/autoconf/tea/teaish.test.tcl @@ -0,0 +1,14 @@ +test-expect 1.0-open { + sqlite3 db :memory: +} {} + +test-assert 1.1-version-3.x { + [string match 3.* [db eval {select sqlite_version()}]] +} + +test-expect 1.2-select { + db eval {select 'hi, world',1,2,3} +} {{hi, world} 1 2 3} + + +test-expect 99.0-db-close {db close} {} diff --git a/autoconf/tea/win/makefile.vc b/autoconf/tea/win/makefile.vc deleted file mode 100644 index da56e811fc..0000000000 --- a/autoconf/tea/win/makefile.vc +++ /dev/null @@ -1,430 +0,0 @@ -# makefile.vc -- -*- Makefile -*- -# -# Microsoft Visual C++ makefile for use with nmake.exe v1.62+ (VC++ 5.0+) -# -# This makefile is based upon the Tcl 8.4 Makefile.vc and modified to -# make it suitable as a general package makefile. Look for the word EDIT -# which marks sections that may need modification. As a minumum you will -# need to change the PROJECT, DOTVERSION and DLLOBJS variables to values -# relevant to your package. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# -# Copyright (c) 1995-1996 Sun Microsystems, Inc. -# Copyright (c) 1998-2000 Ajuba Solutions. -# Copyright (c) 2001 ActiveState Corporation. -# Copyright (c) 2001-2002 David Gravereaux. -# Copyright (c) 2003 Pat Thoyts -# -#------------------------------------------------------------------------- -# RCS: @(#)$Id: makefile.vc,v 1.4 2004/07/26 08:22:05 patthoyts Exp $ -#------------------------------------------------------------------------- - -!if !defined(MSDEVDIR) && !defined(MSVCDIR) && !defined(VCINSTALLDIR) && !defined(MSSDK) && !defined(WINDOWSSDKDIR) -MSG = ^ -You will need to run vcvars32.bat from Developer Studio, first, to setup^ -the environment. Jump to this line to read the new instructions. -!error $(MSG) -!endif - -#------------------------------------------------------------------------------ -# HOW TO USE this makefile: -# -# 1) It is now necessary to have %MSVCDir% set in the environment. This is -# used as a check to see if vcvars32.bat had been run prior to running -# nmake or during the installation of Microsoft Visual C++, MSVCDir had -# been set globally and the PATH adjusted. Either way is valid. -# -# You'll need to run vcvars32.bat contained in the MsDev's vc(98)/bin -# directory to setup the proper environment, if needed, for your current -# setup. This is a needed bootstrap requirement and allows the swapping of -# different environments to be easier. -# -# 2) To use the Platform SDK (not expressly needed), run setenv.bat after -# vcvars32.bat according to the instructions for it. This can also turn on -# the 64-bit compiler, if your SDK has it. -# -# 3) Targets are: -# all -- Builds everything. -# -- Builds the project (eg: nmake sample) -# test -- Builds and runs the test suite. -# install -- Installs the built binaries and libraries to $(INSTALLDIR) -# in an appropriate subdirectory. -# clean/realclean/distclean -- varying levels of cleaning. -# -# 4) Macros usable on the commandline: -# INSTALLDIR= -# Sets where to install Tcl from the built binaries. -# C:\Progra~1\Tcl is assumed when not specified. -# -# OPTS=static,msvcrt,staticpkg,threads,symbols,profile,loimpact,none -# Sets special options for the core. The default is for none. -# Any combination of the above may be used (comma separated). -# 'none' will over-ride everything to nothing. -# -# static = Builds a static library of the core instead of a -# dll. The shell will be static (and large), as well. -# msvcrt = Effects the static option only to switch it from -# using libcmt(d) as the C runtime [by default] to -# msvcrt(d). This is useful for static embedding -# support. -# staticpkg = Effects the static option only to switch -# tclshXX.exe to have the dde and reg extension linked -# inside it. -# threads = Turns on full multithreading support. -# thrdalloc = Use the thread allocator (shared global free pool). -# symbols = Adds symbols for step debugging. -# profile = Adds profiling hooks. Map file is assumed. -# loimpact = Adds a flag for how NT treats the heap to keep memory -# in use, low. This is said to impact alloc performance. -# -# STATS=memdbg,compdbg,none -# Sets optional memory and bytecode compiler debugging code added -# to the core. The default is for none. Any combination of the -# above may be used (comma separated). 'none' will over-ride -# everything to nothing. -# -# memdbg = Enables the debugging memory allocator. -# compdbg = Enables byte compilation logging. -# -# MACHINE=(IX86|IA64|ALPHA) -# Set the machine type used for the compiler, linker, and -# resource compiler. This hook is needed to tell the tools -# when alternate platforms are requested. IX86 is the default -# when not specified. -# -# TMP_DIR= -# OUT_DIR= -# Hooks to allow the intermediate and output directories to be -# changed. $(OUT_DIR) is assumed to be -# $(BINROOT)\(Release|Debug) based on if symbols are requested. -# $(TMP_DIR) will de $(OUT_DIR)\ by default. -# -# TESTPAT= -# Reads the tests requested to be run from this file. -# -# CFG_ENCODING=encoding -# name of encoding for configuration information. Defaults -# to cp1252 -# -# 5) Examples: -# -# Basic syntax of calling nmake looks like this: -# nmake [-nologo] -f makefile.vc [target|macrodef [target|macrodef] [...]] -# -# Standard (no frills) -# c:\tcl_src\win\>c:\progra~1\micros~1\vc98\bin\vcvars32.bat -# Setting environment for using Microsoft Visual C++ tools. -# c:\tcl_src\win\>nmake -f makefile.vc all -# c:\tcl_src\win\>nmake -f makefile.vc install INSTALLDIR=c:\progra~1\tcl -# -# Building for Win64 -# c:\tcl_src\win\>c:\progra~1\micros~1\vc98\bin\vcvars32.bat -# Setting environment for using Microsoft Visual C++ tools. -# c:\tcl_src\win\>c:\progra~1\platfo~1\setenv.bat /pre64 /RETAIL -# Targeting Windows pre64 RETAIL -# c:\tcl_src\win\>nmake -f makefile.vc MACHINE=IA64 -# -#------------------------------------------------------------------------------ -#============================================================================== -############################################################################### -#------------------------------------------------------------------------------ - -!if !exist("makefile.vc") -MSG = ^ -You must run this makefile only from the directory it is in.^ -Please `cd` to its location first. -!error $(MSG) -!endif - -#------------------------------------------------------------------------- -# Project specific information (EDIT) -# -# You should edit this with the name and version of your project. This -# information is used to generate the name of the package library and -# it's install location. -# -# For example, the sample extension is going to build sample04.dll and -# would install it into $(INSTALLDIR)\lib\sample04 -# -# You need to specify the object files that need to be linked into your -# binary here. -# -#------------------------------------------------------------------------- - -PROJECT = sqlite3 -!include "rules.vc" - -# nmakehelp -V will search the file for tag, skips until a -# number and returns all character until a character not in [0-9.ab] -# is read. - -!if [echo REM = This file is generated from Makefile.vc > versions.vc] -!endif -# get project version from row "AC_INIT([sqlite], [3.x.y])" -!if [echo DOTVERSION = \>> versions.vc] \ - && [nmakehlp -V ..\configure.ac AC_INIT >> versions.vc] -!endif -!include "versions.vc" - -VERSION = $(DOTVERSION:.=) -STUBPREFIX = $(PROJECT)stub - -#------------------------------------------------------------------------- -# Target names and paths ( shouldn't need changing ) -#------------------------------------------------------------------------- - -BINROOT = . -ROOT = .. - -PRJIMPLIB = $(OUT_DIR)\$(PROJECT)$(VERSION)$(SUFX).lib -PRJLIBNAME = $(PROJECT).$(EXT) -PRJLIB = $(OUT_DIR)\$(PRJLIBNAME) - -PRJSTUBLIBNAME = $(STUBPREFIX)$(VERSION).lib -PRJSTUBLIB = $(OUT_DIR)\$(PRJSTUBLIBNAME) - -### Make sure we use backslash only. -PRJ_INSTALL_DIR = $(_INSTALLDIR)\$(PROJECT)$(DOTVERSION) -LIB_INSTALL_DIR = $(PRJ_INSTALL_DIR) -BIN_INSTALL_DIR = $(PRJ_INSTALL_DIR) -DOC_INSTALL_DIR = $(PRJ_INSTALL_DIR) -SCRIPT_INSTALL_DIR = $(PRJ_INSTALL_DIR) -INCLUDE_INSTALL_DIR = $(_TCLDIR)\include - -### The following paths CANNOT have spaces in them. -GENERICDIR = $(ROOT)\generic -WINDIR = $(ROOT)\win -LIBDIR = $(ROOT)\library -DOCDIR = $(ROOT)\doc -TOOLSDIR = $(ROOT)\tools -COMPATDIR = $(ROOT)\compat - -### Figure out where the primary source code file(s) is/are. -!if exist("$(ROOT)\..\..\sqlite3.c") && exist("$(ROOT)\..\..\src\tclsqlite.c") -SQL_INCLUDES = -I"$(ROOT)\..\.." -SQLITE_SRCDIR = $(ROOT)\..\.. -TCLSQLITE_SRCDIR = $(ROOT)\..\..\src -DLLOBJS = $(TMP_DIR)\sqlite3.obj $(TMP_DIR)\tclsqlite.obj -!else -TCLSQLITE_SRCDIR = $(ROOT)\generic -DLLOBJS = $(TMP_DIR)\tclsqlite3.obj -!endif - -#--------------------------------------------------------------------- -# Compile flags -#--------------------------------------------------------------------- - -!if !$(DEBUG) -!if $(OPTIMIZING) -### This cranks the optimization level to maximize speed -cdebug = -O2 -Op -Gs -!else -cdebug = -!endif -!else if "$(MACHINE)" == "IA64" -### Warnings are too many, can't support warnings into errors. -cdebug = -Z7 -Od -GZ -!else -cdebug = -Z7 -WX -Od -GZ -!endif - -### Declarations common to all compiler options -cflags = -nologo -c -W3 -D_CRT_SECURE_NO_WARNINGS -YX -Fp$(TMP_DIR)^\ - -!if $(MSVCRT) -!if $(DEBUG) -crt = -MDd -!else -crt = -MD -!endif -!else -!if $(DEBUG) -crt = -MTd -!else -crt = -MT -!endif -!endif - -INCLUDES = $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \ - -I"$(GENERICDIR)" -I"$(ROOT)\.." -BASE_CLFAGS = $(cflags) $(cdebug) $(crt) $(INCLUDES) \ - -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \ - -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 \ - -DSQLITE_ENABLE_FTS4=1 \ - -DSQLITE_ENABLE_FTS5=1 \ - -DSQLITE_3_SUFFIX_ONLY=1 \ - -DSQLITE_ENABLE_RTREE=1 \ - -DSQLITE_ENABLE_GEOPOLY=1 \ - -DSQLITE_ENABLE_MATH_FUNCTIONS=1 \ - -DSQLITE_ENABLE_DESERIALIZE=1 \ - -DSQLITE_ENABLE_DBPAGE_VTAB=1 \ - -DSQLITE_ENABLE_BYTECODE_VTAB=1 \ - -DSQLITE_ENABLE_DBSTAT_VTAB=1 - -CON_CFLAGS = $(cflags) $(cdebug) $(crt) -DCONSOLE -DSQLITE_ENABLE_FTS3=1 -TCL_CFLAGS = -DBUILD_sqlite -DUSE_TCL_STUBS \ - -DPACKAGE_VERSION="\"$(DOTVERSION)\"" $(BASE_CLFAGS) \ - $(OPTDEFINES) - -#--------------------------------------------------------------------- -# Link flags -#--------------------------------------------------------------------- - -!if $(DEBUG) -ldebug = -debug:full -debugtype:cv -!else -ldebug = -release -opt:ref -opt:icf,3 -!endif - -### Declarations common to all linker options -lflags = -nologo -machine:$(MACHINE) $(ldebug) - -!if $(PROFILE) -lflags = $(lflags) -profile -!endif - -!if $(ALIGN98_HACK) && !$(STATIC_BUILD) -### Align sections for PE size savings. -lflags = $(lflags) -opt:nowin98 -!else if !$(ALIGN98_HACK) && $(STATIC_BUILD) -### Align sections for speed in loading by choosing the virtual page size. -lflags = $(lflags) -align:4096 -!endif - -!if $(LOIMPACT) -lflags = $(lflags) -ws:aggressive -!endif - -dlllflags = $(lflags) -dll -conlflags = $(lflags) -subsystem:console -guilflags = $(lflags) -subsystem:windows -baselibs = $(TCLSTUBLIB) - -#--------------------------------------------------------------------- -# TclTest flags -#--------------------------------------------------------------------- - -!IF "$(TESTPAT)" != "" -TESTFLAGS = $(TESTFLAGS) -file $(TESTPAT) -!ENDIF - -#--------------------------------------------------------------------- -# Project specific targets (EDIT) -#--------------------------------------------------------------------- - -all: setup $(PROJECT) -$(PROJECT): setup $(PRJLIB) -install: install-binaries install-libraries install-docs - -# Tests need to ensure we load the right dll file we -# have to handle the output differently on Win9x. -# -!if "$(OS)" == "Windows_NT" || "$(MSVCDIR)" == "IDE" -test: setup $(PROJECT) - set TCL_LIBRARY=$(ROOT)/library - $(TCLSH) << -load $(PRJLIB:\=/) -cd "$(ROOT)/tests" -set argv "$(TESTFLAGS)" -source all.tcl -<< -!else -test: setup $(PROJECT) - echo Please wait while the test results are collected - set TCL_LIBRARY=$(ROOT)/library - $(TCLSH) << >tests.log -load $(PRJLIB:\=/) -cd "$(ROOT)/tests" -set argv "$(TESTFLAGS)" -source all.tcl -<< - type tests.log | more -!endif - -setup: - @if not exist $(OUT_DIR)\nul mkdir $(OUT_DIR) - @if not exist $(TMP_DIR)\nul mkdir $(TMP_DIR) - -$(PRJLIB): $(DLLOBJS) - $(link32) $(dlllflags) -out:$@ $(baselibs) @<< -$** -<< - -@del $*.exp - -$(PRJSTUBLIB): $(PRJSTUBOBJS) - $(lib32) -nologo -out:$@ $(PRJSTUBOBJS) - -#--------------------------------------------------------------------- -# Implicit rules -#--------------------------------------------------------------------- - -$(TMP_DIR)\sqlite3.obj: $(SQLITE_SRCDIR)\sqlite3.c - $(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \ - -c $(SQLITE_SRCDIR)\sqlite3.c - -$(TMP_DIR)\tclsqlite.obj: $(TCLSQLITE_SRCDIR)\tclsqlite.c - $(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \ - -c $(TCLSQLITE_SRCDIR)\tclsqlite.c - -$(TMP_DIR)\tclsqlite3.obj: $(TCLSQLITE_SRCDIR)\tclsqlite3.c - $(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \ - -c $(TCLSQLITE_SRCDIR)\tclsqlite3.c - -{$(WINDIR)}.rc{$(TMP_DIR)}.res: - $(rc32) -fo $@ -r -i "$(GENERICDIR)" -D__WIN32__ \ -!if $(DEBUG) - -d DEBUG \ -!endif -!if $(TCL_THREADS) - -d TCL_THREADS \ -!endif -!if $(STATIC_BUILD) - -d STATIC_BUILD \ -!endif - $< - -.SUFFIXES: -.SUFFIXES:.c .rc - -#--------------------------------------------------------------------- -# Installation. (EDIT) -# -# You may need to modify this section to reflect the final distribution -# of your files and possibly to generate documentation. -# -#--------------------------------------------------------------------- - -install-binaries: - @echo Installing binaries to '$(SCRIPT_INSTALL_DIR)' - @if not exist "$(SCRIPT_INSTALL_DIR)" mkdir "$(SCRIPT_INSTALL_DIR)" - @$(CPY) $(PRJLIB) "$(SCRIPT_INSTALL_DIR)" >NUL - -install-libraries: - @echo Installing libraries to '$(SCRIPT_INSTALL_DIR)' - @if exist $(LIBDIR) $(CPY) $(LIBDIR)\*.tcl "$(SCRIPT_INSTALL_DIR)" - @echo Installing package index in '$(SCRIPT_INSTALL_DIR)' - @type << >"$(SCRIPT_INSTALL_DIR)\pkgIndex.tcl" -package ifneeded $(PROJECT) $(DOTVERSION) \ - [list load [file join $$dir $(PRJLIBNAME)] sqlite3] -<< - -install-docs: - @echo Installing documentation files to '$(DOC_INSTALL_DIR)' - @if exist $(DOCDIR) $(CPY) $(DOCDIR)\*.n "$(DOC_INSTALL_DIR)" - -#--------------------------------------------------------------------- -# Clean up -#--------------------------------------------------------------------- - -clean: - @if exist $(TMP_DIR)\nul $(RMDIR) $(TMP_DIR) - @if exist $(WINDIR)\version.vc del $(WINDIR)\version.vc - -realclean: clean - @if exist $(OUT_DIR)\nul $(RMDIR) $(OUT_DIR) - -distclean: realclean - @if exist $(WINDIR)\nmakehlp.exe del $(WINDIR)\nmakehlp.exe - @if exist $(WINDIR)\nmakehlp.obj del $(WINDIR)\nmakehlp.obj diff --git a/autoconf/tea/win/nmakehlp.c b/autoconf/tea/win/nmakehlp.c deleted file mode 100644 index 2dc33cc657..0000000000 --- a/autoconf/tea/win/nmakehlp.c +++ /dev/null @@ -1,815 +0,0 @@ -/* - * ---------------------------------------------------------------------------- - * nmakehlp.c -- - * - * This is used to fix limitations within nmake and the environment. - * - * Copyright (c) 2002 by David Gravereaux. - * Copyright (c) 2006 by Pat Thoyts - * - * See the file "license.terms" for information on usage and redistribution of - * this file, and for a DISCLAIMER OF ALL WARRANTIES. - * ---------------------------------------------------------------------------- - */ - -#define _CRT_SECURE_NO_DEPRECATE -#include -#ifdef _MSC_VER -#pragma comment (lib, "user32.lib") -#pragma comment (lib, "kernel32.lib") -#endif -#include -#include - -/* - * This library is required for x64 builds with _some_ versions of MSVC - */ -#if defined(_M_IA64) || defined(_M_AMD64) -#if _MSC_VER >= 1400 && _MSC_VER < 1500 -#pragma comment(lib, "bufferoverflowU") -#endif -#endif - -/* ISO hack for dumb VC++ */ -#ifdef _MSC_VER -#define snprintf _snprintf -#endif - - -/* protos */ - -static int CheckForCompilerFeature(const char *option); -static int CheckForLinkerFeature(char **options, int count); -static int IsIn(const char *string, const char *substring); -static int SubstituteFile(const char *substs, const char *filename); -static int QualifyPath(const char *path); -static int LocateDependency(const char *keyfile); -static const char *GetVersionFromFile(const char *filename, const char *match, int numdots); -static DWORD WINAPI ReadFromPipe(LPVOID args); - -/* globals */ - -#define CHUNK 25 -#define STATICBUFFERSIZE 1000 -typedef struct { - HANDLE pipe; - char buffer[STATICBUFFERSIZE]; -} pipeinfo; - -pipeinfo Out = {INVALID_HANDLE_VALUE, ""}; -pipeinfo Err = {INVALID_HANDLE_VALUE, ""}; - -/* - * exitcodes: 0 == no, 1 == yes, 2 == error - */ - -int -main( - int argc, - char *argv[]) -{ - char msg[300]; - DWORD dwWritten; - int chars; - const char *s; - - /* - * Make sure children (cl.exe and link.exe) are kept quiet. - */ - - SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); - - /* - * Make sure the compiler and linker aren't effected by the outside world. - */ - - SetEnvironmentVariable("CL", ""); - SetEnvironmentVariable("LINK", ""); - - if (argc > 1 && *argv[1] == '-') { - switch (*(argv[1]+1)) { - case 'c': - if (argc != 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -c \n" - "Tests for whether cl.exe supports an option\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return CheckForCompilerFeature(argv[2]); - case 'l': - if (argc < 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -l ? ...?\n" - "Tests for whether link.exe supports an option\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return CheckForLinkerFeature(&argv[2], argc-2); - case 'f': - if (argc == 2) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -f \n" - "Find a substring within another\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } else if (argc == 3) { - /* - * If the string is blank, there is no match. - */ - - return 0; - } else { - return IsIn(argv[2], argv[3]); - } - case 's': - if (argc == 2) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -s \n" - "Perform a set of string map type substutitions on a file\n" - "exitcodes: 0\n", - argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return SubstituteFile(argv[2], argv[3]); - case 'V': - if (argc != 4) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -V filename matchstring\n" - "Extract a version from a file:\n" - "eg: pkgIndex.tcl \"package ifneeded http\"", - argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 0; - } - s = GetVersionFromFile(argv[2], argv[3], *(argv[1]+2) - '0'); - if (s && *s) { - printf("%s\n", s); - return 0; - } else - return 1; /* Version not found. Return non-0 exit code */ - - case 'Q': - if (argc != 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -Q path\n" - "Emit the fully qualified path\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return QualifyPath(argv[2]); - - case 'L': - if (argc != 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -L keypath\n" - "Emit the fully qualified path of directory containing keypath\n" - "exitcodes: 0 == success, 1 == not found, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return LocateDependency(argv[2]); - } - } - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -c|-f|-l|-Q|-s|-V ...\n" - "This is a little helper app to equalize shell differences between WinNT and\n" - "Win9x and get nmake.exe to accomplish its job.\n", - argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, &dwWritten, NULL); - return 2; -} - -static int -CheckForCompilerFeature( - const char *option) -{ - STARTUPINFO si; - PROCESS_INFORMATION pi; - SECURITY_ATTRIBUTES sa; - DWORD threadID; - char msg[300]; - BOOL ok; - HANDLE hProcess, h, pipeThreads[2]; - char cmdline[100]; - - hProcess = GetCurrentProcess(); - - ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); - ZeroMemory(&si, sizeof(STARTUPINFO)); - si.cb = sizeof(STARTUPINFO); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = INVALID_HANDLE_VALUE; - - ZeroMemory(&sa, sizeof(SECURITY_ATTRIBUTES)); - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = FALSE; - - /* - * Create a non-inheritible pipe. - */ - - CreatePipe(&Out.pipe, &h, &sa, 0); - - /* - * Dupe the write side, make it inheritible, and close the original. - */ - - DuplicateHandle(hProcess, h, hProcess, &si.hStdOutput, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Same as above, but for the error side. - */ - - CreatePipe(&Err.pipe, &h, &sa, 0); - DuplicateHandle(hProcess, h, hProcess, &si.hStdError, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Base command line. - */ - - lstrcpy(cmdline, "cl.exe -nologo -c -TC -Zs -X -Fp.\\_junk.pch "); - - /* - * Append our option for testing - */ - - lstrcat(cmdline, option); - - /* - * Filename to compile, which exists, but is nothing and empty. - */ - - lstrcat(cmdline, " .\\nul"); - - ok = CreateProcess( - NULL, /* Module name. */ - cmdline, /* Command line. */ - NULL, /* Process handle not inheritable. */ - NULL, /* Thread handle not inheritable. */ - TRUE, /* yes, inherit handles. */ - DETACHED_PROCESS, /* No console for you. */ - NULL, /* Use parent's environment block. */ - NULL, /* Use parent's starting directory. */ - &si, /* Pointer to STARTUPINFO structure. */ - &pi); /* Pointer to PROCESS_INFORMATION structure. */ - - if (!ok) { - DWORD err = GetLastError(); - int chars = snprintf(msg, sizeof(msg) - 1, - "Tried to launch: \"%s\", but got error [%u]: ", cmdline, err); - - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS| - FORMAT_MESSAGE_MAX_WIDTH_MASK, 0L, err, 0, (LPSTR)&msg[chars], - (300-chars), 0); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, lstrlen(msg), &err,NULL); - return 2; - } - - /* - * Close our references to the write handles that have now been inherited. - */ - - CloseHandle(si.hStdOutput); - CloseHandle(si.hStdError); - - WaitForInputIdle(pi.hProcess, 5000); - CloseHandle(pi.hThread); - - /* - * Start the pipe reader threads. - */ - - pipeThreads[0] = CreateThread(NULL, 0, ReadFromPipe, &Out, 0, &threadID); - pipeThreads[1] = CreateThread(NULL, 0, ReadFromPipe, &Err, 0, &threadID); - - /* - * Block waiting for the process to end. - */ - - WaitForSingleObject(pi.hProcess, INFINITE); - CloseHandle(pi.hProcess); - - /* - * Wait for our pipe to get done reading, should it be a little slow. - */ - - WaitForMultipleObjects(2, pipeThreads, TRUE, 500); - CloseHandle(pipeThreads[0]); - CloseHandle(pipeThreads[1]); - - /* - * Look for the commandline warning code in both streams. - * - in MSVC 6 & 7 we get D4002, in MSVC 8 we get D9002. - */ - - return !(strstr(Out.buffer, "D4002") != NULL - || strstr(Err.buffer, "D4002") != NULL - || strstr(Out.buffer, "D9002") != NULL - || strstr(Err.buffer, "D9002") != NULL - || strstr(Out.buffer, "D2021") != NULL - || strstr(Err.buffer, "D2021") != NULL); -} - -static int -CheckForLinkerFeature( - char **options, - int count) -{ - STARTUPINFO si; - PROCESS_INFORMATION pi; - SECURITY_ATTRIBUTES sa; - DWORD threadID; - char msg[300]; - BOOL ok; - HANDLE hProcess, h, pipeThreads[2]; - int i; - char cmdline[255]; - - hProcess = GetCurrentProcess(); - - ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); - ZeroMemory(&si, sizeof(STARTUPINFO)); - si.cb = sizeof(STARTUPINFO); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = INVALID_HANDLE_VALUE; - - ZeroMemory(&sa, sizeof(SECURITY_ATTRIBUTES)); - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = TRUE; - - /* - * Create a non-inheritible pipe. - */ - - CreatePipe(&Out.pipe, &h, &sa, 0); - - /* - * Dupe the write side, make it inheritible, and close the original. - */ - - DuplicateHandle(hProcess, h, hProcess, &si.hStdOutput, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Same as above, but for the error side. - */ - - CreatePipe(&Err.pipe, &h, &sa, 0); - DuplicateHandle(hProcess, h, hProcess, &si.hStdError, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Base command line. - */ - - lstrcpy(cmdline, "link.exe -nologo "); - - /* - * Append our option for testing. - */ - - for (i = 0; i < count; i++) { - lstrcat(cmdline, " \""); - lstrcat(cmdline, options[i]); - lstrcat(cmdline, "\""); - } - - ok = CreateProcess( - NULL, /* Module name. */ - cmdline, /* Command line. */ - NULL, /* Process handle not inheritable. */ - NULL, /* Thread handle not inheritable. */ - TRUE, /* yes, inherit handles. */ - DETACHED_PROCESS, /* No console for you. */ - NULL, /* Use parent's environment block. */ - NULL, /* Use parent's starting directory. */ - &si, /* Pointer to STARTUPINFO structure. */ - &pi); /* Pointer to PROCESS_INFORMATION structure. */ - - if (!ok) { - DWORD err = GetLastError(); - int chars = snprintf(msg, sizeof(msg) - 1, - "Tried to launch: \"%s\", but got error [%u]: ", cmdline, err); - - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS| - FORMAT_MESSAGE_MAX_WIDTH_MASK, 0L, err, 0, (LPSTR)&msg[chars], - (300-chars), 0); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, lstrlen(msg), &err,NULL); - return 2; - } - - /* - * Close our references to the write handles that have now been inherited. - */ - - CloseHandle(si.hStdOutput); - CloseHandle(si.hStdError); - - WaitForInputIdle(pi.hProcess, 5000); - CloseHandle(pi.hThread); - - /* - * Start the pipe reader threads. - */ - - pipeThreads[0] = CreateThread(NULL, 0, ReadFromPipe, &Out, 0, &threadID); - pipeThreads[1] = CreateThread(NULL, 0, ReadFromPipe, &Err, 0, &threadID); - - /* - * Block waiting for the process to end. - */ - - WaitForSingleObject(pi.hProcess, INFINITE); - CloseHandle(pi.hProcess); - - /* - * Wait for our pipe to get done reading, should it be a little slow. - */ - - WaitForMultipleObjects(2, pipeThreads, TRUE, 500); - CloseHandle(pipeThreads[0]); - CloseHandle(pipeThreads[1]); - - /* - * Look for the commandline warning code in the stderr stream. - */ - - return !(strstr(Out.buffer, "LNK1117") != NULL || - strstr(Err.buffer, "LNK1117") != NULL || - strstr(Out.buffer, "LNK4044") != NULL || - strstr(Err.buffer, "LNK4044") != NULL || - strstr(Out.buffer, "LNK4224") != NULL || - strstr(Err.buffer, "LNK4224") != NULL); -} - -static DWORD WINAPI -ReadFromPipe( - LPVOID args) -{ - pipeinfo *pi = (pipeinfo *) args; - char *lastBuf = pi->buffer; - DWORD dwRead; - BOOL ok; - - again: - if (lastBuf - pi->buffer + CHUNK > STATICBUFFERSIZE) { - CloseHandle(pi->pipe); - return (DWORD)-1; - } - ok = ReadFile(pi->pipe, lastBuf, CHUNK, &dwRead, 0L); - if (!ok || dwRead == 0) { - CloseHandle(pi->pipe); - return 0; - } - lastBuf += dwRead; - goto again; - - return 0; /* makes the compiler happy */ -} - -static int -IsIn( - const char *string, - const char *substring) -{ - return (strstr(string, substring) != NULL); -} - -/* - * GetVersionFromFile -- - * Looks for a match string in a file and then returns the version - * following the match where a version is anything acceptable to - * package provide or package ifneeded. - */ - -static const char * -GetVersionFromFile( - const char *filename, - const char *match, - int numdots) -{ - static char szBuffer[100]; - char *szResult = NULL; - FILE *fp = fopen(filename, "rt"); - - if (fp != NULL) { - /* - * Read data until we see our match string. - */ - - while (fgets(szBuffer, sizeof(szBuffer), fp) != NULL) { - LPSTR p, q; - - p = strstr(szBuffer, match); - if (p != NULL) { - /* - * Skip to first digit after the match. - */ - - p += strlen(match); - while (*p && !isdigit((unsigned char)*p)) { - ++p; - } - - /* - * Find ending whitespace. - */ - - q = p; - while (*q && (strchr("0123456789.ab", *q)) && (((!strchr(".ab", *q) - && !strchr("ab", q[-1])) || --numdots))) { - ++q; - } - - *q = 0; - szResult = p; - break; - } - } - fclose(fp); - } - return szResult; -} - -/* - * List helpers for the SubstituteFile function - */ - -typedef struct list_item_t { - struct list_item_t *nextPtr; - char * key; - char * value; -} list_item_t; - -/* insert a list item into the list (list may be null) */ -static list_item_t * -list_insert(list_item_t **listPtrPtr, const char *key, const char *value) -{ - list_item_t *itemPtr = (list_item_t *)malloc(sizeof(list_item_t)); - if (itemPtr) { - itemPtr->key = strdup(key); - itemPtr->value = strdup(value); - itemPtr->nextPtr = NULL; - - while(*listPtrPtr) { - listPtrPtr = &(*listPtrPtr)->nextPtr; - } - *listPtrPtr = itemPtr; - } - return itemPtr; -} - -static void -list_free(list_item_t **listPtrPtr) -{ - list_item_t *tmpPtr, *listPtr = *listPtrPtr; - while (listPtr) { - tmpPtr = listPtr; - listPtr = listPtr->nextPtr; - free(tmpPtr->key); - free(tmpPtr->value); - free(tmpPtr); - } -} - -/* - * SubstituteFile -- - * As windows doesn't provide anything useful like sed and it's unreliable - * to use the tclsh you are building against (consider x-platform builds - - * eg compiling AMD64 target from IX86) we provide a simple substitution - * option here to handle autoconf style substitutions. - * The substitution file is whitespace and line delimited. The file should - * consist of lines matching the regular expression: - * \s*\S+\s+\S*$ - * - * Usage is something like: - * nmakehlp -S << $** > $@ - * @PACKAGE_NAME@ $(PACKAGE_NAME) - * @PACKAGE_VERSION@ $(PACKAGE_VERSION) - * << - */ - -static int -SubstituteFile( - const char *substitutions, - const char *filename) -{ - static char szBuffer[1024], szCopy[1024]; - list_item_t *substPtr = NULL; - FILE *fp, *sp; - - fp = fopen(filename, "rt"); - if (fp != NULL) { - - /* - * Build a list of substutitions from the first filename - */ - - sp = fopen(substitutions, "rt"); - if (sp != NULL) { - while (fgets(szBuffer, sizeof(szBuffer), sp) != NULL) { - unsigned char *ks, *ke, *vs, *ve; - ks = (unsigned char*)szBuffer; - while (ks && *ks && isspace(*ks)) ++ks; - ke = ks; - while (ke && *ke && !isspace(*ke)) ++ke; - vs = ke; - while (vs && *vs && isspace(*vs)) ++vs; - ve = vs; - while (ve && *ve && !(*ve == '\r' || *ve == '\n')) ++ve; - *ke = 0, *ve = 0; - list_insert(&substPtr, (char*)ks, (char*)vs); - } - fclose(sp); - } - - /* debug: dump the list */ -#ifndef NDEBUG - { - int n = 0; - list_item_t *p = NULL; - for (p = substPtr; p != NULL; p = p->nextPtr, ++n) { - fprintf(stderr, "% 3d '%s' => '%s'\n", n, p->key, p->value); - } - } -#endif - - /* - * Run the substitutions over each line of the input - */ - - while (fgets(szBuffer, sizeof(szBuffer), fp) != NULL) { - list_item_t *p = NULL; - for (p = substPtr; p != NULL; p = p->nextPtr) { - char *m = strstr(szBuffer, p->key); - if (m) { - char *cp, *op, *sp; - cp = szCopy; - op = szBuffer; - while (op != m) *cp++ = *op++; - sp = p->value; - while (sp && *sp) *cp++ = *sp++; - op += strlen(p->key); - while (*op) *cp++ = *op++; - *cp = 0; - memcpy(szBuffer, szCopy, sizeof(szCopy)); - } - } - printf("%s", szBuffer); - } - - list_free(&substPtr); - } - fclose(fp); - return 0; -} - -BOOL FileExists(LPCTSTR szPath) -{ -#ifndef INVALID_FILE_ATTRIBUTES - #define INVALID_FILE_ATTRIBUTES ((DWORD)-1) -#endif - DWORD pathAttr = GetFileAttributes(szPath); - return (pathAttr != INVALID_FILE_ATTRIBUTES && - !(pathAttr & FILE_ATTRIBUTE_DIRECTORY)); -} - - -/* - * QualifyPath -- - * - * This composes the current working directory with a provided path - * and returns the fully qualified and normalized path. - * Mostly needed to setup paths for testing. - */ - -static int -QualifyPath( - const char *szPath) -{ - char szCwd[MAX_PATH + 1]; - - GetFullPathName(szPath, sizeof(szCwd)-1, szCwd, NULL); - printf("%s\n", szCwd); - return 0; -} - -/* - * Implements LocateDependency for a single directory. See that command - * for an explanation. - * Returns 0 if found after printing the directory. - * Returns 1 if not found but no errors. - * Returns 2 on any kind of error - * Basically, these are used as exit codes for the process. - */ -static int LocateDependencyHelper(const char *dir, const char *keypath) -{ - HANDLE hSearch; - char path[MAX_PATH+1]; - size_t dirlen; - int keylen, ret; - WIN32_FIND_DATA finfo; - - if (dir == NULL || keypath == NULL) - return 2; /* Have no real error reporting mechanism into nmake */ - dirlen = strlen(dir); - if ((dirlen + 3) > sizeof(path)) - return 2; - strncpy(path, dir, dirlen); - strncpy(path+dirlen, "\\*", 3); /* Including terminating \0 */ - keylen = strlen(keypath); - -#if 0 /* This function is not available in Visual C++ 6 */ - /* - * Use numerics 0 -> FindExInfoStandard, - * 1 -> FindExSearchLimitToDirectories, - * as these are not defined in Visual C++ 6 - */ - hSearch = FindFirstFileEx(path, 0, &finfo, 1, NULL, 0); -#else - hSearch = FindFirstFile(path, &finfo); -#endif - if (hSearch == INVALID_HANDLE_VALUE) - return 1; /* Not found */ - - /* Loop through all subdirs checking if the keypath is under there */ - ret = 1; /* Assume not found */ - do { - int sublen; - /* - * We need to check it is a directory despite the - * FindExSearchLimitToDirectories in the above call. See SDK docs - */ - if ((finfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) - continue; - sublen = strlen(finfo.cFileName); - if ((dirlen+1+sublen+1+keylen+1) > sizeof(path)) - continue; /* Path does not fit, assume not matched */ - strncpy(path+dirlen+1, finfo.cFileName, sublen); - path[dirlen+1+sublen] = '\\'; - strncpy(path+dirlen+1+sublen+1, keypath, keylen+1); - if (FileExists(path)) { - /* Found a match, print to stdout */ - path[dirlen+1+sublen] = '\0'; - QualifyPath(path); - ret = 0; - break; - } - } while (FindNextFile(hSearch, &finfo)); - FindClose(hSearch); - return ret; -} - -/* - * LocateDependency -- - * - * Locates a dependency for a package. - * keypath - a relative path within the package directory - * that is used to confirm it is the correct directory. - * The search path for the package directory is currently only - * the parent and grandparent of the current working directory. - * If found, the command prints - * name_DIRPATH= - * and returns 0. If not found, does not print anything and returns 1. - */ -static int LocateDependency(const char *keypath) -{ - size_t i; - int ret; - static const char *paths[] = {"..", "..\\..", "..\\..\\.."}; - - for (i = 0; i < (sizeof(paths)/sizeof(paths[0])); ++i) { - ret = LocateDependencyHelper(paths[i], keypath); - if (ret == 0) - return ret; - } - return ret; -} - - -/* - * Local variables: - * mode: c - * c-basic-offset: 4 - * fill-column: 78 - * indent-tabs-mode: t - * tab-width: 8 - * End: - */ diff --git a/autoconf/tea/win/rules.vc b/autoconf/tea/win/rules.vc deleted file mode 100644 index 99471053c8..0000000000 --- a/autoconf/tea/win/rules.vc +++ /dev/null @@ -1,711 +0,0 @@ -#------------------------------------------------------------------------------ -# rules.vc -- -# -# Microsoft Visual C++ makefile include for decoding the commandline -# macros. This file does not need editing to build Tcl. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# -# Copyright (c) 2001-2003 David Gravereaux. -# Copyright (c) 2003-2008 Patrick Thoyts -#------------------------------------------------------------------------------ - -!ifndef _RULES_VC -_RULES_VC = 1 - -cc32 = $(CC) # built-in default. -link32 = link -lib32 = lib -rc32 = $(RC) # built-in default. - -!ifndef INSTALLDIR -### Assume the normal default. -_INSTALLDIR = C:\Program Files\Tcl -!else -### Fix the path separators. -_INSTALLDIR = $(INSTALLDIR:/=\) -!endif - -#---------------------------------------------------------- -# Set the proper copy method to avoid overwrite questions -# to the user when copying files and selecting the right -# "delete all" method. -#---------------------------------------------------------- - -!if "$(OS)" == "Windows_NT" -RMDIR = rmdir /S /Q -ERRNULL = 2>NUL -!if ![ver | find "4.0" > nul] -CPY = echo y | xcopy /i >NUL -COPY = copy >NUL -!else -CPY = xcopy /i /y >NUL -COPY = copy /y >NUL -!endif -!else # "$(OS)" != "Windows_NT" -CPY = xcopy /i >_JUNK.OUT # On Win98 NUL does not work here. -COPY = copy >_JUNK.OUT # On Win98 NUL does not work here. -RMDIR = deltree /Y -NULL = \NUL # Used in testing directory existence -ERRNULL = >NUL # Win9x shell cannot redirect stderr -!endif -MKDIR = mkdir - -#------------------------------------------------------------------------------ -# Determine the host and target architectures and compiler version. -#------------------------------------------------------------------------------ - -_HASH=^# -_VC_MANIFEST_EMBED_EXE= -_VC_MANIFEST_EMBED_DLL= -VCVER=0 -!if ![echo VCVERSION=_MSC_VER > vercl.x] \ - && ![echo $(_HASH)if defined(_M_IX86) >> vercl.x] \ - && ![echo ARCH=IX86 >> vercl.x] \ - && ![echo $(_HASH)elif defined(_M_AMD64) >> vercl.x] \ - && ![echo ARCH=AMD64 >> vercl.x] \ - && ![echo $(_HASH)endif >> vercl.x] \ - && ![cl -nologo -TC -P vercl.x $(ERRNULL)] -!include vercl.i -!if ![echo VCVER= ^\> vercl.vc] \ - && ![set /a $(VCVERSION) / 100 - 6 >> vercl.vc] -!include vercl.vc -!endif -!endif -!if ![del $(ERRNUL) /q/f vercl.x vercl.i vercl.vc] -!endif - -!if ![reg query HKLM\Hardware\Description\System\CentralProcessor\0 /v Identifier | findstr /i x86] -NATIVE_ARCH=IX86 -!else -NATIVE_ARCH=AMD64 -!endif - -# Since MSVC8 we must deal with manifest resources. -!if $(VCVERSION) >= 1400 -_VC_MANIFEST_EMBED_EXE=if exist $@.manifest mt -nologo -manifest $@.manifest -outputresource:$@;1 -_VC_MANIFEST_EMBED_DLL=if exist $@.manifest mt -nologo -manifest $@.manifest -outputresource:$@;2 -!endif - -!ifndef MACHINE -MACHINE=$(ARCH) -!endif - -!ifndef CFG_ENCODING -CFG_ENCODING = \"cp1252\" -!endif - -!message =============================================================================== - -#---------------------------------------------------------- -# build the helper app we need to overcome nmake's limiting -# environment. -#---------------------------------------------------------- - -!if !exist(nmakehlp.exe) -!if [$(cc32) -nologo nmakehlp.c -link -subsystem:console > nul] -!endif -!endif - -#---------------------------------------------------------- -# Test for compiler features -#---------------------------------------------------------- - -### test for optimizations -!if [nmakehlp -c -Ot] -!message *** Compiler has 'Optimizations' -OPTIMIZING = 1 -!else -!message *** Compiler does not have 'Optimizations' -OPTIMIZING = 0 -!endif - -OPTIMIZATIONS = - -!if [nmakehlp -c -Ot] -OPTIMIZATIONS = $(OPTIMIZATIONS) -Ot -!endif - -!if [nmakehlp -c -Oi] -OPTIMIZATIONS = $(OPTIMIZATIONS) -Oi -!endif - -!if [nmakehlp -c -Op] -OPTIMIZATIONS = $(OPTIMIZATIONS) -Op -!endif - -!if [nmakehlp -c -fp:strict] -OPTIMIZATIONS = $(OPTIMIZATIONS) -fp:strict -!endif - -!if [nmakehlp -c -Gs] -OPTIMIZATIONS = $(OPTIMIZATIONS) -Gs -!endif - -!if [nmakehlp -c -GS] -OPTIMIZATIONS = $(OPTIMIZATIONS) -GS -!endif - -!if [nmakehlp -c -GL] -OPTIMIZATIONS = $(OPTIMIZATIONS) -GL -!endif - -DEBUGFLAGS = - -!if [nmakehlp -c -RTC1] -DEBUGFLAGS = $(DEBUGFLAGS) -RTC1 -!elseif [nmakehlp -c -GZ] -DEBUGFLAGS = $(DEBUGFLAGS) -GZ -!endif - -COMPILERFLAGS =-W3 -DUNICODE -D_UNICODE - -# In v13 -GL and -YX are incompatible. -!if [nmakehlp -c -YX] -!if ![nmakehlp -c -GL] -OPTIMIZATIONS = $(OPTIMIZATIONS) -YX -!endif -!endif - -!if "$(MACHINE)" == "IX86" -### test for pentium errata -!if [nmakehlp -c -QI0f] -!message *** Compiler has 'Pentium 0x0f fix' -COMPILERFLAGS = $(COMPILERFLAGS) -QI0f -!else -!message *** Compiler does not have 'Pentium 0x0f fix' -!endif -!endif - -!if "$(MACHINE)" == "IA64" -### test for Itanium errata -!if [nmakehlp -c -QIA64_Bx] -!message *** Compiler has 'B-stepping errata workarounds' -COMPILERFLAGS = $(COMPILERFLAGS) -QIA64_Bx -!else -!message *** Compiler does not have 'B-stepping errata workarounds' -!endif -!endif - -!if "$(MACHINE)" == "IX86" -### test for -align:4096, when align:512 will do. -!if [nmakehlp -l -opt:nowin98] -!message *** Linker has 'Win98 alignment problem' -ALIGN98_HACK = 1 -!else -!message *** Linker does not have 'Win98 alignment problem' -ALIGN98_HACK = 0 -!endif -!else -ALIGN98_HACK = 0 -!endif - -LINKERFLAGS = - -!if [nmakehlp -l -ltcg] -LINKERFLAGS =-ltcg -!endif - -#---------------------------------------------------------- -# Decode the options requested. -#---------------------------------------------------------- - -!if "$(OPTS)" == "" || [nmakehlp -f "$(OPTS)" "none"] -STATIC_BUILD = 0 -TCL_THREADS = 1 -DEBUG = 0 -SYMBOLS = 0 -PROFILE = 0 -PGO = 0 -MSVCRT = 0 -LOIMPACT = 0 -TCL_USE_STATIC_PACKAGES = 0 -USE_THREAD_ALLOC = 1 -UNCHECKED = 0 -!else -!if [nmakehlp -f $(OPTS) "static"] -!message *** Doing static -STATIC_BUILD = 1 -!else -STATIC_BUILD = 0 -!endif -!if [nmakehlp -f $(OPTS) "msvcrt"] -!message *** Doing msvcrt -MSVCRT = 1 -!else -MSVCRT = 0 -!endif -!if [nmakehlp -f $(OPTS) "staticpkg"] -!message *** Doing staticpkg -TCL_USE_STATIC_PACKAGES = 1 -!else -TCL_USE_STATIC_PACKAGES = 0 -!endif -!if [nmakehlp -f $(OPTS) "nothreads"] -!message *** Compile explicitly for non-threaded tcl -TCL_THREADS = 0 -!else -TCL_THREADS = 1 -USE_THREAD_ALLOC= 1 -!endif -!if [nmakehlp -f $(OPTS) "symbols"] -!message *** Doing symbols -DEBUG = 1 -!else -DEBUG = 0 -!endif -!if [nmakehlp -f $(OPTS) "pdbs"] -!message *** Doing pdbs -SYMBOLS = 1 -!else -SYMBOLS = 0 -!endif -!if [nmakehlp -f $(OPTS) "profile"] -!message *** Doing profile -PROFILE = 1 -!else -PROFILE = 0 -!endif -!if [nmakehlp -f $(OPTS) "pgi"] -!message *** Doing profile guided optimization instrumentation -PGO = 1 -!elseif [nmakehlp -f $(OPTS) "pgo"] -!message *** Doing profile guided optimization -PGO = 2 -!else -PGO = 0 -!endif -!if [nmakehlp -f $(OPTS) "loimpact"] -!message *** Doing loimpact -LOIMPACT = 1 -!else -LOIMPACT = 0 -!endif -!if [nmakehlp -f $(OPTS) "thrdalloc"] -!message *** Doing thrdalloc -USE_THREAD_ALLOC = 1 -!endif -!if [nmakehlp -f $(OPTS) "tclalloc"] -!message *** Doing tclalloc -USE_THREAD_ALLOC = 0 -!endif -!if [nmakehlp -f $(OPTS) "unchecked"] -!message *** Doing unchecked -UNCHECKED = 1 -!else -UNCHECKED = 0 -!endif -!endif - - -!if !$(STATIC_BUILD) -# Make sure we don't build overly fat DLLs. -MSVCRT = 1 -# We shouldn't statically put the extensions inside the shell when dynamic. -TCL_USE_STATIC_PACKAGES = 0 -!endif - - -#---------------------------------------------------------- -# Figure-out how to name our intermediate and output directories. -# We wouldn't want different builds to use the same .obj files -# by accident. -#---------------------------------------------------------- - -#---------------------------------------- -# Naming convention: -# t = full thread support. -# s = static library (as opposed to an -# import library) -# g = linked to the debug enabled C -# run-time. -# x = special static build when it -# links to the dynamic C run-time. -#---------------------------------------- -SUFX = tsgx - -!if $(DEBUG) -BUILDDIRTOP = Debug -!else -BUILDDIRTOP = Release -!endif - -!if "$(MACHINE)" != "IX86" -BUILDDIRTOP =$(BUILDDIRTOP)_$(MACHINE) -!endif -!if $(VCVER) > 6 -BUILDDIRTOP =$(BUILDDIRTOP)_VC$(VCVER) -!endif - -!if !$(DEBUG) || $(DEBUG) && $(UNCHECKED) -SUFX = $(SUFX:g=) -!endif - -TMP_DIRFULL = .\$(BUILDDIRTOP)\$(PROJECT)_ThreadedDynamicStaticX - -!if !$(STATIC_BUILD) -TMP_DIRFULL = $(TMP_DIRFULL:Static=) -SUFX = $(SUFX:s=) -EXT = dll -!if $(MSVCRT) -TMP_DIRFULL = $(TMP_DIRFULL:X=) -SUFX = $(SUFX:x=) -!endif -!else -TMP_DIRFULL = $(TMP_DIRFULL:Dynamic=) -EXT = lib -!if !$(MSVCRT) -TMP_DIRFULL = $(TMP_DIRFULL:X=) -SUFX = $(SUFX:x=) -!endif -!endif - -!if !$(TCL_THREADS) -TMP_DIRFULL = $(TMP_DIRFULL:Threaded=) -SUFX = $(SUFX:t=) -!endif - -!ifndef TMP_DIR -TMP_DIR = $(TMP_DIRFULL) -!ifndef OUT_DIR -OUT_DIR = .\$(BUILDDIRTOP) -!endif -!else -!ifndef OUT_DIR -OUT_DIR = $(TMP_DIR) -!endif -!endif - - -#---------------------------------------------------------- -# Decode the statistics requested. -#---------------------------------------------------------- - -!if "$(STATS)" == "" || [nmakehlp -f "$(STATS)" "none"] -TCL_MEM_DEBUG = 0 -TCL_COMPILE_DEBUG = 0 -!else -!if [nmakehlp -f $(STATS) "memdbg"] -!message *** Doing memdbg -TCL_MEM_DEBUG = 1 -!else -TCL_MEM_DEBUG = 0 -!endif -!if [nmakehlp -f $(STATS) "compdbg"] -!message *** Doing compdbg -TCL_COMPILE_DEBUG = 1 -!else -TCL_COMPILE_DEBUG = 0 -!endif -!endif - - -#---------------------------------------------------------- -# Decode the checks requested. -#---------------------------------------------------------- - -!if "$(CHECKS)" == "" || [nmakehlp -f "$(CHECKS)" "none"] -TCL_NO_DEPRECATED = 0 -WARNINGS = -W3 -!else -!if [nmakehlp -f $(CHECKS) "nodep"] -!message *** Doing nodep check -TCL_NO_DEPRECATED = 1 -!else -TCL_NO_DEPRECATED = 0 -!endif -!if [nmakehlp -f $(CHECKS) "fullwarn"] -!message *** Doing full warnings check -WARNINGS = -W4 -!if [nmakehlp -l -warn:3] -LINKERFLAGS = $(LINKERFLAGS) -warn:3 -!endif -!else -WARNINGS = -W3 -!endif -!if [nmakehlp -f $(CHECKS) "64bit"] && [nmakehlp -c -Wp64] -!message *** Doing 64bit portability warnings -WARNINGS = $(WARNINGS) -Wp64 -!endif -!endif - -!if $(PGO) > 1 -!if [nmakehlp -l -ltcg:pgoptimize] -LINKERFLAGS = $(LINKERFLAGS:-ltcg=) -ltcg:pgoptimize -!else -MSG=^ -This compiler does not support profile guided optimization. -!error $(MSG) -!endif -!elseif $(PGO) > 0 -!if [nmakehlp -l -ltcg:pginstrument] -LINKERFLAGS = $(LINKERFLAGS:-ltcg=) -ltcg:pginstrument -!else -MSG=^ -This compiler does not support profile guided optimization. -!error $(MSG) -!endif -!endif - -#---------------------------------------------------------- -# Set our defines now armed with our options. -#---------------------------------------------------------- - -OPTDEFINES = -DTCL_CFGVAL_ENCODING=$(CFG_ENCODING) -DSTDC_HEADERS - -!if $(TCL_MEM_DEBUG) -OPTDEFINES = $(OPTDEFINES) -DTCL_MEM_DEBUG -!endif -!if $(TCL_COMPILE_DEBUG) -OPTDEFINES = $(OPTDEFINES) -DTCL_COMPILE_DEBUG -DTCL_COMPILE_STATS -!endif -!if $(TCL_THREADS) -OPTDEFINES = $(OPTDEFINES) -DTCL_THREADS=1 -!if $(USE_THREAD_ALLOC) -OPTDEFINES = $(OPTDEFINES) -DUSE_THREAD_ALLOC=1 -!endif -!endif -!if $(STATIC_BUILD) -OPTDEFINES = $(OPTDEFINES) -DSTATIC_BUILD -!endif -!if $(TCL_NO_DEPRECATED) -OPTDEFINES = $(OPTDEFINES) -DTCL_NO_DEPRECATED -!endif - -!if !$(DEBUG) -OPTDEFINES = $(OPTDEFINES) -DNDEBUG -!if $(OPTIMIZING) -OPTDEFINES = $(OPTDEFINES) -DTCL_CFG_OPTIMIZED -!endif -!endif -!if $(PROFILE) -OPTDEFINES = $(OPTDEFINES) -DTCL_CFG_PROFILED -!endif -!if "$(MACHINE)" == "IA64" || "$(MACHINE)" == "AMD64" -OPTDEFINES = $(OPTDEFINES) -DTCL_CFG_DO64BIT -!endif -!if $(VCVERSION) < 1300 -OPTDEFINES = $(OPTDEFINES) -DNO_STRTOI64 -!endif - -#---------------------------------------------------------- -# Locate the Tcl headers to build against -#---------------------------------------------------------- - -!if "$(PROJECT)" == "tcl" - -_TCL_H = ..\generic\tcl.h - -!else - -# If INSTALLDIR set to tcl root dir then reset to the lib dir. -!if exist("$(_INSTALLDIR)\include\tcl.h") -_INSTALLDIR=$(_INSTALLDIR)\lib -!endif - -!if !defined(TCLDIR) -!if exist("$(_INSTALLDIR)\..\include\tcl.h") -TCLINSTALL = 1 -_TCLDIR = $(_INSTALLDIR)\.. -_TCL_H = $(_INSTALLDIR)\..\include\tcl.h -TCLDIR = $(_INSTALLDIR)\.. -!else -MSG=^ -Failed to find tcl.h. Set the TCLDIR macro. -!error $(MSG) -!endif -!else -_TCLDIR = $(TCLDIR:/=\) -!if exist("$(_TCLDIR)\include\tcl.h") -TCLINSTALL = 1 -_TCL_H = $(_TCLDIR)\include\tcl.h -!elseif exist("$(_TCLDIR)\generic\tcl.h") -TCLINSTALL = 0 -_TCL_H = $(_TCLDIR)\generic\tcl.h -!else -MSG =^ -Failed to find tcl.h. The TCLDIR macro does not appear correct. -!error $(MSG) -!endif -!endif -!endif - -#-------------------------------------------------------------- -# Extract various version numbers from tcl headers -# The generated file is then included in the makefile. -#-------------------------------------------------------------- - -!if [echo REM = This file is generated from rules.vc > versions.vc] -!endif -!if [echo TCL_MAJOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" TCL_MAJOR_VERSION >> versions.vc] -!endif -!if [echo TCL_MINOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" TCL_MINOR_VERSION >> versions.vc] -!endif -!if [echo TCL_PATCH_LEVEL = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" TCL_PATCH_LEVEL >> versions.vc] -!endif - -# If building the tcl core then we need additional package versions -!if "$(PROJECT)" == "tcl" -!if [echo PKG_HTTP_VER = \>> versions.vc] \ - && [nmakehlp -V ..\library\http\pkgIndex.tcl http >> versions.vc] -!endif -!if [echo PKG_TCLTEST_VER = \>> versions.vc] \ - && [nmakehlp -V ..\library\tcltest\pkgIndex.tcl tcltest >> versions.vc] -!endif -!if [echo PKG_MSGCAT_VER = \>> versions.vc] \ - && [nmakehlp -V ..\library\msgcat\pkgIndex.tcl msgcat >> versions.vc] -!endif -!if [echo PKG_PLATFORM_VER = \>> versions.vc] \ - && [nmakehlp -V ..\library\platform\pkgIndex.tcl "platform " >> versions.vc] -!endif -!if [echo PKG_SHELL_VER = \>> versions.vc] \ - && [nmakehlp -V ..\library\platform\pkgIndex.tcl "platform::shell" >> versions.vc] -!endif -!if [echo PKG_DDE_VER = \>> versions.vc] \ - && [nmakehlp -V ..\library\dde\pkgIndex.tcl "dde " >> versions.vc] -!endif -!if [echo PKG_REG_VER =\>> versions.vc] \ - && [nmakehlp -V ..\library\reg\pkgIndex.tcl registry >> versions.vc] -!endif -!endif - -!include versions.vc - -#-------------------------------------------------------------- -# Setup tcl version dependent stuff headers -#-------------------------------------------------------------- - -!if "$(PROJECT)" != "tcl" - -TCL_VERSION = $(TCL_MAJOR_VERSION)$(TCL_MINOR_VERSION) - -!if $(TCL_VERSION) < 81 -TCL_DOES_STUBS = 0 -!else -TCL_DOES_STUBS = 1 -!endif - -!if $(TCLINSTALL) -TCLSH = "$(_TCLDIR)\bin\tclsh$(TCL_VERSION)$(SUFX).exe" -!if !exist($(TCLSH)) && $(TCL_THREADS) -TCLSH = "$(_TCLDIR)\bin\tclsh$(TCL_VERSION)t$(SUFX).exe" -!endif -TCLSTUBLIB = "$(_TCLDIR)\lib\tclstub$(TCL_VERSION).lib" -TCLIMPLIB = "$(_TCLDIR)\lib\tcl$(TCL_VERSION)$(SUFX).lib" -TCL_LIBRARY = $(_TCLDIR)\lib -TCLREGLIB = "$(_TCLDIR)\lib\tclreg13$(SUFX:t=).lib" -TCLDDELIB = "$(_TCLDIR)\lib\tcldde14$(SUFX:t=).lib" -COFFBASE = \must\have\tcl\sources\to\build\this\target -TCLTOOLSDIR = \must\have\tcl\sources\to\build\this\target -TCL_INCLUDES = -I"$(_TCLDIR)\include" -!else -TCLSH = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclsh$(TCL_VERSION)$(SUFX).exe" -!if !exist($(TCLSH)) && $(TCL_THREADS) -TCLSH = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclsh$(TCL_VERSION)t$(SUFX).exe" -!endif -TCLSTUBLIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclstub$(TCL_VERSION).lib" -TCLIMPLIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tcl$(TCL_VERSION)$(SUFX).lib" -TCL_LIBRARY = $(_TCLDIR)\library -TCLREGLIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tclreg13$(SUFX:t=).lib" -TCLDDELIB = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tcldde14$(SUFX:t=).lib" -COFFBASE = "$(_TCLDIR)\win\coffbase.txt" -TCLTOOLSDIR = $(_TCLDIR)\tools -TCL_INCLUDES = -I"$(_TCLDIR)\generic" -I"$(_TCLDIR)\win" -!endif - -!endif - -#------------------------------------------------------------------------- -# Locate the Tk headers to build against -#------------------------------------------------------------------------- - -!if "$(PROJECT)" == "tk" -_TK_H = ..\generic\tk.h -_INSTALLDIR = $(_INSTALLDIR)\.. -!endif - -!ifdef PROJECT_REQUIRES_TK -!if !defined(TKDIR) -!if exist("$(_INSTALLDIR)\..\include\tk.h") -TKINSTALL = 1 -_TKDIR = $(_INSTALLDIR)\.. -_TK_H = $(_TKDIR)\include\tk.h -TKDIR = $(_TKDIR) -!elseif exist("$(_TCLDIR)\include\tk.h") -TKINSTALL = 1 -_TKDIR = $(_TCLDIR) -_TK_H = $(_TKDIR)\include\tk.h -TKDIR = $(_TKDIR) -!endif -!else -_TKDIR = $(TKDIR:/=\) -!if exist("$(_TKDIR)\include\tk.h") -TKINSTALL = 1 -_TK_H = $(_TKDIR)\include\tk.h -!elseif exist("$(_TKDIR)\generic\tk.h") -TKINSTALL = 0 -_TK_H = $(_TKDIR)\generic\tk.h -!else -MSG =^ -Failed to find tk.h. The TKDIR macro does not appear correct. -!error $(MSG) -!endif -!endif -!endif - -#------------------------------------------------------------------------- -# Extract Tk version numbers -#------------------------------------------------------------------------- - -!if defined(PROJECT_REQUIRES_TK) || "$(PROJECT)" == "tk" - -!if [echo TK_MAJOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V $(_TK_H) TK_MAJOR_VERSION >> versions.vc] -!endif -!if [echo TK_MINOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V $(_TK_H) TK_MINOR_VERSION >> versions.vc] -!endif -!if [echo TK_PATCH_LEVEL = \>> versions.vc] \ - && [nmakehlp -V $(_TK_H) TK_PATCH_LEVEL >> versions.vc] -!endif - -!include versions.vc - -TK_DOTVERSION = $(TK_MAJOR_VERSION).$(TK_MINOR_VERSION) -TK_VERSION = $(TK_MAJOR_VERSION)$(TK_MINOR_VERSION) - -!if "$(PROJECT)" != "tk" -!if $(TKINSTALL) -WISH = "$(_TKDIR)\bin\wish$(TK_VERSION)$(SUFX).exe" -TKSTUBLIB = "$(_TKDIR)\lib\tkstub$(TK_VERSION).lib" -TKIMPLIB = "$(_TKDIR)\lib\tk$(TK_VERSION)$(SUFX).lib" -TK_INCLUDES = -I"$(_TKDIR)\include" -!else -WISH = "$(_TKDIR)\win\$(BUILDDIRTOP)\wish$(TCL_VERSION)$(SUFX).exe" -TKSTUBLIB = "$(_TKDIR)\win\$(BUILDDIRTOP)\tkstub$(TCL_VERSION).lib" -TKIMPLIB = "$(_TKDIR)\win\$(BUILDDIRTOP)\tk$(TCL_VERSION)$(SUFX).lib" -TK_INCLUDES = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib" -!endif -!endif - -!endif - -#---------------------------------------------------------- -# Display stats being used. -#---------------------------------------------------------- - -!message *** Intermediate directory will be '$(TMP_DIR)' -!message *** Output directory will be '$(OUT_DIR)' -!message *** Suffix for binaries will be '$(SUFX)' -!message *** Optional defines are '$(OPTDEFINES)' -!message *** Compiler version $(VCVER). Target machine is $(MACHINE) -!message *** Host architecture is $(NATIVE_ARCH) -!message *** Compiler options '$(COMPILERFLAGS) $(OPTIMIZATIONS) $(DEBUGFLAGS) $(WARNINGS)' -!message *** Link options '$(LINKERFLAGS)' - -!endif - diff --git a/autosetup/LICENSE b/autosetup/LICENSE new file mode 100644 index 0000000000..4fe636c9d9 --- /dev/null +++ b/autosetup/LICENSE @@ -0,0 +1,35 @@ +Unless explicitly stated, all files which form part of autosetup +are released under the following license: + +--------------------------------------------------------------------- +autosetup - A build environment "autoconfigurator" + +Copyright (c) 2010-2011, WorkWare Systems + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE WORKWARE SYSTEMS ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WORKWARE +SYSTEMS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation +are those of the authors and should not be interpreted as representing +official policies, either expressed or implied, of WorkWare Systems. diff --git a/autosetup/README.autosetup b/autosetup/README.autosetup new file mode 100644 index 0000000000..3952980480 --- /dev/null +++ b/autosetup/README.autosetup @@ -0,0 +1,11 @@ +README.autosetup created by autosetup v0.7.2 + +This is the autosetup directory for a local install of autosetup. +It contains autosetup, support files and loadable modules. + +*.tcl files in this directory are optional modules which +can be loaded with the 'use' directive. + +*.auto files in this directory are auto-loaded. + +For more information, see https://msteveb.github.io/autosetup/ diff --git a/autosetup/README.md b/autosetup/README.md new file mode 100644 index 0000000000..c8da5c643a --- /dev/null +++ b/autosetup/README.md @@ -0,0 +1,454 @@ +Maintaining Autosetup in the SQLite Tree +======================================================================== + +This document provides some tips and reminders for the SQLite +developers regarding using and maintaining the [Autosetup][]-based +build infrastructure. It is not an [Autosetup][] reference. + +**Table of Contents**: + +- [Autosetup API Reference](#apiref) +- [API Tips](#apitips) +- [Ensuring TCL Compatibility](#tclcompat) +- [Design Conventions](#conventions) + - Symbolic Names of Feature Flags + - Do Not Update Global Shared State +- [Updating Autosetup](#updating) + - ***[Patching Autosetup for Project-local changes](#patching)*** +- [Branch-specific Customization](#branch-customization) + + +------------------------------------------------------------------------ + + +Autosetup API Reference +======================================================================== + +The Autosetup API is quite extensive and can be read either in +the [files in the `autosetup` dir](/dir/autosetup) or using: + +> +``` +$ ./configure --reference | less +``` + +That will include any docs from any TCL files in the `./autosetup` dir +which contain certain (simple) markup defined by autosetup. + +This project's own configuration-related TCL code is spread across the +following files: + +- [proj.tcl][]: project-agnostic utility code for autosetup-driven + projects. This file is designed to be shared between this project, + other projects managed under the SQLite/Hwaci umbrella + (e.g. Fossil), and personal projects of SQLite's developers. It is + essentially an amalgamation of a decade's worth of autosetup-related + utility code. +- [sqlite-config.tcl][]: utility code which is too project-specific + for `proj.tcl`. We split this out of `auto.def` so that it can be + used by both `auto.def` and... +- [auto.def][]: the primary driver for the `./configure` process. + When we talk about "the configure script," we're technically + referring to this file, though it actually contains very little + of the TCL code. +- [autoconf/auto.def][]: the main driver script for the "autoconf" + bundle's configure script. It is essentially a slightly trimmed-down + version of the main `auto.def` file. The `autoconf` dir was ported + from the Autotools to Autosetup in the 3.49.0 dev cycle but retains + the "autoconf" name to minimize downstream disruption. + + + +Autosetup API Tips +======================================================================== + +This section briefly covers only APIs which are frequently useful in +day-to-day maintenance and might not be immediately recognized as such +from a casual perusal of the relevant TCL files. The complete docs of +those with `proj-` prefix can be found in [proj.tcl][] and those with +an `sqlite-` prefix are in [sqlite-config.tcl][]. The others are part +of Autosetup's core packages and are scattered around [the TCL files +in ./autosetup](/dir/autosetup). + +In (mostly) alphabetical order: + +- **`file-isexec filename`**\ + Should be used in place of `[file executable]`, as it will also + check for `${filename}.exe` on Windows platforms. However, on such + platforms it also assumes that _any_ existing file is executable. + +- **`get-env VAR ?default?`**\ + Will fetch an "environment variable" from the first of either: (1) a + KEY=VALUE passed to the configure script or (2) the system's + environment variables. Not to be confused with `getenv`, which only + does the latter and is rarely, if ever, useful in this tree. + - **`proj-get-env VAR ?default?`**\ + Works like `get-env` but will, if that function finds no match, + look for a file named `./.env-$VAR` and, if found, return its + trimmed contents. This can be used, e.g., to set a developer's + local preferences for the default `CFLAGS`.\ + Tip: adding `-O0` to `.env-CFLAGS` reduces rebuild times + considerably at the cost of performance in `make devtest` and the + like. + +- **`proj-fatal msg`**\ + Emits `$msg` to stderr and exits with non-zero. Its differences from + autosetup's `user-error` are purely cosmetic. + +- **`proj-if-opt-truthy flag thenScript ?elseScript?`**\ + Evals `thenScript` if the given `--flag` is truthy, else it + evals the optional `elseScript`. + +- **`proj-indented-notice ?-error? ?-notice? msg`**\ + Breaks its `msg` argument into lines, trims them, and emits them + with consistent indentation. Exactly how it emits depends on the + flags passed to it (or not), as covered in its docs. This will stick + out starkly from normal output and is intended to be used only for + important notices. + +- **`proj-opt-truthy flag`**\ + Returns 1 if `--flag`'s value is "truthy," i.e. one of (1, on, + enabled, yes, true). + +- **`proj-opt-was-provided FLAG`**\ + Returns 1 if `--FLAG` was explicitly provided to configure, + else 0. This distinction can be used to determine, e.g., whether + `--with-readline` was provided or whether we're searching for + readline by default. In the former case, failure to find it should + be treated as fatal, where in the latter case it's not.\ + Unlike most functions which deal with `--flags`, this one does not + validate that `$FLAG` is a registered flag so will not fail fatally + if `$FLAG` is not registered as an Autosetup option. + +- **`proj-val-truthy value`**\ + Returns 1 if `$value` is "truthy," See `proj-opt-truthy` for the definition + of "truthy." + +- **`proj-warn msg`**\ + Emits `$msg` to stderr. Closely-related is autosetup's `user-notice` + (described below). + +- **`sqlite-add-feature-flag ?-shell? FLAG...`**\ + Adds the given feature flag to the CFLAGS which are specific to + building libsqlite3. It's intended to be passed one or more + `-DSQLITE_ENABLE_...`, or similar, flags. If the `-shell` flag is + used then it also passes its arguments to + `sqlite-add-shell-opt`. This is a no-op if `FLAG` is not provided or + is empty. + +- **`sqlite-add-shell-opt FLAG...`**\ + The shell-specific counterpart of `sqlite-add-feature-flag` which + only adds the given flag(s) to the CLI-shell-specific CFLAGS. + +- **`sqlite-configure BUILD-NAME {script}`**\ + This is where all configure `--flags` are defined for all known + build modes ("canonical" or "autoconf"). After processing all flags, + this function runs `$script`, which contains the build-mode-specific + configuration bits, and then runs any finalization bits which are + common to all build modes. The `auto.def` files are intended to contain + exactly two commands: + `use sqlite-config; sqlite-configure BUILD-NAME {script}` + +- **`user-notice msg`**\ + Queues `$msg` to be sent to stderr, but does not emit it until + either `show-notices` is called or the next time autosetup would + output something (it internally calls `show-notices`). This can be + used to generate warnings between a "checking for..." message and + its resulting "yes/no/whatever" message in such a way as to not + spoil the layout of such messages. + + + +Ensuring TCL Compatibility +======================================================================== + +One of the significant benefits of using Autosetup is that (A) this +project uses many TCL scripts in the build process and (B) Autosetup +comes with a TCL interpreter named [JimTCL][]. + +It is important that any TCL files used by the configure process and +makefiles remain compatible with both [JimTCL][] and the canonical +TCL. Though JimTCL has outstanding compatibility with canonical TCL, +it does have a few corners with incompatibilities, e.g. regular +expressions. If a script runs in JimTCL without using any +JimTCL-specific features, then it's a certainty that it will run in +canonical TCL as well. The opposite, however, is not _always_ the +case. + +When [`./configure`](/file/configure) is run, it goes through a +bootstrapping process to find a suitable TCL with which to run the +autosetup framework. The first step involves [finding or building a +TCL shell](/file/autosetup/autosetup-find-tclsh). That will first +search for an available `tclsh` (under several common names, +e.g. `tclsh8.6`) before falling back to compiling the copy of +`jimsh0.c` included in the source tree. i.e. it will prefer to use a +system-installed TCL for running the configure script. Once it finds +(or builds) a TCL shell, it then runs [a sanity test to ensure that +the shell is suitable](/file/autosetup/autosetup-test-tclsh) before +using it to run the main autosetup app. + +There are two simple ways to ensure that running of the configure +process uses JimTCL instead of the canonical `tclsh`, and either +approach provides equally high assurances about configure script +compatibility across TCL implementations: + +1. Build on a system with no `tclsh` installed in the `$PATH`. In that + case, the configure process will fall back to building the in-tree + copy of JimTCL. + +2. Manually build `./jimsh0` in the top of the checkout with:\ + `cc -o jimsh0 autosetup/jimsh0.c`\ + With that in place, the configure script will prefer to use that + before looking for a system-level `tclsh`. Be aware, though, that + `make distclean` will remove that file. + +**Note that `./jimsh0` is distinctly different from the `./jimsh`** +which gets built for code-generation purposes. The latter requires +non-default build flags to enable features which are +platform-dependent, most notably to make its `[file normalize]` work. +This means, for example, that the configure script and its utility +APIs must not use `[file normalize]`, but autosetup provides a +TCL-only implementation of `[file-normalize]` (note the dash) for +portable use in the configure script. Contrariwise, code-generation +scripts invoked via `make` may use `[file normalize]`, as they'll use +`./jimsh` or `tclsh` instead of `./jimsh0`. + + +Known TCL Incompatibilities +------------------------------------------------------------------------ + +A summary of known incompatibilities in JimTCL + +- **CRNL line endings**: prior to 2025-02-05 `fconfigure -translation ...` + was a no-op in JimTCL, and it emits CRNL line endings by default on + Windows. Since then, it supports `-translation binary`, which is + close enough to `-translation lf` for our purposes. When working + with files using the `open` command, it is important to use mode + `"rb"` or `"wb"`, as appropriate, so that the output does not get + CRNL-mangled on Windows. + +- **`file copy`** does not support multiple source files. See + [](/info/61f18c96183867fe) for a workaround. + +- **Regular expressions**: + + - Patterns treat `\nnn` octal values as back-references (which it + does not support). Those can be reformulated as demonstrated in + [](/info/aeac23359bb681c0). + + - `regsub` does not support the `\y` flag. A workaround is demonstrated + in [](/info/c2e5dd791cce3ec4). + + + +Design Conventions +======================================================================== + +This section describes the motivations for the most glaring of the +build's design decisions, in particular how they deviate from +historical, or even widely-conventional, practices. + +Symbolic Names of Feature Flags +------------------------------------------------------------------------ + +Historically, the project's makefile has exclusively used +`UPPER_UNDERSCORE` form for makefile variables. This build, however, +primarily uses `X.y` format, where `X` is often a category label, +e.g. `CFLAGS`, and `y` is the specific instance of that category, +e.g. `CFLAGS.readline`. + +When the configure script exports flags for consumption by filtered +files, e.g. [Makefile.in][] and the generated +`sqlite_cfg.h`, it does so in the more conventional `X_Y` form because +those flags get exported as as C `#define`s to `sqlite_cfg.h`, where +dots are not permitted. + +The `X.y` convention is used in the makefiles primarily because the +person who did the initial port finds that considerably easier on the +eyes and fingers. In practice, the `X_Y` form of such exports is used +exactly once in [Makefile.in][], where it's translated from `@X_Y@` +into into `X.y` form for consumption by [Makefile.in][] and +[main.mk][]. For example: + +> +``` +LDFLAGS.shobj = @SHOBJ_LDFLAGS@ +LDFLAGS.zlib = @LDFLAGS_ZLIB@ +LDFLAGS.math = @LDFLAGS_MATH@ +``` + +(That first one is defined by autosetup, and thus applies "LDFLAGS" as +the suffix rather than the prefix. Which is more legible is a matter +of taste, for which there is no accounting.) + + +Do Not Update Global Shared State +------------------------------------------------------------------------ + +In both the legacy Autotools-driven build and common Autosetup usage, +feature tests performed by the configure script may amend global flags +such as `LIBS`, `LDFLAGS`, and `CFLAGS`[^as-cflags]. That's +appropriate for a makefile which builds a single deliverable, but less +so for makefiles which produce multiple deliverables. Drawbacks of +that approach include: + +- It's unlikely that every single deliverable will require the same + core set of those flags. +- It can be difficult to determine the origin of any given change to + that global state because those changes are hidden behind voodoo + performed outside the immediate visibility of the configure script's + maintainer. +- It can force the maintainers of the configure script to place tests + in a specific order so that the resulting flags get applied at + the correct time and/or in the correct order.\ + (A real-life example: before the approach described below was taken + to collecting build-time flags, the test for `-rpath` had to come + _after_ the test for zlib because the results of the `-rpath` test + implicitly modified global state which broke the zlib feature + test. Because the feature tests no longer (intentionally) modify + shared global state, that is not an issue.) + +In this build, cases where feature tests modify global state in such a +way that it may impact later feature tests are either (A) very +intentionally defined to do so (e.g. the `--with-wasi-sdk` flag has +invasive side-effects) or (B) are oversights (i.e. bugs). + +This tree's [configure script][auto.def], [utility APIs][proj.tcl], +[Makefile.in][], and [main.mk][] therefore strive to separate the +results of any given feature test into its own well-defined +variables. For example: + +- The linker flags for zlib are exported from the configure script as + `LDFLAGS_ZLIB`, which [Makefile.in][] and [main.mk][] then expose as + `LDFLAGS.zlib`. +- `CFLAGS_READLINE` (a.k.a. `CFLAGS.readline`) contains the `CFLAGS` + needed for including `libreadline`, `libedit`, or `linenoise`, and + `LDFLAGS_READLINE` (a.k.a. `LDFLAGS.readline`) is its link-time + counterpart. + +It is then up to the Makefile to apply and order the flags however is +appropriate. + +At the end of the configure script, the global `CFLAGS` _ideally_ +holds only flags which are either relevant to all targets or, failing +that, will have no unintended side-effects on any targets. That said: +clients frequently pass custom `CFLAGS` to `./configure` or `make` to +set library-level feature toggles, e.g. `-DSQLITE_OMIT_FOO`, in which +case there is no practical way to avoid "polluting" the builds of +arbitrary makefile targets with those. _C'est la vie._ + + +[^as-cflags]: But see this article for a detailed discussion of how + autosetup currently deals specifically with CFLAGS: + + + + +Updating Autosetup +======================================================================== + +Updating autosetup is, more often than not, painless. It requires having +a checked-out copy of [the autosetup git repository][autosetup-git]: + +> +``` +$ git clone https://github.com/msteveb/autosetup +$ cd autosetup +# Or, if it's already checked out: +$ git pull +``` + +Then, from the top-most directory of an SQLite checkout: + +> +``` +$ /path/to/autosetup-checkout/autosetup --install . +$ fossil status # show the modified files +``` + +Unless the upgrade made any incompatible changes (which is exceedingly +rare), that's all there is to it. After that's done, **apply a patch +for the change described in the following section**, test the +configure process, and check it in. + + +Patching Autosetup for Project-local Changes +------------------------------------------------------------------------ + +Autosetup reserves the flag name **`--debug`** for its own purposes, +and its own special handling of `--enable-...` flags makes `--debug` +an alias for `--enable-debug`. As this project has a long history of +using `--enable-debug`, we patch autosetup to use the name +`--autosetup-debug` in place of `--debug`. That requires (as of this +writing) four small edits in +[/autosetup/autosetup](/file/autosetup/autosetup), as demonstrated in +[check-in 3296c8d3](/info/3296c8d3). + +If autosetup is upgraded and this patch is _not_ applied the invoking +`./configure` will fail loudly because of the declaration of the +`debug` flag in `auto.def` - duplicated flags are not permitted. + + +Branch-specific Customization +======================================================================== + +Certain vendor-specific branches require slight configure script +customization. Rather than editing `sqlite-config.tcl` for this, +which frequently leads to merge conflicts, the following approach +is recommended: + +In the vendor-specific branch, create a file named +`autosetup/sqlite-custom.tcl`. + +That file should contain the following content... + +If flag customization is required, add: + +> +``` +proc sqlite-custom-flags {} { + # If any existing --flags require different default values + # then call: + options-defaults { + flag-name new-default-value + ... + } + # ^^^ That will replace the default value but will not update + # the --help text, which may lead to some confusion: + # https://github.com/msteveb/autosetup/issues/77 + + return { + {*} { + new-flag-name => {Help text} + ... + } + }; #see below +} +``` + +That function must return either an empty string or a list in the form +used internally by [sqlite-config.tcl][]'s `sqlite-configure`. + +Next, define: + +> +``` +proc sqlite-custom-handle-flags {} { + ... do any custom flag handling here ... +} +``` + +That function, if defined, will be called relatively late in the +configure process, before any filtered files are generated but after +all other significant processing. + + +[Autosetup]: https://msteveb.github.io/autosetup/ +[auto.def]: /file/auto.def +[autoconf/auto.def]: /file/autoconf/auto.def +[autosetup-git]: https://github.com/msteveb/autosetup +[proj.tcl]: /file/autosetup/proj.tcl +[sqlite-config.tcl]: /file/autosetup/sqlite-config.tcl +[Makefile.in]: /file/Makefile.in +[main.mk]: /file/main.mk +[JimTCL]: https://jim.tcl.tk diff --git a/autosetup/autosetup b/autosetup/autosetup new file mode 100755 index 0000000000..c3a31bec58 --- /dev/null +++ b/autosetup/autosetup @@ -0,0 +1,2544 @@ +#!/bin/sh +# Copyright (c) 2006-2011 WorkWare Systems http://www.workware.net.au/ +# All rights reserved +# vim:se syntax=tcl: +# \ +dir=`dirname "$0"`; exec "`$dir/autosetup-find-tclsh`" "$0" "$@" + +# Note that the version has a trailing + on unreleased versions +set autosetup(version) 0.7.2 + +# Can be set to 1 to debug early-init problems +set autosetup(debug) [expr {"--autosetup-debug" in $argv}] + +################################################################## +# +# Main flow of control, option handling +# +proc main {argv} { + global autosetup define + + # There are 3 potential directories involved: + # 1. The directory containing autosetup (this script) + # 2. The directory containing auto.def + # 3. The current directory + + # From this we need to determine: + # a. The path to this script (and related support files) + # b. The path to auto.def + # c. The build directory, where output files are created + + # This is also complicated by the fact that autosetup may + # have been run via the configure wrapper ([getenv WRAPPER] is set) + + # Here are the rules. + # a. This script is $::argv0 + # => dir, prog, exe, libdir + # b. auto.def is in the directory containing the configure wrapper, + # otherwise it is in the current directory. + # => srcdir, autodef + # c. The build directory is the current directory + # => builddir, [pwd] + + # 'misc' is needed before we can do anything, so set a temporary libdir + # in case this is the development version + set autosetup(libdir) [file dirname $::argv0]/lib + use misc + + # (a) + set autosetup(dir) [realdir [file dirname [realpath $::argv0]]] + set autosetup(prog) [file join $autosetup(dir) [file tail $::argv0]] + set autosetup(exe) [getenv WRAPPER $autosetup(prog)] + if {$autosetup(installed)} { + set autosetup(libdir) $autosetup(dir) + } else { + set autosetup(libdir) [file join $autosetup(dir) lib] + } + autosetup_add_dep $autosetup(prog) + + # (b) + if {[getenv WRAPPER ""] eq ""} { + # Invoked directly + set autosetup(srcdir) [pwd] + } else { + # Invoked via the configure wrapper + set autosetup(srcdir) [file-normalize [file dirname $autosetup(exe)]] + } + set autosetup(autodef) [relative-path $autosetup(srcdir)/auto.def] + + # (c) + set autosetup(builddir) [pwd] + + set autosetup(argv) $argv + set autosetup(cmdline) {} + # options is a list of known options + set autosetup(options) {} + # optset is a dictionary of option values set by the user based on getopt + set autosetup(optset) {} + # optdefault is a dictionary of default values + set autosetup(optdefault) {} + # options-defaults is a dictionary of overrides for default values for options + set autosetup(options-defaults) {} + set autosetup(optionhelp) {} + set autosetup(showhelp) 0 + + use util + + # Parse options + use getopt + + # At the is point we don't know what is a valid option + # We simply parse anything that looks like an option + set autosetup(getopt) [getopt argv] + + #"=Core Options:" + options-add { + help:=all => "display help and options. Optional: module name, such as --help=system" + licence license => "display the autosetup license" + version => "display the version of autosetup" + ref:=text manual:=text + reference:=text => "display the autosetup command reference. 'text', 'wiki', 'asciidoc' or 'markdown'" + autosetup-debug => "display debugging output as autosetup runs" + install:=. => "install autosetup to the current or given directory" + } + if {$autosetup(installed)} { + # hidden options so we can produce a nice error + options-add { + sysinstall:path + } + } else { + options-add { + sysinstall:path => "install standalone autosetup to the given directory (e.g.: /usr/local)" + } + } + options-add { + force init:=help => "create initial auto.def, etc. Use --init=help for known types" + # Undocumented options + option-checking=1 + nopager + quiet + timing + conf: + } + + if {[opt-bool version]} { + puts $autosetup(version) + exit 0 + } + + # autosetup --conf=alternate-auto.def + if {[opt-str conf o]} { + set autosetup(autodef) $o + } + + # Debugging output (set this early) + incr autosetup(debug) [opt-bool autosetup-debug] + incr autosetup(force) [opt-bool force] + incr autosetup(msg-quiet) [opt-bool quiet] + incr autosetup(msg-timing) [opt-bool timing] + + # If the local module exists, source it now to allow for + # project-local customisations + if {[file exists $autosetup(libdir)/local.tcl]} { + use local + } + + # Now any auto-load modules + autosetup_load_auto_modules + + if {[opt-str help o]} { + incr autosetup(showhelp) + use help + autosetup_help $o + } + + if {[opt-bool licence license]} { + use help + autosetup_show_license + exit 0 + } + + if {[opt-str {manual ref reference} o]} { + use help + autosetup_reference $o + } + + # Allow combining --install and --init + set earlyexit 0 + if {[opt-str install o]} { + use install + autosetup_install $o + incr earlyexit + } + + if {[opt-str init o]} { + use init + autosetup_init $o + incr earlyexit + } + + if {$earlyexit} { + exit 0 + } + if {[opt-str sysinstall o]} { + use install + autosetup_install $o 1 + exit 0 + } + + if {![file exists $autosetup(autodef)]} { + # Check for invalid option first + options {} + user-error "No auto.def found in \"$autosetup(srcdir)\" (use [file tail $::autosetup(exe)] --init to create one)" + } + + # Parse extra arguments into autosetup(cmdline) + foreach arg $argv { + if {[regexp {([^=]*)=(.*)} $arg -> n v]} { + dict set autosetup(cmdline) $n $v + define $n $v + } else { + user-error "Unexpected parameter: $arg" + } + } + + autosetup_add_dep $autosetup(autodef) + + # Add $argv to CONFIGURE_OPTS + define-append-argv CONFIGURE_OPTS {*}$autosetup(argv) + # Set up AUTOREMAKE to reconfigure with the same args + define-append-argv AUTOREMAKE {*}$autosetup(exe) {*}$autosetup(argv) + + # Log how we were invoked + configlog "Invoked as: [getenv WRAPPER $::argv0] [quote-argv $autosetup(argv)]" + configlog "Tclsh: [info nameofexecutable]" + + # Load auto.def as module "auto.def" + autosetup_load_module auto.def source $autosetup(autodef) + + # Could warn here if options {} was not specified + + show-notices + + if {$autosetup(debug)} { + msg-result "Writing all defines to config.log" + configlog "================ defines ======================" + foreach n [lsort [array names define]] { + configlog "define $n $define($n)" + } + } + + exit 0 +} + +# @section Option Handling + +# @opt-bool ?-nodefault? option ... +# +# Check each of the named, boolean options and if any have been explicitly enabled +# or disabled by the user, return 1 or 0 accordingly. +# +# If the option was specified more than once, the last value wins. +# e.g. With '--enable-foo --disable-foo', '[opt-bool foo]' will return 0 +# +# If no value was specified by the user, returns the default value for the +# first option. If '-nodefault' is given, this behaviour changes and +# -1 is returned instead. +# +proc opt-bool {args} { + set nodefault 0 + if {[lindex $args 0] eq "-nodefault"} { + set nodefault 1 + set args [lrange $args 1 end] + } + option-check-names {*}$args + + foreach opt $args { + if {[dict exists $::autosetup(optset) $opt]} { + return [dict get $::autosetup(optset) $opt] + } + } + + if {$nodefault} { + return -1 + } + # Default value is the default for the first option + return [dict get $::autosetup(optdefault) [lindex $args 0]] +} + +# @opt-val optionlist ?default=""? +# +# Returns a list containing all the values given for the non-boolean options in '$optionlist'. +# There will be one entry in the list for each option given by the user, including if the +# same option was used multiple times. +# +# If no options were set, '$default' is returned (exactly, not as a list). +# +# Note: For most use cases, 'opt-str' should be preferred. +# +proc opt-val {names {default ""}} { + option-check-names {*}$names + + foreach opt $names { + if {[dict exists $::autosetup(optset) $opt]} { + lappend result {*}[dict get $::autosetup(optset) $opt] + } + } + if {[info exists result]} { + return $result + } + return $default +} + +# @opt-str optionlist varname ?default? +# +# Sets '$varname' in the callers scope to the value for one of the given options. +# +# For the list of options given in '$optionlist', if any value is set for any option, +# the option value is taken to be the *last* value of the last option (in the order given). +# +# If no option was given, and a default was specified with 'options-defaults', +# that value is used. +# +# If no 'options-defaults' value was given and '$default' was given, it is used. +# +# If none of the above provided a value, no value is set. +# +# The return value depends on whether '$default' was specified. +# If it was, the option value is returned. +# If it was not, 1 is returns if a value was set, or 0 if not. +# +# Typical usage is as follows: +# +## if {[opt-str {myopt altname} o]} { +## do something with $o +## } +# +# Or: +## define myname [opt-str {myopt altname} o "/usr/local"] +# +proc opt-str {names varname args} { + global autosetup + + option-check-names {*}$names + upvar $varname value + + if {[llength $args]} { + # A default was given, so always return the string value of the option + set default [lindex $args 0] + set retopt 1 + } else { + # No default, so return 0 or 1 to indicate if a value was found + set retopt 0 + } + + foreach opt $names { + if {[dict exists $::autosetup(optset) $opt]} { + set result [lindex [dict get $::autosetup(optset) $opt] end] + } + } + + if {![info exists result]} { + # No user-specified value. Has options-defaults been set? + foreach opt $names { + if {[dict exists $::autosetup(optdefault) $opt]} { + set result [dict get $autosetup(optdefault) $opt] + } + } + } + + if {[info exists result]} { + set value $result + if {$retopt} { + return $value + } + return 1 + } + + if {$retopt} { + set value $default + return $value + } + + return 0 +} + +proc option-check-names {args} { + foreach o $args { + if {$o ni $::autosetup(options)} { + autosetup-error "Request for undeclared option --$o" + } + } +} + +# Parse the option definition in $opts and update +# ::autosetup(setoptions) and ::autosetup(optionhelp) appropriately +# +proc options-add {opts} { + global autosetup + + # First weed out comment lines + set realopts {} + foreach line [split $opts \n] { + if {![string match "#*" [string trimleft $line]]} { + append realopts $line \n + } + } + set opts $realopts + + for {set i 0} {$i < [llength $opts]} {incr i} { + set opt [lindex $opts $i] + if {[string match =* $opt]} { + # This is a special heading + lappend autosetup(optionhelp) [list $opt $autosetup(module)] + continue + } + unset -nocomplain defaultvalue equal value + + #puts "i=$i, opt=$opt" + regexp {^([^:=]*)(:)?(=)?(.*)$} $opt -> name colon equal value + if {$name in $autosetup(options)} { + autosetup-error "Option $name already specified" + } + + #puts "$opt => $name $colon $equal $value" + + # Find the corresponding value in the user options + # and set the default if necessary + if {[string match "-*" $opt]} { + # We no longer support documentation-only options, like "-C " + autosetup-error "Option $opt is not supported" + } elseif {$colon eq ""} { + # Boolean option + lappend autosetup(options) $name + + # Check for override + if {[dict exists $autosetup(options-defaults) $name]} { + # A default was specified with options-defaults, so use it + set value [dict get $autosetup(options-defaults) $name] + } + + if {$value eq "1"} { + set opthelp "--disable-$name" + } else { + set opthelp "--$name" + } + + # Set the default + if {$value eq ""} { + set value 0 + } + set defaultvalue $value + dict set autosetup(optdefault) $name $defaultvalue + + if {[dict exists $autosetup(getopt) $name]} { + # The option was specified by the user. Look at the last value. + lassign [lindex [dict get $autosetup(getopt) $name] end] type setvalue + if {$type eq "str"} { + # Can we convert the value to a boolean? + if {$setvalue in {1 enabled yes}} { + set setvalue 1 + } elseif {$setvalue in {0 disabled no}} { + set setvalue 0 + } else { + user-error "Boolean option $name given as --$name=$setvalue" + } + } + dict set autosetup(optset) $name $setvalue + #puts "Found boolean option --$name=$setvalue" + } + } else { + # String option. + lappend autosetup(options) $name + + if {$equal ne "="} { + # Was the option given as "name:value=default"? + # If so, set $value to the display name and $defaultvalue to the default + # (This is the preferred way to set a default value for a string option) + if {[regexp {^([^=]+)=(.*)$} $value -> value defaultvalue]} { + dict set autosetup(optdefault) $name $defaultvalue + } + } + + # Maybe override the default value + if {[dict exists $autosetup(options-defaults) $name]} { + # A default was specified with options-defaults, so use it + set defaultvalue [dict get $autosetup(options-defaults) $name] + dict set autosetup(optdefault) $name $defaultvalue + } elseif {![info exists defaultvalue]} { + # No default value was given by value=default or options-defaults + # so use the value as the default when the plain option with no + # value is given (.e.g. just --opt instead of --opt=value) + set defaultvalue $value + } + + if {$equal eq "="} { + # String option with optional value + set opthelp "--$name?=$value?" + } else { + # String option with required value + set opthelp "--$name=$value" + } + + # Get the values specified by the user + if {[dict exists $autosetup(getopt) $name]} { + set listvalue {} + + foreach pair [dict get $autosetup(getopt) $name] { + lassign $pair type setvalue + if {$type eq "bool" && $setvalue} { + if {$equal ne "="} { + user-error "Option --$name requires a value" + } + # If given as a boolean, use the default value + set setvalue $defaultvalue + } + lappend listvalue $setvalue + } + + #puts "Found string option --$name=$listvalue" + dict set autosetup(optset) $name $listvalue + } + } + + # Now create the help for this option if appropriate + if {[lindex $opts $i+1] eq "=>"} { + set desc [lindex $opts $i+2] + if {[info exists defaultvalue]} { + set desc [string map [list @default@ $defaultvalue] $desc] + } + # A multi-line description + lappend autosetup(optionhelp) [list $opthelp $autosetup(module) $desc] + incr i 2 + } + } +} + +# @module-options optionlist +# +# Deprecated. Simply use 'options' from within a module. +proc module-options {opts} { + options $opts +} + +proc max {a b} { + expr {$a > $b ? $a : $b} +} + +proc options-wrap-desc {text length firstprefix nextprefix initial} { + set len $initial + set space $firstprefix + foreach word [split $text] { + set word [string trim $word] + if {$word == ""} { + continue + } + if {$len && [string length $space$word] + $len >= $length} { + puts "" + set len 0 + set space $nextprefix + } + incr len [string length $space$word] + puts -nonewline $space$word + set space " " + } + if {$len} { + puts "" + } +} + +# Display options (from $autosetup(optionhelp)) for modules that match +# glob pattern $what +proc options-show {what} { + set local 0 + # Determine the max option width + set max 0 + foreach help $::autosetup(optionhelp) { + lassign $help opt module desc + if {![string match $what $module]} { + continue + } + if {[string match =* $opt] || [string match \n* $desc]} { + continue + } + set max [max $max [string length $opt]] + } + set indent [string repeat " " [expr {$max+4}]] + set cols [getenv COLUMNS 80] + catch { + lassign [exec stty size] _ sttycols + if {[string is integer -strict $sttycols]} { + set cols $sttycols + } + } + incr cols -1 + # Now output + foreach help $::autosetup(optionhelp) { + lassign $help opt module desc + if {![string match $what $module]} { + continue + } + if {$local == 0 && $module eq "auto.def"} { + puts "Local Options:" + incr local + } + if {[string match =* $opt]} { + # Output a special heading line" + puts [string range $opt 1 end] + continue + } + puts -nonewline " [format %-${max}s $opt]" + if {[string match \n* $desc]} { + # Output a pre-formatted help description as-is + puts $desc + } else { + options-wrap-desc [string trim $desc] $cols " " $indent [expr {$max+2}] + } + } +} + +# @options optionspec +# +# Specifies configuration-time options which may be selected by the user +# and checked with 'opt-str' and 'opt-bool'. '$optionspec' contains a series +# of options specifications separated by newlines, as follows: +# +# A boolean option is of the form: +# +## name[=0|1] => "Description of this boolean option" +# +# The default is 'name=0', meaning that the option is disabled by default. +# If 'name=1' is used to make the option enabled by default, the description should reflect +# that with text like "Disable support for ...". +# +# An argument option (one which takes a parameter) is of one of the following forms: +# +## name:value => "Description of this option" +## name:value=default => "Description of this option with a default value" +## name:=value => "Description of this option with an optional value" +# +# If the 'name:value' form is used, the value must be provided with the option (as '--name=myvalue'). +# If the 'name:value=default' form is used, the option has the given default value even if not +# specified by the user. +# If the 'name:=value' form is used, the value is optional and the given value is used +# if it is not provided. +# +# The description may contain '@default@', in which case it will be replaced with the default +# value for the option (taking into account defaults specified with 'options-defaults'. +# +# Undocumented options are also supported by omitting the '=> description'. +# These options are not displayed with '--help' and can be useful for internal options or as aliases. +# +# For example, '--disable-lfs' is an alias for '--disable=largefile': +# +## lfs=1 largefile=1 => "Disable large file support" +# +proc options {optlist} { + global autosetup + + options-add $optlist + + if {$autosetup(showhelp)} { + # If --help, stop now to show help + return -code break + } + + if {$autosetup(module) eq "auto.def"} { + # Check for invalid options + if {[opt-bool option-checking]} { + foreach o [dict keys $::autosetup(getopt)] { + if {$o ni $::autosetup(options)} { + user-error "Unknown option --$o" + } + } + } + } +} + +# @options-defaults dictionary +# +# Specifies a dictionary of options and a new default value for each of those options. +# Use before any 'use' statements in 'auto.def' to change the defaults for +# subsequently included modules. +proc options-defaults {dict} { + foreach {n v} $dict { + dict set ::autosetup(options-defaults) $n $v + } +} + +proc config_guess {} { + if {[file-isexec $::autosetup(dir)/autosetup-config.guess]} { + if {[catch {exec-with-stderr sh $::autosetup(dir)/autosetup-config.guess} alias]} { + user-error $alias + } + return $alias + } else { + configlog "No autosetup-config.guess, so using uname" + string tolower [exec uname -p]-unknown-[exec uname -s][exec uname -r] + } +} + +proc config_sub {alias} { + if {[file-isexec $::autosetup(dir)/autosetup-config.sub]} { + if {[catch {exec-with-stderr sh $::autosetup(dir)/autosetup-config.sub $alias} alias]} { + user-error $alias + } + } + return $alias +} + +# @section Variable Definitions (defines) + +# @define name ?value=1? +# +# Defines the named variable to the given value. +# These (name, value) pairs represent the results of the configuration check +# and are available to be subsequently checked, modified and substituted. +# +proc define {name {value 1}} { + set ::define($name) $value + #dputs "$name <= $value" +} + +# @define-push {name ...} script +# +# Save the values of the given defines, evaluation the script, then restore. +# For example, to avoid updating AS_FLAGS and AS_CXXFLAGS: +## define-push {AS_CFLAGS AS_CXXFLAGS} { +## cc-check-flags -Wno-error +## } +proc define-push {names script} { + array set unset {} + foreach name $names { + if {[is-defined $name]} { + set save($name) [get-define $name] + } else { + set unset($name) 1 + } + } + uplevel 1 $script + array set ::define [array get save] + foreach name [array names unset] { + unset -nocomplain ::define($name) + } +} + +# @undefine name +# +# Undefine the named variable. +# +proc undefine {name} { + unset -nocomplain ::define($name) + #dputs "$name <= " +} + +# @define-append name value ... +# +# Appends the given value(s) to the given "defined" variable. +# If the variable is not defined or empty, it is set to '$value'. +# Otherwise the value is appended, separated by a space. +# Any extra values are similarly appended. +# +# Note that define-append is not designed to add values containing spaces. +# If values may contain spaces, consider define-append-argv instead. +# +proc define-append {name args} { + if {[get-define $name ""] ne ""} { + foreach arg $args { + if {$arg eq ""} { + continue + } + append ::define($name) " " $arg + } + } else { + set ::define($name) [join $args] + } + #dputs "$name += [join $args] => $::define($name)" +} + +# @define-append-argv name value ... +# +# Similar to define-append except designed to construct shell command +# lines, including correct handling of parameters with spaces. +# +# Each non-empty value is quoted if necessary and then appended to the given variable +# if it does not already exist. +# +proc define-append-argv {name args} { + set seen {} + set new {} + foreach val [list {*}[get-define $name ""] {*}$args] { + if {$val ne {} && ![dict exists $seen $val]} { + lappend new [quote-if-needed $val] + dict set seen $val 1 + } + } + set ::define($name) [join $new " "] + #dputs "$name += [join $args] => $::define($name)" +} + +# @get-define name ?default=0? +# +# Returns the current value of the "defined" variable, or '$default' +# if not set. +# +proc get-define {name {default 0}} { + if {[info exists ::define($name)]} { + #dputs "$name => $::define($name)" + return $::define($name) + } + #dputs "$name => $default" + return $default +} + +# @is-defined name +# +# Returns 1 if the given variable is defined. +# +proc is-defined {name} { + info exists ::define($name) +} + +# @is-define-set name +# +# Returns 1 if the given variable is defined and is set +# to a value other than "" or 0 +# +proc is-define-set {name} { + if {[get-define $name] in {0 ""}} { + return 0 + } + return 1 +} + +# @all-defines +# +# Returns a dictionary (name, value list) of all defined variables. +# +# This is suitable for use with 'dict', 'array set' or 'foreach' +# and allows for arbitrary processing of the defined variables. +# +proc all-defines {} { + array get ::define +} + +# @section Environment/Helpers + +# @get-env name default +# +# If '$name' was specified on the command line, return it. +# Otherwise if '$name' was set in the environment, return it. +# Otherwise return '$default'. +# +proc get-env {name default} { + if {[dict exists $::autosetup(cmdline) $name]} { + return [dict get $::autosetup(cmdline) $name] + } + getenv $name $default +} + +# @env-is-set name +# +# Returns 1 if '$name' was specified on the command line or in the environment. +# Note that an empty environment variable is not considered to be set. +# +proc env-is-set {name} { + if {[dict exists $::autosetup(cmdline) $name]} { + return 1 + } + if {[getenv $name ""] ne ""} { + return 1 + } + return 0 +} + +# @readfile filename ?default=""? +# +# Return the contents of the file, without the trailing newline. +# If the file doesn't exist or can't be read, returns '$default'. +# +proc readfile {filename {default_value ""}} { + set result $default_value + catch { + set f [open $filename] + set result [read -nonewline $f] + close $f + } + return $result +} + +# @writefile filename value +# +# Creates the given file containing '$value'. +# Does not add an extra newline. +# +proc writefile {filename value} { + set f [open $filename w] + puts -nonewline $f $value + close $f +} + +proc quote-if-needed {str} { + if {[string match {*[\" ]*} $str]} { + return \"[string map [list \" \\" \\ \\\\] $str]\" + } + return $str +} + +proc quote-argv {argv} { + set args {} + foreach arg $argv { + lappend args [quote-if-needed $arg] + } + join $args +} + +# @list-non-empty list +# +# Returns a copy of the given list with empty elements removed +proc list-non-empty {list} { + set result {} + foreach p $list { + if {$p ne ""} { + lappend result $p + } + } + return $result +} + +# @section Paths, Searching + +# @find-executable-path name +# +# Searches the path for an executable with the given name. +# Note that the name may include some parameters, e.g. 'cc -mbig-endian', +# in which case the parameters are ignored. +# Returns the full path to the executable if found, or "" if not found. +# +proc find-executable-path {name} { + # Ignore any parameters + set name [lindex $name 0] + # The empty string is never a valid executable + if {$name ne ""} { + foreach p [split-path] { + dputs "Looking for $name in $p" + set exec [file join $p $name] + if {[file-isexec $exec]} { + dputs "Found $name -> $exec" + return $exec + } + } + } + return {} +} + +# @find-executable name +# +# Searches the path for an executable with the given name. +# Note that the name may include some parameters, e.g. 'cc -mbig-endian', +# in which case the parameters are ignored. +# Returns 1 if found, or 0 if not. +# +proc find-executable {name} { + if {[find-executable-path $name] eq {}} { + return 0 + } + return 1 +} + +# @find-an-executable ?-required? name ... +# +# Given a list of possible executable names, +# searches for one of these on the path. +# +# Returns the name found, or "" if none found. +# If the first parameter is '-required', an error is generated +# if no executable is found. +# +proc find-an-executable {args} { + set required 0 + if {[lindex $args 0] eq "-required"} { + set args [lrange $args 1 end] + incr required + } + foreach name $args { + if {[find-executable $name]} { + return $name + } + } + if {$required} { + if {[llength $args] == 1} { + user-error "failed to find: [join $args]" + } else { + user-error "failed to find one of: [join $args]" + } + } + return "" +} + +# @section Logging, Messages and Errors + +# @configlog msg +# +# Writes the given message to the configuration log, 'config.log'. +# +proc configlog {msg} { + if {![info exists ::autosetup(logfh)]} { + set ::autosetup(logfh) [open config.log w] + } + puts $::autosetup(logfh) $msg +} + +# @msg-checking msg +# +# Writes the message with no newline to stdout. +# +proc msg-checking {msg} { + if {$::autosetup(msg-quiet) == 0} { + maybe-show-timestamp + puts -nonewline $msg + set ::autosetup(msg-checking) 1 + } +} + +# @msg-result msg +# +# Writes the message to stdout. +# +proc msg-result {msg} { + if {$::autosetup(msg-quiet) == 0} { + maybe-show-timestamp + puts $msg + set ::autosetup(msg-checking) 0 + show-notices + } +} + +# @msg-quiet command ... +# +# 'msg-quiet' evaluates it's arguments as a command with output +# from 'msg-checking' and 'msg-result' suppressed. +# +# This is useful if a check needs to run a subcheck which isn't +# of interest to the user. +proc msg-quiet {args} { + incr ::autosetup(msg-quiet) + set rc [uplevel 1 $args] + incr ::autosetup(msg-quiet) -1 + return $rc +} + +# Will be overridden by 'use misc' +proc error-stacktrace {msg} { + return $msg +} + +proc error-location {msg} { + return $msg +} + +################################################################## +# +# Debugging output +# +proc dputs {msg} { + if {$::autosetup(debug)} { + puts $msg + } +} + +################################################################## +# +# User and system warnings and errors +# +# Usage errors such as wrong command line options + +# @user-error msg +# +# Indicate incorrect usage to the user, including if required components +# or features are not found. +# 'autosetup' exits with a non-zero return code. +# +proc user-error {msg} { + show-notices + puts stderr "Error: $msg" + puts stderr "Try: '[file tail $::autosetup(exe)] --help' for options" + exit 1 +} + +# @user-notice msg +# +# Output the given message to stderr. +# +proc user-notice {msg} { + lappend ::autosetup(notices) $msg +} + +# Incorrect usage in the auto.def file. Identify the location. +proc autosetup-error {msg} { + autosetup-full-error [error-location $msg] +} + +# Like autosetup-error, except $msg is the full error message. +proc autosetup-full-error {msg} { + show-notices + puts stderr $msg + exit 1 +} + +proc show-notices {} { + if {$::autosetup(msg-checking)} { + puts "" + set ::autosetup(msg-checking) 0 + } + flush stdout + if {[info exists ::autosetup(notices)]} { + puts stderr [join $::autosetup(notices) \n] + unset ::autosetup(notices) + } +} + +proc maybe-show-timestamp {} { + if {$::autosetup(msg-timing) && $::autosetup(msg-checking) == 0} { + puts -nonewline [format {[%6.2f] } [expr {([clock millis] - $::autosetup(start)) % 10000 / 1000.0}]] + } +} + +# @autosetup-require-version required +# +# Checks the current version of 'autosetup' against '$required'. +# A fatal error is generated if the current version is less than that required. +# +proc autosetup-require-version {required} { + if {[compare-versions $::autosetup(version) $required] < 0} { + user-error "autosetup version $required is required, but this is $::autosetup(version)" + } +} + +proc autosetup_version {} { + return "autosetup v$::autosetup(version)" +} + +################################################################## +# +# Directory/path handling +# + +proc realdir {dir} { + set oldpwd [pwd] + cd $dir + set pwd [pwd] + cd $oldpwd + return $pwd +} + +# Follow symlinks until we get to something which is not a symlink +proc realpath {path} { + while {1} { + if {[catch { + set path [file readlink $path] + }]} { + # Not a link + break + } + } + return $path +} + +# Convert absolute path, $path into a path relative +# to the given directory (or the current dir, if not given). +# +proc relative-path {path {pwd {}}} { + set diff 0 + set same 0 + set newf {} + set prefix {} + set path [file-normalize $path] + if {$pwd eq ""} { + set pwd [pwd] + } else { + set pwd [file-normalize $pwd] + } + + if {$path eq $pwd} { + return . + } + + # Try to make the filename relative to the current dir + foreach p [split $pwd /] f [split $path /] { + if {$p ne $f} { + incr diff + } elseif {!$diff} { + incr same + } + if {$diff} { + if {$p ne ""} { + # Add .. for sibling or parent dir + lappend prefix .. + } + if {$f ne ""} { + lappend newf $f + } + } + } + if {$same == 1 || [llength $prefix] > 3} { + return $path + } + + file join [join $prefix /] [join $newf /] +} + +# Add filename as a dependency to rerun autosetup +# The name will be normalised (converted to a full path) +# +proc autosetup_add_dep {filename} { + lappend ::autosetup(deps) [file-normalize $filename] +} + +# @section Modules Support + +################################################################## +# +# Library module support +# + +# @use module ... +# +# Load the given library modules. +# e.g. 'use cc cc-shared' +# +# Note that module 'X' is implemented in either 'autosetup/X.tcl' +# or 'autosetup/X/init.tcl' +# +# The latter form is useful for a complex module which requires additional +# support file. In this form, '$::usedir' is set to the module directory +# when it is loaded. +# +proc use {args} { + global autosetup libmodule modsource + + set dirs [list $autosetup(libdir)] + if {[info exists autosetup(srcdir)]} { + lappend dirs $autosetup(srcdir)/autosetup + } + foreach m $args { + if {[info exists libmodule($m)]} { + continue + } + set libmodule($m) 1 + + if {[info exists modsource(${m}.tcl)]} { + autosetup_load_module $m eval $modsource(${m}.tcl) + } else { + set locs [list ${m}.tcl ${m}/init.tcl] + set found 0 + foreach dir $dirs { + foreach loc $locs { + set source $dir/$loc + if {[file exists $source]} { + incr found + break + } + } + if {$found} { + break + } + } + if {$found} { + # For the convenience of the "use" source, point to the directory + # it is being loaded from + set ::usedir [file dirname $source] + autosetup_load_module $m source $source + autosetup_add_dep $source + } else { + autosetup-error "use: No such module: $m" + } + } + } +} + +proc autosetup_load_auto_modules {} { + global autosetup modsource + # First load any embedded auto modules + foreach mod [array names modsource *.auto] { + autosetup_load_module $mod eval $modsource($mod) + } + # Now any external auto modules + foreach file [glob -nocomplain $autosetup(libdir)/*.auto $autosetup(libdir)/*/*.auto] { + autosetup_load_module [file tail $file] source $file + } +} + +# Load module source in the global scope by executing the given command +proc autosetup_load_module {module args} { + global autosetup + set prev $autosetup(module) + set autosetup(module) $module + + if {[catch [list uplevel #0 $args] msg opts] ni {0 2 3}} { + autosetup-full-error [error-dump $msg $opts $::autosetup(debug)] + } + set autosetup(module) $prev +} + +# Initial settings +set autosetup(exe) $::argv0 +set autosetup(istcl) 1 +set autosetup(start) [clock millis] +set autosetup(installed) 0 +set autosetup(sysinstall) 0 +set autosetup(msg-checking) 0 +set autosetup(msg-quiet) 0 +set autosetup(inittypes) {} +set autosetup(module) autosetup + +# Embedded modules are inserted below here +set autosetup(installed) 1 +set autosetup(sysinstall) 0 +# ----- @module asciidoc-formatting.tcl ----- + +set modsource(asciidoc-formatting.tcl) { +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which provides text formatting +# asciidoc format + +use formatting + +proc para {text} { + regsub -all "\[ \t\n\]+" [string trim $text] " " +} +proc title {text} { + underline [para $text] = + nl +} +proc p {text} { + puts [para $text] + nl +} +proc code {text} { + foreach line [parse_code_block $text] { + puts " $line" + } + nl +} +proc codelines {lines} { + foreach line $lines { + puts " $line" + } + nl +} +proc nl {} { + puts "" +} +proc underline {text char} { + regexp "^(\[ \t\]*)(.*)" $text -> indent words + puts $text + puts $indent[string repeat $char [string length $words]] +} +proc section {text} { + underline "[para $text]" - + nl +} +proc subsection {text} { + underline "$text" ~ + nl +} +proc bullet {text} { + puts "* [para $text]" +} +proc indent {text} { + puts " :: " + puts [para $text] +} +proc defn {first args} { + set sep "" + if {$first ne ""} { + puts "${first}::" + } else { + puts " :: " + } + set defn [string trim [join $args \n]] + regsub -all "\n\n" $defn "\n ::\n" defn + puts $defn +} +} + +# ----- @module formatting.tcl ----- + +set modsource(formatting.tcl) { +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which provides common text formatting + +# This is designed for documentation which looks like: +# code {...} +# or +# code { +# ... +# ... +# } +# In the second case, we need to work out the indenting +# and strip it from all lines but preserve the remaining indenting. +# Note that all lines need to be indented with the same initial +# spaces/tabs. +# +# Returns a list of lines with the indenting removed. +# +proc parse_code_block {text} { + # If the text begins with newline, take the following text, + # otherwise just return the original + if {![regexp "^\n(.*)" $text -> text]} { + return [list [string trim $text]] + } + + # And trip spaces off the end + set text [string trimright $text] + + set min 100 + # Examine each line to determine the minimum indent + foreach line [split $text \n] { + if {$line eq ""} { + # Ignore empty lines for the indent calculation + continue + } + regexp "^(\[ \t\]*)" $line -> indent + set len [string length $indent] + if {$len < $min} { + set min $len + } + } + + # Now make a list of lines with this indent removed + set lines {} + foreach line [split $text \n] { + lappend lines [string range $line $min end] + } + + # Return the result + return $lines +} +} + +# ----- @module getopt.tcl ----- + +set modsource(getopt.tcl) { +# Copyright (c) 2006 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Simple getopt module + +# Parse everything out of the argv list which looks like an option +# Everything which doesn't look like an option, or is after --, is left unchanged +# Understands --enable-xxx as a synonym for --xxx to enable the boolean option xxx. +# Understands --disable-xxx to disable the boolean option xxx. +# +# The returned value is a dictionary keyed by option name +# Each value is a list of {type value} ... where type is "bool" or "str". +# The value for a boolean option is 0 or 1. The value of a string option is the value given. +proc getopt {argvname} { + upvar $argvname argv + set nargv {} + + set opts {} + + for {set i 0} {$i < [llength $argv]} {incr i} { + set arg [lindex $argv $i] + + #dputs arg=$arg + + if {$arg eq "--"} { + # End of options + incr i + lappend nargv {*}[lrange $argv $i end] + break + } + + if {[regexp {^--([^=][^=]+)=(.*)$} $arg -> name value]} { + # --name=value + dict lappend opts $name [list str $value] + } elseif {[regexp {^--(enable-|disable-)?([^=]*)$} $arg -> prefix name]} { + if {$prefix in {enable- ""}} { + set value 1 + } else { + set value 0 + } + dict lappend opts $name [list bool $value] + } else { + lappend nargv $arg + } + } + + #puts "getopt: argv=[join $argv] => [join $nargv]" + #array set getopt $opts + #parray getopt + + set argv $nargv + + return $opts +} +} + +# ----- @module help.tcl ----- + +set modsource(help.tcl) { +# Copyright (c) 2010 WorkWare Systems http://workware.net.au/ +# All rights reserved + +# Module which provides usage, help and the command reference + +proc autosetup_help {what} { + use_pager + + puts "Usage: [file tail $::autosetup(exe)] \[options\] \[settings\]\n" + puts "This is [autosetup_version], a build environment \"autoconfigurator\"" + puts "See the documentation online at https://msteveb.github.io/autosetup/\n" + + if {$what in {all local}} { + # Need to load auto.def now + if {[file exists $::autosetup(autodef)]} { + # Load auto.def as module "auto.def" + autosetup_load_module auto.def source $::autosetup(autodef) + } + if {$what eq "all"} { + set what * + } else { + set what auto.def + } + } else { + use $what + puts "Options for module $what:" + } + options-show $what + exit 0 +} + +proc autosetup_show_license {} { + global modsource autosetup + use_pager + + if {[info exists modsource(LICENSE)]} { + puts $modsource(LICENSE) + return + } + foreach dir [list $autosetup(libdir) $autosetup(srcdir)] { + set path [file join $dir LICENSE] + if {[file exists $path]} { + puts [readfile $path] + return + } + } + puts "LICENSE not found" +} + +# If not already paged and stdout is a tty, pipe the output through the pager +# This is done by reinvoking autosetup with --nopager added +proc use_pager {} { + if {![opt-bool nopager] && [getenv PAGER ""] ne "" && [isatty? stdin] && [isatty? stdout]} { + if {[catch { + exec [info nameofexecutable] $::argv0 --nopager {*}$::argv |& {*}[getenv PAGER] >@stdout <@stdin 2>@stderr + } msg opts] == 1} { + if {[dict get $opts -errorcode] eq "NONE"} { + # an internal/exec error + puts stderr $msg + exit 1 + } + } + exit 0 + } +} + +# Outputs the autosetup references in one of several formats +proc autosetup_reference {{type text}} { + + use_pager + + switch -glob -- $type { + wiki {use wiki-formatting} + ascii* {use asciidoc-formatting} + md - markdown {use markdown-formatting} + default {use text-formatting} + } + + title "[autosetup_version] -- Command Reference" + + section {Introduction} + + p { + See https://msteveb.github.io/autosetup/ for the online documentation for 'autosetup'. + This documentation can also be accessed locally with `autosetup --ref`. + } + + p { + 'autosetup' provides a number of built-in commands which + are documented below. These may be used from 'auto.def' to test + for features, define variables, create files from templates and + other similar actions. + } + + automf_command_reference + + exit 0 +} + +proc autosetup_output_block {type lines} { + if {[llength $lines]} { + switch $type { + section { + section $lines + } + subsection { + subsection $lines + } + code { + codelines $lines + } + p { + p [join $lines] + } + list { + foreach line $lines { + bullet $line + } + nl + } + } + } +} + +# Generate a command reference from inline documentation +proc automf_command_reference {} { + lappend files $::autosetup(prog) + lappend files {*}[lsort [glob -nocomplain $::autosetup(libdir)/{*/*.tcl,*.tcl}]] + + # We want to process all non-module files before module files + # and then modules in alphabetical order. + # So examine all files and extract docs into doc($modulename) and doc(_core_) + # + # Each entry is a list of {type data} where $type is one of: section, subsection, code, list, p + # and $data is a string for section, subsection or a list of text lines for other types. + + # XXX: Should commands be in alphabetical order too? Currently they are in file order. + + set doc(_core_) {} + lappend doc(_core_) [list section "Core Commands"] + + foreach file $files { + set modulename [file rootname [file tail $file]] + set current _core_ + set f [open $file] + while {![eof $f]} { + set line [gets $f] + + if {[regexp {^#.*@section (.*)$} $line -> section]} { + lappend doc($current) [list section $section] + continue + } + + # Find embedded module names + if {[regexp {^#.*@module ([^ ]*)} $line -> modulename]} { + continue + } + + # Find lines starting with "# @*" and continuing through the remaining comment lines + if {![regexp {^# @(.*)} $line -> cmd]} { + continue + } + + # Synopsis or command? + if {$cmd eq "synopsis:"} { + set current $modulename + lappend doc($current) [list section "Module: $modulename"] + } else { + lappend doc($current) [list subsection $cmd] + } + + set lines {} + set type p + + # Now the description + while {![eof $f]} { + set line [gets $f] + + if {![regexp {^#(#)? ?(.*)} $line -> hash cmd]} { + break + } + if {$hash eq "#"} { + set t code + } elseif {[regexp {^- (.*)} $cmd -> cmd]} { + set t list + } else { + set t p + } + + #puts "hash=$hash, oldhash=$oldhash, lines=[llength $lines], cmd=$cmd" + + if {$t ne $type || $cmd eq ""} { + # Finish the current block + lappend doc($current) [list $type $lines] + set lines {} + set type $t + } + if {$cmd ne ""} { + lappend lines $cmd + } + } + + lappend doc($current) [list $type $lines] + } + close $f + } + + # Now format and output the results + + # _core_ will sort first + foreach module [lsort [array names doc]] { + foreach item $doc($module) { + autosetup_output_block {*}$item + } + } +} +} + +# ----- @module init.tcl ----- + +set modsource(init.tcl) { +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module to help create auto.def and configure + +proc autosetup_init {type} { + set help 0 + if {$type in {? help}} { + incr help + } elseif {![dict exists $::autosetup(inittypes) $type]} { + puts "Unknown type, --init=$type" + incr help + } + if {$help} { + puts "Use one of the following types (e.g. --init=make)\n" + foreach type [lsort [dict keys $::autosetup(inittypes)]] { + lassign [dict get $::autosetup(inittypes) $type] desc + # XXX: Use the options-show code to wrap the description + puts [format "%-10s %s" $type $desc] + } + return + } + lassign [dict get $::autosetup(inittypes) $type] desc script + + puts "Initialising $type: $desc\n" + + # All initialisations happens in the top level srcdir + cd $::autosetup(srcdir) + + uplevel #0 $script +} + +proc autosetup_add_init_type {type desc script} { + dict set ::autosetup(inittypes) $type [list $desc $script] +} + +# This is for in creating build-system init scripts +# +# If the file doesn't exist, create it containing $contents +# If the file does exist, only overwrite if --force is specified. +# +proc autosetup_check_create {filename contents} { + if {[file exists $filename]} { + if {!$::autosetup(force)} { + puts "I see $filename already exists." + return + } else { + puts "I will overwrite the existing $filename because you used --force." + } + } else { + puts "I don't see $filename, so I will create it." + } + writefile $filename $contents +} +} + +# ----- @module install.tcl ----- + +set modsource(install.tcl) { +# Copyright (c) 2006-2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which can install autosetup + +# autosetup(installed)=1 means that autosetup is not running from source +# autosetup(sysinstall)=1 means that autosetup is running from a sysinstall version +# shared=1 means that we are trying to do a sysinstall. This is only possible from the development source. + +proc autosetup_install {dir {shared 0}} { + global autosetup + if {$shared} { + if {$autosetup(installed) || $autosetup(sysinstall)} { + user-error "Can only --sysinstall from development sources" + } + } elseif {$autosetup(installed) && !$autosetup(sysinstall)} { + user-error "Can't --install from project install" + } + + if {$autosetup(sysinstall)} { + # This is the sysinstall version, so install just uses references + cd $dir + + puts "[autosetup_version] creating configure to use system-installed autosetup" + autosetup_create_configure 1 + puts "Creating autosetup/README.autosetup" + file mkdir autosetup + autosetup_install_readme autosetup/README.autosetup 1 + return + } + + if {[catch { + if {$shared} { + set target $dir/bin/autosetup + set installedas $target + } else { + if {$dir eq "."} { + set installedas autosetup + } else { + set installedas $dir/autosetup + } + cd $dir + file mkdir autosetup + set target autosetup/autosetup + } + set targetdir [file dirname $target] + file mkdir $targetdir + + set f [open $target w] + + set publicmodules {} + + # First the main script, but only up until "CUT HERE" + set in [open $autosetup(dir)/autosetup] + while {[gets $in buf] >= 0} { + if {$buf ne "##-- CUT HERE --##"} { + puts $f $buf + continue + } + + # Insert the static modules here + # i.e. those which don't contain @synopsis: + # All modules are inserted if $shared is set + puts $f "set autosetup(installed) 1" + puts $f "set autosetup(sysinstall) $shared" + foreach file [lsort [glob $autosetup(libdir)/*.{tcl,auto}]] { + set modname [file tail $file] + set ext [file ext $modname] + set buf [readfile $file] + if {!$shared} { + if {$ext eq ".auto" || [string match "*\n# @synopsis:*" $buf]} { + lappend publicmodules $file + continue + } + } + dputs "install: importing lib/[file tail $file]" + puts $f "# ----- @module $modname -----" + puts $f "\nset modsource($modname) \{" + puts $f $buf + puts $f "\}\n" + } + if {$shared} { + foreach {srcname destname} [list $autosetup(libdir)/README.autosetup-lib README.autosetup \ + $autosetup(srcdir)/LICENSE LICENSE] { + dputs "install: importing $srcname as $destname" + puts $f "\nset modsource($destname) \\\n[list [readfile $srcname]\n]\n" + } + } + } + close $in + close $f + catch {exec chmod 755 $target} + + set installfiles {autosetup-config.guess autosetup-config.sub autosetup-test-tclsh} + set removefiles {} + + if {!$shared} { + autosetup_install_readme $targetdir/README.autosetup 0 + + # Install public modules + foreach file $publicmodules { + set tail [file tail $file] + autosetup_install_file $file $targetdir/$tail + } + lappend installfiles jimsh0.c autosetup-find-tclsh LICENSE + lappend removefiles config.guess config.sub test-tclsh find-tclsh + } else { + lappend installfiles {sys-find-tclsh autosetup-find-tclsh} + } + + # Install support files + foreach fileinfo $installfiles { + if {[llength $fileinfo] == 2} { + lassign $fileinfo source dest + } else { + lassign $fileinfo source + set dest $source + } + autosetup_install_file $autosetup(dir)/$source $targetdir/$dest + } + + # Remove obsolete files + foreach file $removefiles { + if {[file exists $targetdir/$file]} { + file delete $targetdir/$file + } + } + } error]} { + user-error "Failed to install autosetup: $error" + } + if {$shared} { + set type "system" + } else { + set type "local" + } + puts "Installed $type [autosetup_version] to $installedas" + + if {!$shared} { + # Now create 'configure' if necessary + autosetup_create_configure 0 + } +} + +proc autosetup_create_configure {shared} { + if {[file exists configure]} { + if {!$::autosetup(force)} { + # Could this be an autosetup configure? + if {![string match "*\nWRAPPER=*" [readfile configure]]} { + puts "I see configure, but not created by autosetup, so I won't overwrite it." + puts "Remove it or use --force to overwrite." + return + } + } else { + puts "I will overwrite the existing configure because you used --force." + } + } else { + puts "I don't see configure, so I will create it." + } + if {$shared} { + writefile configure \ +{#!/bin/sh +WRAPPER="$0"; export WRAPPER; "autosetup" "$@" +} + } else { + writefile configure \ +{#!/bin/sh +dir="`dirname "$0"`/autosetup" +#@@INITCHECK@@# +WRAPPER="$0"; export WRAPPER; exec "`"$dir/autosetup-find-tclsh"`" "$dir/autosetup" "$@" +} + } + catch {exec chmod 755 configure} +} + +# Append the contents of $file to filehandle $f +proc autosetup_install_append {f file} { + dputs "install: include $file" + set in [open $file] + puts $f [read $in] + close $in +} + +proc autosetup_install_file {source target} { + dputs "install: $source => $target" + if {![file exists $source]} { + error "Missing installation file '$source'" + } + writefile $target [readfile $source]\n + # If possible, copy the file mode + file stat $source stat + set mode [format %o [expr {$stat(mode) & 0x1ff}]] + catch {exec chmod $mode $target} +} + +proc autosetup_install_readme {target sysinstall} { + set readme "README.autosetup created by [autosetup_version]\n\n" + if {$sysinstall} { + append readme \ +{This is the autosetup directory for a system install of autosetup. +Loadable modules can be added here. +} + } else { + append readme \ +{This is the autosetup directory for a local install of autosetup. +It contains autosetup, support files and loadable modules. +} +} + + append readme { +*.tcl files in this directory are optional modules which +can be loaded with the 'use' directive. + +*.auto files in this directory are auto-loaded. + +For more information, see https://msteveb.github.io/autosetup/ +} + dputs "install: autosetup/README.autosetup" + writefile $target $readme +} +} + +# ----- @module markdown-formatting.tcl ----- + +set modsource(markdown-formatting.tcl) { +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which provides text formatting +# markdown format (kramdown syntax) + +use formatting + +proc para {text} { + regsub -all "\[ \t\n\]+" [string trim $text] " " text + regsub -all {([^a-zA-Z])'([^']*)'} $text {\1**`\2`**} text + regsub -all {^'([^']*)'} $text {**`\1`**} text + regsub -all {(http[^ \t\n]*)} $text {[\1](\1)} text + return $text +} +proc title {text} { + underline [para $text] = + nl +} +proc p {text} { + puts [para $text] + nl +} +proc codelines {lines} { + puts "~~~~~~~~~~~~" + foreach line $lines { + puts $line + } + puts "~~~~~~~~~~~~" + nl +} +proc code {text} { + puts "~~~~~~~~~~~~" + foreach line [parse_code_block $text] { + puts $line + } + puts "~~~~~~~~~~~~" + nl +} +proc nl {} { + puts "" +} +proc underline {text char} { + regexp "^(\[ \t\]*)(.*)" $text -> indent words + puts $text + puts $indent[string repeat $char [string length $words]] +} +proc section {text} { + underline "[para $text]" - + nl +} +proc subsection {text} { + puts "### `$text`" + nl +} +proc bullet {text} { + puts "* [para $text]" +} +proc defn {first args} { + puts "^" + set defn [string trim [join $args \n]] + if {$first ne ""} { + puts "**${first}**" + puts -nonewline ": " + regsub -all "\n\n" $defn "\n: " defn + } + puts "$defn" +} +} + +# ----- @module misc.tcl ----- + +set modsource(misc.tcl) { +# Copyright (c) 2007-2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module containing misc procs useful to modules +# Largely for platform compatibility + +set autosetup(istcl) [info exists ::tcl_library] +set autosetup(iswin) [string equal windows $tcl_platform(platform)] + +if {$autosetup(iswin)} { + # mingw/windows separates $PATH with semicolons + # and doesn't have an executable bit + proc split-path {} { + split [getenv PATH .] {;} + } + proc file-isexec {exec} { + # Basic test for windows. We ignore .bat + if {[file isfile $exec] || [file isfile $exec.exe]} { + return 1 + } + return 0 + } +} else { + # unix separates $PATH with colons and has and executable bit + proc split-path {} { + split [getenv PATH .] : + } + # Check for an executable file + proc file-isexec {exec} { + if {[file executable $exec] && [file isfile $exec]} { + return 1 + } + return 0 + } +} + +# Assume that exec can return stdout and stderr +proc exec-with-stderr {args} { + exec {*}$args 2>@1 +} + +if {$autosetup(istcl)} { + # Tcl doesn't have the env command + proc getenv {name args} { + if {[info exists ::env($name)]} { + return $::env($name) + } + if {[llength $args]} { + return [lindex $args 0] + } + return -code error "environment variable \"$name\" does not exist" + } + proc isatty? {channel} { + dict exists [fconfigure $channel] -xchar + } + # Jim-compatible stacktrace using info frame + proc stacktrace {} { + set stacktrace {} + # 2 to skip the current frame + for {set i 2} {$i < [info frame]} {incr i} { + set frame [info frame -$i] + if {[dict exists $frame file]} { + # We don't need proc, so use "" + lappend stacktrace "" [dict get $frame file] [dict get $frame line] "" + } + } + return $stacktrace + } +} else { + if {$autosetup(iswin)} { + # On Windows, backslash convert all environment variables + # (Assume that Tcl does this for us) + proc getenv {name args} { + string map {\\ /} [env $name {*}$args] + } + } else { + # Jim on unix is simple + alias getenv env + } + proc isatty? {channel} { + set tty 0 + catch { + # isatty is a recent addition to Jim Tcl + set tty [$channel isatty] + } + return $tty + } +} + +# In case 'file normalize' doesn't exist +# +proc file-normalize {path} { + if {[catch {file normalize $path} result]} { + if {$path eq ""} { + return "" + } + set oldpwd [pwd] + if {[file isdir $path]} { + cd $path + set result [pwd] + } else { + cd [file dirname $path] + set result [file join [pwd] [file tail $path]] + } + cd $oldpwd + } + return $result +} + +# If everything is working properly, the only errors which occur +# should be generated in user code (e.g. auto.def). +# By default, we only want to show the error location in user code. +# We use [info frame] to achieve this, but it works differently on Tcl and Jim. +# +# This is designed to be called for incorrect usage in auto.def, via autosetup-error +# +proc error-location {msg} { + if {$::autosetup(debug)} { + return -code error $msg + } + set vars {p f l cmd} + if {!$::autosetup(istcl) && ![dict exists $::tcl_platform stackFormat]} { + # Older versions of Jim had a 3 element stacktrace + set vars {p f l} + } + foreach $vars [stacktrace] { + if {[string match *.def $f]} { + return "[relative-path $f]:$l: Error: $msg" + } + #puts "Skipping $f:$l" + } + return $msg +} + +# If everything is working properly, the only errors which occur +# should be generated in user code (e.g. auto.def). +# By default, we only want to show the error location in user code. +# We use [info frame] to achieve this, but it works differently on Tcl and Jim. +# +# This is designed to be called for incorrect usage in auto.def, via autosetup-error +# +proc error-stacktrace {msg} { + if {$::autosetup(debug)} { + return -code error $msg + } + # Search back through the stack trace for the first error in a .def file + for {set i 1} {$i < [info level]} {incr i} { + if {$::autosetup(istcl)} { + array set info [info frame -$i] + } else { + lassign [info frame -$i] info(caller) info(file) info(line) + } + if {[string match *.def $info(file)]} { + return "[relative-path $info(file)]:$info(line): Error: $msg" + } + #puts "Skipping $info(file):$info(line)" + } + return $msg +} + +# Given the return from [catch {...} msg opts], returns an appropriate +# error message. A nice one for Jim and a less-nice one for Tcl. +# If 'fulltrace' is set, a full stack trace is provided. +# Otherwise a simple message is provided. +# +# This is designed for developer errors, e.g. in module code or auto.def code +# +# +proc error-dump {msg opts fulltrace} { + if {$::autosetup(istcl)} { + if {$fulltrace} { + return "Error: [dict get $opts -errorinfo]" + } else { + return "Error: $msg" + } + } else { + lassign $opts(-errorinfo) p f l + if {$f ne ""} { + set result "$f:$l: Error: " + } + append result "$msg\n" + if {$fulltrace} { + append result [stackdump $opts(-errorinfo)] + } + + # Remove the trailing newline + string trim $result + } +} +} + +# ----- @module text-formatting.tcl ----- + +set modsource(text-formatting.tcl) { +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which provides text formatting + +use formatting + +proc wordwrap {text length {firstprefix ""} {nextprefix ""}} { + set len 0 + set space $firstprefix + + foreach word [split $text] { + set word [string trim $word] + if {$word eq ""} { + continue + } + if {[info exists partial]} { + append partial " " $word + if {[string first $quote $word] < 0} { + # Haven't found end of quoted word + continue + } + # Finished quoted word + set word $partial + unset partial + unset quote + } else { + set quote [string index $word 0] + if {$quote in {' *}} { + if {[string first $quote $word 1] < 0} { + # Haven't found end of quoted word + # Not a whole word. + set first [string index $word 0] + # Start of quoted word + set partial $word + continue + } + } + } + + if {$len && [string length $space$word] + $len >= $length} { + puts "" + set len 0 + set space $nextprefix + } + incr len [string length $space$word] + + # Use man-page conventions for highlighting 'quoted' and *quoted* + # single words. + # Use x^Hx for *bold* and _^Hx for 'underline'. + # + # less and more will both understand this. + # Pipe through 'col -b' to remove them. + if {[regexp {^'(.*)'(.*)} $word -> quoted after]} { + set quoted [string map {~ " "} $quoted] + regsub -all . $quoted "&\b&" quoted + set word $quoted$after + } elseif {[regexp {^[*](.*)[*](.*)} $word -> quoted after]} { + set quoted [string map {~ " "} $quoted] + regsub -all . $quoted "_\b&" quoted + set word $quoted$after + } + puts -nonewline $space$word + set space " " + } + if {[info exists partial]} { + # Missing end of quote + puts -nonewline $space$partial + } + if {$len} { + puts "" + } +} +proc title {text} { + underline [string trim $text] = + nl +} +proc p {text} { + wordwrap $text 80 + nl +} +proc codelines {lines} { + foreach line $lines { + puts " $line" + } + nl +} +proc nl {} { + puts "" +} +proc underline {text char} { + regexp "^(\[ \t\]*)(.*)" $text -> indent words + puts $text + puts $indent[string repeat $char [string length $words]] +} +proc section {text} { + underline "[string trim $text]" - + nl +} +proc subsection {text} { + underline "$text" ~ + nl +} +proc bullet {text} { + wordwrap $text 76 " * " " " +} +proc indent {text} { + wordwrap $text 76 " " " " +} +proc defn {first args} { + if {$first ne ""} { + underline " $first" ~ + } + foreach p $args { + if {$p ne ""} { + indent $p + } + } +} +} + +# ----- @module util.tcl ----- + +set modsource(util.tcl) { +# Copyright (c) 2012 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which contains miscellaneous utility functions + +# @section Utilities + +# @compare-versions version1 version2 +# +# Versions are of the form 'a.b.c' (may be any number of numeric components) +# +# Compares the two versions and returns: +## -1 if v1 < v2 +## 0 if v1 == v2 +## 1 if v1 > v2 +# +# If one version has fewer components than the other, 0 is substituted to the right. e.g. +## 0.2 < 0.3 +## 0.2.5 > 0.2 +## 1.1 == 1.1.0 +# +proc compare-versions {v1 v2} { + foreach c1 [split $v1 .] c2 [split $v2 .] { + if {$c1 eq ""} { + set c1 0 + } + if {$c2 eq ""} { + set c2 0 + } + if {$c1 < $c2} { + return -1 + } + if {$c1 > $c2} { + return 1 + } + } + return 0 +} + +# @suffix suf list +# +# Takes a list and returns a new list with '$suf' appended +# to each element +# +## suffix .c {a b c} => {a.c b.c c.c} +# +proc suffix {suf list} { + set result {} + foreach p $list { + lappend result $p$suf + } + return $result +} + +# @prefix pre list +# +# Takes a list and returns a new list with '$pre' prepended +# to each element +# +## prefix jim- {a.c b.c} => {jim-a.c jim-b.c} +# +proc prefix {pre list} { + set result {} + foreach p $list { + lappend result $pre$p + } + return $result +} + +# @lpop list +# +# Removes the last entry from the given list and returns it. +proc lpop {listname} { + upvar $listname list + set val [lindex $list end] + set list [lrange $list 0 end-1] + return $val +} +} + +# ----- @module wiki-formatting.tcl ----- + +set modsource(wiki-formatting.tcl) { +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# Module which provides text formatting +# wiki.tcl.tk format output + +use formatting + +proc joinlines {text} { + set lines {} + foreach l [split [string trim $text] \n] { + lappend lines [string trim $l] + } + join $lines +} +proc p {text} { + puts [joinlines $text] + puts "" +} +proc title {text} { + puts "*** [joinlines $text] ***" + puts "" +} +proc codelines {lines} { + puts "======" + foreach line $lines { + puts " $line" + } + puts "======" +} +proc code {text} { + puts "======" + foreach line [parse_code_block $text] { + puts " $line" + } + puts "======" +} +proc nl {} { +} +proc section {text} { + puts "'''$text'''" + puts "" +} +proc subsection {text} { + puts "''$text''" + puts "" +} +proc bullet {text} { + puts " * [joinlines $text]" +} +proc indent {text} { + puts " : [joinlines $text]" +} +proc defn {first args} { + if {$first ne ""} { + indent '''$first''' + } + + foreach p $args { + p $p + } +} +} + + +################################################################## +# +# Entry/Exit +# +if {$autosetup(debug)} { + main $argv +} +if {[catch {main $argv} msg opts] == 1} { + show-notices + autosetup-full-error [error-dump $msg $opts $autosetup(debug)] + if {!$autosetup(debug)} { + puts stderr "Try: '[file tail $autosetup(exe)] --debug' for a full stack trace" + } + exit 1 +} diff --git a/config.guess b/autosetup/autosetup-config.guess old mode 100644 new mode 100755 similarity index 64% rename from config.guess rename to autosetup/autosetup-config.guess index ae713942d8..48a684601b --- a/config.guess +++ b/autosetup/autosetup-config.guess @@ -1,12 +1,14 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2019 Free Software Foundation, Inc. +# Copyright 1992-2024 Free Software Foundation, Inc. -timestamp='2019-05-28' +# shellcheck disable=SC2006,SC2268 # see below for rationale + +timestamp='2024-07-27' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or +# the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -27,17 +29,25 @@ timestamp='2019-05-28' # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess # # Please send patches to . +# The "shellcheck disable" line above the timestamp inhibits complaints +# about features and limitations of the classic Bourne shell that were +# superseded or lifted in POSIX. However, this script identifies a wide +# variety of pre-POSIX systems that do not have POSIX shells at all, and +# even some reasonably current systems (Solaris 10 as case-in-point) still +# have a pre-POSIX /bin/sh. + + me=`echo "$0" | sed -e 's,.*/,,'` usage="\ Usage: $0 [OPTION] -Output the configuration name of the system \`$me' is run on. +Output the configuration name of the system '$me' is run on. Options: -h, --help print this help, then exit @@ -50,13 +60,13 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2019 Free Software Foundation, Inc. +Copyright 1992-2024 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." help=" -Try \`$me --help' for more information." +Try '$me --help' for more information." # Parse command line while test $# -gt 0 ; do @@ -84,13 +94,16 @@ if test $# != 0; then exit 1 fi +# Just in case it came from the environment. +GUESS= + # CC_FOR_BUILD -- compiler used by this script. Note that the use of a # compiler to aid in system detection is discouraged as it requires # temporary files to be created and, as you can see below, it is a # headache to deal with in a portable fashion. -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. +# Historically, 'CC_FOR_BUILD' used to be named 'HOST_CC'. We still +# use 'HOST_CC' if defined, but it is deprecated. # Portable tmp directory creation inspired by the Autoconf team. @@ -99,8 +112,10 @@ tmp= trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 set_cc_for_build() { + # prevent multiple calls if $tmp is already set + test "$tmp" && return 0 : "${TMPDIR=/tmp}" - # shellcheck disable=SC2039 + # shellcheck disable=SC2039,SC3028 { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || @@ -108,9 +123,9 @@ set_cc_for_build() { dummy=$tmp/dummy case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in ,,) echo "int x;" > "$dummy.c" - for driver in cc gcc c89 c99 ; do + for driver in cc gcc c17 c99 c89 ; do if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then - CC_FOR_BUILD="$driver" + CC_FOR_BUILD=$driver break fi done @@ -131,40 +146,57 @@ fi UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown -UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown +UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown -case "$UNAME_SYSTEM" in +case $UNAME_SYSTEM in Linux|GNU|GNU/*) - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - LIBC=gnu + LIBC=unknown set_cc_for_build cat <<-EOF > "$dummy.c" + #if defined(__ANDROID__) + LIBC=android + #else #include #if defined(__UCLIBC__) LIBC=uclibc #elif defined(__dietlibc__) LIBC=dietlibc - #else + #elif defined(__GLIBC__) LIBC=gnu + #elif defined(__LLVM_LIBC__) + LIBC=llvm + #else + #include + /* First heuristic to detect musl libc. */ + #ifdef __DEFINED_va_list + LIBC=musl + #endif + #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` + eval "$cc_set_libc" - # If ldd exists, use it to detect musl libc. - if command -v ldd >/dev/null && \ - ldd --version 2>&1 | grep -q ^musl - then - LIBC=musl + # Second heuristic to detect musl libc. + if [ "$LIBC" = unknown ] && + command -v ldd >/dev/null && + ldd --version 2>&1 | grep -q ^musl; then + LIBC=musl + fi + + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + if [ "$LIBC" = unknown ]; then + LIBC=gnu fi ;; esac # Note: order is significant - the case branches are not exclusive. -case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in +case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, @@ -176,12 +208,12 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". - sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ - "/sbin/$sysctl" 2>/dev/null || \ - "/usr/sbin/$sysctl" 2>/dev/null || \ + /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ + /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ echo unknown)` - case "$UNAME_MACHINE_ARCH" in + case $UNAME_MACHINE_ARCH in + aarch64eb) machine=aarch64_be-unknown ;; armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; @@ -190,13 +222,13 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in earmv*) arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` - machine="${arch}${endian}"-unknown + machine=${arch}${endian}-unknown ;; - *) machine="$UNAME_MACHINE_ARCH"-unknown ;; + *) machine=$UNAME_MACHINE_ARCH-unknown ;; esac # The Operating System including object format, if it has switched # to ELF recently (or will in the future) and ABI. - case "$UNAME_MACHINE_ARCH" in + case $UNAME_MACHINE_ARCH in earm*) os=netbsdelf ;; @@ -217,7 +249,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in ;; esac # Determine ABI tags. - case "$UNAME_MACHINE_ARCH" in + case $UNAME_MACHINE_ARCH in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` @@ -228,7 +260,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. - case "$UNAME_VERSION" in + case $UNAME_VERSION in Debian*) release='-gnu' ;; @@ -239,45 +271,57 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "$machine-${os}${release}${abi-}" - exit ;; + GUESS=$machine-${os}${release}${abi-} + ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE_ARCH-unknown-bitrig$UNAME_RELEASE + ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE_ARCH-unknown-openbsd$UNAME_RELEASE + ;; + *:SecBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/SecBSD.//'` + GUESS=$UNAME_MACHINE_ARCH-unknown-secbsd$UNAME_RELEASE + ;; *:LibertyBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` - echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE_ARCH-unknown-libertybsd$UNAME_RELEASE + ;; *:MidnightBSD:*:*) - echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-unknown-midnightbsd$UNAME_RELEASE + ;; *:ekkoBSD:*:*) - echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-unknown-ekkobsd$UNAME_RELEASE + ;; *:SolidBSD:*:*) - echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-unknown-solidbsd$UNAME_RELEASE + ;; + *:OS108:*:*) + GUESS=$UNAME_MACHINE-unknown-os108_$UNAME_RELEASE + ;; macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd"$UNAME_RELEASE" - exit ;; + GUESS=powerpc-unknown-mirbsd$UNAME_RELEASE + ;; *:MirBSD:*:*) - echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-unknown-mirbsd$UNAME_RELEASE + ;; *:Sortix:*:*) - echo "$UNAME_MACHINE"-unknown-sortix - exit ;; + GUESS=$UNAME_MACHINE-unknown-sortix + ;; + *:Twizzler:*:*) + GUESS=$UNAME_MACHINE-unknown-twizzler + ;; *:Redox:*:*) - echo "$UNAME_MACHINE"-unknown-redox - exit ;; + GUESS=$UNAME_MACHINE-unknown-redox + ;; mips:OSF1:*.*) - echo mips-dec-osf1 - exit ;; + GUESS=mips-dec-osf1 + ;; alpha:OSF1:*:*) + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + trap '' 0 case $UNAME_RELEASE in *4.0) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` @@ -291,7 +335,7 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # covers most systems running today. This code pipes the CPU # types through head -n 1, so we only detect the type of CPU 0. ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` - case "$ALPHA_CPU_TYPE" in + case $ALPHA_CPU_TYPE in "EV4 (21064)") UNAME_MACHINE=alpha ;; "EV4.5 (21064)") @@ -328,117 +372,121 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" - # Reset EXIT trap before exiting to avoid spurious non-zero exit code. - exitcode=$? - trap '' 0 - exit $exitcode ;; + OSF_REL=`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + GUESS=$UNAME_MACHINE-dec-osf$OSF_REL + ;; Amiga*:UNIX_System_V:4.0:*) - echo m68k-unknown-sysv4 - exit ;; + GUESS=m68k-unknown-sysv4 + ;; *:[Aa]miga[Oo][Ss]:*:*) - echo "$UNAME_MACHINE"-unknown-amigaos - exit ;; + GUESS=$UNAME_MACHINE-unknown-amigaos + ;; *:[Mm]orph[Oo][Ss]:*:*) - echo "$UNAME_MACHINE"-unknown-morphos - exit ;; + GUESS=$UNAME_MACHINE-unknown-morphos + ;; *:OS/390:*:*) - echo i370-ibm-openedition - exit ;; + GUESS=i370-ibm-openedition + ;; *:z/VM:*:*) - echo s390-ibm-zvmoe - exit ;; + GUESS=s390-ibm-zvmoe + ;; *:OS400:*:*) - echo powerpc-ibm-os400 - exit ;; + GUESS=powerpc-ibm-os400 + ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix"$UNAME_RELEASE" - exit ;; + GUESS=arm-acorn-riscix$UNAME_RELEASE + ;; arm*:riscos:*:*|arm*:RISCOS:*:*) - echo arm-unknown-riscos - exit ;; + GUESS=arm-unknown-riscos + ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) - echo hppa1.1-hitachi-hiuxmpp - exit ;; + GUESS=hppa1.1-hitachi-hiuxmpp + ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "`(/bin/universe) 2>/dev/null`" = att ; then - echo pyramid-pyramid-sysv3 - else - echo pyramid-pyramid-bsd - fi - exit ;; + case `(/bin/universe) 2>/dev/null` in + att) GUESS=pyramid-pyramid-sysv3 ;; + *) GUESS=pyramid-pyramid-bsd ;; + esac + ;; NILE*:*:*:dcosx) - echo pyramid-pyramid-svr4 - exit ;; + GUESS=pyramid-pyramid-svr4 + ;; DRS?6000:unix:4.0:6*) - echo sparc-icl-nx6 - exit ;; + GUESS=sparc-icl-nx6 + ;; DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) case `/usr/bin/uname -p` in - sparc) echo sparc-icl-nx7; exit ;; - esac ;; + sparc) GUESS=sparc-icl-nx7 ;; + esac + ;; s390x:SunOS:*:*) - echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" - exit ;; + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` + GUESS=$UNAME_MACHINE-ibm-solaris2$SUN_REL + ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" - exit ;; + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` + GUESS=sparc-hal-solaris2$SUN_REL + ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" - exit ;; + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` + GUESS=sparc-sun-solaris2$SUN_REL + ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux"$UNAME_RELEASE" - exit ;; + GUESS=i386-pc-auroraux$UNAME_RELEASE + ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) set_cc_for_build SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if test "$CC_FOR_BUILD" != no_compiler_found; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -m64 -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then SUN_ARCH=x86_64 fi fi - echo "$SUN_ARCH"-pc-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" - exit ;; + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` + GUESS=$SUN_ARCH-pc-solaris2$SUN_REL + ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" - exit ;; + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` + GUESS=sparc-sun-solaris3$SUN_REL + ;; sun4*:SunOS:*:*) - case "`/usr/bin/arch -k`" in + case `/usr/bin/arch -k` in Series*|S4*) UNAME_RELEASE=`uname -v` ;; esac - # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" - exit ;; + # Japanese Language versions have a version number like '4.1.3-JL'. + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` + GUESS=sparc-sun-sunos$SUN_REL + ;; sun3*:SunOS:*:*) - echo m68k-sun-sunos"$UNAME_RELEASE" - exit ;; + GUESS=m68k-sun-sunos$UNAME_RELEASE + ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 - case "`/bin/arch`" in + case `/bin/arch` in sun3) - echo m68k-sun-sunos"$UNAME_RELEASE" + GUESS=m68k-sun-sunos$UNAME_RELEASE ;; sun4) - echo sparc-sun-sunos"$UNAME_RELEASE" + GUESS=sparc-sun-sunos$UNAME_RELEASE ;; esac - exit ;; + ;; aushp:SunOS:*:*) - echo sparc-auspex-sunos"$UNAME_RELEASE" - exit ;; + GUESS=sparc-auspex-sunos$UNAME_RELEASE + ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not # "atarist" or "atariste" at least should have a processor @@ -448,41 +496,41 @@ case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; + GUESS=m68k-atari-mint$UNAME_RELEASE + ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; + GUESS=m68k-atari-mint$UNAME_RELEASE + ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; + GUESS=m68k-atari-mint$UNAME_RELEASE + ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint"$UNAME_RELEASE" - exit ;; + GUESS=m68k-milan-mint$UNAME_RELEASE + ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint"$UNAME_RELEASE" - exit ;; + GUESS=m68k-hades-mint$UNAME_RELEASE + ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint"$UNAME_RELEASE" - exit ;; + GUESS=m68k-unknown-mint$UNAME_RELEASE + ;; m68k:machten:*:*) - echo m68k-apple-machten"$UNAME_RELEASE" - exit ;; + GUESS=m68k-apple-machten$UNAME_RELEASE + ;; powerpc:machten:*:*) - echo powerpc-apple-machten"$UNAME_RELEASE" - exit ;; + GUESS=powerpc-apple-machten$UNAME_RELEASE + ;; RISC*:Mach:*:*) - echo mips-dec-mach_bsd4.3 - exit ;; + GUESS=mips-dec-mach_bsd4.3 + ;; RISC*:ULTRIX:*:*) - echo mips-dec-ultrix"$UNAME_RELEASE" - exit ;; + GUESS=mips-dec-ultrix$UNAME_RELEASE + ;; VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix"$UNAME_RELEASE" - exit ;; + GUESS=vax-dec-ultrix$UNAME_RELEASE + ;; 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix"$UNAME_RELEASE" - exit ;; + GUESS=clipper-intergraph-clix$UNAME_RELEASE + ;; mips:*:*:UMIPS | mips:*:*:RISCos) set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" @@ -510,82 +558,84 @@ EOF dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && SYSTEM_NAME=`"$dummy" "$dummyarg"` && { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos"$UNAME_RELEASE" - exit ;; + GUESS=mips-mips-riscos$UNAME_RELEASE + ;; Motorola:PowerMAX_OS:*:*) - echo powerpc-motorola-powermax - exit ;; + GUESS=powerpc-motorola-powermax + ;; Motorola:*:4.3:PL8-*) - echo powerpc-harris-powermax - exit ;; + GUESS=powerpc-harris-powermax + ;; Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) - echo powerpc-harris-powermax - exit ;; + GUESS=powerpc-harris-powermax + ;; Night_Hawk:Power_UNIX:*:*) - echo powerpc-harris-powerunix - exit ;; + GUESS=powerpc-harris-powerunix + ;; m88k:CX/UX:7*:*) - echo m88k-harris-cxux7 - exit ;; + GUESS=m88k-harris-cxux7 + ;; m88k:*:4*:R4*) - echo m88k-motorola-sysv4 - exit ;; + GUESS=m88k-motorola-sysv4 + ;; m88k:*:3*:R3*) - echo m88k-motorola-sysv3 - exit ;; + GUESS=m88k-motorola-sysv3 + ;; AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] + if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 then - if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ - [ "$TARGET_BINARY_INTERFACE"x = x ] + if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ + test "$TARGET_BINARY_INTERFACE"x = x then - echo m88k-dg-dgux"$UNAME_RELEASE" + GUESS=m88k-dg-dgux$UNAME_RELEASE else - echo m88k-dg-dguxbcs"$UNAME_RELEASE" + GUESS=m88k-dg-dguxbcs$UNAME_RELEASE fi else - echo i586-dg-dgux"$UNAME_RELEASE" + GUESS=i586-dg-dgux$UNAME_RELEASE fi - exit ;; + ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) - echo m88k-dolphin-sysv3 - exit ;; + GUESS=m88k-dolphin-sysv3 + ;; M88*:*:R3*:*) # Delta 88k system running SVR3 - echo m88k-motorola-sysv3 - exit ;; + GUESS=m88k-motorola-sysv3 + ;; XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) - echo m88k-tektronix-sysv3 - exit ;; + GUESS=m88k-tektronix-sysv3 + ;; Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) - echo m68k-tektronix-bsd - exit ;; + GUESS=m68k-tektronix-bsd + ;; *:IRIX*:*:*) - echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" - exit ;; + IRIX_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/g'` + GUESS=mips-sgi-irix$IRIX_REL + ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + GUESS=romp-ibm-aix # uname -m gives an 8 hex-code CPU id + ;; # Note that: echo "'`uname -s`'" gives 'AIX ' i*86:AIX:*:*) - echo i386-ibm-aix - exit ;; + GUESS=i386-ibm-aix + ;; ia64:AIX:*:*) - if [ -x /usr/bin/oslevel ] ; then + if test -x /usr/bin/oslevel ; then IBM_REV=`/usr/bin/oslevel` else - IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" + IBM_REV=$UNAME_VERSION.$UNAME_RELEASE fi - echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" - exit ;; + GUESS=$UNAME_MACHINE-ibm-aix$IBM_REV + ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" #include - main() + int + main () { if (!__power_pc()) exit(1); @@ -595,16 +645,16 @@ EOF EOF if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` then - echo "$SYSTEM_NAME" + GUESS=$SYSTEM_NAME else - echo rs6000-ibm-aix3.2.5 + GUESS=rs6000-ibm-aix3.2.5 fi elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then - echo rs6000-ibm-aix3.2.4 + GUESS=rs6000-ibm-aix3.2.4 else - echo rs6000-ibm-aix3.2 + GUESS=rs6000-ibm-aix3.2 fi - exit ;; + ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then @@ -612,56 +662,56 @@ EOF else IBM_ARCH=powerpc fi - if [ -x /usr/bin/lslpp ] ; then - IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + if test -x /usr/bin/lslpp ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | \ awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else - IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" + IBM_REV=$UNAME_VERSION.$UNAME_RELEASE fi - echo "$IBM_ARCH"-ibm-aix"$IBM_REV" - exit ;; + GUESS=$IBM_ARCH-ibm-aix$IBM_REV + ;; *:AIX:*:*) - echo rs6000-ibm-aix - exit ;; + GUESS=rs6000-ibm-aix + ;; ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) - echo romp-ibm-bsd4.4 - exit ;; + GUESS=romp-ibm-bsd4.4 + ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to - exit ;; # report: romp-ibm BSD 4.3 + GUESS=romp-ibm-bsd$UNAME_RELEASE # 4.3 with uname added to + ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) - echo rs6000-bull-bosx - exit ;; + GUESS=rs6000-bull-bosx + ;; DPX/2?00:B.O.S.:*:*) - echo m68k-bull-sysv3 - exit ;; + GUESS=m68k-bull-sysv3 + ;; 9000/[34]??:4.3bsd:1.*:*) - echo m68k-hp-bsd - exit ;; + GUESS=m68k-hp-bsd + ;; hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) - echo m68k-hp-bsd4.4 - exit ;; + GUESS=m68k-hp-bsd4.4 + ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` - case "$UNAME_MACHINE" in + HPUX_REV=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*.[0B]*//'` + case $UNAME_MACHINE in 9000/31?) HP_ARCH=m68000 ;; 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) - if [ -x /usr/bin/getconf ]; then + if test -x /usr/bin/getconf; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "$sc_cpu_version" in + case $sc_cpu_version in 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 - case "$sc_kernel_bits" in + case $sc_kernel_bits in 32) HP_ARCH=hppa2.0n ;; 64) HP_ARCH=hppa2.0w ;; '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi - if [ "$HP_ARCH" = "" ]; then + if test "$HP_ARCH" = ""; then set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" @@ -669,7 +719,8 @@ EOF #include #include - int main () + int + main () { #if defined(_SC_KERNEL_BITS) long bits = sysconf(_SC_KERNEL_BITS); @@ -700,7 +751,7 @@ EOF test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ "$HP_ARCH" = hppa2.0w ] + if test "$HP_ARCH" = hppa2.0w then set_cc_for_build @@ -721,12 +772,12 @@ EOF HP_ARCH=hppa64 fi fi - echo "$HP_ARCH"-hp-hpux"$HPUX_REV" - exit ;; + GUESS=$HP_ARCH-hp-hpux$HPUX_REV + ;; ia64:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux"$HPUX_REV" - exit ;; + HPUX_REV=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*.[0B]*//'` + GUESS=ia64-hp-hpux$HPUX_REV + ;; 3050*:HI-UX:*:*) set_cc_for_build sed 's/^ //' << EOF > "$dummy.c" @@ -756,36 +807,36 @@ EOF EOF $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } - echo unknown-hitachi-hiuxwe2 - exit ;; + GUESS=unknown-hitachi-hiuxwe2 + ;; 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) - echo hppa1.1-hp-bsd - exit ;; + GUESS=hppa1.1-hp-bsd + ;; 9000/8??:4.3bsd:*:*) - echo hppa1.0-hp-bsd - exit ;; + GUESS=hppa1.0-hp-bsd + ;; *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) - echo hppa1.0-hp-mpeix - exit ;; + GUESS=hppa1.0-hp-mpeix + ;; hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) - echo hppa1.1-hp-osf - exit ;; + GUESS=hppa1.1-hp-osf + ;; hp8??:OSF1:*:*) - echo hppa1.0-hp-osf - exit ;; + GUESS=hppa1.0-hp-osf + ;; i*86:OSF1:*:*) - if [ -x /usr/sbin/sysversion ] ; then - echo "$UNAME_MACHINE"-unknown-osf1mk + if test -x /usr/sbin/sysversion ; then + GUESS=$UNAME_MACHINE-unknown-osf1mk else - echo "$UNAME_MACHINE"-unknown-osf1 + GUESS=$UNAME_MACHINE-unknown-osf1 fi - exit ;; + ;; parisc*:Lites*:*:*) - echo hppa1.1-hp-lites - exit ;; + GUESS=hppa1.1-hp-lites + ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) - echo c1-convex-bsd - exit ;; + GUESS=c1-convex-bsd + ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd @@ -793,17 +844,18 @@ EOF fi exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) - echo c34-convex-bsd - exit ;; + GUESS=c34-convex-bsd + ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) - echo c38-convex-bsd - exit ;; + GUESS=c38-convex-bsd + ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) - echo c4-convex-bsd - exit ;; + GUESS=c4-convex-bsd + ;; CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; + CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` + GUESS=ymp-cray-unicos$CRAY_REL + ;; CRAY*[A-Z]90:*:*:*) echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ @@ -811,114 +863,155 @@ EOF -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) - echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; + CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` + GUESS=t90-cray-unicos$CRAY_REL + ;; CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; + CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` + GUESS=alphaev5-cray-unicosmk$CRAY_REL + ;; CRAY*SV1:*:*:*) - echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; + CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` + GUESS=sv1-cray-unicos$CRAY_REL + ;; *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; + CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` + GUESS=craynv-cray-unicosmp$CRAY_REL + ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; + GUESS=${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL} + ;; 5000:UNIX_System_V:4.*:*) FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; + GUESS=sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL} + ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-pc-bsdi$UNAME_RELEASE + ;; sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi"$UNAME_RELEASE" - exit ;; + GUESS=sparc-unknown-bsdi$UNAME_RELEASE + ;; *:BSD/OS:*:*) - echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-unknown-bsdi$UNAME_RELEASE + ;; arm:FreeBSD:*:*) UNAME_PROCESSOR=`uname -p` set_cc_for_build if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + FREEBSD_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` + GUESS=$UNAME_PROCESSOR-unknown-freebsd$FREEBSD_REL-gnueabi else - echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + FREEBSD_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` + GUESS=$UNAME_PROCESSOR-unknown-freebsd$FREEBSD_REL-gnueabihf fi - exit ;; + ;; *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` - case "$UNAME_PROCESSOR" in + UNAME_PROCESSOR=`uname -p` + case $UNAME_PROCESSOR in amd64) UNAME_PROCESSOR=x86_64 ;; i386) UNAME_PROCESSOR=i586 ;; esac - echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" - exit ;; + FREEBSD_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` + GUESS=$UNAME_PROCESSOR-unknown-freebsd$FREEBSD_REL + ;; i*:CYGWIN*:*) - echo "$UNAME_MACHINE"-pc-cygwin - exit ;; + GUESS=$UNAME_MACHINE-pc-cygwin + ;; *:MINGW64*:*) - echo "$UNAME_MACHINE"-pc-mingw64 - exit ;; + GUESS=$UNAME_MACHINE-pc-mingw64 + ;; *:MINGW*:*) - echo "$UNAME_MACHINE"-pc-mingw32 - exit ;; + GUESS=$UNAME_MACHINE-pc-mingw32 + ;; *:MSYS*:*) - echo "$UNAME_MACHINE"-pc-msys - exit ;; + GUESS=$UNAME_MACHINE-pc-msys + ;; i*:PW*:*) - echo "$UNAME_MACHINE"-pc-pw32 - exit ;; + GUESS=$UNAME_MACHINE-pc-pw32 + ;; + *:SerenityOS:*:*) + GUESS=$UNAME_MACHINE-pc-serenity + ;; *:Interix*:*) - case "$UNAME_MACHINE" in + case $UNAME_MACHINE in x86) - echo i586-pc-interix"$UNAME_RELEASE" - exit ;; + GUESS=i586-pc-interix$UNAME_RELEASE + ;; authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix"$UNAME_RELEASE" - exit ;; + GUESS=x86_64-unknown-interix$UNAME_RELEASE + ;; IA64) - echo ia64-unknown-interix"$UNAME_RELEASE" - exit ;; + GUESS=ia64-unknown-interix$UNAME_RELEASE + ;; esac ;; i*:UWIN*:*) - echo "$UNAME_MACHINE"-pc-uwin - exit ;; + GUESS=$UNAME_MACHINE-pc-uwin + ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-pc-cygwin - exit ;; + GUESS=x86_64-pc-cygwin + ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" - exit ;; + SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` + GUESS=powerpcle-unknown-solaris2$SUN_REL + ;; *:GNU:*:*) # the GNU system - echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" - exit ;; + GNU_ARCH=`echo "$UNAME_MACHINE" | sed -e 's,[-/].*$,,'` + GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's,/.*$,,'` + GUESS=$GNU_ARCH-unknown-$LIBC$GNU_REL + ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" - exit ;; + GNU_SYS=`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"` + GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` + GUESS=$UNAME_MACHINE-unknown-$GNU_SYS$GNU_REL-$LIBC + ;; + x86_64:[Mm]anagarm:*:*|i?86:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-pc-managarm-mlibc" + ;; + *:[Mm]anagarm:*:*) + GUESS="$UNAME_MACHINE-unknown-managarm-mlibc" + ;; *:Minix:*:*) - echo "$UNAME_MACHINE"-unknown-minix - exit ;; + GUESS=$UNAME_MACHINE-unknown-minix + ;; aarch64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __ARM_EABI__ + #ifdef __ARM_PCS_VFP + ABI=eabihf + #else + ABI=eabi + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + eabi | eabihf) CPU=armv8l; LIBCABI=$LIBC$ABI ;; + esac + fi + GUESS=$CPU-unknown-linux-$LIBCABI + ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null` in EV5) UNAME_MACHINE=alphaev5 ;; EV56) UNAME_MACHINE=alphaev56 ;; PCA56) UNAME_MACHINE=alphapca56 ;; @@ -929,60 +1022,72 @@ EOF esac objdump --private-headers /bin/sh | grep -q ld.so.1 if test "$?" = 0 ; then LIBC=gnulibc1 ; fi - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - arc:Linux:*:* | arceb:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + arc:Linux:*:* | arceb:Linux:*:* | arc32:Linux:*:* | arc64:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; arm*:Linux:*:*) set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi + GUESS=$UNAME_MACHINE-unknown-linux-${LIBC}eabi else - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf + GUESS=$UNAME_MACHINE-unknown-linux-${LIBC}eabihf fi fi - exit ;; + ;; avr32*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; cris:Linux:*:*) - echo "$UNAME_MACHINE"-axis-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-axis-linux-$LIBC + ;; crisv32:Linux:*:*) - echo "$UNAME_MACHINE"-axis-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-axis-linux-$LIBC + ;; e2k:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; frv:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; hexagon:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; i*86:Linux:*:*) - echo "$UNAME_MACHINE"-pc-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-pc-linux-$LIBC + ;; ia64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; k1om:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; + kvx:cos:*:*) + GUESS=$UNAME_MACHINE-unknown-cos + ;; + kvx:mbr:*:*) + GUESS=$UNAME_MACHINE-unknown-mbr + ;; + loongarch32:Linux:*:* | loongarch64:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; m32r*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; m68*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; mips:Linux:*:* | mips64:Linux:*:*) set_cc_for_build IS_GLIBC=0 @@ -1027,113 +1132,135 @@ EOF #endif #endif EOF - eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI'`" + cc_set_vars=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI'` + eval "$cc_set_vars" test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } ;; mips64el:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; openrisc*:Linux:*:*) - echo or1k-unknown-linux-"$LIBC" - exit ;; + GUESS=or1k-unknown-linux-$LIBC + ;; or32:Linux:*:* | or1k*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; padre:Linux:*:*) - echo sparc-unknown-linux-"$LIBC" - exit ;; + GUESS=sparc-unknown-linux-$LIBC + ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-"$LIBC" - exit ;; + GUESS=hppa64-unknown-linux-$LIBC + ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; - PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; - *) echo hppa-unknown-linux-"$LIBC" ;; + PA7*) GUESS=hppa1.1-unknown-linux-$LIBC ;; + PA8*) GUESS=hppa2.0-unknown-linux-$LIBC ;; + *) GUESS=hppa-unknown-linux-$LIBC ;; esac - exit ;; + ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-"$LIBC" - exit ;; + GUESS=powerpc64-unknown-linux-$LIBC + ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-"$LIBC" - exit ;; + GUESS=powerpc-unknown-linux-$LIBC + ;; ppc64le:Linux:*:*) - echo powerpc64le-unknown-linux-"$LIBC" - exit ;; + GUESS=powerpc64le-unknown-linux-$LIBC + ;; ppcle:Linux:*:*) - echo powerpcle-unknown-linux-"$LIBC" - exit ;; - riscv32:Linux:*:* | riscv64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=powerpcle-unknown-linux-$LIBC + ;; + riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; s390:Linux:*:* | s390x:Linux:*:*) - echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-ibm-linux-$LIBC + ;; sh64*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; sh*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; tile*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; vax:Linux:*:*) - echo "$UNAME_MACHINE"-dec-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-dec-linux-$LIBC + ;; x86_64:Linux:*:*) - echo "$UNAME_MACHINE"-pc-linux-"$LIBC" - exit ;; + set_cc_for_build + CPU=$UNAME_MACHINE + LIBCABI=$LIBC + if test "$CC_FOR_BUILD" != no_compiler_found; then + ABI=64 + sed 's/^ //' << EOF > "$dummy.c" + #ifdef __i386__ + ABI=x86 + #else + #ifdef __ILP32__ + ABI=x32 + #endif + #endif +EOF + cc_set_abi=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^ABI' | sed 's, ,,g'` + eval "$cc_set_abi" + case $ABI in + x86) CPU=i686 ;; + x32) LIBCABI=${LIBC}x32 ;; + esac + fi + GUESS=$CPU-pc-linux-$LIBCABI + ;; xtensa*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; + GUESS=$UNAME_MACHINE-unknown-linux-$LIBC + ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both # sysname and nodename. - echo i386-sequent-sysv4 - exit ;; + GUESS=i386-sequent-sysv4 + ;; i*86:UNIX_SV:4.2MP:2.*) # Unixware is an offshoot of SVR4, but it has its own version # number series starting with 2... # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. - echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" - exit ;; + GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION + ;; i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility + # If we were able to find 'uname', then EMX Unix compatibility # is probably installed. - echo "$UNAME_MACHINE"-pc-os2-emx - exit ;; + GUESS=$UNAME_MACHINE-pc-os2-emx + ;; i*86:XTS-300:*:STOP) - echo "$UNAME_MACHINE"-unknown-stop - exit ;; + GUESS=$UNAME_MACHINE-unknown-stop + ;; i*86:atheos:*:*) - echo "$UNAME_MACHINE"-unknown-atheos - exit ;; + GUESS=$UNAME_MACHINE-unknown-atheos + ;; i*86:syllable:*:*) - echo "$UNAME_MACHINE"-pc-syllable - exit ;; + GUESS=$UNAME_MACHINE-pc-syllable + ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos"$UNAME_RELEASE" - exit ;; + GUESS=i386-unknown-lynxos$UNAME_RELEASE + ;; i*86:*DOS:*:*) - echo "$UNAME_MACHINE"-pc-msdosdjgpp - exit ;; + GUESS=$UNAME_MACHINE-pc-msdosdjgpp + ;; i*86:*:4.*:*) UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" + GUESS=$UNAME_MACHINE-univel-sysv$UNAME_REL else - echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" + GUESS=$UNAME_MACHINE-pc-sysv$UNAME_REL fi - exit ;; + ;; i*86:*:5:[678]*) # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in @@ -1141,12 +1268,12 @@ EOF *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" - exit ;; + GUESS=$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 @@ -1156,11 +1283,11 @@ EOF && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 - echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" + GUESS=$UNAME_MACHINE-pc-sco$UNAME_REL else - echo "$UNAME_MACHINE"-pc-sysv32 + GUESS=$UNAME_MACHINE-pc-sysv32 fi - exit ;; + ;; pc:*:*:*) # Left here for compatibility: # uname -m prints for DJGPP always 'pc', but it prints nothing about @@ -1168,31 +1295,31 @@ EOF # Note: whatever this is, it MUST be the same as what config.sub # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. - echo i586-pc-msdosdjgpp - exit ;; + GUESS=i586-pc-msdosdjgpp + ;; Intel:Mach:3*:*) - echo i386-pc-mach3 - exit ;; + GUESS=i386-pc-mach3 + ;; paragon:*:*:*) - echo i860-intel-osf1 - exit ;; + GUESS=i860-intel-osf1 + ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 + GUESS=i860-stardent-sysv$UNAME_RELEASE # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 + GUESS=i860-unknown-sysv$UNAME_RELEASE # Unknown i860-SVR4 fi - exit ;; + ;; mini*:CTIX:SYS*5:*) # "miniframe" - echo m68010-convergent-sysv - exit ;; + GUESS=m68010-convergent-sysv + ;; mc68k:UNIX:SYSTEM5:3.51m) - echo m68k-convergent-sysv - exit ;; + GUESS=m68k-convergent-sysv + ;; M680?0:D-NIX:5.3:*) - echo m68k-diab-dnix - exit ;; + GUESS=m68k-diab-dnix + ;; M68*:*:R3V[5678]*:*) test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) @@ -1217,113 +1344,119 @@ EOF /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos"$UNAME_RELEASE" - exit ;; + GUESS=m68k-unknown-lynxos$UNAME_RELEASE + ;; mc68030:UNIX_System_V:4.*:*) - echo m68k-atari-sysv4 - exit ;; + GUESS=m68k-atari-sysv4 + ;; TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos"$UNAME_RELEASE" - exit ;; + GUESS=sparc-unknown-lynxos$UNAME_RELEASE + ;; rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos"$UNAME_RELEASE" - exit ;; + GUESS=rs6000-unknown-lynxos$UNAME_RELEASE + ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos"$UNAME_RELEASE" - exit ;; + GUESS=powerpc-unknown-lynxos$UNAME_RELEASE + ;; SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv"$UNAME_RELEASE" - exit ;; + GUESS=mips-dde-sysv$UNAME_RELEASE + ;; RM*:ReliantUNIX-*:*:*) - echo mips-sni-sysv4 - exit ;; + GUESS=mips-sni-sysv4 + ;; RM*:SINIX-*:*:*) - echo mips-sni-sysv4 - exit ;; + GUESS=mips-sni-sysv4 + ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo "$UNAME_MACHINE"-sni-sysv4 + GUESS=$UNAME_MACHINE-sni-sysv4 else - echo ns32k-sni-sysv + GUESS=ns32k-sni-sysv fi - exit ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + ;; + PENTIUM:*:4.0*:*) # Unisys 'ClearPath HMP IX 4000' SVR4/MP effort # says - echo i586-unisys-sysv4 - exit ;; + GUESS=i586-unisys-sysv4 + ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes . # How about differentiating between stratus architectures? -djm - echo hppa1.1-stratus-sysv4 - exit ;; + GUESS=hppa1.1-stratus-sysv4 + ;; *:*:*:FTX*) # From seanf@swdc.stratus.com. - echo i860-stratus-sysv4 - exit ;; + GUESS=i860-stratus-sysv4 + ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. - echo "$UNAME_MACHINE"-stratus-vos - exit ;; + GUESS=$UNAME_MACHINE-stratus-vos + ;; *:VOS:*:*) # From Paul.Green@stratus.com. - echo hppa1.1-stratus-vos - exit ;; + GUESS=hppa1.1-stratus-vos + ;; mc68*:A/UX:*:*) - echo m68k-apple-aux"$UNAME_RELEASE" - exit ;; + GUESS=m68k-apple-aux$UNAME_RELEASE + ;; news*:NEWS-OS:6*:*) - echo mips-sony-newsos6 - exit ;; + GUESS=mips-sony-newsos6 + ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if [ -d /usr/nec ]; then - echo mips-nec-sysv"$UNAME_RELEASE" + if test -d /usr/nec; then + GUESS=mips-nec-sysv$UNAME_RELEASE else - echo mips-unknown-sysv"$UNAME_RELEASE" + GUESS=mips-unknown-sysv$UNAME_RELEASE fi - exit ;; + ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. - echo powerpc-be-beos - exit ;; + GUESS=powerpc-be-beos + ;; BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. - echo powerpc-apple-beos - exit ;; + GUESS=powerpc-apple-beos + ;; BePC:BeOS:*:*) # BeOS running on Intel PC compatible. - echo i586-pc-beos - exit ;; + GUESS=i586-pc-beos + ;; BePC:Haiku:*:*) # Haiku running on Intel PC compatible. - echo i586-pc-haiku - exit ;; - x86_64:Haiku:*:*) - echo x86_64-unknown-haiku - exit ;; + GUESS=i586-pc-haiku + ;; + ppc:Haiku:*:*) # Haiku running on Apple PowerPC + GUESS=powerpc-apple-haiku + ;; + *:Haiku:*:*) # Haiku modern gcc (not bound by BeOS compat) + GUESS=$UNAME_MACHINE-unknown-haiku + ;; SX-4:SUPER-UX:*:*) - echo sx4-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sx4-nec-superux$UNAME_RELEASE + ;; SX-5:SUPER-UX:*:*) - echo sx5-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sx5-nec-superux$UNAME_RELEASE + ;; SX-6:SUPER-UX:*:*) - echo sx6-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sx6-nec-superux$UNAME_RELEASE + ;; SX-7:SUPER-UX:*:*) - echo sx7-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sx7-nec-superux$UNAME_RELEASE + ;; SX-8:SUPER-UX:*:*) - echo sx8-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sx8-nec-superux$UNAME_RELEASE + ;; SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sx8r-nec-superux$UNAME_RELEASE + ;; SX-ACE:SUPER-UX:*:*) - echo sxace-nec-superux"$UNAME_RELEASE" - exit ;; + GUESS=sxace-nec-superux$UNAME_RELEASE + ;; Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody"$UNAME_RELEASE" - exit ;; + GUESS=powerpc-apple-rhapsody$UNAME_RELEASE + ;; *:Rhapsody:*:*) - echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-apple-rhapsody$UNAME_RELEASE + ;; + arm64:Darwin:*:*) + GUESS=aarch64-apple-darwin$UNAME_RELEASE + ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` case $UNAME_PROCESSOR in @@ -1338,7 +1471,7 @@ EOF else set_cc_for_build fi - if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if test "$CC_FOR_BUILD" != no_compiler_found; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null @@ -1359,109 +1492,122 @@ EOF # uname -m returns i386 or x86_64 UNAME_PROCESSOR=$UNAME_MACHINE fi - echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_PROCESSOR-apple-darwin$UNAME_RELEASE + ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi - echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_PROCESSOR-$UNAME_MACHINE-nto-qnx$UNAME_RELEASE + ;; *:QNX:*:4*) - echo i386-pc-qnx - exit ;; + GUESS=i386-pc-qnx + ;; NEO-*:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk"$UNAME_RELEASE" - exit ;; + GUESS=neo-tandem-nsk$UNAME_RELEASE + ;; NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk"$UNAME_RELEASE" - exit ;; + GUESS=nse-tandem-nsk$UNAME_RELEASE + ;; NSR-*:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk"$UNAME_RELEASE" - exit ;; + GUESS=nsr-tandem-nsk$UNAME_RELEASE + ;; NSV-*:NONSTOP_KERNEL:*:*) - echo nsv-tandem-nsk"$UNAME_RELEASE" - exit ;; + GUESS=nsv-tandem-nsk$UNAME_RELEASE + ;; NSX-*:NONSTOP_KERNEL:*:*) - echo nsx-tandem-nsk"$UNAME_RELEASE" - exit ;; + GUESS=nsx-tandem-nsk$UNAME_RELEASE + ;; *:NonStop-UX:*:*) - echo mips-compaq-nonstopux - exit ;; + GUESS=mips-compaq-nonstopux + ;; BS2000:POSIX*:*:*) - echo bs2000-siemens-sysv - exit ;; + GUESS=bs2000-siemens-sysv + ;; DS/*:UNIX_System_V:*:*) - echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-$UNAME_SYSTEM-$UNAME_RELEASE + ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - # shellcheck disable=SC2154 - if test "$cputype" = 386; then + if test "${cputype-}" = 386; then UNAME_MACHINE=i386 - else - UNAME_MACHINE="$cputype" + elif test "x${cputype-}" != x; then + UNAME_MACHINE=$cputype fi - echo "$UNAME_MACHINE"-unknown-plan9 - exit ;; + GUESS=$UNAME_MACHINE-unknown-plan9 + ;; *:TOPS-10:*:*) - echo pdp10-unknown-tops10 - exit ;; + GUESS=pdp10-unknown-tops10 + ;; *:TENEX:*:*) - echo pdp10-unknown-tenex - exit ;; + GUESS=pdp10-unknown-tenex + ;; KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) - echo pdp10-dec-tops20 - exit ;; + GUESS=pdp10-dec-tops20 + ;; XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) - echo pdp10-xkl-tops20 - exit ;; + GUESS=pdp10-xkl-tops20 + ;; *:TOPS-20:*:*) - echo pdp10-unknown-tops20 - exit ;; + GUESS=pdp10-unknown-tops20 + ;; *:ITS:*:*) - echo pdp10-unknown-its - exit ;; + GUESS=pdp10-unknown-its + ;; SEI:*:*:SEIUX) - echo mips-sei-seiux"$UNAME_RELEASE" - exit ;; + GUESS=mips-sei-seiux$UNAME_RELEASE + ;; *:DragonFly:*:*) - echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" - exit ;; + DRAGONFLY_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` + GUESS=$UNAME_MACHINE-unknown-dragonfly$DRAGONFLY_REL + ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` - case "$UNAME_MACHINE" in - A*) echo alpha-dec-vms ; exit ;; - I*) echo ia64-dec-vms ; exit ;; - V*) echo vax-dec-vms ; exit ;; + case $UNAME_MACHINE in + A*) GUESS=alpha-dec-vms ;; + I*) GUESS=ia64-dec-vms ;; + V*) GUESS=vax-dec-vms ;; esac ;; *:XENIX:*:SysV) - echo i386-pc-xenix - exit ;; + GUESS=i386-pc-xenix + ;; i*86:skyos:*:*) - echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" - exit ;; + SKYOS_REL=`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'` + GUESS=$UNAME_MACHINE-pc-skyos$SKYOS_REL + ;; i*86:rdos:*:*) - echo "$UNAME_MACHINE"-pc-rdos - exit ;; - i*86:AROS:*:*) - echo "$UNAME_MACHINE"-pc-aros - exit ;; + GUESS=$UNAME_MACHINE-pc-rdos + ;; + i*86:Fiwix:*:*) + GUESS=$UNAME_MACHINE-pc-fiwix + ;; + *:AROS:*:*) + GUESS=$UNAME_MACHINE-unknown-aros + ;; x86_64:VMkernel:*:*) - echo "$UNAME_MACHINE"-unknown-esx - exit ;; + GUESS=$UNAME_MACHINE-unknown-esx + ;; amd64:Isilon\ OneFS:*:*) - echo x86_64-unknown-onefs - exit ;; + GUESS=x86_64-unknown-onefs + ;; *:Unleashed:*:*) - echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" - exit ;; + GUESS=$UNAME_MACHINE-unknown-unleashed$UNAME_RELEASE + ;; + *:Ironclad:*:*) + GUESS=$UNAME_MACHINE-unknown-ironclad + ;; esac +# Do we have a guess based on uname results? +if test "x$GUESS" != x; then + echo "$GUESS" + exit +fi + # No uname command or uname output not recognized. set_cc_for_build cat > "$dummy.c" < "$dummy.c" </dev/null && SYSTEM_NAME=`$dummy` && +$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } # Apollos put the system type in the environment. @@ -1601,7 +1748,7 @@ test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } echo "$0: unable to guess system type" >&2 -case "$UNAME_MACHINE:$UNAME_SYSTEM" in +case $UNAME_MACHINE:$UNAME_SYSTEM in mips:Linux | mips64:Linux) # If we got here on MIPS GNU/Linux, output extra information. cat >&2 <&2 <&2 + echo "Invalid configuration '$1': more than four components" >&2 exit 1 ;; *-*-*-*) basic_machine=$field1-$field2 - os=$field3-$field4 + basic_os=$field3-$field4 ;; *-*-*) # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two # parts maybe_os=$field2-$field3 case $maybe_os in - nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ - | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ - | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ - | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) + cloudabi*-eabi* \ + | kfreebsd*-gnu* \ + | knetbsd*-gnu* \ + | kopensolaris*-gnu* \ + | linux-* \ + | managarm-* \ + | netbsd*-eabi* \ + | netbsd*-gnu* \ + | nto-qnx* \ + | os2-emx* \ + | rtmk-nova* \ + | storm-chaos* \ + | uclinux-gnu* \ + | uclinux-uclibc* \ + | windows-* ) basic_machine=$field1 - os=$maybe_os + basic_os=$maybe_os ;; android-linux) basic_machine=$field1-unknown - os=linux-android + basic_os=linux-android ;; *) basic_machine=$field1-$field2 - os=$field3 + basic_os=$field3 ;; esac ;; *-*) - # A lone config we happen to match not fitting any pattern case $field1-$field2 in + # Shorthands that happen to contain a single dash + convex-c[12] | convex-c3[248]) + basic_machine=$field2-convex + basic_os= + ;; decstation-3100) basic_machine=mips-dec - os= + basic_os= ;; *-*) # Second component is usually, but not always the OS case $field2 in - # Prevent following clause from handling this valid os + # Do not treat sunos as a manufacturer sun*os*) basic_machine=$field1 - os=$field2 + basic_os=$field2 ;; # Manufacturers - dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ - | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ - | unicom* | ibm* | next | hp | isi* | apollo | altos* \ - | convergent* | ncr* | news | 32* | 3600* | 3100* \ - | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ - | ultra | tti* | harris | dolphin | highlevel | gould \ - | cbm | ns | masscomp | apple | axis | knuth | cray \ - | microblaze* | sim | cisco \ - | oki | wec | wrs | winbond) + 3100* \ + | 32* \ + | 3300* \ + | 3600* \ + | 7300* \ + | acorn \ + | altos* \ + | apollo \ + | apple \ + | atari \ + | att* \ + | axis \ + | be \ + | bull \ + | cbm \ + | ccur \ + | cisco \ + | commodore \ + | convergent* \ + | convex* \ + | cray \ + | crds \ + | dec* \ + | delta* \ + | dg \ + | digital \ + | dolphin \ + | encore* \ + | gould \ + | harris \ + | highlevel \ + | hitachi* \ + | hp \ + | ibm* \ + | intergraph \ + | isi* \ + | knuth \ + | masscomp \ + | microblaze* \ + | mips* \ + | motorola* \ + | ncr* \ + | news \ + | next \ + | ns \ + | oki \ + | omron* \ + | pc533* \ + | rebel \ + | rom68k \ + | rombug \ + | semi \ + | sequent* \ + | siemens \ + | sgi* \ + | siemens \ + | sim \ + | sni \ + | sony* \ + | stratus \ + | sun \ + | sun[234]* \ + | tektronix \ + | tti* \ + | ultra \ + | unicom* \ + | wec \ + | winbond \ + | wrs) basic_machine=$field1-$field2 - os= + basic_os= + ;; + zephyr*) + basic_machine=$field1-unknown + basic_os=$field2 ;; *) basic_machine=$field1 - os=$field2 + basic_os=$field2 ;; esac ;; @@ -191,450 +279,431 @@ case $1 in case $field1 in 386bsd) basic_machine=i386-pc - os=bsd + basic_os=bsd ;; a29khif) basic_machine=a29k-amd - os=udi + basic_os=udi ;; adobe68k) basic_machine=m68010-adobe - os=scout + basic_os=scout ;; alliant) basic_machine=fx80-alliant - os= + basic_os= ;; altos | altos3068) basic_machine=m68k-altos - os= + basic_os= ;; am29k) basic_machine=a29k-none - os=bsd + basic_os=bsd ;; amdahl) basic_machine=580-amdahl - os=sysv + basic_os=sysv ;; amiga) basic_machine=m68k-unknown - os= + basic_os= ;; amigaos | amigados) basic_machine=m68k-unknown - os=amigaos + basic_os=amigaos ;; amigaunix | amix) basic_machine=m68k-unknown - os=sysv4 + basic_os=sysv4 ;; apollo68) basic_machine=m68k-apollo - os=sysv + basic_os=sysv ;; apollo68bsd) basic_machine=m68k-apollo - os=bsd + basic_os=bsd ;; aros) basic_machine=i386-pc - os=aros + basic_os=aros ;; aux) basic_machine=m68k-apple - os=aux + basic_os=aux ;; balance) basic_machine=ns32k-sequent - os=dynix + basic_os=dynix ;; blackfin) basic_machine=bfin-unknown - os=linux + basic_os=linux ;; cegcc) basic_machine=arm-unknown - os=cegcc - ;; - convex-c1) - basic_machine=c1-convex - os=bsd - ;; - convex-c2) - basic_machine=c2-convex - os=bsd - ;; - convex-c32) - basic_machine=c32-convex - os=bsd - ;; - convex-c34) - basic_machine=c34-convex - os=bsd - ;; - convex-c38) - basic_machine=c38-convex - os=bsd + basic_os=cegcc ;; cray) basic_machine=j90-cray - os=unicos + basic_os=unicos ;; crds | unos) basic_machine=m68k-crds - os= + basic_os= ;; da30) basic_machine=m68k-da30 - os= + basic_os= ;; decstation | pmax | pmin | dec3100 | decstatn) basic_machine=mips-dec - os= + basic_os= ;; delta88) basic_machine=m88k-motorola - os=sysv3 + basic_os=sysv3 ;; dicos) basic_machine=i686-pc - os=dicos + basic_os=dicos ;; djgpp) basic_machine=i586-pc - os=msdosdjgpp + basic_os=msdosdjgpp ;; ebmon29k) basic_machine=a29k-amd - os=ebmon + basic_os=ebmon ;; es1800 | OSE68k | ose68k | ose | OSE) basic_machine=m68k-ericsson - os=ose + basic_os=ose ;; gmicro) basic_machine=tron-gmicro - os=sysv + basic_os=sysv ;; go32) basic_machine=i386-pc - os=go32 + basic_os=go32 ;; h8300hms) basic_machine=h8300-hitachi - os=hms + basic_os=hms ;; h8300xray) basic_machine=h8300-hitachi - os=xray + basic_os=xray ;; h8500hms) basic_machine=h8500-hitachi - os=hms + basic_os=hms ;; harris) basic_machine=m88k-harris - os=sysv3 + basic_os=sysv3 ;; - hp300) + hp300 | hp300hpux) basic_machine=m68k-hp + basic_os=hpux ;; hp300bsd) basic_machine=m68k-hp - os=bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=hpux + basic_os=bsd ;; hppaosf) basic_machine=hppa1.1-hp - os=osf + basic_os=osf ;; hppro) basic_machine=hppa1.1-hp - os=proelf + basic_os=proelf ;; i386mach) basic_machine=i386-mach - os=mach - ;; - vsta) - basic_machine=i386-pc - os=vsta + basic_os=mach ;; isi68 | isi) basic_machine=m68k-isi - os=sysv + basic_os=sysv ;; m68knommu) basic_machine=m68k-unknown - os=linux + basic_os=linux ;; magnum | m3230) basic_machine=mips-mips - os=sysv + basic_os=sysv ;; merlin) basic_machine=ns32k-utek - os=sysv + basic_os=sysv ;; mingw64) basic_machine=x86_64-pc - os=mingw64 + basic_os=mingw64 ;; mingw32) basic_machine=i686-pc - os=mingw32 + basic_os=mingw32 ;; mingw32ce) basic_machine=arm-unknown - os=mingw32ce + basic_os=mingw32ce ;; monitor) basic_machine=m68k-rom68k - os=coff + basic_os=coff ;; morphos) basic_machine=powerpc-unknown - os=morphos + basic_os=morphos ;; moxiebox) basic_machine=moxie-unknown - os=moxiebox + basic_os=moxiebox ;; msdos) basic_machine=i386-pc - os=msdos + basic_os=msdos ;; msys) basic_machine=i686-pc - os=msys + basic_os=msys ;; mvs) basic_machine=i370-ibm - os=mvs + basic_os=mvs ;; nacl) basic_machine=le32-unknown - os=nacl + basic_os=nacl ;; ncr3000) basic_machine=i486-ncr - os=sysv4 + basic_os=sysv4 ;; netbsd386) basic_machine=i386-pc - os=netbsd + basic_os=netbsd ;; netwinder) basic_machine=armv4l-rebel - os=linux + basic_os=linux ;; news | news700 | news800 | news900) basic_machine=m68k-sony - os=newsos + basic_os=newsos ;; news1000) basic_machine=m68030-sony - os=newsos + basic_os=newsos ;; necv70) basic_machine=v70-nec - os=sysv + basic_os=sysv ;; nh3000) basic_machine=m68k-harris - os=cxux + basic_os=cxux ;; nh[45]000) basic_machine=m88k-harris - os=cxux + basic_os=cxux ;; nindy960) basic_machine=i960-intel - os=nindy + basic_os=nindy ;; mon960) basic_machine=i960-intel - os=mon960 + basic_os=mon960 ;; nonstopux) basic_machine=mips-compaq - os=nonstopux + basic_os=nonstopux ;; os400) basic_machine=powerpc-ibm - os=os400 + basic_os=os400 ;; OSE68000 | ose68000) basic_machine=m68000-ericsson - os=ose + basic_os=ose ;; os68k) basic_machine=m68k-none - os=os68k + basic_os=os68k ;; paragon) basic_machine=i860-intel - os=osf + basic_os=osf ;; parisc) basic_machine=hppa-unknown - os=linux + basic_os=linux + ;; + psp) + basic_machine=mipsallegrexel-sony + basic_os=psp ;; pw32) basic_machine=i586-unknown - os=pw32 + basic_os=pw32 ;; rdos | rdos64) basic_machine=x86_64-pc - os=rdos + basic_os=rdos ;; rdos32) basic_machine=i386-pc - os=rdos + basic_os=rdos ;; rom68k) basic_machine=m68k-rom68k - os=coff + basic_os=coff ;; sa29200) basic_machine=a29k-amd - os=udi + basic_os=udi ;; sei) basic_machine=mips-sei - os=seiux + basic_os=seiux ;; sequent) basic_machine=i386-sequent - os= + basic_os= ;; sps7) basic_machine=m68k-bull - os=sysv2 + basic_os=sysv2 ;; st2000) basic_machine=m68k-tandem - os= + basic_os= ;; stratus) basic_machine=i860-stratus - os=sysv4 + basic_os=sysv4 ;; sun2) basic_machine=m68000-sun - os= + basic_os= ;; sun2os3) basic_machine=m68000-sun - os=sunos3 + basic_os=sunos3 ;; sun2os4) basic_machine=m68000-sun - os=sunos4 + basic_os=sunos4 ;; sun3) basic_machine=m68k-sun - os= + basic_os= ;; sun3os3) basic_machine=m68k-sun - os=sunos3 + basic_os=sunos3 ;; sun3os4) basic_machine=m68k-sun - os=sunos4 + basic_os=sunos4 ;; sun4) basic_machine=sparc-sun - os= + basic_os= ;; sun4os3) basic_machine=sparc-sun - os=sunos3 + basic_os=sunos3 ;; sun4os4) basic_machine=sparc-sun - os=sunos4 + basic_os=sunos4 ;; sun4sol2) basic_machine=sparc-sun - os=solaris2 + basic_os=solaris2 ;; sun386 | sun386i | roadrunner) basic_machine=i386-sun - os= + basic_os= ;; sv1) basic_machine=sv1-cray - os=unicos + basic_os=unicos ;; symmetry) basic_machine=i386-sequent - os=dynix + basic_os=dynix ;; t3e) basic_machine=alphaev5-cray - os=unicos + basic_os=unicos ;; t90) basic_machine=t90-cray - os=unicos + basic_os=unicos ;; toad1) basic_machine=pdp10-xkl - os=tops20 + basic_os=tops20 ;; tpf) basic_machine=s390x-ibm - os=tpf + basic_os=tpf ;; udi29k) basic_machine=a29k-amd - os=udi + basic_os=udi ;; ultra3) basic_machine=a29k-nyu - os=sym1 + basic_os=sym1 ;; v810 | necv810) basic_machine=v810-nec - os=none + basic_os=none ;; vaxv) basic_machine=vax-dec - os=sysv + basic_os=sysv ;; vms) basic_machine=vax-dec - os=vms + basic_os=vms + ;; + vsta) + basic_machine=i386-pc + basic_os=vsta ;; vxworks960) basic_machine=i960-wrs - os=vxworks + basic_os=vxworks ;; vxworks68) basic_machine=m68k-wrs - os=vxworks + basic_os=vxworks ;; vxworks29k) basic_machine=a29k-wrs - os=vxworks + basic_os=vxworks ;; xbox) basic_machine=i686-pc - os=mingw32 + basic_os=mingw32 ;; ymp) basic_machine=ymp-cray - os=unicos + basic_os=unicos ;; *) basic_machine=$1 - os= + basic_os= ;; esac ;; @@ -686,27 +755,38 @@ case $basic_machine in bluegene*) cpu=powerpc vendor=ibm - os=cnk + basic_os=cnk ;; decsystem10* | dec10*) cpu=pdp10 vendor=dec - os=tops10 + basic_os=tops10 ;; decsystem20* | dec20*) cpu=pdp10 vendor=dec - os=tops20 + basic_os=tops20 ;; - delta | 3300 | motorola-3300 | motorola-delta \ - | 3300-motorola | delta-motorola) + delta | 3300 | delta-motorola | 3300-motorola | motorola-delta | motorola-3300) cpu=m68k vendor=motorola ;; - dpx2*) + # This used to be dpx2*, but that gets the RS6000-based + # DPX/20 and the x86-based DPX/2-100 wrong. See + # https://oldskool.silicium.org/stations/bull_dpx20.htm + # https://www.feb-patrimoine.com/english/bull_dpx2.htm + # https://www.feb-patrimoine.com/english/unix_and_bull.htm + dpx2 | dpx2[23]00 | dpx2[23]xx) cpu=m68k vendor=bull - os=sysv3 + ;; + dpx2100 | dpx21xx) + cpu=i386 + vendor=bull + ;; + dpx20) + cpu=rs6000 + vendor=bull ;; encore | umax | mmax) cpu=ns32k @@ -715,7 +795,7 @@ case $basic_machine in elxsi) cpu=elxsi vendor=elxsi - os=${os:-bsd} + basic_os=${basic_os:-bsd} ;; fx2800) cpu=i860 @@ -728,7 +808,7 @@ case $basic_machine in h3050r* | hiux*) cpu=hppa1.1 vendor=hitachi - os=hiuxwe2 + basic_os=hiuxwe2 ;; hp3k9[0-9][0-9] | hp9[0-9][0-9]) cpu=hppa1.0 @@ -771,36 +851,36 @@ case $basic_machine in i*86v32) cpu=`echo "$1" | sed -e 's/86.*/86/'` vendor=pc - os=sysv32 + basic_os=sysv32 ;; i*86v4*) cpu=`echo "$1" | sed -e 's/86.*/86/'` vendor=pc - os=sysv4 + basic_os=sysv4 ;; i*86v) cpu=`echo "$1" | sed -e 's/86.*/86/'` vendor=pc - os=sysv + basic_os=sysv ;; i*86sol2) cpu=`echo "$1" | sed -e 's/86.*/86/'` vendor=pc - os=solaris2 + basic_os=solaris2 ;; j90 | j90-cray) cpu=j90 vendor=cray - os=${os:-unicos} + basic_os=${basic_os:-unicos} ;; iris | iris4d) cpu=mips vendor=sgi - case $os in + case $basic_os in irix*) ;; *) - os=irix4 + basic_os=irix4 ;; esac ;; @@ -811,28 +891,16 @@ case $basic_machine in *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) cpu=m68k vendor=atari - os=mint + basic_os=mint ;; news-3600 | risc-news) cpu=mips vendor=sony - os=newsos + basic_os=newsos ;; next | m*-next) cpu=m68k vendor=next - case $os in - openstep*) - ;; - nextstep*) - ;; - ns2*) - os=nextstep2 - ;; - *) - os=nextstep3 - ;; - esac ;; np1) cpu=np1 @@ -841,12 +909,12 @@ case $basic_machine in op50n-* | op60c-*) cpu=hppa1.1 vendor=oki - os=proelf + basic_os=proelf ;; pa-hitachi) cpu=hppa1.1 vendor=hitachi - os=hiuxwe2 + basic_os=hiuxwe2 ;; pbd) cpu=sparc @@ -883,12 +951,12 @@ case $basic_machine in sde) cpu=mipsisa32 vendor=sde - os=${os:-elf} + basic_os=${basic_os:-elf} ;; simso-wrs) cpu=sparclite vendor=wrs - os=vxworks + basic_os=vxworks ;; tower | tower-32) cpu=m68k @@ -905,7 +973,7 @@ case $basic_machine in w89k-*) cpu=hppa1.1 vendor=winbond - os=proelf + basic_os=proelf ;; none) cpu=none @@ -921,12 +989,13 @@ case $basic_machine in ;; *-*) - # shellcheck disable=SC2162 + saved_IFS=$IFS IFS="-" read cpu vendor <&2 + echo "Invalid configuration '$1': machine '$cpu-$vendor' not recognized" 1>&2 exit 1 ;; esac @@ -1278,8 +1491,53 @@ esac # Decode manufacturer-specific aliases for certain operating systems. -if [ x$os != x ] +if test x"$basic_os" != x then + +# First recognize some ad-hoc cases, or perhaps split kernel-os, or else just +# set os. +obj= +case $basic_os in + gnu/linux*) + kernel=linux + os=`echo "$basic_os" | sed -e 's|gnu/linux|gnu|'` + ;; + os2-emx) + kernel=os2 + os=`echo "$basic_os" | sed -e 's|os2-emx|emx|'` + ;; + nto-qnx*) + kernel=nto + os=`echo "$basic_os" | sed -e 's|nto-qnx|qnx|'` + ;; + *-*) + saved_IFS=$IFS + IFS="-" read kernel os <&2 - exit 1 + # No normalization, but not necessarily accepted, that comes below. ;; esac + else # Here we handle the default operating systems that come with various machines. @@ -1533,42 +1744,54 @@ else # will signal an error saying that MANUFACTURER isn't an operating # system, and we'll never get to this point. +kernel= +obj= case $cpu-$vendor in score-*) - os=elf + os= + obj=elf ;; spu-*) - os=elf + os= + obj=elf ;; *-acorn) os=riscix1.2 ;; arm*-rebel) - os=linux + kernel=linux + os=gnu ;; arm*-semi) - os=aout + os= + obj=aout ;; c4x-* | tic4x-*) - os=coff + os= + obj=coff ;; c8051-*) - os=elf + os= + obj=elf ;; clipper-intergraph) os=clix ;; hexagon-*) - os=elf + os= + obj=elf ;; tic54x-*) - os=coff + os= + obj=coff ;; tic55x-*) - os=coff + os= + obj=coff ;; tic6x-*) - os=coff + os= + obj=coff ;; # This must come before the *-dec entry. pdp10-*) @@ -1590,28 +1813,43 @@ case $cpu-$vendor in os=sunos3 ;; m68*-cisco) - os=aout + os= + obj=aout ;; mep-*) - os=elf + os= + obj=elf + ;; + # The -sgi and -siemens entries must be before the mips- entry + # or we get the wrong os. + *-sgi) + os=irix + ;; + *-siemens) + os=sysv4 ;; mips*-cisco) - os=elf + os= + obj=elf ;; - mips*-*) - os=elf + mips*-*|nanomips*-*) + os= + obj=elf ;; or32-*) - os=coff + os= + obj=coff ;; - *-tti) # must be before sparc entry or we get the wrong os. + # This must be before the sparc-* entry or we get the wrong os. + *-tti) os=sysv3 ;; sparc-* | *-sun) os=sunos4.1.1 ;; pru-*) - os=elf + os= + obj=elf ;; *-be) os=beos @@ -1635,7 +1873,7 @@ case $cpu-$vendor in os=hpux ;; *-hitachi) - os=hiux + os=hiuxwe2 ;; i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) os=sysv @@ -1679,12 +1917,6 @@ case $cpu-$vendor in *-encore) os=bsd ;; - *-sgi) - os=irix - ;; - *-siemens) - os=sysv4 - ;; *-masscomp) os=rtu ;; @@ -1692,10 +1924,12 @@ case $cpu-$vendor in os=uxpv ;; *-rom68k) - os=coff + os= + obj=coff ;; *-*bug) - os=coff + os= + obj=coff ;; *-apple) os=macos @@ -1710,84 +1944,406 @@ case $cpu-$vendor in os=none ;; esac + fi +# Now, validate our (potentially fixed-up) individual pieces (OS, OBJ). + +case $os in + # Sometimes we do "kernel-libc", so those need to count as OSes. + llvm* | musl* | newlib* | relibc* | uclibc*) + ;; + # Likewise for "kernel-abi" + eabi* | gnueabi*) + ;; + # VxWorks passes extra cpu info in the 4th filed. + simlinux | simwindows | spe) + ;; + # See `case $cpu-$os` validation below + ghcjs) + ;; + # Now accept the basic system types. + # Each alternative MUST end in a * to match a version number. + abug \ + | aix* \ + | amdhsa* \ + | amigados* \ + | amigaos* \ + | android* \ + | aof* \ + | aos* \ + | aros* \ + | atheos* \ + | auroraux* \ + | aux* \ + | beos* \ + | bitrig* \ + | bme* \ + | bosx* \ + | bsd* \ + | cegcc* \ + | chorusos* \ + | chorusrdb* \ + | clix* \ + | cloudabi* \ + | cnk* \ + | conix* \ + | cos* \ + | cxux* \ + | cygwin* \ + | darwin* \ + | dgux* \ + | dicos* \ + | dnix* \ + | domain* \ + | dragonfly* \ + | drops* \ + | ebmon* \ + | ecoff* \ + | ekkobsd* \ + | emscripten* \ + | emx* \ + | es* \ + | fiwix* \ + | freebsd* \ + | fuchsia* \ + | genix* \ + | genode* \ + | glidix* \ + | gnu* \ + | go32* \ + | haiku* \ + | hcos* \ + | hiux* \ + | hms* \ + | hpux* \ + | ieee* \ + | interix* \ + | ios* \ + | iris* \ + | irix* \ + | ironclad* \ + | isc* \ + | its* \ + | l4re* \ + | libertybsd* \ + | lites* \ + | lnews* \ + | luna* \ + | lynxos* \ + | mach* \ + | macos* \ + | magic* \ + | mbr* \ + | midipix* \ + | midnightbsd* \ + | mingw32* \ + | mingw64* \ + | minix* \ + | mint* \ + | mirbsd* \ + | mks* \ + | mlibc* \ + | mmixware* \ + | mon960* \ + | morphos* \ + | moss* \ + | moxiebox* \ + | mpeix* \ + | mpw* \ + | msdos* \ + | msys* \ + | mvs* \ + | nacl* \ + | netbsd* \ + | netware* \ + | newsos* \ + | nextstep* \ + | nindy* \ + | nonstopux* \ + | nova* \ + | nsk* \ + | nucleus* \ + | nx6 \ + | nx7 \ + | oabi* \ + | ohos* \ + | onefs* \ + | openbsd* \ + | openedition* \ + | openstep* \ + | os108* \ + | os2* \ + | os400* \ + | os68k* \ + | os9* \ + | ose* \ + | osf* \ + | oskit* \ + | osx* \ + | palmos* \ + | phoenix* \ + | plan9* \ + | powermax* \ + | powerunix* \ + | proelf* \ + | psos* \ + | psp* \ + | ptx* \ + | pw32* \ + | qnx* \ + | rdos* \ + | redox* \ + | rhapsody* \ + | riscix* \ + | riscos* \ + | rtems* \ + | rtmk* \ + | rtu* \ + | scout* \ + | secbsd* \ + | sei* \ + | serenity* \ + | sim* \ + | skyos* \ + | solaris* \ + | solidbsd* \ + | sortix* \ + | storm-chaos* \ + | sunos \ + | sunos[34]* \ + | superux* \ + | syllable* \ + | sym* \ + | sysv* \ + | tenex* \ + | tirtos* \ + | toppers* \ + | tops10* \ + | tops20* \ + | tpf* \ + | tvos* \ + | twizzler* \ + | uclinux* \ + | udi* \ + | udk* \ + | ultrix* \ + | unicos* \ + | uniplus* \ + | unleashed* \ + | unos* \ + | uwin* \ + | uxpv* \ + | v88r* \ + |*vms* \ + | vos* \ + | vsta* \ + | vxsim* \ + | vxworks* \ + | wasi* \ + | watchos* \ + | wince* \ + | windiss* \ + | windows* \ + | winnt* \ + | xenix* \ + | xray* \ + | zephyr* \ + | zvmoe* ) + ;; + # This one is extra strict with allowed versions + sco3.2v2 | sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + # This refers to builds using the UEFI calling convention + # (which depends on the architecture) and PE file format. + # Note that this is both a different calling convention and + # different file format than that of GNU-EFI + # (x86_64-w64-mingw32). + uefi) + ;; + none) + ;; + kernel* | msvc* ) + # Restricted further below + ;; + '') + if test x"$obj" = x + then + echo "Invalid configuration '$1': Blank OS only allowed with explicit machine code file format" 1>&2 + fi + ;; + *) + echo "Invalid configuration '$1': OS '$os' not recognized" 1>&2 + exit 1 + ;; +esac + +case $obj in + aout* | coff* | elf* | pe*) + ;; + '') + # empty is fine + ;; + *) + echo "Invalid configuration '$1': Machine code format '$obj' not recognized" 1>&2 + exit 1 + ;; +esac + +# Here we handle the constraint that a (synthetic) cpu and os are +# valid only in combination with each other and nowhere else. +case $cpu-$os in + # The "javascript-unknown-ghcjs" triple is used by GHC; we + # accept it here in order to tolerate that, but reject any + # variations. + javascript-ghcjs) + ;; + javascript-* | *-ghcjs) + echo "Invalid configuration '$1': cpu '$cpu' is not valid with os '$os$obj'" 1>&2 + exit 1 + ;; +esac + +# As a final step for OS-related things, validate the OS-kernel combination +# (given a valid OS), if there is a kernel. +case $kernel-$os-$obj in + linux-gnu*- | linux-android*- | linux-dietlibc*- | linux-llvm*- \ + | linux-mlibc*- | linux-musl*- | linux-newlib*- \ + | linux-relibc*- | linux-uclibc*- | linux-ohos*- ) + ;; + uclinux-uclibc*- | uclinux-gnu*- ) + ;; + managarm-mlibc*- | managarm-kernel*- ) + ;; + windows*-msvc*-) + ;; + -dietlibc*- | -llvm*- | -mlibc*- | -musl*- | -newlib*- | -relibc*- \ + | -uclibc*- ) + # These are just libc implementations, not actual OSes, and thus + # require a kernel. + echo "Invalid configuration '$1': libc '$os' needs explicit kernel." 1>&2 + exit 1 + ;; + -kernel*- ) + echo "Invalid configuration '$1': '$os' needs explicit kernel." 1>&2 + exit 1 + ;; + *-kernel*- ) + echo "Invalid configuration '$1': '$kernel' does not support '$os'." 1>&2 + exit 1 + ;; + *-msvc*- ) + echo "Invalid configuration '$1': '$os' needs 'windows'." 1>&2 + exit 1 + ;; + kfreebsd*-gnu*- | knetbsd*-gnu*- | netbsd*-gnu*- | kopensolaris*-gnu*-) + ;; + vxworks-simlinux- | vxworks-simwindows- | vxworks-spe-) + ;; + nto-qnx*-) + ;; + os2-emx-) + ;; + rtmk-nova-) + ;; + *-eabi*- | *-gnueabi*-) + ;; + none--*) + # None (no kernel, i.e. freestanding / bare metal), + # can be paired with an machine code file format + ;; + -*-) + # Blank kernel with real OS is always fine. + ;; + --*) + # Blank kernel and OS with real machine code file format is always fine. + ;; + *-*-*) + echo "Invalid configuration '$1': Kernel '$kernel' not known to work with OS '$os'." 1>&2 + exit 1 + ;; +esac + # Here we handle the case where we know the os, and the CPU type, but not the # manufacturer. We pick the logical manufacturer. case $vendor in unknown) - case $os in - riscix*) + case $cpu-$os in + *-riscix*) vendor=acorn ;; - sunos*) + *-sunos* | *-solaris*) vendor=sun ;; - cnk*|-aix*) + *-cnk* | *-aix*) vendor=ibm ;; - beos*) + *-beos*) vendor=be ;; - hpux*) + *-hpux*) vendor=hp ;; - mpeix*) + *-mpeix*) vendor=hp ;; - hiux*) + *-hiux*) vendor=hitachi ;; - unos*) + *-unos*) vendor=crds ;; - dgux*) + *-dgux*) vendor=dg ;; - luna*) + *-luna*) vendor=omron ;; - genix*) + *-genix*) vendor=ns ;; - clix*) + *-clix*) vendor=intergraph ;; - mvs* | opened*) + *-mvs* | *-opened*) + vendor=ibm + ;; + *-os400*) vendor=ibm ;; - os400*) + s390-* | s390x-*) vendor=ibm ;; - ptx*) + *-ptx*) vendor=sequent ;; - tpf*) + *-tpf*) vendor=ibm ;; - vxsim* | vxworks* | windiss*) + *-vxsim* | *-vxworks* | *-windiss*) vendor=wrs ;; - aux*) + *-aux*) vendor=apple ;; - hms*) + *-hms*) vendor=hitachi ;; - mpw* | macos*) + *-mpw* | *-macos*) vendor=apple ;; - *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) vendor=atari ;; - vos*) + *-vos*) vendor=stratus ;; esac ;; esac -echo "$cpu-$vendor-$os" +echo "$cpu-$vendor${kernel:+-$kernel}${os:+-$os}${obj:+-$obj}" exit # Local variables: diff --git a/autosetup/autosetup-find-tclsh b/autosetup/autosetup-find-tclsh new file mode 100755 index 0000000000..9f6d6e9402 --- /dev/null +++ b/autosetup/autosetup-find-tclsh @@ -0,0 +1,16 @@ +#!/bin/sh +# Looks for a suitable tclsh or jimsh in the PATH +# If not found, builds a bootstrap jimsh in current dir from source +# Prefer $autosetup_tclsh if is set in the environment (unless ./jimsh0 works) +# If an argument is given, use that as the test instead of autosetup-test-tclsh +d="`dirname "$0"`" +for tclsh in ./jimsh0 $autosetup_tclsh jimsh tclsh tclsh8.5 tclsh8.6 tclsh8.7; do + { $tclsh "$d/${1-autosetup-test-tclsh}"; } 2>/dev/null && exit 0 +done +echo 1>&2 "No installed jimsh or tclsh, building local bootstrap jimsh0" +for cc in ${CC_FOR_BUILD:-cc} gcc; do + { $cc -o jimsh0 "$d/jimsh0.c"; } 2>/dev/null >/dev/null || continue + ./jimsh0 "$d/${1-autosetup-test-tclsh}" && exit 0 +done +echo 1>&2 "No working C compiler found. Tried ${CC_FOR_BUILD:-cc} and gcc." +echo false diff --git a/autosetup/autosetup-test-tclsh b/autosetup/autosetup-test-tclsh new file mode 100644 index 0000000000..75126d2444 --- /dev/null +++ b/autosetup/autosetup-test-tclsh @@ -0,0 +1,20 @@ +# A small Tcl script to verify that the chosen +# interpreter works. Sometimes we might e.g. pick up +# an interpreter for a different arch. +# Outputs the full path to the interpreter + +if {[catch {info version} version] == 0} { + # This is Jim Tcl + if {$version >= 0.72} { + # Ensure that regexp works + regexp (a.*?) a + puts [info nameofexecutable] + exit 0 + } +} elseif {[catch {info tclversion} version] == 0} { + if {$version >= 8.5 && ![string match 8.5a* [info patchlevel]]} { + puts [info nameofexecutable] + exit 0 + } +} +exit 1 diff --git a/autosetup/cc-db.tcl b/autosetup/cc-db.tcl new file mode 100644 index 0000000000..12f1aed2c9 --- /dev/null +++ b/autosetup/cc-db.tcl @@ -0,0 +1,15 @@ +# Copyright (c) 2011 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# @synopsis: +# +# The 'cc-db' module provides a knowledge-base of system idiosyncrasies. +# In general, this module can always be included. + +use cc + +options {} + +# openbsd needs sys/types.h to detect some system headers +cc-include-needs sys/socket.h sys/types.h +cc-include-needs netinet/in.h sys/types.h diff --git a/autosetup/cc-lib.tcl b/autosetup/cc-lib.tcl new file mode 100644 index 0000000000..01a0fb3877 --- /dev/null +++ b/autosetup/cc-lib.tcl @@ -0,0 +1,187 @@ +# Copyright (c) 2011 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# @synopsis: +# +# Provides a library of common tests on top of the 'cc' module. + +use cc + +# @cc-check-lfs +# +# The equivalent of the 'AC_SYS_LARGEFILE' macro. +# +# defines 'HAVE_LFS' if LFS is available, +# and defines '_FILE_OFFSET_BITS=64' if necessary +# +# Returns 1 if 'LFS' is available or 0 otherwise +# +proc cc-check-lfs {} { + cc-check-includes sys/types.h + msg-checking "Checking if -D_FILE_OFFSET_BITS=64 is needed..." + set lfs 1 + if {[msg-quiet cc-with {-includes sys/types.h} {cc-check-sizeof off_t}] == 8} { + msg-result no + } elseif {[msg-quiet cc-with {-includes sys/types.h -cflags -D_FILE_OFFSET_BITS=64} {cc-check-sizeof off_t}] == 8} { + define _FILE_OFFSET_BITS 64 + msg-result yes + } else { + set lfs 0 + msg-result none + } + define-feature lfs $lfs + return $lfs +} + +# @cc-check-endian +# +# The equivalent of the 'AC_C_BIGENDIAN' macro. +# +# defines 'HAVE_BIG_ENDIAN' if endian is known to be big, +# or 'HAVE_LITTLE_ENDIAN' if endian is known to be little. +# +# Returns 1 if determined, or 0 if not. +# +proc cc-check-endian {} { + cc-check-includes sys/types.h sys/param.h + set rc 0 + msg-checking "Checking endian..." + cc-with {-includes {sys/types.h sys/param.h}} { + if {[cctest -code { + #if !defined(BIG_ENDIAN) || !defined(BYTE_ORDER) + #error unknown + #elif BYTE_ORDER != BIG_ENDIAN + #error little + #endif + }]} { + define-feature big-endian + msg-result "big" + set rc 1 + } elseif {[cctest -code { + #if !defined(LITTLE_ENDIAN) || !defined(BYTE_ORDER) + #error unknown + #elif BYTE_ORDER != LITTLE_ENDIAN + #error big + #endif + }]} { + define-feature little-endian + msg-result "little" + set rc 1 + } else { + msg-result "unknown" + } + } + return $rc +} + +# @cc-check-flags flag ?...? +# +# Checks whether the given C/C++ compiler flags can be used. Defines feature +# names prefixed with 'HAVE_CFLAG' and 'HAVE_CXXFLAG' respectively, and +# appends working flags to '-cflags' and 'AS_CFLAGS' or 'AS_CXXFLAGS'. +proc cc-check-flags {args} { + set result 1 + array set opts [cc-get-settings] + switch -exact -- $opts(-lang) { + c++ { + set lang C++ + set prefix CXXFLAG + } + c { + set lang C + set prefix CFLAG + } + default { + autosetup-error "cc-check-flags failed with unknown language: $opts(-lang)" + } + } + foreach flag $args { + msg-checking "Checking whether the $lang compiler accepts $flag..." + if {[cctest -cflags $flag]} { + msg-result yes + define-feature $prefix$flag + cc-with [list -cflags [list $flag]] + define-append AS_${prefix}S $flag + } else { + msg-result no + set result 0 + } + } + return $result +} + +# @cc-check-standards ver ?...? +# +# Checks whether the C/C++ compiler accepts one of the specified '-std=$ver' +# options, and appends the first working one to '-cflags' and 'AS_CFLAGS' or +# 'AS_CXXFLAGS'. +proc cc-check-standards {args} { + array set opts [cc-get-settings] + foreach std $args { + if {[cc-check-flags -std=$std]} { + return $std + } + } + return "" +} + +# Checks whether $keyword is usable as alignof +proc cctest_alignof {keyword} { + msg-checking "Checking for $keyword..." + if {[cctest -code "int x = ${keyword}(char), y = ${keyword}('x');"]} then { + msg-result ok + define-feature $keyword + } else { + msg-result "not found" + } +} + +# @cc-check-c11 +# +# Checks for several C11/C++11 extensions and their alternatives. Currently +# checks for '_Static_assert', '_Alignof', '__alignof__', '__alignof'. +proc cc-check-c11 {} { + msg-checking "Checking for _Static_assert..." + if {[cctest -code { + _Static_assert(1, "static assertions are available"); + }]} then { + msg-result ok + define-feature _Static_assert + } else { + msg-result "not found" + } + + cctest_alignof _Alignof + cctest_alignof __alignof__ + cctest_alignof __alignof +} + +# @cc-check-alloca +# +# The equivalent of the 'AC_FUNC_ALLOCA' macro. +# +# Checks for the existence of 'alloca' +# defines 'HAVE_ALLOCA' and returns 1 if it exists. +proc cc-check-alloca {} { + cc-check-some-feature alloca { + cctest -includes alloca.h -code { alloca (2 * sizeof (int)); } + } +} + +# @cc-signal-return-type +# +# The equivalent of the 'AC_TYPE_SIGNAL' macro. +# +# defines 'RETSIGTYPE' to 'int' or 'void'. +proc cc-signal-return-type {} { + msg-checking "Checking return type of signal handlers..." + cc-with {-includes {sys/types.h signal.h}} { + if {[cctest -code {return *(signal (0, 0)) (0) == 1;}]} { + set type int + } else { + set type void + } + define RETSIGTYPE $type + msg-result $type + } +} diff --git a/autosetup/cc-shared.tcl b/autosetup/cc-shared.tcl new file mode 100644 index 0000000000..1fa200eec1 --- /dev/null +++ b/autosetup/cc-shared.tcl @@ -0,0 +1,115 @@ +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# @synopsis: +# +# The 'cc-shared' module provides support for shared libraries and shared objects. +# It defines the following variables: +# +## SH_CFLAGS Flags to use compiling sources destined for a shared library +## SH_LDFLAGS Flags to use linking (creating) a shared library +## SH_SOPREFIX Prefix to use to set the soname when creating a shared library +## SH_SOFULLPATH Set to 1 if the shared library soname should include the full install path +## SH_SOEXT Extension for shared libs +## SH_SOEXTVER Format for versioned shared libs - %s = version +## SHOBJ_CFLAGS Flags to use compiling sources destined for a shared object +## SHOBJ_LDFLAGS Flags to use linking a shared object, undefined symbols allowed +## SHOBJ_LDFLAGS_R - as above, but all symbols must be resolved +## SH_LINKRPATH Format for setting the rpath when linking an executable, %s = path +## SH_LINKFLAGS Flags to use linking an executable which will load shared objects +## LD_LIBRARY_PATH Environment variable which specifies path to shared libraries +## STRIPLIBFLAGS Arguments to strip a dynamic library + +options {} + +# Defaults: gcc on unix +define SHOBJ_CFLAGS -fPIC +define SHOBJ_LDFLAGS -shared +define SH_CFLAGS -fPIC +define SH_LDFLAGS -shared +define SH_LINKFLAGS -rdynamic +define SH_LINKRPATH "-Wl,-rpath -Wl,%s" +define SH_SOEXT .so +define SH_SOEXTVER .so.%s +define SH_SOPREFIX -Wl,-soname, +define LD_LIBRARY_PATH LD_LIBRARY_PATH +define STRIPLIBFLAGS --strip-unneeded + +# Note: This is a helpful reference for identifying the toolchain +# http://sourceforge.net/apps/mediawiki/predef/index.php?title=Compilers + +switch -glob -- [get-define host] { + *-*-darwin* { + define SHOBJ_CFLAGS "-dynamic -fno-common" + define SHOBJ_LDFLAGS "-bundle -undefined dynamic_lookup" + define SHOBJ_LDFLAGS_R -bundle + define SH_CFLAGS -dynamic + define SH_LDFLAGS -dynamiclib + define SH_LINKFLAGS "" + define SH_SOEXT .dylib + define SH_SOEXTVER .%s.dylib + define SH_SOPREFIX -Wl,-install_name, + define SH_SOFULLPATH + define LD_LIBRARY_PATH DYLD_LIBRARY_PATH + define STRIPLIBFLAGS -x + } + *-*-ming* - *-*-cygwin - *-*-msys { + define SHOBJ_CFLAGS "" + define SHOBJ_LDFLAGS -shared + define SH_CFLAGS "" + define SH_LDFLAGS -shared + define SH_LINKRPATH "" + define SH_LINKFLAGS "" + define SH_SOEXT .dll + define SH_SOEXTVER .dll + define SH_SOPREFIX "" + define LD_LIBRARY_PATH PATH + } + sparc* { + if {[msg-quiet cc-check-decls __SUNPRO_C]} { + msg-result "Found sun stdio compiler" + # sun stdio compiler + # XXX: These haven't been fully tested. + define SHOBJ_CFLAGS -KPIC + define SHOBJ_LDFLAGS "-G" + define SH_CFLAGS -KPIC + define SH_LINKFLAGS -Wl,-export-dynamic + define SH_SOPREFIX -Wl,-h, + } + } + *-*-solaris* { + if {[msg-quiet cc-check-decls __SUNPRO_C]} { + msg-result "Found sun stdio compiler" + # sun stdio compiler + # XXX: These haven't been fully tested. + define SHOBJ_CFLAGS -KPIC + define SHOBJ_LDFLAGS "-G" + define SH_CFLAGS -KPIC + define SH_LINKFLAGS -Wl,-export-dynamic + define SH_SOPREFIX -Wl,-h, + } + } + *-*-hpux* { + define SHOBJ_CFLAGS +z + define SHOBJ_LDFLAGS -b + define SH_CFLAGS +z + define SH_LDFLAGS -b + define SH_LINKFLAGS -Wl,+s + define SH_LINKRPATH "-Wl,+b -Wl,%s" + define SH_SOPREFIX -Wl,+h, + define STRIPLIBFLAGS -Wl,-s + } + *-*-haiku { + define SHOBJ_CFLAGS "" + define SHOBJ_LDFLAGS -shared + define SH_CFLAGS "" + define SH_LDFLAGS -shared + define SH_LINKFLAGS "" + define SH_SOPREFIX "" + define LD_LIBRARY_PATH LIBRARY_PATH + } +} + +if {![is-defined SHOBJ_LDFLAGS_R]} { + define SHOBJ_LDFLAGS_R [get-define SHOBJ_LDFLAGS] +} diff --git a/autosetup/cc.tcl b/autosetup/cc.tcl new file mode 100644 index 0000000000..05c1b1cf40 --- /dev/null +++ b/autosetup/cc.tcl @@ -0,0 +1,758 @@ +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# @synopsis: +# +# The 'cc' module supports checking various 'features' of the C or C++ +# compiler/linker environment. Common commands are 'cc-check-includes', +# 'cc-check-types', 'cc-check-functions', 'cc-with' and 'make-config-header' +# +# The following environment variables are used if set: +# +## CC - C compiler +## CXX - C++ compiler +## CPP - C preprocessor +## CCACHE - Set to "none" to disable automatic use of ccache +## CPPFLAGS - Additional C preprocessor compiler flags (C and C++), before CFLAGS, CXXFLAGS +## CFLAGS - Additional C compiler flags +## CXXFLAGS - Additional C++ compiler flags +## LDFLAGS - Additional compiler flags during linking +## LINKFLAGS - ?How is this different from LDFLAGS? +## LIBS - Additional libraries to use (for all tests) +## CROSS - Tool prefix for cross compilation +# +# The following variables are defined from the corresponding +# environment variables if set. +# +## CC_FOR_BUILD +## LD + +use system + +options {} + +# Checks for the existence of the given function by linking +# +proc cctest_function {function} { + cctest -link 1 -declare "extern void $function\(void);" -code "$function\();" +} + +# Checks for the existence of the given type by compiling +proc cctest_type {type} { + cctest -code "$type _x;" +} + +# Checks for the existence of the given type/structure member. +# e.g. "struct stat.st_mtime" +proc cctest_member {struct_member} { + # split at the first dot + regexp {^([^.]+)[.](.*)$} $struct_member -> struct member + cctest -code "static $struct _s; return sizeof(_s.$member);" +} + +# Checks for the existence of the given define by compiling +# +proc cctest_define {name} { + cctest -code "#ifndef $name\n#error not defined\n#endif" +} + +# Checks for the existence of the given name either as +# a macro (#define) or an rvalue (such as an enum) +# +proc cctest_decl {name} { + cctest -code "#ifndef $name\n(void)$name;\n#endif" +} + +# @cc-check-sizeof type ... +# +# Checks the size of the given types (between 1 and 32, inclusive). +# Defines a variable with the size determined, or 'unknown' otherwise. +# e.g. for type 'long long', defines 'SIZEOF_LONG_LONG'. +# Returns the size of the last type. +# +proc cc-check-sizeof {args} { + foreach type $args { + msg-checking "Checking for sizeof $type..." + set size unknown + # Try the most common sizes first + foreach i {4 8 1 2 16 32} { + if {[cctest -code "static int _x\[sizeof($type) == $i ? 1 : -1\] = { 1 };"]} { + set size $i + break + } + } + msg-result $size + set define [feature-define-name $type SIZEOF_] + define $define $size + } + # Return the last result + get-define $define +} + +# Checks for each feature in $list by using the given script. +# +# When the script is evaluated, $each is set to the feature +# being checked, and $extra is set to any additional cctest args. +# +# Returns 1 if all features were found, or 0 otherwise. +proc cc-check-some-feature {list script} { + set ret 1 + foreach each $list { + if {![check-feature $each $script]} { + set ret 0 + } + } + return $ret +} + +# @cc-check-includes includes ... +# +# Checks that the given include files can be used. +proc cc-check-includes {args} { + cc-check-some-feature $args { + set with {} + if {[dict exists $::autosetup(cc-include-deps) $each]} { + set deps [dict keys [dict get $::autosetup(cc-include-deps) $each]] + msg-quiet cc-check-includes {*}$deps + foreach i $deps { + if {[have-feature $i]} { + lappend with $i + } + } + } + if {[llength $with]} { + cc-with [list -includes $with] { + cctest -includes $each + } + } else { + cctest -includes $each + } + } +} + +# @cc-include-needs include required ... +# +# Ensures that when checking for '$include', a check is first +# made for each '$required' file, and if found, it is included with '#include'. +proc cc-include-needs {file args} { + foreach depfile $args { + dict set ::autosetup(cc-include-deps) $file $depfile 1 + } +} + +# @cc-check-types type ... +# +# Checks that the types exist. +proc cc-check-types {args} { + cc-check-some-feature $args { + cctest_type $each + } +} + +# @cc-check-defines define ... +# +# Checks that the given preprocessor symbols are defined. +proc cc-check-defines {args} { + cc-check-some-feature $args { + cctest_define $each + } +} + +# @cc-check-decls name ... +# +# Checks that each given name is either a preprocessor symbol or rvalue +# such as an enum. Note that the define used is 'HAVE_DECL_xxx' +# rather than 'HAVE_xxx'. +proc cc-check-decls {args} { + set ret 1 + foreach name $args { + msg-checking "Checking for $name..." + set r [cctest_decl $name] + define-feature "decl $name" $r + if {$r} { + msg-result "ok" + } else { + msg-result "not found" + set ret 0 + } + } + return $ret +} + +# @cc-check-functions function ... +# +# Checks that the given functions exist (can be linked). +proc cc-check-functions {args} { + cc-check-some-feature $args { + cctest_function $each + } +} + +# @cc-check-members type.member ... +# +# Checks that the given type/structure members exist. +# A structure member is of the form 'struct stat.st_mtime'. +proc cc-check-members {args} { + cc-check-some-feature $args { + cctest_member $each + } +} + +# @cc-check-function-in-lib function libs ?otherlibs? +# +# Checks that the given function can be found in one of the libs. +# +# First checks for no library required, then checks each of the libraries +# in turn. +# +# If the function is found, the feature is defined and 'lib_$function' is defined +# to '-l$lib' where the function was found, or "" if no library required. +# In addition, '-l$lib' is prepended to the 'LIBS' define. +# +# If additional libraries may be needed for linking, they should be specified +# with '$extralibs' as '-lotherlib1 -lotherlib2'. +# These libraries are not automatically added to 'LIBS'. +# +# Returns 1 if found or 0 if not. +# +proc cc-check-function-in-lib {function libs {otherlibs {}}} { + msg-checking "Checking libs for $function..." + set found 0 + cc-with [list -libs $otherlibs] { + if {[cctest_function $function]} { + msg-result "none needed" + define lib_$function "" + incr found + } else { + foreach lib $libs { + cc-with [list -libs -l$lib] { + if {[cctest_function $function]} { + msg-result -l$lib + define lib_$function -l$lib + # prepend to LIBS + define LIBS "-l$lib [get-define LIBS]" + incr found + break + } + } + } + } + } + define-feature $function $found + if {!$found} { + msg-result "no" + } + return $found +} + +# @cc-check-tools tool ... +# +# Checks for existence of the given compiler tools, taking +# into account any cross compilation prefix. +# +# For example, when checking for 'ar', first 'AR' is checked on the command +# line and then in the environment. If not found, '${host}-ar' or +# simply 'ar' is assumed depending upon whether cross compiling. +# The path is searched for this executable, and if found 'AR' is defined +# to the executable name. +# Note that even when cross compiling, the simple 'ar' is used as a fallback, +# but a warning is generated. This is necessary for some toolchains. +# +# It is an error if the executable is not found. +# +proc cc-check-tools {args} { + foreach tool $args { + set TOOL [string toupper $tool] + set exe [get-env $TOOL [get-define cross]$tool] + if {[find-executable $exe]} { + define $TOOL $exe + continue + } + if {[find-executable $tool]} { + msg-result "Warning: Failed to find $exe, falling back to $tool which may be incorrect" + define $TOOL $tool + continue + } + user-error "Failed to find $exe" + } +} + +# @cc-check-progs prog ... +# +# Checks for existence of the given executables on the path. +# +# For example, when checking for 'grep', the path is searched for +# the executable, 'grep', and if found 'GREP' is defined as 'grep'. +# +# If the executable is not found, the variable is defined as 'false'. +# Returns 1 if all programs were found, or 0 otherwise. +# +proc cc-check-progs {args} { + set failed 0 + foreach prog $args { + set PROG [string toupper $prog] + msg-checking "Checking for $prog..." + if {![find-executable $prog]} { + msg-result no + define $PROG false + incr failed + } else { + msg-result ok + define $PROG $prog + } + } + expr {!$failed} +} + +# @cc-path-progs prog ... +# +# Like cc-check-progs, but sets the define to the full path rather +# than just the program name. +# +proc cc-path-progs {args} { + set failed 0 + foreach prog $args { + set PROG [string toupper $prog] + msg-checking "Checking for $prog..." + set path [find-executable-path $prog] + if {$path eq ""} { + msg-result no + define $PROG false + incr failed + } else { + msg-result $path + define $PROG $path + } + } + expr {!$failed} +} + +# Adds the given settings to $::autosetup(ccsettings) and +# returns the old settings. +# +proc cc-add-settings {settings} { + if {[llength $settings] % 2} { + autosetup-error "settings list is missing a value: $settings" + } + + set prev [cc-get-settings] + # workaround a bug in some versions of jimsh by forcing + # conversion of $prev to a list + llength $prev + + array set new $prev + + foreach {name value} $settings { + switch -exact -- $name { + -cflags - -includes { + # These are given as lists + lappend new($name) {*}[list-non-empty $value] + } + -declare { + lappend new($name) $value + } + -libs { + # Note that new libraries are added before previous libraries + set new($name) [list {*}[list-non-empty $value] {*}$new($name)] + } + -link - -lang - -nooutput { + set new($name) $value + } + -source - -sourcefile - -code { + # XXX: These probably are only valid directly from cctest + set new($name) $value + } + default { + autosetup-error "unknown cctest setting: $name" + } + } + } + + cc-store-settings [array get new] + + return $prev +} + +proc cc-store-settings {new} { + set ::autosetup(ccsettings) $new +} + +proc cc-get-settings {} { + return $::autosetup(ccsettings) +} + +# Similar to cc-add-settings, but each given setting +# simply replaces the existing value. +# +# Returns the previous settings +proc cc-update-settings {args} { + set prev [cc-get-settings] + cc-store-settings [dict merge $prev $args] + return $prev +} + +# @cc-with settings ?{ script }? +# +# Sets the given 'cctest' settings and then runs the tests in '$script'. +# Note that settings such as '-lang' replace the current setting, while +# those such as '-includes' are appended to the existing setting. +# +# If no script is given, the settings become the default for the remainder +# of the 'auto.def' file. +# +## cc-with {-lang c++} { +## # This will check with the C++ compiler +## cc-check-types bool +## cc-with {-includes signal.h} { +## # This will check with the C++ compiler, signal.h and any existing includes. +## ... +## } +## # back to just the C++ compiler +## } +# +# The '-libs' setting is special in that newer values are added *before* earlier ones. +# +## cc-with {-libs {-lc -lm}} { +## cc-with {-libs -ldl} { +## cctest -libs -lsocket ... +## # libs will be in this order: -lsocket -ldl -lc -lm +## } +## } +# +# If you wish to invoke something like cc-check-flags but not have -cflags updated, +# use the following idiom: +# +## cc-with {} { +## cc-check-flags ... +## } +proc cc-with {settings args} { + if {[llength $args] == 0} { + cc-add-settings $settings + } elseif {[llength $args] > 1} { + autosetup-error "usage: cc-with settings ?script?" + } else { + set save [cc-add-settings $settings] + set rc [catch {uplevel 1 [lindex $args 0]} result info] + cc-store-settings $save + if {$rc != 0} { + return -code [dict get $info -code] $result + } + return $result + } +} + +# @cctest ?settings? +# +# Low level C/C++ compiler checker. Compiles and or links a small C program +# according to the arguments and returns 1 if OK, or 0 if not. +# +# Supported settings are: +# +## -cflags cflags A list of flags to pass to the compiler +## -includes list A list of includes, e.g. {stdlib.h stdio.h} +## -declare code Code to declare before main() +## -link 1 Don't just compile, link too +## -lang c|c++ Use the C (default) or C++ compiler +## -libs liblist List of libraries to link, e.g. {-ldl -lm} +## -code code Code to compile in the body of main() +## -source code Compile a complete program. Ignore -includes, -declare and -code +## -sourcefile file Shorthand for -source [readfile [get-define srcdir]/$file] +## -nooutput 1 Treat any compiler output (e.g. a warning) as an error +# +# Unless '-source' or '-sourcefile' is specified, the C program looks like: +# +## #include /* same for remaining includes in the list */ +## declare-code /* any code in -declare, verbatim */ +## int main(void) { +## code /* any code in -code, verbatim */ +## return 0; +## } +# +# And the command line looks like: +# +## CC -cflags CFLAGS CPPFLAGS conftest.c -o conftest.o +## CXX -cflags CXXFLAGS CPPFLAGS conftest.cpp -o conftest.o +# +# And if linking: +# +## CC LDFLAGS -cflags CFLAGS conftest.c -o conftest -libs LIBS +## CXX LDFLAGS -cflags CXXFLAGS conftest.c -o conftest -libs LIBS +# +# Any failures are recorded in 'config.log' +# +proc cctest {args} { + set tmp conftest__ + + # Easiest way to merge in the settings + cc-with $args { + array set opts [cc-get-settings] + } + + if {[info exists opts(-sourcefile)]} { + set opts(-source) [readfile [get-define srcdir]/$opts(-sourcefile) "#error can't find $opts(-sourcefile)"] + } + if {[info exists opts(-source)]} { + set lines $opts(-source) + } else { + foreach i $opts(-includes) { + if {$opts(-code) ne "" && ![feature-checked $i]} { + # Compiling real code with an unchecked header file + # Quickly (and silently) check for it now + + # Remove all -includes from settings before checking + set saveopts [cc-update-settings -includes {}] + msg-quiet cc-check-includes $i + cc-store-settings $saveopts + } + if {$opts(-code) eq "" || [have-feature $i]} { + lappend source "#include <$i>" + } + } + lappend source {*}$opts(-declare) + lappend source "int main(void) {" + lappend source $opts(-code) + lappend source "return 0;" + lappend source "}" + + set lines [join $source \n] + } + + # Build the command line + set cmdline {} + lappend cmdline {*}[get-define CCACHE] + switch -exact -- $opts(-lang) { + c++ { + set src conftest__.cpp + lappend cmdline {*}[get-define CXX] + set cflags [get-define CXXFLAGS] + } + c { + set src conftest__.c + lappend cmdline {*}[get-define CC] + set cflags [get-define CFLAGS] + } + default { + autosetup-error "cctest called with unknown language: $opts(-lang)" + } + } + + if {$opts(-link)} { + lappend cmdline {*}[get-define LDFLAGS] + } else { + lappend cflags {*}[get-define CPPFLAGS] + set tmp conftest__.o + lappend cmdline -c + } + lappend cmdline {*}$opts(-cflags) {*}[get-define cc-default-debug ""] {*}$cflags + lappend cmdline $src -o $tmp + if {$opts(-link)} { + lappend cmdline {*}$opts(-libs) {*}[get-define LIBS] + } + + # At this point we have the complete command line and the + # complete source to be compiled. Get the result from cache if + # we can + if {[info exists ::cc_cache($cmdline,$lines)]} { + msg-checking "(cached) " + set ok $::cc_cache($cmdline,$lines) + if {$::autosetup(debug)} { + configlog "From cache (ok=$ok): [join $cmdline]" + configlog "============" + configlog $lines + configlog "============" + } + return $ok + } + + writefile $src $lines\n + + set ok 1 + set err [catch {exec-with-stderr {*}$cmdline} result errinfo] + if {$err || ($opts(-nooutput) && [string length $result])} { + configlog "Failed: [join $cmdline]" + configlog $result + configlog "============" + configlog "The failed code was:" + configlog $lines + configlog "============" + set ok 0 + } elseif {$::autosetup(debug)} { + configlog "Compiled OK: [join $cmdline]" + configlog "============" + configlog $lines + configlog "============" + } + file delete $src + file delete $tmp + + # cache it + set ::cc_cache($cmdline,$lines) $ok + + return $ok +} + +# @make-autoconf-h outfile ?auto-patterns=HAVE_*? ?bare-patterns=SIZEOF_*? +# +# Deprecated - see 'make-config-header' +proc make-autoconf-h {file {autopatterns {HAVE_*}} {barepatterns {SIZEOF_* HAVE_DECL_*}}} { + user-notice "*** make-autoconf-h is deprecated -- use make-config-header instead" + make-config-header $file -auto $autopatterns -bare $barepatterns +} + +# @make-config-header outfile ?-auto patternlist? ?-bare patternlist? ?-none patternlist? ?-str patternlist? ... +# +# Examines all defined variables which match the given patterns +# and writes an include file, '$file', which defines each of these. +# Variables which match '-auto' are output as follows: +# - defines which have the value '0' are ignored. +# - defines which have integer values are defined as the integer value. +# - any other value is defined as a string, e.g. '"value"' +# Variables which match '-bare' are defined as-is. +# Variables which match '-str' are defined as a string, e.g. '"value"' +# Variables which match '-none' are omitted. +# +# Note that order is important. The first pattern that matches is selected. +# Default behaviour is: +# +## -bare {SIZEOF_* HAVE_DECL_*} -auto HAVE_* -none * +# +# If the file would be unchanged, it is not written. +proc make-config-header {file args} { + set guard _[string toupper [regsub -all {[^a-zA-Z0-9]} [file tail $file] _]] + file mkdir [file dirname $file] + set lines {} + lappend lines "#ifndef $guard" + lappend lines "#define $guard" + + # Add some defaults + lappend args -bare {SIZEOF_* HAVE_DECL_*} -auto HAVE_* + + foreach n [lsort [dict keys [all-defines]]] { + set value [get-define $n] + set type [calc-define-output-type $n $args] + switch -exact -- $type { + -bare { + # Just output the value unchanged + } + -none { + continue + } + -str { + set value \"[string map [list \\ \\\\ \" \\\"] $value]\" + } + -auto { + # Automatically determine the type + if {$value eq "0"} { + lappend lines "/* #undef $n */" + continue + } + if {![string is integer -strict $value]} { + set value \"[string map [list \\ \\\\ \" \\\"] $value]\" + } + } + "" { + continue + } + default { + autosetup-error "Unknown type in make-config-header: $type" + } + } + lappend lines "#define $n $value" + } + lappend lines "#endif" + set buf [join $lines \n] + write-if-changed $file $buf { + msg-result "Created $file" + } +} + +proc calc-define-output-type {name spec} { + foreach {type patterns} $spec { + foreach pattern $patterns { + if {[string match $pattern $name]} { + return $type + } + } + } + return "" +} + +proc cc-init {} { + global autosetup + + # Initialise some values from the environment or commandline or default settings + foreach i {LDFLAGS LIBS CPPFLAGS LINKFLAGS CFLAGS} { + lassign $i var default + define $var [get-env $var $default] + } + + if {[env-is-set CC]} { + # Set by the user, so don't try anything else + set try [list [get-env CC ""]] + } else { + # Try some reasonable options + set try [list [get-define cross]cc [get-define cross]gcc] + } + define CC [find-an-executable {*}$try] + if {[get-define CC] eq ""} { + user-error "Could not find a C compiler. Tried: [join $try ", "]" + } + + define CPP [get-env CPP "[get-define CC] -E"] + + # XXX: Could avoid looking for a C++ compiler until requested + # If CXX isn't found, it is set to the empty string. + if {[env-is-set CXX]} { + define CXX [find-an-executable -required [get-env CXX ""]] + } else { + define CXX [find-an-executable [get-define cross]c++ [get-define cross]g++] + } + + # CXXFLAGS default to CFLAGS if not specified + define CXXFLAGS [get-env CXXFLAGS [get-define CFLAGS]] + + # May need a CC_FOR_BUILD, so look for one + define CC_FOR_BUILD [find-an-executable [get-env CC_FOR_BUILD ""] cc gcc false] + + # These start empty and never come from the user or environment + define AS_CFLAGS "" + define AS_CPPFLAGS "" + define AS_CXXFLAGS "" + + define CCACHE [find-an-executable [get-env CCACHE ccache]] + + # If any of these are set in the environment, propagate them to the AUTOREMAKE commandline + foreach i {CC CXX CCACHE CPP CFLAGS CXXFLAGS CXXFLAGS LDFLAGS LIBS CROSS CPPFLAGS LINKFLAGS CC_FOR_BUILD LD} { + if {[env-is-set $i]} { + # Note: If the variable is set on the command line, get-env will return that value + # so the command line will continue to override the environment + define-append-argv AUTOREMAKE $i=[get-env $i ""] + } + } + + # Initial cctest settings + cc-store-settings {-cflags {} -includes {} -declare {} -link 0 -lang c -libs {} -code {} -nooutput 0} + set autosetup(cc-include-deps) {} + + msg-result "C compiler...[get-define CCACHE] [get-define CC] [get-define CFLAGS] [get-define CPPFLAGS]" + if {[get-define CXX] ne "false"} { + msg-result "C++ compiler...[get-define CCACHE] [get-define CXX] [get-define CXXFLAGS] [get-define CPPFLAGS]" + } + msg-result "Build C compiler...[get-define CC_FOR_BUILD]" + + # On Darwin, we prefer to use -g0 to avoid creating .dSYM directories + # but some compilers may not support it, so test here. + switch -glob -- [get-define host] { + *-*-darwin* { + if {[cctest -cflags {-g0}]} { + define cc-default-debug -g0 + } + } + } + + if {![cc-check-includes stdlib.h]} { + user-error "Compiler does not work. See config.log" + } +} + +cc-init diff --git a/autosetup/find_tclconfig.tcl b/autosetup/find_tclconfig.tcl new file mode 100644 index 0000000000..c3d3df8ec3 --- /dev/null +++ b/autosetup/find_tclconfig.tcl @@ -0,0 +1,24 @@ +# +# Run this TCL script to find and print the pathname for the tclConfig.sh +# file. Used by ../configure +# +if {[catch { + set libdir [tcl::pkgconfig get libdir,install] +}]} { + puts stderr "tclsh too old: does not support tcl::pkgconfig" + exit 1 +} +if {![file exists $libdir]} { + puts stderr "tclsh reported library directory \"$libdir\" does not exist" + exit 1 +} +if {![file exists $libdir/tclConfig.sh]} { + set n1 $libdir/tcl$::tcl_version + if {[file exists $n1/tclConfig.sh]} { + set libdir $n1 + } else { + puts stderr "cannot find tclConfig.sh in either $libdir or $n1" + exit 1 + } +} +puts $libdir diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c new file mode 100644 index 0000000000..1a6453d0c8 --- /dev/null +++ b/autosetup/jimsh0.c @@ -0,0 +1,24506 @@ +/* This is single source file, bootstrap version of Jim Tcl. See http://jim.tcl.tk/ */ +#define JIM_COMPAT +#define JIM_ANSIC +#define JIM_REGEXP +#define HAVE_NO_AUTOCONF +#define JIM_TINY +#define _JIMAUTOCONF_H +#define TCL_LIBRARY "." +#define jim_ext_bootstrap +#define jim_ext_aio +#define jim_ext_readdir +#define jim_ext_regexp +#define jim_ext_file +#define jim_ext_glob +#define jim_ext_exec +#define jim_ext_clock +#define jim_ext_array +#define jim_ext_stdlib +#define jim_ext_tclcompat +#if defined(_MSC_VER) +#define TCL_PLATFORM_OS "windows" +#define TCL_PLATFORM_PLATFORM "windows" +#define TCL_PLATFORM_PATH_SEPARATOR ";" +#define HAVE_MKDIR_ONE_ARG +#define HAVE_SYSTEM +#elif defined(__MINGW32__) +#define TCL_PLATFORM_OS "mingw" +#define TCL_PLATFORM_PLATFORM "windows" +#define TCL_PLATFORM_PATH_SEPARATOR ";" +#define HAVE_MKDIR_ONE_ARG +#define HAVE_SYSTEM +#define HAVE_SYS_TIME_H +#define HAVE_DIRENT_H +#define HAVE_UNISTD_H +#define HAVE_UMASK +#include +#ifndef S_IRWXG +#define S_IRWXG 0 +#endif +#ifndef S_IRWXO +#define S_IRWXO 0 +#endif +#else +#define TCL_PLATFORM_OS "unknown" +#define TCL_PLATFORM_PLATFORM "unix" +#define TCL_PLATFORM_PATH_SEPARATOR ":" +#ifdef _MINIX +#define vfork fork +#define _POSIX_SOURCE +#else +#define _GNU_SOURCE +#endif +#define HAVE_FORK +#define HAVE_WAITPID +#define HAVE_ISATTY +#define HAVE_MKSTEMP +#define HAVE_LINK +#define HAVE_SYS_TIME_H +#define HAVE_DIRENT_H +#define HAVE_UNISTD_H +#define HAVE_UMASK +#define HAVE_PIPE +#define _FILE_OFFSET_BITS 64 +#endif +#define JIM_VERSION 84 +#ifndef JIM_WIN32COMPAT_H +#define JIM_WIN32COMPAT_H + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#if defined(_WIN32) || defined(WIN32) + +#define HAVE_DLOPEN +void *dlopen(const char *path, int mode); +int dlclose(void *handle); +void *dlsym(void *handle, const char *symbol); +char *dlerror(void); + + +#if defined(__MINGW32__) + #define JIM_SPRINTF_DOUBLE_NEEDS_FIX +#endif + +#ifdef _MSC_VER + + +#if _MSC_VER >= 1000 + #pragma warning(disable:4146) +#endif + +#include +#define jim_wide _int64 +#ifndef HAVE_LONG_LONG +#define HAVE_LONG_LONG +#endif +#ifndef LLONG_MAX + #define LLONG_MAX 9223372036854775807I64 +#endif +#ifndef LLONG_MIN + #define LLONG_MIN (-LLONG_MAX - 1I64) +#endif +#define JIM_WIDE_MIN LLONG_MIN +#define JIM_WIDE_MAX LLONG_MAX +#define JIM_WIDE_MODIFIER "I64d" +#define strcasecmp _stricmp +#define strtoull _strtoui64 + +#include + +#include +int gettimeofday(struct timeval *tv, void *unused); + +#define HAVE_OPENDIR +struct dirent { + char *d_name; +}; + +typedef struct DIR { + long handle; + struct _finddata_t info; + struct dirent result; + char *name; +} DIR; + +DIR *opendir(const char *name); +int closedir(DIR *dir); +struct dirent *readdir(DIR *dir); + +#endif + +#endif + +#ifdef __cplusplus +} +#endif + +#endif +#ifndef UTF8_UTIL_H +#define UTF8_UTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + + + +#define MAX_UTF8_LEN 4 + +int utf8_fromunicode(char *p, unsigned uc); + +#ifndef JIM_UTF8 +#include + + +#define utf8_strlen(S, B) ((B) < 0 ? (int)strlen(S) : (B)) +#define utf8_strwidth(S, B) utf8_strlen((S), (B)) +#define utf8_tounicode(S, CP) (*(CP) = (unsigned char)*(S), 1) +#define utf8_getchars(CP, C) (*(CP) = (C), 1) +#define utf8_upper(C) toupper(C) +#define utf8_title(C) toupper(C) +#define utf8_lower(C) tolower(C) +#define utf8_index(C, I) (I) +#define utf8_charlen(C) 1 +#define utf8_prev_len(S, L) 1 +#define utf8_width(C) 1 + +#else + +#endif + +#ifdef __cplusplus +} +#endif + +#endif + +#ifndef __JIM__H +#define __JIM__H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + + +#ifndef HAVE_NO_AUTOCONF +#endif + + + +#ifndef jim_wide +# ifdef HAVE_LONG_LONG +# define jim_wide long long +# ifndef LLONG_MAX +# define LLONG_MAX 9223372036854775807LL +# endif +# ifndef LLONG_MIN +# define LLONG_MIN (-LLONG_MAX - 1LL) +# endif +# define JIM_WIDE_MIN LLONG_MIN +# define JIM_WIDE_MAX LLONG_MAX +# else +# define jim_wide long +# define JIM_WIDE_MIN LONG_MIN +# define JIM_WIDE_MAX LONG_MAX +# endif + + +# ifdef HAVE_LONG_LONG +# define JIM_WIDE_MODIFIER "lld" +# else +# define JIM_WIDE_MODIFIER "ld" +# define strtoull strtoul +# endif +#endif + +#define UCHAR(c) ((unsigned char)(c)) + + + +#define JIM_ABI_VERSION 101 + +#define JIM_OK 0 +#define JIM_ERR 1 +#define JIM_RETURN 2 +#define JIM_BREAK 3 +#define JIM_CONTINUE 4 +#define JIM_SIGNAL 5 +#define JIM_EXIT 6 + +#define JIM_EVAL 7 + +#define JIM_MAX_CALLFRAME_DEPTH 1000 +#define JIM_MAX_EVAL_DEPTH 2000 + + +#define JIM_PRIV_FLAG_SHIFT 20 + +#define JIM_NONE 0 +#define JIM_ERRMSG 1 +#define JIM_ENUM_ABBREV 2 +#define JIM_UNSHARED 4 +#define JIM_MUSTEXIST 8 +#define JIM_NORESULT 16 + + +#define JIM_SUBST_NOVAR 1 +#define JIM_SUBST_NOCMD 2 +#define JIM_SUBST_NOESC 4 +#define JIM_SUBST_FLAG 128 + + +#define JIM_CASESENS 0 +#define JIM_NOCASE 1 +#define JIM_OPT_END 2 + + +#define JIM_PATH_LEN 1024 + + +#define JIM_NOTUSED(V) ((void) V) + +#define JIM_LIBPATH "auto_path" +#define JIM_INTERACTIVE "tcl_interactive" + + +typedef struct Jim_Stack { + int len; + int maxlen; + void **vector; +} Jim_Stack; + + +typedef struct Jim_HashEntry { + void *key; + union { + void *val; + int intval; + } u; + struct Jim_HashEntry *next; +} Jim_HashEntry; + +typedef struct Jim_HashTableType { + unsigned int (*hashFunction)(const void *key); + void *(*keyDup)(void *privdata, const void *key); + void *(*valDup)(void *privdata, const void *obj); + int (*keyCompare)(void *privdata, const void *key1, const void *key2); + void (*keyDestructor)(void *privdata, void *key); + void (*valDestructor)(void *privdata, void *obj); +} Jim_HashTableType; + +typedef struct Jim_HashTable { + Jim_HashEntry **table; + const Jim_HashTableType *type; + void *privdata; + unsigned int size; + unsigned int sizemask; + unsigned int used; + unsigned int collisions; + unsigned int uniq; +} Jim_HashTable; + +typedef struct Jim_HashTableIterator { + Jim_HashTable *ht; + Jim_HashEntry *entry, *nextEntry; + int index; +} Jim_HashTableIterator; + + +#define JIM_HT_INITIAL_SIZE 16 + + +#define Jim_FreeEntryVal(ht, entry) \ + if ((ht)->type->valDestructor) \ + (ht)->type->valDestructor((ht)->privdata, (entry)->u.val) + +#define Jim_SetHashVal(ht, entry, _val_) do { \ + if ((ht)->type->valDup) \ + (entry)->u.val = (ht)->type->valDup((ht)->privdata, (_val_)); \ + else \ + (entry)->u.val = (_val_); \ +} while(0) + +#define Jim_SetHashIntVal(ht, entry, _val_) (entry)->u.intval = (_val_) + +#define Jim_FreeEntryKey(ht, entry) \ + if ((ht)->type->keyDestructor) \ + (ht)->type->keyDestructor((ht)->privdata, (entry)->key) + +#define Jim_SetHashKey(ht, entry, _key_) do { \ + if ((ht)->type->keyDup) \ + (entry)->key = (ht)->type->keyDup((ht)->privdata, (_key_)); \ + else \ + (entry)->key = (void *)(_key_); \ +} while(0) + +#define Jim_CompareHashKeys(ht, key1, key2) \ + (((ht)->type->keyCompare) ? \ + (ht)->type->keyCompare((ht)->privdata, (key1), (key2)) : \ + (key1) == (key2)) + +#define Jim_HashKey(ht, key) ((ht)->type->hashFunction(key) + (ht)->uniq) + +#define Jim_GetHashEntryKey(he) ((he)->key) +#define Jim_GetHashEntryVal(he) ((he)->u.val) +#define Jim_GetHashEntryIntVal(he) ((he)->u.intval) +#define Jim_GetHashTableCollisions(ht) ((ht)->collisions) +#define Jim_GetHashTableSize(ht) ((ht)->size) +#define Jim_GetHashTableUsed(ht) ((ht)->used) + + +typedef struct Jim_Obj { + char *bytes; + const struct Jim_ObjType *typePtr; + int refCount; + int length; + + union { + + jim_wide wideValue; + + int intValue; + + double doubleValue; + + void *ptr; + + struct { + void *ptr1; + void *ptr2; + } twoPtrValue; + + struct { + void *ptr; + int int1; + int int2; + } ptrIntValue; + + struct { + struct Jim_VarVal *vv; + unsigned long callFrameId; + int global; + } varValue; + + struct { + struct Jim_Obj *nsObj; + struct Jim_Cmd *cmdPtr; + unsigned long procEpoch; + } cmdValue; + + struct { + struct Jim_Obj **ele; + int len; + int maxLen; + } listValue; + + struct Jim_Dict *dictValue; + + struct { + int maxLength; + int charLength; + } strValue; + + struct { + unsigned long id; + struct Jim_Reference *refPtr; + } refValue; + + struct { + struct Jim_Obj *fileNameObj; + int lineNumber; + } sourceValue; + + struct { + struct Jim_Obj *varNameObjPtr; + struct Jim_Obj *indexObjPtr; + } dictSubstValue; + struct { + int line; + int argc; + } scriptLineValue; + } internalRep; + struct Jim_Obj *prevObjPtr; + struct Jim_Obj *nextObjPtr; +} Jim_Obj; + + +#define Jim_IncrRefCount(objPtr) \ + ++(objPtr)->refCount +#define Jim_DecrRefCount(interp, objPtr) \ + if (--(objPtr)->refCount <= 0) Jim_FreeObj(interp, objPtr) +#define Jim_IsShared(objPtr) \ + ((objPtr)->refCount > 1) + +#define Jim_FreeNewObj Jim_FreeObj + + +#define Jim_FreeIntRep(i,o) \ + if ((o)->typePtr && (o)->typePtr->freeIntRepProc) \ + (o)->typePtr->freeIntRepProc(i, o) + + +#define Jim_GetIntRepPtr(o) (o)->internalRep.ptr + + +#define Jim_SetIntRepPtr(o, p) \ + (o)->internalRep.ptr = (p) + + +struct Jim_Interp; + +typedef void (Jim_FreeInternalRepProc)(struct Jim_Interp *interp, + struct Jim_Obj *objPtr); +typedef void (Jim_DupInternalRepProc)(struct Jim_Interp *interp, + struct Jim_Obj *srcPtr, Jim_Obj *dupPtr); +typedef void (Jim_UpdateStringProc)(struct Jim_Obj *objPtr); + +typedef struct Jim_ObjType { + const char *name; + Jim_FreeInternalRepProc *freeIntRepProc; + Jim_DupInternalRepProc *dupIntRepProc; + Jim_UpdateStringProc *updateStringProc; + int flags; +} Jim_ObjType; + + +#define JIM_TYPE_NONE 0 +#define JIM_TYPE_REFERENCES 1 + + + +typedef struct Jim_CallFrame { + unsigned long id; + int level; + struct Jim_HashTable vars; + struct Jim_HashTable *staticVars; + struct Jim_CallFrame *parent; + Jim_Obj *const *argv; + int argc; + Jim_Obj *procArgsObjPtr; + Jim_Obj *procBodyObjPtr; + struct Jim_CallFrame *next; + Jim_Obj *nsObj; + Jim_Obj *unused_fileNameObj; + int unused_line; + Jim_Stack *localCommands; + struct Jim_Obj *tailcallObj; + struct Jim_Cmd *tailcallCmd; +} Jim_CallFrame; + + +typedef struct Jim_EvalFrame { + Jim_CallFrame *framePtr; + int level; + int procLevel; + struct Jim_Cmd *cmd; + struct Jim_EvalFrame *parent; + Jim_Obj *const *argv; + int argc; + Jim_Obj *scriptObj; +} Jim_EvalFrame; + +typedef struct Jim_VarVal { + Jim_Obj *objPtr; + struct Jim_CallFrame *linkFramePtr; + int refCount; +} Jim_VarVal; + + +typedef int Jim_CmdProc(struct Jim_Interp *interp, int argc, + Jim_Obj *const *argv); +typedef void Jim_DelCmdProc(struct Jim_Interp *interp, void *privData); + +typedef struct Jim_Dict { + struct JimDictHashEntry { + int offset; + unsigned hash; + } *ht; + unsigned int size; + unsigned int sizemask; + unsigned int uniq; + Jim_Obj **table; + int len; + int maxLen; + unsigned int dummy; +} Jim_Dict; + +typedef struct Jim_Cmd { + int inUse; + int isproc; + struct Jim_Cmd *prevCmd; + Jim_Obj *cmdNameObj; + union { + struct { + + Jim_CmdProc *cmdProc; + Jim_DelCmdProc *delProc; + void *privData; + } native; + struct { + + Jim_Obj *argListObjPtr; + Jim_Obj *bodyObjPtr; + Jim_HashTable *staticVars; + int argListLen; + int reqArity; + int optArity; + int argsPos; + int upcall; + struct Jim_ProcArg { + Jim_Obj *nameObjPtr; + Jim_Obj *defaultObjPtr; + } *arglist; + Jim_Obj *nsObj; + } proc; + } u; +} Jim_Cmd; + + +typedef struct Jim_PrngState { + unsigned char sbox[256]; + unsigned int i, j; +} Jim_PrngState; + +typedef struct Jim_Interp { + Jim_Obj *result; + int unused_errorLine; + Jim_Obj *currentFilenameObj; + int break_level; + int maxCallFrameDepth; + int maxEvalDepth; + int evalDepth; + int returnCode; + int returnLevel; + int exitCode; + long id; + int signal_level; + jim_wide sigmask; + int (*signal_set_result)(struct Jim_Interp *interp, jim_wide sigmask); + Jim_CallFrame *framePtr; + Jim_CallFrame *topFramePtr; + struct Jim_HashTable commands; + unsigned long procEpoch; /* Incremented every time the result + of procedures names lookup caching + may no longer be valid. */ + unsigned long callFrameEpoch; /* Incremented every time a new + callframe is created. This id is used for the + 'ID' field contained in the Jim_CallFrame + structure. */ + int local; + int quitting; + int safeexpr; + Jim_Obj *liveList; + Jim_Obj *freeList; + Jim_Obj *unused_currentScriptObj; + Jim_EvalFrame topEvalFrame; + Jim_EvalFrame *evalFrame; + int procLevel; + Jim_Obj * const *unused_argv; + Jim_Obj *nullScriptObj; + Jim_Obj *emptyObj; + Jim_Obj *trueObj; + Jim_Obj *falseObj; + unsigned long referenceNextId; + struct Jim_HashTable references; + unsigned long lastCollectId; /* reference max Id of the last GC + execution. It's set to ~0 while the collection + is running as sentinel to avoid to recursive + calls via the [collect] command inside + finalizers. */ + jim_wide lastCollectTime; + Jim_Obj *stackTrace; + Jim_Obj *errorProc; + Jim_Obj *unknown; + Jim_Obj *defer; + Jim_Obj *traceCmdObj; + int unknown_called; + int errorFlag; + void *cmdPrivData; /* Used to pass the private data pointer to + a command. It is set to what the user specified + via Jim_CreateCommand(). */ + + Jim_Cmd *oldCmdCache; + int oldCmdCacheSize; + struct Jim_CallFrame *freeFramesList; + struct Jim_HashTable assocData; + Jim_PrngState *prngState; + struct Jim_HashTable packages; + Jim_Stack *loadHandles; +} Jim_Interp; + +#define Jim_SetResultString(i,s,l) Jim_SetResult(i, Jim_NewStringObj(i,s,l)) +#define Jim_SetResultInt(i,intval) Jim_SetResult(i, Jim_NewIntObj(i,intval)) + +#define Jim_SetResultBool(i,b) Jim_SetResultInt(i, b) +#define Jim_SetEmptyResult(i) Jim_SetResult(i, (i)->emptyObj) +#define Jim_GetResult(i) ((i)->result) +#define Jim_CmdPrivData(i) ((i)->cmdPrivData) + +#define Jim_SetResult(i,o) do { \ + Jim_Obj *_resultObjPtr_ = (o); \ + Jim_IncrRefCount(_resultObjPtr_); \ + Jim_DecrRefCount(i,(i)->result); \ + (i)->result = _resultObjPtr_; \ +} while(0) + + +#define Jim_GetId(i) (++(i)->id) + + +#define JIM_REFERENCE_TAGLEN 7 /* The tag is fixed-length, because the reference + string representation must be fixed length. */ +typedef struct Jim_Reference { + Jim_Obj *objPtr; + Jim_Obj *finalizerCmdNamePtr; + char tag[JIM_REFERENCE_TAGLEN+1]; +} Jim_Reference; + + +#define Jim_NewEmptyStringObj(i) Jim_NewStringObj(i, "", 0) +#define Jim_FreeHashTableIterator(iter) Jim_Free(iter) + +#define JIM_EXPORT extern + + + +JIM_EXPORT void *(*Jim_Allocator)(void *ptr, size_t size); + +#define Jim_Free(P) Jim_Allocator((P), 0) +#define Jim_Realloc(P, S) Jim_Allocator((P), (S)) +#define Jim_Alloc(S) Jim_Allocator(NULL, (S)) +JIM_EXPORT char * Jim_StrDup (const char *s); +JIM_EXPORT char *Jim_StrDupLen(const char *s, int l); + + +JIM_EXPORT char **Jim_GetEnviron(void); +JIM_EXPORT void Jim_SetEnviron(char **env); +JIM_EXPORT int Jim_MakeTempFile(Jim_Interp *interp, const char *filename_template, int unlink_file); +#ifndef CLOCK_REALTIME +# define CLOCK_REALTIME 0 +#endif +#ifndef CLOCK_MONOTONIC +# define CLOCK_MONOTONIC 1 +#endif +#ifndef CLOCK_MONOTONIC_RAW +# define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC +#endif +JIM_EXPORT jim_wide Jim_GetTimeUsec(unsigned type); + + +JIM_EXPORT int Jim_Eval(Jim_Interp *interp, const char *script); + + +JIM_EXPORT int Jim_EvalSource(Jim_Interp *interp, const char *filename, int lineno, const char *script); + +#define Jim_Eval_Named(I, S, F, L) Jim_EvalSource((I), (F), (L), (S)) + +JIM_EXPORT int Jim_EvalGlobal(Jim_Interp *interp, const char *script); +JIM_EXPORT int Jim_EvalFile(Jim_Interp *interp, const char *filename); +JIM_EXPORT int Jim_EvalFileGlobal(Jim_Interp *interp, const char *filename); +JIM_EXPORT int Jim_EvalObj (Jim_Interp *interp, Jim_Obj *scriptObjPtr); +JIM_EXPORT int Jim_EvalObjVector (Jim_Interp *interp, int objc, + Jim_Obj *const *objv); +JIM_EXPORT int Jim_EvalObjList(Jim_Interp *interp, Jim_Obj *listObj); +JIM_EXPORT int Jim_EvalObjPrefix(Jim_Interp *interp, Jim_Obj *prefix, + int objc, Jim_Obj *const *objv); +#define Jim_EvalPrefix(i, p, oc, ov) Jim_EvalObjPrefix((i), Jim_NewStringObj((i), (p), -1), (oc), (ov)) +JIM_EXPORT int Jim_EvalNamespace(Jim_Interp *interp, Jim_Obj *scriptObj, Jim_Obj *nsObj); +JIM_EXPORT int Jim_SubstObj (Jim_Interp *interp, Jim_Obj *substObjPtr, + Jim_Obj **resObjPtrPtr, int flags); + + +JIM_EXPORT Jim_Obj *Jim_GetSourceInfo(Jim_Interp *interp, Jim_Obj *objPtr, + int *lineptr); + +JIM_EXPORT void Jim_SetSourceInfo(Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj *fileNameObj, int lineNumber); + + + +JIM_EXPORT void Jim_InitStack(Jim_Stack *stack); +JIM_EXPORT void Jim_FreeStack(Jim_Stack *stack); +JIM_EXPORT int Jim_StackLen(Jim_Stack *stack); +JIM_EXPORT void Jim_StackPush(Jim_Stack *stack, void *element); +JIM_EXPORT void * Jim_StackPop(Jim_Stack *stack); +JIM_EXPORT void * Jim_StackPeek(Jim_Stack *stack); +JIM_EXPORT void Jim_FreeStackElements(Jim_Stack *stack, void (*freeFunc)(void *ptr)); + + +JIM_EXPORT int Jim_InitHashTable (Jim_HashTable *ht, + const Jim_HashTableType *type, void *privdata); +JIM_EXPORT void Jim_ExpandHashTable (Jim_HashTable *ht, + unsigned int size); +JIM_EXPORT int Jim_AddHashEntry (Jim_HashTable *ht, const void *key, + void *val); +JIM_EXPORT int Jim_ReplaceHashEntry (Jim_HashTable *ht, + const void *key, void *val); +JIM_EXPORT int Jim_DeleteHashEntry (Jim_HashTable *ht, + const void *key); +JIM_EXPORT int Jim_FreeHashTable (Jim_HashTable *ht); +JIM_EXPORT Jim_HashEntry * Jim_FindHashEntry (Jim_HashTable *ht, + const void *key); +JIM_EXPORT Jim_HashTableIterator *Jim_GetHashTableIterator + (Jim_HashTable *ht); +JIM_EXPORT Jim_HashEntry * Jim_NextHashEntry + (Jim_HashTableIterator *iter); + + +JIM_EXPORT Jim_Obj * Jim_NewObj (Jim_Interp *interp); +JIM_EXPORT void Jim_FreeObj (Jim_Interp *interp, Jim_Obj *objPtr); +JIM_EXPORT void Jim_InvalidateStringRep (Jim_Obj *objPtr); +JIM_EXPORT Jim_Obj * Jim_DuplicateObj (Jim_Interp *interp, + Jim_Obj *objPtr); +JIM_EXPORT const char * Jim_GetString(Jim_Obj *objPtr, + int *lenPtr); +JIM_EXPORT const char *Jim_String(Jim_Obj *objPtr); +JIM_EXPORT int Jim_Length(Jim_Obj *objPtr); + + +JIM_EXPORT Jim_Obj * Jim_NewStringObj (Jim_Interp *interp, + const char *s, int len); +JIM_EXPORT Jim_Obj *Jim_NewStringObjUtf8(Jim_Interp *interp, + const char *s, int charlen); +JIM_EXPORT Jim_Obj * Jim_NewStringObjNoAlloc (Jim_Interp *interp, + char *s, int len); +JIM_EXPORT void Jim_AppendString (Jim_Interp *interp, Jim_Obj *objPtr, + const char *str, int len); +JIM_EXPORT void Jim_AppendObj (Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj *appendObjPtr); +JIM_EXPORT void Jim_AppendStrings (Jim_Interp *interp, + Jim_Obj *objPtr, ...); +JIM_EXPORT int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr); +JIM_EXPORT int Jim_StringMatchObj (Jim_Interp *interp, Jim_Obj *patternObjPtr, + Jim_Obj *objPtr, int nocase); +JIM_EXPORT Jim_Obj * Jim_StringRangeObj (Jim_Interp *interp, + Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, + Jim_Obj *lastObjPtr); +JIM_EXPORT Jim_Obj * Jim_FormatString (Jim_Interp *interp, + Jim_Obj *fmtObjPtr, int objc, Jim_Obj *const *objv); +JIM_EXPORT Jim_Obj * Jim_ScanString (Jim_Interp *interp, Jim_Obj *strObjPtr, + Jim_Obj *fmtObjPtr, int flags); +JIM_EXPORT int Jim_CompareStringImmediate (Jim_Interp *interp, + Jim_Obj *objPtr, const char *str); +JIM_EXPORT int Jim_StringCompareObj(Jim_Interp *interp, Jim_Obj *firstObjPtr, + Jim_Obj *secondObjPtr, int nocase); +JIM_EXPORT int Jim_Utf8Length(Jim_Interp *interp, Jim_Obj *objPtr); + + +JIM_EXPORT Jim_Obj * Jim_NewReference (Jim_Interp *interp, + Jim_Obj *objPtr, Jim_Obj *tagPtr, Jim_Obj *cmdNamePtr); +JIM_EXPORT Jim_Reference * Jim_GetReference (Jim_Interp *interp, + Jim_Obj *objPtr); +JIM_EXPORT int Jim_SetFinalizer (Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *cmdNamePtr); +JIM_EXPORT int Jim_GetFinalizer (Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj **cmdNamePtrPtr); + + +JIM_EXPORT Jim_Interp * Jim_CreateInterp (void); +JIM_EXPORT void Jim_FreeInterp (Jim_Interp *i); +JIM_EXPORT int Jim_GetExitCode (Jim_Interp *interp); +JIM_EXPORT const char *Jim_ReturnCode(int code); +JIM_EXPORT void Jim_SetResultFormatted(Jim_Interp *interp, const char *format, ...); + + +JIM_EXPORT void Jim_RegisterCoreCommands (Jim_Interp *interp); +JIM_EXPORT int Jim_CreateCommand (Jim_Interp *interp, + const char *cmdName, Jim_CmdProc *cmdProc, void *privData, + Jim_DelCmdProc *delProc); +JIM_EXPORT int Jim_DeleteCommand (Jim_Interp *interp, + Jim_Obj *cmdNameObj); +JIM_EXPORT int Jim_RenameCommand (Jim_Interp *interp, + Jim_Obj *oldNameObj, Jim_Obj *newNameObj); +JIM_EXPORT Jim_Cmd * Jim_GetCommand (Jim_Interp *interp, + Jim_Obj *objPtr, int flags); +JIM_EXPORT int Jim_SetVariable (Jim_Interp *interp, + Jim_Obj *nameObjPtr, Jim_Obj *valObjPtr); +JIM_EXPORT int Jim_SetVariableStr (Jim_Interp *interp, + const char *name, Jim_Obj *objPtr); +JIM_EXPORT int Jim_SetGlobalVariableStr (Jim_Interp *interp, + const char *name, Jim_Obj *objPtr); +JIM_EXPORT int Jim_SetVariableStrWithStr (Jim_Interp *interp, + const char *name, const char *val); +JIM_EXPORT int Jim_SetVariableLink (Jim_Interp *interp, + Jim_Obj *nameObjPtr, Jim_Obj *targetNameObjPtr, + Jim_CallFrame *targetCallFrame); +JIM_EXPORT Jim_Obj * Jim_MakeGlobalNamespaceName(Jim_Interp *interp, + Jim_Obj *nameObjPtr); +JIM_EXPORT Jim_Obj * Jim_GetVariable (Jim_Interp *interp, + Jim_Obj *nameObjPtr, int flags); +JIM_EXPORT Jim_Obj * Jim_GetGlobalVariable (Jim_Interp *interp, + Jim_Obj *nameObjPtr, int flags); +JIM_EXPORT Jim_Obj * Jim_GetVariableStr (Jim_Interp *interp, + const char *name, int flags); +JIM_EXPORT Jim_Obj * Jim_GetGlobalVariableStr (Jim_Interp *interp, + const char *name, int flags); +JIM_EXPORT int Jim_UnsetVariable (Jim_Interp *interp, + Jim_Obj *nameObjPtr, int flags); + + +JIM_EXPORT Jim_CallFrame *Jim_GetCallFrameByLevel(Jim_Interp *interp, + Jim_Obj *levelObjPtr); + + +JIM_EXPORT int Jim_Collect (Jim_Interp *interp); +JIM_EXPORT void Jim_CollectIfNeeded (Jim_Interp *interp); + + +JIM_EXPORT int Jim_GetIndex (Jim_Interp *interp, Jim_Obj *objPtr, + int *indexPtr); + + +JIM_EXPORT Jim_Obj * Jim_NewListObj (Jim_Interp *interp, + Jim_Obj *const *elements, int len); +JIM_EXPORT void Jim_ListInsertElements (Jim_Interp *interp, + Jim_Obj *listPtr, int listindex, int objc, Jim_Obj *const *objVec); +JIM_EXPORT void Jim_ListAppendElement (Jim_Interp *interp, + Jim_Obj *listPtr, Jim_Obj *objPtr); +JIM_EXPORT void Jim_ListAppendList (Jim_Interp *interp, + Jim_Obj *listPtr, Jim_Obj *appendListPtr); +JIM_EXPORT int Jim_ListLength (Jim_Interp *interp, Jim_Obj *objPtr); +JIM_EXPORT int Jim_ListIndex (Jim_Interp *interp, Jim_Obj *listPrt, + int listindex, Jim_Obj **objPtrPtr, int seterr); +JIM_EXPORT Jim_Obj *Jim_ListGetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx); +JIM_EXPORT int Jim_SetListIndex (Jim_Interp *interp, + Jim_Obj *varNamePtr, Jim_Obj *const *indexv, int indexc, + Jim_Obj *newObjPtr); +JIM_EXPORT Jim_Obj * Jim_ConcatObj (Jim_Interp *interp, int objc, + Jim_Obj *const *objv); +JIM_EXPORT Jim_Obj *Jim_ListJoin(Jim_Interp *interp, + Jim_Obj *listObjPtr, const char *joinStr, int joinStrLen); + + +JIM_EXPORT Jim_Obj * Jim_NewDictObj (Jim_Interp *interp, + Jim_Obj *const *elements, int len); +JIM_EXPORT int Jim_DictKey (Jim_Interp *interp, Jim_Obj *dictPtr, + Jim_Obj *keyPtr, Jim_Obj **objPtrPtr, int flags); +JIM_EXPORT int Jim_DictKeysVector (Jim_Interp *interp, + Jim_Obj *dictPtr, Jim_Obj *const *keyv, int keyc, + Jim_Obj **objPtrPtr, int flags); +JIM_EXPORT int Jim_SetDictKeysVector (Jim_Interp *interp, + Jim_Obj *varNamePtr, Jim_Obj *const *keyv, int keyc, + Jim_Obj *newObjPtr, int flags); +JIM_EXPORT Jim_Obj **Jim_DictPairs(Jim_Interp *interp, + Jim_Obj *dictPtr, int *len); +JIM_EXPORT int Jim_DictAddElement(Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj *keyObjPtr, Jim_Obj *valueObjPtr); + +#define JIM_DICTMATCH_KEYS 0x0001 +#define JIM_DICTMATCH_VALUES 0x002 + +JIM_EXPORT int Jim_DictMatchTypes(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *patternObj, int match_type, int return_types); +JIM_EXPORT int Jim_DictSize(Jim_Interp *interp, Jim_Obj *objPtr); +JIM_EXPORT int Jim_DictInfo(Jim_Interp *interp, Jim_Obj *objPtr); +JIM_EXPORT Jim_Obj *Jim_DictMerge(Jim_Interp *interp, int objc, Jim_Obj *const *objv); + + +JIM_EXPORT int Jim_GetReturnCode (Jim_Interp *interp, Jim_Obj *objPtr, + int *intPtr); + + +JIM_EXPORT int Jim_EvalExpression (Jim_Interp *interp, + Jim_Obj *exprObjPtr); +JIM_EXPORT int Jim_GetBoolFromExpr (Jim_Interp *interp, + Jim_Obj *exprObjPtr, int *boolPtr); + + +JIM_EXPORT int Jim_GetBoolean(Jim_Interp *interp, Jim_Obj *objPtr, + int *booleanPtr); + + +JIM_EXPORT int Jim_GetWide (Jim_Interp *interp, Jim_Obj *objPtr, + jim_wide *widePtr); +JIM_EXPORT int Jim_GetWideExpr(Jim_Interp *interp, Jim_Obj *objPtr, + jim_wide *widePtr); +JIM_EXPORT int Jim_GetLong (Jim_Interp *interp, Jim_Obj *objPtr, + long *longPtr); +#define Jim_NewWideObj Jim_NewIntObj +JIM_EXPORT Jim_Obj * Jim_NewIntObj (Jim_Interp *interp, + jim_wide wideValue); + + +JIM_EXPORT int Jim_GetDouble(Jim_Interp *interp, Jim_Obj *objPtr, + double *doublePtr); +JIM_EXPORT void Jim_SetDouble(Jim_Interp *interp, Jim_Obj *objPtr, + double doubleValue); +JIM_EXPORT Jim_Obj * Jim_NewDoubleObj(Jim_Interp *interp, double doubleValue); + + +JIM_EXPORT void Jim_WrongNumArgs (Jim_Interp *interp, int argc, + Jim_Obj *const *argv, const char *msg); +JIM_EXPORT int Jim_GetEnum (Jim_Interp *interp, Jim_Obj *objPtr, + const char * const *tablePtr, int *indexPtr, const char *name, int flags); +JIM_EXPORT int Jim_CheckShowCommands(Jim_Interp *interp, Jim_Obj *objPtr, + const char *const *tablePtr); +JIM_EXPORT int Jim_ScriptIsComplete(Jim_Interp *interp, + Jim_Obj *scriptObj, char *stateCharPtr); + +JIM_EXPORT int Jim_FindByName(const char *name, const char * const array[], size_t len); + + +typedef void (Jim_InterpDeleteProc)(Jim_Interp *interp, void *data); +JIM_EXPORT void * Jim_GetAssocData(Jim_Interp *interp, const char *key); +JIM_EXPORT int Jim_SetAssocData(Jim_Interp *interp, const char *key, + Jim_InterpDeleteProc *delProc, void *data); +JIM_EXPORT int Jim_DeleteAssocData(Jim_Interp *interp, const char *key); +JIM_EXPORT int Jim_CheckAbiVersion(Jim_Interp *interp, int abi_version); + + + + +JIM_EXPORT int Jim_PackageProvide (Jim_Interp *interp, + const char *name, const char *ver, int flags); +JIM_EXPORT int Jim_PackageRequire (Jim_Interp *interp, + const char *name, int flags); +#define Jim_PackageProvideCheck(INTERP, NAME) \ + if (Jim_CheckAbiVersion(INTERP, JIM_ABI_VERSION) == JIM_ERR || Jim_PackageProvide(INTERP, NAME, "1.0", JIM_ERRMSG)) \ + return JIM_ERR + + +JIM_EXPORT void Jim_MakeErrorMessage (Jim_Interp *interp); + + +JIM_EXPORT int Jim_InteractivePrompt (Jim_Interp *interp); +JIM_EXPORT void Jim_HistoryLoad(const char *filename); +JIM_EXPORT void Jim_HistorySave(const char *filename); +JIM_EXPORT char *Jim_HistoryGetline(Jim_Interp *interp, const char *prompt); +JIM_EXPORT void Jim_HistorySetCompletion(Jim_Interp *interp, Jim_Obj *completionCommandObj); +JIM_EXPORT void Jim_HistorySetHints(Jim_Interp *interp, Jim_Obj *hintsCommandObj); +JIM_EXPORT void Jim_HistoryAdd(const char *line); +JIM_EXPORT void Jim_HistoryShow(void); +JIM_EXPORT void Jim_HistorySetMaxLen(int length); +JIM_EXPORT int Jim_HistoryGetMaxLen(void); + + +JIM_EXPORT int Jim_InitStaticExtensions(Jim_Interp *interp); +JIM_EXPORT int Jim_StringToWide(const char *str, jim_wide *widePtr, int base); +JIM_EXPORT int Jim_IsBigEndian(void); + +#define Jim_CheckSignal(i) ((i)->signal_level && (i)->sigmask) +JIM_EXPORT void Jim_SignalSetIgnored(jim_wide mask); + + +JIM_EXPORT int Jim_LoadLibrary(Jim_Interp *interp, const char *pathName); +JIM_EXPORT void Jim_FreeLoadHandles(Jim_Interp *interp); + + +JIM_EXPORT int Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command); + + +JIM_EXPORT int Jim_IsDict(Jim_Obj *objPtr); +JIM_EXPORT int Jim_IsList(Jim_Obj *objPtr); + +#ifdef __cplusplus +} +#endif + +#endif + +#ifndef JIM_SUBCMD_H +#define JIM_SUBCMD_H + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define JIM_MODFLAG_HIDDEN 0x0001 +#define JIM_MODFLAG_FULLARGV 0x0002 + + + +typedef int jim_subcmd_function(Jim_Interp *interp, int argc, Jim_Obj *const *argv); + +typedef struct { + const char *cmd; + const char *args; + jim_subcmd_function *function; + short minargs; + short maxargs; + unsigned short flags; +} jim_subcmd_type; + +#define JIM_DEF_SUBCMD(name, args, minargs, maxargs) { name, args, NULL, minargs, maxargs } +#define JIM_DEF_SUBCMD_HIDDEN(name, args, minargs, maxargs) { name, args, NULL, minargs, maxargs, JIM_MODFLAG_HIDDEN } + +const jim_subcmd_type * +Jim_ParseSubCmd(Jim_Interp *interp, const jim_subcmd_type *command_table, int argc, Jim_Obj *const *argv); + +int Jim_SubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv); + +int Jim_CallSubCmd(Jim_Interp *interp, const jim_subcmd_type *ct, int argc, Jim_Obj *const *argv); + +void Jim_SubCmdArgError(Jim_Interp *interp, const jim_subcmd_type *ct, Jim_Obj *subcmd); + +#ifdef __cplusplus +} +#endif + +#endif +#ifndef JIMREGEXP_H +#define JIMREGEXP_H + + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct { + int rm_so; + int rm_eo; +} regmatch_t; + + +typedef struct regexp { + + int re_nsub; + + + int cflags; + int err; + int regstart; + int reganch; + int regmust; + int regmlen; + int *program; + + + const char *regparse; + int p; + int proglen; + + + int eflags; + const char *start; + const char *reginput; + const char *regbol; + + + regmatch_t *pmatch; + int nmatch; +} regexp; + +typedef regexp regex_t; + +#define REG_EXTENDED 0 +#define REG_NEWLINE 1 +#define REG_ICASE 2 + +#define REG_NOTBOL 16 + +enum { + REG_NOERROR, + REG_NOMATCH, + REG_BADPAT, + REG_ERR_NULL_ARGUMENT, + REG_ERR_UNKNOWN, + REG_ERR_TOO_BIG, + REG_ERR_NOMEM, + REG_ERR_TOO_MANY_PAREN, + REG_ERR_UNMATCHED_PAREN, + REG_ERR_UNMATCHED_BRACES, + REG_ERR_BAD_COUNT, + REG_ERR_JUNK_ON_END, + REG_ERR_OPERAND_COULD_BE_EMPTY, + REG_ERR_NESTED_COUNT, + REG_ERR_INTERNAL, + REG_ERR_COUNT_FOLLOWS_NOTHING, + REG_ERR_INVALID_ESCAPE, + REG_ERR_CORRUPTED, + REG_ERR_NULL_CHAR, + REG_ERR_UNMATCHED_BRACKET, + REG_ERR_NUM +}; + +int jim_regcomp(regex_t *preg, const char *regex, int cflags); +int jim_regexec(regex_t *preg, const char *string, size_t nmatch, regmatch_t pmatch[], int eflags); +size_t jim_regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size); +void jim_regfree(regex_t *preg); + +#ifdef __cplusplus +} +#endif + +#endif +#ifndef JIM_SIGNAL_H +#define JIM_SIGNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +const char *Jim_SignalId(int sig); + +#ifdef __cplusplus +} +#endif + +#endif +#ifndef JIMIOCOMPAT_H +#define JIMIOCOMPAT_H + + +#include +#include +#include + + +void Jim_SetResultErrno(Jim_Interp *interp, const char *msg); + +int Jim_OpenForWrite(const char *filename, int append); + +int Jim_OpenForRead(const char *filename); + +#if defined(__MINGW32__) || defined(_WIN32) + #ifndef STRICT + #define STRICT + #endif + #define WIN32_LEAN_AND_MEAN + #include + #include + #include + #include + + typedef HANDLE phandle_t; + #define JIM_BAD_PHANDLE INVALID_HANDLE_VALUE + + + #define WIFEXITED(STATUS) (((STATUS) & 0xff00) == 0) + #define WEXITSTATUS(STATUS) ((STATUS) & 0x00ff) + #define WIFSIGNALED(STATUS) (((STATUS) & 0xff00) != 0) + #define WTERMSIG(STATUS) (((STATUS) >> 8) & 0xff) + #define WNOHANG 1 + + int Jim_Errno(void); + + long waitpid(phandle_t phandle, int *status, int nohang); + + phandle_t JimWaitPid(long processid, int *status, int nohang); + + long JimProcessPid(phandle_t phandle); + + #define HAVE_PIPE + #define pipe(P) _pipe((P), 0, O_NOINHERIT) + + typedef struct __stat64 jim_stat_t; + #define Jim_Stat _stat64 + #define Jim_FileStat _fstat64 + #define Jim_Lseek _lseeki64 + #define O_TEXT _O_TEXT + #define O_BINARY _O_BINARY + #define Jim_SetMode _setmode + #ifndef STDIN_FILENO + #define STDIN_FILENO 0 + #endif + +#else + #if defined(HAVE_STAT64) + typedef struct stat64 jim_stat_t; + #define Jim_Stat stat64 + #if defined(HAVE_FSTAT64) + #define Jim_FileStat fstat64 + #endif + #if defined(HAVE_LSTAT64) + #define Jim_LinkStat lstat64 + #endif + #else + typedef struct stat jim_stat_t; + #define Jim_Stat stat + #if defined(HAVE_FSTAT) + #define Jim_FileStat fstat + #endif + #if defined(HAVE_LSTAT) + #define Jim_LinkStat lstat + #endif + #endif + #if defined(HAVE_LSEEK64) + #define Jim_Lseek lseek64 + #else + #define Jim_Lseek lseek + #endif + + #if defined(HAVE_UNISTD_H) + #include + #include + #include + + typedef int phandle_t; + #define Jim_Errno() errno + #define JIM_BAD_PHANDLE -1 + #define JimProcessPid(PIDTYPE) (PIDTYPE) + #define JimWaitPid waitpid + + #ifndef HAVE_EXECVPE + #define execvpe(ARG0, ARGV, ENV) execvp(ARG0, ARGV) + #endif + #endif + + #ifndef O_TEXT + #define O_TEXT 0 + #endif + +#endif + +# ifndef MAXPATHLEN +# ifdef PATH_MAX +# define MAXPATHLEN PATH_MAX +# else +# define MAXPATHLEN JIM_PATH_LEN +# endif +# endif + + +int Jim_FileStoreStatData(Jim_Interp *interp, Jim_Obj *varName, const jim_stat_t *sb); + +#endif +int Jim_bootstrapInit(Jim_Interp *interp) +{ + if (Jim_PackageProvide(interp, "bootstrap", "1.0", JIM_ERRMSG)) + return JIM_ERR; + + return Jim_EvalSource(interp, "bootstrap.tcl", 1, +"\n" +"proc package {cmd args} {\n" +" if {$cmd eq \"require\"} {\n" +" foreach path $::auto_path {\n" +" lassign $args pkg\n" +" set pkgpath $path/$pkg.tcl\n" +" if {$path eq \".\"} {\n" +" set pkgpath $pkg.tcl\n" +" }\n" +" if {[file exists $pkgpath]} {\n" +" tailcall uplevel #0 [list source $pkgpath]\n" +" }\n" +" }\n" +" }\n" +"}\n" +"set tcl_platform(bootstrap) 1\n" +); +} +int Jim_initjimshInit(Jim_Interp *interp) +{ + if (Jim_PackageProvide(interp, "initjimsh", "1.0", JIM_ERRMSG)) + return JIM_ERR; + + return Jim_EvalSource(interp, "initjimsh.tcl", 1, +"\n" +"\n" +"\n" +"proc _jimsh_init {} {\n" +" rename _jimsh_init {}\n" +" global jim::exe jim::argv0 tcl_interactive auto_path tcl_platform\n" +"\n" +"\n" +" if {[exists jim::argv0]} {\n" +" if {[string match \"*/*\" $jim::argv0]} {\n" +" set jim::exe [file join [pwd] $jim::argv0]\n" +" } else {\n" +" set jim::argv0 [file tail $jim::argv0]\n" +" set path [split [env PATH \"\"] $tcl_platform(pathSeparator)]\n" +" if {$tcl_platform(platform) eq \"windows\"} {\n" +"\n" +" set path [lmap p [list \"\" {*}$path] { string map {\\\\ /} $p }]\n" +" }\n" +" foreach p $path {\n" +" set exec [file join [pwd] $p $jim::argv0]\n" +" if {[file executable $exec]} {\n" +" set jim::exe $exec\n" +" break\n" +" }\n" +" }\n" +" }\n" +" }\n" +"\n" +"\n" +" lappend p {*}[split [env JIMLIB {}] $tcl_platform(pathSeparator)]\n" +" if {[exists jim::exe]} {\n" +" lappend p [file dirname $jim::exe]\n" +" }\n" +" lappend p {*}$auto_path\n" +" set auto_path $p\n" +"\n" +" if {$tcl_interactive && [env HOME {}] ne \"\"} {\n" +" foreach src {.jimrc jimrc.tcl} {\n" +" if {[file exists [env HOME]/$src]} {\n" +" uplevel #0 source [env HOME]/$src\n" +" break\n" +" }\n" +" }\n" +" }\n" +" return \"\"\n" +"}\n" +"\n" +"if {$tcl_platform(platform) eq \"windows\"} {\n" +" set jim::argv0 [string map {\\\\ /} $jim::argv0]\n" +"}\n" +"\n" +"\n" +"set tcl::autocomplete_commands {array clock debug dict file history info namespace package signal socket string tcl::prefix zlib}\n" +"\n" +"\n" +"\n" +"proc tcl::autocomplete {prefix} {\n" +" if {[set space [string first \" \" $prefix]] != -1} {\n" +" set cmd [string range $prefix 0 $space-1]\n" +" if {$cmd in $::tcl::autocomplete_commands || [info channel $cmd] ne \"\"} {\n" +" set arg [string range $prefix $space+1 end]\n" +"\n" +" return [lmap p [$cmd -commands] {\n" +" if {![string match \"${arg}*\" $p]} continue\n" +" function \"$cmd $p\"\n" +" }]\n" +" }\n" +" }\n" +"\n" +" if {[string match \"source *\" $prefix]} {\n" +" set path [string range $prefix 7 end]\n" +" return [lmap p [glob -nocomplain \"${path}*\"] {\n" +" function \"source $p\"\n" +" }]\n" +" }\n" +"\n" +" return [lmap p [lsort [info commands $prefix*]] {\n" +" if {[string match \"* *\" $p]} {\n" +" continue\n" +" }\n" +" function $p\n" +" }]\n" +"}\n" +"\n" +"\n" +"set tcl::stdhint_commands {array clock debug dict file history info namespace package signal string zlib}\n" +"\n" +"set tcl::stdhint_cols {\n" +" none {0}\n" +" black {30}\n" +" red {31}\n" +" green {32}\n" +" yellow {33}\n" +" blue {34}\n" +" purple {35}\n" +" cyan {36}\n" +" normal {37}\n" +" grey {30 1}\n" +" gray {30 1}\n" +" lred {31 1}\n" +" lgreen {32 1}\n" +" lyellow {33 1}\n" +" lblue {34 1}\n" +" lpurple {35 1}\n" +" lcyan {36 1}\n" +" white {37 1}\n" +"}\n" +"\n" +"\n" +"set tcl::stdhint_col $tcl::stdhint_cols(lcyan)\n" +"\n" +"\n" +"proc tcl::stdhint {string} {\n" +" set result \"\"\n" +" if {[llength $string] >= 2} {\n" +" lassign $string cmd arg\n" +" if {$cmd in $::tcl::stdhint_commands || [info channel $cmd] ne \"\"} {\n" +" catch {\n" +" set help [$cmd -help $arg]\n" +" if {[string match \"Usage: $cmd *\" $help]} {\n" +" set n [llength $string]\n" +" set subcmd [lindex $help $n]\n" +" incr n\n" +" set hint [join [lrange $help $n end]]\n" +" set prefix \"\"\n" +" if {![string match \"* \" $string]} {\n" +" if {$n == 3 && $subcmd ne $arg} {\n" +"\n" +" set prefix \"[string range $subcmd [string length $arg] end] \"\n" +" } else {\n" +" set prefix \" \"\n" +" }\n" +" }\n" +" set result [list $prefix$hint {*}$::tcl::stdhint_col]\n" +" }\n" +" }\n" +" }\n" +" }\n" +" return $result\n" +"}\n" +"\n" +"_jimsh_init\n" +); +} +int Jim_globInit(Jim_Interp *interp) +{ + if (Jim_PackageProvide(interp, "glob", "1.0", JIM_ERRMSG)) + return JIM_ERR; + + return Jim_EvalSource(interp, "glob.tcl", 1, +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"package require readdir\n" +"\n" +"\n" +"proc glob.globdir {dir pattern} {\n" +" if {[file exists $dir/$pattern]} {\n" +"\n" +" return [list $pattern]\n" +" }\n" +"\n" +" set result {}\n" +" set files [readdir $dir]\n" +" lappend files . ..\n" +"\n" +" foreach name $files {\n" +" if {[string match $pattern $name]} {\n" +"\n" +" if {[string index $name 0] eq \".\" && [string index $pattern 0] ne \".\"} {\n" +" continue\n" +" }\n" +" lappend result $name\n" +" }\n" +" }\n" +"\n" +" return $result\n" +"}\n" +"\n" +"\n" +"\n" +"\n" +"proc glob.explode {pattern} {\n" +" set oldexp {}\n" +" set newexp {\"\"}\n" +"\n" +" while 1 {\n" +" set oldexp $newexp\n" +" set newexp {}\n" +" set ob [string first \\{ $pattern]\n" +" set cb [string first \\} $pattern]\n" +"\n" +" if {$ob < $cb && $ob != -1} {\n" +" set mid [string range $pattern 0 $ob-1]\n" +" set subexp [lassign [glob.explode [string range $pattern $ob+1 end]] pattern]\n" +" if {$pattern eq \"\"} {\n" +" error \"unmatched open brace in glob pattern\"\n" +" }\n" +" set pattern [string range $pattern 1 end]\n" +"\n" +" foreach subs $subexp {\n" +" foreach sub [split $subs ,] {\n" +" foreach old $oldexp {\n" +" lappend newexp $old$mid$sub\n" +" }\n" +" }\n" +" }\n" +" } elseif {$cb != -1} {\n" +" set suf [string range $pattern 0 $cb-1]\n" +" set rest [string range $pattern $cb end]\n" +" break\n" +" } else {\n" +" set suf $pattern\n" +" set rest \"\"\n" +" break\n" +" }\n" +" }\n" +"\n" +" foreach old $oldexp {\n" +" lappend newexp $old$suf\n" +" }\n" +" list $rest {*}$newexp\n" +"}\n" +"\n" +"\n" +"\n" +"proc glob.glob {base pattern} {\n" +" set dir [file dirname $pattern]\n" +" if {$pattern eq $dir || $pattern eq \"\"} {\n" +" return [list [file join $base $dir] $pattern]\n" +" } elseif {$pattern eq [file tail $pattern]} {\n" +" set dir \"\"\n" +" }\n" +"\n" +"\n" +" set dirlist [glob.glob $base $dir]\n" +" set pattern [file tail $pattern]\n" +"\n" +"\n" +" set result {}\n" +" foreach {realdir dir} $dirlist {\n" +" if {![file isdir $realdir]} {\n" +" continue\n" +" }\n" +" if {[string index $dir end] ne \"/\" && $dir ne \"\"} {\n" +" append dir /\n" +" }\n" +" foreach name [glob.globdir $realdir $pattern] {\n" +" lappend result [file join $realdir $name] $dir$name\n" +" }\n" +" }\n" +" return $result\n" +"}\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"proc glob {args} {\n" +" set nocomplain 0\n" +" set base \"\"\n" +" set tails 0\n" +"\n" +" set n 0\n" +" foreach arg $args {\n" +" if {[info exists param]} {\n" +" set $param $arg\n" +" unset param\n" +" incr n\n" +" continue\n" +" }\n" +" switch -glob -- $arg {\n" +" -d* {\n" +" set switch $arg\n" +" set param base\n" +" }\n" +" -n* {\n" +" set nocomplain 1\n" +" }\n" +" -ta* {\n" +" set tails 1\n" +" }\n" +" -- {\n" +" incr n\n" +" break\n" +" }\n" +" -* {\n" +" return -code error \"bad option \\\"$arg\\\": must be -directory, -nocomplain, -tails, or --\"\n" +" }\n" +" * {\n" +" break\n" +" }\n" +" }\n" +" incr n\n" +" }\n" +" if {[info exists param]} {\n" +" return -code error \"missing argument to \\\"$switch\\\"\"\n" +" }\n" +" if {[llength $args] <= $n} {\n" +" return -code error \"wrong # args: should be \\\"glob ?options? pattern ?pattern ...?\\\"\"\n" +" }\n" +"\n" +" set args [lrange $args $n end]\n" +"\n" +" set result {}\n" +" foreach pattern $args {\n" +" set escpattern [string map {\n" +" \\\\\\\\ \\x01 \\\\\\{ \\x02 \\\\\\} \\x03 \\\\, \\x04\n" +" } $pattern]\n" +" set patexps [lassign [glob.explode $escpattern] rest]\n" +" if {$rest ne \"\"} {\n" +" return -code error \"unmatched close brace in glob pattern\"\n" +" }\n" +" foreach patexp $patexps {\n" +" set patexp [string map {\n" +" \\x01 \\\\\\\\ \\x02 \\{ \\x03 \\} \\x04 ,\n" +" } $patexp]\n" +" foreach {realname name} [glob.glob $base $patexp] {\n" +" incr n\n" +" if {$tails} {\n" +" lappend result $name\n" +" } else {\n" +" lappend result [file join $base $name]\n" +" }\n" +" }\n" +" }\n" +" }\n" +"\n" +" if {!$nocomplain && [llength $result] == 0} {\n" +" set s $(([llength $args] > 1) ? \"s\" : \"\")\n" +" return -code error \"no files matched glob pattern$s \\\"[join $args]\\\"\"\n" +" }\n" +"\n" +" return $result\n" +"}\n" +); +} +int Jim_stdlibInit(Jim_Interp *interp) +{ + if (Jim_PackageProvide(interp, "stdlib", "1.0", JIM_ERRMSG)) + return JIM_ERR; + + return Jim_EvalSource(interp, "stdlib.tcl", 1, +"\n" +"\n" +"if {![exists -command ref]} {\n" +"\n" +" proc ref {args} {{count 0}} {\n" +" format %08x [incr count]\n" +" }\n" +"}\n" +"\n" +"\n" +"proc lambda {arglist args} {\n" +" tailcall proc [ref {} function lambda.finalizer] $arglist {*}$args\n" +"}\n" +"\n" +"proc lambda.finalizer {name val} {\n" +" rename $name {}\n" +"}\n" +"\n" +"\n" +"proc curry {args} {\n" +" alias [ref {} function lambda.finalizer] {*}$args\n" +"}\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"proc function {value} {\n" +" return $value\n" +"}\n" +"\n" +"\n" +"proc stackdump {stacktrace} {\n" +" set lines {}\n" +" lappend lines \"Traceback (most recent call last):\"\n" +" foreach {cmd l f p} [lreverse $stacktrace] {\n" +" set line {}\n" +" if {$f ne \"\"} {\n" +" append line \" File \\\"$f\\\", line $l\"\n" +" }\n" +" if {$p ne \"\"} {\n" +" append line \", in $p\"\n" +" }\n" +" if {$line ne \"\"} {\n" +" lappend lines $line\n" +" if {$cmd ne \"\"} {\n" +" set nl [string first \\n $cmd 1]\n" +" if {$nl >= 0} {\n" +" set cmd [string range $cmd 0 $nl-1]...\n" +" }\n" +" lappend lines \" $cmd\"\n" +" }\n" +" }\n" +" }\n" +" if {[llength $lines] > 1} {\n" +" return [join $lines \\n]\n" +" }\n" +"}\n" +"\n" +"\n" +"\n" +"proc defer {script} {\n" +" upvar jim::defer v\n" +" lappend v $script\n" +"}\n" +"\n" +"\n" +"\n" +"proc errorInfo {msg {stacktrace \"\"}} {\n" +" if {$stacktrace eq \"\"} {\n" +"\n" +" set stacktrace [info stacktrace]\n" +" }\n" +" lassign $stacktrace p f l cmd\n" +" if {$f ne \"\"} {\n" +" set result \"$f:$l: Error: \"\n" +" }\n" +" append result \"$msg\\n\"\n" +" append result [stackdump $stacktrace]\n" +"\n" +"\n" +" string trim $result\n" +"}\n" +"\n" +"\n" +"\n" +"proc {info nameofexecutable} {} {\n" +" if {[exists ::jim::exe]} {\n" +" return $::jim::exe\n" +" }\n" +"}\n" +"\n" +"\n" +"proc {dict update} {&varName args script} {\n" +" set keys {}\n" +" foreach {n v} $args {\n" +" upvar $v var_$v\n" +" if {[dict exists $varName $n]} {\n" +" set var_$v [dict get $varName $n]\n" +" }\n" +" }\n" +" catch {uplevel 1 $script} msg opts\n" +" if {[info exists varName]} {\n" +" foreach {n v} $args {\n" +" if {[info exists var_$v]} {\n" +" dict set varName $n [set var_$v]\n" +" } else {\n" +" dict unset varName $n\n" +" }\n" +" }\n" +" }\n" +" return {*}$opts $msg\n" +"}\n" +"\n" +"proc {dict replace} {dictionary {args {key value}}} {\n" +" if {[llength ${key value}] % 2} {\n" +" tailcall {dict replace}\n" +" }\n" +" tailcall dict merge $dictionary ${key value}\n" +"}\n" +"\n" +"\n" +"proc {dict lappend} {varName key {args value}} {\n" +" upvar $varName dict\n" +" if {[exists dict] && [dict exists $dict $key]} {\n" +" set list [dict get $dict $key]\n" +" }\n" +" lappend list {*}$value\n" +" dict set dict $key $list\n" +"}\n" +"\n" +"\n" +"proc {dict append} {varName key {args value}} {\n" +" upvar $varName dict\n" +" if {[exists dict] && [dict exists $dict $key]} {\n" +" set str [dict get $dict $key]\n" +" }\n" +" append str {*}$value\n" +" dict set dict $key $str\n" +"}\n" +"\n" +"\n" +"proc {dict incr} {varName key {increment 1}} {\n" +" upvar $varName dict\n" +" if {[exists dict] && [dict exists $dict $key]} {\n" +" set value [dict get $dict $key]\n" +" }\n" +" incr value $increment\n" +" dict set dict $key $value\n" +"}\n" +"\n" +"\n" +"proc {dict remove} {dictionary {args key}} {\n" +" foreach k $key {\n" +" dict unset dictionary $k\n" +" }\n" +" return $dictionary\n" +"}\n" +"\n" +"\n" +"proc {dict for} {vars dictionary script} {\n" +" if {[llength $vars] != 2} {\n" +" return -code error \"must have exactly two variable names\"\n" +" }\n" +" dict size $dictionary\n" +" tailcall foreach $vars $dictionary $script\n" +"}\n" +); +} +int Jim_tclcompatInit(Jim_Interp *interp) +{ + if (Jim_PackageProvide(interp, "tclcompat", "1.0", JIM_ERRMSG)) + return JIM_ERR; + + return Jim_EvalSource(interp, "tclcompat.tcl", 1, +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"set env [env]\n" +"\n" +"\n" +"if {[exists -command stdout]} {\n" +"\n" +" foreach p {gets flush close eof seek tell} {\n" +" proc $p {chan args} {p} {\n" +" tailcall $chan $p {*}$args\n" +" }\n" +" }\n" +" unset p\n" +"\n" +"\n" +"\n" +" proc puts {{-nonewline {}} {chan stdout} msg} {\n" +" if {${-nonewline} ni {-nonewline {}}} {\n" +" tailcall ${-nonewline} puts $msg\n" +" }\n" +" tailcall $chan puts {*}${-nonewline} $msg\n" +" }\n" +"\n" +"\n" +"\n" +"\n" +"\n" +" proc read {{-nonewline {}} chan} {\n" +" if {${-nonewline} ni {-nonewline {}}} {\n" +" tailcall ${-nonewline} read {*}${chan}\n" +" }\n" +" tailcall $chan read {*}${-nonewline}\n" +" }\n" +"\n" +" proc fconfigure {f args} {\n" +" foreach {n v} $args {\n" +" switch -glob -- $n {\n" +" -bl* {\n" +" $f ndelay $(!$v)\n" +" }\n" +" -bu* {\n" +" $f buffering $v\n" +" }\n" +" -tr* {\n" +" $f translation $v\n" +" }\n" +" default {\n" +" return -code error \"fconfigure: unknown option $n\"\n" +" }\n" +" }\n" +" }\n" +" }\n" +"}\n" +"\n" +"\n" +"proc fileevent {args} {\n" +" tailcall {*}$args\n" +"}\n" +"\n" +"\n" +"\n" +"proc parray {arrayname {pattern *} {puts puts}} {\n" +" upvar $arrayname a\n" +"\n" +" set max 0\n" +" foreach name [array names a $pattern]] {\n" +" if {[string length $name] > $max} {\n" +" set max [string length $name]\n" +" }\n" +" }\n" +" incr max [string length $arrayname]\n" +" incr max 2\n" +" foreach name [lsort [array names a $pattern]] {\n" +" $puts [format \"%-${max}s = %s\" $arrayname\\($name\\) $a($name)]\n" +" }\n" +"}\n" +"\n" +"\n" +"proc {file copy} {{force {}} source target} {\n" +" try {\n" +" if {$force ni {{} -force}} {\n" +" error \"bad option \\\"$force\\\": should be -force\"\n" +" }\n" +"\n" +" set in [open $source rb]\n" +"\n" +" if {[file exists $target]} {\n" +" if {$force eq \"\"} {\n" +" error \"error copying \\\"$source\\\" to \\\"$target\\\": file already exists\"\n" +" }\n" +"\n" +" if {$source eq $target} {\n" +" return\n" +" }\n" +"\n" +"\n" +" file stat $source ss\n" +" file stat $target ts\n" +" if {$ss(dev) == $ts(dev) && $ss(ino) == $ts(ino) && $ss(ino)} {\n" +" return\n" +" }\n" +" }\n" +" set out [open $target wb]\n" +" $in copyto $out\n" +" $out close\n" +" } on error {msg opts} {\n" +" incr opts(-level)\n" +" return {*}$opts $msg\n" +" } finally {\n" +" catch {$in close}\n" +" }\n" +"}\n" +"\n" +"\n" +"\n" +"proc popen {cmd {mode r}} {\n" +" lassign [pipe] r w\n" +" try {\n" +" if {[string match \"w*\" $mode]} {\n" +" lappend cmd <@$r &\n" +" set pids [exec {*}$cmd]\n" +" $r close\n" +" set f $w\n" +" } else {\n" +" lappend cmd >@$w &\n" +" set pids [exec {*}$cmd]\n" +" $w close\n" +" set f $r\n" +" }\n" +" lambda {cmd args} {f pids} {\n" +" if {$cmd eq \"pid\"} {\n" +" return $pids\n" +" }\n" +" if {$cmd eq \"close\"} {\n" +" $f close\n" +"\n" +" set retopts {}\n" +" foreach p $pids {\n" +" lassign [wait $p] status - rc\n" +" if {$status eq \"CHILDSTATUS\"} {\n" +" if {$rc == 0} {\n" +" continue\n" +" }\n" +" set msg \"child process exited abnormally\"\n" +" } else {\n" +" set msg \"child killed: received signal\"\n" +" }\n" +" set retopts [list -code error -errorcode [list $status $p $rc] $msg]\n" +" }\n" +" return {*}$retopts\n" +" }\n" +" tailcall $f $cmd {*}$args\n" +" }\n" +" } on error {error opts} {\n" +" $r close\n" +" $w close\n" +" error $error\n" +" }\n" +"}\n" +"\n" +"\n" +"local proc pid {{channelId {}}} {\n" +" if {$channelId eq \"\"} {\n" +" tailcall upcall pid\n" +" }\n" +" if {[catch {$channelId tell}]} {\n" +" return -code error \"can not find channel named \\\"$channelId\\\"\"\n" +" }\n" +" if {[catch {$channelId pid} pids]} {\n" +" return \"\"\n" +" }\n" +" return $pids\n" +"}\n" +"\n" +"\n" +"\n" +"proc throw {code {msg \"\"}} {\n" +" return -code $code $msg\n" +"}\n" +"\n" +"\n" +"proc {file delete force} {path} {\n" +" foreach e [readdir $path] {\n" +" file delete -force $path/$e\n" +" }\n" +" file delete $path\n" +"}\n" +); +} + + +#include +#include +#include +#include +#include +#ifdef HAVE_UNISTD_H +#include +#include +#endif +#ifdef HAVE_UTIL_H +#include +#endif +#ifdef HAVE_PTY_H +#include +#endif + + +#if defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SELECT) && defined(HAVE_NETINET_IN_H) && defined(HAVE_NETDB_H) && defined(HAVE_ARPA_INET_H) +#include +#include +#include +#include +#include +#ifdef HAVE_SYS_UN_H +#include +#endif +#define HAVE_SOCKETS +#elif defined (__MINGW32__) + +#endif + +#if defined(JIM_SSL) +#include +#include +#endif + +#ifdef HAVE_TERMIOS_H +#endif + + +#define AIO_CMD_LEN 32 +#define AIO_DEFAULT_RBUF_LEN 256 +#define AIO_DEFAULT_WBUF_LIMIT (64 * 1024) + +#define AIO_KEEPOPEN 1 +#define AIO_NODELETE 2 +#define AIO_EOF 4 +#define AIO_WBUF_NONE 8 +#define AIO_NONBLOCK 16 + +#define AIO_ONEREAD 32 + +enum wbuftype { + WBUF_OPT_NONE, + WBUF_OPT_LINE, + WBUF_OPT_FULL, +}; + +#if defined(JIM_IPV6) +#define IPV6 1 +#else +#define IPV6 0 +#ifndef PF_INET6 +#define PF_INET6 0 +#endif +#endif +#if defined(HAVE_SYS_UN_H) && defined(PF_UNIX) +#define UNIX_SOCKETS 1 +#else +#define UNIX_SOCKETS 0 +#endif + + + + +static int JimReadableTimeout(int fd, long ms) +{ +#ifdef HAVE_SELECT + int retval; + struct timeval tv; + fd_set rfds; + + FD_ZERO(&rfds); + + FD_SET(fd, &rfds); + tv.tv_sec = ms / 1000; + tv.tv_usec = (ms % 1000) * 1000; + + retval = select(fd + 1, &rfds, NULL, NULL, ms == 0 ? NULL : &tv); + + if (retval > 0) { + return JIM_OK; + } + return JIM_ERR; +#else + return JIM_OK; +#endif +} + + +struct AioFile; + +typedef struct { + int (*writer)(struct AioFile *af, const char *buf, int len); + int (*reader)(struct AioFile *af, char *buf, int len, int pending); + int (*error)(const struct AioFile *af); + const char *(*strerror)(struct AioFile *af); + int (*verify)(struct AioFile *af); +} JimAioFopsType; + +typedef struct AioFile +{ + Jim_Obj *filename; + int wbuft; + int flags; + long timeout; + int fd; + int addr_family; + void *ssl; + const JimAioFopsType *fops; + Jim_Obj *readbuf; + Jim_Obj *writebuf; + char *rbuf; + size_t rbuf_len; + size_t wbuf_limit; +} AioFile; + +static void aio_consume(Jim_Obj *objPtr, int n); + +static int stdio_writer(struct AioFile *af, const char *buf, int len) +{ + int ret = write(af->fd, buf, len); + if (ret < 0 && errno == EPIPE) { + aio_consume(af->writebuf, Jim_Length(af->writebuf)); + } + return ret; +} + +static int stdio_reader(struct AioFile *af, char *buf, int len, int nb) +{ + if (nb || af->timeout == 0 || JimReadableTimeout(af->fd, af->timeout) == JIM_OK) { + + int ret; + + errno = 0; + ret = read(af->fd, buf, len); + if (ret <= 0 && errno != EAGAIN && errno != EINTR) { + af->flags |= AIO_EOF; + } + return ret; + } + errno = ETIMEDOUT; + return -1; +} + +static int stdio_error(const AioFile *af) +{ + if (af->flags & AIO_EOF) { + return JIM_OK; + } + + switch (errno) { + case EAGAIN: + case EINTR: + case ETIMEDOUT: +#ifdef ECONNRESET + case ECONNRESET: +#endif +#ifdef ECONNABORTED + case ECONNABORTED: +#endif + return JIM_OK; + default: + return JIM_ERR; + } +} + +static const char *stdio_strerror(struct AioFile *af) +{ + return strerror(errno); +} + +static const JimAioFopsType stdio_fops = { + stdio_writer, + stdio_reader, + stdio_error, + stdio_strerror, + NULL, +}; + + +static void aio_set_nonblocking(AioFile *af, int nb) +{ +#ifdef O_NDELAY + int old = !!(af->flags & AIO_NONBLOCK); + if (old != nb) { + int fmode = fcntl(af->fd, F_GETFL); + if (nb) { + fmode |= O_NDELAY; + af->flags |= AIO_NONBLOCK; + } + else { + fmode &= ~O_NDELAY; + af->flags &= ~AIO_NONBLOCK; + } + (void)fcntl(af->fd, F_SETFL, fmode); + } +#endif +} + +static int aio_start_nonblocking(AioFile *af) +{ + int old = !!(af->flags & AIO_NONBLOCK); + if (af->timeout) { + aio_set_nonblocking(af, 1); + } + return old; +} + +static int JimAioSubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv); +static AioFile *JimMakeChannel(Jim_Interp *interp, int fd, Jim_Obj *filename, + const char *hdlfmt, int family, int flags); + + +static const char *JimAioErrorString(AioFile *af) +{ + if (af && af->fops) + return af->fops->strerror(af); + + return strerror(errno); +} + +static void JimAioSetError(Jim_Interp *interp, Jim_Obj *name) +{ + AioFile *af = Jim_CmdPrivData(interp); + + if (name) { + Jim_SetResultFormatted(interp, "%#s: %s", name, JimAioErrorString(af)); + } + else { + Jim_SetResultString(interp, JimAioErrorString(af), -1); + } +} + +static int aio_eof(AioFile *af) +{ + return af->flags & AIO_EOF; +} + +static int JimCheckStreamError(Jim_Interp *interp, AioFile *af) +{ + int ret = 0; + if (!aio_eof(af)) { + ret = af->fops->error(af); + if (ret) { + JimAioSetError(interp, af->filename); + } + } + return ret; +} + +static void aio_consume(Jim_Obj *objPtr, int n) +{ + assert(objPtr->bytes); + assert(n <= objPtr->length); + + + memmove(objPtr->bytes, objPtr->bytes + n, objPtr->length - n + 1); + objPtr->length -= n; +} + + +static int aio_flush(Jim_Interp *interp, AioFile *af); + +#ifdef jim_ext_eventloop +static int aio_autoflush(Jim_Interp *interp, void *clientData, int mask) +{ + AioFile *af = clientData; + + aio_flush(interp, af); + if (Jim_Length(af->writebuf) == 0) { + + return -1; + } + return 0; +} +#endif + + +static int aio_flush(Jim_Interp *interp, AioFile *af) +{ + int len; + const char *pt = Jim_GetString(af->writebuf, &len); + if (len) { + int ret = af->fops->writer(af, pt, len); + if (ret > 0) { + + aio_consume(af->writebuf, ret); + } + if (ret < 0) { + return JimCheckStreamError(interp, af); + } + if (Jim_Length(af->writebuf)) { +#ifdef jim_ext_eventloop + void *handler = Jim_FindFileHandler(interp, af->fd, JIM_EVENT_WRITABLE); + if (handler == NULL) { + Jim_CreateFileHandler(interp, af->fd, JIM_EVENT_WRITABLE, aio_autoflush, af, NULL); + return JIM_OK; + } + else if (handler == af) { + + return JIM_OK; + } +#endif + + Jim_SetResultString(interp, "send buffer is full", -1); + return JIM_ERR; + } + } + return JIM_OK; +} + +static int aio_read_len(Jim_Interp *interp, AioFile *af, unsigned flags, int neededLen) +{ + if (!af->readbuf) { + af->readbuf = Jim_NewStringObj(interp, NULL, 0); + } + + if (neededLen >= 0) { + neededLen -= Jim_Length(af->readbuf); + if (neededLen <= 0) { + return JIM_OK; + } + } + + while (neededLen && !aio_eof(af)) { + int retval; + int readlen; + + if (neededLen == -1) { + readlen = af->rbuf_len; + } + else { + readlen = (neededLen > af->rbuf_len ? af->rbuf_len : neededLen); + } + + if (!af->rbuf) { + af->rbuf = Jim_Alloc(af->rbuf_len); + } + retval = af->fops->reader(af, af->rbuf, readlen, flags & AIO_NONBLOCK); + if (retval > 0) { + if (retval) { + Jim_AppendString(interp, af->readbuf, af->rbuf, retval); + } + if (neededLen != -1) { + neededLen -= retval; + } + if (flags & AIO_ONEREAD) { + return JIM_OK; + } + continue; + } + if ((flags & AIO_ONEREAD) || JimCheckStreamError(interp, af)) { + return JIM_ERR; + } + break; + } + + return JIM_OK; +} + +static Jim_Obj *aio_read_consume(Jim_Interp *interp, AioFile *af, int neededLen) +{ + Jim_Obj *objPtr = NULL; + + if (neededLen < 0 || af->readbuf == NULL || Jim_Length(af->readbuf) <= neededLen) { + objPtr = af->readbuf; + af->readbuf = NULL; + } + else if (af->readbuf) { + + int len; + const char *pt = Jim_GetString(af->readbuf, &len); + + objPtr = Jim_NewStringObj(interp, pt, neededLen); + aio_consume(af->readbuf, neededLen); + } + + return objPtr; +} + +static void JimAioDelProc(Jim_Interp *interp, void *privData) +{ + AioFile *af = privData; + + JIM_NOTUSED(interp); + + + aio_flush(interp, af); + Jim_DecrRefCount(interp, af->writebuf); + +#if UNIX_SOCKETS + if (af->addr_family == PF_UNIX && (af->flags & AIO_NODELETE) == 0) { + + Jim_Obj *filenameObj = aio_sockname(interp, af->fd); + if (filenameObj) { + if (Jim_Length(filenameObj)) { + remove(Jim_String(filenameObj)); + } + Jim_FreeNewObj(interp, filenameObj); + } + } +#endif + + Jim_DecrRefCount(interp, af->filename); + +#ifdef jim_ext_eventloop + + Jim_DeleteFileHandler(interp, af->fd, JIM_EVENT_READABLE | JIM_EVENT_WRITABLE | JIM_EVENT_EXCEPTION); +#endif + +#if defined(JIM_SSL) + if (af->ssl != NULL) { + SSL_free(af->ssl); + } +#endif + if (!(af->flags & AIO_KEEPOPEN)) { + close(af->fd); + } + if (af->readbuf) { + Jim_FreeNewObj(interp, af->readbuf); + } + + Jim_Free(af->rbuf); + Jim_Free(af); +} + +static int aio_cmd_read(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + int nonewline = 0; + jim_wide neededLen = -1; + static const char * const options[] = { "-pending", "-nonewline", NULL }; + enum { OPT_PENDING, OPT_NONEWLINE }; + int option; + int nb; + Jim_Obj *objPtr; + + if (argc) { + if (*Jim_String(argv[0]) == '-') { + if (Jim_GetEnum(interp, argv[0], options, &option, NULL, JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + switch (option) { + case OPT_PENDING: + + break; + case OPT_NONEWLINE: + nonewline++; + break; + } + } + else { + if (Jim_GetWide(interp, argv[0], &neededLen) != JIM_OK) + return JIM_ERR; + if (neededLen < 0) { + Jim_SetResultString(interp, "invalid parameter: negative len", -1); + return JIM_ERR; + } + } + argc--; + argv++; + } + if (argc) { + return -1; + } + + + nb = aio_start_nonblocking(af); + + if (aio_read_len(interp, af, nb ? AIO_NONBLOCK : 0, neededLen) != JIM_OK) { + aio_set_nonblocking(af, nb); + return JIM_ERR; + } + objPtr = aio_read_consume(interp, af, neededLen); + + aio_set_nonblocking(af, nb); + + if (objPtr) { + if (nonewline) { + int len; + const char *s = Jim_GetString(objPtr, &len); + + if (len > 0 && s[len - 1] == '\n') { + objPtr->length--; + objPtr->bytes[objPtr->length] = '\0'; + } + } + Jim_SetResult(interp, objPtr); + } + else { + Jim_SetEmptyResult(interp); + } + return JIM_OK; +} + +int Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command) +{ + Jim_Cmd *cmdPtr = Jim_GetCommand(interp, command, JIM_ERRMSG); + + + if (cmdPtr && !cmdPtr->isproc && cmdPtr->u.native.cmdProc == JimAioSubCmdProc) { + return ((AioFile *) cmdPtr->u.native.privData)->fd; + } + Jim_SetResultFormatted(interp, "Not a filehandle: \"%#s\"", command); + return -1; +} + +static int aio_cmd_getfd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + + aio_flush(interp, af); + + Jim_SetResultInt(interp, af->fd); + + return JIM_OK; +} + +static int aio_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + jim_wide count = 0; + jim_wide maxlen = JIM_WIDE_MAX; + int ok = 1; + Jim_Obj *objv[4]; + + if (argc == 2) { + if (Jim_GetWide(interp, argv[1], &maxlen) != JIM_OK) { + return JIM_ERR; + } + } + + objv[0] = argv[0]; + objv[1] = Jim_NewStringObj(interp, "flush", -1); + if (Jim_EvalObjVector(interp, 2, objv) != JIM_OK) { + Jim_SetResultFormatted(interp, "Not a filehandle: \"%#s\"", argv[0]); + return JIM_ERR; + } + + + objv[0] = argv[0]; + objv[1] = Jim_NewStringObj(interp, "puts", -1); + objv[2] = Jim_NewStringObj(interp, "-nonewline", -1); + Jim_IncrRefCount(objv[1]); + Jim_IncrRefCount(objv[2]); + + while (count < maxlen) { + jim_wide len = maxlen - count; + if (len > af->rbuf_len) { + len = af->rbuf_len; + } + if (aio_read_len(interp, af, 0, len) != JIM_OK) { + ok = 0; + break; + } + objv[3] = aio_read_consume(interp, af, len); + count += Jim_Length(objv[3]); + if (Jim_EvalObjVector(interp, 4, objv) != JIM_OK) { + ok = 0; + break; + } + if (aio_eof(af)) { + break; + } + if (count >= 16384 && af->rbuf_len < 65536) { + + af->rbuf_len = 65536; + af->rbuf = Jim_Realloc(af->rbuf, af->rbuf_len); + } + } + + Jim_DecrRefCount(interp, objv[1]); + Jim_DecrRefCount(interp, objv[2]); + + if (!ok) { + return JIM_ERR; + } + + Jim_SetResultInt(interp, count); + + return JIM_OK; +} + +static int aio_cmd_gets(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + Jim_Obj *objPtr = NULL; + int len; + int nb; + unsigned flags = AIO_ONEREAD; + char *nl = NULL; + int offset = 0; + + errno = 0; + + + nb = aio_start_nonblocking(af); + if (nb) { + flags |= AIO_NONBLOCK; + } + + while (!aio_eof(af)) { + if (af->readbuf) { + const char *pt = Jim_GetString(af->readbuf, &len); + nl = memchr(pt + offset, '\n', len - offset); + if (nl) { + + objPtr = Jim_NewStringObj(interp, pt, nl - pt); + + aio_consume(af->readbuf, nl - pt + 1); + break; + } + offset = len; + } + + + if (aio_read_len(interp, af, flags, -1) != JIM_OK) { + break; + } + } + + aio_set_nonblocking(af, nb); + + if (!nl && aio_eof(af) && af->readbuf) { + + objPtr = af->readbuf; + af->readbuf = NULL; + } + else if (!objPtr) { + objPtr = Jim_NewStringObj(interp, NULL, 0); + } + + if (argc) { + if (Jim_SetVariable(interp, argv[0], objPtr) != JIM_OK) { + Jim_FreeNewObj(interp, objPtr); + return JIM_ERR; + } + + len = Jim_Length(objPtr); + + if (!nl && len == 0) { + + len = -1; + } + Jim_SetResultInt(interp, len); + } + else { + Jim_SetResult(interp, objPtr); + } + return JIM_OK; +} + +static int aio_cmd_puts(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + int wlen; + const char *wdata; + Jim_Obj *strObj; + int wnow = 0; + int nl = 1; + + if (argc == 2) { + if (!Jim_CompareStringImmediate(interp, argv[0], "-nonewline")) { + return -1; + } + strObj = argv[1]; + nl = 0; + } + else { + strObj = argv[0]; + } + +#ifdef JIM_MAINTAINER + if (Jim_IsShared(af->writebuf)) { + Jim_DecrRefCount(interp, af->writebuf); + af->writebuf = Jim_DuplicateObj(interp, af->writebuf); + Jim_IncrRefCount(af->writebuf); + } +#endif + Jim_AppendObj(interp, af->writebuf, strObj); + if (nl) { + Jim_AppendString(interp, af->writebuf, "\n", 1); + } + + + wdata = Jim_GetString(af->writebuf, &wlen); + switch (af->wbuft) { + case WBUF_OPT_NONE: + + wnow = 1; + break; + + case WBUF_OPT_LINE: + + if (nl || memchr(wdata, '\n', wlen) != NULL) { + wnow = 1; + } + break; + + case WBUF_OPT_FULL: + if (wlen >= af->wbuf_limit) { + wnow = 1; + } + break; + } + + if (wnow) { + return aio_flush(interp, af); + } + return JIM_OK; +} + +static int aio_cmd_isatty(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ +#ifdef HAVE_ISATTY + AioFile *af = Jim_CmdPrivData(interp); + Jim_SetResultInt(interp, isatty(af->fd)); +#else + Jim_SetResultInt(interp, 0); +#endif + + return JIM_OK; +} + + +static int aio_cmd_flush(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + return aio_flush(interp, af); +} + +static int aio_cmd_eof(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + Jim_SetResultInt(interp, !!aio_eof(af)); + return JIM_OK; +} + +static int aio_cmd_close(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + if (argc == 3) { + int option = -1; +#if defined(HAVE_SOCKETS) + static const char * const options[] = { "r", "w", "-nodelete", NULL }; + enum { OPT_R, OPT_W, OPT_NODELETE }; + + if (Jim_GetEnum(interp, argv[2], options, &option, NULL, JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } +#endif + switch (option) { +#if defined(HAVE_SHUTDOWN) + case OPT_R: + case OPT_W: + if (shutdown(af->fd, option == OPT_R ? SHUT_RD : SHUT_WR) == 0) { + return JIM_OK; + } + JimAioSetError(interp, NULL); + return JIM_ERR; +#endif +#if UNIX_SOCKETS + case OPT_NODELETE: + if (af->addr_family == PF_UNIX) { + af->flags |= AIO_NODELETE; + break; + } + +#endif + default: + Jim_SetResultString(interp, "not supported", -1); + return JIM_ERR; + } + } + + + af->flags &= ~AIO_KEEPOPEN; + + return Jim_DeleteCommand(interp, argv[0]); +} + +static int aio_cmd_seek(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + int orig = SEEK_SET; + jim_wide offset; + + if (argc == 2) { + if (Jim_CompareStringImmediate(interp, argv[1], "start")) + orig = SEEK_SET; + else if (Jim_CompareStringImmediate(interp, argv[1], "current")) + orig = SEEK_CUR; + else if (Jim_CompareStringImmediate(interp, argv[1], "end")) + orig = SEEK_END; + else { + return -1; + } + } + if (Jim_GetWide(interp, argv[0], &offset) != JIM_OK) { + return JIM_ERR; + } + if (orig != SEEK_CUR || offset != 0) { + + aio_flush(interp, af); + } + if (Jim_Lseek(af->fd, offset, orig) == -1) { + JimAioSetError(interp, af->filename); + return JIM_ERR; + } + if (af->readbuf) { + Jim_FreeNewObj(interp, af->readbuf); + af->readbuf = NULL; + } + af->flags &= ~AIO_EOF; + return JIM_OK; +} + +static int aio_cmd_tell(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + Jim_SetResultInt(interp, Jim_Lseek(af->fd, 0, SEEK_CUR)); + return JIM_OK; +} + +static int aio_cmd_filename(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + Jim_SetResult(interp, af->filename); + return JIM_OK; +} + +#ifdef O_NDELAY +static int aio_cmd_ndelay(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + if (argc) { + long nb; + + if (Jim_GetLong(interp, argv[0], &nb) != JIM_OK) { + return JIM_ERR; + } + aio_set_nonblocking(af, nb); + } + Jim_SetResultInt(interp, (af->flags & AIO_NONBLOCK) ? 1 : 0); + return JIM_OK; +} +#endif + + +#ifdef HAVE_FSYNC +static int aio_cmd_sync(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + if (aio_flush(interp, af) != JIM_OK) { + return JIM_ERR; + } + fsync(af->fd); + return JIM_OK; +} +#endif + +static int aio_cmd_buffering(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + Jim_Obj *resultObj; + + static const char * const options[] = { + "none", + "line", + "full", + NULL + }; + + if (argc) { + if (Jim_GetEnum(interp, argv[0], options, &af->wbuft, NULL, JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + + if (af->wbuft == WBUF_OPT_FULL && argc == 2) { + long l; + if (Jim_GetLong(interp, argv[1], &l) != JIM_OK || l <= 0) { + return JIM_ERR; + } + af->wbuf_limit = l; + } + + if (af->wbuft == WBUF_OPT_NONE) { + if (aio_flush(interp, af) != JIM_OK) { + return JIM_ERR; + } + } + + } + + resultObj = Jim_NewListObj(interp, NULL, 0); + Jim_ListAppendElement(interp, resultObj, Jim_NewStringObj(interp, options[af->wbuft], -1)); + if (af->wbuft == WBUF_OPT_FULL) { + Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, af->wbuf_limit)); + } + Jim_SetResult(interp, resultObj); + + return JIM_OK; +} + +static int aio_cmd_translation(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + enum {OPT_BINARY, OPT_TEXT}; + static const char * const options[] = { + "binary", + "text", + NULL + }; + int opt; + + if (Jim_GetEnum(interp, argv[0], options, &opt, NULL, JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } +#if defined(Jim_SetMode) + else { + AioFile *af = Jim_CmdPrivData(interp); + Jim_SetMode(af->fd, opt == OPT_BINARY ? O_BINARY : O_TEXT); + } +#endif + return JIM_OK; +} + +static int aio_cmd_readsize(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + if (argc) { + long l; + if (Jim_GetLong(interp, argv[0], &l) != JIM_OK || l <= 0) { + return JIM_ERR; + } + af->rbuf_len = l; + if (af->rbuf) { + af->rbuf = Jim_Realloc(af->rbuf, af->rbuf_len); + } + } + Jim_SetResultInt(interp, af->rbuf_len); + + return JIM_OK; +} + +#ifdef jim_ext_eventloop +static int aio_cmd_timeout(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ +#ifdef HAVE_SELECT + AioFile *af = Jim_CmdPrivData(interp); + if (argc == 1) { + if (Jim_GetLong(interp, argv[0], &af->timeout) != JIM_OK) { + return JIM_ERR; + } + } + Jim_SetResultInt(interp, af->timeout); + return JIM_OK; +#else + Jim_SetResultString(interp, "timeout not supported", -1); + return JIM_ERR; +#endif +} + +static int aio_eventinfo(Jim_Interp *interp, AioFile * af, unsigned mask, + int argc, Jim_Obj * const *argv) +{ + if (argc == 0) { + + Jim_Obj *objPtr = Jim_FindFileHandler(interp, af->fd, mask); + if (objPtr) { + Jim_SetResult(interp, objPtr); + } + return JIM_OK; + } + + + Jim_DeleteFileHandler(interp, af->fd, mask); + + + if (Jim_Length(argv[0])) { + Jim_CreateScriptFileHandler(interp, af->fd, mask, argv[0]); + } + + return JIM_OK; +} + +static int aio_cmd_readable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + return aio_eventinfo(interp, af, JIM_EVENT_READABLE, argc, argv); +} + +static int aio_cmd_writable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + return aio_eventinfo(interp, af, JIM_EVENT_WRITABLE, argc, argv); +} + +static int aio_cmd_onexception(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + AioFile *af = Jim_CmdPrivData(interp); + + return aio_eventinfo(interp, af, JIM_EVENT_EXCEPTION, argc, argv); +} +#endif + +#if defined(jim_ext_file) && defined(Jim_FileStat) +static int aio_cmd_stat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + AioFile *af = Jim_CmdPrivData(interp); + + if (Jim_FileStat(af->fd, &sb) == -1) { + JimAioSetError(interp, NULL); + return JIM_ERR; + } + return Jim_FileStoreStatData(interp, argc == 0 ? NULL : argv[0], &sb); +} +#endif + + + + +static const jim_subcmd_type aio_command_table[] = { + { "read", + "?-nonewline|len?", + aio_cmd_read, + 0, + 2, + + }, + { "copyto", + "handle ?size?", + aio_cmd_copy, + 1, + 2, + + }, + { "getfd", + NULL, + aio_cmd_getfd, + 0, + 0, + + }, + { "gets", + "?var?", + aio_cmd_gets, + 0, + 1, + + }, + { "puts", + "?-nonewline? str", + aio_cmd_puts, + 1, + 2, + + }, + { "isatty", + NULL, + aio_cmd_isatty, + 0, + 0, + + }, + { "flush", + NULL, + aio_cmd_flush, + 0, + 0, + + }, + { "eof", + NULL, + aio_cmd_eof, + 0, + 0, + + }, + { "close", + "?r(ead)|w(rite)?", + aio_cmd_close, + 0, + 1, + JIM_MODFLAG_FULLARGV, + + }, + { "seek", + "offset ?start|current|end", + aio_cmd_seek, + 1, + 2, + + }, + { "tell", + NULL, + aio_cmd_tell, + 0, + 0, + + }, + { "filename", + NULL, + aio_cmd_filename, + 0, + 0, + + }, +#ifdef O_NDELAY + { "ndelay", + "?0|1?", + aio_cmd_ndelay, + 0, + 1, + + }, +#endif +#ifdef HAVE_FSYNC + { "sync", + NULL, + aio_cmd_sync, + 0, + 0, + + }, +#endif + { "buffering", + "?none|line|full? ?size?", + aio_cmd_buffering, + 0, + 2, + + }, + { "translation", + "binary|text", + aio_cmd_translation, + 1, + 1, + + }, + { "readsize", + "?size?", + aio_cmd_readsize, + 0, + 1, + + }, +#if defined(jim_ext_file) && defined(Jim_FileStat) + { "stat", + "?var?", + aio_cmd_stat, + 0, + 1, + + }, +#endif +#ifdef jim_ext_eventloop + { "readable", + "?readable-script?", + aio_cmd_readable, + 0, + 1, + + }, + { "writable", + "?writable-script?", + aio_cmd_writable, + 0, + 1, + + }, + { "onexception", + "?exception-script?", + aio_cmd_onexception, + 0, + 1, + + }, + { "timeout", + "?ms?", + aio_cmd_timeout, + 0, + 1, + + }, +#endif + { NULL } +}; + +static int JimAioSubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return Jim_CallSubCmd(interp, Jim_ParseSubCmd(interp, aio_command_table, argc, argv), argc, argv); +} + +static int parse_posix_open_mode(Jim_Interp *interp, Jim_Obj *modeObj) +{ + int i; + int flags = 0; + #ifndef O_NOCTTY + + #define O_NOCTTY 0 + #endif + static const char * const modetypes[] = { + "RDONLY", "WRONLY", "RDWR", "APPEND", "BINARY", "CREAT", "EXCL", "NOCTTY", "TRUNC", NULL + }; + static const int modeflags[] = { + O_RDONLY, O_WRONLY, O_RDWR, O_APPEND, 0, O_CREAT, O_EXCL, O_NOCTTY, O_TRUNC, + }; + + for (i = 0; i < Jim_ListLength(interp, modeObj); i++) { + int opt; + Jim_Obj *objPtr = Jim_ListGetIndex(interp, modeObj, i); + if (Jim_GetEnum(interp, objPtr, modetypes, &opt, "access mode", JIM_ERRMSG) != JIM_OK) { + return -1; + } + flags |= modeflags[opt]; + } + return flags; +} + +static int parse_open_mode(Jim_Interp *interp, Jim_Obj *filenameObj, Jim_Obj *modeObj) +{ + + int flags; + const char *mode = Jim_String(modeObj); + if (*mode == 'R' || *mode == 'W') { + return parse_posix_open_mode(interp, modeObj); + } + if (*mode == 'r') { + flags = O_RDONLY; + } + else if (*mode == 'w') { + flags = O_WRONLY | O_CREAT | O_TRUNC; + } + else if (*mode == 'a') { + flags = O_WRONLY | O_CREAT | O_APPEND; + } + else { + Jim_SetResultFormatted(interp, "%s: invalid open mode '%s'", Jim_String(filenameObj), mode); + return -1; + } + mode++; + + if (*mode == 'b') { +#ifdef O_BINARY + flags |= O_BINARY; +#endif + mode++; + } + + if (*mode == 't') { +#ifdef O_TEXT + flags |= O_TEXT; +#endif + mode++; + } + + if (*mode == '+') { + mode++; + + flags &= ~(O_RDONLY | O_WRONLY); + flags |= O_RDWR; + } + + if (*mode == 'x') { + mode++; +#ifdef O_EXCL + flags |= O_EXCL; +#endif + } + + if (*mode == 'F') { + mode++; +#ifdef O_LARGEFILE + flags |= O_LARGEFILE; +#endif + } + + if (*mode == 'e') { + + mode++; + } + return flags; +} + +static int JimAioOpenCommand(Jim_Interp *interp, int argc, + Jim_Obj *const *argv) +{ + int openflags; + const char *filename; + int fd = -1; + int n = 0; + int flags = 0; + + if (argc > 2 && Jim_CompareStringImmediate(interp, argv[2], "-noclose")) { + flags = AIO_KEEPOPEN; + n++; + } + if (argc < 2 || argc > 3 + n) { + Jim_WrongNumArgs(interp, 1, argv, "filename ?-noclose? ?mode?"); + return JIM_ERR; + } + + filename = Jim_String(argv[1]); + +#ifdef jim_ext_tclcompat + { + + + if (*filename == '|') { + Jim_Obj *evalObj[3]; + int i = 0; + + evalObj[i++] = Jim_NewStringObj(interp, "::popen", -1); + evalObj[i++] = Jim_NewStringObj(interp, filename + 1, -1); + if (argc == 3 + n) { + evalObj[i++] = argv[2 + n]; + } + + return Jim_EvalObjVector(interp, i, evalObj); + } + } +#endif + if (argc == 3 + n) { + openflags = parse_open_mode(interp, argv[1], argv[2 + n]); + if (openflags == -1) { + return JIM_ERR; + } + } + else { + openflags = O_RDONLY; + } + fd = open(filename, openflags, 0666); + if (fd < 0) { + JimAioSetError(interp, argv[1]); + return JIM_ERR; + } + + return JimMakeChannel(interp, fd, argv[1], "aio.handle%ld", 0, flags) ? JIM_OK : JIM_ERR; +} + + +static AioFile *JimMakeChannel(Jim_Interp *interp, int fd, Jim_Obj *filename, + const char *hdlfmt, int family, int flags) +{ + AioFile *af; + char buf[AIO_CMD_LEN]; + Jim_Obj *cmdname; + + snprintf(buf, sizeof(buf), hdlfmt, Jim_GetId(interp)); + cmdname = Jim_NewStringObj(interp, buf, -1); + if (!filename) { + filename = cmdname; + } + Jim_IncrRefCount(filename); + + + af = Jim_Alloc(sizeof(*af)); + memset(af, 0, sizeof(*af)); + af->filename = filename; + af->fd = fd; + af->addr_family = family; + af->fops = &stdio_fops; + af->ssl = NULL; + if (flags & AIO_WBUF_NONE) { + af->wbuft = WBUF_OPT_NONE; + } + else { +#ifdef HAVE_ISATTY + af->wbuft = isatty(af->fd) ? WBUF_OPT_LINE : WBUF_OPT_FULL; +#else + af->wbuft = WBUF_OPT_FULL; +#endif + } + +#ifdef FD_CLOEXEC + if ((flags & AIO_KEEPOPEN) == 0) { + (void)fcntl(af->fd, F_SETFD, FD_CLOEXEC); + } +#endif + aio_set_nonblocking(af, !!(flags & AIO_NONBLOCK)); + + af->flags |= flags; + + af->writebuf = Jim_NewStringObj(interp, NULL, 0); + Jim_IncrRefCount(af->writebuf); + af->wbuf_limit = AIO_DEFAULT_WBUF_LIMIT; + af->rbuf_len = AIO_DEFAULT_RBUF_LEN; + + + Jim_CreateCommand(interp, buf, JimAioSubCmdProc, af, JimAioDelProc); + + Jim_SetResult(interp, Jim_MakeGlobalNamespaceName(interp, cmdname)); + + return af; +} + +#if defined(HAVE_PIPE) || (defined(HAVE_SOCKETPAIR) && UNIX_SOCKETS) || defined(HAVE_OPENPTY) +static int JimMakeChannelPair(Jim_Interp *interp, int p[2], Jim_Obj *filename, + const char *hdlfmt, int family, int flags) +{ + if (JimMakeChannel(interp, p[0], filename, hdlfmt, family, flags)) { + Jim_Obj *objPtr = Jim_NewListObj(interp, NULL, 0); + Jim_ListAppendElement(interp, objPtr, Jim_GetResult(interp)); + if (JimMakeChannel(interp, p[1], filename, hdlfmt, family, flags)) { + Jim_ListAppendElement(interp, objPtr, Jim_GetResult(interp)); + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + } + + + close(p[0]); + close(p[1]); + JimAioSetError(interp, NULL); + return JIM_ERR; +} +#endif + +#ifdef HAVE_PIPE +static int JimCreatePipe(Jim_Interp *interp, Jim_Obj *filenameObj, int flags) +{ + int p[2]; + + if (pipe(p) != 0) { + JimAioSetError(interp, NULL); + return JIM_ERR; + } + + return JimMakeChannelPair(interp, p, filenameObj, "aio.pipe%ld", 0, flags); +} + + +static int JimAioPipeCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 1) { + Jim_WrongNumArgs(interp, 1, argv, ""); + return JIM_ERR; + } + return JimCreatePipe(interp, argv[0], 0); +} +#endif + +#ifdef HAVE_OPENPTY +static int JimAioOpenPtyCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int p[2]; + char path[MAXPATHLEN]; + + if (argc != 1) { + Jim_WrongNumArgs(interp, 1, argv, ""); + return JIM_ERR; + } + + if (openpty(&p[0], &p[1], path, NULL, NULL) != 0) { + JimAioSetError(interp, NULL); + return JIM_ERR; + } + + + return JimMakeChannelPair(interp, p, Jim_NewStringObj(interp, path, -1), "aio.pty%ld", 0, 0); + return JimMakeChannelPair(interp, p, Jim_NewStringObj(interp, path, -1), "aio.pty%ld", 0, 0); +} +#endif + + + +int Jim_aioInit(Jim_Interp *interp) +{ + if (Jim_PackageProvide(interp, "aio", "1.0", JIM_ERRMSG)) + return JIM_ERR; + +#if defined(JIM_SSL) + Jim_CreateCommand(interp, "load_ssl_certs", JimAioLoadSSLCertsCommand, NULL, NULL); +#endif + + Jim_CreateCommand(interp, "open", JimAioOpenCommand, NULL, NULL); +#ifdef HAVE_SOCKETS + Jim_CreateCommand(interp, "socket", JimAioSockCommand, NULL, NULL); +#endif +#ifdef HAVE_PIPE + Jim_CreateCommand(interp, "pipe", JimAioPipeCommand, NULL, NULL); +#endif + + + JimMakeChannel(interp, fileno(stdin), NULL, "stdin", 0, AIO_KEEPOPEN); + JimMakeChannel(interp, fileno(stdout), NULL, "stdout", 0, AIO_KEEPOPEN); + JimMakeChannel(interp, fileno(stderr), NULL, "stderr", 0, AIO_KEEPOPEN | AIO_WBUF_NONE); + + return JIM_OK; +} + +#include +#include +#include + + +#ifdef HAVE_DIRENT_H +#include +#endif + +int Jim_ReaddirCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *dirPath; + DIR *dirPtr; + struct dirent *entryPtr; + int nocomplain = 0; + + if (argc == 3 && Jim_CompareStringImmediate(interp, argv[1], "-nocomplain")) { + nocomplain = 1; + } + if (argc != 2 && !nocomplain) { + Jim_WrongNumArgs(interp, 1, argv, "?-nocomplain? dirPath"); + return JIM_ERR; + } + + dirPath = Jim_String(argv[1 + nocomplain]); + + dirPtr = opendir(dirPath); + if (dirPtr == NULL) { + if (nocomplain) { + return JIM_OK; + } + Jim_SetResultString(interp, strerror(errno), -1); + return JIM_ERR; + } + else { + Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); + + while ((entryPtr = readdir(dirPtr)) != NULL) { + if (entryPtr->d_name[0] == '.') { + if (entryPtr->d_name[1] == '\0') { + continue; + } + if ((entryPtr->d_name[1] == '.') && (entryPtr->d_name[2] == '\0')) + continue; + } + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, entryPtr->d_name, -1)); + } + closedir(dirPtr); + + Jim_SetResult(interp, listObj); + + return JIM_OK; + } +} + +int Jim_readdirInit(Jim_Interp *interp) +{ + Jim_PackageProvideCheck(interp, "readdir"); + Jim_CreateCommand(interp, "readdir", Jim_ReaddirCmd, NULL, NULL); + return JIM_OK; +} + +#include +#include + +#if defined(JIM_REGEXP) +#else + #include + #define jim_regcomp regcomp + #define jim_regexec regexec + #define jim_regerror regerror + #define jim_regfree regfree +#endif + +static void FreeRegexpInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + jim_regfree(objPtr->internalRep.ptrIntValue.ptr); + Jim_Free(objPtr->internalRep.ptrIntValue.ptr); +} + +static const Jim_ObjType regexpObjType = { + "regexp", + FreeRegexpInternalRep, + NULL, + NULL, + JIM_TYPE_NONE +}; + +static regex_t *SetRegexpFromAny(Jim_Interp *interp, Jim_Obj *objPtr, unsigned flags) +{ + regex_t *compre; + const char *pattern; + int ret; + + + if (objPtr->typePtr == ®expObjType && + objPtr->internalRep.ptrIntValue.ptr && objPtr->internalRep.ptrIntValue.int1 == flags) { + + return objPtr->internalRep.ptrIntValue.ptr; + } + + + + + pattern = Jim_String(objPtr); + compre = Jim_Alloc(sizeof(regex_t)); + + if ((ret = jim_regcomp(compre, pattern, REG_EXTENDED | flags)) != 0) { + char buf[100]; + + jim_regerror(ret, compre, buf, sizeof(buf)); + Jim_SetResultFormatted(interp, "couldn't compile regular expression pattern: %s", buf); + jim_regfree(compre); + Jim_Free(compre); + return NULL; + } + + Jim_FreeIntRep(interp, objPtr); + + objPtr->typePtr = ®expObjType; + objPtr->internalRep.ptrIntValue.int1 = flags; + objPtr->internalRep.ptrIntValue.ptr = compre; + + return compre; +} + +int Jim_RegexpCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int opt_indices = 0; + int opt_all = 0; + int opt_inline = 0; + regex_t *regex; + int match, i, j; + int offset = 0; + regmatch_t *pmatch = NULL; + int source_len; + int result = JIM_OK; + const char *pattern; + const char *source_str; + int num_matches = 0; + int num_vars; + Jim_Obj *resultListObj = NULL; + int regcomp_flags = 0; + int eflags = 0; + int option; + enum { + OPT_INDICES, OPT_NOCASE, OPT_LINE, OPT_ALL, OPT_INLINE, OPT_START, OPT_END + }; + static const char * const options[] = { + "-indices", "-nocase", "-line", "-all", "-inline", "-start", "--", NULL + }; + + if (argc < 3) { + wrongNumArgs: + Jim_WrongNumArgs(interp, 1, argv, + "?-switch ...? exp string ?matchVar? ?subMatchVar ...?"); + return JIM_ERR; + } + + for (i = 1; i < argc; i++) { + const char *opt = Jim_String(argv[i]); + + if (*opt != '-') { + break; + } + if (Jim_GetEnum(interp, argv[i], options, &option, "switch", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { + return JIM_ERR; + } + if (option == OPT_END) { + i++; + break; + } + switch (option) { + case OPT_INDICES: + opt_indices = 1; + break; + + case OPT_NOCASE: + regcomp_flags |= REG_ICASE; + break; + + case OPT_LINE: + regcomp_flags |= REG_NEWLINE; + break; + + case OPT_ALL: + opt_all = 1; + break; + + case OPT_INLINE: + opt_inline = 1; + break; + + case OPT_START: + if (++i == argc) { + goto wrongNumArgs; + } + if (Jim_GetIndex(interp, argv[i], &offset) != JIM_OK) { + return JIM_ERR; + } + break; + } + } + if (argc - i < 2) { + goto wrongNumArgs; + } + + regex = SetRegexpFromAny(interp, argv[i], regcomp_flags); + if (!regex) { + return JIM_ERR; + } + + pattern = Jim_String(argv[i]); + source_str = Jim_GetString(argv[i + 1], &source_len); + + num_vars = argc - i - 2; + + if (opt_inline) { + if (num_vars) { + Jim_SetResultString(interp, "regexp match variables not allowed when using -inline", + -1); + result = JIM_ERR; + goto done; + } + num_vars = regex->re_nsub + 1; + } + + pmatch = Jim_Alloc((num_vars + 1) * sizeof(*pmatch)); + + if (offset) { + if (offset < 0) { + offset += source_len + 1; + } + if (offset > source_len) { + source_str += source_len; + } + else if (offset > 0) { + source_str += utf8_index(source_str, offset); + } + eflags |= REG_NOTBOL; + } + + if (opt_inline) { + resultListObj = Jim_NewListObj(interp, NULL, 0); + } + + next_match: + match = jim_regexec(regex, source_str, num_vars + 1, pmatch, eflags); + if (match >= REG_BADPAT) { + char buf[100]; + + jim_regerror(match, regex, buf, sizeof(buf)); + Jim_SetResultFormatted(interp, "error while matching pattern: %s", buf); + result = JIM_ERR; + goto done; + } + + if (match == REG_NOMATCH) { + goto done; + } + + num_matches++; + + if (opt_all && !opt_inline) { + + goto try_next_match; + } + + + j = 0; + for (i += 2; opt_inline ? j < num_vars : i < argc; i++, j++) { + Jim_Obj *resultObj; + + if (opt_indices) { + resultObj = Jim_NewListObj(interp, NULL, 0); + } + else { + resultObj = Jim_NewStringObj(interp, "", 0); + } + + if (pmatch[j].rm_so == -1) { + if (opt_indices) { + Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, -1)); + Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, -1)); + } + } + else { + if (opt_indices) { + + int so = utf8_strlen(source_str, pmatch[j].rm_so); + int eo = utf8_strlen(source_str, pmatch[j].rm_eo); + Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, offset + so)); + Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, offset + eo - 1)); + } + else { + Jim_AppendString(interp, resultObj, source_str + pmatch[j].rm_so, pmatch[j].rm_eo - pmatch[j].rm_so); + } + } + + if (opt_inline) { + Jim_ListAppendElement(interp, resultListObj, resultObj); + } + else { + + result = Jim_SetVariable(interp, argv[i], resultObj); + + if (result != JIM_OK) { + Jim_FreeObj(interp, resultObj); + break; + } + } + } + + try_next_match: + if (opt_all && (pattern[0] != '^' || (regcomp_flags & REG_NEWLINE)) && *source_str) { + if (pmatch[0].rm_eo) { + offset += utf8_strlen(source_str, pmatch[0].rm_eo); + source_str += pmatch[0].rm_eo; + } + else { + source_str++; + offset++; + } + if (*source_str) { + eflags = REG_NOTBOL; + goto next_match; + } + } + + done: + if (result == JIM_OK) { + if (opt_inline) { + Jim_SetResult(interp, resultListObj); + } + else { + Jim_SetResultInt(interp, num_matches); + } + } + + Jim_Free(pmatch); + return result; +} + +#define MAX_SUB_MATCHES 50 + +int Jim_RegsubCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int regcomp_flags = 0; + int regexec_flags = 0; + int opt_all = 0; + int opt_command = 0; + int offset = 0; + regex_t *regex; + const char *p; + int result = JIM_OK; + regmatch_t pmatch[MAX_SUB_MATCHES + 1]; + int num_matches = 0; + + int i, j, n; + Jim_Obj *varname; + Jim_Obj *resultObj; + Jim_Obj *cmd_prefix = NULL; + Jim_Obj *regcomp_obj = NULL; + const char *source_str; + int source_len; + const char *replace_str = NULL; + int replace_len; + const char *pattern; + int option; + enum { + OPT_NOCASE, OPT_LINE, OPT_ALL, OPT_START, OPT_COMMAND, OPT_END + }; + static const char * const options[] = { + "-nocase", "-line", "-all", "-start", "-command", "--", NULL + }; + + if (argc < 4) { + wrongNumArgs: + Jim_WrongNumArgs(interp, 1, argv, + "?-switch ...? exp string subSpec ?varName?"); + return JIM_ERR; + } + + for (i = 1; i < argc; i++) { + const char *opt = Jim_String(argv[i]); + + if (*opt != '-') { + break; + } + if (Jim_GetEnum(interp, argv[i], options, &option, "switch", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { + return JIM_ERR; + } + if (option == OPT_END) { + i++; + break; + } + switch (option) { + case OPT_NOCASE: + regcomp_flags |= REG_ICASE; + break; + + case OPT_LINE: + regcomp_flags |= REG_NEWLINE; + break; + + case OPT_ALL: + opt_all = 1; + break; + + case OPT_START: + if (++i == argc) { + goto wrongNumArgs; + } + if (Jim_GetIndex(interp, argv[i], &offset) != JIM_OK) { + return JIM_ERR; + } + break; + + case OPT_COMMAND: + opt_command = 1; + break; + } + } + if (argc - i != 3 && argc - i != 4) { + goto wrongNumArgs; + } + + + regcomp_obj = Jim_DuplicateObj(interp, argv[i]); + Jim_IncrRefCount(regcomp_obj); + regex = SetRegexpFromAny(interp, regcomp_obj, regcomp_flags); + if (!regex) { + Jim_DecrRefCount(interp, regcomp_obj); + return JIM_ERR; + } + pattern = Jim_String(argv[i]); + + source_str = Jim_GetString(argv[i + 1], &source_len); + if (opt_command) { + cmd_prefix = argv[i + 2]; + if (Jim_ListLength(interp, cmd_prefix) == 0) { + Jim_SetResultString(interp, "command prefix must be a list of at least one element", -1); + Jim_DecrRefCount(interp, regcomp_obj); + return JIM_ERR; + } + Jim_IncrRefCount(cmd_prefix); + } + else { + replace_str = Jim_GetString(argv[i + 2], &replace_len); + } + varname = argv[i + 3]; + + + resultObj = Jim_NewStringObj(interp, "", 0); + + if (offset) { + if (offset < 0) { + offset += source_len + 1; + } + if (offset > source_len) { + offset = source_len; + } + else if (offset < 0) { + offset = 0; + } + } + + offset = utf8_index(source_str, offset); + + + Jim_AppendString(interp, resultObj, source_str, offset); + + + n = source_len - offset; + p = source_str + offset; + do { + int match = jim_regexec(regex, p, MAX_SUB_MATCHES, pmatch, regexec_flags); + + if (match >= REG_BADPAT) { + char buf[100]; + + jim_regerror(match, regex, buf, sizeof(buf)); + Jim_SetResultFormatted(interp, "error while matching pattern: %s", buf); + return JIM_ERR; + } + if (match == REG_NOMATCH) { + break; + } + + num_matches++; + + Jim_AppendString(interp, resultObj, p, pmatch[0].rm_so); + + if (opt_command) { + + Jim_Obj *cmdListObj = Jim_DuplicateObj(interp, cmd_prefix); + for (j = 0; j < MAX_SUB_MATCHES; j++) { + if (pmatch[j].rm_so == -1) { + break; + } + else { + Jim_Obj *srcObj = Jim_NewStringObj(interp, p + pmatch[j].rm_so, pmatch[j].rm_eo - pmatch[j].rm_so); + Jim_ListAppendElement(interp, cmdListObj, srcObj); + } + } + Jim_IncrRefCount(cmdListObj); + + result = Jim_EvalObj(interp, cmdListObj); + Jim_DecrRefCount(interp, cmdListObj); + if (result != JIM_OK) { + goto cmd_error; + } + Jim_AppendString(interp, resultObj, Jim_String(Jim_GetResult(interp)), -1); + } + else { + + for (j = 0; j < replace_len; j++) { + int idx; + int c = replace_str[j]; + + if (c == '&') { + idx = 0; + } + else if (c == '\\' && j < replace_len) { + c = replace_str[++j]; + if ((c >= '0') && (c <= '9')) { + idx = c - '0'; + } + else if ((c == '\\') || (c == '&')) { + Jim_AppendString(interp, resultObj, replace_str + j, 1); + continue; + } + else { + Jim_AppendString(interp, resultObj, replace_str + j - 1, (j == replace_len) ? 1 : 2); + continue; + } + } + else { + Jim_AppendString(interp, resultObj, replace_str + j, 1); + continue; + } + if ((idx < MAX_SUB_MATCHES) && pmatch[idx].rm_so != -1 && pmatch[idx].rm_eo != -1) { + Jim_AppendString(interp, resultObj, p + pmatch[idx].rm_so, + pmatch[idx].rm_eo - pmatch[idx].rm_so); + } + } + } + + p += pmatch[0].rm_eo; + n -= pmatch[0].rm_eo; + + + if (!opt_all || n == 0) { + break; + } + + + if ((regcomp_flags & REG_NEWLINE) == 0 && pattern[0] == '^') { + break; + } + + + if (pattern[0] == '\0' && n) { + + Jim_AppendString(interp, resultObj, p, 1); + p++; + n--; + } + + if (pmatch[0].rm_eo == pmatch[0].rm_so) { + + regexec_flags = REG_NOTBOL; + } + else { + regexec_flags = 0; + } + + } while (n); + + Jim_AppendString(interp, resultObj, p, -1); + +cmd_error: + if (result == JIM_OK) { + + if (argc - i == 4) { + result = Jim_SetVariable(interp, varname, resultObj); + + if (result == JIM_OK) { + Jim_SetResultInt(interp, num_matches); + } + else { + Jim_FreeObj(interp, resultObj); + } + } + else { + Jim_SetResult(interp, resultObj); + result = JIM_OK; + } + } + else { + Jim_FreeObj(interp, resultObj); + } + + if (opt_command) { + Jim_DecrRefCount(interp, cmd_prefix); + } + + Jim_DecrRefCount(interp, regcomp_obj); + + return result; +} + +int Jim_regexpInit(Jim_Interp *interp) +{ + Jim_PackageProvideCheck(interp, "regexp"); + Jim_CreateCommand(interp, "regexp", Jim_RegexpCmd, NULL, NULL); + Jim_CreateCommand(interp, "regsub", Jim_RegsubCmd, NULL, NULL); + return JIM_OK; +} + +#include +#include +#include +#include +#include + + +#ifdef HAVE_UTIMES +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#elif defined(_MSC_VER) +#include +#define F_OK 0 +#define W_OK 2 +#define R_OK 4 +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + +#if defined(__MINGW32__) || defined(__MSYS__) || defined(_MSC_VER) +#define ISWINDOWS 1 + +#undef HAVE_SYMLINK +#else +#define ISWINDOWS 0 +#endif + + +#if defined(HAVE_STRUCT_STAT_ST_MTIMESPEC) + #define STAT_MTIME_US(STAT) ((STAT).st_mtimespec.tv_sec * 1000000ll + (STAT).st_mtimespec.tv_nsec / 1000) +#elif defined(HAVE_STRUCT_STAT_ST_MTIM) + #define STAT_MTIME_US(STAT) ((STAT).st_mtim.tv_sec * 1000000ll + (STAT).st_mtim.tv_nsec / 1000) +#endif + + +static void JimFixPath(char *path) +{ + if (ISWINDOWS) { + + char *p = path; + while ((p = strchr(p, '\\')) != NULL) { + *p++ = '/'; + } + } +} + + +static const char *JimGetFileType(int mode) +{ + if (S_ISREG(mode)) { + return "file"; + } + else if (S_ISDIR(mode)) { + return "directory"; + } +#ifdef S_ISCHR + else if (S_ISCHR(mode)) { + return "characterSpecial"; + } +#endif +#ifdef S_ISBLK + else if (S_ISBLK(mode)) { + return "blockSpecial"; + } +#endif +#ifdef S_ISFIFO + else if (S_ISFIFO(mode)) { + return "fifo"; + } +#endif +#ifdef S_ISLNK + else if (S_ISLNK(mode)) { + return "link"; + } +#endif +#ifdef S_ISSOCK + else if (S_ISSOCK(mode)) { + return "socket"; + } +#endif + return "unknown"; +} + +static void AppendStatElement(Jim_Interp *interp, Jim_Obj *listObj, const char *key, jim_wide value) +{ + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, key, -1)); + Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, value)); +} + +int Jim_FileStoreStatData(Jim_Interp *interp, Jim_Obj *varName, const jim_stat_t *sb) +{ + + Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); + + AppendStatElement(interp, listObj, "dev", sb->st_dev); + AppendStatElement(interp, listObj, "ino", sb->st_ino); + AppendStatElement(interp, listObj, "mode", sb->st_mode); + AppendStatElement(interp, listObj, "nlink", sb->st_nlink); + AppendStatElement(interp, listObj, "uid", sb->st_uid); + AppendStatElement(interp, listObj, "gid", sb->st_gid); + AppendStatElement(interp, listObj, "size", sb->st_size); + AppendStatElement(interp, listObj, "atime", sb->st_atime); + AppendStatElement(interp, listObj, "mtime", sb->st_mtime); + AppendStatElement(interp, listObj, "ctime", sb->st_ctime); +#ifdef STAT_MTIME_US + AppendStatElement(interp, listObj, "mtimeus", STAT_MTIME_US(*sb)); +#endif + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "type", -1)); + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, JimGetFileType((int)sb->st_mode), -1)); + + + if (varName) { + Jim_Obj *objPtr; + objPtr = Jim_GetVariable(interp, varName, JIM_NONE); + + if (objPtr) { + Jim_Obj *objv[2]; + + objv[0] = objPtr; + objv[1] = listObj; + + objPtr = Jim_DictMerge(interp, 2, objv); + if (objPtr == NULL) { + + Jim_SetResultFormatted(interp, "can't set \"%#s(dev)\": variable isn't array", varName); + Jim_FreeNewObj(interp, listObj); + return JIM_ERR; + } + + Jim_InvalidateStringRep(objPtr); + + Jim_FreeNewObj(interp, listObj); + listObj = objPtr; + } + Jim_SetVariable(interp, varName, listObj); + } + + + Jim_SetResult(interp, listObj); + + return JIM_OK; +} + +static int JimPathLenNoTrailingSlashes(const char *path, int len) +{ + int i; + for (i = len; i > 1 && path[i - 1] == '/'; i--) { + + if (ISWINDOWS && path[i - 2] == ':') { + + break; + } + } + return i; +} + +static Jim_Obj *JimStripTrailingSlashes(Jim_Interp *interp, Jim_Obj *objPtr) +{ + int len = Jim_Length(objPtr); + const char *path = Jim_String(objPtr); + int i = JimPathLenNoTrailingSlashes(path, len); + if (i != len) { + objPtr = Jim_NewStringObj(interp, path, i); + } + Jim_IncrRefCount(objPtr); + return objPtr; +} + +static int file_cmd_dirname(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr = JimStripTrailingSlashes(interp, argv[0]); + const char *path = Jim_String(objPtr); + const char *p = strrchr(path, '/'); + + if (!p) { + Jim_SetResultString(interp, ".", -1); + } + else if (p[1] == 0) { + + Jim_SetResult(interp, objPtr); + } + else if (p == path) { + Jim_SetResultString(interp, "/", -1); + } + else if (ISWINDOWS && p[-1] == ':') { + + Jim_SetResultString(interp, path, p - path + 1); + } + else { + + int len = JimPathLenNoTrailingSlashes(path, p - path); + Jim_SetResultString(interp, path, len); + } + Jim_DecrRefCount(interp, objPtr); + return JIM_OK; +} + +static int file_cmd_split(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); + const char *path = Jim_String(argv[0]); + + if (*path == '/') { + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "/", 1)); + } + + while (1) { + + while (*path == '/') { + path++; + } + if (*path) { + const char *pt = strchr(path, '/'); + if (pt) { + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, path, pt - path)); + path = pt; + continue; + } + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, path, -1)); + } + break; + } + Jim_SetResult(interp, listObj); + return JIM_OK; +} + +static int file_cmd_rootname(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *path = Jim_String(argv[0]); + const char *lastSlash = strrchr(path, '/'); + const char *p = strrchr(path, '.'); + + if (p == NULL || (lastSlash != NULL && lastSlash > p)) { + Jim_SetResult(interp, argv[0]); + } + else { + Jim_SetResultString(interp, path, p - path); + } + return JIM_OK; +} + +static int file_cmd_extension(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr = JimStripTrailingSlashes(interp, argv[0]); + const char *path = Jim_String(objPtr); + const char *lastSlash = strrchr(path, '/'); + const char *p = strrchr(path, '.'); + + if (p == NULL || (lastSlash != NULL && lastSlash >= p)) { + p = ""; + } + Jim_SetResultString(interp, p, -1); + Jim_DecrRefCount(interp, objPtr); + return JIM_OK; +} + +static int file_cmd_tail(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr = JimStripTrailingSlashes(interp, argv[0]); + const char *path = Jim_String(objPtr); + const char *lastSlash = strrchr(path, '/'); + + if (lastSlash) { + Jim_SetResultString(interp, lastSlash + 1, -1); + } + else { + Jim_SetResult(interp, objPtr); + } + Jim_DecrRefCount(interp, objPtr); + return JIM_OK; +} + +#ifndef HAVE_RESTRICT +#define restrict +#endif + +static char *JimRealPath(const char *restrict path, char *restrict resolved_path, size_t len) +{ +#if defined(HAVE__FULLPATH) + return _fullpath(resolved_path, path, len); +#elif defined(HAVE_REALPATH) + return realpath(path, resolved_path); +#else + return NULL; +#endif +} + +static int file_cmd_normalize(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *path = Jim_String(argv[0]); + char *newname = Jim_Alloc(MAXPATHLEN); + + if (JimRealPath(path, newname, MAXPATHLEN)) { + JimFixPath(newname); + Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, newname, -1)); + return JIM_OK; + } + Jim_Free(newname); + Jim_SetResultFormatted(interp, "can't normalize \"%#s\": %s", argv[0], strerror(errno)); + return JIM_ERR; +} + +static int file_cmd_join(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i; + char *newname = Jim_Alloc(MAXPATHLEN + 1); + char *last = newname; + + *newname = 0; + + + for (i = 0; i < argc; i++) { + int len; + const char *part = Jim_GetString(argv[i], &len); + + if (*part == '/') { + + last = newname; + } + else if (ISWINDOWS && strchr(part, ':')) { + + last = newname; + } + else if (part[0] == '.') { + if (part[1] == '/') { + part += 2; + len -= 2; + } + else if (part[1] == 0 && last != newname) { + + continue; + } + } + + + if (last != newname && last[-1] != '/') { + *last++ = '/'; + } + + if (len) { + if (last + len - newname >= MAXPATHLEN) { + Jim_Free(newname); + Jim_SetResultString(interp, "Path too long", -1); + return JIM_ERR; + } + memcpy(last, part, len); + last += len; + } + + + if (last > newname + 1 && last[-1] == '/') { + + if (!ISWINDOWS || !(last > newname + 2 && last[-2] == ':')) { + *--last = 0; + } + } + } + + *last = 0; + + + + Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, newname, last - newname)); + + return JIM_OK; +} + +static int file_access(Jim_Interp *interp, Jim_Obj *filename, int mode) +{ + Jim_SetResultBool(interp, access(Jim_String(filename), mode) != -1); + + return JIM_OK; +} + +static int file_cmd_readable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return file_access(interp, argv[0], R_OK); +} + +static int file_cmd_writable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return file_access(interp, argv[0], W_OK); +} + +static int file_cmd_executable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ +#ifdef X_OK + return file_access(interp, argv[0], X_OK); +#else + + Jim_SetResultBool(interp, 1); + return JIM_OK; +#endif +} + +static int file_cmd_exists(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return file_access(interp, argv[0], F_OK); +} + +static int file_cmd_delete(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int force = Jim_CompareStringImmediate(interp, argv[0], "-force"); + + if (force || Jim_CompareStringImmediate(interp, argv[0], "--")) { + argc--; + argv++; + } + + while (argc--) { + const char *path = Jim_String(argv[0]); + + if (unlink(path) == -1 && errno != ENOENT) { + if (rmdir(path) == -1) { + + if (!force || Jim_EvalPrefix(interp, "file delete force", 1, argv) != JIM_OK) { + Jim_SetResultFormatted(interp, "couldn't delete file \"%s\": %s", path, + strerror(errno)); + return JIM_ERR; + } + } + } + argv++; + } + return JIM_OK; +} + +#ifdef HAVE_MKDIR_ONE_ARG +#define MKDIR_DEFAULT(PATHNAME) mkdir(PATHNAME) +#else +#define MKDIR_DEFAULT(PATHNAME) mkdir(PATHNAME, 0755) +#endif + +static int mkdir_all(char *path) +{ + int ok = 1; + + + goto first; + + while (ok--) { + + { + char *slash = strrchr(path, '/'); + + if (slash && slash != path) { + *slash = 0; + if (mkdir_all(path) != 0) { + return -1; + } + *slash = '/'; + } + } + first: + if (MKDIR_DEFAULT(path) == 0) { + return 0; + } + if (errno == ENOENT) { + + continue; + } + + if (errno == EEXIST) { + jim_stat_t sb; + + if (Jim_Stat(path, &sb) == 0 && S_ISDIR(sb.st_mode)) { + return 0; + } + + errno = EEXIST; + } + + break; + } + return -1; +} + +static int file_cmd_mkdir(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + while (argc--) { + char *path = Jim_StrDup(Jim_String(argv[0])); + int rc = mkdir_all(path); + + Jim_Free(path); + if (rc != 0) { + Jim_SetResultFormatted(interp, "can't create directory \"%#s\": %s", argv[0], + strerror(errno)); + return JIM_ERR; + } + argv++; + } + return JIM_OK; +} + +static int file_cmd_tempfile(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int fd = Jim_MakeTempFile(interp, (argc >= 1) ? Jim_String(argv[0]) : NULL, 0); + + if (fd < 0) { + return JIM_ERR; + } + close(fd); + + return JIM_OK; +} + +static int file_cmd_rename(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *source; + const char *dest; + int force = 0; + + if (argc == 3) { + if (!Jim_CompareStringImmediate(interp, argv[0], "-force")) { + return -1; + } + force++; + argv++; + argc--; + } + + source = Jim_String(argv[0]); + dest = Jim_String(argv[1]); + + if (!force && access(dest, F_OK) == 0) { + Jim_SetResultFormatted(interp, "error renaming \"%#s\" to \"%#s\": target exists", argv[0], + argv[1]); + return JIM_ERR; + } +#if ISWINDOWS + if (access(dest, F_OK) == 0) { + + remove(dest); + } +#endif + if (rename(source, dest) != 0) { + Jim_SetResultFormatted(interp, "error renaming \"%#s\" to \"%#s\": %s", argv[0], argv[1], + strerror(errno)); + return JIM_ERR; + } + + return JIM_OK; +} + +#if defined(HAVE_LINK) && defined(HAVE_SYMLINK) +static int file_cmd_link(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int ret; + const char *source; + const char *dest; + static const char * const options[] = { "-hard", "-symbolic", NULL }; + enum { OPT_HARD, OPT_SYMBOLIC, }; + int option = OPT_HARD; + + if (argc == 3) { + if (Jim_GetEnum(interp, argv[0], options, &option, NULL, JIM_ENUM_ABBREV | JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + argv++; + argc--; + } + + dest = Jim_String(argv[0]); + source = Jim_String(argv[1]); + + if (option == OPT_HARD) { + ret = link(source, dest); + } + else { + ret = symlink(source, dest); + } + + if (ret != 0) { + Jim_SetResultFormatted(interp, "error linking \"%#s\" to \"%#s\": %s", argv[0], argv[1], + strerror(errno)); + return JIM_ERR; + } + + return JIM_OK; +} +#endif + +static int file_stat(Jim_Interp *interp, Jim_Obj *filename, jim_stat_t *sb) +{ + const char *path = Jim_String(filename); + + if (Jim_Stat(path, sb) == -1) { + Jim_SetResultFormatted(interp, "could not read \"%#s\": %s", filename, strerror(errno)); + return JIM_ERR; + } + return JIM_OK; +} + +#ifdef Jim_LinkStat +static int file_lstat(Jim_Interp *interp, Jim_Obj *filename, jim_stat_t *sb) +{ + const char *path = Jim_String(filename); + + if (Jim_LinkStat(path, sb) == -1) { + Jim_SetResultFormatted(interp, "could not read \"%#s\": %s", filename, strerror(errno)); + return JIM_ERR; + } + return JIM_OK; +} +#else +#define file_lstat file_stat +#endif + +static int file_cmd_atime(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (file_stat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResultInt(interp, sb.st_atime); + return JIM_OK; +} + +static int JimSetFileTimes(Jim_Interp *interp, const char *filename, jim_wide us) +{ +#ifdef HAVE_UTIMES + struct timeval times[2]; + + times[1].tv_sec = times[0].tv_sec = us / 1000000; + times[1].tv_usec = times[0].tv_usec = us % 1000000; + + if (utimes(filename, times) != 0) { + Jim_SetResultFormatted(interp, "can't set time on \"%s\": %s", filename, strerror(errno)); + return JIM_ERR; + } + return JIM_OK; +#else + Jim_SetResultString(interp, "Not implemented", -1); + return JIM_ERR; +#endif +} + +static int file_cmd_mtime(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (argc == 2) { + jim_wide secs; + if (Jim_GetWide(interp, argv[1], &secs) != JIM_OK) { + return JIM_ERR; + } + return JimSetFileTimes(interp, Jim_String(argv[0]), secs * 1000000); + } + if (file_stat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResultInt(interp, sb.st_mtime); + return JIM_OK; +} + +#ifdef STAT_MTIME_US +static int file_cmd_mtimeus(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (argc == 2) { + jim_wide us; + if (Jim_GetWide(interp, argv[1], &us) != JIM_OK) { + return JIM_ERR; + } + return JimSetFileTimes(interp, Jim_String(argv[0]), us); + } + if (file_stat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResultInt(interp, STAT_MTIME_US(sb)); + return JIM_OK; +} +#endif + +static int file_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return Jim_EvalPrefix(interp, "file copy", argc, argv); +} + +static int file_cmd_size(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (file_stat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResultInt(interp, sb.st_size); + return JIM_OK; +} + +static int file_cmd_isdirectory(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + int ret = 0; + + if (file_stat(interp, argv[0], &sb) == JIM_OK) { + ret = S_ISDIR(sb.st_mode); + } + Jim_SetResultInt(interp, ret); + return JIM_OK; +} + +static int file_cmd_isfile(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + int ret = 0; + + if (file_stat(interp, argv[0], &sb) == JIM_OK) { + ret = S_ISREG(sb.st_mode); + } + Jim_SetResultInt(interp, ret); + return JIM_OK; +} + +#ifdef HAVE_GETEUID +static int file_cmd_owned(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + int ret = 0; + + if (file_stat(interp, argv[0], &sb) == JIM_OK) { + ret = (geteuid() == sb.st_uid); + } + Jim_SetResultInt(interp, ret); + return JIM_OK; +} +#endif + +#if defined(HAVE_READLINK) +static int file_cmd_readlink(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *path = Jim_String(argv[0]); + char *linkValue = Jim_Alloc(MAXPATHLEN + 1); + + int linkLength = readlink(path, linkValue, MAXPATHLEN); + + if (linkLength == -1) { + Jim_Free(linkValue); + Jim_SetResultFormatted(interp, "could not read link \"%#s\": %s", argv[0], strerror(errno)); + return JIM_ERR; + } + linkValue[linkLength] = 0; + Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, linkValue, linkLength)); + return JIM_OK; +} +#endif + +static int file_cmd_type(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (file_lstat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResultString(interp, JimGetFileType((int)sb.st_mode), -1); + return JIM_OK; +} + +#ifdef Jim_LinkStat +static int file_cmd_lstat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (file_lstat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + return Jim_FileStoreStatData(interp, argc == 2 ? argv[1] : NULL, &sb); +} +#else +#define file_cmd_lstat file_cmd_stat +#endif + +static int file_cmd_stat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_stat_t sb; + + if (file_stat(interp, argv[0], &sb) != JIM_OK) { + return JIM_ERR; + } + return Jim_FileStoreStatData(interp, argc == 2 ? argv[1] : NULL, &sb); +} + +static const jim_subcmd_type file_command_table[] = { + { "atime", + "name", + file_cmd_atime, + 1, + 1, + + }, + { "mtime", + "name ?time?", + file_cmd_mtime, + 1, + 2, + + }, +#ifdef STAT_MTIME_US + { "mtimeus", + "name ?time?", + file_cmd_mtimeus, + 1, + 2, + + }, +#endif + { "copy", + "?-force? source dest", + file_cmd_copy, + 2, + 3, + + }, + { "dirname", + "name", + file_cmd_dirname, + 1, + 1, + + }, + { "rootname", + "name", + file_cmd_rootname, + 1, + 1, + + }, + { "extension", + "name", + file_cmd_extension, + 1, + 1, + + }, + { "tail", + "name", + file_cmd_tail, + 1, + 1, + + }, + { "split", + "name", + file_cmd_split, + 1, + 1, + + }, + { "normalize", + "name", + file_cmd_normalize, + 1, + 1, + + }, + { "join", + "name ?name ...?", + file_cmd_join, + 1, + -1, + + }, + { "readable", + "name", + file_cmd_readable, + 1, + 1, + + }, + { "writable", + "name", + file_cmd_writable, + 1, + 1, + + }, + { "executable", + "name", + file_cmd_executable, + 1, + 1, + + }, + { "exists", + "name", + file_cmd_exists, + 1, + 1, + + }, + { "delete", + "?-force|--? name ...", + file_cmd_delete, + 1, + -1, + + }, + { "mkdir", + "dir ...", + file_cmd_mkdir, + 1, + -1, + + }, + { "tempfile", + "?template?", + file_cmd_tempfile, + 0, + 1, + + }, + { "rename", + "?-force? source dest", + file_cmd_rename, + 2, + 3, + + }, +#if defined(HAVE_LINK) && defined(HAVE_SYMLINK) + { "link", + "?-symbolic|-hard? newname target", + file_cmd_link, + 2, + 3, + + }, +#endif +#if defined(HAVE_READLINK) + { "readlink", + "name", + file_cmd_readlink, + 1, + 1, + + }, +#endif + { "size", + "name", + file_cmd_size, + 1, + 1, + + }, + { "stat", + "name ?var?", + file_cmd_stat, + 1, + 2, + + }, + { "lstat", + "name ?var?", + file_cmd_lstat, + 1, + 2, + + }, + { "type", + "name", + file_cmd_type, + 1, + 1, + + }, +#ifdef HAVE_GETEUID + { "owned", + "name", + file_cmd_owned, + 1, + 1, + + }, +#endif + { "isdirectory", + "name", + file_cmd_isdirectory, + 1, + 1, + + }, + { "isfile", + "name", + file_cmd_isfile, + 1, + 1, + + }, + { + NULL + } +}; + +static int Jim_CdCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *path; + + if (argc != 2) { + Jim_WrongNumArgs(interp, 1, argv, "dirname"); + return JIM_ERR; + } + + path = Jim_String(argv[1]); + + if (chdir(path) != 0) { + Jim_SetResultFormatted(interp, "couldn't change working directory to \"%s\": %s", path, + strerror(errno)); + return JIM_ERR; + } + return JIM_OK; +} + +static int Jim_PwdCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + char *cwd = Jim_Alloc(MAXPATHLEN); + + if (getcwd(cwd, MAXPATHLEN) == NULL) { + Jim_SetResultString(interp, "Failed to get pwd", -1); + Jim_Free(cwd); + return JIM_ERR; + } + JimFixPath(cwd); + Jim_SetResultString(interp, cwd, -1); + + Jim_Free(cwd); + return JIM_OK; +} + +int Jim_fileInit(Jim_Interp *interp) +{ + Jim_PackageProvideCheck(interp, "file"); + Jim_CreateCommand(interp, "file", Jim_SubCmdProc, (void *)file_command_table, NULL); + Jim_CreateCommand(interp, "pwd", Jim_PwdCmd, NULL, NULL); + Jim_CreateCommand(interp, "cd", Jim_CdCmd, NULL, NULL); + return JIM_OK; +} + +#include +#include + + +#if (!(defined(HAVE_VFORK) || defined(HAVE_FORK)) || !defined(HAVE_WAITPID)) && !defined(__MINGW32__) +static int Jim_ExecCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *cmdlineObj = Jim_NewEmptyStringObj(interp); + int i, j; + int rc; + + + for (i = 1; i < argc; i++) { + int len; + const char *arg = Jim_GetString(argv[i], &len); + + if (i > 1) { + Jim_AppendString(interp, cmdlineObj, " ", 1); + } + if (strpbrk(arg, "\\\" ") == NULL) { + + Jim_AppendString(interp, cmdlineObj, arg, len); + continue; + } + + Jim_AppendString(interp, cmdlineObj, "\"", 1); + for (j = 0; j < len; j++) { + if (arg[j] == '\\' || arg[j] == '"') { + Jim_AppendString(interp, cmdlineObj, "\\", 1); + } + Jim_AppendString(interp, cmdlineObj, &arg[j], 1); + } + Jim_AppendString(interp, cmdlineObj, "\"", 1); + } + rc = system(Jim_String(cmdlineObj)); + + Jim_FreeNewObj(interp, cmdlineObj); + + if (rc) { + Jim_Obj *errorCode = Jim_NewListObj(interp, NULL, 0); + Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, "CHILDSTATUS", -1)); + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, 0)); + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, rc)); + Jim_SetGlobalVariableStr(interp, "errorCode", errorCode); + return JIM_ERR; + } + + return JIM_OK; +} + +int Jim_execInit(Jim_Interp *interp) +{ + Jim_PackageProvideCheck(interp, "exec"); + Jim_CreateCommand(interp, "exec", Jim_ExecCmd, NULL, NULL); + return JIM_OK; +} +#else + + +#include +#include +#include + +struct WaitInfoTable; + +static char **JimOriginalEnviron(void); +static char **JimSaveEnv(char **env); +static void JimRestoreEnv(char **env); +static int JimCreatePipeline(Jim_Interp *interp, int argc, Jim_Obj *const *argv, + phandle_t **pidArrayPtr, int *inPipePtr, int *outPipePtr, int *errFilePtr); +static void JimDetachPids(struct WaitInfoTable *table, int numPids, const phandle_t *pidPtr); +static int JimCleanupChildren(Jim_Interp *interp, int numPids, phandle_t *pidPtr, Jim_Obj *errStrObj); +static int Jim_WaitCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv); + +#if defined(__MINGW32__) +static phandle_t JimStartWinProcess(Jim_Interp *interp, char **argv, char **env, int inputId, int outputId, int errorId); +#endif + +static void Jim_RemoveTrailingNewline(Jim_Obj *objPtr) +{ + int len; + const char *s = Jim_GetString(objPtr, &len); + + if (len > 0 && s[len - 1] == '\n') { + objPtr->length--; + objPtr->bytes[objPtr->length] = '\0'; + } +} + +static int JimAppendStreamToString(Jim_Interp *interp, int fd, Jim_Obj *strObj) +{ + char buf[256]; + int ret = 0; + + while (1) { + int retval = read(fd, buf, sizeof(buf)); + if (retval > 0) { + ret = 1; + Jim_AppendString(interp, strObj, buf, retval); + } + if (retval <= 0) { + break; + } + } + close(fd); + return ret; +} + +static char **JimBuildEnv(Jim_Interp *interp) +{ + int i; + int size; + int num; + int n; + char **envptr; + char *envdata; + + Jim_Obj *objPtr = Jim_GetGlobalVariableStr(interp, "env", JIM_NONE); + + if (!objPtr) { + return JimOriginalEnviron(); + } + + + + num = Jim_ListLength(interp, objPtr); + if (num % 2) { + + num--; + } + size = Jim_Length(objPtr) + 2; + + envptr = Jim_Alloc(sizeof(*envptr) * (num / 2 + 1) + size); + envdata = (char *)&envptr[num / 2 + 1]; + + n = 0; + for (i = 0; i < num; i += 2) { + const char *s1, *s2; + Jim_Obj *elemObj; + + Jim_ListIndex(interp, objPtr, i, &elemObj, JIM_NONE); + s1 = Jim_String(elemObj); + Jim_ListIndex(interp, objPtr, i + 1, &elemObj, JIM_NONE); + s2 = Jim_String(elemObj); + + envptr[n] = envdata; + envdata += sprintf(envdata, "%s=%s", s1, s2); + envdata++; + n++; + } + envptr[n] = NULL; + *envdata = 0; + + return envptr; +} + +static void JimFreeEnv(char **env, char **original_environ) +{ + if (env != original_environ) { + Jim_Free(env); + } +} + +static Jim_Obj *JimMakeErrorCode(Jim_Interp *interp, long pid, int waitStatus, Jim_Obj *errStrObj) +{ + Jim_Obj *errorCode = Jim_NewListObj(interp, NULL, 0); + + if (pid <= 0) { + Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, "NONE", -1)); + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, pid)); + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, -1)); + } + else if (WIFEXITED(waitStatus)) { + Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, "CHILDSTATUS", -1)); + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, pid)); + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, WEXITSTATUS(waitStatus))); + } + else { + const char *type; + const char *action; + const char *signame; + + if (WIFSIGNALED(waitStatus)) { + type = "CHILDKILLED"; + action = "killed"; + signame = Jim_SignalId(WTERMSIG(waitStatus)); + } + else { + type = "CHILDSUSP"; + action = "suspended"; + signame = "none"; + } + + Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, type, -1)); + + if (errStrObj) { + Jim_AppendStrings(interp, errStrObj, "child ", action, " by signal ", Jim_SignalId(WTERMSIG(waitStatus)), "\n", NULL); + } + + Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, pid)); + Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, signame, -1)); + } + return errorCode; +} + +static int JimCheckWaitStatus(Jim_Interp *interp, long pid, int waitStatus, Jim_Obj *errStrObj) +{ + if (WIFEXITED(waitStatus) && WEXITSTATUS(waitStatus) == 0) { + return JIM_OK; + } + Jim_SetGlobalVariableStr(interp, "errorCode", JimMakeErrorCode(interp, pid, waitStatus, errStrObj)); + + return JIM_ERR; +} + + +struct WaitInfo +{ + phandle_t phandle; + int status; + int flags; +}; + + +struct WaitInfoTable { + struct WaitInfo *info; + int size; + int used; + int refcount; +}; + + +#define WI_DETACHED 2 + +#define WAIT_TABLE_GROW_BY 4 + +static void JimFreeWaitInfoTable(struct Jim_Interp *interp, void *privData) +{ + struct WaitInfoTable *table = privData; + + if (--table->refcount == 0) { + Jim_Free(table->info); + Jim_Free(table); + } +} + +static struct WaitInfoTable *JimAllocWaitInfoTable(void) +{ + struct WaitInfoTable *table = Jim_Alloc(sizeof(*table)); + table->info = NULL; + table->size = table->used = 0; + table->refcount = 1; + + return table; +} + +static int JimWaitRemove(struct WaitInfoTable *table, phandle_t phandle) +{ + int i; + + + for (i = 0; i < table->used; i++) { + if (phandle == table->info[i].phandle) { + if (i != table->used - 1) { + table->info[i] = table->info[table->used - 1]; + } + table->used--; + return 0; + } + } + return -1; +} + +static int Jim_ExecCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int outputId; + int errorId; + phandle_t *pidPtr; + int numPids, result; + int child_siginfo = 1; + Jim_Obj *childErrObj; + Jim_Obj *errStrObj; + struct WaitInfoTable *table = Jim_CmdPrivData(interp); + + if (argc > 1 && Jim_CompareStringImmediate(interp, argv[argc - 1], "&")) { + Jim_Obj *listObj; + int i; + + argc--; + numPids = JimCreatePipeline(interp, argc - 1, argv + 1, &pidPtr, NULL, NULL, NULL); + if (numPids < 0) { + return JIM_ERR; + } + + listObj = Jim_NewListObj(interp, NULL, 0); + for (i = 0; i < numPids; i++) { + Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, JimProcessPid(pidPtr[i]))); + } + Jim_SetResult(interp, listObj); + JimDetachPids(table, numPids, pidPtr); + Jim_Free(pidPtr); + return JIM_OK; + } + + numPids = + JimCreatePipeline(interp, argc - 1, argv + 1, &pidPtr, NULL, &outputId, &errorId); + + if (numPids < 0) { + return JIM_ERR; + } + + result = JIM_OK; + + errStrObj = Jim_NewStringObj(interp, "", 0); + + + if (outputId != -1) { + if (JimAppendStreamToString(interp, outputId, errStrObj) < 0) { + result = JIM_ERR; + Jim_SetResultErrno(interp, "error reading from output pipe"); + } + } + + + childErrObj = Jim_NewStringObj(interp, "", 0); + Jim_IncrRefCount(childErrObj); + + if (JimCleanupChildren(interp, numPids, pidPtr, childErrObj) != JIM_OK) { + result = JIM_ERR; + } + + if (errorId != -1) { + int ret; + Jim_Lseek(errorId, 0, SEEK_SET); + ret = JimAppendStreamToString(interp, errorId, errStrObj); + if (ret < 0) { + Jim_SetResultErrno(interp, "error reading from error pipe"); + result = JIM_ERR; + } + else if (ret > 0) { + + child_siginfo = 0; + } + } + + if (child_siginfo) { + + Jim_AppendObj(interp, errStrObj, childErrObj); + } + Jim_DecrRefCount(interp, childErrObj); + + + Jim_RemoveTrailingNewline(errStrObj); + + + Jim_SetResult(interp, errStrObj); + + return result; +} + +static long JimWaitForProcess(struct WaitInfoTable *table, phandle_t phandle, int *statusPtr) +{ + if (JimWaitRemove(table, phandle) == 0) { + + return waitpid(phandle, statusPtr, 0); + } + + + return -1; +} + +static void JimDetachPids(struct WaitInfoTable *table, int numPids, const phandle_t *pidPtr) +{ + int j; + + for (j = 0; j < numPids; j++) { + + int i; + for (i = 0; i < table->used; i++) { + if (pidPtr[j] == table->info[i].phandle) { + table->info[i].flags |= WI_DETACHED; + break; + } + } + } +} + +static int JimGetChannelFd(Jim_Interp *interp, const char *name) +{ + Jim_Obj *objv[2]; + + objv[0] = Jim_NewStringObj(interp, name, -1); + objv[1] = Jim_NewStringObj(interp, "getfd", -1); + + if (Jim_EvalObjVector(interp, 2, objv) == JIM_OK) { + jim_wide fd; + if (Jim_GetWide(interp, Jim_GetResult(interp), &fd) == JIM_OK) { + return fd; + } + } + return -1; +} + +static void JimReapDetachedPids(struct WaitInfoTable *table) +{ + struct WaitInfo *waitPtr; + int count; + int dest; + + if (!table) { + return; + } + + waitPtr = table->info; + dest = 0; + for (count = table->used; count > 0; waitPtr++, count--) { + if (waitPtr->flags & WI_DETACHED) { + int status; + long pid = waitpid(waitPtr->phandle, &status, WNOHANG); + if (pid > 0) { + + table->used--; + continue; + } + } + if (waitPtr != &table->info[dest]) { + table->info[dest] = *waitPtr; + } + dest++; + } +} + +static int Jim_WaitCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + struct WaitInfoTable *table = Jim_CmdPrivData(interp); + int nohang = 0; + long pid; + phandle_t phandle; + int status; + Jim_Obj *errCodeObj; + + + if (argc == 1) { + JimReapDetachedPids(table); + return JIM_OK; + } + + if (argc > 1 && Jim_CompareStringImmediate(interp, argv[1], "-nohang")) { + nohang = 1; + } + if (argc != nohang + 2) { + Jim_WrongNumArgs(interp, 1, argv, "?-nohang? ?pid?"); + return JIM_ERR; + } + if (Jim_GetLong(interp, argv[nohang + 1], &pid) != JIM_OK) { + return JIM_ERR; + } + + + phandle = JimWaitPid(pid, &status, nohang ? WNOHANG : 0); + if (phandle == JIM_BAD_PHANDLE) { + pid = -1; + } +#ifndef __MINGW32__ + else if (pid < 0) { + pid = phandle; + } +#endif + + errCodeObj = JimMakeErrorCode(interp, pid, status, NULL); + + if (phandle != JIM_BAD_PHANDLE && (WIFEXITED(status) || WIFSIGNALED(status))) { + + JimWaitRemove(table, phandle); + } + Jim_SetResult(interp, errCodeObj); + return JIM_OK; +} + +static int Jim_PidCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 1) { + Jim_WrongNumArgs(interp, 1, argv, ""); + return JIM_ERR; + } + + Jim_SetResultInt(interp, (jim_wide)getpid()); + return JIM_OK; +} + +static int +JimCreatePipeline(Jim_Interp *interp, int argc, Jim_Obj *const *argv, phandle_t **pidArrayPtr, + int *inPipePtr, int *outPipePtr, int *errFilePtr) +{ + phandle_t *pidPtr = NULL; /* Points to alloc-ed array holding all + * the pids of child processes. */ + int numPids = 0; /* Actual number of processes that exist + * at *pidPtr right now. */ + int cmdCount; /* Count of number of distinct commands + * found in argc/argv. */ + const char *input = NULL; /* Describes input for pipeline, depending + * on "inputFile". NULL means take input + * from stdin/pipe. */ + int input_len = 0; + +#define FILE_NAME 0 +#define FILE_APPEND 1 +#define FILE_HANDLE 2 +#define FILE_TEXT 3 + + int inputFile = FILE_NAME; /* 1 means input is name of input file. + * 2 means input is filehandle name. + * 0 means input holds actual + * text to be input to command. */ + + int outputFile = FILE_NAME; /* 0 means output is the name of output file. + * 1 means output is the name of output file, and append. + * 2 means output is filehandle name. + * All this is ignored if output is NULL + */ + int errorFile = FILE_NAME; /* 0 means error is the name of error file. + * 1 means error is the name of error file, and append. + * 2 means error is filehandle name. + * All this is ignored if error is NULL + */ + const char *output = NULL; /* Holds name of output file to pipe to, + * or NULL if output goes to stdout/pipe. */ + const char *error = NULL; /* Holds name of stderr file to pipe to, + * or NULL if stderr goes to stderr/pipe. */ + int inputId = -1; + int outputId = -1; + int errorId = -1; + int lastOutputId = -1; + int pipeIds[2]; + int firstArg, lastArg; /* Indexes of first and last arguments in + * current command. */ + int lastBar; + int i; + phandle_t phandle; + char **save_environ; +#if defined(HAVE_EXECVPE) && !defined(__MINGW32__) + char **child_environ; +#endif + struct WaitInfoTable *table = Jim_CmdPrivData(interp); + + + char **arg_array = Jim_Alloc(sizeof(*arg_array) * (argc + 1)); + int arg_count = 0; + + if (inPipePtr != NULL) { + *inPipePtr = -1; + } + if (outPipePtr != NULL) { + *outPipePtr = -1; + } + if (errFilePtr != NULL) { + *errFilePtr = -1; + } + pipeIds[0] = pipeIds[1] = -1; + + cmdCount = 1; + lastBar = -1; + for (i = 0; i < argc; i++) { + const char *arg = Jim_String(argv[i]); + + if (arg[0] == '<') { + inputFile = FILE_NAME; + input = arg + 1; + if (*input == '<') { + inputFile = FILE_TEXT; + input_len = Jim_Length(argv[i]) - 2; + input++; + } + else if (*input == '@') { + inputFile = FILE_HANDLE; + input++; + } + + if (!*input && ++i < argc) { + input = Jim_GetString(argv[i], &input_len); + } + } + else if (arg[0] == '>') { + int dup_error = 0; + + outputFile = FILE_NAME; + + output = arg + 1; + if (*output == '>') { + outputFile = FILE_APPEND; + output++; + } + if (*output == '&') { + + output++; + dup_error = 1; + } + if (*output == '@') { + outputFile = FILE_HANDLE; + output++; + } + if (!*output && ++i < argc) { + output = Jim_String(argv[i]); + } + if (dup_error) { + errorFile = outputFile; + error = output; + } + } + else if (arg[0] == '2' && arg[1] == '>') { + error = arg + 2; + errorFile = FILE_NAME; + + if (*error == '@') { + errorFile = FILE_HANDLE; + error++; + } + else if (*error == '>') { + errorFile = FILE_APPEND; + error++; + } + if (!*error && ++i < argc) { + error = Jim_String(argv[i]); + } + } + else { + if (strcmp(arg, "|") == 0 || strcmp(arg, "|&") == 0) { + if (i == lastBar + 1 || i == argc - 1) { + Jim_SetResultString(interp, "illegal use of | or |& in command", -1); + goto badargs; + } + lastBar = i; + cmdCount++; + } + + arg_array[arg_count++] = (char *)arg; + continue; + } + + if (i >= argc) { + Jim_SetResultFormatted(interp, "can't specify \"%s\" as last word in command", arg); + goto badargs; + } + } + + if (arg_count == 0) { + Jim_SetResultString(interp, "didn't specify command to execute", -1); +badargs: + Jim_Free(arg_array); + return -1; + } + + + save_environ = JimSaveEnv(JimBuildEnv(interp)); + + if (input != NULL) { + if (inputFile == FILE_TEXT) { + inputId = Jim_MakeTempFile(interp, NULL, 1); + if (inputId == -1) { + goto error; + } + if (write(inputId, input, input_len) != input_len) { + Jim_SetResultErrno(interp, "couldn't write temp file"); + close(inputId); + goto error; + } + Jim_Lseek(inputId, 0L, SEEK_SET); + } + else if (inputFile == FILE_HANDLE) { + int fd = JimGetChannelFd(interp, input); + + if (fd < 0) { + goto error; + } + inputId = dup(fd); + } + else { + inputId = Jim_OpenForRead(input); + if (inputId == -1) { + Jim_SetResultFormatted(interp, "couldn't read file \"%s\": %s", input, strerror(Jim_Errno())); + goto error; + } + } + } + else if (inPipePtr != NULL) { + if (pipe(pipeIds) != 0) { + Jim_SetResultErrno(interp, "couldn't create input pipe for command"); + goto error; + } + inputId = pipeIds[0]; + *inPipePtr = pipeIds[1]; + pipeIds[0] = pipeIds[1] = -1; + } + + if (output != NULL) { + if (outputFile == FILE_HANDLE) { + int fd = JimGetChannelFd(interp, output); + if (fd < 0) { + goto error; + } + lastOutputId = dup(fd); + } + else { + lastOutputId = Jim_OpenForWrite(output, outputFile == FILE_APPEND); + if (lastOutputId == -1) { + Jim_SetResultFormatted(interp, "couldn't write file \"%s\": %s", output, strerror(Jim_Errno())); + goto error; + } + } + } + else if (outPipePtr != NULL) { + if (pipe(pipeIds) != 0) { + Jim_SetResultErrno(interp, "couldn't create output pipe"); + goto error; + } + lastOutputId = pipeIds[1]; + *outPipePtr = pipeIds[0]; + pipeIds[0] = pipeIds[1] = -1; + } + + if (error != NULL) { + if (errorFile == FILE_HANDLE) { + if (strcmp(error, "1") == 0) { + + if (lastOutputId != -1) { + errorId = dup(lastOutputId); + } + else { + + error = "stdout"; + } + } + if (errorId == -1) { + int fd = JimGetChannelFd(interp, error); + if (fd < 0) { + goto error; + } + errorId = dup(fd); + } + } + else { + errorId = Jim_OpenForWrite(error, errorFile == FILE_APPEND); + if (errorId == -1) { + Jim_SetResultFormatted(interp, "couldn't write file \"%s\": %s", error, strerror(Jim_Errno())); + goto error; + } + } + } + else if (errFilePtr != NULL) { + errorId = Jim_MakeTempFile(interp, NULL, 1); + if (errorId == -1) { + goto error; + } + *errFilePtr = dup(errorId); + } + + + pidPtr = Jim_Alloc(cmdCount * sizeof(*pidPtr)); + for (firstArg = 0; firstArg < arg_count; numPids++, firstArg = lastArg + 1) { + int pipe_dup_err = 0; + int origErrorId = errorId; + + for (lastArg = firstArg; lastArg < arg_count; lastArg++) { + if (strcmp(arg_array[lastArg], "|") == 0) { + break; + } + if (strcmp(arg_array[lastArg], "|&") == 0) { + pipe_dup_err = 1; + break; + } + } + + if (lastArg == firstArg) { + Jim_SetResultString(interp, "missing command to exec", -1); + goto error; + } + + + arg_array[lastArg] = NULL; + if (lastArg == arg_count) { + outputId = lastOutputId; + lastOutputId = -1; + } + else { + if (pipe(pipeIds) != 0) { + Jim_SetResultErrno(interp, "couldn't create pipe"); + goto error; + } + outputId = pipeIds[1]; + } + + + if (pipe_dup_err) { + errorId = outputId; + } + + + +#ifdef __MINGW32__ + phandle = JimStartWinProcess(interp, &arg_array[firstArg], save_environ, inputId, outputId, errorId); + if (phandle == JIM_BAD_PHANDLE) { + Jim_SetResultFormatted(interp, "couldn't exec \"%s\"", arg_array[firstArg]); + goto error; + } +#else + i = strlen(arg_array[firstArg]); + +#ifdef HAVE_EXECVPE + child_environ = Jim_GetEnviron(); +#endif +#ifdef HAVE_VFORK + phandle = vfork(); +#else + phandle = fork(); +#endif + if (phandle < 0) { + Jim_SetResultErrno(interp, "couldn't fork child process"); + goto error; + } + if (phandle == 0) { + + + if (inputId != -1 && inputId != fileno(stdin)) { + dup2(inputId, fileno(stdin)); + close(inputId); + } + if (outputId != -1 && outputId != fileno(stdout)) { + dup2(outputId, fileno(stdout)); + if (outputId != errorId) { + close(outputId); + } + } + if (errorId != -1 && errorId != fileno(stderr)) { + dup2(errorId, fileno(stderr)); + close(errorId); + } + + if (outPipePtr && *outPipePtr != -1) { + close(*outPipePtr); + } + if (errFilePtr && *errFilePtr != -1) { + close(*errFilePtr); + } + if (pipeIds[0] != -1) { + close(pipeIds[0]); + } + if (lastOutputId != -1) { + close(lastOutputId); + } + + execvpe(arg_array[firstArg], &arg_array[firstArg], child_environ); + + if (write(fileno(stderr), "couldn't exec \"", 15) && + write(fileno(stderr), arg_array[firstArg], i) && + write(fileno(stderr), "\"\n", 2)) { + + } +#ifdef JIM_MAINTAINER + { + + static char *const false_argv[2] = {"false", NULL}; + execvp(false_argv[0],false_argv); + } +#endif + _exit(127); + } +#endif + + + + if (table->used == table->size) { + table->size += WAIT_TABLE_GROW_BY; + table->info = Jim_Realloc(table->info, table->size * sizeof(*table->info)); + } + + table->info[table->used].phandle = phandle; + table->info[table->used].flags = 0; + table->used++; + + pidPtr[numPids] = phandle; + + + errorId = origErrorId; + + + if (inputId != -1) { + close(inputId); + } + if (outputId != -1) { + close(outputId); + } + inputId = pipeIds[0]; + pipeIds[0] = pipeIds[1] = -1; + } + *pidArrayPtr = pidPtr; + + + cleanup: + if (inputId != -1) { + close(inputId); + } + if (lastOutputId != -1) { + close(lastOutputId); + } + if (errorId != -1) { + close(errorId); + } + Jim_Free(arg_array); + + JimRestoreEnv(save_environ); + + return numPids; + + + error: + if ((inPipePtr != NULL) && (*inPipePtr != -1)) { + close(*inPipePtr); + *inPipePtr = -1; + } + if ((outPipePtr != NULL) && (*outPipePtr != -1)) { + close(*outPipePtr); + *outPipePtr = -1; + } + if ((errFilePtr != NULL) && (*errFilePtr != -1)) { + close(*errFilePtr); + *errFilePtr = -1; + } + if (pipeIds[0] != -1) { + close(pipeIds[0]); + } + if (pipeIds[1] != -1) { + close(pipeIds[1]); + } + if (pidPtr != NULL) { + for (i = 0; i < numPids; i++) { + if (pidPtr[i] != JIM_BAD_PHANDLE) { + JimDetachPids(table, 1, &pidPtr[i]); + } + } + Jim_Free(pidPtr); + } + numPids = -1; + goto cleanup; +} + + +static int JimCleanupChildren(Jim_Interp *interp, int numPids, phandle_t *pidPtr, Jim_Obj *errStrObj) +{ + struct WaitInfoTable *table = Jim_CmdPrivData(interp); + int result = JIM_OK; + int i; + + + for (i = 0; i < numPids; i++) { + int waitStatus = 0; + long pid = JimWaitForProcess(table, pidPtr[i], &waitStatus); + if (pid > 0) { + if (JimCheckWaitStatus(interp, pid, waitStatus, errStrObj) != JIM_OK) { + result = JIM_ERR; + } + } + } + Jim_Free(pidPtr); + + return result; +} + +int Jim_execInit(Jim_Interp *interp) +{ + struct WaitInfoTable *waitinfo; + + Jim_PackageProvideCheck(interp, "exec"); + + waitinfo = JimAllocWaitInfoTable(); + Jim_CreateCommand(interp, "exec", Jim_ExecCmd, waitinfo, JimFreeWaitInfoTable); + waitinfo->refcount++; + Jim_CreateCommand(interp, "wait", Jim_WaitCommand, waitinfo, JimFreeWaitInfoTable); + Jim_CreateCommand(interp, "pid", Jim_PidCommand, 0, 0); + + return JIM_OK; +} + +#if defined(__MINGW32__) + + +static int +JimWinFindExecutable(const char *originalName, char fullPath[MAX_PATH]) +{ + int i; + static char extensions[][5] = {".exe", "", ".bat"}; + + for (i = 0; i < (int) (sizeof(extensions) / sizeof(extensions[0])); i++) { + snprintf(fullPath, MAX_PATH, "%s%s", originalName, extensions[i]); + + if (SearchPath(NULL, fullPath, NULL, MAX_PATH, fullPath, NULL) == 0) { + continue; + } + if (GetFileAttributes(fullPath) & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + return 0; + } + + return -1; +} + +static char **JimSaveEnv(char **env) +{ + return env; +} + +static void JimRestoreEnv(char **env) +{ + JimFreeEnv(env, Jim_GetEnviron()); +} + +static char **JimOriginalEnviron(void) +{ + return NULL; +} + +static Jim_Obj * +JimWinBuildCommandLine(Jim_Interp *interp, char **argv) +{ + char *start, *special; + int quote, i; + + Jim_Obj *strObj = Jim_NewStringObj(interp, "", 0); + + for (i = 0; argv[i]; i++) { + if (i > 0) { + Jim_AppendString(interp, strObj, " ", 1); + } + + if (argv[i][0] == '\0') { + quote = 1; + } + else { + quote = 0; + for (start = argv[i]; *start != '\0'; start++) { + if (isspace(UCHAR(*start))) { + quote = 1; + break; + } + } + } + if (quote) { + Jim_AppendString(interp, strObj, "\"" , 1); + } + + start = argv[i]; + for (special = argv[i]; ; ) { + if ((*special == '\\') && (special[1] == '\\' || + special[1] == '"' || (quote && special[1] == '\0'))) { + Jim_AppendString(interp, strObj, start, special - start); + start = special; + while (1) { + special++; + if (*special == '"' || (quote && *special == '\0')) { + + Jim_AppendString(interp, strObj, start, special - start); + break; + } + if (*special != '\\') { + break; + } + } + Jim_AppendString(interp, strObj, start, special - start); + start = special; + } + if (*special == '"') { + if (special == start) { + Jim_AppendString(interp, strObj, "\"", 1); + } + else { + Jim_AppendString(interp, strObj, start, special - start); + } + Jim_AppendString(interp, strObj, "\\\"", 2); + start = special + 1; + } + if (*special == '\0') { + break; + } + special++; + } + Jim_AppendString(interp, strObj, start, special - start); + if (quote) { + Jim_AppendString(interp, strObj, "\"", 1); + } + } + return strObj; +} + +static phandle_t +JimStartWinProcess(Jim_Interp *interp, char **argv, char **env, int inputId, int outputId, int errorId) +{ + STARTUPINFO startInfo; + PROCESS_INFORMATION procInfo; + HANDLE hProcess; + char execPath[MAX_PATH]; + phandle_t phandle = INVALID_HANDLE_VALUE; + Jim_Obj *cmdLineObj; + char *winenv; + + if (JimWinFindExecutable(argv[0], execPath) < 0) { + return phandle; + } + argv[0] = execPath; + + hProcess = GetCurrentProcess(); + cmdLineObj = JimWinBuildCommandLine(interp, argv); + + + ZeroMemory(&startInfo, sizeof(startInfo)); + startInfo.cb = sizeof(startInfo); + startInfo.dwFlags = STARTF_USESTDHANDLES; + startInfo.hStdInput = INVALID_HANDLE_VALUE; + startInfo.hStdOutput= INVALID_HANDLE_VALUE; + startInfo.hStdError = INVALID_HANDLE_VALUE; + + if (inputId == -1) { + inputId = _fileno(stdin); + } + DuplicateHandle(hProcess, (HANDLE)_get_osfhandle(inputId), hProcess, &startInfo.hStdInput, + 0, TRUE, DUPLICATE_SAME_ACCESS); + if (startInfo.hStdInput == INVALID_HANDLE_VALUE) { + goto end; + } + + if (outputId == -1) { + outputId = _fileno(stdout); + } + DuplicateHandle(hProcess, (HANDLE)_get_osfhandle(outputId), hProcess, &startInfo.hStdOutput, + 0, TRUE, DUPLICATE_SAME_ACCESS); + if (startInfo.hStdOutput == INVALID_HANDLE_VALUE) { + goto end; + } + + + if (errorId == -1) { + errorId = _fileno(stderr); + } + DuplicateHandle(hProcess, (HANDLE)_get_osfhandle(errorId), hProcess, &startInfo.hStdError, + 0, TRUE, DUPLICATE_SAME_ACCESS); + if (startInfo.hStdError == INVALID_HANDLE_VALUE) { + goto end; + } + + if (env == NULL) { + + winenv = NULL; + } + else if (env[0] == NULL) { + winenv = (char *)"\0"; + } + else { + winenv = env[0]; + } + + if (!CreateProcess(NULL, (char *)Jim_String(cmdLineObj), NULL, NULL, TRUE, + 0, winenv, NULL, &startInfo, &procInfo)) { + goto end; + } + + + WaitForInputIdle(procInfo.hProcess, 5000); + CloseHandle(procInfo.hThread); + + phandle = procInfo.hProcess; + + end: + Jim_FreeNewObj(interp, cmdLineObj); + if (startInfo.hStdInput != INVALID_HANDLE_VALUE) { + CloseHandle(startInfo.hStdInput); + } + if (startInfo.hStdOutput != INVALID_HANDLE_VALUE) { + CloseHandle(startInfo.hStdOutput); + } + if (startInfo.hStdError != INVALID_HANDLE_VALUE) { + CloseHandle(startInfo.hStdError); + } + return phandle; +} + +#else + +static char **JimOriginalEnviron(void) +{ + return Jim_GetEnviron(); +} + +static char **JimSaveEnv(char **env) +{ + char **saveenv = Jim_GetEnviron(); + Jim_SetEnviron(env); + return saveenv; +} + +static void JimRestoreEnv(char **env) +{ + JimFreeEnv(Jim_GetEnviron(), env); + Jim_SetEnviron(env); +} +#endif +#endif + + +#include +#include +#include +#include + + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +struct clock_options { + int gmt; + const char *format; +}; + +static int parse_clock_options(Jim_Interp *interp, int argc, Jim_Obj *const *argv, struct clock_options *opts) +{ + static const char * const options[] = { "-gmt", "-format", NULL }; + enum { OPT_GMT, OPT_FORMAT, }; + int i; + + for (i = 0; i < argc; i += 2) { + int option; + if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { + return JIM_ERR; + } + switch (option) { + case OPT_GMT: + if (Jim_GetBoolean(interp, argv[i + 1], &opts->gmt) != JIM_OK) { + return JIM_ERR; + } + break; + case OPT_FORMAT: + opts->format = Jim_String(argv[i + 1]); + break; + } + } + return JIM_OK; +} + +static int clock_cmd_format(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + + char buf[100]; + time_t t; + jim_wide seconds; + struct clock_options options = { 0, "%a %b %d %H:%M:%S %Z %Y" }; + struct tm *tm; + + if (Jim_GetWide(interp, argv[0], &seconds) != JIM_OK) { + return JIM_ERR; + } + if (argc % 2 == 0) { + return -1; + } + if (parse_clock_options(interp, argc - 1, argv + 1, &options) == JIM_ERR) { + return JIM_ERR; + } + + t = seconds; + tm = options.gmt ? gmtime(&t) : localtime(&t); + + if (tm == NULL || strftime(buf, sizeof(buf), options.format, tm) == 0) { + Jim_SetResultString(interp, "format string too long or invalid time", -1); + return JIM_ERR; + } + + Jim_SetResultString(interp, buf, -1); + + return JIM_OK; +} + +#ifdef HAVE_STRPTIME +static time_t jim_timegm(const struct tm *tm) +{ + int m = tm->tm_mon + 1; + int y = 1900 + tm->tm_year - (m <= 2); + int era = (y >= 0 ? y : y - 399) / 400; + unsigned yoe = (unsigned)(y - era * 400); + unsigned doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + tm->tm_mday - 1; + unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + long days = (era * 146097 + (int)doe - 719468); + int secs = tm->tm_hour * 3600 + tm->tm_min * 60 + tm->tm_sec; + + return days * 24 * 60 * 60 + secs; +} + +static int clock_cmd_scan(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + char *pt; + struct tm tm; + time_t now = time(NULL); + + struct clock_options options = { 0, NULL }; + + if (argc % 2 == 0) { + return -1; + } + + if (parse_clock_options(interp, argc - 1, argv + 1, &options) == JIM_ERR) { + return JIM_ERR; + } + if (options.format == NULL) { + return -1; + } + + localtime_r(&now, &tm); + + pt = strptime(Jim_String(argv[0]), options.format, &tm); + if (pt == 0 || *pt != 0) { + Jim_SetResultString(interp, "Failed to parse time according to format", -1); + return JIM_ERR; + } + + + tm.tm_isdst = options.gmt ? 0 : -1; + Jim_SetResultInt(interp, options.gmt ? jim_timegm(&tm) : mktime(&tm)); + + return JIM_OK; +} +#endif + +static int clock_cmd_seconds(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_SetResultInt(interp, Jim_GetTimeUsec(CLOCK_REALTIME) / 1000000); + return JIM_OK; +} + +static int clock_cmd_clicks(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_SetResultInt(interp, Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW)); + return JIM_OK; +} + +static int clock_cmd_micros(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_SetResultInt(interp, Jim_GetTimeUsec(CLOCK_REALTIME)); + return JIM_OK; +} + +static int clock_cmd_millis(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_SetResultInt(interp, Jim_GetTimeUsec(CLOCK_REALTIME) / 1000); + return JIM_OK; +} + +static const jim_subcmd_type clock_command_table[] = { + { "clicks", + NULL, + clock_cmd_clicks, + 0, + 0, + + }, + { "format", + "seconds ?-format string? ?-gmt boolean?", + clock_cmd_format, + 1, + 5, + + }, + { "microseconds", + NULL, + clock_cmd_micros, + 0, + 0, + + }, + { "milliseconds", + NULL, + clock_cmd_millis, + 0, + 0, + + }, +#ifdef HAVE_STRPTIME + { "scan", + "str -format format ?-gmt boolean?", + clock_cmd_scan, + 3, + 5, + + }, +#endif + { "seconds", + NULL, + clock_cmd_seconds, + 0, + 0, + + }, + { NULL } +}; + +int Jim_clockInit(Jim_Interp *interp) +{ + Jim_PackageProvideCheck(interp, "clock"); + Jim_CreateCommand(interp, "clock", Jim_SubCmdProc, (void *)clock_command_table, NULL); + return JIM_OK; +} + +#include +#include +#include +#include +#include + + +static int array_cmd_exists(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + + Jim_Obj *dictObj = Jim_GetVariable(interp, argv[0], JIM_UNSHARED); + Jim_SetResultInt(interp, dictObj && Jim_DictSize(interp, dictObj) != -1); + return JIM_OK; +} + +static int array_cmd_get(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); + Jim_Obj *patternObj; + + if (!objPtr) { + return JIM_OK; + } + + patternObj = (argc == 1) ? NULL : argv[1]; + + + if (patternObj == NULL || Jim_CompareStringImmediate(interp, patternObj, "*")) { + if (Jim_IsList(objPtr) && Jim_ListLength(interp, objPtr) % 2 == 0) { + + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + } + + return Jim_DictMatchTypes(interp, objPtr, patternObj, JIM_DICTMATCH_KEYS, JIM_DICTMATCH_KEYS | JIM_DICTMATCH_VALUES); +} + +static int array_cmd_names(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); + + if (!objPtr) { + return JIM_OK; + } + + return Jim_DictMatchTypes(interp, objPtr, argc == 1 ? NULL : argv[1], JIM_DICTMATCH_KEYS, JIM_DICTMATCH_KEYS); +} + +static int array_cmd_unset(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i; + int len; + Jim_Obj *resultObj; + Jim_Obj *objPtr; + Jim_Obj **dictValuesObj; + + if (argc == 1 || Jim_CompareStringImmediate(interp, argv[1], "*")) { + + Jim_UnsetVariable(interp, argv[0], JIM_NONE); + return JIM_OK; + } + + objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); + + if (objPtr == NULL) { + + return JIM_OK; + } + + dictValuesObj = Jim_DictPairs(interp, objPtr, &len); + if (dictValuesObj == NULL) { + + Jim_SetResultString(interp, "", -1); + return JIM_OK; + } + + + resultObj = Jim_NewDictObj(interp, NULL, 0); + + for (i = 0; i < len; i += 2) { + if (!Jim_StringMatchObj(interp, argv[1], dictValuesObj[i], 0)) { + Jim_DictAddElement(interp, resultObj, dictValuesObj[i], dictValuesObj[i + 1]); + } + } + + Jim_SetVariable(interp, argv[0], resultObj); + return JIM_OK; +} + +static int array_cmd_size(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + int len = 0; + + + objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); + if (objPtr) { + len = Jim_DictSize(interp, objPtr); + if (len < 0) { + + Jim_SetResultInt(interp, 0); + return JIM_OK; + } + } + + Jim_SetResultInt(interp, len); + + return JIM_OK; +} + +static int array_cmd_stat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); + if (objPtr) { + return Jim_DictInfo(interp, objPtr); + } + Jim_SetResultFormatted(interp, "\"%#s\" isn't an array", argv[0], NULL); + return JIM_ERR; +} + +static int array_cmd_set(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i; + int len; + Jim_Obj *listObj = argv[1]; + Jim_Obj *dictObj; + + len = Jim_ListLength(interp, listObj); + if (len % 2) { + Jim_SetResultString(interp, "list must have an even number of elements", -1); + return JIM_ERR; + } + + dictObj = Jim_GetVariable(interp, argv[0], JIM_UNSHARED); + if (!dictObj) { + + return Jim_SetVariable(interp, argv[0], listObj); + } + else if (Jim_DictSize(interp, dictObj) < 0) { + return JIM_ERR; + } + + if (Jim_IsShared(dictObj)) { + dictObj = Jim_DuplicateObj(interp, dictObj); + } + + for (i = 0; i < len; i += 2) { + Jim_Obj *nameObj; + Jim_Obj *valueObj; + + Jim_ListIndex(interp, listObj, i, &nameObj, JIM_NONE); + Jim_ListIndex(interp, listObj, i + 1, &valueObj, JIM_NONE); + + Jim_DictAddElement(interp, dictObj, nameObj, valueObj); + } + return Jim_SetVariable(interp, argv[0], dictObj); +} + +static const jim_subcmd_type array_command_table[] = { + { "exists", + "arrayName", + array_cmd_exists, + 1, + 1, + + }, + { "get", + "arrayName ?pattern?", + array_cmd_get, + 1, + 2, + + }, + { "names", + "arrayName ?pattern?", + array_cmd_names, + 1, + 2, + + }, + { "set", + "arrayName list", + array_cmd_set, + 2, + 2, + + }, + { "size", + "arrayName", + array_cmd_size, + 1, + 1, + + }, + { "stat", + "arrayName", + array_cmd_stat, + 1, + 1, + + }, + { "unset", + "arrayName ?pattern?", + array_cmd_unset, + 1, + 2, + + }, + { NULL + } +}; + +int Jim_arrayInit(Jim_Interp *interp) +{ + Jim_PackageProvideCheck(interp, "array"); + Jim_CreateCommand(interp, "array", Jim_SubCmdProc, (void *)array_command_table, NULL); + return JIM_OK; +} +int Jim_InitStaticExtensions(Jim_Interp *interp) +{ +extern int Jim_bootstrapInit(Jim_Interp *); +extern int Jim_aioInit(Jim_Interp *); +extern int Jim_readdirInit(Jim_Interp *); +extern int Jim_regexpInit(Jim_Interp *); +extern int Jim_fileInit(Jim_Interp *); +extern int Jim_globInit(Jim_Interp *); +extern int Jim_execInit(Jim_Interp *); +extern int Jim_clockInit(Jim_Interp *); +extern int Jim_arrayInit(Jim_Interp *); +extern int Jim_stdlibInit(Jim_Interp *); +extern int Jim_tclcompatInit(Jim_Interp *); +Jim_bootstrapInit(interp); +Jim_aioInit(interp); +Jim_readdirInit(interp); +Jim_regexpInit(interp); +Jim_fileInit(interp); +Jim_globInit(interp); +Jim_execInit(interp); +Jim_clockInit(interp); +Jim_arrayInit(interp); +Jim_stdlibInit(interp); +Jim_tclcompatInit(interp); +return JIM_OK; +} +#ifndef JIM_TINY +#define JIM_OPTIMIZATION +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + + +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_EXECINFO_H +#include +#endif +#ifdef HAVE_CRT_EXTERNS_H +#include +#endif + + +#include + + + + + +#ifndef TCL_LIBRARY +#define TCL_LIBRARY "." +#endif +#ifndef TCL_PLATFORM_OS +#define TCL_PLATFORM_OS "unknown" +#endif +#ifndef TCL_PLATFORM_PLATFORM +#define TCL_PLATFORM_PLATFORM "unknown" +#endif +#ifndef TCL_PLATFORM_PATH_SEPARATOR +#define TCL_PLATFORM_PATH_SEPARATOR ":" +#endif + + + + + + + +#ifdef JIM_MAINTAINER +#define JIM_DEBUG_COMMAND +#define JIM_DEBUG_PANIC +#endif + + + +#define JIM_INTEGER_SPACE 24 + +#if defined(DEBUG_SHOW_SCRIPT) || defined(DEBUG_SHOW_SCRIPT_TOKENS) || defined(JIM_DEBUG_COMMAND) || defined(DEBUG_SHOW_SUBST) +static const char *jim_tt_name(int type); +#endif + +#ifdef JIM_DEBUG_PANIC +static void JimPanicDump(int fail_condition, const char *fmt, ...); +#define JimPanic(X) JimPanicDump X +#else +#define JimPanic(X) +#endif + +#ifdef JIM_OPTIMIZATION +static int JimIsWide(Jim_Obj *objPtr); +#define JIM_IF_OPTIM(X) X +#else +#define JIM_IF_OPTIM(X) +#endif + + +static char JimEmptyStringRep[] = ""; + +static void JimFreeCallFrame(Jim_Interp *interp, Jim_CallFrame *cf, int action); +static int ListSetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int listindex, Jim_Obj *newObjPtr, + int flags); +static int Jim_ListIndices(Jim_Interp *interp, Jim_Obj *listPtr, Jim_Obj *const *indexv, int indexc, + Jim_Obj **resultObj, int flags); +static int JimDeleteLocalProcs(Jim_Interp *interp, Jim_Stack *localCommands); +static Jim_Obj *JimExpandDictSugar(Jim_Interp *interp, Jim_Obj *objPtr); +static void SetDictSubstFromAny(Jim_Interp *interp, Jim_Obj *objPtr); +static void JimSetFailedEnumResult(Jim_Interp *interp, const char *arg, const char *badtype, + const char *prefix, const char *const *tablePtr, const char *name); +static int JimCallProcedure(Jim_Interp *interp, Jim_Cmd *cmd, int argc, Jim_Obj *const *argv); +static int JimGetWideNoErr(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr); +static int JimSign(jim_wide w); +static void JimPrngSeed(Jim_Interp *interp, unsigned char *seed, int seedLen); +static void JimRandomBytes(Jim_Interp *interp, void *dest, unsigned int len); +static int JimSetNewVariable(Jim_HashTable *ht, Jim_Obj *nameObjPtr, Jim_VarVal *vv); +static Jim_VarVal *JimFindVariable(Jim_HashTable *ht, Jim_Obj *nameObjPtr); +static int SetVariableFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); + +#define JIM_DICT_SUGAR 100 + + + + +#define JimWideValue(objPtr) (objPtr)->internalRep.wideValue + +#define JimObjTypeName(O) ((O)->typePtr ? (O)->typePtr->name : "none") + +static int utf8_tounicode_case(const char *s, int *uc, int upper) +{ + int l = utf8_tounicode(s, uc); + if (upper) { + *uc = utf8_upper(*uc); + } + return l; +} + +static Jim_Obj *JimPushInterpObjImpl(Jim_Obj **iop, Jim_Obj *no) +{ + Jim_Obj *io = *iop; + Jim_IncrRefCount(no); + *iop = no; + return io; +} + +#define JimPushInterpObj(IO, NO) JimPushInterpObjImpl(&(IO), NO) +#define JimPopInterpObj(I, IO, SO) do { Jim_DecrRefCount(I, IO); IO = SO; } while (0) + + +#define JIM_CHARSET_SCAN 2 +#define JIM_CHARSET_GLOB 0 + +static const char *JimCharsetMatch(const char *pattern, int plen, int c, int flags) +{ + int not = 0; + int pchar; + int match = 0; + int nocase = 0; + int n; + + if (flags & JIM_NOCASE) { + nocase++; + c = utf8_upper(c); + } + + if (flags & JIM_CHARSET_SCAN) { + if (*pattern == '^') { + not++; + pattern++; + plen--; + } + + + if (*pattern == ']') { + goto first; + } + } + + while (plen && *pattern != ']') { + + if (pattern[0] == '\\') { +first: + n = utf8_tounicode_case(pattern, &pchar, nocase); + pattern += n; + plen -= n; + } + else { + + int start; + int end; + + n = utf8_tounicode_case(pattern, &start, nocase); + pattern += n; + plen -= n; + if (pattern[0] == '-' && plen > 1) { + + n = 1 + utf8_tounicode_case(pattern + 1, &end, nocase); + pattern += n; + plen -= n; + + + if ((c >= start && c <= end) || (c >= end && c <= start)) { + match = 1; + } + continue; + } + pchar = start; + } + + if (pchar == c) { + match = 1; + } + } + if (not) { + match = !match; + } + + return match ? pattern : NULL; +} + + + +static int JimGlobMatch(const char *pattern, int plen, const char *string, int slen, int nocase) +{ + int c; + int pchar; + int n; + const char *p; + while (plen) { + switch (pattern[0]) { + case '*': + while (pattern[1] == '*' && plen) { + pattern++; + plen--; + } + pattern++; + plen--; + if (!plen) { + return 1; + } + while (slen) { + + if (JimGlobMatch(pattern, plen, string, slen, nocase)) + return 1; + n = utf8_tounicode(string, &c); + string += n; + slen -= n; + } + return 0; + + case '?': + n = utf8_tounicode(string, &c); + string += n; + slen -= n; + break; + + case '[': { + n = utf8_tounicode(string, &c); + string += n; + slen -= n; + p = JimCharsetMatch(pattern + 1, plen - 1, c, nocase ? JIM_NOCASE : 0); + if (!p) { + return 0; + } + plen -= p - pattern; + pattern = p; + + if (!plen) { + + continue; + } + break; + } + case '\\': + if (pattern[1]) { + pattern++; + plen--; + } + + default: + n = utf8_tounicode_case(string, &c, nocase); + string += n; + slen -= n; + utf8_tounicode_case(pattern, &pchar, nocase); + if (pchar != c) { + return 0; + } + break; + } + n = utf8_tounicode_case(pattern, &pchar, nocase); + pattern += n; + plen -= n; + if (!slen) { + while (*pattern == '*' && plen) { + pattern++; + plen--; + } + break; + } + } + if (!plen && !slen) { + return 1; + } + return 0; +} + +static int JimStringCompareUtf8(const char *s1, int l1, const char *s2, int l2, int nocase) +{ + int minlen = l1; + if (l2 < l1) { + minlen = l2; + } + while (minlen) { + int c1, c2; + s1 += utf8_tounicode_case(s1, &c1, nocase); + s2 += utf8_tounicode_case(s2, &c2, nocase); + if (c1 != c2) { + return JimSign(c1 - c2); + } + minlen--; + } + + if (l1 < l2) { + return -1; + } + if (l1 > l2) { + return 1; + } + return 0; +} + +static int JimStringFirst(const char *s1, int l1, const char *s2, int l2, int idx) +{ + int i; + int l1bytelen; + + if (!l1 || !l2 || l1 > l2) { + return -1; + } + if (idx < 0) + idx = 0; + s2 += utf8_index(s2, idx); + + l1bytelen = utf8_index(s1, l1); + + for (i = idx; i <= l2 - l1; i++) { + int c; + if (memcmp(s2, s1, l1bytelen) == 0) { + return i; + } + s2 += utf8_tounicode(s2, &c); + } + return -1; +} + +static int JimStringLast(const char *s1, int l1, const char *s2, int l2) +{ + const char *p; + + if (!l1 || !l2 || l1 > l2) + return -1; + + + for (p = s2 + l2 - 1; p != s2 - 1; p--) { + if (*p == *s1 && memcmp(s1, p, l1) == 0) { + return p - s2; + } + } + return -1; +} + +#ifdef JIM_UTF8 +static int JimStringLastUtf8(const char *s1, int l1, const char *s2, int l2) +{ + int n = JimStringLast(s1, utf8_index(s1, l1), s2, utf8_index(s2, l2)); + if (n > 0) { + n = utf8_strlen(s2, n); + } + return n; +} +#endif + +static int JimCheckConversion(const char *str, const char *endptr) +{ + if (str[0] == '\0' || str == endptr) { + return JIM_ERR; + } + + if (endptr[0] != '\0') { + while (*endptr) { + if (!isspace(UCHAR(*endptr))) { + return JIM_ERR; + } + endptr++; + } + } + return JIM_OK; +} + +static int JimNumberBase(const char *str, int *base, int *sign) +{ + int i = 0; + + *base = 0; + + while (isspace(UCHAR(str[i]))) { + i++; + } + + if (str[i] == '-') { + *sign = -1; + i++; + } + else { + if (str[i] == '+') { + i++; + } + *sign = 1; + } + + if (str[i] != '0') { + + return 0; + } + + + switch (str[i + 1]) { + case 'x': case 'X': *base = 16; break; + case 'o': case 'O': *base = 8; break; + case 'b': case 'B': *base = 2; break; + case 'd': case 'D': *base = 10; break; + default: return 0; + } + i += 2; + + if (str[i] != '-' && str[i] != '+' && !isspace(UCHAR(str[i]))) { + + return i; + } + + *base = 0; + return 0; +} + +static long jim_strtol(const char *str, char **endptr) +{ + int sign; + int base; + int i = JimNumberBase(str, &base, &sign); + + if (base != 0) { + long value = strtol(str + i, endptr, base); + if (endptr == NULL || *endptr != str + i) { + return value * sign; + } + } + + + return strtol(str, endptr, 10); +} + + +static jim_wide jim_strtoull(const char *str, char **endptr) +{ +#ifdef HAVE_LONG_LONG + int sign; + int base; + int i = JimNumberBase(str, &base, &sign); + + if (base != 0) { + jim_wide value = strtoull(str + i, endptr, base); + if (endptr == NULL || *endptr != str + i) { + return value * sign; + } + } + + + return strtoull(str, endptr, 10); +#else + return (unsigned long)jim_strtol(str, endptr); +#endif +} + +int Jim_StringToWide(const char *str, jim_wide * widePtr, int base) +{ + char *endptr; + + if (base) { + *widePtr = strtoull(str, &endptr, base); + } + else { + *widePtr = jim_strtoull(str, &endptr); + } + + return JimCheckConversion(str, endptr); +} + +int Jim_StringToDouble(const char *str, double *doublePtr) +{ + char *endptr; + + + errno = 0; + + *doublePtr = strtod(str, &endptr); + + return JimCheckConversion(str, endptr); +} + +static jim_wide JimPowWide(jim_wide b, jim_wide e) +{ + jim_wide res = 1; + + + if (b == 1) { + + return 1; + } + if (e < 0) { + if (b != -1) { + return 0; + } + e = -e; + } + while (e) + { + if (e & 1) { + res *= b; + } + e >>= 1; + b *= b; + } + return res; +} + +#ifdef JIM_DEBUG_PANIC +static void JimPanicDump(int condition, const char *fmt, ...) +{ + va_list ap; + + if (!condition) { + return; + } + + va_start(ap, fmt); + + fprintf(stderr, "\nJIM INTERPRETER PANIC: "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n\n"); + va_end(ap); + +#if defined(HAVE_BACKTRACE) + { + void *array[40]; + int size, i; + char **strings; + + size = backtrace(array, 40); + strings = backtrace_symbols(array, size); + for (i = 0; i < size; i++) + fprintf(stderr, "[backtrace] %s\n", strings[i]); + fprintf(stderr, "[backtrace] Include the above lines and the output\n"); + fprintf(stderr, "[backtrace] of 'nm ' in the bug report.\n"); + } +#endif + + exit(1); +} +#endif + + +void *JimDefaultAllocator(void *ptr, size_t size) +{ + if (size == 0) { + free(ptr); + return NULL; + } + else if (ptr) { + return realloc(ptr, size); + } + else { + return malloc(size); + } +} + +void *(*Jim_Allocator)(void *ptr, size_t size) = JimDefaultAllocator; + +char *Jim_StrDup(const char *s) +{ + return Jim_StrDupLen(s, strlen(s)); +} + +char *Jim_StrDupLen(const char *s, int l) +{ + char *copy = Jim_Alloc(l + 1); + + memcpy(copy, s, l); + copy[l] = 0; + return copy; +} + + +jim_wide Jim_GetTimeUsec(unsigned type) +{ + long long now; + struct timeval tv; + +#if defined(HAVE_CLOCK_GETTIME) + struct timespec ts; + + if (clock_gettime(type, &ts) == 0) { + now = ts.tv_sec * 1000000LL + ts.tv_nsec / 1000; + } + else +#endif + { + gettimeofday(&tv, NULL); + + now = tv.tv_sec * 1000000LL + tv.tv_usec; + } + + return now; +} + + + + + +static void JimExpandHashTableIfNeeded(Jim_HashTable *ht); +static unsigned int JimHashTableNextPower(unsigned int size); +static Jim_HashEntry *JimInsertHashEntry(Jim_HashTable *ht, const void *key, int replace); + + + + +unsigned int Jim_IntHashFunction(unsigned int key) +{ + key += ~(key << 15); + key ^= (key >> 10); + key += (key << 3); + key ^= (key >> 6); + key += ~(key << 11); + key ^= (key >> 16); + return key; +} + + +unsigned int Jim_GenHashFunction(const unsigned char *string, int length) +{ + unsigned result = 0; + string += length; + while (length--) { + result += (result << 3) + (unsigned char)(*--string); + } + return result; +} + + + +static void JimResetHashTable(Jim_HashTable *ht) +{ + ht->table = NULL; + ht->size = 0; + ht->sizemask = 0; + ht->used = 0; + ht->collisions = 0; +#ifdef JIM_RANDOMISE_HASH + ht->uniq = (rand() ^ time(NULL) ^ clock()); +#else + ht->uniq = 0; +#endif +} + +static void JimInitHashTableIterator(Jim_HashTable *ht, Jim_HashTableIterator *iter) +{ + iter->ht = ht; + iter->index = -1; + iter->entry = NULL; + iter->nextEntry = NULL; +} + + +int Jim_InitHashTable(Jim_HashTable *ht, const Jim_HashTableType *type, void *privDataPtr) +{ + JimResetHashTable(ht); + ht->type = type; + ht->privdata = privDataPtr; + return JIM_OK; +} + + +void Jim_ExpandHashTable(Jim_HashTable *ht, unsigned int size) +{ + Jim_HashTable n; + unsigned int realsize = JimHashTableNextPower(size), i; + + if (size <= ht->used) + return; + + Jim_InitHashTable(&n, ht->type, ht->privdata); + n.size = realsize; + n.sizemask = realsize - 1; + n.table = Jim_Alloc(realsize * sizeof(Jim_HashEntry *)); + + n.uniq = ht->uniq; + + + memset(n.table, 0, realsize * sizeof(Jim_HashEntry *)); + + n.used = ht->used; + for (i = 0; ht->used > 0; i++) { + Jim_HashEntry *he, *nextHe; + + if (ht->table[i] == NULL) + continue; + + + he = ht->table[i]; + while (he) { + unsigned int h; + + nextHe = he->next; + + h = Jim_HashKey(ht, he->key) & n.sizemask; + he->next = n.table[h]; + n.table[h] = he; + ht->used--; + + he = nextHe; + } + } + assert(ht->used == 0); + Jim_Free(ht->table); + + + *ht = n; +} + +int Jim_AddHashEntry(Jim_HashTable *ht, const void *key, void *val) +{ + Jim_HashEntry *entry = JimInsertHashEntry(ht, key, 0);; + if (entry == NULL) + return JIM_ERR; + + + Jim_SetHashKey(ht, entry, key); + Jim_SetHashVal(ht, entry, val); + return JIM_OK; +} + + +int Jim_ReplaceHashEntry(Jim_HashTable *ht, const void *key, void *val) +{ + int existed; + Jim_HashEntry *entry; + + entry = JimInsertHashEntry(ht, key, 1); + if (entry->key) { + if (ht->type->valDestructor && ht->type->valDup) { + void *newval = ht->type->valDup(ht->privdata, val); + ht->type->valDestructor(ht->privdata, entry->u.val); + entry->u.val = newval; + } + else { + Jim_FreeEntryVal(ht, entry); + Jim_SetHashVal(ht, entry, val); + } + existed = 1; + } + else { + + Jim_SetHashKey(ht, entry, key); + Jim_SetHashVal(ht, entry, val); + existed = 0; + } + + return existed; +} + +int Jim_DeleteHashEntry(Jim_HashTable *ht, const void *key) +{ + if (ht->used) { + unsigned int h = Jim_HashKey(ht, key) & ht->sizemask; + Jim_HashEntry *prevHe = NULL; + Jim_HashEntry *he = ht->table[h]; + + while (he) { + if (Jim_CompareHashKeys(ht, key, he->key)) { + + if (prevHe) + prevHe->next = he->next; + else + ht->table[h] = he->next; + ht->used--; + Jim_FreeEntryKey(ht, he); + Jim_FreeEntryVal(ht, he); + Jim_Free(he); + return JIM_OK; + } + prevHe = he; + he = he->next; + } + } + + return JIM_ERR; +} + +void Jim_ClearHashTable(Jim_HashTable *ht) +{ + unsigned int i; + + + for (i = 0; ht->used > 0; i++) { + Jim_HashEntry *he, *nextHe; + + he = ht->table[i]; + while (he) { + nextHe = he->next; + Jim_FreeEntryKey(ht, he); + Jim_FreeEntryVal(ht, he); + Jim_Free(he); + ht->used--; + he = nextHe; + } + ht->table[i] = NULL; + } +} + +int Jim_FreeHashTable(Jim_HashTable *ht) +{ + Jim_ClearHashTable(ht); + + Jim_Free(ht->table); + + JimResetHashTable(ht); + return JIM_OK; +} + +Jim_HashEntry *Jim_FindHashEntry(Jim_HashTable *ht, const void *key) +{ + Jim_HashEntry *he; + unsigned int h; + + if (ht->used == 0) + return NULL; + h = Jim_HashKey(ht, key) & ht->sizemask; + he = ht->table[h]; + while (he) { + if (Jim_CompareHashKeys(ht, key, he->key)) + return he; + he = he->next; + } + return NULL; +} + +Jim_HashTableIterator *Jim_GetHashTableIterator(Jim_HashTable *ht) +{ + Jim_HashTableIterator *iter = Jim_Alloc(sizeof(*iter)); + JimInitHashTableIterator(ht, iter); + return iter; +} + +Jim_HashEntry *Jim_NextHashEntry(Jim_HashTableIterator *iter) +{ + while (1) { + if (iter->entry == NULL) { + iter->index++; + if (iter->index >= (signed)iter->ht->size) + break; + iter->entry = iter->ht->table[iter->index]; + } + else { + iter->entry = iter->nextEntry; + } + if (iter->entry) { + iter->nextEntry = iter->entry->next; + return iter->entry; + } + } + return NULL; +} + + + + +static void JimExpandHashTableIfNeeded(Jim_HashTable *ht) +{ + if (ht->size == 0) + Jim_ExpandHashTable(ht, JIM_HT_INITIAL_SIZE); + if (ht->size == ht->used) + Jim_ExpandHashTable(ht, ht->size * 2); +} + + +static unsigned int JimHashTableNextPower(unsigned int size) +{ + unsigned int i = JIM_HT_INITIAL_SIZE; + + if (size >= 2147483648U) + return 2147483648U; + while (1) { + if (i >= size) + return i; + i *= 2; + } +} + +static Jim_HashEntry *JimInsertHashEntry(Jim_HashTable *ht, const void *key, int replace) +{ + unsigned int h; + Jim_HashEntry *he; + + + JimExpandHashTableIfNeeded(ht); + + + h = Jim_HashKey(ht, key) & ht->sizemask; + + he = ht->table[h]; + while (he) { + if (Jim_CompareHashKeys(ht, key, he->key)) + return replace ? he : NULL; + he = he->next; + } + + + he = Jim_Alloc(sizeof(*he)); + he->next = ht->table[h]; + ht->table[h] = he; + ht->used++; + he->key = NULL; + + return he; +} + + + +static unsigned int JimStringCopyHTHashFunction(const void *key) +{ + return Jim_GenHashFunction(key, strlen(key)); +} + +static void *JimStringCopyHTDup(void *privdata, const void *key) +{ + return Jim_StrDup(key); +} + +static int JimStringCopyHTKeyCompare(void *privdata, const void *key1, const void *key2) +{ + return strcmp(key1, key2) == 0; +} + +static void JimStringCopyHTKeyDestructor(void *privdata, void *key) +{ + Jim_Free(key); +} + +static const Jim_HashTableType JimPackageHashTableType = { + JimStringCopyHTHashFunction, + JimStringCopyHTDup, + NULL, + JimStringCopyHTKeyCompare, + JimStringCopyHTKeyDestructor, + NULL +}; + +typedef struct AssocDataValue +{ + Jim_InterpDeleteProc *delProc; + void *data; +} AssocDataValue; + +static void JimAssocDataHashTableValueDestructor(void *privdata, void *data) +{ + AssocDataValue *assocPtr = (AssocDataValue *) data; + + if (assocPtr->delProc != NULL) + assocPtr->delProc((Jim_Interp *)privdata, assocPtr->data); + Jim_Free(data); +} + +static const Jim_HashTableType JimAssocDataHashTableType = { + JimStringCopyHTHashFunction, + JimStringCopyHTDup, + NULL, + JimStringCopyHTKeyCompare, + JimStringCopyHTKeyDestructor, + JimAssocDataHashTableValueDestructor +}; + +void Jim_InitStack(Jim_Stack *stack) +{ + stack->len = 0; + stack->maxlen = 0; + stack->vector = NULL; +} + +void Jim_FreeStack(Jim_Stack *stack) +{ + Jim_Free(stack->vector); +} + +int Jim_StackLen(Jim_Stack *stack) +{ + return stack->len; +} + +void Jim_StackPush(Jim_Stack *stack, void *element) +{ + int neededLen = stack->len + 1; + + if (neededLen > stack->maxlen) { + stack->maxlen = neededLen < 20 ? 20 : neededLen * 2; + stack->vector = Jim_Realloc(stack->vector, sizeof(void *) * stack->maxlen); + } + stack->vector[stack->len] = element; + stack->len++; +} + +void *Jim_StackPop(Jim_Stack *stack) +{ + if (stack->len == 0) + return NULL; + stack->len--; + return stack->vector[stack->len]; +} + +void *Jim_StackPeek(Jim_Stack *stack) +{ + if (stack->len == 0) + return NULL; + return stack->vector[stack->len - 1]; +} + +void Jim_FreeStackElements(Jim_Stack *stack, void (*freeFunc) (void *ptr)) +{ + int i; + + for (i = 0; i < stack->len; i++) + freeFunc(stack->vector[i]); +} + + + +#define JIM_TT_NONE 0 +#define JIM_TT_STR 1 +#define JIM_TT_ESC 2 +#define JIM_TT_VAR 3 +#define JIM_TT_DICTSUGAR 4 +#define JIM_TT_CMD 5 + +#define JIM_TT_SEP 6 +#define JIM_TT_EOL 7 +#define JIM_TT_EOF 8 + +#define JIM_TT_LINE 9 +#define JIM_TT_WORD 10 + + +#define JIM_TT_SUBEXPR_START 11 +#define JIM_TT_SUBEXPR_END 12 +#define JIM_TT_SUBEXPR_COMMA 13 +#define JIM_TT_EXPR_INT 14 +#define JIM_TT_EXPR_DOUBLE 15 +#define JIM_TT_EXPR_BOOLEAN 16 + +#define JIM_TT_EXPRSUGAR 17 + + +#define JIM_TT_EXPR_OP 20 + +#define TOKEN_IS_SEP(type) (type >= JIM_TT_SEP && type <= JIM_TT_EOF) + +#define TOKEN_IS_EXPR_START(type) (type == JIM_TT_NONE || type == JIM_TT_SUBEXPR_START || type == JIM_TT_SUBEXPR_COMMA) + +#define TOKEN_IS_EXPR_OP(type) (type >= JIM_TT_EXPR_OP) + +struct JimParseMissing { + int ch; + int line; +}; + +struct JimParserCtx +{ + const char *p; + int len; + int linenr; + const char *tstart; + const char *tend; + int tline; + int tt; + int eof; + int inquote; + int comment; + struct JimParseMissing missing; + const char *errmsg; +}; + +static int JimParseScript(struct JimParserCtx *pc); +static int JimParseSep(struct JimParserCtx *pc); +static int JimParseEol(struct JimParserCtx *pc); +static int JimParseCmd(struct JimParserCtx *pc); +static int JimParseQuote(struct JimParserCtx *pc); +static int JimParseVar(struct JimParserCtx *pc); +static int JimParseBrace(struct JimParserCtx *pc); +static int JimParseStr(struct JimParserCtx *pc); +static int JimParseComment(struct JimParserCtx *pc); +static void JimParseSubCmd(struct JimParserCtx *pc); +static int JimParseSubQuote(struct JimParserCtx *pc); +static Jim_Obj *JimParserGetTokenObj(Jim_Interp *interp, struct JimParserCtx *pc); + +static void JimParserInit(struct JimParserCtx *pc, const char *prg, int len, int linenr) +{ + pc->p = prg; + pc->len = len; + pc->tstart = NULL; + pc->tend = NULL; + pc->tline = 0; + pc->tt = JIM_TT_NONE; + pc->eof = 0; + pc->inquote = 0; + pc->linenr = linenr; + pc->comment = 1; + pc->missing.ch = ' '; + pc->missing.line = linenr; +} + +static int JimParseScript(struct JimParserCtx *pc) +{ + while (1) { + if (!pc->len) { + pc->tstart = pc->p; + pc->tend = pc->p - 1; + pc->tline = pc->linenr; + pc->tt = JIM_TT_EOL; + if (pc->inquote) { + pc->missing.ch = '"'; + } + pc->eof = 1; + return JIM_OK; + } + switch (*(pc->p)) { + case '\\': + if (*(pc->p + 1) == '\n' && !pc->inquote) { + return JimParseSep(pc); + } + pc->comment = 0; + return JimParseStr(pc); + case ' ': + case '\t': + case '\r': + case '\f': + if (!pc->inquote) + return JimParseSep(pc); + pc->comment = 0; + return JimParseStr(pc); + case '\n': + case ';': + pc->comment = 1; + if (!pc->inquote) + return JimParseEol(pc); + return JimParseStr(pc); + case '[': + pc->comment = 0; + return JimParseCmd(pc); + case '$': + pc->comment = 0; + if (JimParseVar(pc) == JIM_ERR) { + + pc->tstart = pc->tend = pc->p++; + pc->len--; + pc->tt = JIM_TT_ESC; + } + return JIM_OK; + case '#': + if (pc->comment) { + JimParseComment(pc); + continue; + } + return JimParseStr(pc); + default: + pc->comment = 0; + return JimParseStr(pc); + } + return JIM_OK; + } +} + +static int JimParseSep(struct JimParserCtx *pc) +{ + pc->tstart = pc->p; + pc->tline = pc->linenr; + while (isspace(UCHAR(*pc->p)) || (*pc->p == '\\' && *(pc->p + 1) == '\n')) { + if (*pc->p == '\n') { + break; + } + if (*pc->p == '\\') { + pc->p++; + pc->len--; + pc->linenr++; + } + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + pc->tt = JIM_TT_SEP; + return JIM_OK; +} + +static int JimParseEol(struct JimParserCtx *pc) +{ + pc->tstart = pc->p; + pc->tline = pc->linenr; + while (isspace(UCHAR(*pc->p)) || *pc->p == ';') { + if (*pc->p == '\n') + pc->linenr++; + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + pc->tt = JIM_TT_EOL; + return JIM_OK; +} + + +static void JimParseSubBrace(struct JimParserCtx *pc) +{ + int level = 1; + + + pc->p++; + pc->len--; + while (pc->len) { + switch (*pc->p) { + case '\\': + if (pc->len > 1) { + if (*++pc->p == '\n') { + pc->linenr++; + } + pc->len--; + } + break; + + case '{': + level++; + break; + + case '}': + if (--level == 0) { + pc->tend = pc->p - 1; + pc->p++; + pc->len--; + return; + } + break; + + case '\n': + pc->linenr++; + break; + } + pc->p++; + pc->len--; + } + pc->missing.ch = '{'; + pc->missing.line = pc->tline; + pc->tend = pc->p - 1; +} + +static int JimParseSubQuote(struct JimParserCtx *pc) +{ + int tt = JIM_TT_STR; + int line = pc->tline; + + + pc->p++; + pc->len--; + while (pc->len) { + switch (*pc->p) { + case '\\': + if (pc->len > 1) { + if (*++pc->p == '\n') { + pc->linenr++; + } + pc->len--; + tt = JIM_TT_ESC; + } + break; + + case '"': + pc->tend = pc->p - 1; + pc->p++; + pc->len--; + return tt; + + case '[': + JimParseSubCmd(pc); + tt = JIM_TT_ESC; + continue; + + case '\n': + pc->linenr++; + break; + + case '$': + tt = JIM_TT_ESC; + break; + } + pc->p++; + pc->len--; + } + pc->missing.ch = '"'; + pc->missing.line = line; + pc->tend = pc->p - 1; + return tt; +} + +static void JimParseSubCmd(struct JimParserCtx *pc) +{ + int level = 1; + int startofword = 1; + int line = pc->tline; + + + pc->p++; + pc->len--; + while (pc->len) { + switch (*pc->p) { + case '\\': + if (pc->len > 1) { + if (*++pc->p == '\n') { + pc->linenr++; + } + pc->len--; + } + break; + + case '[': + level++; + break; + + case ']': + if (--level == 0) { + pc->tend = pc->p - 1; + pc->p++; + pc->len--; + return; + } + break; + + case '"': + if (startofword) { + JimParseSubQuote(pc); + if (pc->missing.ch == '"') { + return; + } + continue; + } + break; + + case '{': + JimParseSubBrace(pc); + startofword = 0; + continue; + + case '\n': + pc->linenr++; + break; + } + startofword = isspace(UCHAR(*pc->p)); + pc->p++; + pc->len--; + } + pc->missing.ch = '['; + pc->missing.line = line; + pc->tend = pc->p - 1; +} + +static int JimParseBrace(struct JimParserCtx *pc) +{ + pc->tstart = pc->p + 1; + pc->tline = pc->linenr; + pc->tt = JIM_TT_STR; + JimParseSubBrace(pc); + return JIM_OK; +} + +static int JimParseCmd(struct JimParserCtx *pc) +{ + pc->tstart = pc->p + 1; + pc->tline = pc->linenr; + pc->tt = JIM_TT_CMD; + JimParseSubCmd(pc); + return JIM_OK; +} + +static int JimParseQuote(struct JimParserCtx *pc) +{ + pc->tstart = pc->p + 1; + pc->tline = pc->linenr; + pc->tt = JimParseSubQuote(pc); + return JIM_OK; +} + +static int JimParseVar(struct JimParserCtx *pc) +{ + + pc->p++; + pc->len--; + +#ifdef EXPRSUGAR_BRACKET + if (*pc->p == '[') { + + JimParseCmd(pc); + pc->tt = JIM_TT_EXPRSUGAR; + return JIM_OK; + } +#endif + + pc->tstart = pc->p; + pc->tt = JIM_TT_VAR; + pc->tline = pc->linenr; + + if (*pc->p == '{') { + pc->tstart = ++pc->p; + pc->len--; + + while (pc->len && *pc->p != '}') { + if (*pc->p == '\n') { + pc->linenr++; + } + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + if (pc->len) { + pc->p++; + pc->len--; + } + } + else { + while (1) { + + if (pc->p[0] == ':' && pc->p[1] == ':') { + while (*pc->p == ':') { + pc->p++; + pc->len--; + } + continue; + } + if (isalnum(UCHAR(*pc->p)) || *pc->p == '_' || UCHAR(*pc->p) >= 0x80) { + pc->p++; + pc->len--; + continue; + } + break; + } + + if (*pc->p == '(') { + int count = 1; + const char *paren = NULL; + + pc->tt = JIM_TT_DICTSUGAR; + + while (count && pc->len) { + pc->p++; + pc->len--; + if (*pc->p == '\\' && pc->len >= 1) { + pc->p++; + pc->len--; + } + else if (*pc->p == '(') { + count++; + } + else if (*pc->p == ')') { + paren = pc->p; + count--; + } + } + if (count == 0) { + pc->p++; + pc->len--; + } + else if (paren) { + + paren++; + pc->len += (pc->p - paren); + pc->p = paren; + } +#ifndef EXPRSUGAR_BRACKET + if (*pc->tstart == '(') { + pc->tt = JIM_TT_EXPRSUGAR; + } +#endif + } + pc->tend = pc->p - 1; + } + if (pc->tstart == pc->p) { + pc->p--; + pc->len++; + return JIM_ERR; + } + return JIM_OK; +} + +static int JimParseStr(struct JimParserCtx *pc) +{ + if (pc->tt == JIM_TT_SEP || pc->tt == JIM_TT_EOL || + pc->tt == JIM_TT_NONE || pc->tt == JIM_TT_STR) { + + if (*pc->p == '{') { + return JimParseBrace(pc); + } + if (*pc->p == '"') { + pc->inquote = 1; + pc->p++; + pc->len--; + + pc->missing.line = pc->tline; + } + } + pc->tstart = pc->p; + pc->tline = pc->linenr; + while (1) { + if (pc->len == 0) { + if (pc->inquote) { + pc->missing.ch = '"'; + } + pc->tend = pc->p - 1; + pc->tt = JIM_TT_ESC; + return JIM_OK; + } + switch (*pc->p) { + case '\\': + if (!pc->inquote && *(pc->p + 1) == '\n') { + pc->tend = pc->p - 1; + pc->tt = JIM_TT_ESC; + return JIM_OK; + } + if (pc->len >= 2) { + if (*(pc->p + 1) == '\n') { + pc->linenr++; + } + pc->p++; + pc->len--; + } + else if (pc->len == 1) { + + pc->missing.ch = '\\'; + } + break; + case '(': + + if (pc->len > 1 && pc->p[1] != '$') { + break; + } + + case ')': + + if (*pc->p == '(' || pc->tt == JIM_TT_VAR) { + if (pc->p == pc->tstart) { + + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + pc->tt = JIM_TT_ESC; + return JIM_OK; + } + break; + + case '$': + case '[': + pc->tend = pc->p - 1; + pc->tt = JIM_TT_ESC; + return JIM_OK; + case ' ': + case '\t': + case '\n': + case '\r': + case '\f': + case ';': + if (!pc->inquote) { + pc->tend = pc->p - 1; + pc->tt = JIM_TT_ESC; + return JIM_OK; + } + else if (*pc->p == '\n') { + pc->linenr++; + } + break; + case '"': + if (pc->inquote) { + pc->tend = pc->p - 1; + pc->tt = JIM_TT_ESC; + pc->p++; + pc->len--; + pc->inquote = 0; + return JIM_OK; + } + break; + } + pc->p++; + pc->len--; + } + return JIM_OK; +} + +static int JimParseComment(struct JimParserCtx *pc) +{ + while (*pc->p) { + if (*pc->p == '\\') { + pc->p++; + pc->len--; + if (pc->len == 0) { + pc->missing.ch = '\\'; + return JIM_OK; + } + if (*pc->p == '\n') { + pc->linenr++; + } + } + else if (*pc->p == '\n') { + pc->p++; + pc->len--; + pc->linenr++; + break; + } + pc->p++; + pc->len--; + } + return JIM_OK; +} + + +static int xdigitval(int c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; +} + +static int odigitval(int c) +{ + if (c >= '0' && c <= '7') + return c - '0'; + return -1; +} + +static int JimEscape(char *dest, const char *s, int slen) +{ + char *p = dest; + int i, len; + + for (i = 0; i < slen; i++) { + switch (s[i]) { + case '\\': + switch (s[i + 1]) { + case 'a': + *p++ = 0x7; + i++; + break; + case 'b': + *p++ = 0x8; + i++; + break; + case 'f': + *p++ = 0xc; + i++; + break; + case 'n': + *p++ = 0xa; + i++; + break; + case 'r': + *p++ = 0xd; + i++; + break; + case 't': + *p++ = 0x9; + i++; + break; + case 'u': + case 'U': + case 'x': + { + unsigned val = 0; + int k; + int maxchars = 2; + + i++; + + if (s[i] == 'U') { + maxchars = 8; + } + else if (s[i] == 'u') { + if (s[i + 1] == '{') { + maxchars = 6; + i++; + } + else { + maxchars = 4; + } + } + + for (k = 0; k < maxchars; k++) { + int c = xdigitval(s[i + k + 1]); + if (c == -1) { + break; + } + val = (val << 4) | c; + } + + if (s[i] == '{') { + if (k == 0 || val > 0x1fffff || s[i + k + 1] != '}') { + + i--; + k = 0; + } + else { + + k++; + } + } + if (k) { + + if (s[i] == 'x') { + *p++ = val; + } + else { + p += utf8_fromunicode(p, val); + } + i += k; + break; + } + + *p++ = s[i]; + } + break; + case 'v': + *p++ = 0xb; + i++; + break; + case '\0': + *p++ = '\\'; + i++; + break; + case '\n': + + *p++ = ' '; + do { + i++; + } while (s[i + 1] == ' ' || s[i + 1] == '\t'); + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + + { + int val = 0; + int c = odigitval(s[i + 1]); + + val = c; + c = odigitval(s[i + 2]); + if (c == -1) { + *p++ = val; + i++; + break; + } + val = (val * 8) + c; + c = odigitval(s[i + 3]); + if (c == -1) { + *p++ = val; + i += 2; + break; + } + val = (val * 8) + c; + *p++ = val; + i += 3; + } + break; + default: + *p++ = s[i + 1]; + i++; + break; + } + break; + default: + *p++ = s[i]; + break; + } + } + len = p - dest; + *p = '\0'; + return len; +} + +static Jim_Obj *JimParserGetTokenObj(Jim_Interp *interp, struct JimParserCtx *pc) +{ + const char *start, *end; + char *token; + int len; + + start = pc->tstart; + end = pc->tend; + len = (end - start) + 1; + if (len < 0) { + len = 0; + } + token = Jim_Alloc(len + 1); + if (pc->tt != JIM_TT_ESC) { + + memcpy(token, start, len); + token[len] = '\0'; + } + else { + + len = JimEscape(token, start, len); + } + + return Jim_NewStringObjNoAlloc(interp, token, len); +} + +static int JimParseListSep(struct JimParserCtx *pc); +static int JimParseListStr(struct JimParserCtx *pc); +static int JimParseListQuote(struct JimParserCtx *pc); + +static int JimParseList(struct JimParserCtx *pc) +{ + if (isspace(UCHAR(*pc->p))) { + return JimParseListSep(pc); + } + switch (*pc->p) { + case '"': + return JimParseListQuote(pc); + + case '{': + return JimParseBrace(pc); + + default: + if (pc->len) { + return JimParseListStr(pc); + } + break; + } + + pc->tstart = pc->tend = pc->p; + pc->tline = pc->linenr; + pc->tt = JIM_TT_EOL; + pc->eof = 1; + return JIM_OK; +} + +static int JimParseListSep(struct JimParserCtx *pc) +{ + pc->tstart = pc->p; + pc->tline = pc->linenr; + while (isspace(UCHAR(*pc->p))) { + if (*pc->p == '\n') { + pc->linenr++; + } + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + pc->tt = JIM_TT_SEP; + return JIM_OK; +} + +static int JimParseListQuote(struct JimParserCtx *pc) +{ + pc->p++; + pc->len--; + + pc->tstart = pc->p; + pc->tline = pc->linenr; + pc->tt = JIM_TT_STR; + + while (pc->len) { + switch (*pc->p) { + case '\\': + pc->tt = JIM_TT_ESC; + if (--pc->len == 0) { + + pc->tend = pc->p; + return JIM_OK; + } + pc->p++; + break; + case '\n': + pc->linenr++; + break; + case '"': + pc->tend = pc->p - 1; + pc->p++; + pc->len--; + return JIM_OK; + } + pc->p++; + pc->len--; + } + + pc->tend = pc->p - 1; + return JIM_OK; +} + +static int JimParseListStr(struct JimParserCtx *pc) +{ + pc->tstart = pc->p; + pc->tline = pc->linenr; + pc->tt = JIM_TT_STR; + + while (pc->len) { + if (isspace(UCHAR(*pc->p))) { + pc->tend = pc->p - 1; + return JIM_OK; + } + if (*pc->p == '\\') { + if (--pc->len == 0) { + + pc->tend = pc->p; + return JIM_OK; + } + pc->tt = JIM_TT_ESC; + pc->p++; + } + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + return JIM_OK; +} + + + +Jim_Obj *Jim_NewObj(Jim_Interp *interp) +{ + Jim_Obj *objPtr; + + + if (interp->freeList != NULL) { + + objPtr = interp->freeList; + interp->freeList = objPtr->nextObjPtr; + } + else { + + objPtr = Jim_Alloc(sizeof(*objPtr)); + } + + objPtr->refCount = 0; + + + objPtr->prevObjPtr = NULL; + objPtr->nextObjPtr = interp->liveList; + if (interp->liveList) + interp->liveList->prevObjPtr = objPtr; + interp->liveList = objPtr; + + return objPtr; +} + +void Jim_FreeObj(Jim_Interp *interp, Jim_Obj *objPtr) +{ + + JimPanic((objPtr->refCount != 0, "!!!Object %p freed with bad refcount %d, type=%s", objPtr, + objPtr->refCount, objPtr->typePtr ? objPtr->typePtr->name : "")); + + + Jim_FreeIntRep(interp, objPtr); + + if (objPtr->bytes != NULL) { + if (objPtr->bytes != JimEmptyStringRep) + Jim_Free(objPtr->bytes); + } + + if (objPtr->prevObjPtr) + objPtr->prevObjPtr->nextObjPtr = objPtr->nextObjPtr; + if (objPtr->nextObjPtr) + objPtr->nextObjPtr->prevObjPtr = objPtr->prevObjPtr; + if (interp->liveList == objPtr) + interp->liveList = objPtr->nextObjPtr; +#ifdef JIM_DISABLE_OBJECT_POOL + Jim_Free(objPtr); +#else + + objPtr->prevObjPtr = NULL; + objPtr->nextObjPtr = interp->freeList; + if (interp->freeList) + interp->freeList->prevObjPtr = objPtr; + interp->freeList = objPtr; + objPtr->refCount = -1; +#endif +} + + +void Jim_InvalidateStringRep(Jim_Obj *objPtr) +{ + if (objPtr->bytes != NULL) { + if (objPtr->bytes != JimEmptyStringRep) + Jim_Free(objPtr->bytes); + } + objPtr->bytes = NULL; +} + + +Jim_Obj *Jim_DuplicateObj(Jim_Interp *interp, Jim_Obj *objPtr) +{ + Jim_Obj *dupPtr; + + dupPtr = Jim_NewObj(interp); + if (objPtr->bytes == NULL) { + + dupPtr->bytes = NULL; + } + else if (objPtr->length == 0) { + dupPtr->bytes = JimEmptyStringRep; + dupPtr->length = 0; + dupPtr->typePtr = NULL; + return dupPtr; + } + else { + dupPtr->bytes = Jim_Alloc(objPtr->length + 1); + dupPtr->length = objPtr->length; + + memcpy(dupPtr->bytes, objPtr->bytes, objPtr->length + 1); + } + + + dupPtr->typePtr = objPtr->typePtr; + if (objPtr->typePtr != NULL) { + if (objPtr->typePtr->dupIntRepProc == NULL) { + dupPtr->internalRep = objPtr->internalRep; + } + else { + + objPtr->typePtr->dupIntRepProc(interp, objPtr, dupPtr); + } + } + return dupPtr; +} + +const char *Jim_GetString(Jim_Obj *objPtr, int *lenPtr) +{ + if (objPtr->bytes == NULL) { + + JimPanic((objPtr->typePtr->updateStringProc == NULL, "UpdateStringProc called against '%s' type.", objPtr->typePtr->name)); + objPtr->typePtr->updateStringProc(objPtr); + } + if (lenPtr) + *lenPtr = objPtr->length; + return objPtr->bytes; +} + + +int Jim_Length(Jim_Obj *objPtr) +{ + if (objPtr->bytes == NULL) { + + Jim_GetString(objPtr, NULL); + } + return objPtr->length; +} + + +const char *Jim_String(Jim_Obj *objPtr) +{ + if (objPtr->bytes == NULL) { + + Jim_GetString(objPtr, NULL); + } + return objPtr->bytes; +} + +static void JimSetStringBytes(Jim_Obj *objPtr, const char *str) +{ + objPtr->bytes = Jim_StrDup(str); + objPtr->length = strlen(str); +} + +static void FreeDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); + +static const Jim_ObjType dictSubstObjType = { + "dict-substitution", + FreeDictSubstInternalRep, + DupDictSubstInternalRep, + NULL, + JIM_TYPE_NONE, +}; + +static void FreeInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); + +static const Jim_ObjType interpolatedObjType = { + "interpolated", + FreeInterpolatedInternalRep, + DupInterpolatedInternalRep, + NULL, + JIM_TYPE_NONE, +}; + +static void FreeInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + Jim_DecrRefCount(interp, objPtr->internalRep.dictSubstValue.indexObjPtr); +} + +static void DupInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + + dupPtr->internalRep = srcPtr->internalRep; + + Jim_IncrRefCount(dupPtr->internalRep.dictSubstValue.indexObjPtr); +} + +static void DupStringInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); +static int SetStringFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); + +static const Jim_ObjType stringObjType = { + "string", + NULL, + DupStringInternalRep, + NULL, + JIM_TYPE_REFERENCES, +}; + +static void DupStringInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + JIM_NOTUSED(interp); + + dupPtr->internalRep.strValue.maxLength = srcPtr->length; + dupPtr->internalRep.strValue.charLength = srcPtr->internalRep.strValue.charLength; +} + +static int SetStringFromAny(Jim_Interp *interp, Jim_Obj *objPtr) +{ + if (objPtr->typePtr != &stringObjType) { + + if (objPtr->bytes == NULL) { + + JimPanic((objPtr->typePtr->updateStringProc == NULL, "UpdateStringProc called against '%s' type.", objPtr->typePtr->name)); + objPtr->typePtr->updateStringProc(objPtr); + } + + Jim_FreeIntRep(interp, objPtr); + + objPtr->typePtr = &stringObjType; + objPtr->internalRep.strValue.maxLength = objPtr->length; + + objPtr->internalRep.strValue.charLength = -1; + } + return JIM_OK; +} + +int Jim_Utf8Length(Jim_Interp *interp, Jim_Obj *objPtr) +{ +#ifdef JIM_UTF8 + SetStringFromAny(interp, objPtr); + + if (objPtr->internalRep.strValue.charLength < 0) { + objPtr->internalRep.strValue.charLength = utf8_strlen(objPtr->bytes, objPtr->length); + } + return objPtr->internalRep.strValue.charLength; +#else + return Jim_Length(objPtr); +#endif +} + + +Jim_Obj *Jim_NewStringObj(Jim_Interp *interp, const char *s, int len) +{ + Jim_Obj *objPtr = Jim_NewObj(interp); + + + if (len == -1) + len = strlen(s); + + if (len == 0) { + objPtr->bytes = JimEmptyStringRep; + } + else { + objPtr->bytes = Jim_StrDupLen(s, len); + } + objPtr->length = len; + + + objPtr->typePtr = NULL; + return objPtr; +} + + +Jim_Obj *Jim_NewStringObjUtf8(Jim_Interp *interp, const char *s, int charlen) +{ +#ifdef JIM_UTF8 + + int bytelen = utf8_index(s, charlen); + + Jim_Obj *objPtr = Jim_NewStringObj(interp, s, bytelen); + + + objPtr->typePtr = &stringObjType; + objPtr->internalRep.strValue.maxLength = bytelen; + objPtr->internalRep.strValue.charLength = charlen; + + return objPtr; +#else + return Jim_NewStringObj(interp, s, charlen); +#endif +} + +Jim_Obj *Jim_NewStringObjNoAlloc(Jim_Interp *interp, char *s, int len) +{ + Jim_Obj *objPtr = Jim_NewObj(interp); + + objPtr->bytes = s; + objPtr->length = (len == -1) ? strlen(s) : len; + objPtr->typePtr = NULL; + return objPtr; +} + +static void StringAppendString(Jim_Obj *objPtr, const char *str, int len) +{ + int needlen; + + if (len == -1) + len = strlen(str); + needlen = objPtr->length + len; + if (objPtr->internalRep.strValue.maxLength < needlen || + objPtr->internalRep.strValue.maxLength == 0) { + needlen *= 2; + + if (needlen < 7) { + needlen = 7; + } + if (objPtr->bytes == JimEmptyStringRep) { + objPtr->bytes = Jim_Alloc(needlen + 1); + } + else { + objPtr->bytes = Jim_Realloc(objPtr->bytes, needlen + 1); + } + objPtr->internalRep.strValue.maxLength = needlen; + } + memcpy(objPtr->bytes + objPtr->length, str, len); + objPtr->bytes[objPtr->length + len] = '\0'; + + if (objPtr->internalRep.strValue.charLength >= 0) { + + objPtr->internalRep.strValue.charLength += utf8_strlen(objPtr->bytes + objPtr->length, len); + } + objPtr->length += len; +} + +void Jim_AppendString(Jim_Interp *interp, Jim_Obj *objPtr, const char *str, int len) +{ + JimPanic((Jim_IsShared(objPtr), "Jim_AppendString called with shared object")); + SetStringFromAny(interp, objPtr); + StringAppendString(objPtr, str, len); +} + +void Jim_AppendObj(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *appendObjPtr) +{ + int len; + const char *str = Jim_GetString(appendObjPtr, &len); + Jim_AppendString(interp, objPtr, str, len); +} + +void Jim_AppendStrings(Jim_Interp *interp, Jim_Obj *objPtr, ...) +{ + va_list ap; + + SetStringFromAny(interp, objPtr); + va_start(ap, objPtr); + while (1) { + const char *s = va_arg(ap, const char *); + + if (s == NULL) + break; + Jim_AppendString(interp, objPtr, s, -1); + } + va_end(ap); +} + +int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr) +{ + if (aObjPtr == bObjPtr) { + return 1; + } + else { + int Alen, Blen; + const char *sA = Jim_GetString(aObjPtr, &Alen); + const char *sB = Jim_GetString(bObjPtr, &Blen); + + return Alen == Blen && memcmp(sA, sB, Alen) == 0; + } +} + +int Jim_StringMatchObj(Jim_Interp *interp, Jim_Obj *patternObjPtr, Jim_Obj *objPtr, int nocase) +{ + int plen, slen; + const char *pattern = Jim_GetString(patternObjPtr, &plen); + const char *string = Jim_GetString(objPtr, &slen); + return JimGlobMatch(pattern, plen, string, slen, nocase); +} + +int Jim_StringCompareObj(Jim_Interp *interp, Jim_Obj *firstObjPtr, Jim_Obj *secondObjPtr, int nocase) +{ + const char *s1 = Jim_String(firstObjPtr); + int l1 = Jim_Utf8Length(interp, firstObjPtr); + const char *s2 = Jim_String(secondObjPtr); + int l2 = Jim_Utf8Length(interp, secondObjPtr); + return JimStringCompareUtf8(s1, l1, s2, l2, nocase); +} + +static int JimRelToAbsIndex(int len, int idx) +{ + if (idx < 0 && idx > -INT_MAX) + return len + idx; + return idx; +} + +static void JimRelToAbsRange(int len, int *firstPtr, int *lastPtr, int *rangeLenPtr) +{ + int rangeLen; + + if (*firstPtr > *lastPtr) { + rangeLen = 0; + } + else { + rangeLen = *lastPtr - *firstPtr + 1; + if (rangeLen) { + if (*firstPtr < 0) { + rangeLen += *firstPtr; + *firstPtr = 0; + } + if (*lastPtr >= len) { + rangeLen -= (*lastPtr - (len - 1)); + *lastPtr = len - 1; + } + } + } + if (rangeLen < 0) + rangeLen = 0; + + *rangeLenPtr = rangeLen; +} + +static int JimStringGetRange(Jim_Interp *interp, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr, + int len, int *first, int *last, int *range) +{ + if (Jim_GetIndex(interp, firstObjPtr, first) != JIM_OK) { + return JIM_ERR; + } + if (Jim_GetIndex(interp, lastObjPtr, last) != JIM_OK) { + return JIM_ERR; + } + *first = JimRelToAbsIndex(len, *first); + *last = JimRelToAbsIndex(len, *last); + JimRelToAbsRange(len, first, last, range); + return JIM_OK; +} + +Jim_Obj *Jim_StringByteRangeObj(Jim_Interp *interp, + Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr) +{ + int first, last; + const char *str; + int rangeLen; + int bytelen; + + str = Jim_GetString(strObjPtr, &bytelen); + + if (JimStringGetRange(interp, firstObjPtr, lastObjPtr, bytelen, &first, &last, &rangeLen) != JIM_OK) { + return NULL; + } + + if (first == 0 && rangeLen == bytelen) { + return strObjPtr; + } + return Jim_NewStringObj(interp, str + first, rangeLen); +} + +Jim_Obj *Jim_StringRangeObj(Jim_Interp *interp, + Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr) +{ +#ifdef JIM_UTF8 + int first, last; + const char *str; + int len, rangeLen; + int bytelen; + + str = Jim_GetString(strObjPtr, &bytelen); + len = Jim_Utf8Length(interp, strObjPtr); + + if (JimStringGetRange(interp, firstObjPtr, lastObjPtr, len, &first, &last, &rangeLen) != JIM_OK) { + return NULL; + } + + if (first == 0 && rangeLen == len) { + return strObjPtr; + } + if (len == bytelen) { + + return Jim_NewStringObj(interp, str + first, rangeLen); + } + return Jim_NewStringObjUtf8(interp, str + utf8_index(str, first), rangeLen); +#else + return Jim_StringByteRangeObj(interp, strObjPtr, firstObjPtr, lastObjPtr); +#endif +} + +Jim_Obj *JimStringReplaceObj(Jim_Interp *interp, + Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr, Jim_Obj *newStrObj) +{ + int first, last; + const char *str; + int len, rangeLen; + Jim_Obj *objPtr; + + len = Jim_Utf8Length(interp, strObjPtr); + + if (JimStringGetRange(interp, firstObjPtr, lastObjPtr, len, &first, &last, &rangeLen) != JIM_OK) { + return NULL; + } + + if (last < first) { + return strObjPtr; + } + + str = Jim_String(strObjPtr); + + + objPtr = Jim_NewStringObjUtf8(interp, str, first); + + + if (newStrObj) { + Jim_AppendObj(interp, objPtr, newStrObj); + } + + + Jim_AppendString(interp, objPtr, str + utf8_index(str, last + 1), len - last - 1); + + return objPtr; +} + +static void JimStrCopyUpperLower(char *dest, const char *str, int uc) +{ + while (*str) { + int c; + str += utf8_tounicode(str, &c); + dest += utf8_getchars(dest, uc ? utf8_upper(c) : utf8_lower(c)); + } + *dest = 0; +} + +static Jim_Obj *JimStringToLower(Jim_Interp *interp, Jim_Obj *strObjPtr) +{ + char *buf; + int len; + const char *str; + + str = Jim_GetString(strObjPtr, &len); + +#ifdef JIM_UTF8 + len *= 2; +#endif + buf = Jim_Alloc(len + 1); + JimStrCopyUpperLower(buf, str, 0); + return Jim_NewStringObjNoAlloc(interp, buf, -1); +} + +static Jim_Obj *JimStringToUpper(Jim_Interp *interp, Jim_Obj *strObjPtr) +{ + char *buf; + const char *str; + int len; + + str = Jim_GetString(strObjPtr, &len); + +#ifdef JIM_UTF8 + len *= 2; +#endif + buf = Jim_Alloc(len + 1); + JimStrCopyUpperLower(buf, str, 1); + return Jim_NewStringObjNoAlloc(interp, buf, -1); +} + +static Jim_Obj *JimStringToTitle(Jim_Interp *interp, Jim_Obj *strObjPtr) +{ + char *buf, *p; + int len; + int c; + const char *str; + + str = Jim_GetString(strObjPtr, &len); + +#ifdef JIM_UTF8 + len *= 2; +#endif + buf = p = Jim_Alloc(len + 1); + + str += utf8_tounicode(str, &c); + p += utf8_getchars(p, utf8_title(c)); + + JimStrCopyUpperLower(p, str, 0); + + return Jim_NewStringObjNoAlloc(interp, buf, -1); +} + +static const char *utf8_memchr(const char *str, int len, int c) +{ +#ifdef JIM_UTF8 + while (len) { + int sc; + int n = utf8_tounicode(str, &sc); + if (sc == c) { + return str; + } + str += n; + len -= n; + } + return NULL; +#else + return memchr(str, c, len); +#endif +} + +static const char *JimFindTrimLeft(const char *str, int len, const char *trimchars, int trimlen) +{ + while (len) { + int c; + int n = utf8_tounicode(str, &c); + + if (utf8_memchr(trimchars, trimlen, c) == NULL) { + + break; + } + str += n; + len -= n; + } + return str; +} + +static const char *JimFindTrimRight(const char *str, int len, const char *trimchars, int trimlen) +{ + str += len; + + while (len) { + int c; + int n = utf8_prev_len(str, len); + + len -= n; + str -= n; + + n = utf8_tounicode(str, &c); + + if (utf8_memchr(trimchars, trimlen, c) == NULL) { + return str + n; + } + } + + return NULL; +} + +static const char default_trim_chars[] = " \t\n\r"; + +static int default_trim_chars_len = sizeof(default_trim_chars); + +static Jim_Obj *JimStringTrimLeft(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *trimcharsObjPtr) +{ + int len; + const char *str = Jim_GetString(strObjPtr, &len); + const char *trimchars = default_trim_chars; + int trimcharslen = default_trim_chars_len; + const char *newstr; + + if (trimcharsObjPtr) { + trimchars = Jim_GetString(trimcharsObjPtr, &trimcharslen); + } + + newstr = JimFindTrimLeft(str, len, trimchars, trimcharslen); + if (newstr == str) { + return strObjPtr; + } + + return Jim_NewStringObj(interp, newstr, len - (newstr - str)); +} + +static Jim_Obj *JimStringTrimRight(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *trimcharsObjPtr) +{ + int len; + const char *trimchars = default_trim_chars; + int trimcharslen = default_trim_chars_len; + const char *nontrim; + + if (trimcharsObjPtr) { + trimchars = Jim_GetString(trimcharsObjPtr, &trimcharslen); + } + + SetStringFromAny(interp, strObjPtr); + + len = Jim_Length(strObjPtr); + nontrim = JimFindTrimRight(strObjPtr->bytes, len, trimchars, trimcharslen); + + if (nontrim == NULL) { + + return Jim_NewEmptyStringObj(interp); + } + if (nontrim == strObjPtr->bytes + len) { + + return strObjPtr; + } + + if (Jim_IsShared(strObjPtr)) { + strObjPtr = Jim_NewStringObj(interp, strObjPtr->bytes, (nontrim - strObjPtr->bytes)); + } + else { + + strObjPtr->bytes[nontrim - strObjPtr->bytes] = 0; + strObjPtr->length = (nontrim - strObjPtr->bytes); + } + + return strObjPtr; +} + +static Jim_Obj *JimStringTrim(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *trimcharsObjPtr) +{ + + Jim_Obj *objPtr = JimStringTrimLeft(interp, strObjPtr, trimcharsObjPtr); + + + strObjPtr = JimStringTrimRight(interp, objPtr, trimcharsObjPtr); + + + if (objPtr != strObjPtr && objPtr->refCount == 0) { + + Jim_FreeNewObj(interp, objPtr); + } + + return strObjPtr; +} + + +#ifdef HAVE_ISASCII +#define jim_isascii isascii +#else +static int jim_isascii(int c) +{ + return !(c & ~0x7f); +} +#endif + +static int JimStringIs(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *strClass, int strict) +{ + static const char * const strclassnames[] = { + "integer", "alpha", "alnum", "ascii", "digit", + "double", "lower", "upper", "space", "xdigit", + "control", "print", "graph", "punct", "boolean", + NULL + }; + enum { + STR_IS_INTEGER, STR_IS_ALPHA, STR_IS_ALNUM, STR_IS_ASCII, STR_IS_DIGIT, + STR_IS_DOUBLE, STR_IS_LOWER, STR_IS_UPPER, STR_IS_SPACE, STR_IS_XDIGIT, + STR_IS_CONTROL, STR_IS_PRINT, STR_IS_GRAPH, STR_IS_PUNCT, STR_IS_BOOLEAN, + }; + int strclass; + int len; + int i; + const char *str; + int (*isclassfunc)(int c) = NULL; + + if (Jim_GetEnum(interp, strClass, strclassnames, &strclass, "class", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { + return JIM_ERR; + } + + str = Jim_GetString(strObjPtr, &len); + if (len == 0) { + Jim_SetResultBool(interp, !strict); + return JIM_OK; + } + + switch (strclass) { + case STR_IS_INTEGER: + { + jim_wide w; + Jim_SetResultBool(interp, JimGetWideNoErr(interp, strObjPtr, &w) == JIM_OK); + return JIM_OK; + } + + case STR_IS_DOUBLE: + { + double d; + Jim_SetResultBool(interp, Jim_GetDouble(interp, strObjPtr, &d) == JIM_OK && errno != ERANGE); + return JIM_OK; + } + + case STR_IS_BOOLEAN: + { + int b; + Jim_SetResultBool(interp, Jim_GetBoolean(interp, strObjPtr, &b) == JIM_OK); + return JIM_OK; + } + + case STR_IS_ALPHA: isclassfunc = isalpha; break; + case STR_IS_ALNUM: isclassfunc = isalnum; break; + case STR_IS_ASCII: isclassfunc = jim_isascii; break; + case STR_IS_DIGIT: isclassfunc = isdigit; break; + case STR_IS_LOWER: isclassfunc = islower; break; + case STR_IS_UPPER: isclassfunc = isupper; break; + case STR_IS_SPACE: isclassfunc = isspace; break; + case STR_IS_XDIGIT: isclassfunc = isxdigit; break; + case STR_IS_CONTROL: isclassfunc = iscntrl; break; + case STR_IS_PRINT: isclassfunc = isprint; break; + case STR_IS_GRAPH: isclassfunc = isgraph; break; + case STR_IS_PUNCT: isclassfunc = ispunct; break; + default: + return JIM_ERR; + } + + for (i = 0; i < len; i++) { + if (!isclassfunc(UCHAR(str[i]))) { + Jim_SetResultBool(interp, 0); + return JIM_OK; + } + } + Jim_SetResultBool(interp, 1); + return JIM_OK; +} + + + +static const Jim_ObjType comparedStringObjType = { + "compared-string", + NULL, + NULL, + NULL, + JIM_TYPE_REFERENCES, +}; + +int Jim_CompareStringImmediate(Jim_Interp *interp, Jim_Obj *objPtr, const char *str) +{ + if (objPtr->typePtr == &comparedStringObjType && objPtr->internalRep.ptr == str) { + return 1; + } + else { + if (strcmp(str, Jim_String(objPtr)) != 0) + return 0; + + if (objPtr->typePtr != &comparedStringObjType) { + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &comparedStringObjType; + } + objPtr->internalRep.ptr = (char *)str; + return 1; + } +} + +static int qsortCompareStringPointers(const void *a, const void *b) +{ + char *const *sa = (char *const *)a; + char *const *sb = (char *const *)b; + + return strcmp(*sa, *sb); +} + + + +static void FreeSourceInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupSourceInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); + +static const Jim_ObjType sourceObjType = { + "source", + FreeSourceInternalRep, + DupSourceInternalRep, + NULL, + JIM_TYPE_REFERENCES, +}; + +void FreeSourceInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + Jim_DecrRefCount(interp, objPtr->internalRep.sourceValue.fileNameObj); +} + +void DupSourceInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + dupPtr->internalRep.sourceValue = srcPtr->internalRep.sourceValue; + Jim_IncrRefCount(dupPtr->internalRep.sourceValue.fileNameObj); +} + +static const Jim_ObjType scriptLineObjType = { + "scriptline", + NULL, + NULL, + NULL, + JIM_NONE, +}; + +static Jim_Obj *JimNewScriptLineObj(Jim_Interp *interp, int argc, int line) +{ + Jim_Obj *objPtr; + +#ifdef DEBUG_SHOW_SCRIPT + char buf[100]; + snprintf(buf, sizeof(buf), "line=%d, argc=%d", line, argc); + objPtr = Jim_NewStringObj(interp, buf, -1); +#else + objPtr = Jim_NewEmptyStringObj(interp); +#endif + objPtr->typePtr = &scriptLineObjType; + objPtr->internalRep.scriptLineValue.argc = argc; + objPtr->internalRep.scriptLineValue.line = line; + + return objPtr; +} + +static void FreeScriptInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupScriptInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); + +static const Jim_ObjType scriptObjType = { + "script", + FreeScriptInternalRep, + DupScriptInternalRep, + NULL, + JIM_TYPE_NONE, +}; + +typedef struct ScriptToken +{ + Jim_Obj *objPtr; + int type; +} ScriptToken; + +typedef struct ScriptObj +{ + ScriptToken *token; + Jim_Obj *fileNameObj; + int len; + int substFlags; + int inUse; /* Used to share a ScriptObj. Currently + only used by Jim_EvalObj() as protection against + shimmering of the currently evaluated object. */ + int firstline; + int linenr; + int missing; +} ScriptObj; + +static void JimSetScriptFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); +static int JimParseCheckMissing(Jim_Interp *interp, int ch); +static ScriptObj *JimGetScript(Jim_Interp *interp, Jim_Obj *objPtr); +static void JimSetErrorStack(Jim_Interp *interp, ScriptObj *script); + +void FreeScriptInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + int i; + struct ScriptObj *script = (void *)objPtr->internalRep.ptr; + + if (--script->inUse != 0) + return; + for (i = 0; i < script->len; i++) { + Jim_DecrRefCount(interp, script->token[i].objPtr); + } + Jim_Free(script->token); + Jim_DecrRefCount(interp, script->fileNameObj); + Jim_Free(script); +} + +void DupScriptInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + JIM_NOTUSED(interp); + JIM_NOTUSED(srcPtr); + + dupPtr->typePtr = NULL; +} + +typedef struct +{ + const char *token; + int len; + int type; + int line; +} ParseToken; + +typedef struct +{ + + ParseToken *list; + int size; + int count; + ParseToken static_list[20]; +} ParseTokenList; + +static void ScriptTokenListInit(ParseTokenList *tokenlist) +{ + tokenlist->list = tokenlist->static_list; + tokenlist->size = sizeof(tokenlist->static_list) / sizeof(ParseToken); + tokenlist->count = 0; +} + +static void ScriptTokenListFree(ParseTokenList *tokenlist) +{ + if (tokenlist->list != tokenlist->static_list) { + Jim_Free(tokenlist->list); + } +} + +static void ScriptAddToken(ParseTokenList *tokenlist, const char *token, int len, int type, + int line) +{ + ParseToken *t; + + if (tokenlist->count == tokenlist->size) { + + tokenlist->size *= 2; + if (tokenlist->list != tokenlist->static_list) { + tokenlist->list = + Jim_Realloc(tokenlist->list, tokenlist->size * sizeof(*tokenlist->list)); + } + else { + + tokenlist->list = Jim_Alloc(tokenlist->size * sizeof(*tokenlist->list)); + memcpy(tokenlist->list, tokenlist->static_list, + tokenlist->count * sizeof(*tokenlist->list)); + } + } + t = &tokenlist->list[tokenlist->count++]; + t->token = token; + t->len = len; + t->type = type; + t->line = line; +} + +static int JimCountWordTokens(struct ScriptObj *script, ParseToken *t) +{ + int expand = 1; + int count = 0; + + + if (t->type == JIM_TT_STR && !TOKEN_IS_SEP(t[1].type)) { + if ((t->len == 1 && *t->token == '*') || (t->len == 6 && strncmp(t->token, "expand", 6) == 0)) { + + expand = -1; + t++; + } + else { + if (script->missing == ' ') { + + script->missing = '}'; + script->linenr = t[1].line; + } + } + } + + + while (!TOKEN_IS_SEP(t->type)) { + t++; + count++; + } + + return count * expand; +} + +static Jim_Obj *JimMakeScriptObj(Jim_Interp *interp, const ParseToken *t) +{ + Jim_Obj *objPtr; + + if (t->type == JIM_TT_ESC && memchr(t->token, '\\', t->len) != NULL) { + + int len = t->len; + char *str = Jim_Alloc(len + 1); + len = JimEscape(str, t->token, len); + objPtr = Jim_NewStringObjNoAlloc(interp, str, len); + } + else { + objPtr = Jim_NewStringObj(interp, t->token, t->len); + } + return objPtr; +} + +static void ScriptObjAddTokens(Jim_Interp *interp, struct ScriptObj *script, + ParseTokenList *tokenlist) +{ + int i; + struct ScriptToken *token; + + int lineargs = 0; + + ScriptToken *linefirst; + int count; + int linenr; + +#ifdef DEBUG_SHOW_SCRIPT_TOKENS + printf("==== Tokens ====\n"); + for (i = 0; i < tokenlist->count; i++) { + printf("[%2d]@%d %s '%.*s'\n", i, tokenlist->list[i].line, jim_tt_name(tokenlist->list[i].type), + tokenlist->list[i].len, tokenlist->list[i].token); + } +#endif + + + count = tokenlist->count; + for (i = 0; i < tokenlist->count; i++) { + if (tokenlist->list[i].type == JIM_TT_EOL) { + count++; + } + } + linenr = script->firstline = tokenlist->list[0].line; + + token = script->token = Jim_Alloc(sizeof(ScriptToken) * count); + + + linefirst = token++; + + for (i = 0; i < tokenlist->count; ) { + + int wordtokens; + + + while (tokenlist->list[i].type == JIM_TT_SEP) { + i++; + } + + wordtokens = JimCountWordTokens(script, tokenlist->list + i); + + if (wordtokens == 0) { + + if (lineargs) { + linefirst->type = JIM_TT_LINE; + linefirst->objPtr = JimNewScriptLineObj(interp, lineargs, linenr); + Jim_IncrRefCount(linefirst->objPtr); + + + lineargs = 0; + linefirst = token++; + } + i++; + continue; + } + else if (wordtokens != 1) { + + token->type = JIM_TT_WORD; + token->objPtr = Jim_NewIntObj(interp, wordtokens); + Jim_IncrRefCount(token->objPtr); + token++; + if (wordtokens < 0) { + + i++; + wordtokens = -wordtokens - 1; + lineargs--; + } + } + + if (lineargs == 0) { + + linenr = tokenlist->list[i].line; + } + lineargs++; + + + while (wordtokens--) { + const ParseToken *t = &tokenlist->list[i++]; + + token->type = t->type; + token->objPtr = JimMakeScriptObj(interp, t); + Jim_IncrRefCount(token->objPtr); + + Jim_SetSourceInfo(interp, token->objPtr, script->fileNameObj, t->line); + token++; + } + } + + if (lineargs == 0) { + token--; + } + + script->len = token - script->token; + + JimPanic((script->len >= count, "allocated script array is too short")); + +#ifdef DEBUG_SHOW_SCRIPT + printf("==== Script (%s) ====\n", Jim_String(script->fileNameObj)); + for (i = 0; i < script->len; i++) { + const ScriptToken *t = &script->token[i]; + printf("[%2d] %s %s\n", i, jim_tt_name(t->type), Jim_String(t->objPtr)); + } +#endif + +} + +int Jim_ScriptIsComplete(Jim_Interp *interp, Jim_Obj *scriptObj, char *stateCharPtr) +{ + ScriptObj *script = JimGetScript(interp, scriptObj); + if (stateCharPtr) { + *stateCharPtr = script->missing; + } + return script->missing == ' ' || script->missing == '}'; +} + +static int JimParseCheckMissing(Jim_Interp *interp, int ch) +{ + const char *msg; + + switch (ch) { + case '\\': + case ' ': + return JIM_OK; + + case '[': + msg = "unmatched \"[\""; + break; + case '{': + msg = "missing close-brace"; + break; + case '}': + msg = "extra characters after close-brace"; + break; + case '"': + default: + msg = "missing quote"; + break; + } + + Jim_SetResultString(interp, msg, -1); + return JIM_ERR; +} + +Jim_Obj *Jim_GetSourceInfo(Jim_Interp *interp, Jim_Obj *objPtr, int *lineptr) +{ + int line; + Jim_Obj *fileNameObj; + + if (objPtr->typePtr == &sourceObjType) { + fileNameObj = objPtr->internalRep.sourceValue.fileNameObj; + line = objPtr->internalRep.sourceValue.lineNumber; + } + else if (objPtr->typePtr == &scriptObjType) { + ScriptObj *script = JimGetScript(interp, objPtr); + fileNameObj = script->fileNameObj; + line = script->firstline; + } + else { + fileNameObj = interp->emptyObj; + line = 1; + } + *lineptr = line; + return fileNameObj; +} + +void Jim_SetSourceInfo(Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj *fileNameObj, int lineNumber) +{ + JimPanic((Jim_IsShared(objPtr), "Jim_SetSourceInfo called with shared object")); + Jim_FreeIntRep(interp, objPtr); + Jim_IncrRefCount(fileNameObj); + objPtr->internalRep.sourceValue.fileNameObj = fileNameObj; + objPtr->internalRep.sourceValue.lineNumber = lineNumber; + objPtr->typePtr = &sourceObjType; +} + +static void SubstObjAddTokens(Jim_Interp *interp, struct ScriptObj *script, + ParseTokenList *tokenlist) +{ + int i; + struct ScriptToken *token; + + token = script->token = Jim_Alloc(sizeof(ScriptToken) * tokenlist->count); + + for (i = 0; i < tokenlist->count; i++) { + const ParseToken *t = &tokenlist->list[i]; + + + token->type = t->type; + token->objPtr = JimMakeScriptObj(interp, t); + Jim_IncrRefCount(token->objPtr); + token++; + } + + script->len = i; +} + +static void JimSetScriptFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) +{ + int scriptTextLen; + const char *scriptText = Jim_GetString(objPtr, &scriptTextLen); + struct JimParserCtx parser; + struct ScriptObj *script; + ParseTokenList tokenlist; + Jim_Obj *fileNameObj; + int line; + + + fileNameObj = Jim_GetSourceInfo(interp, objPtr, &line); + + + ScriptTokenListInit(&tokenlist); + + JimParserInit(&parser, scriptText, scriptTextLen, line); + while (!parser.eof) { + JimParseScript(&parser); + ScriptAddToken(&tokenlist, parser.tstart, parser.tend - parser.tstart + 1, parser.tt, + parser.tline); + } + + + ScriptAddToken(&tokenlist, scriptText + scriptTextLen, 0, JIM_TT_EOF, 0); + + + script = Jim_Alloc(sizeof(*script)); + memset(script, 0, sizeof(*script)); + script->inUse = 1; + script->fileNameObj = fileNameObj; + Jim_IncrRefCount(script->fileNameObj); + script->missing = parser.missing.ch; + script->linenr = parser.missing.line; + + ScriptObjAddTokens(interp, script, &tokenlist); + + + ScriptTokenListFree(&tokenlist); + + + Jim_FreeIntRep(interp, objPtr); + Jim_SetIntRepPtr(objPtr, script); + objPtr->typePtr = &scriptObjType; +} + +static ScriptObj *JimGetScript(Jim_Interp *interp, Jim_Obj *objPtr) +{ + if (objPtr == interp->emptyObj) { + + objPtr = interp->nullScriptObj; + } + + if (objPtr->typePtr != &scriptObjType || ((struct ScriptObj *)Jim_GetIntRepPtr(objPtr))->substFlags) { + JimSetScriptFromAny(interp, objPtr); + } + + return (ScriptObj *)Jim_GetIntRepPtr(objPtr); +} + +void Jim_InterpIncrProcEpoch(Jim_Interp *interp) +{ + interp->procEpoch++; + + + while (interp->oldCmdCache) { + Jim_Cmd *next = interp->oldCmdCache->prevCmd; + Jim_Free(interp->oldCmdCache); + interp->oldCmdCache = next; + } + interp->oldCmdCacheSize = 0; +} + +static void JimIncrCmdRefCount(Jim_Cmd *cmdPtr) +{ + cmdPtr->inUse++; +} + +static void JimDecrCmdRefCount(Jim_Interp *interp, Jim_Cmd *cmdPtr) +{ + if (--cmdPtr->inUse == 0) { + if (cmdPtr->isproc) { + Jim_DecrRefCount(interp, cmdPtr->u.proc.argListObjPtr); + Jim_DecrRefCount(interp, cmdPtr->u.proc.bodyObjPtr); + Jim_DecrRefCount(interp, cmdPtr->u.proc.nsObj); + if (cmdPtr->u.proc.staticVars) { + Jim_FreeHashTable(cmdPtr->u.proc.staticVars); + Jim_Free(cmdPtr->u.proc.staticVars); + } + } + else { + + if (cmdPtr->u.native.delProc) { + cmdPtr->u.native.delProc(interp, cmdPtr->u.native.privData); + } + } + if (cmdPtr->prevCmd) { + + JimDecrCmdRefCount(interp, cmdPtr->prevCmd); + } + + cmdPtr->prevCmd = interp->oldCmdCache; + interp->oldCmdCache = cmdPtr; + if (!interp->quitting && ++interp->oldCmdCacheSize >= 1000) { + Jim_InterpIncrProcEpoch(interp); + } + } +} + +static void JimIncrVarRef(Jim_VarVal *vv) +{ + vv->refCount++; +} + +static void JimDecrVarRef(Jim_Interp *interp, Jim_VarVal *vv) +{ + assert(vv->refCount > 0); + if (--vv->refCount == 0) { + if (vv->objPtr) { + Jim_DecrRefCount(interp, vv->objPtr); + } + Jim_Free(vv); + } +} + +static void JimVariablesHTValDestructor(void *interp, void *val) +{ + JimDecrVarRef(interp, val); +} + +static unsigned int JimObjectHTHashFunction(const void *key) +{ + Jim_Obj *keyObj = (Jim_Obj *)key; + int length; + const char *string; + +#ifdef JIM_OPTIMIZATION + if (JimIsWide(keyObj) && keyObj->bytes == NULL) { + + jim_wide objValue = JimWideValue(keyObj); + if (objValue > INT_MIN && objValue < INT_MAX) { + unsigned result = 0; + unsigned value = (unsigned)objValue; + + if (objValue < 0) { + value = (unsigned)-objValue; + } + + + do { + result += (result << 3) + (value % 10 + '0'); + value /= 10; + } while (value); + + if (objValue < 0) { + result += (result << 3) + '-'; + } + return result; + } + } +#endif + string = Jim_GetString(keyObj, &length); + return Jim_GenHashFunction((const unsigned char *)string, length); +} + +static int JimObjectHTKeyCompare(void *privdata, const void *key1, const void *key2) +{ + return Jim_StringEqObj((Jim_Obj *)key1, (Jim_Obj *)key2); +} + +static void *JimObjectHTKeyValDup(void *privdata, const void *val) +{ + Jim_IncrRefCount((Jim_Obj *)val); + return (void *)val; +} + +static void JimObjectHTKeyValDestructor(void *interp, void *val) +{ + Jim_DecrRefCount(interp, (Jim_Obj *)val); +} + + +static void *JimVariablesHTValDup(void *privdata, const void *val) +{ + JimIncrVarRef((Jim_VarVal *)val); + return (void *)val; +} + +static const Jim_HashTableType JimVariablesHashTableType = { + JimObjectHTHashFunction, + JimObjectHTKeyValDup, + JimVariablesHTValDup, + JimObjectHTKeyCompare, + JimObjectHTKeyValDestructor, + JimVariablesHTValDestructor +}; + + +static const char *Jim_GetStringNoQualifier(Jim_Obj *objPtr, int *length) +{ + int len; + const char *str = Jim_GetString(objPtr, &len); + if (len >= 2 && str[0] == ':' && str[1] == ':') { + while (len && *str == ':') { + len--; + str++; + } + } + *length = len; + return str; +} + +static unsigned int JimCommandsHT_HashFunction(const void *key) +{ + int len; + const char *str = Jim_GetStringNoQualifier((Jim_Obj *)key, &len); + return Jim_GenHashFunction((const unsigned char *)str, len); +} + +static int JimCommandsHT_KeyCompare(void *privdata, const void *key1, const void *key2) +{ + int len1, len2; + const char *str1 = Jim_GetStringNoQualifier((Jim_Obj *)key1, &len1); + const char *str2 = Jim_GetStringNoQualifier((Jim_Obj *)key2, &len2); + return len1 == len2 && memcmp(str1, str2, len1) == 0; +} + +static void JimCommandsHT_ValDestructor(void *interp, void *val) +{ + JimDecrCmdRefCount(interp, val); +} + +static const Jim_HashTableType JimCommandsHashTableType = { + JimCommandsHT_HashFunction, + JimObjectHTKeyValDup, + NULL, + JimCommandsHT_KeyCompare, + JimObjectHTKeyValDestructor, + JimCommandsHT_ValDestructor +}; + + + +Jim_Obj *Jim_MakeGlobalNamespaceName(Jim_Interp *interp, Jim_Obj *nameObjPtr) +{ +#ifdef jim_ext_namespace + Jim_Obj *resultObj; + + const char *name = Jim_String(nameObjPtr); + if (name[0] == ':' && name[1] == ':') { + return nameObjPtr; + } + Jim_IncrRefCount(nameObjPtr); + resultObj = Jim_NewStringObj(interp, "::", -1); + Jim_AppendObj(interp, resultObj, nameObjPtr); + Jim_DecrRefCount(interp, nameObjPtr); + + return resultObj; +#else + return nameObjPtr; +#endif +} + +static Jim_Obj *JimQualifyName(Jim_Interp *interp, Jim_Obj *objPtr) +{ +#ifdef jim_ext_namespace + if (Jim_Length(interp->framePtr->nsObj)) { + int len; + const char *name = Jim_GetString(objPtr, &len); + if (len < 2 || name[0] != ':' || name[1] != ':') { + + objPtr = Jim_DuplicateObj(interp, interp->framePtr->nsObj); + Jim_AppendStrings(interp, objPtr, "::", name, NULL); + } + } +#endif + Jim_IncrRefCount(objPtr); + return objPtr; +} + +static void JimCreateCommand(Jim_Interp *interp, Jim_Obj *nameObjPtr, Jim_Cmd *cmd) +{ + JimPanic((nameObjPtr->refCount == 0, "JimCreateCommand called with zero ref count name")); + + if (interp->local) { + Jim_HashEntry *he = Jim_FindHashEntry(&interp->commands, nameObjPtr); + if (he) { + + cmd->prevCmd = Jim_GetHashEntryVal(he); + Jim_SetHashVal(&interp->commands, he, cmd); + + Jim_InterpIncrProcEpoch(interp); + return; + } + } + + + + Jim_ReplaceHashEntry(&interp->commands, nameObjPtr, cmd); +} + +int Jim_CreateCommandObj(Jim_Interp *interp, Jim_Obj *cmdNameObj, + Jim_CmdProc *cmdProc, void *privData, Jim_DelCmdProc *delProc) +{ + Jim_Cmd *cmdPtr = Jim_Alloc(sizeof(*cmdPtr)); + + + memset(cmdPtr, 0, sizeof(*cmdPtr)); + cmdPtr->inUse = 1; + cmdPtr->u.native.delProc = delProc; + cmdPtr->u.native.cmdProc = cmdProc; + cmdPtr->u.native.privData = privData; + + Jim_IncrRefCount(cmdNameObj); + JimCreateCommand(interp, cmdNameObj, cmdPtr); + Jim_DecrRefCount(interp, cmdNameObj); + + return JIM_OK; +} + + +int Jim_CreateCommand(Jim_Interp *interp, const char *cmdNameStr, + Jim_CmdProc *cmdProc, void *privData, Jim_DelCmdProc *delProc) +{ + return Jim_CreateCommandObj(interp, Jim_NewStringObj(interp, cmdNameStr, -1), cmdProc, privData, delProc); +} + +static int JimCreateProcedureStatics(Jim_Interp *interp, Jim_Cmd *cmdPtr, Jim_Obj *staticsListObjPtr) +{ + int len, i; + + len = Jim_ListLength(interp, staticsListObjPtr); + if (len == 0) { + return JIM_OK; + } + + cmdPtr->u.proc.staticVars = Jim_Alloc(sizeof(Jim_HashTable)); + Jim_InitHashTable(cmdPtr->u.proc.staticVars, &JimVariablesHashTableType, interp); + for (i = 0; i < len; i++) { + Jim_Obj *initObjPtr = NULL; + Jim_Obj *nameObjPtr; + Jim_VarVal *vv = NULL; + Jim_Obj *objPtr = Jim_ListGetIndex(interp, staticsListObjPtr, i); + int subLen = Jim_ListLength(interp, objPtr); + int byref = 0; + + + if (subLen != 1 && subLen != 2) { + Jim_SetResultFormatted(interp, "too many fields in static specifier \"%#s\"", + objPtr); + return JIM_ERR; + } + + nameObjPtr = Jim_ListGetIndex(interp, objPtr, 0); + + + if (subLen == 1) { + int len; + const char *pt = Jim_GetString(nameObjPtr, &len); + if (*pt == '&') { + + nameObjPtr = Jim_NewStringObj(interp, pt + 1, len - 1); + byref = 1; + } + } + Jim_IncrRefCount(nameObjPtr); + + if (subLen == 1) { + switch (SetVariableFromAny(interp, nameObjPtr)) { + case JIM_DICT_SUGAR: + + if (byref) { + Jim_SetResultFormatted(interp, "Can't link to array element \"%#s\"", nameObjPtr); + } + else { + Jim_SetResultFormatted(interp, "Can't initialise array element \"%#s\"", nameObjPtr); + } + Jim_DecrRefCount(interp, nameObjPtr); + return JIM_ERR; + + case JIM_OK: + if (byref) { + vv = nameObjPtr->internalRep.varValue.vv; + } + else { + initObjPtr = Jim_GetVariable(interp, nameObjPtr, JIM_NONE); + } + break; + + case JIM_ERR: + + Jim_SetResultFormatted(interp, + "variable for initialization of static \"%#s\" not found in the local context", + nameObjPtr); + Jim_DecrRefCount(interp, nameObjPtr); + return JIM_ERR; + } + } + else { + initObjPtr = Jim_ListGetIndex(interp, objPtr, 1); + } + + if (vv == NULL) { + vv = Jim_Alloc(sizeof(*vv)); + vv->objPtr = initObjPtr; + Jim_IncrRefCount(vv->objPtr); + vv->linkFramePtr = NULL; + vv->refCount = 0; + } + + if (JimSetNewVariable(cmdPtr->u.proc.staticVars, nameObjPtr, vv) != JIM_OK) { + Jim_SetResultFormatted(interp, + "static variable name \"%#s\" duplicated in statics list", nameObjPtr); + JimIncrVarRef(vv); + JimDecrVarRef(interp, vv); + Jim_DecrRefCount(interp, nameObjPtr); + return JIM_ERR; + } + + Jim_DecrRefCount(interp, nameObjPtr); + } + return JIM_OK; +} + + +#ifdef jim_ext_namespace +static const char *Jim_memrchr(const char *p, int c, int len) +{ + int i; + for (i = len; i > 0; i--) { + if (p[i] == c) { + return p + i; + } + } + return NULL; +} +#endif + +static void JimUpdateProcNamespace(Jim_Interp *interp, Jim_Cmd *cmdPtr, Jim_Obj *nameObjPtr) +{ +#ifdef jim_ext_namespace + if (cmdPtr->isproc) { + int len; + const char *cmdname = Jim_GetStringNoQualifier(nameObjPtr, &len); + + const char *pt = Jim_memrchr(cmdname, ':', len); + if (pt && pt != cmdname && pt[-1] == ':') { + pt++; + Jim_DecrRefCount(interp, cmdPtr->u.proc.nsObj); + cmdPtr->u.proc.nsObj = Jim_NewStringObj(interp, cmdname, pt - cmdname - 2); + Jim_IncrRefCount(cmdPtr->u.proc.nsObj); + + Jim_Obj *tempObj = Jim_NewStringObj(interp, pt, len - (pt - cmdname)); + if (Jim_FindHashEntry(&interp->commands, tempObj)) { + + Jim_InterpIncrProcEpoch(interp); + } + Jim_FreeNewObj(interp, tempObj); + } + } +#endif +} + +static Jim_Cmd *JimCreateProcedureCmd(Jim_Interp *interp, Jim_Obj *argListObjPtr, + Jim_Obj *staticsListObjPtr, Jim_Obj *bodyObjPtr, Jim_Obj *nsObj) +{ + Jim_Cmd *cmdPtr; + int argListLen; + int i; + + argListLen = Jim_ListLength(interp, argListObjPtr); + + + cmdPtr = Jim_Alloc(sizeof(*cmdPtr) + sizeof(struct Jim_ProcArg) * argListLen); + assert(cmdPtr); + memset(cmdPtr, 0, sizeof(*cmdPtr)); + cmdPtr->inUse = 1; + cmdPtr->isproc = 1; + cmdPtr->u.proc.argListObjPtr = argListObjPtr; + cmdPtr->u.proc.argListLen = argListLen; + cmdPtr->u.proc.bodyObjPtr = bodyObjPtr; + cmdPtr->u.proc.argsPos = -1; + cmdPtr->u.proc.arglist = (struct Jim_ProcArg *)(cmdPtr + 1); + cmdPtr->u.proc.nsObj = nsObj ? nsObj : interp->emptyObj; + Jim_IncrRefCount(argListObjPtr); + Jim_IncrRefCount(bodyObjPtr); + Jim_IncrRefCount(cmdPtr->u.proc.nsObj); + + + if (staticsListObjPtr && JimCreateProcedureStatics(interp, cmdPtr, staticsListObjPtr) != JIM_OK) { + goto err; + } + + + + for (i = 0; i < argListLen; i++) { + Jim_Obj *argPtr; + Jim_Obj *nameObjPtr; + Jim_Obj *defaultObjPtr; + int len; + + + argPtr = Jim_ListGetIndex(interp, argListObjPtr, i); + len = Jim_ListLength(interp, argPtr); + if (len == 0) { + Jim_SetResultString(interp, "argument with no name", -1); +err: + JimDecrCmdRefCount(interp, cmdPtr); + return NULL; + } + if (len > 2) { + Jim_SetResultFormatted(interp, "too many fields in argument specifier \"%#s\"", argPtr); + goto err; + } + + if (len == 2) { + + nameObjPtr = Jim_ListGetIndex(interp, argPtr, 0); + defaultObjPtr = Jim_ListGetIndex(interp, argPtr, 1); + } + else { + + nameObjPtr = argPtr; + defaultObjPtr = NULL; + } + + + if (Jim_CompareStringImmediate(interp, nameObjPtr, "args")) { + if (cmdPtr->u.proc.argsPos >= 0) { + Jim_SetResultString(interp, "'args' specified more than once", -1); + goto err; + } + cmdPtr->u.proc.argsPos = i; + } + else { + if (len == 2) { + cmdPtr->u.proc.optArity++; + } + else { + cmdPtr->u.proc.reqArity++; + } + } + + cmdPtr->u.proc.arglist[i].nameObjPtr = nameObjPtr; + cmdPtr->u.proc.arglist[i].defaultObjPtr = defaultObjPtr; + } + + return cmdPtr; +} + +int Jim_DeleteCommand(Jim_Interp *interp, Jim_Obj *nameObj) +{ + int ret = JIM_OK; + + nameObj = JimQualifyName(interp, nameObj); + + if (Jim_DeleteHashEntry(&interp->commands, nameObj) == JIM_ERR) { + Jim_SetResultFormatted(interp, "can't delete \"%#s\": command doesn't exist", nameObj); + ret = JIM_ERR; + } + Jim_DecrRefCount(interp, nameObj); + + return ret; +} + +int Jim_RenameCommand(Jim_Interp *interp, Jim_Obj *oldNameObj, Jim_Obj *newNameObj) +{ + int ret = JIM_ERR; + Jim_HashEntry *he; + Jim_Cmd *cmdPtr; + + if (Jim_Length(newNameObj) == 0) { + return Jim_DeleteCommand(interp, oldNameObj); + } + + + + oldNameObj = JimQualifyName(interp, oldNameObj); + newNameObj = JimQualifyName(interp, newNameObj); + + + he = Jim_FindHashEntry(&interp->commands, oldNameObj); + if (he == NULL) { + Jim_SetResultFormatted(interp, "can't rename \"%#s\": command doesn't exist", oldNameObj); + } + else if (Jim_FindHashEntry(&interp->commands, newNameObj)) { + Jim_SetResultFormatted(interp, "can't rename to \"%#s\": command already exists", newNameObj); + } + else { + cmdPtr = Jim_GetHashEntryVal(he); + if (cmdPtr->prevCmd) { + Jim_SetResultFormatted(interp, "can't rename local command \"%#s\"", oldNameObj); + } + else { + + JimIncrCmdRefCount(cmdPtr); + JimUpdateProcNamespace(interp, cmdPtr, newNameObj); + Jim_AddHashEntry(&interp->commands, newNameObj, cmdPtr); + + + Jim_DeleteHashEntry(&interp->commands, oldNameObj); + + + Jim_InterpIncrProcEpoch(interp); + + ret = JIM_OK; + } + } + + Jim_DecrRefCount(interp, oldNameObj); + Jim_DecrRefCount(interp, newNameObj); + + return ret; +} + + +static void FreeCommandInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + Jim_DecrRefCount(interp, objPtr->internalRep.cmdValue.nsObj); +} + +static void DupCommandInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + dupPtr->internalRep.cmdValue = srcPtr->internalRep.cmdValue; + dupPtr->typePtr = srcPtr->typePtr; + Jim_IncrRefCount(dupPtr->internalRep.cmdValue.nsObj); +} + +static const Jim_ObjType commandObjType = { + "command", + FreeCommandInternalRep, + DupCommandInternalRep, + NULL, + JIM_TYPE_REFERENCES, +}; + +Jim_Cmd *Jim_GetCommand(Jim_Interp *interp, Jim_Obj *objPtr, int flags) +{ + Jim_Cmd *cmd; + + if (objPtr->typePtr == &commandObjType + && objPtr->internalRep.cmdValue.procEpoch == interp->procEpoch +#ifdef jim_ext_namespace + && Jim_StringEqObj(objPtr->internalRep.cmdValue.nsObj, interp->framePtr->nsObj) +#endif + && objPtr->internalRep.cmdValue.cmdPtr->inUse) { + + cmd = objPtr->internalRep.cmdValue.cmdPtr; + } + else { + Jim_Obj *qualifiedNameObj = JimQualifyName(interp, objPtr); + Jim_HashEntry *he = Jim_FindHashEntry(&interp->commands, qualifiedNameObj); +#ifdef jim_ext_namespace + if (he == NULL && Jim_Length(interp->framePtr->nsObj)) { + he = Jim_FindHashEntry(&interp->commands, objPtr); + } +#endif + if (he == NULL) { + if (flags & JIM_ERRMSG) { + Jim_SetResultFormatted(interp, "invalid command name \"%#s\"", objPtr); + } + Jim_DecrRefCount(interp, qualifiedNameObj); + return NULL; + } + cmd = Jim_GetHashEntryVal(he); + + cmd->cmdNameObj = Jim_GetHashEntryKey(he); + + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &commandObjType; + objPtr->internalRep.cmdValue.procEpoch = interp->procEpoch; + objPtr->internalRep.cmdValue.cmdPtr = cmd; + objPtr->internalRep.cmdValue.nsObj = interp->framePtr->nsObj; + Jim_IncrRefCount(interp->framePtr->nsObj); + Jim_DecrRefCount(interp, qualifiedNameObj); + } + while (cmd->u.proc.upcall) { + cmd = cmd->prevCmd; + } + return cmd; +} + + + +static const Jim_ObjType variableObjType = { + "variable", + NULL, + NULL, + NULL, + JIM_TYPE_REFERENCES, +}; + +static int SetVariableFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) +{ + const char *varName; + Jim_CallFrame *framePtr; + int global; + int len; + Jim_VarVal *vv; + + + if (objPtr->typePtr == &variableObjType) { + framePtr = objPtr->internalRep.varValue.global ? interp->topFramePtr : interp->framePtr; + if (objPtr->internalRep.varValue.callFrameId == framePtr->id) { + + return JIM_OK; + } + + } + else if (objPtr->typePtr == &dictSubstObjType) { + return JIM_DICT_SUGAR; + } + + varName = Jim_GetString(objPtr, &len); + + + if (len && varName[len - 1] == ')' && strchr(varName, '(') != NULL) { + return JIM_DICT_SUGAR; + } + + if (varName[0] == ':' && varName[1] == ':') { + while (*varName == ':') { + varName++; + len--; + } + global = 1; + framePtr = interp->topFramePtr; + + Jim_Obj *tempObj = Jim_NewStringObj(interp, varName, len); + vv = JimFindVariable(&framePtr->vars, tempObj); + Jim_FreeNewObj(interp, tempObj); + } + else { + global = 0; + framePtr = interp->framePtr; + + vv = JimFindVariable(&framePtr->vars, objPtr); + if (vv == NULL && framePtr->staticVars) { + + vv = JimFindVariable(framePtr->staticVars, objPtr); + } + } + + if (vv == NULL) { + return JIM_ERR; + } + + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &variableObjType; + objPtr->internalRep.varValue.callFrameId = framePtr->id; + objPtr->internalRep.varValue.vv = vv; + objPtr->internalRep.varValue.global = global; + return JIM_OK; +} + + +static int JimDictSugarSet(Jim_Interp *interp, Jim_Obj *ObjPtr, Jim_Obj *valObjPtr); +static Jim_Obj *JimDictSugarGet(Jim_Interp *interp, Jim_Obj *ObjPtr, int flags); + +static int JimSetNewVariable(Jim_HashTable *ht, Jim_Obj *nameObjPtr, Jim_VarVal *vv) +{ + return Jim_AddHashEntry(ht, nameObjPtr, vv); +} + +static Jim_VarVal *JimFindVariable(Jim_HashTable *ht, Jim_Obj *nameObjPtr) +{ + Jim_HashEntry *he = Jim_FindHashEntry(ht, nameObjPtr); + if (he) { + return (Jim_VarVal *)Jim_GetHashEntryVal(he); + } + return NULL; +} + +static int JimUnsetVariable(Jim_HashTable *ht, Jim_Obj *nameObjPtr) +{ + return Jim_DeleteHashEntry(ht, nameObjPtr); +} + +static Jim_VarVal *JimCreateVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, Jim_Obj *valObjPtr) +{ + const char *name; + Jim_CallFrame *framePtr; + int global; + int len; + + + Jim_VarVal *vv = Jim_Alloc(sizeof(*vv)); + + vv->objPtr = valObjPtr; + Jim_IncrRefCount(valObjPtr); + vv->linkFramePtr = NULL; + vv->refCount = 0; + + name = Jim_GetString(nameObjPtr, &len); + if (name[0] == ':' && name[1] == ':') { + while (*name == ':') { + name++; + len--; + } + framePtr = interp->topFramePtr; + global = 1; + JimSetNewVariable(&framePtr->vars, Jim_NewStringObj(interp, name, len), vv); + } + else { + framePtr = interp->framePtr; + global = 0; + JimSetNewVariable(&framePtr->vars, nameObjPtr, vv); + } + + + Jim_FreeIntRep(interp, nameObjPtr); + nameObjPtr->typePtr = &variableObjType; + nameObjPtr->internalRep.varValue.callFrameId = framePtr->id; + nameObjPtr->internalRep.varValue.vv = vv; + nameObjPtr->internalRep.varValue.global = global; + + return vv; +} + +int Jim_SetVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, Jim_Obj *valObjPtr) +{ + int err; + Jim_VarVal *vv; + + switch (SetVariableFromAny(interp, nameObjPtr)) { + case JIM_DICT_SUGAR: + return JimDictSugarSet(interp, nameObjPtr, valObjPtr); + + case JIM_ERR: + JimCreateVariable(interp, nameObjPtr, valObjPtr); + break; + + case JIM_OK: + vv = nameObjPtr->internalRep.varValue.vv; + if (vv->linkFramePtr == NULL) { + Jim_IncrRefCount(valObjPtr); + Jim_DecrRefCount(interp, vv->objPtr); + vv->objPtr = valObjPtr; + } + else { + Jim_CallFrame *savedCallFrame; + + savedCallFrame = interp->framePtr; + interp->framePtr = vv->linkFramePtr; + err = Jim_SetVariable(interp, vv->objPtr, valObjPtr); + interp->framePtr = savedCallFrame; + if (err != JIM_OK) + return err; + } + } + return JIM_OK; +} + +int Jim_SetVariableStr(Jim_Interp *interp, const char *name, Jim_Obj *objPtr) +{ + Jim_Obj *nameObjPtr; + int result; + + nameObjPtr = Jim_NewStringObj(interp, name, -1); + Jim_IncrRefCount(nameObjPtr); + result = Jim_SetVariable(interp, nameObjPtr, objPtr); + Jim_DecrRefCount(interp, nameObjPtr); + return result; +} + +int Jim_SetGlobalVariableStr(Jim_Interp *interp, const char *name, Jim_Obj *objPtr) +{ + Jim_CallFrame *savedFramePtr; + int result; + + savedFramePtr = interp->framePtr; + interp->framePtr = interp->topFramePtr; + result = Jim_SetVariableStr(interp, name, objPtr); + interp->framePtr = savedFramePtr; + return result; +} + +int Jim_SetVariableStrWithStr(Jim_Interp *interp, const char *name, const char *val) +{ + Jim_Obj *valObjPtr; + int result; + + valObjPtr = Jim_NewStringObj(interp, val, -1); + Jim_IncrRefCount(valObjPtr); + result = Jim_SetVariableStr(interp, name, valObjPtr); + Jim_DecrRefCount(interp, valObjPtr); + return result; +} + +int Jim_SetVariableLink(Jim_Interp *interp, Jim_Obj *nameObjPtr, + Jim_Obj *targetNameObjPtr, Jim_CallFrame *targetCallFrame) +{ + const char *varName; + const char *targetName; + Jim_CallFrame *framePtr; + Jim_VarVal *vv; + int len; + int varnamelen; + + + switch (SetVariableFromAny(interp, nameObjPtr)) { + case JIM_DICT_SUGAR: + + Jim_SetResultFormatted(interp, "bad variable name \"%#s\": upvar won't create a scalar variable that looks like an array element", nameObjPtr); + return JIM_ERR; + + case JIM_OK: + vv = nameObjPtr->internalRep.varValue.vv; + + if (vv->linkFramePtr == NULL) { + Jim_SetResultFormatted(interp, "variable \"%#s\" already exists", nameObjPtr); + return JIM_ERR; + } + + + vv->linkFramePtr = NULL; + break; + } + + + + varName = Jim_GetString(nameObjPtr, &varnamelen); + + if (varName[0] == ':' && varName[1] == ':') { + while (*varName == ':') { + varName++; + varnamelen--; + } + + framePtr = interp->topFramePtr; + } + else { + framePtr = interp->framePtr; + } + + targetName = Jim_GetString(targetNameObjPtr, &len); + if (targetName[0] == ':' && targetName[1] == ':') { + while (*targetName == ':') { + targetName++; + len--; + } + targetNameObjPtr = Jim_NewStringObj(interp, targetName, len); + targetCallFrame = interp->topFramePtr; + } + Jim_IncrRefCount(targetNameObjPtr); + + if (framePtr->level < targetCallFrame->level) { + Jim_SetResultFormatted(interp, + "bad variable name \"%#s\": upvar won't create namespace variable that refers to procedure variable", + nameObjPtr); + Jim_DecrRefCount(interp, targetNameObjPtr); + return JIM_ERR; + } + + + if (framePtr == targetCallFrame) { + Jim_Obj *objPtr = targetNameObjPtr; + + + while (1) { + if (Jim_Length(objPtr) == varnamelen && memcmp(Jim_String(objPtr), varName, varnamelen) == 0) { + Jim_SetResultString(interp, "can't upvar from variable to itself", -1); + Jim_DecrRefCount(interp, targetNameObjPtr); + return JIM_ERR; + } + if (SetVariableFromAny(interp, objPtr) != JIM_OK) + break; + vv = objPtr->internalRep.varValue.vv; + if (vv->linkFramePtr != targetCallFrame) + break; + objPtr = vv->objPtr; + } + } + + + Jim_SetVariable(interp, nameObjPtr, targetNameObjPtr); + + nameObjPtr->internalRep.varValue.vv->linkFramePtr = targetCallFrame; + Jim_DecrRefCount(interp, targetNameObjPtr); + return JIM_OK; +} + +Jim_Obj *Jim_GetVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, int flags) +{ + if (interp->safeexpr) { + return nameObjPtr; + } + switch (SetVariableFromAny(interp, nameObjPtr)) { + case JIM_OK:{ + Jim_VarVal *vv = nameObjPtr->internalRep.varValue.vv; + + if (vv->linkFramePtr == NULL) { + return vv->objPtr; + } + else { + Jim_Obj *objPtr; + + + Jim_CallFrame *savedCallFrame = interp->framePtr; + + interp->framePtr = vv->linkFramePtr; + objPtr = Jim_GetVariable(interp, vv->objPtr, flags); + interp->framePtr = savedCallFrame; + if (objPtr) { + return objPtr; + } + + } + } + break; + + case JIM_DICT_SUGAR: + + return JimDictSugarGet(interp, nameObjPtr, flags); + } + if (flags & JIM_ERRMSG) { + Jim_SetResultFormatted(interp, "can't read \"%#s\": no such variable", nameObjPtr); + } + return NULL; +} + +Jim_Obj *Jim_GetGlobalVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, int flags) +{ + Jim_CallFrame *savedFramePtr; + Jim_Obj *objPtr; + + savedFramePtr = interp->framePtr; + interp->framePtr = interp->topFramePtr; + objPtr = Jim_GetVariable(interp, nameObjPtr, flags); + interp->framePtr = savedFramePtr; + + return objPtr; +} + +Jim_Obj *Jim_GetVariableStr(Jim_Interp *interp, const char *name, int flags) +{ + Jim_Obj *nameObjPtr, *varObjPtr; + + nameObjPtr = Jim_NewStringObj(interp, name, -1); + Jim_IncrRefCount(nameObjPtr); + varObjPtr = Jim_GetVariable(interp, nameObjPtr, flags); + Jim_DecrRefCount(interp, nameObjPtr); + return varObjPtr; +} + +Jim_Obj *Jim_GetGlobalVariableStr(Jim_Interp *interp, const char *name, int flags) +{ + Jim_CallFrame *savedFramePtr; + Jim_Obj *objPtr; + + savedFramePtr = interp->framePtr; + interp->framePtr = interp->topFramePtr; + objPtr = Jim_GetVariableStr(interp, name, flags); + interp->framePtr = savedFramePtr; + + return objPtr; +} + +int Jim_UnsetVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, int flags) +{ + Jim_VarVal *vv; + int retval; + Jim_CallFrame *framePtr; + + retval = SetVariableFromAny(interp, nameObjPtr); + if (retval == JIM_DICT_SUGAR) { + + return JimDictSugarSet(interp, nameObjPtr, NULL); + } + else if (retval == JIM_OK) { + vv = nameObjPtr->internalRep.varValue.vv; + + + if (vv->linkFramePtr) { + framePtr = interp->framePtr; + interp->framePtr = vv->linkFramePtr; + retval = Jim_UnsetVariable(interp, vv->objPtr, JIM_NONE); + interp->framePtr = framePtr; + } + else { + if (nameObjPtr->internalRep.varValue.global) { + int len; + const char *name = Jim_GetString(nameObjPtr, &len); + while (*name == ':') { + name++; + len--; + } + framePtr = interp->topFramePtr; + Jim_Obj *tempObj = Jim_NewStringObj(interp, name, len); + retval = JimUnsetVariable(&framePtr->vars, tempObj); + Jim_FreeNewObj(interp, tempObj); + } + else { + framePtr = interp->framePtr; + retval = JimUnsetVariable(&framePtr->vars, nameObjPtr); + } + + if (retval == JIM_OK) { + + framePtr->id = interp->callFrameEpoch++; + } + } + } + if (retval != JIM_OK && (flags & JIM_ERRMSG)) { + Jim_SetResultFormatted(interp, "can't unset \"%#s\": no such variable", nameObjPtr); + } + return retval; +} + + + +static void JimDictSugarParseVarKey(Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj **varPtrPtr, Jim_Obj **keyPtrPtr) +{ + const char *str, *p; + int len, keyLen; + Jim_Obj *varObjPtr, *keyObjPtr; + + str = Jim_GetString(objPtr, &len); + + p = strchr(str, '('); + JimPanic((p == NULL, "JimDictSugarParseVarKey() called for non-dict-sugar (%s)", str)); + + varObjPtr = Jim_NewStringObj(interp, str, p - str); + + p++; + keyLen = (str + len) - p; + if (str[len - 1] == ')') { + keyLen--; + } + + + keyObjPtr = Jim_NewStringObj(interp, p, keyLen); + + Jim_IncrRefCount(varObjPtr); + Jim_IncrRefCount(keyObjPtr); + *varPtrPtr = varObjPtr; + *keyPtrPtr = keyObjPtr; +} + +static int JimDictSugarSet(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *valObjPtr) +{ + int err; + + SetDictSubstFromAny(interp, objPtr); + + err = Jim_SetDictKeysVector(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr, + &objPtr->internalRep.dictSubstValue.indexObjPtr, 1, valObjPtr, JIM_MUSTEXIST); + + if (err == JIM_OK) { + + Jim_SetEmptyResult(interp); + } + else { + if (!valObjPtr) { + + if (Jim_GetVariable(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr, JIM_NONE)) { + Jim_SetResultFormatted(interp, "can't unset \"%#s\": no such element in array", + objPtr); + return err; + } + } + + Jim_SetResultFormatted(interp, "can't %s \"%#s\": variable isn't array", + (valObjPtr ? "set" : "unset"), objPtr); + } + return err; +} + +static Jim_Obj *JimDictExpandArrayVariable(Jim_Interp *interp, Jim_Obj *varObjPtr, + Jim_Obj *keyObjPtr, int flags) +{ + Jim_Obj *dictObjPtr; + Jim_Obj *resObjPtr = NULL; + int ret; + + dictObjPtr = Jim_GetVariable(interp, varObjPtr, JIM_ERRMSG); + if (!dictObjPtr) { + return NULL; + } + + ret = Jim_DictKey(interp, dictObjPtr, keyObjPtr, &resObjPtr, JIM_NONE); + if (ret != JIM_OK) { + Jim_SetResultFormatted(interp, + "can't read \"%#s(%#s)\": %s array", varObjPtr, keyObjPtr, + ret < 0 ? "variable isn't" : "no such element in"); + } + else if ((flags & JIM_UNSHARED) && Jim_IsShared(dictObjPtr)) { + + Jim_SetVariable(interp, varObjPtr, Jim_DuplicateObj(interp, dictObjPtr)); + } + + return resObjPtr; +} + + +static Jim_Obj *JimDictSugarGet(Jim_Interp *interp, Jim_Obj *objPtr, int flags) +{ + SetDictSubstFromAny(interp, objPtr); + + return JimDictExpandArrayVariable(interp, + objPtr->internalRep.dictSubstValue.varNameObjPtr, + objPtr->internalRep.dictSubstValue.indexObjPtr, flags); +} + + + +void FreeDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + Jim_DecrRefCount(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr); + Jim_DecrRefCount(interp, objPtr->internalRep.dictSubstValue.indexObjPtr); +} + +static void DupDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + + dupPtr->internalRep = srcPtr->internalRep; + + Jim_IncrRefCount(dupPtr->internalRep.dictSubstValue.varNameObjPtr); + Jim_IncrRefCount(dupPtr->internalRep.dictSubstValue.indexObjPtr); +} + + +static void SetDictSubstFromAny(Jim_Interp *interp, Jim_Obj *objPtr) +{ + if (objPtr->typePtr != &dictSubstObjType) { + Jim_Obj *varObjPtr, *keyObjPtr; + + if (objPtr->typePtr == &interpolatedObjType) { + + + varObjPtr = objPtr->internalRep.dictSubstValue.varNameObjPtr; + keyObjPtr = objPtr->internalRep.dictSubstValue.indexObjPtr; + + Jim_IncrRefCount(varObjPtr); + Jim_IncrRefCount(keyObjPtr); + } + else { + JimDictSugarParseVarKey(interp, objPtr, &varObjPtr, &keyObjPtr); + } + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &dictSubstObjType; + objPtr->internalRep.dictSubstValue.varNameObjPtr = varObjPtr; + objPtr->internalRep.dictSubstValue.indexObjPtr = keyObjPtr; + } +} + +static Jim_Obj *JimExpandDictSugar(Jim_Interp *interp, Jim_Obj *objPtr) +{ + Jim_Obj *resObjPtr = NULL; + Jim_Obj *substKeyObjPtr = NULL; + + if (interp->safeexpr) { + return objPtr; + } + + SetDictSubstFromAny(interp, objPtr); + + if (Jim_SubstObj(interp, objPtr->internalRep.dictSubstValue.indexObjPtr, + &substKeyObjPtr, JIM_NONE) + != JIM_OK) { + return NULL; + } + Jim_IncrRefCount(substKeyObjPtr); + resObjPtr = + JimDictExpandArrayVariable(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr, + substKeyObjPtr, 0); + Jim_DecrRefCount(interp, substKeyObjPtr); + + return resObjPtr; +} + + +static Jim_CallFrame *JimCreateCallFrame(Jim_Interp *interp, Jim_CallFrame *parent, Jim_Obj *nsObj) +{ + Jim_CallFrame *cf; + + if (interp->freeFramesList) { + cf = interp->freeFramesList; + interp->freeFramesList = cf->next; + + cf->argv = NULL; + cf->argc = 0; + cf->procArgsObjPtr = NULL; + cf->procBodyObjPtr = NULL; + cf->next = NULL; + cf->staticVars = NULL; + cf->localCommands = NULL; + cf->tailcallObj = NULL; + cf->tailcallCmd = NULL; + } + else { + cf = Jim_Alloc(sizeof(*cf)); + memset(cf, 0, sizeof(*cf)); + + Jim_InitHashTable(&cf->vars, &JimVariablesHashTableType, interp); + } + + cf->id = interp->callFrameEpoch++; + cf->parent = parent; + cf->level = parent ? parent->level + 1 : 0; + cf->nsObj = nsObj; + Jim_IncrRefCount(nsObj); + + return cf; +} + +static int JimDeleteLocalProcs(Jim_Interp *interp, Jim_Stack *localCommands) +{ + + if (localCommands) { + Jim_Obj *cmdNameObj; + + while ((cmdNameObj = Jim_StackPop(localCommands)) != NULL) { + Jim_HashTable *ht = &interp->commands; + Jim_HashEntry *he = Jim_FindHashEntry(ht, cmdNameObj); + if (he) { + Jim_Cmd *cmd = Jim_GetHashEntryVal(he); + if (cmd->prevCmd) { + Jim_Cmd *prevCmd = cmd->prevCmd; + cmd->prevCmd = NULL; + + + JimDecrCmdRefCount(interp, cmd); + + + Jim_SetHashVal(ht, he, prevCmd); + } + else { + Jim_DeleteHashEntry(ht, cmdNameObj); + } + } + Jim_DecrRefCount(interp, cmdNameObj); + } + Jim_FreeStack(localCommands); + Jim_Free(localCommands); + } + return JIM_OK; +} + +static int JimInvokeDefer(Jim_Interp *interp, int retcode) +{ + Jim_Obj *objPtr; + + + if (JimFindVariable(&interp->framePtr->vars, interp->defer) == NULL) { + return retcode; + } + objPtr = Jim_GetVariable(interp, interp->defer, JIM_NONE); + + if (objPtr) { + int ret = JIM_OK; + int i; + int listLen = Jim_ListLength(interp, objPtr); + Jim_Obj *resultObjPtr; + + Jim_IncrRefCount(objPtr); + + resultObjPtr = Jim_GetResult(interp); + Jim_IncrRefCount(resultObjPtr); + Jim_SetEmptyResult(interp); + + + for (i = listLen; i > 0; i--) { + + Jim_Obj *scriptObjPtr = Jim_ListGetIndex(interp, objPtr, i - 1); + ret = Jim_EvalObj(interp, scriptObjPtr); + if (ret != JIM_OK) { + break; + } + } + + if (ret == JIM_OK || retcode == JIM_ERR) { + + Jim_SetResult(interp, resultObjPtr); + } + else { + retcode = ret; + } + + Jim_DecrRefCount(interp, resultObjPtr); + Jim_DecrRefCount(interp, objPtr); + } + return retcode; +} + +#define JIM_FCF_FULL 0 +#define JIM_FCF_REUSE 1 +static void JimFreeCallFrame(Jim_Interp *interp, Jim_CallFrame *cf, int action) + { + JimDeleteLocalProcs(interp, cf->localCommands); + + if (cf->procArgsObjPtr) + Jim_DecrRefCount(interp, cf->procArgsObjPtr); + if (cf->procBodyObjPtr) + Jim_DecrRefCount(interp, cf->procBodyObjPtr); + Jim_DecrRefCount(interp, cf->nsObj); + if (action == JIM_FCF_FULL || cf->vars.size != JIM_HT_INITIAL_SIZE) + Jim_FreeHashTable(&cf->vars); + else { + Jim_ClearHashTable(&cf->vars); + } + cf->next = interp->freeFramesList; + interp->freeFramesList = cf; +} + + + +int Jim_IsBigEndian(void) +{ + union { + unsigned short s; + unsigned char c[2]; + } uval = {0x0102}; + + return uval.c[0] == 1; +} + + +Jim_Interp *Jim_CreateInterp(void) +{ + Jim_Interp *i = Jim_Alloc(sizeof(*i)); + + memset(i, 0, sizeof(*i)); + + i->maxCallFrameDepth = JIM_MAX_CALLFRAME_DEPTH; + i->maxEvalDepth = JIM_MAX_EVAL_DEPTH; + i->lastCollectTime = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW); + + Jim_InitHashTable(&i->commands, &JimCommandsHashTableType, i); +#ifdef JIM_REFERENCES + Jim_InitHashTable(&i->references, &JimReferencesHashTableType, i); +#endif + Jim_InitHashTable(&i->assocData, &JimAssocDataHashTableType, i); + Jim_InitHashTable(&i->packages, &JimPackageHashTableType, NULL); + i->emptyObj = Jim_NewEmptyStringObj(i); + i->trueObj = Jim_NewIntObj(i, 1); + i->falseObj = Jim_NewIntObj(i, 0); + i->framePtr = i->topFramePtr = JimCreateCallFrame(i, NULL, i->emptyObj); + i->result = i->emptyObj; + i->stackTrace = Jim_NewListObj(i, NULL, 0); + i->unknown = Jim_NewStringObj(i, "unknown", -1); + i->defer = Jim_NewStringObj(i, "jim::defer", -1); + i->errorProc = i->emptyObj; + i->nullScriptObj = Jim_NewEmptyStringObj(i); + i->evalFrame = &i->topEvalFrame; + i->currentFilenameObj = Jim_NewEmptyStringObj(i); + Jim_IncrRefCount(i->emptyObj); + Jim_IncrRefCount(i->result); + Jim_IncrRefCount(i->stackTrace); + Jim_IncrRefCount(i->unknown); + Jim_IncrRefCount(i->defer); + Jim_IncrRefCount(i->nullScriptObj); + Jim_IncrRefCount(i->errorProc); + Jim_IncrRefCount(i->trueObj); + Jim_IncrRefCount(i->falseObj); + Jim_IncrRefCount(i->currentFilenameObj); + + + Jim_SetVariableStrWithStr(i, JIM_LIBPATH, TCL_LIBRARY); + Jim_SetVariableStrWithStr(i, JIM_INTERACTIVE, "0"); + + Jim_SetVariableStrWithStr(i, "tcl_platform(engine)", "Jim"); + Jim_SetVariableStrWithStr(i, "tcl_platform(os)", TCL_PLATFORM_OS); + Jim_SetVariableStrWithStr(i, "tcl_platform(platform)", TCL_PLATFORM_PLATFORM); + Jim_SetVariableStrWithStr(i, "tcl_platform(pathSeparator)", TCL_PLATFORM_PATH_SEPARATOR); + Jim_SetVariableStrWithStr(i, "tcl_platform(byteOrder)", Jim_IsBigEndian() ? "bigEndian" : "littleEndian"); + Jim_SetVariableStrWithStr(i, "tcl_platform(threaded)", "0"); + Jim_SetVariableStrWithStr(i, "tcl_platform(bootstrap)", "0"); + Jim_SetVariableStr(i, "tcl_platform(pointerSize)", Jim_NewIntObj(i, sizeof(void *))); + Jim_SetVariableStr(i, "tcl_platform(wordSize)", Jim_NewIntObj(i, sizeof(jim_wide))); + Jim_SetVariableStr(i, "tcl_platform(stackFormat)", Jim_NewIntObj(i, 4)); + + return i; +} + +void Jim_FreeInterp(Jim_Interp *i) +{ + Jim_CallFrame *cf, *cfx; + + Jim_Obj *objPtr, *nextObjPtr; + + i->quitting = 1; + + + for (cf = i->framePtr; cf; cf = cfx) { + + JimInvokeDefer(i, JIM_OK); + cfx = cf->parent; + JimFreeCallFrame(i, cf, JIM_FCF_FULL); + } + + + Jim_FreeHashTable(&i->commands); + + Jim_DecrRefCount(i, i->emptyObj); + Jim_DecrRefCount(i, i->trueObj); + Jim_DecrRefCount(i, i->falseObj); + Jim_DecrRefCount(i, i->result); + Jim_DecrRefCount(i, i->stackTrace); + Jim_DecrRefCount(i, i->errorProc); + Jim_DecrRefCount(i, i->unknown); + Jim_DecrRefCount(i, i->defer); + Jim_DecrRefCount(i, i->nullScriptObj); + Jim_DecrRefCount(i, i->currentFilenameObj); + + + Jim_InterpIncrProcEpoch(i); + +#ifdef JIM_REFERENCES + Jim_FreeHashTable(&i->references); +#endif + Jim_FreeHashTable(&i->packages); + Jim_Free(i->prngState); + Jim_FreeHashTable(&i->assocData); + if (i->traceCmdObj) { + Jim_DecrRefCount(i, i->traceCmdObj); + } + +#ifdef JIM_MAINTAINER + if (i->liveList != NULL) { + objPtr = i->liveList; + + printf("\n-------------------------------------\n"); + printf("Objects still in the free list:\n"); + while (objPtr) { + const char *type = objPtr->typePtr ? objPtr->typePtr->name : "string"; + Jim_String(objPtr); + + if (objPtr->bytes && strlen(objPtr->bytes) > 20) { + printf("%p (%d) %-10s: '%.20s...'\n", + (void *)objPtr, objPtr->refCount, type, objPtr->bytes); + } + else { + printf("%p (%d) %-10s: '%s'\n", + (void *)objPtr, objPtr->refCount, type, objPtr->bytes ? objPtr->bytes : "(null)"); + } + if (objPtr->typePtr == &sourceObjType) { + printf("FILE %s LINE %d\n", + Jim_String(objPtr->internalRep.sourceValue.fileNameObj), + objPtr->internalRep.sourceValue.lineNumber); + } + objPtr = objPtr->nextObjPtr; + } + printf("-------------------------------------\n\n"); + JimPanic((1, "Live list non empty freeing the interpreter! Leak?")); + } +#endif + + + objPtr = i->freeList; + while (objPtr) { + nextObjPtr = objPtr->nextObjPtr; + Jim_Free(objPtr); + objPtr = nextObjPtr; + } + + + for (cf = i->freeFramesList; cf; cf = cfx) { + cfx = cf->next; + if (cf->vars.table) + Jim_FreeHashTable(&cf->vars); + Jim_Free(cf); + } + + + Jim_Free(i); +} + +Jim_CallFrame *Jim_GetCallFrameByLevel(Jim_Interp *interp, Jim_Obj *levelObjPtr) +{ + long level; + const char *str; + Jim_CallFrame *framePtr; + + if (levelObjPtr) { + str = Jim_String(levelObjPtr); + if (str[0] == '#') { + char *endptr; + + level = jim_strtol(str + 1, &endptr); + if (str[1] == '\0' || endptr[0] != '\0') { + level = -1; + } + } + else { + if (Jim_GetLong(interp, levelObjPtr, &level) != JIM_OK || level < 0) { + level = -1; + } + else { + + level = interp->framePtr->level - level; + } + } + } + else { + str = "1"; + level = interp->framePtr->level - 1; + } + + if (level == 0) { + return interp->topFramePtr; + } + if (level > 0) { + + for (framePtr = interp->framePtr; framePtr; framePtr = framePtr->parent) { + if (framePtr->level == level) { + return framePtr; + } + } + } + + Jim_SetResultFormatted(interp, "bad level \"%s\"", str); + return NULL; +} + +static Jim_CallFrame *JimGetCallFrameByInteger(Jim_Interp *interp, long level) +{ + Jim_CallFrame *framePtr; + + if (level == 0) { + return interp->framePtr; + } + + if (level < 0) { + + level = interp->framePtr->level + level; + } + + if (level > 0) { + + for (framePtr = interp->framePtr; framePtr; framePtr = framePtr->parent) { + if (framePtr->level == level) { + return framePtr; + } + } + } + return NULL; +} + +static Jim_EvalFrame *JimGetEvalFrameByProcLevel(Jim_Interp *interp, int proclevel) +{ + Jim_EvalFrame *evalFrame; + + if (proclevel == 0) { + return interp->evalFrame; + } + + if (proclevel < 0) { + + proclevel = interp->procLevel + proclevel; + } + + if (proclevel >= 0) { + + for (evalFrame = interp->evalFrame; evalFrame; evalFrame = evalFrame->parent) { + if (evalFrame->procLevel == proclevel) { + return evalFrame; + } + } + } + return NULL; +} + +static Jim_Obj *JimProcForEvalFrame(Jim_Interp *interp, Jim_EvalFrame *frame) +{ + if (frame == interp->evalFrame || (frame->cmd && frame->cmd->cmdNameObj)) { + Jim_EvalFrame *e; + for (e = frame->parent; e; e = e->parent) { + if (e->cmd && e->cmd->isproc && e->cmd->cmdNameObj) { + break; + } + } + if (e && e->cmd && e->cmd->cmdNameObj) { + return e->cmd->cmdNameObj; + } + } + return NULL; +} + +static void JimAddStackFrame(Jim_Interp *interp, Jim_EvalFrame *frame, Jim_Obj *listObj) +{ + Jim_Obj *procNameObj = JimProcForEvalFrame(interp, frame); + Jim_Obj *fileNameObj = interp->emptyObj; + int linenr = 1; + + if (frame->scriptObj) { + ScriptObj *script = JimGetScript(interp, frame->scriptObj); + fileNameObj = script->fileNameObj; + linenr = script->linenr; + } + + Jim_ListAppendElement(interp, listObj, procNameObj ? procNameObj : interp->emptyObj); + Jim_ListAppendElement(interp, listObj, fileNameObj); + Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, linenr)); + Jim_ListAppendElement(interp, listObj, Jim_NewListObj(interp, frame->argv, frame->argc)); +} + +static void JimSetStackTrace(Jim_Interp *interp, Jim_Obj *stackTraceObj) +{ + + Jim_IncrRefCount(stackTraceObj); + Jim_DecrRefCount(interp, interp->stackTrace); + interp->stackTrace = stackTraceObj; + interp->errorFlag = 1; +} + +static void JimSetErrorStack(Jim_Interp *interp, ScriptObj *script) +{ + if (!interp->errorFlag) { + int i; + Jim_Obj *stackTrace = Jim_NewListObj(interp, NULL, 0); + + if (interp->procLevel == 0 && script) { + Jim_ListAppendElement(interp, stackTrace, interp->emptyObj); + Jim_ListAppendElement(interp, stackTrace, script->fileNameObj); + Jim_ListAppendElement(interp, stackTrace, Jim_NewIntObj(interp, script->linenr)); + Jim_ListAppendElement(interp, stackTrace, interp->emptyObj); + } + else { + for (i = 0; i <= interp->procLevel; i++) { + Jim_EvalFrame *frame = JimGetEvalFrameByProcLevel(interp, -i); + if (frame) { + JimAddStackFrame(interp, frame, stackTrace); + } + } + } + JimSetStackTrace(interp, stackTrace); + } +} + +int Jim_SetAssocData(Jim_Interp *interp, const char *key, Jim_InterpDeleteProc * delProc, + void *data) +{ + AssocDataValue *assocEntryPtr = (AssocDataValue *) Jim_Alloc(sizeof(AssocDataValue)); + + assocEntryPtr->delProc = delProc; + assocEntryPtr->data = data; + return Jim_AddHashEntry(&interp->assocData, key, assocEntryPtr); +} + +void *Jim_GetAssocData(Jim_Interp *interp, const char *key) +{ + Jim_HashEntry *entryPtr = Jim_FindHashEntry(&interp->assocData, key); + + if (entryPtr != NULL) { + AssocDataValue *assocEntryPtr = Jim_GetHashEntryVal(entryPtr); + return assocEntryPtr->data; + } + return NULL; +} + +int Jim_DeleteAssocData(Jim_Interp *interp, const char *key) +{ + return Jim_DeleteHashEntry(&interp->assocData, key); +} + +int Jim_GetExitCode(Jim_Interp *interp) +{ + return interp->exitCode; +} + +static void UpdateStringOfInt(struct Jim_Obj *objPtr); +static int SetIntFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags); + +static const Jim_ObjType intObjType = { + "int", + NULL, + NULL, + UpdateStringOfInt, + JIM_TYPE_NONE, +}; + +static const Jim_ObjType coercedDoubleObjType = { + "coerced-double", + NULL, + NULL, + UpdateStringOfInt, + JIM_TYPE_NONE, +}; + + +static void UpdateStringOfInt(struct Jim_Obj *objPtr) +{ + char buf[JIM_INTEGER_SPACE + 1]; + jim_wide wideValue = JimWideValue(objPtr); + int pos = 0; + + if (wideValue == 0) { + buf[pos++] = '0'; + } + else { + char tmp[JIM_INTEGER_SPACE]; + int num = 0; + int i; + + if (wideValue < 0) { + buf[pos++] = '-'; + i = wideValue % 10; + tmp[num++] = (i > 0) ? (10 - i) : -i; + wideValue /= -10; + } + + while (wideValue) { + tmp[num++] = wideValue % 10; + wideValue /= 10; + } + + for (i = 0; i < num; i++) { + buf[pos++] = '0' + tmp[num - i - 1]; + } + } + buf[pos] = 0; + + JimSetStringBytes(objPtr, buf); +} + +static int SetIntFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags) +{ + jim_wide wideValue; + const char *str; + + if (objPtr->typePtr == &coercedDoubleObjType) { + + objPtr->typePtr = &intObjType; + return JIM_OK; + } + + + str = Jim_String(objPtr); + + if (Jim_StringToWide(str, &wideValue, 0) != JIM_OK) { + if (flags & JIM_ERRMSG) { + Jim_SetResultFormatted(interp, "expected integer but got \"%#s\"", objPtr); + } + return JIM_ERR; + } + if ((wideValue == JIM_WIDE_MIN || wideValue == JIM_WIDE_MAX) && errno == ERANGE) { + Jim_SetResultString(interp, "Integer value too big to be represented", -1); + return JIM_ERR; + } + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &intObjType; + objPtr->internalRep.wideValue = wideValue; + return JIM_OK; +} + +#ifdef JIM_OPTIMIZATION +static int JimIsWide(Jim_Obj *objPtr) +{ + return objPtr->typePtr == &intObjType; +} +#endif + +int Jim_GetWide(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr) +{ + if (objPtr->typePtr != &intObjType && SetIntFromAny(interp, objPtr, JIM_ERRMSG) == JIM_ERR) + return JIM_ERR; + *widePtr = JimWideValue(objPtr); + return JIM_OK; +} + +int Jim_GetWideExpr(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr) +{ + int ret = JIM_OK; + + if (objPtr->typePtr == &sourceObjType || objPtr->typePtr == NULL) { + SetIntFromAny(interp, objPtr, 0); + } + if (objPtr->typePtr == &intObjType) { + *widePtr = JimWideValue(objPtr); + } + else { + JimPanic((interp->safeexpr, "interp->safeexpr is set")); + interp->safeexpr++; + ret = Jim_EvalExpression(interp, objPtr); + interp->safeexpr--; + + if (ret == JIM_OK) { + ret = Jim_GetWide(interp, Jim_GetResult(interp), widePtr); + } + if (ret != JIM_OK) { + Jim_SetResultFormatted(interp, "expected integer expression but got \"%#s\"", objPtr); + } + } + return ret; +} + + +static int JimGetWideNoErr(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr) +{ + if (objPtr->typePtr != &intObjType && SetIntFromAny(interp, objPtr, JIM_NONE) == JIM_ERR) + return JIM_ERR; + *widePtr = JimWideValue(objPtr); + return JIM_OK; +} + +int Jim_GetLong(Jim_Interp *interp, Jim_Obj *objPtr, long *longPtr) +{ + jim_wide wideValue; + int retval; + + retval = Jim_GetWide(interp, objPtr, &wideValue); + if (retval == JIM_OK) { + *longPtr = (long)wideValue; + return JIM_OK; + } + return JIM_ERR; +} + +Jim_Obj *Jim_NewIntObj(Jim_Interp *interp, jim_wide wideValue) +{ + Jim_Obj *objPtr; + + objPtr = Jim_NewObj(interp); + objPtr->typePtr = &intObjType; + objPtr->bytes = NULL; + objPtr->internalRep.wideValue = wideValue; + return objPtr; +} + +#define JIM_DOUBLE_SPACE 30 + +static void UpdateStringOfDouble(struct Jim_Obj *objPtr); +static int SetDoubleFromAny(Jim_Interp *interp, Jim_Obj *objPtr); + +static const Jim_ObjType doubleObjType = { + "double", + NULL, + NULL, + UpdateStringOfDouble, + JIM_TYPE_NONE, +}; + +#if !HAVE_DECL_ISNAN +#undef isnan +#define isnan(X) ((X) != (X)) +#endif +#if !HAVE_DECL_ISINF +#undef isinf +#define isinf(X) (1.0 / (X) == 0.0) +#endif + +static void UpdateStringOfDouble(struct Jim_Obj *objPtr) +{ + double value = objPtr->internalRep.doubleValue; + + if (isnan(value)) { + JimSetStringBytes(objPtr, "NaN"); + return; + } + if (isinf(value)) { + if (value < 0) { + JimSetStringBytes(objPtr, "-Inf"); + } + else { + JimSetStringBytes(objPtr, "Inf"); + } + return; + } + { + char buf[JIM_DOUBLE_SPACE + 1]; + int i; + int len = sprintf(buf, "%.12g", value); + + + for (i = 0; i < len; i++) { + if (buf[i] == '.' || buf[i] == 'e') { +#if defined(JIM_SPRINTF_DOUBLE_NEEDS_FIX) + char *e = strchr(buf, 'e'); + if (e && (e[1] == '-' || e[1] == '+') && e[2] == '0') { + + e += 2; + memmove(e, e + 1, len - (e - buf)); + } +#endif + break; + } + } + if (buf[i] == '\0') { + buf[i++] = '.'; + buf[i++] = '0'; + buf[i] = '\0'; + } + JimSetStringBytes(objPtr, buf); + } +} + +static int SetDoubleFromAny(Jim_Interp *interp, Jim_Obj *objPtr) +{ + double doubleValue; + jim_wide wideValue; + const char *str; + +#ifdef HAVE_LONG_LONG + +#define MIN_INT_IN_DOUBLE -(1LL << 53) +#define MAX_INT_IN_DOUBLE -(MIN_INT_IN_DOUBLE + 1) + + if (objPtr->typePtr == &intObjType + && JimWideValue(objPtr) >= MIN_INT_IN_DOUBLE + && JimWideValue(objPtr) <= MAX_INT_IN_DOUBLE) { + + + objPtr->typePtr = &coercedDoubleObjType; + return JIM_OK; + } +#endif + str = Jim_String(objPtr); + + if (Jim_StringToWide(str, &wideValue, 10) == JIM_OK) { + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &coercedDoubleObjType; + objPtr->internalRep.wideValue = wideValue; + return JIM_OK; + } + else { + + if (Jim_StringToDouble(str, &doubleValue) != JIM_OK) { + Jim_SetResultFormatted(interp, "expected floating-point number but got \"%#s\"", objPtr); + return JIM_ERR; + } + + Jim_FreeIntRep(interp, objPtr); + } + objPtr->typePtr = &doubleObjType; + objPtr->internalRep.doubleValue = doubleValue; + return JIM_OK; +} + +int Jim_GetDouble(Jim_Interp *interp, Jim_Obj *objPtr, double *doublePtr) +{ + if (objPtr->typePtr == &coercedDoubleObjType) { + *doublePtr = JimWideValue(objPtr); + return JIM_OK; + } + if (objPtr->typePtr != &doubleObjType && SetDoubleFromAny(interp, objPtr) == JIM_ERR) + return JIM_ERR; + + if (objPtr->typePtr == &coercedDoubleObjType) { + *doublePtr = JimWideValue(objPtr); + } + else { + *doublePtr = objPtr->internalRep.doubleValue; + } + return JIM_OK; +} + +Jim_Obj *Jim_NewDoubleObj(Jim_Interp *interp, double doubleValue) +{ + Jim_Obj *objPtr; + + objPtr = Jim_NewObj(interp); + objPtr->typePtr = &doubleObjType; + objPtr->bytes = NULL; + objPtr->internalRep.doubleValue = doubleValue; + return objPtr; +} + +static int SetBooleanFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags); + +int Jim_GetBoolean(Jim_Interp *interp, Jim_Obj *objPtr, int * booleanPtr) +{ + if (objPtr->typePtr != &intObjType && SetBooleanFromAny(interp, objPtr, JIM_ERRMSG) == JIM_ERR) + return JIM_ERR; + *booleanPtr = (int) JimWideValue(objPtr); + return JIM_OK; +} + +static const char * const jim_true_false_strings[8] = { + "1", "true", "yes", "on", + "0", "false", "no", "off" +}; + +static const int jim_true_false_lens[8] = { + 1, 4, 3, 2, + 1, 5, 2, 3, +}; + +static int SetBooleanFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags) +{ + int index = Jim_FindByName(Jim_String(objPtr), jim_true_false_strings, + sizeof(jim_true_false_strings) / sizeof(*jim_true_false_strings)); + if (index < 0) { + if (flags & JIM_ERRMSG) { + Jim_SetResultFormatted(interp, "expected boolean but got \"%#s\"", objPtr); + } + return JIM_ERR; + } + + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &intObjType; + + objPtr->internalRep.wideValue = index < 4 ? 1 : 0; + return JIM_OK; +} + +static void ListInsertElements(Jim_Obj *listPtr, int idx, int elemc, Jim_Obj *const *elemVec); +static void ListAppendElement(Jim_Obj *listPtr, Jim_Obj *objPtr); +static void FreeListInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupListInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); +static void UpdateStringOfList(struct Jim_Obj *objPtr); +static int SetListFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); + +static const Jim_ObjType listObjType = { + "list", + FreeListInternalRep, + DupListInternalRep, + UpdateStringOfList, + JIM_TYPE_NONE, +}; + +void FreeListInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + int i; + + for (i = 0; i < objPtr->internalRep.listValue.len; i++) { + Jim_DecrRefCount(interp, objPtr->internalRep.listValue.ele[i]); + } + Jim_Free(objPtr->internalRep.listValue.ele); +} + +void DupListInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + int i; + + JIM_NOTUSED(interp); + + dupPtr->internalRep.listValue.len = srcPtr->internalRep.listValue.len; + dupPtr->internalRep.listValue.maxLen = srcPtr->internalRep.listValue.maxLen; + dupPtr->internalRep.listValue.ele = + Jim_Alloc(sizeof(Jim_Obj *) * srcPtr->internalRep.listValue.maxLen); + memcpy(dupPtr->internalRep.listValue.ele, srcPtr->internalRep.listValue.ele, + sizeof(Jim_Obj *) * srcPtr->internalRep.listValue.len); + for (i = 0; i < dupPtr->internalRep.listValue.len; i++) { + Jim_IncrRefCount(dupPtr->internalRep.listValue.ele[i]); + } + dupPtr->typePtr = &listObjType; +} + +#define JIM_ELESTR_SIMPLE 0 +#define JIM_ELESTR_BRACE 1 +#define JIM_ELESTR_QUOTE 2 +static unsigned char ListElementQuotingType(const char *s, int len) +{ + int i, level, blevel, trySimple = 1; + + + if (len == 0) + return JIM_ELESTR_BRACE; + if (s[0] == '"' || s[0] == '{') { + trySimple = 0; + goto testbrace; + } + for (i = 0; i < len; i++) { + switch (s[i]) { + case ' ': + case '$': + case '"': + case '[': + case ']': + case ';': + case '\\': + case '\r': + case '\n': + case '\t': + case '\f': + case '\v': + trySimple = 0; + + case '{': + case '}': + goto testbrace; + } + } + return JIM_ELESTR_SIMPLE; + + testbrace: + + if (s[len - 1] == '\\') + return JIM_ELESTR_QUOTE; + level = 0; + blevel = 0; + for (i = 0; i < len; i++) { + switch (s[i]) { + case '{': + level++; + break; + case '}': + level--; + if (level < 0) + return JIM_ELESTR_QUOTE; + break; + case '[': + blevel++; + break; + case ']': + blevel--; + break; + case '\\': + if (s[i + 1] == '\n') + return JIM_ELESTR_QUOTE; + else if (s[i + 1] != '\0') + i++; + break; + } + } + if (blevel < 0) { + return JIM_ELESTR_QUOTE; + } + + if (level == 0) { + if (!trySimple) + return JIM_ELESTR_BRACE; + for (i = 0; i < len; i++) { + switch (s[i]) { + case ' ': + case '$': + case '"': + case '[': + case ']': + case ';': + case '\\': + case '\r': + case '\n': + case '\t': + case '\f': + case '\v': + return JIM_ELESTR_BRACE; + break; + } + } + return JIM_ELESTR_SIMPLE; + } + return JIM_ELESTR_QUOTE; +} + +static int BackslashQuoteString(const char *s, int len, char *q) +{ + char *p = q; + + while (len--) { + switch (*s) { + case ' ': + case '$': + case '"': + case '[': + case ']': + case '{': + case '}': + case ';': + case '\\': + *p++ = '\\'; + *p++ = *s++; + break; + case '\n': + *p++ = '\\'; + *p++ = 'n'; + s++; + break; + case '\r': + *p++ = '\\'; + *p++ = 'r'; + s++; + break; + case '\t': + *p++ = '\\'; + *p++ = 't'; + s++; + break; + case '\f': + *p++ = '\\'; + *p++ = 'f'; + s++; + break; + case '\v': + *p++ = '\\'; + *p++ = 'v'; + s++; + break; + default: + *p++ = *s++; + break; + } + } + *p = '\0'; + + return p - q; +} + +static void JimMakeListStringRep(Jim_Obj *objPtr, Jim_Obj **objv, int objc) +{ + #define STATIC_QUOTING_LEN 32 + int i, bufLen, realLength; + const char *strRep; + char *p; + unsigned char *quotingType, staticQuoting[STATIC_QUOTING_LEN]; + + + if (objc > STATIC_QUOTING_LEN) { + quotingType = Jim_Alloc(objc); + } + else { + quotingType = staticQuoting; + } + bufLen = 0; + for (i = 0; i < objc; i++) { + int len; + + strRep = Jim_GetString(objv[i], &len); + quotingType[i] = ListElementQuotingType(strRep, len); + switch (quotingType[i]) { + case JIM_ELESTR_SIMPLE: + if (i != 0 || strRep[0] != '#') { + bufLen += len; + break; + } + + quotingType[i] = JIM_ELESTR_BRACE; + + case JIM_ELESTR_BRACE: + bufLen += len + 2; + break; + case JIM_ELESTR_QUOTE: + bufLen += len * 2; + break; + } + bufLen++; + } + bufLen++; + + + p = objPtr->bytes = Jim_Alloc(bufLen + 1); + realLength = 0; + for (i = 0; i < objc; i++) { + int len, qlen; + + strRep = Jim_GetString(objv[i], &len); + + switch (quotingType[i]) { + case JIM_ELESTR_SIMPLE: + memcpy(p, strRep, len); + p += len; + realLength += len; + break; + case JIM_ELESTR_BRACE: + *p++ = '{'; + memcpy(p, strRep, len); + p += len; + *p++ = '}'; + realLength += len + 2; + break; + case JIM_ELESTR_QUOTE: + if (i == 0 && strRep[0] == '#') { + *p++ = '\\'; + realLength++; + } + qlen = BackslashQuoteString(strRep, len, p); + p += qlen; + realLength += qlen; + break; + } + + if (i + 1 != objc) { + *p++ = ' '; + realLength++; + } + } + *p = '\0'; + objPtr->length = realLength; + + if (quotingType != staticQuoting) { + Jim_Free(quotingType); + } +} + +static void UpdateStringOfList(struct Jim_Obj *objPtr) +{ + JimMakeListStringRep(objPtr, objPtr->internalRep.listValue.ele, objPtr->internalRep.listValue.len); +} + +static int SetListFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) +{ + struct JimParserCtx parser; + const char *str; + int strLen; + Jim_Obj *fileNameObj; + int linenr; + + if (objPtr->typePtr == &listObjType) { + return JIM_OK; + } + + + if (Jim_IsDict(objPtr) && objPtr->bytes == NULL) { + Jim_Dict *dict = objPtr->internalRep.dictValue; + + + objPtr->typePtr = &listObjType; + objPtr->internalRep.listValue.len = dict->len; + objPtr->internalRep.listValue.maxLen = dict->maxLen; + objPtr->internalRep.listValue.ele = dict->table; + + + Jim_Free(dict->ht); + + + Jim_Free(dict); + return JIM_OK; + } + + + fileNameObj = Jim_GetSourceInfo(interp, objPtr, &linenr); + Jim_IncrRefCount(fileNameObj); + + + str = Jim_GetString(objPtr, &strLen); + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &listObjType; + objPtr->internalRep.listValue.len = 0; + objPtr->internalRep.listValue.maxLen = 0; + objPtr->internalRep.listValue.ele = NULL; + + + if (strLen) { + JimParserInit(&parser, str, strLen, linenr); + while (!parser.eof) { + Jim_Obj *elementPtr; + + JimParseList(&parser); + if (parser.tt != JIM_TT_STR && parser.tt != JIM_TT_ESC) + continue; + elementPtr = JimParserGetTokenObj(interp, &parser); + Jim_SetSourceInfo(interp, elementPtr, fileNameObj, parser.tline); + ListAppendElement(objPtr, elementPtr); + } + } + Jim_DecrRefCount(interp, fileNameObj); + return JIM_OK; +} + +Jim_Obj *Jim_NewListObj(Jim_Interp *interp, Jim_Obj *const *elements, int len) +{ + Jim_Obj *objPtr; + + objPtr = Jim_NewObj(interp); + objPtr->typePtr = &listObjType; + objPtr->bytes = NULL; + objPtr->internalRep.listValue.ele = NULL; + objPtr->internalRep.listValue.len = 0; + objPtr->internalRep.listValue.maxLen = 0; + + if (len) { + ListInsertElements(objPtr, 0, len, elements); + } + + return objPtr; +} + +static void JimListGetElements(Jim_Interp *interp, Jim_Obj *listObj, int *listLen, + Jim_Obj ***listVec) +{ + *listLen = Jim_ListLength(interp, listObj); + *listVec = listObj->internalRep.listValue.ele; +} + + +static int JimSign(jim_wide w) +{ + if (w == 0) { + return 0; + } + else if (w < 0) { + return -1; + } + return 1; +} + + +struct lsort_info { + jmp_buf jmpbuf; + Jim_Obj *command; + Jim_Interp *interp; + enum { + JIM_LSORT_ASCII, + JIM_LSORT_NOCASE, + JIM_LSORT_INTEGER, + JIM_LSORT_REAL, + JIM_LSORT_COMMAND, + JIM_LSORT_DICT + } type; + int order; + Jim_Obj **indexv; + int indexc; + int unique; + int (*subfn)(Jim_Obj **, Jim_Obj **); +}; + +static struct lsort_info *sort_info; + +static int ListSortIndexHelper(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + Jim_Obj *lObj, *rObj; + + if (Jim_ListIndices(sort_info->interp, *lhsObj, sort_info->indexv, sort_info->indexc, &lObj, JIM_ERRMSG) != JIM_OK || + Jim_ListIndices(sort_info->interp, *rhsObj, sort_info->indexv, sort_info->indexc, &rObj, JIM_ERRMSG) != JIM_OK) { + longjmp(sort_info->jmpbuf, JIM_ERR); + } + return sort_info->subfn(&lObj, &rObj); +} + + +static int ListSortString(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + return Jim_StringCompareObj(sort_info->interp, *lhsObj, *rhsObj, 0) * sort_info->order; +} + +static int ListSortStringNoCase(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + return Jim_StringCompareObj(sort_info->interp, *lhsObj, *rhsObj, 1) * sort_info->order; +} + +static int ListSortDict(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + + const char *left = Jim_String(*lhsObj); + const char *right = Jim_String(*rhsObj); + + while (1) { + if (isdigit(UCHAR(*left)) && isdigit(UCHAR(*right))) { + + jim_wide lint, rint; + char *lend, *rend; + lint = jim_strtoull(left, &lend); + rint = jim_strtoull(right, &rend); + if (lint != rint) { + return JimSign(lint - rint) * sort_info->order; + } + if (lend -left != rend - right) { + return JimSign((lend - left) - (rend - right)) * sort_info->order; + } + left = lend; + right = rend; + } + else { + int cl, cr; + left += utf8_tounicode_case(left, &cl, 1); + right += utf8_tounicode_case(right, &cr, 1); + if (cl != cr) { + return JimSign(cl - cr) * sort_info->order; + } + if (cl == 0) { + + return Jim_StringCompareObj(sort_info->interp, *lhsObj, *rhsObj, 0) * sort_info->order; + } + } + } +} + +static int ListSortInteger(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + jim_wide lhs = 0, rhs = 0; + + if (Jim_GetWide(sort_info->interp, *lhsObj, &lhs) != JIM_OK || + Jim_GetWide(sort_info->interp, *rhsObj, &rhs) != JIM_OK) { + longjmp(sort_info->jmpbuf, JIM_ERR); + } + + return JimSign(lhs - rhs) * sort_info->order; +} + +static int ListSortReal(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + double lhs = 0, rhs = 0; + + if (Jim_GetDouble(sort_info->interp, *lhsObj, &lhs) != JIM_OK || + Jim_GetDouble(sort_info->interp, *rhsObj, &rhs) != JIM_OK) { + longjmp(sort_info->jmpbuf, JIM_ERR); + } + if (lhs == rhs) { + return 0; + } + if (lhs > rhs) { + return sort_info->order; + } + return -sort_info->order; +} + +static int ListSortCommand(Jim_Obj **lhsObj, Jim_Obj **rhsObj) +{ + Jim_Obj *compare_script; + int rc; + + jim_wide ret = 0; + + + compare_script = Jim_DuplicateObj(sort_info->interp, sort_info->command); + Jim_ListAppendElement(sort_info->interp, compare_script, *lhsObj); + Jim_ListAppendElement(sort_info->interp, compare_script, *rhsObj); + + rc = Jim_EvalObj(sort_info->interp, compare_script); + + if (rc != JIM_OK || Jim_GetWide(sort_info->interp, Jim_GetResult(sort_info->interp), &ret) != JIM_OK) { + longjmp(sort_info->jmpbuf, rc); + } + + return JimSign(ret) * sort_info->order; +} + +static void ListRemoveDuplicates(Jim_Obj *listObjPtr, int (*comp)(Jim_Obj **lhs, Jim_Obj **rhs)) +{ + int src; + int dst = 0; + Jim_Obj **ele = listObjPtr->internalRep.listValue.ele; + + for (src = 1; src < listObjPtr->internalRep.listValue.len; src++) { + if (comp(&ele[dst], &ele[src]) == 0) { + + Jim_DecrRefCount(sort_info->interp, ele[dst]); + } + else { + + dst++; + } + ele[dst] = ele[src]; + } + + + dst++; + if (dst < listObjPtr->internalRep.listValue.len) { + ele[dst] = ele[src]; + } + + + listObjPtr->internalRep.listValue.len = dst; +} + + +static int ListSortElements(Jim_Interp *interp, Jim_Obj *listObjPtr, struct lsort_info *info) +{ + struct lsort_info *prev_info; + + typedef int (qsort_comparator) (const void *, const void *); + int (*fn) (Jim_Obj **, Jim_Obj **); + Jim_Obj **vector; + int len; + int rc; + + JimPanic((Jim_IsShared(listObjPtr), "ListSortElements called with shared object")); + SetListFromAny(interp, listObjPtr); + + + prev_info = sort_info; + sort_info = info; + + vector = listObjPtr->internalRep.listValue.ele; + len = listObjPtr->internalRep.listValue.len; + switch (info->type) { + case JIM_LSORT_ASCII: + fn = ListSortString; + break; + case JIM_LSORT_NOCASE: + fn = ListSortStringNoCase; + break; + case JIM_LSORT_INTEGER: + fn = ListSortInteger; + break; + case JIM_LSORT_REAL: + fn = ListSortReal; + break; + case JIM_LSORT_COMMAND: + fn = ListSortCommand; + break; + case JIM_LSORT_DICT: + fn = ListSortDict; + break; + default: + fn = NULL; + JimPanic((1, "ListSort called with invalid sort type")); + return -1; + } + + if (info->indexc) { + + info->subfn = fn; + fn = ListSortIndexHelper; + } + + if ((rc = setjmp(info->jmpbuf)) == 0) { + qsort(vector, len, sizeof(Jim_Obj *), (qsort_comparator *) fn); + + if (info->unique && len > 1) { + ListRemoveDuplicates(listObjPtr, fn); + } + + Jim_InvalidateStringRep(listObjPtr); + } + sort_info = prev_info; + + return rc; +} + + +static void ListEnsureLength(Jim_Obj *listPtr, int idx) +{ + assert(idx >= 0); + if (idx >= listPtr->internalRep.listValue.maxLen) { + if (idx < 4) { + + idx = 4; + } + listPtr->internalRep.listValue.ele = Jim_Realloc(listPtr->internalRep.listValue.ele, + sizeof(Jim_Obj *) * idx); + + listPtr->internalRep.listValue.maxLen = idx; + } +} + +static void ListInsertElements(Jim_Obj *listPtr, int idx, int elemc, Jim_Obj *const *elemVec) +{ + int currentLen = listPtr->internalRep.listValue.len; + int requiredLen = currentLen + elemc; + int i; + Jim_Obj **point; + + if (elemc == 0) { + + return; + } + + if (requiredLen > listPtr->internalRep.listValue.maxLen) { + if (currentLen) { + + requiredLen *= 2; + } + ListEnsureLength(listPtr, requiredLen); + } + if (idx < 0) { + idx = currentLen; + } + point = listPtr->internalRep.listValue.ele + idx; + memmove(point + elemc, point, (currentLen - idx) * sizeof(Jim_Obj *)); + for (i = 0; i < elemc; ++i) { + point[i] = elemVec[i]; + Jim_IncrRefCount(point[i]); + } + listPtr->internalRep.listValue.len += elemc; +} + +static void ListAppendElement(Jim_Obj *listPtr, Jim_Obj *objPtr) +{ + ListInsertElements(listPtr, -1, 1, &objPtr); +} + +static void ListAppendList(Jim_Obj *listPtr, Jim_Obj *appendListPtr) +{ + ListInsertElements(listPtr, -1, + appendListPtr->internalRep.listValue.len, appendListPtr->internalRep.listValue.ele); +} + +void Jim_ListAppendElement(Jim_Interp *interp, Jim_Obj *listPtr, Jim_Obj *objPtr) +{ + JimPanic((Jim_IsShared(listPtr), "Jim_ListAppendElement called with shared object")); + SetListFromAny(interp, listPtr); + Jim_InvalidateStringRep(listPtr); + ListAppendElement(listPtr, objPtr); +} + +void Jim_ListAppendList(Jim_Interp *interp, Jim_Obj *listPtr, Jim_Obj *appendListPtr) +{ + JimPanic((Jim_IsShared(listPtr), "Jim_ListAppendList called with shared object")); + SetListFromAny(interp, listPtr); + SetListFromAny(interp, appendListPtr); + Jim_InvalidateStringRep(listPtr); + ListAppendList(listPtr, appendListPtr); +} + +int Jim_ListLength(Jim_Interp *interp, Jim_Obj *objPtr) +{ + SetListFromAny(interp, objPtr); + return objPtr->internalRep.listValue.len; +} + +void Jim_ListInsertElements(Jim_Interp *interp, Jim_Obj *listPtr, int idx, + int objc, Jim_Obj *const *objVec) +{ + JimPanic((Jim_IsShared(listPtr), "Jim_ListInsertElement called with shared object")); + SetListFromAny(interp, listPtr); + if (idx >= 0 && idx > listPtr->internalRep.listValue.len) + idx = listPtr->internalRep.listValue.len; + else if (idx < 0) + idx = 0; + Jim_InvalidateStringRep(listPtr); + ListInsertElements(listPtr, idx, objc, objVec); +} + +Jim_Obj *Jim_ListGetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx) +{ + SetListFromAny(interp, listPtr); + if ((idx >= 0 && idx >= listPtr->internalRep.listValue.len) || + (idx < 0 && (-idx - 1) >= listPtr->internalRep.listValue.len)) { + return NULL; + } + if (idx < 0) + idx = listPtr->internalRep.listValue.len + idx; + return listPtr->internalRep.listValue.ele[idx]; +} + +int Jim_ListIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx, Jim_Obj **objPtrPtr, int flags) +{ + *objPtrPtr = Jim_ListGetIndex(interp, listPtr, idx); + if (*objPtrPtr == NULL) { + if (flags & JIM_ERRMSG) { + Jim_SetResultString(interp, "list index out of range", -1); + } + return JIM_ERR; + } + return JIM_OK; +} + +static int Jim_ListIndices(Jim_Interp *interp, Jim_Obj *listPtr, + Jim_Obj *const *indexv, int indexc, Jim_Obj **resultObj, int flags) +{ + int i; + int static_idxes[5]; + int *idxes = static_idxes; + int ret = JIM_OK; + + if (indexc > sizeof(static_idxes) / sizeof(*static_idxes)) { + idxes = Jim_Alloc(indexc * sizeof(*idxes)); + } + + for (i = 0; i < indexc; i++) { + ret = Jim_GetIndex(interp, indexv[i], &idxes[i]); + if (ret != JIM_OK) { + goto err; + } + } + + for (i = 0; i < indexc; i++) { + Jim_Obj *objPtr = Jim_ListGetIndex(interp, listPtr, idxes[i]); + if (!objPtr) { + if (flags & JIM_ERRMSG) { + if (idxes[i] < 0 || idxes[i] > Jim_ListLength(interp, listPtr)) { + Jim_SetResultFormatted(interp, "index \"%#s\" out of range", indexv[i]); + } + else { + Jim_SetResultFormatted(interp, "element %#s missing from sublist \"%#s\"", indexv[i], listPtr); + } + } + return -1; + } + listPtr = objPtr; + } + *resultObj = listPtr; +err: + if (idxes != static_idxes) + Jim_Free(idxes); + return ret; +} + +static int ListSetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx, + Jim_Obj *newObjPtr, int flags) +{ + SetListFromAny(interp, listPtr); + if ((idx >= 0 && idx >= listPtr->internalRep.listValue.len) || + (idx < 0 && (-idx - 1) >= listPtr->internalRep.listValue.len)) { + if (flags & JIM_ERRMSG) { + Jim_SetResultString(interp, "list index out of range", -1); + } + return JIM_ERR; + } + if (idx < 0) + idx = listPtr->internalRep.listValue.len + idx; + Jim_DecrRefCount(interp, listPtr->internalRep.listValue.ele[idx]); + listPtr->internalRep.listValue.ele[idx] = newObjPtr; + Jim_IncrRefCount(newObjPtr); + return JIM_OK; +} + +int Jim_ListSetIndex(Jim_Interp *interp, Jim_Obj *varNamePtr, + Jim_Obj *const *indexv, int indexc, Jim_Obj *newObjPtr) +{ + Jim_Obj *varObjPtr, *objPtr, *listObjPtr; + int shared, i, idx; + + varObjPtr = objPtr = Jim_GetVariable(interp, varNamePtr, JIM_ERRMSG | JIM_UNSHARED); + if (objPtr == NULL) + return JIM_ERR; + if ((shared = Jim_IsShared(objPtr))) + varObjPtr = objPtr = Jim_DuplicateObj(interp, objPtr); + for (i = 0; i < indexc - 1; i++) { + listObjPtr = objPtr; + if (Jim_GetIndex(interp, indexv[i], &idx) != JIM_OK) + goto err; + + objPtr = Jim_ListGetIndex(interp, listObjPtr, idx); + if (objPtr == NULL) { + Jim_SetResultFormatted(interp, "index \"%#s\" out of range", indexv[i]); + goto err; + } + if (Jim_IsShared(objPtr)) { + objPtr = Jim_DuplicateObj(interp, objPtr); + ListSetIndex(interp, listObjPtr, idx, objPtr, JIM_NONE); + } + Jim_InvalidateStringRep(listObjPtr); + } + if (Jim_GetIndex(interp, indexv[indexc - 1], &idx) != JIM_OK) + goto err; + if (ListSetIndex(interp, objPtr, idx, newObjPtr, JIM_ERRMSG) == JIM_ERR) + goto err; + Jim_InvalidateStringRep(objPtr); + Jim_InvalidateStringRep(varObjPtr); + if (Jim_SetVariable(interp, varNamePtr, varObjPtr) != JIM_OK) + goto err; + Jim_SetResult(interp, varObjPtr); + return JIM_OK; + err: + if (shared) { + Jim_FreeNewObj(interp, varObjPtr); + } + return JIM_ERR; +} + +Jim_Obj *Jim_ListJoin(Jim_Interp *interp, Jim_Obj *listObjPtr, const char *joinStr, int joinStrLen) +{ + int i; + int listLen = Jim_ListLength(interp, listObjPtr); + Jim_Obj *resObjPtr = Jim_NewEmptyStringObj(interp); + + for (i = 0; i < listLen; ) { + Jim_AppendObj(interp, resObjPtr, Jim_ListGetIndex(interp, listObjPtr, i)); + if (++i != listLen) { + Jim_AppendString(interp, resObjPtr, joinStr, joinStrLen); + } + } + return resObjPtr; +} + +Jim_Obj *Jim_ConcatObj(Jim_Interp *interp, int objc, Jim_Obj *const *objv) +{ + int i; + + for (i = 0; i < objc; i++) { + if (!Jim_IsList(objv[i])) + break; + } + if (i == objc) { + Jim_Obj *objPtr = Jim_NewListObj(interp, NULL, 0); + + for (i = 0; i < objc; i++) + ListAppendList(objPtr, objv[i]); + return objPtr; + } + else { + + int len = 0, objLen; + char *bytes, *p; + + + for (i = 0; i < objc; i++) { + len += Jim_Length(objv[i]); + } + if (objc) + len += objc - 1; + + p = bytes = Jim_Alloc(len + 1); + for (i = 0; i < objc; i++) { + const char *s = Jim_GetString(objv[i], &objLen); + + + while (objLen && isspace(UCHAR(*s))) { + s++; + objLen--; + len--; + } + + while (objLen && isspace(UCHAR(s[objLen - 1]))) { + + if (objLen > 1 && s[objLen - 2] == '\\') { + break; + } + objLen--; + len--; + } + memcpy(p, s, objLen); + p += objLen; + if (i + 1 != objc) { + if (objLen) + *p++ = ' '; + else { + len--; + } + } + } + *p = '\0'; + return Jim_NewStringObjNoAlloc(interp, bytes, len); + } +} + +Jim_Obj *Jim_ListRange(Jim_Interp *interp, Jim_Obj *listObjPtr, Jim_Obj *firstObjPtr, + Jim_Obj *lastObjPtr) +{ + int first, last; + int len, rangeLen; + + if (Jim_GetIndex(interp, firstObjPtr, &first) != JIM_OK || + Jim_GetIndex(interp, lastObjPtr, &last) != JIM_OK) + return NULL; + len = Jim_ListLength(interp, listObjPtr); + first = JimRelToAbsIndex(len, first); + last = JimRelToAbsIndex(len, last); + JimRelToAbsRange(len, &first, &last, &rangeLen); + if (first == 0 && last == len) { + return listObjPtr; + } + return Jim_NewListObj(interp, listObjPtr->internalRep.listValue.ele + first, rangeLen); +} + +static void FreeDictInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupDictInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); +static void UpdateStringOfDict(struct Jim_Obj *objPtr); +static int SetDictFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); + + +static const Jim_ObjType dictObjType = { + "dict", + FreeDictInternalRep, + DupDictInternalRep, + UpdateStringOfDict, + JIM_TYPE_NONE, +}; + +static void JimFreeDict(Jim_Interp *interp, Jim_Dict *dict) +{ + int i; + for (i = 0; i < dict->len; i++) { + Jim_DecrRefCount(interp, dict->table[i]); + } + Jim_Free(dict->table); + Jim_Free(dict->ht); + Jim_Free(dict); +} + +enum { + DICT_HASH_FIND = -1, + DICT_HASH_REMOVE = -2, + DICT_HASH_ADD = -3, +}; + +static int JimDictHashFind(Jim_Dict *dict, Jim_Obj *keyObjPtr, int op_tvoffset) +{ + unsigned h = (JimObjectHTHashFunction(keyObjPtr) + dict->uniq); + unsigned idx = h & dict->sizemask; + int tvoffset = 0; + unsigned peturb = h; + unsigned first_removed = ~0; + + if (dict->len) { + while ((tvoffset = dict->ht[idx].offset)) { + if (tvoffset == -1) { + if (first_removed == ~0) { + first_removed = idx; + } + } + else if (dict->ht[idx].hash == h) { + if (Jim_StringEqObj(keyObjPtr, dict->table[tvoffset - 1])) { + break; + } + } + + peturb >>= 5; + idx = (5 * idx + 1 + peturb) & dict->sizemask; + } + } + + switch (op_tvoffset) { + case DICT_HASH_FIND: + + break; + case DICT_HASH_REMOVE: + if (tvoffset) { + + dict->ht[idx].offset = -1; + dict->dummy++; + } + + break; + case DICT_HASH_ADD: + if (tvoffset == 0) { + + if (first_removed != ~0) { + idx = first_removed; + dict->dummy--; + } + dict->ht[idx].offset = dict->len + 1; + dict->ht[idx].hash = h; + } + + break; + default: + assert(tvoffset); + + dict->ht[idx].offset = op_tvoffset; + break; + } + + return tvoffset; +} + +static void JimDictExpandHashTable(Jim_Dict *dict, unsigned int size) +{ + int i; + struct JimDictHashEntry *prevht = dict->ht; + int prevsize = dict->size; + + dict->size = JimHashTableNextPower(size); + dict->sizemask = dict->size - 1; + + + dict->ht = Jim_Alloc(dict->size * sizeof(*dict->ht)); + memset(dict->ht, 0, dict->size * sizeof(*dict->ht)); + + + for (i = 0; i < prevsize; i++) { + if (prevht[i].offset > 0) { + + unsigned h = prevht[i].hash; + unsigned idx = h & dict->sizemask; + unsigned peturb = h; + + while (dict->ht[idx].offset) { + peturb >>= 5; + idx = (5 * idx + 1 + peturb) & dict->sizemask; + } + dict->ht[idx].offset = prevht[i].offset; + dict->ht[idx].hash = h; + } + } + Jim_Free(prevht); +} + +static int JimDictAdd(Jim_Dict *dict, Jim_Obj *keyObjPtr) +{ + if (dict->size <= dict->len + dict->dummy) { + JimDictExpandHashTable(dict, dict->size ? dict->size * 2 : 8); + } + return JimDictHashFind(dict, keyObjPtr, DICT_HASH_ADD); +} + +static Jim_Dict *JimDictNew(Jim_Interp *interp, int table_size, int ht_size) +{ + Jim_Dict *dict = Jim_Alloc(sizeof(*dict)); + memset(dict, 0, sizeof(*dict)); + + if (ht_size) { + JimDictExpandHashTable(dict, ht_size); + } + if (table_size) { + dict->table = Jim_Alloc(table_size * sizeof(*dict->table)); + dict->maxLen = table_size; + } +#ifdef JIM_RANDOMISE_HASH + dict->uniq = (rand() ^ time(NULL) ^ clock()); +#endif + return dict; +} + +static void FreeDictInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + JimFreeDict(interp, objPtr->internalRep.dictValue); +} + +static void DupDictInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + Jim_Dict *oldDict = srcPtr->internalRep.dictValue; + int i; + + + Jim_Dict *newDict = JimDictNew(interp, oldDict->maxLen, oldDict->size); + + + for (i = 0; i < oldDict->len; i++) { + newDict->table[i] = oldDict->table[i]; + Jim_IncrRefCount(newDict->table[i]); + } + newDict->len = oldDict->len; + + + newDict->uniq = oldDict->uniq; + + + memcpy(newDict->ht, oldDict->ht, sizeof(*oldDict->ht) * oldDict->size); + + dupPtr->internalRep.dictValue = newDict; + dupPtr->typePtr = &dictObjType; +} + +static void UpdateStringOfDict(struct Jim_Obj *objPtr) +{ + JimMakeListStringRep(objPtr, objPtr->internalRep.dictValue->table, objPtr->internalRep.dictValue->len); +} + +static int SetDictFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) +{ + int listlen; + + if (objPtr->typePtr == &dictObjType) { + return JIM_OK; + } + + if (Jim_IsList(objPtr) && Jim_IsShared(objPtr)) { + Jim_String(objPtr); + } + + listlen = Jim_ListLength(interp, objPtr); + if (listlen % 2) { + Jim_SetResultString(interp, "missing value to go with key", -1); + return JIM_ERR; + } + else { + + Jim_Dict *dict = JimDictNew(interp, 0, listlen); + int i; + + + dict->table = objPtr->internalRep.listValue.ele; + dict->maxLen = objPtr->internalRep.listValue.maxLen; + + + for (i = 0; i < listlen; i += 2) { + int tvoffset = JimDictAdd(dict, dict->table[i]); + if (tvoffset) { + + + Jim_DecrRefCount(interp, dict->table[tvoffset]); + + dict->table[tvoffset] = dict->table[i + 1]; + + Jim_DecrRefCount(interp, dict->table[i]); + } + else { + if (dict->len != i) { + dict->table[dict->len++] = dict->table[i]; + dict->table[dict->len++] = dict->table[i + 1]; + } + else { + dict->len += 2; + } + } + } + + objPtr->typePtr = &dictObjType; + objPtr->internalRep.dictValue = dict; + + return JIM_OK; + } +} + + + +static int DictAddElement(Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj *keyObjPtr, Jim_Obj *valueObjPtr) +{ + Jim_Dict *dict = objPtr->internalRep.dictValue; + if (valueObjPtr == NULL) { + + int tvoffset = JimDictHashFind(dict, keyObjPtr, DICT_HASH_REMOVE); + if (tvoffset) { + + Jim_DecrRefCount(interp, dict->table[tvoffset - 1]); + Jim_DecrRefCount(interp, dict->table[tvoffset]); + dict->len -= 2; + if (tvoffset != dict->len + 1) { + + dict->table[tvoffset - 1] = dict->table[dict->len]; + dict->table[tvoffset] = dict->table[dict->len + 1]; + + + JimDictHashFind(dict, dict->table[tvoffset - 1], tvoffset); + } + return JIM_OK; + } + return JIM_ERR; + } + else { + + int tvoffset = JimDictAdd(dict, keyObjPtr); + if (tvoffset) { + + Jim_IncrRefCount(valueObjPtr); + Jim_DecrRefCount(interp, dict->table[tvoffset]); + dict->table[tvoffset] = valueObjPtr; + } + else { + if (dict->maxLen == dict->len) { + + if (dict->maxLen < 4) { + dict->maxLen = 4; + } + else { + dict->maxLen *= 2; + } + dict->table = Jim_Realloc(dict->table, dict->maxLen * sizeof(*dict->table)); + } + Jim_IncrRefCount(keyObjPtr); + Jim_IncrRefCount(valueObjPtr); + + dict->table[dict->len++] = keyObjPtr; + dict->table[dict->len++] = valueObjPtr; + + } + return JIM_OK; + } +} + +int Jim_DictAddElement(Jim_Interp *interp, Jim_Obj *objPtr, + Jim_Obj *keyObjPtr, Jim_Obj *valueObjPtr) +{ + JimPanic((Jim_IsShared(objPtr), "Jim_DictAddElement called with shared object")); + if (SetDictFromAny(interp, objPtr) != JIM_OK) { + return JIM_ERR; + } + Jim_InvalidateStringRep(objPtr); + return DictAddElement(interp, objPtr, keyObjPtr, valueObjPtr); +} + +Jim_Obj *Jim_NewDictObj(Jim_Interp *interp, Jim_Obj *const *elements, int len) +{ + Jim_Obj *objPtr; + int i; + + JimPanic((len % 2, "Jim_NewDictObj() 'len' argument must be even")); + + objPtr = Jim_NewObj(interp); + objPtr->typePtr = &dictObjType; + objPtr->bytes = NULL; + + objPtr->internalRep.dictValue = JimDictNew(interp, len, len); + for (i = 0; i < len; i += 2) + DictAddElement(interp, objPtr, elements[i], elements[i + 1]); + return objPtr; +} + +int Jim_DictKey(Jim_Interp *interp, Jim_Obj *dictPtr, Jim_Obj *keyPtr, + Jim_Obj **objPtrPtr, int flags) +{ + int tvoffset; + Jim_Dict *dict; + + if (SetDictFromAny(interp, dictPtr) != JIM_OK) { + return -1; + } + dict = dictPtr->internalRep.dictValue; + tvoffset = JimDictHashFind(dict, keyPtr, DICT_HASH_FIND); + if (tvoffset == 0) { + if (flags & JIM_ERRMSG) { + Jim_SetResultFormatted(interp, "key \"%#s\" not known in dictionary", keyPtr); + } + return JIM_ERR; + } + *objPtrPtr = dict->table[tvoffset]; + return JIM_OK; +} + +Jim_Obj **Jim_DictPairs(Jim_Interp *interp, Jim_Obj *dictPtr, int *len) +{ + + if (Jim_IsList(dictPtr)) { + Jim_Obj **table; + JimListGetElements(interp, dictPtr, len, &table); + if (*len % 2 == 0) { + return table; + } + + } + if (SetDictFromAny(interp, dictPtr) != JIM_OK) { + + *len = 1; + return NULL; + } + *len = dictPtr->internalRep.dictValue->len; + return dictPtr->internalRep.dictValue->table; +} + + +int Jim_DictKeysVector(Jim_Interp *interp, Jim_Obj *dictPtr, + Jim_Obj *const *keyv, int keyc, Jim_Obj **objPtrPtr, int flags) +{ + int i; + + if (keyc == 0) { + *objPtrPtr = dictPtr; + return JIM_OK; + } + + for (i = 0; i < keyc; i++) { + Jim_Obj *objPtr; + + int rc = Jim_DictKey(interp, dictPtr, keyv[i], &objPtr, flags); + if (rc != JIM_OK) { + return rc; + } + dictPtr = objPtr; + } + *objPtrPtr = dictPtr; + return JIM_OK; +} + +int Jim_SetDictKeysVector(Jim_Interp *interp, Jim_Obj *varNamePtr, + Jim_Obj *const *keyv, int keyc, Jim_Obj *newObjPtr, int flags) +{ + Jim_Obj *varObjPtr, *objPtr, *dictObjPtr; + int shared, i; + + varObjPtr = objPtr = Jim_GetVariable(interp, varNamePtr, flags); + if (objPtr == NULL) { + if (newObjPtr == NULL && (flags & JIM_MUSTEXIST)) { + + return JIM_ERR; + } + varObjPtr = objPtr = Jim_NewDictObj(interp, NULL, 0); + if (Jim_SetVariable(interp, varNamePtr, objPtr) != JIM_OK) { + Jim_FreeNewObj(interp, varObjPtr); + return JIM_ERR; + } + } + if ((shared = Jim_IsShared(objPtr))) + varObjPtr = objPtr = Jim_DuplicateObj(interp, objPtr); + for (i = 0; i < keyc; i++) { + dictObjPtr = objPtr; + + + if (SetDictFromAny(interp, dictObjPtr) != JIM_OK) { + goto err; + } + + if (i == keyc - 1) { + + if (Jim_DictAddElement(interp, objPtr, keyv[keyc - 1], newObjPtr) != JIM_OK) { + if (newObjPtr || (flags & JIM_MUSTEXIST)) { + goto err; + } + } + break; + } + + + Jim_InvalidateStringRep(dictObjPtr); + if (Jim_DictKey(interp, dictObjPtr, keyv[i], &objPtr, + newObjPtr ? JIM_NONE : JIM_ERRMSG) == JIM_OK) { + if (Jim_IsShared(objPtr)) { + objPtr = Jim_DuplicateObj(interp, objPtr); + DictAddElement(interp, dictObjPtr, keyv[i], objPtr); + } + } + else { + if (newObjPtr == NULL) { + goto err; + } + objPtr = Jim_NewDictObj(interp, NULL, 0); + DictAddElement(interp, dictObjPtr, keyv[i], objPtr); + } + } + + Jim_InvalidateStringRep(objPtr); + Jim_InvalidateStringRep(varObjPtr); + if (Jim_SetVariable(interp, varNamePtr, varObjPtr) != JIM_OK) { + goto err; + } + + if (!(flags & JIM_NORESULT)) { + Jim_SetResult(interp, varObjPtr); + } + return JIM_OK; + err: + if (shared) { + Jim_FreeNewObj(interp, varObjPtr); + } + return JIM_ERR; +} + +static void UpdateStringOfIndex(struct Jim_Obj *objPtr); +static int SetIndexFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); + +static const Jim_ObjType indexObjType = { + "index", + NULL, + NULL, + UpdateStringOfIndex, + JIM_TYPE_NONE, +}; + +static void UpdateStringOfIndex(struct Jim_Obj *objPtr) +{ + if (objPtr->internalRep.intValue == -1) { + JimSetStringBytes(objPtr, "end"); + } + else { + char buf[JIM_INTEGER_SPACE + 1]; + if (objPtr->internalRep.intValue >= 0 || objPtr->internalRep.intValue == -INT_MAX) { + sprintf(buf, "%d", objPtr->internalRep.intValue); + } + else { + + sprintf(buf, "end%d", objPtr->internalRep.intValue + 1); + } + JimSetStringBytes(objPtr, buf); + } +} + +static int SetIndexFromAny(Jim_Interp *interp, Jim_Obj *objPtr) +{ + jim_wide idx; + int end = 0; + const char *str; + Jim_Obj *exprObj = objPtr; + + JimPanic((objPtr->refCount == 0, "SetIndexFromAny() called with zero refcount object")); + + + str = Jim_String(objPtr); + + + if (strncmp(str, "end", 3) == 0) { + end = 1; + str += 3; + idx = 0; + switch (*str) { + case '\0': + exprObj = NULL; + break; + + case '-': + case '+': + exprObj = Jim_NewStringObj(interp, str, -1); + break; + + default: + goto badindex; + } + } + if (exprObj) { + int ret; + Jim_IncrRefCount(exprObj); + ret = Jim_GetWideExpr(interp, exprObj, &idx); + Jim_DecrRefCount(interp, exprObj); + if (ret != JIM_OK) { + goto badindex; + } + } + + if (end) { + if (idx > 0) { + idx = INT_MAX; + } + else { + + idx--; + } + } + else if (idx < 0) { + idx = -INT_MAX; + } + + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &indexObjType; + objPtr->internalRep.intValue = idx; + return JIM_OK; + + badindex: + Jim_SetResultFormatted(interp, + "bad index \"%#s\": must be intexpr or end?[+-]intexpr?", objPtr); + return JIM_ERR; +} + +int Jim_GetIndex(Jim_Interp *interp, Jim_Obj *objPtr, int *indexPtr) +{ + + if (objPtr->typePtr == &intObjType) { + jim_wide val = JimWideValue(objPtr); + + if (val < 0) + *indexPtr = -INT_MAX; + else if (val > INT_MAX) + *indexPtr = INT_MAX; + else + *indexPtr = (int)val; + return JIM_OK; + } + if (objPtr->typePtr != &indexObjType && SetIndexFromAny(interp, objPtr) == JIM_ERR) + return JIM_ERR; + *indexPtr = objPtr->internalRep.intValue; + return JIM_OK; +} + + + +static const char * const jimReturnCodes[] = { + "ok", + "error", + "return", + "break", + "continue", + "signal", + "exit", + "eval", + NULL +}; + +#define jimReturnCodesSize (sizeof(jimReturnCodes)/sizeof(*jimReturnCodes) - 1) + +static const Jim_ObjType returnCodeObjType = { + "return-code", + NULL, + NULL, + NULL, + JIM_TYPE_NONE, +}; + +const char *Jim_ReturnCode(int code) +{ + if (code < 0 || code >= (int)jimReturnCodesSize) { + return "?"; + } + else { + return jimReturnCodes[code]; + } +} + +static int SetReturnCodeFromAny(Jim_Interp *interp, Jim_Obj *objPtr) +{ + int returnCode; + jim_wide wideValue; + + + if (JimGetWideNoErr(interp, objPtr, &wideValue) != JIM_ERR) + returnCode = (int)wideValue; + else if (Jim_GetEnum(interp, objPtr, jimReturnCodes, &returnCode, NULL, JIM_NONE) != JIM_OK) { + Jim_SetResultFormatted(interp, "expected return code but got \"%#s\"", objPtr); + return JIM_ERR; + } + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &returnCodeObjType; + objPtr->internalRep.intValue = returnCode; + return JIM_OK; +} + +int Jim_GetReturnCode(Jim_Interp *interp, Jim_Obj *objPtr, int *intPtr) +{ + if (objPtr->typePtr != &returnCodeObjType && SetReturnCodeFromAny(interp, objPtr) == JIM_ERR) + return JIM_ERR; + *intPtr = objPtr->internalRep.intValue; + return JIM_OK; +} + +static int JimParseExprOperator(struct JimParserCtx *pc); +static int JimParseExprNumber(struct JimParserCtx *pc); +static int JimParseExprIrrational(struct JimParserCtx *pc); +static int JimParseExprBoolean(struct JimParserCtx *pc); + + +enum +{ + + + + JIM_EXPROP_MUL = JIM_TT_EXPR_OP, + JIM_EXPROP_DIV, + JIM_EXPROP_MOD, + JIM_EXPROP_SUB, + JIM_EXPROP_ADD, + JIM_EXPROP_LSHIFT, + JIM_EXPROP_RSHIFT, + JIM_EXPROP_ROTL, + JIM_EXPROP_ROTR, + JIM_EXPROP_LT, + JIM_EXPROP_GT, + JIM_EXPROP_LTE, + JIM_EXPROP_GTE, + JIM_EXPROP_NUMEQ, + JIM_EXPROP_NUMNE, + JIM_EXPROP_BITAND, + JIM_EXPROP_BITXOR, + JIM_EXPROP_BITOR, + JIM_EXPROP_LOGICAND, + JIM_EXPROP_LOGICOR, + JIM_EXPROP_TERNARY, + JIM_EXPROP_COLON, + JIM_EXPROP_POW, + + + JIM_EXPROP_STREQ, + JIM_EXPROP_STRNE, + JIM_EXPROP_STRIN, + JIM_EXPROP_STRNI, + JIM_EXPROP_STRLT, + JIM_EXPROP_STRGT, + JIM_EXPROP_STRLE, + JIM_EXPROP_STRGE, + + + JIM_EXPROP_NOT, + JIM_EXPROP_BITNOT, + JIM_EXPROP_UNARYMINUS, + JIM_EXPROP_UNARYPLUS, + + + JIM_EXPROP_FUNC_INT, + JIM_EXPROP_FUNC_WIDE, + JIM_EXPROP_FUNC_ABS, + JIM_EXPROP_FUNC_DOUBLE, + JIM_EXPROP_FUNC_ROUND, + JIM_EXPROP_FUNC_RAND, + JIM_EXPROP_FUNC_SRAND, + + + JIM_EXPROP_FUNC_SIN, + JIM_EXPROP_FUNC_COS, + JIM_EXPROP_FUNC_TAN, + JIM_EXPROP_FUNC_ASIN, + JIM_EXPROP_FUNC_ACOS, + JIM_EXPROP_FUNC_ATAN, + JIM_EXPROP_FUNC_ATAN2, + JIM_EXPROP_FUNC_SINH, + JIM_EXPROP_FUNC_COSH, + JIM_EXPROP_FUNC_TANH, + JIM_EXPROP_FUNC_CEIL, + JIM_EXPROP_FUNC_FLOOR, + JIM_EXPROP_FUNC_EXP, + JIM_EXPROP_FUNC_LOG, + JIM_EXPROP_FUNC_LOG10, + JIM_EXPROP_FUNC_SQRT, + JIM_EXPROP_FUNC_POW, + JIM_EXPROP_FUNC_HYPOT, + JIM_EXPROP_FUNC_FMOD, +}; + +struct JimExprNode { + int type; + struct Jim_Obj *objPtr; + + struct JimExprNode *left; + struct JimExprNode *right; + struct JimExprNode *ternary; +}; + + +typedef struct Jim_ExprOperator +{ + const char *name; + int (*funcop) (Jim_Interp *interp, struct JimExprNode *opnode); + unsigned char precedence; + unsigned char arity; + unsigned char attr; + unsigned char namelen; +} Jim_ExprOperator; + +static int JimExprGetTerm(Jim_Interp *interp, struct JimExprNode *node, Jim_Obj **objPtrPtr); +static int JimExprGetTermBoolean(Jim_Interp *interp, struct JimExprNode *node); +static int JimExprEvalTermNode(Jim_Interp *interp, struct JimExprNode *node); + +static int JimExprOpNumUnary(Jim_Interp *interp, struct JimExprNode *node) +{ + int intresult = 1; + int rc, bA = 0; + double dA, dC = 0; + jim_wide wA, wC = 0; + Jim_Obj *A; + + if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { + return rc; + } + + if ((A->typePtr != &doubleObjType || A->bytes) && JimGetWideNoErr(interp, A, &wA) == JIM_OK) { + switch (node->type) { + case JIM_EXPROP_FUNC_INT: + case JIM_EXPROP_FUNC_WIDE: + case JIM_EXPROP_FUNC_ROUND: + case JIM_EXPROP_UNARYPLUS: + wC = wA; + break; + case JIM_EXPROP_FUNC_DOUBLE: + dC = wA; + intresult = 0; + break; + case JIM_EXPROP_FUNC_ABS: + wC = wA >= 0 ? wA : -wA; + break; + case JIM_EXPROP_UNARYMINUS: + wC = -wA; + break; + case JIM_EXPROP_NOT: + wC = !wA; + break; + default: + abort(); + } + } + else if ((rc = Jim_GetDouble(interp, A, &dA)) == JIM_OK) { + switch (node->type) { + case JIM_EXPROP_FUNC_INT: + case JIM_EXPROP_FUNC_WIDE: + wC = dA; + break; + case JIM_EXPROP_FUNC_ROUND: + wC = dA < 0 ? (dA - 0.5) : (dA + 0.5); + break; + case JIM_EXPROP_FUNC_DOUBLE: + case JIM_EXPROP_UNARYPLUS: + dC = dA; + intresult = 0; + break; + case JIM_EXPROP_FUNC_ABS: +#ifdef JIM_MATH_FUNCTIONS + dC = fabs(dA); +#else + dC = dA >= 0 ? dA : -dA; +#endif + intresult = 0; + break; + case JIM_EXPROP_UNARYMINUS: + dC = -dA; + intresult = 0; + break; + case JIM_EXPROP_NOT: + wC = !dA; + break; + default: + abort(); + } + } + else if ((rc = Jim_GetBoolean(interp, A, &bA)) == JIM_OK) { + switch (node->type) { + case JIM_EXPROP_NOT: + wC = !bA; + break; + default: + abort(); + } + } + + if (rc == JIM_OK) { + if (intresult) { + Jim_SetResultInt(interp, wC); + } + else { + Jim_SetResult(interp, Jim_NewDoubleObj(interp, dC)); + } + } + + Jim_DecrRefCount(interp, A); + + return rc; +} + +static double JimRandDouble(Jim_Interp *interp) +{ + unsigned long x; + JimRandomBytes(interp, &x, sizeof(x)); + + return (double)x / (double)~0UL; +} + +static int JimExprOpIntUnary(Jim_Interp *interp, struct JimExprNode *node) +{ + jim_wide wA; + Jim_Obj *A; + int rc; + + if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { + return rc; + } + + rc = Jim_GetWide(interp, A, &wA); + if (rc == JIM_OK) { + switch (node->type) { + case JIM_EXPROP_BITNOT: + Jim_SetResultInt(interp, ~wA); + break; + case JIM_EXPROP_FUNC_SRAND: + JimPrngSeed(interp, (unsigned char *)&wA, sizeof(wA)); + Jim_SetResult(interp, Jim_NewDoubleObj(interp, JimRandDouble(interp))); + break; + default: + abort(); + } + } + + Jim_DecrRefCount(interp, A); + + return rc; +} + +static int JimExprOpNone(Jim_Interp *interp, struct JimExprNode *node) +{ + JimPanic((node->type != JIM_EXPROP_FUNC_RAND, "JimExprOpNone only support rand()")); + + Jim_SetResult(interp, Jim_NewDoubleObj(interp, JimRandDouble(interp))); + + return JIM_OK; +} + +#ifdef JIM_MATH_FUNCTIONS +static int JimExprOpDoubleUnary(Jim_Interp *interp, struct JimExprNode *node) +{ + int rc; + double dA, dC; + Jim_Obj *A; + + if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { + return rc; + } + + rc = Jim_GetDouble(interp, A, &dA); + if (rc == JIM_OK) { + switch (node->type) { + case JIM_EXPROP_FUNC_SIN: + dC = sin(dA); + break; + case JIM_EXPROP_FUNC_COS: + dC = cos(dA); + break; + case JIM_EXPROP_FUNC_TAN: + dC = tan(dA); + break; + case JIM_EXPROP_FUNC_ASIN: + dC = asin(dA); + break; + case JIM_EXPROP_FUNC_ACOS: + dC = acos(dA); + break; + case JIM_EXPROP_FUNC_ATAN: + dC = atan(dA); + break; + case JIM_EXPROP_FUNC_SINH: + dC = sinh(dA); + break; + case JIM_EXPROP_FUNC_COSH: + dC = cosh(dA); + break; + case JIM_EXPROP_FUNC_TANH: + dC = tanh(dA); + break; + case JIM_EXPROP_FUNC_CEIL: + dC = ceil(dA); + break; + case JIM_EXPROP_FUNC_FLOOR: + dC = floor(dA); + break; + case JIM_EXPROP_FUNC_EXP: + dC = exp(dA); + break; + case JIM_EXPROP_FUNC_LOG: + dC = log(dA); + break; + case JIM_EXPROP_FUNC_LOG10: + dC = log10(dA); + break; + case JIM_EXPROP_FUNC_SQRT: + dC = sqrt(dA); + break; + default: + abort(); + } + Jim_SetResult(interp, Jim_NewDoubleObj(interp, dC)); + } + + Jim_DecrRefCount(interp, A); + + return rc; +} +#endif + + +static int JimExprOpIntBin(Jim_Interp *interp, struct JimExprNode *node) +{ + jim_wide wA, wB; + int rc; + Jim_Obj *A, *B; + + if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { + return rc; + } + if ((rc = JimExprGetTerm(interp, node->right, &B)) != JIM_OK) { + Jim_DecrRefCount(interp, A); + return rc; + } + + rc = JIM_ERR; + + if (Jim_GetWide(interp, A, &wA) == JIM_OK && Jim_GetWide(interp, B, &wB) == JIM_OK) { + jim_wide wC; + + rc = JIM_OK; + + switch (node->type) { + case JIM_EXPROP_LSHIFT: + wC = wA << wB; + break; + case JIM_EXPROP_RSHIFT: + wC = wA >> wB; + break; + case JIM_EXPROP_BITAND: + wC = wA & wB; + break; + case JIM_EXPROP_BITXOR: + wC = wA ^ wB; + break; + case JIM_EXPROP_BITOR: + wC = wA | wB; + break; + case JIM_EXPROP_MOD: + if (wB == 0) { + wC = 0; + Jim_SetResultString(interp, "Division by zero", -1); + rc = JIM_ERR; + } + else { + int negative = 0; + + if (wB < 0) { + wB = -wB; + wA = -wA; + negative = 1; + } + wC = wA % wB; + if (wC < 0) { + wC += wB; + } + if (negative) { + wC = -wC; + } + } + break; + case JIM_EXPROP_ROTL: + case JIM_EXPROP_ROTR:{ + + unsigned long uA = (unsigned long)wA; + unsigned long uB = (unsigned long)wB; + const unsigned int S = sizeof(unsigned long) * 8; + + + uB %= S; + + if (node->type == JIM_EXPROP_ROTR) { + uB = S - uB; + } + wC = (unsigned long)(uA << uB) | (uA >> (S - uB)); + break; + } + default: + abort(); + } + Jim_SetResultInt(interp, wC); + } + + Jim_DecrRefCount(interp, A); + Jim_DecrRefCount(interp, B); + + return rc; +} + + + +static int JimExprOpBin(Jim_Interp *interp, struct JimExprNode *node) +{ + int rc = JIM_OK; + double dA, dB, dC = 0; + jim_wide wA, wB, wC = 0; + Jim_Obj *A, *B; + + if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { + return rc; + } + if ((rc = JimExprGetTerm(interp, node->right, &B)) != JIM_OK) { + Jim_DecrRefCount(interp, A); + return rc; + } + + if ((A->typePtr != &doubleObjType || A->bytes) && + (B->typePtr != &doubleObjType || B->bytes) && + JimGetWideNoErr(interp, A, &wA) == JIM_OK && JimGetWideNoErr(interp, B, &wB) == JIM_OK) { + + + + switch (node->type) { + case JIM_EXPROP_POW: + case JIM_EXPROP_FUNC_POW: + if (wA == 0 && wB < 0) { + Jim_SetResultString(interp, "exponentiation of zero by negative power", -1); + rc = JIM_ERR; + goto done; + } + wC = JimPowWide(wA, wB); + goto intresult; + case JIM_EXPROP_ADD: + wC = wA + wB; + goto intresult; + case JIM_EXPROP_SUB: + wC = wA - wB; + goto intresult; + case JIM_EXPROP_MUL: + wC = wA * wB; + goto intresult; + case JIM_EXPROP_DIV: + if (wB == 0) { + Jim_SetResultString(interp, "Division by zero", -1); + rc = JIM_ERR; + goto done; + } + else { + if (wB < 0) { + wB = -wB; + wA = -wA; + } + wC = wA / wB; + if (wA % wB < 0) { + wC--; + } + goto intresult; + } + case JIM_EXPROP_LT: + wC = wA < wB; + goto intresult; + case JIM_EXPROP_GT: + wC = wA > wB; + goto intresult; + case JIM_EXPROP_LTE: + wC = wA <= wB; + goto intresult; + case JIM_EXPROP_GTE: + wC = wA >= wB; + goto intresult; + case JIM_EXPROP_NUMEQ: + wC = wA == wB; + goto intresult; + case JIM_EXPROP_NUMNE: + wC = wA != wB; + goto intresult; + } + } + if (Jim_GetDouble(interp, A, &dA) == JIM_OK && Jim_GetDouble(interp, B, &dB) == JIM_OK) { + switch (node->type) { +#ifndef JIM_MATH_FUNCTIONS + case JIM_EXPROP_POW: + case JIM_EXPROP_FUNC_POW: + case JIM_EXPROP_FUNC_ATAN2: + case JIM_EXPROP_FUNC_HYPOT: + case JIM_EXPROP_FUNC_FMOD: + Jim_SetResultString(interp, "unsupported", -1); + rc = JIM_ERR; + goto done; +#else + case JIM_EXPROP_POW: + case JIM_EXPROP_FUNC_POW: + dC = pow(dA, dB); + goto doubleresult; + case JIM_EXPROP_FUNC_ATAN2: + dC = atan2(dA, dB); + goto doubleresult; + case JIM_EXPROP_FUNC_HYPOT: + dC = hypot(dA, dB); + goto doubleresult; + case JIM_EXPROP_FUNC_FMOD: + dC = fmod(dA, dB); + goto doubleresult; +#endif + case JIM_EXPROP_ADD: + dC = dA + dB; + goto doubleresult; + case JIM_EXPROP_SUB: + dC = dA - dB; + goto doubleresult; + case JIM_EXPROP_MUL: + dC = dA * dB; + goto doubleresult; + case JIM_EXPROP_DIV: + if (dB == 0) { +#ifdef INFINITY + dC = dA < 0 ? -INFINITY : INFINITY; +#else + dC = (dA < 0 ? -1.0 : 1.0) * strtod("Inf", NULL); +#endif + } + else { + dC = dA / dB; + } + goto doubleresult; + case JIM_EXPROP_LT: + wC = dA < dB; + goto intresult; + case JIM_EXPROP_GT: + wC = dA > dB; + goto intresult; + case JIM_EXPROP_LTE: + wC = dA <= dB; + goto intresult; + case JIM_EXPROP_GTE: + wC = dA >= dB; + goto intresult; + case JIM_EXPROP_NUMEQ: + wC = dA == dB; + goto intresult; + case JIM_EXPROP_NUMNE: + wC = dA != dB; + goto intresult; + } + } + else { + + + + int i = Jim_StringCompareObj(interp, A, B, 0); + + switch (node->type) { + case JIM_EXPROP_LT: + wC = i < 0; + goto intresult; + case JIM_EXPROP_GT: + wC = i > 0; + goto intresult; + case JIM_EXPROP_LTE: + wC = i <= 0; + goto intresult; + case JIM_EXPROP_GTE: + wC = i >= 0; + goto intresult; + case JIM_EXPROP_NUMEQ: + wC = i == 0; + goto intresult; + case JIM_EXPROP_NUMNE: + wC = i != 0; + goto intresult; + } + } + + rc = JIM_ERR; +done: + Jim_DecrRefCount(interp, A); + Jim_DecrRefCount(interp, B); + return rc; +intresult: + Jim_SetResultInt(interp, wC); + goto done; +doubleresult: + Jim_SetResult(interp, Jim_NewDoubleObj(interp, dC)); + goto done; +} + +static int JimSearchList(Jim_Interp *interp, Jim_Obj *listObjPtr, Jim_Obj *valObj) +{ + int listlen; + int i; + + listlen = Jim_ListLength(interp, listObjPtr); + for (i = 0; i < listlen; i++) { + if (Jim_StringEqObj(Jim_ListGetIndex(interp, listObjPtr, i), valObj)) { + return 1; + } + } + return 0; +} + + + +static int JimExprOpStrBin(Jim_Interp *interp, struct JimExprNode *node) +{ + Jim_Obj *A, *B; + jim_wide wC; + int comp, rc; + + if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { + return rc; + } + if ((rc = JimExprGetTerm(interp, node->right, &B)) != JIM_OK) { + Jim_DecrRefCount(interp, A); + return rc; + } + + switch (node->type) { + case JIM_EXPROP_STREQ: + case JIM_EXPROP_STRNE: + wC = Jim_StringEqObj(A, B); + if (node->type == JIM_EXPROP_STRNE) { + wC = !wC; + } + break; + case JIM_EXPROP_STRLT: + case JIM_EXPROP_STRGT: + case JIM_EXPROP_STRLE: + case JIM_EXPROP_STRGE: + comp = Jim_StringCompareObj(interp, A, B, 0); + if (node->type == JIM_EXPROP_STRLT) { + wC = comp == -1; + } else if (node->type == JIM_EXPROP_STRGT) { + wC = comp == 1; + } else if (node->type == JIM_EXPROP_STRLE) { + wC = comp == -1 || comp == 0; + } else { + wC = comp == 0 || comp == 1; + } + break; + case JIM_EXPROP_STRIN: + wC = JimSearchList(interp, B, A); + break; + case JIM_EXPROP_STRNI: + wC = !JimSearchList(interp, B, A); + break; + default: + abort(); + } + Jim_SetResultInt(interp, wC); + + Jim_DecrRefCount(interp, A); + Jim_DecrRefCount(interp, B); + + return rc; +} + +static int ExprBool(Jim_Interp *interp, Jim_Obj *obj) +{ + long l; + double d; + int b; + int ret = -1; + + + Jim_IncrRefCount(obj); + + if (Jim_GetLong(interp, obj, &l) == JIM_OK) { + ret = (l != 0); + } + else if (Jim_GetDouble(interp, obj, &d) == JIM_OK) { + ret = (d != 0); + } + else if (Jim_GetBoolean(interp, obj, &b) == JIM_OK) { + ret = (b != 0); + } + + Jim_DecrRefCount(interp, obj); + return ret; +} + +static int JimExprOpAnd(Jim_Interp *interp, struct JimExprNode *node) +{ + + int result = JimExprGetTermBoolean(interp, node->left); + + if (result == 1) { + + result = JimExprGetTermBoolean(interp, node->right); + } + if (result == -1) { + return JIM_ERR; + } + Jim_SetResultInt(interp, result); + return JIM_OK; +} + +static int JimExprOpOr(Jim_Interp *interp, struct JimExprNode *node) +{ + + int result = JimExprGetTermBoolean(interp, node->left); + + if (result == 0) { + + result = JimExprGetTermBoolean(interp, node->right); + } + if (result == -1) { + return JIM_ERR; + } + Jim_SetResultInt(interp, result); + return JIM_OK; +} + +static int JimExprOpTernary(Jim_Interp *interp, struct JimExprNode *node) +{ + + int result = JimExprGetTermBoolean(interp, node->left); + + if (result == 1) { + + return JimExprEvalTermNode(interp, node->right); + } + else if (result == 0) { + + return JimExprEvalTermNode(interp, node->ternary); + } + + return JIM_ERR; +} + +enum +{ + OP_FUNC = 0x0001, + OP_RIGHT_ASSOC = 0x0002, +}; + +#define OPRINIT_ATTR(N, P, ARITY, F, ATTR) {N, F, P, ARITY, ATTR, sizeof(N) - 1} +#define OPRINIT(N, P, ARITY, F) OPRINIT_ATTR(N, P, ARITY, F, 0) + +static const struct Jim_ExprOperator Jim_ExprOperators[] = { + OPRINIT("*", 110, 2, JimExprOpBin), + OPRINIT("/", 110, 2, JimExprOpBin), + OPRINIT("%", 110, 2, JimExprOpIntBin), + + OPRINIT("-", 100, 2, JimExprOpBin), + OPRINIT("+", 100, 2, JimExprOpBin), + + OPRINIT("<<", 90, 2, JimExprOpIntBin), + OPRINIT(">>", 90, 2, JimExprOpIntBin), + + OPRINIT("<<<", 90, 2, JimExprOpIntBin), + OPRINIT(">>>", 90, 2, JimExprOpIntBin), + + OPRINIT("<", 80, 2, JimExprOpBin), + OPRINIT(">", 80, 2, JimExprOpBin), + OPRINIT("<=", 80, 2, JimExprOpBin), + OPRINIT(">=", 80, 2, JimExprOpBin), + + OPRINIT("==", 70, 2, JimExprOpBin), + OPRINIT("!=", 70, 2, JimExprOpBin), + + OPRINIT("&", 50, 2, JimExprOpIntBin), + OPRINIT("^", 49, 2, JimExprOpIntBin), + OPRINIT("|", 48, 2, JimExprOpIntBin), + + OPRINIT("&&", 10, 2, JimExprOpAnd), + OPRINIT("||", 9, 2, JimExprOpOr), + OPRINIT_ATTR("?", 5, 3, JimExprOpTernary, OP_RIGHT_ASSOC), + OPRINIT_ATTR(":", 5, 3, NULL, OP_RIGHT_ASSOC), + + + OPRINIT_ATTR("**", 120, 2, JimExprOpBin, OP_RIGHT_ASSOC), + + OPRINIT("eq", 60, 2, JimExprOpStrBin), + OPRINIT("ne", 60, 2, JimExprOpStrBin), + + OPRINIT("in", 55, 2, JimExprOpStrBin), + OPRINIT("ni", 55, 2, JimExprOpStrBin), + + OPRINIT("lt", 75, 2, JimExprOpStrBin), + OPRINIT("gt", 75, 2, JimExprOpStrBin), + OPRINIT("le", 75, 2, JimExprOpStrBin), + OPRINIT("ge", 75, 2, JimExprOpStrBin), + + OPRINIT_ATTR("!", 150, 1, JimExprOpNumUnary, OP_RIGHT_ASSOC), + OPRINIT_ATTR("~", 150, 1, JimExprOpIntUnary, OP_RIGHT_ASSOC), + OPRINIT_ATTR(" -", 150, 1, JimExprOpNumUnary, OP_RIGHT_ASSOC), + OPRINIT_ATTR(" +", 150, 1, JimExprOpNumUnary, OP_RIGHT_ASSOC), + + + + OPRINIT_ATTR("int", 200, 1, JimExprOpNumUnary, OP_FUNC), + OPRINIT_ATTR("wide", 200, 1, JimExprOpNumUnary, OP_FUNC), + OPRINIT_ATTR("abs", 200, 1, JimExprOpNumUnary, OP_FUNC), + OPRINIT_ATTR("double", 200, 1, JimExprOpNumUnary, OP_FUNC), + OPRINIT_ATTR("round", 200, 1, JimExprOpNumUnary, OP_FUNC), + OPRINIT_ATTR("rand", 200, 0, JimExprOpNone, OP_FUNC), + OPRINIT_ATTR("srand", 200, 1, JimExprOpIntUnary, OP_FUNC), + +#ifdef JIM_MATH_FUNCTIONS + OPRINIT_ATTR("sin", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("cos", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("tan", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("asin", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("acos", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("atan", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("atan2", 200, 2, JimExprOpBin, OP_FUNC), + OPRINIT_ATTR("sinh", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("cosh", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("tanh", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("ceil", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("floor", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("exp", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("log", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("log10", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("sqrt", 200, 1, JimExprOpDoubleUnary, OP_FUNC), + OPRINIT_ATTR("pow", 200, 2, JimExprOpBin, OP_FUNC), + OPRINIT_ATTR("hypot", 200, 2, JimExprOpBin, OP_FUNC), + OPRINIT_ATTR("fmod", 200, 2, JimExprOpBin, OP_FUNC), +#endif +}; +#undef OPRINIT +#undef OPRINIT_ATTR + +#define JIM_EXPR_OPERATORS_NUM \ + (sizeof(Jim_ExprOperators)/sizeof(struct Jim_ExprOperator)) + +static int JimParseExpression(struct JimParserCtx *pc) +{ + pc->errmsg = NULL; + + while (1) { + + while (isspace(UCHAR(*pc->p)) || (*(pc->p) == '\\' && *(pc->p + 1) == '\n')) { + if (*pc->p == '\n') { + pc->linenr++; + } + pc->p++; + pc->len--; + } + + if (*pc->p == '#') { + JimParseComment(pc); + + continue; + } + break; + } + + + pc->tline = pc->linenr; + pc->tstart = pc->p; + + if (pc->len == 0) { + pc->tend = pc->p; + pc->tt = JIM_TT_EOL; + pc->eof = 1; + return JIM_OK; + } + switch (*(pc->p)) { + case '(': + pc->tt = JIM_TT_SUBEXPR_START; + goto singlechar; + case ')': + pc->tt = JIM_TT_SUBEXPR_END; + goto singlechar; + case ',': + pc->tt = JIM_TT_SUBEXPR_COMMA; +singlechar: + pc->tend = pc->p; + pc->p++; + pc->len--; + break; + case '[': + return JimParseCmd(pc); + case '$': + if (JimParseVar(pc) == JIM_ERR) + return JimParseExprOperator(pc); + else { + + if (pc->tt == JIM_TT_EXPRSUGAR) { + pc->errmsg = "nesting expr in expr is not allowed"; + return JIM_ERR; + } + return JIM_OK; + } + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '.': + return JimParseExprNumber(pc); + case '"': + return JimParseQuote(pc); + case '{': + return JimParseBrace(pc); + + case 'N': + case 'I': + case 'n': + case 'i': + if (JimParseExprIrrational(pc) == JIM_ERR) + if (JimParseExprBoolean(pc) == JIM_ERR) + return JimParseExprOperator(pc); + break; + case 't': + case 'f': + case 'o': + case 'y': + if (JimParseExprBoolean(pc) == JIM_ERR) + return JimParseExprOperator(pc); + break; + default: + return JimParseExprOperator(pc); + break; + } + return JIM_OK; +} + +static int JimParseExprNumber(struct JimParserCtx *pc) +{ + char *end; + + + pc->tt = JIM_TT_EXPR_INT; + + jim_strtoull(pc->p, (char **)&pc->p); + + if (strchr("eENnIi.", *pc->p) || pc->p == pc->tstart) { + if (strtod(pc->tstart, &end)) { } + if (end == pc->tstart) + return JIM_ERR; + if (end > pc->p) { + + pc->tt = JIM_TT_EXPR_DOUBLE; + pc->p = end; + } + } + pc->tend = pc->p - 1; + pc->len -= (pc->p - pc->tstart); + return JIM_OK; +} + +static int JimParseExprIrrational(struct JimParserCtx *pc) +{ + const char *irrationals[] = { "NaN", "nan", "NAN", "Inf", "inf", "INF", NULL }; + int i; + + for (i = 0; irrationals[i]; i++) { + const char *irr = irrationals[i]; + + if (strncmp(irr, pc->p, 3) == 0) { + pc->p += 3; + pc->len -= 3; + pc->tend = pc->p - 1; + pc->tt = JIM_TT_EXPR_DOUBLE; + return JIM_OK; + } + } + return JIM_ERR; +} + +static int JimParseExprBoolean(struct JimParserCtx *pc) +{ + int i; + for (i = 0; i < sizeof(jim_true_false_strings) / sizeof(*jim_true_false_strings); i++) { + if (strncmp(pc->p, jim_true_false_strings[i], jim_true_false_lens[i]) == 0) { + pc->p += jim_true_false_lens[i]; + pc->len -= jim_true_false_lens[i]; + pc->tend = pc->p - 1; + pc->tt = JIM_TT_EXPR_BOOLEAN; + return JIM_OK; + } + } + return JIM_ERR; +} + +static const struct Jim_ExprOperator *JimExprOperatorInfoByOpcode(int opcode) +{ + static Jim_ExprOperator dummy_op; + if (opcode < JIM_TT_EXPR_OP) { + return &dummy_op; + } + return &Jim_ExprOperators[opcode - JIM_TT_EXPR_OP]; +} + +static int JimParseExprOperator(struct JimParserCtx *pc) +{ + int i; + const struct Jim_ExprOperator *bestOp = NULL; + int bestLen = 0; + + + for (i = 0; i < (signed)JIM_EXPR_OPERATORS_NUM; i++) { + const struct Jim_ExprOperator *op = &Jim_ExprOperators[i]; + + if (op->name[0] != pc->p[0]) { + continue; + } + + if (op->namelen > bestLen && strncmp(op->name, pc->p, op->namelen) == 0) { + bestOp = op; + bestLen = op->namelen; + } + } + if (bestOp == NULL) { + return JIM_ERR; + } + + + if (bestOp->attr & OP_FUNC) { + const char *p = pc->p + bestLen; + int len = pc->len - bestLen; + + while (len && isspace(UCHAR(*p))) { + len--; + p++; + } + if (*p != '(') { + pc->errmsg = "function requires parentheses"; + return JIM_ERR; + } + } + pc->tend = pc->p + bestLen - 1; + pc->p += bestLen; + pc->len -= bestLen; + + pc->tt = (bestOp - Jim_ExprOperators) + JIM_TT_EXPR_OP; + return JIM_OK; +} + + +static void FreeExprInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupExprInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); +static int SetExprFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); + +static const Jim_ObjType exprObjType = { + "expression", + FreeExprInternalRep, + DupExprInternalRep, + NULL, + JIM_TYPE_NONE, +}; + + +struct ExprTree +{ + struct JimExprNode *expr; + struct JimExprNode *nodes; + int len; + int inUse; +}; + +static void ExprTreeFreeNodes(Jim_Interp *interp, struct JimExprNode *nodes, int num) +{ + int i; + for (i = 0; i < num; i++) { + if (nodes[i].objPtr) { + Jim_DecrRefCount(interp, nodes[i].objPtr); + } + } + Jim_Free(nodes); +} + +static void ExprTreeFree(Jim_Interp *interp, struct ExprTree *expr) +{ + ExprTreeFreeNodes(interp, expr->nodes, expr->len); + Jim_Free(expr); +} + +static void FreeExprInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + struct ExprTree *expr = (void *)objPtr->internalRep.ptr; + + if (expr) { + if (--expr->inUse != 0) { + return; + } + + ExprTreeFree(interp, expr); + } +} + +static void DupExprInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + JIM_NOTUSED(interp); + JIM_NOTUSED(srcPtr); + + + dupPtr->typePtr = NULL; +} + +struct ExprBuilder { + int parencount; + int level; + ParseToken *token; + ParseToken *first_token; + Jim_Stack stack; + Jim_Obj *exprObjPtr; + Jim_Obj *fileNameObj; + struct JimExprNode *nodes; + struct JimExprNode *next; +}; + +#ifdef DEBUG_SHOW_EXPR +static void JimShowExprNode(struct JimExprNode *node, int level) +{ + int i; + for (i = 0; i < level; i++) { + printf(" "); + } + if (TOKEN_IS_EXPR_OP(node->type)) { + printf("%s\n", jim_tt_name(node->type)); + if (node->left) { + JimShowExprNode(node->left, level + 1); + } + if (node->right) { + JimShowExprNode(node->right, level + 1); + } + if (node->ternary) { + JimShowExprNode(node->ternary, level + 1); + } + } + else { + printf("[%s] %s\n", jim_tt_name(node->type), Jim_String(node->objPtr)); + } +} +#endif + +#define EXPR_UNTIL_CLOSE 0x0001 +#define EXPR_FUNC_ARGS 0x0002 +#define EXPR_TERNARY 0x0004 + +static int ExprTreeBuildTree(Jim_Interp *interp, struct ExprBuilder *builder, int precedence, int flags, int exp_numterms) { + int rc; + struct JimExprNode *node; + + int exp_stacklen = builder->stack.len + exp_numterms; + + if (builder->level++ > 200) { + Jim_SetResultString(interp, "Expression too complex", -1); + return JIM_ERR; + } + + while (builder->token->type != JIM_TT_EOL) { + ParseToken *t = builder->token++; + int prevtt; + + if (t == builder->first_token) { + prevtt = JIM_TT_NONE; + } + else { + prevtt = t[-1].type; + } + + if (t->type == JIM_TT_SUBEXPR_START) { + if (builder->stack.len == exp_stacklen) { + Jim_SetResultFormatted(interp, "unexpected open parenthesis in expression: \"%#s\"", builder->exprObjPtr); + return JIM_ERR; + } + builder->parencount++; + rc = ExprTreeBuildTree(interp, builder, 0, EXPR_UNTIL_CLOSE, 1); + if (rc != JIM_OK) { + return rc; + } + + } + else if (t->type == JIM_TT_SUBEXPR_END) { + if (!(flags & EXPR_UNTIL_CLOSE)) { + if (builder->stack.len == exp_stacklen && builder->level > 1) { + builder->token--; + builder->level--; + return JIM_OK; + } + Jim_SetResultFormatted(interp, "unexpected closing parenthesis in expression: \"%#s\"", builder->exprObjPtr); + return JIM_ERR; + } + builder->parencount--; + if (builder->stack.len == exp_stacklen) { + + break; + } + } + else if (t->type == JIM_TT_SUBEXPR_COMMA) { + if (!(flags & EXPR_FUNC_ARGS)) { + if (builder->stack.len == exp_stacklen) { + + builder->token--; + builder->level--; + return JIM_OK; + } + Jim_SetResultFormatted(interp, "unexpected comma in expression: \"%#s\"", builder->exprObjPtr); + return JIM_ERR; + } + else { + + if (builder->stack.len > exp_stacklen) { + Jim_SetResultFormatted(interp, "too many arguments to math function"); + return JIM_ERR; + } + } + + } + else if (t->type == JIM_EXPROP_COLON) { + if (!(flags & EXPR_TERNARY)) { + if (builder->level != 1) { + + builder->token--; + builder->level--; + return JIM_OK; + } + Jim_SetResultFormatted(interp, ": without ? in expression: \"%#s\"", builder->exprObjPtr); + return JIM_ERR; + } + if (builder->stack.len == exp_stacklen) { + + builder->token--; + builder->level--; + return JIM_OK; + } + + } + else if (TOKEN_IS_EXPR_OP(t->type)) { + const struct Jim_ExprOperator *op; + + + if (TOKEN_IS_EXPR_OP(prevtt) || TOKEN_IS_EXPR_START(prevtt)) { + if (t->type == JIM_EXPROP_SUB) { + t->type = JIM_EXPROP_UNARYMINUS; + } + else if (t->type == JIM_EXPROP_ADD) { + t->type = JIM_EXPROP_UNARYPLUS; + } + } + + op = JimExprOperatorInfoByOpcode(t->type); + + if (op->precedence < precedence || (!(op->attr & OP_RIGHT_ASSOC) && op->precedence == precedence)) { + + builder->token--; + break; + } + + if (op->attr & OP_FUNC) { + if (builder->token->type != JIM_TT_SUBEXPR_START) { + Jim_SetResultString(interp, "missing arguments for math function", -1); + return JIM_ERR; + } + builder->token++; + if (op->arity == 0) { + if (builder->token->type != JIM_TT_SUBEXPR_END) { + Jim_SetResultString(interp, "too many arguments for math function", -1); + return JIM_ERR; + } + builder->token++; + goto noargs; + } + builder->parencount++; + + + rc = ExprTreeBuildTree(interp, builder, 0, EXPR_FUNC_ARGS | EXPR_UNTIL_CLOSE, op->arity); + } + else if (t->type == JIM_EXPROP_TERNARY) { + + rc = ExprTreeBuildTree(interp, builder, op->precedence, EXPR_TERNARY, 2); + } + else { + rc = ExprTreeBuildTree(interp, builder, op->precedence, 0, 1); + } + + if (rc != JIM_OK) { + return rc; + } + +noargs: + node = builder->next++; + node->type = t->type; + + if (op->arity >= 3) { + node->ternary = Jim_StackPop(&builder->stack); + if (node->ternary == NULL) { + goto missingoperand; + } + } + if (op->arity >= 2) { + node->right = Jim_StackPop(&builder->stack); + if (node->right == NULL) { + goto missingoperand; + } + } + if (op->arity >= 1) { + node->left = Jim_StackPop(&builder->stack); + if (node->left == NULL) { +missingoperand: + Jim_SetResultFormatted(interp, "missing operand to %s in expression: \"%#s\"", op->name, builder->exprObjPtr); + builder->next--; + return JIM_ERR; + + } + } + + + Jim_StackPush(&builder->stack, node); + } + else { + Jim_Obj *objPtr = NULL; + + + + + if (!TOKEN_IS_EXPR_START(prevtt) && !TOKEN_IS_EXPR_OP(prevtt)) { + Jim_SetResultFormatted(interp, "missing operator in expression: \"%#s\"", builder->exprObjPtr); + return JIM_ERR; + } + + + if (t->type == JIM_TT_EXPR_INT || t->type == JIM_TT_EXPR_DOUBLE) { + char *endptr; + if (t->type == JIM_TT_EXPR_INT) { + objPtr = Jim_NewIntObj(interp, jim_strtoull(t->token, &endptr)); + } + else { + objPtr = Jim_NewDoubleObj(interp, strtod(t->token, &endptr)); + } + if (endptr != t->token + t->len) { + + Jim_FreeNewObj(interp, objPtr); + objPtr = NULL; + } + } + + if (!objPtr) { + + objPtr = Jim_NewStringObj(interp, t->token, t->len); + if (t->type == JIM_TT_CMD) { + + Jim_SetSourceInfo(interp, objPtr, builder->fileNameObj, t->line); + } + } + + + node = builder->next++; + node->objPtr = objPtr; + Jim_IncrRefCount(node->objPtr); + node->type = t->type; + Jim_StackPush(&builder->stack, node); + } + } + + if (builder->stack.len == exp_stacklen) { + builder->level--; + return JIM_OK; + } + + if ((flags & EXPR_FUNC_ARGS)) { + Jim_SetResultFormatted(interp, "too %s arguments for math function", (builder->stack.len < exp_stacklen) ? "few" : "many"); + } + else { + if (builder->stack.len < exp_stacklen) { + if (builder->level == 0) { + Jim_SetResultFormatted(interp, "empty expression"); + } + else { + Jim_SetResultFormatted(interp, "syntax error in expression \"%#s\": premature end of expression", builder->exprObjPtr); + } + } + else { + Jim_SetResultFormatted(interp, "extra terms after expression"); + } + } + + return JIM_ERR; +} + +static struct ExprTree *ExprTreeCreateTree(Jim_Interp *interp, const ParseTokenList *tokenlist, Jim_Obj *exprObjPtr, Jim_Obj *fileNameObj) +{ + struct ExprTree *expr; + struct ExprBuilder builder; + int rc; + struct JimExprNode *top = NULL; + + builder.parencount = 0; + builder.level = 0; + builder.token = builder.first_token = tokenlist->list; + builder.exprObjPtr = exprObjPtr; + builder.fileNameObj = fileNameObj; + + builder.nodes = Jim_Alloc(sizeof(struct JimExprNode) * (tokenlist->count - 1)); + memset(builder.nodes, 0, sizeof(struct JimExprNode) * (tokenlist->count - 1)); + builder.next = builder.nodes; + Jim_InitStack(&builder.stack); + + rc = ExprTreeBuildTree(interp, &builder, 0, 0, 1); + + if (rc == JIM_OK) { + top = Jim_StackPop(&builder.stack); + + if (builder.parencount) { + Jim_SetResultString(interp, "missing close parenthesis", -1); + rc = JIM_ERR; + } + } + + + Jim_FreeStack(&builder.stack); + + if (rc != JIM_OK) { + ExprTreeFreeNodes(interp, builder.nodes, builder.next - builder.nodes); + return NULL; + } + + expr = Jim_Alloc(sizeof(*expr)); + expr->inUse = 1; + expr->expr = top; + expr->nodes = builder.nodes; + expr->len = builder.next - builder.nodes; + + assert(expr->len <= tokenlist->count - 1); + + return expr; +} + +static int SetExprFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) +{ + int exprTextLen; + const char *exprText; + struct JimParserCtx parser; + struct ExprTree *expr; + ParseTokenList tokenlist; + int line; + Jim_Obj *fileNameObj; + int rc = JIM_ERR; + + + fileNameObj = Jim_GetSourceInfo(interp, objPtr, &line); + Jim_IncrRefCount(fileNameObj); + + exprText = Jim_GetString(objPtr, &exprTextLen); + + + ScriptTokenListInit(&tokenlist); + + JimParserInit(&parser, exprText, exprTextLen, line); + while (!parser.eof) { + if (JimParseExpression(&parser) != JIM_OK) { + ScriptTokenListFree(&tokenlist); + Jim_SetResultFormatted(interp, "syntax error in expression: \"%#s\"", objPtr); + if (parser.errmsg) { + Jim_AppendStrings(interp, Jim_GetResult(interp), ": ", parser.errmsg, NULL); + } + expr = NULL; + goto err; + } + + ScriptAddToken(&tokenlist, parser.tstart, parser.tend - parser.tstart + 1, parser.tt, + parser.tline); + } + +#ifdef DEBUG_SHOW_EXPR_TOKENS + { + int i; + printf("==== Expr Tokens (%s) ====\n", Jim_String(fileNameObj)); + for (i = 0; i < tokenlist.count; i++) { + printf("[%2d]@%d %s '%.*s'\n", i, tokenlist.list[i].line, jim_tt_name(tokenlist.list[i].type), + tokenlist.list[i].len, tokenlist.list[i].token); + } + } +#endif + + if (tokenlist.count <= 1) { + Jim_SetResultString(interp, "empty expression", -1); + rc = JIM_ERR; + } + else { + rc = JimParseCheckMissing(interp, parser.missing.ch); + } + if (rc != JIM_OK) { + ScriptTokenListFree(&tokenlist); + Jim_DecrRefCount(interp, fileNameObj); + return rc; + } + + + expr = ExprTreeCreateTree(interp, &tokenlist, objPtr, fileNameObj); + + + ScriptTokenListFree(&tokenlist); + + if (!expr) { + goto err; + } + +#ifdef DEBUG_SHOW_EXPR + printf("==== Expr ====\n"); + JimShowExprNode(expr->expr, 0); +#endif + + rc = JIM_OK; + + err: + + Jim_DecrRefCount(interp, fileNameObj); + Jim_FreeIntRep(interp, objPtr); + Jim_SetIntRepPtr(objPtr, expr); + objPtr->typePtr = &exprObjType; + return rc; +} + +static struct ExprTree *JimGetExpression(Jim_Interp *interp, Jim_Obj *objPtr) +{ + if (objPtr->typePtr != &exprObjType) { + if (SetExprFromAny(interp, objPtr) != JIM_OK) { + return NULL; + } + } + return (struct ExprTree *) Jim_GetIntRepPtr(objPtr); +} + +#ifdef JIM_OPTIMIZATION +static Jim_Obj *JimExprIntValOrVar(Jim_Interp *interp, struct JimExprNode *node) +{ + if (node->type == JIM_TT_EXPR_INT) + return node->objPtr; + else if (node->type == JIM_TT_VAR) + return Jim_GetVariable(interp, node->objPtr, JIM_NONE); + else if (node->type == JIM_TT_DICTSUGAR) + return JimExpandDictSugar(interp, node->objPtr); + else + return NULL; +} +#endif + + +static int JimExprEvalTermNode(Jim_Interp *interp, struct JimExprNode *node) +{ + if (TOKEN_IS_EXPR_OP(node->type)) { + const struct Jim_ExprOperator *op = JimExprOperatorInfoByOpcode(node->type); + return op->funcop(interp, node); + } + else { + Jim_Obj *objPtr; + + + switch (node->type) { + case JIM_TT_EXPR_INT: + case JIM_TT_EXPR_DOUBLE: + case JIM_TT_EXPR_BOOLEAN: + case JIM_TT_STR: + Jim_SetResult(interp, node->objPtr); + return JIM_OK; + + case JIM_TT_VAR: + objPtr = Jim_GetVariable(interp, node->objPtr, JIM_ERRMSG); + if (objPtr) { + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + return JIM_ERR; + + case JIM_TT_DICTSUGAR: + objPtr = JimExpandDictSugar(interp, node->objPtr); + if (objPtr) { + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + return JIM_ERR; + + case JIM_TT_ESC: + if (interp->safeexpr) { + return JIM_ERR; + } + if (Jim_SubstObj(interp, node->objPtr, &objPtr, JIM_NONE) == JIM_OK) { + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + return JIM_ERR; + + case JIM_TT_CMD: + if (interp->safeexpr) { + return JIM_ERR; + } + return Jim_EvalObj(interp, node->objPtr); + + default: + + return JIM_ERR; + } + } +} + +static int JimExprGetTerm(Jim_Interp *interp, struct JimExprNode *node, Jim_Obj **objPtrPtr) +{ + int rc = JimExprEvalTermNode(interp, node); + if (rc == JIM_OK) { + *objPtrPtr = Jim_GetResult(interp); + Jim_IncrRefCount(*objPtrPtr); + } + return rc; +} + +static int JimExprGetTermBoolean(Jim_Interp *interp, struct JimExprNode *node) +{ + if (JimExprEvalTermNode(interp, node) == JIM_OK) { + return ExprBool(interp, Jim_GetResult(interp)); + } + return -1; +} + +int Jim_EvalExpression(Jim_Interp *interp, Jim_Obj *exprObjPtr) +{ + struct ExprTree *expr; + int retcode = JIM_OK; + + Jim_IncrRefCount(exprObjPtr); + expr = JimGetExpression(interp, exprObjPtr); + if (!expr) { + retcode = JIM_ERR; + goto done; + } + +#ifdef JIM_OPTIMIZATION + if (!interp->safeexpr) { + Jim_Obj *objPtr; + + + switch (expr->len) { + case 1: + objPtr = JimExprIntValOrVar(interp, expr->expr); + if (objPtr) { + Jim_SetResult(interp, objPtr); + goto done; + } + break; + + case 2: + if (expr->expr->type == JIM_EXPROP_NOT) { + objPtr = JimExprIntValOrVar(interp, expr->expr->left); + + if (objPtr && JimIsWide(objPtr)) { + Jim_SetResult(interp, JimWideValue(objPtr) ? interp->falseObj : interp->trueObj); + goto done; + } + } + break; + + case 3: + objPtr = JimExprIntValOrVar(interp, expr->expr->left); + if (objPtr && JimIsWide(objPtr)) { + Jim_Obj *objPtr2 = JimExprIntValOrVar(interp, expr->expr->right); + if (objPtr2 && JimIsWide(objPtr2)) { + jim_wide wideValueA = JimWideValue(objPtr); + jim_wide wideValueB = JimWideValue(objPtr2); + int cmpRes; + switch (expr->expr->type) { + case JIM_EXPROP_LT: + cmpRes = wideValueA < wideValueB; + break; + case JIM_EXPROP_LTE: + cmpRes = wideValueA <= wideValueB; + break; + case JIM_EXPROP_GT: + cmpRes = wideValueA > wideValueB; + break; + case JIM_EXPROP_GTE: + cmpRes = wideValueA >= wideValueB; + break; + case JIM_EXPROP_NUMEQ: + cmpRes = wideValueA == wideValueB; + break; + case JIM_EXPROP_NUMNE: + cmpRes = wideValueA != wideValueB; + break; + default: + goto noopt; + } + Jim_SetResult(interp, cmpRes ? interp->trueObj : interp->falseObj); + goto done; + } + } + break; + } + } +noopt: +#endif + + expr->inUse++; + + + retcode = JimExprEvalTermNode(interp, expr->expr); + + + Jim_FreeIntRep(interp, exprObjPtr); + exprObjPtr->typePtr = &exprObjType; + Jim_SetIntRepPtr(exprObjPtr, expr); + +done: + Jim_DecrRefCount(interp, exprObjPtr); + + return retcode; +} + +int Jim_GetBoolFromExpr(Jim_Interp *interp, Jim_Obj *exprObjPtr, int *boolPtr) +{ + int retcode = Jim_EvalExpression(interp, exprObjPtr); + + if (retcode == JIM_OK) { + switch (ExprBool(interp, Jim_GetResult(interp))) { + case 0: + *boolPtr = 0; + break; + + case 1: + *boolPtr = 1; + break; + + case -1: + retcode = JIM_ERR; + break; + } + } + return retcode; +} + + + + +typedef struct ScanFmtPartDescr +{ + const char *arg; + const char *prefix; + size_t width; + int pos; + char type; + char modifier; +} ScanFmtPartDescr; + + +typedef struct ScanFmtStringObj +{ + jim_wide size; + char *stringRep; + size_t count; + size_t convCount; + size_t maxPos; + const char *error; + char *scratch; + ScanFmtPartDescr descr[1]; +} ScanFmtStringObj; + + +static void FreeScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); +static void DupScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); +static void UpdateStringOfScanFmt(Jim_Obj *objPtr); + +static const Jim_ObjType scanFmtStringObjType = { + "scanformatstring", + FreeScanFmtInternalRep, + DupScanFmtInternalRep, + UpdateStringOfScanFmt, + JIM_TYPE_NONE, +}; + +void FreeScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) +{ + JIM_NOTUSED(interp); + Jim_Free((char *)objPtr->internalRep.ptr); + objPtr->internalRep.ptr = 0; +} + +void DupScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) +{ + size_t size = (size_t) ((ScanFmtStringObj *) srcPtr->internalRep.ptr)->size; + ScanFmtStringObj *newVec = (ScanFmtStringObj *) Jim_Alloc(size); + + JIM_NOTUSED(interp); + memcpy(newVec, srcPtr->internalRep.ptr, size); + dupPtr->internalRep.ptr = newVec; + dupPtr->typePtr = &scanFmtStringObjType; +} + +static void UpdateStringOfScanFmt(Jim_Obj *objPtr) +{ + JimSetStringBytes(objPtr, ((ScanFmtStringObj *) objPtr->internalRep.ptr)->stringRep); +} + + +static int SetScanFmtFromAny(Jim_Interp *interp, Jim_Obj *objPtr) +{ + ScanFmtStringObj *fmtObj; + char *buffer; + int maxCount, i, approxSize, lastPos = -1; + const char *fmt = Jim_String(objPtr); + int maxFmtLen = Jim_Length(objPtr); + const char *fmtEnd = fmt + maxFmtLen; + int curr; + + Jim_FreeIntRep(interp, objPtr); + + for (i = 0, maxCount = 0; i < maxFmtLen; ++i) + if (fmt[i] == '%') + ++maxCount; + + approxSize = sizeof(ScanFmtStringObj) + +(maxCount + 1) * sizeof(ScanFmtPartDescr) + +maxFmtLen * sizeof(char) + 3 + 1 + + maxFmtLen * sizeof(char) + 1 + + maxFmtLen * sizeof(char) + +(maxCount + 1) * sizeof(char) + +1; + fmtObj = (ScanFmtStringObj *) Jim_Alloc(approxSize); + memset(fmtObj, 0, approxSize); + fmtObj->size = approxSize; + fmtObj->maxPos = 0; + fmtObj->scratch = (char *)&fmtObj->descr[maxCount + 1]; + fmtObj->stringRep = fmtObj->scratch + maxFmtLen + 3 + 1; + memcpy(fmtObj->stringRep, fmt, maxFmtLen); + buffer = fmtObj->stringRep + maxFmtLen + 1; + objPtr->internalRep.ptr = fmtObj; + objPtr->typePtr = &scanFmtStringObjType; + for (i = 0, curr = 0; fmt < fmtEnd; ++fmt) { + int width = 0, skip; + ScanFmtPartDescr *descr = &fmtObj->descr[curr]; + + fmtObj->count++; + descr->width = 0; + + if (*fmt != '%' || fmt[1] == '%') { + descr->type = 0; + descr->prefix = &buffer[i]; + for (; fmt < fmtEnd; ++fmt) { + if (*fmt == '%') { + if (fmt[1] != '%') + break; + ++fmt; + } + buffer[i++] = *fmt; + } + buffer[i++] = 0; + } + + ++fmt; + + if (fmt >= fmtEnd) + goto done; + descr->pos = 0; + if (*fmt == '*') { + descr->pos = -1; + ++fmt; + } + else + fmtObj->convCount++; + + if (sscanf(fmt, "%d%n", &width, &skip) == 1) { + fmt += skip; + + if (descr->pos != -1 && *fmt == '$') { + int prev; + + ++fmt; + descr->pos = width; + width = 0; + + if ((lastPos == 0 && descr->pos > 0) + || (lastPos > 0 && descr->pos == 0)) { + fmtObj->error = "cannot mix \"%\" and \"%n$\" conversion specifiers"; + return JIM_ERR; + } + + for (prev = 0; prev < curr; ++prev) { + if (fmtObj->descr[prev].pos == -1) + continue; + if (fmtObj->descr[prev].pos == descr->pos) { + fmtObj->error = + "variable is assigned by multiple \"%n$\" conversion specifiers"; + return JIM_ERR; + } + } + if (descr->pos < 0) { + fmtObj->error = + "\"%n$\" conversion specifier is negative"; + return JIM_ERR; + } + + if (sscanf(fmt, "%d%n", &width, &skip) == 1) { + descr->width = width; + fmt += skip; + } + if (descr->pos > 0 && (size_t) descr->pos > fmtObj->maxPos) + fmtObj->maxPos = descr->pos; + } + else { + + descr->width = width; + } + } + + if (lastPos == -1) + lastPos = descr->pos; + + if (*fmt == '[') { + int swapped = 1, beg = i, end, j; + + descr->type = '['; + descr->arg = &buffer[i]; + ++fmt; + if (*fmt == '^') + buffer[i++] = *fmt++; + if (*fmt == ']') + buffer[i++] = *fmt++; + while (*fmt && *fmt != ']') + buffer[i++] = *fmt++; + if (*fmt != ']') { + fmtObj->error = "unmatched [ in format string"; + return JIM_ERR; + } + end = i; + buffer[i++] = 0; + + while (swapped) { + swapped = 0; + for (j = beg + 1; j < end - 1; ++j) { + if (buffer[j] == '-' && buffer[j - 1] > buffer[j + 1]) { + char tmp = buffer[j - 1]; + + buffer[j - 1] = buffer[j + 1]; + buffer[j + 1] = tmp; + swapped = 1; + } + } + } + } + else { + + if (fmt < fmtEnd && strchr("hlL", *fmt)) + descr->modifier = tolower((int)*fmt++); + + if (fmt >= fmtEnd) { + fmtObj->error = "missing scan conversion character"; + return JIM_ERR; + } + + descr->type = *fmt; + if (strchr("efgcsndoxui", *fmt) == 0) { + fmtObj->error = "bad scan conversion character"; + return JIM_ERR; + } + else if (*fmt == 'c' && descr->width != 0) { + fmtObj->error = "field width may not be specified in %c " "conversion"; + return JIM_ERR; + } + else if (*fmt == 'u' && descr->modifier == 'l') { + fmtObj->error = "unsigned wide not supported"; + return JIM_ERR; + } + } + curr++; + } + done: + return JIM_OK; +} + + + +#define FormatGetCnvCount(_fo_) \ + ((ScanFmtStringObj*)((_fo_)->internalRep.ptr))->convCount +#define FormatGetMaxPos(_fo_) \ + ((ScanFmtStringObj*)((_fo_)->internalRep.ptr))->maxPos +#define FormatGetError(_fo_) \ + ((ScanFmtStringObj*)((_fo_)->internalRep.ptr))->error + +static Jim_Obj *JimScanAString(Jim_Interp *interp, const char *sdescr, const char *str) +{ + char *buffer = Jim_StrDup(str); + char *p = buffer; + + while (*str) { + int c; + int n; + + if (!sdescr && isspace(UCHAR(*str))) + break; + + n = utf8_tounicode(str, &c); + if (sdescr && !JimCharsetMatch(sdescr, strlen(sdescr), c, JIM_CHARSET_SCAN)) + break; + while (n--) + *p++ = *str++; + } + *p = 0; + return Jim_NewStringObjNoAlloc(interp, buffer, p - buffer); +} + + +static int ScanOneEntry(Jim_Interp *interp, const char *str, int pos, int str_bytelen, + ScanFmtStringObj * fmtObj, long idx, Jim_Obj **valObjPtr) +{ + const char *tok; + const ScanFmtPartDescr *descr = &fmtObj->descr[idx]; + size_t scanned = 0; + size_t anchor = pos; + int i; + Jim_Obj *tmpObj = NULL; + + + *valObjPtr = 0; + if (descr->prefix) { + for (i = 0; pos < str_bytelen && descr->prefix[i]; ++i) { + + if (isspace(UCHAR(descr->prefix[i]))) + while (pos < str_bytelen && isspace(UCHAR(str[pos]))) + ++pos; + else if (descr->prefix[i] != str[pos]) + break; + else + ++pos; + } + if (pos >= str_bytelen) { + return -1; + } + else if (descr->prefix[i] != 0) + return 0; + } + + if (descr->type != 'c' && descr->type != '[' && descr->type != 'n') + while (isspace(UCHAR(str[pos]))) + ++pos; + + + scanned = pos - anchor; + + + if (descr->type == 'n') { + + *valObjPtr = Jim_NewIntObj(interp, anchor + scanned); + } + else if (pos >= str_bytelen) { + + return -1; + } + else if (descr->type == 'c') { + int c; + scanned += utf8_tounicode(&str[pos], &c); + *valObjPtr = Jim_NewIntObj(interp, c); + return scanned; + } + else { + + if (descr->width > 0) { + size_t sLen = utf8_strlen(&str[pos], str_bytelen - pos); + size_t tLen = descr->width > sLen ? sLen : descr->width; + + tmpObj = Jim_NewStringObjUtf8(interp, str + pos, tLen); + tok = tmpObj->bytes; + } + else { + + tok = &str[pos]; + } + switch (descr->type) { + case 'd': + case 'o': + case 'x': + case 'u': + case 'i':{ + char *endp; + jim_wide w; + + int base = descr->type == 'o' ? 8 + : descr->type == 'x' ? 16 : descr->type == 'i' ? 0 : 10; + + + if (base == 0) { + w = jim_strtoull(tok, &endp); + } + else { + w = strtoull(tok, &endp, base); + } + + if (endp != tok) { + + *valObjPtr = Jim_NewIntObj(interp, w); + + + scanned += endp - tok; + } + else { + scanned = *tok ? 0 : -1; + } + break; + } + case 's': + case '[':{ + *valObjPtr = JimScanAString(interp, descr->arg, tok); + scanned += Jim_Length(*valObjPtr); + break; + } + case 'e': + case 'f': + case 'g':{ + char *endp; + double value = strtod(tok, &endp); + + if (endp != tok) { + + *valObjPtr = Jim_NewDoubleObj(interp, value); + + scanned += endp - tok; + } + else { + scanned = *tok ? 0 : -1; + } + break; + } + } + if (tmpObj) { + Jim_FreeNewObj(interp, tmpObj); + } + } + return scanned; +} + + +Jim_Obj *Jim_ScanString(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *fmtObjPtr, int flags) +{ + size_t i, pos; + int scanned = 1; + const char *str = Jim_String(strObjPtr); + int str_bytelen = Jim_Length(strObjPtr); + Jim_Obj *resultList = 0; + Jim_Obj **resultVec = 0; + int resultc; + Jim_Obj *emptyStr = 0; + ScanFmtStringObj *fmtObj; + + + JimPanic((fmtObjPtr->typePtr != &scanFmtStringObjType, "Jim_ScanString() for non-scan format")); + + fmtObj = (ScanFmtStringObj *) fmtObjPtr->internalRep.ptr; + + if (fmtObj->error != 0) { + if (flags & JIM_ERRMSG) + Jim_SetResultString(interp, fmtObj->error, -1); + return 0; + } + + emptyStr = Jim_NewEmptyStringObj(interp); + Jim_IncrRefCount(emptyStr); + + resultList = Jim_NewListObj(interp, NULL, 0); + if (fmtObj->maxPos > 0) { + for (i = 0; i < fmtObj->maxPos; ++i) + Jim_ListAppendElement(interp, resultList, emptyStr); + JimListGetElements(interp, resultList, &resultc, &resultVec); + } + + for (i = 0, pos = 0; i < fmtObj->count; ++i) { + ScanFmtPartDescr *descr = &(fmtObj->descr[i]); + Jim_Obj *value = 0; + + + if (descr->type == 0) + continue; + + if (scanned > 0) + scanned = ScanOneEntry(interp, str, pos, str_bytelen, fmtObj, i, &value); + + if (scanned == -1 && i == 0) + goto eof; + + pos += scanned; + + + if (value == 0) + value = Jim_NewEmptyStringObj(interp); + + if (descr->pos == -1) { + Jim_FreeNewObj(interp, value); + } + else if (descr->pos == 0) + + Jim_ListAppendElement(interp, resultList, value); + else if (resultVec[descr->pos - 1] == emptyStr) { + + Jim_DecrRefCount(interp, resultVec[descr->pos - 1]); + Jim_IncrRefCount(value); + resultVec[descr->pos - 1] = value; + } + else { + + Jim_FreeNewObj(interp, value); + goto err; + } + } + Jim_DecrRefCount(interp, emptyStr); + return resultList; + eof: + Jim_DecrRefCount(interp, emptyStr); + Jim_FreeNewObj(interp, resultList); + return (Jim_Obj *)EOF; + err: + Jim_DecrRefCount(interp, emptyStr); + Jim_FreeNewObj(interp, resultList); + return 0; +} + + +static void JimPrngInit(Jim_Interp *interp) +{ +#define PRNG_SEED_SIZE 256 + int i; + unsigned int *seed; + time_t t = time(NULL); + + interp->prngState = Jim_Alloc(sizeof(Jim_PrngState)); + + seed = Jim_Alloc(PRNG_SEED_SIZE * sizeof(*seed)); + for (i = 0; i < PRNG_SEED_SIZE; i++) { + seed[i] = (rand() ^ t ^ clock()); + } + JimPrngSeed(interp, (unsigned char *)seed, PRNG_SEED_SIZE * sizeof(*seed)); + Jim_Free(seed); +} + + +static void JimRandomBytes(Jim_Interp *interp, void *dest, unsigned int len) +{ + Jim_PrngState *prng; + unsigned char *destByte = (unsigned char *)dest; + unsigned int si, sj, x; + + + if (interp->prngState == NULL) + JimPrngInit(interp); + prng = interp->prngState; + + for (x = 0; x < len; x++) { + prng->i = (prng->i + 1) & 0xff; + si = prng->sbox[prng->i]; + prng->j = (prng->j + si) & 0xff; + sj = prng->sbox[prng->j]; + prng->sbox[prng->i] = sj; + prng->sbox[prng->j] = si; + *destByte++ = prng->sbox[(si + sj) & 0xff]; + } +} + + +static void JimPrngSeed(Jim_Interp *interp, unsigned char *seed, int seedLen) +{ + int i; + Jim_PrngState *prng; + + + if (interp->prngState == NULL) + JimPrngInit(interp); + prng = interp->prngState; + + + for (i = 0; i < 256; i++) + prng->sbox[i] = i; + + for (i = 0; i < seedLen; i++) { + unsigned char t; + + t = prng->sbox[i & 0xFF]; + prng->sbox[i & 0xFF] = prng->sbox[seed[i]]; + prng->sbox[seed[i]] = t; + } + prng->i = prng->j = 0; + + for (i = 0; i < 256; i += seedLen) { + JimRandomBytes(interp, seed, seedLen); + } +} + + +static int Jim_IncrCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_wide wideValue, increment = 1; + Jim_Obj *intObjPtr; + + if (argc != 2 && argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "varName ?increment?"); + return JIM_ERR; + } + if (argc == 3) { + if (Jim_GetWideExpr(interp, argv[2], &increment) != JIM_OK) + return JIM_ERR; + } + intObjPtr = Jim_GetVariable(interp, argv[1], JIM_UNSHARED); + if (!intObjPtr) { + + wideValue = 0; + } + else if (Jim_GetWide(interp, intObjPtr, &wideValue) != JIM_OK) { + return JIM_ERR; + } + if (!intObjPtr || Jim_IsShared(intObjPtr)) { + intObjPtr = Jim_NewIntObj(interp, wideValue + increment); + if (Jim_SetVariable(interp, argv[1], intObjPtr) != JIM_OK) { + Jim_FreeNewObj(interp, intObjPtr); + return JIM_ERR; + } + } + else { + + Jim_InvalidateStringRep(intObjPtr); + JimWideValue(intObjPtr) = wideValue + increment; + + if (argv[1]->typePtr != &variableObjType) { + + Jim_SetVariable(interp, argv[1], intObjPtr); + } + } + Jim_SetResult(interp, intObjPtr); + return JIM_OK; +} + + +#define JIM_EVAL_SARGV_LEN 8 +#define JIM_EVAL_SINTV_LEN 8 + +static int JimTraceCallback(Jim_Interp *interp, const char *type, int argc, Jim_Obj *const *argv) +{ + JimPanic((interp->traceCmdObj == NULL, "xtrace invoked with no object")); + + int ret; + Jim_Obj *nargv[7]; + Jim_Obj *traceCmdObj = interp->traceCmdObj; + Jim_Obj *resultObj = Jim_GetResult(interp); + ScriptObj *script = NULL; + + + + if (interp->evalFrame->scriptObj) { + script = JimGetScript(interp, interp->evalFrame->scriptObj); + } + + nargv[0] = traceCmdObj; + nargv[1] = Jim_NewStringObj(interp, type, -1); + nargv[2] = script ? script->fileNameObj : interp->emptyObj; + nargv[3] = Jim_NewIntObj(interp, script ? script->linenr : 1); + nargv[4] = resultObj; + nargv[5] = argv[0]; + nargv[6] = Jim_NewListObj(interp, argv + 1, argc - 1); + + + interp->traceCmdObj = NULL; + + Jim_IncrRefCount(resultObj); + ret = Jim_EvalObjVector(interp, 7, nargv); + Jim_DecrRefCount(interp, resultObj); + + if (ret == JIM_OK || ret == JIM_RETURN) { + + interp->traceCmdObj = traceCmdObj; + Jim_SetEmptyResult(interp); + ret = JIM_OK; + } + else { + + Jim_DecrRefCount(interp, traceCmdObj); + } + return ret; +} + + +static int JimUnknown(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int retcode; + + if (interp->unknown_called > 50) { + return JIM_ERR; + } + + + + if (Jim_GetCommand(interp, interp->unknown, JIM_NONE) == NULL) + return JIM_ERR; + + interp->unknown_called++; + + retcode = Jim_EvalObjPrefix(interp, interp->unknown, argc, argv); + interp->unknown_called--; + + return retcode; +} + +static void JimPushEvalFrame(Jim_Interp *interp, Jim_EvalFrame *frame, Jim_Obj *scriptObj) +{ + memset(frame, 0, sizeof(*frame)); + frame->parent = interp->evalFrame; + frame->level = frame->parent->level + 1; + frame->procLevel = interp->procLevel; + frame->framePtr = interp->framePtr; + if (scriptObj) { + frame->scriptObj = scriptObj; + } + else { + frame->scriptObj = frame->parent->scriptObj; + } + interp->evalFrame = frame; +#if 0 + if (frame->scriptObj) { + printf("script: %.*s\n", 20, Jim_String(frame->scriptObj)); + } +#endif +} + +static void JimPopEvalFrame(Jim_Interp *interp) +{ + interp->evalFrame = interp->evalFrame->parent; +} + + +static int JimInvokeCommand(Jim_Interp *interp, int objc, Jim_Obj *const *objv) +{ + int retcode; + Jim_Cmd *cmdPtr; + void *prevPrivData; + Jim_Obj *tailcallObj = NULL; + +#if 0 + printf("invoke"); + int j; + for (j = 0; j < objc; j++) { + printf(" '%s'", Jim_String(objv[j])); + } + printf("\n"); +#endif + + cmdPtr = Jim_GetCommand(interp, objv[0], JIM_ERRMSG); + if (cmdPtr == NULL) { + return JimUnknown(interp, objc, objv); + } + JimIncrCmdRefCount(cmdPtr); + + if (interp->evalDepth == interp->maxEvalDepth) { + Jim_SetResultString(interp, "Infinite eval recursion", -1); + retcode = JIM_ERR; + goto out; + } + interp->evalDepth++; + prevPrivData = interp->cmdPrivData; + +tailcall: + + interp->evalFrame->argc = objc; + interp->evalFrame->argv = objv; + interp->evalFrame->cmd = cmdPtr; + + if (!interp->traceCmdObj || + (retcode = JimTraceCallback(interp, "cmd", objc, objv)) == JIM_OK) { + + Jim_SetEmptyResult(interp); + if (cmdPtr->isproc) { + retcode = JimCallProcedure(interp, cmdPtr, objc, objv); + } + else { + interp->cmdPrivData = cmdPtr->u.native.privData; + retcode = cmdPtr->u.native.cmdProc(interp, objc, objv); + } + if (retcode == JIM_ERR) { + JimSetErrorStack(interp, NULL); + } + } + + if (tailcallObj) { + + Jim_DecrRefCount(interp, tailcallObj); + tailcallObj = NULL; + } + + + interp->evalFrame->argc = 0; + interp->evalFrame->argv = NULL; + + + if (retcode == JIM_EVAL && interp->framePtr->tailcallObj) { + JimDecrCmdRefCount(interp, cmdPtr); + + + cmdPtr = interp->framePtr->tailcallCmd; + interp->framePtr->tailcallCmd = NULL; + tailcallObj = interp->framePtr->tailcallObj; + interp->framePtr->tailcallObj = NULL; + objc = tailcallObj->internalRep.listValue.len; + objv = tailcallObj->internalRep.listValue.ele; + goto tailcall; + } + + interp->cmdPrivData = prevPrivData; + interp->evalDepth--; + +out: + JimDecrCmdRefCount(interp, cmdPtr); + + if (retcode == JIM_ERR) { + JimSetErrorStack(interp, NULL); + } + + if (interp->framePtr->tailcallObj) { + JimDecrCmdRefCount(interp, interp->framePtr->tailcallCmd); + Jim_DecrRefCount(interp, interp->framePtr->tailcallObj); + interp->framePtr->tailcallCmd = NULL; + interp->framePtr->tailcallObj = NULL; + } + + return retcode; +} + +int Jim_EvalObjVector(Jim_Interp *interp, int objc, Jim_Obj *const *objv) +{ + int i, retcode; + Jim_EvalFrame frame; + + + for (i = 0; i < objc; i++) + Jim_IncrRefCount(objv[i]); + + + JimPushEvalFrame(interp, &frame, NULL); + + retcode = JimInvokeCommand(interp, objc, objv); + + JimPopEvalFrame(interp); + + + for (i = 0; i < objc; i++) + Jim_DecrRefCount(interp, objv[i]); + + return retcode; +} + +int Jim_EvalObjPrefix(Jim_Interp *interp, Jim_Obj *prefix, int objc, Jim_Obj *const *objv) +{ + int ret; + Jim_Obj **nargv = Jim_Alloc((objc + 1) * sizeof(*nargv)); + + nargv[0] = prefix; + memcpy(&nargv[1], &objv[0], sizeof(nargv[0]) * objc); + ret = Jim_EvalObjVector(interp, objc + 1, nargv); + Jim_Free(nargv); + return ret; +} + +static int JimSubstOneToken(Jim_Interp *interp, const ScriptToken *token, Jim_Obj **objPtrPtr) +{ + Jim_Obj *objPtr; + int ret = JIM_ERR; + + switch (token->type) { + case JIM_TT_STR: + case JIM_TT_ESC: + objPtr = token->objPtr; + break; + case JIM_TT_VAR: + objPtr = Jim_GetVariable(interp, token->objPtr, JIM_ERRMSG); + break; + case JIM_TT_DICTSUGAR: + objPtr = JimExpandDictSugar(interp, token->objPtr); + break; + case JIM_TT_EXPRSUGAR: + ret = Jim_EvalExpression(interp, token->objPtr); + if (ret == JIM_OK) { + objPtr = Jim_GetResult(interp); + } + else { + objPtr = NULL; + } + break; + case JIM_TT_CMD: + ret = Jim_EvalObj(interp, token->objPtr); + if (ret == JIM_OK || ret == JIM_RETURN) { + objPtr = interp->result; + } else { + + objPtr = NULL; + } + break; + default: + JimPanic((1, + "default token type (%d) reached " "in Jim_SubstObj().", token->type)); + objPtr = NULL; + break; + } + if (objPtr) { + *objPtrPtr = objPtr; + return JIM_OK; + } + return ret; +} + +static Jim_Obj *JimInterpolateTokens(Jim_Interp *interp, const ScriptToken * token, int tokens, int flags) +{ + int totlen = 0, i; + Jim_Obj **intv; + Jim_Obj *sintv[JIM_EVAL_SINTV_LEN]; + Jim_Obj *objPtr; + char *s; + + if (tokens <= JIM_EVAL_SINTV_LEN) + intv = sintv; + else + intv = Jim_Alloc(sizeof(Jim_Obj *) * tokens); + + for (i = 0; i < tokens; i++) { + switch (JimSubstOneToken(interp, &token[i], &intv[i])) { + case JIM_OK: + case JIM_RETURN: + break; + case JIM_BREAK: + if (flags & JIM_SUBST_FLAG) { + + tokens = i; + continue; + } + + + case JIM_CONTINUE: + if (flags & JIM_SUBST_FLAG) { + intv[i] = NULL; + continue; + } + + + default: + while (i--) { + Jim_DecrRefCount(interp, intv[i]); + } + if (intv != sintv) { + Jim_Free(intv); + } + return NULL; + } + Jim_IncrRefCount(intv[i]); + Jim_String(intv[i]); + totlen += intv[i]->length; + } + + + if (tokens == 1 && intv[0] && intv == sintv) { + + intv[0]->refCount--; + return intv[0]; + } + + objPtr = Jim_NewStringObjNoAlloc(interp, NULL, 0); + + if (tokens == 4 && token[0].type == JIM_TT_ESC && token[1].type == JIM_TT_ESC + && token[2].type == JIM_TT_VAR) { + + objPtr->typePtr = &interpolatedObjType; + objPtr->internalRep.dictSubstValue.varNameObjPtr = token[0].objPtr; + objPtr->internalRep.dictSubstValue.indexObjPtr = intv[2]; + Jim_IncrRefCount(intv[2]); + } + else if (tokens && intv[0] && intv[0]->typePtr == &sourceObjType) { + + int line; + Jim_Obj *fileNameObj = Jim_GetSourceInfo(interp, intv[0], &line); + Jim_SetSourceInfo(interp, objPtr, fileNameObj, line); + } + + + s = objPtr->bytes = Jim_Alloc(totlen + 1); + objPtr->length = totlen; + for (i = 0; i < tokens; i++) { + if (intv[i]) { + memcpy(s, intv[i]->bytes, intv[i]->length); + s += intv[i]->length; + Jim_DecrRefCount(interp, intv[i]); + } + } + objPtr->bytes[totlen] = '\0'; + + if (intv != sintv) { + Jim_Free(intv); + } + + return objPtr; +} + + +static int JimEvalObjList(Jim_Interp *interp, Jim_Obj *listPtr) +{ + int retcode = JIM_OK; + Jim_EvalFrame frame; + + JimPanic((Jim_IsList(listPtr) == 0, "JimEvalObjList() invoked on non-list.")); + + JimPushEvalFrame(interp, &frame, NULL); + + if (listPtr->internalRep.listValue.len) { + Jim_IncrRefCount(listPtr); + retcode = JimInvokeCommand(interp, + listPtr->internalRep.listValue.len, + listPtr->internalRep.listValue.ele); + Jim_DecrRefCount(interp, listPtr); + } + + JimPopEvalFrame(interp); + + return retcode; +} + +int Jim_EvalObjList(Jim_Interp *interp, Jim_Obj *listPtr) +{ + SetListFromAny(interp, listPtr); + return JimEvalObjList(interp, listPtr); +} + +int Jim_EvalObj(Jim_Interp *interp, Jim_Obj *scriptObjPtr) +{ + int i; + ScriptObj *script; + ScriptToken *token; + int retcode = JIM_OK; + Jim_Obj *sargv[JIM_EVAL_SARGV_LEN], **argv = NULL; + Jim_EvalFrame frame; + + if (Jim_IsList(scriptObjPtr) && scriptObjPtr->bytes == NULL) { + return JimEvalObjList(interp, scriptObjPtr); + } + + Jim_IncrRefCount(scriptObjPtr); + script = JimGetScript(interp, scriptObjPtr); + if (JimParseCheckMissing(interp, script->missing) == JIM_ERR) { + JimSetErrorStack(interp, script); + Jim_DecrRefCount(interp, scriptObjPtr); + return JIM_ERR; + } + + Jim_SetEmptyResult(interp); + + token = script->token; + +#ifdef JIM_OPTIMIZATION + if (script->len == 0) { + Jim_DecrRefCount(interp, scriptObjPtr); + return JIM_OK; + } + if (script->len == 3 + && token[1].objPtr->typePtr == &commandObjType + && token[1].objPtr->internalRep.cmdValue.cmdPtr->isproc == 0 + && token[1].objPtr->internalRep.cmdValue.cmdPtr->u.native.cmdProc == Jim_IncrCoreCommand + && token[2].objPtr->typePtr == &variableObjType) { + + Jim_Obj *objPtr = Jim_GetVariable(interp, token[2].objPtr, JIM_NONE); + + if (objPtr && !Jim_IsShared(objPtr) && objPtr->typePtr == &intObjType) { + JimWideValue(objPtr)++; + Jim_InvalidateStringRep(objPtr); + Jim_DecrRefCount(interp, scriptObjPtr); + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + } +#endif + + script->inUse++; + + JimPushEvalFrame(interp, &frame, scriptObjPtr); + + + interp->errorFlag = 0; + argv = sargv; + + for (i = 0; i < script->len && retcode == JIM_OK; ) { + int argc; + int j; + + + argc = token[i].objPtr->internalRep.scriptLineValue.argc; + script->linenr = token[i].objPtr->internalRep.scriptLineValue.line; + + + if (argc > JIM_EVAL_SARGV_LEN) + argv = Jim_Alloc(sizeof(Jim_Obj *) * argc); + + + i++; + + for (j = 0; j < argc; j++) { + long wordtokens = 1; + int expand = 0; + Jim_Obj *wordObjPtr = NULL; + + if (token[i].type == JIM_TT_WORD) { + wordtokens = JimWideValue(token[i++].objPtr); + if (wordtokens < 0) { + expand = 1; + wordtokens = -wordtokens; + } + } + + if (wordtokens == 1) { + + switch (token[i].type) { + case JIM_TT_ESC: + case JIM_TT_STR: + wordObjPtr = token[i].objPtr; + break; + case JIM_TT_VAR: + wordObjPtr = Jim_GetVariable(interp, token[i].objPtr, JIM_ERRMSG); + break; + case JIM_TT_EXPRSUGAR: + retcode = Jim_EvalExpression(interp, token[i].objPtr); + if (retcode == JIM_OK) { + wordObjPtr = Jim_GetResult(interp); + } + else { + wordObjPtr = NULL; + } + break; + case JIM_TT_DICTSUGAR: + wordObjPtr = JimExpandDictSugar(interp, token[i].objPtr); + break; + case JIM_TT_CMD: + retcode = Jim_EvalObj(interp, token[i].objPtr); + if (retcode == JIM_OK) { + wordObjPtr = Jim_GetResult(interp); + } + break; + default: + JimPanic((1, "default token type reached " "in Jim_EvalObj().")); + } + } + else { + wordObjPtr = JimInterpolateTokens(interp, token + i, wordtokens, JIM_NONE); + } + + if (!wordObjPtr) { + if (retcode == JIM_OK) { + retcode = JIM_ERR; + } + break; + } + + Jim_IncrRefCount(wordObjPtr); + i += wordtokens; + + if (!expand) { + argv[j] = wordObjPtr; + } + else { + + int len = Jim_ListLength(interp, wordObjPtr); + int newargc = argc + len - 1; + int k; + + if (len > 1) { + if (argv == sargv) { + if (newargc > JIM_EVAL_SARGV_LEN) { + argv = Jim_Alloc(sizeof(*argv) * newargc); + memcpy(argv, sargv, sizeof(*argv) * j); + } + } + else { + + argv = Jim_Realloc(argv, sizeof(*argv) * newargc); + } + } + + + for (k = 0; k < len; k++) { + argv[j++] = wordObjPtr->internalRep.listValue.ele[k]; + Jim_IncrRefCount(wordObjPtr->internalRep.listValue.ele[k]); + } + + Jim_DecrRefCount(interp, wordObjPtr); + + + j--; + argc += len - 1; + } + } + + if (retcode == JIM_OK && argc) { + + retcode = JimInvokeCommand(interp, argc, argv); + + if (Jim_CheckSignal(interp)) { + retcode = JIM_SIGNAL; + } + } + + + while (j-- > 0) { + Jim_DecrRefCount(interp, argv[j]); + } + + if (argv != sargv) { + Jim_Free(argv); + argv = sargv; + } + } + + + if (retcode == JIM_ERR) { + JimSetErrorStack(interp, NULL); + } + + JimPopEvalFrame(interp); + + Jim_FreeIntRep(interp, scriptObjPtr); + scriptObjPtr->typePtr = &scriptObjType; + Jim_SetIntRepPtr(scriptObjPtr, script); + Jim_DecrRefCount(interp, scriptObjPtr); + + return retcode; +} + +static int JimSetProcArg(Jim_Interp *interp, Jim_Obj *argNameObj, Jim_Obj *argValObj) +{ + int retcode; + + const char *varname = Jim_String(argNameObj); + if (*varname == '&') { + + Jim_Obj *objPtr; + Jim_CallFrame *savedCallFrame = interp->framePtr; + + interp->framePtr = interp->framePtr->parent; + objPtr = Jim_GetVariable(interp, argValObj, JIM_ERRMSG); + interp->framePtr = savedCallFrame; + if (!objPtr) { + return JIM_ERR; + } + + + objPtr = Jim_NewStringObj(interp, varname + 1, -1); + Jim_IncrRefCount(objPtr); + retcode = Jim_SetVariableLink(interp, objPtr, argValObj, interp->framePtr->parent); + Jim_DecrRefCount(interp, objPtr); + } + else { + retcode = Jim_SetVariable(interp, argNameObj, argValObj); + } + return retcode; +} + +static void JimSetProcWrongArgs(Jim_Interp *interp, Jim_Obj *procNameObj, Jim_Cmd *cmd) +{ + + Jim_Obj *argmsg = Jim_NewStringObj(interp, "", 0); + int i; + + for (i = 0; i < cmd->u.proc.argListLen; i++) { + Jim_AppendString(interp, argmsg, " ", 1); + + if (i == cmd->u.proc.argsPos) { + if (cmd->u.proc.arglist[i].defaultObjPtr) { + + Jim_AppendString(interp, argmsg, "?", 1); + Jim_AppendObj(interp, argmsg, cmd->u.proc.arglist[i].defaultObjPtr); + Jim_AppendString(interp, argmsg, " ...?", -1); + } + else { + + Jim_AppendString(interp, argmsg, "?arg ...?", -1); + } + } + else { + if (cmd->u.proc.arglist[i].defaultObjPtr) { + Jim_AppendString(interp, argmsg, "?", 1); + Jim_AppendObj(interp, argmsg, cmd->u.proc.arglist[i].nameObjPtr); + Jim_AppendString(interp, argmsg, "?", 1); + } + else { + const char *arg = Jim_String(cmd->u.proc.arglist[i].nameObjPtr); + if (*arg == '&') { + arg++; + } + Jim_AppendString(interp, argmsg, arg, -1); + } + } + } + Jim_SetResultFormatted(interp, "wrong # args: should be \"%#s%#s\"", procNameObj, argmsg); +} + +#ifdef jim_ext_namespace +int Jim_EvalNamespace(Jim_Interp *interp, Jim_Obj *scriptObj, Jim_Obj *nsObj) +{ + Jim_CallFrame *callFramePtr; + int retcode; + + + callFramePtr = JimCreateCallFrame(interp, interp->framePtr, nsObj); + callFramePtr->argv = interp->evalFrame->argv; + callFramePtr->argc = interp->evalFrame->argc; + callFramePtr->procArgsObjPtr = NULL; + callFramePtr->procBodyObjPtr = scriptObj; + callFramePtr->staticVars = NULL; + Jim_IncrRefCount(scriptObj); + interp->framePtr = callFramePtr; + + + if (interp->framePtr->level == interp->maxCallFrameDepth) { + Jim_SetResultString(interp, "Too many nested calls. Infinite recursion?", -1); + retcode = JIM_ERR; + } + else { + + retcode = Jim_EvalObj(interp, scriptObj); + } + + + interp->framePtr = interp->framePtr->parent; + JimFreeCallFrame(interp, callFramePtr, JIM_FCF_REUSE); + + return retcode; +} +#endif + +static int JimCallProcedure(Jim_Interp *interp, Jim_Cmd *cmd, int argc, Jim_Obj *const *argv) +{ + Jim_CallFrame *callFramePtr; + int i, d, retcode, optargs; + + + if (argc - 1 < cmd->u.proc.reqArity || + (cmd->u.proc.argsPos < 0 && argc - 1 > cmd->u.proc.reqArity + cmd->u.proc.optArity)) { + JimSetProcWrongArgs(interp, argv[0], cmd); + return JIM_ERR; + } + + if (Jim_Length(cmd->u.proc.bodyObjPtr) == 0) { + + return JIM_OK; + } + + + if (interp->framePtr->level == interp->maxCallFrameDepth) { + Jim_SetResultString(interp, "Too many nested calls. Infinite recursion?", -1); + return JIM_ERR; + } + + + callFramePtr = JimCreateCallFrame(interp, interp->framePtr, cmd->u.proc.nsObj); + callFramePtr->argv = argv; + callFramePtr->argc = argc; + callFramePtr->procArgsObjPtr = cmd->u.proc.argListObjPtr; + callFramePtr->procBodyObjPtr = cmd->u.proc.bodyObjPtr; + callFramePtr->staticVars = cmd->u.proc.staticVars; + + interp->procLevel++; + + Jim_IncrRefCount(cmd->u.proc.argListObjPtr); + Jim_IncrRefCount(cmd->u.proc.bodyObjPtr); + interp->framePtr = callFramePtr; + + + optargs = (argc - 1 - cmd->u.proc.reqArity); + + + i = 1; + for (d = 0; d < cmd->u.proc.argListLen; d++) { + Jim_Obj *nameObjPtr = cmd->u.proc.arglist[d].nameObjPtr; + if (d == cmd->u.proc.argsPos) { + + Jim_Obj *listObjPtr; + int argsLen = 0; + if (cmd->u.proc.reqArity + cmd->u.proc.optArity < argc - 1) { + argsLen = argc - 1 - (cmd->u.proc.reqArity + cmd->u.proc.optArity); + } + listObjPtr = Jim_NewListObj(interp, &argv[i], argsLen); + + + if (cmd->u.proc.arglist[d].defaultObjPtr) { + nameObjPtr =cmd->u.proc.arglist[d].defaultObjPtr; + } + retcode = Jim_SetVariable(interp, nameObjPtr, listObjPtr); + if (retcode != JIM_OK) { + goto badargset; + } + + i += argsLen; + continue; + } + + + if (cmd->u.proc.arglist[d].defaultObjPtr == NULL || optargs-- > 0) { + retcode = JimSetProcArg(interp, nameObjPtr, argv[i++]); + } + else { + + retcode = Jim_SetVariable(interp, nameObjPtr, cmd->u.proc.arglist[d].defaultObjPtr); + } + if (retcode != JIM_OK) { + goto badargset; + } + } + + if (interp->traceCmdObj == NULL || + (retcode = JimTraceCallback(interp, "proc", argc, argv)) == JIM_OK) { + + retcode = Jim_EvalObj(interp, cmd->u.proc.bodyObjPtr); + } + +badargset: + + + retcode = JimInvokeDefer(interp, retcode); + interp->framePtr = interp->framePtr->parent; + JimFreeCallFrame(interp, callFramePtr, JIM_FCF_REUSE); + + + if (retcode == JIM_RETURN) { + if (--interp->returnLevel <= 0) { + retcode = interp->returnCode; + interp->returnCode = JIM_OK; + interp->returnLevel = 0; + } + } + interp->procLevel--; + + return retcode; +} + +int Jim_EvalSource(Jim_Interp *interp, const char *filename, int lineno, const char *script) +{ + int retval; + Jim_Obj *scriptObjPtr; + + scriptObjPtr = Jim_NewStringObj(interp, script, -1); + Jim_IncrRefCount(scriptObjPtr); + if (filename) { + Jim_SetSourceInfo(interp, scriptObjPtr, Jim_NewStringObj(interp, filename, -1), lineno); + } + retval = Jim_EvalObj(interp, scriptObjPtr); + Jim_DecrRefCount(interp, scriptObjPtr); + return retval; +} + +int Jim_Eval(Jim_Interp *interp, const char *script) +{ + return Jim_EvalObj(interp, Jim_NewStringObj(interp, script, -1)); +} + + +int Jim_EvalGlobal(Jim_Interp *interp, const char *script) +{ + int retval; + Jim_CallFrame *savedFramePtr = interp->framePtr; + + interp->framePtr = interp->topFramePtr; + retval = Jim_Eval(interp, script); + interp->framePtr = savedFramePtr; + + return retval; +} + +int Jim_EvalFileGlobal(Jim_Interp *interp, const char *filename) +{ + int retval; + Jim_CallFrame *savedFramePtr = interp->framePtr; + + interp->framePtr = interp->topFramePtr; + retval = Jim_EvalFile(interp, filename); + interp->framePtr = savedFramePtr; + + return retval; +} + +#include + +static Jim_Obj *JimReadTextFile(Jim_Interp *interp, const char *filename) +{ + jim_stat_t sb; + int fd; + char *buf; + int readlen; + + if (Jim_Stat(filename, &sb) == -1 || (fd = open(filename, O_RDONLY | O_TEXT, 0666)) < 0) { + Jim_SetResultFormatted(interp, "couldn't read file \"%s\": %s", filename, strerror(errno)); + return NULL; + } + buf = Jim_Alloc(sb.st_size + 1); + readlen = read(fd, buf, sb.st_size); + close(fd); + if (readlen < 0) { + Jim_Free(buf); + Jim_SetResultFormatted(interp, "failed to load file \"%s\": %s", filename, strerror(errno)); + return NULL; + } + else { + Jim_Obj *objPtr; + buf[readlen] = 0; + + objPtr = Jim_NewStringObjNoAlloc(interp, buf, readlen); + + return objPtr; + } +} + + +int Jim_EvalFile(Jim_Interp *interp, const char *filename) +{ + Jim_Obj *filenameObj; + Jim_Obj *oldFilenameObj; + Jim_Obj *scriptObjPtr; + int retcode; + + scriptObjPtr = JimReadTextFile(interp, filename); + if (!scriptObjPtr) { + return JIM_ERR; + } + + filenameObj = Jim_NewStringObj(interp, filename, -1); + Jim_SetSourceInfo(interp, scriptObjPtr, filenameObj, 1); + + oldFilenameObj = JimPushInterpObj(interp->currentFilenameObj, filenameObj); + + retcode = Jim_EvalObj(interp, scriptObjPtr); + + JimPopInterpObj(interp, interp->currentFilenameObj, oldFilenameObj); + + + if (retcode == JIM_RETURN) { + if (--interp->returnLevel <= 0) { + retcode = interp->returnCode; + interp->returnCode = JIM_OK; + interp->returnLevel = 0; + } + } + + return retcode; +} + +static void JimParseSubst(struct JimParserCtx *pc, int flags) +{ + pc->tstart = pc->p; + pc->tline = pc->linenr; + + if (pc->len == 0) { + pc->tend = pc->p; + pc->tt = JIM_TT_EOL; + pc->eof = 1; + return; + } + if (*pc->p == '[' && !(flags & JIM_SUBST_NOCMD)) { + JimParseCmd(pc); + return; + } + if (*pc->p == '$' && !(flags & JIM_SUBST_NOVAR)) { + if (JimParseVar(pc) == JIM_OK) { + return; + } + + pc->tstart = pc->p; + + pc->p++; + pc->len--; + } + while (pc->len) { + if (*pc->p == '$' && !(flags & JIM_SUBST_NOVAR)) { + break; + } + if (*pc->p == '[' && !(flags & JIM_SUBST_NOCMD)) { + break; + } + if (*pc->p == '\\' && pc->len > 1) { + pc->p++; + pc->len--; + } + pc->p++; + pc->len--; + } + pc->tend = pc->p - 1; + pc->tt = (flags & JIM_SUBST_NOESC) ? JIM_TT_STR : JIM_TT_ESC; +} + + +static int SetSubstFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr, int flags) +{ + int scriptTextLen; + const char *scriptText = Jim_GetString(objPtr, &scriptTextLen); + struct JimParserCtx parser; + struct ScriptObj *script = Jim_Alloc(sizeof(*script)); + ParseTokenList tokenlist; + + + ScriptTokenListInit(&tokenlist); + + JimParserInit(&parser, scriptText, scriptTextLen, 1); + while (1) { + JimParseSubst(&parser, flags); + if (parser.eof) { + + break; + } + ScriptAddToken(&tokenlist, parser.tstart, parser.tend - parser.tstart + 1, parser.tt, + parser.tline); + } + + + script->inUse = 1; + script->substFlags = flags; + script->fileNameObj = interp->emptyObj; + Jim_IncrRefCount(script->fileNameObj); + SubstObjAddTokens(interp, script, &tokenlist); + + + ScriptTokenListFree(&tokenlist); + +#ifdef DEBUG_SHOW_SUBST + { + int i; + + printf("==== Subst ====\n"); + for (i = 0; i < script->len; i++) { + printf("[%2d] %s '%s'\n", i, jim_tt_name(script->token[i].type), + Jim_String(script->token[i].objPtr)); + } + } +#endif + + + Jim_FreeIntRep(interp, objPtr); + Jim_SetIntRepPtr(objPtr, script); + objPtr->typePtr = &scriptObjType; + return JIM_OK; +} + +static ScriptObj *Jim_GetSubst(Jim_Interp *interp, Jim_Obj *objPtr, int flags) +{ + if (objPtr->typePtr != &scriptObjType || ((ScriptObj *)Jim_GetIntRepPtr(objPtr))->substFlags != flags) + SetSubstFromAny(interp, objPtr, flags); + return (ScriptObj *) Jim_GetIntRepPtr(objPtr); +} + +int Jim_SubstObj(Jim_Interp *interp, Jim_Obj *substObjPtr, Jim_Obj **resObjPtrPtr, int flags) +{ + ScriptObj *script; + + JimPanic((substObjPtr->refCount == 0, "Jim_SubstObj() called with zero refcount object")); + + script = Jim_GetSubst(interp, substObjPtr, flags); + + Jim_IncrRefCount(substObjPtr); + script->inUse++; + + *resObjPtrPtr = JimInterpolateTokens(interp, script->token, script->len, flags); + + script->inUse--; + Jim_DecrRefCount(interp, substObjPtr); + if (*resObjPtrPtr == NULL) { + return JIM_ERR; + } + return JIM_OK; +} + +void Jim_WrongNumArgs(Jim_Interp *interp, int argc, Jim_Obj *const *argv, const char *msg) +{ + Jim_Obj *objPtr; + Jim_Obj *listObjPtr; + + JimPanic((argc == 0, "Jim_WrongNumArgs() called with argc=0")); + + listObjPtr = Jim_NewListObj(interp, argv, argc); + + if (msg && *msg) { + Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, msg, -1)); + } + Jim_IncrRefCount(listObjPtr); + objPtr = Jim_ListJoin(interp, listObjPtr, " ", 1); + Jim_DecrRefCount(interp, listObjPtr); + + Jim_SetResultFormatted(interp, "wrong # args: should be \"%#s\"", objPtr); +} + +typedef void JimHashtableIteratorCallbackType(Jim_Interp *interp, Jim_Obj *listObjPtr, + Jim_Obj *keyObjPtr, void *value, Jim_Obj *patternObjPtr, int type); + +#define JimTrivialMatch(pattern) (strpbrk((pattern), "*[?\\") == NULL) + +static Jim_Obj *JimHashtablePatternMatch(Jim_Interp *interp, Jim_HashTable *ht, Jim_Obj *patternObjPtr, + JimHashtableIteratorCallbackType *callback, int type) +{ + Jim_HashEntry *he; + Jim_Obj *listObjPtr = Jim_NewListObj(interp, NULL, 0); + + + if (patternObjPtr && JimTrivialMatch(Jim_String(patternObjPtr))) { + he = Jim_FindHashEntry(ht, patternObjPtr); + if (he) { + callback(interp, listObjPtr, Jim_GetHashEntryKey(he), Jim_GetHashEntryVal(he), + patternObjPtr, type); + } + } + else { + Jim_HashTableIterator htiter; + JimInitHashTableIterator(ht, &htiter); + while ((he = Jim_NextHashEntry(&htiter)) != NULL) { + callback(interp, listObjPtr, Jim_GetHashEntryKey(he), Jim_GetHashEntryVal(he), + patternObjPtr, type); + } + } + return listObjPtr; +} + + +#define JIM_CMDLIST_COMMANDS 0 +#define JIM_CMDLIST_PROCS 1 +#define JIM_CMDLIST_CHANNELS 2 + +static void JimCommandMatch(Jim_Interp *interp, Jim_Obj *listObjPtr, + Jim_Obj *keyObj, void *value, Jim_Obj *patternObj, int type) +{ + Jim_Cmd *cmdPtr = (Jim_Cmd *)value; + + if (type == JIM_CMDLIST_PROCS && !cmdPtr->isproc) { + + return; + } + + Jim_IncrRefCount(keyObj); + + if (type != JIM_CMDLIST_CHANNELS || Jim_AioFilehandle(interp, keyObj) >= 0) { + int match = 1; + if (patternObj) { + int plen, slen; + const char *pattern = Jim_GetStringNoQualifier(patternObj, &plen); + const char *str = Jim_GetStringNoQualifier(keyObj, &slen); +#ifdef JIM_NO_INTROSPECTION + + match = (JimStringCompareUtf8(pattern, plen, str, slen, 0) == 0); +#else + match = JimGlobMatch(pattern, plen, str, slen, 0); +#endif + } + if (match) { + Jim_ListAppendElement(interp, listObjPtr, keyObj); + } + } + Jim_DecrRefCount(interp, keyObj); +} + +static Jim_Obj *JimCommandsList(Jim_Interp *interp, Jim_Obj *patternObjPtr, int type) +{ + return JimHashtablePatternMatch(interp, &interp->commands, patternObjPtr, JimCommandMatch, type); +} + + +#define JIM_VARLIST_GLOBALS 0 +#define JIM_VARLIST_LOCALS 1 +#define JIM_VARLIST_VARS 2 +#define JIM_VARLIST_MASK 0x000f + +#define JIM_VARLIST_VALUES 0x1000 + +static void JimVariablesMatch(Jim_Interp *interp, Jim_Obj *listObjPtr, + Jim_Obj *keyObj, void *value, Jim_Obj *patternObj, int type) +{ + Jim_VarVal *vv = (Jim_VarVal *)value; + + if ((type & JIM_VARLIST_MASK) != JIM_VARLIST_LOCALS || vv->linkFramePtr == NULL) { + if (patternObj == NULL || Jim_StringMatchObj(interp, patternObj, keyObj, 0)) { + Jim_ListAppendElement(interp, listObjPtr, keyObj); + if (type & JIM_VARLIST_VALUES) { + Jim_ListAppendElement(interp, listObjPtr, vv->objPtr); + } + } + } +} + + +static Jim_Obj *JimVariablesList(Jim_Interp *interp, Jim_Obj *patternObjPtr, int mode) +{ + if (mode == JIM_VARLIST_LOCALS && interp->framePtr == interp->topFramePtr) { + return interp->emptyObj; + } + else { + Jim_CallFrame *framePtr = (mode == JIM_VARLIST_GLOBALS) ? interp->topFramePtr : interp->framePtr; + return JimHashtablePatternMatch(interp, &framePtr->vars, patternObjPtr, JimVariablesMatch, + mode); + } +} + +static int JimInfoLevel(Jim_Interp *interp, Jim_Obj *levelObjPtr, Jim_Obj **objPtrPtr) +{ + long level; + + if (Jim_GetLong(interp, levelObjPtr, &level) == JIM_OK) { + Jim_CallFrame *targetCallFrame = JimGetCallFrameByInteger(interp, level); + if (targetCallFrame && targetCallFrame != interp->topFramePtr) { +#ifdef JIM_NO_INTROSPECTION + + *objPtrPtr = Jim_NewListObj(interp, targetCallFrame->argv, 1); +#else + *objPtrPtr = Jim_NewListObj(interp, targetCallFrame->argv, targetCallFrame->argc); +#endif + return JIM_OK; + } + } + Jim_SetResultFormatted(interp, "bad level \"%#s\"", levelObjPtr); + return JIM_ERR; +} + +static int JimInfoFrame(Jim_Interp *interp, Jim_Obj *levelObjPtr, Jim_Obj **objPtrPtr) +{ + long level; + + if (Jim_GetLong(interp, levelObjPtr, &level) == JIM_OK) { + Jim_EvalFrame *frame = JimGetEvalFrameByProcLevel(interp, level); + if (frame) { + Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); + + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "type", -1)); + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "source", -1)); + if (frame->scriptObj) { + ScriptObj *script = JimGetScript(interp, frame->scriptObj); + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "line", -1)); + Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, script->linenr)); + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "file", -1)); + Jim_ListAppendElement(interp, listObj, script->fileNameObj); + } +#ifndef JIM_NO_INTROSPECTION + { + Jim_Obj *cmdObj = Jim_NewListObj(interp, frame->argv, frame->argc); + + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "cmd", -1)); + Jim_ListAppendElement(interp, listObj, cmdObj); + } +#endif + { + Jim_Obj *procNameObj = JimProcForEvalFrame(interp, frame); + if (procNameObj) { + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "proc", -1)); + Jim_ListAppendElement(interp, listObj, procNameObj); + } + } + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "level", -1)); + Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, interp->framePtr->level - frame->framePtr->level)); + + *objPtrPtr = listObj; + return JIM_OK; + } + } + Jim_SetResultFormatted(interp, "bad level \"%#s\"", levelObjPtr); + return JIM_ERR; +} + + +static int Jim_PutsCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 2 && argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "?-nonewline? string"); + return JIM_ERR; + } + if (argc == 3) { + if (!Jim_CompareStringImmediate(interp, argv[1], "-nonewline")) { + Jim_SetResultString(interp, "The second argument must " "be -nonewline", -1); + return JIM_ERR; + } + else { + fputs(Jim_String(argv[2]), stdout); + } + } + else { + puts(Jim_String(argv[1])); + } + return JIM_OK; +} + + +static int JimAddMulHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int op) +{ + jim_wide wideValue, res; + double doubleValue, doubleRes; + int i; + + res = (op == JIM_EXPROP_ADD) ? 0 : 1; + + for (i = 1; i < argc; i++) { + if (Jim_GetWide(interp, argv[i], &wideValue) != JIM_OK) + goto trydouble; + if (op == JIM_EXPROP_ADD) + res += wideValue; + else + res *= wideValue; + } + Jim_SetResultInt(interp, res); + return JIM_OK; + trydouble: + doubleRes = (double)res; + for (; i < argc; i++) { + if (Jim_GetDouble(interp, argv[i], &doubleValue) != JIM_OK) + return JIM_ERR; + if (op == JIM_EXPROP_ADD) + doubleRes += doubleValue; + else + doubleRes *= doubleValue; + } + Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); + return JIM_OK; +} + + +static int JimSubDivHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int op) +{ + jim_wide wideValue, res = 0; + double doubleValue, doubleRes = 0; + int i = 2; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "number ?number ... number?"); + return JIM_ERR; + } + else if (argc == 2) { + if (Jim_GetWide(interp, argv[1], &wideValue) != JIM_OK) { + if (Jim_GetDouble(interp, argv[1], &doubleValue) != JIM_OK) { + return JIM_ERR; + } + else { + if (op == JIM_EXPROP_SUB) + doubleRes = -doubleValue; + else + doubleRes = 1.0 / doubleValue; + Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); + return JIM_OK; + } + } + if (op == JIM_EXPROP_SUB) { + res = -wideValue; + Jim_SetResultInt(interp, res); + } + else { + doubleRes = 1.0 / wideValue; + Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); + } + return JIM_OK; + } + else { + if (Jim_GetWide(interp, argv[1], &res) != JIM_OK) { + if (Jim_GetDouble(interp, argv[1], &doubleRes) + != JIM_OK) { + return JIM_ERR; + } + else { + goto trydouble; + } + } + } + for (i = 2; i < argc; i++) { + if (Jim_GetWide(interp, argv[i], &wideValue) != JIM_OK) { + doubleRes = (double)res; + goto trydouble; + } + if (op == JIM_EXPROP_SUB) + res -= wideValue; + else { + if (wideValue == 0) { + Jim_SetResultString(interp, "Division by zero", -1); + return JIM_ERR; + } + res /= wideValue; + } + } + Jim_SetResultInt(interp, res); + return JIM_OK; + trydouble: + for (; i < argc; i++) { + if (Jim_GetDouble(interp, argv[i], &doubleValue) != JIM_OK) + return JIM_ERR; + if (op == JIM_EXPROP_SUB) + doubleRes -= doubleValue; + else + doubleRes /= doubleValue; + } + Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); + return JIM_OK; +} + + + +static int Jim_AddCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimAddMulHelper(interp, argc, argv, JIM_EXPROP_ADD); +} + + +static int Jim_MulCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimAddMulHelper(interp, argc, argv, JIM_EXPROP_MUL); +} + + +static int Jim_SubCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimSubDivHelper(interp, argc, argv, JIM_EXPROP_SUB); +} + + +static int Jim_DivCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimSubDivHelper(interp, argc, argv, JIM_EXPROP_DIV); +} + + +static int Jim_SetCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 2 && argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "varName ?newValue?"); + return JIM_ERR; + } + if (argc == 2) { + Jim_Obj *objPtr; + + objPtr = Jim_GetVariable(interp, argv[1], JIM_ERRMSG); + if (!objPtr) + return JIM_ERR; + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + if (Jim_SetVariable(interp, argv[1], argv[2]) != JIM_OK) + return JIM_ERR; + Jim_SetResult(interp, argv[2]); + return JIM_OK; +} + +static int Jim_UnsetCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i = 1; + int complain = 1; + + while (i < argc) { + if (Jim_CompareStringImmediate(interp, argv[i], "--")) { + i++; + break; + } + if (Jim_CompareStringImmediate(interp, argv[i], "-nocomplain")) { + complain = 0; + i++; + continue; + } + break; + } + + while (i < argc) { + if (Jim_UnsetVariable(interp, argv[i], complain ? JIM_ERRMSG : JIM_NONE) != JIM_OK + && complain) { + return JIM_ERR; + } + i++; + } + + Jim_SetEmptyResult(interp); + return JIM_OK; +} + +static int JimCheckLoopRetcode(Jim_Interp *interp, int retval) +{ + if (retval == JIM_BREAK || retval == JIM_CONTINUE) { + if (--interp->break_level > 0) { + return 1; + } + } + return 0; +} + + +static int Jim_WhileCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "condition body"); + return JIM_ERR; + } + + + while (1) { + int boolean = 0, retval; + + if ((retval = Jim_GetBoolFromExpr(interp, argv[1], &boolean)) != JIM_OK) + return retval; + if (!boolean) + break; + + if ((retval = Jim_EvalObj(interp, argv[2])) != JIM_OK) { + if (JimCheckLoopRetcode(interp, retval)) { + return retval; + } + switch (retval) { + case JIM_BREAK: + goto out; + case JIM_CONTINUE: + continue; + default: + return retval; + } + } + } + out: + Jim_SetEmptyResult(interp); + return JIM_OK; +} + + +static int Jim_ForCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int retval; + int boolean = 1; + int immediate = 0; + Jim_Obj *varNamePtr = NULL; + Jim_Obj *stopVarNamePtr = NULL; + + if (argc != 5) { + Jim_WrongNumArgs(interp, 1, argv, "start test next body"); + return JIM_ERR; + } + + + if ((retval = Jim_EvalObj(interp, argv[1])) != JIM_OK) { + return retval; + } + + retval = Jim_GetBoolFromExpr(interp, argv[2], &boolean); + + +#ifdef JIM_OPTIMIZATION + if (retval == JIM_OK && boolean) { + ScriptObj *incrScript; + struct ExprTree *expr; + jim_wide stop, currentVal; + Jim_Obj *objPtr; + int cmpOffset; + + + expr = JimGetExpression(interp, argv[2]); + incrScript = JimGetScript(interp, argv[3]); + + + if (incrScript == NULL || incrScript->len != 3 || !expr || expr->len != 3) { + goto evalstart; + } + + if (incrScript->token[1].type != JIM_TT_ESC) { + goto evalstart; + } + + if (expr->expr->type == JIM_EXPROP_LT) { + cmpOffset = 0; + } + else if (expr->expr->type == JIM_EXPROP_LTE) { + cmpOffset = 1; + } + else { + goto evalstart; + } + + if (expr->expr->left->type != JIM_TT_VAR) { + goto evalstart; + } + + if (expr->expr->right->type != JIM_TT_VAR && expr->expr->right->type != JIM_TT_EXPR_INT) { + goto evalstart; + } + + + if (!Jim_CompareStringImmediate(interp, incrScript->token[1].objPtr, "incr")) { + goto evalstart; + } + + + if (!Jim_StringEqObj(incrScript->token[2].objPtr, expr->expr->left->objPtr)) { + goto evalstart; + } + + + if (expr->expr->right->type == JIM_TT_EXPR_INT) { + if (Jim_GetWideExpr(interp, expr->expr->right->objPtr, &stop) == JIM_ERR) { + goto evalstart; + } + } + else { + stopVarNamePtr = expr->expr->right->objPtr; + Jim_IncrRefCount(stopVarNamePtr); + + stop = 0; + } + + + varNamePtr = expr->expr->left->objPtr; + Jim_IncrRefCount(varNamePtr); + + objPtr = Jim_GetVariable(interp, varNamePtr, JIM_NONE); + if (objPtr == NULL || Jim_GetWide(interp, objPtr, ¤tVal) != JIM_OK) { + goto testcond; + } + + + while (retval == JIM_OK) { + + + + + if (stopVarNamePtr) { + objPtr = Jim_GetVariable(interp, stopVarNamePtr, JIM_NONE); + if (objPtr == NULL || Jim_GetWide(interp, objPtr, &stop) != JIM_OK) { + goto testcond; + } + } + + if (currentVal >= stop + cmpOffset) { + break; + } + + + retval = Jim_EvalObj(interp, argv[4]); + if (JimCheckLoopRetcode(interp, retval)) { + immediate++; + goto out; + } + if (retval == JIM_OK || retval == JIM_CONTINUE) { + retval = JIM_OK; + + objPtr = Jim_GetVariable(interp, varNamePtr, JIM_ERRMSG); + + + if (objPtr == NULL) { + retval = JIM_ERR; + goto out; + } + if (!Jim_IsShared(objPtr) && objPtr->typePtr == &intObjType) { + currentVal = ++JimWideValue(objPtr); + Jim_InvalidateStringRep(objPtr); + } + else { + if (Jim_GetWide(interp, objPtr, ¤tVal) != JIM_OK || + Jim_SetVariable(interp, varNamePtr, Jim_NewIntObj(interp, + ++currentVal)) != JIM_OK) { + goto evalnext; + } + } + } + } + goto out; + } + evalstart: +#endif + + while (boolean && (retval == JIM_OK || retval == JIM_CONTINUE)) { + + retval = Jim_EvalObj(interp, argv[4]); + if (JimCheckLoopRetcode(interp, retval)) { + immediate++; + break; + } + if (retval == JIM_OK || retval == JIM_CONTINUE) { + +JIM_IF_OPTIM(evalnext:) + retval = Jim_EvalObj(interp, argv[3]); + if (retval == JIM_OK || retval == JIM_CONTINUE) { + +JIM_IF_OPTIM(testcond:) + retval = Jim_GetBoolFromExpr(interp, argv[2], &boolean); + } + } + } +JIM_IF_OPTIM(out:) + if (stopVarNamePtr) { + Jim_DecrRefCount(interp, stopVarNamePtr); + } + if (varNamePtr) { + Jim_DecrRefCount(interp, varNamePtr); + } + + if (!immediate) { + if (retval == JIM_CONTINUE || retval == JIM_BREAK || retval == JIM_OK) { + Jim_SetEmptyResult(interp); + return JIM_OK; + } + } + + return retval; +} + + +static int Jim_LoopCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int retval; + jim_wide i; + jim_wide limit = 0; + jim_wide incr = 1; + Jim_Obj *bodyObjPtr; + + if (argc < 4 || argc > 6) { + Jim_WrongNumArgs(interp, 1, argv, "var ?first? limit ?incr? body"); + return JIM_ERR; + } + + retval = Jim_GetWideExpr(interp, argv[2], &i); + if (argc > 4 && retval == JIM_OK) { + retval = Jim_GetWideExpr(interp, argv[3], &limit); + } + if (argc > 5 && retval == JIM_OK) { + Jim_GetWideExpr(interp, argv[4], &incr); + } + if (retval != JIM_OK) { + return retval; + } + if (argc == 4) { + limit = i; + i = 0; + } + bodyObjPtr = argv[argc - 1]; + + retval = Jim_SetVariable(interp, argv[1], Jim_NewIntObj(interp, i)); + + while (((i < limit && incr > 0) || (i > limit && incr < 0)) && retval == JIM_OK) { + retval = Jim_EvalObj(interp, bodyObjPtr); + if (JimCheckLoopRetcode(interp, retval)) { + return retval; + } + if (retval == JIM_OK || retval == JIM_CONTINUE) { + Jim_Obj *objPtr = Jim_GetVariable(interp, argv[1], JIM_ERRMSG); + + retval = JIM_OK; + + + i += incr; + + if (objPtr && !Jim_IsShared(objPtr) && objPtr->typePtr == &intObjType) { + if (argv[1]->typePtr != &variableObjType) { + if (Jim_SetVariable(interp, argv[1], objPtr) != JIM_OK) { + return JIM_ERR; + } + } + JimWideValue(objPtr) = i; + Jim_InvalidateStringRep(objPtr); + + if (argv[1]->typePtr != &variableObjType) { + if (Jim_SetVariable(interp, argv[1], objPtr) != JIM_OK) { + retval = JIM_ERR; + break; + } + } + } + else { + objPtr = Jim_NewIntObj(interp, i); + retval = Jim_SetVariable(interp, argv[1], objPtr); + if (retval != JIM_OK) { + Jim_FreeNewObj(interp, objPtr); + } + } + } + } + + if (retval == JIM_OK || retval == JIM_CONTINUE || retval == JIM_BREAK) { + Jim_SetEmptyResult(interp); + return JIM_OK; + } + return retval; +} + +typedef struct { + Jim_Obj *objPtr; + int idx; +} Jim_ListIter; + +static void JimListIterInit(Jim_ListIter *iter, Jim_Obj *objPtr) +{ + iter->objPtr = objPtr; + iter->idx = 0; +} + +static Jim_Obj *JimListIterNext(Jim_Interp *interp, Jim_ListIter *iter) +{ + if (iter->idx >= Jim_ListLength(interp, iter->objPtr)) { + return NULL; + } + return iter->objPtr->internalRep.listValue.ele[iter->idx++]; +} + +static int JimListIterDone(Jim_Interp *interp, Jim_ListIter *iter) +{ + return iter->idx >= Jim_ListLength(interp, iter->objPtr); +} + + +static int JimForeachMapHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int doMap) +{ + int result = JIM_OK; + int i, numargs; + Jim_ListIter twoiters[2]; + Jim_ListIter *iters; + Jim_Obj *script; + Jim_Obj *resultObj; + + if (argc < 4 || argc % 2 != 0) { + Jim_WrongNumArgs(interp, 1, argv, "varList list ?varList list ...? script"); + return JIM_ERR; + } + script = argv[argc - 1]; + numargs = (argc - 1 - 1); + + if (numargs == 2) { + iters = twoiters; + } + else { + iters = Jim_Alloc(numargs * sizeof(*iters)); + } + for (i = 0; i < numargs; i++) { + JimListIterInit(&iters[i], argv[i + 1]); + if (i % 2 == 0 && JimListIterDone(interp, &iters[i])) { + result = JIM_ERR; + } + } + if (result != JIM_OK) { + Jim_SetResultString(interp, "foreach varlist is empty", -1); + goto empty_varlist; + } + + if (doMap) { + resultObj = Jim_NewListObj(interp, NULL, 0); + } + else { + resultObj = interp->emptyObj; + } + Jim_IncrRefCount(resultObj); + + while (1) { + + for (i = 0; i < numargs; i += 2) { + if (!JimListIterDone(interp, &iters[i + 1])) { + break; + } + } + if (i == numargs) { + + break; + } + + + for (i = 0; i < numargs; i += 2) { + Jim_Obj *varName; + + + JimListIterInit(&iters[i], argv[i + 1]); + while ((varName = JimListIterNext(interp, &iters[i])) != NULL) { + Jim_Obj *valObj = JimListIterNext(interp, &iters[i + 1]); + if (!valObj) { + + valObj = interp->emptyObj; + } + + Jim_IncrRefCount(valObj); + result = Jim_SetVariable(interp, varName, valObj); + Jim_DecrRefCount(interp, valObj); + if (result != JIM_OK) { + goto err; + } + } + } + result = Jim_EvalObj(interp, script); + if (JimCheckLoopRetcode(interp, result)) { + goto err; + } + switch (result) { + case JIM_OK: + if (doMap) { + Jim_ListAppendElement(interp, resultObj, interp->result); + } + break; + case JIM_CONTINUE: + break; + case JIM_BREAK: + goto out; + default: + goto err; + } + } + out: + result = JIM_OK; + Jim_SetResult(interp, resultObj); + err: + Jim_DecrRefCount(interp, resultObj); + empty_varlist: + if (numargs > 2) { + Jim_Free(iters); + } + return result; +} + + +static int Jim_ForeachCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimForeachMapHelper(interp, argc, argv, 0); +} + + +static int Jim_LmapCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimForeachMapHelper(interp, argc, argv, 1); +} + + +static int Jim_LassignCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int result = JIM_ERR; + int i; + Jim_ListIter iter; + Jim_Obj *resultObj; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "varList list ?varName ...?"); + return JIM_ERR; + } + + JimListIterInit(&iter, argv[1]); + + for (i = 2; i < argc; i++) { + Jim_Obj *valObj = JimListIterNext(interp, &iter); + result = Jim_SetVariable(interp, argv[i], valObj ? valObj : interp->emptyObj); + if (result != JIM_OK) { + return result; + } + } + + resultObj = Jim_NewListObj(interp, NULL, 0); + while (!JimListIterDone(interp, &iter)) { + Jim_ListAppendElement(interp, resultObj, JimListIterNext(interp, &iter)); + } + + Jim_SetResult(interp, resultObj); + + return JIM_OK; +} + + +static int Jim_IfCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int boolean, retval, current = 1, falsebody = 0; + + if (argc >= 3) { + while (1) { + + if (current >= argc) + goto err; + if ((retval = Jim_GetBoolFromExpr(interp, argv[current++], &boolean)) + != JIM_OK) + return retval; + + if (current >= argc) + goto err; + if (Jim_CompareStringImmediate(interp, argv[current], "then")) + current++; + + if (current >= argc) + goto err; + if (boolean) + return Jim_EvalObj(interp, argv[current]); + + if (++current >= argc) { + Jim_SetResult(interp, Jim_NewEmptyStringObj(interp)); + return JIM_OK; + } + falsebody = current++; + if (Jim_CompareStringImmediate(interp, argv[falsebody], "else")) { + + if (current != argc - 1) + goto err; + return Jim_EvalObj(interp, argv[current]); + } + else if (Jim_CompareStringImmediate(interp, argv[falsebody], "elseif")) + continue; + + else if (falsebody != argc - 1) + goto err; + return Jim_EvalObj(interp, argv[falsebody]); + } + return JIM_OK; + } + err: + Jim_WrongNumArgs(interp, 1, argv, "condition ?then? trueBody ?elseif ...? ?else? falseBody"); + return JIM_ERR; +} + + +int Jim_CommandMatchObj(Jim_Interp *interp, Jim_Obj *commandObj, Jim_Obj *patternObj, + Jim_Obj *stringObj, int flags) +{ + Jim_Obj *parms[5]; + int argc = 0; + long eq; + int rc; + + parms[argc++] = commandObj; + if (flags & JIM_NOCASE) { + parms[argc++] = Jim_NewStringObj(interp, "-nocase", -1); + } + if (flags & JIM_OPT_END) { + parms[argc++] = Jim_NewStringObj(interp, "--", -1); + } + parms[argc++] = patternObj; + parms[argc++] = stringObj; + + rc = Jim_EvalObjVector(interp, argc, parms); + + if (rc != JIM_OK || Jim_GetLong(interp, Jim_GetResult(interp), &eq) != JIM_OK) { + eq = -rc; + } + + return eq; +} + + +static int Jim_SwitchCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + enum { SWITCH_EXACT, SWITCH_GLOB, SWITCH_RE, SWITCH_CMD }; + int matchOpt = SWITCH_EXACT, opt = 1, patCount, i; + int match_flags = 0; + Jim_Obj *command = NULL, *scriptObj = NULL, *strObj; + Jim_Obj **caseList; + + if (argc < 3) { + wrongnumargs: + Jim_WrongNumArgs(interp, 1, argv, "?options? string " + "pattern body ... ?default body? or " "{pattern body ?pattern body ...?}"); + return JIM_ERR; + } + for (opt = 1; opt < argc; ++opt) { + const char *option = Jim_String(argv[opt]); + + if (*option != '-') + break; + else if (strncmp(option, "--", 2) == 0) { + ++opt; + break; + } + else if (strncmp(option, "-exact", 2) == 0) + matchOpt = SWITCH_EXACT; + else if (strncmp(option, "-glob", 2) == 0) + matchOpt = SWITCH_GLOB; + else if (strncmp(option, "-regexp", 2) == 0) { + matchOpt = SWITCH_RE; + match_flags |= JIM_OPT_END; + } + else if (strncmp(option, "-command", 2) == 0) { + matchOpt = SWITCH_CMD; + if ((argc - opt) < 2) + goto wrongnumargs; + command = argv[++opt]; + } + else { + Jim_SetResultFormatted(interp, + "bad option \"%#s\": must be -exact, -glob, -regexp, -command procname or --", + argv[opt]); + return JIM_ERR; + } + if ((argc - opt) < 2) + goto wrongnumargs; + } + strObj = argv[opt++]; + patCount = argc - opt; + if (patCount == 1) { + JimListGetElements(interp, argv[opt], &patCount, &caseList); + } + else + caseList = (Jim_Obj **)&argv[opt]; + if (patCount == 0 || patCount % 2 != 0) + goto wrongnumargs; + for (i = 0; scriptObj == NULL && i < patCount; i += 2) { + Jim_Obj *patObj = caseList[i]; + + if (!Jim_CompareStringImmediate(interp, patObj, "default") + || i < (patCount - 2)) { + switch (matchOpt) { + case SWITCH_EXACT: + if (Jim_StringEqObj(strObj, patObj)) + scriptObj = caseList[i + 1]; + break; + case SWITCH_GLOB: + if (Jim_StringMatchObj(interp, patObj, strObj, 0)) + scriptObj = caseList[i + 1]; + break; + case SWITCH_RE: + command = Jim_NewStringObj(interp, "regexp", -1); + + case SWITCH_CMD:{ + int rc = Jim_CommandMatchObj(interp, command, patObj, strObj, match_flags); + + if (argc - opt == 1) { + JimListGetElements(interp, argv[opt], &patCount, &caseList); + } + + if (rc < 0) { + return -rc; + } + if (rc) + scriptObj = caseList[i + 1]; + break; + } + } + } + else { + scriptObj = caseList[i + 1]; + } + } + for (; i < patCount && Jim_CompareStringImmediate(interp, scriptObj, "-"); i += 2) + scriptObj = caseList[i + 1]; + if (scriptObj && Jim_CompareStringImmediate(interp, scriptObj, "-")) { + Jim_SetResultFormatted(interp, "no body specified for pattern \"%#s\"", caseList[i - 2]); + return JIM_ERR; + } + Jim_SetEmptyResult(interp); + if (scriptObj) { + return Jim_EvalObj(interp, scriptObj); + } + return JIM_OK; +} + + +static int Jim_ListCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *listObjPtr; + + listObjPtr = Jim_NewListObj(interp, argv + 1, argc - 1); + Jim_SetResult(interp, listObjPtr); + return JIM_OK; +} + + +static int Jim_LindexCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + int ret; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "list ?index ...?"); + return JIM_ERR; + } + ret = Jim_ListIndices(interp, argv[1], argv + 2, argc - 2, &objPtr, JIM_NONE); + if (ret < 0) { + ret = JIM_OK; + Jim_SetEmptyResult(interp); + } + else if (ret == JIM_OK) { + Jim_SetResult(interp, objPtr); + } + return ret; +} + + +static int Jim_LlengthCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 2) { + Jim_WrongNumArgs(interp, 1, argv, "list"); + return JIM_ERR; + } + Jim_SetResultInt(interp, Jim_ListLength(interp, argv[1])); + return JIM_OK; +} + + +static int Jim_LsearchCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + static const char * const options[] = { + "-bool", "-not", "-nocase", "-exact", "-glob", "-regexp", "-all", "-inline", "-command", + "-stride", "-index", NULL + }; + enum + { OPT_BOOL, OPT_NOT, OPT_NOCASE, OPT_EXACT, OPT_GLOB, OPT_REGEXP, OPT_ALL, OPT_INLINE, + OPT_COMMAND, OPT_STRIDE, OPT_INDEX }; + int i; + int opt_bool = 0; + int opt_not = 0; + int opt_all = 0; + int opt_inline = 0; + int opt_match = OPT_EXACT; + int listlen; + int rc = JIM_OK; + Jim_Obj *listObjPtr = NULL; + Jim_Obj *commandObj = NULL; + Jim_Obj *indexObj = NULL; + int match_flags = 0; + long stride = 1; + + if (argc < 3) { + wrongargs: + Jim_WrongNumArgs(interp, 1, argv, + "?-exact|-glob|-regexp|-command 'command'? ?-bool|-inline? ?-not? ?-nocase? ?-all? ?-stride len? ?-index val? list value"); + return JIM_ERR; + } + + for (i = 1; i < argc - 2; i++) { + int option; + + if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + switch (option) { + case OPT_BOOL: + opt_bool = 1; + opt_inline = 0; + break; + case OPT_NOT: + opt_not = 1; + break; + case OPT_NOCASE: + match_flags |= JIM_NOCASE; + break; + case OPT_INLINE: + opt_inline = 1; + opt_bool = 0; + break; + case OPT_ALL: + opt_all = 1; + break; + case OPT_REGEXP: + opt_match = option; + match_flags |= JIM_OPT_END; + break; + case OPT_COMMAND: + if (i >= argc - 2) { + goto wrongargs; + } + commandObj = argv[++i]; + + case OPT_EXACT: + case OPT_GLOB: + opt_match = option; + break; + case OPT_INDEX: + if (i >= argc - 2) { + goto wrongargs; + } + indexObj = argv[++i]; + break; + case OPT_STRIDE: + if (i >= argc - 2) { + goto wrongargs; + } + if (Jim_GetLong(interp, argv[++i], &stride) != JIM_OK) { + return JIM_ERR; + } + if (stride < 1) { + Jim_SetResultString(interp, "stride length must be at least 1", -1); + return JIM_ERR; + } + break; + } + } + + argc -= i; + if (argc < 2) { + goto wrongargs; + } + argv += i; + + listlen = Jim_ListLength(interp, argv[0]); + if (listlen % stride) { + Jim_SetResultString(interp, "list size must be a multiple of the stride length", -1); + return JIM_ERR; + } + + if (opt_all) { + listObjPtr = Jim_NewListObj(interp, NULL, 0); + } + if (opt_match == OPT_REGEXP) { + commandObj = Jim_NewStringObj(interp, "regexp", -1); + } + if (commandObj) { + Jim_IncrRefCount(commandObj); + } + + for (i = 0; i < listlen; i += stride) { + int eq = 0; + Jim_Obj *searchListObj; + Jim_Obj *objPtr; + int offset; + + if (indexObj) { + int indexlen = Jim_ListLength(interp, indexObj); + if (stride == 1) { + searchListObj = Jim_ListGetIndex(interp, argv[0], i); + } + else { + searchListObj = Jim_NewListObj(interp, argv[0]->internalRep.listValue.ele + i, stride); + } + Jim_IncrRefCount(searchListObj); + rc = Jim_ListIndices(interp, searchListObj, indexObj->internalRep.listValue.ele, indexlen, &objPtr, JIM_ERRMSG); + if (rc != JIM_OK) { + Jim_DecrRefCount(interp, searchListObj); + rc = JIM_ERR; + goto done; + } + + offset = 0; + } + else { + + searchListObj = argv[0]; + offset = i; + objPtr = Jim_ListGetIndex(interp, searchListObj, i); + Jim_IncrRefCount(searchListObj); + } + + switch (opt_match) { + case OPT_EXACT: + eq = Jim_StringCompareObj(interp, argv[1], objPtr, match_flags) == 0; + break; + + case OPT_GLOB: + eq = Jim_StringMatchObj(interp, argv[1], objPtr, match_flags); + break; + + case OPT_REGEXP: + case OPT_COMMAND: + eq = Jim_CommandMatchObj(interp, commandObj, argv[1], objPtr, match_flags); + if (eq < 0) { + Jim_DecrRefCount(interp, searchListObj); + rc = JIM_ERR; + goto done; + } + break; + } + + + if ((!opt_bool && eq == !opt_not) || (opt_bool && (eq || opt_all))) { + Jim_Obj *resultObj; + + if (opt_bool) { + resultObj = Jim_NewIntObj(interp, eq ^ opt_not); + } + else if (!opt_inline) { + resultObj = Jim_NewIntObj(interp, i); + } + else if (stride == 1) { + resultObj = objPtr; + } + else if (opt_all) { + + ListInsertElements(listObjPtr, -1, stride, + searchListObj->internalRep.listValue.ele + offset); + + resultObj = NULL; + } + else { + resultObj = Jim_NewListObj(interp, searchListObj->internalRep.listValue.ele + offset, stride); + } + + if (opt_all) { + + if (stride == 1) { + Jim_ListAppendElement(interp, listObjPtr, resultObj); + } + } + else { + Jim_SetResult(interp, resultObj); + Jim_DecrRefCount(interp, searchListObj); + goto done; + } + } + Jim_DecrRefCount(interp, searchListObj); + } + + if (opt_all) { + Jim_SetResult(interp, listObjPtr); + listObjPtr = NULL; + } + else { + + if (opt_bool) { + Jim_SetResultBool(interp, opt_not); + } + else if (!opt_inline) { + Jim_SetResultInt(interp, -1); + } + } + + done: + if (listObjPtr) { + Jim_FreeNewObj(interp, listObjPtr); + } + if (commandObj) { + Jim_DecrRefCount(interp, commandObj); + } + return rc; +} + + +static int Jim_LappendCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *listObjPtr; + int new_obj = 0; + int i; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "varName ?value value ...?"); + return JIM_ERR; + } + listObjPtr = Jim_GetVariable(interp, argv[1], JIM_UNSHARED); + if (!listObjPtr) { + + listObjPtr = Jim_NewListObj(interp, NULL, 0); + new_obj = 1; + } + else if (Jim_IsShared(listObjPtr)) { + listObjPtr = Jim_DuplicateObj(interp, listObjPtr); + new_obj = 1; + } + for (i = 2; i < argc; i++) + Jim_ListAppendElement(interp, listObjPtr, argv[i]); + if (Jim_SetVariable(interp, argv[1], listObjPtr) != JIM_OK) { + if (new_obj) + Jim_FreeNewObj(interp, listObjPtr); + return JIM_ERR; + } + Jim_SetResult(interp, listObjPtr); + return JIM_OK; +} + + +static int Jim_LinsertCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int idx, len; + Jim_Obj *listPtr; + + if (argc < 3) { + Jim_WrongNumArgs(interp, 1, argv, "list index ?element ...?"); + return JIM_ERR; + } + listPtr = argv[1]; + if (Jim_IsShared(listPtr)) + listPtr = Jim_DuplicateObj(interp, listPtr); + if (Jim_GetIndex(interp, argv[2], &idx) != JIM_OK) + goto err; + len = Jim_ListLength(interp, listPtr); + if (idx >= len) + idx = len; + else if (idx < 0) + idx = len + idx + 1; + Jim_ListInsertElements(interp, listPtr, idx, argc - 3, &argv[3]); + Jim_SetResult(interp, listPtr); + return JIM_OK; + err: + if (listPtr != argv[1]) { + Jim_FreeNewObj(interp, listPtr); + } + return JIM_ERR; +} + + +static int Jim_LreplaceCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int first, last, len, rangeLen; + Jim_Obj *listObj; + Jim_Obj *newListObj; + + if (argc < 4) { + Jim_WrongNumArgs(interp, 1, argv, "list first last ?element ...?"); + return JIM_ERR; + } + if (Jim_GetIndex(interp, argv[2], &first) != JIM_OK || + Jim_GetIndex(interp, argv[3], &last) != JIM_OK) { + return JIM_ERR; + } + + listObj = argv[1]; + len = Jim_ListLength(interp, listObj); + + first = JimRelToAbsIndex(len, first); + last = JimRelToAbsIndex(len, last); + JimRelToAbsRange(len, &first, &last, &rangeLen); + + + if (first > len) { + first = len; + } + + + newListObj = Jim_NewListObj(interp, listObj->internalRep.listValue.ele, first); + + + ListInsertElements(newListObj, -1, argc - 4, argv + 4); + + + ListInsertElements(newListObj, -1, len - first - rangeLen, listObj->internalRep.listValue.ele + first + rangeLen); + + Jim_SetResult(interp, newListObj); + return JIM_OK; +} + + +static int Jim_LsetCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc < 3) { + Jim_WrongNumArgs(interp, 1, argv, "listVar ?index ...? value"); + return JIM_ERR; + } + else if (argc == 3) { + + if (Jim_SetVariable(interp, argv[1], argv[2]) != JIM_OK) + return JIM_ERR; + Jim_SetResult(interp, argv[2]); + return JIM_OK; + } + return Jim_ListSetIndex(interp, argv[1], argv + 2, argc - 3, argv[argc - 1]); +} + + +static int Jim_LsortCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const argv[]) +{ + static const char * const options[] = { + "-ascii", "-nocase", "-increasing", "-decreasing", "-command", "-integer", "-real", "-index", "-unique", + "-stride", "-dictionary", NULL + }; + enum { + OPT_ASCII, OPT_NOCASE, OPT_INCREASING, OPT_DECREASING, OPT_COMMAND, OPT_INTEGER, OPT_REAL, OPT_INDEX, OPT_UNIQUE, + OPT_STRIDE, OPT_DICT + }; + Jim_Obj *resObj; + int i; + int retCode; + int shared; + long stride = 1; + Jim_Obj **elements; + int listlen; + + struct lsort_info info; + + if (argc < 2) { +wrongargs: + Jim_WrongNumArgs(interp, 1, argv, "?options? list"); + return JIM_ERR; + } + + info.type = JIM_LSORT_ASCII; + info.order = 1; + info.indexc = 0; + info.unique = 0; + info.command = NULL; + info.interp = interp; + + for (i = 1; i < (argc - 1); i++) { + int option; + + if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ENUM_ABBREV | JIM_ERRMSG) + != JIM_OK) + return JIM_ERR; + switch (option) { + case OPT_ASCII: + info.type = JIM_LSORT_ASCII; + break; + case OPT_DICT: + info.type = JIM_LSORT_DICT; + break; + case OPT_NOCASE: + info.type = JIM_LSORT_NOCASE; + break; + case OPT_INTEGER: + info.type = JIM_LSORT_INTEGER; + break; + case OPT_REAL: + info.type = JIM_LSORT_REAL; + break; + case OPT_INCREASING: + info.order = 1; + break; + case OPT_DECREASING: + info.order = -1; + break; + case OPT_UNIQUE: + info.unique = 1; + break; + case OPT_COMMAND: + if (i >= (argc - 2)) { + Jim_SetResultString(interp, "\"-command\" option must be followed by comparison command", -1); + return JIM_ERR; + } + info.type = JIM_LSORT_COMMAND; + info.command = argv[i + 1]; + i++; + break; + case OPT_STRIDE: + if (i >= argc - 2) { + goto wrongargs; + } + if (Jim_GetLong(interp, argv[++i], &stride) != JIM_OK) { + return JIM_ERR; + } + if (stride < 2) { + Jim_SetResultString(interp, "stride length must be at least 2", -1); + return JIM_ERR; + } + break; + case OPT_INDEX: + if (i >= (argc - 2)) { +badindex: + Jim_SetResultString(interp, "\"-index\" option must be followed by list index", -1); + return JIM_ERR; + } + JimListGetElements(interp, argv[i + 1], &info.indexc, &info.indexv); + if (info.indexc == 0) { + goto badindex; + } + i++; + break; + } + } + resObj = argv[argc - 1]; + JimListGetElements(interp, resObj, &listlen, &elements); + if (listlen <= 1) { + + Jim_SetResult(interp, resObj); + return JIM_OK; + } + + if (stride > 1) { + Jim_Obj *tmpListObj; + int i; + + if (listlen % stride) { + Jim_SetResultString(interp, "list size must be a multiple of the stride length", -1); + return JIM_ERR; + } + + tmpListObj = Jim_NewListObj(interp, NULL, 0); + Jim_IncrRefCount(tmpListObj); + for (i = 0; i < listlen; i += stride) { + Jim_ListAppendElement(interp, tmpListObj, Jim_NewListObj(interp, elements + i, stride)); + } + retCode = ListSortElements(interp, tmpListObj, &info); + if (retCode == JIM_OK) { + resObj = Jim_NewListObj(interp, NULL, 0); + + for (i = 0; i < listlen; i += stride) { + Jim_ListAppendList(interp, resObj, Jim_ListGetIndex(interp, tmpListObj, i / stride)); + } + Jim_SetResult(interp, resObj); + } + Jim_DecrRefCount(interp, tmpListObj); + } + else { + if ((shared = Jim_IsShared(resObj))) { + resObj = Jim_DuplicateObj(interp, resObj); + } + retCode = ListSortElements(interp, resObj, &info); + if (retCode == JIM_OK) { + Jim_SetResult(interp, resObj); + } + else if (shared) { + Jim_FreeNewObj(interp, resObj); + } + } + return retCode; +} + + +static int Jim_AppendCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *stringObjPtr; + int i; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "varName ?value ...?"); + return JIM_ERR; + } + if (argc == 2) { + stringObjPtr = Jim_GetVariable(interp, argv[1], JIM_ERRMSG); + if (!stringObjPtr) + return JIM_ERR; + } + else { + int new_obj = 0; + stringObjPtr = Jim_GetVariable(interp, argv[1], JIM_UNSHARED); + if (!stringObjPtr) { + + stringObjPtr = Jim_NewEmptyStringObj(interp); + new_obj = 1; + } + else if (Jim_IsShared(stringObjPtr)) { + new_obj = 1; + stringObjPtr = Jim_DuplicateObj(interp, stringObjPtr); + } + for (i = 2; i < argc; i++) { + Jim_AppendObj(interp, stringObjPtr, argv[i]); + } + if (Jim_SetVariable(interp, argv[1], stringObjPtr) != JIM_OK) { + if (new_obj) { + Jim_FreeNewObj(interp, stringObjPtr); + } + return JIM_ERR; + } + } + Jim_SetResult(interp, stringObjPtr); + return JIM_OK; +} + + + + + +static int Jim_EvalCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int rc; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "arg ?arg ...?"); + return JIM_ERR; + } + + if (argc == 2) { + rc = Jim_EvalObj(interp, argv[1]); + } + else { + rc = Jim_EvalObj(interp, Jim_ConcatObj(interp, argc - 1, argv + 1)); + } + + return rc; +} + + +static int Jim_UplevelCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc >= 2) { + int retcode; + Jim_CallFrame *savedCallFrame, *targetCallFrame; + const char *str; + + + savedCallFrame = interp->framePtr; + + + str = Jim_String(argv[1]); + if ((str[0] >= '0' && str[0] <= '9') || str[0] == '#') { + targetCallFrame = Jim_GetCallFrameByLevel(interp, argv[1]); + argc--; + argv++; + } + else { + targetCallFrame = Jim_GetCallFrameByLevel(interp, NULL); + } + if (targetCallFrame == NULL) { + return JIM_ERR; + } + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv - 1, "?level? command ?arg ...?"); + return JIM_ERR; + } + + interp->framePtr = targetCallFrame; + if (argc == 2) { + retcode = Jim_EvalObj(interp, argv[1]); + } + else { + retcode = Jim_EvalObj(interp, Jim_ConcatObj(interp, argc - 1, argv + 1)); + } + interp->framePtr = savedCallFrame; + return retcode; + } + else { + Jim_WrongNumArgs(interp, 1, argv, "?level? command ?arg ...?"); + return JIM_ERR; + } +} + + +static int Jim_ExprCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int retcode; + + if (argc == 2) { + retcode = Jim_EvalExpression(interp, argv[1]); + } +#ifndef JIM_COMPAT + else { + Jim_WrongNumArgs(interp, 1, argv, "expression"); + retcode = JIM_ERR; + } +#else + else if (argc > 2) { + Jim_Obj *objPtr; + + objPtr = Jim_ConcatObj(interp, argc - 1, argv + 1); + Jim_IncrRefCount(objPtr); + retcode = Jim_EvalExpression(interp, objPtr); + Jim_DecrRefCount(interp, objPtr); + } + else { + Jim_WrongNumArgs(interp, 1, argv, "expression ?...?"); + return JIM_ERR; + } +#endif + return retcode; +} + +static int JimBreakContinueHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int retcode) +{ + if (argc != 1 && argc != 2) { + Jim_WrongNumArgs(interp, 1, argv, "?level?"); + return JIM_ERR; + } + if (argc == 2) { + long level; + int ret = Jim_GetLong(interp, argv[1], &level); + if (ret != JIM_OK) { + return ret; + } + interp->break_level = level; + } + return retcode; +} + + +static int Jim_BreakCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimBreakContinueHelper(interp, argc, argv, JIM_BREAK); +} + + +static int Jim_ContinueCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimBreakContinueHelper(interp, argc, argv, JIM_CONTINUE); +} + + +static int Jim_StacktraceCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *listObj; + int i; + jim_wide skip = 0; + jim_wide last = 0; + + if (argc > 1) { + if (Jim_GetWideExpr(interp, argv[1], &skip) != JIM_OK) { + return JIM_ERR; + } + } + if (argc > 2) { + if (Jim_GetWideExpr(interp, argv[2], &last) != JIM_OK) { + return JIM_ERR; + } + } + + listObj = Jim_NewListObj(interp, NULL, 0); + for (i = skip; i <= interp->procLevel; i++) { + Jim_EvalFrame *frame = JimGetEvalFrameByProcLevel(interp, -i); + if (frame->procLevel < last) { + break; + } + JimAddStackFrame(interp, frame, listObj); + } + Jim_SetResult(interp, listObj); + return JIM_OK; +} + + +static int Jim_ReturnCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i; + Jim_Obj *stackTraceObj = NULL; + Jim_Obj *errorCodeObj = NULL; + int returnCode = JIM_OK; + long level = 1; + + for (i = 1; i < argc - 1; i += 2) { + if (Jim_CompareStringImmediate(interp, argv[i], "-code")) { + if (Jim_GetReturnCode(interp, argv[i + 1], &returnCode) == JIM_ERR) { + return JIM_ERR; + } + } + else if (Jim_CompareStringImmediate(interp, argv[i], "-errorinfo")) { + stackTraceObj = argv[i + 1]; + } + else if (Jim_CompareStringImmediate(interp, argv[i], "-errorcode")) { + errorCodeObj = argv[i + 1]; + } + else if (Jim_CompareStringImmediate(interp, argv[i], "-level")) { + if (Jim_GetLong(interp, argv[i + 1], &level) != JIM_OK || level < 0) { + Jim_SetResultFormatted(interp, "bad level \"%#s\"", argv[i + 1]); + return JIM_ERR; + } + } + else { + break; + } + } + + if (i != argc - 1 && i != argc) { + Jim_WrongNumArgs(interp, 1, argv, + "?-code code? ?-errorinfo stacktrace? ?-level level? ?result?"); + } + + + if (stackTraceObj && returnCode == JIM_ERR) { + JimSetStackTrace(interp, stackTraceObj); + } + + if (errorCodeObj && returnCode == JIM_ERR) { + Jim_SetGlobalVariableStr(interp, "errorCode", errorCodeObj); + } + interp->returnCode = returnCode; + interp->returnLevel = level; + + if (i == argc - 1) { + Jim_SetResult(interp, argv[i]); + } + return level == 0 ? returnCode : JIM_RETURN; +} + + +static int Jim_TailcallCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (interp->framePtr->level == 0) { + Jim_SetResultString(interp, "tailcall can only be called from a proc or lambda", -1); + return JIM_ERR; + } + else if (argc >= 2) { + + Jim_CallFrame *cf = interp->framePtr->parent; + + Jim_Cmd *cmdPtr = Jim_GetCommand(interp, argv[1], JIM_ERRMSG); + if (cmdPtr == NULL) { + return JIM_ERR; + } + + JimPanic((cf->tailcallCmd != NULL, "Already have a tailcallCmd")); + + + JimIncrCmdRefCount(cmdPtr); + cf->tailcallCmd = cmdPtr; + + + JimPanic((cf->tailcallObj != NULL, "Already have a tailcallobj")); + + cf->tailcallObj = Jim_NewListObj(interp, argv + 1, argc - 1); + Jim_IncrRefCount(cf->tailcallObj); + + + return JIM_EVAL; + } + return JIM_OK; +} + +static int JimAliasCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *cmdList; + Jim_Obj *prefixListObj = Jim_CmdPrivData(interp); + + + cmdList = Jim_DuplicateObj(interp, prefixListObj); + Jim_ListInsertElements(interp, cmdList, Jim_ListLength(interp, cmdList), argc - 1, argv + 1); + + return JimEvalObjList(interp, cmdList); +} + +static void JimAliasCmdDelete(Jim_Interp *interp, void *privData) +{ + Jim_Obj *prefixListObj = privData; + Jim_DecrRefCount(interp, prefixListObj); +} + +static int Jim_AliasCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *prefixListObj; + + if (argc < 3) { + Jim_WrongNumArgs(interp, 1, argv, "newname command ?args ...?"); + return JIM_ERR; + } + + prefixListObj = Jim_NewListObj(interp, argv + 2, argc - 2); + Jim_IncrRefCount(prefixListObj); + Jim_SetResult(interp, argv[1]); + + return Jim_CreateCommandObj(interp, argv[1], JimAliasCmd, prefixListObj, JimAliasCmdDelete); +} + + +static int Jim_ProcCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Cmd *cmd; + + if (argc != 4 && argc != 5) { + Jim_WrongNumArgs(interp, 1, argv, "name arglist ?statics? body"); + return JIM_ERR; + } + + if (argc == 4) { + cmd = JimCreateProcedureCmd(interp, argv[2], NULL, argv[3], NULL); + } + else { + cmd = JimCreateProcedureCmd(interp, argv[2], argv[3], argv[4], NULL); + } + + if (cmd) { + + Jim_Obj *nameObjPtr = JimQualifyName(interp, argv[1]); + JimCreateCommand(interp, nameObjPtr, cmd); + + + JimUpdateProcNamespace(interp, cmd, nameObjPtr); + Jim_DecrRefCount(interp, nameObjPtr); + + + Jim_SetResult(interp, argv[1]); + return JIM_OK; + } + return JIM_ERR; +} + + +static int Jim_XtraceCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 2) { + Jim_WrongNumArgs(interp, 1, argv, "callback"); + return JIM_ERR; + } + + if (interp->traceCmdObj) { + Jim_DecrRefCount(interp, interp->traceCmdObj); + interp->traceCmdObj = NULL; + } + + if (Jim_Length(argv[1])) { + + interp->traceCmdObj = argv[1]; + Jim_IncrRefCount(interp->traceCmdObj); + } + return JIM_OK; +} + + +static int Jim_LocalCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int retcode; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "cmd ?args ...?"); + return JIM_ERR; + } + + + interp->local++; + retcode = Jim_EvalObjVector(interp, argc - 1, argv + 1); + interp->local--; + + + + if (retcode == 0) { + Jim_Obj *cmdNameObj = Jim_GetResult(interp); + + if (Jim_GetCommand(interp, cmdNameObj, JIM_ERRMSG) == NULL) { + return JIM_ERR; + } + if (interp->framePtr->localCommands == NULL) { + interp->framePtr->localCommands = Jim_Alloc(sizeof(*interp->framePtr->localCommands)); + Jim_InitStack(interp->framePtr->localCommands); + } + Jim_IncrRefCount(cmdNameObj); + Jim_StackPush(interp->framePtr->localCommands, cmdNameObj); + } + + return retcode; +} + + +static int Jim_UpcallCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "cmd ?args ...?"); + return JIM_ERR; + } + else { + int retcode; + + Jim_Cmd *cmdPtr = Jim_GetCommand(interp, argv[1], JIM_ERRMSG); + if (cmdPtr == NULL || !cmdPtr->isproc || !cmdPtr->prevCmd) { + Jim_SetResultFormatted(interp, "no previous command: \"%#s\"", argv[1]); + return JIM_ERR; + } + + cmdPtr->u.proc.upcall++; + JimIncrCmdRefCount(cmdPtr); + + + retcode = Jim_EvalObjVector(interp, argc - 1, argv + 1); + + + cmdPtr->u.proc.upcall--; + JimDecrCmdRefCount(interp, cmdPtr); + + return retcode; + } +} + + +static int Jim_ApplyCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "lambdaExpr ?arg ...?"); + return JIM_ERR; + } + else { + int ret; + Jim_Cmd *cmd; + Jim_Obj *argListObjPtr; + Jim_Obj *bodyObjPtr; + Jim_Obj *nsObj = NULL; + Jim_Obj **nargv; + + int len = Jim_ListLength(interp, argv[1]); + if (len != 2 && len != 3) { + Jim_SetResultFormatted(interp, "can't interpret \"%#s\" as a lambda expression", argv[1]); + return JIM_ERR; + } + + if (len == 3) { +#ifdef jim_ext_namespace + + nsObj = Jim_ListGetIndex(interp, argv[1], 2); +#else + Jim_SetResultString(interp, "namespaces not enabled", -1); + return JIM_ERR; +#endif + } + argListObjPtr = Jim_ListGetIndex(interp, argv[1], 0); + bodyObjPtr = Jim_ListGetIndex(interp, argv[1], 1); + + cmd = JimCreateProcedureCmd(interp, argListObjPtr, NULL, bodyObjPtr, nsObj); + + if (cmd) { + + nargv = Jim_Alloc((argc - 2 + 1) * sizeof(*nargv)); + nargv[0] = Jim_NewStringObj(interp, "apply lambdaExpr", -1); + Jim_IncrRefCount(nargv[0]); + memcpy(&nargv[1], argv + 2, (argc - 2) * sizeof(*nargv)); + ret = JimCallProcedure(interp, cmd, argc - 2 + 1, nargv); + Jim_DecrRefCount(interp, nargv[0]); + Jim_Free(nargv); + + JimDecrCmdRefCount(interp, cmd); + return ret; + } + return JIM_ERR; + } +} + + + +static int Jim_ConcatCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_SetResult(interp, Jim_ConcatObj(interp, argc - 1, argv + 1)); + return JIM_OK; +} + + +static int Jim_UpvarCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i; + Jim_CallFrame *targetCallFrame; + + + if (argc > 3 && (argc % 2 == 0)) { + targetCallFrame = Jim_GetCallFrameByLevel(interp, argv[1]); + argc--; + argv++; + } + else { + targetCallFrame = Jim_GetCallFrameByLevel(interp, NULL); + } + if (targetCallFrame == NULL) { + return JIM_ERR; + } + + + if (argc < 3) { + Jim_WrongNumArgs(interp, 1, argv, "?level? otherVar localVar ?otherVar localVar ...?"); + return JIM_ERR; + } + + + for (i = 1; i < argc; i += 2) { + if (Jim_SetVariableLink(interp, argv[i + 1], argv[i], targetCallFrame) != JIM_OK) + return JIM_ERR; + } + return JIM_OK; +} + + +static int Jim_GlobalCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int i; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "varName ?varName ...?"); + return JIM_ERR; + } + + if (interp->framePtr->level == 0) + return JIM_OK; + for (i = 1; i < argc; i++) { + + const char *name = Jim_String(argv[i]); + if (name[0] != ':' || name[1] != ':') { + if (Jim_SetVariableLink(interp, argv[i], argv[i], interp->topFramePtr) != JIM_OK) + return JIM_ERR; + } + } + return JIM_OK; +} + +static Jim_Obj *JimStringMap(Jim_Interp *interp, Jim_Obj *mapListObjPtr, + Jim_Obj *objPtr, int nocase) +{ + int numMaps; + const char *str, *noMatchStart = NULL; + int strLen, i; + Jim_Obj *resultObjPtr; + + numMaps = Jim_ListLength(interp, mapListObjPtr); + if (numMaps % 2) { + Jim_SetResultString(interp, "list must contain an even number of elements", -1); + return NULL; + } + + str = Jim_String(objPtr); + strLen = Jim_Utf8Length(interp, objPtr); + + + resultObjPtr = Jim_NewStringObj(interp, "", 0); + while (strLen) { + for (i = 0; i < numMaps; i += 2) { + Jim_Obj *eachObjPtr; + const char *k; + int kl; + + eachObjPtr = Jim_ListGetIndex(interp, mapListObjPtr, i); + k = Jim_String(eachObjPtr); + kl = Jim_Utf8Length(interp, eachObjPtr); + + if (strLen >= kl && kl) { + int rc; + rc = JimStringCompareUtf8(str, kl, k, kl, nocase); + if (rc == 0) { + if (noMatchStart) { + Jim_AppendString(interp, resultObjPtr, noMatchStart, str - noMatchStart); + noMatchStart = NULL; + } + Jim_AppendObj(interp, resultObjPtr, Jim_ListGetIndex(interp, mapListObjPtr, i + 1)); + str += utf8_index(str, kl); + strLen -= kl; + break; + } + } + } + if (i == numMaps) { + int c; + if (noMatchStart == NULL) + noMatchStart = str; + str += utf8_tounicode(str, &c); + strLen--; + } + } + if (noMatchStart) { + Jim_AppendString(interp, resultObjPtr, noMatchStart, str - noMatchStart); + } + return resultObjPtr; +} + + +static int Jim_StringCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int len; + int opt_case = 1; + int option; + static const char * const nocase_options[] = { + "-nocase", NULL + }; + static const char * const nocase_length_options[] = { + "-nocase", "-length", NULL + }; + + enum { + OPT_BYTELENGTH, + OPT_BYTERANGE, + OPT_CAT, + OPT_COMPARE, + OPT_EQUAL, + OPT_FIRST, + OPT_INDEX, + OPT_IS, + OPT_LAST, + OPT_LENGTH, + OPT_MAP, + OPT_MATCH, + OPT_RANGE, + OPT_REPEAT, + OPT_REPLACE, + OPT_REVERSE, + OPT_TOLOWER, + OPT_TOTITLE, + OPT_TOUPPER, + OPT_TRIM, + OPT_TRIMLEFT, + OPT_TRIMRIGHT, + OPT_COUNT + }; + static const jim_subcmd_type cmds[OPT_COUNT + 1] = { + JIM_DEF_SUBCMD("bytelength", "string", 1, 1), + JIM_DEF_SUBCMD("byterange", "string first last", 3, 3), + JIM_DEF_SUBCMD("cat", "?...?", 0, -1), + JIM_DEF_SUBCMD("compare", "?-nocase? ?-length int? string1 string2", 2, 5), + JIM_DEF_SUBCMD("equal", "?-nocase? ?-length int? string1 string2", 2, 5), + JIM_DEF_SUBCMD("first", "subString string ?index?", 2, 3), + JIM_DEF_SUBCMD("index", "string index", 2, 2), + JIM_DEF_SUBCMD("is", "class ?-strict? str", 2, 3), + JIM_DEF_SUBCMD("last", "subString string ?index?", 2, 3), + JIM_DEF_SUBCMD("length","string", 1, 1), + JIM_DEF_SUBCMD("map", "?-nocase? mapList string", 2, 3), + JIM_DEF_SUBCMD("match", "?-nocase? pattern string", 2, 3), + JIM_DEF_SUBCMD("range", "string first last", 3, 3), + JIM_DEF_SUBCMD("repeat", "string count", 2, 2), + JIM_DEF_SUBCMD("replace", "string first last ?string?", 3, 4), + JIM_DEF_SUBCMD("reverse", "string", 1, 1), + JIM_DEF_SUBCMD("tolower", "string", 1, 1), + JIM_DEF_SUBCMD("totitle", "string", 1, 1), + JIM_DEF_SUBCMD("toupper", "string", 1, 1), + JIM_DEF_SUBCMD("trim", "string ?trimchars?", 1, 2), + JIM_DEF_SUBCMD("trimleft", "string ?trimchars?", 1, 2), + JIM_DEF_SUBCMD("trimright", "string ?trimchars?", 1, 2), + { NULL } + }; + const jim_subcmd_type *ct = Jim_ParseSubCmd(interp, cmds, argc, argv); + if (!ct) { + return JIM_ERR; + } + if (ct->function) { + + return ct->function(interp, argc, argv); + } + + option = ct - cmds; + + switch (option) { + case OPT_LENGTH: + Jim_SetResultInt(interp, Jim_Utf8Length(interp, argv[2])); + return JIM_OK; + + case OPT_BYTELENGTH: + Jim_SetResultInt(interp, Jim_Length(argv[2])); + return JIM_OK; + + case OPT_CAT:{ + Jim_Obj *objPtr; + if (argc == 3) { + + objPtr = argv[2]; + } + else { + int i; + + objPtr = Jim_NewStringObj(interp, "", 0); + + for (i = 2; i < argc; i++) { + Jim_AppendObj(interp, objPtr, argv[i]); + } + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + case OPT_COMPARE: + case OPT_EQUAL: + { + + long opt_length = -1; + int n = argc - 4; + int i = 2; + while (n > 0) { + int subopt; + if (Jim_GetEnum(interp, argv[i++], nocase_length_options, &subopt, NULL, + JIM_ENUM_ABBREV) != JIM_OK) { +badcompareargs: + Jim_SubCmdArgError(interp, ct, argv[0]); + return JIM_ERR; + } + if (subopt == 0) { + + opt_case = 0; + n--; + } + else { + + if (n < 2) { + goto badcompareargs; + } + if (Jim_GetLong(interp, argv[i++], &opt_length) != JIM_OK) { + return JIM_ERR; + } + n -= 2; + } + } + if (n) { + goto badcompareargs; + } + argv += argc - 2; + if (opt_length < 0 && option != OPT_COMPARE && opt_case) { + + Jim_SetResultBool(interp, Jim_StringEqObj(argv[0], argv[1])); + } + else { + const char *s1 = Jim_String(argv[0]); + int l1 = Jim_Utf8Length(interp, argv[0]); + const char *s2 = Jim_String(argv[1]); + int l2 = Jim_Utf8Length(interp, argv[1]); + if (opt_length >= 0) { + if (l1 > opt_length) { + l1 = opt_length; + } + if (l2 > opt_length) { + l2 = opt_length; + } + } + n = JimStringCompareUtf8(s1, l1, s2, l2, !opt_case); + Jim_SetResultInt(interp, option == OPT_COMPARE ? n : n == 0); + } + return JIM_OK; + } + + case OPT_MATCH: + if (argc != 4 && + (argc != 5 || + Jim_GetEnum(interp, argv[2], nocase_options, &opt_case, NULL, + JIM_ENUM_ABBREV) != JIM_OK)) { + Jim_WrongNumArgs(interp, 2, argv, "?-nocase? pattern string"); + return JIM_ERR; + } + if (opt_case == 0) { + argv++; + } + Jim_SetResultBool(interp, Jim_StringMatchObj(interp, argv[2], argv[3], !opt_case)); + return JIM_OK; + + case OPT_MAP:{ + Jim_Obj *objPtr; + + if (argc != 4 && + (argc != 5 || + Jim_GetEnum(interp, argv[2], nocase_options, &opt_case, NULL, + JIM_ENUM_ABBREV) != JIM_OK)) { + Jim_WrongNumArgs(interp, 2, argv, "?-nocase? mapList string"); + return JIM_ERR; + } + + if (opt_case == 0) { + argv++; + } + objPtr = JimStringMap(interp, argv[2], argv[3], !opt_case); + if (objPtr == NULL) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + case OPT_RANGE:{ + Jim_Obj *objPtr = Jim_StringRangeObj(interp, argv[2], argv[3], argv[4]); + if (objPtr == NULL) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + case OPT_BYTERANGE:{ + Jim_Obj *objPtr = Jim_StringByteRangeObj(interp, argv[2], argv[3], argv[4]); + if (objPtr == NULL) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + case OPT_REPLACE:{ + Jim_Obj *objPtr = JimStringReplaceObj(interp, argv[2], argv[3], argv[4], argc == 6 ? argv[5] : NULL); + if (objPtr == NULL) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + + case OPT_REPEAT:{ + Jim_Obj *objPtr; + jim_wide count; + + if (Jim_GetWideExpr(interp, argv[3], &count) != JIM_OK) { + return JIM_ERR; + } + objPtr = Jim_NewStringObj(interp, "", 0); + if (count > 0) { + while (count--) { + Jim_AppendObj(interp, objPtr, argv[2]); + } + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + } + + case OPT_REVERSE:{ + char *buf, *p; + const char *str; + int i; + + str = Jim_GetString(argv[2], &len); + buf = Jim_Alloc(len + 1); + assert(buf); + p = buf + len; + *p = 0; + for (i = 0; i < len; ) { + int c; + int l = utf8_tounicode(str, &c); + memcpy(p - l, str, l); + p -= l; + i += l; + str += l; + } + Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, buf, len)); + return JIM_OK; + } + + case OPT_INDEX:{ + int idx; + const char *str; + + if (Jim_GetIndex(interp, argv[3], &idx) != JIM_OK) { + return JIM_ERR; + } + str = Jim_String(argv[2]); + len = Jim_Utf8Length(interp, argv[2]); + idx = JimRelToAbsIndex(len, idx); + if (idx < 0 || idx >= len || str == NULL) { + Jim_SetResultString(interp, "", 0); + } + else if (len == Jim_Length(argv[2])) { + + Jim_SetResultString(interp, str + idx, 1); + } + else { + int c; + int i = utf8_index(str, idx); + Jim_SetResultString(interp, str + i, utf8_tounicode(str + i, &c)); + } + return JIM_OK; + } + + case OPT_FIRST: + case OPT_LAST:{ + int idx = 0, l1, l2; + const char *s1, *s2; + + s1 = Jim_String(argv[2]); + s2 = Jim_String(argv[3]); + l1 = Jim_Utf8Length(interp, argv[2]); + l2 = Jim_Utf8Length(interp, argv[3]); + if (argc == 5) { + if (Jim_GetIndex(interp, argv[4], &idx) != JIM_OK) { + return JIM_ERR; + } + idx = JimRelToAbsIndex(l2, idx); + if (idx < 0) { + idx = 0; + } + } + else if (option == OPT_LAST) { + idx = l2; + } + if (option == OPT_FIRST) { + Jim_SetResultInt(interp, JimStringFirst(s1, l1, s2, l2, idx)); + } + else { +#ifdef JIM_UTF8 + Jim_SetResultInt(interp, JimStringLastUtf8(s1, l1, s2, idx)); +#else + Jim_SetResultInt(interp, JimStringLast(s1, l1, s2, idx)); +#endif + } + return JIM_OK; + } + + case OPT_TRIM: + Jim_SetResult(interp, JimStringTrim(interp, argv[2], argc == 4 ? argv[3] : NULL)); + return JIM_OK; + case OPT_TRIMLEFT: + Jim_SetResult(interp, JimStringTrimLeft(interp, argv[2], argc == 4 ? argv[3] : NULL)); + return JIM_OK; + case OPT_TRIMRIGHT:{ + Jim_SetResult(interp, JimStringTrimRight(interp, argv[2], argc == 4 ? argv[3] : NULL)); + return JIM_OK; + } + + case OPT_TOLOWER: + Jim_SetResult(interp, JimStringToLower(interp, argv[2])); + return JIM_OK; + case OPT_TOUPPER: + Jim_SetResult(interp, JimStringToUpper(interp, argv[2])); + return JIM_OK; + case OPT_TOTITLE: + Jim_SetResult(interp, JimStringToTitle(interp, argv[2])); + return JIM_OK; + + case OPT_IS: + if (argc == 5 && !Jim_CompareStringImmediate(interp, argv[3], "-strict")) { + Jim_SubCmdArgError(interp, ct, argv[0]); + return JIM_ERR; + } + return JimStringIs(interp, argv[argc - 1], argv[2], argc == 5); + } + return JIM_OK; +} + + +static int Jim_TimeCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + long i, count = 1; + jim_wide start, elapsed; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "script ?count?"); + return JIM_ERR; + } + if (argc == 3) { + if (Jim_GetLong(interp, argv[2], &count) != JIM_OK) + return JIM_ERR; + } + if (count < 0) + return JIM_OK; + i = count; + start = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW); + while (i-- > 0) { + int retval; + + retval = Jim_EvalObj(interp, argv[1]); + if (retval != JIM_OK) { + return retval; + } + } + elapsed = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW) - start; + if (elapsed < count * 10) { + Jim_SetResult(interp, Jim_NewDoubleObj(interp, elapsed * 1.0 / count)); + } + else { + Jim_SetResultInt(interp, count == 0 ? 0 : elapsed / count); + } + Jim_AppendString(interp, Jim_GetResult(interp)," microseconds per iteration", -1); + return JIM_OK; +} + + +static int Jim_TimeRateCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + long us = 0; + jim_wide start, delta, overhead; + Jim_Obj *objPtr; + double us_per_iter; + int count; + int n; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "script ?milliseconds?"); + return JIM_ERR; + } + if (argc == 3) { + if (Jim_GetLong(interp, argv[2], &us) != JIM_OK) + return JIM_ERR; + us *= 1000; + } + if (us < 1) { + + us = 1000 * 1000; + } + + + start = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW); + count = 0; + do { + int retval = Jim_EvalObj(interp, argv[1]); + delta = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW) - start; + if (retval != JIM_OK) { + return retval; + } + count++; + } while (delta < us); + + + start = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW); + n = 0; + do { + int retval = Jim_EvalObj(interp, interp->nullScriptObj); + overhead = Jim_GetTimeUsec(CLOCK_MONOTONIC_RAW) - start; + if (retval != JIM_OK) { + return retval; + } + n++; + } while (n < count); + + delta -= overhead; + + us_per_iter = (double)delta / count; + objPtr = Jim_NewListObj(interp, NULL, 0); + + Jim_ListAppendElement(interp, objPtr, Jim_NewStringObj(interp, "us_per_iter", -1)); + Jim_ListAppendElement(interp, objPtr, Jim_NewDoubleObj(interp, us_per_iter)); + Jim_ListAppendElement(interp, objPtr, Jim_NewStringObj(interp, "iters_per_sec", -1)); + Jim_ListAppendElement(interp, objPtr, Jim_NewDoubleObj(interp, 1e6 / us_per_iter)); + Jim_ListAppendElement(interp, objPtr, Jim_NewStringObj(interp, "count", -1)); + Jim_ListAppendElement(interp, objPtr, Jim_NewIntObj(interp, count)); + Jim_ListAppendElement(interp, objPtr, Jim_NewStringObj(interp, "elapsed_us", -1)); + Jim_ListAppendElement(interp, objPtr, Jim_NewIntObj(interp, delta)); + Jim_SetResult(interp, objPtr); + return JIM_OK; +} + + +static int Jim_ExitCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + long exitCode = 0; + + if (argc > 2) { + Jim_WrongNumArgs(interp, 1, argv, "?exitCode?"); + return JIM_ERR; + } + if (argc == 2) { + if (Jim_GetLong(interp, argv[1], &exitCode) != JIM_OK) + return JIM_ERR; + Jim_SetResult(interp, argv[1]); + } + interp->exitCode = exitCode; + return JIM_EXIT; +} + +static int JimMatchReturnCodes(Jim_Interp *interp, Jim_Obj *retcodeListObj, int rc) +{ + int len = Jim_ListLength(interp, retcodeListObj); + int i; + for (i = 0; i < len; i++) { + int returncode; + if (Jim_GetReturnCode(interp, Jim_ListGetIndex(interp, retcodeListObj, i), &returncode) != JIM_OK) { + return JIM_ERR; + } + if (rc == returncode) { + return JIM_OK; + } + } + return -1; +} + + +static int JimCatchTryHelper(Jim_Interp *interp, int istry, int argc, Jim_Obj *const *argv) +{ + static const char * const wrongargs_catchtry[2] = { + "?-?no?code ... --? script ?resultVarName? ?optionVarName?", + "?-?no?code ... --? script ?on|trap codes vars script? ... ?finally script?" + }; + int exitCode = 0; + int i; + int sig = 0; + int ok; + Jim_Obj *finallyScriptObj = NULL; + Jim_Obj *msgVarObj = NULL; + Jim_Obj *optsVarObj = NULL; + Jim_Obj *handlerScriptObj = NULL; + Jim_Obj *errorCodeObj; + int idx; + + + jim_wide ignore_mask = (1 << JIM_EXIT) | (1 << JIM_EVAL) | (1 << JIM_SIGNAL); + static const int max_ignore_code = sizeof(ignore_mask) * 8; + + JimPanic((istry != 0 && istry != 1, "wrong args to JimCatchTryHelper")); + + Jim_SetGlobalVariableStr(interp, "errorCode", Jim_NewStringObj(interp, "NONE", -1)); + + for (i = 1; i < argc - 1; i++) { + const char *arg = Jim_String(argv[i]); + jim_wide option; + int ignore; + + + if (strcmp(arg, "--") == 0) { + i++; + break; + } + if (*arg != '-') { + break; + } + + if (strncmp(arg, "-no", 3) == 0) { + arg += 3; + ignore = 1; + } + else { + arg++; + ignore = 0; + } + + if (Jim_StringToWide(arg, &option, 10) != JIM_OK) { + option = -1; + } + if (option < 0) { + option = Jim_FindByName(arg, jimReturnCodes, jimReturnCodesSize); + } + if (option < 0) { + goto wrongargs; + } + + if (ignore) { + ignore_mask |= ((jim_wide)1 << option); + } + else { + ignore_mask &= (~((jim_wide)1 << option)); + } + } + + idx = i; + + if (argc - idx < 1) { +wrongargs: + Jim_WrongNumArgs(interp, 1, argv, wrongargs_catchtry[istry]); + return JIM_ERR; + } + + if ((ignore_mask & (1 << JIM_SIGNAL)) == 0) { + sig++; + } + + interp->signal_level += sig; + if (Jim_CheckSignal(interp)) { + + exitCode = JIM_SIGNAL; + } + else { + exitCode = Jim_EvalObj(interp, argv[idx]); + + interp->errorFlag = 0; + } + interp->signal_level -= sig; + + errorCodeObj = Jim_GetGlobalVariableStr(interp, "errorCode", JIM_NONE); + + idx++; + if (istry) { + while (idx < argc) { + int option; + int ret; + static const char * const try_options[] = { "on", "trap", "finally", NULL }; + enum { TRY_ON, TRY_TRAP, TRY_FINALLY, }; + + if (Jim_GetEnum(interp, argv[idx], try_options, &option, "handler", JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + switch (option) { + case TRY_ON: + case TRY_TRAP: + if (idx + 4 > argc) { + goto wrongargs; + } + if (option == TRY_ON) { + ret = JimMatchReturnCodes(interp, argv[idx + 1], exitCode); + if (ret > JIM_OK) { + goto wrongargs; + } + } + else if (errorCodeObj) { + int len = Jim_ListLength(interp, argv[idx + 1]); + int i; + + ret = JIM_OK; + + for (i = 0; i < len; i++) { + Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); + Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); + if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { + ret = -1; + break; + } + } + } + else { + + ret = -1; + } + + if (ret == JIM_OK && handlerScriptObj == NULL) { + msgVarObj = Jim_ListGetIndex(interp, argv[idx + 2], 0); + optsVarObj = Jim_ListGetIndex(interp, argv[idx + 2], 1); + handlerScriptObj = argv[idx + 3]; + } + idx += 4; + break; + case TRY_FINALLY: + if (idx + 2 != argc) { + goto wrongargs; + } + finallyScriptObj = argv[idx + 1]; + idx += 2; + break; + } + } + } + else { + if (argc - idx >= 1) { + msgVarObj = argv[idx]; + idx++; + if (argc - idx >= 1) { + optsVarObj = argv[idx]; + idx++; + } + } + } + + + if (exitCode >= 0 && exitCode < max_ignore_code && (((unsigned jim_wide)1 << exitCode) & ignore_mask)) { + + if (finallyScriptObj) { + Jim_EvalObj(interp, finallyScriptObj); + } + return exitCode; + } + + if (sig && exitCode == JIM_SIGNAL) { + + if (interp->signal_set_result) { + interp->signal_set_result(interp, interp->sigmask); + } + else if (!istry) { + Jim_SetResultInt(interp, interp->sigmask); + } + interp->sigmask = 0; + } + + ok = 1; + if (msgVarObj && Jim_Length(msgVarObj)) { + if (Jim_SetVariable(interp, msgVarObj, Jim_GetResult(interp)) != JIM_OK) { + ok = 0; + } + } + if (ok && optsVarObj && Jim_Length(optsVarObj)) { + Jim_Obj *optListObj = Jim_NewListObj(interp, NULL, 0); + + Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-code", -1)); + Jim_ListAppendElement(interp, optListObj, + Jim_NewIntObj(interp, exitCode == JIM_RETURN ? interp->returnCode : exitCode)); + Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-level", -1)); + Jim_ListAppendElement(interp, optListObj, Jim_NewIntObj(interp, interp->returnLevel)); + if (exitCode == JIM_ERR) { + Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-errorinfo", + -1)); + Jim_ListAppendElement(interp, optListObj, interp->stackTrace); + + if (errorCodeObj) { + Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-errorcode", -1)); + Jim_ListAppendElement(interp, optListObj, errorCodeObj); + } + } + if (Jim_SetVariable(interp, optsVarObj, optListObj) != JIM_OK) { + ok = 0; + } + } + if (ok && handlerScriptObj) { + + exitCode = Jim_EvalObj(interp, handlerScriptObj); + } + + if (finallyScriptObj) { + + Jim_Obj *prevResultObj = Jim_GetResult(interp); + Jim_IncrRefCount(prevResultObj); + int ret = Jim_EvalObj(interp, finallyScriptObj); + if (ret == JIM_OK) { + Jim_SetResult(interp, prevResultObj); + } + else { + exitCode = ret; + } + Jim_DecrRefCount(interp, prevResultObj); + } + if (!istry) { + Jim_SetResultInt(interp, exitCode); + exitCode = JIM_OK; + } + return exitCode; +} + + +static int Jim_CatchCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimCatchTryHelper(interp, 0, argc, argv); +} + + +static int Jim_TryCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + return JimCatchTryHelper(interp, 1, argc, argv); +} + + + +static int Jim_RenameCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "oldName newName"); + return JIM_ERR; + } + + return Jim_RenameCommand(interp, argv[1], argv[2]); +} + +#define JIM_DICTMATCH_KEYS 0x0001 +#define JIM_DICTMATCH_VALUES 0x002 + +int Jim_DictMatchTypes(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *patternObj, int match_type, int return_types) +{ + Jim_Obj *listObjPtr; + Jim_Dict *dict; + int i; + + if (SetDictFromAny(interp, objPtr) != JIM_OK) { + return JIM_ERR; + } + dict = objPtr->internalRep.dictValue; + + listObjPtr = Jim_NewListObj(interp, NULL, 0); + + for (i = 0; i < dict->len; i += 2 ) { + Jim_Obj *keyObj = dict->table[i]; + Jim_Obj *valObj = dict->table[i + 1]; + if (patternObj) { + Jim_Obj *matchObj = (match_type == JIM_DICTMATCH_KEYS) ? keyObj : valObj; + if (!Jim_StringMatchObj(interp, patternObj, matchObj, 0)) { + + continue; + } + } + if (return_types & JIM_DICTMATCH_KEYS) { + Jim_ListAppendElement(interp, listObjPtr, keyObj); + } + if (return_types & JIM_DICTMATCH_VALUES) { + Jim_ListAppendElement(interp, listObjPtr, valObj); + } + } + + Jim_SetResult(interp, listObjPtr); + return JIM_OK; +} + +int Jim_DictSize(Jim_Interp *interp, Jim_Obj *objPtr) +{ + if (SetDictFromAny(interp, objPtr) != JIM_OK) { + return -1; + } + return objPtr->internalRep.dictValue->len / 2; +} + +Jim_Obj *Jim_DictMerge(Jim_Interp *interp, int objc, Jim_Obj *const *objv) +{ + Jim_Obj *objPtr = Jim_NewDictObj(interp, NULL, 0); + int i; + + JimPanic((objc == 0, "Jim_DictMerge called with objc=0")); + + + + for (i = 0; i < objc; i++) { + Jim_Obj **table; + int tablelen; + int j; + + table = Jim_DictPairs(interp, objv[i], &tablelen); + if (tablelen && !table) { + Jim_FreeNewObj(interp, objPtr); + return NULL; + } + for (j = 0; j < tablelen; j += 2) { + DictAddElement(interp, objPtr, table[j], table[j + 1]); + } + } + return objPtr; +} + +int Jim_DictInfo(Jim_Interp *interp, Jim_Obj *objPtr) +{ + char buffer[100]; + Jim_Obj *output; + Jim_Dict *dict; + + if (SetDictFromAny(interp, objPtr) != JIM_OK) { + return JIM_ERR; + } + + dict = objPtr->internalRep.dictValue; + + + snprintf(buffer, sizeof(buffer), "%d entries in table, %d buckets", dict->len, dict->size); + output = Jim_NewStringObj(interp, buffer, -1); + Jim_SetResult(interp, output); + return JIM_OK; +} + +static int Jim_EvalEnsemble(Jim_Interp *interp, const char *basecmd, const char *subcmd, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *prefixObj = Jim_NewStringObj(interp, basecmd, -1); + + Jim_AppendString(interp, prefixObj, " ", 1); + Jim_AppendString(interp, prefixObj, subcmd, -1); + + return Jim_EvalObjPrefix(interp, prefixObj, argc, argv); +} + +static int JimDictWith(Jim_Interp *interp, Jim_Obj *dictVarName, Jim_Obj *const *keyv, int keyc, Jim_Obj *scriptObj) +{ + int i; + Jim_Obj *objPtr; + Jim_Obj *dictObj; + Jim_Obj **dictValues; + int len; + int ret = JIM_OK; + + + dictObj = Jim_GetVariable(interp, dictVarName, JIM_ERRMSG); + if (dictObj == NULL || Jim_DictKeysVector(interp, dictObj, keyv, keyc, &objPtr, JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + + dictValues = Jim_DictPairs(interp, objPtr, &len); + if (len && dictValues == NULL) { + return JIM_ERR; + } + for (i = 0; i < len; i += 2) { + if (Jim_SetVariable(interp, dictValues[i], dictValues[i + 1]) == JIM_ERR) { + return JIM_ERR; + } + } + + + if (Jim_Length(scriptObj)) { + ret = Jim_EvalObj(interp, scriptObj); + + + if (ret == JIM_OK && Jim_GetVariable(interp, dictVarName, 0) != NULL) { + + Jim_Obj **newkeyv = Jim_Alloc(sizeof(*newkeyv) * (keyc + 1)); + for (i = 0; i < keyc; i++) { + newkeyv[i] = keyv[i]; + } + + for (i = 0; i < len; i += 2) { + + if (Jim_StringCompareObj(interp, dictVarName, dictValues[i], 0) != 0) { + + objPtr = Jim_GetVariable(interp, dictValues[i], 0); + newkeyv[keyc] = dictValues[i]; + Jim_SetDictKeysVector(interp, dictVarName, newkeyv, keyc + 1, objPtr, JIM_NORESULT); + } + } + Jim_Free(newkeyv); + } + } + + return ret; +} + + +static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + int types = JIM_DICTMATCH_KEYS; + + enum { + OPT_CREATE, + OPT_GET, + OPT_GETDEF, + OPT_GETWITHDEFAULT, + OPT_SET, + OPT_UNSET, + OPT_EXISTS, + OPT_KEYS, + OPT_SIZE, + OPT_INFO, + OPT_MERGE, + OPT_WITH, + OPT_APPEND, + OPT_LAPPEND, + OPT_INCR, + OPT_REMOVE, + OPT_VALUES, + OPT_FOR, + OPT_REPLACE, + OPT_UPDATE, + OPT_COUNT + }; + static const jim_subcmd_type cmds[OPT_COUNT + 1] = { + JIM_DEF_SUBCMD("create", "?key value ...?", 0, -2), + JIM_DEF_SUBCMD("get", "dictionary ?key ...?", 1, -1), + JIM_DEF_SUBCMD_HIDDEN("getdef", "dictionary ?key ...? key default", 3, -1), + JIM_DEF_SUBCMD("getwithdefault", "dictionary ?key ...? key default", 3, -1), + JIM_DEF_SUBCMD("set", "varName key ?key ...? value", 3, -1), + JIM_DEF_SUBCMD("unset", "varName key ?key ...?", 2, -1), + JIM_DEF_SUBCMD("exists", "dictionary key ?key ...?", 2, -1), + JIM_DEF_SUBCMD("keys", "dictionary ?pattern?", 1, 2), + JIM_DEF_SUBCMD("size", "dictionary", 1, 1), + JIM_DEF_SUBCMD("info", "dictionary", 1, 1), + JIM_DEF_SUBCMD("merge", "?...?", 0, -1), + JIM_DEF_SUBCMD("with", "dictVar ?key ...? script", 2, -1), + JIM_DEF_SUBCMD("append", "varName key ?value ...?", 2, -1), + JIM_DEF_SUBCMD("lappend", "varName key ?value ...?", 2, -1), + JIM_DEF_SUBCMD("incr", "varName key ?increment?", 2, 3), + JIM_DEF_SUBCMD("remove", "dictionary ?key ...?", 1, -1), + JIM_DEF_SUBCMD("values", "dictionary ?pattern?", 1, 2), + JIM_DEF_SUBCMD("for", "vars dictionary script", 3, 3), + JIM_DEF_SUBCMD("replace", "dictionary ?key value ...?", 1, -1), + JIM_DEF_SUBCMD("update", "varName ?arg ...? script", 2, -1), + { NULL } + }; + const jim_subcmd_type *ct = Jim_ParseSubCmd(interp, cmds, argc, argv); + if (!ct) { + return JIM_ERR; + } + if (ct->function) { + + return ct->function(interp, argc, argv); + } + + + switch (ct - cmds) { + case OPT_GET: + if (Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, + JIM_ERRMSG) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + + case OPT_GETDEF: + case OPT_GETWITHDEFAULT:{ + int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 4, &objPtr, JIM_ERRMSG); + if (rc == -1) { + + return JIM_ERR; + } + if (rc == JIM_ERR) { + Jim_SetResult(interp, argv[argc - 1]); + } + else { + Jim_SetResult(interp, objPtr); + } + return JIM_OK; + } + + case OPT_SET: + return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG); + + case OPT_EXISTS:{ + int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, JIM_NONE); + if (rc < 0) { + return JIM_ERR; + } + Jim_SetResultBool(interp, rc == JIM_OK); + return JIM_OK; + } + + case OPT_UNSET: + if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_NONE) != JIM_OK) { + return JIM_ERR; + } + return JIM_OK; + + case OPT_VALUES: + types = JIM_DICTMATCH_VALUES; + + case OPT_KEYS: + return Jim_DictMatchTypes(interp, argv[2], argc == 4 ? argv[3] : NULL, types, types); + + case OPT_SIZE: + if (Jim_DictSize(interp, argv[2]) < 0) { + return JIM_ERR; + } + Jim_SetResultInt(interp, Jim_DictSize(interp, argv[2])); + return JIM_OK; + + case OPT_MERGE: + if (argc == 2) { + return JIM_OK; + } + objPtr = Jim_DictMerge(interp, argc - 2, argv + 2); + if (objPtr == NULL) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; + + case OPT_CREATE: + objPtr = Jim_NewDictObj(interp, argv + 2, argc - 2); + Jim_SetResult(interp, objPtr); + return JIM_OK; + + case OPT_INFO: + return Jim_DictInfo(interp, argv[2]); + + case OPT_WITH: + return JimDictWith(interp, argv[2], argv + 3, argc - 4, argv[argc - 1]); + + case OPT_UPDATE: + if (argc < 6 || argc % 2) { + + argc = 2; + } + + default: + return Jim_EvalEnsemble(interp, "dict", Jim_String(argv[1]), argc - 2, argv + 2); + } +} + + +static int Jim_SubstCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + static const char * const options[] = { + "-nobackslashes", "-nocommands", "-novariables", NULL + }; + enum + { OPT_NOBACKSLASHES, OPT_NOCOMMANDS, OPT_NOVARIABLES }; + int i; + int flags = JIM_SUBST_FLAG; + Jim_Obj *objPtr; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "?options? string"); + return JIM_ERR; + } + for (i = 1; i < (argc - 1); i++) { + int option; + + if (Jim_GetEnum(interp, argv[i], options, &option, NULL, + JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { + return JIM_ERR; + } + switch (option) { + case OPT_NOBACKSLASHES: + flags |= JIM_SUBST_NOESC; + break; + case OPT_NOCOMMANDS: + flags |= JIM_SUBST_NOCMD; + break; + case OPT_NOVARIABLES: + flags |= JIM_SUBST_NOVAR; + break; + } + } + if (Jim_SubstObj(interp, argv[argc - 1], &objPtr, flags) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + return JIM_OK; +} + +#ifdef jim_ext_namespace +static int JimIsGlobalNamespace(Jim_Obj *objPtr) +{ + int len; + const char *str = Jim_GetString(objPtr, &len); + return len >= 2 && str[0] == ':' && str[1] == ':'; +} +#endif + + +static int Jim_InfoCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + int mode = 0; + + + enum { + INFO_ALIAS, + INFO_ARGS, + INFO_BODY, + INFO_CHANNELS, + INFO_COMMANDS, + INFO_COMPLETE, + INFO_EXISTS, + INFO_FRAME, + INFO_GLOBALS, + INFO_HOSTNAME, + INFO_LEVEL, + INFO_LOCALS, + INFO_NAMEOFEXECUTABLE, + INFO_PATCHLEVEL, + INFO_PROCS, + INFO_REFERENCES, + INFO_RETURNCODES, + INFO_SCRIPT, + INFO_SOURCE, + INFO_STACKTRACE, + INFO_STATICS, + INFO_VARS, + INFO_VERSION, + INFO_COUNT + }; + static const jim_subcmd_type cmds[INFO_COUNT + 1] = { + JIM_DEF_SUBCMD("alias", "command", 1, 1), + JIM_DEF_SUBCMD("args", "procname", 1, 1), + JIM_DEF_SUBCMD("body", "procname", 1, 1), + JIM_DEF_SUBCMD("channels", "?pattern?", 0, 1), + JIM_DEF_SUBCMD("commands", "?pattern?", 0, 1), + JIM_DEF_SUBCMD("complete", "script ?missing?", 1, 2), + JIM_DEF_SUBCMD("exists", "varName", 1, 1), + JIM_DEF_SUBCMD("frame", "?levelNum?", 0, 1), + JIM_DEF_SUBCMD("globals", "?pattern?", 0, 1), + JIM_DEF_SUBCMD("hostname", NULL, 0, 0), + JIM_DEF_SUBCMD("level", "?levelNum?", 0, 1), + JIM_DEF_SUBCMD("locals", "?pattern?", 0, 1), + JIM_DEF_SUBCMD("nameofexecutable", NULL, 0, 0), + JIM_DEF_SUBCMD("patchlevel", NULL, 0, 0), + JIM_DEF_SUBCMD("procs", "?pattern?", 0, 1), + JIM_DEF_SUBCMD("references", NULL, 0, 0), + JIM_DEF_SUBCMD("returncodes", "?code?", 0, 1), + JIM_DEF_SUBCMD("script", "?filename?", 0, 1), + JIM_DEF_SUBCMD("source", "source ?filename line?", 1, 3), + JIM_DEF_SUBCMD("stacktrace", NULL, 0, 0), + JIM_DEF_SUBCMD("statics", "procname", 1, 1), + JIM_DEF_SUBCMD("vars", "?pattern?", 0, 1), + JIM_DEF_SUBCMD("version", NULL, 0, 0), + { NULL } + }; + const jim_subcmd_type *ct; +#ifdef jim_ext_namespace + int nons = 0; + + if (argc > 2 && Jim_CompareStringImmediate(interp, argv[1], "-nons")) { + + argc--; + argv++; + nons = 1; + } +#endif + ct = Jim_ParseSubCmd(interp, cmds, argc, argv); + if (!ct) { + return JIM_ERR; + } + if (ct->function) { + + return ct->function(interp, argc, argv); + } + + int option = ct - cmds; + + switch (option) { + case INFO_EXISTS: + Jim_SetResultBool(interp, Jim_GetVariable(interp, argv[2], 0) != NULL); + return JIM_OK; + + case INFO_ALIAS:{ + Jim_Cmd *cmdPtr; + + if ((cmdPtr = Jim_GetCommand(interp, argv[2], JIM_ERRMSG)) == NULL) { + return JIM_ERR; + } + if (cmdPtr->isproc || cmdPtr->u.native.cmdProc != JimAliasCmd) { + Jim_SetResultFormatted(interp, "command \"%#s\" is not an alias", argv[2]); + return JIM_ERR; + } + Jim_SetResult(interp, (Jim_Obj *)cmdPtr->u.native.privData); + return JIM_OK; + } + + case INFO_CHANNELS: + mode++; +#ifndef jim_ext_aio + Jim_SetResultString(interp, "aio not enabled", -1); + return JIM_ERR; +#endif + + case INFO_PROCS: + mode++; + + case INFO_COMMANDS: + +#ifdef jim_ext_namespace + if (!nons) { + if (Jim_Length(interp->framePtr->nsObj) || (argc == 3 && JimIsGlobalNamespace(argv[2]))) { + return Jim_EvalPrefix(interp, "namespace info", argc - 1, argv + 1); + } + } +#endif + Jim_SetResult(interp, JimCommandsList(interp, (argc == 3) ? argv[2] : NULL, mode)); + return JIM_OK; + + case INFO_VARS: + mode++; + + case INFO_LOCALS: + mode++; + + case INFO_GLOBALS: + +#ifdef jim_ext_namespace + if (!nons) { + if (Jim_Length(interp->framePtr->nsObj) || (argc == 3 && JimIsGlobalNamespace(argv[2]))) { + return Jim_EvalPrefix(interp, "namespace info", argc - 1, argv + 1); + } + } +#endif + Jim_SetResult(interp, JimVariablesList(interp, argc == 3 ? argv[2] : NULL, mode)); + return JIM_OK; + + case INFO_SCRIPT: + if (argc == 3) { + Jim_IncrRefCount(argv[2]); + Jim_DecrRefCount(interp, interp->currentFilenameObj); + interp->currentFilenameObj = argv[2]; + } + Jim_SetResult(interp, interp->currentFilenameObj); + return JIM_OK; + + case INFO_SOURCE:{ + Jim_Obj *resObjPtr; + Jim_Obj *fileNameObj; + + if (argc == 4) { + Jim_SubCmdArgError(interp, ct, argv[0]); + return JIM_ERR; + } + if (argc == 5) { + jim_wide line; + if (Jim_GetWide(interp, argv[4], &line) != JIM_OK) { + return JIM_ERR; + } + resObjPtr = Jim_NewStringObj(interp, Jim_String(argv[2]), Jim_Length(argv[2])); + Jim_SetSourceInfo(interp, resObjPtr, argv[3], line); + } + else { + int line; + fileNameObj = Jim_GetSourceInfo(interp, argv[2], &line); + resObjPtr = Jim_NewListObj(interp, NULL, 0); + Jim_ListAppendElement(interp, resObjPtr, fileNameObj); + Jim_ListAppendElement(interp, resObjPtr, Jim_NewIntObj(interp, line)); + } + Jim_SetResult(interp, resObjPtr); + return JIM_OK; + } + + case INFO_STACKTRACE: + Jim_SetResult(interp, interp->stackTrace); + return JIM_OK; + + case INFO_LEVEL: + if (argc == 2) { + Jim_SetResultInt(interp, interp->framePtr->level); + } + else { + if (JimInfoLevel(interp, argv[2], &objPtr) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + } + return JIM_OK; + + case INFO_FRAME: + if (argc == 2) { + Jim_SetResultInt(interp, interp->procLevel + 1); + } + else { + if (JimInfoFrame(interp, argv[2], &objPtr) != JIM_OK) { + return JIM_ERR; + } + Jim_SetResult(interp, objPtr); + } + return JIM_OK; + + case INFO_BODY: + case INFO_STATICS: + case INFO_ARGS:{ + Jim_Cmd *cmdPtr; + + if ((cmdPtr = Jim_GetCommand(interp, argv[2], JIM_ERRMSG)) == NULL) { + return JIM_ERR; + } + if (!cmdPtr->isproc) { + Jim_SetResultFormatted(interp, "command \"%#s\" is not a procedure", argv[2]); + return JIM_ERR; + } + switch (option) { +#ifdef JIM_NO_INTROSPECTION + default: + Jim_SetResultString(interp, "unsupported", -1); + return JIM_ERR; +#else + case INFO_BODY: + Jim_SetResult(interp, cmdPtr->u.proc.bodyObjPtr); + break; + case INFO_ARGS: + Jim_SetResult(interp, cmdPtr->u.proc.argListObjPtr); + break; +#endif + case INFO_STATICS: + if (cmdPtr->u.proc.staticVars) { + Jim_SetResult(interp, JimHashtablePatternMatch(interp, cmdPtr->u.proc.staticVars, + NULL, JimVariablesMatch, JIM_VARLIST_LOCALS | JIM_VARLIST_VALUES)); + } + break; + } + return JIM_OK; + } + + case INFO_VERSION: + case INFO_PATCHLEVEL:{ + char buf[(JIM_INTEGER_SPACE * 2) + 1]; + + sprintf(buf, "%d.%d", JIM_VERSION / 100, JIM_VERSION % 100); + Jim_SetResultString(interp, buf, -1); + return JIM_OK; + } + + case INFO_COMPLETE: { + char missing; + + Jim_SetResultBool(interp, Jim_ScriptIsComplete(interp, argv[2], &missing)); + if (missing != ' ' && argc == 4) { + Jim_SetVariable(interp, argv[3], Jim_NewStringObj(interp, &missing, 1)); + } + return JIM_OK; + } + + case INFO_HOSTNAME: + + return Jim_Eval(interp, "os.gethostname"); + + case INFO_NAMEOFEXECUTABLE: + + return Jim_Eval(interp, "{info nameofexecutable}"); + + case INFO_RETURNCODES: + if (argc == 2) { + int i; + Jim_Obj *listObjPtr = Jim_NewListObj(interp, NULL, 0); + + for (i = 0; jimReturnCodes[i]; i++) { + Jim_ListAppendElement(interp, listObjPtr, Jim_NewIntObj(interp, i)); + Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, + jimReturnCodes[i], -1)); + } + + Jim_SetResult(interp, listObjPtr); + } + else if (argc == 3) { + long code; + const char *name; + + if (Jim_GetLong(interp, argv[2], &code) != JIM_OK) { + return JIM_ERR; + } + name = Jim_ReturnCode(code); + if (*name == '?') { + Jim_SetResultInt(interp, code); + } + else { + Jim_SetResultString(interp, name, -1); + } + } + return JIM_OK; + case INFO_REFERENCES: +#ifdef JIM_REFERENCES + return JimInfoReferences(interp, argc, argv); +#else + Jim_SetResultString(interp, "not supported", -1); + return JIM_ERR; +#endif + default: + abort(); + } +} + + +static int Jim_ExistsCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + int result = 0; + + static const char * const options[] = { + "-command", "-proc", "-alias", "-var", NULL + }; + enum + { + OPT_COMMAND, OPT_PROC, OPT_ALIAS, OPT_VAR + }; + int option; + + if (argc == 2) { + option = OPT_VAR; + objPtr = argv[1]; + } + else if (argc == 3) { + if (Jim_GetEnum(interp, argv[1], options, &option, NULL, JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { + return JIM_ERR; + } + objPtr = argv[2]; + } + else { + Jim_WrongNumArgs(interp, 1, argv, "?option? name"); + return JIM_ERR; + } + + if (option == OPT_VAR) { + result = Jim_GetVariable(interp, objPtr, 0) != NULL; + } + else { + + Jim_Cmd *cmd = Jim_GetCommand(interp, objPtr, JIM_NONE); + + if (cmd) { + switch (option) { + case OPT_COMMAND: + result = 1; + break; + + case OPT_ALIAS: + result = cmd->isproc == 0 && cmd->u.native.cmdProc == JimAliasCmd; + break; + + case OPT_PROC: + result = cmd->isproc; + break; + } + } + } + Jim_SetResultBool(interp, result); + return JIM_OK; +} + + +static int Jim_SplitCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *str, *splitChars, *noMatchStart; + int splitLen, strLen; + Jim_Obj *resObjPtr; + int c; + int len; + + if (argc != 2 && argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "string ?splitChars?"); + return JIM_ERR; + } + + str = Jim_GetString(argv[1], &len); + if (len == 0) { + return JIM_OK; + } + strLen = Jim_Utf8Length(interp, argv[1]); + + + if (argc == 2) { + splitChars = " \n\t\r"; + splitLen = 4; + } + else { + splitChars = Jim_String(argv[2]); + splitLen = Jim_Utf8Length(interp, argv[2]); + } + + noMatchStart = str; + resObjPtr = Jim_NewListObj(interp, NULL, 0); + + + if (splitLen) { + Jim_Obj *objPtr; + while (strLen--) { + const char *sc = splitChars; + int scLen = splitLen; + int sl = utf8_tounicode(str, &c); + while (scLen--) { + int pc; + sc += utf8_tounicode(sc, &pc); + if (c == pc) { + objPtr = Jim_NewStringObj(interp, noMatchStart, (str - noMatchStart)); + Jim_ListAppendElement(interp, resObjPtr, objPtr); + noMatchStart = str + sl; + break; + } + } + str += sl; + } + objPtr = Jim_NewStringObj(interp, noMatchStart, (str - noMatchStart)); + Jim_ListAppendElement(interp, resObjPtr, objPtr); + } + else { + Jim_Obj **commonObj = NULL; +#define NUM_COMMON (128 - 9) + while (strLen--) { + int n = utf8_tounicode(str, &c); +#ifdef JIM_OPTIMIZATION + if (c >= 9 && c < 128) { + + c -= 9; + if (!commonObj) { + commonObj = Jim_Alloc(sizeof(*commonObj) * NUM_COMMON); + memset(commonObj, 0, sizeof(*commonObj) * NUM_COMMON); + } + if (!commonObj[c]) { + commonObj[c] = Jim_NewStringObj(interp, str, 1); + } + Jim_ListAppendElement(interp, resObjPtr, commonObj[c]); + str++; + continue; + } +#endif + Jim_ListAppendElement(interp, resObjPtr, Jim_NewStringObjUtf8(interp, str, 1)); + str += n; + } + Jim_Free(commonObj); + } + + Jim_SetResult(interp, resObjPtr); + return JIM_OK; +} + + +static int Jim_JoinCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *joinStr; + int joinStrLen; + + if (argc != 2 && argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "list ?joinString?"); + return JIM_ERR; + } + + if (argc == 2) { + joinStr = " "; + joinStrLen = 1; + } + else { + joinStr = Jim_GetString(argv[2], &joinStrLen); + } + Jim_SetResult(interp, Jim_ListJoin(interp, argv[1], joinStr, joinStrLen)); + return JIM_OK; +} + + +static int Jim_FormatCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + + if (argc < 2) { + Jim_WrongNumArgs(interp, 1, argv, "formatString ?arg arg ...?"); + return JIM_ERR; + } + objPtr = Jim_FormatString(interp, argv[1], argc - 2, argv + 2); + if (objPtr == NULL) + return JIM_ERR; + Jim_SetResult(interp, objPtr); + return JIM_OK; +} + + +static int Jim_ScanCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *listPtr, **outVec; + int outc, i; + + if (argc < 3) { + Jim_WrongNumArgs(interp, 1, argv, "string format ?varName varName ...?"); + return JIM_ERR; + } + if (argv[2]->typePtr != &scanFmtStringObjType) + SetScanFmtFromAny(interp, argv[2]); + if (FormatGetError(argv[2]) != 0) { + Jim_SetResultString(interp, FormatGetError(argv[2]), -1); + return JIM_ERR; + } + if (argc > 3) { + int maxPos = FormatGetMaxPos(argv[2]); + int count = FormatGetCnvCount(argv[2]); + + if (maxPos > argc - 3) { + Jim_SetResultString(interp, "\"%n$\" argument index out of range", -1); + return JIM_ERR; + } + else if (count > argc - 3) { + Jim_SetResultString(interp, "different numbers of variable names and " + "field specifiers", -1); + return JIM_ERR; + } + else if (count < argc - 3) { + Jim_SetResultString(interp, "variable is not assigned by any " + "conversion specifiers", -1); + return JIM_ERR; + } + } + listPtr = Jim_ScanString(interp, argv[1], argv[2], JIM_ERRMSG); + if (listPtr == 0) + return JIM_ERR; + if (argc > 3) { + int rc = JIM_OK; + int count = 0; + + if (listPtr != 0 && listPtr != (Jim_Obj *)EOF) { + int len = Jim_ListLength(interp, listPtr); + + if (len != 0) { + JimListGetElements(interp, listPtr, &outc, &outVec); + for (i = 0; i < outc; ++i) { + if (Jim_Length(outVec[i]) > 0) { + ++count; + if (Jim_SetVariable(interp, argv[3 + i], outVec[i]) != JIM_OK) { + rc = JIM_ERR; + } + } + } + } + Jim_FreeNewObj(interp, listPtr); + } + else { + count = -1; + } + if (rc == JIM_OK) { + Jim_SetResultInt(interp, count); + } + return rc; + } + else { + if (listPtr == (Jim_Obj *)EOF) { + Jim_SetResult(interp, Jim_NewListObj(interp, 0, 0)); + return JIM_OK; + } + Jim_SetResult(interp, listPtr); + } + return JIM_OK; +} + + +static int Jim_ErrorCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + if (argc != 2 && argc != 3) { + Jim_WrongNumArgs(interp, 1, argv, "message ?stacktrace?"); + return JIM_ERR; + } + Jim_SetResult(interp, argv[1]); + if (argc == 3) { + JimSetStackTrace(interp, argv[2]); + return JIM_ERR; + } + return JIM_ERR; +} + + +static int Jim_LrangeCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + + if (argc != 4) { + Jim_WrongNumArgs(interp, 1, argv, "list first last"); + return JIM_ERR; + } + if ((objPtr = Jim_ListRange(interp, argv[1], argv[2], argv[3])) == NULL) + return JIM_ERR; + Jim_SetResult(interp, objPtr); + return JIM_OK; +} + + +static int Jim_LrepeatCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *objPtr; + jim_wide count; + + if (argc < 2 || Jim_GetWideExpr(interp, argv[1], &count) != JIM_OK || count < 0) { + Jim_WrongNumArgs(interp, 1, argv, "count ?value ...?"); + return JIM_ERR; + } + if (count == 0 || argc == 2) { + Jim_SetEmptyResult(interp); + return JIM_OK; + } + + argc -= 2; + argv += 2; + + objPtr = Jim_NewListObj(interp, NULL, 0); + ListEnsureLength(objPtr, argc * count); + while (count--) { + ListInsertElements(objPtr, -1, argc, argv); + } + + Jim_SetResult(interp, objPtr); + return JIM_OK; +} + +char **Jim_GetEnviron(void) +{ +#if defined(HAVE__NSGETENVIRON) + return *_NSGetEnviron(); +#elif defined(_environ) + return _environ; +#else + #if !defined(NO_ENVIRON_EXTERN) + extern char **environ; + #endif + return environ; +#endif +} + +void Jim_SetEnviron(char **env) +{ +#if defined(HAVE__NSGETENVIRON) + *_NSGetEnviron() = env; +#elif defined(_environ) + _environ = env; +#else + #if !defined(NO_ENVIRON_EXTERN) + extern char **environ; + #endif + + environ = env; +#endif +} + + +static int Jim_EnvCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const char *key; + const char *val; + + if (argc == 1) { + char **e = Jim_GetEnviron(); + + int i; + Jim_Obj *listObjPtr = Jim_NewListObj(interp, NULL, 0); + + for (i = 0; e[i]; i++) { + const char *equals = strchr(e[i], '='); + + if (equals) { + Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, e[i], + equals - e[i])); + Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, equals + 1, -1)); + } + } + + Jim_SetResult(interp, listObjPtr); + return JIM_OK; + } + + if (argc > 3) { + Jim_WrongNumArgs(interp, 1, argv, "varName ?default?"); + return JIM_ERR; + } + key = Jim_String(argv[1]); + val = getenv(key); + if (val == NULL) { + if (argc < 3) { + Jim_SetResultFormatted(interp, "environment variable \"%#s\" does not exist", argv[1]); + return JIM_ERR; + } + val = Jim_String(argv[2]); + } + Jim_SetResult(interp, Jim_NewStringObj(interp, val, -1)); + return JIM_OK; +} + + +static int Jim_SourceCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + int retval; + + if (argc != 2) { + Jim_WrongNumArgs(interp, 1, argv, "fileName"); + return JIM_ERR; + } + retval = Jim_EvalFile(interp, Jim_String(argv[1])); + if (retval == JIM_RETURN) + return JIM_OK; + return retval; +} + + +static int Jim_LreverseCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + Jim_Obj *revObjPtr, **ele; + int len; + + if (argc != 2) { + Jim_WrongNumArgs(interp, 1, argv, "list"); + return JIM_ERR; + } + JimListGetElements(interp, argv[1], &len, &ele); + revObjPtr = Jim_NewListObj(interp, NULL, 0); + ListEnsureLength(revObjPtr, len); + len--; + while (len >= 0) + ListAppendElement(revObjPtr, ele[len--]); + Jim_SetResult(interp, revObjPtr); + return JIM_OK; +} + +static int JimRangeLen(jim_wide start, jim_wide end, jim_wide step) +{ + jim_wide len; + + if (step == 0) + return -1; + if (start == end) + return 0; + else if (step > 0 && start > end) + return -1; + else if (step < 0 && end > start) + return -1; + len = end - start; + if (len < 0) + len = -len; + if (step < 0) + step = -step; + len = 1 + ((len - 1) / step); + if (len > INT_MAX) + len = INT_MAX; + return (int)((len < 0) ? -1 : len); +} + + +static int Jim_RangeCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_wide start = 0, end, step = 1; + int len, i; + Jim_Obj *objPtr; + + if (argc < 2 || argc > 4) { + Jim_WrongNumArgs(interp, 1, argv, "?start? end ?step?"); + return JIM_ERR; + } + if (argc == 2) { + if (Jim_GetWideExpr(interp, argv[1], &end) != JIM_OK) + return JIM_ERR; + } + else { + if (Jim_GetWideExpr(interp, argv[1], &start) != JIM_OK || + Jim_GetWideExpr(interp, argv[2], &end) != JIM_OK) + return JIM_ERR; + if (argc == 4 && Jim_GetWideExpr(interp, argv[3], &step) != JIM_OK) + return JIM_ERR; + } + if ((len = JimRangeLen(start, end, step)) == -1) { + Jim_SetResultString(interp, "Invalid (infinite?) range specified", -1); + return JIM_ERR; + } + objPtr = Jim_NewListObj(interp, NULL, 0); + ListEnsureLength(objPtr, len); + for (i = 0; i < len; i++) + ListAppendElement(objPtr, Jim_NewIntObj(interp, start + i * step)); + Jim_SetResult(interp, objPtr); + return JIM_OK; +} + + +static int Jim_RandCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + jim_wide min = 0, max = 0, len, maxMul; + + if (argc < 1 || argc > 3) { + Jim_WrongNumArgs(interp, 1, argv, "?min? max"); + return JIM_ERR; + } + if (argc == 1) { + max = JIM_WIDE_MAX; + } else if (argc == 2) { + if (Jim_GetWideExpr(interp, argv[1], &max) != JIM_OK) + return JIM_ERR; + } else if (argc == 3) { + if (Jim_GetWideExpr(interp, argv[1], &min) != JIM_OK || + Jim_GetWideExpr(interp, argv[2], &max) != JIM_OK) + return JIM_ERR; + } + len = max-min; + if (len < 0) { + Jim_SetResultString(interp, "Invalid arguments (max < min)", -1); + return JIM_ERR; + } + maxMul = JIM_WIDE_MAX - (len ? (JIM_WIDE_MAX%len) : 0); + while (1) { + jim_wide r; + + JimRandomBytes(interp, &r, sizeof(jim_wide)); + if (r < 0 || r >= maxMul) continue; + r = (len == 0) ? 0 : r%len; + Jim_SetResultInt(interp, min+r); + return JIM_OK; + } +} + +static const struct { + const char *name; + Jim_CmdProc *cmdProc; +} Jim_CoreCommandsTable[] = { + {"alias", Jim_AliasCoreCommand}, + {"set", Jim_SetCoreCommand}, + {"unset", Jim_UnsetCoreCommand}, + {"puts", Jim_PutsCoreCommand}, + {"+", Jim_AddCoreCommand}, + {"*", Jim_MulCoreCommand}, + {"-", Jim_SubCoreCommand}, + {"/", Jim_DivCoreCommand}, + {"incr", Jim_IncrCoreCommand}, + {"while", Jim_WhileCoreCommand}, + {"loop", Jim_LoopCoreCommand}, + {"for", Jim_ForCoreCommand}, + {"foreach", Jim_ForeachCoreCommand}, + {"lmap", Jim_LmapCoreCommand}, + {"lassign", Jim_LassignCoreCommand}, + {"if", Jim_IfCoreCommand}, + {"switch", Jim_SwitchCoreCommand}, + {"list", Jim_ListCoreCommand}, + {"lindex", Jim_LindexCoreCommand}, + {"lset", Jim_LsetCoreCommand}, + {"lsearch", Jim_LsearchCoreCommand}, + {"llength", Jim_LlengthCoreCommand}, + {"lappend", Jim_LappendCoreCommand}, + {"linsert", Jim_LinsertCoreCommand}, + {"lreplace", Jim_LreplaceCoreCommand}, + {"lsort", Jim_LsortCoreCommand}, + {"append", Jim_AppendCoreCommand}, + {"eval", Jim_EvalCoreCommand}, + {"uplevel", Jim_UplevelCoreCommand}, + {"expr", Jim_ExprCoreCommand}, + {"break", Jim_BreakCoreCommand}, + {"continue", Jim_ContinueCoreCommand}, + {"proc", Jim_ProcCoreCommand}, + {"xtrace", Jim_XtraceCoreCommand}, + {"concat", Jim_ConcatCoreCommand}, + {"return", Jim_ReturnCoreCommand}, + {"upvar", Jim_UpvarCoreCommand}, + {"global", Jim_GlobalCoreCommand}, + {"string", Jim_StringCoreCommand}, + {"time", Jim_TimeCoreCommand}, + {"timerate", Jim_TimeRateCoreCommand}, + {"exit", Jim_ExitCoreCommand}, + {"catch", Jim_CatchCoreCommand}, + {"try", Jim_TryCoreCommand}, +#ifdef JIM_REFERENCES + {"ref", Jim_RefCoreCommand}, + {"getref", Jim_GetrefCoreCommand}, + {"setref", Jim_SetrefCoreCommand}, + {"finalize", Jim_FinalizeCoreCommand}, + {"collect", Jim_CollectCoreCommand}, +#endif + {"rename", Jim_RenameCoreCommand}, + {"dict", Jim_DictCoreCommand}, + {"subst", Jim_SubstCoreCommand}, + {"info", Jim_InfoCoreCommand}, + {"exists", Jim_ExistsCoreCommand}, + {"split", Jim_SplitCoreCommand}, + {"join", Jim_JoinCoreCommand}, + {"format", Jim_FormatCoreCommand}, + {"scan", Jim_ScanCoreCommand}, + {"error", Jim_ErrorCoreCommand}, + {"lrange", Jim_LrangeCoreCommand}, + {"lrepeat", Jim_LrepeatCoreCommand}, + {"env", Jim_EnvCoreCommand}, + {"source", Jim_SourceCoreCommand}, + {"lreverse", Jim_LreverseCoreCommand}, + {"range", Jim_RangeCoreCommand}, + {"rand", Jim_RandCoreCommand}, + {"tailcall", Jim_TailcallCoreCommand}, + {"local", Jim_LocalCoreCommand}, + {"upcall", Jim_UpcallCoreCommand}, + {"apply", Jim_ApplyCoreCommand}, + {"stacktrace", Jim_StacktraceCoreCommand}, + {NULL, NULL}, +}; + +void Jim_RegisterCoreCommands(Jim_Interp *interp) +{ + int i = 0; + + while (Jim_CoreCommandsTable[i].name != NULL) { + Jim_CreateCommand(interp, + Jim_CoreCommandsTable[i].name, Jim_CoreCommandsTable[i].cmdProc, NULL, NULL); + i++; + } +} + +void Jim_MakeErrorMessage(Jim_Interp *interp) +{ + Jim_Obj *argv[2]; + + argv[0] = Jim_NewStringObj(interp, "errorInfo", -1); + argv[1] = interp->result; + + Jim_EvalObjVector(interp, 2, argv); +} + +static char **JimSortStringTable(const char *const *tablePtr) +{ + int count; + char **tablePtrSorted; + + + for (count = 0; tablePtr[count]; count++) { + } + + + tablePtrSorted = Jim_Alloc(sizeof(char *) * (count + 1)); + memcpy(tablePtrSorted, tablePtr, sizeof(char *) * count); + qsort(tablePtrSorted, count, sizeof(char *), qsortCompareStringPointers); + tablePtrSorted[count] = NULL; + + return tablePtrSorted; +} + +static void JimSetFailedEnumResult(Jim_Interp *interp, const char *arg, const char *badtype, + const char *prefix, const char *const *tablePtr, const char *name) +{ + char **tablePtrSorted; + int i; + + if (name == NULL) { + name = "option"; + } + + Jim_SetResultFormatted(interp, "%s%s \"%s\": must be ", badtype, name, arg); + tablePtrSorted = JimSortStringTable(tablePtr); + for (i = 0; tablePtrSorted[i]; i++) { + if (tablePtrSorted[i + 1] == NULL && i > 0) { + Jim_AppendString(interp, Jim_GetResult(interp), "or ", -1); + } + Jim_AppendStrings(interp, Jim_GetResult(interp), prefix, tablePtrSorted[i], NULL); + if (tablePtrSorted[i + 1]) { + Jim_AppendString(interp, Jim_GetResult(interp), ", ", -1); + } + } + Jim_Free(tablePtrSorted); +} + + +int Jim_CheckShowCommands(Jim_Interp *interp, Jim_Obj *objPtr, const char *const *tablePtr) +{ + if (Jim_CompareStringImmediate(interp, objPtr, "-commands")) { + int i; + char **tablePtrSorted = JimSortStringTable(tablePtr); + Jim_SetResult(interp, Jim_NewListObj(interp, NULL, 0)); + for (i = 0; tablePtrSorted[i]; i++) { + Jim_ListAppendElement(interp, Jim_GetResult(interp), Jim_NewStringObj(interp, tablePtrSorted[i], -1)); + } + Jim_Free(tablePtrSorted); + return JIM_OK; + } + return JIM_ERR; +} + +static const Jim_ObjType getEnumObjType = { + "get-enum", + NULL, + NULL, + NULL, + JIM_TYPE_REFERENCES +}; + +int Jim_GetEnum(Jim_Interp *interp, Jim_Obj *objPtr, + const char *const *tablePtr, int *indexPtr, const char *name, int flags) +{ + const char *bad = "bad "; + const char *const *entryPtr = NULL; + int i; + int match = -1; + int arglen; + const char *arg; + + if (objPtr->typePtr == &getEnumObjType) { + if (objPtr->internalRep.ptrIntValue.ptr == tablePtr && objPtr->internalRep.ptrIntValue.int1 == flags) { + *indexPtr = objPtr->internalRep.ptrIntValue.int2; + return JIM_OK; + } + } + + arg = Jim_GetString(objPtr, &arglen); + + *indexPtr = -1; + + for (entryPtr = tablePtr, i = 0; *entryPtr != NULL; entryPtr++, i++) { + if (Jim_CompareStringImmediate(interp, objPtr, *entryPtr)) { + + match = i; + goto found; + } + if (flags & JIM_ENUM_ABBREV) { + if (strncmp(arg, *entryPtr, arglen) == 0) { + if (*arg == '-' && arglen == 1) { + break; + } + if (match >= 0) { + bad = "ambiguous "; + goto ambiguous; + } + match = i; + } + } + } + + + if (match >= 0) { + found: + + Jim_FreeIntRep(interp, objPtr); + objPtr->typePtr = &getEnumObjType; + objPtr->internalRep.ptrIntValue.ptr = (void *)tablePtr; + objPtr->internalRep.ptrIntValue.int1 = flags; + objPtr->internalRep.ptrIntValue.int2 = match; + + *indexPtr = match; + return JIM_OK; + } + + ambiguous: + if (flags & JIM_ERRMSG) { + JimSetFailedEnumResult(interp, arg, bad, "", tablePtr, name); + } + return JIM_ERR; +} + +int Jim_FindByName(const char *name, const char * const array[], size_t len) +{ + int i; + + for (i = 0; i < (int)len; i++) { + if (array[i] && strcmp(array[i], name) == 0) { + return i; + } + } + return -1; +} + +int Jim_IsDict(Jim_Obj *objPtr) +{ + return objPtr->typePtr == &dictObjType; +} + +int Jim_IsList(Jim_Obj *objPtr) +{ + return objPtr->typePtr == &listObjType; +} + +void Jim_SetResultFormatted(Jim_Interp *interp, const char *format, ...) +{ + + int len = strlen(format); + int extra = 0; + int n = 0; + const char *params[5]; + int nobjparam = 0; + Jim_Obj *objparam[5]; + char *buf; + va_list args; + int i; + + va_start(args, format); + + for (i = 0; i < len && n < 5; i++) { + int l; + + if (strncmp(format + i, "%s", 2) == 0) { + params[n] = va_arg(args, char *); + + l = strlen(params[n]); + } + else if (strncmp(format + i, "%#s", 3) == 0) { + Jim_Obj *objPtr = va_arg(args, Jim_Obj *); + + params[n] = Jim_GetString(objPtr, &l); + objparam[nobjparam++] = objPtr; + Jim_IncrRefCount(objPtr); + } + else { + if (format[i] == '%') { + i++; + } + continue; + } + n++; + extra += l; + } + + len += extra; + buf = Jim_Alloc(len + 1); + len = snprintf(buf, len + 1, format, params[0], params[1], params[2], params[3], params[4]); + + va_end(args); + + Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, buf, len)); + + for (i = 0; i < nobjparam; i++) { + Jim_DecrRefCount(interp, objparam[i]); + } +} + +int Jim_CheckAbiVersion(Jim_Interp *interp, int abi_version) +{ + if (abi_version != JIM_ABI_VERSION) { + Jim_SetResultString(interp, "ABI version mismatch", -1); + return JIM_ERR; + } + return JIM_OK; +} + + +#ifndef jim_ext_package +int Jim_PackageProvide(Jim_Interp *interp, const char *name, const char *ver, int flags) +{ + return JIM_OK; +} +#endif +#ifndef jim_ext_aio +int Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *fhObj) +{ + return -1; +} +#endif + + +#include +#include + + +static int subcmd_null(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + + return JIM_OK; +} + +static const jim_subcmd_type dummy_subcmd = { + "dummy", NULL, subcmd_null, 0, 0, JIM_MODFLAG_HIDDEN +}; + +static Jim_Obj *subcmd_cmd_list(Jim_Interp *interp, const jim_subcmd_type * ct, const char *sep) +{ + + Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); + Jim_Obj *sortCmd[2]; + + for (; ct->cmd; ct++) { + if (!(ct->flags & JIM_MODFLAG_HIDDEN)) { + Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, ct->cmd, -1)); + } + } + + + sortCmd[0] = Jim_NewStringObj(interp, "lsort", -1); + sortCmd[1] = listObj; + + if (Jim_EvalObjVector(interp, 2, sortCmd) == JIM_OK) { + return Jim_ListJoin(interp, Jim_GetResult(interp), sep, strlen(sep)); + } + + return Jim_GetResult(interp); +} + +static void bad_subcmd(Jim_Interp *interp, const jim_subcmd_type * command_table, const char *type, + Jim_Obj *cmd, Jim_Obj *subcmd) +{ + Jim_SetResultFormatted(interp, "%#s, %s command \"%#s\": should be %#s", cmd, type, + subcmd, subcmd_cmd_list(interp, command_table, ", ")); +} + +static void show_cmd_usage(Jim_Interp *interp, const jim_subcmd_type * command_table, int argc, + Jim_Obj *const *argv) +{ + Jim_SetResultFormatted(interp, "Usage: \"%#s command ... \", where command is one of: %#s", + argv[0], subcmd_cmd_list(interp, command_table, ", ")); +} + +static void add_cmd_usage(Jim_Interp *interp, const jim_subcmd_type * ct, Jim_Obj *cmd) +{ + if (cmd) { + Jim_AppendStrings(interp, Jim_GetResult(interp), Jim_String(cmd), " ", NULL); + } + Jim_AppendStrings(interp, Jim_GetResult(interp), ct->cmd, NULL); + if (ct->args && *ct->args) { + Jim_AppendStrings(interp, Jim_GetResult(interp), " ", ct->args, NULL); + } +} + +void Jim_SubCmdArgError(Jim_Interp *interp, const jim_subcmd_type * ct, Jim_Obj *subcmd) +{ + Jim_SetResultString(interp, "wrong # args: should be \"", -1); + add_cmd_usage(interp, ct, subcmd); + Jim_AppendStrings(interp, Jim_GetResult(interp), "\"", NULL); +} + +static const Jim_ObjType subcmdLookupObjType = { + "subcmd-lookup", + NULL, + NULL, + NULL, + JIM_TYPE_REFERENCES +}; + +const jim_subcmd_type *Jim_ParseSubCmd(Jim_Interp *interp, const jim_subcmd_type * command_table, + int argc, Jim_Obj *const *argv) +{ + const jim_subcmd_type *ct; + const jim_subcmd_type *partial = 0; + int cmdlen; + Jim_Obj *cmd; + const char *cmdstr; + int help = 0; + int argsok = 1; + + if (argc < 2) { + Jim_SetResultFormatted(interp, "wrong # args: should be \"%#s command ...\"\n" + "Use \"%#s -help ?command?\" for help", argv[0], argv[0]); + return 0; + } + + cmd = argv[1]; + + + if (cmd->typePtr == &subcmdLookupObjType) { + if (cmd->internalRep.ptrIntValue.ptr == command_table) { + ct = command_table + cmd->internalRep.ptrIntValue.int1; + goto found; + } + } + + + if (Jim_CompareStringImmediate(interp, cmd, "-help")) { + if (argc == 2) { + + show_cmd_usage(interp, command_table, argc, argv); + return &dummy_subcmd; + } + help = 1; + + + cmd = argv[2]; + } + + + if (Jim_CompareStringImmediate(interp, cmd, "-commands")) { + Jim_SetResult(interp, subcmd_cmd_list(interp, command_table, " ")); + return &dummy_subcmd; + } + + cmdstr = Jim_GetString(cmd, &cmdlen); + + for (ct = command_table; ct->cmd; ct++) { + if (Jim_CompareStringImmediate(interp, cmd, ct->cmd)) { + + break; + } + if (strncmp(cmdstr, ct->cmd, cmdlen) == 0) { + if (partial) { + + if (help) { + + show_cmd_usage(interp, command_table, argc, argv); + return &dummy_subcmd; + } + bad_subcmd(interp, command_table, "ambiguous", argv[0], argv[1 + help]); + return 0; + } + partial = ct; + } + continue; + } + + + if (partial && !ct->cmd) { + ct = partial; + } + + if (!ct->cmd) { + + if (help) { + + show_cmd_usage(interp, command_table, argc, argv); + return &dummy_subcmd; + } + bad_subcmd(interp, command_table, "unknown", argv[0], argv[1 + help]); + return 0; + } + + if (help) { + Jim_SetResultString(interp, "Usage: ", -1); + + add_cmd_usage(interp, ct, argv[0]); + return &dummy_subcmd; + } + + + Jim_FreeIntRep(interp, cmd); + cmd->typePtr = &subcmdLookupObjType; + cmd->internalRep.ptrIntValue.ptr = (void *)command_table; + cmd->internalRep.ptrIntValue.int1 = ct - command_table; + +found: + + + if (argc - 2 < ct->minargs) { + argsok = 0; + } + else if (ct->maxargs >= 0 && argc - 2 > ct->maxargs) { + argsok = 0; + } + else if (ct->maxargs < -1 && (argc - 2) % -ct->maxargs != 0) { + + argsok = 0; + } + if (!argsok) { + Jim_SetResultString(interp, "wrong # args: should be \"", -1); + + add_cmd_usage(interp, ct, argv[0]); + Jim_AppendStrings(interp, Jim_GetResult(interp), "\"", NULL); + + return 0; + } + + + return ct; +} + +int Jim_CallSubCmd(Jim_Interp *interp, const jim_subcmd_type * ct, int argc, Jim_Obj *const *argv) +{ + int ret = JIM_ERR; + + if (ct) { + if (ct->flags & JIM_MODFLAG_FULLARGV) { + ret = ct->function(interp, argc, argv); + } + else { + ret = ct->function(interp, argc - 2, argv + 2); + } + if (ret < 0) { + Jim_SubCmdArgError(interp, ct, argv[0]); + ret = JIM_ERR; + } + } + return ret; +} + +int Jim_SubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv) +{ + const jim_subcmd_type *ct = + Jim_ParseSubCmd(interp, (const jim_subcmd_type *)Jim_CmdPrivData(interp), argc, argv); + + return Jim_CallSubCmd(interp, ct, argc, argv); +} + +#include +#include +#include +#include +#include + + +int utf8_fromunicode(char *p, unsigned uc) +{ + if (uc <= 0x7f) { + *p = uc; + return 1; + } + else if (uc <= 0x7ff) { + *p++ = 0xc0 | ((uc & 0x7c0) >> 6); + *p = 0x80 | (uc & 0x3f); + return 2; + } + else if (uc <= 0xffff) { + *p++ = 0xe0 | ((uc & 0xf000) >> 12); + *p++ = 0x80 | ((uc & 0xfc0) >> 6); + *p = 0x80 | (uc & 0x3f); + return 3; + } + + else { + *p++ = 0xf0 | ((uc & 0x1c0000) >> 18); + *p++ = 0x80 | ((uc & 0x3f000) >> 12); + *p++ = 0x80 | ((uc & 0xfc0) >> 6); + *p = 0x80 | (uc & 0x3f); + return 4; + } +} + +#include +#include +#include + + +#define JIM_INTEGER_SPACE 24 +#define MAX_FLOAT_WIDTH 320 + +Jim_Obj *Jim_FormatString(Jim_Interp *interp, Jim_Obj *fmtObjPtr, int objc, Jim_Obj *const *objv) +{ + const char *span, *format, *formatEnd, *msg; + int numBytes = 0, objIndex = 0, gotXpg = 0, gotSequential = 0; + static const char * const mixedXPG = + "cannot mix \"%\" and \"%n$\" conversion specifiers"; + static const char * const badIndex[2] = { + "not enough arguments for all format specifiers", + "\"%n$\" argument index out of range" + }; + int formatLen; + Jim_Obj *resultPtr; + + char *num_buffer = NULL; + int num_buffer_size = 0; + + span = format = Jim_GetString(fmtObjPtr, &formatLen); + formatEnd = format + formatLen; + resultPtr = Jim_NewEmptyStringObj(interp); + + while (format != formatEnd) { + char *end; + int gotMinus, sawFlag; + int gotPrecision, useShort; + long width, precision; + int newXpg; + int ch; + int step; + int doubleType; + char pad = ' '; + char spec[2*JIM_INTEGER_SPACE + 12]; + char *p; + + int formatted_chars; + int formatted_bytes; + const char *formatted_buf; + + step = utf8_tounicode(format, &ch); + format += step; + if (ch != '%') { + numBytes += step; + continue; + } + if (numBytes) { + Jim_AppendString(interp, resultPtr, span, numBytes); + numBytes = 0; + } + + + step = utf8_tounicode(format, &ch); + if (ch == '%') { + span = format; + numBytes = step; + format += step; + continue; + } + + + newXpg = 0; + if (isdigit(ch)) { + int position = strtoul(format, &end, 10); + if (*end == '$') { + newXpg = 1; + objIndex = position - 1; + format = end + 1; + step = utf8_tounicode(format, &ch); + } + } + if (newXpg) { + if (gotSequential) { + msg = mixedXPG; + goto errorMsg; + } + gotXpg = 1; + } else { + if (gotXpg) { + msg = mixedXPG; + goto errorMsg; + } + gotSequential = 1; + } + if ((objIndex < 0) || (objIndex >= objc)) { + msg = badIndex[gotXpg]; + goto errorMsg; + } + + p = spec; + *p++ = '%'; + + gotMinus = 0; + sawFlag = 1; + do { + switch (ch) { + case '-': + gotMinus = 1; + break; + case '0': + pad = ch; + break; + case ' ': + case '+': + case '#': + break; + default: + sawFlag = 0; + continue; + } + *p++ = ch; + format += step; + step = utf8_tounicode(format, &ch); + + } while (sawFlag && (p - spec <= 5)); + + + width = 0; + if (isdigit(ch)) { + width = strtoul(format, &end, 10); + format = end; + step = utf8_tounicode(format, &ch); + } else if (ch == '*') { + if (objIndex >= objc - 1) { + msg = badIndex[gotXpg]; + goto errorMsg; + } + if (Jim_GetLong(interp, objv[objIndex], &width) != JIM_OK) { + goto error; + } + if (width < 0) { + width = -width; + if (!gotMinus) { + *p++ = '-'; + gotMinus = 1; + } + } + objIndex++; + format += step; + step = utf8_tounicode(format, &ch); + } + + + gotPrecision = precision = 0; + if (ch == '.') { + gotPrecision = 1; + format += step; + step = utf8_tounicode(format, &ch); + } + if (isdigit(ch)) { + precision = strtoul(format, &end, 10); + format = end; + step = utf8_tounicode(format, &ch); + } else if (ch == '*') { + if (objIndex >= objc - 1) { + msg = badIndex[gotXpg]; + goto errorMsg; + } + if (Jim_GetLong(interp, objv[objIndex], &precision) != JIM_OK) { + goto error; + } + + + if (precision < 0) { + precision = 0; + } + objIndex++; + format += step; + step = utf8_tounicode(format, &ch); + } + + + useShort = 0; + if (ch == 'h') { + useShort = 1; + format += step; + step = utf8_tounicode(format, &ch); + } else if (ch == 'l') { + + format += step; + step = utf8_tounicode(format, &ch); + if (ch == 'l') { + format += step; + step = utf8_tounicode(format, &ch); + } + } + + format += step; + span = format; + + + if (ch == 'i') { + ch = 'd'; + } + + doubleType = 0; + + switch (ch) { + case '\0': + msg = "format string ended in middle of field specifier"; + goto errorMsg; + case 's': { + formatted_buf = Jim_GetString(objv[objIndex], &formatted_bytes); + formatted_chars = Jim_Utf8Length(interp, objv[objIndex]); + if (gotPrecision && (precision < formatted_chars)) { + + formatted_chars = precision; + formatted_bytes = utf8_index(formatted_buf, precision); + } + break; + } + case 'c': { + jim_wide code; + + if (Jim_GetWide(interp, objv[objIndex], &code) != JIM_OK) { + goto error; + } + + formatted_bytes = utf8_getchars(spec, code); + formatted_buf = spec; + formatted_chars = 1; + break; + } + case 'b': { + unsigned jim_wide w; + int length; + int i; + int j; + + if (Jim_GetWide(interp, objv[objIndex], (jim_wide *)&w) != JIM_OK) { + goto error; + } + length = sizeof(w) * 8; + + + + if (num_buffer_size < length + 1) { + num_buffer_size = length + 1; + num_buffer = Jim_Realloc(num_buffer, num_buffer_size); + } + + j = 0; + for (i = length; i > 0; ) { + i--; + if (w & ((unsigned jim_wide)1 << i)) { + num_buffer[j++] = '1'; + } + else if (j || i == 0) { + num_buffer[j++] = '0'; + } + } + num_buffer[j] = 0; + formatted_chars = formatted_bytes = j; + formatted_buf = num_buffer; + break; + } + + case 'e': + case 'E': + case 'f': + case 'g': + case 'G': + doubleType = 1; + + case 'd': + case 'u': + case 'o': + case 'x': + case 'X': { + jim_wide w; + double d; + int length; + + + if (width) { + p += sprintf(p, "%ld", width); + } + if (gotPrecision) { + p += sprintf(p, ".%ld", precision); + } + + + if (doubleType) { + if (Jim_GetDouble(interp, objv[objIndex], &d) != JIM_OK) { + goto error; + } + length = MAX_FLOAT_WIDTH; + } + else { + if (Jim_GetWide(interp, objv[objIndex], &w) != JIM_OK) { + goto error; + } + length = JIM_INTEGER_SPACE; + if (useShort) { + if (ch == 'd') { + w = (short)w; + } + else { + w = (unsigned short)w; + } + } + *p++ = 'l'; +#ifdef HAVE_LONG_LONG + if (sizeof(long long) == sizeof(jim_wide)) { + *p++ = 'l'; + } +#endif + } + + *p++ = (char) ch; + *p = '\0'; + + + if (width > 10000 || length > 10000 || precision > 10000) { + Jim_SetResultString(interp, "format too long", -1); + goto error; + } + + + + if (width > length) { + length = width; + } + if (gotPrecision) { + length += precision; + } + + + if (num_buffer_size < length + 1) { + num_buffer_size = length + 1; + num_buffer = Jim_Realloc(num_buffer, num_buffer_size); + } + + if (doubleType) { + snprintf(num_buffer, length + 1, spec, d); + } + else { + formatted_bytes = snprintf(num_buffer, length + 1, spec, w); + } + formatted_chars = formatted_bytes = strlen(num_buffer); + formatted_buf = num_buffer; + break; + } + + default: { + + spec[0] = ch; + spec[1] = '\0'; + Jim_SetResultFormatted(interp, "bad field specifier \"%s\"", spec); + goto error; + } + } + + if (!gotMinus) { + while (formatted_chars < width) { + Jim_AppendString(interp, resultPtr, &pad, 1); + formatted_chars++; + } + } + + Jim_AppendString(interp, resultPtr, formatted_buf, formatted_bytes); + + while (formatted_chars < width) { + Jim_AppendString(interp, resultPtr, &pad, 1); + formatted_chars++; + } + + objIndex += gotSequential; + } + if (numBytes) { + Jim_AppendString(interp, resultPtr, span, numBytes); + } + + Jim_Free(num_buffer); + return resultPtr; + + errorMsg: + Jim_SetResultString(interp, msg, -1); + error: + Jim_FreeNewObj(interp, resultPtr); + Jim_Free(num_buffer); + return NULL; +} + + +#if defined(JIM_REGEXP) +#include +#include +#include +#include + + + +#define REG_MAX_PAREN 100 + + + +#define END 0 +#define BOL 1 +#define EOL 2 +#define ANY 3 +#define ANYOF 4 +#define ANYBUT 5 +#define BRANCH 6 +#define BACK 7 +#define EXACTLY 8 +#define NOTHING 9 +#define REP 10 +#define REPMIN 11 +#define REPX 12 +#define REPXMIN 13 +#define BOLX 14 +#define EOLX 15 +#define WORDA 16 +#define WORDZ 17 + +#define OPENNC 1000 +#define OPEN 1001 + + + + +#define CLOSENC 2000 +#define CLOSE 2001 +#define CLOSE_END (CLOSE+REG_MAX_PAREN) + +#define REG_MAGIC 0xFADED00D + + +#define OP(preg, p) (preg->program[p]) +#define NEXT(preg, p) (preg->program[p + 1]) +#define OPERAND(p) ((p) + 2) + + + + +#define FAIL(R,M) { (R)->err = (M); return (M); } +#define ISMULT(c) ((c) == '*' || (c) == '+' || (c) == '?' || (c) == '{') +#define META "^$.[()|?{+*" + +#define HASWIDTH 1 +#define SIMPLE 2 +#define SPSTART 4 +#define WORST 0 + +#define MAX_REP_COUNT 1000000 + +static int reg(regex_t *preg, int paren, int *flagp ); +static int regpiece(regex_t *preg, int *flagp ); +static int regbranch(regex_t *preg, int *flagp ); +static int regatom(regex_t *preg, int *flagp ); +static int regnode(regex_t *preg, int op ); +static int regnext(regex_t *preg, int p ); +static void regc(regex_t *preg, int b ); +static int reginsert(regex_t *preg, int op, int size, int opnd ); +static void regtail(regex_t *preg, int p, int val); +static void regoptail(regex_t *preg, int p, int val ); +static int regopsize(regex_t *preg, int p ); + +static int reg_range_find(const int *string, int c); +static const char *str_find(const char *string, int c, int nocase); +static int prefix_cmp(const int *prog, int proglen, const char *string, int nocase); + + +#ifdef DEBUG +static int regnarrate = 0; +static void regdump(regex_t *preg); +static const char *regprop( int op ); +#endif + + +static int str_int_len(const int *seq) +{ + int n = 0; + while (*seq++) { + n++; + } + return n; +} + +int jim_regcomp(regex_t *preg, const char *exp, int cflags) +{ + int scan; + int longest; + unsigned len; + int flags; + +#ifdef DEBUG + fprintf(stderr, "Compiling: '%s'\n", exp); +#endif + memset(preg, 0, sizeof(*preg)); + + if (exp == NULL) + FAIL(preg, REG_ERR_NULL_ARGUMENT); + + + preg->cflags = cflags; + preg->regparse = exp; + + + preg->proglen = (strlen(exp) + 1) * 5; + preg->program = malloc(preg->proglen * sizeof(int)); + if (preg->program == NULL) + FAIL(preg, REG_ERR_NOMEM); + + regc(preg, REG_MAGIC); + if (reg(preg, 0, &flags) == 0) { + return preg->err; + } + + + if (preg->re_nsub >= REG_MAX_PAREN) + FAIL(preg,REG_ERR_TOO_BIG); + + + preg->regstart = 0; + preg->reganch = 0; + preg->regmust = 0; + preg->regmlen = 0; + scan = 1; + if (OP(preg, regnext(preg, scan)) == END) { + scan = OPERAND(scan); + + + if (OP(preg, scan) == EXACTLY) { + preg->regstart = preg->program[OPERAND(scan)]; + } + else if (OP(preg, scan) == BOL) + preg->reganch++; + + if (flags&SPSTART) { + longest = 0; + len = 0; + for (; scan != 0; scan = regnext(preg, scan)) { + if (OP(preg, scan) == EXACTLY) { + int plen = str_int_len(preg->program + OPERAND(scan)); + if (plen >= len) { + longest = OPERAND(scan); + len = plen; + } + } + } + preg->regmust = longest; + preg->regmlen = len; + } + } + +#ifdef DEBUG + regdump(preg); +#endif + + return 0; +} + +static int reg(regex_t *preg, int paren, int *flagp ) +{ + int ret; + int br; + int ender; + int parno = 0; + int flags; + + *flagp = HASWIDTH; + + + if (paren) { + if (preg->regparse[0] == '?' && preg->regparse[1] == ':') { + + preg->regparse += 2; + parno = -1; + } + else { + parno = ++preg->re_nsub; + } + ret = regnode(preg, OPEN+parno); + } else + ret = 0; + + + br = regbranch(preg, &flags); + if (br == 0) + return 0; + if (ret != 0) + regtail(preg, ret, br); + else + ret = br; + if (!(flags&HASWIDTH)) + *flagp &= ~HASWIDTH; + *flagp |= flags&SPSTART; + while (*preg->regparse == '|') { + preg->regparse++; + br = regbranch(preg, &flags); + if (br == 0) + return 0; + regtail(preg, ret, br); + if (!(flags&HASWIDTH)) + *flagp &= ~HASWIDTH; + *flagp |= flags&SPSTART; + } + + + ender = regnode(preg, (paren) ? CLOSE+parno : END); + regtail(preg, ret, ender); + + + for (br = ret; br != 0; br = regnext(preg, br)) + regoptail(preg, br, ender); + + + if (paren && *preg->regparse++ != ')') { + preg->err = REG_ERR_UNMATCHED_PAREN; + return 0; + } else if (!paren && *preg->regparse != '\0') { + if (*preg->regparse == ')') { + preg->err = REG_ERR_UNMATCHED_PAREN; + return 0; + } else { + preg->err = REG_ERR_JUNK_ON_END; + return 0; + } + } + + return(ret); +} + +static int regbranch(regex_t *preg, int *flagp ) +{ + int ret; + int chain; + int latest; + int flags; + + *flagp = WORST; + + ret = regnode(preg, BRANCH); + chain = 0; + while (*preg->regparse != '\0' && *preg->regparse != ')' && + *preg->regparse != '|') { + latest = regpiece(preg, &flags); + if (latest == 0) + return 0; + *flagp |= flags&HASWIDTH; + if (chain == 0) { + *flagp |= flags&SPSTART; + } + else { + regtail(preg, chain, latest); + } + chain = latest; + } + if (chain == 0) + (void) regnode(preg, NOTHING); + + return(ret); +} + +static int regpiece(regex_t *preg, int *flagp) +{ + int ret; + char op; + int next; + int flags; + int min; + int max; + + ret = regatom(preg, &flags); + if (ret == 0) + return 0; + + op = *preg->regparse; + if (!ISMULT(op)) { + *flagp = flags; + return(ret); + } + + if (!(flags&HASWIDTH) && op != '?') { + preg->err = REG_ERR_OPERAND_COULD_BE_EMPTY; + return 0; + } + + + if (op == '{') { + char *end; + + min = strtoul(preg->regparse + 1, &end, 10); + if (end == preg->regparse + 1) { + preg->err = REG_ERR_BAD_COUNT; + return 0; + } + if (*end == '}') { + max = min; + } + else if (*end == '\0') { + preg->err = REG_ERR_UNMATCHED_BRACES; + return 0; + } + else { + preg->regparse = end; + max = strtoul(preg->regparse + 1, &end, 10); + if (*end != '}') { + preg->err = REG_ERR_UNMATCHED_BRACES; + return 0; + } + } + if (end == preg->regparse + 1) { + max = MAX_REP_COUNT; + } + else if (max < min || max >= 100) { + preg->err = REG_ERR_BAD_COUNT; + return 0; + } + if (min >= 100) { + preg->err = REG_ERR_BAD_COUNT; + return 0; + } + + preg->regparse = strchr(preg->regparse, '}'); + } + else { + min = (op == '+'); + max = (op == '?' ? 1 : MAX_REP_COUNT); + } + + if (preg->regparse[1] == '?') { + preg->regparse++; + next = reginsert(preg, flags & SIMPLE ? REPMIN : REPXMIN, 5, ret); + } + else { + next = reginsert(preg, flags & SIMPLE ? REP: REPX, 5, ret); + } + preg->program[ret + 2] = max; + preg->program[ret + 3] = min; + preg->program[ret + 4] = 0; + + *flagp = (min) ? (WORST|HASWIDTH) : (WORST|SPSTART); + + if (!(flags & SIMPLE)) { + int back = regnode(preg, BACK); + regtail(preg, back, ret); + regtail(preg, next, back); + } + + preg->regparse++; + if (ISMULT(*preg->regparse)) { + preg->err = REG_ERR_NESTED_COUNT; + return 0; + } + + return ret; +} + +static void reg_addrange(regex_t *preg, int lower, int upper) +{ + if (lower > upper) { + reg_addrange(preg, upper, lower); + } + + regc(preg, upper - lower + 1); + regc(preg, lower); +} + +static void reg_addrange_str(regex_t *preg, const char *str) +{ + while (*str) { + reg_addrange(preg, *str, *str); + str++; + } +} + +static int reg_utf8_tounicode_case(const char *s, int *uc, int upper) +{ + int l = utf8_tounicode(s, uc); + if (upper) { + *uc = utf8_upper(*uc); + } + return l; +} + +static int hexdigitval(int c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; +} + +static int parse_hex(const char *s, int n, int *uc) +{ + int val = 0; + int k; + + for (k = 0; k < n; k++) { + int c = hexdigitval(*s++); + if (c == -1) { + break; + } + val = (val << 4) | c; + } + if (k) { + *uc = val; + } + return k; +} + +static int reg_decode_escape(const char *s, int *ch) +{ + int n; + const char *s0 = s; + + *ch = *s++; + + switch (*ch) { + case 'b': *ch = '\b'; break; + case 'e': *ch = 27; break; + case 'f': *ch = '\f'; break; + case 'n': *ch = '\n'; break; + case 'r': *ch = '\r'; break; + case 't': *ch = '\t'; break; + case 'v': *ch = '\v'; break; + case 'u': + if (*s == '{') { + + n = parse_hex(s + 1, 6, ch); + if (n > 0 && s[n + 1] == '}' && *ch >= 0 && *ch <= 0x1fffff) { + s += n + 2; + } + else { + + *ch = 'u'; + } + } + else if ((n = parse_hex(s, 4, ch)) > 0) { + s += n; + } + break; + case 'U': + if ((n = parse_hex(s, 8, ch)) > 0) { + s += n; + } + break; + case 'x': + if ((n = parse_hex(s, 2, ch)) > 0) { + s += n; + } + break; + case '\0': + s--; + *ch = '\\'; + break; + } + return s - s0; +} + +static int regatom(regex_t *preg, int *flagp) +{ + int ret; + int flags; + int nocase = (preg->cflags & REG_ICASE); + + int ch; + int n = reg_utf8_tounicode_case(preg->regparse, &ch, nocase); + + *flagp = WORST; + + preg->regparse += n; + switch (ch) { + + case '^': + ret = regnode(preg, BOL); + break; + case '$': + ret = regnode(preg, EOL); + break; + case '.': + ret = regnode(preg, ANY); + *flagp |= HASWIDTH|SIMPLE; + break; + case '[': { + const char *pattern = preg->regparse; + + if (*pattern == '^') { + ret = regnode(preg, ANYBUT); + pattern++; + } else + ret = regnode(preg, ANYOF); + + + if (*pattern == ']' || *pattern == '-') { + reg_addrange(preg, *pattern, *pattern); + pattern++; + } + + while (*pattern != ']') { + + int start; + int end; + + enum { + CC_ALPHA, CC_ALNUM, CC_SPACE, CC_BLANK, CC_UPPER, CC_LOWER, + CC_DIGIT, CC_XDIGIT, CC_CNTRL, CC_GRAPH, CC_PRINT, CC_PUNCT, + CC_NUM + }; + int cc; + + if (!*pattern) { + preg->err = REG_ERR_UNMATCHED_BRACKET; + return 0; + } + + pattern += reg_utf8_tounicode_case(pattern, &start, nocase); + if (start == '\\') { + + switch (*pattern) { + case 's': + pattern++; + cc = CC_SPACE; + goto cc_switch; + case 'd': + pattern++; + cc = CC_DIGIT; + goto cc_switch; + case 'w': + pattern++; + reg_addrange(preg, '_', '_'); + cc = CC_ALNUM; + goto cc_switch; + } + pattern += reg_decode_escape(pattern, &start); + if (start == 0) { + preg->err = REG_ERR_NULL_CHAR; + return 0; + } + if (start == '\\' && *pattern == 0) { + preg->err = REG_ERR_INVALID_ESCAPE; + return 0; + } + } + if (pattern[0] == '-' && pattern[1] && pattern[1] != ']') { + + pattern += utf8_tounicode(pattern, &end); + pattern += reg_utf8_tounicode_case(pattern, &end, nocase); + if (end == '\\') { + pattern += reg_decode_escape(pattern, &end); + if (end == 0) { + preg->err = REG_ERR_NULL_CHAR; + return 0; + } + if (end == '\\' && *pattern == 0) { + preg->err = REG_ERR_INVALID_ESCAPE; + return 0; + } + } + + reg_addrange(preg, start, end); + continue; + } + if (start == '[' && pattern[0] == ':') { + static const char *character_class[] = { + ":alpha:", ":alnum:", ":space:", ":blank:", ":upper:", ":lower:", + ":digit:", ":xdigit:", ":cntrl:", ":graph:", ":print:", ":punct:", + }; + + for (cc = 0; cc < CC_NUM; cc++) { + n = strlen(character_class[cc]); + if (strncmp(pattern, character_class[cc], n) == 0) { + if (pattern[n] != ']') { + preg->err = REG_ERR_UNMATCHED_BRACKET; + return 0; + } + + pattern += n + 1; + break; + } + } + if (cc != CC_NUM) { +cc_switch: + switch (cc) { + case CC_ALNUM: + reg_addrange(preg, '0', '9'); + + case CC_ALPHA: + if ((preg->cflags & REG_ICASE) == 0) { + reg_addrange(preg, 'a', 'z'); + } + reg_addrange(preg, 'A', 'Z'); + break; + case CC_SPACE: + reg_addrange_str(preg, " \t\r\n\f\v"); + break; + case CC_BLANK: + reg_addrange_str(preg, " \t"); + break; + case CC_UPPER: + reg_addrange(preg, 'A', 'Z'); + break; + case CC_LOWER: + reg_addrange(preg, 'a', 'z'); + break; + case CC_XDIGIT: + reg_addrange(preg, 'a', 'f'); + reg_addrange(preg, 'A', 'F'); + + case CC_DIGIT: + reg_addrange(preg, '0', '9'); + break; + case CC_CNTRL: + reg_addrange(preg, 0, 31); + reg_addrange(preg, 127, 127); + break; + case CC_PRINT: + reg_addrange(preg, ' ', '~'); + break; + case CC_GRAPH: + reg_addrange(preg, '!', '~'); + break; + case CC_PUNCT: + reg_addrange(preg, '!', '/'); + reg_addrange(preg, ':', '@'); + reg_addrange(preg, '[', '`'); + reg_addrange(preg, '{', '~'); + break; + } + continue; + } + } + + reg_addrange(preg, start, start); + } + regc(preg, '\0'); + + if (*pattern) { + pattern++; + } + preg->regparse = pattern; + + *flagp |= HASWIDTH|SIMPLE; + } + break; + case '(': + ret = reg(preg, 1, &flags); + if (ret == 0) + return 0; + *flagp |= flags&(HASWIDTH|SPSTART); + break; + case '\0': + case '|': + case ')': + preg->err = REG_ERR_INTERNAL; + return 0; + case '?': + case '+': + case '*': + case '{': + preg->err = REG_ERR_COUNT_FOLLOWS_NOTHING; + return 0; + case '\\': + ch = *preg->regparse++; + switch (ch) { + case '\0': + preg->err = REG_ERR_INVALID_ESCAPE; + return 0; + case 'A': + ret = regnode(preg, BOLX); + break; + case 'Z': + ret = regnode(preg, EOLX); + break; + case '<': + case 'm': + ret = regnode(preg, WORDA); + break; + case '>': + case 'M': + ret = regnode(preg, WORDZ); + break; + case 'd': + case 'D': + ret = regnode(preg, ch == 'd' ? ANYOF : ANYBUT); + reg_addrange(preg, '0', '9'); + regc(preg, '\0'); + *flagp |= HASWIDTH|SIMPLE; + break; + case 'w': + case 'W': + ret = regnode(preg, ch == 'w' ? ANYOF : ANYBUT); + if ((preg->cflags & REG_ICASE) == 0) { + reg_addrange(preg, 'a', 'z'); + } + reg_addrange(preg, 'A', 'Z'); + reg_addrange(preg, '0', '9'); + reg_addrange(preg, '_', '_'); + regc(preg, '\0'); + *flagp |= HASWIDTH|SIMPLE; + break; + case 's': + case 'S': + ret = regnode(preg, ch == 's' ? ANYOF : ANYBUT); + reg_addrange_str(preg," \t\r\n\f\v"); + regc(preg, '\0'); + *flagp |= HASWIDTH|SIMPLE; + break; + + default: + + + preg->regparse--; + goto de_fault; + } + break; + de_fault: + default: { + int added = 0; + + + preg->regparse -= n; + + ret = regnode(preg, EXACTLY); + + + + while (*preg->regparse && strchr(META, *preg->regparse) == NULL) { + n = reg_utf8_tounicode_case(preg->regparse, &ch, (preg->cflags & REG_ICASE)); + if (ch == '\\' && preg->regparse[n]) { + if (strchr("<>mMwWdDsSAZ", preg->regparse[n])) { + + break; + } + n += reg_decode_escape(preg->regparse + n, &ch); + if (ch == 0) { + preg->err = REG_ERR_NULL_CHAR; + return 0; + } + } + + + if (ISMULT(preg->regparse[n])) { + + if (added) { + + break; + } + + regc(preg, ch); + added++; + preg->regparse += n; + break; + } + + + regc(preg, ch); + added++; + preg->regparse += n; + } + regc(preg, '\0'); + + *flagp |= HASWIDTH; + if (added == 1) + *flagp |= SIMPLE; + break; + } + break; + } + + return(ret); +} + +static void reg_grow(regex_t *preg, int n) +{ + if (preg->p + n >= preg->proglen) { + preg->proglen = (preg->p + n) * 2; + preg->program = realloc(preg->program, preg->proglen * sizeof(int)); + } +} + + +static int regnode(regex_t *preg, int op) +{ + reg_grow(preg, 2); + + + preg->program[preg->p++] = op; + preg->program[preg->p++] = 0; + + + return preg->p - 2; +} + +static void regc(regex_t *preg, int b ) +{ + reg_grow(preg, 1); + preg->program[preg->p++] = b; +} + +static int reginsert(regex_t *preg, int op, int size, int opnd ) +{ + reg_grow(preg, size); + + + memmove(preg->program + opnd + size, preg->program + opnd, sizeof(int) * (preg->p - opnd)); + + memset(preg->program + opnd, 0, sizeof(int) * size); + + preg->program[opnd] = op; + + preg->p += size; + + return opnd + size; +} + +static void regtail(regex_t *preg, int p, int val) +{ + int scan; + int temp; + int offset; + + + scan = p; + for (;;) { + temp = regnext(preg, scan); + if (temp == 0) + break; + scan = temp; + } + + if (OP(preg, scan) == BACK) + offset = scan - val; + else + offset = val - scan; + + preg->program[scan + 1] = offset; +} + + +static void regoptail(regex_t *preg, int p, int val ) +{ + + if (p != 0 && OP(preg, p) == BRANCH) { + regtail(preg, OPERAND(p), val); + } +} + + +static int regtry(regex_t *preg, const char *string ); +static int regmatch(regex_t *preg, int prog); +static int regrepeat(regex_t *preg, int p, int max); + +int jim_regexec(regex_t *preg, const char *string, size_t nmatch, regmatch_t pmatch[], int eflags) +{ + const char *s; + int scan; + + + if (preg == NULL || preg->program == NULL || string == NULL) { + return REG_ERR_NULL_ARGUMENT; + } + + + if (*preg->program != REG_MAGIC) { + return REG_ERR_CORRUPTED; + } + +#ifdef DEBUG + fprintf(stderr, "regexec: %s\n", string); + regdump(preg); +#endif + + preg->eflags = eflags; + preg->pmatch = pmatch; + preg->nmatch = nmatch; + preg->start = string; + + + for (scan = OPERAND(1); scan != 0; scan += regopsize(preg, scan)) { + int op = OP(preg, scan); + if (op == END) + break; + if (op == REPX || op == REPXMIN) + preg->program[scan + 4] = 0; + } + + + if (preg->regmust != 0) { + s = string; + while ((s = str_find(s, preg->program[preg->regmust], preg->cflags & REG_ICASE)) != NULL) { + if (prefix_cmp(preg->program + preg->regmust, preg->regmlen, s, preg->cflags & REG_ICASE) >= 0) { + break; + } + s++; + } + if (s == NULL) + return REG_NOMATCH; + } + + + preg->regbol = string; + + + if (preg->reganch) { + if (eflags & REG_NOTBOL) { + + goto nextline; + } + while (1) { + if (regtry(preg, string)) { + return REG_NOERROR; + } + if (*string) { +nextline: + if (preg->cflags & REG_NEWLINE) { + + string = strchr(string, '\n'); + if (string) { + preg->regbol = ++string; + continue; + } + } + } + return REG_NOMATCH; + } + } + + + s = string; + if (preg->regstart != '\0') { + + while ((s = str_find(s, preg->regstart, preg->cflags & REG_ICASE)) != NULL) { + if (regtry(preg, s)) + return REG_NOERROR; + s++; + } + } + else + + while (1) { + if (regtry(preg, s)) + return REG_NOERROR; + if (*s == '\0') { + break; + } + else { + int c; + s += utf8_tounicode(s, &c); + } + } + + + return REG_NOMATCH; +} + + +static int regtry( regex_t *preg, const char *string ) +{ + int i; + + preg->reginput = string; + + for (i = 0; i < preg->nmatch; i++) { + preg->pmatch[i].rm_so = -1; + preg->pmatch[i].rm_eo = -1; + } + if (regmatch(preg, 1)) { + preg->pmatch[0].rm_so = string - preg->start; + preg->pmatch[0].rm_eo = preg->reginput - preg->start; + return(1); + } else + return(0); +} + +static int prefix_cmp(const int *prog, int proglen, const char *string, int nocase) +{ + const char *s = string; + while (proglen && *s) { + int ch; + int n = reg_utf8_tounicode_case(s, &ch, nocase); + if (ch != *prog) { + return -1; + } + prog++; + s += n; + proglen--; + } + if (proglen == 0) { + return s - string; + } + return -1; +} + +static int reg_range_find(const int *range, int c) +{ + while (*range) { + + if (c >= range[1] && c <= (range[0] + range[1] - 1)) { + return 1; + } + range += 2; + } + return 0; +} + +static const char *str_find(const char *string, int c, int nocase) +{ + if (nocase) { + + c = utf8_upper(c); + } + while (*string) { + int ch; + int n = reg_utf8_tounicode_case(string, &ch, nocase); + if (c == ch) { + return string; + } + string += n; + } + return NULL; +} + +static int reg_iseol(regex_t *preg, int ch) +{ + if (preg->cflags & REG_NEWLINE) { + return ch == '\0' || ch == '\n'; + } + else { + return ch == '\0'; + } +} + +static int regmatchsimplerepeat(regex_t *preg, int scan, int matchmin) +{ + int nextch = '\0'; + const char *save; + int no; + int c; + + int max = preg->program[scan + 2]; + int min = preg->program[scan + 3]; + int next = regnext(preg, scan); + + if (OP(preg, next) == EXACTLY) { + nextch = preg->program[OPERAND(next)]; + } + save = preg->reginput; + no = regrepeat(preg, scan + 5, max); + if (no < min) { + return 0; + } + if (matchmin) { + + max = no; + no = min; + } + + while (1) { + if (matchmin) { + if (no > max) { + break; + } + } + else { + if (no < min) { + break; + } + } + preg->reginput = save + utf8_index(save, no); + reg_utf8_tounicode_case(preg->reginput, &c, (preg->cflags & REG_ICASE)); + + if (reg_iseol(preg, nextch) || c == nextch) { + if (regmatch(preg, next)) { + return(1); + } + } + if (matchmin) { + + no++; + } + else { + + no--; + } + } + return(0); +} + +static int regmatchrepeat(regex_t *preg, int scan, int matchmin) +{ + int *scanpt = preg->program + scan; + + int max = scanpt[2]; + int min = scanpt[3]; + + + if (scanpt[4] < min) { + + scanpt[4]++; + if (regmatch(preg, scan + 5)) { + return 1; + } + scanpt[4]--; + return 0; + } + if (scanpt[4] > max) { + return 0; + } + + if (matchmin) { + + if (regmatch(preg, regnext(preg, scan))) { + return 1; + } + + scanpt[4]++; + if (regmatch(preg, scan + 5)) { + return 1; + } + scanpt[4]--; + return 0; + } + + if (scanpt[4] < max) { + scanpt[4]++; + if (regmatch(preg, scan + 5)) { + return 1; + } + scanpt[4]--; + } + + return regmatch(preg, regnext(preg, scan)); +} + + +static int regmatch(regex_t *preg, int prog) +{ + int scan; + int next; + const char *save; + + scan = prog; + +#ifdef DEBUG + if (scan != 0 && regnarrate) + fprintf(stderr, "%s(\n", regprop(scan)); +#endif + while (scan != 0) { + int n; + int c; +#ifdef DEBUG + if (regnarrate) { + fprintf(stderr, "%3d: %s...\n", scan, regprop(OP(preg, scan))); + } +#endif + next = regnext(preg, scan); + n = reg_utf8_tounicode_case(preg->reginput, &c, (preg->cflags & REG_ICASE)); + + switch (OP(preg, scan)) { + case BOLX: + if ((preg->eflags & REG_NOTBOL)) { + return(0); + } + + case BOL: + if (preg->reginput != preg->regbol) { + return(0); + } + break; + case EOLX: + if (c != 0) { + + return 0; + } + break; + case EOL: + if (!reg_iseol(preg, c)) { + return(0); + } + break; + case WORDA: + + if ((!isalnum(UCHAR(c))) && c != '_') + return(0); + + if (preg->reginput > preg->regbol && + (isalnum(UCHAR(preg->reginput[-1])) || preg->reginput[-1] == '_')) + return(0); + break; + case WORDZ: + + if (preg->reginput > preg->regbol) { + + if (reg_iseol(preg, c) || !(isalnum(UCHAR(c)) || c == '_')) { + c = preg->reginput[-1]; + + if (isalnum(UCHAR(c)) || c == '_') { + break; + } + } + } + + return(0); + + case ANY: + if (reg_iseol(preg, c)) + return 0; + preg->reginput += n; + break; + case EXACTLY: { + int opnd; + int len; + int slen; + + opnd = OPERAND(scan); + len = str_int_len(preg->program + opnd); + + slen = prefix_cmp(preg->program + opnd, len, preg->reginput, preg->cflags & REG_ICASE); + if (slen < 0) { + return(0); + } + preg->reginput += slen; + } + break; + case ANYOF: + if (reg_iseol(preg, c) || reg_range_find(preg->program + OPERAND(scan), c) == 0) { + return(0); + } + preg->reginput += n; + break; + case ANYBUT: + if (reg_iseol(preg, c) || reg_range_find(preg->program + OPERAND(scan), c) != 0) { + return(0); + } + preg->reginput += n; + break; + case NOTHING: + break; + case BACK: + break; + case BRANCH: + if (OP(preg, next) != BRANCH) + next = OPERAND(scan); + else { + do { + save = preg->reginput; + if (regmatch(preg, OPERAND(scan))) { + return(1); + } + preg->reginput = save; + scan = regnext(preg, scan); + } while (scan != 0 && OP(preg, scan) == BRANCH); + return(0); + + } + break; + case REP: + case REPMIN: + return regmatchsimplerepeat(preg, scan, OP(preg, scan) == REPMIN); + + case REPX: + case REPXMIN: + return regmatchrepeat(preg, scan, OP(preg, scan) == REPXMIN); + + case END: + return 1; + + case OPENNC: + case CLOSENC: + return regmatch(preg, next); + + default: + if (OP(preg, scan) >= OPEN+1 && OP(preg, scan) < CLOSE_END) { + save = preg->reginput; + if (regmatch(preg, next)) { + if (OP(preg, scan) < CLOSE) { + int no = OP(preg, scan) - OPEN; + if (no < preg->nmatch && preg->pmatch[no].rm_so == -1) { + preg->pmatch[no].rm_so = save - preg->start; + } + } + else { + int no = OP(preg, scan) - CLOSE; + if (no < preg->nmatch && preg->pmatch[no].rm_eo == -1) { + preg->pmatch[no].rm_eo = save - preg->start; + } + } + return(1); + } + + preg->reginput = save; + return(0); + } + return REG_ERR_INTERNAL; + } + + scan = next; + } + + return REG_ERR_INTERNAL; +} + +static int regrepeat(regex_t *preg, int p, int max) +{ + int count = 0; + const char *scan; + int opnd; + int ch; + int n; + + scan = preg->reginput; + opnd = OPERAND(p); + switch (OP(preg, p)) { + case ANY: + while (!reg_iseol(preg, *scan) && count < max) { + count++; + scan += utf8_charlen(*scan); + } + break; + case EXACTLY: + while (count < max) { + n = reg_utf8_tounicode_case(scan, &ch, preg->cflags & REG_ICASE); + if (preg->program[opnd] != ch) { + break; + } + count++; + scan += n; + } + break; + case ANYOF: + while (count < max) { + n = reg_utf8_tounicode_case(scan, &ch, preg->cflags & REG_ICASE); + if (reg_iseol(preg, ch) || reg_range_find(preg->program + opnd, ch) == 0) { + break; + } + count++; + scan += n; + } + break; + case ANYBUT: + while (count < max) { + n = reg_utf8_tounicode_case(scan, &ch, preg->cflags & REG_ICASE); + if (reg_iseol(preg, ch) || reg_range_find(preg->program + opnd, ch) != 0) { + break; + } + count++; + scan += n; + } + break; + default: + preg->err = REG_ERR_INTERNAL; + count = 0; + break; + } + preg->reginput = scan; + + return(count); +} + +static int regnext(regex_t *preg, int p ) +{ + int offset; + + offset = NEXT(preg, p); + + if (offset == 0) + return 0; + + if (OP(preg, p) == BACK) + return(p-offset); + else + return(p+offset); +} + +static int regopsize(regex_t *preg, int p ) +{ + + switch (OP(preg, p)) { + case REP: + case REPMIN: + case REPX: + case REPXMIN: + return 5; + + case ANYOF: + case ANYBUT: + case EXACTLY: { + int s = p + 2; + while (preg->program[s++]) { + } + return s - p; + } + } + return 2; +} + + +size_t jim_regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size) +{ + static const char *error_strings[] = { + "success", + "no match", + "bad pattern", + "null argument", + "unknown error", + "too big", + "out of memory", + "too many ()", + "parentheses () not balanced", + "braces {} not balanced", + "invalid repetition count(s)", + "extra characters", + "*+ of empty atom", + "nested count", + "internal error", + "count follows nothing", + "invalid escape \\ sequence", + "corrupted program", + "contains null char", + "brackets [] not balanced", + }; + const char *err; + + if (errcode < 0 || errcode >= REG_ERR_NUM) { + err = "Bad error code"; + } + else { + err = error_strings[errcode]; + } + + return snprintf(errbuf, errbuf_size, "%s", err); +} + +void jim_regfree(regex_t *preg) +{ + free(preg->program); +} + +#endif +#include + +void Jim_SetResultErrno(Jim_Interp *interp, const char *msg) +{ + Jim_SetResultFormatted(interp, "%s: %s", msg, strerror(Jim_Errno())); +} + +#if defined(_WIN32) || defined(WIN32) +#include + +int Jim_Errno(void) +{ + switch (GetLastError()) { + case ERROR_FILE_NOT_FOUND: return ENOENT; + case ERROR_PATH_NOT_FOUND: return ENOENT; + case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; + case ERROR_ACCESS_DENIED: return EACCES; + case ERROR_INVALID_HANDLE: return EBADF; + case ERROR_BAD_ENVIRONMENT: return E2BIG; + case ERROR_BAD_FORMAT: return ENOEXEC; + case ERROR_INVALID_ACCESS: return EACCES; + case ERROR_INVALID_DRIVE: return ENOENT; + case ERROR_CURRENT_DIRECTORY: return EACCES; + case ERROR_NOT_SAME_DEVICE: return EXDEV; + case ERROR_NO_MORE_FILES: return ENOENT; + case ERROR_WRITE_PROTECT: return EROFS; + case ERROR_BAD_UNIT: return ENXIO; + case ERROR_NOT_READY: return EBUSY; + case ERROR_BAD_COMMAND: return EIO; + case ERROR_CRC: return EIO; + case ERROR_BAD_LENGTH: return EIO; + case ERROR_SEEK: return EIO; + case ERROR_WRITE_FAULT: return EIO; + case ERROR_READ_FAULT: return EIO; + case ERROR_GEN_FAILURE: return EIO; + case ERROR_SHARING_VIOLATION: return EACCES; + case ERROR_LOCK_VIOLATION: return EACCES; + case ERROR_SHARING_BUFFER_EXCEEDED: return ENFILE; + case ERROR_HANDLE_DISK_FULL: return ENOSPC; + case ERROR_NOT_SUPPORTED: return ENODEV; + case ERROR_REM_NOT_LIST: return EBUSY; + case ERROR_DUP_NAME: return EEXIST; + case ERROR_BAD_NETPATH: return ENOENT; + case ERROR_NETWORK_BUSY: return EBUSY; + case ERROR_DEV_NOT_EXIST: return ENODEV; + case ERROR_TOO_MANY_CMDS: return EAGAIN; + case ERROR_ADAP_HDW_ERR: return EIO; + case ERROR_BAD_NET_RESP: return EIO; + case ERROR_UNEXP_NET_ERR: return EIO; + case ERROR_NETNAME_DELETED: return ENOENT; + case ERROR_NETWORK_ACCESS_DENIED: return EACCES; + case ERROR_BAD_DEV_TYPE: return ENODEV; + case ERROR_BAD_NET_NAME: return ENOENT; + case ERROR_TOO_MANY_NAMES: return ENFILE; + case ERROR_TOO_MANY_SESS: return EIO; + case ERROR_SHARING_PAUSED: return EAGAIN; + case ERROR_REDIR_PAUSED: return EAGAIN; + case ERROR_FILE_EXISTS: return EEXIST; + case ERROR_CANNOT_MAKE: return ENOSPC; + case ERROR_OUT_OF_STRUCTURES: return ENFILE; + case ERROR_ALREADY_ASSIGNED: return EEXIST; + case ERROR_INVALID_PASSWORD: return EPERM; + case ERROR_NET_WRITE_FAULT: return EIO; + case ERROR_NO_PROC_SLOTS: return EAGAIN; + case ERROR_DISK_CHANGE: return EXDEV; + case ERROR_BROKEN_PIPE: return EPIPE; + case ERROR_OPEN_FAILED: return ENOENT; + case ERROR_DISK_FULL: return ENOSPC; + case ERROR_NO_MORE_SEARCH_HANDLES: return EMFILE; + case ERROR_INVALID_TARGET_HANDLE: return EBADF; + case ERROR_INVALID_NAME: return ENOENT; + case ERROR_PROC_NOT_FOUND: return ESRCH; + case ERROR_WAIT_NO_CHILDREN: return ECHILD; + case ERROR_CHILD_NOT_COMPLETE: return ECHILD; + case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; + case ERROR_SEEK_ON_DEVICE: return ESPIPE; + case ERROR_BUSY_DRIVE: return EAGAIN; + case ERROR_DIR_NOT_EMPTY: return EEXIST; + case ERROR_NOT_LOCKED: return EACCES; + case ERROR_BAD_PATHNAME: return ENOENT; + case ERROR_LOCK_FAILED: return EACCES; + case ERROR_ALREADY_EXISTS: return EEXIST; + case ERROR_FILENAME_EXCED_RANGE: return ENAMETOOLONG; + case ERROR_BAD_PIPE: return EPIPE; + case ERROR_PIPE_BUSY: return EAGAIN; + case ERROR_PIPE_NOT_CONNECTED: return EPIPE; + case ERROR_DIRECTORY: return ENOTDIR; + } + return EINVAL; +} + +long JimProcessPid(phandle_t pid) +{ + if (pid == INVALID_HANDLE_VALUE) { + return -1; + } + return GetProcessId(pid); +} + +phandle_t JimWaitPid(long pid, int *status, int nohang) +{ + if (pid > 0) { + HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, pid); + if (h) { + long pid = waitpid(h, status, nohang); + CloseHandle(h); + if (pid > 0) { + return h; + } + } + } + return JIM_BAD_PHANDLE; +} + +long waitpid(phandle_t phandle, int *status, int nohang) +{ + long pid; + DWORD ret = WaitForSingleObject(phandle, nohang ? 0 : INFINITE); + if (ret == WAIT_TIMEOUT || ret == WAIT_FAILED) { + + return -1; + } + GetExitCodeProcess(phandle, &ret); + *status = ret; + + pid = GetProcessId(phandle); + CloseHandle(phandle); + return pid; +} + +int Jim_MakeTempFile(Jim_Interp *interp, const char *filename_template, int unlink_file) +{ + char name[MAX_PATH]; + HANDLE handle; + + if (!GetTempPath(MAX_PATH, name) || !GetTempFileName(name, filename_template ? filename_template : "JIM", 0, name)) { + return -1; + } + + handle = CreateFile(name, GENERIC_READ | GENERIC_WRITE, 0, NULL, + CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY | (unlink_file ? FILE_FLAG_DELETE_ON_CLOSE : 0), + NULL); + + if (handle == INVALID_HANDLE_VALUE) { + goto error; + } + + Jim_SetResultString(interp, name, -1); + return _open_osfhandle((intptr_t)handle, _O_RDWR | _O_TEXT); + + error: + Jim_SetResultErrno(interp, name); + DeleteFile(name); + return -1; +} + +int Jim_OpenForWrite(const char *filename, int append) +{ + if (strcmp(filename, "/dev/null") == 0) { + filename = "nul:"; + } + int fd = _open(filename, _O_WRONLY | _O_CREAT | _O_TEXT | (append ? _O_APPEND : _O_TRUNC), _S_IREAD | _S_IWRITE); + if (fd >= 0 && append) { + + _lseek(fd, 0L, SEEK_END); + } + return fd; +} + +int Jim_OpenForRead(const char *filename) +{ + if (strcmp(filename, "/dev/null") == 0) { + filename = "nul:"; + } + return _open(filename, _O_RDONLY | _O_TEXT, 0); +} + +#elif defined(HAVE_UNISTD_H) + + + +int Jim_MakeTempFile(Jim_Interp *interp, const char *filename_template, int unlink_file) +{ + int fd; + mode_t mask; + Jim_Obj *filenameObj; + + if (filename_template == NULL) { + const char *tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL || *tmpdir == '\0' || access(tmpdir, W_OK) != 0) { + tmpdir = "/tmp/"; + } + filenameObj = Jim_NewStringObj(interp, tmpdir, -1); + if (tmpdir[0] && tmpdir[strlen(tmpdir) - 1] != '/') { + Jim_AppendString(interp, filenameObj, "/", 1); + } + Jim_AppendString(interp, filenameObj, "tcl.tmp.XXXXXX", -1); + } + else { + filenameObj = Jim_NewStringObj(interp, filename_template, -1); + } + + +#ifdef HAVE_UMASK + mask = umask(S_IXUSR | S_IRWXG | S_IRWXO); +#endif +#ifdef HAVE_MKSTEMP + fd = mkstemp(filenameObj->bytes); +#else + if (mktemp(filenameObj->bytes) == NULL) { + fd = -1; + } + else { + fd = open(filenameObj->bytes, O_RDWR | O_CREAT | O_TRUNC); + } +#endif +#ifdef HAVE_UMASK + umask(mask); +#endif + if (fd < 0) { + Jim_SetResultErrno(interp, Jim_String(filenameObj)); + Jim_FreeNewObj(interp, filenameObj); + return -1; + } + if (unlink_file) { + remove(Jim_String(filenameObj)); + } + + Jim_SetResult(interp, filenameObj); + return fd; +} + +int Jim_OpenForWrite(const char *filename, int append) +{ + return open(filename, O_WRONLY | O_CREAT | (append ? O_APPEND : O_TRUNC), 0666); +} + +int Jim_OpenForRead(const char *filename) +{ + return open(filename, O_RDONLY, 0); +} + +#endif + +#if defined(_WIN32) || defined(WIN32) +#ifndef STRICT +#define STRICT +#endif +#define WIN32_LEAN_AND_MEAN +#include + +#if defined(HAVE_DLOPEN_COMPAT) +void *dlopen(const char *path, int mode) +{ + JIM_NOTUSED(mode); + + return (void *)LoadLibraryA(path); +} + +int dlclose(void *handle) +{ + FreeLibrary((HANDLE)handle); + return 0; +} + +void *dlsym(void *handle, const char *symbol) +{ + return GetProcAddress((HMODULE)handle, symbol); +} + +char *dlerror(void) +{ + static char msg[121]; + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), + LANG_NEUTRAL, msg, sizeof(msg) - 1, NULL); + return msg; +} +#endif + +#ifdef _MSC_VER + +#include + + +int gettimeofday(struct timeval *tv, void *unused) +{ + struct _timeb tb; + + _ftime(&tb); + tv->tv_sec = tb.time; + tv->tv_usec = tb.millitm * 1000; + + return 0; +} + + +DIR *opendir(const char *name) +{ + DIR *dir = 0; + + if (name && name[0]) { + size_t base_length = strlen(name); + const char *all = + strchr("/\\", name[base_length - 1]) ? "*" : "/*"; + + if ((dir = (DIR *) Jim_Alloc(sizeof *dir)) != 0 && + (dir->name = (char *)Jim_Alloc(base_length + strlen(all) + 1)) != 0) { + strcat(strcpy(dir->name, name), all); + + if ((dir->handle = (long)_findfirst(dir->name, &dir->info)) != -1) + dir->result.d_name = 0; + else { + Jim_Free(dir->name); + Jim_Free(dir); + dir = 0; + } + } + else { + Jim_Free(dir); + dir = 0; + errno = ENOMEM; + } + } + else { + errno = EINVAL; + } + return dir; +} + +int closedir(DIR * dir) +{ + int result = -1; + + if (dir) { + if (dir->handle != -1) + result = _findclose(dir->handle); + Jim_Free(dir->name); + Jim_Free(dir); + } + if (result == -1) + errno = EBADF; + return result; +} + +struct dirent *readdir(DIR * dir) +{ + struct dirent *result = 0; + + if (dir && dir->handle != -1) { + if (!dir->result.d_name || _findnext(dir->handle, &dir->info) != -1) { + result = &dir->result; + result->d_name = dir->info.name; + } + } + else { + errno = EBADF; + } + return result; +} +#endif +#endif +#include +#include + + + + + + +#ifndef SIGPIPE +#define SIGPIPE 13 +#endif +#ifndef SIGINT +#define SIGINT 2 +#endif + +const char *Jim_SignalId(int sig) +{ + static char buf[10]; + switch (sig) { + case SIGINT: return "SIGINT"; + case SIGPIPE: return "SIGPIPE"; + + } + snprintf(buf, sizeof(buf), "%d", sig); + return buf; +} +#ifndef JIM_BOOTSTRAP_LIB_ONLY +#include +#include +#include + + +#ifdef USE_LINENOISE +#ifdef HAVE_UNISTD_H + #include +#endif +#ifdef HAVE_SYS_STAT_H + #include +#endif +#include "linenoise.h" +#else +#define MAX_LINE_LEN 512 +#endif + +#ifdef USE_LINENOISE +struct JimCompletionInfo { + Jim_Interp *interp; + Jim_Obj *completion_command; + Jim_Obj *hints_command; + +}; + +static struct JimCompletionInfo *JimGetCompletionInfo(Jim_Interp *interp); +static void JimCompletionCallback(const char *prefix, linenoiseCompletions *comp, void *userdata); +static const char completion_callback_assoc_key[] = "interactive-completion"; +static char *JimHintsCallback(const char *prefix, int *color, int *bold, void *userdata); +static void JimFreeHintsCallback(void *hint, void *userdata); +#endif + +char *Jim_HistoryGetline(Jim_Interp *interp, const char *prompt) +{ +#ifdef USE_LINENOISE + struct JimCompletionInfo *compinfo = JimGetCompletionInfo(interp); + char *result; + Jim_Obj *objPtr; + long mlmode = 0; + if (compinfo->completion_command) { + linenoiseSetCompletionCallback(JimCompletionCallback, compinfo); + } + if (compinfo->hints_command) { + linenoiseSetHintsCallback(JimHintsCallback, compinfo); + linenoiseSetFreeHintsCallback(JimFreeHintsCallback); + } + objPtr = Jim_GetVariableStr(interp, "history::multiline", JIM_NONE); + if (objPtr && Jim_GetLong(interp, objPtr, &mlmode) == JIM_NONE) { + linenoiseSetMultiLine(mlmode); + } + + result = linenoise(prompt); + + linenoiseSetCompletionCallback(NULL, NULL); + linenoiseSetHintsCallback(NULL, NULL); + linenoiseSetFreeHintsCallback(NULL); + return result; +#else + int len; + char *line = Jim_Alloc(MAX_LINE_LEN); + + fputs(prompt, stdout); + fflush(stdout); + + if (fgets(line, MAX_LINE_LEN, stdin) == NULL) { + Jim_Free(line); + return NULL; + } + len = strlen(line); + if (len && line[len - 1] == '\n') { + line[len - 1] = '\0'; + } + return line; +#endif +} + +void Jim_HistoryLoad(const char *filename) +{ +#ifdef USE_LINENOISE + linenoiseHistoryLoad(filename); +#endif +} + +void Jim_HistoryAdd(const char *line) +{ +#ifdef USE_LINENOISE + linenoiseHistoryAdd(line); +#endif +} + +void Jim_HistorySave(const char *filename) +{ +#ifdef USE_LINENOISE +#ifdef HAVE_UMASK + mode_t mask; + + mask = umask(S_IXUSR | S_IRWXG | S_IRWXO); +#endif + linenoiseHistorySave(filename); +#ifdef HAVE_UMASK + umask(mask); +#endif +#endif +} + +void Jim_HistoryShow(void) +{ +#ifdef USE_LINENOISE + + int i; + int len; + char **history = linenoiseHistory(&len); + for (i = 0; i < len; i++) { + printf("%4d %s\n", i + 1, history[i]); + } +#endif +} + +void Jim_HistorySetMaxLen(int length) +{ +#ifdef USE_LINENOISE + linenoiseHistorySetMaxLen(length); +#endif +} + +int Jim_HistoryGetMaxLen(void) +{ +#ifdef USE_LINENOISE + return linenoiseHistoryGetMaxLen(); +#endif + return 0; +} + +#ifdef USE_LINENOISE +static void JimCompletionCallback(const char *prefix, linenoiseCompletions *comp, void *userdata) +{ + struct JimCompletionInfo *info = (struct JimCompletionInfo *)userdata; + Jim_Obj *objv[2]; + int ret; + + objv[0] = info->completion_command; + objv[1] = Jim_NewStringObj(info->interp, prefix, -1); + + ret = Jim_EvalObjVector(info->interp, 2, objv); + + + if (ret == JIM_OK) { + int i; + Jim_Obj *listObj = Jim_GetResult(info->interp); + int len = Jim_ListLength(info->interp, listObj); + for (i = 0; i < len; i++) { + linenoiseAddCompletion(comp, Jim_String(Jim_ListGetIndex(info->interp, listObj, i))); + } + } +} + +static char *JimHintsCallback(const char *prefix, int *color, int *bold, void *userdata) +{ + struct JimCompletionInfo *info = (struct JimCompletionInfo *)userdata; + Jim_Obj *objv[2]; + int ret; + char *result = NULL; + + objv[0] = info->hints_command; + objv[1] = Jim_NewStringObj(info->interp, prefix, -1); + + ret = Jim_EvalObjVector(info->interp, 2, objv); + + + if (ret == JIM_OK) { + Jim_Obj *listObj = Jim_GetResult(info->interp); + Jim_IncrRefCount(listObj); + + int len = Jim_ListLength(info->interp, listObj); + if (len >= 1) { + long x; + result = Jim_StrDup(Jim_String(Jim_ListGetIndex(info->interp, listObj, 0))); + if (len >= 2 && Jim_GetLong(info->interp, Jim_ListGetIndex(info->interp, listObj, 1), &x) == JIM_OK) { + *color = x; + } + if (len >= 3 && Jim_GetLong(info->interp, Jim_ListGetIndex(info->interp, listObj, 2), &x) == JIM_OK) { + *bold = x; + } + } + Jim_DecrRefCount(info->interp, listObj); + } + return result; +} + +static void JimFreeHintsCallback(void *hint, void *userdata) +{ + Jim_Free(hint); +} + +static void JimHistoryFreeCompletion(Jim_Interp *interp, void *data) +{ + struct JimCompletionInfo *compinfo = data; + + if (compinfo->completion_command) { + Jim_DecrRefCount(interp, compinfo->completion_command); + } + if (compinfo->hints_command) { + Jim_DecrRefCount(interp, compinfo->hints_command); + } + + Jim_Free(compinfo); +} + +static struct JimCompletionInfo *JimGetCompletionInfo(Jim_Interp *interp) +{ + struct JimCompletionInfo *compinfo = Jim_GetAssocData(interp, completion_callback_assoc_key); + if (compinfo == NULL) { + compinfo = Jim_Alloc(sizeof(*compinfo)); + compinfo->interp = interp; + compinfo->completion_command = NULL; + compinfo->hints_command = NULL; + Jim_SetAssocData(interp, completion_callback_assoc_key, JimHistoryFreeCompletion, compinfo); + } + return compinfo; +} +#endif + +void Jim_HistorySetCompletion(Jim_Interp *interp, Jim_Obj *completionCommandObj) +{ +#ifdef USE_LINENOISE + struct JimCompletionInfo *compinfo = JimGetCompletionInfo(interp); + + if (completionCommandObj) { + + Jim_IncrRefCount(completionCommandObj); + } + if (compinfo->completion_command) { + Jim_DecrRefCount(interp, compinfo->completion_command); + } + compinfo->completion_command = completionCommandObj; +#endif +} + +void Jim_HistorySetHints(Jim_Interp *interp, Jim_Obj *hintsCommandObj) +{ +#ifdef USE_LINENOISE + struct JimCompletionInfo *compinfo = JimGetCompletionInfo(interp); + + if (hintsCommandObj) { + + Jim_IncrRefCount(hintsCommandObj); + } + if (compinfo->hints_command) { + Jim_DecrRefCount(interp, compinfo->hints_command); + } + compinfo->hints_command = hintsCommandObj; +#endif +} + +int Jim_InteractivePrompt(Jim_Interp *interp) +{ + int retcode = JIM_OK; + char *history_file = NULL; +#ifdef USE_LINENOISE + const char *home; + + home = getenv("HOME"); + if (home && isatty(STDIN_FILENO)) { + int history_len = strlen(home) + sizeof("/.jim_history"); + history_file = Jim_Alloc(history_len); + snprintf(history_file, history_len, "%s/.jim_history", home); + Jim_HistoryLoad(history_file); + } + + Jim_HistorySetCompletion(interp, Jim_NewStringObj(interp, "tcl::autocomplete", -1)); + Jim_HistorySetHints(interp, Jim_NewStringObj(interp, "tcl::stdhint", -1)); +#endif + + printf("Welcome to Jim version %d.%d\n", + JIM_VERSION / 100, JIM_VERSION % 100); + Jim_SetVariableStrWithStr(interp, JIM_INTERACTIVE, "1"); + + while (1) { + Jim_Obj *scriptObjPtr; + const char *result; + int reslen; + char prompt[20]; + + if (retcode != JIM_OK) { + const char *retcodestr = Jim_ReturnCode(retcode); + + if (*retcodestr == '?') { + snprintf(prompt, sizeof(prompt) - 3, "[%d] . ", retcode); + } + else { + snprintf(prompt, sizeof(prompt) - 3, "[%s] . ", retcodestr); + } + } + else { + strcpy(prompt, ". "); + } + + scriptObjPtr = Jim_NewStringObj(interp, "", 0); + Jim_IncrRefCount(scriptObjPtr); + while (1) { + char state; + char *line; + + line = Jim_HistoryGetline(interp, prompt); + if (line == NULL) { + if (errno == EINTR) { + continue; + } + Jim_DecrRefCount(interp, scriptObjPtr); + retcode = JIM_OK; + goto out; + } + if (Jim_Length(scriptObjPtr) != 0) { + + Jim_AppendString(interp, scriptObjPtr, "\n", 1); + } + Jim_AppendString(interp, scriptObjPtr, line, -1); + Jim_Free(line); + if (Jim_ScriptIsComplete(interp, scriptObjPtr, &state)) + break; + + snprintf(prompt, sizeof(prompt), "%c> ", state); + } +#ifdef USE_LINENOISE + if (strcmp(Jim_String(scriptObjPtr), "h") == 0) { + + Jim_HistoryShow(); + Jim_DecrRefCount(interp, scriptObjPtr); + continue; + } + + Jim_HistoryAdd(Jim_String(scriptObjPtr)); + if (history_file) { + Jim_HistorySave(history_file); + } +#endif + retcode = Jim_EvalObj(interp, scriptObjPtr); + Jim_DecrRefCount(interp, scriptObjPtr); + + if (retcode == JIM_EXIT) { + break; + } + if (retcode == JIM_ERR) { + Jim_MakeErrorMessage(interp); + } + result = Jim_GetString(Jim_GetResult(interp), &reslen); + if (reslen) { + if (fwrite(result, reslen, 1, stdout) == 0) { + + } + putchar('\n'); + } + } + out: + Jim_Free(history_file); + + return retcode; +} + +#include +#include +#include + + + +extern int Jim_initjimshInit(Jim_Interp *interp); + +static void JimSetArgv(Jim_Interp *interp, int argc, char *const argv[]) +{ + int n; + Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); + + + for (n = 0; n < argc; n++) { + Jim_Obj *obj = Jim_NewStringObj(interp, argv[n], -1); + + Jim_ListAppendElement(interp, listObj, obj); + } + + Jim_SetVariableStr(interp, "argv", listObj); + Jim_SetVariableStr(interp, "argc", Jim_NewIntObj(interp, argc)); +} + +static void JimPrintErrorMessage(Jim_Interp *interp) +{ + Jim_MakeErrorMessage(interp); + fprintf(stderr, "%s\n", Jim_String(Jim_GetResult(interp))); +} + +void usage(const char* executable_name) +{ + printf("jimsh version %d.%d\n", JIM_VERSION / 100, JIM_VERSION % 100); + printf("Usage: %s\n", executable_name); + printf("or : %s [options] [filename]\n", executable_name); + printf("\n"); + printf("Without options: Interactive mode\n"); + printf("\n"); + printf("Options:\n"); + printf(" --version : prints the version string\n"); + printf(" --help : prints this text\n"); + printf(" -e CMD : executes command CMD\n"); + printf(" NOTE: all subsequent options will be passed as arguments to the command\n"); + printf(" [filename|-] : executes the script contained in the named file, or from stdin if \"-\"\n"); + printf(" NOTE: all subsequent options will be passed to the script\n\n"); +} + +int main(int argc, char *const argv[]) +{ + int retcode; + Jim_Interp *interp; + char *const orig_argv0 = argv[0]; + + + if (argc > 1 && strcmp(argv[1], "--version") == 0) { + printf("%d.%d\n", JIM_VERSION / 100, JIM_VERSION % 100); + return 0; + } + else if (argc > 1 && strcmp(argv[1], "--help") == 0) { + usage(argv[0]); + return 0; + } + + + interp = Jim_CreateInterp(); + Jim_RegisterCoreCommands(interp); + + + if (Jim_InitStaticExtensions(interp) != JIM_OK) { + JimPrintErrorMessage(interp); + } + + Jim_SetVariableStrWithStr(interp, "jim::argv0", orig_argv0); + Jim_SetVariableStrWithStr(interp, JIM_INTERACTIVE, argc == 1 ? "1" : "0"); +#ifdef USE_LINENOISE + Jim_SetVariableStrWithStr(interp, "jim::lineedit", "1"); +#else + Jim_SetVariableStrWithStr(interp, "jim::lineedit", "0"); +#endif + retcode = Jim_initjimshInit(interp); + + if (argc == 1) { + + if (retcode == JIM_ERR) { + JimPrintErrorMessage(interp); + } + if (retcode != JIM_EXIT) { + JimSetArgv(interp, 0, NULL); + if (!isatty(STDIN_FILENO)) { + + goto eval_stdin; + } + retcode = Jim_InteractivePrompt(interp); + } + } + else { + + if (argc > 2 && strcmp(argv[1], "-e") == 0) { + + JimSetArgv(interp, argc - 3, argv + 3); + retcode = Jim_Eval(interp, argv[2]); + if (retcode != JIM_ERR) { + int len; + const char *msg = Jim_GetString(Jim_GetResult(interp), &len); + if (fwrite(msg, len, 1, stdout) == 0) { + + } + putchar('\n'); + } + } + else { + Jim_SetVariableStr(interp, "argv0", Jim_NewStringObj(interp, argv[1], -1)); + JimSetArgv(interp, argc - 2, argv + 2); + if (strcmp(argv[1], "-") == 0) { +eval_stdin: + retcode = Jim_Eval(interp, "eval [info source [stdin read] stdin 1]"); + } else { + retcode = Jim_EvalFile(interp, argv[1]); + } + } + if (retcode == JIM_ERR) { + JimPrintErrorMessage(interp); + } + } + if (retcode == JIM_EXIT) { + retcode = Jim_GetExitCode(interp); + } + else if (retcode == JIM_ERR) { + retcode = 1; + } + else { + retcode = 0; + } + Jim_FreeInterp(interp); + return retcode; +} +#endif diff --git a/autosetup/pkg-config.tcl b/autosetup/pkg-config.tcl new file mode 100644 index 0000000000..9ce7111f55 --- /dev/null +++ b/autosetup/pkg-config.tcl @@ -0,0 +1,168 @@ +# Copyright (c) 2016 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# @synopsis: +# +# The 'pkg-config' module allows package information to be found via 'pkg-config'. +# +# If not cross-compiling, the package path should be determined automatically +# by 'pkg-config'. +# If cross-compiling, the default package path is the compiler sysroot. +# If the C compiler doesn't support '-print-sysroot', the path can be supplied +# by the '--sysroot' option or by defining 'SYSROOT'. +# +# 'PKG_CONFIG' may be set to use an alternative to 'pkg-config'. + +use cc + +options { + sysroot:dir => "Override compiler sysroot for pkg-config search path" +} + +# @pkg-config-init ?required? +# +# Initialises the 'pkg-config' system. Unless '$required' is set to 0, +# it is a fatal error if a usable 'pkg-config' is not found . +# +# This command will normally be called automatically as required, +# but it may be invoked explicitly if lack of 'pkg-config' is acceptable. +# +# Returns 1 if ok, or 0 if 'pkg-config' not found/usable (only if '$required' is 0). +# +proc pkg-config-init {{required 1}} { + if {[is-defined HAVE_PKG_CONFIG]} { + return [get-define HAVE_PKG_CONFIG] + } + set found 0 + + define PKG_CONFIG [get-env PKG_CONFIG pkg-config] + msg-checking "Checking for pkg-config..." + + if {[catch {exec [get-define PKG_CONFIG] --version} version]} { + msg-result "[get-define PKG_CONFIG] (not found)" + if {$required} { + user-error "No usable pkg-config" + } + } else { + msg-result $version + define PKG_CONFIG_VERSION $version + + set found 1 + + if {[opt-str sysroot o]} { + define SYSROOT [file-normalize $o] + msg-result "Using specified sysroot [get-define SYSROOT]" + } elseif {[get-define build] ne [get-define host]} { + if {[catch {exec-with-stderr {*}[get-define CC] -print-sysroot} result errinfo] == 0} { + # Use the compiler sysroot, if there is one + define SYSROOT $result + msg-result "Found compiler sysroot $result" + } else { + configlog "[get-define CC] -print-sysroot: $result" + set msg "pkg-config: Cross compiling, but no compiler sysroot and no --sysroot supplied" + if {$required} { + user-error $msg + } else { + msg-result $msg + } + set found 0 + } + } + if {[is-defined SYSROOT]} { + set sysroot [get-define SYSROOT] + + # XXX: It's possible that these should be set only when invoking pkg-config + global env + set env(PKG_CONFIG_DIR) "" + # Supposedly setting PKG_CONFIG_LIBDIR means that PKG_CONFIG_PATH is ignored, + # but it doesn't seem to work that way in practice + set env(PKG_CONFIG_PATH) "" + # Do we need to try /usr/local as well or instead? + set env(PKG_CONFIG_LIBDIR) $sysroot/usr/lib/pkgconfig:$sysroot/usr/share/pkgconfig + set env(PKG_CONFIG_SYSROOT_DIR) $sysroot + } + } + define HAVE_PKG_CONFIG $found + return $found +} + +# @pkg-config module ?requirements? +# +# Use 'pkg-config' to find the given module meeting the given requirements. +# e.g. +# +## pkg-config pango >= 1.37.0 +# +# If found, returns 1 and sets 'HAVE_PKG_PANGO' to 1 along with: +# +## PKG_PANGO_VERSION to the found version +## PKG_PANGO_LIBS to the required libs (--libs-only-l) +## PKG_PANGO_LDFLAGS to the required linker flags (--libs-only-L) +## PKG_PANGO_CFLAGS to the required compiler flags (--cflags) +# +# If not found, returns 0. +# +proc pkg-config {module args} { + set ok [pkg-config-init] + + msg-checking "Checking for $module $args..." + + if {!$ok} { + msg-result "no pkg-config" + return 0 + } + + set pkgconfig [get-define PKG_CONFIG] + + set ret [catch {exec $pkgconfig --modversion "$module $args"} version] + configlog "$pkgconfig --modversion $module $args: $version" + if {$ret} { + msg-result "not found" + return 0 + } + # Sometimes --modversion succeeds but because of dependencies it isn't usable + # This seems to show up with --cflags + set ret [catch {exec $pkgconfig --cflags $module} cflags] + if {$ret} { + msg-result "unusable ($version - see config.log)" + configlog "$pkgconfig --cflags $module" + configlog $cflags + return 0 + } + msg-result $version + set prefix [feature-define-name $module PKG_] + define HAVE_${prefix} + define ${prefix}_VERSION $version + define ${prefix}_CFLAGS $cflags + define ${prefix}_LIBS [exec $pkgconfig --libs-only-l $module] + define ${prefix}_LDFLAGS [exec $pkgconfig --libs-only-L $module] + return 1 +} + +# @pkg-config-get module setting +# +# Convenience access to the results of 'pkg-config'. +# +# For example, '[pkg-config-get pango CFLAGS]' returns +# the value of 'PKG_PANGO_CFLAGS', or '""' if not defined. +proc pkg-config-get {module name} { + set prefix [feature-define-name $module PKG_] + get-define ${prefix}_${name} "" +} + +# @pkg-config-get-var module variable +# +# Return the value of the given variable from the given pkg-config module. +# The module must already have been successfully detected with pkg-config. +# e.g. +# +## if {[pkg-config harfbuzz >= 2.5]} { +## define harfbuzz_libdir [pkg-config-get-var harfbuzz libdir] +## } +# +# Returns the empty string if the variable isn't defined. +proc pkg-config-get-var {module variable} { + set pkgconfig [get-define PKG_CONFIG] + set prefix [feature-define-name $module HAVE_PKG_] + exec $pkgconfig $module --variable $variable +} diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl new file mode 100644 index 0000000000..86f4df44e2 --- /dev/null +++ b/autosetup/proj.tcl @@ -0,0 +1,2549 @@ +######################################################################## +# 2024 September 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# + +# +# ----- @module proj.tcl ----- +# @section Project-agnostic Helper APIs +# + +# +# Routines for Steve Bennett's autosetup which are common to trees +# managed in and around the umbrella of the SQLite project. +# +# The intent is that these routines be relatively generic, independent +# of a given project. +# +# For practical purposes, the copy of this file hosted in the SQLite +# project is the "canonical" one: +# +# https://sqlite.org/src/file/autosetup/proj.tcl +# +# This file was initially derived from one used in the libfossil +# project, authored by the same person who ported it here, and this is +# noted here only as an indication that there are no licensing issues +# despite this code having a handful of near-twins running around a +# handful of third-party source trees. +# +# Design notes: +# +# - Symbols with _ separators are intended for internal use within +# this file, and are not part of the API which auto.def files should +# rely on. Symbols with - separators are public APIs. +# +# - By and large, autosetup prefers to update global state with the +# results of feature checks, e.g. whether the compiler supports flag +# --X. In this developer's opinion that (A) causes more confusion +# than it solves[^1] and (B) adds an unnecessary layer of "voodoo" +# between the autosetup user and its internals. This module, in +# contrast, instead injects the results of its own tests into +# well-defined variables and leaves the integration of those values +# to the caller's discretion. +# +# [1]: As an example: testing for the -rpath flag, using +# cc-check-flags, can break later checks which use +# [cc-check-function-in-lib ...] because the resulting -rpath flag +# implicitly becomes part of those tests. In the case of an rpath +# test, downstream tests may not like the $prefix/lib path added by +# the rpath test. To avoid such problems, we avoid (intentionally) +# updating global state via feature tests. +# + +# +# $proj__Config is an internal-use-only array for storing whatever generic +# internal stuff we need stored. +# +array set ::proj__Config [subst { + self-tests [get-env proj.self-tests 0] + verbose-assert [get-env proj.assert-verbose 0] + isatty [isatty? stdout] +}] + +# +# List of dot-in files to filter in the final stages of +# configuration. Some configuration steps may append to this. Each +# one in this list which exists will trigger the generation of a +# file with that same name, minus the ".in", in the build directory +# (which differ from the source dir in out-of-tree builds). +# +# See: proj-dot-ins-append and proj-dot-ins-process +# +set ::proj__Config(dot-in-files) [list] + +# +# @proj-warn msg +# +# Emits a warning message to stderr. All args are appended with a +# space between each. +# +proc proj-warn {args} { + show-notices + puts stderr [join [list "WARNING:" \[ [proj-scope 1] \]: {*}$args] " "] +} + + +# +# Internal impl of [proj-fatal] and [proj-error]. It must be called +# using tailcall. +# +proc proj__faterr {failMode args} { + show-notices + set lvl 1 + while {"-up" eq [lindex $args 0]} { + set args [lassign $args -] + incr lvl + } + if {$failMode} { + puts stderr [join [list "FATAL:" \[ [proj-scope $lvl] \]: {*}$args]] + exit 1 + } else { + error [join [list in \[ [proj-scope $lvl] \]: {*}$args]] + } +} + +# +# @proj-fatal ?-up...? msg... +# +# Emits an error message to stderr and exits with non-0. All args are +# appended with a space between each. +# +# The calling scope's name is used in the error message. To instead +# use the name of a call higher up in the stack, use -up once for each +# additional level. +# +proc proj-fatal {args} { + tailcall proj__faterr 1 {*}$args +} + +# +# @proj-error ?-up...? msg... +# +# Works like proj-fatal but uses [error] intead of [exit]. +# +proc proj-error {args} { + tailcall proj__faterr 0 {*}$args +} + +# +# @proj-assert script ?message? +# +# Kind of like a C assert: if uplevel of [list expr $script] is false, +# a fatal error is triggered. The error message, by default, includes +# the body of the failed assertion, but if $msg is set then that is +# used instead. +# +proc proj-assert {script {msg ""}} { + if {1 eq $::proj__Config(verbose-assert)} { + msg-result [proj-bold "asserting: $script"] + } + if {![uplevel 1 [list expr $script]]} { + if {"" eq $msg} { + set msg $script + } + tailcall proj__faterr 1 "Assertion failed:" $msg + } +} + +# +# @proj-bold str +# +# If this function believes that the current console might support +# ANSI escape sequences then this returns $str wrapped in a sequence +# to bold that text, else it returns $str as-is. +# +proc proj-bold {args} { + if {$::autosetup(iswin) || !$::proj__Config(isatty)} { + return [join $args] + } + return "\033\[1m${args}\033\[0m" +} + +# +# @proj-indented-notice ?-error? ?-notice? msg +# +# Takes a multi-line message and emits it with consistent indentation. +# It does not perform any line-wrapping of its own. Which output +# routine it uses depends on its flags, defaulting to msg-result. +# For -error and -notice it uses user-notice. +# +# If the -notice flag it used then it emits using [user-notice], which +# means its rendering will (A) go to stderr and (B) be delayed until +# the next time autosetup goes to output a message. +# +# If the -error flag is provided then it renders the message +# immediately to stderr and then exits. +# +# If neither -notice nor -error are used, the message will be sent to +# stdout without delay. +# +proc proj-indented-notice {args} { + set fErr "" + set outFunc "msg-result" + while {[llength $args] > 1} { + switch -exact -- [lindex $args 0] { + -error { + set args [lassign $args fErr] + set outFunc "user-notice" + } + -notice { + set args [lassign $args -] + set outFunc "user-notice" + } + default { + break + } + } + } + set lines [split [join $args] \n] + foreach line $lines { + set line [string trimleft $line] + if {"" eq $line} { + $outFunc $line + } else { + $outFunc " $line" + } + } + if {"" ne $fErr} { + show-notices + exit 1 + } +} + +# +# @proj-is-cross-compiling +# +# Returns 1 if cross-compiling, else 0. +# +proc proj-is-cross-compiling {} { + expr {[get-define host] ne [get-define build]} +} + +# +# @proj-strip-hash-comments value +# +# Expects to receive string input, which it splits on newlines, strips +# out any lines which begin with any number of whitespace followed by +# a '#', and returns a value containing the [append]ed results of each +# remaining line with a \n between each. It does not strip out +# comments which appear after the first non-whitespace character. +# +proc proj-strip-hash-comments {val} { + set x {} + foreach line [split $val \n] { + if {![string match "#*" [string trimleft $line]]} { + append x $line \n + } + } + return $x +} + +# +# @proj-cflags-without-werror +# +# Fetches [define $var], strips out any -Werror entries, and returns +# the new value. This is intended for temporarily stripping -Werror +# from CFLAGS or CPPFLAGS within the scope of a [define-push] block. +# +proc proj-cflags-without-werror {{var CFLAGS}} { + set rv {} + foreach f [get-define $var ""] { + switch -exact -- $f { + -Werror {} + default { lappend rv $f } + } + } + join $rv " " +} + +# +# @proj-check-function-in-lib +# +# A proxy for cc-check-function-in-lib with the following differences: +# +# - Does not make any global changes to the LIBS define. +# +# - Strips out the -Werror flag from CFLAGS before running the test, +# as these feature tests will often fail if -Werror is used. +# +# Returns the result of cc-check-function-in-lib (i.e. true or false). +# The resulting linker flags are stored in the [define] named +# lib_${function}. +# +proc proj-check-function-in-lib {function libs {otherlibs {}}} { + set found 0 + define-push {LIBS CFLAGS} { + #puts "CFLAGS before=[get-define CFLAGS]" + define CFLAGS [proj-cflags-without-werror] + #puts "CFLAGS after =[get-define CFLAGS]" + set found [cc-check-function-in-lib $function $libs $otherlibs] + } + return $found +} + +# +# @proj-search-for-header-dir ?-dirs LIST? ?-subdirs LIST? header +# +# Searches for $header in a combination of dirs and subdirs, specified +# by the -dirs {LIST} and -subdirs {LIST} flags (each of which have +# sane defaults). Returns either the first matching dir or an empty +# string. The return value does not contain the filename part. +# +proc proj-search-for-header-dir {header args} { + set subdirs {include} + set dirs {/usr /usr/local /mingw} +# Debatable: +# if {![proj-is-cross-compiling]} { +# lappend dirs [get-define prefix] +# } + while {[llength $args]} { + switch -exact -- [lindex $args 0] { + -dirs { set args [lassign $args - dirs] } + -subdirs { set args [lassign $args - subdirs] } + default { + proj-error "Unhandled argument: $args" + } + } + } + foreach dir $dirs { + foreach sub $subdirs { + if {[file exists $dir/$sub/$header]} { + return "$dir/$sub" + } + } + } + return "" +} + +# +# @proj-find-executable-path ?-v? binaryName +# +# Works similarly to autosetup's [find-executable-path $binName] but: +# +# - If the first arg is -v, it's verbose about searching, else it's quiet. +# +# Returns the full path to the result or an empty string. +# +proc proj-find-executable-path {args} { + set binName $args + set verbose 0 + if {[lindex $args 0] eq "-v"} { + set verbose 1 + set args [lassign $args - binName] + msg-checking "Looking for $binName ... " + } + set check [find-executable-path $binName] + if {$verbose} { + if {"" eq $check} { + msg-result "not found" + } else { + msg-result $check + } + } + return $check +} + +# +# @proj-bin-define binName ?defName? +# +# Uses [proj-find-executable-path $binName] to (verbosely) search for +# a binary, sets a define (see below) to the result, and returns the +# result (an empty string if not found). +# +# The define'd name is: If $defName is not empty, it is used as-is. If +# $defName is empty then "BIN_X" is used, where X is the upper-case +# form of $binName with any '-' characters replaced with '_'. +# +proc proj-bin-define {binName {defName {}}} { + set check [proj-find-executable-path -v $binName] + if {"" eq $defName} { + set defName "BIN_[string toupper [string map {- _} $binName]]" + } + define $defName $check + return $check +} + +# +# @proj-first-bin-of bin... +# +# Looks for the first binary found of the names passed to this +# function. If a match is found, the full path to that binary is +# returned, else "" is returned. +# +# Despite using cc-path-progs to do the search, this function clears +# any define'd name that function stores for the result (because the +# caller has no sensible way of knowing which [define] name it has +# unless they pass only a single argument). +# +proc proj-first-bin-of {args} { + set rc "" + foreach b $args { + set u [string toupper $b] + # Note that cc-path-progs defines $u to "false" if it finds no + # match. + if {[cc-path-progs $b]} { + set rc [get-define $u] + } + undefine $u + if {"" ne $rc} break + } + return $rc +} + +# +# @proj-opt-was-provided key +# +# Returns 1 if the user specifically provided the given configure flag +# or if it was specifically set using proj-opt-set, else 0. This can +# be used to distinguish between options which have a default value +# and those which were explicitly provided by the user, even if the +# latter is done in a way which uses the default value. +# +# For example, with a configure flag defined like: +# +# { foo-bar:=baz => {its help text} } +# +# This function will, when passed foo-bar, return 1 only if the user +# passes --foo-bar to configure, even if that invocation would resolve +# to the default value of baz. If the user does not explicitly pass in +# --foo-bar (with or without a value) then this returns 0. +# +# Calling [proj-opt-set] is, for purposes of the above, equivalent to +# explicitly passing in the flag. +# +# Note: unlike most functions which deal with configure --flags, this +# one does not validate that $key refers to a pre-defined flag. i.e. +# it accepts arbitrary keys, even those not defined via an [options] +# call. [proj-opt-set] manipulates the internal list of flags, such +# that new options set via that function will cause this function to +# return true. (That's an unintended and unavoidable side-effect, not +# specifically a feature which should be made use of.) +# +proc proj-opt-was-provided {key} { + dict exists $::autosetup(optset) $key +} + +# +# @proj-opt-set flag ?val? +# +# Force-set autosetup option $flag to $val. The value can be fetched +# later with [opt-val], [opt-bool], and friends. +# +# Returns $val. +# +proc proj-opt-set {flag {val 1}} { + if {$flag ni $::autosetup(options)} { + # We have to add this to autosetup(options) or else future calls + # to [opt-bool $flag] will fail validation of $flag. + lappend ::autosetup(options) $flag + } + dict set ::autosetup(optset) $flag $val + return $val +} + +# +# @proj-opt-exists flag +# +# Returns 1 if the given flag has been defined as a legal configure +# option, else returns 0. Options set via proj-opt-set "exist" for +# this purpose even if they were not defined via autosetup's +# [options] function. +# +proc proj-opt-exists {flag} { + expr {$flag in $::autosetup(options)}; +} + +# +# @proj-val-truthy val +# +# Returns 1 if $val appears to be a truthy value, else returns +# 0. Truthy values are any of {1 on true yes enabled} +# +proc proj-val-truthy {val} { + expr {$val in {1 on true yes enabled}} +} + +# +# @proj-opt-truthy flag +# +# Returns 1 if [opt-val $flag] appears to be a truthy value or +# [opt-bool $flag] is true. See proj-val-truthy. +# +proc proj-opt-truthy {flag} { + if {[proj-val-truthy [opt-val $flag]]} { return 1 } + set rc 0 + catch { + # opt-bool will throw if $flag is not a known boolean flag + set rc [opt-bool $flag] + } + return $rc +} + +# +# @proj-if-opt-truthy boolFlag thenScript ?elseScript? +# +# If [proj-opt-truthy $flag] is true, eval $then, else eval $else. +# +proc proj-if-opt-truthy {boolFlag thenScript {elseScript {}}} { + if {[proj-opt-truthy $boolFlag]} { + uplevel 1 $thenScript + } else { + uplevel 1 $elseScript + } +} + +# +# @proj-define-for-opt flag def ?msg? ?iftrue? ?iffalse? +# +# If [proj-opt-truthy $flag] then [define $def $iftrue] else [define +# $def $iffalse]. If $msg is not empty, output [msg-checking $msg] and +# a [msg-results ...] which corresponds to the result. Returns 1 if +# the opt-truthy check passes, else 0. +# +proc proj-define-for-opt {flag def {msg ""} {iftrue 1} {iffalse 0}} { + if {"" ne $msg} { + msg-checking "$msg " + } + set rcMsg "" + set rc 0 + if {[proj-opt-truthy $flag]} { + define $def $iftrue + set rc 1 + } else { + define $def $iffalse + } + switch -- [proj-val-truthy [get-define $def]] { + 0 { set rcMsg no } + 1 { set rcMsg yes } + } + if {"" ne $msg} { + msg-result $rcMsg + } + return $rc +} + +# +# @proj-opt-define-bool ?-v? optName defName ?descr? +# +# Checks [proj-opt-truthy $optName] and calls [define $defName X] +# where X is 0 for false and 1 for true. $descr is an optional +# [msg-checking] argument which defaults to $defName. Returns X. +# +# If args[0] is -v then the boolean semantics are inverted: if +# the option is set, it gets define'd to 0, else 1. Returns the +# define'd value. +# +proc proj-opt-define-bool {args} { + set invert 0 + if {[lindex $args 0] eq "-v"} { + incr invert + lassign $args - optName defName descr + } else { + lassign $args optName defName descr + } + if {"" eq $descr} { + set descr $defName + } + #puts "optName=$optName defName=$defName descr=$descr" + set rc 0 + msg-checking "[join $descr] ... " + set rc [proj-opt-truthy $optName] + if {$invert} { + set rc [expr {!$rc}] + } + msg-result [string map {0 no 1 yes} $rc] + define $defName $rc + return $rc +} + +# +# @proj-check-module-loader +# +# Check for module-loading APIs (libdl/libltdl)... +# +# Looks for libltdl or dlopen(), the latter either in -ldl or built in +# to libc (as it is on some platforms). Returns 1 if found, else +# 0. Either way, it `define`'s: +# +# - HAVE_LIBLTDL to 1 or 0 if libltdl is found/not found +# - HAVE_LIBDL to 1 or 0 if dlopen() is found/not found +# - LDFLAGS_MODULE_LOADER one of ("-lltdl", "-ldl", or ""), noting +# that -ldl may legally be empty on some platforms even if +# HAVE_LIBDL is true (indicating that dlopen() is available without +# extra link flags). LDFLAGS_MODULE_LOADER also gets "-rdynamic" appended +# to it because otherwise trying to open DLLs will result in undefined +# symbol errors. +# +# Note that if it finds LIBLTDL it does not look for LIBDL, so will +# report only that is has LIBLTDL. +# +proc proj-check-module-loader {} { + msg-checking "Looking for module-loader APIs... " + if {99 ne [get-define LDFLAGS_MODULE_LOADER 99]} { + if {1 eq [get-define HAVE_LIBLTDL 0]} { + msg-result "(cached) libltdl" + return 1 + } elseif {1 eq [get-define HAVE_LIBDL 0]} { + msg-result "(cached) libdl" + return 1 + } + # else: wha??? + } + set HAVE_LIBLTDL 0 + set HAVE_LIBDL 0 + set LDFLAGS_MODULE_LOADER "" + set rc 0 + puts "" ;# cosmetic kludge for cc-check-XXX + if {[cc-check-includes ltdl.h] && [cc-check-function-in-lib lt_dlopen ltdl]} { + set HAVE_LIBLTDL 1 + set LDFLAGS_MODULE_LOADER "-lltdl -rdynamic" + msg-result " - Got libltdl." + set rc 1 + } elseif {[cc-with {-includes dlfcn.h} { + cctest -link 1 -declare "extern char* dlerror(void);" -code "dlerror();"}]} { + msg-result " - This system can use dlopen() without -ldl." + set HAVE_LIBDL 1 + set LDFLAGS_MODULE_LOADER "" + set rc 1 + } elseif {[cc-check-includes dlfcn.h]} { + set HAVE_LIBDL 1 + set rc 1 + if {[cc-check-function-in-lib dlopen dl]} { + msg-result " - dlopen() needs libdl." + set LDFLAGS_MODULE_LOADER "-ldl -rdynamic" + } else { + msg-result " - dlopen() not found in libdl. Assuming dlopen() is built-in." + set LDFLAGS_MODULE_LOADER "-rdynamic" + } + } + define HAVE_LIBLTDL $HAVE_LIBLTDL + define HAVE_LIBDL $HAVE_LIBDL + define LDFLAGS_MODULE_LOADER $LDFLAGS_MODULE_LOADER + return $rc +} + +# +# @proj-no-check-module-loader +# +# Sets all flags which would be set by proj-check-module-loader to +# empty/falsy values, as if those checks had failed to find a module +# loader. Intended to be called in place of that function when +# a module loader is explicitly not desired. +# +proc proj-no-check-module-loader {} { + define HAVE_LIBDL 0 + define HAVE_LIBLTDL 0 + define LDFLAGS_MODULE_LOADER "" +} + +# +# @proj-file-content ?-trim? filename +# +# Opens the given file, reads all of its content, and returns it. If +# the first arg is -trim, the contents of the file named by the second +# argument are trimmed before returning them. +# +proc proj-file-content {args} { + set trim 0 + set fname $args + if {"-trim" eq [lindex $args 0]} { + set trim 1 + lassign $args - fname + } + set fp [open $fname rb] + set rc [read $fp] + close $fp + if {$trim} { return [string trim $rc] } + return $rc +} + +# +# @proj-file-conent filename +# +# Returns the contents of the given file as an array of lines, with +# the EOL stripped from each input line. +# +proc proj-file-content-list {fname} { + set fp [open $fname rb] + set rc {} + while { [gets $fp line] >= 0 } { + lappend rc $line + } + close $fp + return $rc +} + +# +# @proj-file-write ?-ro? fname content +# +# Works like autosetup's [writefile] but explicitly uses binary mode +# to avoid EOL translation on Windows. If $fname already exists, it is +# overwritten, even if it's flagged as read-only. +# +proc proj-file-write {args} { + if {"-ro" eq [lindex $args 0]} { + lassign $args ro fname content + } else { + set ro "" + lassign $args fname content + } + file delete -force -- $fname; # in case it's read-only + set f [open $fname wb] + puts -nonewline $f $content + close $f + if {"" ne $ro} { + catch { + exec chmod -w $fname + #file attributes -w $fname; #jimtcl has no 'attributes' + } + } +} + +# +# @proj-check-compile-commands ?-assume-for-clang? ?configFlag? +# +# Checks the compiler for compile_commands.json support. If +# $configFlag is not empty then it is assumed to be the name of an +# autosetup boolean config which controls whether to run/skip this +# check. +# +# If -assume-for-clang is provided and $configFlag is not empty and CC +# matches *clang* and no --$configFlag was explicitly provided to the +# configure script then behave as if --$configFlag had been provided. +# To disable that assumption, either don't pass -assume-for-clang or +# pass --$configFlag=0 to the configure script. (The reason for this +# behavior is that clang supports compile-commands but some other +# compilers report false positives with these tests.) +# +# Returns 1 if supported, else 0, and defines HAVE_COMPILE_COMMANDS to +# that value. Defines MAKE_COMPILATION_DB to "yes" if supported, "no" +# if not. The use of MAKE_COMPILATION_DB is deprecated/discouraged: +# HAVE_COMPILE_COMMANDS is preferred. +# +# ACHTUNG: this test has a long history of false positive results +# because of compilers reacting differently to the -MJ flag. Because +# of this, it is recommended that this support be an opt-in feature, +# rather than an on-by-default default one. That is: in the +# configure script define the option as +# {--the-flag-name=0 => {Enable ....}} +# +proc proj-check-compile-commands {args} { + set i 0 + set configFlag {} + set fAssumeForClang 0 + set doAssume 0 + msg-checking "compile_commands.json support... " + if {"-assume-for-clang" eq [lindex $args 0]} { + lassign $args - configFlag + incr fAssumeForClang + } elseif {1 == [llength $args]} { + lassign $args configFlag + } else { + proj-error "Invalid arguments" + } + if {1 == $fAssumeForClang && "" ne $configFlag} { + if {[string match *clang* [get-define CC]] + && ![proj-opt-was-provided $configFlag] + && ![proj-opt-truthy $configFlag]} { + proj-indented-notice [subst -nocommands -nobackslashes { + CC appears to be clang, so assuming that --$configFlag is likely + to work. To disable this assumption use --$configFlag=0.}] + incr doAssume + } + } + if {!$doAssume && "" ne $configFlag && ![proj-opt-truthy $configFlag]} { + msg-result "check disabled. Use --${configFlag} to enable it." + define HAVE_COMPILE_COMMANDS 0 + define MAKE_COMPILATION_DB no + return 0 + } else { + if {[cctest -lang c -cflags {/dev/null -MJ} -source {}]} { + # This test reportedly incorrectly succeeds on one of + # Martin G.'s older systems. drh also reports a false + # positive on an unspecified older Mac system. + msg-result "compiler supports -MJ. Assuming it's useful for compile_commands.json" + define MAKE_COMPILATION_DB yes; # deprecated + define HAVE_COMPILE_COMMANDS 1 + return 1 + } else { + msg-result "compiler does not support compile_commands.json" + define MAKE_COMPILATION_DB no + define HAVE_COMPILE_COMMANDS 0 + return 0 + } + } +} + +# +# @proj-touch filename +# +# Runs the 'touch' external command on one or more files, ignoring any +# errors. +# +proc proj-touch {filename} { + catch { exec touch {*}$filename } +} + +# +# @proj-make-from-dot-in ?-touch? infile ?outfile? +# +# Uses [make-template] to create makefile(-like) file(s) $outfile from +# $infile but explicitly makes the output read-only, to avoid +# inadvertent editing (who, me?). +# +# If $outfile is empty then: +# +# - If $infile is a 2-element list, it is assumed to be an in/out pair, +# and $outfile is set from the 2nd entry in that list. Else... +# +# - $outfile is set to $infile stripped of its extension. +# +# If the first argument is -touch then the generated file is touched +# to update its timestamp. This can be used as a workaround for +# cases where (A) autosetup does not update the file because it was +# not really modified and (B) the file *really* needs to be updated to +# please the build process. +# +# Failures when running chmod or touch are silently ignored. +# +proc proj-make-from-dot-in {args} { + set fIn "" + set fOut "" + set touch 0 + if {[lindex $args 0] eq "-touch"} { + set touch 1 + lassign $args - fIn fOut + } else { + lassign $args fIn fOut + } + if {"" eq $fOut} { + if {[llength $fIn]>1} { + lassign $fIn fIn fOut + } else { + set fOut [file rootname $fIn] + } + } + #puts "filenames=$filename" + if {[file exists $fOut]} { + catch { exec chmod u+w $fOut } + } + #puts "making template: $fIn ==> $fOut" + #define-push {top_srcdir} { + #puts "--- $fIn $fOut top_srcdir=[get-define top_srcdir]" + make-template $fIn $fOut + #puts "--- $fIn $fOut top_srcdir=[get-define top_srcdir]" + # make-template modifies top_srcdir + #} + if {$touch} { + proj-touch $fOut + } + catch { + exec chmod -w $fOut + #file attributes -w $f; #jimtcl has no 'attributes' + } +} + +# +# @proj-check-profile-flag ?flagname? +# +# Checks for the boolean configure option named by $flagname. If set, +# it checks if $CC seems to refer to gcc. If it does (or appears to) +# then it defines CC_PROFILE_FLAG to "-pg" and returns 1, else it +# defines CC_PROFILE_FLAG to "" and returns 0. +# +# Note that the resulting flag must be added to both CFLAGS and +# LDFLAGS in order for binaries to be able to generate "gmon.out". In +# order to avoid potential problems with escaping, space-containing +# tokens, and interfering with autosetup's use of these vars, this +# routine does not directly modify CFLAGS or LDFLAGS. +# +proc proj-check-profile-flag {{flagname profile}} { + #puts "flagname=$flagname ?[proj-opt-truthy $flagname]?" + if {[proj-opt-truthy $flagname]} { + set CC [get-define CC] + regsub {.*ccache *} $CC "" CC + # ^^^ if CC="ccache gcc" then [exec] treats "ccache gcc" as a + # single binary name and fails. So strip any leading ccache part + # for this purpose. + if { ![catch { exec $CC --version } msg]} { + if {[string first gcc $CC] != -1} { + define CC_PROFILE_FLAG "-pg" + return 1 + } + } + } + define CC_PROFILE_FLAG "" + return 0 +} + +# +# @proj-looks-like-windows ?key? +# +# Returns 1 if this appears to be a Windows environment (MinGw, +# Cygwin, MSys), else returns 0. The optional argument is the name of +# an autosetup define which contains platform name info, defaulting to +# "host" (meaning, somewhat counterintuitively, the target system, not +# the current host). The other legal value is "build" (the build +# machine, i.e. the local host). If $key == "build" then some +# additional checks may be performed which are not applicable when +# $key == "host". +# +proc proj-looks-like-windows {{key host}} { + global autosetup + switch -glob -- [get-define $key] { + *-*-ming* - *-*-cygwin - *-*-msys - *windows* { + return 1 + } + } + if {$key eq "build"} { + # These apply only to the local OS, not a cross-compilation target, + # as the above check potentially can. + if {$::autosetup(iswin)} { return 1 } + if {[find-an-executable cygpath] ne "" || $::tcl_platform(os) eq "Windows NT"} { + return 1 + } + } + return 0 +} + +# +# @proj-looks-like-mac ?key? +# +# Looks at either the 'host' (==compilation target platform) or +# 'build' (==the being-built-on platform) define value and returns if +# if that value seems to indicate that it represents a Mac platform, +# else returns 0. +# +proc proj-looks-like-mac {{key host}} { + switch -glob -- [get-define $key] { + *-*-darwin* { + # https://sqlite.org/forum/forumpost/7b218c3c9f207646 + # There's at least one Linux out there which matches *apple*. + return 1 + } + default { + return 0 + } + } +} + +# +# @proj-exe-extension +# +# Checks autosetup's "host" and "build" defines to see if the build +# host and target are Windows-esque (Cygwin, MinGW, MSys). If the +# build environment is then BUILD_EXEEXT is [define]'d to ".exe", else +# "". If the target, a.k.a. "host", is then TARGET_EXEEXT is +# [define]'d to ".exe", else "". +# +proc proj-exe-extension {} { + set rH "" + set rB "" + if {[proj-looks-like-windows host]} { + set rH ".exe" + } + if {[proj-looks-like-windows build]} { + set rB ".exe" + } + define BUILD_EXEEXT $rB + define TARGET_EXEEXT $rH +} + +# +# @proj-dll-extension +# +# Works like proj-exe-extension except that it defines BUILD_DLLEXT +# and TARGET_DLLEXT to one of (.so, ,dll, .dylib). +# +# Trivia: for .dylib files, the linker needs the -dynamiclib flag +# instead of -shared. +# +proc proj-dll-extension {} { + set inner {{key} { + if {[proj-looks-like-mac $key]} { + return ".dylib" + } + if {[proj-looks-like-windows $key]} { + return ".dll" + } + return ".so" + }} + define BUILD_DLLEXT [apply $inner build] + define TARGET_DLLEXT [apply $inner host] +} + +# +# @proj-lib-extension +# +# Static-library counterpart of proj-dll-extension. Defines +# BUILD_LIBEXT and TARGET_LIBEXT to the conventional static library +# extension for the being-built-on resp. the target platform. +# +proc proj-lib-extension {} { + set inner {{key} { + switch -glob -- [get-define $key] { + *-*-ming* - *-*-cygwin - *-*-msys { + return ".a" + # ^^^ this was ".lib" until 2025-02-07. See + # https://sqlite.org/forum/forumpost/02db2d4240 + } + default { + return ".a" + } + } + }} + define BUILD_LIBEXT [apply $inner build] + define TARGET_LIBEXT [apply $inner host] +} + +# +# @proj-file-extensions +# +# Calls all of the proj-*-extension functions. +# +proc proj-file-extensions {} { + proj-exe-extension + proj-dll-extension + proj-lib-extension +} + +# +# @proj-affirm-files-exist ?-v? filename... +# +# Expects a list of file names. If any one of them does not exist in +# the filesystem, it fails fatally with an informative message. +# Returns the last file name it checks. If the first argument is -v +# then it emits msg-checking/msg-result messages for each file. +# +proc proj-affirm-files-exist {args} { + set rc "" + set verbose 0 + if {[lindex $args 0] eq "-v"} { + set verbose 1 + set args [lrange $args 1 end] + } + foreach f $args { + if {$verbose} { msg-checking "Looking for $f ... " } + if {![file exists $f]} { + user-error "not found: $f" + } + if {$verbose} { msg-result "" } + set rc $f + } + return rc +} + +# +# @proj-check-emsdk +# +# Emscripten is used for doing in-tree builds of web-based WASM stuff, +# as opposed to WASI-based WASM or WASM binaries we import from other +# places. This is only set up for Unix-style OSes and is untested +# anywhere but Linux. Requires that the --with-emsdk flag be +# registered with autosetup. +# +# It looks for the SDK in the location specified by --with-emsdk. +# Values of "" or "auto" mean to check for the environment var EMSDK +# (which gets set by the emsdk_env.sh script from the SDK) or that +# same var passed to configure. +# +# If the given directory is found, it expects to find emsdk_env.sh in +# that directory, as well as the emcc compiler somewhere under there. +# +# If the --with-emsdk[=DIR] flag is explicitly provided and the SDK is +# not found then a fatal error is generated, otherwise failure to find +# the SDK is not fatal. +# +# Defines the following: +# +# - HAVE_EMSDK = 0 or 1 (this function's return value) +# - EMSDK_HOME = "" or top dir of the emsdk +# - EMSDK_ENV_SH = "" or $EMSDK_HOME/emsdk_env.sh +# - BIN_EMCC = "" or $EMSDK_HOME/upstream/emscripten/emcc +# +# Returns 1 if EMSDK_ENV_SH is found, else 0. If EMSDK_HOME is not empty +# but BIN_EMCC is then emcc was not found in the EMSDK_HOME, in which +# case we have to rely on the fact that sourcing $EMSDK_ENV_SH from a +# shell will add emcc to the $PATH. +# +proc proj-check-emsdk {} { + set emsdkHome [opt-val with-emsdk] + define EMSDK_HOME "" + define EMSDK_ENV_SH "" + define BIN_EMCC "" + set hadValue [llength $emsdkHome] + msg-checking "Emscripten SDK? " + if {$emsdkHome in {"" "auto"}} { + # Check the environment. $EMSDK gets set by sourcing emsdk_env.sh. + set emsdkHome [get-env EMSDK ""] + } + set rc 0 + if {$emsdkHome ne ""} { + define EMSDK_HOME $emsdkHome + set emsdkEnv "$emsdkHome/emsdk_env.sh" + if {[file exists $emsdkEnv]} { + msg-result "$emsdkHome" + define EMSDK_ENV_SH $emsdkEnv + set rc 1 + set emcc "$emsdkHome/upstream/emscripten/emcc" + if {[file exists $emcc]} { + define BIN_EMCC $emcc + } + } else { + msg-result "emsdk_env.sh not found in $emsdkHome" + } + } else { + msg-result "not found" + } + if {$hadValue && 0 == $rc} { + # Fail if it was explicitly requested but not found + proj-fatal "Cannot find the Emscripten SDK" + } + define HAVE_EMSDK $rc + return $rc +} + +# +# @proj-cc-check-Wl-flag ?flag ?args?? +# +# Checks whether the given linker flag (and optional arguments) can be +# passed from the compiler to the linker using one of these formats: +# +# - -Wl,flag[,arg1[,...argN]] +# - -Wl,flag -Wl,arg1 ...-Wl,argN +# +# If so, that flag string is returned, else an empty string is +# returned. +# +proc proj-cc-check-Wl-flag {args} { + cc-with {-link 1} { + # Try -Wl,flag,...args + set fli "-Wl" + foreach f $args { append fli ",$f" } + if {[cc-check-flags $fli]} { + return $fli + } + # Try -Wl,flag -Wl,arg1 ...-Wl,argN + set fli "" + foreach f $args { append fli "-Wl,$f " } + if {[cc-check-flags $fli]} { + return [string trim $fli] + } + return "" + } +} + +# +# @proj-check-rpath +# +# Tries various approaches to handling the -rpath link-time +# flag. Defines LDFLAGS_RPATH to that/those flag(s) or an empty +# string. Returns 1 if it finds an option, else 0. +# +# By default, the rpath is set to $prefix/lib. However, if either of +# --exec-prefix=... or --libdir=... are explicitly passed to +# configure then [get-define libdir] is used (noting that it derives +# from exec-prefix by default). +# +proc proj-check-rpath {} { + if {[proj-opt-was-provided libdir] + || [proj-opt-was-provided exec-prefix]} { + set lp "[get-define libdir]" + } else { + set lp "[get-define prefix]/lib" + } + # If we _don't_ use cc-with {} here (to avoid updating the global + # CFLAGS or LIBS or whatever it is that cc-check-flags updates) then + # downstream tests may fail because the resulting rpath gets + # implicitly injected into them. + cc-with {-link 1} { + if {[cc-check-flags "-rpath $lp"]} { + define LDFLAGS_RPATH "-rpath $lp" + } else { + set wl [proj-cc-check-Wl-flag -rpath $lp] + if {"" eq $wl} { + set wl [proj-cc-check-Wl-flag -R$lp] + } + if {"" eq $wl} { + # HP-UX: https://sqlite.org/forum/forumpost/d80ecdaddd + set wl [proj-cc-check-Wl-flag +b $lp] + } + define LDFLAGS_RPATH $wl + } + } + expr {"" ne [get-define LDFLAGS_RPATH]} +} + +# +# @proj-check-soname ?libname? +# +# Checks whether CC supports the -Wl,-soname,lib... flag. If so, it +# returns 1 and defines LDFLAGS_SONAME_PREFIX to the flag's prefix, to +# which the client would need to append "libwhatever.N". If not, it +# returns 0 and defines LDFLAGS_SONAME_PREFIX to an empty string. +# +# The libname argument is only for purposes of running the flag +# compatibility test, and is not included in the resulting +# LDFLAGS_SONAME_PREFIX. It is provided so that clients may +# potentially avoid some end-user confusion by using their own lib's +# name here (which shows up in the "checking..." output). +# +proc proj-check-soname {{libname "libfoo.so.0"}} { + cc-with {-link 1} { + if {[cc-check-flags "-Wl,-soname,${libname}"]} { + define LDFLAGS_SONAME_PREFIX "-Wl,-soname," + return 1 + } elseif {[cc-check-flags "-Wl,+h,${libname}"]} { + # HP-UX: https://sqlite.org/forum/forumpost/d80ecdaddd + define LDFLAGS_SONAME_PREFIX "-Wl,+h," + return 1 + } else { + define LDFLAGS_SONAME_PREFIX "" + return 0 + } + } +} + +# +# @proj-check-fsanitize ?list-of-opts? +# +# Checks whether CC supports -fsanitize=X, where X is each entry of +# the given list of flags. If any of those flags are supported, it +# returns the string "-fsanitize=X..." where X... is a comma-separated +# list of all flags from the original set which are supported. If none +# of the given options are supported then it returns an empty string. +# +# Example: +# +# set f [proj-check-fsanitize {address bounds-check just-testing}] +# +# Will, on many systems, resolve to "-fsanitize=address,bounds-check", +# but may also resolve to "-fsanitize=address". +# +proc proj-check-fsanitize {{opts {address bounds-strict}}} { + set sup {} + foreach opt $opts { + # -nooutput is used because -fsanitize=hwaddress will otherwise + # pass this test on x86_64, but then warn at build time that + # "hwaddress is not supported for this target". + cc-with {-nooutput 1} { + if {[cc-check-flags "-fsanitize=$opt"]} { + lappend sup $opt + } + } + } + if {[llength $sup] > 0} { + return "-fsanitize=[join $sup ,]" + } + return "" +} + +# +# Internal helper for proj-dump-defs-json. Expects to be passed a +# [define] name and the variadic $args which are passed to +# proj-dump-defs-json. If it finds a pattern match for the given +# $name in the various $args, it returns the type flag for that $name, +# e.g. "-str" or "-bare", else returns an empty string. +# +proc proj-defs-type_ {name spec} { + foreach {type patterns} $spec { + foreach pattern $patterns { + if {[string match $pattern $name]} { + return $type + } + } + } + return "" +} + +# +# Internal helper for proj-defs-format_: returns a JSON-ish quoted +# form of the given string-type values. It only performs the most +# basic of escaping. The input must not contain any control +# characters. +# +proc proj-quote-str_ {value} { + return \"[string map [list \\ \\\\ \" \\\"] $value]\" +} + +# +# An internal impl detail of proj-dump-defs-json. Requires a data +# type specifier, as used by make-config-header, and a value. Returns +# the formatted value or the value $::proj__Config(defs-skip) if the caller +# should skip emitting that value. +# +set ::proj__Config(defs-skip) "-proj-defs-format_ sentinel" +proc proj-defs-format_ {type value} { + switch -exact -- $type { + -bare { + # Just output the value unchanged + } + -none { + set value $::proj__Config(defs-skip) + } + -str { + set value [proj-quote-str_ $value] + } + -auto { + # Automatically determine the type + if {![string is integer -strict $value]} { + set value [proj-quote-str_ $value] + } + } + -array { + set ar {} + foreach v $value { + set v [proj-defs-format_ -auto $v] + if {$::proj__Config(defs-skip) ne $v} { + lappend ar $v + } + } + set value "\[ [join $ar {, }] \]" + } + "" { + set value $::proj__Config(defs-skip) + } + default { + proj-fatal "Unknown type in proj-dump-defs-json: $type" + } + } + return $value +} + +# +# @proj-dump-defs-json outfile ...flags +# +# This function works almost identically to autosetup's +# make-config-header but emits its output in JSON form. It is not a +# fully-functional JSON emitter, and will emit broken JSON for +# complicated outputs, but should be sufficient for purposes of +# emitting most configure vars (numbers and simple strings). +# +# In addition to the formatting flags supported by make-config-header, +# it also supports: +# +# -array {patterns...} +# +# Any defines matching the given patterns will be treated as a list of +# values, each of which will be formatted as if it were in an -auto {...} +# set, and the define will be emitted to JSON in the form: +# +# "ITS_NAME": [ "value1", ...valueN ] +# +# Achtung: if a given -array pattern contains values which themselves +# contains spaces... +# +# define-append foo {"-DFOO=bar baz" -DBAR="baz barre"} +# +# will lead to: +# +# ["-DFOO=bar baz", "-DBAR=\"baz", "barre\""] +# +# Neither is especially satisfactory (and the second is useless), and +# handling of such values is subject to change if any such values ever +# _really_ need to be processed by our source trees. +# +proc proj-dump-defs-json {file args} { + file mkdir [file dirname $file] + set lines {} + lappend args -bare {SIZEOF_* HAVE_DECL_*} -auto HAVE_* + foreach n [lsort [dict keys [all-defines]]] { + set type [proj-defs-type_ $n $args] + set value [proj-defs-format_ $type [get-define $n]] + if {$::proj__Config(defs-skip) ne $value} { + lappend lines "\"$n\": ${value}" + } + } + set buf {} + lappend buf [join $lines ",\n"] + write-if-changed $file $buf { + msg-result "Created $file" + } +} + +# +# @proj-xfer-option-aliases map +# +# Expects a list of pairs of configure flags which have been +# registered with autosetup, in this form: +# +# { alias1 => canonical1 +# aliasN => canonicalN ... } +# +# The names must not have their leading -- part and must be in the +# form which autosetup will expect for passing to [opt-val NAME] and +# friends. +# +# Comment lines are permitted in the input. +# +# For each pair of ALIAS and CANONICAL, if --ALIAS is provided but +# --CANONICAL is not, the value of the former is copied to the +# latter. If --ALIAS is not provided, this is a no-op. If both have +# explicitly been provided a fatal usage error is triggered. +# +# Motivation: autosetup enables "hidden aliases" in [options] lists, +# and elides the aliases from --help output but does no further +# handling of them. For example, when --alias is a hidden alias of +# --canonical and a user passes --alias=X, [opt-val canonical] returns +# no value. i.e. the script must check both [opt-val alias] and +# [opt-val canonical]. The intent here is that this function be +# passed such mappings immediately after [options] is called, to carry +# over any values from hidden aliases into their canonical names, such +# that [opt-value canonical] will return X if --alias=X is passed to +# configure. +# +# That said: autosetup's [opt-str] does support alias forms, but it +# requires that the caller know all possible aliases. It's simpler, in +# terms of options handling, if there's only a single canonical name +# which each down-stream call of [opt-...] has to know. +# +proc proj-xfer-options-aliases {mapping} { + foreach {hidden - canonical} [proj-strip-hash-comments $mapping] { + if {[proj-opt-was-provided $hidden]} { + if {[proj-opt-was-provided $canonical]} { + proj-fatal "both --$canonical and its alias --$hidden were used. Use only one or the other." + } else { + proj-opt-set $canonical [opt-val $hidden] + } + } + } +} + +# +# Arguable/debatable... +# +# When _not_ cross-compiling and CC_FOR_BUILD is _not_ explicitly +# specified, force CC_FOR_BUILD to be the same as CC, so that: +# +# ./configure CC=clang +# +# will use CC_FOR_BUILD=clang, instead of cc, for building in-tree +# tools. This is based off of an email discussion and is thought to +# be likely to cause less confusion than seeing 'cc' invocations +# when when the user passes CC=clang. +# +# Sidebar: if we do this before the cc package is installed, it gets +# reverted by that package. Ergo, the cc package init will tell the +# user "Build C compiler...cc" shortly before we tell them otherwise. +# +proc proj-redefine-cc-for-build {} { + if {![proj-is-cross-compiling] + && [get-define CC] ne [get-define CC_FOR_BUILD] + && "nope" eq [get-env CC_FOR_BUILD "nope"]} { + user-notice "Re-defining CC_FOR_BUILD to CC=[get-define CC]. To avoid this, explicitly pass CC_FOR_BUILD=..." + define CC_FOR_BUILD [get-define CC] + } +} + +# +# @proj-which-linenoise headerFile +# +# Attempts to determine whether the given linenoise header file is of +# the "antirez" or "msteveb" flavor. It returns 2 for msteveb, else 1 +# (it does not validate that the header otherwise contains the +# linenoise API). +# +proc proj-which-linenoise {dotH} { + set srcHeader [proj-file-content $dotH] + if {[string match *userdata* $srcHeader]} { + return 2 + } else { + return 1 + } +} + +# +# @proj-remap-autoconf-dir-vars +# +# "Re-map" the autoconf-conventional --XYZdir flags into something +# which is more easily overridable from a make invocation. +# +# Based off of notes in . +# +# Consider: +# +# $ ./configure --prefix=/foo +# $ make install prefix=/blah +# +# In that make invocation, $(libdir) would, at make-time, normally be +# hard-coded to /foo/lib, rather than /blah/lib. That happens because +# autosetup exports conventional $prefix-based values for the numerous +# autoconfig-compatible XYZdir vars at configure-time. What we would +# normally want, however, is that --libdir derives from the make-time +# $(prefix). The distinction between configure-time and make-time is +# the significant factor there. +# +# This function attempts to reconcile those vars in such a way that +# they will derive, at make-time, from $(prefix) in a conventional +# manner unless they are explicitly overridden at configure-time, in +# which case those overrides takes precedence. +# +# Each autoconf-relvant --XYZ flag which is explicitly passed to +# configure is exported as-is, as are those which default to some +# top-level system directory, e.g. /etc or /var. All which derive +# from either $prefix or $exec_prefix are exported in the form of a +# Makefile var reference, e.g. libdir=${exec_prefix}/lib. Ergo, if +# --exec-prefix=FOO is passed to configure, libdir will still derive, +# at make-time, from whatever exec_prefix is passed to make, and will +# use FOO if exec_prefix is not overridden at make-time. Without this +# post-processing, libdir would be cemented in as FOO/lib at +# configure-time, so could be tedious to override properly via a make +# invocation. +# +proc proj-remap-autoconf-dir-vars {} { + set prefix [get-define prefix] + set exec_prefix [get-define exec_prefix $prefix] + # The following var derefs must be formulated such that they are + # legal for use in (A) makefiles, (B) pkgconfig files, and (C) TCL's + # [subst] command. i.e. they must use the form ${X}. + foreach {flag makeVar makeDeref} { + exec-prefix exec_prefix ${prefix} + datadir datadir ${prefix}/share + mandir mandir ${datadir}/man + includedir includedir ${prefix}/include + bindir bindir ${exec_prefix}/bin + libdir libdir ${exec_prefix}/lib + sbindir sbindir ${exec_prefix}/sbin + sysconfdir sysconfdir /etc + sharedstatedir sharedstatedir ${prefix}/com + localstatedir localstatedir /var + runstatedir runstatedir /run + infodir infodir ${datadir}/info + libexecdir libexecdir ${exec_prefix}/libexec + } { + if {[proj-opt-was-provided $flag]} { + define $makeVar [join [opt-val $flag]] + } else { + define $makeVar [join $makeDeref] + } + # Maintenance reminder: the [join] call is to avoid {braces} + # around the output when someone passes in, + # e.g. --libdir=\${prefix}/foo/bar. Debian's SQLite package build + # script does that. + } +} + +# +# @proj-env-file flag ?default? +# +# If a file named .env-$flag exists, this function returns a +# trimmed copy of its contents, else it returns $dflt. The intended +# usage is that things like developer-specific CFLAGS preferences can +# be stored in .env-CFLAGS. +# +proc proj-env-file {flag {dflt ""}} { + set fn ".env-${flag}" + if {[file readable $fn]} { + return [proj-file-content -trim $fn] + } + return $dflt +} + +# +# @proj-get-env var ?default? +# +# Extracts the value of "environment" variable $var from the first of +# the following places where it's defined: +# +# - Passed to configure as $var=... +# - Exists as an environment variable +# - A file named .env-$var (see [proj-env-file]) +# +# If none of those are set, $dflt is returned. +# +proc proj-get-env {var {dflt ""}} { + get-env $var [proj-env-file $var $dflt] +} + +# +# @proj-scope ?lvl? +# +# Returns the name of the _calling_ proc from ($lvl + 1) levels up the +# call stack (where the caller's level will be 1 up from _this_ +# call). If $lvl would resolve to global scope "global scope" is +# returned and if it would be negative then a string indicating such +# is returned (as opposed to throwing an error). +# +proc proj-scope {{lvl 0}} { + #uplevel [expr {$lvl + 1}] {lindex [info level 0] 0} + set ilvl [info level] + set offset [expr {$ilvl - $lvl - 1}] + if { $offset < 0} { + return "invalid scope ($offset)" + } elseif { $offset == 0} { + return "global scope" + } else { + return [lindex [info level $offset] 0] + } +} + +# +# Deprecated name of [proj-scope]. +# +proc proj-current-scope {{lvl 0}} { + puts stderr \ + "Deprecated proj-current-scope called from [proj-scope 1]. Use proj-scope instead." + proj-scope [incr lvl] +} + +# +# Converts parts of tclConfig.sh to autosetup [define]s. +# +# Expects to be passed the name of a value tclConfig.sh or an empty +# string. It converts certain parts of that file's contents to +# [define]s (see the code for the whole list). If $tclConfigSh is an +# empty string then it [define]s the various vars as empty strings. +# +proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { + set shBody {} + set tclVars { + TCL_INCLUDE_SPEC + TCL_LIBS + TCL_LIB_SPEC + TCL_STUB_LIB_SPEC + TCL_EXEC_PREFIX + TCL_PREFIX + TCL_VERSION + TCL_MAJOR_VERSION + TCL_MINOR_VERSION + TCL_PACKAGE_PATH + TCL_PATCH_LEVEL + TCL_SHLIB_SUFFIX + } + # Build a small shell script which proxies the $tclVars from + # $tclConfigSh into autosetup code... + lappend shBody "if test x = \"x${tclConfigSh}\"; then" + foreach v $tclVars { + lappend shBody "$v= ;" + } + lappend shBody "else . \"${tclConfigSh}\"; fi" + foreach v $tclVars { + lappend shBody "echo define $v {\$$v} ;" + } + lappend shBody "exit" + set shBody [join $shBody "\n"] + #puts "shBody=$shBody\n"; exit + eval [exec echo $shBody | sh] +} + +# +# @proj-tweak-default-env-dirs +# +# This function is not useful before [use system] is called to set up +# --prefix and friends. It should be called as soon after [use system] +# as feasible. +# +# For certain target environments, if --prefix is _not_ passed in by +# the user, set the prefix to an environment-specific default. For +# such environments its does [define prefix ...] and [proj-opt-set +# prefix ...], but it does not process vars derived from the prefix, +# e.g. exec-prefix. To do so it is generally necessary to also call +# proj-remap-autoconf-dir-vars late in the config process (immediately +# before ".in" files are filtered). +# +# Similar modifications may be made for --mandir. +# +# Returns >0 if it modifies the environment, else 0. +# +proc proj-tweak-default-env-dirs {} { + set rc 0 + switch -glob -- [get-define host] { + *-haiku { + if {![proj-opt-was-provided prefix]} { + set hdir /boot/home/config/non-packaged + proj-opt-set prefix $hdir + define prefix $hdir + incr rc + } + if {![proj-opt-was-provided mandir]} { + set hdir /boot/system/documentation/man + proj-opt-set mandir $hdir + define mandir $hdir + incr rc + } + } + } + return $rc +} + +# +# @proj-dot-ins-append file ?fileOut ?postProcessScript?? +# +# Queues up an autosetup [make-template]-style file to be processed +# at a later time using [proj-dot-ins-process]. +# +# $file is the input file. If $fileOut is empty then this function +# derives $fileOut from $file, stripping both its directory and +# extension parts. i.e. it defaults to writing the output to the +# current directory (typically $::autosetup(builddir)). +# +# If $postProcessScript is not empty then, during +# [proj-dot-ins-process], it will be eval'd immediately after +# processing the file. In the context of that script, the vars +# $dotInsIn and $dotInsOut will be set to the input and output file +# names. This can be used, for example, to make the output file +# executable or perform validation on its contents: +# +## proj-dot-ins-append my.sh.in my.sh { +## catch {exec chmod u+x $dotInsOut} +## } +# +# See [proj-dot-ins-process], [proj-dot-ins-list] +# +proc proj-dot-ins-append {fileIn args} { + set srcdir $::autosetup(srcdir) + switch -exact -- [llength $args] { + 0 { + lappend fileIn [file rootname [file tail $fileIn]] "" + } + 1 { + lappend fileIn [join $args] "" + } + 2 { + lappend fileIn {*}$args + } + default { + proj-fatal "Too many arguments: $fileIn $args" + } + } + #puts "******* [proj-scope]: adding [llength $fileIn]-length item: $fileIn" + lappend ::proj__Config(dot-in-files) $fileIn +} + +# +# @proj-dot-ins-list +# +# Returns the current list of [proj-dot-ins-append]'d files, noting +# that each entry is a 3-element list of (inputFileName, +# outputFileName, postProcessScript). +# +proc proj-dot-ins-list {} { + return $::proj__Config(dot-in-files) +} + +# +# @proj-dot-ins-process ?-touch? ?-validate? ?-clear? +# +# Each file which has previously been passed to [proj-dot-ins-append] +# is processed, with its passing its in-file out-file names to +# [proj-make-from-dot-in]. +# +# The intent is that a project accumulate any number of files to +# filter and delay their actual filtering until the last stage of the +# configure script, calling this function at that time. +# +# Optional flags: +# +# -touch: gets passed on to [proj-make-from-dot-in] +# +# -validate: after processing each file, before running the file's +# associated script, if any, it runs the file through +# proj-validate-no-unresolved-ats, erroring out if that does. +# +# -clear: after processing, empty the dot-ins list. This effectively +# makes proj-dot-ins-append available for re-use. +# +proc proj-dot-ins-process {args} { + proj-parse-flags args flags { + -touch "" {return "-touch"} + -clear 0 {expr 1} + -validate 0 {expr 1} + } + #puts "args=$args"; parray flags + if {[llength $args] > 0} { + error "Invalid argument to [proj-scope]: $args" + } + foreach f $::proj__Config(dot-in-files) { + proj-assert {3==[llength $f]} \ + "Expecting proj-dot-ins-list to be stored in 3-entry lists. Got: $f" + lassign $f fIn fOut fScript + #puts "DOING $fIn ==> $fOut" + proj-make-from-dot-in {*}$flags(-touch) $fIn $fOut + if {$flags(-validate)} { + proj-validate-no-unresolved-ats $fOut + } + if {"" ne $fScript} { + uplevel 1 [join [list set dotInsIn $fIn \; \ + set dotInsOut $fOut \; \ + eval \{${fScript}\} \; \ + unset dotInsIn dotInsOut]] + } + } + if {$flags(-clear)} { + set ::proj__Config(dot-in-files) [list] + } +} + +# +# @proj-validate-no-unresolved-ats filenames... +# +# For each filename given to it, it validates that the file has no +# unresolved @VAR@ references. If it finds any, it produces an error +# with location information. +# +# Exception: if a filename matches the pattern {*[Mm]ake*} AND a given +# line begins with a # (not including leading whitespace) then that +# line is ignored for purposes of this validation. The intent is that +# @VAR@ inside of makefile comments should not (necessarily) cause +# validation to fail, as it's sometimes convenient to comment out +# sections during development of a configure script and its +# corresponding makefile(s). +# +proc proj-validate-no-unresolved-ats {args} { + foreach f $args { + set lnno 1 + set isMake [string match {*[Mm]ake*} $f] + foreach line [proj-file-content-list $f] { + if {!$isMake || ![string match "#*" [string trimleft $line]]} { + if {[regexp {(@[A-Za-z0-9_\.]+@)} $line match]} { + error "Unresolved reference to $match at line $lnno of $f" + } + } + incr lnno + } + } +} + +# +# @proj-first-file-found tgtVar fileList +# +# Searches $fileList for an existing file. If one is found, its name +# is assigned to tgtVar and 1 is returned, else tgtVar is set to "" +# and 0 is returned. +# +proc proj-first-file-found {tgtVar fileList} { + upvar $tgtVar tgt + foreach f $fileList { + if {[file exists $f]} { + set tgt $f + return 1 + } + } + set tgt "" + return 0 +} + +# +# Defines $defName to contain makefile recipe commands for re-running +# the configure script with its current set of $::argv flags. This +# can be used to automatically reconfigure. +# +proc proj-setup-autoreconfig {defName} { + define $defName \ + [join [list \ + cd \"$::autosetup(builddir)\" \ + && [get-define AUTOREMAKE "error - missing @AUTOREMAKE@"]]] +} + +# +# @prop-append-to defineName args... +# +# A proxy for Autosetup's [define-append]. Appends all non-empty $args +# to [define-append $defineName]. +# +proc proj-define-append {defineName args} { + foreach a $args { + if {"" ne $a} { + define-append $defineName {*}$a + } + } +} + +# +# @prod-define-amend ?-p|-prepend? ?-d|-define? defineName args... +# +# A proxy for Autosetup's [define-append]. +# +# Appends all non-empty $args to the define named by $defineName. If +# one of (-p | -prepend) are used it instead prepends them, in their +# given order, to $defineName. +# +# If -define is used then each argument is assumed to be a [define]'d +# flag and [get-define X ""] is used to fetch it. +# +# Re. linker flags: typically, -lXYZ flags need to be in "reverse" +# order, with each -lY resolving symbols for -lX's to its left. This +# order is largely historical, and not relevant on all environments, +# but it is technically correct and still relevant on some +# environments. +# +# See: proj-append-to +# +proc proj-define-amend {args} { + set defName "" + set prepend 0 + set isdefs 0 + set xargs [list] + foreach arg $args { + switch -exact -- $arg { + "" {} + -p - -prepend { incr prepend } + -d - -define { incr isdefs } + default { + if {"" eq $defName} { + set defName $arg + } else { + lappend xargs $arg + } + } + } + } + if {"" eq $defName} { + proj-error "Missing defineName argument in call from [proj-scope 1]" + } + if {$isdefs} { + set args $xargs + set xargs [list] + foreach arg $args { + lappend xargs [get-define $arg ""] + } + set args $xargs + } +# puts "**** args=$args" +# puts "**** xargs=$xargs" + + set args $xargs + if {$prepend} { + lappend args {*}[get-define $defName ""] + define $defName [join $args]; # join to eliminate {} entries + } else { + proj-define-append $defName {*}$args + } +} + +# +# @proj-define-to-cflag ?-list? ?-quote? ?-zero-undef? defineName... +# +# Treat each argument as the name of a [define] and renders it like a +# CFLAGS value in one of the following forms: +# +# -D$name +# -D$name=integer (strict integer matches only) +# '-D$name=value' (without -quote) +# '-D$name="value"' (with -quote) +# +# It treats integers as numbers and everything else as a quoted +# string, noting that it does not handle strings which themselves +# contain quotes. +# +# The -zero-undef flag causes no -D to be emitted for integer values +# of 0. +# +# By default it returns the result as string of all -D... flags, +# but if passed the -list flag it will return a list of the +# individual CFLAGS. +# +proc proj-define-to-cflag {args} { + set rv {} + proj-parse-flags args flags { + -list 0 {expr 1} + -quote 0 {expr 1} + -zero-undef 0 {expr 1} + } + foreach d $args { + set v [get-define $d ""] + set li {} + if {"" eq $d} { + set v "-D${d}" + } elseif {[string is integer -strict $v]} { + if {!$flags(-zero-undef) || $v ne "0"} { + set v "-D${d}=$v" + } + } elseif {$flags(-quote)} { + set v "'-D${d}=\"$v\"'" + } else { + set v "'-D${d}=$v'" + } + lappend rv $v + } + expr {$flags(-list) ? $rv : [join $rv]} +} + + +if {0} { + # Turns out that autosetup's [options-add] essentially does exactly + # this... + + # A list of lists of Autosetup [options]-format --flags definitions. + # Append to this using [proj-options-add] and use + # [proj-options-combine] to merge them into a single list for passing + # to [options]. + # + set ::proj__Config(extra-options) {} + + # @proj-options-add list + # + # Adds a list of options to the pending --flag processing. It must be + # in the format used by Autosetup's [options] function. + # + # This will have no useful effect if called from after [options] + # is called. + # + # Use [proj-options-combine] to get a combined list of all added + # options. + # + # PS: when writing this i wasn't aware of autosetup's [options-add], + # works quite similarly. Only the timing is different. + proc proj-options-add {list} { + lappend ::proj__Config(extra-options) $list + } + + # @proj-options-combine list1 ?...listN? + # + # Expects each argument to be a list of options compatible with + # autosetup's [options] function. This function concatenates the + # contents of each list into a new top-level list, stripping the outer + # list part of each argument, and returning that list + # + # If passed no arguments, it uses the list generated by calls to + # [proj-options-add]. + proc proj-options-combine {args} { + set rv [list] + if {0 == [llength $args]} { + set args $::proj__Config(extra-options) + } + foreach e $args { + lappend rv {*}$e + } + return $rv + } +}; # proj-options-* + +# Internal cache for use via proj-cache-*. +array set proj__Cache {} + +# +# @proj-cache-key arg {addLevel 0} +# +# Helper to generate cache keys for [proj-cache-*]. +# +# $addLevel should almost always be 0. +# +# Returns a cache key for the given argument: +# +# integer: relative call stack levels to get the scope name of for +# use as a key. [proj-scope [expr {1 + $arg + addLevel}]] is +# then used to generate the key. i.e. the default of 0 uses the +# calling scope's name as the key. +# +# Anything else: returned as-is +# +proc proj-cache-key {arg {addLevel 0}} { + if {[string is integer -strict $arg]} { + return [proj-scope [expr {$arg + $addLevel + 1}]] + } + return $arg +} + +# +# @proj-cache-set ?-key KEY? ?-level 0? value +# +# Sets a feature-check cache entry with the given key. +# +# See proj-cache-key for -key's and -level's semantics, noting that +# this function adds one to -level for purposes of that call. +proc proj-cache-set {args} { + proj-parse-flags args flags { + -key => 0 + -level => 0 + } + lassign $args val + set key [proj-cache-key $flags(-key) [expr {1 + $flags(-level)}]] + #puts "** fcheck set $key = $val" + set ::proj__Cache($key) $val +} + +# +# @proj-cache-remove ?key? ?addLevel? +# +# Removes an entry from the proj-cache. +proc proj-cache-remove {{key 0} {addLevel 0}} { + set key [proj-cache-key $key [expr {1 + $addLevel}]] + set rv "" + if {[info exists ::proj__Cache($key)]} { + set rv $::proj__Cache($key) + unset ::proj__Cache($key) + } + return $rv; +} + +# +# @proj-cache-check ?-key KEY? ?-level LEVEL? tgtVarName +# +# Checks for a feature-check cache entry with the given key. +# +# If the feature-check cache has a matching entry then this function +# assigns its value to tgtVar and returns 1, else it assigns tgtVar to +# "" and returns 0. +# +# See proj-cache-key for $key's and $addLevel's semantics, noting that +# this function adds one to $addLevel for purposes of that call. +proc proj-cache-check {args} { + proj-parse-flags args flags { + -key => 0 + -level => 0 + } + lassign $args tgtVar + upvar $tgtVar tgt + set rc 0 + set key [proj-cache-key $flags(-key) [expr {1 + $flags(-level)}]] + #puts "** fcheck get key=$key" + if {[info exists ::proj__Cache($key)]} { + set tgt $::proj__Cache($key) + incr rc + } else { + set tgt "" + } + return $rc +} + +# +# @proj-coalesce ...args +# +# Returns the first argument which is not empty (eq ""), or an empty +# string on no match. +proc proj-coalesce {args} { + foreach arg $args { + if {"" ne $arg} { + return $arg + } + } + return "" +} + +# +# @proj-parse-flags argvListName targetArrayName {prototype} +# +# A helper to parse flags from proc argument lists. +# +# The first argument is the name of a var holding the args to +# parse. It will be overwritten, possibly with a smaller list. +# +# The second argument is the name of an array variable to create in +# the caller's scope. +# +# The third argument, $prototype, is a description of how to handle +# the flags. Each entry in that list must be in one of the +# following forms: +# +# -flag defaultValue ?-literal|-call|-apply? +# script|number|incr|proc-name|{apply $aLambda} +# +# -flag* ...as above... +# +# -flag => defaultValue ?-call proc-name-and-args|-apply lambdaExpr? +# +# -flag* => ...as above... +# +# :PRAGMA +# +# The first two forms represents a basic flag with no associated +# following argument. The third and fourth forms, called arg-consuming +# flags, extract the value from the following argument in $argvName +# (pneumonic: => points to the next argument.). The :PRAGMA form +# offers a way to configure certain aspects of this call. +# +# If $argv contains any given flag from $prototype, its default value +# is overridden depending on several factors: +# +# - If the -literal flag is used, or the flag's script is a number, +# value is used verbatim. +# +# - Else if the -call flag is used, the argument must be a proc name +# and any leading arguments, e.g. {apply $myLambda}. The proc is passed +# the (flag, value) as arguments (non-consuming flags will get +# passed the flag's current/starting value and consuming flags will +# get the next argument). Its result becomes the result of the +# flag. +# +# - Else if -apply X is used, it's effectively shorthand for -call +# {apply X}. Its argument may either be a $lambaRef or a {{f v} +# {body}} construct. +# +# - Else if $script is one of the following values, it is treated as +# the result of... +# +# - incr: increments the current value of the flag. +# +# - Else $script is eval'd to get its result value. That result +# becomes the new flag value for $tgtArrayName(-flag). This +# function intercepts [return $val] from eval'ing $script. Any +# empty script will result in the flag having "" assigned to it. +# +# Unless the -flag has a trailing asterisk, e.g. -flag*, this function +# assumes that each flag is unique, and using a flag more than once +# causes an error to be triggered. the -flag* forms works similarly +# except that may appear in $argv any number of times: +# +# - For non-arg-consuming flags, each invocation of -flag causes the +# result of $script to overwrite the previous value. e.g. so +# {-flag* {x} {incr foo}} has a default value of x, but passing in +# -flag twice would change it to the result of incrementing foo +# twice. This form can be used to implement, e.g., increasing +# verbosity levels by passing -verbose multiple times. +# +# - For arg-consuming flags, the given flag starts with value X, but +# if the flag is provided in $argv, the default is cleared, then +# each instance of -flag causes its value to be appended to the +# result, so {-flag* => {a b c}} defaults to {a b c}, but passing +# in -flag y -flag z would change it to {y z}, not {a b c y z}.. +# +# By default, the args list is only inspected until the first argument +# which is not described by $prototype. i.e. the first "non-flag" (not +# counting values consumed for flags defined like -flag => default). +# The :all-flags pragma (see below) can modify this behavior. +# +# If a "--" flag is encountered, no more arguments are inspected as +# flags unless the :all-flags pragma (see below) is in effect. The +# first instance of "--" is removed from the target result list but +# all remaining instances of "--" are are passed through. +# +# Any argvName entries not described in $prototype are considered to +# be "non-flags" for purposes of this function, even if they +# ostensibly look like flags. +# +# Returns the number of flags it processed in $argvName, not counting +# "--". +# +# Example: +# +## set args [list -foo -bar {blah} -z 8 9 10 -theEnd] +## proj-parse-flags args flags { +## -foo 0 {expr 1} +## -bar => 0 +## -no-baz 1 {return 0} +## -z 0 2 +## } +# +# After that $flags would contain {-foo 1 -bar {blah} -no-baz 1 -z 2} +# and $args would be {8 9 10 -theEnd}. +# +# Pragmas: +# +# Passing :PRAGMAS to this function may modify how it works. The +# following pragmas are supported (note the leading ":"): +# +# :all-flags indicates that the whole input list should be scanned, +# not stopping at the first non-flag or "--". +# +proc proj-parse-flags {argvName tgtArrayName prototype} { + upvar $argvName argv + upvar $tgtArrayName outFlags + array set flags {}; # staging area + array set blob {}; # holds markers for various per-key state and options + set incrSkip 1; # 1 if we stop at the first non-flag, else 0 + # Parse $prototype for flag definitions... + set n [llength $prototype] + set checkProtoFlag { + #puts "**** checkProtoFlag #$i of $n k=$k fv=$fv" + switch -exact -- $fv { + -literal { + proj-assert {![info exists blob(${k}.consumes)]} + set blob(${k}.script) [list expr [lindex $prototype [incr i]]] + } + -apply { + set fv [lindex $prototype [incr i]] + if {2 == [llength $fv]} { + # Treat this as a lambda literal + set fv [list $fv] + } + lappend blob(${k}.call) "apply $fv" + } + -call { + # arg is either a proc name or {apply $aLambda} + set fv [lindex $prototype [incr i]] + lappend blob(${k}.call) $fv + } + default { + proj-assert {![info exists blob(${k}.consumes)]} + set blob(${k}.script) $fv + } + } + if {$i >= $n} { + proj-error -up "[proj-scope]: Missing argument for $k flag" + } + } + for {set i 0} {$i < $n} {incr i} { + set k [lindex $prototype $i] + #puts "**** #$i of $n k=$k" + + # Check for :PRAGMA... + switch -exact -- $k { + :all-flags { + set incrSkip 0 + continue + } + } + + proj-assert {[string match -* $k]} \ + "Invalid argument: $k" + + if {[string match {*\*} $k]} { + # Re-map -foo* to -foo and flag -foo as a repeatable flag + set k [string map {* ""} $k] + incr blob(${k}.multi) + } + + if {[info exists flags($k)]} { + proj-error -up "[proj-scope]: Duplicated prototype for flag $k" + } + + switch -exact -- [lindex $prototype [expr {$i + 1}]] { + => { + # -flag => DFLT ?-subflag arg? + incr i 2 + if {$i >= $n} { + proj-error -up "[proj-scope]: Missing argument for $k => flag" + } + incr blob(${k}.consumes) + set vi [lindex $prototype $i] + if {$vi in {-apply -call}} { + proj-error -up "[proj-scope]: Missing default value for $k flag" + } else { + set fv [lindex $prototype [expr {$i + 1}]] + if {$fv in {-apply -call}} { + incr i + eval $checkProtoFlag + } + } + } + default { + # -flag VALUE ?flag? SCRIPT + set vi [lindex $prototype [incr i]] + set fv [lindex $prototype [incr i]] + eval $checkProtoFlag + } + } + #puts "**** #$i of $n k=$k vi=$vi" + set flags($k) $vi + } + #puts "-- flags"; parray flags + #puts "-- blob"; parray blob + set rc 0 + set rv {}; # staging area for the target argv value + set skipMode 0 + set n [llength $argv] + # Now look for those flags in $argv... + for {set i 0} {$i < $n} {incr i} { + set arg [lindex $argv $i] + #puts "-- [proj-scope] arg=$arg" + if {$skipMode} { + lappend rv $arg + } elseif {"--" eq $arg} { + # "--" is the conventional way to end processing of args + if {[incr blob(--)] > 1} { + # Elide only the first one + lappend rv $arg + } + incr skipMode $incrSkip + } elseif {[info exists flags($arg)]} { + # A known flag... + set isMulti [info exists blob(${arg}.multi)] + incr blob(${arg}.seen) + if {1 < $blob(${arg}.seen) && !$isMulti} { + proj-error -up [proj-scope] "$arg flag was used multiple times" + } + set vMode 0; # 0=as-is, 1=eval, 2=call + set isConsuming [info exists blob(${arg}.consumes)] + if {$isConsuming} { + incr i + if {$i >= $n} { + proj-error -up [proj-scope] "is missing argument for $arg flag" + } + set vv [lindex $argv $i] + } elseif {[info exists blob(${arg}.script)]} { + set vMode 1 + set vv $blob(${arg}.script) + } else { + set vv $flags($arg) + } + + if {[info exists blob(${arg}.call)]} { + set vMode 2 + set vv [concat {*}$blob(${arg}.call) $arg $vv] + } elseif {$isConsuming} { + proj-assert {!$vMode} + # fall through + } elseif {"" eq $vv || [string is double -strict $vv]} { + set vMode 0 + } elseif {$vv in {incr}} { + set vMode 0 + switch -exact $vv { + incr { + set xx $flags($k); incr xx; set vv $xx; unset xx + } + default { + proj-error "Unhandled \$vv value $vv" + } + } + } else { + set vv [list eval $vv] + set vMode 1 + } + if {$vMode} { + set code [catch [list uplevel 1 $vv] vv xopt] + if {$code ni {0 2}} { + return {*}$xopt $vv + } + } + if {$isConsuming && $isMulti} { + if {1 == $blob(${arg}.seen)} { + # On the first hit, overwrite the default with a new list. + set flags($arg) [list $vv] + } else { + # On subsequent hits, append to the list. + lappend flags($arg) $vv + } + } else { + set flags($arg) $vv + } + incr rc + } else { + # Non-flag + incr skipMode $incrSkip + lappend rv $arg + } + } + set argv $rv + array set outFlags [array get flags] + #puts "-- rv=$rv argv=$argv flags="; parray flags + return $rc +}; # proj-parse-flags + +# +# Older (deprecated) name of proj-parse-flags. +# +proc proj-parse-simple-flags {args} { + tailcall proj-parse-flags {*}$args +} + +if {$::proj__Config(self-tests)} { + set __ova $::proj__Config(verbose-assert); + set ::proj__Config(verbose-assert) 1 + puts "Running [info script] self-tests..." + # proj-cache... + apply {{} { + #proj-warn "Test code for proj-cache" + proj-assert {![proj-cache-check -key here check]} + proj-assert {"here" eq [proj-cache-key here]} + proj-assert {"" eq $check} + proj-cache-set -key here thevalue + proj-assert {[proj-cache-check -key here check]} + proj-assert {"thevalue" eq $check} + + proj-assert {![proj-cache-check check]} + #puts "*** key = ([proj-cache-key 0])" + proj-assert {"" eq $check} + proj-cache-set abc + proj-assert {[proj-cache-check check]} + proj-assert {"abc" eq $check} + + #parray ::proj__Cache; + proj-assert {"" ne [proj-cache-remove]} + proj-assert {![proj-cache-check check]} + proj-assert {"" eq [proj-cache-remove]} + proj-assert {"" eq $check} + }} + + # proj-parse-flags ... + apply {{} { + set foo 3 + set argv {-a "hi - world" -b -b -b -- -a {bye bye} -- -d -D c -a "" --} + proj-parse-flags argv flags { + :all-flags + -a* => "gets overwritten" + -b* 7 {incr foo} + -d 1 0 + -D 0 1 + } + + #puts "-- argv = $argv"; parray flags; + proj-assert {"-- c --" eq $argv} + proj-assert {$flags(-a) eq "{hi - world} {bye bye} {}"} + proj-assert {$foo == 6} + proj-assert {$flags(-b) eq $foo} + proj-assert {$flags(-d) == 0} + proj-assert {$flags(-D) == 1} + set foo 0 + foreach x $flags(-a) { + proj-assert {$x in {{hi - world} {bye bye} {}}} + incr foo + } + proj-assert {3 == $foo} + + set argv {-a {hi world} -b -maybe -- -a {bye bye} -- -b c --} + set foo 0 + proj-parse-flags argv flags { + -a => "aaa" + -b 0 {incr foo} + -maybe no -literal yes + } + #parray flags; puts "--- argv = $argv" + proj-assert {"-a {bye bye} -- -b c --" eq $argv} + proj-assert {$flags(-a) eq "hi world"} + proj-assert {1 == $flags(-b)} + proj-assert {"yes" eq $flags(-maybe)} + + set argv {-f -g -a aaa -M -M -M -L -H -A AAA a b c} + set foo 0 + set myLambda {{flag val} { + proj-assert {$flag in {-f -g -M}} + #puts "myLambda flag=$flag val=$val" + incr val + }} + proc myNonLambda {flag val} { + proj-assert {$flag in {-A -a}} + #puts "myNonLambda flag=$flag val=$val" + concat $val $val + } + proj-parse-flags argv flags { + -f 0 -call {apply $myLambda} + -g 2 -apply $myLambda + -h 3 -apply $myLambda + -H 30 33 + -a => aAAAa -apply {{f v} { + set v + }} + -A => AaaaA -call myNonLambda + -B => 17 -call myNonLambda + -M* 0 -apply $myLambda + -L "" -literal $myLambda + } + rename myNonLambda "" + #puts "--- argv = $argv"; parray flags + proj-assert {$flags(-f) == 1} + proj-assert {$flags(-g) == 3} + proj-assert {$flags(-h) == 3} + proj-assert {$flags(-H) == 33} + proj-assert {$flags(-a) == {aaa}} + proj-assert {$flags(-A) eq "AAA AAA"} + proj-assert {$flags(-B) == 17} + proj-assert {$flags(-M) == 3} + proj-assert {$flags(-L) eq $myLambda} + + set argv {-touch -validate} + proj-parse-flags argv flags { + -touch "" {return "-touch"} + -validate 0 1 + } + #puts "----- argv = $argv"; parray flags + proj-assert {$flags(-touch) eq "-touch"} + proj-assert {$flags(-validate) == 1} + proj-assert {$argv eq {}} + + set argv {-i -i -i} + proj-parse-flags argv flags { + -i* 0 incr + } + proj-assert {3 == $flags(-i)} + }} + set ::proj__Config(verbose-assert) $__ova + unset __ova + puts "Done running [info script] self-tests." +}; # proj- API self-tests diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl new file mode 100644 index 0000000000..7c798b31a2 --- /dev/null +++ b/autosetup/sqlite-config.tcl @@ -0,0 +1,2237 @@ +# This file holds functions for autosetup which are specific to the +# sqlite build tree. They are in this file, instead of auto.def, so +# that they can be reused in the autoconf sub-tree. This file requires +# functions from the project-agnostic proj.tcl. + +if {[string first " " $autosetup(srcdir)] != -1} { + user-error "The pathname of the source tree\ + may not contain space characters" +} +if {[string first " " $autosetup(builddir)] != -1} { + user-error "The pathname of the build directory\ + may not contain space characters" +} + +use proj +# +# We want the package version info to be emitted early on, but doing +# so requires a bit of juggling. We have to [use system] for +# --prefix=... to work and to emit the Host/Build system info, but we +# don't want those to interfere with --help output. +define PACKAGE_VERSION [proj-file-content -trim $::autosetup(srcdir)/VERSION] +if {"--help" ni $::argv} { + msg-result "Configuring SQLite version [get-define PACKAGE_VERSION]" +} +use system ; # Will output "Host System" and "Build System" lines +if {"--help" ni $::argv} { + proj-tweak-default-env-dirs + msg-result "Source dir = $::autosetup(srcdir)" + msg-result "Build dir = $::autosetup(builddir)" + use cc cc-db cc-shared cc-lib pkg-config +} + +# +# Object for communicating certain config-time state across various +# auto.def-related pieces. +array set sqliteConfig [subst [proj-strip-hash-comments { + # + # Gets set by [sqlite-configure] (the main configure script driver). + build-mode unknown + # + # Gets set to 1 when using jimsh for code generation. May affect + # later decisions. + use-jim-for-codegen 0 + # + # Set to 1 when cross-compiling This value may be changed by certain + # build options, so it's important that config code which checks for + # cross-compilation uses this var instead of + # [proj-is-cross-compiling]. + is-cross-compiling [proj-is-cross-compiling] + # + # Pass msg-debug=1 to configure to enable obnoxiously loud output + # from [msg-debug]. + msg-debug-enabled 0 + # + # Output file for --dump-defines. Intended only for build debugging + # and not part of the public build interface. + dump-defines-txt ./config.defines.txt + # + # If not empty then --dump-defines will dump not only + # (dump-defines-txt) but also a JSON file named after this option's + # value. + dump-defines-json "" + + # + # The list of feature --flags which the --all flag implies. This + # requires special handling in a few places. + # + all-flag-enables {fts4 fts5 rtree geopoly session dbpage dbstat carray} + + # + # Default value for the --all flag. Can hypothetically be modified + # by non-canonical builds (it was added for a Tcl extension build + # mode which was eventually removed). + # + all-flag-default 0 +}]] + +######################################################################## +# Processes all configure --flags for this build, run build-specific +# config checks, then finalize the configure process. $buildMode must +# be one of (canonical, autoconf), and others may be added in the +# future. After bootstrapping, $configScript is eval'd in the caller's +# scope, then post-configuration finalization is run. $configScript is +# intended to hold configure code which is specific to the given +# $buildMode, with the caveat that _some_ build-specific code is +# encapsulated in the configuration finalization step. +# +# The intent is that all (or almost all) build-mode-specific +# configuration goes inside the $configScript argument to this +# function, and that an auto.def file contains only two commands: +# +# use sqlite-config +# sqlite-configure BUILD_NAME { build-specific configure script } +# +# There are snippets of build-mode-specific decision-making in +# [sqlite-configure-finalize], which gets run after $configScript. +proc sqlite-configure {buildMode configScript} { + proj-assert {$::sqliteConfig(build-mode) eq "unknown"} \ + "sqlite-configure must not be called more than once" + set allBuildModes {canonical autoconf} + if {$buildMode ni $allBuildModes} { + user-error "Invalid build mode: $buildMode. Expecting one of: $allBuildModes" + } + if {$::sqliteConfig(all-flag-default)} { + set allFlagHelp "Disable these extensions: $::sqliteConfig(all-flag-enables)" + } else { + set allFlagHelp "Enable these extensions: $::sqliteConfig(all-flag-enables)" + } + + set ::sqliteConfig(build-mode) $buildMode + ######################################################################## + # A gentle introduction to flags handling in autosetup + # + # Reference: https://msteveb.github.io/autosetup/developer/ + # + # All configure flags must be described in one or more calls to + # autosetup's [options] and [options-add] functions. The general + # syntax of the single argument to those functions is a list contain + # a mapping of flags to help text: + # + # FLAG => {Help text} + # + # Where FLAG can have any of the following formats: + # + # boolopt => "a boolean option which defaults to disabled" + # boolopt2=1 => "a boolean option which defaults to enabled" + # stringopt: => "an option which takes an argument, e.g. --stringopt=value" + # stringopt:DESCR => As for stringopt: with a description for the value + # stringopt2:=value => "an option where the argument is optional and defaults to 'value'" + # optalias booltopt3 => "a boolean with a hidden alias. --optalias is not shown in --help" + # + # Autosetup does no small amount of specialized handling for flags, + # especially booleans. Each bool-type --FLAG implicitly gets + # --enable-FLAG and --disable-FLAG forms. That can lead lead to some + # confusion when writing help text. For example: + # + # options { json=1 {Disable JSON functions} } + # + # The reason the help text says "disable" is because a boolean option + # which defaults to true is, in the --help text, rendered as: + # + # --disable-json Disable JSON functions + # + # Whereas a bool flag which defaults to false will instead render as: + # + # --enable-FLAG + # + # Non-boolean flags, in contrast, use the names specifically given to + # them in the [options] invocation. e.g. "with-tcl" is the --with-tcl + # flag. + # + # Fetching values for flags: + # + # booleans: use one of: + # - [opt-bool FLAG] is autosetup's built-in command for this, but we + # have some convenience variants: + # - [proj-opt-truthy FLAG] + # - [proj-opt-if-truthy FLAG {THEN} {ELSE}] + # + # Non-boolean (i.e. string) flags: + # - [opt-val FLAG ?default?] + # - [opt-str ...] - see the docs in ./autosetup/autosetup + # + # [proj-opt-was-provided] can be used to determine whether a flag was + # explicitly provided, which is often useful for distinguishing from + # the case of a default value. + ######################################################################## + set allFlags { + # Structure: a list of M {Z} pairs, where M is a descriptive + # option group name and Z is a list of X Y pairs. X is a list of + # $buildMode name(s) to which the Y flags apply, or {*} to apply + # to all builds. Y is a {block} in the form expected by + # autosetup's [options] and [options-add] command. Each block + # which is applicable to $buildMode is passed on to + # [options-add]. The order of each Y and sub-Y is retained, which + # is significant for rendering of --help. + # + # Maintenance note: [options] does not support comments in + # options, but we filter this object through + # [proj-strip-hash-comments] to remove them before passing them on + # to [options]. + + # When writing {help text blocks}, be aware that: + # + # A) autosetup formats them differently if the {block} starts with + # a newline: it starts left-aligned, directly under the --flag, and + # the rest of the block is pasted verbatim rather than + # pretty-printed. + # + # B) Vars and commands are NOT expanded, but we use a [subst] call + # below which will replace (only) $var refs. + + # Options for how to build the library + build-modes { + {canonical autoconf} { + shared=1 => {Disable build of shared library} + static=1 => {Disable build of static library} + } + {canonical} { + amalgamation=1 => {Disable the amalgamation and instead build all files separately} + } + } + + # Library-level features and defaults + lib-features { + {*} { + threadsafe=1 => {Disable mutexing} + with-tempstore:=no => {Use an in-RAM database for temporary tables: never,no,yes,always} + load-extension=1 => {Disable loading of external extensions} + # ^^^ one of the downstream custom builds overrides the load-extension default to 0, which + # confuses the --help text generator. https://github.com/msteveb/autosetup/issues/77 + math=1 => {Disable math functions} + json=1 => {Disable JSON functions} + memsys5 => {Enable MEMSYS5} + memsys3 => {Enable MEMSYS3} + fts3 => {Enable the FTS3 extension} + fts4 => {Enable the FTS4 extension} + fts5 => {Enable the FTS5 extension} + update-limit => {Enable the UPDATE/DELETE LIMIT clause} + geopoly => {Enable the GEOPOLY extension} + rtree => {Enable the RTREE extension} + session => {Enable the SESSION extension} + dbpage => {Enable the sqlite3_dbpage extension} + dbstat => {Enable the sqlite3_dbstat extension} + carray=1 => {Disable the CARRAY extension} + all=$::sqliteConfig(all-flag-default) => {$allFlagHelp} + largefile=1 + => {This legacy flag has no effect on the library but may influence + the generated sqlite_cfg.h by adding #define HAVE_LFS} + } + {canonical} { + column-metadata => {Enable the column metadata APIs} + # ^^^ Affects how sqlite3.c is generated, so is not available in + # the autoconf build. + } + } + + # Options for TCL support + tcl { + {canonical} { + tcl=1 + => {Disable components which require TCL, including all tests. + This tree requires TCL for code generation but can use the in-tree + copy of autosetup/jimsh0.c for that. The SQLite TCL extension and the + test code require a canonical tclsh.} + with-tcl:DIR + => {Directory containing tclConfig.sh or a directory one level up from + that, from which we can derive a directory containing tclConfig.sh. + A dir name of "prefix" is equivalent to the directory specified by + the --prefix flag.} + with-tclsh:PATH + => {Full pathname of tclsh to use. It is used for (A) trying to find + tclConfig.sh and (B) all TCL-based code generation. Use --with-tcl + unless you have a specific need for this flag. Warning: if its + containing dir has multiple tclsh versions, it may select the + wrong tclConfig.sh!} + static-tclsqlite3=0 + => {Statically-link tclsqlite3. This only works if TCL support is + enabled and all requisite libraries are available in + static form. Note that glibc is unable to fully statically + link certain libraries required by tclsqlite3, so this won't + work on most Linux environments.} + } + } + + # Options for line-editing modes for the CLI shell + line-editing { + {canonical autoconf} { + readline=1 + => {Disable readline support} + # --with-readline-lib is a backwards-compatible alias for + # --with-readline-ldflags + with-readline-lib: + with-readline-ldflags:=auto + => {Readline LDFLAGS, e.g. -lreadline -lncurses} + # --with-readline-inc is a backwards-compatible alias for + # --with-readline-cflags. + with-readline-inc: + with-readline-cflags:=auto + => {Readline CFLAGS, e.g. -I/path/to/includes} + with-readline-header:PATH + => {Full path to readline.h, from which --with-readline-cflags will be derived} + with-linenoise:DIR + => {Source directory for linenoise.c and linenoise.h} + editline=0 + => {Enable BSD editline support} + } + } + + # Options for ICU: International Components for Unicode + icu { + {*} { + with-icu-ldflags:LDFLAGS + => {Enable SQLITE_ENABLE_ICU and add the given linker flags for the + ICU libraries. e.g. on Ubuntu systems, try '-licui18n -licuuc -licudata'.} + with-icu-cflags:CFLAGS + => {Apply extra CFLAGS/CPPFLAGS necessary for building with ICU. + e.g. -I/usr/local/include} + with-icu-config:=auto + => {Enable SQLITE_ENABLE_ICU. Value must be one of: auto, pkg-config, + /path/to/icu-config} + icu-collations=0 + => {Enable SQLITE_ENABLE_ICU_COLLATIONS. Requires --with-icu-ldflags=... + or --with-icu-config} + } + } + + # Options for exotic/alternative build modes + alternative-builds { + {canonical autoconf} { + with-wasi-sdk:=/opt/wasi-sdk + => {Top-most dir of the wasi-sdk for a WASI build} + } + + {*} { + # Note that --static-cli-shell has a completely different + # meaning from --static-shell in the autoconf build! + # --[disable-]static-shell is a legacy flag which we can't + # remove without breaking downstream builds. + static-cli-shell=0 + => {Statically-link the sqlite3 CLI shell. + This only works if the requisite libraries are all available in + static form.} + } + + {canonical} { + static-shells=0 + => {Shorthand for --static-cli-shell --static-tclsqlite3} + + with-emsdk:=auto + => {Top-most dir of the Emscripten SDK installation. + Needed only by ext/wasm. Default=EMSDK env var.} + + amalgamation-extra-src:FILES + => {Space-separated list of source files to append as-is to the resulting + sqlite3.c amalgamation file. May be provided multiple times.} + } + } + + # Options primarily for downstream packagers/package maintainers + packaging { + {autoconf} { + # --disable-static-shell: https://sqlite.org/forum/forumpost/cc219ee704 + # Note that this has a different meaning from --static-cli-shell in the + # canonical build! + static-shell=1 + => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} + } + {canonical autoconf} { + rpath=1 => {Disable use of the rpath linker flag} + # soname: https://sqlite.org/src/forumpost/5a3b44f510df8ded + soname:=legacy + => {SONAME for libsqlite3.so. "none", or not using this flag, sets no + soname. "legacy" sets it to its historical value of + libsqlite3.so.0. A value matching the glob "libsqlite3.*" sets + it to that literal value. Any other value is assumed to be a + suffix which gets applied to "libsqlite3.so.", + e.g. --soname=9.10 equates to "libsqlite3.so.9.10".} + # dll-basename: https://sqlite.org/forum/forumpost/828fdfe904 + dll-basename:=auto + => {Specifies the base name of the resulting DLL file. + If not provided, "libsqlite3" is usually assumed but on some platforms + a platform-dependent default is used. On some platforms this flag + gets automatically enabled if it is not provided. Use "default" to + explicitly disable platform-dependent activation on such systems.} + # out-implib: https://sqlite.org/forum/forumpost/0c7fc097b2 + out-implib:=auto + => {Enable use of --out-implib linker flag to generate an + "import library" for the DLL. The output's base name is + specified by this flag's value, with "auto" meaning to figure + out a name automatically. On some platforms this flag gets + automatically enabled if it is not provided. Use "none" to + explicitly disable this feature on such platforms.} + } + } + + # Options mostly for sqlite's own development + developer { + {*} { + # Note that using the --debug/--enable-debug flag here + # requires patching autosetup/autosetup to rename its builtin + # --debug to --autosetup-debug. See details in + # autosetup/README.md#patching. + with-debug=0 + debug=0 + => {Enable debug build flags. This option will impact performance by + as much as 4x, as it includes large numbers of assert()s in + performance-critical loops. Never use --debug for production + builds.} + scanstatus + => {Enable the SQLITE_ENABLE_STMT_SCANSTATUS feature flag} + } + {canonical} { + dev + => {Enable dev-mode build: automatically enables certain other flags} + test-status + => {Enable status of tests} + gcov=0 + => {Enable coverage testing using gcov} + linemacros + => {Enable #line macros in the amalgamation} + dynlink-tools + => {Dynamically link libsqlite3 to certain tools which normally statically embed it} + asan-fsanitize:=auto + => {Comma- or space-separated list of -fsanitize flags for use with the + fuzzcheck-asan tool. Only those which the compiler claims to support + will actually be used. May be provided multiple times.} + } + {*} { + dump-defines=0 + => {Dump autosetup defines to $::sqliteConfig(dump-defines-txt) + (for build debugging)} + } + } + }; # $allFlags + + set allFlags [proj-strip-hash-comments $allFlags] + # ^^^ lappend of [sqlite-custom-flags] introduces weirdness if + # we delay [proj-strip-hash-comments] until after that. + + ######################################################################## + # sqlite-custom.tcl is intended only for vendor-branch-specific + # customization. See autosetup/README.md#branch-customization for + # details. + if {[file exists $::autosetup(libdir)/sqlite-custom.tcl]} { + uplevel 1 {source $::autosetup(libdir)/sqlite-custom.tcl} + } + + if {[llength [info proc sqlite-custom-flags]] > 0} { + # sqlite-custom-flags is assumed to be imported via + # autosetup/sqlite-custom.tcl. + set scf [sqlite-custom-flags] + if {"" ne $scf} { + lappend allFlags sqlite-custom-flags $scf + } + } + + #lappend allFlags just-testing {{*} {soname:=duplicateEntry => {x}}} + + # Filter allFlags to create the set of [options] legal for this build + foreach {group XY} [subst -nobackslashes -nocommands $allFlags] { + foreach {X Y} $XY { + if { $buildMode in $X || "*" in $X } { + options-add $Y + } + } + } + + if {[catch {options {}} msg xopts]} { + # Workaround for + # where [options] behaves oddly on _some_ TCL builds when it's + # called from deeper than the global scope. + dict incr xopts -level + return {*}$xopts $msg + } + sqlite-configure-phase1 $buildMode + uplevel 1 $configScript + sqlite-configure-finalize +}; # sqlite-configure + +######################################################################## +# Runs "phase 1" of the configure process: after initial --flags +# handling but before sqlite-configure's $configScript argument is +# run. $buildMode must be the mode which was passed to +# [sqlite-configure]. +proc sqlite-configure-phase1 {buildMode} { + define PACKAGE_NAME sqlite + define PACKAGE_URL {https://sqlite.org} + define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum + define PACKAGE_STRING "[get-define PACKAGE_NAME] [get-define PACKAGE_VERSION]" + proj-xfer-options-aliases { + # Carry values from hidden --flag aliases over to their canonical + # flag forms. This list must include only options which are common + # to all build modes supported by [sqlite-configure]. + with-readline-inc => with-readline-cflags + with-readline-lib => with-readline-ldflags + with-debug => debug + } + set ::sqliteConfig(msg-debug-enabled) [proj-val-truthy [get-env msg-debug 0]] + proc-debug "msg-debug is enabled" + proj-setup-autoreconfig SQLITE_AUTORECONFIG + proj-file-extensions + if {".exe" eq [get-define TARGET_EXEEXT]} { + define SQLITE_OS_UNIX 0 + define SQLITE_OS_WIN 1 + } else { + define SQLITE_OS_UNIX 1 + define SQLITE_OS_WIN 0 + } + sqlite-setup-default-cflags + define HAVE_LFS 0 + if {[opt-bool largefile]} { + # + # Insofar as we can determine HAVE_LFS has no effect on the + # library. Perhaps it did back in the early 2000's. The + # --enable/disable-largefile flag is retained because it's + # harmless, but it doesn't do anything useful. It does have + # visible side-effects, though: the generated sqlite_cfg.h may (or + # may not) define HAVE_LFS. + cc-check-lfs + } + set srcdir $::autosetup(srcdir) + proj-dot-ins-append $srcdir/Makefile.in + if {[file exists $srcdir/sqlite3.pc.in]} { + proj-dot-ins-append $srcdir/sqlite3.pc.in + } + sqlite-handle-hpux; # must be relatively early so that other config tests can work +}; # sqlite-configure-phase1 + +######################################################################## +# Performs late-stage config steps common to all supported +# $::sqliteConfig(build-mode) values. +proc sqlite-configure-finalize {} { + sqlite-handle-rpath + sqlite-handle-soname + sqlite-handle-threadsafe + sqlite-handle-tempstore + sqlite-handle-load-extension + sqlite-handle-math + sqlite-handle-icu + if {[proj-opt-exists readline]} { + sqlite-handle-line-editing + } + if {[proj-opt-exists shared]} { + proj-define-for-opt shared ENABLE_LIB_SHARED "Build shared library?" + } + if {[proj-opt-exists static]} { + if {![proj-define-for-opt static ENABLE_LIB_STATIC "Build static library?"]} { + # This notice really only applies to the canonical build... + proj-indented-notice { + NOTICE: static lib build may be implicitly re-activated by + other components, e.g. some test apps. + } + } + } + sqlite-handle-env-quirks + sqlite-handle-common-feature-flags + sqlite-finalize-feature-flags + sqlite-process-dot-in-files; # do not [define] anything after this + sqlite-dump-defines +} + +######################################################################## +# Internal config-time debugging output routine. It generates no +# output unless msg-debug=1 is passed to the configure script. +proc msg-debug {msg} { + if {$::sqliteConfig(msg-debug-enabled)} { + puts stderr [proj-bold "** DEBUG: $msg"] + } +} +######################################################################## +# A [msg-debug] proxy which prepends the name of the current proc to +# the debug message. It is not legal to call this from the global +# scope. +proc proc-debug {msg} { + msg-debug "\[[proj-scope 1]\]: $msg" +} + +define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. +define OPT_SHELL {} ; # Feature-related CFLAGS for the sqlite3 CLI app +######################################################################## +# Adds $args, if not empty, to OPT_FEATURE_FLAGS. If the first arg is +# -shell then it strips that arg and passes the remaining args the +# sqlite-add-shell-opt in addition to adding them to +# OPT_FEATURE_FLAGS. This is intended only for holding +# -DSQLITE_ENABLE/OMIT/... flags, but that is not enforced here. +proc sqlite-add-feature-flag {args} { + set shell "" + if {"-shell" eq [lindex $args 0]} { + set args [lassign $args shell] + } + if {"" ne $args} { + if {"" ne $shell} { + sqlite-add-shell-opt {*}$args + } + define-append OPT_FEATURE_FLAGS {*}$args + } +} + +######################################################################## +# Appends $args, if not empty, to OPT_SHELL. +proc sqlite-add-shell-opt {args} { + if {"" ne $args} { + define-append OPT_SHELL {*}$args + } +} + +######################################################################## +# Check for log(3) in libm and die with an error if it is not +# found. $featureName should be the feature name which requires that +# function (it's used only in error messages). defines LDFLAGS_MATH to +# the required linker flags (which may be empty even if the math APIs +# are found, depending on the OS). +proc sqlite-affirm-have-math {featureName} { + if {"" eq [get-define LDFLAGS_MATH ""]} { + if {![msg-quiet proj-check-function-in-lib log m]} { + user-error "Missing math APIs for $featureName" + } + set lfl [get-define lib_log ""] + undefine lib_log + if {"" ne $lfl} { + user-notice "Forcing requirement of $lfl for $featureName" + } + define LDFLAGS_MATH $lfl + } +} + +######################################################################## +# Run checks for required binaries, like ld and ar. In the canonical +# build this must come before [sqlite-handle-wasi-sdk]. +proc sqlite-check-common-bins {} { + cc-check-tools ld ar ; # must come before [sqlite-handle-wasi-sdk] + if {"" eq [proj-bin-define install]} { + proj-warn "Cannot find install binary, so 'make install' will not work." + define BIN_INSTALL false + } +} + +######################################################################## +# Run checks for system-level includes and libs which are common to +# both the canonical build and the "autoconf" bundle. +# +# For the canonical build this must come after +# [sqlite-handle-wasi-sdk], as that function may change the +# environment in ways which affect this. +proc sqlite-check-common-system-deps {} { + # Check for needed/wanted data types + cc-with {-includes stdint.h} \ + {cc-check-types int8_t int16_t int32_t int64_t intptr_t \ + uint8_t uint16_t uint32_t uint64_t uintptr_t} + + # Check for needed/wanted functions + cc-check-functions gmtime_r isnan localtime_r localtime_s \ + usleep utime pread pread64 pwrite pwrite64 + + apply {{} { + set ldrt "" + # Collapse funcs from librt into LDFLAGS_RT. + # Some systems (ex: SunOS) require -lrt in order to use nanosleep + foreach func {fdatasync nanosleep} { + if {[proj-check-function-in-lib $func rt]} { + set ldrt [get-define lib_${func} ""] + undefine lib_${func} + if {"" ne $ldrt} { + break + } + } + } + define LDFLAGS_RT $ldrt + }} + + # Check for needed/wanted headers + cc-check-includes \ + sys/types.h sys/stat.h dlfcn.h unistd.h \ + stdlib.h malloc.h memory.h \ + string.h strings.h \ + inttypes.h + + if {[cc-check-includes zlib.h] && [proj-check-function-in-lib deflate z]} { + # TODO? port over the more sophisticated zlib search from the fossil auto.def + define HAVE_ZLIB 1 + define LDFLAGS_ZLIB -lz + sqlite-add-shell-opt -DSQLITE_HAVE_ZLIB=1 + } else { + define HAVE_ZLIB 0 + define LDFLAGS_ZLIB "" + } +} + +######################################################################## +# Move -DSQLITE_OMIT... and -DSQLITE_ENABLE... flags from CFLAGS and +# CPPFLAGS to OPT_FEATURE_FLAGS and remove them from BUILD_CFLAGS. +proc sqlite-munge-cflags {} { + # Move CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and + # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived + # from the legacy build and was missing the 3.48.0 release (the + # initial Autosetup port). + # https://sqlite.org/forum/forumpost/9801e54665afd728 + # + # Handling of CPPFLAGS, as well as removing ENABLE/OMIT from + # CFLAGS/CPPFLAGS, was missing in the 3.49.0 release as well. + # + # If any configure flags for features are in conflict with + # CFLAGS/CPPFLAGS-specified feature flags, all bets are off. There + # are no guarantees about which one will take precedence. + foreach flagDef {CFLAGS CPPFLAGS} { + set tmp "" + foreach cf [get-define $flagDef ""] { + switch -glob -- $cf { + -DSQLITE_OMIT* - + -DSQLITE_ENABLE* { + sqlite-add-feature-flag $cf + } + default { + lappend tmp $cf + } + } + } + define $flagDef $tmp + } + + # Strip all SQLITE_ENABLE/OMIT flags from BUILD_CFLAGS, + # for compatibility with the legacy build. + set tmp "" + foreach cf [get-define BUILD_CFLAGS ""] { + switch -glob -- $cf { + -DSQLITE_OMIT* - + -DSQLITE_ENABLE* {} + default { + lappend tmp $cf + } + } + } + define BUILD_CFLAGS $tmp +} + +######################################################################### +# Set up the default CFLAGS and BUILD_CFLAGS values. +proc sqlite-setup-default-cflags {} { + ######################################################################## + # We differentiate between two C compilers: the one used for binaries + # which are to run on the build system (in autosetup it's called + # CC_FOR_BUILD and in Makefile.in it's $(B.cc)) and the one used for + # compiling binaries for the target system (CC a.k.a. $(T.cc)). + # Normally they're the same, but they will differ when + # cross-compiling. + # + # When cross-compiling we default to not using the -g flag, based on a + # /chat discussion prompted by + # https://sqlite.org/forum/forumpost/9a67df63eda9925c + set defaultCFlags {-O2} + if {!$::sqliteConfig(is-cross-compiling)} { + lappend defaultCFlags -g + } + define CFLAGS [proj-get-env CFLAGS $defaultCFlags] + # BUILD_CFLAGS is the CFLAGS for CC_FOR_BUILD. + define BUILD_CFLAGS [proj-get-env BUILD_CFLAGS {-g}] + sqlite-munge-cflags +} + +######################################################################## +# Handle various SQLITE_ENABLE/OMIT_... feature flags. +proc sqlite-handle-common-feature-flags {} { + msg-result "Feature flags..." + if {![opt-bool all]} { + # Special handling for --disable-all + foreach flag $::sqliteConfig(all-flag-enables) { + if {![proj-opt-was-provided $flag]} { + proj-opt-set $flag 0 + } + } + } + foreach {boolFlag featureFlag ifSetEvalThis} [proj-strip-hash-comments { + all {} { + # The 'all' option must be first in this list. This impl makes + # an effort to only apply flags which the user did not already + # apply, so that combinations like (--all --disable-geopoly) + # will indeed disable geopoly. There are corner cases where + # flags which depend on each other will behave in non-intuitive + # ways: + # + # --all --disable-rtree + # + # Will NOT disable geopoly, though geopoly depends on rtree. + # The --geopoly flag, though, will automatically re-enable + # --rtree, so --disable-rtree won't actually disable anything in + # that case. + foreach k $::sqliteConfig(all-flag-enables) { + if {![proj-opt-was-provided $k]} { + proj-opt-set $k 1 + } + } + } + fts3 -DSQLITE_ENABLE_FTS3 {sqlite-affirm-have-math fts3} + fts4 -DSQLITE_ENABLE_FTS4 {sqlite-affirm-have-math fts4} + fts5 -DSQLITE_ENABLE_FTS5 {sqlite-affirm-have-math fts5} + geopoly -DSQLITE_ENABLE_GEOPOLY {proj-opt-set rtree} + rtree -DSQLITE_ENABLE_RTREE {} + session {-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK} {} + update-limit -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT {} + memsys5 -DSQLITE_ENABLE_MEMSYS5 {} + memsys3 {} { + if {[opt-bool memsys5]} { + proj-warn "not enabling memsys3 because memsys5 is enabled." + expr 0 + } else { + sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3 + } + } + scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + column-metadata -DSQLITE_ENABLE_COLUMN_METADATA {} + dbpage -DSQLITE_ENABLE_DBPAGE_VTAB {} + dbstat -DSQLITE_ENABLE_DBSTAT_VTAB {} + carray -DSQLITE_ENABLE_CARRAY {} + }] { + if {$boolFlag ni $::autosetup(options)} { + # Skip flags which are in the canonical build but not + # the autoconf bundle. + continue + } + proj-if-opt-truthy $boolFlag { + sqlite-add-feature-flag $featureFlag + if {0 != [eval $ifSetEvalThis] && "all" ne $boolFlag} { + msg-result " + $boolFlag" + } + } { + if {"all" ne $boolFlag} { + msg-result " - $boolFlag" + } + } + } + ######################################################################## + # Invert the above loop's logic for some SQLITE_OMIT_... cases. If + # config option $boolFlag is false, [sqlite-add-feature-flag + # $featureFlag], where $featureFlag is intended to be + # -DSQLITE_OMIT_... + foreach {boolFlag featureFlag} { + json -DSQLITE_OMIT_JSON + } { + if {[proj-opt-truthy $boolFlag]} { + msg-result " + $boolFlag" + } else { + sqlite-add-feature-flag $featureFlag + msg-result " - $boolFlag" + } + } +} + +######################################################################### +# Remove duplicates from the final feature flag sets and show them to +# the user. +proc sqlite-finalize-feature-flags {} { + set oFF [get-define OPT_FEATURE_FLAGS] + if {"" ne $oFF} { + define OPT_FEATURE_FLAGS [lsort -unique $oFF] + msg-result "Library feature flags: [get-define OPT_FEATURE_FLAGS]" + } + set oFF [get-define OPT_SHELL] + if {"" ne $oFF} { + define OPT_SHELL [lsort -unique $oFF] + msg-result "Shell options: [get-define OPT_SHELL]" + } + if {"" ne [set extraSrc [get-define AMALGAMATION_EXTRA_SRC ""]]} { + proj-assert {"canonical" eq $::sqliteConfig(build-mode)} + msg-result "Appending source files to amalgamation: $extraSrc" + } + if {[lsearch [get-define TARGET_DEBUG ""] -DSQLITE_DEBUG=1] > -1} { + msg-result "Note: this is a debug build, so performance will suffer." + } +} + +######################################################################## +# Checks for the --debug flag and [define]s TARGET_DEBUG based on +# that. TARGET_DEBUG is unused in the autoconf build but that is +# arguably a bug. +proc sqlite-handle-debug {} { + msg-checking "SQLITE_DEBUG build? " + proj-if-opt-truthy debug { + define TARGET_DEBUG {-g -DSQLITE_DEBUG=1 -O0 -Wall} + sqlite-add-feature-flag -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE + proj-opt-set memsys5 + msg-result yes + } { + define TARGET_DEBUG {-DNDEBUG} + msg-result no + } +} + +######################################################################## +# "soname" for libsqlite3.so. See discussion at: +# https://sqlite.org/src/forumpost/5a3b44f510df8ded +proc sqlite-handle-soname {} { + define LDFLAGS_LIBSQLITE3_SONAME "" + if {[proj-opt-was-provided soname]} { + set soname [join [opt-val soname] ""] + } else { + # Enabling soname breaks linking for the --dynlink-tools feature, + # and this project has no direct use for soname, so default to + # none. Package maintainers, on the other hand, like to have an + # soname. + set soname none + } + switch -exact -- $soname { + none - "" { return 0 } + legacy { set soname libsqlite3.so.0 } + default { + if {[string match libsqlite3.* $soname]} { + # use it as-is + } else { + # Assume it's a suffix + set soname "libsqlite3.so.${soname}" + } + } + } + proc-debug "soname=$soname" + if {[proj-check-soname $soname]} { + define LDFLAGS_LIBSQLITE3_SONAME [get-define LDFLAGS_SONAME_PREFIX]$soname + msg-result "Setting SONAME using: [get-define LDFLAGS_LIBSQLITE3_SONAME]" + } elseif {[proj-opt-was-provided soname]} { + # --soname was explicitly requested but not available, so fail fatally + proj-fatal "This environment does not support SONAME." + } else { + # --soname was not explicitly requested but not available, so just warn + msg-result "This environment does not support SONAME." + } +} + +######################################################################## +# If --enable-threadsafe is set, this adds -DSQLITE_THREADSAFE=1 to +# OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to the linker flags +# needed for linking pthread (possibly an empty string). If +# --enable-threadsafe is not set, adds -DSQLITE_THREADSAFE=0 to +# OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to an empty string. +proc sqlite-handle-threadsafe {} { + msg-checking "Support threadsafe operation? " + define LDFLAGS_PTHREAD "" + set enable 0 + proj-if-opt-truthy threadsafe { + msg-result "Checking for libs..." + if {[proj-check-function-in-lib pthread_create pthread] + && [proj-check-function-in-lib pthread_mutexattr_init pthread]} { + set enable 1 + define LDFLAGS_PTHREAD [get-define lib_pthread_create] + undefine lib_pthread_create + undefine lib_pthread_mutexattr_init + } elseif {[proj-opt-was-provided threadsafe]} { + user-error "Missing required pthread libraries. Use --disable-threadsafe to disable this check." + } else { + msg-result "pthread support not detected" + } + # Recall that LDFLAGS_PTHREAD might be empty even if pthreads if + # found because it's in -lc on some platforms. + } { + msg-result "Disabled using --disable-threadsafe" + } + sqlite-add-feature-flag -DSQLITE_THREADSAFE=${enable} + return $enable +} + +######################################################################## +# Handles the --with-tempstore flag. +# +# The test fixture likes to set SQLITE_TEMP_STORE on its own, so do +# not set that feature flag unless it was explicitly provided to the +# configure script. +proc sqlite-handle-tempstore {} { + if {[proj-opt-was-provided with-tempstore]} { + set ts [opt-val with-tempstore no] + set tsn 1 + msg-checking "Use an in-RAM database for temporary tables? " + switch -exact -- $ts { + never { set tsn 0 } + no { set tsn 1 } + yes { set tsn 2 } + always { set tsn 3 } + default { + user-error "Invalid --with-tempstore value '$ts'. Use one of: never, no, yes, always" + } + } + msg-result $ts + sqlite-add-feature-flag -DSQLITE_TEMP_STORE=$tsn + } +} + +######################################################################## +# Check for the Emscripten SDK for building the web-based wasm +# components. The core lib and tools do not require this but ext/wasm +# does. Most of the work is done via [proj-check-emsdk], then this +# function adds the following defines: +# +# - EMCC_WRAPPER = "" or top-srcdir/tool/emcc.sh +# - BIN_WASM_OPT = "" or path to wasm-opt +# - BIN_WASM_STRIP = "" or path to wasm-strip +# +# Noting that: +# +# 1) Not finding the SDK is not fatal at this level, nor is failure to +# find one of the related binaries. +# +# 2) wasm-strip is part of the wabt package: +# +# https://github.com/WebAssembly/wabt +# +# and this project requires it for production-mode builds but not dev +# builds. +# +proc sqlite-handle-emsdk {} { + define EMCC_WRAPPER "" + define BIN_WASM_STRIP "" + define BIN_WASM_OPT "" + set srcdir $::autosetup(srcdir) + if {$srcdir ne $::autosetup(builddir)} { + # The EMSDK pieces require writing to the original source tree + # even when doing an out-of-tree build. The ext/wasm pieces do not + # support an out-of-tree build so we treat that case as if EMSDK + # were not found. + msg-result "Out-of tree build: not checking for EMSDK." + return + } + set emccSh $srcdir/tool/emcc.sh + set extWasmConfig $srcdir/ext/wasm/config.make + if {![get-define HAVE_WASI_SDK] && [proj-check-emsdk]} { + define EMCC_WRAPPER $emccSh + set emsdkHome [get-define EMSDK_HOME ""] + proj-assert {"" ne $emsdkHome} + #define EMCC_WRAPPER ""; # just for testing + proj-bin-define wasm-strip + proj-bin-define bash; # ext/wasm/GNUmakefile requires bash + if {[file-isexec $emsdkHome/upstream/bin/wasm-opt]} { + define BIN_WASM_OPT $emsdkHome/upstream/bin/wasm-opt + } else { + # Maybe there's a copy in the path? + proj-bin-define wasm-opt BIN_WASM_OPT + } + proj-dot-ins-append $emccSh.in $emccSh { + catch {exec chmod u+x $dotInsOut} + } + proj-dot-ins-append $extWasmConfig.in $extWasmConfig + } else { + define EMCC_WRAPPER "" + file delete -force -- $emccSh $extWasmConfig + } +} + +######################################################################## +# Internal helper for [sqlite-check-line-editing]. Returns a list of +# potential locations under which readline.h might be found. +# +# On some environments this function may perform extra work to help +# sqlite-check-line-editing figure out how to find libreadline and +# friends. It will communicate those results via means other than the +# result value, e.g. by modifying configure --flags. +proc sqlite-get-readline-dir-list {} { + # Historical note: the dirs list, except for the inclusion of + # $prefix and some platform-specific dirs, originates from the + # legacy configure script. + set dirs [list [get-define prefix]] + switch -glob -- [get-define host] { + *-linux-android { + # Possibly termux + lappend dirs /data/data/com.termux/files/usr + } + *-mingw32 { + lappend dirs /mingw32 /mingw + } + *-mingw64 { + lappend dirs /mingw64 /mingw + } + *-haiku { + lappend dirs /boot/system/develop/headers + if {[opt-val with-readline-ldflags] in {auto ""}} { + # If the user did not supply their own --with-readline-ldflags + # value, hijack that flag to inject options which are known to + # work on Haiku OS installations. + if {"" ne [glob -nocomplain /boot/system/lib/libreadline*]} { + proj-opt-set with-readline-ldflags {-L/boot/system/lib -lreadline} + } + } + } + } + lappend dirs /usr /usr/local /usr/local/readline /usr/contrib + set rv {} + foreach d $dirs { + if {[file isdir $d]} {lappend rv $d} + } + #proc-debug "dirs=$rv" + return $rv +} + +######################################################################## +# sqlite-check-line-editing jumps through proverbial hoops to try to +# find a working line-editing library, setting: +# +# - HAVE_READLINE to 0 or 1 +# - HAVE_LINENOISE to 0, 1, or 2 +# - HAVE_EDITLINE to 0 or 1 +# +# Only one of ^^^ those will be set to non-0. +# +# - LDFLAGS_READLINE = linker flags or empty string +# +# - CFLAGS_READLINE = compilation flags for clients or empty string. +# +# Note that LDFLAGS_READLINE and CFLAGS_READLINE may refer to +# linenoise or editline, not necessarily libreadline. In some cases +# it will set HAVE_READLINE=1 when it's really using editline, for +# reasons described in this function's comments. +# +# Returns a string describing which line-editing approach to use, or +# "none" if no option is available. +# +# Order of checks: +# +# 1) --with-linenoise trumps all others and skips all of the +# complexities involved with the remaining options. +# +# 2) --editline trumps --readline +# +# 3) --disable-readline trumps --readline +# +# 4) Default to automatic search for optional readline +# +# 5) Try to find readline or editline. If it's not found AND the +# corresponding --FEATURE flag was explicitly given then fail +# fatally, else fail non-fatally. +proc sqlite-check-line-editing {} { + msg-result "Checking for line-editing capability..." + define HAVE_READLINE 0 + define HAVE_LINENOISE 0 + define HAVE_EDITLINE 0 + define LDFLAGS_READLINE "" + define CFLAGS_READLINE "" + set failIfNotFound 0 ; # Gets set to 1 for explicit --FEATURE requests + # so that we know whether to fail fatally or not + # if the library is not found. + set libsForReadline {readline edit} ; # -l names to check for readline(). + # The libedit check changes this. + set editLibName "readline" ; # "readline" or "editline" + set editLibDef "HAVE_READLINE" ; # "HAVE_READLINE" or "HAVE_EDITLINE" + set dirLn [opt-val with-linenoise] + if {"" ne $dirLn} { + # Use linenoise from a copy of its sources (not a library)... + if {![file isdir $dirLn]} { + proj-fatal "--with-linenoise value is not a directory" + } + set lnH $dirLn/linenoise.h + if {![file exists $lnH] } { + proj-fatal "Cannot find linenoise.h in $dirLn" + } + set lnC "" + set lnCOpts {linenoise-ship.c linenoise.c} + foreach f $lnCOpts { + if {[file exists $dirLn/$f]} { + set lnC $dirLn/$f + break + } + } + if {"" eq $lnC} { + proj-fatal "Cannot find any of $lnCOpts in $dirLn" + } + set flavor "" + set lnVal [proj-which-linenoise $lnH] + switch -- $lnVal { + 1 { set flavor "antirez" } + 2 { set flavor "msteveb" } + default { + proj-fatal "Cannot determine the flavor of linenoise from $lnH" + } + } + define CFLAGS_READLINE "-I$dirLn $lnC" + define HAVE_LINENOISE $lnVal + sqlite-add-shell-opt -DHAVE_LINENOISE=$lnVal + if {$::sqliteConfig(use-jim-for-codegen) && 2 == $lnVal} { + define-append CFLAGS_JIMSH -DUSE_LINENOISE [get-define CFLAGS_READLINE] + user-notice "Adding linenoise support to jimsh." + } + return "linenoise ($flavor)" + } elseif {[opt-bool editline]} { + # libedit mimics libreadline and on some systems does not have its + # own header installed (instead, that of libreadline is used). + # + # shell.c historically expects HAVE_EDITLINE to be set for + # libedit, but it then expects to see , which + # some system's don't actually have despite having libedit. If we + # end up finding below, we will use + # -DHAVE_EDITLINE=1, else we will use -DHAVE_READLINE=1. In either + # case, we will link against libedit. + set failIfNotFound 1 + set libsForReadline {edit} + set editLibName editline + } elseif {![opt-bool readline]} { + msg-result "Readline support explicitly disabled with --disable-readline" + return "none" + } elseif {[proj-opt-was-provided readline]} { + # If an explicit --[enable-]readline was used, fail if it's not + # found, else treat the feature as optional. + set failIfNotFound 1 + } + + # Transform with-readline-header=X to with-readline-cflags=-I... + set v [opt-val with-readline-header] + proj-opt-set with-readline-header "" + if {"" ne $v} { + if {"auto" eq $v} { + proj-opt-set with-readline-cflags auto + } else { + set v [file dirname $v] + if {[string match */readline $v]} { + # Special case: if the path includes .../readline/readline.h, + # set the -I to one dir up from that because our sources + # #include or . + set v [file dirname $v] + } + proj-opt-set with-readline-cflags "-I$v" + } + } + + # Look for readline.h + set rlInc [opt-val with-readline-cflags auto] + if {"auto" eq $rlInc} { + set rlInc "" + if {$::sqliteConfig(is-cross-compiling)} { + # ^^^ this check is derived from the legacy configure script. + proj-warn "Skipping check for readline.h because we're cross-compiling." + } else { + set dirs [sqlite-get-readline-dir-list] + set subdirs [list \ + include/$editLibName \ + readline] + if {"editline" eq $editLibName} { + lappend subdirs include/readline + # ^^^ editline, on some systems, does not have its own header, + # and uses libreadline's header. + } + lappend subdirs include + set rlInc [proj-search-for-header-dir readline.h \ + -dirs $dirs -subdirs $subdirs] + #proc-debug "rlInc=$rlInc" + if {"" ne $rlInc} { + if {[string match */readline $rlInc]} { + set rlInc [file dirname $rlInc]; # CLI shell: #include + } elseif {[string match */editline $rlInc]} { + set editLibDef HAVE_EDITLINE + set rlInc [file dirname $rlInc]; # CLI shell: #include + } + set rlInc "-I${rlInc}" + } + } + } elseif {"" ne $rlInc && ![string match *-I* $rlInc]} { + proj-fatal "Argument to --with-readline-cflags is intended to be CFLAGS and contain -I..." + } + + # If readline.h was found/specified, look for lib(readline|edit)... + # + # This is not quite straightforward because both libreadline and + # libedit typically require some other library which (according to + # legacy autotools-generated tests) provides tgetent(3). On some + # systems that's built into libreadline/edit, on some (most?) its in + # lib[n]curses, and on some it's in libtermcap. + set rlLib "" + if {"" ne $rlInc} { + set rlLib [opt-val with-readline-ldflags] + #proc-debug "rlLib=$rlLib" + if {$rlLib in {auto ""}} { + set rlLib "" ; # make sure it's not "auto", as we may append to it below + set libTerm ""; # lib with tgetent(3) + if {[proj-check-function-in-lib tgetent [list $editLibName ncurses curses termcap]]} { + # ^^^ that libs list comes from the legacy configure script ^^^ + set libTerm [get-define lib_tgetent] + undefine lib_tgetent + } + if {$editLibName eq $libTerm} { + # tgetent(3) was found in the editing library + set rlLib $libTerm + } elseif {[proj-check-function-in-lib readline $libsForReadline $libTerm]} { + # tgetent(3) was found in an external lib + set rlLib [get-define lib_readline] + lappend rlLib $libTerm + undefine lib_readline + } + } + } + + # If we found a library, configure the build to use it... + if {"" ne $rlLib} { + if {"editline" eq $editLibName && "HAVE_READLINE" eq $editLibDef} { + # Alert the user that, despite outward appearances, we won't be + # linking to the GPL'd libreadline. Presumably that distinction is + # significant for those using --editline. + proj-indented-notice { + NOTE: the local libedit uses so we + will compile with -DHAVE_READLINE=1 but will link with + libedit. + } + } + set rlLib [join $rlLib] + set rlInc [join $rlInc] + define LDFLAGS_READLINE $rlLib + define CFLAGS_READLINE $rlInc + proj-assert {$editLibDef in {HAVE_READLINE HAVE_EDITLINE}} + proj-assert {$editLibName in {readline editline}} + sqlite-add-shell-opt -D${editLibDef}=1 + msg-result "Using $editLibName flags: $rlInc $rlLib" + # Check whether rl_completion_matches() has a signature we can use + # and disable that sub-feature if it doesn't. + if {![cctest -cflags "$rlInc -D${editLibDef}" -libs $rlLib -nooutput 1 \ + -source { + #include + #ifdef HAVE_EDITLINE + #include + #else + #include + #endif + static char * rcg(const char *z, int i){(void)z; (void)i; return 0;} + int main(void) { + char ** x = rl_completion_matches("one", rcg); + (void)x; + return 0; + } + }]} { + proj-warn "readline-style completion disabled due to rl_completion_matches() signature mismatch" + sqlite-add-shell-opt -DSQLITE_OMIT_READLINE_COMPLETION + } + return $editLibName + } + + if {$failIfNotFound} { + proj-fatal "Explicit --$editLibName failed to find a matching library." + } + return "none" +}; # sqlite-check-line-editing + +######################################################################## +# Runs sqlite-check-line-editing and adds a message around it. In the +# canonical build this must not be called before +# sqlite-determine-codegen-tcl for reasons now lost to history (and +# might not still be applicable). +proc sqlite-handle-line-editing {} { + msg-result "Line-editing support for the sqlite3 shell: [sqlite-check-line-editing]" +} + + +######################################################################## +# ICU - International Components for Unicode +# +# Handles these flags: +# +# --with-icu-ldflags=LDFLAGS +# --with-icu-cflags=CFLAGS +# --with-icu-config[=auto | pkg-config | /path/to/icu-config] +# --enable-icu-collations +# +# --with-icu-config values: +# +# - auto: use the first one of (pkg-config, icu-config) found on the +# system. +# - pkg-config: use only pkg-config to determine flags +# - /path/to/icu-config: use that to determine flags +# +# If --with-icu-config is used and neither pkg-config nor icu-config +# are found, fail fatally. +# +# If both --with-icu-ldflags and --with-icu-config are provided, they +# are cumulative. If neither are provided, icu-collations is not +# honored and a warning is emitted if it is provided. +# +# Design note: though we could automatically enable ICU if the +# icu-config binary or (pkg-config icu-io) are found, we specifically +# do not. ICU is always an opt-in feature. +proc sqlite-handle-icu {} { + define LDFLAGS_ICU [join [opt-val with-icu-ldflags ""]] + define CFLAGS_ICU [join [opt-val with-icu-cflags ""]] + if {[proj-opt-was-provided with-icu-config]} { + msg-result "Checking for ICU support..." + set icuConfigBin [opt-val with-icu-config] + set tryIcuConfigBin 1; # set to 0 if we end up using pkg-config + if {$icuConfigBin in {auto pkg-config}} { + if {[pkg-config-init 0] && [pkg-config icu-io]} { + # Maintenance reminder: historical docs say to use both of + # (icu-io, icu-uc). icu-uc lacks a required lib and icu-io has + # all of them on tested OSes. + set tryIcuConfigBin 0 + define LDFLAGS_ICU [get-define PKG_ICU_IO_LDFLAGS] + define-append LDFLAGS_ICU [get-define PKG_ICU_IO_LIBS] + define CFLAGS_ICU [get-define PKG_ICU_IO_CFLAGS] + } elseif {"pkg-config" eq $icuConfigBin} { + proj-fatal "pkg-config cannot find package icu-io" + } else { + proj-assert {"auto" eq $icuConfigBin} + } + } + if {$tryIcuConfigBin} { + if {"auto" eq $icuConfigBin} { + set icuConfigBin [proj-first-bin-of \ + /usr/local/bin/icu-config \ + /usr/bin/icu-config] + if {"" eq $icuConfigBin} { + proj-indented-notice -error { + --with-icu-config=auto cannot find (pkg-config icu-io) or icu-config binary. + On Ubuntu-like systems try: + --with-icu-ldflags='-licui18n -licuuc -licudata' + } + } + } + if {[file-isexec $icuConfigBin]} { + set x [exec $icuConfigBin --ldflags] + if {"" eq $x} { + proj-indented-notice -error \ + [subst { + $icuConfigBin --ldflags returned no data. + On Ubuntu-like systems try: + --with-icu-ldflags='-licui18n -licuuc -licudata' + }] + } + define-append LDFLAGS_ICU $x + set x [exec $icuConfigBin --cppflags] + define-append CFLAGS_ICU $x + } else { + proj-fatal "--with-icu-config=$icuConfigBin does not refer to an executable" + } + } + } + set ldflags [define LDFLAGS_ICU [string trim [get-define LDFLAGS_ICU]]] + set cflags [define CFLAGS_ICU [string trim [get-define CFLAGS_ICU]]] + if {"" ne $ldflags} { + sqlite-add-feature-flag -shell -DSQLITE_ENABLE_ICU + msg-result "Enabling ICU support with flags: $ldflags $cflags" + if {[opt-bool icu-collations]} { + msg-result "Enabling ICU collations." + sqlite-add-feature-flag -shell -DSQLITE_ENABLE_ICU_COLLATIONS + # Recall that shell.c builds with sqlite3.c except in the case + # of --disable-static-shell, a combination we do not + # specifically attempt to account for. + } + } elseif {[opt-bool icu-collations]} { + proj-warn "ignoring --enable-icu-collations because neither --with-icu-ldflags nor --with-icu-config provided any linker flags" + } else { + msg-result "ICU support is disabled." + } +}; # sqlite-handle-icu + + +######################################################################## +# Handles the --enable-load-extension flag. Returns 1 if the support +# is enabled, else 0. If support for that feature is not found, a +# fatal error is triggered if --enable-load-extension is explicitly +# provided, else a loud warning is instead emitted. If +# --disable-load-extension is used, no check is performed. +# +# Makes the following environment changes: +# +# - defines LDFLAGS_DLOPEN to any linker flags needed for this +# feature. It may legally be empty on (A) some systems where +# dlopen() is in libc and (B) certain Unix-esque Windows +# environments which identify as Windows for SQLite's purposes so +# use LoadLibrary(). +# +# - If the feature is not available, adds +# -DSQLITE_OMIT_LOAD_EXTENSION=1 to the feature flags list. +proc sqlite-handle-load-extension {} { + define LDFLAGS_DLOPEN "" + set found 0 + set suffix "" + proj-if-opt-truthy load-extension { + switch -glob -- [get-define host] { + *-*-mingw* - *windows* { + incr found + set suffix "Using LoadLibrary()" + } + default { + set found [proj-check-function-in-lib dlopen dl] + if {$found} { + set suffix [define LDFLAGS_DLOPEN [get-define lib_dlopen]] + undefine lib_dlopen + } else { + if {[proj-opt-was-provided load-extension]} { + # Explicit --enable-load-extension: fail if not found + proj-indented-notice -error { + --enable-load-extension was provided but dlopen() + not found. Use --disable-load-extension to bypass this + check. + } + } else { + # It was implicitly enabled: warn if not found + proj-indented-notice { + WARNING: dlopen() not found, so loadable module support will + be disabled. Use --disable-load-extension to bypass this + check. + } + } + } + } + } + } + if {$found} { + msg-result "Loadable extension support enabled. $suffix" + } else { + msg-result "Disabling loadable extension support. Use --enable-load-extension to enable them." + sqlite-add-feature-flag -DSQLITE_OMIT_LOAD_EXTENSION=1 + } + return $found +} + +######################################################################## +# Handles the --enable-math flag. +proc sqlite-handle-math {} { + proj-if-opt-truthy math { + if {![proj-check-function-in-lib ceil m]} { + user-error "Cannot find libm functions. Use --disable-math to bypass this." + } + define LDFLAGS_MATH [get-define lib_ceil] + undefine lib_ceil + sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS -DSQLITE_ENABLE_PERCENTILE + msg-result "Enabling math SQL functions" + } { + define LDFLAGS_MATH "" + msg-result "Disabling math SQL functions" + } +} + +######################################################################## +# If this OS looks like a Mac, checks for the Mac-specific +# -current_version and -compatibility_version linker flags. Defines +# LDFLAGS_MAC_CVERSION to an empty string and returns 0 if they're not +# supported, else defines that to the linker flags and returns 1. +# +# We don't check this on non-Macs because this whole thing is a +# libtool compatibility kludge to account for a version stamp which +# libtool applied only on Mac platforms. +# +# Based on https://sqlite.org/forum/forumpost/9dfd5b8fd525a5d7. +proc sqlite-handle-mac-cversion {} { + define LDFLAGS_MAC_CVERSION "" + set rc 0 + if {[proj-looks-like-mac]} { + cc-with {-link 1} { + # These version numbers are historical libtool-defined values, not + # library-defined ones + if {[cc-check-flags "-Wl,-current_version,9.6.0"] + && [cc-check-flags "-Wl,-compatibility_version,9.0.0"]} { + define LDFLAGS_MAC_CVERSION "-Wl,-compatibility_version,9.0.0 -Wl,-current_version,9.6.0" + set rc 1 + } elseif {[cc-check-flags "-compatibility_version 9.0.0"] + && [cc-check-flags "-current_version 9.6.0"]} { + define LDFLAGS_MAC_CVERSION "-compatibility_version 9.0.0 -current_version 9.6.0" + set rc 1 + } + } + } + return $rc +} + +######################################################################## +# If this is a Mac platform, check for support for +# -Wl,-install_name,... and, if it's available, define +# LDFLAGS_MAC_INSTALL_NAME to a variant of that string which is +# intended to expand at make-time, else set LDFLAGS_MAC_INSTALL_NAME +# to an empty string. +# +# https://sqlite.org/forum/forumpost/5651662b8875ec0a +proc sqlite-handle-mac-install-name {} { + define LDFLAGS_MAC_INSTALL_NAME ""; # {-Wl,-install_name,"$(install-dir.lib)/$(libsqlite3.DLL)"} + set rc 0 + if {[proj-looks-like-mac]} { + cc-with {-link 1} { + if {[cc-check-flags "-Wl,-install_name,/usr/local/lib/libsqlite3.dylib"]} { + define LDFLAGS_MAC_INSTALL_NAME {-Wl,-install_name,"$(install-dir.lib)/$(libsqlite3.DLL)"} + set rc 1 + } + } + } + return $rc +} + +# +# Checks specific to HP-UX. +# +proc sqlite-handle-hpux {} { + switch -glob -- [get-define host] { + *hpux* { + if {[cc-check-flags "-Ae"]} { + define-append CFLAGS -Ae + } + } + } +} + +######################################################################## +# Handles the --dll-basename configure flag. [define]'s +# SQLITE_DLL_BASENAME to the DLL's preferred base name (minus +# extension). If --dll-basename is not provided (or programmatically +# set - see [sqlite-handle-env-quirks]) then this is always +# "libsqlite3", otherwise it may use a different value based on the +# value of [get-define host]. +proc sqlite-handle-dll-basename {} { + if {[proj-opt-was-provided dll-basename]} { + set dn [join [opt-val dll-basename] ""] + if {$dn in {none default}} { set dn libsqlite3 } + } else { + set dn libsqlite3 + } + if {$dn in {auto ""}} { + switch -glob -- [get-define host] { + *-*-cygwin { set dn cygsqlite3-0 } + *-*-ming* { set dn libsqlite3-0 } + *-*-msys { set dn msys-sqlite3-0 } + default { set dn libsqlite3 } + } + } + define SQLITE_DLL_BASENAME $dn +} + +######################################################################## +# [define]s LDFLAGS_OUT_IMPLIB to either an empty string or to a +# -Wl,... flag for the platform-specific --out-implib flag, which is +# used for building an "import library .dll.a" file on some platforms +# (e.g. msys2, mingw). SQLITE_OUT_IMPLIB is defined to the name of the +# import lib or an empty string. Returns 1 if supported, else 0. +# +# The name of the import library is [define]d in SQLITE_OUT_IMPLIB. +# +# If the configure flag --out-implib is not used (or programmatically +# set) then this simply sets the above-listed defines to empty strings +# (but see [sqlite-handle-env-quirks]). If that flag is used but the +# capability is not available, a fatal error is triggered. +# +# This feature is specifically opt-in because it's supported on far +# more platforms than actually need it and enabling it causes creation +# of libsqlite3.so.a files which are unnecessary in most environments. +# +# Added in response to: https://sqlite.org/forum/forumpost/0c7fc097b2 +# +# Platform notes: +# +# - cygwin sqlite packages historically install no .dll.a file. +# +# - msys2 and mingw sqlite packages historically install +# /usr/lib/libsqlite3.dll.a despite the DLL being in +# /usr/bin. +proc sqlite-handle-out-implib {} { + define LDFLAGS_OUT_IMPLIB "" + define SQLITE_OUT_IMPLIB "" + set rc 0 + if {[proj-opt-was-provided out-implib]} { + set olBaseName [join [opt-val out-implib] ""] + if {$olBaseName in {auto ""}} { + set olBaseName "libsqlite3" ;# [get-define SQLITE_DLL_BASENAME] + # Based on discussions with mingw/msys users, the import lib + # should always be called libsqlite3.dll.a even on platforms + # which rename libsqlite3.dll to something else. + } + if {$olBaseName ne "none"} { + cc-with {-link 1} { + set dll "${olBaseName}[get-define TARGET_DLLEXT]" + set flags [proj-cc-check-Wl-flag --out-implib ${dll}.a] + if {"" ne $flags} { + define LDFLAGS_OUT_IMPLIB $flags + define SQLITE_OUT_IMPLIB ${dll}.a + set rc 1 + } + } + if {!$rc} { + user-error "--out-implib is not supported on this platform" + } + } + } + return $rc +} + +######################################################################## +# If the given platform identifier (defaulting to [get-define host]) +# appears to be one of the Unix-on-Windows environments, returns a +# brief symbolic name for that environment, else returns an empty +# string. +# +# It does not distinguish between msys and msys2, returning msys for +# both. The build does not, as of this writing, specifically support +# msys v1. Similarly, this function returns "mingw" for both "mingw32" +# and "mingw64". +proc sqlite-env-is-unix-on-windows {{envTuple ""}} { + if {"" eq $envTuple} { + set envTuple [get-define host] + } + set name "" + switch -glob -- $envTuple { + *-*-cygwin { set name cygwin } + *-*-ming* { set name mingw } + *-*-msys { set name msys } + } + return $name +} + +######################################################################## +# Performs various tweaks to the build which are only relevant on +# certain platforms, e.g. Mac and "Unix on Windows" platforms (msys2, +# cygwin, ...). +# +# 1) DLL installation: +# +# [define]s SQLITE_DLL_INSTALL_RULES to a symbolic name suffix for a +# set of "make install" rules to use for installation of the DLL +# deliverable. The makefile is tasked with providing rules named +# install-dll-NAME which runs the installation for that set, as well +# as providing a rule named install-dll which resolves to +# install-dll-NAME (perhaps indirectly, depending on whether the DLL +# is (de)activated). +# +# The default value is "unix-generic". +# +# 2) --out-implib: +# +# On platforms where an "import library" is conventionally used but +# --out-implib was not explicitly used, automatically add that flag. +# This conventionally applies only to the "Unix on Windows" +# environments like msys and cygwin. +# +# 3) --dll-basename: +# +# On the same platforms addressed by --out-implib, if --dll-basename +# is not explicitly specified, --dll-basename=auto is implied. +proc sqlite-handle-env-quirks {} { + set instName unix-generic; # name of installation rules set + set autoDll 0; # true if --out-implib/--dll-basename should be implied + set host [get-define host] + switch -glob -- $host { + *apple* - + *darwin* { set instName darwin } + default { + set x [sqlite-env-is-unix-on-windows $host] + if {"" ne $x} { + set instName $x + set autoDll 1 + } + } + } + define SQLITE_DLL_INSTALL_RULES $instName + if {$autoDll} { + if {![proj-opt-was-provided out-implib]} { + # Imply --out-implib=auto + proj-indented-notice [subst -nocommands -nobackslashes { + NOTICE: auto-enabling --out-implib for environment [$host]. + Use --out-implib=none to disable this special case + or --out-implib=auto to squelch this notice. + }] + proj-opt-set out-implib auto + } + if {![proj-opt-was-provided dll-basename]} { + # Imply --dll-basename=auto + proj-indented-notice [subst -nocommands -nobackslashes { + NOTICE: auto-enabling --dll-basename for environment [$host]. + Use --dll-basename=default to disable this special case + or --dll-basename=auto to squelch this notice. + }] + proj-opt-set dll-basename auto + } + } + sqlite-handle-dll-basename + sqlite-handle-out-implib + sqlite-handle-mac-cversion + sqlite-handle-mac-install-name + if {[llength [info proc sqlite-custom-handle-flags]] > 0} { + # sqlite-custom-handle-flags is assumed to be imported via a + # client-specific import: autosetup/sqlite-custom.tcl. + sqlite-custom-handle-flags + } +} + +######################################################################## +# Perform some late-stage work and generate the configure-process +# output file(s). +proc sqlite-process-dot-in-files {} { + ######################################################################## + # "Re-export" the autoconf-conventional --XYZdir flags into something + # which is more easily overridable from a make invocation. See the docs + # for [proj-remap-autoconf-dir-vars] for the explanation of why. + # + # We do this late in the config process, immediately before we export + # the Makefile and other generated files, so that configure tests + # which make make use of the autotools-conventional flags + # (e.g. [proj-check-rpath]) may do so before we "mangle" them here. + proj-remap-autoconf-dir-vars + + proj-dot-ins-process -validate + make-config-header sqlite_cfg.h \ + -bare {SIZEOF_* HAVE_DECL_*} \ + -none {HAVE_CFLAG_* LDFLAGS_* SH_* SQLITE_AUTORECONFIG + TARGET_* USE_GCOV TCL_*} \ + -auto {HAVE_* PACKAGE_*} \ + -none * + proj-touch sqlite_cfg.h ; # help avoid frequent unnecessary @SQLITE_AUTORECONFIG@ +} + +######################################################################## +# Handle --with-wasi-sdk[=DIR] +# +# This must be run relatively early on because it may change the +# toolchain and disable a number of config options. However, in the +# canonical build this must come after [sqlite-check-common-bins]. +proc sqlite-handle-wasi-sdk {} { + set wasiSdkDir [opt-val with-wasi-sdk] ; # ??? [lindex [opt-val with-wasi-sdk] end] + define HAVE_WASI_SDK 0 + if {$wasiSdkDir eq ""} { + return 0 + } elseif {$::sqliteConfig(is-cross-compiling)} { + proj-fatal "Cannot combine --with-wasi-sdk with cross-compilation" + } + msg-result "Checking WASI SDK directory \[$wasiSdkDir]... " + proj-affirm-files-exist -v {*}[prefix "$wasiSdkDir/bin/" {clang wasm-ld ar}] + define HAVE_WASI_SDK 1 + define WASI_SDK_DIR $wasiSdkDir + # Disable numerous options which we know either can't work or are + # not useful in this build... + msg-result "Using wasi-sdk clang. Disabling CLI shell and modifying config flags:" + # Boolean (--enable-/--disable-) flags which must be switched off: + foreach opt { + dynlink-tools + editline + gcov + icu-collations + load-extension + readline + shared + tcl + threadsafe + } { + if {[proj-opt-exists $opt] && [opt-bool $opt]} { + # -^^^^ not all builds define all of these flags + msg-result " --disable-$opt" + proj-opt-set $opt 0 + } + } + # Non-boolean flags which need to be cleared: + foreach opt { + with-emsdk + with-icu-config + with-icu-ldflags + with-icu-cflags + with-linenoise + with-tcl + } { + if {[proj-opt-was-provided $opt]} { + msg-result " removing --$opt" + proj-opt-set $opt "" + } + } + # Remember that we now have a discrepancy between + # $::sqliteConfig(is-cross-compiling) and [proj-is-cross-compiling]. + set ::sqliteConfig(is-cross-compiling) 1 + + # + # Changing --host and --target have no effect here except to + # possibly cause confusion. Autosetup has finished processing them + # by this point. + # + # host_alias=wasm32-wasi + # target=wasm32-wasi + # + # Merely changing CC, LD, and AR to the wasi-sdk's is enough to get + # sqlite3.o building in WASM format. + # + define CC "${wasiSdkDir}/bin/clang" + define LD "${wasiSdkDir}/bin/wasm-ld" + define AR "${wasiSdkDir}/bin/ar" + #define STRIP "${wasiSdkDir}/bin/strip" + return 1 +}; # sqlite-handle-wasi-sdk + +######################################################################## +# TCL... +# +# sqlite-check-tcl performs most of the --with-tcl and --with-tclsh +# handling. Some related bits and pieces are performed before and +# after that function is called. +# +# Important [define]'d vars: +# +# - HAVE_TCL indicates whether we have a tclsh suitable for building +# the TCL SQLite extension and, by extension, the testing +# infrastructure. This must only be 1 for environments where +# tclConfig.sh can be found. +# +# - TCLSH_CMD is the path to the canonical tclsh or "". It never +# refers to jimtcl. +# +# - TCL_CONFIG_SH is the path to tclConfig.sh or "". +# +# - TCLLIBDIR is the dir to which libtclsqlite3 gets installed. +# +# - BTCLSH = the path to the tcl interpreter used for in-tree code +# generation. It may be jimtcl or the canonical tclsh but may not +# be empty - this tree requires TCL to generated numerous +# components. +# +# If --tcl or --with-tcl are provided but no TCL is found, this +# function fails fatally. If they are not explicitly provided then +# failure to find TCL is not fatal but a loud warning will be emitted. +# +proc sqlite-check-tcl {} { + define TCLSH_CMD false ; # Significant is that it exits with non-0 + define HAVE_TCL 0 ; # Will be enabled via --tcl or a successful search + define TCLLIBDIR "" ; # Installation dir for TCL extension lib + define TCL_CONFIG_SH ""; # full path to tclConfig.sh + + # Clear out all vars which would harvest from tclConfig.sh so that + # the late-config validation of @VARS@ works even if --disable-tcl + # is used. + proj-tclConfig-sh-to-autosetup "" + + file delete -force ".tclenv.sh"; # ensure no stale state from previous configures. + if {![opt-bool tcl]} { + proj-indented-notice { + NOTE: TCL is disabled via --disable-tcl. This means that none + of the TCL-based components will be built, including tests + and sqlite3_analyzer. + } + return + } + # TODO: document the steps this is taking. + set srcdir $::autosetup(srcdir) + msg-result "Checking for a suitable tcl... " + proj-assert [proj-opt-truthy tcl] + set use_tcl 1 + set with_tclsh [opt-val with-tclsh] + set with_tcl [opt-val with-tcl] + if {"prefix" eq $with_tcl} { + set with_tcl [get-define prefix] + } + proc-debug "use_tcl ${use_tcl}" + proc-debug "with_tclsh=${with_tclsh}" + proc-debug "with_tcl=$with_tcl" + if {"" eq $with_tclsh && "" eq $with_tcl} { + # If neither --with-tclsh nor --with-tcl are provided, try to find + # a workable tclsh. + set with_tclsh [proj-first-bin-of tclsh9.1 tclsh9.0 tclsh8.6 tclsh] + proc-debug "with_tclsh=${with_tclsh}" + } + + set doConfigLookup 1 ; # set to 0 to test the tclConfig.sh-not-found cases + if {"" ne $with_tclsh} { + # --with-tclsh was provided or found above. Validate it and use it + # to trump any value passed via --with-tcl=DIR. + if {![file-isexec $with_tclsh]} { + proj-fatal "TCL shell $with_tclsh is not executable" + } else { + define TCLSH_CMD $with_tclsh + #msg-result "Using tclsh: $with_tclsh" + } + if {$doConfigLookup && + [catch {exec $with_tclsh $::autosetup(libdir)/find_tclconfig.tcl} result] == 0} { + set with_tcl $result + } + if {"" ne $with_tcl && [file isdir $with_tcl]} { + msg-result "$with_tclsh recommends the tclConfig.sh from $with_tcl" + } else { + proj-warn "$with_tclsh is unable to recommend a tclConfig.sh" + set use_tcl 0 + } + } + set cfg "" + set tclSubdirs {tcl9.1 tcl9.0 tcl8.6 lib} + while {$use_tcl} { + if {"" ne $with_tcl} { + # Ensure that we can find tclConfig.sh under ${with_tcl}/... + if {$doConfigLookup} { + if {[file readable "${with_tcl}/tclConfig.sh"]} { + set cfg "${with_tcl}/tclConfig.sh" + } else { + foreach i $tclSubdirs { + if {[file readable "${with_tcl}/$i/tclConfig.sh"]} { + set cfg "${with_tcl}/$i/tclConfig.sh" + break + } + } + } + } + if {"" eq $cfg} { + proj-fatal "No tclConfig.sh found under ${with_tcl}" + } + } else { + # If we have not yet found a tclConfig.sh file, look in $libdir + # which is set automatically by autosetup or via the --prefix + # command-line option. See + # https://sqlite.org/forum/forumpost/e04e693439a22457 + set libdir [get-define libdir] + if {[file readable "${libdir}/tclConfig.sh"]} { + set cfg "${libdir}/tclConfig.sh" + } else { + foreach i $tclSubdirs { + if {[file readable "${libdir}/$i/tclConfig.sh"]} { + set cfg "${libdir}/$i/tclConfig.sh" + break + } + } + } + if {![file readable $cfg]} { + break + } + } + msg-result "Using tclConfig.sh: $cfg" + break + } + define TCL_CONFIG_SH $cfg + # Export a subset of tclConfig.sh to the current TCL-space. If $cfg + # is an empty string, this emits empty-string entries for the + # various options we're interested in. + proj-tclConfig-sh-to-autosetup $cfg + + if {"" eq $with_tclsh && $cfg ne ""} { + # We have tclConfig.sh but no tclsh. Attempt to locate a tclsh + # based on info from tclConfig.sh. + set tclExecPrefix [get-define TCL_EXEC_PREFIX] + proj-assert {"" ne $tclExecPrefix} + set tryThese [list \ + $tclExecPrefix/bin/tclsh[get-define TCL_VERSION] \ + $tclExecPrefix/bin/tclsh ] + foreach trySh $tryThese { + if {[file-isexec $trySh]} { + set with_tclsh $trySh + break + } + } + if {![file-isexec $with_tclsh]} { + proj-warn "Cannot find a usable tclsh (tried: $tryThese) + } + } + define TCLSH_CMD $with_tclsh + if {$use_tcl} { + # Set up the TCLLIBDIR + # + # 2024-10-28: calculation of TCLLIBDIR is now done via the shell + # in main.mk (search it for T.tcl.env.sh) so that + # static/hand-written makefiles which import main.mk do not have + # to define that before importing main.mk. Even so, we export + # TCLLIBDIR from here, which will cause the canonical makefile to + # use this one rather than to re-calculate it at make-time. + set tcllibdir [get-env TCLLIBDIR ""] + set sq3Ver [get-define PACKAGE_VERSION] + if {"" eq $tcllibdir} { + # Attempt to extract TCLLIBDIR from TCL's $auto_path + if {"" ne $with_tclsh && + [catch {exec echo "puts stdout \$auto_path" | "$with_tclsh"} result] == 0} { + foreach i $result { + if {[file isdir $i]} { + set tcllibdir $i/sqlite${sq3Ver} + break + } + } + } else { + proj-warn "Cannot determine TCLLIBDIR." + # The makefile will fail fatally in this case if a target is + # invoked which requires TCLLIBDIR. + } + } + #if {"" ne $tcllibdir} { msg-result "TCLLIBDIR = ${tcllibdir}"; } + define TCLLIBDIR $tcllibdir + }; # find TCLLIBDIR + + if {[file-isexec $with_tclsh]} { + msg-result "Using tclsh: $with_tclsh" + if {$cfg ne ""} { + define HAVE_TCL 1 + } else { + proj-warn "Found tclsh but no tclConfig.sh." + } + } + show-notices + # If TCL is not found: if it was explicitly requested then fail + # fatally, else just emit a warning. If we can find the APIs needed + # to generate a working JimTCL then that will suffice for build-time + # TCL purposes (see: proc sqlite-determine-codegen-tcl). + if {![get-define HAVE_TCL] && + ([proj-opt-was-provided tcl] || [proj-opt-was-provided with-tcl])} { + proj-fatal "TCL support was requested but no tclConfig.sh could be found." + } + if {"" eq $cfg} { + proj-assert {0 == [get-define HAVE_TCL]} + proj-indented-notice { + WARNING: Cannot find a usable tclConfig.sh file. Use + --with-tcl=DIR to specify a directory where tclConfig.sh can be + found. SQLite does not use TCL internally, but some optional + components require TCL, including tests and sqlite3_analyzer. + } + } +}; # sqlite-check-tcl + +######################################################################## +# sqlite-determine-codegen-tcl checks which TCL to use as a code +# generator. By default, prefer jimsh simply because we have it +# in-tree (it's part of autosetup) unless --with-tclsh=X is used, in +# which case prefer X. +# +# Returns the human-readable name of the TCL it selects. Fails fatally +# if it cannot detect a TCL appropriate for code generation. +# +# Defines: +# +# - BTCLSH = the TCL shell used for code generation. It may set this +# to an unexpanded makefile var name. +# +# - CFLAGS_JIMSH = any flags needed for buildng a BTCLSH-compatible +# jimsh. The defaults may be passed on to configure as +# CFLAGS_JIMSH=... +proc sqlite-determine-codegen-tcl {} { + msg-result "Checking for TCL to use for code generation... " + define CFLAGS_JIMSH [proj-get-env CFLAGS_JIMSH {-O1}] + set cgtcl [opt-val with-tclsh jimsh] + if {"jimsh" ne $cgtcl} { + # When --with-tclsh=X is used, use that for all TCL purposes, + # including in-tree code generation, per developer request. + define BTCLSH "\$(TCLSH_CMD)" + return $cgtcl + } + set flagsToRestore {CC CFLAGS AS_CFLAGS CPPFLAGS AS_CPPFLAGS LDFLAGS LINKFLAGS LIBS CROSS} + define-push $flagsToRestore { + # We have to swap CC to CC_FOR_BUILD for purposes of the various + # [cc-...] tests below. Recall that --with-wasi-sdk may have + # swapped out CC with one which is not appropriate for this block. + # Per consulation with autosetup's creator, doing this properly + # requires us to [define-push] the whole $flagsToRestore list + # (plus a few others which are not relevant in this tree). + # + # These will get set to their previous values at the end of this + # block. + foreach flag $flagsToRestore {define $flag ""} + define CC [get-define CC_FOR_BUILD] + # These headers are technically optional for JimTCL but necessary if + # we want to use it for code generation: + set sysh [cc-check-includes dirent.h sys/time.h] + # jimsh0.c hard-codes #define's for HAVE_DIRENT_H and + # HAVE_SYS_TIME_H on the platforms it supports, so we do not + # need to add -D... flags for those. We check for them here only + # so that we can avoid the situation that we later, at + # make-time, try to compile jimsh but it then fails due to + # missing headers (i.e. fail earlier rather than later). + if {$sysh && [cc-check-functions realpath]} { + define-append CFLAGS_JIMSH -DHAVE_REALPATH + define BTCLSH "\$(JIMSH)" + set ::sqliteConfig(use-jim-for-codegen) 1 + } elseif {$sysh && [cc-check-functions _fullpath]} { + # _fullpath() is a Windows API. It's not entirely clear + # whether we need to add {-DHAVE_SYS_TIME_H -DHAVE_DIRENT_H} + # to CFLAGS_JIMSH in this case. On MinGW32 we definitely do + # not want to because it already hard-codes them. On _MSC_VER + # builds it does not. + define-append CFLAGS_JIMSH -DHAVE__FULLPATH + define BTCLSH "\$(JIMSH)" + set ::sqliteConfig(use-jim-for-codegen) 1 + } elseif {[file-isexec [get-define TCLSH_CMD]]} { + set cgtcl [get-define TCLSH_CMD] + define BTCLSH "\$(TCLSH_CMD)" + } else { + # One last-ditch effort to find TCLSH_CMD: use info from + # tclConfig.sh to try to find a tclsh + if {"" eq [get-define TCLSH_CMD]} { + set tpre [get-define TCL_EXEC_PREFIX] + if {"" ne $tpre} { + set tv [get-define TCL_VERSION] + if {[file-isexec "${tpre}/bin/tclsh${tv}"]} { + define TCLSH_CMD "${tpre}/bin/tclsh${tv}" + } elseif {[file-isexec "${tpre}/bin/tclsh"]} { + define TCLSH_CMD "${tpre}/bin/tclsh" + } + } + } + set cgtcl [get-define TCLSH_CMD] + if {![file-isexec $cgtcl]} { + proj-fatal "Cannot find a tclsh to use for code generation." + } + define BTCLSH "\$(TCLSH_CMD)" + } + }; # /define-push $flagsToRestore + return $cgtcl +}; # sqlite-determine-codegen-tcl + +######################################################################## +# Runs sqlite-check-tcl and, if this is the canonical build, +# sqlite-determine-codegen-tcl. +proc sqlite-handle-tcl {} { + sqlite-check-tcl + if {"canonical" ne $::sqliteConfig(build-mode)} return + msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + + # Determine the base name of the Tcl extension's DLL + # + if {[get-define HAVE_TCL]} { + if {[string match *-cygwin [get-define host]]} { + set libname cyg + } else { + set libname lib + } + if {[get-define TCL_MAJOR_VERSION] > 8} { + append libname tcl9 + } + append libname sqlite + } else { + set libname "" + } + define TCL_EXT_DLL_BASENAME $libname + # The extension is added in the makefile +} + +######################################################################## +# Handle the --enable/disable-rpath flag. +proc sqlite-handle-rpath {} { + # autosetup/cc-shared.tcl sets the rpath flag definition in + # [get-define SH_LINKRPATH], but it does so on a per-platform basis + # rather than as a compiler check. Though we should do a proper + # compiler check (as proj-check-rpath does), we may want to consider + # adopting its approach of clearing the rpath flags for environments + # for which sqlite-env-is-unix-on-windows returns a non-empty + # string. + + # https://sqlite.org/forum/forumpost/13cac3b56516f849 + if {[proj-opt-truthy rpath]} { + proj-check-rpath + } else { + msg-result "Disabling use of rpath." + define LDFLAGS_RPATH "" + } +} + +######################################################################## +# If the --dump-defines configure flag is provided then emit a list of +# all [define] values to config.defines.txt, else do nothing. +proc sqlite-dump-defines {} { + proj-if-opt-truthy dump-defines { + make-config-header $::sqliteConfig(dump-defines-txt) \ + -bare {SQLITE_OS* SQLITE_DEBUG USE_*} \ + -str {BIN_* CC LD AR LDFLAG* OPT_*} \ + -auto {*} + # achtung: ^^^^ whichever SQLITE_OS_foo flag which is set to 0 will + # get _undefined_ here unless it's part of the -bare set. + if {"" ne $::sqliteConfig(dump-defines-json)} { + msg-result "--dump-defines is creating $::sqliteConfig(dump-defines-json)" + ######################################################################## + # Dump config-defines.json... + # Demonstrate (mis?)handling of spaces in JSON-export array values: + # define-append OPT_FOO.list {"-DFOO=bar baz" -DBAR="baz barre"} + define OPT_FEATURE_FLAGS.list [get-define OPT_FEATURE_FLAGS] + define OPT_SHELL.list [get-define OPT_SHELL] + set dumpDefsOpt { + -bare {SIZEOF_* HAVE_DECL_*} + -none {HAVE_CFLAG_* LDFLAGS_* SH_* SQLITE_AUTORECONFIG TARGET_* USE_GCOV TCL_*} + -array {*.list} + -auto {OPT_* PACKAGE_* HAVE_*} + } +# if {$::sqliteConfig(dump-defines-json-include-lowercase)} { +# lappend dumpDefsOpt -none {lib_*} ; # remnants from proj-check-function-in-lib and friends +# lappend dumpDefsOpt -auto {[a-z]*} +# } + lappend dumpDefsOpt -none * + proj-dump-defs-json $::sqliteConfig(dump-defines-json) {*}$dumpDefsOpt + undefine OPT_FEATURE_FLAGS.list + undefine OPT_SHELL.list + } + } +} diff --git a/autosetup/system.tcl b/autosetup/system.tcl new file mode 100644 index 0000000000..05d378afdd --- /dev/null +++ b/autosetup/system.tcl @@ -0,0 +1,420 @@ +# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ +# All rights reserved + +# @synopsis: +# +# This module supports common system interrogation and options +# such as '--host', '--build', '--prefix', and setting 'srcdir', 'builddir', and 'EXEEXT'. +# +# It also support the "feature" naming convention, where searching +# for a feature such as 'sys/type.h' defines 'HAVE_SYS_TYPES_H'. +# +# It defines the following variables, based on '--prefix' unless overridden by the user: +# +## datadir +## sysconfdir +## sharedstatedir +## localstatedir +## infodir +## mandir +## includedir +# +# If '--prefix' is not supplied, it defaults to '/usr/local' unless 'options-defaults { prefix ... }' is used *before* +# including the 'system' module. + +if {[is-defined defaultprefix]} { + user-notice "Note: defaultprefix is deprecated. Use options-defaults to set default options" + options-defaults [list prefix [get-define defaultprefix]] +} + +options { + host:host-alias => {a complete or partial cpu-vendor-opsys for the system where + the application will run (defaults to the same value as --build)} + build:build-alias => {a complete or partial cpu-vendor-opsys for the system + where the application will be built (defaults to the + result of running config.guess)} + prefix:dir=/usr/local => {the target directory for the build (default: '@default@')} + + # These (hidden) options are supported for autoconf/automake compatibility + exec-prefix: + bindir: + sbindir: + includedir: + mandir: + infodir: + libexecdir: + datadir: + libdir: + sysconfdir: + sharedstatedir: + localstatedir: + runstatedir: + maintainer-mode=0 + dependency-tracking=0 + silent-rules=0 + program-prefix: + program-suffix: + program-transform-name: + x-includes: + x-libraries: +} + +# @check-feature name { script } +# +# defines feature '$name' to the return value of '$script', +# which should be 1 if found or 0 if not found. +# +# e.g. the following will define 'HAVE_CONST' to 0 or 1. +# +## check-feature const { +## cctest -code {const int _x = 0;} +## } +proc check-feature {name code} { + msg-checking "Checking for $name..." + set r [uplevel 1 $code] + define-feature $name $r + if {$r} { + msg-result "ok" + } else { + msg-result "not found" + } + return $r +} + +# @have-feature name ?default=0? +# +# Returns the value of feature '$name' if defined, or '$default' if not. +# +# See 'feature-define-name' for how the "feature" name +# is translated into the "define" name. +# +proc have-feature {name {default 0}} { + get-define [feature-define-name $name] $default +} + +# @define-feature name ?value=1? +# +# Sets the feature 'define' to '$value'. +# +# See 'feature-define-name' for how the "feature" name +# is translated into the "define" name. +# +proc define-feature {name {value 1}} { + define [feature-define-name $name] $value +} + +# @feature-checked name +# +# Returns 1 if feature '$name' has been checked, whether true or not. +# +proc feature-checked {name} { + is-defined [feature-define-name $name] +} + +# @feature-define-name name ?prefix=HAVE_? +# +# Converts a "feature" name to the corresponding "define", +# e.g. 'sys/stat.h' becomes 'HAVE_SYS_STAT_H'. +# +# Converts '*' to 'P' and all non-alphanumeric to underscore. +# +proc feature-define-name {name {prefix HAVE_}} { + string toupper $prefix[regsub -all {[^a-zA-Z0-9]} [regsub -all {[*]} $name p] _] +} + +# @write-if-changed filename contents ?script? +# +# If '$filename' doesn't exist, or it's contents are different to '$contents', +# the file is written and '$script' is evaluated. +# +# Otherwise a "file is unchanged" message is displayed. +proc write-if-changed {file buf {script {}}} { + set old [readfile $file ""] + if {$old eq $buf && [file exists $file]} { + msg-result "$file is unchanged" + } else { + writefile $file $buf\n + uplevel 1 $script + } +} + + +# @include-file infile mapping +# +# The core of make-template, called recursively for each @include +# directive found within that template so that this proc's result +# is the fully-expanded template. +# +# The mapping parameter is how we expand @varname@ within the template. +# We do that inline within this step only for @include directives which +# can have variables in the filename arg. A separate substitution pass +# happens when this recursive function returns, expanding the rest of +# the variables. +# +proc include-file {infile mapping} { + # A stack of true/false conditions, one for each nested conditional + # starting with "true" + set condstack {1} + set result {} + set linenum 0 + foreach line [split [readfile $infile] \n] { + incr linenum + if {[regexp {^@(if|else|endif)(\s*)(.*)} $line -> condtype condspace condargs]} { + if {$condtype eq "if"} { + if {[string length $condspace] == 0} { + autosetup-error "$infile:$linenum: Invalid expression: $line" + } + if {[llength $condargs] == 1} { + # ABC => [get-define ABC] ni {0 ""} + # !ABC => [get-define ABC] in {0 ""} + lassign $condargs condvar + if {[regexp {^!(.*)} $condvar -> condvar]} { + set op in + } else { + set op ni + } + set condexpr "\[[list get-define $condvar]\] $op {0 {}}" + } else { + # Translate alphanumeric ABC into [get-define ABC] and leave the + # rest of the expression untouched + regsub -all {([A-Z][[:alnum:]_]*)} $condargs {[get-define \1]} condexpr + } + if {[catch [list expr $condexpr] condval]} { + dputs $condval + autosetup-error "$infile:$linenum: Invalid expression: $line" + } + dputs "@$condtype: $condexpr => $condval" + } + if {$condtype ne "if"} { + if {[llength $condstack] <= 1} { + autosetup-error "$infile:$linenum: Error: @$condtype missing @if" + } elseif {[string length $condargs] && [string index $condargs 0] ne "#"} { + autosetup-error "$infile:$linenum: Error: Extra arguments after @$condtype" + } + } + switch -exact $condtype { + if { + # push condval + lappend condstack $condval + } + else { + # Toggle the last entry + set condval [lpop condstack] + set condval [expr {!$condval}] + lappend condstack $condval + } + endif { + if {[llength $condstack] == 0} { + user-notice "$infile:$linenum: Error: @endif missing @if" + } + lpop condstack + } + } + continue + } + # Only continue if the stack contains all "true" + if {"0" in $condstack} { + continue + } + if {[regexp {^@include\s+(.*)} $line -> filearg]} { + set incfile [string map $mapping $filearg] + if {[file exists $incfile]} { + lappend ::autosetup(deps) [file-normalize $incfile] + lappend result {*}[include-file $incfile $mapping] + } else { + user-error "$infile:$linenum: Include file $incfile is missing" + } + continue + } + if {[regexp {^@define\s+(\w+)\s+(.*)} $line -> var val]} { + define $var $val + continue + } + lappend result $line + } + return $result +} + + +# @make-template template ?outfile? +# +# Reads the input file '/$template' and writes the output file '$outfile' +# (unless unchanged). +# If '$outfile' is blank/omitted, '$template' should end with '.in' which +# is removed to create the output file name. +# +# Each pattern of the form '@define@' is replaced with the corresponding +# "define", if it exists, or left unchanged if not. +# +# The special value '@srcdir@' is substituted with the relative +# path to the source directory from the directory where the output +# file is created, while the special value '@top_srcdir@' is substituted +# with the relative path to the top level source directory. +# +# Conditional sections may be specified as follows: +## @if NAME eq "value" +## lines +## @else +## lines +## @endif +# +# Where 'NAME' is a defined variable name and '@else' is optional. +# Note that variables names *must* start with an uppercase letter. +# If the expression does not match, all lines through '@endif' are ignored. +# +# The alternative forms may also be used: +## @if NAME (true if the variable is defined, but not empty and not "0") +## @if !NAME (opposite of the form above) +## @if +# +# In the general Tcl expression, any words beginning with an uppercase letter +# are translated into [get-define NAME] +# +# Expressions may be nested +# +proc make-template {template {out {}}} { + set infile [file join $::autosetup(srcdir) $template] + + if {![file exists $infile]} { + user-error "Template $template is missing" + } + + # Define this as late as possible + define AUTODEPS $::autosetup(deps) + + if {$out eq ""} { + if {[file ext $template] ne ".in"} { + autosetup-error "make_template $template has no target file and can't guess" + } + set out [file rootname $template] + } + + set outdir [file dirname $out] + + # Make sure the directory exists + file mkdir $outdir + + # Set up srcdir and top_srcdir to be relative to the target dir + define srcdir [relative-path [file join $::autosetup(srcdir) $outdir] $outdir] + define top_srcdir [relative-path $::autosetup(srcdir) $outdir] + + # Build map from global defines to their values so they can be + # substituted into @include file names. + proc build-define-mapping {} { + set mapping {} + foreach {n v} [array get ::define] { + lappend mapping @$n@ $v + } + return $mapping + } + set mapping [build-define-mapping] + + set result [include-file $infile $mapping] + + # Rebuild the define mapping in case we ran across @define + # directives in the template or a file it @included, then + # apply that mapping to the expanded template. + set mapping [build-define-mapping] + write-if-changed $out [string map $mapping [join $result \n]] { + msg-result "Created [relative-path $out] from [relative-path $template]" + } +} + +proc system-init {} { + global autosetup + + # build/host tuples and cross-compilation prefix + opt-str build build "" + define build_alias $build + if {$build eq ""} { + define build [config_guess] + } else { + define build [config_sub $build] + } + + opt-str host host "" + define host_alias $host + if {$host eq ""} { + define host [get-define build] + set cross "" + } else { + define host [config_sub $host] + set cross $host- + } + define cross [get-env CROSS $cross] + + # build/host _cpu, _vendor and _os + foreach type {build host} { + set v [get-define $type] + if {![regexp {^([^-]+)-([^-]+)-(.*)$} $v -> cpu vendor os]} { + user-error "Invalid canonical $type: $v" + } + define ${type}_cpu $cpu + define ${type}_vendor $vendor + define ${type}_os $os + } + + opt-str prefix prefix /usr/local + + # These are for compatibility with autoconf + define target [get-define host] + define prefix $prefix + define builddir $autosetup(builddir) + define srcdir $autosetup(srcdir) + define top_srcdir $autosetup(srcdir) + define abs_top_srcdir [file-normalize $autosetup(srcdir)] + define abs_top_builddir [file-normalize $autosetup(builddir)] + + # autoconf supports all of these + define exec_prefix [opt-str exec-prefix exec_prefix $prefix] + foreach {name defpath} { + bindir /bin + sbindir /sbin + libexecdir /libexec + libdir /lib + } { + define $name [opt-str $name o $exec_prefix$defpath] + } + foreach {name defpath} { + datadir /share + sharedstatedir /com + infodir /share/info + mandir /share/man + includedir /include + } { + define $name [opt-str $name o $prefix$defpath] + } + if {$prefix ne {/usr}} { + opt-str sysconfdir sysconfdir $prefix/etc + } else { + opt-str sysconfdir sysconfdir /etc + } + define sysconfdir $sysconfdir + + define localstatedir [opt-str localstatedir o /var] + define runstatedir [opt-str runstatedir o /run] + + define SHELL [get-env SHELL [find-an-executable sh bash ksh]] + + # These could be used to generate Makefiles following some automake conventions + define AM_SILENT_RULES [opt-bool silent-rules] + define AM_MAINTAINER_MODE [opt-bool maintainer-mode] + define AM_DEPENDENCY_TRACKING [opt-bool dependency-tracking] + + # Windows vs. non-Windows + switch -glob -- [get-define host] { + *-*-ming* - *-*-cygwin - *-*-msys { + define-feature windows + define EXEEXT .exe + } + default { + define EXEEXT "" + } + } + + # Display + msg-result "Host System...[get-define host]" + msg-result "Build System...[get-define build]" +} + +system-init diff --git a/autosetup/teaish/README.txt b/autosetup/teaish/README.txt new file mode 100644 index 0000000000..e11519b042 --- /dev/null +++ b/autosetup/teaish/README.txt @@ -0,0 +1,4 @@ +The *.tcl files in this directory are part of the SQLite's "autoconf" +bundle which are specific to the TEA(-ish) build. During the tarball +generation process, they are copied into /autoconf/autosetup/teaish +(which itself is created as part of that process). diff --git a/autosetup/teaish/core.tcl b/autosetup/teaish/core.tcl new file mode 100644 index 0000000000..c9abfa0626 --- /dev/null +++ b/autosetup/teaish/core.tcl @@ -0,0 +1,2564 @@ +######################################################################## +# 2025 April 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# +######################################################################## +# ----- @module teaish.tcl ----- +# @section TEA-ish ((TCL Extension Architecture)-ish) +# +# Functions in this file with a prefix of teaish__ are +# private/internal APIs. Those with a prefix of teaish- are +# public APIs. +# +# Teaish has a hard dependency on proj.tcl, and any public API members +# of that module are considered legal for use by teaish extensions. +# +# Project home page: https://fossil.wanderinghorse.net/r/teaish + +use proj + +# +# API-internal settings and shared state. +array set teaish__Config [proj-strip-hash-comments { + # + # Teaish's version number, not to be confused with + # teaish__PkgInfo(-version). + # + version 0.1-beta + + # set to 1 to enable some internal debugging output + debug-enabled 0 + + # + # 0 = don't yet have extension's pkgindex + # 0x01 = found TEAISH_EXT_DIR/pkgIndex.tcl.in + # 0x02 = found srcdir/pkgIndex.tcl.in + # 0x10 = found TEAISH_EXT_DIR/pkgIndex.tcl (static file) + # 0x20 = static-pkgIndex.tcl pragma: behave as if 0x10 + # 0x100 = disabled by -tm.tcl.in + # 0x200 = disabled by -tm.tcl + # + # Reminder: it's significant that the bottom 4 bits be + # cases where teaish manages ./pkgIndex.tcl. + # + pkgindex-policy 0 + + # + # The pkginit counterpart of pkgindex-policy: + # + # 0 = no pkginit + # 0x01 = found default X.in: generate X from X.in + # 0x10 = found static pkginit file X + # 0x02 = user-provided X.in generates ./X. + # 0x20 = user-provided static pkginit file X + # + # The 0x0f bits indicate that teaish is responsible for cleaning up + # the (generated) pkginit file. + # + pkginit-policy 0 + # + # 0 = no tm.tcl + # 0x01 = tm.tcl.in + # 0x10 = static tm.tcl + tm-policy 0 + + # + # If 1+ then teaish__verbose will emit messages. + # + verbose 0 + + # + # Mapping of pkginfo -flags to their TEAISH_xxx define (if any). + # This must not be modified after initialization. + # + pkginfo-f2d { + -name TEAISH_NAME + -name.dist TEAISH_DIST_NAME + -name.pkg TEAISH_PKGNAME + -version TEAISH_VERSION + -libDir TEAISH_LIBDIR_NAME + -loadPrefix TEAISH_LOAD_PREFIX + -vsatisfies TEAISH_VSATISFIES + -pkgInit.tcl TEAISH_PKGINIT_TCL + -pkgInit.tcl.in TEAISH_PKGINIT_TCL_IN + -url TEAISH_URL + -tm.tcl TEAISH_TM_TCL + -tm.tcl.in TEAISH_TM_TCL_IN + -options {} + -pragmas {} + -src {} + } + + # + # Queues for use with teaish-checks-queue and teaish-checks-run. + # + queued-checks-pre {} + queued-checks-post {} + + # Whether or not "make dist" parts are enabled. They get enabled + # when building from an extension's dir, disabled when building + # elsewhere. + dist-enabled 1 + # Whether or not "make install" parts are enabled. By default + # they are, but we have a single use case where they're + # both unnecessary and unhelpful, so... + install-enabled 1 + + # By default we enable compilation of a native extension but if the + # extension has no native code or the user wants to take that over + # via teaish.make.in or provide a script-only extension, we will + # elide the default compilation rules if this is 0. + dll-enabled 1 + + # Files to include in the "make dist" bundle. + dist-files {} + + # List of source files for the extension. + extension-src {} + + # Path to the teaish.tcl file. + teaish.tcl {} + + # Dir where teaish.tcl is found. + extension-dir {} + + # Whether the generates TEASH_VSATISFIES_CODE should error out on a + # satisfies error. If 0, it uses return instead of error. + vsatisfies-error 1 + + # Whether or not to allow a "full dist" - a "make dist" build which + # includes both the extension and teaish. By default this is only on + # if the extension dir is teaish's dir. + dist-full-enabled 0 +}] +set teaish__Config(core-dir) $::autosetup(libdir)/teaish + +# +# Array of info managed by teaish-pkginfo-get and friends. Has the +# same set of keys as $teaish__Config(pkginfo-f2d). +# +array set teaish__PkgInfo {} + +# +# Runs {*}$args if $lvl is <= the current verbosity level, else it has +# no side effects. +# +proc teaish__verbose {lvl args} { + if {$lvl <= $::teaish__Config(verbose)} { + {*}$args + } +} + +# +# @teaish-argv-has flags... +# +# Returns true if any arg in $::argv matches any of the given globs, +# else returns false. +# +proc teaish-argv-has {args} { + foreach glob $args { + foreach arg $::argv { + if {[string match $glob $arg]} { + return 1 + } + } + } + return 0 +} + +if {[teaish-argv-has --teaish-verbose --t-v]} { + # Check this early so that we can use verbose-only messages in the + # pre-options-parsing steps. + set ::teaish__Config(verbose) 1 + #teaish__verbose 1 msg-result "--teaish-verbose activated" +} + +msg-quiet use system ; # Outputs "Host System" and "Build System" lines +if {"--help" ni $::argv} { + teaish__verbose 1 msg-result "TEA(ish) Version = $::teaish__Config(version)" + teaish__verbose 1 msg-result "Source dir = $::autosetup(srcdir)" + teaish__verbose 1 msg-result "Build dir = $::autosetup(builddir)" +} + +# +# @teaish-configure-core +# +# Main entry point for the TEA-ish configure process. auto.def's primary +# (ideally only) job should be to call this. +# +proc teaish-configure-core {} { + proj-tweak-default-env-dirs + + set ::teaish__Config(install-mode) [teaish-argv-has --teaish-install*] + set ::teaish__Config(create-ext-mode) \ + [teaish-argv-has --teaish-create-extension=* --t-c-e=*] + set gotExt 0; # True if an extension config is found + if {!$::teaish__Config(create-ext-mode) + && !$::teaish__Config(install-mode)} { + # Don't look for an extension if we're in --t-c-e or --t-i mode + set gotExt [teaish__find_extension] + } + + # + # Set up the core --flags. This needs to come before teaish.tcl is + # sourced so that that file can use teaish-pkginfo-set to append + # options. + # + options-add [proj-strip-hash-comments { + with-tcl:DIR + => {Directory containing tclConfig.sh or a directory one level up from + that, from which we can derive a directory containing tclConfig.sh. + Defaults to the $TCL_HOME environment variable.} + + with-tclsh:PATH + => {Full pathname of tclsh to use. It is used for trying to find + tclConfig.sh. Warning: if its containing dir has multiple tclsh + versions, it may select the wrong tclConfig.sh! + Defaults to the $TCLSH environment variable.} + + tcl-stubs=0 => {Enable use of Tcl stubs library.} + + # TEA has --with-tclinclude but it appears to only be useful for + # building an extension against an uninstalled copy of TCL's own + # source tree. The policy here is that either we get that info + # from tclConfig.sh or we give up. + # + # with-tclinclude:DIR + # => {Specify the directory which contains the tcl.h. This should not + # normally be required, as that information comes from tclConfig.sh.} + + # We _generally_ want to reduce the possibility of flag collisions with + # extensions, and thus use a teaish-... prefix on most flags. However, + # --teaish-extension-dir is frequently needed, so... + # + # As of this spontaneous moment, we'll settle on using --t-A-X to + # abbreviate --teaish-A...-X... flags when doing so is + # unambiguous... + ted: t-e-d: + teaish-extension-dir:DIR + => {Looks for an extension in the given directory instead of the current + dir.} + + t-c-e: + teaish-create-extension:TARGET_DIRECTORY + => {Writes stub files for creating an extension. Will refuse to overwrite + existing files without --teaish-force.} + + t-f + teaish-force + => {Has a context-dependent meaning (autosetup defines --force for its + own use).} + + t-d-d + teaish-dump-defines + => {Dump all configure-defined vars to config.defines.txt} + + t-v:=0 + teaish-verbose:=0 + => {Enable more (often extraneous) messages from the teaish core.} + + t-d + teaish-debug=0 => {Enable teaish-specific debug output} + + t-i + teaish-install:=auto + => {Installs a copy of teaish, including autosetup, to the target dir. + When used with --teaish-create-extension=DIR, a value of "auto" + (no no value) will inherit that directory.} + + #TODO: --teaish-install-extension:=dir as short for + # --t-c-e=dir --t-i + + t-e-p: + teaish-extension-pkginfo:pkginfo + => {For use with --teaish-create-extension. If used, it must be a + list of arguments for use with teaish-pkginfo-set, e.g. + --teaish-extension-pkginfo="-name Foo -version 2.3"} + + t-v-c + teaish-vsatisfies-check=1 + => {Disable the configure-time "vsatisfies" check on the target tclsh.} + + }]; # main options. + + if {$gotExt} { + # We found an extension. Source it... + set ttcl $::teaish__Config(teaish.tcl) + proj-assert {"" ne [teaish-pkginfo-get -name]} + proj-assert {[file exists $ttcl]} \ + "Expecting to have found teaish.(tcl|config) by now" + if {[string match *.tcl $ttcl]} { + uplevel 1 {source $::teaish__Config(teaish.tcl)} + } else { + teaish-pkginfo-set {*}[proj-file-content -trim $ttcl] + } + unset ttcl + # Set up some default values if the extension did not set them. + # This must happen _after_ it's sourced but before + # teaish-configure is called. + array set f2d $::teaish__Config(pkginfo-f2d) + foreach {pflag key type val} { + - TEAISH_CFLAGS -v "" + - TEAISH_LDFLAGS -v "" + - TEAISH_MAKEFILE -v "" + - TEAISH_MAKEFILE_CODE -v "" + - TEAISH_MAKEFILE_IN -v "" + - TEAISH_PKGINDEX_TCL -v "" + - TEAISH_PKGINDEX_TCL_IN -v "" + - TEAISH_PKGINIT_TCL -v "" + - TEAISH_PKGINIT_TCL_IN -v "" + - TEAISH_PKGINIT_TCL_TAIL -v "" + - TEAISH_TEST_TCL -v "" + - TEAISH_TEST_TCL_IN -v "" + + -version - -v 0.0.0 + -name.pkg - -e {set ::teaish__PkgInfo(-name)} + -name.dist - -e {set ::teaish__PkgInfo(-name)} + -libDir - -e { + join [list \ + $::teaish__PkgInfo(-name.pkg) \ + $::teaish__PkgInfo(-version)] "" + } + -loadPrefix - -e { + string totitle $::teaish__PkgInfo(-name.pkg) + } + -vsatisfies - -v {{Tcl 8.5-}} + -pkgInit.tcl - -v "" + -pkgInit.tcl.in - -v "" + -url - -v "" + -tm.tcl - -v "" + -tm.tcl.in - -v "" + -src - -v "" + } { + #proj-assert 0 {Just testing} + set isPIFlag [expr {"-" ne $pflag}] + if {$isPIFlag} { + if {[info exists ::teaish__PkgInfo($pflag)]} { + # Was already set - skip it. + continue; + } + proj-assert {{-} eq $key};# "Unexpected pflag=$pflag key=$key type=$type val=$val" + set key $f2d($pflag) + } + if {"" ne $key} { + if {"" ne [get-define $key ""]} { + # Was already set - skip it. + continue + } + } + switch -exact -- $type { + -v {} + -e { set val [eval $val] } + default { proj-error "Invalid type flag: $type" } + } + #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag" + if {$key ne ""} { + define $key $val + } + if {$isPIFlag} { + set ::teaish__PkgInfo($pflag) $val + } + } + unset isPIFlag pflag key type val + array unset f2d + }; # sourcing extension's teaish.tcl + + if {[llength [info proc teaish-options]] > 0} { + # Add options defined by teaish-options, which is assumed to be + # imported via [teaish-get -teaish-tcl]. + set o [teaish-options] + if {"" ne $o} { + options-add $o + } + } + #set opts [proj-options-combine] + #lappend opts teaish-debug => {x}; #testing dupe entry handling + if {[catch {options {}} msg xopts]} { + # Workaround for + # where [options] behaves oddly on _some_ TCL builds when it's + # called from deeper than the global scope. + dict incr xopts -level + return {*}$xopts $msg + } + + proj-xfer-options-aliases { + t-c-e => teaish-create-extension + t-d => teaish-debug + t-d-d => teaish-dump-defines + ted => teaish-extension-dir + t-e-d => teaish-extension-dir + t-e-p => teaish-extension-pkginfo + t-f => teaish-force + t-i => teaish-install + t-v => teaish-verbose + t-v-c => teaish-vsatisfies-check + } + + scan [opt-val teaish-verbose 0] %d ::teaish__Config(verbose) + set ::teaish__Config(debug-enabled) [opt-bool teaish-debug] + + set exitEarly 0 + if {[proj-opt-was-provided teaish-create-extension]} { + teaish__create_extension [opt-val teaish-create-extension] + incr exitEarly + } + if {$::teaish__Config(install-mode)} { + teaish__install + incr exitEarly + } + + if {$exitEarly} { + file delete -force config.log + return + } + proj-assert {1==$gotExt} "Else we cannot have gotten this far" + + teaish__configure_phase1 +} + + +# +# Internal config-time debugging output routine. It is not legal to +# call this from the global scope. +# +proc teaish-debug {msg} { + if {$::teaish__Config(debug-enabled)} { + puts stderr [proj-bold "** DEBUG: \[[proj-scope 1]\]: $msg"] + } +} + +# +# Runs "phase 1" of the configuration, immediately after processing +# --flags. This is what will import the client-defined teaish.tcl. +# +proc teaish__configure_phase1 {} { + msg-result \ + [join [list "Configuring build of Tcl extension" \ + [proj-bold [teaish-pkginfo-get -name] \ + [teaish-pkginfo-get -version]] "..."]] + + uplevel 1 { + use cc cc-db cc-shared cc-lib; # pkg-config + } + teaish__check_tcl + apply {{} { + # + # If --prefix or --exec-prefix are _not_ provided, use their + # TCL_... counterpart from tclConfig.sh. Caveat: by the time we can + # reach this point, autosetup's system.tcl will have already done + # some non-trivial amount of work with these to create various + # derived values from them, so we temporarily end up with a mishmash + # of autotools-compatibility var values. That will be straightened + # out in the final stage of the configure script via + # [proj-remap-autoconf-dir-vars]. + # + foreach {flag uflag tclVar} { + prefix prefix TCL_PREFIX + exec-prefix exec_prefix TCL_EXEC_PREFIX + } { + if {![proj-opt-was-provided $flag]} { + if {"exec-prefix" eq $flag} { + # If --exec-prefix was not used, ensure that --exec-prefix + # derives from the --prefix we may have just redefined. + set v {${prefix}} + } else { + set v [get-define $tclVar "???"] + teaish__verbose 1 msg-result "Using \$$tclVar for --$flag=$v" + } + proj-assert {"???" ne $v} "Expecting teaish__check_tcl to have defined $tclVar" + #puts "*** $flag $uflag $tclVar = $v" + proj-opt-set $flag $v + define $uflag $v + + # ^^^ As of here, all autotools-compatibility vars which derive + # from --$flag, e.g. --libdir, still derive from the default + # --$flag value which was active when system.tcl was + # included. So long as those flags are not explicitly passed to + # the configure script, those will be straightened out via + # [proj-remap-autoconf-dir-vars]. + } + } + }}; # --[exec-]prefix defaults + teaish__check_common_bins + # + # Set up library file names + # + proj-file-extensions + teaish__define_pkginfo_derived * + + teaish-checks-run -pre + if {[llength [info proc teaish-configure]] > 0} { + # teaish-configure is assumed to be imported via + # teaish.tcl + teaish-configure + } + teaish-checks-run -post + + define TEAISH_USE_STUBS [opt-bool tcl-stubs] + + apply {{} { + # Set up "vsatisfies" code for pkgIndex.tcl.in, + # _teaish.tester.tcl.in, and for a configure-time check. We would + # like to put this before [teaish-checks-run -pre] but it's + # marginally conceivable that a client may need to dynamically + # calculate the vsatisfies and set it via [teaish-configure]. + set vs [get-define TEAISH_VSATISFIES ""] + if {"" eq $vs} return + set code {} + set n 0 + # Treat $vs as a list-of-lists {{Tcl 8.5-} {Foo 1.0- -3.0} ...} + # and generate Tcl which will run package vsatisfies tests with + # that info. + foreach pv $vs { + set n [llength $pv] + if {$n < 2} { + proj-error "-vsatisfies: {$pv} appears malformed. Whole list is: $vs" + } + set pkg [lindex $pv 0] + set vcheck {} + for {set i 1} {$i < $n} {incr i} { + lappend vcheck [lindex $pv $i] + } + if {[opt-bool teaish-vsatisfies-check]} { + set tclsh [get-define TCLSH_CMD] + set vsat "package vsatisfies \[ package provide $pkg \] $vcheck" + set vputs "puts \[ $vsat \]" + #puts "*** vputs = $vputs" + scan [exec echo $vputs | $tclsh] %d vvcheck + if {![info exists vvcheck] || 0 == $vvcheck} { + proj-fatal -up $tclsh "check failed:" $vsat + } + } + if {$::teaish__Config(vsatisfies-error)} { + set vunsat \ + [list error [list Package \ + $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) \ + requires $pv]] + } else { + set vunsat return + } + lappend code \ + [string trim [subst -nocommands \ + {if { ![package vsatisfies [package provide $pkg] $vcheck] } {\n $vunsat\n}}]] + }; # foreach pv + define TEAISH_VSATISFIES_CODE [join $code "\n"] + }}; # vsatisfies + + if {[proj-looks-like-windows]} { + # Without this, linking of an extension will not work on Cygwin or + # Msys2. + msg-result "Using USE_TCL_STUBS for Unix(ish)-on-Windows environment" + teaish-cflags-add -DUSE_TCL_STUBS=1 + } + + #define AS_LIBDIR $::autosetup(libdir) + define TEAISH_TESTUTIL_TCL $::teaish__Config(core-dir)/tester.tcl + + apply {{} { + # + # Ensure we have a pkgIndex.tcl and don't have a stale generated one + # when rebuilding for different --with-tcl=... values. + # + if {!$::teaish__Config(pkgindex-policy)} { + proj-error "Cannot determine which pkgIndex.tcl to use" + } + if {0x300 & $::teaish__Config(pkgindex-policy)} { + teaish__verbose 1 msg-result "pkgIndex disabled by -tm.tcl(.in)" + } else { + set tpi [proj-coalesce \ + [get-define TEAISH_PKGINDEX_TCL_IN] \ + [get-define TEAISH_PKGINDEX_TCL]] + proj-assert {$tpi ne ""} \ + "TEAISH_PKGINDEX_TCL should have been set up by now" + teaish__verbose 1 msg-result "Using pkgIndex from $tpi" + if {0x0f & $::teaish__Config(pkgindex-policy)} { + # Don't leave stale pkgIndex.tcl laying around yet don't delete + # or overwrite a user-managed static pkgIndex.tcl. + file delete -force -- [get-define TEAISH_PKGINDEX_TCL] + proj-dot-ins-append [get-define TEAISH_PKGINDEX_TCL_IN] + } else { + teaish-dist-add [file tail $tpi] + } + } + }}; # $::teaish__Config(pkgindex-policy) + + # + # Ensure we clean up TEAISH_PKGINIT_TCL if needed and @-process + # TEAISH_PKGINIT_TCL_IN if needed. + # + if {0x0f & $::teaish__Config(pkginit-policy)} { + file delete -force -- [get-define TEAISH_PKGINIT_TCL] + proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] \ + [get-define TEAISH_PKGINIT_TCL] + } + if {0x0f & $::teaish__Config(tm-policy)} { + file delete -force -- [get-define TEAISH_TM_TCL] + proj-dot-ins-append [get-define TEAISH_TM_TCL_IN] + } + + apply {{} { + # Queue up any remaining dot-in files + set dotIns [list] + foreach {dIn => dOut} { + TEAISH_TESTER_TCL_IN => TEAISH_TESTER_TCL + TEAISH_TEST_TCL_IN => TEAISH_TEST_TCL + TEAISH_MAKEFILE_IN => TEAISH_MAKEFILE + } { + lappend dotIns [get-define $dIn ""] [get-define $dOut ""] + } + lappend dotIns $::autosetup(srcdir)/Makefile.in Makefile; # must be after TEAISH_MAKEFILE_IN. + # Much later: probably because of timestamps for deps purposes :-? + #puts "dotIns=$dotIns" + foreach {i o} $dotIns { + if {"" ne $i && "" ne $o} { + #puts " pre-dot-ins-append: \[$i\] -> \[$o\]" + proj-dot-ins-append $i $o + } + } + }} + + define TEAISH_DIST_FULL \ + [expr { + $::teaish__Config(dist-enabled) + && $::teaish__Config(dist-full-enabled) + }] + + define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) + define TEAISH_ENABLE_DIST $::teaish__Config(dist-enabled) + define TEAISH_ENABLE_INSTALL $::teaish__Config(install-enabled) + define TEAISH_ENABLE_DLL $::teaish__Config(dll-enabled) + define TEAISH_TCL $::teaish__Config(teaish.tcl) + + define TEAISH_DIST_FILES [join $::teaish__Config(dist-files)] + define TEAISH_EXT_DIR [join $::teaish__Config(extension-dir)] + define TEAISH_EXT_SRC [join $::teaish__Config(extension-src)] + proj-setup-autoreconfig TEAISH_AUTORECONFIG + foreach f { + TEAISH_CFLAGS + TEAISH_LDFLAGS + } { + # Ensure that any of these lists are flattened + define $f [join [get-define $f]] + } + proj-remap-autoconf-dir-vars + set tdefs [teaish__defines_to_list] + define TEAISH__DEFINES_MAP $tdefs; # injected into _teaish.tester.tcl + + # + # NO [define]s after this point! + # + proj-if-opt-truthy teaish-dump-defines { + proj-file-write config.defines.txt $tdefs + } + proj-dot-ins-process -validate + +}; # teaish__configure_phase1 + +# +# Run checks for required binaries. +# +proc teaish__check_common_bins {} { + if {"" eq [proj-bin-define install]} { + proj-warn "Cannot find install binary, so 'make install' will not work." + define BIN_INSTALL false + } + if {"" eq [proj-bin-define zip]} { + proj-warn "Cannot find zip, so 'make dist.zip' will not work." + } + if {"" eq [proj-bin-define tar]} { + proj-warn "Cannot find tar, so 'make dist.tgz' will not work." + } +} + +# +# TCL... +# +# teaish__check_tcl performs most of the --with-tcl and --with-tclsh +# handling. Some related bits and pieces are performed before and +# after that function is called. +# +# Important [define]'d vars: +# +# - TCLSH_CMD is the path to the canonical tclsh or "". +# +# - TCL_CONFIG_SH is the path to tclConfig.sh or "". +# +# - TCLLIBDIR is the dir to which the extension library gets +# - installed. +# +proc teaish__check_tcl {} { + define TCLSH_CMD false ; # Significant is that it exits with non-0 + define TCLLIBDIR "" ; # Installation dir for TCL extension lib + define TCL_CONFIG_SH ""; # full path to tclConfig.sh + + # Clear out all vars which would harvest from tclConfig.sh so that + # the late-config validation of @VARS@ works even if --disable-tcl + # is used. + proj-tclConfig-sh-to-autosetup "" + + # TODO: better document the steps this is taking. + set srcdir $::autosetup(srcdir) + msg-result "Checking for a suitable tcl... " + set use_tcl 1 + set withSh [opt-val with-tclsh [proj-get-env TCLSH]] + set tclHome [opt-val with-tcl [proj-get-env TCL_HOME]] + if {[string match */lib $tclHome]} { + # TEA compatibility kludge: its --with-tcl wants the lib + # dir containing tclConfig.sh. + #proj-warn "Replacing --with-tcl=$tclHome for TEA compatibility" + regsub {/lib^} $tclHome "" tclHome + msg-result "NOTE: stripped /lib suffix from --with-tcl=$tclHome (a TEA-ism)" + } + if {0} { + # This misinteracts with the $TCL_PREFIX default: it will use the + # autosetup-defined --prefix default + if {"prefix" eq $tclHome} { + set tclHome [get-define prefix] + } + } + teaish-debug "use_tcl ${use_tcl}" + teaish-debug "withSh=${withSh}" + teaish-debug "tclHome=$tclHome" + if {"" eq $withSh && "" eq $tclHome} { + # If neither --with-tclsh nor --with-tcl are provided, try to find + # a workable tclsh. + set withSh [proj-first-bin-of tclsh9.1 tclsh9.0 tclsh8.6 tclsh] + teaish-debug "withSh=${withSh}" + } + + set doConfigLookup 1 ; # set to 0 to test the tclConfig.sh-not-found cases + if {"" ne $withSh} { + # --with-tclsh was provided or found above. Validate it and use it + # to trump any value passed via --with-tcl=DIR. + if {![file-isexec $withSh]} { + proj-error "TCL shell $withSh is not executable" + } else { + define TCLSH_CMD $withSh + #msg-result "Using tclsh: $withSh" + } + if {$doConfigLookup && + [catch {exec $withSh $::autosetup(libdir)/find_tclconfig.tcl} result] == 0} { + set tclHome $result + } + if {"" ne $tclHome && [file isdirectory $tclHome]} { + teaish__verbose 1 msg-result "$withSh recommends the tclConfig.sh from $tclHome" + } else { + proj-warn "$withSh is unable to recommend a tclConfig.sh" + set use_tcl 0 + } + } + set cfg "" + set tclSubdirs {tcl9.1 tcl9.0 tcl8.6 tcl8.5 lib} + while {$use_tcl} { + if {"" ne $tclHome} { + # Ensure that we can find tclConfig.sh under ${tclHome}/... + if {$doConfigLookup} { + if {[file readable "${tclHome}/tclConfig.sh"]} { + set cfg "${tclHome}/tclConfig.sh" + } else { + foreach i $tclSubdirs { + if {[file readable "${tclHome}/$i/tclConfig.sh"]} { + set cfg "${tclHome}/$i/tclConfig.sh" + break + } + } + } + } + if {"" eq $cfg} { + proj-error "No tclConfig.sh found under ${tclHome}" + } + } else { + # If we have not yet found a tclConfig.sh file, look in $libdir + # which is set automatically by autosetup or via the --prefix + # command-line option. See + # https://sqlite.org/forum/forumpost/e04e693439a22457 + set libdir [get-define libdir] + if {[file readable "${libdir}/tclConfig.sh"]} { + set cfg "${libdir}/tclConfig.sh" + } else { + foreach i $tclSubdirs { + if {[file readable "${libdir}/$i/tclConfig.sh"]} { + set cfg "${libdir}/$i/tclConfig.sh" + break + } + } + } + if {![file readable $cfg]} { + break + } + } + teaish__verbose 1 msg-result "Using tclConfig.sh = $cfg" + break + }; # while {$use_tcl} + define TCL_CONFIG_SH $cfg + # Export a subset of tclConfig.sh to the current TCL-space. If $cfg + # is an empty string, this emits empty-string entries for the + # various options we're interested in. + proj-tclConfig-sh-to-autosetup $cfg + + if {"" eq $withSh && $cfg ne ""} { + # We have tclConfig.sh but no tclsh. Attempt to locate a tclsh + # based on info from tclConfig.sh. + set tclExecPrefix [get-define TCL_EXEC_PREFIX] + proj-assert {"" ne $tclExecPrefix} + set tryThese [list \ + $tclExecPrefix/bin/tclsh[get-define TCL_VERSION] \ + $tclExecPrefix/bin/tclsh ] + foreach trySh $tryThese { + if {[file-isexec $trySh]} { + set withSh $trySh + break + } + } + if {![file-isexec $withSh]} { + proj-warn "Cannot find a usable tclsh (tried: $tryThese)" + } + } + define TCLSH_CMD $withSh + if {$use_tcl} { + # Set up the TCLLIBDIR + set tcllibdir [get-env TCLLIBDIR ""] + set extDirName [teaish-pkginfo-get -libDir] + if {"" eq $tcllibdir} { + # Attempt to extract TCLLIBDIR from TCL's $auto_path + if {"" ne $withSh && + [catch {exec echo "puts stdout \$auto_path" | "$withSh"} result] == 0} { + foreach i $result { + if {![string match //zip* $i] && [file isdirectory $i]} { + # isdirectory actually passes on //zipfs:/..., but those are + # useless for our purposes + set tcllibdir $i/$extDirName + break + } + } + } else { + proj-error "Cannot determine TCLLIBDIR." + } + } + define TCLLIBDIR $tcllibdir + }; # find TCLLIBDIR + + set gotSh [file-isexec $withSh] + set tmdir ""; # first tcl::tm::list entry + if {$gotSh} { + catch { + set tmli [exec echo {puts [tcl::tm::list]} | $withSh] + # Reminder: this list contains many names of dirs which do not + # exist but are legitimate. If we rely only on an is-dir check, + # we can end up not finding any of the many candidates. + set firstDir "" + foreach d $tmli { + if {"" eq $firstDir && ![string match //*:* $d]} { + # First non-VFS entry, e.g. not //zipfs: + set firstDir $d + } + if {[file isdirectory $d]} { + set tmdir $d + break + } + } + if {"" eq $tmdir} { + set tmdir $firstDir + } + }; # find tcl::tm path + } + define TEAISH_TCL_TM_DIR $tmdir + + # Finally, let's wrap up... + if {$gotSh} { + teaish__verbose 1 msg-result "Using tclsh = $withSh" + if {$cfg ne ""} { + define HAVE_TCL 1 + } else { + proj-warn "Found tclsh but no tclConfig.sh." + } + if {"" eq $tmdir} { + proj-warn "Did not find tcl::tm directory." + } + } + show-notices + # If TCL is not found: if it was explicitly requested then fail + # fatally, else just emit a warning. If we can find the APIs needed + # to generate a working JimTCL then that will suffice for build-time + # TCL purposes (see: proc sqlite-determine-codegen-tcl). + if {!$gotSh} { + proj-error "Did not find tclsh" + } elseif {"" eq $cfg} { + proj-indented-notice -error { + Cannot find a usable tclConfig.sh file. Use --with-tcl=DIR to + specify a directory near which tclConfig.sh can be found, or + --with-tclsh=/path/to/tclsh to allow the tclsh binary to locate + its tclConfig.sh, with the caveat that a symlink to tclsh, or + wrapper script around it, e.g. ~/bin/tclsh -> + $HOME/tcl/9.0/bin/tclsh9.1, may not work because tclsh emits + different library paths for the former than the latter. + } + } + msg-result "Using Tcl [get-define TCL_VERSION] from [get-define TCL_PREFIX]." + teaish__tcl_platform_quirks +}; # teaish__check_tcl + +# +# Perform last-minute platform-specific tweaks to account for quirks. +# +proc teaish__tcl_platform_quirks {} { + define TEAISH_POSTINST_PREREQUIRE "" + switch -glob -- [get-define host] { + *-haiku { + # Haiku's default TCLLIBDIR is "all wrong": it points to a + # read-only virtual filesystem mount-point. We bend it back to + # fit under $TCL_PACKAGE_PATH here. + foreach {k d} { + vj TCL_MAJOR_VERSION + vn TCL_MINOR_VERSION + pp TCL_PACKAGE_PATH + ld TCLLIBDIR + } { + set $k [get-define $d] + } + if {[string match /packages/* $ld]} { + set old $ld + set tail [file tail $ld] + if {8 == $vj} { + set ld "${pp}/tcl${vj}.${vn}/${tail}" + } else { + proj-assert {9 == $vj} + set ld "${pp}/${tail}" + } + define TCLLIBDIR $ld + # [load foo.so], without a directory part, does not work via + # automated tests on Haiku (but works when run + # manually). Similarly, the post-install [package require ...] + # test fails, presumably for a similar reason. We work around + # the former in _teaish.tester.tcl.in. We work around the + # latter by amending the post-install check's ::auto_path (in + # Makefile.in). This code MUST NOT contain any single-quotes. + define TEAISH_POSTINST_PREREQUIRE \ + [join [list set ::auto_path \ + \[ linsert \$::auto_path 0 $ld \] \; \ + ]] + proj-indented-notice [subst -nocommands -nobackslashes { + Haiku users take note: patching target installation dir to match + Tcl's home because Haiku's is not writable. + + Original : $old + Substitute: $ld + }] + } + } + } +}; # teaish__tcl_platform_quirks + +# +# Searches $::argv and/or the build dir and/or the source dir for +# teaish.tcl and friends. Fails if it cannot find teaish.tcl or if +# there are other irreconcilable problems. If it returns 0 then it did +# not find an extension but the --help flag was seen, in which case +# that's not an error. +# +# This does not _load_ the extension, it primarily locates the files +# which make up an extension and fills out no small amount of teaish +# state related to that. +# +proc teaish__find_extension {} { + proj-assert {!$::teaish__Config(install-mode)} + teaish__verbose 1 msg-result "Looking for teaish extension..." + + # Helper for the foreach loop below. + set checkTeaishTcl {{mustHave fid dir} { + set f [file join $dir $fid] + if {[file readable $f]} { + file-normalize $f + } elseif {$mustHave} { + proj-error "Missing required $dir/$fid" + } + }} + + # + # We have to handle some flags manually because the extension must + # be loaded before [options] is run (so that the extension can + # inject its own options). + # + set dirBld $::autosetup(builddir); # dir we're configuring under + set dirSrc $::autosetup(srcdir); # where teaish's configure script lives + set extT ""; # teaish.tcl + set largv {}; # rewritten $::argv + set gotHelpArg 0; # got the --help + foreach arg $::argv { + #puts "*** arg=$arg" + switch -glob -- $arg { + --ted=* - + --t-e-d=* - + --teaish-extension-dir=* { + # Ensure that $extD refers to a directory and contains a + # teaish.tcl. + regexp -- {--[^=]+=(.+)} $arg - extD + set extD [file-normalize $extD] + if {![file isdirectory $extD]} { + proj-error "--teaish-extension-dir value is not a directory: $extD" + } + set extT [apply $checkTeaishTcl 0 teaish.config $extD] + if {"" eq $extT} { + set extT [apply $checkTeaishTcl 1 teaish.tcl $extD] + } + set ::teaish__Config(extension-dir) $extD + } + --help { + incr gotHelpArg + lappend largv $arg + } + default { + lappend largv $arg + } + } + } + set ::argv $largv + + set dirExt $::teaish__Config(extension-dir); # dir with the extension + # + # teaish.tcl is a TCL script which implements various + # interfaces described by this framework. + # + # We use the first one we find in the builddir or srcdir. + # + if {"" eq $extT} { + set flist [list] + proj-assert {$dirExt eq ""} + lappend flist $dirBld/teaish.tcl $dirBld/teaish.config $dirSrc/teaish.tcl + if {![proj-first-file-found extT $flist]} { + if {$gotHelpArg} { + # Tell teaish-configure-core that the lack of extension is not + # an error when --help or --teaish-install is used. + return 0; + } + proj-indented-notice -error " +Did not find any of: $flist + +If you are attempting an out-of-tree build, use + --teaish-extension-dir=/path/to/extension" + } + } + if {![file readable $extT]} { + proj-error "extension tcl file is not readable: $extT" + } + set ::teaish__Config(teaish.tcl) $extT + set dirExt [file dirname $extT] + + set ::teaish__Config(extension-dir) $dirExt + set ::teaish__Config(blddir-is-extdir) [expr {$dirBld eq $dirExt}] + set ::teaish__Config(dist-enabled) $::teaish__Config(blddir-is-extdir); # may change later + set ::teaish__Config(dist-full-enabled) \ + [expr {[file-normalize $::autosetup(srcdir)] + eq [file-normalize $::teaish__Config(extension-dir)]}] + + set addDist {{file} { + teaish-dist-add [file tail $file] + }} + apply $addDist $extT + + teaish__verbose 1 msg-result "Extension dir = [teaish-get -dir]" + teaish__verbose 1 msg-result "Extension config = $extT" + + teaish-pkginfo-set -name [file tail [file dirname $extT]] + + # + # teaish.make[.in] provides some of the info for the main makefile, + # like which source(s) to build and their build flags. + # + # We use the first one of teaish.make.in or teaish.make we find in + # $dirExt. + # + if {[proj-first-file-found extM \ + [list \ + $dirExt/teaish.make.in \ + $dirExt/teaish.make \ + ]]} { + if {[string match *.in $extM]} { + define TEAISH_MAKEFILE_IN $extM + define TEAISH_MAKEFILE _[file rootname [file tail $extM]] + } else { + define TEAISH_MAKEFILE_IN "" + define TEAISH_MAKEFILE $extM + } + apply $addDist $extM + teaish__verbose 1 msg-result "Extension makefile = $extM" + } else { + define TEAISH_MAKEFILE_IN "" + define TEAISH_MAKEFILE "" + } + + # Look for teaish.pkginit.tcl[.in] + set piPolicy 0 + if {[proj-first-file-found extI \ + [list \ + $dirExt/teaish.pkginit.tcl.in \ + $dirExt/teaish.pkginit.tcl \ + ]]} { + if {[string match *.in $extI]} { + # Generate teaish.pkginit.tcl from $extI. + define TEAISH_PKGINIT_TCL_IN $extI + define TEAISH_PKGINIT_TCL [file rootname [file tail $extI]] + set piPolicy 0x01 + } else { + # Assume static $extI. + define TEAISH_PKGINIT_TCL_IN "" + define TEAISH_PKGINIT_TCL $extI + set piPolicy 0x10 + } + apply $addDist $extI + teaish__verbose 1 msg-result "Extension post-load init = $extI" + define TEAISH_PKGINIT_TCL_TAIL \ + [file tail [get-define TEAISH_PKGINIT_TCL]]; # for use in pkgIndex.tcl.in + } + set ::teaish__Config(pkginit-policy) $piPolicy + + # Look for pkgIndex.tcl[.in]... + set piPolicy 0 + if {[proj-first-file-found extPI $dirExt/pkgIndex.tcl.in]} { + # Generate ./pkgIndex.tcl from $extPI. + define TEAISH_PKGINDEX_TCL_IN $extPI + define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] + apply $addDist $extPI + set piPolicy 0x01 + } elseif {$dirExt ne $dirSrc + && [proj-first-file-found extPI $dirSrc/pkgIndex.tcl.in]} { + # Generate ./pkgIndex.tcl from $extPI. + define TEAISH_PKGINDEX_TCL_IN $extPI + define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] + set piPolicy 0x02 + } elseif {[proj-first-file-found extPI $dirExt/pkgIndex.tcl]} { + # Assume $extPI's a static file and use it. + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL $extPI + apply $addDist $extPI + set piPolicy 0x10 + } + # Reminder: we have to delay removal of stale TEAISH_PKGINDEX_TCL + # and the proj-dot-ins-append of TEAISH_PKGINDEX_TCL_IN until much + # later in the process. + set ::teaish__Config(pkgindex-policy) $piPolicy + + # Look for teaish.test.tcl[.in] + proj-assert {"" ne $dirExt} + set flist [list $dirExt/teaish.test.tcl.in $dirExt/teaish.test.tcl] + if {[proj-first-file-found ttt $flist]} { + if {[string match *.in $ttt]} { + # Generate _teaish.test.tcl from $ttt + set xt _[file rootname [file tail $ttt]] + file delete -force -- $xt; # ensure no stale copy is used + define TEAISH_TEST_TCL $xt + define TEAISH_TEST_TCL_IN $ttt + } else { + define TEAISH_TEST_TCL $ttt + define TEAISH_TEST_TCL_IN "" + } + apply $addDist $ttt + } else { + define TEAISH_TEST_TCL "" + define TEAISH_TEST_TCL_IN "" + } + + # Look for _teaish.tester.tcl[.in] + set flist [list $dirExt/_teaish.tester.tcl.in $dirSrc/_teaish.tester.tcl.in] + if {[proj-first-file-found ttt $flist]} { + # Generate teaish.test.tcl from $ttt + set xt [file rootname [file tail $ttt]] + file delete -force -- $xt; # ensure no stale copy is used + define TEAISH_TESTER_TCL $xt + define TEAISH_TESTER_TCL_IN $ttt + if {[lindex $flist 0] eq $ttt} { + apply $addDist $ttt + } + unset ttt xt + } else { + if {[file exists [set ttt [file join $dirSrc _teaish.tester.tcl.in]]]} { + set xt [file rootname [file tail $ttt]] + define TEAISH_TESTER_TCL $xt + define TEAISH_TESTER_TCL_IN $ttt + } else { + define TEAISH_TESTER_TCL "" + define TEAISH_TESTER_TCL_IN "" + } + } + unset flist + + # TEAISH_OUT_OF_EXT_TREE = 1 if we're building from a dir other + # than the extension's home dir. + define TEAISH_OUT_OF_EXT_TREE \ + [expr {[file-normalize $::autosetup(builddir)] ne \ + [file-normalize $::teaish__Config(extension-dir)]}] + return 1 +}; # teaish__find_extension + +# +# @teaish-cflags-add ?-p|prepend? ?-define? cflags... +# +# Equivalent to [proj-define-amend TEAISH_CFLAGS {*}$args]. +# +proc teaish-cflags-add {args} { + proj-define-amend TEAISH_CFLAGS {*}$args +} + +# +# @teaish-define-to-cflag ?flags? defineName...|{defineName...} +# +# Uses [proj-define-to-cflag] to expand a list of [define] keys, each +# one a separate argument, to CFLAGS-style -D... form then appends +# that to the current TEAISH_CFLAGS. +# +# It accepts these flags from proj-define-to-cflag: -quote, +# -zero-undef. It does _not_ support its -list flag. +# +# It accepts its non-flag argument(s) in 2 forms: (1) each arg is a +# single [define] key or (2) its one arg is a list of such keys. +# +# TODO: document teaish's well-defined (as it were) defines for this +# purpose. At a bare minimum: +# +# - TEAISH_NAME +# - TEAISH_PKGNAME +# - TEAISH_VERSION +# - TEAISH_LIBDIR_NAME +# - TEAISH_LOAD_PREFIX +# - TEAISH_URL +# +proc teaish-define-to-cflag {args} { + set flags {} + while {[string match -* [lindex $args 0]]} { + set arg [lindex $args 0] + switch -exact -- $arg { + -quote - + -zero-undef { + lappend flags $arg + set args [lassign $args -] + } + default break + } + } + if {1 == [llength $args]} { + set args [list {*}[lindex $args 0]] + } + #puts "***** flags=$flags args=$args" + teaish-cflags-add [proj-define-to-cflag {*}$flags {*}$args] +} + +# +# @teaish-cflags-for-tea ?...CFLAGS? +# +# Adds several -DPACKAGE_... CFLAGS using the extension's metadata, +# all as quoted strings. Those symbolic names are commonly used in +# TEA-based builds, and this function is intended to simplify porting +# of such builds. The -D... flags added are: +# +# -DPACKAGE_VERSION=... +# -DPACKAGE_NAME=... +# -DPACKAGE_URL=... +# -DPACKAGE_STRING=... +# +# Any arguments are passed-on as-is to teaish-cflags-add. +# +proc teaish-cflags-for-tea {args} { + set name $::teaish__PkgInfo(-name) + set version $::teaish__PkgInfo(-version) + set pstr [join [list $name $version]] + teaish-cflags-add \ + {*}$args \ + '-DPACKAGE_VERSION="$version"' \ + '-DPACKAGE_NAME="$name"' \ + '-DPACKAGE_STRING="$pstr"' \ + '-DPACKAGE_URL="[teaish-get -url]"' +} + +# +# @teaish-ldflags-add ?-p|-prepend? ?-define? ldflags... +# +# Equivalent to [proj-define-amend TEAISH_LDFLAGS {*}$args]. +# +# Typically, -lXYZ flags need to be in "reverse" order, with each -lY +# resolving symbols for -lX's to its left. This order is largely +# historical, and not relevant on all environments, but it is +# technically correct and still relevant on some environments. +# +# See: teaish-ldflags-prepend +# +proc teaish-ldflags-add {args} { + proj-define-amend TEAISH_LDFLAGS {*}$args +} + +# +# @teaish-ldflags-prepend args... +# +# Functionally equivalent to [teaish-ldflags-add -p {*}$args] +# +proc teaish-ldflags-prepend {args} { + teaish-ldflags-add -p {*}$args +} + +# +# @teaish-src-add ?-dist? ?-dir? src-files... +# +# Appends all non-empty $args to the project's list of C/C++ source or +# (in some cases) object files. +# +# If passed -dist then it also passes each filename, as-is, to +# [teaish-dist-add]. +# +# If passed -dir then each src-file has [teaish-get -dir] prepended to +# it before they're added to the list. As often as not, that will be +# the desired behavior so that out-of-tree builds can find the +# sources, but there are cases where it's not desired (e.g. when using +# a source file from outside of the extension's dir, or when adding +# object files (which are typically in the build tree)). +# +proc teaish-src-add {args} { + proj-parse-simple-flags args flags { + -dist 0 {expr 1} + -dir 0 {expr 1} + } + if {$flags(-dist)} { + teaish-dist-add {*}$args + } + if {$flags(-dir)} { + set xargs {} + foreach arg $args { + if {"" ne $arg} { + lappend xargs [file join $::teaish__Config(extension-dir) $arg] + } + } + set args $xargs + } + lappend ::teaish__Config(extension-src) {*}$args +} + +# +# @teaish-dist-add files-or-dirs... +# +# Adds the given files to the list of files to include with the "make +# dist" rules. +# +# This is a no-op when the current build is not in the extension's +# directory, as dist support is disabled in out-of-tree builds. +# +# It is not legal to call this until [teaish-get -dir] has been +# reliably set (via teaish__find_extension). +# +proc teaish-dist-add {args} { + if {$::teaish__Config(blddir-is-extdir)} { + # ^^^ reminder: we ignore $::teaish__Config(dist-enabled) here + # because the client might want to implement their own dist + # rules. + #proj-warn "**** args=$args" + lappend ::teaish__Config(dist-files) {*}$args + } +} + +# teaish-install-add files... +# Equivalent to [proj-define-apend TEAISH_INSTALL_FILES ...]. +#proc teaish-install-add {args} { +# proj-define-amend TEAISH_INSTALL_FILES {*}$args +#} + +# +# @teash-make-add args... +# +# Appends makefile code to the TEAISH_MAKEFILE_CODE define. Each +# arg may be any of: +# +# -tab: emit a literal tab +# -nl: emit a literal newline +# -nltab: short for -nl -tab +# -bnl: emit a backslash-escaped end-of-line +# -bnltab: short for -eol -tab +# +# Anything else is appended verbatim. This function adds no additional +# spacing between each argument nor between subsequent invocations. +# Generally speaking, a series of calls to this function need to +# be sure to end the series with a newline. +proc teaish-make-add {args} { + set out [get-define TEAISH_MAKEFILE_CODE ""] + foreach a $args { + switch -exact -- $a { + -bnl { set a " \\\n" } + -bnltab { set a " \\\n\t" } + -tab { set a "\t" } + -nl { set a "\n" } + -nltab { set a "\n\t" } + } + append out $a + } + define TEAISH_MAKEFILE_CODE $out +} + +# Internal helper to generate a clean/distclean rule name +proc teaish__cleanup_rule {{tgt clean}} { + set x [incr ::teaish__Config(teaish__cleanup_rule-counter-${tgt})] + return ${tgt}-_${x}_ +} + +# @teaish-make-obj ?flags? ?...args? +# +# Uses teaish-make-add to inject makefile rules for $objfile from +# $srcfile, which is assumed to be C code which uses libtcl. Unless +# -recipe is used (see below) it invokes the compiler using the +# makefile-defined $(CC.tcl) which, in the default Makefile.in +# template, includes any flags needed for building against the +# configured Tcl. +# +# This always terminates the resulting code with a newline. +# +# Any arguments after the 2nd may be flags described below or, if no +# -recipe is provided, flags for the compiler call. +# +# -obj obj-filename.o +# +# -src src-filename.c +# +# -recipe {...} +# Uses the trimmed value of {...} as the recipe, prefixing it with +# a single hard-tab character. +# +# -deps {...} +# List of extra files to list as dependencies of $o. +# +# -clean +# Generate cleanup rules as well. +proc teaish-make-obj {args} { + proj-parse-simple-flags args flags { + -clean 0 {expr 1} + -recipe => {} + -deps => {} + -obj => {} + -src => {} + } + #parray flags + if {"" eq $flags(-obj)} { + set args [lassign $args flags(-obj)] + if {"" eq $flags(-obj)} { + proj-error "Missing -obj flag." + } + } + foreach f {-deps -src} { + set flags($f) [string trim [string map {\n " "} $flags($f)]] + } + foreach f {-deps -src} { + set flags($f) [string trim $flags($f)] + } + #parray flags + #puts "-- args=$args" + teaish-make-add \ + "# [proj-scope 1] -> [proj-scope] $flags(-obj) $flags(-src)" -nl \ + "$flags(-obj): $flags(-src) $::teaish__Config(teaish.tcl)" + if {[info exists flags(-deps)]} { + teaish-make-add " " [join $flags(-deps)] + } + teaish-make-add -nltab + if {[info exists flags(-recipe)]} { + teaish-make-add [string trim $flags(-recipe)] -nl + } else { + teaish-make-add [join [list \$(CC.tcl) -c $flags(-src) {*}$args]] -nl + } + if {$flags(-clean)} { + set rule [teaish__cleanup_rule] + teaish-make-add \ + "clean: $rule\n$rule:\n\trm -f \"$flags(-obj)\"\n" + } +} + +# +# @teaish-make-clean ?-r? ?-dist? ...files|{...files} +# +# Adds makefile rules for cleaning up the given files via the "make +# clean" or (if -dist is used) "make distclean" makefile rules. The -r +# flag uses "rm -fr" instead of "rm -f", so be careful with that. +# +# The file names are taken literally as arguments to "rm", so they may +# be shell wildcards to be resolved at cleanup-time. To clean up whole +# directories, pass the -r flag. Each name gets quoted in +# double-quotes, so spaces in names should not be a problem (but +# double-quotes in names will be). +# +proc teaish-make-clean {args} { + if {1 == [llength $args]} { + set args [list {*}[lindex $args 0]] + } + + set tgt clean + set rmflags "-f" + proj-parse-simple-flags args flags { + -dist 0 { + set tgt distclean + } + -r 0 { + set rmflags "-fr" + } + } + set rule [teaish__cleanup_rule $tgt] + teaish-make-add "# [proj-scope 1] -> [proj-scope]: [join $args]\n" + teaish-make-add "${rule}:\n\trm ${rmflags}" + foreach a $args { + teaish-make-add " \"$a\"" + } + teaish-make-add "\n${tgt}: ${rule}\n" +} + +# +# @teaish-make-config-header filename +# +# Invokes autosetup's [make-config-header] and passes it $filename and +# a relatively generic list of options for controlling which defined +# symbols get exported. Clients which need more control over the +# exports can copy/paste/customize this. +# +# The exported file is then passed to [proj-touch] because, in +# practice, that's sometimes necessary to avoid build dependency +# issues. +# +proc teaish-make-config-header {filename} { + make-config-header $filename \ + -none {HAVE_CFLAG_* LDFLAGS_* SH_* TEAISH__* TEAISH_*_CODE} \ + -auto {SIZEOF_* HAVE_* TEAISH_* TCL_*} \ + -none * + proj-touch $filename; # help avoid frequent unnecessary auto-reconfig +} + +# +# @teaish-feature-cache-set $key value +# +# Sets a feature-check cache entry with the given key. +# See proj-cache-set for the key's semantics. $key should +# normally be 0. +# +proc teaish-feature-cache-set {key val} { + proj-cache-set -key $key -level 1 $val +} + +# +# @teaish-feature-cache-check key tgtVarName +# +# Checks for a feature-check cache entry with the given key. +# See proj-cache-set for the key's semantics. +# +# $key should also almost always be 0 but, due to a tclsh +# incompatibility in 1 OS, it cannot have a default value unless it's +# the second argument (but it should be the first one). +# +# If the feature-check cache has a matching entry then this function +# assigns its value to tgtVar and returns 1, else it assigns tgtVar to +# "" and returns 0. +# +# See proj-cache-check for $key's semantics. +# +proc teaish-feature-cache-check {key tgtVar} { + upvar $tgtVar tgt + proj-cache-check -key $key -level 1 tgt +} + +# +# @teaish-check-cached@ ?flags? msg script... +# +# A proxy for feature-test impls which handles caching of a feature +# flag check on per-function basis, using the calling scope's name as +# the cache key. +# +# It emits [msg-checking $msg]. If $msg is empty then it defaults to +# the name of the caller's scope. The -nomsg flag suppresses the +# message for non-cache-hit checks. At the end, it will [msg-result +# "ok"] [msg-result "no"] unless -nostatus is used, in which case the +# caller is responsible for emitting at least a newline when it's +# done. The -msg-0 and -msg-1 flags can be used to change the ok/no +# text. +# +# This function checks for a cache hit before running $script and +# caching the result. If no hit is found then $script is run in the +# calling scope and its result value is stored in the cache. This +# routine will intercept a 'return' from $script. +# +# $script may be a command and its arguments, as opposed to a single +# script block. +# +# Flags: +# +# -nostatus = do not emit "ok" or "no" at the end. This presumes +# that either $script will emit at least one newline before +# returning or the caller will account for it. Because of how this +# function is typically used, -nostatus is not honored when the +# response includes a cached result. +# +# -quiet = disable output from Autosetup's msg-checking and +# msg-result for the duration of the $script check. Note that when +# -quiet is in effect, Autosetup's user-notice can be used to queue +# up output to appear after the check is done. Also note that +# -quiet has no effect on _this_ function, only the $script part. +# +# -nomsg = do not emit $msg for initial check. Like -nostatus, this +# flag is not honored when the response includes a cached result +# because it would otherwise produce no output (which is confusing +# in this context). This is useful when a check runs several other +# verbose checks and they emit all the necessary info. +# +# -msg-0 and -msg-1 MSG = strings to show when the check has failed +# resp. passed. Defaults are "no" and "ok". The 0 and 1 refer to the +# result value from teaish-feature-cache-check. +# +# -key cachekey = set the cache context key. Only needs to be +# explicit when using this function multiple times from a single +# scope. See proj-cache-check and friends for details on the key +# name. Its default is the name of the scope which calls this +# function. +# +proc teaish-check-cached {args} { + proj-parse-simple-flags args flags { + -nostatus 0 {expr 1} + -quiet 0 {expr 1} + -key => 1 + -nomsg 0 {expr 1} + -msg-0 => no + -msg-1 => ok + } + set args [lassign $args msg] + set script [join $args] + if {"" eq $msg} { + set msg [proj-scope 1] + } + if {[teaish-feature-cache-check $flags(-key) check]} { + #if {0 == $flags(-nomsg)} { + msg-checking "${msg} ... (cached) " + #} + #if {!$flags(-nostatus)} { + msg-result $flags(-msg-[expr {0 != ${check}}]) + #} + return $check + } else { + if {0 == $flags(-nomsg)} { + msg-checking "${msg} ... " + } + if {$flags(-quiet)} { + incr ::autosetup(msg-quiet) + } + set code [catch {uplevel 1 $script} rc xopt] + if {$flags(-quiet)} { + incr ::autosetup(msg-quiet) -1 + } + #puts "***** cached-check got code=$code rc=$rc" + if {$code in {0 2}} { + teaish-feature-cache-set 1 $rc + if {!$flags(-nostatus)} { + msg-result $flags(-msg-[expr {0 != ${rc}}]) + } else { + #show-notices; # causes a phantom newline because we're in a + #msg-checking scope, so... + if {[info exists ::autosetup(notices)]} { + show-notices + } + } + } else { + #puts "**** code=$code rc=$rc xopt=$xopt" + teaish-feature-cache-set 1 0 + } + #puts "**** code=$code rc=$rc" + return {*}$xopt $rc + } +} + +# +# Internal helper for teaish__defs_format_: returns a JSON-ish quoted +# form of the given string-type values. +# +# If $asList is true then the return value is in {$value} form. If +# $asList is false it only performs the most basic of escaping and +# the input must not contain any control characters. +# +proc teaish__quote_str {asList value} { + if {$asList} { + return "{${value}}" + } + return \"[string map [list \\ \\\\ \" \\\"] $value]\" +} + +# +# Internal helper for teaish__defines_to_list. Expects to be passed +# a name and the variadic $args which are passed to +# teaish__defines_to_list.. If it finds a pattern match for the +# given $name in the various $args, it returns the type flag for that +# $name, e.g. "-str" or "-bare", else returns an empty string. +# +proc teaish__defs_type {name spec} { + foreach {type patterns} $spec { + foreach pattern $patterns { + if {[string match $pattern $name]} { + return $type + } + } + } + return "" +} + +# +# An internal impl detail. Requires a data type specifier, as used by +# Autosetup's [make-config-header], and a value. Returns the formatted +# value or the value $::teaish__Config(defs-skip) if the caller should +# skip emitting that value. +# +# In addition to -str, -auto, etc., as defined by make-config-header, +# it supports: +# +# -list {...} will cause non-integer values to be quoted in {...} +# instead of quotes. +# +# -autolist {...} works like -auto {...} except that it falls back to +# -list {...} type instead of -str {...} style for non-integers. +# +# -jsarray {...} emits the output in something which, for +# conservative inputs, will be a valid JSON array. It can only +# handle relatively simple values with no control characters in +# them. +# +set teaish__Config(defs-skip) "-teaish__defs_format sentinel" +proc teaish__defs_format {type value} { + switch -exact -- $type { + -bare { + # Just output the value unchanged + } + -none { + set value $::teaish__Config(defs-skip) + } + -str { + set value [teaish__quote_str 0 $value] + } + -auto { + # Automatically determine the type + if {![string is integer -strict $value]} { + set value [teaish__quote_str 0 $value] + } + } + -autolist { + if {![string is integer -strict $value]} { + set value [teaish__quote_str 1 $value] + } + } + -list { + set value [teaish__quote_str 1 $value] + } + -jsarray { + set ar {} + foreach v $value { + if {![string is integer -strict $v]} { + set v [teaish__quote_str 0 $v] + } + if {$::teaish__Config(defs-skip) ne $v} { + lappend ar $v + } + } + set value [concat \[ [join $ar {, }] \]] + } + "" { + # (Much later:) Why do we do this? + set value $::teaish__Config(defs-skip) + } + default { + proj-error \ + "Unknown [proj-scope] -type ($type) called from" \ + [proj-scope 1] + } + } + return $value +} + +# +# Returns Tcl code in the form of code which evaluates to a list of +# configure-time DEFINEs in the form {key val key2 val...}. It may +# misbehave for values which are not numeric or simple strings. Some +# defines are specifically filtered out of the result, either because +# their irrelevant to teaish or because they may be arbitrarily large +# (e.g. makefile content). +# +# The $args are explained in the docs for internal-use-only +# [teaish__defs_format]. The default mode is -autolist. +# +proc teaish__defines_to_list {args} { + set lines {} + lappend lines "\{" + set skipper $::teaish__Config(defs-skip) + set args [list \ + -none { + TEAISH__* + TEAISH_*_CODE + AM_* AS_* + } \ + {*}$args \ + -autolist *] + foreach d [lsort [dict keys [all-defines]]] { + set type [teaish__defs_type $d $args] + set value [teaish__defs_format $type [get-define $d]] + if {$skipper ne $value} { + lappend lines "$d $value" + } + } + lappend lines "\}" + tailcall join $lines "\n" +} + +# +# teaish__pragma ...flags +# +# Offers a way to tweak how teaish's core behaves in some cases, in +# particular those which require changing how the core looks for an +# extension and its files. +# +# Accepts the following flags. Those marked with [L] are safe to use +# during initial loading of tclish.tcl (recall that most teaish APIs +# cannot be used until [teaish-configure] is called). +# +# static-pkgIndex.tcl [L]: Tells teaish that ./pkgIndex.tcl is not +# a generated file, so it will not try to overwrite or delete +# it. Errors out if it does not find pkgIndex.tcl in the +# extension's dir. +# +# no-dist [L]: tells teaish to elide the 'make dist' recipe +# from the generated Makefile. +# +# no-dll [L]: tells teaish to elide the DLL-building recipe +# from the generated Makefile. +# +# no-vsatisfies-error [L]: tells teaish that failure to match the +# -vsatisfies value should simply "return" instead of "error". +# +# no-tester [L]: disables automatic generation of teaish.test.tcl +# even if a copy of _teaish.tester.tcl.in is found. +# +# no-full-dist [L]: changes the "make dist" rules to never include +# a copy of teaish itself. By default it will include itself only +# if the extension lives in the same directory as teaish. +# +# full-dist [L]: changes the "make dist" rules to always include +# a copy of teaish itself. +# +# Emits a warning message for unknown arguments. +# +proc teaish__pragma {args} { + foreach arg $args { + switch -exact -- $arg { + + static-pkgIndex.tcl { + if {$::teaish__Config(tm-policy)} { + proj-fatal -up "Cannot use pragma $arg together with -tm.tcl or -tm.tcl.in." + } + set tpi [file join $::teaish__Config(extension-dir) pkgIndex.tcl] + if {[file exists $tpi]} { + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL $tpi + set ::teaish__Config(pkgindex-policy) 0x20 + } else { + proj-error "pragma $arg: found no package-local pkgIndex.tcl\[.in]" + } + } + + no-dist { + set ::teaish__Config(dist-enabled) 0 + } + + no-install { + set ::teaish__Config(install-enabled) 0 + } + + full-dist { + set ::teaish__Config(dist-full-enabled) 1 + } + + no-full-dist { + set ::teaish__Config(dist-full-enabled) 0 + } + + no-dll { + set ::teaish__Config(dll-enabled) 0 + } + + no-vsatisfies-error { + set ::teaish__Config(vsatisfies-error) 0 + } + + no-tester { + define TEAISH_TESTER_TCL_IN "" + define TEAISH_TESTER_TCL "" + } + + default { + proj-error "Unknown flag: $arg" + } + } + } +} + +# +# @teaish-pkginfo-set ...flags +# +# The way to set up the initial package state. Used like: +# +# teaish-pkginfo-set -name foo -version 0.1.2 +# +# Or: +# +# teaish-pkginfo-set ?-vars|-subst? {-name foo -version 0.1.2} +# +# The latter may be easier to write for a multi-line invocation. +# +# For the second call form, passing the -vars flag tells it to perform +# a [subst] of (only) variables in the {...} part from the calling +# scope. The -subst flag will cause it to [subst] the {...} with +# command substitution as well (but no backslash substitution). When +# using -subst for string concatenation, e.g. with -libDir +# foo[get-version-number], be sure to wrap the value in braces: +# -libDir {foo[get-version-number]}. +# +# Each pkginfo flag corresponds to one piece of extension package +# info. Teaish provides usable default values for all of these flags, +# but at least the -name and -version should be set by clients. +# e.g. the default -name is the directory name the extension lives in, +# which may change (e.g. when building it from a "make dist" bundle). +# +# The flags: +# +# -name theName: The extension's name. It defaults to the name of the +# directory containing the extension. (In TEA this would be the +# PACKAGE_NAME, not to be confused with...) +# +# -name.pkg pkg-provide-name: The extension's name for purposes of +# Tcl_PkgProvide(), [package require], and friends. It defaults to +# the `-name`, and is normally the same, but some projects (like +# SQLite) have a different name here than they do in their +# historical TEA PACKAGE_NAME. +# +# -version version: The extension's package version. Defaults to +# 0.0.0. +# +# -libDir dirName: The base name of the directory into which this +# extension should be installed. It defaults to a concatenation of +# `-name.pkg` and `-version`. +# +# -loadPrefix prefix: For use as the second argument passed to +# Tcl's `load` command in the package-loading process. It defaults +# to title-cased `-name.pkg` because Tcl's `load` plugin system +# expects it in that form. +# +# -options {...}: If provided, it must be a list compatible with +# Autosetup's `options-add` function. These can also be set up via +# `teaish-options`. +# +# -vsatisfies {{...} ...}: Expects a list-of-lists of conditions +# for Tcl's `package vsatisfies` command: each list entry is a +# sub-list of `{PkgName Condition...}`. Teaish inserts those +# checks via its default pkgIndex.tcl.in and _teaish.tester.tcl.in +# templates to verify that the system's package dependencies meet +# these requirements. The default value is `{{Tcl 8.5-}}` (recall +# that it's a list-of-lists), as 8.5 is the minimum Tcl version +# teaish will run on, but some extensions may require newer +# versions or dependencies on other packages. As a special case, +# if `-vsatisfies` is given a single token, e.g. `8.6-`, then it +# is transformed into `{Tcl $thatToken}`, i.e. it checks the Tcl +# version which the package is being run with. If given multiple +# lists, each `package provides` check is run in the given +# order. Failure to meet a `vsatisfies` condition triggers an +# error. +# +# -url {...}: an optional URL for the extension. +# +# -pragmas {...} A list of infrequently-needed lower-level +# directives which can influence teaish, including: +# +# static-pkgIndex.tcl: tells teaish that the client manages their +# own pkgIndex.tcl, so that teaish won't try to overwrite it +# using a template. +# +# no-dist: tells teaish to elide the "make dist" recipe from the +# makefile so that the client can implement it. +# +# no-dll: tells teaish to elide the makefile rules which build +# the DLL, as well as any templated test script and pkgIndex.tcl +# references to them. The intent here is to (A) support +# client-defined build rules for the DLL and (B) eventually +# support script-only extensions. +# +# Unsupported flags or pragmas will trigger an error. +# +# Potential pothole: setting certain state, e.g. -version, after the +# initial call requires recalculating of some [define]s. Any such +# changes should be made as early as possible in teaish-configure so +# that any later use of those [define]s gets recorded properly (not +# with the old value). This is particularly relevant when it is not +# possible to determine the -version or -name until teaish-configure +# has been called, and it's updated dynamically from +# teaish-configure. Notably: +# +# - If -version or -name are updated, -libDir will almost certainly +# need to be explicitly set along with them. +# +# - If -name is updated, -loadPrefix probably needs to be as well. +# +proc teaish-pkginfo-set {args} { + set doVars 0 + set doCommands 0 + set xargs $args + set recalc {} + foreach arg $args { + switch -exact -- $arg { + -vars { + incr doVars + set xargs [lassign $xargs -] + } + -subst { + incr doVars + incr doCommands + set xargs [lassign $xargs -] + } + default { + break + } + } + } + set args $xargs + unset xargs + if {1 == [llength $args] && [llength [lindex $args 0]] > 1} { + # Transform a single {...} arg into the canonical call form + set a [list {*}[lindex $args 0]] + if {$doVars || $doCommands} { + set sflags -nobackslashes + if {!$doCommands} { + lappend sflags -nocommands + } + set a [uplevel 1 [list subst {*}$sflags $a]] + } + set args $a + } + set sentinel "" + set flagDefs [list] + foreach {f d} $::teaish__Config(pkginfo-f2d) { + lappend flagDefs $f => $sentinel + } + proj-parse-simple-flags args flags $flagDefs + if {[llength $args]} { + proj-error -up "Too many (or unknown) arguments to [proj-scope]: $args" + } + foreach {f d} $::teaish__Config(pkginfo-f2d) { + if {$sentinel eq [set v $flags($f)]} continue + switch -exact -- $f { + + -options { + proj-assert {"" eq $d} + options-add $v + } + + -pragmas { + teaish__pragma {*}$v + } + + -vsatisfies { + if {1 == [llength $v] && 1 == [llength [lindex $v 0]]} { + # Transform X to {Tcl $X} + set v [list [join [list Tcl $v]]] + } + define $d $v + } + + -pkgInit.tcl - + -pkgInit.tcl.in { + if {0x22 & $::teaish__Config(pkginit-policy)} { + proj-fatal "Cannot use -pkgInit.tcl(.in) more than once." + } + set x [file join $::teaish__Config(extension-dir) $v] + set tTail [file tail $v] + if {"-pkgInit.tcl.in" eq $f} { + # Generate pkginit file X from X.in + set pI 0x02 + set tIn $x + set tOut [file rootname $tTail] + set other -pkgInit.tcl + } else { + # Static pkginit file X + set pI 0x20 + set tIn "" + set tOut $x + set other -pkgInit.tcl.in + } + set ::teaish__Config(pkginit-policy) $pI + set ::teaish__PkgInfo($other) {} + define TEAISH_PKGINIT_TCL_IN $tIn + define TEAISH_PKGINIT_TCL $tOut + define TEAISH_PKGINIT_TCL_TAIL $tTail + teaish-dist-add $v + set v $x + } + + -src { + set d $::teaish__Config(extension-dir) + foreach f $v { + lappend ::teaish__Config(dist-files) $f + lappend ::teaish__Config(extension-src) $d/$f + lappend ::teaish__PkgInfo(-src) $f + # ^^^ so that default-value initialization in + # teaish-configure-core recognizes that it's been set. + } + } + + -tm.tcl - + -tm.tcl.in { + if {0x30 & $::teaish__Config(pkgindex-policy)} { + proj-fatal "Cannot use $f together with a pkgIndex.tcl." + } elseif {$::teaish__Config(tm-policy)} { + proj-fatal "Cannot use -tm.tcl(.in) more than once." + } + set x [file join $::teaish__Config(extension-dir) $v] + if {"-tm.tcl.in" eq $f} { + # Generate tm file X from X.in + set pT 0x02 + set pI 0x100 + set tIn $x + set tOut [file rootname [file tail $v]] + set other -tm.tcl + } else { + # Static tm file X + set pT 0x20 + set pI 0x200 + set tIn "" + set tOut $x + set other -tm.tcl.in + } + set ::teaish__Config(pkgindex-policy) $pI + set ::teaish__Config(tm-policy) $pT + set ::teaish__PkgInfo($other) {} + define TEAISH_TM_TCL_IN $tIn + define TEAISH_TM_TCL $tOut + define TEAISH_PKGINDEX_TCL "" + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL_TAIL "" + teaish-dist-add $v + teaish__pragma no-dll + set v $x + } + + default { + proj-assert {"" ne $d} + define $d $v + } + } + set ::teaish__PkgInfo($f) $v + if {$f in {-name -version -libDir -loadPrefix}} { + lappend recalc $f + } + } + if {"" ne $recalc} { + teaish__define_pkginfo_derived $recalc + } +} + +# +# @teaish-pkginfo-get ?arg? +# +# If passed no arguments, it returns the extension config info in the +# same form accepted by teaish-pkginfo-set. +# +# If passed one -flagname arg then it returns the value of that config +# option. +# +# Else it treats arg as the name of caller-scoped variable to +# which this function assigns an array containing the configuration +# state of this extension, in the same structure accepted by +# teaish-pkginfo-set. In this case it returns an empty string. +# +proc teaish-pkginfo-get {args} { + set cases {} + set argc [llength $args] + set rv {} + switch -exact $argc { + 0 { + # Return a list of (-flag value) pairs + lappend cases default {{ + if {[info exists ::teaish__PkgInfo($flag)]} { + lappend rv $flag $::teaish__PkgInfo($flag) + } else { + lappend rv $flag [get-define $defName] + } + }} + } + + 1 { + set arg $args + if {[string match -* $arg]} { + # Return the corresponding -flag's value + lappend cases $arg {{ + if {[info exists ::teaish__PkgInfo($flag)]} { + return $::teaish__PkgInfo($flag) + } else { + return [get-define $defName] + } + }} + } else { + # Populate target with an array of (-flag value). + upvar $arg tgt + array set tgt {} + lappend cases default {{ + if {[info exists ::teaish__PkgInfo($flag)]} { + set tgt($flag) $::teaish__PkgInfo($flag) + } else { + set tgt($flag) [get-define $defName] + } + }} + } + } + + default { + proj-error "invalid arg count from [proj-scope 1]" + } + } + + foreach {flag defName} $::teaish__Config(pkginfo-f2d) { + switch -exact -- $flag [join $cases] + } + if {0 == $argc} { return $rv } +} + +# (Re)set some defines based on pkginfo state. $flags is the list of +# pkginfo -flags which triggered this, or "*" for the initial call. +proc teaish__define_pkginfo_derived {flags} { + set all [expr {{*} in $flags}] + if {$all || "-version" in $flags || "-name" in $flags} { + set name $::teaish__PkgInfo(-name) ; # _not_ -name.pkg + if {[info exists ::teaish__PkgInfo(-version)]} { + set pkgver $::teaish__PkgInfo(-version) + set libname "lib" + if {[string match *-cygwin [get-define host]]} { + set libname cyg + } + define TEAISH_DLL8_BASENAME $libname$name$pkgver + define TEAISH_DLL9_BASENAME ${libname}tcl9$name$pkgver + set ext [get-define TARGET_DLLEXT] + define TEAISH_DLL8 [get-define TEAISH_DLL8_BASENAME]$ext + define TEAISH_DLL9 [get-define TEAISH_DLL9_BASENAME]$ext + } + } + if {$all || "-libDir" in $flags} { + if {[info exists ::teaish__PkgInfo(-libDir)]} { + define TCLLIBDIR \ + [file dirname [get-define TCLLIBDIR]]/$::teaish__PkgInfo(-libDir) + } + } +} + +# +# @teaish-checks-queue -pre|-post args... +# +# Queues one or more arbitrary "feature test" functions to be run when +# teaish-checks-run is called. $flag must be one of -pre or -post to +# specify whether the tests should be run before or after +# teaish-configure is run. Each additional arg is the name of a +# feature-test proc. +# +proc teaish-checks-queue {flag args} { + if {$flag ni {-pre -post}} { + proj-error "illegal flag: $flag" + } + lappend ::teaish__Config(queued-checks${flag}) {*}$args +} + +# +# @teaish-checks-run -pre|-post +# +# Runs all feature checks queued using teaish-checks-queue +# then cleares the queue. +# +proc teaish-checks-run {flag} { + if {$flag ni {-pre -post}} { + proj-error "illegal flag: $flag" + } + #puts "*** running $flag: $::teaish__Config(queued-checks${flag})" + set foo 0 + foreach f $::teaish__Config(queued-checks${flag}) { + if {![teaish-feature-cache-check $f foo]} { + set v [$f] + teaish-feature-cache-set $f $v + } + } + set ::teaish__Config(queued-checks${flag}) {} +} + +# +# A general-purpose getter for various teaish state. Requires one +# flag, which determines its result value. Flags marked with [L] below +# are safe for using at load-time, before teaish-pkginfo-set is called +# +# -dir [L]: returns the extension's directory, which may differ from +# the teaish core dir or the build dir. +# +# -teaish-home [L]: the "home" dir of teaish itself, which may +# differ from the extension dir or build dir. +# +# -build-dir [L]: the build directory (typically the current working +# -dir). +# +# Any of the teaish-pkginfo-get/get flags: returns the same as +# teaish-pkginfo-get. Not safe for use until teaish-pkginfo-set has +# been called. +# +# Triggers an error if passed an unknown flag. +# +proc teaish-get {flag} { + #-teaish.tcl {return $::teaish__Config(teaish.tcl)} + switch -exact -- $flag { + -dir { + return $::teaish__Config(extension-dir) + } + -teaish-home { + return $::autosetup(srcdir) + } + -build-dir { + return $::autosetup(builddir) + } + default { + if {[info exists ::teaish__PkgInfo($flag)]} { + return $::teaish__PkgInfo($flag) + } + } + } + proj-error "Unhandled flag: $flag" +} + +# +# Handles --teaish-create-extension=TARGET-DIR +# +proc teaish__create_extension {dir} { + set force [opt-bool teaish-force] + if {"" eq $dir} { + proj-error "--teaish-create-extension=X requires a directory name." + } + file mkdir $dir/generic + set cwd [pwd] + #set dir [file-normalize [file join $cwd $dir]] + teaish__verbose 1 msg-result "Created dir $dir" + cd $dir + if {!$force} { + # Ensure that we don't blindly overwrite anything + foreach f { + generic/teaish.c + teaish.tcl + teaish.make.in + teaish.test.tcl + } { + if {[file exists $f]} { + error "Cowardly refusing to overwrite $dir/$f. Use --teaish-force to overwrite." + } + } + } + + set name [file tail $dir] + set pkgName $name + set version 0.0.1 + set loadPrefix [string totitle $pkgName] + set content {teaish-pkginfo-set } + #puts "0 content=$content" + if {[opt-str teaish-extension-pkginfo epi]} { + set epi [string trim $epi] + if {[string match "*\n*" $epi]} { + set epi "{$epi}" + } elseif {![string match "{*}" $epi]} { + append content "\{" $epi "\}" + } else { + append content $epi + } + #puts "2 content=$content\nepi=$epi" + } else { + append content [subst -nocommands -nobackslashes {{ + -name ${name} + -name.pkg ${pkgName} + -name.dist ${pkgName} + -version ${version} + -loadPrefix $loadPrefix + -libDir ${name}${version} + -vsatisfies {{Tcl 8.5-}} + -url {} + -options {} + -pragmas {full-dist} + }}] + #puts "3 content=$content" + } + #puts "1 content=$content" + append content "\n" { +#proc teaish-options {} { + # Return a list and/or use \[options-add\] to add new + # configure flags. This is called before teaish's + # bootstrapping is finished, so only teaish-* + # APIs which are explicitly noted as being safe + # early on may be used here. Any autosetup-related + # APIs may be used here. + # + # Return an empty string if there are no options to + # add or if they are added using \[options-add\]. + # + # If there are no options to add, this proc need + # not be defined. +#} + +# Called by teaish once bootstrapping is complete. +# This function is responsible for the client-specific +# parts of the configuration process. +proc teaish-configure {} { + teaish-src-add -dir -dist generic/teaish.c + teaish-define-to-cflag -quote TEAISH_PKGNAME TEAISH_VERSION + + # TODO: your code goes here.. +} +}; # $content + proj-file-write teaish.tcl $content + teaish__verbose 1 msg-result "Created teaish.tcl" + + set content "# Teaish test script. +# When this tcl script is invoked via 'make test' it will have loaded +# the package, run any teaish.pkginit.tcl code, and loaded +# autosetup/teaish/tester.tcl. +" + proj-file-write teaish.test.tcl $content + teaish__verbose 1 msg-result "Created teaish.test.tcl" + + set content [subst -nocommands -nobackslashes { +#include +static int +${loadPrefix}_Cmd(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]){ + Tcl_SetObjResult(interp, Tcl_NewStringObj("this is the ${name} extension", -1)); + return TCL_OK; +} + +extern int DLLEXPORT ${loadPrefix}_Init(Tcl_Interp *interp){ + if (Tcl_InitStubs(interp, TCL_VERSION, 0) == NULL) { + return TCL_ERROR; + } + if (Tcl_PkgProvide(interp, TEAISH_PKGNAME, TEAISH_VERSION) == TCL_ERROR) { + return TCL_ERROR; + } + Tcl_CreateObjCommand(interp, TEAISH_PKGNAME, ${loadPrefix}_Cmd, NULL, NULL); + return TCL_OK; +} +}] + proj-file-write generic/teaish.c $content + teaish__verbose 1 msg-result "Created generic/teaish.c" + + set content "# teaish makefile for the ${name} extension +# tx.src = \$(tx.dir)/generic/teaish.c +# tx.LDFLAGS = +# tx.CFLAGS = +" + proj-file-write teaish.make.in $content + teaish__verbose 1 msg-result "Created teaish.make.in" + + msg-result "Created new extension \[$dir\]." + + cd $cwd + set ::teaish__Config(install-ext-dir) $dir +} + +# +# Internal helper for teaish__install +# +proc teaish__install_file {f destDir force} { + set dest $destDir/[file tail $f] + if {[file isdirectory $f]} { + file mkdir $dest + } elseif {!$force && [file exists $dest]} { + array set st1 [file stat $f] + array set st2 [file stat $dest] + if {($st1(mtime) == $st2(mtime)) + && ($st1(size) == $st2(size))} { + if {[file tail $f] in { + pkgIndex.tcl.in + _teaish.tester.tcl.in + }} { + # Assume they're the same. In the scope of the "make dist" + # rules, this happens legitimately when an extension with a + # copy of teaish installed in the same dir assumes that the + # pkgIndex.tcl.in and _teaish.tester.tcl.in belong to the + # extension, whereas teaish believes they belong to teaish. + # So we end up with dupes of those. + return + } + } + proj-error -up "Cowardly refusing to overwrite \[$dest\]." \ + "Use --teaish-force to enable overwriting." + } else { + # file copy -force $f $destDir; # loses +x bit + # + # JimTcl doesn't have [file attribute], so we can't use that here + # (in the context of an autosetup configure script). + exec cp -p $f $dest + } +} + +# +# Installs a copy of teaish, with autosetup, to $dDest, which defaults +# to the --teaish-install=X or --teash-create-extension=X dir. Won't +# overwrite files unless --teaish-force is used. +# +proc teaish__install {{dDest ""}} { + if {$dDest in {auto ""}} { + set dDest [opt-val teaish-install] + if {$dDest in {auto ""}} { + if {[info exists ::teaish__Config(install-ext-dir)]} { + set dDest $::teaish__Config(install-ext-dir) + } + } + } + set force [opt-bool teaish-force] + if {$dDest in {auto ""}} { + proj-error "Cannot determine installation directory." + } elseif {!$force && [file exists $dDest/auto.def]} { + proj-error \ + "Target dir looks like it already contains teaish and/or autosetup: $dDest" \ + "\nUse --teaish-force to overwrite it." + } + + set dSrc $::autosetup(srcdir) + set dAS $::autosetup(libdir) + set dAST $::teaish__Config(core-dir) + set dASTF $dAST/feature + teaish__verbose 1 msg-result "Installing teaish to \[$dDest\]..." + if {$::teaish__Config(verbose)>1} { + msg-result "dSrc = $dSrc" + msg-result "dAS = $dAS" + msg-result "dAST = $dAST" + msg-result "dASTF = $dASTF" + msg-result "dDest = $dDest" + } + + # Dest subdirs... + set ddAS $dDest/autosetup + set ddAST $ddAS/teaish + set ddASTF $ddAST/feature + foreach {srcDir destDir} [list \ + $dAS $ddAS \ + $dAST $ddAST \ + $dASTF $ddASTF \ + ] { + teaish__verbose 1 msg-result "Copying files to $destDir..." + file mkdir $destDir + foreach f [glob -nocomplain -directory $srcDir *] { + if {[string match {*~} $f] || [string match "#*#" [file tail $f]]} { + # Editor-generated backups and emacs lock files + continue + } + teaish__verbose 2 msg-result "\t$f" + teaish__install_file $f $destDir $force + } + } + teaish__verbose 1 msg-result "Copying files to $dDest..." + foreach f { + auto.def configure Makefile.in pkgIndex.tcl.in + _teaish.tester.tcl.in + } { + teaish__verbose 2 msg-result "\t$f" + teaish__install_file $dSrc/$f $dDest $force + } + set ::teaish__Config(install-self-dir) $dDest + msg-result "Teaish $::teaish__Config(version) installed in \[$dDest\]." +} diff --git a/autosetup/teaish/feature.tcl b/autosetup/teaish/feature.tcl new file mode 100644 index 0000000000..6c927d1a77 --- /dev/null +++ b/autosetup/teaish/feature.tcl @@ -0,0 +1,214 @@ +######################################################################## +# 2025 April 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# +######################################################################## +# ----- @module feature-tests.tcl ----- +# @section TEA-ish collection of feature tests. +# +# Functions in this file with a prefix of teaish__ are +# private/internal APIs. Those with a prefix of teaish- are +# public APIs. + + +# @teaish-check-libz +# +# Checks for zlib.h and the function deflate in libz. If found, +# prepends -lz to the extension's ldflags and returns 1, else returns +# 0. It also defines LDFLAGS_LIBZ to the libs flag. +# +proc teaish-check-libz {} { + teaish-check-cached "Checking for libz" { + set rc 0 + if {[msg-quiet cc-check-includes zlib.h] && [msg-quiet proj-check-function-in-lib deflate z]} { + teaish-ldflags-prepend [define LDFLAGS_LIBZ [get-define lib_deflate]] + undefine lib_deflate + incr rc + } + expr $rc + } +} + +# @teaish-check-librt ?funclist? +# +# Checks whether -lrt is needed for any of the given functions. If +# so, appends -lrt via [teaish-ldflags-prepend] and returns 1, else +# returns 0. It also defines LDFLAGS_LIBRT to the libs flag or an +# empty string. +# +# Some systems (ex: SunOS) require -lrt in order to use nanosleep. +# +proc teaish-check-librt {{funclist {fdatasync nanosleep}}} { + teaish-check-cached -nostatus "Checking whether ($funclist) need librt" { + define LDFLAGS_LIBRT "" + foreach func $funclist { + if {[msg-quiet proj-check-function-in-lib $func rt]} { + set ldrt [get-define lib_${func}] + undefine lib_${func} + if {"" ne $ldrt} { + teaish-ldflags-prepend -r [define LDFLAGS_LIBRT $ldrt] + msg-result $ldrt + return 1 + } else { + msg-result "no lib needed" + return 1 + } + } + } + msg-result "not found" + return 0 + } +} + +# @teaish-check-stdint +# +# A thin proxy for [cc-with] which checks for and the +# various fixed-size int types it declares. It defines HAVE_STDINT_T +# to 0 or 1 and (if it's 1) defines HAVE_XYZ_T for each XYZ int type +# to 0 or 1, depending on whether its available. +proc teaish-check-stdint {} { + teaish-check-cached "Checking for stdint.h" { + msg-quiet cc-with {-includes stdint.h} \ + {cc-check-types int8_t int16_t int32_t int64_t intptr_t \ + uint8_t uint16_t uint32_t uint64_t uintptr_t} + } +} + +# @teaish-is-mingw +# +# Returns 1 if building for mingw, else 0. +proc teaish-is-mingw {} { + return [expr { + [string match *mingw* [get-define host]] && + ![file exists /dev/null] + }] +} + +# @teaish-check-libdl +# +# Checks for whether dlopen() can be found and whether it requires +# -ldl for linking. If found, returns 1, defines LDFLAGS_DLOPEN to the +# linker flags (if any), and passes those flags to +# teaish-ldflags-prepend. It unconditionally defines HAVE_DLOPEN to 0 +# or 1 (the its return result value). +proc teaish-check-dlopen {} { + teaish-check-cached -nostatus "Checking for dlopen()" { + set rc 0 + set lfl "" + if {[cc-with {-includes dlfcn.h} { + cctest -link 1 -declare "extern char* dlerror(void);" -code "dlerror();"}]} { + msg-result "-ldl not needed" + incr rc + } elseif {[cc-check-includes dlfcn.h]} { + incr rc + if {[cc-check-function-in-lib dlopen dl]} { + set lfl [get-define lib_dlopen] + undefine lib_dlopen + msg-result " dlopen() needs $lfl" + } else { + msg-result " - dlopen() not found in libdl. Assuming dlopen() is built-in." + } + } else { + msg-result "not found" + } + teaish-ldflags-prepend [define LDFLAGS_DLOPEN $lfl] + define HAVE_DLOPEN $rc + } +} + +# +# @teaish-check-libmath +# +# Handles the --enable-math flag. Returns 1 if found, else 0. +# If found, it prepends -lm (if needed) to the linker flags. +proc teaish-check-libmath {} { + teaish-check-cached "Checking for libc math library" { + set lfl "" + set rc 0 + if {[msg-quiet proj-check-function-in-lib ceil m]} { + incr rc + set lfl [get-define lib_ceil] + undefine lib_ceil + teaish-ldflags-prepend $lfl + msg-checking "$lfl " + } + define LDFLAGS_LIBMATH $lfl + expr $rc + } +} + +# @teaish-import-features ?-flags? feature-names... +# +# For each $name in feature-names... it invokes: +# +# use teaish/feature/$name +# +# to load TEAISH_AUTOSETUP_DIR/feature/$name.tcl +# +# By default, if a proc named teaish-check-${name}-options is defined +# after sourcing a file, it is called and its result is passed to +# proj-append-options. This can be suppressed with the -no-options +# flag. +# +# Flags: +# +# -no-options: disables the automatic running of +# teaish-check-NAME-options, +# +# -run: if the function teaish-check-NAME exists after importing +# then it is called. This flag must not be used when calling this +# function from teaish-options. This trumps both -pre and -post. +# +# -pre: if the function teaish-check-NAME exists after importing +# then it is passed to [teaish-checks-queue -pre]. +# +# -post: works like -pre but instead uses[teaish-checks-queue -post]. +proc teaish-import-features {args} { + set pk "" + set doOpt 1 + proj-parse-simple-flags args flags { + -no-options 0 {set doOpt 0} + -run 0 {expr 1} + -pre 0 {set pk -pre} + -post 0 {set pk -post} + } + # + # TODO: never import the same module more than once. The "use" + # command is smart enough to not do that but we would need to + # remember whether or not any teaish-check-${arg}* procs have been + # called before, and skip them. + # + if {$flags(-run) && "" ne $pk} { + proj-error "Cannot use both -run and $pk" \ + " (called from [proj-scope 1])" + } + + foreach arg $args { + uplevel "use teaish/feature/$arg" + if {$doOpt} { + set n "teaish-check-${arg}-options" + if {[llength [info proc $n]] > 0} { + if {"" ne [set x [$n]]} { + options-add $x + } + } + } + if {$flags(-run)} { + set n "teaish-check-${arg}" + if {[llength [info proc $n]] > 0} { + uplevel 1 $n + } + } elseif {"" ne $pk} { + set n "teaish-check-${arg}" + if {[llength [info proc $n]] > 0} { + teaish-checks-queue {*}$pk $n + } + } + } +} diff --git a/autosetup/teaish/tester.tcl b/autosetup/teaish/tester.tcl new file mode 100644 index 0000000000..a25b366e8d --- /dev/null +++ b/autosetup/teaish/tester.tcl @@ -0,0 +1,293 @@ +######################################################################## +# 2025 April 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# +######################################################################## +# +# Helper routines for running tests on teaish extensions +# +######################################################################## +# ----- @module teaish/tester.tcl ----- +# +# @section TEA-ish Testing APIs. +# +# Though these are part of the autosup dir hierarchy, they are not +# intended to be run from autosetup code. Rather, they're for use +# with/via teaish.tester.tcl and target canonical Tcl only, not JimTcl +# (which the autosetup pieces do target). + +# +# @test-current-scope ?lvl? +# +# Returns the name of the _calling_ proc from ($lvl + 1) levels up the +# call stack (where the caller's level will be 1 up from _this_ +# call). If $lvl would resolve to global scope "global scope" is +# returned and if it would be negative then a string indicating such +# is returned (as opposed to throwing an error). +# +proc test-current-scope {{lvl 0}} { + #uplevel [expr {$lvl + 1}] {lindex [info level 0] 0} + set ilvl [info level] + set offset [expr {$ilvl - $lvl - 1}] + if { $offset < 0} { + return "invalid scope ($offset)" + } elseif { $offset == 0} { + return "global scope" + } else { + return [lindex [info level $offset] 0] + } +} + +# @test-msg +# +# Emits all arugments to stdout. +# +proc test-msg {args} { + puts "$args" +} + +# @test-warn +# +# Emits all arugments to stderr. +# +proc test-warn {args} { + puts stderr "WARNING: $args" +} + +# +# @test-error msg +# +# Triggers a test-failed error with a string describing the calling +# scope and the provided message. +# +proc test-fail {args} { + #puts stderr "ERROR: \[[test-current-scope 1]]: $msg" + #exit 1 + error "FAIL: \[[test-current-scope 1]]: $args" +} + +array set ::test__Counters {} +array set ::test__Config { + verbose-assert 0 verbose-affirm 0 +} + +# Internal impl for affirm and assert. +# +# $args = ?-v? script {msg-on-fail ""} +proc test__affert {failMode args} { + if {$failMode} { + set what assert + } else { + set what affirm + } + set verbose $::test__Config(verbose-$what) + if {"-v" eq [lindex $args 0]} { + lassign $args - script msg + if {1 == [llength $args]} { + # If -v is the only arg, toggle default verbose mode + set ::test__Config(verbose-$what) [expr {!$::test__Config(verbose-$what)}] + return + } + incr verbose + } else { + lassign $args script msg + } + incr ::test__Counters($what) + if {![uplevel 1 expr [list $script]]} { + if {"" eq $msg} { + set msg $script + } + set txt [join [list $what # $::test__Counters($what) "failed:" $msg]] + if {$failMode} { + puts stderr $txt + exit 1 + } else { + error $txt + } + } elseif {$verbose} { + puts stderr [join [list $what # $::test__Counters($what) "passed:" $script]] + } +} + +# +# @affirm ?-v? script ?msg? +# +# Works like a conventional assert method does, but reports failures +# using [error] instead of [exit]. If -v is used, it reports passing +# assertions to stderr. $script is evaluated in the caller's scope as +# an argument to [expr]. +# +proc affirm {args} { + tailcall test__affert 0 {*}$args +} + +# +# @assert ?-v? script ?msg? +# +# Works like [affirm] but exits on error. +# +proc assert {args} { + tailcall test__affert 1 {*}$args +} + +# +# @assert-matches ?-e? pattern ?-e? rhs ?msg? +# +# Equivalent to assert {[string match $pattern $rhs]} except that +# if either of those are prefixed with an -e flag, they are eval'd +# and their results are used. +# +proc assert-matches {args} { + set evalLhs 0 + set evalRhs 0 + if {"-e" eq [lindex $args 0]} { + incr evalLhs + set args [lassign $args -] + } + set args [lassign $args pattern] + if {"-e" eq [lindex $args 0]} { + incr evalRhs + set args [lassign $args -] + } + set args [lassign $args rhs msg] + + if {$evalLhs} { + set pattern [uplevel 1 $pattern] + } + if {$evalRhs} { + set rhs [uplevel 1 $rhs] + } + #puts "***pattern=$pattern\n***rhs=$rhs" + tailcall test__affert 1 \ + [join [list \[ string match [list $pattern] [list $rhs] \]]] $msg + # why does this not work? [list \[ string match [list $pattern] [list $rhs] \]] $msg + # "\[string match [list $pattern] [list $rhs]\]" +} + +# +# @test-assert testId script ?msg? +# +# Works like [assert] but emits $testId to stdout first. +# +proc test-assert {testId script {msg ""}} { + puts "test $testId" + tailcall test__affert 1 $script $msg +} + +# +# @test-expect testId script result +# +# Runs $script in the calling scope and compares its result to +# $result, minus any leading or trailing whitespace. If they differ, +# it triggers an [assert]. +# +proc test-expect {testId script result} { + puts "test $testId" + set x [string trim [uplevel 1 $script]] + set result [string trim $result] + tailcall test__affert 0 [list "{$x}" eq "{$result}"] \ + "\nEXPECTED: <<$result>>\nGOT: <<$x>>" +} + +# +# @test-catch cmd ?...args? +# +# Runs [cmd ...args], repressing any exception except to possibly log +# the failure. Returns 1 if it caught anything, 0 if it didn't. +# +proc test-catch {cmd args} { + if {[catch { + uplevel 1 $cmd {*}$args + } rc xopts]} { + puts "[test-current-scope] ignoring failure of: $cmd [lindex $args 0]: $rc" + return 1 + } + return 0 +} + +# +# @test-catch-matching pattern (script|cmd args...) +# +# Works like test-catch, but it expects its argument(s) to to throw an +# error matching the given string (checked with [string match]). If +# they do not throw, or the error does not match $pattern, this +# function throws, else it returns 1. +# +# If there is no second argument, the $cmd is assumed to be a script, +# and will be eval'd in the caller's scope. +# +# TODO: add -glob and -regex flags to control matching flavor. +# +proc test-catch-matching {pattern cmd args} { + if {[catch { + #puts "**** catch-matching cmd=$cmd args=$args" + if {0 == [llength $args]} { + uplevel 1 $cmd {*}$args + } else { + $cmd {*}$args + } + } rc xopts]} { + if {[string match $pattern $rc]} { + return 1 + } else { + error "[test-current-scope] exception does not match {$pattern}: {$rc}" + } + } + error "[test-current-scope] expecting to see an error matching {$pattern}" +} + +if {![array exists ::teaish__BuildFlags]} { + array set ::teaish__BuildFlags {} +} + +# +# @teaish-build-flag3 flag tgtVar ?dflt? +# +# If the current build has the configure-time flag named $flag set +# then tgtVar is assigned its value and 1 is returned, else tgtVal is +# assigned $dflt and 0 is returned. +# +# Caveat #1: only valid when called in the context of teaish's default +# "make test" recipe, e.g. from teaish.test.tcl. It is not valid from +# a teaish.tcl configure script because (A) the state it relies on +# doesn't fully exist at that point and (B) that level of the API has +# more direct access to the build state. This function requires that +# an external script have populated its internal state, which is +# normally handled via teaish.tester.tcl.in. +# +# Caveat #2: defines in the style of HAVE_FEATURENAME with a value of +# 0 are, by long-standing configure script conventions, treated as +# _undefined_ here. +# +proc teaish-build-flag3 {flag tgtVar {dflt ""}} { + upvar $tgtVar tgt + if {[info exists ::teaish__BuildFlags($flag)]} { + set tgt $::teaish__BuildFlags($flag) + return 1; + } elseif {0==[array size ::teaish__BuildFlags]} { + test-warn \ + "\[[test-current-scope]] was called from " \ + "[test-current-scope 1] without the build flags imported." + } + set tgt $dflt + return 0 +} + +# +# @teaish-build-flag flag ?dflt? +# +# Convenience form of teaish-build-flag3 which returns the +# configure-time-defined value of $flag or "" if it's not defined (or +# if it's an empty string). +# +proc teaish-build-flag {flag {dflt ""}} { + set tgt "" + teaish-build-flag3 $flag tgt $dflt + return $tgt +} diff --git a/configure b/configure index 0f6c18285d..64b60f8b35 100755 --- a/configure +++ b/configure @@ -1,15703 +1,4 @@ -#! /bin/sh -# Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlcipher 3.42.0. -# -# -# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. -# -# -# This configure script is free software; the Free Software Foundation -# gives unlimited permission to copy, distribute and modify it. -## -------------------- ## -## M4sh Initialization. ## -## -------------------- ## - -# Be more Bourne compatible -DUALCASE=1; export DUALCASE # for MKS sh -if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : - emulate sh - NULLCMD=: - # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. - alias -g '${1+"$@"}'='"$@"' - setopt NO_GLOB_SUBST -else - case `(set -o) 2>/dev/null` in #( - *posix*) : - set -o posix ;; #( - *) : - ;; -esac -fi - - -as_nl=' -' -export as_nl -# Printing a long string crashes Solaris 7 /usr/bin/printf. -as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' -as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo -as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo -# Prefer a ksh shell builtin over an external printf program on Solaris, -# but without wasting forks for bash or zsh. -if test -z "$BASH_VERSION$ZSH_VERSION" \ - && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then - as_echo='print -r --' - as_echo_n='print -rn --' -elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then - as_echo='printf %s\n' - as_echo_n='printf %s' -else - if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then - as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' - as_echo_n='/usr/ucb/echo -n' - else - as_echo_body='eval expr "X$1" : "X\\(.*\\)"' - as_echo_n_body='eval - arg=$1; - case $arg in #( - *"$as_nl"*) - expr "X$arg" : "X\\(.*\\)$as_nl"; - arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; - esac; - expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" - ' - export as_echo_n_body - as_echo_n='sh -c $as_echo_n_body as_echo' - fi - export as_echo_body - as_echo='sh -c $as_echo_body as_echo' -fi - -# The user is always right. -if test "${PATH_SEPARATOR+set}" != set; then - PATH_SEPARATOR=: - (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { - (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || - PATH_SEPARATOR=';' - } -fi - - -# IFS -# We need space, tab and new line, in precisely that order. Quoting is -# there to prevent editors from complaining about space-tab. -# (If _AS_PATH_WALK were called with IFS unset, it would disable word -# splitting by setting IFS to empty value.) -IFS=" "" $as_nl" - -# Find who we are. Look in the path if we contain no directory separator. -as_myself= -case $0 in #(( - *[\\/]* ) as_myself=$0 ;; - *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break - done -IFS=$as_save_IFS - - ;; -esac -# We did not find ourselves, most probably we were run as `sh COMMAND' -# in which case we are not to be found in the path. -if test "x$as_myself" = x; then - as_myself=$0 -fi -if test ! -f "$as_myself"; then - $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 - exit 1 -fi - -# Unset variables that we do not need and which cause bugs (e.g. in -# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" -# suppresses any "Segmentation fault" message there. '((' could -# trigger a bug in pdksh 5.2.14. -for as_var in BASH_ENV ENV MAIL MAILPATH -do eval test x\${$as_var+set} = xset \ - && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : -done -PS1='$ ' -PS2='> ' -PS4='+ ' - -# NLS nuisances. -LC_ALL=C -export LC_ALL -LANGUAGE=C -export LANGUAGE - -# CDPATH. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH - -# Use a proper internal environment variable to ensure we don't fall - # into an infinite loop, continuously re-executing ourselves. - if test x"${_as_can_reexec}" != xno && test "x$CONFIG_SHELL" != x; then - _as_can_reexec=no; export _as_can_reexec; - # We cannot yet assume a decent shell, so we have to provide a -# neutralization value for shells without unset; and this also -# works around shells that cannot unset nonexistent variables. -# Preserve -v and -x to the replacement shell. -BASH_ENV=/dev/null -ENV=/dev/null -(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV -case $- in # (((( - *v*x* | *x*v* ) as_opts=-vx ;; - *v* ) as_opts=-v ;; - *x* ) as_opts=-x ;; - * ) as_opts= ;; -esac -exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} -# Admittedly, this is quite paranoid, since all the known shells bail -# out after a failed `exec'. -$as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 -as_fn_exit 255 - fi - # We don't want this to propagate to other subprocesses. - { _as_can_reexec=; unset _as_can_reexec;} -if test "x$CONFIG_SHELL" = x; then - as_bourne_compatible="if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then : - emulate sh - NULLCMD=: - # Pre-4.2 versions of Zsh do word splitting on \${1+\"\$@\"}, which - # is contrary to our usage. Disable this feature. - alias -g '\${1+\"\$@\"}'='\"\$@\"' - setopt NO_GLOB_SUBST -else - case \`(set -o) 2>/dev/null\` in #( - *posix*) : - set -o posix ;; #( - *) : - ;; -esac -fi -" - as_required="as_fn_return () { (exit \$1); } -as_fn_success () { as_fn_return 0; } -as_fn_failure () { as_fn_return 1; } -as_fn_ret_success () { return 0; } -as_fn_ret_failure () { return 1; } - -exitcode=0 -as_fn_success || { exitcode=1; echo as_fn_success failed.; } -as_fn_failure && { exitcode=1; echo as_fn_failure succeeded.; } -as_fn_ret_success || { exitcode=1; echo as_fn_ret_success failed.; } -as_fn_ret_failure && { exitcode=1; echo as_fn_ret_failure succeeded.; } -if ( set x; as_fn_ret_success y && test x = \"\$1\" ); then : - -else - exitcode=1; echo positional parameters were not saved. -fi -test x\$exitcode = x0 || exit 1 -test -x / || exit 1" - as_suggested=" as_lineno_1=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_1a=\$LINENO - as_lineno_2=";as_suggested=$as_suggested$LINENO;as_suggested=$as_suggested" as_lineno_2a=\$LINENO - eval 'test \"x\$as_lineno_1'\$as_run'\" != \"x\$as_lineno_2'\$as_run'\" && - test \"x\`expr \$as_lineno_1'\$as_run' + 1\`\" = \"x\$as_lineno_2'\$as_run'\"' || exit 1 - - test -n \"\${ZSH_VERSION+set}\${BASH_VERSION+set}\" || ( - ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' - ECHO=\$ECHO\$ECHO\$ECHO\$ECHO\$ECHO - ECHO=\$ECHO\$ECHO\$ECHO\$ECHO\$ECHO\$ECHO - PATH=/empty FPATH=/empty; export PATH FPATH - test \"X\`printf %s \$ECHO\`\" = \"X\$ECHO\" \\ - || test \"X\`print -r -- \$ECHO\`\" = \"X\$ECHO\" ) || exit 1 -test \$(( 1 + 1 )) = 2 || exit 1" - if (eval "$as_required") 2>/dev/null; then : - as_have_required=yes -else - as_have_required=no -fi - if test x$as_have_required = xyes && (eval "$as_suggested") 2>/dev/null; then : - -else - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -as_found=false -for as_dir in /bin$PATH_SEPARATOR/usr/bin$PATH_SEPARATOR$PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - as_found=: - case $as_dir in #( - /*) - for as_base in sh bash ksh sh5; do - # Try only shells that exist, to save several forks. - as_shell=$as_dir/$as_base - if { test -f "$as_shell" || test -f "$as_shell.exe"; } && - { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$as_shell"; } 2>/dev/null; then : - CONFIG_SHELL=$as_shell as_have_required=yes - if { $as_echo "$as_bourne_compatible""$as_suggested" | as_run=a "$as_shell"; } 2>/dev/null; then : - break 2 -fi -fi - done;; - esac - as_found=false -done -$as_found || { if { test -f "$SHELL" || test -f "$SHELL.exe"; } && - { $as_echo "$as_bourne_compatible""$as_required" | as_run=a "$SHELL"; } 2>/dev/null; then : - CONFIG_SHELL=$SHELL as_have_required=yes -fi; } -IFS=$as_save_IFS - - - if test "x$CONFIG_SHELL" != x; then : - export CONFIG_SHELL - # We cannot yet assume a decent shell, so we have to provide a -# neutralization value for shells without unset; and this also -# works around shells that cannot unset nonexistent variables. -# Preserve -v and -x to the replacement shell. -BASH_ENV=/dev/null -ENV=/dev/null -(unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV -case $- in # (((( - *v*x* | *x*v* ) as_opts=-vx ;; - *v* ) as_opts=-v ;; - *x* ) as_opts=-x ;; - * ) as_opts= ;; -esac -exec $CONFIG_SHELL $as_opts "$as_myself" ${1+"$@"} -# Admittedly, this is quite paranoid, since all the known shells bail -# out after a failed `exec'. -$as_echo "$0: could not re-execute with $CONFIG_SHELL" >&2 -exit 255 -fi - - if test x$as_have_required = xno; then : - $as_echo "$0: This script requires a shell more modern than all" - $as_echo "$0: the shells that I found on your system." - if test x${ZSH_VERSION+set} = xset ; then - $as_echo "$0: In particular, zsh $ZSH_VERSION has bugs and should" - $as_echo "$0: be upgraded to zsh 4.3.4 or later." - else - $as_echo "$0: Please tell bug-autoconf@gnu.org about your system, -$0: including any error possibly output before this -$0: message. Then install a modern shell, or manually run -$0: the script under such a shell if you do have one." - fi - exit 1 -fi -fi -fi -SHELL=${CONFIG_SHELL-/bin/sh} -export SHELL -# Unset more variables known to interfere with behavior of common tools. -CLICOLOR_FORCE= GREP_OPTIONS= -unset CLICOLOR_FORCE GREP_OPTIONS - -## --------------------- ## -## M4sh Shell Functions. ## -## --------------------- ## -# as_fn_unset VAR -# --------------- -# Portably unset VAR. -as_fn_unset () -{ - { eval $1=; unset $1;} -} -as_unset=as_fn_unset - -# as_fn_set_status STATUS -# ----------------------- -# Set $? to STATUS, without forking. -as_fn_set_status () -{ - return $1 -} # as_fn_set_status - -# as_fn_exit STATUS -# ----------------- -# Exit the shell with STATUS, even in a "trap 0" or "set -e" context. -as_fn_exit () -{ - set +e - as_fn_set_status $1 - exit $1 -} # as_fn_exit - -# as_fn_mkdir_p -# ------------- -# Create "$as_dir" as a directory, including parents if necessary. -as_fn_mkdir_p () -{ - - case $as_dir in #( - -*) as_dir=./$as_dir;; - esac - test -d "$as_dir" || eval $as_mkdir_p || { - as_dirs= - while :; do - case $as_dir in #( - *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( - *) as_qdir=$as_dir;; - esac - as_dirs="'$as_qdir' $as_dirs" - as_dir=`$as_dirname -- "$as_dir" || -$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$as_dir" : 'X\(//\)[^/]' \| \ - X"$as_dir" : 'X\(//\)$' \| \ - X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || -$as_echo X"$as_dir" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ - s//\1/ - q - } - /^X\(\/\/\)[^/].*/{ - s//\1/ - q - } - /^X\(\/\/\)$/{ - s//\1/ - q - } - /^X\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q'` - test -d "$as_dir" && break - done - test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" - - -} # as_fn_mkdir_p - -# as_fn_executable_p FILE -# ----------------------- -# Test if FILE is an executable regular file. -as_fn_executable_p () -{ - test -f "$1" && test -x "$1" -} # as_fn_executable_p -# as_fn_append VAR VALUE -# ---------------------- -# Append the text in VALUE to the end of the definition contained in VAR. Take -# advantage of any shell optimizations that allow amortized linear growth over -# repeated appends, instead of the typical quadratic growth present in naive -# implementations. -if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : - eval 'as_fn_append () - { - eval $1+=\$2 - }' -else - as_fn_append () - { - eval $1=\$$1\$2 - } -fi # as_fn_append - -# as_fn_arith ARG... -# ------------------ -# Perform arithmetic evaluation on the ARGs, and store the result in the -# global $as_val. Take advantage of shells that can avoid forks. The arguments -# must be portable across $(()) and expr. -if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : - eval 'as_fn_arith () - { - as_val=$(( $* )) - }' -else - as_fn_arith () - { - as_val=`expr "$@" || test $? -eq 1` - } -fi # as_fn_arith - - -# as_fn_error STATUS ERROR [LINENO LOG_FD] -# ---------------------------------------- -# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are -# provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with STATUS, using 1 if that was 0. -as_fn_error () -{ - as_status=$1; test $as_status -eq 0 && as_status=1 - if test "$4"; then - as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 - fi - $as_echo "$as_me: error: $2" >&2 - as_fn_exit $as_status -} # as_fn_error - -if expr a : '\(a\)' >/dev/null 2>&1 && - test "X`expr 00001 : '.*\(...\)'`" = X001; then - as_expr=expr -else - as_expr=false -fi - -if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then - as_basename=basename -else - as_basename=false -fi - -if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then - as_dirname=dirname -else - as_dirname=false -fi - -as_me=`$as_basename -- "$0" || -$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ - X"$0" : 'X\(//\)$' \| \ - X"$0" : 'X\(/\)' \| . 2>/dev/null || -$as_echo X/"$0" | - sed '/^.*\/\([^/][^/]*\)\/*$/{ - s//\1/ - q - } - /^X\/\(\/\/\)$/{ - s//\1/ - q - } - /^X\/\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q'` - -# Avoid depending upon Character Ranges. -as_cr_letters='abcdefghijklmnopqrstuvwxyz' -as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' -as_cr_Letters=$as_cr_letters$as_cr_LETTERS -as_cr_digits='0123456789' -as_cr_alnum=$as_cr_Letters$as_cr_digits - - - as_lineno_1=$LINENO as_lineno_1a=$LINENO - as_lineno_2=$LINENO as_lineno_2a=$LINENO - eval 'test "x$as_lineno_1'$as_run'" != "x$as_lineno_2'$as_run'" && - test "x`expr $as_lineno_1'$as_run' + 1`" = "x$as_lineno_2'$as_run'"' || { - # Blame Lee E. McMahon (1931-1989) for sed's syntax. :-) - sed -n ' - p - /[$]LINENO/= - ' <$as_myself | - sed ' - s/[$]LINENO.*/&-/ - t lineno - b - :lineno - N - :loop - s/[$]LINENO\([^'$as_cr_alnum'_].*\n\)\(.*\)/\2\1\2/ - t loop - s/-\n.*// - ' >$as_me.lineno && - chmod +x "$as_me.lineno" || - { $as_echo "$as_me: error: cannot create $as_me.lineno; rerun with a POSIX shell" >&2; as_fn_exit 1; } - - # If we had to re-execute with $CONFIG_SHELL, we're ensured to have - # already done that, so ensure we don't try to do so again and fall - # in an infinite loop. This has already happened in practice. - _as_can_reexec=no; export _as_can_reexec - # Don't try to exec as it changes $[0], causing all sort of problems - # (the dirname of $[0] is not the place where we might find the - # original and so on. Autoconf is especially sensitive to this). - . "./$as_me.lineno" - # Exit status is that of the last command. - exit -} - -ECHO_C= ECHO_N= ECHO_T= -case `echo -n x` in #((((( --n*) - case `echo 'xy\c'` in - *c*) ECHO_T=' ';; # ECHO_T is single tab character. - xy) ECHO_C='\c';; - *) echo `echo ksh88 bug on AIX 6.1` > /dev/null - ECHO_T=' ';; - esac;; -*) - ECHO_N='-n';; -esac - -rm -f conf$$ conf$$.exe conf$$.file -if test -d conf$$.dir; then - rm -f conf$$.dir/conf$$.file -else - rm -f conf$$.dir - mkdir conf$$.dir 2>/dev/null -fi -if (echo >conf$$.file) 2>/dev/null; then - if ln -s conf$$.file conf$$ 2>/dev/null; then - as_ln_s='ln -s' - # ... but there are two gotchas: - # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. - # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -pR'. - ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || - as_ln_s='cp -pR' - elif ln conf$$.file conf$$ 2>/dev/null; then - as_ln_s=ln - else - as_ln_s='cp -pR' - fi -else - as_ln_s='cp -pR' -fi -rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file -rmdir conf$$.dir 2>/dev/null - -if mkdir -p . 2>/dev/null; then - as_mkdir_p='mkdir -p "$as_dir"' -else - test -d ./-p && rmdir ./-p - as_mkdir_p=false -fi - -as_test_x='test -x' -as_executable_p=as_fn_executable_p - -# Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" - -# Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" - -SHELL=${CONFIG_SHELL-/bin/sh} - - -test -n "$DJDIR" || exec 7<&0 &1 - -# Name of the host. -# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, -# so uname gets run too. -ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` - -# -# Initializations. -# -ac_default_prefix=/usr/local -ac_clean_files= -ac_config_libobj_dir=. -LIBOBJS= -cross_compiling=no -subdirs= -MFLAGS= -MAKEFLAGS= - -# Identity of this package. -PACKAGE_NAME='sqlcipher' -PACKAGE_TARNAME='sqlcipher' -PACKAGE_VERSION='3.42.0' -PACKAGE_STRING='sqlcipher 3.42.0' -PACKAGE_BUGREPORT='' -PACKAGE_URL='' - -# Factoring default headers for most tests. -ac_includes_default="\ -#include -#ifdef HAVE_SYS_TYPES_H -# include -#endif -#ifdef HAVE_SYS_STAT_H -# include -#endif -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_STRING_H -# if !defined STDC_HEADERS && defined HAVE_MEMORY_H -# include -# endif -# include -#endif -#ifdef HAVE_STRINGS_H -# include -#endif -#ifdef HAVE_INTTYPES_H -# include -#endif -#ifdef HAVE_STDINT_H -# include -#endif -#ifdef HAVE_UNISTD_H -# include -#endif" - -ac_subst_vars='LTLIBOBJS -LIBOBJS -BUILD_CFLAGS -USE_GCOV -OPT_FEATURE_FLAGS -HAVE_ZLIB -AMALGAMATION_LINE_MACROS -amalgamation_line_macros -USE_AMALGAMATION -TARGET_DEBUG -TARGET_HAVE_EDITLINE -TARGET_HAVE_READLINE -TARGET_READLINE_INC -TARGET_READLINE_LIBS -HAVE_TCL -TCL_SHLIB_SUFFIX -TCL_STUB_LIB_SPEC -TCL_STUB_LIB_FLAG -TCL_STUB_LIB_FILE -TCL_LIB_SPEC -TCL_LIB_FLAG -TCL_LIB_FILE -TCL_INCLUDE_SPEC -TCL_SRC_DIR -TCL_BIN_DIR -TCL_VERSION -TARGET_EXEEXT -SQLITE_OS_WIN -SQLITE_OS_UNIX -BUILD_EXEEXT -TEMP_STORE -ALLOWRELEASE -XTHREADCONNECT -SQLITE_THREADSAFE -BUILD_CC -HAVE_WASI_SDK -RELEASE -VERSION -program_prefix -TCLLIBDIR -TCLSH_CMD -INSTALL_DATA -INSTALL_SCRIPT -INSTALL_PROGRAM -CPP -LT_SYS_LIBRARY_PATH -OTOOL64 -OTOOL -LIPO -NMEDIT -DSYMUTIL -MANIFEST_TOOL -AWK -RANLIB -STRIP -ac_ct_AR -AR -DLLTOOL -OBJDUMP -LN_S -NM -ac_ct_DUMPBIN -DUMPBIN -LD -FGREP -EGREP -GREP -SED -OBJEXT -EXEEXT -ac_ct_CC -CPPFLAGS -LDFLAGS -CFLAGS -CC -host_os -host_vendor -host_cpu -host -build_os -build_vendor -build_cpu -build -LIBTOOL -target_alias -host_alias -build_alias -LIBS -ECHO_T -ECHO_N -ECHO_C -DEFS -mandir -localedir -libdir -psdir -pdfdir -dvidir -htmldir -infodir -docdir -oldincludedir -includedir -runstatedir -localstatedir -sharedstatedir -sysconfdir -datadir -datarootdir -libexecdir -sbindir -bindir -program_transform_name -prefix -exec_prefix -PACKAGE_URL -PACKAGE_BUGREPORT -PACKAGE_STRING -PACKAGE_VERSION -PACKAGE_TARNAME -PACKAGE_NAME -PATH_SEPARATOR -SHELL' -ac_subst_files='' -ac_user_opts=' -enable_option_checking -enable_shared -enable_static -with_pic -enable_fast_install -with_aix_soname -with_gnu_ld -with_sysroot -enable_libtool_lock -enable_largefile -with_wasi_sdk -enable_threadsafe -with_crypto_lib -enable_cross_thread_connections -enable_releasemode -enable_tempstore -enable_tcl -with_tcl -enable_editline -enable_readline -with_readline_lib -with_readline_inc -enable_debug -enable_amalgamation -enable_load_extension -enable_math -enable_json -enable_all -enable_memsys5 -enable_memsys3 -enable_fts3 -enable_fts4 -enable_fts5 -enable_update_limit -enable_geopoly -enable_rtree -enable_session -enable_gcov -' - ac_precious_vars='build_alias -host_alias -target_alias -CC -CFLAGS -LDFLAGS -LIBS -CPPFLAGS -LT_SYS_LIBRARY_PATH -CPP -TCLLIBDIR -amalgamation_line_macros' - - -# Initialize some variables set by options. -ac_init_help= -ac_init_version=false -ac_unrecognized_opts= -ac_unrecognized_sep= -# The variables have the same names as the options, with -# dashes changed to underlines. -cache_file=/dev/null -exec_prefix=NONE -no_create= -no_recursion= -prefix=NONE -program_prefix=NONE -program_suffix=NONE -program_transform_name=s,x,x, -silent= -site= -srcdir= -verbose= -x_includes=NONE -x_libraries=NONE - -# Installation directory options. -# These are left unexpanded so users can "make install exec_prefix=/foo" -# and all the variables that are supposed to be based on exec_prefix -# by default will actually change. -# Use braces instead of parens because sh, perl, etc. also accept them. -# (The list follows the same order as the GNU Coding Standards.) -bindir='${exec_prefix}/bin' -sbindir='${exec_prefix}/sbin' -libexecdir='${exec_prefix}/libexec' -datarootdir='${prefix}/share' -datadir='${datarootdir}' -sysconfdir='${prefix}/etc' -sharedstatedir='${prefix}/com' -localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' -includedir='${prefix}/include' -oldincludedir='/usr/include' -docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' -infodir='${datarootdir}/info' -htmldir='${docdir}' -dvidir='${docdir}' -pdfdir='${docdir}' -psdir='${docdir}' -libdir='${exec_prefix}/lib' -localedir='${datarootdir}/locale' -mandir='${datarootdir}/man' - -ac_prev= -ac_dashdash= -for ac_option -do - # If the previous option needs an argument, assign it. - if test -n "$ac_prev"; then - eval $ac_prev=\$ac_option - ac_prev= - continue - fi - - case $ac_option in - *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; - *=) ac_optarg= ;; - *) ac_optarg=yes ;; - esac - - # Accept the important Cygnus configure options, so we can diagnose typos. - - case $ac_dashdash$ac_option in - --) - ac_dashdash=yes ;; - - -bindir | --bindir | --bindi | --bind | --bin | --bi) - ac_prev=bindir ;; - -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*) - bindir=$ac_optarg ;; - - -build | --build | --buil | --bui | --bu) - ac_prev=build_alias ;; - -build=* | --build=* | --buil=* | --bui=* | --bu=*) - build_alias=$ac_optarg ;; - - -cache-file | --cache-file | --cache-fil | --cache-fi \ - | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c) - ac_prev=cache_file ;; - -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \ - | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*) - cache_file=$ac_optarg ;; - - --config-cache | -C) - cache_file=config.cache ;; - - -datadir | --datadir | --datadi | --datad) - ac_prev=datadir ;; - -datadir=* | --datadir=* | --datadi=* | --datad=*) - datadir=$ac_optarg ;; - - -datarootdir | --datarootdir | --datarootdi | --datarootd | --dataroot \ - | --dataroo | --dataro | --datar) - ac_prev=datarootdir ;; - -datarootdir=* | --datarootdir=* | --datarootdi=* | --datarootd=* \ - | --dataroot=* | --dataroo=* | --dataro=* | --datar=*) - datarootdir=$ac_optarg ;; - - -disable-* | --disable-*) - ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid feature name: $ac_useropt" - ac_useropt_orig=$ac_useropt - ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` - case $ac_user_opts in - *" -"enable_$ac_useropt" -"*) ;; - *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--disable-$ac_useropt_orig" - ac_unrecognized_sep=', ';; - esac - eval enable_$ac_useropt=no ;; - - -docdir | --docdir | --docdi | --doc | --do) - ac_prev=docdir ;; - -docdir=* | --docdir=* | --docdi=* | --doc=* | --do=*) - docdir=$ac_optarg ;; - - -dvidir | --dvidir | --dvidi | --dvid | --dvi | --dv) - ac_prev=dvidir ;; - -dvidir=* | --dvidir=* | --dvidi=* | --dvid=* | --dvi=* | --dv=*) - dvidir=$ac_optarg ;; - - -enable-* | --enable-*) - ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid feature name: $ac_useropt" - ac_useropt_orig=$ac_useropt - ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` - case $ac_user_opts in - *" -"enable_$ac_useropt" -"*) ;; - *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--enable-$ac_useropt_orig" - ac_unrecognized_sep=', ';; - esac - eval enable_$ac_useropt=\$ac_optarg ;; - - -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \ - | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \ - | --exec | --exe | --ex) - ac_prev=exec_prefix ;; - -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \ - | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \ - | --exec=* | --exe=* | --ex=*) - exec_prefix=$ac_optarg ;; - - -gas | --gas | --ga | --g) - # Obsolete; use --with-gas. - with_gas=yes ;; - - -help | --help | --hel | --he | -h) - ac_init_help=long ;; - -help=r* | --help=r* | --hel=r* | --he=r* | -hr*) - ac_init_help=recursive ;; - -help=s* | --help=s* | --hel=s* | --he=s* | -hs*) - ac_init_help=short ;; - - -host | --host | --hos | --ho) - ac_prev=host_alias ;; - -host=* | --host=* | --hos=* | --ho=*) - host_alias=$ac_optarg ;; - - -htmldir | --htmldir | --htmldi | --htmld | --html | --htm | --ht) - ac_prev=htmldir ;; - -htmldir=* | --htmldir=* | --htmldi=* | --htmld=* | --html=* | --htm=* \ - | --ht=*) - htmldir=$ac_optarg ;; - - -includedir | --includedir | --includedi | --included | --include \ - | --includ | --inclu | --incl | --inc) - ac_prev=includedir ;; - -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \ - | --includ=* | --inclu=* | --incl=* | --inc=*) - includedir=$ac_optarg ;; - - -infodir | --infodir | --infodi | --infod | --info | --inf) - ac_prev=infodir ;; - -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*) - infodir=$ac_optarg ;; - - -libdir | --libdir | --libdi | --libd) - ac_prev=libdir ;; - -libdir=* | --libdir=* | --libdi=* | --libd=*) - libdir=$ac_optarg ;; - - -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \ - | --libexe | --libex | --libe) - ac_prev=libexecdir ;; - -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \ - | --libexe=* | --libex=* | --libe=*) - libexecdir=$ac_optarg ;; - - -localedir | --localedir | --localedi | --localed | --locale) - ac_prev=localedir ;; - -localedir=* | --localedir=* | --localedi=* | --localed=* | --locale=*) - localedir=$ac_optarg ;; - - -localstatedir | --localstatedir | --localstatedi | --localstated \ - | --localstate | --localstat | --localsta | --localst | --locals) - ac_prev=localstatedir ;; - -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \ - | --localstate=* | --localstat=* | --localsta=* | --localst=* | --locals=*) - localstatedir=$ac_optarg ;; - - -mandir | --mandir | --mandi | --mand | --man | --ma | --m) - ac_prev=mandir ;; - -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*) - mandir=$ac_optarg ;; - - -nfp | --nfp | --nf) - # Obsolete; use --without-fp. - with_fp=no ;; - - -no-create | --no-create | --no-creat | --no-crea | --no-cre \ - | --no-cr | --no-c | -n) - no_create=yes ;; - - -no-recursion | --no-recursion | --no-recursio | --no-recursi \ - | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) - no_recursion=yes ;; - - -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \ - | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \ - | --oldin | --oldi | --old | --ol | --o) - ac_prev=oldincludedir ;; - -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \ - | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \ - | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*) - oldincludedir=$ac_optarg ;; - - -prefix | --prefix | --prefi | --pref | --pre | --pr | --p) - ac_prev=prefix ;; - -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*) - prefix=$ac_optarg ;; - - -program-prefix | --program-prefix | --program-prefi | --program-pref \ - | --program-pre | --program-pr | --program-p) - ac_prev=program_prefix ;; - -program-prefix=* | --program-prefix=* | --program-prefi=* \ - | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*) - program_prefix=$ac_optarg ;; - - -program-suffix | --program-suffix | --program-suffi | --program-suff \ - | --program-suf | --program-su | --program-s) - ac_prev=program_suffix ;; - -program-suffix=* | --program-suffix=* | --program-suffi=* \ - | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*) - program_suffix=$ac_optarg ;; - - -program-transform-name | --program-transform-name \ - | --program-transform-nam | --program-transform-na \ - | --program-transform-n | --program-transform- \ - | --program-transform | --program-transfor \ - | --program-transfo | --program-transf \ - | --program-trans | --program-tran \ - | --progr-tra | --program-tr | --program-t) - ac_prev=program_transform_name ;; - -program-transform-name=* | --program-transform-name=* \ - | --program-transform-nam=* | --program-transform-na=* \ - | --program-transform-n=* | --program-transform-=* \ - | --program-transform=* | --program-transfor=* \ - | --program-transfo=* | --program-transf=* \ - | --program-trans=* | --program-tran=* \ - | --progr-tra=* | --program-tr=* | --program-t=*) - program_transform_name=$ac_optarg ;; - - -pdfdir | --pdfdir | --pdfdi | --pdfd | --pdf | --pd) - ac_prev=pdfdir ;; - -pdfdir=* | --pdfdir=* | --pdfdi=* | --pdfd=* | --pdf=* | --pd=*) - pdfdir=$ac_optarg ;; - - -psdir | --psdir | --psdi | --psd | --ps) - ac_prev=psdir ;; - -psdir=* | --psdir=* | --psdi=* | --psd=* | --ps=*) - psdir=$ac_optarg ;; - - -q | -quiet | --quiet | --quie | --qui | --qu | --q \ - | -silent | --silent | --silen | --sile | --sil) - silent=yes ;; - - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) - ac_prev=sbindir ;; - -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ - | --sbi=* | --sb=*) - sbindir=$ac_optarg ;; - - -sharedstatedir | --sharedstatedir | --sharedstatedi \ - | --sharedstated | --sharedstate | --sharedstat | --sharedsta \ - | --sharedst | --shareds | --shared | --share | --shar \ - | --sha | --sh) - ac_prev=sharedstatedir ;; - -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \ - | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \ - | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \ - | --sha=* | --sh=*) - sharedstatedir=$ac_optarg ;; - - -site | --site | --sit) - ac_prev=site ;; - -site=* | --site=* | --sit=*) - site=$ac_optarg ;; - - -srcdir | --srcdir | --srcdi | --srcd | --src | --sr) - ac_prev=srcdir ;; - -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*) - srcdir=$ac_optarg ;; - - -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \ - | --syscon | --sysco | --sysc | --sys | --sy) - ac_prev=sysconfdir ;; - -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \ - | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*) - sysconfdir=$ac_optarg ;; - - -target | --target | --targe | --targ | --tar | --ta | --t) - ac_prev=target_alias ;; - -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*) - target_alias=$ac_optarg ;; - - -v | -verbose | --verbose | --verbos | --verbo | --verb) - verbose=yes ;; - - -version | --version | --versio | --versi | --vers | -V) - ac_init_version=: ;; - - -with-* | --with-*) - ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: $ac_useropt" - ac_useropt_orig=$ac_useropt - ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` - case $ac_user_opts in - *" -"with_$ac_useropt" -"*) ;; - *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--with-$ac_useropt_orig" - ac_unrecognized_sep=', ';; - esac - eval with_$ac_useropt=\$ac_optarg ;; - - -without-* | --without-*) - ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` - # Reject names that are not valid shell variable names. - expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: $ac_useropt" - ac_useropt_orig=$ac_useropt - ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` - case $ac_user_opts in - *" -"with_$ac_useropt" -"*) ;; - *) ac_unrecognized_opts="$ac_unrecognized_opts$ac_unrecognized_sep--without-$ac_useropt_orig" - ac_unrecognized_sep=', ';; - esac - eval with_$ac_useropt=no ;; - - --x) - # Obsolete; use --with-x. - with_x=yes ;; - - -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \ - | --x-incl | --x-inc | --x-in | --x-i) - ac_prev=x_includes ;; - -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \ - | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*) - x_includes=$ac_optarg ;; - - -x-libraries | --x-libraries | --x-librarie | --x-librari \ - | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l) - ac_prev=x_libraries ;; - -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \ - | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) - x_libraries=$ac_optarg ;; - - -*) as_fn_error $? "unrecognized option: \`$ac_option' -Try \`$0 --help' for more information" - ;; - - *=*) - ac_envvar=`expr "x$ac_option" : 'x\([^=]*\)='` - # Reject names that are not valid shell variable names. - case $ac_envvar in #( - '' | [0-9]* | *[!_$as_cr_alnum]* ) - as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; - esac - eval $ac_envvar=\$ac_optarg - export $ac_envvar ;; - - *) - # FIXME: should be removed in autoconf 3.0. - $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 - expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && - $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 - : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" - ;; - - esac -done - -if test -n "$ac_prev"; then - ac_option=--`echo $ac_prev | sed 's/_/-/g'` - as_fn_error $? "missing argument to $ac_option" -fi - -if test -n "$ac_unrecognized_opts"; then - case $enable_option_checking in - no) ;; - fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; - *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; - esac -fi - -# Check all directory arguments for consistency. -for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ - datadir sysconfdir sharedstatedir localstatedir includedir \ - oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir -do - eval ac_val=\$$ac_var - # Remove trailing slashes. - case $ac_val in - */ ) - ac_val=`expr "X$ac_val" : 'X\(.*[^/]\)' \| "X$ac_val" : 'X\(.*\)'` - eval $ac_var=\$ac_val;; - esac - # Be sure to have absolute directory names. - case $ac_val in - [\\/$]* | ?:[\\/]* ) continue;; - NONE | '' ) case $ac_var in *prefix ) continue;; esac;; - esac - as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" -done - -# There might be people who depend on the old broken behavior: `$host' -# used to hold the argument of --host etc. -# FIXME: To remove some day. -build=$build_alias -host=$host_alias -target=$target_alias - -# FIXME: To remove some day. -if test "x$host_alias" != x; then - if test "x$build_alias" = x; then - cross_compiling=maybe - elif test "x$build_alias" != "x$host_alias"; then - cross_compiling=yes - fi -fi - -ac_tool_prefix= -test -n "$host_alias" && ac_tool_prefix=$host_alias- - -test "$silent" = yes && exec 6>/dev/null - - -ac_pwd=`pwd` && test -n "$ac_pwd" && -ac_ls_di=`ls -di .` && -ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || - as_fn_error $? "working directory cannot be determined" -test "X$ac_ls_di" = "X$ac_pwd_ls_di" || - as_fn_error $? "pwd does not report name of working directory" - - -# Find the source files, if location was not specified. -if test -z "$srcdir"; then - ac_srcdir_defaulted=yes - # Try the directory containing this script, then the parent directory. - ac_confdir=`$as_dirname -- "$as_myself" || -$as_expr X"$as_myself" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$as_myself" : 'X\(//\)[^/]' \| \ - X"$as_myself" : 'X\(//\)$' \| \ - X"$as_myself" : 'X\(/\)' \| . 2>/dev/null || -$as_echo X"$as_myself" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ - s//\1/ - q - } - /^X\(\/\/\)[^/].*/{ - s//\1/ - q - } - /^X\(\/\/\)$/{ - s//\1/ - q - } - /^X\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q'` - srcdir=$ac_confdir - if test ! -r "$srcdir/$ac_unique_file"; then - srcdir=.. - fi -else - ac_srcdir_defaulted=no -fi -if test ! -r "$srcdir/$ac_unique_file"; then - test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." - as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" -fi -ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" -ac_abs_confdir=`( - cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" - pwd)` -# When building in place, set srcdir=. -if test "$ac_abs_confdir" = "$ac_pwd"; then - srcdir=. -fi -# Remove unnecessary trailing slashes from srcdir. -# Double slashes in file names in object file debugging info -# mess up M-x gdb in Emacs. -case $srcdir in -*/) srcdir=`expr "X$srcdir" : 'X\(.*[^/]\)' \| "X$srcdir" : 'X\(.*\)'`;; -esac -for ac_var in $ac_precious_vars; do - eval ac_env_${ac_var}_set=\${${ac_var}+set} - eval ac_env_${ac_var}_value=\$${ac_var} - eval ac_cv_env_${ac_var}_set=\${${ac_var}+set} - eval ac_cv_env_${ac_var}_value=\$${ac_var} -done - -# -# Report the --help message. -# -if test "$ac_init_help" = "long"; then - # Omit some internal or obsolete options to make the list less imposing. - # This message is too long to be a string in the A/UX 3.1 sh. - cat <<_ACEOF -\`configure' configures sqlcipher 3.42.0 to adapt to many kinds of systems. - -Usage: $0 [OPTION]... [VAR=VALUE]... - -To assign environment variables (e.g., CC, CFLAGS...), specify them as -VAR=VALUE. See below for descriptions of some of the useful variables. - -Defaults for the options are specified in brackets. - -Configuration: - -h, --help display this help and exit - --help=short display options specific to this package - --help=recursive display the short help of all the included packages - -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking ...' messages - --cache-file=FILE cache test results in FILE [disabled] - -C, --config-cache alias for \`--cache-file=config.cache' - -n, --no-create do not create output files - --srcdir=DIR find the sources in DIR [configure dir or \`..'] - -Installation directories: - --prefix=PREFIX install architecture-independent files in PREFIX - [$ac_default_prefix] - --exec-prefix=EPREFIX install architecture-dependent files in EPREFIX - [PREFIX] - -By default, \`make install' will install all the files in -\`$ac_default_prefix/bin', \`$ac_default_prefix/lib' etc. You can specify -an installation prefix other than \`$ac_default_prefix' using \`--prefix', -for instance \`--prefix=\$HOME'. - -For better control, use the options below. - -Fine tuning of the installation directories: - --bindir=DIR user executables [EPREFIX/bin] - --sbindir=DIR system admin executables [EPREFIX/sbin] - --libexecdir=DIR program executables [EPREFIX/libexec] - --sysconfdir=DIR read-only single-machine data [PREFIX/etc] - --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] - --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] - --libdir=DIR object code libraries [EPREFIX/lib] - --includedir=DIR C header files [PREFIX/include] - --oldincludedir=DIR C header files for non-gcc [/usr/include] - --datarootdir=DIR read-only arch.-independent data root [PREFIX/share] - --datadir=DIR read-only architecture-independent data [DATAROOTDIR] - --infodir=DIR info documentation [DATAROOTDIR/info] - --localedir=DIR locale-dependent data [DATAROOTDIR/locale] - --mandir=DIR man documentation [DATAROOTDIR/man] - --docdir=DIR documentation root [DATAROOTDIR/doc/sqlcipher] - --htmldir=DIR html documentation [DOCDIR] - --dvidir=DIR dvi documentation [DOCDIR] - --pdfdir=DIR pdf documentation [DOCDIR] - --psdir=DIR ps documentation [DOCDIR] -_ACEOF - - cat <<\_ACEOF - -System types: - --build=BUILD configure for building on BUILD [guessed] - --host=HOST cross-compile to build programs to run on HOST [BUILD] -_ACEOF -fi - -if test -n "$ac_init_help"; then - case $ac_init_help in - short | recursive ) echo "Configuration of sqlcipher 3.42.0:";; - esac - cat <<\_ACEOF - -Optional Features: - --disable-option-checking ignore unrecognized --enable/--with options - --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) - --enable-FEATURE[=ARG] include FEATURE [ARG=yes] - --enable-shared[=PKGS] build shared libraries [default=yes] - --enable-static[=PKGS] build static libraries [default=yes] - --enable-fast-install[=PKGS] - optimize for fast installation [default=yes] - --disable-libtool-lock avoid locking (might break parallel builds) - --disable-largefile omit support for large files - --disable-threadsafe Disable mutexing - --enable-cross-thread-connections - Allow connection sharing across threads - --enable-releasemode Support libtool link to release mode - --enable-tempstore Use an in-ram database for temporary tables - (never,no,yes,always) - --disable-tcl do not build TCL extension - --enable-editline enable BSD editline support - --disable-readline disable readline support - --enable-debug enable debugging & verbose explain - --disable-amalgamation Disable the amalgamation and instead build all files - separately - --disable-load-extension - Disable loading of external extensions - --disable-math Disable math functions - --disable-json Disable JSON functions - --enable-all Enable FTS4, FTS5, Geopoly, RTree, Sessions - --enable-memsys5 Enable MEMSYS5 - --enable-memsys3 Enable MEMSYS3 - --enable-fts3 Enable the FTS3 extension - --enable-fts4 Enable the FTS4 extension - --enable-fts5 Enable the FTS5 extension - --enable-update-limit Enable the UPDATE/DELETE LIMIT clause - --enable-geopoly Enable the GEOPOLY extension - --enable-rtree Enable the RTREE extension - --enable-session Enable the SESSION extension - --enable-gcov Enable coverage testing using gcov - -Optional Packages: - --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] - --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-pic[=PKGS] try to use only PIC/non-PIC objects [default=use - both] - --with-aix-soname=aix|svr4|both - shared library versioning (aka "SONAME") variant to - provide on AIX, [default=aix]. - --with-gnu-ld assume the C compiler uses GNU ld [default=no] - --with-sysroot[=DIR] Search for dependent libraries within DIR (or the - compiler's sysroot if not specified). - --with-wasi-sdk=DIR directory containing the WASI SDK. Triggers - cross-compile to WASM. - --with-crypto-lib Specify which crypto library to use - --with-tcl=DIR directory containing tcl configuration - (tclConfig.sh) - --with-readline-lib specify readline library - --with-readline-inc specify readline include paths - -Some influential environment variables: - CC C compiler command - CFLAGS C compiler flags - LDFLAGS linker flags, e.g. -L if you have libraries in a - nonstandard directory - LIBS libraries to pass to the linker, e.g. -l - CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if - you have headers in a nonstandard directory - LT_SYS_LIBRARY_PATH - User-defined run-time library search path. - CPP C preprocessor - TCLLIBDIR Where to install tcl plugin - amalgamation_line_macros - - -Use these variables to override the choices made by `configure' or to help -it to find libraries and programs with nonstandard names/locations. - -Report bugs to the package provider. -_ACEOF -ac_status=$? -fi - -if test "$ac_init_help" = "recursive"; then - # If there are subdirs, report their specific --help. - for ac_dir in : $ac_subdirs_all; do test "x$ac_dir" = x: && continue - test -d "$ac_dir" || - { cd "$srcdir" && ac_pwd=`pwd` && srcdir=. && test -d "$ac_dir"; } || - continue - ac_builddir=. - -case "$ac_dir" in -.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; -*) - ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` - # A ".." for each directory in $ac_dir_suffix. - ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` - case $ac_top_builddir_sub in - "") ac_top_builddir_sub=. ac_top_build_prefix= ;; - *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; - esac ;; -esac -ac_abs_top_builddir=$ac_pwd -ac_abs_builddir=$ac_pwd$ac_dir_suffix -# for backward compatibility: -ac_top_builddir=$ac_top_build_prefix - -case $srcdir in - .) # We are building in place. - ac_srcdir=. - ac_top_srcdir=$ac_top_builddir_sub - ac_abs_top_srcdir=$ac_pwd ;; - [\\/]* | ?:[\\/]* ) # Absolute name. - ac_srcdir=$srcdir$ac_dir_suffix; - ac_top_srcdir=$srcdir - ac_abs_top_srcdir=$srcdir ;; - *) # Relative name. - ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix - ac_top_srcdir=$ac_top_build_prefix$srcdir - ac_abs_top_srcdir=$ac_pwd/$srcdir ;; -esac -ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix - - cd "$ac_dir" || { ac_status=$?; continue; } - # Check for guested configure. - if test -f "$ac_srcdir/configure.gnu"; then - echo && - $SHELL "$ac_srcdir/configure.gnu" --help=recursive - elif test -f "$ac_srcdir/configure"; then - echo && - $SHELL "$ac_srcdir/configure" --help=recursive - else - $as_echo "$as_me: WARNING: no configuration information is in $ac_dir" >&2 - fi || ac_status=$? - cd "$ac_pwd" || { ac_status=$?; break; } - done -fi - -test -n "$ac_init_help" && exit $ac_status -if $ac_init_version; then - cat <<\_ACEOF -sqlcipher configure 3.42.0 -generated by GNU Autoconf 2.69 - -Copyright (C) 2012 Free Software Foundation, Inc. -This configure script is free software; the Free Software Foundation -gives unlimited permission to copy, distribute and modify it. -_ACEOF - exit -fi - -## ------------------------ ## -## Autoconf initialization. ## -## ------------------------ ## - -# ac_fn_c_try_compile LINENO -# -------------------------- -# Try to compile conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_compile () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - rm -f conftest.$ac_objext - if { { ac_try="$ac_compile" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_compile") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { - test -z "$ac_c_werror_flag" || - test ! -s conftest.err - } && test -s conftest.$ac_objext; then : - ac_retval=0 -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 -fi - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_compile - -# ac_fn_c_try_link LINENO -# ----------------------- -# Try to link conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_link () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - rm -f conftest.$ac_objext conftest$ac_exeext - if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { - test -z "$ac_c_werror_flag" || - test ! -s conftest.err - } && test -s conftest$ac_exeext && { - test "$cross_compiling" = yes || - test -x conftest$ac_exeext - }; then : - ac_retval=0 -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 -fi - # Delete the IPA/IPO (Inter Procedural Analysis/Optimization) information - # created by the PGI compiler (conftest_ipa8_conftest.oo), as it would - # interfere with the next link command; also delete a directory that is - # left behind by Apple's compiler. We do this before executing the actions. - rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_link - -# ac_fn_c_check_header_compile LINENO HEADER VAR INCLUDES -# ------------------------------------------------------- -# Tests whether HEADER exists and can be compiled using the include files in -# INCLUDES, setting the cache variable VAR accordingly. -ac_fn_c_check_header_compile () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if eval \${$3+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -#include <$2> -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - eval "$3=yes" -else - eval "$3=no" -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - -} # ac_fn_c_check_header_compile - -# ac_fn_c_try_cpp LINENO -# ---------------------- -# Try to preprocess conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_cpp () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_cpp conftest.$ac_ext" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } > conftest.i && { - test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || - test ! -s conftest.err - }; then : - ac_retval=0 -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 -fi - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_cpp - -# ac_fn_c_try_run LINENO -# ---------------------- -# Try to link conftest.$ac_ext, and return whether this succeeded. Assumes -# that executables *can* be run. -ac_fn_c_try_run () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && { ac_try='./conftest$ac_exeext' - { { case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_try") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; }; then : - ac_retval=0 -else - $as_echo "$as_me: program exited with status $ac_status" >&5 - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=$ac_status -fi - rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - as_fn_set_status $ac_retval - -} # ac_fn_c_try_run - -# ac_fn_c_check_func LINENO FUNC VAR -# ---------------------------------- -# Tests whether FUNC exists, setting the cache variable VAR accordingly -ac_fn_c_check_func () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if eval \${$3+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -/* Define $2 to an innocuous variant, in case declares $2. - For example, HP-UX 11i declares gettimeofday. */ -#define $2 innocuous_$2 - -/* System header to define __stub macros and hopefully few prototypes, - which can conflict with char $2 (); below. - Prefer to if __STDC__ is defined, since - exists even on freestanding compilers. */ - -#ifdef __STDC__ -# include -#else -# include -#endif - -#undef $2 - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char $2 (); -/* The GNU C library defines this for functions which it implements - to always fail with ENOSYS. Some functions are actually named - something starting with __ and the normal name is an alias. */ -#if defined __stub_$2 || defined __stub___$2 -choke me -#endif - -int -main () -{ -return $2 (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - eval "$3=yes" -else - eval "$3=no" -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - -} # ac_fn_c_check_func - -# ac_fn_c_check_type LINENO TYPE VAR INCLUDES -# ------------------------------------------- -# Tests whether TYPE exists after having included INCLUDES, setting cache -# variable VAR accordingly. -ac_fn_c_check_type () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if eval \${$3+:} false; then : - $as_echo_n "(cached) " >&6 -else - eval "$3=no" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -int -main () -{ -if (sizeof ($2)) - return 0; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -int -main () -{ -if (sizeof (($2))) - return 0; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - -else - eval "$3=yes" -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - -} # ac_fn_c_check_type - -# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES -# ------------------------------------------------------- -# Tests whether HEADER exists, giving a warning if it cannot be compiled using -# the include files in INCLUDES and setting the cache variable VAR -# accordingly. -ac_fn_c_check_header_mongrel () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if eval \${$3+:} false; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if eval \${$3+:} false; then : - $as_echo_n "(cached) " >&6 -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -else - # Is the header compilable? -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 -$as_echo_n "checking $2 usability... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -#include <$2> -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_header_compiler=yes -else - ac_header_compiler=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 -$as_echo "$ac_header_compiler" >&6; } - -# Is the header present? -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 -$as_echo_n "checking $2 presence... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include <$2> -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - ac_header_preproc=yes -else - ac_header_preproc=no -fi -rm -f conftest.err conftest.i conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 -$as_echo "$ac_header_preproc" >&6; } - -# So? What about this header? -case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( - yes:no: ) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 -$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 -$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} - ;; - no:yes:* ) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 -$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 -$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 -$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 -$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 -$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} - ;; -esac - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if eval \${$3+:} false; then : - $as_echo_n "(cached) " >&6 -else - eval "$3=\$ac_header_compiler" -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -fi - eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno - -} # ac_fn_c_check_header_mongrel -cat >config.log <<_ACEOF -This file contains any messages produced by compilers while -running configure, to aid debugging if configure makes a mistake. - -It was created by sqlcipher $as_me 3.42.0, which was -generated by GNU Autoconf 2.69. Invocation command line was - - $ $0 $@ - -_ACEOF -exec 5>>config.log -{ -cat <<_ASUNAME -## --------- ## -## Platform. ## -## --------- ## - -hostname = `(hostname || uname -n) 2>/dev/null | sed 1q` -uname -m = `(uname -m) 2>/dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` - -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null || echo unknown` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null || echo unknown` - -/bin/arch = `(/bin/arch) 2>/dev/null || echo unknown` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null || echo unknown` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null || echo unknown` -/usr/bin/hostinfo = `(/usr/bin/hostinfo) 2>/dev/null || echo unknown` -/bin/machine = `(/bin/machine) 2>/dev/null || echo unknown` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null || echo unknown` -/bin/universe = `(/bin/universe) 2>/dev/null || echo unknown` - -_ASUNAME - -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - $as_echo "PATH: $as_dir" - done -IFS=$as_save_IFS - -} >&5 - -cat >&5 <<_ACEOF - - -## ----------- ## -## Core tests. ## -## ----------- ## - -_ACEOF - - -# Keep a trace of the command line. -# Strip out --no-create and --no-recursion so they do not pile up. -# Strip out --silent because we don't want to record it for future runs. -# Also quote any args containing shell meta-characters. -# Make two passes to allow for proper duplicate-argument suppression. -ac_configure_args= -ac_configure_args0= -ac_configure_args1= -ac_must_keep_next=false -for ac_pass in 1 2 -do - for ac_arg - do - case $ac_arg in - -no-create | --no-c* | -n | -no-recursion | --no-r*) continue ;; - -q | -quiet | --quiet | --quie | --qui | --qu | --q \ - | -silent | --silent | --silen | --sile | --sil) - continue ;; - *\'*) - ac_arg=`$as_echo "$ac_arg" | sed "s/'/'\\\\\\\\''/g"` ;; - esac - case $ac_pass in - 1) as_fn_append ac_configure_args0 " '$ac_arg'" ;; - 2) - as_fn_append ac_configure_args1 " '$ac_arg'" - if test $ac_must_keep_next = true; then - ac_must_keep_next=false # Got value, back to normal. - else - case $ac_arg in - *=* | --config-cache | -C | -disable-* | --disable-* \ - | -enable-* | --enable-* | -gas | --g* | -nfp | --nf* \ - | -q | -quiet | --q* | -silent | --sil* | -v | -verb* \ - | -with-* | --with-* | -without-* | --without-* | --x) - case "$ac_configure_args0 " in - "$ac_configure_args1"*" '$ac_arg' "* ) continue ;; - esac - ;; - -* ) ac_must_keep_next=true ;; - esac - fi - as_fn_append ac_configure_args " '$ac_arg'" - ;; - esac - done -done -{ ac_configure_args0=; unset ac_configure_args0;} -{ ac_configure_args1=; unset ac_configure_args1;} - -# When interrupted or exit'd, cleanup temporary files, and complete -# config.log. We remove comments because anyway the quotes in there -# would cause problems or look ugly. -# WARNING: Use '\'' to represent an apostrophe within the trap. -# WARNING: Do not start the trap code with a newline, due to a FreeBSD 4.0 bug. -trap 'exit_status=$? - # Save into config.log some information that might help in debugging. - { - echo - - $as_echo "## ---------------- ## -## Cache variables. ## -## ---------------- ##" - echo - # The following way of writing the cache mishandles newlines in values, -( - for ac_var in `(set) 2>&1 | sed -n '\''s/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'\''`; do - eval ac_val=\$$ac_var - case $ac_val in #( - *${as_nl}*) - case $ac_var in #( - *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 -$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; - esac - case $ac_var in #( - _ | IFS | as_nl) ;; #( - BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( - *) { eval $ac_var=; unset $ac_var;} ;; - esac ;; - esac - done - (set) 2>&1 | - case $as_nl`(ac_space='\'' '\''; set) 2>&1` in #( - *${as_nl}ac_space=\ *) - sed -n \ - "s/'\''/'\''\\\\'\'''\''/g; - s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\''\\2'\''/p" - ;; #( - *) - sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" - ;; - esac | - sort -) - echo - - $as_echo "## ----------------- ## -## Output variables. ## -## ----------------- ##" - echo - for ac_var in $ac_subst_vars - do - eval ac_val=\$$ac_var - case $ac_val in - *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; - esac - $as_echo "$ac_var='\''$ac_val'\''" - done | sort - echo - - if test -n "$ac_subst_files"; then - $as_echo "## ------------------- ## -## File substitutions. ## -## ------------------- ##" - echo - for ac_var in $ac_subst_files - do - eval ac_val=\$$ac_var - case $ac_val in - *\'\''*) ac_val=`$as_echo "$ac_val" | sed "s/'\''/'\''\\\\\\\\'\'''\''/g"`;; - esac - $as_echo "$ac_var='\''$ac_val'\''" - done | sort - echo - fi - - if test -s confdefs.h; then - $as_echo "## ----------- ## -## confdefs.h. ## -## ----------- ##" - echo - cat confdefs.h - echo - fi - test "$ac_signal" != 0 && - $as_echo "$as_me: caught signal $ac_signal" - $as_echo "$as_me: exit $exit_status" - } >&5 - rm -f core *.core core.conftest.* && - rm -f -r conftest* confdefs* conf$$* $ac_clean_files && - exit $exit_status -' 0 -for ac_signal in 1 2 13 15; do - trap 'ac_signal='$ac_signal'; as_fn_exit 1' $ac_signal -done -ac_signal=0 - -# confdefs.h avoids OS command line length limits that DEFS can exceed. -rm -f -r conftest* confdefs.h - -$as_echo "/* confdefs.h */" > confdefs.h - -# Predefined preprocessor variables. - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_NAME "$PACKAGE_NAME" -_ACEOF - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_TARNAME "$PACKAGE_TARNAME" -_ACEOF - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_VERSION "$PACKAGE_VERSION" -_ACEOF - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_STRING "$PACKAGE_STRING" -_ACEOF - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_BUGREPORT "$PACKAGE_BUGREPORT" -_ACEOF - -cat >>confdefs.h <<_ACEOF -#define PACKAGE_URL "$PACKAGE_URL" -_ACEOF - - -# Let the site file select an alternate cache file if it wants to. -# Prefer an explicitly selected file to automatically selected ones. -ac_site_file1=NONE -ac_site_file2=NONE -if test -n "$CONFIG_SITE"; then - # We do not want a PATH search for config.site. - case $CONFIG_SITE in #(( - -*) ac_site_file1=./$CONFIG_SITE;; - */*) ac_site_file1=$CONFIG_SITE;; - *) ac_site_file1=./$CONFIG_SITE;; - esac -elif test "x$prefix" != xNONE; then - ac_site_file1=$prefix/share/config.site - ac_site_file2=$prefix/etc/config.site -else - ac_site_file1=$ac_default_prefix/share/config.site - ac_site_file2=$ac_default_prefix/etc/config.site -fi -for ac_site_file in "$ac_site_file1" "$ac_site_file2" -do - test "x$ac_site_file" = xNONE && continue - if test /dev/null != "$ac_site_file" && test -r "$ac_site_file"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 -$as_echo "$as_me: loading site script $ac_site_file" >&6;} - sed 's/^/| /' "$ac_site_file" >&5 - . "$ac_site_file" \ - || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "failed to load site script $ac_site_file -See \`config.log' for more details" "$LINENO" 5; } - fi -done - -if test -r "$cache_file"; then - # Some versions of bash will fail to source /dev/null (special files - # actually), so we avoid doing that. DJGPP emulates it as a regular file. - if test /dev/null != "$cache_file" && test -f "$cache_file"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: loading cache $cache_file" >&5 -$as_echo "$as_me: loading cache $cache_file" >&6;} - case $cache_file in - [\\/]* | ?:[\\/]* ) . "$cache_file";; - *) . "./$cache_file";; - esac - fi -else - { $as_echo "$as_me:${as_lineno-$LINENO}: creating cache $cache_file" >&5 -$as_echo "$as_me: creating cache $cache_file" >&6;} - >$cache_file -fi - -# Check that the precious variables saved in the cache have kept the same -# value. -ac_cache_corrupted=false -for ac_var in $ac_precious_vars; do - eval ac_old_set=\$ac_cv_env_${ac_var}_set - eval ac_new_set=\$ac_env_${ac_var}_set - eval ac_old_val=\$ac_cv_env_${ac_var}_value - eval ac_new_val=\$ac_env_${ac_var}_value - case $ac_old_set,$ac_new_set in - set,) - { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&5 -$as_echo "$as_me: error: \`$ac_var' was set to \`$ac_old_val' in the previous run" >&2;} - ac_cache_corrupted=: ;; - ,set) - { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' was not set in the previous run" >&5 -$as_echo "$as_me: error: \`$ac_var' was not set in the previous run" >&2;} - ac_cache_corrupted=: ;; - ,);; - *) - if test "x$ac_old_val" != "x$ac_new_val"; then - # differences in whitespace do not lead to failure. - ac_old_val_w=`echo x $ac_old_val` - ac_new_val_w=`echo x $ac_new_val` - if test "$ac_old_val_w" != "$ac_new_val_w"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: error: \`$ac_var' has changed since the previous run:" >&5 -$as_echo "$as_me: error: \`$ac_var' has changed since the previous run:" >&2;} - ac_cache_corrupted=: - else - { $as_echo "$as_me:${as_lineno-$LINENO}: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&5 -$as_echo "$as_me: warning: ignoring whitespace changes in \`$ac_var' since the previous run:" >&2;} - eval $ac_var=\$ac_old_val - fi - { $as_echo "$as_me:${as_lineno-$LINENO}: former value: \`$ac_old_val'" >&5 -$as_echo "$as_me: former value: \`$ac_old_val'" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: current value: \`$ac_new_val'" >&5 -$as_echo "$as_me: current value: \`$ac_new_val'" >&2;} - fi;; - esac - # Pass precious variables to config.status. - if test "$ac_new_set" = set; then - case $ac_new_val in - *\'*) ac_arg=$ac_var=`$as_echo "$ac_new_val" | sed "s/'/'\\\\\\\\''/g"` ;; - *) ac_arg=$ac_var=$ac_new_val ;; - esac - case " $ac_configure_args " in - *" '$ac_arg' "*) ;; # Avoid dups. Use of quotes ensures accuracy. - *) as_fn_append ac_configure_args " '$ac_arg'" ;; - esac - fi -done -if $ac_cache_corrupted; then - { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 -$as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} - as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 -fi -## -------------------- ## -## Main body of script. ## -## -------------------- ## - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - - -sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'` -if test "$PACKAGE_VERSION" != "$sqlite_version_sanity_check" ; then -as_fn_error $? "configure script is out of date: - configure \$PACKAGE_VERSION = $PACKAGE_VERSION - top level VERSION file = $sqlite_version_sanity_check -please regen with autoconf" "$LINENO" 5 -fi - -######### -# Programs needed -# -ac_aux_dir= -for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do - if test -f "$ac_dir/install-sh"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/install-sh -c" - break - elif test -f "$ac_dir/install.sh"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/install.sh -c" - break - elif test -f "$ac_dir/shtool"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/shtool install -c" - break - fi -done -if test -z "$ac_aux_dir"; then - as_fn_error $? "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 -fi - -# These three variables are undocumented and unsupported, -# and are intended to be withdrawn in a future Autoconf release. -# They can cause serious problems if a builder's source tree is in a directory -# whose full name contains unusual characters. -ac_config_guess="$SHELL $ac_aux_dir/config.guess" # Please don't use this var. -ac_config_sub="$SHELL $ac_aux_dir/config.sub" # Please don't use this var. -ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. - - -case `pwd` in - *\ * | *\ *) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Libtool does not cope well with whitespace in \`pwd\`" >&5 -$as_echo "$as_me: WARNING: Libtool does not cope well with whitespace in \`pwd\`" >&2;} ;; -esac - - - -macro_version='2.4.6' -macro_revision='2.4.6' - - - - - - - - - - - - - -ltmain=$ac_aux_dir/ltmain.sh - -# Make sure we can run config.sub. -$SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 || - as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking build system type" >&5 -$as_echo_n "checking build system type... " >&6; } -if ${ac_cv_build+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_build_alias=$build_alias -test "x$ac_build_alias" = x && - ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"` -test "x$ac_build_alias" = x && - as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 -ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` || - as_fn_error $? "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 -$as_echo "$ac_cv_build" >&6; } -case $ac_cv_build in -*-*-*) ;; -*) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;; -esac -build=$ac_cv_build -ac_save_IFS=$IFS; IFS='-' -set x $ac_cv_build -shift -build_cpu=$1 -build_vendor=$2 -shift; shift -# Remember, the first character of IFS is used to create $*, -# except with old shells: -build_os=$* -IFS=$ac_save_IFS -case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking host system type" >&5 -$as_echo_n "checking host system type... " >&6; } -if ${ac_cv_host+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test "x$host_alias" = x; then - ac_cv_host=$ac_cv_build -else - ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` || - as_fn_error $? "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 -fi - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_host" >&5 -$as_echo "$ac_cv_host" >&6; } -case $ac_cv_host in -*-*-*) ;; -*) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;; -esac -host=$ac_cv_host -ac_save_IFS=$IFS; IFS='-' -set x $ac_cv_host -shift -host_cpu=$1 -host_vendor=$2 -shift; shift -# Remember, the first character of IFS is used to create $*, -# except with old shells: -host_os=$* -IFS=$ac_save_IFS -case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac - - -# Backslashify metacharacters that are still active within -# double-quoted strings. -sed_quote_subst='s/\(["`$\\]\)/\\\1/g' - -# Same as above, but do not quote variable references. -double_quote_subst='s/\(["`\\]\)/\\\1/g' - -# Sed substitution to delay expansion of an escaped shell variable in a -# double_quote_subst'ed string. -delay_variable_subst='s/\\\\\\\\\\\$/\\\\\\$/g' - -# Sed substitution to delay expansion of an escaped single quote. -delay_single_quote_subst='s/'\''/'\'\\\\\\\'\''/g' - -# Sed substitution to avoid accidental globbing in evaled expressions -no_glob_subst='s/\*/\\\*/g' - -ECHO='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' -ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO -ECHO=$ECHO$ECHO$ECHO$ECHO$ECHO$ECHO - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to print strings" >&5 -$as_echo_n "checking how to print strings... " >&6; } -# Test print first, because it will be a builtin if present. -if test "X`( print -r -- -n ) 2>/dev/null`" = X-n && \ - test "X`print -r -- $ECHO 2>/dev/null`" = "X$ECHO"; then - ECHO='print -r --' -elif test "X`printf %s $ECHO 2>/dev/null`" = "X$ECHO"; then - ECHO='printf %s\n' -else - # Use this function as a fallback that always works. - func_fallback_echo () - { - eval 'cat <<_LTECHO_EOF -$1 -_LTECHO_EOF' - } - ECHO='func_fallback_echo' -fi - -# func_echo_all arg... -# Invoke $ECHO with all args, space-separated. -func_echo_all () -{ - $ECHO "" -} - -case $ECHO in - printf*) { $as_echo "$as_me:${as_lineno-$LINENO}: result: printf" >&5 -$as_echo "printf" >&6; } ;; - print*) { $as_echo "$as_me:${as_lineno-$LINENO}: result: print -r" >&5 -$as_echo "print -r" >&6; } ;; - *) { $as_echo "$as_me:${as_lineno-$LINENO}: result: cat" >&5 -$as_echo "cat" >&6; } ;; -esac - - - - - - - - - - - - - - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}gcc", so it can be a program name with args. -set dummy ${ac_tool_prefix}gcc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}gcc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_CC"; then - ac_ct_CC=$CC - # Extract the first word of "gcc", so it can be a program name with args. -set dummy gcc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="gcc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 -$as_echo "$ac_ct_CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_CC" = x; then - CC="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - CC=$ac_ct_CC - fi -else - CC="$ac_cv_prog_CC" -fi - -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}cc", so it can be a program name with args. -set dummy ${ac_tool_prefix}cc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="${ac_tool_prefix}cc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - fi -fi -if test -z "$CC"; then - # Extract the first word of "cc", so it can be a program name with args. -set dummy cc; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else - ac_prog_rejected=no -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - if test "$as_dir/$ac_word$ac_exec_ext" = "/usr/ucb/cc"; then - ac_prog_rejected=yes - continue - fi - ac_cv_prog_CC="cc" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -if test $ac_prog_rejected = yes; then - # We found a bogon in the path, so make sure we never use it. - set dummy $ac_cv_prog_CC - shift - if test $# != 0; then - # We chose a different compiler from the bogus one. - # However, it has the same basename, so the bogon will be chosen - # first if we set CC to just the basename; use the full file name. - shift - ac_cv_prog_CC="$as_dir/$ac_word${1+' '}$@" - fi -fi -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$CC"; then - if test -n "$ac_tool_prefix"; then - for ac_prog in cl.exe - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$CC"; then - ac_cv_prog_CC="$CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_CC="$ac_tool_prefix$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -CC=$ac_cv_prog_CC -if test -n "$CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CC" >&5 -$as_echo "$CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$CC" && break - done -fi -if test -z "$CC"; then - ac_ct_CC=$CC - for ac_prog in cl.exe -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_CC"; then - ac_cv_prog_ac_ct_CC="$ac_ct_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_CC="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_CC=$ac_cv_prog_ac_ct_CC -if test -n "$ac_ct_CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_CC" >&5 -$as_echo "$ac_ct_CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$ac_ct_CC" && break -done - - if test "x$ac_ct_CC" = x; then - CC="" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - CC=$ac_ct_CC - fi -fi - -fi - - -test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "no acceptable C compiler found in \$PATH -See \`config.log' for more details" "$LINENO" 5; } - -# Provide some information about the compiler. -$as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 -set X $ac_compile -ac_compiler=$2 -for ac_option in --version -v -V -qversion; do - { { ac_try="$ac_compiler $ac_option >&5" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_compiler $ac_option >&5") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - sed '10a\ -... rest of stderr output deleted ... - 10q' conftest.err >conftest.er1 - cat conftest.er1 >&5 - fi - rm -f conftest.er1 conftest.err - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } -done - -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -ac_clean_files_save=$ac_clean_files -ac_clean_files="$ac_clean_files a.out a.out.dSYM a.exe b.out" -# Try to create an executable without -o first, disregard a.out. -# It will help us diagnose broken compilers, and finding out an intuition -# of exeext. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler works" >&5 -$as_echo_n "checking whether the C compiler works... " >&6; } -ac_link_default=`$as_echo "$ac_link" | sed 's/ -o *conftest[^ ]*//'` - -# The possible output files: -ac_files="a.out conftest.exe conftest a.exe a_out.exe b.out conftest.*" - -ac_rmfiles= -for ac_file in $ac_files -do - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; - * ) ac_rmfiles="$ac_rmfiles $ac_file";; - esac -done -rm -f $ac_rmfiles - -if { { ac_try="$ac_link_default" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link_default") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then : - # Autoconf-2.13 could set the ac_cv_exeext variable to `no'. -# So ignore a value of `no', otherwise this would lead to `EXEEXT = no' -# in a Makefile. We should not override ac_cv_exeext if it was cached, -# so that the user can short-circuit this test for compilers unknown to -# Autoconf. -for ac_file in $ac_files '' -do - test -f "$ac_file" || continue - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) - ;; - [ab].out ) - # We found the default executable, but exeext='' is most - # certainly right. - break;; - *.* ) - if test "${ac_cv_exeext+set}" = set && test "$ac_cv_exeext" != no; - then :; else - ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` - fi - # We set ac_cv_exeext here because the later test for it is not - # safe: cross compilers may not add the suffix if given an `-o' - # argument, so we may need to know it at that point already. - # Even if this section looks crufty: it has the advantage of - # actually working. - break;; - * ) - break;; - esac -done -test "$ac_cv_exeext" = no && ac_cv_exeext= - -else - ac_file='' -fi -if test -z "$ac_file"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -$as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "C compiler cannot create executables -See \`config.log' for more details" "$LINENO" 5; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler default output file name" >&5 -$as_echo_n "checking for C compiler default output file name... " >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_file" >&5 -$as_echo "$ac_file" >&6; } -ac_exeext=$ac_cv_exeext - -rm -f -r a.out a.out.dSYM a.exe conftest$ac_cv_exeext b.out -ac_clean_files=$ac_clean_files_save -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of executables" >&5 -$as_echo_n "checking for suffix of executables... " >&6; } -if { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then : - # If both `conftest.exe' and `conftest' are `present' (well, observable) -# catch `conftest.exe'. For instance with Cygwin, `ls conftest' will -# work properly (i.e., refer to `conftest.exe'), while it won't with -# `rm'. -for ac_file in conftest.exe conftest conftest.*; do - test -f "$ac_file" || continue - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM | *.o | *.obj ) ;; - *.* ) ac_cv_exeext=`expr "$ac_file" : '[^.]*\(\..*\)'` - break;; - * ) break;; - esac -done -else - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details" "$LINENO" 5; } -fi -rm -f conftest conftest$ac_cv_exeext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 -$as_echo "$ac_cv_exeext" >&6; } - -rm -f conftest.$ac_ext -EXEEXT=$ac_cv_exeext -ac_exeext=$EXEEXT -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -FILE *f = fopen ("conftest.out", "w"); - return ferror (f) || fclose (f) != 0; - - ; - return 0; -} -_ACEOF -ac_clean_files="$ac_clean_files conftest.out" -# Check that the compiler produces executables we can run. If not, either -# the compiler is broken, or we cross compile. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are cross compiling" >&5 -$as_echo_n "checking whether we are cross compiling... " >&6; } -if test "$cross_compiling" != yes; then - { { ac_try="$ac_link" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_link") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - if { ac_try='./conftest$ac_cv_exeext' - { { case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_try") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; }; then - cross_compiling=no - else - if test "$cross_compiling" = maybe; then - cross_compiling=yes - else - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot run C compiled programs. -If you meant to cross compile, use \`--host'. -See \`config.log' for more details" "$LINENO" 5; } - fi - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $cross_compiling" >&5 -$as_echo "$cross_compiling" >&6; } - -rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out -ac_clean_files=$ac_clean_files_save -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 -$as_echo_n "checking for suffix of object files... " >&6; } -if ${ac_cv_objext+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -rm -f conftest.o conftest.obj -if { { ac_try="$ac_compile" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_compile") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then : - for ac_file in conftest.o conftest.obj conftest.*; do - test -f "$ac_file" || continue; - case $ac_file in - *.$ac_ext | *.xcoff | *.tds | *.d | *.pdb | *.xSYM | *.bb | *.bbg | *.map | *.inf | *.dSYM ) ;; - *) ac_cv_objext=`expr "$ac_file" : '.*\.\(.*\)'` - break;; - esac -done -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - -{ { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot compute suffix of object files: cannot compile -See \`config.log' for more details" "$LINENO" 5; } -fi -rm -f conftest.$ac_cv_objext conftest.$ac_ext -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_objext" >&5 -$as_echo "$ac_cv_objext" >&6; } -OBJEXT=$ac_cv_objext -ac_objext=$OBJEXT -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 -$as_echo_n "checking whether we are using the GNU C compiler... " >&6; } -if ${ac_cv_c_compiler_gnu+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ -#ifndef __GNUC__ - choke me -#endif - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_compiler_gnu=yes -else - ac_compiler_gnu=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -ac_cv_c_compiler_gnu=$ac_compiler_gnu - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_compiler_gnu" >&5 -$as_echo "$ac_cv_c_compiler_gnu" >&6; } -if test $ac_compiler_gnu = yes; then - GCC=yes -else - GCC= -fi -ac_test_CFLAGS=${CFLAGS+set} -ac_save_CFLAGS=$CFLAGS -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 -$as_echo_n "checking whether $CC accepts -g... " >&6; } -if ${ac_cv_prog_cc_g+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_save_c_werror_flag=$ac_c_werror_flag - ac_c_werror_flag=yes - ac_cv_prog_cc_g=no - CFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_prog_cc_g=yes -else - CFLAGS="" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - -else - ac_c_werror_flag=$ac_save_c_werror_flag - CFLAGS="-g" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_prog_cc_g=yes -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - ac_c_werror_flag=$ac_save_c_werror_flag -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_g" >&5 -$as_echo "$ac_cv_prog_cc_g" >&6; } -if test "$ac_test_CFLAGS" = set; then - CFLAGS=$ac_save_CFLAGS -elif test $ac_cv_prog_cc_g = yes; then - if test "$GCC" = yes; then - CFLAGS="-g -O2" - else - CFLAGS="-g" - fi -else - if test "$GCC" = yes; then - CFLAGS="-O2" - else - CFLAGS= - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 -$as_echo_n "checking for $CC option to accept ISO C89... " >&6; } -if ${ac_cv_prog_cc_c89+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_cv_prog_cc_c89=no -ac_save_CC=$CC -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -#include -struct stat; -/* Most of the following tests are stolen from RCS 5.7's src/conf.sh. */ -struct buf { int x; }; -FILE * (*rcsopen) (struct buf *, struct stat *, int); -static char *e (p, i) - char **p; - int i; -{ - return p[i]; -} -static char *f (char * (*g) (char **, int), char **p, ...) -{ - char *s; - va_list v; - va_start (v,p); - s = g (p, va_arg (v,int)); - va_end (v); - return s; -} - -/* OSF 4.0 Compaq cc is some sort of almost-ANSI by default. It has - function prototypes and stuff, but not '\xHH' hex character constants. - These don't provoke an error unfortunately, instead are silently treated - as 'x'. The following induces an error, until -std is added to get - proper ANSI mode. Curiously '\x00'!='x' always comes out true, for an - array size at least. It's necessary to write '\x00'==0 to get something - that's true only with -std. */ -int osf4_cc_array ['\x00' == 0 ? 1 : -1]; - -/* IBM C 6 for AIX is almost-ANSI by default, but it replaces macro parameters - inside strings and character constants. */ -#define FOO(x) 'x' -int xlc6_cc_array[FOO(a) == 'x' ? 1 : -1]; - -int test (int i, double x); -struct s1 {int (*f) (int a);}; -struct s2 {int (*f) (double a);}; -int pairnames (int, char **, FILE *(*)(struct buf *, struct stat *, int), int, int); -int argc; -char **argv; -int -main () -{ -return f (e, argv, 0) != argv[0] || f (e, argv, 1) != argv[1]; - ; - return 0; -} -_ACEOF -for ac_arg in '' -qlanglvl=extc89 -qlanglvl=ansi -std \ - -Ae "-Aa -D_HPUX_SOURCE" "-Xc -D__EXTENSIONS__" -do - CC="$ac_save_CC $ac_arg" - if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_prog_cc_c89=$ac_arg -fi -rm -f core conftest.err conftest.$ac_objext - test "x$ac_cv_prog_cc_c89" != "xno" && break -done -rm -f conftest.$ac_ext -CC=$ac_save_CC - -fi -# AC_CACHE_VAL -case "x$ac_cv_prog_cc_c89" in - x) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: none needed" >&5 -$as_echo "none needed" >&6; } ;; - xno) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 -$as_echo "unsupported" >&6; } ;; - *) - CC="$CC $ac_cv_prog_cc_c89" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_prog_cc_c89" >&5 -$as_echo "$ac_cv_prog_cc_c89" >&6; } ;; -esac -if test "x$ac_cv_prog_cc_c89" != xno; then : - -fi - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a sed that does not truncate output" >&5 -$as_echo_n "checking for a sed that does not truncate output... " >&6; } -if ${ac_cv_path_SED+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ - for ac_i in 1 2 3 4 5 6 7; do - ac_script="$ac_script$as_nl$ac_script" - done - echo "$ac_script" 2>/dev/null | sed 99q >conftest.sed - { ac_script=; unset ac_script;} - if test -z "$SED"; then - ac_path_SED_found=false - # Loop through the user's path and test for each of PROGNAME-LIST - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_prog in sed gsed; do - for ac_exec_ext in '' $ac_executable_extensions; do - ac_path_SED="$as_dir/$ac_prog$ac_exec_ext" - as_fn_executable_p "$ac_path_SED" || continue -# Check for GNU ac_path_SED and select it if it is found. - # Check for GNU $ac_path_SED -case `"$ac_path_SED" --version 2>&1` in -*GNU*) - ac_cv_path_SED="$ac_path_SED" ac_path_SED_found=:;; -*) - ac_count=0 - $as_echo_n 0123456789 >"conftest.in" - while : - do - cat "conftest.in" "conftest.in" >"conftest.tmp" - mv "conftest.tmp" "conftest.in" - cp "conftest.in" "conftest.nl" - $as_echo '' >> "conftest.nl" - "$ac_path_SED" -f conftest.sed < "conftest.nl" >"conftest.out" 2>/dev/null || break - diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break - as_fn_arith $ac_count + 1 && ac_count=$as_val - if test $ac_count -gt ${ac_path_SED_max-0}; then - # Best one so far, save it but keep looking for a better one - ac_cv_path_SED="$ac_path_SED" - ac_path_SED_max=$ac_count - fi - # 10*(2^10) chars as input seems more than enough - test $ac_count -gt 10 && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out;; -esac - - $ac_path_SED_found && break 3 - done - done - done -IFS=$as_save_IFS - if test -z "$ac_cv_path_SED"; then - as_fn_error $? "no acceptable sed could be found in \$PATH" "$LINENO" 5 - fi -else - ac_cv_path_SED=$SED -fi - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_SED" >&5 -$as_echo "$ac_cv_path_SED" >&6; } - SED="$ac_cv_path_SED" - rm -f conftest.sed - -test -z "$SED" && SED=sed -Xsed="$SED -e 1s/^X//" - - - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 -$as_echo_n "checking for grep that handles long lines and -e... " >&6; } -if ${ac_cv_path_GREP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -z "$GREP"; then - ac_path_GREP_found=false - # Loop through the user's path and test for each of PROGNAME-LIST - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_prog in grep ggrep; do - for ac_exec_ext in '' $ac_executable_extensions; do - ac_path_GREP="$as_dir/$ac_prog$ac_exec_ext" - as_fn_executable_p "$ac_path_GREP" || continue -# Check for GNU ac_path_GREP and select it if it is found. - # Check for GNU $ac_path_GREP -case `"$ac_path_GREP" --version 2>&1` in -*GNU*) - ac_cv_path_GREP="$ac_path_GREP" ac_path_GREP_found=:;; -*) - ac_count=0 - $as_echo_n 0123456789 >"conftest.in" - while : - do - cat "conftest.in" "conftest.in" >"conftest.tmp" - mv "conftest.tmp" "conftest.in" - cp "conftest.in" "conftest.nl" - $as_echo 'GREP' >> "conftest.nl" - "$ac_path_GREP" -e 'GREP$' -e '-(cannot match)-' < "conftest.nl" >"conftest.out" 2>/dev/null || break - diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break - as_fn_arith $ac_count + 1 && ac_count=$as_val - if test $ac_count -gt ${ac_path_GREP_max-0}; then - # Best one so far, save it but keep looking for a better one - ac_cv_path_GREP="$ac_path_GREP" - ac_path_GREP_max=$ac_count - fi - # 10*(2^10) chars as input seems more than enough - test $ac_count -gt 10 && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out;; -esac - - $ac_path_GREP_found && break 3 - done - done - done -IFS=$as_save_IFS - if test -z "$ac_cv_path_GREP"; then - as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 - fi -else - ac_cv_path_GREP=$GREP -fi - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_GREP" >&5 -$as_echo "$ac_cv_path_GREP" >&6; } - GREP="$ac_cv_path_GREP" - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 -$as_echo_n "checking for egrep... " >&6; } -if ${ac_cv_path_EGREP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 - then ac_cv_path_EGREP="$GREP -E" - else - if test -z "$EGREP"; then - ac_path_EGREP_found=false - # Loop through the user's path and test for each of PROGNAME-LIST - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_prog in egrep; do - for ac_exec_ext in '' $ac_executable_extensions; do - ac_path_EGREP="$as_dir/$ac_prog$ac_exec_ext" - as_fn_executable_p "$ac_path_EGREP" || continue -# Check for GNU ac_path_EGREP and select it if it is found. - # Check for GNU $ac_path_EGREP -case `"$ac_path_EGREP" --version 2>&1` in -*GNU*) - ac_cv_path_EGREP="$ac_path_EGREP" ac_path_EGREP_found=:;; -*) - ac_count=0 - $as_echo_n 0123456789 >"conftest.in" - while : - do - cat "conftest.in" "conftest.in" >"conftest.tmp" - mv "conftest.tmp" "conftest.in" - cp "conftest.in" "conftest.nl" - $as_echo 'EGREP' >> "conftest.nl" - "$ac_path_EGREP" 'EGREP$' < "conftest.nl" >"conftest.out" 2>/dev/null || break - diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break - as_fn_arith $ac_count + 1 && ac_count=$as_val - if test $ac_count -gt ${ac_path_EGREP_max-0}; then - # Best one so far, save it but keep looking for a better one - ac_cv_path_EGREP="$ac_path_EGREP" - ac_path_EGREP_max=$ac_count - fi - # 10*(2^10) chars as input seems more than enough - test $ac_count -gt 10 && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out;; -esac - - $ac_path_EGREP_found && break 3 - done - done - done -IFS=$as_save_IFS - if test -z "$ac_cv_path_EGREP"; then - as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 - fi -else - ac_cv_path_EGREP=$EGREP -fi - - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_EGREP" >&5 -$as_echo "$ac_cv_path_EGREP" >&6; } - EGREP="$ac_cv_path_EGREP" - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for fgrep" >&5 -$as_echo_n "checking for fgrep... " >&6; } -if ${ac_cv_path_FGREP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if echo 'ab*c' | $GREP -F 'ab*c' >/dev/null 2>&1 - then ac_cv_path_FGREP="$GREP -F" - else - if test -z "$FGREP"; then - ac_path_FGREP_found=false - # Loop through the user's path and test for each of PROGNAME-LIST - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH$PATH_SEPARATOR/usr/xpg4/bin -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_prog in fgrep; do - for ac_exec_ext in '' $ac_executable_extensions; do - ac_path_FGREP="$as_dir/$ac_prog$ac_exec_ext" - as_fn_executable_p "$ac_path_FGREP" || continue -# Check for GNU ac_path_FGREP and select it if it is found. - # Check for GNU $ac_path_FGREP -case `"$ac_path_FGREP" --version 2>&1` in -*GNU*) - ac_cv_path_FGREP="$ac_path_FGREP" ac_path_FGREP_found=:;; -*) - ac_count=0 - $as_echo_n 0123456789 >"conftest.in" - while : - do - cat "conftest.in" "conftest.in" >"conftest.tmp" - mv "conftest.tmp" "conftest.in" - cp "conftest.in" "conftest.nl" - $as_echo 'FGREP' >> "conftest.nl" - "$ac_path_FGREP" FGREP < "conftest.nl" >"conftest.out" 2>/dev/null || break - diff "conftest.out" "conftest.nl" >/dev/null 2>&1 || break - as_fn_arith $ac_count + 1 && ac_count=$as_val - if test $ac_count -gt ${ac_path_FGREP_max-0}; then - # Best one so far, save it but keep looking for a better one - ac_cv_path_FGREP="$ac_path_FGREP" - ac_path_FGREP_max=$ac_count - fi - # 10*(2^10) chars as input seems more than enough - test $ac_count -gt 10 && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out;; -esac - - $ac_path_FGREP_found && break 3 - done - done - done -IFS=$as_save_IFS - if test -z "$ac_cv_path_FGREP"; then - as_fn_error $? "no acceptable fgrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 - fi -else - ac_cv_path_FGREP=$FGREP -fi - - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_FGREP" >&5 -$as_echo "$ac_cv_path_FGREP" >&6; } - FGREP="$ac_cv_path_FGREP" - - -test -z "$GREP" && GREP=grep - - - - - - - - - - - - - - - - - - - -# Check whether --with-gnu-ld was given. -if test "${with_gnu_ld+set}" = set; then : - withval=$with_gnu_ld; test no = "$withval" || with_gnu_ld=yes -else - with_gnu_ld=no -fi - -ac_prog=ld -if test yes = "$GCC"; then - # Check if gcc -print-prog-name=ld gives a path. - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ld used by $CC" >&5 -$as_echo_n "checking for ld used by $CC... " >&6; } - case $host in - *-*-mingw*) - # gcc leaves a trailing carriage return, which upsets mingw - ac_prog=`($CC -print-prog-name=ld) 2>&5 | tr -d '\015'` ;; - *) - ac_prog=`($CC -print-prog-name=ld) 2>&5` ;; - esac - case $ac_prog in - # Accept absolute paths. - [\\/]* | ?:[\\/]*) - re_direlt='/[^/][^/]*/\.\./' - # Canonicalize the pathname of ld - ac_prog=`$ECHO "$ac_prog"| $SED 's%\\\\%/%g'` - while $ECHO "$ac_prog" | $GREP "$re_direlt" > /dev/null 2>&1; do - ac_prog=`$ECHO $ac_prog| $SED "s%$re_direlt%/%"` - done - test -z "$LD" && LD=$ac_prog - ;; - "") - # If it fails, then pretend we aren't using GCC. - ac_prog=ld - ;; - *) - # If it is relative, then search for the first ld in PATH. - with_gnu_ld=unknown - ;; - esac -elif test yes = "$with_gnu_ld"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for GNU ld" >&5 -$as_echo_n "checking for GNU ld... " >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for non-GNU ld" >&5 -$as_echo_n "checking for non-GNU ld... " >&6; } -fi -if ${lt_cv_path_LD+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -z "$LD"; then - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR - for ac_dir in $PATH; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - if test -f "$ac_dir/$ac_prog" || test -f "$ac_dir/$ac_prog$ac_exeext"; then - lt_cv_path_LD=$ac_dir/$ac_prog - # Check to see if the program is GNU ld. I'd rather use --version, - # but apparently some variants of GNU ld only accept -v. - # Break only if it was the GNU/non-GNU ld that we prefer. - case `"$lt_cv_path_LD" -v 2>&1 &5 -$as_echo "$LD" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi -test -z "$LD" && as_fn_error $? "no acceptable ld found in \$PATH" "$LINENO" 5 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the linker ($LD) is GNU ld" >&5 -$as_echo_n "checking if the linker ($LD) is GNU ld... " >&6; } -if ${lt_cv_prog_gnu_ld+:} false; then : - $as_echo_n "(cached) " >&6 -else - # I'd rather use --version here, but apparently some GNU lds only accept -v. -case `$LD -v 2>&1 &5 -$as_echo "$lt_cv_prog_gnu_ld" >&6; } -with_gnu_ld=$lt_cv_prog_gnu_ld - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for BSD- or MS-compatible name lister (nm)" >&5 -$as_echo_n "checking for BSD- or MS-compatible name lister (nm)... " >&6; } -if ${lt_cv_path_NM+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$NM"; then - # Let the user override the test. - lt_cv_path_NM=$NM -else - lt_nm_to_check=${ac_tool_prefix}nm - if test -n "$ac_tool_prefix" && test "$build" = "$host"; then - lt_nm_to_check="$lt_nm_to_check nm" - fi - for lt_tmp_nm in $lt_nm_to_check; do - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR - for ac_dir in $PATH /usr/ccs/bin/elf /usr/ccs/bin /usr/ucb /bin; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - tmp_nm=$ac_dir/$lt_tmp_nm - if test -f "$tmp_nm" || test -f "$tmp_nm$ac_exeext"; then - # Check to see if the nm accepts a BSD-compat flag. - # Adding the 'sed 1q' prevents false positives on HP-UX, which says: - # nm: unknown option "B" ignored - # Tru64's nm complains that /dev/null is an invalid object file - # MSYS converts /dev/null to NUL, MinGW nm treats NUL as empty - case $build_os in - mingw*) lt_bad_file=conftest.nm/nofile ;; - *) lt_bad_file=/dev/null ;; - esac - case `"$tmp_nm" -B $lt_bad_file 2>&1 | sed '1q'` in - *$lt_bad_file* | *'Invalid file or object type'*) - lt_cv_path_NM="$tmp_nm -B" - break 2 - ;; - *) - case `"$tmp_nm" -p /dev/null 2>&1 | sed '1q'` in - */dev/null*) - lt_cv_path_NM="$tmp_nm -p" - break 2 - ;; - *) - lt_cv_path_NM=${lt_cv_path_NM="$tmp_nm"} # keep the first match, but - continue # so that we can try to find one that supports BSD flags - ;; - esac - ;; - esac - fi - done - IFS=$lt_save_ifs - done - : ${lt_cv_path_NM=no} -fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_path_NM" >&5 -$as_echo "$lt_cv_path_NM" >&6; } -if test no != "$lt_cv_path_NM"; then - NM=$lt_cv_path_NM -else - # Didn't find any BSD compatible name lister, look for dumpbin. - if test -n "$DUMPBIN"; then : - # Let the user override the test. - else - if test -n "$ac_tool_prefix"; then - for ac_prog in dumpbin "link -dump" - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_DUMPBIN+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$DUMPBIN"; then - ac_cv_prog_DUMPBIN="$DUMPBIN" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_DUMPBIN="$ac_tool_prefix$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -DUMPBIN=$ac_cv_prog_DUMPBIN -if test -n "$DUMPBIN"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DUMPBIN" >&5 -$as_echo "$DUMPBIN" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$DUMPBIN" && break - done -fi -if test -z "$DUMPBIN"; then - ac_ct_DUMPBIN=$DUMPBIN - for ac_prog in dumpbin "link -dump" -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_DUMPBIN+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_DUMPBIN"; then - ac_cv_prog_ac_ct_DUMPBIN="$ac_ct_DUMPBIN" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_DUMPBIN="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_DUMPBIN=$ac_cv_prog_ac_ct_DUMPBIN -if test -n "$ac_ct_DUMPBIN"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DUMPBIN" >&5 -$as_echo "$ac_ct_DUMPBIN" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$ac_ct_DUMPBIN" && break -done - - if test "x$ac_ct_DUMPBIN" = x; then - DUMPBIN=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - DUMPBIN=$ac_ct_DUMPBIN - fi -fi - - case `$DUMPBIN -symbols -headers /dev/null 2>&1 | sed '1q'` in - *COFF*) - DUMPBIN="$DUMPBIN -symbols -headers" - ;; - *) - DUMPBIN=: - ;; - esac - fi - - if test : != "$DUMPBIN"; then - NM=$DUMPBIN - fi -fi -test -z "$NM" && NM=nm - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the name lister ($NM) interface" >&5 -$as_echo_n "checking the name lister ($NM) interface... " >&6; } -if ${lt_cv_nm_interface+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_nm_interface="BSD nm" - echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:$LINENO: $ac_compile\"" >&5) - (eval "$ac_compile" 2>conftest.err) - cat conftest.err >&5 - (eval echo "\"\$as_me:$LINENO: $NM \\\"conftest.$ac_objext\\\"\"" >&5) - (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) - cat conftest.err >&5 - (eval echo "\"\$as_me:$LINENO: output\"" >&5) - cat conftest.out >&5 - if $GREP 'External.*some_variable' conftest.out > /dev/null; then - lt_cv_nm_interface="MS dumpbin" - fi - rm -f conftest* -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_nm_interface" >&5 -$as_echo "$lt_cv_nm_interface" >&6; } - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ln -s works" >&5 -$as_echo_n "checking whether ln -s works... " >&6; } -LN_S=$as_ln_s -if test "$LN_S" = "ln -s"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no, using $LN_S" >&5 -$as_echo "no, using $LN_S" >&6; } -fi - -# find the maximum length of command line arguments -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking the maximum length of command line arguments" >&5 -$as_echo_n "checking the maximum length of command line arguments... " >&6; } -if ${lt_cv_sys_max_cmd_len+:} false; then : - $as_echo_n "(cached) " >&6 -else - i=0 - teststring=ABCD - - case $build_os in - msdosdjgpp*) - # On DJGPP, this test can blow up pretty badly due to problems in libc - # (any single argument exceeding 2000 bytes causes a buffer overrun - # during glob expansion). Even if it were fixed, the result of this - # check would be larger than it should be. - lt_cv_sys_max_cmd_len=12288; # 12K is about right - ;; - - gnu*) - # Under GNU Hurd, this test is not required because there is - # no limit to the length of command line arguments. - # Libtool will interpret -1 as no limit whatsoever - lt_cv_sys_max_cmd_len=-1; - ;; - - cygwin* | mingw* | cegcc*) - # On Win9x/ME, this test blows up -- it succeeds, but takes - # about 5 minutes as the teststring grows exponentially. - # Worse, since 9x/ME are not pre-emptively multitasking, - # you end up with a "frozen" computer, even though with patience - # the test eventually succeeds (with a max line length of 256k). - # Instead, let's just punt: use the minimum linelength reported by - # all of the supported platforms: 8192 (on NT/2K/XP). - lt_cv_sys_max_cmd_len=8192; - ;; - - mint*) - # On MiNT this can take a long time and run out of memory. - lt_cv_sys_max_cmd_len=8192; - ;; - - amigaos*) - # On AmigaOS with pdksh, this test takes hours, literally. - # So we just punt and use a minimum line length of 8192. - lt_cv_sys_max_cmd_len=8192; - ;; - - bitrig* | darwin* | dragonfly* | freebsd* | netbsd* | openbsd*) - # This has been around since 386BSD, at least. Likely further. - if test -x /sbin/sysctl; then - lt_cv_sys_max_cmd_len=`/sbin/sysctl -n kern.argmax` - elif test -x /usr/sbin/sysctl; then - lt_cv_sys_max_cmd_len=`/usr/sbin/sysctl -n kern.argmax` - else - lt_cv_sys_max_cmd_len=65536 # usable default for all BSDs - fi - # And add a safety zone - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` - ;; - - interix*) - # We know the value 262144 and hardcode it with a safety zone (like BSD) - lt_cv_sys_max_cmd_len=196608 - ;; - - os2*) - # The test takes a long time on OS/2. - lt_cv_sys_max_cmd_len=8192 - ;; - - osf*) - # Dr. Hans Ekkehard Plesser reports seeing a kernel panic running configure - # due to this test when exec_disable_arg_limit is 1 on Tru64. It is not - # nice to cause kernel panics so lets avoid the loop below. - # First set a reasonable default. - lt_cv_sys_max_cmd_len=16384 - # - if test -x /sbin/sysconfig; then - case `/sbin/sysconfig -q proc exec_disable_arg_limit` in - *1*) lt_cv_sys_max_cmd_len=-1 ;; - esac - fi - ;; - sco3.2v5*) - lt_cv_sys_max_cmd_len=102400 - ;; - sysv5* | sco5v6* | sysv4.2uw2*) - kargmax=`grep ARG_MAX /etc/conf/cf.d/stune 2>/dev/null` - if test -n "$kargmax"; then - lt_cv_sys_max_cmd_len=`echo $kargmax | sed 's/.*[ ]//'` - else - lt_cv_sys_max_cmd_len=32768 - fi - ;; - *) - lt_cv_sys_max_cmd_len=`(getconf ARG_MAX) 2> /dev/null` - if test -n "$lt_cv_sys_max_cmd_len" && \ - test undefined != "$lt_cv_sys_max_cmd_len"; then - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 4` - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \* 3` - else - # Make teststring a little bigger before we do anything with it. - # a 1K string should be a reasonable start. - for i in 1 2 3 4 5 6 7 8; do - teststring=$teststring$teststring - done - SHELL=${SHELL-${CONFIG_SHELL-/bin/sh}} - # If test is not a shell built-in, we'll probably end up computing a - # maximum length that is only half of the actual maximum length, but - # we can't tell. - while { test X`env echo "$teststring$teststring" 2>/dev/null` \ - = "X$teststring$teststring"; } >/dev/null 2>&1 && - test 17 != "$i" # 1/2 MB should be enough - do - i=`expr $i + 1` - teststring=$teststring$teststring - done - # Only check the string length outside the loop. - lt_cv_sys_max_cmd_len=`expr "X$teststring" : ".*" 2>&1` - teststring= - # Add a significant safety factor because C++ compilers can tack on - # massive amounts of additional arguments before passing them to the - # linker. It appears as though 1/2 is a usable value. - lt_cv_sys_max_cmd_len=`expr $lt_cv_sys_max_cmd_len \/ 2` - fi - ;; - esac - -fi - -if test -n "$lt_cv_sys_max_cmd_len"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_sys_max_cmd_len" >&5 -$as_echo "$lt_cv_sys_max_cmd_len" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: none" >&5 -$as_echo "none" >&6; } -fi -max_cmd_len=$lt_cv_sys_max_cmd_len - - - - - - -: ${CP="cp -f"} -: ${MV="mv -f"} -: ${RM="rm -f"} - -if ( (MAIL=60; unset MAIL) || exit) >/dev/null 2>&1; then - lt_unset=unset -else - lt_unset=false -fi - - - - - -# test EBCDIC or ASCII -case `echo X|tr X '\101'` in - A) # ASCII based system - # \n is not interpreted correctly by Solaris 8 /usr/ucb/tr - lt_SP2NL='tr \040 \012' - lt_NL2SP='tr \015\012 \040\040' - ;; - *) # EBCDIC based system - lt_SP2NL='tr \100 \n' - lt_NL2SP='tr \r\n \100\100' - ;; -esac - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to convert $build file names to $host format" >&5 -$as_echo_n "checking how to convert $build file names to $host format... " >&6; } -if ${lt_cv_to_host_file_cmd+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $host in - *-*-mingw* ) - case $build in - *-*-mingw* ) # actually msys - lt_cv_to_host_file_cmd=func_convert_file_msys_to_w32 - ;; - *-*-cygwin* ) - lt_cv_to_host_file_cmd=func_convert_file_cygwin_to_w32 - ;; - * ) # otherwise, assume *nix - lt_cv_to_host_file_cmd=func_convert_file_nix_to_w32 - ;; - esac - ;; - *-*-cygwin* ) - case $build in - *-*-mingw* ) # actually msys - lt_cv_to_host_file_cmd=func_convert_file_msys_to_cygwin - ;; - *-*-cygwin* ) - lt_cv_to_host_file_cmd=func_convert_file_noop - ;; - * ) # otherwise, assume *nix - lt_cv_to_host_file_cmd=func_convert_file_nix_to_cygwin - ;; - esac - ;; - * ) # unhandled hosts (and "normal" native builds) - lt_cv_to_host_file_cmd=func_convert_file_noop - ;; -esac - -fi - -to_host_file_cmd=$lt_cv_to_host_file_cmd -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_to_host_file_cmd" >&5 -$as_echo "$lt_cv_to_host_file_cmd" >&6; } - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to convert $build file names to toolchain format" >&5 -$as_echo_n "checking how to convert $build file names to toolchain format... " >&6; } -if ${lt_cv_to_tool_file_cmd+:} false; then : - $as_echo_n "(cached) " >&6 -else - #assume ordinary cross tools, or native build. -lt_cv_to_tool_file_cmd=func_convert_file_noop -case $host in - *-*-mingw* ) - case $build in - *-*-mingw* ) # actually msys - lt_cv_to_tool_file_cmd=func_convert_file_msys_to_w32 - ;; - esac - ;; -esac - -fi - -to_tool_file_cmd=$lt_cv_to_tool_file_cmd -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_to_tool_file_cmd" >&5 -$as_echo "$lt_cv_to_tool_file_cmd" >&6; } - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $LD option to reload object files" >&5 -$as_echo_n "checking for $LD option to reload object files... " >&6; } -if ${lt_cv_ld_reload_flag+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_ld_reload_flag='-r' -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_reload_flag" >&5 -$as_echo "$lt_cv_ld_reload_flag" >&6; } -reload_flag=$lt_cv_ld_reload_flag -case $reload_flag in -"" | " "*) ;; -*) reload_flag=" $reload_flag" ;; -esac -reload_cmds='$LD$reload_flag -o $output$reload_objs' -case $host_os in - cygwin* | mingw* | pw32* | cegcc*) - if test yes != "$GCC"; then - reload_cmds=false - fi - ;; - darwin*) - if test yes = "$GCC"; then - reload_cmds='$LTCC $LTCFLAGS -nostdlib $wl-r -o $output$reload_objs' - else - reload_cmds='$LD$reload_flag -o $output$reload_objs' - fi - ;; -esac - - - - - - - - - -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}objdump", so it can be a program name with args. -set dummy ${ac_tool_prefix}objdump; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_OBJDUMP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$OBJDUMP"; then - ac_cv_prog_OBJDUMP="$OBJDUMP" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_OBJDUMP="${ac_tool_prefix}objdump" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -OBJDUMP=$ac_cv_prog_OBJDUMP -if test -n "$OBJDUMP"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OBJDUMP" >&5 -$as_echo "$OBJDUMP" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_OBJDUMP"; then - ac_ct_OBJDUMP=$OBJDUMP - # Extract the first word of "objdump", so it can be a program name with args. -set dummy objdump; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_OBJDUMP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_OBJDUMP"; then - ac_cv_prog_ac_ct_OBJDUMP="$ac_ct_OBJDUMP" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_OBJDUMP="objdump" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_OBJDUMP=$ac_cv_prog_ac_ct_OBJDUMP -if test -n "$ac_ct_OBJDUMP"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OBJDUMP" >&5 -$as_echo "$ac_ct_OBJDUMP" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_OBJDUMP" = x; then - OBJDUMP="false" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - OBJDUMP=$ac_ct_OBJDUMP - fi -else - OBJDUMP="$ac_cv_prog_OBJDUMP" -fi - -test -z "$OBJDUMP" && OBJDUMP=objdump - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to recognize dependent libraries" >&5 -$as_echo_n "checking how to recognize dependent libraries... " >&6; } -if ${lt_cv_deplibs_check_method+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_file_magic_cmd='$MAGIC_CMD' -lt_cv_file_magic_test_file= -lt_cv_deplibs_check_method='unknown' -# Need to set the preceding variable on all platforms that support -# interlibrary dependencies. -# 'none' -- dependencies not supported. -# 'unknown' -- same as none, but documents that we really don't know. -# 'pass_all' -- all dependencies passed with no checks. -# 'test_compile' -- check by making test program. -# 'file_magic [[regex]]' -- check by looking for files in library path -# that responds to the $file_magic_cmd with a given extended regex. -# If you have 'file' or equivalent on your system and you're not sure -# whether 'pass_all' will *always* work, you probably want this one. - -case $host_os in -aix[4-9]*) - lt_cv_deplibs_check_method=pass_all - ;; - -beos*) - lt_cv_deplibs_check_method=pass_all - ;; - -bsdi[45]*) - lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib)' - lt_cv_file_magic_cmd='/usr/bin/file -L' - lt_cv_file_magic_test_file=/shlib/libc.so - ;; - -cygwin*) - # func_win32_libid is a shell function defined in ltmain.sh - lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' - lt_cv_file_magic_cmd='func_win32_libid' - ;; - -mingw* | pw32*) - # Base MSYS/MinGW do not provide the 'file' command needed by - # func_win32_libid shell function, so use a weaker test based on 'objdump', - # unless we find 'file', for example because we are cross-compiling. - if ( file / ) >/dev/null 2>&1; then - lt_cv_deplibs_check_method='file_magic ^x86 archive import|^x86 DLL' - lt_cv_file_magic_cmd='func_win32_libid' - else - # Keep this pattern in sync with the one in func_win32_libid. - lt_cv_deplibs_check_method='file_magic file format (pei*-i386(.*architecture: i386)?|pe-arm-wince|pe-x86-64)' - lt_cv_file_magic_cmd='$OBJDUMP -f' - fi - ;; - -cegcc*) - # use the weaker test based on 'objdump'. See mingw*. - lt_cv_deplibs_check_method='file_magic file format pe-arm-.*little(.*architecture: arm)?' - lt_cv_file_magic_cmd='$OBJDUMP -f' - ;; - -darwin* | rhapsody*) - lt_cv_deplibs_check_method=pass_all - ;; - -freebsd* | dragonfly*) - if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then - case $host_cpu in - i*86 ) - # Not sure whether the presence of OpenBSD here was a mistake. - # Let's accept both of them until this is cleared up. - lt_cv_deplibs_check_method='file_magic (FreeBSD|OpenBSD|DragonFly)/i[3-9]86 (compact )?demand paged shared library' - lt_cv_file_magic_cmd=/usr/bin/file - lt_cv_file_magic_test_file=`echo /usr/lib/libc.so.*` - ;; - esac - else - lt_cv_deplibs_check_method=pass_all - fi - ;; - -haiku*) - lt_cv_deplibs_check_method=pass_all - ;; - -hpux10.20* | hpux11*) - lt_cv_file_magic_cmd=/usr/bin/file - case $host_cpu in - ia64*) - lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF-[0-9][0-9]) shared object file - IA64' - lt_cv_file_magic_test_file=/usr/lib/hpux32/libc.so - ;; - hppa*64*) - lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|ELF[ -][0-9][0-9])(-bit)?( [LM]SB)? shared object( file)?[, -]* PA-RISC [0-9]\.[0-9]' - lt_cv_file_magic_test_file=/usr/lib/pa20_64/libc.sl - ;; - *) - lt_cv_deplibs_check_method='file_magic (s[0-9][0-9][0-9]|PA-RISC[0-9]\.[0-9]) shared library' - lt_cv_file_magic_test_file=/usr/lib/libc.sl - ;; - esac - ;; - -interix[3-9]*) - # PIC code is broken on Interix 3.x, that's why |\.a not |_pic\.a here - lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so|\.a)$' - ;; - -irix5* | irix6* | nonstopux*) - case $LD in - *-32|*"-32 ") libmagic=32-bit;; - *-n32|*"-n32 ") libmagic=N32;; - *-64|*"-64 ") libmagic=64-bit;; - *) libmagic=never-match;; - esac - lt_cv_deplibs_check_method=pass_all - ;; - -# This must be glibc/ELF. -linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - lt_cv_deplibs_check_method=pass_all - ;; - -netbsd* | netbsdelf*-gnu) - if echo __ELF__ | $CC -E - | $GREP __ELF__ > /dev/null; then - lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|_pic\.a)$' - else - lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so|_pic\.a)$' - fi - ;; - -newos6*) - lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (executable|dynamic lib)' - lt_cv_file_magic_cmd=/usr/bin/file - lt_cv_file_magic_test_file=/usr/lib/libnls.so - ;; - -*nto* | *qnx*) - lt_cv_deplibs_check_method=pass_all - ;; - -openbsd* | bitrig*) - if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`"; then - lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|\.so|_pic\.a)$' - else - lt_cv_deplibs_check_method='match_pattern /lib[^/]+(\.so\.[0-9]+\.[0-9]+|_pic\.a)$' - fi - ;; - -osf3* | osf4* | osf5*) - lt_cv_deplibs_check_method=pass_all - ;; - -rdos*) - lt_cv_deplibs_check_method=pass_all - ;; - -solaris*) - lt_cv_deplibs_check_method=pass_all - ;; - -sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) - lt_cv_deplibs_check_method=pass_all - ;; - -sysv4 | sysv4.3*) - case $host_vendor in - motorola) - lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [ML]SB (shared object|dynamic lib) M[0-9][0-9]* Version [0-9]' - lt_cv_file_magic_test_file=`echo /usr/lib/libc.so*` - ;; - ncr) - lt_cv_deplibs_check_method=pass_all - ;; - sequent) - lt_cv_file_magic_cmd='/bin/file' - lt_cv_deplibs_check_method='file_magic ELF [0-9][0-9]*-bit [LM]SB (shared object|dynamic lib )' - ;; - sni) - lt_cv_file_magic_cmd='/bin/file' - lt_cv_deplibs_check_method="file_magic ELF [0-9][0-9]*-bit [LM]SB dynamic lib" - lt_cv_file_magic_test_file=/lib/libc.so - ;; - siemens) - lt_cv_deplibs_check_method=pass_all - ;; - pc) - lt_cv_deplibs_check_method=pass_all - ;; - esac - ;; - -tpf*) - lt_cv_deplibs_check_method=pass_all - ;; -os2*) - lt_cv_deplibs_check_method=pass_all - ;; -esac - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_deplibs_check_method" >&5 -$as_echo "$lt_cv_deplibs_check_method" >&6; } - -file_magic_glob= -want_nocaseglob=no -if test "$build" = "$host"; then - case $host_os in - mingw* | pw32*) - if ( shopt | grep nocaseglob ) >/dev/null 2>&1; then - want_nocaseglob=yes - else - file_magic_glob=`echo aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ | $SED -e "s/\(..\)/s\/[\1]\/[\1]\/g;/g"` - fi - ;; - esac -fi - -file_magic_cmd=$lt_cv_file_magic_cmd -deplibs_check_method=$lt_cv_deplibs_check_method -test -z "$deplibs_check_method" && deplibs_check_method=unknown - - - - - - - - - - - - - - - - - - - - - - -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}dlltool", so it can be a program name with args. -set dummy ${ac_tool_prefix}dlltool; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_DLLTOOL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$DLLTOOL"; then - ac_cv_prog_DLLTOOL="$DLLTOOL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_DLLTOOL="${ac_tool_prefix}dlltool" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -DLLTOOL=$ac_cv_prog_DLLTOOL -if test -n "$DLLTOOL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DLLTOOL" >&5 -$as_echo "$DLLTOOL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_DLLTOOL"; then - ac_ct_DLLTOOL=$DLLTOOL - # Extract the first word of "dlltool", so it can be a program name with args. -set dummy dlltool; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_DLLTOOL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_DLLTOOL"; then - ac_cv_prog_ac_ct_DLLTOOL="$ac_ct_DLLTOOL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_DLLTOOL="dlltool" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_DLLTOOL=$ac_cv_prog_ac_ct_DLLTOOL -if test -n "$ac_ct_DLLTOOL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DLLTOOL" >&5 -$as_echo "$ac_ct_DLLTOOL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_DLLTOOL" = x; then - DLLTOOL="false" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - DLLTOOL=$ac_ct_DLLTOOL - fi -else - DLLTOOL="$ac_cv_prog_DLLTOOL" -fi - -test -z "$DLLTOOL" && DLLTOOL=dlltool - - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to associate runtime and link libraries" >&5 -$as_echo_n "checking how to associate runtime and link libraries... " >&6; } -if ${lt_cv_sharedlib_from_linklib_cmd+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_sharedlib_from_linklib_cmd='unknown' - -case $host_os in -cygwin* | mingw* | pw32* | cegcc*) - # two different shell functions defined in ltmain.sh; - # decide which one to use based on capabilities of $DLLTOOL - case `$DLLTOOL --help 2>&1` in - *--identify-strict*) - lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib - ;; - *) - lt_cv_sharedlib_from_linklib_cmd=func_cygming_dll_for_implib_fallback - ;; - esac - ;; -*) - # fallback: assume linklib IS sharedlib - lt_cv_sharedlib_from_linklib_cmd=$ECHO - ;; -esac - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_sharedlib_from_linklib_cmd" >&5 -$as_echo "$lt_cv_sharedlib_from_linklib_cmd" >&6; } -sharedlib_from_linklib_cmd=$lt_cv_sharedlib_from_linklib_cmd -test -z "$sharedlib_from_linklib_cmd" && sharedlib_from_linklib_cmd=$ECHO - - - - - - - - -if test -n "$ac_tool_prefix"; then - for ac_prog in ar - do - # Extract the first word of "$ac_tool_prefix$ac_prog", so it can be a program name with args. -set dummy $ac_tool_prefix$ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_AR+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$AR"; then - ac_cv_prog_AR="$AR" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_AR="$ac_tool_prefix$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -AR=$ac_cv_prog_AR -if test -n "$AR"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AR" >&5 -$as_echo "$AR" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$AR" && break - done -fi -if test -z "$AR"; then - ac_ct_AR=$AR - for ac_prog in ar -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_AR+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_AR"; then - ac_cv_prog_ac_ct_AR="$ac_ct_AR" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_AR="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_AR=$ac_cv_prog_ac_ct_AR -if test -n "$ac_ct_AR"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_AR" >&5 -$as_echo "$ac_ct_AR" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$ac_ct_AR" && break -done - - if test "x$ac_ct_AR" = x; then - AR="false" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - AR=$ac_ct_AR - fi -fi - -: ${AR=ar} -: ${AR_FLAGS=cr} - - - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for archiver @FILE support" >&5 -$as_echo_n "checking for archiver @FILE support... " >&6; } -if ${lt_cv_ar_at_file+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_ar_at_file=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - echo conftest.$ac_objext > conftest.lst - lt_ar_try='$AR $AR_FLAGS libconftest.a @conftest.lst >&5' - { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$lt_ar_try\""; } >&5 - (eval $lt_ar_try) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - if test 0 -eq "$ac_status"; then - # Ensure the archiver fails upon bogus file names. - rm -f conftest.$ac_objext libconftest.a - { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$lt_ar_try\""; } >&5 - (eval $lt_ar_try) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - if test 0 -ne "$ac_status"; then - lt_cv_ar_at_file=@ - fi - fi - rm -f conftest.* libconftest.a - -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ar_at_file" >&5 -$as_echo "$lt_cv_ar_at_file" >&6; } - -if test no = "$lt_cv_ar_at_file"; then - archiver_list_spec= -else - archiver_list_spec=$lt_cv_ar_at_file -fi - - - - - - - -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}strip", so it can be a program name with args. -set dummy ${ac_tool_prefix}strip; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_STRIP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$STRIP"; then - ac_cv_prog_STRIP="$STRIP" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_STRIP="${ac_tool_prefix}strip" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -STRIP=$ac_cv_prog_STRIP -if test -n "$STRIP"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STRIP" >&5 -$as_echo "$STRIP" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_STRIP"; then - ac_ct_STRIP=$STRIP - # Extract the first word of "strip", so it can be a program name with args. -set dummy strip; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_STRIP+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_STRIP"; then - ac_cv_prog_ac_ct_STRIP="$ac_ct_STRIP" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_STRIP="strip" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_STRIP=$ac_cv_prog_ac_ct_STRIP -if test -n "$ac_ct_STRIP"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_STRIP" >&5 -$as_echo "$ac_ct_STRIP" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_STRIP" = x; then - STRIP=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - STRIP=$ac_ct_STRIP - fi -else - STRIP="$ac_cv_prog_STRIP" -fi - -test -z "$STRIP" && STRIP=: - - - - - - -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}ranlib", so it can be a program name with args. -set dummy ${ac_tool_prefix}ranlib; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_RANLIB+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$RANLIB"; then - ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_RANLIB="${ac_tool_prefix}ranlib" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -RANLIB=$ac_cv_prog_RANLIB -if test -n "$RANLIB"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $RANLIB" >&5 -$as_echo "$RANLIB" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_RANLIB"; then - ac_ct_RANLIB=$RANLIB - # Extract the first word of "ranlib", so it can be a program name with args. -set dummy ranlib; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_RANLIB+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_RANLIB"; then - ac_cv_prog_ac_ct_RANLIB="$ac_ct_RANLIB" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_RANLIB="ranlib" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_RANLIB=$ac_cv_prog_ac_ct_RANLIB -if test -n "$ac_ct_RANLIB"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_RANLIB" >&5 -$as_echo "$ac_ct_RANLIB" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_RANLIB" = x; then - RANLIB=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - RANLIB=$ac_ct_RANLIB - fi -else - RANLIB="$ac_cv_prog_RANLIB" -fi - -test -z "$RANLIB" && RANLIB=: - - - - - - -# Determine commands to create old-style static archives. -old_archive_cmds='$AR $AR_FLAGS $oldlib$oldobjs' -old_postinstall_cmds='chmod 644 $oldlib' -old_postuninstall_cmds= - -if test -n "$RANLIB"; then - case $host_os in - bitrig* | openbsd*) - old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB -t \$tool_oldlib" - ;; - *) - old_postinstall_cmds="$old_postinstall_cmds~\$RANLIB \$tool_oldlib" - ;; - esac - old_archive_cmds="$old_archive_cmds~\$RANLIB \$tool_oldlib" -fi - -case $host_os in - darwin*) - lock_old_archive_extraction=yes ;; - *) - lock_old_archive_extraction=no ;; -esac - - - - - - - - - - - - - - - - - - - - - -for ac_prog in gawk mawk nawk awk -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_AWK+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$AWK"; then - ac_cv_prog_AWK="$AWK" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_AWK="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -AWK=$ac_cv_prog_AWK -if test -n "$AWK"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 -$as_echo "$AWK" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$AWK" && break -done - - - - - - - - - - - - - - - - - - - -# If no C compiler was specified, use CC. -LTCC=${LTCC-"$CC"} - -# If no C compiler flags were specified, use CFLAGS. -LTCFLAGS=${LTCFLAGS-"$CFLAGS"} - -# Allow CC to be a program name with arguments. -compiler=$CC - - -# Check for command to grab the raw symbol name followed by C symbol from nm. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking command to parse $NM output from $compiler object" >&5 -$as_echo_n "checking command to parse $NM output from $compiler object... " >&6; } -if ${lt_cv_sys_global_symbol_pipe+:} false; then : - $as_echo_n "(cached) " >&6 -else - -# These are sane defaults that work on at least a few old systems. -# [They come from Ultrix. What could be older than Ultrix?!! ;)] - -# Character class describing NM global symbol codes. -symcode='[BCDEGRST]' - -# Regexp to match symbols that can be accessed directly from C. -sympat='\([_A-Za-z][_A-Za-z0-9]*\)' - -# Define system-specific variables. -case $host_os in -aix*) - symcode='[BCDT]' - ;; -cygwin* | mingw* | pw32* | cegcc*) - symcode='[ABCDGISTW]' - ;; -hpux*) - if test ia64 = "$host_cpu"; then - symcode='[ABCDEGRST]' - fi - ;; -irix* | nonstopux*) - symcode='[BCDEGRST]' - ;; -osf*) - symcode='[BCDEGQRST]' - ;; -solaris*) - symcode='[BDRT]' - ;; -sco3.2v5*) - symcode='[DT]' - ;; -sysv4.2uw2*) - symcode='[DT]' - ;; -sysv5* | sco5v6* | unixware* | OpenUNIX*) - symcode='[ABDT]' - ;; -sysv4) - symcode='[DFNSTU]' - ;; -esac - -# If we're using GNU nm, then use its standard symbol codes. -case `$NM -V 2>&1` in -*GNU* | *'with BFD'*) - symcode='[ABCDGIRSTW]' ;; -esac - -if test "$lt_cv_nm_interface" = "MS dumpbin"; then - # Gets list of data symbols to import. - lt_cv_sys_global_symbol_to_import="sed -n -e 's/^I .* \(.*\)$/\1/p'" - # Adjust the below global symbol transforms to fixup imported variables. - lt_cdecl_hook=" -e 's/^I .* \(.*\)$/extern __declspec(dllimport) char \1;/p'" - lt_c_name_hook=" -e 's/^I .* \(.*\)$/ {\"\1\", (void *) 0},/p'" - lt_c_name_lib_hook="\ - -e 's/^I .* \(lib.*\)$/ {\"\1\", (void *) 0},/p'\ - -e 's/^I .* \(.*\)$/ {\"lib\1\", (void *) 0},/p'" -else - # Disable hooks by default. - lt_cv_sys_global_symbol_to_import= - lt_cdecl_hook= - lt_c_name_hook= - lt_c_name_lib_hook= -fi - -# Transform an extracted symbol line into a proper C declaration. -# Some systems (esp. on ia64) link data and code symbols differently, -# so use this general approach. -lt_cv_sys_global_symbol_to_cdecl="sed -n"\ -$lt_cdecl_hook\ -" -e 's/^T .* \(.*\)$/extern int \1();/p'"\ -" -e 's/^$symcode$symcode* .* \(.*\)$/extern char \1;/p'" - -# Transform an extracted symbol line into symbol name and symbol address -lt_cv_sys_global_symbol_to_c_name_address="sed -n"\ -$lt_c_name_hook\ -" -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ -" -e 's/^$symcode$symcode* .* \(.*\)$/ {\"\1\", (void *) \&\1},/p'" - -# Transform an extracted symbol line into symbol name with lib prefix and -# symbol address. -lt_cv_sys_global_symbol_to_c_name_address_lib_prefix="sed -n"\ -$lt_c_name_lib_hook\ -" -e 's/^: \(.*\) .*$/ {\"\1\", (void *) 0},/p'"\ -" -e 's/^$symcode$symcode* .* \(lib.*\)$/ {\"\1\", (void *) \&\1},/p'"\ -" -e 's/^$symcode$symcode* .* \(.*\)$/ {\"lib\1\", (void *) \&\1},/p'" - -# Handle CRLF in mingw tool chain -opt_cr= -case $build_os in -mingw*) - opt_cr=`$ECHO 'x\{0,1\}' | tr x '\015'` # option cr in regexp - ;; -esac - -# Try without a prefix underscore, then with it. -for ac_symprfx in "" "_"; do - - # Transform symcode, sympat, and symprfx into a raw symbol and a C symbol. - symxfrm="\\1 $ac_symprfx\\2 \\2" - - # Write the raw and C identifiers. - if test "$lt_cv_nm_interface" = "MS dumpbin"; then - # Fake it for dumpbin and say T for any non-static function, - # D for any global variable and I for any imported variable. - # Also find C++ and __fastcall symbols from MSVC++, - # which start with @ or ?. - lt_cv_sys_global_symbol_pipe="$AWK '"\ -" {last_section=section; section=\$ 3};"\ -" /^COFF SYMBOL TABLE/{for(i in hide) delete hide[i]};"\ -" /Section length .*#relocs.*(pick any)/{hide[last_section]=1};"\ -" /^ *Symbol name *: /{split(\$ 0,sn,\":\"); si=substr(sn[2],2)};"\ -" /^ *Type *: code/{print \"T\",si,substr(si,length(prfx))};"\ -" /^ *Type *: data/{print \"I\",si,substr(si,length(prfx))};"\ -" \$ 0!~/External *\|/{next};"\ -" / 0+ UNDEF /{next}; / UNDEF \([^|]\)*()/{next};"\ -" {if(hide[section]) next};"\ -" {f=\"D\"}; \$ 0~/\(\).*\|/{f=\"T\"};"\ -" {split(\$ 0,a,/\||\r/); split(a[2],s)};"\ -" s[1]~/^[@?]/{print f,s[1],s[1]; next};"\ -" s[1]~prfx {split(s[1],t,\"@\"); print f,t[1],substr(t[1],length(prfx))}"\ -" ' prfx=^$ac_symprfx" - else - lt_cv_sys_global_symbol_pipe="sed -n -e 's/^.*[ ]\($symcode$symcode*\)[ ][ ]*$ac_symprfx$sympat$opt_cr$/$symxfrm/p'" - fi - lt_cv_sys_global_symbol_pipe="$lt_cv_sys_global_symbol_pipe | sed '/ __gnu_lto/d'" - - # Check to see that the pipe works correctly. - pipe_works=no - - rm -f conftest* - cat > conftest.$ac_ext <<_LT_EOF -#ifdef __cplusplus -extern "C" { -#endif -char nm_test_var; -void nm_test_func(void); -void nm_test_func(void){} -#ifdef __cplusplus -} -#endif -int main(){nm_test_var='a';nm_test_func();return(0);} -_LT_EOF - - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - # Now try to grab the symbols. - nlist=conftest.nm - $ECHO "$as_me:$LINENO: $NM conftest.$ac_objext | $lt_cv_sys_global_symbol_pipe > $nlist" >&5 - if eval "$NM" conftest.$ac_objext \| "$lt_cv_sys_global_symbol_pipe" \> $nlist 2>&5 && test -s "$nlist"; then - # Try sorting and uniquifying the output. - if sort "$nlist" | uniq > "$nlist"T; then - mv -f "$nlist"T "$nlist" - else - rm -f "$nlist"T - fi - - # Make sure that we snagged all the symbols we need. - if $GREP ' nm_test_var$' "$nlist" >/dev/null; then - if $GREP ' nm_test_func$' "$nlist" >/dev/null; then - cat <<_LT_EOF > conftest.$ac_ext -/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ -#if defined _WIN32 || defined __CYGWIN__ || defined _WIN32_WCE -/* DATA imports from DLLs on WIN32 can't be const, because runtime - relocations are performed -- see ld's documentation on pseudo-relocs. */ -# define LT_DLSYM_CONST -#elif defined __osf__ -/* This system does not cope well with relocations in const data. */ -# define LT_DLSYM_CONST -#else -# define LT_DLSYM_CONST const -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -_LT_EOF - # Now generate the symbol file. - eval "$lt_cv_sys_global_symbol_to_cdecl"' < "$nlist" | $GREP -v main >> conftest.$ac_ext' - - cat <<_LT_EOF >> conftest.$ac_ext - -/* The mapping between symbol names and symbols. */ -LT_DLSYM_CONST struct { - const char *name; - void *address; -} -lt__PROGRAM__LTX_preloaded_symbols[] = -{ - { "@PROGRAM@", (void *) 0 }, -_LT_EOF - $SED "s/^$symcode$symcode* .* \(.*\)$/ {\"\1\", (void *) \&\1},/" < "$nlist" | $GREP -v main >> conftest.$ac_ext - cat <<\_LT_EOF >> conftest.$ac_ext - {0, (void *) 0} -}; - -/* This works around a problem in FreeBSD linker */ -#ifdef FREEBSD_WORKAROUND -static const void *lt_preloaded_setup() { - return lt__PROGRAM__LTX_preloaded_symbols; -} -#endif - -#ifdef __cplusplus -} -#endif -_LT_EOF - # Now try linking the two files. - mv conftest.$ac_objext conftstm.$ac_objext - lt_globsym_save_LIBS=$LIBS - lt_globsym_save_CFLAGS=$CFLAGS - LIBS=conftstm.$ac_objext - CFLAGS="$CFLAGS$lt_prog_compiler_no_builtin_flag" - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5 - (eval $ac_link) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && test -s conftest$ac_exeext; then - pipe_works=yes - fi - LIBS=$lt_globsym_save_LIBS - CFLAGS=$lt_globsym_save_CFLAGS - else - echo "cannot find nm_test_func in $nlist" >&5 - fi - else - echo "cannot find nm_test_var in $nlist" >&5 - fi - else - echo "cannot run $lt_cv_sys_global_symbol_pipe" >&5 - fi - else - echo "$progname: failed program was:" >&5 - cat conftest.$ac_ext >&5 - fi - rm -rf conftest* conftst* - - # Do not use the global_symbol_pipe unless it works. - if test yes = "$pipe_works"; then - break - else - lt_cv_sys_global_symbol_pipe= - fi -done - -fi - -if test -z "$lt_cv_sys_global_symbol_pipe"; then - lt_cv_sys_global_symbol_to_cdecl= -fi -if test -z "$lt_cv_sys_global_symbol_pipe$lt_cv_sys_global_symbol_to_cdecl"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: failed" >&5 -$as_echo "failed" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ok" >&5 -$as_echo "ok" >&6; } -fi - -# Response file support. -if test "$lt_cv_nm_interface" = "MS dumpbin"; then - nm_file_list_spec='@' -elif $NM --help 2>/dev/null | grep '[@]FILE' >/dev/null; then - nm_file_list_spec='@' -fi - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for sysroot" >&5 -$as_echo_n "checking for sysroot... " >&6; } - -# Check whether --with-sysroot was given. -if test "${with_sysroot+set}" = set; then : - withval=$with_sysroot; -else - with_sysroot=no -fi - - -lt_sysroot= -case $with_sysroot in #( - yes) - if test yes = "$GCC"; then - lt_sysroot=`$CC --print-sysroot 2>/dev/null` - fi - ;; #( - /*) - lt_sysroot=`echo "$with_sysroot" | sed -e "$sed_quote_subst"` - ;; #( - no|'') - ;; #( - *) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_sysroot" >&5 -$as_echo "$with_sysroot" >&6; } - as_fn_error $? "The sysroot must be an absolute path." "$LINENO" 5 - ;; -esac - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${lt_sysroot:-no}" >&5 -$as_echo "${lt_sysroot:-no}" >&6; } - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a working dd" >&5 -$as_echo_n "checking for a working dd... " >&6; } -if ${ac_cv_path_lt_DD+:} false; then : - $as_echo_n "(cached) " >&6 -else - printf 0123456789abcdef0123456789abcdef >conftest.i -cat conftest.i conftest.i >conftest2.i -: ${lt_DD:=$DD} -if test -z "$lt_DD"; then - ac_path_lt_DD_found=false - # Loop through the user's path and test for each of PROGNAME-LIST - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_prog in dd; do - for ac_exec_ext in '' $ac_executable_extensions; do - ac_path_lt_DD="$as_dir/$ac_prog$ac_exec_ext" - as_fn_executable_p "$ac_path_lt_DD" || continue -if "$ac_path_lt_DD" bs=32 count=1 conftest.out 2>/dev/null; then - cmp -s conftest.i conftest.out \ - && ac_cv_path_lt_DD="$ac_path_lt_DD" ac_path_lt_DD_found=: -fi - $ac_path_lt_DD_found && break 3 - done - done - done -IFS=$as_save_IFS - if test -z "$ac_cv_path_lt_DD"; then - : - fi -else - ac_cv_path_lt_DD=$lt_DD -fi - -rm -f conftest.i conftest2.i conftest.out -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_path_lt_DD" >&5 -$as_echo "$ac_cv_path_lt_DD" >&6; } - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to truncate binary pipes" >&5 -$as_echo_n "checking how to truncate binary pipes... " >&6; } -if ${lt_cv_truncate_bin+:} false; then : - $as_echo_n "(cached) " >&6 -else - printf 0123456789abcdef0123456789abcdef >conftest.i -cat conftest.i conftest.i >conftest2.i -lt_cv_truncate_bin= -if "$ac_cv_path_lt_DD" bs=32 count=1 conftest.out 2>/dev/null; then - cmp -s conftest.i conftest.out \ - && lt_cv_truncate_bin="$ac_cv_path_lt_DD bs=4096 count=1" -fi -rm -f conftest.i conftest2.i conftest.out -test -z "$lt_cv_truncate_bin" && lt_cv_truncate_bin="$SED -e 4q" -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_truncate_bin" >&5 -$as_echo "$lt_cv_truncate_bin" >&6; } - - - - - - - -# Calculate cc_basename. Skip known compiler wrappers and cross-prefix. -func_cc_basename () -{ - for cc_temp in $*""; do - case $cc_temp in - compile | *[\\/]compile | ccache | *[\\/]ccache ) ;; - distcc | *[\\/]distcc | purify | *[\\/]purify ) ;; - \-*) ;; - *) break;; - esac - done - func_cc_basename_result=`$ECHO "$cc_temp" | $SED "s%.*/%%; s%^$host_alias-%%"` -} - -# Check whether --enable-libtool-lock was given. -if test "${enable_libtool_lock+set}" = set; then : - enableval=$enable_libtool_lock; -fi - -test no = "$enable_libtool_lock" || enable_libtool_lock=yes - -# Some flags need to be propagated to the compiler or linker for good -# libtool support. -case $host in -ia64-*-hpux*) - # Find out what ABI is being produced by ac_compile, and set mode - # options accordingly. - echo 'int i;' > conftest.$ac_ext - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - case `/usr/bin/file conftest.$ac_objext` in - *ELF-32*) - HPUX_IA64_MODE=32 - ;; - *ELF-64*) - HPUX_IA64_MODE=64 - ;; - esac - fi - rm -rf conftest* - ;; -*-*-irix6*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. - echo '#line '$LINENO' "configure"' > conftest.$ac_ext - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - if test yes = "$lt_cv_prog_gnu_ld"; then - case `/usr/bin/file conftest.$ac_objext` in - *32-bit*) - LD="${LD-ld} -melf32bsmip" - ;; - *N32*) - LD="${LD-ld} -melf32bmipn32" - ;; - *64-bit*) - LD="${LD-ld} -melf64bmip" - ;; - esac - else - case `/usr/bin/file conftest.$ac_objext` in - *32-bit*) - LD="${LD-ld} -32" - ;; - *N32*) - LD="${LD-ld} -n32" - ;; - *64-bit*) - LD="${LD-ld} -64" - ;; - esac - fi - fi - rm -rf conftest* - ;; - -mips64*-*linux*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. - echo '#line '$LINENO' "configure"' > conftest.$ac_ext - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - emul=elf - case `/usr/bin/file conftest.$ac_objext` in - *32-bit*) - emul="${emul}32" - ;; - *64-bit*) - emul="${emul}64" - ;; - esac - case `/usr/bin/file conftest.$ac_objext` in - *MSB*) - emul="${emul}btsmip" - ;; - *LSB*) - emul="${emul}ltsmip" - ;; - esac - case `/usr/bin/file conftest.$ac_objext` in - *N32*) - emul="${emul}n32" - ;; - esac - LD="${LD-ld} -m $emul" - fi - rm -rf conftest* - ;; - -x86_64-*kfreebsd*-gnu|x86_64-*linux*|powerpc*-*linux*| \ -s390*-*linux*|s390*-*tpf*|sparc*-*linux*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. Note that the listed cases only cover the - # situations where additional linker options are needed (such as when - # doing 32-bit compilation for a host where ld defaults to 64-bit, or - # vice versa); the common cases where no linker options are needed do - # not appear in the list. - echo 'int i;' > conftest.$ac_ext - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - case `/usr/bin/file conftest.o` in - *32-bit*) - case $host in - x86_64-*kfreebsd*-gnu) - LD="${LD-ld} -m elf_i386_fbsd" - ;; - x86_64-*linux*) - case `/usr/bin/file conftest.o` in - *x86-64*) - LD="${LD-ld} -m elf32_x86_64" - ;; - *) - LD="${LD-ld} -m elf_i386" - ;; - esac - ;; - powerpc64le-*linux*) - LD="${LD-ld} -m elf32lppclinux" - ;; - powerpc64-*linux*) - LD="${LD-ld} -m elf32ppclinux" - ;; - s390x-*linux*) - LD="${LD-ld} -m elf_s390" - ;; - sparc64-*linux*) - LD="${LD-ld} -m elf32_sparc" - ;; - esac - ;; - *64-bit*) - case $host in - x86_64-*kfreebsd*-gnu) - LD="${LD-ld} -m elf_x86_64_fbsd" - ;; - x86_64-*linux*) - LD="${LD-ld} -m elf_x86_64" - ;; - powerpcle-*linux*) - LD="${LD-ld} -m elf64lppc" - ;; - powerpc-*linux*) - LD="${LD-ld} -m elf64ppc" - ;; - s390*-*linux*|s390*-*tpf*) - LD="${LD-ld} -m elf64_s390" - ;; - sparc*-*linux*) - LD="${LD-ld} -m elf64_sparc" - ;; - esac - ;; - esac - fi - rm -rf conftest* - ;; - -*-*-sco3.2v5*) - # On SCO OpenServer 5, we need -belf to get full-featured binaries. - SAVE_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -belf" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the C compiler needs -belf" >&5 -$as_echo_n "checking whether the C compiler needs -belf... " >&6; } -if ${lt_cv_cc_needs_belf+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - lt_cv_cc_needs_belf=yes -else - lt_cv_cc_needs_belf=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_cc_needs_belf" >&5 -$as_echo "$lt_cv_cc_needs_belf" >&6; } - if test yes != "$lt_cv_cc_needs_belf"; then - # this is probably gcc 2.8.0, egcs 1.0 or newer; no need for -belf - CFLAGS=$SAVE_CFLAGS - fi - ;; -*-*solaris*) - # Find out what ABI is being produced by ac_compile, and set linker - # options accordingly. - echo 'int i;' > conftest.$ac_ext - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - case `/usr/bin/file conftest.o` in - *64-bit*) - case $lt_cv_prog_gnu_ld in - yes*) - case $host in - i?86-*-solaris*|x86_64-*-solaris*) - LD="${LD-ld} -m elf_x86_64" - ;; - sparc*-*-solaris*) - LD="${LD-ld} -m elf64_sparc" - ;; - esac - # GNU ld 2.21 introduced _sol2 emulations. Use them if available. - if ${LD-ld} -V | grep _sol2 >/dev/null 2>&1; then - LD=${LD-ld}_sol2 - fi - ;; - *) - if ${LD-ld} -64 -r -o conftest2.o conftest.o >/dev/null 2>&1; then - LD="${LD-ld} -64" - fi - ;; - esac - ;; - esac - fi - rm -rf conftest* - ;; -esac - -need_locks=$enable_libtool_lock - -if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}mt", so it can be a program name with args. -set dummy ${ac_tool_prefix}mt; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_MANIFEST_TOOL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$MANIFEST_TOOL"; then - ac_cv_prog_MANIFEST_TOOL="$MANIFEST_TOOL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_MANIFEST_TOOL="${ac_tool_prefix}mt" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -MANIFEST_TOOL=$ac_cv_prog_MANIFEST_TOOL -if test -n "$MANIFEST_TOOL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MANIFEST_TOOL" >&5 -$as_echo "$MANIFEST_TOOL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_MANIFEST_TOOL"; then - ac_ct_MANIFEST_TOOL=$MANIFEST_TOOL - # Extract the first word of "mt", so it can be a program name with args. -set dummy mt; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_MANIFEST_TOOL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_MANIFEST_TOOL"; then - ac_cv_prog_ac_ct_MANIFEST_TOOL="$ac_ct_MANIFEST_TOOL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_MANIFEST_TOOL="mt" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_MANIFEST_TOOL=$ac_cv_prog_ac_ct_MANIFEST_TOOL -if test -n "$ac_ct_MANIFEST_TOOL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_MANIFEST_TOOL" >&5 -$as_echo "$ac_ct_MANIFEST_TOOL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_MANIFEST_TOOL" = x; then - MANIFEST_TOOL=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - MANIFEST_TOOL=$ac_ct_MANIFEST_TOOL - fi -else - MANIFEST_TOOL="$ac_cv_prog_MANIFEST_TOOL" -fi - -test -z "$MANIFEST_TOOL" && MANIFEST_TOOL=mt -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $MANIFEST_TOOL is a manifest tool" >&5 -$as_echo_n "checking if $MANIFEST_TOOL is a manifest tool... " >&6; } -if ${lt_cv_path_mainfest_tool+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_path_mainfest_tool=no - echo "$as_me:$LINENO: $MANIFEST_TOOL '-?'" >&5 - $MANIFEST_TOOL '-?' 2>conftest.err > conftest.out - cat conftest.err >&5 - if $GREP 'Manifest Tool' conftest.out > /dev/null; then - lt_cv_path_mainfest_tool=yes - fi - rm -f conftest* -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_path_mainfest_tool" >&5 -$as_echo "$lt_cv_path_mainfest_tool" >&6; } -if test yes != "$lt_cv_path_mainfest_tool"; then - MANIFEST_TOOL=: -fi - - - - - - - case $host_os in - rhapsody* | darwin*) - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}dsymutil", so it can be a program name with args. -set dummy ${ac_tool_prefix}dsymutil; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_DSYMUTIL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$DSYMUTIL"; then - ac_cv_prog_DSYMUTIL="$DSYMUTIL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_DSYMUTIL="${ac_tool_prefix}dsymutil" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -DSYMUTIL=$ac_cv_prog_DSYMUTIL -if test -n "$DSYMUTIL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $DSYMUTIL" >&5 -$as_echo "$DSYMUTIL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_DSYMUTIL"; then - ac_ct_DSYMUTIL=$DSYMUTIL - # Extract the first word of "dsymutil", so it can be a program name with args. -set dummy dsymutil; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_DSYMUTIL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_DSYMUTIL"; then - ac_cv_prog_ac_ct_DSYMUTIL="$ac_ct_DSYMUTIL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_DSYMUTIL="dsymutil" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_DSYMUTIL=$ac_cv_prog_ac_ct_DSYMUTIL -if test -n "$ac_ct_DSYMUTIL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_DSYMUTIL" >&5 -$as_echo "$ac_ct_DSYMUTIL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_DSYMUTIL" = x; then - DSYMUTIL=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - DSYMUTIL=$ac_ct_DSYMUTIL - fi -else - DSYMUTIL="$ac_cv_prog_DSYMUTIL" -fi - - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}nmedit", so it can be a program name with args. -set dummy ${ac_tool_prefix}nmedit; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_NMEDIT+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$NMEDIT"; then - ac_cv_prog_NMEDIT="$NMEDIT" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_NMEDIT="${ac_tool_prefix}nmedit" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -NMEDIT=$ac_cv_prog_NMEDIT -if test -n "$NMEDIT"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NMEDIT" >&5 -$as_echo "$NMEDIT" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_NMEDIT"; then - ac_ct_NMEDIT=$NMEDIT - # Extract the first word of "nmedit", so it can be a program name with args. -set dummy nmedit; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_NMEDIT+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_NMEDIT"; then - ac_cv_prog_ac_ct_NMEDIT="$ac_ct_NMEDIT" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_NMEDIT="nmedit" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_NMEDIT=$ac_cv_prog_ac_ct_NMEDIT -if test -n "$ac_ct_NMEDIT"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_NMEDIT" >&5 -$as_echo "$ac_ct_NMEDIT" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_NMEDIT" = x; then - NMEDIT=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - NMEDIT=$ac_ct_NMEDIT - fi -else - NMEDIT="$ac_cv_prog_NMEDIT" -fi - - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}lipo", so it can be a program name with args. -set dummy ${ac_tool_prefix}lipo; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_LIPO+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$LIPO"; then - ac_cv_prog_LIPO="$LIPO" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_LIPO="${ac_tool_prefix}lipo" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -LIPO=$ac_cv_prog_LIPO -if test -n "$LIPO"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LIPO" >&5 -$as_echo "$LIPO" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_LIPO"; then - ac_ct_LIPO=$LIPO - # Extract the first word of "lipo", so it can be a program name with args. -set dummy lipo; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_LIPO+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_LIPO"; then - ac_cv_prog_ac_ct_LIPO="$ac_ct_LIPO" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_LIPO="lipo" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_LIPO=$ac_cv_prog_ac_ct_LIPO -if test -n "$ac_ct_LIPO"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_LIPO" >&5 -$as_echo "$ac_ct_LIPO" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_LIPO" = x; then - LIPO=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - LIPO=$ac_ct_LIPO - fi -else - LIPO="$ac_cv_prog_LIPO" -fi - - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}otool", so it can be a program name with args. -set dummy ${ac_tool_prefix}otool; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_OTOOL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$OTOOL"; then - ac_cv_prog_OTOOL="$OTOOL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_OTOOL="${ac_tool_prefix}otool" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -OTOOL=$ac_cv_prog_OTOOL -if test -n "$OTOOL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OTOOL" >&5 -$as_echo "$OTOOL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_OTOOL"; then - ac_ct_OTOOL=$OTOOL - # Extract the first word of "otool", so it can be a program name with args. -set dummy otool; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_OTOOL+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_OTOOL"; then - ac_cv_prog_ac_ct_OTOOL="$ac_ct_OTOOL" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_OTOOL="otool" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_OTOOL=$ac_cv_prog_ac_ct_OTOOL -if test -n "$ac_ct_OTOOL"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OTOOL" >&5 -$as_echo "$ac_ct_OTOOL" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_OTOOL" = x; then - OTOOL=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - OTOOL=$ac_ct_OTOOL - fi -else - OTOOL="$ac_cv_prog_OTOOL" -fi - - if test -n "$ac_tool_prefix"; then - # Extract the first word of "${ac_tool_prefix}otool64", so it can be a program name with args. -set dummy ${ac_tool_prefix}otool64; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_OTOOL64+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$OTOOL64"; then - ac_cv_prog_OTOOL64="$OTOOL64" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_OTOOL64="${ac_tool_prefix}otool64" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -OTOOL64=$ac_cv_prog_OTOOL64 -if test -n "$OTOOL64"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OTOOL64" >&5 -$as_echo "$OTOOL64" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -fi -if test -z "$ac_cv_prog_OTOOL64"; then - ac_ct_OTOOL64=$OTOOL64 - # Extract the first word of "otool64", so it can be a program name with args. -set dummy otool64; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_ac_ct_OTOOL64+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$ac_ct_OTOOL64"; then - ac_cv_prog_ac_ct_OTOOL64="$ac_ct_OTOOL64" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_ac_ct_OTOOL64="otool64" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -ac_ct_OTOOL64=$ac_cv_prog_ac_ct_OTOOL64 -if test -n "$ac_ct_OTOOL64"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_ct_OTOOL64" >&5 -$as_echo "$ac_ct_OTOOL64" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - if test "x$ac_ct_OTOOL64" = x; then - OTOOL64=":" - else - case $cross_compiling:$ac_tool_warned in -yes:) -{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 -$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} -ac_tool_warned=yes ;; -esac - OTOOL64=$ac_ct_OTOOL64 - fi -else - OTOOL64="$ac_cv_prog_OTOOL64" -fi - - - - - - - - - - - - - - - - - - - - - - - - - - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -single_module linker flag" >&5 -$as_echo_n "checking for -single_module linker flag... " >&6; } -if ${lt_cv_apple_cc_single_mod+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_apple_cc_single_mod=no - if test -z "$LT_MULTI_MODULE"; then - # By default we will add the -single_module flag. You can override - # by either setting the environment variable LT_MULTI_MODULE - # non-empty at configure time, or by adding -multi_module to the - # link flags. - rm -rf libconftest.dylib* - echo "int foo(void){return 1;}" > conftest.c - echo "$LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ --dynamiclib -Wl,-single_module conftest.c" >&5 - $LTCC $LTCFLAGS $LDFLAGS -o libconftest.dylib \ - -dynamiclib -Wl,-single_module conftest.c 2>conftest.err - _lt_result=$? - # If there is a non-empty error log, and "single_module" - # appears in it, assume the flag caused a linker warning - if test -s conftest.err && $GREP single_module conftest.err; then - cat conftest.err >&5 - # Otherwise, if the output was created with a 0 exit code from - # the compiler, it worked. - elif test -f libconftest.dylib && test 0 = "$_lt_result"; then - lt_cv_apple_cc_single_mod=yes - else - cat conftest.err >&5 - fi - rm -rf libconftest.dylib* - rm -f conftest.* - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_apple_cc_single_mod" >&5 -$as_echo "$lt_cv_apple_cc_single_mod" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -exported_symbols_list linker flag" >&5 -$as_echo_n "checking for -exported_symbols_list linker flag... " >&6; } -if ${lt_cv_ld_exported_symbols_list+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_ld_exported_symbols_list=no - save_LDFLAGS=$LDFLAGS - echo "_main" > conftest.sym - LDFLAGS="$LDFLAGS -Wl,-exported_symbols_list,conftest.sym" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - lt_cv_ld_exported_symbols_list=yes -else - lt_cv_ld_exported_symbols_list=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$save_LDFLAGS - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_exported_symbols_list" >&5 -$as_echo "$lt_cv_ld_exported_symbols_list" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for -force_load linker flag" >&5 -$as_echo_n "checking for -force_load linker flag... " >&6; } -if ${lt_cv_ld_force_load+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_ld_force_load=no - cat > conftest.c << _LT_EOF -int forced_loaded() { return 2;} -_LT_EOF - echo "$LTCC $LTCFLAGS -c -o conftest.o conftest.c" >&5 - $LTCC $LTCFLAGS -c -o conftest.o conftest.c 2>&5 - echo "$AR cr libconftest.a conftest.o" >&5 - $AR cr libconftest.a conftest.o 2>&5 - echo "$RANLIB libconftest.a" >&5 - $RANLIB libconftest.a 2>&5 - cat > conftest.c << _LT_EOF -int main() { return 0;} -_LT_EOF - echo "$LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a" >&5 - $LTCC $LTCFLAGS $LDFLAGS -o conftest conftest.c -Wl,-force_load,./libconftest.a 2>conftest.err - _lt_result=$? - if test -s conftest.err && $GREP force_load conftest.err; then - cat conftest.err >&5 - elif test -f conftest && test 0 = "$_lt_result" && $GREP forced_load conftest >/dev/null 2>&1; then - lt_cv_ld_force_load=yes - else - cat conftest.err >&5 - fi - rm -f conftest.err libconftest.a conftest conftest.c - rm -rf conftest.dSYM - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_ld_force_load" >&5 -$as_echo "$lt_cv_ld_force_load" >&6; } - case $host_os in - rhapsody* | darwin1.[012]) - _lt_dar_allow_undefined='$wl-undefined ${wl}suppress' ;; - darwin1.*) - _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - darwin*) # darwin 5.x on - # if running on 10.5 or later, the deployment target defaults - # to the OS version, if on x86, and 10.4, the deployment - # target defaults to 10.4. Don't you love it? - case ${MACOSX_DEPLOYMENT_TARGET-10.0},$host in - 10.0,*86*-darwin8*|10.0,*-darwin[91]*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; - 10.[012][,.]*) - _lt_dar_allow_undefined='$wl-flat_namespace $wl-undefined ${wl}suppress' ;; - 10.*) - _lt_dar_allow_undefined='$wl-undefined ${wl}dynamic_lookup' ;; - esac - ;; - esac - if test yes = "$lt_cv_apple_cc_single_mod"; then - _lt_dar_single_mod='$single_module' - fi - if test yes = "$lt_cv_ld_exported_symbols_list"; then - _lt_dar_export_syms=' $wl-exported_symbols_list,$output_objdir/$libname-symbols.expsym' - else - _lt_dar_export_syms='~$NMEDIT -s $output_objdir/$libname-symbols.expsym $lib' - fi - if test : != "$DSYMUTIL" && test no = "$lt_cv_ld_force_load"; then - _lt_dsymutil='~$DSYMUTIL $lib || :' - else - _lt_dsymutil= - fi - ;; - esac - -# func_munge_path_list VARIABLE PATH -# ----------------------------------- -# VARIABLE is name of variable containing _space_ separated list of -# directories to be munged by the contents of PATH, which is string -# having a format: -# "DIR[:DIR]:" -# string "DIR[ DIR]" will be prepended to VARIABLE -# ":DIR[:DIR]" -# string "DIR[ DIR]" will be appended to VARIABLE -# "DIRP[:DIRP]::[DIRA:]DIRA" -# string "DIRP[ DIRP]" will be prepended to VARIABLE and string -# "DIRA[ DIRA]" will be appended to VARIABLE -# "DIR[:DIR]" -# VARIABLE will be replaced by "DIR[ DIR]" -func_munge_path_list () -{ - case x$2 in - x) - ;; - *:) - eval $1=\"`$ECHO $2 | $SED 's/:/ /g'` \$$1\" - ;; - x:*) - eval $1=\"\$$1 `$ECHO $2 | $SED 's/:/ /g'`\" - ;; - *::*) - eval $1=\"\$$1\ `$ECHO $2 | $SED -e 's/.*:://' -e 's/:/ /g'`\" - eval $1=\"`$ECHO $2 | $SED -e 's/::.*//' -e 's/:/ /g'`\ \$$1\" - ;; - *) - eval $1=\"`$ECHO $2 | $SED 's/:/ /g'`\" - ;; - esac -} - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking how to run the C preprocessor" >&5 -$as_echo_n "checking how to run the C preprocessor... " >&6; } -# On Suns, sometimes $CPP names a directory. -if test -n "$CPP" && test -d "$CPP"; then - CPP= -fi -if test -z "$CPP"; then - if ${ac_cv_prog_CPP+:} false; then : - $as_echo_n "(cached) " >&6 -else - # Double quotes because CPP needs to be expanded - for CPP in "$CC -E" "$CC -E -traditional-cpp" "/lib/cpp" - do - ac_preproc_ok=false -for ac_c_preproc_warn_flag in '' yes -do - # Use a header file that comes with gcc, so configuring glibc - # with a fresh cross-compiler works. - # Prefer to if __STDC__ is defined, since - # exists even on freestanding compilers. - # On the NeXT, cc -E runs the code through the compiler's parser, - # not just through cpp. "Syntax error" is here to catch this case. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#ifdef __STDC__ -# include -#else -# include -#endif - Syntax error -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - -else - # Broken: fails on valid input. -continue -fi -rm -f conftest.err conftest.i conftest.$ac_ext - - # OK, works on sane cases. Now check whether nonexistent headers - # can be detected and how. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - # Broken: success on invalid input. -continue -else - # Passes both tests. -ac_preproc_ok=: -break -fi -rm -f conftest.err conftest.i conftest.$ac_ext - -done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.i conftest.err conftest.$ac_ext -if $ac_preproc_ok; then : - break -fi - - done - ac_cv_prog_CPP=$CPP - -fi - CPP=$ac_cv_prog_CPP -else - ac_cv_prog_CPP=$CPP -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $CPP" >&5 -$as_echo "$CPP" >&6; } -ac_preproc_ok=false -for ac_c_preproc_warn_flag in '' yes -do - # Use a header file that comes with gcc, so configuring glibc - # with a fresh cross-compiler works. - # Prefer to if __STDC__ is defined, since - # exists even on freestanding compilers. - # On the NeXT, cc -E runs the code through the compiler's parser, - # not just through cpp. "Syntax error" is here to catch this case. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#ifdef __STDC__ -# include -#else -# include -#endif - Syntax error -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - -else - # Broken: fails on valid input. -continue -fi -rm -f conftest.err conftest.i conftest.$ac_ext - - # OK, works on sane cases. Now check whether nonexistent headers - # can be detected and how. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - # Broken: success on invalid input. -continue -else - # Passes both tests. -ac_preproc_ok=: -break -fi -rm -f conftest.err conftest.i conftest.$ac_ext - -done -# Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.i conftest.err conftest.$ac_ext -if $ac_preproc_ok; then : - -else - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details" "$LINENO" 5; } -fi - -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 -$as_echo_n "checking for ANSI C header files... " >&6; } -if ${ac_cv_header_stdc+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -#include -#include -#include - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_header_stdc=yes -else - ac_cv_header_stdc=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - -if test $ac_cv_header_stdc = yes; then - # SunOS 4.x string.h does not declare mem*, contrary to ANSI. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "memchr" >/dev/null 2>&1; then : - -else - ac_cv_header_stdc=no -fi -rm -f conftest* - -fi - -if test $ac_cv_header_stdc = yes; then - # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - -_ACEOF -if (eval "$ac_cpp conftest.$ac_ext") 2>&5 | - $EGREP "free" >/dev/null 2>&1; then : - -else - ac_cv_header_stdc=no -fi -rm -f conftest* - -fi - -if test $ac_cv_header_stdc = yes; then - # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi. - if test "$cross_compiling" = yes; then : - : -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -#include -#if ((' ' & 0x0FF) == 0x020) -# define ISLOWER(c) ('a' <= (c) && (c) <= 'z') -# define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c)) -#else -# define ISLOWER(c) \ - (('a' <= (c) && (c) <= 'i') \ - || ('j' <= (c) && (c) <= 'r') \ - || ('s' <= (c) && (c) <= 'z')) -# define TOUPPER(c) (ISLOWER(c) ? ((c) | 0x40) : (c)) -#endif - -#define XOR(e, f) (((e) && !(f)) || (!(e) && (f))) -int -main () -{ - int i; - for (i = 0; i < 256; i++) - if (XOR (islower (i), ISLOWER (i)) - || toupper (i) != TOUPPER (i)) - return 2; - return 0; -} -_ACEOF -if ac_fn_c_try_run "$LINENO"; then : - -else - ac_cv_header_stdc=no -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - -fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdc" >&5 -$as_echo "$ac_cv_header_stdc" >&6; } -if test $ac_cv_header_stdc = yes; then - -$as_echo "#define STDC_HEADERS 1" >>confdefs.h - -fi - -# On IRIX 5.3, sys/types and inttypes.h are conflicting. -for ac_header in sys/types.h sys/stat.h stdlib.h string.h memory.h strings.h \ - inttypes.h stdint.h unistd.h -do : - as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` -ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default -" -if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : - cat >>confdefs.h <<_ACEOF -#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 -_ACEOF - -fi - -done - - -for ac_header in dlfcn.h -do : - ac_fn_c_check_header_compile "$LINENO" "dlfcn.h" "ac_cv_header_dlfcn_h" "$ac_includes_default -" -if test "x$ac_cv_header_dlfcn_h" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_DLFCN_H 1 -_ACEOF - -fi - -done - - - - - -# Set options - - - - enable_dlopen=no - - - enable_win32_dll=no - - - # Check whether --enable-shared was given. -if test "${enable_shared+set}" = set; then : - enableval=$enable_shared; p=${PACKAGE-default} - case $enableval in - yes) enable_shared=yes ;; - no) enable_shared=no ;; - *) - enable_shared=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_shared=yes - fi - done - IFS=$lt_save_ifs - ;; - esac -else - enable_shared=yes -fi - - - - - - - - - - # Check whether --enable-static was given. -if test "${enable_static+set}" = set; then : - enableval=$enable_static; p=${PACKAGE-default} - case $enableval in - yes) enable_static=yes ;; - no) enable_static=no ;; - *) - enable_static=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_static=yes - fi - done - IFS=$lt_save_ifs - ;; - esac -else - enable_static=yes -fi - - - - - - - - - - -# Check whether --with-pic was given. -if test "${with_pic+set}" = set; then : - withval=$with_pic; lt_p=${PACKAGE-default} - case $withval in - yes|no) pic_mode=$withval ;; - *) - pic_mode=default - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for lt_pkg in $withval; do - IFS=$lt_save_ifs - if test "X$lt_pkg" = "X$lt_p"; then - pic_mode=yes - fi - done - IFS=$lt_save_ifs - ;; - esac -else - pic_mode=default -fi - - - - - - - - - # Check whether --enable-fast-install was given. -if test "${enable_fast_install+set}" = set; then : - enableval=$enable_fast_install; p=${PACKAGE-default} - case $enableval in - yes) enable_fast_install=yes ;; - no) enable_fast_install=no ;; - *) - enable_fast_install=no - # Look at the argument we got. We use all the common list separators. - lt_save_ifs=$IFS; IFS=$IFS$PATH_SEPARATOR, - for pkg in $enableval; do - IFS=$lt_save_ifs - if test "X$pkg" = "X$p"; then - enable_fast_install=yes - fi - done - IFS=$lt_save_ifs - ;; - esac -else - enable_fast_install=yes -fi - - - - - - - - - shared_archive_member_spec= -case $host,$enable_shared in -power*-*-aix[5-9]*,yes) - { $as_echo "$as_me:${as_lineno-$LINENO}: checking which variant of shared library versioning to provide" >&5 -$as_echo_n "checking which variant of shared library versioning to provide... " >&6; } - -# Check whether --with-aix-soname was given. -if test "${with_aix_soname+set}" = set; then : - withval=$with_aix_soname; case $withval in - aix|svr4|both) - ;; - *) - as_fn_error $? "Unknown argument to --with-aix-soname" "$LINENO" 5 - ;; - esac - lt_cv_with_aix_soname=$with_aix_soname -else - if ${lt_cv_with_aix_soname+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_with_aix_soname=aix -fi - - with_aix_soname=$lt_cv_with_aix_soname -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_aix_soname" >&5 -$as_echo "$with_aix_soname" >&6; } - if test aix != "$with_aix_soname"; then - # For the AIX way of multilib, we name the shared archive member - # based on the bitwidth used, traditionally 'shr.o' or 'shr_64.o', - # and 'shr.imp' or 'shr_64.imp', respectively, for the Import File. - # Even when GNU compilers ignore OBJECT_MODE but need '-maix64' flag, - # the AIX toolchain works better with OBJECT_MODE set (default 32). - if test 64 = "${OBJECT_MODE-32}"; then - shared_archive_member_spec=shr_64 - else - shared_archive_member_spec=shr - fi - fi - ;; -*) - with_aix_soname=aix - ;; -esac - - - - - - - - - - -# This can be used to rebuild libtool when needed -LIBTOOL_DEPS=$ltmain - -# Always use our own libtool. -LIBTOOL='$(SHELL) $(top_builddir)/libtool' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -test -z "$LN_S" && LN_S="ln -s" - - - - - - - - - - - - - - -if test -n "${ZSH_VERSION+set}"; then - setopt NO_GLOB_SUBST -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for objdir" >&5 -$as_echo_n "checking for objdir... " >&6; } -if ${lt_cv_objdir+:} false; then : - $as_echo_n "(cached) " >&6 -else - rm -f .libs 2>/dev/null -mkdir .libs 2>/dev/null -if test -d .libs; then - lt_cv_objdir=.libs -else - # MS-DOS does not allow filenames that begin with a dot. - lt_cv_objdir=_libs -fi -rmdir .libs 2>/dev/null -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_objdir" >&5 -$as_echo "$lt_cv_objdir" >&6; } -objdir=$lt_cv_objdir - - - - - -cat >>confdefs.h <<_ACEOF -#define LT_OBJDIR "$lt_cv_objdir/" -_ACEOF - - - - -case $host_os in -aix3*) - # AIX sometimes has problems with the GCC collect2 program. For some - # reason, if we set the COLLECT_NAMES environment variable, the problems - # vanish in a puff of smoke. - if test set != "${COLLECT_NAMES+set}"; then - COLLECT_NAMES= - export COLLECT_NAMES - fi - ;; -esac - -# Global variables: -ofile=libtool -can_build_shared=yes - -# All known linkers require a '.a' archive for static linking (except MSVC, -# which needs '.lib'). -libext=a - -with_gnu_ld=$lt_cv_prog_gnu_ld - -old_CC=$CC -old_CFLAGS=$CFLAGS - -# Set sane defaults for various variables -test -z "$CC" && CC=cc -test -z "$LTCC" && LTCC=$CC -test -z "$LTCFLAGS" && LTCFLAGS=$CFLAGS -test -z "$LD" && LD=ld -test -z "$ac_objext" && ac_objext=o - -func_cc_basename $compiler -cc_basename=$func_cc_basename_result - - -# Only perform the check for file, if the check method requires it -test -z "$MAGIC_CMD" && MAGIC_CMD=file -case $deplibs_check_method in -file_magic*) - if test "$file_magic_cmd" = '$MAGIC_CMD'; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ${ac_tool_prefix}file" >&5 -$as_echo_n "checking for ${ac_tool_prefix}file... " >&6; } -if ${lt_cv_path_MAGIC_CMD+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $MAGIC_CMD in -[\\/*] | ?:[\\/]*) - lt_cv_path_MAGIC_CMD=$MAGIC_CMD # Let the user override the test with a path. - ;; -*) - lt_save_MAGIC_CMD=$MAGIC_CMD - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR - ac_dummy="/usr/bin$PATH_SEPARATOR$PATH" - for ac_dir in $ac_dummy; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - if test -f "$ac_dir/${ac_tool_prefix}file"; then - lt_cv_path_MAGIC_CMD=$ac_dir/"${ac_tool_prefix}file" - if test -n "$file_magic_test_file"; then - case $deplibs_check_method in - "file_magic "*) - file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"` - MAGIC_CMD=$lt_cv_path_MAGIC_CMD - if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null | - $EGREP "$file_magic_regex" > /dev/null; then - : - else - cat <<_LT_EOF 1>&2 - -*** Warning: the command libtool uses to detect shared libraries, -*** $file_magic_cmd, produces output that libtool cannot recognize. -*** The result is that libtool may fail to recognize shared libraries -*** as such. This will affect the creation of libtool libraries that -*** depend on shared libraries, but programs linked with such libtool -*** libraries will work regardless of this problem. Nevertheless, you -*** may want to report the problem to your system manager and/or to -*** bug-libtool@gnu.org - -_LT_EOF - fi ;; - esac - fi - break - fi - done - IFS=$lt_save_ifs - MAGIC_CMD=$lt_save_MAGIC_CMD - ;; -esac -fi - -MAGIC_CMD=$lt_cv_path_MAGIC_CMD -if test -n "$MAGIC_CMD"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MAGIC_CMD" >&5 -$as_echo "$MAGIC_CMD" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - - - -if test -z "$lt_cv_path_MAGIC_CMD"; then - if test -n "$ac_tool_prefix"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for file" >&5 -$as_echo_n "checking for file... " >&6; } -if ${lt_cv_path_MAGIC_CMD+:} false; then : - $as_echo_n "(cached) " >&6 -else - case $MAGIC_CMD in -[\\/*] | ?:[\\/]*) - lt_cv_path_MAGIC_CMD=$MAGIC_CMD # Let the user override the test with a path. - ;; -*) - lt_save_MAGIC_CMD=$MAGIC_CMD - lt_save_ifs=$IFS; IFS=$PATH_SEPARATOR - ac_dummy="/usr/bin$PATH_SEPARATOR$PATH" - for ac_dir in $ac_dummy; do - IFS=$lt_save_ifs - test -z "$ac_dir" && ac_dir=. - if test -f "$ac_dir/file"; then - lt_cv_path_MAGIC_CMD=$ac_dir/"file" - if test -n "$file_magic_test_file"; then - case $deplibs_check_method in - "file_magic "*) - file_magic_regex=`expr "$deplibs_check_method" : "file_magic \(.*\)"` - MAGIC_CMD=$lt_cv_path_MAGIC_CMD - if eval $file_magic_cmd \$file_magic_test_file 2> /dev/null | - $EGREP "$file_magic_regex" > /dev/null; then - : - else - cat <<_LT_EOF 1>&2 - -*** Warning: the command libtool uses to detect shared libraries, -*** $file_magic_cmd, produces output that libtool cannot recognize. -*** The result is that libtool may fail to recognize shared libraries -*** as such. This will affect the creation of libtool libraries that -*** depend on shared libraries, but programs linked with such libtool -*** libraries will work regardless of this problem. Nevertheless, you -*** may want to report the problem to your system manager and/or to -*** bug-libtool@gnu.org - -_LT_EOF - fi ;; - esac - fi - break - fi - done - IFS=$lt_save_ifs - MAGIC_CMD=$lt_save_MAGIC_CMD - ;; -esac -fi - -MAGIC_CMD=$lt_cv_path_MAGIC_CMD -if test -n "$MAGIC_CMD"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $MAGIC_CMD" >&5 -$as_echo "$MAGIC_CMD" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - else - MAGIC_CMD=: - fi -fi - - fi - ;; -esac - -# Use C for the default configuration in the libtool script - -lt_save_CC=$CC -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - - -# Source file extension for C test sources. -ac_ext=c - -# Object file extension for compiled C test sources. -objext=o -objext=$objext - -# Code to be used in simple compile tests -lt_simple_compile_test_code="int some_variable = 0;" - -# Code to be used in simple link tests -lt_simple_link_test_code='int main(){return(0);}' - - - - - - - -# If no C compiler was specified, use CC. -LTCC=${LTCC-"$CC"} - -# If no C compiler flags were specified, use CFLAGS. -LTCFLAGS=${LTCFLAGS-"$CFLAGS"} - -# Allow CC to be a program name with arguments. -compiler=$CC - -# Save the default compiler, since it gets overwritten when the other -# tags are being tested, and _LT_TAGVAR(compiler, []) is a NOP. -compiler_DEFAULT=$CC - -# save warnings/boilerplate of simple test code -ac_outfile=conftest.$ac_objext -echo "$lt_simple_compile_test_code" >conftest.$ac_ext -eval "$ac_compile" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err -_lt_compiler_boilerplate=`cat conftest.err` -$RM conftest* - -ac_outfile=conftest.$ac_objext -echo "$lt_simple_link_test_code" >conftest.$ac_ext -eval "$ac_link" 2>&1 >/dev/null | $SED '/^$/d; /^ *+/d' >conftest.err -_lt_linker_boilerplate=`cat conftest.err` -$RM -r conftest* - - -if test -n "$compiler"; then - -lt_prog_compiler_no_builtin_flag= - -if test yes = "$GCC"; then - case $cc_basename in - nvcc*) - lt_prog_compiler_no_builtin_flag=' -Xcompiler -fno-builtin' ;; - *) - lt_prog_compiler_no_builtin_flag=' -fno-builtin' ;; - esac - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -fno-rtti -fno-exceptions" >&5 -$as_echo_n "checking if $compiler supports -fno-rtti -fno-exceptions... " >&6; } -if ${lt_cv_prog_compiler_rtti_exceptions+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler_rtti_exceptions=no - ac_outfile=conftest.$ac_objext - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - lt_compiler_flag="-fno-rtti -fno-exceptions" ## exclude from sc_useless_quotes_in_assignment - # Insert the option either (1) after the last *FLAGS variable, or - # (2) before a word containing "conftest.", or (3) at the end. - # Note that $ac_compile itself does not contain backslashes and begins - # with a dollar sign (not a hyphen), so the echo should work correctly. - # The option is referenced via a variable to avoid confusing sed. - lt_compile=`echo "$ac_compile" | $SED \ - -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ - -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ - -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) - (eval "$lt_compile" 2>conftest.err) - ac_status=$? - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - if (exit $ac_status) && test -s "$ac_outfile"; then - # The compiler can only warn and ignore the option if not recognized - # So say no if there are warnings other than the usual output. - $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp - $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 - if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then - lt_cv_prog_compiler_rtti_exceptions=yes - fi - fi - $RM conftest* - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_rtti_exceptions" >&5 -$as_echo "$lt_cv_prog_compiler_rtti_exceptions" >&6; } - -if test yes = "$lt_cv_prog_compiler_rtti_exceptions"; then - lt_prog_compiler_no_builtin_flag="$lt_prog_compiler_no_builtin_flag -fno-rtti -fno-exceptions" -else - : -fi - -fi - - - - - - - lt_prog_compiler_wl= -lt_prog_compiler_pic= -lt_prog_compiler_static= - - - if test yes = "$GCC"; then - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_static='-static' - - case $host_os in - aix*) - # All AIX code is PIC. - if test ia64 = "$host_cpu"; then - # AIX 5 now supports IA64 processor - lt_prog_compiler_static='-Bstatic' - fi - lt_prog_compiler_pic='-fPIC' - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - lt_prog_compiler_pic='-fPIC' - ;; - m68k) - # FIXME: we need at least 68020 code to build shared libraries, but - # adding the '-m68020' flag to GCC prevents building anything better, - # like '-m68040'. - lt_prog_compiler_pic='-m68020 -resident32 -malways-restore-a4' - ;; - esac - ;; - - beos* | irix5* | irix6* | nonstopux* | osf3* | osf4* | osf5*) - # PIC is the default for these OSes. - ;; - - mingw* | cygwin* | pw32* | os2* | cegcc*) - # This hack is so that the source file can tell whether it is being - # built for inclusion in a dll (and should export symbols for example). - # Although the cygwin gcc ignores -fPIC, still need this for old-style - # (--disable-auto-import) libraries - lt_prog_compiler_pic='-DDLL_EXPORT' - case $host_os in - os2*) - lt_prog_compiler_static='$wl-static' - ;; - esac - ;; - - darwin* | rhapsody*) - # PIC is the default on this platform - # Common symbols not allowed in MH_DYLIB files - lt_prog_compiler_pic='-fno-common' - ;; - - haiku*) - # PIC is the default for Haiku. - # The "-static" flag exists, but is broken. - lt_prog_compiler_static= - ;; - - hpux*) - # PIC is the default for 64-bit PA HP-UX, but not for 32-bit - # PA HP-UX. On IA64 HP-UX, PIC is the default but the pic flag - # sets the default TLS model and affects inlining. - case $host_cpu in - hppa*64*) - # +Z the default - ;; - *) - lt_prog_compiler_pic='-fPIC' - ;; - esac - ;; - - interix[3-9]*) - # Interix 3.x gcc -fpic/-fPIC options generate broken code. - # Instead, we relocate shared libraries at runtime. - ;; - - msdosdjgpp*) - # Just because we use GCC doesn't mean we suddenly get shared libraries - # on systems that don't support them. - lt_prog_compiler_can_build_shared=no - enable_shared=no - ;; - - *nto* | *qnx*) - # QNX uses GNU C++, but need to define -shared option too, otherwise - # it will coredump. - lt_prog_compiler_pic='-fPIC -shared' - ;; - - sysv4*MP*) - if test -d /usr/nec; then - lt_prog_compiler_pic=-Kconform_pic - fi - ;; - - *) - lt_prog_compiler_pic='-fPIC' - ;; - esac - - case $cc_basename in - nvcc*) # Cuda Compiler Driver 2.2 - lt_prog_compiler_wl='-Xlinker ' - if test -n "$lt_prog_compiler_pic"; then - lt_prog_compiler_pic="-Xcompiler $lt_prog_compiler_pic" - fi - ;; - esac - else - # PORTME Check for flag to pass linker flags through the system compiler. - case $host_os in - aix*) - lt_prog_compiler_wl='-Wl,' - if test ia64 = "$host_cpu"; then - # AIX 5 now supports IA64 processor - lt_prog_compiler_static='-Bstatic' - else - lt_prog_compiler_static='-bnso -bI:/lib/syscalls.exp' - fi - ;; - - darwin* | rhapsody*) - # PIC is the default on this platform - # Common symbols not allowed in MH_DYLIB files - lt_prog_compiler_pic='-fno-common' - case $cc_basename in - nagfor*) - # NAG Fortran compiler - lt_prog_compiler_wl='-Wl,-Wl,,' - lt_prog_compiler_pic='-PIC' - lt_prog_compiler_static='-Bstatic' - ;; - esac - ;; - - mingw* | cygwin* | pw32* | os2* | cegcc*) - # This hack is so that the source file can tell whether it is being - # built for inclusion in a dll (and should export symbols for example). - lt_prog_compiler_pic='-DDLL_EXPORT' - case $host_os in - os2*) - lt_prog_compiler_static='$wl-static' - ;; - esac - ;; - - hpux9* | hpux10* | hpux11*) - lt_prog_compiler_wl='-Wl,' - # PIC is the default for IA64 HP-UX and 64-bit HP-UX, but - # not for PA HP-UX. - case $host_cpu in - hppa*64*|ia64*) - # +Z the default - ;; - *) - lt_prog_compiler_pic='+Z' - ;; - esac - # Is there a better lt_prog_compiler_static that works with the bundled CC? - lt_prog_compiler_static='$wl-a ${wl}archive' - ;; - - irix5* | irix6* | nonstopux*) - lt_prog_compiler_wl='-Wl,' - # PIC (with -KPIC) is the default. - lt_prog_compiler_static='-non_shared' - ;; - - linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - case $cc_basename in - # old Intel for x86_64, which still supported -KPIC. - ecc*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-static' - ;; - # flang / f18. f95 an alias for gfortran or flang on Debian - flang* | f18* | f95*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-fPIC' - lt_prog_compiler_static='-static' - ;; - # icc used to be incompatible with GCC. - # ICC 10 doesn't accept -KPIC any more. - icc* | ifort*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-fPIC' - lt_prog_compiler_static='-static' - ;; - # Lahey Fortran 8.1. - lf95*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='--shared' - lt_prog_compiler_static='--static' - ;; - nagfor*) - # NAG Fortran compiler - lt_prog_compiler_wl='-Wl,-Wl,,' - lt_prog_compiler_pic='-PIC' - lt_prog_compiler_static='-Bstatic' - ;; - tcc*) - # Fabrice Bellard et al's Tiny C Compiler - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-fPIC' - lt_prog_compiler_static='-static' - ;; - pgcc* | pgf77* | pgf90* | pgf95* | pgfortran*) - # Portland Group compilers (*not* the Pentium gcc compiler, - # which looks to be a dead project) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-fpic' - lt_prog_compiler_static='-Bstatic' - ;; - ccc*) - lt_prog_compiler_wl='-Wl,' - # All Alpha code is PIC. - lt_prog_compiler_static='-non_shared' - ;; - xl* | bgxl* | bgf* | mpixl*) - # IBM XL C 8.0/Fortran 10.1, 11.1 on PPC and BlueGene - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-qpic' - lt_prog_compiler_static='-qstaticlink' - ;; - *) - case `$CC -V 2>&1 | sed 5q` in - *Sun\ Ceres\ Fortran* | *Sun*Fortran*\ [1-7].* | *Sun*Fortran*\ 8.[0-3]*) - # Sun Fortran 8.3 passes all unrecognized flags to the linker - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - lt_prog_compiler_wl='' - ;; - *Sun\ F* | *Sun*Fortran*) - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - lt_prog_compiler_wl='-Qoption ld ' - ;; - *Sun\ C*) - # Sun C 5.9 - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - lt_prog_compiler_wl='-Wl,' - ;; - *Intel*\ [CF]*Compiler*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-fPIC' - lt_prog_compiler_static='-static' - ;; - *Portland\ Group*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-fpic' - lt_prog_compiler_static='-Bstatic' - ;; - esac - ;; - esac - ;; - - newsos6) - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - ;; - - *nto* | *qnx*) - # QNX uses GNU C++, but need to define -shared option too, otherwise - # it will coredump. - lt_prog_compiler_pic='-fPIC -shared' - ;; - - osf3* | osf4* | osf5*) - lt_prog_compiler_wl='-Wl,' - # All OSF/1 code is PIC. - lt_prog_compiler_static='-non_shared' - ;; - - rdos*) - lt_prog_compiler_static='-non_shared' - ;; - - solaris*) - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - case $cc_basename in - f77* | f90* | f95* | sunf77* | sunf90* | sunf95*) - lt_prog_compiler_wl='-Qoption ld ';; - *) - lt_prog_compiler_wl='-Wl,';; - esac - ;; - - sunos4*) - lt_prog_compiler_wl='-Qoption ld ' - lt_prog_compiler_pic='-PIC' - lt_prog_compiler_static='-Bstatic' - ;; - - sysv4 | sysv4.2uw2* | sysv4.3*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - ;; - - sysv4*MP*) - if test -d /usr/nec; then - lt_prog_compiler_pic='-Kconform_pic' - lt_prog_compiler_static='-Bstatic' - fi - ;; - - sysv5* | unixware* | sco3.2v5* | sco5v6* | OpenUNIX*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_pic='-KPIC' - lt_prog_compiler_static='-Bstatic' - ;; - - unicos*) - lt_prog_compiler_wl='-Wl,' - lt_prog_compiler_can_build_shared=no - ;; - - uts4*) - lt_prog_compiler_pic='-pic' - lt_prog_compiler_static='-Bstatic' - ;; - - *) - lt_prog_compiler_can_build_shared=no - ;; - esac - fi - -case $host_os in - # For platforms that do not support PIC, -DPIC is meaningless: - *djgpp*) - lt_prog_compiler_pic= - ;; - *) - lt_prog_compiler_pic="$lt_prog_compiler_pic -DPIC" - ;; -esac - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $compiler option to produce PIC" >&5 -$as_echo_n "checking for $compiler option to produce PIC... " >&6; } -if ${lt_cv_prog_compiler_pic+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler_pic=$lt_prog_compiler_pic -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic" >&5 -$as_echo "$lt_cv_prog_compiler_pic" >&6; } -lt_prog_compiler_pic=$lt_cv_prog_compiler_pic - -# -# Check to make sure the PIC flag actually works. -# -if test -n "$lt_prog_compiler_pic"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler PIC flag $lt_prog_compiler_pic works" >&5 -$as_echo_n "checking if $compiler PIC flag $lt_prog_compiler_pic works... " >&6; } -if ${lt_cv_prog_compiler_pic_works+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler_pic_works=no - ac_outfile=conftest.$ac_objext - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - lt_compiler_flag="$lt_prog_compiler_pic -DPIC" ## exclude from sc_useless_quotes_in_assignment - # Insert the option either (1) after the last *FLAGS variable, or - # (2) before a word containing "conftest.", or (3) at the end. - # Note that $ac_compile itself does not contain backslashes and begins - # with a dollar sign (not a hyphen), so the echo should work correctly. - # The option is referenced via a variable to avoid confusing sed. - lt_compile=`echo "$ac_compile" | $SED \ - -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ - -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ - -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) - (eval "$lt_compile" 2>conftest.err) - ac_status=$? - cat conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - if (exit $ac_status) && test -s "$ac_outfile"; then - # The compiler can only warn and ignore the option if not recognized - # So say no if there are warnings other than the usual output. - $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' >conftest.exp - $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 - if test ! -s conftest.er2 || diff conftest.exp conftest.er2 >/dev/null; then - lt_cv_prog_compiler_pic_works=yes - fi - fi - $RM conftest* - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_pic_works" >&5 -$as_echo "$lt_cv_prog_compiler_pic_works" >&6; } - -if test yes = "$lt_cv_prog_compiler_pic_works"; then - case $lt_prog_compiler_pic in - "" | " "*) ;; - *) lt_prog_compiler_pic=" $lt_prog_compiler_pic" ;; - esac -else - lt_prog_compiler_pic= - lt_prog_compiler_can_build_shared=no -fi - -fi - - - - - - - - - - - -# -# Check to make sure the static flag actually works. -# -wl=$lt_prog_compiler_wl eval lt_tmp_static_flag=\"$lt_prog_compiler_static\" -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler static flag $lt_tmp_static_flag works" >&5 -$as_echo_n "checking if $compiler static flag $lt_tmp_static_flag works... " >&6; } -if ${lt_cv_prog_compiler_static_works+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler_static_works=no - save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS $lt_tmp_static_flag" - echo "$lt_simple_link_test_code" > conftest.$ac_ext - if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then - # The linker can only warn and ignore the option if not recognized - # So say no if there are warnings - if test -s conftest.err; then - # Append any errors to the config.log. - cat conftest.err 1>&5 - $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp - $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 - if diff conftest.exp conftest.er2 >/dev/null; then - lt_cv_prog_compiler_static_works=yes - fi - else - lt_cv_prog_compiler_static_works=yes - fi - fi - $RM -r conftest* - LDFLAGS=$save_LDFLAGS - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_static_works" >&5 -$as_echo "$lt_cv_prog_compiler_static_works" >&6; } - -if test yes = "$lt_cv_prog_compiler_static_works"; then - : -else - lt_prog_compiler_static= -fi - - - - - - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5 -$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; } -if ${lt_cv_prog_compiler_c_o+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler_c_o=no - $RM -r conftest 2>/dev/null - mkdir conftest - cd conftest - mkdir out - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - - lt_compiler_flag="-o out/conftest2.$ac_objext" - # Insert the option either (1) after the last *FLAGS variable, or - # (2) before a word containing "conftest.", or (3) at the end. - # Note that $ac_compile itself does not contain backslashes and begins - # with a dollar sign (not a hyphen), so the echo should work correctly. - lt_compile=`echo "$ac_compile" | $SED \ - -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ - -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ - -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) - (eval "$lt_compile" 2>out/conftest.err) - ac_status=$? - cat out/conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - if (exit $ac_status) && test -s out/conftest2.$ac_objext - then - # The compiler can only warn and ignore the option if not recognized - # So say no if there are warnings - $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp - $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 - if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then - lt_cv_prog_compiler_c_o=yes - fi - fi - chmod u+w . 2>&5 - $RM conftest* - # SGI C++ compiler will create directory out/ii_files/ for - # template instantiation - test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files - $RM out/* && rmdir out - cd .. - $RM -r conftest - $RM conftest* - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o" >&5 -$as_echo "$lt_cv_prog_compiler_c_o" >&6; } - - - - - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $compiler supports -c -o file.$ac_objext" >&5 -$as_echo_n "checking if $compiler supports -c -o file.$ac_objext... " >&6; } -if ${lt_cv_prog_compiler_c_o+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler_c_o=no - $RM -r conftest 2>/dev/null - mkdir conftest - cd conftest - mkdir out - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - - lt_compiler_flag="-o out/conftest2.$ac_objext" - # Insert the option either (1) after the last *FLAGS variable, or - # (2) before a word containing "conftest.", or (3) at the end. - # Note that $ac_compile itself does not contain backslashes and begins - # with a dollar sign (not a hyphen), so the echo should work correctly. - lt_compile=`echo "$ac_compile" | $SED \ - -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ - -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ - -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:$LINENO: $lt_compile\"" >&5) - (eval "$lt_compile" 2>out/conftest.err) - ac_status=$? - cat out/conftest.err >&5 - echo "$as_me:$LINENO: \$? = $ac_status" >&5 - if (exit $ac_status) && test -s out/conftest2.$ac_objext - then - # The compiler can only warn and ignore the option if not recognized - # So say no if there are warnings - $ECHO "$_lt_compiler_boilerplate" | $SED '/^$/d' > out/conftest.exp - $SED '/^$/d; /^ *+/d' out/conftest.err >out/conftest.er2 - if test ! -s out/conftest.er2 || diff out/conftest.exp out/conftest.er2 >/dev/null; then - lt_cv_prog_compiler_c_o=yes - fi - fi - chmod u+w . 2>&5 - $RM conftest* - # SGI C++ compiler will create directory out/ii_files/ for - # template instantiation - test -d out/ii_files && $RM out/ii_files/* && rmdir out/ii_files - $RM out/* && rmdir out - cd .. - $RM -r conftest - $RM conftest* - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler_c_o" >&5 -$as_echo "$lt_cv_prog_compiler_c_o" >&6; } - - - - -hard_links=nottested -if test no = "$lt_cv_prog_compiler_c_o" && test no != "$need_locks"; then - # do not overwrite the value of need_locks provided by the user - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can lock with hard links" >&5 -$as_echo_n "checking if we can lock with hard links... " >&6; } - hard_links=yes - $RM conftest* - ln conftest.a conftest.b 2>/dev/null && hard_links=no - touch conftest.a - ln conftest.a conftest.b 2>&5 || hard_links=no - ln conftest.a conftest.b 2>/dev/null && hard_links=no - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $hard_links" >&5 -$as_echo "$hard_links" >&6; } - if test no = "$hard_links"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: '$CC' does not support '-c -o', so 'make -j' may be unsafe" >&5 -$as_echo "$as_me: WARNING: '$CC' does not support '-c -o', so 'make -j' may be unsafe" >&2;} - need_locks=warn - fi -else - need_locks=no -fi - - - - - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $compiler linker ($LD) supports shared libraries" >&5 -$as_echo_n "checking whether the $compiler linker ($LD) supports shared libraries... " >&6; } - - runpath_var= - allow_undefined_flag= - always_export_symbols=no - archive_cmds= - archive_expsym_cmds= - compiler_needs_object=no - enable_shared_with_static_runtimes=no - export_dynamic_flag_spec= - export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED '\''s/.* //'\'' | sort | uniq > $export_symbols' - hardcode_automatic=no - hardcode_direct=no - hardcode_direct_absolute=no - hardcode_libdir_flag_spec= - hardcode_libdir_separator= - hardcode_minus_L=no - hardcode_shlibpath_var=unsupported - inherit_rpath=no - link_all_deplibs=unknown - module_cmds= - module_expsym_cmds= - old_archive_from_new_cmds= - old_archive_from_expsyms_cmds= - thread_safe_flag_spec= - whole_archive_flag_spec= - # include_expsyms should be a list of space-separated symbols to be *always* - # included in the symbol list - include_expsyms= - # exclude_expsyms can be an extended regexp of symbols to exclude - # it will be wrapped by ' (' and ')$', so one must not match beginning or - # end of line. Example: 'a|bc|.*d.*' will exclude the symbols 'a' and 'bc', - # as well as any symbol that contains 'd'. - exclude_expsyms='_GLOBAL_OFFSET_TABLE_|_GLOBAL__F[ID]_.*' - # Although _GLOBAL_OFFSET_TABLE_ is a valid symbol C name, most a.out - # platforms (ab)use it in PIC code, but their linkers get confused if - # the symbol is explicitly referenced. Since portable code cannot - # rely on this symbol name, it's probably fine to never include it in - # preloaded symbol tables. - # Exclude shared library initialization/finalization symbols. - extract_expsyms_cmds= - - case $host_os in - cygwin* | mingw* | pw32* | cegcc*) - # FIXME: the MSVC++ port hasn't been tested in a loooong time - # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. - if test yes != "$GCC"; then - with_gnu_ld=no - fi - ;; - interix*) - # we just hope/assume this is gcc and not c89 (= MSVC++) - with_gnu_ld=yes - ;; - openbsd* | bitrig*) - with_gnu_ld=no - ;; - linux* | k*bsd*-gnu | gnu*) - link_all_deplibs=no - ;; - esac - - ld_shlibs=yes - - # On some targets, GNU ld is compatible enough with the native linker - # that we're better off using the native interface for both. - lt_use_gnu_ld_interface=no - if test yes = "$with_gnu_ld"; then - case $host_os in - aix*) - # The AIX port of GNU ld has always aspired to compatibility - # with the native linker. However, as the warning in the GNU ld - # block says, versions before 2.19.5* couldn't really create working - # shared libraries, regardless of the interface used. - case `$LD -v 2>&1` in - *\ \(GNU\ Binutils\)\ 2.19.5*) ;; - *\ \(GNU\ Binutils\)\ 2.[2-9]*) ;; - *\ \(GNU\ Binutils\)\ [3-9]*) ;; - *) - lt_use_gnu_ld_interface=yes - ;; - esac - ;; - *) - lt_use_gnu_ld_interface=yes - ;; - esac - fi - - if test yes = "$lt_use_gnu_ld_interface"; then - # If archive_cmds runs LD, not CC, wlarc should be empty - wlarc='$wl' - - # Set some defaults for GNU ld with shared library support. These - # are reset later if shared libraries are not supported. Putting them - # here allows them to be overridden if necessary. - runpath_var=LD_RUN_PATH - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - export_dynamic_flag_spec='$wl--export-dynamic' - # ancient GNU ld didn't support --whole-archive et. al. - if $LD --help 2>&1 | $GREP 'no-whole-archive' > /dev/null; then - whole_archive_flag_spec=$wlarc'--whole-archive$convenience '$wlarc'--no-whole-archive' - else - whole_archive_flag_spec= - fi - supports_anon_versioning=no - case `$LD -v | $SED -e 's/(^)\+)\s\+//' 2>&1` in - *GNU\ gold*) supports_anon_versioning=yes ;; - *\ [01].* | *\ 2.[0-9].* | *\ 2.10.*) ;; # catch versions < 2.11 - *\ 2.11.93.0.2\ *) supports_anon_versioning=yes ;; # RH7.3 ... - *\ 2.11.92.0.12\ *) supports_anon_versioning=yes ;; # Mandrake 8.2 ... - *\ 2.11.*) ;; # other 2.11 versions - *) supports_anon_versioning=yes ;; - esac - - # See if GNU ld supports shared libraries. - case $host_os in - aix[3-9]*) - # On AIX/PPC, the GNU linker is very broken - if test ia64 != "$host_cpu"; then - ld_shlibs=no - cat <<_LT_EOF 1>&2 - -*** Warning: the GNU linker, at least up to release 2.19, is reported -*** to be unable to reliably create shared libraries on AIX. -*** Therefore, libtool is disabling shared libraries support. If you -*** really care for shared libraries, you may want to install binutils -*** 2.20 or above, or modify your PATH so that a non-GNU linker is found. -*** You will then need to restart the configuration process. - -_LT_EOF - fi - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - archive_expsym_cmds='' - ;; - m68k) - archive_cmds='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' - hardcode_libdir_flag_spec='-L$libdir' - hardcode_minus_L=yes - ;; - esac - ;; - - beos*) - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - allow_undefined_flag=unsupported - # Joseph Beckenbach says some releases of gcc - # support --undefined. This deserves some investigation. FIXME - archive_cmds='$CC -nostart $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - else - ld_shlibs=no - fi - ;; - - cygwin* | mingw* | pw32* | cegcc*) - # _LT_TAGVAR(hardcode_libdir_flag_spec, ) is actually meaningless, - # as there is no search path for DLLs. - hardcode_libdir_flag_spec='-L$libdir' - export_dynamic_flag_spec='$wl--export-all-symbols' - allow_undefined_flag=unsupported - always_export_symbols=no - enable_shared_with_static_runtimes=yes - export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[BCDGRS][ ]/s/.*[ ]\([^ ]*\)/\1 DATA/;s/^.*[ ]__nm__\([^ ]*\)[ ][^ ]*/\1 DATA/;/^I[ ]/d;/^[AITW][ ]/s/.* //'\'' | sort | uniq > $export_symbols' - exclude_expsyms='[_]+GLOBAL_OFFSET_TABLE_|[_]+GLOBAL__[FID]_.*|[_]+head_[A-Za-z0-9_]+_dll|[A-Za-z0-9_]+_dll_iname' - - if $LD --help 2>&1 | $GREP 'auto-import' > /dev/null; then - archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags -o $output_objdir/$soname $wl--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' - # If the export-symbols file already is a .def file, use it as - # is; otherwise, prepend EXPORTS... - archive_expsym_cmds='if test DEF = "`$SED -n -e '\''s/^[ ]*//'\'' -e '\''/^\(;.*\)*$/d'\'' -e '\''s/^\(EXPORTS\|LIBRARY\)\([ ].*\)*$/DEF/p'\'' -e q $export_symbols`" ; then - cp $export_symbols $output_objdir/$soname.def; - else - echo EXPORTS > $output_objdir/$soname.def; - cat $export_symbols >> $output_objdir/$soname.def; - fi~ - $CC -shared $output_objdir/$soname.def $libobjs $deplibs $compiler_flags -o $output_objdir/$soname $wl--enable-auto-image-base -Xlinker --out-implib -Xlinker $lib' - else - ld_shlibs=no - fi - ;; - - haiku*) - archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - link_all_deplibs=yes - ;; - - os2*) - hardcode_libdir_flag_spec='-L$libdir' - hardcode_minus_L=yes - allow_undefined_flag=unsupported - shrext_cmds=.dll - archive_cmds='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - emxexp $libobjs | $SED /"_DLL_InitTerm"/d >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - archive_expsym_cmds='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - prefix_cmds="$SED"~ - if test EXPORTS = "`$SED 1q $export_symbols`"; then - prefix_cmds="$prefix_cmds -e 1d"; - fi~ - prefix_cmds="$prefix_cmds -e \"s/^\(.*\)$/_\1/g\""~ - cat $export_symbols | $prefix_cmds >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - old_archive_From_new_cmds='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' - enable_shared_with_static_runtimes=yes - ;; - - interix[3-9]*) - hardcode_direct=no - hardcode_shlibpath_var=no - hardcode_libdir_flag_spec='$wl-rpath,$libdir' - export_dynamic_flag_spec='$wl-E' - # Hack: On Interix 3.x, we cannot compile PIC because of a broken gcc. - # Instead, shared libraries are loaded at an image base (0x10000000 by - # default) and relocated if they conflict, which is a slow very memory - # consuming and fragmenting process. To avoid this, we pick a random, - # 256 KiB-aligned image base between 0x50000000 and 0x6FFC0000 at link - # time. Moving up from 0x10000000 also allows more sbrk(2) space. - archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - archive_expsym_cmds='sed "s|^|_|" $export_symbols >$output_objdir/$soname.expsym~$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-h,$soname $wl--retain-symbols-file,$output_objdir/$soname.expsym $wl--image-base,`expr ${RANDOM-$$} % 4096 / 2 \* 262144 + 1342177280` -o $lib' - ;; - - gnu* | linux* | tpf* | k*bsd*-gnu | kopensolaris*-gnu) - tmp_diet=no - if test linux-dietlibc = "$host_os"; then - case $cc_basename in - diet\ *) tmp_diet=yes;; # linux-dietlibc with static linking (!diet-dyn) - esac - fi - if $LD --help 2>&1 | $EGREP ': supported targets:.* elf' > /dev/null \ - && test no = "$tmp_diet" - then - tmp_addflag=' $pic_flag' - tmp_sharedflag='-shared' - case $cc_basename,$host_cpu in - pgcc*) # Portland Group C compiler - whole_archive_flag_spec='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - tmp_addflag=' $pic_flag' - ;; - pgf77* | pgf90* | pgf95* | pgfortran*) - # Portland Group f77 and f90 compilers - whole_archive_flag_spec='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - tmp_addflag=' $pic_flag -Mnomain' ;; - ecc*,ia64* | icc*,ia64*) # Intel C compiler on ia64 - tmp_addflag=' -i_dynamic' ;; - efc*,ia64* | ifort*,ia64*) # Intel Fortran compiler on ia64 - tmp_addflag=' -i_dynamic -nofor_main' ;; - ifc* | ifort*) # Intel Fortran compiler - tmp_addflag=' -nofor_main' ;; - lf95*) # Lahey Fortran 8.1 - whole_archive_flag_spec= - tmp_sharedflag='--shared' ;; - nagfor*) # NAGFOR 5.3 - tmp_sharedflag='-Wl,-shared' ;; - xl[cC]* | bgxl[cC]* | mpixl[cC]*) # IBM XL C 8.0 on PPC (deal with xlf below) - tmp_sharedflag='-qmkshrobj' - tmp_addflag= ;; - nvcc*) # Cuda Compiler Driver 2.2 - whole_archive_flag_spec='$wl--whole-archive`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - compiler_needs_object=yes - ;; - esac - case `$CC -V 2>&1 | sed 5q` in - *Sun\ C*) # Sun C 5.9 - whole_archive_flag_spec='$wl--whole-archive`new_convenience=; for conv in $convenience\"\"; do test -z \"$conv\" || new_convenience=\"$new_convenience,$conv\"; done; func_echo_all \"$new_convenience\"` $wl--no-whole-archive' - compiler_needs_object=yes - tmp_sharedflag='-G' ;; - *Sun\ F*) # Sun Fortran 8.3 - tmp_sharedflag='-G' ;; - esac - archive_cmds='$CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - - if test yes = "$supports_anon_versioning"; then - archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ - echo "local: *; };" >> $output_objdir/$libname.ver~ - $CC '"$tmp_sharedflag""$tmp_addflag"' $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-version-script $wl$output_objdir/$libname.ver -o $lib' - fi - - case $cc_basename in - tcc*) - export_dynamic_flag_spec='-rdynamic' - ;; - xlf* | bgf* | bgxlf* | mpixlf*) - # IBM XL Fortran 10.1 on PPC cannot create shared libs itself - whole_archive_flag_spec='--whole-archive$convenience --no-whole-archive' - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - archive_cmds='$LD -shared $libobjs $deplibs $linker_flags -soname $soname -o $lib' - if test yes = "$supports_anon_versioning"; then - archive_expsym_cmds='echo "{ global:" > $output_objdir/$libname.ver~ - cat $export_symbols | sed -e "s/\(.*\)/\1;/" >> $output_objdir/$libname.ver~ - echo "local: *; };" >> $output_objdir/$libname.ver~ - $LD -shared $libobjs $deplibs $linker_flags -soname $soname -version-script $output_objdir/$libname.ver -o $lib' - fi - ;; - esac - else - ld_shlibs=no - fi - ;; - - netbsd* | netbsdelf*-gnu) - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - archive_cmds='$LD -Bshareable $libobjs $deplibs $linker_flags -o $lib' - wlarc= - else - archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - fi - ;; - - solaris*) - if $LD -v 2>&1 | $GREP 'BFD 2\.8' > /dev/null; then - ld_shlibs=no - cat <<_LT_EOF 1>&2 - -*** Warning: The releases 2.8.* of the GNU linker cannot reliably -*** create shared libraries on Solaris systems. Therefore, libtool -*** is disabling shared libraries support. We urge you to upgrade GNU -*** binutils to release 2.9.1 or newer. Another option is to modify -*** your PATH or compiler configuration so that the native linker is -*** used, and then restart. - -_LT_EOF - elif $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - else - ld_shlibs=no - fi - ;; - - sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX*) - case `$LD -v 2>&1` in - *\ [01].* | *\ 2.[0-9].* | *\ 2.1[0-5].*) - ld_shlibs=no - cat <<_LT_EOF 1>&2 - -*** Warning: Releases of the GNU linker prior to 2.16.91.0.3 cannot -*** reliably create shared libraries on SCO systems. Therefore, libtool -*** is disabling shared libraries support. We urge you to upgrade GNU -*** binutils to release 2.16.91.0.3 or newer. Another option is to modify -*** your PATH or compiler configuration so that the native linker is -*** used, and then restart. - -_LT_EOF - ;; - *) - # For security reasons, it is highly recommended that you always - # use absolute paths for naming shared libraries, and exclude the - # DT_RUNPATH tag from executables and libraries. But doing so - # requires that you compile everything twice, which is a pain. - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - else - ld_shlibs=no - fi - ;; - esac - ;; - - sunos4*) - archive_cmds='$LD -assert pure-text -Bshareable -o $lib $libobjs $deplibs $linker_flags' - wlarc= - hardcode_direct=yes - hardcode_shlibpath_var=no - ;; - - *) - if $LD --help 2>&1 | $GREP ': supported targets:.* elf' > /dev/null; then - archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname $wl-retain-symbols-file $wl$export_symbols -o $lib' - else - ld_shlibs=no - fi - ;; - esac - - if test no = "$ld_shlibs"; then - runpath_var= - hardcode_libdir_flag_spec= - export_dynamic_flag_spec= - whole_archive_flag_spec= - fi - else - # PORTME fill in a description of your system's linker (not GNU ld) - case $host_os in - aix3*) - allow_undefined_flag=unsupported - always_export_symbols=yes - archive_expsym_cmds='$LD -o $output_objdir/$soname $libobjs $deplibs $linker_flags -bE:$export_symbols -T512 -H512 -bM:SRE~$AR $AR_FLAGS $lib $output_objdir/$soname' - # Note: this linker hardcodes the directories in LIBPATH if there - # are no directories specified by -L. - hardcode_minus_L=yes - if test yes = "$GCC" && test -z "$lt_prog_compiler_static"; then - # Neither direct hardcoding nor static linking is supported with a - # broken collect2. - hardcode_direct=unsupported - fi - ;; - - aix[4-9]*) - if test ia64 = "$host_cpu"; then - # On IA64, the linker does run time linking by default, so we don't - # have to do anything special. - aix_use_runtimelinking=no - exp_sym_flag='-Bexport' - no_entry_flag= - else - # If we're using GNU nm, then we don't want the "-C" option. - # -C means demangle to GNU nm, but means don't demangle to AIX nm. - # Without the "-l" option, or with the "-B" option, AIX nm treats - # weak defined symbols like other global defined symbols, whereas - # GNU nm marks them as "W". - # While the 'weak' keyword is ignored in the Export File, we need - # it in the Import File for the 'aix-soname' feature, so we have - # to replace the "-B" option with "-P" for AIX nm. - if $NM -V 2>&1 | $GREP 'GNU' > /dev/null; then - export_symbols_cmds='$NM -Bpg $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W")) && (substr(\$ 3,1,1) != ".")) { if (\$ 2 == "W") { print \$ 3 " weak" } else { print \$ 3 } } }'\'' | sort -u > $export_symbols' - else - export_symbols_cmds='`func_echo_all $NM | $SED -e '\''s/B\([^B]*\)$/P\1/'\''` -PCpgl $libobjs $convenience | awk '\''{ if (((\$ 2 == "T") || (\$ 2 == "D") || (\$ 2 == "B") || (\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) && (substr(\$ 1,1,1) != ".")) { if ((\$ 2 == "W") || (\$ 2 == "V") || (\$ 2 == "Z")) { print \$ 1 " weak" } else { print \$ 1 } } }'\'' | sort -u > $export_symbols' - fi - aix_use_runtimelinking=no - - # Test if we are trying to use run time linking or normal - # AIX style linking. If -brtl is somewhere in LDFLAGS, we - # have runtime linking enabled, and use it for executables. - # For shared libraries, we enable/disable runtime linking - # depending on the kind of the shared library created - - # when "with_aix_soname,aix_use_runtimelinking" is: - # "aix,no" lib.a(lib.so.V) shared, rtl:no, for executables - # "aix,yes" lib.so shared, rtl:yes, for executables - # lib.a static archive - # "both,no" lib.so.V(shr.o) shared, rtl:yes - # lib.a(lib.so.V) shared, rtl:no, for executables - # "both,yes" lib.so.V(shr.o) shared, rtl:yes, for executables - # lib.a(lib.so.V) shared, rtl:no - # "svr4,*" lib.so.V(shr.o) shared, rtl:yes, for executables - # lib.a static archive - case $host_os in aix4.[23]|aix4.[23].*|aix[5-9]*) - for ld_flag in $LDFLAGS; do - if (test x-brtl = "x$ld_flag" || test x-Wl,-brtl = "x$ld_flag"); then - aix_use_runtimelinking=yes - break - fi - done - if test svr4,no = "$with_aix_soname,$aix_use_runtimelinking"; then - # With aix-soname=svr4, we create the lib.so.V shared archives only, - # so we don't have lib.a shared libs to link our executables. - # We have to force runtime linking in this case. - aix_use_runtimelinking=yes - LDFLAGS="$LDFLAGS -Wl,-brtl" - fi - ;; - esac - - exp_sym_flag='-bexport' - no_entry_flag='-bnoentry' - fi - - # When large executables or shared objects are built, AIX ld can - # have problems creating the table of contents. If linking a library - # or program results in "error TOC overflow" add -mminimal-toc to - # CXXFLAGS/CFLAGS for g++/gcc. In the cases where that is not - # enough to fix the problem, add -Wl,-bbigtoc to LDFLAGS. - - archive_cmds='' - hardcode_direct=yes - hardcode_direct_absolute=yes - hardcode_libdir_separator=':' - link_all_deplibs=yes - file_list_spec='$wl-f,' - case $with_aix_soname,$aix_use_runtimelinking in - aix,*) ;; # traditional, no import file - svr4,* | *,yes) # use import file - # The Import File defines what to hardcode. - hardcode_direct=no - hardcode_direct_absolute=no - ;; - esac - - if test yes = "$GCC"; then - case $host_os in aix4.[012]|aix4.[012].*) - # We only want to do this on AIX 4.2 and lower, the check - # below for broken collect2 doesn't work under 4.3+ - collect2name=`$CC -print-prog-name=collect2` - if test -f "$collect2name" && - strings "$collect2name" | $GREP resolve_lib_name >/dev/null - then - # We have reworked collect2 - : - else - # We have old collect2 - hardcode_direct=unsupported - # It fails to find uninstalled libraries when the uninstalled - # path is not listed in the libpath. Setting hardcode_minus_L - # to unsupported forces relinking - hardcode_minus_L=yes - hardcode_libdir_flag_spec='-L$libdir' - hardcode_libdir_separator= - fi - ;; - esac - shared_flag='-shared' - if test yes = "$aix_use_runtimelinking"; then - shared_flag="$shared_flag "'$wl-G' - fi - # Need to ensure runtime linking is disabled for the traditional - # shared library, or the linker may eventually find shared libraries - # /with/ Import File - we do not want to mix them. - shared_flag_aix='-shared' - shared_flag_svr4='-shared $wl-G' - else - # not using gcc - if test ia64 = "$host_cpu"; then - # VisualAge C++, Version 5.5 for AIX 5L for IA-64, Beta 3 Release - # chokes on -Wl,-G. The following line is correct: - shared_flag='-G' - else - if test yes = "$aix_use_runtimelinking"; then - shared_flag='$wl-G' - else - shared_flag='$wl-bM:SRE' - fi - shared_flag_aix='$wl-bM:SRE' - shared_flag_svr4='$wl-G' - fi - fi - - export_dynamic_flag_spec='$wl-bexpall' - # It seems that -bexpall does not export symbols beginning with - # underscore (_), so it is better to generate a list of symbols to export. - always_export_symbols=yes - if test aix,yes = "$with_aix_soname,$aix_use_runtimelinking"; then - # Warning - without using the other runtime loading flags (-brtl), - # -berok will link without error, but may produce a broken library. - allow_undefined_flag='-berok' - # Determine the default libpath from the value encoded in an - # empty executable. - if test set = "${lt_cv_aix_libpath+set}"; then - aix_libpath=$lt_cv_aix_libpath -else - if ${lt_cv_aix_libpath_+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - - lt_aix_libpath_sed=' - /Import File Strings/,/^$/ { - /^0/ { - s/^0 *\([^ ]*\) *$/\1/ - p - } - }' - lt_cv_aix_libpath_=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` - # Check for a 64-bit object if we didn't find anything. - if test -z "$lt_cv_aix_libpath_"; then - lt_cv_aix_libpath_=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` - fi -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - if test -z "$lt_cv_aix_libpath_"; then - lt_cv_aix_libpath_=/usr/lib:/lib - fi - -fi - - aix_libpath=$lt_cv_aix_libpath_ -fi - - hardcode_libdir_flag_spec='$wl-blibpath:$libdir:'"$aix_libpath" - archive_expsym_cmds='$CC -o $output_objdir/$soname $libobjs $deplibs $wl'$no_entry_flag' $compiler_flags `if test -n "$allow_undefined_flag"; then func_echo_all "$wl$allow_undefined_flag"; else :; fi` $wl'$exp_sym_flag:\$export_symbols' '$shared_flag - else - if test ia64 = "$host_cpu"; then - hardcode_libdir_flag_spec='$wl-R $libdir:/usr/lib:/lib' - allow_undefined_flag="-z nodefs" - archive_expsym_cmds="\$CC $shared_flag"' -o $output_objdir/$soname $libobjs $deplibs '"\$wl$no_entry_flag"' $compiler_flags $wl$allow_undefined_flag '"\$wl$exp_sym_flag:\$export_symbols" - else - # Determine the default libpath from the value encoded in an - # empty executable. - if test set = "${lt_cv_aix_libpath+set}"; then - aix_libpath=$lt_cv_aix_libpath -else - if ${lt_cv_aix_libpath_+:} false; then : - $as_echo_n "(cached) " >&6 -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - - lt_aix_libpath_sed=' - /Import File Strings/,/^$/ { - /^0/ { - s/^0 *\([^ ]*\) *$/\1/ - p - } - }' - lt_cv_aix_libpath_=`dump -H conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` - # Check for a 64-bit object if we didn't find anything. - if test -z "$lt_cv_aix_libpath_"; then - lt_cv_aix_libpath_=`dump -HX64 conftest$ac_exeext 2>/dev/null | $SED -n -e "$lt_aix_libpath_sed"` - fi -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - if test -z "$lt_cv_aix_libpath_"; then - lt_cv_aix_libpath_=/usr/lib:/lib - fi - -fi - - aix_libpath=$lt_cv_aix_libpath_ -fi - - hardcode_libdir_flag_spec='$wl-blibpath:$libdir:'"$aix_libpath" - # Warning - without using the other run time loading flags, - # -berok will link without error, but may produce a broken library. - no_undefined_flag=' $wl-bernotok' - allow_undefined_flag=' $wl-berok' - if test yes = "$with_gnu_ld"; then - # We only use this code for GNU lds that support --whole-archive. - whole_archive_flag_spec='$wl--whole-archive$convenience $wl--no-whole-archive' - else - # Exported symbols can be pulled into shared objects from archives - whole_archive_flag_spec='$convenience' - fi - archive_cmds_need_lc=yes - archive_expsym_cmds='$RM -r $output_objdir/$realname.d~$MKDIR $output_objdir/$realname.d' - # -brtl affects multiple linker settings, -berok does not and is overridden later - compiler_flags_filtered='`func_echo_all "$compiler_flags " | $SED -e "s%-brtl\\([, ]\\)%-berok\\1%g"`' - if test svr4 != "$with_aix_soname"; then - # This is similar to how AIX traditionally builds its shared libraries. - archive_expsym_cmds="$archive_expsym_cmds"'~$CC '$shared_flag_aix' -o $output_objdir/$realname.d/$soname $libobjs $deplibs $wl-bnoentry '$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$AR $AR_FLAGS $output_objdir/$libname$release.a $output_objdir/$realname.d/$soname' - fi - if test aix != "$with_aix_soname"; then - archive_expsym_cmds="$archive_expsym_cmds"'~$CC '$shared_flag_svr4' -o $output_objdir/$realname.d/$shared_archive_member_spec.o $libobjs $deplibs $wl-bnoentry '$compiler_flags_filtered'$wl-bE:$export_symbols$allow_undefined_flag~$STRIP -e $output_objdir/$realname.d/$shared_archive_member_spec.o~( func_echo_all "#! $soname($shared_archive_member_spec.o)"; if test shr_64 = "$shared_archive_member_spec"; then func_echo_all "# 64"; else func_echo_all "# 32"; fi; cat $export_symbols ) > $output_objdir/$realname.d/$shared_archive_member_spec.imp~$AR $AR_FLAGS $output_objdir/$soname $output_objdir/$realname.d/$shared_archive_member_spec.o $output_objdir/$realname.d/$shared_archive_member_spec.imp' - else - # used by -dlpreopen to get the symbols - archive_expsym_cmds="$archive_expsym_cmds"'~$MV $output_objdir/$realname.d/$soname $output_objdir' - fi - archive_expsym_cmds="$archive_expsym_cmds"'~$RM -r $output_objdir/$realname.d' - fi - fi - ;; - - amigaos*) - case $host_cpu in - powerpc) - # see comment about AmigaOS4 .so support - archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags $wl-soname $wl$soname -o $lib' - archive_expsym_cmds='' - ;; - m68k) - archive_cmds='$RM $output_objdir/a2ixlibrary.data~$ECHO "#define NAME $libname" > $output_objdir/a2ixlibrary.data~$ECHO "#define LIBRARY_ID 1" >> $output_objdir/a2ixlibrary.data~$ECHO "#define VERSION $major" >> $output_objdir/a2ixlibrary.data~$ECHO "#define REVISION $revision" >> $output_objdir/a2ixlibrary.data~$AR $AR_FLAGS $lib $libobjs~$RANLIB $lib~(cd $output_objdir && a2ixlibrary -32)' - hardcode_libdir_flag_spec='-L$libdir' - hardcode_minus_L=yes - ;; - esac - ;; - - bsdi[45]*) - export_dynamic_flag_spec=-rdynamic - ;; - - cygwin* | mingw* | pw32* | cegcc*) - # When not using gcc, we currently assume that we are using - # Microsoft Visual C++. - # hardcode_libdir_flag_spec is actually meaningless, as there is - # no search path for DLLs. - case $cc_basename in - cl*) - # Native MSVC - hardcode_libdir_flag_spec=' ' - allow_undefined_flag=unsupported - always_export_symbols=yes - file_list_spec='@' - # Tell ltmain to make .lib files, not .a files. - libext=lib - # Tell ltmain to make .dll files, not .so files. - shrext_cmds=.dll - # FIXME: Setting linknames here is a bad hack. - archive_cmds='$CC -o $output_objdir/$soname $libobjs $compiler_flags $deplibs -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~linknames=' - archive_expsym_cmds='if test DEF = "`$SED -n -e '\''s/^[ ]*//'\'' -e '\''/^\(;.*\)*$/d'\'' -e '\''s/^\(EXPORTS\|LIBRARY\)\([ ].*\)*$/DEF/p'\'' -e q $export_symbols`" ; then - cp "$export_symbols" "$output_objdir/$soname.def"; - echo "$tool_output_objdir$soname.def" > "$output_objdir/$soname.exp"; - else - $SED -e '\''s/^/-link -EXPORT:/'\'' < $export_symbols > $output_objdir/$soname.exp; - fi~ - $CC -o $tool_output_objdir$soname $libobjs $compiler_flags $deplibs "@$tool_output_objdir$soname.exp" -Wl,-DLL,-IMPLIB:"$tool_output_objdir$libname.dll.lib"~ - linknames=' - # The linker will not automatically build a static lib if we build a DLL. - # _LT_TAGVAR(old_archive_from_new_cmds, )='true' - enable_shared_with_static_runtimes=yes - exclude_expsyms='_NULL_IMPORT_DESCRIPTOR|_IMPORT_DESCRIPTOR_.*' - export_symbols_cmds='$NM $libobjs $convenience | $global_symbol_pipe | $SED -e '\''/^[BCDGRS][ ]/s/.*[ ]\([^ ]*\)/\1,DATA/'\'' | $SED -e '\''/^[AITW][ ]/s/.*[ ]//'\'' | sort | uniq > $export_symbols' - # Don't use ranlib - old_postinstall_cmds='chmod 644 $oldlib' - postlink_cmds='lt_outputfile="@OUTPUT@"~ - lt_tool_outputfile="@TOOL_OUTPUT@"~ - case $lt_outputfile in - *.exe|*.EXE) ;; - *) - lt_outputfile=$lt_outputfile.exe - lt_tool_outputfile=$lt_tool_outputfile.exe - ;; - esac~ - if test : != "$MANIFEST_TOOL" && test -f "$lt_outputfile.manifest"; then - $MANIFEST_TOOL -manifest "$lt_tool_outputfile.manifest" -outputresource:"$lt_tool_outputfile" || exit 1; - $RM "$lt_outputfile.manifest"; - fi' - ;; - *) - # Assume MSVC wrapper - hardcode_libdir_flag_spec=' ' - allow_undefined_flag=unsupported - # Tell ltmain to make .lib files, not .a files. - libext=lib - # Tell ltmain to make .dll files, not .so files. - shrext_cmds=.dll - # FIXME: Setting linknames here is a bad hack. - archive_cmds='$CC -o $lib $libobjs $compiler_flags `func_echo_all "$deplibs" | $SED '\''s/ -lc$//'\''` -link -dll~linknames=' - # The linker will automatically build a .lib file if we build a DLL. - old_archive_from_new_cmds='true' - # FIXME: Should let the user specify the lib program. - old_archive_cmds='lib -OUT:$oldlib$oldobjs$old_deplibs' - enable_shared_with_static_runtimes=yes - ;; - esac - ;; - - darwin* | rhapsody*) - - - archive_cmds_need_lc=no - hardcode_direct=no - hardcode_automatic=yes - hardcode_shlibpath_var=unsupported - if test yes = "$lt_cv_ld_force_load"; then - whole_archive_flag_spec='`for conv in $convenience\"\"; do test -n \"$conv\" && new_convenience=\"$new_convenience $wl-force_load,$conv\"; done; func_echo_all \"$new_convenience\"`' - - else - whole_archive_flag_spec='' - fi - link_all_deplibs=yes - allow_undefined_flag=$_lt_dar_allow_undefined - case $cc_basename in - ifort*|nagfor*) _lt_dar_can_shared=yes ;; - *) _lt_dar_can_shared=$GCC ;; - esac - if test yes = "$_lt_dar_can_shared"; then - output_verbose_link_cmd=func_echo_all - archive_cmds="\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dsymutil" - module_cmds="\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dsymutil" - archive_expsym_cmds="sed 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC -dynamiclib \$allow_undefined_flag -o \$lib \$libobjs \$deplibs \$compiler_flags -install_name \$rpath/\$soname \$verstring $_lt_dar_single_mod$_lt_dar_export_syms$_lt_dsymutil" - module_expsym_cmds="sed -e 's|^|_|' < \$export_symbols > \$output_objdir/\$libname-symbols.expsym~\$CC \$allow_undefined_flag -o \$lib -bundle \$libobjs \$deplibs \$compiler_flags$_lt_dar_export_syms$_lt_dsymutil" - - else - ld_shlibs=no - fi - - ;; - - dgux*) - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_libdir_flag_spec='-L$libdir' - hardcode_shlibpath_var=no - ;; - - # FreeBSD 2.2.[012] allows us to include c++rt0.o to get C++ constructor - # support. Future versions do this automatically, but an explicit c++rt0.o - # does not break anything, and helps significantly (at the cost of a little - # extra space). - freebsd2.2*) - archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags /usr/lib/c++rt0.o' - hardcode_libdir_flag_spec='-R$libdir' - hardcode_direct=yes - hardcode_shlibpath_var=no - ;; - - # Unfortunately, older versions of FreeBSD 2 do not have this feature. - freebsd2.*) - archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' - hardcode_direct=yes - hardcode_minus_L=yes - hardcode_shlibpath_var=no - ;; - - # FreeBSD 3 and greater uses gcc -shared to do shared libraries. - freebsd* | dragonfly*) - archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - hardcode_libdir_flag_spec='-R$libdir' - hardcode_direct=yes - hardcode_shlibpath_var=no - ;; - - hpux9*) - if test yes = "$GCC"; then - archive_cmds='$RM $output_objdir/$soname~$CC -shared $pic_flag $wl+b $wl$install_libdir -o $output_objdir/$soname $libobjs $deplibs $compiler_flags~test "x$output_objdir/$soname" = "x$lib" || mv $output_objdir/$soname $lib' - else - archive_cmds='$RM $output_objdir/$soname~$LD -b +b $install_libdir -o $output_objdir/$soname $libobjs $deplibs $linker_flags~test "x$output_objdir/$soname" = "x$lib" || mv $output_objdir/$soname $lib' - fi - hardcode_libdir_flag_spec='$wl+b $wl$libdir' - hardcode_libdir_separator=: - hardcode_direct=yes - - # hardcode_minus_L: Not really in the search PATH, - # but as the default location of the library. - hardcode_minus_L=yes - export_dynamic_flag_spec='$wl-E' - ;; - - hpux10*) - if test yes,no = "$GCC,$with_gnu_ld"; then - archive_cmds='$CC -shared $pic_flag $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags' - else - archive_cmds='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags' - fi - if test no = "$with_gnu_ld"; then - hardcode_libdir_flag_spec='$wl+b $wl$libdir' - hardcode_libdir_separator=: - hardcode_direct=yes - hardcode_direct_absolute=yes - export_dynamic_flag_spec='$wl-E' - # hardcode_minus_L: Not really in the search PATH, - # but as the default location of the library. - hardcode_minus_L=yes - fi - ;; - - hpux11*) - if test yes,no = "$GCC,$with_gnu_ld"; then - case $host_cpu in - hppa*64*) - archive_cmds='$CC -shared $wl+h $wl$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - ia64*) - archive_cmds='$CC -shared $pic_flag $wl+h $wl$soname $wl+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' - ;; - *) - archive_cmds='$CC -shared $pic_flag $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags' - ;; - esac - else - case $host_cpu in - hppa*64*) - archive_cmds='$CC -b $wl+h $wl$soname -o $lib $libobjs $deplibs $compiler_flags' - ;; - ia64*) - archive_cmds='$CC -b $wl+h $wl$soname $wl+nodefaultrpath -o $lib $libobjs $deplibs $compiler_flags' - ;; - *) - - # Older versions of the 11.00 compiler do not understand -b yet - # (HP92453-01 A.11.01.20 doesn't, HP92453-01 B.11.X.35175-35176.GP does) - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $CC understands -b" >&5 -$as_echo_n "checking if $CC understands -b... " >&6; } -if ${lt_cv_prog_compiler__b+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_prog_compiler__b=no - save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS -b" - echo "$lt_simple_link_test_code" > conftest.$ac_ext - if (eval $ac_link 2>conftest.err) && test -s conftest$ac_exeext; then - # The linker can only warn and ignore the option if not recognized - # So say no if there are warnings - if test -s conftest.err; then - # Append any errors to the config.log. - cat conftest.err 1>&5 - $ECHO "$_lt_linker_boilerplate" | $SED '/^$/d' > conftest.exp - $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 - if diff conftest.exp conftest.er2 >/dev/null; then - lt_cv_prog_compiler__b=yes - fi - else - lt_cv_prog_compiler__b=yes - fi - fi - $RM -r conftest* - LDFLAGS=$save_LDFLAGS - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_prog_compiler__b" >&5 -$as_echo "$lt_cv_prog_compiler__b" >&6; } - -if test yes = "$lt_cv_prog_compiler__b"; then - archive_cmds='$CC -b $wl+h $wl$soname $wl+b $wl$install_libdir -o $lib $libobjs $deplibs $compiler_flags' -else - archive_cmds='$LD -b +h $soname +b $install_libdir -o $lib $libobjs $deplibs $linker_flags' -fi - - ;; - esac - fi - if test no = "$with_gnu_ld"; then - hardcode_libdir_flag_spec='$wl+b $wl$libdir' - hardcode_libdir_separator=: - - case $host_cpu in - hppa*64*|ia64*) - hardcode_direct=no - hardcode_shlibpath_var=no - ;; - *) - hardcode_direct=yes - hardcode_direct_absolute=yes - export_dynamic_flag_spec='$wl-E' - - # hardcode_minus_L: Not really in the search PATH, - # but as the default location of the library. - hardcode_minus_L=yes - ;; - esac - fi - ;; - - irix5* | irix6* | nonstopux*) - if test yes = "$GCC"; then - archive_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - # Try to use the -exported_symbol ld option, if it does not - # work, assume that -exports_file does not work either and - # implicitly export all symbols. - # This should be the same for all languages, so no per-tag cache variable. - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether the $host_os linker accepts -exported_symbol" >&5 -$as_echo_n "checking whether the $host_os linker accepts -exported_symbol... " >&6; } -if ${lt_cv_irix_exported_symbol+:} false; then : - $as_echo_n "(cached) " >&6 -else - save_LDFLAGS=$LDFLAGS - LDFLAGS="$LDFLAGS -shared $wl-exported_symbol ${wl}foo $wl-update_registry $wl/dev/null" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -int foo (void) { return 0; } -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - lt_cv_irix_exported_symbol=yes -else - lt_cv_irix_exported_symbol=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$save_LDFLAGS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_irix_exported_symbol" >&5 -$as_echo "$lt_cv_irix_exported_symbol" >&6; } - if test yes = "$lt_cv_irix_exported_symbol"; then - archive_expsym_cmds='$CC -shared $pic_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations $wl-exports_file $wl$export_symbols -o $lib' - fi - link_all_deplibs=no - else - archive_cmds='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - archive_expsym_cmds='$CC -shared $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -exports_file $export_symbols -o $lib' - fi - archive_cmds_need_lc='no' - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - hardcode_libdir_separator=: - inherit_rpath=yes - link_all_deplibs=yes - ;; - - linux*) - case $cc_basename in - tcc*) - # Fabrice Bellard et al's Tiny C Compiler - ld_shlibs=yes - archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - ;; - esac - ;; - - netbsd* | netbsdelf*-gnu) - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - archive_cmds='$LD -Bshareable -o $lib $libobjs $deplibs $linker_flags' # a.out - else - archive_cmds='$LD -shared -o $lib $libobjs $deplibs $linker_flags' # ELF - fi - hardcode_libdir_flag_spec='-R$libdir' - hardcode_direct=yes - hardcode_shlibpath_var=no - ;; - - newsos6) - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_direct=yes - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - hardcode_libdir_separator=: - hardcode_shlibpath_var=no - ;; - - *nto* | *qnx*) - ;; - - openbsd* | bitrig*) - if test -f /usr/libexec/ld.so; then - hardcode_direct=yes - hardcode_shlibpath_var=no - hardcode_direct_absolute=yes - if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`"; then - archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags $wl-retain-symbols-file,$export_symbols' - hardcode_libdir_flag_spec='$wl-rpath,$libdir' - export_dynamic_flag_spec='$wl-E' - else - archive_cmds='$CC -shared $pic_flag -o $lib $libobjs $deplibs $compiler_flags' - hardcode_libdir_flag_spec='$wl-rpath,$libdir' - fi - else - ld_shlibs=no - fi - ;; - - os2*) - hardcode_libdir_flag_spec='-L$libdir' - hardcode_minus_L=yes - allow_undefined_flag=unsupported - shrext_cmds=.dll - archive_cmds='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - emxexp $libobjs | $SED /"_DLL_InitTerm"/d >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - archive_expsym_cmds='$ECHO "LIBRARY ${soname%$shared_ext} INITINSTANCE TERMINSTANCE" > $output_objdir/$libname.def~ - $ECHO "DESCRIPTION \"$libname\"" >> $output_objdir/$libname.def~ - $ECHO "DATA MULTIPLE NONSHARED" >> $output_objdir/$libname.def~ - $ECHO EXPORTS >> $output_objdir/$libname.def~ - prefix_cmds="$SED"~ - if test EXPORTS = "`$SED 1q $export_symbols`"; then - prefix_cmds="$prefix_cmds -e 1d"; - fi~ - prefix_cmds="$prefix_cmds -e \"s/^\(.*\)$/_\1/g\""~ - cat $export_symbols | $prefix_cmds >> $output_objdir/$libname.def~ - $CC -Zdll -Zcrtdll -o $output_objdir/$soname $libobjs $deplibs $compiler_flags $output_objdir/$libname.def~ - emximp -o $lib $output_objdir/$libname.def' - old_archive_From_new_cmds='emximp -o $output_objdir/${libname}_dll.a $output_objdir/$libname.def' - enable_shared_with_static_runtimes=yes - ;; - - osf3*) - if test yes = "$GCC"; then - allow_undefined_flag=' $wl-expect_unresolved $wl\*' - archive_cmds='$CC -shared$allow_undefined_flag $libobjs $deplibs $compiler_flags $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - else - allow_undefined_flag=' -expect_unresolved \*' - archive_cmds='$CC -shared$allow_undefined_flag $libobjs $deplibs $compiler_flags -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - fi - archive_cmds_need_lc='no' - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - hardcode_libdir_separator=: - ;; - - osf4* | osf5*) # as osf3* with the addition of -msym flag - if test yes = "$GCC"; then - allow_undefined_flag=' $wl-expect_unresolved $wl\*' - archive_cmds='$CC -shared$allow_undefined_flag $pic_flag $libobjs $deplibs $compiler_flags $wl-msym $wl-soname $wl$soname `test -n "$verstring" && func_echo_all "$wl-set_version $wl$verstring"` $wl-update_registry $wl$output_objdir/so_locations -o $lib' - hardcode_libdir_flag_spec='$wl-rpath $wl$libdir' - else - allow_undefined_flag=' -expect_unresolved \*' - archive_cmds='$CC -shared$allow_undefined_flag $libobjs $deplibs $compiler_flags -msym -soname $soname `test -n "$verstring" && func_echo_all "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib' - archive_expsym_cmds='for i in `cat $export_symbols`; do printf "%s %s\\n" -exported_symbol "\$i" >> $lib.exp; done; printf "%s\\n" "-hidden">> $lib.exp~ - $CC -shared$allow_undefined_flag $wl-input $wl$lib.exp $compiler_flags $libobjs $deplibs -soname $soname `test -n "$verstring" && $ECHO "-set_version $verstring"` -update_registry $output_objdir/so_locations -o $lib~$RM $lib.exp' - - # Both c and cxx compiler support -rpath directly - hardcode_libdir_flag_spec='-rpath $libdir' - fi - archive_cmds_need_lc='no' - hardcode_libdir_separator=: - ;; - - solaris*) - no_undefined_flag=' -z defs' - if test yes = "$GCC"; then - wlarc='$wl' - archive_cmds='$CC -shared $pic_flag $wl-z ${wl}text $wl-h $wl$soname -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -shared $pic_flag $wl-z ${wl}text $wl-M $wl$lib.exp $wl-h $wl$soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' - else - case `$CC -V 2>&1` in - *"Compilers 5.0"*) - wlarc='' - archive_cmds='$LD -G$allow_undefined_flag -h $soname -o $lib $libobjs $deplibs $linker_flags' - archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $LD -G$allow_undefined_flag -M $lib.exp -h $soname -o $lib $libobjs $deplibs $linker_flags~$RM $lib.exp' - ;; - *) - wlarc='$wl' - archive_cmds='$CC -G$allow_undefined_flag -h $soname -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='echo "{ global:" > $lib.exp~cat $export_symbols | $SED -e "s/\(.*\)/\1;/" >> $lib.exp~echo "local: *; };" >> $lib.exp~ - $CC -G$allow_undefined_flag -M $lib.exp -h $soname -o $lib $libobjs $deplibs $compiler_flags~$RM $lib.exp' - ;; - esac - fi - hardcode_libdir_flag_spec='-R$libdir' - hardcode_shlibpath_var=no - case $host_os in - solaris2.[0-5] | solaris2.[0-5].*) ;; - *) - # The compiler driver will combine and reorder linker options, - # but understands '-z linker_flag'. GCC discards it without '$wl', - # but is careful enough not to reorder. - # Supported since Solaris 2.6 (maybe 2.5.1?) - if test yes = "$GCC"; then - whole_archive_flag_spec='$wl-z ${wl}allextract$convenience $wl-z ${wl}defaultextract' - else - whole_archive_flag_spec='-z allextract$convenience -z defaultextract' - fi - ;; - esac - link_all_deplibs=yes - ;; - - sunos4*) - if test sequent = "$host_vendor"; then - # Use $CC to link under sequent, because it throws in some extra .o - # files that make .init and .fini sections work. - archive_cmds='$CC -G $wl-h $soname -o $lib $libobjs $deplibs $compiler_flags' - else - archive_cmds='$LD -assert pure-text -Bstatic -o $lib $libobjs $deplibs $linker_flags' - fi - hardcode_libdir_flag_spec='-L$libdir' - hardcode_direct=yes - hardcode_minus_L=yes - hardcode_shlibpath_var=no - ;; - - sysv4) - case $host_vendor in - sni) - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_direct=yes # is this really true??? - ;; - siemens) - ## LD is ld it makes a PLAMLIB - ## CC just makes a GrossModule. - archive_cmds='$LD -G -o $lib $libobjs $deplibs $linker_flags' - reload_cmds='$CC -r -o $output$reload_objs' - hardcode_direct=no - ;; - motorola) - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_direct=no #Motorola manual says yes, but my tests say they lie - ;; - esac - runpath_var='LD_RUN_PATH' - hardcode_shlibpath_var=no - ;; - - sysv4.3*) - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_shlibpath_var=no - export_dynamic_flag_spec='-Bexport' - ;; - - sysv4*MP*) - if test -d /usr/nec; then - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_shlibpath_var=no - runpath_var=LD_RUN_PATH - hardcode_runpath_var=yes - ld_shlibs=yes - fi - ;; - - sysv4*uw2* | sysv5OpenUNIX* | sysv5UnixWare7.[01].[10]* | unixware7* | sco3.2v5.0.[024]*) - no_undefined_flag='$wl-z,text' - archive_cmds_need_lc=no - hardcode_shlibpath_var=no - runpath_var='LD_RUN_PATH' - - if test yes = "$GCC"; then - archive_cmds='$CC -shared $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='$CC -shared $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - else - archive_cmds='$CC -G $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='$CC -G $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - fi - ;; - - sysv5* | sco3.2v5* | sco5v6*) - # Note: We CANNOT use -z defs as we might desire, because we do not - # link with -lc, and that would cause any symbols used from libc to - # always be unresolved, which means just about no library would - # ever link correctly. If we're not using GNU ld we use -z text - # though, which does catch some bad symbols but isn't as heavy-handed - # as -z defs. - no_undefined_flag='$wl-z,text' - allow_undefined_flag='$wl-z,nodefs' - archive_cmds_need_lc=no - hardcode_shlibpath_var=no - hardcode_libdir_flag_spec='$wl-R,$libdir' - hardcode_libdir_separator=':' - link_all_deplibs=yes - export_dynamic_flag_spec='$wl-Bexport' - runpath_var='LD_RUN_PATH' - - if test yes = "$GCC"; then - archive_cmds='$CC -shared $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='$CC -shared $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - else - archive_cmds='$CC -G $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - archive_expsym_cmds='$CC -G $wl-Bexport:$export_symbols $wl-h,$soname -o $lib $libobjs $deplibs $compiler_flags' - fi - ;; - - uts4*) - archive_cmds='$LD -G -h $soname -o $lib $libobjs $deplibs $linker_flags' - hardcode_libdir_flag_spec='-L$libdir' - hardcode_shlibpath_var=no - ;; - - *) - ld_shlibs=no - ;; - esac - - if test sni = "$host_vendor"; then - case $host in - sysv4 | sysv4.2uw2* | sysv4.3* | sysv5*) - export_dynamic_flag_spec='$wl-Blargedynsym' - ;; - esac - fi - fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ld_shlibs" >&5 -$as_echo "$ld_shlibs" >&6; } -test no = "$ld_shlibs" && can_build_shared=no - -with_gnu_ld=$with_gnu_ld - - - - - - - - - - - - - - - -# -# Do we need to explicitly link libc? -# -case "x$archive_cmds_need_lc" in -x|xyes) - # Assume -lc should be added - archive_cmds_need_lc=yes - - if test yes,yes = "$GCC,$enable_shared"; then - case $archive_cmds in - *'~'*) - # FIXME: we may have to deal with multi-command sequences. - ;; - '$CC '*) - # Test whether the compiler implicitly links with -lc since on some - # systems, -lgcc has to come before -lc. If gcc already passes -lc - # to ld, don't add -lc before -lgcc. - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether -lc should be explicitly linked in" >&5 -$as_echo_n "checking whether -lc should be explicitly linked in... " >&6; } -if ${lt_cv_archive_cmds_need_lc+:} false; then : - $as_echo_n "(cached) " >&6 -else - $RM conftest* - echo "$lt_simple_compile_test_code" > conftest.$ac_ext - - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 - (eval $ac_compile) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } 2>conftest.err; then - soname=conftest - lib=conftest - libobjs=conftest.$ac_objext - deplibs= - wl=$lt_prog_compiler_wl - pic_flag=$lt_prog_compiler_pic - compiler_flags=-v - linker_flags=-v - verstring= - output_objdir=. - libname=conftest - lt_save_allow_undefined_flag=$allow_undefined_flag - allow_undefined_flag= - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$archive_cmds 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1\""; } >&5 - (eval $archive_cmds 2\>\&1 \| $GREP \" -lc \" \>/dev/null 2\>\&1) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } - then - lt_cv_archive_cmds_need_lc=no - else - lt_cv_archive_cmds_need_lc=yes - fi - allow_undefined_flag=$lt_save_allow_undefined_flag - else - cat conftest.err 1>&5 - fi - $RM conftest* - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_archive_cmds_need_lc" >&5 -$as_echo "$lt_cv_archive_cmds_need_lc" >&6; } - archive_cmds_need_lc=$lt_cv_archive_cmds_need_lc - ;; - esac - fi - ;; -esac - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking dynamic linker characteristics" >&5 -$as_echo_n "checking dynamic linker characteristics... " >&6; } - -if test yes = "$GCC"; then - case $host_os in - darwin*) lt_awk_arg='/^libraries:/,/LR/' ;; - *) lt_awk_arg='/^libraries:/' ;; - esac - case $host_os in - mingw* | cegcc*) lt_sed_strip_eq='s|=\([A-Za-z]:\)|\1|g' ;; - *) lt_sed_strip_eq='s|=/|/|g' ;; - esac - lt_search_path_spec=`$CC -print-search-dirs | awk $lt_awk_arg | $SED -e "s/^libraries://" -e $lt_sed_strip_eq` - case $lt_search_path_spec in - *\;*) - # if the path contains ";" then we assume it to be the separator - # otherwise default to the standard path separator (i.e. ":") - it is - # assumed that no part of a normal pathname contains ";" but that should - # okay in the real world where ";" in dirpaths is itself problematic. - lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED 's/;/ /g'` - ;; - *) - lt_search_path_spec=`$ECHO "$lt_search_path_spec" | $SED "s/$PATH_SEPARATOR/ /g"` - ;; - esac - # Ok, now we have the path, separated by spaces, we can step through it - # and add multilib dir if necessary... - lt_tmp_lt_search_path_spec= - lt_multi_os_dir=/`$CC $CPPFLAGS $CFLAGS $LDFLAGS -print-multi-os-directory 2>/dev/null` - # ...but if some path component already ends with the multilib dir we assume - # that all is fine and trust -print-search-dirs as is (GCC 4.2? or newer). - case "$lt_multi_os_dir; $lt_search_path_spec " in - "/; "* | "/.; "* | "/./; "* | *"$lt_multi_os_dir "* | *"$lt_multi_os_dir/ "*) - lt_multi_os_dir= - ;; - esac - for lt_sys_path in $lt_search_path_spec; do - if test -d "$lt_sys_path$lt_multi_os_dir"; then - lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path$lt_multi_os_dir" - elif test -n "$lt_multi_os_dir"; then - test -d "$lt_sys_path" && \ - lt_tmp_lt_search_path_spec="$lt_tmp_lt_search_path_spec $lt_sys_path" - fi - done - lt_search_path_spec=`$ECHO "$lt_tmp_lt_search_path_spec" | awk ' -BEGIN {RS = " "; FS = "/|\n";} { - lt_foo = ""; - lt_count = 0; - for (lt_i = NF; lt_i > 0; lt_i--) { - if ($lt_i != "" && $lt_i != ".") { - if ($lt_i == "..") { - lt_count++; - } else { - if (lt_count == 0) { - lt_foo = "/" $lt_i lt_foo; - } else { - lt_count--; - } - } - } - } - if (lt_foo != "") { lt_freq[lt_foo]++; } - if (lt_freq[lt_foo] == 1) { print lt_foo; } -}'` - # AWK program above erroneously prepends '/' to C:/dos/paths - # for these hosts. - case $host_os in - mingw* | cegcc*) lt_search_path_spec=`$ECHO "$lt_search_path_spec" |\ - $SED 's|/\([A-Za-z]:\)|\1|g'` ;; - esac - sys_lib_search_path_spec=`$ECHO "$lt_search_path_spec" | $lt_NL2SP` -else - sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib" -fi -library_names_spec= -libname_spec='lib$name' -soname_spec= -shrext_cmds=.so -postinstall_cmds= -postuninstall_cmds= -finish_cmds= -finish_eval= -shlibpath_var= -shlibpath_overrides_runpath=unknown -version_type=none -dynamic_linker="$host_os ld.so" -sys_lib_dlsearch_path_spec="/lib /usr/lib" -need_lib_prefix=unknown -hardcode_into_libs=no - -# when you set need_version to no, make sure it does not cause -set_version -# flags to be left without arguments -need_version=unknown - - - -case $host_os in -aix3*) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname.a' - shlibpath_var=LIBPATH - - # AIX 3 has no versioning support, so we append a major version to the name. - soname_spec='$libname$release$shared_ext$major' - ;; - -aix[4-9]*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - hardcode_into_libs=yes - if test ia64 = "$host_cpu"; then - # AIX 5 supports IA64 - library_names_spec='$libname$release$shared_ext$major $libname$release$shared_ext$versuffix $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - else - # With GCC up to 2.95.x, collect2 would create an import file - # for dependence libraries. The import file would start with - # the line '#! .'. This would cause the generated library to - # depend on '.', always an invalid library. This was fixed in - # development snapshots of GCC prior to 3.0. - case $host_os in - aix4 | aix4.[01] | aix4.[01].*) - if { echo '#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 97)' - echo ' yes ' - echo '#endif'; } | $CC -E - | $GREP yes > /dev/null; then - : - else - can_build_shared=no - fi - ;; - esac - # Using Import Files as archive members, it is possible to support - # filename-based versioning of shared library archives on AIX. While - # this would work for both with and without runtime linking, it will - # prevent static linking of such archives. So we do filename-based - # shared library versioning with .so extension only, which is used - # when both runtime linking and shared linking is enabled. - # Unfortunately, runtime linking may impact performance, so we do - # not want this to be the default eventually. Also, we use the - # versioned .so libs for executables only if there is the -brtl - # linker flag in LDFLAGS as well, or --with-aix-soname=svr4 only. - # To allow for filename-based versioning support, we need to create - # libNAME.so.V as an archive file, containing: - # *) an Import File, referring to the versioned filename of the - # archive as well as the shared archive member, telling the - # bitwidth (32 or 64) of that shared object, and providing the - # list of exported symbols of that shared object, eventually - # decorated with the 'weak' keyword - # *) the shared object with the F_LOADONLY flag set, to really avoid - # it being seen by the linker. - # At run time we better use the real file rather than another symlink, - # but for link time we create the symlink libNAME.so -> libNAME.so.V - - case $with_aix_soname,$aix_use_runtimelinking in - # AIX (on Power*) has no versioning support, so currently we cannot hardcode correct - # soname into executable. Probably we can add versioning support to - # collect2, so additional links can be useful in future. - aix,yes) # traditional libtool - dynamic_linker='AIX unversionable lib.so' - # If using run time linking (on AIX 4.2 or later) use lib.so - # instead of lib.a to let people know that these are not - # typical AIX shared libraries. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - ;; - aix,no) # traditional AIX only - dynamic_linker='AIX lib.a(lib.so.V)' - # We preserve .a as extension for shared libraries through AIX4.2 - # and later when we are not doing run time linking. - library_names_spec='$libname$release.a $libname.a' - soname_spec='$libname$release$shared_ext$major' - ;; - svr4,*) # full svr4 only - dynamic_linker="AIX lib.so.V($shared_archive_member_spec.o)" - library_names_spec='$libname$release$shared_ext$major $libname$shared_ext' - # We do not specify a path in Import Files, so LIBPATH fires. - shlibpath_overrides_runpath=yes - ;; - *,yes) # both, prefer svr4 - dynamic_linker="AIX lib.so.V($shared_archive_member_spec.o), lib.a(lib.so.V)" - library_names_spec='$libname$release$shared_ext$major $libname$shared_ext' - # unpreferred sharedlib libNAME.a needs extra handling - postinstall_cmds='test -n "$linkname" || linkname="$realname"~func_stripname "" ".so" "$linkname"~$install_shared_prog "$dir/$func_stripname_result.$libext" "$destdir/$func_stripname_result.$libext"~test -z "$tstripme" || test -z "$striplib" || $striplib "$destdir/$func_stripname_result.$libext"' - postuninstall_cmds='for n in $library_names $old_library; do :; done~func_stripname "" ".so" "$n"~test "$func_stripname_result" = "$n" || func_append rmfiles " $odir/$func_stripname_result.$libext"' - # We do not specify a path in Import Files, so LIBPATH fires. - shlibpath_overrides_runpath=yes - ;; - *,no) # both, prefer aix - dynamic_linker="AIX lib.a(lib.so.V), lib.so.V($shared_archive_member_spec.o)" - library_names_spec='$libname$release.a $libname.a' - soname_spec='$libname$release$shared_ext$major' - # unpreferred sharedlib libNAME.so.V and symlink libNAME.so need extra handling - postinstall_cmds='test -z "$dlname" || $install_shared_prog $dir/$dlname $destdir/$dlname~test -z "$tstripme" || test -z "$striplib" || $striplib $destdir/$dlname~test -n "$linkname" || linkname=$realname~func_stripname "" ".a" "$linkname"~(cd "$destdir" && $LN_S -f $dlname $func_stripname_result.so)' - postuninstall_cmds='test -z "$dlname" || func_append rmfiles " $odir/$dlname"~for n in $old_library $library_names; do :; done~func_stripname "" ".a" "$n"~func_append rmfiles " $odir/$func_stripname_result.so"' - ;; - esac - shlibpath_var=LIBPATH - fi - ;; - -amigaos*) - case $host_cpu in - powerpc) - # Since July 2007 AmigaOS4 officially supports .so libraries. - # When compiling the executable, add -use-dynld -Lsobjs: to the compileline. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - ;; - m68k) - library_names_spec='$libname.ixlibrary $libname.a' - # Create ${libname}_ixlibrary.a entries in /sys/libs. - finish_eval='for lib in `ls $libdir/*.ixlibrary 2>/dev/null`; do libname=`func_echo_all "$lib" | $SED '\''s%^.*/\([^/]*\)\.ixlibrary$%\1%'\''`; $RM /sys/libs/${libname}_ixlibrary.a; $show "cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a"; cd /sys/libs && $LN_S $lib ${libname}_ixlibrary.a || exit 1; done' - ;; - esac - ;; - -beos*) - library_names_spec='$libname$shared_ext' - dynamic_linker="$host_os ld.so" - shlibpath_var=LIBRARY_PATH - ;; - -bsdi[45]*) - version_type=linux # correct to gnu/linux during the next big refactor - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - finish_cmds='PATH="\$PATH:/sbin" ldconfig $libdir' - shlibpath_var=LD_LIBRARY_PATH - sys_lib_search_path_spec="/shlib /usr/lib /usr/X11/lib /usr/contrib/lib /lib /usr/local/lib" - sys_lib_dlsearch_path_spec="/shlib /usr/lib /usr/local/lib" - # the default ld.so.conf also contains /usr/contrib/lib and - # /usr/X11R6/lib (/usr/X11 is a link to /usr/X11R6), but let us allow - # libtool to hard-code these into programs - ;; - -cygwin* | mingw* | pw32* | cegcc*) - version_type=windows - shrext_cmds=.dll - need_version=no - need_lib_prefix=no - - case $GCC,$cc_basename in - yes,*) - # gcc - library_names_spec='$libname.dll.a' - # DLL is installed to $(libdir)/../bin by postinstall_cmds - postinstall_cmds='base_file=`basename \$file`~ - dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\$base_file'\''i; echo \$dlname'\''`~ - dldir=$destdir/`dirname \$dlpath`~ - test -d \$dldir || mkdir -p \$dldir~ - $install_prog $dir/$dlname \$dldir/$dlname~ - chmod a+x \$dldir/$dlname~ - if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then - eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; - fi' - postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ - dlpath=$dir/\$dldll~ - $RM \$dlpath' - shlibpath_overrides_runpath=yes - - case $host_os in - cygwin*) - # Cygwin DLLs use 'cyg' prefix rather than 'lib' - soname_spec='`echo $libname | sed -e 's/^lib/cyg/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' - - sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/lib/w32api" - ;; - mingw* | cegcc*) - # MinGW DLLs use traditional 'lib' prefix - soname_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' - ;; - pw32*) - # pw32 DLLs use 'pw' prefix rather than 'lib' - library_names_spec='`echo $libname | sed -e 's/^lib/pw/'``echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' - ;; - esac - dynamic_linker='Win32 ld.exe' - ;; - - *,cl*) - # Native MSVC - libname_spec='$name' - soname_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext' - library_names_spec='$libname.dll.lib' - - case $build_os in - mingw*) - sys_lib_search_path_spec= - lt_save_ifs=$IFS - IFS=';' - for lt_path in $LIB - do - IFS=$lt_save_ifs - # Let DOS variable expansion print the short 8.3 style file name. - lt_path=`cd "$lt_path" 2>/dev/null && cmd //C "for %i in (".") do @echo %~si"` - sys_lib_search_path_spec="$sys_lib_search_path_spec $lt_path" - done - IFS=$lt_save_ifs - # Convert to MSYS style. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | sed -e 's|\\\\|/|g' -e 's| \\([a-zA-Z]\\):| /\\1|g' -e 's|^ ||'` - ;; - cygwin*) - # Convert to unix form, then to dos form, then back to unix form - # but this time dos style (no spaces!) so that the unix form looks - # like /cygdrive/c/PROGRA~1:/cygdr... - sys_lib_search_path_spec=`cygpath --path --unix "$LIB"` - sys_lib_search_path_spec=`cygpath --path --dos "$sys_lib_search_path_spec" 2>/dev/null` - sys_lib_search_path_spec=`cygpath --path --unix "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` - ;; - *) - sys_lib_search_path_spec=$LIB - if $ECHO "$sys_lib_search_path_spec" | $GREP ';[c-zC-Z]:/' >/dev/null; then - # It is most probably a Windows format PATH. - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e 's/;/ /g'` - else - sys_lib_search_path_spec=`$ECHO "$sys_lib_search_path_spec" | $SED -e "s/$PATH_SEPARATOR/ /g"` - fi - # FIXME: find the short name or the path components, as spaces are - # common. (e.g. "Program Files" -> "PROGRA~1") - ;; - esac - - # DLL is installed to $(libdir)/../bin by postinstall_cmds - postinstall_cmds='base_file=`basename \$file`~ - dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\$base_file'\''i; echo \$dlname'\''`~ - dldir=$destdir/`dirname \$dlpath`~ - test -d \$dldir || mkdir -p \$dldir~ - $install_prog $dir/$dlname \$dldir/$dlname' - postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; echo \$dlname'\''`~ - dlpath=$dir/\$dldll~ - $RM \$dlpath' - shlibpath_overrides_runpath=yes - dynamic_linker='Win32 link.exe' - ;; - - *) - # Assume MSVC wrapper - library_names_spec='$libname`echo $release | $SED -e 's/[.]/-/g'`$versuffix$shared_ext $libname.lib' - dynamic_linker='Win32 ld.exe' - ;; - esac - # FIXME: first we should search . and the directory the executable is in - shlibpath_var=PATH - ;; - -darwin* | rhapsody*) - dynamic_linker="$host_os dyld" - version_type=darwin - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$major$shared_ext $libname$shared_ext' - soname_spec='$libname$release$major$shared_ext' - shlibpath_overrides_runpath=yes - shlibpath_var=DYLD_LIBRARY_PATH - shrext_cmds='`test .$module = .yes && echo .so || echo .dylib`' - - sys_lib_search_path_spec="$sys_lib_search_path_spec /usr/local/lib" - sys_lib_dlsearch_path_spec='/usr/local/lib /lib /usr/lib' - ;; - -dgux*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - ;; - -freebsd* | dragonfly*) - # DragonFly does not have aout. When/if they implement a new - # versioning mechanism, adjust this. - if test -x /usr/bin/objformat; then - objformat=`/usr/bin/objformat` - else - case $host_os in - freebsd[23].*) objformat=aout ;; - *) objformat=elf ;; - esac - fi - version_type=freebsd-$objformat - case $version_type in - freebsd-elf*) - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - need_version=no - need_lib_prefix=no - ;; - freebsd-*) - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - need_version=yes - ;; - esac - shlibpath_var=LD_LIBRARY_PATH - case $host_os in - freebsd2.*) - shlibpath_overrides_runpath=yes - ;; - freebsd3.[01]* | freebsdelf3.[01]*) - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - ;; - freebsd3.[2-9]* | freebsdelf3.[2-9]* | \ - freebsd4.[0-5] | freebsdelf4.[0-5] | freebsd4.1.1 | freebsdelf4.1.1) - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - ;; - *) # from 4.6 on, and DragonFly - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - ;; - esac - ;; - -haiku*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - dynamic_linker="$host_os runtime_loader" - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LIBRARY_PATH - shlibpath_overrides_runpath=no - sys_lib_dlsearch_path_spec='/boot/home/config/lib /boot/common/lib /boot/system/lib' - hardcode_into_libs=yes - ;; - -hpux9* | hpux10* | hpux11*) - # Give a soname corresponding to the major version so that dld.sl refuses to - # link against other versions. - version_type=sunos - need_lib_prefix=no - need_version=no - case $host_cpu in - ia64*) - shrext_cmds='.so' - hardcode_into_libs=yes - dynamic_linker="$host_os dld.so" - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - if test 32 = "$HPUX_IA64_MODE"; then - sys_lib_search_path_spec="/usr/lib/hpux32 /usr/local/lib/hpux32 /usr/local/lib" - sys_lib_dlsearch_path_spec=/usr/lib/hpux32 - else - sys_lib_search_path_spec="/usr/lib/hpux64 /usr/local/lib/hpux64" - sys_lib_dlsearch_path_spec=/usr/lib/hpux64 - fi - ;; - hppa*64*) - shrext_cmds='.sl' - hardcode_into_libs=yes - dynamic_linker="$host_os dld.sl" - shlibpath_var=LD_LIBRARY_PATH # How should we handle SHLIB_PATH - shlibpath_overrides_runpath=yes # Unless +noenvvar is specified. - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - sys_lib_search_path_spec="/usr/lib/pa20_64 /usr/ccs/lib/pa20_64" - sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec - ;; - *) - shrext_cmds='.sl' - dynamic_linker="$host_os dld.sl" - shlibpath_var=SHLIB_PATH - shlibpath_overrides_runpath=no # +s is required to enable SHLIB_PATH - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - ;; - esac - # HP-UX runs *really* slowly unless shared libraries are mode 555, ... - postinstall_cmds='chmod 555 $lib' - # or fails outright, so override atomically: - install_override_mode=555 - ;; - -interix[3-9]*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - dynamic_linker='Interix 3.x ld.so.1 (PE, like ELF)' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - ;; - -irix5* | irix6* | nonstopux*) - case $host_os in - nonstopux*) version_type=nonstopux ;; - *) - if test yes = "$lt_cv_prog_gnu_ld"; then - version_type=linux # correct to gnu/linux during the next big refactor - else - version_type=irix - fi ;; - esac - need_lib_prefix=no - need_version=no - soname_spec='$libname$release$shared_ext$major' - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$release$shared_ext $libname$shared_ext' - case $host_os in - irix5* | nonstopux*) - libsuff= shlibsuff= - ;; - *) - case $LD in # libtool.m4 will add one of these switches to LD - *-32|*"-32 "|*-melf32bsmip|*"-melf32bsmip ") - libsuff= shlibsuff= libmagic=32-bit;; - *-n32|*"-n32 "|*-melf32bmipn32|*"-melf32bmipn32 ") - libsuff=32 shlibsuff=N32 libmagic=N32;; - *-64|*"-64 "|*-melf64bmip|*"-melf64bmip ") - libsuff=64 shlibsuff=64 libmagic=64-bit;; - *) libsuff= shlibsuff= libmagic=never-match;; - esac - ;; - esac - shlibpath_var=LD_LIBRARY${shlibsuff}_PATH - shlibpath_overrides_runpath=no - sys_lib_search_path_spec="/usr/lib$libsuff /lib$libsuff /usr/local/lib$libsuff" - sys_lib_dlsearch_path_spec="/usr/lib$libsuff /lib$libsuff" - hardcode_into_libs=yes - ;; - -# No shared lib support for Linux oldld, aout, or coff. -linux*oldld* | linux*aout* | linux*coff*) - dynamic_linker=no - ;; - -linux*android*) - version_type=none # Android doesn't support versioned libraries. - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext' - soname_spec='$libname$release$shared_ext' - finish_cmds= - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - - # This implies no fast_install, which is unacceptable. - # Some rework will be needed to allow for fast_install - # before this can be enabled. - hardcode_into_libs=yes - - dynamic_linker='Android linker' - # Don't embed -rpath directories since the linker doesn't support them. - hardcode_libdir_flag_spec='-L$libdir' - ;; - -# This must be glibc/ELF. -linux* | k*bsd*-gnu | kopensolaris*-gnu | gnu*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - finish_cmds='PATH="\$PATH:/sbin" ldconfig -n $libdir' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - - # Some binutils ld are patched to set DT_RUNPATH - if ${lt_cv_shlibpath_overrides_runpath+:} false; then : - $as_echo_n "(cached) " >&6 -else - lt_cv_shlibpath_overrides_runpath=no - save_LDFLAGS=$LDFLAGS - save_libdir=$libdir - eval "libdir=/foo; wl=\"$lt_prog_compiler_wl\"; \ - LDFLAGS=\"\$LDFLAGS $hardcode_libdir_flag_spec\"" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - if ($OBJDUMP -p conftest$ac_exeext) 2>/dev/null | grep "RUNPATH.*$libdir" >/dev/null; then : - lt_cv_shlibpath_overrides_runpath=yes -fi -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - LDFLAGS=$save_LDFLAGS - libdir=$save_libdir - -fi - - shlibpath_overrides_runpath=$lt_cv_shlibpath_overrides_runpath - - # This implies no fast_install, which is unacceptable. - # Some rework will be needed to allow for fast_install - # before this can be enabled. - hardcode_into_libs=yes - - # Ideally, we could use ldconfig to report *all* directores which are - # searched for libraries, however this is still not possible. Aside from not - # being certain /sbin/ldconfig is available, command - # 'ldconfig -N -X -v | grep ^/' on 64bit Fedora does not report /usr/lib64, - # even though it is searched at run-time. Try to do the best guess by - # appending ld.so.conf contents (and includes) to the search path. - if test -f /etc/ld.so.conf; then - lt_ld_extra=`awk '/^include / { system(sprintf("cd /etc; cat %s 2>/dev/null", \$2)); skip = 1; } { if (!skip) print \$0; skip = 0; }' < /etc/ld.so.conf | $SED -e 's/#.*//;/^[ ]*hwcap[ ]/d;s/[:, ]/ /g;s/=[^=]*$//;s/=[^= ]* / /g;s/"//g;/^$/d' | tr '\n' ' '` - sys_lib_dlsearch_path_spec="/lib /usr/lib $lt_ld_extra" - fi - - # We used to test for /lib/ld.so.1 and disable shared libraries on - # powerpc, because MkLinux only supported shared libraries with the - # GNU dynamic linker. Since this was broken with cross compilers, - # most powerpc-linux boxes support dynamic linking these days and - # people can always --disable-shared, the test was removed, and we - # assume the GNU/Linux dynamic linker is in use. - dynamic_linker='GNU/Linux ld.so' - ;; - -netbsdelf*-gnu) - version_type=linux - need_lib_prefix=no - need_version=no - library_names_spec='${libname}${release}${shared_ext}$versuffix ${libname}${release}${shared_ext}$major ${libname}${shared_ext}' - soname_spec='${libname}${release}${shared_ext}$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - dynamic_linker='NetBSD ld.elf_so' - ;; - -netbsd*) - version_type=sunos - need_lib_prefix=no - need_version=no - if echo __ELF__ | $CC -E - | $GREP __ELF__ >/dev/null; then - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' - dynamic_linker='NetBSD (a.out) ld.so' - else - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - dynamic_linker='NetBSD ld.elf_so' - fi - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - ;; - -newsos6) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - ;; - -*nto* | *qnx*) - version_type=qnx - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - dynamic_linker='ldqnx.so' - ;; - -openbsd* | bitrig*) - version_type=sunos - sys_lib_dlsearch_path_spec=/usr/lib - need_lib_prefix=no - if test -z "`echo __ELF__ | $CC -E - | $GREP __ELF__`"; then - need_version=no - else - need_version=yes - fi - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - finish_cmds='PATH="\$PATH:/sbin" ldconfig -m $libdir' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - ;; - -os2*) - libname_spec='$name' - version_type=windows - shrext_cmds=.dll - need_version=no - need_lib_prefix=no - # OS/2 can only load a DLL with a base name of 8 characters or less. - soname_spec='`test -n "$os2dllname" && libname="$os2dllname"; - v=$($ECHO $release$versuffix | tr -d .-); - n=$($ECHO $libname | cut -b -$((8 - ${#v})) | tr . _); - $ECHO $n$v`$shared_ext' - library_names_spec='${libname}_dll.$libext' - dynamic_linker='OS/2 ld.exe' - shlibpath_var=BEGINLIBPATH - sys_lib_search_path_spec="/lib /usr/lib /usr/local/lib" - sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec - postinstall_cmds='base_file=`basename \$file`~ - dlpath=`$SHELL 2>&1 -c '\''. $dir/'\''\$base_file'\''i; $ECHO \$dlname'\''`~ - dldir=$destdir/`dirname \$dlpath`~ - test -d \$dldir || mkdir -p \$dldir~ - $install_prog $dir/$dlname \$dldir/$dlname~ - chmod a+x \$dldir/$dlname~ - if test -n '\''$stripme'\'' && test -n '\''$striplib'\''; then - eval '\''$striplib \$dldir/$dlname'\'' || exit \$?; - fi' - postuninstall_cmds='dldll=`$SHELL 2>&1 -c '\''. $file; $ECHO \$dlname'\''`~ - dlpath=$dir/\$dldll~ - $RM \$dlpath' - ;; - -osf3* | osf4* | osf5*) - version_type=osf - need_lib_prefix=no - need_version=no - soname_spec='$libname$release$shared_ext$major' - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - sys_lib_search_path_spec="/usr/shlib /usr/ccs/lib /usr/lib/cmplrs/cc /usr/lib /usr/local/lib /var/shlib" - sys_lib_dlsearch_path_spec=$sys_lib_search_path_spec - ;; - -rdos*) - dynamic_linker=no - ;; - -solaris*) - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - # ldd complains unless libraries are executable - postinstall_cmds='chmod +x $lib' - ;; - -sunos4*) - version_type=sunos - library_names_spec='$libname$release$shared_ext$versuffix $libname$shared_ext$versuffix' - finish_cmds='PATH="\$PATH:/usr/etc" ldconfig $libdir' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - if test yes = "$with_gnu_ld"; then - need_lib_prefix=no - fi - need_version=yes - ;; - -sysv4 | sysv4.3*) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - case $host_vendor in - sni) - shlibpath_overrides_runpath=no - need_lib_prefix=no - runpath_var=LD_RUN_PATH - ;; - siemens) - need_lib_prefix=no - ;; - motorola) - need_lib_prefix=no - need_version=no - shlibpath_overrides_runpath=no - sys_lib_search_path_spec='/lib /usr/lib /usr/ccs/lib' - ;; - esac - ;; - -sysv4*MP*) - if test -d /usr/nec; then - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$shared_ext.$versuffix $libname$shared_ext.$major $libname$shared_ext' - soname_spec='$libname$shared_ext.$major' - shlibpath_var=LD_LIBRARY_PATH - fi - ;; - -sysv5* | sco3.2v5* | sco5v6* | unixware* | OpenUNIX* | sysv4*uw2*) - version_type=sco - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=yes - hardcode_into_libs=yes - if test yes = "$with_gnu_ld"; then - sys_lib_search_path_spec='/usr/local/lib /usr/gnu/lib /usr/ccs/lib /usr/lib /lib' - else - sys_lib_search_path_spec='/usr/ccs/lib /usr/lib' - case $host_os in - sco3.2v5*) - sys_lib_search_path_spec="$sys_lib_search_path_spec /lib" - ;; - esac - fi - sys_lib_dlsearch_path_spec='/usr/lib' - ;; - -tpf*) - # TPF is a cross-target only. Preferred cross-host = GNU/Linux. - version_type=linux # correct to gnu/linux during the next big refactor - need_lib_prefix=no - need_version=no - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - shlibpath_var=LD_LIBRARY_PATH - shlibpath_overrides_runpath=no - hardcode_into_libs=yes - ;; - -uts4*) - version_type=linux # correct to gnu/linux during the next big refactor - library_names_spec='$libname$release$shared_ext$versuffix $libname$release$shared_ext$major $libname$shared_ext' - soname_spec='$libname$release$shared_ext$major' - shlibpath_var=LD_LIBRARY_PATH - ;; - -*) - dynamic_linker=no - ;; -esac -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $dynamic_linker" >&5 -$as_echo "$dynamic_linker" >&6; } -test no = "$dynamic_linker" && can_build_shared=no - -variables_saved_for_relink="PATH $shlibpath_var $runpath_var" -if test yes = "$GCC"; then - variables_saved_for_relink="$variables_saved_for_relink GCC_EXEC_PREFIX COMPILER_PATH LIBRARY_PATH" -fi - -if test set = "${lt_cv_sys_lib_search_path_spec+set}"; then - sys_lib_search_path_spec=$lt_cv_sys_lib_search_path_spec -fi - -if test set = "${lt_cv_sys_lib_dlsearch_path_spec+set}"; then - sys_lib_dlsearch_path_spec=$lt_cv_sys_lib_dlsearch_path_spec -fi - -# remember unaugmented sys_lib_dlsearch_path content for libtool script decls... -configure_time_dlsearch_path=$sys_lib_dlsearch_path_spec - -# ... but it needs LT_SYS_LIBRARY_PATH munging for other configure-time code -func_munge_path_list sys_lib_dlsearch_path_spec "$LT_SYS_LIBRARY_PATH" - -# to be used as default LT_SYS_LIBRARY_PATH value in generated libtool -configure_time_lt_sys_library_path=$LT_SYS_LIBRARY_PATH - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking how to hardcode library paths into programs" >&5 -$as_echo_n "checking how to hardcode library paths into programs... " >&6; } -hardcode_action= -if test -n "$hardcode_libdir_flag_spec" || - test -n "$runpath_var" || - test yes = "$hardcode_automatic"; then - - # We can hardcode non-existent directories. - if test no != "$hardcode_direct" && - # If the only mechanism to avoid hardcoding is shlibpath_var, we - # have to relink, otherwise we might link with an installed library - # when we should be linking with a yet-to-be-installed one - ## test no != "$_LT_TAGVAR(hardcode_shlibpath_var, )" && - test no != "$hardcode_minus_L"; then - # Linking always hardcodes the temporary library directory. - hardcode_action=relink - else - # We can link without hardcoding, and we can hardcode nonexisting dirs. - hardcode_action=immediate - fi -else - # We cannot hardcode anything, or else we can only hardcode existing - # directories. - hardcode_action=unsupported -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $hardcode_action" >&5 -$as_echo "$hardcode_action" >&6; } - -if test relink = "$hardcode_action" || - test yes = "$inherit_rpath"; then - # Fast installation is not supported - enable_fast_install=no -elif test yes = "$shlibpath_overrides_runpath" || - test no = "$enable_shared"; then - # Fast installation is not necessary - enable_fast_install=needless -fi - - - - - - - if test yes != "$enable_dlopen"; then - enable_dlopen=unknown - enable_dlopen_self=unknown - enable_dlopen_self_static=unknown -else - lt_cv_dlopen=no - lt_cv_dlopen_libs= - - case $host_os in - beos*) - lt_cv_dlopen=load_add_on - lt_cv_dlopen_libs= - lt_cv_dlopen_self=yes - ;; - - mingw* | pw32* | cegcc*) - lt_cv_dlopen=LoadLibrary - lt_cv_dlopen_libs= - ;; - - cygwin*) - lt_cv_dlopen=dlopen - lt_cv_dlopen_libs= - ;; - - darwin*) - # if libdl is installed we need to link against it - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 -$as_echo_n "checking for dlopen in -ldl... " >&6; } -if ${ac_cv_lib_dl_dlopen+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-ldl $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char dlopen (); -int -main () -{ -return dlopen (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_dl_dlopen=yes -else - ac_cv_lib_dl_dlopen=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 -$as_echo "$ac_cv_lib_dl_dlopen" >&6; } -if test "x$ac_cv_lib_dl_dlopen" = xyes; then : - lt_cv_dlopen=dlopen lt_cv_dlopen_libs=-ldl -else - - lt_cv_dlopen=dyld - lt_cv_dlopen_libs= - lt_cv_dlopen_self=yes - -fi - - ;; - - tpf*) - # Don't try to run any link tests for TPF. We know it's impossible - # because TPF is a cross-compiler, and we know how we open DSOs. - lt_cv_dlopen=dlopen - lt_cv_dlopen_libs= - lt_cv_dlopen_self=no - ;; - - *) - ac_fn_c_check_func "$LINENO" "shl_load" "ac_cv_func_shl_load" -if test "x$ac_cv_func_shl_load" = xyes; then : - lt_cv_dlopen=shl_load -else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for shl_load in -ldld" >&5 -$as_echo_n "checking for shl_load in -ldld... " >&6; } -if ${ac_cv_lib_dld_shl_load+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-ldld $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char shl_load (); -int -main () -{ -return shl_load (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_dld_shl_load=yes -else - ac_cv_lib_dld_shl_load=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_shl_load" >&5 -$as_echo "$ac_cv_lib_dld_shl_load" >&6; } -if test "x$ac_cv_lib_dld_shl_load" = xyes; then : - lt_cv_dlopen=shl_load lt_cv_dlopen_libs=-ldld -else - ac_fn_c_check_func "$LINENO" "dlopen" "ac_cv_func_dlopen" -if test "x$ac_cv_func_dlopen" = xyes; then : - lt_cv_dlopen=dlopen -else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 -$as_echo_n "checking for dlopen in -ldl... " >&6; } -if ${ac_cv_lib_dl_dlopen+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-ldl $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char dlopen (); -int -main () -{ -return dlopen (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_dl_dlopen=yes -else - ac_cv_lib_dl_dlopen=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 -$as_echo "$ac_cv_lib_dl_dlopen" >&6; } -if test "x$ac_cv_lib_dl_dlopen" = xyes; then : - lt_cv_dlopen=dlopen lt_cv_dlopen_libs=-ldl -else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -lsvld" >&5 -$as_echo_n "checking for dlopen in -lsvld... " >&6; } -if ${ac_cv_lib_svld_dlopen+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lsvld $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char dlopen (); -int -main () -{ -return dlopen (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_svld_dlopen=yes -else - ac_cv_lib_svld_dlopen=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_svld_dlopen" >&5 -$as_echo "$ac_cv_lib_svld_dlopen" >&6; } -if test "x$ac_cv_lib_svld_dlopen" = xyes; then : - lt_cv_dlopen=dlopen lt_cv_dlopen_libs=-lsvld -else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dld_link in -ldld" >&5 -$as_echo_n "checking for dld_link in -ldld... " >&6; } -if ${ac_cv_lib_dld_dld_link+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-ldld $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char dld_link (); -int -main () -{ -return dld_link (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_dld_dld_link=yes -else - ac_cv_lib_dld_dld_link=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dld_dld_link" >&5 -$as_echo "$ac_cv_lib_dld_dld_link" >&6; } -if test "x$ac_cv_lib_dld_dld_link" = xyes; then : - lt_cv_dlopen=dld_link lt_cv_dlopen_libs=-ldld -fi - - -fi - - -fi - - -fi - - -fi - - -fi - - ;; - esac - - if test no = "$lt_cv_dlopen"; then - enable_dlopen=no - else - enable_dlopen=yes - fi - - case $lt_cv_dlopen in - dlopen) - save_CPPFLAGS=$CPPFLAGS - test yes = "$ac_cv_header_dlfcn_h" && CPPFLAGS="$CPPFLAGS -DHAVE_DLFCN_H" - - save_LDFLAGS=$LDFLAGS - wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $export_dynamic_flag_spec\" - - save_LIBS=$LIBS - LIBS="$lt_cv_dlopen_libs $LIBS" - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program can dlopen itself" >&5 -$as_echo_n "checking whether a program can dlopen itself... " >&6; } -if ${lt_cv_dlopen_self+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test yes = "$cross_compiling"; then : - lt_cv_dlopen_self=cross -else - lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 - lt_status=$lt_dlunknown - cat > conftest.$ac_ext <<_LT_EOF -#line $LINENO "configure" -#include "confdefs.h" - -#if HAVE_DLFCN_H -#include -#endif - -#include - -#ifdef RTLD_GLOBAL -# define LT_DLGLOBAL RTLD_GLOBAL -#else -# ifdef DL_GLOBAL -# define LT_DLGLOBAL DL_GLOBAL -# else -# define LT_DLGLOBAL 0 -# endif -#endif - -/* We may have to define LT_DLLAZY_OR_NOW in the command line if we - find out it does not work in some platform. */ -#ifndef LT_DLLAZY_OR_NOW -# ifdef RTLD_LAZY -# define LT_DLLAZY_OR_NOW RTLD_LAZY -# else -# ifdef DL_LAZY -# define LT_DLLAZY_OR_NOW DL_LAZY -# else -# ifdef RTLD_NOW -# define LT_DLLAZY_OR_NOW RTLD_NOW -# else -# ifdef DL_NOW -# define LT_DLLAZY_OR_NOW DL_NOW -# else -# define LT_DLLAZY_OR_NOW 0 -# endif -# endif -# endif -# endif -#endif - -/* When -fvisibility=hidden is used, assume the code has been annotated - correspondingly for the symbols needed. */ -#if defined __GNUC__ && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)) || (__GNUC__ > 3)) -int fnord () __attribute__((visibility("default"))); -#endif - -int fnord () { return 42; } -int main () -{ - void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW); - int status = $lt_dlunknown; - - if (self) - { - if (dlsym (self,"fnord")) status = $lt_dlno_uscore; - else - { - if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore; - else puts (dlerror ()); - } - /* dlclose (self); */ - } - else - puts (dlerror ()); - - return status; -} -_LT_EOF - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5 - (eval $ac_link) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && test -s "conftest$ac_exeext" 2>/dev/null; then - (./conftest; exit; ) >&5 2>/dev/null - lt_status=$? - case x$lt_status in - x$lt_dlno_uscore) lt_cv_dlopen_self=yes ;; - x$lt_dlneed_uscore) lt_cv_dlopen_self=yes ;; - x$lt_dlunknown|x*) lt_cv_dlopen_self=no ;; - esac - else : - # compilation failed - lt_cv_dlopen_self=no - fi -fi -rm -fr conftest* - - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_dlopen_self" >&5 -$as_echo "$lt_cv_dlopen_self" >&6; } - - if test yes = "$lt_cv_dlopen_self"; then - wl=$lt_prog_compiler_wl eval LDFLAGS=\"\$LDFLAGS $lt_prog_compiler_static\" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a statically linked program can dlopen itself" >&5 -$as_echo_n "checking whether a statically linked program can dlopen itself... " >&6; } -if ${lt_cv_dlopen_self_static+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test yes = "$cross_compiling"; then : - lt_cv_dlopen_self_static=cross -else - lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 - lt_status=$lt_dlunknown - cat > conftest.$ac_ext <<_LT_EOF -#line $LINENO "configure" -#include "confdefs.h" - -#if HAVE_DLFCN_H -#include -#endif - -#include - -#ifdef RTLD_GLOBAL -# define LT_DLGLOBAL RTLD_GLOBAL -#else -# ifdef DL_GLOBAL -# define LT_DLGLOBAL DL_GLOBAL -# else -# define LT_DLGLOBAL 0 -# endif -#endif - -/* We may have to define LT_DLLAZY_OR_NOW in the command line if we - find out it does not work in some platform. */ -#ifndef LT_DLLAZY_OR_NOW -# ifdef RTLD_LAZY -# define LT_DLLAZY_OR_NOW RTLD_LAZY -# else -# ifdef DL_LAZY -# define LT_DLLAZY_OR_NOW DL_LAZY -# else -# ifdef RTLD_NOW -# define LT_DLLAZY_OR_NOW RTLD_NOW -# else -# ifdef DL_NOW -# define LT_DLLAZY_OR_NOW DL_NOW -# else -# define LT_DLLAZY_OR_NOW 0 -# endif -# endif -# endif -# endif -#endif - -/* When -fvisibility=hidden is used, assume the code has been annotated - correspondingly for the symbols needed. */ -#if defined __GNUC__ && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3)) || (__GNUC__ > 3)) -int fnord () __attribute__((visibility("default"))); -#endif - -int fnord () { return 42; } -int main () -{ - void *self = dlopen (0, LT_DLGLOBAL|LT_DLLAZY_OR_NOW); - int status = $lt_dlunknown; - - if (self) - { - if (dlsym (self,"fnord")) status = $lt_dlno_uscore; - else - { - if (dlsym( self,"_fnord")) status = $lt_dlneed_uscore; - else puts (dlerror ()); - } - /* dlclose (self); */ - } - else - puts (dlerror ()); - - return status; -} -_LT_EOF - if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_link\""; } >&5 - (eval $ac_link) 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } && test -s "conftest$ac_exeext" 2>/dev/null; then - (./conftest; exit; ) >&5 2>/dev/null - lt_status=$? - case x$lt_status in - x$lt_dlno_uscore) lt_cv_dlopen_self_static=yes ;; - x$lt_dlneed_uscore) lt_cv_dlopen_self_static=yes ;; - x$lt_dlunknown|x*) lt_cv_dlopen_self_static=no ;; - esac - else : - # compilation failed - lt_cv_dlopen_self_static=no - fi -fi -rm -fr conftest* - - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $lt_cv_dlopen_self_static" >&5 -$as_echo "$lt_cv_dlopen_self_static" >&6; } - fi - - CPPFLAGS=$save_CPPFLAGS - LDFLAGS=$save_LDFLAGS - LIBS=$save_LIBS - ;; - esac - - case $lt_cv_dlopen_self in - yes|no) enable_dlopen_self=$lt_cv_dlopen_self ;; - *) enable_dlopen_self=unknown ;; - esac - - case $lt_cv_dlopen_self_static in - yes|no) enable_dlopen_self_static=$lt_cv_dlopen_self_static ;; - *) enable_dlopen_self_static=unknown ;; - esac -fi - - - - - - - - - - - - - - - - - -striplib= -old_striplib= -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether stripping libraries is possible" >&5 -$as_echo_n "checking whether stripping libraries is possible... " >&6; } -if test -n "$STRIP" && $STRIP -V 2>&1 | $GREP "GNU strip" >/dev/null; then - test -z "$old_striplib" && old_striplib="$STRIP --strip-debug" - test -z "$striplib" && striplib="$STRIP --strip-unneeded" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else -# FIXME - insert some real tests, host_os isn't really good enough - case $host_os in - darwin*) - if test -n "$STRIP"; then - striplib="$STRIP -x" - old_striplib="$STRIP -S" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - fi - ;; - *) - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - ;; - esac -fi - - - - - - - - - - - - - # Report what library types will actually be built - { $as_echo "$as_me:${as_lineno-$LINENO}: checking if libtool supports shared libraries" >&5 -$as_echo_n "checking if libtool supports shared libraries... " >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $can_build_shared" >&5 -$as_echo "$can_build_shared" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build shared libraries" >&5 -$as_echo_n "checking whether to build shared libraries... " >&6; } - test no = "$can_build_shared" && enable_shared=no - - # On AIX, shared libraries and static libraries use the same namespace, and - # are all built from PIC. - case $host_os in - aix3*) - test yes = "$enable_shared" && enable_static=no - if test -n "$RANLIB"; then - archive_cmds="$archive_cmds~\$RANLIB \$lib" - postinstall_cmds='$RANLIB $lib' - fi - ;; - - aix[4-9]*) - if test ia64 != "$host_cpu"; then - case $enable_shared,$with_aix_soname,$aix_use_runtimelinking in - yes,aix,yes) ;; # shared object as lib.so file only - yes,svr4,*) ;; # shared object as lib.so archive member only - yes,*) enable_static=no ;; # shared object in lib.a archive as well - esac - fi - ;; - esac - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_shared" >&5 -$as_echo "$enable_shared" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build static libraries" >&5 -$as_echo_n "checking whether to build static libraries... " >&6; } - # Make sure either enable_shared or enable_static is yes. - test yes = "$enable_shared" || enable_static=yes - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_static" >&5 -$as_echo "$enable_static" >&6; } - - - - -fi -ac_ext=c -ac_cpp='$CPP $CPPFLAGS' -ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' -ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' -ac_compiler_gnu=$ac_cv_c_compiler_gnu - -CC=$lt_save_CC - - - - - - - - - - - - - - - - ac_config_commands="$ac_config_commands libtool" - - - - -# Only expand once: - - -# Find a good install program. We prefer a C program (faster), -# so one script is as good as another. But avoid the broken or -# incompatible versions: -# SysV /etc/install, /usr/sbin/install -# SunOS /usr/etc/install -# IRIX /sbin/install -# AIX /bin/install -# AmigaOS /C/install, which installs bootblocks on floppy discs -# AIX 4 /usr/bin/installbsd, which doesn't work without a -g flag -# AFS /usr/afsws/bin/install, which mishandles nonexistent args -# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff" -# OS/2's system install, which has a completely different semantic -# ./install, which can be erroneously created by make from ./install.sh. -# Reject install programs that cannot install multiple files. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 -$as_echo_n "checking for a BSD-compatible install... " >&6; } -if test -z "$INSTALL"; then -if ${ac_cv_path_install+:} false; then : - $as_echo_n "(cached) " >&6 -else - as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - # Account for people who put trailing slashes in PATH elements. -case $as_dir/ in #(( - ./ | .// | /[cC]/* | \ - /etc/* | /usr/sbin/* | /usr/etc/* | /sbin/* | /usr/afsws/bin/* | \ - ?:[\\/]os2[\\/]install[\\/]* | ?:[\\/]OS2[\\/]INSTALL[\\/]* | \ - /usr/ucb/* ) ;; - *) - # OSF1 and SCO ODT 3.0 have their own names for install. - # Don't use installbsd from OSF since it installs stuff as root - # by default. - for ac_prog in ginstall scoinst install; do - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_prog$ac_exec_ext"; then - if test $ac_prog = install && - grep dspmsg "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then - # AIX install. It has an incompatible calling convention. - : - elif test $ac_prog = install && - grep pwplus "$as_dir/$ac_prog$ac_exec_ext" >/dev/null 2>&1; then - # program-specific install script used by HP pwplus--don't use. - : - else - rm -rf conftest.one conftest.two conftest.dir - echo one > conftest.one - echo two > conftest.two - mkdir conftest.dir - if "$as_dir/$ac_prog$ac_exec_ext" -c conftest.one conftest.two "`pwd`/conftest.dir" && - test -s conftest.one && test -s conftest.two && - test -s conftest.dir/conftest.one && - test -s conftest.dir/conftest.two - then - ac_cv_path_install="$as_dir/$ac_prog$ac_exec_ext -c" - break 3 - fi - fi - fi - done - done - ;; -esac - - done -IFS=$as_save_IFS - -rm -rf conftest.one conftest.two conftest.dir - -fi - if test "${ac_cv_path_install+set}" = set; then - INSTALL=$ac_cv_path_install - else - # As a last resort, use the slow shell script. Don't cache a - # value for INSTALL within a source directory, because that will - # break other packages using the cache if that directory is - # removed, or if the value is a relative name. - INSTALL=$ac_install_sh - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $INSTALL" >&5 -$as_echo "$INSTALL" >&6; } - -# Use test -z because SunOS4 sh mishandles braces in ${var-val}. -# It thinks the first close brace ends the variable substitution. -test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}' - -test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' - -test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' - - -######### -# Enable large file support (if special flags are necessary) -# -# Check whether --enable-largefile was given. -if test "${enable_largefile+set}" = set; then : - enableval=$enable_largefile; -fi - -if test "$enable_largefile" != no; then - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for special C compiler options needed for large files" >&5 -$as_echo_n "checking for special C compiler options needed for large files... " >&6; } -if ${ac_cv_sys_largefile_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_cv_sys_largefile_CC=no - if test "$GCC" != yes; then - ac_save_CC=$CC - while :; do - # IRIX 6.2 and later do not support large files by default, - # so use the C compiler's -n32 option if that helps. - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; -int -main () -{ - - ; - return 0; -} -_ACEOF - if ac_fn_c_try_compile "$LINENO"; then : - break -fi -rm -f core conftest.err conftest.$ac_objext - CC="$CC -n32" - if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_sys_largefile_CC=' -n32'; break -fi -rm -f core conftest.err conftest.$ac_objext - break - done - CC=$ac_save_CC - rm -f conftest.$ac_ext - fi -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_largefile_CC" >&5 -$as_echo "$ac_cv_sys_largefile_CC" >&6; } - if test "$ac_cv_sys_largefile_CC" != no; then - CC=$CC$ac_cv_sys_largefile_CC - fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _FILE_OFFSET_BITS value needed for large files" >&5 -$as_echo_n "checking for _FILE_OFFSET_BITS value needed for large files... " >&6; } -if ${ac_cv_sys_file_offset_bits+:} false; then : - $as_echo_n "(cached) " >&6 -else - while :; do - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_sys_file_offset_bits=no; break -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#define _FILE_OFFSET_BITS 64 -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_sys_file_offset_bits=64; break -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - ac_cv_sys_file_offset_bits=unknown - break -done -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_file_offset_bits" >&5 -$as_echo "$ac_cv_sys_file_offset_bits" >&6; } -case $ac_cv_sys_file_offset_bits in #( - no | unknown) ;; - *) -cat >>confdefs.h <<_ACEOF -#define _FILE_OFFSET_BITS $ac_cv_sys_file_offset_bits -_ACEOF -;; -esac -rm -rf conftest* - if test $ac_cv_sys_file_offset_bits = unknown; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _LARGE_FILES value needed for large files" >&5 -$as_echo_n "checking for _LARGE_FILES value needed for large files... " >&6; } -if ${ac_cv_sys_large_files+:} false; then : - $as_echo_n "(cached) " >&6 -else - while :; do - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_sys_large_files=no; break -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#define _LARGE_FILES 1 -#include - /* Check that off_t can represent 2**63 - 1 correctly. - We can't simply define LARGE_OFF_T to be 9223372036854775807, - since some C++ compilers masquerading as C compilers - incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) - int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 - && LARGE_OFF_T % 2147483647 == 1) - ? 1 : -1]; -int -main () -{ - - ; - return 0; -} -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_sys_large_files=1; break -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - ac_cv_sys_large_files=unknown - break -done -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sys_large_files" >&5 -$as_echo "$ac_cv_sys_large_files" >&6; } -case $ac_cv_sys_large_files in #( - no | unknown) ;; - *) -cat >>confdefs.h <<_ACEOF -#define _LARGE_FILES $ac_cv_sys_large_files -_ACEOF -;; -esac -rm -rf conftest* - fi - - -fi - - -######### -# Check for needed/wanted data types -ac_fn_c_check_type "$LINENO" "int8_t" "ac_cv_type_int8_t" "$ac_includes_default" -if test "x$ac_cv_type_int8_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_INT8_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "int16_t" "ac_cv_type_int16_t" "$ac_includes_default" -if test "x$ac_cv_type_int16_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_INT16_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "int32_t" "ac_cv_type_int32_t" "$ac_includes_default" -if test "x$ac_cv_type_int32_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_INT32_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "int64_t" "ac_cv_type_int64_t" "$ac_includes_default" -if test "x$ac_cv_type_int64_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_INT64_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "intptr_t" "ac_cv_type_intptr_t" "$ac_includes_default" -if test "x$ac_cv_type_intptr_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_INTPTR_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "uint8_t" "ac_cv_type_uint8_t" "$ac_includes_default" -if test "x$ac_cv_type_uint8_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_UINT8_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "uint16_t" "ac_cv_type_uint16_t" "$ac_includes_default" -if test "x$ac_cv_type_uint16_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_UINT16_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "uint32_t" "ac_cv_type_uint32_t" "$ac_includes_default" -if test "x$ac_cv_type_uint32_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_UINT32_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "uint64_t" "ac_cv_type_uint64_t" "$ac_includes_default" -if test "x$ac_cv_type_uint64_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_UINT64_T 1 -_ACEOF - - -fi -ac_fn_c_check_type "$LINENO" "uintptr_t" "ac_cv_type_uintptr_t" "$ac_includes_default" -if test "x$ac_cv_type_uintptr_t" = xyes; then : - -cat >>confdefs.h <<_ACEOF -#define HAVE_UINTPTR_T 1 -_ACEOF - - -fi - - -######### -# Check for needed/wanted headers -for ac_header in sys/types.h stdlib.h stdint.h inttypes.h malloc.h -do : - as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` -ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : - cat >>confdefs.h <<_ACEOF -#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 -_ACEOF - -fi - -done - - -######### -# Figure out whether or not we have these functions -# -for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64 -do : - as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` -ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" -if eval test \"x\$"$as_ac_var"\" = x"yes"; then : - cat >>confdefs.h <<_ACEOF -#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 -_ACEOF - -fi -done - - -######### -# By default, we use the amalgamation (this may be changed below...) -# -USE_AMALGAMATION=1 - -######### -# See whether we can run specific tclsh versions known to work well; -# if not, then we fall back to plain tclsh. -# TODO: try other versions before falling back? -# -for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_TCLSH_CMD+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$TCLSH_CMD"; then - ac_cv_prog_TCLSH_CMD="$TCLSH_CMD" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_TCLSH_CMD="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -TCLSH_CMD=$ac_cv_prog_TCLSH_CMD -if test -n "$TCLSH_CMD"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD" >&5 -$as_echo "$TCLSH_CMD" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$TCLSH_CMD" && break -done -test -n "$TCLSH_CMD" || TCLSH_CMD="none" - -if test "$TCLSH_CMD" = "none"; then - # If we can't find a local tclsh, then building the amalgamation will fail. - # We act as though --disable-amalgamation has been used. - echo "Warning: can't find tclsh - defaulting to non-amalgamation build." - USE_AMALGAMATION=0 - TCLSH_CMD="tclsh" -fi - - - -if test "x${TCLLIBDIR+set}" != "xset" ; then - TCLLIBDIR='$(libdir)' - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do - if test -d $i ; then - TCLLIBDIR=$i - break - fi - done - TCLLIBDIR="${TCLLIBDIR}/sqlite3" -fi - -######### -# Set up an appropriate program prefix -# -if test "$program_prefix" = "NONE"; then - program_prefix="" -fi - - -VERSION=`cat $srcdir/VERSION | sed 's/^\([0-9]*\.*[0-9]*\).*/\1/'` -{ $as_echo "$as_me:${as_lineno-$LINENO}: Version set to $VERSION" >&5 -$as_echo "$as_me: Version set to $VERSION" >&6;} - -RELEASE=`cat $srcdir/VERSION` -{ $as_echo "$as_me:${as_lineno-$LINENO}: Release set to $RELEASE" >&5 -$as_echo "$as_me: Release set to $RELEASE" >&6;} - - -########## -# Handle --with-wasi-sdk=DIR -# -# This must be early because it changes the toolchain. -# - -# Check whether --with-wasi-sdk was given. -if test "${with_wasi_sdk+set}" = set; then : - withval=$with_wasi_sdk; with_wasisdk=${withval} -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for WASI SDK directory" >&5 -$as_echo_n "checking for WASI SDK directory... " >&6; } -if ${ac_cv_c_wasi_sdk+:} false; then : - $as_echo_n "(cached) " >&6 -else - - # First check to see if --with-tcl was specified. - if test x"${with_wasi_sdk}" != x ; then - if ! test -d "${with_wasi_sdk}" ; then - as_fn_error $? "${with_wasi_sdk} directory doesn't exist" "$LINENO" 5 - fi - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL" >&5 -$as_echo "${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL" >&6; } - use_wasi_sdk=yes - else - use_wasi_sdk=no - fi - -fi - -if test "${use_wasi_sdk}" = "no" ; then - HAVE_WASI_SDK="" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -else - HAVE_WASI_SDK=1 -# Changing --host and --target have no effect here except to possibly -# cause confusion. autoconf has finished processing them by this -# point. -# -# host_alias=wasm32-wasi -# target=wasm32-wasi -# -# Merely changing CC and LD to the wasi-sdk's is enough to get -# sqlite3.o building in WASM format. - CC="${with_wasi_sdk}/bin/clang" - LD="${with_wasi_sdk}/bin/wasm-ld" - RANLIB="${with_wasi_sdk}/bin/llvm-ranlib" - cross_compiling=yes - enable_threadsafe=no - use_tcl=no - enable_tcl=no - # libtool is apparently hard-coded to use gcc for linking DLLs, so - # we disable the DLL build... - enable_shared=no - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi - - - -######### -# Locate a compiler for the build machine. This compiler should -# generate command-line programs that run on the build machine. -# -if test x"$cross_compiling" = xno; then - BUILD_CC=$CC - BUILD_CFLAGS=$CFLAGS -else - if test "${BUILD_CC+set}" != set; then - for ac_prog in gcc cc cl -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_BUILD_CC+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$BUILD_CC"; then - ac_cv_prog_BUILD_CC="$BUILD_CC" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_BUILD_CC="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -BUILD_CC=$ac_cv_prog_BUILD_CC -if test -n "$BUILD_CC"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $BUILD_CC" >&5 -$as_echo "$BUILD_CC" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$BUILD_CC" && break -done - - fi - if test "${BUILD_CFLAGS+set}" != set; then - BUILD_CFLAGS="-g" - fi -fi - - -########## -# Do we want to support multithreaded use of sqlite -# -# Check whether --enable-threadsafe was given. -if test "${enable_threadsafe+set}" = set; then : - enableval=$enable_threadsafe; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support threadsafe operation" >&5 -$as_echo_n "checking whether to support threadsafe operation... " >&6; } -if test "$enable_threadsafe" = "no"; then - SQLITE_THREADSAFE=0 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -else - SQLITE_THREADSAFE=1 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi - - -if test "$SQLITE_THREADSAFE" = "1"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 -$as_echo_n "checking for library containing pthread_create... " >&6; } -if ${ac_cv_search_pthread_create+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char pthread_create (); -int -main () -{ -return pthread_create (); - ; - return 0; -} -_ACEOF -for ac_lib in '' pthread; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_pthread_create=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_pthread_create+:} false; then : - break -fi -done -if ${ac_cv_search_pthread_create+:} false; then : - -else - ac_cv_search_pthread_create=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 -$as_echo "$ac_cv_search_pthread_create" >&6; } -ac_res=$ac_cv_search_pthread_create -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5 -$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; } -if ${ac_cv_search_pthread_mutexattr_init+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char pthread_mutexattr_init (); -int -main () -{ -return pthread_mutexattr_init (); - ; - return 0; -} -_ACEOF -for ac_lib in '' pthread; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_pthread_mutexattr_init=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_pthread_mutexattr_init+:} false; then : - break -fi -done -if ${ac_cv_search_pthread_mutexattr_init+:} false; then : - -else - ac_cv_search_pthread_mutexattr_init=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5 -$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; } -ac_res=$ac_cv_search_pthread_mutexattr_init -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - -fi - -########## -# Which crypto library do we use -# - -# Check whether --with-crypto-lib was given. -if test "${with_crypto_lib+set}" = set; then : - withval=$with_crypto_lib; crypto_lib=$withval -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for crypto library to use" >&5 -$as_echo_n "checking for crypto library to use... " >&6; } -if test "$crypto_lib" = "none"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: none" >&5 -$as_echo "none" >&6; } -else - if test "$crypto_lib" = "commoncrypto"; then - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_CC" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_CC" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: commoncrypto" >&5 -$as_echo "commoncrypto" >&6; } - else - if test "$crypto_lib" = "libtomcrypt"; then - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_LIBTOMCRYPT" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_LIBTOMCRYPT" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: libtomcrypt" >&5 -$as_echo "libtomcrypt" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for register_cipher in -ltomcrypt" >&5 -$as_echo_n "checking for register_cipher in -ltomcrypt... " >&6; } -if ${ac_cv_lib_tomcrypt_register_cipher+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-ltomcrypt $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char register_cipher (); -int -main () -{ -return register_cipher (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_tomcrypt_register_cipher=yes -else - ac_cv_lib_tomcrypt_register_cipher=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_tomcrypt_register_cipher" >&5 -$as_echo "$ac_cv_lib_tomcrypt_register_cipher" >&6; } -if test "x$ac_cv_lib_tomcrypt_register_cipher" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBTOMCRYPT 1 -_ACEOF - - LIBS="-ltomcrypt $LIBS" - -else - as_fn_error $? "Library crypto not found. Install libtomcrypt!\"" "$LINENO" 5 -fi - - else - if test "$crypto_lib" = "nss"; then - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_NSS" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_NSS" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: nss3" >&5 -$as_echo "nss3" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PK11_Decrypt in -lnss3" >&5 -$as_echo_n "checking for PK11_Decrypt in -lnss3... " >&6; } -if ${ac_cv_lib_nss3_PK11_Decrypt+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lnss3 $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char PK11_Decrypt (); -int -main () -{ -return PK11_Decrypt (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_nss3_PK11_Decrypt=yes -else - ac_cv_lib_nss3_PK11_Decrypt=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_nss3_PK11_Decrypt" >&5 -$as_echo "$ac_cv_lib_nss3_PK11_Decrypt" >&6; } -if test "x$ac_cv_lib_nss3_PK11_Decrypt" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBNSS3 1 -_ACEOF - - LIBS="-lnss3 $LIBS" - -else - as_fn_error $? "Library crypto not found. Install nss!\"" "$LINENO" 5 -fi - - else - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_OPENSSL" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_OPENSSL" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: openssl" >&5 -$as_echo "openssl" >&6; } - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for HMAC_Init_ex in -lcrypto" >&5 -$as_echo_n "checking for HMAC_Init_ex in -lcrypto... " >&6; } -if ${ac_cv_lib_crypto_HMAC_Init_ex+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lcrypto $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char HMAC_Init_ex (); -int -main () -{ -return HMAC_Init_ex (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_crypto_HMAC_Init_ex=yes -else - ac_cv_lib_crypto_HMAC_Init_ex=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_crypto_HMAC_Init_ex" >&5 -$as_echo "$ac_cv_lib_crypto_HMAC_Init_ex" >&6; } -if test "x$ac_cv_lib_crypto_HMAC_Init_ex" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBCRYPTO 1 -_ACEOF - - LIBS="-lcrypto $LIBS" - -else - as_fn_error $? "Library crypto not found. Install openssl!\"" "$LINENO" 5 -fi - - fi - fi - fi -fi - -########## -# Do we want to allow a connection created in one thread to be used -# in another thread. This does not work on many Linux systems (ex: RedHat 9) -# due to bugs in the threading implementations. This is thus off by default. -# -# Check whether --enable-cross-thread-connections was given. -if test "${enable_cross_thread_connections+set}" = set; then : - enableval=$enable_cross_thread_connections; -else - enable_xthreadconnect=no -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to allow connections to be shared across threads" >&5 -$as_echo_n "checking whether to allow connections to be shared across threads... " >&6; } -if test "$enable_xthreadconnect" = "no"; then - XTHREADCONNECT='' - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -else - XTHREADCONNECT='-DSQLITE_ALLOW_XTHREAD_CONNECT=1' - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi - - -########## -# Do we want to support release -# -# Check whether --enable-releasemode was given. -if test "${enable_releasemode+set}" = set; then : - enableval=$enable_releasemode; -else - enable_releasemode=no -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support shared library linked as release mode or not" >&5 -$as_echo_n "checking whether to support shared library linked as release mode or not... " >&6; } -if test "$enable_releasemode" = "no"; then - ALLOWRELEASE="" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -else - ALLOWRELEASE="-release `cat $srcdir/VERSION`" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi - - -########## -# Do we want temporary databases in memory -# -# Check whether --enable-tempstore was given. -if test "${enable_tempstore+set}" = set; then : - enableval=$enable_tempstore; -else - enable_tempstore=no -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to use an in-ram database for temporary tables" >&5 -$as_echo_n "checking whether to use an in-ram database for temporary tables... " >&6; } -case "$enable_tempstore" in - never ) - TEMP_STORE=0 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: never" >&5 -$as_echo "never" >&6; } - ;; - no ) - TEMP_STORE=1 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - ;; - yes ) - TEMP_STORE=2 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - ;; - always ) - TEMP_STORE=3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: always" >&5 -$as_echo "always" >&6; } - ;; - * ) - TEMP_STORE=1 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - ;; -esac - - - -########### -# Lots of things are different if we are compiling for Windows using -# the CYGWIN environment. So check for that special case and handle -# things accordingly. -# -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if executables have the .exe suffix" >&5 -$as_echo_n "checking if executables have the .exe suffix... " >&6; } -if test "$config_BUILD_EXEEXT" = ".exe"; then - CYGWIN=yes - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: unknown" >&5 -$as_echo "unknown" >&6; } -fi -if test "$CYGWIN" != "yes"; then - -case $host_os in - *cygwin* ) CYGWIN=yes;; - * ) CYGWIN=no;; -esac - -fi -if test "$CYGWIN" = "yes"; then - BUILD_EXEEXT=.exe -else - BUILD_EXEEXT=$EXEEXT -fi -if test x"$cross_compiling" = xno; then - TARGET_EXEEXT=$BUILD_EXEEXT -else - TARGET_EXEEXT=$config_TARGET_EXEEXT -fi -if test "$TARGET_EXEEXT" = ".exe"; then - SQLITE_OS_UNIX=0 - SQLITE_OS_WIN=1 - CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1" -else - SQLITE_OS_UNIX=1 - SQLITE_OS_WIN=0 - CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1" -fi - - - - - - -########## -# Figure out all the parameters needed to compile against Tcl. -# -# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG -# macros in the in the tcl.m4 file of the standard TCL distribution. -# Those macros could not be used directly since we have to make some -# minor changes to accomodate systems that do not have TCL installed. -# -# Check whether --enable-tcl was given. -if test "${enable_tcl+set}" = set; then : - enableval=$enable_tcl; use_tcl=$enableval -else - use_tcl=yes -fi - -if test "${use_tcl}" = "yes" ; then - -# Check whether --with-tcl was given. -if test "${with_tcl+set}" = set; then : - withval=$with_tcl; with_tclconfig=${withval} -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Tcl configuration" >&5 -$as_echo_n "checking for Tcl configuration... " >&6; } - if ${ac_cv_c_tclconfig+:} false; then : - $as_echo_n "(cached) " >&6 -else - - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)` - else - as_fn_error $? "${with_tclconfig} directory doesn't contain tclConfig.sh" "$LINENO" 5 - fi - fi - - # Start autosearch by asking tclsh - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # On ubuntu 14.10, $auto_path on tclsh is not quite correct. - # So try again after applying corrections. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # Recent versions of Xcode on Macs hid the tclConfig.sh file - # in a strange place. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ../tcl[8-9].[0-9]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ../../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ../../tcl[8-9].[0-9]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ../../../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ../../../tcl[8-9].[0-9]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - `ls -d ${libdir} 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i; pwd)` - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[8-9].[0-9] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[8-9].[0-9]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - -fi - - - if test x"${ac_cv_c_tclconfig}" = x ; then - use_tcl=no - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Can't find Tcl configuration definitions" >&5 -$as_echo "$as_me: WARNING: Can't find Tcl configuration definitions" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&5 -$as_echo "$as_me: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&5 -$as_echo "$as_me: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&2;} - else - TCL_BIN_DIR=${ac_cv_c_tclconfig} - { $as_echo "$as_me:${as_lineno-$LINENO}: result: found $TCL_BIN_DIR/tclConfig.sh" >&5 -$as_echo "found $TCL_BIN_DIR/tclConfig.sh" >&6; } - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for existence of $TCL_BIN_DIR/tclConfig.sh" >&5 -$as_echo_n "checking for existence of $TCL_BIN_DIR/tclConfig.sh... " >&6; } - if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading" >&5 -$as_echo "loading" >&6; } - . $TCL_BIN_DIR/tclConfig.sh - else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: file not found" >&5 -$as_echo "file not found" >&6; } - fi - - # - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - # - - if test -f $TCL_BIN_DIR/Makefile ; then - TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC} - TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC} - TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH} - fi - - # - # eval is required to do the TCL_DBGX substitution - # - - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - - - - - - - - - - - - - - - fi -fi -if test "${use_tcl}" = "no" ; then - HAVE_TCL="" -else - HAVE_TCL=1 -fi - - -########## -# Figure out what C libraries are required to compile programs -# that use "readline()" library. -# -TARGET_READLINE_LIBS="" -TARGET_READLINE_INC="" -TARGET_HAVE_READLINE=0 -TARGET_HAVE_EDITLINE=0 -# Check whether --enable-editline was given. -if test "${enable_editline+set}" = set; then : - enableval=$enable_editline; with_editline=$enableval -else - with_editline=auto -fi - -# Check whether --enable-readline was given. -if test "${enable_readline+set}" = set; then : - enableval=$enable_readline; with_readline=$enableval -else - with_readline=auto -fi - - -if test x"$with_editline" != xno; then - sLIBS=$LIBS - LIBS="" - TARGET_HAVE_EDITLINE=1 - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing readline" >&5 -$as_echo_n "checking for library containing readline... " >&6; } -if ${ac_cv_search_readline+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char readline (); -int -main () -{ -return readline (); - ; - return 0; -} -_ACEOF -for ac_lib in '' edit; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_readline=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_readline+:} false; then : - break -fi -done -if ${ac_cv_search_readline+:} false; then : - -else - ac_cv_search_readline=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_readline" >&5 -$as_echo "$ac_cv_search_readline" >&6; } -ac_res=$ac_cv_search_readline -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - with_readline=no -else - TARGET_HAVE_EDITLINE=0 -fi - - TARGET_READLINE_LIBS=$LIBS - LIBS=$sLIBS -fi -if test x"$with_readline" != xno; then - found="yes" - - -# Check whether --with-readline-lib was given. -if test "${with_readline_lib+set}" = set; then : - withval=$with_readline_lib; with_readline_lib=$withval -else - with_readline_lib="auto" -fi - - if test "x$with_readline_lib" = xauto; then - save_LIBS="$LIBS" - LIBS="" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing tgetent" >&5 -$as_echo_n "checking for library containing tgetent... " >&6; } -if ${ac_cv_search_tgetent+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char tgetent (); -int -main () -{ -return tgetent (); - ; - return 0; -} -_ACEOF -for ac_lib in '' readline ncurses curses termcap; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_tgetent=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_tgetent+:} false; then : - break -fi -done -if ${ac_cv_search_tgetent+:} false; then : - -else - ac_cv_search_tgetent=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_tgetent" >&5 -$as_echo "$ac_cv_search_tgetent" >&6; } -ac_res=$ac_cv_search_tgetent -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - term_LIBS="$LIBS" -else - term_LIBS="" -fi - - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for readline in -lreadline" >&5 -$as_echo_n "checking for readline in -lreadline... " >&6; } -if ${ac_cv_lib_readline_readline+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-lreadline $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char readline (); -int -main () -{ -return readline (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_readline_readline=yes -else - ac_cv_lib_readline_readline=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_readline_readline" >&5 -$as_echo "$ac_cv_lib_readline_readline" >&6; } -if test "x$ac_cv_lib_readline_readline" = xyes; then : - TARGET_READLINE_LIBS="-lreadline" -else - found="no" -fi - - TARGET_READLINE_LIBS="$TARGET_READLINE_LIBS $term_LIBS" - LIBS="$save_LIBS" - else - TARGET_READLINE_LIBS="$with_readline_lib" - fi - - -# Check whether --with-readline-inc was given. -if test "${with_readline_inc+set}" = set; then : - withval=$with_readline_inc; with_readline_inc=$withval -else - with_readline_inc="auto" -fi - - if test "x$with_readline_inc" = xauto; then - ac_fn_c_check_header_mongrel "$LINENO" "readline.h" "ac_cv_header_readline_h" "$ac_includes_default" -if test "x$ac_cv_header_readline_h" = xyes; then : - found="yes" -else - - found="no" - if test "$cross_compiling" != yes; then - for dir in /usr /usr/local /usr/local/readline /usr/contrib /mingw; do - for subdir in include include/readline; do - as_ac_File=`$as_echo "ac_cv_file_$dir/$subdir/readline.h" | $as_tr_sh` -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $dir/$subdir/readline.h" >&5 -$as_echo_n "checking for $dir/$subdir/readline.h... " >&6; } -if eval \${$as_ac_File+:} false; then : - $as_echo_n "(cached) " >&6 -else - test "$cross_compiling" = yes && - as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 -if test -r "$dir/$subdir/readline.h"; then - eval "$as_ac_File=yes" -else - eval "$as_ac_File=no" -fi -fi -eval ac_res=\$$as_ac_File - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -if eval test \"x\$"$as_ac_File"\" = x"yes"; then : - found=yes -fi - - if test "$found" = "yes"; then - TARGET_READLINE_INC="-I$dir/$subdir" - break - fi - done - test "$found" = "yes" && break - done - fi - -fi - - - else - TARGET_READLINE_INC="$with_readline_inc" - fi - - if test x"$found" = xno; then - TARGET_READLINE_LIBS="" - TARGET_READLINE_INC="" - TARGET_HAVE_READLINE=0 - else - TARGET_HAVE_READLINE=1 - fi -fi - - - - - - -########## -# Figure out what C libraries are required to compile programs -# that use "fdatasync()" function. -# -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing fdatasync" >&5 -$as_echo_n "checking for library containing fdatasync... " >&6; } -if ${ac_cv_search_fdatasync+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char fdatasync (); -int -main () -{ -return fdatasync (); - ; - return 0; -} -_ACEOF -for ac_lib in '' rt; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_fdatasync=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_fdatasync+:} false; then : - break -fi -done -if ${ac_cv_search_fdatasync+:} false; then : - -else - ac_cv_search_fdatasync=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_fdatasync" >&5 -$as_echo "$ac_cv_search_fdatasync" >&6; } -ac_res=$ac_cv_search_fdatasync -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - - -######### -# check for debug enabled -# Check whether --enable-debug was given. -if test "${enable_debug+set}" = set; then : - enableval=$enable_debug; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking build type" >&5 -$as_echo_n "checking build type... " >&6; } -if test "${enable_debug}" = "yes" ; then - TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug" >&5 -$as_echo "debug" >&6; } -else - TARGET_DEBUG="-DNDEBUG" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: release" >&5 -$as_echo "release" >&6; } -fi - - -######### -# See whether we should use the amalgamation to build - -# Check whether --enable-amalgamation was given. -if test "${enable_amalgamation+set}" = set; then : - enableval=$enable_amalgamation; -fi - -if test "${enable_amalgamation}" = "no" ; then - USE_AMALGAMATION=0 -fi - - -######### -# By default, amalgamation sqlite3.c will have #line directives. -# This is a build option not shown by ./configure --help -# To control it, use configure option: amalgamation_line_macros=? -# where ? is no to suppress #line directives or yes to create them. -AMALGAMATION_LINE_MACROS=--linemacros=1 - - -if test "${amalgamation_line_macros+set}" = set; then : - enableval=$amalgamation_line_macros; -fi -if test "${amalgamation_line_macros}" = "yes" ; then - AMALGAMATION_LINE_MACROS=--linemacros=1 -fi -if test "${amalgamation_line_macros}" = "no" ; then - AMALGAMATION_LINE_MACROS=--linemacros=0 -fi - -######### -# Look for zlib. Only needed by extensions and by the sqlite3.exe shell -for ac_header in zlib.h -do : - ac_fn_c_check_header_mongrel "$LINENO" "zlib.h" "ac_cv_header_zlib_h" "$ac_includes_default" -if test "x$ac_cv_header_zlib_h" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_ZLIB_H 1 -_ACEOF - -fi - -done - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing deflate" >&5 -$as_echo_n "checking for library containing deflate... " >&6; } -if ${ac_cv_search_deflate+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char deflate (); -int -main () -{ -return deflate (); - ; - return 0; -} -_ACEOF -for ac_lib in '' z; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_deflate=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_deflate+:} false; then : - break -fi -done -if ${ac_cv_search_deflate+:} false; then : - -else - ac_cv_search_deflate=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_deflate" >&5 -$as_echo "$ac_cv_search_deflate" >&6; } -ac_res=$ac_cv_search_deflate -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1" -else - HAVE_ZLIB="" -fi - - - -######### -# See whether we should allow loadable extensions -# Check whether --enable-load-extension was given. -if test "${enable_load_extension+set}" = set; then : - enableval=$enable_load_extension; -else - enable_load_extension=yes -fi - -if test "${enable_load_extension}" = "yes" ; then - OPT_FEATURE_FLAGS="" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing dlopen" >&5 -$as_echo_n "checking for library containing dlopen... " >&6; } -if ${ac_cv_search_dlopen+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char dlopen (); -int -main () -{ -return dlopen (); - ; - return 0; -} -_ACEOF -for ac_lib in '' dl; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_dlopen=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_dlopen+:} false; then : - break -fi -done -if ${ac_cv_search_dlopen+:} false; then : - -else - ac_cv_search_dlopen=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_dlopen" >&5 -$as_echo "$ac_cv_search_dlopen" >&6; } -ac_res=$ac_cv_search_dlopen -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - -else - OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" -fi - -########## -# Do we want to support math functions -# -# Check whether --enable-math was given. -if test "${enable_math+set}" = set; then : - enableval=$enable_math; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support math functions" >&5 -$as_echo_n "checking whether to support math functions... " >&6; } -if test "$enable_math" = "no"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MATH_FUNCTIONS" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing ceil" >&5 -$as_echo_n "checking for library containing ceil... " >&6; } -if ${ac_cv_search_ceil+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char ceil (); -int -main () -{ -return ceil (); - ; - return 0; -} -_ACEOF -for ac_lib in '' m; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_ceil=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_ceil+:} false; then : - break -fi -done -if ${ac_cv_search_ceil+:} false; then : - -else - ac_cv_search_ceil=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ceil" >&5 -$as_echo "$ac_cv_search_ceil" >&6; } -ac_res=$ac_cv_search_ceil -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - -fi - -########## -# Do we want to support JSON functions -# -# Check whether --enable-json was given. -if test "${enable_json+set}" = set; then : - enableval=$enable_json; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support JSON functions" >&5 -$as_echo_n "checking whether to support JSON functions... " >&6; } -if test "$enable_json" = "no"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_OMIT_JSON" -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -fi - -######## -# The --enable-all argument is short-hand to enable -# multiple extensions. -# Check whether --enable-all was given. -if test "${enable_all+set}" = set; then : - enableval=$enable_all; -fi - - -########## -# Do we want to support memsys3 and/or memsys5 -# -# Check whether --enable-memsys5 was given. -if test "${enable_memsys5+set}" = set; then : - enableval=$enable_memsys5; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5 -$as_echo_n "checking whether to support MEMSYS5... " >&6; } -if test "${enable_memsys5}" = "yes"; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi -# Check whether --enable-memsys3 was given. -if test "${enable_memsys3+set}" = set; then : - enableval=$enable_memsys3; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5 -$as_echo_n "checking whether to support MEMSYS3... " >&6; } -if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - -######### -# See whether we should enable Full Text Search extensions -# Check whether --enable-fts3 was given. -if test "${enable_fts3+set}" = set; then : - enableval=$enable_fts3; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support FTS3" >&5 -$as_echo_n "checking whether to support FTS3... " >&6; } -if test "${enable_fts3}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi -# Check whether --enable-fts4 was given. -if test "${enable_fts4+set}" = set; then : - enableval=$enable_fts4; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support FTS4" >&5 -$as_echo_n "checking whether to support FTS4... " >&6; } -if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 -$as_echo_n "checking for library containing log... " >&6; } -if ${ac_cv_search_log+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char log (); -int -main () -{ -return log (); - ; - return 0; -} -_ACEOF -for ac_lib in '' m; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_log=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_log+:} false; then : - break -fi -done -if ${ac_cv_search_log+:} false; then : - -else - ac_cv_search_log=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_log" >&5 -$as_echo "$ac_cv_search_log" >&6; } -ac_res=$ac_cv_search_log -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi -# Check whether --enable-fts5 was given. -if test "${enable_fts5+set}" = set; then : - enableval=$enable_fts5; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support FTS5" >&5 -$as_echo_n "checking whether to support FTS5... " >&6; } -if test "${enable_fts5}" = "yes" -o "${enable_all}" = "yes" ; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5" - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 -$as_echo_n "checking for library containing log... " >&6; } -if ${ac_cv_search_log+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char log (); -int -main () -{ -return log (); - ; - return 0; -} -_ACEOF -for ac_lib in '' m; do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO"; then : - ac_cv_search_log=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext - if ${ac_cv_search_log+:} false; then : - break -fi -done -if ${ac_cv_search_log+:} false; then : - -else - ac_cv_search_log=no -fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_log" >&5 -$as_echo "$ac_cv_search_log" >&6; } -ac_res=$ac_cv_search_log -if test "$ac_res" != no; then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - -fi - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - -######### -# See whether we should enable the LIMIT clause on UPDATE and DELETE -# statements. -# Check whether --enable-update-limit was given. -if test "${enable_update_limit+set}" = set; then : - enableval=$enable_update_limit; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support LIMIT on UPDATE and DELETE statements" >&5 -$as_echo_n "checking whether to support LIMIT on UPDATE and DELETE statements... " >&6; } -if test "${enable_update_limit}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - -######### -# See whether we should enable GEOPOLY -# Check whether --enable-geopoly was given. -if test "${enable_geopoly+set}" = set; then : - enableval=$enable_geopoly; enable_geopoly=yes -else - enable_geopoly=no -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support GEOPOLY" >&5 -$as_echo_n "checking whether to support GEOPOLY... " >&6; } -if test "${enable_geopoly}" = "yes" -o "${enable_all}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY" - enable_rtree=yes - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - -######### -# See whether we should enable RTREE -# Check whether --enable-rtree was given. -if test "${enable_rtree+set}" = set; then : - enableval=$enable_rtree; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support RTREE" >&5 -$as_echo_n "checking whether to support RTREE... " >&6; } -if test "${enable_rtree}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - -######### -# See whether we should enable the SESSION extension -# Check whether --enable-session was given. -if test "${enable_session+set}" = set; then : - enableval=$enable_session; -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support SESSION" >&5 -$as_echo_n "checking whether to support SESSION... " >&6; } -if test "${enable_session}" = "yes" -o "${enable_all}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION" - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - -######### -# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter -for option in $CFLAGS $CPPFLAGS -do - case $option in - -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";; - -DSQLITE_ENABLE*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";; - esac -done - - - -# attempt to remove any OMITS and ENABLES from the $(CFLAGS) parameter -ac_temp_CFLAGS="" -for option in $CFLAGS -do - case $option in - -DSQLITE_OMIT*) ;; - -DSQLITE_ENABLE*) ;; - *) ac_temp_CFLAGS="$ac_temp_CFLAGS $option";; - esac -done -CFLAGS=$ac_temp_CFLAGS - - -# attempt to remove any OMITS and ENABLES from the $(CPPFLAGS) parameter -ac_temp_CPPFLAGS="" -for option in $CPPFLAGS -do - case $option in - -DSQLITE_OMIT*) ;; - -DSQLITE_ENABLE*) ;; - *) ac_temp_CPPFLAGS="$ac_temp_CPPFLAGS $option";; - esac -done -CPPFLAGS=$ac_temp_CPPFLAGS - - -# attempt to remove any OMITS and ENABLES from the $(BUILD_CFLAGS) parameter -ac_temp_BUILD_CFLAGS="" -for option in $BUILD_CFLAGS -do - case $option in - -DSQLITE_OMIT*) ;; - -DSQLITE_ENABLE*) ;; - *) ac_temp_BUILD_CFLAGS="$ac_temp_BUILD_CFLAGS $option";; - esac -done -BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS - - -######### -# See whether we should use GCOV -# Check whether --enable-gcov was given. -if test "${enable_gcov+set}" = set; then : - enableval=$enable_gcov; -fi - -if test "${use_gcov}" = "yes" ; then - USE_GCOV=1 -else - USE_GCOV=0 -fi - - -######### -# Enable/disabled amalagamation line macros -######## -AMALGAMATION_LINE_MACROS=--linemacros=0 -if test "${amalgamation_line_macros}" = "yes" ; then - AMALGAMATION_LINE_MACROS=--linemacros=1 -fi -if test "${amalgamation_line_macros}" = "no" ; then - AMALGAMATION_LINE_MACROS=--linemacros=0 -fi - - -######### -# Output the config header -ac_config_headers="$ac_config_headers sqlite_cfg.h" - - -######### -# Generate the output files. -# - -ac_config_files="$ac_config_files Makefile sqlcipher.pc" - -cat >confcache <<\_ACEOF -# This file is a shell script that caches the results of configure -# tests run on this system so they can be shared between configure -# scripts and configure runs, see configure's option --config-cache. -# It is not useful on other systems. If it contains results you don't -# want to keep, you may remove or edit it. -# -# config.status only pays attention to the cache file if you give it -# the --recheck option to rerun configure. -# -# `ac_cv_env_foo' variables (set or unset) will be overridden when -# loading this file, other *unset* `ac_cv_foo' will be assigned the -# following values. - -_ACEOF - -# The following way of writing the cache mishandles newlines in values, -# but we know of no workaround that is simple, portable, and efficient. -# So, we kill variables containing newlines. -# Ultrix sh set writes to stderr and can't be redirected directly, -# and sets the high bit in the cache file unless we assign to the vars. -( - for ac_var in `(set) 2>&1 | sed -n 's/^\([a-zA-Z_][a-zA-Z0-9_]*\)=.*/\1/p'`; do - eval ac_val=\$$ac_var - case $ac_val in #( - *${as_nl}*) - case $ac_var in #( - *_cv_*) { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cache variable $ac_var contains a newline" >&5 -$as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; - esac - case $ac_var in #( - _ | IFS | as_nl) ;; #( - BASH_ARGV | BASH_SOURCE) eval $ac_var= ;; #( - *) { eval $ac_var=; unset $ac_var;} ;; - esac ;; - esac - done - - (set) 2>&1 | - case $as_nl`(ac_space=' '; set) 2>&1` in #( - *${as_nl}ac_space=\ *) - # `set' does not quote correctly, so add quotes: double-quote - # substitution turns \\\\ into \\, and sed turns \\ into \. - sed -n \ - "s/'/'\\\\''/g; - s/^\\([_$as_cr_alnum]*_cv_[_$as_cr_alnum]*\\)=\\(.*\\)/\\1='\\2'/p" - ;; #( - *) - # `set' quotes correctly as required by POSIX, so do not add quotes. - sed -n "/^[_$as_cr_alnum]*_cv_[_$as_cr_alnum]*=/p" - ;; - esac | - sort -) | - sed ' - /^ac_cv_env_/b end - t clear - :clear - s/^\([^=]*\)=\(.*[{}].*\)$/test "${\1+set}" = set || &/ - t end - s/^\([^=]*\)=\(.*\)$/\1=${\1=\2}/ - :end' >>confcache -if diff "$cache_file" confcache >/dev/null 2>&1; then :; else - if test -w "$cache_file"; then - if test "x$cache_file" != "x/dev/null"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 -$as_echo "$as_me: updating cache $cache_file" >&6;} - if test ! -f "$cache_file" || test -h "$cache_file"; then - cat confcache >"$cache_file" - else - case $cache_file in #( - */* | ?:*) - mv -f confcache "$cache_file"$$ && - mv -f "$cache_file"$$ "$cache_file" ;; #( - *) - mv -f confcache "$cache_file" ;; - esac - fi - fi - else - { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 -$as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} - fi -fi -rm -f confcache - -test "x$prefix" = xNONE && prefix=$ac_default_prefix -# Let make expand exec_prefix. -test "x$exec_prefix" = xNONE && exec_prefix='${prefix}' - -DEFS=-DHAVE_CONFIG_H - -ac_libobjs= -ac_ltlibobjs= -U= -for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue - # 1. Remove the extension, and $U if already installed. - ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' - ac_i=`$as_echo "$ac_i" | sed "$ac_script"` - # 2. Prepend LIBOBJDIR. When used with automake>=1.10 LIBOBJDIR - # will be set to the directory where LIBOBJS objects are built. - as_fn_append ac_libobjs " \${LIBOBJDIR}$ac_i\$U.$ac_objext" - as_fn_append ac_ltlibobjs " \${LIBOBJDIR}$ac_i"'$U.lo' -done -LIBOBJS=$ac_libobjs - -LTLIBOBJS=$ac_ltlibobjs - - - -: "${CONFIG_STATUS=./config.status}" -ac_write_fail=0 -ac_clean_files_save=$ac_clean_files -ac_clean_files="$ac_clean_files $CONFIG_STATUS" -{ $as_echo "$as_me:${as_lineno-$LINENO}: creating $CONFIG_STATUS" >&5 -$as_echo "$as_me: creating $CONFIG_STATUS" >&6;} -as_write_fail=0 -cat >$CONFIG_STATUS <<_ASEOF || as_write_fail=1 -#! $SHELL -# Generated by $as_me. -# Run this file to recreate the current configuration. -# Compiler output produced by configure, useful for debugging -# configure, is in config.log if it exists. - -debug=false -ac_cs_recheck=false -ac_cs_silent=false - -SHELL=\${CONFIG_SHELL-$SHELL} -export SHELL -_ASEOF -cat >>$CONFIG_STATUS <<\_ASEOF || as_write_fail=1 -## -------------------- ## -## M4sh Initialization. ## -## -------------------- ## - -# Be more Bourne compatible -DUALCASE=1; export DUALCASE # for MKS sh -if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : - emulate sh - NULLCMD=: - # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. - alias -g '${1+"$@"}'='"$@"' - setopt NO_GLOB_SUBST -else - case `(set -o) 2>/dev/null` in #( - *posix*) : - set -o posix ;; #( - *) : - ;; -esac -fi - - -as_nl=' -' -export as_nl -# Printing a long string crashes Solaris 7 /usr/bin/printf. -as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' -as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo -as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo -# Prefer a ksh shell builtin over an external printf program on Solaris, -# but without wasting forks for bash or zsh. -if test -z "$BASH_VERSION$ZSH_VERSION" \ - && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then - as_echo='print -r --' - as_echo_n='print -rn --' -elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then - as_echo='printf %s\n' - as_echo_n='printf %s' -else - if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then - as_echo_body='eval /usr/ucb/echo -n "$1$as_nl"' - as_echo_n='/usr/ucb/echo -n' - else - as_echo_body='eval expr "X$1" : "X\\(.*\\)"' - as_echo_n_body='eval - arg=$1; - case $arg in #( - *"$as_nl"*) - expr "X$arg" : "X\\(.*\\)$as_nl"; - arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; - esac; - expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" - ' - export as_echo_n_body - as_echo_n='sh -c $as_echo_n_body as_echo' - fi - export as_echo_body - as_echo='sh -c $as_echo_body as_echo' -fi - -# The user is always right. -if test "${PATH_SEPARATOR+set}" != set; then - PATH_SEPARATOR=: - (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { - (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || - PATH_SEPARATOR=';' - } -fi - - -# IFS -# We need space, tab and new line, in precisely that order. Quoting is -# there to prevent editors from complaining about space-tab. -# (If _AS_PATH_WALK were called with IFS unset, it would disable word -# splitting by setting IFS to empty value.) -IFS=" "" $as_nl" - -# Find who we are. Look in the path if we contain no directory separator. -as_myself= -case $0 in #(( - *[\\/]* ) as_myself=$0 ;; - *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - test -r "$as_dir/$0" && as_myself=$as_dir/$0 && break - done -IFS=$as_save_IFS - - ;; -esac -# We did not find ourselves, most probably we were run as `sh COMMAND' -# in which case we are not to be found in the path. -if test "x$as_myself" = x; then - as_myself=$0 -fi -if test ! -f "$as_myself"; then - $as_echo "$as_myself: error: cannot find myself; rerun with an absolute file name" >&2 - exit 1 -fi - -# Unset variables that we do not need and which cause bugs (e.g. in -# pre-3.0 UWIN ksh). But do not cause bugs in bash 2.01; the "|| exit 1" -# suppresses any "Segmentation fault" message there. '((' could -# trigger a bug in pdksh 5.2.14. -for as_var in BASH_ENV ENV MAIL MAILPATH -do eval test x\${$as_var+set} = xset \ - && ( (unset $as_var) || exit 1) >/dev/null 2>&1 && unset $as_var || : -done -PS1='$ ' -PS2='> ' -PS4='+ ' - -# NLS nuisances. -LC_ALL=C -export LC_ALL -LANGUAGE=C -export LANGUAGE - -# CDPATH. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH - - -# as_fn_error STATUS ERROR [LINENO LOG_FD] -# ---------------------------------------- -# Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are -# provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with STATUS, using 1 if that was 0. -as_fn_error () -{ - as_status=$1; test $as_status -eq 0 && as_status=1 - if test "$4"; then - as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 - fi - $as_echo "$as_me: error: $2" >&2 - as_fn_exit $as_status -} # as_fn_error - - -# as_fn_set_status STATUS -# ----------------------- -# Set $? to STATUS, without forking. -as_fn_set_status () -{ - return $1 -} # as_fn_set_status - -# as_fn_exit STATUS -# ----------------- -# Exit the shell with STATUS, even in a "trap 0" or "set -e" context. -as_fn_exit () -{ - set +e - as_fn_set_status $1 - exit $1 -} # as_fn_exit - -# as_fn_unset VAR -# --------------- -# Portably unset VAR. -as_fn_unset () -{ - { eval $1=; unset $1;} -} -as_unset=as_fn_unset -# as_fn_append VAR VALUE -# ---------------------- -# Append the text in VALUE to the end of the definition contained in VAR. Take -# advantage of any shell optimizations that allow amortized linear growth over -# repeated appends, instead of the typical quadratic growth present in naive -# implementations. -if (eval "as_var=1; as_var+=2; test x\$as_var = x12") 2>/dev/null; then : - eval 'as_fn_append () - { - eval $1+=\$2 - }' -else - as_fn_append () - { - eval $1=\$$1\$2 - } -fi # as_fn_append - -# as_fn_arith ARG... -# ------------------ -# Perform arithmetic evaluation on the ARGs, and store the result in the -# global $as_val. Take advantage of shells that can avoid forks. The arguments -# must be portable across $(()) and expr. -if (eval "test \$(( 1 + 1 )) = 2") 2>/dev/null; then : - eval 'as_fn_arith () - { - as_val=$(( $* )) - }' -else - as_fn_arith () - { - as_val=`expr "$@" || test $? -eq 1` - } -fi # as_fn_arith - - -if expr a : '\(a\)' >/dev/null 2>&1 && - test "X`expr 00001 : '.*\(...\)'`" = X001; then - as_expr=expr -else - as_expr=false -fi - -if (basename -- /) >/dev/null 2>&1 && test "X`basename -- / 2>&1`" = "X/"; then - as_basename=basename -else - as_basename=false -fi - -if (as_dir=`dirname -- /` && test "X$as_dir" = X/) >/dev/null 2>&1; then - as_dirname=dirname -else - as_dirname=false -fi - -as_me=`$as_basename -- "$0" || -$as_expr X/"$0" : '.*/\([^/][^/]*\)/*$' \| \ - X"$0" : 'X\(//\)$' \| \ - X"$0" : 'X\(/\)' \| . 2>/dev/null || -$as_echo X/"$0" | - sed '/^.*\/\([^/][^/]*\)\/*$/{ - s//\1/ - q - } - /^X\/\(\/\/\)$/{ - s//\1/ - q - } - /^X\/\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q'` - -# Avoid depending upon Character Ranges. -as_cr_letters='abcdefghijklmnopqrstuvwxyz' -as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ' -as_cr_Letters=$as_cr_letters$as_cr_LETTERS -as_cr_digits='0123456789' -as_cr_alnum=$as_cr_Letters$as_cr_digits - -ECHO_C= ECHO_N= ECHO_T= -case `echo -n x` in #((((( --n*) - case `echo 'xy\c'` in - *c*) ECHO_T=' ';; # ECHO_T is single tab character. - xy) ECHO_C='\c';; - *) echo `echo ksh88 bug on AIX 6.1` > /dev/null - ECHO_T=' ';; - esac;; -*) - ECHO_N='-n';; -esac - -rm -f conf$$ conf$$.exe conf$$.file -if test -d conf$$.dir; then - rm -f conf$$.dir/conf$$.file -else - rm -f conf$$.dir - mkdir conf$$.dir 2>/dev/null -fi -if (echo >conf$$.file) 2>/dev/null; then - if ln -s conf$$.file conf$$ 2>/dev/null; then - as_ln_s='ln -s' - # ... but there are two gotchas: - # 1) On MSYS, both `ln -s file dir' and `ln file dir' fail. - # 2) DJGPP < 2.04 has no symlinks; `ln -s' creates a wrapper executable. - # In both cases, we have to default to `cp -pR'. - ln -s conf$$.file conf$$.dir 2>/dev/null && test ! -f conf$$.exe || - as_ln_s='cp -pR' - elif ln conf$$.file conf$$ 2>/dev/null; then - as_ln_s=ln - else - as_ln_s='cp -pR' - fi -else - as_ln_s='cp -pR' -fi -rm -f conf$$ conf$$.exe conf$$.dir/conf$$.file conf$$.file -rmdir conf$$.dir 2>/dev/null - - -# as_fn_mkdir_p -# ------------- -# Create "$as_dir" as a directory, including parents if necessary. -as_fn_mkdir_p () -{ - - case $as_dir in #( - -*) as_dir=./$as_dir;; - esac - test -d "$as_dir" || eval $as_mkdir_p || { - as_dirs= - while :; do - case $as_dir in #( - *\'*) as_qdir=`$as_echo "$as_dir" | sed "s/'/'\\\\\\\\''/g"`;; #'( - *) as_qdir=$as_dir;; - esac - as_dirs="'$as_qdir' $as_dirs" - as_dir=`$as_dirname -- "$as_dir" || -$as_expr X"$as_dir" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$as_dir" : 'X\(//\)[^/]' \| \ - X"$as_dir" : 'X\(//\)$' \| \ - X"$as_dir" : 'X\(/\)' \| . 2>/dev/null || -$as_echo X"$as_dir" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ - s//\1/ - q - } - /^X\(\/\/\)[^/].*/{ - s//\1/ - q - } - /^X\(\/\/\)$/{ - s//\1/ - q - } - /^X\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q'` - test -d "$as_dir" && break - done - test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" - - -} # as_fn_mkdir_p -if mkdir -p . 2>/dev/null; then - as_mkdir_p='mkdir -p "$as_dir"' -else - test -d ./-p && rmdir ./-p - as_mkdir_p=false -fi - - -# as_fn_executable_p FILE -# ----------------------- -# Test if FILE is an executable regular file. -as_fn_executable_p () -{ - test -f "$1" && test -x "$1" -} # as_fn_executable_p -as_test_x='test -x' -as_executable_p=as_fn_executable_p - -# Sed expression to map a string onto a valid CPP name. -as_tr_cpp="eval sed 'y%*$as_cr_letters%P$as_cr_LETTERS%;s%[^_$as_cr_alnum]%_%g'" - -# Sed expression to map a string onto a valid variable name. -as_tr_sh="eval sed 'y%*+%pp%;s%[^_$as_cr_alnum]%_%g'" - - -exec 6>&1 -## ----------------------------------- ## -## Main body of $CONFIG_STATUS script. ## -## ----------------------------------- ## -_ASEOF -test $as_write_fail = 0 && chmod +x $CONFIG_STATUS || ac_write_fail=1 - -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -# Save the log message, to keep $0 and so on meaningful, and to -# report actual input values of CONFIG_FILES etc. instead of their -# values after options handling. -ac_log=" -This file was extended by sqlcipher $as_me 3.42.0, which was -generated by GNU Autoconf 2.69. Invocation command line was - - CONFIG_FILES = $CONFIG_FILES - CONFIG_HEADERS = $CONFIG_HEADERS - CONFIG_LINKS = $CONFIG_LINKS - CONFIG_COMMANDS = $CONFIG_COMMANDS - $ $0 $@ - -on `(hostname || uname -n) 2>/dev/null | sed 1q` -" - -_ACEOF - -case $ac_config_files in *" -"*) set x $ac_config_files; shift; ac_config_files=$*;; -esac - -case $ac_config_headers in *" -"*) set x $ac_config_headers; shift; ac_config_headers=$*;; -esac - - -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -# Files that config.status was made for. -config_files="$ac_config_files" -config_headers="$ac_config_headers" -config_commands="$ac_config_commands" - -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -ac_cs_usage="\ -\`$as_me' instantiates files and other configuration actions -from templates according to the current configuration. Unless the files -and actions are specified as TAGs, all are instantiated by default. - -Usage: $0 [OPTION]... [TAG]... - - -h, --help print this help, then exit - -V, --version print version number and configuration settings, then exit - --config print configuration, then exit - -q, --quiet, --silent - do not print progress messages - -d, --debug don't remove temporary files - --recheck update $as_me by reconfiguring in the same conditions - --file=FILE[:TEMPLATE] - instantiate the configuration file FILE - --header=FILE[:TEMPLATE] - instantiate the configuration header FILE - -Configuration files: -$config_files - -Configuration headers: -$config_headers - -Configuration commands: -$config_commands - -Report bugs to the package provider." - -_ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" -ac_cs_version="\\ -sqlcipher config.status 3.42.0 -configured by $0, generated by GNU Autoconf 2.69, - with options \\"\$ac_cs_config\\" - -Copyright (C) 2012 Free Software Foundation, Inc. -This config.status script is free software; the Free Software Foundation -gives unlimited permission to copy, distribute and modify it." - -ac_pwd='$ac_pwd' -srcdir='$srcdir' -INSTALL='$INSTALL' -AWK='$AWK' -test -n "\$AWK" || AWK=awk -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -# The default lists apply if the user does not specify any file. -ac_need_defaults=: -while test $# != 0 -do - case $1 in - --*=?*) - ac_option=`expr "X$1" : 'X\([^=]*\)='` - ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` - ac_shift=: - ;; - --*=) - ac_option=`expr "X$1" : 'X\([^=]*\)='` - ac_optarg= - ac_shift=: - ;; - *) - ac_option=$1 - ac_optarg=$2 - ac_shift=shift - ;; - esac - - case $ac_option in - # Handling of the options. - -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r) - ac_cs_recheck=: ;; - --version | --versio | --versi | --vers | --ver | --ve | --v | -V ) - $as_echo "$ac_cs_version"; exit ;; - --config | --confi | --conf | --con | --co | --c ) - $as_echo "$ac_cs_config"; exit ;; - --debug | --debu | --deb | --de | --d | -d ) - debug=: ;; - --file | --fil | --fi | --f ) - $ac_shift - case $ac_optarg in - *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; - '') as_fn_error $? "missing file argument" ;; - esac - as_fn_append CONFIG_FILES " '$ac_optarg'" - ac_need_defaults=false;; - --header | --heade | --head | --hea ) - $ac_shift - case $ac_optarg in - *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; - esac - as_fn_append CONFIG_HEADERS " '$ac_optarg'" - ac_need_defaults=false;; - --he | --h) - # Conflict between --help and --header - as_fn_error $? "ambiguous option: \`$1' -Try \`$0 --help' for more information.";; - --help | --hel | -h ) - $as_echo "$ac_cs_usage"; exit ;; - -q | -quiet | --quiet | --quie | --qui | --qu | --q \ - | -silent | --silent | --silen | --sile | --sil | --si | --s) - ac_cs_silent=: ;; - - # This is an error. - -*) as_fn_error $? "unrecognized option: \`$1' -Try \`$0 --help' for more information." ;; - - *) as_fn_append ac_config_targets " $1" - ac_need_defaults=false ;; - - esac - shift -done - -ac_configure_extra_args= - -if $ac_cs_silent; then - exec 6>/dev/null - ac_configure_extra_args="$ac_configure_extra_args --silent" -fi - -_ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -if \$ac_cs_recheck; then - set X $SHELL '$0' $ac_configure_args \$ac_configure_extra_args --no-create --no-recursion - shift - \$as_echo "running CONFIG_SHELL=$SHELL \$*" >&6 - CONFIG_SHELL='$SHELL' - export CONFIG_SHELL - exec "\$@" -fi - -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -exec 5>>config.log -{ - echo - sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX -## Running $as_me. ## -_ASBOX - $as_echo "$ac_log" -} >&5 - -_ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -# -# INIT-COMMANDS -# - - -# The HP-UX ksh and POSIX shell print the target directory to stdout -# if CDPATH is set. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH - -sed_quote_subst='$sed_quote_subst' -double_quote_subst='$double_quote_subst' -delay_variable_subst='$delay_variable_subst' -macro_version='`$ECHO "$macro_version" | $SED "$delay_single_quote_subst"`' -macro_revision='`$ECHO "$macro_revision" | $SED "$delay_single_quote_subst"`' -enable_shared='`$ECHO "$enable_shared" | $SED "$delay_single_quote_subst"`' -enable_static='`$ECHO "$enable_static" | $SED "$delay_single_quote_subst"`' -pic_mode='`$ECHO "$pic_mode" | $SED "$delay_single_quote_subst"`' -enable_fast_install='`$ECHO "$enable_fast_install" | $SED "$delay_single_quote_subst"`' -shared_archive_member_spec='`$ECHO "$shared_archive_member_spec" | $SED "$delay_single_quote_subst"`' -SHELL='`$ECHO "$SHELL" | $SED "$delay_single_quote_subst"`' -ECHO='`$ECHO "$ECHO" | $SED "$delay_single_quote_subst"`' -PATH_SEPARATOR='`$ECHO "$PATH_SEPARATOR" | $SED "$delay_single_quote_subst"`' -host_alias='`$ECHO "$host_alias" | $SED "$delay_single_quote_subst"`' -host='`$ECHO "$host" | $SED "$delay_single_quote_subst"`' -host_os='`$ECHO "$host_os" | $SED "$delay_single_quote_subst"`' -build_alias='`$ECHO "$build_alias" | $SED "$delay_single_quote_subst"`' -build='`$ECHO "$build" | $SED "$delay_single_quote_subst"`' -build_os='`$ECHO "$build_os" | $SED "$delay_single_quote_subst"`' -SED='`$ECHO "$SED" | $SED "$delay_single_quote_subst"`' -Xsed='`$ECHO "$Xsed" | $SED "$delay_single_quote_subst"`' -GREP='`$ECHO "$GREP" | $SED "$delay_single_quote_subst"`' -EGREP='`$ECHO "$EGREP" | $SED "$delay_single_quote_subst"`' -FGREP='`$ECHO "$FGREP" | $SED "$delay_single_quote_subst"`' -LD='`$ECHO "$LD" | $SED "$delay_single_quote_subst"`' -NM='`$ECHO "$NM" | $SED "$delay_single_quote_subst"`' -LN_S='`$ECHO "$LN_S" | $SED "$delay_single_quote_subst"`' -max_cmd_len='`$ECHO "$max_cmd_len" | $SED "$delay_single_quote_subst"`' -ac_objext='`$ECHO "$ac_objext" | $SED "$delay_single_quote_subst"`' -exeext='`$ECHO "$exeext" | $SED "$delay_single_quote_subst"`' -lt_unset='`$ECHO "$lt_unset" | $SED "$delay_single_quote_subst"`' -lt_SP2NL='`$ECHO "$lt_SP2NL" | $SED "$delay_single_quote_subst"`' -lt_NL2SP='`$ECHO "$lt_NL2SP" | $SED "$delay_single_quote_subst"`' -lt_cv_to_host_file_cmd='`$ECHO "$lt_cv_to_host_file_cmd" | $SED "$delay_single_quote_subst"`' -lt_cv_to_tool_file_cmd='`$ECHO "$lt_cv_to_tool_file_cmd" | $SED "$delay_single_quote_subst"`' -reload_flag='`$ECHO "$reload_flag" | $SED "$delay_single_quote_subst"`' -reload_cmds='`$ECHO "$reload_cmds" | $SED "$delay_single_quote_subst"`' -OBJDUMP='`$ECHO "$OBJDUMP" | $SED "$delay_single_quote_subst"`' -deplibs_check_method='`$ECHO "$deplibs_check_method" | $SED "$delay_single_quote_subst"`' -file_magic_cmd='`$ECHO "$file_magic_cmd" | $SED "$delay_single_quote_subst"`' -file_magic_glob='`$ECHO "$file_magic_glob" | $SED "$delay_single_quote_subst"`' -want_nocaseglob='`$ECHO "$want_nocaseglob" | $SED "$delay_single_quote_subst"`' -DLLTOOL='`$ECHO "$DLLTOOL" | $SED "$delay_single_quote_subst"`' -sharedlib_from_linklib_cmd='`$ECHO "$sharedlib_from_linklib_cmd" | $SED "$delay_single_quote_subst"`' -AR='`$ECHO "$AR" | $SED "$delay_single_quote_subst"`' -AR_FLAGS='`$ECHO "$AR_FLAGS" | $SED "$delay_single_quote_subst"`' -archiver_list_spec='`$ECHO "$archiver_list_spec" | $SED "$delay_single_quote_subst"`' -STRIP='`$ECHO "$STRIP" | $SED "$delay_single_quote_subst"`' -RANLIB='`$ECHO "$RANLIB" | $SED "$delay_single_quote_subst"`' -old_postinstall_cmds='`$ECHO "$old_postinstall_cmds" | $SED "$delay_single_quote_subst"`' -old_postuninstall_cmds='`$ECHO "$old_postuninstall_cmds" | $SED "$delay_single_quote_subst"`' -old_archive_cmds='`$ECHO "$old_archive_cmds" | $SED "$delay_single_quote_subst"`' -lock_old_archive_extraction='`$ECHO "$lock_old_archive_extraction" | $SED "$delay_single_quote_subst"`' -CC='`$ECHO "$CC" | $SED "$delay_single_quote_subst"`' -CFLAGS='`$ECHO "$CFLAGS" | $SED "$delay_single_quote_subst"`' -compiler='`$ECHO "$compiler" | $SED "$delay_single_quote_subst"`' -GCC='`$ECHO "$GCC" | $SED "$delay_single_quote_subst"`' -lt_cv_sys_global_symbol_pipe='`$ECHO "$lt_cv_sys_global_symbol_pipe" | $SED "$delay_single_quote_subst"`' -lt_cv_sys_global_symbol_to_cdecl='`$ECHO "$lt_cv_sys_global_symbol_to_cdecl" | $SED "$delay_single_quote_subst"`' -lt_cv_sys_global_symbol_to_import='`$ECHO "$lt_cv_sys_global_symbol_to_import" | $SED "$delay_single_quote_subst"`' -lt_cv_sys_global_symbol_to_c_name_address='`$ECHO "$lt_cv_sys_global_symbol_to_c_name_address" | $SED "$delay_single_quote_subst"`' -lt_cv_sys_global_symbol_to_c_name_address_lib_prefix='`$ECHO "$lt_cv_sys_global_symbol_to_c_name_address_lib_prefix" | $SED "$delay_single_quote_subst"`' -lt_cv_nm_interface='`$ECHO "$lt_cv_nm_interface" | $SED "$delay_single_quote_subst"`' -nm_file_list_spec='`$ECHO "$nm_file_list_spec" | $SED "$delay_single_quote_subst"`' -lt_sysroot='`$ECHO "$lt_sysroot" | $SED "$delay_single_quote_subst"`' -lt_cv_truncate_bin='`$ECHO "$lt_cv_truncate_bin" | $SED "$delay_single_quote_subst"`' -objdir='`$ECHO "$objdir" | $SED "$delay_single_quote_subst"`' -MAGIC_CMD='`$ECHO "$MAGIC_CMD" | $SED "$delay_single_quote_subst"`' -lt_prog_compiler_no_builtin_flag='`$ECHO "$lt_prog_compiler_no_builtin_flag" | $SED "$delay_single_quote_subst"`' -lt_prog_compiler_pic='`$ECHO "$lt_prog_compiler_pic" | $SED "$delay_single_quote_subst"`' -lt_prog_compiler_wl='`$ECHO "$lt_prog_compiler_wl" | $SED "$delay_single_quote_subst"`' -lt_prog_compiler_static='`$ECHO "$lt_prog_compiler_static" | $SED "$delay_single_quote_subst"`' -lt_cv_prog_compiler_c_o='`$ECHO "$lt_cv_prog_compiler_c_o" | $SED "$delay_single_quote_subst"`' -need_locks='`$ECHO "$need_locks" | $SED "$delay_single_quote_subst"`' -MANIFEST_TOOL='`$ECHO "$MANIFEST_TOOL" | $SED "$delay_single_quote_subst"`' -DSYMUTIL='`$ECHO "$DSYMUTIL" | $SED "$delay_single_quote_subst"`' -NMEDIT='`$ECHO "$NMEDIT" | $SED "$delay_single_quote_subst"`' -LIPO='`$ECHO "$LIPO" | $SED "$delay_single_quote_subst"`' -OTOOL='`$ECHO "$OTOOL" | $SED "$delay_single_quote_subst"`' -OTOOL64='`$ECHO "$OTOOL64" | $SED "$delay_single_quote_subst"`' -libext='`$ECHO "$libext" | $SED "$delay_single_quote_subst"`' -shrext_cmds='`$ECHO "$shrext_cmds" | $SED "$delay_single_quote_subst"`' -extract_expsyms_cmds='`$ECHO "$extract_expsyms_cmds" | $SED "$delay_single_quote_subst"`' -archive_cmds_need_lc='`$ECHO "$archive_cmds_need_lc" | $SED "$delay_single_quote_subst"`' -enable_shared_with_static_runtimes='`$ECHO "$enable_shared_with_static_runtimes" | $SED "$delay_single_quote_subst"`' -export_dynamic_flag_spec='`$ECHO "$export_dynamic_flag_spec" | $SED "$delay_single_quote_subst"`' -whole_archive_flag_spec='`$ECHO "$whole_archive_flag_spec" | $SED "$delay_single_quote_subst"`' -compiler_needs_object='`$ECHO "$compiler_needs_object" | $SED "$delay_single_quote_subst"`' -old_archive_from_new_cmds='`$ECHO "$old_archive_from_new_cmds" | $SED "$delay_single_quote_subst"`' -old_archive_from_expsyms_cmds='`$ECHO "$old_archive_from_expsyms_cmds" | $SED "$delay_single_quote_subst"`' -archive_cmds='`$ECHO "$archive_cmds" | $SED "$delay_single_quote_subst"`' -archive_expsym_cmds='`$ECHO "$archive_expsym_cmds" | $SED "$delay_single_quote_subst"`' -module_cmds='`$ECHO "$module_cmds" | $SED "$delay_single_quote_subst"`' -module_expsym_cmds='`$ECHO "$module_expsym_cmds" | $SED "$delay_single_quote_subst"`' -with_gnu_ld='`$ECHO "$with_gnu_ld" | $SED "$delay_single_quote_subst"`' -allow_undefined_flag='`$ECHO "$allow_undefined_flag" | $SED "$delay_single_quote_subst"`' -no_undefined_flag='`$ECHO "$no_undefined_flag" | $SED "$delay_single_quote_subst"`' -hardcode_libdir_flag_spec='`$ECHO "$hardcode_libdir_flag_spec" | $SED "$delay_single_quote_subst"`' -hardcode_libdir_separator='`$ECHO "$hardcode_libdir_separator" | $SED "$delay_single_quote_subst"`' -hardcode_direct='`$ECHO "$hardcode_direct" | $SED "$delay_single_quote_subst"`' -hardcode_direct_absolute='`$ECHO "$hardcode_direct_absolute" | $SED "$delay_single_quote_subst"`' -hardcode_minus_L='`$ECHO "$hardcode_minus_L" | $SED "$delay_single_quote_subst"`' -hardcode_shlibpath_var='`$ECHO "$hardcode_shlibpath_var" | $SED "$delay_single_quote_subst"`' -hardcode_automatic='`$ECHO "$hardcode_automatic" | $SED "$delay_single_quote_subst"`' -inherit_rpath='`$ECHO "$inherit_rpath" | $SED "$delay_single_quote_subst"`' -link_all_deplibs='`$ECHO "$link_all_deplibs" | $SED "$delay_single_quote_subst"`' -always_export_symbols='`$ECHO "$always_export_symbols" | $SED "$delay_single_quote_subst"`' -export_symbols_cmds='`$ECHO "$export_symbols_cmds" | $SED "$delay_single_quote_subst"`' -exclude_expsyms='`$ECHO "$exclude_expsyms" | $SED "$delay_single_quote_subst"`' -include_expsyms='`$ECHO "$include_expsyms" | $SED "$delay_single_quote_subst"`' -prelink_cmds='`$ECHO "$prelink_cmds" | $SED "$delay_single_quote_subst"`' -postlink_cmds='`$ECHO "$postlink_cmds" | $SED "$delay_single_quote_subst"`' -file_list_spec='`$ECHO "$file_list_spec" | $SED "$delay_single_quote_subst"`' -variables_saved_for_relink='`$ECHO "$variables_saved_for_relink" | $SED "$delay_single_quote_subst"`' -need_lib_prefix='`$ECHO "$need_lib_prefix" | $SED "$delay_single_quote_subst"`' -need_version='`$ECHO "$need_version" | $SED "$delay_single_quote_subst"`' -version_type='`$ECHO "$version_type" | $SED "$delay_single_quote_subst"`' -runpath_var='`$ECHO "$runpath_var" | $SED "$delay_single_quote_subst"`' -shlibpath_var='`$ECHO "$shlibpath_var" | $SED "$delay_single_quote_subst"`' -shlibpath_overrides_runpath='`$ECHO "$shlibpath_overrides_runpath" | $SED "$delay_single_quote_subst"`' -libname_spec='`$ECHO "$libname_spec" | $SED "$delay_single_quote_subst"`' -library_names_spec='`$ECHO "$library_names_spec" | $SED "$delay_single_quote_subst"`' -soname_spec='`$ECHO "$soname_spec" | $SED "$delay_single_quote_subst"`' -install_override_mode='`$ECHO "$install_override_mode" | $SED "$delay_single_quote_subst"`' -postinstall_cmds='`$ECHO "$postinstall_cmds" | $SED "$delay_single_quote_subst"`' -postuninstall_cmds='`$ECHO "$postuninstall_cmds" | $SED "$delay_single_quote_subst"`' -finish_cmds='`$ECHO "$finish_cmds" | $SED "$delay_single_quote_subst"`' -finish_eval='`$ECHO "$finish_eval" | $SED "$delay_single_quote_subst"`' -hardcode_into_libs='`$ECHO "$hardcode_into_libs" | $SED "$delay_single_quote_subst"`' -sys_lib_search_path_spec='`$ECHO "$sys_lib_search_path_spec" | $SED "$delay_single_quote_subst"`' -configure_time_dlsearch_path='`$ECHO "$configure_time_dlsearch_path" | $SED "$delay_single_quote_subst"`' -configure_time_lt_sys_library_path='`$ECHO "$configure_time_lt_sys_library_path" | $SED "$delay_single_quote_subst"`' -hardcode_action='`$ECHO "$hardcode_action" | $SED "$delay_single_quote_subst"`' -enable_dlopen='`$ECHO "$enable_dlopen" | $SED "$delay_single_quote_subst"`' -enable_dlopen_self='`$ECHO "$enable_dlopen_self" | $SED "$delay_single_quote_subst"`' -enable_dlopen_self_static='`$ECHO "$enable_dlopen_self_static" | $SED "$delay_single_quote_subst"`' -old_striplib='`$ECHO "$old_striplib" | $SED "$delay_single_quote_subst"`' -striplib='`$ECHO "$striplib" | $SED "$delay_single_quote_subst"`' - -LTCC='$LTCC' -LTCFLAGS='$LTCFLAGS' -compiler='$compiler_DEFAULT' - -# A function that is used when there is no print builtin or printf. -func_fallback_echo () -{ - eval 'cat <<_LTECHO_EOF -\$1 -_LTECHO_EOF' -} - -# Quote evaled strings. -for var in SHELL \ -ECHO \ -PATH_SEPARATOR \ -SED \ -GREP \ -EGREP \ -FGREP \ -LD \ -NM \ -LN_S \ -lt_SP2NL \ -lt_NL2SP \ -reload_flag \ -OBJDUMP \ -deplibs_check_method \ -file_magic_cmd \ -file_magic_glob \ -want_nocaseglob \ -DLLTOOL \ -sharedlib_from_linklib_cmd \ -AR \ -AR_FLAGS \ -archiver_list_spec \ -STRIP \ -RANLIB \ -CC \ -CFLAGS \ -compiler \ -lt_cv_sys_global_symbol_pipe \ -lt_cv_sys_global_symbol_to_cdecl \ -lt_cv_sys_global_symbol_to_import \ -lt_cv_sys_global_symbol_to_c_name_address \ -lt_cv_sys_global_symbol_to_c_name_address_lib_prefix \ -lt_cv_nm_interface \ -nm_file_list_spec \ -lt_cv_truncate_bin \ -lt_prog_compiler_no_builtin_flag \ -lt_prog_compiler_pic \ -lt_prog_compiler_wl \ -lt_prog_compiler_static \ -lt_cv_prog_compiler_c_o \ -need_locks \ -MANIFEST_TOOL \ -DSYMUTIL \ -NMEDIT \ -LIPO \ -OTOOL \ -OTOOL64 \ -shrext_cmds \ -export_dynamic_flag_spec \ -whole_archive_flag_spec \ -compiler_needs_object \ -with_gnu_ld \ -allow_undefined_flag \ -no_undefined_flag \ -hardcode_libdir_flag_spec \ -hardcode_libdir_separator \ -exclude_expsyms \ -include_expsyms \ -file_list_spec \ -variables_saved_for_relink \ -libname_spec \ -library_names_spec \ -soname_spec \ -install_override_mode \ -finish_eval \ -old_striplib \ -striplib; do - case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in - *[\\\\\\\`\\"\\\$]*) - eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED \\"\\\$sed_quote_subst\\"\\\`\\\\\\"" ## exclude from sc_prohibit_nested_quotes - ;; - *) - eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" - ;; - esac -done - -# Double-quote double-evaled strings. -for var in reload_cmds \ -old_postinstall_cmds \ -old_postuninstall_cmds \ -old_archive_cmds \ -extract_expsyms_cmds \ -old_archive_from_new_cmds \ -old_archive_from_expsyms_cmds \ -archive_cmds \ -archive_expsym_cmds \ -module_cmds \ -module_expsym_cmds \ -export_symbols_cmds \ -prelink_cmds \ -postlink_cmds \ -postinstall_cmds \ -postuninstall_cmds \ -finish_cmds \ -sys_lib_search_path_spec \ -configure_time_dlsearch_path \ -configure_time_lt_sys_library_path; do - case \`eval \\\\\$ECHO \\\\""\\\\\$\$var"\\\\"\` in - *[\\\\\\\`\\"\\\$]*) - eval "lt_\$var=\\\\\\"\\\`\\\$ECHO \\"\\\$\$var\\" | \\\$SED -e \\"\\\$double_quote_subst\\" -e \\"\\\$sed_quote_subst\\" -e \\"\\\$delay_variable_subst\\"\\\`\\\\\\"" ## exclude from sc_prohibit_nested_quotes - ;; - *) - eval "lt_\$var=\\\\\\"\\\$\$var\\\\\\"" - ;; - esac -done - -ac_aux_dir='$ac_aux_dir' - -# See if we are running on zsh, and set the options that allow our -# commands through without removal of \ escapes INIT. -if test -n "\${ZSH_VERSION+set}"; then - setopt NO_GLOB_SUBST -fi - - - PACKAGE='$PACKAGE' - VERSION='$VERSION' - RM='$RM' - ofile='$ofile' - - - - -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 - -# Handling of arguments. -for ac_config_target in $ac_config_targets -do - case $ac_config_target in - "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;; - "sqlite_cfg.h") CONFIG_HEADERS="$CONFIG_HEADERS sqlite_cfg.h" ;; - "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; - "sqlcipher.pc") CONFIG_FILES="$CONFIG_FILES sqlcipher.pc" ;; - - *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; - esac -done - - -# If the user did not use the arguments to specify the items to instantiate, -# then the envvar interface is used. Set only those that are not. -# We use the long form for the default assignment because of an extremely -# bizarre bug on SunOS 4.1.3. -if $ac_need_defaults; then - test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files - test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers - test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands -fi - -# Have a temporary directory for convenience. Make it in the build tree -# simply because there is no reason against having it here, and in addition, -# creating and moving files from /tmp can sometimes cause problems. -# Hook for its removal unless debugging. -# Note that there is a small window in which the directory will not be cleaned: -# after its creation but before its name has been assigned to `$tmp'. -$debug || -{ - tmp= ac_tmp= - trap 'exit_status=$? - : "${ac_tmp:=$tmp}" - { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status -' 0 - trap 'as_fn_exit 1' 1 2 13 15 -} -# Create a (secure) tmp directory for tmp files. - -{ - tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && - test -d "$tmp" -} || -{ - tmp=./conf$$-$RANDOM - (umask 077 && mkdir "$tmp") -} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 -ac_tmp=$tmp - -# Set up the scripts for CONFIG_FILES section. -# No need to generate them if there are no CONFIG_FILES. -# This happens for instance with `./config.status config.h'. -if test -n "$CONFIG_FILES"; then - - -ac_cr=`echo X | tr X '\015'` -# On cygwin, bash can eat \r inside `` if the user requested igncr. -# But we know of no other shell where ac_cr would be empty at this -# point, so we can use a bashism as a fallback. -if test "x$ac_cr" = x; then - eval ac_cr=\$\'\\r\' -fi -ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` -if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then - ac_cs_awk_cr='\\r' -else - ac_cs_awk_cr=$ac_cr -fi - -echo 'BEGIN {' >"$ac_tmp/subs1.awk" && -_ACEOF - - -{ - echo "cat >conf$$subs.awk <<_ACEOF" && - echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && - echo "_ACEOF" -} >conf$$subs.sh || - as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 -ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` -ac_delim='%!_!# ' -for ac_last_try in false false false false false :; do - . ./conf$$subs.sh || - as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 - - ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` - if test $ac_delim_n = $ac_delim_num; then - break - elif $ac_last_try; then - as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 - else - ac_delim="$ac_delim!$ac_delim _$ac_delim!! " - fi -done -rm -f conf$$subs.sh - -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && -_ACEOF -sed -n ' -h -s/^/S["/; s/!.*/"]=/ -p -g -s/^[^!]*!// -:repl -t repl -s/'"$ac_delim"'$// -t delim -:nl -h -s/\(.\{148\}\)..*/\1/ -t more1 -s/["\\]/\\&/g; s/^/"/; s/$/\\n"\\/ -p -n -b repl -:more1 -s/["\\]/\\&/g; s/^/"/; s/$/"\\/ -p -g -s/.\{148\}// -t nl -:delim -h -s/\(.\{148\}\)..*/\1/ -t more2 -s/["\\]/\\&/g; s/^/"/; s/$/"/ -p -b -:more2 -s/["\\]/\\&/g; s/^/"/; s/$/"\\/ -p -g -s/.\{148\}// -t delim -' >$CONFIG_STATUS || ac_write_fail=1 -rm -f conf$$subs.awk -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -_ACAWK -cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && - for (key in S) S_is_set[key] = 1 - FS = "" - -} -{ - line = $ 0 - nfields = split(line, field, "@") - substed = 0 - len = length(field[1]) - for (i = 2; i < nfields; i++) { - key = field[i] - keylen = length(key) - if (S_is_set[key]) { - value = S[key] - line = substr(line, 1, len) "" value "" substr(line, len + keylen + 3) - len += length(value) + length(field[++i]) - substed = 1 - } else - len += 1 + keylen - } - - print line -} - -_ACAWK -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then - sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" -else - cat -fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ - || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 -_ACEOF - -# VPATH may cause trouble with some makes, so we remove sole $(srcdir), -# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and -# trailing colons and then remove the whole line if VPATH becomes empty -# (actually we leave an empty line to preserve line numbers). -if test "x$srcdir" = x.; then - ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ -h -s/// -s/^/:/ -s/[ ]*$/:/ -s/:\$(srcdir):/:/g -s/:\${srcdir}:/:/g -s/:@srcdir@:/:/g -s/^:*// -s/:*$// -x -s/\(=[ ]*\).*/\1/ -G -s/\n// -s/^[^=]*=[ ]*$// -}' -fi - -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -fi # test -n "$CONFIG_FILES" - -# Set up the scripts for CONFIG_HEADERS section. -# No need to generate them if there are no CONFIG_HEADERS. -# This happens for instance with `./config.status Makefile'. -if test -n "$CONFIG_HEADERS"; then -cat >"$ac_tmp/defines.awk" <<\_ACAWK || -BEGIN { -_ACEOF - -# Transform confdefs.h into an awk script `defines.awk', embedded as -# here-document in config.status, that substitutes the proper values into -# config.h.in to produce config.h. - -# Create a delimiter string that does not exist in confdefs.h, to ease -# handling of long lines. -ac_delim='%!_!# ' -for ac_last_try in false false :; do - ac_tt=`sed -n "/$ac_delim/p" confdefs.h` - if test -z "$ac_tt"; then - break - elif $ac_last_try; then - as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 - else - ac_delim="$ac_delim!$ac_delim _$ac_delim!! " - fi -done - -# For the awk script, D is an array of macro values keyed by name, -# likewise P contains macro parameters if any. Preserve backslash -# newline sequences. - -ac_word_re=[_$as_cr_Letters][_$as_cr_alnum]* -sed -n ' -s/.\{148\}/&'"$ac_delim"'/g -t rset -:rset -s/^[ ]*#[ ]*define[ ][ ]*/ / -t def -d -:def -s/\\$// -t bsnl -s/["\\]/\\&/g -s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ -D["\1"]=" \3"/p -s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2"/p -d -:bsnl -s/["\\]/\\&/g -s/^ \('"$ac_word_re"'\)\(([^()]*)\)[ ]*\(.*\)/P["\1"]="\2"\ -D["\1"]=" \3\\\\\\n"\\/p -t cont -s/^ \('"$ac_word_re"'\)[ ]*\(.*\)/D["\1"]=" \2\\\\\\n"\\/p -t cont -d -:cont -n -s/.\{148\}/&'"$ac_delim"'/g -t clear -:clear -s/\\$// -t bsnlc -s/["\\]/\\&/g; s/^/"/; s/$/"/p -d -:bsnlc -s/["\\]/\\&/g; s/^/"/; s/$/\\\\\\n"\\/p -b cont -' >$CONFIG_STATUS || ac_write_fail=1 - -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 - for (key in D) D_is_set[key] = 1 - FS = "" -} -/^[\t ]*#[\t ]*(define|undef)[\t ]+$ac_word_re([\t (]|\$)/ { - line = \$ 0 - split(line, arg, " ") - if (arg[1] == "#") { - defundef = arg[2] - mac1 = arg[3] - } else { - defundef = substr(arg[1], 2) - mac1 = arg[2] - } - split(mac1, mac2, "(") #) - macro = mac2[1] - prefix = substr(line, 1, index(line, defundef) - 1) - if (D_is_set[macro]) { - # Preserve the white space surrounding the "#". - print prefix "define", macro P[macro] D[macro] - next - } else { - # Replace #undef with comments. This is necessary, for example, - # in the case of _POSIX_SOURCE, which is predefined and required - # on some systems where configure will not decide to define it. - if (defundef == "undef") { - print "/*", prefix defundef, macro, "*/" - next - } - } -} -{ print } -_ACAWK -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 - as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 -fi # test -n "$CONFIG_HEADERS" - - -eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS" -shift -for ac_tag -do - case $ac_tag in - :[FHLC]) ac_mode=$ac_tag; continue;; - esac - case $ac_mode$ac_tag in - :[FHL]*:*);; - :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; - :[FH]-) ac_tag=-:-;; - :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; - esac - ac_save_IFS=$IFS - IFS=: - set x $ac_tag - IFS=$ac_save_IFS - shift - ac_file=$1 - shift - - case $ac_mode in - :L) ac_source=$1;; - :[FH]) - ac_file_inputs= - for ac_f - do - case $ac_f in - -) ac_f="$ac_tmp/stdin";; - *) # Look for the file first in the build tree, then in the source tree - # (if the path is not absolute). The absolute path cannot be DOS-style, - # because $ac_f cannot contain `:'. - test -f "$ac_f" || - case $ac_f in - [\\/$]*) false;; - *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; - esac || - as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; - esac - case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac - as_fn_append ac_file_inputs " '$ac_f'" - done - - # Let's still pretend it is `configure' which instantiates (i.e., don't - # use $as_me), people would be surprised to read: - # /* config.h. Generated by config.status. */ - configure_input='Generated from '` - $as_echo "$*" | sed 's|^[^:]*/||;s|:[^:]*/|, |g' - `' by configure.' - if test x"$ac_file" != x-; then - configure_input="$ac_file. $configure_input" - { $as_echo "$as_me:${as_lineno-$LINENO}: creating $ac_file" >&5 -$as_echo "$as_me: creating $ac_file" >&6;} - fi - # Neutralize special characters interpreted by sed in replacement strings. - case $configure_input in #( - *\&* | *\|* | *\\* ) - ac_sed_conf_input=`$as_echo "$configure_input" | - sed 's/[\\\\&|]/\\\\&/g'`;; #( - *) ac_sed_conf_input=$configure_input;; - esac - - case $ac_tag in - *:-:* | *:-) cat >"$ac_tmp/stdin" \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; - esac - ;; - esac - - ac_dir=`$as_dirname -- "$ac_file" || -$as_expr X"$ac_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ - X"$ac_file" : 'X\(//\)[^/]' \| \ - X"$ac_file" : 'X\(//\)$' \| \ - X"$ac_file" : 'X\(/\)' \| . 2>/dev/null || -$as_echo X"$ac_file" | - sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ - s//\1/ - q - } - /^X\(\/\/\)[^/].*/{ - s//\1/ - q - } - /^X\(\/\/\)$/{ - s//\1/ - q - } - /^X\(\/\).*/{ - s//\1/ - q - } - s/.*/./; q'` - as_dir="$ac_dir"; as_fn_mkdir_p - ac_builddir=. - -case "$ac_dir" in -.) ac_dir_suffix= ac_top_builddir_sub=. ac_top_build_prefix= ;; -*) - ac_dir_suffix=/`$as_echo "$ac_dir" | sed 's|^\.[\\/]||'` - # A ".." for each directory in $ac_dir_suffix. - ac_top_builddir_sub=`$as_echo "$ac_dir_suffix" | sed 's|/[^\\/]*|/..|g;s|/||'` - case $ac_top_builddir_sub in - "") ac_top_builddir_sub=. ac_top_build_prefix= ;; - *) ac_top_build_prefix=$ac_top_builddir_sub/ ;; - esac ;; -esac -ac_abs_top_builddir=$ac_pwd -ac_abs_builddir=$ac_pwd$ac_dir_suffix -# for backward compatibility: -ac_top_builddir=$ac_top_build_prefix - -case $srcdir in - .) # We are building in place. - ac_srcdir=. - ac_top_srcdir=$ac_top_builddir_sub - ac_abs_top_srcdir=$ac_pwd ;; - [\\/]* | ?:[\\/]* ) # Absolute name. - ac_srcdir=$srcdir$ac_dir_suffix; - ac_top_srcdir=$srcdir - ac_abs_top_srcdir=$srcdir ;; - *) # Relative name. - ac_srcdir=$ac_top_build_prefix$srcdir$ac_dir_suffix - ac_top_srcdir=$ac_top_build_prefix$srcdir - ac_abs_top_srcdir=$ac_pwd/$srcdir ;; -esac -ac_abs_srcdir=$ac_abs_top_srcdir$ac_dir_suffix - - - case $ac_mode in - :F) - # - # CONFIG_FILE - # - - case $INSTALL in - [\\/$]* | ?:[\\/]* ) ac_INSTALL=$INSTALL ;; - *) ac_INSTALL=$ac_top_build_prefix$INSTALL ;; - esac -_ACEOF - -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -# If the template does not know about datarootdir, expand it. -# FIXME: This hack should be removed a few years after 2.60. -ac_datarootdir_hack=; ac_datarootdir_seen= -ac_sed_dataroot=' -/datarootdir/ { - p - q -} -/@datadir@/p -/@docdir@/p -/@infodir@/p -/@localedir@/p -/@mandir@/p' -case `eval "sed -n \"\$ac_sed_dataroot\" $ac_file_inputs"` in -*datarootdir*) ac_datarootdir_seen=yes;; -*@datadir@*|*@docdir@*|*@infodir@*|*@localedir@*|*@mandir@*) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&5 -$as_echo "$as_me: WARNING: $ac_file_inputs seems to ignore the --datarootdir setting" >&2;} -_ACEOF -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 - ac_datarootdir_hack=' - s&@datadir@&$datadir&g - s&@docdir@&$docdir&g - s&@infodir@&$infodir&g - s&@localedir@&$localedir&g - s&@mandir@&$mandir&g - s&\\\${datarootdir}&$datarootdir&g' ;; -esac -_ACEOF - -# Neutralize VPATH when `$srcdir' = `.'. -# Shell code in configure.ac might set extrasub. -# FIXME: do we really want to maintain this feature? -cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -ac_sed_extra="$ac_vpsub -$extrasub -_ACEOF -cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 -:t -/@[a-zA-Z_][a-zA-Z_0-9]*@/!b -s|@configure_input@|$ac_sed_conf_input|;t t -s&@top_builddir@&$ac_top_builddir_sub&;t t -s&@top_build_prefix@&$ac_top_build_prefix&;t t -s&@srcdir@&$ac_srcdir&;t t -s&@abs_srcdir@&$ac_abs_srcdir&;t t -s&@top_srcdir@&$ac_top_srcdir&;t t -s&@abs_top_srcdir@&$ac_abs_top_srcdir&;t t -s&@builddir@&$ac_builddir&;t t -s&@abs_builddir@&$ac_abs_builddir&;t t -s&@abs_top_builddir@&$ac_abs_top_builddir&;t t -s&@INSTALL@&$ac_INSTALL&;t t -$ac_datarootdir_hack -" -eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ - >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 - -test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && - { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && - { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ - "$ac_tmp/out"`; test -z "$ac_out"; } && - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined" >&5 -$as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined" >&2;} - - rm -f "$ac_tmp/stdin" - case $ac_file in - -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; - *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; - esac \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 - ;; - :H) - # - # CONFIG_HEADER - # - if test x"$ac_file" != x-; then - { - $as_echo "/* $configure_input */" \ - && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" - } >"$ac_tmp/config.h" \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 - if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then - { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 -$as_echo "$as_me: $ac_file is unchanged" >&6;} - else - rm -f "$ac_file" - mv "$ac_tmp/config.h" "$ac_file" \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 - fi - else - $as_echo "/* $configure_input */" \ - && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \ - || as_fn_error $? "could not create -" "$LINENO" 5 - fi - ;; - - :C) { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5 -$as_echo "$as_me: executing $ac_file commands" >&6;} - ;; - esac - - - case $ac_file$ac_mode in - "libtool":C) - - # See if we are running on zsh, and set the options that allow our - # commands through without removal of \ escapes. - if test -n "${ZSH_VERSION+set}"; then - setopt NO_GLOB_SUBST - fi - - cfgfile=${ofile}T - trap "$RM \"$cfgfile\"; exit 1" 1 2 15 - $RM "$cfgfile" - - cat <<_LT_EOF >> "$cfgfile" -#! $SHELL -# Generated automatically by $as_me ($PACKAGE) $VERSION -# NOTE: Changes made to this file will be lost: look at ltmain.sh. - -# Provide generalized library-building support services. -# Written by Gordon Matzigkeit, 1996 - -# Copyright (C) 2014 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# GNU Libtool is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of of the License, or -# (at your option) any later version. -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program or library that is built -# using GNU Libtool, you may include this file under the same -# distribution terms that you use for the rest of that program. -# -# GNU Libtool is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -# The names of the tagged configurations supported by this script. -available_tags='' - -# Configured defaults for sys_lib_dlsearch_path munging. -: \${LT_SYS_LIBRARY_PATH="$configure_time_lt_sys_library_path"} - -# ### BEGIN LIBTOOL CONFIG - -# Which release of libtool.m4 was used? -macro_version=$macro_version -macro_revision=$macro_revision - -# Whether or not to build shared libraries. -build_libtool_libs=$enable_shared - -# Whether or not to build static libraries. -build_old_libs=$enable_static - -# What type of objects to build. -pic_mode=$pic_mode - -# Whether or not to optimize for fast installation. -fast_install=$enable_fast_install - -# Shared archive member basename,for filename based shared library versioning on AIX. -shared_archive_member_spec=$shared_archive_member_spec - -# Shell to use when invoking shell scripts. -SHELL=$lt_SHELL - -# An echo program that protects backslashes. -ECHO=$lt_ECHO - -# The PATH separator for the build system. -PATH_SEPARATOR=$lt_PATH_SEPARATOR - -# The host system. -host_alias=$host_alias -host=$host -host_os=$host_os - -# The build system. -build_alias=$build_alias -build=$build -build_os=$build_os - -# A sed program that does not truncate output. -SED=$lt_SED - -# Sed that helps us avoid accidentally triggering echo(1) options like -n. -Xsed="\$SED -e 1s/^X//" - -# A grep program that handles long lines. -GREP=$lt_GREP - -# An ERE matcher. -EGREP=$lt_EGREP - -# A literal string matcher. -FGREP=$lt_FGREP - -# A BSD- or MS-compatible name lister. -NM=$lt_NM - -# Whether we need soft or hard links. -LN_S=$lt_LN_S - -# What is the maximum length of a command? -max_cmd_len=$max_cmd_len - -# Object file suffix (normally "o"). -objext=$ac_objext - -# Executable file suffix (normally ""). -exeext=$exeext - -# whether the shell understands "unset". -lt_unset=$lt_unset - -# turn spaces into newlines. -SP2NL=$lt_lt_SP2NL - -# turn newlines into spaces. -NL2SP=$lt_lt_NL2SP - -# convert \$build file names to \$host format. -to_host_file_cmd=$lt_cv_to_host_file_cmd - -# convert \$build files to toolchain format. -to_tool_file_cmd=$lt_cv_to_tool_file_cmd - -# An object symbol dumper. -OBJDUMP=$lt_OBJDUMP - -# Method to check whether dependent libraries are shared objects. -deplibs_check_method=$lt_deplibs_check_method - -# Command to use when deplibs_check_method = "file_magic". -file_magic_cmd=$lt_file_magic_cmd - -# How to find potential files when deplibs_check_method = "file_magic". -file_magic_glob=$lt_file_magic_glob - -# Find potential files using nocaseglob when deplibs_check_method = "file_magic". -want_nocaseglob=$lt_want_nocaseglob - -# DLL creation program. -DLLTOOL=$lt_DLLTOOL - -# Command to associate shared and link libraries. -sharedlib_from_linklib_cmd=$lt_sharedlib_from_linklib_cmd - -# The archiver. -AR=$lt_AR - -# Flags to create an archive. -AR_FLAGS=$lt_AR_FLAGS - -# How to feed a file listing to the archiver. -archiver_list_spec=$lt_archiver_list_spec - -# A symbol stripping program. -STRIP=$lt_STRIP - -# Commands used to install an old-style archive. -RANLIB=$lt_RANLIB -old_postinstall_cmds=$lt_old_postinstall_cmds -old_postuninstall_cmds=$lt_old_postuninstall_cmds - -# Whether to use a lock for old archive extraction. -lock_old_archive_extraction=$lock_old_archive_extraction - -# A C compiler. -LTCC=$lt_CC - -# LTCC compiler flags. -LTCFLAGS=$lt_CFLAGS - -# Take the output of nm and produce a listing of raw symbols and C names. -global_symbol_pipe=$lt_lt_cv_sys_global_symbol_pipe - -# Transform the output of nm in a proper C declaration. -global_symbol_to_cdecl=$lt_lt_cv_sys_global_symbol_to_cdecl - -# Transform the output of nm into a list of symbols to manually relocate. -global_symbol_to_import=$lt_lt_cv_sys_global_symbol_to_import - -# Transform the output of nm in a C name address pair. -global_symbol_to_c_name_address=$lt_lt_cv_sys_global_symbol_to_c_name_address - -# Transform the output of nm in a C name address pair when lib prefix is needed. -global_symbol_to_c_name_address_lib_prefix=$lt_lt_cv_sys_global_symbol_to_c_name_address_lib_prefix - -# The name lister interface. -nm_interface=$lt_lt_cv_nm_interface - -# Specify filename containing input files for \$NM. -nm_file_list_spec=$lt_nm_file_list_spec - -# The root where to search for dependent libraries,and where our libraries should be installed. -lt_sysroot=$lt_sysroot - -# Command to truncate a binary pipe. -lt_truncate_bin=$lt_lt_cv_truncate_bin - -# The name of the directory that contains temporary libtool files. -objdir=$objdir - -# Used to examine libraries when file_magic_cmd begins with "file". -MAGIC_CMD=$MAGIC_CMD - -# Must we lock files when doing compilation? -need_locks=$lt_need_locks - -# Manifest tool. -MANIFEST_TOOL=$lt_MANIFEST_TOOL - -# Tool to manipulate archived DWARF debug symbol files on Mac OS X. -DSYMUTIL=$lt_DSYMUTIL - -# Tool to change global to local symbols on Mac OS X. -NMEDIT=$lt_NMEDIT - -# Tool to manipulate fat objects and archives on Mac OS X. -LIPO=$lt_LIPO - -# ldd/readelf like tool for Mach-O binaries on Mac OS X. -OTOOL=$lt_OTOOL - -# ldd/readelf like tool for 64 bit Mach-O binaries on Mac OS X 10.4. -OTOOL64=$lt_OTOOL64 - -# Old archive suffix (normally "a"). -libext=$libext - -# Shared library suffix (normally ".so"). -shrext_cmds=$lt_shrext_cmds - -# The commands to extract the exported symbol list from a shared archive. -extract_expsyms_cmds=$lt_extract_expsyms_cmds - -# Variables whose values should be saved in libtool wrapper scripts and -# restored at link time. -variables_saved_for_relink=$lt_variables_saved_for_relink - -# Do we need the "lib" prefix for modules? -need_lib_prefix=$need_lib_prefix - -# Do we need a version for libraries? -need_version=$need_version - -# Library versioning type. -version_type=$version_type - -# Shared library runtime path variable. -runpath_var=$runpath_var - -# Shared library path variable. -shlibpath_var=$shlibpath_var - -# Is shlibpath searched before the hard-coded library search path? -shlibpath_overrides_runpath=$shlibpath_overrides_runpath - -# Format of library name prefix. -libname_spec=$lt_libname_spec - -# List of archive names. First name is the real one, the rest are links. -# The last name is the one that the linker finds with -lNAME -library_names_spec=$lt_library_names_spec - -# The coded name of the library, if different from the real name. -soname_spec=$lt_soname_spec - -# Permission mode override for installation of shared libraries. -install_override_mode=$lt_install_override_mode - -# Command to use after installation of a shared archive. -postinstall_cmds=$lt_postinstall_cmds - -# Command to use after uninstallation of a shared archive. -postuninstall_cmds=$lt_postuninstall_cmds - -# Commands used to finish a libtool library installation in a directory. -finish_cmds=$lt_finish_cmds - -# As "finish_cmds", except a single script fragment to be evaled but -# not shown. -finish_eval=$lt_finish_eval - -# Whether we should hardcode library paths into libraries. -hardcode_into_libs=$hardcode_into_libs - -# Compile-time system search path for libraries. -sys_lib_search_path_spec=$lt_sys_lib_search_path_spec - -# Detected run-time system search path for libraries. -sys_lib_dlsearch_path_spec=$lt_configure_time_dlsearch_path - -# Explicit LT_SYS_LIBRARY_PATH set during ./configure time. -configure_time_lt_sys_library_path=$lt_configure_time_lt_sys_library_path - -# Whether dlopen is supported. -dlopen_support=$enable_dlopen - -# Whether dlopen of programs is supported. -dlopen_self=$enable_dlopen_self - -# Whether dlopen of statically linked programs is supported. -dlopen_self_static=$enable_dlopen_self_static - -# Commands to strip libraries. -old_striplib=$lt_old_striplib -striplib=$lt_striplib - - -# The linker used to build libraries. -LD=$lt_LD - -# How to create reloadable object files. -reload_flag=$lt_reload_flag -reload_cmds=$lt_reload_cmds - -# Commands used to build an old-style archive. -old_archive_cmds=$lt_old_archive_cmds - -# A language specific compiler. -CC=$lt_compiler - -# Is the compiler the GNU compiler? -with_gcc=$GCC - -# Compiler flag to turn off builtin functions. -no_builtin_flag=$lt_lt_prog_compiler_no_builtin_flag - -# Additional compiler flags for building library objects. -pic_flag=$lt_lt_prog_compiler_pic - -# How to pass a linker flag through the compiler. -wl=$lt_lt_prog_compiler_wl - -# Compiler flag to prevent dynamic linking. -link_static_flag=$lt_lt_prog_compiler_static - -# Does compiler simultaneously support -c and -o options? -compiler_c_o=$lt_lt_cv_prog_compiler_c_o - -# Whether or not to add -lc for building shared libraries. -build_libtool_need_lc=$archive_cmds_need_lc - -# Whether or not to disallow shared libs when runtime libs are static. -allow_libtool_libs_with_static_runtimes=$enable_shared_with_static_runtimes - -# Compiler flag to allow reflexive dlopens. -export_dynamic_flag_spec=$lt_export_dynamic_flag_spec - -# Compiler flag to generate shared objects directly from archives. -whole_archive_flag_spec=$lt_whole_archive_flag_spec - -# Whether the compiler copes with passing no objects directly. -compiler_needs_object=$lt_compiler_needs_object - -# Create an old-style archive from a shared archive. -old_archive_from_new_cmds=$lt_old_archive_from_new_cmds - -# Create a temporary old-style archive to link instead of a shared archive. -old_archive_from_expsyms_cmds=$lt_old_archive_from_expsyms_cmds - -# Commands used to build a shared archive. -archive_cmds=$lt_archive_cmds -archive_expsym_cmds=$lt_archive_expsym_cmds - -# Commands used to build a loadable module if different from building -# a shared archive. -module_cmds=$lt_module_cmds -module_expsym_cmds=$lt_module_expsym_cmds - -# Whether we are building with GNU ld or not. -with_gnu_ld=$lt_with_gnu_ld - -# Flag that allows shared libraries with undefined symbols to be built. -allow_undefined_flag=$lt_allow_undefined_flag - -# Flag that enforces no undefined symbols. -no_undefined_flag=$lt_no_undefined_flag - -# Flag to hardcode \$libdir into a binary during linking. -# This must work even if \$libdir does not exist -hardcode_libdir_flag_spec=$lt_hardcode_libdir_flag_spec - -# Whether we need a single "-rpath" flag with a separated argument. -hardcode_libdir_separator=$lt_hardcode_libdir_separator - -# Set to "yes" if using DIR/libNAME\$shared_ext during linking hardcodes -# DIR into the resulting binary. -hardcode_direct=$hardcode_direct - -# Set to "yes" if using DIR/libNAME\$shared_ext during linking hardcodes -# DIR into the resulting binary and the resulting library dependency is -# "absolute",i.e impossible to change by setting \$shlibpath_var if the -# library is relocated. -hardcode_direct_absolute=$hardcode_direct_absolute - -# Set to "yes" if using the -LDIR flag during linking hardcodes DIR -# into the resulting binary. -hardcode_minus_L=$hardcode_minus_L - -# Set to "yes" if using SHLIBPATH_VAR=DIR during linking hardcodes DIR -# into the resulting binary. -hardcode_shlibpath_var=$hardcode_shlibpath_var - -# Set to "yes" if building a shared library automatically hardcodes DIR -# into the library and all subsequent libraries and executables linked -# against it. -hardcode_automatic=$hardcode_automatic - -# Set to yes if linker adds runtime paths of dependent libraries -# to runtime path list. -inherit_rpath=$inherit_rpath - -# Whether libtool must link a program against all its dependency libraries. -link_all_deplibs=$link_all_deplibs - -# Set to "yes" if exported symbols are required. -always_export_symbols=$always_export_symbols - -# The commands to list exported symbols. -export_symbols_cmds=$lt_export_symbols_cmds - -# Symbols that should not be listed in the preloaded symbols. -exclude_expsyms=$lt_exclude_expsyms - -# Symbols that must always be exported. -include_expsyms=$lt_include_expsyms - -# Commands necessary for linking programs (against libraries) with templates. -prelink_cmds=$lt_prelink_cmds - -# Commands necessary for finishing linking programs. -postlink_cmds=$lt_postlink_cmds - -# Specify filename containing input files. -file_list_spec=$lt_file_list_spec - -# How to hardcode a shared library path into an executable. -hardcode_action=$hardcode_action - -# ### END LIBTOOL CONFIG - -_LT_EOF - - cat <<'_LT_EOF' >> "$cfgfile" - -# ### BEGIN FUNCTIONS SHARED WITH CONFIGURE - -# func_munge_path_list VARIABLE PATH -# ----------------------------------- -# VARIABLE is name of variable containing _space_ separated list of -# directories to be munged by the contents of PATH, which is string -# having a format: -# "DIR[:DIR]:" -# string "DIR[ DIR]" will be prepended to VARIABLE -# ":DIR[:DIR]" -# string "DIR[ DIR]" will be appended to VARIABLE -# "DIRP[:DIRP]::[DIRA:]DIRA" -# string "DIRP[ DIRP]" will be prepended to VARIABLE and string -# "DIRA[ DIRA]" will be appended to VARIABLE -# "DIR[:DIR]" -# VARIABLE will be replaced by "DIR[ DIR]" -func_munge_path_list () -{ - case x$2 in - x) - ;; - *:) - eval $1=\"`$ECHO $2 | $SED 's/:/ /g'` \$$1\" - ;; - x:*) - eval $1=\"\$$1 `$ECHO $2 | $SED 's/:/ /g'`\" - ;; - *::*) - eval $1=\"\$$1\ `$ECHO $2 | $SED -e 's/.*:://' -e 's/:/ /g'`\" - eval $1=\"`$ECHO $2 | $SED -e 's/::.*//' -e 's/:/ /g'`\ \$$1\" - ;; - *) - eval $1=\"`$ECHO $2 | $SED 's/:/ /g'`\" - ;; - esac -} - - -# Calculate cc_basename. Skip known compiler wrappers and cross-prefix. -func_cc_basename () -{ - for cc_temp in $*""; do - case $cc_temp in - compile | *[\\/]compile | ccache | *[\\/]ccache ) ;; - distcc | *[\\/]distcc | purify | *[\\/]purify ) ;; - \-*) ;; - *) break;; - esac - done - func_cc_basename_result=`$ECHO "$cc_temp" | $SED "s%.*/%%; s%^$host_alias-%%"` -} - - -# ### END FUNCTIONS SHARED WITH CONFIGURE - -_LT_EOF - - case $host_os in - aix3*) - cat <<\_LT_EOF >> "$cfgfile" -# AIX sometimes has problems with the GCC collect2 program. For some -# reason, if we set the COLLECT_NAMES environment variable, the problems -# vanish in a puff of smoke. -if test set != "${COLLECT_NAMES+set}"; then - COLLECT_NAMES= - export COLLECT_NAMES -fi -_LT_EOF - ;; - esac - - -ltmain=$ac_aux_dir/ltmain.sh - - - # We use sed instead of cat because bash on DJGPP gets confused if - # if finds mixed CR/LF and LF-only lines. Since sed operates in - # text mode, it properly converts lines to CR/LF. This bash problem - # is reportedly fixed, but why not run on old versions too? - sed '$q' "$ltmain" >> "$cfgfile" \ - || (rm -f "$cfgfile"; exit 1) - - mv -f "$cfgfile" "$ofile" || - (rm -f "$ofile" && cp "$cfgfile" "$ofile" && rm -f "$cfgfile") - chmod +x "$ofile" - - ;; - - esac -done # for ac_tag - - -as_fn_exit 0 -_ACEOF -ac_clean_files=$ac_clean_files_save - -test $ac_write_fail = 0 || - as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 - - -# configure is writing to config.log, and then calls config.status. -# config.status does its own redirection, appending to config.log. -# Unfortunately, on DOS this fails, as config.log is still kept open -# by configure, so config.status won't be able to write to it; its -# output is simply discarded. So we exec the FD to /dev/null, -# effectively closing config.log, so it can be properly (re)opened and -# appended to by config.status. When coming back to configure, we -# need to make the FD available again. -if test "$no_create" != yes; then - ac_cs_success=: - ac_config_status_args= - test "$silent" = yes && - ac_config_status_args="$ac_config_status_args --quiet" - exec 5>/dev/null - $SHELL $CONFIG_STATUS $ac_config_status_args || ac_cs_success=false - exec 5>>config.log - # Use ||, not &&, to avoid exiting from the if with $? = 1, which - # would make configure fail if this is the last instruction. - $ac_cs_success || as_fn_exit 1 -fi -if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 -$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} -fi - +#!/bin/sh +dir="`dirname "$0"`/autosetup" +#@@INITCHECK@@# +WRAPPER="$0"; export WRAPPER; exec "`"$dir/autosetup-find-tclsh"`" "$dir/autosetup" "$@" diff --git a/configure.ac b/configure.ac deleted file mode 100644 index 4be94470e4..0000000000 --- a/configure.ac +++ /dev/null @@ -1,951 +0,0 @@ -# -# The build process allows for using a cross-compiler. But the default -# action is to target the same platform that we are running on. The -# configure script needs to discover the following properties of the -# build and target systems: -# -# srcdir -# -# The is the name of the directory that contains the -# "configure" shell script. All source files are -# located relative to this directory. -# -# bindir -# -# The name of the directory where executables should be -# written by the "install" target of the makefile. -# -# program_prefix -# -# Add this prefix to the names of all executables that run -# on the target machine. Default: "" -# -# ENABLE_SHARED -# -# True if shared libraries should be generated. -# -# BUILD_CC -# -# The name of a command that is used to convert C -# source files into executables that run on the build -# platform. -# -# BUILD_CFLAGS -# -# Switches that the build compiler needs in order to construct -# command-line programs. -# -# BUILD_LIBS -# -# Libraries that the build compiler needs in order to construct -# command-line programs. -# -# BUILD_EXEEXT -# -# The filename extension for executables on the build -# platform. "" for Unix and ".exe" for Windows. -# -# TCL_* -# -# Lots of values are read in from the tclConfig.sh script, -# if that script is available. This values are used for -# constructing and installing the TCL extension. -# -# TARGET_READLINE_LIBS -# -# This is the library directives passed to the target linker -# that cause the executable to link against the readline library. -# This might be a switch like "-lreadline" or pathnames of library -# file like "../../src/libreadline.a". -# -# TARGET_READLINE_INC -# -# This variables define the directory that contain header -# files for the readline library. If the compiler is able -# to find on its own, then this can be blank. -# -# TARGET_EXEEXT -# -# The filename extension for executables on the -# target platform. "" for Unix and ".exe" for windows. -# -# This configure.in file is easy to reuse on other projects. Just -# change the argument to AC_INIT. And disable any features that -# you don't need (for example BLT) by erasing or commenting out -# the corresponding code. -# -AC_INIT(sqlcipher, m4_esyscmd([cat VERSION | tr -d '\n'])) - -dnl Make sure the local VERSION file matches this configure script -sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'` -if test "$PACKAGE_VERSION" != "$sqlite_version_sanity_check" ; then -AC_MSG_ERROR([configure script is out of date: - configure \$PACKAGE_VERSION = $PACKAGE_VERSION - top level VERSION file = $sqlite_version_sanity_check -please regen with autoconf]) -fi - -######### -# Programs needed -# -LT_INIT -AC_PROG_INSTALL - -######### -# Enable large file support (if special flags are necessary) -# -AC_SYS_LARGEFILE - -######### -# Check for needed/wanted data types -AC_CHECK_TYPES([int8_t, int16_t, int32_t, int64_t, intptr_t, uint8_t, - uint16_t, uint32_t, uint64_t, uintptr_t]) - -######### -# Check for needed/wanted headers -AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h]) - -######### -# Figure out whether or not we have these functions -# -AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64]) - -######### -# By default, we use the amalgamation (this may be changed below...) -# -USE_AMALGAMATION=1 - -######### -# See whether we can run specific tclsh versions known to work well; -# if not, then we fall back to plain tclsh. -# TODO: try other versions before falling back? -# -AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none) -if test "$TCLSH_CMD" = "none"; then - # If we can't find a local tclsh, then building the amalgamation will fail. - # We act as though --disable-amalgamation has been used. - echo "Warning: can't find tclsh - defaulting to non-amalgamation build." - USE_AMALGAMATION=0 - TCLSH_CMD="tclsh" -fi -AC_SUBST(TCLSH_CMD) - -AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin]) -if test "x${TCLLIBDIR+set}" != "xset" ; then - TCLLIBDIR='$(libdir)' - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do - if test -d $i ; then - TCLLIBDIR=$i - break - fi - done - TCLLIBDIR="${TCLLIBDIR}/sqlite3" -fi - -######### -# Set up an appropriate program prefix -# -if test "$program_prefix" = "NONE"; then - program_prefix="" -fi -AC_SUBST(program_prefix) - -VERSION=[`cat $srcdir/VERSION | sed 's/^\([0-9]*\.*[0-9]*\).*/\1/'`] -AC_MSG_NOTICE(Version set to $VERSION) -AC_SUBST(VERSION) -RELEASE=`cat $srcdir/VERSION` -AC_MSG_NOTICE(Release set to $RELEASE) -AC_SUBST(RELEASE) - -########## -# Handle --with-wasi-sdk=DIR -# -# This must be early because it changes the toolchain. -# -AC_ARG_WITH(wasi-sdk, -AS_HELP_STRING([--with-wasi-sdk=DIR], - [directory containing the WASI SDK. Triggers cross-compile to WASM.]), with_wasisdk=${withval}) -AC_MSG_CHECKING([for WASI SDK directory]) -AC_CACHE_VAL(ac_cv_c_wasi_sdk,[ - # First check to see if --with-tcl was specified. - if test x"${with_wasi_sdk}" != x ; then - if ! test -d "${with_wasi_sdk}" ; then - AC_MSG_ERROR([${with_wasi_sdk} directory doesn't exist]) - fi - AC_MSG_RESULT([${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL]) - use_wasi_sdk=yes - else - use_wasi_sdk=no - fi -]) -if test "${use_wasi_sdk}" = "no" ; then - HAVE_WASI_SDK="" - AC_MSG_RESULT([no]) -else - HAVE_WASI_SDK=1 -# Changing --host and --target have no effect here except to possibly -# cause confusion. autoconf has finished processing them by this -# point. -# -# host_alias=wasm32-wasi -# target=wasm32-wasi -# -# Merely changing CC and LD to the wasi-sdk's is enough to get -# sqlite3.o building in WASM format. - CC="${with_wasi_sdk}/bin/clang" - LD="${with_wasi_sdk}/bin/wasm-ld" - RANLIB="${with_wasi_sdk}/bin/llvm-ranlib" - cross_compiling=yes - enable_threadsafe=no - use_tcl=no - enable_tcl=no - # libtool is apparently hard-coded to use gcc for linking DLLs, so - # we disable the DLL build... - enable_shared=no - AC_MSG_RESULT([yes]) -fi -AC_SUBST(HAVE_WASI_SDK) - - -######### -# Locate a compiler for the build machine. This compiler should -# generate command-line programs that run on the build machine. -# -if test x"$cross_compiling" = xno; then - BUILD_CC=$CC - BUILD_CFLAGS=$CFLAGS -else - if test "${BUILD_CC+set}" != set; then - AC_CHECK_PROGS(BUILD_CC, gcc cc cl) - fi - if test "${BUILD_CFLAGS+set}" != set; then - BUILD_CFLAGS="-g" - fi -fi -AC_SUBST(BUILD_CC) - -########## -# Do we want to support multithreaded use of sqlite -# -AC_ARG_ENABLE(threadsafe, -AS_HELP_STRING([--disable-threadsafe],[Disable mutexing])) -AC_MSG_CHECKING([whether to support threadsafe operation]) -if test "$enable_threadsafe" = "no"; then - SQLITE_THREADSAFE=0 - AC_MSG_RESULT([no]) -else - SQLITE_THREADSAFE=1 - AC_MSG_RESULT([yes]) -fi -AC_SUBST(SQLITE_THREADSAFE) - -if test "$SQLITE_THREADSAFE" = "1"; then - AC_SEARCH_LIBS(pthread_create, pthread) - AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) -fi - -########## -# Which crypto library do we use -# -AC_ARG_WITH([crypto-lib], -AC_HELP_STRING([--with-crypto-lib],[Specify which crypto library to use]), -crypto_lib=$withval) -AC_MSG_CHECKING([for crypto library to use]) -if test "$crypto_lib" = "none"; then - AC_MSG_RESULT([none]) -else - if test "$crypto_lib" = "commoncrypto"; then - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_CC" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_CC" - AC_MSG_RESULT([commoncrypto]) - else - if test "$crypto_lib" = "libtomcrypt"; then - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_LIBTOMCRYPT" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_LIBTOMCRYPT" - AC_MSG_RESULT([libtomcrypt]) - AC_CHECK_LIB([tomcrypt], [register_cipher], , - AC_MSG_ERROR([Library crypto not found. Install libtomcrypt!"])) - else - if test "$crypto_lib" = "nss"; then - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_NSS" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_NSS" - AC_MSG_RESULT([nss3]) - AC_CHECK_LIB([nss3], [PK11_Decrypt], , - AC_MSG_ERROR([Library crypto not found. Install nss!"])) - else - CFLAGS="$CFLAGS -DSQLCIPHER_CRYPTO_OPENSSL" - BUILD_CFLAGS="$BUILD_CFLAGS -DSQLCIPHER_CRYPTO_OPENSSL" - AC_MSG_RESULT([openssl]) - AC_CHECK_LIB([crypto], [HMAC_Init_ex], , - AC_MSG_ERROR([Library crypto not found. Install openssl!"])) - fi - fi - fi -fi - -########## -# Do we want to allow a connection created in one thread to be used -# in another thread. This does not work on many Linux systems (ex: RedHat 9) -# due to bugs in the threading implementations. This is thus off by default. -# -AC_ARG_ENABLE(cross-thread-connections, -AC_HELP_STRING([--enable-cross-thread-connections],[Allow connection sharing across threads]),,enable_xthreadconnect=no) -AC_MSG_CHECKING([whether to allow connections to be shared across threads]) -if test "$enable_xthreadconnect" = "no"; then - XTHREADCONNECT='' - AC_MSG_RESULT([no]) -else - XTHREADCONNECT='-DSQLITE_ALLOW_XTHREAD_CONNECT=1' - AC_MSG_RESULT([yes]) -fi -AC_SUBST(XTHREADCONNECT) - -########## -# Do we want to support release -# -AC_ARG_ENABLE(releasemode, -AS_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no) -AC_MSG_CHECKING([whether to support shared library linked as release mode or not]) -if test "$enable_releasemode" = "no"; then - ALLOWRELEASE="" - AC_MSG_RESULT([no]) -else - ALLOWRELEASE="-release `cat $srcdir/VERSION`" - AC_MSG_RESULT([yes]) -fi -AC_SUBST(ALLOWRELEASE) - -########## -# Do we want temporary databases in memory -# -AC_ARG_ENABLE(tempstore, -AS_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no) -AC_MSG_CHECKING([whether to use an in-ram database for temporary tables]) -case "$enable_tempstore" in - never ) - TEMP_STORE=0 - AC_MSG_RESULT([never]) - ;; - no ) - TEMP_STORE=1 - AC_MSG_RESULT([no]) - ;; - yes ) - TEMP_STORE=2 - AC_MSG_RESULT([yes]) - ;; - always ) - TEMP_STORE=3 - AC_MSG_RESULT([always]) - ;; - * ) - TEMP_STORE=1 - AC_MSG_RESULT([no]) - ;; -esac - -AC_SUBST(TEMP_STORE) - -########### -# Lots of things are different if we are compiling for Windows using -# the CYGWIN environment. So check for that special case and handle -# things accordingly. -# -AC_MSG_CHECKING([if executables have the .exe suffix]) -if test "$config_BUILD_EXEEXT" = ".exe"; then - CYGWIN=yes - AC_MSG_RESULT(yes) -else - AC_MSG_RESULT(unknown) -fi -if test "$CYGWIN" != "yes"; then - m4_warn([obsolete], -[AC_CYGWIN is obsolete: use AC_CANONICAL_HOST and check if $host_os -matches *cygwin*])dnl -AC_CANONICAL_HOST -case $host_os in - *cygwin* ) CYGWIN=yes;; - * ) CYGWIN=no;; -esac - -fi -if test "$CYGWIN" = "yes"; then - BUILD_EXEEXT=.exe -else - BUILD_EXEEXT=$EXEEXT -fi -if test x"$cross_compiling" = xno; then - TARGET_EXEEXT=$BUILD_EXEEXT -else - TARGET_EXEEXT=$config_TARGET_EXEEXT -fi -if test "$TARGET_EXEEXT" = ".exe"; then - SQLITE_OS_UNIX=0 - SQLITE_OS_WIN=1 - CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1" -else - SQLITE_OS_UNIX=1 - SQLITE_OS_WIN=0 - CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1" -fi - -AC_SUBST(BUILD_EXEEXT) -AC_SUBST(SQLITE_OS_UNIX) -AC_SUBST(SQLITE_OS_WIN) -AC_SUBST(TARGET_EXEEXT) - -########## -# Figure out all the parameters needed to compile against Tcl. -# -# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG -# macros in the in the tcl.m4 file of the standard TCL distribution. -# Those macros could not be used directly since we have to make some -# minor changes to accomodate systems that do not have TCL installed. -# -AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]), - [use_tcl=$enableval],[use_tcl=yes]) -if test "${use_tcl}" = "yes" ; then - AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval}) - AC_MSG_CHECKING([for Tcl configuration]) - AC_CACHE_VAL(ac_cv_c_tclconfig,[ - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)` - else - AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh]) - fi - fi - - # Start autosearch by asking tclsh - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # On ubuntu 14.10, $auto_path on tclsh is not quite correct. - # So try again after applying corrections. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # Recent versions of Xcode on Macs hid the tclConfig.sh file - # in a strange place. - if test x"${ac_cv_c_tclconfig}" = x ; then - if test x"$cross_compiling" = xno; then - for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - `ls -d ${libdir} 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i; pwd)` - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null` - do - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig=`(cd $i/unix; pwd)` - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tclconfig}" = x ; then - use_tcl=no - AC_MSG_WARN(Can't find Tcl configuration definitions) - AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***) - AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***) - else - TCL_BIN_DIR=${ac_cv_c_tclconfig} - AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh) - - AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh]) - if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then - AC_MSG_RESULT([loading]) - . $TCL_BIN_DIR/tclConfig.sh - else - AC_MSG_RESULT([file not found]) - fi - - # - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - # - - if test -f $TCL_BIN_DIR/Makefile ; then - TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC} - TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC} - TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH} - fi - - # - # eval is required to do the TCL_DBGX substitution - # - - eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\"" - eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\"" - eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\"" - - eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\"" - eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\"" - eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\"" - - AC_SUBST(TCL_VERSION) - AC_SUBST(TCL_BIN_DIR) - AC_SUBST(TCL_SRC_DIR) - AC_SUBST(TCL_INCLUDE_SPEC) - - AC_SUBST(TCL_LIB_FILE) - AC_SUBST(TCL_LIB_FLAG) - AC_SUBST(TCL_LIB_SPEC) - - AC_SUBST(TCL_STUB_LIB_FILE) - AC_SUBST(TCL_STUB_LIB_FLAG) - AC_SUBST(TCL_STUB_LIB_SPEC) - AC_SUBST(TCL_SHLIB_SUFFIX) - fi -fi -if test "${use_tcl}" = "no" ; then - HAVE_TCL="" -else - HAVE_TCL=1 -fi -AC_SUBST(HAVE_TCL) - -########## -# Figure out what C libraries are required to compile programs -# that use "readline()" library. -# -TARGET_READLINE_LIBS="" -TARGET_READLINE_INC="" -TARGET_HAVE_READLINE=0 -TARGET_HAVE_EDITLINE=0 -AC_ARG_ENABLE([editline], - [AS_HELP_STRING([--enable-editline],[enable BSD editline support])], - [with_editline=$enableval], - [with_editline=auto]) -AC_ARG_ENABLE([readline], - [AS_HELP_STRING([--disable-readline],[disable readline support])], - [with_readline=$enableval], - [with_readline=auto]) - -if test x"$with_editline" != xno; then - sLIBS=$LIBS - LIBS="" - TARGET_HAVE_EDITLINE=1 - AC_SEARCH_LIBS(readline,edit,[with_readline=no],[TARGET_HAVE_EDITLINE=0]) - TARGET_READLINE_LIBS=$LIBS - LIBS=$sLIBS -fi -if test x"$with_readline" != xno; then - found="yes" - - AC_ARG_WITH([readline-lib], - [AS_HELP_STRING([--with-readline-lib],[specify readline library])], - [with_readline_lib=$withval], - [with_readline_lib="auto"]) - if test "x$with_readline_lib" = xauto; then - save_LIBS="$LIBS" - LIBS="" - AC_SEARCH_LIBS(tgetent, [readline ncurses curses termcap], [term_LIBS="$LIBS"], [term_LIBS=""]) - AC_CHECK_LIB([readline], [readline], [TARGET_READLINE_LIBS="-lreadline"], [found="no"]) - TARGET_READLINE_LIBS="$TARGET_READLINE_LIBS $term_LIBS" - LIBS="$save_LIBS" - else - TARGET_READLINE_LIBS="$with_readline_lib" - fi - - AC_ARG_WITH([readline-inc], - [AS_HELP_STRING([--with-readline-inc],[specify readline include paths])], - [with_readline_inc=$withval], - [with_readline_inc="auto"]) - if test "x$with_readline_inc" = xauto; then - AC_CHECK_HEADER(readline.h, [found="yes"], [ - found="no" - if test "$cross_compiling" != yes; then - for dir in /usr /usr/local /usr/local/readline /usr/contrib /mingw; do - for subdir in include include/readline; do - AC_CHECK_FILE($dir/$subdir/readline.h, found=yes) - if test "$found" = "yes"; then - TARGET_READLINE_INC="-I$dir/$subdir" - break - fi - done - test "$found" = "yes" && break - done - fi - ]) - else - TARGET_READLINE_INC="$with_readline_inc" - fi - - if test x"$found" = xno; then - TARGET_READLINE_LIBS="" - TARGET_READLINE_INC="" - TARGET_HAVE_READLINE=0 - else - TARGET_HAVE_READLINE=1 - fi -fi - -AC_SUBST(TARGET_READLINE_LIBS) -AC_SUBST(TARGET_READLINE_INC) -AC_SUBST(TARGET_HAVE_READLINE) -AC_SUBST(TARGET_HAVE_EDITLINE) - -########## -# Figure out what C libraries are required to compile programs -# that use "fdatasync()" function. -# -AC_SEARCH_LIBS(fdatasync, [rt]) - -######### -# check for debug enabled -AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain])) -AC_MSG_CHECKING([build type]) -if test "${enable_debug}" = "yes" ; then - TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0" - AC_MSG_RESULT([debug]) -else - TARGET_DEBUG="-DNDEBUG" - AC_MSG_RESULT([release]) -fi -AC_SUBST(TARGET_DEBUG) - -######### -# See whether we should use the amalgamation to build - -AC_ARG_ENABLE(amalgamation, AS_HELP_STRING([--disable-amalgamation], - [Disable the amalgamation and instead build all files separately])) -if test "${enable_amalgamation}" = "no" ; then - USE_AMALGAMATION=0 -fi -AC_SUBST(USE_AMALGAMATION) - -######### -# By default, amalgamation sqlite3.c will have #line directives. -# This is a build option not shown by ./configure --help -# To control it, use configure option: amalgamation_line_macros=? -# where ? is no to suppress #line directives or yes to create them. -AMALGAMATION_LINE_MACROS=--linemacros=1 -AC_ARG_VAR(amalgamation_line_macros,) -AC_SUBST(AMALGAMATION_LINE_MACROS) -if test "${amalgamation_line_macros+set}" = set; then : - enableval=$amalgamation_line_macros; -fi -if test "${amalgamation_line_macros}" = "yes" ; then - AMALGAMATION_LINE_MACROS=--linemacros=1 -fi -if test "${amalgamation_line_macros}" = "no" ; then - AMALGAMATION_LINE_MACROS=--linemacros=0 -fi - -######### -# Look for zlib. Only needed by extensions and by the sqlite3.exe shell -AC_CHECK_HEADERS(zlib.h) -AC_SEARCH_LIBS(deflate, z, [HAVE_ZLIB="-DSQLITE_HAVE_ZLIB=1"], [HAVE_ZLIB=""]) -AC_SUBST(HAVE_ZLIB) - -######### -# See whether we should allow loadable extensions -AC_ARG_ENABLE(load-extension, AS_HELP_STRING([--disable-load-extension], - [Disable loading of external extensions]),,[enable_load_extension=yes]) -if test "${enable_load_extension}" = "yes" ; then - OPT_FEATURE_FLAGS="" - AC_SEARCH_LIBS(dlopen, dl) -else - OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" -fi - -########## -# Do we want to support math functions -# -AC_ARG_ENABLE(math, -AS_HELP_STRING([--disable-math],[Disable math functions])) -AC_MSG_CHECKING([whether to support math functions]) -if test "$enable_math" = "no"; then - AC_MSG_RESULT([no]) -else - AC_MSG_RESULT([yes]) - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MATH_FUNCTIONS" - AC_SEARCH_LIBS(ceil, m) -fi - -########## -# Do we want to support JSON functions -# -AC_ARG_ENABLE(json, -AS_HELP_STRING([--disable-json],[Disable JSON functions])) -AC_MSG_CHECKING([whether to support JSON functions]) -if test "$enable_json" = "no"; then - AC_MSG_RESULT([no]) - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_OMIT_JSON" -else - AC_MSG_RESULT([yes]) -fi - -######## -# The --enable-all argument is short-hand to enable -# multiple extensions. -AC_ARG_ENABLE(all, AS_HELP_STRING([--enable-all], - [Enable FTS4, FTS5, Geopoly, RTree, Sessions])) - -########## -# Do we want to support memsys3 and/or memsys5 -# -AC_ARG_ENABLE(memsys5, - AS_HELP_STRING([--enable-memsys5],[Enable MEMSYS5])) -AC_MSG_CHECKING([whether to support MEMSYS5]) -if test "${enable_memsys5}" = "yes"; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi -AC_ARG_ENABLE(memsys3, - AS_HELP_STRING([--enable-memsys3],[Enable MEMSYS3])) -AC_MSG_CHECKING([whether to support MEMSYS3]) -if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi - -######### -# See whether we should enable Full Text Search extensions -AC_ARG_ENABLE(fts3, AS_HELP_STRING([--enable-fts3], - [Enable the FTS3 extension])) -AC_MSG_CHECKING([whether to support FTS3]) -if test "${enable_fts3}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS3" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi -AC_ARG_ENABLE(fts4, AS_HELP_STRING([--enable-fts4], - [Enable the FTS4 extension])) -AC_MSG_CHECKING([whether to support FTS4]) -if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then - AC_MSG_RESULT([yes]) - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS4" - AC_SEARCH_LIBS([log],[m]) -else - AC_MSG_RESULT([no]) -fi -AC_ARG_ENABLE(fts5, AS_HELP_STRING([--enable-fts5], - [Enable the FTS5 extension])) -AC_MSG_CHECKING([whether to support FTS5]) -if test "${enable_fts5}" = "yes" -o "${enable_all}" = "yes" ; then - AC_MSG_RESULT([yes]) - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_FTS5" - AC_SEARCH_LIBS([log],[m]) -else - AC_MSG_RESULT([no]) -fi - -######### -# See whether we should enable the LIMIT clause on UPDATE and DELETE -# statements. -AC_ARG_ENABLE(update-limit, AS_HELP_STRING([--enable-update-limit], - [Enable the UPDATE/DELETE LIMIT clause])) -AC_MSG_CHECKING([whether to support LIMIT on UPDATE and DELETE statements]) -if test "${enable_update_limit}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi - -######### -# See whether we should enable GEOPOLY -AC_ARG_ENABLE(geopoly, AS_HELP_STRING([--enable-geopoly], - [Enable the GEOPOLY extension]), - [enable_geopoly=yes],[enable_geopoly=no]) -AC_MSG_CHECKING([whether to support GEOPOLY]) -if test "${enable_geopoly}" = "yes" -o "${enable_all}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_GEOPOLY" - enable_rtree=yes - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi - -######### -# See whether we should enable RTREE -AC_ARG_ENABLE(rtree, AS_HELP_STRING([--enable-rtree], - [Enable the RTREE extension])) -AC_MSG_CHECKING([whether to support RTREE]) -if test "${enable_rtree}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_RTREE" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi - -######### -# See whether we should enable the SESSION extension -AC_ARG_ENABLE(session, AS_HELP_STRING([--enable-session], - [Enable the SESSION extension])) -AC_MSG_CHECKING([whether to support SESSION]) -if test "${enable_session}" = "yes" -o "${enable_all}" = "yes" ; then - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_SESSION" - OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_PREUPDATE_HOOK" - AC_MSG_RESULT([yes]) -else - AC_MSG_RESULT([no]) -fi - -######### -# attempt to duplicate any OMITS and ENABLES into the ${OPT_FEATURE_FLAGS} parameter -for option in $CFLAGS $CPPFLAGS -do - case $option in - -DSQLITE_OMIT*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";; - -DSQLITE_ENABLE*) OPT_FEATURE_FLAGS="$OPT_FEATURE_FLAGS $option";; - esac -done -AC_SUBST(OPT_FEATURE_FLAGS) - - -# attempt to remove any OMITS and ENABLES from the $(CFLAGS) parameter -ac_temp_CFLAGS="" -for option in $CFLAGS -do - case $option in - -DSQLITE_OMIT*) ;; - -DSQLITE_ENABLE*) ;; - *) ac_temp_CFLAGS="$ac_temp_CFLAGS $option";; - esac -done -CFLAGS=$ac_temp_CFLAGS - - -# attempt to remove any OMITS and ENABLES from the $(CPPFLAGS) parameter -ac_temp_CPPFLAGS="" -for option in $CPPFLAGS -do - case $option in - -DSQLITE_OMIT*) ;; - -DSQLITE_ENABLE*) ;; - *) ac_temp_CPPFLAGS="$ac_temp_CPPFLAGS $option";; - esac -done -CPPFLAGS=$ac_temp_CPPFLAGS - - -# attempt to remove any OMITS and ENABLES from the $(BUILD_CFLAGS) parameter -ac_temp_BUILD_CFLAGS="" -for option in $BUILD_CFLAGS -do - case $option in - -DSQLITE_OMIT*) ;; - -DSQLITE_ENABLE*) ;; - *) ac_temp_BUILD_CFLAGS="$ac_temp_BUILD_CFLAGS $option";; - esac -done -BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS - - -######### -# See whether we should use GCOV -AC_ARG_ENABLE(gcov, AS_HELP_STRING([--enable-gcov], - [Enable coverage testing using gcov])) -if test "${use_gcov}" = "yes" ; then - USE_GCOV=1 -else - USE_GCOV=0 -fi -AC_SUBST(USE_GCOV) - -######### -# Enable/disabled amalagamation line macros -######## -AMALGAMATION_LINE_MACROS=--linemacros=0 -if test "${amalgamation_line_macros}" = "yes" ; then - AMALGAMATION_LINE_MACROS=--linemacros=1 -fi -if test "${amalgamation_line_macros}" = "no" ; then - AMALGAMATION_LINE_MACROS=--linemacros=0 -fi -AC_SUBST(AMALGAMATION_LINE_MACROS) - -######### -# Output the config header -AC_CONFIG_HEADERS(sqlite_cfg.h) - -######### -# Generate the output files. -# -AC_SUBST(BUILD_CFLAGS) -AC_CONFIG_FILES([ -Makefile -sqlcipher.pc -]) -AC_OUTPUT diff --git a/contrib/sqlitecon.tcl b/contrib/sqlitecon.tcl index b5dbcafc2a..78463a1ffa 100644 --- a/contrib/sqlitecon.tcl +++ b/contrib/sqlitecon.tcl @@ -567,7 +567,7 @@ proc sqlitecon::Cut w { } } -# Do a paste opeation. +# Do a paste operation. # proc sqlitecon::Paste w { if {[sqlitecon::canCut $w]==1} { diff --git a/doc/compile-for-unix.md b/doc/compile-for-unix.md new file mode 100644 index 0000000000..ce76b97bae --- /dev/null +++ b/doc/compile-for-unix.md @@ -0,0 +1,70 @@ +# Notes On Compiling SQLite On All Kinds Of Unix + +Here are step-by-step instructions on how to build SQLite from +canonical source on any modern machine that isn't Windows. These +notes are tested (on 2024-10-11) on Ubuntu and on MacOS, but they +are general and should work on most any modern unix platform. +See the companion document ([](./compile-for-windows.md>)) for +guidance on building for Windows. + + 1. Install a C-compiler. GCC or Clang both work fine. If you are + reading this document, you've probably already done that. + + 2. *(Optional):* Install TCL development libraries. In this note, + we'll do a private install in the $HOME/local directory, + but you can make adjustments to install TCL wherever you like. + This document assumes you are working with TCL version 9.0. + See also the [](./tcl-extension-testing.md) document that contains + more details on compiling Tcl for use with SQLite. +
    +
  1. Get the TCL source archive, perhaps from + + or . +
  2. Untar the source archive. CD into the "unix/" subfolder + of the source tree. +
  3. Run: `mkdir $HOME/local` +
  4. Run: `./configure --prefix=$HOME/local` +
  5. Run: `make install` +
+

+ As of 2024-10-25, TCL is not longer required for many + common build targets, such as "sqlite3.c" or the "sqlite3" + command-line tool. So you can skip this step if that is all + you want to build. TCL is still required to run "make test" + and similar, or to build the TCL extension, of course. + + 4. Download the SQLite source tree and unpack it. CD into the + toplevel directory of the source tree. + + 5. Run: `./configure --enable-all --with-tclsh=$HOME/local/bin/tclsh9.0` + + You do not need to use --with-tclsh if the tclsh you want to use is the + first one on your PATH or if you are building without TCL. + + 6. Run the "`Makefile`" makefile with an appropriate target. + Examples: +

    +
  • `make sqlite3.c` +
  • `make sqlite3` +
  • `make sqldiff` +
  • `make sqlite3_rsync` +
+

None of the targets above require TCL. TCL is needed + for the following targets: +

    +
  • `make tclextension-install` +
  • `make devtest` +
  • `make releasetest` +
  • `make sqlite3_analyzer` +
+ + It is not required that you run the "tclextension-install" target prior to + running tests. However, the tests will run more smoothly if you do. + The version of SQLite used for the TCL extension does *not* need to + correspond to the version of SQLite under test. So you can install the + SQLite TCL extension once, and then use it to test many different versions + of SQLite. + + + 7. For a debugging build of the CLI, where the ".treetrace" and ".wheretrace" + commands work, add the the --with-debug argument to configure. diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md new file mode 100644 index 0000000000..0e59c83fed --- /dev/null +++ b/doc/compile-for-windows.md @@ -0,0 +1,182 @@ +# Notes On Compiling SQLite On Windows 11 + +Below are step-by-step instructions on how to build SQLite from +canonical source on a new Windows 11 PC, as of 2025-10-31. +See [](./compile-for-unix.md) for a similar guide for unix-like +systems, including MacOS. + + 1. Install Microsoft Visual Studio. The free "community edition" + will work fine. Do a standard install for C++ development. + SQLite only needs the + "cl" compiler and the "nmake" build tool. +
  • Note: + VS2015 or later is required for the procedures below to + all work. You *might* be able to get the build to work with + earlier versions of MSVC, but in that case the TCL installation + of step 3 will be required, since the "jimsh0.c" program of + Autosetup that is used as a substitute for "tclsh.exe" won't + compile with versions of Visual Studio prior to VS2015. In any + event, building SQLite from canonical source code on Windows + is not supported for earlier versions of Visual Studio.
+ + 2. Under the "Start" menu, find "All Apps" then go to "Visual Studio 20XX" + and find "x64 Native Tools Command Prompt for VS 20XX". Pin that + application to your task bar, as you will use it a lot. Bring up + an instance of this command prompt and do all of the subsequent steps + in that "x64 Native Tools" command prompt. (Or use "x86" if you want + a 32-bit build. Or use "ARM64" if you want to do a build for Windows + on ARM.) The subsequent steps will not work in a vanilla + DOS prompt. Nor will they work in PowerShell. + + 3. *(Optional):* Install TCL development libraries. + This note assumes that you will + install the TCL development libraries in the "`c:\Tcl`" directory. + Make adjustments + if you want TCL installed somewhere else. SQLite needs both the + "tclsh90.exe" command-line tool as part of the build process, and + the "tcl90.lib" and "tclstub.lib" libraries in order to run tests. + This document assumes you are working with TCL version 9.0. + See [](./tcl-extension-testing.md#windows) for guidance on how + to compile TCL version 8.6 for use with SQLite. +
    +
  1. Get the TCL source archive, perhaps from + + or . +
  2. Untar or unzip the source archive. CD into the "win/" subfolder + of the source tree. +
  3. Run: `nmake /f makefile.vc INSTALLDIR=c:\Tcl release` +
  4. Run: `nmake /f makefile.vc INSTALLDIR=c:\Tcl install`
    + Notes: +
      +
    1. The previous two `nmake` commands must be run separately. +
    2. Also, the INSTALLDIR=... argument is required on both. +
    +
  5. Optional: CD to `c:\Tcl\bin` and make a copy of + `tclsh90.exe` over into just `tclsh.exe`. +
  6. Optional: + Add `c:\Tcl\bin` to your %PATH%. To do this, go to Settings + and search for "path". Select "edit environment variables for + your account" and modify your default PATH accordingly. + You will need to close and reopen your command prompts after + making this change. +
+ + As of 2024-10-25, TCL is not longer required for many + common build targets, such as "sqlite3.c" or the "sqlite3.exe" + command-line tool. So you can skip this step if that is all + you want to build. TCL is still required to run "make test" + and similar, or to build the TCL extension, of course. + + 4. Download the SQLite source tree and unpack it. CD into the + toplevel directory of the source tree. + + 5. Run the "`Makefile.msc`" makefile with an appropriate target. + Examples: +
    +
  • `nmake /f makefile.msc` +
  • `nmake /f makefile.msc sqlite3.c` +
  • `nmake /f makefile.msc sqlite3.exe` +
  • `nmake /f makefile.msc sqldiff.exe` +
  • `nmake /f makefile.msc sqlite3_rsync.exe` +
+

No TCL is required for the nmake targets above. But for the ones + that follow, you will need a TCL installation, as described in step 3 + above. If you install TCL in some directory other than C:\\Tcl, then + you will also need to add the "TCLDIR=<dir>" option on the + nmake command line to tell nmake where your TCL is installed. +

    +
  • `nmake /f makefile.msc tclextension-install` +
  • `nmake /f makefile.msc devtest` +
  • `nmake /f makefile.msc releasetest` +
  • `nmake /f makefile.msc sqlite3_analyzer.exe` +
+ + It is not required that you run the "tclextension-install" target prior to + running tests. However, the tests will run more smoothly if you do. + The version of SQLite used for the TCL extension does *not* need to + correspond to the version of SQLite under test. So you can install the + SQLite TCL extension once, and then use it to test many different versions + of SQLite. + + + 7. For a debugging build of the CLI, where the ".treetrace" and ".wheretrace" + commands work, add the DEBUG=3 argument to nmake. Like this: +
    +
  • `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe` +
+ + +## 32-bit Builds + +Doing a 32-bit build is just like doing a 64-bit build with the +following minor changes: + + 1. Use the "x86 Native Tools Command Prompt" instead of + "x64 Native Tools Command Prompt". "**x86**" instead of "**x64**". + + 2. Use a different installation directory for TCL. + The recommended directory is `c:\tcl32`. Thus you end up + with two TCL builds: +
    +
  • `c:\tcl` ← 64-bit (the default) +
  • `c:\tcl32` ← 32-bit +
+ + 3. Ensure that `c:\tcl32\bin` comes before `c:\tcl\bin` on + your PATH environment variable. You can achieve this using + a command like: +
    +
  • `set PATH=c:\tcl32\bin;%PATH%` +
+ +## Building a DLL + +The command the developers use for building the deliverable DLL on the +[download page](https://sqlite.org/download.html) is as follows: + +> nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1" + +That command generates both the sqlite3.dll and sqlite3.def files. The same +command works for both 32-bit and 64-bit builds. + +## Statically Linking The TCL Library + +Some utility programs associated with SQLite need to be linked +with TCL in order to function. The [sqlite3_analyzer.exe program](https://sqlite.org/sqlanalyze.html) +is an example. You can build as described above, and then +enter: + +> nmake /f Makefile.msc sqlite3_analyzer.exe + +And you will end up with a working executable. However, that executable +will depend on having the "tcl98.dll" library somewhere on your %PATH%. +Use the following steps to build an executable that has the TCL library +statically linked so that it does not depend on separate DLL: + + 1. Use the appropriate "Command Prompt" window - either x86 or + x64, depending on whether you want a 32-bit or 64-bit executable. + + 2. Untar the TCL source tarball into a fresh directory. CD into + the "win/" subfolder. + + 3. Run: `nmake /f makefile.vc OPTS=static shell` + + 4. CD into the "Release*" subfolder that is created (note the + wildcard - the full name of the directory might vary). There + you will find the "tcl90s.lib" file. Copy this file into the + same directory that you put the "tcl90.lib" on your initial + installation. (In this document, that directory is + "`C:\Tcl32\lib`" for 32-bit builds and + "`C:\Tcl\lib`" for 64-bit builds.) + + 5. CD into your SQLite source code directory and build the desired + utility program, but add the following extra argument to the + nmake command line: + 
STATICALLY_LINK_TCL=1
+

So, for example, to build a statically linked version of + sqlite3_analyzer.exe, you might type: + 

nmake /f Makefile.msc STATICALLY_LINK_TCL=1 sqlite3_analyzer.exe
+ + 6. After your executable is built, you can verify that it does not + depend on the TCL DLL by running: + 
dumpbin /dependents sqlite3_analyzer.exe
diff --git a/doc/jsonb.md b/doc/jsonb.md new file mode 100644 index 0000000000..63ce77b170 --- /dev/null +++ b/doc/jsonb.md @@ -0,0 +1,289 @@ +# The JSONB Format + +This document describes SQLite's JSONB binary encoding of +JSON. + +## 1.0 What Is JSONB? + +Beginning with version 3.45.0 (circa 2024-01-01), SQLite supports an +alternative binary encoding of JSON which we call "JSONB". JSONB is +a binary format that stored as a BLOB. + +The advantage of JSONB over ordinary text RFC 8259 JSON is that JSONB +is both slightly smaller (by between 5% and 10% in most cases) and +can be processed in less than half the number of CPU cycles. The built-in +[JSON SQL functions] of SQLite can accept either ordinary text JSON +or the binary JSONB encoding for any of their JSON inputs. + +The "JSONB" name is inspired by [PostgreSQL](https://postgresql.org), but the +on-disk format for SQLite's JSONB is not the same as PostgreSQL's. +The two formats have the same name, but they have wildly different internal +representations and are not in any way binary compatible. + +The central idea behind this JSONB specification is that each element +begins with a header that includes the size and type of that element. +The header takes the place of punctuation such as double-quotes, +curly-brackes, square-brackets, commas, and colons. Since the size +and type of each element is contained in its header, the element can +be read faster since it is no longer necessary to carefully scan forward +looking for the closing delimiter. The payload of JSONB is the same +as for corresponding text JSON. The same payload bytes occur in the +same order. The only real difference between JSONB and ordinary text +JSON is that JSONB includes a binary header on +each element and omits delimiter and separator punctuation. + +### 1.1 Internal Use Only + +The details of the JSONB are not intended to be visible to application +developers. Application developers should look at JSONB as an opaque BLOB +used internally by SQLite. Nevertheless, we want the format to be backwards +compatible across all future versions of SQLite. To that end, the format +is documented by this file in the source tree. But this file should be +used only by SQLite core developers, not by developers of applications +that only use SQLite. + +## 2.0 The Purpose Of This Document + +JSONB is not intended as an external format to be used by +applications. JSONB is designed for internal use by SQLite only. +Programmers do not need to understand the JSONB format in order to +use it effectively. +Applications should access JSONB only through the [JSON SQL functions], +not by looking at individual bytes of the BLOB. + +However, JSONB is intended to be portable and backwards compatible +for all future versions of SQLite. In other words, you should not have +to export and reimport your SQLite database files when you upgrade to +a newer SQLite version. For that reason, the JSONB format needs to +be well-defined. + +This document is therefore similar in purpose to the +[SQLite database file format] document that describes the on-disk +format of an SQLite database file. Applications are not expected +to directly read and write the bits and bytes of SQLite database files. +The SQLite database file format is carefully documented so that it +can be stable and enduring. In the same way, the JSONB representation +of JSON is documented here so that it too can be stable and enduring, +not so that applications can read or writes individual bytes. + +## 3.0 Encoding + +JSONB is a direct translation of the underlying text JSON. The difference +is that JSONB uses a binary encoding that is faster to parse compared to +the detailed syntax of text JSON. + +Each JSON element is encoded as a header and a payload. The header +determines type of element (string, numeric, boolean, null, object, or +array) and the size of the payload. The header can be between 1 and +9 bytes in size. The payload can be any size from zero bytes up to the +maximum allowed BLOB size. + +### 3.1 Payload Size + +The upper four bits of the first byte of the header determine size of the +header and possibly also the size of the payload. +If the upper four bits have a value between 0 and 11, then the header is +exactly one byte in size and the payload size is determined by those +upper four bits. If the upper four bits have a value between 12 and 15, +that means that the total header size is 2, 3, 5, or 9 bytes and the +payload size is unsigned big-endian integer that is contained in the +subsequent bytes. The size integer is the one byte that following the +initial header byte if the upper four bits +are 12, two bytes if the upper bits are 13, four bytes if the upper bits +are 14, and eight bytes if the upper bits are 15. The current design +of SQLite does not support BLOB values larger than 2GiB, so the eight-byte +variant of the payload size integer will never be used by the current code. +The eight-byte payload size integer is included in the specification +to allow for future expansion. + +The header for an element does *not* need to be in its simplest +form. For example, consider the JSON numeric value "`1`". +That element can be encode in five different ways: + + * `0x13 0x31` + * `0xc3 0x01 0x31` + * `0xd3 0x00 0x01 0x31` + * `0xe3 0x00 0x00 0x00 0x01 0x31` + * `0xf3 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x31` + +The shortest encoding is preferred, of course, and usually happens with +primitive elements such as numbers. However the total size of an array +or object might not be known exactly when the header of the element is +first generated. It is convenient to reserve space for the largest +possible header and then go back and fill in the correct payload size +at the end. This technique can result in array or object headers that +are larger than absolutely necessary. + +### 3.2 Element Type + +The least-significant four bits of the first byte of the header (the first +byte masked against 0x0f) determine element type. The following codes are +used: + +
    +

  1. NULL → +The element is a JSON "null". The payload size for a true JSON NULL must +must be zero. Future versions of SQLite might extend the JSONB format +with elements that have a zero element type but a non-zero size. In that +way, legacy versions of SQLite will interpret the element as a NULL +for backwards compatibility while newer versions will interpret the +element in some other way. + +

  2. TRUE → +The element is a JSON "true". The payload size must be zero for a actual +"true" value. Elements with type 1 and a non-zero payload size are +reserved for future expansion. Legacy implementations that see an element +type of 1 with a non-zero payload size should continue to interpret that +element as "true" for compatibility. + +

  3. FALSE → +The element is a JSON "false". The payload size must be zero for a actual +"false" value. Elements with type 2 and a non-zero payload size are +reserved for future expansion. Legacy implementations that see an element +type of 2 with a non-zero payload size should continue to interpret that +element as "false" for compatibility. + +

  4. INT → +The element is a JSON integer value in the canonical +RFC 8259 format, without extensions. The payload is the ASCII +text representation of that numeric value. + +

  5. INT5 → +The element is a JSON integer literal in hexadecimal notation. +The payload is the ASCII text representation of the literal. +Because the payload is in a non-standard format, it will need +to be translated when the JSONB is converted into RFC 8259 text JSON. + +

  6. FLOAT → +The element is a JSON floating-point value in the canonical +RFC 8259 format, without extensions. The payload is the ASCII +text representation of that numeric value. + +

  7. FLOAT5 → +The element is a JSON floating-point value that is not in the +canonical JSON format but rather the extended JSON5 format. +The payload is the ASCII text representation of that numeric value. +Because the payload is in a non-standard format, it will need to +be translated when the JSONB is converted into RFC 8259 text JSON. + +

  8. TEXT → +The element is a JSON string value that does not contain +any escapes nor any characters that need to be escaped for either SQL or +JSON. The payload is the UTF8 text representation of the string value. +The payload does not include string delimiters. + +

  9. TEXTJ → +The element is a JSON string value that contains +RFC 8259 character escapes (such as "\n" or "\u0020"). +Those escapes will need to be translated into actual UTF8 if this element +is [json_extract|extracted] into SQL. +The payload is the UTF8 text representation of the escaped string value. +The payload does not include string delimiters. + +

  10. TEXT5 → +The element is a JSON string value that contains +character escapes, including some character escapes that part of JSON5 +and which are not found in the canonical RFC 8259 spec. +Those escapes will need to be translated into standard JSON prior to +rendering the JSON as text, or into their actual UTF8 characters if this +element is [json_extract|extracted] into SQL. +The payload is the UTF8 text representation of the escaped string value. +The payload does not include string delimiters. + +

  11. TEXTRAW → +The element is a JSON string value that contains +UTF8 characters that need to be escaped if this string is rendered into +standard JSON text. +The payload does not include string delimiters. + +

  12. ARRAY → +The element is a JSON array. The payload contains +JSONB elements that comprise values contained within the array. + +

  13. OBJECT → +The element is a JSON object. The payload contains +pairs of JSONB elements that comprise entries for the JSON object. +The first element in each pair must be a string (types 7 through 10). +The second element of each pair may be any types, including nested +arrays or objects. + +

  14. RESERVED-13 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. + +

  15. RESERVED-14 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. + +

  16. RESERVED-15 → +Reserved for future expansion. Legacy implements that encounter this +element type should raise an error. +

+ +Element types outside the range of 0 to 12 are reserved for future +expansion. The current implement raises an error if see an element type +other than those listed above. However, future versions of SQLite might +use of the three remaining element types to implement indexing or similar +optimizations, to speed up lookup against large JSON arrays and/or objects. + +### 3.3 Design Rationale For Element Types + +A key goal of JSONB is that it should be quick to translate +to and from text JSON and/or be constructed from SQL values. +When converting from text into JSONB, we do not want the +converter subroutine to burn CPU cycles converting elements +values into some standard format which might never be used. +Format conversion is "lazy" - it is deferred until actually +needed. This has implications for the JSONB format design: + + 1. Numeric values are stored as text, not a numbers. The values are + a direct copy of the text JSON values from which they are derived. + + 2. There are multiple element types depending on the details of value + formats. For example, INT is used for pure RFC-8259 integer + literals and INT5 exists for JSON5 extensions such as hexadecimal + notation. FLOAT is used for pure RFC-8259 floating point literals + and FLOAT5 is used for JSON5 extensions. There are four different + representations of strings, depending on where the string came from + and how special characters within the string are escaped. + +A second goal of JSONB is that it should be capable of serving as the +"parse tree" for JSON when a JSON value is being processed by the +various [JSON SQL functions] built into SQLite. Before JSONB was +developed, operations such [json_replace()] and [json_patch()] +and similar worked in three stages: + + + 1. Translate the text JSON into a internal format that is + easier to scan and edit. + 2. Perform the requested operation on the JSON. + 3. Translate the internal format back into text. + +JSONB seeks to serve as the internal format directly - bypassing +the first and third stages of that process. Since most of the CPU +cycles are spent on the first and third stages, that suggests that +JSONB processing will be much faster than text JSON processing. + +So when processing JSONB, only the second stage of the three-stage +process is required. But when processing text JSON, it is still necessary +to do stages one and three. If JSONB is to be used as the internal +binary representation, this is yet another reason to store numeric +values as text. Storing numbers as text minimizes the amount of +conversion work needed for stages one and three. This is also why +there are four different representations of text in JSONB. Different +text representations are used for text coming from different sources +(RFC-8259 JSON, JSON5, or SQL string values) and conversions only +happen if and when they are actually needed. + +### 3.4 Valid JSONB BLOBs + +A valid JSONB BLOB consists of a single JSON element. The element must +exactly fill the BLOB. This one element is often a JSON object or array +and those usually contain additional elements as its payload, but the +element can be a primitive value such a string, number, boolean, or null. + +When the built-in JSON functions are attempting to determine if a BLOB +argument is a JSONB or just a random BLOB, they look at the header of +the outer element to see that it is well-formed and that the element +completely fills the BLOB. If these conditions are met, then the BLOB +is accepted as a JSONB value. diff --git a/doc/lemon.html b/doc/lemon.html index 16aea8784b..965f305c04 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -683,6 +683,7 @@

4.4 Special Directives

  • %endif
  • %extra_argument
  • %fallback +
  • %free
  • %if
  • %ifdef
  • %ifndef @@ -693,6 +694,7 @@

    4.4 Special Directives

  • %parse_accept
  • %parse_failure
  • %right +
  • %realloc
  • %stack_overflow
  • %stack_size
  • %start_symbol @@ -844,7 +846,7 @@

    4.4.7 The %fallback directive

    would have generated a syntax error.

    The %fallback directive was added to support robust parsing of SQL -syntax in SQLite. +syntax in SQLite. The SQL language contains a large assortment of keywords, each of which appears as a different token to the language parser. SQL contains so many keywords that it can be difficult for programmers to keep up with @@ -879,7 +881,7 @@

    4.4.8 The %if directive and its friends

    Grammar text in between "%ifdef MACRO" and the next nested "%endif" is ignored unless the "-DMACRO" command-line option is used. Grammar text -betwen "%ifndef MACRO" and the next nested "%endif" is +between "%ifndef MACRO" and the next nested "%endif" is included except when the "-DMACRO" command-line option is used.

    The text in between "%if CONDITIONAL" and its @@ -1200,6 +1202,21 @@

    4.4.25 The %wildcard directive

    the wildcard token and some other token, the other token is always used. The wildcard token is only matched if there are no alternatives.

    + +

    4.4.26 The %realloc and %free directives

    + +

    The %realloc and %free directives defines function +that allocate and free heap memory. The signatures of these functions +should be the same as the realloc() and free() functions from the standard +C library. + +

    If both of these functions are defined +then these functions are used to allocate and free +memory for supplemental parser stack space, if the initial +parse stack space is exceeded. The initial parser stack size +is specified by either %stack_size or the +-DYYSTACKDEPTH compile-time flag. +

    5.0 Error Processing

    @@ -1224,13 +1241,14 @@

    5.0 Error Processing

    first syntax error, of course, if there are no instances of the "error" non-terminal in your grammar.

    +

    6.0 History of Lemon

    Lemon was originally written by Richard Hipp sometime in the late 1980s on a Sun4 Workstation using K&R C. -There was a companion LL(1) parser generator program named "Lime", the -source code to which as been lost.

    +There was a companion LL(1) parser generator program named "Lime". +The Lime source code has been lost.

    The lemon.c source file was originally many separate files that were compiled together to generate the "lemon" executable. Sometime in the diff --git a/doc/pager-invariants.txt b/doc/pager-invariants.txt index 44444dad54..0fea0a698d 100644 --- a/doc/pager-invariants.txt +++ b/doc/pager-invariants.txt @@ -45,7 +45,7 @@ *** Definition: Two databases (or the same database at two points it time) are said to be "logically equivalent" if they give the same answer to all queries. Note in particular the content of freelist leaf - pages can be changed arbitarily without effecting the logical equivalence + pages can be changed arbitrarily without effecting the logical equivalence of the database. (7) At any time, if any subset, including the empty set and the total set, diff --git a/doc/tcl-extension-testing.md b/doc/tcl-extension-testing.md new file mode 100644 index 0000000000..eb2a8c3a3b --- /dev/null +++ b/doc/tcl-extension-testing.md @@ -0,0 +1,264 @@ +# Test Procedures For The SQLite TCL Extension + +## 1.0 Background + +The SQLite TCL extension logic (in the +"[tclsqlite.c](/file/src/tclsqlite.c)" source +file) is statically linked into "textfixture" executable +which is the program used to do most of the testing +associated with "make test", "make devtest", and/or +"make releasetest". So the functionality of the SQLite +TCL extension is thoroughly vetted during normal testing. The +procedures below are designed to test the loadable extension +aspect of the SQLite TCL extension, and in particular to verify +that the "make tclextension-install" build target works and that +an ordinary tclsh can subsequently run "package require sqlite3". + +This procedure can also be used as a template for how to set up +a local TCL+SQLite development environment. In other words, it +can be be used as a guide on how to compile per-user copies of +Tcl that are used to develop, test, and debug SQLite. In that +case, perhaps make minor changes to the procedure such as: + + * Make TCLBUILD directory is permanent. + * Enable debugging symbols on the Tcl library build. + * Reduce the optimization level to -O0 for easier debugging. + * Also compile "wish" to go with each "tclsh". + + + +## 2.0 Testing On Unix-like Systems (Including Mac) + +See also the [](./compile-for-unix.md) document which provides another +perspective on how to compile SQLite on unix-like systems. + +### 2.1 Setup + +

      +
    1. + [Fossil][] installed. +
    2. Check out source code and set environment variables: +
        +
      1. **TCLSOURCE** → + The top-level directory of a [Fossil][] check-out of the + [TCL source tree][tcl-fossil]. +
      2. **SQLITESOURCE** → + A Fossil check-out of the SQLite source tree. +
      3. **TCLHOME** → + A directory that does not exist at the start of the test and which + will be deleted at the end of the test, and that will contain the + test builds of the TCL libraries and the SQLite TCL Extensions. + It is the top-most installation directory, i.e. the one provided + to Tcl's `./configure --prefix=/path/to/tcl`. +
      4. **TCLVERSION** → + The `X.Y`-form version of Tcl being used: 8.6, 9.0, 9.1... +
      +
    + +### 2.2 Testing TCL 8.x and 9.x on unix + +From a checked-out copy of [the core Tcl tree][tcl-fossil] + +
      +
    1. `TCLVERSION=8.6`
      + ↑ A version of your choice. This process has been tested with + values of 8.6, 9.0, and 9.1 (as of 2025-04-16). The out-of-life + version 8.5 fails some of `make devtest` for undetermined reasons. +
    2. `TCLHOME=$HOME/tcl/$TCLVERSION` +
    3. `TCLSOURCE=/path/to/tcl/checkout` +
    4. `SQLITESOURCE=/path/to/sqlite/checkout` +
    5. `rm -fr $TCLHOME`
      + ↑ Ensure that no stale Tcl installation is laying around. +
    6. `cd $TCLSOURCE` +
    7. `fossil up core-8-6-branch`
      + ↑ The branch corresponding to `$TCLVERSION`, e.g. + `core-9-0-branch` or `trunk`. +
    8. `fossil clean -x` +
    9. `cd unix` +
    10. `./configure --prefix=$TCLHOME --disable-shared`
      + ↑ The `--disable-shared` is to avoid the need to set `LD_LIBRARY_PATH` + when using this Tcl build. +
    11. `make install` +
    12. `cd $SQLITESOURCE` +
    13. `fossil clean -x` +
    14. `./configure --with-tcl=$TCLHOME --all` +
    15. `make tclextension-install`
      + ↑ Verify extension installed at + `$TCLHOME/lib/tcl${TCLVERSION}/sqlite`. +
    16. `make tclextension-list`
      + ↑ Verify TCL extension correctly installed. +
    17. `make tclextension-verify`
      + ↑ Verify that the correct version is installed. +
    18. `$TCLHOME/bin/tclsh[89].[0-9] test/testrunner.tcl release --explain`
      + ↑ Verify thousands of lines of output with no errors. Or + consider running "devtest" without --explain instead of "release". +
    + +### 2.3 Cleanup + +
      +
    1. `rm -rf $TCLHOME` +
    + + +## 3.0 Testing On Windows + +See also the [](./compile-for-windows.md) document which provides another +perspective on how to compile SQLite on Windows. + +### 3.1 Setup for Windows + +(These docs are not as up-to-date as the Unix docs, above.) + +
      +
    1. + [Fossil][] installed. +
    2. + Unix-like command-line tools installed. Example: + [unxutils](https://unxutils.sourceforge.net/) +
    3. [Visual Studio](https://visualstudio.microsoft.com/vs/community/) + installed. VS2015 or later required. +
    4. Check out source code and set environment variables. +
        +
      1. **TCLSOURCE** → + The top-level directory of a Fossil check-out of the TCL source tree. +
      2. **SQLITESOURCE** → + A Fossil check-out of the SQLite source tree. +
      3. **TCLBUILD** → + A directory that does not exist at the start of the test and which + will be deleted at the end of the test, and that will contain the + test builds of the TCL libraries and the SQLite TCL Extensions. +
      4. **ORIGINALPATH** → + The original value of %PATH%. In other words, set as follows: + `set ORIGINALPATH %PATH%` +
      +
    + +### 3.2 Testing TCL 8.6 on Windows + +
      +
    1. `mkdir %TCLBUILD%\tcl86` +
    2. `cd %TCLSOURCE%\win` +
    3. `fossil up core-8-6-16`
      + ↑ Or some other version of Tcl8.6. +
    4. `fossil clean -x` +
    5. `set INSTALLDIR=%TCLBUILD%\tcl86` +
    6. `nmake /f makefile.vc release`
      + ⇅ You *must* invoke the "release" and "install" targets + using separate "nmake" commands or tclsh86t.exe won't be + installed. +
    7. `nmake /f makefile.vc install` +
    8. `cd %SQLITESOURCE%` +
    9. `fossil clean -x` +
    10. `set TCLDIR=%TCLBUILD%\tcl86` +
    11. `set PATH=%TCLBUILD%\tcl86\bin;%ORIGINALPATH%` +
    12. `set TCLSH_CMD=%TCLBUILD%\tcl86\bin\tclsh86t.exe` +
    13. `nmake /f Makefile.msc tclextension-install`
      + ↑ Verify extension installed at %TCLBUILD%\\tcl86\\lib\\tcl8.6\\sqlite3.* +
    14. `nmake /f Makefile.msc tclextension-verify` +
    15. `tclsh86t test/testrunner.tcl release --explain`
      + ↑ Verify thousands of lines of output with no errors. Or + consider running "devtest" without --explain instead of "release". +
    + +### 3.3 Testing TCL 9.0 on Windows + +
      +
    1. `mkdir %TCLBUILD%\tcl90` +
    2. `cd %TCLSOURCE%\win` +
    3. `fossil up core-9-0-0`
      + ↑ Or some other version of Tcl9 +
    4. `fossil clean -x` +
    5. `set INSTALLDIR=%TCLBUILD%\tcl90` +
    6. `nmake /f makefile.vc release`
      + ⇅ You *must* invoke the "release" and "install" targets + using separate "nmake" commands or tclsh90.exe won't be + installed. +
    7. `nmake /f makefile.vc install` +
    8. `cd %SQLITESOURCE%` +
    9. `fossil clean -x` +
    10. `set TCLDIR=%TCLBUILD%\tcl90` +
    11. `set PATH=%TCLBUILD%\tcl90\bin;%ORIGINALPATH%` +
    12. `set TCLSH_CMD=%TCLBUILD%\tcl90\bin\tclsh90.exe` +
    13. `nmake /f Makefile.msc tclextension-install`
      + ↑ Verify extension installed at %TCLBUILD%\\tcl90\\lib\\sqlite3.* +
    14. `nmake /f Makefile.msc tclextension-verify` +
    15. `tclsh90 test/testrunner.tcl release --explain`
      + ↑ Verify thousands of lines of output with no errors. Or + consider running "devtest" without --explain instead of "release". +
    + +### 3.4 Cleanup + +
      +
    1. `rm -rf %TCLBUILD%` +
    + +## 4.0 Testing the TEA(ish) Build (unix only) + +This part requires following the setup instructions for Unix systems, +at the top of this document. + +The former TEA, now TEA(ish), build of this extension uses the same +code as the builds described above but is provided in a form more +convenient for downstream Tcl users. + +It lives in `autoconf/tea` and, as part of the `autoconf` bundle, +_cannot be tested directly from the canonical tree_. Instead it has to +be packaged. + +### 4.1 Teaish Setup + +Follow the same Tcl- and environment-related related setup described +in the first section of this document, up to and including the +installation of Tcl (unless, of course, it was already installed using +those same instructions). + +### 4.2 Teaish Testing + +
      +
    1. `cd $SQLITESOURCE` +
    2. Run either `make snapshot-tarball` or `make amalgamation-tarball` + ↑ + Those steps will leave behind a temp dir called `mkpkg_tmp_dir`, + under which the extension is most readily reached. It can optionally + be extracted from the generated tarball, but that tarball was + generated from this dir, and reusing this dir is a time saver + during development. +
    3. `cd mkpkg_tmp/tea` +
    4. `./configure --with-tcl=$TCLHOME` +
    5. `make test install`
      + ↑ Should run to completion without any errors. +
    6. `make uninstall`
      + ↑ Will uninstall the extension. This _can_ be run + in the same invocation as the `install` target, but only + if the `-j#` make flag is _not_ used. If it is, the + install/uninstall steps will race and make a mess of things. + Parallel builds do not help in this build, anyway, as there's + only a single C file to compile. +
    + +When actively developing and testing the teaish build, which requires +going through the tarball generation, there's a caveat about the +`mkpkg_tmp_dir` dir: it will be deleted every time a tarball is +built, the shell console which is parked in that +directory for testing needs to add `cd $PWD &&` to the start of the +build commands, like: + +> +``` +[user@host:.../mkpkg_tmp_dir/tea]$ \ + cd $PWD && ./configure CFLAGS=-O0 --with-tcl=$TCLHOME \ + && make test install uninstall +``` + +### 4.3 Teaish Cleanup + + +
      +
    1. `rm -rf $TCLHOME` +
    2. `cd $SQLITESOURCE; rm -fr mkpkg_tmp_dir; fossil clean -x` +
    + +[Fossil]: https://fossil-scm.org/home +[tcl-fossil]: https://core.tcl-lang.org/tcl diff --git a/doc/testrunner.md b/doc/testrunner.md new file mode 100644 index 0000000000..d1696e9d1d --- /dev/null +++ b/doc/testrunner.md @@ -0,0 +1,404 @@ + + +# The testrunner.tcl Script + + + + +# 1. Overview + +The testrunner.tcl program is a Tcl script used to run multiple SQLite +tests in parallel, thus reducing testing time on multi-core machines. +It supports the following types of tests: + + * Tcl test scripts. + + * Fuzzcheck tests, including using an external fuzzcheck database. + + * Tests run with `make` commands. Examples: + - `make devtest` + - `make releasetest` + - `make sdevtest` + - `make testrunner` + +The testrunner.tcl program stores output of all tests and builds run in +log file **testrunner.log**, created in the current working directory. +Search this file to find details of errors. Suggested search commands: + + * `grep "^!" testrunner.log` + * `grep failed testrunner.log` + +The testrunner.tcl program also populates SQLite database **testrunner.db**. +This database contains details of all tests run, running and to be run. +A useful query might be: + +``` + SELECT * FROM script WHERE state='failed' +``` + +You can get a summary of errors in a prior run by invoking commands like +these: + +``` + tclsh $(TESTDIR)/testrunner.tcl errors + tclsh $(TESTDIR)/testrunner.tcl errors -v +``` + +Running the command: + +``` + tclsh $(TESTDIR)/testrunner.tcl status +``` + +in the directory containing the testrunner.db database runs various queries +to produce a succinct report on the state of a running testrunner.tcl script. +A good way to keep and eye on test progress is to run either of the two +following commands: + +``` + watch tclsh $(TESTDIR)/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status -d 2 +``` + +Both of the commands above accomplish about the same thing, but the second +one has the advantage of not requiring "watch" to be installed on your +system. + +Sometimes testrunner.tcl uses the `testfixture` binary that it is run with +to run tests (see "Binary Tests" below). Sometimes it builds testfixture and +other binaries in specific configurations to test (see "Source Tests"). + + +# 2. Binary Tests + +The commands described in this section all run various combinations of the Tcl +test scripts using the `testfixture` binary used to run the testrunner.tcl +script (i.e. they do not invoke the compiler to build new binaries, or the +`make` command to run tests that are not Tcl scripts). The procedure to run +these tests is therefore: + + 1. Build the "testfixture" (or "testfixture.exe" for windows) binary using + whatever method seems convenient. + + 2. Test the binary built in step 1 by running testrunner.tcl with it, + perhaps with various options. + +The following sub-sections describe the various options that can be +passed to testrunner.tcl to test binary testfixture builds. + + +## 2.1. Organization of Tcl Tests + +Tcl tests are stored in files that match the pattern *\*.test*. They are +found in both the $TOP/test/ directory, and in the various sub-directories +of the $TOP/ext/ directory of the source tree. Not all *\*.test* files +contain Tcl tests - a handful are Tcl scripts designed to invoke other +*\*.test* files. + +The **veryquick** set of tests is a subset of all Tcl test scripts in the +source tree. In includes most tests, but excludes some that are very slow. +Almost all fault-injection tests (those that test the response of the library +to OOM or IO errors) are excluded. It is defined in source file +*test/permutations.test*. + +The **full** set of tests includes all Tcl test scripts in the source tree. +To run a "full" test is to run all Tcl test scripts that can be found in the +source tree. + +File *permutations.test* defines various test "permutations". A permutation +consists of: + + * A subset of Tcl test scripts, and + + * Runtime configuration to apply before running each test script + (e.g. enabling auto-vacuum, or disable lookaside). + +Running **all** tests is to run all tests in the full test set, plus a dozen +or so permutations. The specific permutations that are run as part of "all" +are defined in file *testrunner_data.tcl*. + + +## 2.2. Commands to Run Tests + +To run the "veryquick" test set, use either of the following: + +``` + ./testfixture $TESTDIR/testrunner.tcl + ./testfixture $TESTDIR/testrunner.tcl veryquick +``` + +To run the "full" test suite: + +``` + ./testfixture $TESTDIR/testrunner.tcl full +``` + +To run the subset of the "full" test suite for which the test file name matches +a specified pattern (e.g. all tests that start with "fts5"), either of: + +``` + ./testfixture $TESTDIR/testrunner.tcl fts5% + ./testfixture $TESTDIR/testrunner.tcl 'fts5*' +``` + +Strictly speaking, for a test to be run the pattern must match the script +filename, not including the directory, using the rules of Tcl's +\[string match\] command. Except that before the matching is done, any "%" +characters specified as part of the pattern are transformed to "\*". + + +To run "all" tests (full + permutations): + +``` + ./testfixture $TESTDIR/testrunner.tcl all +``` + + +## 2.3. Investigating Binary Test Failures + +If a test fails, testrunner.tcl reports name of the Tcl test script and, if +applicable, the name of the permutation, to stdout. This information can also +be retrieved from either *testrunner.log* or *testrunner.db*. + +If there is no permutation, the individual test script may be run with: + +``` + ./testfixture $PATH_TO_SCRIPT +``` + +Or, if the failure occured as part of a permutation: + +``` + ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT +``` + +TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT? + + +# 3. Source Code Tests + +The commands described in this section invoke the C compiler to build +binaries from the source tree, then use those binaries to run Tcl and +other tests. The advantages of this are that: + + * it is possible to test multiple build configurations with a single + command, and + + * it ensures that tests are always run using binaries created with the + same set of compiler options. + +The testrunner.tcl commands described in this section may be run using +either a *testfixture* (or testfixture.exe) build, or with any other Tcl +shell that supports SQLite 3.31.1 or newer via "package require sqlite3". + +TODO: ./configure + Makefile.msc build systems. + + +## 3.1. Commands to Run SQLite Tests + +The **mdevtest** command is equivalent to running the veryquick tests and +the `make fuzztest` target once for each of two --enable-all builds - one +with debugging enabled and one without: + +``` + tclsh $TESTDIR/testrunner.tcl mdevtest +``` + +In other words, it is equivalent to running: + +``` + $TOP/configure --enable-all --enable-debug + make fuzztest + make testfixture + ./testfixture $TOP/test/testrunner.tcl veryquick + + # Then, after removing files created by the tests above: + $TOP/configure --enable-all OPTS="-O0" + make fuzztest + make testfixture + ./testfixture $TOP/test/testrunner.tcl veryquick +``` + +The **sdevtest** command is identical to the mdevtest command, except that the +second of the two builds is a sanitizer build. Specifically, this means that +OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0": + +``` + tclsh $TESTDIR/testrunner.tcl sdevtest +``` + +The **release** command runs lots of tests under lots of builds. It runs +different combinations of builds and tests depending on whether it is run +on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details +of the specific tests run. + +``` + tclsh $TESTDIR/testrunner.tcl release +``` + +As with source code tests, one or more patterns +may be appended to any of the above commands (mdevtest, sdevtest or release). +Pattern matching is used for both Tcl tests and fuzz tests. + +``` + tclsh $TESTDIR/testrunner.tcl release rtree% +``` + + +## 3.2. Running ZipVFS Tests + +testrunner.tcl can build a zipvfs-enabled testfixture and use it to run +tests from the Zipvfs project with the following command: + +``` + tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS +``` + +This can be combined with any of "mdevtest", "sdevtest" or "release" to +test both SQLite and Zipvfs with a single command: + +``` + tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest +``` + + +## 3.3. Investigating Source Code Test Failures + +Investigating a test failure that occurs during source code testing is a +two step process: + + 1. Recreating the build configuration in which the test failed, and + + 2. Re-running the actual test. + +To recreate a build configuration, use the testrunner.tcl **script** command +to create a build script. A build script is a bash script on Linux or OSX, or +a dos \*.bat file on windows. For example: + +``` + # Create a script that recreates build configuration "Device-One" on + # Linux or OSX: + tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh + + # Create a script that recreates build configuration "Have-Not" on Windows: + tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat +``` + +The generated bash or \*.bat file script accepts a single argument - a makefile +target to build. This may be used either to run a `make` command test directly, +or else to build a testfixture (or testfixture.exe) binary with which to +run a Tcl test script, as described above. + + +## 3.4 External Fuzzcheck Databases + +Testrunner.tcl will also run fuzzcheck against an external (out of tree) +database, for example fuzzcheck databases generated by dbsqlfuzz. To do +this, simply add the "`--fuzzdb` *FILENAME*" command-line option or set +the FUZZDB environment variable to the name of the external +database. For large external databases, testrunner.tcl will automatically use +the "`--slice`" command-line option of fuzzcheck to divide the work up into +multiple jobs, to increase parallelism. + +Thus, for example, to run a full releasetest including an external +dbsqlfuzz database, run a command like one of these: + +``` + tclsh test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db + FUZZDB=../fuzz/20250415.db make releasetest + nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest +``` + +The patternlist option to testrunner.tcl will match against fuzzcheck +databases. So if you want to run *only* tests involving the external +database, you can use a command something like this: + +``` + tclsh test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db +``` + + +# 4. Extra testrunner.tcl Options + +The testrunner.tcl script options in this section may be used with both source +code and binary tests. + +The **--buildonly** option instructs testrunner.tcl just to build the binaries +required by a test, not to run any actual tests. For example: + +``` + # Build binaries required by release test. + tclsh $TESTDIR/testrunner.tcl --buildonly release" +``` + +The **--dryrun** option prevents testrunner.tcl from building any binaries +or running any tests. Instead, it just writes the shell commands that it +would normally execute into the testrunner.log file. Example: + +``` + # Log the shell commmands that make up the mdevtest test. + tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" +``` + +The **--explain** option is similar to --dryrun in that it prevents +testrunner.tcl from building any binaries or running any tests. The +difference is that --explain prints on standard output a human-readable +summary of all the builds and tests that would have been run. + +``` + # Show what builds and tests would have been run + tclsh $TESTDIR/testrunner.tcl --explain mdevtest +``` + +The **--status** option uses VT100 escape sequences to display the test +status full-screen. This is similar to running +"`watch test/testrunner status`" in a separate window, just more convenient. +Unfortunately, this option does not work correctly on Windows, due to the +sketchy implementation of VT100 escapes on the Windows console. + + +# 5. Controlling CPU Core Utilization + +When running either binary or source code tests, testrunner.tcl reports the +number of jobs it intends to use to stdout. e.g. + +``` + $ ./testfixture $TESTDIR/testrunner.tcl + splitting work across 16 jobs + ... more output ... +``` + +By default, testfixture.tcl attempts to set the number of jobs to the number +of real cores on the machine. This can be overridden using the "--jobs" (or -j) +switch: + +``` + $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8 + splitting work across 8 jobs + ... more output ... +``` + +The number of jobs may also be changed while an instance of testrunner.tcl is +running by exucuting the following command from the directory containing the +testrunner.log and testrunner.db files: + +``` + $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS +``` diff --git a/doc/vfs-shm.txt b/doc/vfs-shm.txt index c1f125a120..a483e9b159 100644 --- a/doc/vfs-shm.txt +++ b/doc/vfs-shm.txt @@ -1,6 +1,6 @@ The 5 states of an historical rollback lock as implemented by the xLock, xUnlock, and xCheckReservedLock methods of the sqlite3_io_methods -objec are: +object are: UNLOCKED SHARED @@ -58,7 +58,7 @@ The meanings of the various wal-index locking states is as follows: A particular lock manager implementation may coalesce one or more of the wal-index locking states, though with a reduction in concurrency. -For example, an implemention might implement only exclusive locking, +For example, an implementation might implement only exclusive locking, in which case all states would be equivalent to CHECKPOINT, meaning that only one reader or one writer or one checkpointer could be active at a time. Or, an implementation might combine READ and READ_FULL into diff --git a/doc/wal-lock.md b/doc/wal-lock.md index d74bb88b63..8df7cc836c 100644 --- a/doc/wal-lock.md +++ b/doc/wal-lock.md @@ -12,7 +12,7 @@ facilitates transfer of OS priority between processes when a high priority process is blocked by a lower priority one. Only read/write clients use blocking locks. Clients that have read-only access -to the \*-shm file nevery use blocking locks. +to the \*-shm file never use blocking locks. Threads or processes that access a single database at a time never deadlock as a result of blocking database locks. But it is of course possible for threads diff --git a/ext/README.md b/ext/README.md index 933a33d053..78312819ab 100644 --- a/ext/README.md +++ b/ext/README.md @@ -1,6 +1,6 @@ ## Loadable Extensions -Various [loadable extensions](https://www.sqlite.org/loadext.html) for +Various [loadable extensions](https://sqlite.org/loadext.html) for SQLite are found in subfolders. Most subfolders are dedicated to a single loadable extension (for diff --git a/ext/async/README.txt b/ext/async/README.txt deleted file mode 100644 index f62fa2fc17..0000000000 --- a/ext/async/README.txt +++ /dev/null @@ -1,170 +0,0 @@ -NOTE (2012-11-29): - -The functionality implemented by this extension has been superseded -by WAL-mode. This module is no longer supported or maintained. The -code is retained for historical reference only. - ------------------------------------------------------------------------------- - -Normally, when SQLite writes to a database file, it waits until the write -operation is finished before returning control to the calling application. -Since writing to the file-system is usually very slow compared with CPU -bound operations, this can be a performance bottleneck. This directory -contains an extension that causes SQLite to perform all write requests -using a separate thread running in the background. Although this does not -reduce the overall system resources (CPU, disk bandwidth etc.) at all, it -allows SQLite to return control to the caller quickly even when writing to -the database, eliminating the bottleneck. - - 1. Functionality - - 1.1 How it Works - 1.2 Limitations - 1.3 Locking and Concurrency - - 2. Compilation and Usage - - 3. Porting - - - -1. FUNCTIONALITY - - With asynchronous I/O, write requests are handled by a separate thread - running in the background. This means that the thread that initiates - a database write does not have to wait for (sometimes slow) disk I/O - to occur. The write seems to happen very quickly, though in reality - it is happening at its usual slow pace in the background. - - Asynchronous I/O appears to give better responsiveness, but at a price. - You lose the Durable property. With the default I/O backend of SQLite, - once a write completes, you know that the information you wrote is - safely on disk. With the asynchronous I/O, this is not the case. If - your program crashes or if a power loss occurs after the database - write but before the asynchronous write thread has completed, then the - database change might never make it to disk and the next user of the - database might not see your change. - - You lose Durability with asynchronous I/O, but you still retain the - other parts of ACID: Atomic, Consistent, and Isolated. Many - appliations get along fine without the Durablity. - - 1.1 How it Works - - Asynchronous I/O works by creating a special SQLite "vfs" structure - and registering it with sqlite3_vfs_register(). When files opened via - this vfs are written to (using the vfs xWrite() method), the data is not - written directly to disk, but is placed in the "write-queue" to be - handled by the background thread. - - When files opened with the asynchronous vfs are read from - (using the vfs xRead() method), the data is read from the file on - disk and the write-queue, so that from the point of view of - the vfs reader the xWrite() appears to have already completed. - - The special vfs is registered (and unregistered) by calls to the - API functions sqlite3async_initialize() and sqlite3async_shutdown(). - See section "Compilation and Usage" below for details. - - 1.2 Limitations - - In order to gain experience with the main ideas surrounding asynchronous - IO, this implementation is deliberately kept simple. Additional - capabilities may be added in the future. - - For example, as currently implemented, if writes are happening at a - steady stream that exceeds the I/O capability of the background writer - thread, the queue of pending write operations will grow without bound. - If this goes on for long enough, the host system could run out of memory. - A more sophisticated module could to keep track of the quantity of - pending writes and stop accepting new write requests when the queue of - pending writes grows too large. - - 1.3 Locking and Concurrency - - Multiple connections from within a single process that use this - implementation of asynchronous IO may access a single database - file concurrently. From the point of view of the user, if all - connections are from within a single process, there is no difference - between the concurrency offered by "normal" SQLite and SQLite - using the asynchronous backend. - - If file-locking is enabled (it is enabled by default), then connections - from multiple processes may also read and write the database file. - However concurrency is reduced as follows: - - * When a connection using asynchronous IO begins a database - transaction, the database is locked immediately. However the - lock is not released until after all relevant operations - in the write-queue have been flushed to disk. This means - (for example) that the database may remain locked for some - time after a "COMMIT" or "ROLLBACK" is issued. - - * If an application using asynchronous IO executes transactions - in quick succession, other database users may be effectively - locked out of the database. This is because when a BEGIN - is executed, a database lock is established immediately. But - when the corresponding COMMIT or ROLLBACK occurs, the lock - is not released until the relevant part of the write-queue - has been flushed through. As a result, if a COMMIT is followed - by a BEGIN before the write-queue is flushed through, the database - is never unlocked,preventing other processes from accessing - the database. - - File-locking may be disabled at runtime using the sqlite3async_control() - API (see below). This may improve performance when an NFS or other - network file-system, as the synchronous round-trips to the server be - required to establish file locks are avoided. However, if multiple - connections attempt to access the same database file when file-locking - is disabled, application crashes and database corruption is a likely - outcome. - - -2. COMPILATION AND USAGE - - The asynchronous IO extension consists of a single file of C code - (sqlite3async.c), and a header file (sqlite3async.h) that defines the - C API used by applications to activate and control the modules - functionality. - - To use the asynchronous IO extension, compile sqlite3async.c as - part of the application that uses SQLite. Then use the API defined - in sqlite3async.h to initialize and configure the module. - - The asynchronous IO VFS API is described in detail in comments in - sqlite3async.h. Using the API usually consists of the following steps: - - 1. Register the asynchronous IO VFS with SQLite by calling the - sqlite3async_initialize() function. - - 2. Create a background thread to perform write operations and call - sqlite3async_run(). - - 3. Use the normal SQLite API to read and write to databases via - the asynchronous IO VFS. - - Refer to sqlite3async.h for details. - - -3. PORTING - - Currently the asynchronous IO extension is compatible with win32 systems - and systems that support the pthreads interface, including Mac OSX, Linux, - and other varieties of Unix. - - To port the asynchronous IO extension to another platform, the user must - implement mutex and condition variable primitives for the new platform. - Currently there is no externally available interface to allow this, but - modifying the code within sqlite3async.c to include the new platforms - concurrency primitives is relatively easy. Search within sqlite3async.c - for the comment string "PORTING FUNCTIONS" for details. Then implement - new versions of each of the following: - - static void async_mutex_enter(int eMutex); - static void async_mutex_leave(int eMutex); - static void async_cond_wait(int eCond, int eMutex); - static void async_cond_signal(int eCond); - static void async_sched_yield(void); - - The functionality required of each of the above functions is described - in comments in sqlite3async.c. diff --git a/ext/async/sqlite3async.c b/ext/async/sqlite3async.c deleted file mode 100644 index eed7c8d738..0000000000 --- a/ext/async/sqlite3async.c +++ /dev/null @@ -1,1706 +0,0 @@ -/* -** 2005 December 14 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** $Id: sqlite3async.c,v 1.7 2009/07/18 11:52:04 danielk1977 Exp $ -** -** This file contains the implementation of an asynchronous IO backend -** for SQLite. -*/ - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ASYNCIO) - -#include "sqlite3async.h" -#include "sqlite3.h" -#include -#include -#include - -/* Useful macros used in several places */ -#define MIN(x,y) ((x)<(y)?(x):(y)) -#define MAX(x,y) ((x)>(y)?(x):(y)) - -#ifndef SQLITE_AMALGAMATION -/* Macro to mark parameters as unused and silence compiler warnings. */ -#define UNUSED_PARAMETER(x) (void)(x) -#endif - -/* Forward references */ -typedef struct AsyncWrite AsyncWrite; -typedef struct AsyncFile AsyncFile; -typedef struct AsyncFileData AsyncFileData; -typedef struct AsyncFileLock AsyncFileLock; -typedef struct AsyncLock AsyncLock; - -/* Enable for debugging */ -#ifndef NDEBUG -#include -static int sqlite3async_trace = 0; -# define ASYNC_TRACE(X) if( sqlite3async_trace ) asyncTrace X -static void asyncTrace(const char *zFormat, ...){ - char *z; - va_list ap; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - fprintf(stderr, "[%d] %s", 0 /* (int)pthread_self() */, z); - sqlite3_free(z); -} -#else -# define ASYNC_TRACE(X) -#endif - -/* -** THREAD SAFETY NOTES -** -** Basic rules: -** -** * Both read and write access to the global write-op queue must be -** protected by the async.queueMutex. As are the async.ioError and -** async.nFile variables. -** -** * The async.pLock list and all AsyncLock and AsyncFileLock -** structures must be protected by the async.lockMutex mutex. -** -** * The file handles from the underlying system are not assumed to -** be thread safe. -** -** * See the last two paragraphs under "The Writer Thread" for -** an assumption to do with file-handle synchronization by the Os. -** -** Deadlock prevention: -** -** There are three mutex used by the system: the "writer" mutex, -** the "queue" mutex and the "lock" mutex. Rules are: -** -** * It is illegal to block on the writer mutex when any other mutex -** are held, and -** -** * It is illegal to block on the queue mutex when the lock mutex -** is held. -** -** i.e. mutex's must be grabbed in the order "writer", "queue", "lock". -** -** File system operations (invoked by SQLite thread): -** -** xOpen -** xDelete -** xFileExists -** -** File handle operations (invoked by SQLite thread): -** -** asyncWrite, asyncClose, asyncTruncate, asyncSync -** -** The operations above add an entry to the global write-op list. They -** prepare the entry, acquire the async.queueMutex momentarily while -** list pointers are manipulated to insert the new entry, then release -** the mutex and signal the writer thread to wake up in case it happens -** to be asleep. -** -** -** asyncRead, asyncFileSize. -** -** Read operations. Both of these read from both the underlying file -** first then adjust their result based on pending writes in the -** write-op queue. So async.queueMutex is held for the duration -** of these operations to prevent other threads from changing the -** queue in mid operation. -** -** -** asyncLock, asyncUnlock, asyncCheckReservedLock -** -** These primitives implement in-process locking using a hash table -** on the file name. Files are locked correctly for connections coming -** from the same process. But other processes cannot see these locks -** and will therefore not honor them. -** -** -** The writer thread: -** -** The async.writerMutex is used to make sure only there is only -** a single writer thread running at a time. -** -** Inside the writer thread is a loop that works like this: -** -** WHILE (write-op list is not empty) -** Do IO operation at head of write-op list -** Remove entry from head of write-op list -** END WHILE -** -** The async.queueMutex is always held during the test, and when the entry is removed from the head -** of the write-op list. Sometimes it is held for the interim -** period (while the IO is performed), and sometimes it is -** relinquished. It is relinquished if (a) the IO op is an -** ASYNC_CLOSE or (b) when the file handle was opened, two of -** the underlying systems handles were opened on the same -** file-system entry. -** -** If condition (b) above is true, then one file-handle -** (AsyncFile.pBaseRead) is used exclusively by sqlite threads to read the -** file, the other (AsyncFile.pBaseWrite) by sqlite3_async_flush() -** threads to perform write() operations. This means that read -** operations are not blocked by asynchronous writes (although -** asynchronous writes may still be blocked by reads). -** -** This assumes that the OS keeps two handles open on the same file -** properly in sync. That is, any read operation that starts after a -** write operation on the same file system entry has completed returns -** data consistent with the write. We also assume that if one thread -** reads a file while another is writing it all bytes other than the -** ones actually being written contain valid data. -** -** If the above assumptions are not true, set the preprocessor symbol -** SQLITE_ASYNC_TWO_FILEHANDLES to 0. -*/ - - -#ifndef NDEBUG -# define TESTONLY( X ) X -#else -# define TESTONLY( X ) -#endif - -/* -** PORTING FUNCTIONS -** -** There are two definitions of the following functions. One for pthreads -** compatible systems and one for Win32. These functions isolate the OS -** specific code required by each platform. -** -** The system uses three mutexes and a single condition variable. To -** block on a mutex, async_mutex_enter() is called. The parameter passed -** to async_mutex_enter(), which must be one of ASYNC_MUTEX_LOCK, -** ASYNC_MUTEX_QUEUE or ASYNC_MUTEX_WRITER, identifies which of the three -** mutexes to lock. Similarly, to unlock a mutex, async_mutex_leave() is -** called with a parameter identifying the mutex being unlocked. Mutexes -** are not recursive - it is an error to call async_mutex_enter() to -** lock a mutex that is already locked, or to call async_mutex_leave() -** to unlock a mutex that is not currently locked. -** -** The async_cond_wait() and async_cond_signal() functions are modelled -** on the pthreads functions with similar names. The first parameter to -** both functions is always ASYNC_COND_QUEUE. When async_cond_wait() -** is called the mutex identified by the second parameter must be held. -** The mutex is unlocked, and the calling thread simultaneously begins -** waiting for the condition variable to be signalled by another thread. -** After another thread signals the condition variable, the calling -** thread stops waiting, locks mutex eMutex and returns. The -** async_cond_signal() function is used to signal the condition variable. -** It is assumed that the mutex used by the thread calling async_cond_wait() -** is held by the caller of async_cond_signal() (otherwise there would be -** a race condition). -** -** It is guaranteed that no other thread will call async_cond_wait() when -** there is already a thread waiting on the condition variable. -** -** The async_sched_yield() function is called to suggest to the operating -** system that it would be a good time to shift the current thread off the -** CPU. The system will still work if this function is not implemented -** (it is not currently implemented for win32), but it might be marginally -** more efficient if it is. -*/ -static void async_mutex_enter(int eMutex); -static void async_mutex_leave(int eMutex); -static void async_cond_wait(int eCond, int eMutex); -static void async_cond_signal(int eCond); -static void async_sched_yield(void); - -/* -** There are also two definitions of the following. async_os_initialize() -** is called when the asynchronous VFS is first installed, and os_shutdown() -** is called when it is uninstalled (from within sqlite3async_shutdown()). -** -** For pthreads builds, both of these functions are no-ops. For win32, -** they provide an opportunity to initialize and finalize the required -** mutex and condition variables. -** -** If async_os_initialize() returns other than zero, then the initialization -** fails and SQLITE_ERROR is returned to the user. -*/ -static int async_os_initialize(void); -static void async_os_shutdown(void); - -/* Values for use as the 'eMutex' argument of the above functions. The -** integer values assigned to these constants are important for assert() -** statements that verify that mutexes are locked in the correct order. -** Specifically, it is unsafe to try to lock mutex N while holding a lock -** on mutex M if (M<=N). -*/ -#define ASYNC_MUTEX_LOCK 0 -#define ASYNC_MUTEX_QUEUE 1 -#define ASYNC_MUTEX_WRITER 2 - -/* Values for use as the 'eCond' argument of the above functions. */ -#define ASYNC_COND_QUEUE 0 - -/************************************************************************* -** Start of OS specific code. -*/ -#if SQLITE_OS_WIN || defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) - -#include - -/* The following block contains the win32 specific code. */ - -#define mutex_held(X) (GetCurrentThreadId()==primitives.aHolder[X]) - -static struct AsyncPrimitives { - int isInit; - DWORD aHolder[3]; - CRITICAL_SECTION aMutex[3]; - HANDLE aCond[1]; -} primitives = { 0 }; - -static int async_os_initialize(void){ - if( !primitives.isInit ){ - primitives.aCond[0] = CreateEvent(NULL, TRUE, FALSE, 0); - if( primitives.aCond[0]==NULL ){ - return 1; - } - InitializeCriticalSection(&primitives.aMutex[0]); - InitializeCriticalSection(&primitives.aMutex[1]); - InitializeCriticalSection(&primitives.aMutex[2]); - primitives.isInit = 1; - } - return 0; -} -static void async_os_shutdown(void){ - if( primitives.isInit ){ - DeleteCriticalSection(&primitives.aMutex[0]); - DeleteCriticalSection(&primitives.aMutex[1]); - DeleteCriticalSection(&primitives.aMutex[2]); - CloseHandle(primitives.aCond[0]); - primitives.isInit = 0; - } -} - -/* The following block contains the Win32 specific code. */ -static void async_mutex_enter(int eMutex){ - assert( eMutex==0 || eMutex==1 || eMutex==2 ); - assert( eMutex!=2 || (!mutex_held(0) && !mutex_held(1) && !mutex_held(2)) ); - assert( eMutex!=1 || (!mutex_held(0) && !mutex_held(1)) ); - assert( eMutex!=0 || (!mutex_held(0)) ); - EnterCriticalSection(&primitives.aMutex[eMutex]); - TESTONLY( primitives.aHolder[eMutex] = GetCurrentThreadId(); ) -} -static void async_mutex_leave(int eMutex){ - assert( eMutex==0 || eMutex==1 || eMutex==2 ); - assert( mutex_held(eMutex) ); - TESTONLY( primitives.aHolder[eMutex] = 0; ) - LeaveCriticalSection(&primitives.aMutex[eMutex]); -} -static void async_cond_wait(int eCond, int eMutex){ - ResetEvent(primitives.aCond[eCond]); - async_mutex_leave(eMutex); - WaitForSingleObject(primitives.aCond[eCond], INFINITE); - async_mutex_enter(eMutex); -} -static void async_cond_signal(int eCond){ - assert( mutex_held(ASYNC_MUTEX_QUEUE) ); - SetEvent(primitives.aCond[eCond]); -} -static void async_sched_yield(void){ - Sleep(0); -} -#else - -/* The following block contains the pthreads specific code. */ -#include -#include - -#define mutex_held(X) pthread_equal(primitives.aHolder[X], pthread_self()) - -static int async_os_initialize(void) {return 0;} -static void async_os_shutdown(void) {} - -static struct AsyncPrimitives { - pthread_mutex_t aMutex[3]; - pthread_cond_t aCond[1]; - pthread_t aHolder[3]; -} primitives = { - { PTHREAD_MUTEX_INITIALIZER, - PTHREAD_MUTEX_INITIALIZER, - PTHREAD_MUTEX_INITIALIZER - } , { - PTHREAD_COND_INITIALIZER - } , { 0, 0, 0 } -}; - -static void async_mutex_enter(int eMutex){ - assert( eMutex==0 || eMutex==1 || eMutex==2 ); - assert( eMutex!=2 || (!mutex_held(0) && !mutex_held(1) && !mutex_held(2)) ); - assert( eMutex!=1 || (!mutex_held(0) && !mutex_held(1)) ); - assert( eMutex!=0 || (!mutex_held(0)) ); - pthread_mutex_lock(&primitives.aMutex[eMutex]); - TESTONLY( primitives.aHolder[eMutex] = pthread_self(); ) -} -static void async_mutex_leave(int eMutex){ - assert( eMutex==0 || eMutex==1 || eMutex==2 ); - assert( mutex_held(eMutex) ); - TESTONLY( primitives.aHolder[eMutex] = 0; ) - pthread_mutex_unlock(&primitives.aMutex[eMutex]); -} -static void async_cond_wait(int eCond, int eMutex){ - assert( eMutex==0 || eMutex==1 || eMutex==2 ); - assert( mutex_held(eMutex) ); - TESTONLY( primitives.aHolder[eMutex] = 0; ) - pthread_cond_wait(&primitives.aCond[eCond], &primitives.aMutex[eMutex]); - TESTONLY( primitives.aHolder[eMutex] = pthread_self(); ) -} -static void async_cond_signal(int eCond){ - assert( mutex_held(ASYNC_MUTEX_QUEUE) ); - pthread_cond_signal(&primitives.aCond[eCond]); -} -static void async_sched_yield(void){ - sched_yield(); -} -#endif -/* -** End of OS specific code. -*************************************************************************/ - -#define assert_mutex_is_held(X) assert( mutex_held(X) ) - - -#ifndef SQLITE_ASYNC_TWO_FILEHANDLES -/* #define SQLITE_ASYNC_TWO_FILEHANDLES 0 */ -#define SQLITE_ASYNC_TWO_FILEHANDLES 1 -#endif - -/* -** State information is held in the static variable "async" defined -** as the following structure. -** -** Both async.ioError and async.nFile are protected by async.queueMutex. -*/ -static struct TestAsyncStaticData { - AsyncWrite *pQueueFirst; /* Next write operation to be processed */ - AsyncWrite *pQueueLast; /* Last write operation on the list */ - AsyncLock *pLock; /* Linked list of all AsyncLock structures */ - volatile int ioDelay; /* Extra delay between write operations */ - volatile int eHalt; /* One of the SQLITEASYNC_HALT_XXX values */ - volatile int bLockFiles; /* Current value of "lockfiles" parameter */ - int ioError; /* True if an IO error has occurred */ - int nFile; /* Number of open files (from sqlite pov) */ -} async = { 0,0,0,0,0,1,0,0 }; - -/* Possible values of AsyncWrite.op */ -#define ASYNC_NOOP 0 -#define ASYNC_WRITE 1 -#define ASYNC_SYNC 2 -#define ASYNC_TRUNCATE 3 -#define ASYNC_CLOSE 4 -#define ASYNC_DELETE 5 -#define ASYNC_OPENEXCLUSIVE 6 -#define ASYNC_UNLOCK 7 - -/* Names of opcodes. Used for debugging only. -** Make sure these stay in sync with the macros above! -*/ -static const char *azOpcodeName[] = { - "NOOP", "WRITE", "SYNC", "TRUNCATE", "CLOSE", "DELETE", "OPENEX", "UNLOCK" -}; - -/* -** Entries on the write-op queue are instances of the AsyncWrite -** structure, defined here. -** -** The interpretation of the iOffset and nByte variables varies depending -** on the value of AsyncWrite.op: -** -** ASYNC_NOOP: -** No values used. -** -** ASYNC_WRITE: -** iOffset -> Offset in file to write to. -** nByte -> Number of bytes of data to write (pointed to by zBuf). -** -** ASYNC_SYNC: -** nByte -> flags to pass to sqlite3OsSync(). -** -** ASYNC_TRUNCATE: -** iOffset -> Size to truncate file to. -** nByte -> Unused. -** -** ASYNC_CLOSE: -** iOffset -> Unused. -** nByte -> Unused. -** -** ASYNC_DELETE: -** iOffset -> Contains the "syncDir" flag. -** nByte -> Number of bytes of zBuf points to (file name). -** -** ASYNC_OPENEXCLUSIVE: -** iOffset -> Value of "delflag". -** nByte -> Number of bytes of zBuf points to (file name). -** -** ASYNC_UNLOCK: -** nByte -> Argument to sqlite3OsUnlock(). -** -** -** For an ASYNC_WRITE operation, zBuf points to the data to write to the file. -** This space is sqlite3_malloc()d along with the AsyncWrite structure in a -** single blob, so is deleted when sqlite3_free() is called on the parent -** structure. -*/ -struct AsyncWrite { - AsyncFileData *pFileData; /* File to write data to or sync */ - int op; /* One of ASYNC_xxx etc. */ - sqlite_int64 iOffset; /* See above */ - int nByte; /* See above */ - char *zBuf; /* Data to write to file (or NULL if op!=ASYNC_WRITE) */ - AsyncWrite *pNext; /* Next write operation (to any file) */ -}; - -/* -** An instance of this structure is created for each distinct open file -** (i.e. if two handles are opened on the one file, only one of these -** structures is allocated) and stored in the async.aLock hash table. The -** keys for async.aLock are the full pathnames of the opened files. -** -** AsyncLock.pList points to the head of a linked list of AsyncFileLock -** structures, one for each handle currently open on the file. -** -** If the opened file is not a main-database (the SQLITE_OPEN_MAIN_DB is -** not passed to the sqlite3OsOpen() call), or if async.bLockFiles is -** false, variables AsyncLock.pFile and AsyncLock.eLock are never used. -** Otherwise, pFile is a file handle opened on the file in question and -** used to obtain the file-system locks required by database connections -** within this process. -** -** See comments above the asyncLock() function for more details on -** the implementation of database locking used by this backend. -*/ -struct AsyncLock { - char *zFile; - int nFile; - sqlite3_file *pFile; - int eLock; - AsyncFileLock *pList; - AsyncLock *pNext; /* Next in linked list headed by async.pLock */ -}; - -/* -** An instance of the following structure is allocated along with each -** AsyncFileData structure (see AsyncFileData.lock), but is only used if the -** file was opened with the SQLITE_OPEN_MAIN_DB. -*/ -struct AsyncFileLock { - int eLock; /* Internally visible lock state (sqlite pov) */ - int eAsyncLock; /* Lock-state with write-queue unlock */ - AsyncFileLock *pNext; -}; - -/* -** The AsyncFile structure is a subclass of sqlite3_file used for -** asynchronous IO. -** -** All of the actual data for the structure is stored in the structure -** pointed to by AsyncFile.pData, which is allocated as part of the -** sqlite3OsOpen() using sqlite3_malloc(). The reason for this is that the -** lifetime of the AsyncFile structure is ended by the caller after OsClose() -** is called, but the data in AsyncFileData may be required by the -** writer thread after that point. -*/ -struct AsyncFile { - sqlite3_io_methods *pMethod; - AsyncFileData *pData; -}; -struct AsyncFileData { - char *zName; /* Underlying OS filename - used for debugging */ - int nName; /* Number of characters in zName */ - sqlite3_file *pBaseRead; /* Read handle to the underlying Os file */ - sqlite3_file *pBaseWrite; /* Write handle to the underlying Os file */ - AsyncFileLock lock; /* Lock state for this handle */ - AsyncLock *pLock; /* AsyncLock object for this file system entry */ - AsyncWrite closeOp; /* Preallocated close operation */ -}; - -/* -** Add an entry to the end of the global write-op list. pWrite should point -** to an AsyncWrite structure allocated using sqlite3_malloc(). The writer -** thread will call sqlite3_free() to free the structure after the specified -** operation has been completed. -** -** Once an AsyncWrite structure has been added to the list, it becomes the -** property of the writer thread and must not be read or modified by the -** caller. -*/ -static void addAsyncWrite(AsyncWrite *pWrite){ - /* We must hold the queue mutex in order to modify the queue pointers */ - if( pWrite->op!=ASYNC_UNLOCK ){ - async_mutex_enter(ASYNC_MUTEX_QUEUE); - } - - /* Add the record to the end of the write-op queue */ - assert( !pWrite->pNext ); - if( async.pQueueLast ){ - assert( async.pQueueFirst ); - async.pQueueLast->pNext = pWrite; - }else{ - async.pQueueFirst = pWrite; - } - async.pQueueLast = pWrite; - ASYNC_TRACE(("PUSH %p (%s %s %d)\n", pWrite, azOpcodeName[pWrite->op], - pWrite->pFileData ? pWrite->pFileData->zName : "-", pWrite->iOffset)); - - if( pWrite->op==ASYNC_CLOSE ){ - async.nFile--; - } - - /* The writer thread might have been idle because there was nothing - ** on the write-op queue for it to do. So wake it up. */ - async_cond_signal(ASYNC_COND_QUEUE); - - /* Drop the queue mutex */ - if( pWrite->op!=ASYNC_UNLOCK ){ - async_mutex_leave(ASYNC_MUTEX_QUEUE); - } -} - -/* -** Increment async.nFile in a thread-safe manner. -*/ -static void incrOpenFileCount(void){ - /* We must hold the queue mutex in order to modify async.nFile */ - async_mutex_enter(ASYNC_MUTEX_QUEUE); - if( async.nFile==0 ){ - async.ioError = SQLITE_OK; - } - async.nFile++; - async_mutex_leave(ASYNC_MUTEX_QUEUE); -} - -/* -** This is a utility function to allocate and populate a new AsyncWrite -** structure and insert it (via addAsyncWrite() ) into the global list. -*/ -static int addNewAsyncWrite( - AsyncFileData *pFileData, - int op, - sqlite3_int64 iOffset, - int nByte, - const char *zByte -){ - AsyncWrite *p; - if( op!=ASYNC_CLOSE && async.ioError ){ - return async.ioError; - } - p = sqlite3_malloc(sizeof(AsyncWrite) + (zByte?nByte:0)); - if( !p ){ - /* The upper layer does not expect operations like OsWrite() to - ** return SQLITE_NOMEM. This is partly because under normal conditions - ** SQLite is required to do rollback without calling malloc(). So - ** if malloc() fails here, treat it as an I/O error. The above - ** layer knows how to handle that. - */ - return SQLITE_IOERR; - } - p->op = op; - p->iOffset = iOffset; - p->nByte = nByte; - p->pFileData = pFileData; - p->pNext = 0; - if( zByte ){ - p->zBuf = (char *)&p[1]; - memcpy(p->zBuf, zByte, nByte); - }else{ - p->zBuf = 0; - } - addAsyncWrite(p); - return SQLITE_OK; -} - -/* -** Close the file. This just adds an entry to the write-op list, the file is -** not actually closed. -*/ -static int asyncClose(sqlite3_file *pFile){ - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - - /* Unlock the file, if it is locked */ - async_mutex_enter(ASYNC_MUTEX_LOCK); - p->lock.eLock = 0; - async_mutex_leave(ASYNC_MUTEX_LOCK); - - addAsyncWrite(&p->closeOp); - return SQLITE_OK; -} - -/* -** Implementation of sqlite3OsWrite() for asynchronous files. Instead of -** writing to the underlying file, this function adds an entry to the end of -** the global AsyncWrite list. Either SQLITE_OK or SQLITE_NOMEM may be -** returned. -*/ -static int asyncWrite( - sqlite3_file *pFile, - const void *pBuf, - int amt, - sqlite3_int64 iOff -){ - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - return addNewAsyncWrite(p, ASYNC_WRITE, iOff, amt, pBuf); -} - -/* -** Read data from the file. First we read from the filesystem, then adjust -** the contents of the buffer based on ASYNC_WRITE operations in the -** write-op queue. -** -** This method holds the mutex from start to finish. -*/ -static int asyncRead( - sqlite3_file *pFile, - void *zOut, - int iAmt, - sqlite3_int64 iOffset -){ - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - int rc = SQLITE_OK; - sqlite3_int64 filesize = 0; - sqlite3_file *pBase = p->pBaseRead; - sqlite3_int64 iAmt64 = (sqlite3_int64)iAmt; - - /* Grab the write queue mutex for the duration of the call */ - async_mutex_enter(ASYNC_MUTEX_QUEUE); - - /* If an I/O error has previously occurred in this virtual file - ** system, then all subsequent operations fail. - */ - if( async.ioError!=SQLITE_OK ){ - rc = async.ioError; - goto asyncread_out; - } - - if( pBase->pMethods ){ - sqlite3_int64 nRead; - rc = pBase->pMethods->xFileSize(pBase, &filesize); - if( rc!=SQLITE_OK ){ - goto asyncread_out; - } - nRead = MIN(filesize - iOffset, iAmt64); - if( nRead>0 ){ - rc = pBase->pMethods->xRead(pBase, zOut, (int)nRead, iOffset); - ASYNC_TRACE(("READ %s %d bytes at %d\n", p->zName, nRead, iOffset)); - } - } - - if( rc==SQLITE_OK ){ - AsyncWrite *pWrite; - char *zName = p->zName; - - for(pWrite=async.pQueueFirst; pWrite; pWrite = pWrite->pNext){ - if( pWrite->op==ASYNC_WRITE && ( - (pWrite->pFileData==p) || - (zName && pWrite->pFileData->zName==zName) - )){ - sqlite3_int64 nCopy; - sqlite3_int64 nByte64 = (sqlite3_int64)pWrite->nByte; - - /* Set variable iBeginIn to the offset in buffer pWrite->zBuf[] from - ** which data should be copied. Set iBeginOut to the offset within - ** the output buffer to which data should be copied. If either of - ** these offsets is a negative number, set them to 0. - */ - sqlite3_int64 iBeginOut = (pWrite->iOffset-iOffset); - sqlite3_int64 iBeginIn = -iBeginOut; - if( iBeginIn<0 ) iBeginIn = 0; - if( iBeginOut<0 ) iBeginOut = 0; - - filesize = MAX(filesize, pWrite->iOffset+nByte64); - - nCopy = MIN(nByte64-iBeginIn, iAmt64-iBeginOut); - if( nCopy>0 ){ - memcpy(&((char *)zOut)[iBeginOut], &pWrite->zBuf[iBeginIn], (size_t)nCopy); - ASYNC_TRACE(("OVERREAD %d bytes at %d\n", nCopy, iBeginOut+iOffset)); - } - } - } - } - -asyncread_out: - async_mutex_leave(ASYNC_MUTEX_QUEUE); - if( rc==SQLITE_OK && filesize<(iOffset+iAmt) ){ - rc = SQLITE_IOERR_SHORT_READ; - } - return rc; -} - -/* -** Truncate the file to nByte bytes in length. This just adds an entry to -** the write-op list, no IO actually takes place. -*/ -static int asyncTruncate(sqlite3_file *pFile, sqlite3_int64 nByte){ - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - return addNewAsyncWrite(p, ASYNC_TRUNCATE, nByte, 0, 0); -} - -/* -** Sync the file. This just adds an entry to the write-op list, the -** sync() is done later by sqlite3_async_flush(). -*/ -static int asyncSync(sqlite3_file *pFile, int flags){ - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - return addNewAsyncWrite(p, ASYNC_SYNC, 0, flags, 0); -} - -/* -** Read the size of the file. First we read the size of the file system -** entry, then adjust for any ASYNC_WRITE or ASYNC_TRUNCATE operations -** currently in the write-op list. -** -** This method holds the mutex from start to finish. -*/ -int asyncFileSize(sqlite3_file *pFile, sqlite3_int64 *piSize){ - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - int rc = SQLITE_OK; - sqlite3_int64 s = 0; - sqlite3_file *pBase; - - async_mutex_enter(ASYNC_MUTEX_QUEUE); - - /* Read the filesystem size from the base file. If pMethods is NULL, this - ** means the file hasn't been opened yet. In this case all relevant data - ** must be in the write-op queue anyway, so we can omit reading from the - ** file-system. - */ - pBase = p->pBaseRead; - if( pBase->pMethods ){ - rc = pBase->pMethods->xFileSize(pBase, &s); - } - - if( rc==SQLITE_OK ){ - AsyncWrite *pWrite; - for(pWrite=async.pQueueFirst; pWrite; pWrite = pWrite->pNext){ - if( pWrite->op==ASYNC_DELETE - && p->zName - && strcmp(p->zName, pWrite->zBuf)==0 - ){ - s = 0; - }else if( pWrite->pFileData && ( - (pWrite->pFileData==p) - || (p->zName && pWrite->pFileData->zName==p->zName) - )){ - switch( pWrite->op ){ - case ASYNC_WRITE: - s = MAX(pWrite->iOffset + (sqlite3_int64)(pWrite->nByte), s); - break; - case ASYNC_TRUNCATE: - s = MIN(s, pWrite->iOffset); - break; - } - } - } - *piSize = s; - } - async_mutex_leave(ASYNC_MUTEX_QUEUE); - return rc; -} - -/* -** Lock or unlock the actual file-system entry. -*/ -static int getFileLock(AsyncLock *pLock){ - int rc = SQLITE_OK; - AsyncFileLock *pIter; - int eRequired = 0; - - if( pLock->pFile ){ - for(pIter=pLock->pList; pIter; pIter=pIter->pNext){ - assert(pIter->eAsyncLock>=pIter->eLock); - if( pIter->eAsyncLock>eRequired ){ - eRequired = pIter->eAsyncLock; - assert(eRequired>=0 && eRequired<=SQLITE_LOCK_EXCLUSIVE); - } - } - - if( eRequired>pLock->eLock ){ - rc = pLock->pFile->pMethods->xLock(pLock->pFile, eRequired); - if( rc==SQLITE_OK ){ - pLock->eLock = eRequired; - } - } - else if( eRequiredeLock && eRequired<=SQLITE_LOCK_SHARED ){ - rc = pLock->pFile->pMethods->xUnlock(pLock->pFile, eRequired); - if( rc==SQLITE_OK ){ - pLock->eLock = eRequired; - } - } - } - - return rc; -} - -/* -** Return the AsyncLock structure from the global async.pLock list -** associated with the file-system entry identified by path zName -** (a string of nName bytes). If no such structure exists, return 0. -*/ -static AsyncLock *findLock(const char *zName, int nName){ - AsyncLock *p = async.pLock; - while( p && (p->nFile!=nName || memcmp(p->zFile, zName, nName)) ){ - p = p->pNext; - } - return p; -} - -/* -** The following two methods - asyncLock() and asyncUnlock() - are used -** to obtain and release locks on database files opened with the -** asynchronous backend. -*/ -static int asyncLock(sqlite3_file *pFile, int eLock){ - int rc = SQLITE_OK; - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - - if( p->zName ){ - async_mutex_enter(ASYNC_MUTEX_LOCK); - if( p->lock.eLockpLock; - AsyncFileLock *pIter; - assert(pLock && pLock->pList); - for(pIter=pLock->pList; pIter; pIter=pIter->pNext){ - if( pIter!=&p->lock && ( - (eLock==SQLITE_LOCK_EXCLUSIVE && pIter->eLock>=SQLITE_LOCK_SHARED) || - (eLock==SQLITE_LOCK_PENDING && pIter->eLock>=SQLITE_LOCK_RESERVED) || - (eLock==SQLITE_LOCK_RESERVED && pIter->eLock>=SQLITE_LOCK_RESERVED) || - (eLock==SQLITE_LOCK_SHARED && pIter->eLock>=SQLITE_LOCK_PENDING) - )){ - rc = SQLITE_BUSY; - } - } - if( rc==SQLITE_OK ){ - p->lock.eLock = eLock; - p->lock.eAsyncLock = MAX(p->lock.eAsyncLock, eLock); - } - assert(p->lock.eAsyncLock>=p->lock.eLock); - if( rc==SQLITE_OK ){ - rc = getFileLock(pLock); - } - } - async_mutex_leave(ASYNC_MUTEX_LOCK); - } - - ASYNC_TRACE(("LOCK %d (%s) rc=%d\n", eLock, p->zName, rc)); - return rc; -} -static int asyncUnlock(sqlite3_file *pFile, int eLock){ - int rc = SQLITE_OK; - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - if( p->zName ){ - AsyncFileLock *pLock = &p->lock; - async_mutex_enter(ASYNC_MUTEX_QUEUE); - async_mutex_enter(ASYNC_MUTEX_LOCK); - pLock->eLock = MIN(pLock->eLock, eLock); - rc = addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0); - async_mutex_leave(ASYNC_MUTEX_LOCK); - async_mutex_leave(ASYNC_MUTEX_QUEUE); - } - return rc; -} - -/* -** This function is called when the pager layer first opens a database file -** and is checking for a hot-journal. -*/ -static int asyncCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - int ret = 0; - AsyncFileLock *pIter; - AsyncFileData *p = ((AsyncFile *)pFile)->pData; - - async_mutex_enter(ASYNC_MUTEX_LOCK); - for(pIter=p->pLock->pList; pIter; pIter=pIter->pNext){ - if( pIter->eLock>=SQLITE_LOCK_RESERVED ){ - ret = 1; - break; - } - } - async_mutex_leave(ASYNC_MUTEX_LOCK); - - ASYNC_TRACE(("CHECK-LOCK %d (%s)\n", ret, p->zName)); - *pResOut = ret; - return SQLITE_OK; -} - -/* -** sqlite3_file_control() implementation. -*/ -static int asyncFileControl(sqlite3_file *id, int op, void *pArg){ - switch( op ){ - case SQLITE_FCNTL_LOCKSTATE: { - async_mutex_enter(ASYNC_MUTEX_LOCK); - *(int*)pArg = ((AsyncFile*)id)->pData->lock.eLock; - async_mutex_leave(ASYNC_MUTEX_LOCK); - return SQLITE_OK; - } - } - return SQLITE_NOTFOUND; -} - -/* -** Return the device characteristics and sector-size of the device. It -** is tricky to implement these correctly, as this backend might -** not have an open file handle at this point. -*/ -static int asyncSectorSize(sqlite3_file *pFile){ - UNUSED_PARAMETER(pFile); - return 512; -} -static int asyncDeviceCharacteristics(sqlite3_file *pFile){ - UNUSED_PARAMETER(pFile); - return 0; -} - -static int unlinkAsyncFile(AsyncFileData *pData){ - AsyncFileLock **ppIter; - int rc = SQLITE_OK; - - if( pData->zName ){ - AsyncLock *pLock = pData->pLock; - for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){ - if( (*ppIter)==&pData->lock ){ - *ppIter = pData->lock.pNext; - break; - } - } - if( !pLock->pList ){ - AsyncLock **pp; - if( pLock->pFile ){ - pLock->pFile->pMethods->xClose(pLock->pFile); - } - for(pp=&async.pLock; *pp!=pLock; pp=&((*pp)->pNext)); - *pp = pLock->pNext; - sqlite3_free(pLock); - }else{ - rc = getFileLock(pLock); - } - } - - return rc; -} - -/* -** The parameter passed to this function is a copy of a 'flags' parameter -** passed to this modules xOpen() method. This function returns true -** if the file should be opened asynchronously, or false if it should -** be opened immediately. -** -** If the file is to be opened asynchronously, then asyncOpen() will add -** an entry to the event queue and the file will not actually be opened -** until the event is processed. Otherwise, the file is opened directly -** by the caller. -*/ -static int doAsynchronousOpen(int flags){ - return (flags&SQLITE_OPEN_CREATE) && ( - (flags&SQLITE_OPEN_MAIN_JOURNAL) || - (flags&SQLITE_OPEN_TEMP_JOURNAL) || - (flags&SQLITE_OPEN_DELETEONCLOSE) - ); -} - -/* -** Open a file. -*/ -static int asyncOpen( - sqlite3_vfs *pAsyncVfs, - const char *zName, - sqlite3_file *pFile, - int flags, - int *pOutFlags -){ - static sqlite3_io_methods async_methods = { - 1, /* iVersion */ - asyncClose, /* xClose */ - asyncRead, /* xRead */ - asyncWrite, /* xWrite */ - asyncTruncate, /* xTruncate */ - asyncSync, /* xSync */ - asyncFileSize, /* xFileSize */ - asyncLock, /* xLock */ - asyncUnlock, /* xUnlock */ - asyncCheckReservedLock, /* xCheckReservedLock */ - asyncFileControl, /* xFileControl */ - asyncSectorSize, /* xSectorSize */ - asyncDeviceCharacteristics /* xDeviceCharacteristics */ - }; - - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - AsyncFile *p = (AsyncFile *)pFile; - int nName = 0; - int rc = SQLITE_OK; - int nByte; - AsyncFileData *pData; - AsyncLock *pLock = 0; - char *z; - int isAsyncOpen = doAsynchronousOpen(flags); - - /* If zName is NULL, then the upper layer is requesting an anonymous file. - ** Otherwise, allocate enough space to make a copy of the file name (along - ** with the second nul-terminator byte required by xOpen). - */ - if( zName ){ - nName = (int)strlen(zName); - } - - nByte = ( - sizeof(AsyncFileData) + /* AsyncFileData structure */ - 2 * pVfs->szOsFile + /* AsyncFileData.pBaseRead and pBaseWrite */ - nName + 2 /* AsyncFileData.zName */ - ); - z = sqlite3_malloc(nByte); - if( !z ){ - return SQLITE_NOMEM; - } - memset(z, 0, nByte); - pData = (AsyncFileData*)z; - z += sizeof(pData[0]); - pData->pBaseRead = (sqlite3_file*)z; - z += pVfs->szOsFile; - pData->pBaseWrite = (sqlite3_file*)z; - pData->closeOp.pFileData = pData; - pData->closeOp.op = ASYNC_CLOSE; - - if( zName ){ - z += pVfs->szOsFile; - pData->zName = z; - pData->nName = nName; - memcpy(pData->zName, zName, nName); - } - - if( !isAsyncOpen ){ - int flagsout; - rc = pVfs->xOpen(pVfs, pData->zName, pData->pBaseRead, flags, &flagsout); - if( rc==SQLITE_OK - && (flagsout&SQLITE_OPEN_READWRITE) - && (flags&SQLITE_OPEN_EXCLUSIVE)==0 - ){ - rc = pVfs->xOpen(pVfs, pData->zName, pData->pBaseWrite, flags, 0); - } - if( pOutFlags ){ - *pOutFlags = flagsout; - } - } - - async_mutex_enter(ASYNC_MUTEX_LOCK); - - if( zName && rc==SQLITE_OK ){ - pLock = findLock(pData->zName, pData->nName); - if( !pLock ){ - int nByte = pVfs->szOsFile + sizeof(AsyncLock) + pData->nName + 1; - pLock = (AsyncLock *)sqlite3_malloc(nByte); - if( pLock ){ - memset(pLock, 0, nByte); - if( async.bLockFiles && (flags&SQLITE_OPEN_MAIN_DB) ){ - pLock->pFile = (sqlite3_file *)&pLock[1]; - rc = pVfs->xOpen(pVfs, pData->zName, pLock->pFile, flags, 0); - if( rc!=SQLITE_OK ){ - sqlite3_free(pLock); - pLock = 0; - } - } - if( pLock ){ - pLock->nFile = pData->nName; - pLock->zFile = &((char *)(&pLock[1]))[pVfs->szOsFile]; - memcpy(pLock->zFile, pData->zName, pLock->nFile); - pLock->pNext = async.pLock; - async.pLock = pLock; - } - }else{ - rc = SQLITE_NOMEM; - } - } - } - - if( rc==SQLITE_OK ){ - p->pMethod = &async_methods; - p->pData = pData; - - /* Link AsyncFileData.lock into the linked list of - ** AsyncFileLock structures for this file. - */ - if( zName ){ - pData->lock.pNext = pLock->pList; - pLock->pList = &pData->lock; - pData->zName = pLock->zFile; - } - }else{ - if( pData->pBaseRead->pMethods ){ - pData->pBaseRead->pMethods->xClose(pData->pBaseRead); - } - if( pData->pBaseWrite->pMethods ){ - pData->pBaseWrite->pMethods->xClose(pData->pBaseWrite); - } - sqlite3_free(pData); - } - - async_mutex_leave(ASYNC_MUTEX_LOCK); - - if( rc==SQLITE_OK ){ - pData->pLock = pLock; - } - - if( rc==SQLITE_OK && isAsyncOpen ){ - rc = addNewAsyncWrite(pData, ASYNC_OPENEXCLUSIVE, (sqlite3_int64)flags,0,0); - if( rc==SQLITE_OK ){ - if( pOutFlags ) *pOutFlags = flags; - }else{ - async_mutex_enter(ASYNC_MUTEX_LOCK); - unlinkAsyncFile(pData); - async_mutex_leave(ASYNC_MUTEX_LOCK); - sqlite3_free(pData); - } - } - if( rc!=SQLITE_OK ){ - p->pMethod = 0; - }else{ - incrOpenFileCount(); - } - - return rc; -} - -/* -** Implementation of sqlite3OsDelete. Add an entry to the end of the -** write-op queue to perform the delete. -*/ -static int asyncDelete(sqlite3_vfs *pAsyncVfs, const char *z, int syncDir){ - UNUSED_PARAMETER(pAsyncVfs); - return addNewAsyncWrite(0, ASYNC_DELETE, syncDir, (int)strlen(z)+1, z); -} - -/* -** Implementation of sqlite3OsAccess. This method holds the mutex from -** start to finish. -*/ -static int asyncAccess( - sqlite3_vfs *pAsyncVfs, - const char *zName, - int flags, - int *pResOut -){ - int rc; - int ret; - AsyncWrite *p; - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - - assert(flags==SQLITE_ACCESS_READWRITE - || flags==SQLITE_ACCESS_READ - || flags==SQLITE_ACCESS_EXISTS - ); - - async_mutex_enter(ASYNC_MUTEX_QUEUE); - rc = pVfs->xAccess(pVfs, zName, flags, &ret); - if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){ - for(p=async.pQueueFirst; p; p = p->pNext){ - if( p->op==ASYNC_DELETE && 0==strcmp(p->zBuf, zName) ){ - ret = 0; - }else if( p->op==ASYNC_OPENEXCLUSIVE - && p->pFileData->zName - && 0==strcmp(p->pFileData->zName, zName) - ){ - ret = 1; - } - } - } - ASYNC_TRACE(("ACCESS(%s): %s = %d\n", - flags==SQLITE_ACCESS_READWRITE?"read-write": - flags==SQLITE_ACCESS_READ?"read":"exists" - , zName, ret) - ); - async_mutex_leave(ASYNC_MUTEX_QUEUE); - *pResOut = ret; - return rc; -} - -/* -** Fill in zPathOut with the full path to the file identified by zPath. -*/ -static int asyncFullPathname( - sqlite3_vfs *pAsyncVfs, - const char *zPath, - int nPathOut, - char *zPathOut -){ - int rc; - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - rc = pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); - - /* Because of the way intra-process file locking works, this backend - ** needs to return a canonical path. The following block assumes the - ** file-system uses unix style paths. - */ - if( rc==SQLITE_OK ){ - int i, j; - char *z = zPathOut; - int n = (int)strlen(z); - while( n>1 && z[n-1]=='/' ){ n--; } - for(i=j=0; i0 && z[j-1]!='/' ){ j--; } - if( j>0 ){ j--; } - i += 2; - continue; - } - } - z[j++] = z[i]; - } - z[j] = 0; - } - - return rc; -} -static void *asyncDlOpen(sqlite3_vfs *pAsyncVfs, const char *zPath){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - return pVfs->xDlOpen(pVfs, zPath); -} -static void asyncDlError(sqlite3_vfs *pAsyncVfs, int nByte, char *zErrMsg){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - pVfs->xDlError(pVfs, nByte, zErrMsg); -} -static void (*asyncDlSym( - sqlite3_vfs *pAsyncVfs, - void *pHandle, - const char *zSymbol -))(void){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - return pVfs->xDlSym(pVfs, pHandle, zSymbol); -} -static void asyncDlClose(sqlite3_vfs *pAsyncVfs, void *pHandle){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - pVfs->xDlClose(pVfs, pHandle); -} -static int asyncRandomness(sqlite3_vfs *pAsyncVfs, int nByte, char *zBufOut){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - return pVfs->xRandomness(pVfs, nByte, zBufOut); -} -static int asyncSleep(sqlite3_vfs *pAsyncVfs, int nMicro){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - return pVfs->xSleep(pVfs, nMicro); -} -static int asyncCurrentTime(sqlite3_vfs *pAsyncVfs, double *pTimeOut){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)pAsyncVfs->pAppData; - return pVfs->xCurrentTime(pVfs, pTimeOut); -} - -static sqlite3_vfs async_vfs = { - 1, /* iVersion */ - sizeof(AsyncFile), /* szOsFile */ - 0, /* mxPathname */ - 0, /* pNext */ - SQLITEASYNC_VFSNAME, /* zName */ - 0, /* pAppData */ - asyncOpen, /* xOpen */ - asyncDelete, /* xDelete */ - asyncAccess, /* xAccess */ - asyncFullPathname, /* xFullPathname */ - asyncDlOpen, /* xDlOpen */ - asyncDlError, /* xDlError */ - asyncDlSym, /* xDlSym */ - asyncDlClose, /* xDlClose */ - asyncRandomness, /* xDlError */ - asyncSleep, /* xDlSym */ - asyncCurrentTime /* xDlClose */ -}; - -/* -** This procedure runs in a separate thread, reading messages off of the -** write queue and processing them one by one. -** -** If async.writerHaltNow is true, then this procedure exits -** after processing a single message. -** -** If async.writerHaltWhenIdle is true, then this procedure exits when -** the write queue is empty. -** -** If both of the above variables are false, this procedure runs -** indefinately, waiting for operations to be added to the write queue -** and processing them in the order in which they arrive. -** -** An artifical delay of async.ioDelay milliseconds is inserted before -** each write operation in order to simulate the effect of a slow disk. -** -** Only one instance of this procedure may be running at a time. -*/ -static void asyncWriterThread(void){ - sqlite3_vfs *pVfs = (sqlite3_vfs *)(async_vfs.pAppData); - AsyncWrite *p = 0; - int rc = SQLITE_OK; - int holdingMutex = 0; - - async_mutex_enter(ASYNC_MUTEX_WRITER); - - while( async.eHalt!=SQLITEASYNC_HALT_NOW ){ - int doNotFree = 0; - sqlite3_file *pBase = 0; - - if( !holdingMutex ){ - async_mutex_enter(ASYNC_MUTEX_QUEUE); - } - while( (p = async.pQueueFirst)==0 ){ - if( async.eHalt!=SQLITEASYNC_HALT_NEVER ){ - async_mutex_leave(ASYNC_MUTEX_QUEUE); - break; - }else{ - ASYNC_TRACE(("IDLE\n")); - async_cond_wait(ASYNC_COND_QUEUE, ASYNC_MUTEX_QUEUE); - ASYNC_TRACE(("WAKEUP\n")); - } - } - if( p==0 ) break; - holdingMutex = 1; - - /* Right now this thread is holding the mutex on the write-op queue. - ** Variable 'p' points to the first entry in the write-op queue. In - ** the general case, we hold on to the mutex for the entire body of - ** the loop. - ** - ** However in the cases enumerated below, we relinquish the mutex, - ** perform the IO, and then re-request the mutex before removing 'p' from - ** the head of the write-op queue. The idea is to increase concurrency with - ** sqlite threads. - ** - ** * An ASYNC_CLOSE operation. - ** * An ASYNC_OPENEXCLUSIVE operation. For this one, we relinquish - ** the mutex, call the underlying xOpenExclusive() function, then - ** re-aquire the mutex before seting the AsyncFile.pBaseRead - ** variable. - ** * ASYNC_SYNC and ASYNC_WRITE operations, if - ** SQLITE_ASYNC_TWO_FILEHANDLES was set at compile time and two - ** file-handles are open for the particular file being "synced". - */ - if( async.ioError!=SQLITE_OK && p->op!=ASYNC_CLOSE ){ - p->op = ASYNC_NOOP; - } - if( p->pFileData ){ - pBase = p->pFileData->pBaseWrite; - if( - p->op==ASYNC_CLOSE || - p->op==ASYNC_OPENEXCLUSIVE || - (pBase->pMethods && (p->op==ASYNC_SYNC || p->op==ASYNC_WRITE) ) - ){ - async_mutex_leave(ASYNC_MUTEX_QUEUE); - holdingMutex = 0; - } - if( !pBase->pMethods ){ - pBase = p->pFileData->pBaseRead; - } - } - - switch( p->op ){ - case ASYNC_NOOP: - break; - - case ASYNC_WRITE: - assert( pBase ); - ASYNC_TRACE(("WRITE %s %d bytes at %d\n", - p->pFileData->zName, p->nByte, p->iOffset)); - rc = pBase->pMethods->xWrite(pBase, (void *)(p->zBuf), p->nByte, p->iOffset); - break; - - case ASYNC_SYNC: - assert( pBase ); - ASYNC_TRACE(("SYNC %s\n", p->pFileData->zName)); - rc = pBase->pMethods->xSync(pBase, p->nByte); - break; - - case ASYNC_TRUNCATE: - assert( pBase ); - ASYNC_TRACE(("TRUNCATE %s to %d bytes\n", - p->pFileData->zName, p->iOffset)); - rc = pBase->pMethods->xTruncate(pBase, p->iOffset); - break; - - case ASYNC_CLOSE: { - AsyncFileData *pData = p->pFileData; - ASYNC_TRACE(("CLOSE %s\n", p->pFileData->zName)); - if( pData->pBaseWrite->pMethods ){ - pData->pBaseWrite->pMethods->xClose(pData->pBaseWrite); - } - if( pData->pBaseRead->pMethods ){ - pData->pBaseRead->pMethods->xClose(pData->pBaseRead); - } - - /* Unlink AsyncFileData.lock from the linked list of AsyncFileLock - ** structures for this file. Obtain the async.lockMutex mutex - ** before doing so. - */ - async_mutex_enter(ASYNC_MUTEX_LOCK); - rc = unlinkAsyncFile(pData); - async_mutex_leave(ASYNC_MUTEX_LOCK); - - if( !holdingMutex ){ - async_mutex_enter(ASYNC_MUTEX_QUEUE); - holdingMutex = 1; - } - assert_mutex_is_held(ASYNC_MUTEX_QUEUE); - async.pQueueFirst = p->pNext; - sqlite3_free(pData); - doNotFree = 1; - break; - } - - case ASYNC_UNLOCK: { - AsyncWrite *pIter; - AsyncFileData *pData = p->pFileData; - int eLock = p->nByte; - - /* When a file is locked by SQLite using the async backend, it is - ** locked within the 'real' file-system synchronously. When it is - ** unlocked, an ASYNC_UNLOCK event is added to the write-queue to - ** unlock the file asynchronously. The design of the async backend - ** requires that the 'real' file-system file be locked from the - ** time that SQLite first locks it (and probably reads from it) - ** until all asynchronous write events that were scheduled before - ** SQLite unlocked the file have been processed. - ** - ** This is more complex if SQLite locks and unlocks the file multiple - ** times in quick succession. For example, if SQLite does: - ** - ** lock, write, unlock, lock, write, unlock - ** - ** Each "lock" operation locks the file immediately. Each "write" - ** and "unlock" operation adds an event to the event queue. If the - ** second "lock" operation is performed before the first "unlock" - ** operation has been processed asynchronously, then the first - ** "unlock" cannot be safely processed as is, since this would mean - ** the file was unlocked when the second "write" operation is - ** processed. To work around this, when processing an ASYNC_UNLOCK - ** operation, SQLite: - ** - ** 1) Unlocks the file to the minimum of the argument passed to - ** the xUnlock() call and the current lock from SQLite's point - ** of view, and - ** - ** 2) Only unlocks the file at all if this event is the last - ** ASYNC_UNLOCK event on this file in the write-queue. - */ - assert( holdingMutex==1 ); - assert( async.pQueueFirst==p ); - for(pIter=async.pQueueFirst->pNext; pIter; pIter=pIter->pNext){ - if( pIter->pFileData==pData && pIter->op==ASYNC_UNLOCK ) break; - } - if( !pIter ){ - async_mutex_enter(ASYNC_MUTEX_LOCK); - pData->lock.eAsyncLock = MIN( - pData->lock.eAsyncLock, MAX(pData->lock.eLock, eLock) - ); - assert(pData->lock.eAsyncLock>=pData->lock.eLock); - rc = getFileLock(pData->pLock); - async_mutex_leave(ASYNC_MUTEX_LOCK); - } - break; - } - - case ASYNC_DELETE: - ASYNC_TRACE(("DELETE %s\n", p->zBuf)); - rc = pVfs->xDelete(pVfs, p->zBuf, (int)p->iOffset); - if( rc==SQLITE_IOERR_DELETE_NOENT ) rc = SQLITE_OK; - break; - - case ASYNC_OPENEXCLUSIVE: { - int flags = (int)p->iOffset; - AsyncFileData *pData = p->pFileData; - ASYNC_TRACE(("OPEN %s flags=%d\n", p->zBuf, (int)p->iOffset)); - assert(pData->pBaseRead->pMethods==0 && pData->pBaseWrite->pMethods==0); - rc = pVfs->xOpen(pVfs, pData->zName, pData->pBaseRead, flags, 0); - assert( holdingMutex==0 ); - async_mutex_enter(ASYNC_MUTEX_QUEUE); - holdingMutex = 1; - break; - } - - default: assert(!"Illegal value for AsyncWrite.op"); - } - - /* If we didn't hang on to the mutex during the IO op, obtain it now - ** so that the AsyncWrite structure can be safely removed from the - ** global write-op queue. - */ - if( !holdingMutex ){ - async_mutex_enter(ASYNC_MUTEX_QUEUE); - holdingMutex = 1; - } - /* ASYNC_TRACE(("UNLINK %p\n", p)); */ - if( p==async.pQueueLast ){ - async.pQueueLast = 0; - } - if( !doNotFree ){ - assert_mutex_is_held(ASYNC_MUTEX_QUEUE); - async.pQueueFirst = p->pNext; - sqlite3_free(p); - } - assert( holdingMutex ); - - /* An IO error has occurred. We cannot report the error back to the - ** connection that requested the I/O since the error happened - ** asynchronously. The connection has already moved on. There - ** really is nobody to report the error to. - ** - ** The file for which the error occurred may have been a database or - ** journal file. Regardless, none of the currently queued operations - ** associated with the same database should now be performed. Nor should - ** any subsequently requested IO on either a database or journal file - ** handle for the same database be accepted until the main database - ** file handle has been closed and reopened. - ** - ** Furthermore, no further IO should be queued or performed on any file - ** handle associated with a database that may have been part of a - ** multi-file transaction that included the database associated with - ** the IO error (i.e. a database ATTACHed to the same handle at some - ** point in time). - */ - if( rc!=SQLITE_OK ){ - async.ioError = rc; - } - - if( async.ioError && !async.pQueueFirst ){ - async_mutex_enter(ASYNC_MUTEX_LOCK); - if( 0==async.pLock ){ - async.ioError = SQLITE_OK; - } - async_mutex_leave(ASYNC_MUTEX_LOCK); - } - - /* Drop the queue mutex before continuing to the next write operation - ** in order to give other threads a chance to work with the write queue. - */ - if( !async.pQueueFirst || !async.ioError ){ - async_mutex_leave(ASYNC_MUTEX_QUEUE); - holdingMutex = 0; - if( async.ioDelay>0 ){ - pVfs->xSleep(pVfs, async.ioDelay*1000); - }else{ - async_sched_yield(); - } - } - } - - async_mutex_leave(ASYNC_MUTEX_WRITER); - return; -} - -/* -** Install the asynchronous VFS. -*/ -int sqlite3async_initialize(const char *zParent, int isDefault){ - int rc = SQLITE_OK; - if( async_vfs.pAppData==0 ){ - sqlite3_vfs *pParent = sqlite3_vfs_find(zParent); - if( !pParent || async_os_initialize() ){ - rc = SQLITE_ERROR; - }else if( SQLITE_OK!=(rc = sqlite3_vfs_register(&async_vfs, isDefault)) ){ - async_os_shutdown(); - }else{ - async_vfs.pAppData = (void *)pParent; - async_vfs.mxPathname = ((sqlite3_vfs *)async_vfs.pAppData)->mxPathname; - } - } - return rc; -} - -/* -** Uninstall the asynchronous VFS. -*/ -void sqlite3async_shutdown(void){ - if( async_vfs.pAppData ){ - async_os_shutdown(); - sqlite3_vfs_unregister((sqlite3_vfs *)&async_vfs); - async_vfs.pAppData = 0; - } -} - -/* -** Process events on the write-queue. -*/ -void sqlite3async_run(void){ - asyncWriterThread(); -} - -/* -** Control/configure the asynchronous IO system. -*/ -int sqlite3async_control(int op, ...){ - int rc = SQLITE_OK; - va_list ap; - va_start(ap, op); - switch( op ){ - case SQLITEASYNC_HALT: { - int eWhen = va_arg(ap, int); - if( eWhen!=SQLITEASYNC_HALT_NEVER - && eWhen!=SQLITEASYNC_HALT_NOW - && eWhen!=SQLITEASYNC_HALT_IDLE - ){ - rc = SQLITE_MISUSE; - break; - } - async.eHalt = eWhen; - async_mutex_enter(ASYNC_MUTEX_QUEUE); - async_cond_signal(ASYNC_COND_QUEUE); - async_mutex_leave(ASYNC_MUTEX_QUEUE); - break; - } - - case SQLITEASYNC_DELAY: { - int iDelay = va_arg(ap, int); - if( iDelay<0 ){ - rc = SQLITE_MISUSE; - break; - } - async.ioDelay = iDelay; - break; - } - - case SQLITEASYNC_LOCKFILES: { - int bLock = va_arg(ap, int); - async_mutex_enter(ASYNC_MUTEX_QUEUE); - if( async.nFile || async.pQueueFirst ){ - async_mutex_leave(ASYNC_MUTEX_QUEUE); - rc = SQLITE_MISUSE; - break; - } - async.bLockFiles = bLock; - async_mutex_leave(ASYNC_MUTEX_QUEUE); - break; - } - - case SQLITEASYNC_GET_HALT: { - int *peWhen = va_arg(ap, int *); - *peWhen = async.eHalt; - break; - } - case SQLITEASYNC_GET_DELAY: { - int *piDelay = va_arg(ap, int *); - *piDelay = async.ioDelay; - break; - } - case SQLITEASYNC_GET_LOCKFILES: { - int *piDelay = va_arg(ap, int *); - *piDelay = async.bLockFiles; - break; - } - - default: - rc = SQLITE_ERROR; - break; - } - va_end(ap); - return rc; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ASYNCIO) */ diff --git a/ext/async/sqlite3async.h b/ext/async/sqlite3async.h deleted file mode 100644 index 13b23bc6a2..0000000000 --- a/ext/async/sqlite3async.h +++ /dev/null @@ -1,222 +0,0 @@ - -#ifndef __SQLITEASYNC_H_ -#define __SQLITEASYNC_H_ 1 - -/* -** Make sure we can call this stuff from C++. -*/ -#ifdef __cplusplus -extern "C" { -#endif - -#define SQLITEASYNC_VFSNAME "sqlite3async" - -/* -** THREAD SAFETY NOTES: -** -** Of the four API functions in this file, the following are not threadsafe: -** -** sqlite3async_initialize() -** sqlite3async_shutdown() -** -** Care must be taken that neither of these functions is called while -** another thread may be calling either any sqlite3async_XXX() function -** or an sqlite3_XXX() API function related to a database handle that -** is using the asynchronous IO VFS. -** -** These functions: -** -** sqlite3async_run() -** sqlite3async_control() -** -** are threadsafe. It is quite safe to call either of these functions even -** if another thread may also be calling one of them or an sqlite3_XXX() -** function related to a database handle that uses the asynchronous IO VFS. -*/ - -/* -** Initialize the asynchronous IO VFS and register it with SQLite using -** sqlite3_vfs_register(). If the asynchronous VFS is already initialized -** and registered, this function is a no-op. The asynchronous IO VFS -** is registered as "sqlite3async". -** -** The asynchronous IO VFS does not make operating system IO requests -** directly. Instead, it uses an existing VFS implementation for all -** required file-system operations. If the first parameter to this function -** is NULL, then the current default VFS is used for IO. If it is not -** NULL, then it must be the name of an existing VFS. In other words, the -** first argument to this function is passed to sqlite3_vfs_find() to -** locate the VFS to use for all real IO operations. This VFS is known -** as the "parent VFS". -** -** If the second parameter to this function is non-zero, then the -** asynchronous IO VFS is registered as the default VFS for all SQLite -** database connections within the process. Otherwise, the asynchronous IO -** VFS is only used by connections opened using sqlite3_open_v2() that -** specifically request VFS "sqlite3async". -** -** If a parent VFS cannot be located, then SQLITE_ERROR is returned. -** In the unlikely event that operating system specific initialization -** fails (win32 systems create the required critical section and event -** objects within this function), then SQLITE_ERROR is also returned. -** Finally, if the call to sqlite3_vfs_register() returns an error, then -** the error code is returned to the user by this function. In all three -** of these cases, intialization has failed and the asynchronous IO VFS -** is not registered with SQLite. -** -** Otherwise, if no error occurs, SQLITE_OK is returned. -*/ -int sqlite3async_initialize(const char *zParent, int isDefault); - -/* -** This function unregisters the asynchronous IO VFS using -** sqlite3_vfs_unregister(). -** -** On win32 platforms, this function also releases the small number of -** critical section and event objects created by sqlite3async_initialize(). -*/ -void sqlite3async_shutdown(void); - -/* -** This function may only be called when the asynchronous IO VFS is -** installed (after a call to sqlite3async_initialize()). It processes -** zero or more queued write operations before returning. It is expected -** (but not required) that this function will be called by a different -** thread than those threads that use SQLite. The "background thread" -** that performs IO. -** -** How many queued write operations are performed before returning -** depends on the global setting configured by passing the SQLITEASYNC_HALT -** verb to sqlite3async_control() (see below for details). By default -** this function never returns - it processes all pending operations and -** then blocks waiting for new ones. -** -** If multiple simultaneous calls are made to sqlite3async_run() from two -** or more threads, then the calls are serialized internally. -*/ -void sqlite3async_run(void); - -/* -** This function may only be called when the asynchronous IO VFS is -** installed (after a call to sqlite3async_initialize()). It is used -** to query or configure various parameters that affect the operation -** of the asynchronous IO VFS. At present there are three parameters -** supported: -** -** * The "halt" parameter, which configures the circumstances under -** which the sqlite3async_run() parameter is configured. -** -** * The "delay" parameter. Setting the delay parameter to a non-zero -** value causes the sqlite3async_run() function to sleep for the -** configured number of milliseconds between each queued write -** operation. -** -** * The "lockfiles" parameter. This parameter determines whether or -** not the asynchronous IO VFS locks the database files it operates -** on. Disabling file locking can improve throughput. -** -** This function is always passed two arguments. When setting the value -** of a parameter, the first argument must be one of SQLITEASYNC_HALT, -** SQLITEASYNC_DELAY or SQLITEASYNC_LOCKFILES. The second argument must -** be passed the new value for the parameter as type "int". -** -** When querying the current value of a paramter, the first argument must -** be one of SQLITEASYNC_GET_HALT, GET_DELAY or GET_LOCKFILES. The second -** argument to this function must be of type (int *). The current value -** of the queried parameter is copied to the memory pointed to by the -** second argument. For example: -** -** int eCurrentHalt; -** int eNewHalt = SQLITEASYNC_HALT_IDLE; -** -** sqlite3async_control(SQLITEASYNC_HALT, eNewHalt); -** sqlite3async_control(SQLITEASYNC_GET_HALT, &eCurrentHalt); -** assert( eNewHalt==eCurrentHalt ); -** -** See below for more detail on each configuration parameter. -** -** SQLITEASYNC_HALT: -** -** This is used to set the value of the "halt" parameter. The second -** argument must be one of the SQLITEASYNC_HALT_XXX symbols defined -** below (either NEVER, IDLE and NOW). -** -** If the parameter is set to NEVER, then calls to sqlite3async_run() -** never return. This is the default setting. If the parameter is set -** to IDLE, then calls to sqlite3async_run() return as soon as the -** queue of pending write operations is empty. If the parameter is set -** to NOW, then calls to sqlite3async_run() return as quickly as -** possible, without processing any pending write requests. -** -** If an attempt is made to set this parameter to an integer value other -** than SQLITEASYNC_HALT_NEVER, IDLE or NOW, then sqlite3async_control() -** returns SQLITE_MISUSE and the current value of the parameter is not -** modified. -** -** Modifying the "halt" parameter affects calls to sqlite3async_run() -** made by other threads that are currently in progress. -** -** SQLITEASYNC_DELAY: -** -** This is used to set the value of the "delay" parameter. If set to -** a non-zero value, then after completing a pending write request, the -** sqlite3async_run() function sleeps for the configured number of -** milliseconds. -** -** If an attempt is made to set this parameter to a negative value, -** sqlite3async_control() returns SQLITE_MISUSE and the current value -** of the parameter is not modified. -** -** Modifying the "delay" parameter affects calls to sqlite3async_run() -** made by other threads that are currently in progress. -** -** SQLITEASYNC_LOCKFILES: -** -** This is used to set the value of the "lockfiles" parameter. This -** parameter must be set to either 0 or 1. If set to 1, then the -** asynchronous IO VFS uses the xLock() and xUnlock() methods of the -** parent VFS to lock database files being read and/or written. If -** the parameter is set to 0, then these locks are omitted. -** -** This parameter may only be set when there are no open database -** connections using the VFS and the queue of pending write requests -** is empty. Attempting to set it when this is not true, or to set it -** to a value other than 0 or 1 causes sqlite3async_control() to return -** SQLITE_MISUSE and the value of the parameter to remain unchanged. -** -** If this parameter is set to zero, then it is only safe to access the -** database via the asynchronous IO VFS from within a single process. If -** while writing to the database via the asynchronous IO VFS the database -** is also read or written from within another process, or via another -** connection that does not use the asynchronous IO VFS within the same -** process, the results are undefined (and may include crashes or database -** corruption). -** -** Alternatively, if this parameter is set to 1, then it is safe to access -** the database from multiple connections within multiple processes using -** either the asynchronous IO VFS or the parent VFS directly. -*/ -int sqlite3async_control(int op, ...); - -/* -** Values that can be used as the first argument to sqlite3async_control(). -*/ -#define SQLITEASYNC_HALT 1 -#define SQLITEASYNC_GET_HALT 2 -#define SQLITEASYNC_DELAY 3 -#define SQLITEASYNC_GET_DELAY 4 -#define SQLITEASYNC_LOCKFILES 5 -#define SQLITEASYNC_GET_LOCKFILES 6 - -/* -** If the first argument to sqlite3async_control() is SQLITEASYNC_HALT, -** the second argument should be one of the following. -*/ -#define SQLITEASYNC_HALT_NEVER 0 /* Never halt (default value) */ -#define SQLITEASYNC_HALT_NOW 1 /* Halt as soon as possible */ -#define SQLITEASYNC_HALT_IDLE 2 /* Halt when write-queue is empty */ - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif -#endif /* ifndef __SQLITEASYNC_H_ */ diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index dee4eb9ec0..0c3b512af0 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -8,6 +8,7 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# TESTRUNNER: shell # # The focus of this file is testing the CLI shell tool. Specifically, # the ".recommend" command. @@ -401,6 +402,19 @@ do_setup_rec_test $tn.19.0 { SCAN t1 USING COVERING INDEX t1_idx_01a7214e } +ifcapable fts5 { + do_setup_rec_test $tn.20.0 { + CREATE VIRTUAL TABLE ft USING fts5(a); + CREATE TABLE t1(x, y); + } { + SELECT * FROM ft, t1 WHERE a=x + } { + CREATE INDEX t1_idx_00000078 ON t1(x); + SCAN ft VIRTUAL TABLE INDEX 0: + SEARCH t1 USING INDEX t1_idx_00000078 (x=?) + } +} + } proc do_candidates_test {tn sql res} { @@ -427,6 +441,8 @@ do_execsql_test 5.0 { WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100) INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s; + + CREATE INDEX i1 ON t1( lower(a) ); } do_candidates_test 5.1 { SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?) @@ -456,6 +472,7 @@ do_execsql_test 5.3 { ANALYZE; SELECT * FROM sqlite_stat1 ORDER BY 1, 2; } { + t1 i1 {100 50} t1 t1_idx_00000061 {100 50} t1 t1_idx_00000062 {100 20} t1 t1_idx_000123a7 {100 50 17} @@ -464,4 +481,129 @@ do_execsql_test 5.3 { t2 t2_idx_0001295b {100 20 5} } +do_catchsql_test 5.4 { + SELECT sqlite_expert_rem(123, 123); +} {1 {no such function: sqlite_expert_rem}} +do_catchsql_test 5.5 { + SELECT sqlite_expert_sample(); +} {1 {no such function: sqlite_expert_sample}} + +if 0 { +do_test expert1-6.0 { + catchcmd :memory: { +.expert +select base64(''); +.expert +select name from pragma_collation_list order by name collate uint; +} +} {0 {(no new indexes) + +SCAN CONSTANT ROW + +(no new indexes) + +SCAN pragma_collation_list VIRTUAL TABLE INDEX 0: +USE TEMP B-TREE FOR ORDER BY +}} +} + +do_execsql_test 6.0 { + CREATE TABLE x1(a, b, c, d); + CREATE INDEX x1ab ON x1(a, lower(b)); + CREATE INDEX x1dcba ON x1(d, b+c, a); +} + +do_candidates_test 6.1 { + SELECT * FROM x1 WHERE b=? ORDER BY a; +} { + CREATE INDEX x1_idx_0001267f ON x1(b, a); + CREATE INDEX x1_idx_00000062 ON x1(b); +} + +#------------------------------------------------------------------------- +ifcapable fts5 { + reset_db + do_execsql_test 7.0 { + CREATE VIRTUAL TABLE ft USING fts5(a); + CREATE TABLE t1(x, y); + } + + do_candidates_test 7.1 { + SELECT * FROM ft, t1 WHERE a=x + } { + CREATE INDEX t1_idx_00000078 ON t1(x); + } + + register_tcl_module db + proc vtab_command {method args} { + global G + + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c);" + } + + xBestIndex { + return [list] + } + + xFilter { + return [list sql "SELECT rowid, * FROM t0"] + } + } + + return {} + } + + do_execsql_test 7.2 { + CREATE TABLE t0(a, b, c); + INSERT INTO t0 VALUES(1, 2, 3), (11, 22, 33); + CREATE VIRTUAL TABLE t2 USING tcl(vtab_command); + } + + do_execsql_test 7.3 { + SELECT * FROM t2 + } { + 1 2 3 + 11 22 33 + } + + do_candidates_test 7.4 { + SELECT * FROM ft, t1 WHERE a=x + } { + CREATE INDEX t1_idx_00000078 ON t1(x); + } + + do_test 7.5 { + set expert [sqlite3_expert_new db] + list [catch { $expert sql "SELECT * FROM ft, t2 WHERE b=1" } msg] $msg + } {1 {no such table: t2}} + $expert destroy + + reset_db + do_execsql_test 7.6 { + BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS 'bfts_idx_data'(id INTEGER PRIMARY KEY, block BLOB); + CREATE TABLE IF NOT EXISTS 'fts_idx_data'(id INTEGER PRIMARY KEY, block BLOB); + INSERT INTO fts_idx_data VALUES(1,X''); + INSERT INTO fts_idx_data VALUES(10,X'00000000ff000001000000'); + CREATE TABLE IF NOT EXISTS 'fts_idx_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID; + CREATE TABLE IF NOT EXISTS 'fts_idx_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER); + CREATE TABLE IF NOT EXISTS 'fts_idx_config'(k PRIMARY KEY, v) WITHOUT ROWID; + INSERT INTO fts_idx_config VALUES('version',4); + PRAGMA writable_schema=ON; + INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)VALUES('table','fts_idx','fts_idx',0,'CREATE VIRTUAL TABLE fts_idx USING fts5(Title, Description, Channel, Tags, content='''', contentless_delete=1)'); + + CREATE TABLE f(x BLOB, y); + COMMIT; + PRAGMA writable_schema = RESET; + } + + do_candidates_test 7.4 { + SELECT * FROM fts_idx, f WHERE x = fts_idx.Channel + } { + CREATE INDEX f_idx_00000078 ON f(x); + } +} + finish_test diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index c01feff58c..c430c3ae95 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -32,7 +32,7 @@ #endif /* !defined(SQLITE_AMALGAMATION) */ -#ifndef SQLITE_OMIT_VIRTUALTABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -172,11 +172,11 @@ struct sqlite3expert { ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. */ -static void *idxMalloc(int *pRc, int nByte){ +static void *idxMalloc(int *pRc, i64 nByte){ void *pRet; assert( *pRc==SQLITE_OK ); assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); + pRet = sqlite3_malloc64(nByte); if( pRet ){ memset(pRet, 0, nByte); }else{ @@ -243,7 +243,7 @@ static int idxHashAdd( return 1; } } - pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); + pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + (i64)nKey+1 + (i64)nVal+1); if( pEntry ){ pEntry->zKey = (char*)&pEntry[1]; memcpy(pEntry->zKey, zKey, nKey); @@ -378,15 +378,15 @@ struct ExpertCsr { }; static char *expertDequote(const char *zIn){ - int n = STRLEN(zIn); - char *zRet = sqlite3_malloc(n); + i64 n = STRLEN(zIn); + char *zRet = sqlite3_malloc64(n); assert( zIn[0]=='\'' ); assert( zIn[n-1]=='\'' ); if( zRet ){ - int iOut = 0; - int iIn = 0; + i64 iOut = 0; + i64 iIn = 0; for(iIn=1; iIn<(n-1); iIn++){ if( zIn[iIn]=='\'' ){ assert( zIn[iIn+1]=='\'' ); @@ -626,7 +626,7 @@ static int expertFilter( pCsr->pData = 0; if( rc==SQLITE_OK ){ rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg, - "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName + "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName ); } @@ -662,6 +662,7 @@ static int idxRegisterVtab(sqlite3expert *p){ 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0, /* xIntegrity */ }; return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p); @@ -698,7 +699,7 @@ static int idxGetTableInfo( sqlite3_stmt *p1 = 0; int nCol = 0; int nTab; - int nByte; + i64 nByte; IdxTable *pNew = 0; int rc, rc2; char *pCsr = 0; @@ -790,14 +791,14 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ va_list ap; char *zAppend = 0; char *zRet = 0; - int nIn = zIn ? STRLEN(zIn) : 0; - int nAppend = 0; + i64 nIn = zIn ? STRLEN(zIn) : 0; + i64 nAppend = 0; va_start(ap, zFmt); if( *pRc==SQLITE_OK ){ zAppend = sqlite3_vmprintf(zFmt, ap); if( zAppend ){ nAppend = STRLEN(zAppend); - zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); + zRet = (char*)sqlite3_malloc64(nIn + nAppend + 1); } if( zAppend && zRet ){ if( nIn ) memcpy(zRet, zIn, nIn); @@ -1391,6 +1392,66 @@ static int idxProcessTriggers(sqlite3expert *p, char **pzErr){ return rc; } +/* +** This function tests if the schema of the main database of database handle +** db contains an object named zTab. Assuming no error occurs, output parameter +** (*pbContains) is set to true if zTab exists, or false if it does not. +** +** Or, if an error occurs, an SQLite error code is returned. The final value +** of (*pbContains) is undefined in this case. +*/ +static int expertDbContainsObject( + sqlite3 *db, + const char *zTab, + int *pbContains /* OUT: True if object exists */ +){ + const char *zSql = "SELECT 1 FROM sqlite_schema WHERE name = ?"; + sqlite3_stmt *pSql = 0; + int rc = SQLITE_OK; + int ret = 0; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pSql, 1, zTab, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pSql) ){ + ret = 1; + } + rc = sqlite3_finalize(pSql); + } + + *pbContains = ret; + return rc; +} + +/* +** Execute SQL command zSql using database handle db. If no error occurs, +** set (*pzErr) to NULL and return SQLITE_OK. +** +** If an error does occur, return an SQLite error code and set (*pzErr) to +** point to a buffer containing an English language error message. Except, +** if the error message begins with "no such module:", then ignore the +** error and return as if the SQL statement had succeeded. +** +** This is used to copy as much of the database schema as possible while +** ignoring any errors related to missing virtual table modules. +*/ +static int expertSchemaSql(sqlite3 *db, const char *zSql, char **pzErr){ + int rc = SQLITE_OK; + char *zErr = 0; + + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK && zErr ){ + int nErr = STRLEN(zErr); + if( nErr>=15 && memcmp(zErr, "no such module:", 15)==0 ){ + sqlite3_free(zErr); + rc = SQLITE_OK; + zErr = 0; + } + } + + *pzErr = zErr; + return rc; +} static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ int rc = idxRegisterVtab(p); @@ -1402,26 +1463,35 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){ ** 2) Create the equivalent virtual table in dbv. */ rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg, - "SELECT type, name, sql, 1 FROM sqlite_schema " - "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' " + "SELECT type, name, sql, 1, " + " substr(sql,1,14)=='create virtual' COLLATE nocase " + "FROM sqlite_schema " + "WHERE type IN ('table','view') AND " + " substr(name,1,7)!='sqlite_' COLLATE nocase " " UNION ALL " - "SELECT type, name, sql, 2 FROM sqlite_schema " + "SELECT type, name, sql, 2, 0 FROM sqlite_schema " "WHERE type = 'trigger'" " AND tbl_name IN(SELECT name FROM sqlite_schema WHERE type = 'view') " - "ORDER BY 4, 1" + "ORDER BY 4, 5 DESC, 1" ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){ const char *zType = (const char*)sqlite3_column_text(pSchema, 0); const char *zName = (const char*)sqlite3_column_text(pSchema, 1); const char *zSql = (const char*)sqlite3_column_text(pSchema, 2); + int bVirtual = sqlite3_column_int(pSchema, 4); + int bExists = 0; if( zType==0 || zName==0 ) continue; - if( zType[0]=='v' || zType[1]=='r' ){ - if( zSql ) rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg); + rc = expertDbContainsObject(p->dbv, zName, &bExists); + if( rc || bExists ) continue; + + if( zType[0]=='v' || zType[1]=='r' || bVirtual ){ + /* A view. Or a trigger on a view. */ + if( zSql ) rc = expertSchemaSql(p->dbv, zSql, pzErrmsg); }else{ IdxTable *pTab; rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(pTab!=0) ){ int i; char *zInner = 0; char *zOuter = 0; @@ -1492,14 +1562,14 @@ struct IdxRemCtx { int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ i64 iVal; /* SQLITE_INTEGER value */ double rVal; /* SQLITE_FLOAT value */ - int nByte; /* Bytes of space allocated at z */ - int n; /* Size of buffer z */ + i64 nByte; /* Bytes of space allocated at z */ + i64 n; /* Size of buffer z */ char *z; /* SQLITE_TEXT/BLOB value */ } aSlot[1]; }; /* -** Implementation of scalar function rem(). +** Implementation of scalar function sqlite_expert_rem(). */ static void idxRemFunc( sqlite3_context *pCtx, @@ -1512,7 +1582,7 @@ static void idxRemFunc( assert( argc==2 ); iSlot = sqlite3_value_int(argv[0]); - assert( iSlot<=p->nSlot ); + assert( iSlotnSlot ); pSlot = &p->aSlot[iSlot]; switch( pSlot->eType ){ @@ -1529,11 +1599,13 @@ static void idxRemFunc( break; case SQLITE_BLOB: - sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + assert( pSlot->n <= 0x7fffffff ); + sqlite3_result_blob(pCtx, pSlot->z, (int)pSlot->n, SQLITE_TRANSIENT); break; case SQLITE_TEXT: - sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + assert( pSlot->n <= 0x7fffffff ); + sqlite3_result_text(pCtx, pSlot->z, (int)pSlot->n, SQLITE_TRANSIENT); break; } @@ -1553,10 +1625,10 @@ static void idxRemFunc( case SQLITE_BLOB: case SQLITE_TEXT: { - int nByte = sqlite3_value_bytes(argv[1]); + i64 nByte = sqlite3_value_bytes(argv[1]); const void *pData = 0; if( nByte>pSlot->nByte ){ - char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); + char *zNew = (char*)sqlite3_realloc64(pSlot->z, nByte*2); if( zNew==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -1611,7 +1683,7 @@ static int idxPopulateOneStat1( int nCol = 0; int i; sqlite3_stmt *pQuery = 0; - int *aStat = 0; + i64 *aStat = 0; int rc = SQLITE_OK; assert( p->iSample>0 ); @@ -1622,8 +1694,15 @@ static int idxPopulateOneStat1( const char *zComma = zCols==0 ? "" : ", "; const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0); const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1); + if( zName==0 ){ + /* This index contains an expression. Ignore it. */ + sqlite3_free(zCols); + sqlite3_free(zOrder); + return sqlite3_reset(pIndexXInfo); + } zCols = idxAppendText(&rc, zCols, - "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl + "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", + zComma, zName, nCol, zName, zColl ); zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol); } @@ -1650,7 +1729,7 @@ static int idxPopulateOneStat1( sqlite3_free(zQuery); if( rc==SQLITE_OK ){ - aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); + aStat = (i64*)idxMalloc(&rc, sizeof(i64)*(nCol+1)); } if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ IdxHashEntry *pEntry; @@ -1667,11 +1746,11 @@ static int idxPopulateOneStat1( } if( rc==SQLITE_OK ){ - int s0 = aStat[0]; - zStat = sqlite3_mprintf("%d", s0); + i64 s0 = aStat[0]; + zStat = sqlite3_mprintf("%lld", s0); if( zStat==0 ) rc = SQLITE_NOMEM; for(i=1; rc==SQLITE_OK && i<=nCol; i++){ - zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]); + zStat = idxAppendText(&rc, zStat, " %lld", (s0+aStat[i]/2) / aStat[i]); } } @@ -1750,24 +1829,24 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); if( rc==SQLITE_OK ){ - int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); + i64 nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); } if( rc==SQLITE_OK ){ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv); - rc = sqlite3_create_function( - dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 + rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", + 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0 ); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 + rc = sqlite3_create_function(p->db, "sqlite_expert_sample", + 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0 ); } if( rc==SQLITE_OK ){ - pCtx->nSlot = nMax+1; + pCtx->nSlot = (i64)nMax+1; rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); } if( rc==SQLITE_OK ){ @@ -1814,10 +1893,95 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0); } + sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0); + sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0); + sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0); return rc; } +/* +** Define and possibly pretend to use a useless collation sequence. +** This pretense allows expert to accept SQL using custom collations. +*/ +int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){ + (void)up1; + (void)up2; + (void)up3; + (void)up4; + (void)up5; + assert(0); /* VDBE should never be run. */ + return 0; +} +/* And a callback to register above upon actual need */ +void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){ + (void)up1; + sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0); +} + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) +/* +** dummy functions for no-op implementation of UDFs during expert's work +*/ +void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){ + (void)up1; + (void)up2; + (void)up3; + assert(0); /* VDBE should never be run. */ +} +void dummyUDFvalue(sqlite3_context *up1){ + (void)up1; + assert(0); /* VDBE should never be run. */ +} + +/* +** Register UDFs from user database with another. +*/ +int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){ + sqlite3_stmt *pStmt; + int rc = sqlite3_prepare_v2(dbSrc, + "SELECT name,type,enc,narg,flags " + "FROM pragma_function_list() " + "WHERE builtin==0", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + int nargs = sqlite3_column_int(pStmt,3); + int flags = sqlite3_column_int(pStmt,4); + const char *name = (char*)sqlite3_column_text(pStmt,0); + const char *type = (char*)sqlite3_column_text(pStmt,1); + const char *enc = (char*)sqlite3_column_text(pStmt,2); + if( name==0 || type==0 || enc==0 ){ + /* no-op. Only happens on OOM */ + }else{ + int ienc = SQLITE_UTF8; + int rcf = SQLITE_ERROR; + if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE; + else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE; + ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY)); + if( strcmp(type,"w")==0 ){ + rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0, + dummyUDF,dummyUDFvalue,0,0,0); + }else if( strcmp(type,"a")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + 0,dummyUDF,dummyUDFvalue); + }else if( strcmp(type,"s")==0 ){ + rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0, + dummyUDF,0,0); + } + if( rcf!=SQLITE_OK ){ + rc = rcf; + break; + } + } + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + return rc; +} +#endif + /* ** Allocate a new sqlite3expert object. */ @@ -1844,18 +2008,38 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0); } } - + + /* Allow custom collations to be dealt with through prepare. */ + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS); + if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS); + +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \ + && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + /* Register UDFs from database [db] with [dbm] and [dbv]. */ + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbm); + } + if( rc==SQLITE_OK ){ + rc = registerUDFs(pNew->db, pNew->dbv); + } +#endif /* Copy the entire schema of database [db] into [dbm]. */ if( rc==SQLITE_OK ){ sqlite3_stmt *pSql = 0; rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, - "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'" - " AND sql NOT LIKE 'CREATE VIRTUAL %%'" + "SELECT sql, name, substr(sql,1,14)=='create virtual' COLLATE nocase" + " FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' COLLATE nocase" + " ORDER BY 3 DESC, rowid" ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); - if( zSql ) rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg); + const char *zName = (const char*)sqlite3_column_text(pSql, 1); + int bExists = 0; + rc = expertDbContainsObject(pNew->dbm, zName, &bExists); + if( rc==SQLITE_OK && zSql && bExists==0 ){ + rc = expertSchemaSql(pNew->dbm, zSql, pzErrmsg); + } } idxFinalize(&rc, pSql); } @@ -1870,7 +2054,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew); } - /* If an error has occurred, free the new object and reutrn NULL. Otherwise, + /* If an error has occurred, free the new object and return NULL. Otherwise, ** return the new sqlite3expert handle. */ if( rc!=SQLITE_OK ){ sqlite3_expert_destroy(pNew); @@ -1920,12 +2104,16 @@ int sqlite3_expert_sql( while( rc==SQLITE_OK && zStmt && zStmt[0] ){ sqlite3_stmt *pStmt = 0; + /* Ensure that the provided statement compiles against user's DB. */ + rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt); + if( rc!=SQLITE_OK ) break; + sqlite3_finalize(pStmt); rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt); if( rc==SQLITE_OK ){ if( pStmt ){ IdxStatement *pNew; const char *z = sqlite3_sql(pStmt); - int n = STRLEN(z); + i64 n = STRLEN(z); pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); if( rc==SQLITE_OK ){ pNew->zSql = (char*)&pNew[1]; diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c index 064c1908a9..4383d7c7bb 100644 --- a/ext/expert/test_expert.c +++ b/ext/expert/test_expert.c @@ -16,15 +16,7 @@ #include "sqlite3expert.h" #include #include - -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -36,7 +28,7 @@ static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ Tcl_CmdInfo info; if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ - Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), NULL); return TCL_ERROR; } diff --git a/ext/fts3/README.syntax b/ext/fts3/README.syntax index 01bc80c5fb..d32ae384c5 100644 --- a/ext/fts3/README.syntax +++ b/ext/fts3/README.syntax @@ -62,20 +62,20 @@ matches rows that contain both the "engineering" and "consultancy" tokens in the same column with not more than 10 other words between them. It does not matter which of the two terms occurs first in the document, only that - they be seperated by only 10 tokens or less. The user may also specify + they be separated by only 10 tokens or less. The user may also specify a different required proximity by adding "/N" immediately after the NEAR operator, where N is an integer. For example: MATCH 'engineering NEAR/5 consultancy' - searches for a row containing an instance of each specified token seperated + searches for a row containing an instance of each specified token separated by not more than 5 other tokens. More than one NEAR operator can be used in as sequence. For example this query: MATCH 'reliable NEAR/2 engineering NEAR/5 consultancy' searches for a row that contains an instance of the token "reliable" - seperated by not more than two tokens from an instance of "engineering", + separated by not more than two tokens from an instance of "engineering", which is in turn separated by not more than 5 other tokens from an instance of the term "consultancy". Phrases enclosed in quotes may also be used as arguments to the NEAR operator. @@ -146,7 +146,7 @@ MATCH '(hello world) OR (simple example)' matches documents that contain both "hello" and "world", and documents - that contain both "simple" and "example". It is not possible to forumlate + that contain both "simple" and "example". It is not possible to formulate such a query using the standard syntax. 2) Instead of separating tokens and phrases by whitespace, an AND operator @@ -174,7 +174,7 @@ 4) Unlike in the standard syntax, where the OR operator has a higher precedence than the implicit AND operator, when using the enhanced - syntax implicit and explict AND operators have a higher precedence + syntax implicit and explicit AND operators have a higher precedence than OR operators. Using the enhanced syntax, the following two queries are equivalent: diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 43a9daf60d..f178abafed 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -87,7 +87,7 @@ ** Here, array { X } means zero or more occurrences of X, adjacent in ** memory. A "position" is an index of a token in the token stream ** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals +** in the same logical place as the position element, and act as sentinels ** ending a position list array. POS_END is 0. POS_COLUMN is 1. ** The positions numbers are not stored literally but rather as two more ** than the difference from the prior position, or the just the position plus @@ -295,12 +295,6 @@ # define SQLITE_CORE 1 #endif -#include -#include -#include -#include -#include -#include #include "fts3.h" #ifndef SQLITE_CORE @@ -640,6 +634,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS); /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); @@ -2343,10 +2338,15 @@ static int fts3PoslistPhraseMerge( if( *p1==POS_COLUMN ){ p1++; p1 += fts3GetVarint32(p1, &iCol1); + /* iCol1==0 indicates corruption. Column 0 does not have a POS_COLUMN + ** entry, so this is actually end-of-doclist. */ + if( iCol1==0 ) return 0; } if( *p2==POS_COLUMN ){ p2++; p2 += fts3GetVarint32(p2, &iCol2); + /* As above, iCol2==0 indicates corruption. */ + if( iCol2==0 ) return 0; } while( 1 ){ @@ -2633,7 +2633,7 @@ static int fts3DoclistOrMerge( ** sizes of the two inputs, plus enough space for exactly one of the input ** docids to grow. ** - ** A symetric argument may be made if the doclists are in descending + ** A symmetric argument may be made if the doclists are in descending ** order. */ aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); @@ -3889,6 +3889,8 @@ static int fts3RenameMethod( rc = sqlite3Fts3PendingTermsFlush(p); } + p->bIgnoreSavepoint = 1; + if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", @@ -3916,6 +3918,8 @@ static int fts3RenameMethod( "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); + + p->bIgnoreSavepoint = 0; return rc; } @@ -3926,12 +3930,28 @@ static int fts3RenameMethod( */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint<=iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + + if( pTab->bIgnoreSavepoint==0 ){ + if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){ + char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", + pTab->zDb, pTab->zName, pTab->zName + ); + if( zSql ){ + pTab->bIgnoreSavepoint = 1; + rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); + pTab->bIgnoreSavepoint = 0; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } } return rc; } @@ -3942,12 +3962,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint >= iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint-1 ); + pTab->iSavepoint = iSavepoint; return SQLITE_OK; } @@ -3957,11 +3976,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** Discard the contents of the pending terms table. */ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; + Fts3Table *pTab = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); + assert( pTab->inTransaction ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + sqlite3Fts3PendingTermsClear(pTab); + } return SQLITE_OK; } @@ -3980,8 +4001,42 @@ static int fts3ShadowName(const char *zName){ return 0; } +/* +** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual +** table. +*/ +static int fts3IntegrityMethod( + sqlite3_vtab *pVtab, /* The virtual table to be checked */ + const char *zSchema, /* Name of schema in which pVtab lives */ + const char *zTabname, /* Name of the pVTab table */ + int isQuick, /* True if this is a quick_check */ + char **pzErr /* Write error message here */ +){ + Fts3Table *p = (Fts3Table*)pVtab; + int rc = SQLITE_OK; + int bOk = 0; + + UNUSED_PARAMETER(isQuick); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( rc!=SQLITE_CORRUPT_VTAB ); + if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS%d table %s.%s: %s", + p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", + p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + + + static const sqlite3_module fts3Module = { - /* iVersion */ 3, + /* iVersion */ 4, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, @@ -4005,6 +4060,7 @@ static const sqlite3_module fts3Module = { /* xRelease */ fts3ReleaseMethod, /* xRollbackTo */ fts3RollbackToMethod, /* xShadowName */ fts3ShadowName, + /* xIntegrity */ fts3IntegrityMethod, }; /* @@ -4376,7 +4432,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); + aOut = (char *)sqlite3Fts3MallocZero(((i64)nPoslist)+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; @@ -4675,7 +4731,7 @@ static int incrPhraseTokenNext( ** ** * does not contain any deferred tokens. ** -** Advance it to the next matching documnent in the database and populate +** Advance it to the next matching document in the database and populate ** the Fts3Doclist.pList and nList fields. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to @@ -5461,7 +5517,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ nTmp += p->pRight->pPhrase->doclist.nList; } nTmp += p->pPhrase->doclist.nList; - aTmp = sqlite3_malloc64(nTmp*2); + aTmp = sqlite3_malloc64(nTmp*2 + FTS3_VARINT_MAX); if( !aTmp ){ *pRc = SQLITE_NOMEM; res = 0; @@ -5682,7 +5738,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){ } /* -** Restart interation for expression pExpr so that the next call to +** Restart iteration for expression pExpr so that the next call to ** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** @@ -5725,6 +5781,24 @@ static void fts3EvalRestart( } } +/* +** Expression node pExpr is an MSR phrase. This function restarts pExpr +** so that it is a regular phrase query, not an MSR. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +*/ +int sqlite3Fts3MsrCancel(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc = SQLITE_OK; + if( pExpr->bEof==0 ){ + i64 iDocid = pExpr->iDocid; + fts3EvalRestart(pCsr, pExpr, &rc); + while( rc==SQLITE_OK && pExpr->iDocid!=iDocid ){ + fts3EvalNextRow(pCsr, pExpr, &rc); + if( pExpr->bEof ) rc = FTS_CORRUPT_VTAB; + } + } + return rc; +} + /* ** After allocating the Fts3Expr.aMI[] array for each phrase in the ** expression rooted at pExpr, the cursor iterates through all rows matched @@ -6112,7 +6186,7 @@ int sqlite3Fts3Corrupt(){ } #endif -#if !SQLITE_CORE +#if !defined(SQLITE_CORE) /* ** Initialize API pointer table, if required. */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 3a8a884f92..e98b90a753 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -14,10 +14,20 @@ #ifndef _FTSINT_H #define _FTSINT_H -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif +#include +#include +#include +#include +#include +#include + /* FTS3/FTS4 require virtual tables */ #ifdef SQLITE_OMIT_VIRTUALTABLE # undef SQLITE_ENABLE_FTS3 @@ -37,7 +47,7 @@ /* If not building as part of the core, include sqlite3ext.h. */ #ifndef SQLITE_CORE -# include "sqlite3ext.h" +# include "sqlite3ext.h" SQLITE_EXTENSION_INIT3 #endif @@ -179,13 +189,6 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ */ #define UNUSED_PARAMETER(x) (void)(x) -/* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - /* ** The TESTONLY macro is used to enclose variable declarations or ** other bits of code that are needed to support the arguments @@ -202,6 +205,19 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define deliberate_fall_through +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + + #endif /* SQLITE_AMALGAMATION */ #ifdef SQLITE_DEBUG @@ -265,6 +281,7 @@ struct Fts3Table { int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + int iSavepoint; /* ** The following array of hash tables is used to buffer pending index @@ -305,7 +322,7 @@ struct Fts3Table { #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - /* True to disable the incremental doclist optimization. This is controled + /* True to disable the incremental doclist optimization. This is controlled ** by special insert command 'test-no-incr-doclist'. */ int bNoIncrDoclist; @@ -357,7 +374,7 @@ struct Fts3Cursor { /* ** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** Actually, Fts3Cursor.eSearch can be greater than or equal to ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index ** of the column to be searched. For example, in ** @@ -430,9 +447,13 @@ struct Fts3Phrase { */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ + Fts3PhraseToken aToken[FLEXARRAY]; /* One for each token in the phrase */ }; +/* Size (in bytes) of an Fts3Phrase object large enough to hold N tokens */ +#define SZ_FTS3PHRASE(N) \ + (offsetof(Fts3Phrase,aToken)+(N)*sizeof(Fts3PhraseToken)) + /* ** A tree of these objects forms the RHS of a MATCH operator. ** @@ -639,6 +660,7 @@ int sqlite3Fts3MsrIncrNext( int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); +int sqlite3Fts3MsrCancel(Fts3Cursor*, Fts3Expr*); /* fts3_tokenize_vtab.c */ int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); @@ -652,5 +674,7 @@ int sqlite3FtsUnicodeIsdiacritic(int); int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); + #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index d3b194c942..439d579366 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -545,7 +545,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index ea8167c595..681d4e8625 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -161,6 +161,23 @@ int sqlite3Fts3OpenTokenizer( */ static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); +/* +** Search buffer z[], size n, for a '"' character. Or, if enable_parenthesis +** is defined, search for '(' and ')' as well. Return the index of the first +** such character in the buffer. If there is no such character, return -1. +*/ +static int findBarredChar(const char *z, int n){ + int ii; + for(ii=0; iiiLangid, z, i, &pCursor); + *pnConsumed = n; + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; @@ -202,7 +212,18 @@ static int getNextToken( rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + /* Check that this tokenization did not gobble up any " characters. Or, + ** if enable_parenthesis is true, that it did not gobble up any + ** open or close parenthesis characters either. If it did, call + ** getNextToken() again, but pass only that part of the input buffer + ** up to the first such character. */ + int iBarred = findBarredChar(z, iEnd); + if( iBarred>=0 ){ + pModule->xClose(pCursor); + return getNextToken(pParse, iCol, z, iBarred, ppExpr, pnConsumed); + } + + nByte = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1) + nToken; pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; @@ -212,7 +233,7 @@ static int getNextToken( pRet->pPhrase->nToken = 1; pRet->pPhrase->iColumn = iCol; pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + pRet->pPhrase->aToken[0].z = (char*)&pRet->pPhrase->aToken[1]; memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); if( iEnd=0 ){ + *pnConsumed = iBarred; + } rc = SQLITE_OK; } @@ -283,9 +308,9 @@ static int getNextString( Fts3Expr *p = 0; sqlite3_tokenizer_cursor *pCursor = 0; char *zTemp = 0; - int nTemp = 0; + i64 nTemp = 0; - const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + const int nSpace = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1); int nToken = 0; /* The final Fts3Expr data structure, including the Fts3Phrase, @@ -319,10 +344,11 @@ static int getNextString( Fts3PhraseToken *pToken; p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); - if( !p ) goto no_mem; - zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); - if( !zTemp ) goto no_mem; + if( !zTemp || !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } assert( nToken==ii ); pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; @@ -337,9 +363,6 @@ static int getNextString( nToken = ii+1; } } - - pModule->xClose(pCursor); - pCursor = 0; } if( rc==SQLITE_DONE ){ @@ -347,7 +370,10 @@ static int getNextString( char *zBuf = 0; p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); - if( !p ) goto no_mem; + if( !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; p->pPhrase = (Fts3Phrase *)&p[1]; @@ -355,11 +381,9 @@ static int getNextString( p->pPhrase->nToken = nToken; zBuf = (char *)&p->pPhrase->aToken[nToken]; + assert( nTemp==0 || zTemp ); if( zTemp ){ memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); - }else{ - assert( nTemp==0 ); } for(jj=0; jjpPhrase->nToken; jj++){ @@ -369,17 +393,17 @@ static int getNextString( rc = SQLITE_OK; } - *ppExpr = p; - return rc; -no_mem: - + getnextstring_out: if( pCursor ){ pModule->xClose(pCursor); } sqlite3_free(zTemp); - sqlite3_free(p); - *ppExpr = 0; - return SQLITE_NOMEM; + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + p = 0; + } + *ppExpr = p; + return rc; } /* @@ -658,7 +682,7 @@ static int fts3ExprParse( /* The isRequirePhrase variable is set to true if a phrase or ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** binary operator (AND, OR, NOT or NEAR) is encountered when ** isRequirePhrase is set, this is a syntax error. */ if( !isPhrase && isRequirePhrase ){ @@ -1240,7 +1264,6 @@ static void fts3ExprTestCommon( } if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ - sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 63e55b3dc9..1918be4cb7 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -187,7 +187,7 @@ static void fts3HashInsertElement( } -/* Resize the hash table so that it cantains "new_size" buckets. +/* Resize the hash table so that it contains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. ** diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index fbe7913020..35e81b74af 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -256,7 +256,7 @@ static int star_oh(const char *z){ /* ** If the word ends with zFrom and xCond() is true for the stem -** of the word that preceeds the zFrom ending, then change the +** of the word that precedes the zFrom ending, then change the ** ending to zTo. ** ** The input word *pz and zFrom are both in reverse order. zTo diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 227c5f00f6..62e27d30bf 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -17,10 +17,6 @@ #include #include -#ifndef SQLITE_AMALGAMATION -typedef sqlite3_int64 i64; -#endif - /* ** Characters that may appear in the second argument to matchinfo(). */ @@ -108,9 +104,13 @@ struct MatchinfoBuffer { int nElem; int bGlobal; /* Set if global data is loaded */ char *zMatchinfo; - u32 aMatchinfo[1]; + u32 aMI[FLEXARRAY]; }; +/* Size (in bytes) of a MatchinfoBuffer sufficient for N elements */ +#define SZ_MATCHINFOBUFFER(N) \ + (offsetof(MatchinfoBuffer,aMI)+(((N)+1)/2)*sizeof(u64)) + /* ** The snippet() and offsets() functions both return text values. An instance @@ -135,13 +135,13 @@ struct StrBuffer { static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) - + sizeof(MatchinfoBuffer); + + SZ_MATCHINFOBUFFER(1); sqlite3_int64 nStr = strlen(zMatchinfo); pRet = sqlite3Fts3MallocZero(nByte + nStr+1); if( pRet ){ - pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + pRet->aMI[0] = (u8*)(&pRet->aMI[1]) - (u8*)pRet; + pRet->aMI[1+nElem] = pRet->aMI[0] + sizeof(u32)*((int)nElem+1); pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; @@ -155,10 +155,10 @@ static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ static void fts3MIBufferFree(void *p){ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + assert( (u32*)p==&pBuf->aMI[1] + || (u32*)p==&pBuf->aMI[pBuf->nElem+2] ); - if( (u32*)p==&pBuf->aMatchinfo[1] ){ + if( (u32*)p==&pBuf->aMI[1] ){ pBuf->aRef[1] = 0; }else{ pBuf->aRef[2] = 0; @@ -175,18 +175,18 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ if( p->aRef[1]==0 ){ p->aRef[1] = 1; - aOut = &p->aMatchinfo[1]; + aOut = &p->aMI[1]; xRet = fts3MIBufferFree; } else if( p->aRef[2]==0 ){ p->aRef[2] = 1; - aOut = &p->aMatchinfo[p->nElem+2]; + aOut = &p->aMI[p->nElem+2]; xRet = fts3MIBufferFree; }else{ aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); if( aOut ){ xRet = sqlite3_free; - if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + if( p->bGlobal ) memcpy(aOut, &p->aMI[1], p->nElem*sizeof(u32)); } } @@ -196,7 +196,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ p->bGlobal = 1; - memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); + memcpy(&p->aMI[2+p->nElem], &p->aMI[1], p->nElem*sizeof(u32)); } /* @@ -398,6 +398,7 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){ return 1; } + assert( pIter->nSnippet>=0 ); pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; for(i=0; inPhrase; i++){ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; @@ -446,7 +447,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } @@ -610,7 +611,7 @@ static int fts3StringAppend( } /* If there is insufficient space allocated at StrBuffer.z, use realloc() - ** to grow the buffer until so that it is big enough to accomadate the + ** to grow the buffer until so that it is big enough to accommodate the ** appended data. */ if( pStr->n+nAppend+1>=pStr->nAlloc ){ @@ -1022,16 +1023,16 @@ static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ break; case FTS3_MATCHINFO_LHITS: - nVal = pInfo->nCol * pInfo->nPhrase; + nVal = (size_t)pInfo->nCol * pInfo->nPhrase; break; case FTS3_MATCHINFO_LHITS_BM: - nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + nVal = (size_t)pInfo->nPhrase * ((pInfo->nCol + 31) / 32); break; default: assert( cArg==FTS3_MATCHINFO_HITS ); - nVal = pInfo->nCol * pInfo->nPhrase * 3; + nVal = (size_t)pInfo->nCol * pInfo->nPhrase * 3; break; } @@ -1585,6 +1586,22 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ return rc; } +/* +** If expression pExpr is a phrase expression that uses an MSR query, +** restart it as a regular, non-incremental query. Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + TermOffsetCtx *p = (TermOffsetCtx*)ctx; + int rc = SQLITE_OK; + UNUSED_PARAMETER(iPhrase); + if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){ + rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr); + pExpr->pPhrase->bIncr = 0; + } + return rc; +} + /* ** Implementation of offsets() function. */ @@ -1621,6 +1638,12 @@ void sqlite3Fts3Offsets( sCtx.iDocid = pCsr->iPrevId; sCtx.pCsr = pCsr; + /* If a query restart will be required, do it here, rather than later of + ** after pointers to poslist buffers that may be invalidated by a restart + ** have been saved. */ + rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx); + if( rc!=SQLITE_OK ) goto offsets_out; + /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index 47e244e22c..655dd9f35a 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -78,6 +78,8 @@ static int fts3termConnectMethod( iIndex = atoi(argv[4]); argc--; } + + *ppVtab = 0; /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ @@ -95,12 +97,17 @@ static int fts3termConnectMethod( rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); if( rc!=SQLITE_OK ) return rc; - nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3termTable *)sqlite3_malloc64(nByte); + nByte = sizeof(Fts3termTable); + p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, (size_t)nByte); - p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero( + sizeof(Fts3Table) + nDb + nFts3 + 2 + ); + if( p->pFts3Tab==0 ){ + sqlite3_free(p); + return SQLITE_NOMEM; + } p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; @@ -130,6 +137,7 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_finalize(pFts3->aStmt[i]); } sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(pFts3); sqlite3_free(p); return SQLITE_OK; } @@ -362,7 +370,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 49a8476bf3..70bccf0c52 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -18,14 +18,7 @@ ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. */ -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #include #include @@ -167,7 +160,8 @@ static int SQLITE_TCLAPI fts3_near_match_cmd( Tcl_Obj *pPhrasecount = 0; Tcl_Obj **apExprToken; - int nExprToken; + Tcl_Size nExprToken; + Tcl_Size nn; UNUSED_PARAMETER(clientData); @@ -201,37 +195,40 @@ static int SQLITE_TCLAPI fts3_near_match_cmd( } } - rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken); + rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken); + doc.nToken = (int)nn; if( rc!=TCL_OK ) goto near_match_out; doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken)); for(ii=0; iiNM_MAX_TOKEN ){ - Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + Tcl_AppendResult(interp, "Too many tokens in phrase", NULL); rc = TCL_ERROR; goto near_match_out; } - for(jj=0; jjz = Tcl_GetStringFromObj(apToken[jj], &pT->n); + pT->z = Tcl_GetStringFromObj(apToken[jj], &nn); + pT->n = (int)nn; } - aPhrase[ii].nToken = nToken; + aPhrase[ii].nToken = (int)nToken; } for(ii=1; iizInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; @@ -445,7 +445,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index eab3f513e5..24c237a89d 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -226,11 +226,7 @@ int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif +#include "tclsqlite.h" #include /* diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 393f8a8717..19dff31f00 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3325,7 +3325,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - sqlite3Fts3PendingTermsClear(p); /* Determine the auto-incr-merge setting if unknown. If enabled, ** estimate the number of leaf blocks of content to be written @@ -3347,6 +3346,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = sqlite3_reset(pStmt); } } + + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } return rc; } @@ -3711,8 +3714,8 @@ struct NodeWriter { ** to an appendable b-tree segment. */ struct IncrmergeWriter { - int nLeafEst; /* Space allocated for leaf blocks */ - int nWork; /* Number of leaf pages flushed */ + i64 nLeafEst; /* Space allocated for leaf blocks */ + i64 nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ @@ -3953,7 +3956,7 @@ static int fts3IncrmergePush( ** ** It is assumed that the buffer associated with pNode is already large ** enough to accommodate the new entry. The buffer associated with pPrev -** is extended by this function if requrired. +** is extended by this function if required. ** ** If an error (i.e. OOM condition) occurs, an SQLite error code is ** returned. Otherwise, SQLITE_OK. @@ -3978,6 +3981,8 @@ static int fts3AppendToNode( blobGrowBuffer(pPrev, nTerm, &rc); if( rc!=SQLITE_OK ) return rc; + assert( pPrev!=0 ); + assert( pPrev->a!=0 ); nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); nSuffix = nTerm - nPrefix; @@ -4034,9 +4039,13 @@ static int fts3IncrmergeAppend( nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; /* If the current block is not empty, and if adding this term/doclist - ** to the current block would make it larger than Fts3Table.nNodeSize - ** bytes, write this block out to the database. */ - if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + ** to the current block would make it larger than Fts3Table.nNodeSize bytes, + ** and if there is still room for another leaf page, write this block out to + ** the database. */ + if( pLeaf->block.n>0 + && (pLeaf->block.n + nSpace)>p->nNodeSize + && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst) + ){ rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); pWriter->nWork++; @@ -4347,6 +4356,7 @@ static int fts3IncrmergeLoad( for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ NodeReader reader; + memset(&reader, 0, sizeof(reader)); pNode = &pWriter->aNodeWriter[i]; if( pNode->block.a){ @@ -4367,7 +4377,7 @@ static int fts3IncrmergeLoad( rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc - ); + ); if( rc==SQLITE_OK ){ memcpy(pNode->block.a, aBlock, nBlock); pNode->block.n = nBlock; @@ -4451,7 +4461,7 @@ static int fts3IncrmergeWriter( ){ int rc; /* Return Code */ int i; /* Iterator variable */ - int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + i64 nLeafEst = 0; /* Blocks allocated for leaf nodes */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ @@ -4461,7 +4471,7 @@ static int fts3IncrmergeWriter( sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ - nLeafEst = sqlite3_column_int(pLeafEst, 0); + nLeafEst = sqlite3_column_int64(pLeafEst, 0); } rc = sqlite3_reset(pLeafEst); } @@ -5217,7 +5227,7 @@ static u64 fts3ChecksumIndex( int rc; u64 cksum = 0; - assert( *pRc==SQLITE_OK ); + if( *pRc ) return 0; memset(&filter, 0, sizeof(filter)); memset(&csr, 0, sizeof(csr)); @@ -5284,7 +5294,7 @@ static u64 fts3ChecksumIndex( ** If an error occurs (e.g. an OOM or IO error), return an SQLite error ** code. The final value of *pbOk is undefined in this case. */ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ +int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int rc = SQLITE_OK; /* Return code */ u64 cksum1 = 0; /* Checksum based on FTS index contents */ u64 cksum2 = 0; /* Checksum based on %_content contents */ @@ -5362,7 +5372,12 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -5402,7 +5417,7 @@ static int fts3DoIntegrityCheck( ){ int rc; int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } @@ -5432,8 +5447,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoIncrmerge(p, &zVal[6]); }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ rc = fts3DoAutoincrmerge(p, &zVal[10]); + }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - }else{ + else{ int v; if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ v = atoi(&zVal[9]); @@ -5451,8 +5469,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; rc = SQLITE_OK; } -#endif } +#endif return rc; } @@ -5601,7 +5619,7 @@ int sqlite3Fts3DeferToken( /* ** SQLite value pRowid contains the rowid of a row that may or may not be ** present in the FTS3 table. If it is, delete it and adjust the contents -** of subsiduary data structures accordingly. +** of subsidiary data structures accordingly. */ static int fts3DeleteByRowid( Fts3Table *p, diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 58d90c68c7..3bf866ef74 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -628,6 +628,9 @@ proc print_categories {lMap} { $caseP $caseS $caseZ + + default: + return 1; } return 0; } @@ -890,7 +893,7 @@ proc print_test_main {} { puts "\}" } -# Proces the command line arguments. Exit early if they are not to +# Process the command line arguments. Exit early if they are not to # our liking. # proc usage {} { diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index 2320d70b7d..6ee71c262c 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -82,7 +82,7 @@ proc get_struct_docs {data names} { set current_doc "" } set subject n/a - regexp {^ *([[:alpha:]]*)} $line -> subject + regexp {^ *([[:alnum:]_]*)} $line -> subject if {[lsearch $names $subject]>=0} { set current_header $subject } else { @@ -108,8 +108,11 @@ proc get_tokenizer_docs {data} { append res "
    $line

    \n" continue } + if {[regexp {FTS5_TOKENIZER} $line]} { + set line

    + } if {[regexp {SYNONYM SUPPORT} $line]} { - set line "

    Synonym Support

    " + set line "

    Synonym Support

    " } if {[string trim $line] == ""} { append res "

    \n" @@ -223,10 +226,12 @@ proc main {data} { Fts5ExtensionApi { set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"] set map [list] + set lKey [list] foreach {k v} [get_struct_members $data] { if {[string match x* $k]==0} continue - lappend map $k "$k" + lappend lKey $k } + foreach k [lsort -decr $lKey] { lappend map $k "$k" } output [string map $map $struct] } diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 081e534f3f..907ea232a4 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -55,8 +55,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken @@ -88,8 +88,11 @@ struct Fts5PhraseIter { ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -99,8 +102,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -116,12 +121,13 @@ struct Fts5PhraseIter { ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. Returns SQLITE_OK if successful, or an error -** code (i.e. SQLITE_NOMEM) if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -147,6 +153,10 @@ struct Fts5PhraseIter { ** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -228,6 +238,10 @@ struct Fts5PhraseIter { ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -261,9 +275,80 @@ struct Fts5PhraseIter { ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API may be slow in some cases if the token identified by parameters +** iIdx and iToken matched a prefix token in the query. In most cases, the +** first call to this API for each prefix token in the query is forced +** to scan the portion of the full-text index that matches the prefix +** token to collect the extra data required by this API. If the prefix +** token matches a large number of token instances in the document set, +** this may be a performance problem. +** +** If the user knows in advance that a query may use this API for a +** prefix token, FTS5 may be configured to collect all required data as part +** of the initial querying of the full-text index, avoiding the second scan +** entirely. This also causes prefix queries that do not use this API to +** run more slowly and use more memory. FTS5 may be configured in this way +** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] +** option, or on a per-query basis using the +** [fts5_insttoken | fts5_insttoken()] user function. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -298,6 +383,22 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; /* @@ -318,7 +419,7 @@ struct Fts5ExtensionApi { ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -342,7 +443,7 @@ struct Fts5ExtensionApi { ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -366,6 +467,13 @@ struct Fts5ExtensionApi { ** on a columnsize=0 database. ** ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -389,6 +497,30 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**

      +**
    • There is no "iVersion" field, and +**
    • The xTokenize() method does not take a locale argument. +**
    +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -492,11 +624,38 @@ struct Fts5ExtensionApi { ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ typedef struct fts5_tokenizer fts5_tokenizer; struct fts5_tokenizer { int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); @@ -516,6 +675,7 @@ struct fts5_tokenizer { ); }; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -535,13 +695,13 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -550,7 +710,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -558,10 +718,29 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5d05da875e..d5404535cc 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -20,6 +20,7 @@ SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_AMALGAMATION @@ -59,7 +60,34 @@ typedef sqlite3_uint64 u64; # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +/* +** This macro is used in a single assert() within fts5 to check that an +** allocation is aligned to an 8-byte boundary. But it is a complicated +** macro to get right for multiple platforms without generating warnings. +** So instead of reproducing the entire definition from sqliteInt.h, we +** just do without this assert() for the rare non-amalgamation builds. +*/ +#define EIGHT_BYTE_ALIGNMENT(x) 1 + +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + +#endif /* SQLITE_AMALGAMATION */ + +/* +** Constants for the largest and smallest possible 32-bit signed integers. +*/ +# define LARGEST_INT32 ((int)(0x7fffffff)) +# define SMALLEST_INT32 ((int)((-1) - LARGEST_INT32)) /* Truncate very long tokens to this many bytes. Hard limit is ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset @@ -131,10 +159,11 @@ typedef struct Fts5Colset Fts5Colset; */ struct Fts5Colset { int nCol; - int aiCol[1]; + int aiCol[FLEXARRAY]; }; - +/* Size (int bytes) of a complete Fts5Colset object with N columns. */ +#define SZ_FTS5COLSET(N) (sizeof(i64)*((N+2)/2)) /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code @@ -142,6 +171,18 @@ struct Fts5Colset { */ typedef struct Fts5Config Fts5Config; +typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; + +struct Fts5TokenizerConfig { + Fts5Tokenizer *pTok; + fts5_tokenizer_v2 *pApi2; + fts5_tokenizer *pApi1; + const char **azArg; + int nArg; + int ePattern; /* FTS_PATTERN_XXX constant */ + const char *pLocale; /* Current locale to use */ + int nLocale; /* Size of pLocale in bytes */ +}; /* ** An instance of the following structure encodes all information that can @@ -154,6 +195,10 @@ typedef struct Fts5Config Fts5Config; ** attempt to merge together. A value of 1 sets the object to use the ** compile time default. Zero disables auto-merge altogether. ** +** bContentlessDelete: +** True if the contentless_delete option was present in the CREATE +** VIRTUAL TABLE statement. +** ** zContent: ** ** zContentRowid: @@ -177,9 +222,12 @@ typedef struct Fts5Config Fts5Config; ** ** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); ** +** bLocale: +** Set to true if locale=1 was specified when the table was created. */ struct Fts5Config { sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* Global fts5 object for handle db */ char *zDb; /* Database holding FTS index (e.g. "main") */ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ @@ -188,15 +236,18 @@ struct Fts5Config { int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ + int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */ char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int bTokendata; /* "tokendata=" option value (dflt==0) */ + int bLocale; /* "locale=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; - Fts5Tokenizer *pTok; - fts5_tokenizer *pTokApi; + Fts5TokenizerConfig t; int bLock; /* True when table is preparing statement */ - int ePattern; /* FTS_PATTERN_XXX constant */ + /* Values loaded from the %_config table */ int iVersion; /* fts5 file format 'version' */ @@ -209,6 +260,8 @@ struct Fts5Config { char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ int bSecureDelete; /* 'secure-delete' */ + int nDeleteMerge; /* 'deletemerge' */ + int bPrefixInsttoken; /* 'prefix-insttoken' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -224,9 +277,10 @@ struct Fts5Config { #define FTS5_CURRENT_VERSION 4 #define FTS5_CURRENT_VERSION_SECUREDELETE 5 -#define FTS5_CONTENT_NORMAL 0 -#define FTS5_CONTENT_NONE 1 -#define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_CONTENT_NORMAL 0 +#define FTS5_CONTENT_NONE 1 +#define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_CONTENT_UNINDEXED 3 #define FTS5_DETAIL_FULL 0 #define FTS5_DETAIL_NONE 1 @@ -261,6 +315,8 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); int sqlite3Fts5ConfigParseRank(const char*, char**, char**); +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...); + /* ** End of interface to code in fts5_config.c. **************************************************************************/ @@ -305,7 +361,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); void sqlite3Fts5Put32(u8*, int); int sqlite3Fts5Get32(const u8*); -#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) +#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF) #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) typedef struct Fts5PoslistReader Fts5PoslistReader; @@ -378,17 +434,19 @@ struct Fts5IndexIter { /* ** Values used as part of the flags argument passed to IndexQuery(). */ -#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ -#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ -#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ -#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* The following are used internally by the fts5_index.c module. They are ** defined here only to make it easier to avoid clashes with the flags ** above. */ -#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 -#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 -#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080 +#define FTS5INDEX_QUERY_SCANONETERM 0x0100 /* ** Create/destroy an Fts5Index object. @@ -457,6 +515,17 @@ void *sqlite3Fts5StructureRef(Fts5Index*); void sqlite3Fts5StructureRelease(void*); int sqlite3Fts5StructureTest(Fts5Index*, void*); +/* +** Used by xInstToken(): +*/ +int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +); /* ** Insert or remove data to or from the index. Each time a document is @@ -531,6 +600,16 @@ int sqlite3Fts5IndexReset(Fts5Index *p); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); +int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); + +void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); + +/* Used to populate hash tables for xInstToken in detail=none/column mode. */ +int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff +); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ @@ -574,18 +653,20 @@ struct Fts5Table { Fts5Index *pIndex; /* Full-text index */ }; -int sqlite3Fts5GetTokenizer( - Fts5Global*, - const char **azArg, - int nArg, - Fts5Config*, - char **pzErr -); +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig); Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); int sqlite3Fts5FlushToDisk(Fts5Table*); +void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc); + +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); +int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, + const char **ppText, int *pnText, const char **ppLoc, int *pnLoc +); + /* ** End of interface to code in fts5.c. **************************************************************************/ @@ -615,6 +696,11 @@ int sqlite3Fts5HashWrite( */ void sqlite3Fts5HashClear(Fts5Hash*); +/* +** Return true if the hash is empty, false otherwise. +*/ +int sqlite3Fts5HashIsEmpty(Fts5Hash*); + int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ int nPre, @@ -631,11 +717,13 @@ void sqlite3Fts5HashScanNext(Fts5Hash*); int sqlite3Fts5HashScanEof(Fts5Hash*); void sqlite3Fts5HashScanEntry(Fts5Hash *, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ); + /* ** End of interface to code in fts5_hash.c. **************************************************************************/ @@ -658,8 +746,8 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); -int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); +int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int); +int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg); @@ -684,6 +772,9 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); int sqlite3Fts5StorageReset(Fts5Storage *p); +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*); +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ @@ -730,7 +821,7 @@ int sqlite3Fts5ExprPattern( ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); ** } */ -int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc); int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); int sqlite3Fts5ExprEof(Fts5Expr*); i64 sqlite3Fts5ExprRowid(Fts5Expr*); @@ -756,6 +847,10 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); +int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); +int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*); +void sqlite3Fts5ExprClearTokens(Fts5Expr*); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by @@ -832,6 +927,7 @@ int sqlite3Fts5TokenizerPattern( int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), Fts5Tokenizer *pTok ); +int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*); /* ** End of interface to code in fts5_tokenizer.c. **************************************************************************/ diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index b178f47334..95b33ea318 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -110,15 +110,19 @@ static int fts5CInstIterInit( */ typedef struct HighlightContext HighlightContext; struct HighlightContext { - CInstIter iter; /* Coalesced Instance Iterator */ - int iPos; /* Current token offset in zIn[] */ + /* Constant parameters to fts5HighlightCb() */ int iRangeStart; /* First token to include */ int iRangeEnd; /* If non-zero, last token to include */ const char *zOpen; /* Opening highlight */ const char *zClose; /* Closing highlight */ const char *zIn; /* Input text */ int nIn; /* Size of input text in bytes */ - int iOff; /* Current offset within zIn[] */ + + /* Variables modified by fts5HighlightCb() */ + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iOff; /* Have copied up to this offset in zIn[] */ + int bOpen; /* True if highlight is open */ char *zOut; /* Output value */ }; @@ -151,8 +155,8 @@ static int fts5HighlightCb( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStartOff, /* Start offset of token */ - int iEndOff /* End offset of token */ + int iStartOff, /* Start byte offset of token */ + int iEndOff /* End byte offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; @@ -168,35 +172,61 @@ static int fts5HighlightCb( if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } - if( iPos==p->iter.iStart ){ + /* If the parenthesis is open, and this token is not part of the current + ** phrase, and the starting byte offset of this token is past the point + ** that has currently been copied into the output buffer, close the + ** parenthesis. */ + if( p->bOpen + && (iPos<=p->iter.iStart || p->iter.iStart<0) + && iStartOff>p->iOff + ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } + + /* If this is the start of a new phrase, and the highlight is not open: + ** + ** * copy text from the input up to the start of the phrase, and + ** * open the highlight. + */ + if( iPos==p->iter.iStart && p->bOpen==0 ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); fts5HighlightAppend(&rc, p, p->zOpen, -1); p->iOff = iStartOff; + p->bOpen = 1; } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd>=0 && p->iter.iStartiRangeStart ){ + if( p->bOpen==0 ){ + assert( p->iRangeEnd>=0 ); fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->bOpen = 1; } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); - fts5HighlightAppend(&rc, p, p->zClose, -1); p->iOff = iEndOff; + if( rc==SQLITE_OK ){ rc = fts5CInstIterNext(&p->iter); } } - if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){ - fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); - p->iOff = iEndOff; - if( iPos>=p->iter.iStart && iPositer.iEnd ){ + if( iPos==p->iRangeEnd ){ + if( p->bOpen ){ + if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + } fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; } + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; } return rc; } + /* ** Implementation of highlight() function. */ @@ -223,14 +253,26 @@ static void fts5HighlightFunction( ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); - - if( ctx.zIn ){ + if( rc==SQLITE_RANGE ){ + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = SQLITE_OK; + }else if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb + ); + } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); } fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); @@ -424,6 +466,8 @@ static void fts5SnippetFunction( memset(&sFinder, 0, sizeof(Fts5SFinder)); for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); if( rc!=SQLITE_OK ) break; - rc = pApi->xTokenize(pFts, - sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize_v2(pFts, + sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb ); if( rc!=SQLITE_OK ) break; rc = pApi->xColumnSize(pFts, i, &nDocsize); @@ -490,6 +536,9 @@ static void fts5SnippetFunction( rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); } if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iBestCol */ + int nLoc = 0; /* Bytes in pLoc */ + if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } @@ -508,7 +557,15 @@ static void fts5SnippetFunction( } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb + ); + } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); } if( ctx.iRangeEnd>=(nColSize-1) ){ fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); @@ -610,7 +667,7 @@ static int fts5Bm25GetData( ** under consideration. ** ** The problem with this is that if (N < 2*nHit), the IDF is - ** negative. Which is undesirable. So the mimimum allowable IDF is + ** negative. Which is undesirable. So the minimum allowable IDF is ** (1e-6) - roughly the same as a term that appears in just over ** half of set of 5,000,000 documents. */ double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); @@ -689,6 +746,53 @@ static void fts5Bm25Function( } } +/* +** Implementation of fts5_get_locale() function. +*/ +static void fts5GetLocaleFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + int iCol = 0; + int eType = 0; + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + /* xColumnLocale() must be available */ + assert( pApi->iVersion>=4 ); + + if( nVal!=1 ){ + const char *z = "wrong number of arguments to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + eType = sqlite3_value_numeric_type(apVal[0]); + if( eType!=SQLITE_INTEGER ){ + const char *z = "non-integer argument passed to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ + sqlite3_result_error_code(pCtx, SQLITE_RANGE); + return; + } + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + return; + } + + sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT); +} + int sqlite3Fts5AuxInit(fts5_api *pApi){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ @@ -696,9 +800,10 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { - { "snippet", 0, fts5SnippetFunction, 0 }, - { "highlight", 0, fts5HighlightFunction, 0 }, - { "bm25", 0, fts5Bm25Function, 0 }, + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index b9614e1290..afcd83b6ba 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -68,6 +68,7 @@ void sqlite3Fts5BufferAppendBlob( ){ if( nData ){ if( fts5BufferGrow(pRc, pBuf, nData) ) return; + assert( pBuf->p!=0 ); memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; } @@ -169,6 +170,7 @@ int sqlite3Fts5PoslistNext64( i64 *piOff /* IN/OUT: Current offset */ ){ int i = *pi; + assert( a!=0 || i==0 ); if( i>=n ){ /* EOF */ *piOff = -1; @@ -176,6 +178,7 @@ int sqlite3Fts5PoslistNext64( }else{ i64 iOff = *piOff; u32 iVal; + assert( a!=0 ); fts5FastGetVarint32(a, i, iVal); if( iVal<=1 ){ if( iVal==0 ){ @@ -305,7 +308,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ ** * The 52 upper and lower case ASCII characters, and ** * The 10 integer ASCII characters. ** * The underscore character "_" (0x5F). -** * The unicode "subsitute" character (0x1A). +** * The unicode "substitute" character (0x1A). */ int sqlite3Fts5IsBareword(char t){ u8 aBareword[128] = { diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7a4c7b8177..eea82b046d 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -22,6 +22,8 @@ #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ + /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (64*1024) @@ -232,7 +234,6 @@ static int fts5ConfigSetEnum( ** eventually free any such error message using sqlite3_free(). */ static int fts5ConfigParseSpecial( - Fts5Global *pGlobal, Fts5Config *pConfig, /* Configuration object to update */ const char *zCmd, /* Special command to parse */ const char *zArg, /* Argument to parse */ @@ -240,6 +241,7 @@ static int fts5ConfigParseSpecial( ){ int rc = SQLITE_OK; int nCmd = (int)strlen(zCmd); + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; const char *p; @@ -296,12 +298,11 @@ static int fts5ConfigParseSpecial( if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ const char *p = (const char*)zArg; sqlite3_int64 nArg = strlen(zArg) + 1; - char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); - char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); - char *pSpace = pDel; + char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg); - if( azArg && pSpace ){ - if( pConfig->pTok ){ + if( azArg ){ + char *pSpace = (char*)&azArg[nArg]; + if( pConfig->t.azArg ){ *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); rc = SQLITE_ERROR; }else{ @@ -324,16 +325,14 @@ static int fts5ConfigParseSpecial( *pzErr = sqlite3_mprintf("parse error in tokenize directive"); rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, (int)nArg, pConfig, - pzErr - ); + pConfig->t.azArg = (const char**)azArg; + pConfig->t.nArg = nArg; + azArg = 0; } } } - sqlite3_free(azArg); - sqlite3_free(pDel); + return rc; } @@ -352,6 +351,26 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessDelete = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessUnindexed = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ if( pConfig->zContentRowid ){ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); @@ -372,6 +391,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed locale=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bLocale = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, @@ -386,20 +415,20 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed tokendata=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bTokendata = (zArg[0]=='1'); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } -/* -** Allocate an instance of the default tokenizer ("simple") at -** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error -** code if an error occurs. -*/ -static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ - assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); - return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0); -} - /* ** Gobble up the first bareword or quoted word from the input buffer zIn. ** Return a pointer to the character immediately following the last in @@ -459,7 +488,8 @@ static int fts5ConfigParseColumn( Fts5Config *p, char *zCol, char *zArg, - char **pzErr + char **pzErr, + int *pbUnindexed ){ int rc = SQLITE_OK; if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) @@ -470,6 +500,7 @@ static int fts5ConfigParseColumn( }else if( zArg ){ if( 0==sqlite3_stricmp(zArg, "unindexed") ){ p->abUnindexed[p->nCol] = 1; + *pbUnindexed = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); rc = SQLITE_ERROR; @@ -490,11 +521,26 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); if( p->eContent!=FTS5_CONTENT_NONE ){ + assert( p->eContent==FTS5_CONTENT_EXTERNAL + || p->eContent==FTS5_CONTENT_NORMAL + || p->eContent==FTS5_CONTENT_UNINDEXED + ); for(i=0; inCol; i++){ if( p->eContent==FTS5_CONTENT_EXTERNAL ){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); - }else{ + }else if( p->eContent==FTS5_CONTENT_NORMAL || p->abUnindexed[i] ){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } + if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){ + for(i=0; inCol; i++){ + if( p->abUnindexed[i]==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); } } } @@ -528,10 +574,12 @@ int sqlite3Fts5ConfigParse( Fts5Config *pRet; /* New object to return */ int i; sqlite3_int64 nByte; + int bUnindexed = 0; /* True if there are one or more UNINDEXED */ *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); + pRet->pGlobal = pGlobal; pRet->db = db; pRet->iCookie = -1; @@ -580,13 +628,13 @@ int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; }else{ if( bOption ){ - rc = fts5ConfigParseSpecial(pGlobal, pRet, + rc = fts5ConfigParseSpecial(pRet, ALWAYS(zOne)?zOne:"", zTwo?zTwo:"", pzErr ); }else{ - rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr); + rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr, &bUnindexed); zOne = 0; } } @@ -596,21 +644,52 @@ int sqlite3Fts5ConfigParse( sqlite3_free(zTwo); } - /* If a tokenizer= option was successfully parsed, the tokenizer has - ** already been allocated. Otherwise, allocate an instance of the default - ** tokenizer (unicode61) now. */ - if( rc==SQLITE_OK && pRet->pTok==0 ){ - rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); + /* We only allow contentless_delete=1 if the table is indeed contentless. */ + if( rc==SQLITE_OK + && pRet->bContentlessDelete + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_delete=1 if columnsize=0 is not present. + ** + ** This restriction may be removed at some point. + */ + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 is incompatible with columnsize=0" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_unindexed=1 if the table is actually a + ** contentless one. + */ + if( rc==SQLITE_OK + && pRet->bContentlessUnindexed + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_unindexed=1 requires a contentless table" + ); + rc = SQLITE_ERROR; } /* If no zContent option was specified, fill in the default values. */ if( rc==SQLITE_OK && pRet->zContent==0 ){ const char *zTail = 0; - assert( pRet->eContent==FTS5_CONTENT_NORMAL - || pRet->eContent==FTS5_CONTENT_NONE + assert( pRet->eContent==FTS5_CONTENT_NORMAL + || pRet->eContent==FTS5_CONTENT_NONE ); if( pRet->eContent==FTS5_CONTENT_NORMAL ){ zTail = "content"; + }else if( bUnindexed && pRet->bContentlessUnindexed ){ + pRet->eContent = FTS5_CONTENT_UNINDEXED; + zTail = "content"; }else if( pRet->bColumnsize ){ zTail = "docsize"; } @@ -644,9 +723,14 @@ int sqlite3Fts5ConfigParse( void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ if( pConfig ){ int i; - if( pConfig->pTok ){ - pConfig->pTokApi->xDelete(pConfig->pTok); + if( pConfig->t.pTok ){ + if( pConfig->t.pApi1 ){ + pConfig->t.pApi1->xDelete(pConfig->t.pTok); + }else{ + pConfig->t.pApi2->xDelete(pConfig->t.pTok); + } } + sqlite3_free((char*)pConfig->t.azArg); sqlite3_free(pConfig->zDb); sqlite3_free(pConfig->zName); for(i=0; inCol; i++){ @@ -721,10 +805,24 @@ int sqlite3Fts5Tokenize( void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ - if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize( - pConfig->pTok, pCtx, flags, pText, nText, xToken - ); + int rc = SQLITE_OK; + if( pText ){ + if( pConfig->t.pTok==0 ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } + if( rc==SQLITE_OK ){ + if( pConfig->t.pApi1 ){ + rc = pConfig->t.pApi1->xTokenize( + pConfig->t.pTok, pCtx, flags, pText, nText, xToken + ); + }else{ + rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, + pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken + ); + } + } + } + return rc; } /* @@ -890,6 +988,18 @@ int sqlite3Fts5ConfigSetValue( } } + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ + int nVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nVal = sqlite3_value_int(pVal); + }else{ + *pbBadkey = 1; + } + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; + if( nVal>100 ) nVal = 0; + pConfig->nDeleteMerge = nVal; + } + else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; @@ -916,6 +1026,19 @@ int sqlite3Fts5ConfigSetValue( }else{ pConfig->bSecureDelete = (bVal ? 1 : 0); } + } + + else if( 0==sqlite3_stricmp(zKey, "insttoken") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bPrefixInsttoken = (bVal ? 1 : 0); + } + }else{ *pbBadkey = 1; } @@ -938,6 +1061,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ @@ -965,13 +1089,10 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE ){ rc = SQLITE_ERROR; - if( pConfig->pzErrmsg ){ - assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " - "(found %d, expected %d or %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE - ); - } + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE + ); }else{ pConfig->iVersion = iVersion; } @@ -981,3 +1102,26 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ } return rc; } + +/* +** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer +** containing the error message created using printf() style formatting +** string zFmt and its trailing arguments. +*/ +void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ + va_list ap; /* ... printf arguments */ + char *zMsg = 0; + + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + if( pConfig->pzErrmsg ){ + assert( *pConfig->pzErrmsg==0 ); + *pConfig->pzErrmsg = zMsg; + }else{ + sqlite3_free(zMsg); + } + + va_end(ap); +} + + diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 0e018420d0..352df81f4f 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -54,7 +54,7 @@ struct Fts5Expr { /* ** eType: -** Expression node type. Always one of: +** Expression node type. Usually one of: ** ** FTS5_AND (nChild, apChild valid) ** FTS5_OR (nChild, apChild valid) @@ -62,6 +62,10 @@ struct Fts5Expr { ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) ** +** An expression node with eType==0 may also exist. It always matches zero +** rows. This is created when a phrase containing no tokens is parsed. +** e.g. "". +** ** iHeight: ** Distance from this node to furthest leaf. This is always 0 for nodes ** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one @@ -82,9 +86,13 @@ struct Fts5ExprNode { /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ - Fts5ExprNode *apChild[1]; /* Array of child nodes */ + Fts5ExprNode *apChild[FLEXARRAY]; /* Array of child nodes */ }; +/* Size (in bytes) of an Fts5ExprNode object that holds up to N children */ +#define SZ_FTS5EXPRNODE(N) \ + (offsetof(Fts5ExprNode,apChild) + (N)*sizeof(Fts5ExprNode*)) + #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) /* @@ -100,7 +108,9 @@ struct Fts5ExprNode { struct Fts5ExprTerm { u8 bPrefix; /* True for a prefix term */ u8 bFirst; /* True if token must be first in column */ - char *zTerm; /* nul-terminated term */ + char *pTerm; /* Term data */ + int nQueryTerm; /* Effective size of term in bytes */ + int nFullTerm; /* Size of term in bytes incl. tokendata */ Fts5IndexIter *pIter; /* Iterator for this term */ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; @@ -113,9 +123,13 @@ struct Fts5ExprPhrase { Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ - Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ + Fts5ExprTerm aTerm[FLEXARRAY]; /* Terms that make up this phrase */ }; +/* Size (in bytes) of an Fts5ExprPhrase object that holds up to N terms */ +#define SZ_FTS5EXPRPHRASE(N) \ + (offsetof(Fts5ExprPhrase,aTerm) + (N)*sizeof(Fts5ExprTerm)) + /* ** One or more phrases that must appear within a certain token distance of ** each other within each matching document. @@ -124,9 +138,12 @@ struct Fts5ExprNearset { int nNear; /* NEAR parameter */ Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ int nPhrase; /* Number of entries in aPhrase[] array */ - Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ + Fts5ExprPhrase *apPhrase[FLEXARRAY]; /* Array of phrase pointers */ }; +/* Size (in bytes) of an Fts5ExprNearset object covering up to N phrases */ +#define SZ_FTS5EXPRNEARSET(N) \ + (offsetof(Fts5ExprNearset,apPhrase)+(N)*sizeof(Fts5ExprPhrase*)) /* ** Parse context. @@ -280,12 +297,13 @@ int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert( sParse.pExpr || sParse.rc!=SQLITE_OK ); assert_expr_depth_ok(sParse.rc, sParse.pExpr); /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ - if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ - int n = sizeof(Fts5Colset); + if( sParse.rc==SQLITE_OK && iColnCol ){ + int n = SZ_FTS5COLSET(1); Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); if( pColset ){ pColset->nCol = 1; @@ -301,15 +319,7 @@ int sqlite3Fts5ExprNew( sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ - if( !sParse.pExpr ){ - const int nByte = sizeof(Fts5ExprNode); - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); - if( pNew->pRoot ){ - pNew->pRoot->bEof = 1; - } - }else{ - pNew->pRoot = sParse.pExpr; - } + pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; @@ -322,7 +332,11 @@ int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } @@ -967,7 +981,7 @@ static int fts5ExprNearInitAll( p->pIter = 0; } rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), + pExpr->pIndex, p->pTerm, p->nQueryTerm, (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), pNear->pColset, @@ -1123,7 +1137,7 @@ static int fts5ExprNodeTest_STRING( } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - if( pIter->iRowid==iLast || pIter->bEof ) continue; + if( pIter->iRowid==iLast ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; @@ -1535,7 +1549,13 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ -int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ +int sqlite3Fts5ExprFirst( + Fts5Expr *p, + Fts5Index *pIdx, + i64 iFirst, + i64 iLast, + int bDesc +){ Fts5ExprNode *pRoot = p->pRoot; int rc; /* Return code */ @@ -1557,6 +1577,9 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ assert( pRoot->bEof==0 ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } return rc; } @@ -1604,7 +1627,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ Fts5ExprTerm *pSyn; Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; - sqlite3_free(pTerm->zTerm); + sqlite3_free(pTerm->pTerm); sqlite3Fts5IterClose(pTerm->pIter); for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; @@ -1645,12 +1668,9 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( Fts5ExprNearset *pRet = 0; if( pParse->rc==SQLITE_OK ){ - if( pPhrase==0 ){ - return pNear; - } if( pNear==0 ){ sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(SZALLOC+1); pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -1661,7 +1681,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( int nNew = pNear->nPhrase + SZALLOC; sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(nNew+1); pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -1702,6 +1722,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + Fts5Config *pConfig; int rc; }; @@ -1735,8 +1756,12 @@ static int fts5ParseTokenize( rc = SQLITE_NOMEM; }else{ memset(pSyn, 0, (size_t)nByte); - pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); - memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + pSyn->nFullTerm = pSyn->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata ){ + pSyn->nQueryTerm = (int)strlen(pSyn->pTerm); + } + memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; } @@ -1747,12 +1772,12 @@ static int fts5ParseTokenize( int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + SZ_FTS5EXPRPHRASE(nNew+1) ); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + if( pPhrase==0 ) memset(pNew, 0, SZ_FTS5EXPRPHRASE(1)); pCtx->pPhrase = pPhrase = pNew; pNew->nTerm = nNew - SZALLOC; } @@ -1761,7 +1786,11 @@ static int fts5ParseTokenize( if( rc==SQLITE_OK ){ pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->nFullTerm = pTerm->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ + pTerm->nQueryTerm = (int)strlen(pTerm->pTerm); + } } } @@ -1828,6 +1857,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( memset(&sCtx, 0, sizeof(TokenCtx)); sCtx.pPhrase = pAppend; + sCtx.pConfig = pConfig; rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ @@ -1855,10 +1885,11 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( if( sCtx.pPhrase==0 ){ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, SZ_FTS5EXPRPHRASE(1)); }else if( sCtx.pPhrase->nTerm ){ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; } + assert( pParse->apPhrase!=0 ); pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; } @@ -1875,30 +1906,32 @@ int sqlite3Fts5ExprClonePhrase( Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ + if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + rc = SQLITE_RANGE; + }else{ + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + } if( rc==SQLITE_OK ){ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNode)); + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRNODE(1)); } if( rc==SQLITE_OK ){ - pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + SZ_FTS5EXPRNEARSET(2)); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; if( pColsetOrig ){ sqlite3_int64 nByte; Fts5Colset *pColset; - nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + nByte = SZ_FTS5COLSET(pColsetOrig->nCol); pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); if( pColset ){ memcpy(pColset, pColsetOrig, (size_t)nByte); @@ -1907,26 +1940,27 @@ int sqlite3Fts5ExprClonePhrase( } } - if( pOrig->nTerm ){ - int i; /* Used to iterate through phrase terms */ - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), - 0, 0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + if( rc==SQLITE_OK ){ + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRPHRASE(1)); } - }else{ - /* This happens when parsing a token or quoted phrase that contains - ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){ @@ -1990,7 +2024,8 @@ void sqlite3Fts5ParseSetDistance( ); return; } - nNear = nNear * 10 + (p->p[i] - '0'); + if( nNear<214748363 ) nNear = nNear * 10 + (p->p[i] - '0'); + /* ^^^^^^^^^^^^^^^--- Prevent integer overflow */ } }else{ nNear = FTS5_DEFAULT_NEARDIST; @@ -2019,7 +2054,7 @@ static Fts5Colset *fts5ParseColset( assert( pParse->rc==SQLITE_OK ); assert( iCol>=0 && iColpConfig->nCol ); - pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + pNew = sqlite3_realloc64(p, SZ_FTS5COLSET(nCol+1)); if( pNew==0 ){ pParse->rc = SQLITE_NOMEM; }else{ @@ -2054,7 +2089,7 @@ Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ int nCol = pParse->pConfig->nCol; pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, - sizeof(Fts5Colset) + sizeof(int)*nCol + SZ_FTS5COLSET(nCol+1) ); if( pRet ){ int i; @@ -2115,7 +2150,7 @@ Fts5Colset *sqlite3Fts5ParseColset( static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ Fts5Colset *pRet; if( pOrig ){ - sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + sqlite3_int64 nByte = SZ_FTS5COLSET(pOrig->nCol); pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); if( pRet ){ memcpy(pRet, pOrig, (size_t)nByte); @@ -2242,6 +2277,9 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } } +/* +** Add pSub as a child of p. +*/ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ @@ -2280,7 +2318,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( assert( pNear->nPhrase==1 ); assert( pParse->bPhraseToAnd ); - nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + nByte = SZ_FTS5EXPRNODE(nTerm+1); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ pRet->eType = FTS5_AND; @@ -2290,17 +2328,19 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( pParse->nPhrase--; for(ii=0; iirc, sizeof(Fts5ExprPhrase) + &pParse->rc, SZ_FTS5EXPRPHRASE(1) ); if( pPhrase ){ if( parseGrowPhraseArray(pParse) ){ fts5ExprPhraseFree(pPhrase); }else{ + Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii]; + Fts5ExprTerm *pTo = &pPhrase->aTerm[0]; pParse->apPhrase[pParse->nPhrase++] = pPhrase; pPhrase->nTerm = 1; - pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup( - &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1 - ); + pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm); + pTo->nQueryTerm = p->nQueryTerm; + pTo->nFullTerm = p->nFullTerm; pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) ); @@ -2357,7 +2397,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode( if( pRight->eType==eType ) nChild += pRight->nChild-1; } - nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + nByte = SZ_FTS5EXPRNODE(nChild); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ @@ -2384,19 +2424,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode( "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); - sqlite3_free(pRet); + sqlite3Fts5ParseNodeFree(pRet); pRet = 0; + pNear = 0; + assert( pLeft==0 && pRight==0 ); } } }else{ + assert( pNear==0 ); fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); + pLeft = pRight = 0; if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ sqlite3Fts5ParseError(pParse, "fts5 expression tree is too large (maximum depth %d)", SQLITE_FTS5_MAX_EXPR_DEPTH ); - sqlite3_free(pRet); + sqlite3Fts5ParseNodeFree(pRet); pRet = 0; } } @@ -2434,6 +2478,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( assert( pRight->eType==FTS5_STRING || pRight->eType==FTS5_TERM || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -2447,6 +2492,8 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( ); if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>0 ); assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); sqlite3Fts5ParseNodeFree(pRight); pRet = pLeft; @@ -2477,7 +2524,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( return pRet; } -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ sqlite3_int64 nByte = 0; Fts5ExprTerm *p; @@ -2485,16 +2532,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ /* Determine the maximum amount of space required. */ for(p=pTerm; p; p=p->pSynonym){ - nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; + nByte += pTerm->nQueryTerm * 2 + 3 + 2; } zQuoted = sqlite3_malloc64(nByte); if( zQuoted ){ int i = 0; for(p=pTerm; p; p=p->pSynonym){ - char *zIn = p->zTerm; + char *zIn = p->pTerm; + char *zEnd = &zIn[p->nQueryTerm]; zQuoted[i++] = '"'; - while( *zIn ){ + while( zInnTerm; iTerm++){ - char *zTerm = pPhrase->aTerm[iTerm].zTerm; - zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + Fts5ExprTerm *p = &pPhrase->aTerm[iTerm]; + zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", + p->nQueryTerm, p->pTerm + ); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); } @@ -2583,6 +2633,8 @@ static char *fts5ExprPrintTcl( if( zRet==0 ) return 0; } + }else if( pExpr->eType==0 ){ + zRet = sqlite3_mprintf("{}"); }else{ char const *zOp = 0; int i; @@ -2844,14 +2896,14 @@ static void fts5ExprFold( sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); } } -#endif /* ifdef SQLITE_TEST */ +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called during initialization to register the fts5_expr() scalar ** UDF with the SQLite handle passed as the only argument. */ int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) struct Fts5ExprFunc { const char *z; void (*x)(sqlite3_context*,int,sqlite3_value**); @@ -2972,6 +3024,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +/* +** pToken is a buffer nToken bytes in size that may or may not contain +** an embedded 0x00 byte. If it does, return the number of bytes in +** the buffer before the 0x00. If it does not, return nToken. +*/ +static int fts5QueryTerm(const char *pToken, int nToken){ + int ii; + for(ii=0; iipExpr; int i; + int nQuery = nToken; + i64 iRowid = pExpr->pRoot->iRowid; UNUSED_PARAM2(iUnused1, iUnused2); - if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE; + if( pExpr->pConfig->bTokendata ){ + nQuery = fts5QueryTerm(pToken, nQuery); + } if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ - Fts5ExprTerm *pTerm; + Fts5ExprTerm *pT; if( p->aPopulator[i].bOk==0 ) continue; - for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = (int)strlen(pTerm->zTerm); - if( (nTerm==nToken || (nTermbPrefix)) - && memcmp(pTerm->zTerm, pToken, nTerm)==0 + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + if( (pT->nQueryTerm==nQuery || (pT->nQueryTermbPrefix)) + && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); + if( rc==SQLITE_OK && (pExpr->pConfig->bTokendata || pT->bPrefix) ){ + int iCol = p->iOff>>32; + int iTokOff = p->iOff & 0x7FFFFFFF; + rc = sqlite3Fts5IndexIterWriteTokendata( + pT->pIter, pToken, nToken, iRowid, iCol, iTokOff + ); + } if( rc ) return rc; break; } @@ -3052,6 +3126,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ + case 0: case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); @@ -3133,3 +3208,79 @@ int sqlite3Fts5ExprPhraseCollist( return rc; } + +/* +** Does the work of the fts5_api.xQueryToken() API method. +*/ +int sqlite3Fts5ExprQueryToken( + Fts5Expr *pExpr, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + + *ppOut = pPhrase->aTerm[iToken].pTerm; + *pnOut = pPhrase->aTerm[iToken].nFullTerm; + return SQLITE_OK; +} + +/* +** Does the work of the fts5_api.xInstToken() API method. +*/ +int sqlite3Fts5ExprInstToken( + Fts5Expr *pExpr, + i64 iRowid, + int iPhrase, + int iCol, + int iOff, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + Fts5ExprTerm *pTerm = 0; + int rc = SQLITE_OK; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + pTerm = &pPhrase->aTerm[iToken]; + if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm, + iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; + } + return rc; +} + +/* +** Clear the token mappings for all Fts5IndexIter objects managed by +** the expression passed as the only argument. +*/ +void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ + int ii; + for(ii=0; iinPhrase; ii++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterClearTokendata(pT->pIter); + } + } +} diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index bc9244fc01..a33dec9a92 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -20,7 +20,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; /* ** This file contains the implementation of an in-memory hash table used -** to accumuluate "term -> doclist" content before it is flused to a level-0 +** to accumulate "term -> doclist" content before it is flushed to a level-0 ** segment. */ @@ -36,10 +36,15 @@ struct Fts5Hash { /* ** Each entry in the hash table is represented by an object of the -** following type. Each object, its key (a nul-terminated string) and -** its current data are stored in a single memory allocation. The -** key immediately follows the object in memory. The position list -** data immediately follows the key data in memory. +** following type. Each object, its key, and its current data are stored +** in a single memory allocation. The key immediately follows the object +** in memory. The position list data immediately follows the key data +** in memory. +** +** The key is Fts5HashEntry.nKey bytes in size. It consists of a single +** byte identifying the index (either the main term index or a prefix-index), +** followed by the term data. For example: "0token". There is no +** nul-terminator - in this case nKey=6. ** ** The data that follows the key is in a similar, but not identical format ** to the doclist data stored in the database. It is: @@ -72,7 +77,7 @@ struct Fts5HashEntry { }; /* -** Eqivalent to: +** Equivalent to: ** ** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } */ @@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){ unsigned int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), - (int)strlen(fts5EntryKey(p))); + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } @@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite( for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ char *zKey = fts5EntryKey(p); if( zKey[0]==bByte - && p->nKey==nToken + && p->nKey==nToken+1 && memcmp(&zKey[1], pToken, nToken)==0 ){ break; @@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite( zKey[0] = bByte; memcpy(&zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); - p->nKey = nToken; + p->nKey = nToken+1; zKey[nToken+1] = '\0'; - p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); + p->nData = nToken+1 + sizeof(Fts5HashEntry); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge( *ppOut = p1; p1 = 0; }else{ - int i = 0; char *zKey1 = fts5EntryKey(p1); char *zKey2 = fts5EntryKey(p2); - while( zKey1[i]==zKey2[i] ) i++; + int nMin = MIN(p1->nKey, p2->nKey); + + int cmp = memcmp(zKey1, zKey2, nMin); + if( cmp==0 ){ + cmp = p1->nKey - p2->nKey; + } + assert( cmp!=0 ); - if( ((u8)zKey1[i])>((u8)zKey2[i]) ){ + if( cmp>0 ){ /* p2 is smaller */ *ppOut = p2; ppOut = &p2->pScanNext; @@ -432,10 +441,8 @@ static Fts5HashEntry *fts5HashEntryMerge( } /* -** Extract all tokens from hash table iHash and link them into a list -** in sorted order. The hash table is cleared before returning. It is -** the responsibility of the caller to free the elements of the returned -** list. +** Link all tokens from hash table iHash into a list in sorted order. The +** tokens are not removed from the hash table. */ static int fts5HashEntrySort( Fts5Hash *pHash, @@ -457,7 +464,7 @@ static int fts5HashEntrySort( Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ if( pTerm==0 - || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) ){ Fts5HashEntry *pEntry = pIter; pEntry->pScanNext = 0; @@ -475,7 +482,6 @@ static int fts5HashEntrySort( pList = fts5HashEntryMerge(pList, ap[i]); } - pHash->nEntry = 0; sqlite3_free(ap); *ppSorted = pList; return SQLITE_OK; @@ -497,12 +503,11 @@ int sqlite3Fts5HashQuery( for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ zKey = fts5EntryKey(p); - assert( p->nKey+1==(int)strlen(zKey) ); - if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break; + if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break; } if( p ){ - int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1; + int nHashPre = sizeof(Fts5HashEntry) + nTerm; int nList = p->nData - nHashPre; u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); if( pRet ){ @@ -529,6 +534,28 @@ int sqlite3Fts5HashScanInit( return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); } +#ifdef SQLITE_DEBUG +static int fts5HashCount(Fts5Hash *pHash){ + int nEntry = 0; + int ii; + for(ii=0; iinSlot; ii++){ + Fts5HashEntry *p = 0; + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ + nEntry++; + } + } + return nEntry; +} +#endif + +/* +** Return true if the hash table is empty, false otherwise. +*/ +int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ + assert( pHash->nEntry==fts5HashCount(pHash) ); + return pHash->nEntry==0; +} + void sqlite3Fts5HashScanNext(Fts5Hash *p){ assert( !sqlite3Fts5HashScanEof(p) ); p->pScan = p->pScan->pScanNext; @@ -541,19 +568,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){ void sqlite3Fts5HashScanEntry( Fts5Hash *pHash, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ char *zKey = fts5EntryKey(p); - int nTerm = (int)strlen(zKey); + int nTerm = p->nKey; fts5HashAddPoslistSize(pHash, p, 0); *pzTerm = zKey; - *ppDoclist = (const u8*)&zKey[nTerm+1]; - *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); + *pnTerm = nTerm; + *ppDoclist = (const u8*)&zKey[nTerm]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm); }else{ *pzTerm = 0; + *pnTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index eaeeeff4f7..acd0570a5d 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -56,6 +56,24 @@ #define FTS5_MAX_LEVEL 64 +/* +** There are two versions of the format used for the structure record: +** +** 1. the legacy format, that may be read by all fts5 versions, and +** +** 2. the V2 format, which is used by contentless_delete=1 databases. +** +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy +** format structure record contains a varint - the number of levels in +** the structure. Whereas a V2 structure record contains the constant +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a +** varint has to be at least 16256 to begin with "0xFF". And the default +** maximum number of levels is 64. +** +** See below for more on structure record formats. +*/ +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" + /* ** Details: ** @@ -63,7 +81,7 @@ ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** -** , contains the following 5 types of records. See the comments surrounding +** , contains the following 6 types of records. See the comments surrounding ** the FTS5_*_ROWID macros below for a description of how %_data rowids are ** assigned to each fo them. ** @@ -71,13 +89,13 @@ ** ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists -** of a single 32-bit configuration cookie value followed by a list of -** SQLite varints. If the FTS table features more than one index (because -** there are one or more prefix indexes), it is guaranteed that all share -** the same cookie value. +** of a single 32-bit configuration cookie value followed by a list of +** SQLite varints. +** +** If the structure record is a V2 record, the configuration cookie is +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. ** -** Immediately following the configuration cookie, the record begins with -** three varints: +** Next, the record continues with three varints: ** ** + number of levels, ** + total number of segments on all levels, @@ -92,6 +110,12 @@ ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** +** Then, for V2 structures only: +** +** + lower origin counter value, +** + upper origin counter value, +** + the number of tombstone hash pages. +** ** 2. The Averages Record: ** ** A single record within the %_data table. The data is a list of varints. @@ -207,6 +231,38 @@ ** * A list of delta-encoded varints - the first rowid on each subsequent ** child page. ** +** 6. Tombstone Hash Page +** +** These records are only ever present in contentless_delete=1 tables. +** There are zero or more of these associated with each segment. They +** are used to store the tombstone rowids for rows contained in the +** associated segments. +** +** The set of nHashPg tombstone hash pages associated with a single +** segment together form a single hash table containing tombstone rowids. +** To find the page of the hash on which a key might be stored: +** +** iPg = (rowid % nHashPg) +** +** Then, within page iPg, which has nSlot slots: +** +** iSlot = (rowid / nHashPg) % nSlot +** +** Each tombstone hash page begins with an 8 byte header: +** +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the +** first tombstone hash page for each segment (iPg=0). If set, +** the hash table contains rowid 0. If clear, it does not. +** Rowid 0 is handled specially. +** 2-bytes: unused. +** 4-bytes: Big-endian integer containing number of entries on page. +** +** Following this are nSlot 4 or 8 byte slots (depending on the key-size +** in the first byte of the page header). The number of slots may be +** determined based on the size of the page record and the key-size: +** +** nSlot = (nByte - 8) / key-size */ /* @@ -240,6 +296,7 @@ #define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) #ifdef SQLITE_DEBUG int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } @@ -266,6 +323,9 @@ typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; +typedef struct Fts5TokenDataIter Fts5TokenDataIter; +typedef struct Fts5TokenDataMap Fts5TokenDataMap; +typedef struct Fts5TombstoneArray Fts5TombstoneArray; struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ @@ -275,6 +335,12 @@ struct Fts5Data { /* ** One object per %_data table. +** +** nContentlessDelete: +** The number of contentless delete operations since the most recent +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked +** so that extra auto-merge work can be done by fts5IndexFlush() to +** account for the delete operations. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ @@ -289,9 +355,12 @@ struct Fts5Index { int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ + int nContentlessDelete; /* Number of contentless delete ops */ + int nPendingRow; /* Number of INSERT in hash table */ /* Error state. */ int rc; /* Current error code */ + int flushRc; /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ @@ -300,6 +369,7 @@ struct Fts5Index { sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ sqlite3_stmt *pIdxSelect; + sqlite3_stmt *pIdxNextSelect; int nRead; /* Total number of blocks read */ sqlite3_stmt *pDeleteFromIdx; @@ -323,11 +393,23 @@ struct Fts5DoclistIter { ** The contents of the "structure" record for each index are represented ** using an Fts5Structure record in memory. Which uses instances of the ** other Fts5StructureXXX types as components. +** +** nOriginCntr: +** This value is set to non-zero for structure records created for +** contentlessdelete=1 tables only. In that case it represents the +** origin value to apply to the next top-level segment created. */ struct Fts5StructureSegment { int iSegid; /* Segment id */ int pgnoFirst; /* First leaf page number in segment */ int pgnoLast; /* Last leaf page number in segment */ + + /* contentlessdelete=1 tables only: */ + u64 iOrigin1; + u64 iOrigin2; + int nPgTombstone; /* Number of tombstone hash table pages */ + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ + u64 nEntry; /* Number of rows in this segment */ }; struct Fts5StructureLevel { int nMerge; /* Number of segments in incr-merge */ @@ -337,11 +419,16 @@ struct Fts5StructureLevel { struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ + u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ - Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ + Fts5StructureLevel aLevel[FLEXARRAY]; /* Array of nLevel level objects */ }; +/* Size (in bytes) of an Fts5Structure object holding up to N levels */ +#define SZ_FTS5STRUCTURE(N) \ + (offsetof(Fts5Structure,aLevel) + (N)*sizeof(Fts5StructureLevel)) + /* ** An object of type Fts5SegWriter is used to write to segments. */ @@ -425,6 +512,13 @@ struct Fts5CResult { ** ** iTermIdx: ** Index of current term on iTermLeafPgno. +** +** apTombstone/nTombstone: +** These are used for contentless_delete=1 tables only. When the cursor +** is first allocated, the apTombstone[] array is allocated so that it +** is large enough for all tombstones hash pages associated with the +** segment. The pages themselves are loaded lazily from the database as +** they are required. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -433,6 +527,7 @@ struct Fts5SegIter { Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ i64 iLeafOffset; /* Byte offset within current leaf */ + Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */ /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); @@ -459,6 +554,49 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption found reading blob %lld from table \"%s\"", + iRowid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ROWID(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid) + +static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption on page %d, segment %d, table \"%s\"", + pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ITER(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter) + +static int fts5IndexCorruptIdx(Fts5Index *pIdx){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption in table \"%s\"", pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_IDX(pIdx) fts5IndexCorruptIdx(pIdx) + + +/* +** Array of tombstone pages. Reference counted. +*/ +struct Fts5TombstoneArray { + int nRef; /* Number of pointers to this object */ + int nTombstone; + Fts5Data *apTombstone[FLEXARRAY]; /* Array of tombstone pages */ +}; + +/* Size (in bytes) of an Fts5TombstoneArray holding up to N tombstones */ +#define SZ_FTS5TOMBSTONEARRAY(N) \ + (offsetof(Fts5TombstoneArray,apTombstone)+(N)*sizeof(Fts5Data*)) + /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. @@ -503,9 +641,16 @@ struct Fts5SegIter { ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. +** +** pColset: +** If not NULL, points to an object containing a set of column indices. +** Only matches that occur in one of these columns will be returned. +** The Fts5Iter does not own the Fts5Colset object, and so it is not +** freed when the iterator is closed - it is owned by the upper layer. */ struct Fts5Iter { Fts5IndexIter base; /* Base class containing output vars */ + Fts5TokenDataIter *pTokenDataIter; Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ @@ -520,9 +665,11 @@ struct Fts5Iter { i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ - Fts5SegIter aSeg[1]; /* Array of segment iterators */ + Fts5SegIter aSeg[FLEXARRAY]; /* Array of segment iterators */ }; +/* Size (in bytes) of an Fts5Iter object holding up to N segment iterators */ +#define SZ_FTS5ITER(N) (offsetof(Fts5Iter,aSeg)+(N)*sizeof(Fts5SegIter)) /* ** An instance of the following type is used to iterate through the contents @@ -550,9 +697,13 @@ struct Fts5DlidxLvl { struct Fts5DlidxIter { int nLvl; int iSegid; - Fts5DlidxLvl aLvl[1]; + Fts5DlidxLvl aLvl[FLEXARRAY]; }; +/* Size (in bytes) of an Fts5DlidxIter object with up to N levels */ +#define SZ_FTS5DLIDXITER(N) \ + (offsetof(Fts5DlidxIter,aLvl)+(N)*sizeof(Fts5DlidxLvl)) + static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -562,6 +713,60 @@ static u16 fts5GetU16(const u8 *aIn){ return ((u16)aIn[0] << 8) + aIn[1]; } +/* +** The only argument points to a buffer at least 8 bytes in size. This +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian +** unsigned integer and returns the result. +*/ +static u64 fts5GetU64(u8 *a){ + return ((u64)a[0] << 56) + + ((u64)a[1] << 48) + + ((u64)a[2] << 40) + + ((u64)a[3] << 32) + + ((u64)a[4] << 24) + + ((u64)a[5] << 16) + + ((u64)a[6] << 8) + + ((u64)a[7] << 0); +} + +/* +** The only argument points to a buffer at least 4 bytes in size. This +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian +** unsigned integer and returns the result. +*/ +static u32 fts5GetU32(const u8 *a){ + return ((u32)a[0] << 24) + + ((u32)a[1] << 16) + + ((u32)a[2] << 8) + + ((u32)a[3] << 0); +} + +/* +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU64(u8 *a, u64 iVal){ + a[0] = ((iVal >> 56) & 0xFF); + a[1] = ((iVal >> 48) & 0xFF); + a[2] = ((iVal >> 40) & 0xFF); + a[3] = ((iVal >> 32) & 0xFF); + a[4] = ((iVal >> 24) & 0xFF); + a[5] = ((iVal >> 16) & 0xFF); + a[6] = ((iVal >> 8) & 0xFF); + a[7] = ((iVal >> 0) & 0xFF); +} + +/* +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU32(u8 *a, u32 iVal){ + a[0] = ((iVal >> 24) & 0xFF); + a[1] = ((iVal >> 16) & 0xFF); + a[2] = ((iVal >> 8) & 0xFF); + a[3] = ((iVal >> 0) & 0xFF); +} + /* ** Allocate and return a buffer at least nByte bytes in size. ** @@ -618,11 +823,13 @@ static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ /* ** Close the read-only blob handle, if it is open. */ -void sqlite3Fts5IndexCloseReader(Fts5Index *p){ +static void fts5IndexCloseReader(Fts5Index *p){ if( p->pReader ){ + int rc; sqlite3_blob *pReader = p->pReader; p->pReader = 0; - sqlite3_blob_close(pReader); + rc = sqlite3_blob_close(pReader); + if( p->rc==SQLITE_OK ) p->rc = rc; } } @@ -647,7 +854,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ assert( p->pReader==0 ); p->pReader = pBlob; if( rc!=SQLITE_OK ){ - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); } if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } @@ -666,16 +873,17 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ ** All the reasons those functions might return SQLITE_ERROR - missing ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ - if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT_ROWID(p, iRowid); if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ - int nByte = sqlite3_blob_bytes(p->pReader); - sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + i64 nByte = sqlite3_blob_bytes(p->pReader); + i64 szData = (sizeof(Fts5Data) + 7) & ~7; + i64 nAlloc = szData + nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); if( pRet ){ pRet->nn = nByte; - aOut = pRet->p = (u8*)&pRet[1]; + aOut = pRet->p = (u8*)pRet + szData; }else{ rc = SQLITE_NOMEM; } @@ -698,6 +906,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ } assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) ); return pRet; } @@ -714,7 +923,7 @@ static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = fts5DataRead(p, iRowid); if( pRet ){ if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); fts5DataRelease(pRet); pRet = 0; } @@ -729,9 +938,13 @@ static int fts5IndexPrepareStmt( ){ if( p->rc==SQLITE_OK ){ if( zSql ){ - p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, + int rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB, ppStmt, 0); + /* If this prepare() call fails with SQLITE_ERROR, then one of the + ** %_idx or %_data tables has been removed or modified. Call this + ** corruption. */ + p->rc = (rc==SQLITE_ERROR ? SQLITE_CORRUPT : rc); }else{ p->rc = SQLITE_NOMEM; } @@ -789,10 +1002,17 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ /* ** Remove all records associated with segment iSegid. */ -static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ + int iSegid = pSeg->iSegid; i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; fts5DataDelete(p, iFirst, iLast); + + if( pSeg->nPgTombstone ){ + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); + fts5DataDelete(p, iTomb1, iTomb2); + } if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( @@ -851,7 +1071,7 @@ int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){ static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){ Fts5Structure *p = *pp; if( *pRc==SQLITE_OK && p->nRef>1 ){ - i64 nByte = sizeof(Fts5Structure)+(p->nLevel-1)*sizeof(Fts5StructureLevel); + i64 nByte = SZ_FTS5STRUCTURE(p->nLevel); Fts5Structure *pNew; pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte); if( pNew ){ @@ -903,11 +1123,19 @@ static int fts5StructureDecode( int nSegment = 0; sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ + u64 nOriginCntr = 0; /* Largest origin value seen so far */ /* Grab the cookie value */ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); i = 4; + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ + i += 4; + bStructureV2 = 1; + } + /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); @@ -917,10 +1145,7 @@ static int fts5StructureDecode( ){ return FTS5_CORRUPT; } - nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ - ); + nByte = SZ_FTS5STRUCTURE(nLevel); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ @@ -958,6 +1183,14 @@ static int fts5StructureDecode( i += fts5GetVarint32(&pData[i], pSeg->iSegid); i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( bStructureV2 ){ + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntry); + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); + } if( pSeg->pgnoLastpgnoFirst ){ rc = FTS5_CORRUPT; break; @@ -968,6 +1201,9 @@ static int fts5StructureDecode( } } if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + if( bStructureV2 ){ + pRet->nOriginCntr = nOriginCntr+1; + } if( rc!=SQLITE_OK ){ fts5StructureRelease(pRet); @@ -989,10 +1225,7 @@ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; - sqlite3_int64 nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ - ); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(nLevel+2); pStruct = sqlite3_realloc64(pStruct, nByte); if( pStruct ){ @@ -1049,8 +1282,14 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->rc==SQLITE_OK ){ + if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + }else if( p->rc==SQLITE_CORRUPT_VTAB ){ + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName + ); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ @@ -1180,6 +1419,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); memset(&buf, 0, sizeof(Fts5Buffer)); @@ -1188,9 +1428,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; - if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ sqlite3Fts5Put32(buf.p, iCookie); buf.n = 4; + if( pStruct->nOriginCntr>0 ){ + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); + } fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); @@ -1204,9 +1447,17 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ assert( pLvl->nMerge<=pLvl->nSeg ); for(iSeg=0; iSegnSeg; iSeg++){ - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); + if( pStruct->nOriginCntr>0 ){ + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); + } } } @@ -1349,9 +1600,9 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ } if( iOffnn ){ - i64 iVal; + u64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; - iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + iOff += fts5GetVarint(&pData->p[iOff], &iVal); pLvl->iRowid += iVal; pLvl->iOff = iOff; }else{ @@ -1519,7 +1770,7 @@ static Fts5DlidxIter *fts5DlidxIterInit( int bDone = 0; for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ - sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + sqlite3_int64 nByte = SZ_FTS5DLIDXITER(i+1); Fts5DlidxIter *pNew; pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); @@ -1661,7 +1912,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_ITER(p, pIter); return; } iOff = 4; @@ -1693,7 +1944,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ iOff += fts5GetVarint32(&a[iOff], nNew); if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->term.n = nKeep; @@ -1729,6 +1980,25 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ } } +/* +** Allocate a tombstone hash page array object (pIter->pTombArray) for +** the iterator passed as the second argument. If an OOM error occurs, +** leave an error in the Fts5Index object. +*/ +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ + const i64 nTomb = (i64)pIter->pSeg->nPgTombstone; + if( nTomb>0 ){ + i64 nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); + Fts5TombstoneArray *pNew; + pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pNew ){ + pNew->nTombstone = nTomb; + pNew->nRef = 1; + pIter->pTombArray = pNew; + } + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in ** segment pSeg. The iterator is left pointing to the first entry when @@ -1770,6 +2040,7 @@ static void fts5SegIterInit( pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); + fts5SegIterAllocTombstone(p, pIter); } } @@ -1803,6 +2074,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ while( 1 ){ u64 iDelta = 0; + if( i>=n ) break; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ if( i=pNew->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); }else{ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -1965,7 +2237,7 @@ static void fts5SegIterNext_None( if( iOffiEndofDoclist ){ /* Next entry is on the current page */ - i64 iDelta; + u64 iDelta; iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); pIter->iLeafOffset = iOff; pIter->iRowid += iDelta; @@ -1980,15 +2252,16 @@ static void fts5SegIterNext_None( }else{ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList; sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); if( pList==0 ) goto next_none_eof; pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList; - sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -2054,11 +2327,12 @@ static void fts5SegIterNext( }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList = 0; assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); } if( pList==0 ){ fts5DataRelease(pIter->pLeaf); @@ -2068,8 +2342,7 @@ static void fts5SegIterNext( pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), - (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); *pbNewTerm = 1; } @@ -2101,7 +2374,7 @@ static void fts5SegIterNext( } assert_nc( iOffszLeaf ); if( iOff>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } } @@ -2209,18 +2482,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - iOff = fts5LeafFirstRowidOff(pLast); - if( iOff>pLast->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; + if( p->rc==SQLITE_OK ){ + iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + FTS5_CORRUPT_ITER(p, pIter); + return; + } + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; - if( fts5LeafIsTermless(pLast) ){ - pIter->iEndofDoclist = pLast->nn+1; - }else{ - pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } } } @@ -2290,7 +2565,7 @@ static void fts5LeafSeek( iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -2333,7 +2608,7 @@ static void fts5LeafSeek( iOff = iTermOff; if( iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -2355,7 +2630,7 @@ static void fts5LeafSeek( iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; }else{ nKeep = 0; @@ -2370,7 +2645,7 @@ static void fts5LeafSeek( search_success: if( (i64)iOff+nNew>n || nNew<1 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->iLeafOffset = iOff + nNew; @@ -2455,7 +2730,7 @@ static void fts5SegIterSeekInit( fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); } - if( p->rc==SQLITE_OK && bGe==0 ){ + if( p->rc==SQLITE_OK && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM)) ){ pIter->flags |= FTS5_SEGITER_ONETERM; if( pIter->pLeaf ){ if( flags & FTS5INDEX_QUERY_DESC ){ @@ -2471,6 +2746,9 @@ static void fts5SegIterSeekInit( } fts5SegIterSetNext(p, pIter); + if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM) ){ + fts5SegIterAllocTombstone(p, pIter); + } /* Either: ** @@ -2487,6 +2765,79 @@ static void fts5SegIterSeekInit( ); } + +/* +** SQL used by fts5SegIterNextInit() to find the page to open. +*/ +static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){ + if( p->pIdxNextSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term>? ORDER BY term ASC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + + } + return p->pIdxNextSelect; +} + +/* +** This is similar to fts5SegIterSeekInit(), except that it initializes +** the segment iterator to point to the first term following the page +** with pToken/nToken on it. +*/ +static void fts5SegIterNextInit( + Fts5Index *p, + const char *pTerm, int nTerm, + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = -1; /* Page of segment to open */ + int bDlidx = 0; + sqlite3_stmt *pSel = 0; /* SELECT to find iPg */ + + pSel = fts5IdxNextStmt(p); + if( pSel ){ + assert( p->rc==SQLITE_OK ); + sqlite3_bind_int(pSel, 1, pSeg->iSegid); + sqlite3_bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC); + + if( sqlite3_step(pSel)==SQLITE_ROW ){ + i64 val = sqlite3_column_int64(pSel, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pSel); + sqlite3_bind_null(pSel, 2); + if( p->rc ) return; + } + + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + pIter->flags |= FTS5_SEGITER_ONETERM; + if( iPg>=0 ){ + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + fts5SegIterSetNext(p, pIter); + } + if( pIter->pLeaf ){ + const u8 *a = pIter->pLeaf->p; + int iTermOff = 0; + + pIter->iPgidxOff = pIter->pLeaf->szLeaf; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff); + pIter->iLeafOffset = iTermOff; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + if( bDlidx ) fts5SegIterLoadDlidx(p, pIter); + + assert( p->rc!=SQLITE_OK || + fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0 + ); + } +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within the ** in-memory hash table. If there is no such term in the hash-table, the @@ -2513,14 +2864,21 @@ static void fts5SegIterHashInit( const u8 *pList = 0; p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); - sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); - n = (z ? (int)strlen((const char*)z) : 0); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList); if( pList ){ pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf ){ pLeaf->p = (u8*)pList; } } + + /* The call to sqlite3Fts5HashScanInit() causes the hash table to + ** fill the size field of all existing position lists. This means they + ** can no longer be appended to. Since the only scenario in which they + ** can be appended to is if the previous operation on this table was + ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this + ** possibility altogether. */ + p->bDelete = 0; }else{ p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), (const char*)pTerm, nTerm, (void**)&pLeaf, &nList @@ -2551,6 +2909,37 @@ static void fts5SegIterHashInit( fts5SegIterSetNext(p, pIter); } +/* +** Array ap[] contains n elements. Release each of these elements using +** fts5DataRelease(). Then free the array itself using sqlite3_free(). +*/ +static void fts5IndexFreeArray(Fts5Data **ap, int n){ + if( ap ){ + int ii; + for(ii=0; iinRef--; + if( p->nRef<=0 ){ + int ii; + for(ii=0; iinTombstone; ii++){ + fts5DataRelease(p->apTombstone[ii]); + } + sqlite3_free(p); + } + } +} + /* ** Zero the iterator passed as the only argument. */ @@ -2558,6 +2947,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); fts5DataRelease(pIter->pLeaf); fts5DataRelease(pIter->pNextLeaf); + fts5TombstoneArrayDelete(pIter->pTombArray); fts5DlidxIterFree(pIter->pDlidx); sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); @@ -2691,7 +3081,6 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ - p1->bDel = p2->bDel; return i2; } res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; @@ -2721,7 +3110,7 @@ static void fts5SegIterGotoPage( assert( iLeafPgno>pIter->iLeafPgno ); if( iLeafPgno>pIter->pSeg->pgnoLast ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; @@ -2736,7 +3125,7 @@ static void fts5SegIterGotoPage( u8 *a = pIter->pLeaf->p; int n = pIter->pLeaf->szLeaf; if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; @@ -2803,7 +3192,6 @@ static void fts5SegIterNextFrom( }while( p->rc==SQLITE_OK ); } - /* ** Free the iterator object passed as the second argument. */ @@ -2895,6 +3283,85 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){ pIter->iSwitchRowid = pSeg->iRowid; } +/* +** The argument to this macro must be an Fts5Data structure containing a +** tombstone hash page. This macro returns the key-size of the hash-page. +*/ +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) + +#define TOMBSTONE_NSLOT(pPg) \ + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) + +/* +** Query a single tombstone hash table for rowid iRowid. Return true if +** it is found or false otherwise. The tombstone hash table is one of +** nHashTable tables. +*/ +static int fts5IndexTombstoneQuery( + Fts5Data *pHash, /* Hash table page to query */ + int nHashTable, /* Number of pages attached to segment */ + u64 iRowid /* Rowid to query hash for */ +){ + const int szKey = TOMBSTONE_KEYSIZE(pHash); + const int nSlot = TOMBSTONE_NSLOT(pHash); + int iSlot = (iRowid / nHashTable) % nSlot; + int nCollide = nSlot; + + if( iRowid==0 ){ + return pHash->p[1]; + }else if( szKey==4 ){ + u32 *aSlot = (u32*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + }else{ + u64 *aSlot = (u64*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + } + + return 0; +} + +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + Fts5TombstoneArray *pArray = pSeg->pTombArray; + + if( pSeg->pLeaf && pArray ){ + /* Figure out which page the rowid might be present on. */ + int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone; + assert( iPg>=0 ); + + /* If tombstone hash page iPg has not yet been loaded from the + ** database, load it now. */ + if( pArray->apTombstone[iPg]==0 ){ + pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) + ); + if( pArray->apTombstone[iPg]==0 ) return 0; + } + + return fts5IndexTombstoneQuery( + pArray->apTombstone[iPg], + pArray->nTombstone, + pSeg->iRowid + ); + } + + return 0; +} + /* ** Move the iterator to the next entry. ** @@ -2932,7 +3399,9 @@ static void fts5MultiIterNext( fts5AssertMultiIterSetup(p, pIter); assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); - if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + if( (pIter->bSkipEmpty==0 || pSeg->nPos) + && 0==fts5MultiIterIsDeleted(pIter) + ){ pIter->xSetOutputs(pIter, pSeg); return; } @@ -2964,7 +3433,9 @@ static void fts5MultiIterNext2( } fts5AssertMultiIterSetup(p, pIter); - }while( fts5MultiIterIsEmpty(p, pIter) ); + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) + && (p->rc==SQLITE_OK) + ); } } @@ -2977,12 +3448,11 @@ static Fts5Iter *fts5MultiIterAlloc( int nSeg ){ Fts5Iter *pNew; - int nSlot; /* Power of two >= nSeg */ + i64 nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlotaSeg[] */ + SZ_FTS5ITER(nSlot) + /* pNew + pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ @@ -3134,7 +3604,7 @@ static void fts5ChunkIterate( if( nRem<=0 ){ break; }else if( pSeg->pSeg==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); return; }else{ pgno++; @@ -3422,6 +3892,32 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ } } +/* +** All the component segment-iterators of pIter have been set up. This +** functions finishes setup for iterator pIter itself. +*/ +static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){ + int iIter; + for(iIter=pIter->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pIter, iEq, iIter); + } + } + fts5MultiIterSetEof(pIter); + fts5AssertMultiIterSetup(p, pIter); + + if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter)) + || fts5MultiIterIsDeleted(pIter) + ){ + fts5MultiIterNext(p, pIter, 0, 0); + }else if( pIter->base.bEof==0 ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + pIter->xSetOutputs(pIter, pSeg); + } +} /* ** Allocate a new Fts5Iter object. @@ -3503,29 +3999,12 @@ static void fts5MultiIterNew( assert( iIter==nSeg ); } - /* If the above was successful, each component iterators now points + /* If the above was successful, each component iterator now points ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ - for(iIter=pNew->nSeg-1; iIter>0; iIter--){ - int iEq; - if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - Fts5SegIter *pSeg = &pNew->aSeg[iEq]; - if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); - fts5MultiIterAdvanced(p, pNew, iEq, iIter); - } - } - fts5MultiIterSetEof(pNew); - fts5AssertMultiIterSetup(p, pNew); - - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ - fts5MultiIterNext(p, pNew, 0, 0); - }else if( pNew->base.bEof==0 ){ - Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; - pNew->xSetOutputs(pNew, pSeg); - } - + fts5MultiIterFinishSetup(p, pNew); }else{ fts5MultiIterFree(pNew); *ppOut = 0; @@ -3550,7 +4029,6 @@ static void fts5MultiIterNew2( pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; - pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; @@ -3697,7 +4175,10 @@ static void fts5IndexDiscardData(Fts5Index *p){ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; + p->nPendingRow = 0; + p->flushRc = SQLITE_OK; } + p->nContentlessDelete = 0; } /* @@ -3911,7 +4392,7 @@ static void fts5WriteDlidxAppend( } if( pDlidx->bPrevValid ){ - iVal = iRowid - pDlidx->iPrev; + iVal = (u64)iRowid - (u64)pDlidx->iPrev; }else{ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); @@ -4098,7 +4579,7 @@ static void fts5WriteAppendPoslistData( const u8 *a = aData; int n = nData; - assert( p->pConfig->pgsz>0 ); + assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK ); while( p->rc==SQLITE_OK && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz ){ @@ -4226,7 +4707,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ ** a single page has been assigned to more than one segment. In ** this case a prior iteration of this loop may have corrupted the ** segment currently being trimmed. */ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iLeafRowid); }else{ fts5BufferZero(&buf); fts5BufferGrow(&p->rc, &buf, pData->nn); @@ -4334,6 +4815,12 @@ static void fts5IndexMergeLevel( /* Read input from all segments in the input level */ nInput = pLvl->nSeg; + + /* Set the range of origins that will go into the output segment. */ + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; + } } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); @@ -4393,8 +4880,11 @@ static void fts5IndexMergeLevel( int i; /* Remove the redundant segments from the %_data table */ + assert( pSeg->nEntry==0 ); for(i=0; iaSeg[i].iSegid); + Fts5StructureSegment *pOld = &pLvl->aSeg[i]; + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); + fts5DataRemoveSegment(p, pOld); } /* Remove the redundant segments from the input level */ @@ -4420,6 +4910,48 @@ static void fts5IndexMergeLevel( if( pnRem ) *pnRem -= writer.nLeafWritten; } +/* +** If this is not a contentless_delete=1 table, or if the 'deletemerge' +** configuration option is set to 0, then this function always returns -1. +** Otherwise, it searches the structure object passed as the second argument +** for a level suitable for merging due to having a large number of +** tombstones in the tombstone hash. If one is found, its index is returned. +** Otherwise, if there is no suitable level, -1. +*/ +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ + Fts5Config *pConfig = p->pConfig; + int iRet = -1; + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ + int ii; + int nBest = 0; + + for(ii=0; iinLevel; ii++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; + i64 nEntry = 0; + i64 nTomb = 0; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + nEntry += pLvl->aSeg[iSeg].nEntry; + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; + } + assert_nc( nEntry>0 || pLvl->nSeg==0 ); + if( nEntry>0 ){ + int nPercent = (nTomb * 100) / nEntry; + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ + iRet = ii; + nBest = nPercent; + } + } + + /* If pLvl is already the input level to an ongoing merge, look no + ** further for a merge candidate. The caller should be allowed to + ** continue merging from pLvl first. */ + if( pLvl->nMerge ) break; + } + } + return iRet; +} + /* ** Do up to nPg pages of automerge work on the index. ** @@ -4439,14 +4971,15 @@ static int fts5IndexMerge( int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ - /* Set iBestLvl to the level to read input segments from. */ + /* Set iBestLvl to the level to read input segments from. Or to -1 if + ** there is no level suitable to merge segments from. */ assert( pStruct->nLevel>0 ); for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; if( pLvl->nMerge ){ if( pLvl->nMerge>nBest ){ iBestLvl = iLvl; - nBest = pLvl->nMerge; + nBest = nMin; } break; } @@ -4455,22 +4988,18 @@ static int fts5IndexMerge( iBestLvl = iLvl; } } - - /* If nBest is still 0, then the index must be empty. */ -#ifdef SQLITE_DEBUG - for(iLvl=0; nBest==0 && iLvlnLevel; iLvl++){ - assert( pStruct->aLevel[iLvl].nSeg==0 ); + if( nBestaLevel[iBestLvl].nMerge==0 ){ - break; - } + if( iBestLvl<0 ) break; bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } + + if( nMin==1 ) nMin = 2; } *ppStruct = pStruct; return bRet; @@ -4529,6 +5058,14 @@ static int fts5IndexReturn(Fts5Index *p){ return rc; } +/* +** Close the read-only blob handle, if it is open. +*/ +void sqlite3Fts5IndexCloseReader(Fts5Index *p){ + fts5IndexCloseReader(p); + fts5IndexReturn(p); +} + typedef struct Fts5FlushCtx Fts5FlushCtx; struct Fts5FlushCtx { Fts5Index *pIdx; @@ -4636,8 +5173,8 @@ static void fts5SecureDeleteOverflow( pLeaf = 0; }else if( bDetailNone ){ break; - }else if( iNext>=pLeaf->szLeaf || iNext<4 ){ - p->rc = FTS5_CORRUPT; + }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ + FTS5_CORRUPT_ROWID(p, iRowid); break; }else{ int nShift = iNext - 4; @@ -4655,9 +5192,13 @@ static void fts5SecureDeleteOverflow( int i1 = pLeaf->szLeaf; int i2 = 0; + i1 += fts5GetVarint32(&aPg[i1], iFirst); + if( iFirstrc, (pLeaf->nn-pLeaf->szLeaf)+2); if( aIdx==0 ) break; - i1 += fts5GetVarint32(&aPg[i1], iFirst); i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); if( i1nn ){ memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); @@ -4702,7 +5243,6 @@ static void fts5DoSecureDelete( int iPgIdx = pSeg->pLeaf->szLeaf; u64 iDelta = 0; - u64 iNextDelta = 0; int iNextOff = 0; int iOff = 0; int nIdx = 0; @@ -4710,12 +5250,10 @@ static void fts5DoSecureDelete( int bLastInDoclist = 0; int iIdx = 0; int iStart = 0; - int iKeyOff = 0; - int iPrevKeyOff = 0; int iDelKeyOff = 0; /* Offset of deleted key, if any */ nIdx = nPg-iPgIdx; - aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + aIdx = sqlite3Fts5MallocZero(&p->rc, ((i64)nIdx)+16); if( p->rc ) return; memcpy(aIdx, &aPg[iPgIdx], nIdx); @@ -4736,10 +5274,21 @@ static void fts5DoSecureDelete( ** This block sets the following variables: ** ** iStart: + ** The offset of the first byte of the rowid or delta-rowid + ** value for the doclist entry being removed. + ** ** iDelta: + ** The value of the rowid or delta-rowid value for the doclist + ** entry being removed. + ** + ** iNextOff: + ** The offset of the next entry following the position list + ** for the one being removed. If the position list for this + ** entry overflows onto the next leaf page, this value will be + ** greater than pLeaf->szLeaf. */ { - int iSOP; + int iSOP; /* Start-Of-Position-list */ if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ iStart = pSeg->iTermLeafOffset; }else{ @@ -4775,47 +5324,81 @@ static void fts5DoSecureDelete( } iOff = iStart; + + /* If the position-list for the entry being removed flows over past + ** the end of this page, delete the portion of the position-list on the + ** next page and beyond. + ** + ** Set variable bLastInDoclist to true if this entry happens + ** to be the last rowid in the doclist for its term. */ if( iNextOff>=iPgIdx ){ int pgno = pSeg->iLeafPgno+1; fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); iNextOff = iPgIdx; - }else{ - /* Set bLastInDoclist to true if the entry being removed is the last - ** in its doclist. */ - for(iIdx=0, iKeyOff=0; iIdxbDel==0 ){ + if( iNextOff!=iPgIdx ){ + /* Loop through the page-footer. If iNextOff (offset of the + ** entry following the one we are removing) is equal to the + ** offset of a key on this page, then the entry is the last + ** in its doclist. */ + int iKeyOff = 0; + for(iIdx=0; iIdxbDel ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta); + aPg[iOff++] = 0x01; + }else if( bLastInDoclist==0 ){ if( iNextOff!=iPgIdx ){ + u64 iNextDelta = 0; iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta); iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta); } }else if( - iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + pSeg->iLeafPgno==pSeg->iTermLeafPgno + && iStart==pSeg->iTermLeafOffset ){ /* The entry being removed was the only position list in its ** doclist. Therefore the term needs to be removed as well. */ int iKey = 0; - for(iIdx=0, iKeyOff=0; iIdx(u32)iStart ) break; iKeyOff += iVal; } + assert_nc( iKey>=1 ); + /* Set iDelKeyOff to the value of the footer entry to remove from + ** the page. */ iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + /* This is the only position-list associated with the term, and there + ** is another term following it on this page. So the subsequent term + ** needs to be moved to replace the term associated with the entry + ** being removed. */ int nPrefix = 0; int nSuffix = 0; int nPrefix2 = 0; @@ -4834,13 +5417,15 @@ static void fts5DoSecureDelete( nSuffix = (nPrefix2 + nSuffix2) - nPrefix; if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>nPrefix ){ + if( nPrefix2>pSeg->term.n ){ + FTS5_CORRUPT_IDX(p); + }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); } @@ -4850,97 +5435,126 @@ static void fts5DoSecureDelete( } } }else if( iStart==4 ){ - int iPgno; - - assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); - /* The entry being removed may be the only position list in - ** its doclist. */ - for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ - Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); - int bEmpty = (pPg && pPg->nn==4); - fts5DataRelease(pPg); - if( bEmpty==0 ) break; - } - - if( iPgno==pSeg->iTermLeafPgno ){ - i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); - Fts5Data *pTerm = fts5DataRead(p, iId); - if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ - u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; - int nTermIdx = pTerm->nn - pTerm->szLeaf; - int iTermIdx = 0; - int iTermOff = 0; - - while( 1 ){ - u32 iVal = 0; - int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); - iTermOff += iVal; - if( (iTermIdx+nByte)>=nTermIdx ) break; - iTermIdx += nByte; - } - nTermIdx = iTermIdx; + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); - } + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); } - fts5DataRelease(pTerm); } + fts5DataRelease(pTerm); } + } - if( p->rc==SQLITE_OK ){ - const int nMove = nPg - iNextOff; - int nShift = 0; + /* Assuming no error has occurred, this block does final edits to the + ** leaf page before writing it back to disk. Input variables are: + ** + ** nPg: Total initial size of leaf page. + ** iPgIdx: Initial offset of page footer. + ** + ** iOff: Offset to move data to + ** iNextOff: Offset to move data from + */ + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; /* Number of bytes to move */ + int nShift = iNextOff - iOff; /* Distance to move them */ - memmove(&aPg[iOff], &aPg[iNextOff], nMove); - iPgIdx -= (iNextOff - iOff); - nPg = iPgIdx; - fts5PutU16(&aPg[2], iPgIdx); + int iPrevKeyOut = 0; + int iKeyIn = 0; - nShift = iNextOff - iOff; - for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ - iKeyOff -= nShift; - nShift = 0; - } - nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); - iPrevKeyOff = iKeyOff; - } - } + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= nShift; + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); - if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + for(iIdx=0; iIdxiOff ? nShift : 0)); + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut); + iPrevKeyOut = iKeyOut; } + } - assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); - fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); } - sqlite3_free(aIdx); + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + } + sqlite3_free(aIdx); } /* ** This is called as part of flushing a delete to disk in 'secure-delete' ** mode. It edits the segments within the database described by argument ** pStruct to remove the entries for term zTerm, rowid iRowid. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** has occurred. Any error code is also stored in the Fts5Index handle. */ -static void fts5FlushSecureDelete( +static int fts5FlushSecureDelete( Fts5Index *p, Fts5Structure *pStruct, const char *zTerm, + int nTerm, i64 iRowid ){ const int f = FTS5INDEX_QUERY_SKIPHASH; - int nTerm = (int)strlen(zTerm); Fts5Iter *pIter = 0; /* Used to find term instance */ + /* If the version number has not been set to SECUREDELETE, do so now. */ + if( p->pConfig->iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE ){ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pStmt = 0; + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "REPLACE INTO %Q.'%q_config' VALUES ('version', %d)", + pConfig->zDb, pConfig->zName, FTS5_CURRENT_VERSION_SECUREDELETE + )); + if( p->rc==SQLITE_OK ){ + int rc; + sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc; + pConfig->iCookie++; + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); if( fts5MultiIterEof(p, pIter)==0 ){ i64 iThis = fts5MultiIterRowid(pIter); @@ -4958,6 +5572,7 @@ static void fts5FlushSecureDelete( } fts5MultiIterFree(pIter); + return p->rc; } @@ -4977,184 +5592,199 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); - iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); - if( iSegid ){ - const int pgsz = p->pConfig->pgsz; - int eDetail = p->pConfig->eDetail; - int bSecureDelete = p->pConfig->bSecureDelete; - Fts5StructureSegment *pSeg; /* New segment within pStruct */ - Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ - Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ - - Fts5SegWriter writer; - fts5WriteInit(p, &writer, iSegid); - - pBuf = &writer.writer.buf; - pPgidx = &writer.writer.pgidx; - - /* fts5WriteInit() should have initialized the buffers to (most likely) - ** the maximum space required. */ - assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - - /* Begin scanning through hash table entries. This loop runs once for each - ** term/doclist currently stored within the hash table. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); - } - while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ - const char *zTerm; /* Buffer containing term */ - int nTerm; /* Size of zTerm in bytes */ - const u8 *pDoclist; /* Pointer to doclist for this term */ - int nDoclist; /* Size of doclist in bytes */ - - /* Get the term and doclist for this entry. */ - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - nTerm = (int)strlen(zTerm); - if( bSecureDelete==0 ){ - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; - assert( writer.bFirstRowidInPage==0 ); - } - - if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ - /* The entire doclist will fit on the current leaf. */ - fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); - }else{ - int bTermWritten = !bSecureDelete; - i64 iRowid = 0; - i64 iPrev = 0; - int iOff = 0; - - /* The entire doclist will not fit on this leaf. The following - ** loop iterates through the poslists that make up the current - ** doclist. */ - while( p->rc==SQLITE_OK && iOffpConfig->pgsz; + int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); + + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ + + /* Get the term and doclist for this entry. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } + + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + int bTermWritten = !bSecureDelete; + i64 iRowid = 0; + i64 iPrev = 0; + int iOff = 0; + + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ iOff++; - nDoclist = 0; - }else{ continue; } } - }else if( (pDoclist[iOff] & 0x01) ){ - fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); - if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ - iOff++; - continue; - } } - } - - if( p->rc==SQLITE_OK && bTermWritten==0 ){ - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - bTermWritten = 1; - assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); - } - - if( writer.bFirstRowidInPage ){ - fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); - writer.bFirstRowidInPage = 0; - fts5WriteDlidxAppend(p, &writer, iRowid); - }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); - } - if( p->rc!=SQLITE_OK ) break; - assert( pBuf->n<=pBuf->nSpace ); - iPrev = iRowid; - - if( eDetail==FTS5_DETAIL_NONE ){ - if( iOffp[pBuf->n++] = 0; - iOff++; + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } + + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + u64 iRowidDelta = (u64)iRowid - (u64)iPrev; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta); + } + if( p->rc!=SQLITE_OK ) break; + assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ if( iOffp[pBuf->n++] = 0; iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); } - } - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - }else{ - int bDummy; - int nPos; - int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); - nCopy += nPos; - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); - } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); + int bDel = 0; + int nPos = 0; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel); + if( bDel && bSecureDelete ){ + fts5BufferAppendVarint(&p->rc, pBuf, nPos*2); + iOff += nCopy; + nCopy = nPos; + }else{ + nCopy += nPos; + } + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; } - if( iPos>=nCopy ) break; } + iOff += nCopy; } - iOff += nCopy; } } + + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); } - - /* TODO2: Doclist terminator written here. */ - /* pBuf->p[pBuf->n++] = '\0'; */ - assert( pBuf->n<=pBuf->nSpace ); - if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); - } - sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &pgnoLast); - - assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); - if( pgnoLast>0 ){ - /* Update the Fts5Structure. It is written back to the database by the - ** fts5StructureRelease() call below. */ - if( pStruct->nLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); - } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; + fts5WriteFinish(p, &writer, &pgnoLast); + + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pStruct->nOriginCntr; + pSeg->iOrigin2 = pStruct->nOriginCntr; + pSeg->nEntry = p->nPendingRow; + pStruct->nOriginCntr++; + } + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructurePromote(p, 0, pStruct); } } - fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); @@ -5165,10 +5795,21 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ - if( p->nPendingData ){ + if( p->flushRc ){ + p->rc = p->flushRc; + return; + } + if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); - p->nPendingData = 0; fts5FlushOneHash(p); + if( p->rc==SQLITE_OK ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + p->nPendingRow = 0; + p->nContentlessDelete = 0; + }else if( p->nPendingData || p->nContentlessDelete ){ + p->flushRc = p->rc; + } } } @@ -5177,31 +5818,37 @@ static Fts5Structure *fts5IndexOptimizeStruct( Fts5Structure *pStruct ){ Fts5Structure *pNew = 0; - sqlite3_int64 nByte = sizeof(Fts5Structure); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(1); int nSeg = pStruct->nSegment; int i; /* Figure out if this structure requires optimization. A structure does ** not require optimization if either: ** - ** + it consists of fewer than two segments, or - ** + all segments are on the same level, or - ** + all segments except one are currently inputs to a merge operation. + ** 1. it consists of fewer than two segments, or + ** 2. all segments are on the same level, or + ** 3. all segments except one are currently inputs to a merge operation. ** - ** In the first case, return NULL. In the second, increment the ref-count - ** on *pStruct and return a copy of the pointer to it. + ** In the first case, if there are no tombstone hash pages, return NULL. In + ** the second, increment the ref-count on *pStruct and return a copy of the + ** pointer to it. */ - if( nSeg<2 ) return 0; + if( nSeg==0 ) return 0; for(i=0; inLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; - if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + int nMerge = pStruct->aLevel[i].nMerge; + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ + return 0; + } fts5StructureRef(pStruct); return pStruct; } assert( pStruct->aLevel[i].nMerge<=nThis ); } - nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); + assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -5210,6 +5857,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; + pNew->nOriginCntr = pStruct->nOriginCntr; pLvl = &pNew->aLevel[pNew->nLevel-1]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ @@ -5240,7 +5888,9 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); + assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 ); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || pStruct!=0 ); fts5StructureInvalidate(p); if( pStruct ){ @@ -5269,7 +5919,10 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ ** INSERT command. */ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct = fts5StructureRead(p); + Fts5Structure *pStruct = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; fts5StructureInvalidate(p); @@ -5277,8 +5930,8 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); pStruct = pNew; - nMin = 2; - nMerge = nMerge*-1; + nMin = 1; + nMerge = (nMerge==SMALLEST_INT32 ? LARGEST_INT32 : (nMerge*-1)); } if( pStruct && pStruct->nLevel ){ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ @@ -5564,7 +6217,7 @@ static void fts5MergePrefixLists( } if( pHead==0 || pHead->pNext==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); break; } @@ -5601,7 +6254,7 @@ static void fts5MergePrefixLists( assert_nc( tmp.n+nTail<=nTmp ); assert( tmp.n+nTail<=nTmp+nMerge*10 ); if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_IDX(p); break; } fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); @@ -5636,6 +6289,387 @@ static void fts5MergePrefixLists( *p1 = out; } + +/* +** Iterate through a range of entries in the FTS index, invoking the xVisit +** callback for each of them. +** +** Parameter pToken points to an nToken buffer containing an FTS index term +** (i.e. a document term with the preceding 1 byte index identifier - +** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits +** all entries for terms that have pToken/nToken as a prefix. If bPrefix +** is false, then only entries with pToken/nToken as the entire key are +** visited. +** +** If the current table is a tokendata=1 table, then if bPrefix is true then +** each index term is treated separately. However, if bPrefix is false, then +** all index terms corresponding to pToken/nToken are collapsed into a single +** term before the callback is invoked. +** +** The callback invoked for each entry visited is specified by paramter xVisit. +** Each time it is invoked, it is passed a pointer to the Fts5Index object, +** a copy of the 7th paramter to this function (pCtx) and a pointer to the +** iterator that indicates the current entry. If the current entry is the +** first with a new term (i.e. different from that of the previous entry, +** including the very first term), then the final two parameters are passed +** a pointer to the term and its size in bytes, respectively. If the current +** entry is not the first associated with its term, these two parameters +** are passed 0. +** +** If parameter pColset is not NULL, then it is used to filter entries before +** the callback is invoked. +*/ +static int fts5VisitEntries( + Fts5Index *p, /* Fts5 index object */ + Fts5Colset *pColset, /* Columns filter to apply, or NULL */ + u8 *pToken, /* Buffer containing token */ + int nToken, /* Size of buffer pToken in bytes */ + int bPrefix, /* True for a prefix scan */ + void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int), + void *pCtx /* Passed as second argument to xVisit() */ +){ + const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN : 0) + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ + int bNewTerm = 1; + Fts5Structure *pStruct = fts5StructureRead(p); + + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nNew = 0; + const u8 *pNew = 0; + + p1->xSetOutputs(p1, pSeg); + if( p->rc ) break; + + if( bNewTerm ){ + nNew = pSeg->term.n; + pNew = pSeg->term.p; + if( nNewrc; +} + + +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits (so all iRowid fields are the same). +** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an +** array of these for the entire query (in which case iRowid fields may take +** a variety of values). +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +** +** iRowid: +** Rowid for the current entry. +** +** iPos: +** Position of current entry within row. In the usual ((iCol<<32)+iOff) +** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()). +** +** iIter: +** If the Fts5TokenDataIter iterator that the entry is part of is +** actually an iterator (i.e. with nIter>0, not just a container for +** Fts5TokenDataMap structures), then this variable is an index into +** the apIter[] array. The corresponding term is that which the iterator +** at apIter[iIter] currently points to. +** +** Or, if the Fts5TokenDataIter iterator is just a container object +** (nIter==0), then iIter is an index into the term.p[] buffer where +** the term is stored. +** +** nByte: +** In the case where iIter is an index into term.p[], this variable +** is the size of the term in bytes. If iIter is an index into apIter[], +** this variable is unused. +*/ +struct Fts5TokenDataMap { + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ + int nByte; /* Length of token in bytes (or 0) */ +}; + +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +** +** This object serves two purposes. The first is as a container for an array +** of Fts5TokenDataMap structures, which are used to find the token required +** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and +** aMap[] variables. +*/ +struct Fts5TokenDataIter { + int nMapAlloc; /* Allocated size of aMap[] in entries */ + int nMap; /* Number of valid entries in aMap[] */ + Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ + + /* The following are used for prefix-queries only. */ + Fts5Buffer terms; + + /* The following are used for other full-token tokendata queries only. */ + int nIter; + int nIterAlloc; + Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; + Fts5Iter *apIter[FLEXARRAY]; +}; + +/* Size in bytes of an Fts5TokenDataIter object holding up to N iterators */ +#define SZ_FTS5TOKENDATAITER(N) \ + (offsetof(Fts5TokenDataIter,apIter) + (N)*sizeof(Fts5Iter)) + +/* +** The two input arrays - a1[] and a2[] - are in sorted order. This function +** merges the two arrays together and writes the result to output array +** aOut[]. aOut[] is guaranteed to be large enough to hold the result. +** +** Duplicate entries are copied into the output. So the size of the output +** array is always (n1+n2) entries. +*/ +static void fts5TokendataMerge( + Fts5TokenDataMap *a1, int n1, /* Input array 1 */ + Fts5TokenDataMap *a2, int n2, /* Input array 2 */ + Fts5TokenDataMap *aOut /* Output array */ +){ + int i1 = 0; + int i2 = 0; + + assert( n1>=0 && n2>=0 ); + while( i1=n2 || (i1rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nAlloc = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->aMap[pT->nMap].nByte = nByte; + pT->nMap++; + } +} + +/* +** Sort the contents of the pT->aMap[] array. +** +** The sorting algorithm requires a malloc(). If this fails, an error code +** is left in Fts5Index.rc before returning. +*/ +static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ + Fts5TokenDataMap *aTmp = 0; + int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + + aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( aTmp ){ + Fts5TokenDataMap *a1 = pT->aMap; + Fts5TokenDataMap *a2 = aTmp; + i64 nHalf; + + for(nHalf=1; nHalfnMap; nHalf=nHalf*2){ + int i1; + for(i1=0; i1nMap; i1+=(nHalf*2)){ + int n1 = MIN(nHalf, pT->nMap-i1); + int n2 = MIN(nHalf, pT->nMap-i1-n1); + fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]); + } + SWAPVAL(Fts5TokenDataMap*, a1, a2); + } + + if( a1!=pT->aMap ){ + memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap)); + } + sqlite3_free(aTmp); + +#ifdef SQLITE_DEBUG + { + int ii; + for(ii=1; iinMap; ii++){ + Fts5TokenDataMap *p1 = &pT->aMap[ii-1]; + Fts5TokenDataMap *p2 = &pT->aMap[ii]; + assert( p1->iRowidiRowid + || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos) + ); + } + } +#endif + } +} + +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + fts5BufferFree(&pSet->terms); + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + + +/* +** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata() +** to pass data to prefixIterSetupTokendataCb(). +*/ +typedef struct TokendataSetupCtx TokendataSetupCtx; +struct TokendataSetupCtx { + Fts5TokenDataIter *pT; /* Object being populated with mappings */ + int iTermOff; /* Offset of current term in terms.p[] */ + int nTermByte; /* Size of current term in bytes */ +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This +** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each +** position in the current position-list. It doesn't matter that some of +** these may be out of order - they will be sorted later. +*/ +static void prefixIterSetupTokendataCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx; + int iPosOff = 0; + i64 iPos = 0; + + if( pNew ){ + pSetup->nTermByte = nNew-1; + pSetup->iTermOff = pSetup->pT->terms.n; + fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1); + } + + while( 0==sqlite3Fts5PoslistNext64( + p1->base.pData, p1->base.nData, &iPosOff, &iPos + ) ){ + fts5TokendataIterAppendMap(p, + pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos + ); + } +} + + +/* +** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries(). +*/ +typedef struct PrefixSetupCtx PrefixSetupCtx; +struct PrefixSetupCtx { + void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); + i64 iLastRowid; + int nMerge; + Fts5Buffer *aBuf; + int nBuf; + Fts5Buffer doclist; + TokendataSetupCtx *pTokendata; +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIter() +*/ +static void prefixIterSetupCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx; + const int nMerge = pSetup->nMerge; + + if( p1->base.nData>0 ){ + if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){ + int i; + for(i=0; p->rc==SQLITE_OK && pSetup->doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=pSetup->nBuf ); + for(iStore=i1; iStoreaBuf[iStore].n==0 ){ + fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]); + fts5BufferZero(&pSetup->doclist); + break; + } + } + if( iStore==i1+nMerge ){ + pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]); + for(iStore=i1; iStoreaBuf[iStore]); + } + } + } + pSetup->iLastRowid = 0; + } + + pSetup->xAppend( + p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist + ); + pSetup->iLastRowid = p1->base.iRowid; + } + + if( pSetup->pTokendata ){ + prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew); + } +} + static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ @@ -5643,129 +6677,91 @@ static void fts5SetupPrefixIter( u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ - Fts5Iter **ppIter /* OUT: New iterator */ + Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; - Fts5Buffer *aBuf; - int nBuf = 32; - int nMerge = 1; + PrefixSetupCtx s; + TokendataSetupCtx s2; + + memset(&s, 0, sizeof(s)); + memset(&s2, 0, sizeof(s2)); + + s.nMerge = 1; + s.iLastRowid = 0; + s.nBuf = 32; + if( iIdx==0 + && p->pConfig->eDetail==FTS5_DETAIL_FULL + && p->pConfig->bPrefixInsttoken + ){ + s.pTokendata = &s2; + s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, SZ_FTS5TOKENDATAITER(1)); + } - void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ - xMerge = fts5MergeRowidLists; - xAppend = fts5AppendRowid; + s.xMerge = fts5MergeRowidLists; + s.xAppend = fts5AppendRowid; }else{ - nMerge = FTS5_MERGE_NLIST-1; - nBuf = nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ - xMerge = fts5MergePrefixLists; - xAppend = fts5AppendPoslist; + s.nMerge = FTS5_MERGE_NLIST-1; + s.nBuf = s.nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ + s.xMerge = fts5MergePrefixLists; + s.xAppend = fts5AppendPoslist; } - aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); + s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) ); - if( aBuf && pStruct ){ - const int flags = FTS5INDEX_QUERY_SCAN - | FTS5INDEX_QUERY_SKIPEMPTY - | FTS5INDEX_QUERY_NOOUTPUT; + if( p->rc==SQLITE_OK ){ + void *pCtx = (void*)&s; int i; - i64 iLastRowid = 0; - Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; - Fts5Buffer doclist; - int bNewTerm = 1; - memset(&doclist, 0, sizeof(doclist)); + /* If iIdx is non-zero, then it is the number of a prefix-index for + ** prefixes 1 character longer than the prefix being queried for. That + ** index contains all the doclists required, except for the one + ** corresponding to the prefix itself. That one is extracted from the + ** main term index here. */ if( iIdx!=0 ){ - int dummy = 0; - const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; pToken[0] = FTS5_MAIN_PREFIX; - fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); - for(; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &dummy) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - p1->xSetOutputs(p1, pSeg); - if( p1->base.nData ){ - xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; - } - } - fts5MultiIterFree(p1); + fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx); } pToken[0] = FTS5_MAIN_PREFIX + iIdx; - fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); - for( /* no-op */ ; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &bNewTerm) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - int nTerm = pSeg->term.n; - const u8 *pTerm = pSeg->term.p; - p1->xSetOutputs(p1, pSeg); - - assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); - if( bNewTerm ){ - if( nTermbase.nData==0 ) continue; - - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ - for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ - int i1 = i*nMerge; - int iStore; - assert( i1+nMerge<=nBuf ); - for(iStore=i1; iStorebase.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; - } - - assert( (nBuf%nMerge)==0 ); - for(i=0; irc==SQLITE_OK ){ - xMerge(p, &doclist, nMerge, &aBuf[i]); + s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]); } - for(iFree=i; iFreerc!=SQLITE_OK ); if( pData ){ pData->p = (u8*)&pData[1]; - pData->nn = pData->szLeaf = doclist.n; - if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); + pData->nn = pData->szLeaf = s.doclist.n; + if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } - fts5BufferFree(&doclist); + + assert( (*ppIter)!=0 || p->rc!=SQLITE_OK ); + if( p->rc==SQLITE_OK && s.pTokendata ){ + fts5TokendataIterSortMap(p, s2.pT); + (*ppIter)->pTokenDataIter = s2.pT; + s2.pT = 0; + } } + fts5TokendataIterDelete(s2.pT); + fts5BufferFree(&s.doclist); fts5StructureRelease(pStruct); - sqlite3_free(aBuf); + sqlite3_free(s.aBuf); } @@ -5784,13 +6780,16 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ /* Flush the hash table to disk if required */ if( iRowidiWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) - || (p->nPendingData > p->pConfig->nHashSize) + || (p->nPendingData > p->pConfig->nHashSize) ){ fts5IndexFlush(p); } p->iWriteRowid = iRowid; p->bDelete = bDelete; + if( bDelete==0 ){ + p->nPendingRow++; + } return fts5IndexReturn(p); } @@ -5800,7 +6799,7 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ int sqlite3Fts5IndexSync(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); return fts5IndexReturn(p); } @@ -5811,11 +6810,10 @@ int sqlite3Fts5IndexSync(Fts5Index *p){ ** records must be invalidated. */ int sqlite3Fts5IndexRollback(Fts5Index *p){ - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); fts5IndexDiscardData(p); fts5StructureInvalidate(p); - /* assert( p->rc==SQLITE_OK ); */ - return SQLITE_OK; + return fts5IndexReturn(p); } /* @@ -5824,12 +6822,20 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ - Fts5Structure s; + Fts5Structure *pTmp; + union { + Fts5Structure sFts; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + } uFts; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - memset(&s, 0, sizeof(Fts5Structure)); + pTmp = &uFts.sFts; + memset(uFts.tmpSpace, 0, sizeof(uFts.tmpSpace)); + if( p->pConfig->bContentlessDelete ){ + pTmp->nOriginCntr = 1; + } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); - fts5StructureWrite(p, &s); + fts5StructureWrite(p, pTmp); return fts5IndexReturn(p); } @@ -5891,6 +6897,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pIdxNextSelect); sqlite3_finalize(p->pDataVersion); sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); @@ -5986,6 +6993,387 @@ int sqlite3Fts5IndexWrite( return rc; } +/* +** pToken points to a buffer of size nToken bytes containing a search +** term, including the index number at the start, used on a tokendata=1 +** table. This function returns true if the term in buffer pBuf matches +** token pToken/nToken. +*/ +static int fts5IsTokendataPrefix( + Fts5Buffer *pBuf, + const u8 *pToken, + int nToken +){ + return ( + pBuf->n>=nToken + && 0==memcmp(pBuf->p, pToken, nToken) + && (pBuf->n==nToken || pBuf->p[nToken]==0x00) + ); +} + +/* +** Ensure the segment-iterator passed as the only argument points to EOF. +*/ +static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; +} + +static void fts5IterClose(Fts5IndexIter *pIndexIter){ + if( pIndexIter ){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *pIndex = pIter->pIndex; + fts5TokendataIterDelete(pIter->pTokenDataIter); + fts5MultiIterFree(pIter); + fts5IndexCloseReader(pIndex); + } +} + +/* +** This function appends iterator pAppend to Fts5TokenDataIter pIn and +** returns the result. +*/ +static Fts5TokenDataIter *fts5AppendTokendataIter( + Fts5Index *p, /* Index object (for error code) */ + Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */ + Fts5Iter *pAppend /* Append this iterator */ +){ + Fts5TokenDataIter *pRet = pIn; + + if( p->rc==SQLITE_OK ){ + if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ + int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); + Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + if( pIn==0 ) memset(pNew, 0, nByte); + pRet = pNew; + pNew->nIterAlloc = nAlloc; + } + } + } + if( p->rc ){ + fts5IterClose((Fts5IndexIter*)pAppend); + }else{ + pRet->apIter[pRet->nIter++] = pAppend; + } + assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc ); + + return pRet; +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function sets the iterator output +** variables (pIter->base.*) according to the contents of the current +** row. +*/ +static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ + int ii; + int nHit = 0; + i64 iRowid = SMALLEST_INT64; + int iMin = 0; + + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + + pIter->base.nData = 0; + pIter->base.pData = 0; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 ){ + if( nHit==0 || p->base.iRowidbase.iRowid; + nHit = 1; + pIter->base.pData = p->base.pData; + pIter->base.nData = p->base.nData; + iMin = ii; + }else if( p->base.iRowid==iRowid ){ + nHit++; + } + } + } + + if( nHit==0 ){ + pIter->base.bEof = 1; + }else{ + int eDetail = pIter->pIndex->pConfig->eDetail; + pIter->base.bEof = 0; + pIter->base.iRowid = iRowid; + + if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1); + }else + if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ + int nReader = 0; + int nByte = 0; + i64 iPrev = 0; + + /* Allocate array of iterators if they are not already allocated. */ + if( pT->aPoslistReader==0 ){ + pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero( + &pIter->pIndex->rc, + pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)) + ); + if( pT->aPoslistReader==0 ) return; + pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter]; + } + + /* Populate an iterator for each poslist that will be merged */ + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( iRowid==p->base.iRowid ){ + pT->aPoslistToIter[nReader] = ii; + sqlite3Fts5PoslistReaderInit( + p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++] + ); + nByte += p->base.nData; + } + } + + /* Ensure the output buffer is large enough */ + if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10) ){ + return; + } + + /* Ensure the token-mapping is large enough */ + if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ + int nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + pT->aMap, nNew*sizeof(Fts5TokenDataMap) + ); + if( aNew==0 ){ + pIter->pIndex->rc = SQLITE_NOMEM; + return; + } + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pIter->poslist.n = 0; + + while( 1 ){ + i64 iMinPos = LARGEST_INT64; + + /* Find smallest position */ + iMin = 0; + for(ii=0; iiaPoslistReader[ii]; + if( pReader->bEof==0 ){ + if( pReader->iPosiPos; + iMin = ii; + } + } + } + + /* If all readers were at EOF, break out of the loop. */ + if( iMinPos==LARGEST_INT64 ) break; + + sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos); + sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]); + + if( eDetail==FTS5_DETAIL_FULL ){ + pT->aMap[pT->nMap].iPos = iMinPos; + pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin]; + pT->aMap[pT->nMap].iRowid = iRowid; + pT->nMap++; + } + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + } +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function advances the iterator. If +** argument bFrom is false, then the iterator is advanced to the next +** entry. Or, if bFrom is true, it is advanced to the first entry with +** a rowid of iFrom or greater. +*/ +static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ + int ii; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 + && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidbase.bEof==0 + && p->base.iRowidrc==SQLITE_OK + ){ + fts5MultiIterNext(pIndex, p, 0, 0); + } + } + } + + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } +} + +/* +** If the segment-iterator passed as the first argument is at EOF, then +** set pIter->term to a copy of buffer pTerm. +*/ +static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ + if( pIter && pIter->aSeg[0].pLeaf==0 ){ + fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p); + } +} + +/* +** This function sets up an iterator to use for a non-prefix query on a +** tokendata=1 table. +*/ +static Fts5Iter *fts5SetupTokendataIter( + Fts5Index *p, /* FTS index to query */ + const u8 *pToken, /* Buffer containing query term */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset /* Colset to filter on */ +){ + Fts5Iter *pRet = 0; + Fts5TokenDataIter *pSet = 0; + Fts5Structure *pStruct = 0; + const int flags = FTS5INDEX_QUERY_SCANONETERM | FTS5INDEX_QUERY_SCAN; + + Fts5Buffer bSeek = {0, 0, 0}; + Fts5Buffer *pSmall = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + + while( p->rc==SQLITE_OK ){ + Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; + Fts5Iter *pNew = 0; + Fts5SegIter *pNewIter = 0; + Fts5SegIter *pPrevIter = 0; + + int iLvl, iSeg, ii; + + pNew = fts5MultiIterAlloc(p, pStruct->nSegment); + if( pSmall ){ + fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); + fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); + }else{ + fts5BufferSet(&p->rc, &bSeek, nToken, pToken); + } + if( p->rc ){ + fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + pNewIter = &pNew->aSeg[0]; + pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + int bDone = 0; + + if( pPrevIter ){ + if( fts5BufferCompare(pSmall, &pPrevIter->term) ){ + memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter)); + memset(pPrevIter, 0, sizeof(Fts5SegIter)); + bDone = 1; + }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){ + fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter); + bDone = 1; + } + } + + if( bDone==0 ){ + fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + } + + if( pPrevIter ){ + if( pPrevIter->pTombArray ){ + pNewIter->pTombArray = pPrevIter->pTombArray; + pNewIter->pTombArray->nRef++; + } + }else{ + fts5SegIterAllocTombstone(p, pNewIter); + } + + pNewIter++; + if( pPrevIter ) pPrevIter++; + if( p->rc ) break; + } + } + fts5TokendataSetTermIfEof(pPrev, pSmall); + + pNew->bSkipEmpty = 1; + pNew->pColset = pColset; + fts5IterSetOutputCb(&p->rc, pNew); + + /* Loop through all segments in the new iterator. Find the smallest + ** term that any segment-iterator points to. Iterator pNew will be + ** used for this term. Also, set any iterator that points to a term that + ** does not match pToken/nToken to point to EOF */ + pSmall = 0; + for(ii=0; iinSeg; ii++){ + Fts5SegIter *pII = &pNew->aSeg[ii]; + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ + fts5SegIterSetEOF(pII); + } + if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ + pSmall = &pII->term; + } + } + + /* If pSmall is still NULL at this point, then the new iterator does + ** not point to any terms that match the query. So delete it and break + ** out of the loop - all required iterators have been collected. */ + if( pSmall==0 ){ + fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + /* Append this iterator to the set and continue. */ + pSet = fts5AppendTokendataIter(p, pSet, pNew); + } + + if( p->rc==SQLITE_OK && pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Iter *pIter = pSet->apIter[ii]; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM; + } + fts5MultiIterFinishSetup(p, pIter); + } + } + + if( p->rc==SQLITE_OK ){ + pRet = fts5MultiIterAlloc(p, 0); + } + if( pRet ){ + pRet->nSeg = 0; + pRet->pTokenDataIter = pSet; + if( pSet ){ + fts5IterSetOutputsTokendata(pRet); + }else{ + pRet->base.bEof = 1; + } + }else{ + fts5TokendataIterDelete(pSet); + } + + fts5StructureRelease(pStruct); + fts5BufferFree(&bSeek); + return pRet; +} + /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -6007,8 +7395,19 @@ int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ int iPrefixIdx = 0; /* +1 prefix index */ + int bTokendata = pConfig->bTokendata; + assert( buf.p!=0 ); if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + /* The NOTOKENDATA flag is set when each token in a tokendata=1 table + ** should be treated individually, instead of merging all those with + ** a common prefix into a single entry. This is used, for example, by + ** queries performed as part of an integrity-check, or by the fts5vocab + ** module. */ + if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ + bTokendata = 0; + } + /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to ** greater than pConfig->nPrefix to indicate that the query will be @@ -6034,7 +7433,10 @@ int sqlite3Fts5IndexQuery( } } - if( iIdx<=pConfig->nPrefix ){ + if( bTokendata && iIdx==0 ){ + buf.p[0] = FTS5_MAIN_PREFIX; + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); + }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); @@ -6045,7 +7447,7 @@ int sqlite3Fts5IndexQuery( fts5StructureRelease(pStruct); } }else{ - /* Scan multiple terms in the main index */ + /* Scan multiple terms in the main index for a prefix query. */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); if( pRet==0 ){ @@ -6061,9 +7463,9 @@ int sqlite3Fts5IndexQuery( } if( p->rc ){ - sqlite3Fts5IterClose((Fts5IndexIter*)pRet); + fts5IterClose((Fts5IndexIter*)pRet); pRet = 0; - sqlite3Fts5IndexCloseReader(p); + fts5IndexCloseReader(p); } *ppIter = (Fts5IndexIter*)pRet; @@ -6081,7 +7483,12 @@ int sqlite3Fts5IndexQuery( int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); - fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); + fts5TokendataIterNext(pIter, 0, 0); + }else{ + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + } return fts5IndexReturn(pIter->pIndex); } @@ -6114,7 +7521,12 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ */ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); + fts5TokendataIterNext(pIter, 1, iMatch); + }else{ + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + } return fts5IndexReturn(pIter->pIndex); } @@ -6129,15 +7541,185 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } +/* +** pIter is a prefix query. This function populates pIter->pTokenDataIter +** with an Fts5TokenDataIter object containing mappings for all rows +** matched by the query. +*/ +static int fts5SetupPrefixIterTokendata( + Fts5Iter *pIter, + const char *pToken, /* Token prefix to search for */ + int nToken /* Size of pToken in bytes */ +){ + Fts5Index *p = pIter->pIndex; + Fts5Buffer token = {0, 0, 0}; + TokendataSetupCtx ctx; + + memset(&ctx, 0, sizeof(ctx)); + + fts5BufferGrow(&p->rc, &token, nToken+1); + assert( token.p!=0 || p->rc!=SQLITE_OK ); + ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); + + if( p->rc==SQLITE_OK ){ + + /* Fill in the token prefix to search for */ + token.p[0] = FTS5_MAIN_PREFIX; + memcpy(&token.p[1], pToken, nToken); + token.n = nToken+1; + + fts5VisitEntries( + p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx + ); + + fts5TokendataIterSortMap(p, ctx.pT); + } + + if( p->rc==SQLITE_OK ){ + pIter->pTokenDataIter = ctx.pT; + }else{ + fts5TokendataIterDelete(ctx.pT); + } + fts5BufferFree(&token); + + return fts5IndexReturn(p); +} + +/* +** This is used by xInstToken() to access the token at offset iOff, column +** iCol of row iRowid. The token is returned via output variables *ppOut +** and *pnOut. The iterator passed as the first argument must be a tokendata=1 +** iterator (pIter->pTokenDataIter!=0). +** +** pToken/nToken: +*/ +int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + i64 iPos = (((i64)iCol)<<32) + iOff; + Fts5TokenDataMap *aMap = 0; + int i1 = 0; + int i2 = 0; + int iTest = 0; + + assert( pT || (pToken && pIter->nSeg>0) ); + if( pT==0 ){ + int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken); + if( rc!=SQLITE_OK ) return rc; + pT = pIter->pTokenDataIter; + } + + i2 = pT->nMap; + aMap = pT->aMap; + + while( i2>i1 ){ + iTest = (i1 + i2) / 2; + + if( aMap[iTest].iRowidiRowid ){ + i2 = iTest; + }else{ + if( aMap[iTest].iPosiPos ){ + i2 = iTest; + }else{ + break; + } + } + } + + if( i2>i1 ){ + if( pIter->nSeg==0 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + }else{ + Fts5TokenDataMap *p = &aMap[iTest]; + *ppOut = (const char*)&pT->terms.p[p->iIter]; + *pnOut = aMap[iTest].nByte; + } + } + + return SQLITE_OK; +} + +/* +** Clear any existing entries from the token-map associated with the +** iterator passed as the only argument. +*/ +void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter && pIter->pTokenDataIter + && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL) + ){ + pIter->pTokenDataIter->nMap = 0; + } +} + +/* +** Set a token-mapping for the iterator passed as the first argument. This +** is used in detail=column or detail=none mode when a token is requested +** using the xInstToken() API. In this case the caller tokenizers the +** current row and configures the token-mapping via multiple calls to this +** function. +*/ +int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, int iCol, int iOff +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *p = pIter->pIndex; + i64 iPos = (((i64)iCol)<<32) + iOff; + + assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); + assert( pIter->pTokenDataIter || pIter->nSeg>0 ); + if( pIter->nSeg>0 ){ + /* This is a prefix term iterator. */ + if( pT==0 ){ + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); + pIter->pTokenDataIter = pT; + } + if( pT ){ + fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos); + fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken); + } + }else{ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos); + } + } + return fts5IndexReturn(p); +} + /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ - Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - Fts5Index *pIndex = pIter->pIndex; - fts5MultiIterFree(pIter); - sqlite3Fts5IndexCloseReader(pIndex); + Fts5Index *pIndex = ((Fts5Iter*)pIndexIter)->pIndex; + fts5IterClose(pIndexIter); + fts5IndexReturn(pIndex); } } @@ -6219,6 +7801,347 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ return fts5IndexReturn(p); } +/* +** Retrieve the origin value that will be used for the segment currently +** being accumulated in the in-memory hash table when it is flushed to +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to +** the queried value. Or, if an error occurs, an error code is returned +** and the final value of (*piOrigin) is undefined. +*/ +int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + *piOrigin = pStruct->nOriginCntr; + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/* +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages +** associated with the same segment. This function adds rowid iRowid to +** the hash table. The caller is required to guarantee that there is at +** least one free slot on the page. +** +** If parameter bForce is false and the hash table is deemed to be full +** (more than half of the slots are occupied), then non-zero is returned +** and iRowid not inserted. Or, if bForce is true or if the hash table page +** is not full, iRowid is inserted and zero returned. +*/ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ + const int szKey = TOMBSTONE_KEYSIZE(pPg); + const int nSlot = TOMBSTONE_NSLOT(pPg); + const int nElem = fts5GetU32(&pPg->p[4]); + int iSlot = (iRowid / nPg) % nSlot; + int nCollide = nSlot; + + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; + if( iRowid==0 ){ + pPg->p[1] = 0x01; + return 0; + } + + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + + fts5PutU32(&pPg->p[4], nElem+1); + if( szKey==4 ){ + u32 *aSlot = (u32*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); + }else{ + u64 *aSlot = (u64*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU64((u8*)&aSlot[iSlot], iRowid); + } + + return 0; +} + +/* +** This function attempts to build a new hash containing all the keys +** currently in the tombstone hash table for segment pSeg. The new +** hash will be stored in the nOut buffers passed in array apOut[]. +** All pages of the new hash use key-size szKey (4 or 8). +** +** Return 0 if the hash is successfully rebuilt into the nOut pages. +** Or non-zero if it is not (because one page became overfull). In this +** case the caller should retry with a larger nOut parameter. +** +** Parameter pData1 is page iPg1 of the hash table being rebuilt. +*/ +static int fts5IndexTombstoneRehash( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int nOut, /* Number of output pages */ + Fts5Data **apOut /* Array of output hash pages */ +){ + int ii; + int res = 0; + + /* Initialize the headers of all the output pages */ + for(ii=0; iip[0] = szKey; + fts5PutU32(&apOut[ii]->p[4], 0); + } + + /* Loop through the current pages of the hash table. */ + for(ii=0; res==0 && iinPgTombstone; ii++){ + Fts5Data *pData = 0; /* Page ii of the current hash table */ + Fts5Data *pFree = 0; /* Free this at the end of the loop */ + + if( iPg1==ii ){ + pData = pData1; + }else{ + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); + } + + if( pData ){ + int szKeyIn = TOMBSTONE_KEYSIZE(pData); + int nSlotIn = (pData->nn - 8) / szKeyIn; + int iIn; + for(iIn=0; iInp[8]; + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); + }else{ + u64 *aSlot = (u64*)&pData->p[8]; + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); + } + + /* If iVal is not 0 at this point, insert it into the new hash table */ + if( iVal ){ + Fts5Data *pPg = apOut[(iVal % nOut)]; + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); + if( res ) break; + } + } + + /* If this is page 0 of the old hash, copy the rowid-0-flag from the + ** old hash to the new. */ + if( ii==0 ){ + apOut[0]->p[1] = pData->p[1]; + } + } + fts5DataRelease(pFree); + } + + return res; +} + +/* +** This is called to rebuild the hash table belonging to segment pSeg. +** If parameter pData1 is not NULL, then one page of the existing hash table +** has already been loaded - pData1, which is page iPg1. The key-size for +** the new hash table is szKey (4 or 8). +** +** If successful, the new hash table is not written to disk. Instead, +** output parameter (*pnOut) is set to the number of pages in the new +** hash table, and (*papOut) to point to an array of buffers containing +** the new page data. +** +** If an error occurs, an error code is left in the Fts5Index object and +** both output parameters set to 0 before returning. +*/ +static void fts5IndexTombstoneRebuild( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int *pnOut, /* OUT: Number of output pages */ + Fts5Data ***papOut /* OUT: Output hash pages */ +){ + const int MINSLOT = 32; + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); + int nSlot = 0; /* Number of slots in each output page */ + int nOut = 0; + + /* Figure out how many output pages (nOut) and how many slots per + ** page (nSlot). There are three possibilities: + ** + ** 1. The hash table does not yet exist. In this case the new hash + ** table will consist of a single page with MINSLOT slots. + ** + ** 2. The hash table exists but is currently a single page. In this + ** case an attempt is made to grow the page to accommodate the new + ** entry. The page is allowed to grow up to nSlotPerPage (see above) + ** slots. + ** + ** 3. The hash table already consists of more than one page, or of + ** a single page already so large that it cannot be grown. In this + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage + ** slots each, where nPg is the current number of pages in the + ** hash table. + */ + if( pSeg->nPgTombstone==0 ){ + /* Case 1. */ + nOut = 1; + nSlot = MINSLOT; + }else if( pSeg->nPgTombstone==1 ){ + /* Case 2. */ + int nElem = (int)fts5GetU32(&pData1->p[4]); + assert( pData1 && iPg1==0 ); + nOut = 1; + nSlot = MAX(nElem*4, MINSLOT); + if( nSlot>nSlotPerPage ) nOut = 0; + } + if( nOut==0 ){ + /* Case 3. */ + nOut = (pSeg->nPgTombstone * 2 + 1); + nSlot = nSlotPerPage; + } + + /* Allocate the required array and output pages */ + while( 1 ){ + int res = 0; + int ii = 0; + int szPage = 0; + Fts5Data **apOut = 0; + + /* Allocate space for the new hash table */ + assert( nSlot>=MINSLOT ); + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); + szPage = 8 + nSlot*szKey; + for(ii=0; iirc, + sizeof(Fts5Data)+szPage + ); + if( pNew ){ + pNew->nn = szPage; + pNew->p = (u8*)&pNew[1]; + apOut[ii] = pNew; + } + } + + /* Rebuild the hash table. */ + if( p->rc==SQLITE_OK ){ + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); + } + if( res==0 ){ + if( p->rc ){ + fts5IndexFreeArray(apOut, nOut); + apOut = 0; + nOut = 0; + } + *pnOut = nOut; + *papOut = apOut; + break; + } + + /* If control flows to here, it was not possible to rebuild the hash + ** table. Free all buffers and then try again with more pages. */ + assert( p->rc==SQLITE_OK ); + fts5IndexFreeArray(apOut, nOut); + nSlot = nSlotPerPage; + nOut = nOut*2 + 1; + } +} + + +/* +** Add a tombstone for rowid iRowid to segment pSeg. +*/ +static void fts5IndexTombstoneAdd( + Fts5Index *p, + Fts5StructureSegment *pSeg, + u64 iRowid +){ + Fts5Data *pPg = 0; + int iPg = -1; + int szKey = 0; + int nHash = 0; + Fts5Data **apHash = 0; + + p->nContentlessDelete++; + + if( pSeg->nPgTombstone>0 ){ + iPg = iRowid % pSeg->nPgTombstone; + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); + if( pPg==0 ){ + assert( p->rc!=SQLITE_OK ); + return; + } + + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); + fts5DataRelease(pPg); + return; + } + } + + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; + if( iRowid>0xFFFFFFFF ) szKey = 8; + + /* Rebuild the hash table */ + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); + + /* If all has succeeded, write the new rowid into one of the new hash + ** table pages, then write them all out to disk. */ + if( nHash ){ + int ii = 0; + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); + for(ii=0; iiiSegid, ii); + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); + } + pSeg->nPgTombstone = nHash; + fts5StructureWrite(p, p->pStruct); + } + + fts5DataRelease(pPg); + fts5IndexFreeArray(apHash, nHash); +} + +/* +** Add iRowid to the tombstone list of the segment or segments that contain +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite +** error code otherwise. +*/ +int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ + int iLvl; + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + int iSeg; + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ + if( bFound==0 ){ + pSeg->nEntryTombstone++; + bFound = 1; + } + fts5IndexTombstoneAdd(p, pSeg, iRowid); + } + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} /************************************************************************* ************************************************************************** @@ -6302,7 +8225,9 @@ static int fts5QueryCksum( int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIter = 0; - int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); + int rc = sqlite3Fts5IndexQuery( + p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA), 0, &pIter + ); while( rc==SQLITE_OK && ALWAYS(pIter!=0) && 0==sqlite3Fts5IterEof(pIter) ){ i64 rowid = pIter->iRowid; @@ -6324,7 +8249,7 @@ static int fts5QueryCksum( rc = sqlite3Fts5IterNext(pIter); } } - sqlite3Fts5IterClose(pIter); + fts5IterClose(pIter); *pCksum = cksum; return rc; @@ -6365,19 +8290,27 @@ static int fts5TestUtf8(const char *z, int n){ /* ** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. +** +** This function sets output variable (*pbFail) to true if the test fails. Or +** leaves it unchanged if the test succeeds. */ static void fts5TestTerm( Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, - u64 *pCksum + u64 *pCksum, + int *pbFail ){ int rc = p->rc; if( pPrev->n==0 ){ fts5BufferSet(&rc, pPrev, n, (const u8*)z); }else - if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + if( *pbFail==0 + && rc==SQLITE_OK + && (pPrev->n!=n || memcmp(pPrev->p, z, n)) + && (p->pHash==0 || p->pHash->nEntry==0) + ){ u64 cksum3 = *pCksum; const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ @@ -6427,7 +8360,7 @@ static void fts5TestTerm( fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ - rc = FTS5_CORRUPT; + *pbFail = 1; } *pCksum = cksum3; } @@ -6436,7 +8369,7 @@ static void fts5TestTerm( #else # define fts5TestDlidxReverse(x,y,z) -# define fts5TestTerm(u,v,w,x,y,z) +# define fts5TestTerm(t,u,v,w,x,y,z) #endif /* @@ -6461,15 +8394,18 @@ static void fts5IndexIntegrityCheckEmpty( for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ - if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) + || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)) + ){ + FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + } } fts5DataRelease(pLeaf); } } -static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ - int iTermOff = 0; +static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){ + i64 iTermOff = 0; int ii; Fts5Buffer buf1 = {0,0,0}; @@ -6478,7 +8414,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ ii = pLeaf->szLeaf; while( iinn && p->rc==SQLITE_OK ){ int res; - int iOff; + i64 iOff; int nIncr; ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); @@ -6486,12 +8422,12 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff = iTermOff; if( iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else if( iTermOff==nIncr ){ int nByte; iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); } @@ -6500,7 +8436,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ buf1.n = nKeep; fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); @@ -6508,7 +8444,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ if( p->rc==SQLITE_OK ){ res = fts5BufferCompare(&buf1, &buf2); - if( res<=0 ) p->rc = FTS5_CORRUPT; + if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid); } } fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); @@ -6569,7 +8505,7 @@ static void fts5IndexIntegrityCheckSegment( ** entry even if all the terms are removed from it by secure-delete ** operations. */ }else{ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); } }else{ @@ -6581,15 +8517,15 @@ static void fts5IndexIntegrityCheckSegment( iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; - if( res<0 ) p->rc = FTS5_CORRUPT; + if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow); } - fts5IntegrityCheckPgidx(p, pLeaf); + fts5IntegrityCheckPgidx(p, iRow, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -6619,7 +8555,7 @@ static void fts5IndexIntegrityCheckSegment( iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ - if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) FTS5_CORRUPT_ROWID(p, iKey); fts5DataRelease(pLeaf); } } @@ -6634,12 +8570,12 @@ static void fts5IndexIntegrityCheckSegment( int iRowidOff = fts5LeafFirstRowidOff(pLeaf); ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); }else if( bSecureDelete==0 || iRowidOff>0 ){ i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowidrc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); } } fts5DataRelease(pLeaf); @@ -6691,6 +8627,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ + int bTestFail = 0; #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; @@ -6733,7 +8670,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ - fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail); if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ @@ -6751,15 +8688,26 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ } } } - fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail); fts5MultiIterFree(pIter); - if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; - - fts5StructureRelease(pStruct); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ){ + p->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName + ); + } #ifdef SQLITE_DEBUG + /* In SQLITE_DEBUG builds, expensive extra checks were run as part of + ** the integrity-check above. If no other errors were detected, but one + ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */ + if( p->rc==SQLITE_OK && bTestFail ){ + p->rc = FTS5_CORRUPT; + } fts5BufferFree(&term); #endif + + fts5StructureRelease(pStruct); fts5BufferFree(&poslist); return fts5IndexReturn(p); } @@ -6770,13 +8718,14 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ ** function only. */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Decode a segment-data rowid from the %_data table. This function is ** the opposite of macro FTS5_SEGMENT_ROWID(). */ static void fts5DecodeRowid( i64 iRowid, /* Rowid from %_data table */ + int *pbTombstone, /* OUT: Tombstone hash flag */ int *piSegid, /* OUT: Segment id */ int *pbDlidx, /* OUT: Dlidx flag */ int *piHeight, /* OUT: Height */ @@ -6792,13 +8741,16 @@ static void fts5DecodeRowid( iRowid >>= FTS5_DATA_DLI_B; *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); + iRowid >>= FTS5_DATA_ID_B; + + *pbTombstone = (int)(iRowid & 0x0001); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ - fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid components */ + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ @@ -6808,14 +8760,16 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", - bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", + bTomb ? "tombstone " : "", + iSegid, iHeight, iPgno ); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, @@ -6830,16 +8784,22 @@ static void fts5DebugStructure( ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); + if( pSeg->iOrigin1>0 ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", + pSeg->iOrigin1, pSeg->iOrigin2 + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -6864,9 +8824,9 @@ static void fts5DecodeStructure( fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** @@ -6889,9 +8849,9 @@ static void fts5DecodeAverages( zSpace = " "; } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Buffer (a/n) is assumed to contain a list of serialized varints. Read ** each varint and append its string representation to buffer pBuf. Return @@ -6908,9 +8868,9 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ } return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The start of buffer (a/n) contains the start of a doclist. The doclist ** may or may not finish within the buffer. This function appends a text @@ -6943,9 +8903,9 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ return iOff; } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This function is part of the fts5_decode() debugging function. It is ** only ever used with detail=none tables. @@ -6986,9 +8946,27 @@ static void fts5DecodeRowidList( sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); } } -#endif /* SQLITE_TEST */ +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static void fts5BufferAppendTerm(int *pRc, Fts5Buffer *pBuf, Fts5Buffer *pTerm){ + int ii; + fts5BufferGrow(pRc, pBuf, pTerm->n*2 + 1); + if( *pRc==SQLITE_OK ){ + for(ii=0; iin; ii++){ + if( pTerm->p[ii]==0x00 ){ + pBuf->p[pBuf->n++] = '\\'; + pBuf->p[pBuf->n++] = '0'; + }else{ + pBuf->p[pBuf->n++] = pTerm->p[ii]; + } + } + pBuf->p[pBuf->n] = 0x00; + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The implementation of user-defined scalar function fts5_decode(). */ @@ -6999,6 +8977,7 @@ static void fts5DecodeFunction( ){ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + int bTomb; const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ @@ -7016,12 +8995,12 @@ static void fts5DecodeFunction( ** buffer overreads even if the record is corrupt. */ n = sqlite3_value_bytes(apVal[1]); aBlob = sqlite3_value_blob(apVal[1]); - nSpace = n + FTS5_DATA_ZERO_PADDING; + nSpace = ((i64)n) + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; if( n>0 ) memcpy(a, aBlob, n); - fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( bDlidx ){ @@ -7040,6 +9019,28 @@ static void fts5DecodeFunction( " %d(%lld)", lvl.iLeafPgno, lvl.iRowid ); } + }else if( bTomb ){ + u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; iiestimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + break; + } + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + int rc = SQLITE_OK; + Fts5StructVcsr *pNew = 0; + + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + *ppCsr = (sqlite3_vtab_cursor*)pNew; + + return SQLITE_OK; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + fts5StructureRelease(pCsr->pStruct); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + + assert( pCsr->pStruct ); + pCsr->iSeg++; + pCsr->iRowid++; + while( pCsr->iLevelnLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){ + pCsr->iLevel++; + pCsr->iSeg = 0; + } + if( pCsr->iLevel>=p->nLevel ){ + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + return pCsr->pStruct==0; +} + +static int fts5structRowidMethod( + sqlite3_vtab_cursor *cur, + sqlite_int64 *piRowid +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + *piRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int fts5structColumnMethod( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; + + switch( i ){ + case 0: /* level */ + sqlite3_result_int(ctx, pCsr->iLevel); + break; + case 1: /* segment */ + sqlite3_result_int(ctx, pCsr->iSeg); + break; + case 2: /* merge */ + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); + break; + case 3: /* segid */ + sqlite3_result_int(ctx, pSeg->iSegid); + break; + case 4: /* leaf1 */ + sqlite3_result_int(ctx, pSeg->pgnoFirst); + break; + case 5: /* leaf2 */ + sqlite3_result_int(ctx, pSeg->pgnoLast); + break; + case 6: /* origin1 */ + sqlite3_result_int64(ctx, pSeg->iOrigin1); + break; + case 7: /* origin2 */ + sqlite3_result_int64(ctx, pSeg->iOrigin2); + break; + case 8: /* npgtombstone */ + sqlite3_result_int(ctx, pSeg->nPgTombstone); + break; + case 9: /* nentrytombstone */ + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); + break; + case 10: /* nentry */ + sqlite3_result_int64(ctx, pSeg->nEntry); + break; + } + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int fts5structFilterMethod( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; + int rc = SQLITE_OK; + + const u8 *aBlob = 0; + int nBlob = 0; + + assert( argc==1 ); + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + + nBlob = sqlite3_value_bytes(argv[0]); + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); + if( rc==SQLITE_OK ){ + pCsr->iLevel = 0; + pCsr->iRowid = 0; + pCsr->iSeg = -1; + rc = fts5structNextMethod(pVtabCursor); + } + + return rc; +} + +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called as part of registering the FTS5 module with database @@ -7244,7 +9474,7 @@ static void fts5RowidFunction( ** SQLite error code is returned instead. */ int sqlite3Fts5IndexInit(sqlite3 *db){ -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); @@ -7261,6 +9491,37 @@ int sqlite3Fts5IndexInit(sqlite3 *db){ db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } + + if( rc==SQLITE_OK ){ + static const sqlite3_module fts5structure_module = { + 0, /* iVersion */ + 0, /* xCreate */ + fts5structConnectMethod, /* xConnect */ + fts5structBestIndexMethod, /* xBestIndex */ + fts5structDisconnectMethod, /* xDisconnect */ + 0, /* xDestroy */ + fts5structOpenMethod, /* xOpen */ + fts5structCloseMethod, /* xClose */ + fts5structFilterMethod, /* xFilter */ + fts5structNextMethod, /* xNext */ + fts5structEofMethod, /* xEof */ + fts5structColumnMethod, /* xColumn */ + fts5structRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); + } return rc; #else return SQLITE_OK; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 13921ce49e..f45b9ef906 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -83,8 +83,18 @@ struct Fts5Global { Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ Fts5Cursor *pCsr; /* First in list of all open cursors */ + u32 aLocaleHdr[4]; }; +/* +** Size of header on fts5_locale() values. And macro to access a buffer +** containing a copy of the header from an Fts5Config pointer. +*/ +#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) +#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) + +#define FTS5_INSTTOKEN_SUBTYPE 73 + /* ** Each auxiliary function registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part @@ -103,11 +113,28 @@ struct Fts5Auxiliary { ** Each tokenizer module registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part ** of the Fts5Global.pTok list. +** +** bV2Native: +** True if the tokenizer was registered using xCreateTokenizer_v2(), false +** for xCreateTokenizer(). If this variable is true, then x2 is populated +** with the routines as supplied by the caller and x1 contains synthesized +** wrapper routines. In this case the user-data pointer passed to +** x1.xCreate should be a pointer to the Fts5TokenizerModule structure, +** not a copy of pUserData. +** +** Of course, if bV2Native is false, then x1 contains the real routines and +** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule +** object should be passed to x2.xCreate. +** +** The synthesized wrapper routines are necessary for xFindTokenizer(_v2) +** calls. */ struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ - fts5_tokenizer x; /* Tokenizer functions */ + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; @@ -117,6 +144,8 @@ struct Fts5FullTable { Fts5Storage *pStorage; /* Document store */ Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ + int iSavepoint; /* Successful xSavepoint()+1 */ + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -141,9 +170,11 @@ struct Fts5Sorter { i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ - int aIdx[1]; /* Offsets into aPoslist for current row */ + int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */ }; +/* Size (int bytes) of an Fts5Sorter object with N indexes */ +#define SZ_FTS5SORTER(N) (offsetof(Fts5Sorter,nIdx)+((N+2)/2)*sizeof(i64)) /* ** Virtual-table cursor object. @@ -193,7 +224,7 @@ struct Fts5Cursor { Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ - /* Cache used by auxiliary functions xInst() and xInstCount() */ + /* Cache used by auxiliary API functions xInst() and xInstCount() */ Fts5PoslistReader *aInstIter; /* One for each phrase */ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ @@ -304,10 +335,16 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ #endif /* -** Return true if pTab is a contentless table. +** Return true if pTab is a contentless table. If parameter bIncludeUnindexed +** is true, this includes contentless tables that store UNINDEXED columns +** only. */ -static int fts5IsContentless(Fts5FullTable *pTab){ - return pTab->p.pConfig->eContent==FTS5_CONTENT_NONE; +static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){ + int eContent = pTab->p.pConfig->eContent; + return ( + eContent==FTS5_CONTENT_NONE + || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED) + ); } /* @@ -375,8 +412,12 @@ static int fts5InitVtab( assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); } if( rc==SQLITE_OK ){ + pConfig->pzErrmsg = pzErr; pTab->p.pConfig = pConfig; pTab->pGlobal = pGlobal; + if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } } /* Open the index sub-system */ @@ -398,13 +439,17 @@ static int fts5InitVtab( /* Load the initial configuration */ if( rc==SQLITE_OK ){ - assert( pConfig->pzErrmsg==0 ); - pConfig->pzErrmsg = pzErr; - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); - sqlite3Fts5IndexRollback(pTab->p.pIndex); - pConfig->pzErrmsg = 0; + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1); + } + + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); } + if( pConfig ) pConfig->pzErrmsg = 0; if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab); pTab = 0; @@ -466,16 +511,27 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ #endif } +static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008002 ) +#endif + { + pIdxInfo->estimatedRows = nRow; + } +#endif +} + static int fts5UsePatternMatch( Fts5Config *pConfig, struct sqlite3_index_constraint *p ){ assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB ); assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE ); - if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ + if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ return 1; } - if( pConfig->ePattern==FTS5_PATTERN_LIKE + if( pConfig->t.ePattern==FTS5_PATTERN_LIKE && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB) ){ return 1; @@ -522,10 +578,10 @@ static int fts5UsePatternMatch( ** This function ensures that there is at most one "r" or "=". And that if ** there exists an "=" then there is no "<" or ">". ** -** Costs are assigned as follows: +** If an unusable MATCH operator is present in the WHERE clause, then +** SQLITE_CONSTRAINT is returned. ** -** a) If an unusable MATCH operator is present in the WHERE clause, the -** cost is unconditionally set to 1e50 (a really big number). +** Costs are assigned as follows: ** ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: @@ -558,7 +614,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ int bSeenEq = 0; int bSeenGt = 0; int bSeenLt = 0; - int bSeenMatch = 0; + int nSeenMatch = 0; int bSeenRank = 0; @@ -589,21 +645,19 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* A MATCH operator or equivalent */ if( p->usable==0 || iCol<0 ){ /* As there exists an unusable MATCH constraint this is an - ** unusable plan. Set a prohibitively high cost. */ - pInfo->estimatedCost = 1e50; - assert( iIdxStr < pInfo->nConstraint*6 + 1 ); + ** unusable plan. Return SQLITE_CONSTRAINT. */ idxStr[iIdxStr] = 0; - return SQLITE_OK; + return SQLITE_CONSTRAINT; }else{ if( iCol==nCol+1 ){ if( bSeenRank ) continue; idxStr[iIdxStr++] = 'r'; bSeenRank = 1; - }else if( iCol>=0 ){ - bSeenMatch = 1; + }else{ + nSeenMatch++; idxStr[iIdxStr++] = 'M'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); - idxStr += strlen(&idxStr[iIdxStr]); + iIdxStr += (int)strlen(&idxStr[iIdxStr]); assert( idxStr[iIdxStr]=='\0' ); } pInfo->aConstraintUsage[i].argvIndex = ++iCons; @@ -617,10 +671,12 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ idxStr += strlen(&idxStr[iIdxStr]); pInfo->aConstraintUsage[i].argvIndex = ++iCons; assert( idxStr[iIdxStr]=='\0' ); + nSeenMatch++; }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){ idxStr[iIdxStr++] = '='; bSeenEq = 1; pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; } } } @@ -647,12 +703,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } idxStr[iIdxStr] = '\0'; - /* Set idxFlags flags for the ORDER BY clause */ + /* Set idxFlags flags for the ORDER BY clause + ** + ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC". + */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; - if( iSort==(pConfig->nCol+1) && bSeenMatch ){ + if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){ idxFlags |= FTS5_BI_ORDER_RANK; - }else if( iSort==-1 ){ + }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ idxFlags |= FTS5_BI_ORDER_ROWID; } if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ @@ -665,14 +724,21 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0; - if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo); - }else if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0; - }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0; + pInfo->estimatedCost = nSeenMatch ? 1000.0 : 25.0; + fts5SetUniqueFlag(pInfo); + fts5SetEstimatedRows(pInfo, 1); }else{ - pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0; + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 5000.0 : 750000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 7500.0 : 2250000.0; + }else{ + pInfo->estimatedCost = nSeenMatch ? 10000.0 : 3000000.0; + } + for(i=1; iestimatedCost *= 0.4; + } + fts5SetEstimatedRows(pInfo, (i64)(pInfo->estimatedCost / 4.0)); } pInfo->idxNum = idxFlags; @@ -871,7 +937,9 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ int bDesc = pCsr->bDesc; i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc + ); if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ *pbSkip = 1; } @@ -904,6 +972,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ ); assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table, + ** clear any token mappings accumulated at the fts5_index.c level. In + ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH, + ** we need to retain the mappings for the entire query. */ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata + ){ + sqlite3Fts5ExprClearTokens(pCsr->pExpr); + } + if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; @@ -938,6 +1016,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ } }else{ rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE); } break; } @@ -967,7 +1046,7 @@ static int fts5PrepareStatement( rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, SQLITE_PREPARE_PERSISTENT, &pRet, 0); if( rc!=SQLITE_OK ){ - *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } @@ -991,7 +1070,7 @@ static int fts5CursorFirstSorted( const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); - nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + nByte = SZ_FTS5SORTER(nPhrase); pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, (size_t)nByte); @@ -1032,7 +1111,9 @@ static int fts5CursorFirstSorted( static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ int rc; Fts5Expr *pExpr = pCsr->pExpr; - rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc + ); if( sqlite3Fts5ExprEof(pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } @@ -1191,6 +1272,145 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ return iDefault; } +/* +** Set the error message on the virtual table passed as the first argument. +*/ +static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + sqlite3_free(p->p.base.zErrMsg); + p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale +** specified by pLocale/nLocale. The buffer indicated by pLocale must remain +** valid until after the final call to sqlite3Fts5Tokenize() that will use +** the locale. +*/ +static void sqlite3Fts5SetLocale( + Fts5Config *pConfig, + const char *zLocale, + int nLocale +){ + Fts5TokenizerConfig *pT = &pConfig->t; + pT->pLocale = zLocale; + pT->nLocale = nLocale; +} + +/* +** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale(). +*/ +void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ + sqlite3Fts5SetLocale(pConfig, 0, 0); +} + +/* +** Return true if the value passed as the only argument is an +** fts5_locale() value. +*/ +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ + int ret = 0; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case. + ** If the blob was created using zeroblob(), then sqlite3_value_blob() + ** may call malloc(). If this malloc() fails, then the values returned + ** by both value_blob() and value_bytes() will be 0. If value_bytes() were + ** called first, then the NULL pointer returned by value_blob() might + ** be dereferenced. */ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( nBlob>FTS5_LOCALE_HDR_SIZE + && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) + ){ + ret = 1; + } + } + return ret; +} + +/* +** Value pVal is guaranteed to be an fts5_locale() value, according to +** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale +** from the value and returns them separately. +** +** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set +** to point to buffers containing the text and locale, as utf-8, +** respectively. In this case output parameters (*pnText) and (*pnLoc) are +** set to the sizes in bytes of these two buffers. +** +** Or, if an error occurs, then an SQLite error code is returned. The final +** value of the four output parameters is undefined in this case. +*/ +int sqlite3Fts5DecodeLocaleValue( + sqlite3_value *pVal, + const char **ppText, + int *pnText, + const char **ppLoc, + int *pnLoc +){ + const char *p = sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + int nLoc = 0; + + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + assert( n>FTS5_LOCALE_HDR_SIZE ); + + for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){ + if( nLoc==(n-1) ){ + return SQLITE_MISMATCH; + } + } + *ppLoc = &p[FTS5_LOCALE_HDR_SIZE]; + *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE; + + *ppText = &p[nLoc+1]; + *pnText = n - nLoc - 1; + return SQLITE_OK; +} + +/* +** Argument pVal is the text of a full-text search expression. It may or +** may not have been wrapped by fts5_locale(). This function extracts +** the text of the expression, and sets output variable (*pzText) to +** point to a nul-terminated buffer containing the expression. +** +** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called +** to set the tokenizer to use the specified locale. +** +** If output variable (*pbFreeAndReset) is set to true, then the caller +** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer +** locale, and (b) call sqlite3_free() to free (*pzText). +*/ +static int fts5ExtractExprText( + Fts5Config *pConfig, /* Fts5 configuration */ + sqlite3_value *pVal, /* Value to extract expression text from */ + char **pzText, /* OUT: nul-terminated buffer of text */ + int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ +){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText); + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + } + *pbFreeAndReset = 1; + }else{ + *pzText = (char*)sqlite3_value_text(pVal); + *pbFreeAndReset = 0; + } + + return rc; +} + + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -1221,17 +1441,12 @@ static int fts5FilterMethod( sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ int iCol; /* Column on LHS of MATCH operator */ char **pzErrmsg = pConfig->pzErrmsg; + int bPrefixInsttoken = pConfig->bPrefixInsttoken; int i; int iIdxStr = 0; Fts5Expr *pExpr = 0; - if( pConfig->bLock ){ - pTab->p.base.zErrMsg = sqlite3_mprintf( - "recursively defined fts5 content table" - ); - return SQLITE_ERROR; - } - + assert( pConfig->bLock==0 ); if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); @@ -1255,8 +1470,17 @@ static int fts5FilterMethod( pRank = apVal[i]; break; case 'M': { - const char *zText = (const char*)sqlite3_value_text(apVal[i]); + char *zText = 0; + int bFreeAndReset = 0; + int bInternal = 0; + + rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset); + if( rc!=SQLITE_OK ) goto filter_out; if( zText==0 ) zText = ""; + if( sqlite3_value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE ){ + pConfig->bPrefixInsttoken = 1; + } + iCol = 0; do{ iCol = iCol*10 + (idxStr[iIdxStr]-'0'); @@ -1268,7 +1492,7 @@ static int fts5FilterMethod( ** indicates that the MATCH expression is not a full text query, ** but a request for an internal parameter. */ rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); - goto filter_out; + bInternal = 1; }else{ char **pzErr = &pTab->p.base.zErrMsg; rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); @@ -1276,9 +1500,15 @@ static int fts5FilterMethod( rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); pExpr = 0; } - if( rc!=SQLITE_OK ) goto filter_out; } + if( bFreeAndReset ){ + sqlite3_free(zText); + sqlite3Fts5ClearLocale(pConfig); + } + + if( bInternal || rc!=SQLITE_OK ) goto filter_out; + break; } case 'L': @@ -1329,6 +1559,9 @@ static int fts5FilterMethod( pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); } + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) goto filter_out; + if( pTab->pSortCsr ){ /* If pSortCsr is non-NULL, then this call is being made as part of ** processing for a "... MATCH ORDER BY rank" query (ePlan is @@ -1351,6 +1584,7 @@ static int fts5FilterMethod( pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); }else if( pCsr->pExpr ){ + assert( rc==SQLITE_OK ); rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ if( bOrderByRank ){ @@ -1362,9 +1596,7 @@ static int fts5FilterMethod( } } }else if( pConfig->zContent==0 ){ - *pConfig->pzErrmsg = sqlite3_mprintf( - "%s: table does not support scanning", pConfig->zName - ); + fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName); rc = SQLITE_ERROR; }else{ /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup @@ -1388,6 +1620,7 @@ static int fts5FilterMethod( filter_out: sqlite3Fts5ExprFree(pExpr); pConfig->pzErrmsg = pzErrmsg; + pConfig->bPrefixInsttoken = bPrefixInsttoken; return rc; } @@ -1407,9 +1640,13 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ assert( pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE + || pCsr->ePlan==FTS5_PLAN_SCAN + || pCsr->ePlan==FTS5_PLAN_ROWID ); if( pCsr->pSorter ){ return pCsr->pSorter->iRowid; + }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){ + return sqlite3_column_int64(pCsr->pStmt, 0); }else{ return sqlite3Fts5ExprRowid(pCsr->pExpr); } @@ -1426,25 +1663,16 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ int ePlan = pCsr->ePlan; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); - switch( ePlan ){ - case FTS5_PLAN_SPECIAL: - *pRowid = 0; - break; - - case FTS5_PLAN_SOURCE: - case FTS5_PLAN_MATCH: - case FTS5_PLAN_SORTED_MATCH: - *pRowid = fts5CursorRowid(pCsr); - break; - - default: - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - break; + if( ePlan==FTS5_PLAN_SPECIAL ){ + *pRowid = 0; + }else{ + *pRowid = fts5CursorRowid(pCsr); } return SQLITE_OK; } + /* ** If the cursor requires seeking (bSeekRequired flag is set), seek it. ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. @@ -1481,8 +1709,13 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK ){ rc = FTS5_CORRUPT; + fts5SetVtabError((Fts5FullTable*)pTab, + "fts5: missing row %lld from content table %s", + fts5CursorRowid(pCsr), + pTab->pConfig->zContent + ); }else if( pTab->pConfig->pzErrmsg ){ - *pTab->pConfig->pzErrmsg = sqlite3_mprintf( + fts5SetVtabError((Fts5FullTable*)pTab, "%s", sqlite3_errmsg(pTab->pConfig->db) ); } @@ -1491,14 +1724,6 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ return rc; } -static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ - va_list ap; /* ... printf arguments */ - va_start(ap, zFormat); - assert( p->p.base.zErrMsg==0 ); - p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); - va_end(ap); -} - /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: @@ -1522,6 +1747,7 @@ static int fts5SpecialInsert( Fts5Config *pConfig = pTab->p.pConfig; int rc = SQLITE_OK; int bError = 0; + int bLoadConfig = 0; if( 0==sqlite3_stricmp("delete-all", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ @@ -1533,8 +1759,9 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ - if( pConfig->eContent==FTS5_CONTENT_NONE ){ + if( fts5IsContentless(pTab, 1) ){ fts5SetVtabError(pTab, "'rebuild' may not be used with a contentless fts5 table" ); @@ -1542,6 +1769,7 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ rc = sqlite3Fts5StorageOptimize(pTab->pStorage); }else if( 0==sqlite3_stricmp("merge", zCmd) ){ @@ -1554,8 +1782,13 @@ static int fts5SpecialInsert( }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ pConfig->bPrefixIndex = sqlite3_value_int(pVal); #endif + }else if( 0==sqlite3_stricmp("flush", zCmd) ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); }else{ - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } if( rc==SQLITE_OK ){ rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError); } @@ -1567,6 +1800,12 @@ static int fts5SpecialInsert( } } } + + if( rc==SQLITE_OK && bLoadConfig ){ + pTab->p.pConfig->iCookie--; + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + return rc; } @@ -1578,7 +1817,7 @@ static int fts5SpecialDelete( int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0); } return rc; } @@ -1591,7 +1830,7 @@ static void fts5StorageInsert( ){ int rc = *pRc; if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); @@ -1599,6 +1838,67 @@ static void fts5StorageInsert( *pRc = rc; } +/* +** +** This function is called when the user attempts an UPDATE on a contentless +** table. Parameter bRowidModified is true if the UPDATE statement modifies +** the rowid value. Parameter apVal[] contains the new values for each user +** defined column of the fts5 table. pConfig is the configuration object of the +** table being updated (guaranteed to be contentless). The contentless_delete=1 +** and contentless_unindexed=1 options may or may not be set. +** +** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite +** error code if it cannot. In this case an error message is also loaded into +** pConfig. Output parameter (*pbContent) is set to true if the caller should +** update the %_content table only - not the FTS index or any other shadow +** table. This occurs when an UPDATE modifies only UNINDEXED columns of the +** table. +** +** An UPDATE may proceed if: +** +** * The only columns modified are UNINDEXED columns, or +** +** * The contentless_delete=1 option was specified and all of the indexed +** columns (not a subset) have been modified. +*/ +static int fts5ContentlessUpdate( + Fts5Config *pConfig, + sqlite3_value **apVal, + int bRowidModified, + int *pbContent +){ + int ii; + int bSeenIndex = 0; /* Have seen modified indexed column */ + int bSeenIndexNC = 0; /* Have seen unmodified indexed column */ + int rc = SQLITE_OK; + + for(ii=0; iinCol; ii++){ + if( pConfig->abUnindexed[ii]==0 ){ + if( sqlite3_value_nochange(apVal[ii]) ){ + bSeenIndexNC++; + }else{ + bSeenIndex++; + } + } + } + + if( bSeenIndex==0 && bRowidModified==0 ){ + *pbContent = 1; + }else{ + if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){ + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, + (pConfig->bContentlessDelete ? + "%s a subset of columns on fts5 contentless-delete table: %s" : + "%s contentless fts5 table: %s") + , "cannot UPDATE", pConfig->zName + ); + } + } + + return rc; +} + /* ** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be @@ -1623,8 +1923,6 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ - int bUpdateOrDelete = 0; - /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -1636,7 +1934,7 @@ static int fts5UpdateMethod( ); assert( pTab->p.pConfig->pzErrmsg==0 ); if( pConfig->pgsz==0 ){ - rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie); if( rc!=SQLITE_OK ) return rc; } @@ -1654,7 +1952,14 @@ static int fts5UpdateMethod( if( pConfig->eContent!=FTS5_CONTENT_NORMAL && 0==sqlite3_stricmp("delete", z) ){ - rc = fts5SpecialDelete(pTab, apVal); + if( pConfig->bContentlessDelete ){ + fts5SetVtabError(pTab, + "'delete' may not be used with a contentless_delete=1 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = fts5SpecialDelete(pTab, apVal); + } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } @@ -1671,90 +1976,111 @@ static int fts5UpdateMethod( ** Cases 3 and 4 may violate the rowid constraint. */ int eConflict = SQLITE_ABORT; - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ eConflict = sqlite3_vtab_on_conflict(pConfig->db); } assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); assert( nArg!=1 || eType0==SQLITE_INTEGER ); - /* Filter out attempts to run UPDATE or DELETE on contentless tables. - ** This is not suported. */ - if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ - pTab->p.base.zErrMsg = sqlite3_mprintf( - "cannot %s contentless fts5 table: %s", - (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName - ); - rc = SQLITE_ERROR; - } - /* DELETE */ - else if( nArg==1 ){ - i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); - bUpdateOrDelete = 1; + if( nArg==1 ){ + /* It is only possible to DELETE from a contentless table if the + ** contentless_delete=1 flag is set. */ + if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){ + fts5SetVtabError(pTab, + "cannot DELETE from contentless fts5 table: %s", pConfig->zName + ); + rc = SQLITE_ERROR; + }else{ + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0); + } } /* INSERT or UPDATE */ else{ int eType1 = sqlite3_value_numeric_type(apVal[1]); - if( eType1!=SQLITE_INTEGER && eType1!=SQLITE_NULL ){ - rc = SQLITE_MISMATCH; + /* It is an error to write an fts5_locale() value to a table without + ** the locale=1 option. */ + if( pConfig->bLocale==0 ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + rc = SQLITE_MISMATCH; + goto update_out; + } + } } - else if( eType0!=SQLITE_INTEGER ){ - /* If this is a REPLACE, first remove the current entry (if any) */ + if( eType0!=SQLITE_INTEGER ){ + /* An INSERT statement. If the conflict-mode is REPLACE, first remove + ** the current entry (if any). */ if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); - bUpdateOrDelete = 1; + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); } /* UPDATE */ else{ + Fts5Storage *pStorage = pTab->pStorage; i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ - if( eType1==SQLITE_INTEGER && iOld!=iNew ){ + int bContent = 0; /* Content only update */ + + /* If this is a contentless table (including contentless_unindexed=1 + ** tables), check if the UPDATE may proceed. */ + if( fts5IsContentless(pTab, 1) ){ + rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent); + if( rc!=SQLITE_OK ) goto update_out; + } + + if( eType1!=SQLITE_INTEGER ){ + rc = SQLITE_MISMATCH; + }else if( iOld!=iNew ){ + assert( bContent==0 ); if( eConflict==SQLITE_REPLACE ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0); } fts5StorageInsert(&rc, pTab, apVal, pRowid); }else{ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid); } if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal,*pRowid); + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid); + } + } + }else if( bContent ){ + /* This occurs when an UPDATE on a contentless table affects *only* + ** UNINDEXED columns. This is a no-op for contentless_unindexed=0 + ** tables, or a write to the %_content table only for =1 tables. */ + assert( fts5IsContentless(pTab, 1) ); + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid); } }else{ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); fts5StorageInsert(&rc, pTab, apVal, pRowid); } - bUpdateOrDelete = 1; + sqlite3Fts5StorageReleaseDeleteRow(pStorage); } } } - if( rc==SQLITE_OK - && bUpdateOrDelete - && pConfig->bSecureDelete - && pConfig->iVersion==FTS5_CURRENT_VERSION - ){ - rc = sqlite3Fts5StorageConfigValue( - pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE - ); - if( rc==SQLITE_OK ){ - pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; - } - } - + update_out: pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -1767,8 +2093,7 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_SYNC, 0); pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - fts5TripCursors(pTab); - rc = sqlite3Fts5StorageSync(pTab->pStorage); + rc = sqlite3Fts5FlushToDisk(&pTab->p); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -1777,9 +2102,11 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); - fts5NewTransaction((Fts5FullTable*)pVtab); - return SQLITE_OK; + int rc = fts5NewTransaction((Fts5FullTable*)pVtab); + if( rc==SQLITE_OK ){ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); + } + return rc; } /* @@ -1802,6 +2129,7 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); + pTab->p.pConfig->pgsz = 0; return rc; } @@ -1833,17 +2161,40 @@ static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } -static int fts5ApiTokenize( +/* +** Implementation of xTokenize_v2() API. +*/ +static int fts5ApiTokenize_v2( Fts5Context *pCtx, const char *pText, int nText, + const char *pLoc, int nLoc, void *pUserData, int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize( - pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + int rc = SQLITE_OK; + + sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pTab->pConfig, + FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken ); + sqlite3Fts5SetLocale(pTab->pConfig, 0, 0); + + return rc; +} + +/* +** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0 +** passed as the locale. +*/ +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -1856,6 +2207,49 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +/* +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This +** function extracts the text value of column iCol of the current row. +** Additionally, if there is an associated locale, it invokes +** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller +** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point +** after this function returns. +** +** If successful, (*ppText) is set to point to a buffer containing the text +** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that +** buffer in bytes. It is not guaranteed to be nul-terminated. If an error +** occurs, an SQLite error code is returned. The final values of the two +** output parameters are undefined in this case. +*/ +static int fts5TextFromStmt( + Fts5Config *pConfig, + sqlite3_stmt *pStmt, + int iCol, + const char **ppText, + int *pnText +){ + sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1); + const char *pLoc = 0; + int nLoc = 0; + int rc = SQLITE_OK; + + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc); + }else{ + *ppText = (const char*)sqlite3_value_text(pVal); + *pnText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol); + nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol); + } + } + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + return rc; +} + static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, @@ -1864,46 +2258,69 @@ static int fts5ApiColumnText( ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) - || pCsr->ePlan==FTS5_PLAN_SPECIAL - ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pTab->pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn); + sqlite3Fts5ClearLocale(pTab->pConfig); } } return rc; } +/* +** This is called by various API functions - xInst, xPhraseFirst, +** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase +** of the current row. This function works for both detail=full tables (in +** which case the position-list was read from the fts index) or for other +** detail= modes if the row content is available. +*/ static int fts5CsrPoslist( - Fts5Cursor *pCsr, - int iPhrase, - const u8 **pa, - int *pn + Fts5Cursor *pCsr, /* Fts5 cursor object */ + int iPhrase, /* Phrase to find position list for */ + const u8 **pa, /* OUT: Pointer to position list buffer */ + int *pn /* OUT: Size of (*pa) in bytes */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - + if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ + rc = SQLITE_RANGE; + }else if( pConfig->eDetail!=FTS5_DETAIL_FULL + && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + ){ + *pa = 0; + *pn = 0; + return SQLITE_OK; + }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = fts5SeekCursor(pCsr, 0); + } for(i=0; inCol && rc==SQLITE_OK; i++){ - int n; const char *z; - rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } + sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -1914,13 +2331,18 @@ static int fts5CsrPoslist( CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ - Fts5Sorter *pSorter = pCsr->pSorter; - int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - *pn = pSorter->aIdx[iPhrase] - i1; - *pa = &pSorter->aPoslist[i1]; + if( rc==SQLITE_OK ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } }else{ - *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pa = 0; + *pn = 0; } return rc; @@ -1991,7 +2413,8 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); - if( aInst[1]<0 || aInst[1]>=nCol ){ + assert( aInst[1]>=0 ); + if( aInst[1]>=nCol ){ rc = FTS5_CORRUPT; break; } @@ -2029,12 +2452,6 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; -#if 0 - }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ - *piPhrase = pCsr->aInst[iIdx*3]; - *piCol = pCsr->aInst[iIdx*3 + 2]; - *piOff = -1; -#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -2075,7 +2492,7 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ if( pConfig->bColumnsize ){ i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); - }else if( pConfig->zContent==0 ){ + }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ int i; for(i=0; inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ @@ -2084,17 +2501,19 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ } }else{ int i; + rc = fts5SeekCursor(pCsr, 0); for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ - const char *z; int n; - void *p = (void*)(&pCsr->aColumnSize[i]); + const char *z = 0; + int n = 0; pCsr->aColumnSize[i] = 0; - rc = fts5ApiColumnText(pCtx, i, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize( - pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, + z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); } + sqlite3Fts5ClearLocale(pConfig); } } } @@ -2174,11 +2593,10 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ } static void fts5ApiPhraseNext( - Fts5Context *pUnused, + Fts5Context *pCtx, Fts5PhraseIter *pIter, int *piCol, int *piOff ){ - UNUSED_PARAM(pUnused); if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; @@ -2186,8 +2604,12 @@ static void fts5ApiPhraseNext( int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ + /* Avoid returning a (*piCol) value that is too large for the table, + ** even if the position-list is corrupt. The caller might not be + ** expecting it. */ + int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol; pIter->a += fts5GetVarint32(pIter->a, iVal); - *piCol = iVal; + *piCol = (iVal>=nCol ? nCol-1 : iVal); *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } @@ -2289,13 +2711,96 @@ static int fts5ApiPhraseFirstColumn( return rc; } +/* +** xQueryToken() API implemenetation. +*/ +static int fts5ApiQueryToken( + Fts5Context* pCtx, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); +} -static int fts5ApiQueryPhrase(Fts5Context*, int, void*, - int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) +/* +** xInstToken() API implemenetation. +*/ +static int fts5ApiInstToken( + Fts5Context *pCtx, + int iIdx, + int iToken, + const char **ppOut, int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; + }else{ + int iPhrase = pCsr->aInst[iIdx*3]; + int iCol = pCsr->aInst[iIdx*3 + 1]; + int iOff = pCsr->aInst[iIdx*3 + 2]; + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5ExprInstToken( + pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut + ); + } + } + return rc; +} + + +static int fts5ApiQueryPhrase(Fts5Context*, int, void*, + int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); +/* +** The xColumnLocale() API. +*/ +static int fts5ApiColumnLocale( + Fts5Context *pCtx, + int iCol, + const char **pzLocale, + int *pnLocale +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + *pzLocale = 0; + *pnLocale = 0; + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( + pConfig->abUnindexed[iCol]==0 + && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + && pConfig->bLocale + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + const char *zDummy = 0; + int nDummy = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy); + if( rc==SQLITE_OK ){ + *pzLocale = pConfig->t.pLocale; + *pnLocale = pConfig->t.nLocale; + } + sqlite3Fts5ClearLocale(pConfig); + } + } + + return rc; +} + static const Fts5ExtensionApi sFts5Api = { - 2, /* iVersion */ + 4, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -2315,6 +2820,10 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, + fts5ApiQueryToken, + fts5ApiInstToken, + fts5ApiColumnLocale, + fts5ApiTokenize_v2 }; /* @@ -2365,6 +2874,7 @@ static void fts5ApiInvoke( sqlite3_value **argv ){ assert( pCsr->pAux==0 ); + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; @@ -2378,6 +2888,21 @@ static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ return pCsr; } +/* +** Parameter zFmt is a printf() style formatting string. This function +** formats it using the trailing arguments and returns the result as +** an error message to the context passed as the first argument. +*/ +static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + va_end(ap); +} + static void fts5ApiCallback( sqlite3_context *context, int argc, @@ -2393,12 +2918,13 @@ static void fts5ApiCallback( iCsrId = sqlite3_value_int64(argv[0]); pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); - if( pCsr==0 || pCsr->ePlan==0 ){ - char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){ + fts5ResultError(context, "no such cursor: %lld", iCsrId); }else{ + sqlite3_vtab *pTab = pCsr->base.pVtab; fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + sqlite3_free(pTab->zErrMsg); + pTab->zErrMsg = 0; } } @@ -2516,8 +3042,8 @@ static int fts5ColumnMethod( ** auxiliary function. */ sqlite3_result_int64(pCtx, pCsr->iCsrId); }else if( iCol==pConfig->nCol+1 ){ - /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); }else if( @@ -2528,14 +3054,32 @@ static int fts5ColumnMethod( fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); } } - }else if( !fts5IsContentless(pTab) ){ - pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - rc = fts5SeekCursor(pCsr, 1); - if( rc==SQLITE_OK ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + }else{ + if( !sqlite3_vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT); + } + sqlite3Fts5ClearLocale(pConfig); + }else{ + sqlite3_result_value(pCtx, pVal); + } + } + + pConfig->pzErrmsg = 0; } - pConfig->pzErrmsg = 0; } + return rc; } @@ -2573,8 +3117,10 @@ static int fts5RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ + int rc; Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - return sqlite3Fts5StorageRename(pTab->pStorage, zName); + rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); + return rc; } int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ @@ -2588,9 +3134,15 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ ** Flush the contents of the pending-terms table to disk. */ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint); - return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); + rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } + return rc; } /* @@ -2599,9 +3151,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint); - return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); + if( (iSavepoint+1)iSavepoint ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint; + } + } + return rc; } /* @@ -2611,11 +3170,14 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ */ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts5FullTable *pTab = (Fts5FullTable*)pVtab; - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + int rc = SQLITE_OK; fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); - pTab->p.pConfig->pgsz = 0; - return sqlite3Fts5StorageRollback(pTab->pStorage); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + pTab->p.pConfig->pgsz = 0; + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + } + return rc; } /* @@ -2657,47 +3219,210 @@ static int fts5CreateAux( } /* -** Register a new tokenizer. This is the implementation of the -** fts5_api.xCreateTokenizer() method. +** This function is used by xCreateTokenizer_v2() and xCreateTokenizer(). +** It allocates and partially populates a new Fts5TokenizerModule object. +** The new object is already linked into the Fts5Global context before +** returning. +** +** If successful, SQLITE_OK is returned and a pointer to the new +** Fts5TokenizerModule object returned via output parameter (*ppNew). All +** that is required is for the caller to fill in the methods in +** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native +** as appropriate. +** +** If an error occurs, an SQLite error code is returned and the final value +** of (*ppNew) undefined. */ -static int fts5CreateTokenizer( - fts5_api *pApi, /* Global context (one per db handle) */ +static int fts5NewTokenizerModule( + Fts5Global *pGlobal, /* Global context (one per db handle) */ const char *zName, /* Name of new function */ void *pUserData, /* User data for aux. function */ - fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ - void(*xDestroy)(void*) /* Destructor for pUserData */ + void(*xDestroy)(void*), /* Destructor for pUserData */ + Fts5TokenizerModule **ppNew ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; - Fts5TokenizerModule *pNew; - sqlite3_int64 nName; /* Size of zName and its \0 terminator */ - sqlite3_int64 nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ nName = strlen(zName) + 1; nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc64(nByte); + *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); if( pNew ){ - memset(pNew, 0, (size_t)nByte); pNew->zName = (char*)&pNew[1]; memcpy(pNew->zName, zName, nName); pNew->pUserData = pUserData; - pNew->x = *pTokenizer; pNew->xDestroy = xDestroy; pNew->pNext = pGlobal->pTok; pGlobal->pTok = pNew; if( pNew->pNext==0 ){ pGlobal->pDfltTok = pNew; } + } + + return rc; +} + +/* +** An instance of this type is used as the Fts5Tokenizer object for +** wrapper tokenizers - those that provide access to a v1 tokenizer via +** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer +** via the fts5_tokenizer API. +*/ +typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; +struct Fts5VtoVTokenizer { + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ + Fts5Tokenizer *pReal; +}; + +/* +** Create a wrapper tokenizer. The context argument pCtx points to the +** Fts5TokenizerModule object. +*/ +static int fts5VtoVCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx; + Fts5VtoVTokenizer *pNew = 0; + int rc = SQLITE_OK; + + pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + if( rc==SQLITE_OK ){ + pNew->x1 = pMod->x1; + pNew->x2 = pMod->x2; + pNew->bV2Native = pMod->bV2Native; + if( pMod->bV2Native ){ + rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + }else{ + rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + } + } + + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Delete an Fts5VtoVTokenizer wrapper tokenizer. +*/ +static void fts5VtoVDelete(Fts5Tokenizer *pTok){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + if( p ){ + if( p->bV2Native ){ + p->x2.xDelete(p->pReal); + }else{ + p->x1.xDelete(p->pReal); + } + sqlite3_free(p); + } +} + + +/* +** xTokenizer method for a wrapper tokenizer that offers the v1 interface +** (no support for locales). +*/ +static int fts5V1toV2Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native ); + return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** xTokenizer method for a wrapper tokenizer that offers the v2 interface +** (with locale support). +*/ +static int fts5V2toV1Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native==0 ); + UNUSED_PARAM2(pLocale,nLocale); + return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer_v2() method. +*/ +static int fts5CreateTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + + if( pTokenizer->iVersion>2 ){ + rc = SQLITE_ERROR; }else{ - rc = SQLITE_NOMEM; + Fts5TokenizerModule *pNew = 0; + rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew); + if( pNew ){ + pNew->x2 = *pTokenizer; + pNew->bV2Native = 1; + pNew->x1.xCreate = fts5VtoVCreate; + pNew->x1.xTokenize = fts5V1toV2Tokenize; + pNew->x1.xDelete = fts5VtoVDelete; + } } return rc; } +/* +** The fts5_api.xCreateTokenizer() method. +*/ +static int fts5CreateTokenizer( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5TokenizerModule *pNew = 0; + int rc = SQLITE_OK; + + rc = fts5NewTokenizerModule( + (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew + ); + if( pNew ){ + pNew->x1 = *pTokenizer; + pNew->x2.xCreate = fts5VtoVCreate; + pNew->x2.xTokenize = fts5V2toV1Tokenize; + pNew->x2.xDelete = fts5VtoVDelete; + } + return rc; +} + +/* +** Search the global context passed as the first argument for a tokenizer +** module named zName. If found, return a pointer to the Fts5TokenizerModule +** object. Otherwise, return NULL. +*/ static Fts5TokenizerModule *fts5LocateTokenizer( - Fts5Global *pGlobal, - const char *zName + Fts5Global *pGlobal, /* Global (one per db handle) object */ + const char *zName /* Name of tokenizer module to find */ ){ Fts5TokenizerModule *pMod = 0; @@ -2712,6 +3437,36 @@ static Fts5TokenizerModule *fts5LocateTokenizer( return pMod; } +/* +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer_v2() method. +*/ +static int fts5FindTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of tokenizer */ + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *ppTokenizer = &pMod->x2; + }else{ + *ppTokenizer = 0; + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. @@ -2727,53 +3482,75 @@ static int fts5FindTokenizer( pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); if( pMod ){ - *pTokenizer = pMod->x; - *ppUserData = pMod->pUserData; + if( pMod->bV2Native==0 ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *pTokenizer = pMod->x1; }else{ - memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + memset(pTokenizer, 0, sizeof(*pTokenizer)); + *ppUserData = 0; rc = SQLITE_ERROR; } return rc; } -int sqlite3Fts5GetTokenizer( - Fts5Global *pGlobal, - const char **azArg, - int nArg, - Fts5Config *pConfig, - char **pzErr -){ - Fts5TokenizerModule *pMod; +/* +** Attempt to instantiate the tokenizer. +*/ +int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ + const char **azArg = pConfig->t.azArg; + const int nArg = pConfig->t.nArg; + Fts5TokenizerModule *pMod = 0; int rc = SQLITE_OK; - pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]); }else{ - rc = pMod->x.xCreate( - pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; + if( pMod->bV2Native ){ + xCreate = pMod->x2.xCreate; + pConfig->t.pApi2 = &pMod->x2; + }else{ + pConfig->t.pApi1 = &pMod->x1; + xCreate = pMod->x1.xCreate; + } + + rc = xCreate(pMod->pUserData, + (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok ); - pConfig->pTokApi = &pMod->x; + if( rc!=SQLITE_OK ){ - if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor"); - }else{ - pConfig->ePattern = sqlite3Fts5TokenizerPattern( - pMod->x.xCreate, pConfig->pTok + if( rc!=SQLITE_NOMEM ){ + sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor"); + } + }else if( pMod->bV2Native==0 ){ + pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( + pMod->x1.xCreate, pConfig->t.pTok ); } } if( rc!=SQLITE_OK ){ - pConfig->pTokApi = 0; - pConfig->pTok = 0; + pConfig->t.pApi1 = 0; + pConfig->t.pApi2 = 0; + pConfig->t.pTok = 0; } return rc; } + +/* +** xDestroy callback passed to sqlite3_create_module(). This is invoked +** when the db handle is being closed. Free memory associated with +** tokenizers and aux functions registered with this db handle. +*/ static void fts5ModuleDestroy(void *pCtx){ Fts5TokenizerModule *pTok, *pNextTok; Fts5Auxiliary *pAux, *pNextAux; @@ -2794,6 +3571,10 @@ static void fts5ModuleDestroy(void *pCtx){ sqlite3_free(pGlobal); } +/* +** Implementation of the fts5() function used by clients to obtain the +** API pointer. +*/ static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ @@ -2820,6 +3601,81 @@ static void fts5SourceIdFunc( sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT); } +/* +** Implementation of fts5_locale(LOCALE, TEXT) function. +** +** If parameter LOCALE is NULL, or a zero-length string, then a copy of +** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as +** text, and the value returned is a blob consisting of: +** +** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER). +** * The LOCALE, as utf-8 text, followed by +** * 0x00, followed by +** * The TEXT, as utf-8 text. +** +** There is no final nul-terminator following the TEXT value. +*/ +static void fts5LocaleFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + const char *zLocale = 0; + i64 nLocale = 0; + const char *zText = 0; + i64 nText = 0; + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + + zLocale = (const char*)sqlite3_value_text(apArg[0]); + nLocale = sqlite3_value_bytes(apArg[0]); + + zText = (const char*)sqlite3_value_text(apArg[1]); + nText = sqlite3_value_bytes(apArg[1]); + + if( zLocale==0 || zLocale[0]=='\0' ){ + sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT); + }else{ + Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); + u8 *pBlob = 0; + u8 *pCsr = 0; + i64 nBlob = 0; + + nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; + pBlob = (u8*)sqlite3_malloc64(nBlob); + if( pBlob==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + + pCsr = pBlob; + memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE); + pCsr += FTS5_LOCALE_HDR_SIZE; + memcpy(pCsr, zLocale, nLocale); + pCsr += nLocale; + (*pCsr++) = 0x00; + if( zText ) memcpy(pCsr, zText, nText); + assert( &pCsr[nText]==&pBlob[nBlob] ); + + sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); + } +} + +/* +** Implementation of fts5_insttoken() function. +*/ +static void fts5InsttokenFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + assert( nArg==1 ); + (void)nArg; + sqlite3_result_value(pCtx, apArg[0]); + sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE); +} + /* ** Return true if zName is the extension on one of the shadow tables used ** by this module. @@ -2835,9 +3691,48 @@ static int fts5ShadowName(const char *zName){ return 0; } +/* +** Run an integrity check on the FTS5 data structures. Return a string +** if anything is found amiss. Return a NULL pointer if everything is +** OK. +*/ +static int fts5IntegrityMethod( + sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */ + const char *zSchema, /* Name of schema in which this table lives */ + const char *zTabname, /* Name of the table itself */ + int isQuick, /* True if this is a quick-check */ + char **pzErr /* Write error message here */ +){ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc; + + assert( pzErr!=0 && *pzErr==0 ); + UNUSED_PARAM(isQuick); + assert( pTab->p.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } + }else if( (rc&0xff)==SQLITE_CORRUPT ){ + rc = SQLITE_OK; + } + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; + + return rc; +} + static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { - /* iVersion */ 3, + /* iVersion */ 4, /* xCreate */ fts5CreateMethod, /* xConnect */ fts5ConnectMethod, /* xBestIndex */ fts5BestIndexMethod, @@ -2860,7 +3755,8 @@ static int fts5Init(sqlite3 *db){ /* xSavepoint */ fts5SavepointMethod, /* xRelease */ fts5ReleaseMethod, /* xRollbackTo */ fts5RollbackToMethod, - /* xShadowName */ fts5ShadowName + /* xShadowName */ fts5ShadowName, + /* xIntegrity */ fts5IntegrityMethod }; int rc; @@ -2873,10 +3769,22 @@ static int fts5Init(sqlite3 *db){ void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 2; + pGlobal->api.iVersion = 3; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; + pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; + pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; + + /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector. + ** The constants below were generated randomly. */ + sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr); + pGlobal->aLocaleHdr[0] ^= 0xF924976D; + pGlobal->aLocaleHdr[1] ^= 0x16596E13; + pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA; + pGlobal->aLocaleHdr[3] ^= 0x9B03A67F; + assert( sizeof(pGlobal->aLocaleHdr)==16 ); + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); @@ -2895,6 +3803,20 @@ static int fts5Init(sqlite3 *db){ p, fts5SourceIdFunc, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_locale", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE, + p, fts5LocaleFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_insttoken", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5InsttokenFunc, 0, 0 + ); + } } /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file @@ -2902,8 +3824,8 @@ static int fts5Init(sqlite3 *db){ ** its entry point to enable the matchinfo() demo. */ #ifdef SQLITE_FTS5_ENABLE_TEST_MI if( rc==SQLITE_OK ){ - extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); - rc = sqlite3Fts5TestRegisterMatchinfo(db); + extern int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api*); + rc = sqlite3Fts5TestRegisterMatchinfoAPI(&pGlobal->api); } #endif diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 02b98d9e44..76820e85b3 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -16,13 +16,40 @@ #include "fts5Int.h" +/* +** pSavedRow: +** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it +** does a by-rowid lookup to retrieve a single row from the %_content +** table or equivalent external-content table/view. +** +** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original +** values for a row being UPDATEd. In that case, the SQL statement is +** not reset and pSavedRow is set to point at it. This is so that the +** insert operation that follows the delete may access the original +** row values for any new values for which sqlite3_value_nochange() returns +** true. i.e. if the user executes: +** +** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1); +** ... +** UPDATE fts SET a=?, b=? WHERE rowid=?; +** +** then the value passed to the xUpdate() method of this table as the +** new.c value is an sqlite3_value_nochange() value. So in this case it +** must be read from the saved row stored in Fts5Storage.pSavedRow. +** +** This is necessary - using sqlite3_value_nochange() instead of just having +** SQLite pass the original value back via xUpdate() - so as not to discard +** any locale information associated with such values. +** +*/ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ i64 nTotalRow; /* Total number of rows in FTS table */ i64 *aTotalSize; /* Total sizes of each column */ - sqlite3_stmt *aStmt[11]; + sqlite3_stmt *pSavedRow; + sqlite3_stmt *aStmt[12]; }; @@ -36,14 +63,15 @@ struct Fts5Storage { # error "FTS5_STMT_LOOKUP mismatch" #endif -#define FTS5_STMT_INSERT_CONTENT 3 -#define FTS5_STMT_REPLACE_CONTENT 4 -#define FTS5_STMT_DELETE_CONTENT 5 -#define FTS5_STMT_REPLACE_DOCSIZE 6 -#define FTS5_STMT_DELETE_DOCSIZE 7 -#define FTS5_STMT_LOOKUP_DOCSIZE 8 -#define FTS5_STMT_REPLACE_CONFIG 9 -#define FTS5_STMT_SCAN 10 +#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_INSERT_CONTENT 4 +#define FTS5_STMT_REPLACE_CONTENT 5 +#define FTS5_STMT_DELETE_CONTENT 6 +#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_DELETE_DOCSIZE 8 +#define FTS5_STMT_LOOKUP_DOCSIZE 9 +#define FTS5_STMT_REPLACE_CONFIG 10 +#define FTS5_STMT_SCAN 11 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -73,14 +101,15 @@ static int fts5StorageGetStmt( "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ - "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ - "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ "SELECT %s FROM %s AS T", /* SCAN */ @@ -88,6 +117,8 @@ static int fts5StorageGetStmt( Fts5Config *pC = p->pConfig; char *zSql = 0; + assert( ArraySize(azStmt)==ArraySize(p->aStmt) ); + switch( eStmt ){ case FTS5_STMT_SCAN: zSql = sqlite3_mprintf(azStmt[eStmt], @@ -104,6 +135,7 @@ static int fts5StorageGetStmt( break; case FTS5_STMT_LOOKUP: + case FTS5_STMT_LOOKUP2: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent, pC->zContentRowid ); @@ -111,23 +143,51 @@ static int fts5StorageGetStmt( case FTS5_STMT_INSERT_CONTENT: case FTS5_STMT_REPLACE_CONTENT: { - int nCol = pC->nCol + 1; - char *zBind; + char *zBind = 0; int i; - zBind = sqlite3_malloc64(1 + nCol*2); - if( zBind ){ - for(i=0; ieContent==FTS5_CONTENT_NORMAL + || pC->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Add bindings for the "c*" columns - those that store the actual + ** table content. If eContent==NORMAL, then there is one binding + ** for each column. Or, if eContent==UNINDEXED, then there are only + ** bindings for the UNINDEXED columns. */ + for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){ + if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1); } - zBind[i*2-1] = '\0'; - zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); - sqlite3_free(zBind); } + + /* Add bindings for any "l*" columns. Only non-UNINDEXED columns + ** require these. */ + if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){ + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pC->abUnindexed[i]==0 ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2); + } + } + } + + zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind); + sqlite3_free(zBind); break; } + case FTS5_STMT_REPLACE_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, + (pC->bContentlessDelete ? ",?" : "") + ); + break; + + case FTS5_STMT_LOOKUP_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], + (pC->bContentlessDelete ? ",origin" : ""), + pC->zDb, pC->zName + ); + break; + default: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); break; @@ -137,7 +197,7 @@ static int fts5StorageGetStmt( rc = SQLITE_NOMEM; }else{ int f = SQLITE_PREPARE_PERSISTENT; - if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB; + if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB; p->pConfig->bLock++; rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); p->pConfig->bLock--; @@ -145,6 +205,11 @@ static int fts5StorageGetStmt( if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); } + if( rc==SQLITE_ERROR && eStmt>FTS5_STMT_LOOKUP2 && eStmtpIndex = pIndex; if( bCreate ){ - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ int nDefn = 32 + pConfig->nCol*10; - char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10); + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20); if( zDefn==0 ){ rc = SQLITE_NOMEM; }else{ @@ -308,8 +375,20 @@ int sqlite3Fts5StorageOpen( sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); iOff = (int)strlen(zDefn); for(i=0; inCol; i++){ - sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); - iOff += (int)strlen(&zDefn[iOff]); + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->abUnindexed[i] + ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + if( pConfig->bLocale ){ + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } } rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } @@ -317,9 +396,11 @@ int sqlite3Fts5StorageOpen( } if( rc==SQLITE_OK && pConfig->bColumnsize ){ - rc = sqlite3Fts5CreateTable( - pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr - ); + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; + if( pConfig->bContentlessDelete ){ + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; + } + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( @@ -384,58 +465,129 @@ static int fts5StorageInsertCallback( return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } +/* +** This function is used as part of an UPDATE statement that modifies the +** rowid of a row. In that case, this function is called first to set +** Fts5Storage.pSavedRow to point to a statement that may be used to +** access the original values of the row being deleted - iDel. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if row iDel does not exist. In this case +** pSavedRow is not set and SQLITE_OK returned. +*/ +int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ + int rc = SQLITE_OK; + sqlite3_stmt *pSeek = 0; + + assert( p->pSavedRow==0 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + rc = sqlite3_reset(pSeek); + }else{ + p->pSavedRow = pSeek; + } + } + + return rc; +} + /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. +** +** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left +** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access +** the original values of the row being deleted. This is used by UPDATE +** statements. */ static int fts5StorageDeleteFromIndex( Fts5Storage *p, i64 iDel, - sqlite3_value **apVal + sqlite3_value **apVal, + int bSaveRow /* True to set pSavedRow */ ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int rc2; /* sqlite3_reset() return code */ int iCol; Fts5InsertCtx ctx; + assert( bSaveRow==0 || apVal==0 ); + assert( bSaveRow==0 || bSaveRow==1 ); + assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 ); + if( apVal==0 ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pSeek, 1, iDel); - if( sqlite3_step(pSeek)!=SQLITE_ROW ){ - return sqlite3_reset(pSeek); + if( p->pSavedRow && bSaveRow ){ + pSeek = p->pSavedRow; + p->pSavedRow = 0; + }else{ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } } } ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ - const char *zText; - int nText; + sqlite3_value *pVal = 0; + sqlite3_value *pFree = 0; + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + assert( pSeek==0 || apVal==0 ); assert( pSeek!=0 || apVal!=0 ); if( pSeek ){ - zText = (const char*)sqlite3_column_text(pSeek, iCol); - nText = sqlite3_column_bytes(pSeek, iCol); - }else if( ALWAYS(apVal) ){ - zText = (const char*)sqlite3_value_text(apVal[iCol-1]); - nText = sqlite3_value_bytes(apVal[iCol-1]); + pVal = sqlite3_column_value(pSeek, iCol); }else{ - continue; + pVal = apVal[iCol-1]; } - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, - zText, nText, (void*)&ctx, fts5StorageInsertCallback - ); - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; - if( p->aTotalSize[iCol-1]<0 ){ - rc = FTS5_CORRUPT; + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + if( sqlite3_value_type(pVal)!=SQLITE_TEXT ){ + /* Make a copy of the value to work with. This is because the call + ** to sqlite3_value_text() below forces the type of the value to + ** SQLITE_TEXT, and we may need to use it again later. */ + pFree = pVal = sqlite3_value_dup(pVal); + if( pVal==0 ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol+pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } + } } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + pText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ + rc = FTS5_CORRUPT; + } + sqlite3Fts5ClearLocale(pConfig); + } + sqlite3_value_free(pFree); } } if( rc==SQLITE_OK && p->nTotalRow<1 ){ @@ -444,11 +596,62 @@ static int fts5StorageDeleteFromIndex( p->nTotalRow--; } - rc2 = sqlite3_reset(pSeek); - if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bSaveRow ){ + assert( p->pSavedRow==0 ); + p->pSavedRow = pSeek; + }else{ + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + } return rc; } +/* +** Reset any saved statement pSavedRow. Zero pSavedRow as well. This +** should be called by the xUpdate() method of the fts5 table before +** returning from any operation that may have set Fts5Storage.pSavedRow. +*/ +void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + assert( pStorage->pSavedRow==0 + || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] + ); + sqlite3_reset(pStorage->pSavedRow); + pStorage->pSavedRow = 0; +} + +/* +** This function is called to process a DELETE on a contentless_delete=1 +** table. It adds the tombstone required to delete the entry with rowid +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, +** an SQLite error code. +*/ +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ + i64 iOrigin = 0; + sqlite3_stmt *pLookup = 0; + int rc = SQLITE_OK; + + assert( p->pConfig->bContentlessDelete ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE + || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Look up the origin of the document in the %_docsize table. Store + ** this in stack variable iOrigin. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLookup, 1, iDel); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + iOrigin = sqlite3_column_int64(pLookup, 1); + } + rc = sqlite3_reset(pLookup); + } + + if( rc==SQLITE_OK && iOrigin!=0 ){ + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + } + + return rc; +} /* ** Insert a record into the %_docsize table. Specifically, do: @@ -469,6 +672,13 @@ static int fts5StorageInsertDocsize( rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pReplace, 1, iRowid); + if( p->pConfig->bContentlessDelete ){ + i64 iOrigin = 0; + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); + sqlite3_bind_int64(pReplace, 3, iOrigin); + } + } + if( rc==SQLITE_OK ){ sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); @@ -526,7 +736,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){ /* ** Remove a row from the FTS table. */ -int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ +int sqlite3Fts5StorageDelete( + Fts5Storage *p, /* Storage object */ + i64 iDel, /* Rowid to delete from table */ + sqlite3_value **apVal, /* Optional - values to remove from index */ + int bSaveRow /* If true, set pSavedRow for deleted row */ +){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; @@ -536,7 +751,21 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ /* Delete the index records */ if( rc==SQLITE_OK ){ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + } + + if( rc==SQLITE_OK ){ + if( p->pConfig->bContentlessDelete ){ + rc = fts5StorageContentlessDelete(p, iDel); + if( rc==SQLITE_OK + && bSaveRow + && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ + rc = sqlite3Fts5StorageFindDeleteRow(p, iDel); + } + }else{ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow); + } } /* Delete the %_docsize record */ @@ -550,7 +779,9 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ } /* Delete the %_content record */ - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); } @@ -582,8 +813,13 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, - "DELETE FROM %Q.'%q_docsize';", - pConfig->zDb, pConfig->zName + "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName + ); + } + + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName ); } @@ -613,7 +849,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ } if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ @@ -624,14 +860,36 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1); - int nText = sqlite3_column_bytes(pScan, ctx.iCol+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback - ); + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pLoc in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -697,6 +955,7 @@ static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ */ int sqlite3Fts5StorageContentInsert( Fts5Storage *p, + int bReplace, /* True to use REPLACE instead of INSERT */ sqlite3_value **apVal, i64 *piRowid ){ @@ -704,7 +963,9 @@ int sqlite3Fts5StorageContentInsert( int rc = SQLITE_OK; /* Insert the new row into the %_content table. */ - if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && pConfig->eContent!=FTS5_CONTENT_UNINDEXED + ){ if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ *piRowid = sqlite3_value_int64(apVal[1]); }else{ @@ -713,9 +974,52 @@ int sqlite3Fts5StorageContentInsert( }else{ sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ int i; /* Counter variable */ - rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); + + assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT ); + assert( bReplace==0 || bReplace==1 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT+bReplace, &pInsert, 0); + if( pInsert ) sqlite3_clear_bindings(pInsert); + + /* Bind the rowid value */ + sqlite3_bind_value(pInsert, 1, apVal[1]); + + /* Loop through values for user-defined columns. i=2 is the leftmost + ** user-defined column. As is column 1 of pSavedRow. */ + for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + int bUnindexed = pConfig->abUnindexed[i-2]; + if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){ + sqlite3_value *pVal = apVal[i]; + + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + /* This is an UPDATE statement, and user-defined column (i-2) was not + ** modified. Retrieve the value from Fts5Storage.pSavedRow. */ + pVal = sqlite3_column_value(p->pSavedRow, i-1); + if( pConfig->bLocale && bUnindexed==0 ){ + sqlite3_bind_value(pInsert, pConfig->nCol + i, + sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1) + ); + } + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + const char *pLoc = 0; + int nText = 0; + int nLoc = 0; + assert( pConfig->bLocale ); + + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( bUnindexed==0 ){ + int iLoc = pConfig->nCol + i; + sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT); + } + } + + continue; + } + + rc = sqlite3_bind_value(pInsert, i, pVal); + } } if( rc==SQLITE_OK ){ sqlite3_step(pInsert); @@ -750,14 +1054,38 @@ int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]); - int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageInsertCallback - ); + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pText in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = apVal[ctx.iCol+2]; + if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ + pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol); + nLoc = sqlite3_column_bytes(p->pSavedRow, iCol); + } + }else{ + pVal = apVal[ctx.iCol+2]; + } + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -921,29 +1249,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } for(i=0; rc==SQLITE_OK && inCol; i++){ - if( pConfig->abUnindexed[i] ) continue; - ctx.iCol = i; - ctx.szCol = 0; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - rc = sqlite3Fts5TermsetNew(&ctx.pTermset); - } - if( rc==SQLITE_OK ){ - const char *zText = (const char*)sqlite3_column_text(pScan, i+1); - int nText = sqlite3_column_bytes(pScan, i+1); - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - zText, nText, - (void*)&ctx, - fts5StorageIntegrityCallback - ); - } - if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ - rc = FTS5_CORRUPT; - } - aTotalSize[i] += ctx.szCol; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; + if( pConfig->abUnindexed[i]==0 ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + sqlite3_value *pVal = sqlite3_column_value(pScan, i+1); + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue( + pVal, &pText, &nText, &pLoc, &nLoc + ); + }else{ + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = i + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + ctx.iCol = i; + ctx.szCol = 0; + + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } + + /* If this is not a columnsize=0 database, check that the number + ** of tokens in the value matches the aColSize[] value read from + ** the %_docsize table. */ + if( rc==SQLITE_OK + && pConfig->bColumnsize + && ctx.szCol!=aColSize[i] + ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } } } sqlite3Fts5TermsetFree(ctx.pTermset); @@ -1124,7 +1484,9 @@ int sqlite3Fts5StorageSync(Fts5Storage *p){ i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); if( p->bTotalsValid ){ rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; + if( rc==SQLITE_OK ){ + p->bTotalsValid = 0; + } } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexSync(p->pIndex); diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 80c600dbb1..25cd5c0633 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -14,20 +14,14 @@ #ifdef SQLITE_TEST -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -# ifndef SQLITE_TCLAPI -# define SQLITE_TCLAPI -# endif -#endif +#include "tclsqlite.h" #ifdef SQLITE_ENABLE_FTS5 #include "fts5.h" #include #include +#include #ifdef SQLITE_DEBUG extern int sqlite3_fts5_may_be_corrupt; @@ -103,14 +97,14 @@ static int SQLITE_TCLAPI f5tDbAndApi( rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); return TCL_ERROR; } sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0); sqlite3_step(pStmt); if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); return TCL_ERROR; } @@ -244,6 +238,10 @@ static int SQLITE_TCLAPI xF5tApi( { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ + + { "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */ + { "xInstToken", 2, "IDX ITERM" }, /* 19 */ + { "xColumnLocale", 1, "COL" }, /* 20 */ { 0, 0, 0} }; @@ -294,12 +292,12 @@ static int SQLITE_TCLAPI xF5tApi( break; } CASE(3, "xTokenize") { - int nText; + Tcl_Size nText; char *zText = Tcl_GetStringFromObj(objv[2], &nText); F5tFunction ctx; ctx.interp = interp; ctx.pScript = objv[3]; - rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb); + rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb); if( rc==SQLITE_OK ){ Tcl_ResetResult(interp); } @@ -395,7 +393,7 @@ static int SQLITE_TCLAPI xF5tApi( CASE(12, "xSetAuxdata") { F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); if( pData==0 ){ - Tcl_AppendResult(interp, "out of memory", 0); + Tcl_AppendResult(interp, "out of memory", (char*)0); return TCL_ERROR; } pData->pObj = objv[2]; @@ -455,7 +453,7 @@ static int SQLITE_TCLAPI xF5tApi( rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ @@ -500,6 +498,52 @@ static int SQLITE_TCLAPI xF5tApi( break; } + CASE(18, "xQueryToken") { + const char *pTerm = 0; + int nTerm = 0; + int iPhrase = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + + CASE(19, "xInstToken") { + const char *pTerm = 0; + int nTerm = 0; + int iIdx = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + + CASE(20, "xColumnLocale") { + const char *z = 0; + int n = 0; + int iCol; + if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ + return TCL_ERROR; + } + rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n); + if( rc==SQLITE_OK && z ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); + } + break; + } + default: assert( 0 ); break; @@ -570,15 +614,16 @@ static void xF5tFunction( sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1); }else{ Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); - int n; const char *zType = (pVar->typePtr ? pVar->typePtr->name : ""); char c = zType[0]; if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ /* Only return a BLOB type if the Tcl variable is a bytearray and ** has no string representation. */ - unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n); - sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT); + Tcl_Size nn; + unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn); + sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT); }else if( c=='b' && strcmp(zType,"boolean")==0 ){ + int n; Tcl_GetIntFromObj(0, pVar, &n); sqlite3_result_int(pCtx, n); }else if( c=='d' && strcmp(zType,"double")==0 ){ @@ -591,8 +636,9 @@ static void xF5tFunction( Tcl_GetWideIntFromObj(0, pVar, &v); sqlite3_result_int64(pCtx, v); }else{ - unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); - sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT); + Tcl_Size nn; + unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn); + sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT); } } } @@ -638,7 +684,7 @@ static int SQLITE_TCLAPI f5tCreateFunction( pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy ); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); return TCL_ERROR; } @@ -684,8 +730,9 @@ static int SQLITE_TCLAPI f5tTokenize( int objc, Tcl_Obj *CONST objv[] ){ - char *zText; - int nText; + char *pCopy = 0; + char *zText = 0; + Tcl_Size nText = 0; sqlite3 *db = 0; fts5_api *pApi = 0; Fts5Tokenizer *pTok = 0; @@ -694,7 +741,7 @@ static int SQLITE_TCLAPI f5tTokenize( void *pUserdata; int rc; - int nArg; + Tcl_Size nArg; const char **azArg; F5tTokenizeCtx ctx; @@ -705,7 +752,7 @@ static int SQLITE_TCLAPI f5tTokenize( if( objc==5 ){ char *zOpt = Tcl_GetString(objv[1]); if( strcmp("-subst", zOpt) ){ - Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0); + Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0); return TCL_ERROR; } } @@ -714,7 +761,7 @@ static int SQLITE_TCLAPI f5tTokenize( return TCL_ERROR; } if( nArg==0 ){ - Tcl_AppendResult(interp, "no such tokenizer: ", 0); + Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0); Tcl_Free((void*)azArg); return TCL_ERROR; } @@ -722,32 +769,43 @@ static int SQLITE_TCLAPI f5tTokenize( rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0); + Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0); return TCL_ERROR; } - rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok); + rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0); + Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0); return TCL_ERROR; } + if( nText>0 ){ + pCopy = sqlite3_malloc(nText); + if( pCopy==0 ){ + tokenizer.xDelete(pTok); + Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); + return TCL_ERROR; + }else{ + memcpy(pCopy, zText, nText); + } + } + pRet = Tcl_NewObj(); Tcl_IncrRefCount(pRet); ctx.bSubst = (objc==5); ctx.pRet = pRet; - ctx.zInput = zText; + ctx.zInput = pCopy; rc = tokenizer.xTokenize( - pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2 + pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, pCopy,(int)nText, xTokenizeCb2 ); tokenizer.xDelete(pTok); + sqlite3_free(pCopy); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0); + Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0); Tcl_DecrRefCount(pRet); return TCL_ERROR; } - Tcl_Free((void*)azArg); Tcl_SetObjResult(interp, pRet); Tcl_DecrRefCount(pRet); @@ -766,18 +824,32 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; int (*xToken)(void*, int, const char*, int, int, int); + F5tTokenizerInstance *pInst; }; struct F5tTokenizerModule { Tcl_Interp *interp; Tcl_Obj *pScript; + void *pParentCtx; + fts5_tokenizer_v2 parent_v2; + fts5_tokenizer parent; F5tTokenizerContext *pContext; }; +/* +** zLocale: +** Within a call to xTokenize_v2(), pLocale/nLocale store the locale +** passed to the call by fts5. This can be retrieved by a Tcl tokenize +** script using [sqlite3_fts5_locale]. +*/ struct F5tTokenizerInstance { Tcl_Interp *interp; Tcl_Obj *pScript; + F5tTokenizerModule *pModule; + Fts5Tokenizer *pParent; F5tTokenizerContext *pContext; + const char *pLocale; + int nLocale; }; static int f5tTokenizerCreate( @@ -786,11 +858,20 @@ static int f5tTokenizerCreate( int nArg, Fts5Tokenizer **ppOut ){ + Fts5Tokenizer *pParent = 0; F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; Tcl_Obj *pEval; int rc = TCL_OK; int i; + assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 ); + if( pMod->parent_v2.xCreate ){ + rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + if( pMod->parent.xCreate ){ + rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent); + } + pEval = Tcl_DuplicateObj(pMod->pScript); Tcl_IncrRefCount(pEval); for(i=0; rc==TCL_OK && iinterp = pMod->interp; pInst->pScript = Tcl_GetObjResult(pMod->interp); pInst->pContext = pMod->pContext; + pInst->pParent = pParent; + pInst->pModule = pMod; Tcl_IncrRefCount(pInst->pScript); *ppOut = (Fts5Tokenizer*)pInst; } @@ -820,11 +903,21 @@ static int f5tTokenizerCreate( static void f5tTokenizerDelete(Fts5Tokenizer *p){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; - Tcl_DecrRefCount(pInst->pScript); - ckfree((char *)pInst); + if( pInst ){ + if( pInst->pParent ){ + if( pInst->pModule->parent_v2.xDelete ){ + pInst->pModule->parent_v2.xDelete(pInst->pParent); + }else{ + pInst->pModule->parent.xDelete(pInst->pParent); + } + } + Tcl_DecrRefCount(pInst->pScript); + ckfree((char *)pInst); + } } -static int f5tTokenizerTokenize( + +static int f5tTokenizerReallyTokenize( Fts5Tokenizer *p, void *pCtx, int flags, @@ -832,6 +925,7 @@ static int f5tTokenizerTokenize( int (*xToken)(void*, int, const char*, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + F5tTokenizerInstance *pOldInst = 0; void *pOldCtx; int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; @@ -840,9 +934,11 @@ static int f5tTokenizerTokenize( pOldCtx = pInst->pContext->pCtx; xOldToken = pInst->pContext->xToken; + pOldInst = pInst->pContext->pInst; pInst->pContext->pCtx = pCtx; pInst->pContext->xToken = xToken; + pInst->pContext->pInst = pInst; assert( flags==FTS5_TOKENIZE_DOCUMENT @@ -878,9 +974,105 @@ static int f5tTokenizerTokenize( pInst->pContext->pCtx = pOldCtx; pInst->pContext->xToken = xOldToken; + pInst->pContext->pInst = pOldInst; return rc; } +typedef struct CallbackCtx CallbackCtx; +struct CallbackCtx { + Fts5Tokenizer *p; + void *pCtx; + int flags; + int (*xToken)(void*, int, const char*, int, int, int); +}; + +static int f5tTokenizeCallback( + void *pCtx, + int tflags, + const char *z, int n, + int iStart, int iEnd +){ + CallbackCtx *p = (CallbackCtx*)pCtx; + return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken); +} + +static int f5tTokenizerTokenize_v2( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + const char *pLoc, int nLoc, + int (*xToken)(void*, int, const char*, int, int, int) +){ + int rc = SQLITE_OK; + F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; + + pInst->pLocale = pLoc; + pInst->nLocale = nLoc; + + if( pInst->pParent ){ + CallbackCtx ctx; + ctx.p = p; + ctx.pCtx = pCtx; + ctx.flags = flags; + ctx.xToken = xToken; + if( pInst->pModule->parent_v2.xTokenize ){ + rc = pInst->pModule->parent_v2.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, + pLoc, nLoc, f5tTokenizeCallback + ); + }else{ + rc = pInst->pModule->parent.xTokenize( + pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback + ); + } + }else{ + rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); + } + + pInst->pLocale = 0; + pInst->nLocale = 0; + return rc; +} +static int f5tTokenizerTokenize( + Fts5Tokenizer *p, + void *pCtx, + int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** sqlite3_fts5_locale +*/ +static int SQLITE_TCLAPI f5tTokenizerLocale( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; + + if( objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, ""); + return TCL_ERROR; + } + + if( p->xToken==0 ){ + Tcl_AppendResult(interp, + "sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0 + ); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, + Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale) + ); + return TCL_OK; +} + /* ** sqlite3_fts5_token ?-colocated? TEXT START END */ @@ -893,13 +1085,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn( F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; int iStart; int iEnd; - int nToken; + Tcl_Size nToken; int tflags = 0; char *zToken; int rc; if( objc==5 ){ - int nArg; + Tcl_Size nArg; char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ tflags |= FTS5_TOKEN_COLOCATED; @@ -919,12 +1111,12 @@ static int SQLITE_TCLAPI f5tTokenizerReturn( if( p->xToken==0 ){ Tcl_AppendResult(interp, - "sqlite3_fts5_token may only be used by tokenizer callback", 0 + "sqlite3_fts5_token may only be used by tokenizer callback", (char*)0 ); return TCL_ERROR; } - rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return rc==SQLITE_OK ? TCL_OK : TCL_ERROR; @@ -966,32 +1158,112 @@ static int SQLITE_TCLAPI f5tCreateTokenizer( fts5_api *pApi; char *zName; Tcl_Obj *pScript; - fts5_tokenizer t; F5tTokenizerModule *pMod; - int rc; - - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT"); + int rc = SQLITE_OK; + int bV2 = 0; /* True to use _v2 API */ + int iVersion = 2; /* Value for _v2.iVersion */ + const char *zParent = 0; /* Name of parent tokenizer, if any */ + int ii = 0; + + if( objc<4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT"); return TCL_ERROR; } - if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ - return TCL_ERROR; + + /* Parse any options. Set stack variables bV2 and zParent. */ + for(ii=1; iiinterp = interp; pMod->pScript = pScript; - pMod->pContext = pContext; Tcl_IncrRefCount(pScript); - rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer); + pMod->pContext = pContext; + if( zParent ){ + if( bV2 ){ + fts5_tokenizer_v2 *pParent = 0; + rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent); + if( rc==SQLITE_OK ){ + memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2)); + pMod->parent_v2.xDelete(0); + } + }else{ + rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent); + if( rc==SQLITE_OK ){ + pMod->parent.xDelete(0); + } + } + } + + if( rc==SQLITE_OK ){ + void *pModCtx = (void*)pMod; + if( bV2==0 ){ + fts5_tokenizer t; + t.xCreate = f5tTokenizerCreate; + t.xTokenize = f5tTokenizerTokenize; + t.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer); + }else{ + fts5_tokenizer_v2 t2; + memset(&t2, 0, sizeof(t2)); + t2.iVersion = iVersion; + t2.xCreate = f5tTokenizerCreate; + t2.xTokenize = f5tTokenizerTokenize_v2; + t2.xDelete = f5tTokenizerDelete; + rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); + } + } + if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0); + Tcl_AppendResult(interp, ( + bV2 ? "error in fts5_api.xCreateTokenizer_v2()" + : "error in fts5_api.xCreateTokenizer()" + ), (char*)0); return TCL_ERROR; } @@ -1048,7 +1320,7 @@ static int SQLITE_TCLAPI f5tTokenHash( Tcl_Obj *CONST objv[] ){ char *z; - int n; + Tcl_Size n; unsigned int iVal; int nSlot; @@ -1061,7 +1333,7 @@ static int SQLITE_TCLAPI f5tTokenHash( } z = Tcl_GetStringFromObj(objv[2], &n); - iVal = f5t_fts5HashKey(nSlot, z, n); + iVal = f5t_fts5HashKey(nSlot, z, (int)n); Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); return TCL_OK; } @@ -1117,6 +1389,313 @@ static int SQLITE_TCLAPI f5tRegisterTok( return TCL_OK; } +typedef struct OriginTextCtx OriginTextCtx; +struct OriginTextCtx { + sqlite3 *db; + fts5_api *pApi; +}; + +typedef struct OriginTextTokenizer OriginTextTokenizer; +struct OriginTextTokenizer { + Fts5Tokenizer *pTok; /* Underlying tokenizer object */ + fts5_tokenizer tokapi; /* API implementation for pTok */ +}; + +/* +** Delete the OriginTextCtx object indicated by the only argument. +*/ +static void f5tOrigintextTokenizerDelete(void *pCtx){ + OriginTextCtx *p = (OriginTextCtx*)pCtx; + ckfree((char*)p); +} + +static int f5tOrigintextCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + OriginTextCtx *p = (OriginTextCtx*)pCtx; + OriginTextTokenizer *pTok = 0; + void *pTokCtx = 0; + int rc = SQLITE_OK; + + pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); + if( pTok==0 ){ + rc = SQLITE_NOMEM; + }else if( nArg<1 ){ + rc = SQLITE_ERROR; + }else{ + /* Locate the underlying tokenizer */ + rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi); + } + + /* Create the new tokenizer instance */ + if( rc==SQLITE_OK ){ + rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pTok); + pTok = 0; + } + *ppOut = (Fts5Tokenizer*)pTok; + return rc; +} + +static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){ + OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; + if( p->pTok ){ + p->tokapi.xDelete(p->pTok); + } + sqlite3_free(p); +} + +typedef struct OriginTextCb OriginTextCb; +struct OriginTextCb { + void *pCtx; + const char *pText; + int nText; + int (*xToken)(void *, int, const char *, int, int, int); + + char *aBuf; /* Buffer to use */ + int nBuf; /* Allocated size of aBuf[] */ +}; + +static int xOriginToken( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input */ +){ + OriginTextCb *p = (OriginTextCb*)pCtx; + int ret = 0; + + if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){ + /* Token exactly matches document text. Pass it through as is. */ + ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); + }else{ + int nReq = nToken + 1 + (iEnd-iStart); + if( nReq>p->nBuf ){ + sqlite3_free(p->aBuf); + p->aBuf = sqlite3_malloc(nReq*2); + if( p->aBuf==0 ) return SQLITE_NOMEM; + p->nBuf = nReq*2; + } + + memcpy(p->aBuf, pToken, nToken); + p->aBuf[nToken] = '\0'; + memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart); + ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd); + } + + return ret; +} + + +static int f5tOrigintextTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, int) +){ + OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; + OriginTextCb cb; + int ret; + + memset(&cb, 0, sizeof(cb)); + cb.pCtx = pCtx; + cb.pText = pText; + cb.nText = nText; + cb.xToken = xToken; + + ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken); + sqlite3_free(cb.aBuf); + return ret; +} + +/* +** sqlite3_fts5_register_origintext DB +** +** Description... +*/ +static int SQLITE_TCLAPI f5tRegisterOriginText( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + fts5_api *pApi = 0; + int rc; + fts5_tokenizer tok = {0, 0, 0}; + OriginTextCtx *pCtx = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR; + + pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx)); + pCtx->db = db; + pCtx->pApi = pApi; + + tok.xCreate = f5tOrigintextCreate; + tok.xDelete = f5tOrigintextDelete; + tok.xTokenize = f5tOrigintextTokenize; + rc = pApi->xCreateTokenizer( + pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete + ); + + Tcl_ResetResult(interp); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** This function is used to DROP an fts5 table. It works even if the data +** structures fts5 stores within the database are corrupt, which sometimes +** prevents a straight "DROP TABLE" command from succeeding. +** +** The first parameter is the database handle to use for the DROP TABLE +** operation. The second is the name of the database to drop the fts5 table +** from (i.e. "main", "temp" or the name of an attached database). The +** third parameter is the name of the fts5 table to drop. +** +** SQLITE_OK is returned if the table is successfully dropped. Or, if an +** error occurs, an SQLite error code. +*/ +static int sqlite3_fts5_drop_corrupt_table( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zTab /* Name of fts5 table to drop */ +){ + int rc = SQLITE_OK; + int bDef = 0; + + rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef); + if( rc==SQLITE_OK ){ + char *zScript = sqlite3_mprintf( + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_config';" + "INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');" + "INSERT INTO %Q.'%q_config' VALUES('version', 4);" + "DROP TABLE %Q.'%q';", + zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab + ); + + if( zScript==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); + rc = sqlite3_exec(db, zScript, 0, 0, 0); + if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0); + sqlite3_free(zScript); + } + } + + return rc; +} + +/* +** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE +** +** Description... +*/ +static int SQLITE_TCLAPI f5tDropCorruptTable( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + const char *zTab = 0; + int rc = SQLITE_OK; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + zTab = Tcl_GetString(objv[3]); + + rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); + return TCL_ERROR; + } + + return TCL_OK; +} + +/* +** Free a buffer returned to SQLite by the str() function. +*/ +void f5tFree(void *p){ + char *x = (char *)p; + ckfree(&x[-8]); +} + +/* +** Implementation of str(). +*/ +void f5tStrFunc(sqlite3_context *pCtx, int nArg, sqlite3_value **apArg){ + const char *zText = 0; + assert( nArg==1 ); + + zText = (const char*)sqlite3_value_text(apArg[0]); + if( zText ){ + sqlite3_int64 nText = strlen(zText); + char *zCopy = (char*)ckalloc(nText+8); + if( zCopy==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + zCopy += 8; + memcpy(zCopy, zText, nText); + sqlite3_result_text64(pCtx, zCopy, nText, f5tFree, SQLITE_UTF8); + } + } +} + +/* +** sqlite3_fts5_register_str DB +** +** Register the str() function with database handle DB. str() interprets +** its only argument as text and returns a copy of the value in a +** non-nul-terminated buffer. +*/ +static int SQLITE_TCLAPI f5tRegisterStr( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbPointer(interp, objv[1], &db) ){ + return TCL_ERROR; + } + + sqlite3_create_function(db, "str", 1, SQLITE_UTF8, 0, f5tStrFunc, 0, 0); + + return TCL_OK; +} + /* ** Entry point. */ @@ -1128,12 +1707,16 @@ int Fts5tcl_Init(Tcl_Interp *interp){ } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, + { "sqlite3_fts5_locale", f5tTokenizerLocale, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, - { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 } + { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }, + { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }, + { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }, + { "sqlite3_fts5_register_str", f5tRegisterStr, 0 } }; int i; F5tTokenizerContext *pContext; diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 6f2d6e7ea2..e8648a4d22 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -14,7 +14,7 @@ ** versions of FTS5. It contains the implementation of an FTS5 auxiliary ** function very similar to the FTS4 function matchinfo(): ** -** https://www.sqlite.org/fts3.html#matchinfo +** https://sqlite.org/fts3.html#matchinfo ** ** Known differences are that: ** @@ -393,17 +393,13 @@ static void fts5MatchinfoFunc( } } -int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ - int rc; /* Return code */ - fts5_api *pApi; /* FTS5 API functions */ - - /* Extract the FTS5 API pointer from the database handle. The - ** fts5_api_from_db() function above is copied verbatim from the - ** FTS5 documentation. Refer there for details. */ - rc = fts5_api_from_db(db, &pApi); - if( rc!=SQLITE_OK ) return rc; +/* +** Register "matchinfo" with global API object pApi. +*/ +int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api *pApi){ + int rc; - /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered + /* If fts5_api_from_db() returned NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** ** Also check that the fts5_api object is version 2 or newer. @@ -418,4 +414,20 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ return rc; } +/* +** Register "matchinfo" with database handle db. +*/ +int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ + int rc; /* Return code */ + fts5_api *pApi; /* FTS5 API functions */ + + /* Extract the FTS5 API pointer from the database handle. The + ** fts5_api_from_db() function above is copied verbatim from the + ** FTS5 documentation. Refer there for details. */ + rc = fts5_api_from_db(db, &pApi); + if( rc!=SQLITE_OK ) return rc; + + return sqlite3Fts5TestRegisterMatchinfoAPI(pApi); +} + #endif /* SQLITE_ENABLE_FTS5 */ diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index a5d839da66..994d304dc6 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -472,7 +472,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index e61d6b1edd..b8a1136465 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -198,7 +198,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { c = *(zIn++); \ if( c>=0xc0 ){ \ c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + while( zIn=0xc0 ){ \ + while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + typedef struct Unicode61Tokenizer Unicode61Tokenizer; struct Unicode61Tokenizer { unsigned char aTokenChar[128]; /* ASCII range token characters */ @@ -380,7 +386,6 @@ static int fts5UnicodeCreate( zCat = azArg[i+1]; } } - if( rc==SQLITE_OK ){ rc = unicodeSetCategories(p, zCat); } @@ -410,7 +415,6 @@ static int fts5UnicodeCreate( rc = SQLITE_ERROR; } } - }else{ rc = SQLITE_NOMEM; } @@ -549,7 +553,7 @@ static int fts5UnicodeTokenize( typedef struct PorterTokenizer PorterTokenizer; struct PorterTokenizer { - fts5_tokenizer tokenizer; /* Parent tokenizer module */ + fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */ Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; }; @@ -561,7 +565,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){ if( pTok ){ PorterTokenizer *p = (PorterTokenizer*)pTok; if( p->pTokenizer ){ - p->tokenizer.xDelete(p->pTokenizer); + p->tokenizer_v2.xDelete(p->pTokenizer); } sqlite3_free(p); } @@ -580,6 +584,7 @@ static int fts5PorterCreate( PorterTokenizer *pRet; void *pUserdata = 0; const char *zBase = "unicode61"; + fts5_tokenizer_v2 *pV2 = 0; if( nArg>0 ){ zBase = azArg[0]; @@ -588,14 +593,15 @@ static int fts5PorterCreate( pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); }else{ rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ int nArg2 = (nArg>0 ? nArg-1 : 0); - const char **azArg2 = (nArg2 ? &azArg[1] : 0); - rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + const char **az2 = (nArg2 ? &azArg[1] : 0); + memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2)); + rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer); } if( rc!=SQLITE_OK ){ @@ -1246,6 +1252,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, + const char *pLoc, int nLoc, int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; @@ -1253,8 +1260,8 @@ static int fts5PorterTokenize( sCtx.xToken = xToken; sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; - return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + return p->tokenizer_v2.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb ); } @@ -1264,6 +1271,7 @@ static int fts5PorterTokenize( typedef struct TrigramTokenizer TrigramTokenizer; struct TrigramTokenizer { int bFold; /* True to fold to lower-case */ + int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */ }; /* @@ -1283,28 +1291,46 @@ static int fts5TriCreate( Fts5Tokenizer **ppOut ){ int rc = SQLITE_OK; - TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + TrigramTokenizer *pNew = 0; UNUSED_PARAM(pUnused); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + if( nArg%2 ){ + rc = SQLITE_ERROR; }else{ int i; - pNew->bFold = 1; - for(i=0; rc==SQLITE_OK && ibFold = 1; + pNew->iFoldParam = 0; + + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); + } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } }else{ - pNew->bFold = (zArg[0]=='0'); + rc = SQLITE_ERROR; } - }else{ + } + + if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ rc = SQLITE_ERROR; } - } - if( rc!=SQLITE_OK ){ - fts5TriDelete((Fts5Tokenizer*)pNew); - pNew = 0; + + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } } } *ppOut = (Fts5Tokenizer*)pNew; @@ -1324,40 +1350,65 @@ static int fts5TriTokenize( TrigramTokenizer *p = (TrigramTokenizer*)pTok; int rc = SQLITE_OK; char aBuf[32]; + char *zOut = aBuf; + int ii; const unsigned char *zIn = (const unsigned char*)pText; - const unsigned char *zEof = &zIn[nText]; - u32 iCode; + const unsigned char *zEof = (zIn ? &zIn[nText] : 0); + u32 iCode = 0; + int aStart[3]; /* Input offset of each character in aBuf[] */ UNUSED_PARAM(unusedFlags); - while( 1 ){ - char *zOut = aBuf; - int iStart = zIn - (const unsigned char*)pText; - const unsigned char *zNext; - - READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) break; - zNext = zIn; - if( zInbFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); - WRITE_UTF8(zOut, iCode); + + /* Populate aBuf[] with the characters for the first trigram. */ + for(ii=0; ii<3; ii++){ + do { + aStart[ii] = zIn - (const unsigned char*)pText; + if( zIn>=zEof ) return SQLITE_OK; READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) break; - }else{ - break; - } - if( zInbFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); - WRITE_UTF8(zOut, iCode); + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + WRITE_UTF8(zOut, iCode); + } + + /* At the start of each iteration of this loop: + ** + ** aBuf: Contains 3 characters. The 3 characters of the next trigram. + ** zOut: Points to the byte following the last character in aBuf. + ** aStart[3]: Contains the byte offset in the input text corresponding + ** to the start of each of the three characters in the buffer. + */ + assert( zIn<=zEof ); + while( 1 ){ + int iNext; /* Start of character following current tri */ + const char *z1; + + /* Read characters from the input up until the first non-diacritic */ + do { + iNext = zIn - (const unsigned char*)pText; + if( zIn>=zEof ){ + iCode = 0; + break; + } READ_UTF8(zIn, zEof, iCode); - if( iCode==0 ) break; - if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0); - WRITE_UTF8(zOut, iCode); - }else{ - break; - } - rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf); - if( rc!=SQLITE_OK ) break; - zIn = zNext; + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + + /* Pass the current trigram back to fts5 */ + rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext); + if( iCode==0 || rc!=SQLITE_OK ) break; + + /* Remove the first character from buffer aBuf[]. Append the character + ** with codepoint iCode. */ + z1 = aBuf; + FTS5_SKIP_UTF8(z1); + memmove(aBuf, z1, zOut - z1); + zOut -= (z1 - aBuf); + WRITE_UTF8(zOut, iCode); + + /* Update the aStart[] array */ + aStart[0] = aStart[1]; + aStart[1] = aStart[2]; + aStart[2] = iNext; } return rc; @@ -1380,11 +1431,23 @@ int sqlite3Fts5TokenizerPattern( ){ if( xCreate==fts5TriCreate ){ TrigramTokenizer *p = (TrigramTokenizer*)pTok; - return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + if( p->iFoldParam==0 ){ + return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + } } return FTS5_PATTERN_NONE; } +/* +** Return true if the tokenizer described by p->azArg[] is the trigram +** tokenizer. This tokenizer needs to be loaded before xBestIndex is +** called for the first time in order to correctly handle LIKE/GLOB. +*/ +int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){ + return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram")); +} + + /* ** Register all built-in tokenizers with FTS5. */ @@ -1395,7 +1458,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ } aBuiltin[] = { { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, - { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, }; @@ -1410,6 +1472,19 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ 0 ); } - + if( rc==SQLITE_OK ){ + fts5_tokenizer_v2 sPorter = { + 2, + fts5PorterCreate, + fts5PorterDelete, + fts5PorterTokenize + }; + rc = pApi->xCreateTokenizer_v2(pApi, + "porter", + (void*)pApi, + &sPorter, + 0 + ); + } return rc; } diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 3e97264fa8..2133d5d5b8 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -364,6 +364,9 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ default: return 1; } break; + + default: + return 1; } return 0; } @@ -775,4 +778,3 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ } aAscii[0] = 0; /* 0x00 is never a token character */ } - diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 18774c4e4a..3a6a968f7c 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -64,6 +64,7 @@ struct Fts5VocabCursor { int nLeTerm; /* Size of zLeTerm in bytes */ char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + int colUsed; /* Copy of sqlite3_index_info.colUsed */ /* These are used by 'col' tables only */ int iCol; @@ -90,9 +91,11 @@ struct Fts5VocabCursor { /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. */ -#define FTS5_VOCAB_TERM_EQ 0x01 -#define FTS5_VOCAB_TERM_GE 0x02 -#define FTS5_VOCAB_TERM_LE 0x04 +#define FTS5_VOCAB_TERM_EQ 0x0100 +#define FTS5_VOCAB_TERM_GE 0x0200 +#define FTS5_VOCAB_TERM_LE 0x0400 + +#define FTS5_VOCAB_COLUSED_MASK 0xFF /* @@ -190,12 +193,12 @@ static int fts5VocabInitVtab( *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); rc = SQLITE_ERROR; }else{ - int nByte; /* Bytes of space to allocate */ + i64 nByte; /* Bytes of space to allocate */ const char *zDb = bDb ? argv[3] : argv[1]; const char *zTab = bDb ? argv[4] : argv[3]; const char *zType = bDb ? argv[5] : argv[4]; - int nDb = (int)strlen(zDb)+1; - int nTab = (int)strlen(zTab)+1; + i64 nDb = strlen(zDb)+1; + i64 nTab = strlen(zTab)+1; int eType = 0; rc = fts5VocabTableType(zType, pzErr, &eType); @@ -269,11 +272,13 @@ static int fts5VocabBestIndexMethod( int iTermEq = -1; int iTermGe = -1; int iTermLe = -1; - int idxNum = 0; + int idxNum = (int)pInfo->colUsed; int nArg = 0; UNUSED_PARAM(pUnused); + assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed ); + for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; if( p->usable==0 ) continue; @@ -365,7 +370,7 @@ static int fts5VocabOpenMethod( if( rc==SQLITE_OK ){ pVTab->zErrMsg = sqlite3_mprintf( "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl - ); + ); rc = SQLITE_ERROR; } }else{ @@ -391,7 +396,12 @@ static int fts5VocabOpenMethod( return rc; } +/* +** Restore cursor pCsr to the state it was in immediately after being +** created by the xOpen() method. +*/ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ + int nCol = pCsr->pFts5->pConfig->nCol; pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); sqlite3Fts5StructureRelease(pCsr->pStruct); @@ -401,6 +411,12 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->nLeTerm = -1; pCsr->zLeTerm = 0; pCsr->bEof = 0; + pCsr->iCol = 0; + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + pCsr->colUsed = 0; + memset(pCsr->aCnt, 0, sizeof(i64)*nCol); + memset(pCsr->aDoc, 0, sizeof(i64)*nCol); } /* @@ -525,9 +541,19 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ switch( pTab->eType ){ case FTS5_VOCAB_ROW: - if( eDetail==FTS5_DETAIL_FULL ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; + /* Do not bother counting the number of instances if the "cnt" + ** column is not being read (according to colUsed). */ + if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){ + while( iPosaCnt[] */ + pCsr->aCnt[0]++; + } } } pCsr->aDoc[0]++; @@ -625,11 +651,12 @@ static int fts5VocabFilterMethod( if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK); if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); nTerm = sqlite3_value_bytes(pEq); - f = 0; + f = FTS5INDEX_QUERY_NOTOKENDATA; }else{ if( pGe ){ zTerm = (const char *)sqlite3_value_text(pGe); @@ -783,7 +810,8 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, - /* xShadowName */ 0 + /* xShadowName */ 0, + /* xIntegrity */ 0 }; void *p = (void*)pGlobal; diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 9c012932da..8ea87dbdd1 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -51,6 +51,10 @@ proc fts5_test_poslist2 {cmd} { sort_poslist $res } +proc fts5_test_insttoken {cmd iInst iToken} { + $cmd xInstToken $iInst $iToken +} + proc fts5_test_collist {cmd} { set res [list] @@ -61,6 +65,12 @@ proc fts5_test_collist {cmd} { set res } +proc fts5_collist {cmd iPhrase} { + set res [list] + $cmd xPhraseColumnForeach $iPhrase c { lappend res $c } + set res +} + proc fts5_test_columnsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -69,6 +79,13 @@ proc fts5_test_columnsize {cmd} { set res } +proc fts5_columntext {cmd iCol} { + $cmd xColumnText $iCol +} +proc fts5_columnlocale {cmd iCol} { + $cmd xColumnLocale $iCol +} + proc fts5_test_columntext {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -77,6 +94,14 @@ proc fts5_test_columntext {cmd} { set res } +proc fts5_test_columnlocale {cmd} { + set res [list] + for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { + lappend res [$cmd xColumnLocale $i] + } + set res +} + proc fts5_test_columntotalsize {cmd} { set res [list] for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { @@ -104,6 +129,10 @@ proc fts5_test_rowcount {cmd} { $cmd xRowCount } +proc fts5_test_rowid {cmd} { + $cmd xRowid +} + proc test_queryphrase_cb {cnt cmd} { upvar $cnt L for {set i 0} {$i < [$cmd xInstCount]} {incr i} { @@ -125,6 +154,13 @@ proc fts5_test_queryphrase {cmd} { set res } +proc fts5_queryphrase {cmd iPhrase} { + set cnt [list] + for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 } + $cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt] + set cnt +} + proc fts5_test_phrasecount {cmd} { $cmd xPhraseCount } @@ -144,16 +180,23 @@ proc fts5_aux_test_functions {db} { foreach f { fts5_test_columnsize fts5_test_columntext + fts5_test_columnlocale fts5_test_columntotalsize fts5_test_poslist fts5_test_poslist2 fts5_test_collist + fts5_test_insttoken fts5_test_tokenize fts5_test_rowcount + fts5_test_rowid fts5_test_all fts5_test_queryphrase fts5_test_phrasecount + fts5_columntext + fts5_columnlocale + fts5_queryphrase + fts5_collist } { sqlite3_fts5_create_function $db $f $f } @@ -438,6 +481,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } +proc foreach_tokenizer_mode {prefix script} { + set saved $::testprefix + foreach {d mapping} { + "" {} + "-origintext" {, tokenize="origintext unicode61", tokendata=1} + } { + set s [string map [list %TOKENIZER% $mapping] $script] + set ::testprefix "$prefix$d" + reset_db + sqlite3_fts5_register_origintext db + uplevel $s + } + set ::testprefix $saved +} #------------------------------------------------------------------------- # Convert a poslist of the type returned by fts5_test_poslist() to a diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 59ce4f6a1f..184cb77b84 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -22,6 +22,7 @@ ifcapable !fts5 { } foreach_detail_mode $::testprefix { +foreach_tokenizer_mode $::testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); @@ -44,7 +45,7 @@ do_execsql_test 1.1 { # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); @@ -65,14 +66,17 @@ foreach w {a b c d e f} { do_execsql_test 2.4 { INSERT INTO t1(t1) VALUES('integrity-check'); -} + PRAGMA integrity_check; + PRAGMA integrity_check(t1); +} {ok ok} #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); } foreach {i x y} { 1 {g f d b f} {h h e i a} @@ -88,14 +92,16 @@ foreach {i x y} { } { do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) } do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') } + do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -118,8 +124,9 @@ foreach {i x y} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 5.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -135,15 +142,16 @@ foreach {i x y} { 10 {ddd abcde dddd dd c} {dddd c c d abcde} } { do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) } - do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') } + do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok if {[set_test_counter errors]} break } #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -178,6 +186,7 @@ do_execsql_test 6.6 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db expr srand(0) do_execsql_test 7.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y,z); @@ -219,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 8.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); @@ -233,6 +243,7 @@ do_execsql_test 8.1 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db expr srand(0) @@ -277,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 10.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); } set d10 { 1 {g f d b f} {h h e i a} @@ -311,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%); } {1 {reserved fts5 column name: rank}} do_catchsql_test 11.2 { - CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%); } {1 {reserved fts5 table name: rank}} do_catchsql_test 11.3 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%); } {1 {reserved fts5 column name: rowid}} #------------------------------------------------------------------------- # do_execsql_test 12.1 { - CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); } {} do_catchsql_test 12.2 { @@ -338,8 +350,9 @@ do_test 12.3 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 13.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); } {} @@ -362,8 +375,9 @@ do_execsql_test 13.6 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 14.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz xyz xyz xyz xyz xyz' @@ -414,7 +428,7 @@ do_execsql_test 15.1 { } do_catchsql_test 15.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: checksum mismatch for table "t1"}} #------------------------------------------------------------------------- # @@ -426,7 +440,7 @@ do_execsql_test 16.1 { proc funk {} { db eval { UPDATE n1_config SET v=50 WHERE k='version' } set fd [db incrblob main n1_data block 10] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary # puts -nonewline $fd "\x44\x45" close $fd } @@ -439,15 +453,16 @@ db func funk funk # statement no longer fails. # do_catchsql_test 16.2 { - SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' + SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' } {0 {{} -1e-06 {}}} # {1 {SQL logic error}} #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO b2 VALUES('a'); INSERT INTO b2 VALUES('b'); INSERT INTO b2 VALUES('c'); @@ -463,8 +478,9 @@ do_test 17.2 { if {[string match n* %DETAIL%]==0} { reset_db + sqlite3_fts5_register_origintext db do_execsql_test 17.3 { - CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); INSERT INTO c2 VALUES('x x x', 'x x x'); SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; } {1} @@ -473,8 +489,9 @@ if {[string match n* %DETAIL%]==0} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%); INSERT INTO uio VALUES(NULL); INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; @@ -521,8 +538,8 @@ do_execsql_test 17.9 { #-------------------------------------------------------------------- # do_execsql_test 18.1 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); - CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%); + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1 VALUES('abc*', NULL); INSERT INTO t2 VALUES(1, 'abcdefg'); } @@ -537,8 +554,9 @@ do_execsql_test 18.3 { # fts5 table in the temp schema. # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 19.0 { - CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; @@ -548,8 +566,9 @@ do_execsql_test 19.0 { # Test that 6 and 7 byte varints can be read. # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 20.0 { - CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%); } set ::ids [list \ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] @@ -567,7 +586,7 @@ do_test 20.1 { # do_execsql_test 21.0 { CREATE TEMP TABLE t8(a, b); - CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%); } do_execsql_test 21.1 { @@ -578,7 +597,7 @@ do_execsql_test 21.1 { } do_execsql_test 22.0 { - CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t9(rowid, x) VALUES(2, 'bbb'); BEGIN; INSERT INTO t9(rowid, x) VALUES(1, 'aaa'); @@ -593,7 +612,7 @@ do_execsql_test 22.1 { #------------------------------------------------------------------------- do_execsql_test 23.0 { - CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%); CREATE TABLE t11(x); } do_execsql_test 23.1 { @@ -605,7 +624,7 @@ do_execsql_test 23.2 { #------------------------------------------------------------------------- do_execsql_test 24.0 { - CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t12 VALUES('aaaa'); } do_execsql_test 24.1 { @@ -615,6 +634,9 @@ do_execsql_test 24.1 { INSERT INTO t12 VALUES('aaaa'); END; } +execsql_pp { + SELECT rowid, hex(block) FROM t12_data +} do_execsql_test 24.2 { INSERT INTO t12(t12) VALUES('integrity-check'); } @@ -624,7 +646,7 @@ do_execsql_test 24.3 { #------------------------------------------------------------------------- do_execsql_test 25.0 { - CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%); } do_execsql_test 25.1 { BEGIN; @@ -635,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB } +} } expand_all_sql db diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 5aa7456586..a74c0f8884 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ab -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -294,6 +294,39 @@ do_execsql_test 7.0 { INSERT INTO x1 VALUES($doc); } +#------------------------------------------------------------------------- +# Forum post: https://sqlite.org/forum/forumpost/ea4d8c9acb +# +reset_db +do_execsql_test 8.0 { + PRAGMA encoding = 'UTF-16le'; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} +set v [db one {SELECT x'2a12'}] +do_execsql_test 8.1 { + INSERT INTO vt0 VALUES ($v); +} +do_execsql_test 8.2 { + SELECT quote(c0) FROM vt0 +} {X'2A12'} +do_execsql_test 8.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} {} +reset_db +do_execsql_test 8.4 { + PRAGMA encoding = 'UTF-16le'; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} +do_execsql_test 8.5 { + INSERT INTO vt0 VALUES (x'2a12'); +} +do_execsql_test 8.6 { + SELECT quote(c0) FROM vt0 +} {X'2A12'} +do_execsql_test 8.7 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} {} + } ;# foreach_detail_mode... diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index f3a914653f..4628e909c1 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ac -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index 524da6deae..27806a4c0c 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -18,7 +18,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ad -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ae.test b/ext/fts5/test/fts5ae.test index d9f132ca97..205a59a69f 100644 --- a/ext/fts5/test/fts5ae.test +++ b/ext/fts5/test/fts5ae.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ae -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index 3d79295092..9c95ef2daa 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -18,7 +18,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5af -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ag.test b/ext/fts5/test/fts5ag.test index 9ead957c9d..42cd913784 100644 --- a/ext/fts5/test/fts5ag.test +++ b/ext/fts5/test/fts5ag.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ag -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 0004351375..bf9c9e9dbc 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ah -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -163,6 +163,17 @@ do_execsql_test 1.8.2 { SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; } {10000} +do_execsql_test 1.8.3 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text'; +} {4999} +do_execsql_test 1.8.4 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text'; +} {0} + +do_catchsql_test 1.9 { + SELECT * FROM t1('*xy'); +} {1 {unknown special query: xy}} + } ;# foreach_detail_mode #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5ai.test b/ext/fts5/test/fts5ai.test index 20e1069398..a6576d3afc 100644 --- a/ext/fts5/test/fts5ai.test +++ b/ext/fts5/test/fts5ai.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ai -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5aj.test b/ext/fts5/test/fts5aj.test index 50dae20162..e802306b38 100644 --- a/ext/fts5/test/fts5aj.test +++ b/ext/fts5/test/fts5aj.test @@ -19,7 +19,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5aj -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test index e248f2a328..253f14fc79 100644 --- a/ext/fts5/test/fts5ak.test +++ b/ext/fts5/test/fts5ak.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ak -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index 842d991a37..7187ad67c7 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5al -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -293,6 +293,16 @@ do_catchsql_test 4.4.4 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL } {1 {parse error in rank function: }} +# Check that the second and subsequent rank= constraints are ignored. +# +do_catchsql_test 4.3.3 { + SELECT *, rank FROM t3 + WHERE t3 MATCH 'a' AND + rank MATCH 'nosuch()' AND + rank MATCH 'rowidmod(3)' + ORDER BY rank ASC +} {1 {unable to use function MATCH in the requested context}} + } ;# foreach_detail_mode diff --git a/ext/fts5/test/fts5alter.test b/ext/fts5/test/fts5alter.test index 67f948cbbe..bb5f78dc86 100644 --- a/ext/fts5/test/fts5alter.test +++ b/ext/fts5/test/fts5alter.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5alter -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5auto.test b/ext/fts5/test/fts5auto.test index 79d432b812..b771af912e 100644 --- a/ext/fts5/test/fts5auto.test +++ b/ext/fts5/test/fts5auto.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5auto -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test index 561067c4bc..960dbc5117 100644 --- a/ext/fts5/test/fts5aux.test +++ b/ext/fts5/test/fts5aux.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5aux -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -307,5 +307,98 @@ do_catchsql_test 10.1.4 { SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid } {1 {unable to use function firstcol in the requested context}} -finish_test +#------------------------------------------------------------------------- +# Test that xInstCount() works from within an xPhraseQuery() callback. +# +reset_db + +proc xCallback {cmd} { + incr ::hitcount [$cmd xInstCount] + return SQLITE_OK +} +proc fts5_hitcount {cmd} { + set ::hitcount 0 + $cmd xQueryPhrase 0 xCallback + return $::hitcount +} +sqlite3_fts5_create_function db fts5_hitcount fts5_hitcount + +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE x1 USING fts5(z); + INSERT INTO x1 VALUES('one two three'); + INSERT INTO x1 VALUES('one two one three one'); + INSERT INTO x1 VALUES('one two three'); +} + +do_execsql_test 11.2 { + SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1; +} {5} + +#------------------------------------------------------------------------- +# Test that xColumnText returns SQLITE_RANGE when it should. +# +reset_db +fts5_aux_test_functions db +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + INSERT INTO t1 VALUES('one', 'two', 'three'); + INSERT INTO t1 VALUES('one', 'one', 'one'); + INSERT INTO t1 VALUES('two', 'two', 'two'); + INSERT INTO t1 VALUES('three', 'three', 'three'); +} +do_catchsql_test 12.1.1 { + SELECT fts5_columntext(t1, -1) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 3) FROM t1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.1.2 { + SELECT fts5_columntext(t1, 1) FROM t1('one AND two'); +} {0 two} + +do_catchsql_test 12.2.1 { + SELECT fts5_queryphrase(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.2 { + SELECT fts5_queryphrase(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.2.3 { + SELECT fts5_queryphrase(t1, 1) FROM t1('one AND two'); +} {0 {{1 2 1}}} + +do_catchsql_test 12.3.1 { + SELECT fts5_collist(t1, -1) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.2 { + SELECT fts5_collist(t1, 2) FROM t1('one AND two'); +} {1 SQLITE_RANGE} +do_catchsql_test 12.3.3 { + SELECT fts5_collist(t1, 1) FROM t1('one AND two'); +} {0 1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +do_catchsql_test 13.2 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {0 {{a b c} {d e f}}} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +do_catchsql_test 13.4 { + SELECT highlight(t1, 0, '[', ']') FROM t1 +} {1 {SQL logic error}} + +finish_test diff --git a/ext/fts5/test/fts5aux2.test b/ext/fts5/test/fts5aux2.test new file mode 100644 index 0000000000..2352970ec7 --- /dev/null +++ b/ext/fts5/test/fts5aux2.test @@ -0,0 +1,71 @@ +# 2024 June 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the auxiliary function APIs. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aux + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('a b', 'c d'); + INSERT INTO x1 VALUES('d e', 'a b'); + INSERT INTO x1 VALUES('a b', 'e f'); + INSERT INTO x1 VALUES('d e', 'c d'); +} + +fts5_aux_test_functions db +do_execsql_test 1.1 { + SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2 +} [list [list {*}{ + columnsize {2 2} + columntext {{d e} {a b}} + columntotalsize {8 8} + poslist {} + tokenize {{d e} {a b}} + rowcount 4 +}]] + +do_execsql_test 1.2 { + SELECT fts5_test_columntext(x1) FROM x1 +} { + {{a b} {c d}} + {{d e} {a b}} + {{a b} {e f}} + {{d e} {c d}} +} + +do_execsql_test 1.3 { + SELECT fts5_test_rowid(x1) FROM x1 +} { + 1 2 3 4 +} +do_execsql_test 1.4 { + SELECT fts5_test_phrasecount(x1) FROM x1 +} { + 0 0 0 0 +} +do_catchsql_test 1.5 { + SELECT fts5_queryphrase(x1, 0) FROM x1 +} {1 SQLITE_RANGE} +do_execsql_test 1.6 { + SELECT fts5_test_rowcount(x1) FROM x1 +} {4 4 4 4} + + +finish_test diff --git a/ext/fts5/test/fts5auxdata.test b/ext/fts5/test/fts5auxdata.test index a2a41704c5..7f99fed316 100644 --- a/ext/fts5/test/fts5auxdata.test +++ b/ext/fts5/test/fts5auxdata.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5auxdata -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5bigid.test b/ext/fts5/test/fts5bigid.test new file mode 100644 index 0000000000..ae20ec641e --- /dev/null +++ b/ext/fts5/test/fts5bigid.test @@ -0,0 +1,62 @@ +# 2023 May 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5bigid + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set nRow 20000 + +proc do_ascdesc_test {tn query} { + set ::lAsc [db eval { SELECT rowid FROM x1($query) }] + set ::lDesc [db eval { SELECT rowid FROM x1($query) ORDER BY rowid DESC }] + do_test $tn.1 { lsort -integer $::lAsc } $::lAsc + do_test $tn.2 { lsort -integer -decr $::lDesc } $::lDesc + do_test $tn.3 { lsort -integer $::lDesc } $::lAsc +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); +} + +do_test 1.1 { + for {set ii 0} {$ii < $nRow} {incr ii} { + db eval { + REPLACE INTO x1(rowid, a) VALUES(random(), 'movement at the station'); + } + } +} {} + +do_ascdesc_test 1.2 "the" + +do_execsql_test 1.3 { + DELETE FROM x1 +} + +do_test 1.4 { + for {set ii 0} {$ii < $nRow} {incr ii} { + db eval { + INSERT INTO x1(rowid, a) VALUES( + $ii + 0x6FFFFFFFFFFFFFFF, 'movement at the station' + ); + } + } +} {} + +do_ascdesc_test 1.5 "movement" + +finish_test diff --git a/ext/fts5/test/fts5bigpl.test b/ext/fts5/test/fts5bigpl.test index 2c9df11b1f..9e3d86c0e6 100644 --- a/ext/fts5/test/fts5bigpl.test +++ b/ext/fts5/test/fts5bigpl.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5bigpl -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test new file mode 100644 index 0000000000..9348554104 --- /dev/null +++ b/ext/fts5/test/fts5blob.test @@ -0,0 +1,166 @@ +# 2024 July 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file verifies that: +# +# * blob values may be written to locale=0 tables. +# +# * blob values - other than fts5_locale() values - may not be written +# to locale=0 tables. This is an SQLITE_MISMATCH error +# +# * blob values may be returned by queries on the external-content table +# of a locale=0 table. +# +# * blob values not may be returned by queries on the external-content +# table of a locale=1 table, apart from fts5_locale() blobs. This is an +# SQLITE_MISMATCH error. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5blob + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Test that blobs may be stored in normal locale=0 tables. +# +foreach {tn enc} { + 1 utf8 + 2 utf16 +} { + reset_db + fts5_aux_test_functions db + + execsql "PRAGMA encoding = $enc" + + execsql " + CREATE VIRTUAL TABLE t1 USING fts5(x, y); + " + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546'); + INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546'); + INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz'); + } + + do_execsql_test 1.$tn.1 { + SELECT rowid, quote(x), quote(y) FROM t1 + } { + 1 555 X'0000000041424320444546' + 2 666 X'41424300444546' + 3 777 'xyz' + } + + do_execsql_test 1.$tn.2 { + DELETE FROM t1 WHERE rowid=2; + DELETE FROM t1 WHERE rowid=1; + } + + do_execsql_test 1.$tn.3 { + PRAGMA integrity_check; + } {ok} +} + +#-------------------------------------------------------------------------- +# Test that a blob may be stored and retrieved in an unindexed column of +# a regular table with locale=1. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1); + INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546'); +} + +do_execsql_test 2.1 { + select rowid, x, quote(y) FROM t1 +} { + 12 twelve X'0000000041424320444546' +} + +#-------------------------------------------------------------------------- +# Test that blobs may not be written to any type of table with locale=1 +# set. Except, they may be written to UNINDEXED columns. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {0 {}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {0 {}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456'); +} {0 {}} + + +#-------------------------------------------------------------------------- +# Test that fts5_locale() values may not be written to any type of table +# without locale=1 set. Even to an UNINDEXED column. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b); + + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2); + CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=); + + CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED); +} + +do_catchsql_test 3.1 { + INSERT INTO x1(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.2 { + INSERT INTO x2(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.3 { + INSERT INTO x3(rowid, a, b) + VALUES(113, 'hello world', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} +do_catchsql_test 3.4 { + INSERT INTO x4(rowid, a, b, c) + VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc')); +} {1 {fts5_locale() requires locale=1}} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} + +foreach {tn sql} { + 1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') } + 2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') } + 3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') } +} { + do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}} +} + + +finish_test + + diff --git a/ext/fts5/test/fts5cat.test b/ext/fts5/test/fts5cat.test index 483f64bfef..71e2abe3ae 100644 --- a/ext/fts5/test/fts5cat.test +++ b/ext/fts5/test/fts5cat.test @@ -55,5 +55,22 @@ do_execsql_test 1.5 { SELECT * FROM t4t } {สนามกีฬา 1 1} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 " + CREATE VIRTUAL TABLE x1 USING fts5(c, + tokenize=\"unicode61 categories ' \t'\"); +" + +do_catchsql_test 2.1 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tMYZ'\"); +" {1 {error in tokenizer constructor}} + +do_catchsql_test 2.2 " + CREATE VIRTUAL TABLE x2 USING fts5(c, + tokenize=\"unicode61 categories 'N*\t\tXYZ'\"); +" {1 {error in tokenizer constructor}} + finish_test diff --git a/ext/fts5/test/fts5circref.test b/ext/fts5/test/fts5circref.test index ea992195af..8732fa17dd 100644 --- a/ext/fts5/test/fts5circref.test +++ b/ext/fts5/test/fts5circref.test @@ -72,7 +72,7 @@ foreach {tn schema sql} { } { db_restore_and_reopen do_execsql_test 1.1.$tn.1 $schema - do_catchsql_test 1.1.$tn.2 $sql {1 {SQL logic error}} + do_catchsql_test 1.1.$tn.2 $sql {1 {database disk image is malformed}} db close } diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test index 7243743b51..e5429572c5 100644 --- a/ext/fts5/test/fts5colset.test +++ b/ext/fts5/test/fts5colset.test @@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 4.1 { SELECT * FROM t1 WHERE rowid MATCH 'a' - } {1 {unable to use function MATCH in the requested context}} + } {1 {no query solution}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test index 2b03d575aa..7af49184b8 100644 --- a/ext/fts5/test/fts5columnsize.test +++ b/ext/fts5/test/fts5columnsize.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5columnsize -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5config.test b/ext/fts5/test/fts5config.test index 35894c6bb0..28f3146ea3 100644 --- a/ext/fts5/test/fts5config.test +++ b/ext/fts5/test/fts5config.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5config -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5conflict.test b/ext/fts5/test/fts5conflict.test index 644db53a1e..b5bf0a1160 100644 --- a/ext/fts5/test/fts5conflict.test +++ b/ext/fts5/test/fts5conflict.test @@ -65,4 +65,44 @@ do_execsql_test 2.1 { INSERT INTO fts_idx(fts_idx) VALUES('integrity-check'); } +#------------------------------------------------------------------------- +# Tests for OR IGNORE conflict handling. +# +reset_db +foreach_detail_mode $::testprefix { + + do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(xyz, detail=%DETAIL%); + + BEGIN; + INSERT INTO t1(rowid, xyz) VALUES(13, 'thirteen documents'); + INSERT INTO t1(rowid, xyz) VALUES(14, 'fourteen documents'); + INSERT INTO t1(rowid, xyz) VALUES(15, 'fifteen documents'); + COMMIT; + } + + set db_cksum [cksum] + foreach {tn sql} { + 1 { + INSERT OR IGNORE INTO t1(rowid, xyz) VALUES(14, 'new text'); + } + 2 { + UPDATE OR IGNORE t1 SET rowid=13 WHERE rowid=15; + } + 3 { + INSERT OR IGNORE INTO t1(rowid, xyz) + SELECT 13, 'some text' + UNION ALL + SELECT 14, 'some text' + UNION ALL + SELECT 15, 'some text' + } + } { + do_execsql_test 3.1.$tn.1 $sql + do_test 3.1.$tn.2 { cksum } $db_cksum + } + +} + + finish_test diff --git a/ext/fts5/test/fts5content.test b/ext/fts5/test/fts5content.test index 74a74e2ad0..05b5cc6113 100644 --- a/ext/fts5/test/fts5content.test +++ b/ext/fts5/test/fts5content.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5content -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -293,5 +293,76 @@ do_catchsql_test 7.2.5 { SELECT * FROM t1('abc') ORDER BY rank; } {1 {recursively defined fts5 content table}} +#--------------------------------------------------------------------------- +# Check that if the content table is a view, and that view contains an +# error, a reasonable error message is returned if the user tries to +# read from the view via the fts5 table. +# +reset_db +do_execsql_test 8.1 { + CREATE VIEW a1 AS + SELECT 1 AS r, text_value(1) AS t + UNION ALL + SELECT 2 AS r, text_value(2) AS t; + + CREATE VIRTUAL TABLE t1 USING fts5(t, content='a1', content_rowid='r'); +} + +foreach {tn sql} { + 1 "SELECT * FROM t1" + 2 "INSERT INTO t1(t1) VALUES('rebuild')" + 3 "SELECT * FROM t1 WHERE rowid=1" +} { + do_catchsql_test 8.2.$tn $sql {1 {no such function: text_value}} +} + +proc text_value {i} { + if {$i==1} { return "one" } + if {$i==2} { return "two" } + return "many" +} +db func text_value text_value + +do_execsql_test 8.3.1 { SELECT * FROM t1 } {one two} +do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') } +do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one} +do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one two three'); + INSERT INTO t1 VALUES(2, 'one two three'); + + CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a); + INSERT INTO ft(ft) VALUES('rebuild'); +} + +do_execsql_test 9.2 { + SELECT rowid, b FROM ft('two'); +} { + 1 {one two three} + 2 {one two three} +} + +do_execsql_test 9.3 { + DELETE FROM t1 WHERE a=2; +} + +do_catchsql_test 9.4 { + SELECT rowid FROM ft('two'); +} {0 {1 2}} + +do_catchsql_test 9.5 { + SELECT * FROM ft('two'); +} {1 {fts5: missing row 2 from content table 'main'.'t1'}} + +fts5_aux_test_functions db + +do_catchsql_test 9.6 { + SELECT rowid, fts5_columntext(ft, 0) FROM ft('two'); +} {1 SQLITE_CORRUPT_VTAB} + finish_test diff --git a/ext/fts5/test/fts5contentless.test b/ext/fts5/test/fts5contentless.test new file mode 100644 index 0000000000..991e9888fc --- /dev/null +++ b/ext/fts5/test/fts5contentless.test @@ -0,0 +1,290 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Check that it is not possible to specify "contentless_delete=1" for +# anything other than a contentless table. +# +set res(0) {0 {}} +set res(1) {1 {contentless_delete=1 requires a contentless table}} +foreach {tn sql bError} { + 1 "(a, b, contentless_delete=1)" 1 + 2 "(a, b, contentless_delete=1, content=abc)" 1 + 3 "(a, b, contentless_delete=1, content=)" 0 + 4 "(content=, contentless_delete=1, a)" 0 + 5 "(content='', contentless_delete=1, hello)" 0 +} { + execsql { BEGIN } + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError) + execsql { ROLLBACK } +} + +# Check that it is not possible to specify "contentless_delete=1" +# along with columnsize=1. +# +set res(0) {0 {}} +set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}} +foreach {tn sql bError} { + 2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 +} { + execsql { BEGIN } + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError) + execsql { ROLLBACK } +} + +# Check that if contentless_delete=1 is specified, then the "origin" +# column is added to the %_docsize table. +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(c, content=''); + CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1); +} +do_execsql_test 3.1 { + SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize'); +} { + {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} + {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)} +} + +do_execsql_test 3.2.1 { + SELECT hex(block) FROM x1_data WHERE id=10 +} {00000000000000} +do_execsql_test 3.2.2 { + SELECT hex(block) FROM x2_data WHERE id=10 +} {00000000FF000001000000} + +do_execsql_test 3.3 { + INSERT INTO x2 VALUES('first text'); + INSERT INTO x2 VALUES('second text'); +} +do_execsql_test 3.4 { + SELECT id, origin FROM x2_docsize +} {1 1 2 2} +do_execsql_test 3.5 { + SELECT level, segment, loc1, loc2 FROM fts5_structure( + (SELECT block FROM x2_data WHERE id=10) + ) +} { + 0 0 1 1 + 0 1 2 2 +} +do_execsql_test 3.6 { + INSERT INTO x2(x2) VALUES('optimize'); +} +do_execsql_test 3.7 { + SELECT level, segment, loc1, loc2 FROM fts5_structure( + (SELECT block FROM x2_data WHERE id=10) + ) +} { + 1 0 1 2 +} + +do_execsql_test 3.8 { + DELETE FROM x2 WHERE rowid=2; +} + +do_execsql_test 3.9 { + SELECT rowid FROM x2('text') +} {1} + +#-------------------------------------------------------------------------- +reset_db +proc document {n} { + set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + set ret [list] + for {set ii 0} {$ii < $n} {incr ii} { + lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set ret +} + +set nRow 1000 + +do_execsql_test 4.0 { + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 100); +} +do_test 4.1 { + for {set ii 0} {$ii < $nRow} {incr ii} { + set doc [document 6] + execsql { + INSERT INTO t1 VALUES($doc); + INSERT INTO ft VALUES($doc); + } + } +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.2.$v { set L1 } $L2 +} + +do_test 4.3 { + for {set ii 1} {$ii < $nRow} {incr ii 2} { + execsql { + DELETE FROM ft WHERE rowid=$ii; + DELETE FROM t1 WHERE rowid=$ii; + } + } +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.4.$v { set L1 } $L2 +} + +do_execsql_test 4.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} {} + +foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} { + set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}] + set L2 [execsql {SELECT rowid FROM ft($v)}] + do_test 4.6.$v { set L1 } $L2 +} + +#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data } + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(rowid, x) VALUES(1, 'one two three'); + INSERT INTO ft(rowid, x) VALUES(2, 'one two four'); + INSERT INTO ft(rowid, x) VALUES(3, 'one two five'); + INSERT INTO ft(rowid, x) VALUES(4, 'one two seven'); + INSERT INTO ft(rowid, x) VALUES(5, 'one two eight'); +} + +do_execsql_test 5.1 { + DELETE FROM ft WHERE rowid=2 +} + +do_execsql_test 5.2 { + SELECT rowid FROM ft +} {1 3 4 5} + +do_catchsql_test 5.3 { + UPDATE ft SET x='four six' WHERE rowid=3 +} {0 {}} + +do_execsql_test 5.4 { + SELECT rowid FROM ft('one'); +} {1 4 5} + +do_execsql_test 5.5 { + REPLACE INTO ft(rowid, x) VALUES(3, 'four six'); + SELECT rowid FROM ft('one'); +} {1 4 5} + +do_execsql_test 5.6 { + REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven'); + SELECT rowid FROM ft('one'); +} {1 4 5 6} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(rowid, x) VALUES(1, 'one two three'); + INSERT INTO ft(rowid, x) VALUES(2, 'one two four'); +} + +do_test 6.1 { + db eval { SELECT rowid FROM ft('one two') } { + if {$rowid==1} { + db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') } + } + } +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); +} + +set lRowid [list -450 0 1 2 42] + +do_test 7.1 { + execsql BEGIN + foreach r $lRowid { + execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); } + } + execsql COMMIT +} {} + +do_test 7.2 { + execsql BEGIN + foreach r $lRowid { + execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); } + } + execsql COMMIT +} {} + +do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {} +do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 8.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {1 {'delete' may not be used with a contentless_delete=1 table}} + +do_execsql_test 8.2 { + BEGIN; + INSERT INTO ft(rowid, x) VALUES(3, 'four four four'); + DELETE FROM ft WHERE rowid=3; + COMMIT; + SELECT rowid FROM ft('four'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0); + INSERT INTO ft VALUES('hello world'); + INSERT INTO ft VALUES('one two three'); +} + +do_catchsql_test 9.1 { + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world'); +} {0 {}} + +do_catchsql_test 9.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2); +} {1 {malformed contentless_delete=... directive}} + +do_catchsql_test 9.3 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11); +} {1 {malformed contentless_delete=... directive}} + +finish_test diff --git a/ext/fts5/test/fts5contentless2.test b/ext/fts5/test/fts5contentless2.test new file mode 100644 index 0000000000..248534bce4 --- /dev/null +++ b/ext/fts5/test/fts5contentless2.test @@ -0,0 +1,207 @@ +# 2023 July 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc vocab {} { + list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp +} + +proc document {nToken} { + set doc [list] + set vocab [vocab] + for {set ii 0} {$ii < $nToken} {incr ii} { + lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set doc +} +db func document document + +proc contains {doc token} { + expr {[lsearch $doc $token]>=0} +} +db func contains contains + +proc do_compare_tables_test {tn} { + uplevel [list do_test $tn { + foreach v [vocab] { + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }] + set l2 [execsql { SELECT rowid FROM t2($v) }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 1]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set w "[string range $v 0 0]*" + set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }] + set l2 [execsql { SELECT rowid FROM t2($w) }] + if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" } + + set l1 [execsql { + SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC + }] + set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }] + if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" } + } + set {} {} + } {}] +} + +proc lshuffle {in} { + set L [list] + set ret [list] + foreach elem $in { lappend L [list [expr rand()] $elem] } + foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] } + set ret +} + +expr srand(0) + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5( + doc, prefix=2, content=, contentless_delete=1 + ); + + CREATE TABLE t1(doc); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE rowid = old.rowid; + END; +} + +set SMALLEST64 -9223372036854775808 +set LARGEST64 9223372036854775807 + +foreach {tn r1 r2} { + 1 0 50 + 2 $SMALLEST64 $SMALLEST64+50 + 3 $LARGEST64-50 $LARGEST64 + 4 -50 -1 +} { + set r1 [expr $r1] + set r2 [expr $r2] + + do_test 1.1.$tn { + execsql BEGIN + for {set ii $r1} {$ii <= $r2} {incr ii} { + execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); } + } + execsql COMMIT + } {} +} +do_test 1.2 { + db eval { SELECT rowid, doc FROM t1 } { + execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) } + } +} {} + +foreach {tn rowid} { + 1 $SMALLEST64 + 2 0 + 3 -5 + 4 -30 + 5 $LARGEST64 + 6 $LARGEST64-1 +} { + set rowid [expr $rowid] + do_execsql_test 1.3.$tn.1 { + DELETE FROM t1 WHERE rowid=$rowid + } + do_compare_tables_test 1.3.$tn.2 +} + +set iTest 1 +foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] { + if {($iTest % 50)==0} { + execsql { INSERT INTO t2(t2) VALUES('optimize') } + } + if {($iTest % 5)==0} { + execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) } + } + do_execsql_test 1.4.$iTest.1($r) { + DELETE FROM t1 WHERE rowid=$r + } + do_compare_tables_test 1.4.$iTest.2 + incr iTest +} + +do_execsql_test 1.5 { + SELECT * FROM t1 +} {} + +#------------------------------------------------------------------------- +reset_db +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s; +} + +do_execsql_test 2.1 { + BEGIN; + DELETE FROM t2 WHERE rowid=32; + DELETE FROM t2 WHERE rowid=64; + DELETE FROM t2 WHERE rowid=96; + DELETE FROM t2 WHERE rowid=128; + DELETE FROM t2 WHERE rowid=160; + DELETE FROM t2 WHERE rowid=192; + COMMIT; +} + +do_execsql_test 2.2 { + SELECT * FROM t2('128'); +} {} + +#------------------------------------------------------------------------- + +foreach {tn step} { + 1 3 + 2 7 + 3 15 +} { + set step [expr $step] + + reset_db + db func document document + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1); + INSERT INTO t2(t2, rank) VALUES('pgsz', 100); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s; + } + do_execsql_test 3.$tn.1 { + DELETE FROM t2 WHERE (rowid % $step)==0 + } + do_execsql_test 3.$tn.2 { + SELECT * FROM t2( $step * 5 ) + } {} +} + + + +finish_test diff --git a/ext/fts5/test/fts5contentless3.test b/ext/fts5/test/fts5contentless3.test new file mode 100644 index 0000000000..693840da82 --- /dev/null +++ b/ext/fts5/test/fts5contentless3.test @@ -0,0 +1,195 @@ +# 2023 July 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + BEGIN; + INSERT INTO ft VALUES('one one one'); + INSERT INTO ft VALUES('two two two'); + INSERT INTO ft VALUES('three three three'); + INSERT INTO ft VALUES('four four four'); + INSERT INTO ft VALUES('five five five'); + INSERT INTO ft VALUES('six six six'); + INSERT INTO ft VALUES('seven seven seven'); + INSERT INTO ft VALUES('eight eight eight'); + INSERT INTO ft VALUES('nine nine nine'); + COMMIT; + + DELETE FROM ft WHERE rowid=3; +} + +proc myhex {hex} { binary decode hex $hex } +db func myhex myhex + +do_execsql_test 1.1 { + UPDATE ft_data SET block = + myhex('04000000 00000001' || + '01020304 01020304 01020304 01020304' || + '01020304 01020304 01020304 01020304' + ) + WHERE id = (SELECT max(id) FROM ft_data); +} + +do_execsql_test 1.2 { + DELETE FROM ft WHERE rowid=1 +} + +do_execsql_test 1.3 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.3 { + UPDATE ft_data SET block = + myhex('08000000 00000001' || + '0000000001020304 0000000001020304 0000000001020304 0000000001020304' || + '0000000001020304 0000000001020304 0000000001020304 0000000001020304' + ) + WHERE id = (SELECT max(id) FROM ft_data); +} + +do_execsql_test 1.4 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.5 { + DELETE FROM ft WHERE rowid=4 +} + +do_execsql_test 1.6 { + UPDATE ft_data SET block = myhex('04000000 00000000') + WHERE id = (SELECT max(id) FROM ft_data); +} +do_execsql_test 1.7 { + SELECT rowid FROM ft('two'); +} {2} + +do_execsql_test 1.8 { + UPDATE ft_data SET block = myhex('04000000 00000000') + WHERE id = (SELECT max(id) FROM ft_data); +} +do_execsql_test 1.9 { + DELETE FROM ft WHERE rowid=8 +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + INSERT INTO ft VALUES('one one one'); + INSERT INTO ft VALUES('two two two'); + INSERT INTO ft VALUES('three three three'); + INSERT INTO ft VALUES('four four four'); + INSERT INTO ft VALUES('five five five'); + INSERT INTO ft VALUES('six six six'); + INSERT INTO ft VALUES('seven seven seven'); + INSERT INTO ft VALUES('eight eight eight'); + INSERT INTO ft VALUES('nine nine nine'); +} + +do_execsql_test 2.1 { + INSERT INTO ft(ft) VALUES('optimize'); +} +do_execsql_test 2.2 { + SELECT count(*) FROM ft_data +} {3} +do_execsql_test 2.3 { + DELETE FROM ft WHERE rowid=5 +} +do_execsql_test 2.4 { + SELECT count(*) FROM ft_data +} {4} + +# Check that an 'optimize' works (rewrites the index) if there is a single +# segment with one or more tombstone hash pages. +do_execsql_test 2.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} +do_execsql_test 2.6 { + SELECT count(*) FROM ft_data +} {3} + +# Check that an 'optimize' is a no-op if there is a single segment +# and no tombstone hash pages. +do_execsql_test 2.7 { + INSERT INTO ft(ft) VALUES('optimize'); + SELECT rowid FROM ft_data; +} [db eval {SELECT rowid FROM ft_data}] + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s; + INSERT INTO ft(ft) VALUES('optimize'); +} + +do_execsql_test 3.1 { + SELECT count(*) FROM ft_data +} {200} + +do_execsql_test 3.2 { + DELETE FROM ft WHERE (rowid % 50)==0; + SELECT count(*) FROM ft_data; +} {203} + +do_execsql_test 3.3 { + INSERT INTO ft(ft, rank) VALUES('merge', 500); + SELECT rowid FROM ft_data; +} [db eval {SELECT rowid FROM ft_data}] + +do_execsql_test 3.4 { + INSERT INTO ft(ft, rank) VALUES('merge', -1000); + SELECT count(*) FROM ft_data; +} {197} + +do_execsql_test 3.5 { + DELETE FROM ft WHERE (rowid % 50)==1; + SELECT count(*) FROM ft_data; +} {200} + +do_execsql_test 3.6 { + SELECT level, segment, npgtombstone FROM fts5_structure( + (SELECT block FROM ft_data WHERE id=10) + ) +} {1 0 3} + +do_test 3.6 { + while 1 { + set nChange [db total_changes] + execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) } + if {([db total_changes] - $nChange)<2} break + } +} {} + +do_execsql_test 3.7 { + SELECT level, segment, npgtombstone FROM fts5_structure( + (SELECT block FROM ft_data WHERE id=10) + ) +} {2 0 0} + + +finish_test diff --git a/ext/fts5/test/fts5contentless4.test b/ext/fts5/test/fts5contentless4.test new file mode 100644 index 0000000000..7fdf8c4b01 --- /dev/null +++ b/ext/fts5/test/fts5contentless4.test @@ -0,0 +1,247 @@ +# 2023 July 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc document {n} { + set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z] + set ret [list] + for {set ii 0} {$ii < $n} {incr ii} { + lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]] + } + set ret +} +db func document document + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + INSERT INTO ft(ft, rank) VALUES('pgsz', 240); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO ft SELECT document(12) FROM s; +} + +do_execsql_test 1.1 { + INSERT INTO ft(ft) VALUES('optimize'); +} + +do_execsql_test 1.2 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 0 1000 0} + +do_execsql_test 1.3 { + DELETE FROM ft WHERE rowid < 50 +} + +do_execsql_test 1.4 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 0 1000 49} + +do_execsql_test 1.5 { + DELETE FROM ft WHERE rowid < 1000 +} + +do_execsql_test 1.6 { + SELECT level, segment, nentry, nentrytombstone FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {1 0 1 0} + +#-------------------------------------------------------------------------- +reset_db +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); +} + +do_test 2.1 { + for {set ii 0} {$ii < 5000} {incr ii} { + execsql { INSERT INTO ft VALUES( document(12) ) } + } +} {} + +do_execsql_test 2.2 { + SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {5000} + +for {set ii 5000} {$ii >= 0} {incr ii -100} { + do_execsql_test 2.3.$ii { + DELETE FROM ft WHERE rowid > $ii + } + do_execsql_test 2.3.$ii.2 { + SELECT + CAST((total(nentry) - total(nentrytombstone)) AS integer) + FROM + fts5_structure( (SELECT block FROM ft_data WHERE id=10) ) + } $ii +} + +execsql_pp { + SELECT * FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} + +do_test 2.4 { + for {set ii 0} {$ii < 5000} {incr ii} { + execsql { INSERT INTO ft VALUES( document(12) ) } + } +} {} + +for {set ii 1} {$ii <= 5000} {incr ii 10} { + do_execsql_test 2.3.$ii { + DELETE FROM ft WHERE rowid = $ii; + INSERT INTO ft VALUES( document(12) ); + INSERT INTO ft(ft, rank) VALUES('merge', -10); + } + + do_execsql_test 2.3.$ii.2 { + SELECT + CAST((total(nentry) - total(nentrytombstone)) AS integer) + FROM + fts5_structure( (SELECT block FROM ft_data WHERE id=10) ) + } 5000 +} + +#------------------------------------------------------------------------- +reset_db +db func document document +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT document(12) FROM s; +} + +do_catchsql_test 3.1 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text'); +} {1 {SQL logic error}} +do_catchsql_test 3.2 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 50); +} {0 {}} +do_execsql_test 3.3 { + SELECT * FROM ft_config WHERE k='deletemerge' +} {deletemerge 50} +do_catchsql_test 3.4 { + INSERT INTO ft(ft, rank) VALUES('deletemerge', 101); +} {0 {}} +do_execsql_test 3.5 { + SELECT * FROM ft_config WHERE k='deletemerge' +} {deletemerge 101} + +do_execsql_test 3.6 { + DELETE FROM ft WHERE rowid<95 +} + +do_execsql_test 3.7 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {94 100} + +do_execsql_test 3.8 { + DELETE FROM ft WHERE rowid=95 +} + +do_execsql_test 3.9 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {95 100} + +do_execsql_test 3.10 { + DELETE FROM ft; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT document(12) FROM s; + INSERT INTO ft(ft, rank) VALUES('deletemerge', 50); +} + +do_execsql_test 3.11 { + DELETE FROM ft WHERE rowid<95 +} + +do_execsql_test 3.12 { + SELECT nentrytombstone, nentry FROM fts5_structure(( + SELECT block FROM ft_data WHERE id=10 + )) +} {0 6} + +#------------------------------------------------------------------------- +reset_db +db func document document +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1); + INSERT INTO x1(x1, rank) VALUES('usermerge', 16); + INSERT INTO x1(x1, rank) VALUES('deletemerge', 40); + INSERT INTO x1 VALUES('one'); + INSERT INTO x1 VALUES('two'); + INSERT INTO x1 VALUES('three'); + INSERT INTO x1 VALUES('four'); + INSERT INTO x1 VALUES('five'); + INSERT INTO x1 VALUES('six'); + INSERT INTO x1 VALUES('seven'); + INSERT INTO x1 VALUES('eight'); + INSERT INTO x1 VALUES('nine'); + INSERT INTO x1 VALUES('ten'); +} + +do_execsql_test 4.1 { + SELECT level, segment FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) +} { + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 +} + +for {set ii 1} {$ii < 4} {incr ii} { + do_execsql_test 4.2.$ii { + DELETE FROM x1 WHERE rowid = $ii; + INSERT INTO x1(x1, rank) VALUES('merge', 5); + SELECT level, segment FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) + } { + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 + } +} + +do_execsql_test 4.3 { + DELETE FROM x1 WHERE rowid = $ii; + INSERT INTO x1(x1, rank) VALUES('merge', 5); + SELECT level, segment, nentry FROM fts5_structure(( + SELECT block FROM x1_data WHERE id=10 + )) +} { + 1 0 6 +} + +finish_test diff --git a/ext/fts5/test/fts5contentless5.test b/ext/fts5/test/fts5contentless5.test new file mode 100644 index 0000000000..86d0753286 --- /dev/null +++ b/ext/fts5/test/fts5contentless5.test @@ -0,0 +1,111 @@ +# 2023 August 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the content= and content_rowid= options. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5contentless5 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +unset -nocomplain res + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1); + INSERT INTO t1 VALUES('A', 'B', 'C'); + INSERT INTO t1 VALUES('D', 'E', 'F'); + INSERT INTO t1 VALUES('G', 'H', 'I'); +} + +do_execsql_test 1.01 { + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES('x', 'y'); +} + +# explain_i "UPDATE t1 SET a='a' WHERE t1.rowid=1" +#breakpoint +#explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL" + +#breakpoint +#explain_i "UPDATE t1 SET a='a' WHERE b IS NULL AND rowid=?" + +foreach {tn up err} { + 1 "UPDATE t1 SET a='a', b='b', c='c' WHERE rowid=1" 0 + 2 "UPDATE t1 SET a='a', b='b' WHERE rowid=1" 1 + 3 "UPDATE t1 SET b='b', c='c' WHERE rowid=1" 1 + 4 "UPDATE t1 SET a='a', c='c' WHERE rowid=1" 1 + 5 "UPDATE t1 SET a='a', c='c' WHERE t1.rowid=1 AND b IS NULL" 1 + 6 "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1" 1 + 7 "UPDATE t1 SET a='a', b='b', c='c' FROM t2 WHERE t1.rowid=1" 0 +} { + + set res(0) {0 {}} + set res(1) {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: t1}} + do_catchsql_test 1.$tn $up $res($err) +} + +#------------------------------------------------------------------------- +reset_db + +proc random {n} { expr {abs(int(rand()*$n))} } +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} +proc vocab {} { + list abc def ghi jkl mno pqr stu vwx yza +} +proc term {} { + select_one [vocab] +} +proc document {} { + set nTerm [expr [random 3] + 7] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, contentless_delete=1, content=''); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); +} + +do_test 2.1 { + for {set ii 1} {$ii < 12} {incr ii} { + db transaction { + for {set jj 0} {$jj < 10} {incr jj} { + set doc [document] + execsql { INSERT INTO ft VALUES($doc); } + } + } + } +} {} + +do_test 2.2 { + foreach r [db eval {SELECT rowid FROM ft}] { + execsql { DELETE FROM ft WHERE rowid=$r } + } +} {} + +set doc [document] +do_execsql_test 2.3 { + INSERT INTO ft VALUES($doc) +} + + +finish_test diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 5f13513ec7..8788bc2ed6 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -47,7 +47,10 @@ do_test 1.3 { DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } -} {1 {database disk image is malformed}} +} {1 {fts5: corruption found reading blob 137438953476 from table "t1"}} +do_execsql_test 1.3b { + PRAGMA integrity_check(t1); +} {{fts5: corruption found reading blob 137438953476 from table "t1"}} do_test 1.4 { db_restore_and_reopen @@ -57,7 +60,7 @@ do_test 1.4 { rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } -} {1 {database disk image is malformed}} +} {1 {fts5: corruption found reading blob 137438953476 from table "t1"}} db_restore_and_reopen #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} @@ -95,6 +98,27 @@ sqlite3_db_config db DEFENSIVE 0 do_catchsql_test 3.1 { DELETE FROM t3_content WHERE rowid = 3; SELECT * FROM t3 WHERE t3 MATCH 'o'; +} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}} + +#-------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('one two three'); + INSERT INTO t2 VALUES('four five six'); + INSERT INTO t2 VALUES('seven eight nine'); + INSERT INTO t2 VALUES('ten eleven twelve'); +} +do_execsql_test 4.1 { + SELECT hex(block) FROM t2_data WHERE id=1; +} {040C} +do_execsql_test 4.2 { + UPDATE t2_data SET block = X'0402' WHERE id=1 +} +breakpoint +do_catchsql_test 4.3 { + DELETE FROM t2 WHERE rowid=3 } {1 {database disk image is malformed}} finish_test diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index a815320b76..fd2a841c7e 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -100,6 +100,7 @@ set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}] set all [db eval {SELECT rowid FROM t1}] sqlite3_db_config db DEFENSIVE 0 +unset -nocomplain res for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { do_execsql_test 2.$i.1 { BEGIN; @@ -108,12 +109,12 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { do_catchsql_test 2.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check'); - } {1 {database disk image is malformed}} + } {/1.*fts5: corruption.*/} do_test 2.$i.3 { set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}] expr { - $res=="1 {database disk image is malformed}" + [string match {*fts5: corruption*} $res] || $res=="0 {$all}" } } 1 @@ -152,21 +153,24 @@ foreach {tn hdr} { execsql BEGIN set fd [db incrblob main x3_data block $rowid] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary set existing [read $fd [string length $hdr]] seek $fd 0 puts -nonewline $fd $hdr close $fd set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}] - if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + if {[string match {*fts5: corruption*} $res]} {incr nCorrupt} set {} 1 } {1} if {($tn2 % 10)==0 && $existing != $hdr} { do_test 3.$tn.$tn2.2 { catchsql { INSERT INTO x3(x3) VALUES('integrity-check') } - } {1 {database disk image is malformed}} + } {/.*fts5: corruption.*/} + do_execsql_test 3.$tn.$tn2.3 { + PRAGMA integrity_check(x3); + } {/.*fts5: corruption.*/} } execsql ROLLBACK @@ -205,7 +209,7 @@ foreach {tn nCut} { set res [catchsql { SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC }] - if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + if {[string match {*fts5: corruption*} $res]} {incr nCorrupt} set {} 1 } {1} @@ -235,7 +239,7 @@ foreach {tn hdr} { execsql BEGIN set fd [db incrblob main x5_data block $rowid] - fconfigure $fd -encoding binary -translation binary + fconfigure $fd -translation binary puts -nonewline $fd $hdr close $fd diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index f9a95665c4..20be7c45cf 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt3 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -102,7 +102,7 @@ proc do_3_test {tn} { list [ catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg ] $msg - } {1 {database disk image is malformed}} + } {/.*fts5: corruption.*/} catch { db eval ROLLBACK } } } @@ -273,7 +273,7 @@ do_execsql_test 6.1.1 { } do_catchsql_test 6.1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------- reset_db @@ -289,7 +289,7 @@ do_execsql_test 6.2.1 { } do_catchsql_test 6.2.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------- reset_db @@ -308,7 +308,7 @@ do_execsql_test 6.3.1 { } do_catchsql_test 6.3.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.3 { ROLLBACK; BEGIN; @@ -319,7 +319,7 @@ do_execsql_test 6.3.3 { } do_catchsql_test 6.3.3 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.4 { ROLLBACK; BEGIN; @@ -330,7 +330,7 @@ do_execsql_test 6.3.4 { } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.6 { ROLLBACK; BEGIN; @@ -341,7 +341,7 @@ do_execsql_test 6.3.6 { } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------------------------------------------------------------------------ @@ -374,7 +374,7 @@ do_test 7.1 { db eval BEGIN db eval {DELETE FROM t5_data WHERE rowid = $i} set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ] - if {$r != "1 {database disk image is malformed}"} { error $r } + if {![string match {*fts5: corruption*} $r]} { error $r } db eval ROLLBACK } } {} @@ -399,7 +399,7 @@ do_test 9.1.1 { } {} do_catchsql_test 9.1.2 { SELECT * FROM t1('one AND two'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_test 9.2.1 { set blob "12345678" ;# cookie @@ -411,7 +411,7 @@ do_test 9.2.1 { } {} do_catchsql_test 9.2.2 { SELECT * FROM t1('one AND two'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -497,7 +497,7 @@ do_test 10.0 { } {} do_catchsql_test 10.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -678,13 +678,13 @@ do_test 12.0 { | end c2.db }]} {} -do_catchsql_test 11.1 { +do_catchsql_test 12.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} -do_catchsql_test 11.2 { +do_catchsql_test 12.2 { INSERT INTO t1(t1, rank) VALUES('merge', 500); -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -870,7 +870,7 @@ do_test 14.0 { do_catchsql_test 14.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #--------------------------------------------------------------------------- # @@ -1040,7 +1040,7 @@ do_test 16.0 { do_catchsql_test 16.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1126,7 +1126,7 @@ do_test 17.0 { do_catchsql_test 17.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1435,7 +1435,7 @@ do_test 18.0 { do_catchsql_test 18.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1546,7 +1546,7 @@ do_test 19.0 { do_catchsql_test 19.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1630,7 +1630,7 @@ do_test 20.0 { do_catchsql_test 20.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -1764,7 +1764,7 @@ do_test 21.0 { do_catchsql_test 21.1 { DELETE FROM t1 WHERE t1 MATCH 'ab*ndon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -2100,7 +2100,7 @@ do_test 22.0 { do_catchsql_test 22.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -2211,7 +2211,7 @@ do_test 23.0 { do_catchsql_test 23.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -2429,7 +2429,7 @@ do_test 24.0 { do_catchsql_test 24.1 { UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thread*'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 24.2 { INSERT INTO t1(t1) VALUES('integrity-check'); @@ -2518,7 +2518,7 @@ do_test 25.0 { do_catchsql_test 25.1 { INSERT INTO t1(t1) VALUES('rebuild'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_execsql_test 25.2 { PRAGMA page_size=512; @@ -3011,7 +3011,7 @@ do_test 27.0 { do_catchsql_test 27.1 { DELETE FROM t1 WHERE a MATCH 'fts*'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 { highlight(t1, 2, '[', ']') FROM t1('g + h') WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; -} {1 {vtable constructor failed: t1}} +} {/.*fts5: corrupt.*/} do_catchsql_test 32.2 { SELECT * FROM t3; @@ -4267,7 +4267,7 @@ do_test 35.0 { do_catchsql_test 35.1 { SELECT * FROM t1 WHERE t1 MATCH 'e*'; -} {1 {database disk image is malformed}} +} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}} #------------------------------------------------------------------------- reset_db @@ -5351,7 +5351,7 @@ do_execsql_test 41.0 { do_catchsql_test 41.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 41.2 { INSERT INTO t1(t1) VALUES('integrity-check'); @@ -5573,7 +5573,7 @@ do_test 42.0 { do_catchsql_test 42.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: checksum mismatch for table "t1"}} #------------------------------------------------------------------------- reset_db @@ -5813,7 +5813,7 @@ do_execsql_test 44.1 { do_catchsql_test 44.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 44.3 { SELECT snippet(t1, -1, '.', '..', '', 2 ) FROM t1('g h') ORDER BY rank; @@ -6644,7 +6644,7 @@ do_test 48.0 { do_catchsql_test 48.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: corruption on page 1, segment 1, table "t1"}} #-------------------------------------------------------------------------- reset_db @@ -6917,7 +6917,6 @@ REPLACE INTO t1_data VALUES(1,X'2eb1182424'); REPLACE INTO t1_data VALUES(10,X'000000000102080002010101020107'); INSERT INTO t1_data VALUES(137438953473,X'0000032b0230300102060102060102061f0203010203010203010832303136303630390102070102070102070101340102050102050102050101350102040102040102040207303030303030301c023d010204010204010662696e6172790306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020108636f6d70696c657201020201020201020201066462737461740702030102030102030204656275670402020102020102020107656e61626c6507020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020202087874656e73696f6e1f02040102040102040104667473340a02030102030102030401350d020301020301020301036763630102030102030102030206656f706f6c7910020301020301020301056a736f6e3113020301020301020301046c6f61641f020301020301020301036d61781c02020102020102020205656d6f72791c020301020301020304047379733516020301020301020301066e6f6361736502060102020306010202030601020213060102020306010202030601020203060102020306010202030601020203060102020306010202030601020201046f6d69741f0202010202010202010572747265651902030102030102030402696d010601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202010a7468726561647361666522020201020201020201047674616207020401020401020401017801060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102ad060101020106010102010601010201060101020106010101010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020415130c0c124413110f47130efc0e11100f0e100f440f1040150f'); INSERT INTO t1_data VALUES(274877906945,X'00000e96023030011a042319320d3b123d812b5a31120110446e581b66814a05010a4537814274010e8102815c810f3d0104846d01081581204401103741043c59416b44010a404655265301103f73811a11114213010a821235820f020135030484320201360104816a020162020484550302390301710a04824a020166030483690201670704837d0201690404822602016a0504825c02026b620504817502016f0904810d0303e79c88060482760201700a04826302017204048155020373c2be050481130201770204846202027962050482710202c2ba010482140203e58496070483330204e8b2b879010483710101310110545c0c814b0e3a6501082c815d5b011a2a0e2f0d765c3d686014061d0d0112810733112c2e82141101048313010e5c6f632e813e42010c811882370548010e19158146822f1f01104d364a708146135a010a237b0a55210201610904841703027678090481270201620304810002026374060484660202657a0704827602016601048351090483540301660704814b02016b03025f0304c582caba0204816602016c01025f0302cebc0904843e02016e0804821802016f0a04817503016f070483100201720304822c020484380201740404842e0102460201770104812f0204836c02027a6f040483110202cebc02048267040161020484650205d5bd62cebc0604845b0204f2a580880204842a0206f38184a179670502750204f696a3aa0a04814601013201063330390110812281378114600d010c03716c5e822d010e81226b542a814d010a72740f83000108813a1e0b010c5681046f812c010c07814a777328011664244219531b1a2f811e4a010c4d81557c7f1b0201300704810702013307048230010484050202367807048175020239710804832502016204026b0204814d020363306108048262020265650504817602026667070483150201690704832f0301360a04814d02016c08024702016d0304843e0303cfb2630204814002016e0804837503016203048416030370c2be0304821b02016f0604834b0201700504816b030175070210020273660604841c020676c2bac2b2640604830a02017704027d02017808048141010482700201790504811d0202c2ba0502470206ca8d73ecbab9010483340204f09e9ab504048367010133010c3e04814f82250114812e814b2e0411811305010c811337811e6e010c82085e2b0e5d010c61812054811c01148122451b0781050c813d010c17823762643e011e080c1720814a10364306143b0d33260112810f0c2a810c816b13010a8163810e470201370404811102056176c2aa36050481530202646e0904846202026a730404827402016b0a020c02036f616b0404830a0201750504820d02017605025e0201770a04820702027a73050482460202c2ba0604824a020483330203cba434040483200203cebc790304847e01013401124181442c1d091f81580108601d8336011081320a2b8125820001123b0b81158116811f070110078112817a817308010e6682410d810e2601122d0d6413378147351e01105081021d3525812d01128246510a622204054101105c1b620e81302b05020130020483480104822702013102027e020132030483270201350304844802023770030207020261710604823f0201640802570204830a020265770304831f020168070483210201690204825b02016f020481280704835402057037e18b8d0904810802017304048439030172020481440303c2bd6b0a02630201760202490804815e0201770304816e050483550201780704816902017a070483280207c2b2f093aabc780a0482240301b303024e0301bc0604837a0202cb800204834a0202cebc04048201040484410203d3ad770a04814f0202d5a508048371010135011630817e0f81040d2c041552010c813d3b7e8115010c40692182693a01121f810d810d0a32814701101d1d1f642281742e01068229240110811231810a387f4c01100f50810f8165810d0114811f26443152593c104a010e641e1a3357820e020132030481540201340204815402023778040207020163060481020204815b020164020483010201670a0481540304f0948f870904811a0201690704835502016d0604832203027b01022803017a0a025102016e020484260404816002026f690502680301720104834e02017208024f02021d020475e69c8e0504814c0202767602025f0302de870a04837b020178090483200104835003026a72090484400201790504837204022302017a0104836b0202c2aa0a0481070303bcc2b3090484370203c7866403027501013601087c158303011212814305813e7b0e090118141a1c49713a211e0c74630f010a59826d8113011203328166037781561a01101d7f1d2a1f822533010e820e070f7b40160110811f40292c813226010a2d20824a32010a81418158670201300304840a0201660404817d02016d0804826902016e080626817f0104820b02016f0404825a02017003023b020272750804840c0301760a04844d02017403025b020175080484200202766e060482360303c2ba6c030482220202c2aa0204810e0301b20204835703048421040484240301bc0802410202d2a1010482630204e1b18f3704048354010137010c08816337812f01101382211532424d39010881248123010e7724810267815f011081236029813e273301101b7b29812a5b813b01128150810324814b220b01060c8417010e165b6c81708117010a1782346f6c0201380804816803016b0604840102013908023f0201610204816c0201630302760201640304832604023e0204833902016502048203020266770804821a0201670a04830002026964080484500304f29e9eb70802250201700204811b0201710904832d0201730304826b05048403020174010481000205776a62c2b2050482630201790202260206c2b2eaaeb464050485020301b3080482480303bac7af0a0484550203c695650804822a0202ceb90a0481170202dbae05023f010138011819814a2703390a61090c6912011a21181304812523811b5f164e050114110f35128123423f810c010c817573817c03010e7182590c812b0401142503597e6e0e2f3a3759011252813a811a2b75091a010882162a31010e17450a81048279010858658208020231670a048205020361c2b906023b02016301048236020164050482520201650904833d0201670904811b0201690604825002026a7a0604837c02016e0204832002017002026d0302c789010481020201720504835e0201740604810002017502020a0702630302676306025f0303e4a0a70102640203786a75010484440201790104841402047ac2be72070481340207c2ba3766673576090482790301bd07048142010481600202c7b309027a0401740604823b0202d2bf0304830f0204e989a6300a02600204f4bd91b60702120101390106518369010e19254641823711010c258267288121010e817c810d2b17250110810a578133812f4c0110415681067b288121010e0881208119347101140b8131543c8100343d1101088203813e01100d742e3230820f3802013006048107020134080481440202356501021a02013808048147020162020482230201630304833c030162050483390206656cf093b5bd070484140202677303048502020769e3ad9669c2b90304847402016b0804836402016c0404841f02016d010481250904825202016e06025402016f0704842a0201700a04834a0201720a0483530201750304822e020676f097b18374030482140201780604833d0202c2b9010481550301be070483720204826f0202ca80060481630202d5a80504833f0101610114551047810e130a78660c011a364611206c0b13080705733d5501240f08070c090b0c20813d1471042e4351131e011204412d814f0913104201263036110d060b1f811a301b0f4e1a29092f181c012808071e221a2a81075b320503065a0f140c1a0a26011c07231d0e6f3715063b760c6b091501121111303e3a71566d6d010e0867814d816a0c01181e18240d41724d221b3f384b0201300204830b060483080302c2b30204837f0201310a026e020134050481560301730902690201360406827c0f0201370204825d06020a0201390904815b0304f3bfb2a70a04822c0201610404810e020262660604841a03017608025b02016309088112825f020164090482310201650a0484480302396f01025d02026774010482090302df9b06048321020168070661813303016f03048248020169030483610504814001048401010483460301300704824203016a0204824402016a0504813303021d03017209048412030277380804824502016c0104814c02036ec2ba0804835702016f0204811d030176020238020270360a04840d02017107048201030469ca99690602350201720304812d0104845e02017304022c02017409022b020175050484140302caaf0402410201760404831f020177010484180704845203026f6d04024b02017a030482660202c2aa0204810a0301b30a04817b0302bd76020483780302be6a0302440202c58207025e0202c69901027a0302ad77010483200206c993f099b183070484270203caa1660204841a0204f29788ac0804831d0204f4ba9f950504843c010162011c0e33810216341c2413042130780501184d373e53131f2f052907423e010c830e3781390e011c1320461f81041b811b041e15243d011e241b10816c310b130c3133033b0741011a11816d3139100c13140b395848011c580e411a06304306810a3138330d011a441707092c70140c1643813920010c73653581374f010c826c81210f0402013803024f07048172020161020481650204810c020162030483470301370404813602016307048379020483280201640304815501048176010481060201650104812a0104841f0201660504821f0201670302700201680804846403016b0a04831d0301700904845502016908025c02016a010483560904827602026b6306023402016c0404832602016d040484410204825702016e0504831603027831020482160302c2be0504827d02026f6a05048121020171030232020483220304845402023a0201720204845c0304846e03013607048224020174060484480201780704844303016f0604814f045807070a0707070707080709070709070808090a5c07080708080b07060a06080707070b0a0b0808070b0a0b0a55070b08080a0908080707060709070709070706080c060b07070c0a66070b08080609070607080c0909660b06070707080a0807070b0b0707080a0b07070d0607080c0908630707070b07070a070d060b0707090a07080b080a070809085f0707070c0706080706070809080f06080a5807070607060e070807080907070b070b060c0709090807690808070707070708070608070709070809070a0d0b07070809095b070707070707070c080d07070b06070707070c07080b0808811a0b08060706080a070a07080609070707080808071307070a0708070907060807090b06060707070b0707080708070707080c090a0a81080a0b07070b0f0b0706070707060b07070b07080808110b070707'); -INSERT INTO t1_data VALUES(274877906946,X'00000e880330627a020482240202c2aa0a04833f0301b30704844b0302b9650704824f0301ba0204845f0202c9820a0483640202d194060482300203e19cbd0904844b0203e691b4050483510205e78dadde9b0104821201016301142a6c033b8151085c094601140b813d49313f81110e1c011681163611221527257f5d38011c150f22811a0a3c12350631238117011c3e26420b402c1d81080c40150b2f01181c3143382640273d60132e070118663b1d162a1b0e2e8111393e0110821117310e52811c01141a2f49810181391f2b130112323c0305812a6f2e390201320204842702023334020481340201350202610201360304844603023362040484470203376a360a04826b02013808026203016f0704830d02013902025502016106088170827a0301320a04820403016c0404831502016204048327050484030201630a04814302016401048349020484760302430301640204845d02016504048249020367c2bd07020a0302cebc0902150201680a02500201690204846102016a06024c0301770504842b02016b0104830e0704811803023370040483580301710404845f02016c0504844f0204820d0204837f0302c2aa0104833702016d0104844c02016e0804834f03026c6a07025702046fecbd9a01023a020270330204830a0301740304837c020271350204811e0201720706833b310206736ef09289b70104832b020174070483290301320204827c02017608022802017806025b0302c2b30904835202027978080483040303c9b56f0904846a0202c2aa04048127050482120301b30504813a0301b901024204016a0704840c0202c5820704823c0202c999010482470202cebc0602400203db91670602730202dca7050482760203e1a3950304817e0203e786a702048273010164011a0612105b292b817c1211080d5a01147c1d420b35451c36811a011e0e168117081c0c2e051d474055192d011e02050a1c81180420250f815f300f21011c02316a37143321443a10042d54230112761428810e4750054101101805072b8215294e0116680f0f5381445a3e0b070901224e4a41210c361c281b101c43051325130f01185a1e19108106300f2e3f4538020130060481370202327305022002013405048168020335377a03021802013704027b02013804048260020161030882118101030263650904814d0201620604822502016303048419030135090483240201640602280301380404817e0201650404823c0304f097ac9f010482680201660a0267030566e2b6936f0104821c0201680704813302016b0604832002016c01026a0301610204843202016d0804845e03026d6d03023c020270730a04817f0304c2be797a0804832e020271710504835f02017203021b0201740204825a02017706027202017805048451010683572e03016e0a04814c0201790304811702017a080484450302c2b90204837b0202c2aa0604825b0303b273630104841c0301b9050485040301bc0a027b0303bd37680502670202c98b0204826b0203cfa1740504823a0203d199610202350203e3bf87040483570204f1baaba90504817301016501120b8104392d0d20180f011645213f292e4d0d082f8165011e0b400c07341b2329307f193338173a012055292409050c560a272a0f4403245718011a1c3a183f1c43264c3126060829012081208102043a044d0621650b180e150e011a066c030e513d7d265e1313130c0118171953040457347b114d191901261b1c060c26090d6f0d332a1519096e03101d1d012207342f1f2c7e2517251d0f310d2a17081e02013005020a02013308048247020135030483660201380704841b030132030482180201610102600604825f0304c2bdc2ba02023a02016209021f0201630604813002056663cebc610604841e0201670504816a0104842703037177310604833f03027872080482350201680604825e02016a040483320104840d02016b0304813f02026c6408020602016d070481590104837d03016c08020d02016e060484630301780404815c02016f0104826b0804825e02017008027b0302c2aa0504847c0201710404836d0201730402510302cebc08048338020375c7bf05048344020177010482660304822303026479070481630204786ec2ba0204814f0202796a0a04834f02017a01048407030484660604810603026561020483180204c2b278390504813d0301b304024f04026536010483110302b9330604813e0301be0304840b040484560202c4a702022d0206c6a5f7ada9990402350202c79f090481180202caa60502140204cebcc2bd080483320202db900a0481250205f4b5aa9079040484360204f7b985bd0204835e01016601128101285c096981190e01121f813f0d431a8135530114698102813228492f190a011260161881328101812601188155780d813257050c0b04060114161681340772811b5e25011c4505810e13290b253a0c0c0a1a4b011a3714133e1235812b136b062c0b011a6b591356810c3c240906250b1001148127810d413e0e81090d020231680504822c0302c2bc0204830c0201360104840c0303656a740902110302d1950504824f0201370902130201390a0482530303e0a9ae0904844c0201610804810d0201620a04810c03023039030481330302356902048268020163060483470201650504822d02016706048200030483560302713509025002026a79010484410804825902016c0504822002016e07020d0304843a02016f0204842d0201710304837604048361030482430201720904840f020173020482520804810b02017406068425320201760202420201780804845d02037979650704814802017a080481110301780804812c0202c2bd070484600202cebc0508813082410102770401610202250202ddb40302310205f19e9a937a030482410101670118365558195a0a062d0581260a011881068143330844041f0a1851011a2025141e1081204f550e077521011a193b1f58351912265681220821011812070528472f4e2f407f204a01124d1e1f811b810d7b4d01180f1d3481034a35580a12811f011c2303340d1470150778070c812331011620411939703c032915143f01104281116a3d323c6a0201350604842902013703023f0201390a04816e0201610604826402016202023f0201630a04843302016409048258020165020482620404814702016602061b833f0201670404827d03026369020227020169040483490301670604847602016a0604845d0404840902016c04024601088150822b010482350204830a0301690304844902016d0202710704820002016f03048509020482380304836603026e74010483580201700504817a020171020481080204826a0201720304837b0202410302160302c2b3080484550201730904816e020174040248010484280304834e020275730404836e020176060484720104815c02017709026a020178060275040483790201790504821501026c030170080483770304cfb269710704815102017a0204813303016f090483160202c2aa050481400301ba0804810b0401630804830d0301be0604844d0202c8a30a02110203cba0640204816a0202cebf060482420204e487856e030481080204ec97bd6d080484080205f09eb3a0770502260208f687999931ed878703048424010168011a12460e090c036e151b812e065501161708411982151738471f35011a2d1c0678340c1f04425c21200c010c2a087f255d4a011a0d0b6c33814a212c3a0a401b1e011c501f2381010a0481201c0c6012280118150b5228520e0a036c1c8123011a15810a060408030a81563f381601185b1b06212a143f332a60160e011a221b1e62411d2048090e0b0f5502033072350804826a020131020482530201320304823b0201330104814502033677380204813102026174020483540304dbbf6f620404835d0201620504846a0104831803017103048323020164060483740201660a02410201690102130104821402016a0504823f02016c0404832e02016d09022602026e640a04822702016f070482000301750402670201710304813106020d0202726304048220020173060484530201750504831e020483400302c2bd0704843a020177020483470203786371030483740201790904810002017a0302300202c2b9080483280301bc010481700303bd33720304825e0205c99973c2bd040483160203d5a6330a04842b0204e7b3b3300904813c0205f099a68f72090209010169011c21101d4b2d0e0e066b4253074c140118070a0910447556030833541d01163733816837402b3909122501183c5b1139102e2d430c662334011e27050f21621230323503332b6a0332011e1e07031843202e6e3c2850094d410c01163955220b16812d24521212011681250b0a3505460481176f011a2c09266b162968051c0a1481170116022e1e820c352037263a070201310104825d0602110303696869020484070201320204826e0201610304832203016f040232020162050642843f03048336020484540301370804833c03026c6105027a0203636165090483120201640502770304833e03017107022f0301780702470201650a04811e020167080238020168040481160102230404826b030170040483000201690304836f0302766c0304811402016b0504812e02026c6108027702016d0308827d81530604837302026e790904842602016f06048208020170060481680302320201710204812902017307048255020274320104822a02017506026803016e0a04821303017207025f0302c2b90504834a020177020483130201780604836b0402210301320604847302017a010483130202c2aa0804823c0301b9030482600301ba0104845304016c0504837e0202c3b8080484600204cf9d6379020483660202d3860704812e0203e3a4be0402560203e58784010481210204f09e95ac0102580204f5aea5890a023301016a01123428131a1f6c81445601141e227c1a7b5f1918810301182318812e17455605460d811c011a28820221311a6e12093f050a0c0120082c0b0f1362074457460c3b070d5132011c2143052a20133d160a358117591f01103136813b136e6247011c100e4c28060d16815a320a3e11070124462c03582e262d45110804113326040808070807080809090b81050708060708090607060907070b070e07070807060706070b08070f0807070709080708080c070706060808090c07060708080708080909811307070708060709080707070607070a060b070706070707080a080607060c070707080809070608080908090a812406070707070a0906070b0b0908070b07070b0607070b0608070608090b080a080f080a0608080b070b08070a080b0a810408080708080607090707080807070b070c070a070f070b080607090707080d06070b810b070607070607070b08070707070b14070a0f08070b0d08070e080b060a0a070a0707080707070709080a0a0a0e810f0907070709080a0b0707060a0707060807060a08070b08070907060807090b090a0a81140a09070706100707090a060607060e07070807070d08070a07070806070608070a070708070707080a0808090909'); INSERT INTO t1_data VALUES(274877906947,X'00080e7f073c23110a1a18392f66090524183704276d6703306a320404824e030164080483520305c2bd7ac2bc0604815a0201360704833202016106021f020482400201630304822a0708817e8204030173040483500201640404824803016d0804824a03017709023002016606048367020268680a04815802016902088339811804027f0302656e0704834e0303d5a5370604816702036a3366090484470303c2ba660904826e02016b0904837c02016c07021403026c610604835802016d0204816802016e0104831202016f0104822f020270720602060201710704822202017206048174020273690204824602017409020c020175090482140201760a0482720301660404824403016a090484290201790404845703025d0203c2bc33040484620301bd0304824c020484540202c78607022403019a010482380202ca87070484390202d39d030485050203e184940404831b0203e6a881060483480203e8b18c0a04816d0203ee8d850104814801016b0110467257393c81272c011a053e815d3b190517064524521f011c3823590a8115372004313b1f3216011a5a20780b102d0804426916112c011a182f810781082d12137026161501221a180516811611051c131207811515173501180320112581062e05621c1407011c2d0e0617811522062208065a21520114582841621e6c203f1e2001161647411a272533815b1c2602013009048309020232630104835a0301720104817f0201330604836f0302ddb5080482560202347a07048102020135020483460104827b02043678ca800a04835f0201370404814b0104846002016103048246010482220301700204833f0201620404824d060481150201650304824f02016606088110834c0201670604821d0303c2be66010481790201680404843b030176050482270201690a04830e02016e0904844202016f040481010301630304822f020270640204822f03016e0704845802027177090482710206736ec2b2796a0104832e0306dab1d485377004048304020174050481700201750a0212020378627604048164030173080483190201790704833d0204823a02017a0506820e67030178070484530202c2b2060481500104823f020483030301b3020484310301bc04027e0402caaf0a026a0301be040482590204842e0202540202c79f0804824d0202cfa30804815a0204f29a92970204823301016c01140f63351a0a653b650d22011c09117a3e1538123537046a15043101141310082f49052f772b0c011c11121781583c2a5010133228241301287f3e0a2b1244080503060a100f413b4f0d070e2a01103e4e1f04814e7b1601183d0404052877111f230f811d01123a100f053e5c076910011a031732102381243d1b1727507301180e5d273e810803812e0f192a02013301048271070481330204821d020134080263020135060481280201610104830a0201640604826d020165060483050201670204841c0504841c0304841b0201680a04845602016a0104811c01024f030481080204813102016b0204837008024502016d0404836c04068207780301670704842302016f0404821203016d040484490301720404837e0201700104821d03048407030165050483050201720a04811602017307023502017407020503016f080484240302c2b90504821b020175090484090201770a088119822503026d6905048300020178030482680604812a0201790104830c0204833a0303d9a06806022002017a060482600203c2aa33030481560301b904020a0301bd0504820f0202d0b90904817c0202d3820202200202daa9080482030203de966e02024b0202df9d080484350204f098b0a20604845e01016d01220304456608322258060a031d4c38340f090112310c070e4238626e6601124a318109030513812f0118240d561e533742188113101b01160b24444b224d44814d4806011c05774e483410330d23541b28090401141f29062581131e221b6d011e81053a037a03320b0e4c24360d2310011a0e321d3c141825111d54637a1c0114093d3c2e58571a35293a0201350104840a03017701048330020136060217020138030483370201610a0482650201620504815e0201630704827701048201020164080483690201660804846703016904048113020167070483080201680504837d05022c0302cdb10a04815f0201690104833a0404824302026a360a04823b02016b0a04813502016d0504831a0204833803021a02026e360404825e02016f080484140201720304844b0404816603056ff09d899b0304823f020275390204816e0301780a04824202017604088308812703027902027770050482040201790104827e040482750204812902017a060483030304c2bdc2b30104836f0203c2aa62040484040301b903021c0302bd6b090484300301be0704814d0202c99402025603049a65656b090484020202ca92090482060203d19a730504844a0203d49f690804836e0202dfa8020482710204f09180860704822901016e011a0c0b8104243647521f43231f36011a2e1b33432c3d0b414905054d17011010573a6c0a816c1801160e063582340a5239050b06011a4481063d1b67250f2044200839012044591d1857291214135814101a1b361d011225067e8147111a4a4301166b13362e17195f3812186f01141c465b032b290406373301182a152a2281300f8107054e3f02023274080481770305c2bacf8168020481450201340604832f0201350704842e02013605020e0201380404841d0201610404810d020483750201620304812b020484230301610804834503026c6a0304816d0201630102380305613577337405048359020165040482720201660904826202066736f094b0af0a0482250201680104811f02016b0304847202016c0404822403016f0904822c0302c2b301025002016d0504817b01023f02016e020483090802040303e7bda10804832d02036f6b740404811402017005048419020484220202716506026303026b760904830a020172080482430304706c73620504825f02017308048413020174080481070201760104827f0204836e020477e7b89a0104840e02017a030483700206c2aa35657065050482740301b30804842b04046cc2be78090481040301b903020d0301bc010484260904813f0203c7a5620302330203c99f36050481010301a30704815b0202ca8b090483250202cdbb0604820a0202cebc0102170401380304842b0207eca2a6f29c87950904824001016f01221d17052b58101241060e3a201f1021633a0114816919811c142443100801280426080e2620042a812c531a490e121707131710011273432e493347811a340112195f671f46721c325e0118380c052b812822478107600b0116021c21821b2019263433040126021b05351b2a286b05181f071b5628111a330a012014533e073d0c0e5469141d1e2734050901220318051b44412803632e0642370e0a3a2b020131070481770201320a04812f0202346e04022b020136030483590304f09a81b60404834702016105048210020162030205030167010268020163010481540604820202048300020264310902420201650804834b02016703048247030365c6b602048205030573f098b890030481450201690204832802016a0a04826703016c0104825e02016b0604815e03016c03048334030677c2aa74c2bc09023d02016c0304823903027777060484540303df866c0104815b02016d0204811f0303796f7704020302016e0204814d07024a02016f0902680201700604840a0104831a0204835a020172070484440201730a023703026b660604830e030278790304815b0201750904822402017704025b020178030482350307f4b2a3896a343407026b0201790902720302633409020402017a020484590302dea004025f0202c2b20204816b050481200202de900402160204ee85a5770204822c0101700114143a0d391a60812d4e09011a2f313104201c372c3a3411321b011a268140144226334145050d1c4d01164e081f20671f088107237901186b123c1f6d07261e2b732e210116511116342a3d32376e083001106882257a0a17141101163039192b0c05812d735f3b01262a3e0841030b17181411051e0a18530e272b6d01182f322b260e24581d5381050f02013104048353020132050482370301690204843e02013302020d0201340104841f0201350a048139020137050482770201380204833a02016105025a0504832f0201620304836d020163050484100304832003016c030481290201640304837e020482490304822b010482290201660a04827002046964dc960204833102016c0704814502016d010482000201700104817e02037176760904821f020473eb91a708048152020174090482770201750404831e01063a825d03017a0904826a020276730a0254020177080260030277630104815c020178070481220202020301720804841202017a0204834c0202c2aa040484010301ba080482580202c6a3020481320203cdbf690502790202ce90070483140301bc030481470205d1a371cebc060481590203d2976a0404830c0203dfba6e0604814b01017101163732393b8120422f054b0e010e030b211d815d1c01165641757c080d81311d090e0112816581542d2313054301224e07121706516606080e39102d231c4b39010a2d81402d5e011a132527428114080d6e1111721c011a814a1a341538251023100d1c4c011e22182622623712411e38162a182d3b01142b67611981470f1f1f250201310a04824e0202336207048217020238730204815a0303cf886d06025a02033962620404833803016f0a04814e020161060483140302726d050483450301790904810c02036376690504811c02016403024d020165010483280802550301650904827f0304ebb8b561070482340201670302670301660804810e020168040482340201690704844d0302616404020f02016a030481060301700704827802016b070481240104814b0302c2bd0504816102016c0604837f03017a0404837902036dc2b90804810002016e0904821602016f0304812c03016401024b02017103048233060483600303e5848e04023a020172040481050305f3978aa06c070481151708070b070a0d0707070607080c080909090706080707070707070806070707070a090b070708080909090981140708070708080b0a0b0b070b070907090707070707070807080c0c070609070b0807100706070e08080a81110f06070707070f07120a0c070707070b0707060607080709080b0b080709060708070808080a810f0707060707070b070707070a080b08070e08070b0b08070c080f070a09060807070a080909080a810b080b070706070b0b0708060b07070c07070707070a0a09090b0708070a07070b0a070c070a060b080907080807070d8123070707070a0706060f070707090b07070707070b07080907080a060f070608080706070c060707070c070a810f07070706070707070a070b0713070a070707090a070c070706080a07070807080808070b0909810607080808090707080709060a070a060707070707070b080707090707060b0807'); INSERT INTO t1_data VALUES(274877906948,X'00000e8a0330717304048359030134050481100203756371070482190201760704817b0301770804821d0201770204844d0201780204836c0404826103017504026a0202c2aa020482420301b305022c040267390602570301b9080481540301bc0102290206c99cf6b5aa80080481430202cebc02026e02048120010172011a1c2f15158108048125463f251d010811811539011412423105812181171549011847284a30234e5b33042632120118351e8113817d0f2b220d111901264f104a211004061d0a2a0b35121a0a2118341f011c81160a1b030d2a0610243e445f0c011c6f0c1e3b1768141e322717500b140110537f810169811625011a492847203e210f532c16480627020135020481780302caae0104811a020261330904846c0201620804812d0201630404814a0201650704837503017301048276020168050213020169010484540604842b0302796f0504833302016a0304831b02016b0604701302016d0604815302016e070483630204815202027071020481520201710104835b02037266700704843002027362040481490201740904817d020175020481040304f59e9c9407048218020176060484060204776dc2aa020483070201780504812503016601048159020279790704840502027a730904826c030178030482740202c2aa04023d010483540301bd06026b0203cab877010483290202cdbf060482410202cebc04025c0401690204827f04016c0904840e0202cf880104835d0203dfbe6c01025b0204f0aeb7b2030481680205f1a7b5bb390504826a010173011a22810d12415003071f81181839011a3220221511546d810012052b57011a0c4274300d154e81111f041e10011e293f4213051b2276560817312811170120092136122418370e4e782b3912080f3201262b3b340f222b0c09142a0822116a135c1c130c0114320c4e385a0d0415075f01163543340f06362381133c0c012224180981742048191d110e180e180d310f011a20632450281f043027114b034e0203316a61030482160201360104826e0201380a0484570201610104844d0504814a0201620904820f020164010481630304810403027761040258020165080481550302c9b70204827a0204677367690102660201680104831b020169020483760301720a04814502016a04020c02016b04023405023c030269630504840502016c08048463020170030482670202716d080482000201720304835c03016507024d020174030482520504821503016505048207020175040208030137060481710201760a04833d0201780404832302027a6f080481200202c2ba0204812c0301bd0404821e0205c3a66865730702540202c7890804816f0205cebcc7af730504815c0202d2930202540202db89020481160203e8b8a00304825d0205f0958db331070481620305989b8569780a04813d0204f69299a5020210010174011a7b3829100a4e511f1a281c17140114812626032c372634234c01140a520e815a810815200501123e4f3531042d57615b011a041f3e64070f1f1913274a20770114811d0f5d743e0634161c01162c2782130c1b810520280d01164a513110480b402b810d13011e522d08042c1146137012201e810512011a290903182c05301e5d811944290201390a04836d0301610104836e02016208027803036cc2bc07048261020263640a0481480301730704813602016408022b020165050484310301720702260301780402500203666d73070484470201670104825b02016a040481590304836702016c0304835601025e02016d060484110301340204813702016f04021802017006068336280201710304813507025902017201026508022903016d0604812f0201740304827e03016a010482440201750404834b02017604021a010484150504836a02057773c2b2380602520201780204823e0302cebc080484040303d2956403048171020179020484240204813c0202c2b303048307020484410301be0204816f030484250203c798680104843d03019c030482570204e19ea86a020482350302a0950a0482280204e5a4bc780304810e0101750126090a35030a03220a1731630f31252f0c4b1e31011e39200e3715282a03103b56090f6b1501121d4916246e6d460d6501162609380406361e816d203f011a22166008124f58202e182025150114390f3a25713f0e3f715c011a5a11191123466025710c313312011e3c191326811c1444055f1f5109051201143b106f1181000d068155012043381381020d81080d0603171824260a0201300404844a02013207024203026b390204833803066eeebabb35660604842102013308020302023f02013503048243020436716b66040481440201380a04843d020261690704836002016203048209030484670201630a04841d020264790104822b0201650502340302c2bd060484300201670202620201680704810d020169010484430104843402026b67040481540306eea3ad77c2aa0a04836702016c020258040482270104830a02016d0404824102036e716a0604843e02016f0104832a04020c0204836803036530650a04817402017005025b0301630604843d020171070275020273720404827002017504048133010484120301370102270201760104814f03026203016d0904844802017701048375080481220301660202330301700a04827f020378c2b2030484710202796806023c0301730704813d02017a0404815f0104817202048407010483390202c2b204026802048254040266640204831e0301b3090482230302b963060482010401750a0483290306ba35f2999dac09020f0203ca926e090483350203cdb4780302350202cebc0a020f0204d7a7696d010483100206f097bc996d71040481480101760114185b2258291610821c0e01160272173107154f5b813722011a81020c200e1826250d39811f07011a7911152a2a45131504422c81070120050d3f5b23342e3e4139032a3813042d0116592d1c15630c0c0a814649011c1a362f5c4a35511f0804033e372b01102981262a352e8205010e0b4b6282388106011e26810a2d125f361a12170d1721311e0201300204832b0201320104811b050483790201350204832202016404024c0202657a090483710202666307022703016c07048362030277750604842f0201670304843d0104844c0203686f7a07027202016a0102430204847502046ce0a2b20704810302016d0204827b02026e330804826c02016f0104835502017102027606026c020172060483490202736505048371020174030484130302387602025d02017502048345020376346b0904825c0201770904814b0303c2bd720804812b020178040483600201790804816402017a08026e0202c2b90704811c0301bd0504821e0204cdbcc7a108021a0202cfb8080481490204f09f96a50204842101017701180207232d37812d0c045c4a0a011a06163b3408171c52213a26592201206d08581605811a171e0c0a1347104914011282181324082b73320f01122f6e811d2c3d410a44011e551414206a092f133f333d150a3e0f011e235b170e37060627471b13373b3e27011a0e1e816b270c10102d53381045011a060e1e254d044932651234691e011a158138300a04810c0a8121071802013003027707048214020131080481370201320a04835502013304048271020538ceb369650802310201390a0481440202616c0502570302c7a104026c0201620804811c02016301048168020241020164040481560104820b020165080484460102400202686c060232020269670604827302016a0108810c826e0704824e02026b6c0604816902016c0304831c02016d0304811903016407022b02016e01022e0604845a030237780702040301710804815102016f0104842903017a0704826902017003048445020482080202713002024e0201720804822003016202022302017303048111010482790204812d03067479c39f66700a0210020174020229020175090484430301690604820402017606020202017a01021b0104843a0202c2b90604842b0301bd030484570202c69b0504815d0202c8a30a02240202ceb8090484690301bc0404832f0202df85020481230101780116812b0a16810e4b045a3b2a01205b1305811134092f62072343100f0f05011e5734152612030b4c4134123009361601121781653207780a6a0d01164a25210824138107738139011481341f088158060c8133010e5920193a4c2331011a0510358101231a1b3609702732011a2f07631610033436810256174c011a1342040a58110721378139101602033067750802720201320604834303017109048244020133090481700203366f79010483520201390404810902016301048411050483420201640a048432020265310704832b02026774040481000302d5b2010259020369756c0504832902016c07048365030233700904824102016d010482670404834c0504830c02016e0104831604048120030169090481750303eba6990104835f02016f0604834c030379c2ba0a0481560201700404815e06048256020174040482370201760a04820d0201770604811c02017a0404812e06020b0202c2b9030481660301bd0a04816c0301be030483620206c99b6d7777750304835b0206ceb0646b66610402490202d8bf030482250206e8bfbc626964080482510204f09c9a9e0404830f010179010e2081335661371c01220e4e2718124f0d0649812b0b0a063b040b011402741d1235810805211a011409161d732b8106325f6a01182e330325068107703728302b011e3723081c0d0a3f810c183e061b067f0106834a12011a044030185a1e810704220a0541011245602b0e421441817801144b03811a1a29614e224b02013003048139020132050485050304f09caba6010482230201350104822708048413020138020483430201610402230201620902440108812181070301300304815e020263710404831002016403048226010483660604823c020165010484510201670104816407048418030334c2b20a0481120201690404814c02016a010481500904810f02036b75610304836402016d01027902016f0304817b03056f6373cebc0a0483010201700104817d020171050482680104843b0302383203048128040807090707070b0608060707060c0b810e07080807070707060b080707070b0807090807070a070a07070808070b06090807070708080a0b81230907070b070b0707080907070706090807070807060b07060707070808070a080b0708090b0b09810a0707060908070607060609070b0a070706080a09070707070e0a0708090b0c0b09070a080a811a0706080c09070a07080b0708060806070b080c0e07090e09060706080b060a070b0607090707130b080708070b0908070a0c810d070b0706080707080b080a0a070807090708070707090709070706080709080a81170a0707070a070707070a0b0a07080d080707060a070707070b0707060f0b060707060a08070807080708810d0807070709070b070808070907080f0b070907090b0707070a0807070c0b080c0a810107070a0b07060c07080f070b09070b0906070b070b'); INSERT INTO t1_data VALUES(274877906949,X'00000e5c033079720404826c0404833002021b03026f6b020482100201740904842e010484150303c2b36f0204840e0201760704826b02017708048273020578f48ba5b50a0481400201790904845902017a050483280304c2bac2bd0404841e0203c2aa680904843a0301bc070481100301be0304820304016a050481630202ca8103027b0202cb860604840d0204ceb56e370204832e0302bc740202520202cf8d07027e0205d8ae39c2aa0104813c0204e887b3770404816a0204f1bfb0970204832f01017a0118101e282f07045961813a193e011e69162f0d2b051c060f084460063053011e06810c20330d0733815c220515220c011210290a7e07810c3a18011a1f2f064a19155212472781047c010a123e45825501166c6062182718167131092f0112331b812c0b6e81470b01184f3a230d45261e271c36111701140a8128456b291248391a0201300104840d0204812502013401048318020137070483070201380904823b03016704026402013907023403016e060481380201610204812f0404824504045243020162050481150104824003016a0804827a020163070484590204817903017a02048209020164020484200203653077030482610302737601048424020166070481730201670304841e0504840e03016b0504825302016901048235010484400404841002016a010483680404845f02046bc2b97407023302026c7a0704812d02027161070209030378cebc0a021f020172040483750404815f0201730304813c030131020482040201740704824d0302793901027502017604048423020177040620813103048313020179030484540204833c04022503017a0904826302017a050483530203c2b2660404840f0307ba6272f397bd92010483070301be020482760202c6b9050482410203c9a3650604812c0202cbae020482180202d38c0704812f0204f096adb7070483490204f6a69c8b030484390102c2aa010c815e81147969010c811c827f0417010e81077a4c03815f010a2e8100820c010a810d148140010c0a7481201a13010c0b831525323d010a8206358129010c21637a33812701083c8340390301300504820a0304326939770404841c03016107048334030165030484150302696c0704810103016c0704812a03027178090481620301730504830e0301750504846303017802021c0301790304836803017a0604834e040135030482650302cdbf010483330304ceb23169070484090303e4849a0504817d0201b201088219744601082210832c010629846001121c8137182211816232010a81668122690108817281080110816a433581102f3e010e7481001b3481190106814e1b010e3646823a810e070301310504840b030161010481720301660604822f0301680804821003016a0704844803016b0104845b03016c0504823403026f700802340301710804826004026d3106024a03017509068458070301770a04836c04016b01021f0301790a0482790302c2b20a02270401b3040482130303ceae6e010484330302df9e040482100201b3010881656e750106820b50010e81434a27048153010a812b068122010843810c0401048211010a50817d812e010c811a8163810801061e832c010c811f81418101030163030481210301640504831102048104030482440301680a048202040271300404841303066b6576cebc6a0104843603026f7a08026d0303756a7808023e030176040483610301770704814403027973020484360302c2b906020c0401bc010484490401bd0804835c0302d38c0304846f0304eaae96750a0484020201b9010c3b824018322d010e0468315a5c817901067d583401060b810e010865118169010c0e07826b813a01066f8265010c3b8239633852010e61161e7030821b01068241210301350804843d0301620302480301690402610403e7a1910704812103016d080269030673c2bac2b36c090483070301760504821d02048454030278350702620302c2ba010483320304ca897a750804813b0401b5070482350303cebc6d050483120302d199020483020402ad660604840c0302d2a1010482550302daa30202340201ba01026d01085119826a010c2f5e82008110010882028221010481090108821b812d01068128460208810b8264010c810971812d440301310304820e0301320504825e05048221030333756d010482720301340502700301640604813d03016607020c0301690204823a03016b0104812203016c0704842103016d0304835f03016e0a04813c03016f0304834d030173060482660401710604810f0301740104843c0303d48174040482690201bc010482690108813c83290114090f816045242e148111010a810616822701048456010e2c81167a638115010a3a3b83204001067a8367010c8114127a2265010a824f7f5c230301300604833e0302616f020483560301640104827d090481280301650502150301660804841a03016804048221030169030481120301720104827803017606023d04016e020481700302c2aa0904836c0401ba090484100201bd01088240224f010843813674020a81185068620106822a44010e14154a8101825e010c19814a1b826c010c81221f81651b010a815a4d812c01082d7281160301300804843c030231300304836703013509048353030365c2bc0304814d0302667504027a0301670604831503016c09025d03016d03048178030172020483620301730104816b0402c2b30404823203017401027204016a08048451030175090237040177090481690301770302190301780a04823d03027a350a0481260302c2aa060481620401b3090481090304f098a1a30a04825d0201be010a0d730b816f010c821d81236c2b010c6f8208358119010681236801087d4e831e0108823c8235010667794b010683165e010e05812e3d3c820d010c81148123817403043531c2ba0204824c03083875c2bceba7957a03020803016302020804036dcebc0304814e0301640304843704033379790504823203016b08023603016d070481090304825804013707027a03016f080484530301710a04842203017809025e0304c2aa6135010481460401b90a04836b0401bc0704814f0306c6ab66e3afb9080482660103c39f350304821f0201b0050481110301680602030201b8040481310604810403016c0a024803017108024b0201be06048323030482650301300504812d0304f48990ae060482740103c491680a0483700301760a0481290201a7010484090202b177020482380201b3030482280102c5800602090301360104826a02018202048173040483780301370104814102018b090482730201930504810f0102c680020483790404830c020183020228020185030484400704820b02028d7a0a04821c0201920404835e02019507048437010484070301760204836102019a020214080483100303d795680504820602019e030481050201a30404820a05048436030177070484290201a50704843b0201a8060482690304816f03027735020483630202ab79050484430201b6060481050201b904024003016d070481130201bd090484350201bf06048361030163020481580102c781040484560201830604833a020286650404842a0301750a04821a020189070483370303e0b9b30904836602018c0104816c04020602021d030171050484570201960a04842d03016b0204821903016c04021e03016e0204845002019a0a04844402019d0704835103013103021702039f77750704827e0201a1090483340201a301022b0404831d0304616363750502520302cfb2040483720201a50a0481530202ad760902310201af0104811d0908816281390301640404843c0201b3080483240201bb030483480201bd040484290304840a0203bf646a0404843f03017a040482450102c89d05020f0201a1090483550201a3060483550201ab0904813203017a04024a0201ad030482210201b10204824302025d0303656577030483600202b46e010482240201b6090484410204bce39f9d020484570102c9870604813202018b0804843603017a06027102018d080481130204840c02018f0104836309026202029164020484430201930504816d0201970304843402039c6c73070483700201a0070483470201a203048262030364d7960a0484010201a80304832a060484390202a96a080481170303c69532090481440201ac040214020481180202ae770604832c0201b0090482300201b40804845a0201b5070482110302716f010482160201ba0504837b0201bb02048149030165050483250201bc07088105833002067f824d0302d19f0402330201bd04024d03017a0904814f0102ca800a048321020181040481580201820604821e020383693707048417020287630704840002018a0904834c0104835c02018b06022f02018c0704814702018d0704810a0104831a020490e7b38308048423020191020482510201920304832f030135030483710201950304833207026802019902026b02019b0904832a02019d09024c02019f0802210201a4010484080202a5710a0481680203a777670804831e0301790902100201ae0402700104834e0201af040484650202b673080484190201b705048221010483520201b808048465030167010484400104cb81cab90104834802018602048317020689777568cf920804842602018b0404817b020191050484200202a0720304833e0201a3040625827f0201a4030485080102cdb1070881408205020483700201b30404840e0202b735080482640201b80504815a0201b90a0482370206bb31cebc6c730304842f030133070483010103ce80370304826d0201900202770201ae0504844e0201b1050484080201b20904815f0201b3040483430201b40702600201b50804823b0201b90304846e0302c9a1070483300201ba0404844d0104847e0201bb050482780201bc010882378223010e811105814d8134010a8142823f2d0106811c52010a6e1814816b02088168824b010a438137812d011019060c6b812f811c010a81314e811b03033366660404825503013502048263030238620904820703016303048120040f080b0907070b07070a0907070707080a07070b0a0a81060b0707070606070f0b070b07070908070b070f0b090807080b07070707070c0e0707090d07080908080a0a50070a0707080708070706070707080a094d07070707070707070707080706070707090844070f07080c0708070708070707080a4707060609060c0b07080a07090808080737070b09060706070707070707070707094807080b0607070707060708074107080709070706070707080607060706070808070a460a0d06090709060b060707060a07070c0907060b06060b070a090707080707070b0707070c060b08070b070a09070b07070b08080706070707070807080707090d070707060707070609070a090807070d0707070b09070707070706070a0908070a0807060b0a080707090707090b08090a08070707080707070e07060708070709080b06070b0a0707070a06070606070809060a07080b07070a070c07070808070e070807070c07090607070707060707080b0743090708'); @@ -6976,7 +6975,7 @@ COMMIT; do_catchsql_test 51.1 { SELECT max(rowid)==0 FROM t1('e*'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -8752,7 +8751,7 @@ do_test 60.0 { do_catchsql_test 60.2 { SELECT (matchinfo(t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ' -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- do_test 61.0 { @@ -8958,7 +8957,6 @@ do_catchsql_test 61.2 { SELECT * FROM t3 ORDER BY rowid; } {/*malformed database schema*/} -breakpoint #------------------------------------------------------------------------- do_test 62.0 { sqlite3 db {} @@ -9774,7 +9772,7 @@ do_test 66.0 { do_catchsql_test 66.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -10108,7 +10106,7 @@ do_test 68.0 { do_catchsql_test 68.1 { PRAGMA reverse_unordered_selects=ON; INSERT INTO t1(t1) SELECT x FROM t2; -} {1 {database disk image is malformed}} +} {1 {fts5: corruption on page 1, segment 1, table "t1"}} #------------------------------------------------------------------------- reset_db @@ -10324,7 +10322,7 @@ do_test 69.0 { do_catchsql_test 69.2 { SELECT * FROM t1 WHERE a MATCH 'fx*' -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10507,7 +10505,7 @@ do_test 71.0 { do_catchsql_test 71.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10634,9 +10632,9 @@ do_catchsql_test 72.1 { INSERT INTO ttt(ttt) VALUES('integrity-check'); } {1 {database disk image is malformed}} -do_catchsql_test 72.1 { +do_catchsql_test 72.2 { SELECT 1 FROM ttt('e* NOT ee*'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10762,12 +10760,13 @@ do_test 73.0 { do_catchsql_test 73.1 { SELECT snippet(ttt,ttt, NOT 54 ), * FROM ttt('e* NOT ee*e* NOT ee* NOT ee*e* NOT e*') ; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db do_test 74.0 { sqlite3 db {} + sqlite3_fts5_register_matchinfo db db deserialize [decode_hexdb { | size 106496 pagesize 4096 filename x.db | page 1 offset 0 @@ -14587,14 +14586,19 @@ do_test 74.0 { | end x.db }]} {} -do_catchsql_test 74.1 { - SELECT rowid, quote(matchinfo(t1,'p�xyb... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 07 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 02 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 22 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 01 f3 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 b3 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 0f .............6.. +| 3168: f0 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 03 05 07 1b 88 80 80 80 80 0a 03 00 3c dd 00 00 ............<... +| 3200: 18 c2 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 90 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 0c 52 02 30 31 02 06 01 01 04 01 01 32 4...R.01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 00 f1 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 74 .....3eac......t +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 11 02 70 62 01 02 05 04 09 18 84 80 ......pb........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 0d c0 00 16 12 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 5b 01 04 04 06 1b 84 80 80 80 ..f...[......... +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 01 f8 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 02 4e 01 03 34 1e .....4t ...N..4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 19 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 60 82 16 08 04 01 2i.......2`..... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 83 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 g.z.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end crash-c4a4c5492615bd.db +}]} {} + + +do_catchsql_test 83.1 { + SELECT * FROM t1('R*R*R*R*R*R*R*R*') WHERE (a,b)<=(current_date,0 BETWEEN 'a'<>11 AND '') ORDER BY rowid DESC; +} {/.*fts5: corruption found/} + +#------------------------------------------------------------------------- +reset_db +do_test 84.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 53248 pagesize 4096 filename c1a.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0d .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 0d 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 05 00 00 00 02 0f f1 00 00 00 00 0c 0f fb 0f f1 ................ +| 4064: 00 00 0b 01 03 00 1c 81 3a 84 5e 81 3a 81 3a 0a ........:.^.:.:. +| 4080: 0a 00 00 00 0b 84 80 80 80 80 01 00 00 00 0a 0a ................ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 10 offset 36864 +| 0: 0d 00 00 00 02 0f e2 00 0f e2 0f ef 00 00 00 00 ................ +| 4064: 00 00 0b 01 03 00 1c 81 3a 84 5e 81 3a 81 3a 0f ........:.^.:.:. +| 4080: 0a 03 00 24 00 00 00 00 01 01 02 00 01 01 01 09 ...$............ +| page 11 offset 40960 +| 0: 0d 00 00 00 01 00 22 00 00 22 00 00 00 00 00 00 ................ +| 32: 00 00 9f 56 84 80 80 80 80 01 04 00 bf 30 00 00 ...V.........0.. +| 48: 0f 58 02 30 30 19 02 05 01 02 05 01 02 05 16 02 .X.00........... +| 64: 05 01 02 05 01 02 05 61 02 05 01 02 05 01 02 05 .......a........ +| 80: 13 02 05 01 02 05 01 02 05 0d 02 03 01 02 03 01 ................ +| 96: 02 03 02 09 78 66 66 66 66 66 66 66 65 81 17 02 ....xfffffffe... +| 112: 05 01 02 05 01 02 05 01 01 31 04 02 04 01 02 04 .........1...... +| 128: 01 02 04 01 02 05 01 02 05 01 02 05 0d 02 06 01 ................ +| 144: 02 06 01 02 06 02 01 30 79 02 04 01 02 04 01 02 .......0y....... +| 160: 04 03 02 30 30 2b 02 05 01 02 05 01 02 05 58 02 ...00+........X. +| 176: 05 01 02 05 01 02 05 01 02 05 01 02 05 01 02 05 ................ +| 192: 16 02 05 01 02 05 01 02 05 05 06 30 30 30 30 30 ...........00000 +| 208: 30 81 0b 02 04 01 02 04 01 02 04 10 02 05 01 02 0............... +| 224: 05 01 02 05 03 02 32 34 76 02 05 01 02 05 01 02 ......24v....... +| 240: 05 02 01 38 07 02 04 01 02 04 01 02 04 01 01 32 ...8...........2 +| 256: 28 02 04 01 02 04 01 02 04 04 02 05 01 02 05 01 (............... +| 272: 02 05 02 01 30 1f 02 05 01 02 05 01 02 05 03 02 ....0........... +| 288: 30 30 10 02 05 01 02 05 01 02 05 6a 02 04 01 02 00.........j.... +| 304: 04 01 02 04 02 08 35 30 30 30 30 30 30 30 81 26 ......50000000.& +| 320: 02 05 01 02 05 01 02 05 01 01 33 07 02 06 01 02 ..........3..... +| 336: 06 01 02 06 81 2c 02 04 01 02 04 01 02 04 02 04 .....,.......... +| 352: 32 37 36 36 81 23 02 05 01 02 05 01 02 05 01 01 2766.#.......... +| 368: 34 13 02 05 01 02 05 01 02 05 02 03 30 39 36 1c 4...........096. +| 384: 02 05 01 02 05 01 02 05 07 02 05 01 02 05 01 02 ................ +| 400: 05 01 03 35 30 30 7f 02 05 01 02 05 01 02 05 04 ...500.......... +| 416: 02 30 30 81 0e 02 06 01 02 06 01 02 06 06 03 30 .00............0 +| 432: 30 30 81 11 02 04 01 02 04 01 02 04 01 05 36 35 00............65 +| 448: 35 33 36 81 1a 02 05 01 02 05 01 02 05 01 04 38 536............8 +| 464: 31 39 32 81 02 02 06 01 02 06 01 02 06 01 05 61 192............a +| 480: 6c 6c 6f 77 01 02 02 01 02 02 01 02 02 02 02 72 llow...........r +| 496: 67 81 08 02 04 01 02 04 01 02 04 02 05 74 6f 6d g............tom +| 512: 69 63 04 02 02 01 02 02 01 02 02 03 06 74 61 63 ic...........tac +| 528: 68 65 64 79 02 03 01 02 03 01 02 03 02 0d 75 74 hedy..........ut +| 544: 6f 63 68 65 63 6b 70 6f 69 6e 74 2b 02 04 01 02 ocheckpoint+.... +| 560: 04 01 02 04 05 06 76 61 63 75 75 6d 0d 02 03 01 ......vacuum.... +| 576: 02 03 01 02 03 01 06 62 69 6e 61 72 79 03 06 01 .......binary... +| 592: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 608: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 624: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 640: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 656: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 672: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 688: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 704: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 720: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 736: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 752: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 768: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 784: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 800: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 816: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 832: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 848: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 864: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 880: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 896: 01 02 02 02 07 79 74 65 63 6f 64 65 37 02 03 01 .....ytecode7... +| 912: 02 03 01 02 03 01 05 63 61 63 68 65 10 02 03 01 .......cache.... +| 928: 02 03 01 01 03 02 04 6c 61 6e 67 07 02 03 01 02 .......lang..... +| 944: 03 01 02 03 02 05 6f 6c 75 6d 6e 7c 02 03 01 02 ......olumn|.... +| 960: 03 01 02 03 03 06 6d 6d 65 6e 74 73 43 02 04 01 ......mmentsC... +| 976: 02 04 01 02 04 04 05 70 69 6c 65 72 07 02 02 01 .......piler.... +| 992: 02 02 01 02 02 05 04 6f 75 6e 64 7f 02 03 01 02 .......ound..... +| 1008: 03 01 02 03 03 03 75 6e 74 81 17 02 04 01 02 04 ......unt....... +| 1024: 01 02 04 02 05 75 72 73 6f 72 3a 02 03 01 02 03 .....ursor:..... +| 1040: 01 02 03 01 06 64 62 70 61 67 65 3d 02 03 01 02 .....dbpage=.... +| 1056: 03 01 02 03 03 04 73 74 61 74 40 02 03 01 02 03 ......stat@..... +| 1072: 01 02 03 02 04 65 62 75 67 0a 02 02 01 02 02 01 .....ebug....... +| 1088: 02 02 03 05 66 61 75 6c 74 0d 02 02 01 02 02 01 ....fault....... +| 1104: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1120: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1136: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1152: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1168: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1184: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1200: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 4f 02 ..............O. +| 1216: 03 01 02 03 01 02 03 03 03 70 74 68 81 05 02 04 .........pth.... +| 1232: 01 02 04 01 02 04 19 02 04 01 02 04 01 02 04 02 ................ +| 1248: 05 69 72 65 63 74 34 02 02 01 02 02 01 02 02 01 .irect4......... +| 1264: 06 65 6e 61 62 6c 65 37 02 02 01 02 02 01 02 02 .enable7........ +| 1280: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1296: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1312: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1328: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1344: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1360: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1376: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1392: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1408: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1424: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1440: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1456: 02 01 02 02 02 06 78 70 6c 61 69 6e 43 02 03 01 ......xplainC... +| 1472: 02 03 01 02 03 04 01 72 81 05 02 03 01 02 03 01 .......r........ +| 1488: 02 03 03 07 74 65 6e 73 69 6f 6e 81 2f 02 04 01 ....tension./... +| 1504: 02 04 01 02 04 01 04 66 69 6c 65 13 02 03 01 02 .......file..... +| 1520: 03 01 02 03 02 05 6f 72 6d 61 74 13 02 04 01 02 ......ormat..... +| 1536: 04 01 02 04 02 03 74 73 33 46 02 03 01 02 03 01 ......ts3F...... +| 1552: 02 03 01 02 03 01 02 03 01 02 03 04 01 34 4c 02 .............4L. +| 1568: 03 01 02 03 01 02 03 04 01 35 4f 02 03 01 02 03 .........5O..... +| 1584: 01 02 03 02 03 75 6e 63 5e 02 05 01 02 05 01 02 .....unc^....... +| 1600: 05 05 04 74 69 6f 6e 73 02 05 01 02 05 01 02 05 ...tions........ +| 1616: 13 02 03 01 02 03 01 02 03 09 01 73 55 02 04 01 ...........sU... +| 1632: 02 04 01 02 04 01 07 67 65 6f 70 6f 6c 79 52 02 .......geopolyR. +| 1648: 03 01 02 03 01 02 03 01 05 68 69 6e 74 73 3a 02 .........hints:. +| 1664: 04 01 02 04 01 02 04 02 03 6f 6f 6b 61 02 04 01 .........ooka... +| 1680: 02 04 01 02 04 01 02 69 6e 01 02 04 01 02 04 01 .......in....... +| 1696: 02 04 03 04 69 74 73 7a 1f 02 04 01 02 04 01 02 ....itsz........ +| 1712: 04 03 08 74 72 69 6e 73 69 63 73 04 02 03 01 02 ...trinsics..... +| 1728: 03 01 02 03 01 07 6a 6f 75 72 6e 61 6c 16 02 03 ......journal... +| 1744: 01 02 03 01 02 03 01 06 6c 65 6e 67 74 68 81 0b ........length.. +| 1760: 02 03 01 02 03 01 02 03 01 02 05 01 02 05 01 02 ................ +| 1776: 05 0d 02 04 01 02 04 01 02 04 02 03 69 6b 65 81 ............ike. +| 1792: 0e 02 03 01 02 03 01 02 03 03 03 6d 69 74 16 02 ...........mit.. +| 1808: 05 01 02 05 01 02 05 5e 02 04 01 02 04 01 02 04 .......^........ +| 1824: 02 03 6f 61 64 81 2f 02 03 01 02 03 01 02 03 01 ..oad./......... +| 1840: 06 6d 61 6c 6c 6f 63 76 02 02 01 02 02 01 02 02 .mallocv........ +| 1856: 3a 02 03 01 02 03 01 02 03 03 02 74 68 55 02 03 :..........thU.. +| 1872: 01 02 03 01 02 03 03 01 78 79 02 02 01 02 02 01 ........xy...... +| 1888: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1904: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1920: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1936: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1952: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1968: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1984: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 2000: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 2016: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 2032: 02 02 02 05 65 6d 6f 72 79 81 11 02 03 01 02 03 ....emory....... +| 2048: 01 02 03 04 04 73 79 73 35 58 02 03 01 02 03 01 .....sys5X...... +| 2064: 02 03 02 03 6d 61 70 19 02 03 01 02 03 01 02 03 ....map......... +| 2080: 79 02 03 01 02 03 01 02 03 02 04 75 74 65 78 81 y..........utex. +| 2096: 2c 02 02 01 02 02 01 02 02 01 06 6e 6f 63 61 73 ,..........nocas +| 2112: 65 02 06 01 02 02 03 06 01 02 02 03 06 01 02 02 e............... +| 2128: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2144: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2160: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2176: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2192: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2208: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2224: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2240: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2256: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2272: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2288: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2304: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2320: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2336: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2352: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2368: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2384: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2400: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2416: 02 02 03 06 01 02 02 03 07 72 6d 61 6c 69 7a 65 .........rmalize +| 2432: 5b 02 03 01 02 03 01 02 03 02 05 75 6d 62 65 72 [..........umber +| 2448: 81 23 02 04 01 02 04 01 02 04 01 06 6f 66 66 73 .#..........offs +| 2464: 65 74 5e 02 03 01 02 03 01 02 03 02 03 6d 69 74 et^..........mit +| 2480: 81 2c 02 03 01 02 03 01 02 03 01 02 02 01 02 02 .,.............. +| 2496: 01 02 02 02 01 70 81 26 02 04 01 02 04 01 02 04 .....p.&........ +| 2512: 02 07 76 65 72 66 6c 6f 77 34 02 03 01 02 03 01 ..verflow4...... +| 2528: 02 03 01 04 70 61 67 65 1c 02 03 01 02 03 01 02 ....page........ +| 2544: 03 64 02 04 01 02 04 01 02 04 13 02 03 01 02 03 .d.............. +| 2560: 01 02 03 01 02 03 01 02 03 01 02 03 03 09 72 65 ..............re +| 2576: 6e 74 68 65 73 69 73 49 02 04 01 02 04 01 02 04 nthesisI........ +| 2592: 03 05 74 74 65 72 6e 81 0e 02 04 01 02 04 01 02 ..ttern......... +| 2608: 04 02 05 63 61 63 68 65 1f 02 03 01 02 03 01 02 ...cache........ +| 2624: 03 02 08 72 65 75 70 64 61 74 65 61 02 03 01 02 ...reupdatea.... +| 2640: 03 01 02 03 01 04 72 65 61 64 34 02 04 01 02 04 ......read4..... +| 2656: 01 02 04 03 07 63 75 72 73 69 76 65 22 02 03 01 .....cursive.... +| 2672: 02 03 01 02 03 02 04 6f 77 69 64 01 02 03 01 02 .......owid..... +| 2688: 03 01 02 03 02 04 74 72 65 65 64 02 03 01 02 03 ......treed..... +| 2704: 01 02 03 04 02 69 6d 01 06 01 02 02 03 06 01 02 .....im......... +| 2720: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2736: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2752: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2768: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2784: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2800: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2816: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2832: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2848: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2864: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2880: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2896: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2912: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2928: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2944: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2960: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2976: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2992: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3008: 01 02 02 03 06 01 02 02 03 06 01 02 02 01 0a 73 ...............s +| 3024: 63 61 6e 73 74 61 74 75 73 70 02 04 01 02 04 01 canstatusp...... +| 3040: 02 04 02 05 65 63 74 6f 72 25 02 03 01 02 03 01 ....ector%...... +| 3056: 02 03 03 04 6c 65 63 74 7f 02 04 01 02 04 01 02 ....lect........ +| 3072: 04 03 05 73 73 69 6f 6e 67 02 03 01 02 03 01 02 ...ssiong....... +| 3088: 03 02 03 69 7a 65 10 02 04 01 02 04 01 02 04 04 ...ize.......... +| 3104: 02 04 01 02 04 01 02 04 01 02 04 01 02 04 01 02 ................ +| 3120: 04 01 02 04 01 02 04 01 02 04 07 02 04 01 02 04 ................ +| 3136: 01 02 04 5b 02 05 01 02 05 01 02 05 10 02 04 01 ...[............ +| 3152: 02 04 01 02 04 04 02 04 01 02 04 01 02 04 02 03 ................ +| 3168: 6f 66 74 76 02 03 01 02 03 01 02 03 02 02 71 6c oftv..........ql +| 3184: 5e 02 04 01 02 04 01 02 04 13 02 04 01 02 04 01 ^............... +| 3200: 02 04 28 02 03 01 02 03 01 02 03 02 04 74 61 74 ..(..........tat +| 3216: 34 6a 02 03 01 02 03 01 02 03 03 02 6d 74 70 02 4j..........mtp. +| 3232: 03 01 02 03 01 02 03 05 04 76 74 61 62 6d 02 03 .........vtabm.. +| 3248: 01 02 03 01 02 03 03 03 6f 72 65 81 35 02 03 01 ........ore.5... +| 3264: 02 03 01 02 03 02 0a 79 6e 63 68 72 6f 6e 6f 75 .......ynchronou +| 3280: 73 28 02 03 01 02 03 01 02 03 04 02 04 01 02 04 s(.............. +| 3296: 01 02 04 03 04 73 74 65 6d 81 32 02 02 01 02 02 .....stem.2..... +| 3312: 01 02 02 01 04 74 65 6d 70 81 35 02 02 01 02 02 .....temp.5..... +| 3328: 01 02 02 02 06 68 72 65 61 64 73 31 02 04 01 02 .....hreads1.... +| 3344: 04 01 02 04 76 02 04 01 02 04 01 02 04 08 03 61 ....v..........a +| 3360: 66 65 81 38 02 02 01 02 02 01 02 02 02 06 72 69 fe.8..........ri +| 3376: 67 67 65 72 81 20 02 03 01 02 03 01 02 03 08 01 gger. .......... +| 3392: 73 22 02 04 01 02 04 01 02 04 01 07 75 6e 6b 6e s...........unkn +| 3408: 6f 77 6e 73 02 03 01 02 03 01 02 03 01 08 76 61 owns..........va +| 3424: 72 69 61 62 6c 65 81 23 02 03 01 02 03 01 02 03 riable.#........ +| 3440: 02 03 64 62 65 81 26 02 03 01 02 03 01 02 03 02 ..dbe.&......... +| 3456: 03 69 65 77 01 02 05 01 02 05 01 02 05 02 03 74 .iew...........t +| 3472: 61 62 37 02 04 01 02 04 01 02 04 04 02 04 01 02 ab7............. +| 3488: 04 01 02 04 01 02 04 01 02 04 01 02 04 01 03 77 ...............w +| 3504: 61 6c 2b 02 03 01 02 03 01 02 03 01 02 03 01 02 al+............. +| 3520: 03 01 02 03 02 05 6f 72 6b 65 72 31 02 03 01 02 ......orker1.... +| 3536: 03 01 02 03 76 02 03 01 02 03 01 02 03 01 01 78 ....v..........x +| 3552: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3568: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3584: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3600: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3616: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3632: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3648: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3664: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3680: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3696: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3712: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3728: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3744: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3760: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3776: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3792: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3808: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3824: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3840: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3856: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3872: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3888: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3904: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3920: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3936: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3952: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3968: 06 01 01 02 01 06 04 30 15 1e 0c 28 1b 0d 0c 15 .......0...(.... +| 3984: 0c 16 14 16 10 0c 17 0e 0e 0f 11 10 10 0e 10 11 ................ +| 4000: 18 11 82 3e 12 10 0f 10 11 10 0f 0f 10 11 0f 0f ...>............ +| 4016: 81 05 18 10 81 45 11 0d 13 0f 10 17 0c 0c 0e 18 .....E.......... +| 4032: 0c 12 10 0e 0d 0f 13 12 24 0f 17 0f 1a 0d 81 1c ........$....... +| 4048: 11 0f 17 10 82 3e 12 11 11 18 0d 12 2a 14 11 10 .....>......*... +| 4064: 13 0f 12 0f 0f 82 3a 15 10 0f 10 4d 0e 1f 0f 0d ......:....M.... +| 4080: 0f 0f 1e 10 10 1a 0f 12 0c 12 14 0f 0e 20 17 19 ............. .. +| page 12 offset 45056 +| 0: 0d 00 00 00 01 0d f4 00 0d f4 00 00 00 00 00 00 ................ +| 3568: 00 00 00 00 84 04 84 80 80 80 80 02 04 00 88 0c ................ +| 3584: 00 07 02 00 01 01 02 56 06 01 01 02 01 06 01 01 .......V........ +| 3600: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3616: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3632: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3648: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3664: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3680: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3696: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3712: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3728: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3744: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3760: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3776: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3792: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3808: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3824: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3840: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3856: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3872: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3888: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3904: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3920: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3936: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3952: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3968: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3984: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4000: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 4016: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 4032: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4048: 52 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 R............... +| 4064: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4080: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| end c1a.txt.db +}]} {} + +do_catchsql_test 84.1 { + SELECT * FROM t1('R*R*x') ORDER BY rowid DESC; +} {1 {fts5: corruption found reading blob 137438953475 from table "t1"}} + +sqlite3_fts5_may_be_corrupt 0 +finish_test diff --git a/ext/fts5/test/fts5corrupt4.test b/ext/fts5/test/fts5corrupt4.test index b31f4d96e9..0505250c2f 100644 --- a/ext/fts5/test/fts5corrupt4.test +++ b/ext/fts5/test/fts5corrupt4.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt4 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index 16682b1325..4b21a9ff74 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -15,9 +15,9 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5corrupt3 +set testprefix fts5corrupt5 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -237,7 +237,7 @@ do_test 1.0 { do_catchsql_test 1.1 { SELECT * FROM t1('R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -450,7 +450,7 @@ do_test 2.0 { do_catchsql_test 2.1 { SELECT * FROM t1('R*R*R*R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -569,7 +569,7 @@ do_test 3.0 { do_catchsql_test 3.1 { UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thra*T'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -793,6 +793,1131 @@ do_catchsql_test 4.5 { REPLACE INTO t1(rowid,a,b,rowid) VALUES(200,1,2,3); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 5.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 28672 pagesize 4096 filename crash-0c6d3451d11597.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 04 ................ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63 R PRIMARY KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 AR. KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00 ................ +| 16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64 ck.....ft.....nd +| 4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f 00 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00 ................ +| 16: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 10 03 0f d6 00 0f f4 10 e1 0f d6 00 00 ................ +| 16: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00 heck....optim... +| end crash-0c6d3451d11597.db +}]} {} + +do_execsql_test 5.1 { + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); +} +do_catchsql_test 5.4 { + UPDATE t1 SET content=randomblob(500); +} {/.*fts5: corrupt.*/} + +#------------------------------------------------------------------------- +reset_db +do_test 6.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-42fa37b694d45a.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d ...............m +| 112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00 .....N.......... +| 3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05 WITHOUT ROWID[. +| 3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21 Y, sz BLOB)U...! +| 3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65 !.wtablet1_conte +| 3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45 ntt1_content.CRE +| 3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f ATE TABLE 't1_co +| 3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45 ntent'(id INTEGE +| 3824: 52 20 50 52 49 4d 41 52 49 20 4b 45 59 2c 20 63 R PRIMARI KEY, c +| 3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 0)i.......-table +| 3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 t1_idxt1_idx.CRE +| 3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 ATE TABLE 't1_id +| 3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 x'(segid, term, +| 3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 pgno, PRIMARY KE +| 3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 Y(segid, term)) +| 3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 WITHOUT ROWIDU.. +| 3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 ......tablet1_da +| 3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 tat1_data.CREATE +| 3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 TABLE 't1_data' +| 4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 AR. KEY, block B +| 4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c LOB):......ctabl +| 4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54 et1t1CREATE VIRT +| 4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49 UAL TABLE t1 USI +| 4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29 NG fts5(content) +| page 2 offset 4096 +| 0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd f0 00 ................ +| 16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80 .............$.. +| 4032: 80 80 80 01 03 00 4e 00 10 00 1e 06 30 61 62 61 ......N.....0aba +| 4048: 63 6c 01 02 02 04 02 66 74 02 5f 02 04 04 6e 64 cl.....ft._...nd +| 4064: 6f 6e 02 02 02 04 0a 07 05 01 03 00 10 03 03 0f on.............. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f 00 01 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00 ................ +| 4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00 .....abandon.... +| 4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b .abaft.....aback +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f ee 00 0f fa 0f 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03 ................ +| 4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 10 03 0f d6 00 0f 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c ..........rebuil +| 4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63 d...+integrity-c +| 4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00 heck....optim... +| page 8 offset 28672 +| 0: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| end crash-42fa37b694d45a.db +}]} {} + +do_execsql_test 6.1 { + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); +} +do_catchsql_test 6.2 { + UPDATE t1 SET content=randomblob(500) WHERE t1; +} {1 {constraint failed}} + +#------------------------------------------------------------------------- +reset_db +do_test 7.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40 ..........b....@ +| 112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44 ...u.....f.....D +| 128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00 .....b.......... +| 2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74 ..C......utablet +| 2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41 4t4CREATE VIRTUA +| 2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47 L TABLE t4 USING +| 2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c fts5vocab('t2', +| 2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74 'row')C......ut +| 2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56 ablet3t3CREATE V +| 3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20 IRTUAL TABLE t3 +| 3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28 USING fts5vocab( +| 3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17 't1', 'row')V... +| 3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66 ....tablet2_conf +| 3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41 igt2_config.CREA +| 3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e TE TABLE 't2_con +| 3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b fig'(k PRIMARY K +| 3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52 EY, v) WITHOUT R +| 3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62 OWID^...!!...tab +| 3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 let2_contentt2_c +| 3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41 ontent.CREATE TA +| 3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27 BLE 't2_content' +| 3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d (id INTEGER PRIM +| 3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c ARY KEY, c0, c1, +| 3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62 c2)i.......-tab +| 3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43 let2_idxt2_idx.C +| 3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f REATE TABLE 't2_ +| 3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d idx'(segid, term +| 3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 , pgno, PRIMARY +| 3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 KEY(segid, term) +| 3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 ) WITHOUT ROWIDU +| 3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f ........tablet2_ +| 3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41 datat2_data.CREA +| 3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74 TE TABLE 't2_dat +| 3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 a'(id INTEGER PR +| 3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b IMARY KEY, block +| 3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74 BLOB)X........t +| 3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56 ablet2t2CREATE V +| 3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20 IRTUAL TABLE t2 +| 3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b USING fts5('a',[ +| 3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f b],.c.,detail=no +| 3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 ne,columnsize=0) +| 3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06 configt1_config. +| 3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81 UT ROWID[...!!.. +| 3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 .tablet1_docsize +| 3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54 t1_docsize.CREAT +| 3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61 BLOB)^...!!...ta +| 3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f blet1_contentt1_ +| 3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54 content.CREATE T +| 3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74 ABLE 't1_content +| 3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 '(id INTEGER PRI +| 3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 MARY KEY, c0, c1 +| 3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61 , c2)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20 USING fts5(a,b +| 4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65 unindexed,c,toke +| 4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63 nize=.porter asc +| 4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29 ii.,tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c ..h............| +| 16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .*.............. +| 3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00 .......0........ +| 3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80 ..........7..... +| 3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01 ...t.....0a..... +| 3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01 b.....c.....g... +| 3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01 ....h.......i... +| 3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00 ............*... +| 3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80 ............%... +| 3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02 .....P.....0g... +| 3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02 .....h........i. +| 4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03 ........7....... +| 4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01 .t.....0a.....b. +| 4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02 ....c.....g..... +| 4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04 ..h.......i..... +| 4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00 ................ +| 4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 3 offset 8192 +| 0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c ................ +| 4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 5 offset 16384 +| 0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03 ................ +| 4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03 ................ +| page 6 offset 20480 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 7 offset 24576 +| 0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00 ................ +| 3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84 ..............A. +| 4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61 ............4.0a +| 4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01 .....b.....c.... +| 4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01 .d...e...f...g.. +| 4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06 ...h.....i...... +| 4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f ................ +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 8 offset 28672 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 9 offset 32768 +| 0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03 ................ +| 4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67 .....a b cg h ig +| 4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69 h i.......g h i +| 4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17 a b cg h i...... +| 4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69 .a b cd e fg h i +| page 10 offset 36864 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d8b4a99207c10b.db +}]} {} + +do_catchsql_test 7.1 { + SELECT snippet(t1, -1, '.', '..', '[', ']'), + highlight(t1, 2, '[', ']') + FROM t1('g + h') + WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db +do_test 8.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40 ...............@ +| 112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00 ...h............ +| 3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f V.......tablet1_ +| 3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05 configt1_config. +| 3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41 _config'(k PRIMA +| 3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f RY KEY, v) WITHO +| 3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81 UT ROWIDk...!!.. +| 3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65 !tablet1_docsize +| 3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54 t1_docsize.CREAT +| 3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73 E TABLE 't1_docs +| 3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20 ize'(id INTEGER +| 3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20 PRIMARY KEY, sz +| 3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54 BLOB, origin INT +| 3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61 EGER)i.......-ta +| 3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03 blet1_idxt1_idx. +| 3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 _idx'(segid, ter +| 3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 m, pgno, PRIMARY +| 3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d KEY(segid, term +| 3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 )) WITHOUT ROWID +| 3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 U........tablet1 +| 3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45 _datat1_data.CRE +| 3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61 ATE TABLE 't1_da +| 3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 ta'(id INTEGER P +| 3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 RIMARY KEY, bloc +| 3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b k BLOB)g.......; +| 4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20 tablet1t1CREATE +| 4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31 VIRTUAL TABLE t1 +| 4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62 USING fts5(a, b +| 4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f , content='', co +| 4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65 ntentless_delete +| 4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29 =1, tokendata=1) +| page 2 offset 4096 +| 0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00 .............1.. +| 16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00 ...........6.... +| 3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00 ................ +| 3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01 ..).........V... +| 3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02 ..0aaa.......... +| 3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02 .............. . +| 3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01 ................ +| 3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04 @............... +| 3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 ................ +| 3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62 ......bc.......b +| 3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01 bb.............. +| 3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06 .......... ..... +| 3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06 ............@... +| 3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 ................ +| 3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63 ...............c +| 3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04 d.......ccc..... +| 3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a ................ +| 3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a ... ............ +| 3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64 ......de.......d +| 3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01 dd.............. +| 3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01 ...........ef... +| 3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08 ....eee......... +| 4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66 ......fg.......f +| 4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01 Vf..!..!B..gh... +| 4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06 ....ggg..A..hi.. +| 4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c .......P........ +| 4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22 .........w...... +| 4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 4 offset 12288 +| 0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0 ................ +| 16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01 ................ +| 4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01 ......?......... +| 4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01 ................ +| 4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| end crash-d57c01958e48ab.db +}]} {} + +do_catchsql_test 8.1 { + SELECT rowid FROM t1('a* NOT ý‘') ; +} {0 {1 2 3 4 5 6 7}} + +#------------------------------------------------------------------------- +reset_db +do_test 9.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 32768 pagesize 4096 filename crash-c76a16c24c8ba6.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08 .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00 (a,b,c)......... +| page 3 offset 8192 +| 0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00 ................ +| 3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18 .....J.......... +| 3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06 ...+.00......... +| 3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30 ...........20160 +| 3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01 609...........4. +| 3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02 ..........5..... +| 3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04 ......0000000... +| 3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06 ........binary.. +| 3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70 ............comp +| 3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64 iler...........d +| 3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04 bstat........... +| 3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65 ebug...........e +| 3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02 nable........... +| 3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02 ......xtension.. +| 3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03 .........fts4... +| 3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01 ........5....... +| 3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03 ....gcc......... +| 3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02 ..eopoly........ +| 3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02 ...json1........ +| 3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03 ...load......... +| 3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05 ..max........... +| 3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04 emory........... +| 3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e sys5...........n +| 3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03 ocase........... +| 3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02 ...omit......... +| 3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03 ..rtree......... +| 3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06 ..im............ +| 3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56 ..threadsafe.W4V +| 3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02 .d.F...vtab..... +| 3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01 ......x......... +| 3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02 ................ +| 3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3936: 01 02 01 06 01 01 10 01 06 01 01 02 01 06 01 01 ................ +| 3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c ................ +| 4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f .D...G.......... +| 4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f D..@.......$Z$$. +| 4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01 ...$............ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 5 offset 16384 +| 0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74 ....$..........t +| 16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5 .a.N./.......... +| 32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5 ...t.[.@.$...... +| 48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb .......h.O.5.... +| 64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a .......x.W.>.$.. +| 3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f ...........$..%. +| 3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49 .THREADSAFE=0XBI +| 3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41 NARY.#..%..THREA +| 3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22 DSAFE=0XNOCASE.. +| 3136: 05 00 25 0f 17 54 48 52 45 41 44 53 31 46 45 3d ..%..THREADS1FE= +| 3152: 30 58 52 64 52 49 4d 1f 21 05 00 33 0f 19 4f 4d 0XRdRIM.!..3..OM +| 3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f IT LOAD EXTENSIO +| 3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f NXBINARY. ..3..O +| 3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 MIT LOAD EXTENSI +| 3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17 ONXNOCASE....3.. +| 3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 OMIT LOAD EXTENS +| 3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19 IONXRTRIM....3.. +| 3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30 MAX MEMORY=50000 +| 3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f 000XBINARY....3. +| 3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 .MAX MEMORY=5000 +| 3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33 0000XNOCASE....3 +| 3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 ..MAX MEMORY=500 +| 3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25 00000XRTRIM....% +| 3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42 ..ENABLE RTREEXB +| 3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17 LE RTREEXNOCASE. +| 3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52 ...%..ENABLE RTR +| 3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45 EEXRTRIM....)..E +| 3440: 4e 41 42 4b 45 20 4d 45 4d 53 59 53 35 58 42 49 NABKE MEMSYS5XBI +| 3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c NARY....)..ENABL +| 3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45 B`-EMSYS5XNOCASE +| 3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45 ....)..ENABLE ME +| 3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25 MSYS5XRTRIM....% +| 3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42 ..ENABLE JSON1XB +| 3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42 INARY....%..ENAB +| 3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17 LE JSON1XNOCASE. +| 3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f ...%..ENABLE JSO +| 3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45 N1XRTRIM....)..E +| 3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49 NABLE GEOPOLYXBI +| 3616: 4e 41 52 59 1a 11 05 00 39 0f 19 45 4e 41 42 4c NARY....9..ENABL +| 3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45 E GEOPOLYXNOCASE +| 3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45 ....)..ENABLE GE +| 3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23 OPOLYXRTRIM....# +| 3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49 ..ENABLE FTS5XBI +| 3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c NARY....#..ENABL +| 3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05 E FTS5XNOCASE... +| 3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58 .#..ENABLE FTS5X +| 3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42 RTRIM....#..ENAB +| 3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b LE FTS4XBINARY.. +| 3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34 ..#..ENABLE FTS4 +| 3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e XNOCASE....#..EN +| 3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e ABLE FTS4XRTRIM. +| 3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e TAT VTABXBINARY. +| 3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d TAT VT$.HNOCASE. +| 3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53 ...1..ENABLE DBS +| 3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06 TAT VTABXRTRIM.. +| 3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52 .....DEBUGXBINAR +| 3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f Y.......DEBUGXNO +| 3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47 CASE.......DEBUG +| 3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d XRTRIM'...C..COM +| 3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20 PILER=gcc-5.4.0 +| 4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27 20160609XBINARY' +| 4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3c 67 ...C..COMPILER 10; +} {X'0000001A04306162630102020101620202020101640206030303040806'} + +do_execsql_test 2.2 { + UPDATE t1_data SET + block=X'0000001A04306162630102025501620202020101640206030303040806' + WHERE id>10 +} + +do_catchsql_test 2.3 { + DELETE FROM t1 WHERE rowid = 1 +} {/.*fts5: corrupt.*/} + finish_test diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test new file mode 100644 index 0000000000..471a1b0e39 --- /dev/null +++ b/ext/fts5/test/fts5corrupt8.test @@ -0,0 +1,147 @@ +# 2024 Aug 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt8 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} + +do_execsql_test 1.1 { + UPDATE t1_data SET block='hello world' WHERE id=10 +} + +db close +sqlite3 db test.db + +do_catchsql_test 1.2 { + SELECT * FROM t1 +} {1 {fts5: corrupt structure record for table "t1"}} +do_catchsql_test 1.3 { + DROP TABLE t1 +} {0 {}} +do_execsql_test 1.4 { + SELECT * FROM sqlite_schema +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} +do_execsql_test 2.1 { + UPDATE t1_config SET v=555 WHERE k='version' +} +db close +sqlite3 db test.db +do_catchsql_test 2.2 { + SELECT * FROM t1 +} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}} +do_catchsql_test 2.3 { + DROP TABLE t1 +} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}} +do_test 2.4 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_execsql_test 2.5 { + SELECT * FROM sqlite_schema +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} +do_execsql_test 3.1 { + DELETE FROM t1_config; +} +db close +sqlite3 db test.db +do_catchsql_test 3.2 { + SELECT * FROM t1 +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} +do_catchsql_test 3.3 { + DROP TABLE t1 +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} + + +do_test 3.4 { + sqlite3_db_config db DEFENSIVE 1 +} {1} +do_test 3.5 { + sqlite3_fts5_drop_corrupt_table db main t1 +} {} +do_test 3.6 { + sqlite3_db_config db DEFENSIVE -1 +} {1} +do_execsql_test 3.7 { + SELECT * FROM sqlite_schema +} + +#------------------------------------------------------------------------- +reset_db + +proc hex_to_blob {hex} { + binary encode hex $hex +} +db func hex_to_blob hex_to_blob + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1); + BEGIN; + INSERT INTO x1(rowid, x) VALUES(1, 'a b c d e f g h'); + INSERT INTO x1(rowid, x) VALUES(2, 'a b c d e f g h'); + COMMIT; + DELETE FROM x1 WHERE rowid=1; +} + +do_execsql_test 4.1 { + SELECT hex(block) FROM x1_data WHERE id=10 +} { + 00000000FF00000101010200010101010101010102 +} + +do_execsql_test 4.2.1 { + UPDATE x1_data SET block= + X'00000000FF00000101010200010101010101819C9B95A8000102' + WHERE id=10; +} + +do_catchsql_test 4.2.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +do_execsql_test 4.3.1 { + UPDATE x1_data SET block= + X'00000000FF000001010102000101010101019282AFF9A0000102' + WHERE id=10; +} + +do_catchsql_test 4.3.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +do_execsql_test 4.4.1 { + UPDATE x1_data SET block= + X'00000000FF000001010102000101010101018181808080130102' + WHERE id=10; +} + +do_catchsql_test 4.3.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +finish_test + diff --git a/ext/fts5/test/fts5corruptbig.test b/ext/fts5/test/fts5corruptbig.test new file mode 100644 index 0000000000..6019f17eee --- /dev/null +++ b/ext/fts5/test/fts5corruptbig.test @@ -0,0 +1,53 @@ +# 2025 October 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on really large position lists. Those that require +# 4 or 5 byte position-list size varints. Because of the amount of memory +# required, these tests only run on 64-bit platforms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corruptbig + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if { $tcl_platform(wordSize)<8 } { + finish_test + return +} + +if { $SQLITE_MAX_LENGTH!=0x7FFFFFFF } { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} + +do_execsql_test 1.1 { + UPDATE t1_data SET block = zeroblob(2147483640) WHERE id=10; +} + +do_execsql_test 1.2 { + SELECT id, length(block) FROM t1_data +} {1 0 10 2147483640} + +do_catchsql_test 1.3 { + SELECT * FROM t1('abc') +} {1 {out of memory}} + +finish_test + diff --git a/ext/fts5/test/fts5delete.test b/ext/fts5/test/fts5delete.test index 1214fec4f4..024f89594c 100644 --- a/ext/fts5/test/fts5delete.test +++ b/ext/fts5/test/fts5delete.test @@ -113,5 +113,58 @@ do_execsql_test 3.4 { INSERT INTO tx(tx) VALUES('integrity-check'); } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, + content='', contentless_unindexed=1 + ); + CREATE VIRTUAL TABLE ft2 USING fts5(a, b UNINDEXED, + content='', contentless_unindexed=1, contentless_delete=1 + ); + + INSERT INTO ft1(rowid, a, b) VALUES(1, 'one', 'i'); + INSERT INTO ft1(rowid, a, b) VALUES(2, 'two', 'ii'); + INSERT INTO ft1(rowid, a, b) VALUES(3, 'three', 'iii'); + INSERT INTO ft2(rowid, a, b) VALUES(1, 'one', 'i'); + INSERT INTO ft2(rowid, a, b) VALUES(2, 'two', 'ii'); + INSERT INTO ft2(rowid, a, b) VALUES(3, 'three', 'iii'); +} + +do_catchsql_test 4.1 { + DELETE FROM ft1 WHERE rowid=2 +} {1 {cannot DELETE from contentless fts5 table: ft1}} +do_catchsql_test 4.2 { + DELETE FROM ft2 WHERE rowid=2 +} {0 {}} + +do_catchsql_test 4.3 { + INSERT INTO ft1(ft1, rowid, a) VALUES('delete', 2, 'two'); +} {0 {}} +do_catchsql_test 4.2 { + INSERT INTO ft2(ft2, rowid, a) VALUES('delete', 2, 'two'); +} {1 {'delete' may not be used with a contentless_delete=1 table}} + +do_execsql_test 4.3 { + SELECT rowid, * FROM ft1; +} { + 1 {} i + 3 {} iii +} +do_execsql_test 4.4 { + SELECT rowid, * FROM ft2; +} { + 1 {} i + 3 {} iii +} + +do_execsql_test 4.5 { + SELECT * FROM ft1_content +} {1 i 3 iii} + +do_execsql_test 4.6 { + SELECT * FROM ft2_content +} {1 i 3 iii} + finish_test diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 1fb95a9004..db82db1c2b 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5dlidx -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5doclist.test b/ext/fts5/test/fts5doclist.test index 08b773f6f5..5b1becb514 100644 --- a/ext/fts5/test/fts5doclist.test +++ b/ext/fts5/test/fts5doclist.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5doclist -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test index 3ccbd7d7a2..49c2f2753a 100644 --- a/ext/fts5/test/fts5ea.test +++ b/ext/fts5/test/fts5ea.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ea -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index ce40e471aa..bee9683c3c 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5eb -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -86,15 +86,18 @@ do_execsql_test 3.0 { INSERT INTO e1 VALUES ('just a few words with a / inside'); } do_execsql_test 3.1 { - SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; + SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; } {1 -1e-06} do_execsql_test 3.2 { SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"' } 1 do_execsql_test 3.3 { - SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; + SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; } {1 -1e-06} +do_execsql_test 3.4 " + SELECT fts5_expr_tcl('e AND \" \"'); +" {{AND [nearset -- {e}] [{}]}} finish_test diff --git a/ext/fts5/test/fts5expr.test b/ext/fts5/test/fts5expr.test new file mode 100644 index 0000000000..49be61d9c4 --- /dev/null +++ b/ext/fts5/test/fts5expr.test @@ -0,0 +1,52 @@ +# 2024 August 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5expr + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test'); +} + +do_execsql_test 1.1 { + SELECT rowid FROM x1('expr'); +} {113} + +for {set ii 0} {$ii < 300} {incr ii} { + set expr "expr " + append expr [string repeat "NOT abcd " $ii] + + if {$ii<257} { + set res {0 113} + } else { + set res {1 {fts5 expression tree is too large (maximum depth 256)}} + } + do_catchsql_test 1.1.$ii { + SELECT rowid FROM x1($expr) + } $res +} + +do_execsql_test 1.2 { + SELECT rowid FROM x1 WHERE a MATCH '"..."' +} {} + +finish_test + diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index 1d0d5c9b7c..2b4f6c4d2a 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -90,7 +90,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] do_faultsim_test 4 -faults oom-* -body { db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} } -test { - faultsim_test_result {0 {0 {} 3}} + faultsim_test_result {0 {0 {} 2}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index a39063a356..1aacddce9f 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5fault6 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test index 5afab77541..dc060a1592 100644 --- a/ext/fts5/test/fts5fault8.test +++ b/ext/fts5/test/fts5fault8.test @@ -57,7 +57,6 @@ foreach_detail_mode $testprefix { } ;# foreach_detail_mode... - do_execsql_test 4.0 { CREATE VIRTUAL TABLE x2 USING fts5(a); INSERT INTO x2(x2, rank) VALUES('crisismerge', 2); @@ -80,5 +79,18 @@ do_faultsim_test 4 -faults oom-* -prep { faultsim_test_result {0 {}} {1 SQLITE_NOMEM} } +set TMPDBERROR {1 {unable to open a temporary database file for storing temporary tables}} + +do_faultsim_test 5 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA temp_store = memory } +} -body { + execsql { PRAGMA integrity_check } +} -test { + if {[string match {*error code=7*} $testresult]==0} { + faultsim_test_result {0 ok} {1 SQLITE_NOMEM} $::TMPDBERROR + } +} + finish_test diff --git a/ext/fts5/test/fts5faultF.test b/ext/fts5/test/fts5faultF.test new file mode 100644 index 0000000000..96cc2b083f --- /dev/null +++ b/ext/fts5/test/fts5faultF.test @@ -0,0 +1,111 @@ +# 2023 July 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. Particularly those that may occur +# when using contentless_delete=1 databases. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultF + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen +} -body { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) + } +} -test { + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d'); + COMMIT; + DELETE FROM t1 WHERE rowid IN (2, 4); +} + +do_faultsim_test 2 -prep { + sqlite3 db test.db + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + SELECT rowid FROM t1('b c'); + } +} -test { + faultsim_test_result {0 {1 3}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d'); + INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d'); + COMMIT; +} + +faultsim_save_and_close +do_faultsim_test 3 -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d'); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s; +} + +do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 } + +faultsim_save_and_close +do_faultsim_test 4 -faults oom-t* -prep { + faultsim_restore_and_reopen + execsql { SELECT rowid FROM t1 } +} -body { + execsql { + DELETE FROM t1 WHERE rowid < 100 + } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test + diff --git a/ext/fts5/test/fts5faultG.test b/ext/fts5/test/fts5faultG.test new file mode 100644 index 0000000000..9110c6336d --- /dev/null +++ b/ext/fts5/test/fts5faultG.test @@ -0,0 +1,76 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultG + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultG + + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a); + INSERT INTO t1 VALUES('test renaming the table'); + INSERT INTO t1 VALUES(' after it has been written'); + INSERT INTO t1 VALUES(' actually other stuff instead'); +} +faultsim_save_and_close +do_faultsim_test 1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + } +} -body { + execsql { + DELETE FROM t1; + } +} -test { + catchsql { COMMIT } + faultsim_integrity_check + faultsim_test_result {0 {}} +} + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, content=, contentless_delete=1); + BEGIN; + INSERT INTO t1 VALUES('here''s some text'); + INSERT INTO t1 VALUES('useful stuff, text'); + INSERT INTO t1 VALUES('what would we do without text!'); + COMMIT; +} +faultsim_save_and_close +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + DELETE FROM t1 WHERE rowid=2; + } +} -body { + execsql { + INSERT INTO t1(t1) VALUES('optimize'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {}} +} + + + +finish_test diff --git a/ext/fts5/test/fts5faultH.test b/ext/fts5/test/fts5faultH.test new file mode 100644 index 0000000000..0cbbf7f5ef --- /dev/null +++ b/ext/fts5/test/fts5faultH.test @@ -0,0 +1,150 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultG + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultH + +sqlite3_fts5_register_origintext db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + + BEGIN; + INSERT INTO t1 VALUES('oNe tWo thRee'); + INSERT INTO t1 VALUES('One Two Three'); + INSERT INTO t1 VALUES('onE twO threE'); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES('one two three'); + INSERT INTO t1 VALUES('one two three'); + INSERT INTO t1 VALUES('one two three'); + COMMIT; +} + +do_faultsim_test 1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('three'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {1 2 3 4 5 6}} +} + + +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 24}} +} + +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB'); + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 3.1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 35}} +} +do_faultsim_test 3.2 -faults oom* -prep { +} -body { + execsql { + SELECT count(*) FROM t1('BBB'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 27} +} + + +finish_test diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test new file mode 100644 index 0000000000..a2b04af8f5 --- /dev/null +++ b/ext/fts5/test/fts5faultI.test @@ -0,0 +1,349 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultI + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultI + +do_execsql_test 1.0 { + PRAGMA encoding = utf16; + CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1); + INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram'); +} + +faultsim_save_and_close +faultsim_restore_and_reopen + +do_faultsim_test 1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1(fts5_locale('en_US', 'origintext')); + } +} -test { + faultsim_test_result {0 1} +} + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + SELECT * FROM t1('ascii'); + } +} -body { + execsql { + UPDATE t1 SET rowid=rowid+1; + } +} -test { + faultsim_test_result {0 {}} +} + +fts5_aux_test_functions db +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*'); + } +} -test { + faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('token token token'); + } +} -body { + execsql { + INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO w1 VALUES('one'); + SAVEPOINT one; + INSERT INTO w1 VALUES('two'); + ROLLBACK TO one; + } + +} -body { + execsql { + INSERT INTO w1 VALUES('string'); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db + +do_faultsim_test 5 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two'); + } +} -test { + faultsim_test_result {0 two} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); +} +fts5_aux_test_functions db +faultsim_save_and_close + +do_faultsim_test 6 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO w1 VALUES('four five six'); + SAVEPOINT abc; + INSERT INTO w1 VALUES('seven eight nine'); + SAVEPOINT def; + INSERT INTO w1 VALUES('ten eleven twelve'); + } +} -body { + execsql { + RELEASE abc; + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE w1 USING fts5(a); + INSERT INTO w1 VALUES('one two three'); + INSERT INTO w1 VALUES('three two one'); + DELETE FROM w1_content WHERE rowid=1; +} + +faultsim_save_and_close + +do_faultsim_test 7 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM w1 } +} -body { + execsql { + PRAGMA integrity_check; + } +} -test { +} + +#------------------------------------------------------------------------- +reset_db +fts5_tclnum_register db +fts5_aux_test_functions db + +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize = "tclnum query", detail=columns + ); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_faultsim_test 8.1 -faults oom* -prep { +} -body { + execsql { + SELECT fts5_test_collist (ft) FROM ft('one two'); + } +} -test { + faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +do_faultsim_test 8.2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM ft('one two') ORDER BY rank; + } +} -test { + faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + INSERT INTO ft VALUES('one two three i ii iii'); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +faultsim_save_and_close + +do_faultsim_test 9.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE ft SET rowid=4 WHERE rowid=1 + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 9.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 1} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, locale=1); + INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii')); + INSERT INTO ft VALUES('four five six iv v vi'); + INSERT INTO ft VALUES('eight nine ten viii ix x'); +} {} + +do_execsql_test 10.1 { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' +} {hello} + +faultsim_save_and_close +do_faultsim_test 10.1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three' + } +} -test { + faultsim_test_result {0 hello} +} + +breakpoint +faultsim_save_and_close +do_faultsim_test 10.2 -faults oom-t* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO ft VALUES(zeroblob(10000)); + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE f1 USING fts5(content); + CREATE TABLE g1(id, content); + INSERT INTO g1 VALUES(30000, 'a b c'); + INSERT INTO g1 VALUES(40000, 'd e f'); +} + +faultsim_save_and_close + +do_faultsim_test 11 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + INSERT INTO f1(rowid, content) SELECT id, content FROM g1; + } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +reset_db + +ifcapable foreignkey { + do_execsql_test 12.0 { + CREATE VIRTUAL TABLE f1 USING fts5(content); + CREATE TABLE p1(a INTEGER PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 DEFERRABLE INITIALLY DEFERRED); + } + + faultsim_save_and_close + + do_faultsim_test 11 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + PRAGMA foreign_keys = 1; + BEGIN; + INSERT INTO c1 VALUES(123); + SAVEPOINT xyz; + } + } -body { + execsql { + INSERT INTO f1 VALUES('a b c'); + ROLLBACK TO xyz; + COMMIT; + } + } -test { + execsql { SELECT 123 } + faultsim_test_result \ + {1 {FOREIGN KEY constraint failed}} \ + {1 {out of memory}} \ + {1 {constraint failed}} + } +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('abc def', X'123456'); +} +faultsim_save_and_close + + +do_faultsim_test 13 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE t1 SET a='def abc' + } +} -test { + faultsim_test_result {0 {}} +} + +finish_test + diff --git a/ext/fts5/test/fts5first.test b/ext/fts5/test/fts5first.test index 357672de68..492681eed7 100644 --- a/ext/fts5/test/fts5first.test +++ b/ext/fts5/test/fts5first.test @@ -22,6 +22,7 @@ do_execsql_test 1.0 { CREATE VIRTUAL TABLE x1 USING fts5(a, b); } +unset -nocomplain res foreach {tn expr ok} { 1 {^abc} 1 2 {^abc + def} 1 diff --git a/ext/fts5/test/fts5full.test b/ext/fts5/test/fts5full.test index 751e874c3b..76fdc0288f 100644 --- a/ext/fts5/test/fts5full.test +++ b/ext/fts5/test/fts5full.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5full -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5hash.test b/ext/fts5/test/fts5hash.test index 5df55f226f..b3d8b562c8 100644 --- a/ext/fts5/test/fts5hash.test +++ b/ext/fts5/test/fts5hash.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5hash -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 4038830861..4bf120c446 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5integrity -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -37,6 +37,12 @@ do_execsql_test 2.1 { INSERT INTO yy(yy) VALUES('integrity-check'); } +db close +sqlite3 db test.db +do_execsql_test 2.1 { + INSERT INTO yy(yy) VALUES('integrity-check'); +} + #-------------------------------------------------------------------- # do_execsql_test 3.0 { @@ -77,6 +83,9 @@ do_catchsql_test 4.2 { UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3; INSERT INTO aa(aa) VALUES('integrity-check'); } {1 {database disk image is malformed}} +do_execsql_test 4.2.1 { + PRAGMA integrity_check(aa); +} {{malformed inverted index for FTS5 table main.aa}} do_catchsql_test 4.3 { ROLLBACK; @@ -150,6 +159,7 @@ do_execsql_test 5.3 { INSERT INTO gg(gg) VALUES('integrity-check'); } +unset -nocomplain res do_test 5.4.1 { set ok 0 for {set i 0} {$i < 10000} {incr i} { @@ -317,4 +327,92 @@ do_catchsql_test 10.5.3 { INSERT INTO vt0(vt0) VALUES('integrity-check'); } {0 {}} +reset_db +proc slang {in} {return [string map {th d e eh} $in]} +db function slang -deterministic -innocuous slang +do_execsql_test 11.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b))); + INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog'); + SELECT c FROM t1; +} {{deh quick fox jumps ovehr deh lazy brown dog}} + +do_execsql_test 11.1 { + CREATE VIRTUAL TABLE t2 USING fts5(content="t1", c); + INSERT INTO t2(t2) VALUES('rebuild'); + SELECT rowid FROM t2 WHERE t2 MATCH 'deh'; +} {1} + +do_execsql_test 11.2 { + PRAGMA integrity_check(t2); +} {ok} +db close +sqlite3 db test.db + +# FIX ME? +# +# FTS5 integrity-check does not care if the content table is unreadable or +# does not exist. It only looks for internal inconsistencies in the +# inverted index. +# +do_execsql_test 11.3 { + PRAGMA integrity_check(t2); +} {ok} +do_execsql_test 11.4 { + DROP TABLE t1; + PRAGMA integrity_check(t2); +} {ok} + +#------------------------------------------------------------------- +reset_db + +do_execsql_test 12.1 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('one', 'two'); + INSERT INTO x1 VALUES('three', 'four'); + INSERT INTO x1 VALUES('five', 'six'); +} + +do_execsql_test 12.2 { + PRAGMA integrity_check +} {ok} + +db close +sqlite3 db test.db -readonly 1 + +explain_i { + PRAGMA integrity_check + } +do_execsql_test 12.3 { + PRAGMA integrity_check +} {ok} + + +#------------------------------------------------------------------- +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii); + INSERT INTO t1 VALUES('a b c'), ('d e f'); + PRAGMA integrity_check; +} {ok} + +db close +sqlite3 db test.db +do_catchsql_test 13.2 { + PRAGMA integrity_check; +} {0 ok} + +do_execsql_test 13.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)' + WHERE name = 't1'; +} + +db close +sqlite3 db test.db +breakpoint +do_catchsql_test 13.4 { + PRAGMA integrity_check; +} {1 {SQL logic error}} + + finish_test diff --git a/ext/fts5/test/fts5integrity2.test b/ext/fts5/test/fts5integrity2.test new file mode 100644 index 0000000000..968be3bddf --- /dev/null +++ b/ext/fts5/test/fts5integrity2.test @@ -0,0 +1,56 @@ +# 2024 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests focused on the integrity-check procedure. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5integrity2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a, detail='none'); + BEGIN; + INSERT INTO t2(rowid, a) VALUES(-1, 'hello world'); + INSERT INTO t2(rowid, a) VALUES(9223372036854775807, 'hello world'); + COMMIT; +} + +do_execsql_test 2.1 { + SELECT rowid FROM t2('hello AND world'); +} {-1 9223372036854775807} + +#------------------------------------------------------------------------- +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none'); + CREATE TABLE r1(r); + + WITH c(x) AS (VALUES(1) UNION SELECT x<<1 FROM c) + INSERT INTO r1(r) SELECT -1-x FROM c; + + INSERT INTO t1(rowid, a) SELECT r, 'abc' FROM r1; +} + +do_execsql_test 2.1 { + PRAGMA integrity_check; +} {ok} + +do_execsql_test 2.2 { + SELECT rowid FROM t1('abc') ORDER BY +rowid; +} [db eval {SELECT r FROM r1 ORDER BY r}] + + +finish_test diff --git a/ext/fts5/test/fts5interrupt.test b/ext/fts5/test/fts5interrupt.test index ca682852c4..67ef5f7e97 100644 --- a/ext/fts5/test/fts5interrupt.test +++ b/ext/fts5/test/fts5interrupt.test @@ -33,6 +33,7 @@ proc progress_handler {args} { return 0 } +unset -nocomplain res foreach {tn sql} { 1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') } 2 { COMMIT } @@ -64,4 +65,3 @@ foreach {tn sql} { } finish_test - diff --git a/ext/fts5/test/fts5join.test b/ext/fts5/test/fts5join.test new file mode 100644 index 0000000000..e4d3b69b79 --- /dev/null +++ b/ext/fts5/test/fts5join.test @@ -0,0 +1,69 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5join + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE vt USING fts5(x); + INSERT INTO vt VALUES('abc'); + INSERT INTO vt VALUES('xyz'); + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TIMESTAMP); + INSERT INTO t1 VALUES(1, 1), (2, 2); + CREATE INDEX i1 ON t1(b); +} + +# set sqlite_where_trace [expr 0xFFF] + +do_eqp_test 1.1 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid ORDER BY t1.rowid; +} { + QUERY PLAN + |--SCAN t1 + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + +do_eqp_test 1.2 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid AND b>? ORDER BY b LIMIT 10 +} { + QUERY PLAN + |--SEARCH t1 USING COVERING INDEX i1 (b>?) + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + +do_eqp_test 1.3 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid AND b>? +} { + QUERY PLAN + |--SEARCH t1 USING COVERING INDEX i1 (b>?) + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + +do_eqp_test 1.4 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid ORDER BY b +} { + QUERY PLAN + |--SCAN t1 USING COVERING INDEX i1 + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + + +finish_test diff --git a/ext/fts5/test/fts5lastrowid.test b/ext/fts5/test/fts5lastrowid.test index d152a8f09b..75866139d3 100644 --- a/ext/fts5/test/fts5lastrowid.test +++ b/ext/fts5/test/fts5lastrowid.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5lastrowid -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5leftjoin.test b/ext/fts5/test/fts5leftjoin.test index 4ef6a8961b..69a172bd45 100644 --- a/ext/fts5/test/fts5leftjoin.test +++ b/ext/fts5/test/fts5leftjoin.test @@ -40,4 +40,53 @@ do_execsql_test 1.2 { SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc') } {1 abc 2 abc} + +do_execsql_test 1.3 { + DELETE FROM t1; + INSERT INTO t1 VALUES(14); +} + +do_execsql_test 1.4 { + SELECT * FROM vt LEFT JOIN t1 ON vt.rowid = 1; +} { + abc 14 + xyz {} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t0 USING fts5(a,b); + INSERT INTO t0(a,b)VALUES(1,0); + CREATE TABLE t1(x); +} + +do_execsql_test 2.1 { + SELECT * FROM t0 LEFT JOIN t1; +} {1 0 {}} + +breakpoint +do_catchsql_test 2.2 { + SELECT * FROM t0 LEFT JOIN t1 ON t0.b MATCH '1'; +} {1 {no query solution}} + +do_execsql_test 2.3 { + SELECT * FROM t0 LEFT JOIN t1 ON +b MATCH '1'; +} {1 0 {}} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t0 USING fts5(c0, c1); + INSERT INTO t0(c0,c1) VALUES (1,0); +} + +do_catchsql_test 3.1 { + SELECT * FROM t0 + LEFT JOIN ( SELECT 0 AS col_0 ) + ON ((((t0.c1 MATCH '1')AND(CASE WHEN t0.c0 THEN CAST(t0.c1 AS INTEGER) ELSE 1 END)))); +} {1 {no query solution}} + + finish_test diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test new file mode 100644 index 0000000000..e5799fb7fd --- /dev/null +++ b/ext/fts5/test/fts5locale.test @@ -0,0 +1,748 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5locale + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc transform_token {locale token} { + switch -- $locale { + reverse { + set ret "" + foreach c [split $token ""] { + set ret "$c$ret" + } + set token $ret + } + + default { + # no-op + } + } + + set token +} + +proc tcl_create {args} { return "tcl_tokenize" } +proc tcl_tokenize {tflags text} { + set iToken 1 + set bSkip 0 + if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 } + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + incr iToken + if {(($iToken) % ($bSkip + 1))} continue + + set w [transform_token [sqlite3_fts5_locale] $w] + sqlite3_fts5_token $w $iStart $iEnd + } +} + +#------------------------------------------------------------------------- +# Check that queries can have a locale attached to them. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('cba'); +} {} + +do_execsql_test 1.1 { + SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') ); +} {1 abc} + +do_execsql_test 1.2 { + SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); +} {2 cba} + + +#------------------------------------------------------------------------- +# Test that the locale= option exists and seems to accept values. And +# that fts5_locale() values may only be inserted into an internal-content +# table if the locale=1 option was specified. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl); + + CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance); +} + +do_catchsql_test 2.2.1 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2); +} {1 {malformed locale=... directive}} +do_catchsql_test 2.2.2 { + CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111); +} {1 {malformed locale=... directive}} + +do_catchsql_test 2.3 { + INSERT INTO b1(b1, rank) VALUES('locale', 0); +} {1 {SQL logic error}} + +do_execsql_test 2.4.1 { + INSERT INTO b1 VALUES('abc', 'one two three'); +} + +do_execsql_test 2.4.2 { + INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); +} + +do_execsql_test 2.5 { + INSERT INTO b2 VALUES('abc', 'one two three'); +} + +do_catchsql_test 2.6 { + INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six')); +} {1 {fts5_locale() requires locale=1}} + +do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1} +do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {} +do_execsql_test 2.9 { SELECT rowid FROM b1('ruof') } 2 +do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2 + +do_execsql_test 2.11 { + SELECT x, quote(y) FROM b1 +} { + abc {'one two three'} + def {'four five six'} +} + +do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { + {'four five six'} +} + +do_execsql_test 2.13 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.14 { + INSERT INTO b1(b1) VALUES('rebuild'); +} +do_execsql_test 2.15 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.16 { + DELETE FROM b1 WHERE rowid=2 +} +do_execsql_test 2.17 { + INSERT INTO b1(b1) VALUES('integrity-check'); +} + +do_execsql_test 2.18 { + INSERT INTO b1(rowid, x, y) VALUES( + test_setsubtype(45, 76), 'abc def', 'def abc' + ); +} + +#------------------------------------------------------------------------- +# Test the 'delete' command with contentless tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1); + CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance); + + INSERT INTO c1 VALUES('hello world'); + INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.2 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + eerht eno hello owt world +} + +do_execsql_test 3.3 { + INSERT INTO c1(c1, rowid, x) + VALUES('delete', 2, fts5_locale('reverse', 'one two three') ); +} + +do_execsql_test 3.4 { + SELECT DISTINCT term FROM c2 ORDER BY 1 +} { + hello world +} + +#------------------------------------------------------------------------- +# Test that an UPDATE that updates a subset of the columns does not +# magically discard the locale from those columns not updated. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl); + CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance); + + INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def'); + INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello')); +} + +do_execsql_test 4.2 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def ghi olleh +} + +do_execsql_test 4.3 { + UPDATE d1 SET x='jkl' WHERE rowid=2; +} + +do_execsql_test 4.4 { + SELECT DISTINCT term FROM d2 ORDER BY 1 +} { + abc def jkl olleh +} + +do_execsql_test 4.5 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 2 jkl hello +} + +do_execsql_test 4.6 { + UPDATE d1 SET rowid=4 WHERE rowid=2 +} + +do_execsql_test 4.7 { + SELECT rowid, * FROM d1 +} { + 1 abc def + 4 jkl hello +} + +fts5_aux_test_functions db + +do_execsql_test 4.8.1 { + SELECT fts5_test_columntext(d1) FROM d1('jkl') +} {{jkl hello}} +do_execsql_test 4.8.2 { + SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{jkl hello}} + +do_execsql_test 4.9 { + SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello')) +} {{{} reverse}} + +do_execsql_test 4.10 { + SELECT fts5_test_columnlocale(d1) FROM d1 +} { + {{} {}} + {{} reverse} +} + +#------------------------------------------------------------------------- +# Test that if an fts5_locale() value is written to an UNINDEXED +# column it is stored as text. This is so that blobs and other values +# can also be stored as is. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, y UNINDEXED, locale=1, tokenize=tcl + ); + + INSERT INTO t1(rowid, x, y) VALUES(111, + fts5_locale('reverse', 'one two three'), + fts5_locale('reverse', 'four five six') + ); +} + +do_execsql_test 5.2 { + SELECT rowid, x, y FROM t1 +} { + 111 {one two three} {four five six} +} + +do_execsql_test 5.3 { + SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content +} { + text text text +} + +#------------------------------------------------------------------------- + +foreach {tn opt} { + 1 {} + 2 {, columnsize=0} +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 6.$tn.1 " + CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt); + " + + do_execsql_test 6.$tn.2 { + INSERT INTO y1(rowid, t) VALUES + (1, fts5_locale('second', 'the city of London')), + (2, fts5_locale('second', 'shall have all the old')), + (3, fts5_locale('second', 'Liberties and Customs')), + (4, fts5_locale('second', 'which it hath been used to have')); + } + + fts5_aux_test_functions db + + do_execsql_test 6.$tn.3 { + SELECT fts5_test_columnsize(y1) FROM y1 + } { + 2 3 2 4 + } + + do_execsql_test 6.$tn.4 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 6.$tn.5 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); + } { + 2 3 + } + + do_execsql_test 6.$tn.6 { + SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); + } { + 4 4 + } + + do_execsql_test 6.$tn.7 { + SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); + } { + 4 {which it hath been used to [have]} + } + + do_execsql_test 6.$tn.8 { + SELECT rowid, + highlight(y1, 0, '[', ']'), + snippet(y1, 0, '[', ']', '...', 10) + FROM y1('Liberties + Customs'); + } { + 3 {[Liberties and Customs]} + {[Liberties and Customs]} + } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); +} +do_catchsql_test 6.1 { + INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world')); +} {1 {fts5_locale() requires locale=1}} + +do_execsql_test 6.2 { + SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') ); +} {text text} + +#-------------------------------------------------------------------------- +# Test that fts5_locale() works with external-content tables. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 7.1 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT); + CREATE VIEW v1 AS + SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1; + + CREATE VIRTUAL TABLE ft USING fts5( + bb, tt, locale=1, tokenize=tcl, content=v1 + ); + + INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL); + INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse'); + INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second'); +} + +do_execsql_test 7.2 { + INSERT INTO ft(ft) VALUES('rebuild'); + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 7.3 { + SELECT rowid, quote(bb), quote(tt) FROM ft +} { + 1 NULL {'one two three'} + 2 '7800616263' {'four five six'} + 3 '000000007800616263' {'seven eight nine'} +} + +do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } +do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2 + +fts5_aux_test_functions db + +do_execsql_test 7.6 { + SELECT fts5_test_columnlocale(ft) FROM ft; +} { + {{} {}} {{} reverse} {{} second} +} + +#------------------------------------------------------------------------- +# Test that the porter tokenizer works with locales. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl"); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + + INSERT INTO ft(rowid, tt) VALUES + (111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')), + (222, fts5_locale('reverse', 'This value may also be set')); +} + +do_execsql_test 8.1 { + SELECT DISTINCT term FROM vocab ORDER BY 1 +} { + a eb eulav osla sihT te the token yam +} + +#------------------------------------------------------------------------- +# Test that position-lists (used by xInst, xPhraseFirst etc.) work with +# locales and modes other than detail=full. +# +foreach {tn detail} { + 1 detail=full + 2 detail=none + 3 detail=column +} { + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + do_execsql_test 9.$tn.0 " + CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail); + " + do_execsql_test 9.$tn.1 { + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); + INSERT INTO ft(rowid, tt) VALUES + (-1, fts5_locale('second', 'it is an ancient mariner')); + } + + do_execsql_test 9.$tn.2 { + SELECT DISTINCT term FROM vocab + } {an it mariner} + + do_execsql_test 9.$tn.3 { + SELECT highlight(ft, 0, '[', ']') FROM ft('mariner') + } {{it is an ancient [mariner]}} +} + +#------------------------------------------------------------------------- +# Check some corrupt fts5_locale() blob formats are detected. +# +foreach_detail_mode $::testprefix { + + reset_db + sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create + fts5_aux_test_functions db + do_execsql_test 10.1 { + CREATE TABLE x1(ii INTEGER PRIMARY KEY, x); + CREATE VIRTUAL TABLE ft USING fts5(x, + content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0 + ); + + CREATE VIRTUAL TABLE ft2 USING fts5( + x, locale=1, detail=%DETAIL%, columnsize=0 + ); + } + + foreach {tn v} { + 1 X'001152' + 2 X'0011223344' + 3 X'00E0B2EB68656c6c6f' + 4 X'00E0B2EB0068656c6c6f' + } { + do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') } + do_execsql_test 10.2.$tn.1 { DELETE FROM x1; } + do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) " + + do_catchsql_test 10.2.$tn.3 { + INSERT INTO ft(ft) VALUES('rebuild'); + } {0 {}} + + do_catchsql_test 10.2.$tn.4 " + SELECT * FROM ft( test_setsubtype($v, 76) ); + " {1 {fts5: syntax error near ""}} + + do_execsql_test 10.2.$tn.5 { + INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); + } + + if {"%DETAIL%"=="full"} { + do_execsql_test 10.2.$tn.6 { + SELECT fts5_test_poslist(ft) FROM ft('world'); + } {0.0.1} + + do_execsql_test 10.2.$tn.7.1 { + SELECT fts5_test_columnsize(ft) FROM ft('world'); + } {1} + + do_execsql_test 10.2.$tn.7.2 { + SELECT fts5_test_columnlocale(ft) FROM ft('world'); + } {{{}}} + } + + do_catchsql_test 10.2.$tn.8 { + SELECT count(*) FROM ft('hello') + } {0 1} + + do_catchsql_test 10.2.$tn.9 { + PRAGMA integrity_check; + } {0 ok} + + do_execsql_test 10.2.$tn.10 { + DELETE FROM x1; + INSERT INTO x1(ii, x) VALUES(1, 'hello world'); + } + + do_catchsql_test 10.2.$tn.11 " + INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) + " {0 {}} + + do_catchsql_test 10.2.$tn.12 " + INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) + " {0 {}} + + do_execsql_test 10.2.$tn.13 { + INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); + } + do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v" + + do_catchsql_test 10.2.$tn.15 { + PRAGMA integrity_check; + } {0 {{malformed inverted index for FTS5 table main.ft2}}} + + do_execsql_test 10.2.$tn.16 { + DELETE FROM ft2_content; + INSERT INTO ft2(ft2) VALUES('rebuild'); + } + } + +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1); + INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three')); +} + +do_catchsql_test 11.1 { + SELECT fts5_columnlocale(x1, -1) FROM x1('two'); +} {1 SQLITE_RANGE} +do_catchsql_test 11.2 { + SELECT fts5_columnlocale(x1, 1) FROM x1('two'); +} {1 SQLITE_RANGE} + +#------------------------------------------------------------------------- +# +reset_db +do_test 12.0 { + list [catch { + sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create + } msg] $msg +} {1 {error in fts5_api.xCreateTokenizer_v2()}} + +#------------------------------------------------------------------------- +# Tests for auxiliary function fts5_get_locale(). +# +reset_db + +# Check that if the table does not support locale=1, fts5_get_locale() +# always returns NULL. +do_execsql_test 13.1.0 { + CREATE VIRTUAL TABLE nolocale USING fts5(a, b); + INSERT INTO nolocale VALUES('one two three', 'four five six'); + INSERT INTO nolocale VALUES('three two one', 'seven eight nine'); +} +do_execsql_test 13.1.1 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale; +} {1 1} +do_execsql_test 13.1.2 { + SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two'); +} {1} +do_execsql_test 13.1.3 { + SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two'); +} {1 1} +do_execsql_test 13.1.4 { + SELECT + fts5_get_locale(nolocale, 1) IS NULL + FROM nolocale('three AND two') ORDER BY rank +} {1 1} +do_catchsql_test 13.1.5 { + SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.6 { + SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two'); +} {1 {column index out of range}} +do_catchsql_test 13.1.7 { + SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.8 { + SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two'); +} {1 {wrong number of arguments to function fts5_get_locale()}} +do_catchsql_test 13.1.9 { + SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two'); +} {1 {non-integer argument passed to function fts5_get_locale()}} + + +# Check that if the table does support locale=1, fts5_get_locale() +# returns the locale of the identified row/column. +do_execsql_test 13.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); + INSERT INTO ft VALUES( + fts5_locale('th_TH', 'one two three'), 'four five six seven' + ); + INSERT INTO ft VALUES( + 'three two one', fts5_locale('en_AU', 'seven eight nine') + ); +} + +do_execsql_test 13.2.1 { + SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.2 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') +} { 'th_TH' NULL NULL 'en_AU' } +do_execsql_test 13.2.3 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rank +} { NULL 'en_AU' 'th_TH' NULL } +do_execsql_test 13.2.4 { + SELECT + quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_execsql_test 13.2.5 { + SELECT + quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} { 'th_TH' NULL NULL 'en_AU' } + +do_catchsql_test 13.2.6 { + SELECT + quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} +do_catchsql_test 13.2.7 { + SELECT + quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) + FROM ft('one AND three') ORDER BY rowid +} {1 {non-integer argument passed to function fts5_get_locale()}} + +#------------------------------------------------------------------------- +# Check that UPDATE statements that may affect more than one row work. +# +reset_db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); +} + +do_execsql_test 14.2 { + INSERT INTO ft VALUES('hello', 'world'); +} + +do_execsql_test 14.3 { + UPDATE ft SET b = fts5_locale('en_AU', 'world'); +} + +do_execsql_test 14.4 { + INSERT INTO ft VALUES(X'abcd', X'1234'); +} {} + +do_execsql_test 14.5 { + SELECT quote(a), quote(b) FROM ft +} {'hello' 'world' X'ABCD' X'1234'} + +do_execsql_test 14.6 { + DELETE FROM ft; + INSERT INTO ft VALUES(NULL, 'null'); + INSERT INTO ft VALUES(123, 'int'); + INSERT INTO ft VALUES(345.0, 'real'); + INSERT INTO ft VALUES('abc', 'text'); + INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text'); + + SELECT a, typeof(a), b FROM ft +} { + {} null null + 123 integer int + 345.0 real real + abc text text + def text text +} + +do_execsql_test 14.7 { + SELECT quote(c0), typeof(c0) FROM ft_content +} { + NULL null + 123 integer + 345.0 real + 'abc' text + 'def' text +} + +#------------------------------------------------------------------------- +# Check that inserting UNINDEXED columns between indexed columns of a +# locale=1 table does not cause a problem. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db + +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl); +} + +do_execsql_test 15.2 { + INSERT INTO ft VALUES('one', 'two', 'three'); + INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three')); +} + +do_execsql_test 15.3 { + SELECT c2, l2 FROM ft_content +} {three {} three loc} + +do_execsql_test 15.4 { + SELECT c, fts5_columnlocale(ft, 2) FROM ft +} {three {} three loc} + + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 570693373f..a3bce869fb 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -517,7 +517,31 @@ fts5_aux_test_functions db do_execsql_test 15.3 { SELECT fts5_test_all(t1) FROM t1 LIMIT 1; } { - {columnsize {0 0} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2} + {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2} } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 16.0 { + CREATE TABLE t1(x); + BEGIN EXCLUSIVE; +} + +do_test 16.1 { + set rc [catch { + sqlite3 db2 test.db + db2 eval {SELECT * FROM t1} + } errmsg] + lappend rc $errmsg +} {1 {database is locked}} + +do_execsql_test 16.2 { + ROLLBACK; +} + +do_test 16.3 { + catchsql { SELECT * FROM t1 } db2 +} {0 {}} + finish_test diff --git a/ext/fts5/test/fts5merge.test b/ext/fts5/test/fts5merge.test index 3b86167b0d..09c18245f3 100644 --- a/ext/fts5/test/fts5merge.test +++ b/ext/fts5/test/fts5merge.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5merge -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -238,6 +238,22 @@ do_execsql_test 6.3 { INSERT INTO g1(g1) VALUES('integrity-check'); } +#-------------------------------------------------------------------------- +# Check that passing -2147483648 as the parameter to a merge command +# does not cause a signed integer overflow error. +# +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE f1 USING fts5(a); +} +do_execsql_test 7.1 { + INSERT INTO f1 VALUES('one two three'); + INSERT INTO f1 VALUES('four five six'); + INSERT INTO f1 VALUES('seven eight nine'); +} +do_execsql_test 7.2 { + INSERT INTO f1(f1, rank) VALUES('merge', -2147483648); +} finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index da3f652697..817be9560c 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -35,21 +35,21 @@ do_catchsql_test 1.1.2 { do_catchsql_test 1.2.1 { SELECT highlight(t1, 4, '', '') FROM t1('*id'); -} {0 {{}}} +} {1 {no such cursor: 4}} do_catchsql_test 1.2.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*id')); -} {0 {}} +} {1 {no such cursor: 6}} do_catchsql_test 1.3.1 { SELECT highlight(t1, 4, '', '') FROM t1('*reads'); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} do_catchsql_test 1.3.2 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} db close sqlite3 db test.db @@ -57,7 +57,12 @@ sqlite3 db test.db do_catchsql_test 1.3.3 { SELECT a FROM t1 WHERE rank = (SELECT highlight(t1, 4, '', '') FROM t1('*reads')); -} {1 {no such cursor: 1}} +} {1 {no such cursor: 0}} + +fts5_aux_test_functions db +do_catchsql_test 1.3.4 { + SELECT fts5_columntext(t1) FROM t1('*reads'); +} {1 {no such cursor: 0}} #------------------------------------------------------------------------- reset_db @@ -91,7 +96,6 @@ do_execsql_test 2.2.1 { INSERT INTO vt0(c0) VALUES ('xyz'); } -breakpoint do_execsql_test 2.2.2 { ALTER TABLE t0 RENAME TO t1; } @@ -424,10 +428,12 @@ do_execsql_test -db db2 15.3 { SAVEPOINT one; } {} do_execsql_test 15.4 END -do_test 15.4 { +do_test 15.5 { list [catch { db2 eval COMMIT } msg] $msg } {0 {}} +db2 close + #------------------------------------------------------------------------- reset_db forcedelete test.db2 @@ -469,6 +475,8 @@ do_execsql_test -db db2 16.6 { SELECT * FROM x1 } {abc def} +db2 close + #------------------------------------------------------------------------- reset_db do_execsql_test 17.1 { @@ -496,6 +504,198 @@ do_execsql_test 17.5 { SELECT c0 FROM t0 WHERE c0 GLOB '*faul*'; } {assertionfaultproblem} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 18.0 { + BEGIN; + CREATE VIRTUAL TABLE t1 USING fts5(text); + ALTER TABLE t1 RENAME TO t2; +} + +do_execsql_test 18.1 { + DROP TABLE t2; +} + +do_execsql_test 18.2 { + COMMIT; +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 19.0 { + CREATE VIRTUAL TABLE t1 USING fts5(text); + CREATE TABLE t2(text); + BEGIN; + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES('three'); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES('three'); + SAVEPOINT one; + INSERT INTO t2 VALUES('one'); + INSERT INTO t2 VALUES('two'); + INSERT INTO t2 VALUES('three'); + ROLLBACK TO one; + COMMIT; +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a); + INSERT INTO x1(rowid, a) VALUES + (1, 'a b c d'), + (2, 'x b c d'), + (3, 'x y z d'), + (4, 'a y c x'); +} + +do_execsql_test 20.1 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'b'; +} {1} + +do_execsql_test 20.2 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'y'; +} {4} + +do_execsql_test 20.3 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR x1 MATCH 'y'; +} {1 4 3} + +do_execsql_test 20.4 { + SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR (x1 MATCH 'y' AND x1 MATCH 'd'); +} {1 4 3} + +do_execsql_test 20.5 { + SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd'); +} {3 1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 21.0 { + CREATE TABLE t1(ii INTEGER, x TEXT, y TEXT); + CREATE VIRTUAL TABLE xyz USING fts5(content_rowid=ii, content=t1, x, y); + INSERT INTO t1 VALUES(1, 'one', 'i'); + INSERT INTO t1 VALUES(2, 'two', 'ii'); + INSERT INTO t1 VALUES(3, 'tree', 'iii'); + INSERT INTO xyz(xyz) VALUES('rebuild'); +} + +do_execsql_test 21.1 { + UPDATE xyz SET y='TWO' WHERE rowid=2; + UPDATE t1 SET y='TWO' WHERE ii=2; +} + +do_execsql_test 21.2 { + PRAGMA integrity_check +} {ok} + +sqlite3_db_config db DEFENSIVE 1 +do_execsql_test 21.3 { + CREATE TABLE xyz_notashadow(x, y); + DROP TABLE xyz_notashadow; +} +sqlite3_db_config db DEFENSIVE 0 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 22.0 { + SELECT fts5(NULL); +} {{}} +do_execsql_test 22.1 { + SELECT count(*) FROM ( + SELECT fts5_source_id() + ) +} {1} +execsql_pp { + SELECT fts5_source_id() +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1 VALUES('one + two + three'); + INSERT INTO x1 VALUES('one + xyz + three'); + INSERT INTO x1 VALUES('xyz + two + xyz'); +} +do_execsql_test 23.1 { + SELECT rowid FROM x1('one + two + three'); +} {1} + +do_execsql_test 23.2 { + SELECT rowid FROM x1('^".." AND one'); +} {} + +do_execsql_test 23.3 { + SELECT rowid FROM x1('abc NEAR ".." NEAR def'); +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 24.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none'); + INSERT INTO t1(a) VALUES('a'); +} + +do_execsql_test 24.2 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ) ORDER BY 1; +} {-1e-06} + +do_execsql_test 24.3 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT � NOT def') ) ORDER BY 1; +} {-1e-06} + +do_execsql_test 24.4 { + SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ); +} {-1e-06} + +#------------------------------------------------------------------------- +reset_db +fts5_aux_test_functions db + +do_execsql_test 25.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none', content=''); + INSERT INTO t1(a) VALUES('a b c'); +} + +do_execsql_test 25.0 { + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; +} {{}} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 26.0 { + PRAGMA foreign_keys = ON; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + CREATE TABLE t2(y INTEGER PRIMARY KEY, + z INTEGER REFERENCES t1(x) DEFERRABLE INITIALLY DEFERRED + ); + CREATE VIRTUAL TABLE t3 USING fts5(a, b, content='', tokendata=1); +} + +do_execsql_test 26.1 { + BEGIN; + INSERT INTO t2 VALUES(1,111); + INSERT INTO t3 VALUES(3,3); + PRAGMA defer_foreign_keys=ON; + DELETE FROM t2 WHERE y+1; + COMMIT; +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 27.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b); + INSERT INTO ft1(rowid, a, b) VALUES(3, '3', '3'); +} + +do_execsql_test 27.1 { + SELECT * FROM ft1 WHERE rowid=3 AND b MATCH 'hello'; +} finish_test diff --git a/ext/fts5/test/fts5near.test b/ext/fts5/test/fts5near.test index bbe144a898..318a169488 100644 --- a/ext/fts5/test/fts5near.test +++ b/ext/fts5/test/fts5near.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5near -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5onepass.test b/ext/fts5/test/fts5onepass.test index 01021ed348..b334096754 100644 --- a/ext/fts5/test/fts5onepass.test +++ b/ext/fts5/test/fts5onepass.test @@ -38,15 +38,15 @@ foreach {tn sql uses} { 1.2 { DELETE FROM ft WHERE rowid=? } 0 1.3 { DELETE FROM ft WHERE rowid=? } 0 1.4 { DELETE FROM ft WHERE ft MATCH '1' } 1 - 1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 - 1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 + 1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 0 + 1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 0 2.1 { UPDATE ft SET content='a b c' } 1 2.2 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 2.3 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 2.4 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' } 1 - 2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 - 2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 + 2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 0 + 2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 0 } { do_test 1.$tn { sql_uses_stmt db $sql } $uses } diff --git a/ext/fts5/test/fts5optimize.test b/ext/fts5/test/fts5optimize.test index e0f0fd7242..610bf439c9 100644 --- a/ext/fts5/test/fts5optimize.test +++ b/ext/fts5/test/fts5optimize.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5optimize -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5optimize2.test b/ext/fts5/test/fts5optimize2.test index a0782ee790..57f4e96b99 100644 --- a/ext/fts5/test/fts5optimize2.test +++ b/ext/fts5/test/fts5optimize2.test @@ -9,13 +9,13 @@ # #*********************************************************************** # -# TESTRUNNER: slow +# TESTRUNNER: superslow # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5optimize2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -42,23 +42,4 @@ do_execsql_test 1.2 { SELECT count(*) FROM t1('mno') } $nLoop -do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(x); - INSERT INTO t2(t2, rank) VALUES('pgsz', 32); -} - -do_test 2.1 { - for {set ii 0} {$ii < $nLoop} {incr ii} { - execsql { - INSERT INTO t2 VALUES('abc def ghi'); - INSERT INTO t2 VALUES('jkl mno pqr'); - INSERT INTO t2(t2, rank) VALUES('merge', -1); - } - } -} {} - -do_execsql_test 2.2 { - SELECT count(*) FROM t2('mno') -} $nLoop - finish_test diff --git a/ext/fts5/test/fts5optimize3.test b/ext/fts5/test/fts5optimize3.test new file mode 100644 index 0000000000..79e62f9f22 --- /dev/null +++ b/ext/fts5/test/fts5optimize3.test @@ -0,0 +1,45 @@ +# 2023 Aug 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: superslow +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5optimize2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set nLoop 2500 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2(t2, rank) VALUES('pgsz', 32); +} + +do_test 1.1 { + for {set ii 0} {$ii < $nLoop} {incr ii} { + execsql { + INSERT INTO t2 VALUES('abc def ghi'); + INSERT INTO t2 VALUES('jkl mno pqr'); + INSERT INTO t2(t2, rank) VALUES('merge', -1); + } + } +} {} + +do_execsql_test 1.2 { + SELECT count(*) FROM t2('mno') +} $nLoop + +finish_test diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test new file mode 100644 index 0000000000..be77cbfca5 --- /dev/null +++ b/ext/fts5/test/fts5origintext.test @@ -0,0 +1,357 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { +foreach {tn insttoken} { + 1 0 + 2 1 +} { +reset_db + +sqlite3_fts5_register_origintext db +do_execsql_test $tn.1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", detail=%DETAIL% + ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); +} + +do_execsql_test $tn.1.1 { + INSERT INTO ft VALUES('Hello world'); +} + +do_execsql_test $tn.1.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +proc b {x} { string map [list "\0" "."] $x } +db func b b + +do_execsql_test $tn.1.3 { + select b(term) from vocab; +} { + hello.Hello + world +} + +do_execsql_test $tn.1.4 { + SELECT rowid FROM ft('Hello'); +} {1} + +#------------------------------------------------------------------------- +reset_db + +# Return a random integer between 0 and n-1. +# +proc random {n} { + expr {abs(int(rand()*$n))} +} + +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} + +proc term {} { + set first_letter { + a b c d e f g h i j k l m n o p q r s t u v w x y z + A B C D E F G H I J K L M N O P Q R S T U V W X Y Z + } + + set term [select_one $first_letter] + append term [random 100] +} + +proc document {} { + set nTerm [expr [random 5] + 5] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +sqlite3_fts5_register_origintext db +do_execsql_test $tn.2.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", detail=%DETAIL% + ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); + INSERT INTO ft(ft, rank) VALUES('pgsz', 128); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); +} + +do_test $tn.2.1 { + for {set ii 0} {$ii < 500} {incr ii} { + execsql { INSERT INTO ft VALUES( document() ) } + } +} {} + +do_execsql_test $tn.2.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test $tn.2.3 { + INSERT INTO ft(ft, rank) VALUES('merge', 16); +} + +do_execsql_test $tn.2.4 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test $tn.2.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} + +#------------------------------------------------------------------------- +reset_db + +sqlite3_fts5_register_origintext db +do_execsql_test $tn.3.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", detail=%DETAIL% + ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); + + INSERT INTO ft(rowid, x) VALUES(1, 'hello'); + INSERT INTO ft(rowid, x) VALUES(2, 'Hello'); + INSERT INTO ft(rowid, x) VALUES(3, 'HELLO'); +} + +#proc b {x} { string map [list "\0" "."] $x } +#db func b b +#execsql_pp { SELECT b(term) FROM vocab } + +do_execsql_test $tn.3.1.1 { SELECT rowid FROM ft('hello') } 1 +do_execsql_test $tn.3.1.2 { SELECT rowid FROM ft('Hello') } 2 +do_execsql_test $tn.3.1.3 { SELECT rowid FROM ft('HELLO') } 3 + +do_execsql_test $tn.3.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, + tokenize="origintext unicode61", + tokendata=1, + detail=%DETAIL% + ); + INSERT INTO ft2(ft2, rank) VALUES('insttoken', $insttoken); + CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance); + + INSERT INTO ft2(rowid, x) VALUES(1, 'hello'); + INSERT INTO ft2(rowid, x) VALUES(2, 'Hello'); + INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO'); + + INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo'); +} + +#proc b {x} { string map [list "\0" "."] $x } +#db func b b +#execsql_pp { SELECT b(term) FROM vocab } + +do_execsql_test $tn.3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3} +do_execsql_test $tn.3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3} +do_execsql_test $tn.3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} + +do_execsql_test $tn.3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} + +do_execsql_test $tn.3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} { + 3 2 1 +} +do_execsql_test $tn.3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} { + 3 2 1 +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +proc querytoken {cmd iPhrase iToken} { + set txt [$cmd xQueryToken $iPhrase $iToken] + string map [list "\0" "."] $txt +} +sqlite3_fts5_create_function db querytoken querytoken + +do_execsql_test $tn.4.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL% + ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); + INSERT INTO ft VALUES('one two three four'); +} + +do_execsql_test $tn.4.1 { + SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO') +} {1 two.TwO} +do_execsql_test $tn.4.2 { + SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE') +} {1 one} +do_execsql_test $tn.4.3 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE') +} {1 two.TWO} + +if {"%DETAIL%"=="full"} { + # Phrase queries are only supported for detail=full. + # + do_execsql_test $tn.4.4 { + SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') + } {1 three.ThreE} + do_catchsql_test $tn.4.5 { + SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') + } {1 SQLITE_RANGE} + do_catchsql_test $tn.4.6 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') + } {1 SQLITE_RANGE} + do_catchsql_test $tn.4.7 { + SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') + } {1 SQLITE_RANGE} +} + +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +proc insttoken {cmd iIdx iToken} { + set txt [$cmd xInstToken $iIdx $iToken] + string map [list "\0" "."] $txt +} +sqlite3_fts5_create_function db insttoken insttoken +fts5_aux_test_functions db + +do_execsql_test $tn.5.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL% + ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); + INSERT INTO ft VALUES('one ONE One oNe oNE one'); +} + +do_execsql_test $tn.5.1 { + SELECT insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0), + insttoken(ft, 3, 0), + insttoken(ft, 4, 0), + insttoken(ft, 5, 0) + FROM ft('one'); +} { + one one.ONE one.One one.oNe one.oNE one +} + +do_execsql_test $tn.5.2 { + SELECT insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0), + insttoken(ft, 3, 0), + insttoken(ft, 4, 0), + insttoken(ft, 5, 0) + FROM ft('on*'); +} { + one one.ONE one.One one.oNe one.oNE one +} + +do_execsql_test $tn.5.3 { + SELECT insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0), + insttoken(ft, 3, 0), + insttoken(ft, 4, 0), + insttoken(ft, 5, 0) + FROM ft(fts5_insttoken('on*')); +} { + one one.ONE one.One one.oNe one.oNE one +} + +do_execsql_test $tn.5.4 { + SELECT insttoken(ft, 1, 0) FROM ft('one'); +} { + one.ONE +} + +do_execsql_test $tn.5.5 { + SELECT fts5_test_poslist(ft) FROM ft('one'); +} { + {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5} +} + +#------------------------------------------------------------------------- +# Test the xInstToken() API with: +# +# * a non tokendata=1 table. +# * prefix queries. +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test $tn.6.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0 + ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); + + INSERT INTO ft VALUES('One Two', 'Three two'); + INSERT INTO ft VALUES('three Three', 'one One'); +} +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +do_execsql_test $tn.6.1 { + SELECT rowid, tokens(ft) FROM ft('One'); +} {1 one.One 2 one.One} + +do_execsql_test $tn.6.2 { + SELECT rowid, tokens(ft) FROM ft('on*'); +} {1 one.One 2 {one one.One}} + +do_execsql_test $tn.6.3 { + SELECT rowid, tokens(ft) FROM ft('Three*'); +} {1 three.Three 2 three.Three} + +fts5_aux_test_functions db +do_catchsql_test $tn.6.4 { + SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one'); +} {1 SQLITE_RANGE} + +do_catchsql_test $tn.6.5 { + SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one'); +} {1 SQLITE_RANGE} + +do_catchsql_test $tn.6.6 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2); +} {1 {malformed tokendata=... directive}} +do_catchsql_test $tn.6.7 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11); +} {1 {malformed tokendata=... directive}} + +} +} + +finish_test + diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test new file mode 100644 index 0000000000..a8c7172344 --- /dev/null +++ b/ext/fts5/test/fts5origintext2.test @@ -0,0 +1,146 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft VALUES('Hello'); + INSERT INTO ft VALUES('hello'); + INSERT INTO ft VALUES('HELLO'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('World'); + INSERT INTO ft VALUES('world'); + INSERT INTO ft VALUES('WORLD'); + COMMIT; +} + +do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3} +do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6} +do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9} + +do_execsql_test 1.5 { + SELECT count(*) FROM ft_data +} 3 + +do_execsql_test 1.6 { + DELETE FROM ft; + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s; + INSERT INTO ft VALUES ('hELLO hELLO hELLO'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('World World World World World World World'); + INSERT INTO ft VALUES('world world world world world world world'); + INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD'); + INSERT INTO ft VALUES('World World World World World World World'); + INSERT INTO ft VALUES('world world world world world world world'); + INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD'); + COMMIT; +} + +do_execsql_test 1.7 { + SELECT count(*) FROM ft_data; +} 23 + +do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101} + +do_execsql_test 1.9 { + DELETE FROM ft; + INSERT INTO ft(ft) VALUES('optimize'); + SELECT count(*) FROM ft_data; +} {2} +do_execsql_test 1.10 { + BEGIN; + INSERT INTO ft VALUES('Hello'); + INSERT INTO ft VALUES('hello'); + INSERT INTO ft VALUES('HELLO'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('World'); + INSERT INTO ft VALUES('world'); + INSERT INTO ft VALUES('WORLD'); +} + +do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3} +do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6} +do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9} +do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3} + +#------------------------------------------------------------------------ +reset_db +sqlite3_fts5_register_origintext db +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING fts5( + v, tokenize="origintext unicode61", tokendata=1, detail=none + ); + + INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy'); + INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy'); +} + +do_execsql_test 2.1 { + SELECT tokens(x1) FROM x1('xxx'); +} { + {xxx xxx.Xxx xxx.XXX} {xxx xxx} +} + +do_execsql_test 2.2 { + UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1; +} + +do_execsql_test 2.3 { + SELECT tokens(x1) FROM x1('xxx'); +} { + {xxx {} xxx} {xxx xxx} +} + +finish_test + diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test new file mode 100644 index 0000000000..351ab1f617 --- /dev/null +++ b/ext/fts5/test/fts5origintext3.test @@ -0,0 +1,141 @@ +# 2023 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + foreach {tn insttoken} { + 1 0 + 2 1 + } { + + reset_db + + sqlite3_fts5_register_origintext db + fts5_aux_test_functions db + proc insttoken {cmd iIdx iToken} { + set txt [$cmd xInstToken $iIdx $iToken] + string map [list "\0" "."] $txt + } + sqlite3_fts5_create_function db insttoken insttoken + + do_execsql_test $tn.1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + } + + do_execsql_test $tn.1.0.1 { + INSERT INTO ft(ft, rank) VALUES('insttoken', 1); + } + + do_execsql_test $tn.1.1 { + INSERT INTO ft VALUES('Hello world HELLO WORLD hello'); + } + + do_execsql_test $tn.1.2 { + SELECT fts5_test_poslist(ft) FROM ft('hello'); + } {{0.0.0 0.0.2 0.0.4}} + + do_execsql_test $tn.1.3 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hello'); + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.3.1 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hel*'); + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.4 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hello') ORDER BY rank; + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.5 { + CREATE VIRTUAL TABLE ft2 USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE'); + INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO'); + INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE'); + } + + do_execsql_test $tn.1.6 { + SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank; + } {three.THREE 3 three 1 three 2} + + do_execsql_test $tn.1.7 { + INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb'); + INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB'); + } + + do_execsql_test $tn.1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24} + do_execsql_test $tn.1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24} + + do_execsql_test $tn.2.0 { + CREATE VIRTUAL TABLE ft3 USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + prefix=2 + ); + } + do_execsql_test $tn.2.1 { + INSERT INTO ft3(rowid, x) VALUES(1, 'one'); + INSERT INTO ft3(rowid, x) VALUES(2, 'ONE'); + INSERT INTO ft3(rowid, x) VALUES(3, 'ONT'); + INSERT INTO ft3(rowid, x) VALUES(4, 'on'); + INSERT INTO ft3(rowid, x) VALUES(5, 'On'); + } + + do_execsql_test $tn.2.2 { + SELECT rowid FROM ft3('on*'); + } {1 2 3 4 5} + + do_execsql_test $tn.2.3 { + SELECT rowid, insttoken(ft3, 0, 0) FROM ft3('on*'); + } {1 one 2 one.ONE 3 ont.ONT 4 on 5 on.On} + + } +} + +finish_test + diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test new file mode 100644 index 0000000000..3b907ba2cc --- /dev/null +++ b/ext/fts5/test/fts5origintext4.test @@ -0,0 +1,80 @@ +# 2023 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext4 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# The tests below verify that a doclist-index is used to limit the number +# of pages loaded into the cache. It does this by querying sqlite3_db_status() +# for the amount of memory used by the pager cache. +# +# memsubsys1 effectively limits the page-cache to 24 pages. Which masks +# the effect tested by the tests in this file. And "mmap" prevents the +# cache from being used, also preventing these tests from working. +# +if {[permutation]=="memsubsys1" || [permutation]=="mmap"} { + finish_test + return +} + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + PRAGMA page_size = 4096; + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft SELECT 'the first thing'; + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000 + ) + INSERT INTO ft SELECT 'The second thing' FROM s; + + INSERT INTO ft SELECT 'the first thing'; + COMMIT; + INSERT INTO ft(ft) VALUES('optimize'); +} + +foreach {tn sql expr} { + 1 { SELECT rowid FROM ft('the') } {$mem > 250000} + 2 { SELECT rowid FROM ft('first') } {$mem < 50000} + 3 { SELECT rowid FROM ft('the first') } {$mem < 50000} +} { + db close + sqlite3 db test.db + sqlite3_fts5_register_origintext db + + execsql $sql + do_test 1.2.$tn { + set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1] + expr $expr + } 1 +} + +proc b {x} { string map [list "\0" "."] $x } +db func b b +# execsql_pp { SELECT segid, b(term), pgno from ft_idx } + +finish_test + diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test new file mode 100644 index 0000000000..848cc15b5c --- /dev/null +++ b/ext/fts5/test/fts5origintext5.test @@ -0,0 +1,273 @@ +# 2023 Dec 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for tables that use both tokendata=1 and contentless_delete=1. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Return a random integer between 0 and n-1. +# +proc random {n} { expr {abs(int(rand()*$n))} } + +# Select an element of the list passed as the only argument at random and +# return it. +# +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} + +# Given a term that consists entirely of alphabet characters, return all +# permutations of the term using upper and lower case characters. e.g. +# +# "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba} +# +proc casify {term {lRet {{}}}} { + if {$term==""} { return $lRet } + set t [string range $term 1 end] + set f1 [string toupper [string range $term 0 0]] + set f2 [string tolower [string range $term 0 0]] + set ret [list] + foreach x $lRet { + lappend ret "$x$f1" + lappend ret "$x$f2" + } + return [casify $t $ret] +} + +proc vocab {} { + list abc def ghi jkl mno pqr stu vwx yza +} + +# Return a random 3 letter term. +# +proc term {} { + if {[info exists ::expanded_vocab]==0} { + foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] } + } + + select_one $::expanded_vocab +} + +# Return a document - between 3 and 10 terms. +# +proc document {} { + set nTerm [expr [random 3] + 7] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +#------------------------------------------------------------------------- + +expr srand(6) + +set NDOC 200 +set NLOOP 50 + +sqlite3_fts5_register_origintext db + +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +proc rankfunc {cmd} { + $cmd xRowid +} +sqlite3_fts5_create_function db rankfunc rankfunc + +proc ctrl_tokens {term args} { + set ret [list] + set term [string tolower $term] + foreach doc $args { + foreach a $doc { + if {[string tolower $a]==$term} { + if {$a==$term} { + lappend ret $a + } else { + lappend ret [string tolower $a].$a + } + } + } + } + set ret +} +db func ctrl_tokens ctrl_tokens + +proc do_all_vocab_test {tn} { + foreach ::v [concat [vocab] nnn] { + set answer [execsql { + SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%' + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft) FROM ft($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank + } $answer + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + + CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()'); +} +do_test 1.1 { + for {set ii 0} {$ii < $NDOC} {incr ii} { + set doc [document] + execsql { + INSERT INTO ft(rowid, x) VALUES($ii, $doc); + INSERT INTO ctrl(id, x) VALUES($ii, $doc); + } + } +} {} + +#execsql_pp { SELECT * FROM ctrl } +#execsql_pp { SELECT * FROM ft } +#fts5_aux_test_functions db +#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); } + +do_all_vocab_test 1.2 + +for {set ii 0} {$ii < $NLOOP} {incr ii} { + set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }] + foreach r $lRowid { + execsql { DELETE FROM ft WHERE rowid = $r } + execsql { DELETE FROM ctrl WHERE rowid = $r } + + set doc [document] + execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) } + execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) } + } + do_all_vocab_test 1.3.$ii +} + +#------------------------------------------------------------------------- + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts5( + x, y, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + + CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT); + INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64); + INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()'); +} +do_test 2.1 { + for {set ii 0} {$ii < $NDOC} {incr ii} { + set doc1 [document] + set doc2 [document] + execsql { + INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2); + INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2); + } + } +} {} + +proc do_all_vocab_test2 {tn} { + foreach ::v [vocab] { + set answer [execsql { + SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2 + WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%'; + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft2) FROM ft2($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank + } $answer + } +} + +do_all_vocab_test2 2.2 + +for {set ii 0} {$ii < $NLOOP} {incr ii} { + set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }] + foreach r $lRowid { + execsql { DELETE FROM ft2 WHERE rowid = $r } + execsql { DELETE FROM ctrl2 WHERE rowid = $r } + + set doc1 [document] + set doc2 [document] + execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) } + execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) } + } + do_all_vocab_test 2.3.$ii +} + +#------------------------------------------------------------------------- + +unset -nocomplain ::expanded_vocab +proc vocab {} { + list abcde fghij klmno +} + +proc do_all_vocab_test3 {tn} { + foreach ::v [concat [vocab] nnn] { + set answer [execsql { + SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%' + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft3) FROM ft3($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank + } $answer + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft3 USING fts5( + w, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()'); + CREATE TABLE ctrl3(w); +} + +do_execsql_test 3.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2 + ) + INSERT INTO ctrl3 SELECT document() FROM s; + INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3; +} + +do_all_vocab_test3 3.2 + + +finish_test + diff --git a/ext/fts5/test/fts5origintext6.test b/ext/fts5/test/fts5origintext6.test new file mode 100644 index 0000000000..7b27e310b9 --- /dev/null +++ b/ext/fts5/test/fts5origintext6.test @@ -0,0 +1,209 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext6 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc insert_data {tbl} { + db eval " + INSERT INTO $tbl (rowid, x, y) VALUES + (1, 'ChH BDd HhG efc BjJ BGi GBG FdD','ciJ AFf ADf fBJ fhC GFI JEH fcA'), + (2, 'deg AIG Fie jII cCd Hbf igF fEE','GeA Ija gJg EDc HFi DDI dCf aDd'), + (3, 'IJC hga deC Jfa Aeg hfh CcH dfb','ajD hgC Jaf IfH CHe jIG AjD adF'), + (4, 'FiH GJH IDA AiG bBc CGG Eih bIH','hHg JaH aii IHE Ggd gcH gji CGc'), + (5, 'ceg CAd jFI GAB BGg EeC IdH acG','bBC eIG ifH eDE Adj bjb GCj ebA'), + (6, 'Eac Fbh aFF Eea jeG EIj HCc JJH','hbd giE Gfe eiI dEF abE cJf cAb'), + (7, 'dic hAc jEC AiG FEF jHc HiD HBI','aEd ebE Gfi AJG EBA faj GiG jjE'), + (8, 'Fca iEe EgE jjJ gce ijf EGc EBi','gaI dhH bFg CFc HeC CjI Jfg ccH'), + (9, 'cfd iaa HCf iHJ HjG ffh ABb ibi','CfG bia Dai eii Ejg Jeg fCg hDb'), + (10, 'Jjf hJC IID HJj bGB EbJ cgg eBj','jci jhi JAF jIg Bei Bcd cAC AJd'), + (11, 'egG Cdi bFf fEB hfH jDH jia Efd','FAd eCg fAi aiC baC eJG acF iGE'), + (12, 'Ada Gde CJI ADG gJA Cbb ccF iAB','eAE ajC FBB ccd Jgh fJg ieg hGE'), + (13, 'gBb fDG Jdd HdD fiJ Bed Cig iGg','heC FeI iaj gdg ebB giC HaD FIe'), + (14, 'FiI iDd Ffe igI bgB EJf FHG hDF','cjC AeI abf Fah cbJ ffH jEb aib'), + (15, 'jaF hBI jIH Gdh FEc Fij hgj jFh','dGA ADH feh AAI AfJ DbC gBi hGH'), + (16, 'gjH BGg iGj aFE CAH edI idf HEH','hIf DDg fjB hGi cHF BCH FjG Bgd'), + (17, 'iaI JGH hji gcj Dda eeG jDd CBi','cHg jeh caG gIc feF ihG hgJ Abj'), + (18, 'jHI iDB eFf AiH EFB CDb IAj GbC','Ghe dEI gdI jai gib dAG BIa djb'), + (19, 'abI fHG Ccf aAc FDa fiC agF bdB','afi hde IgE bGF cfg DHD diE aca'), + (20, 'IFh eDJ jfh cDg dde JGJ GAf fIJ','IBa EfH faE aeI FIF baJ FGj EIH'), + (21, 'Dee bFC bBA dEI CEj aJI ghA dCH','hBA ddA HJh dfj egI Dij dFE bGE'), + (22, 'JFE BCj FgA afc Jda FGD iHJ HDh','eAI jHe BHD Gah bbD Bgj gbh eGB'), + (23, 'edE CJE FjG aFI edA Cea FId iFe','ABG jcA ddj EEc Dcg hAI agA biA'), + (24, 'AgE cfc eef cGh aFB DcH efJ hcH','eGF HaB diG fgi bdc iGJ FGJ fFB'), + (25, 'aCa AgI GhC DDI hGJ Hgc Gcg bbG','iID Fga jHa jIj idj DFD bAC AFJ'), + (26, 'gjC JGh Fge faa eCA iGG gHE Gai','bDi hFE BbI DHD Adb Fgi hCa Hij'), + (27, 'Eji jEI jhF DFC afH cDh AGc dHA','IDe GcA ChF DIb Bif HfH agD DGh'), + (28, 'gDD AEE Dfg ICf Cbi JdE jgH eEi','eEb dBG FDE jgf cAI FaJ jaA cDd'), + (29, 'cbe Gec hgB Egi bca dHg bAJ jBf','EFB DgD GJc fDb EeE bBA GFC Hbe'), + (30, 'Adc eHB afI hDc Bhh baE hcJ BBd','JAH deg bcF Dab Bgj Gbb JHi FIB'), + (31, 'agF dIj AJJ Hfg cCG hED Igc fHC','JEf eia dHf Ggc Agj geD bEE Gei'), + (32, 'DAd cCe cbJ FjG gJe gba dJA GCf','eAf hFc bGE ABI hHA IcE abF CCE'), + (33, 'fFh jJe DhJ cDJ EBi AfD eFI IhG','fEG GCc Bjd EFF ggg CFe EHd ciB'), + (34, 'Ejb BjI eAF HaD eEJ FaG Eda AHC','Iah hgD EJG fdD cIE Daj IFf eJh'), + (35, 'aHG eCe FjA djJ dAJ jiJ IaE GGB','Acg iEF JfB FIC Eei ggj dic Iii'), + (36, 'Fdb EDF GaF JjB ehH IgC hgi DCG','cag DHI Fah hAJ bbh egG Hia hgJ'), + (37, 'HGg icC JEC AFJ Ddh dhi hfC Ich','fEg bED Bff hCJ EiA cIf bfG cGA'), + (38, 'aEJ jGI BCi FaA ebA BHj cIJ GcC','dCH ADd bGB cFE AgF geD cbG jIc'), + (39, 'JFB bBi heA BFA hgB Ahj EIE CgI','EIJ JFG FJE GeA Hdg HeH ACh GiA'), + (40, 'agB DDC CED igC Dfc DhI eiC fHi','dAB dcg iJF cej Fcc cAc AfB Fdd'), + (41, 'BdF DHj Ege hcG DEd eFa dCf gBb','FBG ChB cej iGd Hbh fCc Ibe Abh'), + (42, 'Bgc DjI cbC jGD bdb hHB IJA IJH','heg cii abb IGf eDe hJc dii fcE'), + (43, 'fhf ECa FiA aDh Jbf CiB Jhe ajD','GFE bIF aeD gDE BIE Jea DfC BEc'), + (44, 'GjE dBj DbJ ICF aDh EEH Ejb jFb','dJj aEc IBg bEG Faf fjA hjf FAF'), + (45, 'BfA efd IIJ AHG dDF eGg dIJ Gcb','Bfj jeb Ahc dAE ACH Dfb ieb dhC'), + (46, 'Ibj ege geC dJh CIi hbD EAG fGA','DEb BFe Bjg FId Fhg HeF JAc BbE'), + (47, 'dhB afC hgG bEJ aIe Cbe iEE JCD','bdg Ajc FGA jbh Jge iAj fIA jbE'), + (48, 'egH iDi bfH iiI hGC jFF Hfd AHB','bjE Beb iCc haB gIH Dea bga dfd'), + (49, 'jgf chc jGc Baj HBb jdE hgh heI','FFB aBd iEB EIG HGf Bbj EIi JbI'), + (50, 'jhe EGi ajA fbH geh EHe FdC bij','jDE bBC gbH HeE dcH iBH IFE AHi'), + (51, 'aCb JiD cgJ Bjj iAI Hbe IAF FhH','ijf bhE Jdf FED dCH bbG HcJ ebH'); + " +} + +foreach_detail_mode $testprefix { +foreach external {0 1 2} { + reset_db + + proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret + } + sqlite3_fts5_create_function db tokens tokens + sqlite3_fts5_register_origintext db + + set E(0) internal + set E(1) external + set E(2) contentless + set e $E($external) + + db eval { CREATE TABLE ex(x, y) } + switch -- $external { + 0 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + } + } + + 1 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + content=ex + ); + } + } + + 2 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + content= + ); + } + } + } + insert_data ex + insert_data ft + + proc prefixquery {prefix bInst bYOnly} { + set ret [list] + db eval { SELECT rowid, x, y FROM ex ORDER BY rowid } { + set row [list] + set bSeen 0 + + set T [concat $x $y] + if {$bYOnly} { set T $y } + + foreach w $T { + if {[string match -nocase $prefix $w]} { + set bSeen 1 + if {$bInst} { + set v [string tolower $w] + if {$w != $v} { append v ".$w" } + lappend row $v + } + } + } + + if {$bSeen} { + lappend ret $rowid + lappend ret $row + } + } + + set ret + } + + proc do_prefixquery_test {tn prefix} { + set bInst [expr {$::e!="contentless" || "%DETAIL%"=="full"}] + set expect [prefixquery $prefix $bInst 0] + set expect2 [prefixquery $prefix $bInst 1] + + uplevel [list do_execsql_test $tn.1 " + SELECT rowid, tokens(ft) FROM ft('$prefix') + " $expect] + uplevel [list do_execsql_test $tn.2 " + SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('$prefix')) + " $expect] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) } + uplevel [list do_execsql_test $tn.3 " + SELECT rowid, tokens(ft) FROM ft('$prefix') + " $expect] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) } + + if {"%DETAIL%"!="none"} { + uplevel [list do_execsql_test $tn.4 " + SELECT rowid, tokens(ft) FROM ft('y: $prefix') + " $expect2] + uplevel [list do_execsql_test $tn.5 " + SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('y: $prefix')) + " $expect2] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) } + uplevel [list do_execsql_test $tn.6 " + SELECT rowid, tokens(ft) FROM ft('y: $prefix') + " $expect2] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) } + } + } + + do_prefixquery_test 1.$e.1 a* + do_prefixquery_test 1.$e.2 b* + do_prefixquery_test 1.$e.3 c* + do_prefixquery_test 1.$e.4 d* + do_prefixquery_test 1.$e.5 e* + do_prefixquery_test 1.$e.6 f* + do_prefixquery_test 1.$e.7 g* + do_prefixquery_test 1.$e.8 h* + do_prefixquery_test 1.$e.9 i* + do_prefixquery_test 1.$e.10 j* +}} + + + +finish_test + diff --git a/ext/fts5/test/fts5phrase.test b/ext/fts5/test/fts5phrase.test index 10598ccf43..708cdfd83e 100644 --- a/ext/fts5/test/fts5phrase.test +++ b/ext/fts5/test/fts5phrase.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5phrase -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -93,15 +93,21 @@ foreach {tn cols tokens} { 10 {b} "i e" 11 {a} "i e" } { - set fts "{$cols}:[join $tokens +]" set where [list] foreach c $cols { lappend where "pmatch($c, '$tokens')" } set where [join $where " OR "] - set res [db eval "SELECT rowid FROM t3 WHERE $where"] - do_execsql_test "1.$tn.$fts->([llength $res] rows)" { - SELECT rowid FROM t3($fts) - } $res + foreach fts [list \ + "{$cols}:[join $tokens +]" \ + "{$cols}:NEAR([join $tokens +])" \ + "{$cols}:NEAR([join $tokens +],1)" \ + "{$cols}:NEAR([join $tokens +],111)" \ + ] { + set res [db eval "SELECT rowid FROM t3 WHERE $where"] + do_execsql_test "1.$tn.$fts->([llength $res] rows)" { + SELECT rowid FROM t3($fts) + } $res + } } do_execsql_test 2.0 { diff --git a/ext/fts5/test/fts5plan.test b/ext/fts5/test/fts5plan.test index 6862acf179..57d5254a35 100644 --- a/ext/fts5/test/fts5plan.test +++ b/ext/fts5/test/fts5plan.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5plan -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5porter.test b/ext/fts5/test/fts5porter.test index c7b1ce6f3f..de1c3e15a3 100644 --- a/ext/fts5/test/fts5porter.test +++ b/ext/fts5/test/fts5porter.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5porter -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5porter2.test b/ext/fts5/test/fts5porter2.test index 6e81b2d310..556060baa3 100644 --- a/ext/fts5/test/fts5porter2.test +++ b/ext/fts5/test/fts5porter2.test @@ -18,7 +18,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5porter2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5prefix.test b/ext/fts5/test/fts5prefix.test index 279f312f22..7a29628ea1 100644 --- a/ext/fts5/test/fts5prefix.test +++ b/ext/fts5/test/fts5prefix.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5prefix -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5prefix2.test b/ext/fts5/test/fts5prefix2.test index bf16e81a73..0860b3cddd 100644 --- a/ext/fts5/test/fts5prefix2.test +++ b/ext/fts5/test/fts5prefix2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5prefix2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -52,6 +52,36 @@ do_execsql_test 2.1 { SELECT * FROM t2('to*'); } {top to tommy} +#------------------------------------------------------------------------- + +foreach {tn newrowid} { + 1 122 + 2 123 + 3 124 +} { + reset_db + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t12 USING fts5(x); + INSERT INTO t12(rowid, x) VALUES(123, 'wwww'); + } + do_execsql_test 3.$tn.1 { + BEGIN; + DELETE FROM t12 WHERE rowid=123; + SELECT * FROM t12('wwww*'); + INSERT INTO t12(rowid, x) VALUES($newrowid, 'wwww'); + SELECT * FROM t12('wwww*'); + END; + } {wwww} + do_execsql_test 3.$tn.2 { + INSERT INTO t12(t12) VALUES('integrity-check'); + } + do_execsql_test 3.$tn.3 { + SELECT rowid FROM t12('wwww*'); + } $newrowid +} + +finish_test + finish_test diff --git a/ext/fts5/test/fts5query.test b/ext/fts5/test/fts5query.test index 5237e8e250..4e8bab8cf7 100644 --- a/ext/fts5/test/fts5query.test +++ b/ext/fts5/test/fts5query.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5query -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5rank.test b/ext/fts5/test/fts5rank.test index 22534e8e03..7a700cb97f 100644 --- a/ext/fts5/test/fts5rank.test +++ b/ext/fts5/test/fts5rank.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rank -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -180,4 +180,28 @@ do_execsql_test 6.1 { {table table table} {the table names.} {rank on an fts5 table} } + +#------------------------------------------------------------------------- +# forum post: https://sqlite.org/forum/forumpost/a2dd636330 +# +reset_db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t USING fts5 (a, b); + INSERT INTO t (a, b) VALUES ('data1', 'sentence1'), ('data2', 'sentence2'); + INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)'); +} + +sqlite3 db2 test.db +do_execsql_test -db db2 1.1 { + SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK; +} {data1 sentence1 1 data2 sentence2 1} + +do_execsql_test 1.2 { + INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)'); +} +do_execsql_test -db db2 1.3 { + SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK; +} {data1 sentence1 1 data2 sentence2 1} +db2 close + finish_test diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test index ae881c02f0..065d16b910 100644 --- a/ext/fts5/test/fts5rebuild.test +++ b/ext/fts5/test/fts5rebuild.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rebuild -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -46,7 +46,7 @@ do_execsql_test 1.5 { do_catchsql_test 1.6 { INSERT INTO f1(f1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('rebuild'); diff --git a/ext/fts5/test/fts5restart.test b/ext/fts5/test/fts5restart.test index db2c62b675..da58fe3aed 100644 --- a/ext/fts5/test/fts5restart.test +++ b/ext/fts5/test/fts5restart.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5restart -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -29,6 +29,7 @@ do_execsql_test 1.0 { # Run the 'optimize' command. Check that it does not disturb ongoing # full-text queries. # +unset -nocomplain lRowid do_test 1.1 { for {set i 1} {$i < 1000} {incr i} { execsql { INSERT INTO f1 VALUES('a b c d e') } diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test index 8935ecfea7..e4e4f6b844 100644 --- a/ext/fts5/test/fts5rowid.test +++ b/ext/fts5/test/fts5rowid.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rowid -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5savepoint.test b/ext/fts5/test/fts5savepoint.test index e431f9f5fd..bf66052282 100644 --- a/ext/fts5/test/fts5savepoint.test +++ b/ext/fts5/test/fts5savepoint.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5savepoint -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -48,7 +48,7 @@ do_catchsql_test 2.0 { SAVEPOINT two; INSERT INTO ft1 VALUES('b'); COMMIT; -} {1 {SQL logic error}} +} {1 {database disk image is malformed}} reset_db ifcapable fts3 { @@ -71,7 +71,7 @@ ifcapable fts3 { do_catchsql_test 3.2 { DROP TABLE vt1; - } {1 {SQL logic error}} + } {0 {}} do_execsql_test 3.3 { SAVEPOINT x; diff --git a/ext/fts5/test/fts5secure.test b/ext/fts5/test/fts5secure.test index 50d84cef79..7314946162 100644 --- a/ext/fts5/test/fts5secure.test +++ b/ext/fts5/test/fts5secure.test @@ -273,6 +273,76 @@ do_execsql_test 5.3 { do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2 do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2 +#------------------------------------------------------------------------- +# Tests for the bug fixed by https://sqlite.org/src/info/4b60a1c3 +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE fts USING fts5(content); + INSERT INTO fts(fts, rank) VALUES ('secure-delete', 1); + INSERT INTO fts(rowid, content) VALUES + (3407, 'profile profile profile profile profile profile profile profile pull pulling pulling really'); + DELETE FROM fts WHERE rowid IS 3407; + INSERT INTO fts(fts) VALUES ('integrity-check'); +} + +foreach {tn detail} { + 1 full + 2 column + 3 none +} { + do_execsql_test 6.1.$detail " + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=$detail); + " + + do_execsql_test 6.2.$detail { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + for {set ii 1} {$ii < 100} {incr ii} { + do_execsql_test 6.3.$detail.$ii.1 { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'word1'); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i0} { - set idx [expr int(abs(rand()) * [llength $in])] - lappend out [lindex $in $idx] - set in [lreplace $in $idx $idx] + proc newdoc {} { + for {set i 0} {$i<8} {incr i} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] + } + set ret + } + db func newdoc newdoc + + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE fff USING fts5(y); + INSERT INTO fff(fff, rank) VALUES('pgsz', 64); + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; } - set out -} - -#dump fff -set iTest 1 -foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { - #if {$iTest==1} { dump fff } - #if {$iTest==1} { breakpoint } - do_execsql_test 3.1.$iTest.$ii { - DELETE FROM fff WHERE rowid=$ii; + execsql $cfg + + proc lshuffle {in} { + set out [list] + while {[llength $in]>0} { + set idx [expr int(abs(rand()) * [llength $in])] + lappend out [lindex $in $idx] + set in [lreplace $in $idx $idx] + } + set out } - #if {$iTest==1} { dump fff } - if {($iTest % 20)==0} { - do_execsql_test 3.1.$iTest.$ii.ic { - INSERT INTO fff(fff) VALUES('integrity-check'); + + #dump fff + + set iTest 1 + foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { + #if {$iTest==1} { dump fff } + #if {$iTest==1} { breakpoint } + do_execsql_test 3.$tn.1.$iTest.$ii { + DELETE FROM fff WHERE rowid=$ii; + } + #if {$iTest==1} { dump fff } + if {($iTest % 20)==0} { + do_execsql_test 3.$tn.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } } + #if {$iTest==1} { break } + incr iTest } - #if {$iTest==1} { break } - incr iTest } #execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC } diff --git a/ext/fts5/test/fts5secure6.test b/ext/fts5/test/fts5secure6.test index e561a43f7d..e2f4ceabc8 100644 --- a/ext/fts5/test/fts5secure6.test +++ b/ext/fts5/test/fts5secure6.test @@ -18,7 +18,7 @@ db progress 1 progress_handler set ::PHC 0 proc progress_handler {args} { incr ::PHC - if {($::PHC % 100000)==0} breakpoint + # if {($::PHC % 100000)==0} breakpoint return 0 } @@ -50,6 +50,92 @@ do_test 1.3 { expr $phc(1)*5 < $phc(2) } {1} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) +} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(-100000, 'abc def ghi'); + INSERT INTO t1(rowid, x) VALUES(-99999, 'abc def ghi'); + INSERT INTO t1(rowid, x) VALUES(9223372036854775800, 'abc def ghi'); + COMMIT; +} + +do_execsql_test 2.2 { + SELECT rowid FROM t1('def') +} {-100000 -99999 9223372036854775800} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) +} + +do_execsql_test 3.1 { + BEGIN; + INSERT INTO t1(rowid, x) + VALUES(51869, 'when whenever where weress what turn'), + (51871, 'to were'); + COMMIT; +} + +do_execsql_test 3.2 { + DELETE FROM t1 WHERE rowid=51871; + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(rowid, x) VALUES(10, 'one two'); +} +do_execsql_test 4.1 { + UPDATE t1 SET x = 'one three' WHERE rowid=10; + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 4.2 { + DELETE FROM t1 WHERE rowid=10; +} +do_execsql_test 4.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(content); + + INSERT INTO t1(t1,rank) VALUES('secure-delete',1); + INSERT INTO t1 VALUES('active'),('boomer'),('atom'),('atomic'), + ('alpha channel backup abandon test aback boomer atom alpha active'); + DELETE FROM t1 WHERE t1 MATCH 'abandon'; +} + +do_execsql_test 5.1 { + INSERT INTO t1(t1) VALUES('rebuild'); +} + +do_execsql_test 5.2 { + DELETE FROM t1 WHERE rowid NOTNULL<5; +} + +db close +sqlite3 db test.db + +do_execsql_test 5.3 { + PRAGMA integrity_check; +} {ok} + finish_test diff --git a/ext/fts5/test/fts5secure7.test b/ext/fts5/test/fts5secure7.test new file mode 100644 index 0000000000..16a044f538 --- /dev/null +++ b/ext/fts5/test/fts5secure7.test @@ -0,0 +1,116 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# TESTRUNNER: slow +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure7 + + +set NVOCAB 500 +set NDOC [expr 1000] + +set NREP 100 +set nDeletePerRep [expr 5] + +set VOCAB [list] + +proc select_one {list} { + set n [llength $list] + lindex $list [expr {abs(int(rand()*$n))}] +} + +proc init_vocab {} { + set L [split "abcdefghijklmnopqrstuvwxyz" {}] + set nL [llength $L] + for {set i 0} {$i < $::NVOCAB} {incr i} { + set n [expr {6 + int(rand()*8)}] + set word "" + for {set j 0} {$j < $n} {incr j} { + append word [select_one $L] + } + lappend ::VOCAB $word + } +} + +proc get_word {} { + select_one $::VOCAB +} + +proc get_document {nWord} { + set ret [list] + for {set i 0} {$i < $nWord} {incr i} { + lappend ret [get_word] + } + return $ret +} + +init_vocab + +db func document [list get_document 12] + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(body); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 1.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC + ) + INSERT INTO t1 SELECT document() FROM s; +} + +for {set iRep 0} {$iRep < $NREP} {incr iRep} { + set lRowid [db eval {SELECT rowid FROM t1}] + for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} { + set idx [select_one $lRowid] + db eval { + DELETE FROM t1 WHERE rowid=$idx + } + } + db eval { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep + ) + INSERT INTO t1 SELECT document() FROM s; + } + do_execsql_test 1.2.$iRep { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} + +reset_db +db func document [list get_document 12] +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(body); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + INSERT INTO t1(t1, rank) VALUES('pgsz', 128); +} +do_execsql_test 2.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC + ) + INSERT INTO t1 SELECT document() FROM s; +} +for {set ii 0} {$ii < $NDOC} {incr ii} { + set lRowid [db eval {SELECT rowid FROM t1}] + set idx [select_one $lRowid] + db eval { DELETE FROM t1 WHERE rowid=$idx } + do_execsql_test 2.2.$ii { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} + +finish_test + + diff --git a/ext/fts5/test/fts5secure8.test b/ext/fts5/test/fts5secure8.test new file mode 100644 index 0000000000..8b65b7c59f --- /dev/null +++ b/ext/fts5/test/fts5secure8.test @@ -0,0 +1,71 @@ +# 2023 Nov 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure8 + +proc sql_repeat {txt n} { + string repeat $txt $n +} +db func repeat sql_repeat + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x); + + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + + INSERT INTO ft(rowid, x) VALUES(100, 'hello world'); + INSERT INTO ft(rowid, x) VALUES(200, 'one day'); + + BEGIN; + INSERT INTO ft(rowid, x) VALUES(45, 'one two three'); + UPDATE ft SET x = repeat('hello world ', 500) WHERE rowid=100; + COMMIT +} + +do_execsql_test 1.1 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); + DELETE FROM ft WHERE rowid=100; +} + +do_execsql_test 1.2 { + PRAGMA integrity_check; +} {ok} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5 ( + name, + content='' + ); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + INSERT INTO xyz (rowid, name) VALUES(1, 'A'); + INSERT INTO xyz (rowid, name) VALUES(2, 'A'); + INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A'); +} + +do_execsql_test 2.1 { + pragma quick_check; +} {ok} + +do_catchsql_test 2.2 { + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 'hello world'); +} {1 {SQL logic error}} + + + + + +finish_test + + diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test index 63874ece5d..2959ab65ce 100644 --- a/ext/fts5/test/fts5securefault.test +++ b/ext/fts5/test/fts5securefault.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5securefault -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. return_if_no_fts5 do_execsql_test 1.0 { diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 936bbb2549..ad59bf0d9e 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -350,7 +350,7 @@ do_execsql_test 14.3 { do_execsql_test 14.4 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' -} {0 {} 3} +} {0 {} 2} #------------------------------------------------------------------------- reset_db @@ -480,4 +480,33 @@ do_execsql_test 22.0 { do_catchsql_test 22.1 {SELECT * FROM x1('')} {1 {fts5: syntax error near ""}} do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.1 { + BEGIN; + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); + INSERT INTO x1 VALUES('a b c d'); +} + +do_execsql_test 23.2 { + SELECT count(*) FROM x1_data; +} {2} + +do_execsql_test 23.3 { + INSERT INTO x1(x1) VALUES('flush'); + SELECT count(*) FROM x1_data; +} {3} + +do_execsql_test 23.4 { + ROLLBACK; + SELECT count(*) FROM x1_data; +} {2} + + finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index e57cea70fa..01c590c9f7 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -343,7 +343,9 @@ do_execsql_test 17.0 { INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); COMMIT; } -do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 } +do_execsql_test 17.1 { + SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 +} do_execsql_test 17.2 { BEGIN; INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test index 0d4972b372..bc3ebfc7ca 100644 --- a/ext/fts5/test/fts5simple3.test +++ b/ext/fts5/test/fts5simple3.test @@ -13,7 +13,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5simple3 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -81,7 +81,7 @@ do_execsql_test 3.0 { } #------------------------------------------------------------------------- -# Test that a crash occuring when the second or subsequent tokens in a +# Test that a crash occurring when the second or subsequent tokens in a # phrase matched zero rows has been fixed. # do_execsql_test 4.0 { diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 86610ee9eb..55e2f186a9 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5synonym -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5synonym2.test b/ext/fts5/test/fts5synonym2.test index 8bbfb07566..ec8b750c57 100644 --- a/ext/fts5/test/fts5synonym2.test +++ b/ext/fts5/test/fts5synonym2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5synonym2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -42,7 +42,7 @@ proc fts5_test_bothlist {cmd} { } sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist -proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] } +proc fts5_rowid {cmd} { expr [$cmd xRowid] } sqlite3_fts5_create_function db fts5_rowid fts5_rowid do_execsql_test 1.$tok.0.1 " @@ -122,6 +122,9 @@ foreach {tn expr} { 4.1 "NEAR(one two, 2)" 4.2 "NEAR(one two three, 2)" 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" + + 5.1 "one + two" + 5.2 "1 + two" } { if {[fts5_expr_ok $expr ss]==0} { do_test 1.$tok.$tn.OMITTED { list } [list] diff --git a/ext/fts5/test/fts5tokendata.test b/ext/fts5/test/fts5tokendata.test new file mode 100644 index 0000000000..7f75f4fa8e --- /dev/null +++ b/ext/fts5/test/fts5tokendata.test @@ -0,0 +1,105 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokendata + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + + sqlite3_fts5_register_origintext db + fts5_aux_test_functions db + proc b {x} { string map [list "\0" "."] $x } + db func b b + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, tokendata=1, + tokenize="origintext unicode61", detail=%DETAIL% + ); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); + } + + do_execsql_test 1.1 { + INSERT INTO ft(rowid, a, b) VALUES + (1, 'Pedagog Pedal Pedant', 'Peculier Day Today'), + (2, 'Pedant pedantic pecked', 'Peck Penalize Pen'); + + INSERT INTO ft(rowid, a, b) VALUES + (3, 'Penalty Pence Penciled', 'One Two Three'), + (4, 'Pedant Pedal Pedant', 'Peculier Day Today'); + } + + do_execsql_test 1.2 { + SELECT DISTINCT b(term) FROM vocab + } { + day.Day one.One peck.Peck pecked peculier.Peculier pedagog.Pedagog + pedal.Pedal pedant.Pedant pedantic pen.Pen penalize.Penalize + penalty.Penalty pence.Pence penciled.Penciled three.Three + today.Today two.Two + } + + do_execsql_test 1.3.1 { + SELECT rowid FROM ft('pe*') + } { + 1 2 3 4 + } + + do_execsql_test 1.3.2 { + SELECT rowid FROM ft('pe*') ORDER BY rowid DESC + } { + 4 3 2 1 + } + + if {"%DETAIL%"!="none"} { + do_execsql_test 1.3.3 { + SELECT rowid FROM ft WHERE a MATCH 'pe*' ORDER BY rowid DESC + } { + 4 3 2 1 + } + } + + do_execsql_test 1.4 { + SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pedant') + } { + 1 pedant.Pedant + 2 pedant.Pedant + 4 pedant.Pedant + } + + do_execsql_test 1.5 { + SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pe*') + } { + 1 pedagog.Pedagog + 2 pedant.Pedant + 3 penalty.Penalty + 4 pedant.Pedant + } + + do_execsql_test 1.6 { + SELECT rowid, fts5_test_poslist(ft) FROM ft('pe*') + } { + 1 {0.0.0 0.0.1 0.0.2 0.1.0} + 2 {0.0.0 0.0.1 0.0.2 0.1.0 0.1.1 0.1.2} + 3 {0.0.0 0.0.1 0.0.2} + 4 {0.0.0 0.0.1 0.0.2 0.1.0} + } +} + +finish_test + diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 27370657ed..a828e3a22b 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5tokenizer -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -300,5 +300,75 @@ set ::flags [list] do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} do_test 9.5.2 { set ::flags } {query} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 10.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61); + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="unicode61 error");' + WHERE name = 'x1'; +} + +db close +sqlite3 db test.db + +do_catchsql_test 10.2 { + SELECT * FROM x1('abc'); +} {1 {error in tokenizer constructor}} + +do_catchsql_test 10.3 { + INSERT INTO x1 VALUES('abc'); +} {1 {error in tokenizer constructor}} + +do_execsql_test 10.4 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize="nosuch error");' + WHERE name = 'x1'; +} + +db close +sqlite3 db test.db + +do_catchsql_test 10.5 { + SELECT * FROM x1('abc'); +} {1 {no such tokenizer: nosuch}} +do_catchsql_test 10.6 { + INSERT INTO x1 VALUES('abc'); +} {1 {no such tokenizer: nosuch}} + +do_execsql_test 10.7 { + DROP TABLE x1; + SELECT * FROM sqlite_schema; +} + +reset_db +do_execsql_test 10.8 { + CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=unicode61); + INSERT INTO x1 VALUES('a b c'), ('d e f'), ('a b c'); + CREATE VIRTUAL TABLE x1v USING fts5vocab(x1, row); + + PRAGMA writable_schema = 1; + UPDATE sqlite_schema + SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(x, tokenize=simplify);' + WHERE name = 'x1'; +} + +do_execsql_test 10.9 { + SELECT * FROM x1v +} { + a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1 +} + +db close +sqlite3 db test.db + +do_execsql_test 10.10 { + SELECT * FROM x1v +} { + a 2 2 b 2 2 c 2 2 d 1 1 e 1 1 f 1 1 +} finish_test diff --git a/ext/fts5/test/fts5tokenizer2.test b/ext/fts5/test/fts5tokenizer2.test new file mode 100644 index 0000000000..4fe31d22c4 --- /dev/null +++ b/ext/fts5/test/fts5tokenizer2.test @@ -0,0 +1,109 @@ +# 2023 Nov 03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer +proc get_tst_tokenizer {args} { + return "tst_tokenizer" +} +proc tst_tokenizer {flags txt} { + set token "" + set lTok [list] + + foreach c [split $txt {}] { + if {$token==""} { + append token $c + } else { + set t1 [string is upper $token] + set t2 [string is upper $c] + + if {$t1!=$t2} { + lappend lTok $token + set token "" + } + append token $c + } + } + if {$token!=""} { lappend lTok $token } + + set iOff 0 + foreach t $lTok { + set n [string length $t] + sqlite3_fts5_token $t $iOff [expr $iOff+$n] + incr iOff $n + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('AAdontBBmess'); +} + +do_execsql_test 1.2 { + SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB'); +} {AAdont>BB', '<') FROM t1('BB'); +} {AAdont>BB', '<') FROM t1('AA'); +} {>AA', '<') FROM t1('dont'); +} {AA>dont', '<') FROM t1('mess'); +} {AAdontBB>mess<} + +do_execsql_test 1.7 { + SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess'); +} {AAdont>BBmess<} + +# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd +# Error handling of tokenize= arguments. +# +foreach {n tkz} { + 1 {ascii none} + 2 {unicode61 none} + 3 {porter none} + 4 {trigram none} + 5 {ascii none 0} + 6 {unicode61 none 0} + 7 {porter none 0} + 8 {trigram none 0} +} { + db eval {DROP TABLE IF EXISTS t2;} + do_catchsql_test 2.$n " + DROP TABLE IF EXISTS t2; + CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz'); + " {1 {error in tokenizer constructor}} +} + + +finish_test diff --git a/ext/fts5/test/fts5tokenizer3.test b/ext/fts5/test/fts5tokenizer3.test new file mode 100644 index 0000000000..5cdab743c2 --- /dev/null +++ b/ext/fts5/test/fts5tokenizer3.test @@ -0,0 +1,77 @@ +# 2024 Aug 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on the built-in fts5 tokenizers. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokenizer3 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +proc get_sod {args} { return "split_on_dot" } +proc get_lowercase {args} { return "lowercase" } + +proc lowercase {flags txt} { + set n [string length $txt] + sqlite3_fts5_token [string tolower $txt] 0 $n + return 0 +} + +proc split_on_dot {flags txt} { + set iOff 0 + foreach t [split $txt "."] { + set n [string length $txt] + sqlite3_fts5_token $t $iOff [expr $iOff+$n] + incr iOff [expr {$n+1}] + } + return "" +} + +foreach {tn script} { + 1 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 2 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod + } + 3 { + sqlite3_fts5_create_tokenizer db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } + 4 { + sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase + sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod + } +} { + reset_db + eval $script + + do_execsql_test 1.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot); + CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance); + INSERT INTO t1 VALUES('ABC.Def.ghi'); + } + + do_execsql_test 1.$tn.1 { + SELECT term FROM t1vocab ORDER BY 1 + } {abc def ghi} +} + + +finish_test diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test index 951daf1440..377e3f7813 100644 --- a/ext/fts5/test/fts5trigram.test +++ b/ext/fts5/test/fts5trigram.test @@ -56,6 +56,7 @@ foreach {tn like res} { 7 {ABCDEFG%} 1 8 {%รุงเ%} 2 9 {%งเ%} 2 + 10 {%"งเ"%} {} } { do_execsql_test 1.3.$tn { SELECT rowid FROM t1 WHERE y LIKE $like @@ -69,6 +70,9 @@ do_execsql_test 2.0 { INSERT INTO t1 VALUES('abcdefghijklm'); INSERT INTO t1 VALUES('กรุงเทพมหานคร'); } +do_catchsql_test 2.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram case_sensitive'); +} {1 {error in tokenizer constructor}} foreach {tn s res} { 1 abc "(abc)defghijklm" @@ -197,6 +201,12 @@ do_eqp_test 6.3 { do_eqp_test 6.4 { SELECT * FROM ci1 WHERE x GLOB ? } {VIRTUAL TABLE INDEX 0:G0} +do_eqp_test 6.5 { + SELECT * FROM ci1 WHERE x < ? +} {{SCAN ci1 VIRTUAL TABLE INDEX 0:}} +do_eqp_test 6.6 { + SELECT * FROM ci0 WHERE x < ? +} {{SCAN ci0 VIRTUAL TABLE INDEX 0:}} reset_db do_execsql_test 7.0 { @@ -206,7 +216,7 @@ do_execsql_test 7.0 { (20, "жираф.png"), (30, "cat.png"), (40, "кот.png"), - (50, "misic-🎵-.mp3"); + (50, "misic-🎵-.mp3"); } do_execsql_test 7.1 { SELECT rowid FROM f WHERE +filename GLOB '*ир*'; @@ -215,4 +225,142 @@ do_execsql_test 7.2 { SELECT rowid FROM f WHERE filename GLOB '*ир*'; } {20} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram); + INSERT INTO t1 VALUES('abcdefghijklm'); +} + +foreach {tn match res} { + 1 "abc ghi" "(abc)def(ghi)jklm" + 2 "def ghi" "abc(defghi)jklm" + 3 "efg ghi" "abcd(efghi)jklm" + 4 "efghi" "abcd(efghi)jklm" + 5 "abcd jklm" "(abcd)efghi(jklm)" + 6 "ijkl jklm" "abcdefgh(ijklm)" + 7 "ijk ijkl hijk" "abcdefg(hijkl)m" + +} { + do_execsql_test 8.1.$tn { + SELECT highlight(t1, 0, '(', ')') FROM t1($match) + } $res +} + +do_execsql_test 8.2 { + CREATE VIRTUAL TABLE ft2 USING fts5(a, tokenize="trigram"); + INSERT INTO ft2 VALUES('abc x cde'); + INSERT INTO ft2 VALUES('abc cde'); + INSERT INTO ft2 VALUES('abcde'); +} + +do_execsql_test 8.3 { + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'abc AND cde'; +} { + {[abc] x [cde]} + {[abc] [cde]} + {[abcde]} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + tokenize=trigram + ); + + INSERT INTO t1(rowid, a12) VALUES(111, 'thats a tricky case though'); + INSERT INTO t1(rowid, a12) VALUES(222, 'the query planner cannot do'); +} + +do_execsql_test 9.1 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' +} {111} + +do_execsql_test 9.2 { + SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' AND a12 LIKE '%case%' +} {111} + +do_execsql_test 9.3 { + SELECT rowid FROM t1 WHERE a12 LIKE NULL +} {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=trigram); +} + +do_test 10.1 { + foreach {val} { + "abc \UFFjkl\UFF" + "abc \UFFFjkl\UFFF" + "abc \UFFFFjkl\UFFFF" + "abc \UFFFFFjkl\UFFFFF" + "\UFFjkl\UFF abc" + "\UFFFjkl\UFFF abc" + "\UFFFFjkl\UFFFF abc" + "\UFFFFFjkl\UFFFFF abc" + "\U10001jkl\U10001 abc" + } { + execsql { INSERT INTO t1 VALUES( $val ) } + } +} {} + +do_test 10.2 { + foreach {val} { + X'E18000626320646566' + X'61EDA0806320646566' + X'61EDA0806320646566' + X'61EFBFBE6320646566' + X'76686920E18000626320646566' + X'7668692061EDA0806320646566' + X'7668692061EDA0806320646566' + X'7668692061EFBFBE6320646566' + } { + execsql " INSERT INTO t1 VALUES( $val ) " + } +} {} + +do_test 10.3 { + set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}] + set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}] + set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}] + execsql { + INSERT INTO t1 VALUES($a); + INSERT INTO t1 VALUES($b); + INSERT INTO t1 VALUES($c); + INSERT INTO t1 VALUES($d); + + INSERT INTO t1 VALUES('abcd' || $a); + INSERT INTO t1 VALUES('abcd' || $b); + INSERT INTO t1 VALUES('abcd' || $c); + INSERT INTO t1 VALUES('abcd' || $d); + } +} {} + +do_execsql_test 11.0 { + CREATE VIRTUAL TABLE t4 USING fts5(y, tokenize=trigram); +} +sqlite3_fts5_register_str db +do_execsql_test 11.1 { + INSERT INTO t4 VALUES( str('') ); +} + +do_test 12.0 { + sqlite3_fts5_tokenize db trigram "abcd" +} {abc 0 3 bcd 1 4} + +do_test 12.1 { + sqlite3_fts5_tokenize db trigram "a" +} {} + +do_test 12.2 { + sqlite3_fts5_tokenize db trigram "" +} {} + finish_test + diff --git a/ext/fts5/test/fts5trigram2.test b/ext/fts5/test/fts5trigram2.test new file mode 100644 index 0000000000..c81684a22b --- /dev/null +++ b/ext/fts5/test/fts5trigram2.test @@ -0,0 +1,137 @@ +# 2023 October 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Tests for the fts5 "trigram" tokenizer. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5trigram2 + +do_execsql_test 1.0 " + CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1'); + INSERT INTO t1 VALUES('abc\u0303defghijklm'); + INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm'); +" +do_catchsql_test 1.0.1 { + CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram remove_diacritics'); +} {1 {error in tokenizer constructor}} + +do_execsql_test 1.1 { + SELECT highlight(t1, 0, '(', ')') FROM t1('abc'); +} [list \ + "(abc\u0303)defghijklm" \ + "(a\u0303b\u0303c\u0303)defghijklm" \ +] + +do_execsql_test 1.2 { + SELECT highlight(t1, 0, '(', ')') FROM t1('bcde'); +} [list \ + "a(bc\u0303de)fghijklm" \ + "a\u0303(b\u0303c\u0303de)fghijklm" \ +] + +do_execsql_test 1.3 { + SELECT highlight(t1, 0, '(', ')') FROM t1('cdef'); +} [list \ + "ab(c\u0303def)ghijklm" \ + "a\u0303b\u0303(c\u0303def)ghijklm" \ +] + +do_execsql_test 1.4 { + SELECT highlight(t1, 0, '(', ')') FROM t1('def'); +} [list \ + "abc\u0303(def)ghijklm" \ + "a\u0303b\u0303c\u0303(def)ghijklm" \ +] + + +#------------------------------------------------------------------------- +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE t2 USING fts5( + z, tokenize='trigram case_sensitive 1 remove_diacritics 1' + ); +} {1 {error in tokenizer constructor}} + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t2 USING fts5( + z, tokenize='trigram case_sensitive 0 remove_diacritics 1' + ); +} +do_execsql_test 2.2 " + INSERT INTO t2 VALUES('\u00E3bcdef'); + INSERT INTO t2 VALUES('b\u00E3cdef'); + INSERT INTO t2 VALUES('bc\u00E3def'); + INSERT INTO t2 VALUES('bcd\u00E3ef'); +" + +do_execsql_test 2.3 { + SELECT highlight(t2, 0, '(', ')') FROM t2('abc'); +} "(\u00E3bc)def" +do_execsql_test 2.4 { + SELECT highlight(t2, 0, '(', ')') FROM t2('bac'); +} "(b\u00E3c)def" +do_execsql_test 2.5 { + SELECT highlight(t2, 0, '(', ')') FROM t2('bca'); +} "(bc\u00E3)def" +do_execsql_test 2.6 " + SELECT highlight(t2, 0, '(', ')') FROM t2('\u00E3bc'); +" "(\u00E3bc)def" + +#------------------------------------------------------------------------- +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5( + z, tokenize='trigram remove_diacritics 1' + ); +} {} +do_execsql_test 3.1 " + INSERT INTO t3 VALUES ('\u0303abc\u0303'); +" +do_execsql_test 3.2 { + SELECT highlight(t3, 0, '(', ')') FROM t3('abc'); +} "\u0303(abc\u0303)" + +#------------------------------------------------------------------------- +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram); +} {} + +do_execsql_test 4.1 { + INSERT INTO t4 VALUES('ABCD'); + INSERT INTO t4 VALUES('DEFG'); +} {} + +db close +sqlite3 db test.db + +do_eqp_test 4.1 { + SELECT rowid FROM t4 WHERE z LIKE '%abc%' +} {VIRTUAL TABLE INDEX 0:L0} + +do_execsql_test 4.2 { + SELECT rowid FROM t4 WHERE z LIKE '%abc%' +} {1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t5 USING fts5( + c1, tokenize='trigram', detail='none' + ); + INSERT INTO t5(rowid, c1) VALUES(1, 'abc_____xyx_yxz'); + INSERT INTO t5(rowid, c1) VALUES(2, 'abc_____xyxz'); + INSERT INTO t5(rowid, c1) VALUES(3, 'ac_____xyxz'); +} {} +do_execsql_test 5.1 { + SELECT rowid FROM t5 WHERE c1 LIKE 'abc%xyxz' +} {2} + +finish_test diff --git a/ext/fts5/test/fts5ubsan.test b/ext/fts5/test/fts5ubsan.test index 2dc0aa7bd4..76382a1e15 100644 --- a/ext/fts5/test/fts5ubsan.test +++ b/ext/fts5/test/fts5ubsan.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5ubsan -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5unicode.test b/ext/fts5/test/fts5unicode.test index e2d0f60124..f10e0d02d8 100644 --- a/ext/fts5/test/fts5unicode.test +++ b/ext/fts5/test/fts5unicode.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unicode -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -60,6 +60,7 @@ do_execsql_test 2.1 " # require 17 or more bits to store). # +unset -nocomplain A B C D set A [db one {SELECT char(0x1F75E)}] ;# Type So set B [db one {SELECT char(0x1F5FD)}] ;# Type So set C [db one {SELECT char(0x2F802)}] ;# Type Lo diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test index 662b9dd87b..7a49a1d83f 100644 --- a/ext/fts5/test/fts5unicode2.test +++ b/ext/fts5/test/fts5unicode2.test @@ -17,7 +17,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unicode2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -116,6 +116,7 @@ set docs [list { connected by OR. }] +unset -nocomplain map set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS @@ -470,119 +471,23 @@ do_execsql_test 8.2.3 { } {2 4} #------------------------------------------------------------------------- -# -if 0 { -foreach {tn sql} { - 1 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]); - CREATE VIRTUAL TABLE t6 USING fts4( - tokenize=unicode61 [tokenchars=="] "tokenchars=[]"); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]); - } - 2 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= ."); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]"); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4"); - } - 3 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .'); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]'); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4'); - } - 4 { - CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`); - CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`); - CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`); - } -} { - do_execsql_test 9.$tn.0 { - DROP TABLE IF EXISTS t5; - DROP TABLE IF EXISTS t5aux; - DROP TABLE IF EXISTS t6; - DROP TABLE IF EXISTS t6aux; - DROP TABLE IF EXISTS t7; - DROP TABLE IF EXISTS t7aux; - } - do_execsql_test 9.$tn.1 $sql - - do_execsql_test 9.$tn.2 { - CREATE VIRTUAL TABLE t5aux USING fts4aux(t5); - INSERT INTO t5 VALUES('one two three/four.five.six'); - SELECT * FROM t5aux; - } { - four.five.six * 1 1 four.five.six 0 1 1 - {one two three} * 1 1 {one two three} 0 1 1 - } - - do_execsql_test 9.$tn.3 { - CREATE VIRTUAL TABLE t6aux USING fts4aux(t6); - INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta'); - SELECT * FROM t6aux; - } { - {alpha=beta"gamma} * 1 1 {alpha=beta"gamma} 0 1 1 - {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1 - } - - do_execsql_test 9.$tn.4 { - CREATE VIRTUAL TABLE t7aux USING fts4aux(t7); - INSERT INTO t7 VALUES('alephxbeth\xC4gimel'); - SELECT * FROM t7aux; - } { - aleph * 1 1 aleph 0 1 1 - beth * 1 1 beth 0 1 1 - gimel * 1 1 gimel 0 1 1 - } -} - -# Check that multiple options are handled correctly. -# -do_execsql_test 10.1 { - DROP TABLE IF EXISTS t1; - CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61 - "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy" - "separators=a" "separators=a" "tokenchars=a" "tokenchars=a" - ); - - INSERT INTO t1 VALUES('oneatwoxthreeyfour'); - INSERT INTO t1 VALUES('a.single=word'); - CREATE VIRTUAL TABLE t1aux USING fts4aux(t1); - SELECT * FROM t1aux; -} { - .single=word * 1 1 .single=word 0 1 1 - four * 1 1 four 0 1 1 - one * 1 1 one 0 1 1 - three * 1 1 three 0 1 1 - two * 1 1 two 0 1 1 -} - -# Test that case folding happens after tokenization, not before. -# -do_execsql_test 10.2 { - DROP TABLE IF EXISTS t2; - CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB"); - INSERT INTO t2 VALUES('oneatwoBthree'); - INSERT INTO t2 VALUES('onebtwoAthree'); - CREATE VIRTUAL TABLE t2aux USING fts4aux(t2); - SELECT * FROM t2aux; -} { - one * 1 1 one 0 1 1 - onebtwoathree * 1 1 onebtwoathree 0 1 1 - three * 1 1 three 0 1 1 - two * 1 1 two 0 1 1 -} -# Test that the tokenchars and separators options work with the -# fts3tokenize table. -# -do_execsql_test 11.1 { - CREATE VIRTUAL TABLE ft1 USING fts3tokenize( - "unicode61", "tokenchars=@.", "separators=1234567890" - ); - SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road'; +foreach {tn val bErr} { + 1 0 0 + 2 1 0 + 3 2 0 + 4 3 1 + 5 11 1 } { - berlin@street sydney.road -} - + reset_db + set aRes(0) {0 {}} + set aRes(1) {1 {error in tokenizer constructor}} + set res $aRes($bErr) + do_catchsql_test 9.1.$tn " + CREATE VIRTUAL TABLE bl USING fts5( + s, tokenize='trigram remove_diacritics $val' + ); + " $res } finish_test diff --git a/ext/fts5/test/fts5unicode3.test b/ext/fts5/test/fts5unicode3.test index 30eb3c4166..ddb61a9997 100644 --- a/ext/fts5/test/fts5unicode3.test +++ b/ext/fts5/test/fts5unicode3.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5unicode4.test b/ext/fts5/test/fts5unicode4.test index dfd7f5a254..f006d6c0a6 100644 --- a/ext/fts5/test/fts5unicode4.test +++ b/ext/fts5/test/fts5unicode4.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unicode4 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -28,4 +28,34 @@ do_execsql_test 1.1 { INSERT INTO sss VALUES('まりや'); } +foreach {tn enc tok} { + 1 utf-8 ascii + 2 utf-16 ascii + 3 utf-8 unicode61 + 4 utf-16 unicode61 +} { + reset_db + + do_execsql_test 1.$tn.0 " + PRAGMA encoding = '$enc'; + CREATE VIRTUAL TABLE vt2 USING fts5(c0, c1, tokenize=$tok); + " + + do_execsql_test 1.$tn.1 { + INSERT INTO vt2(c0, c1) VALUES ('bhal', x'17db'); + } + + do_execsql_test 1.$tn.2 { + UPDATE vt2 SET c0='bhal'; + } + + do_execsql_test 1.$tn.3 { + INSERT INTO vt2(vt2) VALUES('integrity-check') + } + + do_execsql_test 1.$tn.4 { + SELECT quote(c1) FROM vt2 + } {X'17DB'} +} + finish_test diff --git a/ext/fts5/test/fts5unindexed.test b/ext/fts5/test/fts5unindexed.test index 8b72c4c776..5099a89693 100644 --- a/ext/fts5/test/fts5unindexed.test +++ b/ext/fts5/test/fts5unindexed.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5unindexed -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5unindexed2.test b/ext/fts5/test/fts5unindexed2.test new file mode 100644 index 0000000000..c0abfc3980 --- /dev/null +++ b/ext/fts5/test/fts5unindexed2.test @@ -0,0 +1,297 @@ +# 2024 Sep 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file focus on "unindexed" columns in contentless +# tables. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5unindexed2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts5( + a, b UNINDEXED, content=, contentless_unindexed=1 + ); +} {} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('abc def', 'ghi jkl'); +} + +do_execsql_test 1.3 { + SELECT rowid, a, b FROM t1 +} {1 {} {ghi jkl}} + +do_execsql_test 1.4 { + INSERT INTO t1(rowid, a, b) VALUES(11, 'hello world', 'one two three'); +} + +do_execsql_test 1.5 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'abc def', 'ghi jkl'); +} + +do_execsql_test 1.6 { + SELECT rowid, a, b FROM t1 +} { + 11 {} {one two three} +} + +do_execsql_test 1.7 { + PRAGMA integrity_check +} {ok} + +do_execsql_test 1.8 { + INSERT INTO t1(rowid, a, b) VALUES(12, 'abc def', 'ghi jkl'); +} + +do_execsql_test 1.9 { + SELECT rowid, a, b FROM t1('def') +} {12 {} {ghi jkl}} + +do_execsql_test 1.10 { + SELECT rowid, a, b FROM t1('def OR hello') ORDER BY rank +} {11 {} {one two three} 12 {} {ghi jkl}} + +do_execsql_test 1.11 { + SELECT rowid, a, b FROM t1 WHERE rowid=11 +} {11 {} {one two three}} + +do_execsql_test 1.12 { + SELECT rowid, a, b FROM t1 +} {11 {} {one two three} 12 {} {ghi jkl}} + + +fts5_aux_test_functions db +do_execsql_test 1.12.2 { + SELECT rowid, fts5_test_columntext(t1) FROM t1('def OR hello') +} {11 {{} {one two three}} 12 {{} {ghi jkl}}} + +do_execsql_test 1.13 { + INSERT INTO t1(t1) VALUES('delete-all'); +} + +do_execsql_test 1.14 { + SELECT rowid, a, b FROM t1 +} + +do_execsql_test 1.15 { + PRAGMA integrity_check +} {ok} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t4 USING fts5( + x, y UNINDEXED, z, columnsize=0, content='', contentless_unindexed=1 + ); +} + +do_execsql_test 2.1 { + INSERT INTO t4(rowid, x, y, z) VALUES(1, 'a a', 'b b b', 'c'); +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5( + a UNINDEXED, b, c UNINDEXED, d, content=, contentless_delete=1, + contentless_unindexed=1 + ); +} + +do_execsql_test 3.1 { + INSERT INTO x1(rowid, a, b, c, d) VALUES(131, 'aaa', 'bbb', 'ccc', 'ddd'); +} + +do_execsql_test 3.2 { + SELECT * FROM x1 +} {aaa {} ccc {}} + +do_execsql_test 3.3 { + INSERT INTO x1(rowid, a, b, c, d) VALUES(1000, 'AAA', 'BBB', 'CCC', 'DDD'); +} + +do_execsql_test 3.4 { + SELECT rowid, * FROM x1 +} { + 131 aaa {} ccc {} + 1000 AAA {} CCC {} +} + +do_execsql_test 3.5 { + DELETE FROM x1 WHERE rowid=131; + SELECT rowid, * FROM x1 +} { + 1000 AAA {} CCC {} +} + +do_execsql_test 3.6 { + INSERT INTO x1(rowid, a, b, c, d) VALUES(112, 'aaa', 'bbb', 'ccc', 'ddd'); + SELECT rowid, * FROM x1 +} { + 112 aaa {} ccc {} + 1000 AAA {} CCC {} +} + +do_execsql_test 3.7 { + UPDATE x1 SET b='hello', d='world', rowid=1120 WHERE rowid=112 +} + +do_execsql_test 3.8 { + SELECT rowid, * FROM x1 +} { + 1000 AAA {} CCC {} + 1120 aaa {} ccc {} +} + +do_execsql_test 3.9 { + SELECT rowid, * FROM x1('hello'); +} { + 1120 aaa {} ccc {} +} + +do_execsql_test 3.9 { + SELECT rowid, * FROM x1('bbb'); +} { + 1000 AAA {} CCC {} +} + +fts5_aux_test_functions db +do_execsql_test 3.10 { + SELECT rowid, fts5_test_columntext(x1) FROM x1('b*') +} {1000 {AAA {} CCC {}}} + +#------------------------------------------------------------------------- +# Check that if contentless_unindexed=1 is not specified, the values +# of UNINDEXED columns are not stored in the database. +# +# Also check that contentless_unindexed=1 is not allowed unless the table +# is actually contentless. +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, c UNINDEXED, content=''); + INSERT INTO ft VALUES('one', 'two', 'three'); + SELECT rowid, * FROM ft; +} {1 {} {} {}} + +do_execsql_test 4.1 { + SELECT name FROM sqlite_schema ORDER BY 1 +} { + ft ft_config ft_data ft_docsize ft_idx +} + +do_catchsql_test 4.2 { + CREATE VIRTUAL TABLE ft2 USING fts5( + a, b, c UNINDEXED, contentless_unindexed=1 + ); +} {1 {contentless_unindexed=1 requires a contentless table}} + +do_catchsql_test 4.3 { + DELETE FROM ft WHERE rowid=1 +} {1 {cannot DELETE from contentless fts5 table: ft}} + +#------------------------------------------------------------------------- +# Check that the usual restrictions on contentless tables apply to +# contentless_unindexed=1 tables. +# +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ft USING fts5( + a, b UNINDEXED, c, content='', contentless_unindexed=1 + ); + INSERT INTO ft VALUES('one', 'two', 'three'); + INSERT INTO ft VALUES('four', 'five', 'six'); + INSERT INTO ft VALUES('seven', 'eight', 'nine'); + SELECT rowid, * FROM ft; +} { + 1 {} two {} + 2 {} five {} + 3 {} eight {} +} + +do_execsql_test 5.1 { + PRAGMA integrity_check +} {ok} + +do_catchsql_test 5.2 { + DELETE FROM ft WHERE rowid=2 +} {1 {cannot DELETE from contentless fts5 table: ft}} + +do_execsql_test 5.3 { + SELECT rowid, * FROM ft('six') +} { + 2 {} five {} +} + +do_catchsql_test 5.4 { + UPDATE ft SET a='x', b='y', c='z' WHERE rowid=3 +} {1 {cannot UPDATE contentless fts5 table: ft}} + +fts5_aux_test_functions db + +do_execsql_test 5.5 { + SELECT fts5_test_columntext(ft) FROM ft WHERE rowid=3 +} { + {{} eight {}} +} +do_execsql_test 5.6 { + SELECT fts5_test_columntext(ft) FROM ft('three'); +} { + {{} two {}} +} + +#------------------------------------------------------------------------- +# Check that it is possible to UPDATE a contentless_unindexed=1 table +# if the only columns being modified are UNINDEXED. +# +# If the contentless_unindexed=1 table is also contentless_delete=1, then +# it is also possible to update indexed columns - but only if *all* indexed +# columns are updated. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, c UNINDEXED, d, + contentless_unindexed=1, content='' + ); + + INSERT INTO ft1(rowid, a, b, c, d) VALUES + (100, 'x y', 'b1', 'c1', 'a b'), + (200, 'c d', 'b2', 'c2', 'a b'), + (300, 'e f', 'b3', 'c3', 'a b'); +} + +do_execsql_test 6.1 { + UPDATE ft1 SET b='b1.1', c='c1.1' WHERE rowid=100; +} +do_execsql_test 6.2 { + UPDATE ft1 SET b='b2.1' WHERE rowid=200; +} +do_execsql_test 6.3 { + UPDATE ft1 SET c='c3.1' WHERE rowid=300; +} + +do_execsql_test 6.4 { + SELECT rowid, a, b, c, d FROM ft1 +} { + 100 {} b1.1 c1.1 {} + 200 {} b2.1 c2 {} + 300 {} b3 c3.1 {} +} + +finish_test + diff --git a/ext/fts5/test/fts5update2.test b/ext/fts5/test/fts5update2.test new file mode 100644 index 0000000000..d04af4800d --- /dev/null +++ b/ext/fts5/test/fts5update2.test @@ -0,0 +1,177 @@ +# 2024 Sep 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5update2 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +#------------------------------------------------------------------------- +# Test that the various types of UPDATE statement are handled correctly +# by different table types. +# +foreach_detail_mode $testprefix { +foreach {tn cu} { + 1 0 + 2 1 +} { + reset_db + do_execsql_test 1.$tn.1 " + CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, c UNINDEXED, d, + content='', + contentless_unindexed=$cu, + detail=%DETAIL% + ); + CREATE VIRTUAL TABLE ft2 USING fts5(a, b UNINDEXED, c UNINDEXED, d, + content='', + contentless_unindexed=$cu, contentless_delete=1, + detail=%DETAIL% + ); + " + + do_execsql_test 1.$tn.2 { + INSERT INTO ft1(rowid, a, b, c, d) VALUES(1, 'a1', 'b1', 'c1', 'd1'); + INSERT INTO ft1(rowid, a, b, c, d) VALUES(2, 'a2', 'b2', 'c2', 'd2'); + INSERT INTO ft1(rowid, a, b, c, d) VALUES(3, 'a3', 'b3', 'c3', 'd3'); + + INSERT INTO ft2(rowid, a, b, c, d) VALUES(1, 'a1', 'b1', 'c1', 'd1'); + INSERT INTO ft2(rowid, a, b, c, d) VALUES(2, 'a2', 'b2', 'c2', 'd2'); + INSERT INTO ft2(rowid, a, b, c, d) VALUES(3, 'a3', 'b3', 'c3', 'd3'); + } + + # It should be possible to update a subset of the UNINDEXED columns of + # a contentless table. Regardless of whether or not contentless_unindexed=1 + # or contentless_delete=1 is set. + do_execsql_test 1.$tn.3 { + UPDATE ft1 SET b=b||'.1'; + UPDATE ft2 SET b=b||'.1'; + } + do_execsql_test 1.$tn.4 { + UPDATE ft1 SET b=b||'.2', c=c||'.2'; + UPDATE ft2 SET b=b||'.2', c=c||'.2'; + } + + set res(0) { + 1 {} {} {} {} + 2 {} {} {} {} + 3 {} {} {} {} + } + set res(1) { + 1 {} b1.1.2 c1.2 {} + 2 {} b2.1.2 c2.2 {} + 3 {} b3.1.2 c3.2 {} + } + + do_execsql_test 1.$tn.5 { + SELECT rowid, * FROM ft2 + } $res($cu) + + do_execsql_test 1.6.1 { SELECT rowid FROM ft1('a2') } {2} + do_execsql_test 1.6.2 { SELECT rowid FROM ft2('a2') } {2} + + # It should be possible to update all indexed columns (but no other subset) + # if the contentless_delete=1 option is set, as it is for "ft2". + do_execsql_test 1.$tn.7 { + UPDATE ft2 SET a='a22', d='d22' WHERE rowid=2; + } + do_execsql_test 1.$tn.8 { SELECT rowid FROM ft2('a22 AND d22') } {2} + + do_execsql_test 1.$tn.9 { + UPDATE ft2 SET a='a33', d='d33', b='b3' WHERE rowid=3; + } + + set res(1) { + 1 {} b1.1.2 c1.2 {} + 2 {} b2.1.2 c2.2 {} + 3 {} b3 c3.2 {} + } + do_execsql_test 1.$tn.10 { + SELECT rowid, * FROM ft2 + } $res($cu) + + do_catchsql_test 1.$tn.11 { + UPDATE ft2 SET a='a11' WHERE rowid=1 + } {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: ft2}} + do_catchsql_test 1.$tn.12 { + UPDATE ft2 SET d='d11' WHERE rowid=1 + } {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: ft2}} + + # It is not possible to update the values of indexed columns if + # contentless_delete=1 is not set. + do_catchsql_test 1.$tn.13 { + UPDATE ft1 SET a='a11' WHERE rowid=1 + } {1 {cannot UPDATE contentless fts5 table: ft1}} + do_catchsql_test 1.$tn.14 { + UPDATE ft1 SET d='d11' WHERE rowid=1 + } {1 {cannot UPDATE contentless fts5 table: ft1}} + + # It should be possible to update the rowid if contentless_delete=1 is + # set and all indexed columns are updated. + do_execsql_test 1.$tn.15 { + UPDATE ft2 SET a='aXone', d='dXone', rowid=11 WHERE rowid=1 + } + + set res(0) { + 2 {} {} {} {} + 3 {} {} {} {} + 11 {} {} {} {} + } + set res(1) { + 2 {} b2.1.2 c2.2 {} + 3 {} b3 c3.2 {} + 11 {} b1.1.2 c1.2 {} + } + do_execsql_test 1.$tn.16 { + SELECT rowid, * FROM ft2 + } $res($cu) + + # Should not be possible to update the rowid of a contentless_delete=1 + # table if no indexed columns are updated. + do_catchsql_test 1.$tn.17 { + UPDATE ft2 SET rowid=12 WHERE rowid=11 + } {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: ft2}} + do_catchsql_test 1.$tn.18 { + UPDATE ft1 SET rowid=12 WHERE rowid=1 + } {1 {cannot UPDATE contentless fts5 table: ft1}} + + do_execsql_test 1.$tn.19 { + UPDATE ft2 SET a='aXtwo', d='dXtwo', c='newval', rowid=12 WHERE rowid=2 + } {} + + set res(0) { + 3 {} {} {} {} + 11 {} {} {} {} + 12 {} {} {} {} + } + set res(1) { + 3 {} b3 c3.2 {} + 11 {} b1.1.2 c1.2 {} + 12 {} b2.1.2 newval {} + } + do_execsql_test 1.$tn.20 { + SELECT rowid, * FROM ft2 + } $res($cu) + + do_execsql_test 1.$tn.21 { + SELECT rowid, * FROM ft2('aXtwo AND dXtwo') + } [lrange $res($cu) 10 end] + +}} ;# end of [foreach_detail_mode] loop + +finish_test diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 79fd94e6bc..58dd9fe14e 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5version -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -112,7 +112,7 @@ do_execsql_test 2.1 { do_execsql_test 2.2 { SELECT v FROM xyz_config WHERE k='version'; -} {5} +} {4} do_execsql_test 2.3 { ROLLBACK TO one; diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index c457c5c210..b1644527ea 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5vocab -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -513,6 +513,7 @@ do_execsql_test 10.5 { INSERT INTO ft(a) VALUES('4 5 6'); } +unset -nocomplain x res do_test 10.6 { set res [list] db eval { SELECT rowid FROM ft('4') } x { diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test index 6f7aad329c..58416a7e90 100644 --- a/ext/fts5/test/fts5vocab2.test +++ b/ext/fts5/test/fts5vocab2.test @@ -15,7 +15,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5vocab2 -# If SQLITE_ENABLE_FTS5 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return @@ -280,6 +280,52 @@ do_catchsql_test 5.2 { INSERT INTO t1 SELECT randomblob(3000) FROM v1 } {1 {query aborted}} +#------------------------------------------------------------------------- +reset_db +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 6.0 { + BEGIN TRANSACTION; + CREATE VIRTUAL TABLE t1 USING fts5(a,b unindexed,c,tokenize="porter ascii",tokendata=1); + REPLACE INTO t1_data VALUES(1,X'03090009'); + REPLACE INTO t1_data VALUES(10,X'000000000103030003010101020101030101'); + REPLACE INTO t1_data VALUES(137438953473,X'0000002e023061010202010162010203010163010204010167010601020201016801060102030101690106010204040606060808'); + REPLACE INTO t1_data VALUES(274877906945,X'0000001f013067020802010202010168020803010203010169020804010204040909'); + REPLACE INTO t1_data VALUES(412316860417,X'0000002e023061030202010162030203010163030204010167030601020201016803060102030101690306010204040606060808'); + COMMIT; +} + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE t3 USING fts5vocab('t1', 'row'); +} + +do_catchsql_test 6.2 { + SELECT * FROM t3; +} {1 {database disk image is malformed}} + +sqlite3_fts5_may_be_corrupt 0 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, col); + + INSERT INTO t1 VALUES('xx', 'xx'); + + CREATE TABLE x1(t); + INSERT INTO x1 VALUES('xx'); + INSERT INTO x1 VALUES('xx'); + + SELECT term, col FROM v1; +} { + xx a xx b +} + +do_execsql_test 7.1 { + SELECT * FROM x1 WHERE 'a'=(SELECT col FROM v1 WHERE term=t) +} {xx xx} + finish_test diff --git a/ext/fts5/tool/mkfts5c.tcl b/ext/fts5/tool/mkfts5c.tcl index b1a55fa4ae..6f20a0cd73 100644 --- a/ext/fts5/tool/mkfts5c.tcl +++ b/ext/fts5/tool/mkfts5c.tcl @@ -2,7 +2,7 @@ # restart with tclsh \ exec tclsh "$0" "$@" -set srcdir [file dirname [file dirname [info script]]] +set srcdir [file dirname [file dirname [file normalize [info script]]]] set G(src) [string map [list %dir% $srcdir] { %dir%/fts5.h %dir%/fts5Int.h @@ -23,7 +23,27 @@ set G(src) [string map [list %dir% $srcdir] { }] set G(hdr) { - +/* +** This, the "fts5.c" source file, is a composite file that is itself +** assembled from the following files: +** +** fts5.h +** fts5Int.h +** fts5parse.h <--- Generated from fts5parse.y by Lemon +** fts5parse.c <--- Generated from fts5parse.y by Lemon +** fts5_aux.c +** fts5_buffer.c +** fts5_config.c +** fts5_expr.c +** fts5_hash.c +** fts5_index.c +** fts5_main.c +** fts5_storage.c +** fts5_tokenize.c +** fts5_unicode2.c +** fts5_varint.c +** fts5_vocab.c +*/ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) @@ -33,10 +53,16 @@ set G(hdr) { # undef NDEBUG #endif +#ifdef HAVE_STDINT_H +#include +#endif +#ifdef HAVE_INTTYPES_H +#include +#endif } set G(footer) { - +/* Here ends the fts5.c composite file. */ #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ } diff --git a/ext/icu/README.txt b/ext/icu/README.txt index c3c35a2c6f..40def24662 100644 --- a/ext/icu/README.txt +++ b/ext/icu/README.txt @@ -1,19 +1,18 @@ - This directory contains source code for the SQLite "ICU" extension, an integration of the "International Components for Unicode" library with SQLite. Documentation follows. 1. Features - + 1.1 SQL Scalars upper() and lower() 1.2 Unicode Aware LIKE Operator 1.3 ICU Collation Sequences 1.4 SQL REGEXP Operator - + 2. Compilation and Usage - + 3. Bugs, Problems and Security Issues - + 3.1 The "case_sensitive_like" Pragma 3.2 The SQLITE_MAX_LIKE_PATTERN_LENGTH Macro 3.3 Collation Sequence Security Issue @@ -23,10 +22,10 @@ SQLite. Documentation follows. 1.1 SQL Scalars upper() and lower() - SQLite's built-in implementations of these two functions only + SQLite's built-in implementations of these two functions only provide case mapping for the 26 letters used in the English language. The ICU based functions provided by this extension - provide case mapping, where defined, for the full range of + provide case mapping, where defined, for the full range of unicode characters. ICU provides two types of case mapping, "general" case mapping and @@ -36,7 +35,7 @@ SQLite. Documentation follows. http://www.icu-project.org/userguide/caseMappings.html http://www.icu-project.org/userguide/posix.html#case_mappings - To utilise "general" case mapping, the upper() or lower() scalar + To utilise "general" case mapping, the upper() or lower() scalar functions are invoked with one argument: upper('abc') -> 'ABC' @@ -57,7 +56,7 @@ SQLite. Documentation follows. operator understands case equivalence for the 26 letters of the English language alphabet. The implementation of LIKE included in this extension uses the ICU function u_foldCase() to provide case - independent comparisons for the full range of unicode characters. + independent comparisons for the full range of unicode characters. The U_FOLD_CASE_DEFAULT flag is passed to u_foldCase(), meaning the dotless 'I' character used in the Turkish language is considered @@ -66,9 +65,9 @@ SQLite. Documentation follows. 1.3 ICU Collation Sequences - A special SQL scalar function, icu_load_collation() is provided that + A special SQL scalar function, icu_load_collation() is provided that may be used to register ICU collation sequences with SQLite. It - is always called with exactly two arguments, the ICU locale + is always called with exactly two arguments, the ICU locale identifying the collation sequence to ICU, and the name of the SQLite collation sequence to create. For example, to create an SQLite collation sequence named "turkish" using Turkish language @@ -87,7 +86,7 @@ SQLite. Documentation follows. australian_penpal_name TEXT COLLATE australian, turkish_penpal_name TEXT COLLATE turkish ); - + 1.4 SQL REGEXP Operator This extension provides an implementation of the SQL binary @@ -116,7 +115,7 @@ SQLite. Documentation follows. and use it as a dynamically loadable SQLite extension. To do this using gcc on *nix: - gcc -fPIC -shared icu.c `pkg-config --libs --cflags icu-uc icu-io` \ + gcc -fPIC -shared icu.c `pkg-config --libs --cflags icu-io` \ -o libSqliteIcu.so You may need to add "-I" flags so that gcc can find sqlite3ext.h @@ -124,6 +123,11 @@ SQLite. Documentation follows. loaded into sqlite in the same way as any other dynamically loadable extension. + As of version 3.48, it can be enabled in the canonical build process + by passing one of --with-icu-config or --with-icu-ldflags to the + configure script, optionally together with --enable-icu-collations. + See the configure --help for more details. + 3 BUGS, PROBLEMS AND SECURITY ISSUES @@ -144,13 +148,13 @@ SQLite. Documentation follows. SQLITE_MAX_LIKE_PATTERN_LENGTH macro as the maximum length of a pattern in bytes (irrespective of encoding). The default value is defined in internal header file "limits.h". - - The ICU extension LIKE implementation suffers from the same + + The ICU extension LIKE implementation suffers from the same problem and uses the same solution. However, since the ICU extension code does not include the SQLite file "limits.h", modifying the default value therein does not affect the ICU extension. The default value of SQLITE_MAX_LIKE_PATTERN_LENGTH used by - the ICU extension LIKE operator is 50000, defined in source + the ICU extension LIKE operator is 50000, defined in source file "icu.c". 3.3 Collation Sequence Security diff --git a/ext/icu/icu.c b/ext/icu/icu.c index e745ab0253..50110072b5 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -471,7 +471,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -486,7 +486,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; i +#include + +#include +#include + +/* +** nKeyVal: +** The number of values that make up the 'key' for the current pCheck +** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. +*/ +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + char *zObj; /* Current object. Or NULL. */ + + sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; + int nKeyVal; + + char *zMessage; + int bCorruptSchema; + + int rc; /* Error code */ + char *zErr; /* Error message */ + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + p->rc = sqlite3_errcode(p->db); + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; + } + pRet = intckPrepare(p, zSql); + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + intckStep(p, pStmt); + intckFinalize(p, pStmt); +} + +/* +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + va_end(ap); + return zRet; +} + +/* +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. +*/ +static void intckSaveKey(sqlite3_intck *p){ + int ii; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; + + assert( p->pCheck ); + assert( p->zKey==0 ); + + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); + } + + pStmt = intckPrepare(p, zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + + sqlite3_free(zSql); + intckFinalize(p, pXinfo); +} + +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + + pStmt = intckPrepareFmt(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +/* +** Return true if argument c is an ascii whitespace character. +*/ +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = (int)strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); +*/ +static void intckParseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( SQLITE_ROW==intckStep(p, pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepareFmt(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ +static char *intckCheckObjectSql( + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ + int *pnKeyVal /* OUT: Number of key-values for this scan */ +){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat(a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', '), " + " count(*)" + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepareFmt(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "" + ", whereclause(w_c) AS (%s)" + "" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\n END AS error_message'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" + "" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" + " ), t.n" + " FROM case_statement, wrapper_with ww, thiskey t" + ")" + + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " + , p->zDb, p->zDb, zObj, zObj + , zPrev ? zPrev : "VALUES('')", zCommon + ); + }else{ + pStmt = intckPrepareFmt(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + "" + + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k, n) AS (" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" + ", main_select(m, n) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" + " ), thiskey.n" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" + ")" + + "SELECT m, n FROM main_select", + p->zDb, zObj, zPrev, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } + } + intckFinalize(p, pStmt); + + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 1"); + return zRet; +} + +/* +** Open a new integrity-check object. +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = (int)strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } + } + + *ppOut = pNew; + return rc; +} + +/* +** Free the integrity-check object. +*/ +void sqlite3_intck_close(sqlite3_intck *p){ + if( p ){ + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); + sqlite3_free(p); + } +} + +/* +** Step the integrity-check object. +*/ +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; + }else{ + p->rc = SQLITE_DONE; + } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, "%s", + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; + } + } + + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Normal case, do nothing. */ + }else{ + intckFinalize(p, p->pCheck); + p->pCheck = 0; + p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } + } + } + } + + return p->rc; +} + +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ +const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +/* +** Return the error code and message. +*/ +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + if( pzErr ) *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ +int sqlite3_intck_unlock(sqlite3_intck *p){ + if( p->rc==SQLITE_OK && p->pCheck ){ + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSaveKey(p); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; +} + +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } + return p->zTestSql; +} diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h new file mode 100644 index 0000000000..e08a86f289 --- /dev/null +++ b/ext/intck/sqlite3intck.h @@ -0,0 +1,171 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ +typedef struct sqlite3_intck sqlite3_intck; + +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ +int sqlite3_intck_open( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ +); + +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); + +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_step(sqlite3_intck *pCk); + +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); + +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c new file mode 100644 index 0000000000..d3a619b503 --- /dev/null +++ b/ext/intck/test_intck.c @@ -0,0 +1,233 @@ +/* +** 2010 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Code for testing all sorts of SQLite interfaces. This code +** is not included in the SQLite library. +*/ + +#include "sqlite3.h" +#include "sqlite3intck.h" +#include "tclsqlite.h" +#include +#include + +/* In test1.c */ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +const char *sqlite3ErrName(int); + +typedef struct TestIntck TestIntck; +struct TestIntck { + sqlite3_intck *intck; +}; + +static int testIntckCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct Subcmd { + const char *zName; + int nArg; + const char *zExpect; + } aCmd[] = { + {"close", 0, ""}, /* 0 */ + {"step", 0, ""}, /* 1 */ + {"message", 0, ""}, /* 2 */ + {"error", 0, ""}, /* 3 */ + {"unlock", 0, ""}, /* 4 */ + {"test_sql", 1, ""}, /* 5 */ + {0 , 0} + }; + int rc = TCL_OK; + int iIdx = -1; + TestIntck *p = (TestIntck*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); + return TCL_ERROR; + } + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aCmd, sizeof(aCmd[0]), "SUB-COMMAND", 0, &iIdx + ); + if( rc ) return rc; + + if( objc!=2+aCmd[iIdx].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aCmd[iIdx].zExpect); + return TCL_ERROR; + } + + switch( iIdx ){ + case 0: assert( 0==strcmp("close", aCmd[iIdx].zName) ); { + Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0)); + break; + } + + case 1: assert( 0==strcmp("step", aCmd[iIdx].zName) ); { + rc = sqlite3_intck_step(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 2: assert( 0==strcmp("message", aCmd[iIdx].zName) ); { + const char *z = sqlite3_intck_message(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(z ? z : "", -1)); + break; + } + + case 3: assert( 0==strcmp("error", aCmd[iIdx].zName) ); { + const char *zErr = 0; + Tcl_Obj *pRes; + rc = sqlite3_intck_error(p->intck, 0); + pRes = Tcl_NewObj(); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(sqlite3ErrName(rc), -1) + ); + sqlite3_intck_error(p->intck, &zErr); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(zErr ? zErr : 0, -1) + ); + Tcl_SetObjResult(interp, pRes); + break; + } + + case 4: assert( 0==strcmp("unlock", aCmd[iIdx].zName) ); { + rc = sqlite3_intck_unlock(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 5: assert( 0==strcmp("test_sql", aCmd[iIdx].zName) ); { + const char *zObj = Tcl_GetString(objv[2]); + const char *zSql = sqlite3_intck_test_sql(p->intck, zObj[0] ? zObj : 0); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zSql, -1)); + break; + } + } + + return TCL_OK; +} + +/* +** Destructor for commands created by test_sqlite3_intck(). +*/ +static void testIntckFree(void *clientData){ + TestIntck *p = (TestIntck*)clientData; + sqlite3_intck_close(p->intck); + ckfree(p); +} + +/* +** tclcmd: sqlite3_intck DB DBNAME +*/ +static int test_sqlite3_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char zName[64]; + int iName = 0; + Tcl_CmdInfo info; + TestIntck *p = 0; + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + + p = (TestIntck*)ckalloc(sizeof(TestIntck)); + memset(p, 0, sizeof(TestIntck)); + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = 0; + + rc = sqlite3_intck_open(db, zDb, &p->intck); + if( rc!=SQLITE_OK ){ + ckfree(p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errstr(rc), -1)); + return TCL_ERROR; + } + + do { + sprintf(zName, "intck%d", iName++); + }while( Tcl_GetCommandInfo(interp, zName, &info)!=0 ); + Tcl_CreateObjCommand(interp, zName, testIntckCmd, (void*)p, testIntckFree); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zName, -1)); + + return TCL_OK; +} + +/* +** tclcmd: test_do_intck DB DBNAME +*/ +static int test_do_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + sqlite3_intck *pCk = 0; + Tcl_Obj *pRet = 0; + const char *zErr = 0; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + rc = sqlite3_intck_open(db, zDb, &pCk); + if( rc==SQLITE_OK ){ + while( sqlite3_intck_step(pCk)==SQLITE_OK ){ + const char *zMsg = sqlite3_intck_message(pCk); + if( zMsg ){ + Tcl_ListObjAppendElement(interp, pRet, Tcl_NewStringObj(zMsg, -1)); + } + } + rc = sqlite3_intck_error(pCk, &zErr); + } + if( rc!=SQLITE_OK ){ + if( zErr ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + }else{ + Tcl_SetObjResult(interp, pRet); + } + Tcl_DecrRefCount(pRet); + sqlite3_intck_close(pCk); + sqlite3_intck_close(0); + return rc ? TCL_ERROR : TCL_OK; +} + +int Sqlitetestintck_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_intck", test_sqlite3_intck, 0, 0); + Tcl_CreateObjCommand(interp, "test_do_intck", test_do_intck, 0, 0); + return TCL_OK; +} diff --git a/ext/jni/GNUmakefile b/ext/jni/GNUmakefile new file mode 100644 index 0000000000..f2fc7fb7bd --- /dev/null +++ b/ext/jni/GNUmakefile @@ -0,0 +1,505 @@ +# Quick-and-dirty makefile to bootstrap the sqlite3-jni project. This +# build assumes a Linux-like system. +default: all + +JAVA_HOME ?= $(HOME)/jdk/current +# e.g. /usr/lib/jvm/default-javajava-19-openjdk-amd64 +JDK_HOME ?= $(JAVA_HOME) +# ^^^ JDK_HOME is not as widely used as JAVA_HOME +bin.jar := $(JDK_HOME)/bin/jar +bin.java := $(JDK_HOME)/bin/java +bin.javac := $(JDK_HOME)/bin/javac +bin.javadoc := $(JDK_HOME)/bin/javadoc +ifeq (,$(wildcard $(JDK_HOME))) +$(error set JDK_HOME to the top-most dir of your JDK installation.) +endif +MAKEFILE := $(lastword $(MAKEFILE_LIST)) +$(MAKEFILE): + +package.jar := sqlite3-jni.jar + +dir.top := ../.. +dir.tool := ../../tool +dir.jni := $(patsubst %/,%,$(dir $(MAKEFILE))) +dir.src := $(dir.jni)/src +dir.src.c := $(dir.src)/c +dir.bld := $(dir.jni)/bld +dir.bld.c := $(dir.bld) +dir.src.jni := $(dir.src)/org/sqlite/jni +dir.src.capi := $(dir.src.jni)/capi +dir.src.fts5 := $(dir.src.jni)/fts5 +dir.tests := $(dir.src)/tests +mkdir ?= mkdir -p +$(dir.bld.c): + $(mkdir) $@ + +javac.flags ?= -Xlint:unchecked -Xlint:deprecation +java.flags ?= +javac.flags += -encoding utf8 +# -------------^^^^^^^^^^^^^^ required for Windows builds +jnicheck ?= 1 +ifeq (1,$(jnicheck)) + java.flags += -Xcheck:jni +endif + +classpath := $(dir.src) +CLEAN_FILES := $(package.jar) +DISTCLEAN_FILES := $(dir.jni)/*~ $(dir.src.c)/*~ $(dir.src.jni)/*~ + +sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h +.NOTPARALLEL: $(sqlite3-jni.h) +CApi.java := $(dir.src.capi)/CApi.java +SQLTester.java := $(dir.src.capi)/SQLTester.java +CApi.class := $(CApi.java:.java=.class) +SQLTester.class := $(SQLTester.java:.java=.class) + +######################################################################## +# The future of FTS5 customization in this API is as yet unclear. +# The pieces are all in place, and are all thin proxies so not much +# complexity, but some semantic changes were required in porting +# which are largely untested. +# +# Reminder: this flag influences the contents of $(sqlite3-jni.h), +# which is checked in. Please do not check in changes to that file in +# which the fts5 APIs have been stripped unless that feature is +# intended to be stripped for good. +enable.fts5 ?= 1 + +ifeq (,$(wildcard $(dir.tests)/*)) + enable.tester := 0 +else + enable.tester := 1 +endif + +# bin.version-info = binary to output various sqlite3 version info +# building the distribution zip file. +bin.version-info := $(dir.top)/version-info +.NOTPARALLEL: $(bin.version-info) +$(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile + $(MAKE) -C $(dir.top) version-info + +# Be explicit about which Java files to compile so that we can work on +# in-progress files without requiring them to be in a compilable state. +JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\ + Experimental.java \ + NotNull.java \ + Nullable.java \ +) $(patsubst %,$(dir.src.capi)/%,\ + AbstractCollationCallback.java \ + AggregateFunction.java \ + AuthorizerCallback.java \ + AutoExtensionCallback.java \ + BusyHandlerCallback.java \ + CollationCallback.java \ + CollationNeededCallback.java \ + CommitHookCallback.java \ + ConfigLogCallback.java \ + ConfigSqlLogCallback.java \ + NativePointerHolder.java \ + OutputPointer.java \ + PrepareMultiCallback.java \ + PreupdateHookCallback.java \ + ProgressHandlerCallback.java \ + ResultCode.java \ + RollbackHookCallback.java \ + ScalarFunction.java \ + SQLFunction.java \ + CallbackProxy.java \ + CApi.java \ + TableColumnMetadata.java \ + TraceV2Callback.java \ + UpdateHookCallback.java \ + ValueHolder.java \ + WindowFunction.java \ + XDestroyCallback.java \ + sqlite3.java \ + sqlite3_blob.java \ + sqlite3_context.java \ + sqlite3_stmt.java \ + sqlite3_value.java \ +) $(patsubst %,$(dir.src.jni)/wrapper1/%,\ + AggregateFunction.java \ + ScalarFunction.java \ + SqlFunction.java \ + Sqlite.java \ + SqliteException.java \ + ValueHolder.java \ + WindowFunction.java \ +) + +JAVA_FILES.unittest := $(patsubst %,$(dir.src.jni)/%,\ + capi/Tester1.java \ + wrapper1/Tester2.java \ +) +ifeq (1,$(enable.fts5)) + JAVA_FILES.unittest += $(patsubst %,$(dir.src.fts5)/%,\ + TesterFts5.java \ + ) + JAVA_FILES.main += $(patsubst %,$(dir.src.fts5)/%,\ + fts5_api.java \ + fts5_extension_function.java \ + fts5_tokenizer.java \ + Fts5.java \ + Fts5Context.java \ + Fts5ExtensionApi.java \ + Fts5PhraseIter.java \ + Fts5Tokenizer.java \ + XTokenizeCallback.java \ + ) +endif +JAVA_FILES.tester := $(SQLTester.java) +JAVA_FILES.package.info := \ + $(dir.src.jni)/package-info.java \ + $(dir.src.jni)/annotation/package-info.java + +CLASS_FILES.main := $(JAVA_FILES.main:.java=.class) +CLASS_FILES.unittest := $(JAVA_FILES.unittest:.java=.class) +CLASS_FILES.tester := $(JAVA_FILES.tester:.java=.class) + +JAVA_FILES += $(JAVA_FILES.main) $(JAVA_FILES.unittest) +ifeq (1,$(enable.tester)) + JAVA_FILES += $(JAVA_FILES.tester) +endif + +CLASS_FILES := +define CLASSFILE_DEPS +all: $(1).class +$(1).class: $(1).java +CLASS_FILES += $(1).class +endef +$(foreach B,$(basename \ + $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.tester)),\ + $(eval $(call CLASSFILE_DEPS,$(B)))) +$(CLASS_FILES): $(MAKEFILE) + $(bin.javac) $(javac.flags) -h $(dir.bld.c) -cp $(classpath) $(JAVA_FILES) + +#.PHONY: classfiles + +######################################################################## +# Set up sqlite3.c and sqlite3.h... +# +# To build with SEE (https://sqlite.org/see), either put sqlite3-see.c +# in the top of this build tree or pass +# sqlite3.c=PATH_TO_sqlite3-see.c to the build. Note that only +# encryption modules with no 3rd-party dependencies will currently +# work here: AES256-OFB, AES128-OFB, and AES128-CCM. Not +# coincidentally, those 3 modules are included in the sqlite3-see.c +# bundle. +# +# A custom sqlite3.c must not have any spaces in its name. +# $(sqlite3.canonical.c) must point to the sqlite3.c in +# the sqlite3 canonical source tree, as that source file +# is required for certain utility and test code. +sqlite3.canonical.c := $(firstword $(wildcard $(dir.src.c)/sqlite3.c) $(dir.top)/sqlite3.c) +sqlite3.canonical.h := $(firstword $(wildcard $(dir.src.c)/sqlite3.h) $(dir.top)/sqlite3.h) +sqlite3.c := $(sqlite3.canonical.c) +sqlite3.h := $(sqlite3.canonical.h) +#ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c) 2>/dev/null)) +# SQLITE_C_IS_SEE := 0 +#else +# SQLITE_C_IS_SEE := 1 +# $(info This is an SEE build.) +#endif + +.NOTPARALLEL: $(sqlite3.h) +$(sqlite3.h): + $(MAKE) -C $(dir.top) sqlite3.c +$(sqlite3.c): $(sqlite3.h) + +opt.threadsafe ?= 1 +opt.fatal-oom ?= 1 +opt.debug ?= 1 +opt.metrics ?= 1 +SQLITE_OPT = \ + -DSQLITE_THREADSAFE=$(opt.threadsafe) \ + -DSQLITE_TEMP_STORE=2 \ + -DSQLITE_USE_URI=1 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_C=$(sqlite3.c) \ + -DSQLITE_JNI_FATAL_OOM=$(opt.fatal-oom) \ + -DSQLITE_JNI_ENABLE_METRICS=$(opt.metrics) + +opt.extras ?= 1 +ifeq (1,$(opt.extras)) +SQLITE_OPT += -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_NORMALIZE \ + -DSQLITE_ENABLE_SQLLOG \ + -DSQLITE_ENABLE_COLUMN_METADATA +endif + +ifeq (1,$(opt.debug)) + SQLITE_OPT += -DSQLITE_DEBUG -g -DDEBUG -UNDEBUG +else + SQLITE_OPT += -Os +endif + +ifeq (1,$(enable.fts5)) + SQLITE_OPT += -DSQLITE_ENABLE_FTS5 +endif + +sqlite3-jni.c := $(dir.src.c)/sqlite3-jni.c +sqlite3-jni.o := $(dir.bld.c)/sqlite3-jni.o +sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h +package.dll := $(dir.bld.c)/libsqlite3-jni.so +# All javac-generated .h files must be listed in $(sqlite3-jni.h.in): +sqlite3-jni.h.in := +# $(java.with.jni) lists all Java files which contain JNI decls: +java.with.jni := +define ADD_JNI_H +sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h +java.with.jni += $(1)/$(2).java +$$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h: $(1)/$(2).java +endef +# Invoke ADD_JNI_H once for each Java file which includes JNI +# declarations: +$(eval $(call ADD_JNI_H,$(dir.src.capi),CApi,_capi)) +$(eval $(call ADD_JNI_H,$(dir.src.capi),SQLTester,_capi)) +ifeq (1,$(enable.fts5)) + $(eval $(call ADD_JNI_H,$(dir.src.fts5),Fts5ExtensionApi,_fts5)) + $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_api,_fts5)) + $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_tokenizer,_fts5)) +endif +$(sqlite3-jni.h.in): $(dir.bld.c) + +#package.dll.cfiles := +package.dll.cflags = \ + -std=c99 \ + -fPIC \ + -I. \ + -I$(dir $(sqlite3.h)) \ + -I$(dir.src.c) \ + -I$(JDK_HOME)/include \ + $(patsubst %,-I%,$(patsubst %.h,,$(wildcard $(JDK_HOME)/include/*))) \ + -Wall +# The gross $(patsubst...) above is to include the platform-specific +# subdir which lives under $(JDK_HOME)/include and is a required +# include path for client-level code. +# +# Using (-Wall -Wextra) triggers an untennable number of +# gcc warnings from sqlite3.c for mundane things like +# unused parameters. +######################################################################## +ifeq (1,$(enable.tester)) + package.dll.cflags += -DSQLITE_JNI_ENABLE_SQLTester +endif + +$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE) + @cat $(sqlite3-jni.h.in) > $@.tmp + @if cmp $@ $@.tmp >/dev/null; then \ + rm -f $@.tmp; \ + echo "$@ not modified"; \ + else \ + mv $@.tmp $@; \ + echo "Updated $@"; \ + fi + @if [ x1 != x$(enable.fts5) ]; then \ + echo "*** REMINDER:"; \ + echo "*** enable.fts5=0, so please do not check in changes to $@."; \ + fi + +$(package.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h) +$(package.dll): $(sqlite3-jni.c) $(MAKEFILE) + $(CC) $(package.dll.cflags) $(SQLITE_OPT) \ + $(sqlite3-jni.c) -shared -o $@ +all: $(package.dll) + +.PHONY: test test-one +Tester1.flags ?= +Tester2.flags ?= +test.flags.jvm = -ea -Djava.library.path=$(dir.bld.c) \ + $(java.flags) -cp $(classpath) +test.deps := $(CLASS_FILES) $(package.dll) +test-one: $(test.deps) + $(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) + $(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 $(Tester2.flags) +test-sqllog: $(test.deps) + @echo "Testing with -sqllog..." + $(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) -sqllog +test-mt: $(test.deps) + @echo "Testing in multi-threaded mode:"; + $(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 \ + -t 7 -r 50 -shuffle $(Tester1.flags) + $(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 \ + -t 7 -r 50 -shuffle $(Tester2.flags) + +test: test-one test-mt +tests: test test-sqllog + +tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test)) +tester.flags ?= # --verbose +.PHONY: tester tester-local tester-ext +ifeq (1,$(enable.tester)) +tester-local: $(CLASS_FILES.tester) $(package.dll) + $(bin.java) -ea -Djava.library.path=$(dir.bld.c) \ + $(java.flags) -cp $(classpath) \ + org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.scripts) +tester: tester-local +else +tester: + @echo "SQLTester support is disabled." +endif + +tester.extdir.default := $(dir.tests)/ext +tester.extdir ?= $(tester.extdir.default) +tester.extern-scripts := $(wildcard $(tester.extdir)/*.test) +ifneq (,$(tester.extern-scripts)) +tester-ext: + $(bin.java) -ea -Djava.library.path=$(dir.bld.c) \ + $(java.flags) -cp $(classpath) \ + org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.extern-scripts) +else +tester-ext: + @echo "******************************************************"; \ + echo "*** Include the out-of-tree test suite in the 'tester'"; \ + echo "*** target by either symlinking its directory to"; \ + echo "*** $(tester.extdir.default) or passing it to make"; \ + echo "*** as tester.extdir=/path/to/that/dir."; \ + echo "******************************************************"; +endif + +tester-ext: tester-local +tester: tester-ext +tests: tester +######################################################################## +# Build each SQLITE_THREADMODE variant and run all tests against them. +multitest: clean +define MULTIOPT +multitest: multitest-$(1) +multitest-$(1): + $$(MAKE) opt.debug=$$(opt.debug) $(patsubst %,opt.%,$(2)) \ + tests clean enable.fts5=1 +endef + +$(eval $(call MULTIOPT,01,threadsafe=0 oom=1)) +$(eval $(call MULTIOPT,00,threadsafe=0 oom=0)) +$(eval $(call MULTIOPT,11,threadsafe=1 oom=1)) +$(eval $(call MULTIOPT,10,threadsafe=1 oom=0)) +$(eval $(call MULTIOPT,21,threadsafe=2 oom=1)) +$(eval $(call MULTIOPT,20,threadsafe=2 oom=0)) + + +######################################################################## +# jar bundle... +package.jar.in := $(abspath $(dir.src)/jar.in) +CLEAN_FILES += $(package.jar.in) +JAVA_FILES.jar := $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.package.info) +CLASS_FILES.jar := $(filter-out %/package-info.class,$(JAVA_FILES.jar:.java=.class)) +$(package.jar.in): $(package.dll) $(MAKEFILE) + ls -1 \ + $(dir.src.jni)/*/*.java $(dir.src.jni)/*/*.class \ + | sed -e 's,^$(dir.src)/,,' | sort > $@ + +$(package.jar): $(CLASS_FILES.jar) $(MAKEFILE) $(package.jar.in) + @rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~ + cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.capi.Tester1 @$(package.jar.in) + @ls -la $@ + @echo "To use this jar you will need the -Djava.library.path=DIR/CONTAINING/libsqlite3-jni.so flag." + @echo "e.g. java -Djava.library.path=bld -jar $@" + +jar: $(package.jar) +run-jar: $(package.jar) $(package.dll) + $(bin.java) -Djava.library.path=$(dir.bld) -jar $(package.jar) $(run-jar.flags) + +######################################################################## +# javadoc... +dir.doc := $(dir.jni)/javadoc +doc.index := $(dir.doc)/index.html +javadoc.exclude := -exclude org.sqlite.jni.fts5 +# ^^^^ 2023-09-13: elide the fts5 parts from the public docs for +# the time being, as it's not clear where the Java bindings for +# those bits are going. +# javadoc.exclude += -exclude org.sqlite.jni.capi +# ^^^^ exclude the capi API only for certain builds (TBD) +$(doc.index): $(JAVA_FILES.main) $(MAKEFILE) + @if [ -d $(dir.doc) ]; then rm -fr $(dir.doc)/*; fi + $(bin.javadoc) -cp $(classpath) -d $(dir.doc) -quiet \ + -subpackages org.sqlite.jni $(javadoc.exclude) + @echo "javadoc output is in $@" + +.PHONY: doc javadoc docserve +.FORCE: doc +doc: $(doc.index) +javadoc: $(doc.index) +# Force rebuild of docs +redoc: + @rm -f $(doc.index) + @$(MAKE) doc +docserve: $(doc.index) + cd $(dir.doc) && althttpd -max-age 1 -page index.html +######################################################################## +# Clean up... +CLEAN_FILES += $(dir.bld.c)/* \ + $(dir.src.jni)/*.class \ + $(dir.src.jni)/*/*.class \ + $(package.dll) \ + hs_err_pid*.log + +.PHONY: clean distclean +clean: + -rm -f $(CLEAN_FILES) +distclean: clean + -rm -f $(DISTCLEAN_FILES) + -rm -fr $(dir.bld.c) $(dir.doc) + +######################################################################## +# distribution bundle rules... + +ifeq (,$(filter snapshot,$(MAKECMDGOALS))) +dist-name-prefix := sqlite-jni +else +dist-name-prefix := sqlite-jni-snapshot-$(shell /usr/bin/date +%Y%m%d) +endif +dist-name := $(dist-name-prefix)-TEMP + + +dist-dir.top := $(dist-name) +dist-dir.src := $(dist-dir.top)/src +dist.top.extras := \ + README.md + +.PHONY: dist snapshot + +dist: \ + $(bin.version-info) $(sqlite3.canonical.c) \ + $(package.jar) $(MAKEFILE) + @echo "Making end-user deliverables..." + @echo "****************************************************************************"; \ + echo "*** WARNING: be sure to build this with JDK8 (javac 1.8) for compatibility."; \ + echo "*** reasons!"; $$($(bin.javac) -version); \ + echo "****************************************************************************" + @rm -fr $(dist-dir.top) + @mkdir -p $(dist-dir.src) + @cp -p $(dist.top.extras) $(dist-dir.top)/. + @cp -p jar-dist.make $(dist-dir.top)/Makefile + @cp -p $(dir.src.c)/*.[ch] $(dist-dir.src)/. + @cp -p $(sqlite3.canonical.c) $(sqlite3.canonical.h) $(dist-dir.src)/. + @set -e; \ + vnum=$$($(bin.version-info) --download-version); \ + vjar=$$($(bin.version-info) --version); \ + vdir=$(dist-name-prefix)-$$vnum; \ + arczip=$$vdir.zip; \ + cp -p $(package.jar) $(dist-dir.top)/sqlite3-jni-$${vjar}.jar; \ + echo "Making $$arczip ..."; \ + rm -fr $$arczip $$vdir; \ + mv $(dist-dir.top) $$vdir; \ + zip -qr $$arczip $$vdir; \ + rm -fr $$vdir; \ + ls -la $$arczip; \ + set +e; \ + unzip -lv $$arczip || echo "Missing unzip app? Not fatal." + +snapshot: dist + +.PHONY: dist-clean +clean: dist-clean +dist-clean: + rm -fr $(dist-name) $(wildcard sqlite-jni-*.zip) diff --git a/ext/jni/README.md b/ext/jni/README.md new file mode 100644 index 0000000000..5ad79fce9e --- /dev/null +++ b/ext/jni/README.md @@ -0,0 +1,316 @@ +SQLite3 via JNI +======================================================================== + +This directory houses a Java Native Interface (JNI) binding for the +sqlite3 API. If you are reading this from the distribution ZIP file, +links to resources in the canonical source tree will note work. The +canonical copy of this file can be browsed at: + + + +Technical support is available in the forum: + + + + +> **FOREWARNING:** this subproject is very much in development and + subject to any number of changes. Please do not rely on any + information about its API until this disclaimer is removed. The JNI + bindings released with version 3.43 are a "tech preview." Once + finalized, strong backward compatibility guarantees will apply. + +Project goals/requirements: + +- A [1-to-1(-ish) mapping of the C API](#1to1ish) to Java via JNI, + insofar as cross-language semantics allow for. A closely-related + goal is that [the C documentation](https://sqlite.org/c3ref/intro.html) + should be usable as-is, insofar as possible, for the JNI binding. + +- Support Java as far back as version 8 (2014). + +- Environment-independent. Should work everywhere both Java + and SQLite3 do. + +- No 3rd-party dependencies beyond the JDK. That includes no + build-level dependencies for specific IDEs and toolchains. We + welcome the addition of build files for arbitrary environments + insofar as they neither interfere with each other nor become + a maintenance burden for the sqlite developers. + +Non-goals: + +- Creation of high-level OO wrapper APIs. Clients are free to create + them off of the C-style API. + +- Virtual tables are unlikely to be supported due to the amount of + glue code needed to fit them into Java. + +- Support for mixed-mode operation, where client code accesses SQLite + both via the Java-side API and the C API via their own native + code. Such cases would be a minefield of potential mis-interactions + between this project's JNI bindings and mixed-mode client code. + + +Hello World +----------------------------------------------------------------------- + +```java +import org.sqlite.jni.*; +import static org.sqlite.jni.CApi.*; + +... + +final sqlite3 db = sqlite3_open(":memory:"); +try { + final int rc = sqlite3_errcode(db); + if( 0 != rc ){ + if( null != db ){ + System.out.print("Error opening db: "+sqlite3_errmsg(db)); + }else{ + System.out.print("Error opening db: rc="+rc); + } + ... handle error ... + } + // ... else use the db ... +}finally{ + // ALWAYS close databases using sqlite3_close() or sqlite3_close_v2() + // when done with them. All of their active statement handles must + // first have been passed to sqlite3_finalize(). + sqlite3_close_v2(db); +} +``` + + +Building +======================================================================== + +The canonical builds assumes a Linux-like environment and requires: + +- GNU Make +- A JDK supporting Java 8 or higher +- A modern C compiler. gcc and clang should both work. + +Put simply: + +```console +$ export JAVA_HOME=/path/to/jdk/root +$ make +$ make test +$ make clean +``` + +The jar distribution can be created with `make jar`, but note that it +does not contain the binary DLL file. A different DLL is needed for +each target platform. + + + +One-to-One(-ish) Mapping to C +======================================================================== + +This JNI binding aims to provide as close to a 1-to-1 experience with +the C API as cross-language semantics allow. Interface changes are +necessarily made where cross-language semantics do not allow a 1-to-1, +and judiciously made where a 1-to-1 mapping would be unduly cumbersome +to use in Java. In all cases, this binding makes every effort to +provide semantics compatible with the C API documentation even if the +interface to those semantics is slightly different. Any cases which +deviate from those semantics (either removing or adding semantics) are +clearly documented. + +Where it makes sense to do so for usability, Java-side overloads are +provided which accept or return data in alternative forms or provide +sensible default argument values. In all such cases they are thin +proxies around the corresponding C APIs and do not introduce new +semantics. + +In a few cases, Java-specific capabilities have been added in +new APIs, all of which have "_java" somewhere in their names. +Examples include: + +- `sqlite3_result_java_object()` +- `sqlite3_column_java_object()` +- `sqlite3_value_java_object()` + +which, as one might surmise, collectively enable the passing of +arbitrary Java objects from user-defined SQL functions through to the +caller. + + +Golden Rule: Garbage Collection Cannot Free SQLite Resources +------------------------------------------------------------------------ + +It is important that all databases and prepared statement handles get +cleaned up by client code. A database cannot be closed if it has open +statement handles. `sqlite3_close()` fails if the db cannot be closed +whereas `sqlite3_close_v2()` recognizes that case and marks the db as +a "zombie," pending finalization when the library detects that all +pending statements have been closed. Be aware that Java garbage +collection _cannot_ close a database or finalize a prepared statement. +Those things require explicit API calls. + +Classes for which it is sensible support Java's `AutoCloseable` +interface so can be used with try-with-resources constructs. + + +Golden Rule #2: _Never_ Throw from Callbacks (Unless...) +------------------------------------------------------------------------ + +All routines in this API, barring explicitly documented exceptions, +retain C-like semantics. For example, they are not permitted to throw +or propagate exceptions and must return error information (if any) via +result codes or `null`. The only cases where the C-style APIs may +throw is through client-side misuse, e.g. passing in a null where it +may cause a `NullPointerException`. The APIs clearly mark function +parameters which should not be null, but does not generally actively +defend itself against such misuse. Some C-style APIs explicitly accept +`null` as a no-op for usability's sake, and some of the JNI APIs +deliberately return an error code, instead of segfaulting, when passed +a `null`. + +Client-defined callbacks _must never throw exceptions_ unless _very +explicitly documented_ as being throw-safe. Exceptions are generally +reserved for higher-level bindings which are constructed to +specifically deal with them and ensure that they do not leak C-level +resources. In some cases, callback handlers are permitted to throw, in +which cases they get translated to C-level result codes and/or +messages. If a callback which is not permitted to throw throws, its +exception may trigger debug output but will otherwise be suppressed. + +The reason some callbacks are permitted to throw and others not is +because all such callbacks act as proxies for C function callback +interfaces and some of those interfaces have no error-reporting +mechanism. Those which are capable of propagating errors back through +the library convert exceptions from callbacks into corresponding +C-level error information. Those which cannot propagate errors +necessarily suppress any exceptions in order to maintain the C-style +semantics of the APIs. + + +Unwieldy Constructs are Re-mapped +------------------------------------------------------------------------ + +Some constructs, when modelled 1-to-1 from C to Java, are unduly +clumsy to work with in Java because they try to shoehorn C's way of +doing certain things into Java's wildly different ways. The following +subsections cover those, starting with a verbose explanation and +demonstration of where such changes are "really necessary"... + +### Custom Collations + +A prime example of where interface changes for Java are necessary for +usability is [registration of a custom +collation](https://sqlite.org/c3ref/create_collation.html): + +```c +// C: +int sqlite3_create_collation(sqlite3 * db, const char * name, int eTextRep, + void *pUserData, + int (*xCompare)(void*,int,void const *,int,void const *)); + +int sqlite3_create_collation_v2(sqlite3 * db, const char * name, int eTextRep, + void *pUserData, + int (*xCompare)(void*,int,void const *,int,void const *), + void (*xDestroy)(void*)); +``` + +The `pUserData` object is optional client-defined state for the +`xCompare()` and/or `xDestroy()` callback functions, both of which are +passed that object as their first argument. That data is passed around +"externally" in C because that's how C models the world. If we were to +bind that part as-is to Java, the result would be awkward to use (^Yes, +we tried this.): + +```java +// Java: +int sqlite3_create_collation(sqlite3 db, String name, int eTextRep, + Object pUserData, xCompareType xCompare); + +int sqlite3_create_collation_v2(sqlite3 db, String name, int eTextRep, + Object pUserData, + xCompareType xCompare, xDestroyType xDestroy); +``` + +The awkwardness comes from (A) having two distinctly different objects +for callbacks and (B) having their internal state provided separately, +which is ill-fitting in Java. For the sake of usability, C APIs which +follow that pattern use a slightly different Java interface: + +```java +int sqlite3_create_collation(sqlite3 db, String name, int eTextRep, + SomeCallbackType collation); +``` + +Where the `Collation` class has an abstract `call()` method and +no-op `xDestroy()` method which can be overridden if needed, leading to +a much more Java-esque usage: + +```java +int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new SomeCallbackType(){ + + // Required comparison function: + @Override public int call(byte[] lhs, byte[] rhs){ ... } + + // Optional finalizer function: + @Override public void xDestroy(){ ... } + + // Optional local state: + private String localState1 = + "This is local state. There are many like it, but this one is mine."; + private MyStateType localState2 = new MyStateType(); + ... +}); +``` + +Noting that: + +- It is possible to bind in call-scope-local state via closures, if + desired, as opposed to packing it into the Collation object. + +- No capabilities of the C API are lost or unduly obscured via the + above API reshaping, so power users need not make any compromises. + +- In the specific example above, `sqlite3_create_collation_v2()` + becomes superfluous because the provided interface effectively + provides both the v1 and v2 interfaces, the difference being that + overriding the `xDestroy()` method effectively gives it v2 + semantics. + + +### User-defined SQL Functions (a.k.a. UDFs) + +The [`sqlite3_create_function()`](https://sqlite.org/c3ref/create_function.html) +family of APIs make heavy use of function pointers to provide +client-defined callbacks, necessitating interface changes in the JNI +binding. The Java API has only one core function-registration function: + +```java +int sqlite3_create_function(sqlite3 db, String funcName, int nArgs, + int encoding, SQLFunction func); +``` + +> Design question: does the encoding argument serve any purpose in + Java? That's as-yet undetermined. If not, it will be removed. + +`SQLFunction` is not used directly, but is instead instantiated via +one of its three subclasses: + +- `ScalarFunction` implements simple scalar functions using but a + single callback. +- `AggregateFunction` implements aggregate functions using two + callbacks. +- `WindowFunction` implements window functions using four + callbacks. + +Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/capi/Tester1.java) for +`SQLFunction` for how it's used. + +Reminder: see the disclaimer at the top of this document regarding the +in-flux nature of this API. + +### And so on... + +Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and +`sqlite3_update_hook()`, use interfaces similar to those shown above. +Despite the changes in signature, the JNI layer makes every effort to +provide the same semantics as the C API documentation suggests. diff --git a/ext/jni/jar-dist.make b/ext/jni/jar-dist.make new file mode 100644 index 0000000000..7596c99f3f --- /dev/null +++ b/ext/jni/jar-dist.make @@ -0,0 +1,60 @@ +#!/this/is/make +#^^^^ help emacs out +# +# This is a POSIX-make-compatible makefile for building the sqlite3 +# JNI library from "dist" zip file. It must be edited to set the +# proper top-level JDK directory and, depending on the platform, add a +# platform-specific -I directory. It should build as-is with any +# 2020s-era version of gcc or clang. It requires JDK version 8 or +# higher and that JAVA_HOME points to the top-most installation +# directory of that JDK. On Ubuntu-style systems the JDK is typically +# installed under /usr/lib/jvm/java-VERSION-PLATFORM. + +default: all + +JAVA_HOME = /usr/lib/jvm/java-1.8.0-openjdk-amd64 +CFLAGS = \ + -fPIC \ + -Isrc \ + -I$(JAVA_HOME)/include \ + -I$(JAVA_HOME)/include/linux \ + -I$(JAVA_HOME)/include/apple \ + -I$(JAVA_HOME)/include/bsd \ + -Wall + +SQLITE_OPT = \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_THREADSAFE=1 \ + -DSQLITE_TEMP_STORE=2 \ + -DSQLITE_USE_URI=1 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_DEBUG + +sqlite3-jni.dll = libsqlite3-jni.so +$(sqlite3-jni.dll): + @echo "************************************************************************"; \ + echo "*** If this fails to build, be sure to edit this makefile ***"; \ + echo "*** to configure it for your system. ***"; \ + echo "************************************************************************" + $(CC) $(CFLAGS) $(SQLITE_OPT) \ + src/sqlite3-jni.c -shared -o $@ + @echo "Now try running it with: make test" + +test.flags = -Djava.library.path=. sqlite3-jni-*.jar +test: $(sqlite3-jni.dll) + java -jar $(test.flags) + java -jar $(test.flags) -t 7 -r 10 -shuffle + +clean: + -rm -f $(sqlite3-jni.dll) + +all: $(sqlite3-jni.dll) diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c new file mode 100644 index 0000000000..f130eff042 --- /dev/null +++ b/ext/jni/src/c/sqlite3-jni.c @@ -0,0 +1,6360 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements the JNI bindings declared in +** org.sqlite.jni.capi.CApi (from which sqlite3-jni.h is generated). +*/ + +/* +** If you found this comment by searching the code for +** CallStaticObjectMethod because it appears in console output then +** you're probably the victim of an OpenJDK bug: +** +** https://bugs.openjdk.org/browse/JDK-8130659 +** +** It's known to happen with OpenJDK v8 but not with v19. It was +** triggered by this code long before it made any use of +** CallStaticObjectMethod(). +*/ + +/* +** Define any SQLITE_... config defaults we want if they aren't +** overridden by the builder. Please keep these alphabetized. +*/ + +/**********************************************************************/ +/* SQLITE_D... */ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +# define SQLITE_DEFAULT_CACHE_SIZE -16384 +#endif +#if !defined(SQLITE_DEFAULT_PAGE_SIZE) +# define SQLITE_DEFAULT_PAGE_SIZE 8192 +#endif +#ifndef SQLITE_DQS +# define SQLITE_DQS 0 +#endif + +/**********************************************************************/ +/* SQLITE_ENABLE_... */ +/* +** Unconditionally enable API_ARMOR in the JNI build. It ensures that +** public APIs behave predictable in the face of passing illegal NULLs +** or ranges which might otherwise invoke undefined behavior. +*/ +#undef SQLITE_ENABLE_API_ARMOR +#define SQLITE_ENABLE_API_ARMOR 1 + +#ifndef SQLITE_ENABLE_BYTECODE_VTAB +# define SQLITE_ENABLE_BYTECODE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBPAGE_VTAB +# define SQLITE_ENABLE_DBPAGE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBSTAT_VTAB +# define SQLITE_ENABLE_DBSTAT_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif +#ifndef SQLITE_ENABLE_MATH_FUNCTIONS +# define SQLITE_ENABLE_MATH_FUNCTIONS 1 +#endif +#ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC +# define SQLITE_ENABLE_OFFSET_SQL_FUNC 1 +#endif +#ifndef SQLITE_ENABLE_RTREE +# define SQLITE_ENABLE_RTREE 1 +#endif +//#ifndef SQLITE_ENABLE_SESSION +//# define SQLITE_ENABLE_SESSION 1 +//#endif +#ifndef SQLITE_ENABLE_STMTVTAB +# define SQLITE_ENABLE_STMTVTAB 1 +#endif +//#ifndef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +//# define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +//#endif + +/**********************************************************************/ +/* SQLITE_J... */ +#ifdef SQLITE_JNI_FATAL_OOM +#if !SQLITE_JNI_FATAL_OOM +#undef SQLITE_JNI_FATAL_OOM +#endif +#endif + +/**********************************************************************/ +/* SQLITE_O... */ +#ifndef SQLITE_OMIT_DEPRECATED +# define SQLITE_OMIT_DEPRECATED 1 +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE +# define SQLITE_OMIT_SHARED_CACHE 1 +#endif +#ifdef SQLITE_OMIT_UTF16 +/* UTF16 is required for java */ +# undef SQLITE_OMIT_UTF16 1 +#endif + +/**********************************************************************/ +/* SQLITE_S... */ +#ifndef SQLITE_STRICT_SUBTYPE +# define SQLITE_STRICT_SUBTYPE 1 +#endif + +/**********************************************************************/ +/* SQLITE_T... */ +#ifndef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 2 +#endif +#ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 1 +#endif + +/**********************************************************************/ +/* SQLITE_USE_... */ +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 1 +#endif + + +/* +** Which sqlite3.c we're using needs to be configurable to enable +** building against a custom copy, e.g. the SEE variant. We have to +** include sqlite3.c, as opposed to sqlite3.h, in order to get access +** to some internal details like SQLITE_MAX_... and friends. This +** increases the rebuild time considerably but we need this in order +** to access some internal functionality and keep the to-Java-exported +** values of SQLITE_MAX_... and SQLITE_LIMIT_... in sync with the C +** build. +*/ +#ifndef SQLITE_C +# define SQLITE_C sqlite3.c +#endif +#define INC__STRINGIFY_(f) #f +#define INC__STRINGIFY(f) INC__STRINGIFY_(f) +#include INC__STRINGIFY(SQLITE_C) +#undef INC__STRINGIFY_ +#undef INC__STRINGIFY +#undef SQLITE_C + +/* +** End of the sqlite3 lib setup. What follows is JNI-specific. +*/ + +#include "sqlite3-jni.h" +#include +#include /* only for testing/debugging */ +#include /* intptr_t for 32-bit builds */ + +/* Only for debugging */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/* +** Creates a verbose JNI function name. Suffix must be +** the JNI-mangled form of the function's name, minus the +** prefix seen in this macro. +** +** If you get java.lang.UnsatisfiedLinkError when calling newly-added +** native bindings, be sure that the mangled name is correct. It can +** be found in the generated sqlite3-jni.h. +*/ +#define JniFuncName(Suffix) \ + Java_org_sqlite_jni_capi_CApi_sqlite3_ ## Suffix + +/* Prologue for JNI function declarations and definitions. */ +#define JniDecl(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL JniFuncName(Suffix) + +/* +** S3JniApi's intent is that CFunc be the name(s) of the C API func(s) +** the being-declared JNI function is wrapping, making it easier to +** find those bindings' JNI-side entry points. The other args are for +** JniDecl. See the many examples in this file. +*/ +#define S3JniApi(CFunc,ReturnType,Suffix) JniDecl(ReturnType,Suffix) + +/* +** S3JniCast_L2P and P2L cast jlong (64-bit) to/from pointers. This is +** required for casting warning-free on 32-bit builds, where we +** otherwise get complaints that we're casting between different-sized +** int types. +** +** This use of intptr_t is the _only_ reason we require +** which, in turn, requires building with -std=c99 (or later). +** +** See also: the notes for LongPtrGet_T. +*/ +#define S3JniCast_L2P(JLongAsPtr) (void*)((intptr_t)(JLongAsPtr)) +#define S3JniCast_P2L(PTR) (jlong)((intptr_t)(PTR)) + +/* +** Shortcuts for the first 2 parameters to all JNI bindings. +** +** The type of the jSelf arg differs, but no docs seem to mention +** this: for static methods it's of type jclass and for non-static +** it's jobject. jobject actually works for all funcs, in the sense +** that it compiles and runs so long as we don't use jSelf (which is +** only rarely needed in this code), but to be pedantically correct we +** need the proper type in the signature. +** +** https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/design.html#jni_interface_functions_and_pointers +*/ +#define JniArgsEnvObj JNIEnv * env, jobject jSelf +#define JniArgsEnvClass JNIEnv * env, jclass jKlazz +/* +** Helpers to account for -Xcheck:jni warnings about not having +** checked for exceptions. +*/ +#define S3JniIfThrew if( (*env)->ExceptionCheck(env) ) +#define S3JniExceptionClear (*env)->ExceptionClear(env) +#define S3JniExceptionReport (*env)->ExceptionDescribe(env) +#define S3JniExceptionIgnore S3JniIfThrew S3JniExceptionClear +#define S3JniExceptionWarnIgnore \ + S3JniIfThrew {S3JniExceptionReport; S3JniExceptionClear;}(void)0 +#define S3JniExceptionWarnCallbackThrew(STR) \ + MARKER(("WARNING: " STR " MUST NOT THROW.\n")); \ + (*env)->ExceptionDescribe(env) + +/** To be used for cases where we're _really_ not expecting an + exception, e.g. looking up well-defined Java class members. */ +#define S3JniExceptionIsFatal(MSG) S3JniIfThrew {\ + S3JniExceptionReport; S3JniExceptionClear; \ + (*env)->FatalError(env, MSG); \ + } + +/* +** Declares local var env = s3jni_env(). All JNI calls involve a +** JNIEnv somewhere, always named env, and many of our macros assume +** env is in scope. Where it's not, but should be, use this to make it +** so. +*/ +#define S3JniDeclLocal_env JNIEnv * const env = s3jni_env() + +/* Fail fatally with an OOM message. */ +static inline void s3jni_oom(JNIEnv * const env){ + (*env)->FatalError(env, "SQLite3 JNI is out of memory.") /* does not return */; +} + +/* +** sqlite3_malloc() proxy which fails fatally on OOM. This should +** only be used for routines which manage global state and have no +** recovery strategy for OOM. For sqlite3 API which can reasonably +** return SQLITE_NOMEM, s3jni_malloc() should be used instead. +*/ +static void * s3jni_malloc_or_die(JNIEnv * const env, size_t n){ + void * const rv = sqlite3_malloc(n); + if( n && !rv ) s3jni_oom(env); + return rv; +} + +/* +** Works like sqlite3_malloc() unless built with SQLITE_JNI_FATAL_OOM, +** in which case it calls s3jni_oom() on OOM. +*/ +#ifdef SQLITE_JNI_FATAL_OOM +#define s3jni_malloc(SIZE) s3jni_malloc_or_die(env, SIZE) +#else +#define s3jni_malloc(SIZE) sqlite3_malloc(((void)env,(SIZE))) +/* the ((void)env) trickery here is to avoid ^^^^^^ an otherwise + unused arg in at least one place. */ +#endif + +/* +** Works like sqlite3_realloc() unless built with SQLITE_JNI_FATAL_OOM, +** in which case it calls s3jni_oom() on OOM. +*/ +#ifdef SQLITE_JNI_FATAL_OOM +static void * s3jni_realloc_or_die(JNIEnv * const env, void * p, size_t n){ + void * const rv = sqlite3_realloc(p, (int)n); + if( n && !rv ) s3jni_oom(env); + return rv; +} +#define s3jni_realloc(MEM,SIZE) s3jni_realloc_or_die(env, (MEM), (SIZE)) +#else +#define s3jni_realloc(MEM,SIZE) sqlite3_realloc((MEM), ((void)env, (SIZE))) +#endif + +/* Fail fatally if !EXPR. */ +#define s3jni_oom_fatal(EXPR) if( !(EXPR) ) s3jni_oom(env) +/* Maybe fail fatally if !EXPR. */ +#ifdef SQLITE_JNI_FATAL_OOM +#define s3jni_oom_check s3jni_oom_fatal +#else +#define s3jni_oom_check(EXPR) +#endif +//#define S3JniDb_oom(pDb,EXPR) ((EXPR) ? sqlite3OomFault(pDb) : 0) + +#define s3jni_db_oom(pDb) (void)((pDb) ? ((pDb)->mallocFailed=1) : 0) + +/* Helpers for Java value reference management. */ +static jobject s3jni_ref_global(JNIEnv * const env, jobject const v){ + jobject const rv = v ? (*env)->NewGlobalRef(env, v) : NULL; + s3jni_oom_fatal( v ? !!rv : 1 ); + return rv; +} +static jobject s3jni_ref_local(JNIEnv * const env, jobject const v){ + jobject const rv = v ? (*env)->NewLocalRef(env, v) : NULL; + s3jni_oom_fatal( v ? !!rv : 1 ); + return rv; +} +static inline void s3jni_unref_global(JNIEnv * const env, jobject const v){ + if( v ) (*env)->DeleteGlobalRef(env, v); +} +static inline void s3jni_unref_local(JNIEnv * const env, jobject const v){ + if( v ) (*env)->DeleteLocalRef(env, v); +} +#define S3JniRefGlobal(VAR) s3jni_ref_global(env, (VAR)) +#define S3JniRefLocal(VAR) s3jni_ref_local(env, (VAR)) +#define S3JniUnrefGlobal(VAR) s3jni_unref_global(env, (VAR)) +#define S3JniUnrefLocal(VAR) s3jni_unref_local(env, (VAR)) + +/* +** Lookup key type for use with s3jni_nphop() and a cache of a +** frequently-needed Java-side class reference and one or two Java +** class member IDs. +*/ +typedef struct S3JniNphOp S3JniNphOp; +struct S3JniNphOp { + const int index /* index into S3JniGlobal.nph[] */; + const char * const zName /* Full Java name of the class */; + const char * const zMember /* Name of member property */; + const char * const zTypeSig /* JNI type signature of zMember */; + /* + ** klazz is a global ref to the class represented by pRef. + ** + ** According to: + ** + ** https://developer.ibm.com/articles/j-jni/ + ** + ** > ... the IDs returned for a given class don't change for the + ** lifetime of the JVM process. But the call to get the field or + ** method can require significant work in the JVM, because fields + ** and methods might have been inherited from superclasses, making + ** the JVM walk up the class hierarchy to find them. Because the + ** IDs are the same for a given class, you should look them up + ** once and then reuse them. Similarly, looking up class objects + ** can be expensive, so they should be cached as well. + */ + jclass klazz; + volatile jfieldID fidValue /* NativePointerHolder.nativePointer or + ** OutputPointer.T.value */; + volatile jmethodID midCtor /* klazz's no-arg constructor. Used by + ** NativePointerHolder_new(). */; +}; + +/* +** Cache keys for each concrete NativePointerHolder subclasses and +** OutputPointer.T types. The members are to be used with s3jni_nphop() +** and friends, and each one's member->index corresponds to its index +** in the S3JniGlobal.nph[] array. +*/ +static const struct { + const S3JniNphOp sqlite3; + const S3JniNphOp sqlite3_backup; + const S3JniNphOp sqlite3_blob; + const S3JniNphOp sqlite3_context; + const S3JniNphOp sqlite3_stmt; + const S3JniNphOp sqlite3_value; + const S3JniNphOp OutputPointer_Bool; + const S3JniNphOp OutputPointer_Int32; + const S3JniNphOp OutputPointer_Int64; + const S3JniNphOp OutputPointer_sqlite3; + const S3JniNphOp OutputPointer_sqlite3_blob; + const S3JniNphOp OutputPointer_sqlite3_stmt; + const S3JniNphOp OutputPointer_sqlite3_value; + const S3JniNphOp OutputPointer_String; +#ifdef SQLITE_ENABLE_FTS5 + const S3JniNphOp OutputPointer_ByteArray; + const S3JniNphOp Fts5Context; + const S3JniNphOp Fts5ExtensionApi; + const S3JniNphOp fts5_api; + const S3JniNphOp fts5_tokenizer; + const S3JniNphOp Fts5Tokenizer; +#endif +} S3JniNphOps = { +#define MkRef(INDEX, KLAZZ, MEMBER, SIG) \ + { INDEX, "org/sqlite/jni/" KLAZZ, MEMBER, SIG } +/* NativePointerHolder ref */ +#define RefN(INDEX, KLAZZ) MkRef(INDEX, KLAZZ, "nativePointer", "J") +/* OutputPointer.T ref */ +#define RefO(INDEX, KLAZZ, SIG) MkRef(INDEX, KLAZZ, "value", SIG) + RefN(0, "capi/sqlite3"), + RefN(1, "capi/sqlite3_backup"), + RefN(2, "capi/sqlite3_blob"), + RefN(3, "capi/sqlite3_context"), + RefN(4, "capi/sqlite3_stmt"), + RefN(5, "capi/sqlite3_value"), + RefO(6, "capi/OutputPointer$Bool", "Z"), + RefO(7, "capi/OutputPointer$Int32", "I"), + RefO(8, "capi/OutputPointer$Int64", "J"), + RefO(9, "capi/OutputPointer$sqlite3", + "Lorg/sqlite/jni/capi/sqlite3;"), + RefO(10, "capi/OutputPointer$sqlite3_blob", + "Lorg/sqlite/jni/capi/sqlite3_blob;"), + RefO(11, "capi/OutputPointer$sqlite3_stmt", + "Lorg/sqlite/jni/capi/sqlite3_stmt;"), + RefO(12, "capi/OutputPointer$sqlite3_value", + "Lorg/sqlite/jni/capi/sqlite3_value;"), + RefO(13, "capi/OutputPointer$String", "Ljava/lang/String;"), +#ifdef SQLITE_ENABLE_FTS5 + RefO(14, "capi/OutputPointer$ByteArray", "[B"), + RefN(15, "fts5/Fts5Context"), + RefN(16, "fts5/Fts5ExtensionApi"), + RefN(17, "fts5/fts5_api"), + RefN(18, "fts5/fts5_tokenizer"), + RefN(19, "fts5/Fts5Tokenizer") +#endif +#undef MkRef +#undef RefN +#undef RefO +}; + +#define S3JniNph(T) &S3JniNphOps.T + +enum { + /* + ** Size of the NativePointerHolder cache. Need enough space for + ** (only) the library's NativePointerHolder and OutputPointer types, + ** a fixed count known at build-time. This value needs to be + ** exactly the number of S3JniNphOp entries in the S3JniNphOps + ** object. + */ + S3Jni_NphCache_size = sizeof(S3JniNphOps) / sizeof(S3JniNphOp) +}; + +/* +** State for binding C callbacks to Java methods. +*/ +typedef struct S3JniHook S3JniHook; +struct S3JniHook{ + jobject jObj /* global ref to Java instance */; + jmethodID midCallback /* callback method. Signature depends on + ** jObj's type */; + /* We lookup the jObj.xDestroy() method as-needed for contexts which + ** support custom finalizers. Fundamentally we can support them for + ** any Java type, but we only want to expose support for them where + ** the C API does. */ + jobject jExtra /* Global ref to a per-hook-type value */; + int doXDestroy /* If true then S3JniHook_unref() will call + jObj->xDestroy() if it's available. */; + S3JniHook * pNext /* Next entry in S3Global.hooks.aFree */; +}; +/* For clean bitwise-copy init of local instances. */ +static const S3JniHook S3JniHook_empty = {0,0,0,0,0}; + +/* +** Per-(sqlite3*) state for various JNI bindings. This state is +** allocated as needed, cleaned up in sqlite3_close(_v2)(), and +** recycled when possible. +** +** Trivia: vars and parameters of this type are often named "ps" +** because this class used to have a name for which that abbreviation +** made sense. +*/ +typedef struct S3JniDb S3JniDb; +struct S3JniDb { + sqlite3 *pDb /* The associated db handle */; + jobject jDb /* A global ref of the output object which gets + returned from sqlite3_open(_v2)(). We need this in + order to have an object to pass to routines like + sqlite3_collation_needed()'s callback, or else we + have to dynamically create one for that purpose, + which would be fine except that it would be a + different instance (and maybe even a different + class) than the one the user may expect to + receive. */; + char * zMainDbName /* Holds the string allocated on behalf of + SQLITE_DBCONFIG_MAINDBNAME. */; + struct { + S3JniHook busyHandler; + S3JniHook collationNeeded; + S3JniHook commit; + S3JniHook progress; + S3JniHook rollback; + S3JniHook trace; + S3JniHook update; + S3JniHook auth; +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + S3JniHook preUpdate; +#endif + } hooks; +#ifdef SQLITE_ENABLE_FTS5 + /* FTS5-specific state */ + struct { + jobject jApi /* global ref to s3jni_fts5_api_from_db() */; + } fts; +#endif + S3JniDb * pNext /* Next entry in SJG.perDb.aFree */; +}; + +static const char * const S3JniDb_clientdata_key = "S3JniDb"; +#define S3JniDb_from_clientdata(pDb) \ + (pDb ? sqlite3_get_clientdata(pDb, S3JniDb_clientdata_key) : 0) + +/* +** Cache for per-JNIEnv (i.e. per-thread) data. +** +** Trivia: vars and parameters of this type are often named "jc" +** because this class used to have a name for which that abbreviation +** made sense. +*/ +typedef struct S3JniEnv S3JniEnv; +struct S3JniEnv { + JNIEnv *env /* JNIEnv in which this cache entry was created */; + /* + ** pdbOpening is used to coordinate the Java/DB connection of a + ** being-open()'d db in the face of auto-extensions. + ** Auto-extensions run before we can bind the C db to its Java + ** representation, but auto-extensions require that binding to pass + ** on to their Java-side callbacks. We handle this as follows: + ** + ** - In the JNI side of sqlite3_open(), allocate the Java side of + ** that connection and set pdbOpening to point to that + ** object. + ** + ** - Call sqlite3_open(), which triggers the auto-extension + ** handler. That handler uses pdbOpening to connect the native + ** db handle which it receives with pdbOpening. + ** + ** - When sqlite3_open() returns, check whether pdbOpening->pDb is + ** NULL. If it isn't, auto-extension handling set it up. If it + ** is, complete the Java/C binding unless sqlite3_open() returns + ** a NULL db, in which case free pdbOpening. + */ + S3JniDb * pdbOpening; + S3JniEnv * pNext /* Next entry in SJG.envCache.aHead or + SJG.envCache.aFree */; +}; + +/* +** State for proxying sqlite3_auto_extension() in Java. This was +** initially a separate class from S3JniHook and now the older name is +** retained for readability in the APIs which use this, as well as for +** its better code-searchability. +*/ +typedef S3JniHook S3JniAutoExtension; + +/* +** Type IDs for SQL function categories. +*/ +enum UDFType { + UDF_UNKNOWN_TYPE = 0/*for error propagation*/, + UDF_SCALAR, + UDF_AGGREGATE, + UDF_WINDOW +}; + +/* +** State for binding Java-side UDFs. +*/ +typedef struct S3JniUdf S3JniUdf; +struct S3JniUdf { + jobject jObj /* SQLFunction instance */; + char * zFuncName /* Only for error reporting and debug logging */; + enum UDFType type /* UDF type */; + /** Method IDs for the various UDF methods. */ + jmethodID jmidxFunc /* xFunc method (scalar) */; + jmethodID jmidxStep /* xStep method (aggregate/window) */; + jmethodID jmidxFinal /* xFinal method (aggregate/window) */; + jmethodID jmidxValue /* xValue method (window) */; + jmethodID jmidxInverse /* xInverse method (window) */; + S3JniUdf * pNext /* Next entry in SJG.udf.aFree. */; +}; + +#if defined(SQLITE_JNI_ENABLE_METRICS) && 0==SQLITE_JNI_ENABLE_METRICS +# undef SQLITE_JNI_ENABLE_METRICS +#endif + +/* +** If true, modifying S3JniGlobal.metrics is protected by a mutex, +** else it isn't. +*/ +#ifdef SQLITE_DEBUG +# define S3JNI_METRICS_MUTEX SQLITE_THREADSAFE +#else +# define S3JNI_METRICS_MUTEX 0 +#endif +#ifndef SQLITE_JNI_ENABLE_METRICS +# undef S3JNI_METRICS_MUTEX +# define S3JNI_METRICS_MUTEX 0 +#endif + +/* +** Global state, e.g. caches and metrics. +*/ +typedef struct S3JniGlobalType S3JniGlobalType; +struct S3JniGlobalType { + /* + ** According to: https://developer.ibm.com/articles/j-jni/ + ** + ** > A thread can get a JNIEnv by calling GetEnv() using the JNI + ** invocation interface through a JavaVM object. The JavaVM object + ** itself can be obtained by calling the JNI GetJavaVM() method + ** using a JNIEnv object and can be cached and shared across + ** threads. Caching a copy of the JavaVM object enables any thread + ** with access to the cached object to get access to its own + ** JNIEnv when necessary. + */ + JavaVM * jvm; + /* + ** Global mutex. It must not be used for anything which might call + ** back into the JNI layer. + */ + sqlite3_mutex * mutex; + /* + ** Cache of references to Java classes and method IDs for + ** NativePointerHolder subclasses and OutputPointer.T types. + */ + struct { + S3JniNphOp list[S3Jni_NphCache_size]; + sqlite3_mutex * mutex; /* mutex for this->list */ + volatile void const * locker; /* sanity-checking-only context object + for this->mutex */ + } nph; + /* + ** Cache of per-thread state. + */ + struct { + S3JniEnv * aHead /* Linked list of in-use instances */; + S3JniEnv * aFree /* Linked list of free instances */; + sqlite3_mutex * mutex /* mutex for aHead and aFree. */; + volatile void const * locker /* env mutex is held on this + object's behalf. Used only for + sanity checking. */; + } envCache; + /* + ** Per-db state. This can move into the core library once we can tie + ** client-defined state to db handles there. + */ + struct { + S3JniDb * aFree /* Linked list of free instances */; + sqlite3_mutex * mutex /* mutex for aHead and aFree */; + volatile void const * locker + /* perDb mutex is held on this object's behalf. Used only for + sanity checking. Note that the mutex is at the class level, not + instance level. */; + } perDb; + struct { + S3JniUdf * aFree /* Head of the free-item list. Guarded by global + mutex. */; + } udf; + /* + ** Refs to global classes and methods. Obtained during static init + ** and never released. + */ + struct { + jclass cLong /* global ref to java.lang.Long */; + jclass cString /* global ref to java.lang.String */; + jobject oCharsetUtf8 /* global ref to StandardCharset.UTF_8 */; + jmethodID ctorLong1 /* the Long(long) constructor */; + jmethodID ctorStringBA /* the String(byte[],Charset) constructor */; + jmethodID stringGetBytes /* the String.getBytes(Charset) method */; + + /* + ByteBuffer may or may not be supported via JNI on any given + platform: + + https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/functions.html#nio_support + + We only store a ref to byteBuffer.klazz if JNI support for + ByteBuffer is available (which we determine during static init). + */ + struct { + jclass klazz /* global ref to java.nio.ByteBuffer */; + jmethodID midAlloc /* ByteBuffer.allocateDirect() */; + jmethodID midLimit /* ByteBuffer.limit() */; + } byteBuffer; + } g; + /* + ** The list of Java-side auto-extensions + ** (org.sqlite.jni.capi.AutoExtensionCallback objects). + */ + struct { + S3JniAutoExtension *aExt /* The auto-extension list. It is + maintained such that all active + entries are in the first contiguous + nExt array elements. */; + int nAlloc /* number of entries allocated for aExt, + as distinct from the number of active + entries. */; + int nExt /* number of active entries in aExt, all in the + first nExt'th array elements. */; + sqlite3_mutex * mutex /* mutex for manipulation/traversal of aExt */; + volatile const void * locker /* object on whose behalf the mutex + is held. Only for sanity checking + in debug builds. */; + } autoExt; +#ifdef SQLITE_ENABLE_FTS5 + struct { + volatile jobject jExt /* Global ref to Java singleton for the + Fts5ExtensionApi instance. */; + struct { + jfieldID fidA /* Fts5Phrase::a member */; + jfieldID fidB /* Fts5Phrase::b member */; + } jPhraseIter; + } fts5; +#endif + struct { +#ifdef SQLITE_ENABLE_SQLLOG + S3JniHook sqllog /* sqlite3_config(SQLITE_CONFIG_SQLLOG) callback */; +#endif + S3JniHook configlog /* sqlite3_config(SQLITE_CONFIG_LOG) callback */; + S3JniHook * aFree /* free-item list, for recycling. */; + sqlite3_mutex * mutex /* mutex for aFree */; + volatile const void * locker /* object on whose behalf the mutex + is held. Only for sanity checking + in debug builds. */; + } hook; +#ifdef SQLITE_JNI_ENABLE_METRICS + /* Internal metrics. */ + struct { + volatile unsigned nEnvHit; + volatile unsigned nEnvMiss; + volatile unsigned nEnvAlloc; + volatile unsigned nMutexEnv /* number of times envCache.mutex was entered for + a S3JniEnv operation. */; + volatile unsigned nMutexNph /* number of times SJG.mutex was entered */; + volatile unsigned nMutexHook /* number of times SJG.mutex hooks.was entered */; + volatile unsigned nMutexPerDb /* number of times perDb.mutex was entered */; + volatile unsigned nMutexAutoExt /* number of times autoExt.mutex was entered */; + volatile unsigned nMutexGlobal /* number of times global mutex was entered. */; + volatile unsigned nMutexUdf /* number of times global mutex was entered + for UDFs. */; + volatile unsigned nDestroy /* xDestroy() calls across all types */; + volatile unsigned nPdbAlloc /* Number of S3JniDb alloced. */; + volatile unsigned nPdbRecycled /* Number of S3JniDb reused. */; + volatile unsigned nUdfAlloc /* Number of S3JniUdf alloced. */; + volatile unsigned nUdfRecycled /* Number of S3JniUdf reused. */; + volatile unsigned nHookAlloc /* Number of S3JniHook alloced. */; + volatile unsigned nHookRecycled /* Number of S3JniHook reused. */; + struct { + /* Number of calls for each type of UDF callback. */ + volatile unsigned nFunc; + volatile unsigned nStep; + volatile unsigned nFinal; + volatile unsigned nValue; + volatile unsigned nInverse; + } udf; + unsigned nMetrics /* Total number of mutex-locked + metrics increments. */; +#if S3JNI_METRICS_MUTEX + sqlite3_mutex * mutex; +#endif + } metrics; +#endif /* SQLITE_JNI_ENABLE_METRICS */ +}; +static S3JniGlobalType S3JniGlobal = {}; +#define SJG S3JniGlobal + +/* Increments *p, possibly protected by a mutex. */ +#ifndef SQLITE_JNI_ENABLE_METRICS +#define s3jni_incr(PTR) +#elif S3JNI_METRICS_MUTEX +static void s3jni_incr( volatile unsigned int * const p ){ + sqlite3_mutex_enter(SJG.metrics.mutex); + ++SJG.metrics.nMetrics; + ++(*p); + sqlite3_mutex_leave(SJG.metrics.mutex); +} +#else +#define s3jni_incr(PTR) ++(*(PTR)) +#endif + +/* Helpers for working with specific mutexes. */ +#if SQLITE_THREADSAFE +#define s3jni_mutex_enter2(M, Metric) \ + sqlite3_mutex_enter( M ); \ + s3jni_incr( &SJG.metrics.Metric ) +#define s3jni_mutex_leave2(M) \ + sqlite3_mutex_leave( M ) + +#define s3jni_mutex_enter(M, L, Metric) \ + assert( (void*)env != (void*)L && "Invalid use of " #L); \ + s3jni_mutex_enter2( M, Metric ); \ + L = env +#define s3jni_mutex_leave(M, L) \ + assert( (void*)env == (void*)L && "Invalid use of " #L); \ + L = 0; \ + s3jni_mutex_leave2( M ) + +#define S3JniEnv_mutex_assertLocked \ + assert( 0 != SJG.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) +#define S3JniEnv_mutex_assertLocker \ + assert( (env) == SJG.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) +#define S3JniEnv_mutex_assertNotLocker \ + assert( (env) != SJG.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) + +#define S3JniEnv_mutex_enter \ + s3jni_mutex_enter( SJG.envCache.mutex, SJG.envCache.locker, nMutexEnv ) +#define S3JniEnv_mutex_leave \ + s3jni_mutex_leave( SJG.envCache.mutex, SJG.envCache.locker ) + +#define S3JniAutoExt_mutex_enter \ + s3jni_mutex_enter( SJG.autoExt.mutex, SJG.autoExt.locker, nMutexAutoExt ) +#define S3JniAutoExt_mutex_leave \ + s3jni_mutex_leave( SJG.autoExt.mutex, SJG.autoExt.locker ) +#define S3JniAutoExt_mutex_assertLocker \ + assert( env == SJG.autoExt.locker && "Misuse of S3JniGlobal.autoExt.mutex" ) + +#define S3JniGlobal_mutex_enter \ + s3jni_mutex_enter2( SJG.mutex, nMutexGlobal ) +#define S3JniGlobal_mutex_leave \ + s3jni_mutex_leave2( SJG.mutex ) + +#define S3JniHook_mutex_enter \ + s3jni_mutex_enter( SJG.hook.mutex, SJG.hook.locker, nMutexHook ) +#define S3JniHook_mutex_leave \ + s3jni_mutex_leave( SJG.hook.mutex, SJG.hook.locker ) + +#define S3JniNph_mutex_enter \ + s3jni_mutex_enter( SJG.nph.mutex, SJG.nph.locker, nMutexNph ) +#define S3JniNph_mutex_leave \ + s3jni_mutex_leave( SJG.nph.mutex, SJG.nph.locker ) + +#define S3JniDb_mutex_assertLocker \ + assert( (env) == SJG.perDb.locker && "Misuse of S3JniGlobal.perDb.mutex" ) +#define S3JniDb_mutex_enter \ + s3jni_mutex_enter( SJG.perDb.mutex, SJG.perDb.locker, nMutexPerDb ) +#define S3JniDb_mutex_leave \ + s3jni_mutex_leave( SJG.perDb.mutex, SJG.perDb.locker ) + +#else /* SQLITE_THREADSAFE==0 */ +#define S3JniAutoExt_mutex_assertLocker +#define S3JniAutoExt_mutex_enter +#define S3JniAutoExt_mutex_leave +#define S3JniDb_mutex_assertLocker +#define S3JniDb_mutex_enter +#define S3JniDb_mutex_leave +#define S3JniEnv_mutex_assertLocked +#define S3JniEnv_mutex_assertLocker +#define S3JniEnv_mutex_assertNotLocker +#define S3JniEnv_mutex_enter +#define S3JniEnv_mutex_leave +#define S3JniGlobal_mutex_enter +#define S3JniGlobal_mutex_leave +#define S3JniHook_mutex_enter +#define S3JniHook_mutex_leave +#define S3JniNph_mutex_enter +#define S3JniNph_mutex_leave +#endif + +/* Helpers for jstring and jbyteArray. */ +static const char * s3jni__jstring_to_mutf8(JNIEnv * const env, jstring v ){ + const char *z = v ? (*env)->GetStringUTFChars(env, v, NULL) : 0; + s3jni_oom_check( v ? !!z : !z ); + return z; +} + +#define s3jni_jstring_to_mutf8(ARG) s3jni__jstring_to_mutf8(env, (ARG)) +#define s3jni_mutf8_release(ARG,VAR) if( VAR ) (*env)->ReleaseStringUTFChars(env, ARG, VAR) + +/* +** If jBA is not NULL then its GetByteArrayElements() value is +** returned. If jBA is not NULL and nBA is not NULL then *nBA is set +** to the GetArrayLength() of jBA. If GetByteArrayElements() requires +** an allocation and that allocation fails then this function either +** fails fatally or returns 0, depending on build-time options. + */ +static jbyte * s3jni__jbyteArray_bytes2(JNIEnv * const env, jbyteArray jBA, jsize * nBA ){ + jbyte * const rv = jBA ? (*env)->GetByteArrayElements(env, jBA, NULL) : 0; + s3jni_oom_check( jBA ? !!rv : 1 ); + if( jBA && nBA ) *nBA = (*env)->GetArrayLength(env, jBA); + return rv; +} + +#define s3jni_jbyteArray_bytes2(jByteArray,ptrToSz) \ + s3jni__jbyteArray_bytes2(env, (jByteArray), (ptrToSz)) +#define s3jni_jbyteArray_bytes(jByteArray) s3jni__jbyteArray_bytes2(env, (jByteArray), 0) +#define s3jni_jbyteArray_release(jByteArray,jBytes) \ + if( jBytes ) (*env)->ReleaseByteArrayElements(env, jByteArray, jBytes, JNI_ABORT) +#define s3jni_jbyteArray_commit(jByteArray,jBytes) \ + if( jBytes ) (*env)->ReleaseByteArrayElements(env, jByteArray, jBytes, JNI_COMMIT) + +/* +** If jbb is-a java.nio.Buffer object and the JNI environment supports +** it, *pBuf is set to the buffer's memory and *pN is set to its +** limit() (as opposed to its capacity()). If jbb is NULL, not a +** Buffer, or the JNI environment does not support that operation, +** *pBuf is set to 0 and *pN is set to 0. +** +** Note that the length of the buffer can be larger than SQLITE_LIMIT +** but this function does not know what byte range of the buffer is +** required so cannot check for that violation. The caller is required +** to ensure that any to-be-bind()ed range fits within SQLITE_LIMIT. +** +** Sidebar: it is unfortunate that we cannot get ByteBuffer.limit() +** via a JNI method like we can for ByteBuffer.capacity(). We instead +** have to call back into Java to get the limit(). Depending on how +** the ByteBuffer is used, the limit and capacity might be the same, +** but when reusing a buffer, the limit may well change whereas the +** capacity is fixed. The problem with, e.g., read()ing blob data to a +** ByteBuffer's memory based on its capacity is that Java-level code +** is restricted to accessing the range specified in +** ByteBuffer.limit(). If we were to honor only the capacity, we +** could end up writing to, or reading from, parts of a ByteBuffer +** which client code itself cannot access without explicitly modifying +** the limit. The penalty we pay for this correctness is that we must +** call into Java to get the limit() of every ByteBuffer we work with. +** +** An alternative to having to call into ByteBuffer.limit() from here +** would be to add private native impls of all ByteBuffer-using +** methods, each of which adds a jint parameter which _must_ be set to +** theBuffer.limit() by public Java APIs which use those private impls +** to do the real work. +*/ +static void s3jni__get_nio_buffer(JNIEnv * const env, jobject jbb, void **pBuf, jint * pN ){ + *pBuf = 0; + *pN = 0; + if( jbb ){ + *pBuf = (*env)->GetDirectBufferAddress(env, jbb); + if( *pBuf ){ + /* + ** Maintenance reminder: do not use + ** (*env)->GetDirectBufferCapacity(env,jbb), even though it + ** would be much faster, for reasons explained in this + ** function's comments. + */ + *pN = (*env)->CallIntMethod(env, jbb, SJG.g.byteBuffer.midLimit); + S3JniExceptionIsFatal("Error calling ByteBuffer.limit() method."); + } + } +} +#define s3jni_get_nio_buffer(JOBJ,vpOut,jpOut) \ + s3jni__get_nio_buffer(env,(JOBJ),(vpOut),(jpOut)) + +/* +** Returns the current JNIEnv object. Fails fatally if it cannot find +** the object. +*/ +static JNIEnv * s3jni_env(void){ + JNIEnv * env = 0; + if( (*SJG.jvm)->GetEnv(SJG.jvm, (void **)&env, + JNI_VERSION_1_8) ){ + fprintf(stderr, "Fatal error: cannot get current JNIEnv.\n"); + abort(); + } + return env; +} + +/* +** Fetches the S3JniGlobal.envCache row for the given env, allocating a +** row if needed. When a row is allocated, its state is initialized +** insofar as possible. Calls (*env)->FatalError() if allocation of an +** entry fails. That's hypothetically possible but "shouldn't happen." +*/ +static S3JniEnv * S3JniEnv__get(JNIEnv * const env){ + struct S3JniEnv * row; + S3JniEnv_mutex_enter; + row = SJG.envCache.aHead; + for( ; row; row = row->pNext ){ + if( row->env == env ){ + s3jni_incr( &SJG.metrics.nEnvHit ); + S3JniEnv_mutex_leave; + return row; + } + } + s3jni_incr( &SJG.metrics.nEnvMiss ); + row = SJG.envCache.aFree; + if( row ){ + SJG.envCache.aFree = row->pNext; + }else{ + row = s3jni_malloc_or_die(env, sizeof(*row)); + s3jni_incr( &SJG.metrics.nEnvAlloc ); + } + memset(row, 0, sizeof(*row)); + row->pNext = SJG.envCache.aHead; + SJG.envCache.aHead = row; + row->env = env; + + S3JniEnv_mutex_leave; + return row; +} + +#define S3JniEnv_get() S3JniEnv__get(env) + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own Java/JNI bindings. +** +** For purposes of certain hand-crafted JNI function bindings, we +** need a way of reporting errors which is consistent with the rest of +** the C API, as opposed to throwing Java exceptions. To that end, this +** internal-use-only function is a thin proxy around +** sqlite3ErrorWithMessage(). The intent is that it only be used from +** JNI bindings such as sqlite3_prepare_v2/v3(), and definitely not +** from client code. +** +** Returns err_code. +*/ +static int s3jni_db_error(sqlite3* const db, int err_code, + const char * const zMsg){ + if( db!=0 ){ + if( 0==zMsg ){ + sqlite3Error(db, err_code); + }else{ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + } + } + return err_code; +} + +/* +** Creates a new jByteArray of length nP, copies p's contents into it, +** and returns that byte array (NULL on OOM unless fail-fast alloc +** errors are enabled). p may be NULL, in which case the array is +** created but no bytes are filled. +*/ +static jbyteArray s3jni__new_jbyteArray(JNIEnv * const env, + const void * const p, int nP){ + jbyteArray jba = (*env)->NewByteArray(env, (jint)nP); + + s3jni_oom_check( jba ); + if( jba && p ){ + (*env)->SetByteArrayRegion(env, jba, 0, (jint)nP, (const jbyte*)p); + } + return jba; +} + +#define s3jni_new_jbyteArray(P,n) s3jni__new_jbyteArray(env, P, n) + + +/* +** Uses the java.lang.String(byte[],Charset) constructor to create a +** new String from UTF-8 string z. n is the number of bytes to +** copy. If n<0 then sqlite3Strlen30() is used to calculate it. +** +** Returns NULL if z is NULL or on OOM, else returns a new jstring +** owned by the caller. +** +** Sidebar: this is a painfully inefficient way to convert from +** standard UTF-8 to a Java string, but JNI offers only algorithms for +** working with MUTF-8, not UTF-8. +*/ +static jstring s3jni__utf8_to_jstring(JNIEnv * const env, + const char * const z, int n){ + jstring rv = NULL; + if( 0==n || (n<0 && z && !z[0]) ){ + /* Fast-track the empty-string case via the MUTF-8 API. We could + hypothetically do this for any strings where n<4 and z is + NUL-terminated and none of z[0..3] are NUL bytes. */ + rv = (*env)->NewStringUTF(env, ""); + s3jni_oom_check( rv ); + }else if( z ){ + jbyteArray jba; + if( n<0 ) n = sqlite3Strlen30(z); + jba = s3jni_new_jbyteArray((unsigned const char *)z, n); + if( jba ){ + rv = (*env)->NewObject(env, SJG.g.cString, SJG.g.ctorStringBA, + jba, SJG.g.oCharsetUtf8); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + } + S3JniUnrefLocal(jba); + } + s3jni_oom_check( rv ); + } + return rv; +} +#define s3jni_utf8_to_jstring(CStr,n) s3jni__utf8_to_jstring(env, CStr, n) + +/* +** Converts the given java.lang.String object into a NUL-terminated +** UTF-8 C-string by calling jstr.getBytes(StandardCharset.UTF_8). +** Returns NULL if jstr is NULL or on allocation error. If jstr is not +** NULL and nLen is not NULL then nLen is set to the length of the +** returned string, not including the terminating NUL. If jstr is not +** NULL and it returns NULL, this indicates an allocation error. In +** that case, if nLen is not NULL then it is either set to 0 (if +** fetching of jstr's bytes fails to allocate) or set to what would +** have been the length of the string had C-string allocation +** succeeded. +** +** The returned memory is allocated from sqlite3_malloc() and +** ownership is transferred to the caller. +*/ +static char * s3jni__jstring_to_utf8(JNIEnv * const env, + jstring jstr, int *nLen){ + jbyteArray jba; + jsize nBA; + char *rv; + + if( !jstr ) return 0; + jba = (*env)->CallObjectMethod(env, jstr, SJG.g.stringGetBytes, + SJG.g.oCharsetUtf8); + + if( (*env)->ExceptionCheck(env) || !jba + /* order of these checks is significant for -Xlint:jni */ ) { + S3JniExceptionReport; + s3jni_oom_check( jba ); + if( nLen ) *nLen = 0; + return 0; + } + nBA = (*env)->GetArrayLength(env, jba); + if( nLen ) *nLen = (int)nBA; + rv = s3jni_malloc( nBA + 1 ); + if( rv ){ + (*env)->GetByteArrayRegion(env, jba, 0, nBA, (jbyte*)rv); + rv[nBA] = 0; + } + S3JniUnrefLocal(jba); + return rv; +} +#define s3jni_jstring_to_utf8(JStr,n) s3jni__jstring_to_utf8(env, JStr, n) + +/* +** Expects to be passed a pointer from sqlite3_column_text16() or +** sqlite3_value_text16() and a byte-length value from +** sqlite3_column_bytes16() or sqlite3_value_bytes16(). It creates a +** Java String of exactly half that character length, returning NULL +** if !p or (*env)->NewString() fails. +*/ +static jstring s3jni_text16_to_jstring(JNIEnv * const env, const void * const p, int nP){ + jstring const rv = p + ? (*env)->NewString(env, (const jchar *)p, (jsize)(nP/2)) + : NULL; + s3jni_oom_check( p ? !!rv : 1 ); + return rv; +} + +/* +** Creates a new ByteBuffer instance with a capacity of n. assert()s +** that SJG.g.byteBuffer.klazz is not 0 and n>0. +*/ +static jobject s3jni__new_ByteBuffer(JNIEnv * const env, int n){ + jobject rv = 0; + assert( SJG.g.byteBuffer.klazz ); + assert( SJG.g.byteBuffer.midAlloc ); + assert( n > 0 ); + rv = (*env)->CallStaticObjectMethod(env, SJG.g.byteBuffer.klazz, + SJG.g.byteBuffer.midAlloc, (jint)n); + S3JniIfThrew { + S3JniExceptionReport; + S3JniExceptionClear; + } + s3jni_oom_check( rv ); + return rv; +} + +/* +** If n>0 and sqlite3_jni_supports_nio() is true then this creates a +** new ByteBuffer object and copies n bytes from p to it. Returns NULL +** if n is 0, sqlite3_jni_supports_nio() is false, or on allocation +** error (unless fatal alloc failures are enabled). +*/ +static jobject s3jni__blob_to_ByteBuffer(JNIEnv * const env, + const void * p, int n){ + jobject rv = NULL; + assert( n >= 0 ); + if( 0==n || !SJG.g.byteBuffer.klazz ){ + return NULL; + } + rv = s3jni__new_ByteBuffer(env, n); + if( rv ){ + void * tgt = (*env)->GetDirectBufferAddress(env, rv); + memcpy(tgt, p, (size_t)n); + } + return rv; +} + + +/* +** Requires jx to be a Throwable. Calls its toString() method and +** returns its value converted to a UTF-8 string. The caller owns the +** returned string and must eventually sqlite3_free() it. Returns 0 +** if there is a problem fetching the info or on OOM. +** +** Design note: we use toString() instead of getMessage() because the +** former includes the exception type's name: +** +** Exception e = new RuntimeException("Hi"); +** System.out.println(e.toString()); // java.lang.RuntimeException: Hi +** System.out.println(e.getMessage()); // Hi +*/ +static char * s3jni_exception_error_msg(JNIEnv * const env, jthrowable jx){ + jmethodID mid; + jstring msg; + char * zMsg; + jclass const klazz = (*env)->GetObjectClass(env, jx); + mid = (*env)->GetMethodID(env, klazz, "toString", "()Ljava/lang/String;"); + S3JniUnrefLocal(klazz); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + return 0; + } + msg = (*env)->CallObjectMethod(env, jx, mid); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + return 0; + } + zMsg = s3jni_jstring_to_utf8( msg, 0); + S3JniUnrefLocal(msg); + return zMsg; +} + +/* +** Extracts env's current exception, sets ps->pDb's error message to +** its message string, and clears the exception. If errCode is non-0, +** it is used as-is, else SQLITE_ERROR is assumed. If there's a +** problem extracting the exception's message, it's treated as +** non-fatal and zDfltMsg is used in its place. +** +** Locks the global S3JniDb mutex. +** +** This must only be called if a JNI exception is pending. +** +** Returns errCode unless it is 0, in which case SQLITE_ERROR is +** returned. +*/ +static int s3jni__db_exception(JNIEnv * const env, sqlite3 * const pDb, + int errCode, const char *zDfltMsg){ + jthrowable const ex = (*env)->ExceptionOccurred(env); + + if( 0==errCode ) errCode = SQLITE_ERROR; + if( ex ){ + char * zMsg; + S3JniExceptionClear; + zMsg = s3jni_exception_error_msg(env, ex); + s3jni_db_error(pDb, errCode, zMsg ? zMsg : zDfltMsg); + sqlite3_free(zMsg); + S3JniUnrefLocal(ex); + }else if( zDfltMsg ){ + s3jni_db_error(pDb, errCode, zDfltMsg); + } + return errCode; +} +#define s3jni_db_exception(pDb,ERRCODE,DFLTMSG) \ + s3jni__db_exception(env, (pDb), (ERRCODE), (DFLTMSG) ) + +/* +** Extracts the (void xDestroy()) method from jObj and applies it to +** jObj. If jObj is NULL, this is a no-op. The lack of an xDestroy() +** method is silently ignored. Any exceptions thrown by xDestroy() +** trigger a warning to stdout or stderr and then the exception is +** suppressed. +*/ +static void s3jni__call_xDestroy(JNIEnv * const env, jobject jObj){ + if( jObj ){ + jclass const klazz = (*env)->GetObjectClass(env, jObj); + jmethodID method = (*env)->GetMethodID(env, klazz, "xDestroy", "()V"); + + S3JniUnrefLocal(klazz); + if( method ){ + s3jni_incr( &SJG.metrics.nDestroy ); + (*env)->CallVoidMethod(env, jObj, method); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("xDestroy() callback"); + S3JniExceptionClear; + } + }else{ + /* Non-fatal. */ + S3JniExceptionClear; + } + } +} +#define s3jni_call_xDestroy(JOBJ) s3jni__call_xDestroy(env, (JOBJ)) + +/* +** Internal helper for many hook callback impls. Locks the S3JniDb +** mutex, makes a copy of src into dest, with a some differences: (1) +** if src->jObj or src->jExtra are not NULL then dest will be a new +** LOCAL ref to it instead of a copy of the prior GLOBAL ref. (2) +** dest->doXDestroy is always false. +** +** If dest->jObj is not NULL when this returns then the caller is +** obligated to eventually free the new ref by passing *dest to +** S3JniHook_localundup(). The dest pointer must NOT be passed to +** S3JniHook_unref(), as that routine assumes that dest->jObj/jExtra +** are GLOBAL refs (it's illegal to try to unref the wrong ref type). +** +** Background: when running a hook we need a call-local copy lest +** another thread modify the hook while we're running it. That copy +** has to have its own Java reference, but it need only be call-local. +*/ +static void S3JniHook__localdup( JNIEnv * const env, S3JniHook const * const src, + S3JniHook * const dest ){ + S3JniHook_mutex_enter; + *dest = *src; + if(src->jObj) dest->jObj = S3JniRefLocal(src->jObj); + if(src->jExtra) dest->jExtra = S3JniRefLocal(src->jExtra); + dest->doXDestroy = 0; + S3JniHook_mutex_leave; +} +#define S3JniHook_localdup(src,dest) S3JniHook__localdup(env,src,dest) + +static void S3JniHook__localundup( JNIEnv * const env, S3JniHook * const h ){ + S3JniUnrefLocal(h->jObj); + S3JniUnrefLocal(h->jExtra); + *h = S3JniHook_empty; +} +#define S3JniHook_localundup(HOOK) S3JniHook__localundup(env, &(HOOK)) + +/* +** Removes any Java references from s and clears its state. If +** doXDestroy is true and s->jObj is not NULL, s->jObj +** is passed to s3jni_call_xDestroy() before any references are +** cleared. It is legal to call this when the object has no Java +** references. s must not be NULL. +*/ +static void S3JniHook__unref(JNIEnv * const env, S3JniHook * const s){ + if( s->jObj ){ + if( s->doXDestroy ){ + s3jni_call_xDestroy(s->jObj); + } + S3JniUnrefGlobal(s->jObj); + S3JniUnrefGlobal(s->jExtra); + }else{ + assert( !s->jExtra ); + } + *s = S3JniHook_empty; +} +#define S3JniHook_unref(hook) S3JniHook__unref(env, (hook)) + +/* +** Allocates one blank S3JniHook object from the recycling bin, if +** available, else from the heap. Returns NULL or dies on OOM, +** depending on build options. Locks on SJG.hooks.mutex. +*/ +static S3JniHook *S3JniHook__alloc(JNIEnv * const env){ + S3JniHook * p = 0; + S3JniHook_mutex_enter; + if( SJG.hook.aFree ){ + p = SJG.hook.aFree; + SJG.hook.aFree = p->pNext; + p->pNext = 0; + s3jni_incr(&SJG.metrics.nHookRecycled); + } + S3JniHook_mutex_leave; + if( 0==p ){ + p = s3jni_malloc(sizeof(S3JniHook)); + if( p ){ + s3jni_incr(&SJG.metrics.nHookAlloc); + } + } + if( p ){ + *p = S3JniHook_empty; + } + return p; +} +#define S3JniHook_alloc() S3JniHook__alloc(env) + +/* +** The rightful fate of all results from S3JniHook_alloc(). Locks on +** SJG.hook.mutex. +*/ +static void S3JniHook__free(JNIEnv * const env, S3JniHook * const p){ + if(p){ + assert( !p->pNext ); + S3JniHook_unref(p); + S3JniHook_mutex_enter; + p->pNext = SJG.hook.aFree; + SJG.hook.aFree = p; + S3JniHook_mutex_leave; + } +} +#define S3JniHook_free(hook) S3JniHook__free(env, hook) + +#if 0 +/* S3JniHook__free() without the lock: caller must hold the global mutex */ +static void S3JniHook__free_unlocked(JNIEnv * const env, S3JniHook * const p){ + if(p){ + assert( !p->pNext ); + assert( p->pNext != SJG.hook.aFree ); + S3JniHook_unref(p); + p->pNext = SJG.hook.aFree; + SJG.hook.aFree = p; + } +} +#define S3JniHook_free_unlocked(hook) S3JniHook__free_unlocked(env, hook) +#endif + +/* +** Clears all of s's state. Requires that that the caller has locked +** S3JniGlobal.perDb.mutex. Make sure to do anything needed with +** s->pNext and s->pPrev before calling this, as this clears them. +*/ +static void S3JniDb_clear(JNIEnv * const env, S3JniDb * const s){ + S3JniDb_mutex_assertLocker; + sqlite3_free( s->zMainDbName ); +#define UNHOOK(MEMBER) \ + S3JniHook_unref(&s->hooks.MEMBER) + UNHOOK(auth); + UNHOOK(busyHandler); + UNHOOK(collationNeeded); + UNHOOK(commit); + UNHOOK(progress); + UNHOOK(rollback); + UNHOOK(trace); + UNHOOK(update); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + UNHOOK(preUpdate); +#endif +#undef UNHOOK + S3JniUnrefGlobal(s->jDb); + memset(s, 0, sizeof(S3JniDb)); +} + +/* +** Clears s's state and moves it to the free-list. Requires that +** S3JniGlobal.perDb.mutex is locked. +*/ +static void S3JniDb__set_aside_unlocked(JNIEnv * const env, S3JniDb * const s){ + assert( s ); + S3JniDb_mutex_assertLocker; + if( s ){ + S3JniDb_clear(env, s); + s->pNext = SJG.perDb.aFree; + SJG.perDb.aFree = s; + } +} +#define S3JniDb_set_aside_unlocked(JniDb) S3JniDb__set_aside_unlocked(env, JniDb) + +static void S3JniDb__set_aside(JNIEnv * const env, S3JniDb * const s){ + S3JniDb_mutex_enter; + S3JniDb_set_aside_unlocked(s); + S3JniDb_mutex_leave; +} +#define S3JniDb_set_aside(JNIDB) S3JniDb__set_aside(env, JNIDB) + +/* +** Uncache any state for the given JNIEnv, clearing all Java +** references the cache owns. Returns true if env was cached and false +** if it was not found in the cache. Ownership of the S3JniEnv object +** associated with the given argument is transferred to this function, +** which makes it free for re-use. +** +** Requires that the env mutex be locked. +*/ +static int S3JniEnv_uncache(JNIEnv * const env){ + struct S3JniEnv * row; + struct S3JniEnv * pPrev = 0; + + S3JniEnv_mutex_assertLocked; + row = SJG.envCache.aHead; + for( ; row; pPrev = row, row = row->pNext ){ + if( row->env == env ){ + break; + } + } + if( !row ){ + return 0; + } + if( pPrev) pPrev->pNext = row->pNext; + else{ + assert( SJG.envCache.aHead == row ); + SJG.envCache.aHead = row->pNext; + } + memset(row, 0, sizeof(S3JniEnv)); + row->pNext = SJG.envCache.aFree; + SJG.envCache.aFree = row; + return 1; +} + +/* +** Fetches the given nph-ref from cache the cache and returns the +** object with its klazz member set. This is an O(1) operation except +** on the first call for a given pRef, during which pRef->klazz and +** pRef->pRef are initialized thread-safely. In the latter case it's +** still effectively O(1), but with a much longer 1. +** +** It is up to the caller to populate the other members of the +** returned object if needed, taking care to lock the modification +** with S3JniNph_mutex_enter/leave. +** +** This simple cache catches >99% of searches in the current +** (2023-07-31) tests. +*/ +static S3JniNphOp * s3jni__nphop(JNIEnv * const env, S3JniNphOp const* pRef){ + S3JniNphOp * const pNC = &SJG.nph.list[pRef->index]; + + assert( (void*)pRef>=(void*)&S3JniNphOps && (void*)pRef<(void*)(&S3JniNphOps + 1) + && "pRef is out of range" ); + assert( pRef->index>=0 + && (pRef->index < (sizeof(S3JniNphOps) / sizeof(S3JniNphOp))) + && "pRef->index is out of range" ); + if( !pNC->klazz ){ + S3JniNph_mutex_enter; + if( !pNC->klazz ){ + jclass const klazz = (*env)->FindClass(env, pRef->zName); + //printf("FindClass %s\n", pRef->zName); + S3JniExceptionIsFatal("FindClass() unexpectedly threw"); + pNC->klazz = S3JniRefGlobal(klazz); + } + S3JniNph_mutex_leave; + } + assert( pNC->klazz ); + return pNC; +} + +#define s3jni_nphop(PRef) s3jni__nphop(env, PRef) + +/* +** Common code for accessor functions for NativePointerHolder and +** OutputPointer types. pRef must be a pointer from S3JniNphOps. jOut +** must be an instance of that class (Java's type safety takes care of +** that requirement). If necessary, this fetches the jfieldID for +** jOut's pRef->zMember, which must be of the type represented by the +** JNI type signature pRef->zTypeSig, and stores it in +** S3JniGlobal.nph.list[pRef->index]. Fails fatally if the pRef->zMember +** property is not found, as that presents a serious internal misuse. +** +** Property lookups are cached on a per-pRef basis. +*/ +static jfieldID s3jni_nphop_field(JNIEnv * const env, S3JniNphOp const* pRef){ + S3JniNphOp * const pNC = s3jni_nphop(pRef); + + if( !pNC->fidValue ){ + S3JniNph_mutex_enter; + if( !pNC->fidValue ){ + pNC->fidValue = (*env)->GetFieldID(env, pNC->klazz, + pRef->zMember, pRef->zTypeSig); + S3JniExceptionIsFatal("Code maintenance required: missing " + "required S3JniNphOp::fidValue."); + } + S3JniNph_mutex_leave; + } + assert( pNC->fidValue ); + return pNC->fidValue; +} + +/* +** Sets a native ptr value in NativePointerHolder object jNph, +** which must be of the native type described by pRef. jNph +** may not be NULL. +*/ +static void NativePointerHolder__set(JNIEnv * const env, S3JniNphOp const* pRef, + jobject jNph, const void * p){ + assert( jNph ); + (*env)->SetLongField(env, jNph, s3jni_nphop_field(env, pRef), + S3JniCast_P2L(p)); + S3JniExceptionIsFatal("Could not set NativePointerHolder.nativePointer."); +} + +#define NativePointerHolder_set(PREF,JNPH,P) \ + NativePointerHolder__set(env, PREF, JNPH, P) + +/* +** Fetches a native ptr value from NativePointerHolder object jNph, +** which must be of the native type described by pRef. This is a +** no-op if jNph is NULL. +*/ +static void * NativePointerHolder__get(JNIEnv * env, jobject jNph, + S3JniNphOp const* pRef){ + void * rv = 0; + if( jNph ){ + rv = S3JniCast_L2P( + (*env)->GetLongField(env, jNph, s3jni_nphop_field(env, pRef)) + ); + S3JniExceptionIsFatal("Cannot fetch NativePointerHolder.nativePointer."); + } + return rv; +} + +#define NativePointerHolder_get(JOBJ,NPHREF) \ + NativePointerHolder__get(env, (JOBJ), (NPHREF)) + +/* +** Helpers for extracting pointers from jobjects, noting that we rely +** on the corresponding Java interfaces having already done the +** type-checking. OBJ must be a jobject referring to a +** NativePointerHolder, where T matches PtrGet_T. Don't use these +** in contexts where that's not the case. Note that these aren't +** type-safe in the strictest sense: +** +** sqlite3 * s = PtrGet_sqlite3_stmt(...) +** +** will work, despite the incorrect macro name, so long as the +** argument is a Java sqlite3 object, as this operation only has void +** pointers to work with. +*/ +#define PtrGet_T(T,JOBJ) (T*)NativePointerHolder_get((JOBJ), S3JniNph(T)) +#define PtrGet_sqlite3(JOBJ) PtrGet_T(sqlite3, (JOBJ)) +#define PtrGet_sqlite3_backup(JOBJ) PtrGet_T(sqlite3_backup, (JOBJ)) +#define PtrGet_sqlite3_blob(JOBJ) PtrGet_T(sqlite3_blob, (JOBJ)) +#define PtrGet_sqlite3_context(JOBJ) PtrGet_T(sqlite3_context, (JOBJ)) +#define PtrGet_sqlite3_stmt(JOBJ) PtrGet_T(sqlite3_stmt, (JOBJ)) +#define PtrGet_sqlite3_value(JOBJ) PtrGet_T(sqlite3_value, (JOBJ)) +/* +** LongPtrGet_T(X,Y) expects X to be an unqualified sqlite3 struct +** type name and Y to be a native pointer to such an object in the +** form of a jlong value. The jlong is simply cast to (X*). This +** approach is, as of 2023-09-27, supplanting the former approach. We +** now do the native pointer extraction in the Java side, rather than +** the C side, because it's reportedly significantly faster. The +** intptr_t part here is necessary for compatibility with (at least) +** ARM32. +** +** 2023-11-09: testing has not revealed any measurable performance +** difference between the approach of passing type T to C compared to +** passing pointer-to-T to C, and adding support for the latter +** everywhere requires significantly more code. As of this writing, the +** older/simpler approach is being applied except for (A) where the +** newer approach has already been applied and (B) hot-spot APIs where +** a difference of microseconds (i.e. below our testing measurement +** threshold) might add up. +*/ +#define LongPtrGet_T(T,JLongAsPtr) (T*)((intptr_t)((JLongAsPtr))) +#define LongPtrGet_sqlite3(JLongAsPtr) LongPtrGet_T(sqlite3,(JLongAsPtr)) +#define LongPtrGet_sqlite3_backup(JLongAsPtr) LongPtrGet_T(sqlite3_backup,(JLongAsPtr)) +#define LongPtrGet_sqlite3_blob(JLongAsPtr) LongPtrGet_T(sqlite3_blob,(JLongAsPtr)) +#define LongPtrGet_sqlite3_stmt(JLongAsPtr) LongPtrGet_T(sqlite3_stmt,(JLongAsPtr)) +#define LongPtrGet_sqlite3_value(JLongAsPtr) LongPtrGet_T(sqlite3_value,(JLongAsPtr)) +/* +** Extracts the new S3JniDb instance from the free-list, or allocates +** one if needed, associates it with pDb, and returns. Returns NULL +** on OOM. The returned object MUST, on success of the calling +** operation, subsequently be associated with jDb via +** NativePointerHolder_set() or freed using S3JniDb_set_aside(). +*/ +static S3JniDb * S3JniDb_alloc(JNIEnv * const env, jobject jDb){ + S3JniDb * rv = 0; + S3JniDb_mutex_enter; + if( SJG.perDb.aFree ){ + rv = SJG.perDb.aFree; + SJG.perDb.aFree = rv->pNext; + rv->pNext = 0; + s3jni_incr( &SJG.metrics.nPdbRecycled ); + } + S3JniDb_mutex_leave; + if( 0==rv ){ + rv = s3jni_malloc(sizeof(S3JniDb)); + if( rv ){ + s3jni_incr( &SJG.metrics.nPdbAlloc ); + } + } + if( rv ){ + memset(rv, 0, sizeof(S3JniDb)); + rv->jDb = S3JniRefGlobal(jDb); + } + return rv; +} + +/* +** Returns the S3JniDb object for the given org.sqlite.jni.capi.sqlite3 +** object, or NULL if jDb is NULL, no pointer can be extracted +** from it, or no matching entry can be found. +*/ +static S3JniDb * S3JniDb__from_java(JNIEnv * const env, jobject jDb){ + sqlite3 * const pDb = jDb ? PtrGet_sqlite3(jDb) : 0; + return pDb ? S3JniDb_from_clientdata(pDb) : 0; +} +#define S3JniDb_from_java(jObject) S3JniDb__from_java(env,(jObject)) + +/* +** S3JniDb finalizer for use with sqlite3_set_clientdata(). +*/ +static void S3JniDb_xDestroy(void *p){ + S3JniDeclLocal_env; + S3JniDb * const ps = p; + assert( !ps->pNext && "Else ps is already in the free-list."); + S3JniDb_set_aside(ps); +} + +/* +** Evaluates to the S3JniDb object for the given sqlite3 object, or +** NULL if pDb is NULL or was not initialized via the JNI interfaces. +*/ +#define S3JniDb_from_c(sqlite3Ptr) \ + ((sqlite3Ptr) ? S3JniDb_from_clientdata(sqlite3Ptr) : 0) +#define S3JniDb_from_jlong(sqlite3PtrAsLong) \ + S3JniDb_from_c(LongPtrGet_T(sqlite3,sqlite3PtrAsLong)) + +/* +** Unref any Java-side state in (S3JniAutoExtension*) AX and zero out +** AX. +*/ +#define S3JniAutoExtension_clear(AX) S3JniHook_unref(AX); + +/* +** Initializes a pre-allocated S3JniAutoExtension object. Returns +** non-0 if there is an error collecting the required state from +** jAutoExt (which must be an AutoExtensionCallback object). On error, +** it passes ax to S3JniAutoExtension_clear(). +*/ +static int S3JniAutoExtension_init(JNIEnv *const env, + S3JniAutoExtension * const ax, + jobject const jAutoExt){ + jclass const klazz = (*env)->GetObjectClass(env, jAutoExt); + + S3JniAutoExt_mutex_assertLocker; + *ax = S3JniHook_empty; + ax->midCallback = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/capi/sqlite3;)I"); + S3JniUnrefLocal(klazz); + S3JniExceptionWarnIgnore; + if( !ax->midCallback ){ + S3JniAutoExtension_clear(ax); + return SQLITE_ERROR; + } + ax->jObj = S3JniRefGlobal(jAutoExt); + return 0; +} + +/* +** Sets the value property of the OutputPointer.Bool jOut object to +** v. +*/ +static void OutputPointer_set_Bool(JNIEnv * const env, jobject const jOut, + int v){ + (*env)->SetBooleanField(env, jOut, s3jni_nphop_field( + env, S3JniNph(OutputPointer_Bool) + ), v ? JNI_TRUE : JNI_FALSE ); + S3JniExceptionIsFatal("Cannot set OutputPointer.Bool.value"); +} + +/* +** Sets the value property of the OutputPointer.Int32 jOut object to +** v. +*/ +static void OutputPointer_set_Int32(JNIEnv * const env, jobject const jOut, + int v){ + (*env)->SetIntField(env, jOut, s3jni_nphop_field( + env, S3JniNph(OutputPointer_Int32) + ), (jint)v); + S3JniExceptionIsFatal("Cannot set OutputPointer.Int32.value"); +} + +/* +** Sets the value property of the OutputPointer.Int64 jOut object to +** v. +*/ +static void OutputPointer_set_Int64(JNIEnv * const env, jobject const jOut, + jlong v){ + (*env)->SetLongField(env, jOut, s3jni_nphop_field( + env, S3JniNph(OutputPointer_Int64) + ), v); + S3JniExceptionIsFatal("Cannot set OutputPointer.Int64.value"); +} + +/* +** Internal helper for OutputPointer_set_TYPE() where TYPE is an +** Object type. +*/ +static void OutputPointer_set_obj(JNIEnv * const env, + S3JniNphOp const * const pRef, + jobject const jOut, + jobject v){ + (*env)->SetObjectField(env, jOut, s3jni_nphop_field(env, pRef), v); + S3JniExceptionIsFatal("Cannot set OutputPointer.T.value"); +} + +#ifdef SQLITE_ENABLE_FTS5 +#if 0 +/* +** Sets the value property of the OutputPointer.ByteArray jOut object +** to v. +*/ +static void OutputPointer_set_ByteArray(JNIEnv * const env, jobject const jOut, + jbyteArray const v){ + OutputPointer_set_obj(env, S3JniNph(OutputPointer_ByteArray), jOut, v); +} +#endif +#endif /* SQLITE_ENABLE_FTS5 */ + +/* +** Sets the value property of the OutputPointer.String jOut object to +** v. +*/ +static void OutputPointer_set_String(JNIEnv * const env, jobject const jOut, + jstring const v){ + OutputPointer_set_obj(env, S3JniNph(OutputPointer_String), jOut, v); +} + +/* +** Returns true if eTextRep is a valid sqlite3 encoding constant, else +** returns false. +*/ +static int encodingTypeIsValid(int eTextRep){ + switch( eTextRep ){ + case SQLITE_UTF8: case SQLITE_UTF16: + case SQLITE_UTF16LE: case SQLITE_UTF16BE: + return 1; + default: + return 0; + } +} + +/* For use with sqlite3_result_pointer(), sqlite3_value_pointer(), + sqlite3_bind_java_object(), and sqlite3_column_java_object(). */ +static const char * const s3jni__value_jref_key = "org.sqlite.jni.capi.ResultJavaVal"; + +/* +** If v is not NULL, it must be a jobject global reference. Its +** reference is relinquished. +*/ +static void S3Jni_jobject_finalizer(void *v){ + if( v ){ + S3JniDeclLocal_env; + S3JniUnrefGlobal((jobject)v); + } +} + +/* +** Returns a new Java instance of the class referred to by pRef, which +** MUST be interface-compatible with NativePointerHolder and MUST have +** a no-arg constructor. The NativePointerHolder_set() method is +** passed the new Java object (which must not be NULL) and pNative +** (which may be NULL). Hypothetically returns NULL if Java fails to +** allocate, but the JNI docs are not entirely clear on that detail. +** +** Always use a static pointer from the S3JniNphOps struct for the +** 2nd argument. +*/ +static jobject NativePointerHolder_new(JNIEnv * const env, + S3JniNphOp const * pRef, + const void * pNative){ + jobject rv = 0; + S3JniNphOp * const pNC = s3jni_nphop(pRef); + if( !pNC->midCtor ){ + S3JniNph_mutex_enter; + if( !pNC->midCtor ){ + pNC->midCtor = (*env)->GetMethodID(env, pNC->klazz, "", "()V"); + S3JniExceptionIsFatal("Cannot find constructor for class."); + } + S3JniNph_mutex_leave; + } + rv = (*env)->NewObject(env, pNC->klazz, pNC->midCtor); + S3JniExceptionIsFatal("No-arg constructor threw."); + s3jni_oom_check(rv); + if( rv ) NativePointerHolder_set(pRef, rv, pNative); + return rv; +} + +static inline jobject new_java_sqlite3(JNIEnv * const env, sqlite3 *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3), sv); +} +static inline jobject new_java_sqlite3_backup(JNIEnv * const env, sqlite3_backup *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_backup), sv); +} +static inline jobject new_java_sqlite3_blob(JNIEnv * const env, sqlite3_blob *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_blob), sv); +} +static inline jobject new_java_sqlite3_context(JNIEnv * const env, sqlite3_context *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_context), sv); +} +static inline jobject new_java_sqlite3_stmt(JNIEnv * const env, sqlite3_stmt *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_stmt), sv); +} +static inline jobject new_java_sqlite3_value(JNIEnv * const env, sqlite3_value *sv){ + return NativePointerHolder_new(env, S3JniNph(sqlite3_value), sv); +} + +/* Helper typedefs for UDF callback types. */ +typedef void (*udf_xFunc_f)(sqlite3_context*,int,sqlite3_value**); +typedef void (*udf_xStep_f)(sqlite3_context*,int,sqlite3_value**); +typedef void (*udf_xFinal_f)(sqlite3_context*); +/*typedef void (*udf_xValue_f)(sqlite3_context*);*/ +/*typedef void (*udf_xInverse_f)(sqlite3_context*,int,sqlite3_value**);*/ + +/* +** Allocate a new S3JniUdf (User-defined Function) and associate it +** with the SQLFunction-type jObj. Returns NULL on OOM. If the +** returned object's type==UDF_UNKNOWN_TYPE then the type of UDF was +** not unambiguously detected based on which callback members it has, +** which falls into the category of user error. +** +** The caller must arrange for the returned object to eventually be +** passed to S3JniUdf_free(). +*/ +static S3JniUdf * S3JniUdf_alloc(JNIEnv * const env, jobject jObj){ + S3JniUdf * s = 0; + + S3JniGlobal_mutex_enter; + s3jni_incr(&SJG.metrics.nMutexUdf); + if( SJG.udf.aFree ){ + s = SJG.udf.aFree; + SJG.udf.aFree = s->pNext; + s->pNext = 0; + s3jni_incr(&SJG.metrics.nUdfRecycled); + } + S3JniGlobal_mutex_leave; + if( !s ){ + s = s3jni_malloc( sizeof(*s)); + s3jni_incr(&SJG.metrics.nUdfAlloc); + } + if( s ){ + const char * zFSI = /* signature for xFunc, xStep, xInverse */ + "(Lorg/sqlite/jni/capi/sqlite3_context;[Lorg/sqlite/jni/capi/sqlite3_value;)V"; + const char * zFV = /* signature for xFinal, xValue */ + "(Lorg/sqlite/jni/capi/sqlite3_context;)V"; + jclass const klazz = (*env)->GetObjectClass(env, jObj); + + memset(s, 0, sizeof(*s)); + s->jObj = S3JniRefGlobal(jObj); + +#define FGET(FuncName,FuncSig,Field) \ + s->Field = (*env)->GetMethodID(env, klazz, FuncName, FuncSig); \ + if( !s->Field ) (*env)->ExceptionClear(env) + + FGET("xFunc", zFSI, jmidxFunc); + FGET("xStep", zFSI, jmidxStep); + FGET("xFinal", zFV, jmidxFinal); + FGET("xValue", zFV, jmidxValue); + FGET("xInverse", zFSI, jmidxInverse); +#undef FGET + + S3JniUnrefLocal(klazz); + if( s->jmidxFunc ) s->type = UDF_SCALAR; + else if( s->jmidxStep && s->jmidxFinal ){ + s->type = (s->jmidxValue && s->jmidxInverse) + ? UDF_WINDOW : UDF_AGGREGATE; + }else{ + s->type = UDF_UNKNOWN_TYPE; + } + } + return s; +} + +/* +** Frees up all resources owned by s, clears its state, then either +** caches it for reuse (if cacheIt is true) or frees it. The former +** requires locking the global mutex, so it must not be held when this +** is called. +*/ +static void S3JniUdf_free(JNIEnv * const env, S3JniUdf * const s, + int cacheIt){ + assert( !s->pNext ); + if( s->jObj ){ + s3jni_call_xDestroy(s->jObj); + S3JniUnrefGlobal(s->jObj); + sqlite3_free(s->zFuncName); + assert( !s->pNext ); + memset(s, 0, sizeof(*s)); + } + if( cacheIt ){ + S3JniGlobal_mutex_enter; + s->pNext = S3JniGlobal.udf.aFree; + S3JniGlobal.udf.aFree = s; + S3JniGlobal_mutex_leave; + }else{ + sqlite3_free( s ); + } +} + +/* Finalizer for sqlite3_create_function() and friends. */ +static void S3JniUdf_finalizer(void * s){ + S3JniUdf_free(s3jni_env(), (S3JniUdf*)s, 1); +} + +/* +** Helper for processing args to UDF handlers with signature +** (sqlite3_context*,int,sqlite3_value**). +*/ +typedef struct { + jobject jcx /* sqlite3_context */; + jobjectArray jargv /* sqlite3_value[] */; +} udf_jargs; + +/* +** Converts the given (cx, argc, argv) into arguments for the given +** UDF, writing the result (Java wrappers for cx and argv) in the +** final 2 arguments. Returns 0 on success, SQLITE_NOMEM on allocation +** error. On error *jCx and *jArgv will be set to 0. The output +** objects are of type org.sqlite.jni.capi.sqlite3_context and +** array-of-org.sqlite.jni.capi.sqlite3_value, respectively. +*/ +static int udf_args(JNIEnv *env, + sqlite3_context * const cx, + int argc, sqlite3_value**argv, + jobject * jCx, jobjectArray *jArgv){ + jobjectArray ja = 0; + jobject jcx = new_java_sqlite3_context(env, cx); + jint i; + *jCx = 0; + *jArgv = 0; + if( !jcx ) goto error_oom; + ja = (*env)->NewObjectArray( + env, argc, s3jni_nphop(S3JniNph(sqlite3_value))->klazz, + NULL); + s3jni_oom_check( ja ); + if( !ja ) goto error_oom; + for(i = 0; i < argc; ++i){ + jobject jsv = new_java_sqlite3_value(env, argv[i]); + if( !jsv ) goto error_oom; + (*env)->SetObjectArrayElement(env, ja, i, jsv); + S3JniUnrefLocal(jsv)/*ja has a ref*/; + } + *jCx = jcx; + *jArgv = ja; + return 0; +error_oom: + S3JniUnrefLocal(jcx); + S3JniUnrefLocal(ja); + return SQLITE_NOMEM; +} + +/* +** Requires that jCx and jArgv are sqlite3_context +** resp. array-of-sqlite3_value values initialized by udf_args(). The +** latter will be 0-and-NULL for UDF types with no arguments. This +** function zeroes out the nativePointer member of jCx and each entry +** in jArgv. This is a safety-net precaution to avoid undefined +** behavior if a Java-side UDF holds a reference to its context or one +** of its arguments. This MUST be called from any function which +** successfully calls udf_args(), after calling the corresponding UDF +** and checking its exception status, or which Java-wraps a +** sqlite3_context for use with a UDF(ish) call. It MUST NOT be called +** in any other case. +*/ +static void udf_unargs(JNIEnv *env, jobject jCx, int argc, jobjectArray jArgv){ + int i = 0; + assert(jCx); + NativePointerHolder_set(S3JniNph(sqlite3_context), jCx, 0); + for( ; i < argc; ++i ){ + jobject jsv = (*env)->GetObjectArrayElement(env, jArgv, i); + /* + ** There is a potential Java-triggerable case of Undefined + ** Behavior here, but it would require intentional misuse of the + ** API: + ** + ** If a Java UDF grabs an sqlite3_value from its argv and then + ** assigns that element to null, it becomes unreachable to us so + ** we cannot clear out its pointer. That Java-side object's + ** getNativePointer() will then refer to a stale value, so passing + ** it into (e.g.) sqlite3_value_SOMETHING() would invoke UB. + ** + ** High-level wrappers can avoid that possibility if they do not + ** expose sqlite3_value directly to clients (as is the case in + ** org.sqlite.jni.wrapper1.SqlFunction). + ** + ** One potential (but expensive) workaround for this would be to + ** privately store a duplicate argv array in each sqlite3_context + ** wrapper object, and clear the native pointers from that copy. + */ + assert(jsv && "Someone illegally modified a UDF argument array."); + if( jsv ){ + NativePointerHolder_set(S3JniNph(sqlite3_value), jsv, 0); + } + } +} + + +/* +** Must be called immediately after a Java-side UDF callback throws. +** If translateToErr is true then it sets the exception's message in +** the result error using sqlite3_result_error(). If translateToErr is +** false then it emits a warning that the function threw but should +** not do so. In either case, it clears the exception state. +** +** Returns SQLITE_NOMEM if an allocation fails, else SQLITE_ERROR. In +** the former case it calls sqlite3_result_error_nomem(). +*/ +static int udf_report_exception(JNIEnv * const env, int translateToErr, + sqlite3_context * cx, + const char *zFuncName, const char *zFuncType ){ + jthrowable const ex = (*env)->ExceptionOccurred(env); + int rc = SQLITE_ERROR; + + assert(ex && "This must only be called when a Java exception is pending."); + if( translateToErr ){ + char * zMsg; + char * z; + + S3JniExceptionClear; + zMsg = s3jni_exception_error_msg(env, ex); + z = sqlite3_mprintf("Client-defined SQL function %s.%s() threw: %s", + zFuncName ? zFuncName : "", zFuncType, + zMsg ? zMsg : "Unknown exception" ); + sqlite3_free(zMsg); + if( z ){ + sqlite3_result_error(cx, z, -1); + sqlite3_free(z); + }else{ + sqlite3_result_error_nomem(cx); + rc = SQLITE_NOMEM; + } + }else{ + S3JniExceptionWarnCallbackThrew("client-defined SQL function"); + S3JniExceptionClear; + } + S3JniUnrefLocal(ex); + return rc; +} + +/* +** Sets up the state for calling a Java-side xFunc/xStep/xInverse() +** UDF, calls it, and returns 0 on success. +*/ +static int udf_xFSI(sqlite3_context* const pCx, int argc, + sqlite3_value** const argv, S3JniUdf * const s, + jmethodID xMethodID, const char * const zFuncType){ + S3JniDeclLocal_env; + udf_jargs args = {0,0}; + int rc = udf_args(env, pCx, argc, argv, &args.jcx, &args.jargv); + + if( 0 == rc ){ + (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); + S3JniIfThrew{ + rc = udf_report_exception(env, 'F'==zFuncType[1]/*xFunc*/, pCx, + s->zFuncName, zFuncType); + } + udf_unargs(env, args.jcx, argc, args.jargv); + } + S3JniUnrefLocal(args.jcx); + S3JniUnrefLocal(args.jargv); + return rc; +} + +/* +** Sets up the state for calling a Java-side xFinal/xValue() UDF, +** calls it, and returns 0 on success. +*/ +static int udf_xFV(sqlite3_context* cx, S3JniUdf * s, + jmethodID xMethodID, + const char *zFuncType){ + S3JniDeclLocal_env; + jobject jcx = new_java_sqlite3_context(env, cx); + int rc = 0; + int const isFinal = 'F'==zFuncType[1]/*xFinal*/; + + if( jcx ){ + (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx); + S3JniIfThrew{ + rc = udf_report_exception(env, isFinal, cx, s->zFuncName, + zFuncType); + } + udf_unargs(env, jcx, 0, 0); + S3JniUnrefLocal(jcx); + }else{ + if( isFinal ) sqlite3_result_error_nomem(cx); + rc = SQLITE_NOMEM; + } + return rc; +} + +/* Proxy for C-to-Java xFunc. */ +static void udf_xFunc(sqlite3_context* cx, int argc, + sqlite3_value** argv){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nFunc ); + udf_xFSI(cx, argc, argv, s, s->jmidxFunc, "xFunc"); +} +/* Proxy for C-to-Java xStep. */ +static void udf_xStep(sqlite3_context* cx, int argc, + sqlite3_value** argv){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nStep ); + udf_xFSI(cx, argc, argv, s, s->jmidxStep, "xStep"); +} +/* Proxy for C-to-Java xFinal. */ +static void udf_xFinal(sqlite3_context* cx){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nFinal ); + udf_xFV(cx, s, s->jmidxFinal, "xFinal"); +} +/* Proxy for C-to-Java xValue. */ +static void udf_xValue(sqlite3_context* cx){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nValue ); + udf_xFV(cx, s, s->jmidxValue, "xValue"); +} +/* Proxy for C-to-Java xInverse. */ +static void udf_xInverse(sqlite3_context* cx, int argc, + sqlite3_value** argv){ + S3JniUdf * const s = (S3JniUdf*)sqlite3_user_data(cx); + s3jni_incr( &SJG.metrics.udf.nInverse ); + udf_xFSI(cx, argc, argv, s, s->jmidxInverse, "xInverse"); +} + + +//////////////////////////////////////////////////////////////////////// +// What follows is the JNI/C bindings. They are in alphabetical order +// except for this macro-generated subset which are kept together +// (alphabetized) here at the front... +//////////////////////////////////////////////////////////////////////// + +/** Create a trivial JNI wrapper for (int CName(void)). */ +#define WRAP_INT_VOID(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass){ \ + return (jint)CName(); \ + } +/** Create a trivial JNI wrapper for (int CName(int)). */ +#define WRAP_INT_INT(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jint arg){ \ + return (jint)CName((int)arg); \ + } +/* +** Create a trivial JNI wrapper for (const mutf8_string * +** CName(void)). This is only valid for functions which are known to +** return ASCII or text which is equivalent in UTF-8 and MUTF-8. +*/ +#define WRAP_MUTF8_VOID(JniNameSuffix,CName) \ + JniDecl(jstring,JniNameSuffix)(JniArgsEnvClass){ \ + jstring const rv = (*env)->NewStringUTF( env, CName() ); \ + s3jni_oom_check(rv); \ + return rv; \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3_stmt*)). */ +#define WRAP_INT_STMT(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpStmt){ \ + return (jint)CName(LongPtrGet_sqlite3_stmt(jpStmt)); \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3_stmt*,int)). */ +#define WRAP_INT_STMT_INT(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpStmt, jint n){ \ + return (jint)CName(LongPtrGet_sqlite3_stmt(jpStmt), (int)n); \ + } +/** Create a trivial JNI wrapper for (boolean CName(sqlite3_stmt*)). */ +#define WRAP_BOOL_STMT(JniNameSuffix,CName) \ + JniDecl(jboolean,JniNameSuffix)(JniArgsEnvClass, jobject jStmt){ \ + return CName(PtrGet_sqlite3_stmt(jStmt)) ? JNI_TRUE : JNI_FALSE; \ + } +/** Create a trivial JNI wrapper for (jstring CName(sqlite3_stmt*,int)). */ +#define WRAP_STR_STMT_INT(JniNameSuffix,CName) \ + JniDecl(jstring,JniNameSuffix)(JniArgsEnvClass, jlong jpStmt, jint ndx){ \ + return s3jni_utf8_to_jstring( \ + CName(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx), \ + -1); \ + } +/** Create a trivial JNI wrapper for (boolean CName(sqlite3*)). */ +#define WRAP_BOOL_DB(JniNameSuffix,CName) \ + JniDecl(jboolean,JniNameSuffix)(JniArgsEnvClass, jlong jpDb){ \ + return CName(LongPtrGet_sqlite3(jpDb)) ? JNI_TRUE : JNI_FALSE; \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3*)). */ +#define WRAP_INT_DB(JniNameSuffix,CName) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpDb){ \ + return (jint)CName(LongPtrGet_sqlite3(jpDb)); \ + } +/** Create a trivial JNI wrapper for (int64 CName(sqlite3*)). */ +#define WRAP_INT64_DB(JniNameSuffix,CName) \ + JniDecl(jlong,JniNameSuffix)(JniArgsEnvClass, jlong jpDb){ \ + return (jlong)CName(LongPtrGet_sqlite3(jpDb)); \ + } +/** Create a trivial JNI wrapper for (jstring CName(sqlite3*,int)). */ +#define WRAP_STR_DB_INT(JniNameSuffix,CName) \ + JniDecl(jstring,JniNameSuffix)(JniArgsEnvClass, jlong jpDb, jint ndx){ \ + return s3jni_utf8_to_jstring( \ + CName(LongPtrGet_sqlite3(jpDb), (int)ndx), \ + -1); \ + } +/** Create a trivial JNI wrapper for (int CName(sqlite3_value*)). */ +#define WRAP_INT_SVALUE(JniNameSuffix,CName,DfltOnNull) \ + JniDecl(jint,JniNameSuffix)(JniArgsEnvClass, jlong jpSValue){ \ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSValue); \ + return (jint)(sv ? CName(sv): DfltOnNull); \ + } +/** Create a trivial JNI wrapper for (boolean CName(sqlite3_value*)). */ +#define WRAP_BOOL_SVALUE(JniNameSuffix,CName,DfltOnNull) \ + JniDecl(jboolean,JniNameSuffix)(JniArgsEnvClass, jlong jpSValue){ \ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSValue); \ + return (jint)(sv ? CName(sv) : DfltOnNull) \ + ? JNI_TRUE : JNI_FALSE; \ + } + +WRAP_INT_DB(1changes, sqlite3_changes) +WRAP_INT64_DB(1changes64, sqlite3_changes64) +WRAP_INT_STMT(1clear_1bindings, sqlite3_clear_bindings) +WRAP_INT_STMT_INT(1column_1bytes, sqlite3_column_bytes) +WRAP_INT_STMT_INT(1column_1bytes16, sqlite3_column_bytes16) +WRAP_INT_STMT(1column_1count, sqlite3_column_count) +WRAP_STR_STMT_INT(1column_1decltype, sqlite3_column_decltype) +WRAP_STR_STMT_INT(1column_1name, sqlite3_column_name) +#ifdef SQLITE_ENABLE_COLUMN_METADATA +WRAP_STR_STMT_INT(1column_1database_1name, sqlite3_column_database_name) +WRAP_STR_STMT_INT(1column_1origin_1name, sqlite3_column_origin_name) +WRAP_STR_STMT_INT(1column_1table_1name, sqlite3_column_table_name) +#endif +WRAP_INT_STMT_INT(1column_1type, sqlite3_column_type) +WRAP_INT_STMT(1data_1count, sqlite3_data_count) +WRAP_STR_DB_INT(1db_1name, sqlite3_db_name) +WRAP_INT_DB(1error_1offset, sqlite3_error_offset) +WRAP_INT_DB(1extended_1errcode, sqlite3_extended_errcode) +WRAP_BOOL_DB(1get_1autocommit, sqlite3_get_autocommit) +WRAP_MUTF8_VOID(1libversion, sqlite3_libversion) +WRAP_INT_VOID(1libversion_1number, sqlite3_libversion_number) +WRAP_INT_VOID(1keyword_1count, sqlite3_keyword_count) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +WRAP_INT_DB(1preupdate_1blobwrite, sqlite3_preupdate_blobwrite) +WRAP_INT_DB(1preupdate_1count, sqlite3_preupdate_count) +WRAP_INT_DB(1preupdate_1depth, sqlite3_preupdate_depth) +#endif +WRAP_INT_INT(1release_1memory, sqlite3_release_memory) +WRAP_INT_INT(1sleep, sqlite3_sleep) +WRAP_MUTF8_VOID(1sourceid, sqlite3_sourceid) +WRAP_BOOL_STMT(1stmt_1busy, sqlite3_stmt_busy) +WRAP_INT_STMT_INT(1stmt_1explain, sqlite3_stmt_explain) +WRAP_INT_STMT(1stmt_1isexplain, sqlite3_stmt_isexplain) +WRAP_BOOL_STMT(1stmt_1readonly, sqlite3_stmt_readonly) +WRAP_INT_DB(1system_1errno, sqlite3_system_errno) +WRAP_INT_VOID(1threadsafe, sqlite3_threadsafe) +WRAP_INT_DB(1total_1changes, sqlite3_total_changes) +WRAP_INT64_DB(1total_1changes64, sqlite3_total_changes64) +WRAP_INT_SVALUE(1value_1encoding, sqlite3_value_encoding,SQLITE_UTF8) +WRAP_BOOL_SVALUE(1value_1frombind, sqlite3_value_frombind,0) +WRAP_INT_SVALUE(1value_1nochange, sqlite3_value_nochange,0) +WRAP_INT_SVALUE(1value_1numeric_1type, sqlite3_value_numeric_type,SQLITE_NULL) +WRAP_INT_SVALUE(1value_1subtype, sqlite3_value_subtype,0) +WRAP_INT_SVALUE(1value_1type, sqlite3_value_type,SQLITE_NULL) + +#undef WRAP_BOOL_DB +#undef WRAP_BOOL_STMT +#undef WRAP_BOOL_SVALUE +#undef WRAP_INT64_DB +#undef WRAP_INT_DB +#undef WRAP_INT_INT +#undef WRAP_INT_STMT +#undef WRAP_INT_STMT_INT +#undef WRAP_INT_SVALUE +#undef WRAP_INT_VOID +#undef WRAP_MUTF8_VOID +#undef WRAP_STR_STMT_INT +#undef WRAP_STR_DB_INT + +S3JniApi(sqlite3_aggregate_context(),jlong,1aggregate_1context)( + JniArgsEnvClass, jobject jCx, jboolean initialize +){ + sqlite3_context * const pCx = PtrGet_sqlite3_context(jCx); + void * const p = pCx + ? sqlite3_aggregate_context(pCx, (int)(initialize + ? (int)sizeof(void*) + : 0)) + : 0; + return S3JniCast_P2L(p); +} + +/* +** Central auto-extension runner for auto-extensions created in Java. +*/ +static int s3jni_run_java_auto_extensions(sqlite3 *pDb, const char **pzErr, + const struct sqlite3_api_routines *ignored){ + int rc = 0; + unsigned i, go = 1; + JNIEnv * env = 0; + S3JniDb * ps; + S3JniEnv * jc; + + if( 0==SJG.autoExt.nExt ) return 0; + env = s3jni_env(); + jc = S3JniEnv_get(); + S3JniDb_mutex_enter; + ps = jc->pdbOpening ? jc->pdbOpening : S3JniDb_from_c(pDb); + if( !ps ){ + *pzErr = sqlite3_mprintf("Unexpected arrival of null S3JniDb in " + "auto-extension runner."); + S3JniDb_mutex_leave; + return SQLITE_ERROR; + } + assert( ps->jDb ); + if( !ps->pDb ){ + assert( jc->pdbOpening == ps ); + rc = sqlite3_set_clientdata(pDb, S3JniDb_clientdata_key, + ps, 0/* we'll re-set this after open() + completes. */); + if( rc ){ + S3JniDb_mutex_leave; + return rc; + } + } + else{ + assert( ps == jc->pdbOpening ); + jc->pdbOpening = 0; + } + S3JniDb_mutex_leave; + NativePointerHolder_set(S3JniNph(sqlite3), ps->jDb, pDb) + /* As of here, the Java/C connection is complete except for the + (temporary) lack of finalizer for the ps object. */; + ps->pDb = pDb; + for( i = 0; go && 0==rc; ++i ){ + S3JniAutoExtension ax = S3JniHook_empty + /* We need a copy of the auto-extension object, with our own + ** local reference to it, to avoid a race condition with another + ** thread manipulating the list during the call and invaliding + ** what ax references. */; + S3JniAutoExt_mutex_enter; + if( i >= SJG.autoExt.nExt ){ + go = 0; + }else{ + S3JniHook_localdup(&SJG.autoExt.aExt[i], &ax); + } + S3JniAutoExt_mutex_leave; + if( ax.jObj ){ + rc = (*env)->CallIntMethod(env, ax.jObj, ax.midCallback, ps->jDb); + S3JniHook_localundup(ax); + S3JniIfThrew { + jthrowable const ex = (*env)->ExceptionOccurred(env); + char * zMsg; + S3JniExceptionClear; + zMsg = s3jni_exception_error_msg(env, ex); + S3JniUnrefLocal(ex); + *pzErr = sqlite3_mprintf("auto-extension threw: %s", zMsg); + sqlite3_free(zMsg); + rc = SQLITE_ERROR; + } + } + } + return rc; +} + +S3JniApi(sqlite3_auto_extension(),jint,1auto_1extension)( + JniArgsEnvClass, jobject jAutoExt +){ + int i; + S3JniAutoExtension * ax = 0; + int rc = 0; + + if( !jAutoExt ) return SQLITE_MISUSE; + S3JniAutoExt_mutex_enter; + for( i = 0; i < SJG.autoExt.nExt; ++i ){ + /* Look for a match. */ + ax = &SJG.autoExt.aExt[i]; + if( ax->jObj && (*env)->IsSameObject(env, ax->jObj, jAutoExt) ){ + /* same object, so this is a no-op. */ + S3JniAutoExt_mutex_leave; + return 0; + } + } + if( i == SJG.autoExt.nExt ){ + assert( SJG.autoExt.nExt <= SJG.autoExt.nAlloc ); + if( SJG.autoExt.nExt == SJG.autoExt.nAlloc ){ + /* Allocate another slot. */ + unsigned n = 1 + SJG.autoExt.nAlloc; + S3JniAutoExtension * const aNew = + s3jni_realloc( SJG.autoExt.aExt, n * sizeof(*ax) ); + if( !aNew ){ + rc = SQLITE_NOMEM; + }else{ + SJG.autoExt.aExt = aNew; + ++SJG.autoExt.nAlloc; + } + } + if( 0==rc ){ + ax = &SJG.autoExt.aExt[SJG.autoExt.nExt]; + rc = S3JniAutoExtension_init(env, ax, jAutoExt); + assert( rc ? (0==ax->jObj && 0==ax->midCallback) + : (0!=ax->jObj && 0!=ax->midCallback) ); + } + } + if( 0==rc ){ + static int once = 0; + if( 0==once && ++once ){ + rc = sqlite3_auto_extension( + (void(*)(void))s3jni_run_java_auto_extensions + /* Reminder: the JNI binding of sqlite3_reset_auto_extension() + ** does not call the core-lib impl. It only clears Java-side + ** auto-extensions. */ + ); + if( rc ){ + assert( ax ); + S3JniAutoExtension_clear(ax); + } + } + if( 0==rc ){ + ++SJG.autoExt.nExt; + } + } + S3JniAutoExt_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_backup_finish(),jint,1backup_1finish)( + JniArgsEnvClass, jlong jpBack +){ + int rc = 0; + if( jpBack!=0 ){ + rc = sqlite3_backup_finish( LongPtrGet_sqlite3_backup(jpBack) ); + } + return rc; +} + +S3JniApi(sqlite3_backup_init(),jobject,1backup_1init)( + JniArgsEnvClass, jlong jpDbDest, jstring jTDest, + jlong jpDbSrc, jstring jTSrc +){ + sqlite3 * const pDest = LongPtrGet_sqlite3(jpDbDest); + sqlite3 * const pSrc = LongPtrGet_sqlite3(jpDbSrc); + char * const zDest = s3jni_jstring_to_utf8(jTDest, 0); + char * const zSrc = s3jni_jstring_to_utf8(jTSrc, 0); + jobject rv = 0; + + if( pDest && pSrc && zDest && zSrc ){ + sqlite3_backup * const pB = + sqlite3_backup_init(pDest, zDest, pSrc, zSrc); + if( pB ){ + rv = new_java_sqlite3_backup(env, pB); + if( !rv ){ + sqlite3_backup_finish( pB ); + } + } + } + sqlite3_free(zDest); + sqlite3_free(zSrc); + return rv; +} + +S3JniApi(sqlite3_backup_pagecount(),jint,1backup_1pagecount)( + JniArgsEnvClass, jlong jpBack +){ + return sqlite3_backup_pagecount(LongPtrGet_sqlite3_backup(jpBack)); +} + +S3JniApi(sqlite3_backup_remaining(),jint,1backup_1remaining)( + JniArgsEnvClass, jlong jpBack +){ + return sqlite3_backup_remaining(LongPtrGet_sqlite3_backup(jpBack)); +} + +S3JniApi(sqlite3_backup_step(),jint,1backup_1step)( + JniArgsEnvClass, jlong jpBack, jint nPage +){ + return sqlite3_backup_step(LongPtrGet_sqlite3_backup(jpBack), (int)nPage); +} + +S3JniApi(sqlite3_bind_blob(),jint,1bind_1blob)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jbyteArray baData, jint nMax +){ + jsize nBA = 0; + jbyte * const pBuf = baData ? s3jni_jbyteArray_bytes2(baData, &nBA) : 0; + int rc; + if( pBuf ){ + if( nMax>nBA ){ + nMax = nBA; + } + rc = sqlite3_bind_blob(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, + pBuf, (int)nMax, SQLITE_TRANSIENT); + s3jni_jbyteArray_release(baData, pBuf); + }else{ + rc = baData + ? SQLITE_NOMEM + : sqlite3_bind_null( LongPtrGet_sqlite3_stmt(jpStmt), ndx ); + } + return (jint)rc; +} + +/** + Helper for use with s3jni_setup_nio_args(). +*/ +struct S3JniNioArgs { + jobject jBuf; /* input - ByteBuffer */ + jint iOffset; /* input - byte offset */ + jint iHowMany; /* input - byte count to bind/read/write */ + jint nBuf; /* output - jBuf's buffer size */ + void * p; /* output - jBuf's buffer memory */ + void * pStart; /* output - offset of p to bind/read/write */ + int nOut; /* output - number of bytes from pStart to bind/read/write */ +}; +typedef struct S3JniNioArgs S3JniNioArgs; +static const S3JniNioArgs S3JniNioArgs_empty = { + 0,0,0,0,0,0,0 +}; + +/* +** Internal helper for sqlite3_bind_nio_buffer(), +** sqlite3_result_nio_buffer(), and similar methods which take a +** ByteBuffer object as either input or output. Populates pArgs and +** returns 0 on success, non-0 if the operation should fail. The +** caller is required to check for SJG.g.byteBuffer.klazz!=0 before calling +** this and reporting it in a way appropriate for that routine. This +** function may assert() that SJG.g.byteBuffer.klazz is not 0. +** +** The (jBuffer, iOffset, iHowMany) arguments are the (ByteBuffer, offset, +** length) arguments to the bind/result method. +** +** If iHowMany is negative then it's treated as "until the end" and +** the calculated slice is trimmed to fit if needed. If iHowMany is +** positive and extends past the end of jBuffer then SQLITE_ERROR is +** returned. +** +** Returns 0 if everything looks to be in order, else some SQLITE_... +** result code +*/ +static int s3jni_setup_nio_args( + JNIEnv *env, S3JniNioArgs * pArgs, + jobject jBuffer, jint iOffset, jint iHowMany +){ + jlong iEnd = 0; + const int bAllowTruncate = iHowMany<0; + *pArgs = S3JniNioArgs_empty; + pArgs->jBuf = jBuffer; + pArgs->iOffset = iOffset; + pArgs->iHowMany = iHowMany; + assert( SJG.g.byteBuffer.klazz ); + if( pArgs->iOffset<0 ){ + return SQLITE_ERROR + /* SQLITE_MISUSE or SQLITE_RANGE would fit better but we use + SQLITE_ERROR for consistency with the code documented for a + negative target blob offset in sqlite3_blob_read/write(). */; + } + s3jni_get_nio_buffer(pArgs->jBuf, &pArgs->p, &pArgs->nBuf); + if( !pArgs->p ){ + return SQLITE_MISUSE; + }else if( pArgs->iOffset>=pArgs->nBuf ){ + pArgs->pStart = 0; + pArgs->nOut = 0; + return 0; + } + assert( pArgs->nBuf > 0 ); + assert( pArgs->iOffset < pArgs->nBuf ); + iEnd = pArgs->iHowMany<0 + ? pArgs->nBuf - pArgs->iOffset + : pArgs->iOffset + pArgs->iHowMany; + if( iEnd>(jlong)pArgs->nBuf ){ + if( bAllowTruncate ){ + iEnd = pArgs->nBuf - pArgs->iOffset; + }else{ + return SQLITE_ERROR + /* again: for consistency with blob_read/write(), though + SQLITE_MISUSE or SQLITE_RANGE would be a better fit. */; + } + } + if( iEnd - pArgs->iOffset > (jlong)SQLITE_MAX_LENGTH ){ + return SQLITE_TOOBIG; + } + assert( pArgs->iOffset >= 0 ); + assert( iEnd > pArgs->iOffset ); + pArgs->pStart = pArgs->p + pArgs->iOffset; + pArgs->nOut = (int)(iEnd - pArgs->iOffset); + assert( pArgs->nOut > 0 ); + assert( (pArgs->pStart + pArgs->nOut) <= (pArgs->p + pArgs->nBuf) ); + return 0; +} + +S3JniApi(sqlite3_bind_nio_buffer(),jint,1bind_1nio_1buffer)( + JniArgsEnvClass, jobject jpStmt, jint ndx, jobject jBuffer, + jint iOffset, jint iN +){ + sqlite3_stmt * pStmt = PtrGet_sqlite3_stmt(jpStmt); + S3JniNioArgs args; + int rc; + if( !pStmt || !SJG.g.byteBuffer.klazz ) return SQLITE_MISUSE; + rc = s3jni_setup_nio_args(env, &args, jBuffer, iOffset, iN); + if(rc){ + return rc; + }else if( !args.pStart || !args.nOut ){ + return sqlite3_bind_null(pStmt, ndx); + } + return sqlite3_bind_blob( pStmt, (int)ndx, args.pStart, + args.nOut, SQLITE_TRANSIENT ); +} + +S3JniApi(sqlite3_bind_double(),jint,1bind_1double)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jdouble val +){ + return (jint)sqlite3_bind_double(LongPtrGet_sqlite3_stmt(jpStmt), + (int)ndx, (double)val); +} + +S3JniApi(sqlite3_bind_int(),jint,1bind_1int)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jint val +){ + return (jint)sqlite3_bind_int(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, (int)val); +} + +S3JniApi(sqlite3_bind_int64(),jint,1bind_1int64)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jlong val +){ + return (jint)sqlite3_bind_int64(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, (sqlite3_int64)val); +} + +/* +** Bind a new global ref to Object `val` using sqlite3_bind_pointer(). +*/ +S3JniApi(sqlite3_bind_java_object(),jint,1bind_1java_1object)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jobject val +){ + sqlite3_stmt * const pStmt = LongPtrGet_sqlite3_stmt(jpStmt); + int rc = SQLITE_MISUSE; + + if(pStmt){ + jobject const rv = S3JniRefGlobal(val); + if( rv ){ + rc = sqlite3_bind_pointer(pStmt, ndx, rv, s3jni__value_jref_key, + S3Jni_jobject_finalizer); + }else if(val){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_bind_null(pStmt, ndx); + } + } + return rc; +} + +S3JniApi(sqlite3_bind_null(),jint,1bind_1null)( + JniArgsEnvClass, jlong jpStmt, jint ndx +){ + return (jint)sqlite3_bind_null(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_bind_parameter_count(),jint,1bind_1parameter_1count)( + JniArgsEnvClass, jlong jpStmt +){ + return (jint)sqlite3_bind_parameter_count(LongPtrGet_sqlite3_stmt(jpStmt)); +} + +S3JniApi(sqlite3_bind_parameter_index(),jint,1bind_1parameter_1index)( + JniArgsEnvClass, jlong jpStmt, jbyteArray jName +){ + int rc = 0; + jbyte * const pBuf = s3jni_jbyteArray_bytes(jName); + if( pBuf ){ + rc = sqlite3_bind_parameter_index(LongPtrGet_sqlite3_stmt(jpStmt), + (const char *)pBuf); + s3jni_jbyteArray_release(jName, pBuf); + } + return rc; +} + +S3JniApi(sqlite3_bind_parameter_name(),jstring,1bind_1parameter_1name)( + JniArgsEnvClass, jlong jpStmt, jint ndx +){ + const char *z = + sqlite3_bind_parameter_name(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx); + return z ? s3jni_utf8_to_jstring(z, -1) : 0; +} + +/* +** Impl of sqlite3_bind_text/text16(). +*/ +static int s3jni__bind_text(int is16, JNIEnv *env, jlong jpStmt, jint ndx, + jbyteArray baData, jint nMax){ + jsize nBA = 0; + jbyte * const pBuf = + baData ? s3jni_jbyteArray_bytes2(baData, &nBA) : 0; + int rc; + if( pBuf ){ + if( nMax>nBA ){ + nMax = nBA; + } + /* Note that we rely on the Java layer having assured that baData + is NUL-terminated if nMax is negative. In order to avoid UB for + such cases, we do not expose the byte-limit arguments in the + public API. */ + rc = is16 + ? sqlite3_bind_text16(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, + pBuf, (int)nMax, SQLITE_TRANSIENT) + : sqlite3_bind_text(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx, + (const char *)pBuf, + (int)nMax, SQLITE_TRANSIENT); + }else{ + rc = baData + ? sqlite3_bind_null(LongPtrGet_sqlite3_stmt(jpStmt), (int)ndx) + : SQLITE_NOMEM; + } + s3jni_jbyteArray_release(baData, pBuf); + return (jint)rc; + +} + +S3JniApi(sqlite3_bind_text(),jint,1bind_1text)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jbyteArray baData, jint nMax +){ + return s3jni__bind_text(0, env, jpStmt, ndx, baData, nMax); +} + +S3JniApi(sqlite3_bind_text16(),jint,1bind_1text16)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jbyteArray baData, jint nMax +){ + return s3jni__bind_text(1, env, jpStmt, ndx, baData, nMax); +} + +S3JniApi(sqlite3_bind_value(),jint,1bind_1value)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jlong jpValue +){ + int rc = 0; + sqlite3_stmt * pStmt = LongPtrGet_sqlite3_stmt(jpStmt); + if( pStmt ){ + sqlite3_value *v = LongPtrGet_sqlite3_value(jpValue); + if( v ){ + rc = sqlite3_bind_value(pStmt, (int)ndx, v); + }else{ + rc = sqlite3_bind_null(pStmt, (int)ndx); + } + }else{ + rc = SQLITE_MISUSE; + } + return (jint)rc; +} + +S3JniApi(sqlite3_bind_zeroblob(),jint,1bind_1zeroblob)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jint n +){ + return (jint)sqlite3_bind_zeroblob(LongPtrGet_sqlite3_stmt(jpStmt), + (int)ndx, (int)n); +} + +S3JniApi(sqlite3_bind_zeroblob64(),jint,1bind_1zeroblob64)( + JniArgsEnvClass, jlong jpStmt, jint ndx, jlong n +){ + return (jint)sqlite3_bind_zeroblob64(LongPtrGet_sqlite3_stmt(jpStmt), + (int)ndx, (sqlite3_uint64)n); +} + +S3JniApi(sqlite3_blob_bytes(),jint,1blob_1bytes)( + JniArgsEnvClass, jlong jpBlob +){ + return sqlite3_blob_bytes(LongPtrGet_sqlite3_blob(jpBlob)); +} + +S3JniApi(sqlite3_blob_close(),jint,1blob_1close)( + JniArgsEnvClass, jlong jpBlob +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + return b ? (jint)sqlite3_blob_close(b) : SQLITE_MISUSE; +} + +S3JniApi(sqlite3_blob_open(),jint,1blob_1open)( + JniArgsEnvClass, jlong jpDb, jstring jDbName, jstring jTbl, jstring jCol, + jlong jRowId, jint flags, jobject jOut +){ + sqlite3 * const db = LongPtrGet_sqlite3(jpDb); + sqlite3_blob * pBlob = 0; + char * zDbName = 0, * zTableName = 0, * zColumnName = 0; + int rc; + + if( !db || !jDbName || !jTbl || !jCol ) return SQLITE_MISUSE; + zDbName = s3jni_jstring_to_utf8(jDbName,0); + zTableName = zDbName ? s3jni_jstring_to_utf8(jTbl,0) : 0; + zColumnName = zTableName ? s3jni_jstring_to_utf8(jCol,0) : 0; + rc = zColumnName + ? sqlite3_blob_open(db, zDbName, zTableName, zColumnName, + (sqlite3_int64)jRowId, (int)flags, &pBlob) + : SQLITE_NOMEM; + if( 0==rc ){ + jobject rv = new_java_sqlite3_blob(env, pBlob); + if( !rv ){ + sqlite3_blob_close(pBlob); + rc = SQLITE_NOMEM; + } + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3_blob), jOut, rv); + } + sqlite3_free(zDbName); + sqlite3_free(zTableName); + sqlite3_free(zColumnName); + return rc; +} + +S3JniApi(sqlite3_blob_read(),jint,1blob_1read)( + JniArgsEnvClass, jlong jpBlob, jbyteArray jTgt, jint iOffset +){ + jbyte * const pBa = s3jni_jbyteArray_bytes(jTgt); + int rc = jTgt ? (pBa ? SQLITE_MISUSE : SQLITE_NOMEM) : SQLITE_MISUSE; + if( pBa ){ + jsize const nTgt = (*env)->GetArrayLength(env, jTgt); + rc = sqlite3_blob_read(LongPtrGet_sqlite3_blob(jpBlob), pBa, + (int)nTgt, (int)iOffset); + if( 0==rc ){ + s3jni_jbyteArray_commit(jTgt, pBa); + }else{ + s3jni_jbyteArray_release(jTgt, pBa); + } + } + return rc; +} + +S3JniApi(sqlite3_blob_read_nio_buffer(),jint,1blob_1read_1nio_1buffer)( + JniArgsEnvClass, jlong jpBlob, jint iSrcOff, jobject jBB, jint iTgtOff, jint iHowMany +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + S3JniNioArgs args; + int rc; + if( !b || !SJG.g.byteBuffer.klazz || iHowMany<0 ){ + return SQLITE_MISUSE; + }else if( iTgtOff<0 || iSrcOff<0 ){ + return SQLITE_ERROR + /* for consistency with underlying sqlite3_blob_read() */; + }else if( 0==iHowMany ){ + return 0; + } + rc = s3jni_setup_nio_args(env, &args, jBB, iTgtOff, iHowMany); + if(rc){ + return rc; + }else if( !args.pStart || !args.nOut ){ + return 0; + } + assert( args.iHowMany>0 ); + return sqlite3_blob_read( b, args.pStart, (int)args.nOut, (int)iSrcOff ); +} + +S3JniApi(sqlite3_blob_reopen(),jint,1blob_1reopen)( + JniArgsEnvClass, jlong jpBlob, jlong iNewRowId +){ + return (jint)sqlite3_blob_reopen(LongPtrGet_sqlite3_blob(jpBlob), + (sqlite3_int64)iNewRowId); +} + +S3JniApi(sqlite3_blob_write(),jint,1blob_1write)( + JniArgsEnvClass, jlong jpBlob, jbyteArray jBa, jint iOffset +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + jbyte * const pBuf = b ? s3jni_jbyteArray_bytes(jBa) : 0; + const jsize nBA = pBuf ? (*env)->GetArrayLength(env, jBa) : 0; + int rc = SQLITE_MISUSE; + if(b && pBuf){ + rc = sqlite3_blob_write( b, pBuf, (int)nBA, (int)iOffset ); + } + s3jni_jbyteArray_release(jBa, pBuf); + return (jint)rc; +} + +S3JniApi(sqlite3_blob_write_nio_buffer(),jint,1blob_1write_1nio_1buffer)( + JniArgsEnvClass, jlong jpBlob, jint iTgtOff, jobject jBB, jint iSrcOff, jint iHowMany +){ + sqlite3_blob * const b = LongPtrGet_sqlite3_blob(jpBlob); + S3JniNioArgs args; + int rc; + if( !b || !SJG.g.byteBuffer.klazz ){ + return SQLITE_MISUSE; + }else if( iTgtOff<0 || iSrcOff<0 ){ + return SQLITE_ERROR + /* for consistency with underlying sqlite3_blob_write() */; + }else if( 0==iHowMany ){ + return 0; + } + rc = s3jni_setup_nio_args(env, &args, jBB, iSrcOff, iHowMany); + if(rc){ + return rc; + }else if( !args.pStart || !args.nOut ){ + return 0; + } + return sqlite3_blob_write( b, args.pStart, (int)args.nOut, (int)iTgtOff ); +} + +/* Central C-to-Java busy handler proxy. */ +static int s3jni_busy_handler(void* pState, int n){ + S3JniDb * const ps = (S3JniDb *)pState; + int rc = 0; + S3JniDeclLocal_env; + S3JniHook hook; + + S3JniHook_localdup(&ps->hooks.busyHandler, &hook); + if( hook.jObj ){ + rc = (*env)->CallIntMethod(env, hook.jObj, + hook.midCallback, (jint)n); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("sqlite3_busy_handler() callback"); + rc = s3jni_db_exception(ps->pDb, SQLITE_ERROR, + "sqlite3_busy_handler() callback threw."); + } + S3JniHook_localundup(hook); + } + return rc; +} + +S3JniApi(sqlite3_busy_handler(),jint,1busy_1handler)( + JniArgsEnvClass, jlong jpDb, jobject jBusy +){ + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + S3JniHook * const pHook = ps ? &ps->hooks.busyHandler : 0; + S3JniHook hook = S3JniHook_empty; + int rc = 0; + + if( !ps ) return (jint)SQLITE_MISUSE; + S3JniDb_mutex_enter; + if( jBusy ){ + if( pHook->jObj && (*env)->IsSameObject(env, pHook->jObj, jBusy) ){ + /* Same object - this is a no-op. */ + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jBusy); + hook.jObj = S3JniRefGlobal(jBusy); + hook.midCallback = (*env)->GetMethodID(env, klazz, "call", "(I)I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + rc = SQLITE_ERROR; + } + } + } + if( 0==rc ){ + if( jBusy ){ + if( hook.jObj ){ /* Replace handler */ + rc = sqlite3_busy_handler(ps->pDb, s3jni_busy_handler, ps); + if( 0==rc ){ + S3JniHook_unref(pHook); + *pHook = hook /* transfer Java ref ownership */; + hook = S3JniHook_empty; + } + }/* else no-op */ + }else{ /* Clear handler */ + rc = sqlite3_busy_handler(ps->pDb, 0, 0); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + } + } + S3JniHook_unref(&hook); + S3JniDb_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_busy_timeout(),jint,1busy_1timeout)( + JniArgsEnvClass, jlong jpDb, jint ms +){ + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + int rc = SQLITE_MISUSE; + if( ps ){ + S3JniDb_mutex_enter; + S3JniHook_unref(&ps->hooks.busyHandler); + rc = sqlite3_busy_timeout(ps->pDb, (int)ms); + S3JniDb_mutex_leave; + } + return rc; +} + +S3JniApi(sqlite3_cancel_auto_extension(),jboolean,1cancel_1auto_1extension)( + JniArgsEnvClass, jobject jAutoExt +){ + S3JniAutoExtension * ax; + jboolean rc = JNI_FALSE; + int i; + + if( !jAutoExt ){ + return rc; + } + S3JniAutoExt_mutex_enter; + /* This algo corresponds to the one in the core. */ + for( i = SJG.autoExt.nExt-1; i >= 0; --i ){ + ax = &SJG.autoExt.aExt[i]; + if( ax->jObj && (*env)->IsSameObject(env, ax->jObj, jAutoExt) ){ + S3JniAutoExtension_clear(ax); + /* Move final entry into this slot. */ + --SJG.autoExt.nExt; + *ax = SJG.autoExt.aExt[SJG.autoExt.nExt]; + SJG.autoExt.aExt[SJG.autoExt.nExt] = S3JniHook_empty; + assert( !SJG.autoExt.aExt[SJG.autoExt.nExt].jObj ); + rc = JNI_TRUE; + break; + } + } + S3JniAutoExt_mutex_leave; + return rc; +} + +/* Wrapper for sqlite3_close(_v2)(). */ +static jint s3jni_close_db(JNIEnv * const env, jlong jpDb, int version){ + int rc = 0; + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + + assert(version == 1 || version == 2); + if( ps ){ + rc = 1==version + ? (jint)sqlite3_close(ps->pDb) + : (jint)sqlite3_close_v2(ps->pDb); + } + return (jint)rc; +} + +S3JniApi(sqlite3_close(),jint,1close)(JniArgsEnvClass, jlong pDb){ + return s3jni_close_db(env, pDb, 1); +} + +S3JniApi(sqlite3_close_v2(),jint,1close_1v2)(JniArgsEnvClass, jlong pDb){ + return s3jni_close_db(env, pDb, 2); +} + +/* +** Assumes z is an array of unsigned short and returns the index in +** that array of the first element with the value 0. +*/ +static unsigned int s3jni_utf16_strlen(void const * z){ + unsigned int i = 0; + const unsigned short * p = z; + while( p[i] ) ++i; + return i; +} + +/* Descriptive alias for use with sqlite3_collation_needed(). */ +typedef S3JniHook S3JniCollationNeeded; + +/* Central C-to-Java sqlite3_collation_needed16() hook impl. */ +static void s3jni_collation_needed_impl16(void *pState, sqlite3 *pDb, + int eTextRep, const void * z16Name){ + S3JniCollationNeeded * const pHook = pState; + S3JniDeclLocal_env; + S3JniHook hook; + + S3JniHook_localdup(pHook, &hook); + if( hook.jObj ){ + unsigned int const nName = s3jni_utf16_strlen(z16Name); + jstring jName = (*env)->NewString(env, (jchar const *)z16Name, nName); + + s3jni_oom_check( jName ); + assert( hook.jExtra ); + S3JniIfThrew{ + S3JniExceptionClear; + }else if( hook.jExtra ){ + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, + hook.jExtra, (jint)eTextRep, jName); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("sqlite3_collation_needed() callback"); + } + } + S3JniUnrefLocal(jName); + S3JniHook_localundup(hook); + } +} + +S3JniApi(sqlite3_collation_needed(),jint,1collation_1needed)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + S3JniDb * ps; + S3JniCollationNeeded * pHook; + int rc = 0; + + S3JniDb_mutex_enter; + ps = S3JniDb_from_jlong(jpDb); + if( !ps ){ + S3JniDb_mutex_leave; + return SQLITE_MISUSE; + } + pHook = &ps->hooks.collationNeeded; + if( pHook->jObj && jHook && + (*env)->IsSameObject(env, pHook->jObj, jHook) ){ + /* no-op */ + }else if( !jHook ){ + rc = sqlite3_collation_needed(ps->pDb, 0, 0); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jHook); + jmethodID const xCallback = (*env)->GetMethodID( + env, klazz, "call", "(Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)V" + ); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + rc = s3jni_db_exception(ps->pDb, SQLITE_MISUSE, + "Cannot not find matching call() in " + "CollationNeededCallback object."); + }else{ + rc = sqlite3_collation_needed16(ps->pDb, pHook, + s3jni_collation_needed_impl16); + if( 0==rc ){ + S3JniHook_unref(pHook); + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jHook); + pHook->jExtra = S3JniRefGlobal(ps->jDb); + } + } + } + S3JniDb_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_column_blob(),jbyteArray,1column_1blob)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + void const * const p = sqlite3_column_blob(pStmt, (int)ndx); + int const n = p ? sqlite3_column_bytes(pStmt, (int)ndx) : 0; + + return p ? s3jni_new_jbyteArray(p, n) : 0; +} + +S3JniApi(sqlite3_column_double(),jdouble,1column_1double)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + return (jdouble)sqlite3_column_double(PtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_column_int(),jint,1column_1int)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + return (jint)sqlite3_column_int(PtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_column_int64(),jlong,1column_1int64)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + return (jlong)sqlite3_column_int64(PtrGet_sqlite3_stmt(jpStmt), (int)ndx); +} + +S3JniApi(sqlite3_column_java_object(),jobject,1column_1java_1object)( + JniArgsEnvClass, jlong jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = LongPtrGet_sqlite3_stmt(jpStmt); + jobject rv = 0; + if( stmt ){ + sqlite3 * const db = sqlite3_db_handle(stmt); + sqlite3_value * sv; + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + sv = sqlite3_column_value(stmt, (int)ndx); + if( sv ){ + rv = S3JniRefLocal( + sqlite3_value_pointer(sv, s3jni__value_jref_key) + ); + } + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + } + return rv; +} + +S3JniApi(sqlite3_column_nio_buffer(),jobject,1column_1nio_1buffer)( + JniArgsEnvClass, jobject jStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jStmt); + jobject rv = 0; + if( stmt ){ + const void * const p = sqlite3_column_blob(stmt, (int)ndx); + if( p ){ + const int n = sqlite3_column_bytes(stmt, (int)ndx); + rv = s3jni__blob_to_ByteBuffer(env, p, n); + } + } + return rv; +} + +S3JniApi(sqlite3_column_text(),jbyteArray,1column_1text)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jpStmt); + const unsigned char * const p = stmt ? sqlite3_column_text(stmt, (int)ndx) : 0; + const int n = p ? sqlite3_column_bytes(stmt, (int)ndx) : 0; + return p ? s3jni_new_jbyteArray(p, n) : NULL; +} + +#if 0 +// this impl might prove useful. +S3JniApi(sqlite3_column_text(),jstring,1column_1text)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jpStmt); + const unsigned char * const p = stmt ? sqlite3_column_text(stmt, (int)ndx) : 0; + const int n = p ? sqlite3_column_bytes(stmt, (int)ndx) : 0; + return p ? s3jni_utf8_to_jstring( (const char *)p, n) : 0; +} +#endif + +S3JniApi(sqlite3_column_text16(),jstring,1column_1text16)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_stmt * const stmt = PtrGet_sqlite3_stmt(jpStmt); + const void * const p = stmt ? sqlite3_column_text16(stmt, (int)ndx) : 0; + const int n = p ? sqlite3_column_bytes16(stmt, (int)ndx) : 0; + return s3jni_text16_to_jstring(env, p, n); +} + +S3JniApi(sqlite3_column_value(),jobject,1column_1value)( + JniArgsEnvClass, jobject jpStmt, jint ndx +){ + sqlite3_value * const sv = + sqlite3_column_value(PtrGet_sqlite3_stmt(jpStmt), (int)ndx) + /* reminder: returns an SQL NULL if jpStmt==NULL */; + return new_java_sqlite3_value(env, sv); +} + +/* +** Impl for commit hooks (if isCommit is true) or rollback hooks. +*/ +static int s3jni_commit_rollback_hook_impl(int isCommit, S3JniDb * const ps){ + S3JniDeclLocal_env; + int rc = 0; + S3JniHook hook; + + S3JniHook_localdup(isCommit + ? &ps->hooks.commit : &ps->hooks.rollback, + &hook); + if( hook.jObj ){ + rc = isCommit + ? (int)(*env)->CallIntMethod(env, hook.jObj, hook.midCallback) + : (int)((*env)->CallVoidMethod(env, hook.jObj, hook.midCallback), 0); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, SQLITE_ERROR, + isCommit + ? "Commit hook callback threw" + : "Rollback hook callback threw"); + } + S3JniHook_localundup(hook); + } + return rc; +} + +/* C-to-Java commit hook wrapper. */ +static int s3jni_commit_hook_impl(void *pP){ + return s3jni_commit_rollback_hook_impl(1, pP); +} + +/* C-to-Java rollback hook wrapper. */ +static void s3jni_rollback_hook_impl(void *pP){ + (void)s3jni_commit_rollback_hook_impl(0, pP); +} + +/* +** Proxy for sqlite3_commit_hook() (if isCommit is true) or +** sqlite3_rollback_hook(). +*/ +static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, + jlong jpDb, jobject jHook){ + S3JniDb * ps; + jobject pOld = 0; /* previous hook */ + S3JniHook * pHook; /* ps->hooks.commit|rollback */ + + S3JniDb_mutex_enter; + ps = S3JniDb_from_jlong(jpDb); + if( !ps ){ + s3jni_db_error(ps->pDb, SQLITE_MISUSE, 0); + S3JniDb_mutex_leave; + return 0; + } + pHook = isCommit ? &ps->hooks.commit : &ps->hooks.rollback; + pOld = pHook->jObj; + if( pOld && jHook && + (*env)->IsSameObject(env, pOld, jHook) ){ + /* No-op. */ + }else if( !jHook ){ + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + *pHook = S3JniHook_empty; + if( isCommit ) sqlite3_commit_hook(ps->pDb, 0, 0); + else sqlite3_rollback_hook(ps->pDb, 0, 0); + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jHook); + jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", + isCommit ? "()I" : "()V"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionReport; + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching call() method in" + "hook object."); + }else{ + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jHook); + if( isCommit ) sqlite3_commit_hook(ps->pDb, s3jni_commit_hook_impl, ps); + else sqlite3_rollback_hook(ps->pDb, s3jni_rollback_hook_impl, ps); + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + } + } + S3JniDb_mutex_leave; + return pOld; +} + +S3JniApi(sqlite3_commit_hook(),jobject,1commit_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + return s3jni_commit_rollback_hook(1, env, jpDb, jHook); +} + +S3JniApi(sqlite3_compileoption_get(),jstring,1compileoption_1get)( + JniArgsEnvClass, jint n +){ + const char * z = sqlite3_compileoption_get(n); + jstring const rv = z ? (*env)->NewStringUTF( env, z ) : 0; + /* We know these to be ASCII, so MUTF-8 is fine. */; + s3jni_oom_check(z ? !!rv : 1); + return rv; +} + +S3JniApi(sqlite3_compileoption_used(),jboolean,1compileoption_1used)( + JniArgsEnvClass, jstring name +){ + const char *zUtf8 = s3jni_jstring_to_mutf8(name) + /* We know these to be ASCII, so MUTF-8 is fine (and + hypothetically faster to convert). */; + const jboolean rc = + 0==sqlite3_compileoption_used(zUtf8) ? JNI_FALSE : JNI_TRUE; + s3jni_mutf8_release(name, zUtf8); + return rc; +} + +S3JniApi(sqlite3_complete(),jint,1complete)( + JniArgsEnvClass, jbyteArray jSql +){ + jbyte * const pBuf = s3jni_jbyteArray_bytes(jSql); + const jsize nBA = pBuf ? (*env)->GetArrayLength(env, jSql) : 0; + int rc; + + assert( (nBA>0 ? 0==pBuf[nBA-1] : (pBuf ? 0==*pBuf : 1)) + && "Byte array is not NUL-terminated." ); + rc = (pBuf && 0==pBuf[(nBA ? nBA-1 : 0)]) + ? sqlite3_complete( (const char *)pBuf ) + : (jSql ? SQLITE_NOMEM : SQLITE_MISUSE); + s3jni_jbyteArray_release(jSql, pBuf); + return rc; +} + +S3JniApi(sqlite3_config() /*for a small subset of options.*/ + sqlite3_config__enable()/* internal name to avoid name-mangling issues*/, + jint,1config_1_1enable)(JniArgsEnvClass, jint n){ + switch( n ){ + case SQLITE_CONFIG_SINGLETHREAD: + case SQLITE_CONFIG_MULTITHREAD: + case SQLITE_CONFIG_SERIALIZED: + return sqlite3_config( n ); + default: + return SQLITE_MISUSE; + } +} +/* C-to-Java SQLITE_CONFIG_LOG wrapper. */ +static void s3jni_config_log(void *ignored, int errCode, const char *z){ + S3JniDeclLocal_env; + S3JniHook hook = S3JniHook_empty; + + S3JniHook_localdup(&SJG.hook.configlog, &hook); + if( hook.jObj ){ + jstring const jArg1 = z ? s3jni_utf8_to_jstring(z, -1) : 0; + if( z ? !!jArg1 : 1 ){ + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, errCode, jArg1); + } + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("SQLITE_CONFIG_LOG callback"); + S3JniExceptionClear; + } + S3JniHook_localundup(hook); + S3JniUnrefLocal(jArg1); + } +} + +S3JniApi(sqlite3_config() /* for SQLITE_CONFIG_LOG */ + sqlite3_config__config_log() /* internal name */, + jint, 1config_1_1CONFIG_1LOG +)(JniArgsEnvClass, jobject jLog){ + S3JniHook * const pHook = &SJG.hook.configlog; + int rc = 0; + + S3JniGlobal_mutex_enter; + if( !jLog ){ + rc = sqlite3_config( SQLITE_CONFIG_LOG, NULL, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + }else if( pHook->jObj && (*env)->IsSameObject(env, jLog, pHook->jObj) ){ + /* No-op */ + }else { + jclass const klazz = (*env)->GetObjectClass(env, jLog); + jmethodID const midCallback = (*env)->GetMethodID(env, klazz, "call", + "(ILjava/lang/String;)V"); + S3JniUnrefLocal(klazz); + if( midCallback ){ + rc = sqlite3_config( SQLITE_CONFIG_LOG, s3jni_config_log, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + pHook->midCallback = midCallback; + pHook->jObj = S3JniRefGlobal(jLog); + } + }else{ + S3JniExceptionWarnIgnore; + rc = SQLITE_ERROR; + } + } + S3JniGlobal_mutex_leave; + return rc; +} + +#ifdef SQLITE_ENABLE_SQLLOG +/* C-to-Java SQLITE_CONFIG_SQLLOG wrapper. */ +static void s3jni_config_sqllog(void *ignored, sqlite3 *pDb, const char *z, int op){ + jobject jArg0 = 0; + jstring jArg1 = 0; + S3JniDeclLocal_env; + S3JniDb * const ps = S3JniDb_from_c(pDb); + S3JniHook hook = S3JniHook_empty; + + if( ps ){ + S3JniHook_localdup(&SJG.hook.sqllog, &hook); + } + if( !hook.jObj ) return; + jArg0 = S3JniRefLocal(ps->jDb); + switch( op ){ + case 0: /* db opened */ + case 1: /* SQL executed */ + jArg1 = s3jni_utf8_to_jstring( z, -1); + break; + case 2: /* db closed */ + break; + default: + (*env)->FatalError(env, "Unhandled 4th arg to SQLITE_CONFIG_SQLLOG."); + break; + } + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, jArg0, jArg1, op); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("SQLITE_CONFIG_SQLLOG callback"); + S3JniExceptionClear; + } + S3JniHook_localundup(hook); + S3JniUnrefLocal(jArg0); + S3JniUnrefLocal(jArg1); +} +//! Requirement of SQLITE_CONFIG_SQLLOG. +void sqlite3_init_sqllog(void){ + sqlite3_config( SQLITE_CONFIG_SQLLOG, s3jni_config_sqllog, 0 ); +} +#endif + +S3JniApi(sqlite3_config() /* for SQLITE_CONFIG_SQLLOG */ + sqlite3_config__SQLLOG() /*internal name*/, + jint, 1config_1_1SQLLOG +)(JniArgsEnvClass, jobject jLog){ +#ifndef SQLITE_ENABLE_SQLLOG + return SQLITE_MISUSE; +#else + S3JniHook * const pHook = &SJG.hook.sqllog; + int rc = 0; + + S3JniGlobal_mutex_enter; + if( !jLog ){ + rc = sqlite3_config( SQLITE_CONFIG_SQLLOG, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + } + }else if( pHook->jObj && (*env)->IsSameObject(env, jLog, pHook->jObj) ){ + /* No-op */ + }else { + jclass const klazz = (*env)->GetObjectClass(env, jLog); + jmethodID const midCallback = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/capi/sqlite3;" + "Ljava/lang/String;" + "I)V"); + S3JniUnrefLocal(klazz); + if( midCallback ){ + rc = sqlite3_config( SQLITE_CONFIG_SQLLOG, s3jni_config_sqllog, NULL ); + if( 0==rc ){ + S3JniHook_unref(pHook); + pHook->midCallback = midCallback; + pHook->jObj = S3JniRefGlobal(jLog); + } + }else{ + S3JniExceptionWarnIgnore; + rc = SQLITE_ERROR; + } + } + S3JniGlobal_mutex_leave; + return rc; +#endif +} + +S3JniApi(sqlite3_context_db_handle(),jobject,1context_1db_1handle)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_context * const pCx = PtrGet_sqlite3_context(jpCx); + sqlite3 * const pDb = pCx ? sqlite3_context_db_handle(pCx) : 0; + S3JniDb * const ps = pDb ? S3JniDb_from_c(pDb) : 0; + return ps ? ps->jDb : 0; +} + +/* +** State for CollationCallbacks. This used to be its own separate +** type, but has since been consolidated with S3JniHook. It retains +** its own typedef for code legibility and searchability reasons. +*/ +typedef S3JniHook S3JniCollationCallback; + +/* +** Proxy for Java-side CollationCallback.xCompare() callbacks. +*/ +static int CollationCallback_xCompare(void *pArg, int nLhs, const void *lhs, + int nRhs, const void *rhs){ + S3JniCollationCallback * const pCC = pArg; + S3JniDeclLocal_env; + jint rc = 0; + if( pCC->jObj ){ + jbyteArray jbaLhs = s3jni_new_jbyteArray(lhs, (jint)nLhs); + jbyteArray jbaRhs = jbaLhs + ? s3jni_new_jbyteArray(rhs, (jint)nRhs) : 0; + if( !jbaRhs ){ + S3JniUnrefLocal(jbaLhs); + /* We have no recovery strategy here. */ + s3jni_oom_check( jbaRhs ); + return 0; + } + rc = (*env)->CallIntMethod(env, pCC->jObj, pCC->midCallback, + jbaLhs, jbaRhs); + S3JniExceptionIgnore; + S3JniUnrefLocal(jbaLhs); + S3JniUnrefLocal(jbaRhs); + } + return (int)rc; +} + +/* CollationCallback finalizer for use by the sqlite3 internals. */ +static void CollationCallback_xDestroy(void *pArg){ + S3JniCollationCallback * const pCC = pArg; + S3JniDeclLocal_env; + S3JniHook_free(pCC); +} + +S3JniApi(sqlite3_create_collation() sqlite3_create_collation_v2(), + jint,1create_1collation +)(JniArgsEnvClass, jobject jDb, jstring name, jint eTextRep, + jobject oCollation){ + int rc; + S3JniDb * ps; + + if( !jDb || !name || !encodingTypeIsValid(eTextRep) ){ + return (jint)SQLITE_MISUSE; + } + S3JniDb_mutex_enter; + ps = S3JniDb_from_java(jDb); + jclass const klazz = (*env)->GetObjectClass(env, oCollation); + jmethodID const midCallback = + (*env)->GetMethodID(env, klazz, "call", "([B[B)I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew{ + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Could not get call() method from " + "CollationCallback object."); + }else{ + char * const zName = s3jni_jstring_to_utf8(name, 0); + S3JniCollationCallback * const pCC = + zName ? S3JniHook_alloc() : 0; + if( pCC ){ + rc = sqlite3_create_collation_v2(ps->pDb, zName, (int)eTextRep, + pCC, CollationCallback_xCompare, + CollationCallback_xDestroy); + if( 0==rc ){ + pCC->midCallback = midCallback; + pCC->jObj = S3JniRefGlobal(oCollation); + pCC->doXDestroy = 1; + }else{ + CollationCallback_xDestroy(pCC); + } + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zName); + } + S3JniDb_mutex_leave; + return (jint)rc; +} + +S3JniApi(sqlite3_create_function() sqlite3_create_function_v2() + sqlite3_create_window_function(), + jint,1create_1function +)(JniArgsEnvClass, jobject jDb, jstring jFuncName, jint nArg, + jint eTextRep, jobject jFunctor){ + S3JniUdf * s = 0; + int rc; + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + char * zFuncName = 0; + + if( !pDb || !jFuncName ){ + return SQLITE_MISUSE; + }else if( !encodingTypeIsValid(eTextRep) ){ + return s3jni_db_error(pDb, SQLITE_FORMAT, + "Invalid function encoding option."); + } + s = S3JniUdf_alloc(env, jFunctor); + if( !s ) return SQLITE_NOMEM; + + if( UDF_UNKNOWN_TYPE==s->type ){ + rc = s3jni_db_error(pDb, SQLITE_MISUSE, + "Cannot unambiguously determine function type."); + S3JniUdf_free(env, s, 1); + goto error_cleanup; + } + zFuncName = s3jni_jstring_to_utf8(jFuncName,0); + if( !zFuncName ){ + rc = SQLITE_NOMEM; + S3JniUdf_free(env, s, 1); + goto error_cleanup; + } + s->zFuncName = zFuncName /* pass on ownership */; + if( UDF_WINDOW == s->type ){ + rc = sqlite3_create_window_function(pDb, zFuncName, nArg, eTextRep, s, + udf_xStep, udf_xFinal, udf_xValue, + udf_xInverse, S3JniUdf_finalizer); + }else{ + udf_xFunc_f xFunc = 0; + udf_xStep_f xStep = 0; + udf_xFinal_f xFinal = 0; + if( UDF_SCALAR == s->type ){ + xFunc = udf_xFunc; + }else{ + assert( UDF_AGGREGATE == s->type ); + xStep = udf_xStep; + xFinal = udf_xFinal; + } + rc = sqlite3_create_function_v2(pDb, zFuncName, nArg, eTextRep, s, + xFunc, xStep, xFinal, S3JniUdf_finalizer); + } +error_cleanup: + /* Reminder: on sqlite3_create_function() error, s will be + ** destroyed via create_function(). */ + return (jint)rc; +} + + +S3JniApi(sqlite3_db_config() /*for MAINDBNAME*/, + jint,1db_1config__Lorg_sqlite_jni_capi_sqlite3_2ILjava_lang_String_2 +)(JniArgsEnvClass, jobject jDb, jint op, jstring jStr){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc; + char *zStr; + + switch( (ps && jStr) ? op : 0 ){ + case SQLITE_DBCONFIG_MAINDBNAME: + S3JniDb_mutex_enter + /* Protect against a race in modifying/freeing + ps->zMainDbName. */; + zStr = s3jni_jstring_to_utf8( jStr, 0); + if( zStr ){ + rc = sqlite3_db_config(ps->pDb, (int)op, zStr); + if( rc ){ + sqlite3_free( zStr ); + }else{ + sqlite3_free( ps->zMainDbName ); + ps->zMainDbName = zStr; + } + }else{ + rc = SQLITE_NOMEM; + } + S3JniDb_mutex_leave; + break; + case 0: + default: + rc = SQLITE_MISUSE; + } + return rc; +} + +S3JniApi( + sqlite3_db_config(), + /* WARNING: openjdk v19 creates a different mangled name for this + ** function than openjdk v8 does. We account for that by exporting + ** both versions of the name. */ + jint,1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2 +)( + JniArgsEnvClass, jobject jDb, jint op, jint onOff, jobject jOut +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc; + switch( ps ? op : 0 ){ + case SQLITE_DBCONFIG_ENABLE_FKEY: + case SQLITE_DBCONFIG_ENABLE_TRIGGER: + case SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: + case SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: + case SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: + case SQLITE_DBCONFIG_ENABLE_QPSG: + case SQLITE_DBCONFIG_TRIGGER_EQP: + case SQLITE_DBCONFIG_RESET_DATABASE: + case SQLITE_DBCONFIG_DEFENSIVE: + case SQLITE_DBCONFIG_WRITABLE_SCHEMA: + case SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: + case SQLITE_DBCONFIG_DQS_DML: + case SQLITE_DBCONFIG_DQS_DDL: + case SQLITE_DBCONFIG_ENABLE_VIEW: + case SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: + case SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case SQLITE_DBCONFIG_STMT_SCANSTATUS: + case SQLITE_DBCONFIG_REVERSE_SCANORDER: { + int pOut = 0; + rc = sqlite3_db_config( ps->pDb, (int)op, onOff, &pOut ); + if( 0==rc && jOut ){ + OutputPointer_set_Int32(env, jOut, pOut); + } + break; + } + default: + rc = SQLITE_MISUSE; + } + return (jint)rc; +} + +/* +** This is a workaround for openjdk v19 (and possibly others) encoding +** this function's name differently than JDK v8 does. If we do not +** install both names for this function then Java will not be able to +** find the function in both environments. +*/ +JniDecl(jint,1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_00024Int32_2)( + JniArgsEnvClass, jobject jDb, jint op, jint onOff, jobject jOut +){ + return JniFuncName(1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2)( + env, jKlazz, jDb, op, onOff, jOut + ); +} + +S3JniApi(sqlite3_db_filename(),jstring,1db_1filename)( + JniArgsEnvClass, jobject jDb, jstring jDbName +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + char *zDbName; + jstring jRv = 0; + int nStr = 0; + + if( !ps || !jDbName ){ + return 0; + } + zDbName = s3jni_jstring_to_utf8( jDbName, &nStr); + if( zDbName ){ + char const * zRv = sqlite3_db_filename(ps->pDb, zDbName); + sqlite3_free(zDbName); + if( zRv ){ + jRv = s3jni_utf8_to_jstring( zRv, -1); + } + } + return jRv; +} + +S3JniApi(sqlite3_db_handle(),jobject,1db_1handle)( + JniArgsEnvClass, jobject jpStmt +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + sqlite3 * const pDb = pStmt ? sqlite3_db_handle(pStmt) : 0; + S3JniDb * const ps = pDb ? S3JniDb_from_c(pDb) : 0; + return ps ? ps->jDb : 0; +} + +S3JniApi(sqlite3_db_readonly(),jint,1db_1readonly)( + JniArgsEnvClass, jobject jDb, jstring jDbName +){ + int rc = 0; + S3JniDb * const ps = S3JniDb_from_java(jDb); + char *zDbName = jDbName ? s3jni_jstring_to_utf8( jDbName, 0 ) : 0; + rc = sqlite3_db_readonly(ps ? ps->pDb : 0, zDbName); + sqlite3_free(zDbName); + return (jint)rc; +} + +S3JniApi(sqlite3_db_release_memory(),jint,1db_1release_1memory)( + JniArgsEnvClass, jobject jDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + return pDb ? sqlite3_db_release_memory(pDb) : SQLITE_MISUSE; +} + +S3JniApi(sqlite3_db_status(),jint,1db_1status)( + JniArgsEnvClass, jobject jDb, jint op, jobject jOutCurrent, + jobject jOutHigh, jboolean reset +){ + int iCur = 0, iHigh = 0; + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + int rc = sqlite3_db_status( pDb, op, &iCur, &iHigh, reset ); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCurrent, iCur); + OutputPointer_set_Int32(env, jOutHigh, iHigh); + } + return (jint)rc; +} + +S3JniApi(sqlite3_errcode(),jint,1errcode)( + JniArgsEnvClass, jobject jpDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + return pDb ? sqlite3_errcode(pDb) : SQLITE_MISUSE; +} + +S3JniApi(sqlite3_errmsg(),jstring,1errmsg)( + JniArgsEnvClass, jobject jpDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + return pDb ? s3jni_utf8_to_jstring( sqlite3_errmsg(pDb), -1) : 0 + /* We don't use errmsg16() directly only because it would cause an + additional level of internal encoding in sqlite3. The end + effect should be identical to using errmsg16(), however. */; +} + +S3JniApi(sqlite3_set_errmsg(),jint,1set_1errmsg)( + JniArgsEnvClass, jobject jpDb, jint errCode, jstring msg +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + const char *zUtf8; + jint rc; + if( !pDb ) return SQLITE_MISUSE; + zUtf8 = msg ? s3jni_jstring_to_mutf8(msg) : NULL; + rc = sqlite3_set_errmsg(pDb, (int)errCode, zUtf8); + s3jni_mutf8_release(msg, zUtf8); + return rc; +} + +S3JniApi(sqlite3_errstr(),jstring,1errstr)( + JniArgsEnvClass, jint rcCode +){ + jstring rv; + const char * z = sqlite3_errstr((int)rcCode); + if( !z ){ + /* This hypothetically cannot happen, but we'll behave like the + low-level library would in such a case... */ + z = "unknown error"; + } + rv = (*env)->NewStringUTF(env, z) + /* We know these values to be plain ASCII, so pose no MUTF-8 + ** incompatibility */; + s3jni_oom_check( rv ); + return rv; +} + +#ifndef SQLITE_ENABLE_NORMALIZE +/* Dummy stub for sqlite3_normalized_sql(). Never called. */ +static const char * sqlite3_normalized_sql(sqlite3_stmt *s){ + S3JniDeclLocal_env; + (*env)->FatalError(env, "dummy sqlite3_normalized_sql() was " + "impossibly called.") /* does not return */; + return 0; +} +#endif + +/* +** Impl for sqlite3_expanded_sql() (if isExpanded is true) and +** sqlite3_normalized_sql(). +*/ +static jstring s3jni_xn_sql(int isExpanded, JNIEnv *env, jobject jpStmt){ + jstring rv = 0; + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + + if( pStmt ){ + char * zSql = isExpanded + ? sqlite3_expanded_sql(pStmt) + : (char*)sqlite3_normalized_sql(pStmt); + s3jni_oom_fatal(zSql); + if( zSql ){ + rv = s3jni_utf8_to_jstring(zSql, -1); + if( isExpanded ) sqlite3_free(zSql); + } + } + return rv; +} + +S3JniApi(sqlite3_expanded_sql(),jstring,1expanded_1sql)( + JniArgsEnvClass, jobject jpStmt +){ + return s3jni_xn_sql(1, env, jpStmt); +} + +S3JniApi(sqlite3_normalized_sql(),jstring,1normalized_1sql)( + JniArgsEnvClass, jobject jpStmt +){ +#ifdef SQLITE_ENABLE_NORMALIZE + return s3jni_xn_sql(0, env, jpStmt); +#else + return 0; +#endif +} + +S3JniApi(sqlite3_extended_result_codes(),jint,1extended_1result_1codes)( + JniArgsEnvClass, jobject jpDb, jboolean onoff +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + int const rc = pDb + ? sqlite3_extended_result_codes(pDb, onoff ? 1 : 0) + : SQLITE_MISUSE; + return rc; +} + +S3JniApi(sqlite3_finalize(),jint,1finalize)( + JniArgsEnvClass, jlong jpStmt +){ + return jpStmt + ? sqlite3_finalize(LongPtrGet_sqlite3_stmt(jpStmt)) + : 0; +} + +S3JniApi(sqlite3_get_auxdata(),jobject,1get_1auxdata)( + JniArgsEnvClass, jobject jCx, jint n +){ + return sqlite3_get_auxdata(PtrGet_sqlite3_context(jCx), (int)n); +} + +S3JniApi(sqlite3_initialize(),jint,1initialize)( + JniArgsEnvClass +){ + return sqlite3_initialize(); +} + +S3JniApi(sqlite3_interrupt(),void,1interrupt)( + JniArgsEnvClass, jobject jpDb +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + if( pDb ){ + sqlite3_interrupt(pDb); + } +} + +S3JniApi(sqlite3_is_interrupted(),jboolean,1is_1interrupted)( + JniArgsEnvClass, jobject jpDb +){ + int rc = 0; + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + if( pDb ){ + rc = sqlite3_is_interrupted(pDb); + } + return rc ? JNI_TRUE : JNI_FALSE; +} + +/* +** Uncaches the current JNIEnv from the S3JniGlobal state, clearing +** any resources owned by that cache entry and making that slot +** available for re-use. +*/ +S3JniApi(sqlite3_java_uncache_thread(), jboolean, 1java_1uncache_1thread)( + JniArgsEnvClass +){ + int rc; + S3JniEnv_mutex_enter; + rc = S3JniEnv_uncache(env); + S3JniEnv_mutex_leave; + return rc ? JNI_TRUE : JNI_FALSE; +} + +S3JniApi(sqlite3_jni_db_error(), jint, 1jni_1db_1error)( + JniArgsEnvClass, jobject jDb, jint jRc, jstring jStr +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc = SQLITE_MISUSE; + if( ps ){ + char *zStr; + zStr = jStr + ? s3jni_jstring_to_utf8( jStr, 0) + : NULL; + rc = s3jni_db_error( ps->pDb, (int)jRc, zStr ); + sqlite3_free(zStr); + } + return rc; +} + +S3JniApi(sqlite3_jni_supports_nio(), jboolean,1jni_1supports_1nio)( + JniArgsEnvClass +){ + return SJG.g.byteBuffer.klazz ? JNI_TRUE : JNI_FALSE; +} + + +S3JniApi(sqlite3_keyword_check(),jboolean,1keyword_1check)( + JniArgsEnvClass, jstring jWord +){ + int nWord = 0; + char * zWord = s3jni_jstring_to_utf8(jWord, &nWord); + int rc = 0; + + s3jni_oom_check(jWord ? !!zWord : 1); + if( zWord && nWord ){ + rc = sqlite3_keyword_check(zWord, nWord); + } + sqlite3_free(zWord); + return rc ? JNI_TRUE : JNI_FALSE; +} + +S3JniApi(sqlite3_keyword_name(),jstring,1keyword_1name)( + JniArgsEnvClass, jint ndx +){ + const char * zWord = 0; + int n = 0; + jstring rv = 0; + + if( 0==sqlite3_keyword_name(ndx, &zWord, &n) ){ + rv = s3jni_utf8_to_jstring(zWord, n); + } + return rv; +} + + +S3JniApi(sqlite3_last_insert_rowid(),jlong,1last_1insert_1rowid)( + JniArgsEnvClass, jobject jpDb +){ + return (jlong)sqlite3_last_insert_rowid(PtrGet_sqlite3(jpDb)); +} + +S3JniApi(sqlite3_limit(),jint,1limit)( + JniArgsEnvClass, jobject jpDb, jint id, jint newVal +){ + jint rc = 0; + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + if( pDb ){ + rc = sqlite3_limit( pDb, (int)id, (int)newVal ); + } + return rc; +} + +/* Pre-open() code common to sqlite3_open[_v2](). */ +static int s3jni_open_pre(JNIEnv * const env, S3JniEnv **jc, + jstring jDbName, char **zDbName, + S3JniDb ** ps){ + int rc = 0; + jobject jDb = 0; + + *jc = S3JniEnv_get(); + if( !*jc ){ + rc = SQLITE_NOMEM; + goto end; + } + *zDbName = jDbName ? s3jni_jstring_to_utf8( jDbName, 0) : 0; + if( jDbName && !*zDbName ){ + rc = SQLITE_NOMEM; + goto end; + } + jDb = new_java_sqlite3(env, 0); + if( !jDb ){ + sqlite3_free(*zDbName); + *zDbName = 0; + rc = SQLITE_NOMEM; + goto end; + } + *ps = S3JniDb_alloc(env, jDb); + if( *ps ){ + (*jc)->pdbOpening = *ps; + }else{ + S3JniUnrefLocal(jDb); + rc = SQLITE_NOMEM; + } +end: + return rc; +} + +/* +** Post-open() code common to both the sqlite3_open() and +** sqlite3_open_v2() bindings. ps->jDb must be the +** org.sqlite.jni.capi.sqlite3 object which will hold the db's native +** pointer. theRc must be the result code of the open() op. If +** *ppDb is NULL then ps is set aside and its state cleared, +** else ps is associated with *ppDb. If *ppDb is not NULL then +** ps->jDb is stored in jOut (an OutputPointer.sqlite3 instance). +** +** Must be called if s3jni_open_pre() succeeds and must not be called +** if it doesn't. +** +** Returns theRc. +*/ +static int s3jni_open_post(JNIEnv * const env, S3JniEnv * const jc, + S3JniDb * ps, sqlite3 **ppDb, + jobject jOut, int theRc){ + int rc = 0; + jc->pdbOpening = 0; + if( *ppDb ){ + assert(ps->jDb); + if( 0==ps->pDb ){ + ps->pDb = *ppDb; + NativePointerHolder_set(S3JniNph(sqlite3), ps->jDb, *ppDb); + }else{ + assert( ps->pDb==*ppDb + && "Set up via s3jni_run_java_auto_extensions()" ); + } + rc = sqlite3_set_clientdata(ps->pDb, S3JniDb_clientdata_key, + ps, S3JniDb_xDestroy) + /* As of here, the Java/C connection is complete */; + }else{ + S3JniDb_set_aside(ps); + ps = 0; + } + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3), + jOut, ps ? ps->jDb : 0); + return theRc ? theRc : rc; +} + +S3JniApi(sqlite3_open(),jint,1open)( + JniArgsEnvClass, jstring strName, jobject jOut +){ + sqlite3 * pOut = 0; + char *zName = 0; + S3JniDb * ps = 0; + S3JniEnv * jc = 0; + int rc; + + if( 0==jOut ) return SQLITE_MISUSE; + rc = s3jni_open_pre(env, &jc, strName, &zName, &ps); + if( 0==rc ){ + rc = s3jni_open_post(env, jc, ps, &pOut, jOut, + sqlite3_open(zName, &pOut)); + assert(rc==0 ? pOut!=0 : 1); + sqlite3_free(zName); + } + return (jint)rc; +} + +S3JniApi(sqlite3_open_v2(),jint,1open_1v2)( + JniArgsEnvClass, jstring strName, + jobject jOut, jint flags, jstring strVfs +){ + sqlite3 * pOut = 0; + char *zName = 0; + S3JniDb * ps = 0; + S3JniEnv * jc = 0; + char *zVfs = 0; + int rc; + + if( 0==jOut ) return SQLITE_MISUSE; + rc = s3jni_open_pre(env, &jc, strName, &zName, &ps); + if( 0==rc ){ + if( strVfs ){ + zVfs = s3jni_jstring_to_utf8( strVfs, 0); + if( !zVfs ){ + rc = SQLITE_NOMEM; + } + } + if( 0==rc ){ + rc = sqlite3_open_v2(zName, &pOut, (int)flags, zVfs); + } + rc = s3jni_open_post(env, jc, ps, &pOut, jOut, rc); + } + assert(rc==0 ? pOut!=0 : 1); + sqlite3_free(zName); + sqlite3_free(zVfs); + return (jint)rc; +} + +/* Proxy for the sqlite3_prepare[_v2/3]() family. */ +static jint sqlite3_jni_prepare_v123( int prepVersion, JNIEnv * const env, + jclass self, + jlong jpDb, jbyteArray baSql, + jint nMax, jint prepFlags, + jobject jOutStmt, jobject outTail){ + sqlite3_stmt * pStmt = 0; + jobject jStmt = 0; + const char * zTail = 0; + sqlite3 * const pDb = LongPtrGet_sqlite3(jpDb); + jbyte * const pBuf = pDb ? s3jni_jbyteArray_bytes(baSql) : 0; + int rc = SQLITE_ERROR; + + assert(prepVersion==1 || prepVersion==2 || prepVersion==3); + if( !pDb || !jOutStmt ){ + rc = SQLITE_MISUSE; + goto end; + }else if( !pBuf ){ + rc = baSql ? SQLITE_NOMEM : SQLITE_MISUSE; + goto end; + } + jStmt = new_java_sqlite3_stmt(env, 0); + if( !jStmt ){ + rc = SQLITE_NOMEM; + goto end; + } + switch( prepVersion ){ + case 1: rc = sqlite3_prepare(pDb, (const char *)pBuf, + (int)nMax, &pStmt, &zTail); + break; + case 2: rc = sqlite3_prepare_v2(pDb, (const char *)pBuf, + (int)nMax, &pStmt, &zTail); + break; + case 3: rc = sqlite3_prepare_v3(pDb, (const char *)pBuf, + (int)nMax, (unsigned int)prepFlags, + &pStmt, &zTail); + break; + default: + assert(!"Invalid prepare() version"); + } +end: + s3jni_jbyteArray_release(baSql,pBuf); + if( 0==rc ){ + if( 0!=outTail ){ + /* Noting that pBuf is deallocated now but its address is all we need for + ** what follows... */ + assert(zTail ? ((void*)zTail>=(void*)pBuf) : 1); + assert(zTail ? (((int)((void*)zTail - (void*)pBuf)) >= 0) : 1); + OutputPointer_set_Int32( + env, outTail, (int)(zTail ? (zTail - (const char *)pBuf) : 0) + ); + } + if( pStmt ){ + NativePointerHolder_set(S3JniNph(sqlite3_stmt), jStmt, pStmt); + }else{ + /* Happens for comments and whitespace. */ + S3JniUnrefLocal(jStmt); + jStmt = 0; + } + }else{ + S3JniUnrefLocal(jStmt); + jStmt = 0; + } + if( jOutStmt ){ + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3_stmt), + jOutStmt, jStmt); + } + return (jint)rc; +} +S3JniApi(sqlite3_prepare(),jint,1prepare)( + JNIEnv * const env, jclass self, jlong jpDb, jbyteArray baSql, + jint nMax, jobject jOutStmt, jobject outTail +){ + return sqlite3_jni_prepare_v123(1, env, self, jpDb, baSql, nMax, 0, + jOutStmt, outTail); +} +S3JniApi(sqlite3_prepare_v2(),jint,1prepare_1v2)( + JNIEnv * const env, jclass self, jlong jpDb, jbyteArray baSql, + jint nMax, jobject jOutStmt, jobject outTail +){ + return sqlite3_jni_prepare_v123(2, env, self, jpDb, baSql, nMax, 0, + jOutStmt, outTail); +} +S3JniApi(sqlite3_prepare_v3(),jint,1prepare_1v3)( + JNIEnv * const env, jclass self, jlong jpDb, jbyteArray baSql, + jint nMax, jint prepFlags, jobject jOutStmt, jobject outTail +){ + return sqlite3_jni_prepare_v123(3, env, self, jpDb, baSql, nMax, + prepFlags, jOutStmt, outTail); +} + +/* +** Impl for C-to-Java of the callbacks for both sqlite3_update_hook() +** and sqlite3_preupdate_hook(). The differences are that for +** update_hook(): +** +** - pDb is NULL +** - iKey1 is the row ID +** - iKey2 is unused +*/ +static void s3jni_updatepre_hook_impl(void * pState, sqlite3 *pDb, int opId, + const char *zDb, const char *zTable, + sqlite3_int64 iKey1, sqlite3_int64 iKey2){ + S3JniDb * const ps = pState; + S3JniDeclLocal_env; + jstring jDbName; + jstring jTable; + const int isPre = 0!=pDb; + S3JniHook hook; + + S3JniHook_localdup(isPre ? +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + &ps->hooks.preUpdate +#else + &S3JniHook_empty +#endif + : &ps->hooks.update, &hook); + if( !hook.jObj ){ + return; + } + jDbName = s3jni_utf8_to_jstring( zDb, -1); + jTable = jDbName ? s3jni_utf8_to_jstring( zTable, -1) : 0; + S3JniIfThrew { + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_NOMEM, 0); + }else{ + assert( hook.jObj ); + assert( hook.midCallback ); + assert( ps->jDb ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( isPre ) (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, + ps->jDb, (jint)opId, jDbName, jTable, + (jlong)iKey1, (jlong)iKey2); + else +#endif + (*env)->CallVoidMethod(env, hook.jObj, hook.midCallback, + (jint)opId, jDbName, jTable, (jlong)iKey1); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("sqlite3_(pre)update_hook() callback"); + s3jni_db_exception(ps->pDb, 0, + "sqlite3_(pre)update_hook() callback threw"); + } + } + S3JniUnrefLocal(jDbName); + S3JniUnrefLocal(jTable); + S3JniHook_localundup(hook); +} + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +static void s3jni_preupdate_hook_impl(void * pState, sqlite3 *pDb, int opId, + const char *zDb, const char *zTable, + sqlite3_int64 iKey1, sqlite3_int64 iKey2){ + return s3jni_updatepre_hook_impl(pState, pDb, opId, zDb, zTable, + iKey1, iKey2); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +static void s3jni_update_hook_impl(void * pState, int opId, const char *zDb, + const char *zTable, sqlite3_int64 nRowid){ + return s3jni_updatepre_hook_impl(pState, NULL, opId, zDb, zTable, nRowid, 0); +} + +#if !defined(SQLITE_ENABLE_PREUPDATE_HOOK) +/* We need no-op impls for preupdate_{count,depth,blobwrite}() */ +S3JniApi(sqlite3_preupdate_blobwrite(),jint,1preupdate_1blobwrite)( + JniArgsEnvClass, jlong jDb){ return SQLITE_MISUSE; } +S3JniApi(sqlite3_preupdate_count(),jint,1preupdate_1count)( + JniArgsEnvClass, jlong jDb){ return SQLITE_MISUSE; } +S3JniApi(sqlite3_preupdate_depth(),jint,1preupdate_1depth)( + JniArgsEnvClass, jlong jDb){ return SQLITE_MISUSE; } +#endif /* !SQLITE_ENABLE_PREUPDATE_HOOK */ + +/* +** JNI wrapper for both sqlite3_update_hook() and +** sqlite3_preupdate_hook() (if isPre is true). +*/ +static jobject s3jni_updatepre_hook(JNIEnv * env, int isPre, jlong jpDb, jobject jHook){ + S3JniDb * const ps = S3JniDb_from_jlong(jpDb); + jclass klazz; + jobject pOld = 0; + jmethodID xCallback; + S3JniHook * pHook; + + if( !ps ) return 0; + S3JniDb_mutex_enter; + pHook = isPre ? +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + &ps->hooks.preUpdate +#else + 0 +#endif + : &ps->hooks.update; + if( !pHook ){ + goto end; + } + pOld = pHook->jObj; + if( pOld && jHook && (*env)->IsSameObject(env, pOld, jHook) ){ + goto end; + } + if( !jHook ){ + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + *pHook = S3JniHook_empty; +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( isPre ) sqlite3_preupdate_hook(ps->pDb, 0, 0); + else +#endif + sqlite3_update_hook(ps->pDb, 0, 0); + goto end; + } + klazz = (*env)->GetObjectClass(env, jHook); + xCallback = isPre + ? (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/capi/sqlite3;" + "I" + "Ljava/lang/String;" + "Ljava/lang/String;" + "JJ)V") + : (*env)->GetMethodID(env, klazz, "call", + "(ILjava/lang/String;Ljava/lang/String;J)V"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching callback on " + "(pre)update hook object."); + }else{ + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jHook); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( isPre ) sqlite3_preupdate_hook(ps->pDb, s3jni_preupdate_hook_impl, ps); + else +#endif + sqlite3_update_hook(ps->pDb, s3jni_update_hook_impl, ps); + if( pOld ){ + jobject tmp = S3JniRefLocal(pOld); + S3JniUnrefGlobal(pOld); + pOld = tmp; + } + } +end: + S3JniDb_mutex_leave; + return pOld; +} + + +S3JniApi(sqlite3_preupdate_hook(),jobject,1preupdate_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + return s3jni_updatepre_hook(env, 1, jpDb, jHook); +#else + return NULL; +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +} + +/* Impl for sqlite3_preupdate_{new,old}(). */ +static int s3jni_preupdate_newold(JNIEnv * const env, int isNew, jlong jpDb, + jint iCol, jobject jOut){ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3 * const pDb = LongPtrGet_sqlite3(jpDb); + int rc = SQLITE_MISUSE; + if( pDb ){ + sqlite3_value * pOut = 0; + int (*fOrig)(sqlite3*,int,sqlite3_value**) = + isNew ? sqlite3_preupdate_new : sqlite3_preupdate_old; + rc = fOrig(pDb, (int)iCol, &pOut); + if( 0==rc ){ + jobject pWrap = new_java_sqlite3_value(env, pOut); + if( !pWrap ){ + rc = SQLITE_NOMEM; + } + OutputPointer_set_obj(env, S3JniNph(OutputPointer_sqlite3_value), + jOut, pWrap); + S3JniUnrefLocal(pWrap); + } + } + return rc; +#else + return SQLITE_MISUSE; +#endif +} + +S3JniApi(sqlite3_preupdate_new(),jint,1preupdate_1new)( + JniArgsEnvClass, jlong jpDb, jint iCol, jobject jOut +){ + return s3jni_preupdate_newold(env, 1, jpDb, iCol, jOut); +} + +S3JniApi(sqlite3_preupdate_old(),jint,1preupdate_1old)( + JniArgsEnvClass, jlong jpDb, jint iCol, jobject jOut +){ + return s3jni_preupdate_newold(env, 0, jpDb, iCol, jOut); +} + + +/* Central C-to-Java sqlite3_progress_handler() proxy. */ +static int s3jni_progress_handler_impl(void *pP){ + S3JniDb * const ps = (S3JniDb *)pP; + int rc = 0; + S3JniDeclLocal_env; + S3JniHook hook; + + S3JniHook_localdup(&ps->hooks.progress, &hook); + if( hook.jObj ){ + rc = (int)(*env)->CallIntMethod(env, hook.jObj, hook.midCallback); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, rc, + "sqlite3_progress_handler() callback threw"); + } + S3JniHook_localundup(hook); + } + return rc; +} + +S3JniApi(sqlite3_progress_handler(),void,1progress_1handler)( + JniArgsEnvClass,jobject jDb, jint n, jobject jProgress +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + S3JniHook * const pHook = ps ? &ps->hooks.progress : 0; + + if( !ps ) return; + S3JniDb_mutex_enter; + if( n<1 || !jProgress ){ + S3JniHook_unref(pHook); + sqlite3_progress_handler(ps->pDb, 0, 0, 0); + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jProgress); + jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", "()I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionClear; + s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching xCallback() on " + "ProgressHandler object."); + }else{ + S3JniUnrefGlobal(pHook->jObj); + pHook->midCallback = xCallback; + pHook->jObj = S3JniRefGlobal(jProgress); + sqlite3_progress_handler(ps->pDb, (int)n, s3jni_progress_handler_impl, ps); + } + } + S3JniDb_mutex_leave; +} + +S3JniApi(sqlite3_randomness(),void,1randomness)( + JniArgsEnvClass, jbyteArray jTgt +){ + jbyte * const jba = s3jni_jbyteArray_bytes(jTgt); + if( jba ){ + jsize const nTgt = (*env)->GetArrayLength(env, jTgt); + sqlite3_randomness( (int)nTgt, jba ); + s3jni_jbyteArray_commit(jTgt, jba); + } +} + + +S3JniApi(sqlite3_reset(),jint,1reset)( + JniArgsEnvClass, jobject jpStmt +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + return pStmt ? sqlite3_reset(pStmt) : SQLITE_MISUSE; +} + +/* Clears all entries from S3JniGlobal.autoExt. */ +static void s3jni_reset_auto_extension(JNIEnv *env){ + int i; + S3JniAutoExt_mutex_enter; + for( i = 0; i < SJG.autoExt.nExt; ++i ){ + S3JniAutoExtension_clear( &SJG.autoExt.aExt[i] ); + } + SJG.autoExt.nExt = 0; + S3JniAutoExt_mutex_leave; +} + +S3JniApi(sqlite3_reset_auto_extension(),void,1reset_1auto_1extension)( + JniArgsEnvClass +){ + s3jni_reset_auto_extension(env); +} + +/* Impl for sqlite3_result_text/blob() and friends. */ +static void result_blob_text(int as64 /* true for text64/blob64() mode */, + int eTextRep /* 0 for blobs, else SQLITE_UTF... */, + JNIEnv * const env, sqlite3_context *pCx, + jbyteArray jBa, jlong nMax){ + int const asBlob = 0==eTextRep; + if( !pCx ){ + /* We should arguably emit a warning here. But where to log it? */ + return; + }else if( jBa ){ + jbyte * const pBuf = s3jni_jbyteArray_bytes(jBa); + jsize nBA = (*env)->GetArrayLength(env, jBa); + if( nMax>=0 && nBA>(jsize)nMax ){ + nBA = (jsize)nMax; + /** + From the sqlite docs: + + > If the 3rd parameter to any of the sqlite3_result_text* + interfaces other than sqlite3_result_text64() is negative, + then SQLite computes the string length itself by searching + the 2nd parameter for the first zero character. + + Note that the text64() interfaces take an unsigned value for + the length, which Java does not support. This binding takes + the approach of passing on negative values to the C API, + which will in turn fail with SQLITE_TOOBIG at some later + point (recall that the sqlite3_result_xyz() family do not + have result values). + */ + } + if( as64 ){ /* 64-bit... */ + static const jsize nLimit64 = + SQLITE_MAX_ALLOCATION_SIZE/*only _kinda_ arbitrary*/; + if( nBA > nLimit64 ){ + sqlite3_result_error_toobig(pCx); + }else if( asBlob ){ + sqlite3_result_blob64(pCx, pBuf, (sqlite3_uint64)nBA, + SQLITE_TRANSIENT); + }else{ /* text64... */ + if( encodingTypeIsValid(eTextRep) ){ + sqlite3_result_text64(pCx, (const char *)pBuf, + (sqlite3_uint64)nBA, + SQLITE_TRANSIENT, eTextRep); + }else{ + sqlite3_result_error_code(pCx, SQLITE_FORMAT); + } + } + }else{ /* 32-bit... */ + static const jsize nLimit = SQLITE_MAX_ALLOCATION_SIZE; + if( nBA > nLimit ){ + sqlite3_result_error_toobig(pCx); + }else if( asBlob ){ + sqlite3_result_blob(pCx, pBuf, (int)nBA, + SQLITE_TRANSIENT); + }else{ + switch( eTextRep ){ + case SQLITE_UTF8: + sqlite3_result_text(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + case SQLITE_UTF16: + sqlite3_result_text16(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCx, (const char *)pBuf, (int)nBA, + SQLITE_TRANSIENT); + break; + } + } + s3jni_jbyteArray_release(jBa, pBuf); + } + }else{ + sqlite3_result_null(pCx); + } +} + +S3JniApi(sqlite3_result_blob(),void,1result_1blob)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jint nMax +){ + return result_blob_text(0, 0, env, PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_blob64(),void,1result_1blob64)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jlong nMax +){ + return result_blob_text(1, 0, env, PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_double(),void,1result_1double)( + JniArgsEnvClass, jobject jpCx, jdouble v +){ + sqlite3_result_double(PtrGet_sqlite3_context(jpCx), v); +} + +S3JniApi(sqlite3_result_error(),void,1result_1error)( + JniArgsEnvClass, jobject jpCx, jbyteArray baMsg, jint eTextRep +){ + const char * zUnspecified = "Unspecified error."; + jsize const baLen = (*env)->GetArrayLength(env, baMsg); + jbyte * const pjBuf = baMsg ? s3jni_jbyteArray_bytes(baMsg) : NULL; + switch( pjBuf ? eTextRep : SQLITE_UTF8 ){ + case SQLITE_UTF8: { + const char *zMsg = pjBuf ? (const char *)pjBuf : zUnspecified; + int const n = pjBuf ? (int)baLen : (int)sqlite3Strlen30(zMsg); + sqlite3_result_error(PtrGet_sqlite3_context(jpCx), zMsg, n); + break; + } + case SQLITE_UTF16: { + const void *zMsg = pjBuf; + sqlite3_result_error16(PtrGet_sqlite3_context(jpCx), zMsg, (int)baLen); + break; + } + default: + sqlite3_result_error(PtrGet_sqlite3_context(jpCx), + "Invalid encoding argument passed " + "to sqlite3_result_error().", -1); + break; + } + s3jni_jbyteArray_release(baMsg,pjBuf); +} + +S3JniApi(sqlite3_result_error_code(),void,1result_1error_1code)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_error_code(PtrGet_sqlite3_context(jpCx), (int)v); +} + +S3JniApi(sqlite3_result_error_nomem(),void,1result_1error_1nomem)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_result_error_nomem(PtrGet_sqlite3_context(jpCx)); +} + +S3JniApi(sqlite3_result_error_toobig(),void,1result_1error_1toobig)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_result_error_toobig(PtrGet_sqlite3_context(jpCx)); +} + +S3JniApi(sqlite3_result_int(),void,1result_1int)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_int(PtrGet_sqlite3_context(jpCx), (int)v); +} + +S3JniApi(sqlite3_result_int64(),void,1result_1int64)( + JniArgsEnvClass, jobject jpCx, jlong v +){ + sqlite3_result_int64(PtrGet_sqlite3_context(jpCx), (sqlite3_int64)v); +} + +S3JniApi(sqlite3_result_java_object(),void,1result_1java_1object)( + JniArgsEnvClass, jobject jpCx, jobject v +){ + sqlite3_context * pCx = PtrGet_sqlite3_context(jpCx); + if( !pCx ) return; + else if( v ){ + jobject const rjv = S3JniRefGlobal(v); + if( rjv ){ + sqlite3_result_pointer(pCx, rjv, + s3jni__value_jref_key, S3Jni_jobject_finalizer); + }else{ + sqlite3_result_error_nomem(PtrGet_sqlite3_context(jpCx)); + } + }else{ + sqlite3_result_null(PtrGet_sqlite3_context(jpCx)); + } +} + +S3JniApi(sqlite3_result_nio_buffer(),void,1result_1nio_1buffer)( + JniArgsEnvClass, jobject jpCtx, jobject jBuffer, + jint iOffset, jint iN +){ + sqlite3_context * pCx = PtrGet_sqlite3_context(jpCtx); + int rc; + S3JniNioArgs args; + if( !pCx ){ + return; + }else if( !SJG.g.byteBuffer.klazz ){ + sqlite3_result_error( + pCx, "This JVM does not support JNI access to ByteBuffers.", -1 + ); + return; + } + rc = s3jni_setup_nio_args(env, &args, jBuffer, iOffset, iN); + if(rc){ + if( iOffset<0 ){ + sqlite3_result_error(pCx, "Start index may not be negative.", -1); + }else if( SQLITE_TOOBIG==rc ){ + sqlite3_result_error_toobig(pCx); + }else{ + sqlite3_result_error( + pCx, "Invalid arguments to sqlite3_result_nio_buffer().", -1 + ); + } + }else if( !args.pStart || !args.nOut ){ + sqlite3_result_null(pCx); + }else{ + sqlite3_result_blob(pCx, args.pStart, args.nOut, SQLITE_TRANSIENT); + } +} + + +S3JniApi(sqlite3_result_null(),void,1result_1null)( + JniArgsEnvClass, jobject jpCx +){ + sqlite3_result_null(PtrGet_sqlite3_context(jpCx)); +} + +S3JniApi(sqlite3_result_subtype(),void,1result_1subtype)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_subtype(PtrGet_sqlite3_context(jpCx), (unsigned int)v); +} + + +S3JniApi(sqlite3_result_text(),void,1result_1text)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jint nMax +){ + return result_blob_text(0, SQLITE_UTF8, env, + PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_text64(),void,1result_1text64)( + JniArgsEnvClass, jobject jpCx, jbyteArray jBa, jlong nMax, + jint eTextRep +){ + return result_blob_text(1, eTextRep, env, + PtrGet_sqlite3_context(jpCx), jBa, nMax); +} + +S3JniApi(sqlite3_result_value(),void,1result_1value)( + JniArgsEnvClass, jobject jpCx, jobject jpSVal +){ + sqlite3_result_value(PtrGet_sqlite3_context(jpCx), + PtrGet_sqlite3_value(jpSVal)); +} + +S3JniApi(sqlite3_result_zeroblob(),void,1result_1zeroblob)( + JniArgsEnvClass, jobject jpCx, jint v +){ + sqlite3_result_zeroblob(PtrGet_sqlite3_context(jpCx), (int)v); +} + +S3JniApi(sqlite3_result_zeroblob64(),jint,1result_1zeroblob64)( + JniArgsEnvClass, jobject jpCx, jlong v +){ + return (jint)sqlite3_result_zeroblob64(PtrGet_sqlite3_context(jpCx), + (sqlite3_int64)v); +} + +S3JniApi(sqlite3_rollback_hook(),jobject,1rollback_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + return s3jni_commit_rollback_hook(0, env, jpDb, jHook); +} + +/* Callback for sqlite3_set_authorizer(). */ +int s3jni_xAuth(void* pState, int op,const char*z0, const char*z1, + const char*z2,const char*z3){ + S3JniDb * const ps = pState; + S3JniDeclLocal_env; + S3JniHook hook; + int rc = 0; + + S3JniHook_localdup(&ps->hooks.auth, &hook ); + if( hook.jObj ){ + jstring const s0 = z0 ? s3jni_utf8_to_jstring( z0, -1) : 0; + jstring const s1 = z1 ? s3jni_utf8_to_jstring( z1, -1) : 0; + jstring const s2 = z2 ? s3jni_utf8_to_jstring( z2, -1) : 0; + jstring const s3 = z3 ? s3jni_utf8_to_jstring( z3, -1) : 0; + + rc = (*env)->CallIntMethod(env, hook.jObj, hook.midCallback, (jint)op, + s0, s1, s3, s3); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, rc, "sqlite3_set_authorizer() callback"); + } + S3JniUnrefLocal(s0); + S3JniUnrefLocal(s1); + S3JniUnrefLocal(s2); + S3JniUnrefLocal(s3); + S3JniHook_localundup(hook); + } + return rc; +} + +S3JniApi(sqlite3_set_authorizer(),jint,1set_1authorizer)( + JniArgsEnvClass,jobject jDb, jobject jHook +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + S3JniHook * const pHook = ps ? &ps->hooks.auth : 0; + int rc = 0; + + if( !ps ) return SQLITE_MISUSE; + S3JniDb_mutex_enter; + if( !jHook ){ + S3JniHook_unref(pHook); + rc = sqlite3_set_authorizer( ps->pDb, 0, 0 ); + }else{ + jclass klazz; + if( pHook->jObj ){ + if( (*env)->IsSameObject(env, pHook->jObj, jHook) ){ + /* Same object - this is a no-op. */ + S3JniDb_mutex_leave; + return 0; + } + S3JniHook_unref(pHook); + } + pHook->jObj = S3JniRefGlobal(jHook); + klazz = (*env)->GetObjectClass(env, jHook); + pHook->midCallback = (*env)->GetMethodID(env, klazz, + "call", + "(I" + "Ljava/lang/String;" + "Ljava/lang/String;" + "Ljava/lang/String;" + "Ljava/lang/String;" + ")I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Error setting up Java parts of authorizer hook."); + }else{ + rc = sqlite3_set_authorizer(ps->pDb, s3jni_xAuth, ps); + } + if( rc ) S3JniHook_unref(pHook); + } + S3JniDb_mutex_leave; + return rc; +} + +S3JniApi(sqlite3_set_auxdata(),void,1set_1auxdata)( + JniArgsEnvClass, jobject jCx, jint n, jobject jAux +){ + sqlite3_set_auxdata(PtrGet_sqlite3_context(jCx), (int)n, + S3JniRefGlobal(jAux), S3Jni_jobject_finalizer); +} + +S3JniApi(sqlite3_set_last_insert_rowid(),void,1set_1last_1insert_1rowid)( + JniArgsEnvClass, jobject jpDb, jlong rowId +){ + sqlite3_set_last_insert_rowid(PtrGet_sqlite3(jpDb), + (sqlite3_int64)rowId); +} + +S3JniApi(sqlite3_shutdown(),jint,1shutdown)( + JniArgsEnvClass +){ + s3jni_reset_auto_extension(env); +#ifdef SQLITE_ENABLE_SQLLOG + S3JniHook_unref(&SJG.hook.sqllog); +#endif + S3JniHook_unref(&SJG.hook.configlog); + /* Free up S3JniDb recycling bin. */ + S3JniDb_mutex_enter; { + while( S3JniGlobal.perDb.aFree ){ + S3JniDb * const d = S3JniGlobal.perDb.aFree; + S3JniGlobal.perDb.aFree = d->pNext; + S3JniDb_clear(env, d); + sqlite3_free(d); + } + } S3JniDb_mutex_leave; + S3JniGlobal_mutex_enter; { + /* Free up S3JniUdf recycling bin. */ + while( S3JniGlobal.udf.aFree ){ + S3JniUdf * const u = S3JniGlobal.udf.aFree; + S3JniGlobal.udf.aFree = u->pNext; + u->pNext = 0; + S3JniUdf_free(env, u, 0); + } + } S3JniGlobal_mutex_leave; + S3JniHook_mutex_enter; { + /* Free up S3JniHook recycling bin. */ + while( S3JniGlobal.hook.aFree ){ + S3JniHook * const u = S3JniGlobal.hook.aFree; + S3JniGlobal.hook.aFree = u->pNext; + u->pNext = 0; + assert( !u->doXDestroy ); + assert( !u->jObj ); + assert( !u->jExtra ); + sqlite3_free( u ); + } + } S3JniHook_mutex_leave; + /* Free up env cache. */ + S3JniEnv_mutex_enter; { + while( SJG.envCache.aHead ){ + S3JniEnv_uncache( SJG.envCache.aHead->env ); + } + } S3JniEnv_mutex_leave; + /* Do not clear S3JniGlobal.jvm or S3JniGlobal.g: it's legal to + ** restart the lib. */ + return sqlite3_shutdown(); +} + +S3JniApi(sqlite3_status(),jint,1status)( + JniArgsEnvClass, jint op, jobject jOutCurrent, jobject jOutHigh, + jboolean reset +){ + int iCur = 0, iHigh = 0; + int rc = sqlite3_status( op, &iCur, &iHigh, reset ); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCurrent, iCur); + OutputPointer_set_Int32(env, jOutHigh, iHigh); + } + return (jint)rc; +} + +S3JniApi(sqlite3_status64(),jint,1status64)( + JniArgsEnvClass, jint op, jobject jOutCurrent, jobject jOutHigh, + jboolean reset +){ + sqlite3_int64 iCur = 0, iHigh = 0; + int rc = sqlite3_status64( op, &iCur, &iHigh, reset ); + if( 0==rc ){ + OutputPointer_set_Int64(env, jOutCurrent, iCur); + OutputPointer_set_Int64(env, jOutHigh, iHigh); + } + return (jint)rc; +} + +S3JniApi(sqlite3_stmt_status(),jint,1stmt_1status)( + JniArgsEnvClass, jobject jStmt, jint op, jboolean reset +){ + return sqlite3_stmt_status(PtrGet_sqlite3_stmt(jStmt), + (int)op, reset ? 1 : 0); +} + + +static int s3jni_strlike_glob(int isLike, JNIEnv *const env, + jbyteArray baG, jbyteArray baT, jint escLike){ + int rc = 0; + jbyte * const pG = s3jni_jbyteArray_bytes(baG); + jbyte * const pT = s3jni_jbyteArray_bytes(baT); + + /* Note that we're relying on the byte arrays having been + NUL-terminated on the Java side. */ + rc = isLike + ? sqlite3_strlike((const char *)pG, (const char *)pT, + (unsigned int)escLike) + : sqlite3_strglob((const char *)pG, (const char *)pT); + s3jni_jbyteArray_release(baG, pG); + s3jni_jbyteArray_release(baT, pT); + return rc; +} + +S3JniApi(sqlite3_strglob(),jint,1strglob)( + JniArgsEnvClass, jbyteArray baG, jbyteArray baT +){ + return s3jni_strlike_glob(0, env, baG, baT, 0); +} + +S3JniApi(sqlite3_strlike(),jint,1strlike)( + JniArgsEnvClass, jbyteArray baG, jbyteArray baT, jint escChar +){ + return s3jni_strlike_glob(1, env, baG, baT, escChar); +} + +S3JniApi(sqlite3_sql(),jstring,1sql)( + JniArgsEnvClass, jobject jpStmt +){ + sqlite3_stmt * const pStmt = PtrGet_sqlite3_stmt(jpStmt); + jstring rv = 0; + if( pStmt ){ + const char * zSql = 0; + zSql = sqlite3_sql(pStmt); + rv = s3jni_utf8_to_jstring( zSql, -1); + } + return rv; +} + +S3JniApi(sqlite3_step(),jint,1step)( + JniArgsEnvClass, jlong jpStmt +){ + sqlite3_stmt * const pStmt = LongPtrGet_sqlite3_stmt(jpStmt); + return pStmt ? (jint)sqlite3_step(pStmt) : (jint)SQLITE_MISUSE; +} + +S3JniApi(sqlite3_table_column_metadata(),jint,1table_1column_1metadata)( + JniArgsEnvClass, jobject jDb, jstring jDbName, jstring jTableName, + jstring jColumnName, jobject jDataType, jobject jCollSeq, jobject jNotNull, + jobject jPrimaryKey, jobject jAutoinc +){ + sqlite3 * const db = PtrGet_sqlite3(jDb); + char * zDbName = 0, * zTableName = 0, * zColumnName = 0; + const char * pzCollSeq = 0; + const char * pzDataType = 0; + int pNotNull = 0, pPrimaryKey = 0, pAutoinc = 0; + int rc; + + if( !db || !jDbName || !jTableName ) return SQLITE_MISUSE; + zDbName = s3jni_jstring_to_utf8(jDbName,0); + zTableName = zDbName ? s3jni_jstring_to_utf8(jTableName,0) : 0; + zColumnName = (zTableName && jColumnName) + ? s3jni_jstring_to_utf8(jColumnName,0) : 0; + rc = zTableName + ? sqlite3_table_column_metadata(db, zDbName, zTableName, + zColumnName, &pzDataType, &pzCollSeq, + &pNotNull, &pPrimaryKey, &pAutoinc) + : SQLITE_NOMEM; + if( 0==rc ){ + jstring jseq = jCollSeq + ? (pzCollSeq ? s3jni_utf8_to_jstring(pzCollSeq, -1) : 0) + : 0; + jstring jdtype = jDataType + ? (pzDataType ? s3jni_utf8_to_jstring(pzDataType, -1) : 0) + : 0; + if( (jCollSeq && pzCollSeq && !jseq) + || (jDataType && pzDataType && !jdtype) ){ + rc = SQLITE_NOMEM; + }else{ + if( jNotNull ) OutputPointer_set_Bool(env, jNotNull, pNotNull); + if( jPrimaryKey ) OutputPointer_set_Bool(env, jPrimaryKey, pPrimaryKey); + if( jAutoinc ) OutputPointer_set_Bool(env, jAutoinc, pAutoinc); + if( jCollSeq ) OutputPointer_set_String(env, jCollSeq, jseq); + if( jDataType ) OutputPointer_set_String(env, jDataType, jdtype); + } + S3JniUnrefLocal(jseq); + S3JniUnrefLocal(jdtype); + } + sqlite3_free(zDbName); + sqlite3_free(zTableName); + sqlite3_free(zColumnName); + return rc; +} + +static int s3jni_trace_impl(unsigned traceflag, void *pC, void *pP, void *pX){ + S3JniDb * const ps = (S3JniDb *)pC; + S3JniDeclLocal_env; + jobject jX = NULL /* the tracer's X arg */; + jobject jP = NULL /* the tracer's P arg */; + jobject jPUnref = NULL /* potentially a local ref to jP */; + int rc = 0; + S3JniHook hook; + + S3JniHook_localdup(&ps->hooks.trace, &hook ); + if( !hook.jObj ){ + return 0; + } + switch( traceflag ){ + case SQLITE_TRACE_STMT: + jX = s3jni_utf8_to_jstring( (const char *)pX, -1); + if( !jX ) rc = SQLITE_NOMEM; + break; + case SQLITE_TRACE_PROFILE: + jX = (*env)->NewObject(env, SJG.g.cLong, SJG.g.ctorLong1, + (jlong)*((sqlite3_int64*)pX)); + // hmm. ^^^ (*pX) really is zero. + // MARKER(("profile time = %llu\n", *((sqlite3_int64*)pX))); + s3jni_oom_check( jX ); + if( !jX ) rc = SQLITE_NOMEM; + break; + case SQLITE_TRACE_ROW: + break; + case SQLITE_TRACE_CLOSE: + jP = jPUnref = S3JniRefLocal(ps->jDb); + break; + default: + assert(!"cannot happen - unknown trace flag"); + rc = SQLITE_ERROR; + } + if( 0==rc ){ + if( !jP ){ + /* Create a new temporary sqlite3_stmt wrapper */ + jP = jPUnref = new_java_sqlite3_stmt(env, pP); + if( !jP ){ + rc = SQLITE_NOMEM; + } + } + if( 0==rc ){ + assert(jP); + rc = (int)(*env)->CallIntMethod(env, hook.jObj, hook.midCallback, + (jint)traceflag, jP, jX); + S3JniIfThrew{ + rc = s3jni_db_exception(ps->pDb, SQLITE_ERROR, + "sqlite3_trace_v2() callback threw."); + } + } + } + S3JniUnrefLocal(jPUnref); + S3JniUnrefLocal(jX); + S3JniHook_localundup(hook); + return rc; +} + +S3JniApi(sqlite3_trace_v2(),jint,1trace_1v2)( + JniArgsEnvClass,jobject jDb, jint traceMask, jobject jTracer +){ + S3JniDb * const ps = S3JniDb_from_java(jDb); + int rc; + + if( !ps ) return SQLITE_MISUSE; + if( !traceMask || !jTracer ){ + S3JniDb_mutex_enter; + rc = (jint)sqlite3_trace_v2(ps->pDb, 0, 0, 0); + S3JniHook_unref(&ps->hooks.trace); + S3JniDb_mutex_leave; + }else{ + jclass const klazz = (*env)->GetObjectClass(env, jTracer); + S3JniHook hook = S3JniHook_empty; + hook.midCallback = (*env)->GetMethodID( + env, klazz, "call", "(ILjava/lang/Object;Ljava/lang/Object;)I" + ); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionClear; + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Cannot not find matching call() on " + "TracerCallback object."); + }else{ + hook.jObj = S3JniRefGlobal(jTracer); + S3JniDb_mutex_enter; + rc = sqlite3_trace_v2(ps->pDb, (unsigned)traceMask, s3jni_trace_impl, ps); + if( 0==rc ){ + S3JniHook_unref(&ps->hooks.trace); + ps->hooks.trace = hook /* transfer ownership of reference */; + }else{ + S3JniHook_unref(&hook); + } + S3JniDb_mutex_leave; + } + } + return rc; +} + +S3JniApi(sqlite3_txn_state(),jint,1txn_1state)( + JniArgsEnvClass,jobject jDb, jstring jSchema +){ + sqlite3 * const pDb = PtrGet_sqlite3(jDb); + int rc = SQLITE_MISUSE; + if( pDb ){ + char * zSchema = jSchema + ? s3jni_jstring_to_utf8(jSchema, 0) + : 0; + if( !jSchema || (zSchema && jSchema) ){ + rc = sqlite3_txn_state(pDb, zSchema); + sqlite3_free(zSchema); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +S3JniApi(sqlite3_update_hook(),jobject,1update_1hook)( + JniArgsEnvClass, jlong jpDb, jobject jHook +){ + return s3jni_updatepre_hook(env, 0, jpDb, jHook); +} + + +S3JniApi(sqlite3_value_blob(),jbyteArray,1value_1blob)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const jbyte * pBytes = sv ? sqlite3_value_blob(sv) : 0; + int const nLen = pBytes ? sqlite3_value_bytes(sv) : 0; + + s3jni_oom_check( nLen ? !!pBytes : 1 ); + return pBytes + ? s3jni_new_jbyteArray(pBytes, nLen) + : NULL; +} + +S3JniApi(sqlite3_value_bytes(),jint,1value_1bytes)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return sv ? sqlite3_value_bytes(sv) : 0; +} + +S3JniApi(sqlite3_value_bytes16(),jint,1value_1bytes16)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return sv ? sqlite3_value_bytes16(sv) : 0; +} + + +S3JniApi(sqlite3_value_double(),jdouble,1value_1double)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return (jdouble) (sv ? sqlite3_value_double(sv) : 0.0); +} + + +S3JniApi(sqlite3_value_dup(),jobject,1value_1dup)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + sqlite3_value * const sd = sv ? sqlite3_value_dup(sv) : 0; + jobject rv = sd ? new_java_sqlite3_value(env, sd) : 0; + if( sd && !rv ) { + /* OOM */ + sqlite3_value_free(sd); + } + return rv; +} + +S3JniApi(sqlite3_value_free(),void,1value_1free)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + if( sv ){ + sqlite3_value_free(sv); + } +} + +S3JniApi(sqlite3_value_int(),jint,1value_1int)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return (jint) (sv ? sqlite3_value_int(sv) : 0); +} + +S3JniApi(sqlite3_value_int64(),jlong,1value_1int64)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return (jlong) (sv ? sqlite3_value_int64(sv) : 0LL); +} + +S3JniApi(sqlite3_value_java_object(),jobject,1value_1java_1object)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + return sv + ? sqlite3_value_pointer(sv, s3jni__value_jref_key) + : 0; +} + +S3JniApi(sqlite3_value_nio_buffer(),jobject,1value_1nio_1buffer)( + JniArgsEnvClass, jobject jVal +){ + sqlite3_value * const sv = PtrGet_sqlite3_value(jVal); + jobject rv = 0; + if( sv ){ + const void * const p = sqlite3_value_blob(sv); + if( p ){ + const int n = sqlite3_value_bytes(sv); + rv = s3jni__blob_to_ByteBuffer(env, p, n); + } + } + return rv; +} + +S3JniApi(sqlite3_value_text(),jbyteArray,1value_1text)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const unsigned char * const p = sv ? sqlite3_value_text(sv) : 0; + int const n = p ? sqlite3_value_bytes(sv) : 0; + return p ? s3jni_new_jbyteArray(p, n) : 0; +} + +#if 0 +// this impl might prove useful. +S3JniApi(sqlite3_value_text(),jstring,1value_1text)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const unsigned char * const p = sv ? sqlite3_value_text(sv) : 0; + int const n = p ? sqlite3_value_bytes(sv) : 0; + return p ? s3jni_utf8_to_jstring( (const char *)p, n) : 0; +} +#endif + +S3JniApi(sqlite3_value_text16(),jstring,1value_1text16)( + JniArgsEnvClass, jlong jpSVal +){ + sqlite3_value * const sv = LongPtrGet_sqlite3_value(jpSVal); + const int n = sv ? sqlite3_value_bytes16(sv) : 0; + const void * const p = sv ? sqlite3_value_text16(sv) : 0; + return p ? s3jni_text16_to_jstring(env, p, n) : 0; +} + +JniDecl(void,1jni_1internal_1details)(JniArgsEnvClass){ + MARKER(("\nVarious bits of internal info:\n")); + puts("FTS5 is " +#ifdef SQLITE_ENABLE_FTS5 + "available" +#else + "unavailable" +#endif + "." + ); + puts("sizeofs:"); +#define SO(T) printf("\tsizeof(" #T ") = %u\n", (unsigned)sizeof(T)) + SO(void*); + SO(jmethodID); + SO(jfieldID); + SO(S3JniEnv); + SO(S3JniHook); + SO(S3JniDb); + SO(S3JniNphOps); + printf("\t(^^^ %u NativePointerHolder/OutputPointer.T types)\n", + (unsigned)S3Jni_NphCache_size); + SO(S3JniGlobal); + SO(S3JniGlobal.nph); + SO(S3JniGlobal.metrics); + SO(S3JniAutoExtension); + SO(S3JniUdf); +#undef SO +#ifdef SQLITE_JNI_ENABLE_METRICS + printf("Cache info:\n"); + printf("\tJNIEnv cache: %u allocs, %u misses, %u hits\n", + SJG.metrics.nEnvAlloc, SJG.metrics.nEnvMiss, + SJG.metrics.nEnvHit); + printf("Mutex entry:" + "\n\tglobal = %u" + "\n\tenv = %u" + "\n\tnph = %u for S3JniNphOp init" + "\n\thook = %u" + "\n\tperDb = %u" + "\n\tautoExt list = %u" + "\n\tS3JniUdf = %u (free-list)" + "\n\tmetrics = %u\n", + SJG.metrics.nMutexGlobal, SJG.metrics.nMutexEnv, + SJG.metrics.nMutexNph, SJG.metrics.nMutexHook, + SJG.metrics.nMutexPerDb, SJG.metrics.nMutexAutoExt, + SJG.metrics.nMutexUdf, SJG.metrics.nMetrics); + puts("Allocs:"); + printf("\tS3JniDb: %u alloced (*%u = %u bytes), %u recycled\n", + SJG.metrics.nPdbAlloc, (unsigned) sizeof(S3JniDb), + (unsigned)(SJG.metrics.nPdbAlloc * sizeof(S3JniDb)), + SJG.metrics.nPdbRecycled); + printf("\tS3JniUdf: %u alloced (*%u = %u bytes), %u recycled\n", + SJG.metrics.nUdfAlloc, (unsigned) sizeof(S3JniUdf), + (unsigned)(SJG.metrics.nUdfAlloc * sizeof(S3JniUdf)), + SJG.metrics.nUdfRecycled); + printf("\tS3JniHook: %u alloced (*%u = %u bytes), %u recycled\n", + SJG.metrics.nHookAlloc, (unsigned) sizeof(S3JniHook), + (unsigned)(SJG.metrics.nHookAlloc * sizeof(S3JniHook)), + SJG.metrics.nHookRecycled); + printf("\tS3JniEnv: %u alloced (*%u = %u bytes)\n", + SJG.metrics.nEnvAlloc, (unsigned) sizeof(S3JniEnv), + (unsigned)(SJG.metrics.nEnvAlloc * sizeof(S3JniEnv))); + puts("Java-side UDF calls:"); +#define UDF(T) printf("\t%-8s = %u\n", "x" #T, SJG.metrics.udf.n##T) + UDF(Func); UDF(Step); UDF(Final); UDF(Value); UDF(Inverse); +#undef UDF + printf("xDestroy calls across all callback types: %u\n", + SJG.metrics.nDestroy); +#else + puts("Built without SQLITE_JNI_ENABLE_METRICS."); +#endif +} + +//////////////////////////////////////////////////////////////////////// +// End of the sqlite3_... API bindings. Next up, FTS5... +//////////////////////////////////////////////////////////////////////// +#ifdef SQLITE_ENABLE_FTS5 + +/* Creates a verbose JNI Fts5 function name. */ +#define JniFuncNameFtsXA(Suffix) \ + Java_org_sqlite_jni_fts5_Fts5ExtensionApi_ ## Suffix +#define JniFuncNameFtsApi(Suffix) \ + Java_org_sqlite_jni_fts5_fts5_1api_ ## Suffix +#define JniFuncNameFtsTok(Suffix) \ + Java_org_sqlite_jni_fts5_fts5_tokenizer_ ## Suffix + +#define JniDeclFtsXA(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JniFuncNameFtsXA(Suffix) +#define JniDeclFtsApi(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JniFuncNameFtsApi(Suffix) +#define JniDeclFtsTok(ReturnType,Suffix) \ + JNIEXPORT ReturnType JNICALL \ + JniFuncNameFtsTok(Suffix) + +#define PtrGet_fts5_api(OBJ) NativePointerHolder_get(OBJ,S3JniNph(fts5_api)) +#define PtrGet_fts5_tokenizer(OBJ) NativePointerHolder_get(OBJ,S3JniNph(fts5_tokenizer)) +#define PtrGet_Fts5Context(OBJ) NativePointerHolder_get(OBJ,S3JniNph(Fts5Context)) +#define PtrGet_Fts5Tokenizer(OBJ) NativePointerHolder_get(OBJ,S3JniNph(Fts5Tokenizer)) +#define s3jni_ftsext() &sFts5Api/*singleton from sqlite3.c*/ +#define Fts5ExtDecl Fts5ExtensionApi const * const ext = s3jni_ftsext() + +/** + State for binding Java-side FTS5 auxiliary functions. +*/ +typedef struct { + jobject jObj /* functor instance */; + jobject jUserData /* 2nd arg to JNI binding of + xCreateFunction(), ostensibly the 3rd arg + to the lib-level xCreateFunction(), except + that we necessarily use that slot for a + Fts5JniAux instance. */; + char * zFuncName /* Only for error reporting and debug logging */; + jmethodID jmid /* callback member's method ID */; +} Fts5JniAux; + +static void Fts5JniAux_free(Fts5JniAux * const s){ + S3JniDeclLocal_env; + if( env ){ + /*MARKER(("FTS5 aux function cleanup: %s\n", s->zFuncName));*/ + s3jni_call_xDestroy(s->jObj); + S3JniUnrefGlobal(s->jObj); + S3JniUnrefGlobal(s->jUserData); + } + sqlite3_free(s->zFuncName); + sqlite3_free(s); +} + +static void Fts5JniAux_xDestroy(void *p){ + if( p ) Fts5JniAux_free(p); +} + +static Fts5JniAux * Fts5JniAux_alloc(JNIEnv * const env, jobject jObj){ + Fts5JniAux * s = s3jni_malloc( sizeof(Fts5JniAux)); + + if( s ){ + jclass klazz; + memset(s, 0, sizeof(Fts5JniAux)); + s->jObj = S3JniRefGlobal(jObj); + klazz = (*env)->GetObjectClass(env, jObj); + s->jmid = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/fts5/Fts5ExtensionApi;" + "Lorg/sqlite/jni/fts5/Fts5Context;" + "Lorg/sqlite/jni/capi/sqlite3_context;" + "[Lorg/sqlite/jni/capi/sqlite3_value;)V"); + S3JniUnrefLocal(klazz); + S3JniIfThrew{ + S3JniExceptionReport; + S3JniExceptionClear; + Fts5JniAux_free(s); + s = 0; + } + } + return s; +} + +static inline jobject new_java_Fts5Context(JNIEnv * const env, Fts5Context *sv){ + return NativePointerHolder_new(env, S3JniNph(Fts5Context), sv); +} +static inline jobject new_java_fts5_api(JNIEnv * const env, fts5_api *sv){ + return NativePointerHolder_new(env, S3JniNph(fts5_api), sv); +} + +/* +** Returns a per-JNIEnv global ref to the Fts5ExtensionApi singleton +** instance, or NULL on OOM. +*/ +static jobject s3jni_getFts5ExtensionApi(JNIEnv * const env){ + if( !SJG.fts5.jExt ){ + S3JniGlobal_mutex_enter; + if( !SJG.fts5.jExt ){ + jobject const pNPH = NativePointerHolder_new( + env, S3JniNph(Fts5ExtensionApi), s3jni_ftsext() + ); + if( pNPH ){ + SJG.fts5.jExt = S3JniRefGlobal(pNPH); + S3JniUnrefLocal(pNPH); + } + } + S3JniGlobal_mutex_leave; + } + return SJG.fts5.jExt; +} + +/* +** Returns a pointer to the fts5_api instance for database connection +** db. If an error occurs, returns NULL and leaves an error in the +** database handle (accessible using sqlite3_errcode()/errmsg()). +*/ +static fts5_api *s3jni_fts5_api_from_db(sqlite3 *db){ + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0) ){ + sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", NULL); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +JniDeclFtsApi(jobject,getInstanceForDb)(JniArgsEnvClass,jobject jDb){ + S3JniDb * const ps = S3JniDb_from_java(jDb); +#if 0 + jobject rv = 0; + if( !ps ) return 0; + else if( ps->fts.jApi ){ + rv = ps->fts.jApi; + }else{ + fts5_api * const pApi = s3jni_fts5_api_from_db(ps->pDb); + if( pApi ){ + rv = new_java_fts5_api(env, pApi); + ps->fts.jApi = rv ? S3JniRefGlobal(rv) : 0; + } + } + return rv; +#else + if( ps && !ps->fts.jApi ){ + S3JniDb_mutex_enter; + if( !ps->fts.jApi ){ + fts5_api * const pApi = s3jni_fts5_api_from_db(ps->pDb); + if( pApi ){ + jobject const rv = new_java_fts5_api(env, pApi); + ps->fts.jApi = rv ? S3JniRefGlobal(rv) : 0; + } + } + S3JniDb_mutex_leave; + } + return ps ? ps->fts.jApi : 0; +#endif +} + + +JniDeclFtsXA(jobject,getInstance)(JniArgsEnvClass){ + return s3jni_getFts5ExtensionApi(env); +} + +JniDeclFtsXA(jint,xColumnCount)(JniArgsEnvObj,jobject jCtx){ + Fts5ExtDecl; + return (jint)ext->xColumnCount(PtrGet_Fts5Context(jCtx)); +} + +JniDeclFtsXA(jint,xColumnSize)(JniArgsEnvObj,jobject jCtx, jint iIdx, jobject jOut32){ + Fts5ExtDecl; + int n1 = 0; + int const rc = ext->xColumnSize(PtrGet_Fts5Context(jCtx), (int)iIdx, &n1); + if( 0==rc ) OutputPointer_set_Int32(env, jOut32, n1); + return rc; +} + +JniDeclFtsXA(jint,xColumnText)(JniArgsEnvObj,jobject jCtx, jint iCol, + jobject jOut){ + Fts5ExtDecl; + const char *pz = 0; + int pn = 0; + int rc = ext->xColumnText(PtrGet_Fts5Context(jCtx), (int)iCol, + &pz, &pn); + if( 0==rc ){ + jstring jstr = pz ? s3jni_utf8_to_jstring( pz, pn) : 0; + if( pz ){ + if( jstr ){ + OutputPointer_set_String(env, jOut, jstr); + S3JniUnrefLocal(jstr)/*jOut has a reference*/; + }else{ + rc = SQLITE_NOMEM; + } + } + } + return (jint)rc; +} + +JniDeclFtsXA(jint,xColumnTotalSize)(JniArgsEnvObj,jobject jCtx, jint iCol, jobject jOut64){ + Fts5ExtDecl; + sqlite3_int64 nOut = 0; + int const rc = ext->xColumnTotalSize(PtrGet_Fts5Context(jCtx), (int)iCol, &nOut); + if( 0==rc && jOut64 ) OutputPointer_set_Int64(env, jOut64, (jlong)nOut); + return (jint)rc; +} + +/* +** Proxy for fts5_extension_function instances plugged in via +** fts5_api::xCreateFunction(). +*/ +static void s3jni_fts5_extension_function(Fts5ExtensionApi const *pApi, + Fts5Context *pFts, + sqlite3_context *pCx, + int argc, + sqlite3_value **argv){ + Fts5JniAux * const pAux = pApi->xUserData(pFts); + jobject jpCx = 0; + jobjectArray jArgv = 0; + jobject jpFts = 0; + jobject jFXA; + int rc; + S3JniDeclLocal_env; + + assert(pAux); + jFXA = s3jni_getFts5ExtensionApi(env); + if( !jFXA ) goto error_oom; + jpFts = new_java_Fts5Context(env, pFts); + if( !jpFts ) goto error_oom; + rc = udf_args(env, pCx, argc, argv, &jpCx, &jArgv); + if( rc ) goto error_oom; + (*env)->CallVoidMethod(env, pAux->jObj, pAux->jmid, + jFXA, jpFts, jpCx, jArgv); + S3JniIfThrew{ + udf_report_exception(env, 1, pCx, pAux->zFuncName, "call"); + } + udf_unargs(env, jpCx, argc, jArgv); + S3JniUnrefLocal(jpFts); + S3JniUnrefLocal(jpCx); + S3JniUnrefLocal(jArgv); + return; +error_oom: + s3jni_db_oom( sqlite3_context_db_handle(pCx) ); + assert( !jArgv ); + assert( !jpCx ); + S3JniUnrefLocal(jpFts); + sqlite3_result_error_nomem(pCx); + return; +} + +JniDeclFtsApi(jint,xCreateFunction)(JniArgsEnvObj, jstring jName, + jobject jUserData, jobject jFunc){ + fts5_api * const pApi = PtrGet_fts5_api(jSelf); + int rc; + char * zName; + Fts5JniAux * pAux; + + assert(pApi); + zName = s3jni_jstring_to_utf8( jName, 0); + if(!zName) return SQLITE_NOMEM; + pAux = Fts5JniAux_alloc(env, jFunc); + if( pAux ){ + rc = pApi->xCreateFunction(pApi, zName, pAux, + s3jni_fts5_extension_function, + Fts5JniAux_xDestroy); + }else{ + rc = SQLITE_NOMEM; + } + if( 0==rc ){ + pAux->jUserData = jUserData ? S3JniRefGlobal(jUserData) : 0; + pAux->zFuncName = zName; + }else{ + sqlite3_free(zName); + } + return (jint)rc; +} + + +typedef struct S3JniFts5AuxData S3JniFts5AuxData; +/* +** TODO: this middle-man struct is no longer necessary. Consider +** removing it and passing around jObj itself instead. +*/ +struct S3JniFts5AuxData { + jobject jObj; +}; + +static void S3JniFts5AuxData_xDestroy(void *x){ + if( x ){ + S3JniFts5AuxData * const p = x; + if( p->jObj ){ + S3JniDeclLocal_env; + s3jni_call_xDestroy(p->jObj); + S3JniUnrefGlobal(p->jObj); + } + sqlite3_free(x); + } +} + +JniDeclFtsXA(jobject,xGetAuxdata)(JniArgsEnvObj,jobject jCtx, jboolean bClear){ + Fts5ExtDecl; + jobject rv = 0; + S3JniFts5AuxData * const pAux = ext->xGetAuxdata(PtrGet_Fts5Context(jCtx), bClear); + if( pAux ){ + if( bClear ){ + if( pAux->jObj ){ + rv = S3JniRefLocal(pAux->jObj); + S3JniUnrefGlobal(pAux->jObj); + } + /* Note that we do not call xDestroy() in this case. */ + sqlite3_free(pAux); + }else{ + rv = pAux->jObj; + } + } + return rv; +} + +JniDeclFtsXA(jint,xInst)(JniArgsEnvObj,jobject jCtx, jint iIdx, jobject jOutPhrase, + jobject jOutCol, jobject jOutOff){ + Fts5ExtDecl; + int n1 = 0, n2 = 2, n3 = 0; + int const rc = ext->xInst(PtrGet_Fts5Context(jCtx), (int)iIdx, &n1, &n2, &n3); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutPhrase, n1); + OutputPointer_set_Int32(env, jOutCol, n2); + OutputPointer_set_Int32(env, jOutOff, n3); + } + return rc; +} + +JniDeclFtsXA(jint,xInstCount)(JniArgsEnvObj,jobject jCtx, jobject jOut32){ + Fts5ExtDecl; + int nOut = 0; + int const rc = ext->xInstCount(PtrGet_Fts5Context(jCtx), &nOut); + if( 0==rc && jOut32 ) OutputPointer_set_Int32(env, jOut32, nOut); + return (jint)rc; +} + +JniDeclFtsXA(jint,xPhraseCount)(JniArgsEnvObj,jobject jCtx){ + Fts5ExtDecl; + return (jint)ext->xPhraseCount(PtrGet_Fts5Context(jCtx)); +} + +/* Copy the 'a' and 'b' fields from pSrc to Fts5PhraseIter object jIter. */ +static void s3jni_phraseIter_NToJ(JNIEnv *const env, + Fts5PhraseIter const * const pSrc, + jobject jIter){ + S3JniGlobalType * const g = &S3JniGlobal; + assert(g->fts5.jPhraseIter.fidA); + (*env)->SetLongField(env, jIter, g->fts5.jPhraseIter.fidA, + S3JniCast_P2L(pSrc->a)); + S3JniExceptionIsFatal("Cannot set Fts5PhraseIter.a field."); + (*env)->SetLongField(env, jIter, g->fts5.jPhraseIter.fidB, + S3JniCast_P2L(pSrc->b)); + S3JniExceptionIsFatal("Cannot set Fts5PhraseIter.b field."); +} + +/* Copy the 'a' and 'b' fields from Fts5PhraseIter object jIter to pDest. */ +static void s3jni_phraseIter_JToN(JNIEnv *const env, jobject jIter, + Fts5PhraseIter * const pDest){ + S3JniGlobalType * const g = &S3JniGlobal; + assert(g->fts5.jPhraseIter.fidA); + pDest->a = S3JniCast_L2P( + (*env)->GetLongField(env, jIter, g->fts5.jPhraseIter.fidA) + ); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.a field."); + pDest->b = S3JniCast_L2P( + (*env)->GetLongField(env, jIter, g->fts5.jPhraseIter.fidB) + ); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.b field."); +} + +JniDeclFtsXA(jint,xPhraseFirst)(JniArgsEnvObj,jobject jCtx, jint iPhrase, + jobject jIter, jobject jOutCol, + jobject jOutOff){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int rc, iCol = 0, iOff = 0; + rc = ext->xPhraseFirst(PtrGet_Fts5Context(jCtx), (int)iPhrase, + &iter, &iCol, &iOff); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCol, iCol); + OutputPointer_set_Int32(env, jOutOff, iOff); + s3jni_phraseIter_NToJ(env, &iter, jIter); + } + return rc; +} + +JniDeclFtsXA(jint,xPhraseFirstColumn)(JniArgsEnvObj,jobject jCtx, jint iPhrase, + jobject jIter, jobject jOutCol){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int rc, iCol = 0; + rc = ext->xPhraseFirstColumn(PtrGet_Fts5Context(jCtx), (int)iPhrase, + &iter, &iCol); + if( 0==rc ){ + OutputPointer_set_Int32(env, jOutCol, iCol); + s3jni_phraseIter_NToJ(env, &iter, jIter); + } + return rc; +} + +JniDeclFtsXA(void,xPhraseNext)(JniArgsEnvObj,jobject jCtx, jobject jIter, + jobject jOutCol, jobject jOutOff){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int iCol = 0, iOff = 0; + s3jni_phraseIter_JToN(env, jIter, &iter); + ext->xPhraseNext(PtrGet_Fts5Context(jCtx), &iter, &iCol, &iOff); + OutputPointer_set_Int32(env, jOutCol, iCol); + OutputPointer_set_Int32(env, jOutOff, iOff); + s3jni_phraseIter_NToJ(env, &iter, jIter); +} + +JniDeclFtsXA(void,xPhraseNextColumn)(JniArgsEnvObj,jobject jCtx, jobject jIter, + jobject jOutCol){ + Fts5ExtDecl; + Fts5PhraseIter iter; + int iCol = 0; + s3jni_phraseIter_JToN(env, jIter, &iter); + ext->xPhraseNextColumn(PtrGet_Fts5Context(jCtx), &iter, &iCol); + OutputPointer_set_Int32(env, jOutCol, iCol); + s3jni_phraseIter_NToJ(env, &iter, jIter); +} + + +JniDeclFtsXA(jint,xPhraseSize)(JniArgsEnvObj,jobject jCtx, jint iPhrase){ + Fts5ExtDecl; + return (jint)ext->xPhraseSize(PtrGet_Fts5Context(jCtx), (int)iPhrase); +} + +/* State for use with xQueryPhrase() and xTokenize(). */ +struct s3jni_xQueryPhraseState { + Fts5ExtensionApi const * ext; + jmethodID midCallback; /* jCallback->call() method */ + jobject jCallback; /* Fts5ExtensionApi.XQueryPhraseCallback instance */ + jobject jFcx; /* (Fts5Context*) for xQueryPhrase() + callback. This is NOT the instance that is + passed to xQueryPhrase(), it's the one + created by xQueryPhrase() for use by its + callback. */ + /* State for xTokenize() */ + struct { + const char * zPrev; + int nPrev; + jbyteArray jba; + } tok; +}; + +static int s3jni_xQueryPhrase(const Fts5ExtensionApi *xapi, + Fts5Context * pFcx, void *pData){ + struct s3jni_xQueryPhraseState * const s = pData; + S3JniDeclLocal_env; + + if( !s->jFcx ){ + s->jFcx = new_java_Fts5Context(env, pFcx); + if( !s->jFcx ) return SQLITE_NOMEM; + } + int rc = (int)(*env)->CallIntMethod(env, s->jCallback, s->midCallback, + SJG.fts5.jExt, s->jFcx); + S3JniIfThrew{ + S3JniExceptionWarnCallbackThrew("xQueryPhrase() callback"); + S3JniExceptionClear; + rc = SQLITE_ERROR; + } + return rc; +} + +JniDeclFtsXA(jint,xQueryPhrase)(JniArgsEnvObj,jobject jFcx, jint iPhrase, + jobject jCallback){ + Fts5ExtDecl; + int rc; + struct s3jni_xQueryPhraseState s; + jclass klazz = jCallback ? (*env)->GetObjectClass(env, jCallback) : NULL; + + if( !klazz ) return SQLITE_MISUSE; + s.jCallback = jCallback; + s.jFcx = 0; + s.ext = ext; + s.midCallback = (*env)->GetMethodID(env, klazz, "call", + "(Lorg/sqlite/jni/fts5/Fts5ExtensionApi;" + "Lorg/sqlite/jni/fts5/Fts5Context;)I"); + S3JniUnrefLocal(klazz); + S3JniExceptionIsFatal("Could not extract xQueryPhraseCallback.call() method."); + rc = ext->xQueryPhrase(PtrGet_Fts5Context(jFcx), iPhrase, &s, + s3jni_xQueryPhrase); + S3JniUnrefLocal(s.jFcx); + return (jint)rc; +} + + +JniDeclFtsXA(jint,xRowCount)(JniArgsEnvObj,jobject jCtx, jobject jOut64){ + Fts5ExtDecl; + sqlite3_int64 nOut = 0; + int const rc = ext->xRowCount(PtrGet_Fts5Context(jCtx), &nOut); + if( 0==rc && jOut64 ) OutputPointer_set_Int64(env, jOut64, (jlong)nOut); + return (jint)rc; +} + +JniDeclFtsXA(jlong,xRowid)(JniArgsEnvObj,jobject jCtx){ + Fts5ExtDecl; + return (jlong)ext->xRowid(PtrGet_Fts5Context(jCtx)); +} + +JniDeclFtsXA(jint,xSetAuxdata)(JniArgsEnvObj,jobject jCtx, jobject jAux){ + Fts5ExtDecl; + int rc; + S3JniFts5AuxData * pAux; + + pAux = s3jni_malloc( sizeof(*pAux)); + if( !pAux ){ + if( jAux ){ + /* Emulate how xSetAuxdata() behaves when it cannot alloc + ** its auxdata wrapper. */ + s3jni_call_xDestroy(jAux); + } + return SQLITE_NOMEM; + } + pAux->jObj = S3JniRefGlobal(jAux); + rc = ext->xSetAuxdata(PtrGet_Fts5Context(jCtx), pAux, + S3JniFts5AuxData_xDestroy); + return rc; +} + +/* xToken() impl for xTokenize(). */ +static int s3jni_xTokenize_xToken(void *p, int tFlags, const char* z, + int nZ, int iStart, int iEnd){ + int rc; + S3JniDeclLocal_env; + struct s3jni_xQueryPhraseState * const s = p; + jbyteArray jba; + + S3JniUnrefLocal(s->tok.jba); + s->tok.zPrev = z; + s->tok.nPrev = nZ; + s->tok.jba = s3jni_new_jbyteArray(z, nZ); + if( !s->tok.jba ) return SQLITE_NOMEM; + jba = s->tok.jba; + rc = (int)(*env)->CallIntMethod(env, s->jCallback, s->midCallback, + (jint)tFlags, jba, (jint)iStart, + (jint)iEnd); + S3JniIfThrew { + S3JniExceptionWarnCallbackThrew("xTokenize() callback"); + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Proxy for Fts5ExtensionApi.xTokenize() and +** fts5_tokenizer.xTokenize() +*/ +static jint s3jni_fts5_xTokenize(JniArgsEnvObj, S3JniNphOp const *pRef, + jint tokFlags, jobject jFcx, + jbyteArray jbaText, jobject jCallback){ + Fts5ExtDecl; + struct s3jni_xQueryPhraseState s; + int rc = 0; + jbyte * const pText = jCallback ? s3jni_jbyteArray_bytes(jbaText) : 0; + jsize nText = pText ? (*env)->GetArrayLength(env, jbaText) : 0; + jclass const klazz = jCallback ? (*env)->GetObjectClass(env, jCallback) : NULL; + + if( !klazz ) return SQLITE_MISUSE; + memset(&s, 0, sizeof(s)); + s.jCallback = jCallback; + s.jFcx = jFcx; + s.ext = ext; + s.midCallback = (*env)->GetMethodID(env, klazz, "call", "(I[BII)I"); + S3JniUnrefLocal(klazz); + S3JniIfThrew { + S3JniExceptionReport; + S3JniExceptionClear; + s3jni_jbyteArray_release(jbaText, pText); + return SQLITE_ERROR; + } + s.tok.jba = S3JniRefLocal(jbaText); + s.tok.zPrev = (const char *)pText; + s.tok.nPrev = (int)nText; + if( pRef == S3JniNph(Fts5ExtensionApi) ){ + rc = ext->xTokenize(PtrGet_Fts5Context(jFcx), + (const char *)pText, (int)nText, + &s, s3jni_xTokenize_xToken); + }else if( pRef == S3JniNph(fts5_tokenizer) ){ + fts5_tokenizer * const pTok = PtrGet_fts5_tokenizer(jSelf); + rc = pTok->xTokenize(PtrGet_Fts5Tokenizer(jFcx), &s, tokFlags, + (const char *)pText, (int)nText, + s3jni_xTokenize_xToken); + }else{ + (*env)->FatalError(env, "This cannot happen. Maintenance required."); + } + if( s.tok.jba ){ + assert( s.tok.zPrev ); + S3JniUnrefLocal(s.tok.jba); + } + s3jni_jbyteArray_release(jbaText, pText); + return (jint)rc; +} + +JniDeclFtsXA(jint,xTokenize)(JniArgsEnvObj,jobject jFcx, jbyteArray jbaText, + jobject jCallback){ + return s3jni_fts5_xTokenize(env, jSelf, S3JniNph(Fts5ExtensionApi), + 0, jFcx, jbaText, jCallback); +} + +JniDeclFtsTok(jint,xTokenize)(JniArgsEnvObj,jobject jFcx, jint tokFlags, + jbyteArray jbaText, jobject jCallback){ + return s3jni_fts5_xTokenize(env, jSelf, S3JniNph(Fts5Tokenizer), + tokFlags, jFcx, jbaText, jCallback); +} + + +JniDeclFtsXA(jobject,xUserData)(JniArgsEnvObj,jobject jFcx){ + Fts5ExtDecl; + Fts5JniAux * const pAux = ext->xUserData(PtrGet_Fts5Context(jFcx)); + return pAux ? pAux->jUserData : 0; +} + +#endif /* SQLITE_ENABLE_FTS5 */ + +//////////////////////////////////////////////////////////////////////// +// End of the main API bindings. Start of SQLTester bits... +//////////////////////////////////////////////////////////////////////// + +#ifdef SQLITE_JNI_ENABLE_SQLTester +typedef struct SQLTesterJni SQLTesterJni; +struct SQLTesterJni { + sqlite3_int64 nDup; +}; +static SQLTesterJni SQLTester = { + 0 +}; + +static void SQLTester_dup_destructor(void*pToFree){ + u64 *p = (u64*)pToFree; + assert( p!=0 ); + p--; + assert( p[0]==0x2bbf4b7c ); + p[0] = 0; + p[1] = 0; + sqlite3_free(p); +} + +/* +** Implementation of +** +** dup(TEXT) +** +** This SQL function simply makes a copy of its text argument. But it +** returns the result using a custom destructor, in order to provide +** tests for the use of Mem.xDel() in the SQLite VDBE. +*/ +static void SQLTester_dup_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + u64 *pOut; + char *z; + int n = sqlite3_value_bytes(argv[0]); + SQLTesterJni * const p = (SQLTesterJni *)sqlite3_user_data(context); + S3JniDeclLocal_env; + + ++p->nDup; + if( n>0 && (pOut = s3jni_malloc( (n+16)&~7 ))!=0 ){ + pOut[0] = 0x2bbf4b7c; + z = (char*)&pOut[1]; + memcpy(z, sqlite3_value_text(argv[0]), n); + z[n] = 0; + sqlite3_result_text(context, z, n, SQLTester_dup_destructor); + } + return; +} + +/* +** Return the number of calls to the dup() SQL function since the +** SQLTester context was opened or since the last dup_count() call. +*/ +static void SQLTester_dup_count_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SQLTesterJni * const p = (SQLTesterJni *)sqlite3_user_data(context); + sqlite3_result_int64(context, p->nDup); + p->nDup = 0; +} + +/* +** Return non-zero if string z matches glob pattern zGlob and zero if the +** pattern does not match. +** +** To repeat: +** +** zero == no match +** non-zero == match +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front, or a hexadecimal +** literal of the form 0x... +*/ +static int SQLTester_strnotglob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && SQLTester_strnotglob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( SQLTester_strnotglob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( z[0]=='0' + && (z[1]=='x' || z[1]=='X') + && sqlite3Isxdigit(z[2]) + ){ + z += 3; + while( sqlite3Isxdigit(z[0]) ){ z++; } + }else{ + if( (z[0]=='-' || z[0]=='+') && sqlite3Isdigit(z[1]) ) z++; + if( !sqlite3Isdigit(z[0]) ) return 0; + z++; + while( sqlite3Isdigit(z[0]) ){ z++; } + } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + return *z==0; +} + +JNIEXPORT jint JNICALL +Java_org_sqlite_jni_capi_SQLTester_strglob( + JniArgsEnvClass, jbyteArray baG, jbyteArray baT +){ + int rc = 0; + jbyte * const pG = s3jni_jbyteArray_bytes(baG); + jbyte * const pT = pG ? s3jni_jbyteArray_bytes(baT) : 0; + + s3jni_oom_fatal(pT); + /* Note that we're relying on the byte arrays having been + NUL-terminated on the Java side. */ + rc = !SQLTester_strnotglob((const char *)pG, (const char *)pT); + s3jni_jbyteArray_release(baG, pG); + s3jni_jbyteArray_release(baT, pT); + return rc; +} + + +static int SQLTester_auto_extension(sqlite3 *pDb, const char **pzErr, + const struct sqlite3_api_routines *ignored){ + sqlite3_create_function(pDb, "dup", 1, SQLITE_UTF8, &SQLTester, + SQLTester_dup_func, 0, 0); + sqlite3_create_function(pDb, "dup_count", 0, SQLITE_UTF8, &SQLTester, + SQLTester_dup_count_func, 0, 0); + return 0; +} + +JNIEXPORT void JNICALL +Java_org_sqlite_jni_capi_SQLTester_installCustomExtensions(JniArgsEnvClass){ + sqlite3_auto_extension( (void(*)(void))SQLTester_auto_extension ); +} + +#endif /* SQLITE_JNI_ENABLE_SQLTester */ +//////////////////////////////////////////////////////////////////////// +// End of SQLTester bindings. Start of lower-level bits. +//////////////////////////////////////////////////////////////////////// + +/* +** Called during static init of the CApi class to set up global +** state. +*/ +JNIEXPORT void JNICALL +Java_org_sqlite_jni_capi_CApi_init(JniArgsEnvClass){ + jclass klazz; + + memset(&S3JniGlobal, 0, sizeof(S3JniGlobal)); + if( (*env)->GetJavaVM(env, &SJG.jvm) ){ + (*env)->FatalError(env, "GetJavaVM() failure shouldn't be possible."); + return; + } + + /* Grab references to various global classes and objects... */ + SJG.g.cLong = S3JniRefGlobal((*env)->FindClass(env,"java/lang/Long")); + S3JniExceptionIsFatal("Error getting reference to Long class."); + SJG.g.ctorLong1 = (*env)->GetMethodID(env, SJG.g.cLong, + "", "(J)V"); + S3JniExceptionIsFatal("Error getting reference to Long constructor."); + + SJG.g.cString = S3JniRefGlobal((*env)->FindClass(env,"java/lang/String")); + S3JniExceptionIsFatal("Error getting reference to String class."); + SJG.g.ctorStringBA = + (*env)->GetMethodID(env, SJG.g.cString, + "", "([BLjava/nio/charset/Charset;)V"); + S3JniExceptionIsFatal("Error getting reference to String(byte[],Charset) ctor."); + SJG.g.stringGetBytes = + (*env)->GetMethodID(env, SJG.g.cString, + "getBytes", "(Ljava/nio/charset/Charset;)[B"); + S3JniExceptionIsFatal("Error getting reference to String.getBytes(Charset)."); + + { /* java.nio.charset.StandardCharsets.UTF_8 */ + jfieldID fUtf8; + klazz = (*env)->FindClass(env,"java/nio/charset/StandardCharsets"); + S3JniExceptionIsFatal("Error getting reference to StandardCharsets class."); + fUtf8 = (*env)->GetStaticFieldID(env, klazz, "UTF_8", + "Ljava/nio/charset/Charset;"); + S3JniExceptionIsFatal("Error getting StandardCharsets.UTF_8 field."); + SJG.g.oCharsetUtf8 = + S3JniRefGlobal((*env)->GetStaticObjectField(env, klazz, fUtf8)); + S3JniExceptionIsFatal("Error getting reference to StandardCharsets.UTF_8."); + S3JniUnrefLocal(klazz); + } + +#ifdef SQLITE_ENABLE_FTS5 + klazz = (*env)->FindClass(env, "org/sqlite/jni/fts5/Fts5PhraseIter"); + S3JniExceptionIsFatal("Error getting reference to org.sqlite.jni.fts5.Fts5PhraseIter."); + SJG.fts5.jPhraseIter.fidA = (*env)->GetFieldID(env, klazz, "a", "J"); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.a field."); + SJG.fts5.jPhraseIter.fidB = (*env)->GetFieldID(env, klazz, "b", "J"); + S3JniExceptionIsFatal("Cannot get Fts5PhraseIter.b field."); + S3JniUnrefLocal(klazz); +#endif + + SJG.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.mutex ); + SJG.hook.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.hook.mutex ); + SJG.nph.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.nph.mutex ); + SJG.envCache.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.envCache.mutex ); + SJG.perDb.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.perDb.mutex ); + SJG.autoExt.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.autoExt.mutex ); + +#if S3JNI_METRICS_MUTEX + SJG.metrics.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + s3jni_oom_fatal( SJG.metrics.mutex ); +#endif + + { + /* Test whether this JVM supports direct memory access via + ByteBuffer. */ + unsigned char buf[16] = {0}; + jobject bb = (*env)->NewDirectByteBuffer(env, buf, 16); + if( bb ){ + SJG.g.byteBuffer.klazz = S3JniRefGlobal((*env)->GetObjectClass(env, bb)); + SJG.g.byteBuffer.midAlloc = (*env)->GetStaticMethodID( + env, SJG.g.byteBuffer.klazz, "allocateDirect", "(I)Ljava/nio/ByteBuffer;" + ); + S3JniExceptionIsFatal("Error getting ByteBuffer.allocateDirect() method."); + SJG.g.byteBuffer.midLimit = (*env)->GetMethodID( + env, SJG.g.byteBuffer.klazz, "limit", "()I" + ); + S3JniExceptionIsFatal("Error getting ByteBuffer.limit() method."); + S3JniUnrefLocal(bb); + }else{ + SJG.g.byteBuffer.klazz = 0; + SJG.g.byteBuffer.midAlloc = 0; + } + } + + sqlite3_shutdown() + /* So that it becomes legal for Java-level code to call + ** sqlite3_config(). */; +} diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h new file mode 100644 index 0000000000..81af5cbde1 --- /dev/null +++ b/ext/jni/src/c/sqlite3-jni.h @@ -0,0 +1,2469 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_capi_CApi */ + +#ifndef _Included_org_sqlite_jni_capi_CApi +#define _Included_org_sqlite_jni_capi_CApi +#ifdef __cplusplus +extern "C" { +#endif +#undef org_sqlite_jni_capi_CApi_SQLITE_ACCESS_EXISTS +#define org_sqlite_jni_capi_CApi_SQLITE_ACCESS_EXISTS 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READWRITE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READ +#define org_sqlite_jni_capi_CApi_SQLITE_ACCESS_READ 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_DENY +#define org_sqlite_jni_capi_CApi_SQLITE_DENY 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_IGNORE +#define org_sqlite_jni_capi_CApi_SQLITE_IGNORE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_INDEX 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TABLE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_INDEX 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TABLE 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_TRIGGER 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TEMP_VIEW 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_TRIGGER 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_VIEW 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_DELETE +#define org_sqlite_jni_capi_CApi_SQLITE_DELETE 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_INDEX 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TABLE 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_INDEX 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TABLE 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_TRIGGER 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TEMP_VIEW 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_TRIGGER 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_VIEW 17L +#undef org_sqlite_jni_capi_CApi_SQLITE_INSERT +#define org_sqlite_jni_capi_CApi_SQLITE_INSERT 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_PRAGMA +#define org_sqlite_jni_capi_CApi_SQLITE_PRAGMA 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_READ +#define org_sqlite_jni_capi_CApi_SQLITE_READ 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_SELECT +#define org_sqlite_jni_capi_CApi_SQLITE_SELECT 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRANSACTION +#define org_sqlite_jni_capi_CApi_SQLITE_TRANSACTION 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_UPDATE +#define org_sqlite_jni_capi_CApi_SQLITE_UPDATE 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_ATTACH +#define org_sqlite_jni_capi_CApi_SQLITE_ATTACH 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_DETACH +#define org_sqlite_jni_capi_CApi_SQLITE_DETACH 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_ALTER_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_ALTER_TABLE 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_REINDEX +#define org_sqlite_jni_capi_CApi_SQLITE_REINDEX 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_ANALYZE +#define org_sqlite_jni_capi_CApi_SQLITE_ANALYZE 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_CREATE_VTABLE +#define org_sqlite_jni_capi_CApi_SQLITE_CREATE_VTABLE 29L +#undef org_sqlite_jni_capi_CApi_SQLITE_DROP_VTABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DROP_VTABLE 30L +#undef org_sqlite_jni_capi_CApi_SQLITE_FUNCTION +#define org_sqlite_jni_capi_CApi_SQLITE_FUNCTION 31L +#undef org_sqlite_jni_capi_CApi_SQLITE_SAVEPOINT +#define org_sqlite_jni_capi_CApi_SQLITE_SAVEPOINT 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_RECURSIVE +#define org_sqlite_jni_capi_CApi_SQLITE_RECURSIVE 33L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATIC +#define org_sqlite_jni_capi_CApi_SQLITE_STATIC 0LL +#undef org_sqlite_jni_capi_CApi_SQLITE_TRANSIENT +#define org_sqlite_jni_capi_CApi_SQLITE_TRANSIENT -1LL +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETSTART_INVERT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETSTART_INVERT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_NOSAVEPOINT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_NOSAVEPOINT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_INVERT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_INVERT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_IGNORENOOP +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESETAPPLY_IGNORENOOP 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_DATA +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_DATA 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_NOTFOUND +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_NOTFOUND 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONFLICT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONFLICT 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONSTRAINT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_CONSTRAINT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_FOREIGN_KEY +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_FOREIGN_KEY 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_OMIT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_OMIT 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_REPLACE +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_REPLACE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_ABORT +#define org_sqlite_jni_capi_CApi_SQLITE_CHANGESET_ABORT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SINGLETHREAD +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SINGLETHREAD 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MULTITHREAD +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MULTITHREAD 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SERIALIZED +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SERIALIZED 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MALLOC +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MALLOC 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMALLOC +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMALLOC 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SCRATCH +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SCRATCH 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PAGECACHE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PAGECACHE 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_HEAP +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_HEAP 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMSTATUS +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMSTATUS 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MUTEX 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETMUTEX 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOOKASIDE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOOKASIDE 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOG +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_LOG 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_URI +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_URI 17L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE2 +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE2 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE2 +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_GETPCACHE2 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_COVERING_INDEX_SCAN +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_COVERING_INDEX_SCAN 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SQLLOG +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SQLLOG 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MMAP_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MMAP_SIZE 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_WIN32_HEAPSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_WIN32_HEAPSIZE 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE_HDRSZ +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PCACHE_HDRSZ 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PMASZ +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_PMASZ 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_STMTJRNL_SPILL +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_STMTJRNL_SPILL 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SMALL_MALLOC +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SMALL_MALLOC 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SORTERREF_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_SORTERREF_SIZE 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMDB_MAXSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_CONFIG_MEMDB_MAXSIZE 29L +#undef org_sqlite_jni_capi_CApi_SQLITE_INTEGER +#define org_sqlite_jni_capi_CApi_SQLITE_INTEGER 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_FLOAT +#define org_sqlite_jni_capi_CApi_SQLITE_FLOAT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_TEXT +#define org_sqlite_jni_capi_CApi_SQLITE_TEXT 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_BLOB +#define org_sqlite_jni_capi_CApi_SQLITE_BLOB 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_NULL +#define org_sqlite_jni_capi_CApi_SQLITE_NULL 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAINDBNAME +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAINDBNAME 1000L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LOOKASIDE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LOOKASIDE 1001L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FKEY +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FKEY 1002L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_TRIGGER 1003L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_QPSG +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_QPSG 1007L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRIGGER_EQP +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRIGGER_EQP 1008L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_RESET_DATABASE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_RESET_DATABASE 1009L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DEFENSIVE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DEFENSIVE 1010L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_WRITABLE_SCHEMA +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_ALTER_TABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DML +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DML 1013L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DDL +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_DQS_DDL 1014L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_VIEW +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_ENABLE_VIEW 1015L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRUSTED_SCHEMA +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_STMT_SCANSTATUS +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_STMT_SCANSTATUS 1018L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_REVERSE_SCANORDER +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_REVERSE_SCANORDER 1019L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAX +#define org_sqlite_jni_capi_CApi_SQLITE_DBCONFIG_MAX 1019L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_USED 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_SCHEMA_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_SCHEMA_USED 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_STMT_USED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_STMT_USED 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_HIT +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_HIT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_HIT +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_HIT 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_MISS +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_MISS 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_WRITE 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_DEFERRED_FKS +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_DEFERRED_FKS 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF8 +#define org_sqlite_jni_capi_CApi_SQLITE_UTF8 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16LE +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16LE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16BE +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16BE 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16 +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_UTF16_ALIGNED +#define org_sqlite_jni_capi_CApi_SQLITE_UTF16_ALIGNED 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCKSTATE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCKSTATE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_GET_LOCKPROXYFILE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_GET_LOCKPROXYFILE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SET_LOCKPROXYFILE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SET_LOCKPROXYFILE 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LAST_ERRNO +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LAST_ERRNO 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_HINT +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_HINT 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CHUNK_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CHUNK_SIZE 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_FILE_POINTER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_FILE_POINTER 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC_OMITTED +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC_OMITTED 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_AV_RETRY +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_AV_RETRY 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PERSIST_WAL +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PERSIST_WAL 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_OVERWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_OVERWRITE 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFSNAME +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFSNAME 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_POWERSAFE_OVERWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_POWERSAFE_OVERWRITE 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PRAGMA +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PRAGMA 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BUSYHANDLER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BUSYHANDLER 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TEMPFILENAME +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TEMPFILENAME 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_MMAP_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_MMAP_SIZE 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TRACE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_TRACE 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_HAS_MOVED +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_HAS_MOVED 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SYNC 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_PHASETWO +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_PHASETWO 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_SET_HANDLE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_SET_HANDLE 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WAL_BLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WAL_BLOCK 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ZIPVFS +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ZIPVFS 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RBU +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RBU 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFS_POINTER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_VFS_POINTER 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_JOURNAL_POINTER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_JOURNAL_POINTER 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_GET_HANDLE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_WIN32_GET_HANDLE 29L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PDB +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_PDB 30L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BEGIN_ATOMIC_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_ATOMIC_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCK_TIMEOUT +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_LOCK_TIMEOUT 34L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_DATA_VERSION +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_DATA_VERSION 35L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_LIMIT +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_SIZE_LIMIT 36L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_DONE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_DONE 37L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESERVE_BYTES +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESERVE_BYTES 38L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_START +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKPT_START 39L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_EXTERNAL_READER +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_EXTERNAL_READER 40L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKSM_FILE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_CKSM_FILE 41L +#undef org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESET_CACHE +#define org_sqlite_jni_capi_CApi_SQLITE_FCNTL_RESET_CACHE 42L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_NONE +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_NONE 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_SHARED +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_SHARED 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_RESERVED +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_RESERVED 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_PENDING +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_PENDING 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCK_EXCLUSIVE +#define org_sqlite_jni_capi_CApi_SQLITE_LOCK_EXCLUSIVE 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC512 +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC512 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC1K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC1K 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC2K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC2K 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC4K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC4K 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC8K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC8K 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC16K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC16K 64L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC32K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC32K 128L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC64K +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_ATOMIC64K 256L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SAFE_APPEND +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SAFE_APPEND 512L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SEQUENTIAL +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_SEQUENTIAL 1024L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 2048L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_POWERSAFE_OVERWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_POWERSAFE_OVERWRITE 4096L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_IMMUTABLE +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_IMMUTABLE 8192L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOCAP_BATCH_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOCAP_BATCH_ATOMIC 16384L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LENGTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LENGTH 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_SQL_LENGTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_SQL_LENGTH 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COLUMN +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COLUMN 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_EXPR_DEPTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_EXPR_DEPTH 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COMPOUND_SELECT +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_COMPOUND_SELECT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VDBE_OP +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VDBE_OP 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_FUNCTION_ARG +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_FUNCTION_ARG 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_ATTACHED +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_ATTACHED 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LIKE_PATTERN_LENGTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VARIABLE_NUMBER +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_VARIABLE_NUMBER 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_TRIGGER_DEPTH +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_TRIGGER_DEPTH 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_LIMIT_WORKER_THREADS +#define org_sqlite_jni_capi_CApi_SQLITE_LIMIT_WORKER_THREADS 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_READONLY +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_READONLY 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_READWRITE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_READWRITE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_CREATE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_CREATE 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_URI +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_URI 64L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_MEMORY +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_MEMORY 128L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOMUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOMUTEX 32768L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_FULLMUTEX +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_FULLMUTEX 65536L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_SHAREDCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_SHAREDCACHE 131072L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_PRIVATECACHE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_PRIVATECACHE 262144L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOFOLLOW +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_NOFOLLOW 16777216L +#undef org_sqlite_jni_capi_CApi_SQLITE_OPEN_EXRESCODE +#define org_sqlite_jni_capi_CApi_SQLITE_OPEN_EXRESCODE 33554432L +#undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_PERSISTENT +#define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_PERSISTENT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NO_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_PREPARE_NO_VTAB 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_OK +#define org_sqlite_jni_capi_CApi_SQLITE_OK 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_INTERNAL +#define org_sqlite_jni_capi_CApi_SQLITE_INTERNAL 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_PERM +#define org_sqlite_jni_capi_CApi_SQLITE_PERM 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_ABORT +#define org_sqlite_jni_capi_CApi_SQLITE_ABORT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCKED +#define org_sqlite_jni_capi_CApi_SQLITE_LOCKED 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOMEM +#define org_sqlite_jni_capi_CApi_SQLITE_NOMEM 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_INTERRUPT +#define org_sqlite_jni_capi_CApi_SQLITE_INTERRUPT 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR 10L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT 11L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTFOUND +#define org_sqlite_jni_capi_CApi_SQLITE_NOTFOUND 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_FULL +#define org_sqlite_jni_capi_CApi_SQLITE_FULL 13L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN 14L +#undef org_sqlite_jni_capi_CApi_SQLITE_PROTOCOL +#define org_sqlite_jni_capi_CApi_SQLITE_PROTOCOL 15L +#undef org_sqlite_jni_capi_CApi_SQLITE_EMPTY +#define org_sqlite_jni_capi_CApi_SQLITE_EMPTY 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_SCHEMA +#define org_sqlite_jni_capi_CApi_SQLITE_SCHEMA 17L +#undef org_sqlite_jni_capi_CApi_SQLITE_TOOBIG +#define org_sqlite_jni_capi_CApi_SQLITE_TOOBIG 18L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT 19L +#undef org_sqlite_jni_capi_CApi_SQLITE_MISMATCH +#define org_sqlite_jni_capi_CApi_SQLITE_MISMATCH 20L +#undef org_sqlite_jni_capi_CApi_SQLITE_MISUSE +#define org_sqlite_jni_capi_CApi_SQLITE_MISUSE 21L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOLFS +#define org_sqlite_jni_capi_CApi_SQLITE_NOLFS 22L +#undef org_sqlite_jni_capi_CApi_SQLITE_AUTH +#define org_sqlite_jni_capi_CApi_SQLITE_AUTH 23L +#undef org_sqlite_jni_capi_CApi_SQLITE_FORMAT +#define org_sqlite_jni_capi_CApi_SQLITE_FORMAT 24L +#undef org_sqlite_jni_capi_CApi_SQLITE_RANGE +#define org_sqlite_jni_capi_CApi_SQLITE_RANGE 25L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTADB +#define org_sqlite_jni_capi_CApi_SQLITE_NOTADB 26L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTICE +#define org_sqlite_jni_capi_CApi_SQLITE_NOTICE 27L +#undef org_sqlite_jni_capi_CApi_SQLITE_WARNING +#define org_sqlite_jni_capi_CApi_SQLITE_WARNING 28L +#undef org_sqlite_jni_capi_CApi_SQLITE_ROW +#define org_sqlite_jni_capi_CApi_SQLITE_ROW 100L +#undef org_sqlite_jni_capi_CApi_SQLITE_DONE +#define org_sqlite_jni_capi_CApi_SQLITE_DONE 101L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR_MISSING_COLLSEQ +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR_MISSING_COLLSEQ 257L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR_RETRY +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR_RETRY 513L +#undef org_sqlite_jni_capi_CApi_SQLITE_ERROR_SNAPSHOT +#define org_sqlite_jni_capi_CApi_SQLITE_ERROR_SNAPSHOT 769L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_READ +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_READ 266L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHORT_READ +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHORT_READ 522L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_WRITE 778L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSYNC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSYNC 1034L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_FSYNC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_FSYNC 1290L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_TRUNCATE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_TRUNCATE 1546L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSTAT +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_FSTAT 1802L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_UNLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_UNLOCK 2058L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_RDLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_RDLOCK 2314L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE 2570L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_BLOCKED +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_BLOCKED 2826L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_NOMEM +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_NOMEM 3082L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_ACCESS +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_ACCESS 3338L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CHECKRESERVEDLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CHECKRESERVEDLOCK 3594L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_LOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_LOCK 3850L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CLOSE 4106L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DIR_CLOSE 4362L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMOPEN +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMOPEN 4618L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMSIZE 4874L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMLOCK 5130L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMMAP +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SHMMAP 5386L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_SEEK +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_SEEK 5642L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE_NOENT +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DELETE_NOENT 5898L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_MMAP +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_MMAP 6154L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_GETTEMPPATH +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_GETTEMPPATH 6410L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CONVPATH +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CONVPATH 6666L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_VNODE +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_VNODE 6922L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_AUTH +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_AUTH 7178L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_BEGIN_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_BEGIN_ATOMIC 7434L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_COMMIT_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_COMMIT_ATOMIC 7690L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_ROLLBACK_ATOMIC +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_ROLLBACK_ATOMIC 7946L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_DATA +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_DATA 8202L +#undef org_sqlite_jni_capi_CApi_SQLITE_IOERR_CORRUPTFS +#define org_sqlite_jni_capi_CApi_SQLITE_IOERR_CORRUPTFS 8458L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCKED_SHAREDCACHE +#define org_sqlite_jni_capi_CApi_SQLITE_LOCKED_SHAREDCACHE 262L +#undef org_sqlite_jni_capi_CApi_SQLITE_LOCKED_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_LOCKED_VTAB 518L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY_RECOVERY +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY_RECOVERY 261L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY_SNAPSHOT +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY_SNAPSHOT 517L +#undef org_sqlite_jni_capi_CApi_SQLITE_BUSY_TIMEOUT +#define org_sqlite_jni_capi_CApi_SQLITE_BUSY_TIMEOUT 773L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_NOTEMPDIR +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_NOTEMPDIR 270L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_ISDIR +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_ISDIR 526L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_FULLPATH +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_FULLPATH 782L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_CONVPATH +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_CONVPATH 1038L +#undef org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_SYMLINK +#define org_sqlite_jni_capi_CApi_SQLITE_CANTOPEN_SYMLINK 1550L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_VTAB 267L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_SEQUENCE +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_SEQUENCE 523L +#undef org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_INDEX +#define org_sqlite_jni_capi_CApi_SQLITE_CORRUPT_INDEX 779L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_RECOVERY +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_RECOVERY 264L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTLOCK +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTLOCK 520L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_ROLLBACK 776L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_DBMOVED +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_DBMOVED 1032L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTINIT +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_CANTINIT 1288L +#undef org_sqlite_jni_capi_CApi_SQLITE_READONLY_DIRECTORY +#define org_sqlite_jni_capi_CApi_SQLITE_READONLY_DIRECTORY 1544L +#undef org_sqlite_jni_capi_CApi_SQLITE_ABORT_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_ABORT_ROLLBACK 516L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_CHECK +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_CHECK 275L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_COMMITHOOK +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_COMMITHOOK 531L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FOREIGNKEY +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FOREIGNKEY 787L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FUNCTION +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_FUNCTION 1043L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_NOTNULL +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_NOTNULL 1299L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PRIMARYKEY +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PRIMARYKEY 1555L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_TRIGGER +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_TRIGGER 1811L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_UNIQUE +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_UNIQUE 2067L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_VTAB +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_VTAB 2323L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_ROWID +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_ROWID 2579L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PINNED +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_PINNED 2835L +#undef org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_DATATYPE +#define org_sqlite_jni_capi_CApi_SQLITE_CONSTRAINT_DATATYPE 3091L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_WAL +#define org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_WAL 283L +#undef org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_NOTICE_RECOVER_ROLLBACK 539L +#undef org_sqlite_jni_capi_CApi_SQLITE_WARNING_AUTOINDEX +#define org_sqlite_jni_capi_CApi_SQLITE_WARNING_AUTOINDEX 284L +#undef org_sqlite_jni_capi_CApi_SQLITE_AUTH_USER +#define org_sqlite_jni_capi_CApi_SQLITE_AUTH_USER 279L +#undef org_sqlite_jni_capi_CApi_SQLITE_OK_LOAD_PERMANENTLY +#define org_sqlite_jni_capi_CApi_SQLITE_OK_LOAD_PERMANENTLY 256L +#undef org_sqlite_jni_capi_CApi_SQLITE_SERIALIZE_NOCOPY +#define org_sqlite_jni_capi_CApi_SQLITE_SERIALIZE_NOCOPY 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_FREEONCLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_FREEONCLOSE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_READONLY +#define org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_READONLY 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_RESIZEABLE +#define org_sqlite_jni_capi_CApi_SQLITE_DESERIALIZE_RESIZEABLE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_SESSION_CONFIG_STRMSIZE +#define org_sqlite_jni_capi_CApi_SQLITE_SESSION_CONFIG_STRMSIZE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_SESSION_OBJCONFIG_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_SESSION_OBJCONFIG_SIZE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_MEMORY_USED +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_MEMORY_USED 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_USED +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_USED 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_OVERFLOW +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_OVERFLOW 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_SIZE 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PARSER_STACK +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PARSER_STACK 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_SIZE +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_PAGECACHE_SIZE 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_COUNT +#define org_sqlite_jni_capi_CApi_SQLITE_STATUS_MALLOC_COUNT 9L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FULLSCAN_STEP +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FULLSCAN_STEP 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_SORT +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_SORT 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_AUTOINDEX +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_AUTOINDEX 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_VM_STEP +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_VM_STEP 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_REPREPARE +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_REPREPARE 5L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_RUN +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_RUN 6L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_MISS +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_MISS 7L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_HIT +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_FILTER_HIT 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_MEMUSED +#define org_sqlite_jni_capi_CApi_SQLITE_STMTSTATUS_MEMUSED 99L +#undef org_sqlite_jni_capi_CApi_SQLITE_SYNC_NORMAL +#define org_sqlite_jni_capi_CApi_SQLITE_SYNC_NORMAL 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_SYNC_FULL +#define org_sqlite_jni_capi_CApi_SQLITE_SYNC_FULL 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_SYNC_DATAONLY +#define org_sqlite_jni_capi_CApi_SQLITE_SYNC_DATAONLY 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_STMT +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_STMT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_PROFILE +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_PROFILE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_ROW +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_ROW 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_TRACE_CLOSE +#define org_sqlite_jni_capi_CApi_SQLITE_TRACE_CLOSE 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_TXN_NONE +#define org_sqlite_jni_capi_CApi_SQLITE_TXN_NONE 0L +#undef org_sqlite_jni_capi_CApi_SQLITE_TXN_READ +#define org_sqlite_jni_capi_CApi_SQLITE_TXN_READ 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_TXN_WRITE +#define org_sqlite_jni_capi_CApi_SQLITE_TXN_WRITE 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_DETERMINISTIC +#define org_sqlite_jni_capi_CApi_SQLITE_DETERMINISTIC 2048L +#undef org_sqlite_jni_capi_CApi_SQLITE_DIRECTONLY +#define org_sqlite_jni_capi_CApi_SQLITE_DIRECTONLY 524288L +#undef org_sqlite_jni_capi_CApi_SQLITE_SUBTYPE +#define org_sqlite_jni_capi_CApi_SQLITE_SUBTYPE 1048576L +#undef org_sqlite_jni_capi_CApi_SQLITE_INNOCUOUS +#define org_sqlite_jni_capi_CApi_SQLITE_INNOCUOUS 2097152L +#undef org_sqlite_jni_capi_CApi_SQLITE_RESULT_SUBTYPE +#define org_sqlite_jni_capi_CApi_SQLITE_RESULT_SUBTYPE 16777216L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_SCAN_UNIQUE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_SCAN_UNIQUE 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_EQ +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_EQ 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GT 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LE 8L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LT 16L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GE 32L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_MATCH +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_MATCH 64L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIKE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIKE 65L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GLOB +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_GLOB 66L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_REGEXP +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_REGEXP 67L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_NE +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_NE 68L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOT 69L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOTNULL +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNULL +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_ISNULL 71L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_IS +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_IS 72L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIMIT +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_LIMIT 73L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_OFFSET +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_OFFSET 74L +#undef org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_FUNCTION +#define org_sqlite_jni_capi_CApi_SQLITE_INDEX_CONSTRAINT_FUNCTION 150L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_CONSTRAINT_SUPPORT +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_CONSTRAINT_SUPPORT 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_INNOCUOUS +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_INNOCUOUS 2L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_DIRECTONLY +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_DIRECTONLY 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_VTAB_USES_ALL_SCHEMAS +#define org_sqlite_jni_capi_CApi_SQLITE_VTAB_USES_ALL_SCHEMAS 4L +#undef org_sqlite_jni_capi_CApi_SQLITE_ROLLBACK +#define org_sqlite_jni_capi_CApi_SQLITE_ROLLBACK 1L +#undef org_sqlite_jni_capi_CApi_SQLITE_FAIL +#define org_sqlite_jni_capi_CApi_SQLITE_FAIL 3L +#undef org_sqlite_jni_capi_CApi_SQLITE_REPLACE +#define org_sqlite_jni_capi_CApi_SQLITE_REPLACE 5L +/* + * Class: org_sqlite_jni_capi_CApi + * Method: init + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_init + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_java_uncache_thread + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1java_1uncache_1thread + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_jni_supports_nio + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1supports_1nio + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_jni_db_error + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1db_1error + (JNIEnv *, jclass, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_aggregate_context + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Z)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1aggregate_1context + (JNIEnv *, jclass, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_auto_extension + * Signature: (Lorg/sqlite/jni/capi/AutoExtensionCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1auto_1extension + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_finish + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1finish + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_init + * Signature: (JLjava/lang/String;JLjava/lang/String;)Lorg/sqlite/jni/capi/sqlite3_backup; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1init + (JNIEnv *, jclass, jlong, jstring, jlong, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_pagecount + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1pagecount + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_remaining + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1remaining + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_backup_step + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1backup_1step + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_blob + * Signature: (JI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1blob + (JNIEnv *, jclass, jlong, jint, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_double + * Signature: (JID)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1double + (JNIEnv *, jclass, jlong, jint, jdouble); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_int + * Signature: (JII)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1int + (JNIEnv *, jclass, jlong, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_int64 + * Signature: (JIJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1int64 + (JNIEnv *, jclass, jlong, jint, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_java_object + * Signature: (JILjava/lang/Object;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1java_1object + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;ILjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1nio_1buffer + (JNIEnv *, jclass, jobject, jint, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_null + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1null + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_parameter_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1parameter_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_parameter_index + * Signature: (J[B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1parameter_1index + (JNIEnv *, jclass, jlong, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_parameter_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1parameter_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_text + * Signature: (JI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1text + (JNIEnv *, jclass, jlong, jint, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_text16 + * Signature: (JI[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1text16 + (JNIEnv *, jclass, jlong, jint, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_value + * Signature: (JIJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1value + (JNIEnv *, jclass, jlong, jint, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_zeroblob + * Signature: (JII)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1zeroblob + (JNIEnv *, jclass, jlong, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_bind_zeroblob64 + * Signature: (JIJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1bind_1zeroblob64 + (JNIEnv *, jclass, jlong, jint, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_bytes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1bytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_close + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1close + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_open + * Signature: (JLjava/lang/String;Ljava/lang/String;Ljava/lang/String;JILorg/sqlite/jni/capi/OutputPointer/sqlite3_blob;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1open + (JNIEnv *, jclass, jlong, jstring, jstring, jstring, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_read + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1read + (JNIEnv *, jclass, jlong, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_read_nio_buffer + * Signature: (JILjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1read_1nio_1buffer + (JNIEnv *, jclass, jlong, jint, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_reopen + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1reopen + (JNIEnv *, jclass, jlong, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_write + * Signature: (J[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1write + (JNIEnv *, jclass, jlong, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_blob_write_nio_buffer + * Signature: (JILjava/nio/ByteBuffer;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1blob_1write_1nio_1buffer + (JNIEnv *, jclass, jlong, jint, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_busy_handler + * Signature: (JLorg/sqlite/jni/capi/BusyHandlerCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1busy_1handler + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_busy_timeout + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1busy_1timeout + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_cancel_auto_extension + * Signature: (Lorg/sqlite/jni/capi/AutoExtensionCallback;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1cancel_1auto_1extension + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_changes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1changes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_changes64 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1changes64 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_clear_bindings + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1clear_1bindings + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_close + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1close + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_close_v2 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1close_1v2 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_blob + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1blob + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_bytes + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1bytes + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_bytes16 + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1bytes16 + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_database_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1database_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_decltype + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1decltype + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_double + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)D + */ +JNIEXPORT jdouble JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1double + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_int + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1int + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_int64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1int64 + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_java_object + * Signature: (JI)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1java_1object + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Ljava/nio/ByteBuffer; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1nio_1buffer + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_origin_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1origin_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_table_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1table_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_text + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1text + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_text16 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1text16 + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_type + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1type + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_column_value + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;I)Lorg/sqlite/jni/capi/sqlite3_value; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1column_1value + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_collation_needed + * Signature: (JLorg/sqlite/jni/capi/CollationNeededCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1collation_1needed + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_commit_hook + * Signature: (JLorg/sqlite/jni/capi/CommitHookCallback;)Lorg/sqlite/jni/capi/CommitHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1commit_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_compileoption_get + * Signature: (I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1compileoption_1get + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_compileoption_used + * Signature: (Ljava/lang/String;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1compileoption_1used + (JNIEnv *, jclass, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_complete + * Signature: ([B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1complete + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_config__enable + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1enable + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_config__CONFIG_LOG + * Signature: (Lorg/sqlite/jni/capi/ConfigLogCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1CONFIG_1LOG + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_config__SQLLOG + * Signature: (Lorg/sqlite/jni/capi/ConfigSqlLogCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1config_1_1SQLLOG + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_context_db_handle + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)Lorg/sqlite/jni/capi/sqlite3; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1context_1db_1handle + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_create_collation + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;ILorg/sqlite/jni/capi/CollationCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1create_1collation + (JNIEnv *, jclass, jobject, jstring, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_create_function + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;IILorg/sqlite/jni/capi/SQLFunction;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1create_1function + (JNIEnv *, jclass, jobject, jstring, jint, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_data_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1data_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_config + * Signature: (Lorg/sqlite/jni/capi/sqlite3;IILorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2 + (JNIEnv *, jclass, jobject, jint, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_config + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2ILjava_lang_String_2 + (JNIEnv *, jclass, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_name + * Signature: (JI)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1name + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_filename + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1filename + (JNIEnv *, jclass, jobject, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_handle + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Lorg/sqlite/jni/capi/sqlite3; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1handle + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_readonly + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1readonly + (JNIEnv *, jclass, jobject, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_release_memory + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1release_1memory + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_db_status + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1status + (JNIEnv *, jclass, jobject, jint, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_errcode + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errcode + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_errmsg + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errmsg + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_errmsg + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1errmsg + (JNIEnv *, jclass, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_error_offset + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1error_1offset + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_errstr + * Signature: (I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errstr + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_expanded_sql + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1expanded_1sql + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_extended_errcode + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1errcode + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_extended_result_codes + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1extended_1result_1codes + (JNIEnv *, jclass, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_get_autocommit + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1get_1autocommit + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_get_auxdata + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1get_1auxdata + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_finalize + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1finalize + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_initialize + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1initialize + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_interrupt + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1interrupt + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_is_interrupted + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1is_1interrupted + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_keyword_check + * Signature: (Ljava/lang/String;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1keyword_1check + (JNIEnv *, jclass, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_keyword_count + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1keyword_1count + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_keyword_name + * Signature: (I)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1keyword_1name + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_last_insert_rowid + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1last_1insert_1rowid + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_libversion + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1libversion + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_libversion_number + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1libversion_1number + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_limit + * Signature: (Lorg/sqlite/jni/capi/sqlite3;II)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1limit + (JNIEnv *, jclass, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_normalized_sql + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1normalized_1sql + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_open + * Signature: (Ljava/lang/String;Lorg/sqlite/jni/capi/OutputPointer/sqlite3;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1open + (JNIEnv *, jclass, jstring, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_open_v2 + * Signature: (Ljava/lang/String;Lorg/sqlite/jni/capi/OutputPointer/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1open_1v2 + (JNIEnv *, jclass, jstring, jobject, jint, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_prepare + * Signature: (J[BILorg/sqlite/jni/capi/OutputPointer/sqlite3_stmt;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1prepare + (JNIEnv *, jclass, jlong, jbyteArray, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_prepare_v2 + * Signature: (J[BILorg/sqlite/jni/capi/OutputPointer/sqlite3_stmt;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1prepare_1v2 + (JNIEnv *, jclass, jlong, jbyteArray, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_prepare_v3 + * Signature: (J[BIILorg/sqlite/jni/capi/OutputPointer/sqlite3_stmt;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1prepare_1v3 + (JNIEnv *, jclass, jlong, jbyteArray, jint, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_blobwrite + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1blobwrite + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_count + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1count + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_depth + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1depth + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_hook + * Signature: (JLorg/sqlite/jni/capi/PreupdateHookCallback;)Lorg/sqlite/jni/capi/PreupdateHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_new + * Signature: (JILorg/sqlite/jni/capi/OutputPointer/sqlite3_value;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1new + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_preupdate_old + * Signature: (JILorg/sqlite/jni/capi/OutputPointer/sqlite3_value;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1preupdate_1old + (JNIEnv *, jclass, jlong, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_progress_handler + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILorg/sqlite/jni/capi/ProgressHandlerCallback;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1progress_1handler + (JNIEnv *, jclass, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_randomness + * Signature: ([B)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1randomness + (JNIEnv *, jclass, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_release_memory + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1release_1memory + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_reset + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1reset + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_reset_auto_extension + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1reset_1auto_1extension + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_double + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;D)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1double + (JNIEnv *, jclass, jobject, jdouble); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error + (JNIEnv *, jclass, jobject, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error_toobig + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1toobig + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error_nomem + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1nomem + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_error_code + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1error_1code + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_int + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1int + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_int64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;J)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1int64 + (JNIEnv *, jclass, jobject, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_java_object + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Ljava/lang/Object;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1java_1object + (JNIEnv *, jclass, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Ljava/nio/ByteBuffer;II)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1nio_1buffer + (JNIEnv *, jclass, jobject, jobject, jint, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_null + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1null + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_subtype + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1subtype + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_value + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;Lorg/sqlite/jni/capi/sqlite3_value;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1value + (JNIEnv *, jclass, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_zeroblob + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;I)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1zeroblob + (JNIEnv *, jclass, jobject, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_zeroblob64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1zeroblob64 + (JNIEnv *, jclass, jobject, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_blob + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1blob + (JNIEnv *, jclass, jobject, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_blob64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BJ)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1blob64 + (JNIEnv *, jclass, jobject, jbyteArray, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_text + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1text + (JNIEnv *, jclass, jobject, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_result_text64 + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;[BJI)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1result_1text64 + (JNIEnv *, jclass, jobject, jbyteArray, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_rollback_hook + * Signature: (JLorg/sqlite/jni/capi/RollbackHookCallback;)Lorg/sqlite/jni/capi/RollbackHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1rollback_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_authorizer + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Lorg/sqlite/jni/capi/AuthorizerCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1authorizer + (JNIEnv *, jclass, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_auxdata + * Signature: (Lorg/sqlite/jni/capi/sqlite3_context;ILjava/lang/Object;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1auxdata + (JNIEnv *, jclass, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_last_insert_rowid + * Signature: (Lorg/sqlite/jni/capi/sqlite3;J)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1last_1insert_1rowid + (JNIEnv *, jclass, jobject, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_shutdown + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1shutdown + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_sleep + * Signature: (I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1sleep + (JNIEnv *, jclass, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_sourceid + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1sourceid + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_sql + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1sql + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_status + * Signature: (ILorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1status + (JNIEnv *, jclass, jint, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_status64 + * Signature: (ILorg/sqlite/jni/capi/OutputPointer/Int64;Lorg/sqlite/jni/capi/OutputPointer/Int64;Z)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1status64 + (JNIEnv *, jclass, jint, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_step + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1step + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_busy + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1busy + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_explain + * Signature: (JI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1explain + (JNIEnv *, jclass, jlong, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_isexplain + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1isexplain + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_readonly + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1readonly + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_stmt_status + * Signature: (Lorg/sqlite/jni/capi/sqlite3_stmt;IZ)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1stmt_1status + (JNIEnv *, jclass, jobject, jint, jboolean); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_strglob + * Signature: ([B[B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1strglob + (JNIEnv *, jclass, jbyteArray, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_strlike + * Signature: ([B[BI)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1strlike + (JNIEnv *, jclass, jbyteArray, jbyteArray, jint); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_system_errno + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1system_1errno + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_table_column_metadata + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Lorg/sqlite/jni/capi/OutputPointer/String;Lorg/sqlite/jni/capi/OutputPointer/String;Lorg/sqlite/jni/capi/OutputPointer/Bool;Lorg/sqlite/jni/capi/OutputPointer/Bool;Lorg/sqlite/jni/capi/OutputPointer/Bool;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1table_1column_1metadata + (JNIEnv *, jclass, jobject, jstring, jstring, jstring, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_threadsafe + * Signature: ()I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1threadsafe + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_total_changes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1total_1changes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_total_changes64 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1total_1changes64 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_trace_v2 + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILorg/sqlite/jni/capi/TraceV2Callback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1trace_1v2 + (JNIEnv *, jclass, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_txn_state + * Signature: (Lorg/sqlite/jni/capi/sqlite3;Ljava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1txn_1state + (JNIEnv *, jclass, jobject, jstring); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_update_hook + * Signature: (JLorg/sqlite/jni/capi/UpdateHookCallback;)Lorg/sqlite/jni/capi/UpdateHookCallback; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1update_1hook + (JNIEnv *, jclass, jlong, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_blob + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1blob + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_bytes + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1bytes + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_bytes16 + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1bytes16 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_double + * Signature: (J)D + */ +JNIEXPORT jdouble JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1double + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_dup + * Signature: (J)Lorg/sqlite/jni/capi/sqlite3_value; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1dup + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_encoding + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1encoding + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_free + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1free + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_frombind + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1frombind + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_int + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1int + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_int64 + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1int64 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_java_object + * Signature: (J)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1java_1object + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_nio_buffer + * Signature: (Lorg/sqlite/jni/capi/sqlite3_value;)Ljava/nio/ByteBuffer; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1nio_1buffer + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_nochange + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1nochange + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_numeric_type + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1numeric_1type + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_subtype + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1subtype + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_text + * Signature: (J)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1text + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_text16 + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1text16 + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_value_type + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1value_1type + (JNIEnv *, jclass, jlong); + +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_jni_internal_details + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1jni_1internal_1details + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_capi_SQLTester */ + +#ifndef _Included_org_sqlite_jni_capi_SQLTester +#define _Included_org_sqlite_jni_capi_SQLTester +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_capi_SQLTester + * Method: strglob + * Signature: ([B[B)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_SQLTester_strglob + (JNIEnv *, jclass, jbyteArray, jbyteArray); + +/* + * Class: org_sqlite_jni_capi_SQLTester + * Method: installCustomExtensions + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_capi_SQLTester_installCustomExtensions + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_Fts5ExtensionApi */ + +#ifndef _Included_org_sqlite_jni_fts5_Fts5ExtensionApi +#define _Included_org_sqlite_jni_fts5_Fts5ExtensionApi +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: getInstance + * Signature: ()Lorg/sqlite/jni/fts5/Fts5ExtensionApi; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_getInstance + (JNIEnv *, jclass); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnCount + (JNIEnv *, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnSize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnSize + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnText + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnText + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xColumnTotalSize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/Int64;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xColumnTotalSize + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xGetAuxdata + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Z)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xGetAuxdata + (JNIEnv *, jobject, jobject, jboolean); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xInst + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xInst + (JNIEnv *, jobject, jobject, jint, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xInstCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xInstCount + (JNIEnv *, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseCount + (JNIEnv *, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseFirst + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseFirst + (JNIEnv *, jobject, jobject, jint, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseFirstColumn + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseFirstColumn + (JNIEnv *, jobject, jobject, jint, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseNext + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;Lorg/sqlite/jni/capi/OutputPointer/Int32;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseNext + (JNIEnv *, jobject, jobject, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseNextColumn + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/fts5/Fts5PhraseIter;Lorg/sqlite/jni/capi/OutputPointer/Int32;)V + */ +JNIEXPORT void JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseNextColumn + (JNIEnv *, jobject, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xPhraseSize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;I)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xPhraseSize + (JNIEnv *, jobject, jobject, jint); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xQueryPhrase + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;ILorg/sqlite/jni/fts5/Fts5ExtensionApi/XQueryPhraseCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xQueryPhrase + (JNIEnv *, jobject, jobject, jint, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xRowCount + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Lorg/sqlite/jni/capi/OutputPointer/Int64;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xRowCount + (JNIEnv *, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xRowid + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)J + */ +JNIEXPORT jlong JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xRowid + (JNIEnv *, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xSetAuxdata + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;Ljava/lang/Object;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xSetAuxdata + (JNIEnv *, jobject, jobject, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xTokenize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;[BLorg/sqlite/jni/fts5/XTokenizeCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xTokenize + (JNIEnv *, jobject, jobject, jbyteArray, jobject); + +/* + * Class: org_sqlite_jni_fts5_Fts5ExtensionApi + * Method: xUserData + * Signature: (Lorg/sqlite/jni/fts5/Fts5Context;)Ljava/lang/Object; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_Fts5ExtensionApi_xUserData + (JNIEnv *, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_fts5_api */ + +#ifndef _Included_org_sqlite_jni_fts5_fts5_api +#define _Included_org_sqlite_jni_fts5_fts5_api +#ifdef __cplusplus +extern "C" { +#endif +#undef org_sqlite_jni_fts5_fts5_api_iVersion +#define org_sqlite_jni_fts5_fts5_api_iVersion 2L +/* + * Class: org_sqlite_jni_fts5_fts5_api + * Method: getInstanceForDb + * Signature: (Lorg/sqlite/jni/capi/sqlite3;)Lorg/sqlite/jni/fts5/fts5_api; + */ +JNIEXPORT jobject JNICALL Java_org_sqlite_jni_fts5_fts5_1api_getInstanceForDb + (JNIEnv *, jclass, jobject); + +/* + * Class: org_sqlite_jni_fts5_fts5_api + * Method: xCreateFunction + * Signature: (Ljava/lang/String;Ljava/lang/Object;Lorg/sqlite/jni/fts5/fts5_extension_function;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_fts5_1api_xCreateFunction + (JNIEnv *, jobject, jstring, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_sqlite_jni_fts5_fts5_tokenizer */ + +#ifndef _Included_org_sqlite_jni_fts5_fts5_tokenizer +#define _Included_org_sqlite_jni_fts5_fts5_tokenizer +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_sqlite_jni_fts5_fts5_tokenizer + * Method: xTokenize + * Signature: (Lorg/sqlite/jni/fts5/Fts5Tokenizer;I[BLorg/sqlite/jni/fts5/XTokenizeCallback;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_fts5_fts5_1tokenizer_xTokenize + (JNIEnv *, jobject, jobject, jint, jbyteArray, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/ext/jni/src/org/sqlite/jni/annotation/Experimental.java b/ext/jni/src/org/sqlite/jni/annotation/Experimental.java new file mode 100644 index 0000000000..190435c858 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/Experimental.java @@ -0,0 +1,30 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file houses the Experimental annotation for the sqlite3 C API. +*/ +package org.sqlite.jni.annotation; +import java.lang.annotation.*; + +/** + This annotation is for flagging methods, constructors, and types + which are expressly experimental and subject to any amount of + change or outright removal. Client code should not rely on such + features. +*/ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target({ + ElementType.METHOD, + ElementType.CONSTRUCTOR, + ElementType.TYPE +}) +public @interface Experimental{} diff --git a/ext/jni/src/org/sqlite/jni/annotation/NotNull.java b/ext/jni/src/org/sqlite/jni/annotation/NotNull.java new file mode 100644 index 0000000000..2873082446 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/NotNull.java @@ -0,0 +1,71 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file houses the NotNull annotation for the sqlite3 C API. +*/ +package org.sqlite.jni.annotation; +import java.lang.annotation.*; + +/** + This annotation is for flagging parameters which may not legally be + null or point to closed/finalized C-side resources. + +

    In the case of Java types which map directly to C struct types + (e.g. {@link org.sqlite.jni.capi.sqlite3}, {@link + org.sqlite.jni.capi.sqlite3_stmt}, and {@link + org.sqlite.jni.capi.sqlite3_context}), a closed/finalized resource + is also considered to be null for purposes this annotation because + the C-side effect of passing such a handle is the same as if null + is passed.

    + +

    When used in the context of Java interfaces which are called + from the C APIs, this annotation communicates that the C API will + never pass a null value to the callback for that parameter.

    + +

    Passing a null, for this annotation's definition of null, for + any parameter marked with this annotation specifically invokes + undefined behavior (see below).

    + +

    Passing 0 (i.e. C NULL) or a negative value for any long-type + parameter marked with this annotation specifically invokes undefined + behavior (see below). Such values are treated as C pointers in the + JNI layer.

    + +

    Undefined behaviour: the JNI build uses the {@code + SQLITE_ENABLE_API_ARMOR} build flag, meaning that the C code + invoked with invalid NULL pointers and the like will not invoke + undefined behavior in the conventional C sense, but may, for + example, return result codes which are not documented for the + affected APIs or may otherwise behave unpredictably. In no known + cases will such arguments result in C-level code dereferencing a + NULL pointer or accessing out-of-bounds (or otherwise invalid) + memory. In other words, they may cause unexpected behavior but + should never cause an outright crash or security issue.

    + +

    Note that the C-style API does not throw any exceptions on its + own because it has a no-throw policy in order to retain its C-style + semantics, but it may trigger NullPointerExceptions (or similar) if + passed a null for a parameter flagged with this annotation.

    + +

    This annotation is informational only. No policy is in place to + programmatically ensure that NotNull is conformed to in client + code.

    + +

    This annotation is solely for the use by the classes in the + org.sqlite.jni package and subpackages, but is made public so that + javadoc will link to it from the annotated functions. It is not + part of the public API and client-level code must not rely on + it.

    +*/ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target(ElementType.PARAMETER) +public @interface NotNull{} diff --git a/ext/jni/src/org/sqlite/jni/annotation/Nullable.java b/ext/jni/src/org/sqlite/jni/annotation/Nullable.java new file mode 100644 index 0000000000..e3fa30efc9 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/Nullable.java @@ -0,0 +1,33 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file houses the Nullable annotation for the sqlite3 C API. +*/ +package org.sqlite.jni.annotation; +import java.lang.annotation.*; + +/** + This annotation is for flagging parameters which may legally be + null, noting that they may behave differently if passed null but + are prepared to expect null as a value. When used in the context of + callback methods which are called into from the C APIs, this + annotation communicates that the C API may pass a null value to the + callback. + +

    This annotation is solely for the use by the classes in this + package but is made public so that javadoc will link to it from the + annotated functions. It is not part of the public API and + client-level code must not rely on it. +*/ +@Documented +@Retention(RetentionPolicy.SOURCE) +@Target(ElementType.PARAMETER) +public @interface Nullable{} diff --git a/ext/jni/src/org/sqlite/jni/annotation/package-info.java b/ext/jni/src/org/sqlite/jni/annotation/package-info.java new file mode 100644 index 0000000000..20ac7a3017 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/annotation/package-info.java @@ -0,0 +1,17 @@ +/* +** 2023-09-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +/** + This package houses annotations specific to the JNI bindings of the + SQLite3 C API. +*/ +package org.sqlite.jni.annotation; diff --git a/ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java b/ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java new file mode 100644 index 0000000000..925536636e --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java @@ -0,0 +1,34 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.NotNull; + +/** + An implementation of {@link CollationCallback} which provides a + no-op xDestroy() method. +*/ +public abstract class AbstractCollationCallback + implements CollationCallback, XDestroyCallback { + /** + Must compare the given byte arrays and return the result using + {@code memcmp()} semantics. + */ + public abstract int call(@NotNull byte[] lhs, @NotNull byte[] rhs); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. This implementation does nothing. + */ + public void xDestroy(){} +} diff --git a/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java b/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java new file mode 100644 index 0000000000..912f6ed5b5 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java @@ -0,0 +1,138 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + + +/** + A SQLFunction implementation for aggregate functions. Its T is the + data type of its "accumulator" state, an instance of which is + intended to be be managed using the getAggregateState() and + takeAggregateState() methods. +*/ +public abstract class AggregateFunction implements SQLFunction { + + /** + As for the xStep() argument of the C API's + sqlite3_create_function(). If this function throws, the + exception is not propagated and a warning might be emitted to a + debugging channel. + */ + public abstract void xStep(sqlite3_context cx, sqlite3_value[] args); + + /** + As for the xFinal() argument of the C API's sqlite3_create_function(). + If this function throws, it is translated into an sqlite3_result_error(). + */ + public abstract void xFinal(sqlite3_context cx); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. + */ + public void xDestroy() {} + + /** + PerContextState assists aggregate and window functions in + managing their accumulator state across calls to the UDF's + callbacks. + +

    T must be of a type which can be legally stored as a value in + java.util.HashMap. + +

    If a given aggregate or window function is called multiple times + in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)..., + then the clients need some way of knowing which call is which so + that they can map their state between their various UDF callbacks + and reset it via xFinal(). This class takes care of such + mappings. + +

    This class works by mapping + sqlite3_context.getAggregateContext() to a single piece of + state, of a client-defined type (the T part of this class), which + persists across a "matching set" of the UDF's callbacks. + +

    This class is a helper providing commonly-needed functionality + - it is not required for use with aggregate or window functions. + Client UDFs are free to perform such mappings using custom + approaches. The provided {@link AggregateFunction} and {@link + WindowFunction} classes use this. + */ + public static final class PerContextState { + private final java.util.Map> map + = new java.util.HashMap<>(); + + /** + Should be called from a UDF's xStep(), xValue(), and xInverse() + methods, passing it that method's first argument and an initial + value for the persistent state. If there is currently no + mapping for the given context within the map, one is created + using the given initial value, else the existing one is used + and the 2nd argument is ignored. It returns a ValueHolder + which can be used to modify that state directly without + requiring that the client update the underlying map's entry. + +

    The caller is obligated to eventually call + takeAggregateState() to clear the mapping. + */ + public ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ + final Long key = cx.getAggregateContext(true); + ValueHolder rc = null==key ? null : map.get(key); + if( null==rc ){ + map.put(key, rc = new ValueHolder<>(initialValue)); + } + return rc; + } + + /** + Should be called from a UDF's xFinal() method and passed that + method's first argument. This function removes the value + associated with cx.getAggregateContext() from the map and + returns it, returning null if no other UDF method has been + called to set up such a mapping. The latter condition will be + the case if a UDF is used in a statement which has no result + rows. + */ + public T takeAggregateState(sqlite3_context cx){ + final ValueHolder h = map.remove(cx.getAggregateContext(false)); + return null==h ? null : h.value; + } + } + + /** Per-invocation state for the UDF. */ + private final PerContextState map = new PerContextState<>(); + + /** + To be called from the implementation's xStep() method, as well + as the xValue() and xInverse() methods of the {@link WindowFunction} + subclass, to fetch the current per-call UDF state. On the + first call to this method for any given sqlite3_context + argument, the context is set to the given initial value. On all other + calls, the 2nd argument is ignored. + + @see AggregateFunction.PerContextState#getAggregateState + */ + protected final ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ + return map.getAggregateState(cx, initialValue); + } + + /** + To be called from the implementation's xFinal() method to fetch + the final state of the UDF and remove its mapping. + + see AggregateFunction.PerContextState#takeAggregateState + */ + protected final T takeAggregateState(sqlite3_context cx){ + return map.takeAggregateState(cx); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java b/ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java new file mode 100644 index 0000000000..298e3a5900 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java @@ -0,0 +1,29 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.*; + +/** + Callback for use with {@link CApi#sqlite3_set_authorizer}. +*/ +public interface AuthorizerCallback extends CallbackProxy { + /** + Must function as described for the C-level + sqlite3_set_authorizer() callback. If it throws, the error is + converted to a db-level error and the exception is suppressed. + */ + int call(int opId, @Nullable String s1, @Nullable String s2, + @Nullable String s3, @Nullable String s4); + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java b/ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java new file mode 100644 index 0000000000..7a54132d29 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java @@ -0,0 +1,40 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with the {@link CApi#sqlite3_auto_extension} + family of APIs. +*/ +public interface AutoExtensionCallback extends CallbackProxy { + /** + Must function as described for a C-level + sqlite3_auto_extension() callback. + +

    This callback may throw and the exception's error message will + be set as the db's error string. + +

    Tips for implementations: + +

    - Opening a database from an auto-extension handler will lead to + an endless recursion of the auto-handler triggering itself + indirectly for each newly-opened database. + +

    - If this routine is stateful, it may be useful to make the + overridden method synchronized. + +

    - Results are undefined if the given db is closed by an auto-extension. + */ + int call(sqlite3 db); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java b/ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java new file mode 100644 index 0000000000..00223f0b66 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_busy_handler}. +*/ +public interface BusyHandlerCallback extends CallbackProxy { + /** + Must function as documented for the C-level + sqlite3_busy_handler() callback argument, minus the (void*) + argument the C-level function requires. + */ + int call(int n); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java new file mode 100644 index 0000000000..0b840c3623 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -0,0 +1,2897 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file declares the main JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import java.util.Arrays; +import java.nio.charset.StandardCharsets; +import org.sqlite.jni.annotation.*; + +/** + This class contains the entire C-style sqlite3 JNI API binding, + minus a few bits and pieces declared in other files. For client-side + use, a static import is recommended: + + 

    {@code
+  import static org.sqlite.jni.capi.CApi.*;
+  }
    + +

    The C-side part can be found in sqlite3-jni.c. + +

    Only functions which materially differ from their C counterparts + are documented here, and only those material differences are + documented. The C documentation is otherwise applicable for these + APIs: + +

    https://sqlite.org/c3ref/intro.html + +

    A handful of Java-specific APIs have been added which are + documented here. A number of convenience overloads are provided + which are not documented but whose semantics map 1-to-1 in an + intuitive manner. e.g. {@link + #sqlite3_result_set(sqlite3_context,int)} is equivalent to {@link + #sqlite3_result_int}, and sqlite3_result_set() has many + type-specific overloads. + +

    Notes regarding Java's Modified UTF-8 vs standard UTF-8: + +

    SQLite internally uses UTF-8 encoding, whereas Java natively uses + UTF-16. Java JNI has routines for converting to and from UTF-8, + but JNI uses what its docs call modified UTF-8 (see links below) + Care must be taken when converting Java strings to or from standard + UTF-8 to ensure that the proper conversion is performed. In short, + Java's {@code String.getBytes(StandardCharsets.UTF_8)} performs the proper + conversion in Java, and there are no JNI C APIs for that conversion + (JNI's {@code NewStringUTF()} requires its input to be in MUTF-8). + +

    The known consequences and limitations this discrepancy places on + the SQLite3 JNI binding include: + +

      + +
    • C functions which take C-style strings without a length argument + require special care when taking input from Java. In particular, + Java strings converted to byte arrays for encoding purposes are not + NUL-terminated, and conversion to a Java byte array must sometimes + be careful to add one. Functions which take a length do not require + this so long as the length is provided. Search the CApi class + for "\0" for examples. + +
    + +

    Further reading: + +

    https://stackoverflow.com/questions/57419723 +

    https://stackoverflow.com/questions/7921016 +

    https://itecnote.com/tecnote/java-getting-true-utf-8-characters-in-java-jni/ +

    https://docs.oracle.com/javase/8/docs/api/java/lang/Character.html#unicode +

    https://docs.oracle.com/javase/8/docs/api/java/io/DataInput.html#modified-utf-8 + +*/ +public final class CApi { + static { + System.loadLibrary("sqlite3-jni"); + } + //! Not used + private CApi(){} + //! Called from static init code. + private static native void init(); + + /** + Returns a nul-terminated copy of s as a UTF-8-encoded byte array, + or null if s is null. + */ + private static byte[] nulTerminateUtf8(String s){ + return null==s ? null : (s+"\0").getBytes(StandardCharsets.UTF_8); + } + + /** + Each thread which uses the SQLite3 JNI APIs should call + sqlite3_jni_uncache_thread() when it is done with the library - + either right before it terminates or when it finishes using the + SQLite API. This will clean up any cached per-thread info. + +

    This process does not close any databases or finalize + any prepared statements because their ownership does not depend on + a given thread. For proper library behavior, and to + avoid C-side leaks, be sure to finalize all statements and close + all databases before calling this function. + +

    Calling this from the main application thread is not strictly + required. Additional threads must call this before ending or they + will leak cache entries in the C heap, which in turn may keep + numerous Java-side global references active. + +

    This routine returns false without side effects if the current + JNIEnv is not cached, else returns true, but this information is + primarily for testing of the JNI bindings and is not information + which client-level code can use to make any informed + decisions. Its return type and semantics are not considered + stable and may change at any time. + */ + public static native boolean sqlite3_java_uncache_thread(); + + /** + Returns true if this JVM has JNI-level support for C-level direct + memory access using java.nio.ByteBuffer, else returns false. + */ + @Experimental + public static native boolean sqlite3_jni_supports_nio(); + + /** + For internal use only. Sets the given db's error code and + (optionally) string. If rc is 0, it defaults to SQLITE_ERROR. + + On success it returns rc. On error it may return a more serious + code, such as SQLITE_NOMEM. Returns SQLITE_MISUSE if db is null. + */ + static native int sqlite3_jni_db_error(@NotNull sqlite3 db, + int rc, @Nullable String msg); + + /** + Convenience overload which uses e.toString() as the error + message. + */ + static int sqlite3_jni_db_error(@NotNull sqlite3 db, + int rc, @NotNull Exception e){ + return sqlite3_jni_db_error(db, rc, e.toString()); + } + + ////////////////////////////////////////////////////////////////////// + // Maintenance reminder: please keep the sqlite3_.... functions + // alphabetized. The SQLITE_... values. on the other hand, are + // grouped by category. + + /** + Functions exactly like the native form except that (A) the 2nd + argument is a boolean instead of an int and (B) the returned + value is not a pointer address and is only intended for use as a + per-UDF-call lookup key in a higher-level data structure. + +

    Passing a true second argument is analogous to passing some + unspecified small, non-0 positive value to the C API and passing + false is equivalent to passing 0 to the C API. + +

    Like the C API, it returns 0 if allocation fails or if + initialize is false and no prior aggregate context was allocated + for cx. If initialize is true then it returns 0 only on + allocation error. In all cases, 0 is considered the sentinel + "not a key" value. + */ + public static native long sqlite3_aggregate_context(sqlite3_context cx, boolean initialize); + + /** + Functions almost as documented for the C API, with these + exceptions: + +

    - The callback interface is shorter because of + cross-language differences. Specifically, 3rd argument to the C + auto-extension callback interface is unnecessary here. + +

    The C API docs do not specifically say so, but if the list of + auto-extensions is manipulated from an auto-extension, it is + undefined which, if any, auto-extensions will subsequently + execute for the current database. That is, doing so will result + in unpredictable, but not undefined, behavior. + +

    See the AutoExtension class docs for more information. + */ + public static native int sqlite3_auto_extension(@NotNull AutoExtensionCallback callback); + + private static native int sqlite3_backup_finish(@NotNull long ptrToBackup); + + public static int sqlite3_backup_finish(@NotNull sqlite3_backup b){ + return null==b ? 0 : sqlite3_backup_finish(b.clearNativePointer()); + } + + private static native sqlite3_backup sqlite3_backup_init( + @NotNull long ptrToDbDest, @NotNull String destSchemaName, + @NotNull long ptrToDbSrc, @NotNull String srcSchemaName + ); + + public static sqlite3_backup sqlite3_backup_init( + @NotNull sqlite3 dbDest, @NotNull String destSchemaName, + @NotNull sqlite3 dbSrc, @NotNull String srcSchemaName + ){ + return sqlite3_backup_init( dbDest.getNativePointer(), destSchemaName, + dbSrc.getNativePointer(), srcSchemaName ); + } + + private static native int sqlite3_backup_pagecount(@NotNull long ptrToBackup); + + public static int sqlite3_backup_pagecount(@NotNull sqlite3_backup b){ + return sqlite3_backup_pagecount(b.getNativePointer()); + } + + private static native int sqlite3_backup_remaining(@NotNull long ptrToBackup); + + public static int sqlite3_backup_remaining(@NotNull sqlite3_backup b){ + return sqlite3_backup_remaining(b.getNativePointer()); + } + + private static native int sqlite3_backup_step(@NotNull long ptrToBackup, int nPage); + + public static int sqlite3_backup_step(@NotNull sqlite3_backup b, int nPage){ + return sqlite3_backup_step(b.getNativePointer(), nPage); + } + + private static native int sqlite3_bind_blob( + @NotNull long ptrToStmt, int ndx, @Nullable byte[] data, int n + ); + + /** + If n is negative, SQLITE_MISUSE is returned. If n>data.length + then n is silently truncated to data.length. + */ + public static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data, int n + ){ + return sqlite3_bind_blob(stmt.getNativePointer(), ndx, data, n); + } + + public static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data + ){ + return (null==data) + ? sqlite3_bind_null(stmt.getNativePointer(), ndx) + : sqlite3_bind_blob(stmt.getNativePointer(), ndx, data, data.length); + } + + /** + Convenience overload which is a simple proxy for + sqlite3_bind_nio_buffer(). + */ + @Experimental + /*public*/ static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data, + int begin, int n + ){ + return sqlite3_bind_nio_buffer(stmt, ndx, data, begin, n); + } + + /** + Convenience overload which is equivalent to passing its arguments + to sqlite3_bind_nio_buffer() with the values 0 and -1 for the + final two arguments. + */ + @Experimental + /*public*/ static int sqlite3_bind_blob( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data + ){ + return sqlite3_bind_nio_buffer(stmt, ndx, data, 0, -1); + } + + private static native int sqlite3_bind_double( + @NotNull long ptrToStmt, int ndx, double v + ); + + public static int sqlite3_bind_double( + @NotNull sqlite3_stmt stmt, int ndx, double v + ){ + return sqlite3_bind_double(stmt.getNativePointer(), ndx, v); + } + + private static native int sqlite3_bind_int( + @NotNull long ptrToStmt, int ndx, int v + ); + + public static int sqlite3_bind_int( + @NotNull sqlite3_stmt stmt, int ndx, int v + ){ + return sqlite3_bind_int(stmt.getNativePointer(), ndx, v); + } + + private static native int sqlite3_bind_int64( + @NotNull long ptrToStmt, int ndx, long v + ); + + public static int sqlite3_bind_int64(@NotNull sqlite3_stmt stmt, int ndx, long v){ + return sqlite3_bind_int64( stmt.getNativePointer(), ndx, v ); + } + + private static native int sqlite3_bind_java_object( + @NotNull long ptrToStmt, int ndx, @Nullable Object o + ); + + /** + Binds the contents of the given buffer object as a blob. + + The byte range of the buffer may be restricted by providing a + start index and a number of bytes. beginPos may not be negative. + Negative howMany is interpreted as the remainder of the buffer + past the given start position, up to the buffer's limit() (as + opposed its capacity()). + + If beginPos+howMany would extend past the limit() of the buffer + then SQLITE_ERROR is returned. + + If any of the following are true, this function behaves like + sqlite3_bind_null(): the buffer is null, beginPos is at or past + the end of the buffer, howMany is 0, or the calculated slice of + the blob has a length of 0. + + If ndx is out of range, it returns SQLITE_RANGE, as documented + for sqlite3_bind_blob(). If beginPos is negative or if + sqlite3_jni_supports_nio() returns false then SQLITE_MISUSE is + returned. Note that this function is bound (as it were) by the + SQLITE_LIMIT_LENGTH constraint and SQLITE_TOOBIG is returned if + the resulting slice of the buffer exceeds that limit. + + This function does not modify the buffer's streaming-related + cursors. + + If the buffer is modified in a separate thread while this + operation is running, results are undefined and will likely + result in corruption of the bound data or a segmentation fault. + + Design note: this function should arguably take a java.nio.Buffer + instead of ByteBuffer, but it can only operate on "direct" + buffers and the only such class offered by Java is (apparently) + ByteBuffer. + + @see https://docs.oracle.com/javase/8/docs/api/java/nio/Buffer.html + */ + @Experimental + /*public*/ static native int sqlite3_bind_nio_buffer( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data, + int beginPos, int howMany + ); + + /** + Convenience overload which binds the given buffer's entire + contents, up to its limit() (as opposed to its capacity()). + */ + @Experimental + /*public*/ static int sqlite3_bind_nio_buffer( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable java.nio.ByteBuffer data + ){ + return sqlite3_bind_nio_buffer(stmt, ndx, data, 0, -1); + } + + /** + Binds the given object at the given index. If o is null then this behaves like + sqlite3_bind_null(). + + @see #sqlite3_result_java_object + */ + public static int sqlite3_bind_java_object( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable Object o + ){ + return sqlite3_bind_java_object(stmt.getNativePointer(), ndx, o); + } + + private static native int sqlite3_bind_null(@NotNull long ptrToStmt, int ndx); + + public static int sqlite3_bind_null(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_bind_null(stmt.getNativePointer(), ndx); + } + + private static native int sqlite3_bind_parameter_count(@NotNull long ptrToStmt); + + public static int sqlite3_bind_parameter_count(@NotNull sqlite3_stmt stmt){ + return sqlite3_bind_parameter_count(stmt.getNativePointer()); + } + + /** + Requires that paramName be a NUL-terminated UTF-8 string. + + This overload is private because: (A) to keep users from + inadvertently passing non-NUL-terminated byte arrays (an easy + thing to do). (B) it is cheaper to NUL-terminate the + String-to-byte-array conversion in the public-facing Java-side + overload than to do that in C, so that signature is the + public-facing one. + */ + private static native int sqlite3_bind_parameter_index( + @NotNull long ptrToStmt, @NotNull byte[] paramName + ); + + public static int sqlite3_bind_parameter_index( + @NotNull sqlite3_stmt stmt, @NotNull String paramName + ){ + final byte[] utf8 = nulTerminateUtf8(paramName); + return null==utf8 ? 0 : sqlite3_bind_parameter_index(stmt.getNativePointer(), utf8); + } + + private static native String sqlite3_bind_parameter_name( + @NotNull long ptrToStmt, int index + ); + + public static String sqlite3_bind_parameter_name(@NotNull sqlite3_stmt stmt, int index){ + return sqlite3_bind_parameter_name(stmt.getNativePointer(), index); + } + + private static native int sqlite3_bind_text( + @NotNull long ptrToStmt, int ndx, @Nullable byte[] utf8, int maxBytes + ); + + /** + Works like the C-level sqlite3_bind_text() but assumes + SQLITE_TRANSIENT for the final C API parameter. The byte array is + assumed to be in UTF-8 encoding. + +

    If data is not null and maxBytes>utf8.length then maxBytes is + silently truncated to utf8.length. If maxBytes is negative then + results are undefined if data is not null and does not contain a + NUL byte. + */ + static int sqlite3_bind_text( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] utf8, int maxBytes + ){ + return sqlite3_bind_text(stmt.getNativePointer(), ndx, utf8, maxBytes); + } + + /** + Converts data, if not null, to a UTF-8-encoded byte array and + binds it as such, returning the result of the C-level + sqlite3_bind_null() or sqlite3_bind_text(). + */ + public static int sqlite3_bind_text( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable String data + ){ + if( null==data ) return sqlite3_bind_null(stmt.getNativePointer(), ndx); + final byte[] utf8 = data.getBytes(StandardCharsets.UTF_8); + return sqlite3_bind_text(stmt.getNativePointer(), ndx, utf8, utf8.length); + } + + /** + Requires that utf8 be null or in UTF-8 encoding. + */ + public static int sqlite3_bind_text( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] utf8 + ){ + return ( null==utf8 ) + ? sqlite3_bind_null(stmt.getNativePointer(), ndx) + : sqlite3_bind_text(stmt.getNativePointer(), ndx, utf8, utf8.length); + } + + private static native int sqlite3_bind_text16( + @NotNull long ptrToStmt, int ndx, @Nullable byte[] data, int maxBytes + ); + + /** + Identical to the sqlite3_bind_text() overload with the same + signature but requires that its input be encoded in UTF-16 in + platform byte order. + */ + static int sqlite3_bind_text16( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data, int maxBytes + ){ + return sqlite3_bind_text16(stmt.getNativePointer(), ndx, data, maxBytes); + } + + /** + Converts its string argument to UTF-16 and binds it as such, returning + the result of the C-side function of the same name. The 3rd argument + may be null. + */ + public static int sqlite3_bind_text16( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable String data + ){ + if(null == data) return sqlite3_bind_null(stmt, ndx); + final byte[] bytes = data.getBytes(StandardCharsets.UTF_16); + return sqlite3_bind_text16(stmt.getNativePointer(), ndx, bytes, bytes.length); + } + + /** + Requires that data be null or in UTF-16 encoding in platform byte + order. Returns the result of the C-level sqlite3_bind_null() or + sqlite3_bind_text16(). + */ + public static int sqlite3_bind_text16( + @NotNull sqlite3_stmt stmt, int ndx, @Nullable byte[] data + ){ + return (null == data) + ? sqlite3_bind_null(stmt.getNativePointer(), ndx) + : sqlite3_bind_text16(stmt.getNativePointer(), ndx, data, data.length); + } + + private static native int sqlite3_bind_value(@NotNull long ptrToStmt, int ndx, long ptrToValue); + + /** + Functions like the C-level sqlite3_bind_value(), or + sqlite3_bind_null() if val is null. + */ + public static int sqlite3_bind_value(@NotNull sqlite3_stmt stmt, int ndx, sqlite3_value val){ + return sqlite3_bind_value(stmt.getNativePointer(), ndx, + null==val ? 0L : val.getNativePointer()); + } + + private static native int sqlite3_bind_zeroblob(@NotNull long ptrToStmt, int ndx, int n); + + public static int sqlite3_bind_zeroblob(@NotNull sqlite3_stmt stmt, int ndx, int n){ + return sqlite3_bind_zeroblob(stmt.getNativePointer(), ndx, n); + } + + private static native int sqlite3_bind_zeroblob64( + @NotNull long ptrToStmt, int ndx, long n + ); + + public static int sqlite3_bind_zeroblob64(@NotNull sqlite3_stmt stmt, int ndx, long n){ + return sqlite3_bind_zeroblob64(stmt.getNativePointer(), ndx, n); + } + + private static native int sqlite3_blob_bytes(@NotNull long ptrToBlob); + + public static int sqlite3_blob_bytes(@NotNull sqlite3_blob blob){ + return sqlite3_blob_bytes(blob.getNativePointer()); + } + + private static native int sqlite3_blob_close(@Nullable long ptrToBlob); + + public static int sqlite3_blob_close(@Nullable sqlite3_blob blob){ + return null==blob ? 0 : sqlite3_blob_close(blob.clearNativePointer()); + } + + private static native int sqlite3_blob_open( + @NotNull long ptrToDb, @NotNull String dbName, + @NotNull String tableName, @NotNull String columnName, + long iRow, int flags, @NotNull OutputPointer.sqlite3_blob out + ); + + public static int sqlite3_blob_open( + @NotNull sqlite3 db, @NotNull String dbName, + @NotNull String tableName, @NotNull String columnName, + long iRow, int flags, @NotNull OutputPointer.sqlite3_blob out + ){ + return sqlite3_blob_open(db.getNativePointer(), dbName, tableName, + columnName, iRow, flags, out); + } + + /** + Convenience overload. + */ + public static sqlite3_blob sqlite3_blob_open( + @NotNull sqlite3 db, @NotNull String dbName, + @NotNull String tableName, @NotNull String columnName, + long iRow, int flags ){ + final OutputPointer.sqlite3_blob out = new OutputPointer.sqlite3_blob(); + sqlite3_blob_open(db.getNativePointer(), dbName, tableName, columnName, + iRow, flags, out); + return out.take(); + } + + private static native int sqlite3_blob_read( + @NotNull long ptrToBlob, @NotNull byte[] target, int srcOffset + ); + + /** + As per C's sqlite3_blob_read(), but writes its output to the + given byte array. Note that the final argument is the offset of + the source buffer, not the target array. + */ + public static int sqlite3_blob_read( + @NotNull sqlite3_blob src, @NotNull byte[] target, int srcOffset + ){ + return sqlite3_blob_read(src.getNativePointer(), target, srcOffset); + } + + /** + An internal level of indirection. + */ + @Experimental + private static native int sqlite3_blob_read_nio_buffer( + @NotNull long ptrToBlob, int srcOffset, + @NotNull java.nio.ByteBuffer tgt, int tgtOffset, int howMany + ); + + /** + Reads howMany bytes from offset srcOffset of src into position + tgtOffset of tgt. + + Returns SQLITE_MISUSE if src is null, tgt is null, or + sqlite3_jni_supports_nio() returns false. Returns SQLITE_ERROR if + howMany or either offset are negative. If argument validation + succeeds, it returns the result of the underlying call to + sqlite3_blob_read() (0 on success). + */ + @Experimental + /*public*/ static int sqlite3_blob_read_nio_buffer( + @NotNull sqlite3_blob src, int srcOffset, + @NotNull java.nio.ByteBuffer tgt, int tgtOffset, int howMany + ){ + return (JNI_SUPPORTS_NIO && src!=null && tgt!=null) + ? sqlite3_blob_read_nio_buffer( + src.getNativePointer(), srcOffset, tgt, tgtOffset, howMany + ) + : SQLITE_MISUSE; + } + + /** + Convenience overload which reads howMany bytes from position + srcOffset of src and returns the result as a new ByteBuffer. + + srcOffset may not be negative. If howMany is negative, it is + treated as all bytes following srcOffset. + + Returns null if sqlite3_jni_supports_nio(), any arguments are + invalid, if the number of bytes to read is 0 or is larger than + the src blob, or the underlying call to sqlite3_blob_read() fails + for any reason. + */ + @Experimental + /*public*/ static java.nio.ByteBuffer sqlite3_blob_read_nio_buffer( + @NotNull sqlite3_blob src, int srcOffset, int howMany + ){ + if( !JNI_SUPPORTS_NIO || src==null ) return null; + else if( srcOffset<0 ) return null; + final int nB = sqlite3_blob_bytes(src); + if( srcOffset>=nB ) return null; + else if( howMany<0 ) howMany = nB - srcOffset; + if( srcOffset + howMany > nB ) return null; + final java.nio.ByteBuffer tgt = + java.nio.ByteBuffer.allocateDirect(howMany); + final int rc = sqlite3_blob_read_nio_buffer( + src.getNativePointer(), srcOffset, tgt, 0, howMany + ); + return 0==rc ? tgt : null; + } + + /** + Overload alias for sqlite3_blob_read_nio_buffer(). + */ + @Experimental + /*public*/ static int sqlite3_blob_read( + @NotNull sqlite3_blob src, int srcOffset, + @NotNull java.nio.ByteBuffer tgt, + int tgtOffset, int howMany + ){ + return sqlite3_blob_read_nio_buffer( + src, srcOffset, tgt, tgtOffset, howMany + ); + } + + /** + Convenience overload which uses 0 for both src and tgt offsets + and reads a number of bytes equal to the smaller of + sqlite3_blob_bytes(src) and tgt.limit(). + + On success it sets tgt.limit() to the number of bytes read. On + error, tgt.limit() is not modified. + + Returns 0 on success. Returns SQLITE_MISUSE is either argument is + null or sqlite3_jni_supports_nio() returns false. Else it returns + the result of the underlying call to sqlite3_blob_read(). + */ + @Experimental + /*public*/ static int sqlite3_blob_read( + @NotNull sqlite3_blob src, + @NotNull java.nio.ByteBuffer tgt + ){ + if(!JNI_SUPPORTS_NIO || src==null || tgt==null) return SQLITE_MISUSE; + final int nSrc = sqlite3_blob_bytes(src); + final int nTgt = tgt.limit(); + final int nRead = nTgt T sqlite3_column_java_object( + @NotNull sqlite3_stmt stmt, int ndx, @NotNull Class type + ){ + final Object o = sqlite3_column_java_object(stmt, ndx); + return type.isInstance(o) ? (T)o : null; + } + + private static native String sqlite3_column_name(@NotNull long ptrToStmt, int ndx); + + public static String sqlite3_column_name(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_name(stmt.getNativePointer(), ndx); + } + + /** + A variant of sqlite3_column_blob() which returns the blob as a + ByteBuffer object. Returns null if its argument is null, if + sqlite3_jni_supports_nio() is false, or if sqlite3_column_blob() + would return null for the same inputs. + */ + @Experimental + /*public*/ static native java.nio.ByteBuffer sqlite3_column_nio_buffer( + @NotNull sqlite3_stmt stmt, int ndx + ); + + private static native String sqlite3_column_origin_name(@NotNull long ptrToStmt, int ndx); + + /** + Only available if built with SQLITE_ENABLE_COLUMN_METADATA. + */ + public static String sqlite3_column_origin_name(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_origin_name(stmt.getNativePointer(), ndx); + } + + private static native String sqlite3_column_table_name(@NotNull long ptrToStmt, int ndx); + + /** + Only available if built with SQLITE_ENABLE_COLUMN_METADATA. + */ + public static String sqlite3_column_table_name(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_table_name(stmt.getNativePointer(), ndx); + } + + /** + Functions identially to the C API, and this note is just to + stress that the returned bytes are encoded as UTF-8. It returns + null if the underlying C-level sqlite3_column_text() returns NULL + or on allocation error. + + @see #sqlite3_column_text16(sqlite3_stmt,int) + */ + public static native byte[] sqlite3_column_text( + @NotNull sqlite3_stmt stmt, int ndx + ); + + public static native String sqlite3_column_text16( + @NotNull sqlite3_stmt stmt, int ndx + ); + + // The real utility of this function is questionable. + // /** + // Returns a Java value representation based on the value of + // sqlite_value_type(). For integer types it returns either Integer + // or Long, depending on whether the value will fit in an + // Integer. For floating-point values it always returns type Double. + + // If the column was bound using sqlite3_result_java_object() then + // that value, as an Object, is returned. + // */ + // public static Object sqlite3_column_to_java(@NotNull sqlite3_stmt stmt, + // int ndx){ + // sqlite3_value v = sqlite3_column_value(stmt, ndx); + // Object rv = null; + // if(null == v) return v; + // v = sqlite3_value_dup(v)/*need a protected value*/; + // if(null == v) return v /* OOM error in C */; + // if(112/* 'p' */ == sqlite3_value_subtype(v)){ + // rv = sqlite3_value_java_object(v); + // }else{ + // switch(sqlite3_value_type(v)){ + // case SQLITE_INTEGER: { + // final long i = sqlite3_value_int64(v); + // rv = (i<=0x7fffffff && i>=-0x7fffffff-1) + // ? new Integer((int)i) : new Long(i); + // break; + // } + // case SQLITE_FLOAT: rv = new Double(sqlite3_value_double(v)); break; + // case SQLITE_BLOB: rv = sqlite3_value_blob(v); break; + // case SQLITE_TEXT: rv = sqlite3_value_text16(v); break; + // default: break; + // } + // } + // sqlite3_value_free(v); + // return rv; + // } + + private static native int sqlite3_column_type(@NotNull long ptrToStmt, int ndx); + + public static int sqlite3_column_type(@NotNull sqlite3_stmt stmt, int ndx){ + return sqlite3_column_type(stmt.getNativePointer(), ndx); + } + + public static native sqlite3_value sqlite3_column_value( + @NotNull sqlite3_stmt stmt, int ndx + ); + + private static native int sqlite3_collation_needed( + @NotNull long ptrToDb, @Nullable CollationNeededCallback callback + ); + + /** + This functions like C's sqlite3_collation_needed16() because + Java's string type is inherently compatible with that interface. + */ + public static int sqlite3_collation_needed( + @NotNull sqlite3 db, @Nullable CollationNeededCallback callback + ){ + return sqlite3_collation_needed(db.getNativePointer(), callback); + } + + private static native CommitHookCallback sqlite3_commit_hook( + @NotNull long ptrToDb, @Nullable CommitHookCallback hook + ); + + public static CommitHookCallback sqlite3_commit_hook( + @NotNull sqlite3 db, @Nullable CommitHookCallback hook + ){ + return sqlite3_commit_hook(db.getNativePointer(), hook); + } + + public static native String sqlite3_compileoption_get(int n); + + public static native boolean sqlite3_compileoption_used(String optName); + + /** + This implementation is private because it's too easy to pass it + non-NUL-terminated byte arrays from client code. + */ + private static native int sqlite3_complete( + @NotNull byte[] nulTerminatedUtf8Sql + ); + + /** + Unlike the C API, this returns SQLITE_MISUSE if its argument is + null (as opposed to invoking UB). + */ + public static int sqlite3_complete(@NotNull String sql){ + return sqlite3_complete( nulTerminateUtf8(sql) ); + } + + /** + Internal level of indirection for sqlite3_config(int). + */ + private static native int sqlite3_config__enable(int op); + + /** + Internal level of indirection for sqlite3_config(ConfigLogCallback). + */ + private static native int sqlite3_config__CONFIG_LOG( + @Nullable ConfigLogCallback logger + ); + + /** + Internal level of indirection for sqlite3_config(ConfigSqlLogCallback). + */ + private static native int sqlite3_config__SQLLOG( + @Nullable ConfigSqlLogCallback logger + ); + + /** +

    Works like in the C API with the exception that it only supports + the following subset of configuration flags: + +

    SQLITE_CONFIG_SINGLETHREAD + SQLITE_CONFIG_MULTITHREAD + SQLITE_CONFIG_SERIALIZED + +

    Others may be added in the future. It returns SQLITE_MISUSE if + given an argument it does not handle. + +

    Note that sqlite3_config() is not threadsafe with regards to + the rest of the library. This must not be called when any other + library APIs are being called. + */ + public static int sqlite3_config(int op){ + return sqlite3_config__enable(op); + } + + /** + If the native library was built with SQLITE_ENABLE_SQLLOG defined + then this acts as a proxy for C's + sqlite3_config(SQLITE_CONFIG_SQLLOG,...). This sets or clears the + logger. If installation of a logger fails, any previous logger is + retained. + +

    If not built with SQLITE_ENABLE_SQLLOG defined, this returns + SQLITE_MISUSE. + +

    Note that sqlite3_config() is not threadsafe with regards to + the rest of the library. This must not be called when any other + library APIs are being called. + */ + public static int sqlite3_config( @Nullable ConfigSqlLogCallback logger ){ + return sqlite3_config__SQLLOG(logger); + } + + /** + The sqlite3_config() overload for handling the SQLITE_CONFIG_LOG + option. + */ + public static int sqlite3_config( @Nullable ConfigLogCallback logger ){ + return sqlite3_config__CONFIG_LOG(logger); + } + + /** + Unlike the C API, this returns null if its argument is + null (as opposed to invoking UB). + */ + public static native sqlite3 sqlite3_context_db_handle( + @NotNull sqlite3_context cx + ); + + public static native int sqlite3_create_collation( + @NotNull sqlite3 db, @NotNull String name, int eTextRep, + @NotNull CollationCallback col + ); + + /** + The Java counterpart to the C-native sqlite3_create_function(), + sqlite3_create_function_v2(), and + sqlite3_create_window_function(). Which one it behaves like + depends on which methods the final argument implements. See + SQLFunction's subclasses (ScalarFunction, AggregateFunction, + and WindowFunction) for details. + +

    Unlike the C API, this returns SQLITE_MISUSE null if its db or + functionName arguments are null (as opposed to invoking UB). + */ + public static native int sqlite3_create_function( + @NotNull sqlite3 db, @NotNull String functionName, + int nArg, int eTextRep, @NotNull SQLFunction func + ); + + private static native int sqlite3_data_count(@NotNull long ptrToStmt); + + public static int sqlite3_data_count(@NotNull sqlite3_stmt stmt){ + return sqlite3_data_count(stmt.getNativePointer()); + } + + /** + Overload for sqlite3_db_config() calls which take (int,int*) + variadic arguments. Returns SQLITE_MISUSE if op is not one of the + SQLITE_DBCONFIG_... options which uses this call form. + +

    Unlike the C API, this returns SQLITE_MISUSE if its db argument + is null (as opposed to invoking UB). + */ + public static native int sqlite3_db_config( + @NotNull sqlite3 db, int op, int onOff, @Nullable OutputPointer.Int32 out + ); + + /** + Overload for sqlite3_db_config() calls which take a (const char*) + variadic argument. As of SQLite3 v3.43 the only such option is + SQLITE_DBCONFIG_MAINDBNAME. Returns SQLITE_MISUSE if op is not + SQLITE_DBCONFIG_MAINDBNAME, but that set of options may be + extended in future versions. + */ + public static native int sqlite3_db_config( + @NotNull sqlite3 db, int op, @NotNull String val + ); + + private static native String sqlite3_db_name(@NotNull long ptrToDb, int ndx); + + public static String sqlite3_db_name(@NotNull sqlite3 db, int ndx){ + return null==db ? null : sqlite3_db_name(db.getNativePointer(), ndx); + } + + public static native String sqlite3_db_filename( + @NotNull sqlite3 db, @NotNull String dbName + ); + + public static native sqlite3 sqlite3_db_handle(@NotNull sqlite3_stmt stmt); + + public static native int sqlite3_db_readonly(@NotNull sqlite3 db, String dbName); + + public static native int sqlite3_db_release_memory(sqlite3 db); + + public static native int sqlite3_db_status( + @NotNull sqlite3 db, int op, @NotNull OutputPointer.Int32 pCurrent, + @NotNull OutputPointer.Int32 pHighwater, boolean reset + ); + + public static native int sqlite3_errcode(@NotNull sqlite3 db); + + public static native String sqlite3_errmsg(@NotNull sqlite3 db); + + /** Added in 3.51.0. */ + public static native int sqlite3_set_errmsg(@NotNull sqlite3 db, + int resultCode, + String msg); + + private static native int sqlite3_error_offset(@NotNull long ptrToDb); + + /** + Caveat: the returned byte offset values assume UTF-8-encoded + inputs, so won't always match character offsets in Java Strings. + */ + public static int sqlite3_error_offset(@NotNull sqlite3 db){ + return sqlite3_error_offset(db.getNativePointer()); + } + + public static native String sqlite3_errstr(int resultCode); + + public static native String sqlite3_expanded_sql(@NotNull sqlite3_stmt stmt); + + private static native int sqlite3_extended_errcode(@NotNull long ptrToDb); + + public static int sqlite3_extended_errcode(@NotNull sqlite3 db){ + return sqlite3_extended_errcode(db.getNativePointer()); + } + + public static native int sqlite3_extended_result_codes( + @NotNull sqlite3 db, boolean on + ); + + private static native boolean sqlite3_get_autocommit(@NotNull long ptrToDb); + + public static boolean sqlite3_get_autocommit(@NotNull sqlite3 db){ + return sqlite3_get_autocommit(db.getNativePointer()); + } + + public static native Object sqlite3_get_auxdata( + @NotNull sqlite3_context cx, int n + ); + + private static native int sqlite3_finalize(long ptrToStmt); + + public static int sqlite3_finalize(@NotNull sqlite3_stmt stmt){ + return null==stmt ? 0 : sqlite3_finalize(stmt.clearNativePointer()); + } + + public static native int sqlite3_initialize(); + + public static native void sqlite3_interrupt(@NotNull sqlite3 db); + + public static native boolean sqlite3_is_interrupted(@NotNull sqlite3 db); + + public static native boolean sqlite3_keyword_check(@NotNull String word); + + public static native int sqlite3_keyword_count(); + + public static native String sqlite3_keyword_name(int index); + + + public static native long sqlite3_last_insert_rowid(@NotNull sqlite3 db); + + public static native String sqlite3_libversion(); + + public static native int sqlite3_libversion_number(); + + public static native int sqlite3_limit(@NotNull sqlite3 db, int id, int newVal); + + /** + Only available if built with SQLITE_ENABLE_NORMALIZE. If not, it always + returns null. + */ + public static native String sqlite3_normalized_sql(@NotNull sqlite3_stmt stmt); + + /** + Works like its C counterpart and makes the native pointer of the + underling (sqlite3*) object available via + ppDb.getNativePointer(). That pointer is necessary for looking up + the JNI-side native, but clients need not pay it any + heed. Passing the object to sqlite3_close() or sqlite3_close_v2() + will clear that pointer mapping. + +

    Recall that even if opening fails, the output pointer might be + non-null. Any error message about the failure will be in that + object and it is up to the caller to sqlite3_close() that + db handle. + */ + public static native int sqlite3_open( + @Nullable String filename, @NotNull OutputPointer.sqlite3 ppDb + ); + + /** + Convenience overload which returns its db handle directly. The returned + object might not have been successfully opened: use sqlite3_errcode() to + check whether it is in an error state. + +

    Ownership of the returned value is passed to the caller, who must eventually + pass it to sqlite3_close() or sqlite3_close_v2(). + */ + public static sqlite3 sqlite3_open(@Nullable String filename){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + sqlite3_open(filename, out); + return out.take(); + } + + public static native int sqlite3_open_v2( + @Nullable String filename, @NotNull OutputPointer.sqlite3 ppDb, + int flags, @Nullable String zVfs + ); + + /** + Has the same semantics as the sqlite3-returning sqlite3_open() + but uses sqlite3_open_v2() instead of sqlite3_open(). + */ + public static sqlite3 sqlite3_open_v2(@Nullable String filename, int flags, + @Nullable String zVfs){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + sqlite3_open_v2(filename, out, flags, zVfs); + return out.take(); + } + + /** + The sqlite3_prepare() family of functions require slightly + different signatures than their native counterparts, but (A) they + retain functionally equivalent semantics and (B) overloading + allows us to install several convenience forms. + +

    All of them which take their SQL in the form of a byte[] require + that it be in UTF-8 encoding unless explicitly noted otherwise. + +

    The forms which take a "tail" output pointer return (via that + output object) the index into their SQL byte array at which the + end of the first SQL statement processed by the call was + found. That's fundamentally how the C APIs work but making use of + that value requires more copying of the input SQL into + consecutively smaller arrays in order to consume all of + it. (There is an example of doing that in this project's Tester1 + class.) For that vast majority of uses, that capability is not + necessary, however, and overloads are provided which gloss over + that. + +

    Results are undefined if maxBytes>sqlUtf8.length. + +

    This routine is private because its maxBytes value is not + strictly necessary in the Java interface, as sqlUtf8.length tells + us the information we need. Making this public would give clients + more ways to shoot themselves in the foot without providing any + real utility. + */ + private static native int sqlite3_prepare( + @NotNull long ptrToDb, @NotNull byte[] sqlUtf8, int maxBytes, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ); + + /** + Works like the canonical sqlite3_prepare() but its "tail" output + argument is returned as the index offset into the given + UTF-8-encoded byte array at which SQL parsing stopped. The + semantics are otherwise identical to the C API counterpart. + +

    Several overloads provided simplified call signatures. + */ + public static int sqlite3_prepare( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ){ + return sqlite3_prepare(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, pTailOffset); + } + + public static int sqlite3_prepare( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + return sqlite3_prepare(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, null); + } + + public static int sqlite3_prepare( + @NotNull sqlite3 db, @NotNull String sql, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + final byte[] utf8 = sql.getBytes(StandardCharsets.UTF_8); + return sqlite3_prepare(db.getNativePointer(), utf8, utf8.length, + outStmt, null); + } + + /** + Convenience overload which returns its statement handle directly, + or null on error or when reading only whitespace or + comments. sqlite3_errcode() can be used to determine whether + there was an error or the input was empty. Ownership of the + returned object is passed to the caller, who must eventually pass + it to sqlite3_finalize(). + */ + public static sqlite3_stmt sqlite3_prepare( + @NotNull sqlite3 db, @NotNull String sql + ){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + sqlite3_prepare(db, sql, out); + return out.take(); + } + /** + @see #sqlite3_prepare + */ + private static native int sqlite3_prepare_v2( + @NotNull long ptrToDb, @NotNull byte[] sqlUtf8, int maxBytes, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ); + + /** + Works like the canonical sqlite3_prepare_v2() but its "tail" + output parameter is returned as the index offset into the given + byte array at which SQL parsing stopped. + */ + public static int sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ){ + return sqlite3_prepare_v2(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, pTailOffset); + } + + public static int sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + return sqlite3_prepare_v2(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + outStmt, null); + } + + public static int sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull String sql, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + final byte[] utf8 = sql.getBytes(StandardCharsets.UTF_8); + return sqlite3_prepare_v2(db.getNativePointer(), utf8, utf8.length, + outStmt, null); + } + + /** + Works identically to the sqlite3_stmt-returning sqlite3_prepare() + but uses sqlite3_prepare_v2(). + */ + public static sqlite3_stmt sqlite3_prepare_v2( + @NotNull sqlite3 db, @NotNull String sql + ){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + sqlite3_prepare_v2(db, sql, out); + return out.take(); + } + + /** + @see #sqlite3_prepare + */ + private static native int sqlite3_prepare_v3( + @NotNull long ptrToDb, @NotNull byte[] sqlUtf8, int maxBytes, + int prepFlags, @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ); + + /** + Works like the canonical sqlite3_prepare_v2() but its "tail" + output parameter is returned as the index offset into the given + byte array at which SQL parsing stopped. + */ + public static int sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, int prepFlags, + @NotNull OutputPointer.sqlite3_stmt outStmt, + @Nullable OutputPointer.Int32 pTailOffset + ){ + return sqlite3_prepare_v3(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + prepFlags, outStmt, pTailOffset); + } + + /** + Convenience overload which elides the seldom-used pTailOffset + parameter. + */ + public static int sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, int prepFlags, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + return sqlite3_prepare_v3(db.getNativePointer(), sqlUtf8, sqlUtf8.length, + prepFlags, outStmt, null); + } + + /** + Convenience overload which elides the seldom-used pTailOffset + parameter and converts the given string to UTF-8 before passing + it on. + */ + public static int sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull String sql, int prepFlags, + @NotNull OutputPointer.sqlite3_stmt outStmt + ){ + final byte[] utf8 = sql.getBytes(StandardCharsets.UTF_8); + return sqlite3_prepare_v3(db.getNativePointer(), utf8, utf8.length, + prepFlags, outStmt, null); + } + + /** + Works identically to the sqlite3_stmt-returning sqlite3_prepare() + but uses sqlite3_prepare_v3(). + */ + public static sqlite3_stmt sqlite3_prepare_v3( + @NotNull sqlite3 db, @NotNull String sql, int prepFlags + ){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + sqlite3_prepare_v3(db, sql, prepFlags, out); + return out.take(); + } + + /** + A convenience wrapper around sqlite3_prepare_v3() which accepts + an arbitrary amount of input provided as a UTF-8-encoded byte + array. It loops over the input bytes looking for + statements. Each one it finds is passed to p.call(), passing + ownership of it to that function. If p.call() returns 0, looping + continues, else the loop stops and p.call()'s result code is + returned. If preparation of any given segment fails, looping + stops and that result code is returned. + +

    If p.call() throws, the exception is converted to a db-level + error and a non-0 code is returned, in order to retain the + C-style error semantics of the API. + +

    How each statement is handled, including whether it is finalized + or not, is up to the callback object. e.g. the callback might + collect them for later use. If it does not collect them then it + must finalize them. See PrepareMultiCallback.Finalize for a + simple proxy which does that. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + int prepFlags, + @NotNull PrepareMultiCallback p){ + final OutputPointer.Int32 oTail = new OutputPointer.Int32(); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + int rc = 0; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + while( 0==rc && pos0 ){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, + sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v3(db, sqlChunk, prepFlags, outStmt, oTail); + if( 0!=rc ) break; + pos = oTail.value; + stmt = outStmt.take(); + if( null==stmt ){ + // empty statement (whitespace/comments) + continue; + } + try{ + rc = p.call(stmt); + }catch(Exception e){ + rc = sqlite3_jni_db_error( db, SQLITE_ERROR, e ); + } + } + return rc; + } + + /** + Convenience overload which accepts its SQL as a String and uses + no statement-preparation flags. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull byte[] sqlUtf8, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, sqlUtf8, 0, p); + } + + /** + Convenience overload which accepts its SQL as a String. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String sql, int prepFlags, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi( + db, sql.getBytes(StandardCharsets.UTF_8), prepFlags, p + ); + } + + /** + Convenience overload which accepts its SQL as a String and uses + no statement-preparation flags. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String sql, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, sql, 0, p); + } + + /** + Convenience overload which accepts its SQL as a String + array. They will be concatenated together as-is, with no + separator, and passed on to one of the other overloads. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String[] sql, int prepFlags, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, String.join("",sql), prepFlags, p); + } + + /** + Convenience overload which uses no statement-preparation flags. + */ + public static int sqlite3_prepare_multi( + @NotNull sqlite3 db, @NotNull String[] sql, + @NotNull PrepareMultiCallback p){ + return sqlite3_prepare_multi(db, sql, 0, p); + } + + private static native int sqlite3_preupdate_blobwrite(@NotNull long ptrToDb); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_blobwrite(), else it returns + SQLITE_MISUSE with no side effects. + */ + public static int sqlite3_preupdate_blobwrite(@NotNull sqlite3 db){ + return sqlite3_preupdate_blobwrite(db.getNativePointer()); + } + + private static native int sqlite3_preupdate_count(@NotNull long ptrToDb); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_count(), else it returns + SQLITE_MISUSE with no side effects. + */ + public static int sqlite3_preupdate_count(@NotNull sqlite3 db){ + return sqlite3_preupdate_count(db.getNativePointer()); + } + + private static native int sqlite3_preupdate_depth(@NotNull long ptrToDb); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_depth(), else it returns + SQLITE_MISUSE with no side effects. + */ + public static int sqlite3_preupdate_depth(@NotNull sqlite3 db){ + return sqlite3_preupdate_depth(db.getNativePointer()); + } + + private static native PreupdateHookCallback sqlite3_preupdate_hook( + @NotNull long ptrToDb, @Nullable PreupdateHookCallback hook + ); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, this + acts as a proxy for C's sqlite3_preupdate_hook(), else it returns null + with no side effects. + */ + public static PreupdateHookCallback sqlite3_preupdate_hook( + @NotNull sqlite3 db, @Nullable PreupdateHookCallback hook + ){ + return sqlite3_preupdate_hook(db.getNativePointer(), hook); + } + + private static native int sqlite3_preupdate_new(@NotNull long ptrToDb, int col, + @NotNull OutputPointer.sqlite3_value out); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, + this acts as a proxy for C's sqlite3_preupdate_new(), else it + returns SQLITE_MISUSE with no side effects. + + WARNING: client code _must not_ hold a reference to the returned + sqlite3_value object beyond the scope of the preupdate hook in + which this function is called. Doing so will leave the client + holding a stale pointer, the address of which could point to + anything at all after the pre-update hook is complete. This API + has no way to record such objects and clear/invalidate them at + the end of a pre-update hook. We "could" add infrastructure to do + so, but would require significant levels of bookkeeping. + */ + public static int sqlite3_preupdate_new(@NotNull sqlite3 db, int col, + @NotNull OutputPointer.sqlite3_value out){ + return sqlite3_preupdate_new(db.getNativePointer(), col, out); + } + + /** + Convenience wrapper for the 3-arg sqlite3_preupdate_new() which returns + null on error. + */ + public static sqlite3_value sqlite3_preupdate_new(@NotNull sqlite3 db, int col){ + final OutputPointer.sqlite3_value out = new OutputPointer.sqlite3_value(); + sqlite3_preupdate_new(db.getNativePointer(), col, out); + return out.take(); + } + + private static native int sqlite3_preupdate_old(@NotNull long ptrToDb, int col, + @NotNull OutputPointer.sqlite3_value out); + + /** + If the C API was built with SQLITE_ENABLE_PREUPDATE_HOOK defined, + this acts as a proxy for C's sqlite3_preupdate_old(), else it + returns SQLITE_MISUSE with no side effects. + + WARNING: see warning in sqlite3_preupdate_new() regarding the + potential for stale sqlite3_value handles. + */ + public static int sqlite3_preupdate_old(@NotNull sqlite3 db, int col, + @NotNull OutputPointer.sqlite3_value out){ + return sqlite3_preupdate_old(db.getNativePointer(), col, out); + } + + /** + Convenience wrapper for the 3-arg sqlite3_preupdate_old() which returns + null on error. + */ + public static sqlite3_value sqlite3_preupdate_old(@NotNull sqlite3 db, int col){ + final OutputPointer.sqlite3_value out = new OutputPointer.sqlite3_value(); + sqlite3_preupdate_old(db.getNativePointer(), col, out); + return out.take(); + } + + public static native void sqlite3_progress_handler( + @NotNull sqlite3 db, int n, @Nullable ProgressHandlerCallback h + ); + + public static native void sqlite3_randomness(byte[] target); + + public static native int sqlite3_release_memory(int n); + + public static native int sqlite3_reset(@NotNull sqlite3_stmt stmt); + + /** + Works like the C API except that it has no side effects if auto + extensions are currently running. (The JNI-level list of + extensions cannot be manipulated while it is being traversed.) + */ + public static native void sqlite3_reset_auto_extension(); + + public static native void sqlite3_result_double( + @NotNull sqlite3_context cx, double v + ); + + /** + The main sqlite3_result_error() impl of which all others are + proxies. eTextRep must be one of SQLITE_UTF8 or SQLITE_UTF16 and + msg must be encoded correspondingly. Any other eTextRep value + results in the C-level sqlite3_result_error() being called with a + complaint about the invalid argument. + */ + private static native void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull byte[] msg, int eTextRep + ); + + public static void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull byte[] utf8 + ){ + sqlite3_result_error(cx, utf8, SQLITE_UTF8); + } + + public static void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull String msg + ){ + final byte[] utf8 = msg.getBytes(StandardCharsets.UTF_8); + sqlite3_result_error(cx, utf8, SQLITE_UTF8); + } + + public static void sqlite3_result_error16( + @NotNull sqlite3_context cx, @NotNull byte[] utf16 + ){ + sqlite3_result_error(cx, utf16, SQLITE_UTF16); + } + + public static void sqlite3_result_error16( + @NotNull sqlite3_context cx, @NotNull String msg + ){ + final byte[] utf16 = msg.getBytes(StandardCharsets.UTF_16); + sqlite3_result_error(cx, utf16, SQLITE_UTF16); + } + + /** + Equivalent to passing e.toString() to {@link + #sqlite3_result_error(sqlite3_context,String)}. Note that + toString() is used instead of getMessage() because the former + prepends the exception type name to the message. + */ + public static void sqlite3_result_error( + @NotNull sqlite3_context cx, @NotNull Exception e + ){ + sqlite3_result_error(cx, e.toString()); + } + + public static native void sqlite3_result_error_toobig( + @NotNull sqlite3_context cx + ); + + public static native void sqlite3_result_error_nomem( + @NotNull sqlite3_context cx + ); + + public static native void sqlite3_result_error_code( + @NotNull sqlite3_context cx, int c + ); + + public static native void sqlite3_result_int( + @NotNull sqlite3_context cx, int v + ); + + public static native void sqlite3_result_int64( + @NotNull sqlite3_context cx, long v + ); + + /** + Binds the SQL result to the given object, or {@link + #sqlite3_result_null} if {@code o} is null. Use {@link + #sqlite3_value_java_object} to fetch it. + +

    This is implemented in terms of C's sqlite3_result_pointer(), + but that function is not exposed to JNI because (A) + cross-language semantic mismatch and (B) Java doesn't need that + argument for its intended purpose (type safety). + + @see #sqlite3_value_java_object + @see #sqlite3_bind_java_object + */ + public static native void sqlite3_result_java_object( + @NotNull sqlite3_context cx, @NotNull Object o + ); + + /** + Similar to sqlite3_bind_nio_buffer(), this works like + sqlite3_result_blob() but accepts a java.nio.ByteBuffer as its + input source. See sqlite3_bind_nio_buffer() for the semantics of + the second and subsequent arguments. + + If cx is null then this function will silently fail. If + sqlite3_jni_supports_nio() returns false or iBegin is negative, + an error result is set. If (begin+n) extends beyond the end of + the buffer, it is silently truncated to fit. + + If any of the following apply, this function behaves like + sqlite3_result_null(): the blob is null, the resulting slice of + the blob is empty. + + If the resulting slice of the buffer exceeds SQLITE_LIMIT_LENGTH + then this function behaves like sqlite3_result_error_toobig(). + */ + @Experimental + /*public*/ static native void sqlite3_result_nio_buffer( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob, + int begin, int n + ); + + /** + Convenience overload which uses the whole input object + as the result blob content. + */ + @Experimental + /*public*/ static void sqlite3_result_nio_buffer( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob + ){ + sqlite3_result_nio_buffer(cx, blob, 0, -1); + } + + public static native void sqlite3_result_null( + @NotNull sqlite3_context cx + ); + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Boolean v + ){ + sqlite3_result_int(cx, v ? 1 : 0); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, boolean v + ){ + sqlite3_result_int(cx, v ? 1 : 0); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Double v + ){ + sqlite3_result_double(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, double v + ){ + sqlite3_result_double(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Integer v + ){ + sqlite3_result_int(cx, v); + } + + public static void sqlite3_result_set(@NotNull sqlite3_context cx, int v){ + sqlite3_result_int(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @NotNull Long v + ){ + sqlite3_result_int64(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, long v + ){ + sqlite3_result_int64(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @Nullable String v + ){ + if( null==v ) sqlite3_result_null(cx); + else sqlite3_result_text(cx, v); + } + + public static void sqlite3_result_set( + @NotNull sqlite3_context cx, @Nullable byte[] blob + ){ + if( null==blob ) sqlite3_result_null(cx); + else sqlite3_result_blob(cx, blob, blob.length); + } + + public static native void sqlite3_result_subtype( + @NotNull sqlite3_context cx, int val + ); + + public static native void sqlite3_result_value( + @NotNull sqlite3_context cx, @NotNull sqlite3_value v + ); + + public static native void sqlite3_result_zeroblob( + @NotNull sqlite3_context cx, int n + ); + + public static native int sqlite3_result_zeroblob64( + @NotNull sqlite3_context cx, long n + ); + + /** + This overload is private because its final parameter is arguably + unnecessary in Java. + */ + private static native void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable byte[] blob, int maxLen + ); + + public static void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable byte[] blob + ){ + sqlite3_result_blob(cx, blob, (int)(null==blob ? 0 : blob.length)); + } + + /** + Convenience overload which behaves like + sqlite3_result_nio_buffer(). + */ + @Experimental + /*public*/ static void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob, + int begin, int n + ){ + sqlite3_result_nio_buffer(cx, blob, begin, n); + } + + /** + Convenience overload which behaves like the two-argument overload of + sqlite3_result_nio_buffer(). + */ + @Experimental + /*public*/ static void sqlite3_result_blob( + @NotNull sqlite3_context cx, @Nullable java.nio.ByteBuffer blob + ){ + sqlite3_result_nio_buffer(cx, blob); + } + + /** + Binds the given text using C's sqlite3_result_blob64() unless: + +

      + +
    • @param blob is null: translates to sqlite3_result_null()
      • + +
    • @param blob is too large: translates to + sqlite3_result_error_toobig()
      • + +
    + +

    If @param maxLen is larger than blob.length, it is truncated + to that value. If it is negative, results are undefined.

    + +

    This overload is private because its final parameter is + arguably unnecessary in Java.

    + */ + private static native void sqlite3_result_blob64( + @NotNull sqlite3_context cx, @Nullable byte[] blob, long maxLen + ); + + public static void sqlite3_result_blob64( + @NotNull sqlite3_context cx, @Nullable byte[] blob + ){ + sqlite3_result_blob64(cx, blob, (long)(null==blob ? 0 : blob.length)); + } + + /** + This overload is private because its final parameter is + arguably unnecessary in Java. + */ + private static native void sqlite3_result_text( + @NotNull sqlite3_context cx, @Nullable byte[] utf8, int maxLen + ); + + public static void sqlite3_result_text( + @NotNull sqlite3_context cx, @Nullable byte[] utf8 + ){ + sqlite3_result_text(cx, utf8, null==utf8 ? 0 : utf8.length); + } + + public static void sqlite3_result_text( + @NotNull sqlite3_context cx, @Nullable String text + ){ + if(null == text) sqlite3_result_null(cx); + else{ + final byte[] utf8 = text.getBytes(StandardCharsets.UTF_8); + sqlite3_result_text(cx, utf8, utf8.length); + } + } + + /** + Binds the given text using C's sqlite3_result_text64() unless: + +
      + +
    • text is null: translates to a call to {@link + #sqlite3_result_null}
      • + +
    • text is too large: translates to a call to + {@link #sqlite3_result_error_toobig}
      • + +
    • The @param encoding argument has an invalid value: translates to + {@link sqlite3_result_error_code} with code SQLITE_FORMAT.
      • + +
    + + If maxLength (in bytes, not characters) is larger than + text.length, it is silently truncated to text.length. If it is + negative, results are undefined. If text is null, the subsequent + arguments are ignored. + + This overload is private because its maxLength parameter is + arguably unnecessary in Java. + */ + private static native void sqlite3_result_text64( + @NotNull sqlite3_context cx, @Nullable byte[] text, + long maxLength, int encoding + ); + + /** + Sets the current UDF result to the given bytes, which are assumed + be encoded in UTF-16 using the platform's byte order. + */ + public static void sqlite3_result_text16( + @NotNull sqlite3_context cx, @Nullable byte[] utf16 + ){ + if(null == utf16) sqlite3_result_null(cx); + else sqlite3_result_text64(cx, utf16, utf16.length, SQLITE_UTF16); + } + + public static void sqlite3_result_text16( + @NotNull sqlite3_context cx, @Nullable String text + ){ + if(null == text) sqlite3_result_null(cx); + else{ + final byte[] b = text.getBytes(StandardCharsets.UTF_16); + sqlite3_result_text64(cx, b, b.length, SQLITE_UTF16); + } + } + + private static native RollbackHookCallback sqlite3_rollback_hook( + @NotNull long ptrToDb, @Nullable RollbackHookCallback hook + ); + + public static RollbackHookCallback sqlite3_rollback_hook( + @NotNull sqlite3 db, @Nullable RollbackHookCallback hook + ){ + return sqlite3_rollback_hook(db.getNativePointer(), hook); + } + + public static native int sqlite3_set_authorizer( + @NotNull sqlite3 db, @Nullable AuthorizerCallback auth + ); + + public static native void sqlite3_set_auxdata( + @NotNull sqlite3_context cx, int n, @Nullable Object data + ); + + public static native void sqlite3_set_last_insert_rowid( + @NotNull sqlite3 db, long rowid + ); + + + /** + In addition to calling the C-level sqlite3_shutdown(), the JNI + binding also cleans up all stale per-thread state managed by the + library, as well as any registered auto-extensions, and frees up + various bits of memory. Calling this while database handles or + prepared statements are still active will leak resources. Trying + to use those objects after this routine is called invoked + undefined behavior. + */ + public static synchronized native int sqlite3_shutdown(); + + public static native int sqlite3_sleep(int ms); + + public static native String sqlite3_sourceid(); + + public static native String sqlite3_sql(@NotNull sqlite3_stmt stmt); + + //! Consider removing this. We can use sqlite3_status64() instead, + // or use that one's impl with this one's name. + public static native int sqlite3_status( + int op, @NotNull OutputPointer.Int32 pCurrent, + @NotNull OutputPointer.Int32 pHighwater, boolean reset + ); + + public static native int sqlite3_status64( + int op, @NotNull OutputPointer.Int64 pCurrent, + @NotNull OutputPointer.Int64 pHighwater, boolean reset + ); + + private static native int sqlite3_step(@NotNull long ptrToStmt); + + public static int sqlite3_step(@NotNull sqlite3_stmt stmt){ + return null==stmt ? SQLITE_MISUSE : sqlite3_step(stmt.getNativePointer()); + } + + public static native boolean sqlite3_stmt_busy(@NotNull sqlite3_stmt stmt); + + private static native int sqlite3_stmt_explain(@NotNull long ptrToStmt, int op); + + public static int sqlite3_stmt_explain(@NotNull sqlite3_stmt stmt, int op){ + return null==stmt ? SQLITE_MISUSE : sqlite3_stmt_explain(stmt.getNativePointer(), op); + } + + private static native int sqlite3_stmt_isexplain(@NotNull long ptrToStmt); + + public static int sqlite3_stmt_isexplain(@NotNull sqlite3_stmt stmt){ + return null==stmt ? 0 : sqlite3_stmt_isexplain(stmt.getNativePointer()); + } + + public static native boolean sqlite3_stmt_readonly(@NotNull sqlite3_stmt stmt); + + public static native int sqlite3_stmt_status( + @NotNull sqlite3_stmt stmt, int op, boolean reset + ); + + /** + Internal impl of the public sqlite3_strglob() method. Neither + argument may be null and both must be NUL-terminated UTF-8. + + This overload is private because: (A) to keep users from + inadvertently passing non-NUL-terminated byte arrays (an easy + thing to do). (B) it is cheaper to NUL-terminate the + String-to-byte-array conversion in the Java implementation + (sqlite3_strglob(String,String)) than to do that in C, so that + signature is the public-facing one. + */ + private static native int sqlite3_strglob( + @NotNull byte[] glob, @NotNull byte[] nulTerminatedUtf8 + ); + + public static int sqlite3_strglob( + @NotNull String glob, @NotNull String txt + ){ + return sqlite3_strglob(nulTerminateUtf8(glob), + nulTerminateUtf8(txt)); + } + + /** + The LIKE counterpart of the private sqlite3_strglob() method. + */ + private static native int sqlite3_strlike( + @NotNull byte[] glob, @NotNull byte[] nulTerminatedUtf8, + int escChar + ); + + public static int sqlite3_strlike( + @NotNull String glob, @NotNull String txt, char escChar + ){ + return sqlite3_strlike(nulTerminateUtf8(glob), + nulTerminateUtf8(txt), + (int)escChar); + } + + private static native int sqlite3_system_errno(@NotNull long ptrToDb); + + public static int sqlite3_system_errno(@NotNull sqlite3 db){ + return sqlite3_system_errno(db.getNativePointer()); + } + + public static native int sqlite3_table_column_metadata( + @NotNull sqlite3 db, @NotNull String zDbName, + @NotNull String zTableName, @NotNull String zColumnName, + @Nullable OutputPointer.String pzDataType, + @Nullable OutputPointer.String pzCollSeq, + @Nullable OutputPointer.Bool pNotNull, + @Nullable OutputPointer.Bool pPrimaryKey, + @Nullable OutputPointer.Bool pAutoinc + ); + + /** + Convenience overload which returns its results via a single + output object. If this function returns non-0 (error), the the + contents of the output object are not modified. + */ + public static int sqlite3_table_column_metadata( + @NotNull sqlite3 db, @NotNull String zDbName, + @NotNull String zTableName, @NotNull String zColumnName, + @NotNull TableColumnMetadata out){ + return sqlite3_table_column_metadata( + db, zDbName, zTableName, zColumnName, + out.pzDataType, out.pzCollSeq, out.pNotNull, + out.pPrimaryKey, out.pAutoinc); + } + + /** + Convenience overload which returns the column metadata object on + success and null on error. + */ + public static TableColumnMetadata sqlite3_table_column_metadata( + @NotNull sqlite3 db, @NotNull String zDbName, + @NotNull String zTableName, @NotNull String zColumnName){ + final TableColumnMetadata out = new TableColumnMetadata(); + return 0==sqlite3_table_column_metadata( + db, zDbName, zTableName, zColumnName, out + ) ? out : null; + } + + public static native int sqlite3_threadsafe(); + + private static native int sqlite3_total_changes(@NotNull long ptrToDb); + + public static int sqlite3_total_changes(@NotNull sqlite3 db){ + return sqlite3_total_changes(db.getNativePointer()); + } + + private static native long sqlite3_total_changes64(@NotNull long ptrToDb); + + public static long sqlite3_total_changes64(@NotNull sqlite3 db){ + return sqlite3_total_changes64(db.getNativePointer()); + } + + /** + Works like C's sqlite3_trace_v2() except that the 3rd argument to that + function is elided here because the roles of that functions' 3rd and 4th + arguments are encapsulated in the final argument to this function. + +

    Unlike the C API, which is documented as always returning 0, + this implementation returns non-0 if initialization of the tracer + mapping state fails (e.g. on OOM). + */ + public static native int sqlite3_trace_v2( + @NotNull sqlite3 db, int traceMask, @Nullable TraceV2Callback tracer + ); + + public static native int sqlite3_txn_state( + @NotNull sqlite3 db, @Nullable String zSchema + ); + + private static native UpdateHookCallback sqlite3_update_hook( + @NotNull long ptrToDb, @Nullable UpdateHookCallback hook + ); + + public static UpdateHookCallback sqlite3_update_hook( + @NotNull sqlite3 db, @Nullable UpdateHookCallback hook + ){ + return sqlite3_update_hook(db.getNativePointer(), hook); + } + + /* + Note that: + + void * sqlite3_user_data(sqlite3_context*) + + Is not relevant in the JNI binding, as its feature is replaced by + the ability to pass an object, including any relevant state, to + sqlite3_create_function(). + */ + + private static native byte[] sqlite3_value_blob(@NotNull long ptrToValue); + + public static byte[] sqlite3_value_blob(@NotNull sqlite3_value v){ + return sqlite3_value_blob(v.getNativePointer()); + } + + private static native int sqlite3_value_bytes(@NotNull long ptrToValue); + + public static int sqlite3_value_bytes(@NotNull sqlite3_value v){ + return sqlite3_value_bytes(v.getNativePointer()); + } + + private static native int sqlite3_value_bytes16(@NotNull long ptrToValue); + + public static int sqlite3_value_bytes16(@NotNull sqlite3_value v){ + return sqlite3_value_bytes16(v.getNativePointer()); + } + + private static native double sqlite3_value_double(@NotNull long ptrToValue); + + public static double sqlite3_value_double(@NotNull sqlite3_value v){ + return sqlite3_value_double(v.getNativePointer()); + } + + private static native sqlite3_value sqlite3_value_dup(@NotNull long ptrToValue); + + public static sqlite3_value sqlite3_value_dup(@NotNull sqlite3_value v){ + return sqlite3_value_dup(v.getNativePointer()); + } + + private static native int sqlite3_value_encoding(@NotNull long ptrToValue); + + public static int sqlite3_value_encoding(@NotNull sqlite3_value v){ + return sqlite3_value_encoding(v.getNativePointer()); + } + + private static native void sqlite3_value_free(@Nullable long ptrToValue); + + public static void sqlite3_value_free(@Nullable sqlite3_value v){ + if( null!=v ) sqlite3_value_free(v.clearNativePointer()); + } + + private static native boolean sqlite3_value_frombind(@NotNull long ptrToValue); + + public static boolean sqlite3_value_frombind(@NotNull sqlite3_value v){ + return sqlite3_value_frombind(v.getNativePointer()); + } + + private static native int sqlite3_value_int(@NotNull long ptrToValue); + + public static int sqlite3_value_int(@NotNull sqlite3_value v){ + return sqlite3_value_int(v.getNativePointer()); + } + + private static native long sqlite3_value_int64(@NotNull long ptrToValue); + + public static long sqlite3_value_int64(@NotNull sqlite3_value v){ + return sqlite3_value_int64(v.getNativePointer()); + } + + private static native Object sqlite3_value_java_object(@NotNull long ptrToValue); + + /** + If the given value was set using {@link + #sqlite3_result_java_object} then this function returns that + object, else it returns null. + +

    It is up to the caller to inspect the object to determine its + type, and cast it if necessary. + */ + public static Object sqlite3_value_java_object(@NotNull sqlite3_value v){ + return sqlite3_value_java_object(v.getNativePointer()); + } + + /** + A variant of sqlite3_value_java_object() which returns the + fetched object cast to T if the object is an instance of the + given Class, else it returns null. + */ + @SuppressWarnings("unchecked") + public static T sqlite3_value_java_object(@NotNull sqlite3_value v, + @NotNull Class type){ + final Object o = sqlite3_value_java_object(v); + return type.isInstance(o) ? (T)o : null; + } + + /** + A variant of sqlite3_column_blob() which returns the blob as a + ByteBuffer object. Returns null if its argument is null, if + sqlite3_jni_supports_nio() is false, or if sqlite3_value_blob() + would return null for the same input. + */ + @Experimental + /*public*/ static native java.nio.ByteBuffer sqlite3_value_nio_buffer( + @NotNull sqlite3_value v + ); + + private static native int sqlite3_value_nochange(@NotNull long ptrToValue); + + public static int sqlite3_value_nochange(@NotNull sqlite3_value v){ + return sqlite3_value_nochange(v.getNativePointer()); + } + + private static native int sqlite3_value_numeric_type(@NotNull long ptrToValue); + + public static int sqlite3_value_numeric_type(@NotNull sqlite3_value v){ + return sqlite3_value_numeric_type(v.getNativePointer()); + } + + private static native int sqlite3_value_subtype(@NotNull long ptrToValue); + + public static int sqlite3_value_subtype(@NotNull sqlite3_value v){ + return sqlite3_value_subtype(v.getNativePointer()); + } + + private static native byte[] sqlite3_value_text(@NotNull long ptrToValue); + + /** + Functions identially to the C API, and this note is just to + stress that the returned bytes are encoded as UTF-8. It returns + null if the underlying C-level sqlite3_value_text() returns NULL + or on allocation error. + */ + public static byte[] sqlite3_value_text(@NotNull sqlite3_value v){ + return sqlite3_value_text(v.getNativePointer()); + } + + private static native String sqlite3_value_text16(@NotNull long ptrToValue); + + public static String sqlite3_value_text16(@NotNull sqlite3_value v){ + return sqlite3_value_text16(v.getNativePointer()); + } + + private static native int sqlite3_value_type(@NotNull long ptrToValue); + + public static int sqlite3_value_type(@NotNull sqlite3_value v){ + return sqlite3_value_type(v.getNativePointer()); + } + + /** + This is NOT part of the public API. It exists solely as a place + for this code's developers to collect internal metrics and such. + It has no stable interface. It may go way or change behavior at + any time. + */ + public static native void sqlite3_jni_internal_details(); + + ////////////////////////////////////////////////////////////////////// + // SQLITE_... constants follow... + + // version info + public static final int SQLITE_VERSION_NUMBER = sqlite3_libversion_number(); + public static final String SQLITE_VERSION = sqlite3_libversion(); + public static final String SQLITE_SOURCE_ID = sqlite3_sourceid(); + + // access + public static final int SQLITE_ACCESS_EXISTS = 0; + public static final int SQLITE_ACCESS_READWRITE = 1; + public static final int SQLITE_ACCESS_READ = 2; + + // authorizer + public static final int SQLITE_DENY = 1; + public static final int SQLITE_IGNORE = 2; + public static final int SQLITE_CREATE_INDEX = 1; + public static final int SQLITE_CREATE_TABLE = 2; + public static final int SQLITE_CREATE_TEMP_INDEX = 3; + public static final int SQLITE_CREATE_TEMP_TABLE = 4; + public static final int SQLITE_CREATE_TEMP_TRIGGER = 5; + public static final int SQLITE_CREATE_TEMP_VIEW = 6; + public static final int SQLITE_CREATE_TRIGGER = 7; + public static final int SQLITE_CREATE_VIEW = 8; + public static final int SQLITE_DELETE = 9; + public static final int SQLITE_DROP_INDEX = 10; + public static final int SQLITE_DROP_TABLE = 11; + public static final int SQLITE_DROP_TEMP_INDEX = 12; + public static final int SQLITE_DROP_TEMP_TABLE = 13; + public static final int SQLITE_DROP_TEMP_TRIGGER = 14; + public static final int SQLITE_DROP_TEMP_VIEW = 15; + public static final int SQLITE_DROP_TRIGGER = 16; + public static final int SQLITE_DROP_VIEW = 17; + public static final int SQLITE_INSERT = 18; + public static final int SQLITE_PRAGMA = 19; + public static final int SQLITE_READ = 20; + public static final int SQLITE_SELECT = 21; + public static final int SQLITE_TRANSACTION = 22; + public static final int SQLITE_UPDATE = 23; + public static final int SQLITE_ATTACH = 24; + public static final int SQLITE_DETACH = 25; + public static final int SQLITE_ALTER_TABLE = 26; + public static final int SQLITE_REINDEX = 27; + public static final int SQLITE_ANALYZE = 28; + public static final int SQLITE_CREATE_VTABLE = 29; + public static final int SQLITE_DROP_VTABLE = 30; + public static final int SQLITE_FUNCTION = 31; + public static final int SQLITE_SAVEPOINT = 32; + public static final int SQLITE_RECURSIVE = 33; + + // blob finalizers: these should, because they are treated as + // special pointer values in C, ideally have the same sizeof() as + // the platform's (void*), but we can't know that size from here. + public static final long SQLITE_STATIC = 0; + public static final long SQLITE_TRANSIENT = -1; + + // changeset + public static final int SQLITE_CHANGESETSTART_INVERT = 2; + public static final int SQLITE_CHANGESETAPPLY_NOSAVEPOINT = 1; + public static final int SQLITE_CHANGESETAPPLY_INVERT = 2; + public static final int SQLITE_CHANGESETAPPLY_IGNORENOOP = 4; + public static final int SQLITE_CHANGESET_DATA = 1; + public static final int SQLITE_CHANGESET_NOTFOUND = 2; + public static final int SQLITE_CHANGESET_CONFLICT = 3; + public static final int SQLITE_CHANGESET_CONSTRAINT = 4; + public static final int SQLITE_CHANGESET_FOREIGN_KEY = 5; + public static final int SQLITE_CHANGESET_OMIT = 0; + public static final int SQLITE_CHANGESET_REPLACE = 1; + public static final int SQLITE_CHANGESET_ABORT = 2; + + // config + public static final int SQLITE_CONFIG_SINGLETHREAD = 1; + public static final int SQLITE_CONFIG_MULTITHREAD = 2; + public static final int SQLITE_CONFIG_SERIALIZED = 3; + public static final int SQLITE_CONFIG_MALLOC = 4; + public static final int SQLITE_CONFIG_GETMALLOC = 5; + public static final int SQLITE_CONFIG_SCRATCH = 6; + public static final int SQLITE_CONFIG_PAGECACHE = 7; + public static final int SQLITE_CONFIG_HEAP = 8; + public static final int SQLITE_CONFIG_MEMSTATUS = 9; + public static final int SQLITE_CONFIG_MUTEX = 10; + public static final int SQLITE_CONFIG_GETMUTEX = 11; + public static final int SQLITE_CONFIG_LOOKASIDE = 13; + public static final int SQLITE_CONFIG_PCACHE = 14; + public static final int SQLITE_CONFIG_GETPCACHE = 15; + public static final int SQLITE_CONFIG_LOG = 16; + public static final int SQLITE_CONFIG_URI = 17; + public static final int SQLITE_CONFIG_PCACHE2 = 18; + public static final int SQLITE_CONFIG_GETPCACHE2 = 19; + public static final int SQLITE_CONFIG_COVERING_INDEX_SCAN = 20; + public static final int SQLITE_CONFIG_SQLLOG = 21; + public static final int SQLITE_CONFIG_MMAP_SIZE = 22; + public static final int SQLITE_CONFIG_WIN32_HEAPSIZE = 23; + public static final int SQLITE_CONFIG_PCACHE_HDRSZ = 24; + public static final int SQLITE_CONFIG_PMASZ = 25; + public static final int SQLITE_CONFIG_STMTJRNL_SPILL = 26; + public static final int SQLITE_CONFIG_SMALL_MALLOC = 27; + public static final int SQLITE_CONFIG_SORTERREF_SIZE = 28; + public static final int SQLITE_CONFIG_MEMDB_MAXSIZE = 29; + + // data types + public static final int SQLITE_INTEGER = 1; + public static final int SQLITE_FLOAT = 2; + public static final int SQLITE_TEXT = 3; + public static final int SQLITE_BLOB = 4; + public static final int SQLITE_NULL = 5; + + // db config + public static final int SQLITE_DBCONFIG_MAINDBNAME = 1000; + public static final int SQLITE_DBCONFIG_LOOKASIDE = 1001; + public static final int SQLITE_DBCONFIG_ENABLE_FKEY = 1002; + public static final int SQLITE_DBCONFIG_ENABLE_TRIGGER = 1003; + public static final int SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER = 1004; + public static final int SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION = 1005; + public static final int SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE = 1006; + public static final int SQLITE_DBCONFIG_ENABLE_QPSG = 1007; + public static final int SQLITE_DBCONFIG_TRIGGER_EQP = 1008; + public static final int SQLITE_DBCONFIG_RESET_DATABASE = 1009; + public static final int SQLITE_DBCONFIG_DEFENSIVE = 1010; + public static final int SQLITE_DBCONFIG_WRITABLE_SCHEMA = 1011; + public static final int SQLITE_DBCONFIG_LEGACY_ALTER_TABLE = 1012; + public static final int SQLITE_DBCONFIG_DQS_DML = 1013; + public static final int SQLITE_DBCONFIG_DQS_DDL = 1014; + public static final int SQLITE_DBCONFIG_ENABLE_VIEW = 1015; + public static final int SQLITE_DBCONFIG_LEGACY_FILE_FORMAT = 1016; + public static final int SQLITE_DBCONFIG_TRUSTED_SCHEMA = 1017; + public static final int SQLITE_DBCONFIG_STMT_SCANSTATUS = 1018; + public static final int SQLITE_DBCONFIG_REVERSE_SCANORDER = 1019; + public static final int SQLITE_DBCONFIG_MAX = 1019; + + // db status + public static final int SQLITE_DBSTATUS_LOOKASIDE_USED = 0; + public static final int SQLITE_DBSTATUS_CACHE_USED = 1; + public static final int SQLITE_DBSTATUS_SCHEMA_USED = 2; + public static final int SQLITE_DBSTATUS_STMT_USED = 3; + public static final int SQLITE_DBSTATUS_LOOKASIDE_HIT = 4; + public static final int SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE = 5; + public static final int SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL = 6; + public static final int SQLITE_DBSTATUS_CACHE_HIT = 7; + public static final int SQLITE_DBSTATUS_CACHE_MISS = 8; + public static final int SQLITE_DBSTATUS_CACHE_WRITE = 9; + public static final int SQLITE_DBSTATUS_DEFERRED_FKS = 10; + public static final int SQLITE_DBSTATUS_CACHE_USED_SHARED = 11; + public static final int SQLITE_DBSTATUS_CACHE_SPILL = 12; + public static final int SQLITE_DBSTATUS_MAX = 12; + + // encodings + public static final int SQLITE_UTF8 = 1; + public static final int SQLITE_UTF16LE = 2; + public static final int SQLITE_UTF16BE = 3; + public static final int SQLITE_UTF16 = 4; + public static final int SQLITE_UTF16_ALIGNED = 8; + + // fcntl + public static final int SQLITE_FCNTL_LOCKSTATE = 1; + public static final int SQLITE_FCNTL_GET_LOCKPROXYFILE = 2; + public static final int SQLITE_FCNTL_SET_LOCKPROXYFILE = 3; + public static final int SQLITE_FCNTL_LAST_ERRNO = 4; + public static final int SQLITE_FCNTL_SIZE_HINT = 5; + public static final int SQLITE_FCNTL_CHUNK_SIZE = 6; + public static final int SQLITE_FCNTL_FILE_POINTER = 7; + public static final int SQLITE_FCNTL_SYNC_OMITTED = 8; + public static final int SQLITE_FCNTL_WIN32_AV_RETRY = 9; + public static final int SQLITE_FCNTL_PERSIST_WAL = 10; + public static final int SQLITE_FCNTL_OVERWRITE = 11; + public static final int SQLITE_FCNTL_VFSNAME = 12; + public static final int SQLITE_FCNTL_POWERSAFE_OVERWRITE = 13; + public static final int SQLITE_FCNTL_PRAGMA = 14; + public static final int SQLITE_FCNTL_BUSYHANDLER = 15; + public static final int SQLITE_FCNTL_TEMPFILENAME = 16; + public static final int SQLITE_FCNTL_MMAP_SIZE = 18; + public static final int SQLITE_FCNTL_TRACE = 19; + public static final int SQLITE_FCNTL_HAS_MOVED = 20; + public static final int SQLITE_FCNTL_SYNC = 21; + public static final int SQLITE_FCNTL_COMMIT_PHASETWO = 22; + public static final int SQLITE_FCNTL_WIN32_SET_HANDLE = 23; + public static final int SQLITE_FCNTL_WAL_BLOCK = 24; + public static final int SQLITE_FCNTL_ZIPVFS = 25; + public static final int SQLITE_FCNTL_RBU = 26; + public static final int SQLITE_FCNTL_VFS_POINTER = 27; + public static final int SQLITE_FCNTL_JOURNAL_POINTER = 28; + public static final int SQLITE_FCNTL_WIN32_GET_HANDLE = 29; + public static final int SQLITE_FCNTL_PDB = 30; + public static final int SQLITE_FCNTL_BEGIN_ATOMIC_WRITE = 31; + public static final int SQLITE_FCNTL_COMMIT_ATOMIC_WRITE = 32; + public static final int SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE = 33; + public static final int SQLITE_FCNTL_LOCK_TIMEOUT = 34; + public static final int SQLITE_FCNTL_DATA_VERSION = 35; + public static final int SQLITE_FCNTL_SIZE_LIMIT = 36; + public static final int SQLITE_FCNTL_CKPT_DONE = 37; + public static final int SQLITE_FCNTL_RESERVE_BYTES = 38; + public static final int SQLITE_FCNTL_CKPT_START = 39; + public static final int SQLITE_FCNTL_EXTERNAL_READER = 40; + public static final int SQLITE_FCNTL_CKSM_FILE = 41; + public static final int SQLITE_FCNTL_RESET_CACHE = 42; + + // flock + public static final int SQLITE_LOCK_NONE = 0; + public static final int SQLITE_LOCK_SHARED = 1; + public static final int SQLITE_LOCK_RESERVED = 2; + public static final int SQLITE_LOCK_PENDING = 3; + public static final int SQLITE_LOCK_EXCLUSIVE = 4; + + // iocap + public static final int SQLITE_IOCAP_ATOMIC = 1; + public static final int SQLITE_IOCAP_ATOMIC512 = 2; + public static final int SQLITE_IOCAP_ATOMIC1K = 4; + public static final int SQLITE_IOCAP_ATOMIC2K = 8; + public static final int SQLITE_IOCAP_ATOMIC4K = 16; + public static final int SQLITE_IOCAP_ATOMIC8K = 32; + public static final int SQLITE_IOCAP_ATOMIC16K = 64; + public static final int SQLITE_IOCAP_ATOMIC32K = 128; + public static final int SQLITE_IOCAP_ATOMIC64K = 256; + public static final int SQLITE_IOCAP_SAFE_APPEND = 512; + public static final int SQLITE_IOCAP_SEQUENTIAL = 1024; + public static final int SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN = 2048; + public static final int SQLITE_IOCAP_POWERSAFE_OVERWRITE = 4096; + public static final int SQLITE_IOCAP_IMMUTABLE = 8192; + public static final int SQLITE_IOCAP_BATCH_ATOMIC = 16384; + + // limits + public static final int SQLITE_LIMIT_LENGTH = 0; + public static final int SQLITE_LIMIT_SQL_LENGTH = 1; + public static final int SQLITE_LIMIT_COLUMN = 2; + public static final int SQLITE_LIMIT_EXPR_DEPTH = 3; + public static final int SQLITE_LIMIT_COMPOUND_SELECT = 4; + public static final int SQLITE_LIMIT_VDBE_OP = 5; + public static final int SQLITE_LIMIT_FUNCTION_ARG = 6; + public static final int SQLITE_LIMIT_ATTACHED = 7; + public static final int SQLITE_LIMIT_LIKE_PATTERN_LENGTH = 8; + public static final int SQLITE_LIMIT_VARIABLE_NUMBER = 9; + public static final int SQLITE_LIMIT_TRIGGER_DEPTH = 10; + public static final int SQLITE_LIMIT_WORKER_THREADS = 11; + + // open flags + + public static final int SQLITE_OPEN_READONLY = 0x00000001 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_READWRITE = 0x00000002 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_CREATE = 0x00000004 /* Ok for sqlite3_open_v2() */; + //public static final int SQLITE_OPEN_DELETEONCLOSE = 0x00000008 /* VFS only */; + //public static final int SQLITE_OPEN_EXCLUSIVE = 0x00000010 /* VFS only */; + //public static final int SQLITE_OPEN_AUTOPROXY = 0x00000020 /* VFS only */; + public static final int SQLITE_OPEN_URI = 0x00000040 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_MEMORY = 0x00000080 /* Ok for sqlite3_open_v2() */; + //public static final int SQLITE_OPEN_MAIN_DB = 0x00000100 /* VFS only */; + //public static final int SQLITE_OPEN_TEMP_DB = 0x00000200 /* VFS only */; + //public static final int SQLITE_OPEN_TRANSIENT_DB = 0x00000400 /* VFS only */; + //public static final int SQLITE_OPEN_MAIN_JOURNAL = 0x00000800 /* VFS only */; + //public static final int SQLITE_OPEN_TEMP_JOURNAL = 0x00001000 /* VFS only */; + //public static final int SQLITE_OPEN_SUBJOURNAL = 0x00002000 /* VFS only */; + //public static final int SQLITE_OPEN_SUPER_JOURNAL = 0x00004000 /* VFS only */; + public static final int SQLITE_OPEN_NOMUTEX = 0x00008000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_FULLMUTEX = 0x00010000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_SHAREDCACHE = 0x00020000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_PRIVATECACHE = 0x00040000 /* Ok for sqlite3_open_v2() */; + //public static final int SQLITE_OPEN_WAL = 0x00080000 /* VFS only */; + public static final int SQLITE_OPEN_NOFOLLOW = 0x01000000 /* Ok for sqlite3_open_v2() */; + public static final int SQLITE_OPEN_EXRESCODE = 0x02000000 /* Extended result codes */; + + // prepare flags + public static final int SQLITE_PREPARE_PERSISTENT = 1; + public static final int SQLITE_PREPARE_NO_VTAB = 4; + + // result codes + public static final int SQLITE_OK = 0; + public static final int SQLITE_ERROR = 1; + public static final int SQLITE_INTERNAL = 2; + public static final int SQLITE_PERM = 3; + public static final int SQLITE_ABORT = 4; + public static final int SQLITE_BUSY = 5; + public static final int SQLITE_LOCKED = 6; + public static final int SQLITE_NOMEM = 7; + public static final int SQLITE_READONLY = 8; + public static final int SQLITE_INTERRUPT = 9; + public static final int SQLITE_IOERR = 10; + public static final int SQLITE_CORRUPT = 11; + public static final int SQLITE_NOTFOUND = 12; + public static final int SQLITE_FULL = 13; + public static final int SQLITE_CANTOPEN = 14; + public static final int SQLITE_PROTOCOL = 15; + public static final int SQLITE_EMPTY = 16; + public static final int SQLITE_SCHEMA = 17; + public static final int SQLITE_TOOBIG = 18; + public static final int SQLITE_CONSTRAINT = 19; + public static final int SQLITE_MISMATCH = 20; + public static final int SQLITE_MISUSE = 21; + public static final int SQLITE_NOLFS = 22; + public static final int SQLITE_AUTH = 23; + public static final int SQLITE_FORMAT = 24; + public static final int SQLITE_RANGE = 25; + public static final int SQLITE_NOTADB = 26; + public static final int SQLITE_NOTICE = 27; + public static final int SQLITE_WARNING = 28; + public static final int SQLITE_ROW = 100; + public static final int SQLITE_DONE = 101; + public static final int SQLITE_ERROR_MISSING_COLLSEQ = 257; + public static final int SQLITE_ERROR_RETRY = 513; + public static final int SQLITE_ERROR_SNAPSHOT = 769; + public static final int SQLITE_IOERR_READ = 266; + public static final int SQLITE_IOERR_SHORT_READ = 522; + public static final int SQLITE_IOERR_WRITE = 778; + public static final int SQLITE_IOERR_FSYNC = 1034; + public static final int SQLITE_IOERR_DIR_FSYNC = 1290; + public static final int SQLITE_IOERR_TRUNCATE = 1546; + public static final int SQLITE_IOERR_FSTAT = 1802; + public static final int SQLITE_IOERR_UNLOCK = 2058; + public static final int SQLITE_IOERR_RDLOCK = 2314; + public static final int SQLITE_IOERR_DELETE = 2570; + public static final int SQLITE_IOERR_BLOCKED = 2826; + public static final int SQLITE_IOERR_NOMEM = 3082; + public static final int SQLITE_IOERR_ACCESS = 3338; + public static final int SQLITE_IOERR_CHECKRESERVEDLOCK = 3594; + public static final int SQLITE_IOERR_LOCK = 3850; + public static final int SQLITE_IOERR_CLOSE = 4106; + public static final int SQLITE_IOERR_DIR_CLOSE = 4362; + public static final int SQLITE_IOERR_SHMOPEN = 4618; + public static final int SQLITE_IOERR_SHMSIZE = 4874; + public static final int SQLITE_IOERR_SHMLOCK = 5130; + public static final int SQLITE_IOERR_SHMMAP = 5386; + public static final int SQLITE_IOERR_SEEK = 5642; + public static final int SQLITE_IOERR_DELETE_NOENT = 5898; + public static final int SQLITE_IOERR_MMAP = 6154; + public static final int SQLITE_IOERR_GETTEMPPATH = 6410; + public static final int SQLITE_IOERR_CONVPATH = 6666; + public static final int SQLITE_IOERR_VNODE = 6922; + public static final int SQLITE_IOERR_AUTH = 7178; + public static final int SQLITE_IOERR_BEGIN_ATOMIC = 7434; + public static final int SQLITE_IOERR_COMMIT_ATOMIC = 7690; + public static final int SQLITE_IOERR_ROLLBACK_ATOMIC = 7946; + public static final int SQLITE_IOERR_DATA = 8202; + public static final int SQLITE_IOERR_CORRUPTFS = 8458; + public static final int SQLITE_LOCKED_SHAREDCACHE = 262; + public static final int SQLITE_LOCKED_VTAB = 518; + public static final int SQLITE_BUSY_RECOVERY = 261; + public static final int SQLITE_BUSY_SNAPSHOT = 517; + public static final int SQLITE_BUSY_TIMEOUT = 773; + public static final int SQLITE_CANTOPEN_NOTEMPDIR = 270; + public static final int SQLITE_CANTOPEN_ISDIR = 526; + public static final int SQLITE_CANTOPEN_FULLPATH = 782; + public static final int SQLITE_CANTOPEN_CONVPATH = 1038; + public static final int SQLITE_CANTOPEN_SYMLINK = 1550; + public static final int SQLITE_CORRUPT_VTAB = 267; + public static final int SQLITE_CORRUPT_SEQUENCE = 523; + public static final int SQLITE_CORRUPT_INDEX = 779; + public static final int SQLITE_READONLY_RECOVERY = 264; + public static final int SQLITE_READONLY_CANTLOCK = 520; + public static final int SQLITE_READONLY_ROLLBACK = 776; + public static final int SQLITE_READONLY_DBMOVED = 1032; + public static final int SQLITE_READONLY_CANTINIT = 1288; + public static final int SQLITE_READONLY_DIRECTORY = 1544; + public static final int SQLITE_ABORT_ROLLBACK = 516; + public static final int SQLITE_CONSTRAINT_CHECK = 275; + public static final int SQLITE_CONSTRAINT_COMMITHOOK = 531; + public static final int SQLITE_CONSTRAINT_FOREIGNKEY = 787; + public static final int SQLITE_CONSTRAINT_FUNCTION = 1043; + public static final int SQLITE_CONSTRAINT_NOTNULL = 1299; + public static final int SQLITE_CONSTRAINT_PRIMARYKEY = 1555; + public static final int SQLITE_CONSTRAINT_TRIGGER = 1811; + public static final int SQLITE_CONSTRAINT_UNIQUE = 2067; + public static final int SQLITE_CONSTRAINT_VTAB = 2323; + public static final int SQLITE_CONSTRAINT_ROWID = 2579; + public static final int SQLITE_CONSTRAINT_PINNED = 2835; + public static final int SQLITE_CONSTRAINT_DATATYPE = 3091; + public static final int SQLITE_NOTICE_RECOVER_WAL = 283; + public static final int SQLITE_NOTICE_RECOVER_ROLLBACK = 539; + public static final int SQLITE_WARNING_AUTOINDEX = 284; + public static final int SQLITE_AUTH_USER = 279; + public static final int SQLITE_OK_LOAD_PERMANENTLY = 256; + + // serialize + public static final int SQLITE_SERIALIZE_NOCOPY = 1; + public static final int SQLITE_DESERIALIZE_FREEONCLOSE = 1; + public static final int SQLITE_DESERIALIZE_READONLY = 4; + public static final int SQLITE_DESERIALIZE_RESIZEABLE = 2; + + // session + public static final int SQLITE_SESSION_CONFIG_STRMSIZE = 1; + public static final int SQLITE_SESSION_OBJCONFIG_SIZE = 1; + + // sqlite3 status + public static final int SQLITE_STATUS_MEMORY_USED = 0; + public static final int SQLITE_STATUS_PAGECACHE_USED = 1; + public static final int SQLITE_STATUS_PAGECACHE_OVERFLOW = 2; + public static final int SQLITE_STATUS_MALLOC_SIZE = 5; + public static final int SQLITE_STATUS_PARSER_STACK = 6; + public static final int SQLITE_STATUS_PAGECACHE_SIZE = 7; + public static final int SQLITE_STATUS_MALLOC_COUNT = 9; + + // stmt status + public static final int SQLITE_STMTSTATUS_FULLSCAN_STEP = 1; + public static final int SQLITE_STMTSTATUS_SORT = 2; + public static final int SQLITE_STMTSTATUS_AUTOINDEX = 3; + public static final int SQLITE_STMTSTATUS_VM_STEP = 4; + public static final int SQLITE_STMTSTATUS_REPREPARE = 5; + public static final int SQLITE_STMTSTATUS_RUN = 6; + public static final int SQLITE_STMTSTATUS_FILTER_MISS = 7; + public static final int SQLITE_STMTSTATUS_FILTER_HIT = 8; + public static final int SQLITE_STMTSTATUS_MEMUSED = 99; + + // sync flags + public static final int SQLITE_SYNC_NORMAL = 2; + public static final int SQLITE_SYNC_FULL = 3; + public static final int SQLITE_SYNC_DATAONLY = 16; + + // tracing flags + public static final int SQLITE_TRACE_STMT = 1; + public static final int SQLITE_TRACE_PROFILE = 2; + public static final int SQLITE_TRACE_ROW = 4; + public static final int SQLITE_TRACE_CLOSE = 8; + + // transaction state + public static final int SQLITE_TXN_NONE = 0; + public static final int SQLITE_TXN_READ = 1; + public static final int SQLITE_TXN_WRITE = 2; + + // udf flags + public static final int SQLITE_DETERMINISTIC = 0x000000800; + public static final int SQLITE_DIRECTONLY = 0x000080000; + public static final int SQLITE_SUBTYPE = 0x000100000; + public static final int SQLITE_INNOCUOUS = 0x000200000; + public static final int SQLITE_RESULT_SUBTYPE = 0x001000000; + + // virtual tables + public static final int SQLITE_INDEX_SCAN_UNIQUE = 1; + public static final int SQLITE_INDEX_CONSTRAINT_EQ = 2; + public static final int SQLITE_INDEX_CONSTRAINT_GT = 4; + public static final int SQLITE_INDEX_CONSTRAINT_LE = 8; + public static final int SQLITE_INDEX_CONSTRAINT_LT = 16; + public static final int SQLITE_INDEX_CONSTRAINT_GE = 32; + public static final int SQLITE_INDEX_CONSTRAINT_MATCH = 64; + public static final int SQLITE_INDEX_CONSTRAINT_LIKE = 65; + public static final int SQLITE_INDEX_CONSTRAINT_GLOB = 66; + public static final int SQLITE_INDEX_CONSTRAINT_REGEXP = 67; + public static final int SQLITE_INDEX_CONSTRAINT_NE = 68; + public static final int SQLITE_INDEX_CONSTRAINT_ISNOT = 69; + public static final int SQLITE_INDEX_CONSTRAINT_ISNOTNULL = 70; + public static final int SQLITE_INDEX_CONSTRAINT_ISNULL = 71; + public static final int SQLITE_INDEX_CONSTRAINT_IS = 72; + public static final int SQLITE_INDEX_CONSTRAINT_LIMIT = 73; + public static final int SQLITE_INDEX_CONSTRAINT_OFFSET = 74; + public static final int SQLITE_INDEX_CONSTRAINT_FUNCTION = 150; + public static final int SQLITE_VTAB_CONSTRAINT_SUPPORT = 1; + public static final int SQLITE_VTAB_INNOCUOUS = 2; + public static final int SQLITE_VTAB_DIRECTONLY = 3; + public static final int SQLITE_VTAB_USES_ALL_SCHEMAS = 4; + public static final int SQLITE_ROLLBACK = 1; + public static final int SQLITE_FAIL = 3; + public static final int SQLITE_REPLACE = 5; + static { + init(); + } + /* Must come after static init(). */ + private static final boolean JNI_SUPPORTS_NIO = sqlite3_jni_supports_nio(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java b/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java new file mode 100644 index 0000000000..04000a3f31 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java @@ -0,0 +1,45 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +/** + This marker interface exists solely for use as a documentation and + class-grouping tool. It should be applied to interfaces or + classes which have a call() method implementing some specific + callback interface on behalf of the C library. + +

    Unless very explicitly documented otherwise, callbacks must + never throw. Any which do throw but should not might trigger debug + output regarding the error, but the exception will not be + propagated. For callback interfaces which support returning error + info to the core, the JNI binding will convert any exceptions to + C-level error information. For callback interfaces which do not + support returning error information, all exceptions will + necessarily be suppressed in order to retain the C-style no-throw + semantics and avoid invoking undefined behavior in the C layer. + +

    Callbacks of this style follow a common naming convention: + +

    1) They use the UpperCamelCase form of the C function they're + proxying for, minus the {@code sqlite3_} prefix, plus a {@code + Callback} suffix. e.g. {@code sqlite3_busy_handler()}'s callback is + named {@code BusyHandlerCallback}. Exceptions are made where that + would potentially be ambiguous, e.g. {@link ConfigSqlLogCallback} + instead of {@code ConfigCallback} because the {@code + sqlite3_config()} interface may need to support more callback types + in the future. + +

    2) They all have a {@code call()} method but its signature is + callback-specific. +*/ +public interface CallbackProxy {} diff --git a/ext/jni/src/org/sqlite/jni/capi/CollationCallback.java b/ext/jni/src/org/sqlite/jni/capi/CollationCallback.java new file mode 100644 index 0000000000..ed8bd09475 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CollationCallback.java @@ -0,0 +1,35 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.NotNull; + +/** + Callback for use with {@link CApi#sqlite3_create_collation}. + + @see AbstractCollationCallback +*/ +public interface CollationCallback + extends CallbackProxy, XDestroyCallback { + /** + Must compare the given byte arrays and return the result using + {@code memcmp()} semantics. + */ + int call(@NotNull byte[] lhs, @NotNull byte[] rhs); + + /** + Called by SQLite when the collation is destroyed. If a collation + requires custom cleanup, override this method. + */ + void xDestroy(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java b/ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java new file mode 100644 index 0000000000..ffd7fa94ab --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java @@ -0,0 +1,29 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_collation_needed}. +*/ +public interface CollationNeededCallback extends CallbackProxy { + /** + Has the same semantics as the C-level sqlite3_create_collation() + callback. + +

    Because the C API has no mechanism for reporting errors + from this callbacks, any exceptions thrown by this callback + are suppressed. + */ + void call(sqlite3 db, int eTextRep, String collationName); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java new file mode 100644 index 0000000000..e1e55c78d2 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_commit_hook}. +*/ +public interface CommitHookCallback extends CallbackProxy { + /** + Works as documented for the C-level sqlite3_commit_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + int call(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b/ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java new file mode 100644 index 0000000000..6513b0730d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java @@ -0,0 +1,25 @@ +/* +** 2023-08-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A callback for use with sqlite3_config(). +*/ +public interface ConfigLogCallback { + /** + Must function as described for a C-level callback for + {@link CApi#sqlite3_config(ConfigLogCallback)}, with the slight signature change. + */ + void call(int errCode, String msg); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java b/ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java new file mode 100644 index 0000000000..a5530b49a4 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java @@ -0,0 +1,25 @@ +/* +** 2023-08-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A callback for use with sqlite3_config(). +*/ +public interface ConfigSqlLogCallback { + /** + Must function as described for a C-level callback for + {@link CApi#sqlite3_config(ConfigSqlLogCallback)}, with the slight signature change. + */ + void call(sqlite3 db, String msg, int msgType ); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b/ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java new file mode 100644 index 0000000000..e82909e424 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java @@ -0,0 +1,46 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A helper for passing pointers between JNI C code and Java, in + particular for output pointers of high-level object types in the + sqlite3 C API, e.g. (sqlite3**) and (sqlite3_stmt**). This is + intended to be subclassed and the ContextType is intended to be the + class which is doing the subclassing. The intent of the ContextType + is strictly to provide some level of type safety by avoiding that + NativePointerHolder is not inadvertently passed to an incompatible + function signature. + + These objects do not own the pointer they refer to. They are + intended simply to communicate that pointer between C and Java. +*/ +public class NativePointerHolder { + //! Only set from JNI, where access permissions don't matter. + private volatile long nativePointer = 0; + /** + For use ONLY by package-level APIs which act as proxies for + close/finalize operations. Such ops must call this to zero out + the pointer so that this object is not carrying a stale + pointer. This function returns the prior value of the pointer and + sets it to 0. + */ + final long clearNativePointer() { + final long rv = nativePointer; + nativePointer= 0; + return rv; + } + + public final long getNativePointer(){ return nativePointer; } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java b/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java new file mode 100644 index 0000000000..f50d0c57cb --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java @@ -0,0 +1,253 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Helper classes for handling JNI output pointers. + +

    We do not use a generic OutputPointer because working with those + from the native JNI code is unduly quirky due to a lack of + autoboxing at that level. + +

    The usage is similar for all of these types: + + 

    {@code
+   OutputPointer.sqlite3 out = new OutputPointer.sqlite3();
+   assert( null==out.get() );
+   int rc = sqlite3_open(":memory:", out);
+   if( 0!=rc ) ... error;
+   assert( null!=out.get() );
+   sqlite3 db = out.take();
+   assert( null==out.get() );
+   }
    + +

    With the minor exception that the primitive types permit direct + access to the object's value via the `value` property, whereas the + JNI-level opaque types do not permit client-level code to set that + property. + +

    Warning: do not share instances of these classes across + threads. Doing so may lead to corrupting sqlite3-internal state. +*/ +public final class OutputPointer { + + /** + Output pointer for use with routines, such as sqlite3_open(), + which return a database handle via an output pointer. These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3 { + private org.sqlite.jni.capi.sqlite3 value; + /** Initializes with a null value. */ + public sqlite3(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public org.sqlite.jni.capi.sqlite3 get(){return value;} + /** Equivalent to calling get() then clear(). */ + public org.sqlite.jni.capi.sqlite3 take(){ + final org.sqlite.jni.capi.sqlite3 v = value; + value = null; + return v; + } + } + + /** + Output pointer for sqlite3_blob_open(). These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3_blob { + private org.sqlite.jni.capi.sqlite3_blob value; + /** Initializes with a null value. */ + public sqlite3_blob(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public org.sqlite.jni.capi.sqlite3_blob get(){return value;} + /** Equivalent to calling get() then clear(). */ + public org.sqlite.jni.capi.sqlite3_blob take(){ + final org.sqlite.jni.capi.sqlite3_blob v = value; + value = null; + return v; + } + } + + /** + Output pointer for use with routines, such as sqlite3_prepare(), + which return a statement handle via an output pointer. These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3_stmt { + private org.sqlite.jni.capi.sqlite3_stmt value; + /** Initializes with a null value. */ + public sqlite3_stmt(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public org.sqlite.jni.capi.sqlite3_stmt get(){return value;} + /** Equivalent to calling get() then clear(). */ + public org.sqlite.jni.capi.sqlite3_stmt take(){ + final org.sqlite.jni.capi.sqlite3_stmt v = value; + value = null; + return v; + } + } + + /** + Output pointer for use with routines, such as sqlite3_prepupdate_new(), + which return a sqlite3_value handle via an output pointer. These + pointers can only be set by the JNI layer, not by client-level + code. + */ + public static final class sqlite3_value { + private org.sqlite.jni.capi.sqlite3_value value; + /** Initializes with a null value. */ + public sqlite3_value(){value = null;} + /** Sets the current value to null. */ + public void clear(){value = null;} + /** Returns the current value. */ + public org.sqlite.jni.capi.sqlite3_value get(){return value;} + /** Equivalent to calling get() then clear(). */ + public org.sqlite.jni.capi.sqlite3_value take(){ + final org.sqlite.jni.capi.sqlite3_value v = value; + value = null; + return v; + } + } + + /** + Output pointer for use with native routines which return booleans + via integer output pointers. + */ + public static final class Bool { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public boolean value; + /** Initializes with the value 0. */ + public Bool(){this(false);} + /** Initializes with the value v. */ + public Bool(boolean v){value = v;} + /** Returns the current value. */ + public boolean get(){return value;} + /** Sets the current value to v. */ + public void set(boolean v){value = v;} + } + + /** + Output pointer for use with native routines which return integers via + output pointers. + */ + public static final class Int32 { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public int value; + /** Initializes with the value 0. */ + public Int32(){this(0);} + /** Initializes with the value v. */ + public Int32(int v){value = v;} + /** Returns the current value. */ + public int get(){return value;} + /** Sets the current value to v. */ + public void set(int v){value = v;} + } + + /** + Output pointer for use with native routines which return 64-bit integers + via output pointers. + */ + public static final class Int64 { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public long value; + /** Initializes with the value 0. */ + public Int64(){this(0);} + /** Initializes with the value v. */ + public Int64(long v){value = v;} + /** Returns the current value. */ + public long get(){return value;} + /** Sets the current value. */ + public void set(long v){value = v;} + } + + /** + Output pointer for use with native routines which return strings via + output pointers. + */ + public static final class String { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public java.lang.String value; + /** Initializes with a null value. */ + public String(){this(null);} + /** Initializes with the value v. */ + public String(java.lang.String v){value = v;} + /** Returns the current value. */ + public java.lang.String get(){return value;} + /** Sets the current value. */ + public void set(java.lang.String v){value = v;} + } + + /** + Output pointer for use with native routines which return byte + arrays via output pointers. + */ + public static final class ByteArray { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public byte[] value; + /** Initializes with the value null. */ + public ByteArray(){this(null);} + /** Initializes with the value v. */ + public ByteArray(byte[] v){value = v;} + /** Returns the current value. */ + public byte[] get(){return value;} + /** Sets the current value. */ + public void set(byte[] v){value = v;} + } + + /** + Output pointer for use with native routines which return + blobs via java.nio.ByteBuffer. + + See {@link org.sqlite.jni.capi.CApi#sqlite3_jni_supports_nio} + */ + public static final class ByteBuffer { + /** + This is public for ease of use. Accessors are provided for + consistency with the higher-level types. + */ + public java.nio.ByteBuffer value; + /** Initializes with the value null. */ + public ByteBuffer(){this(null);} + /** Initializes with the value v. */ + public ByteBuffer(java.nio.ByteBuffer v){value = v;} + /** Returns the current value. */ + public java.nio.ByteBuffer get(){return value;} + /** Sets the current value. */ + public void set(java.nio.ByteBuffer v){value = v;} + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java b/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java new file mode 100644 index 0000000000..35bb069c49 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java @@ -0,0 +1,81 @@ +/* +** 2023-09-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_prepare_multi}. +*/ +public interface PrepareMultiCallback extends CallbackProxy { + + /** + Gets passed a sqlite3_stmt which it may handle in arbitrary ways, + transferring ownership of it to this function. + + sqlite3_prepare_multi() will _not_ finalize st - it is up + to the call() implementation how st is handled. + + Must return 0 on success or an SQLITE_... code on error. If it + throws, sqlite3_prepare_multi() will transform the exception into + a db-level error in order to retain the C-style error semantics + of the API. + + See the {@link Finalize} class for a wrapper which finalizes the + statement after calling a proxy PrepareMultiCallback. + */ + int call(sqlite3_stmt st); + + /** + A PrepareMultiCallback impl which wraps a separate impl and finalizes + any sqlite3_stmt passed to its callback. + */ + public static final class Finalize implements PrepareMultiCallback { + private final PrepareMultiCallback p; + /** + p is the proxy to call() when this.call() is called. + */ + public Finalize( PrepareMultiCallback p ){ + this.p = p; + } + /** + Calls the call() method of the proxied callback and either returns its + result or propagates an exception. Either way, it passes its argument to + sqlite3_finalize() before returning. + */ + @Override public int call(sqlite3_stmt st){ + try { + return this.p.call(st); + }finally{ + CApi.sqlite3_finalize(st); + } + } + } + + /** + A PrepareMultiCallback impl which steps entirely through a result set, + ignoring all non-error results. + */ + final class StepAll implements PrepareMultiCallback { + public StepAll(){} + /** + Calls sqlite3_step() on st until it returns something other than + SQLITE_ROW. If the final result is SQLITE_DONE then 0 is returned, + else the result of the final step is returned. + */ + @Override public int call(sqlite3_stmt st){ + int rc = CApi.SQLITE_DONE; + while( CApi.SQLITE_ROW == (rc = CApi.sqlite3_step(st)) ){} + return CApi.SQLITE_DONE==rc ? 0 : rc; + } + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java new file mode 100644 index 0000000000..38f7c5613e --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java @@ -0,0 +1,27 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_preupdate_hook}. +*/ +public interface PreupdateHookCallback extends CallbackProxy { + /** + Must function as described for the C-level sqlite3_preupdate_hook() + callback. If it throws, the exception is translated to a + db-level error and the exception is suppressed. + */ + void call(sqlite3 db, int op, String dbName, String dbTable, + long iKey1, long iKey2 ); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java b/ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java new file mode 100644 index 0000000000..464baa2e3d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java @@ -0,0 +1,27 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_progress_handler}. +*/ +public interface ProgressHandlerCallback extends CallbackProxy { + /** + Works as documented for the C-level sqlite3_progress_handler() callback. + +

    If it throws, the exception message is passed on to the db and + the exception is suppressed. + */ + int call(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ResultCode.java b/ext/jni/src/org/sqlite/jni/capi/ResultCode.java new file mode 100644 index 0000000000..5a8b2e6a18 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ResultCode.java @@ -0,0 +1,155 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + This enum contains all of the core and "extended" result codes used + by the sqlite3 library. It is provided not for use with the C-style + API (with which it won't work) but for higher-level code which may + find it useful to map SQLite result codes to human-readable names. +*/ +public enum ResultCode { + SQLITE_OK(CApi.SQLITE_OK), + SQLITE_ERROR(CApi.SQLITE_ERROR), + SQLITE_INTERNAL(CApi.SQLITE_INTERNAL), + SQLITE_PERM(CApi.SQLITE_PERM), + SQLITE_ABORT(CApi.SQLITE_ABORT), + SQLITE_BUSY(CApi.SQLITE_BUSY), + SQLITE_LOCKED(CApi.SQLITE_LOCKED), + SQLITE_NOMEM(CApi.SQLITE_NOMEM), + SQLITE_READONLY(CApi.SQLITE_READONLY), + SQLITE_INTERRUPT(CApi.SQLITE_INTERRUPT), + SQLITE_IOERR(CApi.SQLITE_IOERR), + SQLITE_CORRUPT(CApi.SQLITE_CORRUPT), + SQLITE_NOTFOUND(CApi.SQLITE_NOTFOUND), + SQLITE_FULL(CApi.SQLITE_FULL), + SQLITE_CANTOPEN(CApi.SQLITE_CANTOPEN), + SQLITE_PROTOCOL(CApi.SQLITE_PROTOCOL), + SQLITE_EMPTY(CApi.SQLITE_EMPTY), + SQLITE_SCHEMA(CApi.SQLITE_SCHEMA), + SQLITE_TOOBIG(CApi.SQLITE_TOOBIG), + SQLITE_CONSTRAINT(CApi.SQLITE_CONSTRAINT), + SQLITE_MISMATCH(CApi.SQLITE_MISMATCH), + SQLITE_MISUSE(CApi.SQLITE_MISUSE), + SQLITE_NOLFS(CApi.SQLITE_NOLFS), + SQLITE_AUTH(CApi.SQLITE_AUTH), + SQLITE_FORMAT(CApi.SQLITE_FORMAT), + SQLITE_RANGE(CApi.SQLITE_RANGE), + SQLITE_NOTADB(CApi.SQLITE_NOTADB), + SQLITE_NOTICE(CApi.SQLITE_NOTICE), + SQLITE_WARNING(CApi.SQLITE_WARNING), + SQLITE_ROW(CApi.SQLITE_ROW), + SQLITE_DONE(CApi.SQLITE_DONE), + SQLITE_ERROR_MISSING_COLLSEQ(CApi.SQLITE_ERROR_MISSING_COLLSEQ), + SQLITE_ERROR_RETRY(CApi.SQLITE_ERROR_RETRY), + SQLITE_ERROR_SNAPSHOT(CApi.SQLITE_ERROR_SNAPSHOT), + SQLITE_IOERR_READ(CApi.SQLITE_IOERR_READ), + SQLITE_IOERR_SHORT_READ(CApi.SQLITE_IOERR_SHORT_READ), + SQLITE_IOERR_WRITE(CApi.SQLITE_IOERR_WRITE), + SQLITE_IOERR_FSYNC(CApi.SQLITE_IOERR_FSYNC), + SQLITE_IOERR_DIR_FSYNC(CApi.SQLITE_IOERR_DIR_FSYNC), + SQLITE_IOERR_TRUNCATE(CApi.SQLITE_IOERR_TRUNCATE), + SQLITE_IOERR_FSTAT(CApi.SQLITE_IOERR_FSTAT), + SQLITE_IOERR_UNLOCK(CApi.SQLITE_IOERR_UNLOCK), + SQLITE_IOERR_RDLOCK(CApi.SQLITE_IOERR_RDLOCK), + SQLITE_IOERR_DELETE(CApi.SQLITE_IOERR_DELETE), + SQLITE_IOERR_BLOCKED(CApi.SQLITE_IOERR_BLOCKED), + SQLITE_IOERR_NOMEM(CApi.SQLITE_IOERR_NOMEM), + SQLITE_IOERR_ACCESS(CApi.SQLITE_IOERR_ACCESS), + SQLITE_IOERR_CHECKRESERVEDLOCK(CApi.SQLITE_IOERR_CHECKRESERVEDLOCK), + SQLITE_IOERR_LOCK(CApi.SQLITE_IOERR_LOCK), + SQLITE_IOERR_CLOSE(CApi.SQLITE_IOERR_CLOSE), + SQLITE_IOERR_DIR_CLOSE(CApi.SQLITE_IOERR_DIR_CLOSE), + SQLITE_IOERR_SHMOPEN(CApi.SQLITE_IOERR_SHMOPEN), + SQLITE_IOERR_SHMSIZE(CApi.SQLITE_IOERR_SHMSIZE), + SQLITE_IOERR_SHMLOCK(CApi.SQLITE_IOERR_SHMLOCK), + SQLITE_IOERR_SHMMAP(CApi.SQLITE_IOERR_SHMMAP), + SQLITE_IOERR_SEEK(CApi.SQLITE_IOERR_SEEK), + SQLITE_IOERR_DELETE_NOENT(CApi.SQLITE_IOERR_DELETE_NOENT), + SQLITE_IOERR_MMAP(CApi.SQLITE_IOERR_MMAP), + SQLITE_IOERR_GETTEMPPATH(CApi.SQLITE_IOERR_GETTEMPPATH), + SQLITE_IOERR_CONVPATH(CApi.SQLITE_IOERR_CONVPATH), + SQLITE_IOERR_VNODE(CApi.SQLITE_IOERR_VNODE), + SQLITE_IOERR_AUTH(CApi.SQLITE_IOERR_AUTH), + SQLITE_IOERR_BEGIN_ATOMIC(CApi.SQLITE_IOERR_BEGIN_ATOMIC), + SQLITE_IOERR_COMMIT_ATOMIC(CApi.SQLITE_IOERR_COMMIT_ATOMIC), + SQLITE_IOERR_ROLLBACK_ATOMIC(CApi.SQLITE_IOERR_ROLLBACK_ATOMIC), + SQLITE_IOERR_DATA(CApi.SQLITE_IOERR_DATA), + SQLITE_IOERR_CORRUPTFS(CApi.SQLITE_IOERR_CORRUPTFS), + SQLITE_LOCKED_SHAREDCACHE(CApi.SQLITE_LOCKED_SHAREDCACHE), + SQLITE_LOCKED_VTAB(CApi.SQLITE_LOCKED_VTAB), + SQLITE_BUSY_RECOVERY(CApi.SQLITE_BUSY_RECOVERY), + SQLITE_BUSY_SNAPSHOT(CApi.SQLITE_BUSY_SNAPSHOT), + SQLITE_BUSY_TIMEOUT(CApi.SQLITE_BUSY_TIMEOUT), + SQLITE_CANTOPEN_NOTEMPDIR(CApi.SQLITE_CANTOPEN_NOTEMPDIR), + SQLITE_CANTOPEN_ISDIR(CApi.SQLITE_CANTOPEN_ISDIR), + SQLITE_CANTOPEN_FULLPATH(CApi.SQLITE_CANTOPEN_FULLPATH), + SQLITE_CANTOPEN_CONVPATH(CApi.SQLITE_CANTOPEN_CONVPATH), + SQLITE_CANTOPEN_SYMLINK(CApi.SQLITE_CANTOPEN_SYMLINK), + SQLITE_CORRUPT_VTAB(CApi.SQLITE_CORRUPT_VTAB), + SQLITE_CORRUPT_SEQUENCE(CApi.SQLITE_CORRUPT_SEQUENCE), + SQLITE_CORRUPT_INDEX(CApi.SQLITE_CORRUPT_INDEX), + SQLITE_READONLY_RECOVERY(CApi.SQLITE_READONLY_RECOVERY), + SQLITE_READONLY_CANTLOCK(CApi.SQLITE_READONLY_CANTLOCK), + SQLITE_READONLY_ROLLBACK(CApi.SQLITE_READONLY_ROLLBACK), + SQLITE_READONLY_DBMOVED(CApi.SQLITE_READONLY_DBMOVED), + SQLITE_READONLY_CANTINIT(CApi.SQLITE_READONLY_CANTINIT), + SQLITE_READONLY_DIRECTORY(CApi.SQLITE_READONLY_DIRECTORY), + SQLITE_ABORT_ROLLBACK(CApi.SQLITE_ABORT_ROLLBACK), + SQLITE_CONSTRAINT_CHECK(CApi.SQLITE_CONSTRAINT_CHECK), + SQLITE_CONSTRAINT_COMMITHOOK(CApi.SQLITE_CONSTRAINT_COMMITHOOK), + SQLITE_CONSTRAINT_FOREIGNKEY(CApi.SQLITE_CONSTRAINT_FOREIGNKEY), + SQLITE_CONSTRAINT_FUNCTION(CApi.SQLITE_CONSTRAINT_FUNCTION), + SQLITE_CONSTRAINT_NOTNULL(CApi.SQLITE_CONSTRAINT_NOTNULL), + SQLITE_CONSTRAINT_PRIMARYKEY(CApi.SQLITE_CONSTRAINT_PRIMARYKEY), + SQLITE_CONSTRAINT_TRIGGER(CApi.SQLITE_CONSTRAINT_TRIGGER), + SQLITE_CONSTRAINT_UNIQUE(CApi.SQLITE_CONSTRAINT_UNIQUE), + SQLITE_CONSTRAINT_VTAB(CApi.SQLITE_CONSTRAINT_VTAB), + SQLITE_CONSTRAINT_ROWID(CApi.SQLITE_CONSTRAINT_ROWID), + SQLITE_CONSTRAINT_PINNED(CApi.SQLITE_CONSTRAINT_PINNED), + SQLITE_CONSTRAINT_DATATYPE(CApi.SQLITE_CONSTRAINT_DATATYPE), + SQLITE_NOTICE_RECOVER_WAL(CApi.SQLITE_NOTICE_RECOVER_WAL), + SQLITE_NOTICE_RECOVER_ROLLBACK(CApi.SQLITE_NOTICE_RECOVER_ROLLBACK), + SQLITE_WARNING_AUTOINDEX(CApi.SQLITE_WARNING_AUTOINDEX), + SQLITE_AUTH_USER(CApi.SQLITE_AUTH_USER), + SQLITE_OK_LOAD_PERMANENTLY(CApi.SQLITE_OK_LOAD_PERMANENTLY); + + public final int value; + + ResultCode(int rc){ + value = rc; + ResultCodeMap.set(rc, this); + } + + /** + Returns the entry from this enum for the given result code, or + null if no match is found. + */ + public static ResultCode getEntryForInt(int rc){ + return ResultCodeMap.get(rc); + } + + /** + Internal level of indirection required because we cannot initialize + static enum members in an enum before the enum constructor is + invoked. + */ + private static final class ResultCodeMap { + private static final java.util.Map i2e + = new java.util.HashMap<>(); + private static void set(int rc, ResultCode e){ i2e.put(rc, e); } + private static ResultCode get(int rc){ return i2e.get(rc); } + } + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java new file mode 100644 index 0000000000..cf9c4b6e7a --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_rollback_hook}. +*/ +public interface RollbackHookCallback extends CallbackProxy { + /** + Must function as documented for the C-level sqlite3_rollback_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + void call(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/SQLFunction.java b/ext/jni/src/org/sqlite/jni/capi/SQLFunction.java new file mode 100644 index 0000000000..7ad1381a7a --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/SQLFunction.java @@ -0,0 +1,36 @@ +/* +** 2023-07-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + SQLFunction is used in conjunction with the + sqlite3_create_function() JNI-bound API to give that native code + access to the callback functions needed in order to implement SQL + functions in Java. + +

    + + This class is not used by itself, but is a marker base class. The + three UDF types are modelled by the inner classes Scalar, + Aggregate, and Window. Most simply, clients may subclass + those, or create anonymous classes from them, to implement + UDFs. Clients are free to create their own classes for use with + UDFs, so long as they conform to the public interfaces defined by + those three classes. The JNI layer only actively relies on the + SQLFunction base class and the method names and signatures used by + the UDF callback interfaces. +*/ +public interface SQLFunction { + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/SQLTester.java b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java new file mode 100644 index 0000000000..bc2e75f8be --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java @@ -0,0 +1,1449 @@ +/* +** 2023-08-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the main application entry pointer for the +** SQLTester framework. +*/ +package org.sqlite.jni.capi; +import java.util.ArrayList; +import java.util.Arrays; +import java.nio.charset.StandardCharsets; +import java.util.regex.*; +import static org.sqlite.jni.capi.CApi.*; + +/** + Modes for how to escape (or not) column values and names from + SQLTester.execSql() to the result buffer output. +*/ +enum ResultBufferMode { + //! Do not append to result buffer + NONE, + //! Append output escaped. + ESCAPED, + //! Append output as-is + ASIS +} + +/** + Modes to specify how to emit multi-row output from + SQLTester.execSql() to the result buffer. +*/ +enum ResultRowMode { + //! Keep all result rows on one line, space-separated. + ONELINE, + //! Add a newline between each result row. + NEWLINE +} + +/** + Base exception type for test-related failures. +*/ +class SQLTesterException extends RuntimeException { + private boolean bFatal = false; + + SQLTesterException(String msg){ + super(msg); + } + + protected SQLTesterException(String msg, boolean fatal){ + super(msg); + bFatal = fatal; + } + + /** + Indicates whether the framework should consider this exception + type as immediately fatal to the test run or not. + */ + final boolean isFatal(){ return bFatal; } +} + +class DbException extends SQLTesterException { + DbException(sqlite3 db, int rc, boolean closeDb){ + super("DB error #"+rc+": "+sqlite3_errmsg(db),true); + if( closeDb ) sqlite3_close_v2(db); + } + DbException(sqlite3 db, int rc){ + this(db, rc, false); + } +} + +/** + Generic test-failed exception. + */ +class TestScriptFailed extends SQLTesterException { + public TestScriptFailed(TestScript ts, String msg){ + super(ts.getOutputPrefix()+": "+msg, true); + } +} + +/** + Thrown when an unknown test command is encountered in a script. +*/ +class UnknownCommand extends SQLTesterException { + public UnknownCommand(TestScript ts, String cmd){ + super(ts.getOutputPrefix()+": unknown command: "+cmd, false); + } +} + +/** + Thrown when an "incompatible directive" is found in a script. This + can be the presence of a C-preprocessor construct, specific + metadata tags within a test script's header, or specific test + constructs which are incompatible with this particular + implementation. +*/ +class IncompatibleDirective extends SQLTesterException { + public IncompatibleDirective(TestScript ts, String line){ + super(ts.getOutputPrefix()+": incompatible directive: "+line, false); + } +} + +/** + Console output utility class. +*/ +class Outer { + private int verbosity = 0; + + static void out(Object val){ + System.out.print(val); + } + + Outer out(Object... vals){ + for(Object v : vals) out(v); + return this; + } + + Outer outln(Object... vals){ + out(vals).out("\n"); + return this; + } + + Outer verbose(Object... vals){ + if(verbosity>0){ + out("VERBOSE",(verbosity>1 ? "+: " : ": ")).outln(vals); + } + return this; + } + + void setVerbosity(int level){ + verbosity = level; + } + + int getVerbosity(){ + return verbosity; + } + + public boolean isVerbose(){return verbosity > 0;} + +} + +/** +

    This class provides an application which aims to implement the + rudimentary SQL-driven test tool described in the accompanying + {@code test-script-interpreter.md}. + +

    This class is an internal testing tool, not part of the public + interface but is (A) in the same package as the library because + access permissions require it to be so and (B) the JDK8 javadoc + offers no way to filter individual classes out of the doc + generation process (it can only exclude packages, but see (A)). + +

    An instance of this application provides a core set of services + which TestScript instances use for processing testing logic. + TestScripts, in turn, delegate the concrete test work to Command + objects, which the TestScript parses on their behalf. +*/ +public class SQLTester { + //! List of input script files. + private final java.util.List listInFiles = new ArrayList<>(); + //! Console output utility. + private final Outer outer = new Outer(); + //! Test input buffer. + private final StringBuilder inputBuffer = new StringBuilder(); + //! Test result buffer. + private final StringBuilder resultBuffer = new StringBuilder(); + //! Buffer for REQUIRED_PROPERTIES pragmas. + private final StringBuilder dbInitSql = new StringBuilder(); + //! Output representation of SQL NULL. + private String nullView = "nil"; + //! Total tests run. + private int nTotalTest = 0; + //! Total test script files run. + private int nTestFile = 0; + //! Number of scripts which were aborted. + private int nAbortedScript = 0; + //! Incremented by test case handlers + private int nTest = 0; + //! True to enable column name output from execSql() + private boolean emitColNames; + //! True to keep going regardless of how a test fails. + private boolean keepGoing = false; + //! The list of available db handles. + private final sqlite3[] aDb = new sqlite3[7]; + //! Index into aDb of the current db. + private int iCurrentDb = 0; + //! Name of the default db, re-created for each script. + private final String initialDbName = "test.db"; + + + public SQLTester(){ + reset(); + } + + void setVerbosity(int level){ + this.outer.setVerbosity( level ); + } + int getVerbosity(){ + return this.outer.getVerbosity(); + } + boolean isVerbose(){ + return this.outer.isVerbose(); + } + + void outputColumnNames(boolean b){ emitColNames = b; } + + void verbose(Object... vals){ + outer.verbose(vals); + } + + void outln(Object... vals){ + outer.outln(vals); + } + + void out(Object... vals){ + outer.out(vals); + } + + //! Adds the given test script to the to-test list. + public void addTestScript(String filename){ + listInFiles.add(filename); + //verbose("Added file ",filename); + } + + private void setupInitialDb() throws DbException { + if( null==aDb[0] ){ + Util.unlink(initialDbName); + openDb(0, initialDbName, true); + }else{ + outln("WARNING: setupInitialDb() unexpectedly ", + "triggered while it is opened."); + } + } + + static final String[] startEmoji = { + "🚴", "🏄", "🏇", "🤸", "⛹", "🏊", "⛷", "🧗", "🏋" + }; + static final int nStartEmoji = startEmoji.length; + static int iStartEmoji = 0; + + private static String nextStartEmoji(){ + return startEmoji[iStartEmoji++ % nStartEmoji]; + } + + public void runTests() throws Exception { + final long tStart = System.currentTimeMillis(); + for(String f : listInFiles){ + reset(); + ++nTestFile; + final TestScript ts = new TestScript(f); + outln(nextStartEmoji(), " starting [",f,"]"); + boolean threw = false; + final long timeStart = System.currentTimeMillis(); + try{ + ts.run(this); + }catch(SQLTesterException e){ + threw = true; + outln("🔥EXCEPTION: ",e.getClass().getSimpleName(),": ",e.getMessage()); + ++nAbortedScript; + if( keepGoing ) outln("Continuing anyway because of the keep-going option."); + else if( e.isFatal() ) throw e; + }finally{ + final long timeEnd = System.currentTimeMillis(); + outln("🏁",(threw ? "❌" : "✅")," ",nTest," test(s) in ", + (timeEnd-timeStart),"ms."); + } + } + final long tEnd = System.currentTimeMillis(); + outln("Total run-time: ",(tEnd-tStart),"ms"); + Util.unlink(initialDbName); + } + + private StringBuilder clearBuffer(StringBuilder b){ + b.setLength(0); + return b; + } + + StringBuilder clearInputBuffer(){ + return clearBuffer(inputBuffer); + } + + StringBuilder clearResultBuffer(){ + return clearBuffer(resultBuffer); + } + + StringBuilder getInputBuffer(){ return inputBuffer; } + + void appendInput(String n, boolean addNL){ + inputBuffer.append(n); + if(addNL) inputBuffer.append('\n'); + } + + void appendResult(String n, boolean addNL){ + resultBuffer.append(n); + if(addNL) resultBuffer.append('\n'); + } + + void appendDbInitSql(String n) throws DbException { + dbInitSql.append(n).append('\n'); + if( null!=getCurrentDb() ){ + //outln("RUNNING DB INIT CODE: ",n); + execSql(null, true, ResultBufferMode.NONE, null, n); + } + } + String getDbInitSql(){ return dbInitSql.toString(); } + + String getInputText(){ return inputBuffer.toString(); } + + String getResultText(){ return resultBuffer.toString(); } + + private String takeBuffer(StringBuilder b){ + final String rc = b.toString(); + clearBuffer(b); + return rc; + } + + String takeInputBuffer(){ return takeBuffer(inputBuffer); } + + String takeResultBuffer(){ return takeBuffer(resultBuffer); } + + int getCurrentDbId(){ return iCurrentDb; } + + SQLTester affirmDbId(int n) throws IndexOutOfBoundsException { + if(n<0 || n>=aDb.length){ + throw new IndexOutOfBoundsException("illegal db number: "+n); + } + return this; + } + + sqlite3 setCurrentDb(int n){ + affirmDbId(n); + iCurrentDb = n; + return this.aDb[n]; + } + + sqlite3 getCurrentDb(){ return aDb[iCurrentDb]; } + + sqlite3 getDbById(int id){ + return affirmDbId(id).aDb[id]; + } + + void closeDb(int id) { + final sqlite3 db = affirmDbId(id).aDb[id]; + if( null != db ){ + sqlite3_close_v2(db); + aDb[id] = null; + } + } + + void closeDb() { closeDb(iCurrentDb); } + + void closeAllDbs(){ + for(int i = 0; i 0){ + //outln("RUNNING DB INIT CODE: ",dbInitSql.toString()); + rc = execSql(db, false, ResultBufferMode.NONE, + null, dbInitSql.toString()); + } + if( 0!=rc ){ + throw new DbException(db, rc, true); + } + return aDb[iCurrentDb] = db; + } + + sqlite3 openDb(int slot, String name, boolean createIfNeeded) throws DbException { + affirmDbId(slot); + iCurrentDb = slot; + return openDb(name, createIfNeeded); + } + + /** + Resets all tester context state except for that related to + tracking running totals. + */ + void reset(){ + clearInputBuffer(); + clearResultBuffer(); + clearBuffer(dbInitSql); + closeAllDbs(); + nTest = 0; + nullView = "nil"; + emitColNames = false; + iCurrentDb = 0; + //dbInitSql.append("SELECT 1;"); + } + + void setNullValue(String v){nullView = v;} + + /** + If true, encountering an unknown command in a script causes the + remainder of the script to be skipped, rather than aborting the + whole script run. + */ + boolean skipUnknownCommands(){ + // Currently hard-coded. Potentially a flag someday. + return true; + } + + void incrementTestCounter(){ ++nTest; ++nTotalTest; } + + //! "Special" characters - we have to escape output if it contains any. + static final Pattern patternSpecial = Pattern.compile( + "[\\x00-\\x20\\x22\\x5c\\x7b\\x7d]" + ); + //! Either of '{' or '}'. + static final Pattern patternSquiggly = Pattern.compile("[{}]"); + + /** + Returns v or some escaped form of v, as defined in the tester's + spec doc. + */ + String escapeSqlValue(String v){ + if( "".equals(v) ) return "{}"; + Matcher m = patternSpecial.matcher(v); + if( !m.find() ){ + return v /* no escaping needed */; + } + m = patternSquiggly.matcher(v); + if( !m.find() ){ + return "{"+v+"}"; + } + final StringBuilder sb = new StringBuilder("\""); + final int n = v.length(); + for(int i = 0; i < n; ++i){ + final char ch = v.charAt(i); + switch(ch){ + case '\\': sb.append("\\\\"); break; + case '"': sb.append("\\\""); break; + default: + //verbose("CHAR ",(int)ch," ",ch," octal=",String.format("\\%03o", (int)ch)); + if( (int)ch < 32 ) sb.append(String.format("\\%03o", (int)ch)); + else sb.append(ch); + break; + } + } + sb.append("\""); + return sb.toString(); + } + + private void appendDbErr(sqlite3 db, StringBuilder sb, int rc){ + sb.append(org.sqlite.jni.capi.ResultCode.getEntryForInt(rc)).append(' '); + final String msg = escapeSqlValue(sqlite3_errmsg(db)); + if( '{' == msg.charAt(0) ){ + sb.append(msg); + }else{ + sb.append('{').append(msg).append('}'); + } + } + + /** + Runs SQL on behalf of test commands and outputs the results following + the very specific rules of the test framework. + + If db is null, getCurrentDb() is assumed. If throwOnError is true then + any db-side error will result in an exception, else they result in + the db's result code. + + appendMode specifies how/whether to append results to the result + buffer. rowMode specifies whether to output all results in a + single line or one line per row. If appendMode is + ResultBufferMode.NONE then rowMode is ignored and may be null. + */ + public int execSql(sqlite3 db, boolean throwOnError, + ResultBufferMode appendMode, ResultRowMode rowMode, + String sql) throws SQLTesterException { + if( null==db && null==aDb[0] ){ + // Delay opening of the initial db to enable tests to change its + // name and inject on-connect code via, e.g., the MEMDB + // directive. this setup as the potential to misinteract with + // auto-extension timing and must be done carefully. + setupInitialDb(); + } + final OutputPointer.Int32 oTail = new OutputPointer.Int32(); + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + final byte[] sqlUtf8 = sql.getBytes(StandardCharsets.UTF_8); + if( null==db ) db = getCurrentDb(); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + int rc = 0; + sqlite3_stmt stmt = null; + int spacing = 0 /* emit a space for --result if>0 */ ; + final StringBuilder sb = (ResultBufferMode.NONE==appendMode) + ? null : resultBuffer; + //outln("sqlChunk len= = ",sqlChunk.length); + try{ + while(pos < sqlChunk.length){ + if(pos > 0){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, + sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v2(db, sqlChunk, outStmt, oTail); + /*outln("PREPARE rc ",rc," oTail=",oTail.get(),": ", + new String(sqlChunk,StandardCharsets.UTF_8),"\n");*/ + if( 0!=rc ){ + if(throwOnError){ + throw new DbException(db, rc); + }else if( null!=sb ){ + appendDbErr(db, sb, rc); + } + break; + } + pos = oTail.value; + stmt = outStmt.take(); + if( null == stmt ){ + // empty statement was parsed. + continue; + } + if( null!=sb ){ + // Add the output to the result buffer... + final int nCol = sqlite3_column_count(stmt); + String colName = null, val = null; + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){ + for(int i = 0; i < nCol; ++i){ + if( spacing++ > 0 ) sb.append(' '); + if( emitColNames ){ + colName = sqlite3_column_name(stmt, i); + switch(appendMode){ + case ASIS: + sb.append( colName ); + break; + case ESCAPED: + sb.append( escapeSqlValue(colName) ); + break; + default: + throw new SQLTesterException("Unhandled ResultBufferMode: "+appendMode); + } + sb.append(' '); + } + val = sqlite3_column_text16(stmt, i); + if( null==val ){ + sb.append( nullView ); + continue; + } + switch(appendMode){ + case ASIS: + sb.append( val ); + break; + case ESCAPED: + sb.append( escapeSqlValue(val) ); + break; + default: + throw new SQLTesterException("Unhandled ResultBufferMode: "+appendMode); + } + } + if( ResultRowMode.NEWLINE == rowMode ){ + spacing = 0; + sb.append('\n'); + } + } + }else{ + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){} + } + sqlite3_finalize(stmt); + stmt = null; + if(SQLITE_ROW==rc || SQLITE_DONE==rc) rc = 0; + else if( rc!=0 ){ + if( null!=sb ){ + appendDbErr(db, sb, rc); + } + break; + } + } + }finally{ + sqlite3_reset(stmt + /* In order to trigger an exception in the + INSERT...RETURNING locking scenario: + https://sqlite.org/forum/forumpost/36f7a2e7494897df */); + sqlite3_finalize(stmt); + } + if( 0!=rc && throwOnError ){ + throw new DbException(db, rc); + } + return rc; + } + + public static void main(String[] argv) throws Exception{ + installCustomExtensions(); + boolean dumpInternals = false; + final SQLTester t = new SQLTester(); + for(String a : argv){ + if(a.startsWith("-")){ + final String flag = a.replaceFirst("-+",""); + if( flag.equals("verbose") ){ + // Use --verbose up to 3 times + t.setVerbosity(t.getVerbosity() + 1); + }else if( flag.equals("keep-going") ){ + t.keepGoing = true; + }else if( flag.equals("internals") ){ + dumpInternals = true; + }else{ + throw new IllegalArgumentException("Unhandled flag: "+flag); + } + continue; + } + t.addTestScript(a); + } + final AutoExtensionCallback ax = new AutoExtensionCallback() { + private final SQLTester tester = t; + @Override public int call(sqlite3 db){ + final String init = tester.getDbInitSql(); + if( !init.isEmpty() ){ + tester.execSql(db, true, ResultBufferMode.NONE, null, init); + } + return 0; + } + }; + sqlite3_auto_extension(ax); + try { + t.runTests(); + }finally{ + sqlite3_cancel_auto_extension(ax); + t.outln("Processed ",t.nTotalTest," test(s) in ",t.nTestFile," file(s)."); + if( t.nAbortedScript > 0 ){ + t.outln("Aborted ",t.nAbortedScript," script(s)."); + } + if( dumpInternals ){ + sqlite3_jni_internal_details(); + } + } + } + + /** + Internal impl of the public strglob() method. Neither argument + may be NULL and both _MUST_ be NUL-terminated. + */ + private static native int strglob(byte[] glob, byte[] txt); + + /** + Works essentially the same as sqlite3_strglob() except that the + glob character '#' matches a sequence of one or more digits. It + does not match when it appears at the start or middle of a series + of digits, e.g. "#23" or "1#3", but will match at the end, + e.g. "12#". + */ + static int strglob(String glob, String txt){ + return strglob( + (glob+"\0").getBytes(StandardCharsets.UTF_8), + (txt+"\0").getBytes(StandardCharsets.UTF_8) + ); + } + + /** + Sets up C-side components needed by the test framework. This must + not be called until main() is triggered so that it does not + interfere with library clients who don't use this class. + */ + static native void installCustomExtensions(); + static { + System.loadLibrary("sqlite3-jni") + /* Interestingly, when SQLTester is the main app, we have to + load that lib from here. The same load from CApi does + not happen early enough. Without this, + installCustomExtensions() is an unresolved symbol. */; + } + +} + +/** + General utilities for the SQLTester bits. +*/ +final class Util { + + //! Throws a new T, appending all msg args into a string for the message. + static void toss(Class errorType, Object... msg) throws Exception { + StringBuilder sb = new StringBuilder(); + for(Object s : msg) sb.append(s); + final java.lang.reflect.Constructor ctor = + errorType.getConstructor(String.class); + throw ctor.newInstance(sb.toString()); + } + + static void toss(Object... msg) throws Exception{ + toss(RuntimeException.class, msg); + } + + //! Tries to delete the given file, silently ignoring failure. + static void unlink(String filename){ + try{ + final java.io.File f = new java.io.File(filename); + f.delete(); + }catch(Exception e){ + /* ignore */ + } + } + + /** + Appends all entries in argv[1..end] into a space-separated + string, argv[0] is not included because it's expected to be a + command name. + */ + static String argvToString(String[] argv){ + StringBuilder sb = new StringBuilder(); + for(int i = 1; i < argv.length; ++i ){ + if( i>1 ) sb.append(" "); + sb.append( argv[i] ); + } + return sb.toString(); + } + +} + +/** + Base class for test script commands. It provides a set of utility + APIs for concrete command implementations. + + Each subclass must have a public no-arg ctor and must implement + the process() method which is abstract in this class. + + Commands are intended to be stateless, except perhaps for counters + and similar internals. Specifically, no state which changes the + behavior between any two invocations of process() should be + retained. +*/ +abstract class Command { + protected Command(){} + + /** + Must process one command-unit of work and either return + (on success) or throw (on error). + + The first two arguments specify the context of the test. The TestScript + provides the content of the test and the SQLTester providers the sandbox + in which that script is being evaluated. + + argv is a list with the command name followed by any arguments to + that command. The argcCheck() method from this class provides + very basic argc validation. + */ + public abstract void process( + SQLTester st, TestScript ts, String[] argv + ) throws Exception; + + /** + If argv.length-1 (-1 because the command's name is in argv[0]) does not + fall in the inclusive range (min,max) then this function throws. Use + a max value of -1 to mean unlimited. + */ + protected final void argcCheck(TestScript ts, String[] argv, int min, int max){ + int argc = argv.length-1; + if(argc=0 && argc>max)){ + if( min==max ){ + ts.toss(argv[0]," requires exactly ",min," argument(s)"); + }else if(max>0){ + ts.toss(argv[0]," requires ",min,"-",max," arguments."); + }else{ + ts.toss(argv[0]," requires at least ",min," arguments."); + } + } + } + + /** + Equivalent to argcCheck(argv,argc,argc). + */ + protected final void argcCheck(TestScript ts, String[] argv, int argc){ + argcCheck(ts, argv, argc, argc); + } +} + +//! --close command +class CloseDbCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,0,1); + int id; + if(argv.length>1){ + String arg = argv[1]; + if("all".equals(arg)){ + t.closeAllDbs(); + return; + } + else{ + id = Integer.parseInt(arg); + } + }else{ + id = t.getCurrentDbId(); + } + t.closeDb(id); + } +} + +//! --column-names command +class ColumnNamesCommand extends Command { + public void process( + SQLTester st, TestScript ts, String[] argv + ){ + argcCheck(ts,argv,1); + st.outputColumnNames( Integer.parseInt(argv[1])!=0 ); + } +} + +//! --db command +class DbCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,1); + t.setCurrentDb( Integer.parseInt(argv[1]) ); + } +} + +//! --glob command +class GlobCommand extends Command { + private boolean negate = false; + public GlobCommand(){} + protected GlobCommand(boolean negate){ this.negate = negate; } + + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,1,-1); + t.incrementTestCounter(); + final String sql = t.takeInputBuffer(); + int rc = t.execSql(null, true, ResultBufferMode.ESCAPED, + ResultRowMode.ONELINE, sql); + final String result = t.getResultText(); + final String sArgs = Util.argvToString(argv); + //t2.verbose2(argv[0]," rc = ",rc," result buffer:\n", result,"\nargs:\n",sArgs); + final String glob = Util.argvToString(argv); + rc = SQLTester.strglob(glob, result); + if( (negate && 0==rc) || (!negate && 0!=rc) ){ + ts.toss(argv[0], " mismatch: ", glob," vs input: ",result); + } + } +} + +//! --json command +class JsonCommand extends ResultCommand { + public JsonCommand(){ super(ResultBufferMode.ASIS); } +} + +//! --json-block command +class JsonBlockCommand extends TableResultCommand { + public JsonBlockCommand(){ super(true); } +} + +//! --new command +class NewDbCommand extends OpenDbCommand { + public NewDbCommand(){ super(true); } + public void process(SQLTester t, TestScript ts, String[] argv){ + if(argv.length>1){ + Util.unlink(argv[1]); + } + super.process(t, ts, argv); + } + +} + +//! Placeholder dummy/no-op/unimplemented commands +class NoopCommand extends Command { + private boolean verbose = false; + public NoopCommand(boolean verbose){ + this.verbose = verbose; + } + public NoopCommand(){} + public void process(SQLTester t, TestScript ts, String[] argv){ + if( this.verbose ){ + t.outln("Skipping unhandled command: "+argv[0]); + } + } +} + +//! --notglob command +class NotGlobCommand extends GlobCommand { + public NotGlobCommand(){ + super(true); + } +} + +//! --null command +class NullCommand extends Command { + public void process( + SQLTester st, TestScript ts, String[] argv + ){ + argcCheck(ts,argv,1); + st.setNullValue( argv[1] ); + } +} + +//! --open command +class OpenDbCommand extends Command { + private boolean createIfNeeded = false; + public OpenDbCommand(){} + protected OpenDbCommand(boolean c){createIfNeeded = c;} + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,1); + t.openDb(argv[1], createIfNeeded); + } +} + +//! --print command +class PrintCommand extends Command { + public void process( + SQLTester st, TestScript ts, String[] argv + ){ + st.out(ts.getOutputPrefix(),": "); + if( 1==argv.length ){ + st.out( st.getInputText() ); + }else{ + st.outln( Util.argvToString(argv) ); + } + } +} + +//! --result command +class ResultCommand extends Command { + private final ResultBufferMode bufferMode; + protected ResultCommand(ResultBufferMode bm){ bufferMode = bm; } + public ResultCommand(){ this(ResultBufferMode.ESCAPED); } + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,0,-1); + t.incrementTestCounter(); + final String sql = t.takeInputBuffer(); + //ts.verbose2(argv[0]," SQL =\n",sql); + int rc = t.execSql(null, false, bufferMode, ResultRowMode.ONELINE, sql); + final String result = t.getResultText().trim(); + final String sArgs = argv.length>1 ? Util.argvToString(argv) : ""; + if( !result.equals(sArgs) ){ + t.outln(argv[0]," FAILED comparison. Result buffer:\n", + result,"\nExpected result:\n",sArgs); + ts.toss(argv[0]+" comparison failed."); + } + } +} + +//! --run command +class RunCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,0,1); + final sqlite3 db = (1==argv.length) + ? t.getCurrentDb() : t.getDbById( Integer.parseInt(argv[1]) ); + final String sql = t.takeInputBuffer(); + final int rc = t.execSql(db, false, ResultBufferMode.NONE, + ResultRowMode.ONELINE, sql); + if( 0!=rc && t.isVerbose() ){ + String msg = sqlite3_errmsg(db); + ts.verbose1(argv[0]," non-fatal command error #",rc,": ", + msg,"\nfor SQL:\n",sql); + } + } +} + +//! --tableresult command +class TableResultCommand extends Command { + private final boolean jsonMode; + protected TableResultCommand(boolean jsonMode){ this.jsonMode = jsonMode; } + public TableResultCommand(){ this(false); } + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,0); + t.incrementTestCounter(); + String body = ts.fetchCommandBody(t); + if( null==body ) ts.toss("Missing ",argv[0]," body."); + body = body.trim(); + if( !body.endsWith("\n--end") ){ + ts.toss(argv[0], " must be terminated with --end."); + }else{ + body = body.substring(0, body.length()-6); + } + final String[] globs = body.split("\\s*\\n\\s*"); + if( globs.length < 1 ){ + ts.toss(argv[0], " requires 1 or more ", + (jsonMode ? "json snippets" : "globs"),"."); + } + final String sql = t.takeInputBuffer(); + t.execSql(null, true, + jsonMode ? ResultBufferMode.ASIS : ResultBufferMode.ESCAPED, + ResultRowMode.NEWLINE, sql); + final String rbuf = t.getResultText(); + final String[] res = rbuf.split("\n"); + if( res.length != globs.length ){ + ts.toss(argv[0], " failure: input has ", res.length, + " row(s) but expecting ",globs.length); + } + for(int i = 0; i < res.length; ++i){ + final String glob = globs[i].replaceAll("\\s+"," ").trim(); + //ts.verbose2(argv[0]," <<",glob,">> vs <<",res[i],">>"); + if( jsonMode ){ + if( !glob.equals(res[i]) ){ + ts.toss(argv[0], " json <<",glob, ">> does not match: <<", + res[i],">>"); + } + }else if( 0 != SQLTester.strglob(glob, res[i]) ){ + ts.toss(argv[0], " glob <<",glob,">> does not match: <<",res[i],">>"); + } + } + } +} + +//! --testcase command +class TestCaseCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,1); + ts.setTestCaseName(argv[1]); + t.clearResultBuffer(); + t.clearInputBuffer(); + } +} + +//! --verbosity command +class VerbosityCommand extends Command { + public void process(SQLTester t, TestScript ts, String[] argv){ + argcCheck(ts,argv,1); + ts.setVerbosity( Integer.parseInt(argv[1]) ); + } +} + +class CommandDispatcher { + + private static final java.util.Map commandMap = + new java.util.HashMap<>(); + + /** + Returns a (cached) instance mapped to name, or null if no match + is found. + */ + static Command getCommandByName(String name){ + Command rv = commandMap.get(name); + if( null!=rv ) return rv; + switch(name){ + case "close": rv = new CloseDbCommand(); break; + case "column-names": rv = new ColumnNamesCommand(); break; + case "db": rv = new DbCommand(); break; + case "glob": rv = new GlobCommand(); break; + case "json": rv = new JsonCommand(); break; + case "jsonglob": rv = new NoopCommand(true); break; + case "json-block": rv = new JsonBlockCommand(); break; + case "new": rv = new NewDbCommand(); break; + case "notglob": rv = new NotGlobCommand(); break; + case "null": rv = new NullCommand(); break; + case "oom": rv = new NoopCommand(); break; + case "open": rv = new OpenDbCommand(); break; + case "print": rv = new PrintCommand(); break; + case "result": rv = new ResultCommand(); break; + case "run": rv = new RunCommand(); break; + case "stmt-cache": rv = new NoopCommand(); break; + case "tableresult": rv = new TableResultCommand(); break; + case "testcase": rv = new TestCaseCommand(); break; + case "verbosity": rv = new VerbosityCommand(); break; + default: rv = null; break; + } + if( null!=rv ) commandMap.put(name, rv); + return rv; + } + + /** + Treats argv[0] as a command name, looks it up with + getCommandByName(), and calls process() on that instance, passing + it arguments given to this function. + */ + static void dispatch(SQLTester tester, TestScript ts, String[] argv) throws Exception{ + final Command cmd = getCommandByName(argv[0]); + if(null == cmd){ + throw new UnknownCommand(ts, argv[0]); + } + cmd.process(tester, ts, argv); + } +} + + +/** + This class represents a single test script. It handles (or + delegates) its the reading-in and parsing, but the details of + evaluation are delegated elsewhere. +*/ +class TestScript { + //! input file + private String filename = null; + //! Name pulled from the SCRIPT_MODULE_NAME directive of the file + private String moduleName = null; + //! Current test case name. + private String testCaseName = null; + //! Content buffer state. + private final Cursor cur = new Cursor(); + //! Utility for console output. + private final Outer outer = new Outer(); + + //! File content and parse state. + private static final class Cursor { + private final StringBuilder sb = new StringBuilder(); + byte[] src = null; + //! Current position in this.src. + int pos = 0; + //! Current line number. Starts at 0 for internal reasons and will + // line up with 1-based reality once parsing starts. + int lineNo = 0 /* yes, zero */; + //! Putback value for this.pos. + int putbackPos = 0; + //! Putback line number + int putbackLineNo = 0; + //! Peeked-to pos, used by peekLine() and consumePeeked(). + int peekedPos = 0; + //! Peeked-to line number. + int peekedLineNo = 0; + + //! Restore parsing state to the start of the stream. + void rewind(){ + sb.setLength(0); + pos = lineNo = putbackPos = putbackLineNo = peekedPos = peekedLineNo = 0 + /* kinda missing memset() about now. */; + } + } + + private byte[] readFile(String filename) throws Exception { + return java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(filename)); + } + + /** + Initializes the script with the content of the given file. + Throws if it cannot read the file. + */ + public TestScript(String filename) throws Exception{ + this.filename = filename; + setVerbosity(2); + cur.src = readFile(filename); + } + + public String getFilename(){ + return filename; + } + + public String getModuleName(){ + return moduleName; + } + + /** + Verbosity level 0 produces no debug/verbose output. Level 1 produces + some and level 2 produces more. + */ + public void setVerbosity(int level){ + outer.setVerbosity(level); + } + + public String getOutputPrefix(){ + String rc = "["+(moduleName==null ? "" : moduleName)+"]"; + if( null!=testCaseName ) rc += "["+testCaseName+"]"; + if( null!=filename ) rc += "["+filename+"]"; + return rc + " line "+ cur.lineNo; + } + + static final String[] verboseLabel = {"🔈",/*"🔉",*/"🔊","📢"}; + //! Output vals only if level<=current verbosity level. + private TestScript verboseN(int level, Object... vals){ + final int verbosity = outer.getVerbosity(); + if(verbosity>=level){ + outer.out( verboseLabel[level-1], getOutputPrefix(), " ",level,": " + ).outln(vals); + } + return this; + } + + TestScript verbose1(Object... vals){return verboseN(1,vals);} + TestScript verbose2(Object... vals){return verboseN(2,vals);} + TestScript verbose3(Object... vals){return verboseN(3,vals);} + + private void reset(){ + testCaseName = null; + cur.rewind(); + } + + void setTestCaseName(String n){ testCaseName = n; } + + /** + Returns the next line from the buffer, minus the trailing EOL. + + Returns null when all input is consumed. Throws if it reads + illegally-encoded input, e.g. (non-)characters in the range + 128-256. + */ + String getLine(){ + if( cur.pos==cur.src.length ){ + return null /* EOF */; + } + cur.putbackPos = cur.pos; + cur.putbackLineNo = cur.lineNo; + cur.sb.setLength(0); + final boolean skipLeadingWs = false; + byte b = 0, prevB = 0; + int i = cur.pos; + if(skipLeadingWs) { + /* Skip any leading spaces, including newlines. This will eliminate + blank lines. */ + for(; i < cur.src.length; ++i, prevB=b){ + b = cur.src[i]; + switch((int)b){ + case 32/*space*/: case 9/*tab*/: case 13/*CR*/: continue; + case 10/*NL*/: ++cur.lineNo; continue; + default: break; + } + break; + } + if( i==cur.src.length ){ + return null /* EOF */; + } + } + boolean doBreak = false; + final byte[] aChar = {0,0,0,0} /* multi-byte char buffer */; + int nChar = 0 /* number of bytes in the char */; + for(; i < cur.src.length && !doBreak; ++i){ + b = cur.src[i]; + switch( (int)b ){ + case 13/*CR*/: continue; + case 10/*NL*/: + ++cur.lineNo; + if(cur.sb.length()>0) doBreak = true; + // Else it's an empty string + break; + default: + /* Multi-byte chars need to be gathered up and appended at + one time. Appending individual bytes to the StringBuffer + appends their integer value. */ + nChar = 1; + switch( b & 0xF0 ){ + case 0xC0: nChar = 2; break; + case 0xE0: nChar = 3; break; + case 0xF0: nChar = 4; break; + default: + if( b > 127 ) this.toss("Invalid character (#"+(int)b+")."); + break; + } + if( 1==nChar ){ + cur.sb.append((char)b); + }else{ + for(int x = 0; x < nChar; ++x) aChar[x] = cur.src[i+x]; + cur.sb.append(new String(Arrays.copyOf(aChar, nChar), + StandardCharsets.UTF_8)); + i += nChar-1; + } + break; + } + } + cur.pos = i; + final String rv = cur.sb.toString(); + if( i==cur.src.length && rv.isEmpty() ){ + return null /* EOF */; + } + return rv; + }/*getLine()*/ + + /** + Fetches the next line then resets the cursor to its pre-call + state. consumePeeked() can be used to consume this peeked line + without having to re-parse it. + */ + String peekLine(){ + final int oldPos = cur.pos; + final int oldPB = cur.putbackPos; + final int oldPBL = cur.putbackLineNo; + final int oldLine = cur.lineNo; + try{ return getLine(); } + finally{ + cur.peekedPos = cur.pos; + cur.peekedLineNo = cur.lineNo; + cur.pos = oldPos; + cur.lineNo = oldLine; + cur.putbackPos = oldPB; + cur.putbackLineNo = oldPBL; + } + } + + /** + Only valid after calling peekLine() and before calling getLine(). + This places the cursor to the position it would have been at had + the peekLine() had been fetched with getLine(). + */ + void consumePeeked(){ + cur.pos = cur.peekedPos; + cur.lineNo = cur.peekedLineNo; + } + + /** + Restores the cursor to the position it had before the previous + call to getLine(). + */ + void putbackLine(){ + cur.pos = cur.putbackPos; + cur.lineNo = cur.putbackLineNo; + } + + private boolean checkRequiredProperties(SQLTester t, String[] props) throws SQLTesterException{ + if( true ) return false; + int nOk = 0; + for(String rp : props){ + verbose1("REQUIRED_PROPERTIES: ",rp); + switch(rp){ + case "RECURSIVE_TRIGGERS": + t.appendDbInitSql("pragma recursive_triggers=on;"); + ++nOk; + break; + case "TEMPSTORE_FILE": + /* This _assumes_ that the lib is built with SQLITE_TEMP_STORE=1 or 2, + which we just happen to know is the case */ + t.appendDbInitSql("pragma temp_store=1;"); + ++nOk; + break; + case "TEMPSTORE_MEM": + /* This _assumes_ that the lib is built with SQLITE_TEMP_STORE=1 or 2, + which we just happen to know is the case */ + t.appendDbInitSql("pragma temp_store=0;"); + ++nOk; + break; + case "AUTOVACUUM": + t.appendDbInitSql("pragma auto_vacuum=full;"); + ++nOk; + case "INCRVACUUM": + t.appendDbInitSql("pragma auto_vacuum=incremental;"); + ++nOk; + default: + break; + } + } + return props.length == nOk; + } + + private static final Pattern patternRequiredProperties = + Pattern.compile(" REQUIRED_PROPERTIES:[ \\t]*(\\S.*)\\s*$"); + private static final Pattern patternScriptModuleName = + Pattern.compile(" SCRIPT_MODULE_NAME:[ \\t]*(\\S+)\\s*$"); + private static final Pattern patternMixedModuleName = + Pattern.compile(" ((MIXED_)?MODULE_NAME):[ \\t]*(\\S+)\\s*$"); + private static final Pattern patternCommand = + Pattern.compile("^--(([a-z-]+)( .*)?)$"); + + /** + Looks for "directives." If a compatible one is found, it is + processed and this function returns. If an incompatible one is found, + a description of it is returned and processing of the test must + end immediately. + */ + private void checkForDirective( + SQLTester tester, String line + ) throws IncompatibleDirective { + if(line.startsWith("#")){ + throw new IncompatibleDirective(this, "C-preprocessor input: "+line); + }else if(line.startsWith("---")){ + new IncompatibleDirective(this, "triple-dash: "+line); + } + Matcher m = patternScriptModuleName.matcher(line); + if( m.find() ){ + moduleName = m.group(1); + return; + } + m = patternRequiredProperties.matcher(line); + if( m.find() ){ + final String rp = m.group(1); + if( ! checkRequiredProperties( tester, rp.split("\\s+") ) ){ + throw new IncompatibleDirective(this, "REQUIRED_PROPERTIES: "+rp); + } + } + m = patternMixedModuleName.matcher(line); + if( m.find() ){ + throw new IncompatibleDirective(this, m.group(1)+": "+m.group(3)); + } + if( line.contains("\n|") ){ + throw new IncompatibleDirective(this, "newline-pipe combination."); + } + return; + } + + boolean isCommandLine(String line, boolean checkForImpl){ + final Matcher m = patternCommand.matcher(line); + boolean rc = m.find(); + if( rc && checkForImpl ){ + rc = null!=CommandDispatcher.getCommandByName(m.group(2)); + } + return rc; + } + + /** + If line looks like a command, returns an argv for that command + invocation, else returns null. + */ + String[] getCommandArgv(String line){ + final Matcher m = patternCommand.matcher(line); + return m.find() ? m.group(1).trim().split("\\s+") : null; + } + + /** + Fetches lines until the next recognized command. Throws if + checkForDirective() does. Returns null if there is no input or + it's only whitespace. The returned string retains all whitespace. + + Note that "subcommands", --command-like constructs in the body + which do not match a known command name are considered to be + content, not commands. + */ + String fetchCommandBody(SQLTester tester){ + final StringBuilder sb = new StringBuilder(); + String line; + while( (null != (line = peekLine())) ){ + checkForDirective(tester, line); + if( isCommandLine(line, true) ) break; + else { + sb.append(line).append("\n"); + consumePeeked(); + } + } + line = sb.toString(); + return line.trim().isEmpty() ? null : line; + } + + private void processCommand(SQLTester t, String[] argv) throws Exception{ + verbose1("running command: ",argv[0], " ", Util.argvToString(argv)); + if(outer.getVerbosity()>1){ + final String input = t.getInputText(); + if( !input.isEmpty() ) verbose3("Input buffer = ",input); + } + CommandDispatcher.dispatch(t, this, argv); + } + + void toss(Object... msg) throws TestScriptFailed { + StringBuilder sb = new StringBuilder(); + for(Object s : msg) sb.append(s); + throw new TestScriptFailed(this, sb.toString()); + } + + /** + Runs this test script in the context of the given tester object. + */ + public boolean run(SQLTester tester) throws Exception { + reset(); + setVerbosity(tester.getVerbosity()); + String line, directive; + String[] argv; + while( null != (line = getLine()) ){ + verbose3("input line: ",line); + checkForDirective(tester, line); + argv = getCommandArgv(line); + if( null!=argv ){ + processCommand(tester, argv); + continue; + } + tester.appendInput(line,true); + } + return true; + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java b/ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java new file mode 100644 index 0000000000..95541bdcba --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java @@ -0,0 +1,33 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + + +/** + A SQLFunction implementation for scalar functions. +*/ +public abstract class ScalarFunction implements SQLFunction { + /** + As for the xFunc() argument of the C API's + sqlite3_create_function(). If this function throws, it is + translated into an sqlite3_result_error(). + */ + public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. This default implementation does nothing. + */ + public void xDestroy() {} +} diff --git a/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java b/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java new file mode 100644 index 0000000000..54808cd1ca --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java @@ -0,0 +1,35 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper object for use with sqlite3_table_column_metadata(). + They are populated only via that interface. +*/ +public final class TableColumnMetadata { + final OutputPointer.Bool pNotNull = new OutputPointer.Bool(); + final OutputPointer.Bool pPrimaryKey = new OutputPointer.Bool(); + final OutputPointer.Bool pAutoinc = new OutputPointer.Bool(); + final OutputPointer.String pzCollSeq = new OutputPointer.String(); + final OutputPointer.String pzDataType = new OutputPointer.String(); + + public TableColumnMetadata(){ + } + + public String getDataType(){ return pzDataType.value; } + public String getCollation(){ return pzCollSeq.value; } + public boolean isNotNull(){ return pNotNull.value; } + public boolean isPrimaryKey(){ return pPrimaryKey.value; } + public boolean isAutoincrement(){ return pAutoinc.value; } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java new file mode 100644 index 0000000000..9d14c954b8 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java @@ -0,0 +1,2207 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a set of tests for the sqlite3 JNI bindings. +*/ +package org.sqlite.jni.capi; +import static org.sqlite.jni.capi.CApi.*; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + An annotation for Tester1 tests which we do not want to run in + reflection-driven test mode because either they are not suitable + for multi-threaded threaded mode or we have to control their execution + order. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface ManualTest{} +/** + Annotation for Tester1 tests which mark those which must be skipped + in multi-threaded mode. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface SingleThreadOnly{} + +/** + Annotation for Tester1 tests which must only be run if + sqlite3_jni_supports_nio() is true. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface RequiresJniNio{} + +public class Tester1 implements Runnable { + //! True when running in multi-threaded mode. + private static boolean mtMode = false; + //! True to sleep briefly between tests. + private static boolean takeNaps = false; + //! True to shuffle the order of the tests. + private static boolean shuffle = false; + //! True to dump the list of to-run tests to stdout. + private static int listRunTests = 0; + //! True to squelch all out() and outln() output. + private static boolean quietMode = false; + //! Total number of runTests() calls. + private static int nTestRuns = 0; + //! List of test*() methods to run. + private static List testMethods = null; + //! List of exceptions collected by run() + private static final List listErrors = new ArrayList<>(); + private static final class Metrics { + //! Number of times createNewDb() (or equivalent) is invoked. + volatile int dbOpen = 0; + } + + private final Integer tId; + + Tester1(Integer id){ + tId = id; + } + + static final Metrics metrics = new Metrics(); + + public static synchronized void outln(){ + if( !quietMode ){ + System.out.println(); + } + } + + public static synchronized void outPrefix(){ + if( !quietMode ){ + System.out.print(Thread.currentThread().getName()+": "); + } + } + + public static synchronized void outln(Object val){ + if( !quietMode ){ + outPrefix(); + System.out.println(val); + } + } + + public static synchronized void out(Object val){ + if( !quietMode ){ + System.out.print(val); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void out(Object... vals){ + if( !quietMode ){ + outPrefix(); + for(Object v : vals) out(v); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void outln(Object... vals){ + if( !quietMode ){ + out(vals); out("\n"); + } + } + + static volatile int affirmCount = 0; + public static synchronized int affirm(Boolean v, String comment){ + ++affirmCount; + if( false ) assert( v /* prefer assert over exception if it's enabled because + the JNI layer sometimes has to suppress exceptions, + so they might be squelched on their way back to the + top. */); + if( !v ) throw new RuntimeException(comment); + return affirmCount; + } + + public static void affirm(Boolean v){ + affirm(v, "Affirmation failed."); + } + + @SingleThreadOnly /* because it's thread-agnostic */ + private void test1(){ + affirm(sqlite3_libversion_number() == SQLITE_VERSION_NUMBER); + } + + public static sqlite3 createNewDb(){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + int rc = sqlite3_open(":memory:", out); + ++metrics.dbOpen; + sqlite3 db = out.take(); + if( 0!=rc ){ + final String msg = + null==db ? sqlite3_errstr(rc) : sqlite3_errmsg(db); + sqlite3_close(db); + throw new RuntimeException("Opening db failed: "+msg); + } + affirm( null == out.get() ); + affirm( 0 != db.getNativePointer() ); + rc = sqlite3_busy_timeout(db, 2000); + affirm( 0 == rc ); + return db; + } + + public static void execSql(sqlite3 db, String[] sql){ + execSql(db, String.join("", sql)); + } + + public static int execSql(sqlite3 db, boolean throwOnError, String sql){ + OutputPointer.Int32 oTail = new OutputPointer.Int32(); + final byte[] sqlUtf8 = sql.getBytes(StandardCharsets.UTF_8); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + int rc = 0; + sqlite3_stmt stmt = null; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + while(pos < sqlChunk.length){ + if(pos > 0){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, + sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v2(db, sqlChunk, outStmt, oTail); + if(throwOnError) affirm(0 == rc); + else if( 0!=rc ) break; + pos = oTail.value; + stmt = outStmt.take(); + if( null == stmt ){ + // empty statement was parsed. + continue; + } + affirm(0 != stmt.getNativePointer()); + while( SQLITE_ROW == (rc = sqlite3_step(stmt)) ){ + } + sqlite3_finalize(stmt); + affirm(0 == stmt.getNativePointer()); + if(0!=rc && SQLITE_ROW!=rc && SQLITE_DONE!=rc){ + break; + } + } + sqlite3_finalize(stmt); + if(SQLITE_ROW==rc || SQLITE_DONE==rc) rc = 0; + if( 0!=rc && throwOnError){ + throw new RuntimeException("db op failed with rc=" + +rc+": "+sqlite3_errmsg(db)); + } + return rc; + } + + public static void execSql(sqlite3 db, String sql){ + execSql(db, true, sql); + } + + public static sqlite3_stmt prepare(sqlite3 db, boolean throwOnError, String sql){ + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + int rc = sqlite3_prepare_v2(db, sql, outStmt); + if( throwOnError ){ + affirm( 0 == rc ); + } + final sqlite3_stmt rv = outStmt.take(); + affirm( null == outStmt.get() ); + if( throwOnError ){ + affirm( 0 != rv.getNativePointer() ); + } + return rv; + } + + public static sqlite3_stmt prepare(sqlite3 db, String sql){ + return prepare(db, true, sql); + } + + private void showCompileOption(){ + int i = 0; + String optName; + outln("compile options:"); + for( ; null != (optName = sqlite3_compileoption_get(i)); ++i){ + outln("\t"+optName+"\t (used="+ + sqlite3_compileoption_used(optName)+")"); + } + } + + private void testCompileOption(){ + int i = 0; + String optName; + for( ; null != (optName = sqlite3_compileoption_get(i)); ++i){ + } + affirm( i > 10 ); + affirm( null==sqlite3_compileoption_get(-1) ); + } + + private void testOpenDb1(){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + int rc = sqlite3_open(":memory:", out); + ++metrics.dbOpen; + sqlite3 db = out.get(); + affirm(0 == rc); + affirm(db.getNativePointer()!=0); + sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, null) + /* This function has different mangled names in jdk8 vs jdk19, + and this call is here to ensure that the build fails + if it cannot find both names. */; + + affirm( 0==sqlite3_db_readonly(db,"main") ); + affirm( 0==sqlite3_db_readonly(db,null) ); + affirm( 0>sqlite3_db_readonly(db,"nope") ); + affirm( 0>sqlite3_db_readonly(null,null) ); + affirm( 0==sqlite3_last_insert_rowid(null) ); + + // These interrupt checks are only to make sure that the JNI binding + // has the proper exported symbol names. They don't actually test + // anything useful. + affirm( !sqlite3_is_interrupted(db) ); + sqlite3_interrupt(db); + affirm( sqlite3_is_interrupted(db) ); + sqlite3_close_v2(db); + affirm(0 == db.getNativePointer()); + } + + private void testOpenDb2(){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + int rc = sqlite3_open_v2(":memory:", out, + SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE, null); + ++metrics.dbOpen; + affirm(0 == rc); + sqlite3 db = out.get(); + affirm(0 != db.getNativePointer()); + sqlite3_close_v2(db); + affirm(0 == db.getNativePointer()); + } + + private void testPrepare123(){ + sqlite3 db = createNewDb(); + int rc; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + rc = sqlite3_prepare(db, "CREATE TABLE t1(a);", outStmt); + affirm(0 == rc); + sqlite3_stmt stmt = outStmt.take(); + affirm(0 != stmt.getNativePointer()); + affirm( !sqlite3_stmt_readonly(stmt) ); + affirm( db == sqlite3_db_handle(stmt) ); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE == rc); + sqlite3_finalize(stmt); + affirm( null == sqlite3_db_handle(stmt) ); + affirm(0 == stmt.getNativePointer()); + + { /* Demonstrate how to use the "zTail" option of + sqlite3_prepare() family of functions. */ + OutputPointer.Int32 oTail = new OutputPointer.Int32(); + final byte[] sqlUtf8 = + "CREATE TABLE t2(a); INSERT INTO t2(a) VALUES(1),(2),(3)" + .getBytes(StandardCharsets.UTF_8); + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + while(pos < sqlChunk.length){ + if(pos > 0){ + sqlChunk = Arrays.copyOfRange(sqlChunk, pos, sqlChunk.length); + } + //outln("SQL chunk #"+n+" length = "+sqlChunk.length+", pos = "+pos); + if( 0==sqlChunk.length ) break; + rc = sqlite3_prepare_v2(db, sqlChunk, outStmt, oTail); + affirm(0 == rc); + stmt = outStmt.get(); + pos = oTail.value; + /*outln("SQL tail pos = "+pos+". Chunk = "+ + (new String(Arrays.copyOfRange(sqlChunk,0,pos), + StandardCharsets.UTF_8)));*/ + switch(n){ + case 1: affirm(19 == pos); break; + case 2: affirm(36 == pos); break; + default: affirm( false /* can't happen */ ); + + } + ++n; + affirm(0 != stmt.getNativePointer()); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE == rc); + sqlite3_finalize(stmt); + affirm(0 == stmt.getNativePointer()); + } + } + + + rc = sqlite3_prepare_v3(db, "INSERT INTO t2(a) VALUES(1),(2),(3)", + 0, outStmt); + affirm(0 == rc); + stmt = outStmt.get(); + affirm(0 != stmt.getNativePointer()); + sqlite3_finalize(stmt); + affirm(0 == stmt.getNativePointer() ); + + affirm( 0==sqlite3_errcode(db) ); + stmt = sqlite3_prepare(db, "intentional error"); + affirm( null==stmt ); + affirm( 0!=sqlite3_errcode(db) ); + affirm( 0==sqlite3_errmsg(db).indexOf("near \"intentional\"") ); + sqlite3_finalize(stmt); + stmt = sqlite3_prepare(db, "/* empty input*/\n-- comments only"); + affirm( null==stmt ); + affirm( 0==sqlite3_errcode(db) ); + sqlite3_close_v2(db); + } + + private void testBindFetchInt(){ + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(:a);"); + affirm(1 == sqlite3_bind_parameter_count(stmt)); + final int paramNdx = sqlite3_bind_parameter_index(stmt, ":a"); + affirm(1 == paramNdx); + affirm( ":a".equals(sqlite3_bind_parameter_name(stmt, paramNdx))); + int total1 = 0; + long rowid = -1; + int changes = sqlite3_changes(db); + int changesT = sqlite3_total_changes(db); + long changes64 = sqlite3_changes64(db); + long changesT64 = sqlite3_total_changes64(db); + int rc; + for(int i = 99; i < 102; ++i ){ + total1 += i; + rc = sqlite3_bind_int(stmt, paramNdx, i); + affirm(0 == rc); + rc = sqlite3_step(stmt); + sqlite3_reset(stmt); + affirm(SQLITE_DONE == rc); + long x = sqlite3_last_insert_rowid(db); + affirm(x > rowid); + rowid = x; + } + sqlite3_finalize(stmt); + affirm(300 == total1); + affirm(sqlite3_changes(db) > changes); + affirm(sqlite3_total_changes(db) > changesT); + affirm(sqlite3_changes64(db) > changes64); + affirm(sqlite3_total_changes64(db) > changesT64); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + affirm( sqlite3_stmt_readonly(stmt) ); + affirm( !sqlite3_stmt_busy(stmt) ); + if( sqlite3_compileoption_used("ENABLE_COLUMN_METADATA") ){ + /* Unlike in native C code, JNI won't trigger an + UnsatisfiedLinkError until these are called (on Linux, at + least). */ + affirm("t".equals(sqlite3_column_table_name(stmt,0))); + affirm("main".equals(sqlite3_column_database_name(stmt,0))); + affirm("a".equals(sqlite3_column_origin_name(stmt,0))); + } + + int total2 = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + affirm( sqlite3_stmt_busy(stmt) ); + total2 += sqlite3_column_int(stmt, 0); + sqlite3_value sv = sqlite3_column_value(stmt, 0); + affirm( null != sv ); + affirm( 0 != sv.getNativePointer() ); + affirm( SQLITE_INTEGER == sqlite3_value_type(sv) ); + } + affirm( !sqlite3_stmt_busy(stmt) ); + sqlite3_finalize(stmt); + affirm(total1 == total2); + + // sqlite3_value_frombind() checks... + stmt = prepare(db, "SELECT 1, ?"); + sqlite3_bind_int(stmt, 1, 2); + rc = sqlite3_step(stmt); + affirm( SQLITE_ROW==rc ); + affirm( !sqlite3_value_frombind(sqlite3_column_value(stmt, 0)) ); + affirm( sqlite3_value_frombind(sqlite3_column_value(stmt, 1)) ); + sqlite3_finalize(stmt); + + sqlite3_close_v2(db); + affirm(0 == db.getNativePointer()); + } + + private void testBindFetchInt64(){ + try (sqlite3 db = createNewDb()){ + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + long total1 = 0; + for(long i = 0xffffffff; i < 0xffffffff + 3; ++i ){ + total1 += i; + sqlite3_bind_int64(stmt, 1, i); + sqlite3_step(stmt); + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + long total2 = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + total2 += sqlite3_column_int64(stmt, 0); + } + sqlite3_finalize(stmt); + affirm(total1 == total2); + //sqlite3_close_v2(db); + } + } + + private void testBindFetchDouble(){ + try (sqlite3 db = createNewDb()){ + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + double total1 = 0; + for(double i = 1.5; i < 5.0; i = i + 1.0 ){ + total1 += i; + sqlite3_bind_double(stmt, 1, i); + sqlite3_step(stmt); + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + double total2 = 0; + int counter = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + ++counter; + total2 += sqlite3_column_double(stmt, 0); + } + affirm(4 == counter); + sqlite3_finalize(stmt); + affirm(total2<=total1+0.01 && total2>=total1-0.01); + //sqlite3_close_v2(db); + } + } + + private void testBindFetchText(){ + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + String[] list1 = { "hell🤩", "w😃rld", "!🤩" }; + int rc; + int n = 0; + for( String e : list1 ){ + rc = (0==n) + ? sqlite3_bind_text(stmt, 1, e) + : sqlite3_bind_text16(stmt, 1, e); + affirm(0 == rc); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE==rc); + sqlite3_reset(stmt); + } + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + StringBuilder sbuf = new StringBuilder(); + n = 0; + final boolean tryNio = sqlite3_jni_supports_nio(); + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final sqlite3_value sv = sqlite3_value_dup(sqlite3_column_value(stmt,0)); + final String txt = sqlite3_column_text16(stmt, 0); + sbuf.append( txt ); + affirm( txt.equals(new String( + sqlite3_column_text(stmt, 0), + StandardCharsets.UTF_8 + )) ); + affirm( txt.length() < sqlite3_value_bytes(sv) ); + affirm( txt.equals(new String( + sqlite3_value_text(sv), + StandardCharsets.UTF_8)) ); + affirm( txt.length() == sqlite3_value_bytes16(sv)/2 ); + affirm( txt.equals(sqlite3_value_text16(sv)) ); + if( tryNio ){ + java.nio.ByteBuffer bu = sqlite3_value_nio_buffer(sv); + byte ba[] = sqlite3_value_blob(sv); + affirm( ba.length == bu.capacity() ); + int i = 0; + for( byte b : ba ){ + affirm( b == bu.get(i++) ); + } + } + sqlite3_value_free(sv); + ++n; + } + sqlite3_finalize(stmt); + affirm(3 == n); + affirm("w😃rldhell🤩!🤩".contentEquals(sbuf)); + + try( sqlite3_stmt stmt2 = prepare(db, "SELECT ?, ?") ){ + rc = sqlite3_bind_text(stmt2, 1, ""); + affirm( 0==rc ); + rc = sqlite3_bind_text(stmt2, 2, (String)null); + affirm( 0==rc ); + rc = sqlite3_step(stmt2); + affirm( SQLITE_ROW==rc ); + byte[] colBa = sqlite3_column_text(stmt2, 0); + affirm( 0==colBa.length ); + colBa = sqlite3_column_text(stmt2, 1); + affirm( null==colBa ); + //sqlite3_finalize(stmt); + } + + if(true){ + sqlite3_close_v2(db); + }else{ + // Let the Object.finalize() override deal with it. + } + } + + private void testBindFetchBlob(){ + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + byte[] list1 = { 0x32, 0x33, 0x34 }; + int rc = sqlite3_bind_blob(stmt, 1, list1); + affirm( 0==rc ); + rc = sqlite3_step(stmt); + affirm(SQLITE_DONE == rc); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a DESC;"); + int n = 0; + int total = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + byte[] blob = sqlite3_column_blob(stmt, 0); + affirm(3 == blob.length); + int i = 0; + for(byte b : blob){ + affirm(b == list1[i++]); + total += b; + } + ++n; + } + sqlite3_finalize(stmt); + affirm(1 == n); + affirm(total == 0x32 + 0x33 + 0x34); + sqlite3_close_v2(db); + } + + @RequiresJniNio + private void testBindByteBuffer(){ + /* TODO: these tests need to be much more extensive to check the + begin/end range handling. */ + + java.nio.ByteBuffer zeroCheck = + java.nio.ByteBuffer.allocateDirect(0); + affirm( null != zeroCheck ); + zeroCheck = null; + sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a)"); + + final java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocateDirect(10); + buf.put((byte)0x31)/*note that we'll skip this one*/ + .put((byte)0x32) + .put((byte)0x33) + .put((byte)0x34) + .put((byte)0x35)/*we'll skip this one too*/; + + final int expectTotal = buf.get(1) + buf.get(2) + buf.get(3); + sqlite3_stmt stmt = prepare(db, "INSERT INTO t(a) VALUES(?);"); + affirm( SQLITE_ERROR == sqlite3_bind_blob(stmt, 1, buf, -1, 0), + "Buffer offset may not be negative." ); + affirm( 0 == sqlite3_bind_blob(stmt, 1, buf, 1, 3) ); + affirm( SQLITE_DONE == sqlite3_step(stmt) ); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t;"); + int total = 0; + affirm( SQLITE_ROW == sqlite3_step(stmt) ); + byte blob[] = sqlite3_column_blob(stmt, 0); + java.nio.ByteBuffer nioBlob = + sqlite3_column_nio_buffer(stmt, 0); + affirm(3 == blob.length); + affirm(blob.length == nioBlob.capacity()); + affirm(blob.length == nioBlob.limit()); + int i = 0; + for(byte b : blob){ + affirm( i<=3 ); + affirm(b == buf.get(1 + i)); + affirm(b == nioBlob.get(i)); + ++i; + total += b; + } + affirm( SQLITE_DONE == sqlite3_step(stmt) ); + sqlite3_finalize(stmt); + affirm(total == expectTotal); + + SQLFunction func = + new ScalarFunction(){ + public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + sqlite3_result_blob(cx, buf, 1, 3); + } + }; + + affirm( 0 == sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func) ); + stmt = prepare(db, "SELECT myfunc()"); + affirm( SQLITE_ROW == sqlite3_step(stmt) ); + blob = sqlite3_column_blob(stmt, 0); + affirm(3 == blob.length); + i = 0; + total = 0; + for(byte b : blob){ + affirm( i<=3 ); + affirm(b == buf.get(1 + i++)); + total += b; + } + affirm( SQLITE_DONE == sqlite3_step(stmt) ); + sqlite3_finalize(stmt); + affirm(total == expectTotal); + + sqlite3_close_v2(db); + } + + private void testSql(){ + sqlite3 db = createNewDb(); + sqlite3_stmt stmt = prepare(db, "SELECT 1"); + affirm( "SELECT 1".equals(sqlite3_sql(stmt)) ); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT ?"); + sqlite3_bind_text(stmt, 1, "hell😃"); + final String expect = "SELECT 'hell😃'"; + affirm( expect.equals(sqlite3_expanded_sql(stmt)) ); + String n = sqlite3_normalized_sql(stmt); + affirm( null==n || "SELECT?;".equals(n) ); + sqlite3_finalize(stmt); + sqlite3_close(db); + } + + private void testCollation(){ + final sqlite3 db = createNewDb(); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + final ValueHolder xDestroyCalled = new ValueHolder<>(0); + final CollationCallback myCollation = new CollationCallback() { + private final String myState = + "this is local state. There is much like it, but this is mine."; + @Override + // Reverse-sorts its inputs... + public int call(byte[] lhs, byte[] rhs){ + int len = lhs.length > rhs.length ? rhs.length : lhs.length; + int c = 0, i = 0; + for(i = 0; i < len; ++i){ + c = lhs[i] - rhs[i]; + if(0 != c) break; + } + if(0==c){ + if(i < lhs.length) c = 1; + else if(i < rhs.length) c = -1; + } + return -c; + } + @Override + public void xDestroy() { + // Just demonstrates that xDestroy is called. + ++xDestroyCalled.value; + } + }; + final CollationNeededCallback collLoader = new CollationNeededCallback(){ + @Override + public void call(sqlite3 dbArg, int eTextRep, String collationName){ + affirm(dbArg == db/* as opposed to a temporary object*/); + sqlite3_create_collation(dbArg, "reversi", eTextRep, myCollation); + } + }; + int rc = sqlite3_collation_needed(db, collLoader); + affirm( 0 == rc ); + rc = sqlite3_collation_needed(db, collLoader); + affirm( 0 == rc /* Installing the same object again is a no-op */); + sqlite3_stmt stmt = prepare(db, "SELECT a FROM t ORDER BY a COLLATE reversi"); + int counter = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final String val = sqlite3_column_text16(stmt, 0); + ++counter; + //outln("REVERSI'd row#"+counter+": "+val); + switch(counter){ + case 1: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 3: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + sqlite3_finalize(stmt); + stmt = prepare(db, "SELECT a FROM t ORDER BY a"); + counter = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final String val = sqlite3_column_text16(stmt, 0); + ++counter; + //outln("Non-REVERSI'd row#"+counter+": "+val); + switch(counter){ + case 3: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 1: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + sqlite3_finalize(stmt); + affirm( 0 == xDestroyCalled.value ); + rc = sqlite3_collation_needed(db, null); + affirm( 0 == rc ); + sqlite3_close_v2(db); + affirm( 0 == db.getNativePointer() ); + affirm( 1 == xDestroyCalled.value ); + } + + @SingleThreadOnly /* because it's thread-agnostic */ + private void testToUtf8(){ + /** + https://docs.oracle.com/javase/8/docs/api/java/nio/charset/Charset.html + + Let's ensure that we can convert to standard UTF-8 in Java code + (noting that the JNI native API has no way to do this). + */ + final byte[] ba = "a \0 b".getBytes(StandardCharsets.UTF_8); + affirm( 5 == ba.length /* as opposed to 6 in modified utf-8 */); + } + + private void testStatus(){ + final OutputPointer.Int64 cur64 = new OutputPointer.Int64(); + final OutputPointer.Int64 high64 = new OutputPointer.Int64(); + final OutputPointer.Int32 cur32 = new OutputPointer.Int32(); + final OutputPointer.Int32 high32 = new OutputPointer.Int32(); + final sqlite3 db = createNewDb(); + execSql(db, "create table t(a); insert into t values(1),(2),(3)"); + + int rc = sqlite3_status(SQLITE_STATUS_MEMORY_USED, cur32, high32, false); + affirm( 0 == rc ); + affirm( cur32.value > 0 ); + affirm( high32.value >= cur32.value ); + + rc = sqlite3_status64(SQLITE_STATUS_MEMORY_USED, cur64, high64, false); + affirm( 0 == rc ); + affirm( cur64.value > 0 ); + affirm( high64.value >= cur64.value ); + + cur32.value = 0; + high32.value = 1; + rc = sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, cur32, high32, false); + affirm( 0 == rc ); + affirm( cur32.value > 0 ); + affirm( high32.value == 0 /* always 0 for SCHEMA_USED */ ); + + sqlite3_close_v2(db); + } + + private void testUdf1(){ + final sqlite3 db = createNewDb(); + // These ValueHolders are just to confirm that the func did what we want... + final ValueHolder xDestroyCalled = new ValueHolder<>(false); + final ValueHolder xFuncAccum = new ValueHolder<>(0); + final ValueHolder neverEverDoThisInClientCode = new ValueHolder<>(null); + final ValueHolder neverEverDoThisInClientCode2 = new ValueHolder<>(null); + + // Create an SQLFunction instance using one of its 3 subclasses: + // Scalar, Aggregate, or Window: + SQLFunction func = + // Each of the 3 subclasses requires a different set of + // functions, all of which must be implemented. Anonymous + // classes are a convenient way to implement these. + new ScalarFunction(){ + public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + affirm(db == sqlite3_context_db_handle(cx)); + if( null==neverEverDoThisInClientCode.value ){ + /* !!!NEVER!!! hold a reference to an sqlite3_value or + sqlite3_context object like this in client code! They + are ONLY legal for the duration of their single + call. We do it here ONLY to test that the defenses + against clients doing this are working. */ + neverEverDoThisInClientCode2.value = cx; + neverEverDoThisInClientCode.value = args; + } + int result = 0; + for( sqlite3_value v : args ) result += sqlite3_value_int(v); + xFuncAccum.value += result;// just for post-run testing + sqlite3_result_int(cx, result); + } + /* OPTIONALLY override xDestroy... */ + public void xDestroy(){ + xDestroyCalled.value = true; + } + }; + + // Register and use the function... + int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); + affirm(0 == rc); + affirm(0 == xFuncAccum.value); + final sqlite3_stmt stmt = prepare(db, "SELECT myfunc(1,2,3)"); + int n = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + affirm( 6 == sqlite3_column_int(stmt, 0) ); + ++n; + } + sqlite3_finalize(stmt); + affirm(1 == n); + affirm(6 == xFuncAccum.value); + affirm( !xDestroyCalled.value ); + affirm( null!=neverEverDoThisInClientCode.value ); + affirm( null!=neverEverDoThisInClientCode2.value ); + affirm( 0 xFuncAccum = new ValueHolder<>(0); + + SQLFunction funcAgg = new AggregateFunction(){ + @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){ + /** Throwing from here should emit loud noise on stdout or stderr + but the exception is suppressed because we have no way to inform + sqlite about it from these callbacks. */ + //throw new RuntimeException("Throwing from an xStep"); + } + @Override public void xFinal(sqlite3_context cx){ + throw new RuntimeException("Throwing from an xFinal"); + } + }; + int rc = sqlite3_create_function(db, "myagg", 1, SQLITE_UTF8, funcAgg); + affirm(0 == rc); + affirm(0 == xFuncAccum.value); + sqlite3_stmt stmt = prepare(db, "SELECT myagg(1)"); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + affirm( 0 != rc ); + affirm( sqlite3_errmsg(db).indexOf("an xFinal") > 0 ); + + SQLFunction funcSc = new ScalarFunction(){ + @Override public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + throw new RuntimeException("Throwing from an xFunc"); + } + }; + rc = sqlite3_create_function(db, "mysca", 0, SQLITE_UTF8, funcSc); + affirm(0 == rc); + affirm(0 == xFuncAccum.value); + stmt = prepare(db, "SELECT mysca()"); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + affirm( 0 != rc ); + affirm( sqlite3_errmsg(db).indexOf("an xFunc") > 0 ); + rc = sqlite3_create_function(db, "mysca", 1, -1, funcSc); + affirm( SQLITE_FORMAT==rc, "invalid encoding value." ); + sqlite3_close_v2(db); + } + + @SingleThreadOnly + private void testUdfJavaObject(){ + affirm( !mtMode ); + final sqlite3 db = createNewDb(); + final ValueHolder testResult = new ValueHolder<>(db); + final ValueHolder boundObj = new ValueHolder<>(42); + final SQLFunction func = new ScalarFunction(){ + public void xFunc(sqlite3_context cx, sqlite3_value args[]){ + sqlite3_result_java_object(cx, testResult.value); + affirm( sqlite3_value_java_object(args[0]) == boundObj ); + } + }; + int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); + affirm(0 == rc); + sqlite3_stmt stmt = prepare(db, "select myfunc(?)"); + affirm( 0 != stmt.getNativePointer() ); + affirm( testResult.value == db ); + rc = sqlite3_bind_java_object(stmt, 1, boundObj); + affirm( 0==rc ); + int n = 0; + if( SQLITE_ROW == sqlite3_step(stmt) ){ + affirm( testResult.value == sqlite3_column_java_object(stmt, 0) ); + affirm( testResult.value == sqlite3_column_java_object(stmt, 0, sqlite3.class) ); + affirm( null == sqlite3_column_java_object(stmt, 0, sqlite3_stmt.class) ); + affirm( null == sqlite3_column_java_object(stmt,1) ); + final sqlite3_value v = sqlite3_column_value(stmt, 0); + affirm( testResult.value == sqlite3_value_java_object(v) ); + affirm( testResult.value == sqlite3_value_java_object(v, sqlite3.class) ); + affirm( testResult.value == + sqlite3_value_java_object(v, testResult.value.getClass()) ); + affirm( testResult.value == sqlite3_value_java_object(v, Object.class) ); + affirm( null == sqlite3_value_java_object(v, String.class) ); + ++n; + } + sqlite3_finalize(stmt); + affirm( 1 == n ); + affirm( 0==sqlite3_db_release_memory(db) ); + sqlite3_close_v2(db); + } + + private void testUdfAggregate(){ + final sqlite3 db = createNewDb(); + final ValueHolder xFinalNull = + // To confirm that xFinal() is called with no aggregate state + // when the corresponding result set is empty. + new ValueHolder<>(false); + final ValueHolder neverEverDoThisInClientCode = new ValueHolder<>(null); + final ValueHolder neverEverDoThisInClientCode2 = new ValueHolder<>(null); + SQLFunction func = new AggregateFunction(){ + @Override + public void xStep(sqlite3_context cx, sqlite3_value[] args){ + if( null==neverEverDoThisInClientCode.value ){ + /* !!!NEVER!!! hold a reference to an sqlite3_value or + sqlite3_context object like this in client code! They + are ONLY legal for the duration of their single + call. We do it here ONLY to test that the defenses + against clients doing this are working. */ + neverEverDoThisInClientCode.value = args; + } + final ValueHolder agg = this.getAggregateState(cx, 0); + agg.value += sqlite3_value_int(args[0]); + affirm( agg == this.getAggregateState(cx, 0) ); + } + @Override + public void xFinal(sqlite3_context cx){ + if( null==neverEverDoThisInClientCode2.value ){ + neverEverDoThisInClientCode2.value = cx; + } + final Integer v = this.takeAggregateState(cx); + if(null == v){ + xFinalNull.value = true; + sqlite3_result_null(cx); + }else{ + sqlite3_result_int(cx, v); + } + } + }; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES(1),(2),(3)"); + int rc = sqlite3_create_function(db, "myfunc", 1, SQLITE_UTF8, func); + affirm(0 == rc); + sqlite3_stmt stmt = prepare(db, "select myfunc(a), myfunc(a+10) from t"); + affirm( 0==sqlite3_stmt_status(stmt, SQLITE_STMTSTATUS_RUN, false) ); + int n = 0; + if( SQLITE_ROW == sqlite3_step(stmt) ){ + int v = sqlite3_column_int(stmt, 0); + affirm( 6 == v ); + int v2 = sqlite3_column_int(stmt, 1); + affirm( 30+v == v2 ); + ++n; + } + affirm( 1==n ); + affirm(!xFinalNull.value); + affirm( null!=neverEverDoThisInClientCode.value ); + affirm( null!=neverEverDoThisInClientCode2.value ); + affirm( 0(){ + + private void xStepInverse(sqlite3_context cx, int v){ + this.getAggregateState(cx,0).value += v; + } + @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){ + this.xStepInverse(cx, sqlite3_value_int(args[0])); + } + @Override public void xInverse(sqlite3_context cx, sqlite3_value[] args){ + this.xStepInverse(cx, -sqlite3_value_int(args[0])); + } + + private void xFinalValue(sqlite3_context cx, Integer v){ + if(null == v) sqlite3_result_null(cx); + else sqlite3_result_int(cx, v); + } + @Override public void xFinal(sqlite3_context cx){ + xFinalValue(cx, this.takeAggregateState(cx)); + } + @Override public void xValue(sqlite3_context cx){ + xFinalValue(cx, this.getAggregateState(cx,null).value); + } + }; + int rc = sqlite3_create_function(db, "winsumint", 1, SQLITE_UTF8, func); + affirm( 0 == rc ); + execSql(db, new String[] { + "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", + "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" + }); + final sqlite3_stmt stmt = prepare(db, + "SELECT x, winsumint(y) OVER ("+ + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING"+ + ") AS sum_y "+ + "FROM twin ORDER BY x;"); + int n = 0; + while( SQLITE_ROW == sqlite3_step(stmt) ){ + final String s = sqlite3_column_text16(stmt, 0); + final int i = sqlite3_column_int(stmt, 1); + switch(++n){ + case 1: affirm( "a".equals(s) && 9==i ); break; + case 2: affirm( "b".equals(s) && 12==i ); break; + case 3: affirm( "c".equals(s) && 16==i ); break; + case 4: affirm( "d".equals(s) && 12==i ); break; + case 5: affirm( "e".equals(s) && 9==i ); break; + default: affirm( false /* cannot happen */ ); + } + } + sqlite3_finalize(stmt); + affirm( 5 == n ); + sqlite3_close_v2(db); + } + + private void listBoundMethods(){ + if(false){ + final java.lang.reflect.Field[] declaredFields = + CApi.class.getDeclaredFields(); + outln("Bound constants:\n"); + for(java.lang.reflect.Field field : declaredFields) { + if(java.lang.reflect.Modifier.isStatic(field.getModifiers())) { + outln("\t",field.getName()); + } + } + } + final java.lang.reflect.Method[] declaredMethods = + CApi.class.getDeclaredMethods(); + final java.util.List funcList = new java.util.ArrayList<>(); + for(java.lang.reflect.Method m : declaredMethods){ + if((m.getModifiers() & java.lang.reflect.Modifier.STATIC) != 0){ + final String name = m.getName(); + if(name.startsWith("sqlite3_")){ + funcList.add(name); + } + } + } + int count = 0; + java.util.Collections.sort(funcList); + for(String n : funcList){ + ++count; + outln("\t",n,"()"); + } + outln(count," functions named sqlite3_*."); + } + + private void testTrace(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + /* Ensure that characters outside of the UTF BMP survive the trip + from Java to sqlite3 and back to Java. (At no small efficiency + penalty.) */ + final String nonBmpChar = "😃"; + int rc = sqlite3_trace_v2( + db, SQLITE_TRACE_STMT | SQLITE_TRACE_PROFILE + | SQLITE_TRACE_ROW | SQLITE_TRACE_CLOSE, + new TraceV2Callback(){ + @Override public int call(int traceFlag, Object pNative, Object x){ + ++counter.value; + //outln("TRACE "+traceFlag+" pNative = "+pNative.getClass().getName()); + switch(traceFlag){ + case SQLITE_TRACE_STMT: + affirm(pNative instanceof sqlite3_stmt); + //outln("TRACE_STMT sql = "+x); + affirm(x instanceof String); + affirm( ((String)x).indexOf(nonBmpChar) > 0 ); + break; + case SQLITE_TRACE_PROFILE: + affirm(pNative instanceof sqlite3_stmt); + affirm(x instanceof Long); + //outln("TRACE_PROFILE time = "+x); + break; + case SQLITE_TRACE_ROW: + affirm(pNative instanceof sqlite3_stmt); + affirm(null == x); + //outln("TRACE_ROW = "+sqlite3_column_text16((sqlite3_stmt)pNative, 0)); + break; + case SQLITE_TRACE_CLOSE: + affirm(pNative instanceof sqlite3); + affirm(null == x); + break; + default: + affirm(false /*cannot happen*/); + break; + } + return 0; + } + }); + affirm( 0==rc ); + execSql(db, "SELECT coalesce(null,null,'"+nonBmpChar+"'); "+ + "SELECT 'w"+nonBmpChar+"orld'"); + affirm( 6 == counter.value ); + sqlite3_close_v2(db); + affirm( 7 == counter.value ); + } + + @SingleThreadOnly /* because threads inherently break this test */ + private static void testBusy(){ + final String dbName = "_busy-handler.db"; + try{ + final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3(); + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); + + int rc = sqlite3_open(dbName, outDb); + ++metrics.dbOpen; + affirm( 0 == rc ); + final sqlite3 db1 = outDb.get(); + execSql(db1, "CREATE TABLE IF NOT EXISTS t(a)"); + rc = sqlite3_open(dbName, outDb); + ++metrics.dbOpen; + affirm( 0 == rc ); + affirm( outDb.get() != db1 ); + final sqlite3 db2 = outDb.get(); + + affirm( "main".equals( sqlite3_db_name(db1, 0) ) ); + rc = sqlite3_db_config(db1, SQLITE_DBCONFIG_MAINDBNAME, "foo"); + affirm( sqlite3_db_filename(db1, "foo").endsWith(dbName) ); + affirm( "foo".equals( sqlite3_db_name(db1, 0) ) ); + affirm( SQLITE_MISUSE == sqlite3_db_config(db1, 0, 0, null) ); + + final ValueHolder xBusyCalled = new ValueHolder<>(0); + BusyHandlerCallback handler = new BusyHandlerCallback(){ + @Override public int call(int n){ + //outln("busy handler #"+n); + return n > 2 ? 0 : ++xBusyCalled.value; + } + }; + rc = sqlite3_busy_handler(db2, handler); + affirm(0 == rc); + + // Force a locked condition... + execSql(db1, "BEGIN EXCLUSIVE"); + rc = sqlite3_prepare_v2(db2, "SELECT * from t", outStmt); + affirm( SQLITE_BUSY == rc); + affirm( null == outStmt.get() ); + affirm( 3 == xBusyCalled.value ); + sqlite3_close_v2(db1); + sqlite3_close_v2(db2); + }finally{ + try{(new java.io.File(dbName)).delete();} + catch(Exception e){/* ignore */} + } + } + + private void testProgress(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + sqlite3_progress_handler(db, 1, new ProgressHandlerCallback(){ + @Override public int call(){ + ++counter.value; + return 0; + } + }); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( counter.value > 0 ); + int nOld = counter.value; + sqlite3_progress_handler(db, 0, null); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( nOld == counter.value ); + sqlite3_close_v2(db); + } + + private void testCommitHook(){ + final sqlite3 db = createNewDb(); + sqlite3_extended_result_codes(db, true); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder hookResult = new ValueHolder<>(0); + final CommitHookCallback theHook = new CommitHookCallback(){ + @Override public int call(){ + ++counter.value; + return hookResult.value; + } + }; + CommitHookCallback oldHook = sqlite3_commit_hook(db, theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 2 == counter.value ); + execSql(db, "BEGIN; SELECT 1; SELECT 2; COMMIT;"); + affirm( 2 == counter.value /* NOT invoked if no changes are made */ ); + execSql(db, "BEGIN; update t set a='d' where a='c'; COMMIT;"); + affirm( 3 == counter.value ); + oldHook = sqlite3_commit_hook(db, theHook); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='e' where a='d'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = sqlite3_commit_hook(db, null); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='f' where a='e'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = sqlite3_commit_hook(db, null); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='g' where a='f'; COMMIT;"); + affirm( 4 == counter.value ); + + final CommitHookCallback newHook = new CommitHookCallback(){ + @Override public int call(){return 0;} + }; + oldHook = sqlite3_commit_hook(db, newHook); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='h' where a='g'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = sqlite3_commit_hook(db, theHook); + affirm( newHook == oldHook ); + execSql(db, "BEGIN; update t set a='i' where a='h'; COMMIT;"); + affirm( 5 == counter.value ); + hookResult.value = SQLITE_ERROR; + int rc = execSql(db, false, "BEGIN; update t set a='j' where a='i'; COMMIT;"); + affirm( SQLITE_CONSTRAINT_COMMITHOOK == rc ); + affirm( 6 == counter.value ); + sqlite3_close_v2(db); + } + + private void testUpdateHook(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder expectedOp = new ValueHolder<>(0); + final UpdateHookCallback theHook = new UpdateHookCallback(){ + @Override + public void call(int opId, String dbName, String tableName, long rowId){ + ++counter.value; + if( 0!=expectedOp.value ){ + affirm( expectedOp.value == opId ); + } + } + }; + UpdateHookCallback oldHook = sqlite3_update_hook(db, theHook); + affirm( null == oldHook ); + expectedOp.value = SQLITE_INSERT; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 3 == counter.value ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='d' where a='c';"); + affirm( 4 == counter.value ); + oldHook = sqlite3_update_hook(db, theHook); + affirm( theHook == oldHook ); + expectedOp.value = SQLITE_DELETE; + execSql(db, "DELETE FROM t where a='d'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_update_hook(db, null); + affirm( theHook == oldHook ); + execSql(db, "update t set a='e' where a='b';"); + affirm( 5 == counter.value ); + oldHook = sqlite3_update_hook(db, null); + affirm( null == oldHook ); + + final UpdateHookCallback newHook = new UpdateHookCallback(){ + @Override public void call(int opId, String dbName, String tableName, long rowId){ + } + }; + oldHook = sqlite3_update_hook(db, newHook); + affirm( null == oldHook ); + execSql(db, "update t set a='h' where a='a'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_update_hook(db, theHook); + affirm( newHook == oldHook ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='i' where a='h'"); + affirm( 6 == counter.value ); + sqlite3_close_v2(db); + } + + /** + This test is functionally identical to testUpdateHook(), only with a + different callback type. + */ + private void testPreUpdateHook(){ + if( !sqlite3_compileoption_used("ENABLE_PREUPDATE_HOOK") ){ + //outln("Skipping testPreUpdateHook(): no pre-update hook support."); + return; + } + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder expectedOp = new ValueHolder<>(0); + final PreupdateHookCallback theHook = new PreupdateHookCallback(){ + @Override + public void call(sqlite3 db, int opId, String dbName, String dbTable, + long iKey1, long iKey2 ){ + ++counter.value; + switch( opId ){ + case SQLITE_UPDATE: + affirm( 0 < sqlite3_preupdate_count(db) ); + affirm( null != sqlite3_preupdate_new(db, 0) ); + affirm( null != sqlite3_preupdate_old(db, 0) ); + break; + case SQLITE_INSERT: + affirm( null != sqlite3_preupdate_new(db, 0) ); + break; + case SQLITE_DELETE: + affirm( null != sqlite3_preupdate_old(db, 0) ); + break; + default: + break; + } + if( 0!=expectedOp.value ){ + affirm( expectedOp.value == opId ); + } + } + }; + PreupdateHookCallback oldHook = sqlite3_preupdate_hook(db, theHook); + affirm( null == oldHook ); + expectedOp.value = SQLITE_INSERT; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 3 == counter.value ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='d' where a='c';"); + affirm( 4 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, theHook); + affirm( theHook == oldHook ); + expectedOp.value = SQLITE_DELETE; + execSql(db, "DELETE FROM t where a='d'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, null); + affirm( theHook == oldHook ); + execSql(db, "update t set a='e' where a='b';"); + affirm( 5 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, null); + affirm( null == oldHook ); + + final PreupdateHookCallback newHook = new PreupdateHookCallback(){ + @Override + public void call(sqlite3 db, int opId, String dbName, + String tableName, long iKey1, long iKey2){ + } + }; + oldHook = sqlite3_preupdate_hook(db, newHook); + affirm( null == oldHook ); + execSql(db, "update t set a='h' where a='a'"); + affirm( 5 == counter.value ); + oldHook = sqlite3_preupdate_hook(db, theHook); + affirm( newHook == oldHook ); + expectedOp.value = SQLITE_UPDATE; + execSql(db, "update t set a='i' where a='h'"); + affirm( 6 == counter.value ); + + sqlite3_close_v2(db); + } + + private void testRollbackHook(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final RollbackHookCallback theHook = new RollbackHookCallback(){ + @Override public void call(){ + ++counter.value; + } + }; + RollbackHookCallback oldHook = sqlite3_rollback_hook(db, theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 0 == counter.value ); + execSql(db, false, "BEGIN; SELECT 1; SELECT 2; ROLLBACK;"); + affirm( 1 == counter.value /* contra to commit hook, is invoked if no changes are made */ ); + + final RollbackHookCallback newHook = new RollbackHookCallback(){ + @Override public void call(){return;} + }; + oldHook = sqlite3_rollback_hook(db, newHook); + affirm( theHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 1 == counter.value ); + oldHook = sqlite3_rollback_hook(db, theHook); + affirm( newHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 2 == counter.value ); + int rc = execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 0 == rc ); + affirm( 3 == counter.value ); + sqlite3_close_v2(db); + } + + /** + If FTS5 is available, runs FTS5 tests, else returns with no side + effects. If it is available but loading of the FTS5 bits fails, + it throws. + */ + @SuppressWarnings("unchecked") + @SingleThreadOnly /* because the Fts5 parts are not yet known to be + thread-safe */ + private void testFts5() throws Exception { + if( !sqlite3_compileoption_used("ENABLE_FTS5") ){ + //outln("SQLITE_ENABLE_FTS5 is not set. Skipping FTS5 tests."); + return; + } + Exception err = null; + try { + Class t = Class.forName("org.sqlite.jni.fts5.TesterFts5"); + java.lang.reflect.Constructor ctor = t.getConstructor(); + ctor.setAccessible(true); + final long timeStart = System.currentTimeMillis(); + ctor.newInstance() /* will run all tests */; + final long timeEnd = System.currentTimeMillis(); + outln("FTS5 Tests done in ",(timeEnd - timeStart),"ms"); + }catch(ClassNotFoundException e){ + outln("FTS5 classes not loaded."); + err = e; + }catch(NoSuchMethodException e){ + outln("FTS5 tester ctor not found."); + err = e; + }catch(Exception e){ + outln("Instantiation of FTS5 tester threw."); + err = e; + } + if( null != err ){ + outln("Exception: "+err); + err.printStackTrace(); + throw err; + } + } + + private void testAuthorizer(){ + final sqlite3 db = createNewDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder authRc = new ValueHolder<>(0); + final AuthorizerCallback auth = new AuthorizerCallback(){ + public int call(int op, String s0, String s1, String s2, String s3){ + ++counter.value; + //outln("xAuth(): "+s0+" "+s1+" "+s2+" "+s3); + return authRc.value; + } + }; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + sqlite3_set_authorizer(db, auth); + execSql(db, "UPDATE t SET a=1"); + affirm( 1 == counter.value ); + authRc.value = SQLITE_DENY; + int rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( SQLITE_AUTH==rc ); + sqlite3_set_authorizer(db, null); + rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( 0==rc ); + // TODO: expand these tests considerably + sqlite3_close(db); + } + + @SingleThreadOnly /* because multiple threads legitimately make these + results unpredictable */ + private synchronized void testAutoExtension(){ + final ValueHolder val = new ValueHolder<>(0); + final ValueHolder toss = new ValueHolder<>(null); + final AutoExtensionCallback ax = new AutoExtensionCallback(){ + @Override public int call(sqlite3 db){ + ++val.value; + if( null!=toss.value ){ + throw new RuntimeException(toss.value); + } + return 0; + } + }; + int rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + sqlite3_close(createNewDb()); + affirm( 1==val.value ); + sqlite3_close(createNewDb()); + affirm( 2==val.value ); + sqlite3_reset_auto_extension(); + sqlite3_close(createNewDb()); + affirm( 2==val.value ); + rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + // Must not add a new entry + rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + sqlite3_close( createNewDb() ); + affirm( 3==val.value ); + + sqlite3 db = createNewDb(); + affirm( 4==val.value ); + execSql(db, "ATTACH ':memory:' as foo"); + affirm( 4==val.value, "ATTACH uses the same connection, not sub-connections." ); + sqlite3_close(db); + db = null; + + affirm( sqlite3_cancel_auto_extension(ax) ); + affirm( !sqlite3_cancel_auto_extension(ax) ); + sqlite3_close(createNewDb()); + affirm( 4==val.value ); + rc = sqlite3_auto_extension( ax ); + affirm( 0==rc ); + Exception err = null; + toss.value = "Throwing from auto_extension."; + try{ + sqlite3_close(createNewDb()); + }catch(Exception e){ + err = e; + } + affirm( err!=null ); + affirm( err.getMessage().indexOf(toss.value)>0 ); + toss.value = null; + + val.value = 0; + final AutoExtensionCallback ax2 = new AutoExtensionCallback(){ + @Override public int call(sqlite3 db){ + ++val.value; + return 0; + } + }; + rc = sqlite3_auto_extension( ax2 ); + affirm( 0 == rc ); + sqlite3_close(createNewDb()); + affirm( 2 == val.value ); + affirm( sqlite3_cancel_auto_extension(ax) ); + affirm( !sqlite3_cancel_auto_extension(ax) ); + sqlite3_close(createNewDb()); + affirm( 3 == val.value ); + rc = sqlite3_auto_extension( ax ); + affirm( 0 == rc ); + sqlite3_close(createNewDb()); + affirm( 5 == val.value ); + affirm( sqlite3_cancel_auto_extension(ax2) ); + affirm( !sqlite3_cancel_auto_extension(ax2) ); + sqlite3_close(createNewDb()); + affirm( 6 == val.value ); + rc = sqlite3_auto_extension( ax2 ); + affirm( 0 == rc ); + sqlite3_close(createNewDb()); + affirm( 8 == val.value ); + + sqlite3_reset_auto_extension(); + sqlite3_close(createNewDb()); + affirm( 8 == val.value ); + affirm( !sqlite3_cancel_auto_extension(ax) ); + affirm( !sqlite3_cancel_auto_extension(ax2) ); + sqlite3_close(createNewDb()); + affirm( 8 == val.value ); + } + + + private void testColumnMetadata(){ + final sqlite3 db = createNewDb(); + execSql(db, new String[] { + "CREATE TABLE t(a duck primary key not null collate noCase); ", + "INSERT INTO t(a) VALUES(1),(2),(3);" + }); + OutputPointer.Bool bNotNull = new OutputPointer.Bool(); + OutputPointer.Bool bPrimaryKey = new OutputPointer.Bool(); + OutputPointer.Bool bAutoinc = new OutputPointer.Bool(); + OutputPointer.String zCollSeq = new OutputPointer.String(); + OutputPointer.String zDataType = new OutputPointer.String(); + int rc = sqlite3_table_column_metadata( + db, "main", "t", "a", zDataType, zCollSeq, + bNotNull, bPrimaryKey, bAutoinc); + affirm( 0==rc ); + affirm( bPrimaryKey.value ); + affirm( !bAutoinc.value ); + affirm( bNotNull.value ); + affirm( "noCase".equals(zCollSeq.value) ); + affirm( "duck".equals(zDataType.value) ); + + TableColumnMetadata m = + sqlite3_table_column_metadata(db, "main", "t", "a"); + affirm( null != m ); + affirm( bPrimaryKey.value == m.isPrimaryKey() ); + affirm( bAutoinc.value == m.isAutoincrement() ); + affirm( bNotNull.value == m.isNotNull() ); + affirm( zCollSeq.value.equals(m.getCollation()) ); + affirm( zDataType.value.equals(m.getDataType()) ); + + affirm( null == sqlite3_table_column_metadata(db, "nope", "t", "a") ); + affirm( null == sqlite3_table_column_metadata(db, "main", "nope", "a") ); + + m = sqlite3_table_column_metadata(db, "main", "t", null) + /* Check only for existence of table */; + affirm( null != m ); + affirm( m.isPrimaryKey() ); + affirm( !m.isAutoincrement() ); + affirm( !m.isNotNull() ); + affirm( "BINARY".equalsIgnoreCase(m.getCollation()) ); + affirm( "INTEGER".equalsIgnoreCase(m.getDataType()) ); + + sqlite3_close_v2(db); + } + + private void testTxnState(){ + final sqlite3 db = createNewDb(); + affirm( SQLITE_TXN_NONE == sqlite3_txn_state(db, null) ); + affirm( sqlite3_get_autocommit(db) ); + execSql(db, "BEGIN;"); + affirm( !sqlite3_get_autocommit(db) ); + affirm( SQLITE_TXN_NONE == sqlite3_txn_state(db, null) ); + execSql(db, "SELECT * FROM sqlite_schema;"); + affirm( SQLITE_TXN_READ == sqlite3_txn_state(db, "main") ); + execSql(db, "CREATE TABLE t(a);"); + affirm( SQLITE_TXN_WRITE == sqlite3_txn_state(db, null) ); + execSql(db, "ROLLBACK;"); + affirm( SQLITE_TXN_NONE == sqlite3_txn_state(db, null) ); + sqlite3_close_v2(db); + } + + + private void testExplain(){ + final sqlite3 db = createNewDb(); + sqlite3_stmt stmt = prepare(db,"SELECT 1"); + + affirm( 0 == sqlite3_stmt_isexplain(stmt) ); + int rc = sqlite3_stmt_explain(stmt, 1); + affirm( 1 == sqlite3_stmt_isexplain(stmt) ); + rc = sqlite3_stmt_explain(stmt, 2); + affirm( 2 == sqlite3_stmt_isexplain(stmt) ); + sqlite3_finalize(stmt); + sqlite3_close_v2(db); + } + + private void testLimit(){ + final sqlite3 db = createNewDb(); + int v; + + v = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + affirm( v > 0 ); + affirm( v == sqlite3_limit(db, SQLITE_LIMIT_LENGTH, v-1) ); + affirm( v-1 == sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1) ); + sqlite3_close_v2(db); + } + + private void testComplete(){ + affirm( 0==sqlite3_complete("select 1") ); + affirm( 0!=sqlite3_complete("select 1;") ); + affirm( 0!=sqlite3_complete("nope 'nope' 'nope' 1;"), "Yup" ); + } + + private void testKeyword(){ + final int n = sqlite3_keyword_count(); + affirm( n>0 ); + affirm( !sqlite3_keyword_check("_nope_") ); + affirm( sqlite3_keyword_check("seLect") ); + affirm( null!=sqlite3_keyword_name(0) ); + affirm( null!=sqlite3_keyword_name(n-1) ); + affirm( null==sqlite3_keyword_name(n) ); + } + + private void testBackup(){ + final sqlite3 dbDest = createNewDb(); + + try (sqlite3 dbSrc = createNewDb()) { + execSql(dbSrc, new String[]{ + "pragma page_size=512; VACUUM;", + "create table t(a);", + "insert into t(a) values(1),(2),(3);" + }); + affirm( null==sqlite3_backup_init(dbSrc,"main",dbSrc,"main") ); + try (sqlite3_backup b = sqlite3_backup_init(dbDest,"main",dbSrc,"main")) { + affirm( null!=b ); + affirm( b.getNativePointer()!=0 ); + int rc; + while( SQLITE_DONE!=(rc = sqlite3_backup_step(b, 1)) ){ + affirm( 0==rc ); + } + affirm( sqlite3_backup_pagecount(b) > 0 ); + rc = sqlite3_backup_finish(b); + affirm( 0==rc ); + affirm( b.getNativePointer()==0 ); + } + } + + try (sqlite3_stmt stmt = prepare(dbDest,"SELECT sum(a) from t")) { + sqlite3_step(stmt); + affirm( sqlite3_column_int(stmt,0) == 6 ); + } + sqlite3_close_v2(dbDest); + } + + private void testRandomness(){ + byte[] foo = new byte[20]; + int i = 0; + for( byte b : foo ){ + i += b; + } + affirm( i==0 ); + sqlite3_randomness(foo); + for( byte b : foo ){ + if(b!=0) ++i; + } + affirm( i!=0, "There's a very slight chance that 0 is actually correct." ); + } + + private void testBlobOpen(){ + final sqlite3 db = createNewDb(); + + execSql(db, "CREATE TABLE T(a BLOB);" + +"INSERT INTO t(rowid,a) VALUES(1, 'def'),(2, 'XYZ');" + ); + final OutputPointer.sqlite3_blob pOut = new OutputPointer.sqlite3_blob(); + int rc = sqlite3_blob_open(db, "main", "t", "a", + sqlite3_last_insert_rowid(db), 1, pOut); + affirm( 0==rc ); + sqlite3_blob b = pOut.take(); + affirm( null!=b ); + affirm( 0!=b.getNativePointer() ); + affirm( 3==sqlite3_blob_bytes(b) ); + rc = sqlite3_blob_write( b, new byte[] {100, 101, 102 /*"DEF"*/}, 0); + affirm( 0==rc ); + rc = sqlite3_blob_close(b); + affirm( 0==rc ); + rc = sqlite3_blob_close(b); + affirm( 0!=rc ); + affirm( 0==b.getNativePointer() ); + sqlite3_stmt stmt = prepare(db,"SELECT length(a), a FROM t ORDER BY a"); + affirm( SQLITE_ROW == sqlite3_step(stmt) ); + affirm( 3 == sqlite3_column_int(stmt,0) ); + affirm( "def".equals(sqlite3_column_text16(stmt,1)) ); + sqlite3_finalize(stmt); + + b = sqlite3_blob_open(db, "main", "t", "a", + sqlite3_last_insert_rowid(db), 0); + affirm( null!=b ); + rc = sqlite3_blob_reopen(b, 2); + affirm( 0==rc ); + final byte[] tgt = new byte[3]; + rc = sqlite3_blob_read(b, tgt, 0); + affirm( 0==rc ); + affirm( 100==tgt[0] && 101==tgt[1] && 102==tgt[2], "DEF" ); + rc = sqlite3_blob_close(b); + affirm( 0==rc ); + + if( !sqlite3_jni_supports_nio() ){ + outln("WARNING: skipping tests for ByteBuffer-using sqlite3_blob APIs ", + "because this platform lacks that support."); + sqlite3_close_v2(db); + return; + } + /* Sanity checks for the java.nio.ByteBuffer-taking overloads of + sqlite3_blob_read/write(). */ + execSql(db, "UPDATE t SET a=zeroblob(10)"); + b = sqlite3_blob_open(db, "main", "t", "a", 1, 1); + affirm( null!=b ); + java.nio.ByteBuffer bb = java.nio.ByteBuffer.allocateDirect(10); + for( byte i = 0; i < 10; ++i ){ + bb.put((int)i, (byte)(48+i & 0xff)); + } + rc = sqlite3_blob_write(b, 1, bb, 1, 10); + affirm( rc==SQLITE_ERROR, "b length < (srcOffset + bb length)" ); + rc = sqlite3_blob_write(b, -1, bb); + affirm( rc==SQLITE_ERROR, "Target offset may not be negative" ); + rc = sqlite3_blob_write(b, 0, bb, -1, -1); + affirm( rc==SQLITE_ERROR, "Source offset may not be negative" ); + rc = sqlite3_blob_write(b, 1, bb, 1, 8); + affirm( rc==0 ); + // b's contents: 0 49 50 51 52 53 54 55 56 0 + // ascii: 0 '1' '2' '3' '4' '5' '6' '7' '8' 0 + byte br[] = new byte[10]; + java.nio.ByteBuffer bbr = + java.nio.ByteBuffer.allocateDirect(bb.limit()); + rc = sqlite3_blob_read( b, br, 0 ); + affirm( rc==0 ); + rc = sqlite3_blob_read( b, bbr ); + affirm( rc==0 ); + java.nio.ByteBuffer bbr2 = sqlite3_blob_read_nio_buffer(b, 0, 12); + affirm( null==bbr2, "Read size is too big"); + bbr2 = sqlite3_blob_read_nio_buffer(b, -1, 3); + affirm( null==bbr2, "Source offset is negative"); + bbr2 = sqlite3_blob_read_nio_buffer(b, 5, 6); + affirm( null==bbr2, "Read pos+size is too big"); + bbr2 = sqlite3_blob_read_nio_buffer(b, 4, 7); + affirm( null==bbr2, "Read pos+size is too big"); + bbr2 = sqlite3_blob_read_nio_buffer(b, 4, 6); + affirm( null!=bbr2 ); + java.nio.ByteBuffer bbr3 = + java.nio.ByteBuffer.allocateDirect(2 * bb.limit()); + java.nio.ByteBuffer bbr4 = + java.nio.ByteBuffer.allocateDirect(5); + rc = sqlite3_blob_read( b, bbr3 ); + affirm( rc==0 ); + rc = sqlite3_blob_read( b, bbr4 ); + affirm( rc==0 ); + affirm( sqlite3_blob_bytes(b)==bbr3.limit() ); + affirm( 5==bbr4.limit() ); + sqlite3_blob_close(b); + affirm( 0==br[0] ); + affirm( 0==br[9] ); + affirm( 0==bbr.get(0) ); + affirm( 0==bbr.get(9) ); + affirm( bbr2.limit() == 6 ); + affirm( 0==bbr3.get(0) ); + { + Exception ex = null; + try{ bbr3.get(11); } + catch(Exception e){ex = e;} + affirm( ex instanceof IndexOutOfBoundsException, + "bbr3.limit() was reset by read()" ); + ex = null; + } + affirm( 0==bbr4.get(0) ); + for( int i = 1; i < 9; ++i ){ + affirm( br[i] == 48 + i ); + affirm( br[i] == bbr.get(i) ); + affirm( br[i] == bbr3.get(i) ); + if( i>3 ){ + affirm( br[i] == bbr2.get(i-4) ); + } + if( i < bbr4.limit() ){ + affirm( br[i] == bbr4.get(i) ); + } + } + sqlite3_close_v2(db); + } + + private void testPrepareMulti(){ + final sqlite3 db = createNewDb(); + final String[] sql = { + "create table t(","a)", + "; insert into t(a) values(1),(2),(3);", + "select a from t;" + }; + final List liStmt = new ArrayList<>(); + final PrepareMultiCallback proxy = new PrepareMultiCallback.StepAll(); + final ValueHolder toss = new ValueHolder<>(null); + PrepareMultiCallback m = new PrepareMultiCallback() { + @Override public int call(sqlite3_stmt st){ + liStmt.add(st); + if( null!=toss.value ){ + throw new RuntimeException(toss.value); + } + return proxy.call(st); + } + }; + int rc = sqlite3_prepare_multi(db, sql, m); + affirm( 0==rc ); + affirm( liStmt.size() == 3 ); + for( sqlite3_stmt st : liStmt ){ + sqlite3_finalize(st); + } + toss.value = "This is an exception."; + rc = sqlite3_prepare_multi(db, "SELECT 1", m); + affirm( SQLITE_ERROR==rc ); + affirm( sqlite3_errmsg(db).indexOf(toss.value)>0 ); + sqlite3_close_v2(db); + } + + private void testSetErrmsg(){ + final sqlite3 db = createNewDb(); + + int rc = sqlite3_set_errmsg(db, SQLITE_RANGE, "nope"); + affirm( 0==rc ); + affirm( SQLITE_MISUSE == sqlite3_set_errmsg(null, 0, null) ); + affirm( "nope".equals(sqlite3_errmsg(db)) ); + affirm( SQLITE_RANGE == sqlite3_errcode(db) ); + rc = sqlite3_set_errmsg(db, 0, null); + affirm( "not an error".equals(sqlite3_errmsg(db)) ); + affirm( 0 == sqlite3_errcode(db) ); + sqlite3_close_v2(db); + } + + /* Copy/paste/rename this to add new tests. */ + private void _testTemplate(){ + final sqlite3 db = createNewDb(); + sqlite3_stmt stmt = prepare(db,"SELECT 1"); + sqlite3_finalize(stmt); + sqlite3_close_v2(db); + } + + + @ManualTest /* we really only want to run this test manually */ + private void testSleep(){ + out("Sleeping briefly... "); + sqlite3_sleep(600); + outln("Woke up."); + } + + private void nap() throws InterruptedException { + if( takeNaps ){ + Thread.sleep(java.util.concurrent.ThreadLocalRandom.current().nextInt(3, 17), 0); + } + } + + @ManualTest /* because we only want to run this test on demand */ + private void testFail(){ + affirm( false, "Intentional failure." ); + } + + private void runTests(boolean fromThread) throws Exception { + if(false) showCompileOption(); + List mlist = testMethods; + affirm( null!=mlist ); + if( shuffle ){ + mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) ); + java.util.Collections.shuffle(mlist); + } + if( (!fromThread && listRunTests>0) || listRunTests>1 ){ + synchronized(this.getClass()){ + if( !fromThread ){ + out("Initial test"," list: "); + for(java.lang.reflect.Method m : testMethods){ + out(m.getName()+" "); + } + outln(); + outln("(That list excludes some which are hard-coded to run.)"); + } + out("Running"," tests: "); + for(java.lang.reflect.Method m : mlist){ + out(m.getName()+" "); + } + outln(); + } + } + for(java.lang.reflect.Method m : mlist){ + nap(); + try{ + m.invoke(this); + }catch(java.lang.reflect.InvocationTargetException e){ + outln("FAILURE: ",m.getName(),"(): ", e.getCause()); + throw e; + } + } + synchronized( this.getClass() ){ + ++nTestRuns; + } + } + + public void run() { + try { + runTests(0!=this.tId); + }catch(Exception e){ + synchronized( listErrors ){ + listErrors.add(e); + } + }finally{ + affirm( sqlite3_java_uncache_thread() ); + affirm( !sqlite3_java_uncache_thread() ); + } + } + + /** + Runs the basic sqlite3 JNI binding sanity-check suite. + + CLI flags: + + -q|-quiet: disables most test output. + + -t|-thread N: runs the tests in N threads + concurrently. Default=1. + + -r|-repeat N: repeats the tests in a loop N times, each one + consisting of the -thread value's threads. + + -shuffle: randomizes the order of most of the test functions. + + -naps: sleep small random intervals between tests in order to add + some chaos for cross-thread contention. + + + -list-tests: outputs the list of tests being run, minus some + which are hard-coded. In multi-threaded mode, use this twice to + to emit the list run by each thread (which may differ from the initial + list, in particular if -shuffle is used). + + -fail: forces an exception to be thrown during the test run. Use + with -shuffle to make its appearance unpredictable. + + -v: emit some developer-mode info at the end. + */ + public static void main(String[] args) throws Exception { + int nThread = 1; + boolean doSomethingForDev = false; + int nRepeat = 1; + boolean forceFail = false; + boolean sqlLog = false; + boolean configLog = false; + boolean squelchTestOutput = false; + for( int i = 0; i < args.length; ){ + String arg = args[i++]; + if(arg.startsWith("-")){ + arg = arg.replaceFirst("-+",""); + if(arg.equals("v")){ + doSomethingForDev = true; + //listBoundMethods(); + }else if(arg.equals("t") || arg.equals("thread")){ + nThread = Integer.parseInt(args[i++]); + }else if(arg.equals("r") || arg.equals("repeat")){ + nRepeat = Integer.parseInt(args[i++]); + }else if(arg.equals("shuffle")){ + shuffle = true; + }else if(arg.equals("list-tests")){ + ++listRunTests; + }else if(arg.equals("fail")){ + forceFail = true; + }else if(arg.equals("sqllog")){ + sqlLog = true; + }else if(arg.equals("configlog")){ + configLog = true; + }else if(arg.equals("naps")){ + takeNaps = true; + }else if(arg.equals("q") || arg.equals("quiet")){ + squelchTestOutput = true; + }else{ + throw new IllegalArgumentException("Unhandled flag:"+arg); + } + } + } + + if( sqlLog ){ + if( sqlite3_compileoption_used("ENABLE_SQLLOG") ){ + final ConfigSqlLogCallback log = new ConfigSqlLogCallback() { + @Override public void call(sqlite3 db, String msg, int op){ + switch(op){ + case 0: outln("Opening db: ",db); break; + case 1: outln("SQL ",db,": ",msg); break; + case 2: outln("Closing db: ",db); break; + } + } + }; + int rc = sqlite3_config( log ); + affirm( 0==rc ); + rc = sqlite3_config( (ConfigSqlLogCallback)null ); + affirm( 0==rc ); + rc = sqlite3_config( log ); + affirm( 0==rc ); + }else{ + outln("WARNING: -sqllog is not active because library was built ", + "without SQLITE_ENABLE_SQLLOG."); + } + } + if( configLog ){ + final ConfigLogCallback log = new ConfigLogCallback() { + @Override public void call(int code, String msg){ + outln("ConfigLogCallback: ",ResultCode.getEntryForInt(code),": ", msg); + } + }; + int rc = sqlite3_config( log ); + affirm( 0==rc ); + rc = sqlite3_config( (ConfigLogCallback)null ); + affirm( 0==rc ); + rc = sqlite3_config( log ); + affirm( 0==rc ); + } + + quietMode = squelchTestOutput; + outln("If you just saw warning messages regarding CallStaticObjectMethod, ", + "you are very likely seeing the side effects of a known openjdk8 ", + "bug. It is unsightly but does not affect the library."); + + { + // Build list of tests to run from the methods named test*(). + testMethods = new ArrayList<>(); + int nSkipped = 0; + for(final java.lang.reflect.Method m : Tester1.class.getDeclaredMethods()){ + final String name = m.getName(); + if( name.equals("testFail") ){ + if( forceFail ){ + testMethods.add(m); + } + }else if( m.isAnnotationPresent( RequiresJniNio.class ) + && !sqlite3_jni_supports_nio() ){ + outln("Skipping test for lack of JNI java.nio.ByteBuffer support: ", + name,"()\n"); + ++nSkipped; + }else if( !m.isAnnotationPresent( ManualTest.class ) ){ + if( nThread>1 && m.isAnnotationPresent( SingleThreadOnly.class ) ){ + out("Skipping test in multi-thread mode: ",name,"()\n"); + ++nSkipped; + }else if( name.startsWith("test") ){ + testMethods.add(m); + } + } + } + } + + final long timeStart = System.currentTimeMillis(); + int nLoop = 0; + switch( sqlite3_threadsafe() ){ /* Sanity checking */ + case 0: + affirm( SQLITE_ERROR==sqlite3_config( SQLITE_CONFIG_SINGLETHREAD ), + "Could not switch to single-thread mode." ); + affirm( SQLITE_ERROR==sqlite3_config( SQLITE_CONFIG_MULTITHREAD ), + "Could switch to multithread mode." ); + affirm( SQLITE_ERROR==sqlite3_config( SQLITE_CONFIG_SERIALIZED ), + "Could not switch to serialized threading mode." ); + outln("This is a single-threaded build. Not using threads."); + nThread = 1; + break; + case 1: + case 2: + affirm( 0==sqlite3_config( SQLITE_CONFIG_SINGLETHREAD ), + "Could not switch to single-thread mode." ); + affirm( 0==sqlite3_config( SQLITE_CONFIG_MULTITHREAD ), + "Could not switch to multithread mode." ); + affirm( 0==sqlite3_config( SQLITE_CONFIG_SERIALIZED ), + "Could not switch to serialized threading mode." ); + break; + default: + affirm( false, "Unhandled SQLITE_THREADSAFE value." ); + } + outln("libversion_number: ", + sqlite3_libversion_number(),"\n", + sqlite3_libversion(),"\n",SQLITE_SOURCE_ID,"\n", + "SQLITE_THREADSAFE=",sqlite3_threadsafe()); + outln("JVM NIO support? ",sqlite3_jni_supports_nio() ? "YES" : "NO"); + final boolean showLoopCount = (nRepeat>1 && nThread>1); + if( showLoopCount ){ + outln("Running ",nRepeat," loop(s) with ",nThread," thread(s) each."); + } + if( takeNaps ) outln("Napping between tests is enabled."); + for( int n = 0; n < nRepeat; ++n ){ + ++nLoop; + if( showLoopCount ) out((1==nLoop ? "" : " ")+nLoop); + if( nThread<=1 ){ + new Tester1(0).runTests(false); + continue; + } + Tester1.mtMode = true; + final ExecutorService ex = Executors.newFixedThreadPool( nThread ); + for( int i = 0; i < nThread; ++i ){ + ex.submit( new Tester1(i), i ); + } + ex.shutdown(); + try{ + ex.awaitTermination(nThread*200, java.util.concurrent.TimeUnit.MILLISECONDS); + ex.shutdownNow(); + }catch (InterruptedException ie){ + ex.shutdownNow(); + Thread.currentThread().interrupt(); + } + if( !listErrors.isEmpty() ){ + quietMode = false; + outln("TEST ERRORS:"); + Exception err = null; + for( Exception e : listErrors ){ + e.printStackTrace(); + if( null==err ) err = e; + } + if( null!=err ) throw err; + } + } + if( showLoopCount ) outln(); + quietMode = false; + + final long timeEnd = System.currentTimeMillis(); + outln("Tests done. Metrics across ",nTestRuns," total iteration(s):"); + outln("\tAssertions checked: ",affirmCount); + outln("\tDatabases opened: ",metrics.dbOpen); + if( doSomethingForDev ){ + sqlite3_jni_internal_details(); + } + affirm( 0==sqlite3_release_memory(1) ); + sqlite3_shutdown(); + int nMethods = 0; + int nNatives = 0; + final java.lang.reflect.Method[] declaredMethods = + CApi.class.getDeclaredMethods(); + for(java.lang.reflect.Method m : declaredMethods){ + final int mod = m.getModifiers(); + if( 0!=(mod & java.lang.reflect.Modifier.STATIC) ){ + final String name = m.getName(); + if(name.startsWith("sqlite3_")){ + ++nMethods; + if( 0!=(mod & java.lang.reflect.Modifier.NATIVE) ){ + ++nNatives; + } + } + } + } + outln("\tCApi.sqlite3_*() methods: "+ + nMethods+" total, with "+ + nNatives+" native, "+ + (nMethods - nNatives)+" Java" + ); + outln("\tTotal test time = " + +(timeEnd - timeStart)+"ms"); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java b/ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java new file mode 100644 index 0000000000..56465a2c0a --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java @@ -0,0 +1,50 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; +import org.sqlite.jni.annotation.Nullable; + +/** + Callback for use with {@link CApi#sqlite3_trace_v2}. +*/ +public interface TraceV2Callback extends CallbackProxy { + /** + Called by sqlite3 for various tracing operations, as per + sqlite3_trace_v2(). Note that this interface elides the 2nd + argument to the native trace callback, as that role is better + filled by instance-local state. + +

    These callbacks may throw, in which case their exceptions are + converted to C-level error information. + +

    The 2nd argument to this function, if non-null, will be a an + sqlite3 or sqlite3_stmt object, depending on the first argument + (see below). + +

    The final argument to this function is the "X" argument + documented for sqlite3_trace() and sqlite3_trace_v2(). Its type + depends on value of the first argument: + +

    - SQLITE_TRACE_STMT: pNative is a sqlite3_stmt. pX is a String + containing the prepared SQL. + +

    - SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long + holding an approximate number of nanoseconds the statement took + to run. + +

    - SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null. + +

    - SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null. + */ + int call(int traceFlag, Object pNative, @Nullable Object pX); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java b/ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java new file mode 100644 index 0000000000..e3d491f67e --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java @@ -0,0 +1,26 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for use with {@link CApi#sqlite3_update_hook}. +*/ +public interface UpdateHookCallback extends CallbackProxy { + /** + Must function as described for the C-level sqlite3_update_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + void call(int opId, String dbName, String tableName, long rowId); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/ValueHolder.java b/ext/jni/src/org/sqlite/jni/capi/ValueHolder.java new file mode 100644 index 0000000000..0a469fea9a --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/ValueHolder.java @@ -0,0 +1,27 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the ValueHolder utility class for the sqlite3 +** JNI bindings. +*/ +package org.sqlite.jni.capi; + +/** + A helper class which simply holds a single value. Its primary use + is for communicating values out of anonymous classes, as doing so + requires a "final" reference, as well as communicating aggregate + SQL function state across calls to such functions. +*/ +public class ValueHolder { + public T value; + public ValueHolder(){} + public ValueHolder(T v){value = v;} +} diff --git a/ext/jni/src/org/sqlite/jni/capi/WindowFunction.java b/ext/jni/src/org/sqlite/jni/capi/WindowFunction.java new file mode 100644 index 0000000000..eaf1bb9a35 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/WindowFunction.java @@ -0,0 +1,39 @@ +/* +** 2023-08-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + + +/** + A SQLFunction implementation for window functions. Note that + WindowFunction inherits from {@link AggregateFunction} and each + instance is required to implement the inherited abstract methods + from that class. See {@link AggregateFunction} for information on + managing the UDF's invocation-specific state. +*/ +public abstract class WindowFunction extends AggregateFunction { + + /** + As for the xInverse() argument of the C API's + sqlite3_create_window_function(). If this function throws, the + exception is not propagated and a warning might be emitted + to a debugging channel. + */ + public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args); + + /** + As for the xValue() argument of the C API's sqlite3_create_window_function(). + See xInverse() for the fate of any exceptions this throws. + */ + public abstract void xValue(sqlite3_context cx); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java b/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java new file mode 100644 index 0000000000..ce6c6a6abf --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java @@ -0,0 +1,37 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file declares JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + Callback for a hook called by SQLite when certain client-provided + state are destroyed. It gets its name from the pervasive use of + the symbol name xDestroy() for this purpose in the C API + documentation. +*/ +public interface XDestroyCallback { + /** + Must perform any cleanup required by this object. Must not + throw. Must not call back into the sqlite3 API, else it might + invoke a deadlock. + + WARNING: as a rule, it is never safe to register individual + instances with this interface multiple times in the + library. e.g., do not register the same CollationCallback with + multiple arities or names using sqlite3_create_collation(). If + this rule is violated, the library will eventually try to free + each individual reference, leading to memory corruption or a + crash via duplicate free(). + */ + void xDestroy(); +} diff --git a/ext/jni/src/org/sqlite/jni/capi/package-info.java b/ext/jni/src/org/sqlite/jni/capi/package-info.java new file mode 100644 index 0000000000..127f380675 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/package-info.java @@ -0,0 +1,89 @@ +/** + This package houses a JNI binding to the SQLite3 C API. + +

    The primary interfaces are in {@link + org.sqlite.jni.capi.CApi}.

    + +

    API Goals and Requirements

    + +
      + +
    • A 1-to-1(-ish) mapping of the C API to Java via JNI, insofar + as cross-language semantics allow for. A closely-related goal is + that the C + documentation should be usable as-is, insofar as possible, + for most of the JNI binding. As a rule, undocumented symbols in + the Java interface behave as documented for their C API + counterpart. Only semantic differences and Java-specific features + are documented here.
      • + +
    • Support Java as far back as version 8 (2014).
      • + +
    • Environment-independent. Should work everywhere both Java and + SQLite3 do.
      • + +
    • No 3rd-party dependencies beyond the JDK. That includes no + build-level dependencies for specific IDEs and toolchains. We + welcome the addition of build files for arbitrary environments + insofar as they neither interfere with each other nor become a + maintenance burden for the sqlite developers.
      • + +
    + +

    Non-Goals

    + +
      + +
    • Creation of high-level OO wrapper APIs. Clients are free to + create them off of the C-style API.
      • + +
    • Support for mixed-mode operation, where client code accesses + SQLite both via the Java-side API and the C API via their own + native code. In such cases, proxy functionalities (primarily + callback handler wrappers of all sorts) may fail because the + C-side use of the SQLite APIs will bypass those proxies.
      • + +
    + +

    State of this API

    + +

    As of version 3.43, this software is in "tech preview" form. We + tentatively plan to stamp it as stable with the 3.44 release.

    + +

    Threading Considerations

    + +

    This API is, if built with SQLITE_THREADSAFE set to 1 or 2, + thread-safe, insofar as the C API guarantees, with some addenda:

    + +
      + +
    • It is not legal to use Java-facing SQLite3 resource handles + (sqlite3, sqlite3_stmt, etc) from multiple threads concurrently, + nor to use any database-specific resources concurrently in a + thread separate from the one the database is currently in use + in. i.e. do not use a sqlite3_stmt in thread #2 when thread #1 is + using the database which prepared that handle. + +
      Violating this will eventually corrupt the JNI-level bindings + between Java's and C's view of the database. This is a limitation + of the JNI bindings, not the lower-level library. +
      • + +
    • It is legal to use a given handle, and database-specific + resources, across threads, so long as no two threads pass + resources owned by the same database into the library + concurrently. +
      • + +
    + +

    Any number of threads may, of course, create and use any number + of database handles they wish. Care only needs to be taken when + those handles or their associated resources cross threads, or...

    + +

    When built with SQLITE_THREADSAFE=0 then no threading guarantees + are provided and multi-threaded use of the library will provoke + undefined behavior.

    + +*/ +package org.sqlite.jni.capi; diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3.java new file mode 100644 index 0000000000..cc6f2e6e8d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3.java @@ -0,0 +1,43 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for communicating C-level (sqlite3*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java + and C via JNI. +*/ +public final class sqlite3 extends NativePointerHolder + implements AutoCloseable { + + // Only invoked from JNI + private sqlite3(){} + + public String toString(){ + final long ptr = getNativePointer(); + if( 0==ptr ){ + return sqlite3.class.getSimpleName()+"@null"; + } + final String fn = CApi.sqlite3_db_filename(this, "main"); + return sqlite3.class.getSimpleName() + +"@"+String.format("0x%08x",ptr) + +"["+((null == fn) ? "" : fn)+"]" + ; + } + + @Override public void close(){ + CApi.sqlite3_close_v2(this); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java new file mode 100644 index 0000000000..0ef75c17eb --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java @@ -0,0 +1,31 @@ +/* +** 2023-09-03 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for passing C-level (sqlite3_backup*) instances around in + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class sqlite3_backup extends NativePointerHolder + implements AutoCloseable { + // Only invoked from JNI. + private sqlite3_backup(){} + + @Override public void close(){ + CApi.sqlite3_backup_finish(this); + } + +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java new file mode 100644 index 0000000000..bdc0200af4 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java @@ -0,0 +1,30 @@ +/* +** 2023-09-03 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for passing C-level (sqlite3_blob*) instances around in + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class sqlite3_blob extends NativePointerHolder + implements AutoCloseable { + // Only invoked from JNI. + private sqlite3_blob(){} + + @Override public void close(){ + CApi.sqlite3_blob_close(this); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java new file mode 100644 index 0000000000..82ec49af16 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java @@ -0,0 +1,79 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + sqlite3_context instances are used in conjunction with user-defined + SQL functions (a.k.a. UDFs). +*/ +public final class sqlite3_context extends NativePointerHolder { + private Long aggregateContext = null; + + /** + getAggregateContext() corresponds to C's + sqlite3_aggregate_context(), with a slightly different interface + to account for cross-language differences. It serves the same + purposes in a slightly different way: it provides a key which is + stable across invocations of a UDF's callbacks, such that all + calls into those callbacks can determine which "set" of those + calls they belong to. + +

    Note that use of this method is not a requirement for proper use + of this class. sqlite3_aggregate_context() can also be used. + +

    If the argument is true and the aggregate context has not yet + been set up, it will be initialized and fetched on demand, else it + won't. The intent is that xStep(), xValue(), and xInverse() + methods pass true and xFinal() methods pass false. + +

    This function treats numeric 0 as null, always returning null instead + of 0. + +

    If this object is being used in the context of an aggregate or + window UDF, this function returns a non-0 value which is distinct + for each set of UDF callbacks from a single invocation of the + UDF, otherwise it returns 0. The returned value is only only + valid within the context of execution of a single SQL statement, + and must not be re-used by future invocations of the UDF in + different SQL statements. + +

    Consider this SQL, where MYFUNC is a user-defined aggregate function: + + 

    {@code
+     SELECT MYFUNC(A), MYFUNC(B) FROM T;
+     }
    + +

    The xStep() and xFinal() methods of the callback need to be able + to differentiate between those two invocations in order to + perform their work properly. The value returned by + getAggregateContext() will be distinct for each of those + invocations of MYFUNC() and is intended to be used as a lookup + key for mapping callback invocations to whatever client-defined + state is needed by the UDF. + +

    There is one case where this will return null in the context + of an aggregate or window function: if the result set has no + rows, the UDF's xFinal() will be called without any other x...() + members having been called. In that one case, no aggregate + context key will have been generated. xFinal() implementations + need to be prepared to accept that condition as legal. + */ + public synchronized Long getAggregateContext(boolean initIfNeeded){ + if( aggregateContext==null ){ + aggregateContext = CApi.sqlite3_aggregate_context(this, initIfNeeded); + if( !initIfNeeded && null==aggregateContext ) aggregateContext = 0L; + } + return (null==aggregateContext || 0!=aggregateContext) ? aggregateContext : null; + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java new file mode 100644 index 0000000000..564891c727 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java @@ -0,0 +1,30 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +/** + A wrapper for communicating C-level (sqlite3_stmt*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class sqlite3_stmt extends NativePointerHolder + implements AutoCloseable { + // Only invoked from JNI. + private sqlite3_stmt(){} + + @Override public void close(){ + CApi.sqlite3_finalize(this); + } +} diff --git a/ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java b/ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java new file mode 100644 index 0000000000..a4772f0f63 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java @@ -0,0 +1,19 @@ +/* +** 2023-07-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.capi; + +public final class sqlite3_value extends NativePointerHolder { + //! Invoked only from JNI. + private sqlite3_value(){} +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5.java new file mode 100644 index 0000000000..0dceeafd2e --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5.java @@ -0,0 +1,32 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; + +/** + INCOMPLETE AND COMPLETELY UNTESTED. + + A utility object for holding FTS5-specific types and constants + which are used by multiple FTS5 classes. +*/ +public final class Fts5 { + /* Not used */ + private Fts5(){} + + + public static final int FTS5_TOKENIZE_QUERY = 0x0001; + public static final int FTS5_TOKENIZE_PREFIX = 0x0002; + public static final int FTS5_TOKENIZE_DOCUMENT = 0x0004; + public static final int FTS5_TOKENIZE_AUX = 0x0008; + public static final int FTS5_TOKEN_COLOCATED = 0x0001; +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java new file mode 100644 index 0000000000..439b477910 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java @@ -0,0 +1,24 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.*; + +/** + A wrapper for communicating C-level (Fts5Context*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class Fts5Context extends NativePointerHolder { +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java new file mode 100644 index 0000000000..f409f4961d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java @@ -0,0 +1,96 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.*; +import org.sqlite.jni.annotation.*; + +/** +*/ +public final class Fts5ExtensionApi extends NativePointerHolder { + //! Only called from JNI + private Fts5ExtensionApi(){} + private final int iVersion = 2; + + /* Callback type for used by xQueryPhrase(). */ + public interface XQueryPhraseCallback { + int call(Fts5ExtensionApi fapi, Fts5Context cx); + } + + /** + Returns the singleton instance of this class. + */ + public static native Fts5ExtensionApi getInstance(); + + public native int xColumnCount(@NotNull Fts5Context fcx); + + public native int xColumnSize(@NotNull Fts5Context cx, int iCol, + @NotNull OutputPointer.Int32 pnToken); + + public native int xColumnText(@NotNull Fts5Context cx, int iCol, + @NotNull OutputPointer.String txt); + + public native int xColumnTotalSize(@NotNull Fts5Context fcx, int iCol, + @NotNull OutputPointer.Int64 pnToken); + + public native Object xGetAuxdata(@NotNull Fts5Context cx, boolean clearIt); + + public native int xInst(@NotNull Fts5Context cx, int iIdx, + @NotNull OutputPointer.Int32 piPhrase, + @NotNull OutputPointer.Int32 piCol, + @NotNull OutputPointer.Int32 piOff); + + public native int xInstCount(@NotNull Fts5Context fcx, + @NotNull OutputPointer.Int32 pnInst); + + public native int xPhraseCount(@NotNull Fts5Context fcx); + + public native int xPhraseFirst(@NotNull Fts5Context cx, int iPhrase, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol, + @NotNull OutputPointer.Int32 iOff); + + public native int xPhraseFirstColumn(@NotNull Fts5Context cx, int iPhrase, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol); + public native void xPhraseNext(@NotNull Fts5Context cx, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol, + @NotNull OutputPointer.Int32 iOff); + public native void xPhraseNextColumn(@NotNull Fts5Context cx, + @NotNull Fts5PhraseIter iter, + @NotNull OutputPointer.Int32 iCol); + public native int xPhraseSize(@NotNull Fts5Context fcx, int iPhrase); + + public native int xQueryPhrase(@NotNull Fts5Context cx, int iPhrase, + @NotNull XQueryPhraseCallback callback); + public native int xRowCount(@NotNull Fts5Context fcx, + @NotNull OutputPointer.Int64 nRow); + + public native long xRowid(@NotNull Fts5Context cx); + /* Note that the JNI binding lacks the C version's xDelete() + callback argument. Instead, if pAux has an xDestroy() method, it + is called if the FTS5 API finalizes the aux state (including if + allocation of storage for the auxdata fails). Any reference to + pAux held by the JNI layer will be relinquished regardless of + whether pAux has an xDestroy() method. */ + + public native int xSetAuxdata(@NotNull Fts5Context cx, @Nullable Object pAux); + + public native int xTokenize(@NotNull Fts5Context cx, @NotNull byte[] pText, + @NotNull XTokenizeCallback callback); + + public native Object xUserData(Fts5Context cx); + //^^^ returns the pointer passed as the 3rd arg to the C-level + // fts5_api::xCreateFunction(). +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java new file mode 100644 index 0000000000..5774eb5936 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java @@ -0,0 +1,25 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.NativePointerHolder; + +/** + A wrapper for C-level Fts5PhraseIter. They are only modified and + inspected by native-level code. +*/ +public final class Fts5PhraseIter extends NativePointerHolder { + //! Updated and used only by native code. + private long a; + private long b; +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java new file mode 100644 index 0000000000..b72e5d0fc0 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java @@ -0,0 +1,31 @@ +/* +** 2023-08-05x +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.NativePointerHolder; + +/** + INCOMPLETE AND COMPLETELY UNTESTED. + + A wrapper for communicating C-level (Fts5Tokenizer*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. + + At the C level, the Fts5Tokenizer type is essentially a void + pointer used specifically for tokenizers. +*/ +public final class Fts5Tokenizer extends NativePointerHolder { + //! Only called from JNI. + private Fts5Tokenizer(){} +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java new file mode 100644 index 0000000000..4d97ced47d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java @@ -0,0 +1,841 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a set of tests for the sqlite3 JNI bindings. +*/ +package org.sqlite.jni.fts5; +import java.util.*; +import static org.sqlite.jni.capi.CApi.*; +import static org.sqlite.jni.capi.Tester1.*; +import org.sqlite.jni.capi.*; +import java.nio.charset.StandardCharsets; + +public class TesterFts5 { + + private static void test1(){ + final Fts5ExtensionApi fea = Fts5ExtensionApi.getInstance(); + affirm( null != fea ); + affirm( fea.getNativePointer() != 0 ); + affirm( fea == Fts5ExtensionApi.getInstance() )/*singleton*/; + + sqlite3 db = createNewDb(); + fts5_api fApi = fts5_api.getInstanceForDb(db); + affirm( fApi != null ); + affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ ); + + execSql(db, new String[] { + "CREATE VIRTUAL TABLE ft USING fts5(a, b);", + "INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');", + "INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');" + }); + + final String pUserData = "This is pUserData"; + final int outputs[] = {0, 0}; + final fts5_extension_function func = new fts5_extension_function(){ + @Override public void call(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]){ + final int nCols = ext.xColumnCount(fCx); + affirm( 2 == nCols ); + affirm( nCols == argv.length ); + affirm( ext.xUserData(fCx) == pUserData ); + final OutputPointer.String op = new OutputPointer.String(); + final OutputPointer.Int32 colsz = new OutputPointer.Int32(); + final OutputPointer.Int64 colTotalSz = new OutputPointer.Int64(); + for(int i = 0; i < nCols; ++i ){ + int rc = ext.xColumnText(fCx, i, op); + affirm( 0 == rc ); + final String val = op.value; + affirm( val.equals(sqlite3_value_text16(argv[i])) ); + rc = ext.xColumnSize(fCx, i, colsz); + affirm( 0==rc ); + affirm( 3==sqlite3_value_bytes(argv[i]) ); + rc = ext.xColumnTotalSize(fCx, i, colTotalSz); + affirm( 0==rc ); + } + ++outputs[0]; + } + public void xDestroy(){ + outputs[1] = 1; + } + }; + + int rc = fApi.xCreateFunction("myaux", pUserData, func); + affirm( 0==rc ); + + affirm( 0==outputs[0] ); + execSql(db, "select myaux(ft,a,b) from ft;"); + affirm( 2==outputs[0] ); + affirm( 0==outputs[1] ); + sqlite3_close_v2(db); + affirm( 1==outputs[1] ); + } + + /* + ** Argument sql is a string containing one or more SQL statements + ** separated by ";" characters. This function executes each of these + ** statements against the database passed as the first argument. If + ** no error occurs, the results of the SQL script are returned as + ** an array of strings. If an error does occur, a RuntimeException is + ** thrown. + */ + private static String[] sqlite3_exec(sqlite3 db, String sql) { + List aOut = new ArrayList<>(); + + /* Iterate through the list of SQL statements. For each, step through + ** it and add any results to the aOut[] array. */ + int rc = sqlite3_prepare_multi(db, sql, new PrepareMultiCallback() { + @Override public int call(sqlite3_stmt pStmt){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int ii; + for(ii=0; ii, ); + */ + class fts5_aux implements fts5_extension_function { + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length>1 ){ + throw new RuntimeException("fts5_aux: wrong number of args"); + } + + boolean bClear = (argv.length==1); + Object obj = ext.xGetAuxdata(fCx, bClear); + if( obj instanceof String ){ + sqlite3_result_text16(pCx, (String)obj); + } + + if( argv.length==1 ){ + String val = sqlite3_value_text16(argv[0]); + if( !val.isEmpty() ){ + ext.xSetAuxdata(fCx, val); + } + } + } + public void xDestroy(){ } + } + + /* + ** fts5_inst(); + ** + ** This is used to test the xInstCount() and xInst() APIs. It returns a + ** text value containing a Tcl list with xInstCount() elements. Each + ** element is itself a list of 3 integers - the phrase number, column + ** number and token offset returned by each call to xInst(). + */ + fts5_extension_function fts5_inst = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_inst: wrong number of args"); + } + + OutputPointer.Int32 pnInst = new OutputPointer.Int32(); + OutputPointer.Int32 piPhrase = new OutputPointer.Int32(); + OutputPointer.Int32 piCol = new OutputPointer.Int32(); + OutputPointer.Int32 piOff = new OutputPointer.Int32(); + String ret = ""; + + int rc = ext.xInstCount(fCx, pnInst); + int nInst = pnInst.get(); + int ii; + + for(ii=0; rc==SQLITE_OK && ii0 ) ret += " "; + ret += "{"+piPhrase.get()+" "+piCol.get()+" "+piOff.get()+"}"; + } + + sqlite3_result_text(pCx, ret); + } + public void xDestroy(){ } + }; + + /* + ** fts5_pinst(); + ** + ** Like SQL function fts5_inst(), except using the following + ** + ** xPhraseCount + ** xPhraseFirst + ** xPhraseNext + */ + fts5_extension_function fts5_pinst = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_pinst: wrong number of args"); + } + + OutputPointer.Int32 piCol = new OutputPointer.Int32(); + OutputPointer.Int32 piOff = new OutputPointer.Int32(); + String ret = ""; + int rc = SQLITE_OK; + + int nPhrase = ext.xPhraseCount(fCx); + int ii; + + for(ii=0; rc==SQLITE_OK && ii=0; + ext.xPhraseNext(fCx, pIter, piCol, piOff) + ){ + if( !ret.isEmpty() ) ret += " "; + ret += "{"+ii+" "+piCol.get()+" "+piOff.get()+"}"; + } + } + + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_pinst: rc=" + rc); + }else{ + sqlite3_result_text(pCx, ret); + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_pcolinst(); + ** + ** Like SQL function fts5_pinst(), except using the following + ** + ** xPhraseFirstColumn + ** xPhraseNextColumn + */ + fts5_extension_function fts5_pcolinst = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_pcolinst: wrong number of args"); + } + + OutputPointer.Int32 piCol = new OutputPointer.Int32(); + String ret = ""; + int rc = SQLITE_OK; + + int nPhrase = ext.xPhraseCount(fCx); + int ii; + + for(ii=0; rc==SQLITE_OK && ii=0; + ext.xPhraseNextColumn(fCx, pIter, piCol) + ){ + if( !ret.isEmpty() ) ret += " "; + ret += "{"+ii+" "+piCol.get()+"}"; + } + } + + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_pcolinst: rc=" + rc); + }else{ + sqlite3_result_text(pCx, ret); + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_rowcount(); + */ + fts5_extension_function fts5_rowcount = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=0 ){ + throw new RuntimeException("fts5_rowcount: wrong number of args"); + } + OutputPointer.Int64 pnRow = new OutputPointer.Int64(); + + int rc = ext.xRowCount(fCx, pnRow); + if( rc==SQLITE_OK ){ + sqlite3_result_int64(pCx, pnRow.get()); + }else{ + throw new RuntimeException("fts5_rowcount: rc=" + rc); + } + } + public void xDestroy(){ } + }; + + /* + ** fts5_phrasesize(); + */ + fts5_extension_function fts5_phrasesize = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=1 ){ + throw new RuntimeException("fts5_phrasesize: wrong number of args"); + } + int iPhrase = sqlite3_value_int(argv[0]); + + int sz = ext.xPhraseSize(fCx, iPhrase); + sqlite3_result_int(pCx, sz); + } + public void xDestroy(){ } + }; + + /* + ** fts5_phrasehits(, ); + ** + ** Use the xQueryPhrase() API to determine how many hits, in total, + ** there are for phrase in the database. + */ + fts5_extension_function fts5_phrasehits = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=1 ){ + throw new RuntimeException("fts5_phrasesize: wrong number of args"); + } + int iPhrase = sqlite3_value_int(argv[0]); + int rc = SQLITE_OK; + + class MyCallback implements Fts5ExtensionApi.XQueryPhraseCallback { + public int nRet = 0; + public int getRet() { return nRet; } + + @Override + public int call(Fts5ExtensionApi fapi, Fts5Context cx){ + OutputPointer.Int32 pnInst = new OutputPointer.Int32(); + int rc = fapi.xInstCount(cx, pnInst); + nRet += pnInst.get(); + return rc; + } + }; + + MyCallback xCall = new MyCallback(); + rc = ext.xQueryPhrase(fCx, iPhrase, xCall); + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_phrasehits: rc=" + rc); + } + sqlite3_result_int(pCx, xCall.getRet()); + } + public void xDestroy(){ } + }; + + /* + ** fts5_tokenize(, ) + */ + fts5_extension_function fts5_tokenize = new fts5_extension_function(){ + @Override public void call( + Fts5ExtensionApi ext, + Fts5Context fCx, + sqlite3_context pCx, + sqlite3_value argv[] + ){ + if( argv.length!=1 ){ + throw new RuntimeException("fts5_tokenize: wrong number of args"); + } + byte[] utf8 = sqlite3_value_text(argv[0]); + int rc = SQLITE_OK; + + class MyCallback implements XTokenizeCallback { + private List myList = new ArrayList<>(); + + public String getval() { + return String.join("+", myList); + } + + @Override + public int call(int tFlags, byte[] txt, int iStart, int iEnd){ + try { + String str = new String(txt, StandardCharsets.UTF_8); + myList.add(str); + } catch (Exception e) { + } + return SQLITE_OK; + } + }; + + MyCallback xCall = new MyCallback(); + ext.xTokenize(fCx, utf8, xCall); + sqlite3_result_text16(pCx, xCall.getval()); + + if( rc!=SQLITE_OK ){ + throw new RuntimeException("fts5_tokenize: rc=" + rc); + } + } + public void xDestroy(){ } + }; + + fts5_api api = fts5_api.getInstanceForDb(db); + api.xCreateFunction("fts5_rowid", fts5_rowid); + api.xCreateFunction("fts5_columncount", fts5_columncount); + api.xCreateFunction("fts5_columnsize", fts5_columnsize); + api.xCreateFunction("fts5_columntext", fts5_columntext); + api.xCreateFunction("fts5_columntotalsize", fts5_columntsize); + + api.xCreateFunction("fts5_aux1", new fts5_aux()); + api.xCreateFunction("fts5_aux2", new fts5_aux()); + + api.xCreateFunction("fts5_inst", fts5_inst); + api.xCreateFunction("fts5_pinst", fts5_pinst); + api.xCreateFunction("fts5_pcolinst", fts5_pcolinst); + api.xCreateFunction("fts5_rowcount", fts5_rowcount); + api.xCreateFunction("fts5_phrasesize", fts5_phrasesize); + api.xCreateFunction("fts5_phrasehits", fts5_phrasehits); + api.xCreateFunction("fts5_tokenize", fts5_tokenize); + } + /* + ** Test of various Fts5ExtensionApi methods + */ + private static void test2(){ + + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(a, b);" + + "INSERT INTO ft(rowid, a, b) VALUES(-9223372036854775808, 'x', 'x');" + + "INSERT INTO ft(rowid, a, b) VALUES(0, 'x', 'x');" + + "INSERT INTO ft(rowid, a, b) VALUES(1, 'x y z', 'x y z');" + + "INSERT INTO ft(rowid, a, b) VALUES(2, 'x y z', 'x z');" + + "INSERT INTO ft(rowid, a, b) VALUES(3, 'x y z', 'x y z');" + + "INSERT INTO ft(rowid, a, b) VALUES(9223372036854775807, 'x', 'x');" + ); + + create_test_functions(db); + + /* Test that fts5_rowid() seems to work */ + do_execsql_test(db, + "SELECT rowid==fts5_rowid(ft) FROM ft('x')", + "[1, 1, 1, 1, 1, 1]" + ); + + /* Test fts5_columncount() */ + do_execsql_test(db, + "SELECT fts5_columncount(ft) FROM ft('x')", + "[2, 2, 2, 2, 2, 2]" + ); + + /* Test fts5_columnsize() */ + do_execsql_test(db, + "SELECT fts5_columnsize(ft, 0) FROM ft('x') ORDER BY rowid", + "[1, 1, 3, 3, 3, 1]" + ); + do_execsql_test(db, + "SELECT fts5_columnsize(ft, 1) FROM ft('x') ORDER BY rowid", + "[1, 1, 3, 2, 3, 1]" + ); + do_execsql_test(db, + "SELECT fts5_columnsize(ft, -1) FROM ft('x') ORDER BY rowid", + "[2, 2, 6, 5, 6, 2]" + ); + + /* Test that xColumnSize() returns SQLITE_RANGE if the column number + ** is out-of range */ + try { + do_execsql_test(db, + "SELECT fts5_columnsize(ft, 2) FROM ft('x') ORDER BY rowid" + ); + } catch( RuntimeException e ){ + affirm( e.getMessage().matches(".*column index out of range") ); + } + + /* Test fts5_columntext() */ + do_execsql_test(db, + "SELECT fts5_columntext(ft, 0) FROM ft('x') ORDER BY rowid", + "[x, x, x y z, x y z, x y z, x]" + ); + do_execsql_test(db, + "SELECT fts5_columntext(ft, 1) FROM ft('x') ORDER BY rowid", + "[x, x, x y z, x z, x y z, x]" + ); + boolean threw = false; + try{ + /* columntext() used to return NULLs when given an out-of bounds column + but now results in a range error. */ + do_execsql_test(db, + "SELECT fts5_columntext(ft, 2) FROM ft('x') ORDER BY rowid", + "[null, null, null, null, null, null]" + ); + }catch(Exception e){ + threw = true; + affirm( e.getMessage().matches(".*column index out of range") ); + } + affirm( threw ); + threw = false; + + /* Test fts5_columntotalsize() */ + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, 0) FROM ft('x') ORDER BY rowid", + "[12, 12, 12, 12, 12, 12]" + ); + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, 1) FROM ft('x') ORDER BY rowid", + "[11, 11, 11, 11, 11, 11]" + ); + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, -1) FROM ft('x') ORDER BY rowid", + "[23, 23, 23, 23, 23, 23]" + ); + + /* Test that xColumnTotalSize() returns SQLITE_RANGE if the column + ** number is out-of range */ + try { + do_execsql_test(db, + "SELECT fts5_columntotalsize(ft, 2) FROM ft('x') ORDER BY rowid" + ); + } catch( RuntimeException e ){ + affirm( e.getMessage().matches(".*column index out of range") ); + } + + do_execsql_test(db, + "SELECT rowid, fts5_rowcount(ft) FROM ft('z')", + "[1, 6, 2, 6, 3, 6]" + ); + + sqlite3_close_v2(db); + } + + /* + ** Test of various Fts5ExtensionApi methods + */ + private static void test3(){ + + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(a, b);" + + "INSERT INTO ft(a, b) VALUES('the one', 1);" + + "INSERT INTO ft(a, b) VALUES('the two', 2);" + + "INSERT INTO ft(a, b) VALUES('the three', 3);" + + "INSERT INTO ft(a, b) VALUES('the four', '');" + ); + create_test_functions(db); + + /* Test fts5_aux1() + fts5_aux2() - users of xGetAuxdata and xSetAuxdata */ + do_execsql_test(db, + "SELECT fts5_aux1(ft, a) FROM ft('the')", + "[null, the one, the two, the three]" + ); + do_execsql_test(db, + "SELECT fts5_aux2(ft, b) FROM ft('the')", + "[null, 1, 2, 3]" + ); + do_execsql_test(db, + "SELECT fts5_aux1(ft, a), fts5_aux2(ft, b) FROM ft('the')", + "[null, null, the one, 1, the two, 2, the three, 3]" + ); + do_execsql_test(db, + "SELECT fts5_aux1(ft, b), fts5_aux1(ft) FROM ft('the')", + "[null, 1, 1, 2, 2, 3, 3, null]" + ); + } + + /* + ** Test of various Fts5ExtensionApi methods + */ + private static void test4(){ + + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + create_test_functions(db); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(a, b);" + + "INSERT INTO ft(a, b) VALUES('one two three', 'two three four');" + + "INSERT INTO ft(a, b) VALUES('two three four', 'three four five');" + + "INSERT INTO ft(a, b) VALUES('three four five', 'four five six');" + ); + + + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('two')", + "[{0 0 1} {0 1 0}, {0 0 0}]" + ); + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('four')", + "[{0 1 2}, {0 0 2} {0 1 1}, {0 0 1} {0 1 0}]" + ); + + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('a OR b OR four')", + "[{2 1 2}, {2 0 2} {2 1 1}, {2 0 1} {2 1 0}]" + ); + do_execsql_test(db, + "SELECT fts5_inst(ft) FROM ft('two four')", + "[{0 0 1} {0 1 0} {1 1 2}, {0 0 0} {1 0 2} {1 1 1}]" + ); + + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('two')", + "[{0 0 1} {0 1 0}, {0 0 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('four')", + "[{0 1 2}, {0 0 2} {0 1 1}, {0 0 1} {0 1 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('a OR b OR four')", + "[{2 1 2}, {2 0 2} {2 1 1}, {2 0 1} {2 1 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pinst(ft) FROM ft('two four')", + "[{0 0 1} {0 1 0} {1 1 2}, {0 0 0} {1 0 2} {1 1 1}]" + ); + + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('two')", + "[{0 0} {0 1}, {0 0}]" + ); + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('four')", + "[{0 1}, {0 0} {0 1}, {0 0} {0 1}]" + ); + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('a OR b OR four')", + "[{2 1}, {2 0} {2 1}, {2 0} {2 1}]" + ); + do_execsql_test(db, + "SELECT fts5_pcolinst(ft) FROM ft('two four')", + "[{0 0} {0 1} {1 1}, {0 0} {1 0} {1 1}]" + ); + + do_execsql_test(db, + "SELECT fts5_phrasesize(ft, 0) FROM ft('four five six') LIMIT 1;", + "[1]" + ); + do_execsql_test(db, + "SELECT fts5_phrasesize(ft, 0) FROM ft('four + five + six') LIMIT 1;", + "[3]" + ); + + + sqlite3_close_v2(db); + } + + private static void test5(){ + /* Open db and populate an fts5 table */ + sqlite3 db = createNewDb(); + create_test_functions(db); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(x, b);" + + "INSERT INTO ft(x) VALUES('one two three four five six seven eight');" + + "INSERT INTO ft(x) VALUES('one two one four one six one eight');" + + "INSERT INTO ft(x) VALUES('one two three four five six seven eight');" + ); + + do_execsql_test(db, + "SELECT fts5_phrasehits(ft, 0) FROM ft('one') LIMIT 1", + "[6]" + ); + + sqlite3_close_v2(db); + } + + private static void test6(){ + sqlite3 db = createNewDb(); + create_test_functions(db); + do_execsql_test(db, + "CREATE VIRTUAL TABLE ft USING fts5(x, b);" + + "INSERT INTO ft(x) VALUES('one two three four five six seven eight');" + ); + + do_execsql_test(db, + "SELECT fts5_tokenize(ft, 'abc def ghi') FROM ft('one')", + "[abc+def+ghi]" + ); + do_execsql_test(db, + "SELECT fts5_tokenize(ft, 'it''s BEEN a...') FROM ft('one')", + "[it+s+been+a]" + ); + + sqlite3_close_v2(db); + } + + private static synchronized void runTests(){ + test1(); + test2(); + test3(); + test4(); + test5(); + test6(); + } + + public TesterFts5(){ + runTests(); + } +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java b/ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java new file mode 100644 index 0000000000..3aa514f314 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java @@ -0,0 +1,22 @@ +/* +** 2023-08-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; + + +/** + Callback type for use with xTokenize() variants. +*/ +public interface XTokenizeCallback { + int call(int tFlags, byte[] txt, int iStart, int iEnd); +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_api.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_api.java new file mode 100644 index 0000000000..d7d2da430d --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_api.java @@ -0,0 +1,76 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.annotation.*; +import org.sqlite.jni.capi.*; + +/** + A wrapper for communicating C-level (fts5_api*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class fts5_api extends NativePointerHolder { + /* Only invoked from JNI */ + private fts5_api(){} + + public static final int iVersion = 2; + + /** + Returns the fts5_api instance associated with the given db, or + null if something goes horribly wrong. + */ + public static synchronized native fts5_api getInstanceForDb(@NotNull sqlite3 db); + + public synchronized native int xCreateFunction(@NotNull String name, + @Nullable Object userData, + @NotNull fts5_extension_function xFunction); + + /** + Convenience overload which passes null as the 2nd argument to the + 3-parameter form. + */ + public int xCreateFunction(@NotNull String name, + @NotNull fts5_extension_function xFunction){ + return xCreateFunction(name, null, xFunction); + } + + // /* Create a new auxiliary function */ + // int (*xCreateFunction)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_extension_function xFunction, + // void (*xDestroy)(void*) + // ); + + // Still potentially todo: + + // int (*xCreateTokenizer)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_tokenizer *pTokenizer, + // void (*xDestroy)(void*) + // ); + + // /* Find an existing tokenizer */ + // int (*xFindTokenizer)( + // fts5_api *pApi, + // const char *zName, + // void **ppContext, + // fts5_tokenizer *pTokenizer + // ); + +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java new file mode 100644 index 0000000000..6e98f64ff3 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java @@ -0,0 +1,51 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.sqlite3_context; +import org.sqlite.jni.capi.sqlite3_value; + +/** + JNI-level wrapper for C's fts5_extension_function type. +*/ +public interface fts5_extension_function { + // typedef void (*fts5_extension_function)( + // const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + // Fts5Context *pFts, /* First arg to pass to pApi functions */ + // sqlite3_context *pCtx, /* Context for returning result/error */ + // int nVal, /* Number of values in apVal[] array */ + // sqlite3_value **apVal /* Array of trailing arguments */ + // ); + + /** + The callback implementation, corresponding to the xFunction + argument of C's fts5_api::xCreateFunction(). + */ + void call(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]); + /** + Is called when this function is destroyed by sqlite3. Typically + this function will be empty. + */ + void xDestroy(); + + /** + A base implementation of fts5_extension_function() which has a + no-op xDestroy() method. + */ + abstract class Abstract implements fts5_extension_function { + @Override public abstract void call(Fts5ExtensionApi ext, Fts5Context fCx, + sqlite3_context pCx, sqlite3_value argv[]); + @Override public void xDestroy(){} + } +} diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java new file mode 100644 index 0000000000..f4ada4dc30 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java @@ -0,0 +1,49 @@ +/* +** 2023-08-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the JNI bindings for the sqlite3 C API. +*/ +package org.sqlite.jni.fts5; +import org.sqlite.jni.capi.NativePointerHolder; +import org.sqlite.jni.annotation.NotNull; + +/** + A wrapper for communicating C-level (fts5_tokenizer*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java and C + via JNI. +*/ +public final class fts5_tokenizer extends NativePointerHolder { + /* Only invoked by JNI */ + private fts5_tokenizer(){} + + // int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + // void (*xDelete)(Fts5Tokenizer*); + + public native int xTokenize(@NotNull Fts5Tokenizer t, int tokFlags, + @NotNull byte pText[], + @NotNull XTokenizeCallback callback); + + + // int (*xTokenize)(Fts5Tokenizer*, + // void *pCtx, + // int flags, /* Mask of FTS5_TOKENIZE_* flags */ + // const char *pText, int nText, + // int (*xToken)( + // void *pCtx, /* Copy of 2nd argument to xTokenize() */ + // int tflags, /* Mask of FTS5_TOKEN_* flags */ + // const char *pToken, /* Pointer to buffer containing token */ + // int nToken, /* Size of token in bytes */ + // int iStart, /* Byte offset of token within input text */ + // int iEnd /* Byte offset of end of token within input text */ + // ) + // ); +} diff --git a/ext/jni/src/org/sqlite/jni/test-script-interpreter.md b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md new file mode 100644 index 0000000000..cc7b7e7f9a --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md @@ -0,0 +1,270 @@ +# Specifications For A Rudimentary SQLite Test Script Interpreter + +## Overview + +The purpose of the Test Script Interpreter is to read and interpret +script files that contain SQL commands and desired results. The +interpreter will check results and report any discrepancies found. + +The test script files are ASCII text files. The filename always ends with +".test". Each script is evaluated independently; context does not carry +forward from one script to the next. So, for example, the --null command +run in one test script does not cause any changes in the behavior of +subsequent test scripts. All open database connections are closed +at the end of each test script. All database files created by a test +script are deleted when the script finishes. + +## Parsing Rules: + + 1. The test script is read line by line, where a line is a sequence of + characters that runs up to the next '\\n' (0x0a) character or until + the end of the file. There is never a need to read ahead past the + end of the current line. + + 2. If any line contains the string " MODULE_NAME:" (with a space before + the initial "M") or "MIXED_MODULE_NAME:" then that test script is + incompatible with this spec. Processing of the test script should + end immediately. There is no need to read any more of the file. + In verbose mode, the interpreter might choose to emit an informational + messages saying that the test script was abandoned due to an + incompatible module type. + + 3. If any line contains the string "SCRIPT_MODULE_NAME:" then the input + script is known to be of the correct type for this specification and + processing may continue. The "MODULE_NAME" checking in steps 2 and 3 + may optionally be discontinued after sighting a "SCRIPT_MODULE_NAME". + + 4. If any line contains "REQUIRED_PROPERTIES:" and that substring is followed + by any non-whitespace text, then the script is not compatible with this + spec. Processing should stop immediately. In verbose mode, the + interpreter might choose to emit an information message saying that the + test script was abandoned due to unsupported requirement properties. + + 5. If any line begins with the "\|" (0x7c) character, that indicates that + the input script is not compatible with this specification. Processing + of the script should stop immediately. In verbose mode, the interpreter + might choose to emit an informational message indicating that the + test script was abandoned because it contained "a dbtotxt format database + specification". + + 6. Any line that begins with "#" is a C-preprocessor line. The interpreter + described by this spec does not know how to deal with C-preprocessor lines. + Hence, processing should be abandoned. In verbose mode, the interpreter + might emit an informational message similar to + "script NAME abandoned due to C-preprocessor line: ..." + + 7. If a line begins with exactly two minus signs followed by a + lowercase letter, that is a command. Process commands as described + below. + + 8. All other lines should be accumulated into the "input buffer". + The various commands will have access to this input buffer. + Some commands will reset the buffer. + +## Initialization + +The initial state of the interpreter at the start of processing each script +is as if the following command sequence had been run: + +> ~~~ +--close all +--db 0 +--new test.db +--null nil +~~~ + +In words, all database connections are closed except for connection 0 (the +default) which is open on an empty database named "test.db". The string +"nil" is displayed for NULL column values. + +The only context carried forward after the evaluation of one test script +into the evaluation of the next test script is the count of the number of +tests run and the number of failures seen. + +## Commands: + +Each command looks like an SQL comment. The command begins at the left +margin (no leading space) and starts with exactly 2 minus signs ("-"). +The command name consists of lowercase letters and maybe a "-" or two. +Some commands have arguments. + +The arguments are separated from the command name by one or more spaces. + +Commands have access to the input buffer and might reset the input buffer. +The command can also optionally read (and consume) additional text from +script that comes after the command. + +Unknown or unrecognized commands indicate that the script contains features +that are not (yet) supported by this specification. Processing of the +script should terminate immediately. When this happens and when the +interpreter is in a "verbose" mode, the interpreter might choose to emit +an informational message along the lines of "test script NAME abandoned +due to unsupported command: --whatever". + +The initial implementation will only recognize a few commands. Other +commands may be added later. The following is the initial set of +commands: + +### The --testcase command + +Every test case starts with a --testcase command. The --testcase +command resets both the "input buffer" and the "result buffer". The +argument to the --testcase command is the name of the test case. That +test case name is used for logging and debugging and when printing +errors. The input buffer is set to the body of the test case. + +### The --result command + +The --result command tries to execute the text in the input buffer as SQL. +For each row of result coming out of this SQL, the text of that result is +appended to the "result buffer". If a result row contains multiple columns, +the columns are processed from left to right. For each column, text is +appended to the result buffer according to the following rules: + + * If the result buffer already contains some text, append a space. + (In this way, all column values and all row values are separated from + each other by a single space.) + + * If sqlite3_column_text() returns NULL, then append "nil" - or + some other text that is specified by the --null command - and skip + all subsequent rules. + + * If sqlite3_column_text() is an empty string, append `{}` to the + result buffer and skip all subsequent rules. + + * If sqlite3_column_text() does not contain any special + characters, append it to the result buffer without any + formatting and skip all subsequent rules. Special characters are: + 0x00 to 0x20 (inclusive), double-quote (0x22), backslash (0x5c), + curly braces (0x7b and 0x7d). + + * If sqlite3_column_text() does not contains curly braces, then put + the text inside of `{...}` and append it and skip all subsequent rules. + + * Append the text within double-quotes (`"..."`) and within the text + escape '"' and '\\' by prepending a single '\\' and escape any + control characters (characters less than 0x20) using octal notation: + '\\NNN'. + +If an error is encountered while running the SQL, then append the +symbolic C-preprocessor name for the error +code (ex: "SQLITE_CONSTRAINT") as if it were a column value. Then append +the error message text as if it where a column value. Then stop processing. + +After the SQL text has been run, compare the content of the result buffer +against the argument to the --result command and report a testing error if +there are any differences. + +The --result command resets the input buffer, but it does not reset +the result buffer. This distinction does not matter for the --result +command itself, but it is important for related commands like --glob +and --notglob. Sometimes test cases will contains a bunch of SQL +followed by multiple --glob and/or --notglob statements. All of the +globs should be evaluated against the result buffer, but the SQL should +only be run once. This is accomplished by resetting the input buffer +but not the result buffer. + +### The --glob command + +The --glob command works just like --result except that the argument to +--glob is interpreted as a TEST-GLOB pattern and the results are compared +using that glob pattern rather than using strcmp(). Other than that, +the two operate the same. + +The TEST-GLOB pattern is slightly different for a standard GLOB: + + * The '*' character matches zero or more characters. + + * The '?' character matches any single character + + * The '[...]' character sequence machines a single character + in between the brackets. + + * The '#' character matches one or more digits (This is the main + difference between standard unix-glob and TEST-GLOB. unix-glob + does not have this feature. It was added to because it comes + up a lot during SQLite testing.) + +### The --notglob command + +The --notglob command works just like --glob except that it reports an +error if the GLOB does match, rather than if the GLOB does not match. + +### The --oom command + +This command is to be used for out-of-memory testing. It means that +OOM errors should be simulated to ensure that SQLite is able to deal with +them. This command can be silently ignored for now. We might add support +for this later. + +### The --tableresult command + +The --tableresult command works like --glob except that the GLOB pattern +to be matched is taken from subsequent lines of the input script up to +the next --end. Every span of one or more whitespace characters in this +pattern text is collapsed into a single space (0x20). +Leading and trailing whitespace are removed from the pattern. +The --end that ends the GLOB pattern is not part of the GLOB pattern, but +the --end is consumed from the script input. + +### The --new and --open commands + +The --new and --open commands cause a database file to be opened. +The name of the file is the argument to the command. The --new command +opens an initially empty database (it deletes the file before opening it) +whereas the --open command opens an existing database if it already +exists. + +### The --db command + +The script interpreter can have up to 7 different SQLite database +connections open at a time. The --db command is used to switch between +them. The argument to --db is an integer between 0 and 6 that selects +which database connection to use moving forward. + +### The --close command + +The --close command causes an existing database connection to close. +This command is a no-op if the database connection is not currently +open. There can be up to 7 different database connections, numbered 0 +through 6. The number of the database connection to close is an +argument to the --close command, which will fail if an out-of-range +value is provided. Or if the argument to --close is "all" then all +open database connections are closed. If passed no argument, the +currently-active database is assumed. + +### The --null command + +The NULL command changes the text that is used to represent SQL NULL +values in the result buffer. + +### The --run command + +The --run command executes text in the input buffer as if it where SQL. +However, nothing is added to the result buffer. Any output from the SQL +is silently ignored. Errors in the SQL are silently ignored. + +The --run command normally executes the SQL in the current database +connection. However, if --run has an argument that is an integer between +0 and 6 then the SQL is run in the alternative database connection specified +by that argument. + +### The --json and --json-block commands + +The --json and --json-block commands work like --result and --tableresult, +respectively. The difference is that column values are appended to the +result buffer literally, without ever enclosing the values in `{...}` or +`"..."` and without escaping any characters in the column value and comparison +is always an exact strcmp() not a GLOB. + +### The --print command + +The --print command emits both its arguments and its body (if any) to +stdout, indenting each line of output. + +### The --column-names command + +The --column-names command requires 0 or 1 as an argument, to disable +resp. enable it, and modifies SQL execution to include column names +in output. When this option is on, each column value emitted gets +prefixed by its column name, with a single space between them. diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java new file mode 100644 index 0000000000..fc63b53542 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java @@ -0,0 +1,144 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; + +/** + EXPERIMENTAL/INCOMPLETE/UNTESTED + + A SqlFunction implementation for aggregate functions. The T type + represents the type of data accumulated by this aggregate while it + works. e.g. a SUM()-like UDF might use Integer or Long and a + CONCAT()-like UDF might use a StringBuilder or a List. +*/ +public abstract class AggregateFunction implements SqlFunction { + + /** + As for the xStep() argument of the C API's + sqlite3_create_function(). If this function throws, the + exception is reported via sqlite3_result_error(). + */ + public abstract void xStep(SqlFunction.Arguments args); + + /** + As for the xFinal() argument of the C API's + sqlite3_create_function(). If this function throws, it is + translated into sqlite3_result_error(). + + Note that the passed-in object will not actually contain any + arguments for xFinal() but will contain the context object needed + for setting the call's result or error state. + */ + public abstract void xFinal(SqlFunction.Arguments args); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. + */ + public void xDestroy() {} + + /** + PerContextState assists aggregate and window functions in + managing their accumulator state across calls to the UDF's + callbacks. + +

    T must be of a type which can be legally stored as a value in + java.util.HashMap. + +

    If a given aggregate or window function is called multiple times + in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)..., + then the clients need some way of knowing which call is which so + that they can map their state between their various UDF callbacks + and reset it via xFinal(). This class takes care of such + mappings. + +

    This class works by mapping + sqlite3_context.getAggregateContext() to a single piece of + state, of a client-defined type (the T part of this class), which + persists across a "matching set" of the UDF's callbacks. + +

    This class is a helper providing commonly-needed functionality + - it is not required for use with aggregate or window functions. + Client UDFs are free to perform such mappings using custom + approaches. The provided {@link AggregateFunction} and {@link + WindowFunction} classes use this. + */ + public static final class PerContextState { + private final java.util.Map> map + = new java.util.HashMap<>(); + + /** + Should be called from a UDF's xStep(), xValue(), and xInverse() + methods, passing it that method's first argument and an initial + value for the persistent state. If there is currently no + mapping for the given context within the map, one is created + using the given initial value, else the existing one is used + and the 2nd argument is ignored. It returns a ValueHolder + which can be used to modify that state directly without + requiring that the client update the underlying map's entry. + +

    The caller is obligated to eventually call + takeAggregateState() to clear the mapping. + */ + public ValueHolder getAggregateState(SqlFunction.Arguments args, T initialValue){ + final Long key = args.getContext().getAggregateContext(true); + ValueHolder rc = null==key ? null : map.get(key); + if( null==rc ){ + map.put(key, rc = new ValueHolder<>(initialValue)); + } + return rc; + } + + /** + Should be called from a UDF's xFinal() method and passed that + method's first argument. This function removes the value + associated with with the arguments' aggregate context from the + map and returns it, returning null if no other UDF method has + been called to set up such a mapping. The latter condition will + be the case if a UDF is used in a statement which has no result + rows. + */ + public T takeAggregateState(SqlFunction.Arguments args){ + final ValueHolder h = map.remove(args.getContext().getAggregateContext(false)); + return null==h ? null : h.value; + } + } + + /** Per-invocation state for the UDF. */ + private final PerContextState map = new PerContextState<>(); + + /** + To be called from the implementation's xStep() method, as well + as the xValue() and xInverse() methods of the {@link WindowFunction} + subclass, to fetch the current per-call UDF state. On the + first call to this method for any given sqlite3_context + argument, the context is set to the given initial value. On all other + calls, the 2nd argument is ignored. + + @see SQLFunction.PerContextState#getAggregateState + */ + protected final ValueHolder getAggregateState(SqlFunction.Arguments args, T initialValue){ + return map.getAggregateState(args, initialValue); + } + + /** + To be called from the implementation's xFinal() method to fetch + the final state of the UDF and remove its mapping. + + see SQLFunction.PerContextState#takeAggregateState + */ + protected final T takeAggregateState(SqlFunction.Arguments args){ + return map.takeAggregateState(args); + } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java new file mode 100644 index 0000000000..c616ae7393 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java @@ -0,0 +1,33 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; + +/** + The SqlFunction type for scalar SQL functions. +*/ +public abstract class ScalarFunction implements SqlFunction { + /** + As for the xFunc() argument of the C API's + sqlite3_create_function(). If this function throws, it is + translated into an sqlite3_result_error(). + */ + public abstract void xFunc(SqlFunction.Arguments args); + + /** + Optionally override to be notified when the UDF is finalized by + SQLite. This default implementation does nothing. + */ + public void xDestroy() {} + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java new file mode 100644 index 0000000000..bb0fd0ccd4 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java @@ -0,0 +1,318 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.capi.sqlite3_context; +import org.sqlite.jni.capi.sqlite3_value; + +/** + Base marker interface for SQLite's three types of User-Defined SQL + Functions (UDFs): Scalar, Aggregate, and Window functions. +*/ +public interface SqlFunction { + + int DETERMINISTIC = CApi.SQLITE_DETERMINISTIC; + int INNOCUOUS = CApi.SQLITE_INNOCUOUS; + int DIRECTONLY = CApi.SQLITE_DIRECTONLY; + int SUBTYPE = CApi.SQLITE_SUBTYPE; + int RESULT_SUBTYPE = CApi.SQLITE_RESULT_SUBTYPE; + int UTF8 = CApi.SQLITE_UTF8; + int UTF16 = CApi.SQLITE_UTF16; + + /** + The Arguments type is an abstraction on top of the lower-level + UDF function argument types. It provides _most_ of the functionality + of the lower-level interface, insofar as possible without "leaking" + those types into this API. + */ + final class Arguments implements Iterable{ + private final sqlite3_context cx; + private final sqlite3_value args[]; + public final int length; + + /** + Must be passed the context and arguments for the UDF call this + object is wrapping. Intended to be used by internal proxy + classes which "convert" the lower-level interface into this + package's higher-level interface, e.g. ScalarAdapter and + AggregateAdapter. + + Passing null for the args is equivalent to passing a length-0 + array. + */ + Arguments(sqlite3_context cx, sqlite3_value args[]){ + this.cx = cx; + this.args = args==null ? new sqlite3_value[0] : args; + this.length = this.args.length; + } + + /** + Returns the sqlite3_value at the given argument index or throws + an IllegalArgumentException exception if ndx is out of range. + */ + private sqlite3_value valueAt(int ndx){ + if(ndx<0 || ndx>=args.length){ + throw new IllegalArgumentException( + "SQL function argument index "+ndx+" is out of range." + ); + } + return args[ndx]; + } + + //! Returns the underlying sqlite3_context for these arguments. + sqlite3_context getContext(){return cx;} + + /** + Returns the Sqlite (db) object associated with this UDF call, + or null if the UDF is somehow called without such an object or + the db has been closed in an untimely manner (e.g. closed by a + UDF call). + */ + public Sqlite getDb(){ + return Sqlite.fromNative( CApi.sqlite3_context_db_handle(cx) ); + } + + public int getArgCount(){ return args.length; } + + public int getInt(int argNdx){return CApi.sqlite3_value_int(valueAt(argNdx));} + public long getInt64(int argNdx){return CApi.sqlite3_value_int64(valueAt(argNdx));} + public double getDouble(int argNdx){return CApi.sqlite3_value_double(valueAt(argNdx));} + public byte[] getBlob(int argNdx){return CApi.sqlite3_value_blob(valueAt(argNdx));} + public byte[] getText(int argNdx){return CApi.sqlite3_value_text(valueAt(argNdx));} + public String getText16(int argNdx){return CApi.sqlite3_value_text16(valueAt(argNdx));} + public int getBytes(int argNdx){return CApi.sqlite3_value_bytes(valueAt(argNdx));} + public int getBytes16(int argNdx){return CApi.sqlite3_value_bytes16(valueAt(argNdx));} + public Object getObject(int argNdx){return CApi.sqlite3_value_java_object(valueAt(argNdx));} + public T getObject(int argNdx, Class type){ + return CApi.sqlite3_value_java_object(valueAt(argNdx), type); + } + + public int getType(int argNdx){return CApi.sqlite3_value_type(valueAt(argNdx));} + public int getSubtype(int argNdx){return CApi.sqlite3_value_subtype(valueAt(argNdx));} + public int getNumericType(int argNdx){return CApi.sqlite3_value_numeric_type(valueAt(argNdx));} + public int getNoChange(int argNdx){return CApi.sqlite3_value_nochange(valueAt(argNdx));} + public boolean getFromBind(int argNdx){return CApi.sqlite3_value_frombind(valueAt(argNdx));} + public int getEncoding(int argNdx){return CApi.sqlite3_value_encoding(valueAt(argNdx));} + + public void resultInt(int v){ CApi.sqlite3_result_int(cx, v); } + public void resultInt64(long v){ CApi.sqlite3_result_int64(cx, v); } + public void resultDouble(double v){ CApi.sqlite3_result_double(cx, v); } + public void resultError(String msg){CApi.sqlite3_result_error(cx, msg);} + public void resultError(Exception e){CApi.sqlite3_result_error(cx, e);} + public void resultErrorTooBig(){CApi.sqlite3_result_error_toobig(cx);} + public void resultErrorCode(int rc){CApi.sqlite3_result_error_code(cx, rc);} + public void resultObject(Object o){CApi.sqlite3_result_java_object(cx, o);} + public void resultNull(){CApi.sqlite3_result_null(cx);} + /** + Analog to sqlite3_result_value(), using the Value object at the + given argument index. + */ + public void resultArg(int argNdx){CApi.sqlite3_result_value(cx, valueAt(argNdx));} + public void resultSubtype(int subtype){CApi.sqlite3_result_subtype(cx, subtype);} + public void resultZeroBlob(long n){ + // Throw on error? If n is too big, + // sqlite3_result_error_toobig() is automatically called. + CApi.sqlite3_result_zeroblob64(cx, n); + } + + public void resultBlob(byte[] blob){CApi.sqlite3_result_blob(cx, blob);} + public void resultText(byte[] utf8){CApi.sqlite3_result_text(cx, utf8);} + public void resultText(String txt){CApi.sqlite3_result_text(cx, txt);} + public void resultText16(byte[] utf16){CApi.sqlite3_result_text16(cx, utf16);} + public void resultText16(String txt){CApi.sqlite3_result_text16(cx, txt);} + + /** + Callbacks should invoke this on OOM errors, instead of throwing + OutOfMemoryError, because the latter cannot be propagated + through the C API. + */ + public void resultNoMem(){CApi.sqlite3_result_error_nomem(cx);} + + /** + Analog to sqlite3_set_auxdata() but throws if argNdx is out of + range. + */ + public void setAuxData(int argNdx, Object o){ + /* From the API docs: https://sqlite.org/c3ref/get_auxdata.html + + The value of the N parameter to these interfaces should be + non-negative. Future enhancements may make use of negative N + values to define new kinds of function caching behavior. + */ + valueAt(argNdx); + CApi.sqlite3_set_auxdata(cx, argNdx, o); + } + + /** + Analog to sqlite3_get_auxdata() but throws if argNdx is out of + range. + */ + public Object getAuxData(int argNdx){ + valueAt(argNdx); + return CApi.sqlite3_get_auxdata(cx, argNdx); + } + + /** + Represents a single SqlFunction argument. Primarily intended + for use with the Arguments class's Iterable interface. + */ + public final static class Arg { + private final Arguments a; + private final int ndx; + /* Only for use by the Arguments class. */ + private Arg(Arguments a, int ndx){ + this.a = a; + this.ndx = ndx; + } + /** Returns this argument's index in its parent argument list. */ + public int getIndex(){return ndx;} + public int getInt(){return a.getInt(ndx);} + public long getInt64(){return a.getInt64(ndx);} + public double getDouble(){return a.getDouble(ndx);} + public byte[] getBlob(){return a.getBlob(ndx);} + public byte[] getText(){return a.getText(ndx);} + public String getText16(){return a.getText16(ndx);} + public int getBytes(){return a.getBytes(ndx);} + public int getBytes16(){return a.getBytes16(ndx);} + public Object getObject(){return a.getObject(ndx);} + public T getObject(Class type){ return a.getObject(ndx, type); } + public int getType(){return a.getType(ndx);} + public Object getAuxData(){return a.getAuxData(ndx);} + public void setAuxData(Object o){a.setAuxData(ndx, o);} + } + + @Override + public java.util.Iterator iterator(){ + final Arg[] proxies = new Arg[args.length]; + for( int i = 0; i < args.length; ++i ){ + proxies[i] = new Arg(this, i); + } + return java.util.Arrays.stream(proxies).iterator(); + } + + } + + /** + Internal-use adapter for wrapping this package's ScalarFunction + for use with the org.sqlite.jni.capi.ScalarFunction interface. + */ + final class ScalarAdapter extends org.sqlite.jni.capi.ScalarFunction { + private final ScalarFunction impl; + ScalarAdapter(ScalarFunction impl){ + this.impl = impl; + } + /** + Proxies this.impl.xFunc(), adapting the call arguments to that + function's signature. If the proxy throws, it's translated to + sqlite_result_error() with the exception's message. + */ + public void xFunc(sqlite3_context cx, sqlite3_value[] args){ + try{ + impl.xFunc( new SqlFunction.Arguments(cx, args) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + public void xDestroy(){ + impl.xDestroy(); + } + } + + /** + Internal-use adapter for wrapping this package's AggregateFunction + for use with the org.sqlite.jni.capi.AggregateFunction interface. + */ + class AggregateAdapter extends org.sqlite.jni.capi.AggregateFunction { + /*cannot be final without duplicating the whole body in WindowAdapter*/ + private final AggregateFunction impl; + AggregateAdapter(AggregateFunction impl){ + this.impl = impl; + } + + /** + Proxies this.impl.xStep(), adapting the call arguments to that + function's signature. If the proxied function throws, it is + translated to sqlite_result_error() with the exception's + message. + */ + public void xStep(sqlite3_context cx, sqlite3_value[] args){ + try{ + impl.xStep( new SqlFunction.Arguments(cx, args) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + /** + As for the xFinal() argument of the C API's + sqlite3_create_function(). If the proxied function throws, it + is translated into a sqlite3_result_error(). + */ + public void xFinal(sqlite3_context cx){ + try{ + impl.xFinal( new SqlFunction.Arguments(cx, null) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + public void xDestroy(){ + impl.xDestroy(); + } + } + + /** + Internal-use adapter for wrapping this package's WindowFunction + for use with the org.sqlite.jni.capi.WindowFunction interface. + */ + final class WindowAdapter extends AggregateAdapter { + private final WindowFunction impl; + WindowAdapter(WindowFunction impl){ + super(impl); + this.impl = impl; + } + + /** + Proxies this.impl.xInverse(), adapting the call arguments to that + function's signature. If the proxied function throws, it is + translated to sqlite_result_error() with the exception's + message. + */ + public void xInverse(sqlite3_context cx, sqlite3_value[] args){ + try{ + impl.xInverse( new SqlFunction.Arguments(cx, args) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + /** + As for the xValue() argument of the C API's sqlite3_create_window_function(). + If the proxied function throws, it is translated into a sqlite3_result_error(). + */ + public void xValue(sqlite3_context cx){ + try{ + impl.xValue( new SqlFunction.Arguments(cx, null) ); + }catch(Exception e){ + CApi.sqlite3_result_error(cx, e); + } + } + + public void xDestroy(){ + impl.xDestroy(); + } + } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java new file mode 100644 index 0000000000..d259e0ce62 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java @@ -0,0 +1,1994 @@ +/* +** 2023-10-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import java.nio.charset.StandardCharsets; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.capi.sqlite3; +import org.sqlite.jni.capi.sqlite3_stmt; +import org.sqlite.jni.capi.sqlite3_backup; +import org.sqlite.jni.capi.sqlite3_blob; +import org.sqlite.jni.capi.OutputPointer; + +/** + This class represents a database connection, analog to the C-side + sqlite3 class but with added argument validation, exceptions, and + similar "smoothing of sharp edges" to make the API safe to use from + Java. It also acts as a namespace for other types for which + individual instances are tied to a specific database connection. +*/ +public final class Sqlite implements AutoCloseable { + private sqlite3 db; + private static final boolean JNI_SUPPORTS_NIO = + CApi.sqlite3_jni_supports_nio(); + + // Result codes + public static final int OK = CApi.SQLITE_OK; + public static final int ERROR = CApi.SQLITE_ERROR; + public static final int INTERNAL = CApi.SQLITE_INTERNAL; + public static final int PERM = CApi.SQLITE_PERM; + public static final int ABORT = CApi.SQLITE_ABORT; + public static final int BUSY = CApi.SQLITE_BUSY; + public static final int LOCKED = CApi.SQLITE_LOCKED; + public static final int NOMEM = CApi.SQLITE_NOMEM; + public static final int READONLY = CApi.SQLITE_READONLY; + public static final int INTERRUPT = CApi.SQLITE_INTERRUPT; + public static final int IOERR = CApi.SQLITE_IOERR; + public static final int CORRUPT = CApi.SQLITE_CORRUPT; + public static final int NOTFOUND = CApi.SQLITE_NOTFOUND; + public static final int FULL = CApi.SQLITE_FULL; + public static final int CANTOPEN = CApi.SQLITE_CANTOPEN; + public static final int PROTOCOL = CApi.SQLITE_PROTOCOL; + public static final int EMPTY = CApi.SQLITE_EMPTY; + public static final int SCHEMA = CApi.SQLITE_SCHEMA; + public static final int TOOBIG = CApi.SQLITE_TOOBIG; + public static final int CONSTRAINT = CApi. SQLITE_CONSTRAINT; + public static final int MISMATCH = CApi.SQLITE_MISMATCH; + public static final int MISUSE = CApi.SQLITE_MISUSE; + public static final int NOLFS = CApi.SQLITE_NOLFS; + public static final int AUTH = CApi.SQLITE_AUTH; + public static final int FORMAT = CApi.SQLITE_FORMAT; + public static final int RANGE = CApi.SQLITE_RANGE; + public static final int NOTADB = CApi.SQLITE_NOTADB; + public static final int NOTICE = CApi.SQLITE_NOTICE; + public static final int WARNING = CApi.SQLITE_WARNING; + public static final int ROW = CApi.SQLITE_ROW; + public static final int DONE = CApi.SQLITE_DONE; + public static final int ERROR_MISSING_COLLSEQ = CApi.SQLITE_ERROR_MISSING_COLLSEQ; + public static final int ERROR_RETRY = CApi.SQLITE_ERROR_RETRY; + public static final int ERROR_SNAPSHOT = CApi.SQLITE_ERROR_SNAPSHOT; + public static final int IOERR_READ = CApi.SQLITE_IOERR_READ; + public static final int IOERR_SHORT_READ = CApi.SQLITE_IOERR_SHORT_READ; + public static final int IOERR_WRITE = CApi.SQLITE_IOERR_WRITE; + public static final int IOERR_FSYNC = CApi.SQLITE_IOERR_FSYNC; + public static final int IOERR_DIR_FSYNC = CApi.SQLITE_IOERR_DIR_FSYNC; + public static final int IOERR_TRUNCATE = CApi.SQLITE_IOERR_TRUNCATE; + public static final int IOERR_FSTAT = CApi.SQLITE_IOERR_FSTAT; + public static final int IOERR_UNLOCK = CApi.SQLITE_IOERR_UNLOCK; + public static final int IOERR_RDLOCK = CApi.SQLITE_IOERR_RDLOCK; + public static final int IOERR_DELETE = CApi.SQLITE_IOERR_DELETE; + public static final int IOERR_BLOCKED = CApi.SQLITE_IOERR_BLOCKED; + public static final int IOERR_NOMEM = CApi.SQLITE_IOERR_NOMEM; + public static final int IOERR_ACCESS = CApi.SQLITE_IOERR_ACCESS; + public static final int IOERR_CHECKRESERVEDLOCK = CApi.SQLITE_IOERR_CHECKRESERVEDLOCK; + public static final int IOERR_LOCK = CApi.SQLITE_IOERR_LOCK; + public static final int IOERR_CLOSE = CApi.SQLITE_IOERR_CLOSE; + public static final int IOERR_DIR_CLOSE = CApi.SQLITE_IOERR_DIR_CLOSE; + public static final int IOERR_SHMOPEN = CApi.SQLITE_IOERR_SHMOPEN; + public static final int IOERR_SHMSIZE = CApi.SQLITE_IOERR_SHMSIZE; + public static final int IOERR_SHMLOCK = CApi.SQLITE_IOERR_SHMLOCK; + public static final int IOERR_SHMMAP = CApi.SQLITE_IOERR_SHMMAP; + public static final int IOERR_SEEK = CApi.SQLITE_IOERR_SEEK; + public static final int IOERR_DELETE_NOENT = CApi.SQLITE_IOERR_DELETE_NOENT; + public static final int IOERR_MMAP = CApi.SQLITE_IOERR_MMAP; + public static final int IOERR_GETTEMPPATH = CApi.SQLITE_IOERR_GETTEMPPATH; + public static final int IOERR_CONVPATH = CApi.SQLITE_IOERR_CONVPATH; + public static final int IOERR_VNODE = CApi.SQLITE_IOERR_VNODE; + public static final int IOERR_AUTH = CApi.SQLITE_IOERR_AUTH; + public static final int IOERR_BEGIN_ATOMIC = CApi.SQLITE_IOERR_BEGIN_ATOMIC; + public static final int IOERR_COMMIT_ATOMIC = CApi.SQLITE_IOERR_COMMIT_ATOMIC; + public static final int IOERR_ROLLBACK_ATOMIC = CApi.SQLITE_IOERR_ROLLBACK_ATOMIC; + public static final int IOERR_DATA = CApi.SQLITE_IOERR_DATA; + public static final int IOERR_CORRUPTFS = CApi.SQLITE_IOERR_CORRUPTFS; + public static final int LOCKED_SHAREDCACHE = CApi.SQLITE_LOCKED_SHAREDCACHE; + public static final int LOCKED_VTAB = CApi.SQLITE_LOCKED_VTAB; + public static final int BUSY_RECOVERY = CApi.SQLITE_BUSY_RECOVERY; + public static final int BUSY_SNAPSHOT = CApi.SQLITE_BUSY_SNAPSHOT; + public static final int BUSY_TIMEOUT = CApi.SQLITE_BUSY_TIMEOUT; + public static final int CANTOPEN_NOTEMPDIR = CApi.SQLITE_CANTOPEN_NOTEMPDIR; + public static final int CANTOPEN_ISDIR = CApi.SQLITE_CANTOPEN_ISDIR; + public static final int CANTOPEN_FULLPATH = CApi.SQLITE_CANTOPEN_FULLPATH; + public static final int CANTOPEN_CONVPATH = CApi.SQLITE_CANTOPEN_CONVPATH; + public static final int CANTOPEN_SYMLINK = CApi.SQLITE_CANTOPEN_SYMLINK; + public static final int CORRUPT_VTAB = CApi.SQLITE_CORRUPT_VTAB; + public static final int CORRUPT_SEQUENCE = CApi.SQLITE_CORRUPT_SEQUENCE; + public static final int CORRUPT_INDEX = CApi.SQLITE_CORRUPT_INDEX; + public static final int READONLY_RECOVERY = CApi.SQLITE_READONLY_RECOVERY; + public static final int READONLY_CANTLOCK = CApi.SQLITE_READONLY_CANTLOCK; + public static final int READONLY_ROLLBACK = CApi.SQLITE_READONLY_ROLLBACK; + public static final int READONLY_DBMOVED = CApi.SQLITE_READONLY_DBMOVED; + public static final int READONLY_CANTINIT = CApi.SQLITE_READONLY_CANTINIT; + public static final int READONLY_DIRECTORY = CApi.SQLITE_READONLY_DIRECTORY; + public static final int ABORT_ROLLBACK = CApi.SQLITE_ABORT_ROLLBACK; + public static final int CONSTRAINT_CHECK = CApi.SQLITE_CONSTRAINT_CHECK; + public static final int CONSTRAINT_COMMITHOOK = CApi.SQLITE_CONSTRAINT_COMMITHOOK; + public static final int CONSTRAINT_FOREIGNKEY = CApi.SQLITE_CONSTRAINT_FOREIGNKEY; + public static final int CONSTRAINT_FUNCTION = CApi.SQLITE_CONSTRAINT_FUNCTION; + public static final int CONSTRAINT_NOTNULL = CApi.SQLITE_CONSTRAINT_NOTNULL; + public static final int CONSTRAINT_PRIMARYKEY = CApi.SQLITE_CONSTRAINT_PRIMARYKEY; + public static final int CONSTRAINT_TRIGGER = CApi.SQLITE_CONSTRAINT_TRIGGER; + public static final int CONSTRAINT_UNIQUE = CApi.SQLITE_CONSTRAINT_UNIQUE; + public static final int CONSTRAINT_VTAB = CApi.SQLITE_CONSTRAINT_VTAB; + public static final int CONSTRAINT_ROWID = CApi.SQLITE_CONSTRAINT_ROWID; + public static final int CONSTRAINT_PINNED = CApi.SQLITE_CONSTRAINT_PINNED; + public static final int CONSTRAINT_DATATYPE = CApi.SQLITE_CONSTRAINT_DATATYPE; + public static final int NOTICE_RECOVER_WAL = CApi.SQLITE_NOTICE_RECOVER_WAL; + public static final int NOTICE_RECOVER_ROLLBACK = CApi.SQLITE_NOTICE_RECOVER_ROLLBACK; + public static final int WARNING_AUTOINDEX = CApi.SQLITE_WARNING_AUTOINDEX; + public static final int AUTH_USER = CApi.SQLITE_AUTH_USER; + public static final int OK_LOAD_PERMANENTLY = CApi.SQLITE_OK_LOAD_PERMANENTLY; + + // sqlite3_open() flags + public static final int OPEN_READWRITE = CApi.SQLITE_OPEN_READWRITE; + public static final int OPEN_CREATE = CApi.SQLITE_OPEN_CREATE; + public static final int OPEN_EXRESCODE = CApi.SQLITE_OPEN_EXRESCODE; + + // transaction state + public static final int TXN_NONE = CApi.SQLITE_TXN_NONE; + public static final int TXN_READ = CApi.SQLITE_TXN_READ; + public static final int TXN_WRITE = CApi.SQLITE_TXN_WRITE; + + // sqlite3_status() ops + public static final int STATUS_MEMORY_USED = CApi.SQLITE_STATUS_MEMORY_USED; + public static final int STATUS_PAGECACHE_USED = CApi.SQLITE_STATUS_PAGECACHE_USED; + public static final int STATUS_PAGECACHE_OVERFLOW = CApi.SQLITE_STATUS_PAGECACHE_OVERFLOW; + public static final int STATUS_MALLOC_SIZE = CApi.SQLITE_STATUS_MALLOC_SIZE; + public static final int STATUS_PARSER_STACK = CApi.SQLITE_STATUS_PARSER_STACK; + public static final int STATUS_PAGECACHE_SIZE = CApi.SQLITE_STATUS_PAGECACHE_SIZE; + public static final int STATUS_MALLOC_COUNT = CApi.SQLITE_STATUS_MALLOC_COUNT; + + // sqlite3_db_status() ops + public static final int DBSTATUS_LOOKASIDE_USED = CApi.SQLITE_DBSTATUS_LOOKASIDE_USED; + public static final int DBSTATUS_CACHE_USED = CApi.SQLITE_DBSTATUS_CACHE_USED; + public static final int DBSTATUS_SCHEMA_USED = CApi.SQLITE_DBSTATUS_SCHEMA_USED; + public static final int DBSTATUS_STMT_USED = CApi.SQLITE_DBSTATUS_STMT_USED; + public static final int DBSTATUS_LOOKASIDE_HIT = CApi.SQLITE_DBSTATUS_LOOKASIDE_HIT; + public static final int DBSTATUS_LOOKASIDE_MISS_SIZE = CApi.SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE; + public static final int DBSTATUS_LOOKASIDE_MISS_FULL = CApi.SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL; + public static final int DBSTATUS_CACHE_HIT = CApi.SQLITE_DBSTATUS_CACHE_HIT; + public static final int DBSTATUS_CACHE_MISS = CApi.SQLITE_DBSTATUS_CACHE_MISS; + public static final int DBSTATUS_CACHE_WRITE = CApi.SQLITE_DBSTATUS_CACHE_WRITE; + public static final int DBSTATUS_DEFERRED_FKS = CApi.SQLITE_DBSTATUS_DEFERRED_FKS; + public static final int DBSTATUS_CACHE_USED_SHARED = CApi.SQLITE_DBSTATUS_CACHE_USED_SHARED; + public static final int DBSTATUS_CACHE_SPILL = CApi.SQLITE_DBSTATUS_CACHE_SPILL; + + // Limits + public static final int LIMIT_LENGTH = CApi.SQLITE_LIMIT_LENGTH; + public static final int LIMIT_SQL_LENGTH = CApi.SQLITE_LIMIT_SQL_LENGTH; + public static final int LIMIT_COLUMN = CApi.SQLITE_LIMIT_COLUMN; + public static final int LIMIT_EXPR_DEPTH = CApi.SQLITE_LIMIT_EXPR_DEPTH; + public static final int LIMIT_COMPOUND_SELECT = CApi.SQLITE_LIMIT_COMPOUND_SELECT; + public static final int LIMIT_VDBE_OP = CApi.SQLITE_LIMIT_VDBE_OP; + public static final int LIMIT_FUNCTION_ARG = CApi.SQLITE_LIMIT_FUNCTION_ARG; + public static final int LIMIT_ATTACHED = CApi.SQLITE_LIMIT_ATTACHED; + public static final int LIMIT_LIKE_PATTERN_LENGTH = CApi.SQLITE_LIMIT_LIKE_PATTERN_LENGTH; + public static final int LIMIT_VARIABLE_NUMBER = CApi.SQLITE_LIMIT_VARIABLE_NUMBER; + public static final int LIMIT_TRIGGER_DEPTH = CApi.SQLITE_LIMIT_TRIGGER_DEPTH; + public static final int LIMIT_WORKER_THREADS = CApi.SQLITE_LIMIT_WORKER_THREADS; + + // sqlite3_prepare_v3() flags + public static final int PREPARE_PERSISTENT = CApi.SQLITE_PREPARE_PERSISTENT; + public static final int PREPARE_NO_VTAB = CApi.SQLITE_PREPARE_NO_VTAB; + + // sqlite3_trace_v2() flags + public static final int TRACE_STMT = CApi.SQLITE_TRACE_STMT; + public static final int TRACE_PROFILE = CApi.SQLITE_TRACE_PROFILE; + public static final int TRACE_ROW = CApi.SQLITE_TRACE_ROW; + public static final int TRACE_CLOSE = CApi.SQLITE_TRACE_CLOSE; + public static final int TRACE_ALL = TRACE_STMT | TRACE_PROFILE | TRACE_ROW | TRACE_CLOSE; + + // sqlite3_db_config() ops + public static final int DBCONFIG_ENABLE_FKEY = CApi.SQLITE_DBCONFIG_ENABLE_FKEY; + public static final int DBCONFIG_ENABLE_TRIGGER = CApi.SQLITE_DBCONFIG_ENABLE_TRIGGER; + public static final int DBCONFIG_ENABLE_FTS3_TOKENIZER = CApi.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER; + public static final int DBCONFIG_ENABLE_LOAD_EXTENSION = CApi.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION; + public static final int DBCONFIG_NO_CKPT_ON_CLOSE = CApi.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE; + public static final int DBCONFIG_ENABLE_QPSG = CApi.SQLITE_DBCONFIG_ENABLE_QPSG; + public static final int DBCONFIG_TRIGGER_EQP = CApi.SQLITE_DBCONFIG_TRIGGER_EQP; + public static final int DBCONFIG_RESET_DATABASE = CApi.SQLITE_DBCONFIG_RESET_DATABASE; + public static final int DBCONFIG_DEFENSIVE = CApi.SQLITE_DBCONFIG_DEFENSIVE; + public static final int DBCONFIG_WRITABLE_SCHEMA = CApi.SQLITE_DBCONFIG_WRITABLE_SCHEMA; + public static final int DBCONFIG_LEGACY_ALTER_TABLE = CApi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE; + public static final int DBCONFIG_DQS_DML = CApi.SQLITE_DBCONFIG_DQS_DML; + public static final int DBCONFIG_DQS_DDL = CApi.SQLITE_DBCONFIG_DQS_DDL; + public static final int DBCONFIG_ENABLE_VIEW = CApi.SQLITE_DBCONFIG_ENABLE_VIEW; + public static final int DBCONFIG_LEGACY_FILE_FORMAT = CApi.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT; + public static final int DBCONFIG_TRUSTED_SCHEMA = CApi.SQLITE_DBCONFIG_TRUSTED_SCHEMA; + public static final int DBCONFIG_STMT_SCANSTATUS = CApi.SQLITE_DBCONFIG_STMT_SCANSTATUS; + public static final int DBCONFIG_REVERSE_SCANORDER = CApi.SQLITE_DBCONFIG_REVERSE_SCANORDER; + + // sqlite3_config() ops + public static final int CONFIG_SINGLETHREAD = CApi.SQLITE_CONFIG_SINGLETHREAD; + public static final int CONFIG_MULTITHREAD = CApi.SQLITE_CONFIG_MULTITHREAD; + public static final int CONFIG_SERIALIZED = CApi.SQLITE_CONFIG_SERIALIZED; + + // Encodings + public static final int UTF8 = CApi.SQLITE_UTF8; + public static final int UTF16 = CApi.SQLITE_UTF16; + public static final int UTF16LE = CApi.SQLITE_UTF16LE; + public static final int UTF16BE = CApi.SQLITE_UTF16BE; + /* We elide the UTF16_ALIGNED from this interface because it + is irrelevant for the Java interface. */ + + // SQL data type IDs + public static final int INTEGER = CApi.SQLITE_INTEGER; + public static final int FLOAT = CApi.SQLITE_FLOAT; + public static final int TEXT = CApi.SQLITE_TEXT; + public static final int BLOB = CApi.SQLITE_BLOB; + public static final int NULL = CApi.SQLITE_NULL; + + // Authorizer codes. + public static final int DENY = CApi.SQLITE_DENY; + public static final int IGNORE = CApi.SQLITE_IGNORE; + public static final int CREATE_INDEX = CApi.SQLITE_CREATE_INDEX; + public static final int CREATE_TABLE = CApi.SQLITE_CREATE_TABLE; + public static final int CREATE_TEMP_INDEX = CApi.SQLITE_CREATE_TEMP_INDEX; + public static final int CREATE_TEMP_TABLE = CApi.SQLITE_CREATE_TEMP_TABLE; + public static final int CREATE_TEMP_TRIGGER = CApi.SQLITE_CREATE_TEMP_TRIGGER; + public static final int CREATE_TEMP_VIEW = CApi.SQLITE_CREATE_TEMP_VIEW; + public static final int CREATE_TRIGGER = CApi.SQLITE_CREATE_TRIGGER; + public static final int CREATE_VIEW = CApi.SQLITE_CREATE_VIEW; + public static final int DELETE = CApi.SQLITE_DELETE; + public static final int DROP_INDEX = CApi.SQLITE_DROP_INDEX; + public static final int DROP_TABLE = CApi.SQLITE_DROP_TABLE; + public static final int DROP_TEMP_INDEX = CApi.SQLITE_DROP_TEMP_INDEX; + public static final int DROP_TEMP_TABLE = CApi.SQLITE_DROP_TEMP_TABLE; + public static final int DROP_TEMP_TRIGGER = CApi.SQLITE_DROP_TEMP_TRIGGER; + public static final int DROP_TEMP_VIEW = CApi.SQLITE_DROP_TEMP_VIEW; + public static final int DROP_TRIGGER = CApi.SQLITE_DROP_TRIGGER; + public static final int DROP_VIEW = CApi.SQLITE_DROP_VIEW; + public static final int INSERT = CApi.SQLITE_INSERT; + public static final int PRAGMA = CApi.SQLITE_PRAGMA; + public static final int READ = CApi.SQLITE_READ; + public static final int SELECT = CApi.SQLITE_SELECT; + public static final int TRANSACTION = CApi.SQLITE_TRANSACTION; + public static final int UPDATE = CApi.SQLITE_UPDATE; + public static final int ATTACH = CApi.SQLITE_ATTACH; + public static final int DETACH = CApi.SQLITE_DETACH; + public static final int ALTER_TABLE = CApi.SQLITE_ALTER_TABLE; + public static final int REINDEX = CApi.SQLITE_REINDEX; + public static final int ANALYZE = CApi.SQLITE_ANALYZE; + public static final int CREATE_VTABLE = CApi.SQLITE_CREATE_VTABLE; + public static final int DROP_VTABLE = CApi.SQLITE_DROP_VTABLE; + public static final int FUNCTION = CApi.SQLITE_FUNCTION; + public static final int SAVEPOINT = CApi.SQLITE_SAVEPOINT; + public static final int RECURSIVE = CApi.SQLITE_RECURSIVE; + + //! Used only by the open() factory functions. + private Sqlite(sqlite3 db){ + this.db = db; + } + + /** Maps org.sqlite.jni.capi.sqlite3 to Sqlite instances. */ + private static final java.util.Map nativeToWrapper + = new java.util.HashMap<>(); + + + /** + When any given thread is done using the SQLite library, calling + this will free up any native-side resources which may be + associated specifically with that thread. This is not strictly + necessary, in particular in applications which only use SQLite + from a single thread, but may help free some otherwise errant + resources. + + Calling into SQLite from a given thread after this has been + called in that thread is harmless. The library will simply start + to re-cache certain state for that thread. + + Contrariwise, failing to call this will effectively leak a small + amount of cached state for the thread, which may add up to + significant amounts if the application uses SQLite from many + threads. + + This must never be called while actively using SQLite from this + thread, e.g. from within a query loop or a callback which is + operating on behalf of the library. + */ + static void uncacheThread(){ + CApi.sqlite3_java_uncache_thread(); + } + + /** + Returns the Sqlite object associated with the given sqlite3 + object, or null if there is no such mapping. + */ + static Sqlite fromNative(sqlite3 low){ + synchronized(nativeToWrapper){ + return nativeToWrapper.get(low); + } + } + + /** + Returns a newly-opened db connection or throws SqliteException if + opening fails. All arguments are as documented for + sqlite3_open_v2(). + + Design question: do we want static factory functions or should + this be reformulated as a constructor? + */ + public static Sqlite open(String filename, int flags, String vfsName){ + final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); + final int rc = CApi.sqlite3_open_v2(filename, out, flags, vfsName); + final sqlite3 n = out.take(); + if( 0!=rc ){ + if( null==n ) throw new SqliteException(rc); + final SqliteException ex = new SqliteException(n); + n.close(); + throw ex; + } + final Sqlite rv = new Sqlite(n); + synchronized(nativeToWrapper){ + nativeToWrapper.put(n, rv); + } + runAutoExtensions(rv); + return rv; + } + + public static Sqlite open(String filename, int flags){ + return open(filename, flags, null); + } + + public static Sqlite open(String filename){ + return open(filename, OPEN_READWRITE|OPEN_CREATE, null); + } + + public static String libVersion(){ + return CApi.sqlite3_libversion(); + } + + public static int libVersionNumber(){ + return CApi.sqlite3_libversion_number(); + } + + public static String libSourceId(){ + return CApi.sqlite3_sourceid(); + } + + /** + Returns the value of the native library's build-time value of the + SQLITE_THREADSAFE build option. + */ + public static int libThreadsafe(){ + return CApi.sqlite3_threadsafe(); + } + + /** + Analog to sqlite3_compileoption_get(). + */ + public static String compileOptionGet(int n){ + return CApi.sqlite3_compileoption_get(n); + } + + /** + Analog to sqlite3_compileoption_used(). + */ + public static boolean compileOptionUsed(String optName){ + return CApi.sqlite3_compileoption_used(optName); + } + + private static final boolean hasNormalizeSql = + compileOptionUsed("ENABLE_NORMALIZE"); + + private static final boolean hasSqlLog = + compileOptionUsed("ENABLE_SQLLOG"); + + /** + Throws UnsupportedOperationException if check is false. + flag is expected to be the name of an SQLITE_ENABLE_... + build flag. + */ + private static void checkSupported(boolean check, String flag){ + if( !check ){ + throw new UnsupportedOperationException( + "Library was built without "+flag + ); + } + } + + /** + Analog to sqlite3_complete(). + */ + public static boolean isCompleteStatement(String sql){ + switch(CApi.sqlite3_complete(sql)){ + case 0: return false; + case CApi.SQLITE_MISUSE: + throw new IllegalArgumentException("Input may not be null."); + case CApi.SQLITE_NOMEM: + throw new OutOfMemoryError(); + default: + return true; + } + } + + public static int keywordCount(){ + return CApi.sqlite3_keyword_count(); + } + + public static boolean keywordCheck(String word){ + return CApi.sqlite3_keyword_check(word); + } + + public static String keywordName(int index){ + return CApi.sqlite3_keyword_name(index); + } + + public static boolean strglob(String glob, String txt){ + return 0==CApi.sqlite3_strglob(glob, txt); + } + + public static boolean strlike(String glob, String txt, char escChar){ + return 0==CApi.sqlite3_strlike(glob, txt, escChar); + } + + /** + Output object for use with status() and libStatus(). + */ + public static final class Status { + /** The current value for the requested status() or libStatus() metric. */ + long current; + /** The peak value for the requested status() or libStatus() metric. */ + long peak; + } + + /** + As per sqlite3_status64(), but returns its current and high-water + results as a Status object. Throws if the first argument is + not one of the STATUS_... constants. + */ + public static Status libStatus(int op, boolean resetStats){ + org.sqlite.jni.capi.OutputPointer.Int64 pCurrent = + new org.sqlite.jni.capi.OutputPointer.Int64(); + org.sqlite.jni.capi.OutputPointer.Int64 pHighwater = + new org.sqlite.jni.capi.OutputPointer.Int64(); + checkRcStatic( CApi.sqlite3_status64(op, pCurrent, pHighwater, resetStats) ); + final Status s = new Status(); + s.current = pCurrent.value; + s.peak = pHighwater.value; + return s; + } + + /** + As per sqlite3_db_status(), but returns its current and + high-water results as a Status object. Throws if the first + argument is not one of the DBSTATUS_... constants or on any other + misuse. + */ + public Status status(int op, boolean resetStats){ + org.sqlite.jni.capi.OutputPointer.Int32 pCurrent = + new org.sqlite.jni.capi.OutputPointer.Int32(); + org.sqlite.jni.capi.OutputPointer.Int32 pHighwater = + new org.sqlite.jni.capi.OutputPointer.Int32(); + checkRc( CApi.sqlite3_db_status(thisDb(), op, pCurrent, pHighwater, resetStats) ); + final Status s = new Status(); + s.current = pCurrent.value; + s.peak = pHighwater.value; + return s; + } + + @Override public void close(){ + if(null!=this.db){ + synchronized(nativeToWrapper){ + nativeToWrapper.remove(this.db); + } + this.db.close(); + this.db = null; + } + } + + /** + Returns this object's underlying native db handle, or null if + this instance has been closed. This is very specifically not + public. + */ + sqlite3 nativeHandle(){ return this.db; } + + private sqlite3 thisDb(){ + if( null==db || 0==db.getNativePointer() ){ + throw new IllegalArgumentException("This database instance is closed."); + } + return this.db; + } + + // private byte[] stringToUtf8(String s){ + // return s==null ? null : s.getBytes(StandardCharsets.UTF_8); + // } + + /** + If rc!=0, throws an SqliteException. If this db is currently + opened and has non-0 sqlite3_errcode(), the error state is + extracted from it, else only the string form of rc is used. It is + the caller's responsibility to filter out non-error codes such as + SQLITE_ROW and SQLITE_DONE before calling this. + + As a special case, if rc is SQLITE_NOMEM, an OutOfMemoryError is + thrown. + */ + private void checkRc(int rc){ + if( 0!=rc ){ + if( CApi.SQLITE_NOMEM==rc ){ + throw new OutOfMemoryError(); + }else if( null==db || 0==CApi.sqlite3_errcode(db) ){ + throw new SqliteException(rc); + }else{ + throw new SqliteException(db); + } + } + } + + /** + Like checkRc() but behaves as if that function were + called with a null db object. + */ + private static void checkRcStatic(int rc){ + if( 0!=rc ){ + if( CApi.SQLITE_NOMEM==rc ){ + throw new OutOfMemoryError(); + }else{ + throw new SqliteException(rc); + } + } + } + + /** + Toggles the use of extended result codes on or off. By default + they are turned off, but they can be enabled by default by + including the OPEN_EXRESCODE flag when opening a database. + + Because this API reports db-side errors using exceptions, + enabling this may change the values returned by + SqliteException.errcode(). + */ + public void useExtendedResultCodes(boolean on){ + checkRc( CApi.sqlite3_extended_result_codes(thisDb(), on) ); + } + + /** + Analog to sqlite3_prepare_v3(), this prepares the first SQL + statement from the given input string and returns it as a + Stmt. It throws an SqliteException if preparation fails or an + IllegalArgumentException if the input is empty (e.g. contains + only comments or whitespace). + + The first argument must be SQL input in UTF-8 encoding. + + prepFlags must be 0 or a bitmask of the PREPARE_... constants. + + For processing multiple statements from a single input, use + prepareMulti(). + + Design note: though the C-level API succeeds with a null + statement object for empty inputs, that approach is cumbersome to + use in higher-level APIs because every prepared statement has to + be checked for null before using it. + */ + public Stmt prepare(byte utf8Sql[], int prepFlags){ + final OutputPointer.sqlite3_stmt out = new OutputPointer.sqlite3_stmt(); + final int rc = CApi.sqlite3_prepare_v3(thisDb(), utf8Sql, prepFlags, out); + checkRc(rc); + final sqlite3_stmt q = out.take(); + if( null==q ){ + /* The C-level API treats input which is devoid of SQL + statements (e.g. all comments or an empty string) as success + but returns a NULL sqlite3_stmt object. In higher-level APIs, + wrapping a "successful NULL" object that way is tedious to + use because it forces clients and/or wrapper-level code to + check for that unusual case. In practice, higher-level + bindings are generally better-served by treating empty SQL + input as an error. */ + throw new IllegalArgumentException("Input contains no SQL statements."); + } + return new Stmt(this, q); + } + + /** + Equivalent to prepare(X, prepFlags), where X is + sql.getBytes(StandardCharsets.UTF_8). + */ + public Stmt prepare(String sql, int prepFlags){ + return prepare( sql.getBytes(StandardCharsets.UTF_8), prepFlags ); + } + + /** + Equivalent to prepare(sql, 0). + */ + public Stmt prepare(String sql){ + return prepare(sql, 0); + } + + + /** + Callback type for use with prepareMulti(). + */ + public interface PrepareMulti { + /** + Gets passed a Stmt which it may handle in arbitrary ways. + Ownership of st is passed to this function. It must throw on + error. + */ + void call(Sqlite.Stmt st); + } + + /** + A PrepareMulti implementation which calls another PrepareMulti + object and then finalizes its statement. + */ + public static class PrepareMultiFinalize implements PrepareMulti { + private final PrepareMulti pm; + /** + Proxies the given PrepareMulti via this object's call() method. + */ + public PrepareMultiFinalize(PrepareMulti proxy){ + this.pm = proxy; + } + /** + Passes st to the call() method of the object this one proxies, + then finalizes st, propagating any exceptions from call() after + finalizing st. + */ + @Override public void call(Stmt st){ + try{ pm.call(st); } + finally{ st.finalizeStmt(); } + } + } + + /** + Equivalent to prepareMulti(sql,0,visitor). + */ + public void prepareMulti(String sql, PrepareMulti visitor){ + prepareMulti( sql, 0, visitor ); + } + + /** + Equivalent to prepareMulti(X,prepFlags,visitor), where X is + sql.getBytes(StandardCharsets.UTF_8). + */ + public void prepareMulti(String sql, int prepFlags, PrepareMulti visitor){ + prepareMulti(sql.getBytes(StandardCharsets.UTF_8), prepFlags, visitor); + } + + /** + A variant of prepare() which can handle multiple SQL statements + in a single input string. For each statement in the given string, + the statement is passed to visitor.call() a single time, passing + ownership of the statement to that function. This function does + not step() or close() statements - those operations are left to + caller or the visitor function. + + Unlike prepare(), this function does not fail if the input + contains only whitespace or SQL comments. In that case it is up + to the caller to arrange for that to be an error (if desired). + + PrepareMultiFinalize offers a proxy which finalizes each + statement after it is passed to another client-defined visitor. + + Be aware that certain legal SQL constructs may fail in the + preparation phase, before the corresponding statement can be + stepped. Most notably, authorizer checks which disallow access to + something in a statement behave that way. + */ + public void prepareMulti(byte sqlUtf8[], int prepFlags, PrepareMulti visitor){ + int pos = 0, n = 1; + byte[] sqlChunk = sqlUtf8; + final org.sqlite.jni.capi.OutputPointer.sqlite3_stmt outStmt = + new org.sqlite.jni.capi.OutputPointer.sqlite3_stmt(); + final org.sqlite.jni.capi.OutputPointer.Int32 oTail = + new org.sqlite.jni.capi.OutputPointer.Int32(); + while( pos < sqlChunk.length ){ + sqlite3_stmt stmt; + if( pos>0 ){ + sqlChunk = java.util.Arrays.copyOfRange(sqlChunk, pos, sqlChunk.length); + } + if( 0==sqlChunk.length ) break; + checkRc( + CApi.sqlite3_prepare_v3(db, sqlChunk, prepFlags, outStmt, oTail) + ); + pos = oTail.value; + stmt = outStmt.take(); + if( null==stmt ){ + /* empty statement, e.g. only comments or whitespace, was parsed. */ + continue; + } + visitor.call(new Stmt(this, stmt)); + } + } + + public void createFunction(String name, int nArg, int eTextRep, ScalarFunction f){ + int rc = CApi.sqlite3_create_function(thisDb(), name, nArg, eTextRep, + new SqlFunction.ScalarAdapter(f)); + if( 0!=rc ) throw new SqliteException(db); + } + + public void createFunction(String name, int nArg, ScalarFunction f){ + this.createFunction(name, nArg, CApi.SQLITE_UTF8, f); + } + + public void createFunction(String name, int nArg, int eTextRep, AggregateFunction f){ + int rc = CApi.sqlite3_create_function(thisDb(), name, nArg, eTextRep, + new SqlFunction.AggregateAdapter(f)); + if( 0!=rc ) throw new SqliteException(db); + } + + public void createFunction(String name, int nArg, AggregateFunction f){ + this.createFunction(name, nArg, CApi.SQLITE_UTF8, f); + } + + public void createFunction(String name, int nArg, int eTextRep, WindowFunction f){ + int rc = CApi.sqlite3_create_function(thisDb(), name, nArg, eTextRep, + new SqlFunction.WindowAdapter(f)); + if( 0!=rc ) throw new SqliteException(db); + } + + public void createFunction(String name, int nArg, WindowFunction f){ + this.createFunction(name, nArg, CApi.SQLITE_UTF8, f); + } + + public long changes(){ + return CApi.sqlite3_changes64(thisDb()); + } + + public long totalChanges(){ + return CApi.sqlite3_total_changes64(thisDb()); + } + + public long lastInsertRowId(){ + return CApi.sqlite3_last_insert_rowid(thisDb()); + } + + public void setLastInsertRowId(long rowId){ + CApi.sqlite3_set_last_insert_rowid(thisDb(), rowId); + } + + public void interrupt(){ + CApi.sqlite3_interrupt(thisDb()); + } + + public boolean isInterrupted(){ + return CApi.sqlite3_is_interrupted(thisDb()); + } + + public boolean isAutoCommit(){ + return CApi.sqlite3_get_autocommit(thisDb()); + } + + /** + Analog to sqlite3_txn_state(). Returns one of TXN_NONE, TXN_READ, + or TXN_WRITE to denote this database's current transaction state + for the given schema name (or the most restrictive state of any + schema if zSchema is null). + */ + public int transactionState(String zSchema){ + return CApi.sqlite3_txn_state(thisDb(), zSchema); + } + + /** + Analog to sqlite3_db_name(). Returns null if passed an unknown + index. + */ + public String dbName(int dbNdx){ + return CApi.sqlite3_db_name(thisDb(), dbNdx); + } + + /** + Analog to sqlite3_db_filename(). Returns null if passed an + unknown db name. + */ + public String dbFileName(String dbName){ + return CApi.sqlite3_db_filename(thisDb(), dbName); + } + + /** + Analog to sqlite3_db_config() for the call forms which take one + of the boolean-type db configuration flags (namely the + DBCONFIG_... constants defined in this class). On success it + returns the result of that underlying call. Throws on error. + */ + public boolean dbConfig(int op, boolean on){ + org.sqlite.jni.capi.OutputPointer.Int32 pOut = + new org.sqlite.jni.capi.OutputPointer.Int32(); + checkRc( CApi.sqlite3_db_config(thisDb(), op, on ? 1 : 0, pOut) ); + return pOut.get()!=0; + } + + /** + Analog to the variant of sqlite3_db_config() for configuring the + SQLITE_DBCONFIG_MAINDBNAME option. Throws on error. + */ + public void setMainDbName(String name){ + checkRc( + CApi.sqlite3_db_config(thisDb(), CApi.SQLITE_DBCONFIG_MAINDBNAME, + name) + ); + } + + /** + Analog to sqlite3_db_readonly() but throws an SqliteException + with result code SQLITE_NOTFOUND if given an unknown database + name. + */ + public boolean readOnly(String dbName){ + final int rc = CApi.sqlite3_db_readonly(thisDb(), dbName); + if( 0==rc ) return false; + else if( rc>0 ) return true; + throw new SqliteException(CApi.SQLITE_NOTFOUND); + } + + /** + Analog to sqlite3_db_release_memory(). + */ + public void releaseMemory(){ + CApi.sqlite3_db_release_memory(thisDb()); + } + + /** + Analog to sqlite3_release_memory(). + */ + public static int libReleaseMemory(int n){ + return CApi.sqlite3_release_memory(n); + } + + /** + Analog to sqlite3_limit(). limitId must be one of the + LIMIT_... constants. + + Returns the old limit for the given option. If newLimit is + negative, it returns the old limit without modifying the limit. + + If sqlite3_limit() returns a negative value, this function throws + an SqliteException with the SQLITE_RANGE result code but no + further error info (because that case does not qualify as a + db-level error). Such errors may indicate an invalid argument + value or an invalid range for newLimit (the underlying function + does not differentiate between those). + */ + public int limit(int limitId, int newLimit){ + final int rc = CApi.sqlite3_limit(thisDb(), limitId, newLimit); + if( rc<0 ){ + throw new SqliteException(CApi.SQLITE_RANGE); + } + return rc; + } + + /** + Analog to sqlite3_errstr(). + */ + static String errstr(int resultCode){ + return CApi.sqlite3_errstr(resultCode); + } + + /** + A wrapper object for use with tableColumnMetadata(). They are + created and populated only via that interface. + */ + public final class TableColumnMetadata { + Boolean pNotNull = null; + Boolean pPrimaryKey = null; + Boolean pAutoinc = null; + String pzCollSeq = null; + String pzDataType = null; + + private TableColumnMetadata(){} + + public String getDataType(){ return pzDataType; } + public String getCollation(){ return pzCollSeq; } + public boolean isNotNull(){ return pNotNull; } + public boolean isPrimaryKey(){ return pPrimaryKey; } + public boolean isAutoincrement(){ return pAutoinc; } + } + + /** + Returns data about a database, table, and (optionally) column + (which may be null), as per sqlite3_table_column_metadata(). + Throws if passed invalid arguments, else returns the result as a + new TableColumnMetadata object. + */ + TableColumnMetadata tableColumnMetadata( + String zDbName, String zTableName, String zColumnName + ){ + org.sqlite.jni.capi.OutputPointer.String pzDataType + = new org.sqlite.jni.capi.OutputPointer.String(); + org.sqlite.jni.capi.OutputPointer.String pzCollSeq + = new org.sqlite.jni.capi.OutputPointer.String(); + org.sqlite.jni.capi.OutputPointer.Bool pNotNull + = new org.sqlite.jni.capi.OutputPointer.Bool(); + org.sqlite.jni.capi.OutputPointer.Bool pPrimaryKey + = new org.sqlite.jni.capi.OutputPointer.Bool(); + org.sqlite.jni.capi.OutputPointer.Bool pAutoinc + = new org.sqlite.jni.capi.OutputPointer.Bool(); + final int rc = CApi.sqlite3_table_column_metadata( + thisDb(), zDbName, zTableName, zColumnName, + pzDataType, pzCollSeq, pNotNull, pPrimaryKey, pAutoinc + ); + checkRc(rc); + TableColumnMetadata rv = new TableColumnMetadata(); + rv.pzDataType = pzDataType.value; + rv.pzCollSeq = pzCollSeq.value; + rv.pNotNull = pNotNull.value; + rv.pPrimaryKey = pPrimaryKey.value; + rv.pAutoinc = pAutoinc.value; + return rv; + } + + public interface TraceCallback { + /** + Called by sqlite3 for various tracing operations, as per + sqlite3_trace_v2(). Note that this interface elides the 2nd + argument to the native trace callback, as that role is better + filled by instance-local state. + +

    These callbacks may throw, in which case their exceptions are + converted to C-level error information. + +

    The 2nd argument to this function, if non-null, will be a an + Sqlite or Sqlite.Stmt object, depending on the first argument + (see below). + +

    The final argument to this function is the "X" argument + documented for sqlite3_trace() and sqlite3_trace_v2(). Its type + depends on value of the first argument: + +

    - SQLITE_TRACE_STMT: pNative is a Sqlite.Stmt. pX is a String + containing the prepared SQL. + +

    - SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long + holding an approximate number of nanoseconds the statement took + to run. + +

    - SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null. + +

    - SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null. + */ + void call(int traceFlag, Object pNative, Object pX); + } + + /** + Analog to sqlite3_trace_v2(). traceMask must be a mask of the + TRACE_... constants. Pass a null callback to remove tracing. + + Throws on error. + */ + public void trace(int traceMask, TraceCallback callback){ + final Sqlite self = this; + final org.sqlite.jni.capi.TraceV2Callback tc = + (null==callback) ? null : new org.sqlite.jni.capi.TraceV2Callback(){ + @SuppressWarnings("unchecked") + @Override public int call(int flag, Object pNative, Object pX){ + switch(flag){ + case TRACE_ROW: + case TRACE_PROFILE: + case TRACE_STMT: + callback.call(flag, Sqlite.Stmt.fromNative((sqlite3_stmt)pNative), pX); + break; + case TRACE_CLOSE: + callback.call(flag, self, pX); + break; + } + return 0; + } + }; + checkRc( CApi.sqlite3_trace_v2(thisDb(), traceMask, tc) ); + } + + /** + Corresponds to the sqlite3_stmt class. Use Sqlite.prepare() to + create new instances. + */ + public static final class Stmt implements AutoCloseable { + private Sqlite _db; + private sqlite3_stmt stmt; + + /** Only called by the prepare() factory functions. */ + Stmt(Sqlite db, sqlite3_stmt stmt){ + this._db = db; + this.stmt = stmt; + synchronized(nativeToWrapper){ + nativeToWrapper.put(this.stmt, this); + } + } + + sqlite3_stmt nativeHandle(){ + return stmt; + } + + /** Maps org.sqlite.jni.capi.sqlite3_stmt to Stmt instances. */ + private static final java.util.Map nativeToWrapper + = new java.util.HashMap<>(); + + /** + Returns the Stmt object associated with the given sqlite3_stmt + object, or null if there is no such mapping. + */ + static Stmt fromNative(sqlite3_stmt low){ + synchronized(nativeToWrapper){ + return nativeToWrapper.get(low); + } + } + + /** + If this statement is still opened, its low-level handle is + returned, else an IllegalArgumentException is thrown. + */ + private sqlite3_stmt thisStmt(){ + if( null==stmt || 0==stmt.getNativePointer() ){ + throw new IllegalArgumentException("This Stmt has been finalized."); + } + return stmt; + } + + /** Throws if n is out of range of this statement's result column + count. Intended to be used by the columnXyz() methods. */ + private sqlite3_stmt checkColIndex(int n){ + if(n<0 || n>=columnCount()){ + throw new IllegalArgumentException("Column index "+n+" is out of range."); + } + return thisStmt(); + } + + /** + Corresponds to sqlite3_finalize(), but we cannot override the + name finalize() here because this one requires a different + signature. It does not throw on error here because "destructors + do not throw." If it returns non-0, the object is still + finalized, but the result code is an indication that something + went wrong in a prior call into the statement's API, as + documented for sqlite3_finalize(). + */ + public int finalizeStmt(){ + int rc = 0; + if( null!=stmt ){ + synchronized(nativeToWrapper){ + nativeToWrapper.remove(this.stmt); + } + CApi.sqlite3_finalize(stmt); + stmt = null; + _db = null; + } + return rc; + } + + @Override public void close(){ + finalizeStmt(); + } + + /** + Throws if rc is any value other than 0, SQLITE_ROW, or + SQLITE_DONE, else returns rc. Error state for the exception is + extracted from this statement object (if it's opened) or the + string form of rc. + */ + private int checkRc(int rc){ + switch(rc){ + case 0: + case CApi.SQLITE_ROW: + case CApi.SQLITE_DONE: return rc; + default: + if( null==stmt ) throw new SqliteException(rc); + else throw new SqliteException(this); + } + } + + /** + Works like sqlite3_step() but returns true for SQLITE_ROW, + false for SQLITE_DONE, and throws SqliteException for any other + result. + */ + public boolean step(){ + switch(checkRc(CApi.sqlite3_step(thisStmt()))){ + case CApi.SQLITE_ROW: return true; + case CApi.SQLITE_DONE: return false; + default: + throw new IllegalStateException( + "This \"cannot happen\": all possible result codes were checked already." + ); + } + } + + /** + Works like sqlite3_step(), returning the same result codes as + that function unless throwOnError is true, in which case it + will throw an SqliteException for any result codes other than + Sqlite.ROW or Sqlite.DONE. + + The utility of this overload over the no-argument one is the + ability to handle BUSY and LOCKED errors more easily. + */ + public int step(boolean throwOnError){ + final int rc = (null==stmt) + ? Sqlite.MISUSE + : CApi.sqlite3_step(stmt); + return throwOnError ? checkRc(rc) : rc; + } + + /** + Returns the Sqlite which prepared this statement, or null if + this statement has been finalized. + */ + public Sqlite getDb(){ return this._db; } + + /** + Works like sqlite3_reset() but throws on error. + */ + public void reset(){ + checkRc(CApi.sqlite3_reset(thisStmt())); + } + + public boolean isBusy(){ + return CApi.sqlite3_stmt_busy(thisStmt()); + } + + public boolean isReadOnly(){ + return CApi.sqlite3_stmt_readonly(thisStmt()); + } + + public String sql(){ + return CApi.sqlite3_sql(thisStmt()); + } + + public String expandedSql(){ + return CApi.sqlite3_expanded_sql(thisStmt()); + } + + /** + Analog to sqlite3_stmt_explain() but throws if op is invalid. + */ + public void explain(int op){ + checkRc(CApi.sqlite3_stmt_explain(thisStmt(), op)); + } + + /** + Analog to sqlite3_stmt_isexplain(). + */ + public int isExplain(){ + return CApi.sqlite3_stmt_isexplain(thisStmt()); + } + + /** + Analog to sqlite3_normalized_sql(), but throws + UnsupportedOperationException if the library was built without + the SQLITE_ENABLE_NORMALIZE flag. + */ + public String normalizedSql(){ + Sqlite.checkSupported(hasNormalizeSql, "SQLITE_ENABLE_NORMALIZE"); + return CApi.sqlite3_normalized_sql(thisStmt()); + } + + public void clearBindings(){ + CApi.sqlite3_clear_bindings( thisStmt() ); + } + public void bindInt(int ndx, int val){ + checkRc(CApi.sqlite3_bind_int(thisStmt(), ndx, val)); + } + public void bindInt64(int ndx, long val){ + checkRc(CApi.sqlite3_bind_int64(thisStmt(), ndx, val)); + } + public void bindDouble(int ndx, double val){ + checkRc(CApi.sqlite3_bind_double(thisStmt(), ndx, val)); + } + public void bindObject(int ndx, Object o){ + checkRc(CApi.sqlite3_bind_java_object(thisStmt(), ndx, o)); + } + public void bindNull(int ndx){ + checkRc(CApi.sqlite3_bind_null(thisStmt(), ndx)); + } + public int bindParameterCount(){ + return CApi.sqlite3_bind_parameter_count(thisStmt()); + } + public int bindParameterIndex(String paramName){ + return CApi.sqlite3_bind_parameter_index(thisStmt(), paramName); + } + public String bindParameterName(int ndx){ + return CApi.sqlite3_bind_parameter_name(thisStmt(), ndx); + } + public void bindText(int ndx, byte[] utf8){ + checkRc(CApi.sqlite3_bind_text(thisStmt(), ndx, utf8)); + } + public void bindText(int ndx, String asUtf8){ + checkRc(CApi.sqlite3_bind_text(thisStmt(), ndx, asUtf8)); + } + public void bindText16(int ndx, byte[] utf16){ + checkRc(CApi.sqlite3_bind_text16(thisStmt(), ndx, utf16)); + } + public void bindText16(int ndx, String asUtf16){ + checkRc(CApi.sqlite3_bind_text16(thisStmt(), ndx, asUtf16)); + } + public void bindZeroBlob(int ndx, int n){ + checkRc(CApi.sqlite3_bind_zeroblob(thisStmt(), ndx, n)); + } + public void bindBlob(int ndx, byte[] bytes){ + checkRc(CApi.sqlite3_bind_blob(thisStmt(), ndx, bytes)); + } + + public byte[] columnBlob(int ndx){ + return CApi.sqlite3_column_blob( checkColIndex(ndx), ndx ); + } + public byte[] columnText(int ndx){ + return CApi.sqlite3_column_text( checkColIndex(ndx), ndx ); + } + public String columnText16(int ndx){ + return CApi.sqlite3_column_text16( checkColIndex(ndx), ndx ); + } + public int columnBytes(int ndx){ + return CApi.sqlite3_column_bytes( checkColIndex(ndx), ndx ); + } + public int columnBytes16(int ndx){ + return CApi.sqlite3_column_bytes16( checkColIndex(ndx), ndx ); + } + public int columnInt(int ndx){ + return CApi.sqlite3_column_int( checkColIndex(ndx), ndx ); + } + public long columnInt64(int ndx){ + return CApi.sqlite3_column_int64( checkColIndex(ndx), ndx ); + } + public double columnDouble(int ndx){ + return CApi.sqlite3_column_double( checkColIndex(ndx), ndx ); + } + public int columnType(int ndx){ + return CApi.sqlite3_column_type( checkColIndex(ndx), ndx ); + } + public String columnDeclType(int ndx){ + return CApi.sqlite3_column_decltype( checkColIndex(ndx), ndx ); + } + /** + Analog to sqlite3_column_count() but throws if this statement + has been finalized. + */ + public int columnCount(){ + /* We cannot reliably cache the column count in a class + member because an ALTER TABLE from a separate statement + can invalidate that count and we have no way, short of + installing a COMMIT handler or the like, of knowing when + to re-read it. We cannot install such a handler without + interfering with a client's ability to do so. */ + return CApi.sqlite3_column_count(thisStmt()); + } + public int columnDataCount(){ + return CApi.sqlite3_data_count( thisStmt() ); + } + public Object columnObject(int ndx){ + return CApi.sqlite3_column_java_object( checkColIndex(ndx), ndx ); + } + public T columnObject(int ndx, Class type){ + return CApi.sqlite3_column_java_object( checkColIndex(ndx), ndx, type ); + } + public String columnName(int ndx){ + return CApi.sqlite3_column_name( checkColIndex(ndx), ndx ); + } + public String columnDatabaseName(int ndx){ + return CApi.sqlite3_column_database_name( checkColIndex(ndx), ndx ); + } + public String columnOriginName(int ndx){ + return CApi.sqlite3_column_origin_name( checkColIndex(ndx), ndx ); + } + public String columnTableName(int ndx){ + return CApi.sqlite3_column_table_name( checkColIndex(ndx), ndx ); + } + } /* Stmt class */ + + /** + Interface for auto-extensions, as per the + sqlite3_auto_extension() API. + + Design note: the chicken/egg timing of auto-extension execution + requires that this feature be entirely re-implemented in Java + because the C-level API has no access to the Sqlite type so + cannot pass on an object of that type while the database is being + opened. One side effect of this reimplementation is that this + class's list of auto-extensions is 100% independent of the + C-level list so, e.g., clearAutoExtensions() will have no effect + on auto-extensions added via the C-level API and databases opened + from that level of API will not be passed to this level's + AutoExtension instances. + */ + public interface AutoExtension { + public void call(Sqlite db); + } + + private static final java.util.Set autoExtensions = + new java.util.LinkedHashSet<>(); + + /** + Passes db to all auto-extensions. If any one of them throws, + db.close() is called before the exception is propagated. + */ + private static void runAutoExtensions(Sqlite db){ + AutoExtension list[]; + synchronized(autoExtensions){ + /* Avoid that modifications to the AutoExtension list from within + auto-extensions affect this execution of this list. */ + list = autoExtensions.toArray(new AutoExtension[0]); + } + try { + for( AutoExtension ax : list ) ax.call(db); + }catch(Exception e){ + db.close(); + throw e; + } + } + + /** + Analog to sqlite3_auto_extension(), adds the given object to the + list of auto-extensions if it is not already in that list. The + given object will be run as part of Sqlite.open(), and passed the + being-opened database. If the extension throws then open() will + fail. + + This API does not guaranty whether or not manipulations made to + the auto-extension list from within auto-extension callbacks will + affect the current traversal of the auto-extension list. Whether + or not they do is unspecified and subject to change between + versions. e.g. if an AutoExtension calls addAutoExtension(), + whether or not the new extension will be run on the being-opened + database is undefined. + + Note that calling Sqlite.open() from an auto-extension will + necessarily result in recursion loop and (eventually) a stack + overflow. + */ + public static void addAutoExtension( AutoExtension e ){ + if( null==e ){ + throw new IllegalArgumentException("AutoExtension may not be null."); + } + synchronized(autoExtensions){ + autoExtensions.add(e); + } + } + + /** + Removes the given object from the auto-extension list if it is in + that list, otherwise this has no side-effects beyond briefly + locking that list. + */ + public static void removeAutoExtension( AutoExtension e ){ + synchronized(autoExtensions){ + autoExtensions.remove(e); + } + } + + /** + Removes all auto-extensions which were added via addAutoExtension(). + */ + public static void clearAutoExtensions(){ + synchronized(autoExtensions){ + autoExtensions.clear(); + } + } + + /** + Encapsulates state related to the sqlite3 backup API. Use + Sqlite.initBackup() to create new instances. + */ + public static final class Backup implements AutoCloseable { + private sqlite3_backup b; + private Sqlite dbTo; + private Sqlite dbFrom; + + Backup(Sqlite dbDest, String schemaDest,Sqlite dbSrc, String schemaSrc){ + this.dbTo = dbDest; + this.dbFrom = dbSrc; + b = CApi.sqlite3_backup_init(dbDest.nativeHandle(), schemaDest, + dbSrc.nativeHandle(), schemaSrc); + if(null==b) toss(); + } + + private void toss(){ + int rc = CApi.sqlite3_errcode(dbTo.nativeHandle()); + if(0!=rc) throw new SqliteException(dbTo); + rc = CApi.sqlite3_errcode(dbFrom.nativeHandle()); + if(0!=rc) throw new SqliteException(dbFrom); + throw new SqliteException(CApi.SQLITE_ERROR); + } + + private sqlite3_backup getNative(){ + if( null==b ) throw new IllegalStateException("This Backup is already closed."); + return b; + } + /** + If this backup is still active, this completes the backup and + frees its native resources, otherwise it this is a no-op. + */ + public void finish(){ + if( null!=b ){ + CApi.sqlite3_backup_finish(b); + b = null; + dbTo = null; + dbFrom = null; + } + } + + /** Equivalent to finish(). */ + @Override public void close(){ + this.finish(); + } + + /** + Analog to sqlite3_backup_step(). Returns 0 if stepping succeeds + or, Sqlite.DONE if the end is reached, Sqlite.BUSY if one of + the databases is busy, Sqlite.LOCKED if one of the databases is + locked, and throws for any other result code or if this object + has been closed. Note that BUSY and LOCKED are not necessarily + permanent errors, so do not trigger an exception. + */ + public int step(int pageCount){ + final int rc = CApi.sqlite3_backup_step(getNative(), pageCount); + switch(rc){ + case 0: + case Sqlite.DONE: + case Sqlite.BUSY: + case Sqlite.LOCKED: + return rc; + default: + toss(); + return CApi.SQLITE_ERROR/*not reached*/; + } + } + + /** + Analog to sqlite3_backup_pagecount(). + */ + public int pageCount(){ + return CApi.sqlite3_backup_pagecount(getNative()); + } + + /** + Analog to sqlite3_backup_remaining(). + */ + public int remaining(){ + return CApi.sqlite3_backup_remaining(getNative()); + } + } + + /** + Analog to sqlite3_backup_init(). If schemaSrc is null, "main" is + assumed. Throws if either this db or dbSrc (the source db) are + not opened, if either of schemaDest or schemaSrc are null, or if + the underlying call to sqlite3_backup_init() fails. + + The returned object must eventually be cleaned up by either + arranging for it to be auto-closed (e.g. using + try-with-resources) or by calling its finish() method. + */ + public Backup initBackup(String schemaDest, Sqlite dbSrc, String schemaSrc){ + thisDb(); + dbSrc.thisDb(); + if( null==schemaSrc || null==schemaDest ){ + throw new IllegalArgumentException( + "Neither the source nor destination schema name may be null." + ); + } + return new Backup(this, schemaDest, dbSrc, schemaSrc); + } + + + /** + Callback type for use with createCollation(). + */ + public interface Collation { + /** + Called by the SQLite core to compare inputs. Implementations + must compare its two arguments using memcmp(3) semantics. + + Warning: the SQLite core has no mechanism for reporting errors + from custom collations and its workflow does not accommodate + propagation of exceptions from callbacks. Any exceptions thrown + from collations will be silently suppressed and sorting results + will be unpredictable. + */ + int call(byte[] lhs, byte[] rhs); + } + + /** + Analog to sqlite3_create_collation(). + + Throws if name is null or empty, c is null, or the encoding flag + is invalid. The encoding must be one of the UTF8, UTF16, UTF16LE, + or UTF16BE constants. + */ + public void createCollation(String name, int encoding, Collation c){ + thisDb(); + if( null==name || name.isEmpty()){ + throw new IllegalArgumentException("Collation name may not be null or empty."); + } + if( null==c ){ + throw new IllegalArgumentException("Collation may not be null."); + } + switch(encoding){ + case UTF8: + case UTF16: + case UTF16LE: + case UTF16BE: + break; + default: + throw new IllegalArgumentException("Invalid Collation encoding."); + } + checkRc( + CApi.sqlite3_create_collation( + thisDb(), name, encoding, new org.sqlite.jni.capi.CollationCallback(){ + @Override public int call(byte[] lhs, byte[] rhs){ + try{return c.call(lhs, rhs);} + catch(Exception e){return 0;} + } + @Override public void xDestroy(){} + } + ) + ); + } + + /** + Callback for use with onCollationNeeded(). + */ + public interface CollationNeeded { + /** + Must behave as documented for the callback for + sqlite3_collation_needed(). + + Warning: the C API has no mechanism for reporting or + propagating errors from this callback, so any exceptions it + throws are suppressed. + */ + void call(Sqlite db, int encoding, String collationName); + } + + /** + Sets up the given object to be called by the SQLite core when it + encounters a collation name which it does not know. Pass a null + object to disconnect the object from the core. This replaces any + existing collation-needed loader, or is a no-op if the given + object is already registered. Throws if registering the loader + fails. + */ + public void onCollationNeeded( CollationNeeded cn ){ + org.sqlite.jni.capi.CollationNeededCallback cnc = null; + if( null!=cn ){ + cnc = new org.sqlite.jni.capi.CollationNeededCallback(){ + @Override public void call(sqlite3 db, int encoding, String collationName){ + final Sqlite xdb = Sqlite.fromNative(db); + if(null!=xdb) cn.call(xdb, encoding, collationName); + } + }; + } + checkRc( CApi.sqlite3_collation_needed(thisDb(), cnc) ); + } + + /** + Callback for use with busyHandler(). + */ + public interface BusyHandler { + /** + Must function as documented for the C-level + sqlite3_busy_handler() callback argument, minus the (void*) + argument the C-level function requires. + + If this function throws, it is translated to a database-level + error. + */ + int call(int n); + } + + /** + Analog to sqlite3_busy_timeout(). + */ + public void setBusyTimeout(int ms){ + checkRc(CApi.sqlite3_busy_timeout(thisDb(), ms)); + } + + /** + Analog to sqlite3_busy_handler(). If b is null then any + current handler is cleared. + */ + public void setBusyHandler( BusyHandler b ){ + org.sqlite.jni.capi.BusyHandlerCallback bhc = null; + if( null!=b ){ + /*bhc = new org.sqlite.jni.capi.BusyHandlerCallback(){ + @Override public int call(int n){ + return b.call(n); + } + };*/ + bhc = b::call; + } + checkRc( CApi.sqlite3_busy_handler(thisDb(), bhc) ); + } + + public interface CommitHook { + /** + Must behave as documented for the C-level sqlite3_commit_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + int call(); + } + + /** + A level of indirection to permit setCommitHook() to have similar + semantics as the C API, returning the previous hook. The caveat + is that if the low-level API is used to install a hook, it will + have a different hook type than Sqlite.CommitHook so + setCommitHook() will return null instead of that object. + */ + private static class CommitHookProxy + implements org.sqlite.jni.capi.CommitHookCallback { + final CommitHook commitHook; + CommitHookProxy(CommitHook ch){ + this.commitHook = ch; + } + @Override public int call(){ + return commitHook.call(); + } + } + + /** + Analog to sqlite3_commit_hook(). Returns the previous hook, if + any (else null). Throws if this db is closed. + + Minor caveat: if a commit hook is set on this object's underlying + db handle using the lower-level SQLite API, this function may + return null when replacing it, despite there being a hook, + because it will have a different callback type. So long as the + handle is only manipulated via the high-level API, this caveat + does not apply. + */ + public CommitHook setCommitHook( CommitHook c ){ + CommitHookProxy chp = null; + if( null!=c ){ + chp = new CommitHookProxy(c); + } + final org.sqlite.jni.capi.CommitHookCallback rv = + CApi.sqlite3_commit_hook(thisDb(), chp); + return (rv instanceof CommitHookProxy) + ? ((CommitHookProxy)rv).commitHook + : null; + } + + + public interface RollbackHook { + /** + Must behave as documented for the C-level sqlite3_rollback_hook() + callback. If it throws, the exception is translated into + a db-level error. + */ + void call(); + } + + /** + A level of indirection to permit setRollbackHook() to have similar + semantics as the C API, returning the previous hook. The caveat + is that if the low-level API is used to install a hook, it will + have a different hook type than Sqlite.RollbackHook so + setRollbackHook() will return null instead of that object. + */ + private static class RollbackHookProxy + implements org.sqlite.jni.capi.RollbackHookCallback { + final RollbackHook rollbackHook; + RollbackHookProxy(RollbackHook ch){ + this.rollbackHook = ch; + } + @Override public void call(){rollbackHook.call();} + } + + /** + Analog to sqlite3_rollback_hook(). Returns the previous hook, if + any (else null). Throws if this db is closed. + + Minor caveat: if a rollback hook is set on this object's underlying + db handle using the lower-level SQLite API, this function may + return null when replacing it, despite there being a hook, + because it will have a different callback type. So long as the + handle is only manipulated via the high-level API, this caveat + does not apply. + */ + public RollbackHook setRollbackHook( RollbackHook c ){ + RollbackHookProxy chp = null; + if( null!=c ){ + chp = new RollbackHookProxy(c); + } + final org.sqlite.jni.capi.RollbackHookCallback rv = + CApi.sqlite3_rollback_hook(thisDb(), chp); + return (rv instanceof RollbackHookProxy) + ? ((RollbackHookProxy)rv).rollbackHook + : null; + } + + public interface UpdateHook { + /** + Must function as described for the C-level sqlite3_update_hook() + callback. + */ + void call(int opId, String dbName, String tableName, long rowId); + } + + /** + A level of indirection to permit setUpdateHook() to have similar + semantics as the C API, returning the previous hook. The caveat + is that if the low-level API is used to install a hook, it will + have a different hook type than Sqlite.UpdateHook so + setUpdateHook() will return null instead of that object. + */ + private static class UpdateHookProxy + implements org.sqlite.jni.capi.UpdateHookCallback { + final UpdateHook updateHook; + UpdateHookProxy(UpdateHook ch){ + this.updateHook = ch; + } + @Override public void call(int opId, String dbName, String tableName, long rowId){ + updateHook.call(opId, dbName, tableName, rowId); + } + } + + /** + Analog to sqlite3_update_hook(). Returns the previous hook, if + any (else null). Throws if this db is closed. + + Minor caveat: if a update hook is set on this object's underlying + db handle using the lower-level SQLite API, this function may + return null when replacing it, despite there being a hook, + because it will have a different callback type. So long as the + handle is only manipulated via the high-level API, this caveat + does not apply. + */ + public UpdateHook setUpdateHook( UpdateHook c ){ + UpdateHookProxy chp = null; + if( null!=c ){ + chp = new UpdateHookProxy(c); + } + final org.sqlite.jni.capi.UpdateHookCallback rv = + CApi.sqlite3_update_hook(thisDb(), chp); + return (rv instanceof UpdateHookProxy) + ? ((UpdateHookProxy)rv).updateHook + : null; + } + + + /** + Callback interface for use with setProgressHandler(). + */ + public interface ProgressHandler { + /** + Must behave as documented for the C-level sqlite3_progress_handler() + callback. If it throws, the exception is translated into + a db-level error. + */ + int call(); + } + + /** + Analog to sqlite3_progress_handler(), sets the current progress + handler or clears it if p is null. + + Note that this API, in contrast to setUpdateHook(), + setRollbackHook(), and setCommitHook(), cannot return the + previous handler. That inconsistency is part of the lower-level C + API. + */ + public void setProgressHandler( int n, ProgressHandler p ){ + org.sqlite.jni.capi.ProgressHandlerCallback phc = null; + if( null!=p ){ + /*phc = new org.sqlite.jni.capi.ProgressHandlerCallback(){ + @Override public int call(){ return p.call(); } + };*/ + phc = p::call; + } + CApi.sqlite3_progress_handler( thisDb(), n, phc ); + } + + + /** + Callback for use with setAuthorizer(). + */ + public interface Authorizer { + /** + Must function as described for the C-level + sqlite3_set_authorizer() callback. If it throws, the error is + converted to a db-level error and the exception is suppressed. + */ + int call(int opId, String s1, String s2, String s3, String s4); + } + + /** + Analog to sqlite3_set_authorizer(), this sets the current + authorizer callback, or clears if it passed null. + */ + public void setAuthorizer( Authorizer a ) { + org.sqlite.jni.capi.AuthorizerCallback ac = null; + if( null!=a ){ + /*ac = new org.sqlite.jni.capi.AuthorizerCallback(){ + @Override public int call(int opId, String s1, String s2, String s3, String s4){ + return a.call(opId, s1, s2, s3, s4); + } + };*/ + ac = a::call; + } + checkRc( CApi.sqlite3_set_authorizer( thisDb(), ac ) ); + } + + /** + Object type for use with blobOpen() + */ + public final class Blob implements AutoCloseable { + private Sqlite db; + private sqlite3_blob b; + Blob(Sqlite db, sqlite3_blob b){ + this.db = db; + this.b = b; + } + + /** + If this blob is still opened, its low-level handle is + returned, else an IllegalArgumentException is thrown. + */ + private sqlite3_blob thisBlob(){ + if( null==b || 0==b.getNativePointer() ){ + throw new IllegalArgumentException("This Blob has been finalized."); + } + return b; + } + + /** + Analog to sqlite3_blob_close(). + */ + @Override public void close(){ + if( null!=b ){ + CApi.sqlite3_blob_close(b); + b = null; + db = null; + } + } + + /** + Throws if the JVM does not have JNI-level support for + ByteBuffer. + */ + private void checkNio(){ + if( !Sqlite.JNI_SUPPORTS_NIO ){ + throw new UnsupportedOperationException( + "This JVM does not support JNI access to ByteBuffer." + ); + } + } + /** + Analog to sqlite3_blob_reopen() but throws on error. + */ + public void reopen(long newRowId){ + db.checkRc( CApi.sqlite3_blob_reopen(thisBlob(), newRowId) ); + } + + /** + Analog to sqlite3_blob_write() but throws on error. + */ + public void write( byte[] bytes, int atOffset ){ + db.checkRc( CApi.sqlite3_blob_write(thisBlob(), bytes, atOffset) ); + } + + /** + Analog to sqlite3_blob_read() but throws on error. + */ + public void read( byte[] dest, int atOffset ){ + db.checkRc( CApi.sqlite3_blob_read(thisBlob(), dest, atOffset) ); + } + + /** + Analog to sqlite3_blob_bytes(). + */ + public int bytes(){ + return CApi.sqlite3_blob_bytes(thisBlob()); + } + } + + /** + Analog to sqlite3_blob_open(). Returns a Blob object for the + given database, table, column, and rowid. The blob is opened for + read-write mode if writeable is true, else it is read-only. + + The returned object must eventually be freed, before this + database is closed, by either arranging for it to be auto-closed + or calling its close() method. + + Throws on error. + */ + public Blob blobOpen(String dbName, String tableName, String columnName, + long iRow, boolean writeable){ + final OutputPointer.sqlite3_blob out = new OutputPointer.sqlite3_blob(); + checkRc( + CApi.sqlite3_blob_open(thisDb(), dbName, tableName, columnName, + iRow, writeable ? 1 : 0, out) + ); + return new Blob(this, out.take()); + } + + /** + Callback for use with libConfigLog(). + */ + public interface ConfigLog { + /** + Must function as described for a C-level callback for + sqlite3_config()'s SQLITE_CONFIG_LOG callback, with the slight + signature change. Any exceptions thrown from this callback are + necessarily suppressed. + */ + void call(int errCode, String msg); + } + + /** + Analog to sqlite3_config() with the SQLITE_CONFIG_LOG option, + this sets or (if log is null) clears the current logger. + */ + public static void libConfigLog(ConfigLog log){ + final org.sqlite.jni.capi.ConfigLogCallback l = + null==log + ? null + /*: new org.sqlite.jni.capi.ConfigLogCallback() { + @Override public void call(int errCode, String msg){ + log.call(errCode, msg); + } + };*/ + : log::call; + checkRcStatic(CApi.sqlite3_config(l)); + } + + /** + Callback for use with libConfigSqlLog(). + */ + public interface ConfigSqlLog { + /** + Must function as described for a C-level callback for + sqlite3_config()'s SQLITE_CONFIG_SQLLOG callback, with the + slight signature change. Any exceptions thrown from this + callback are necessarily suppressed. + */ + void call(Sqlite db, String msg, int msgType); + } + + /** + Analog to sqlite3_config() with the SQLITE_CONFIG_SQLLOG option, + this sets or (if log is null) clears the current logger. + + If SQLite is built without SQLITE_ENABLE_SQLLOG defined then this + will throw an UnsupportedOperationException. + */ + public static void libConfigSqlLog(ConfigSqlLog log){ + Sqlite.checkSupported(hasNormalizeSql, "SQLITE_ENABLE_SQLLOG"); + final org.sqlite.jni.capi.ConfigSqlLogCallback l = + null==log + ? null + : new org.sqlite.jni.capi.ConfigSqlLogCallback() { + @Override public void call(sqlite3 db, String msg, int msgType){ + try{ + log.call(fromNative(db), msg, msgType); + }catch(Exception e){ + /* Suppressed */ + } + } + }; + checkRcStatic(CApi.sqlite3_config(l)); + } + + /** + Analog to the C-level sqlite3_config() with one of the + SQLITE_CONFIG_... constants defined as CONFIG_... in this + class. Throws on error, including passing of an unknown option or + if a specified option is not supported by the underlying build of + the SQLite library. + */ + public static void libConfigOp( int op ){ + checkRcStatic(CApi.sqlite3_config(op)); + } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java b/ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java new file mode 100644 index 0000000000..9b4440f190 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java @@ -0,0 +1,85 @@ +/* +** 2023-10-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; +import org.sqlite.jni.capi.CApi; +import org.sqlite.jni.capi.sqlite3; + +/** + A wrapper for communicating C-level (sqlite3*) instances with + Java. These wrappers do not own their associated pointer, they + simply provide a type-safe way to communicate it between Java + and C via JNI. +*/ +public final class SqliteException extends java.lang.RuntimeException { + private int errCode = CApi.SQLITE_ERROR; + private int xerrCode = CApi.SQLITE_ERROR; + private int errOffset = -1; + private int sysErrno = 0; + + /** + Records the given error string and uses SQLITE_ERROR for both the + error code and extended error code. + */ + public SqliteException(String msg){ + super(msg); + } + + /** + Uses sqlite3_errstr(sqlite3ResultCode) for the error string and + sets both the error code and extended error code to the given + value. This approach includes no database-level information and + systemErrno() will be 0, so is intended only for use with sqlite3 + APIs for which a result code is not an error but which the + higher-level wrapper should treat as one. + */ + public SqliteException(int sqlite3ResultCode){ + super(CApi.sqlite3_errstr(sqlite3ResultCode)); + errCode = xerrCode = sqlite3ResultCode; + } + + /** + Records the current error state of db (which must not be null and + must refer to an opened db object). Note that this does not close + the db. + + Design note: closing the db on error is really only useful during + a failed db-open operation, and the place(s) where that can + happen are inside this library, not client-level code. + */ + SqliteException(sqlite3 db){ + super(CApi.sqlite3_errmsg(db)); + errCode = CApi.sqlite3_errcode(db); + xerrCode = CApi.sqlite3_extended_errcode(db); + errOffset = CApi.sqlite3_error_offset(db); + sysErrno = CApi.sqlite3_system_errno(db); + } + + /** + Records the current error state of db (which must not be null and + must refer to an open database). + */ + public SqliteException(Sqlite db){ + this(db.nativeHandle()); + } + + public SqliteException(Sqlite.Stmt stmt){ + this(stmt.getDb()); + } + + public int errcode(){ return errCode; } + public int extendedErrcode(){ return xerrCode; } + public int errorOffset(){ return errOffset; } + public int systemErrno(){ return sysErrno; } + +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java b/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java new file mode 100644 index 0000000000..528e1f61c6 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java @@ -0,0 +1,1212 @@ +/* +** 2023-10-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains a set of tests for the sqlite3 JNI bindings. +*/ +package org.sqlite.jni.wrapper1; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.sqlite.jni.capi.CApi; + +/** + An annotation for Tester2 tests which we do not want to run in + reflection-driven test mode because either they are not suitable + for multi-threaded threaded mode or we have to control their execution + order. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface ManualTest{} +/** + Annotation for Tester2 tests which mark those which must be skipped + in multi-threaded mode. +*/ +@java.lang.annotation.Retention(java.lang.annotation.RetentionPolicy.RUNTIME) +@java.lang.annotation.Target({java.lang.annotation.ElementType.METHOD}) +@interface SingleThreadOnly{} + +public class Tester2 implements Runnable { + //! True when running in multi-threaded mode. + private static boolean mtMode = false; + //! True to sleep briefly between tests. + private static boolean takeNaps = false; + //! True to shuffle the order of the tests. + private static boolean shuffle = false; + //! True to dump the list of to-run tests to stdout. + private static int listRunTests = 0; + //! True to squelch all out() and outln() output. + private static boolean quietMode = false; + //! Total number of runTests() calls. + private static int nTestRuns = 0; + //! List of test*() methods to run. + private static List testMethods = null; + //! List of exceptions collected by run() + private static final List listErrors = new ArrayList<>(); + private static final class Metrics { + //! Number of times createNewDb() (or equivalent) is invoked. + volatile int dbOpen = 0; + } + + //! Instance ID. + private final Integer tId; + + Tester2(Integer id){ + tId = id; + } + + static final Metrics metrics = new Metrics(); + + public static synchronized void outln(){ + if( !quietMode ){ + System.out.println(); + } + } + + public static synchronized void outPrefix(){ + if( !quietMode ){ + System.out.print(Thread.currentThread().getName()+": "); + } + } + + public static synchronized void outln(Object val){ + if( !quietMode ){ + outPrefix(); + System.out.println(val); + } + } + + public static synchronized void out(Object val){ + if( !quietMode ){ + System.out.print(val); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void out(Object... vals){ + if( !quietMode ){ + outPrefix(); + for(Object v : vals) out(v); + } + } + + @SuppressWarnings("unchecked") + public static synchronized void outln(Object... vals){ + if( !quietMode ){ + out(vals); out("\n"); + } + } + + static volatile int affirmCount = 0; + public static synchronized int affirm(Boolean v, String comment){ + ++affirmCount; + if( false ) assert( v /* prefer assert over exception if it's enabled because + the JNI layer sometimes has to suppress exceptions, + so they might be squelched on their way back to the + top. */); + if( !v ) throw new RuntimeException(comment); + return affirmCount; + } + + public static void affirm(Boolean v){ + affirm(v, "Affirmation failed."); + } + + + public static void execSql(Sqlite db, String sql[]){ + execSql(db, String.join("", sql)); + } + + /** + Executes all SQL statements in the given string. If throwOnError + is true then it will throw for any prepare/step errors, else it + will return the corresponding non-0 result code. + */ + public static int execSql(Sqlite dbw, boolean throwOnError, String sql){ + final ValueHolder rv = new ValueHolder<>(0); + final Sqlite.PrepareMulti pm = new Sqlite.PrepareMulti(){ + @Override public void call(Sqlite.Stmt stmt){ + try{ + while( Sqlite.ROW == (rv.value = stmt.step(throwOnError)) ){} + } + finally{ stmt.finalizeStmt(); } + } + }; + try { + dbw.prepareMulti(sql, pm); + }catch(SqliteException se){ + if( throwOnError ){ + throw se; + }else{ + /* This error (likely) happened in the prepare() phase and we + need to preempt it. */ + rv.value = se.errcode(); + } + } + return (rv.value==Sqlite.DONE) ? 0 : rv.value; + } + + static void execSql(Sqlite db, String sql){ + execSql(db, true, sql); + } + + @SingleThreadOnly /* because it's thread-agnostic */ + private void test1(){ + affirm(Sqlite.libVersionNumber() == CApi.SQLITE_VERSION_NUMBER); + } + + private void nap() throws InterruptedException { + if( takeNaps ){ + Thread.sleep(java.util.concurrent.ThreadLocalRandom.current().nextInt(3, 17), 0); + } + } + + Sqlite openDb(String name){ + final Sqlite db = Sqlite.open(name, Sqlite.OPEN_READWRITE| + Sqlite.OPEN_CREATE| + Sqlite.OPEN_EXRESCODE); + ++metrics.dbOpen; + return db; + } + + Sqlite openDb(){ return openDb(":memory:"); } + + void testOpenDb1(){ + Sqlite db = openDb(); + affirm( 0!=db.nativeHandle().getNativePointer() ); + affirm( "main".equals( db.dbName(0) ) ); + db.setMainDbName("foo"); + affirm( "foo".equals( db.dbName(0) ) ); + affirm( db.dbConfig(Sqlite.DBCONFIG_DEFENSIVE, true) + /* The underlying function has different mangled names in jdk8 + vs jdk19, and this call is here to ensure that the build + fails if it cannot find both names. */ ); + affirm( !db.dbConfig(Sqlite.DBCONFIG_DEFENSIVE, false) ); + SqliteException ex = null; + try{ db.dbConfig(0, false); } + catch(SqliteException e){ ex = e; } + affirm( null!=ex ); + ex = null; + db.close(); + affirm( null==db.nativeHandle() ); + + try{ db = openDb("/no/such/dir/.../probably"); } + catch(SqliteException e){ ex = e; } + affirm( ex!=null ); + affirm( ex.errcode() != 0 ); + affirm( ex.extendedErrcode() != 0 ); + affirm( ex.errorOffset() < 0 ); + // there's no reliable way to predict what ex.systemErrno() might be + } + + void testPrepare1(){ + try (Sqlite db = openDb()) { + Sqlite.Stmt stmt = db.prepare("SELECT ?1"); + Exception e = null; + affirm( null!=stmt.nativeHandle() ); + affirm( db == stmt.getDb() ); + affirm( 1==stmt.bindParameterCount() ); + affirm( "?1".equals(stmt.bindParameterName(1)) ); + affirm( null==stmt.bindParameterName(2) ); + stmt.bindInt64(1, 1); + stmt.bindDouble(1, 1.1); + stmt.bindObject(1, db); + stmt.bindNull(1); + stmt.bindText(1, new byte[] {32,32,32}); + stmt.bindText(1, "123"); + stmt.bindText16(1, "123".getBytes(StandardCharsets.UTF_16)); + stmt.bindText16(1, "123"); + stmt.bindZeroBlob(1, 8); + stmt.bindBlob(1, new byte[] {1,2,3,4}); + stmt.bindInt(1, 17); + try{ stmt.bindInt(2,1); } + catch(Exception ex){ e = ex; } + affirm( null!=e ); + e = null; + affirm( stmt.step() ); + try{ stmt.columnInt(1); } + catch(Exception ex){ e = ex; } + affirm( null!=e ); + e = null; + affirm( 17 == stmt.columnInt(0) ); + affirm( 17L == stmt.columnInt64(0) ); + affirm( 17.0 == stmt.columnDouble(0) ); + affirm( "17".equals(stmt.columnText16(0)) ); + affirm( !stmt.step() ); + stmt.reset(); + affirm( Sqlite.ROW==stmt.step(false) ); + affirm( !stmt.step() ); + affirm( 0 == stmt.finalizeStmt() ); + affirm( null==stmt.nativeHandle() ); + + stmt = db.prepare("SELECT ?"); + stmt.bindObject(1, db); + affirm( Sqlite.ROW == stmt.step(false) ); + affirm( db==stmt.columnObject(0) ); + affirm( db==stmt.columnObject(0, Sqlite.class ) ); + affirm( null==stmt.columnObject(0, Sqlite.Stmt.class ) ); + affirm( 0==stmt.finalizeStmt() ) + /* getting a non-0 out of sqlite3_finalize() is tricky */; + affirm( null==stmt.nativeHandle() ); + } + } + + void testUdfScalar(){ + final ValueHolder xDestroyCalled = new ValueHolder<>(0); + try (Sqlite db = openDb()) { + execSql(db, "create table t(a); insert into t(a) values(1),(2),(3)"); + final ValueHolder vh = new ValueHolder<>(0); + final ScalarFunction f = new ScalarFunction(){ + public void xFunc(SqlFunction.Arguments args){ + affirm( db == args.getDb() ); + for( SqlFunction.Arguments.Arg arg : args ){ + vh.value += arg.getInt(); + } + args.resultInt(vh.value); + } + public void xDestroy(){ + ++xDestroyCalled.value; + } + }; + db.createFunction("myfunc", -1, f); + Sqlite.Stmt q = db.prepare("select myfunc(1,2,3)"); + affirm( q.step() ); + affirm( 6 == vh.value ); + affirm( 6 == q.columnInt(0) ); + q.finalizeStmt(); + affirm( 0 == xDestroyCalled.value ); + vh.value = 0; + q = db.prepare("select myfunc(-1,-2,-3)"); + affirm( q.step() ); + affirm( -6 == vh.value ); + affirm( -6 == q.columnInt(0) ); + affirm( 0 == xDestroyCalled.value ); + q.finalizeStmt(); + } + affirm( 1 == xDestroyCalled.value ); + } + + void testUdfAggregate(){ + final ValueHolder xDestroyCalled = new ValueHolder<>(0); + Sqlite.Stmt q = null; + try (Sqlite db = openDb()) { + execSql(db, "create table t(a); insert into t(a) values(1),(2),(3)"); + final AggregateFunction f = new AggregateFunction(){ + public void xStep(SqlFunction.Arguments args){ + final ValueHolder agg = this.getAggregateState(args, 0); + for( SqlFunction.Arguments.Arg arg : args ){ + agg.value += arg.getInt(); + } + } + public void xFinal(SqlFunction.Arguments args){ + final Integer v = this.takeAggregateState(args); + if( null==v ) args.resultNull(); + else args.resultInt(v); + } + public void xDestroy(){ + ++xDestroyCalled.value; + } + }; + db.createFunction("summer", 1, f); + q = db.prepare( + "with cte(v) as ("+ + "select 3 union all select 5 union all select 7"+ + ") select summer(v), summer(v+1) from cte" + /* ------------------^^^^^^^^^^^ ensures that we're handling + sqlite3_aggregate_context() properly. */ + ); + affirm( q.step() ); + affirm( 15==q.columnInt(0) ); + q.finalizeStmt(); + q = null; + affirm( 0 == xDestroyCalled.value ); + db.createFunction("summerN", -1, f); + + q = db.prepare("select summerN(1,8,9), summerN(2,3,4)"); + affirm( q.step() ); + affirm( 18==q.columnInt(0) ); + affirm( 9==q.columnInt(1) ); + q.finalizeStmt(); + q = null; + + }/*db*/ + finally{ + if( null!=q ) q.finalizeStmt(); + } + affirm( 2 == xDestroyCalled.value + /* because we've bound the same instance twice */ ); + } + + private void testUdfWindow(){ + final Sqlite db = openDb(); + /* Example window function, table, and results taken from: + https://sqlite.org/windowfunctions.html#udfwinfunc */ + final WindowFunction func = new WindowFunction(){ + //! Impl of xStep() and xInverse() + private void xStepInverse(SqlFunction.Arguments args, int v){ + this.getAggregateState(args,0).value += v; + } + @Override public void xStep(SqlFunction.Arguments args){ + this.xStepInverse(args, args.getInt(0)); + } + @Override public void xInverse(SqlFunction.Arguments args){ + this.xStepInverse(args, -args.getInt(0)); + } + //! Impl of xFinal() and xValue() + private void xFinalValue(SqlFunction.Arguments args, Integer v){ + if(null == v) args.resultNull(); + else args.resultInt(v); + } + @Override public void xFinal(SqlFunction.Arguments args){ + xFinalValue(args, this.takeAggregateState(args)); + affirm( null == this.getAggregateState(args,null).value ); + } + @Override public void xValue(SqlFunction.Arguments args){ + xFinalValue(args, this.getAggregateState(args,null).value); + } + }; + db.createFunction("winsumint", 1, func); + execSql(db, new String[] { + "CREATE TEMP TABLE twin(x, y); INSERT INTO twin VALUES", + "('a', 4),('b', 5),('c', 3),('d', 8),('e', 1)" + }); + final Sqlite.Stmt stmt = db.prepare( + "SELECT x, winsumint(y) OVER ("+ + "ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING"+ + ") AS sum_y "+ + "FROM twin ORDER BY x;" + ); + int n = 0; + while( stmt.step() ){ + final String s = stmt.columnText16(0); + final int i = stmt.columnInt(1); + switch(++n){ + case 1: affirm( "a".equals(s) && 9==i ); break; + case 2: affirm( "b".equals(s) && 12==i ); break; + case 3: affirm( "c".equals(s) && 16==i ); break; + case 4: affirm( "d".equals(s) && 12==i ); break; + case 5: affirm( "e".equals(s) && 9==i ); break; + default: affirm( false /* cannot happen */ ); + } + } + stmt.close(); + affirm( 5 == n ); + db.close(); + } + + private void testKeyword(){ + final int n = Sqlite.keywordCount(); + affirm( n>0 ); + affirm( !Sqlite.keywordCheck("_nope_") ); + affirm( Sqlite.keywordCheck("seLect") ); + affirm( null!=Sqlite.keywordName(0) ); + affirm( null!=Sqlite.keywordName(n-1) ); + affirm( null==Sqlite.keywordName(n) ); + } + + + private void testExplain(){ + final Sqlite db = openDb(); + Sqlite.Stmt q = db.prepare("SELECT 1"); + affirm( 0 == q.isExplain() ); + q.explain(0); + affirm( 0 == q.isExplain() ); + q.explain(1); + affirm( 1 == q.isExplain() ); + q.explain(2); + affirm( 2 == q.isExplain() ); + Exception ex = null; + try{ + q.explain(-1); + }catch(Exception e){ + ex = e; + } + affirm( ex instanceof SqliteException ); + q.finalizeStmt(); + db.close(); + } + + + private void testTrace(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + /* Ensure that characters outside of the UTF BMP survive the trip + from Java to sqlite3 and back to Java. (At no small efficiency + penalty.) */ + final String nonBmpChar = "😃"; + db.trace( + Sqlite.TRACE_ALL, + new Sqlite.TraceCallback(){ + @Override public void call(int traceFlag, Object pNative, Object x){ + ++counter.value; + //outln("TRACE "+traceFlag+" pNative = "+pNative.getClass().getName()); + switch(traceFlag){ + case Sqlite.TRACE_STMT: + affirm(pNative instanceof Sqlite.Stmt); + //outln("TRACE_STMT sql = "+x); + affirm(x instanceof String); + affirm( ((String)x).indexOf(nonBmpChar) > 0 ); + break; + case Sqlite.TRACE_PROFILE: + affirm(pNative instanceof Sqlite.Stmt); + affirm(x instanceof Long); + //outln("TRACE_PROFILE time = "+x); + break; + case Sqlite.TRACE_ROW: + affirm(pNative instanceof Sqlite.Stmt); + affirm(null == x); + //outln("TRACE_ROW = "+sqlite3_column_text16((sqlite3_stmt)pNative, 0)); + break; + case Sqlite.TRACE_CLOSE: + affirm(pNative instanceof Sqlite); + affirm(null == x); + break; + default: + affirm(false /*cannot happen*/); + break; + } + } + }); + execSql(db, "SELECT coalesce(null,null,'"+nonBmpChar+"'); "+ + "SELECT 'w"+nonBmpChar+"orld'"); + affirm( 6 == counter.value ); + db.close(); + affirm( 7 == counter.value ); + } + + private void testStatus(){ + final Sqlite db = openDb(); + execSql(db, "create table t(a); insert into t values(1),(2),(3)"); + + Sqlite.Status s = Sqlite.libStatus(Sqlite.STATUS_MEMORY_USED, false); + affirm( s.current > 0 ); + affirm( s.peak >= s.current ); + + s = db.status(Sqlite.DBSTATUS_SCHEMA_USED, false); + affirm( s.current > 0 ); + affirm( s.peak == 0 /* always 0 for SCHEMA_USED */ ); + + db.close(); + } + + @SingleThreadOnly /* because multiple threads legitimately make these + results unpredictable */ + private synchronized void testAutoExtension(){ + final ValueHolder val = new ValueHolder<>(0); + final ValueHolder toss = new ValueHolder<>(null); + final Sqlite.AutoExtension ax = new Sqlite.AutoExtension(){ + @Override public void call(Sqlite db){ + ++val.value; + if( null!=toss.value ){ + throw new RuntimeException(toss.value); + } + } + }; + Sqlite.addAutoExtension(ax); + openDb().close(); + affirm( 1==val.value ); + openDb().close(); + affirm( 2==val.value ); + Sqlite.clearAutoExtensions(); + openDb().close(); + affirm( 2==val.value ); + + Sqlite.addAutoExtension( ax ); + Sqlite.addAutoExtension( ax ); // Must not add a second entry + Sqlite.addAutoExtension( ax ); // or a third one + openDb().close(); + affirm( 3==val.value ); + + Sqlite db = openDb(); + affirm( 4==val.value ); + execSql(db, "ATTACH ':memory:' as foo"); + affirm( 4==val.value, "ATTACH uses the same connection, not sub-connections." ); + db.close(); + db = null; + + Sqlite.removeAutoExtension(ax); + openDb().close(); + affirm( 4==val.value ); + Sqlite.addAutoExtension(ax); + Exception err = null; + toss.value = "Throwing from auto_extension."; + try{ + openDb(); + }catch(Exception e){ + err = e; + } + affirm( err!=null ); + affirm( err.getMessage().contains(toss.value) ); + toss.value = null; + + val.value = 0; + final Sqlite.AutoExtension ax2 = new Sqlite.AutoExtension(){ + @Override public void call(Sqlite db){ + ++val.value; + } + }; + Sqlite.addAutoExtension(ax2); + openDb().close(); + affirm( 2 == val.value ); + Sqlite.removeAutoExtension(ax); + openDb().close(); + affirm( 3 == val.value ); + Sqlite.addAutoExtension(ax); + openDb().close(); + affirm( 5 == val.value ); + Sqlite.removeAutoExtension(ax2); + openDb().close(); + affirm( 6 == val.value ); + Sqlite.addAutoExtension(ax2); + openDb().close(); + affirm( 8 == val.value ); + + Sqlite.clearAutoExtensions(); + openDb().close(); + affirm( 8 == val.value ); + } + + private void testBackup(){ + final Sqlite dbDest = openDb(); + + try (Sqlite dbSrc = openDb()) { + execSql(dbSrc, new String[]{ + "pragma page_size=512; VACUUM;", + "create table t(a);", + "insert into t(a) values(1),(2),(3);" + }); + Exception e = null; + try { + dbSrc.initBackup("main",dbSrc,"main"); + }catch(Exception x){ + e = x; + } + affirm( e instanceof SqliteException ); + e = null; + try (Sqlite.Backup b = dbDest.initBackup("main",dbSrc,"main")) { + affirm( null!=b ); + int rc; + while( Sqlite.DONE!=(rc = b.step(1)) ){ + affirm( 0==rc ); + } + affirm( b.pageCount() > 0 ); + b.finish(); + } + } + + try (Sqlite.Stmt q = dbDest.prepare("SELECT sum(a) from t")) { + q.step(); + affirm( q.columnInt(0) == 6 ); + } + dbDest.close(); + } + + private void testCollation(){ + final Sqlite db = openDb(); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + final Sqlite.Collation myCollation = new Sqlite.Collation() { + private final String myState = + "this is local state. There is much like it, but this is mine."; + @Override + // Reverse-sorts its inputs... + public int call(byte[] lhs, byte[] rhs){ + int len = lhs.length > rhs.length ? rhs.length : lhs.length; + int c = 0, i = 0; + for(i = 0; i < len; ++i){ + c = lhs[i] - rhs[i]; + if(0 != c) break; + } + if(0==c){ + if(i < lhs.length) c = 1; + else if(i < rhs.length) c = -1; + } + return -c; + } + }; + final Sqlite.CollationNeeded collLoader = new Sqlite.CollationNeeded(){ + @Override + public void call(Sqlite dbArg, int eTextRep, String collationName){ + affirm(dbArg == db); + db.createCollation("reversi", eTextRep, myCollation); + } + }; + db.onCollationNeeded(collLoader); + Sqlite.Stmt stmt = db.prepare("SELECT a FROM t ORDER BY a COLLATE reversi"); + int counter = 0; + while( stmt.step() ){ + final String val = stmt.columnText16(0); + ++counter; + switch(counter){ + case 1: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 3: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + stmt.finalizeStmt(); + stmt = db.prepare("SELECT a FROM t ORDER BY a"); + counter = 0; + while( stmt.step() ){ + final String val = stmt.columnText16(0); + ++counter; + //outln("Non-REVERSI'd row#"+counter+": "+val); + switch(counter){ + case 3: affirm("c".equals(val)); break; + case 2: affirm("b".equals(val)); break; + case 1: affirm("a".equals(val)); break; + } + } + affirm(3 == counter); + stmt.finalizeStmt(); + db.onCollationNeeded(null); + db.close(); + } + + @SingleThreadOnly /* because threads inherently break this test */ + private void testBusy(){ + final String dbName = "_busy-handler.db"; + try{ + Sqlite db1 = openDb(dbName); + ++metrics.dbOpen; + execSql(db1, "CREATE TABLE IF NOT EXISTS t(a)"); + Sqlite db2 = openDb(dbName); + ++metrics.dbOpen; + + final ValueHolder xBusyCalled = new ValueHolder<>(0); + Sqlite.BusyHandler handler = new Sqlite.BusyHandler(){ + @Override public int call(int n){ + return n > 2 ? 0 : ++xBusyCalled.value; + } + }; + db2.setBusyHandler(handler); + + // Force a locked condition... + execSql(db1, "BEGIN EXCLUSIVE"); + int rc = 0; + SqliteException ex = null; + try{ + db2.prepare("SELECT * from t"); + }catch(SqliteException x){ + ex = x; + } + affirm( null!=ex ); + affirm( Sqlite.BUSY == ex.errcode() ); + affirm( 3 == xBusyCalled.value ); + db1.close(); + db2.close(); + }finally{ + try{(new java.io.File(dbName)).delete();} + catch(Exception e){/* ignore */} + } + } + + private void testCommitHook(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder hookResult = new ValueHolder<>(0); + final Sqlite.CommitHook theHook = new Sqlite.CommitHook(){ + @Override public int call(){ + ++counter.value; + return hookResult.value; + } + }; + Sqlite.CommitHook oldHook = db.setCommitHook(theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 2 == counter.value ); + execSql(db, "BEGIN; SELECT 1; SELECT 2; COMMIT;"); + affirm( 2 == counter.value /* NOT invoked if no changes are made */ ); + execSql(db, "BEGIN; update t set a='d' where a='c'; COMMIT;"); + affirm( 3 == counter.value ); + oldHook = db.setCommitHook(theHook); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='e' where a='d'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = db.setCommitHook(null); + affirm( theHook == oldHook ); + execSql(db, "BEGIN; update t set a='f' where a='e'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = db.setCommitHook(null); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='g' where a='f'; COMMIT;"); + affirm( 4 == counter.value ); + + final Sqlite.CommitHook newHook = new Sqlite.CommitHook(){ + @Override public int call(){return 0;} + }; + oldHook = db.setCommitHook(newHook); + affirm( null == oldHook ); + execSql(db, "BEGIN; update t set a='h' where a='g'; COMMIT;"); + affirm( 4 == counter.value ); + oldHook = db.setCommitHook(theHook); + affirm( newHook == oldHook ); + execSql(db, "BEGIN; update t set a='i' where a='h'; COMMIT;"); + affirm( 5 == counter.value ); + hookResult.value = Sqlite.ERROR; + int rc = execSql(db, false, "BEGIN; update t set a='j' where a='i'; COMMIT;"); + affirm( Sqlite.CONSTRAINT_COMMITHOOK == rc ); + affirm( 6 == counter.value ); + db.close(); + } + + private void testRollbackHook(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final Sqlite.RollbackHook theHook = new Sqlite.RollbackHook(){ + @Override public void call(){ + ++counter.value; + } + }; + Sqlite.RollbackHook oldHook = db.setRollbackHook(theHook); + affirm( null == oldHook ); + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 0 == counter.value ); + execSql(db, false, "BEGIN; SELECT 1; SELECT 2; ROLLBACK;"); + affirm( 1 == counter.value /* contra to commit hook, is invoked if no changes are made */ ); + + final Sqlite.RollbackHook newHook = new Sqlite.RollbackHook(){ + @Override public void call(){} + }; + oldHook = db.setRollbackHook(newHook); + affirm( theHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 1 == counter.value ); + oldHook = db.setRollbackHook(theHook); + affirm( newHook == oldHook ); + execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 2 == counter.value ); + int rc = execSql(db, false, "BEGIN; SELECT 1; ROLLBACK;"); + affirm( 0 == rc ); + affirm( 3 == counter.value ); + db.close(); + } + + private void testUpdateHook(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder expectedOp = new ValueHolder<>(0); + final Sqlite.UpdateHook theHook = new Sqlite.UpdateHook(){ + @Override + public void call(int opId, String dbName, String tableName, long rowId){ + ++counter.value; + if( 0!=expectedOp.value ){ + affirm( expectedOp.value == opId ); + } + } + }; + Sqlite.UpdateHook oldHook = db.setUpdateHook(theHook); + affirm( null == oldHook ); + expectedOp.value = Sqlite.INSERT; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + affirm( 3 == counter.value ); + expectedOp.value = Sqlite.UPDATE; + execSql(db, "update t set a='d' where a='c';"); + affirm( 4 == counter.value ); + oldHook = db.setUpdateHook(theHook); + affirm( theHook == oldHook ); + expectedOp.value = Sqlite.DELETE; + execSql(db, "DELETE FROM t where a='d'"); + affirm( 5 == counter.value ); + oldHook = db.setUpdateHook(null); + affirm( theHook == oldHook ); + execSql(db, "update t set a='e' where a='b';"); + affirm( 5 == counter.value ); + oldHook = db.setUpdateHook(null); + affirm( null == oldHook ); + + final Sqlite.UpdateHook newHook = new Sqlite.UpdateHook(){ + @Override public void call(int opId, String dbName, String tableName, long rowId){ + } + }; + oldHook = db.setUpdateHook(newHook); + affirm( null == oldHook ); + execSql(db, "update t set a='h' where a='a'"); + affirm( 5 == counter.value ); + oldHook = db.setUpdateHook(theHook); + affirm( newHook == oldHook ); + expectedOp.value = Sqlite.UPDATE; + execSql(db, "update t set a='i' where a='h'"); + affirm( 6 == counter.value ); + db.close(); + } + + private void testProgress(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + db.setProgressHandler(1, new Sqlite.ProgressHandler(){ + @Override public int call(){ + ++counter.value; + return 0; + } + }); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( counter.value > 0 ); + int nOld = counter.value; + db.setProgressHandler(0, null); + execSql(db, "SELECT 1; SELECT 2;"); + affirm( nOld == counter.value ); + db.close(); + } + + private void testAuthorizer(){ + final Sqlite db = openDb(); + final ValueHolder counter = new ValueHolder<>(0); + final ValueHolder authRc = new ValueHolder<>(0); + final Sqlite.Authorizer auth = new Sqlite.Authorizer(){ + public int call(int op, String s0, String s1, String s2, String s3){ + ++counter.value; + //outln("xAuth(): "+s0+" "+s1+" "+s2+" "+s3); + return authRc.value; + } + }; + execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')"); + db.setAuthorizer(auth); + execSql(db, "UPDATE t SET a=1"); + affirm( 1 == counter.value ); + authRc.value = Sqlite.DENY; + int rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( Sqlite.AUTH==rc ); + db.setAuthorizer(null); + rc = execSql(db, false, "UPDATE t SET a=2"); + affirm( 0==rc ); + db.close(); + } + + private void testBlobOpen(){ + final Sqlite db = openDb(); + + execSql(db, "CREATE TABLE T(a BLOB);" + +"INSERT INTO t(rowid,a) VALUES(1, 'def'),(2, 'XYZ');" + ); + Sqlite.Blob b = db.blobOpen("main", "t", "a", + db.lastInsertRowId(), true); + affirm( 3==b.bytes() ); + b.write(new byte[] {100, 101, 102 /*"DEF"*/}, 0); + b.close(); + Sqlite.Stmt stmt = db.prepare("SELECT length(a), a FROM t ORDER BY a"); + affirm( stmt.step() ); + affirm( 3 == stmt.columnInt(0) ); + affirm( "def".equals(stmt.columnText16(1)) ); + stmt.finalizeStmt(); + + b = db.blobOpen("main", "t", "a", db.lastInsertRowId(), false); + final byte[] tgt = new byte[3]; + b.read( tgt, 0 ); + affirm( 100==tgt[0] && 101==tgt[1] && 102==tgt[2], "DEF" ); + execSql(db,"UPDATE t SET a=zeroblob(10) WHERE rowid=2"); + b.close(); + b = db.blobOpen("main", "t", "a", db.lastInsertRowId(), true); + byte[] bw = new byte[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + }; + b.write(bw, 0); + byte[] br = new byte[10]; + b.read(br, 0); + for( int i = 0; i < br.length; ++i ){ + affirm(bw[i] == br[i]); + } + b.close(); + db.close(); + } + + void testPrepareMulti(){ + final ValueHolder fCount = new ValueHolder<>(0); + final ValueHolder mCount = new ValueHolder<>(0); + try (Sqlite db = openDb()) { + execSql(db, "create table t(a); insert into t(a) values(1),(2),(3)"); + db.createFunction("counter", -1, new ScalarFunction(){ + @Override public void xFunc(SqlFunction.Arguments args){ + ++fCount.value; + args.resultNull(); + } + } + ); + final Sqlite.PrepareMulti pm = new Sqlite.PrepareMultiFinalize( + new Sqlite.PrepareMulti() { + @Override public void call(Sqlite.Stmt q){ + ++mCount.value; + while(q.step()){} + } + } + ); + final String sql = "select counter(*) from t;"+ + "select counter(*) from t; /* comment */"+ + "select counter(*) from t; -- comment\n" + ; + db.prepareMulti(sql, pm); + } + affirm( 3 == mCount.value ); + affirm( 9 == fCount.value ); + } + + + /* Copy/paste/rename this to add new tests. */ + private void _testTemplate(){ + try (Sqlite db = openDb()) { + Sqlite.Stmt stmt = db.prepare("SELECT 1"); + stmt.finalizeStmt(); + } + } + + private void runTests(boolean fromThread) throws Exception { + List mlist = testMethods; + affirm( null!=mlist ); + if( shuffle ){ + mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) ); + java.util.Collections.shuffle(mlist); + } + if( (!fromThread && listRunTests>0) || listRunTests>1 ){ + synchronized(this.getClass()){ + if( !fromThread ){ + out("Initial test"," list: "); + for(java.lang.reflect.Method m : testMethods){ + out(m.getName()+" "); + } + outln(); + outln("(That list excludes some which are hard-coded to run.)"); + } + out("Running"," tests: "); + for(java.lang.reflect.Method m : mlist){ + out(m.getName()+" "); + } + outln(); + } + } + for(java.lang.reflect.Method m : mlist){ + nap(); + try{ + m.invoke(this); + }catch(java.lang.reflect.InvocationTargetException e){ + outln("FAILURE: ",m.getName(),"(): ", e.getCause()); + throw e; + } + } + synchronized( this.getClass() ){ + ++nTestRuns; + } + } + + public void run() { + try { + runTests(0!=this.tId); + }catch(Exception e){ + synchronized( listErrors ){ + listErrors.add(e); + } + }finally{ + Sqlite.uncacheThread(); + } + } + + /** + Runs the basic sqlite3 JNI binding sanity-check suite. + + CLI flags: + + -q|-quiet: disables most test output. + + -t|-thread N: runs the tests in N threads + concurrently. Default=1. + + -r|-repeat N: repeats the tests in a loop N times, each one + consisting of the -thread value's threads. + + -shuffle: randomizes the order of most of the test functions. + + -naps: sleep small random intervals between tests in order to add + some chaos for cross-thread contention. + + -list-tests: outputs the list of tests being run, minus some + which are hard-coded. In multi-threaded mode, use this twice to + to emit the list run by each thread (which may differ from the initial + list, in particular if -shuffle is used). + + -fail: forces an exception to be thrown during the test run. Use + with -shuffle to make its appearance unpredictable. + + -v: emit some developer-mode info at the end. + */ + public static void main(String[] args) throws Exception { + int nThread = 1; + int nRepeat = 1; + boolean doSomethingForDev = false; + boolean forceFail = false; + boolean sqlLog = false; + boolean configLog = false; + boolean squelchTestOutput = false; + for( int i = 0; i < args.length; ){ + String arg = args[i++]; + if(arg.startsWith("-")){ + arg = arg.replaceFirst("-+",""); + if(arg.equals("v")){ + doSomethingForDev = true; + //listBoundMethods(); + }else if(arg.equals("t") || arg.equals("thread")){ + nThread = Integer.parseInt(args[i++]); + }else if(arg.equals("r") || arg.equals("repeat")){ + nRepeat = Integer.parseInt(args[i++]); + }else if(arg.equals("shuffle")){ + shuffle = true; + }else if(arg.equals("list-tests")){ + ++listRunTests; + }else if(arg.equals("fail")){ + forceFail = true; + }else if(arg.equals("sqllog")){ + sqlLog = true; + }else if(arg.equals("configlog")){ + configLog = true; + }else if(arg.equals("naps")){ + takeNaps = true; + }else if(arg.equals("q") || arg.equals("quiet")){ + squelchTestOutput = true; + }else{ + throw new IllegalArgumentException("Unhandled flag:"+arg); + } + } + } + + if( sqlLog ){ + if( Sqlite.compileOptionUsed("ENABLE_SQLLOG") ){ + Sqlite.libConfigSqlLog( new Sqlite.ConfigSqlLog() { + @Override public void call(Sqlite db, String msg, int op){ + switch(op){ + case 0: outln("Opening db: ",db); break; + case 1: outln("SQL ",db,": ",msg); break; + case 2: outln("Closing db: ",db); break; + } + } + } + ); + }else{ + outln("WARNING: -sqllog is not active because library was built ", + "without SQLITE_ENABLE_SQLLOG."); + } + } + if( configLog ){ + Sqlite.libConfigLog( new Sqlite.ConfigLog() { + @Override public void call(int code, String msg){ + outln("ConfigLog: ",Sqlite.errstr(code),": ", msg); + } + } + ); + } + + quietMode = squelchTestOutput; + outln("If you just saw warning messages regarding CallStaticObjectMethod, ", + "you are very likely seeing the side effects of a known openjdk8 ", + "bug. It is unsightly but does not affect the library."); + + { + // Build list of tests to run from the methods named test*(). + testMethods = new ArrayList<>(); + int nSkipped = 0; + for(final java.lang.reflect.Method m : Tester2.class.getDeclaredMethods()){ + final String name = m.getName(); + if( name.equals("testFail") ){ + if( forceFail ){ + testMethods.add(m); + } + }else if( !m.isAnnotationPresent( ManualTest.class ) ){ + if( nThread>1 && m.isAnnotationPresent( SingleThreadOnly.class ) ){ + if( 0==nSkipped++ ){ + out("Skipping tests in multi-thread mode:"); + } + out(" "+name+"()"); + }else if( name.startsWith("test") ){ + testMethods.add(m); + } + } + } + if( nSkipped>0 ) out("\n"); + } + + final long timeStart = System.currentTimeMillis(); + outln("libversion_number: ", + Sqlite.libVersionNumber(),"\n", + Sqlite.libVersion(),"\n",Sqlite.libSourceId(),"\n", + "SQLITE_THREADSAFE=",CApi.sqlite3_threadsafe()); + final boolean showLoopCount = (nRepeat>1 && nThread>1); + if( showLoopCount ){ + outln("Running ",nRepeat," loop(s) with ",nThread," thread(s) each."); + } + if( takeNaps ) outln("Napping between tests is enabled."); + int nLoop = 0; + for( int n = 0; n < nRepeat; ++n ){ + ++nLoop; + if( showLoopCount ) out((1==nLoop ? "" : " ")+nLoop); + if( nThread<=1 ){ + new Tester2(0).runTests(false); + continue; + } + Tester2.mtMode = true; + final ExecutorService ex = Executors.newFixedThreadPool( nThread ); + for( int i = 0; i < nThread; ++i ){ + ex.submit( new Tester2(i), i ); + } + ex.shutdown(); + try{ + ex.awaitTermination(nThread*200, java.util.concurrent.TimeUnit.MILLISECONDS); + ex.shutdownNow(); + }catch (InterruptedException ie){ + ex.shutdownNow(); + Thread.currentThread().interrupt(); + } + if( !listErrors.isEmpty() ){ + quietMode = false; + outln("TEST ERRORS:"); + Exception err = null; + for( Exception e : listErrors ){ + e.printStackTrace(); + if( null==err ) err = e; + } + if( null!=err ) throw err; + } + } + if( showLoopCount ) outln(); + quietMode = false; + + final long timeEnd = System.currentTimeMillis(); + outln("Tests done. Metrics across ",nTestRuns," total iteration(s):"); + outln("\tAssertions checked: ",affirmCount); + outln("\tDatabases opened: ",metrics.dbOpen); + if( doSomethingForDev ){ + CApi.sqlite3_jni_internal_details(); + } + affirm( 0==Sqlite.libReleaseMemory(1) ); + CApi.sqlite3_shutdown(); + int nMethods = 0; + int nNatives = 0; + int nCanonical = 0; + final java.lang.reflect.Method[] declaredMethods = + CApi.class.getDeclaredMethods(); + for(java.lang.reflect.Method m : declaredMethods){ + final int mod = m.getModifiers(); + if( 0!=(mod & java.lang.reflect.Modifier.STATIC) ){ + final String name = m.getName(); + if(name.startsWith("sqlite3_")){ + ++nMethods; + if( 0!=(mod & java.lang.reflect.Modifier.NATIVE) ){ + ++nNatives; + } + } + } + } + outln("\tCApi.sqlite3_*() methods: "+ + nMethods+" total, with "+ + nNatives+" native, "+ + (nMethods - nNatives)+" Java" + ); + outln("\tTotal test time = " + +(timeEnd - timeStart)+"ms"); + } +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java b/ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java new file mode 100644 index 0000000000..7549bb97b2 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java @@ -0,0 +1,25 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the ValueHolder utility class. +*/ +package org.sqlite.jni.wrapper1; + +/** + A helper class which simply holds a single value. Its primary use + is for communicating values out of anonymous callbacks, as doing so + requires a "final" reference. +*/ +public class ValueHolder { + public T value; + public ValueHolder(){} + public ValueHolder(T v){value = v;} +} diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java new file mode 100644 index 0000000000..a3905567d4 --- /dev/null +++ b/ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java @@ -0,0 +1,42 @@ +/* +** 2023-10-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file is part of the wrapper1 interface for sqlite3. +*/ +package org.sqlite.jni.wrapper1; + +/** + A SqlFunction implementation for window functions. The T type + represents the type of data accumulated by this function while it + works. e.g. a SUM()-like UDF might use Integer or Long and a + CONCAT()-like UDF might use a StringBuilder or a List. +*/ +public abstract class WindowFunction extends AggregateFunction { + + /** + As for the xInverse() argument of the C API's + sqlite3_create_window_function(). If this function throws, the + exception is reported via sqlite3_result_error(). + */ + public abstract void xInverse(SqlFunction.Arguments args); + + /** + As for the xValue() argument of the C API's + sqlite3_create_window_function(). If this function throws, it is + translated into sqlite3_result_error(). + + Note that the passed-in object will not actually contain any + arguments for xValue() but will contain the context object needed + for setting the call's result or error state. + */ + public abstract void xValue(SqlFunction.Arguments args); + +} diff --git a/ext/jni/src/tests/000-000-sanity.test b/ext/jni/src/tests/000-000-sanity.test new file mode 100644 index 0000000000..4ccbece31c --- /dev/null +++ b/ext/jni/src/tests/000-000-sanity.test @@ -0,0 +1,53 @@ +/* +** This is a comment. There are many like it but this one is mine. +** +** SCRIPT_MODULE_NAME: sanity-check +** xMIXED_MODULE_NAME: mixed-module +** xMODULE_NAME: module-name +** xREQUIRED_PROPERTIES: small fast reliable +** xREQUIRED_PROPERTIES: RECURSIVE_TRIGGERS +** xREQUIRED_PROPERTIES: TEMPSTORE_FILE TEMPSTORE_MEM +** xREQUIRED_PROPERTIES: AUTOVACUUM INCRVACUUM +** +*/ +--print starting up 😃 +--close all +--oom +--db 0 +--new my.db +--null zilch +--testcase 1.0 +SELECT 1, null; +--result 1 zilch +--glob *zil* +--notglob *ZIL* +SELECT 1, 2; +intentional error; +--run +--testcase json-1 +SELECT json_array(1,2,3) +--json [1,2,3] +--testcase tableresult-1 + select 1, 'a'; + select 2, 'b'; +--tableresult + # [a-z] + 2 b +--end +--testcase json-block-1 + select json_array(1,2,3); + select json_object('a',1,'b',2); +--json-block + [1,2,3] + {"a":1,"b":2} +--end +--testcase col-names-on +--column-names 1 + select 1 as 'a', 2 as 'b'; +--result a 1 b 2 +--testcase col-names-off +--column-names 0 + select 1 as 'a', 2 as 'b'; +--result 1 2 +--close +--print reached the end 😃 diff --git a/ext/jni/src/tests/000-001-ignored.test b/ext/jni/src/tests/000-001-ignored.test new file mode 100644 index 0000000000..5af852e197 --- /dev/null +++ b/ext/jni/src/tests/000-001-ignored.test @@ -0,0 +1,9 @@ +/* +** This script must be marked as ignored because it contains +** content which triggers that condition. +** +** SCRIPT_MODULE_NAME: ignored +** +*/ + +| diff --git a/ext/jni/src/tests/900-001-fts.test b/ext/jni/src/tests/900-001-fts.test new file mode 100644 index 0000000000..65285e86b0 --- /dev/null +++ b/ext/jni/src/tests/900-001-fts.test @@ -0,0 +1,12 @@ +/* +** SCRIPT_MODULE_NAME: fts5-sanity-checks +** xREQUIRED_PROPERTIES: FTS5 +** +*/ + +--testcase 1.0 +CREATE VIRTUAL TABLE email USING fts5(sender, title, body); +insert into email values('fred','Help!','Dear Sir...'); +insert into email values('barney','Assistance','Dear Madam...'); +select * from email where email match 'assistance'; +--result barney Assistance {Dear Madam...} diff --git a/ext/lsm1/Makefile b/ext/lsm1/Makefile deleted file mode 100644 index 7056432d2d..0000000000 --- a/ext/lsm1/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# -# This Makefile is designed for use with main.mk in the root directory of -# this project. After including main.mk, the users makefile should contain: -# -# LSMDIR=$(TOP)/ext/lsm1/ -# LSMOPTS=-fPIC -# include $(LSMDIR)/Makefile -# -# The most useful targets are [lsmtest] and [lsm.so]. -# - -LSMOBJ = \ - lsm_ckpt.o \ - lsm_file.o \ - lsm_log.o \ - lsm_main.o \ - lsm_mem.o \ - lsm_mutex.o \ - lsm_shared.o \ - lsm_sorted.o \ - lsm_str.o \ - lsm_tree.o \ - lsm_unix.o \ - lsm_win32.o \ - lsm_varint.o \ - lsm_vtab.o - -LSMHDR = \ - $(LSMDIR)/lsm.h \ - $(LSMDIR)/lsmInt.h - -LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \ - $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c \ - $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c \ - $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c \ - $(LSMDIR)/lsm-test/lsmtest9.c \ - $(LSMDIR)/lsm-test/lsmtest_datasource.c \ - $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c \ - $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \ - $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \ - $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c - - -# all: lsm.so - -LSMOPTS += -fPIC -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB - -lsm.so: $(LSMOBJ) - $(TCCX) -shared -fPIC -o lsm.so $(LSMOBJ) - -%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h - $(TCCX) $(LSMOPTS) -c $< - -lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o - # $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc - $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz diff --git a/ext/lsm1/Makefile.msc b/ext/lsm1/Makefile.msc deleted file mode 100644 index 3e5a3b3310..0000000000 --- a/ext/lsm1/Makefile.msc +++ /dev/null @@ -1,102 +0,0 @@ -# -# This Makefile is designed for use with Makefile.msc in the root directory -# of this project. The Makefile.msc should contain: -# -# LSMDIR=$(TOP)\ext\lsm1 -# !INCLUDE $(LSMDIR)\Makefile.msc -# -# The most useful targets are [lsmtest.exe] and [lsm.dll]. -# - -LSMOBJ = \ - lsm_ckpt.lo \ - lsm_file.lo \ - lsm_log.lo \ - lsm_main.lo \ - lsm_mem.lo \ - lsm_mutex.lo \ - lsm_shared.lo \ - lsm_sorted.lo \ - lsm_str.lo \ - lsm_tree.lo \ - lsm_unix.lo \ - lsm_win32.lo \ - lsm_varint.lo \ - lsm_vtab.lo - -LSMHDR = \ - $(LSMDIR)\lsm.h \ - $(LSMDIR)\lsmInt.h - -LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c \ - $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c \ - $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c \ - $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c \ - $(LSMDIR)\lsm-test\lsmtest9.c \ - $(LSMDIR)\lsm-test\lsmtest_datasource.c \ - $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c \ - $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \ - $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \ - $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c - -# all: lsm.dll lsmtest.exe - -LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR) - -!IF $(DEBUG)>2 -LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1 -!ENDIF - -!IF $(MEMDEBUG)!=0 -LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1 -!ENDIF - -lsm_ckpt.lo: $(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c - -lsm_file.lo: $(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c - -lsm_log.lo: $(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c - -lsm_main.lo: $(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c - -lsm_mem.lo: $(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c - -lsm_mutex.lo: $(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c - -lsm_shared.lo: $(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c - -lsm_sorted.lo: $(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c - -lsm_str.lo: $(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c - -lsm_tree.lo: $(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c - -lsm_unix.lo: $(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c - -lsm_win32.lo: $(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c - -lsm_varint.lo: $(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c - -lsm_vtab.lo: $(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c - -lsm.dll: $(LSMOBJ) - $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ) - copy /Y $@ $(LSMDIR)\$@ - -lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ) - $(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ) - copy /Y $@ $(LSMDIR)\$@ diff --git a/ext/lsm1/lsm-test/README b/ext/lsm1/lsm-test/README deleted file mode 100644 index 80654ee97e..0000000000 --- a/ext/lsm1/lsm-test/README +++ /dev/null @@ -1,40 +0,0 @@ - - -Organization of test case files: - - lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a - database file, then verify that the contents of the database can - be queried. - - lsmtest2.c: Crash tests. Tests that attempt to verify that the database - recovers correctly following an application or system crash. - - lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of - transactions and sub-transactions. - - lsmtest4.c: Multi-client tests. - - lsmtest5.c: Multi-client tests with a different thread for each client. - - lsmtest6.c: OOM injection tests. - - lsmtest7.c: API tests. - - lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that - the system recovers and other clients proceed unaffected if - a process fails in the middle of a write transaction. - - The difference from lsmtest2.c is that this file tests - live-recovery (recovery from a failure that occurs while other - clients are still running) whereas lsmtest2.c tests recovery - from a system or power failure. - - lsmtest9.c: More data tests. These focus on testing that calling - lsm_work(nMerge=1) to compact the database does not corrupt it. - In other words, that databases containing block-redirects - can be read and written. - - - - - diff --git a/ext/lsm1/lsm-test/lsmtest.h b/ext/lsm1/lsm-test/lsmtest.h deleted file mode 100644 index ca60424add..0000000000 --- a/ext/lsm1/lsm-test/lsmtest.h +++ /dev/null @@ -1,303 +0,0 @@ - -#ifndef __WRAPPER_INT_H_ -#define __WRAPPER_INT_H_ - -#include "lsmtest_tdb.h" -#include "sqlite3.h" -#include "lsm.h" - -#include -#include -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef _WIN32 -# include "windows.h" -# define gettimeofday win32GetTimeOfDay -# define F_OK (0) -# define sleep(sec) Sleep(1000 * (sec)) -# define usleep(usec) Sleep(((usec) + 999) / 1000) -# ifdef _MSC_VER -# include -# define snprintf _snprintf -# define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) -# define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) -# define __va_copy(dst,src) ((dst) = (src)) -# define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1) -# else -# error Unsupported C compiler for Windows. -# endif -int win32GetTimeOfDay(struct timeval *, void *); -#endif - -#ifndef _LSM_INT_H -typedef unsigned int u32; -typedef unsigned char u8; -typedef long long int i64; -typedef unsigned long long int u64; -#endif - - -#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) - -#define MIN(x,y) ((x)<(y) ? (x) : (y)) -#define MAX(x,y) ((x)>(y) ? (x) : (y)) - -#define unused_parameter(x) (void)(x) - -#define TESTDB_DEFAULT_PAGE_SIZE 4096 -#define TESTDB_DEFAULT_CACHE_SIZE 2048 - -#ifndef _O_BINARY -# define _O_BINARY (0) -#endif - -/* -** Ideally, these should be in wrapper.c. But they are here instead so that -** they can be used by the C++ database wrappers in wrapper2.cc. -*/ -typedef struct DatabaseMethods DatabaseMethods; -struct TestDb { - DatabaseMethods const *pMethods; /* Database methods */ - const char *zLibrary; /* Library name for tdb_open() */ -}; -struct DatabaseMethods { - int (*xClose)(TestDb *); - int (*xWrite)(TestDb *, void *, int , void *, int); - int (*xDelete)(TestDb *, void *, int); - int (*xDeleteRange)(TestDb *, void *, int, void *, int); - int (*xFetch)(TestDb *, void *, int, void **, int *); - int (*xScan)(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) - ); - int (*xBegin)(TestDb *, int); - int (*xCommit)(TestDb *, int); - int (*xRollback)(TestDb *, int); -}; - -/* -** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the -** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but -** the primary interface is the C++ API. -*/ -int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb); -int test_kc_close(TestDb *); -int test_kc_write(TestDb *, void *, int , void *, int); -int test_kc_delete(TestDb *, void *, int); -int test_kc_delete_range(TestDb *, void *, int, void *, int); -int test_kc_fetch(TestDb *, void *, int, void **, int *); -int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - -int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_mdb_close(TestDb *); -int test_mdb_write(TestDb *, void *, int , void *, int); -int test_mdb_delete(TestDb *, void *, int); -int test_mdb_fetch(TestDb *, void *, int, void **, int *); -int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - -/* -** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. -** The wrapper for lsm is a bit more involved than the others, as it -** includes code for a couple of different lsm configurations, and for -** various types of fault injection and robustness testing. -*/ -int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb); - -int tdb_lsm_configure(lsm_db *, const char *); - -/* Functions in lsmtest_tdb4.c */ -int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb); - - -/* Functions in testutil.c. */ -int testPrngInit(void); -u32 testPrngValue(u32 iVal); -void testPrngArray(u32 iVal, u32 *aOut, int nOut); -void testPrngString(u32 iVal, char *aOut, int nOut); - -void testErrorInit(int argc, char **); -void testPrintError(const char *zFormat, ...); -void testPrintUsage(const char *zArgs); -void testPrintFUsage(const char *zFormat, ...); -void testTimeInit(void); -int testTimeGet(void); - -/* Functions in testmem.c. */ -void testMallocInstall(lsm_env *pEnv); -void testMallocUninstall(lsm_env *pEnv); -void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *); -void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *); -void testMallocOomEnable(lsm_env *pEnv, int); - -/* lsmtest.c */ -TestDb *testOpen(const char *zSystem, int, int *pRc); -void testReopen(TestDb **ppDb, int *pRc); -void testClose(TestDb **ppDb); - -void testFetch(TestDb *, void *, int, void *, int, int *); -void testWrite(TestDb *, void *, int, void *, int, int *); -void testDelete(TestDb *, void *, int, int *); -void testDeleteRange(TestDb *, void *, int, void *, int, int *); -void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc); -void testFetchStr(TestDb *, const char *, const char *, int *pRc); - -void testBegin(TestDb *pDb, int iTrans, int *pRc); -void testCommit(TestDb *pDb, int iTrans, int *pRc); - -void test_failed(void); - -char *testMallocPrintf(const char *zFormat, ...); -char *testMallocVPrintf(const char *zFormat, va_list ap); -int testGlobMatch(const char *zPattern, const char *zStr); - -void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *); -void testFetchCompare(TestDb *, TestDb *, void *, int, int *); - -void *testMalloc(int); -void *testMallocCopy(void *pCopy, int nByte); -void *testRealloc(void *, int); -void testFree(void *); - -/* lsmtest_bt.c */ -int do_bt(int nArg, char **azArg); - -/* testio.c */ -int testVfsConfigureDb(TestDb *pDb); - -/* testfunc.c */ -int do_show(int nArg, char **azArg); -int do_work(int nArg, char **azArg); - -/* testio.c */ -int do_io(int nArg, char **azArg); - -/* lsmtest2.c */ -void do_crash_test(const char *zPattern, int *pRc); -int do_rollback_test(int nArg, char **azArg); - -/* test3.c */ -void test_rollback(const char *zSystem, const char *zPattern, int *pRc); - -/* test4.c */ -void test_mc(const char *zSystem, const char *zPattern, int *pRc); - -/* test5.c */ -void test_mt(const char *zSystem, const char *zPattern, int *pRc); - -/* lsmtest6.c */ -void test_oom(const char *zPattern, int *pRc); -void testDeleteLsmdb(const char *zFile); - -void testSaveDb(const char *zFile, const char *zAuxExt); -void testRestoreDb(const char *zFile, const char *zAuxExt); -void testCopyLsmdb(const char *zFrom, const char *zTo); - -/* lsmtest7.c */ -void test_api(const char *zPattern, int *pRc); - -/* lsmtest8.c */ -void do_writer_crash_test(const char *zPattern, int *pRc); - -/************************************************************************* -** Interface to functionality in test_datasource.c. -*/ -typedef struct Datasource Datasource; -typedef struct DatasourceDefn DatasourceDefn; - -struct DatasourceDefn { - int eType; /* A TEST_DATASOURCE_* value */ - int nMinKey; /* Minimum key size */ - int nMaxKey; /* Maximum key size */ - int nMinVal; /* Minimum value size */ - int nMaxVal; /* Maximum value size */ -}; - -#define TEST_DATASOURCE_RANDOM 1 -#define TEST_DATASOURCE_SEQUENCE 2 - -char *testDatasourceName(const DatasourceDefn *); -Datasource *testDatasourceNew(const DatasourceDefn *); -void testDatasourceFree(Datasource *); -void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *); -/* End of test_datasource.c interface. -*************************************************************************/ -void testDatasourceFetch( - TestDb *pDb, /* Database handle */ - Datasource *pData, - int iKey, - int *pRc /* IN/OUT: Error code */ -); - -void testWriteDatasource(TestDb *, Datasource *, int, int *); -void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *); -void testDeleteDatasource(TestDb *, Datasource *, int, int *); -void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *); - - -/* test1.c */ -void test_data_1(const char *, const char *, int *pRc); -void test_data_2(const char *, const char *, int *pRc); -void test_data_3(const char *, const char *, int *pRc); -void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); -void testCaseProgress(int, int, int, int *); -int testCaseNDot(void); - -void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); -int testControlDb(TestDb **ppDb); - -typedef struct CksumDb CksumDb; -CksumDb *testCksumArrayNew(Datasource *, int, int, int); -char *testCksumArrayGet(CksumDb *, int); -void testCksumArrayFree(CksumDb *); -void testCaseStart(int *pRc, char *zFmt, ...); -void testCaseFinish(int rc); -void testCaseSkip(void); -int testCaseBegin(int *, const char *, const char *, ...); - -#define TEST_CKSUM_BYTES 29 -int testCksumDatabase(TestDb *pDb, char *zOut); -int testCountDatabase(TestDb *pDb); -void testCompareInt(int, int, int *); -void testCompareStr(const char *z1, const char *z2, int *pRc); - -/* lsmtest9.c */ -void test_data_4(const char *, const char *, int *pRc); - - -/* -** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. -*/ -#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) -int testArgSelectX(void *, const char *, int, const char *, int *); - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif - -#endif diff --git a/ext/lsm1/lsm-test/lsmtest1.c b/ext/lsm1/lsm-test/lsmtest1.c deleted file mode 100644 index 1ce2cc0588..0000000000 --- a/ext/lsm1/lsm-test/lsmtest1.c +++ /dev/null @@ -1,656 +0,0 @@ - -#include "lsmtest.h" - -#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE -#define DATA_RANDOM TEST_DATASOURCE_RANDOM - -typedef struct Datatest1 Datatest1; -typedef struct Datatest2 Datatest2; - -/* -** An instance of the following structure contains parameters used to -** customize the test function in this file. Test procedure: -** -** 1. Create a data-source based on the "datasource definition" vars. -** -** 2. Insert nRow key value pairs into the database. -** -** 3. Delete all keys from the database. Deletes are done in the same -** order as the inserts. -** -** During steps 2 and 3 above, after each Datatest1.nVerify inserts or -** deletes, the following: -** -** a. Run Datasource.nTest key lookups and check the results are as expected. -** -** b. If Datasource.bTestScan is true, run a handful (8) of range -** queries (scanning forwards and backwards). Check that the results -** are as expected. -** -** c. Close and reopen the database. Then run (a) and (b) again. -*/ -struct Datatest1 { - /* Datasource definition */ - DatasourceDefn defn; - - /* Test procedure parameters */ - int nRow; /* Number of rows to insert then delete */ - int nVerify; /* How often to verify the db contents */ - int nTest; /* Number of keys to test (0==all) */ - int bTestScan; /* True to do scan tests */ -}; - -/* -** An instance of the following data structure is used to describe the -** second type of test case in this file. The chief difference between -** these tests and those described by Datatest1 is that these tests also -** experiment with range-delete operations. Tests proceed as follows: -** -** 1. Open the datasource described by Datatest2.defn. -** -** 2. Open a connection on an empty database. -** -** 3. Do this Datatest2.nIter times: -** -** a) Insert Datatest2.nWrite key-value pairs from the datasource. -** -** b) Select two pseudo-random keys and use them as the start -** and end points of a range-delete operation. -** -** c) Verify that the contents of the database are as expected (see -** below for details). -** -** d) Close and then reopen the database handle. -** -** e) Verify that the contents of the database are still as expected. -** -** The inserts and range deletes are run twice - once on the database being -** tested and once using a control system (sqlite3, kc etc. - something that -** works). In order to verify that the contents of the db being tested are -** correct, the test runs a bunch of scans and lookups on both the test and -** control databases. If the results are the same, the test passes. -*/ -struct Datatest2 { - DatasourceDefn defn; - int nRange; - int nWrite; /* Number of writes per iteration */ - int nIter; /* Total number of iterations to run */ -}; - -/* -** Generate a unique name for the test case pTest with database system -** zSystem. -*/ -static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", - zSystem, zData, bRecover, pTest->nRow, pTest->nVerify - ); - testFree(zData); - return zRet; -} - -int testControlDb(TestDb **ppDb){ -#ifdef HAVE_KYOTOCABINET - return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); -#else - return tdb_open("sqlite3", "", 1, ppDb); -#endif -} - -void testDatasourceFetch( - TestDb *pDb, /* Database handle */ - Datasource *pData, - int iKey, - int *pRc /* IN/OUT: Error code */ -){ - void *pKey; int nKey; /* Database key to query for */ - void *pVal; int nVal; /* Expected result of query */ - - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testFetch(pDb, pKey, nKey, pVal, nVal, pRc); -} - -/* -** This function is called to test that the contents of database pDb -** are as expected. In this case, expected is defined as containing -** key-value pairs iFirst through iLast, inclusive, from data source -** pData. In other words, a loop like the following could be used to -** construct a database with identical contents from scratch. -** -** for(i=iFirst; i<=iLast; i++){ -** testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); -** // insert (pKey, nKey) -> (pVal, nVal) into database -** } -** -** The key domain consists of keys 0 to (nRow-1), inclusive, from -** data source pData. For both scan and lookup tests, keys are selected -** pseudo-randomly from within this set. -** -** This function runs nLookupTest lookup tests and nScanTest scan tests. -** -** A lookup test consists of selecting a key from the domain and querying -** pDb for it. The test fails if the presence of the key and, if present, -** the associated value do not match the expectations defined above. -** -** A scan test involves selecting a key from the domain and running -** the following queries: -** -** 1. Scan all keys equal to or greater than the key, in ascending order. -** 2. Scan all keys equal to or smaller than the key, in descending order. -** -** Additionally, if nLookupTest is greater than zero, the following are -** run once: -** -** 1. Scan all keys in the db, in ascending order. -** 2. Scan all keys in the db, in descending order. -** -** As you would assume, the test fails if the returned values do not match -** expectations. -*/ -void testDbContents( - TestDb *pDb, /* Database handle being tested */ - Datasource *pData, /* pDb contains data from here */ - int nRow, /* Size of key domain */ - int iFirst, /* Index of first key from pData in pDb */ - int iLast, /* Index of last key from pData in pDb */ - int nLookupTest, /* Number of lookup tests to run */ - int nScanTest, /* Number of scan tests to run */ - int *pRc /* IN/OUT: Error code */ -){ - int j; - int rc = *pRc; - - if( rc==0 && nScanTest ){ - TestDb *pDb2 = 0; - - /* Open a control db (i.e. one that we assume works) */ - rc = testControlDb(&pDb2); - - for(j=iFirst; rc==0 && j<=iLast; j++){ - void *pKey; int nKey; /* Database key to insert */ - void *pVal; int nVal; /* Database value to insert */ - testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal); - rc = tdb_write(pDb2, pKey, nKey, pVal, nVal); - } - - if( rc==0 ){ - int iKey1; - int iKey2; - void *pKey1; int nKey1; /* Start key */ - void *pKey2; int nKey2; /* Final key */ - - iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow; - iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow; - testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); - pKey1 = testMalloc(nKey1+1); - memcpy(pKey1, pKey2, nKey1+1); - testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); - - testScanCompare(pDb2, pDb, 0, 0, 0, 0, 0, &rc); - testScanCompare(pDb2, pDb, 0, 0, 0, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0, &rc); - testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 1, 0, 0, 0, 0, &rc); - testScanCompare(pDb2, pDb, 1, 0, 0, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0, &rc); - testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc); - testFree(pKey1); - } - tdb_close(pDb2); - } - - /* Test some lookups. */ - for(j=0; rc==0 && j=nRow ){ - iKey = j; - }else{ - iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow; - } - - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - if( iFirst>iKey || iKey>iLast ){ - pVal = 0; - nVal = -1; - } - - testFetch(pDb, pKey, nKey, pVal, nVal, &rc); - } - - *pRc = rc; -} - -/* -** This function should be called during long running test cases to output -** the progress dots (...) to stdout. -*/ -void testCaseProgress(int i, int n, int nDot, int *piDot){ - int iDot = *piDot; - while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){ - printf("."); - fflush(stdout); - iDot++; - } - *piDot = iDot; -} - -int testCaseNDot(void){ return 20; } - -#if 0 -static void printScanCb( - void *pCtx, void *pKey, int nKey, void *pVal, int nVal -){ - printf("%s\n", (char *)pKey); - fflush(stdout); -} -#endif - -void testReopenRecover(TestDb **ppDb, int *pRc){ - if( *pRc==0 ){ - const char *zLib = tdb_library_name(*ppDb); - const char *zDflt = tdb_default_db(zLib); - testCopyLsmdb(zDflt, "bak.db"); - testClose(ppDb); - testCopyLsmdb("bak.db", zDflt); - *pRc = tdb_open(zLib, 0, 0, ppDb); - } -} - - -static void doDataTest1( - const char *zSystem, /* Database system to test */ - int bRecover, - Datatest1 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int i; - int iDot; - int rc = LSM_OK; - Datasource *pData; - TestDb *pDb; - int iToggle = 0; - - /* Start the test case, open a database and allocate the datasource. */ - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - - i = 0; - iDot = 0; - while( rc==LSM_OK && inRow ){ - - /* Insert some data */ - testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc); - i += p->nVerify; - - if( iToggle ) testBegin(pDb, 1, &rc); - /* Check that the db content is correct. */ - testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); - if( iToggle ) testCommit(pDb, 0, &rc); - iToggle = (iToggle+1)%2; - - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - - /* Check that the db content is still correct. */ - testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); - } - - i = 0; - iDot = 0; - while( rc==LSM_OK && inRow ){ - - /* Delete some entries */ - testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc); - i += p->nVerify; - - /* Check that the db content is correct. */ - testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); - - /* Close and reopen the database. */ - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - - /* Check that the db content is still correct. */ - testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); - } - - /* Free the datasource, close the database and finish the test case. */ - testDatasourceFree(pData); - tdb_close(pDb); - testCaseFinish(rc); - *pRc = rc; -} - - -void test_data_1( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest1 aTest[] = { - { {DATA_RANDOM, 500,600, 1000,2000}, 1000, 100, 10, 0}, - { {DATA_RANDOM, 20,25, 100,200}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 100,200}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 10,20}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 1000,2000}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,100, 10000,20000}, 100, 25, 100, 1}, - { {DATA_RANDOM, 80,100, 10,20}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 5000,6000, 10,20}, 100, 25, 100, 1}, - { {DATA_SEQUENTIAL, 5,10, 10,20}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,10, 100,200}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,10, 1000,2000}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,100, 10000,20000}, 100, 25, 100, 1}, - { {DATA_RANDOM, 10,10, 100,100}, 100000, 1000, 100, 0}, - { {DATA_SEQUENTIAL, 10,10, 100,100}, 100000, 1000, 100, 0}, - }; - - int i; - int bRecover; - - for(bRecover=0; bRecover<2; bRecover++){ - if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; - for(i=0; *pRc==LSM_OK && idefn); - rc = testControlDb(&pControl); - - if( tdb_lsm(pDb) ){ - int nBuf = 32 * 1024 * 1024; - lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); - } - - for(i=0; rc==0 && inIter; i++){ - void *pKey1; int nKey1; - void *pKey2; int nKey2; - int ii; - int nRange = MIN(p->nIter*p->nWrite, p->nRange); - - for(ii=0; rc==0 && iinWrite; ii++){ - int iKey = (i*p->nWrite + ii) % p->nRange; - testWriteDatasource(pControl, pData, iKey, &rc); - testWriteDatasource(pDb, pData, iKey, &rc); - } - - testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0); - pKey1 = testMallocCopy(pKey1, nKey1); - testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0); - - testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc); - testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc); - testFree(pKey1); - - testCompareDb(pData, nRange, i, pControl, pDb, &rc); - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - testCompareDb(pData, nRange, i, pControl, pDb, &rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", - zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter - ); - testFree(zData); - return zRet; -} - -void test_data_2( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest2 aTest[] = { - /* defn, nRange, nWrite, nIter */ - { {DATA_RANDOM, 20,25, 100,200}, 10000, 10, 50 }, - { {DATA_RANDOM, 20,25, 100,200}, 10000, 200, 50 }, - { {DATA_RANDOM, 20,25, 100,200}, 100, 10, 1000 }, - { {DATA_RANDOM, 20,25, 100,200}, 100, 200, 50 }, - }; - - int i; - int bRecover; - - for(bRecover=0; bRecover<2; bRecover++){ - if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; - for(i=0; *pRc==LSM_OK && i> 24) & 0xFF; - aBuf[1] = (iVal >> 16) & 0xFF; - aBuf[2] = (iVal >> 8) & 0xFF; - aBuf[3] = (iVal >> 0) & 0xFF; -} - -void dt3PutKey(u8 *aBuf, int iKey){ - assert( iKey<100000 && iKey>=0 ); - sprintf((char *)aBuf, "%.5d", iKey); -} - -static void doDataTest3( - const char *zSystem, /* Database system to test */ - Datatest3 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int iDot = 0; - int rc = *pRc; - TestDb *pDb; - u8 *abPresent; /* Array of boolean */ - char *aVal; /* Buffer to hold values */ - int i; - u32 iSeq = 10; /* prng counter */ - - abPresent = (u8 *)testMalloc(p->nRange+1); - aVal = (char *)testMalloc(p->nValMax+1); - pDb = testOpen(zSystem, 1, &rc); - - for(i=0; inIter && rc==0; i++){ - int ii; - - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - - /* Perform nWrite inserts */ - for(ii=0; iinWrite; ii++){ - u8 aKey[6]; - u32 iKey; - int nVal; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin; - testPrngString(testPrngValue(iSeq++), aVal, nVal); - dt3PutKey(aKey, iKey); - - testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc); - abPresent[iKey] = 1; - } - - /* Perform nDelete deletes */ - for(ii=0; iinDelete; ii++){ - u8 aKey1[6]; - u8 aKey2[6]; - u32 iKey; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - dt3PutKey(aKey1, iKey-1); - dt3PutKey(aKey2, iKey+1); - - testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc); - abPresent[iKey] = 0; - } - - testReopen(&pDb, &rc); - - for(ii=1; rc==0 && ii<=p->nRange; ii++){ - int nDbVal; - void *pDbVal; - u8 aKey[6]; - int dbrc; - - dt3PutKey(aKey, ii); - dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal); - testCompareInt(0, dbrc, &rc); - - if( abPresent[ii] ){ - testCompareInt(1, (nDbVal>0), &rc); - }else{ - testCompareInt(1, (nDbVal<0), &rc); - } - } - } - - testClose(&pDb); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName3(const char *zSystem, Datatest3 *p){ - return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)", - zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, - p->nValMin, p->nValMax - ); -} - -void test_data_3( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest3 aTest[] = { - /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */ - { 100, 1000, 5, 5, 50, 100 }, - { 100, 1000, 2, 2, 5, 10 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && inRow++; - for(i=0; icksum1 += ((u8 *)pKey)[i]; - p->cksum2 += p->cksum1; - } - for(i=0; icksum1 += ((u8 *)pVal)[i]; - p->cksum2 += p->cksum1; - } -} - -/* -** tdb_scan() callback used by testCountDatabase() -*/ -static void scanCountDb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - Cksum *p = (Cksum *)pCtx; - p->nRow++; - - unused_parameter(pKey); - unused_parameter(nKey); - unused_parameter(pVal); - unused_parameter(nVal); -} - - -/* -** Iterate through the entire contents of database pDb. Write a checksum -** string based on the db contents into buffer zOut before returning. A -** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size: -** -** * 32-bit integer (10 bytes) -** * 1 space (1 byte) -** * 32-bit hex (8 bytes) -** * 1 space (1 byte) -** * 32-bit hex (8 bytes) -** * nul-terminator (1 byte) -** -** The number of entries in the database is returned. -*/ -int testCksumDatabase( - TestDb *pDb, /* Database handle */ - char *zOut /* Buffer to write checksum to */ -){ - Cksum cksum; - memset(&cksum, 0, sizeof(Cksum)); - tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb); - sprintf(zOut, "%d %x %x", - cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2 - ); - assert( strlen(zOut)0 ); */ - if( testrc==0 ) testrc = lsm_checkpoint(db, 0); - } - tdb_close(pDb); - - /* Check that the database content is still correct */ - testCompareCksumLsmdb(DBNAME, - bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc); - } - - testCksumArrayFree(pCksumDb); - testDatasourceFree(pData); -} - -/* -** This test verifies that if a system crash occurs while committing a -** transaction to the log file, no earlier transactions are lost or damaged. -*/ -static void crash_test2(int bCompress, int *pRc){ - const char *DBNAME = "testdb.lsm"; - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; - - const int nIter = 200; - const int nInsert = 20; - - int i; - int iDot = 0; - Datasource *pData; - CksumDb *pCksumDb; - TestDb *pDb; - - /* Allocate datasource. And calculate the expected checksums. */ - pData = testDatasourceNew(&defn); - pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1); - - /* Setup and save the initial database. */ - testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); - - for(i=0; izTest) ){ - p->x(p->bCompress, pRc); - testCaseFinish(*pRc); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest3.c b/ext/lsm1/lsm-test/lsmtest3.c deleted file mode 100644 index 760dec300f..0000000000 --- a/ext/lsm1/lsm-test/lsmtest3.c +++ /dev/null @@ -1,238 +0,0 @@ - - -/* -** This file contains tests related to the explicit rollback of database -** transactions and sub-transactions. -*/ - - -/* -** Repeat 2000 times (until the db contains 100,000 entries): -** -** 1. Open a transaction and insert 500 rows, opening a nested -** sub-transaction each 100 rows. -** -** 2. Roll back to each sub-transaction savepoint. Check the database -** checksum looks Ok. -** -** 3. Every second iteration, roll back the main transaction. Check the -** db checksum is correct. Every other iteration, commit the main -** transaction (increasing the size of the db by 100 rows). -*/ - - -#include "lsmtest.h" - -struct CksumDb { - int nFirst; - int nLast; - int nStep; - char **azCksum; -}; - -CksumDb *testCksumArrayNew( - Datasource *pData, - int nFirst, - int nLast, - int nStep -){ - TestDb *pDb; - CksumDb *pRet; - int i; - int nEntry; - int rc = 0; - - assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 ); - - pRet = malloc(sizeof(CksumDb)); - memset(pRet, 0, sizeof(CksumDb)); - pRet->nFirst = nFirst; - pRet->nLast = nLast; - pRet->nStep = nStep; - nEntry = 1 + ((nLast - nFirst) / nStep); - - /* Allocate space so that azCksum is an array of nEntry pointers to - ** buffers each TEST_CKSUM_BYTES in size. */ - pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES)); - for(i=0; iazCksum[nEntry]); - pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES]; - } - - tdb_open("lsm", "tempdb.lsm", 1, &pDb); - testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc); - for(i=0; iazCksum[i]); - if( i==nEntry ) break; - testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc); - } - - tdb_close(pDb); - - return pRet; -} - -char *testCksumArrayGet(CksumDb *p, int nRow){ - int i; - assert( nRow>=p->nFirst ); - assert( nRow<=p->nLast ); - assert( ((nRow-p->nFirst) % p->nStep)==0 ); - - i = (nRow - p->nFirst) / p->nStep; - return p->azCksum[i]; -} - -void testCksumArrayFree(CksumDb *p){ - free(p->azCksum); - memset(p, 0x55, sizeof(*p)); - free(p); -} - -/* End of CksumDb code. -**************************************************************************/ - -/* -** Test utility function. Write key-value pair $i from datasource pData -** into database pDb. -*/ -void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); - testWrite(pDb, pKey, nKey, pVal, nVal, pRc); -} - -/* -** Test utility function. Delete datasource pData key $i from database pDb. -*/ -void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ - void *pKey; int nKey; - testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); - testDelete(pDb, pKey, nKey, pRc); -} - -/* -** This function inserts nWrite key/value pairs into database pDb - the -** nWrite key value pairs starting at iFirst from data source pData. -*/ -void testWriteDatasourceRange( - TestDb *pDb, /* Database to write to */ - Datasource *pData, /* Data source to read values from */ - int iFirst, /* Index of first key/value pair */ - int nWrite, /* Number of key/value pairs to write */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - for(i=0; i2 && rc==0; iTrans--){ - tdb_rollback(pDb, iTrans); - nCurrent -= 100; - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - } - - if( i%2 ){ - tdb_rollback(pDb, 0); - nCurrent -= 100; - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - }else{ - tdb_commit(pDb, 0); - } - } - testCaseFinish(rc); - - skip_rollback_test: - tdb_close(pDb); - testCksumArrayFree(pCksum); - return rc; -} - -void test_rollback( - const char *zSystem, - const char *zPattern, - int *pRc -){ - if( *pRc==0 ){ - int bRun = 1; - - if( zPattern ){ - char *zName = getName(zSystem); - bRun = testGlobMatch(zPattern, zName); - testFree(zName); - } - - if( bRun ){ - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 }; - Datasource *pData = testDatasourceNew(&defn); - *pRc = rollback_test_1(zSystem, pData); - testDatasourceFree(pData); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest4.c b/ext/lsm1/lsm-test/lsmtest4.c deleted file mode 100644 index a47241db92..0000000000 --- a/ext/lsm1/lsm-test/lsmtest4.c +++ /dev/null @@ -1,127 +0,0 @@ - -/* -** This file contains test cases involving multiple database clients. -*/ - -#include "lsmtest.h" - -/* -** The following code implements test cases "mc1.*". -** -** This test case uses one writer and $nReader readers. All connections -** are driven by a single thread. All connections are opened at the start -** of the test and remain open until the test is finished. -** -** The test consists of $nStep steps. Each step the following is performed: -** -** 1. The writer inserts $nWriteStep records into the db. -** -** 2. The writer checks that the contents of the db are as expected. -** -** 3. Each reader that currently has an open read transaction also checks -** that the contents of the db are as expected (according to the snapshot -** the read transaction is reading - see below). -** -** After step 1, reader 1 opens a read transaction. After step 2, reader -** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1 -** closes the current read transaction and opens a new one. And so on. -** The result is that at step N (for N > $nReader), there exists a reader -** with an open read transaction reading the snapshot committed following -** steps (N-$nReader-1) to N. -*/ -typedef struct Mctest Mctest; -struct Mctest { - DatasourceDefn defn; /* Datasource to use */ - int nStep; /* Total number of steps in test */ - int nWriteStep; /* Number of rows to insert each step */ - int nReader; /* Number of read connections */ -}; -static void do_mc_test( - const char *zSystem, /* Database system to test */ - Mctest *pTest, - int *pRc /* IN/OUT: return code */ -){ - const int nDomain = pTest->nStep * pTest->nWriteStep; - Datasource *pData; /* Source of data */ - TestDb *pDb; /* First database connection (writer) */ - int iReader; /* Used to iterate through aReader */ - int iStep; /* Current step in test */ - int iDot = 0; /* Current step in test */ - - /* Array of reader connections */ - struct Reader { - TestDb *pDb; /* Connection handle */ - int iLast; /* Current snapshot contains keys 0..iLast */ - } *aReader; - - /* Create a data source */ - pData = testDatasourceNew(&pTest->defn); - - /* Open the writer connection */ - pDb = testOpen(zSystem, 1, pRc); - - /* Allocate aReader */ - aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader); - for(iReader=0; iReadernReader; iReader++){ - aReader[iReader].pDb = testOpen(zSystem, 0, pRc); - } - - for(iStep=0; iStepnStep; iStep++){ - int iLast; - int iBegin; /* Start read trans using aReader[iBegin] */ - - /* Insert nWriteStep more records into the database */ - int iFirst = iStep*pTest->nWriteStep; - testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc); - - /* Check that the db is Ok according to the writer */ - iLast = (iStep+1) * pTest->nWriteStep - 1; - testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc); - - /* Have reader (iStep % nReader) open a read transaction here. */ - iBegin = (iStep % pTest->nReader); - if( iBeginnReader && aReader[iReader].iLast; iReader++){ - iLast = aReader[iReader].iLast; - testDbContents( - aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc - ); - } - - /* Report progress */ - testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot); - } - - /* Close all readers */ - for(iReader=0; iReadernReader; iReader++){ - testClose(&aReader[iReader].pDb); - } - testFree(aReader); - - /* Close the writer-connection and free the datasource */ - testClose(&pDb); - testDatasourceFree(pData); -} - - -void test_mc( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - Mctest aTest[] = { - { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 }, - }; - - for(i=0; i "k.0000000045". -** -** As well as the key/value pairs, the database also contains checksum -** entries. The checksums form a hierarchy - for every F key/value -** entries there is one level 1 checksum. And for each F level 1 checksums -** there is one level 2 checksum. And so on. -** -** Checksum keys are encoded as the two byte "c." followed by the -** checksum level, followed by a 10 digit decimal number containing -** the value of the first key that contributes to the checksum value. -** For example, assuming F==10, the level 1 checksum that spans keys -** 10 to 19 is "c.1.0000000010". -** -** Clients may perform one of two operations on the database: a read -** or a write. -** -** READ OPERATIONS: -** -** A read operation scans a range of F key/value pairs. It computes -** the expected checksum and then compares the computed value to the -** actual value stored in the level 1 checksum entry. It then scans -** the group of F level 1 checksums, and compares the computed checksum -** to the associated level 2 checksum value, and so on until the -** highest level checksum value has been verified. -** -** If a checksum ever fails to match the expected value, the test -** has failed. -** -** WRITE OPERATIONS: -** -** A write operation involves writing (possibly clobbering) a single -** key/value pair. The associated level 1 checksum is then recalculated -** updated. Then the level 2 checksum, and so on until the highest -** level checksum has been modified. -** -** All updates occur inside a single transaction. -** -** INTERFACE: -** -** The interface used by test cases to read and write the db consists -** of type DbParameters and the following functions: -** -** dbReadOperation() -** dbWriteOperation() -*/ - -#include "lsmtest.h" - -typedef struct DbParameters DbParameters; -struct DbParameters { - int nFanout; /* Checksum fanout (F) */ - int nKey; /* Size of key space (N) */ -}; - -#define DB_KEY_BYTES (2+5+10+1) - -/* -** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. -** This function populates the buffer with a nul-terminated key string -** corresponding to key iKey. -*/ -static void dbFormatKey( - DbParameters *pParam, - int iLevel, - int iKey, /* Key value */ - char *aBuf /* Write key string here */ -){ - if( iLevel==0 ){ - snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey); - }else{ - int f = 1; - int i; - for(i=0; inFanout; - snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f)); - } -} - -/* -** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. -** This function populates the buffer with the string representation of -** checksum value iVal. -*/ -static void dbFormatCksumValue(u32 iVal, char *aBuf){ - snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal); -} - -/* -** Return the highest level of checksum in the database described -** by *pParam. -*/ -static int dbMaxLevel(DbParameters *pParam){ - int iMax; - int n = 1; - for(iMax=0; nnKey; iMax++){ - n = n * pParam->nFanout; - } - return iMax; -} - -static void dbCksum( - void *pCtx, /* IN/OUT: Pointer to u32 containing cksum */ - void *pKey, int nKey, /* Database key. Unused. */ - void *pVal, int nVal /* Database value. Checksum this. */ -){ - u8 *aVal = (u8 *)pVal; - u32 *pCksum = (u32 *)pCtx; - u32 cksum = *pCksum; - int i; - - unused_parameter(pKey); - unused_parameter(nKey); - - for(i=0; inFanout entries at level -** iLevel-1. -*/ -static u32 dbComputeCksum( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - int iLevel, /* Level of checksum to compute */ - int iKey, /* Compute checksum for this key */ - int *pRc /* IN/OUT: Error code */ -){ - u32 cksum = 0; - if( *pRc==0 ){ - int nFirst; - int nLast; - int iFirst = 0; - int iLast = 0; - int i; - int f = 1; - char zFirst[DB_KEY_BYTES]; - char zLast[DB_KEY_BYTES]; - - assert( iLevel>=1 ); - for(i=0; inFanout; - - iFirst = f*(iKey/f); - iLast = iFirst + f - 1; - dbFormatKey(pParam, iLevel-1, iFirst, zFirst); - dbFormatKey(pParam, iLevel-1, iLast, zLast); - nFirst = strlen(zFirst); - nLast = strlen(zLast); - - *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum); - } - - return cksum; -} - -static void dbReadOperation( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - void (*xDelay)(void *), - void *pDelayCtx, - int iKey, /* Key to read */ - int *pRc /* IN/OUT: Error code */ -){ - const int iMax = dbMaxLevel(pParam); - int i; - - if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc); - for(i=1; *pRc==0 && i<=iMax; i++){ - char zCksum[DB_KEY_BYTES]; - char zKey[DB_KEY_BYTES]; - u32 iCksum = 0; - - iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); - if( iCksum ){ - if( xDelay && i==1 ) xDelay(pDelayCtx); - dbFormatCksumValue(iCksum, zCksum); - dbFormatKey(pParam, i, iKey, zKey); - testFetchStr(pDb, zKey, zCksum, pRc); - } - } - if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); -} - -static int dbWriteOperation( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - int iKey, /* Key to write to */ - const char *zValue, /* Nul-terminated value to write */ - int *pRc /* IN/OUT: Error code */ -){ - const int iMax = dbMaxLevel(pParam); - char zKey[DB_KEY_BYTES]; - int i; - int rc; - - assert( iKey>=0 && iKeynKey ); - dbFormatKey(pParam, 0, iKey, zKey); - - /* Open a write transaction. This may fail - SQLITE4_BUSY */ - if( *pRc==0 && tdb_transaction_support(pDb) ){ - rc = tdb_begin(pDb, 2); - if( rc==5 ) return 0; - *pRc = rc; - } - - testWriteStr(pDb, zKey, zValue, pRc); - for(i=1; i<=iMax; i++){ - char zCksum[DB_KEY_BYTES]; - u32 iCksum = 0; - - iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); - dbFormatCksumValue(iCksum, zCksum); - dbFormatKey(pParam, i, iKey, zKey); - testWriteStr(pDb, zKey, zCksum, pRc); - } - if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); - return 1; -} - -/************************************************************************* -** The following block contains testXXX() functions that implement a -** wrapper around the systems native multi-thread support. There are no -** synchronization primitives - just functions to launch and join -** threads. Wrapper functions are: -** -** testThreadSupport() -** -** testThreadInit() -** testThreadShutdown() -** testThreadLaunch() -** testThreadWait() -** -** testThreadSetHalt() -** testThreadGetHalt() -** testThreadSetResult() -** testThreadGetResult() -** -** testThreadEnterMutex() -** testThreadLeaveMutex() -*/ -typedef struct ThreadSet ThreadSet; -#ifdef LSM_MUTEX_PTHREADS - -#include -#include - -typedef struct Thread Thread; -struct Thread { - int rc; - char *zMsg; - pthread_t id; - void (*xMain)(ThreadSet *, int, void *); - void *pCtx; - ThreadSet *pThreadSet; -}; - -struct ThreadSet { - int bHalt; /* Halt flag */ - int nThread; /* Number of threads */ - Thread *aThread; /* Array of Thread structures */ - pthread_mutex_t mutex; /* Mutex used for cheating */ -}; - -/* -** Return true if this build supports threads, or false otherwise. If -** this function returns false, no other testThreadXXX() functions should -** be called. -*/ -static int testThreadSupport(){ return 1; } - -/* -** Allocate and return a thread-set handle with enough space allocated -** to handle up to nMax threads. Each call to this function should be -** matched by a call to testThreadShutdown() to delete the object. -*/ -static ThreadSet *testThreadInit(int nMax){ - int nByte; /* Total space to allocate */ - ThreadSet *p; /* Return value */ - - nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax; - p = (ThreadSet *)testMalloc(nByte); - p->nThread = nMax; - p->aThread = (Thread *)&p[1]; - pthread_mutex_init(&p->mutex, 0); - - return p; -} - -/* -** Delete a thread-set object and release all resources held by it. -*/ -static void testThreadShutdown(ThreadSet *p){ - int i; - for(i=0; inThread; i++){ - testFree(p->aThread[i].zMsg); - } - pthread_mutex_destroy(&p->mutex); - testFree(p); -} - -static void *ttMain(void *pArg){ - Thread *pThread = (Thread *)pArg; - int iThread; - iThread = (pThread - pThread->pThreadSet->aThread); - pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx); - return 0; -} - -/* -** Launch a new thread. -*/ -static int testThreadLaunch( - ThreadSet *p, - int iThread, - void (*xMain)(ThreadSet *, int, void *), - void *pCtx -){ - int rc; - Thread *pThread; - - assert( iThread>=0 && iThreadnThread ); - - pThread = &p->aThread[iThread]; - assert( pThread->pThreadSet==0 ); - pThread->xMain = xMain; - pThread->pCtx = pCtx; - pThread->pThreadSet = p; - rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread); - - return rc; -} - -/* -** Set the thread-set "halt" flag. -*/ -static void testThreadSetHalt(ThreadSet *pThreadSet){ - pThreadSet->bHalt = 1; -} - -/* -** Return the current value of the thread-set "halt" flag. -*/ -static int testThreadGetHalt(ThreadSet *pThreadSet){ - return pThreadSet->bHalt; -} - -static void testThreadSleep(ThreadSet *pThreadSet, int nMs){ - int nRem = nMs; - while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){ - usleep(50000); - nRem -= 50; - } -} - -/* -** Wait for all threads launched to finish before returning. If nMs -** is greater than zero, set the "halt" flag to tell all threads -** to halt after waiting nMs milliseconds. -*/ -static void testThreadWait(ThreadSet *pThreadSet, int nMs){ - int i; - - testThreadSleep(pThreadSet, nMs); - testThreadSetHalt(pThreadSet); - for(i=0; inThread; i++){ - Thread *pThread = &pThreadSet->aThread[i]; - if( pThread->xMain ){ - pthread_join(pThread->id, 0); - } - } -} - -/* -** Set the result for thread iThread. -*/ -static void testThreadSetResult( - ThreadSet *pThreadSet, /* Thread-set handle */ - int iThread, /* Set result for this thread */ - int rc, /* Result error code */ - char *zFmt, /* Result string format */ - ... /* Result string formatting args... */ -){ - va_list ap; - - testFree(pThreadSet->aThread[iThread].zMsg); - pThreadSet->aThread[iThread].rc = rc; - pThreadSet->aThread[iThread].zMsg = 0; - if( zFmt ){ - va_start(ap, zFmt); - pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap); - va_end(ap); - } -} - -/* -** Retrieve the result for thread iThread. -*/ -static int testThreadGetResult( - ThreadSet *pThreadSet, /* Thread-set handle */ - int iThread, /* Get result for this thread */ - const char **pzRes /* OUT: Pointer to result string */ -){ - if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg; - return pThreadSet->aThread[iThread].rc; -} - -/* -** Enter and leave the test case mutex. -*/ -#if 0 -static void testThreadEnterMutex(ThreadSet *p){ - pthread_mutex_lock(&p->mutex); -} -static void testThreadLeaveMutex(ThreadSet *p){ - pthread_mutex_unlock(&p->mutex); -} -#endif -#endif - -#if !defined(LSM_MUTEX_PTHREADS) -static int testThreadSupport(){ return 0; } - -#define testThreadInit(a) 0 -#define testThreadShutdown(a) -#define testThreadLaunch(a,b,c,d) 0 -#define testThreadWait(a,b) -#define testThreadSetHalt(a) -#define testThreadGetHalt(a) 0 -#define testThreadGetResult(a,b,c) 0 -#define testThreadSleep(a,b) 0 - -static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){ - unused_parameter(a); - unused_parameter(b); - unused_parameter(c); - unused_parameter(d); -} -#endif -/* End of threads wrapper. -*************************************************************************/ - -/************************************************************************* -** Below this point is the third part of this file - the implementation -** of the mt1.* tests. -*/ -typedef struct Mt1Test Mt1Test; -struct Mt1Test { - DbParameters param; /* Description of database to read/write */ - int nReadwrite; /* Number of read/write threads */ - int nFastReader; /* Number of fast reader threads */ - int nSlowReader; /* Number of slow reader threads */ - int nMs; /* How long to run for */ - const char *zSystem; /* Database system to test */ -}; - -typedef struct Mt1DelayCtx Mt1DelayCtx; -struct Mt1DelayCtx { - ThreadSet *pSet; /* Threadset to sleep within */ - int nMs; /* Sleep in ms */ -}; - -static void xMt1Delay(void *pCtx){ - Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx; - testThreadSleep(p->pSet, p->nMs); -} - -#define MT1_THREAD_RDWR 0 -#define MT1_THREAD_SLOW 1 -#define MT1_THREAD_FAST 2 - -static void xMt1Work(lsm_db *pDb, void *pCtx){ -#if 0 - char *z = 0; - lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z); - printf("%s\n", z); - fflush(stdout); -#endif -} - -/* -** This is the main() proc for all threads in test case "mt1". -*/ -static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){ - Mt1Test *p = (Mt1Test *)pCtx; /* Test parameters */ - Mt1DelayCtx delay; - int nRead = 0; /* Number of calls to dbReadOperation() */ - int nWrite = 0; /* Number of completed database writes */ - int rc = 0; /* Error code */ - int iPrng; /* Prng argument variable */ - TestDb *pDb; /* Database handle */ - int eType; - - delay.pSet = pThreadSet; - delay.nMs = 0; - if( iThreadnReadwrite ){ - eType = MT1_THREAD_RDWR; - }else if( iThread<(p->nReadwrite+p->nFastReader) ){ - eType = MT1_THREAD_FAST; - }else{ - eType = MT1_THREAD_SLOW; - delay.nMs = (p->nMs / 20); - } - - /* Open a new database connection. Initialize the pseudo-random number - ** argument based on the thread number. */ - iPrng = testPrngValue(iThread); - pDb = testOpen(p->zSystem, 0, &rc); - - if( rc==0 ){ - tdb_lsm_config_work_hook(pDb, xMt1Work, 0); - } - - /* Loop until either an error occurs or some other thread sets the - ** halt flag. */ - while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){ - int iKey; - - /* Perform a read operation on an arbitrarily selected key. */ - iKey = (testPrngValue(iPrng++) % p->param.nKey); - dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc); - if( rc ) continue; - nRead++; - - /* Attempt to write an arbitrary key value pair (and update the associated - ** checksum entries). dbWriteOperation() returns 1 if the write is - ** successful, or 0 if it failed with an LSM_BUSY error. */ - if( eType==MT1_THREAD_RDWR ){ - char aValue[50]; - char aRnd[25]; - - iKey = (testPrngValue(iPrng++) % p->param.nKey); - testPrngString(iPrng, aRnd, sizeof(aRnd)); - iPrng += sizeof(aRnd); - snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd); - nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc); - } - } - testClose(&pDb); - - /* If an error has occured, set the thread error code and the threadset - ** halt flag to tell the other test threads to halt. Otherwise, set the - ** thread error code to 0 and post a message with the number of read - ** and write operations completed. */ - if( rc ){ - testThreadSetResult(pThreadSet, iThread, rc, 0); - testThreadSetHalt(pThreadSet); - }else{ - testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite); - } -} - -static void do_test_mt1( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Mt1Test aTest[] = { - /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */ - { {10, 1000}, 4, 0, 0, 10000, 0 }, - { {10, 1000}, 4, 4, 2, 100000, 0 }, - { {10, 100000}, 4, 0, 0, 10000, 0 }, - { {10, 100000}, 4, 4, 2, 100000, 0 }, - }; - int i; - - for(i=0; *pRc==0 && iparam.nFanout, p->param.nKey, - p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader - ); - if( bRun ){ - TestDb *pDb; - ThreadSet *pSet; - int iThread; - int nThread; - - p->zSystem = zSystem; - pDb = testOpen(zSystem, 1, pRc); - - nThread = p->nReadwrite + p->nFastReader + p->nSlowReader; - pSet = testThreadInit(nThread); - for(iThread=0; *pRc==0 && iThreadnMs); - for(iThread=0; *pRc==0 && iThreadiNext = 1; - p->bEnable = 1; - p->nFail = 1; - p->pEnv = tdb_lsm_env(); -} - -static void xOomHook(OomTest *p){ - p->nFail++; -} - -static int testOomContinue(OomTest *p){ - if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){ - return 0; - } - p->nFail = 0; - testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p); - return 1; -} - -static void testOomEnable(OomTest *p, int bEnable){ - p->bEnable = bEnable; - testMallocOomEnable(p->pEnv, bEnable); -} - -static void testOomNext(OomTest *p){ - p->iNext++; -} - -static int testOomHit(OomTest *p){ - return (p->nFail>0); -} - -static int testOomFinish(OomTest *p){ - return p->rc; -} - -static void testOomAssert(OomTest *p, int bVal){ - if( bVal==0 ){ - test_failed(); - p->rc = 1; - } -} - -/* -** Test that the error code matches the state of the OomTest object passed -** as the first argument. Specifically, check that rc is LSM_NOMEM if an -** OOM error has already been injected, or LSM_OK if not. -*/ -static void testOomAssertRc(OomTest *p, int rc){ - testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM); - testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 ); -} - -static void testOomOpen( - OomTest *pOom, - const char *zName, - lsm_db **ppDb, - int *pRc -){ - if( *pRc==LSM_OK ){ - int rc; - rc = lsm_new(tdb_lsm_env(), ppDb); - if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName); - testOomAssertRc(pOom, rc); - *pRc = rc; - } -} - -static void testOomFetch( - OomTest *pOom, - lsm_db *pDb, - void *pKey, int nKey, - void *pVal, int nVal, - int *pRc -){ - testOomAssertRc(pOom, *pRc); - if( *pRc==LSM_OK ){ - lsm_cursor *pCsr; - int rc; - - rc = lsm_csr_open(pDb, &pCsr); - if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0); - testOomAssertRc(pOom, rc); - - if( rc==LSM_OK ){ - const void *p; int n; - testOomAssert(pOom, lsm_csr_valid(pCsr)); - - rc = lsm_csr_key(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) ); - } - - if( rc==LSM_OK ){ - const void *p; int n; - testOomAssert(pOom, lsm_csr_valid(pCsr)); - - rc = lsm_csr_value(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) ); - } - - lsm_csr_close(pCsr); - *pRc = rc; - } -} - -static void testOomWrite( - OomTest *pOom, - lsm_db *pDb, - void *pKey, int nKey, - void *pVal, int nVal, - int *pRc -){ - testOomAssertRc(pOom, *pRc); - if( *pRc==LSM_OK ){ - int rc; - - rc = lsm_insert(pDb, pKey, nKey, pVal, nVal); - testOomAssertRc(pOom, rc); - - *pRc = rc; - } -} - - -static void testOomFetchStr( - OomTest *pOom, - lsm_db *pDb, - const char *zKey, - const char *zVal, - int *pRc -){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); -} - -static void testOomFetchData( - OomTest *pOom, - lsm_db *pDb, - Datasource *pData, - int iKey, - int *pRc -){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc); -} - -static void testOomWriteStr( - OomTest *pOom, - lsm_db *pDb, - const char *zKey, - const char *zVal, - int *pRc -){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); -} - -static void testOomWriteData( - OomTest *pOom, - lsm_db *pDb, - Datasource *pData, - int iKey, - int *pRc -){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc); -} - -static void testOomScan( - OomTest *pOom, - lsm_db *pDb, - int bReverse, - const void *pKey, int nKey, - int nScan, - int *pRc -){ - if( *pRc==0 ){ - int rc; - int iScan = 0; - lsm_cursor *pCsr; - int (*xAdvance)(lsm_cursor *) = 0; - - - rc = lsm_csr_open(pDb, &pCsr); - testOomAssertRc(pOom, rc); - - if( rc==LSM_OK ){ - if( bReverse ){ - rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE); - xAdvance = lsm_csr_prev; - }else{ - rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE); - xAdvance = lsm_csr_next; - } - } - testOomAssertRc(pOom, rc); - - while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan "one" -** "two" -> "four" -** "three" -> "nine" -** "four" -> "sixteen" -** "five" -> "twentyfive" -** "six" -> "thirtysix" -** "seven" -> "fourtynine" -** "eight" -> "sixtyfour" -*/ -static void setup_populate_db(void){ - const char *azStr[] = { - "one", "one", - "two", "four", - "three", "nine", - "four", "sixteen", - "five", "twentyfive", - "six", "thirtysix", - "seven", "fourtynine", - "eight", "sixtyfour", - }; - int rc; - int ii; - lsm_db *pDb; - - testDeleteLsmdb(LSMTEST6_TESTDB); - - rc = lsm_new(tdb_lsm_env(), &pDb); - if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); - - for(ii=0; rc==LSM_OK && iiiInsStart, pStep->nIns, pRc); - testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc); - if( *pRc==0 ){ - int nSave = -1; - int nBuf = 64; - lsm_db *db = tdb_lsm(pDb); - - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); - lsm_begin(db, 1); - lsm_commit(db, 0); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); - - *pRc = lsm_work(db, 0, 0, 0); - if( *pRc==0 ){ - *pRc = lsm_checkpoint(db, 0); - } - } -} - -static void doSetupStepArray( - TestDb *pDb, - Datasource *pData, - const SetupStep *aStep, - int nStep -){ - int i; - for(i=0; i -void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){ - if( *pRc==0 ){ - FILE *fd; - fd = fopen(zFile, "rb"); - if( fd==0 ){ - *pRc = 1; - }else{ - if( 0!=fseek(fd, iOff, SEEK_SET) ){ - *pRc = 1; - }else{ - assert( nByte>=0 ); - if( (size_t)nByte!=fread(pOut, 1, nByte, fd) ){ - *pRc = 1; - } - } - fclose(fd); - } - } -} - -void testWriteFile( - const char *zFile, - int iOff, - void *pOut, - int nByte, - int *pRc -){ - if( *pRc==0 ){ - FILE *fd; - fd = fopen(zFile, "r+b"); - if( fd==0 ){ - *pRc = 1; - }else{ - if( 0!=fseek(fd, iOff, SEEK_SET) ){ - *pRc = 1; - }else{ - assert( nByte>=0 ); - if( (size_t)nByte!=fwrite(pOut, 1, nByte, fd) ){ - *pRc = 1; - } - } - fclose(fd); - } - } -} - -static ShmHeader *getShmHeader(const char *zDb){ - int rc = 0; - char *zShm = testMallocPrintf("%s-shm", zDb); - ShmHeader *pHdr; - - pHdr = testMalloc(sizeof(ShmHeader)); - testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc); - assert( rc==0 ); - - return pHdr; -} - -/* -** This function makes a copy of the three files associated with LSM -** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db", -** "test.db-log" and "test.db-shm"). -** -** It then opens a new database connection to the copy with the xLock() call -** instrumented so that it appears that some other process already connected -** to the db (holding a shared lock on DMS2). This prevents recovery from -** running. Then: -** -** 1) Check that the checksum of the database is zCksum. -** 2) Write a few keys to the database. Then delete the same keys. -** 3) Check that the checksum is zCksum. -** 4) Flush the db to disk and run a checkpoint. -** 5) Check once more that the checksum is still zCksum. -*/ -static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){ - if( *pRc==LSM_OK ){ - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500}; - Datasource *pData; - const char *zCopy = "testcopy.lsm"; - char zCksum2[TEST_CKSUM_BYTES]; - TestDb *pDb = 0; - int rc; - - pData = testDatasourceNew(&defn); - - testCopyLsmdb(zDb, zCopy); - rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb); - if( rc==0 ){ - ShmHeader *pHdr; - lsm_db *db; - testCksumDatabase(pDb, zCksum2); - testCompareStr(zCksum, zCksum2, &rc); - - testWriteDatasourceRange(pDb, pData, 1, 10, &rc); - testDeleteDatasourceRange(pDb, pData, 1, 10, &rc); - - /* Test that the two tree-headers are now consistent. */ - pHdr = getShmHeader(zCopy); - if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){ - rc = 1; - } - testFree(pHdr); - - if( rc==0 ){ - int nBuf = 64; - db = tdb_lsm(pDb); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); - lsm_begin(db, 1); - lsm_commit(db, 0); - rc = lsm_work(db, 0, 0, 0); - } - - testCksumDatabase(pDb, zCksum2); - testCompareStr(zCksum, zCksum2, &rc); - } - - testDatasourceFree(pData); - testClose(&pDb); - testDeleteLsmdb(zCopy); - *pRc = rc; - } -} - -static void doWriterCrash1(int *pRc){ - const int nWrite = 2000; - const int nStep = 10; - const int iWriteStart = 20000; - int rc = 0; - TestDb *pDb = 0; - Datasource *pData = 0; - - rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); - if( rc==0 ){ - int iDot = 0; - char zCksum[TEST_CKSUM_BYTES]; - int i; - setupDatabase1(pDb, &pData); - testCksumDatabase(pDb, zCksum); - testBegin(pDb, 2, &rc); - for(i=0; rc==0 && ihdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum1, &rc); - - /* If both tree-headers are valid, tree-header-1 is used. */ - memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); - memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If tree-header 1 is invalid, tree-header-2 is used */ - memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->hdr1.aCksum[0] = 5; - pHdr2->hdr1.aCksum[0] = 6; - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If tree-header 2 is invalid, tree-header-1 is used */ - memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); - pHdr2->hdr2.aCksum[0] = 5; - pHdr2->hdr2.aCksum[0] = 6; - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - testFree(pHdr1); - testFree(pHdr2); - testClose(&pDb); - } - - *pRc = rc; -} - -void do_writer_crash_test(const char *zPattern, int *pRc){ - struct Test { - const char *zName; - void (*xFunc)(int *); - } aTest[] = { - { "writercrash1.lsm", doWriterCrash1 }, - { "writercrash2.lsm", doWriterCrash2 }, - }; - int i; - for(i=0; izName) ){ - p->xFunc(pRc); - testCaseFinish(*pRc); - } - } - -} diff --git a/ext/lsm1/lsm-test/lsmtest9.c b/ext/lsm1/lsm-test/lsmtest9.c deleted file mode 100644 index b01de0d4e5..0000000000 --- a/ext/lsm1/lsm-test/lsmtest9.c +++ /dev/null @@ -1,140 +0,0 @@ - -#include "lsmtest.h" - -#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE -#define DATA_RANDOM TEST_DATASOURCE_RANDOM - -typedef struct Datatest4 Datatest4; - -/* -** Test overview: -** -** 1. Insert (Datatest4.nRec) records into a database. -** -** 2. Repeat (Datatest4.nRepeat) times: -** -** 2a. Delete 2/3 of the records in the database. -** -** 2b. Run lsm_work(nMerge=1). -** -** 2c. Insert as many records as were deleted in 2a. -** -** 2d. Check database content is as expected. -** -** 2e. If (Datatest4.bReopen) is true, close and reopen the database. -*/ -struct Datatest4 { - /* Datasource definition */ - DatasourceDefn defn; - - int nRec; - int nRepeat; - int bReopen; -}; - -static void doDataTest4( - const char *zSystem, /* Database system to test */ - Datatest4 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - lsm_db *db = 0; - TestDb *pDb; - TestDb *pControl; - Datasource *pData; - int i; - int rc = 0; - int iDot = 0; - int bMultiThreaded = 0; /* True for MT LSM database */ - - int nRecOn3 = (p->nRec / 3); - int iData = 0; - - /* Start the test case, open a database and allocate the datasource. */ - rc = testControlDb(&pControl); - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - if( rc==0 ){ - db = tdb_lsm(pDb); - bMultiThreaded = tdb_lsm_multithread(pDb); - } - - testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); - testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); - - for(i=0; rc==0 && inRepeat; i++){ - - testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); - testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); - - if( db ){ - int nDone; -#if 0 - fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); -#endif - do { - nDone = 0; - rc = lsm_work(db, 1, (1<<30), &nDone); - }while( rc==0 && nDone>0 ); - if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK; -#if 0 - fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); -#endif - } - -if( i+1nRepeat ){ - iData += (nRecOn3*2); - testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); - testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); - - testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); - - /* If Datatest4.bReopen is true, close and reopen the database */ - if( p->bReopen ){ - testReopen(&pDb, &rc); - if( rc==0 ) db = tdb_lsm(pDb); - } -} - - /* Update the progress dots... */ - testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName4(const char *zSystem, Datatest4 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", - zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen - ); - testFree(zData); - return zRet; -} - -void test_data_4( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest4 aTest[] = { - /* defn, nRec, nRepeat, bReopen */ - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && ieType ){ - case TEST_DATASOURCE_RANDOM: { - int nRange = (1 + p->nMaxKey - p->nMinKey); - nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; - testPrngString((u32)iData, p->aKey, nKey); - break; - } - case TEST_DATASOURCE_SEQUENCE: - nKey = sprintf(p->aKey, "%012d", iData); - break; - } - *ppKey = p->aKey; - *pnKey = nKey; - } - if( ppVal ){ - u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal; - testPrngString((u32)~iData, p->aVal, (int)nVal); - *ppVal = p->aVal; - *pnVal = (int)nVal; - } -} - -void testDatasourceFree(Datasource *p){ - testFree(p); -} - -/* -** Return a pointer to a nul-terminated string that corresponds to the -** contents of the datasource-definition passed as the first argument. -** The caller should eventually free the returned pointer using testFree(). -*/ -char *testDatasourceName(const DatasourceDefn *p){ - char *zRet; - zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)", - (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"), - p->nMinKey, p->nMaxKey, - p->nMinVal, p->nMaxVal - ); - return zRet; -} - -Datasource *testDatasourceNew(const DatasourceDefn *pDefn){ - Datasource *p; - int nMinKey; - int nMaxKey; - int nMinVal; - int nMaxVal; - - if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){ - nMinKey = 128; - nMaxKey = 128; - }else{ - nMinKey = MAX(0, pDefn->nMinKey); - nMaxKey = MAX(nMinKey, pDefn->nMaxKey); - } - nMinVal = MAX(0, pDefn->nMinVal); - nMaxVal = MAX(nMinVal, pDefn->nMaxVal); - - p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1); - p->eType = pDefn->eType; - p->nMinKey = nMinKey; - p->nMinVal = nMinVal; - p->nMaxKey = nMaxKey; - p->nMaxVal = nMaxVal; - - p->aKey = (char *)&p[1]; - p->aVal = &p->aKey[nMaxKey]; - return p; -}; diff --git a/ext/lsm1/lsm-test/lsmtest_func.c b/ext/lsm1/lsm-test/lsmtest_func.c deleted file mode 100644 index eb8346aa83..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_func.c +++ /dev/null @@ -1,177 +0,0 @@ - -#include "lsmtest.h" - - -int do_work(int nArg, char **azArg){ - struct Option { - const char *zName; - } aOpt [] = { - { "-nmerge" }, - { "-nkb" }, - { 0 } - }; - - lsm_db *pDb; - int rc; - int i; - const char *zDb; - int nMerge = 1; - int nKB = (1<<30); - - if( nArg==0 ) goto usage; - zDb = azArg[nArg-1]; - for(i=0; i<(nArg-1); i++){ - int iSel; - rc = testArgSelect(aOpt, "option", azArg[i], &iSel); - if( rc ) return rc; - switch( iSel ){ - case 0: - i++; - if( i==(nArg-1) ) goto usage; - nMerge = atoi(azArg[i]); - break; - case 1: - i++; - if( i==(nArg-1) ) goto usage; - nKB = atoi(azArg[i]); - break; - } - } - - rc = lsm_new(0, &pDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - }else{ - rc = lsm_open(pDb, zDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - }else{ - int n = -1; - lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n); - n = n*2; - lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n); - - rc = lsm_work(pDb, nMerge, nKB, 0); - if( rc!=LSM_OK ){ - testPrintError("lsm_work(): rc=%d\n", rc); - } - } - } - if( rc==LSM_OK ){ - rc = lsm_checkpoint(pDb, 0); - } - - lsm_close(pDb); - return rc; - - usage: - testPrintUsage("?-optimize? ?-n N? DATABASE"); - return -1; -} - - -/* -** lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO?? -*/ -int do_show(int nArg, char **azArg){ - lsm_db *pDb; - int rc; - const char *zDb; - - int eOpt = LSM_INFO_DB_STRUCTURE; - unsigned int iPg = 0; - int bConfig = 0; - const char *zConfig = ""; - - struct Option { - const char *zName; - int bConfig; - int eOpt; - } aOpt [] = { - { "array", 0, LSM_INFO_ARRAY_STRUCTURE }, - { "array-pages", 0, LSM_INFO_ARRAY_PAGES }, - { "blocksize", 1, LSM_CONFIG_BLOCK_SIZE }, - { "pagesize", 1, LSM_CONFIG_PAGE_SIZE }, - { "freelist", 0, LSM_INFO_FREELIST }, - { "page-ascii", 0, LSM_INFO_PAGE_ASCII_DUMP }, - { "page-hex", 0, LSM_INFO_PAGE_HEX_DUMP }, - { 0, 0 } - }; - - char *z = 0; - int iDb = 0; /* Index of DATABASE in azArg[] */ - - /* Check if there is a "-config" option: */ - if( nArg>2 && strlen(azArg[0])>1 - && memcmp(azArg[0], "-config", strlen(azArg[0]))==0 - ){ - zConfig = azArg[1]; - iDb = 2; - } - if( nArg<(iDb+1) ) goto usage; - - if( nArg>(iDb+1) ){ - rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt); - if( rc!=0 ) return rc; - bConfig = aOpt[eOpt].bConfig; - eOpt = aOpt[eOpt].eOpt; - if( (bConfig==0 && eOpt==LSM_INFO_FREELIST) - || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE) - || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE) - ){ - if( nArg!=(iDb+2) ) goto usage; - }else{ - if( nArg!=(iDb+3) ) goto usage; - iPg = atoi(azArg[iDb+2]); - } - } - zDb = azArg[iDb]; - - rc = lsm_new(0, &pDb); - tdb_lsm_configure(pDb, zConfig); - if( rc!=LSM_OK ){ - testPrintError("lsm_new(): rc=%d\n", rc); - }else{ - rc = lsm_open(pDb, zDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - } - } - - if( rc==LSM_OK ){ - if( bConfig==0 ){ - switch( eOpt ){ - case LSM_INFO_DB_STRUCTURE: - case LSM_INFO_FREELIST: - rc = lsm_info(pDb, eOpt, &z); - break; - case LSM_INFO_ARRAY_STRUCTURE: - case LSM_INFO_ARRAY_PAGES: - case LSM_INFO_PAGE_ASCII_DUMP: - case LSM_INFO_PAGE_HEX_DUMP: - rc = lsm_info(pDb, eOpt, iPg, &z); - break; - default: - assert( !"no chance" ); - } - - if( rc==LSM_OK ){ - printf("%s\n", z ? z : ""); - fflush(stdout); - } - lsm_free(lsm_get_env(pDb), z); - }else{ - int iRes = -1; - lsm_config(pDb, eOpt, &iRes); - printf("%d\n", iRes); - fflush(stdout); - } - } - - lsm_close(pDb); - return rc; - - usage: - testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?"); - return -1; -} diff --git a/ext/lsm1/lsm-test/lsmtest_io.c b/ext/lsm1/lsm-test/lsmtest_io.c deleted file mode 100644 index 7aa5d10948..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_io.c +++ /dev/null @@ -1,248 +0,0 @@ - -/* -** SUMMARY -** -** This file implements the 'io' subcommand of the test program. It is used -** for testing the performance of various combinations of write() and fsync() -** system calls. All operations occur on a single file, which may or may not -** exist when a test is started. -** -** A test consists of a series of commands. Each command is either a write -** or an fsync. A write is specified as "@", where -** is the amount of data written, and is the offset of the file -** to write to. An or an is specified as an integer number -** of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of -** KB, MB or GB, respectively. An fsync is simply "S". All commands are -** case-insensitive. -** -** Example test program: -** -** 2M@6M 1492K@4M S 4096@4K S -** -** This program writes 2 MB of data starting at the offset 6MB offset of -** the file, followed by 1492 KB of data written at the 4MB offset of the -** file, followed by a call to fsync(), a write of 4KB of data at byte -** offset 4096, and finally another call to fsync(). -** -** Commands may either be specified on the command line (one command per -** command line argument) or read from stdin. Commands read from stdin -** must be separated by white-space. -** -** COMMAND LINE INVOCATION -** -** The sub-command implemented in this file must be invoked with at least -** two arguments - the path to the file to write to and the page-size to -** use for writing. If there are more than two arguments, then each -** subsequent argument is assumed to be a test command. If there are exactly -** two arguments, the test commands are read from stdin. -** -** A write command does not result in a single call to system call write(). -** Instead, the specified region is written sequentially using one or -** more calls to write(), each of which writes not more than one page of -** data. For example, if the page-size is 4KB, the command "2M@6M" results -** in 512 calls to write(), each of which writes 4KB of data. -** -** EXAMPLES -** -** Two equivalent examples: -** -** $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S -** 3544K written in 129 ms -** $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 -** 3544K written in 127 ms -** -*/ - -#include "lsmtest.h" - -typedef struct IoContext IoContext; - -struct IoContext { - int fd; - int nWrite; -}; - -/* -** As isspace(3) -*/ -static int safe_isspace(char c){ - if( c&0x80) return 0; - return isspace(c); -} - -/* -** As isdigit(3) -*/ -static int safe_isdigit(char c){ - if( c&0x80) return 0; - return isdigit(c); -} - -static i64 getNextSize(char *zIn, char **pzOut, int *pRc){ - i64 iRet = 0; - if( *pRc==0 ){ - char *z = zIn; - - if( !safe_isdigit(*z) ){ - *pRc = 1; - return 0; - } - - /* Process digits */ - while( safe_isdigit(*z) ){ - iRet = iRet*10 + (*z - '0'); - z++; - } - - /* Process suffix */ - switch( *z ){ - case 'k': case 'K': - iRet = iRet * 1024; - z++; - break; - - case 'm': case 'M': - iRet = iRet * 1024 * 1024; - z++; - break; - - case 'g': case 'G': - iRet = iRet * 1024 * 1024 * 1024; - z++; - break; - } - - if( pzOut ) *pzOut = z; - } - return iRet; -} - -static int doOneCmd( - IoContext *pCtx, - u8 *aData, - int pgsz, - char *zCmd, - char **pzOut -){ - char c; - char *z = zCmd; - - while( safe_isspace(*z) ) z++; - c = *z; - - if( c==0 ){ - if( pzOut ) *pzOut = z; - return 0; - } - - if( c=='s' || c=='S' ){ - if( pzOut ) *pzOut = &z[1]; - return fdatasync(pCtx->fd); - } - - if( safe_isdigit(c) ){ - i64 iOff = 0; - int nByte = 0; - int rc = 0; - int nPg; - int iPg; - - nByte = (int)getNextSize(z, &z, &rc); - if( rc || *z!='@' ) goto bad_command; - z++; - iOff = getNextSize(z, &z, &rc); - if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command; - if( pzOut ) *pzOut = z; - - nPg = (nByte+pgsz-1) / pgsz; - lseek(pCtx->fd, (off_t)iOff, SEEK_SET); - for(iPg=0; iPgfd, aData, pgsz); - } - pCtx->nWrite += nByte/1024; - - return 0; - } - - bad_command: - testPrintError("unrecognized command: %s", zCmd); - return 1; -} - -static int readStdin(char **pzOut){ - int nAlloc = 128; - char *zOut = 0; - int nOut = 0; - - while( !feof(stdin) ){ - int nRead; - - nAlloc = nAlloc*2; - zOut = realloc(zOut, nAlloc); - nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin); - - if( nRead==0 ) break; - nOut += nRead; - zOut[nOut] = '\0'; - } - - *pzOut = zOut; - return 0; -} - -int do_io(int nArg, char **azArg){ - IoContext ctx; - int pgsz; - char *zFile; - char *zPgsz; - int i; - int rc = 0; - - char *zStdin = 0; - char *z; - - u8 *aData; - - memset(&ctx, 0, sizeof(IoContext)); - if( nArg<2 ){ - testPrintUsage("FILE PGSZ ?CMD-1 ...?"); - return -1; - } - zFile = azArg[0]; - zPgsz = azArg[1]; - - pgsz = (int)getNextSize(zPgsz, 0, &rc); - if( pgsz<=0 ){ - testPrintError("Ridiculous page size: %d", pgsz); - return -1; - } - aData = malloc(pgsz); - memset(aData, 0x77, pgsz); - - ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644); - if( ctx.fd<0 ){ - perror("open: "); - return -1; - } - - if( nArg==2 ){ - readStdin(&zStdin); - testTimeInit(); - z = zStdin; - while( *z && rc==0 ){ - rc = doOneCmd(&ctx, aData, pgsz, z, &z); - } - }else{ - testTimeInit(); - for(i=2; i - -void test_failed(){ - assert( 0 ); - return; -} - -#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__) -static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){ - if( rc ){ - *pRc = rc; - fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc); - test_failed(); - } -} - -static int lsm_memcmp(u8 *a, u8 *b, int c){ - int i; - for(i=0; i0 && lsm_memcmp(pVal, pDbVal, nVal))) ){ - testSetError(1); - } - } -} - -void testWrite( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - void *pVal, int nVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; -static int nCall = 0; -nCall++; - rc = tdb_write(pDb, pKey, nKey, pVal, nVal); - testSetError(rc); - } -} -void testDelete( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; - *pRc = rc = tdb_delete(pDb, pKey, nKey); - testSetError(rc); - } -} -void testDeleteRange( - TestDb *pDb, /* Database handle */ - void *pKey1, int nKey1, - void *pKey2, int nKey2, - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; - *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2); - testSetError(rc); - } -} - -void testBegin(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_begin(pDb, iTrans); - testSetError(rc); - } -} -void testCommit(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_commit(pDb, iTrans); - testSetError(rc); - } -} -#if 0 /* unused */ -static void testRollback(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_rollback(pDb, iTrans); - testSetError(rc); - } -} -#endif - -void testWriteStr( - TestDb *pDb, /* Database handle */ - const char *zKey, /* Key to query database for */ - const char *zVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - int nVal = (zVal ? strlen(zVal) : 0); - testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); -} - -#if 0 /* unused */ -static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){ - testDelete(pDb, (void *)zKey, strlen(zKey), pRc); -} -#endif -void testFetchStr( - TestDb *pDb, /* Database handle */ - const char *zKey, /* Key to query database for */ - const char *zVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - int nVal = (zVal ? strlen(zVal) : 0); - testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); -} - -void testFetchCompare( - TestDb *pControl, - TestDb *pDb, - void *pKey, int nKey, - int *pRc -){ - int rc; - void *pDbVal1; - void *pDbVal2; - int nDbVal1; - int nDbVal2; - - static int nCall = 0; - nCall++; - - rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1); - testSetError(rc); - - rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2); - testSetError(rc); - - if( *pRc==0 - && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1))) - ){ - testSetError(1); - } -} - -typedef struct ScanResult ScanResult; -struct ScanResult { - TestDb *pDb; - - int nRow; - u32 cksum1; - u32 cksum2; - void *pKey1; int nKey1; - void *pKey2; int nKey2; - - int bReverse; - int nPrevKey; - u8 aPrevKey[256]; -}; - -static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){ - int res; - res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2)); - if( res==0 ){ - res = nKey1 - nKey2; - } - return res; -} - -int test_scan_debug = 0; - -static void scanCompareCb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - ScanResult *p = (ScanResult *)pCtx; - u8 *aKey = (u8 *)pKey; - u8 *aVal = (u8 *)pVal; - int i; - - if( test_scan_debug ){ - printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey); - fflush(stdout); - } -#if 0 - if( test_scan_debug ) printf("%.20s\n", (char *)pVal); -#endif - -#if 0 - /* Check tdb_fetch() matches */ - int rc = 0; - testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc); - assert( rc==0 ); -#endif - - /* Update the checksum data */ - p->nRow++; - for(i=0; icksum1 += ((int)aKey[i] << (i&0x0F)); - p->cksum2 += p->cksum1; - } - for(i=0; icksum1 += ((int)aVal[i] << (i&0x0F)); - p->cksum2 += p->cksum1; - } - - /* Check that the delivered row is not out of order. */ - if( nKey<(int)sizeof(p->aPrevKey) ){ - if( p->nPrevKey ){ - int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey); - if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){ - testPrintError("Returned key out of order at %s:%d\n", - __FILE__, __LINE__ - ); - } - } - - p->nPrevKey = nKey; - memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey)); - } - - /* Check that the delivered row is within range. */ - if( p->pKey1 && ( - (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0) - || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey) - )){ - testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__); - } - if( p->pKey2 && ( - (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0) - || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2=0 ); - zRet = (char *)testMalloc(nByte+1); - vsnprintf(zRet, nByte+1, zFormat, ap); - return zRet; -} - -char *testMallocPrintf(const char *zFormat, ...){ - va_list ap; - char *zRet; - - va_start(ap, zFormat); - zRet = testMallocVPrintf(zFormat, ap); - va_end(ap); - - return zRet; -} - - -/* -** A wrapper around malloc(3). -** -** This function should be used for all allocations made by test procedures. -** It has the following properties: -** -** * Test code may assume that allocations may not fail. -** * Returned memory is always zeroed. -** -** Allocations made using testMalloc() should be freed using testFree(). -*/ -void *testMalloc(int n){ - u8 *p = (u8*)malloc(n + 8); - memset(p, 0, n+8); - *(int*)p = n; - return (void*)&p[8]; -} - -void *testMallocCopy(void *pCopy, int nByte){ - void *pRet = testMalloc(nByte); - memcpy(pRet, pCopy, nByte); - return pRet; -} - -void *testRealloc(void *ptr, int n){ - if( ptr ){ - u8 *p = (u8*)ptr - 8; - int nOrig = *(int*)p; - p = (u8*)realloc(p, n+8); - if( nOrig1 ){ - testPrintError("Usage: test ?PATTERN?\n"); - return 1; - } - if( nArg==1 ){ - zPattern = azArg[0]; - } - - for(j=0; tdb_system_name(j); j++){ - rc = 0; - - test_data_1(tdb_system_name(j), zPattern, &rc); - test_data_2(tdb_system_name(j), zPattern, &rc); - test_data_3(tdb_system_name(j), zPattern, &rc); - test_data_4(tdb_system_name(j), zPattern, &rc); - test_rollback(tdb_system_name(j), zPattern, &rc); - test_mc(tdb_system_name(j), zPattern, &rc); - test_mt(tdb_system_name(j), zPattern, &rc); - - if( rc ) nFail++; - } - - rc = 0; - test_oom(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - test_api(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - do_crash_test(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - do_writer_crash_test(zPattern, &rc); - if( rc ) nFail++; - - return (nFail!=0); -} - -static lsm_db *configure_lsm_db(TestDb *pDb){ - lsm_db *pLsm; - pLsm = tdb_lsm(pDb); - if( pLsm ){ - tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4"); - } - return pLsm; -} - -typedef struct WriteHookEvent WriteHookEvent; -struct WriteHookEvent { - i64 iOff; - int nData; - int nUs; -}; -WriteHookEvent prev = {0, 0, 0}; - -static void flushPrev(FILE *pOut){ - if( prev.nData ){ - fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs); - prev.nData = 0; - } -} - -#if 0 /* unused */ -static void do_speed_write_hook2( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - FILE *pOut = (FILE *)pCtx; - if( bLog ) return; - - if( prev.nData && nData && iOff==prev.iOff+prev.nData ){ - prev.nData += nData; - prev.nUs += nUs; - }else{ - flushPrev(pOut); - if( nData==0 ){ - fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs); - }else{ - prev.iOff = iOff; - prev.nData = nData; - prev.nUs = nUs; - } - } -} -#endif - -#define ST_REPEAT 0 -#define ST_WRITE 1 -#define ST_PAUSE 2 -#define ST_FETCH 3 -#define ST_SCAN 4 -#define ST_NSCAN 5 -#define ST_KEYSIZE 6 -#define ST_VALSIZE 7 -#define ST_TRANS 8 - - -static void print_speed_test_help(){ - printf( -"\n" -"Repeat the following $repeat times:\n" -" 1. Insert $write key-value pairs. One transaction for each write op.\n" -" 2. Pause for $pause ms.\n" -" 3. Perform $fetch queries on the database.\n" -"\n" -" Keys are $keysize bytes in size. Values are $valsize bytes in size\n" -" Both keys and values are pseudo-randomly generated\n" -"\n" -"Options are:\n" -" -repeat $repeat (default value 10)\n" -" -write $write (default value 10000)\n" -" -pause $pause (default value 0)\n" -" -fetch $fetch (default value 0)\n" -" -keysize $keysize (default value 12)\n" -" -valsize $valsize (default value 100)\n" -" -system $system (default value \"lsm\")\n" -" -trans $trans (default value 0)\n" -"\n" -); -} - -int do_speed_test2(int nArg, char **azArg){ - struct Option { - const char *zOpt; - int eVal; - int iDefault; - } aOpt[] = { - { "-repeat", ST_REPEAT, 10}, - { "-write", ST_WRITE, 10000}, - { "-pause", ST_PAUSE, 0}, - { "-fetch", ST_FETCH, 0}, - { "-scan", ST_SCAN, 0}, - { "-nscan", ST_NSCAN, 0}, - { "-keysize", ST_KEYSIZE, 12}, - { "-valsize", ST_VALSIZE, 100}, - { "-trans", ST_TRANS, 0}, - { "-system", -1, 0}, - { "help", -2, 0}, - {0, 0, 0} - }; - int i; - int aParam[9]; - int rc = 0; - int bReadonly = 0; - int nContent = 0; - - TestDb *pDb; - Datasource *pData; - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 }; - char *zSystem = ""; - int bLsm = 1; - FILE *pLog = 0; - -#ifdef NDEBUG - /* If NDEBUG is defined, disable the dynamic memory related checks in - ** lsmtest_mem.c. They slow things down. */ - testMallocUninstall(tdb_lsm_env()); -#endif - - /* Initialize aParam[] with default values. */ - for(i=0; i=0 ){ - aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]); - }else{ - zSystem = azArg[i+1]; - bLsm = 0; -#if 0 - for(j=0; zSystem[j]; j++){ - if( zSystem[j]=='=' ) bLsm = 1; - } -#endif - } - } - - printf("#"); - for(i=0; i=0 ){ - printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]); - }else if( aOpt[i].eVal==-1 ){ - printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem); - } - } - } - printf("\n"); - - defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE]; - defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE]; - pData = testDatasourceNew(&defn); - - if( aParam[ST_WRITE]==0 ){ - bReadonly = 1; - } - - if( bLsm ){ - rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb); - }else{ - pDb = testOpen(zSystem, !bReadonly, &rc); - } - if( rc!=0 ) return rc; - if( bReadonly ){ - nContent = testCountDatabase(pDb); - } - -#if 0 - pLog = fopen("/tmp/speed.log", "w"); - tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); -#endif - - for(i=0; i=nArg ){ - testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); - return 1; - } - if( aOpt[iSel].isSwitch==1 ){ - nRow = atoi(azArg[i]); - } - if( aOpt[iSel].isSwitch==2 ){ - nSleep = atoi(azArg[i]); - } - if( aOpt[iSel].isSwitch==3 ){ - struct Mode { - const char *zMode; - int doReadTest; - int doWriteTest; - } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}}; - int iMode; - rc = testArgSelect(aMode, "option", azArg[i], &iMode); - if( rc ) return rc; - doReadTest = aMode[iMode].doReadTest; - doWriteTest = aMode[iMode].doWriteTest; - } - if( aOpt[iSel].isSwitch==4 ){ - /* The "-out FILE" switch. This option is used to specify a file to - ** write the gnuplot script to. */ - zOut = azArg[i]; - } - }else{ - /* A db name */ - rc = testArgSelect(aOpt, "system", azArg[i], &iSel); - if( rc ) return rc; - sys_mask |= (1< 100000) ? 100000 : nSelStep; - - aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep); - aWrite = malloc(sizeof(int) * nRow/nStep); - aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep); - - /* This loop collects the INSERT speed data. */ - if( doWriteTest ){ - printf("Writing output to file \"%s\".\n", zOut); - - for(j=0; aSys[j].zLibrary; j++){ - FILE *pLog = 0; - TestDb *pDb; /* Database being tested */ - lsm_db *pLsm; - int iDot = 0; - - if( ((1<nData ){ - fprintf(pHook->pOut, "write %s %d %d\n", - (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData - ); - pHook->nData = 0; - fflush(pHook->pOut); - } -} - -static void do_insert_write_hook( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - InsertWriteHook *pHook = (InsertWriteHook *)pCtx; - if( bLog ) return; - - if( nData==0 ){ - flushHook(pHook); - fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db")); - }else if( pHook->nData - && bLog==pHook->bLog - && iOff==(pHook->iOff+pHook->nData) - ){ - pHook->nData += nData; - }else{ - flushHook(pHook); - pHook->bLog = bLog; - pHook->iOff = iOff; - pHook->nData = nData; - } -} - -static int do_replay(int nArg, char **azArg){ - char aBuf[4096]; - FILE *pInput; - FILE *pClose = 0; - const char *zDb; - - lsm_env *pEnv; - lsm_file *pOut; - int rc; - - if( nArg!=2 ){ - testPrintError("Usage: replay WRITELOG FILE\n"); - return 1; - } - - if( strcmp(azArg[0], "-")==0 ){ - pInput = stdin; - }else{ - pClose = pInput = fopen(azArg[0], "r"); - } - zDb = azArg[1]; - pEnv = tdb_lsm_env(); - rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); - if( rc!=LSM_OK ) return rc; - - while( feof(pInput)==0 ){ - char zLine[80]; - fgets(zLine, sizeof(zLine)-1, pInput); - zLine[sizeof(zLine)-1] = '\0'; - - if( 0==memcmp("sync db", zLine, 7) ){ - rc = pEnv->xSync(pOut); - if( rc!=0 ) break; - }else{ - int iOff; - int nData; - int nMatch; - nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData); - if( nMatch==2 ){ - int i; - for(i=0; ixWrite(pOut, iOff+i, aBuf, sizeof(aBuf)); - if( rc!=0 ) break; - } - } - } - } - if( pClose ) fclose(pClose); - pEnv->xClose(pOut); - - return rc; -} - -static int do_insert(int nArg, char **azArg){ - const char *zDb = "lsm"; - TestDb *pDb = 0; - int i; - int rc; - const int nRow = 1 * 1000 * 1000; - - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 }; - Datasource *pData = 0; - - if( nArg>1 ){ - testPrintError("Usage: insert ?DATABASE?\n"); - return 1; - } - if( nArg==1 ){ zDb = azArg[0]; } - - testMallocUninstall(tdb_lsm_env()); - for(i=0; zDb[i] && zDb[i]!='='; i++); - if( zDb[i] ){ - rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb); - }else{ - rc = tdb_open(zDb, 0, 1, &pDb); - } - - if( rc!=0 ){ - testPrintError("Error opening db \"%s\": %d\n", zDb, rc); - }else{ - InsertWriteHook hook; - memset(&hook, 0, sizeof(hook)); - hook.pOut = fopen("writelog.txt", "w"); - - pData = testDatasourceNew(&defn); - tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0); - tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook); - - if( rc==0 ){ - for(i=0; i -#include - -static void lsmtest_rusage_report(void){ - struct rusage r; - memset(&r, 0, sizeof(r)); - - getrusage(RUSAGE_SELF, &r); - printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", - (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock - ); -} -#else -static void lsmtest_rusage_report(void){ - /* no-op */ -} -#endif - -int main(int argc, char **argv){ - struct TestFunc { - const char *zName; - int bRusageReport; - int (*xFunc)(int, char **); - } aTest[] = { - {"random", 1, do_random_tests}, - {"writespeed", 1, do_writer_test}, - {"io", 1, st_do_io}, - - {"insert", 1, do_insert}, - {"replay", 1, do_replay}, - - {"speed", 1, do_speed_tests}, - {"speed2", 1, do_speed_test2}, - {"show", 0, st_do_show}, - {"work", 1, st_do_work}, - {"test", 1, do_test}, - - {0, 0} - }; - int rc; /* Return Code */ - int iFunc; /* Index into aTest[] */ - - int nLeakAlloc = 0; /* Allocations leaked by lsm */ - int nLeakByte = 0; /* Bytes leaked by lsm */ - -#ifdef LSM_DEBUG_MEM - FILE *pReport = 0; /* lsm malloc() report file */ - const char *zReport = "malloc.txt generated"; -#else - const char *zReport = "malloc.txt NOT generated"; -#endif - - testMallocInstall(tdb_lsm_env()); - - if( argc<2 ){ - testPrintError("Usage: %s sub-command ?args...?\n", argv[0]); - return -1; - } - - /* Initialize error reporting */ - testErrorInit(argc, argv); - - /* Initialize PRNG system */ - testPrngInit(); - - rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc); - if( rc==0 ){ - rc = aTest[iFunc].xFunc(argc-2, &argv[2]); - } - -#ifdef LSM_DEBUG_MEM - pReport = fopen("malloc.txt", "w"); - testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport); - fclose(pReport); -#else - testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0); -#endif - - if( nLeakAlloc ){ - testPrintError("Leaked %d bytes in %d allocations (%s)\n", - nLeakByte, nLeakAlloc, zReport - ); - if( rc==0 ) rc = -1; - } - testMallocUninstall(tdb_lsm_env()); - - if( aTest[iFunc].bRusageReport ){ - lsmtest_rusage_report(); - } - return rc; -} diff --git a/ext/lsm1/lsm-test/lsmtest_mem.c b/ext/lsm1/lsm-test/lsmtest_mem.c deleted file mode 100644 index 4c35e849f2..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_mem.c +++ /dev/null @@ -1,409 +0,0 @@ - -#include -#include -#include - -#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) - -#define MIN(x,y) ((x)<(y) ? (x) : (y)) - -typedef unsigned int u32; -typedef unsigned char u8; -typedef long long int i64; -typedef unsigned long long int u64; - -#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM) - extern int backtrace(void**,int); - extern void backtrace_symbols_fd(void*const*,int,int); -# define TM_BACKTRACE 12 -#else -# define backtrace(A,B) 1 -# define backtrace_symbols_fd(A,B,C) -#endif - - -typedef struct TmBlockHdr TmBlockHdr; -typedef struct TmAgg TmAgg; -typedef struct TmGlobal TmGlobal; - -struct TmGlobal { - /* Linked list of all currently outstanding allocations. And a table of - ** all allocations, past and present, indexed by backtrace() info. */ - TmBlockHdr *pFirst; -#ifdef TM_BACKTRACE - TmAgg *aHash[10000]; -#endif - - /* Underlying malloc/realloc/free functions */ - void *(*xMalloc)(int); /* underlying malloc(3) function */ - void *(*xRealloc)(void *, int); /* underlying realloc(3) function */ - void (*xFree)(void *); /* underlying free(3) function */ - - /* Mutex to protect pFirst and aHash */ - void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */ - void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */ - void (*xDelMutex)(TmGlobal*); /* Call this to delete mutex */ - void *pMutex; /* Mutex handle */ - - void *(*xSaveMalloc)(void *, size_t); - void *(*xSaveRealloc)(void *, void *, size_t); - void (*xSaveFree)(void *, void *); - - /* OOM injection scheduling. If nCountdown is greater than zero when a - ** malloc attempt is made, it is decremented. If this means nCountdown - ** transitions from 1 to 0, then the allocation fails. If bPersist is true - ** when this happens, nCountdown is then incremented back to 1 (so that the - ** next attempt fails too). - */ - int nCountdown; - int bPersist; - int bEnable; - void (*xHook)(void *); - void *pHookCtx; -}; - -struct TmBlockHdr { - TmBlockHdr *pNext; - TmBlockHdr *pPrev; - int nByte; -#ifdef TM_BACKTRACE - TmAgg *pAgg; -#endif - u32 iForeGuard; -}; - -#ifdef TM_BACKTRACE -struct TmAgg { - int nAlloc; /* Number of allocations at this path */ - int nByte; /* Total number of bytes allocated */ - int nOutAlloc; /* Number of outstanding allocations */ - int nOutByte; /* Number of outstanding bytes */ - void *aFrame[TM_BACKTRACE]; /* backtrace() output */ - TmAgg *pNext; /* Next object in hash-table collision */ -}; -#endif - -#define FOREGUARD 0x80F5E153 -#define REARGUARD 0xE4676B53 -static const u32 rearguard = REARGUARD; - -#define ROUND8(x) (((x)+7)&~7) - -#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) )) - -static void lsmtest_oom_error(void){ - static int nErr = 0; - nErr++; -} - -static void tmEnterMutex(TmGlobal *pTm){ - pTm->xEnterMutex(pTm); -} -static void tmLeaveMutex(TmGlobal *pTm){ - pTm->xLeaveMutex(pTm); -} - -static void *tmMalloc(TmGlobal *pTm, int nByte){ - TmBlockHdr *pNew; /* New allocation header block */ - u8 *pUser; /* Return value */ - int nReq; /* Total number of bytes requested */ - - assert( sizeof(rearguard)==4 ); - nReq = BLOCK_HDR_SIZE + nByte + 4; - pNew = (TmBlockHdr *)pTm->xMalloc(nReq); - memset(pNew, 0, sizeof(TmBlockHdr)); - - tmEnterMutex(pTm); - assert( pTm->nCountdown>=0 ); - assert( pTm->bPersist==0 || pTm->bPersist==1 ); - - if( pTm->bEnable && pTm->nCountdown==1 ){ - /* Simulate an OOM error. */ - lsmtest_oom_error(); - pTm->xFree(pNew); - pTm->nCountdown = pTm->bPersist; - if( pTm->xHook ) pTm->xHook(pTm->pHookCtx); - pUser = 0; - }else{ - if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--; - - pNew->iForeGuard = FOREGUARD; - pNew->nByte = nByte; - pNew->pNext = pTm->pFirst; - - if( pTm->pFirst ){ - pTm->pFirst->pPrev = pNew; - } - pTm->pFirst = pNew; - - pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE]; - memset(pUser, 0x56, nByte); - memcpy(&pUser[nByte], &rearguard, 4); - -#ifdef TM_BACKTRACE - { - TmAgg *pAgg; - int i; - u32 iHash = 0; - void *aFrame[TM_BACKTRACE]; - memset(aFrame, 0, sizeof(aFrame)); - backtrace(aFrame, TM_BACKTRACE); - - for(i=0; iaHash); - - for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){ - if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break; - } - if( !pAgg ){ - pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg)); - memset(pAgg, 0, sizeof(TmAgg)); - memcpy(pAgg->aFrame, aFrame, sizeof(aFrame)); - pAgg->pNext = pTm->aHash[iHash]; - pTm->aHash[iHash] = pAgg; - } - pAgg->nAlloc++; - pAgg->nByte += nByte; - pAgg->nOutAlloc++; - pAgg->nOutByte += nByte; - pNew->pAgg = pAgg; - } -#endif - } - - tmLeaveMutex(pTm); - return pUser; -} - -static void tmFree(TmGlobal *pTm, void *p){ - if( p ){ - TmBlockHdr *pHdr; - u8 *pUser = (u8 *)p; - - tmEnterMutex(pTm); - pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); - assert( pHdr->iForeGuard==FOREGUARD ); - assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) ); - - if( pHdr->pPrev ){ - assert( pHdr->pPrev->pNext==pHdr ); - pHdr->pPrev->pNext = pHdr->pNext; - }else{ - assert( pHdr==pTm->pFirst ); - pTm->pFirst = pHdr->pNext; - } - if( pHdr->pNext ){ - assert( pHdr->pNext->pPrev==pHdr ); - pHdr->pNext->pPrev = pHdr->pPrev; - } - -#ifdef TM_BACKTRACE - pHdr->pAgg->nOutAlloc--; - pHdr->pAgg->nOutByte -= pHdr->nByte; -#endif - - tmLeaveMutex(pTm); - memset(pUser, 0x58, pHdr->nByte); - memset(pHdr, 0x57, sizeof(TmBlockHdr)); - pTm->xFree(pHdr); - } -} - -static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){ - void *pNew; - - pNew = tmMalloc(pTm, nByte); - if( pNew && p ){ - TmBlockHdr *pHdr; - u8 *pUser = (u8 *)p; - pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); - memcpy(pNew, p, MIN(nByte, pHdr->nByte)); - tmFree(pTm, p); - } - return pNew; -} - -static void tmMallocOom( - TmGlobal *pTm, - int nCountdown, - int bPersist, - void (*xHook)(void *), - void *pHookCtx -){ - assert( nCountdown>=0 ); - assert( bPersist==0 || bPersist==1 ); - pTm->nCountdown = nCountdown; - pTm->bPersist = bPersist; - pTm->xHook = xHook; - pTm->pHookCtx = pHookCtx; - pTm->bEnable = 1; -} - -static void tmMallocOomEnable( - TmGlobal *pTm, - int bEnable -){ - pTm->bEnable = bEnable; -} - -static void tmMallocCheck( - TmGlobal *pTm, - int *pnLeakAlloc, - int *pnLeakByte, - FILE *pFile -){ - TmBlockHdr *pHdr; - int nLeak = 0; - int nByte = 0; - - if( pTm==0 ) return; - - for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){ - nLeak++; - nByte += pHdr->nByte; - } - if( pnLeakAlloc ) *pnLeakAlloc = nLeak; - if( pnLeakByte ) *pnLeakByte = nByte; - -#ifdef TM_BACKTRACE - if( pFile ){ - int i; - fprintf(pFile, "LEAKS\n"); - for(i=0; iaHash); i++){ - TmAgg *pAgg; - for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ - if( pAgg->nOutAlloc ){ - int j; - fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc); - for(j=0; jaFrame[j]); - } - fprintf(pFile, "\n"); - } - } - } - fprintf(pFile, "\nALLOCATIONS\n"); - for(i=0; iaHash); i++){ - TmAgg *pAgg; - for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ - int j; - fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc); - for(j=0; jaFrame[j]); - fprintf(pFile, "\n"); - } - } - } -#else - (void)pFile; -#endif -} - - -#include "lsm.h" -#include "stdlib.h" - -typedef struct LsmMutex LsmMutex; -struct LsmMutex { - lsm_env *pEnv; - lsm_mutex *pMutex; -}; - -static void tmLsmMutexEnter(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)pTm->pMutex; - p->pEnv->xMutexEnter(p->pMutex); -} -static void tmLsmMutexLeave(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)(pTm->pMutex); - p->pEnv->xMutexLeave(p->pMutex); -} -static void tmLsmMutexDel(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)pTm->pMutex; - pTm->xFree(p); -} -static void *tmLsmMalloc(int n){ return malloc(n); } -static void tmLsmFree(void *ptr){ free(ptr); } -static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); } - -static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ - return tmMalloc((TmGlobal *)(p->pMemCtx), n); -} -static void tmLsmEnvFree(lsm_env *p, void *ptr){ - tmFree((TmGlobal *)(p->pMemCtx), ptr); -} -static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ - return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n); -} - -void testMallocInstall(lsm_env *pEnv){ - TmGlobal *pGlobal; - LsmMutex *pMutex; - assert( pEnv->pMemCtx==0 ); - - /* Allocate and populate a TmGlobal structure. */ - pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal)); - memset(pGlobal, 0, sizeof(TmGlobal)); - pGlobal->xMalloc = tmLsmMalloc; - pGlobal->xRealloc = tmLsmRealloc; - pGlobal->xFree = tmLsmFree; - pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex)); - pMutex->pEnv = pEnv; - pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex); - pGlobal->xEnterMutex = tmLsmMutexEnter; - pGlobal->xLeaveMutex = tmLsmMutexLeave; - pGlobal->xDelMutex = tmLsmMutexDel; - pGlobal->pMutex = (void *)pMutex; - - pGlobal->xSaveMalloc = pEnv->xMalloc; - pGlobal->xSaveRealloc = pEnv->xRealloc; - pGlobal->xSaveFree = pEnv->xFree; - - /* Set up pEnv to the use the new TmGlobal */ - pEnv->pMemCtx = (void *)pGlobal; - pEnv->xMalloc = tmLsmEnvMalloc; - pEnv->xRealloc = tmLsmEnvRealloc; - pEnv->xFree = tmLsmEnvFree; -} - -void testMallocUninstall(lsm_env *pEnv){ - TmGlobal *p = (TmGlobal *)pEnv->pMemCtx; - pEnv->pMemCtx = 0; - if( p ){ - pEnv->xMalloc = p->xSaveMalloc; - pEnv->xRealloc = p->xSaveRealloc; - pEnv->xFree = p->xSaveFree; - p->xDelMutex(p); - tmLsmFree(p); - } -} - -void testMallocCheck( - lsm_env *pEnv, - int *pnLeakAlloc, - int *pnLeakByte, - FILE *pFile -){ - if( pEnv->pMemCtx==0 ){ - *pnLeakAlloc = 0; - *pnLeakByte = 0; - }else{ - tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile); - } -} - -void testMallocOom( - lsm_env *pEnv, - int nCountdown, - int bPersist, - void (*xHook)(void *), - void *pHookCtx -){ - TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); - tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx); -} - -void testMallocOomEnable(lsm_env *pEnv, int bEnable){ - TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); - tmMallocOomEnable(pTm, bEnable); -} diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.c b/ext/lsm1/lsm-test/lsmtest_tdb.c deleted file mode 100644 index 8f63f64acb..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb.c +++ /dev/null @@ -1,846 +0,0 @@ - -/* -** This program attempts to test the correctness of some facets of the -** LSM database library. Specifically, that the contents of the database -** are maintained correctly during a series of inserts and deletes. -*/ - - -#include "lsmtest_tdb.h" -#include "lsm.h" - -#include "lsmtest.h" - -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include - - -typedef struct SqlDb SqlDb; - -static int error_transaction_function(TestDb *p, int iLevel){ - unused_parameter(p); - unused_parameter(iLevel); - return -1; -} - - -/************************************************************************* -** Begin wrapper for LevelDB. -*/ -#ifdef HAVE_LEVELDB - -#include - -typedef struct LevelDb LevelDb; -struct LevelDb { - TestDb base; - leveldb_t *db; - leveldb_options_t *pOpt; - leveldb_writeoptions_t *pWriteOpt; - leveldb_readoptions_t *pReadOpt; - - char *pVal; -}; - -static int test_leveldb_close(TestDb *pTestDb){ - LevelDb *pDb = (LevelDb *)pTestDb; - - leveldb_close(pDb->db); - leveldb_writeoptions_destroy(pDb->pWriteOpt); - leveldb_readoptions_destroy(pDb->pReadOpt); - leveldb_options_destroy(pDb->pOpt); - free(pDb->pVal); - free(pDb); - - return 0; -} - -static int test_leveldb_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr); - return (zErr!=0); -} - -static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr); - return (zErr!=0); -} - -static int test_leveldb_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - size_t nVal = 0; - - if( pKey==0 ) return 0; - free(pDb->pVal); - pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr); - *ppVal = (void *)(pDb->pVal); - if( pDb->pVal==0 ){ - *pnVal = -1; - }else{ - *pnVal = (int)nVal; - } - - return (zErr!=0); -} - -static int test_leveldb_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *, void *, int , void *, int) -){ - LevelDb *pDb = (LevelDb *)pTestDb; - leveldb_iterator_t *iter; - - iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt); - - if( bReverse==0 ){ - if( pKey1 ){ - leveldb_iter_seek(iter, pKey1, nKey1); - }else{ - leveldb_iter_seek_to_first(iter); - } - }else{ - if( pKey2 ){ - leveldb_iter_seek(iter, pKey2, nKey2); - - if( leveldb_iter_valid(iter)==0 ){ - leveldb_iter_seek_to_last(iter); - }else{ - const char *k; size_t n; - int res; - k = leveldb_iter_key(iter, &n); - res = memcmp(k, pKey2, MIN(n, nKey2)); - if( res==0 ) res = n - nKey2; - assert( res>=0 ); - if( res>0 ){ - leveldb_iter_prev(iter); - } - } - }else{ - leveldb_iter_seek_to_last(iter); - } - } - - - while( leveldb_iter_valid(iter) ){ - const char *k; size_t n; - const char *v; size_t n2; - int res; - - k = leveldb_iter_key(iter, &n); - if( bReverse==0 && pKey2 ){ - res = memcmp(k, pKey2, MIN(n, nKey2)); - if( res==0 ) res = n - nKey2; - if( res>0 ) break; - } - if( bReverse!=0 && pKey1 ){ - res = memcmp(k, pKey1, MIN(n, nKey1)); - if( res==0 ) res = n - nKey1; - if( res<0 ) break; - } - - v = leveldb_iter_value(iter, &n2); - - xCallback(pCtx, (void *)k, n, (void *)v, n2); - - if( bReverse==0 ){ - leveldb_iter_next(iter); - }else{ - leveldb_iter_prev(iter); - } - } - - leveldb_iter_destroy(iter); - return 0; -} - -static int test_leveldb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods LeveldbMethods = { - test_leveldb_close, - test_leveldb_write, - test_leveldb_delete, - 0, - test_leveldb_fetch, - test_leveldb_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - LevelDb *pLevelDb; - char *zErr = 0; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); - memset(pLevelDb, 0, sizeof(LevelDb)); - - pLevelDb->pOpt = leveldb_options_create(); - leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1); - pLevelDb->pWriteOpt = leveldb_writeoptions_create(); - pLevelDb->pReadOpt = leveldb_readoptions_create(); - - pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr); - - if( zErr ){ - test_leveldb_close((TestDb *)pLevelDb); - *ppDb = 0; - return 1; - } - - *ppDb = (TestDb *)pLevelDb; - pLevelDb->base.pMethods = &LeveldbMethods; - return 0; -} -#endif /* HAVE_LEVELDB */ -/* -** End wrapper for LevelDB. -*************************************************************************/ - -#ifdef HAVE_KYOTOCABINET -static int kc_close(TestDb *pTestDb){ - return test_kc_close(pTestDb); -} - -static int kc_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - return test_kc_write(pTestDb, pKey, nKey, pVal, nVal); -} - -static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){ - return test_kc_delete(pTestDb, pKey, nKey); -} - -static int kc_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2); -} - -static int kc_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - if( pKey==0 ) return LSM_OK; - return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal); -} - -static int kc_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - return test_kc_scan( - pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback - ); -} - -static int kc_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods KcdbMethods = { - kc_close, - kc_write, - kc_delete, - kc_delete_range, - kc_fetch, - kc_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - int rc; - TestDb *pTestDb = 0; - - rc = test_kc_open(zFilename, bClear, &pTestDb); - if( rc!=0 ){ - *ppDb = 0; - return rc; - } - pTestDb->pMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_KYOTOCABINET */ -/* -** End wrapper for Kyoto cabinet. -*************************************************************************/ - -#ifdef HAVE_MDB -static int mdb_close(TestDb *pTestDb){ - return test_mdb_close(pTestDb); -} - -static int mdb_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal); -} - -static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){ - return test_mdb_delete(pTestDb, pKey, nKey); -} - -static int mdb_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - if( pKey==0 ) return LSM_OK; - return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal); -} - -static int mdb_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - return test_mdb_scan( - pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback - ); -} - -static int mdb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods KcdbMethods = { - mdb_close, - mdb_write, - mdb_delete, - 0, - mdb_fetch, - mdb_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - int rc; - TestDb *pTestDb = 0; - - rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb); - if( rc!=0 ){ - *ppDb = 0; - return rc; - } - pTestDb->pMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_MDB */ - -/************************************************************************* -** Begin wrapper for SQLite. -*/ - -/* -** nOpenTrans: -** The number of open nested transactions, in the same sense as used -** by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this -** value is 0, there are no transactions open at all. If it is 1, then -** there is a read transaction. If it is 2 or greater, then there are -** (nOpenTrans-1) nested write transactions open. -*/ -struct SqlDb { - TestDb base; - sqlite3 *db; - sqlite3_stmt *pInsert; - sqlite3_stmt *pDelete; - sqlite3_stmt *pDeleteRange; - sqlite3_stmt *pFetch; - sqlite3_stmt *apScan[8]; - - int nOpenTrans; - - /* Used by sql_fetch() to allocate space for results */ - int nAlloc; - u8 *aAlloc; -}; - -static int sql_close(TestDb *pTestDb){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_finalize(pDb->pInsert); - sqlite3_finalize(pDb->pDelete); - sqlite3_finalize(pDb->pDeleteRange); - sqlite3_finalize(pDb->pFetch); - sqlite3_finalize(pDb->apScan[0]); - sqlite3_finalize(pDb->apScan[1]); - sqlite3_finalize(pDb->apScan[2]); - sqlite3_finalize(pDb->apScan[3]); - sqlite3_finalize(pDb->apScan[4]); - sqlite3_finalize(pDb->apScan[5]); - sqlite3_finalize(pDb->apScan[6]); - sqlite3_finalize(pDb->apScan[7]); - sqlite3_close(pDb->db); - free((char *)pDb->aAlloc); - free((char *)pDb); - return SQLITE_OK; -} - -static int sql_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC); - sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC); - sqlite3_step(pDb->pInsert); - return sqlite3_reset(pDb->pInsert); -} - -static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC); - sqlite3_step(pDb->pDelete); - return sqlite3_reset(pDb->pDelete); -} - -static int sql_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC); - sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC); - sqlite3_step(pDb->pDeleteRange); - return sqlite3_reset(pDb->pDeleteRange); -} - -static int sql_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - SqlDb *pDb = (SqlDb *)pTestDb; - int rc; - - sqlite3_reset(pDb->pFetch); - if( pKey==0 ){ - assert( ppVal==0 ); - assert( pnVal==0 ); - return LSM_OK; - } - - sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC); - rc = sqlite3_step(pDb->pFetch); - if( rc==SQLITE_ROW ){ - int nVal = sqlite3_column_bytes(pDb->pFetch, 0); - u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0); - - if( nVal>pDb->nAlloc ){ - free(pDb->aAlloc); - pDb->aAlloc = (u8 *)malloc(nVal*2); - pDb->nAlloc = nVal*2; - } - memcpy(pDb->aAlloc, aVal, nVal); - *pnVal = nVal; - *ppVal = (void *)pDb->aAlloc; - }else{ - *pnVal = -1; - *ppVal = 0; - } - - rc = sqlite3_reset(pDb->pFetch); - return rc; -} - -static int sql_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_stmt *pScan; - - assert( bReverse==1 || bReverse==0 ); - pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4]; - - if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC); - if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC); - - while( SQLITE_ROW==sqlite3_step(pScan) ){ - void *pKey; int nKey; - void *pVal; int nVal; - - nKey = sqlite3_column_bytes(pScan, 0); - pKey = (void *)sqlite3_column_blob(pScan, 0); - nVal = sqlite3_column_bytes(pScan, 1); - pVal = (void *)sqlite3_column_blob(pScan, 1); - - xCallback(pCtx, pKey, nKey, pVal, nVal); - } - return sqlite3_reset(pScan); -} - -static int sql_begin(TestDb *pTestDb, int iLevel){ - int i; - SqlDb *pDb = (SqlDb *)pTestDb; - - /* iLevel==0 is a no-op */ - if( iLevel==0 ) return 0; - - /* If there are no transactions at all open, open a read transaction. */ - if( pDb->nOpenTrans==0 ){ - int rc = sqlite3_exec(pDb->db, - "BEGIN; SELECT * FROM sqlite_schema LIMIT 1;" , 0, 0, 0 - ); - if( rc!=0 ) return rc; - pDb->nOpenTrans = 1; - } - - /* Open any required write transactions */ - for(i=pDb->nOpenTrans; idb, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=SQLITE_OK ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_commit(TestDb *pTestDb, int iLevel){ - SqlDb *pDb = (SqlDb *)pTestDb; - assert( iLevel>=0 ); - - /* Close the read transaction if requested. */ - if( pDb->nOpenTrans>=1 && iLevel==0 ){ - int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0); - if( rc!=0 ) return rc; - pDb->nOpenTrans = 0; - } - - /* Close write transactions as required */ - if( pDb->nOpenTrans>iLevel ){ - char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=0 ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_rollback(TestDb *pTestDb, int iLevel){ - SqlDb *pDb = (SqlDb *)pTestDb; - assert( iLevel>=0 ); - - if( pDb->nOpenTrans>=1 && iLevel==0 ){ - /* Close the read transaction if requested. */ - int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0); - if( rc!=0 ) return rc; - }else if( pDb->nOpenTrans>1 && iLevel==1 ){ - /* Or, rollback and close the top-level write transaction */ - int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0); - if( rc!=0 ) return rc; - }else{ - /* Or, just roll back some nested transactions */ - char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=0 ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods SqlMethods = { - sql_close, - sql_write, - sql_delete, - sql_delete_range, - sql_fetch, - sql_scan, - sql_begin, - sql_commit, - sql_rollback - }; - const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)"; - const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)"; - const char *zDelete = "DELETE FROM t1 WHERE k = ?"; - const char *zRange = "DELETE FROM t1 WHERE k>? AND k= ?1 ORDER BY k"; - const char *zScan3 = "SELECT * FROM t1 ORDER BY k"; - - const char *zScan4 = - "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC"; - const char *zScan5 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC"; - const char *zScan6 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC"; - const char *zScan7 = "SELECT * FROM t1 ORDER BY k DESC"; - - int rc; - SqlDb *pDb; - char *zPragma; - - if( bClear && zFilename && zFilename[0] ){ - unlink(zFilename); - } - - pDb = (SqlDb *)malloc(sizeof(SqlDb)); - memset(pDb, 0, sizeof(SqlDb)); - pDb->base.pMethods = &SqlMethods; - - if( 0!=(rc = sqlite3_open(zFilename, &pDb->db)) - || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0)) - ){ - *ppDb = 0; - sql_close((TestDb *)pDb); - return rc; - } - - zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE); - sqlite3_exec(pDb->db, zPragma, 0, 0, 0); - sqlite3_free(zPragma); - zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE); - sqlite3_exec(pDb->db, zPragma, 0, 0, 0); - sqlite3_free(zPragma); - - /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */ - sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); - sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0); - sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0); - if( zSpec ){ - rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0); - if( rc!=SQLITE_OK ){ - sql_close((TestDb *)pDb); - return rc; - } - } - - *ppDb = (TestDb *)pDb; - return 0; -} -/* -** End wrapper for SQLite. -*************************************************************************/ - -/************************************************************************* -** Begin exported functions. -*/ -static struct Lib { - const char *zName; - const char *zDefaultDb; - int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb); -} aLib[] = { - { "sqlite3", "testdb.sqlite", sql_open }, - { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, - { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, - { "lsm_lomem2", "testdb.lsm_lomem2", test_lsm_lomem2_open }, -#ifdef HAVE_ZLIB - { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, -#endif - { "lsm", "testdb.lsm", test_lsm_open }, -#ifdef LSM_MUTEX_PTHREADS - { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, - { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, -#endif -#ifdef HAVE_LEVELDB - { "leveldb", "testdb.leveldb", test_leveldb_open }, -#endif -#ifdef HAVE_KYOTOCABINET - { "kyotocabinet", "testdb.kc", kc_open }, -#endif -#ifdef HAVE_MDB - { "mdb", "./testdb.mdb", mdb_open } -#endif -}; - -const char *tdb_system_name(int i){ - if( i<0 || i>=ArraySize(aLib) ) return 0; - return aLib[i].zName; -} - -const char *tdb_default_db(const char *zSys){ - int i; - for(i=0; izLibrary = aLib[i].zName; - } - break; - } - } - - if( rc ){ - /* Failed to find the requested database library. Return an error. */ - *ppDb = 0; - } - return rc; -} - -int tdb_close(TestDb *pDb){ - if( pDb ){ - return pDb->pMethods->xClose(pDb); - } - return 0; -} - -int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal); -} - -int tdb_delete(TestDb *pDb, void *pKey, int nKey){ - return pDb->pMethods->xDelete(pDb, pKey, nKey); -} - -int tdb_delete_range( - TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2 -){ - return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2); -} - -int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){ - return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal); -} - -int tdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to scan in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - return pDb->pMethods->xScan( - pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback - ); -} - -int tdb_begin(TestDb *pDb, int iLevel){ - return pDb->pMethods->xBegin(pDb, iLevel); -} -int tdb_commit(TestDb *pDb, int iLevel){ - return pDb->pMethods->xCommit(pDb, iLevel); -} -int tdb_rollback(TestDb *pDb, int iLevel){ - return pDb->pMethods->xRollback(pDb, iLevel); -} - -int tdb_transaction_support(TestDb *pDb){ - return (pDb->pMethods->xBegin != error_transaction_function); -} - -const char *tdb_library_name(TestDb *pDb){ - return pDb->zLibrary; -} - -/* -** End exported functions. -*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.h b/ext/lsm1/lsm-test/lsmtest_tdb.h deleted file mode 100644 index c55b6e2f80..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb.h +++ /dev/null @@ -1,174 +0,0 @@ - -/* -** This file is the interface to a very simple database library used for -** testing. The interface is similar to that of the LSM. The main virtue -** of this library is that the same API may be used to access a key-value -** store implemented by LSM, SQLite or another database system. Which -** makes it easy to use for correctness and performance tests. -*/ - -#ifndef __WRAPPER_H_ -#define __WRAPPER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "lsm.h" - -typedef struct TestDb TestDb; - -/* -** Open a new database connection. The first argument is the name of the -** database library to use. e.g. something like: -** -** "sqlite3" -** "lsm" -** -** See function tdb_system_name() for a list of available database systems. -** -** The second argument is the name of the database to open (e.g. a filename). -** -** If the third parameter is non-zero, then any existing database by the -** name of zDb is removed before opening a new one. If it is zero, then an -** existing database may be opened. -*/ -int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb); - -/* -** Close a database handle. -*/ -int tdb_close(TestDb *pDb); - -/* -** Write a new key/value into the database. -*/ -int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal); - -/* -** Delete a key from the database. -*/ -int tdb_delete(TestDb *pDb, void *pKey, int nKey); - -/* -** Delete a range of keys from the database. -*/ -int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2); - -/* -** Query the database for key (pKey/nKey). If no entry is found, set *ppVal -** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal -** to a pointer to and size of the value associated with (pKey/nKey). -*/ -int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal); - -/* -** Open and close nested transactions. Currently, these functions only -** work for SQLite3 and LSM systems. Use the tdb_transaction_support() -** function to determine if a given TestDb handle supports these methods. -** -** These functions and the iLevel parameter follow the same conventions as -** the SQLite 4 transaction interface. Note that this is slightly different -** from the way LSM does things. As follows: -** -** tdb_begin(): -** A successful call to tdb_begin() with (iLevel>1) guarantees that -** there are at least (iLevel-1) write transactions open. If iLevel==1, -** then it guarantees that at least a read-transaction is open. Calling -** tdb_begin() with iLevel==0 is a no-op. -** -** tdb_commit(): -** A successful call to tdb_commit() with (iLevel>1) guarantees that -** there are at most (iLevel-1) write transactions open. If iLevel==1, -** then it guarantees that there are no write transactions open (although -** a read-transaction may remain open). Calling tdb_commit() with -** iLevel==0 ensures that all transactions, read or write, have been -** closed and committed. -** -** tdb_rollback(): -** This call is similar to tdb_commit(), except that instead of committing -** transactions, it reverts them. For example, calling tdb_rollback() with -** iLevel==2 ensures that there is at most one write transaction open, and -** restores the database to the state that it was in when that transaction -** was opened. -** -** In other words, tdb_commit() just closes transactions - tdb_rollback() -** closes transactions and then restores the database to the state it -** was in before those transactions were even opened. -*/ -int tdb_begin(TestDb *pDb, int iLevel); -int tdb_commit(TestDb *pDb, int iLevel); -int tdb_rollback(TestDb *pDb, int iLevel); - -/* -** Return true if transactions are supported, or false otherwise. -*/ -int tdb_transaction_support(TestDb *pDb); - -/* -** Return the name of the database library (as passed to tdb_open()) used -** by the handled passed as the first argument. -*/ -const char *tdb_library_name(TestDb *pDb); - -/* -** Scan a range of database keys. Invoke the callback function for each -** key visited. -*/ -int tdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to scan in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -); - -const char *tdb_system_name(int i); -const char *tdb_default_db(const char *zSys); - -int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb); - -/* -** If the TestDb handle passed as an argument is a wrapper around an LSM -** database, return the LSM handle. Otherwise, if the argument is some other -** database system, return NULL. -*/ -lsm_db *tdb_lsm(TestDb *pDb); - -/* -** Return true if the db passed as an argument is a multi-threaded LSM -** connection. -*/ -int tdb_lsm_multithread(TestDb *pDb); - -/* -** Return a pointer to the lsm_env object used by all lsm database -** connections initialized as a copy of the object returned by -** lsm_default_env(). It may be modified (e.g. to override functions) -** if the caller can guarantee that it is not already in use. -*/ -lsm_env *tdb_lsm_env(void); - -/* -** The following functions only work with LSM database handles. It is -** illegal to call them with any other type of database handle specified -** as an argument. -*/ -void tdb_lsm_enable_log(TestDb *pDb, int bEnable); -void tdb_lsm_application_crash(TestDb *pDb); -void tdb_lsm_prepare_system_crash(TestDb *pDb); -void tdb_lsm_system_crash(TestDb *pDb); -void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync); - - -void tdb_lsm_safety(TestDb *pDb, int eMode); -void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *); -void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*); -int tdb_lsm_config_str(TestDb *pDb, const char *zStr); - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif - -#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb2.cc b/ext/lsm1/lsm-test/lsmtest_tdb2.cc deleted file mode 100644 index 86ebb49583..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb2.cc +++ /dev/null @@ -1,369 +0,0 @@ - - -#include "lsmtest.h" -#include - -#ifdef HAVE_KYOTOCABINET -#include "kcpolydb.h" -extern "C" { - struct KcDb { - TestDb base; - kyotocabinet::TreeDB* db; - char *pVal; - }; -} - -int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){ - KcDb *pKcDb; - int ok; - int rc = 0; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pKcDb = (KcDb *)malloc(sizeof(KcDb)); - memset(pKcDb, 0, sizeof(KcDb)); - - - pKcDb->db = new kyotocabinet::TreeDB(); - pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE); - pKcDb->db->tune_page_cache( - TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE - ); - ok = pKcDb->db->open(zFilename, - kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE - ); - if( ok==0 ){ - free(pKcDb); - pKcDb = 0; - rc = 1; - } - - *ppDb = (TestDb *)pKcDb; - return rc; -} - -int test_kc_close(TestDb *pDb){ - KcDb *pKcDb = (KcDb *)pDb; - if( pKcDb->pVal ){ - delete [] pKcDb->pVal; - } - pKcDb->db->close(); - delete pKcDb->db; - free(pKcDb); - return 0; -} - -int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - KcDb *pKcDb = (KcDb *)pDb; - int ok; - - ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal); - return (ok ? 0 : 1); -} - -int test_kc_delete(TestDb *pDb, void *pKey, int nKey){ - KcDb *pKcDb = (KcDb *)pDb; - int ok; - - ok = pKcDb->db->remove((const char *)pKey, nKey); - return (ok ? 0 : 1); -} - -int test_kc_delete_range( - TestDb *pDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - int res; - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - - while( 1 ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - - pKey = pCur->get(&nKey, &pVal, &nVal); - if( pKey==0 ) break; - -#ifndef NDEBUG - if( pKey1 ){ - res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); - assert( res>0 || (res==0 && nKey>nKey1) ); - } -#endif - - if( pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2remove(); - delete [] pKey; - } - - delete pCur; - return 0; -} - -int test_kc_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - KcDb *pKcDb = (KcDb *)pDb; - size_t nVal; - - if( pKcDb->pVal ){ - delete [] pKcDb->pVal; - pKcDb->pVal = 0; - } - - pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal); - if( pKcDb->pVal ){ - *ppVal = pKcDb->pVal; - *pnVal = nVal; - }else{ - *ppVal = 0; - *pnVal = -1; - } - - return 0; -} - -int test_kc_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - int res; - - if( bReverse==0 ){ - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - }else{ - if( pKey2 ){ - res = pCur->jump_back((const char *)pKey2, nKey2); - }else{ - res = pCur->jump_back(); - } - } - - while( res ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - pKey = pCur->get(&nKey, &pVal, &nVal); - - if( bReverse==0 && pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2nKey) ){ - delete [] pKey; - break; - } - } - - xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal); - delete [] pKey; - - if( bReverse ){ - res = pCur->step_back(); - }else{ - res = pCur->step(); - } - } - - delete pCur; - return 0; -} -#endif /* HAVE_KYOTOCABINET */ - -#ifdef HAVE_MDB -#include "lmdb.h" - -extern "C" { - struct MdbDb { - TestDb base; - MDB_env *env; - MDB_dbi dbi; - }; -} - -int test_mdb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - MDB_txn *txn; - MdbDb *pMdb; - int rc; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pMdb = (MdbDb *)malloc(sizeof(MdbDb)); - memset(pMdb, 0, sizeof(MdbDb)); - - rc = mdb_env_create(&pMdb->env); - if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); - if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); - if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_open(txn, NULL, 0, &pMdb->dbi); - mdb_txn_commit(txn); - } - - *ppDb = (TestDb *)pMdb; - return rc; -} - -int test_mdb_close(TestDb *pDb){ - MdbDb *pMdb = (MdbDb *)pDb; - - mdb_close(pMdb->env, pMdb->dbi); - mdb_env_close(pMdb->env); - free(pMdb); - return 0; -} - -int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val val; - MDB_val key; - MDB_txn *txn; - - val.mv_size = nVal; - val.mv_data = pVal; - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_del(txn, pMdb->dbi, &key, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_val val = {0, 0}; - rc = mdb_get(txn, pMdb->dbi, &key, &val); - if( rc==MDB_NOTFOUND ){ - rc = 0; - *ppVal = 0; - *pnVal = -1; - }else{ - *ppVal = val.mv_data; - *pnVal = val.mv_size; - } - mdb_txn_commit(txn); - } - - return rc; -} - -int test_mdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - MdbDb *pMdb = (MdbDb *)pDb; - int rc; - MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; - MDB_txn *txn; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_cursor *csr; - MDB_val key = {0, 0}; - MDB_val val = {0, 0}; - - rc = mdb_cursor_open(txn, pMdb->dbi, &csr); - if( rc==0 ){ - while( mdb_cursor_get(csr, &key, &val, op)==0 ){ - xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); - } - mdb_cursor_close(csr); - } - } - - return rc; -} - -#endif /* HAVE_MDB */ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb3.c b/ext/lsm1/lsm-test/lsmtest_tdb3.c deleted file mode 100644 index e29497af20..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb3.c +++ /dev/null @@ -1,1429 +0,0 @@ - -#include "lsmtest_tdb.h" -#include "lsm.h" -#include "lsmtest.h" - -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include - -#ifndef _WIN32 -# include -#endif - -typedef struct LsmDb LsmDb; -typedef struct LsmWorker LsmWorker; -typedef struct LsmFile LsmFile; - -#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) -#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) - -#ifdef LSM_MUTEX_PTHREADS -#include - -#define LSMTEST_THREAD_CKPT 1 -#define LSMTEST_THREAD_WORKER 2 -#define LSMTEST_THREAD_WORKER_AC 3 - -/* -** There are several different types of worker threads that run in different -** test configurations, depending on the value of LsmWorker.eType. -** -** 1. Checkpointer. -** 2. Worker with auto-checkpoint. -** 3. Worker without auto-checkpoint. -*/ -struct LsmWorker { - LsmDb *pDb; /* Main database structure */ - lsm_db *pWorker; /* Worker database handle */ - pthread_t worker_thread; /* Worker thread */ - pthread_cond_t worker_cond; /* Condition var the worker waits on */ - pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ - int bDoWork; /* Set to true by client when there is work */ - int worker_rc; /* Store error code here */ - int eType; /* LSMTEST_THREAD_XXX constant */ - int bBlock; -}; -#else -struct LsmWorker { int worker_rc; int bBlock; }; -#endif - -static void mt_shutdown(LsmDb *); - -lsm_env *tdb_lsm_env(void){ - static int bInit = 0; - static lsm_env env; - if( bInit==0 ){ - memcpy(&env, lsm_default_env(), sizeof(env)); - bInit = 1; - } - return &env; -} - -typedef struct FileSector FileSector; -typedef struct FileData FileData; - -struct FileSector { - u8 *aOld; /* Old data for this sector */ -}; - -struct FileData { - int nSector; /* Allocated size of apSector[] array */ - FileSector *aSector; /* Array of file sectors */ -}; - -/* -** bPrepareCrash: -** If non-zero, the file wrappers maintain enough in-memory data to -** simulate the effect of a power-failure on the file-system (i.e. that -** unsynced sectors may be written, not written, or overwritten with -** arbitrary data when the crash occurs). -** -** bCrashed: -** Set to true after a crash is simulated. Once this variable is true, all -** VFS methods other than xClose() return LSM_IOERR as soon as they are -** called (without affecting the contents of the file-system). -** -** env: -** The environment object used by all lsm_db* handles opened by this -** object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx -** always points to the containing LsmDb structure. -*/ -struct LsmDb { - TestDb base; /* Base class - methods table */ - lsm_env env; /* Environment used by connection db */ - char *zName; /* Database file name */ - lsm_db *db; /* LSM database handle */ - - lsm_cursor *pCsr; /* Cursor held open during read transaction */ - void *pBuf; /* Buffer for tdb_fetch() output */ - int nBuf; /* Allocated (not used) size of pBuf */ - - /* Crash testing related state */ - int bCrashed; /* True once a crash has occurred */ - int nAutoCrash; /* Number of syncs until a crash */ - int bPrepareCrash; /* True to store writes in memory */ - - /* Unsynced data (while crash testing) */ - int szSector; /* Assumed size of disk sectors (512B) */ - FileData aFile[2]; /* Database and log file data */ - - /* Other test instrumentation */ - int bNoRecovery; /* If true, assume DMS2 is locked */ - - /* Work hook redirection */ - void (*xWork)(lsm_db *, void *); - void *pWorkCtx; - - /* IO logging hook */ - void (*xWriteHook)(void *, int, lsm_i64, int, int); - void *pWriteCtx; - - /* Worker threads (for lsm_mt) */ - int nMtMinCkpt; - int nMtMaxCkpt; - int eMode; - int nWorker; - LsmWorker *aWorker; -}; - -#define LSMTEST_MODE_SINGLETHREAD 1 -#define LSMTEST_MODE_BACKGROUND_CKPT 2 -#define LSMTEST_MODE_BACKGROUND_WORK 3 -#define LSMTEST_MODE_BACKGROUND_BOTH 4 - -/************************************************************************* -************************************************************************** -** Begin test VFS code. -*/ - -struct LsmFile { - lsm_file *pReal; /* Real underlying file */ - int bLog; /* True for log file. False for db file */ - LsmDb *pDb; /* Database handle that uses this file */ -}; - -static int testEnvFullpath( - lsm_env *pEnv, /* Environment for current LsmDb */ - const char *zFile, /* Relative path name */ - char *zOut, /* Output buffer */ - int *pnOut /* IN/OUT: Size of output buffer */ -){ - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut); -} - -static int testEnvOpen( - lsm_env *pEnv, /* Environment for current LsmDb */ - const char *zFile, /* Name of file to open */ - int flags, - lsm_file **ppFile /* OUT: New file handle object */ -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; - int rc; /* Return Code */ - LsmFile *pRet; /* The new file handle */ - int nFile; /* Length of string zFile in bytes */ - - nFile = strlen(zFile); - pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); - pRet->pDb = pDb; - pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); - - rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); - if( rc!=LSM_OK ){ - testFree(pRet); - pRet = 0; - } - - *ppFile = (lsm_file *)pRet; - return rc; -} - -static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - if( p->pDb->bCrashed ) return LSM_IOERR; - return pRealEnv->xRead(p->pReal, iOff, pData, nData); -} - -static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - LsmDb *pDb = p->pDb; - - if( pDb->bCrashed ) return LSM_IOERR; - - if( pDb->bPrepareCrash ){ - FileData *pData2 = &pDb->aFile[p->bLog]; - int iFirst; - int iLast; - int iSector; - - iFirst = (int)(iOff / pDb->szSector); - iLast = (int)((iOff + nData - 1) / pDb->szSector); - - if( pData2->nSector<(iLast+1) ){ - int nNew = ( ((iLast + 1) + 63) / 64 ) * 64; - assert( nNew>iLast ); - pData2->aSector = (FileSector *)testRealloc( - pData2->aSector, nNew*sizeof(FileSector) - ); - memset(&pData2->aSector[pData2->nSector], - 0, (nNew - pData2->nSector) * sizeof(FileSector) - ); - pData2->nSector = nNew; - } - - for(iSector=iFirst; iSector<=iLast; iSector++){ - if( pData2->aSector[iSector].aOld==0 ){ - u8 *aOld = (u8 *)testMalloc(pDb->szSector); - pRealEnv->xRead( - p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector - ); - pData2->aSector[iSector].aOld = aOld; - } - } - } - - if( pDb->xWriteHook ){ - int rc; - int nUs; - struct timeval t1; - struct timeval t2; - - gettimeofday(&t1, 0); - assert( nData>0 ); - rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData); - gettimeofday(&t2, 0); - - nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); - pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs); - return rc; - } - - return pRealEnv->xWrite(p->pReal, iOff, pData, nData); -} - -static void doSystemCrash(LsmDb *pDb); - -static int testEnvSync(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - LsmDb *pDb = p->pDb; - FileData *pData = &pDb->aFile[p->bLog]; - int i; - - if( pDb->bCrashed ) return LSM_IOERR; - - if( pDb->nAutoCrash ){ - pDb->nAutoCrash--; - if( pDb->nAutoCrash==0 ){ - doSystemCrash(pDb); - pDb->bCrashed = 1; - return LSM_IOERR; - } - } - - if( pDb->bPrepareCrash ){ - for(i=0; inSector; i++){ - testFree(pData->aSector[i].aOld); - pData->aSector[i].aOld = 0; - } - } - - if( pDb->xWriteHook ){ - int rc; - int nUs; - struct timeval t1; - struct timeval t2; - - gettimeofday(&t1, 0); - rc = pRealEnv->xSync(p->pReal); - gettimeofday(&t2, 0); - - nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); - pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs); - return rc; - } - - return pRealEnv->xSync(p->pReal); -} - -static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - if( p->pDb->bCrashed ) return LSM_IOERR; - return pRealEnv->xTruncate(p->pReal, iOff); -} - -static int testEnvSectorSize(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xSectorSize(p->pReal); -} - -static int testEnvRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut); -} - -static int testEnvFileid( - lsm_file *pFile, - void *ppOut, - int *pnOut -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xFileid(p->pReal, ppOut, pnOut); -} - -static int testEnvClose(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - - pRealEnv->xClose(p->pReal); - testFree(p); - return LSM_OK; -} - -static int testEnvUnlink(lsm_env *pEnv, const char *zFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - unused_parameter(pEnv); - return pRealEnv->xUnlink(pRealEnv, zFile); -} - -static int testEnvLock(lsm_file *pFile, int iLock, int eType){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - - if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ - return LSM_BUSY; - } - return pRealEnv->xLock(p->pReal, iLock, eType); -} - -static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - - if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ - return LSM_BUSY; - } - return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType); -} - -static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp); -} - -static void testEnvShmBarrier(void){ -} - -static int testEnvShmUnmap(lsm_file *pFile, int bDel){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xShmUnmap(p->pReal, bDel); -} - -static int testEnvSleep(lsm_env *pEnv, int us){ - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xSleep(pRealEnv, us); -} - -static void doSystemCrash(LsmDb *pDb){ - lsm_env *pEnv = tdb_lsm_env(); - int iFile; - int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector; - - char *zFile = pDb->zName; - char *zFree = 0; - - for(iFile=0; iFile<2; iFile++){ - lsm_file *pFile = 0; - int i; - - pEnv->xOpen(pEnv, zFile, 0, &pFile); - for(i=0; iaFile[iFile].nSector; i++){ - u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; - if( aOld ){ - int iOpt = testPrngValue(iSeed++) % 3; - switch( iOpt ){ - case 0: - break; - - case 1: - testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4); - /* Fall-through */ - - case 2: - pEnv->xWrite( - pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector - ); - break; - } - testFree(aOld); - pDb->aFile[iFile].aSector[i].aOld = 0; - } - } - pEnv->xClose(pFile); - zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName); - } - - sqlite3_free(zFree); -} -/* -** End test VFS code. -************************************************************************** -*************************************************************************/ - -/************************************************************************* -************************************************************************** -** Begin test compression hooks. -*/ - -#ifdef HAVE_ZLIB -#include - -static int testZipBound(void *pCtx, int nSrc){ - return compressBound(nSrc); -} - -static int testZipCompress( - void *pCtx, /* Context pointer */ - char *aOut, int *pnOut, /* OUT: Buffer containing compressed data */ - const char *aIn, int nIn /* Buffer containing input data */ -){ - uLongf n = *pnOut; /* In/out buffer size for compress() */ - int rc; /* compress() return code */ - - rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); - *pnOut = n; - return (rc==Z_OK ? 0 : LSM_ERROR); -} - -static int testZipUncompress( - void *pCtx, /* Context pointer */ - char *aOut, int *pnOut, /* OUT: Buffer containing uncompressed data */ - const char *aIn, int nIn /* Buffer containing input data */ -){ - uLongf n = *pnOut; /* In/out buffer size for uncompress() */ - int rc; /* uncompress() return code */ - - rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); - *pnOut = n; - return (rc==Z_OK ? 0 : LSM_ERROR); -} - -static int testConfigureCompression(lsm_db *pDb){ - static lsm_compress zip = { - 0, /* Context pointer (unused) */ - 1, /* Id value */ - testZipBound, /* xBound method */ - testZipCompress, /* xCompress method */ - testZipUncompress /* xUncompress method */ - }; - return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); -} -#endif /* ifdef HAVE_ZLIB */ - -/* -** End test compression hooks. -************************************************************************** -*************************************************************************/ - -static int test_lsm_close(TestDb *pTestDb){ - int i; - int rc = LSM_OK; - LsmDb *pDb = (LsmDb *)pTestDb; - - lsm_csr_close(pDb->pCsr); - lsm_close(pDb->db); - - /* If this is a multi-threaded database, wait on the worker threads. */ - mt_shutdown(pDb); - for(i=0; inWorker && rc==LSM_OK; i++){ - rc = pDb->aWorker[i].worker_rc; - } - - for(i=0; iaFile[0].nSector; i++){ - testFree(pDb->aFile[0].aSector[i].aOld); - } - testFree(pDb->aFile[0].aSector); - for(i=0; iaFile[1].nSector; i++){ - testFree(pDb->aFile[1].aSector[i].aOld); - } - testFree(pDb->aFile[1].aSector); - - memset(pDb, sizeof(LsmDb), 0x11); - testFree((char *)pDb->pBuf); - testFree((char *)pDb); - return rc; -} - -static void mt_signal_worker(LsmDb*, int); - -static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ - int nSleep = 0; - int nKB; - int rc; - - do { - nKB = 0; - rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc!=LSM_OK || nKBnMtMaxCkpt ) break; -#ifdef LSM_MUTEX_PTHREADS - mt_signal_worker(pDb, - (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1) - ); -#endif - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int waitOnWorker(LsmDb *pDb){ - int rc; - int nLimit = -1; - int nSleep = 0; - - rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); - do { - int nOld, nNew, rc2; - rc2 = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); - if( rc2!=LSM_OK ) return rc2; - if( nOld==0 || nNew<(nLimit/2) ) break; -#ifdef LSM_MUTEX_PTHREADS - mt_signal_worker(pDb, 0); -#endif - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int test_lsm_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - LsmDb *pDb = (LsmDb *)pTestDb; - int rc = LSM_OK; - - if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ - rc = waitOnCheckpointer(pDb, pDb->db); - }else if( - pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK - || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH - ){ - rc = waitOnWorker(pDb); - } - - if( rc==LSM_OK ){ - rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); - } - return rc; -} - -static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ - LsmDb *pDb = (LsmDb *)pTestDb; - return lsm_delete(pDb->db, pKey, nKey); -} - -static int test_lsm_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - LsmDb *pDb = (LsmDb *)pTestDb; - return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2); -} - -static int test_lsm_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - LsmDb *pDb = (LsmDb *)pTestDb; - lsm_cursor *csr; - - if( pKey==0 ) return LSM_OK; - - if( pDb->pCsr==0 ){ - rc = lsm_csr_open(pDb->db, &csr); - if( rc!=LSM_OK ) return rc; - }else{ - csr = pDb->pCsr; - } - - rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ); - if( rc==LSM_OK ){ - if( lsm_csr_valid(csr) ){ - const void *pVal; int nVal; - rc = lsm_csr_value(csr, &pVal, &nVal); - if( nVal>pDb->nBuf ){ - testFree(pDb->pBuf); - pDb->pBuf = testMalloc(nVal*2); - pDb->nBuf = nVal*2; - } - memcpy(pDb->pBuf, pVal, nVal); - *ppVal = pDb->pBuf; - *pnVal = nVal; - }else{ - *ppVal = 0; - *pnVal = -1; - } - } - if( pDb->pCsr==0 ){ - lsm_csr_close(csr); - } - return rc; -} - -static int test_lsm_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - LsmDb *pDb = (LsmDb *)pTestDb; - lsm_cursor *csr; - lsm_cursor *csr2 = 0; - int rc; - - if( pDb->pCsr==0 ){ - rc = lsm_csr_open(pDb->db, &csr); - if( rc!=LSM_OK ) return rc; - }else{ - rc = LSM_OK; - csr = pDb->pCsr; - } - - /* To enhance testing, if both pLast and pFirst are defined, seek the - ** cursor to the "end" boundary here. Then the next block seeks it to - ** the "start" ready for the scan. The point is to test that cursors - ** can be reused. */ - if( pLast && pFirst ){ - if( bReverse ){ - rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE); - }else{ - rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE); - } - } - - if( bReverse ){ - if( pLast ){ - rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE); - }else{ - rc = lsm_csr_last(csr); - } - }else{ - if( pFirst ){ - rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_GE); - }else{ - rc = lsm_csr_first(csr); - } - } - - while( rc==LSM_OK && lsm_csr_valid(csr) ){ - const void *pKey; int nKey; - const void *pVal; int nVal; - int cmp; - - lsm_csr_key(csr, &pKey, &nKey); - lsm_csr_value(csr, &pVal, &nVal); - - if( bReverse && pFirst ){ - cmp = memcmp(pFirst, pKey, MIN(nKey, nFirst)); - if( cmp>0 || (cmp==0 && nFirst>nKey) ) break; - }else if( bReverse==0 && pLast ){ - cmp = memcmp(pLast, pKey, MIN(nKey, nLast)); - if( cmp<0 || (cmp==0 && nLastpCsr==0 ){ - lsm_csr_close(csr); - } - return rc; -} - -static int test_lsm_begin(TestDb *pTestDb, int iLevel){ - int rc = LSM_OK; - LsmDb *pDb = (LsmDb *)pTestDb; - - /* iLevel==0 is a no-op. */ - if( iLevel==0 ) return 0; - - if( pDb->pCsr==0 ) rc = lsm_csr_open(pDb->db, &pDb->pCsr); - if( rc==LSM_OK && iLevel>1 ){ - rc = lsm_begin(pDb->db, iLevel-1); - } - - return rc; -} -static int test_lsm_commit(TestDb *pTestDb, int iLevel){ - LsmDb *pDb = (LsmDb *)pTestDb; - - /* If iLevel==0, close any open read transaction */ - if( iLevel==0 && pDb->pCsr ){ - lsm_csr_close(pDb->pCsr); - pDb->pCsr = 0; - } - - /* If iLevel==0, close any open read transaction */ - return lsm_commit(pDb->db, MAX(0, iLevel-1)); -} -static int test_lsm_rollback(TestDb *pTestDb, int iLevel){ - LsmDb *pDb = (LsmDb *)pTestDb; - - /* If iLevel==0, close any open read transaction */ - if( iLevel==0 && pDb->pCsr ){ - lsm_csr_close(pDb->pCsr); - pDb->pCsr = 0; - } - - return lsm_rollback(pDb->db, MAX(0, iLevel-1)); -} - -/* -** A log message callback registered with lsm connections. Prints all -** messages to stderr. -*/ -static void xLog(void *pCtx, int rc, const char *z){ - unused_parameter(rc); - /* fprintf(stderr, "lsm: rc=%d \"%s\"\n", rc, z); */ - if( pCtx ) fprintf(stderr, "%s: ", (char *)pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -static void xWorkHook(lsm_db *db, void *pArg){ - LsmDb *p = (LsmDb *)pArg; - if( p->xWork ) p->xWork(db, p->pWorkCtx); -} - -#define TEST_NO_RECOVERY -1 -#define TEST_COMPRESSION -3 - -#define TEST_MT_MODE -2 -#define TEST_MT_MIN_CKPT -4 -#define TEST_MT_MAX_CKPT -5 - - -int test_lsm_config_str( - LsmDb *pLsm, - lsm_db *db, - int bWorker, - const char *zStr, - int *pnThread -){ - struct CfgParam { - const char *zParam; - int bWorker; - int eParam; - } aParam[] = { - { "autoflush", 0, LSM_CONFIG_AUTOFLUSH }, - { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, - { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, - { "safety", 0, LSM_CONFIG_SAFETY }, - { "autowork", 0, LSM_CONFIG_AUTOWORK }, - { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, - { "mmap", 0, LSM_CONFIG_MMAP }, - { "use_log", 0, LSM_CONFIG_USE_LOG }, - { "automerge", 0, LSM_CONFIG_AUTOMERGE }, - { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, - { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, - { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE }, - { "test_no_recovery", 0, TEST_NO_RECOVERY }, - { "bg_min_ckpt", 0, TEST_NO_RECOVERY }, - - { "mt_mode", 0, TEST_MT_MODE }, - { "mt_min_ckpt", 0, TEST_MT_MIN_CKPT }, - { "mt_max_ckpt", 0, TEST_MT_MAX_CKPT }, - -#ifdef HAVE_ZLIB - { "compression", 0, TEST_COMPRESSION }, -#endif - { 0, 0 } - }; - const char *z = zStr; - int nThread = 1; - - if( zStr==0 ) return 0; - - assert( db ); - while( z[0] ){ - const char *zStart; - - /* Skip whitespace */ - while( *z==' ' ) z++; - zStart = z; - - while( *z && *z!='=' ) z++; - if( *z ){ - int eParam; - int i; - int iVal; - int iMul = 1; - int rc; - char zParam[32]; - int nParam = z-zStart; - if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; - - memcpy(zParam, zStart, nParam); - zParam[nParam] = '\0'; - rc = testArgSelect(aParam, "param", zParam, &i); - if( rc!=0 ) return rc; - eParam = aParam[i].eParam; - - z++; - zStart = z; - while( *z>='0' && *z<='9' ) z++; - if( *z=='k' || *z=='K' ){ - iMul = 1; - z++; - }else if( *z=='M' || *z=='M' ){ - iMul = 1024; - z++; - } - nParam = z-zStart; - if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; - memcpy(zParam, zStart, nParam); - zParam[nParam] = '\0'; - iVal = atoi(zParam) * iMul; - - if( eParam>0 ){ - if( bWorker || aParam[i].bWorker==0 ){ - lsm_config(db, eParam, &iVal); - } - }else{ - switch( eParam ){ - case TEST_NO_RECOVERY: - if( pLsm ) pLsm->bNoRecovery = iVal; - break; - case TEST_MT_MODE: - if( pLsm ) nThread = iVal; - break; - case TEST_MT_MIN_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal*1024; - break; - case TEST_MT_MAX_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal*1024; - break; -#ifdef HAVE_ZLIB - case TEST_COMPRESSION: - testConfigureCompression(db); - break; -#endif - } - } - }else if( z!=zStart ){ - goto syntax_error; - } - } - - if( pnThread ) *pnThread = nThread; - if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){ - pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt; - } - - return 0; - syntax_error: - testPrintError("syntax error at: \"%s\"\n", z); - return 1; -} - -int tdb_lsm_config_str(TestDb *pDb, const char *zStr){ - int rc = 0; - if( tdb_lsm(pDb) ){ -#ifdef LSM_MUTEX_PTHREADS - int i; -#endif - LsmDb *pLsm = (LsmDb *)pDb; - - rc = test_lsm_config_str(pLsm, pLsm->db, 0, zStr, 0); -#ifdef LSM_MUTEX_PTHREADS - for(i=0; rc==0 && inWorker; i++){ - rc = test_lsm_config_str(0, pLsm->aWorker[i].pWorker, 1, zStr, 0); - } -#endif - } - return rc; -} - -int tdb_lsm_configure(lsm_db *db, const char *zConfig){ - return test_lsm_config_str(0, db, 0, zConfig, 0); -} - -static int testLsmStartWorkers(LsmDb *, int, const char *, const char *); - -static int testLsmOpen( - const char *zCfg, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods LsmMethods = { - test_lsm_close, - test_lsm_write, - test_lsm_delete, - test_lsm_delete_range, - test_lsm_fetch, - test_lsm_scan, - test_lsm_begin, - test_lsm_commit, - test_lsm_rollback - }; - - int rc; - int nFilename; - LsmDb *pDb; - - /* If the bClear flag is set, delete any existing database. */ - assert( zFilename); - if( bClear ) testDeleteLsmdb(zFilename); - nFilename = strlen(zFilename); - - pDb = (LsmDb *)testMalloc(sizeof(LsmDb) + nFilename + 1); - memset(pDb, 0, sizeof(LsmDb)); - pDb->base.pMethods = &LsmMethods; - pDb->zName = (char *)&pDb[1]; - memcpy(pDb->zName, zFilename, nFilename + 1); - - /* Default the sector size used for crash simulation to 512 bytes. - ** Todo: There should be an OS method to obtain this value - just as - ** there is in SQLite. For now, LSM assumes that it is smaller than - ** the page size (default 4KB). - */ - pDb->szSector = 256; - - /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ - pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; - pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; - - memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); - pDb->env.pVfsCtx = (void *)pDb; - pDb->env.xFullpath = testEnvFullpath; - pDb->env.xOpen = testEnvOpen; - pDb->env.xRead = testEnvRead; - pDb->env.xWrite = testEnvWrite; - pDb->env.xTruncate = testEnvTruncate; - pDb->env.xSync = testEnvSync; - pDb->env.xSectorSize = testEnvSectorSize; - pDb->env.xRemap = testEnvRemap; - pDb->env.xFileid = testEnvFileid; - pDb->env.xClose = testEnvClose; - pDb->env.xUnlink = testEnvUnlink; - pDb->env.xLock = testEnvLock; - pDb->env.xTestLock = testEnvTestLock; - pDb->env.xShmBarrier = testEnvShmBarrier; - pDb->env.xShmMap = testEnvShmMap; - pDb->env.xShmUnmap = testEnvShmUnmap; - pDb->env.xSleep = testEnvSleep; - - rc = lsm_new(&pDb->env, &pDb->db); - if( rc==LSM_OK ){ - int nThread = 1; - lsm_config_log(pDb->db, xLog, 0); - lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); - - rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); - if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); - - pDb->eMode = nThread; -#ifdef LSM_MUTEX_PTHREADS - if( rc==LSM_OK && nThread>1 ){ - testLsmStartWorkers(pDb, nThread, zFilename, zCfg); - } -#endif - - if( rc!=LSM_OK ){ - test_lsm_close((TestDb *)pDb); - pDb = 0; - } - } - - *ppDb = (TestDb *)pDb; - return rc; -} - -int test_lsm_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return testLsmOpen(zSpec, zFilename, bClear, ppDb); -} - -int test_lsm_small_open( - const char *zSpec, - const char *zFile, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "page_size=256 block_size=64 mmap=1024"; - return testLsmOpen(zCfg, zFile, bClear, ppDb); -} - -int test_lsm_lomem_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - /* "max_freelist=4 autocheckpoint=32" */ - const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32" - "mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_lomem2_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - /* "max_freelist=4 autocheckpoint=32" */ - const char *zCfg = - "page_size=512 block_size=64 autoflush=0 mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_zip_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32 compression=1 mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -lsm_db *tdb_lsm(TestDb *pDb){ - if( pDb->pMethods->xClose==test_lsm_close ){ - return ((LsmDb *)pDb)->db; - } - return 0; -} - -int tdb_lsm_multithread(TestDb *pDb){ - int ret = 0; - if( tdb_lsm(pDb) ){ - ret = ((LsmDb*)pDb)->eMode!=LSMTEST_MODE_SINGLETHREAD; - } - return ret; -} - -void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ - lsm_db *db = tdb_lsm(pDb); - if( db ){ - lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); - } -} - -void tdb_lsm_application_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bCrashed = 1; - } -} - -void tdb_lsm_prepare_system_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bPrepareCrash = 1; - } -} - -void tdb_lsm_system_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bCrashed = 1; - doSystemCrash(p); - } -} - -void tdb_lsm_safety(TestDb *pDb, int eMode){ - assert( eMode==LSM_SAFETY_OFF - || eMode==LSM_SAFETY_NORMAL - || eMode==LSM_SAFETY_FULL - ); - if( tdb_lsm(pDb) ){ - int iParam = eMode; - LsmDb *p = (LsmDb *)pDb; - lsm_config(p->db, LSM_CONFIG_SAFETY, &iParam); - } -} - -void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync){ - assert( iSync>0 ); - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->nAutoCrash = iSync; - p->bPrepareCrash = 1; - } -} - -void tdb_lsm_config_work_hook( - TestDb *pDb, - void (*xWork)(lsm_db *, void *), - void *pWorkCtx -){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->xWork = xWork; - p->pWorkCtx = pWorkCtx; - } -} - -void tdb_lsm_write_hook( - TestDb *pDb, - void (*xWrite)(void *, int, lsm_i64, int, int), - void *pWriteCtx -){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->xWriteHook = xWrite; - p->pWriteCtx = pWriteCtx; - } -} - -int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb){ - return testLsmOpen(zCfg, zDb, bClear, ppDb); -} - -#ifdef LSM_MUTEX_PTHREADS - -/* -** Signal worker thread iWorker that there may be work to do. -*/ -static void mt_signal_worker(LsmDb *pDb, int iWorker){ - LsmWorker *p = &pDb->aWorker[iWorker]; - pthread_mutex_lock(&p->worker_mutex); - p->bDoWork = 1; - pthread_cond_signal(&p->worker_cond); - pthread_mutex_unlock(&p->worker_mutex); -} - -/* -** This routine is used as the main() for all worker threads. -*/ -static void *worker_main(void *pArg){ - LsmWorker *p = (LsmWorker *)pArg; - lsm_db *pWorker; /* Connection to access db through */ - - pthread_mutex_lock(&p->worker_mutex); - while( (pWorker = p->pWorker) ){ - int rc = LSM_OK; - - /* Do some work. If an error occurs, exit. */ - - pthread_mutex_unlock(&p->worker_mutex); - if( p->eType==LSMTEST_THREAD_CKPT ){ - int nKB = 0; - rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ - rc = lsm_checkpoint(pWorker, 0); - } - }else{ - int nWrite; - do { - - if( p->eType==LSMTEST_THREAD_WORKER ){ - waitOnCheckpointer(p->pDb, pWorker); - } - - nWrite = 0; - rc = lsm_work(pWorker, 0, 256, &nWrite); - - if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){ - mt_signal_worker(p->pDb, 1); - } - }while( nWrite && p->pWorker ); - } - pthread_mutex_lock(&p->worker_mutex); - - if( rc!=LSM_OK && rc!=LSM_BUSY ){ - p->worker_rc = rc; - break; - } - - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection - ** is being closed. */ - if( p->pWorker && p->bDoWork==0 ){ - pthread_cond_wait(&p->worker_cond, &p->worker_mutex); - } - p->bDoWork = 0; - } - pthread_mutex_unlock(&p->worker_mutex); - - return 0; -} - - -static void mt_stop_worker(LsmDb *pDb, int iWorker){ - LsmWorker *p = &pDb->aWorker[iWorker]; - if( p->pWorker ){ - void *pDummy; - lsm_db *pWorker; - - /* Signal the worker to stop */ - pthread_mutex_lock(&p->worker_mutex); - pWorker = p->pWorker; - p->pWorker = 0; - pthread_cond_signal(&p->worker_cond); - pthread_mutex_unlock(&p->worker_mutex); - - /* Join the worker thread. */ - pthread_join(p->worker_thread, &pDummy); - - /* Free resources allocated in mt_start_worker() */ - pthread_cond_destroy(&p->worker_cond); - pthread_mutex_destroy(&p->worker_mutex); - lsm_close(pWorker); - } -} - -static void mt_shutdown(LsmDb *pDb){ - int i; - for(i=0; inWorker; i++){ - mt_stop_worker(pDb, i); - } -} - -/* -** This callback is invoked by LSM when the client database writes to -** the database file (i.e. to flush the contents of the in-memory tree). -** This implies there may be work to do on the database, so signal -** the worker threads. -*/ -static void mt_client_work_hook(lsm_db *db, void *pArg){ - LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - - /* Invoke the user level work-hook, if any. */ - if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); - - /* Wake up worker thread 0. */ - mt_signal_worker(pDb, 0); -} - -static void mt_worker_work_hook(lsm_db *db, void *pArg){ - LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - - /* Invoke the user level work-hook, if any. */ - if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); -} - -/* -** Launch worker thread iWorker for database connection pDb. -*/ -static int mt_start_worker( - LsmDb *pDb, /* Main database structure */ - int iWorker, /* Worker number to start */ - const char *zFilename, /* File name of database to open */ - const char *zCfg, /* Connection configuration string */ - int eType /* Type of worker thread */ -){ - int rc = 0; /* Return code */ - LsmWorker *p; /* Object to initialize */ - - assert( iWorkernWorker ); - assert( eType==LSMTEST_THREAD_CKPT - || eType==LSMTEST_THREAD_WORKER - || eType==LSMTEST_THREAD_WORKER_AC - ); - - p = &pDb->aWorker[iWorker]; - p->eType = eType; - p->pDb = pDb; - - /* Open the worker connection */ - if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); - if( zCfg ){ - test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0); - } - if( rc==0 ) rc = lsm_open(p->pWorker, zFilename); - lsm_config_log(p->pWorker, xLog, (void *)"worker"); - - /* Configure the work-hook */ - if( rc==0 ){ - lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb); - } - - if( eType==LSMTEST_THREAD_WORKER ){ - test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0); - } - - /* Kick off the worker thread. */ - if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0); - if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0); - if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); - - return rc; -} - - -static int testLsmStartWorkers( - LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg -){ - int rc; - - if( eModel<1 || eModel>4 ) return 1; - if( eModel==1 ) return 0; - - /* Configure a work-hook for the client connection. Worker 0 is signalled - ** every time the users connection writes to the database. */ - lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); - - /* Allocate space for two worker connections. They may not both be - ** used, but both are allocated. */ - pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); - memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); - - switch( eModel ){ - case LSMTEST_MODE_BACKGROUND_CKPT: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); - break; - - case LSMTEST_MODE_BACKGROUND_WORK: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC); - break; - - case LSMTEST_MODE_BACKGROUND_BOTH: - pDb->nWorker = 2; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER); - if( rc==0 ){ - rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT); - } - break; - } - - return rc; -} - - -int test_lsm_mt2( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "mt_mode=2"; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_mt3( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "mt_mode=4"; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -#else -static void mt_shutdown(LsmDb *pDb) { - unused_parameter(pDb); -} -int test_lsm_mt(const char *zFilename, int bClear, TestDb **ppDb){ - unused_parameter(zFilename); - unused_parameter(bClear); - unused_parameter(ppDb); - testPrintError("threads unavailable - recompile with LSM_MUTEX_PTHREADS\n"); - return 1; -} -#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb4.c b/ext/lsm1/lsm-test/lsmtest_tdb4.c deleted file mode 100644 index 1f92928522..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb4.c +++ /dev/null @@ -1,980 +0,0 @@ - -/* -** This file contains the TestDb bt wrapper. -*/ - -#include "lsmtest_tdb.h" -#include "lsmtest.h" -#include -#include "bt.h" - -#include - -typedef struct BtDb BtDb; -typedef struct BtFile BtFile; - -/* Background checkpointer interface (see implementations below). */ -typedef struct bt_ckpter bt_ckpter; -static int bgc_attach(BtDb *pDb, const char*); -static int bgc_detach(BtDb *pDb); - -/* -** Each database or log file opened by a database handle is wrapped by -** an object of the following type. -*/ -struct BtFile { - BtDb *pBt; /* Database handle that opened this file */ - bt_env *pVfs; /* Underlying VFS */ - bt_file *pFile; /* File handle belonging to underlying VFS */ - int nSectorSize; /* Size of sectors in bytes */ - int nSector; /* Allocated size of nSector array */ - u8 **apSector; /* Original sector data */ -}; - -/* -** nCrashSync: -** If this value is non-zero, then a "crash-test" is running. If -** nCrashSync==1, then the crash is simulated during the very next -** call to the xSync() VFS method (on either the db or log file). -** If nCrashSync==2, the following call to xSync(), and so on. -** -** bCrash: -** After a crash is simulated, this variable is set. Any subsequent -** attempts to write to a file or modify the file system in any way -** fail once this is set. All the caller can do is close the connection. -** -** bFastInsert: -** If this variable is set to true, then a BT_CONTROL_FAST_INSERT_OP -** control is issued before each callto BtReplace() or BtCsrOpen(). -*/ -struct BtDb { - TestDb base; /* Base class */ - bt_db *pBt; /* bt database handle */ - sqlite4_env *pEnv; /* SQLite environment (for malloc/free) */ - bt_env *pVfs; /* Underlying VFS */ - int bFastInsert; /* True to use fast-insert */ - - /* Space for bt_fetch() results */ - u8 *aBuffer; /* Space to store results */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ - int nRef; - - /* Background checkpointer used by mt connections */ - bt_ckpter *pCkpter; - - /* Stuff used for crash test simulation */ - BtFile *apFile[2]; /* Database and log files used by pBt */ - bt_env env; /* Private VFS for this object */ - int nCrashSync; /* Number of syncs until crash (see above) */ - int bCrash; /* True once a crash has been simulated */ -}; - -static int btVfsFullpath( - sqlite4_env *pEnv, - bt_env *pVfs, - const char *z, - char **pzOut -){ - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - if( pBt->bCrash ) return SQLITE4_IOERR; - return pBt->pVfs->xFullpath(pEnv, pBt->pVfs, z, pzOut); -} - -static int btVfsOpen( - sqlite4_env *pEnv, - bt_env *pVfs, - const char *zFile, - int flags, bt_file **ppFile -){ - BtFile *p; - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - int rc; - - if( pBt->bCrash ) return SQLITE4_IOERR; - - p = (BtFile*)testMalloc(sizeof(BtFile)); - if( !p ) return SQLITE4_NOMEM; - if( flags & BT_OPEN_DATABASE ){ - pBt->apFile[0] = p; - }else if( flags & BT_OPEN_LOG ){ - pBt->apFile[1] = p; - } - if( (flags & BT_OPEN_SHARED)==0 ){ - p->pBt = pBt; - } - p->pVfs = pBt->pVfs; - - rc = pBt->pVfs->xOpen(pEnv, pVfs, zFile, flags, &p->pFile); - if( rc!=SQLITE4_OK ){ - testFree(p); - p = 0; - }else{ - pBt->nRef++; - } - - *ppFile = (bt_file*)p; - return rc; -} - -static int btVfsSize(bt_file *pFile, sqlite4_int64 *piRes){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xSize(p->pFile, piRes); -} - -static int btVfsRead(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xRead(p->pFile, iOff, pBuf, nBuf); -} - -static int btFlushSectors(BtFile *p, int iFile){ - sqlite4_int64 iSz; - int rc; - int i; - u8 *aTmp = 0; - - rc = p->pBt->pVfs->xSize(p->pFile, &iSz); - for(i=0; rc==SQLITE4_OK && inSector; i++){ - if( p->pBt->bCrash && p->apSector[i] ){ - - /* The system is simulating a crash. There are three choices for - ** this sector: - ** - ** 1) Leave it as it is (simulating a successful write), - ** 2) Restore the original data (simulating a lost write), - ** 3) Populate the disk sector with garbage data. - */ - sqlite4_int64 iSOff = p->nSectorSize*i; - int nWrite = MIN(p->nSectorSize, iSz - iSOff); - - if( nWrite ){ - u8 *aWrite = 0; - int iOpt = (testPrngValue(i) % 3) + 1; - if( iOpt==1 ){ - aWrite = p->apSector[i]; - }else if( iOpt==3 ){ - if( aTmp==0 ) aTmp = testMalloc(p->nSectorSize); - aWrite = aTmp; - testPrngArray(i*13, (u32*)aWrite, nWrite/sizeof(u32)); - } - -#if 0 -fprintf(stderr, "handle sector %d of %s with %s\n", i, - iFile==0 ? "db" : "log", - iOpt==1 ? "rollback" : iOpt==2 ? "write" : "omit" -); -fflush(stderr); -#endif - - if( aWrite ){ - rc = p->pBt->pVfs->xWrite(p->pFile, iSOff, aWrite, nWrite); - } - } - } - testFree(p->apSector[i]); - p->apSector[i] = 0; - } - - testFree(aTmp); - return rc; -} - -static int btSaveSectors(BtFile *p, sqlite4_int64 iOff, int nBuf){ - int rc; - sqlite4_int64 iSz; /* Size of file on disk */ - int iFirst; /* First sector affected */ - int iSector; /* Current sector */ - int iLast; /* Last sector affected */ - - if( p->nSectorSize==0 ){ - p->nSectorSize = p->pBt->pVfs->xSectorSize(p->pFile); - if( p->nSectorSize<512 ) p->nSectorSize = 512; - } - iLast = (iOff+nBuf-1) / p->nSectorSize; - iFirst = iOff / p->nSectorSize; - - rc = p->pBt->pVfs->xSize(p->pFile, &iSz); - for(iSector=iFirst; rc==SQLITE4_OK && iSector<=iLast; iSector++){ - int nRead; - sqlite4_int64 iSOff = iSector * p->nSectorSize; - u8 *aBuf = testMalloc(p->nSectorSize); - nRead = MIN(p->nSectorSize, (iSz - iSOff)); - if( nRead>0 ){ - rc = p->pBt->pVfs->xRead(p->pFile, iSOff, aBuf, nRead); - } - - while( rc==SQLITE4_OK && iSector>=p->nSector ){ - int nNew = p->nSector + 32; - u8 **apNew = (u8**)testMalloc(nNew * sizeof(u8*)); - memcpy(apNew, p->apSector, p->nSector*sizeof(u8*)); - testFree(p->apSector); - p->apSector = apNew; - p->nSector = nNew; - } - - p->apSector[iSector] = aBuf; - } - - return rc; -} - -static int btVfsWrite(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - if( p->pBt && p->pBt->nCrashSync ){ - btSaveSectors(p, iOff, nBuf); - } - return p->pVfs->xWrite(p->pFile, iOff, pBuf, nBuf); -} - -static int btVfsTruncate(bt_file *pFile, sqlite4_int64 iOff){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xTruncate(p->pFile, iOff); -} - -static int btVfsSync(bt_file *pFile){ - int rc = SQLITE4_OK; - BtFile *p = (BtFile*)pFile; - BtDb *pBt = p->pBt; - - if( pBt ){ - if( pBt->bCrash ) return SQLITE4_IOERR; - if( pBt->nCrashSync ){ - pBt->nCrashSync--; - pBt->bCrash = (pBt->nCrashSync==0); - if( pBt->bCrash ){ - btFlushSectors(pBt->apFile[0], 0); - btFlushSectors(pBt->apFile[1], 1); - rc = SQLITE4_IOERR; - }else{ - btFlushSectors(p, 0); - } - } - } - - if( rc==SQLITE4_OK ){ - rc = p->pVfs->xSync(p->pFile); - } - return rc; -} - -static int btVfsSectorSize(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - return p->pVfs->xSectorSize(p->pFile); -} - -static void btDeref(BtDb *p){ - p->nRef--; - assert( p->nRef>=0 ); - if( p->nRef<=0 ) testFree(p); -} - -static int btVfsClose(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - BtDb *pBt = p->pBt; - int rc; - if( pBt ){ - btFlushSectors(p, 0); - if( p==pBt->apFile[0] ) pBt->apFile[0] = 0; - if( p==pBt->apFile[1] ) pBt->apFile[1] = 0; - } - testFree(p->apSector); - rc = p->pVfs->xClose(p->pFile); -#if 0 - btDeref(p->pBt); -#endif - testFree(p); - return rc; -} - -static int btVfsUnlink(sqlite4_env *pEnv, bt_env *pVfs, const char *zFile){ - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - if( pBt->bCrash ) return SQLITE4_IOERR; - return pBt->pVfs->xUnlink(pEnv, pBt->pVfs, zFile); -} - -static int btVfsLock(bt_file *pFile, int iLock, int eType){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xLock(p->pFile, iLock, eType); -} - -static int btVfsTestLock(bt_file *pFile, int iLock, int nLock, int eType){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xTestLock(p->pFile, iLock, nLock, eType); -} - -static int btVfsShmMap(bt_file *pFile, int iChunk, int sz, void **ppOut){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xShmMap(p->pFile, iChunk, sz, ppOut); -} - -static void btVfsShmBarrier(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - return p->pVfs->xShmBarrier(p->pFile); -} - -static int btVfsShmUnmap(bt_file *pFile, int bDelete){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xShmUnmap(p->pFile, bDelete); -} - -static int bt_close(TestDb *pTestDb){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtClose(p->pBt); - free(p->aBuffer); - if( p->apFile[0] ) p->apFile[0]->pBt = 0; - if( p->apFile[1] ) p->apFile[1]->pBt = 0; - bgc_detach(p); - testFree(p); - return rc; -} - -static int btMinTransaction(BtDb *p, int iMin, int *piLevel){ - int iLevel; - int rc = SQLITE4_OK; - - iLevel = sqlite4BtTransactionLevel(p->pBt); - if( iLevelpBt, iMin); - *piLevel = iLevel; - }else{ - *piLevel = -1; - } - - return rc; -} -static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){ - int rc = rcin; - if( iLevel>=0 ){ - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCommit(p->pBt, iLevel); - }else{ - sqlite4BtRollback(p->pBt, iLevel); - } - assert( iLevel==sqlite4BtTransactionLevel(p->pBt) ); - } - return rc; -} - -static int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){ - BtDb *p = (BtDb*)pTestDb; - int iLevel; - int rc; - - rc = btMinTransaction(p, 2, &iLevel); - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV); - rc = btRestoreTransaction(p, iLevel, rc); - } - return rc; -} - -static int bt_delete(TestDb *pTestDb, void *pK, int nK){ - return bt_write(pTestDb, pK, nK, 0, -1); -} - -static int bt_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int rc = SQLITE4_OK; - int iLevel; - - rc = btMinTransaction(p, 2, &iLevel); - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - } - while( rc==SQLITE4_OK ){ - const void *pK; - int n; - int nCmp; - int res; - - rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE); - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - if( rc!=SQLITE4_OK ) break; - - rc = sqlite4BtCsrKey(pCsr, &pK, &n); - if( rc!=SQLITE4_OK ) break; - - nCmp = MIN(n, nKey1); - res = memcmp(pKey1, pK, nCmp); - assert( res<0 || (res==0 && nKey1<=n) ); - if( res==0 && nKey1==n ){ - rc = sqlite4BtCsrNext(pCsr); - if( rc!=SQLITE4_OK ) break; - rc = sqlite4BtCsrKey(pCsr, &pK, &n); - if( rc!=SQLITE4_OK ) break; - } - - nCmp = MIN(n, nKey2); - res = memcmp(pKey2, pK, nCmp); - if( res<0 || (res==0 && nKey2<=n) ) break; - - rc = sqlite4BtDelete(pCsr); - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - sqlite4BtCsrClose(pCsr); - - rc = btRestoreTransaction(p, iLevel, rc); - return rc; -} - -static int bt_fetch( - TestDb *pTestDb, - void *pK, int nK, - void **ppVal, int *pnVal -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int iLevel; - int rc = SQLITE4_OK; - - iLevel = sqlite4BtTransactionLevel(p->pBt); - if( iLevel==0 ){ - rc = sqlite4BtBegin(p->pBt, 1); - if( rc!=SQLITE4_OK ) return rc; - } - - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ); - if( rc==SQLITE4_OK ){ - const void *pV = 0; - int nV = 0; - rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); - if( rc==SQLITE4_OK ){ - if( nV>p->nBuffer ){ - free(p->aBuffer); - p->aBuffer = (u8*)malloc(nV*2); - p->nBuffer = nV*2; - } - memcpy(p->aBuffer, pV, nV); - *pnVal = nV; - *ppVal = (void*)(p->aBuffer); - } - - }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){ - *ppVal = 0; - *pnVal = -1; - rc = SQLITE4_OK; - } - sqlite4BtCsrClose(pCsr); - } - - if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0); - return rc; -} - -static int bt_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int rc; - int iLevel; - - rc = btMinTransaction(p, 1, &iLevel); - - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - } - if( rc==SQLITE4_OK ){ - if( bReverse ){ - if( pLast ){ - rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE); - }else{ - rc = sqlite4BtCsrLast(pCsr); - } - }else{ - rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE); - } - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - - while( rc==SQLITE4_OK ){ - const void *pK = 0; int nK = 0; - const void *pV = 0; int nV = 0; - - rc = sqlite4BtCsrKey(pCsr, &pK, &nK); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); - } - - if( rc!=SQLITE4_OK ) break; - if( bReverse ){ - if( pFirst ){ - int res; - int nCmp = MIN(nK, nFirst); - res = memcmp(pFirst, pK, nCmp); - if( res>0 || (res==0 && nKnLast) ) break; - } - } - - xCallback(pCtx, (void*)pK, nK, (void*)pV, nV); - if( bReverse ){ - rc = sqlite4BtCsrPrev(pCsr); - }else{ - rc = sqlite4BtCsrNext(pCsr); - } - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - sqlite4BtCsrClose(pCsr); - } - - rc = btRestoreTransaction(p, iLevel, rc); - return rc; -} - -static int bt_begin(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtBegin(p->pBt, iLvl); - return rc; -} - -static int bt_commit(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtCommit(p->pBt, iLvl); - return rc; -} - -static int bt_rollback(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtRollback(p->pBt, iLvl); - return rc; -} - -static int testParseOption( - const char **pzIn, /* IN/OUT: pointer to next option */ - const char **pzOpt, /* OUT: nul-terminated option name */ - const char **pzArg, /* OUT: nul-terminated option argument */ - char *pSpace /* Temporary space for output params */ -){ - const char *p = *pzIn; - const char *pStart; - int n; - - char *pOut = pSpace; - - while( *p==' ' ) p++; - pStart = p; - while( *p && *p!='=' ) p++; - if( *p==0 ) return 1; - - n = (p - pStart); - memcpy(pOut, pStart, n); - *pzOpt = pOut; - pOut += n; - *pOut++ = '\0'; - - p++; - pStart = p; - while( *p && *p!=' ' ) p++; - n = (p - pStart); - - memcpy(pOut, pStart, n); - *pzArg = pOut; - pOut += n; - *pOut++ = '\0'; - - *pzIn = p; - return 0; -} - -static int testParseInt(const char *z, int *piVal){ - int i = 0; - const char *p = z; - - while( *p>='0' && *p<='9' ){ - i = i*10 + (*p - '0'); - p++; - } - if( *p=='K' || *p=='k' ){ - i = i * 1024; - p++; - }else if( *p=='M' || *p=='m' ){ - i = i * 1024 * 1024; - p++; - } - - if( *p ) return SQLITE4_ERROR; - *piVal = i; - return SQLITE4_OK; -} - -static int testBtConfigure(BtDb *pDb, const char *zCfg, int *pbMt){ - int rc = SQLITE4_OK; - - if( zCfg ){ - struct CfgParam { - const char *zParam; - int eParam; - } aParam[] = { - { "safety", BT_CONTROL_SAFETY }, - { "autockpt", BT_CONTROL_AUTOCKPT }, - { "multiproc", BT_CONTROL_MULTIPROC }, - { "blksz", BT_CONTROL_BLKSZ }, - { "pagesz", BT_CONTROL_PAGESZ }, - { "mt", -1 }, - { "fastinsert", -2 }, - { 0, 0 } - }; - const char *z = zCfg; - int n = strlen(z); - char *aSpace; - const char *zOpt; - const char *zArg; - - aSpace = (char*)testMalloc(n+2); - while( rc==SQLITE4_OK && 0==testParseOption(&z, &zOpt, &zArg, aSpace) ){ - int i; - int iVal; - rc = testArgSelect(aParam, "param", zOpt, &i); - if( rc!=SQLITE4_OK ) break; - - rc = testParseInt(zArg, &iVal); - if( rc!=SQLITE4_OK ) break; - - switch( aParam[i].eParam ){ - case -1: - *pbMt = iVal; - break; - case -2: - pDb->bFastInsert = 1; - break; - default: - rc = sqlite4BtControl(pDb->pBt, aParam[i].eParam, (void*)&iVal); - break; - } - } - testFree(aSpace); - } - - return rc; -} - - -int test_bt_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - - static const DatabaseMethods SqlMethods = { - bt_close, - bt_write, - bt_delete, - bt_delete_range, - bt_fetch, - bt_scan, - bt_begin, - bt_commit, - bt_rollback - }; - BtDb *p = 0; - bt_db *pBt = 0; - int rc; - sqlite4_env *pEnv = sqlite4_env_default(); - - if( bClear && zFilename && zFilename[0] ){ - char *zLog = sqlite3_mprintf("%s-wal", zFilename); - unlink(zFilename); - unlink(zLog); - sqlite3_free(zLog); - } - - rc = sqlite4BtNew(pEnv, 0, &pBt); - if( rc==SQLITE4_OK ){ - int mt = 0; /* True for multi-threaded connection */ - - p = (BtDb*)testMalloc(sizeof(BtDb)); - p->base.pMethods = &SqlMethods; - p->pBt = pBt; - p->pEnv = pEnv; - p->nRef = 1; - - p->env.pVfsCtx = (void*)p; - p->env.xFullpath = btVfsFullpath; - p->env.xOpen = btVfsOpen; - p->env.xSize = btVfsSize; - p->env.xRead = btVfsRead; - p->env.xWrite = btVfsWrite; - p->env.xTruncate = btVfsTruncate; - p->env.xSync = btVfsSync; - p->env.xSectorSize = btVfsSectorSize; - p->env.xClose = btVfsClose; - p->env.xUnlink = btVfsUnlink; - p->env.xLock = btVfsLock; - p->env.xTestLock = btVfsTestLock; - p->env.xShmMap = btVfsShmMap; - p->env.xShmBarrier = btVfsShmBarrier; - p->env.xShmUnmap = btVfsShmUnmap; - - sqlite4BtControl(pBt, BT_CONTROL_GETVFS, (void*)&p->pVfs); - sqlite4BtControl(pBt, BT_CONTROL_SETVFS, (void*)&p->env); - - rc = testBtConfigure(p, zSpec, &mt); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtOpen(pBt, zFilename); - } - - if( rc==SQLITE4_OK && mt ){ - int nAuto = 0; - rc = bgc_attach(p, zSpec); - sqlite4BtControl(pBt, BT_CONTROL_AUTOCKPT, (void*)&nAuto); - } - } - - if( rc!=SQLITE4_OK && p ){ - bt_close(&p->base); - } - - *ppDb = &p->base; - return rc; -} - -int test_fbt_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return test_bt_open("fast=1", zFilename, bClear, ppDb); -} - -int test_fbts_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return test_bt_open("fast=1 blksz=32K pagesz=512", zFilename, bClear, ppDb); -} - - -void tdb_bt_prepare_sync_crash(TestDb *pTestDb, int iSync){ - BtDb *p = (BtDb*)pTestDb; - assert( pTestDb->pMethods->xClose==bt_close ); - assert( p->bCrash==0 ); - p->nCrashSync = iSync; -} - -bt_db *tdb_bt(TestDb *pDb){ - if( pDb->pMethods->xClose==bt_close ){ - return ((BtDb *)pDb)->pBt; - } - return 0; -} - -/************************************************************************* -** Beginning of code for background checkpointer. -*/ - -struct bt_ckpter { - sqlite4_buffer file; /* File name */ - sqlite4_buffer spec; /* Options */ - int nLogsize; /* Minimum log size to checkpoint */ - int nRef; /* Number of clients */ - - int bDoWork; /* Set by client threads */ - pthread_t ckpter_thread; /* Checkpointer thread */ - pthread_cond_t ckpter_cond; /* Condition var the ckpter waits on */ - pthread_mutex_t ckpter_mutex; /* Mutex used with ckpter_cond */ - - bt_ckpter *pNext; /* Next object in list at gBgc.pCkpter */ -}; - -static struct GlobalBackgroundCheckpointer { - bt_ckpter *pCkpter; /* Linked list of checkpointers */ -} gBgc; - -static void *bgc_main(void *pArg){ - BtDb *pDb = 0; - int rc; - int mt; - bt_ckpter *pCkpter = (bt_ckpter*)pArg; - - rc = test_bt_open("", (char*)pCkpter->file.p, 0, (TestDb**)&pDb); - assert( rc==SQLITE4_OK ); - rc = testBtConfigure(pDb, (char*)pCkpter->spec.p, &mt); - - while( pCkpter->nRef>0 ){ - bt_db *db = pDb->pBt; - int nLog = 0; - - sqlite4BtBegin(db, 1); - sqlite4BtCommit(db, 0); - sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); - - if( nLog>=pCkpter->nLogsize ){ - int rc; - bt_checkpoint ckpt; - memset(&ckpt, 0, sizeof(bt_checkpoint)); - ckpt.nFrameBuffer = nLog/2; - rc = sqlite4BtControl(db, BT_CONTROL_CHECKPOINT, (void*)&ckpt); - assert( rc==SQLITE4_OK ); - sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); - } - - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection - ** is being closed. */ - pthread_mutex_lock(&pCkpter->ckpter_mutex); - if( pCkpter->bDoWork==0 ){ - pthread_cond_wait(&pCkpter->ckpter_cond, &pCkpter->ckpter_mutex); - } - pCkpter->bDoWork = 0; - pthread_mutex_unlock(&pCkpter->ckpter_mutex); - } - - if( pDb ) bt_close((TestDb*)pDb); - return 0; -} - -static void bgc_logsize_cb(void *pCtx, int nLogsize){ - bt_ckpter *p = (bt_ckpter*)pCtx; - if( nLogsize>=p->nLogsize ){ - pthread_mutex_lock(&p->ckpter_mutex); - p->bDoWork = 1; - pthread_cond_signal(&p->ckpter_cond); - pthread_mutex_unlock(&p->ckpter_mutex); - } -} - -static int bgc_attach(BtDb *pDb, const char *zSpec){ - int rc; - int n; - bt_info info; - bt_ckpter *pCkpter; - - /* Figure out the full path to the database opened by handle pDb. */ - info.eType = BT_INFO_FILENAME; - info.pgno = 0; - sqlite4_buffer_init(&info.output, 0); - rc = sqlite4BtControl(pDb->pBt, BT_CONTROL_INFO, (void*)&info); - if( rc!=SQLITE4_OK ) return rc; - - sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); - - /* Search for an existing bt_ckpter object. */ - n = info.output.n; - for(pCkpter=gBgc.pCkpter; pCkpter; pCkpter=pCkpter->pNext){ - if( n==pCkpter->file.n && 0==memcmp(info.output.p, pCkpter->file.p, n) ){ - break; - } - } - - /* Failed to find a suitable checkpointer. Create a new one. */ - if( pCkpter==0 ){ - bt_logsizecb cb; - - pCkpter = testMalloc(sizeof(bt_ckpter)); - memcpy(&pCkpter->file, &info.output, sizeof(sqlite4_buffer)); - info.output.p = 0; - pCkpter->pNext = gBgc.pCkpter; - pCkpter->nLogsize = 1000; - gBgc.pCkpter = pCkpter; - pCkpter->nRef = 1; - - sqlite4_buffer_init(&pCkpter->spec, 0); - rc = sqlite4_buffer_set(&pCkpter->spec, zSpec, strlen(zSpec)+1); - assert( rc==SQLITE4_OK ); - - /* Kick off the checkpointer thread. */ - if( rc==0 ) rc = pthread_cond_init(&pCkpter->ckpter_cond, 0); - if( rc==0 ) rc = pthread_mutex_init(&pCkpter->ckpter_mutex, 0); - if( rc==0 ){ - rc = pthread_create(&pCkpter->ckpter_thread, 0, bgc_main, (void*)pCkpter); - } - assert( rc==0 ); /* todo: Fix this */ - - /* Set up the logsize callback for the client thread */ - cb.pCtx = (void*)pCkpter; - cb.xLogsize = bgc_logsize_cb; - sqlite4BtControl(pDb->pBt, BT_CONTROL_LOGSIZECB, (void*)&cb); - }else{ - pCkpter->nRef++; - } - - /* Assuming a checkpointer was encountered or effected, attach the - ** connection to it. */ - if( pCkpter ){ - pDb->pCkpter = pCkpter; - } - - sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); - sqlite4_buffer_clear(&info.output); - return rc; -} - -static int bgc_detach(BtDb *pDb){ - int rc = SQLITE4_OK; - bt_ckpter *pCkpter = pDb->pCkpter; - if( pCkpter ){ - int bShutdown = 0; /* True if this is the last reference */ - - sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); - pCkpter->nRef--; - if( pCkpter->nRef==0 ){ - bt_ckpter **pp; - - *pp = pCkpter->pNext; - for(pp=&gBgc.pCkpter; *pp!=pCkpter; pp=&((*pp)->pNext)); - bShutdown = 1; - } - sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); - - if( bShutdown ){ - void *pDummy; - - /* Signal the checkpointer thread. */ - pthread_mutex_lock(&pCkpter->ckpter_mutex); - pCkpter->bDoWork = 1; - pthread_cond_signal(&pCkpter->ckpter_cond); - pthread_mutex_unlock(&pCkpter->ckpter_mutex); - - /* Join the checkpointer thread. */ - pthread_join(pCkpter->ckpter_thread, &pDummy); - pthread_cond_destroy(&pCkpter->ckpter_cond); - pthread_mutex_destroy(&pCkpter->ckpter_mutex); - - sqlite4_buffer_clear(&pCkpter->file); - sqlite4_buffer_clear(&pCkpter->spec); - testFree(pCkpter); - } - - pDb->pCkpter = 0; - } - return rc; -} - -/* -** End of background checkpointer. -*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_util.c b/ext/lsm1/lsm-test/lsmtest_util.c deleted file mode 100644 index adab8a53e8..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_util.c +++ /dev/null @@ -1,223 +0,0 @@ - -#include "lsmtest.h" -#include -#include -#include -#ifndef _WIN32 -# include -#endif - -/* -** Global variables used within this module. -*/ -static struct TestutilGlobal { - char **argv; - int argc; -} g = {0, 0}; - -static struct TestutilRnd { - unsigned int aRand1[2048]; /* Bits 0..10 */ - unsigned int aRand2[2048]; /* Bits 11..21 */ - unsigned int aRand3[1024]; /* Bits 22..31 */ -} r; - -/************************************************************************* -** The following block is a copy of the implementation of SQLite function -** sqlite3_randomness. This version has two important differences: -** -** 1. It always uses the same seed. So the sequence of random data output -** is the same for every run of the program. -** -** 2. It is not threadsafe. -*/ -static struct sqlite3PrngType { - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ -} sqlite3Prng = { - 0xAF, 0x28, - { - 0x71, 0xF5, 0xB4, 0x6E, 0x80, 0xAB, 0x1D, 0xB8, - 0xFB, 0xB7, 0x49, 0xBF, 0xFF, 0x72, 0x2D, 0x14, - 0x79, 0x09, 0xE3, 0x78, 0x76, 0xB0, 0x2C, 0x0A, - 0x8E, 0x23, 0xEE, 0xDF, 0xE0, 0x9A, 0x2F, 0x67, - 0xE1, 0xBE, 0x0E, 0xA7, 0x08, 0x97, 0xEB, 0x77, - 0x78, 0xBA, 0x9D, 0xCA, 0x49, 0x4C, 0x60, 0x9A, - 0xF6, 0xBD, 0xDA, 0x7F, 0xBC, 0x48, 0x58, 0x52, - 0xE5, 0xCD, 0x83, 0x72, 0x23, 0x52, 0xFF, 0x6D, - 0xEF, 0x0F, 0x82, 0x29, 0xA0, 0x83, 0x3F, 0x7D, - 0xA4, 0x88, 0x31, 0xE7, 0x88, 0x92, 0x3B, 0x9B, - 0x3B, 0x2C, 0xC2, 0x4C, 0x71, 0xA2, 0xB0, 0xEA, - 0x36, 0xD0, 0x00, 0xF1, 0xD3, 0x39, 0x17, 0x5D, - 0x2A, 0x7A, 0xE4, 0xAD, 0xE1, 0x64, 0xCE, 0x0F, - 0x9C, 0xD9, 0xF5, 0xED, 0xB0, 0x22, 0x5E, 0x62, - 0x97, 0x02, 0xA3, 0x8C, 0x67, 0x80, 0xFC, 0x88, - 0x14, 0x0B, 0x15, 0x10, 0x0F, 0xC7, 0x40, 0xD4, - 0xF1, 0xF9, 0x0E, 0x1A, 0xCE, 0xB9, 0x1E, 0xA1, - 0x72, 0x8E, 0xD7, 0x78, 0x39, 0xCD, 0xF4, 0x5D, - 0x2A, 0x59, 0x26, 0x34, 0xF2, 0x73, 0x0B, 0xA0, - 0x02, 0x51, 0x2C, 0x03, 0xA3, 0xA7, 0x43, 0x13, - 0xE8, 0x98, 0x2B, 0xD2, 0x53, 0xF8, 0xEE, 0x91, - 0x7D, 0xE7, 0xE3, 0xDA, 0xD5, 0xBB, 0xC0, 0x92, - 0x9D, 0x98, 0x01, 0x2C, 0xF9, 0xB9, 0xA0, 0xEB, - 0xCF, 0x32, 0xFA, 0x01, 0x49, 0xA5, 0x1D, 0x9A, - 0x76, 0x86, 0x3F, 0x40, 0xD4, 0x89, 0x8F, 0x9C, - 0xE2, 0xE3, 0x11, 0x31, 0x37, 0xB2, 0x49, 0x28, - 0x35, 0xC0, 0x99, 0xB6, 0xD0, 0xBC, 0x66, 0x35, - 0xF7, 0x83, 0x5B, 0xD7, 0x37, 0x1A, 0x2B, 0x18, - 0xA6, 0xFF, 0x8D, 0x7C, 0x81, 0xA8, 0xFC, 0x9E, - 0xC4, 0xEC, 0x80, 0xD0, 0x98, 0xA7, 0x76, 0xCC, - 0x9C, 0x2F, 0x7B, 0xFF, 0x8E, 0x0E, 0xBB, 0x90, - 0xAE, 0x13, 0x06, 0xF5, 0x1C, 0x4E, 0x52, 0xF7 - } -}; - -/* Generate and return single random byte */ -static unsigned char randomByte(void){ - unsigned char t; - sqlite3Prng.i++; - t = sqlite3Prng.s[sqlite3Prng.i]; - sqlite3Prng.j += t; - sqlite3Prng.s[sqlite3Prng.i] = sqlite3Prng.s[sqlite3Prng.j]; - sqlite3Prng.s[sqlite3Prng.j] = t; - t += sqlite3Prng.s[sqlite3Prng.i]; - return sqlite3Prng.s[t]; -} - -/* -** Return N random bytes. -*/ -static void randomBlob(int nBuf, unsigned char *zBuf){ - int i; - for(i=0; i>11) & 0x000007FF] ^ - r.aRand3[(iVal>>22) & 0x000003FF] - ; -} - -void testPrngArray(unsigned int iVal, unsigned int *aOut, int nOut){ - int i; - for(i=0; izName; - pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] - ){ - if( zPrev ){ testPrintError("%s, ", zPrev); } - zPrev = pEntry->zName; - } - testPrintError("or %s\n", zPrev); -} - -int testArgSelectX( - void *aData, - const char *zType, - int sz, - const char *zArg, - int *piOut -){ - struct Entry { const char *zName; }; - struct Entry *pEntry; - int nArg = strlen(zArg); - - int i = 0; - int iOut = -1; - int nOut = 0; - - for(pEntry=(struct Entry *)aData; - pEntry->zName; - pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] - ){ - int nName = strlen(pEntry->zName); - if( nArg<=nName && memcmp(pEntry->zName, zArg, nArg)==0 ){ - iOut = i; - if( nName==nArg ){ - nOut = 1; - break; - } - nOut++; - } - i++; - } - - if( nOut!=1 ){ - argError(aData, zType, sz, zArg); - }else{ - *piOut = iOut; - } - return (nOut!=1); -} - -struct timeval zero_time; - -void testTimeInit(void){ - gettimeofday(&zero_time, 0); -} - -int testTimeGet(void){ - struct timeval now; - gettimeofday(&now, 0); - return - (((int)now.tv_sec - (int)zero_time.tv_sec)*1000) + - (((int)now.tv_usec - (int)zero_time.tv_usec)/1000); -} diff --git a/ext/lsm1/lsm-test/lsmtest_win32.c b/ext/lsm1/lsm-test/lsmtest_win32.c deleted file mode 100644 index 9472723368..0000000000 --- a/ext/lsm1/lsm-test/lsmtest_win32.c +++ /dev/null @@ -1,30 +0,0 @@ - -#include "lsmtest.h" - -#ifdef _WIN32 - -#define TICKS_PER_SECOND (10000000) -#define TICKS_PER_MICROSECOND (10) -#define TICKS_UNIX_EPOCH (116444736000000000LL) - -int win32GetTimeOfDay( - struct timeval *tp, - void *tzp -){ - FILETIME fileTime; - ULONGLONG ticks; - ULONGLONG unixTicks; - - unused_parameter(tzp); - memset(&fileTime, 0, sizeof(FILETIME)); - GetSystemTimeAsFileTime(&fileTime); - ticks = (ULONGLONG)fileTime.dwHighDateTime << 32; - ticks |= (ULONGLONG)fileTime.dwLowDateTime; - unixTicks = ticks - TICKS_UNIX_EPOCH; - tp->tv_sec = (long)(unixTicks / TICKS_PER_SECOND); - unixTicks -= ((ULONGLONG)tp->tv_sec * TICKS_PER_SECOND); - tp->tv_usec = (long)(unixTicks / TICKS_PER_MICROSECOND); - - return 0; -} -#endif diff --git a/ext/lsm1/lsm.h b/ext/lsm1/lsm.h deleted file mode 100644 index 48701c4c5e..0000000000 --- a/ext/lsm1/lsm.h +++ /dev/null @@ -1,684 +0,0 @@ -/* -** 2011-08-10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file defines the LSM API. -*/ -#ifndef _LSM_H -#define _LSM_H -#include -#ifdef __cplusplus -extern "C" { -#endif - -/* -** Opaque handle types. -*/ -typedef struct lsm_compress lsm_compress; /* Compression library functions */ -typedef struct lsm_compress_factory lsm_compress_factory; -typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ -typedef struct lsm_db lsm_db; /* Database connection handle */ -typedef struct lsm_env lsm_env; /* Runtime environment */ -typedef struct lsm_file lsm_file; /* OS file handle */ -typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ - -/* 64-bit integer type used for file offsets. */ -typedef long long int lsm_i64; /* 64-bit signed integer type */ - -/* Candidate values for the 3rd argument to lsm_env.xLock() */ -#define LSM_LOCK_UNLOCK 0 -#define LSM_LOCK_SHARED 1 -#define LSM_LOCK_EXCL 2 - -/* Flags for lsm_env.xOpen() */ -#define LSM_OPEN_READONLY 0x0001 - -/* -** CAPI: Database Runtime Environment -** -** Run-time environment used by LSM -*/ -struct lsm_env { - int nByte; /* Size of this structure in bytes */ - int iVersion; /* Version number of this structure (1) */ - /****** file i/o ***********************************************/ - void *pVfsCtx; - int (*xFullpath)(lsm_env*, const char *, char *, int *); - int (*xOpen)(lsm_env*, const char *, int flags, lsm_file **); - int (*xRead)(lsm_file *, lsm_i64, void *, int); - int (*xWrite)(lsm_file *, lsm_i64, void *, int); - int (*xTruncate)(lsm_file *, lsm_i64); - int (*xSync)(lsm_file *); - int (*xSectorSize)(lsm_file *); - int (*xRemap)(lsm_file *, lsm_i64, void **, lsm_i64*); - int (*xFileid)(lsm_file *, void *pBuf, int *pnBuf); - int (*xClose)(lsm_file *); - int (*xUnlink)(lsm_env*, const char *); - int (*xLock)(lsm_file*, int, int); - int (*xTestLock)(lsm_file*, int, int, int); - int (*xShmMap)(lsm_file*, int, int, void **); - void (*xShmBarrier)(void); - int (*xShmUnmap)(lsm_file*, int); - /****** memory allocation ****************************************/ - void *pMemCtx; - void *(*xMalloc)(lsm_env*, size_t); /* malloc(3) function */ - void *(*xRealloc)(lsm_env*, void *, size_t); /* realloc(3) function */ - void (*xFree)(lsm_env*, void *); /* free(3) function */ - size_t (*xSize)(lsm_env*, void *); /* xSize function */ - /****** mutexes ****************************************************/ - void *pMutexCtx; - int (*xMutexStatic)(lsm_env*,int,lsm_mutex**); /* Obtain a static mutex */ - int (*xMutexNew)(lsm_env*, lsm_mutex**); /* Get a new dynamic mutex */ - void (*xMutexDel)(lsm_mutex *); /* Delete an allocated mutex */ - void (*xMutexEnter)(lsm_mutex *); /* Grab a mutex */ - int (*xMutexTry)(lsm_mutex *); /* Attempt to obtain a mutex */ - void (*xMutexLeave)(lsm_mutex *); /* Leave a mutex */ - int (*xMutexHeld)(lsm_mutex *); /* Return true if mutex is held */ - int (*xMutexNotHeld)(lsm_mutex *); /* Return true if mutex not held */ - /****** other ****************************************************/ - int (*xSleep)(lsm_env*, int microseconds); - - /* New fields may be added in future releases, in which case the - ** iVersion value will increase. */ -}; - -/* -** Values that may be passed as the second argument to xMutexStatic. -*/ -#define LSM_MUTEX_GLOBAL 1 -#define LSM_MUTEX_HEAP 2 - -/* -** CAPI: LSM Error Codes -*/ -#define LSM_OK 0 -#define LSM_ERROR 1 -#define LSM_BUSY 5 -#define LSM_NOMEM 7 -#define LSM_READONLY 8 -#define LSM_IOERR 10 -#define LSM_CORRUPT 11 -#define LSM_FULL 13 -#define LSM_CANTOPEN 14 -#define LSM_PROTOCOL 15 -#define LSM_MISUSE 21 - -#define LSM_MISMATCH 50 - - -#define LSM_IOERR_NOENT (LSM_IOERR | (1<<8)) - -/* -** CAPI: Creating and Destroying Database Connection Handles -** -** Open and close a database connection handle. -*/ -int lsm_new(lsm_env*, lsm_db **ppDb); -int lsm_close(lsm_db *pDb); - -/* -** CAPI: Connecting to a Database -*/ -int lsm_open(lsm_db *pDb, const char *zFilename); - -/* -** CAPI: Obtaining pointers to database environments -** -** Return a pointer to the environment used by the database connection -** passed as the first argument. Assuming the argument is valid, this -** function always returns a valid environment pointer - it cannot fail. -*/ -lsm_env *lsm_get_env(lsm_db *pDb); - -/* -** The lsm_default_env() function returns a pointer to the default LSM -** environment for the current platform. -*/ -lsm_env *lsm_default_env(void); - - -/* -** CAPI: Configuring a database connection. -** -** The lsm_config() function is used to configure a database connection. -*/ -int lsm_config(lsm_db *, int, ...); - -/* -** The following values may be passed as the second argument to lsm_config(). -** -** LSM_CONFIG_AUTOFLUSH: -** A read/write integer parameter. -** -** This value determines the amount of data allowed to accumulate in a -** live in-memory tree before it is marked as old. After committing a -** transaction, a connection checks if the size of the live in-memory tree, -** including data structure overhead, is greater than the value of this -** option in KB. If it is, and there is not already an old in-memory tree, -** the live in-memory tree is marked as old. -** -** The maximum allowable value is 1048576 (1GB). There is no minimum -** value. If this parameter is set to zero, then an attempt is made to -** mark the live in-memory tree as old after each transaction is committed. -** -** The default value is 1024 (1MB). -** -** LSM_CONFIG_PAGE_SIZE: -** A read/write integer parameter. This parameter may only be set before -** lsm_open() has been called. -** -** LSM_CONFIG_BLOCK_SIZE: -** A read/write integer parameter. -** -** This parameter may only be set before lsm_open() has been called. It -** must be set to a power of two between 64 and 65536, inclusive (block -** sizes between 64KB and 64MB). -** -** If the connection creates a new database, the block size of the new -** database is set to the value of this option in KB. After lsm_open() -** has been called, querying this parameter returns the actual block -** size of the opened database. -** -** The default value is 1024 (1MB blocks). -** -** LSM_CONFIG_SAFETY: -** A read/write integer parameter. Valid values are 0, 1 (the default) -** and 2. This parameter determines how robust the database is in the -** face of a system crash (e.g. a power failure or operating system -** crash). As follows: -** -** 0 (off): No robustness. A system crash may corrupt the database. -** -** 1 (normal): Some robustness. A system crash may not corrupt the -** database file, but recently committed transactions may -** be lost following recovery. -** -** 2 (full): Full robustness. A system crash may not corrupt the -** database file. Following recovery the database file -** contains all successfully committed transactions. -** -** LSM_CONFIG_AUTOWORK: -** A read/write integer parameter. -** -** LSM_CONFIG_AUTOCHECKPOINT: -** A read/write integer parameter. -** -** If this option is set to non-zero value N, then a checkpoint is -** automatically attempted after each N KB of data have been written to -** the database file. -** -** The amount of uncheckpointed data already written to the database file -** is a global parameter. After performing database work (writing to the -** database file), the process checks if the total amount of uncheckpointed -** data exceeds the value of this paramter. If so, a checkpoint is performed. -** This means that this option may cause the connection to perform a -** checkpoint even if the current connection has itself written very little -** data into the database file. -** -** The default value is 2048 (checkpoint every 2MB). -** -** LSM_CONFIG_MMAP: -** A read/write integer parameter. If this value is set to 0, then the -** database file is accessed using ordinary read/write IO functions. Or, -** if it is set to 1, then the database file is memory mapped and accessed -** that way. If this parameter is set to any value N greater than 1, then -** up to the first N KB of the file are memory mapped, and any remainder -** accessed using read/write IO. -** -** The default value is 1 on 64-bit platforms and 32768 on 32-bit platforms. -** -** -** LSM_CONFIG_USE_LOG: -** A read/write boolean parameter. True (the default) to use the log -** file normally. False otherwise. -** -** LSM_CONFIG_AUTOMERGE: -** A read/write integer parameter. The minimum number of segments to -** merge together at a time. Default value 4. -** -** LSM_CONFIG_MAX_FREELIST: -** A read/write integer parameter. The maximum number of free-list -** entries that are stored in a database checkpoint (the others are -** stored elsewhere in the database). -** -** There is no reason for an application to configure or query this -** parameter. It is only present because configuring a small value -** makes certain parts of the lsm code easier to test. -** -** LSM_CONFIG_MULTIPLE_PROCESSES: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() has been called. If true, the library uses shared-memory -** and posix advisory locks to co-ordinate access by clients from within -** multiple processes. Otherwise, if false, all database clients must be -** located in the same process. The default value is true. -** -** LSM_CONFIG_SET_COMPRESSION: -** Set the compression methods used to compress and decompress database -** content. The argument to this option should be a pointer to a structure -** of type lsm_compress. The lsm_config() method takes a copy of the -** structures contents. -** -** This option may only be used before lsm_open() is called. Invoking it -** after lsm_open() has been called results in an LSM_MISUSE error. -** -** LSM_CONFIG_GET_COMPRESSION: -** Query the compression methods used to compress and decompress database -** content. -** -** LSM_CONFIG_SET_COMPRESSION_FACTORY: -** Configure a factory method to be invoked in case of an LSM_MISMATCH -** error. -** -** LSM_CONFIG_READONLY: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() is called. -*/ -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 -#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 -#define LSM_CONFIG_READONLY 16 - -#define LSM_SAFETY_OFF 0 -#define LSM_SAFETY_NORMAL 1 -#define LSM_SAFETY_FULL 2 - -/* -** CAPI: Compression and/or Encryption Hooks -*/ -struct lsm_compress { - void *pCtx; - unsigned int iId; - int (*xBound)(void *, int nSrc); - int (*xCompress)(void *, char *, int *, const char *, int); - int (*xUncompress)(void *, char *, int *, const char *, int); - void (*xFree)(void *pCtx); -}; - -struct lsm_compress_factory { - void *pCtx; - int (*xFactory)(void *, lsm_db *, unsigned int); - void (*xFree)(void *pCtx); -}; - -#define LSM_COMPRESSION_EMPTY 0 -#define LSM_COMPRESSION_NONE 1 - -/* -** CAPI: Allocating and Freeing Memory -** -** Invoke the memory allocation functions that belong to environment -** pEnv. Or the system defaults if no memory allocation functions have -** been registered. -*/ -void *lsm_malloc(lsm_env*, size_t); -void *lsm_realloc(lsm_env*, void *, size_t); -void lsm_free(lsm_env*, void *); - -/* -** CAPI: Querying a Connection For Operational Data -** -** Query a database connection for operational statistics or data. -*/ -int lsm_info(lsm_db *, int, ...); - -int lsm_get_user_version(lsm_db *, unsigned int *); -int lsm_set_user_version(lsm_db *, unsigned int); - -/* -** The following values may be passed as the second argument to lsm_info(). -** -** LSM_INFO_NWRITE: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages written to -** the database file during the lifetime of this connection. -** -** LSM_INFO_NREAD: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages read from -** the database file during the lifetime of this connection. -** -** LSM_INFO_DB_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure reflecting the -** current structure of the database file. Specifically, the current state -** of the worker snapshot. The returned string should be eventually freed -** by the caller using lsm_free(). -** -** The returned list contains one element for each level in the database, -** in order from most to least recent. Each element contains a -** single element for each segment comprising the corresponding level, -** starting with the lhs segment, then each of the rhs segments (if any) -** in order from most to least recent. -** -** Each segment element is itself a list of 4 integer values, as follows: -** -**

    1. First page of segment -**
    2. Last page of segment -**
    3. Root page of segment (if applicable) -**
    4. Total number of pages in segment -**
    -** -** LSM_INFO_ARRAY_STRUCTURE: -** There should be two arguments passed following this option (i.e. a -** total of four arguments passed to lsm_info()). The first argument -** should be the page number of the first page in a database array -** (perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second -** trailing argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string that must -** be eventually freed using lsm_free() by the caller. -** -** The output string contains the text representation of a Tcl list of -** integers. Each pair of integers represent a range of pages used by -** the identified array. For example, if the array occupies database -** pages 993 to 1024, then pages 2048 to 2777, then the returned string -** will be "993 1024 2048 2777". -** -** If the specified integer argument does not correspond to the first -** page of any database array, LSM_ERROR is returned and the output -** pointer is set to a NULL value. -** -** LSM_INFO_LOG_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list of six integers that describe -** the current structure of the log file. -** -** LSM_INFO_ARRAY_PAGES: -** -** LSM_INFO_PAGE_ASCII_DUMP: -** As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed -** with calls that specify this option - an integer page number and a -** (char **) used to return a nul-terminated string that must be later -** freed using lsm_free(). In this case the output string is populated -** with a human-readable description of the page content. -** -** If the page cannot be decoded, it is not an error. In this case the -** human-readable output message will report the systems failure to -** interpret the page data. -** -** LSM_INFO_PAGE_HEX_DUMP: -** This argument is similar to PAGE_ASCII_DUMP, except that keys and -** values are represented using hexadecimal notation instead of ascii. -** -** LSM_INFO_FREELIST: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list containing one element for each -** free block in the database. The element itself consists of two -** integers - the block number and the id of the snapshot that freed it. -** -** LSM_INFO_CHECKPOINT_SIZE: -** The third argument should be of type (int *). The location pointed to -** by this argument is populated with the number of KB written to the -** database file since the most recent checkpoint. -** -** LSM_INFO_TREE_SIZE: -** If this value is passed as the second argument to an lsm_info() call, it -** should be followed by two arguments of type (int *) (for a total of four -** arguments). -** -** At any time, there are either one or two tree structures held in shared -** memory that new database clients will access (there may also be additional -** tree structures being used by older clients - this API does not provide -** information on them). One tree structure - the current tree - is used to -** accumulate new data written to the database. The other tree structure - -** the old tree - is a read-only tree holding older data and may be flushed -** to disk at any time. -** -** Assuming no error occurs, the location pointed to by the first of the two -** (int *) arguments is set to the size of the old in-memory tree in KB. -** The second is set to the size of the current, or live in-memory tree. -** -** LSM_INFO_COMPRESSION_ID: -** This value should be followed by a single argument of type -** (unsigned int *). If successful, the location pointed to is populated -** with the database compression id before returning. -*/ -#define LSM_INFO_NWRITE 1 -#define LSM_INFO_NREAD 2 -#define LSM_INFO_DB_STRUCTURE 3 -#define LSM_INFO_LOG_STRUCTURE 4 -#define LSM_INFO_ARRAY_STRUCTURE 5 -#define LSM_INFO_PAGE_ASCII_DUMP 6 -#define LSM_INFO_PAGE_HEX_DUMP 7 -#define LSM_INFO_FREELIST 8 -#define LSM_INFO_ARRAY_PAGES 9 -#define LSM_INFO_CHECKPOINT_SIZE 10 -#define LSM_INFO_TREE_SIZE 11 -#define LSM_INFO_FREELIST_SIZE 12 -#define LSM_INFO_COMPRESSION_ID 13 - - -/* -** CAPI: Opening and Closing Write Transactions -** -** These functions are used to open and close transactions and nested -** sub-transactions. -** -** The lsm_begin() function is used to open transactions and sub-transactions. -** A successful call to lsm_begin() ensures that there are at least iLevel -** nested transactions open. To open a top-level transaction, pass iLevel=1. -** To open a sub-transaction within the top-level transaction, iLevel=2. -** Passing iLevel=0 is a no-op. -** -** lsm_commit() is used to commit transactions and sub-transactions. A -** successful call to lsm_commit() ensures that there are at most iLevel -** nested transactions open. To commit a top-level transaction, pass iLevel=0. -** To commit all sub-transactions inside the main transaction, pass iLevel=1. -** -** Function lsm_rollback() is used to roll back transactions and -** sub-transactions. A successful call to lsm_rollback() restores the database -** to the state it was in when the iLevel'th nested sub-transaction (if any) -** was first opened. And then closes transactions to ensure that there are -** at most iLevel nested transactions open. Passing iLevel=0 rolls back and -** closes the top-level transaction. iLevel=1 also rolls back the top-level -** transaction, but leaves it open. iLevel=2 rolls back the sub-transaction -** nested directly inside the top-level transaction (and leaves it open). -*/ -int lsm_begin(lsm_db *pDb, int iLevel); -int lsm_commit(lsm_db *pDb, int iLevel); -int lsm_rollback(lsm_db *pDb, int iLevel); - -/* -** CAPI: Writing to a Database -** -** Write a new value into the database. If a value with a duplicate key -** already exists it is replaced. -*/ -int lsm_insert(lsm_db*, const void *pKey, int nKey, const void *pVal, int nVal); - -/* -** Delete a value from the database. No error is returned if the specified -** key value does not exist in the database. -*/ -int lsm_delete(lsm_db *, const void *pKey, int nKey); - -/* -** Delete all database entries with keys that are greater than (pKey1/nKey1) -** and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and -** (pKey2/nKey2) themselves, if they exist in the database, are not deleted. -** -** Return LSM_OK if successful, or an LSM error code otherwise. -*/ -int lsm_delete_range(lsm_db *, - const void *pKey1, int nKey1, const void *pKey2, int nKey2 -); - -/* -** CAPI: Explicit Database Work and Checkpointing -** -** This function is called by a thread to work on the database structure. -*/ -int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); - -int lsm_flush(lsm_db *pDb); - -/* -** Attempt to checkpoint the current database snapshot. Return an LSM -** error code if an error occurs or LSM_OK otherwise. -** -** If the current snapshot has already been checkpointed, calling this -** function is a no-op. In this case if pnKB is not NULL, *pnKB is -** set to 0. Or, if the current snapshot is successfully checkpointed -** by this function and pbKB is not NULL, *pnKB is set to the number -** of bytes written to the database file since the previous checkpoint -** (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). -*/ -int lsm_checkpoint(lsm_db *pDb, int *pnKB); - -/* -** CAPI: Opening and Closing Database Cursors -** -** Open and close a database cursor. -*/ -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); -int lsm_csr_close(lsm_cursor *pCsr); - -/* -** CAPI: Positioning Database Cursors -** -** If the fourth parameter is LSM_SEEK_EQ, LSM_SEEK_GE or LSM_SEEK_LE, -** this function searches the database for an entry with key (pKey/nKey). -** If an error occurs, an LSM error code is returned. Otherwise, LSM_OK. -** -** If no error occurs and the requested key is present in the database, the -** cursor is left pointing to the entry with the specified key. Or, if the -** specified key is not present in the database the state of the cursor -** depends on the value passed as the final parameter, as follows: -** -** LSM_SEEK_EQ: -** The cursor is left at EOF (invalidated). A call to lsm_csr_valid() -** returns non-zero. -** -** LSM_SEEK_LE: -** The cursor is left pointing to the largest key in the database that -** is smaller than (pKey/nKey). If the database contains no keys smaller -** than (pKey/nKey), the cursor is left at EOF. -** -** LSM_SEEK_GE: -** The cursor is left pointing to the smallest key in the database that -** is larger than (pKey/nKey). If the database contains no keys larger -** than (pKey/nKey), the cursor is left at EOF. -** -** If the fourth parameter is LSM_SEEK_LEFAST, this function searches the -** database in a similar manner to LSM_SEEK_LE, with two differences: -** -**
    1. Even if a key can be found (the cursor is not left at EOF), the -** lsm_csr_value() function may not be used (attempts to do so return -** LSM_MISUSE). -** -**
    2. The key that the cursor is left pointing to may be one that has -** been recently deleted from the database. In this case it is -** guaranteed that the returned key is larger than any key currently -** in the database that is less than or equal to (pKey/nKey). -**
    -** -** LSM_SEEK_LEFAST requests are intended to be used to allocate database -** keys. -*/ -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek); - -int lsm_csr_first(lsm_cursor *pCsr); -int lsm_csr_last(lsm_cursor *pCsr); - -/* -** Advance the specified cursor to the next or previous key in the database. -** Return LSM_OK if successful, or an LSM error code otherwise. -** -** Functions lsm_csr_seek(), lsm_csr_first() and lsm_csr_last() are "seek" -** functions. Whether or not lsm_csr_next and lsm_csr_prev may be called -** successfully also depends on the most recent seek function called on -** the cursor. Specifically: -** -**
      -**
    • At least one seek function must have been called on the cursor. -**
    • To call lsm_csr_next(), the most recent call to a seek function must -** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -** LSM_SEEK_GE. -**
    • To call lsm_csr_prev(), the most recent call to a seek function must -** have been either lsm_csr_last() or a call to lsm_csr_seek() specifying -** LSM_SEEK_LE. -**
    -** -** Otherwise, if the above conditions are not met when lsm_csr_next or -** lsm_csr_prev is called, LSM_MISUSE is returned and the cursor position -** remains unchanged. -*/ -int lsm_csr_next(lsm_cursor *pCsr); -int lsm_csr_prev(lsm_cursor *pCsr); - -/* -** Values that may be passed as the fourth argument to lsm_csr_seek(). -*/ -#define LSM_SEEK_LEFAST -2 -#define LSM_SEEK_LE -1 -#define LSM_SEEK_EQ 0 -#define LSM_SEEK_GE 1 - -/* -** CAPI: Extracting Data From Database Cursors -** -** Retrieve data from a database cursor. -*/ -int lsm_csr_valid(lsm_cursor *pCsr); -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey); -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal); - -/* -** If no error occurs, this function compares the database key passed via -** the pKey/nKey arguments with the key that the cursor passed as the first -** argument currently points to. If the cursors key is less than, equal to -** or greater than pKey/nKey, *piRes is set to less than, equal to or greater -** than zero before returning. LSM_OK is returned in this case. -** -** Or, if an error occurs, an LSM error code is returned and the final -** value of *piRes is undefined. If the cursor does not point to a valid -** key when this function is called, LSM_MISUSE is returned. -*/ -int lsm_csr_cmp(lsm_cursor *pCsr, const void *pKey, int nKey, int *piRes); - -/* -** CAPI: Change these!! -** -** Configure a callback to which debugging and other messages should -** be directed. Only useful for debugging lsm. -*/ -void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); - -/* -** Configure a callback that is invoked if the database connection ever -** writes to the database file. -*/ -void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); - -/* ENDOFAPI */ -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif -#endif /* ifndef _LSM_H */ diff --git a/ext/lsm1/lsmInt.h b/ext/lsm1/lsmInt.h deleted file mode 100644 index 5060366d0d..0000000000 --- a/ext/lsm1/lsmInt.h +++ /dev/null @@ -1,997 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Internal structure definitions for the LSM module. -*/ -#ifndef _LSM_INT_H -#define _LSM_INT_H - -#include "lsm.h" -#include -#include - -#include -#include -#include -#include - -#ifdef _WIN32 -# ifdef _MSC_VER -# define snprintf _snprintf -# endif -#else -# include -#endif - -#ifdef NDEBUG -# ifdef LSM_DEBUG_EXPENSIVE -# undef LSM_DEBUG_EXPENSIVE -# endif -# ifdef LSM_DEBUG -# undef LSM_DEBUG -# endif -#else -# ifndef LSM_DEBUG -# define LSM_DEBUG -# endif -#endif - -/* #define LSM_DEBUG_EXPENSIVE 1 */ - -/* -** Default values for various data structure parameters. These may be -** overridden by calls to lsm_config(). -*/ -#define LSM_DFLT_PAGE_SIZE (4 * 1024) -#define LSM_DFLT_BLOCK_SIZE (1 * 1024 * 1024) -#define LSM_DFLT_AUTOFLUSH (1 * 1024 * 1024) -#define LSM_DFLT_AUTOCHECKPOINT (i64)(2 * 1024 * 1024) -#define LSM_DFLT_AUTOWORK 1 -#define LSM_DFLT_LOG_SIZE (128*1024) -#define LSM_DFLT_AUTOMERGE 4 -#define LSM_DFLT_SAFETY LSM_SAFETY_NORMAL -#define LSM_DFLT_MMAP (LSM_IS_64_BIT ? 1 : 32768) -#define LSM_DFLT_MULTIPLE_PROCESSES 1 -#define LSM_DFLT_USE_LOG 1 - -/* Initial values for log file checksums. These are only used if the -** database file does not contain a valid checkpoint. */ -#define LSM_CKSUM0_INIT 42 -#define LSM_CKSUM1_INIT 42 - -/* "mmap" mode is currently only used in environments with 64-bit address -** spaces. The following macro is used to test for this. */ -#define LSM_IS_64_BIT (sizeof(void*)==8) - -#define LSM_AUTOWORK_QUANT 32 - -typedef struct Database Database; -typedef struct DbLog DbLog; -typedef struct FileSystem FileSystem; -typedef struct Freelist Freelist; -typedef struct FreelistEntry FreelistEntry; -typedef struct Level Level; -typedef struct LogMark LogMark; -typedef struct LogRegion LogRegion; -typedef struct LogWriter LogWriter; -typedef struct LsmString LsmString; -typedef struct Mempool Mempool; -typedef struct Merge Merge; -typedef struct MergeInput MergeInput; -typedef struct MetaPage MetaPage; -typedef struct MultiCursor MultiCursor; -typedef struct Page Page; -typedef struct Redirect Redirect; -typedef struct Segment Segment; -typedef struct SegmentMerger SegmentMerger; -typedef struct ShmChunk ShmChunk; -typedef struct ShmHeader ShmHeader; -typedef struct ShmReader ShmReader; -typedef struct Snapshot Snapshot; -typedef struct TransMark TransMark; -typedef struct Tree Tree; -typedef struct TreeCursor TreeCursor; -typedef struct TreeHeader TreeHeader; -typedef struct TreeMark TreeMark; -typedef struct TreeRoot TreeRoot; - -#ifndef _SQLITEINT_H_ -typedef unsigned char u8; -typedef unsigned short int u16; -typedef unsigned int u32; -typedef lsm_i64 i64; -typedef unsigned long long int u64; -#endif - -/* A page number is a 64-bit integer. */ -typedef i64 LsmPgno; - -#ifdef LSM_DEBUG -int lsmErrorBkpt(int); -#else -# define lsmErrorBkpt(x) (x) -#endif - -#define LSM_PROTOCOL_BKPT lsmErrorBkpt(LSM_PROTOCOL) -#define LSM_IOERR_BKPT lsmErrorBkpt(LSM_IOERR) -#define LSM_NOMEM_BKPT lsmErrorBkpt(LSM_NOMEM) -#define LSM_CORRUPT_BKPT lsmErrorBkpt(LSM_CORRUPT) -#define LSM_MISUSE_BKPT lsmErrorBkpt(LSM_MISUSE) - -#define unused_parameter(x) (void)(x) -#define array_size(x) (sizeof(x)/sizeof(x[0])) - - -/* The size of each shared-memory chunk */ -#define LSM_SHM_CHUNK_SIZE (32*1024) - -/* The number of bytes reserved at the start of each shm chunk for MM. */ -#define LSM_SHM_CHUNK_HDR (sizeof(ShmChunk)) - -/* The number of available read locks. */ -#define LSM_LOCK_NREADER 6 - -/* The number of available read-write client locks. */ -#define LSM_LOCK_NRWCLIENT 16 - -/* Lock definitions. -*/ -#define LSM_LOCK_DMS1 1 /* Serialize connect/disconnect ops */ -#define LSM_LOCK_DMS2 2 /* Read-write connections */ -#define LSM_LOCK_DMS3 3 /* Read-only connections */ -#define LSM_LOCK_WRITER 4 -#define LSM_LOCK_WORKER 5 -#define LSM_LOCK_CHECKPOINTER 6 -#define LSM_LOCK_ROTRANS 7 -#define LSM_LOCK_READER(i) ((i) + LSM_LOCK_ROTRANS + 1) -#define LSM_LOCK_RWCLIENT(i) ((i) + LSM_LOCK_READER(LSM_LOCK_NREADER)) - -#define LSM_N_LOCK LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT) - -/* -** Meta-page size and usable size. -*/ -#define LSM_META_PAGE_SIZE 4096 - -#define LSM_META_RW_PAGE_SIZE (LSM_META_PAGE_SIZE - LSM_N_LOCK) - -/* -** Hard limit on the number of free-list entries that may be stored in -** a checkpoint (the remainder are stored as a system record in the LSM). -** See also LSM_CONFIG_MAX_FREELIST. -*/ -#define LSM_MAX_FREELIST_ENTRIES 24 - -#define LSM_MAX_BLOCK_REDIRECTS 16 - -#define LSM_ATTEMPTS_BEFORE_PROTOCOL 10000 - - -/* -** Each entry stored in the LSM (or in-memory tree structure) has an -** associated mask of the following flags. -*/ -#define LSM_START_DELETE 0x01 /* Start of open-ended delete range */ -#define LSM_END_DELETE 0x02 /* End of open-ended delete range */ -#define LSM_POINT_DELETE 0x04 /* Delete this key */ -#define LSM_INSERT 0x08 /* Insert this key and value */ -#define LSM_SEPARATOR 0x10 /* True if entry is separator key only */ -#define LSM_SYSTEMKEY 0x20 /* True if entry is a system key (FREELIST) */ - -#define LSM_CONTIGUOUS 0x40 /* Used in lsm_tree.c */ - -/* -** A string that can grow by appending. -*/ -struct LsmString { - lsm_env *pEnv; /* Run-time environment */ - int n; /* Size of string. -1 indicates error */ - int nAlloc; /* Space allocated for z[] */ - char *z; /* The string content */ -}; - -typedef struct LsmFile LsmFile; -struct LsmFile { - lsm_file *pFile; - LsmFile *pNext; -}; - -/* -** An instance of the following type is used to store an ordered list of -** u32 values. -** -** Note: This is a place-holder implementation. It should be replaced by -** a version that avoids making a single large allocation when the array -** contains a large number of values. For this reason, the internals of -** this object should only manipulated by the intArrayXXX() functions in -** lsm_tree.c. -*/ -typedef struct IntArray IntArray; -struct IntArray { - int nAlloc; - int nArray; - u32 *aArray; -}; - -struct Redirect { - int n; /* Number of redirects */ - struct RedirectEntry { - int iFrom; - int iTo; - } *a; -}; - -/* -** An instance of this structure represents a point in the history of the -** tree structure to roll back to. Refer to comments in lsm_tree.c for -** details. -*/ -struct TreeMark { - u32 iRoot; /* Offset of root node in shm file */ - u32 nHeight; /* Current height of tree structure */ - u32 iWrite; /* Write offset in shm file */ - u32 nChunk; /* Number of chunks in shared-memory file */ - u32 iFirst; /* First chunk in linked list */ - u32 iNextShmid; /* Next id to allocate */ - int iRollback; /* Index in lsm->rollback to revert to */ -}; - -/* -** An instance of this structure represents a point in the database log. -*/ -struct LogMark { - i64 iOff; /* Offset into log (see lsm_log.c) */ - int nBuf; /* Size of in-memory buffer here */ - u8 aBuf[8]; /* Bytes of content in aBuf[] */ - u32 cksum0; /* Checksum 0 at offset (iOff-nBuf) */ - u32 cksum1; /* Checksum 1 at offset (iOff-nBuf) */ -}; - -struct TransMark { - TreeMark tree; - LogMark log; -}; - -/* -** A structure that defines the start and end offsets of a region in the -** log file. The size of the region in bytes is (iEnd - iStart), so if -** iEnd==iStart the region is zero bytes in size. -*/ -struct LogRegion { - i64 iStart; /* Start of region in log file */ - i64 iEnd; /* End of region in log file */ -}; - -struct DbLog { - u32 cksum0; /* Checksum 0 at offset iOff */ - u32 cksum1; /* Checksum 1 at offset iOff */ - i64 iSnapshotId; /* Log space has been reclaimed to this ss */ - LogRegion aRegion[3]; /* Log file regions (see docs in lsm_log.c) */ -}; - -struct TreeRoot { - u32 iRoot; - u32 nHeight; - u32 nByte; /* Total size of this tree in bytes */ - u32 iTransId; -}; - -/* -** Tree header structure. -*/ -struct TreeHeader { - u32 iUsedShmid; /* Id of first shm chunk used by this tree */ - u32 iNextShmid; /* Shm-id of next chunk allocated */ - u32 iFirst; /* Chunk number of smallest shm-id */ - u32 nChunk; /* Number of chunks in shared-memory file */ - TreeRoot root; /* Root and height of current tree */ - u32 iWrite; /* Write offset in shm file */ - TreeRoot oldroot; /* Root and height of the previous tree */ - u32 iOldShmid; /* Last shm-id used by previous tree */ - u32 iUsrVersion; /* get/set_user_version() value */ - i64 iOldLog; /* Log offset associated with old tree */ - u32 oldcksum0; - u32 oldcksum1; - DbLog log; /* Current layout of log file */ - u32 aCksum[2]; /* Checksums 1 and 2. */ -}; - -/* -** Database handle structure. -** -** mLock: -** A bitmask representing the locks currently held by the connection. -** An LSM database supports N distinct locks, where N is some number less -** than or equal to 32. Locks are numbered starting from 1 (see the -** definitions for LSM_LOCK_WRITER and co.). -** -** The least significant 32-bits in mLock represent EXCLUSIVE locks. The -** most significant are SHARED locks. So, if a connection holds a SHARED -** lock on lock region iLock, then the following is true: -** -** (mLock & ((iLock+32-1) << 1)) -** -** Or for an EXCLUSIVE lock: -** -** (mLock & ((iLock-1) << 1)) -** -** pCsr: -** Points to the head of a linked list that contains all currently open -** cursors. Once this list becomes empty, the user has no outstanding -** cursors and the database handle can be successfully closed. -** -** pCsrCache: -** This list contains cursor objects that have been closed using -** lsm_csr_close(). Each time a cursor is closed, it is shifted from -** the pCsr list to this list. When a new cursor is opened, this list -** is inspected to see if there exists a cursor object that can be -** reused. This is an optimization only. -*/ -struct lsm_db { - - /* Database handle configuration */ - lsm_env *pEnv; /* runtime environment */ - int (*xCmp)(void *, int, void *, int); /* Compare function */ - - /* Values configured by calls to lsm_config */ - int eSafety; /* LSM_SAFETY_OFF, NORMAL or FULL */ - int bAutowork; /* Configured by LSM_CONFIG_AUTOWORK */ - int nTreeLimit; /* Configured by LSM_CONFIG_AUTOFLUSH */ - int nMerge; /* Configured by LSM_CONFIG_AUTOMERGE */ - int bUseLog; /* Configured by LSM_CONFIG_USE_LOG */ - int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ - int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ - int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ - int iMmap; /* Configured by LSM_CONFIG_MMAP */ - i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ - int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ - int bReadonly; /* Configured by LSM_CONFIG_READONLY */ - lsm_compress compress; /* Compression callbacks */ - lsm_compress_factory factory; /* Compression callback factory */ - - /* Sub-system handles */ - FileSystem *pFS; /* On-disk portion of database */ - Database *pDatabase; /* Database shared data */ - - int iRwclient; /* Read-write client lock held (-1 == none) */ - - /* Client transaction context */ - Snapshot *pClient; /* Client snapshot */ - int iReader; /* Read lock held (-1 == unlocked) */ - int bRoTrans; /* True if a read-only db trans is open */ - MultiCursor *pCsr; /* List of all open cursors */ - LogWriter *pLogWriter; /* Context for writing to the log file */ - int nTransOpen; /* Number of opened write transactions */ - int nTransAlloc; /* Allocated size of aTrans[] array */ - TransMark *aTrans; /* Array of marks for transaction rollback */ - IntArray rollback; /* List of tree-nodes to roll back */ - int bDiscardOld; /* True if lsmTreeDiscardOld() was called */ - - MultiCursor *pCsrCache; /* List of all closed cursors */ - - /* Worker context */ - Snapshot *pWorker; /* Worker snapshot (or NULL) */ - Freelist *pFreelist; /* See sortedNewToplevel() */ - int bUseFreelist; /* True to use pFreelist */ - int bIncrMerge; /* True if currently doing a merge */ - - int bInFactory; /* True if within factory.xFactory() */ - - /* Debugging message callback */ - void (*xLog)(void *, int, const char *); - void *pLogCtx; - - /* Work done notification callback */ - void (*xWork)(lsm_db *, void *); - void *pWorkCtx; - - u64 mLock; /* Mask of current locks. See lsmShmLock(). */ - lsm_db *pNext; /* Next connection to same database */ - - int nShm; /* Size of apShm[] array */ - void **apShm; /* Shared memory chunks */ - ShmHeader *pShmhdr; /* Live shared-memory header */ - TreeHeader treehdr; /* Local copy of tree-header */ - u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)]; -}; - -struct Segment { - LsmPgno iFirst; /* First page of this run */ - LsmPgno iLastPg; /* Last page of this run */ - LsmPgno iRoot; /* Root page number (if any) */ - LsmPgno nSize; /* Size of this run in pages */ - - Redirect *pRedirect; /* Block redirects (or NULL) */ -}; - -/* -** iSplitTopic/pSplitKey/nSplitKey: -** If nRight>0, this buffer contains a copy of the largest key that has -** already been written to the left-hand-side of the level. -*/ -struct Level { - Segment lhs; /* Left-hand (main) segment */ - int nRight; /* Size of apRight[] array */ - Segment *aRhs; /* Old segments being merged into this */ - int iSplitTopic; /* Split key topic (if nRight>0) */ - void *pSplitKey; /* Pointer to split-key (if nRight>0) */ - int nSplitKey; /* Number of bytes in split-key */ - - u16 iAge; /* Number of times data has been written */ - u16 flags; /* Mask of LEVEL_XXX bits */ - Merge *pMerge; /* Merge operation currently underway */ - Level *pNext; /* Next level in tree */ -}; - -/* -** The Level.flags field is set to a combination of the following bits. -** -** LEVEL_FREELIST_ONLY: -** Set if the level consists entirely of free-list entries. -** -** LEVEL_INCOMPLETE: -** This is set while a new toplevel level is being constructed. It is -** never set for any level other than a new toplevel. -*/ -#define LEVEL_FREELIST_ONLY 0x0001 -#define LEVEL_INCOMPLETE 0x0002 - - -/* -** A structure describing an ongoing merge. There is an instance of this -** structure for every Level currently undergoing a merge in the worker -** snapshot. -** -** It is assumed that code that uses an instance of this structure has -** access to the associated Level struct. -** -** iOutputOff: -** The byte offset to write to next within the last page of the -** output segment. -*/ -struct MergeInput { - LsmPgno iPg; /* Page on which next input is stored */ - int iCell; /* Cell containing next input to merge */ -}; -struct Merge { - int nInput; /* Number of input runs being merged */ - MergeInput *aInput; /* Array nInput entries in size */ - MergeInput splitkey; /* Location in file of current splitkey */ - int nSkip; /* Number of separators entries to skip */ - int iOutputOff; /* Write offset on output page */ - LsmPgno iCurrentPtr; /* Current pointer value */ -}; - -/* -** The first argument to this macro is a pointer to a Segment structure. -** Returns true if the structure instance indicates that the separators -** array is valid. -*/ -#define segmentHasSeparators(pSegment) ((pSegment)->sep.iFirst>0) - -/* -** The values that accompany the lock held by a database reader. -*/ -struct ShmReader { - u32 iTreeId; - i64 iLsmId; -}; - -/* -** An instance of this structure is stored in the first shared-memory -** page. The shared-memory header. -** -** bWriter: -** Immediately after opening a write transaction taking the WRITER lock, -** each writer client sets this flag. It is cleared right before the -** WRITER lock is relinquished. If a subsequent writer finds that this -** flag is already set when a write transaction is opened, this indicates -** that a previous writer failed mid-transaction. -** -** iMetaPage: -** If the database file does not contain a valid, synced, checkpoint, this -** value is set to 0. Otherwise, it is set to the meta-page number that -** contains the most recently written checkpoint (either 1 or 2). -** -** hdr1, hdr2: -** The two copies of the in-memory tree header. Two copies are required -** in case a writer fails while updating one of them. -*/ -struct ShmHeader { - u32 aSnap1[LSM_META_PAGE_SIZE / 4]; - u32 aSnap2[LSM_META_PAGE_SIZE / 4]; - u32 bWriter; - u32 iMetaPage; - TreeHeader hdr1; - TreeHeader hdr2; - ShmReader aReader[LSM_LOCK_NREADER]; -}; - -/* -** An instance of this structure is stored at the start of each shared-memory -** chunk except the first (which is the header chunk - see above). -*/ -struct ShmChunk { - u32 iShmid; - u32 iNext; -}; - -/* -** Maximum number of shared-memory chunks allowed in the *-shm file. Since -** each shared-memory chunk is 32KB in size, this is a theoretical limit only. -*/ -#define LSM_MAX_SHMCHUNKS (1<<30) - -/* Return true if shm-sequence "a" is larger than or equal to "b" */ -#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < LSM_MAX_SHMCHUNKS) - -#define LSM_APPLIST_SZ 4 - -/* -** An instance of the following structure stores the in-memory part of -** the current free block list. This structure is to the free block list -** as the in-memory tree is to the users database content. The contents -** of the free block list is found by merging the in-memory components -** with those stored in the LSM, just as the contents of the database is -** found by merging the in-memory tree with the user data entries in the -** LSM. -** -** Each FreelistEntry structure in the array represents either an insert -** or delete operation on the free-list. For deletes, the FreelistEntry.iId -** field is set to -1. For inserts, it is set to zero or greater. -** -** The array of FreelistEntry structures is always sorted in order of -** block number (ascending). -** -** When the in-memory free block list is written into the LSM, each insert -** operation is written separately. The entry key is the bitwise inverse -** of the block number as a 32-bit big-endian integer. This is done so that -** the entries in the LSM are sorted in descending order of block id. -** The associated value is the snapshot id, formated as a varint. -*/ -struct Freelist { - FreelistEntry *aEntry; /* Free list entries */ - int nEntry; /* Number of valid slots in aEntry[] */ - int nAlloc; /* Allocated size of aEntry[] */ -}; -struct FreelistEntry { - u32 iBlk; /* Block number */ - i64 iId; /* Largest snapshot id to use this block */ -}; - -/* -** A snapshot of a database. A snapshot contains all the information required -** to read or write a database file on disk. See the description of struct -** Database below for futher details. -*/ -struct Snapshot { - Database *pDatabase; /* Database this snapshot belongs to */ - u32 iCmpId; /* Id of compression scheme */ - Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ - i64 iId; /* Snapshot id */ - i64 iLogOff; /* Log file offset */ - Redirect redirect; /* Block redirection array */ - - /* Used by worker snapshots only */ - int nBlock; /* Number of blocks in database file */ - LsmPgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ - Freelist freelist; /* Free block list */ - u32 nWrite; /* Total number of pages written to disk */ -}; -#define LSM_INITIAL_SNAPSHOT_ID 11 - -/* -** Functions from file "lsm_ckpt.c". -*/ -int lsmCheckpointWrite(lsm_db *, u32 *); -int lsmCheckpointLevels(lsm_db *, int, void **, int *); -int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal); - -int lsmCheckpointRecover(lsm_db *); -int lsmCheckpointDeserialize(lsm_db *, int, u32 *, Snapshot **); - -int lsmCheckpointLoadWorker(lsm_db *pDb); -int lsmCheckpointStore(lsm_db *pDb, int); - -int lsmCheckpointLoad(lsm_db *pDb, int *); -int lsmCheckpointLoadOk(lsm_db *pDb, int); -int lsmCheckpointClientCacheOk(lsm_db *); - -u32 lsmCheckpointNBlock(u32 *); -i64 lsmCheckpointId(u32 *, int); -u32 lsmCheckpointNWrite(u32 *, int); -i64 lsmCheckpointLogOffset(u32 *); -int lsmCheckpointPgsz(u32 *); -int lsmCheckpointBlksz(u32 *); -void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); -void lsmCheckpointZeroLogoffset(lsm_db *); - -int lsmCheckpointSaveWorker(lsm_db *pDb, int); -int lsmDatabaseFull(lsm_db *pDb); -int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite); - -int lsmCheckpointSize(lsm_db *db, int *pnByte); - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId); - -/* -** Functions from file "lsm_tree.c". -*/ -int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); -void lsmTreeRelease(lsm_env *, Tree *); -int lsmTreeInit(lsm_db *); -int lsmTreeRepair(lsm_db *); - -void lsmTreeMakeOld(lsm_db *pDb); -void lsmTreeDiscardOld(lsm_db *pDb); -int lsmTreeHasOld(lsm_db *pDb); - -int lsmTreeSize(lsm_db *); -int lsmTreeEndTransaction(lsm_db *pDb, int bCommit); -int lsmTreeLoadHeader(lsm_db *pDb, int *); -int lsmTreeLoadHeaderOk(lsm_db *, int); - -int lsmTreeInsert(lsm_db *pDb, void *pKey, int nKey, void *pVal, int nVal); -int lsmTreeDelete(lsm_db *db, void *pKey1, int nKey1, void *pKey2, int nKey2); -void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark); -void lsmTreeMark(lsm_db *pDb, TreeMark *pMark); - -int lsmTreeCursorNew(lsm_db *pDb, int, TreeCursor **); -void lsmTreeCursorDestroy(TreeCursor *); - -int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes); -int lsmTreeCursorNext(TreeCursor *pCsr); -int lsmTreeCursorPrev(TreeCursor *pCsr); -int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast); -void lsmTreeCursorReset(TreeCursor *pCsr); -int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey); -int lsmTreeCursorFlags(TreeCursor *pCsr); -int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal); -int lsmTreeCursorValid(TreeCursor *pCsr); -int lsmTreeCursorSave(TreeCursor *pCsr); - -void lsmFlagsToString(int flags, char *zFlags); - -/* -** Functions from file "mem.c". -*/ -void *lsmMalloc(lsm_env*, size_t); -void lsmFree(lsm_env*, void *); -void *lsmRealloc(lsm_env*, void *, size_t); -void *lsmReallocOrFree(lsm_env*, void *, size_t); -void *lsmReallocOrFreeRc(lsm_env *, void *, size_t, int *); - -void *lsmMallocZeroRc(lsm_env*, size_t, int *); -void *lsmMallocRc(lsm_env*, size_t, int *); - -void *lsmMallocZero(lsm_env *pEnv, size_t); -char *lsmMallocStrdup(lsm_env *pEnv, const char *); - -/* -** Functions from file "lsm_mutex.c". -*/ -int lsmMutexStatic(lsm_env*, int, lsm_mutex **); -int lsmMutexNew(lsm_env*, lsm_mutex **); -void lsmMutexDel(lsm_env*, lsm_mutex *); -void lsmMutexEnter(lsm_env*, lsm_mutex *); -int lsmMutexTry(lsm_env*, lsm_mutex *); -void lsmMutexLeave(lsm_env*, lsm_mutex *); - -#ifndef NDEBUG -int lsmMutexHeld(lsm_env *, lsm_mutex *); -int lsmMutexNotHeld(lsm_env *, lsm_mutex *); -#endif - -/************************************************************************** -** Start of functions from "lsm_file.c". -*/ -int lsmFsOpen(lsm_db *, const char *, int); -int lsmFsOpenLog(lsm_db *, int *); -void lsmFsCloseLog(lsm_db *); -void lsmFsClose(FileSystem *); - -int lsmFsUnmap(FileSystem *); - -int lsmFsConfigure(lsm_db *db); - -int lsmFsBlockSize(FileSystem *); -void lsmFsSetBlockSize(FileSystem *, int); -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom); - -int lsmFsPageSize(FileSystem *); -void lsmFsSetPageSize(FileSystem *, int); - -int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); - -/* Creating, populating, gobbling and deleting sorted runs. */ -void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int); -int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); -int lsmFsSortedFinish(FileSystem *, Segment *); -int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); -int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); - -/* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ -lsm_env *lsmFsEnv(FileSystem *); -lsm_env *lsmPageEnv(Page *); -FileSystem *lsmPageFS(Page *); - -int lsmFsSectorSize(FileSystem *); - -void lsmSortedSplitkey(lsm_db *, Level *, int *); - -/* Reading sorted run content. */ -int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); -int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **); -int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); - -u8 *lsmFsPageData(Page *, int *); -int lsmFsPageRelease(Page *); -int lsmFsPagePersist(Page *); -void lsmFsPageRef(Page *); -LsmPgno lsmFsPageNumber(Page *); - -int lsmFsNRead(FileSystem *); -int lsmFsNWrite(FileSystem *); - -int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **); -int lsmFsMetaPageRelease(MetaPage *); -u8 *lsmFsMetaPageData(MetaPage *, int *); - -#ifdef LSM_DEBUG -int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg); -int lsmFsIntegrityCheck(lsm_db *); -#endif - -LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno); - -int lsmFsPageWritable(Page *); - -/* Functions to read, write and sync the log file. */ -int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); -int lsmFsSyncLog(FileSystem *pFS); -int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); -int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte); -int lsmFsCloseAndDeleteLog(FileSystem *pFS); - -LsmFile *lsmFsDeferClose(FileSystem *pFS); - -/* And to sync the db file */ -int lsmFsSyncDb(FileSystem *, int); - -void lsmFsFlushWaiting(FileSystem *, int *); - -/* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz); -int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut); -int lsmConfigMmap(lsm_db *pDb, int *piParam); - -int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **); -int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); -int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); -int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int); - -int lsmEnvShmMap(lsm_env *, lsm_file *, int, int, void **); -void lsmEnvShmBarrier(lsm_env *); -void lsmEnvShmUnmap(lsm_env *, lsm_file *, int); - -void lsmEnvSleep(lsm_env *, int); - -int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); - -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *); - -void lsmFsPurgeCache(FileSystem *); - -/* -** End of functions from "lsm_file.c". -**************************************************************************/ - -/* -** Functions from file "lsm_sorted.c". -*/ -int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **); -void lsmSortedCleanup(lsm_db *); -int lsmSortedAutoWork(lsm_db *, int nUnit); - -int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmSaveWorker(lsm_db *, int); - -int lsmFlushTreeToDisk(lsm_db *pDb); - -void lsmSortedRemap(lsm_db *pDb); - -void lsmSortedFreeLevel(lsm_env *pEnv, Level *); - -int lsmSortedAdvanceAll(lsm_db *pDb); - -int lsmSortedLoadMerge(lsm_db *, Level *, u32 *, int *); -int lsmSortedLoadFreelist(lsm_db *pDb, void **, int *); - -void *lsmSortedSplitKey(Level *pLevel, int *pnByte); - -void lsmSortedSaveTreeCursors(lsm_db *); - -int lsmMCursorNew(lsm_db *, MultiCursor **); -void lsmMCursorClose(MultiCursor *, int); -int lsmMCursorSeek(MultiCursor *, int, void *, int , int); -int lsmMCursorFirst(MultiCursor *); -int lsmMCursorPrev(MultiCursor *); -int lsmMCursorLast(MultiCursor *); -int lsmMCursorValid(MultiCursor *); -int lsmMCursorNext(MultiCursor *); -int lsmMCursorKey(MultiCursor *, void **, int *); -int lsmMCursorValue(MultiCursor *, void **, int *); -int lsmMCursorType(MultiCursor *, int *); -lsm_db *lsmMCursorDb(MultiCursor *); -void lsmMCursorFreeCache(lsm_db *); - -int lsmSaveCursors(lsm_db *pDb); -int lsmRestoreCursors(lsm_db *pDb); - -void lsmSortedDumpStructure(lsm_db *pDb, Snapshot *, int, int, const char *); -void lsmFsDumpBlocklists(lsm_db *); - -void lsmSortedExpandBtreePage(Page *pPg, int nOrig); - -void lsmPutU32(u8 *, u32); -u32 lsmGetU32(u8 *); -u64 lsmGetU64(u8 *); - -/* -** Functions from "lsm_varint.c". -*/ -int lsmVarintPut32(u8 *, int); -int lsmVarintGet32(u8 *, int *); -int lsmVarintPut64(u8 *aData, i64 iVal); -int lsmVarintGet64(const u8 *aData, i64 *piVal); - -int lsmVarintLen64(i64); - -int lsmVarintLen32(int); -int lsmVarintSize(u8 c); - -/* -** Functions from file "main.c". -*/ -void lsmLogMessage(lsm_db *, int, const char *, ...); -int lsmInfoFreelist(lsm_db *pDb, char **pzOut); - -/* -** Functions from file "lsm_log.c". -*/ -int lsmLogBegin(lsm_db *pDb); -int lsmLogWrite(lsm_db *, int, void *, int, void *, int); -int lsmLogCommit(lsm_db *); -void lsmLogEnd(lsm_db *pDb, int bCommit); -void lsmLogTell(lsm_db *, LogMark *); -void lsmLogSeek(lsm_db *, LogMark *); -void lsmLogClose(lsm_db *); - -int lsmLogRecover(lsm_db *); -int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); - -/* Valid values for the second argument to lsmLogWrite(). */ -#define LSM_WRITE 0x06 -#define LSM_DELETE 0x08 -#define LSM_DRANGE 0x0A - -/************************************************************************** -** Functions from file "lsm_shared.c". -*/ - -int lsmDbDatabaseConnect(lsm_db*, const char *); -void lsmDbDatabaseRelease(lsm_db *); - -int lsmBeginReadTrans(lsm_db *); -int lsmBeginWriteTrans(lsm_db *); -int lsmBeginFlush(lsm_db *); - -int lsmDetectRoTrans(lsm_db *db, int *); -int lsmBeginRoTrans(lsm_db *db); - -int lsmBeginWork(lsm_db *); -void lsmFinishWork(lsm_db *, int, int *); - -int lsmFinishRecovery(lsm_db *); -void lsmFinishReadTrans(lsm_db *); -int lsmFinishWriteTrans(lsm_db *, int); -int lsmFinishFlush(lsm_db *, int); - -int lsmSnapshotSetFreelist(lsm_db *, int *, int); - -Snapshot *lsmDbSnapshotClient(lsm_db *); -Snapshot *lsmDbSnapshotWorker(lsm_db *); - -void lsmSnapshotSetCkptid(Snapshot *, i64); - -Level *lsmDbSnapshotLevel(Snapshot *); -void lsmDbSnapshotSetLevel(Snapshot *, Level *); - -void lsmDbRecoveryComplete(lsm_db *, int); - -int lsmBlockAllocate(lsm_db *, int, int *); -int lsmBlockFree(lsm_db *, int); -int lsmBlockRefree(lsm_db *, int); - -void lsmFreelistDeltaBegin(lsm_db *); -void lsmFreelistDeltaEnd(lsm_db *); -int lsmFreelistDelta(lsm_db *pDb); - -DbLog *lsmDatabaseLog(lsm_db *pDb); - -#ifdef LSM_DEBUG - int lsmHoldingClientMutex(lsm_db *pDb); - int lsmShmAssertLock(lsm_db *db, int iLock, int eOp); - int lsmShmAssertWorker(lsm_db *db); -#endif - -void lsmFreeSnapshot(lsm_env *, Snapshot *); - - -/* Candidate values for the 3rd argument to lsmShmLock() */ -#define LSM_LOCK_UNLOCK 0 -#define LSM_LOCK_SHARED 1 -#define LSM_LOCK_EXCL 2 - -int lsmShmCacheChunks(lsm_db *db, int nChunk); -int lsmShmLock(lsm_db *db, int iLock, int eOp, int bBlock); -int lsmShmTestLock(lsm_db *db, int iLock, int nLock, int eOp); -void lsmShmBarrier(lsm_db *db); - -#ifdef LSM_DEBUG -void lsmShmHasLock(lsm_db *db, int iLock, int eOp); -#else -# define lsmShmHasLock(x,y,z) -#endif - -int lsmReadlock(lsm_db *, i64 iLsm, u32 iShmMin, u32 iShmMax); - -int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse); -int lsmTreeInUse(lsm_db *db, u32 iLsmId, int *pbInUse); -int lsmFreelistAppend(lsm_env *pEnv, Freelist *p, int iBlk, i64 iId); - -int lsmDbMultiProc(lsm_db *); -void lsmDbDeferredClose(lsm_db *, lsm_file *, LsmFile *); -LsmFile *lsmDbRecycleFd(lsm_db *); - -int lsmWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmCheckCompressionId(lsm_db *, u32); - - -/************************************************************************** -** functions in lsm_str.c -*/ -void lsmStringInit(LsmString*, lsm_env *pEnv); -int lsmStringExtend(LsmString*, int); -int lsmStringAppend(LsmString*, const char *, int); -void lsmStringVAppendf(LsmString*, const char *zFormat, va_list, va_list); -void lsmStringAppendf(LsmString*, const char *zFormat, ...); -void lsmStringClear(LsmString*); -char *lsmMallocPrintf(lsm_env*, const char*, ...); -int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n); - -int lsmStrlen(const char *zName); - - - -/* -** Round up a number to the next larger multiple of 8. This is used -** to force 8-byte alignment on 64-bit architectures. -*/ -#define ROUND8(x) (((x)+7)&~7) - -#define LSM_MIN(x,y) ((x)>(y) ? (y) : (x)) -#define LSM_MAX(x,y) ((x)>(y) ? (x) : (y)) - -#endif diff --git a/ext/lsm1/lsm_ckpt.c b/ext/lsm1/lsm_ckpt.c deleted file mode 100644 index 1c4f788ad6..0000000000 --- a/ext/lsm1/lsm_ckpt.c +++ /dev/null @@ -1,1239 +0,0 @@ -/* -** 2011-09-11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains code to read and write checkpoints. -** -** A checkpoint represents the database layout at a single point in time. -** It includes a log offset. When an existing database is opened, the -** current state is determined by reading the newest checkpoint and updating -** it with all committed transactions from the log that follow the specified -** offset. -*/ -#include "lsmInt.h" - -/* -** CHECKPOINT BLOB FORMAT: -** -** A checkpoint blob is a series of unsigned 32-bit integers stored in -** big-endian byte order. As follows: -** -** Checkpoint header (see the CKPT_HDR_XXX #defines): -** -** 1. The checkpoint id MSW. -** 2. The checkpoint id LSW. -** 3. The number of integer values in the entire checkpoint, including -** the two checksum values. -** 4. The compression scheme id. -** 5. The total number of blocks in the database. -** 6. The block size. -** 7. The number of levels. -** 8. The nominal database page size. -** 9. The number of pages (in total) written to the database file. -** -** Log pointer: -** -** 1. The log offset MSW. -** 2. The log offset LSW. -** 3. Log checksum 0. -** 4. Log checksum 1. -** -** Note that the "log offset" is not the literal byte offset. Instead, -** it is the byte offset multiplied by 2, with least significant bit -** toggled each time the log pointer value is changed. This is to make -** sure that this field changes each time the log pointer is updated, -** even if the log file itself is disabled. See lsmTreeMakeOld(). -** -** See ckptExportLog() and ckptImportLog(). -** -** Append points: -** -** 8 integers (4 * 64-bit page numbers). See ckptExportAppendlist(). -** -** For each level in the database, a level record. Formatted as follows: -** -** 0. Age of the level (least significant 16-bits). And flags mask (most -** significant 16-bits). -** 1. The number of right-hand segments (nRight, possibly 0), -** 2. Segment record for left-hand segment (8 integers defined below), -** 3. Segment record for each right-hand segment (8 integers defined below), -** 4. If nRight>0, The number of segments involved in the merge -** 5. if nRight>0, Current nSkip value (see Merge structure defn.), -** 6. For each segment in the merge: -** 5a. Page number of next cell to read during merge (this field -** is 64-bits - 2 integers) -** 5b. Cell number of next cell to read during merge -** 7. Page containing current split-key (64-bits - 2 integers). -** 8. Cell within page containing current split-key. -** 9. Current pointer value (64-bits - 2 integers). -** -** The block redirect array: -** -** 1. Number of redirections (maximum LSM_MAX_BLOCK_REDIRECTS). -** 2. For each redirection: -** a. "from" block number -** b. "to" block number -** -** The in-memory freelist entries. Each entry is either an insert or a -** delete. The in-memory freelist is to the free-block-list as the -** in-memory tree is to the users database content. -** -** 1. Number of free-list entries stored in checkpoint header. -** 2. Number of free blocks (in total). -** 3. Total number of blocks freed during database lifetime. -** 4. For each entry: -** 2a. Block number of free block. -** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, -** or the associated checkpoint id for an insert. -** -** The checksum: -** -** 1. Checksum value 1. -** 2. Checksum value 2. -** -** In the above, a segment record consists of the following four 64-bit -** fields (converted to 2 * u32 by storing the MSW followed by LSW): -** -** 1. First page of array, -** 2. Last page of array, -** 3. Root page of array (or 0), -** 4. Size of array in pages. -*/ - -/* -** LARGE NUMBERS OF LEVEL RECORDS: -** -** A limit on the number of rhs segments that may be present in the database -** file. Defining this limit ensures that all level records fit within -** the 4096 byte limit for checkpoint blobs. -** -** The number of right-hand-side segments in a database is counted as -** follows: -** -** * For each level in the database not undergoing a merge, add 1. -** -** * For each level in the database that is undergoing a merge, add -** the number of segments on the rhs of the level. -** -** A level record not undergoing a merge is 10 integers. A level record -** with nRhs rhs segments and (nRhs+1) input segments (i.e. including the -** separators from the next level) is (11*nRhs+20) integers. The maximum -** per right-hand-side level is therefore 21 integers. So the maximum -** size of all level records in a checkpoint is 21*40=820 integers. -** -** TODO: Before pointer values were changed from 32 to 64 bits, the above -** used to come to 420 bytes - leaving significant space for a free-list -** prefix. No more. To fix this, reduce the size of the level records in -** a db snapshot, and improve management of the free-list tail in -** lsm_sorted.c. -*/ -#define LSM_MAX_RHS_SEGMENTS 40 - -/* -** LARGE NUMBERS OF FREELIST ENTRIES: -** -** There is also a limit (LSM_MAX_FREELIST_ENTRIES - defined in lsmInt.h) -** on the number of free-list entries stored in a checkpoint. Since each -** free-list entry consists of 3 integers, the maximum free-list size is -** 3*100=300 integers. Combined with the limit on rhs segments defined -** above, this ensures that a checkpoint always fits within a 4096 byte -** meta page. -** -** If the database contains more than 100 free blocks, the "overflow" flag -** in the checkpoint header is set and the remainder are stored in the -** system FREELIST entry in the LSM (along with user data). The value -** accompanying the FREELIST key in the LSM is, like a checkpoint, an array -** of 32-bit big-endian integers. As follows: -** -** For each entry: -** a. Block number of free block. -** b. MSW of associated checkpoint id. -** c. LSW of associated checkpoint id. -** -** The number of entries is not required - it is implied by the size of the -** value blob containing the integer array. -** -** Note that the limit defined by LSM_MAX_FREELIST_ENTRIES is a hard limit. -** The actual value used may be configured using LSM_CONFIG_MAX_FREELIST. -*/ - -/* -** The argument to this macro must be of type u32. On a little-endian -** architecture, it returns the u32 value that results from interpreting -** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by intepreting the 4 bytes -** of the input value as a little-endian integer. -*/ -#define BYTESWAP32(x) ( \ - (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ - + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ -) - -static const int one = 1; -#define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) - -/* Sizes, in integers, of various parts of the checkpoint. */ -#define CKPT_HDR_SIZE 9 -#define CKPT_LOGPTR_SIZE 4 -#define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) - -/* A #define to describe each integer in the checkpoint header. */ -#define CKPT_HDR_ID_MSW 0 -#define CKPT_HDR_ID_LSW 1 -#define CKPT_HDR_NCKPT 2 -#define CKPT_HDR_CMPID 3 -#define CKPT_HDR_NBLOCK 4 -#define CKPT_HDR_BLKSZ 5 -#define CKPT_HDR_NLEVEL 6 -#define CKPT_HDR_PGSZ 7 -#define CKPT_HDR_NWRITE 8 - -#define CKPT_HDR_LO_MSW 9 -#define CKPT_HDR_LO_LSW 10 -#define CKPT_HDR_LO_CKSUM1 11 -#define CKPT_HDR_LO_CKSUM2 12 - -typedef struct CkptBuffer CkptBuffer; - -/* -** Dynamic buffer used to accumulate data for a checkpoint. -*/ -struct CkptBuffer { - lsm_env *pEnv; - int nAlloc; - u32 *aCkpt; -}; - -/* -** Calculate the checksum of the checkpoint specified by arguments aCkpt and -** nCkpt. Store the checksum in *piCksum1 and *piCksum2 before returning. -** -** The value of the nCkpt parameter includes the two checksum values at -** the end of the checkpoint. They are not used as inputs to the checksum -** calculation. The checksum is based on the array of (nCkpt-2) integers -** at aCkpt[]. -*/ -static void ckptChecksum(u32 *aCkpt, u32 nCkpt, u32 *piCksum1, u32 *piCksum2){ - u32 i; - u32 cksum1 = 1; - u32 cksum2 = 2; - - if( nCkpt % 2 ){ - cksum1 += aCkpt[nCkpt-3] & 0x0000FFFF; - cksum2 += aCkpt[nCkpt-3] & 0xFFFF0000; - } - - for(i=0; (i+3)=p->nAlloc ){ - int nNew = LSM_MAX(8, iIdx*2); - p->aCkpt = (u32 *)lsmReallocOrFree(p->pEnv, p->aCkpt, nNew*sizeof(u32)); - if( !p->aCkpt ){ - *pRc = LSM_NOMEM_BKPT; - return; - } - p->nAlloc = nNew; - } - p->aCkpt[iIdx] = iVal; -} - -/* -** Argument aInt points to an array nInt elements in size. Switch the -** endian-ness of each element of the array. -*/ -static void ckptChangeEndianness(u32 *aInt, int nInt){ - if( LSM_LITTLE_ENDIAN ){ - int i; - for(i=0; iaCkpt, nCkpt+2, &aCksum[0], &aCksum[1]); - ckptSetValue(p, nCkpt, aCksum[0], pRc); - ckptSetValue(p, nCkpt+1, aCksum[1], pRc); - } -} - -static void ckptAppend64(CkptBuffer *p, int *piOut, i64 iVal, int *pRc){ - int iOut = *piOut; - ckptSetValue(p, iOut++, (iVal >> 32) & 0xFFFFFFFF, pRc); - ckptSetValue(p, iOut++, (iVal & 0xFFFFFFFF), pRc); - *piOut = iOut; -} - -static i64 ckptRead64(u32 *a){ - return (((i64)a[0]) << 32) + (i64)a[1]; -} - -static i64 ckptGobble64(u32 *a, int *piIn){ - int iIn = *piIn; - *piIn += 2; - return ckptRead64(&a[iIn]); -} - - -/* -** Append a 6-value segment record corresponding to pSeg to the checkpoint -** buffer passed as the third argument. -*/ -static void ckptExportSegment( - Segment *pSeg, - CkptBuffer *p, - int *piOut, - int *pRc -){ - ckptAppend64(p, piOut, pSeg->iFirst, pRc); - ckptAppend64(p, piOut, pSeg->iLastPg, pRc); - ckptAppend64(p, piOut, pSeg->iRoot, pRc); - ckptAppend64(p, piOut, pSeg->nSize, pRc); -} - -static void ckptExportLevel( - Level *pLevel, /* Level object to serialize */ - CkptBuffer *p, /* Append new level record to this ckpt */ - int *piOut, /* IN/OUT: Size of checkpoint so far */ - int *pRc /* IN/OUT: Error code */ -){ - int iOut = *piOut; - Merge *pMerge; - - pMerge = pLevel->pMerge; - ckptSetValue(p, iOut++, (u32)pLevel->iAge + (u32)(pLevel->flags<<16), pRc); - ckptSetValue(p, iOut++, pLevel->nRight, pRc); - ckptExportSegment(&pLevel->lhs, p, &iOut, pRc); - - assert( (pLevel->nRight>0)==(pMerge!=0) ); - if( pMerge ){ - int i; - for(i=0; inRight; i++){ - ckptExportSegment(&pLevel->aRhs[i], p, &iOut, pRc); - } - assert( pMerge->nInput==pLevel->nRight - || pMerge->nInput==pLevel->nRight+1 - ); - ckptSetValue(p, iOut++, pMerge->nInput, pRc); - ckptSetValue(p, iOut++, pMerge->nSkip, pRc); - for(i=0; inInput; i++){ - ckptAppend64(p, &iOut, pMerge->aInput[i].iPg, pRc); - ckptSetValue(p, iOut++, pMerge->aInput[i].iCell, pRc); - } - ckptAppend64(p, &iOut, pMerge->splitkey.iPg, pRc); - ckptSetValue(p, iOut++, pMerge->splitkey.iCell, pRc); - ckptAppend64(p, &iOut, pMerge->iCurrentPtr, pRc); - } - - *piOut = iOut; -} - -/* -** Populate the log offset fields of the checkpoint buffer. 4 values. -*/ -static void ckptExportLog( - lsm_db *pDb, - int bFlush, - CkptBuffer *p, - int *piOut, - int *pRc -){ - int iOut = *piOut; - - assert( iOut==CKPT_HDR_LO_MSW ); - - if( bFlush ){ - i64 iOff = pDb->treehdr.iOldLog; - ckptAppend64(p, &iOut, iOff, pRc); - ckptSetValue(p, iOut++, pDb->treehdr.oldcksum0, pRc); - ckptSetValue(p, iOut++, pDb->treehdr.oldcksum1, pRc); - }else{ - for(; iOut<=CKPT_HDR_LO_CKSUM2; iOut++){ - ckptSetValue(p, iOut, pDb->pShmhdr->aSnap2[iOut], pRc); - } - } - - assert( *pRc || iOut==CKPT_HDR_LO_CKSUM2+1 ); - *piOut = iOut; -} - -static void ckptExportAppendlist( - lsm_db *db, /* Database connection */ - CkptBuffer *p, /* Checkpoint buffer to write to */ - int *piOut, /* IN/OUT: Offset within checkpoint buffer */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - LsmPgno *aiAppend = db->pWorker->aiAppend; - - for(i=0; ipFS; /* File system object */ - Snapshot *pSnap = pDb->pWorker; /* Worker snapshot */ - int nLevel = 0; /* Number of levels in checkpoint */ - int iLevel; /* Used to count out nLevel levels */ - int iOut = 0; /* Current offset in aCkpt[] */ - Level *pLevel; /* Level iterator */ - int i; /* Iterator used while serializing freelist */ - CkptBuffer ckpt; - - /* Initialize the output buffer */ - memset(&ckpt, 0, sizeof(CkptBuffer)); - ckpt.pEnv = pDb->pEnv; - iOut = CKPT_HDR_SIZE; - - /* Write the log offset into the checkpoint. */ - ckptExportLog(pDb, bLog, &ckpt, &iOut, &rc); - - /* Write the append-point list */ - ckptExportAppendlist(pDb, &ckpt, &iOut, &rc); - - /* Figure out how many levels will be written to the checkpoint. */ - for(pLevel=lsmDbSnapshotLevel(pSnap); pLevel; pLevel=pLevel->pNext) nLevel++; - - /* Serialize nLevel levels. */ - iLevel = 0; - for(pLevel=lsmDbSnapshotLevel(pSnap); iLevelpNext){ - ckptExportLevel(pLevel, &ckpt, &iOut, &rc); - iLevel++; - } - - /* Write the block-redirect list */ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.n, &rc); - for(i=0; iredirect.n; i++){ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iFrom, &rc); - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iTo, &rc); - } - - /* Write the freelist */ - assert( pSnap->freelist.nEntry<=pDb->nMaxFreelist ); - if( rc==LSM_OK ){ - int nFree = pSnap->freelist.nEntry; - ckptSetValue(&ckpt, iOut++, nFree, &rc); - for(i=0; ifreelist.aEntry[i]; - ckptSetValue(&ckpt, iOut++, p->iBlk, &rc); - ckptSetValue(&ckpt, iOut++, (p->iId >> 32) & 0xFFFFFFFF, &rc); - ckptSetValue(&ckpt, iOut++, p->iId & 0xFFFFFFFF, &rc); - } - } - - /* Write the checkpoint header */ - assert( iId>=0 ); - assert( pSnap->iCmpId==pDb->compress.iId - || pSnap->iCmpId==LSM_COMPRESSION_EMPTY - ); - ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); - ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); - ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); - ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); - ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); - ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); - - if( bCksum ){ - ckptAddChecksum(&ckpt, iOut, &rc); - }else{ - ckptSetValue(&ckpt, iOut, 0, &rc); - ckptSetValue(&ckpt, iOut+1, 0, &rc); - } - iOut += 2; - assert( iOut<=1024 ); - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): id=%lld freelist: %d", iId, pSnap->freelist.nEntry - ); - for(i=0; ifreelist.nEntry; i++){ - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): iBlk=%d id=%lld", - pSnap->freelist.aEntry[i].iBlk, - pSnap->freelist.aEntry[i].iId - ); - } -#endif - - *ppCkpt = (void *)ckpt.aCkpt; - if( pnCkpt ) *pnCkpt = sizeof(u32)*iOut; - return rc; -} - - -/* -** Helper function for ckptImport(). -*/ -static void ckptNewSegment( - u32 *aIn, - int *piIn, - Segment *pSegment /* Populate this structure */ -){ - assert( pSegment->iFirst==0 && pSegment->iLastPg==0 ); - assert( pSegment->nSize==0 && pSegment->iRoot==0 ); - pSegment->iFirst = ckptGobble64(aIn, piIn); - pSegment->iLastPg = ckptGobble64(aIn, piIn); - pSegment->iRoot = ckptGobble64(aIn, piIn); - pSegment->nSize = ckptGobble64(aIn, piIn); - assert( pSegment->iFirst ); -} - -static int ckptSetupMerge(lsm_db *pDb, u32 *aInt, int *piIn, Level *pLevel){ - Merge *pMerge; /* Allocated Merge object */ - int nInput; /* Number of input segments in merge */ - int iIn = *piIn; /* Next value to read from aInt[] */ - int i; /* Iterator variable */ - int nByte; /* Number of bytes to allocate */ - - /* Allocate the Merge object. If malloc() fails, return LSM_NOMEM. */ - nInput = (int)aInt[iIn++]; - nByte = sizeof(Merge) + sizeof(MergeInput) * nInput; - pMerge = (Merge *)lsmMallocZero(pDb->pEnv, nByte); - if( !pMerge ) return LSM_NOMEM_BKPT; - pLevel->pMerge = pMerge; - - /* Populate the Merge object. */ - pMerge->aInput = (MergeInput *)&pMerge[1]; - pMerge->nInput = nInput; - pMerge->iOutputOff = -1; - pMerge->nSkip = (int)aInt[iIn++]; - for(i=0; iaInput[i].iPg = ckptGobble64(aInt, &iIn); - pMerge->aInput[i].iCell = (int)aInt[iIn++]; - } - pMerge->splitkey.iPg = ckptGobble64(aInt, &iIn); - pMerge->splitkey.iCell = (int)aInt[iIn++]; - pMerge->iCurrentPtr = ckptGobble64(aInt, &iIn); - - /* Set *piIn and return LSM_OK. */ - *piIn = iIn; - return LSM_OK; -} - - -static int ckptLoadLevels( - lsm_db *pDb, - u32 *aIn, - int *piIn, - int nLevel, - Level **ppLevel -){ - int i; - int rc = LSM_OK; - Level *pRet = 0; - Level **ppNext; - int iIn = *piIn; - - ppNext = &pRet; - for(i=0; rc==LSM_OK && ipEnv, sizeof(Level), &rc); - if( rc==LSM_OK ){ - pLevel->iAge = (u16)(aIn[iIn] & 0x0000FFFF); - pLevel->flags = (u16)((aIn[iIn]>>16) & 0x0000FFFF); - iIn++; - pLevel->nRight = aIn[iIn++]; - if( pLevel->nRight ){ - int nByte = sizeof(Segment) * pLevel->nRight; - pLevel->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - *ppNext = pLevel; - ppNext = &pLevel->pNext; - - /* Allocate the main segment */ - ckptNewSegment(aIn, &iIn, &pLevel->lhs); - - /* Allocate each of the right-hand segments, if any */ - for(iRight=0; iRightnRight; iRight++){ - ckptNewSegment(aIn, &iIn, &pLevel->aRhs[iRight]); - } - - /* Set up the Merge object, if required */ - if( pLevel->nRight>0 ){ - rc = ckptSetupMerge(pDb, aIn, &iIn, pLevel); - } - } - } - } - - if( rc!=LSM_OK ){ - /* An OOM must have occurred. Free any level structures allocated and - ** return the error to the caller. */ - lsmSortedFreeLevel(pDb->pEnv, pRet); - pRet = 0; - } - - *ppLevel = pRet; - *piIn = iIn; - return rc; -} - - -int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal){ - int rc = LSM_OK; - if( nVal>0 ){ - u32 *aIn; - - aIn = lsmMallocRc(pDb->pEnv, nVal, &rc); - if( aIn ){ - Level *pLevel = 0; - Level *pParent; - - int nIn; - int nLevel; - int iIn = 1; - memcpy(aIn, pVal, nVal); - nIn = nVal / sizeof(u32); - - ckptChangeEndianness(aIn, nIn); - nLevel = aIn[0]; - rc = ckptLoadLevels(pDb, aIn, &iIn, nLevel, &pLevel); - lsmFree(pDb->pEnv, aIn); - assert( rc==LSM_OK || pLevel==0 ); - if( rc==LSM_OK ){ - pParent = lsmDbSnapshotLevel(pDb->pWorker); - assert( pParent ); - while( pParent->pNext ) pParent = pParent->pNext; - pParent->pNext = pLevel; - } - } - } - - return rc; -} - -/* -** Return the data for the LEVELS record. -** -** The size of the checkpoint that can be stored in the database header -** must not exceed 1024 32-bit integers. Normally, it does not. However, -** if it does, part of the checkpoint must be stored in the LSM. This -** routine returns that part. -*/ -int lsmCheckpointLevels( - lsm_db *pDb, /* Database handle */ - int nLevel, /* Number of levels to write to blob */ - void **paVal, /* OUT: Pointer to LEVELS blob */ - int *pnVal /* OUT: Size of LEVELS blob in bytes */ -){ - Level *p; /* Used to iterate through levels */ - int nAll= 0; - int rc; - int i; - int iOut; - CkptBuffer ckpt; - assert( nLevel>0 ); - - for(p=lsmDbSnapshotLevel(pDb->pWorker); p; p=p->pNext) nAll++; - - assert( nAll>nLevel ); - nAll -= nLevel; - for(p=lsmDbSnapshotLevel(pDb->pWorker); p && nAll>0; p=p->pNext) nAll--; - - memset(&ckpt, 0, sizeof(CkptBuffer)); - ckpt.pEnv = pDb->pEnv; - - ckptSetValue(&ckpt, 0, nLevel, &rc); - iOut = 1; - for(i=0; rc==LSM_OK && ipNext; - } - assert( rc!=LSM_OK || p==0 ); - - if( rc==LSM_OK ){ - ckptChangeEndianness(ckpt.aCkpt, iOut); - *paVal = (void *)ckpt.aCkpt; - *pnVal = iOut * sizeof(u32); - }else{ - *pnVal = 0; - *paVal = 0; - } - - return rc; -} - -/* -** Read the checkpoint id from meta-page pPg. -*/ -static i64 ckptLoadId(MetaPage *pPg){ - i64 ret = 0; - if( pPg ){ - int nData; - u8 *aData = lsmFsMetaPageData(pPg, &nData); - ret = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32) + - ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); - } - return ret; -} - -/* -** Return true if the buffer passed as an argument contains a valid -** checkpoint. -*/ -static int ckptChecksumOk(u32 *aCkpt){ - u32 nCkpt = aCkpt[CKPT_HDR_NCKPT]; - u32 cksum1; - u32 cksum2; - - if( nCkpt(LSM_META_RW_PAGE_SIZE)/sizeof(u32) ){ - return 0; - } - ckptChecksum(aCkpt, nCkpt, &cksum1, &cksum2); - return (cksum1==aCkpt[nCkpt-2] && cksum2==aCkpt[nCkpt-1]); -} - -/* -** Attempt to load a checkpoint from meta page iMeta. -** -** This function is a no-op if *pRc is set to any value other than LSM_OK -** when it is called. If an error occurs, *pRc is set to an LSM error code -** before returning. -** -** If no error occurs and the checkpoint is successfully loaded, copy it to -** ShmHeader.aSnap1[] and ShmHeader.aSnap2[], and set ShmHeader.iMetaPage -** to indicate its origin. In this case return 1. Or, if the checkpoint -** cannot be loaded (because the checksum does not compute), return 0. -*/ -static int ckptTryLoad(lsm_db *pDb, MetaPage *pPg, u32 iMeta, int *pRc){ - int bLoaded = 0; /* Return value */ - if( *pRc==LSM_OK ){ - int rc = LSM_OK; /* Error code */ - u32 *aCkpt = 0; /* Pointer to buffer containing checkpoint */ - u32 nCkpt; /* Number of elements in aCkpt[] */ - int nData; /* Bytes of data in aData[] */ - u8 *aData; /* Meta page data */ - - aData = lsmFsMetaPageData(pPg, &nData); - nCkpt = (u32)lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); - if( nCkpt<=nData/sizeof(u32) && nCkpt>CKPT_HDR_NCKPT ){ - aCkpt = (u32 *)lsmMallocRc(pDb->pEnv, nCkpt*sizeof(u32), &rc); - } - if( aCkpt ){ - memcpy(aCkpt, aData, nCkpt*sizeof(u32)); - ckptChangeEndianness(aCkpt, nCkpt); - if( ckptChecksumOk(aCkpt) ){ - ShmHeader *pShm = pDb->pShmhdr; - memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); - memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); - memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); - pShm->iMetaPage = iMeta; - bLoaded = 1; - } - } - - lsmFree(pDb->pEnv, aCkpt); - *pRc = rc; - } - return bLoaded; -} - -/* -** Initialize the shared-memory header with an empty snapshot. This function -** is called when no valid snapshot can be found in the database header. -*/ -static void ckptLoadEmpty(lsm_db *pDb){ - u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The redirected block list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ - }; - u32 nCkpt = array_size(aCkpt); - ShmHeader *pShm = pDb->pShmhdr; - - aCkpt[CKPT_HDR_NCKPT] = nCkpt; - aCkpt[CKPT_HDR_BLKSZ] = pDb->nDfltBlksz; - aCkpt[CKPT_HDR_PGSZ] = pDb->nDfltPgsz; - ckptChecksum(aCkpt, array_size(aCkpt), &aCkpt[nCkpt-2], &aCkpt[nCkpt-1]); - - memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); - memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); - memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); -} - -/* -** This function is called as part of database recovery to initialize the -** ShmHeader.aSnap1[] and ShmHeader.aSnap2[] snapshots. -*/ -int lsmCheckpointRecover(lsm_db *pDb){ - int rc = LSM_OK; /* Return Code */ - i64 iId1; /* Id of checkpoint on meta-page 1 */ - i64 iId2; /* Id of checkpoint on meta-page 2 */ - int bLoaded = 0; /* True once checkpoint has been loaded */ - int cmp; /* True if (iId2>iId1) */ - MetaPage *apPg[2] = {0, 0}; /* Meta-pages 1 and 2 */ - - rc = lsmFsMetaPageGet(pDb->pFS, 0, 1, &apPg[0]); - if( rc==LSM_OK ) rc = lsmFsMetaPageGet(pDb->pFS, 0, 2, &apPg[1]); - - iId1 = ckptLoadId(apPg[0]); - iId2 = ckptLoadId(apPg[1]); - cmp = (iId2 > iId1); - bLoaded = ckptTryLoad(pDb, apPg[cmp?1:0], (cmp?2:1), &rc); - if( bLoaded==0 ){ - bLoaded = ckptTryLoad(pDb, apPg[cmp?0:1], (cmp?1:2), &rc); - } - - /* The database does not contain a valid checkpoint. Initialize the shared - ** memory header with an empty checkpoint. */ - if( bLoaded==0 ){ - ckptLoadEmpty(pDb); - } - - lsmFsMetaPageRelease(apPg[0]); - lsmFsMetaPageRelease(apPg[1]); - - return rc; -} - -/* -** Store the snapshot in pDb->aSnapshot[] in meta-page iMeta. -*/ -int lsmCheckpointStore(lsm_db *pDb, int iMeta){ - MetaPage *pPg = 0; - int rc; - - assert( iMeta==1 || iMeta==2 ); - rc = lsmFsMetaPageGet(pDb->pFS, 1, iMeta, &pPg); - if( rc==LSM_OK ){ - u8 *aData; - int nData; - int nCkpt; - - nCkpt = (int)pDb->aSnapshot[CKPT_HDR_NCKPT]; - aData = lsmFsMetaPageData(pPg, &nData); - memcpy(aData, pDb->aSnapshot, nCkpt*sizeof(u32)); - ckptChangeEndianness((u32 *)aData, nCkpt); - rc = lsmFsMetaPageRelease(pPg); - } - - return rc; -} - -/* -** Copy the current client snapshot from shared-memory to pDb->aSnapshot[]. -*/ -int lsmCheckpointLoad(lsm_db *pDb, int *piRead){ - int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; - ShmHeader *pShm = pDb->pShmhdr; - while( (nRem--)>0 ){ - int nInt; - - nInt = pShm->aSnap1[CKPT_HDR_NCKPT]; - if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ - memcpy(pDb->aSnapshot, pShm->aSnap1, nInt*sizeof(u32)); - if( ckptChecksumOk(pDb->aSnapshot) ){ - if( piRead ) *piRead = 1; - return LSM_OK; - } - } - - nInt = pShm->aSnap2[CKPT_HDR_NCKPT]; - if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ - memcpy(pDb->aSnapshot, pShm->aSnap2, nInt*sizeof(u32)); - if( ckptChecksumOk(pDb->aSnapshot) ){ - if( piRead ) *piRead = 2; - return LSM_OK; - } - } - - lsmShmBarrier(pDb); - } - return LSM_PROTOCOL_BKPT; -} - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ - int rc; - - assert( db->pClient==0 && db->pWorker==0 ); - rc = lsmCheckpointLoad(db, 0); - if( rc==LSM_OK ){ - *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; - } - - return rc; -} - -int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ - u32 *aShm; - assert( iSnap==1 || iSnap==2 ); - aShm = (iSnap==1) ? pDb->pShmhdr->aSnap1 : pDb->pShmhdr->aSnap2; - return (lsmCheckpointId(pDb->aSnapshot, 0)==lsmCheckpointId(aShm, 0) ); -} - -int lsmCheckpointClientCacheOk(lsm_db *pDb){ - return ( pDb->pClient - && pDb->pClient->iId==lsmCheckpointId(pDb->aSnapshot, 0) - && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap1, 0) - && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap2, 0) - ); -} - -int lsmCheckpointLoadWorker(lsm_db *pDb){ - int rc; - ShmHeader *pShm = pDb->pShmhdr; - int nInt1; - int nInt2; - - /* Must be holding the WORKER lock to do this. Or DMS2. */ - assert( - lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) - ); - - /* Check that the two snapshots match. If not, repair them. */ - nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; - nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; - if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ - if( ckptChecksumOk(pShm->aSnap1) ){ - memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); - }else if( ckptChecksumOk(pShm->aSnap2) ){ - memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); - }else{ - return LSM_PROTOCOL_BKPT; - } - } - - rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); - if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; - - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); - } - -#if 0 - assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); -#endif - return rc; -} - -int lsmCheckpointDeserialize( - lsm_db *pDb, - int bInclFreelist, /* If true, deserialize free-list */ - u32 *aCkpt, - Snapshot **ppSnap -){ - int rc = LSM_OK; - Snapshot *pNew; - - pNew = (Snapshot *)lsmMallocZeroRc(pDb->pEnv, sizeof(Snapshot), &rc); - if( rc==LSM_OK ){ - Level *pLvl; - int nFree; - int i; - int nLevel = (int)aCkpt[CKPT_HDR_NLEVEL]; - int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; - - pNew->iId = lsmCheckpointId(aCkpt, 0); - pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; - pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; - rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); - pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); - pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; - - /* Make a copy of the append-list */ - for(i=0; iaiAppend[i] = ckptRead64(a); - } - - /* Read the block-redirect list */ - pNew->redirect.n = aCkpt[iIn++]; - if( pNew->redirect.n ){ - pNew->redirect.a = lsmMallocZeroRc(pDb->pEnv, - (sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS), &rc - ); - if( rc==LSM_OK ){ - for(i=0; iredirect.n; i++){ - pNew->redirect.a[i].iFrom = aCkpt[iIn++]; - pNew->redirect.a[i].iTo = aCkpt[iIn++]; - } - } - for(pLvl=pNew->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - if( pLvl->nRight ){ - pLvl->aRhs[pLvl->nRight-1].pRedirect = &pNew->redirect; - }else{ - pLvl->lhs.pRedirect = &pNew->redirect; - } - } - - /* Copy the free-list */ - if( rc==LSM_OK && bInclFreelist ){ - nFree = aCkpt[iIn++]; - if( nFree ){ - pNew->freelist.aEntry = (FreelistEntry *)lsmMallocZeroRc( - pDb->pEnv, sizeof(FreelistEntry)*nFree, &rc - ); - if( rc==LSM_OK ){ - int j; - for(j=0; jfreelist.aEntry[j]; - p->iBlk = aCkpt[iIn++]; - p->iId = ((i64)(aCkpt[iIn])<<32) + aCkpt[iIn+1]; - iIn += 2; - } - pNew->freelist.nEntry = pNew->freelist.nAlloc = nFree; - } - } - } - } - - if( rc!=LSM_OK ){ - lsmFreeSnapshot(pDb->pEnv, pNew); - pNew = 0; - } - - *ppSnap = pNew; - return rc; -} - -/* -** Connection pDb must be the worker connection in order to call this -** function. It returns true if the database already contains the maximum -** number of levels or false otherwise. -** -** This is used when flushing the in-memory tree to disk. If the database -** is already full, then the caller should invoke lsm_work() or similar -** until it is not full before creating a new level by flushing the in-memory -** tree to disk. Limiting the number of levels in the database ensures that -** the records describing them always fit within the checkpoint blob. -*/ -int lsmDatabaseFull(lsm_db *pDb){ - Level *p; - int nRhs = 0; - - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); - assert( pDb->pWorker ); - - for(p=pDb->pWorker->pLevel; p; p=p->pNext){ - nRhs += (p->nRight ? p->nRight : 1); - } - - return (nRhs >= LSM_MAX_RHS_SEGMENTS); -} - -/* -** The connection passed as the only argument is currently the worker -** connection. Some work has been performed on the database by the connection, -** but no new snapshot has been written into shared memory. -** -** This function updates the shared-memory worker and client snapshots with -** the new snapshot produced by the work performed by pDb. -** -** If successful, LSM_OK is returned. Otherwise, if an error occurs, an LSM -** error code is returned. -*/ -int lsmCheckpointSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *pSnap = pDb->pWorker; - ShmHeader *pShm = pDb->pShmhdr; - void *p = 0; - int n = 0; - int rc; - - pSnap->iId++; - rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId, 1, &p, &n); - if( rc!=LSM_OK ) return rc; - assert( ckptChecksumOk((u32 *)p) ); - - assert( n<=LSM_META_RW_PAGE_SIZE ); - memcpy(pShm->aSnap2, p, n); - lsmShmBarrier(pDb); - memcpy(pShm->aSnap1, p, n); - lsmFree(pDb->pEnv, p); - - /* assert( lsmFsIntegrityCheck(pDb) ); */ - return LSM_OK; -} - -/* -** This function is used to determine the snapshot-id of the most recently -** checkpointed snapshot. Variable ShmHeader.iMetaPage indicates which of -** the two meta-pages said snapshot resides on (if any). -** -** If successful, this function loads the snapshot from the meta-page, -** verifies its checksum and sets *piId to the snapshot-id before returning -** LSM_OK. Or, if the checksum attempt fails, *piId is set to zero and -** LSM_OK returned. If an error occurs, an LSM error code is returned and -** the final value of *piId is undefined. -*/ -int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite){ - int rc = LSM_OK; - MetaPage *pPg; - u32 iMeta; - - iMeta = pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ - rc = lsmFsMetaPageGet(pDb->pFS, 0, iMeta, &pPg); - if( rc==LSM_OK ){ - int nCkpt; - int nData; - u8 *aData; - - aData = lsmFsMetaPageData(pPg, &nData); - assert( nData==LSM_META_RW_PAGE_SIZE ); - nCkpt = lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); - if( nCkpt<(LSM_META_RW_PAGE_SIZE/sizeof(u32)) ){ - u32 *aCopy = lsmMallocRc(pDb->pEnv, sizeof(u32) * nCkpt, &rc); - if( aCopy ){ - memcpy(aCopy, aData, nCkpt*sizeof(u32)); - ckptChangeEndianness(aCopy, nCkpt); - if( ckptChecksumOk(aCopy) ){ - if( piId ) *piId = lsmCheckpointId(aCopy, 0); - if( piLog ) *piLog = (lsmCheckpointLogOffset(aCopy) >> 1); - if( pnWrite ) *pnWrite = aCopy[CKPT_HDR_NWRITE]; - } - lsmFree(pDb->pEnv, aCopy); - } - } - lsmFsMetaPageRelease(pPg); - } - } - - if( (iMeta!=1 && iMeta!=2) || rc!=LSM_OK || pDb->pShmhdr->iMetaPage!=iMeta ){ - if( piId ) *piId = 0; - if( piLog ) *piLog = 0; - if( pnWrite ) *pnWrite = 0; - } - return rc; -} - -/* -** Return the checkpoint-id of the checkpoint array passed as the first -** argument to this function. If the second argument is true, then assume -** that the checkpoint is made up of 32-bit big-endian integers. If it -** is false, assume that the integers are in machine byte order. -*/ -i64 lsmCheckpointId(u32 *aCkpt, int bDisk){ - i64 iId; - if( bDisk ){ - u8 *aData = (u8 *)aCkpt; - iId = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32); - iId += ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); - }else{ - iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; - } - return iId; -} - -u32 lsmCheckpointNBlock(u32 *aCkpt){ - return aCkpt[CKPT_HDR_NBLOCK]; -} - -u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ - if( bDisk ){ - return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); - }else{ - return aCkpt[CKPT_HDR_NWRITE]; - } -} - -i64 lsmCheckpointLogOffset(u32 *aCkpt){ - return ((i64)aCkpt[CKPT_HDR_LO_MSW] << 32) + (i64)aCkpt[CKPT_HDR_LO_LSW]; -} - -int lsmCheckpointPgsz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_PGSZ]; } - -int lsmCheckpointBlksz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_BLKSZ]; } - -void lsmCheckpointLogoffset( - u32 *aCkpt, - DbLog *pLog -){ - pLog->aRegion[2].iStart = (lsmCheckpointLogOffset(aCkpt) >> 1); - - pLog->cksum0 = aCkpt[CKPT_HDR_LO_CKSUM1]; - pLog->cksum1 = aCkpt[CKPT_HDR_LO_CKSUM2]; - pLog->iSnapshotId = lsmCheckpointId(aCkpt, 0); -} - -void lsmCheckpointZeroLogoffset(lsm_db *pDb){ - u32 nCkpt; - - nCkpt = pDb->aSnapshot[CKPT_HDR_NCKPT]; - assert( nCkpt>CKPT_HDR_NCKPT ); - assert( nCkpt==pDb->pShmhdr->aSnap1[CKPT_HDR_NCKPT] ); - assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap1, nCkpt*sizeof(u32)) ); - assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap2, nCkpt*sizeof(u32)) ); - - pDb->aSnapshot[CKPT_HDR_LO_MSW] = 0; - pDb->aSnapshot[CKPT_HDR_LO_LSW] = 0; - ckptChecksum(pDb->aSnapshot, nCkpt, - &pDb->aSnapshot[nCkpt-2], &pDb->aSnapshot[nCkpt-1] - ); - - memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); - memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); -} - -/* -** Set the output variable to the number of KB of data written into the -** database file since the most recent checkpoint. -*/ -int lsmCheckpointSize(lsm_db *db, int *pnKB){ - int rc = LSM_OK; - u32 nSynced; - - /* Set nSynced to the number of pages that had been written when the - ** database was last checkpointed. */ - rc = lsmCheckpointSynced(db, 0, 0, &nSynced); - - if( rc==LSM_OK ){ - u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; - u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; - *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); - } - - return rc; -} diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c deleted file mode 100644 index fd78835bbb..0000000000 --- a/ext/lsm1/lsm_file.c +++ /dev/null @@ -1,3311 +0,0 @@ -/* -** 2011-08-26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** NORMAL DATABASE FILE FORMAT -** -** The following database file format concepts are used by the code in -** this file to read and write the database file. -** -** Pages: -** -** A database file is divided into pages. The first 8KB of the file consists -** of two 4KB meta-pages. The meta-page size is not configurable. The -** remainder of the file is made up of database pages. The default database -** page size is 4KB. Database pages are aligned to page-size boundaries, -** so if the database page size is larger than 8KB there is a gap between -** the end of the meta pages and the start of the database pages. -** -** Database pages are numbered based on their position in the file. Page N -** begins at byte offset ((N-1)*pgsz). This means that page 1 does not -** exist - since it would always overlap with the meta pages. If the -** page-size is (say) 512 bytes, then the first usable page in the database -** is page 33. -** -** It is assumed that the first two meta pages and the data that follows -** them are located on different disk sectors. So that if a power failure -** while writing to a meta page there is no risk of damage to the other -** meta page or any other part of the database file. TODO: This may need -** to be revisited. -** -** Blocks: -** -** The database file is also divided into blocks. The default block size is -** 1MB. When writing to the database file, an attempt is made to write data -** in contiguous block-sized chunks. -** -** The first and last page on each block are special in that they are 4 -** bytes smaller than all other pages. This is because the last four bytes -** of space on the first and last pages of each block are reserved for -** pointers to other blocks (i.e. a 32-bit block number). -** -** Runs: -** -** A run is a sequence of pages that the upper layer uses to store a -** sorted array of database keys (and accompanying data - values, FC -** pointers and so on). Given a page within a run, it is possible to -** navigate to the next page in the run as follows: -** -** a) if the current page is not the last in a block, the next page -** in the run is located immediately after the current page, OR -** -** b) if the current page is the last page in a block, the next page -** in the run is the first page on the block identified by the -** block pointer stored in the last 4 bytes of the current block. -** -** It is possible to navigate to the previous page in a similar fashion, -** using the block pointer embedded in the last 4 bytes of the first page -** of each block as required. -** -** The upper layer is responsible for identifying by page number the -** first and last page of any run that it needs to navigate - there are -** no "end-of-run" markers stored or identified by this layer. This is -** necessary as clients reading different database snapshots may access -** different subsets of a run. -** -** THE LOG FILE -** -** This file opens and closes the log file. But it does not contain any -** logic related to the log file format. Instead, it exports the following -** functions that are used by the code in lsm_log.c to read and write the -** log file: -** -** lsmFsOpenLog -** lsmFsWriteLog -** lsmFsSyncLog -** lsmFsReadLog -** lsmFsTruncateLog -** lsmFsCloseAndDeleteLog -** -** COMPRESSED DATABASE FILE FORMAT -** -** The compressed database file format is very similar to the normal format. -** The file still begins with two 4KB meta-pages (which are never compressed). -** It is still divided into blocks. -** -** The first and last four bytes of each block are reserved for 32-bit -** pointer values. Similar to the way four bytes are carved from the end of -** the first and last page of each block in uncompressed databases. From -** the point of view of the upper layer, all pages are the same size - this -** is different from the uncompressed format where the first and last pages -** on each block are 4 bytes smaller than the others. -** -** Pages are stored in variable length compressed form, as follows: -** -** * 3-byte size field containing the size of the compressed page image -** in bytes. The most significant bit of each byte of the size field -** is always set. The remaining 7 bits are used to store a 21-bit -** integer value (in big-endian order - the first byte in the field -** contains the most significant 7 bits). Since the maximum allowed -** size of a compressed page image is (2^17 - 1) bytes, there are -** actually 4 unused bits in the size field. -** -** In other words, if the size of the compressed page image is nSz, -** the header can be serialized as follows: -** -** u8 aHdr[3] -** aHdr[0] = 0x80 | (u8)(nSz >> 14); -** aHdr[1] = 0x80 | (u8)(nSz >> 7); -** aHdr[2] = 0x80 | (u8)(nSz >> 0); -** -** * Compressed page image. -** -** * A second copy of the 3-byte record header. -** -** A page number is a byte offset into the database file. So the smallest -** possible page number is 8192 (immediately after the two meta-pages). -** The first and root page of a segment are identified by a page number -** corresponding to the byte offset of the first byte in the corresponding -** page record. The last page of a segment is identified by the byte offset -** of the last byte in its record. -** -** Unlike uncompressed pages, compressed page records may span blocks. -** -** Sometimes, in order to avoid touching sectors that contain synced data -** when writing, it is necessary to insert unused space between compressed -** page records. This can be done as follows: -** -** * For less than 6 bytes of empty space, the first and last byte -** of the free space contain the total number of free bytes. For -** example: -** -** Block of 4 free bytes: 0x04 0x?? 0x?? 0x04 -** Block of 2 free bytes: 0x02 0x02 -** A single free byte: 0x01 -** -** * For 6 or more bytes of empty space, a record similar to a -** compressed page record is added to the segment. A padding record -** is distinguished from a compressed page record by the most -** significant bit of the second byte of the size field, which is -** cleared instead of set. -*/ -#include "lsmInt.h" - -#include -#include -#include - -/* -** File-system object. Each database connection allocates a single instance -** of the following structure. It is used for all access to the database and -** log files. -** -** The database file may be accessed via two methods - using mmap() or using -** read() and write() calls. In the general case both methods are used - a -** prefix of the file is mapped into memory and the remainder accessed using -** read() and write(). This is helpful when accessing very large files (or -** files that may grow very large during the lifetime of a database -** connection) on systems with 32-bit address spaces. However, it also requires -** that this object manage two distinct types of Page objects simultaneously - -** those that carry pointers to the mapped file and those that carry arrays -** populated by read() calls. -** -** pFree: -** The head of a singly-linked list that containing currently unused Page -** structures suitable for use as mmap-page handles. Connected by the -** Page.pFreeNext pointers. -** -** pMapped: -** The head of a singly-linked list that contains all pages that currently -** carry pointers to the mapped region. This is used if the region is -** every remapped - the pointers carried by existing pages can be adjusted -** to account for the remapping. Connected by the Page.pMappedNext pointers. -** -** pWaiting: -** When the upper layer wishes to append a new b-tree page to a segment, -** it allocates a Page object that carries a malloc'd block of memory - -** regardless of the mmap-related configuration. The page is not assigned -** a page number at first. When the upper layer has finished constructing -** the page contents, it calls lsmFsPagePersist() to assign a page number -** to it. At this point it is likely that N pages have been written to the -** segment, the (N+1)th page is still outstanding and the b-tree page is -** assigned page number (N+2). To avoid writing page (N+2) before page -** (N+1), the recently completed b-tree page is held in the singly linked -** list headed by pWaiting until page (N+1) has been written. -** -** Function lsmFsFlushWaiting() is responsible for eventually writing -** waiting pages to disk. -** -** apHash/nHash: -** Hash table used to store all Page objects that carry malloc'd arrays, -** except those b-tree pages that have not yet been assigned page numbers. -** Once they have been assigned page numbers - they are added to this -** hash table. -** -** Hash table overflow chains are connected using the Page.pHashNext -** pointers. -** -** pLruFirst, pLruLast: -** The first and last entries in a doubly-linked list of pages. This -** list contains all pages with malloc'd data that are present in the -** hash table and have a ref-count of zero. -*/ -struct FileSystem { - lsm_db *pDb; /* Database handle that owns this object */ - lsm_env *pEnv; /* Environment pointer */ - char *zDb; /* Database file name */ - char *zLog; /* Database file name */ - int nMetasize; /* Size of meta pages in bytes */ - int nMetaRwSize; /* Read/written size of meta pages in bytes */ - i64 nPagesize; /* Database page-size in bytes */ - i64 nBlocksize; /* Database block-size in bytes */ - - /* r/w file descriptors for both files. */ - LsmFile *pLsmFile; /* Used after lsm_close() to link into list */ - lsm_file *fdDb; /* Database file */ - lsm_file *fdLog; /* Log file */ - int szSector; /* Database file sector size */ - - /* If this is a compressed database, a pointer to the compression methods. - ** For an uncompressed database, a NULL pointer. */ - lsm_compress *pCompress; - u8 *aIBuffer; /* Buffer to compress to */ - u8 *aOBuffer; /* Buffer to uncompress from */ - int nBuffer; /* Allocated size of above buffers in bytes */ - - /* mmap() page related things */ - i64 nMapLimit; /* Maximum bytes of file to map */ - void *pMap; /* Current mapping of database file */ - i64 nMap; /* Bytes mapped at pMap */ - Page *pFree; /* Unused Page structures */ - Page *pMapped; /* List of Page structs that point to pMap */ - - /* Page cache parameters for non-mmap() pages */ - int nCacheMax; /* Configured cache size (in pages) */ - int nCacheAlloc; /* Current cache size (in pages) */ - Page *pLruFirst; /* Head of the LRU list */ - Page *pLruLast; /* Tail of the LRU list */ - int nHash; /* Number of hash slots in hash table */ - Page **apHash; /* nHash Hash slots */ - Page *pWaiting; /* b-tree pages waiting to be written */ - - /* Statistics */ - int nOut; /* Number of outstanding pages */ - int nWrite; /* Total number of pages written */ - int nRead; /* Total number of pages read */ -}; - -/* -** Database page handle. -** -** pSeg: -** When lsmFsSortedAppend() is called on a compressed database, the new -** page is not assigned a page number or location in the database file -** immediately. Instead, these are assigned by the lsmFsPagePersist() call -** right before it writes the compressed page image to disk. -** -** The lsmFsSortedAppend() function sets the pSeg pointer to point to the -** segment that the new page will be a part of. It is unset by -** lsmFsPagePersist() after the page is written to disk. -*/ -struct Page { - u8 *aData; /* Buffer containing page data */ - int nData; /* Bytes of usable data at aData[] */ - LsmPgno iPg; /* Page number */ - int nRef; /* Number of outstanding references */ - int flags; /* Combination of PAGE_XXX flags */ - Page *pHashNext; /* Next page in hash table slot */ - Page *pLruNext; /* Next page in LRU list */ - Page *pLruPrev; /* Previous page in LRU list */ - FileSystem *pFS; /* File system that owns this page */ - - /* Only used in compressed database mode: */ - int nCompress; /* Compressed size (or 0 for uncomp. db) */ - int nCompressPrev; /* Compressed size of prev page */ - Segment *pSeg; /* Segment this page will be written to */ - - /* Pointers for singly linked lists */ - Page *pWaitingNext; /* Next page in FileSystem.pWaiting list */ - Page *pFreeNext; /* Next page in FileSystem.pFree list */ - Page *pMappedNext; /* Next page in FileSystem.pMapped list */ -}; - -/* -** Meta-data page handle. There are two meta-data pages at the start of -** the database file, each FileSystem.nMetasize bytes in size. -*/ -struct MetaPage { - int iPg; /* Either 1 or 2 */ - int bWrite; /* Write back to db file on release */ - u8 *aData; /* Pointer to buffer */ - FileSystem *pFS; /* FileSystem that owns this page */ -}; - -/* -** Values for LsmPage.flags -*/ -#define PAGE_DIRTY 0x00000001 /* Set if page is dirty */ -#define PAGE_FREE 0x00000002 /* Set if Page.aData requires lsmFree() */ -#define PAGE_HASPREV 0x00000004 /* Set if page is first on uncomp. block */ - -/* -** Number of pgsz byte pages omitted from the start of block 1. The start -** of block 1 contains two 4096 byte meta pages (8192 bytes in total). -*/ -#define BLOCK1_HDR_SIZE(pgsz) LSM_MAX(1, 8192/(pgsz)) - -/* -** If NDEBUG is not defined, set a breakpoint in function lsmIoerrBkpt() -** to catch IO errors (any error returned by a VFS method). -*/ -#ifndef NDEBUG -static void lsmIoerrBkpt(void){ - static int nErr = 0; - nErr++; -} -static int IOERR_WRAPPER(int rc){ - if( rc!=LSM_OK ) lsmIoerrBkpt(); - return rc; -} -#else -# define IOERR_WRAPPER(rc) (rc) -#endif - -#ifdef NDEBUG -# define assert_lists_are_ok(x) -#else -static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash); - -static void assert_lists_are_ok(FileSystem *pFS){ -#if 0 - Page *p; - - assert( pFS->nMapLimit>=0 ); - - /* Check that all pages in the LRU list have nRef==0, pointers to buffers - ** in heap memory, and corresponding entries in the hash table. */ - for(p=pFS->pLruFirst; p; p=p->pLruNext){ - assert( p==pFS->pLruFirst || p->pLruPrev!=0 ); - assert( p==pFS->pLruLast || p->pLruNext!=0 ); - assert( p->pLruPrev==0 || p->pLruPrev->pLruNext==p ); - assert( p->pLruNext==0 || p->pLruNext->pLruPrev==p ); - assert( p->nRef==0 ); - assert( p->flags & PAGE_FREE ); - assert( p==fsPageFindInHash(pFS, p->iPg, 0) ); - } -#endif -} -#endif - -/* -** Wrappers around the VFS methods of the lsm_env object: -** -** lsmEnvOpen() -** lsmEnvRead() -** lsmEnvWrite() -** lsmEnvSync() -** lsmEnvSectorSize() -** lsmEnvClose() -** lsmEnvTruncate() -** lsmEnvUnlink() -** lsmEnvRemap() -*/ -int lsmEnvOpen(lsm_env *pEnv, const char *zFile, int flags, lsm_file **ppNew){ - return pEnv->xOpen(pEnv, zFile, flags, ppNew); -} - -static int lsmEnvRead( - lsm_env *pEnv, - lsm_file *pFile, - lsm_i64 iOff, - void *pRead, - int nRead -){ - return IOERR_WRAPPER( pEnv->xRead(pFile, iOff, pRead, nRead) ); -} - -static int lsmEnvWrite( - lsm_env *pEnv, - lsm_file *pFile, - lsm_i64 iOff, - const void *pWrite, - int nWrite -){ - return IOERR_WRAPPER( pEnv->xWrite(pFile, iOff, (void *)pWrite, nWrite) ); -} - -static int lsmEnvSync(lsm_env *pEnv, lsm_file *pFile){ - return IOERR_WRAPPER( pEnv->xSync(pFile) ); -} - -static int lsmEnvSectorSize(lsm_env *pEnv, lsm_file *pFile){ - return pEnv->xSectorSize(pFile); -} - -int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile){ - return IOERR_WRAPPER( pEnv->xClose(pFile) ); -} - -static int lsmEnvTruncate(lsm_env *pEnv, lsm_file *pFile, lsm_i64 nByte){ - return IOERR_WRAPPER( pEnv->xTruncate(pFile, nByte) ); -} - -static int lsmEnvUnlink(lsm_env *pEnv, const char *zDel){ - return IOERR_WRAPPER( pEnv->xUnlink(pEnv, zDel) ); -} - -static int lsmEnvRemap( - lsm_env *pEnv, - lsm_file *pFile, - i64 szMin, - void **ppMap, - i64 *pszMap -){ - return pEnv->xRemap(pFile, szMin, ppMap, pszMap); -} - -int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock){ - if( pFile==0 ) return LSM_OK; - return pEnv->xLock(pFile, iLock, eLock); -} - -int lsmEnvTestLock( - lsm_env *pEnv, - lsm_file *pFile, - int iLock, - int nLock, - int eLock -){ - return pEnv->xTestLock(pFile, iLock, nLock, eLock); -} - -int lsmEnvShmMap( - lsm_env *pEnv, - lsm_file *pFile, - int iChunk, - int sz, - void **ppOut -){ - return pEnv->xShmMap(pFile, iChunk, sz, ppOut); -} - -void lsmEnvShmBarrier(lsm_env *pEnv){ - pEnv->xShmBarrier(); -} - -void lsmEnvShmUnmap(lsm_env *pEnv, lsm_file *pFile, int bDel){ - pEnv->xShmUnmap(pFile, bDel); -} - -void lsmEnvSleep(lsm_env *pEnv, int nUs){ - pEnv->xSleep(pEnv, nUs); -} - - -/* -** Write the contents of string buffer pStr into the log file, starting at -** offset iOff. -*/ -int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr){ - assert( pFS->fdLog ); - return lsmEnvWrite(pFS->pEnv, pFS->fdLog, iOff, pStr->z, pStr->n); -} - -/* -** fsync() the log file. -*/ -int lsmFsSyncLog(FileSystem *pFS){ - assert( pFS->fdLog ); - return lsmEnvSync(pFS->pEnv, pFS->fdLog); -} - -/* -** Read nRead bytes of data starting at offset iOff of the log file. Append -** the results to string buffer pStr. -*/ -int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr){ - int rc; /* Return code */ - assert( pFS->fdLog ); - rc = lsmStringExtend(pStr, nRead); - if( rc==LSM_OK ){ - rc = lsmEnvRead(pFS->pEnv, pFS->fdLog, iOff, &pStr->z[pStr->n], nRead); - pStr->n += nRead; - } - return rc; -} - -/* -** Truncate the log file to nByte bytes in size. -*/ -int lsmFsTruncateLog(FileSystem *pFS, i64 nByte){ - if( pFS->fdLog==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); -} - -/* -** Truncate the db file to nByte bytes in size. -*/ -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ - if( pFS->fdDb==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); -} - -/* -** Close the log file. Then delete it from the file-system. This function -** is called during database shutdown only. -*/ -int lsmFsCloseAndDeleteLog(FileSystem *pFS){ - char *zDel; - - if( pFS->fdLog ){ - lsmEnvClose(pFS->pEnv, pFS->fdLog ); - pFS->fdLog = 0; - } - - zDel = lsmMallocPrintf(pFS->pEnv, "%s-log", pFS->zDb); - if( zDel ){ - lsmEnvUnlink(pFS->pEnv, zDel); - lsmFree(pFS->pEnv, zDel); - } - return LSM_OK; -} - -/* -** Return true if page iReal of the database should be accessed using mmap. -** False otherwise. -*/ -static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){ - return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit); -} - -/* -** Given that there are currently nHash slots in the hash table, return -** the hash key for file iFile, page iPg. -*/ -static int fsHashKey(int nHash, LsmPgno iPg){ - return (iPg % nHash); -} - -/* -** This is a helper function for lsmFsOpen(). It opens a single file on -** disk (either the database or log file). -*/ -static lsm_file *fsOpenFile( - FileSystem *pFS, /* File system object */ - int bReadonly, /* True to open this file read-only */ - int bLog, /* True for log, false for db */ - int *pRc /* IN/OUT: Error code */ -){ - lsm_file *pFile = 0; - if( *pRc==LSM_OK ){ - int flags = (bReadonly ? LSM_OPEN_READONLY : 0); - const char *zPath = (bLog ? pFS->zLog : pFS->zDb); - - *pRc = lsmEnvOpen(pFS->pEnv, zPath, flags, &pFile); - } - return pFile; -} - -/* -** If it is not already open, this function opens the log file. It returns -** LSM_OK if successful (or if the log file was already open) or an LSM -** error code otherwise. -** -** The log file must be opened before any of the following may be called: -** -** lsmFsWriteLog -** lsmFsSyncLog -** lsmFsReadLog -*/ -int lsmFsOpenLog(lsm_db *db, int *pbOpen){ - int rc = LSM_OK; - FileSystem *pFS = db->pFS; - - if( 0==pFS->fdLog ){ - pFS->fdLog = fsOpenFile(pFS, db->bReadonly, 1, &rc); - - if( rc==LSM_IOERR_NOENT && db->bReadonly ){ - rc = LSM_OK; - } - } - - if( pbOpen ) *pbOpen = (pFS->fdLog!=0); - return rc; -} - -/* -** Close the log file, if it is open. -*/ -void lsmFsCloseLog(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS->fdLog ){ - lsmEnvClose(pFS->pEnv, pFS->fdLog); - pFS->fdLog = 0; - } -} - -/* -** Open a connection to a database stored within the file-system. -** -** If parameter bReadonly is true, then open a read-only file-descriptor -** on the database file. It is possible that bReadonly will be false even -** if the user requested that pDb be opened read-only. This is because the -** file-descriptor may later on be recycled by a read-write connection. -** If the db file can be opened for read-write access, it always is. Parameter -** bReadonly is only ever true if it has already been determined that the -** db can only be opened for read-only access. -** -** Return LSM_OK if successful or an lsm error code otherwise. -*/ -int lsmFsOpen( - lsm_db *pDb, /* Database connection to open fd for */ - const char *zDb, /* Full path to database file */ - int bReadonly /* True to open db file read-only */ -){ - FileSystem *pFS; - int rc = LSM_OK; - int nDb = strlen(zDb); - int nByte; - - assert( pDb->pFS==0 ); - assert( pDb->pWorker==0 && pDb->pClient==0 ); - - nByte = sizeof(FileSystem) + nDb+1 + nDb+4+1; - pFS = (FileSystem *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - if( pFS ){ - LsmFile *pLsmFile; - pFS->zDb = (char *)&pFS[1]; - pFS->zLog = &pFS->zDb[nDb+1]; - pFS->nPagesize = LSM_DFLT_PAGE_SIZE; - pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; - pFS->nMetasize = LSM_META_PAGE_SIZE; - pFS->nMetaRwSize = LSM_META_RW_PAGE_SIZE; - pFS->pDb = pDb; - pFS->pEnv = pDb->pEnv; - - /* Make a copy of the database and log file names. */ - memcpy(pFS->zDb, zDb, nDb+1); - memcpy(pFS->zLog, zDb, nDb); - memcpy(&pFS->zLog[nDb], "-log", 5); - - /* Allocate the hash-table here. At some point, it should be changed - ** so that it can grow dynamicly. */ - pFS->nCacheMax = 2048*1024 / pFS->nPagesize; - pFS->nHash = 4096; - pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc); - - /* Open the database file */ - pLsmFile = lsmDbRecycleFd(pDb); - if( pLsmFile ){ - pFS->pLsmFile = pLsmFile; - pFS->fdDb = pLsmFile->pFile; - memset(pLsmFile, 0, sizeof(LsmFile)); - }else{ - pFS->pLsmFile = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmFile), &rc); - if( rc==LSM_OK ){ - pFS->fdDb = fsOpenFile(pFS, bReadonly, 0, &rc); - } - } - - if( rc!=LSM_OK ){ - lsmFsClose(pFS); - pFS = 0; - }else{ - pFS->szSector = lsmEnvSectorSize(pFS->pEnv, pFS->fdDb); - } - } - - pDb->pFS = pFS; - return rc; -} - -/* -** Configure the file-system object according to the current values of -** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. -*/ -int lsmFsConfigure(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS ){ - lsm_env *pEnv = pFS->pEnv; - Page *pPg; - - assert( pFS->nOut==0 ); - assert( pFS->pWaiting==0 ); - assert( pFS->pMapped==0 ); - - /* Reset any compression/decompression buffers already allocated */ - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - pFS->nBuffer = 0; - - /* Unmap the file, if it is currently mapped */ - if( pFS->pMap ){ - lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - pFS->nMapLimit = 0; - } - - /* Free all allocated page structures */ - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - assert( pPg->flags & PAGE_FREE ); - lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - pPg = pFS->pFree; - while( pPg ){ - Page *pNext = pPg->pFreeNext; - lsmFree(pEnv, pPg); - pPg = pNext; - } - - /* Zero pointers that point to deleted page objects */ - pFS->nCacheAlloc = 0; - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - pFS->pFree = 0; - if( pFS->apHash ){ - memset(pFS->apHash, 0, pFS->nHash*sizeof(pFS->apHash[0])); - } - - /* Configure the FileSystem object */ - if( db->compress.xCompress ){ - pFS->pCompress = &db->compress; - pFS->nMapLimit = 0; - }else{ - pFS->pCompress = 0; - if( db->iMmap==1 ){ - /* Unlimited */ - pFS->nMapLimit = (i64)1 << 60; - }else{ - /* iMmap is a limit in KB. Set nMapLimit to the same value in bytes. */ - pFS->nMapLimit = (i64)db->iMmap * 1024; - } - } - } - - return LSM_OK; -} - -/* -** Close and destroy a FileSystem object. -*/ -void lsmFsClose(FileSystem *pFS){ - if( pFS ){ - Page *pPg; - lsm_env *pEnv = pFS->pEnv; - - assert( pFS->nOut==0 ); - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - pPg = pFS->pFree; - while( pPg ){ - Page *pNext = pPg->pFreeNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - if( pFS->fdDb ) lsmEnvClose(pFS->pEnv, pFS->fdDb ); - if( pFS->fdLog ) lsmEnvClose(pFS->pEnv, pFS->fdLog ); - lsmFree(pEnv, pFS->pLsmFile); - lsmFree(pEnv, pFS->apHash); - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - lsmFree(pEnv, pFS); - } -} - -/* -** This function is called when closing a database handle (i.e. lsm_close()) -** if there exist other connections to the same database within this process. -** In that case the file-descriptor open on the database file is not closed -** when the FileSystem object is destroyed, as this would cause any POSIX -** locks held by the other connections to be silently dropped (see "man close" -** for details). Instead, the file-descriptor is stored in a list by the -** lsm_shared.c module until it is either closed or reused. -** -** This function returns a pointer to an object that can be linked into -** the list described above. The returned object now 'owns' the database -** file descriptr, so that when the FileSystem object is destroyed, it -** will not be closed. -** -** This function may be called at most once in the life-time of a -** FileSystem object. The results of any operations involving the database -** file descriptor are undefined once this function has been called. -** -** None of this is necessary on non-POSIX systems. But we do it anyway in -** the name of using as similar code as possible on all platforms. -*/ -LsmFile *lsmFsDeferClose(FileSystem *pFS){ - LsmFile *p = pFS->pLsmFile; - assert( p->pNext==0 ); - p->pFile = pFS->fdDb; - pFS->fdDb = 0; - pFS->pLsmFile = 0; - return p; -} - -/* -** Allocate a buffer and populate it with the output of the xFileid() -** method of the database file handle. If successful, set *ppId to point -** to the buffer and *pnId to the number of bytes in the buffer and return -** LSM_OK. Otherwise, set *ppId and *pnId to zero and return an LSM -** error code. -*/ -int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId){ - lsm_env *pEnv = pDb->pEnv; - FileSystem *pFS = pDb->pFS; - int rc; - int nId = 0; - void *pId; - - rc = pEnv->xFileid(pFS->fdDb, 0, &nId); - pId = lsmMallocZeroRc(pEnv, nId, &rc); - if( rc==LSM_OK ) rc = pEnv->xFileid(pFS->fdDb, pId, &nId); - - if( rc!=LSM_OK ){ - lsmFree(pEnv, pId); - pId = 0; - nId = 0; - } - - *ppId = pId; - *pnId = nId; - return rc; -} - -/* -** Return the nominal page-size used by this file-system. Actual pages -** may be smaller or larger than this value. -*/ -int lsmFsPageSize(FileSystem *pFS){ - return pFS->nPagesize; -} - -/* -** Return the block-size used by this file-system. -*/ -int lsmFsBlockSize(FileSystem *pFS){ - return pFS->nBlocksize; -} - -/* -** Configure the nominal page-size used by this file-system. Actual -** pages may be smaller or larger than this value. -*/ -void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){ - pFS->nPagesize = nPgsz; - pFS->nCacheMax = 2048*1024 / pFS->nPagesize; -} - -/* -** Configure the block-size used by this file-system. -*/ -void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){ - pFS->nBlocksize = nBlocksize; -} - -/* -** Return the page number of the first page on block iBlock. Blocks are -** numbered starting from 1. -** -** For a compressed database, page numbers are byte offsets. The first -** page on each block is the byte offset immediately following the 4-byte -** "previous block" pointer at the start of each block. -*/ -static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){ - LsmPgno iPg; - if( pFS->pCompress ){ - if( iBlock==1 ){ - iPg = pFS->nMetasize * 2 + 4; - }else{ - iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4; - } - }else{ - const i64 nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - if( iBlock==1 ){ - iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize); - }else{ - iPg = 1 + (iBlock-1) * nPagePerBlock; - } - } - return iPg; -} - -/* -** Return the page number of the last page on block iBlock. Blocks are -** numbered starting from 1. -** -** For a compressed database, page numbers are byte offsets. The first -** page on each block is the byte offset of the byte immediately before -** the 4-byte "next block" pointer at the end of each block. -*/ -static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){ - if( pFS->pCompress ){ - return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4; - }else{ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - return iBlock * nPagePerBlock; - } -} - -/* -** Return the block number of the block that page iPg is located on. -** Blocks are numbered starting from 1. -*/ -static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){ - if( pFS->pCompress ){ - return (int)((iPg / pFS->nBlocksize) + 1); - }else{ - return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize))); - } -} - -/* -** Return true if page iPg is the last page on its block. -** -** This function is only called in non-compressed database mode. -*/ -static int fsIsLast(FileSystem *pFS, LsmPgno iPg){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - assert( !pFS->pCompress ); - return ( iPg && (iPg % nPagePerBlock)==0 ); -} - -/* -** Return true if page iPg is the first page on its block. -** -** This function is only called in non-compressed database mode. -*/ -static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - assert( !pFS->pCompress ); - return ( (iPg % nPagePerBlock)==1 - || (iPgnData; - } - return pPage->aData; -} - -/* -** Return the page number of a page. -*/ -LsmPgno lsmFsPageNumber(Page *pPage){ - /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */ - return pPage ? pPage->iPg : 0; -} - -/* -** Page pPg is currently part of the LRU list belonging to pFS. Remove -** it from the list. pPg->pLruNext and pPg->pLruPrev are cleared by this -** operation. -*/ -static void fsPageRemoveFromLru(FileSystem *pFS, Page *pPg){ - assert( pPg->pLruNext || pPg==pFS->pLruLast ); - assert( pPg->pLruPrev || pPg==pFS->pLruFirst ); - if( pPg->pLruNext ){ - pPg->pLruNext->pLruPrev = pPg->pLruPrev; - }else{ - pFS->pLruLast = pPg->pLruPrev; - } - if( pPg->pLruPrev ){ - pPg->pLruPrev->pLruNext = pPg->pLruNext; - }else{ - pFS->pLruFirst = pPg->pLruNext; - } - pPg->pLruPrev = 0; - pPg->pLruNext = 0; -} - -/* -** Page pPg is not currently part of the LRU list belonging to pFS. Add it. -*/ -static void fsPageAddToLru(FileSystem *pFS, Page *pPg){ - assert( pPg->pLruNext==0 && pPg->pLruPrev==0 ); - pPg->pLruPrev = pFS->pLruLast; - if( pPg->pLruPrev ){ - pPg->pLruPrev->pLruNext = pPg; - }else{ - pFS->pLruFirst = pPg; - } - pFS->pLruLast = pPg; -} - -/* -** Page pPg is currently stored in the apHash/nHash hash table. Remove it. -*/ -static void fsPageRemoveFromHash(FileSystem *pFS, Page *pPg){ - int iHash; - Page **pp; - - iHash = fsHashKey(pFS->nHash, pPg->iPg); - for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext); - *pp = pPg->pHashNext; - pPg->pHashNext = 0; -} - -/* -** Free a Page object allocated by fsPageBuffer(). -*/ -static void fsPageBufferFree(Page *pPg){ - pPg->pFS->nCacheAlloc--; - lsmFree(pPg->pFS->pEnv, pPg->aData); - lsmFree(pPg->pFS->pEnv, pPg); -} - - -/* -** Purge the cache of all non-mmap pages with nRef==0. -*/ -void lsmFsPurgeCache(FileSystem *pFS){ - Page *pPg; - - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - assert( pPg->flags & PAGE_FREE ); - fsPageRemoveFromHash(pFS, pPg); - fsPageBufferFree(pPg); - pPg = pNext; - } - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - - assert( pFS->nCacheAlloc<=pFS->nOut && pFS->nCacheAlloc>=0 ); -} - -/* -** Search the hash-table for page iPg. If an entry is round, return a pointer -** to it. Otherwise, return NULL. -** -** Either way, if argument piHash is not NULL set *piHash to the hash slot -** number that page iPg would be stored in before returning. -*/ -static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){ - Page *p; /* Return value */ - int iHash = fsHashKey(pFS->nHash, iPg); - - if( piHash ) *piHash = iHash; - for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ - if( p->iPg==iPg) break; - } - return p; -} - -/* -** Allocate and return a non-mmap Page object. If there are already -** nCacheMax such Page objects outstanding, try to recycle an existing -** Page instead. -*/ -static int fsPageBuffer( - FileSystem *pFS, - Page **ppOut -){ - int rc = LSM_OK; - Page *pPage = 0; - if( pFS->pLruFirst==0 || pFS->nCacheAllocnCacheMax ){ - /* Allocate a new Page object */ - pPage = lsmMallocZero(pFS->pEnv, sizeof(Page)); - if( !pPage ){ - rc = LSM_NOMEM_BKPT; - }else{ - pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize); - if( !pPage->aData ){ - lsmFree(pFS->pEnv, pPage); - rc = LSM_NOMEM_BKPT; - pPage = 0; - }else{ - pFS->nCacheAlloc++; - } - } - }else{ - /* Reuse an existing Page object */ - u8 *aData; - pPage = pFS->pLruFirst; - aData = pPage->aData; - fsPageRemoveFromLru(pFS, pPage); - fsPageRemoveFromHash(pFS, pPage); - - memset(pPage, 0, sizeof(Page)); - pPage->aData = aData; - } - - if( pPage ){ - pPage->flags = PAGE_FREE; - } - *ppOut = pPage; - return rc; -} - -/* -** Assuming *pRc is initially LSM_OK, attempt to ensure that the -** memory-mapped region is at least iSz bytes in size. If it is not already, -** iSz bytes in size, extend it and update the pointers associated with any -** outstanding Page objects. -** -** If *pRc is not LSM_OK when this function is called, it is a no-op. -** Otherwise, *pRc is set to an lsm error code if an error occurs, or -** left unmodified otherwise. -** -** This function is never called in compressed database mode. -*/ -static void fsGrowMapping( - FileSystem *pFS, /* File system object */ - i64 iSz, /* Minimum size to extend mapping to */ - int *pRc /* IN/OUT: Error code */ -){ - assert( PAGE_HASPREV==4 ); - - if( *pRc==LSM_OK && iSz>pFS->nMap ){ - int rc; - u8 *aOld = pFS->pMap; - rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, iSz, &pFS->pMap, &pFS->nMap); - if( rc==LSM_OK && pFS->pMap!=aOld ){ - Page *pFix; - i64 iOff = (u8 *)pFS->pMap - aOld; - for(pFix=pFS->pMapped; pFix; pFix=pFix->pMappedNext){ - pFix->aData += iOff; - } - lsmSortedRemap(pFS->pDb); - } - *pRc = rc; - } -} - -/* -** If it is mapped, unmap the database file. -*/ -int lsmFsUnmap(FileSystem *pFS){ - int rc = LSM_OK; - if( pFS ){ - rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - } - return rc; -} - -/* -** fsync() the database file. -*/ -int lsmFsSyncDb(FileSystem *pFS, int nBlock){ - return lsmEnvSync(pFS->pEnv, pFS->fdDb); -} - -/* -** If block iBlk has been redirected according to the redirections in the -** object passed as the first argument, return the destination block to -** which it is redirected. Otherwise, return a copy of iBlk. -*/ -static int fsRedirectBlock(Redirect *p, int iBlk){ - if( p ){ - int i; - for(i=0; in; i++){ - if( iBlk==p->a[i].iFrom ) return p->a[i].iTo; - } - } - assert( iBlk!=0 ); - return iBlk; -} - -/* -** If page iPg has been redirected according to the redirections in the -** object passed as the second argument, return the destination page to -** which it is redirected. Otherwise, return a copy of iPg. -*/ -LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){ - LsmPgno iReal = iPg; - - if( pRedir ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - int iBlk = fsPageToBlock(pFS, iPg); - int i; - for(i=0; in; i++){ - int iFrom = pRedir->a[i].iFrom; - if( iFrom>iBlk ) break; - if( iFrom==iBlk ){ - int iTo = pRedir->a[i].iTo; - iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; - if( iTo==1 ){ - iReal += (fsFirstPageOnBlock(pFS, 1)-1); - } - break; - } - } - } - - assert( iReal!=0 ); - return iReal; -} - -/* Required by the circular fsBlockNext<->fsPageGet dependency. */ -static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *); - -/* -** Parameter iBlock is a database file block. This function reads the value -** stored in the blocks "next block" pointer and stores it in *piNext. -** LSM_OK is returned if everything is successful, or an LSM error code -** otherwise. -*/ -static int fsBlockNext( - FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ - int iBlock, /* Read field from this block */ - int *piNext /* OUT: Next block in linked list */ -){ - int rc; - int iRead; /* Read block from here */ - - if( pSeg ){ - iRead = fsRedirectBlock(pSeg->pRedirect, iBlock); - }else{ - iRead = iBlock; - } - - assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); - if( pFS->pCompress ){ - i64 iOff; /* File offset to read data from */ - u8 aNext[4]; /* 4-byte pointer read from db file */ - - iOff = (i64)iRead * pFS->nBlocksize - sizeof(aNext); - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aNext, sizeof(aNext)); - if( rc==LSM_OK ){ - *piNext = (int)lsmGetU32(aNext); - } - }else{ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - Page *pLast; - rc = fsPageGet(pFS, 0, iRead*nPagePerBlock, 0, &pLast, 0); - if( rc==LSM_OK ){ - *piNext = lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmFsPageRelease(pLast); - } - } - - if( pSeg ){ - *piNext = fsRedirectBlock(pSeg->pRedirect, *piNext); - } - return rc; -} - -/* -** Return the page number of the last page on the same block as page iPg. -*/ -LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){ - return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg)); -} - -/* -** Read nData bytes of data from offset iOff of the database file into -** buffer aData. If this means reading past the end of a block, follow -** the block pointer to the next block and continue reading. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** This function is only called in compressed database mode. -*/ -static int fsReadData( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection */ - i64 iOff, /* Read data from this offset */ - u8 *aData, /* Buffer to read data into */ - int nData /* Number of bytes to read */ -){ - i64 iEob; /* End of block */ - int nRead; - int rc; - - assert( pFS->pCompress ); - - iEob = fsLastPageOnPagesBlock(pFS, iOff) + 1; - nRead = (int)LSM_MIN(iEob - iOff, nData); - - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nRead); - if( rc==LSM_OK && nRead!=nData ){ - int iBlk; - - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - if( rc==LSM_OK ){ - i64 iOff2 = fsFirstPageOnBlock(pFS, iBlk); - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff2, &aData[nRead], nData-nRead); - } - } - - return rc; -} - -/* -** Parameter iBlock is a database file block. This function reads the value -** stored in the blocks "previous block" pointer and stores it in *piPrev. -** LSM_OK is returned if everything is successful, or an LSM error code -** otherwise. -*/ -static int fsBlockPrev( - FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ - int iBlock, /* Read field from this block */ - int *piPrev /* OUT: Previous block in linked list */ -){ - int rc = LSM_OK; /* Return code */ - - assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); - assert( iBlock>0 ); - - if( pFS->pCompress ){ - i64 iOff = fsFirstPageOnBlock(pFS, iBlock) - 4; - u8 aPrev[4]; /* 4-byte pointer read from db file */ - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aPrev, sizeof(aPrev)); - if( rc==LSM_OK ){ - Redirect *pRedir = (pSeg ? pSeg->pRedirect : 0); - *piPrev = fsRedirectBlock(pRedir, (int)lsmGetU32(aPrev)); - } - }else{ - assert( 0 ); - } - return rc; -} - -/* -** Encode and decode routines for record size fields. -*/ -static void putRecordSize(u8 *aBuf, int nByte, int bFree){ - aBuf[0] = (u8)(nByte >> 14) | 0x80; - aBuf[1] = ((u8)(nByte >> 7) & 0x7F) | (bFree ? 0x00 : 0x80); - aBuf[2] = (u8)nByte | 0x80; -} -static int getRecordSize(u8 *aBuf, int *pbFree){ - int nByte; - nByte = (aBuf[0] & 0x7F) << 14; - nByte += (aBuf[1] & 0x7F) << 7; - nByte += (aBuf[2] & 0x7F); - *pbFree = !(aBuf[1] & 0x80); - return nByte; -} - -/* -** Subtract iSub from database file offset iOff and set *piRes to the -** result. If doing so means passing the start of a block, follow the -** block pointer stored in the first 4 bytes of the block. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -static int fsSubtractOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iSub, - i64 *piRes -){ - i64 iStart; - int iBlk = 0; - int rc; - - assert( pFS->pCompress ); - - iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); - if( (iOff-iSub)>=iStart ){ - *piRes = (iOff-iSub); - return LSM_OK; - } - - rc = fsBlockPrev(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - *piRes = fsLastPageOnBlock(pFS, iBlk) - iSub + (iOff - iStart + 1); - return rc; -} - -/* -** Add iAdd to database file offset iOff and set *piRes to the -** result. If doing so means passing the end of a block, follow the -** block pointer stored in the last 4 bytes of the block. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -static int fsAddOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iAdd, - i64 *piRes -){ - i64 iEob; - int iBlk; - int rc; - - assert( pFS->pCompress ); - - iEob = fsLastPageOnPagesBlock(pFS, iOff); - if( (iOff+iAdd)<=iEob ){ - *piRes = (iOff+iAdd); - return LSM_OK; - } - - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); - return rc; -} - -/* -** If it is not already allocated, allocate either the FileSystem.aOBuffer (if -** bWrite is true) or the FileSystem.aIBuffer (if bWrite is false). Return -** LSM_OK if successful if the attempt to allocate memory fails. -*/ -static int fsAllocateBuffer(FileSystem *pFS, int bWrite){ - u8 **pp; /* Pointer to either aIBuffer or aOBuffer */ - - assert( pFS->pCompress ); - - /* If neither buffer has been allocated, figure out how large they - ** should be. Store this value in FileSystem.nBuffer. */ - if( pFS->nBuffer==0 ){ - assert( pFS->aIBuffer==0 && pFS->aOBuffer==0 ); - pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize); - if( pFS->nBuffer<(pFS->szSector+6) ){ - pFS->nBuffer = pFS->szSector+6; - } - } - - pp = (bWrite ? &pFS->aOBuffer : &pFS->aIBuffer); - if( *pp==0 ){ - *pp = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); - if( *pp==0 ) return LSM_NOMEM_BKPT; - } - - return LSM_OK; -} - -/* -** This function is only called in compressed database mode. It reads and -** uncompresses the compressed data for page pPg from the database and -** populates the pPg->aData[] buffer and pPg->nCompress field. -** -** It is possible that instead of a page record, there is free space -** at offset pPg->iPgno. In this case no data is read from the file, but -** output variable *pnSpace is set to the total number of free bytes. -** -** LSM_OK is returned if successful, or an LSM error code otherwise. -*/ -static int fsReadPagedata( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* pPg is part of this segment */ - Page *pPg, /* Page to read and uncompress data for */ - int *pnSpace /* OUT: Total bytes of free space */ -){ - lsm_compress *p = pFS->pCompress; - i64 iOff = pPg->iPg; - u8 aSz[3]; - int rc; - - assert( p && pPg->nCompress==0 ); - - if( fsAllocateBuffer(pFS, 0) ) return LSM_NOMEM; - - rc = fsReadData(pFS, pSeg, iOff, aSz, sizeof(aSz)); - - if( rc==LSM_OK ){ - int bFree; - if( aSz[0] & 0x80 ){ - pPg->nCompress = (int)getRecordSize(aSz, &bFree); - }else{ - pPg->nCompress = (int)aSz[0] - sizeof(aSz)*2; - bFree = 1; - } - if( bFree ){ - if( pnSpace ){ - *pnSpace = pPg->nCompress + sizeof(aSz)*2; - }else{ - rc = LSM_CORRUPT_BKPT; - } - }else{ - rc = fsAddOffset(pFS, pSeg, iOff, 3, &iOff); - if( rc==LSM_OK ){ - if( pPg->nCompress>pFS->nBuffer ){ - rc = LSM_CORRUPT_BKPT; - }else{ - rc = fsReadData(pFS, pSeg, iOff, pFS->aIBuffer, pPg->nCompress); - } - if( rc==LSM_OK ){ - int n = pFS->nPagesize; - rc = p->xUncompress(p->pCtx, - (char *)pPg->aData, &n, - (const char *)pFS->aIBuffer, pPg->nCompress - ); - if( rc==LSM_OK && n!=pPg->pFS->nPagesize ){ - rc = LSM_CORRUPT_BKPT; - } - } - } - } - } - return rc; -} - -/* -** Return a handle for a database page. -** -** If this file-system object is accessing a compressed database it may be -** that there is no page record at database file offset iPg. Instead, there -** may be a free space record. In this case, set *ppPg to NULL and *pnSpace -** to the total number of free bytes before returning. -** -** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code. -*/ -static int fsPageGet( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection to use (or NULL) */ - LsmPgno iPg, /* Page id */ - int noContent, /* True to not load content from disk */ - Page **ppPg, /* OUT: New page handle */ - int *pnSpace /* OUT: Bytes of free space */ -){ - Page *p; - int iHash; - int rc = LSM_OK; - - /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is - ** not NULL, and the block containing iPg has been redirected, then iReal - ** is the page number after redirection. */ - LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); - - assert_lists_are_ok(pFS); - assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); - assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); - *ppPg = 0; - - /* Search the hash-table for the page */ - p = fsPageFindInHash(pFS, iReal, &iHash); - - if( p ){ - assert( p->flags & PAGE_FREE ); - if( p->nRef==0 ) fsPageRemoveFromLru(pFS, p); - }else{ - - if( fsMmapPage(pFS, iReal) ){ - i64 iEnd = (i64)iReal * pFS->nPagesize; - fsGrowMapping(pFS, iEnd, &rc); - if( rc!=LSM_OK ) return rc; - - if( pFS->pFree ){ - p = pFS->pFree; - pFS->pFree = p->pFreeNext; - assert( p->nRef==0 ); - }else{ - p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); - if( rc ) return rc; - p->pFS = pFS; - } - p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (iReal-1)]; - p->iPg = iReal; - - /* This page now carries a pointer to the mapping. Link it in to - ** the FileSystem.pMapped list. */ - assert( p->pMappedNext==0 ); - p->pMappedNext = pFS->pMapped; - pFS->pMapped = p; - - assert( pFS->pCompress==0 ); - assert( (p->flags & PAGE_FREE)==0 ); - }else{ - rc = fsPageBuffer(pFS, &p); - if( rc==LSM_OK ){ - int nSpace = 0; - p->iPg = iReal; - p->nRef = 0; - p->pFS = pFS; - assert( p->flags==0 || p->flags==PAGE_FREE ); - -#ifdef LSM_DEBUG - memset(p->aData, 0x56, pFS->nPagesize); -#endif - assert( p->pLruNext==0 && p->pLruPrev==0 ); - if( noContent==0 ){ - if( pFS->pCompress ){ - rc = fsReadPagedata(pFS, pSeg, p, &nSpace); - }else{ - int nByte = pFS->nPagesize; - i64 iOff = (i64)(iReal-1) * pFS->nPagesize; - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, p->aData, nByte); - } - pFS->nRead++; - } - - /* If the xRead() call was successful (or not attempted), link the - ** page into the page-cache hash-table. Otherwise, if it failed, - ** free the buffer. */ - if( rc==LSM_OK && nSpace==0 ){ - p->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = p; - }else{ - fsPageBufferFree(p); - p = 0; - if( pnSpace ) *pnSpace = nSpace; - } - } - } - - assert( (rc==LSM_OK && (p || (pnSpace && *pnSpace))) - || (rc!=LSM_OK && p==0) - ); - } - - if( rc==LSM_OK && p ){ - if( pFS->pCompress==0 && (fsIsLast(pFS, iReal) || fsIsFirst(pFS, iReal)) ){ - p->nData = pFS->nPagesize - 4; - if( fsIsFirst(pFS, iReal) && p->nRef==0 ){ - p->aData += 4; - p->flags |= PAGE_HASPREV; - } - }else{ - p->nData = pFS->nPagesize; - } - pFS->nOut += (p->nRef==0); - p->nRef++; - } - *ppPg = p; - return rc; -} - -/* -** Read the 64-bit checkpoint id of the checkpoint currently stored on meta -** page iMeta of the database file. If no error occurs, store the id value -** in *piVal and return LSM_OK. Otherwise, return an LSM error code and leave -** *piVal unmodified. -** -** If a checkpointer connection is currently updating meta-page iMeta, or an -** earlier checkpointer crashed while doing so, the value read into *piVal -** may be garbage. It is the callers responsibility to deal with this. -*/ -int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){ - FileSystem *pFS = db->pFS; - int rc = LSM_OK; - - assert( iMeta==1 || iMeta==2 ); - if( pFS->nMapLimit>0 ){ - fsGrowMapping(pFS, iMeta*LSM_META_PAGE_SIZE, &rc); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(&((u8 *)pFS->pMap)[(iMeta-1)*LSM_META_PAGE_SIZE]); - } - }else{ - MetaPage *pMeta = 0; - rc = lsmFsMetaPageGet(pFS, 0, iMeta, &pMeta); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(pMeta->aData); - lsmFsMetaPageRelease(pMeta); - } - } - - return rc; -} - - -/* -** Return true if the first or last page of segment pRun falls between iFirst -** and iLast, inclusive, and pRun is not equal to pIgnore. -*/ -static int fsRunEndsBetween( - Segment *pRun, - Segment *pIgnore, - LsmPgno iFirst, - LsmPgno iLast -){ - return (pRun!=pIgnore && ( - (pRun->iFirst>=iFirst && pRun->iFirst<=iLast) - || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast) - )); -} - -/* -** Return true if level pLevel contains a segment other than pIgnore for -** which the first or last page is between iFirst and iLast, inclusive. -*/ -static int fsLevelEndsBetween( - Level *pLevel, - Segment *pIgnore, - LsmPgno iFirst, - LsmPgno iLast -){ - int i; - - if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){ - return 1; - } - for(i=0; inRight; i++){ - if( fsRunEndsBetween(&pLevel->aRhs[i], pIgnore, iFirst, iLast) ){ - return 1; - } - } - - return 0; -} - -/* -** Block iBlk is no longer in use by segment pIgnore. If it is not in use -** by any other segment, move it to the free block list. -*/ -static int fsFreeBlock( - FileSystem *pFS, /* File system object */ - Snapshot *pSnapshot, /* Worker snapshot */ - Segment *pIgnore, /* Ignore this run when searching */ - int iBlk /* Block number of block to free */ -){ - int rc = LSM_OK; /* Return code */ - LsmPgno iFirst; /* First page on block iBlk */ - LsmPgno iLast; /* Last page on block iBlk */ - Level *pLevel; /* Used to iterate through levels */ - - int iIn; /* Used to iterate through append points */ - int iOut = 0; /* Used to output append points */ - LsmPgno *aApp = pSnapshot->aiAppend; - - iFirst = fsFirstPageOnBlock(pFS, iBlk); - iLast = fsLastPageOnBlock(pFS, iBlk); - - /* Check if any other run in the snapshot has a start or end page - ** within this block. If there is such a run, return early. */ - for(pLevel=lsmDbSnapshotLevel(pSnapshot); pLevel; pLevel=pLevel->pNext){ - if( fsLevelEndsBetween(pLevel, pIgnore, iFirst, iLast) ){ - return LSM_OK; - } - } - - /* Remove any entries that lie on this block from the append-list. */ - for(iIn=0; iIniLast ){ - aApp[iOut++] = aApp[iIn]; - } - } - while( iOutpDb, iBlk); - } - return rc; -} - -/* -** Delete or otherwise recycle the blocks currently occupied by run pDel. -*/ -int lsmFsSortedDelete( - FileSystem *pFS, - Snapshot *pSnapshot, - int bZero, /* True to zero the Segment structure */ - Segment *pDel -){ - if( pDel->iFirst ){ - int rc = LSM_OK; - - int iBlk; - int iLastBlk; - - iBlk = fsPageToBlock(pFS, pDel->iFirst); - iLastBlk = fsPageToBlock(pFS, pDel->iLastPg); - - /* Mark all blocks currently used by this sorted run as free */ - while( iBlk && rc==LSM_OK ){ - int iNext = 0; - if( iBlk!=iLastBlk ){ - rc = fsBlockNext(pFS, pDel, iBlk, &iNext); - }else if( bZero==0 && pDel->iLastPg!=fsLastPageOnBlock(pFS, iLastBlk) ){ - break; - } - rc = fsFreeBlock(pFS, pSnapshot, pDel, iBlk); - iBlk = iNext; - } - - if( pDel->pRedirect ){ - assert( pDel->pRedirect==&pSnapshot->redirect ); - pSnapshot->redirect.n = 0; - } - - if( bZero ) memset(pDel, 0, sizeof(Segment)); - } - return LSM_OK; -} - -/* -** aPgno is an array containing nPgno page numbers. Return the smallest page -** number from the array that falls on block iBlk. Or, if none of the pages -** in aPgno[] fall on block iBlk, return 0. -*/ -static LsmPgno firstOnBlock( - FileSystem *pFS, - int iBlk, - LsmPgno *aPgno, - int nPgno -){ - LsmPgno iRet = 0; - int i; - for(i=0; ipRedirect, iPg)); -} - -/* -** Return true if the second argument is not NULL and any of the first -** last or root pages lie on a redirected block. -*/ -static int fsSegmentRedirects(FileSystem *pFS, Segment *p){ - return (p && ( - fsPageRedirects(pFS, p, p->iFirst) - || fsPageRedirects(pFS, p, p->iRoot) - || fsPageRedirects(pFS, p, p->iLastPg) - )); -} -#endif - -/* -** Argument aPgno is an array of nPgno page numbers. All pages belong to -** the segment pRun. This function gobbles from the start of the run to the -** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is -** the new first page of the run). -*/ -void lsmFsGobble( - lsm_db *pDb, - Segment *pRun, - LsmPgno *aPgno, - int nPgno -){ - int rc = LSM_OK; - FileSystem *pFS = pDb->pFS; - Snapshot *pSnapshot = pDb->pWorker; - int iBlk; - - assert( pRun->nSize>0 ); - assert( 0==fsSegmentRedirects(pFS, pRun) ); - assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) ); - - iBlk = fsPageToBlock(pFS, pRun->iFirst); - pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); - - while( rc==LSM_OK ){ - int iNext = 0; - LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); - if( iFirst ){ - pRun->iFirst = iFirst; - break; - } - rc = fsBlockNext(pFS, pRun, iBlk, &iNext); - if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk); - pRun->nSize -= ( - 1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk) - ); - iBlk = iNext; - } - - pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); - assert( pRun->nSize>0 ); -} - -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number (byte offset) of a page within segment -** pSeg. The page record, including all headers, is nByte bytes in size. -** Before returning, set *piNext to the page number of the next page in -** the segment, or to zero if iPg is the last. -** -** In other words, do: -** -** *piNext = iPg + nByte; -** -** But take block overflow and redirection into account. -*/ -static int fsNextPageOffset( - FileSystem *pFS, /* File system object */ - Segment *pSeg, /* Segment to move within */ - LsmPgno iPg, /* Offset of current page */ - int nByte, /* Size of current page including headers */ - LsmPgno *piNext /* OUT: Offset of next page. Or zero (EOF) */ -){ - LsmPgno iNext; - int rc; - - assert( pFS->pCompress ); - - rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); - if( pSeg && iNext==pSeg->iLastPg ){ - iNext = 0; - }else if( rc==LSM_OK ){ - rc = fsAddOffset(pFS, pSeg, iNext, 1, &iNext); - } - - *piNext = iNext; - return rc; -} - -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number of a pagethat appears in segment pSeg. -** This function determines the page number of the previous page in the -** same run. *piPrev is set to the previous page number before returning. -** -** LSM_OK is returned if no error occurs. Otherwise, an lsm error code. -** If any value other than LSM_OK is returned, then the final value of -** *piPrev is undefined. -*/ -static int fsGetPageBefore( - FileSystem *pFS, - Segment *pSeg, - LsmPgno iPg, - LsmPgno *piPrev -){ - u8 aSz[3]; - int rc; - i64 iRead; - - assert( pFS->pCompress ); - - rc = fsSubtractOffset(pFS, pSeg, iPg, sizeof(aSz), &iRead); - if( rc==LSM_OK ) rc = fsReadData(pFS, pSeg, iRead, aSz, sizeof(aSz)); - - if( rc==LSM_OK ){ - int bFree; - int nSz; - if( aSz[2] & 0x80 ){ - nSz = getRecordSize(aSz, &bFree) + sizeof(aSz)*2; - }else{ - nSz = (int)(aSz[2] & 0x7F); - bFree = 1; - } - rc = fsSubtractOffset(pFS, pSeg, iPg, nSz, piPrev); - } - - return rc; -} - -/* -** The first argument to this function is a valid reference to a database -** file page that is part of a sorted run. If parameter eDir is -1, this -** function attempts to locate and load the previous page in the same run. -** Or, if eDir is +1, it attempts to find the next page in the same run. -** The results of passing an eDir value other than positive or negative one -** are undefined. -** -** If parameter pRun is not NULL then it must point to the run that page -** pPg belongs to. In this case, if pPg is the first or last page of the -** run, and the request is for the previous or next page, respectively, -** *ppNext is set to NULL before returning LSM_OK. If pRun is NULL, then it -** is assumed that the next or previous page, as requested, exists. -** -** If the previous/next page does exist and is successfully loaded, *ppNext -** is set to point to it and LSM_OK is returned. Otherwise, if an error -** occurs, *ppNext is set to NULL and and lsm error code returned. -** -** Page references returned by this function should be released by the -** caller using lsmFsPageRelease(). -*/ -int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ - int rc = LSM_OK; - FileSystem *pFS = pPg->pFS; - LsmPgno iPg = pPg->iPg; - - assert( 0==fsSegmentRedirects(pFS, pRun) ); - if( pFS->pCompress ){ - int nSpace = pPg->nCompress + 2*3; - - do { - if( eDir>0 ){ - rc = fsNextPageOffset(pFS, pRun, iPg, nSpace, &iPg); - }else{ - if( iPg==pRun->iFirst ){ - iPg = 0; - }else{ - rc = fsGetPageBefore(pFS, pRun, iPg, &iPg); - } - } - - nSpace = 0; - if( iPg!=0 ){ - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, &nSpace); - assert( (*ppNext==0)==(rc!=LSM_OK || nSpace>0) ); - }else{ - *ppNext = 0; - } - }while( nSpace>0 && rc==LSM_OK ); - - }else{ - Redirect *pRedir = pRun ? pRun->pRedirect : 0; - assert( eDir==1 || eDir==-1 ); - if( eDir<0 ){ - if( pRun && iPg==pRun->iFirst ){ - *ppNext = 0; - return LSM_OK; - }else if( fsIsFirst(pFS, iPg) ){ - assert( pPg->flags & PAGE_HASPREV ); - iPg = fsLastPageOnBlock(pFS, lsmGetU32(&pPg->aData[-4])); - }else{ - iPg--; - } - }else{ - if( pRun ){ - if( iPg==pRun->iLastPg ){ - *ppNext = 0; - return LSM_OK; - } - } - - if( fsIsLast(pFS, iPg) ){ - int iBlk = fsRedirectBlock( - pRedir, lsmGetU32(&pPg->aData[pFS->nPagesize-4]) - ); - iPg = fsFirstPageOnBlock(pFS, iBlk); - }else{ - iPg++; - } - } - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, 0); - } - - return rc; -} - -/* -** This function is called when creating a new segment to determine if the -** first part of it can be written following an existing segment on an -** already allocated block. If it is possible, the page number of the first -** page to use for the new segment is returned. Otherwise zero. -** -** If argument pLvl is not NULL, then this function will not attempt to -** start the new segment immediately following any segment that is part -** of the right-hand-side of pLvl. -*/ -static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){ - int i; - LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; - LsmPgno iRet = 0; - - for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ - if( (iRet = aiAppend[i]) ){ - if( pLvl ){ - int iBlk = fsPageToBlock(pFS, iRet); - int j; - for(j=0; iRet && jnRight; j++){ - if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ - iRet = 0; - } - } - } - if( iRet ) aiAppend[i] = 0; - } - } - return iRet; -} - -/* -** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and -** return a pointer to it. The page is writable until either -** lsmFsPagePersist() is called on it or the ref-count drops to zero. -*/ -int lsmFsSortedAppend( - FileSystem *pFS, - Snapshot *pSnapshot, - Level *pLvl, - int bDefer, - Page **ppOut -){ - int rc = LSM_OK; - Page *pPg = 0; - LsmPgno iApp = 0; - LsmPgno iNext = 0; - Segment *p = &pLvl->lhs; - LsmPgno iPrev = p->iLastPg; - - *ppOut = 0; - assert( p->pRedirect==0 ); - - if( pFS->pCompress || bDefer ){ - /* In compressed database mode the page is not assigned a page number - ** or location in the database file at this point. This will be done - ** by the lsmFsPagePersist() call. */ - rc = fsPageBuffer(pFS, &pPg); - if( rc==LSM_OK ){ - pPg->pFS = pFS; - pPg->pSeg = p; - pPg->iPg = 0; - pPg->flags |= PAGE_DIRTY; - pPg->nData = pFS->nPagesize; - assert( pPg->aData ); - if( pFS->pCompress==0 ) pPg->nData -= 4; - - pPg->nRef = 1; - pFS->nOut++; - } - }else{ - if( iPrev==0 ){ - iApp = findAppendPoint(pFS, pLvl); - }else if( fsIsLast(pFS, iPrev) ){ - int iNext2; - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iPrev), &iNext2); - if( rc!=LSM_OK ) return rc; - iApp = fsFirstPageOnBlock(pFS, iNext2); - }else{ - iApp = iPrev + 1; - } - - /* If this is the first page allocated, or if the page allocated is the - ** last in the block, also allocate the next block here. */ - if( iApp==0 || fsIsLast(pFS, iApp) ){ - int iNew; /* New block number */ - - rc = lsmBlockAllocate(pFS->pDb, 0, &iNew); - if( rc!=LSM_OK ) return rc; - if( iApp==0 ){ - iApp = fsFirstPageOnBlock(pFS, iNew); - }else{ - iNext = fsFirstPageOnBlock(pFS, iNew); - } - } - - /* Grab the new page. */ - pPg = 0; - rc = fsPageGet(pFS, 0, iApp, 1, &pPg, 0); - assert( rc==LSM_OK || pPg==0 ); - - /* If this is the first or last page of a block, fill in the pointer - ** value at the end of the new page. */ - if( rc==LSM_OK ){ - p->nSize++; - p->iLastPg = iApp; - if( p->iFirst==0 ) p->iFirst = iApp; - pPg->flags |= PAGE_DIRTY; - - if( fsIsLast(pFS, iApp) ){ - lsmPutU32(&pPg->aData[pFS->nPagesize-4], fsPageToBlock(pFS, iNext)); - }else if( fsIsFirst(pFS, iApp) ){ - lsmPutU32(&pPg->aData[-4], fsPageToBlock(pFS, iPrev)); - } - } - } - - *ppOut = pPg; - return rc; -} - -/* -** Mark the segment passed as the second argument as finished. Once a segment -** is marked as finished it is not possible to append any further pages to -** it. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ - int rc = LSM_OK; - if( p && p->iLastPg ){ - assert( p->pRedirect==0 ); - - /* Check if the last page of this run happens to be the last of a block. - ** If it is, then an extra block has already been allocated for this run. - ** Shift this extra block back to the free-block list. - ** - ** Otherwise, add the first free page in the last block used by the run - ** to the lAppend list. - */ - if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ - int i; - LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; - for(i=0; iiLastPg+1; - break; - } - } - }else if( pFS->pCompress==0 ){ - Page *pLast; - rc = fsPageGet(pFS, 0, p->iLastPg, 0, &pLast, 0); - if( rc==LSM_OK ){ - int iBlk = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmBlockRefree(pFS->pDb, iBlk); - lsmFsPageRelease(pLast); - } - }else{ - int iBlk = 0; - rc = fsBlockNext(pFS, p, fsPageToBlock(pFS, p->iLastPg), &iBlk); - if( rc==LSM_OK ){ - lsmBlockRefree(pFS->pDb, iBlk); - } - } - } - return rc; -} - -/* -** Obtain a reference to page number iPg. -** -** Return LSM_OK if successful, or an lsm error code if an error occurs. -*/ -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){ - return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); -} - -/* -** Obtain a reference to the last page in the segment passed as the -** second argument. -** -** Return LSM_OK if successful, or an lsm error code if an error occurs. -*/ -int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){ - int rc; - LsmPgno iPg = pSeg->iLastPg; - if( pFS->pCompress ){ - int nSpace; - iPg++; - do { - nSpace = 0; - rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg); - if( rc==LSM_OK ){ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, &nSpace); - } - }while( rc==LSM_OK && nSpace>0 ); - - }else{ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); - } - return rc; -} - -/* -** Return a reference to meta-page iPg. If successful, LSM_OK is returned -** and *ppPg populated with the new page reference. The reference should -** be released by the caller using lsmFsPageRelease(). -** -** Otherwise, if an error occurs, *ppPg is set to NULL and an LSM error -** code is returned. -*/ -int lsmFsMetaPageGet( - FileSystem *pFS, /* File-system connection */ - int bWrite, /* True for write access, false for read */ - int iPg, /* Either 1 or 2 */ - MetaPage **ppPg /* OUT: Pointer to MetaPage object */ -){ - int rc = LSM_OK; - MetaPage *pPg; - assert( iPg==1 || iPg==2 ); - - pPg = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); - - if( pPg ){ - i64 iOff = (iPg-1) * pFS->nMetasize; - if( pFS->nMapLimit>0 ){ - fsGrowMapping(pFS, 2*pFS->nMetasize, &rc); - pPg->aData = (u8 *)(pFS->pMap) + iOff; - }else{ - pPg->aData = lsmMallocRc(pFS->pEnv, pFS->nMetasize, &rc); - if( rc==LSM_OK && bWrite==0 ){ - rc = lsmEnvRead( - pFS->pEnv, pFS->fdDb, iOff, pPg->aData, pFS->nMetaRwSize - ); - } -#ifndef NDEBUG - /* pPg->aData causes an uninitialized access via a downstreadm write(). - After discussion on this list, this memory should not, for performance - reasons, be memset. However, tracking down "real" misuse is more - difficult with this "false" positive, so it is set when NDEBUG. - */ - else if( rc==LSM_OK ){ - memset( pPg->aData, 0x77, pFS->nMetasize ); - } -#endif - } - - if( rc!=LSM_OK ){ - if( pFS->nMapLimit==0 ) lsmFree(pFS->pEnv, pPg->aData); - lsmFree(pFS->pEnv, pPg); - pPg = 0; - }else{ - pPg->iPg = iPg; - pPg->bWrite = bWrite; - pPg->pFS = pFS; - } - } - - *ppPg = pPg; - return rc; -} - -/* -** Release a meta-page reference obtained via a call to lsmFsMetaPageGet(). -*/ -int lsmFsMetaPageRelease(MetaPage *pPg){ - int rc = LSM_OK; - if( pPg ){ - FileSystem *pFS = pPg->pFS; - - if( pFS->nMapLimit==0 ){ - if( pPg->bWrite ){ - i64 iOff = (pPg->iPg==2 ? pFS->nMetasize : 0); - int nWrite = pFS->nMetaRwSize; - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, pPg->aData, nWrite); - } - lsmFree(pFS->pEnv, pPg->aData); - } - - lsmFree(pFS->pEnv, pPg); - } - return rc; -} - -/* -** Return a pointer to a buffer containing the data associated with the -** meta-page passed as the first argument. If parameter pnData is not NULL, -** set *pnData to the size of the meta-page in bytes before returning. -*/ -u8 *lsmFsMetaPageData(MetaPage *pPg, int *pnData){ - if( pnData ) *pnData = pPg->pFS->nMetaRwSize; - return pPg->aData; -} - -/* -** Return true if page is currently writable. This is used in assert() -** statements only. -*/ -#ifndef NDEBUG -int lsmFsPageWritable(Page *pPg){ - return (pPg->flags & PAGE_DIRTY) ? 1 : 0; -} -#endif - -/* -** This is called when block iFrom is being redirected to iTo. If page -** number (*piPg) lies on block iFrom, then calculate the equivalent -** page on block iTo and set *piPg to this value before returning. -*/ -static void fsMovePage( - FileSystem *pFS, /* File system object */ - int iTo, /* Destination block */ - int iFrom, /* Source block */ - LsmPgno *piPg /* IN/OUT: Page number */ -){ - LsmPgno iPg = *piPg; - if( iFrom==fsPageToBlock(pFS, iPg) ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; - } -} - -/* -** Copy the contents of block iFrom to block iTo. -** -** It is safe to assume that there are no outstanding references to pages -** on block iTo. And that block iFrom is not currently being written. In -** other words, the data can be read and written directly. -*/ -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){ - Snapshot *p = pFS->pDb->pWorker; - int rc = LSM_OK; - int i; - i64 nMap; - - i64 iFromOff = (i64)(iFrom-1) * pFS->nBlocksize; - i64 iToOff = (i64)(iTo-1) * pFS->nBlocksize; - - assert( iTo!=1 ); - assert( iFrom>iTo ); - - /* Grow the mapping as required. */ - nMap = LSM_MIN(pFS->nMapLimit, (i64)iFrom * pFS->nBlocksize); - fsGrowMapping(pFS, nMap, &rc); - - if( rc==LSM_OK ){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - int nSz = pFS->nPagesize; - u8 *aBuf = 0; - u8 *aData = 0; - - for(i=0; rc==LSM_OK && inMapLimit ){ - u8 *aMap = (u8 *)(pFS->pMap); - aData = &aMap[iOff]; - }else{ - if( aBuf==0 ){ - aBuf = (u8 *)lsmMallocRc(pFS->pEnv, nSz, &rc); - if( aBuf==0 ) break; - } - aData = aBuf; - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - - /* Copy aData to the to page */ - if( rc==LSM_OK ){ - iOff = iToOff + i*nSz; - if( (iOff+nSz)<=pFS->nMapLimit ){ - u8 *aMap = (u8 *)(pFS->pMap); - memcpy(&aMap[iOff], aData, nSz); - }else{ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - } - } - lsmFree(pFS->pEnv, aBuf); - lsmFsPurgeCache(pFS); - } - - /* Update append-point list if necessary */ - for(i=0; iaiAppend[i]); - } - - /* Update the Segment structure itself */ - fsMovePage(pFS, iTo, iFrom, &pSeg->iFirst); - fsMovePage(pFS, iTo, iFrom, &pSeg->iLastPg); - fsMovePage(pFS, iTo, iFrom, &pSeg->iRoot); - - return rc; -} - -/* -** Append raw data to a segment. Return the database file offset that the -** data is written to (this may be used as the page number if the data -** being appended is a new page record). -** -** This function is only used in compressed database mode. -*/ -static LsmPgno fsAppendData( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Segment to append to */ - const u8 *aData, /* Buffer containing data to write */ - int nData, /* Size of buffer aData[] in bytes */ - int *pRc /* IN/OUT: Error code */ -){ - LsmPgno iRet = 0; - int rc = *pRc; - assert( pFS->pCompress ); - if( rc==LSM_OK ){ - int nRem = 0; - int nWrite = 0; - LsmPgno iLastOnBlock; - LsmPgno iApp = pSeg->iLastPg+1; - - /* If this is the first data written into the segment, find an append-point - ** or allocate a new block. */ - if( iApp==1 ){ - pSeg->iFirst = iApp = findAppendPoint(pFS, 0); - if( iApp==0 ){ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); - } - } - iRet = iApp; - - /* Write as much data as is possible at iApp (usually all of it). */ - iLastOnBlock = fsLastPageOnPagesBlock(pFS, iApp); - if( rc==LSM_OK ){ - int nSpace = (int)(iLastOnBlock - iApp + 1); - nWrite = LSM_MIN(nData, nSpace); - nRem = nData - nWrite; - assert( nWrite>=0 ); - if( nWrite!=0 ){ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite); - } - iApp += nWrite; - } - - /* If required, allocate a new block and write the rest of the data - ** into it. Set the next and previous block pointers to link the new - ** block to the old. */ - assert( nRem<=0 || (iApp-1)==iLastOnBlock ); - if( rc==LSM_OK && (iApp-1)==iLastOnBlock ){ - u8 aPtr[4]; /* Space to serialize a u32 */ - int iBlk; /* New block number */ - - if( nWrite>0 ){ - /* Allocate a new block. */ - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - - /* Set the "next" pointer on the old block */ - if( rc==LSM_OK ){ - assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 ); - lsmPutU32(aPtr, iBlk); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr)); - } - - /* Set the "prev" pointer on the new block */ - if( rc==LSM_OK ){ - LsmPgno iWrite; - lsmPutU32(aPtr, fsPageToBlock(pFS, iApp)); - iWrite = fsFirstPageOnBlock(pFS, iBlk); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr)); - if( nRem>0 ) iApp = iWrite; - } - }else{ - /* The next block is already allocated. */ - assert( nRem>0 ); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iApp), &iBlk); - iRet = iApp = fsFirstPageOnBlock(pFS, iBlk); - } - - /* Write the remaining data into the new block */ - if( rc==LSM_OK && nRem>0 ){ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, &aData[nWrite], nRem); - iApp += nRem; - } - } - - pSeg->iLastPg = iApp-1; - *pRc = rc; - } - - return iRet; -} - -/* -** This function is only called in compressed database mode. It -** compresses the contents of page pPg and writes the result to the -** buffer at pFS->aOBuffer. The size of the compressed data is stored in -** pPg->nCompress. -** -** If buffer pFS->aOBuffer[] has not been allocated then this function -** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. -*/ -static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ - lsm_compress *p = pFS->pCompress; - - if( fsAllocateBuffer(pFS, 1) ) return LSM_NOMEM; - assert( pPg->nData==pFS->nPagesize ); - - pPg->nCompress = pFS->nBuffer; - return p->xCompress(p->pCtx, - (char *)pFS->aOBuffer, &pPg->nCompress, - (const char *)pPg->aData, pPg->nData - ); -} - -/* -** Append a new page to segment pSeg. Set output variable *piNew to the -** page number of the new page before returning. -** -** If the new page is the last on its block, then the 'next' block that -** will be used by the segment is allocated here too. In this case output -** variable *piNext is set to the block number of the next block. -** -** If the new page is the first on its block but not the first in the -** entire segment, set output variable *piPrev to the block number of -** the previous block in the segment. -** -** LSM_OK is returned if successful, or an lsm error code otherwise. If -** any value other than LSM_OK is returned, then the final value of all -** output variables is undefined. -*/ -static int fsAppendPage( - FileSystem *pFS, - Segment *pSeg, - LsmPgno *piNew, - int *piPrev, - int *piNext -){ - LsmPgno iPrev = pSeg->iLastPg; - int rc; - assert( iPrev!=0 ); - - *piPrev = 0; - *piNext = 0; - - if( fsIsLast(pFS, iPrev) ){ - /* Grab the first page on the next block (which has already be - ** allocated). In this case set *piPrev to tell the caller to set - ** the "previous block" pointer in the first 4 bytes of the page. - */ - int iNext; - int iBlk = fsPageToBlock(pFS, iPrev); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, iBlk, &iNext); - if( rc!=LSM_OK ) return rc; - *piNew = fsFirstPageOnBlock(pFS, iNext); - *piPrev = iBlk; - }else{ - *piNew = iPrev+1; - if( fsIsLast(pFS, *piNew) ){ - /* Allocate the next block here. */ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - if( rc!=LSM_OK ) return rc; - *piNext = iBlk; - } - } - - pSeg->nSize++; - pSeg->iLastPg = *piNew; - return LSM_OK; -} - -/* -** Flush all pages in the FileSystem.pWaiting list to disk. -*/ -void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){ - int rc = *pRc; - Page *pPg; - - pPg = pFS->pWaiting; - pFS->pWaiting = 0; - - while( pPg ){ - Page *pNext = pPg->pWaitingNext; - if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg); - assert( pPg->nRef==1 ); - lsmFsPageRelease(pPg); - pPg = pNext; - } - *pRc = rc; -} - -/* -** If there exists a hash-table entry associated with page iPg, remove it. -*/ -static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){ - Page *p; - int iHash = fsHashKey(pFS->nHash, iPg); - - for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext); - - if( p ){ - assert( p->nRef==0 || (p->flags & PAGE_FREE)==0 ); - fsPageRemoveFromHash(pFS, p); - p->iPg = 0; - iHash = fsHashKey(pFS->nHash, 0); - p->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = p; - } -} - -/* -** If the page passed as an argument is dirty, update the database file -** (or mapping of the database file) with its current contents and mark -** the page as clean. -** -** Return LSM_OK if the operation is a success, or an LSM error code -** otherwise. -*/ -int lsmFsPagePersist(Page *pPg){ - int rc = LSM_OK; - if( pPg && (pPg->flags & PAGE_DIRTY) ){ - FileSystem *pFS = pPg->pFS; - - if( pFS->pCompress ){ - int iHash; /* Hash key of assigned page number */ - u8 aSz[3]; /* pPg->nCompress as a 24-bit big-endian */ - assert( pPg->pSeg && pPg->iPg==0 && pPg->nCompress==0 ); - - /* Compress the page image. */ - rc = fsCompressIntoBuffer(pFS, pPg); - - /* Serialize the compressed size into buffer aSz[] */ - putRecordSize(aSz, pPg->nCompress, 0); - - /* Write the serialized page record into the database file. */ - pPg->iPg = fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - fsAppendData(pFS, pPg->pSeg, pFS->aOBuffer, pPg->nCompress, &rc); - fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - - /* Now that it has a page number, insert the page into the hash table */ - iHash = fsHashKey(pFS->nHash, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - - pPg->pSeg->nSize += (sizeof(aSz) * 2) + pPg->nCompress; - - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - }else{ - - if( pPg->iPg==0 ){ - /* No page number has been assigned yet. This occurs with pages used - ** in the b-tree hierarchy. They were not assigned page numbers when - ** they were created as doing so would cause this call to - ** lsmFsPagePersist() to write an out-of-order page. Instead a page - ** number is assigned here so that the page data will be appended - ** to the current segment. - */ - Page **pp; - int iPrev = 0; - int iNext = 0; - int iHash; - - assert( pPg->pSeg->iFirst ); - assert( pPg->flags & PAGE_FREE ); - assert( (pPg->flags & PAGE_HASPREV)==0 ); - assert( pPg->nData==pFS->nPagesize-4 ); - - rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); - if( rc!=LSM_OK ) return rc; - - assert( pPg->flags & PAGE_FREE ); - iHash = fsHashKey(pFS->nHash, pPg->iPg); - fsRemoveHashEntry(pFS, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); - - if( iPrev ){ - assert( iNext==0 ); - memmove(&pPg->aData[4], pPg->aData, pPg->nData); - lsmPutU32(pPg->aData, iPrev); - pPg->flags |= PAGE_HASPREV; - pPg->aData += 4; - }else if( iNext ){ - assert( iPrev==0 ); - lsmPutU32(&pPg->aData[pPg->nData], iNext); - }else{ - int nData = pPg->nData; - pPg->nData += 4; - lsmSortedExpandBtreePage(pPg, nData); - } - - pPg->nRef++; - for(pp=&pFS->pWaiting; *pp; pp=&(*pp)->pWaitingNext); - *pp = pPg; - assert( pPg->pWaitingNext==0 ); - - }else{ - i64 iOff; /* Offset to write within database file */ - - iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); - if( fsMmapPage(pFS, pPg->iPg)==0 ){ - u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); - }else if( pPg->flags & PAGE_FREE ){ - fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); - if( rc==LSM_OK ){ - u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; - u8 *aFrom = pPg->aData - (pPg->flags & PAGE_HASPREV); - memcpy(aTo, aFrom, pFS->nPagesize); - lsmFree(pFS->pEnv, aFrom); - pFS->nCacheAlloc--; - pPg->aData = aTo + (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_FREE; - fsPageRemoveFromHash(pFS, pPg); - pPg->pMappedNext = pFS->pMapped; - pFS->pMapped = pPg; - } - } - - lsmFsFlushWaiting(pFS, &rc); - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - } - } - } - - return rc; -} - -/* -** For non-compressed databases, this function is a no-op. For compressed -** databases, it adds a padding record to the segment passed as the third -** argument. -** -** The size of the padding records is selected so that the last byte -** written is the last byte of a disk sector. This means that if a -** snapshot is taken and checkpointed, subsequent worker processes will -** not write to any sector that contains checkpointed data. -*/ -int lsmFsSortedPadding( - FileSystem *pFS, - Snapshot *pSnapshot, - Segment *pSeg -){ - int rc = LSM_OK; - if( pFS->pCompress && pSeg->iFirst ){ - LsmPgno iLast2; - LsmPgno iLast = pSeg->iLastPg; /* Current last page of segment */ - int nPad; /* Bytes of padding required */ - u8 aSz[3]; - - iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1; - assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) ); - nPad = (int)(iLast2 - iLast); - - if( iLast2>fsLastPageOnPagesBlock(pFS, iLast) ){ - nPad -= 4; - } - assert( nPad>=0 ); - - if( nPad>=6 ){ - pSeg->nSize += nPad; - nPad -= 6; - putRecordSize(aSz, nPad, 1); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - memset(pFS->aOBuffer, 0, nPad); - fsAppendData(pFS, pSeg, pFS->aOBuffer, nPad, &rc); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - }else if( nPad>0 ){ - u8 aBuf[5] = {0,0,0,0,0}; - aBuf[0] = (u8)nPad; - aBuf[nPad-1] = (u8)nPad; - fsAppendData(pFS, pSeg, aBuf, nPad, &rc); - } - - assert( rc!=LSM_OK - || pSeg->iLastPg==fsLastPageOnPagesBlock(pFS, pSeg->iLastPg) - || ((pSeg->iLastPg + 1) % pFS->szSector)==0 - ); - } - - return rc; -} - - -/* -** Increment the reference count on the page object passed as the first -** argument. -*/ -void lsmFsPageRef(Page *pPg){ - if( pPg ){ - pPg->nRef++; - } -} - -/* -** Release a page-reference obtained using fsPageGet(). -*/ -int lsmFsPageRelease(Page *pPg){ - int rc = LSM_OK; - if( pPg ){ - assert( pPg->nRef>0 ); - pPg->nRef--; - if( pPg->nRef==0 ){ - FileSystem *pFS = pPg->pFS; - rc = lsmFsPagePersist(pPg); - pFS->nOut--; - - assert( pPg->pFS->pCompress - || fsIsFirst(pPg->pFS, pPg->iPg)==0 - || (pPg->flags & PAGE_HASPREV) - ); - pPg->aData -= (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_HASPREV; - - if( (pPg->flags & PAGE_FREE)==0 ){ - /* Removed from mapped list */ - Page **pp; - for(pp=&pFS->pMapped; (*pp)!=pPg; pp=&(*pp)->pMappedNext); - *pp = pPg->pMappedNext; - pPg->pMappedNext = 0; - - /* Add to free list */ - pPg->pFreeNext = pFS->pFree; - pFS->pFree = pPg; - }else{ - fsPageAddToLru(pFS, pPg); - } - } - } - - return rc; -} - -/* -** Return the total number of pages read from the database file. -*/ -int lsmFsNRead(FileSystem *pFS){ return pFS->nRead; } - -/* -** Return the total number of pages written to the database file. -*/ -int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } - -/* -** Return a copy of the environment pointer used by the file-system object. -*/ -lsm_env *lsmFsEnv(FileSystem *pFS){ - return pFS->pEnv; -} - -/* -** Return a copy of the environment pointer used by the file-system object -** to which this page belongs. -*/ -lsm_env *lsmPageEnv(Page *pPg) { - return pPg->pFS->pEnv; -} - -/* -** Return a pointer to the file-system object associated with the Page -** passed as the only argument. -*/ -FileSystem *lsmPageFS(Page *pPg){ - return pPg->pFS; -} - -/* -** Return the sector-size as reported by the log file handle. -*/ -int lsmFsSectorSize(FileSystem *pFS){ - return pFS->szSector; -} - -/* -** Helper function for lsmInfoArrayStructure(). -*/ -static Segment *startsWith(Segment *pRun, LsmPgno iFirst){ - return (iFirst==pRun->iFirst) ? pRun : 0; -} - -/* -** Return the segment that starts with page iFirst, if any. If no such segment -** can be found, return NULL. -*/ -static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){ - Level *pLvl; /* Used to iterate through db levels */ - Segment *pSeg = 0; /* Pointer to segment to return */ - - for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){ - if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){ - int i; - for(i=0; inRight; i++){ - if( (pSeg = startsWith(&pLvl->aRhs[i], iFirst)) ) break; - } - } - } - - return pSeg; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_STRUCTURE) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayStructure( - lsm_db *pDb, - int bBlock, /* True for block numbers only */ - LsmPgno iFirst, - char **pzOut -){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pArray = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pArray = findSegment(pWorker, iFirst); - - if( pArray==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - FileSystem *pFS = pDb->pFS; - LsmString str; - int iBlk; - int iLastBlk; - - iBlk = fsPageToBlock(pFS, pArray->iFirst); - iLastBlk = fsPageToBlock(pFS, pArray->iLastPg); - - lsmStringInit(&str, pDb->pEnv); - if( bBlock ){ - lsmStringAppendf(&str, "%d", iBlk); - while( iBlk!=iLastBlk ){ - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", iBlk); - } - }else{ - lsmStringAppendf(&str, "%d", pArray->iFirst); - while( iBlk!=iLastBlk ){ - lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); - } - lsmStringAppendf(&str, " %d", pArray->iLastPg); - } - - *pzOut = str.z; - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -int lsmFsSegmentContainsPg( - FileSystem *pFS, - Segment *pSeg, - LsmPgno iPg, - int *pbRes -){ - Redirect *pRedir = pSeg->pRedirect; - int rc = LSM_OK; - int iBlk; - int iLastBlk; - int iPgBlock; /* Block containing page iPg */ - - iPgBlock = fsPageToBlock(pFS, pSeg->iFirst); - iBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iFirst)); - iLastBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iLastPg)); - - while( iBlk!=iLastBlk && iBlk!=iPgBlock && rc==LSM_OK ){ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - } - - *pbRes = (iBlk==iPgBlock); - return rc; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pSeg = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pSeg = findSegment(pWorker, iFirst); - - if( pSeg==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - Page *pPg = 0; - FileSystem *pFS = pDb->pFS; - LsmString str; - - lsmStringInit(&str, pDb->pEnv); - rc = lsmFsDbPageGet(pFS, pSeg, iFirst, &pPg); - while( rc==LSM_OK && pPg ){ - Page *pNext = 0; - lsmStringAppendf(&str, " %lld", lsmFsPageNumber(pPg)); - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, str.z); - }else{ - *pzOut = str.z; - } - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -/* -** The following macros are used by the integrity-check code. Associated with -** each block in the database is an 8-bit bit mask (the entry in the aUsed[] -** array). As the integrity-check meanders through the database, it sets the -** following bits to indicate how each block is used. -** -** INTEGRITY_CHECK_FIRST_PG: -** First page of block is in use by sorted run. -** -** INTEGRITY_CHECK_LAST_PG: -** Last page of block is in use by sorted run. -** -** INTEGRITY_CHECK_USED: -** At least one page of the block is in use by a sorted run. -** -** INTEGRITY_CHECK_FREE: -** The free block list contains an entry corresponding to this block. -*/ -#define INTEGRITY_CHECK_FIRST_PG 0x01 -#define INTEGRITY_CHECK_LAST_PG 0x02 -#define INTEGRITY_CHECK_USED 0x04 -#define INTEGRITY_CHECK_FREE 0x08 - -/* -** Helper function for lsmFsIntegrityCheck() -*/ -static void checkBlocks( - FileSystem *pFS, - Segment *pSeg, - int bExtra, /* If true, count the "next" block if any */ - int nUsed, - u8 *aUsed -){ - if( pSeg ){ - if( pSeg && pSeg->nSize>0 ){ - int rc; - int iBlk; /* Current block (during iteration) */ - int iLastBlk; /* Last block of segment */ - int iFirstBlk; /* First block of segment */ - int bLastIsLastOnBlock; /* True iLast is the last on its block */ - - assert( 0==fsSegmentRedirects(pFS, pSeg) ); - iBlk = iFirstBlk = fsPageToBlock(pFS, pSeg->iFirst); - iLastBlk = fsPageToBlock(pFS, pSeg->iLastPg); - - bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==pSeg->iLastPg); - assert( iBlk>0 ); - - do { - /* iBlk is a part of this sorted run. */ - aUsed[iBlk-1] |= INTEGRITY_CHECK_USED; - - /* If the first page of this block is also part of the segment, - ** set the flag to indicate that the first page of iBlk is in use. - */ - if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst || iBlk!=iFirstBlk ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; - } - - /* Unless the sorted run finishes before the last page on this block, - ** the last page of this block is also in use. */ - if( iBlk!=iLastBlk || bLastIsLastOnBlock ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_LAST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Special case. The sorted run being scanned is the output run of - ** a level currently undergoing an incremental merge. The sorted - ** run ends on the last page of iBlk, but the next block has already - ** been allocated. So mark it as in use as well. */ - if( iBlk==iLastBlk && bLastIsLastOnBlock && bExtra ){ - int iExtra = 0; - rc = fsBlockNext(pFS, pSeg, iBlk, &iExtra); - assert( rc==LSM_OK ); - - assert( aUsed[iExtra-1]==0 ); - aUsed[iExtra-1] |= INTEGRITY_CHECK_USED; - aUsed[iExtra-1] |= INTEGRITY_CHECK_FIRST_PG; - aUsed[iExtra-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Move on to the next block in the sorted run. Or set iBlk to zero - ** in order to break out of the loop if this was the last block in - ** the run. */ - if( iBlk==iLastBlk ){ - iBlk = 0; - }else{ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - assert( rc==LSM_OK ); - } - }while( iBlk ); - } - } -} - -typedef struct CheckFreelistCtx CheckFreelistCtx; -struct CheckFreelistCtx { - u8 *aUsed; - int nBlock; -}; -static int checkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - CheckFreelistCtx *p = (CheckFreelistCtx *)pCtx; - - assert( iBlk>=1 ); - assert( iBlk<=p->nBlock ); - assert( p->aUsed[iBlk-1]==0 ); - p->aUsed[iBlk-1] = INTEGRITY_CHECK_FREE; - return 0; -} - -/* -** This function checks that all blocks in the database file are accounted -** for. For each block, exactly one of the following must be true: -** -** + the block is part of a sorted run, or -** + the block is on the free-block list -** -** This function also checks that there are no references to blocks with -** out-of-range block numbers. -** -** If no errors are found, non-zero is returned. If an error is found, an -** assert() fails. -*/ -int lsmFsIntegrityCheck(lsm_db *pDb){ - CheckFreelistCtx ctx; - FileSystem *pFS = pDb->pFS; - int i; - int rc; - Freelist freelist = {0, 0, 0}; - u8 *aUsed; - Level *pLevel; - Snapshot *pWorker = pDb->pWorker; - int nBlock = pWorker->nBlock; - -#if 0 - static int nCall = 0; - nCall++; - printf("%d calls\n", nCall); -#endif - - aUsed = lsmMallocZero(pDb->pEnv, nBlock); - if( aUsed==0 ){ - /* Malloc has failed. Since this function is only called within debug - ** builds, this probably means the user is running an OOM injection test. - ** Regardless, it will not be possible to run the integrity-check at this - ** time, so assume the database is Ok and return non-zero. */ - return 1; - } - - for(pLevel=pWorker->pLevel; pLevel; pLevel=pLevel->pNext){ - int j; - checkBlocks(pFS, &pLevel->lhs, (pLevel->nRight!=0), nBlock, aUsed); - for(j=0; jnRight; j++){ - checkBlocks(pFS, &pLevel->aRhs[j], 0, nBlock, aUsed); - } - } - - /* Mark all blocks in the free-list as used */ - ctx.aUsed = aUsed; - ctx.nBlock = nBlock; - rc = lsmWalkFreelist(pDb, 0, checkFreelistCb, (void *)&ctx); - - if( rc==LSM_OK ){ - for(i=0; ipEnv, aUsed); - lsmFree(pDb->pEnv, freelist.aEntry); - - return 1; -} - -#ifndef NDEBUG -/* -** Return true if pPg happens to be the last page in segment pSeg. Or false -** otherwise. This function is only invoked as part of assert() conditions. -*/ -int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){ - if( pPg->pFS->pCompress ){ - LsmPgno iNext = 0; - int rc; - rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext); - return (rc!=LSM_OK || iNext==0); - } - return (pPg->iPg==pSeg->iLastPg); -} -#endif diff --git a/ext/lsm1/lsm_log.c b/ext/lsm1/lsm_log.c deleted file mode 100644 index a66e40bccd..0000000000 --- a/ext/lsm1/lsm_log.c +++ /dev/null @@ -1,1156 +0,0 @@ -/* -** 2011-08-13 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of LSM database logging. Logging -** has one purpose in LSM - to make transactions durable. -** -** When data is written to an LSM database, it is initially stored in an -** in-memory tree structure. Since this structure is in volatile memory, -** if a power failure or application crash occurs it may be lost. To -** prevent loss of data in this case, each time a record is written to the -** in-memory tree an equivalent record is appended to the log on disk. -** If a power failure or application crash does occur, data can be recovered -** by reading the log. -** -** A log file consists of the following types of records representing data -** written into the database: -** -** LOG_WRITE: A key-value pair written to the database. -** LOG_DELETE: A delete key issued to the database. -** LOG_COMMIT: A transaction commit. -** -** And the following types of records for ancillary purposes.. -** -** LOG_EOF: A record indicating the end of a log file. -** LOG_PAD1: A single byte padding record. -** LOG_PAD2: An N byte padding record (N>1). -** LOG_JUMP: A pointer to another offset within the log file. -** -** Each transaction written to the log contains one or more LOG_WRITE and/or -** LOG_DELETE records, followed by a LOG_COMMIT record. The LOG_COMMIT record -** contains an 8-byte checksum based on all previous data written to the -** log file. -** -** LOG CHECKSUMS & RECOVERY -** -** Checksums are found in two types of log records: LOG_COMMIT and -** LOG_CKSUM records. In order to recover content from a log, a client -** reads each record from the start of the log, calculating a checksum as -** it does. Each time a LOG_COMMIT or LOG_CKSUM is encountered, the -** recovery process verifies that the checksum stored in the log -** matches the calculated checksum. If it does not, the recovery process -** can stop reading the log. -** -** If a recovery process reads records (other than COMMIT or CKSUM) -** consisting of at least LSM_CKSUM_MAXDATA bytes, then the next record in -** the log must be either a LOG_CKSUM or LOG_COMMIT record. If it is -** not, the recovery process also stops reading the log. -** -** To recover the log file, it must be read twice. The first time to -** determine the location of the last valid commit record. And the second -** time to load data into the in-memory tree. -** -** Todo: Surely there is a better way... -** -** LOG WRAPPING -** -** If the log file were never deleted or wrapped, it would be possible to -** read it from start to end each time is required recovery (i.e each time -** the number of database clients changes from 0 to 1). Effectively reading -** the entire history of the database each time. This would quickly become -** inefficient. Additionally, since the log file would grow without bound, -** it wastes storage space. -** -** Instead, part of each checkpoint written into the database file contains -** a log offset (and other information required to read the log starting at -** at this offset) at which to begin recovery. Offset $O. -** -** Once a checkpoint has been written and synced into the database file, it -** is guaranteed that no recovery process will need to read any data before -** offset $O of the log file. It is therefore safe to begin overwriting -** any data that occurs before offset $O. -** -** This implementation separates the log into three regions mapped into -** the log file - regions 0, 1 and 2. During recovery, regions are read -** in ascending order (i.e. 0, then 1, then 2). Each region is zero or -** more bytes in size. -** -** |---1---|..|--0--|.|--2--|.... -** -** New records are always appended to the end of region 2. -** -** Initially (when it is empty), all three regions are zero bytes in size. -** Each of them are located at the beginning of the file. As records are -** added to the log, region 2 grows, so that the log consists of a zero -** byte region 1, followed by a zero byte region 0, followed by an N byte -** region 2. After one or more checkpoints have been written to disk, -** the start point of region 2 is moved to $O. For example: -** -** A) ||.........|--2--|.... -** -** (both regions 0 and 1 are 0 bytes in size at offset 0). -** -** Eventually, the log wraps around to write new records into the start. -** At this point, region 2 is renamed to region 0. Region 0 is renamed -** to region 2. After appending a few records to the new region 2, the -** log file looks like this: -** -** B) ||--2--|...|--0--|.... -** -** (region 1 is still 0 bytes in size, located at offset 0). -** -** Any checkpoints made at this point may reduce the size of region 0. -** However, if they do not, and region 2 expands so that it is about to -** overwrite the start of region 0, then region 2 is renamed to region 1, -** and a new region 2 created at the end of the file following the existing -** region 0. -** -** C) |---1---|..|--0--|.|-2-| -** -** In this state records are appended to region 2 until checkpoints have -** contracted regions 0 AND 1 UNTil they are both zero bytes in size. They -** are then shifted to the start of the log file, leaving the system in -** the equivalent of state A above. -** -** Alternatively, state B may transition directly to state A if the size -** of region 0 is reduced to zero bytes before region 2 threatens to -** encroach upon it. -** -** LOG_PAD1 & LOG_PAD2 RECORDS -** -** PAD1 and PAD2 records may appear in a log file at any point. They allow -** a process writing the log file align the beginning of transactions with -** the beginning of disk sectors, which increases robustness. -** -** RECORD FORMATS: -** -** LOG_EOF: * A single 0x00 byte. -** -** LOG_PAD1: * A single 0x01 byte. -** -** LOG_PAD2: * A single 0x02 byte, followed by -** * The number of unused bytes (N) as a varint, -** * An N byte block of unused space. -** -** LOG_COMMIT: * A single 0x03 byte. -** * An 8-byte checksum. -** -** LOG_JUMP: * A single 0x04 byte. -** * Absolute file offset to jump to, encoded as a varint. -** -** LOG_WRITE: * A single 0x06 or 0x07 byte, -** * The number of bytes in the key, encoded as a varint, -** * The number of bytes in the value, encoded as a varint, -** * If the first byte was 0x07, an 8 byte checksum. -** * The key data, -** * The value data. -** -** LOG_DELETE: * A single 0x08 or 0x09 byte, -** * The number of bytes in the key, encoded as a varint, -** * If the first byte was 0x09, an 8 byte checksum. -** * The key data. -** -** Varints are as described in lsm_varint.c (SQLite 4 format). -** -** CHECKSUMS: -** -** The checksum is calculated using two 32-bit unsigned integers, s0 and -** s1. The initial value for both is 42. It is updated each time a record -** is written into the log file by treating the encoded (binary) record as -** an array of 32-bit little-endian integers. Then, if x[] is the integer -** array, updating the checksum accumulators as follows: -** -** for i from 0 to n-1 step 2: -** s0 += x[i] + s1; -** s1 += x[i+1] + s0; -** endfor -** -** If the record is not an even multiple of 8-bytes in size it is padded -** with zeroes to make it so before the checksum is updated. -** -** The checksum stored in a COMMIT, WRITE or DELETE is based on all bytes -** up to the start of the 8-byte checksum itself, including the COMMIT, -** WRITE or DELETE fields that appear before the checksum in the record. -** -** VARINT FORMAT -** -** See lsm_varint.c. -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -/* Log record types */ -#define LSM_LOG_EOF 0x00 -#define LSM_LOG_PAD1 0x01 -#define LSM_LOG_PAD2 0x02 -#define LSM_LOG_COMMIT 0x03 -#define LSM_LOG_JUMP 0x04 - -#define LSM_LOG_WRITE 0x06 -#define LSM_LOG_WRITE_CKSUM 0x07 - -#define LSM_LOG_DELETE 0x08 -#define LSM_LOG_DELETE_CKSUM 0x09 - -#define LSM_LOG_DRANGE 0x0A -#define LSM_LOG_DRANGE_CKSUM 0x0B - -/* Require a checksum every 32KB. */ -#define LSM_CKSUM_MAXDATA (32*1024) - -/* Do not wrap a log file smaller than this in bytes. */ -#define LSM_MIN_LOGWRAP (128*1024) - -/* -** szSector: -** Commit records must be aligned to end on szSector boundaries. If -** the safety-mode is set to NORMAL or OFF, this value is 1. Otherwise, -** if the safety-mode is set to FULL, it is the size of the file-system -** sectors as reported by lsmFsSectorSize(). -*/ -struct LogWriter { - u32 cksum0; /* Checksum 0 at offset iOff */ - u32 cksum1; /* Checksum 1 at offset iOff */ - int iCksumBuf; /* Bytes of buf that have been checksummed */ - i64 iOff; /* Offset at start of buffer buf */ - int szSector; /* Sector size for this transaction */ - LogRegion jump; /* Avoid writing to this region */ - i64 iRegion1End; /* End of first region written by trans */ - i64 iRegion2Start; /* Start of second regions written by trans */ - LsmString buf; /* Buffer containing data not yet written */ -}; - -/* -** Return the result of interpreting the first 4 bytes in buffer aIn as -** a 32-bit unsigned little-endian integer. -*/ -static u32 getU32le(u8 *aIn){ - return ((u32)aIn[3] << 24) - + ((u32)aIn[2] << 16) - + ((u32)aIn[1] << 8) - + ((u32)aIn[0]); -} - - -/* -** This function is the same as logCksum(), except that pointer "a" need -** not be aligned to an 8-byte boundary or padded with zero bytes. This -** version is slower, but sometimes more convenient to use. -*/ -static void logCksumUnaligned( - char *z, /* Input buffer */ - int n, /* Size of input buffer in bytes */ - u32 *pCksum0, /* IN/OUT: Checksum value 1 */ - u32 *pCksum1 /* IN/OUT: Checksum value 2 */ -){ - u8 *a = (u8 *)z; - u32 cksum0 = *pCksum0; - u32 cksum1 = *pCksum1; - int nIn = (n/8) * 8; - int i; - - assert( n>0 ); - for(i=0; inIn ); - memcpy(aBuf, &a[nIn], n-nIn); - cksum0 += getU32le(aBuf) + cksum1; - cksum1 += getU32le(&aBuf[4]) + cksum0; - } - - *pCksum0 = cksum0; - *pCksum1 = cksum1; -} - -/* -** Update pLog->cksum0 and pLog->cksum1 so that the first nBuf bytes in the -** write buffer (pLog->buf) are included in the checksum. -*/ -static void logUpdateCksum(LogWriter *pLog, int nBuf){ - assert( (pLog->iCksumBuf % 8)==0 ); - assert( pLog->iCksumBuf<=nBuf ); - assert( (nBuf % 8)==0 || nBuf==pLog->buf.n ); - if( nBuf>pLog->iCksumBuf ){ - logCksumUnaligned( - &pLog->buf.z[pLog->iCksumBuf], nBuf-pLog->iCksumBuf, - &pLog->cksum0, &pLog->cksum1 - ); - } - pLog->iCksumBuf = nBuf; -} - -static i64 firstByteOnSector(LogWriter *pLog, i64 iOff){ - return (iOff / pLog->szSector) * pLog->szSector; -} -static i64 lastByteOnSector(LogWriter *pLog, i64 iOff){ - return firstByteOnSector(pLog, iOff) + pLog->szSector - 1; -} - -/* -** If possible, reclaim log file space. Log file space is reclaimed after -** a snapshot that points to the same data in the database file is synced -** into the db header. -*/ -static int logReclaimSpace(lsm_db *pDb){ - int rc; - int iMeta; - int bRotrans; /* True if there exists some ro-trans */ - - /* Test if there exists some other connection with a read-only transaction - ** open. If there does, then log file space may not be reclaimed. */ - rc = lsmDetectRoTrans(pDb, &bRotrans); - if( rc!=LSM_OK || bRotrans ) return rc; - - iMeta = (int)pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ - DbLog *pLog = &pDb->treehdr.log; - i64 iSyncedId; - - /* Read the snapshot-id of the snapshot stored on meta-page iMeta. Note - ** that in theory, the value read is untrustworthy (due to a race - ** condition - see comments above lsmFsReadSyncedId()). So it is only - ** ever used to conclude that no log space can be reclaimed. If it seems - ** to indicate that it may be possible to reclaim log space, a - ** second call to lsmCheckpointSynced() (which does return trustworthy - ** values) is made below to confirm. */ - rc = lsmFsReadSyncedId(pDb, iMeta, &iSyncedId); - - if( rc==LSM_OK && pLog->iSnapshotId!=iSyncedId ){ - i64 iSnapshotId = 0; - i64 iOff = 0; - rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); - if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; - if( iOff>=p->iStart && iOff<=p->iEnd ) break; - p->iStart = 0; - p->iEnd = 0; - } - assert( iRegion<3 ); - pLog->aRegion[iRegion].iStart = iOff; - pLog->iSnapshotId = iSnapshotId; - } - } - } - return rc; -} - -/* -** This function is called when a write-transaction is first opened. It -** is assumed that the caller is holding the client-mutex when it is -** called. -** -** Before returning, this function allocates the LogWriter object that -** will be used to write to the log file during the write transaction. -** LSM_OK is returned if no error occurs, otherwise an LSM error code. -*/ -int lsmLogBegin(lsm_db *pDb){ - int rc = LSM_OK; - LogWriter *pNew; - LogRegion *aReg; - - if( pDb->bUseLog==0 ) return LSM_OK; - - /* If the log file has not yet been opened, open it now. Also allocate - ** the LogWriter structure, if it has not already been allocated. */ - rc = lsmFsOpenLog(pDb, 0); - if( pDb->pLogWriter==0 ){ - pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); - if( pNew ){ - lsmStringInit(&pNew->buf, pDb->pEnv); - rc = lsmStringExtend(&pNew->buf, 2); - } - pDb->pLogWriter = pNew; - }else{ - pNew = pDb->pLogWriter; - assert( (u8 *)(&pNew[1])==(u8 *)(&((&pNew->buf)[1])) ); - memset(pNew, 0, ((u8 *)&pNew->buf) - (u8 *)pNew); - pNew->buf.n = 0; - } - - if( rc==LSM_OK ){ - /* The following call detects whether or not a new snapshot has been - ** synced into the database file. If so, it updates the contents of - ** the pDb->treehdr.log structure to reclaim any space in the log - ** file that is no longer required. - ** - ** TODO: Calling this every transaction is overkill. And since the - ** call has to read and checksum a snapshot from the database file, - ** it is expensive. It would be better to figure out a way so that - ** this is only called occasionally - say for every 32KB written to - ** the log file. - */ - rc = logReclaimSpace(pDb); - } - if( rc!=LSM_OK ){ - lsmLogClose(pDb); - return rc; - } - - /* Set the effective sector-size for this transaction. Sectors are assumed - ** to be one byte in size if the safety-mode is OFF or NORMAL, or as - ** reported by lsmFsSectorSize if it is FULL. */ - if( pDb->eSafety==LSM_SAFETY_FULL ){ - pNew->szSector = lsmFsSectorSize(pDb->pFS); - assert( pNew->szSector>0 ); - }else{ - pNew->szSector = 1; - } - - /* There are now three scenarios: - ** - ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins - ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap - ** around to the start and write data into the start of the log file. - ** - ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the - ** file than region 0. In this case, append data to region 2, but - ** remember to jump over region 1 if required. - ** - ** 3) Region 2 is the last in the file. Append to it. - */ - aReg = &pDb->treehdr.log.aRegion[0]; - - assert( aReg[0].iEnd==0 || aReg[0].iEnd>aReg[0].iStart ); - assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); - - pNew->cksum0 = pDb->treehdr.log.cksum0; - pNew->cksum1 = pDb->treehdr.log.cksum1; - - if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ - /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP - ** into the log file in this case. Pad it out to 8 bytes using a PAD2 - ** record so that the checksums can be updated immediately. */ - u8 aJump[] = { - LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 - }; - - lsmStringBinAppend(&pNew->buf, aJump, sizeof(aJump)); - logUpdateCksum(pNew, pNew->buf.n); - rc = lsmFsWriteLog(pDb->pFS, aReg[2].iEnd, &pNew->buf); - pNew->iCksumBuf = pNew->buf.n = 0; - - aReg[2].iEnd += 8; - pNew->jump = aReg[0] = aReg[2]; - aReg[2].iStart = aReg[2].iEnd = 0; - }else if( aReg[1].iEnd==0 && aReg[2].iEndiOff = aReg[2].iEnd; - pNew->jump = aReg[0]; - }else{ - /* Case 3. */ - assert( aReg[2].iStart>=aReg[0].iEnd && aReg[2].iStart>=aReg[1].iEnd ); - pNew->iOff = aReg[2].iEnd; - } - - if( pNew->jump.iStart ){ - i64 iRound; - assert( pNew->jump.iStart>pNew->iOff ); - - iRound = firstByteOnSector(pNew, pNew->jump.iStart); - if( iRound>pNew->iOff ) pNew->jump.iStart = iRound; - pNew->jump.iEnd = lastByteOnSector(pNew, pNew->jump.iEnd); - } - - assert( pDb->pLogWriter==pNew ); - return rc; -} - -/* -** This function is called when a write-transaction is being closed. -** Parameter bCommit is true if the transaction is being committed, -** or false otherwise. The caller must hold the client-mutex to call -** this function. -** -** A call to this function deletes the LogWriter object allocated by -** lsmLogBegin(). If the transaction is being committed, the shared state -** in *pLog is updated before returning. -*/ -void lsmLogEnd(lsm_db *pDb, int bCommit){ - DbLog *pLog; - LogWriter *p; - p = pDb->pLogWriter; - - if( p==0 ) return; - pLog = &pDb->treehdr.log; - - if( bCommit ){ - pLog->aRegion[2].iEnd = p->iOff; - pLog->cksum0 = p->cksum0; - pLog->cksum1 = p->cksum1; - if( p->iRegion1End ){ - /* This happens when the transaction had to jump over some other - ** part of the log. */ - assert( pLog->aRegion[1].iEnd==0 ); - assert( pLog->aRegion[2].iStartiRegion1End ); - pLog->aRegion[1].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[1].iEnd = p->iRegion1End; - pLog->aRegion[2].iStart = p->iRegion2Start; - } - } -} - -static int jumpIfRequired( - lsm_db *pDb, - LogWriter *pLog, - int nReq, - int *pbJump -){ - /* Determine if it is necessary to add an LSM_LOG_JUMP to jump over the - ** jump region before writing the LSM_LOG_WRITE or DELETE record. This - ** is necessary if there is insufficient room between the current offset - ** and the jump region to fit the new WRITE/DELETE record and the largest - ** possible JUMP record with up to 7 bytes of padding (a total of 17 - ** bytes). */ - if( (pLog->jump.iStart > (pLog->iOff + pLog->buf.n)) - && (pLog->jump.iStart < (pLog->iOff + pLog->buf.n + (nReq + 17))) - ){ - int rc; /* Return code */ - i64 iJump; /* Offset to jump to */ - u8 aJump[10]; /* Encoded jump record */ - int nJump; /* Valid bytes in aJump[] */ - int nPad; /* Bytes of padding required */ - - /* Serialize the JUMP record */ - iJump = pLog->jump.iEnd+1; - aJump[0] = LSM_LOG_JUMP; - nJump = 1 + lsmVarintPut64(&aJump[1], iJump); - - /* Adding padding to the contents of the buffer so that it will be a - ** multiple of 8 bytes in size after the JUMP record is appended. This - ** is not strictly required, it just makes the keeping the running - ** checksum up to date in this file a little simpler. */ - nPad = (pLog->buf.n + nJump) % 8; - if( nPad ){ - u8 aPad[7] = {0,0,0,0,0,0,0}; - nPad = 8-nPad; - if( nPad==1 ){ - aPad[0] = LSM_LOG_PAD1; - }else{ - aPad[0] = LSM_LOG_PAD2; - aPad[1] = (u8)(nPad-2); - } - rc = lsmStringBinAppend(&pLog->buf, aPad, nPad); - if( rc!=LSM_OK ) return rc; - } - - /* Append the JUMP record to the buffer. Then flush the buffer to disk - ** and update the checksums. The next write to the log file (assuming - ** there is no transaction rollback) will be to offset iJump (just past - ** the jump region). */ - rc = lsmStringBinAppend(&pLog->buf, aJump, nJump); - if( rc!=LSM_OK ) return rc; - assert( (pLog->buf.n % 8)==0 ); - rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); - if( rc!=LSM_OK ) return rc; - logUpdateCksum(pLog, pLog->buf.n); - pLog->iRegion1End = (pLog->iOff + pLog->buf.n); - pLog->iRegion2Start = iJump; - pLog->iOff = iJump; - pLog->iCksumBuf = pLog->buf.n = 0; - if( pbJump ) *pbJump = 1; - } - - return LSM_OK; -} - -static int logCksumAndFlush(lsm_db *pDb){ - int rc; /* Return code */ - LogWriter *pLog = pDb->pLogWriter; - - /* Calculate the checksum value. Append it to the buffer. */ - logUpdateCksum(pLog, pLog->buf.n); - lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum0); - pLog->buf.n += 4; - lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum1); - pLog->buf.n += 4; - - /* Write the contents of the buffer to disk. */ - rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); - pLog->iOff += pLog->buf.n; - pLog->iCksumBuf = pLog->buf.n = 0; - - return rc; -} - -/* -** Write the contents of the log-buffer to disk. Then write either a CKSUM -** or COMMIT record, depending on the value of parameter eType. -*/ -static int logFlush(lsm_db *pDb, int eType){ - int rc; - int nReq; - LogWriter *pLog = pDb->pLogWriter; - - assert( eType==LSM_LOG_COMMIT ); - assert( pLog ); - - /* Commit record is always 9 bytes in size. */ - nReq = 9; - if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ) nReq += pLog->szSector + 17; - rc = jumpIfRequired(pDb, pLog, nReq, 0); - - /* If this is a COMMIT, add padding to the log so that the COMMIT record - ** is aligned against the end of a disk sector. In other words, add padding - ** so that the first byte following the COMMIT record lies on a different - ** sector. */ - if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ){ - int nPad; /* Bytes of padding to add */ - - /* Determine the value of nPad. */ - nPad = ((pLog->iOff + pLog->buf.n + 9) % pLog->szSector); - if( nPad ) nPad = pLog->szSector - nPad; - rc = lsmStringExtend(&pLog->buf, nPad); - if( rc!=LSM_OK ) return rc; - - while( nPad ){ - if( nPad==1 ){ - pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD1; - nPad = 0; - }else{ - int n = LSM_MIN(200, nPad-2); - pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD2; - pLog->buf.z[pLog->buf.n++] = (char)n; - nPad -= 2; - memset(&pLog->buf.z[pLog->buf.n], 0x2B, n); - pLog->buf.n += n; - nPad -= n; - } - } - } - - /* Make sure there is room in the log-buffer to add the CKSUM or COMMIT - ** record. Then add the first byte of it. */ - rc = lsmStringExtend(&pLog->buf, 9); - if( rc!=LSM_OK ) return rc; - pLog->buf.z[pLog->buf.n++] = (char)eType; - memset(&pLog->buf.z[pLog->buf.n], 0, 8); - - rc = logCksumAndFlush(pDb); - - /* If this is a commit and synchronous=full, sync the log to disk. */ - if( rc==LSM_OK && eType==LSM_LOG_COMMIT && pDb->eSafety==LSM_SAFETY_FULL ){ - rc = lsmFsSyncLog(pDb->pFS); - } - return rc; -} - -/* -** Append an LSM_LOG_WRITE (if nVal>=0) or LSM_LOG_DELETE (if nVal<0) -** record to the database log. -*/ -int lsmLogWrite( - lsm_db *pDb, /* Database handle */ - int eType, - void *pKey, int nKey, /* Database key to write to log */ - void *pVal, int nVal /* Database value (or nVal<0) to write */ -){ - int rc = LSM_OK; - LogWriter *pLog; /* Log object to write to */ - int nReq; /* Bytes of space required in log */ - int bCksum = 0; /* True to embed a checksum in this record */ - - assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE ); - assert( LSM_LOG_WRITE==LSM_WRITE ); - assert( LSM_LOG_DELETE==LSM_DELETE ); - assert( LSM_LOG_DRANGE==LSM_DRANGE ); - assert( (eType==LSM_LOG_DELETE)==(nVal<0) ); - - if( pDb->bUseLog==0 ) return LSM_OK; - pLog = pDb->pLogWriter; - - /* Determine how many bytes of space are required, assuming that a checksum - ** will be embedded in this record (even though it may not be). */ - nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey; - if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal; - - /* Jump over the jump region if required. Set bCksum to true to tell the - ** code below to include a checksum in the record if either (a) writing - ** this record would mean that more than LSM_CKSUM_MAXDATA bytes of data - ** have been written to the log since the last checksum, or (b) the jump - ** is taken. */ - rc = jumpIfRequired(pDb, pLog, nReq, &bCksum); - if( (pLog->buf.n+nReq) > LSM_CKSUM_MAXDATA ) bCksum = 1; - - if( rc==LSM_OK ){ - rc = lsmStringExtend(&pLog->buf, nReq); - } - if( rc==LSM_OK ){ - u8 *a = (u8 *)&pLog->buf.z[pLog->buf.n]; - - /* Write the record header - the type byte followed by either 1 (for - ** DELETE) or 2 (for WRITE) varints. */ - assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) ); - assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) ); - assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) ); - *(a++) = (u8)eType | (u8)bCksum; - a += lsmVarintPut32(a, nKey); - if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal); - - if( bCksum ){ - pLog->buf.n = (a - (u8 *)pLog->buf.z); - rc = logCksumAndFlush(pDb); - a = (u8 *)&pLog->buf.z[pLog->buf.n]; - } - - memcpy(a, pKey, nKey); - a += nKey; - if( eType!=LSM_LOG_DELETE ){ - memcpy(a, pVal, nVal); - a += nVal; - } - pLog->buf.n = a - (u8 *)pLog->buf.z; - assert( pLog->buf.n<=pLog->buf.nAlloc ); - } - - return rc; -} - -/* -** Append an LSM_LOG_COMMIT record to the database log. -*/ -int lsmLogCommit(lsm_db *pDb){ - if( pDb->bUseLog==0 ) return LSM_OK; - return logFlush(pDb, LSM_LOG_COMMIT); -} - -/* -** Store the current offset and other checksum related information in the -** structure *pMark. Later, *pMark can be passed to lsmLogSeek() to "rewind" -** the LogWriter object to the current log file offset. This is used when -** rolling back savepoint transactions. -*/ -void lsmLogTell( - lsm_db *pDb, /* Database handle */ - LogMark *pMark /* Populate this object with current offset */ -){ - LogWriter *pLog; - int nCksum; - - if( pDb->bUseLog==0 ) return; - pLog = pDb->pLogWriter; - nCksum = pLog->buf.n & 0xFFFFFFF8; - logUpdateCksum(pLog, nCksum); - assert( pLog->iCksumBuf==nCksum ); - pMark->nBuf = pLog->buf.n - nCksum; - memcpy(pMark->aBuf, &pLog->buf.z[nCksum], pMark->nBuf); - - pMark->iOff = pLog->iOff + pLog->buf.n; - pMark->cksum0 = pLog->cksum0; - pMark->cksum1 = pLog->cksum1; -} - -/* -** Seek (rewind) back to the log file offset stored by an ealier call to -** lsmLogTell() in *pMark. -*/ -void lsmLogSeek( - lsm_db *pDb, /* Database handle */ - LogMark *pMark /* Object containing log offset to seek to */ -){ - LogWriter *pLog; - - if( pDb->bUseLog==0 ) return; - pLog = pDb->pLogWriter; - - assert( pMark->iOff<=pLog->iOff+pLog->buf.n ); - if( (pMark->iOff & 0xFFFFFFF8)>=pLog->iOff ){ - pLog->buf.n = (int)(pMark->iOff - pLog->iOff); - pLog->iCksumBuf = (pLog->buf.n & 0xFFFFFFF8); - }else{ - pLog->buf.n = pMark->nBuf; - memcpy(pLog->buf.z, pMark->aBuf, pMark->nBuf); - pLog->iCksumBuf = 0; - pLog->iOff = pMark->iOff - pMark->nBuf; - } - pLog->cksum0 = pMark->cksum0; - pLog->cksum1 = pMark->cksum1; - - if( pMark->iOff > pLog->iRegion1End ) pLog->iRegion1End = 0; - if( pMark->iOff > pLog->iRegion2Start ) pLog->iRegion2Start = 0; -} - -/* -** This function does the work for an lsm_info(LOG_STRUCTURE) request. -*/ -int lsmInfoLogStructure(lsm_db *pDb, char **pzVal){ - int rc = LSM_OK; - char *zVal = 0; - - /* If there is no read or write transaction open, read the latest - ** tree-header from shared-memory to report on. If necessary, update - ** it based on the contents of the database header. - ** - ** No locks are taken here - these are passive read operations only. - */ - if( pDb->pCsr==0 && pDb->nTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK ) rc = logReclaimSpace(pDb); - } - - if( rc==LSM_OK ){ - DbLog *pLog = &pDb->treehdr.log; - zVal = lsmMallocPrintf(pDb->pEnv, - "%d %d %d %d %d %d", - (int)pLog->aRegion[0].iStart, (int)pLog->aRegion[0].iEnd, - (int)pLog->aRegion[1].iStart, (int)pLog->aRegion[1].iEnd, - (int)pLog->aRegion[2].iStart, (int)pLog->aRegion[2].iEnd - ); - if( !zVal ) rc = LSM_NOMEM_BKPT; - } - - *pzVal = zVal; - return rc; -} - -/************************************************************************* -** Begin code for log recovery. -*/ - -typedef struct LogReader LogReader; -struct LogReader { - FileSystem *pFS; /* File system to read from */ - i64 iOff; /* File offset at end of buf content */ - int iBuf; /* Current read offset in buf */ - LsmString buf; /* Buffer containing file content */ - - int iCksumBuf; /* Offset in buf corresponding to cksum[01] */ - u32 cksum0; /* Checksum 0 at offset iCksumBuf */ - u32 cksum1; /* Checksum 1 at offset iCksumBuf */ -}; - -static void logReaderBlob( - LogReader *p, /* Log reader object */ - LsmString *pBuf, /* Dynamic storage, if required */ - int nBlob, /* Number of bytes to read */ - u8 **ppBlob, /* OUT: Pointer to blob read */ - int *pRc /* IN/OUT: Error code */ -){ - static const int LOG_READ_SIZE = 512; - int rc = *pRc; /* Return code */ - int nReq = nBlob; /* Bytes required */ - - while( rc==LSM_OK && nReq>0 ){ - int nAvail; /* Bytes of data available in p->buf */ - if( p->buf.n==p->iBuf ){ - int nCksum; /* Total bytes requiring checksum */ - int nCarry = 0; /* Total bytes requiring checksum */ - - nCksum = p->iBuf - p->iCksumBuf; - if( nCksum>0 ){ - nCarry = nCksum % 8; - nCksum = ((nCksum / 8) * 8); - if( nCksum>0 ){ - logCksumUnaligned( - &p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1 - ); - } - } - if( nCarry>0 ) memcpy(p->buf.z, &p->buf.z[p->iBuf-nCarry], nCarry); - p->buf.n = nCarry; - p->iBuf = nCarry; - - rc = lsmFsReadLog(p->pFS, p->iOff, LOG_READ_SIZE, &p->buf); - if( rc!=LSM_OK ) break; - p->iCksumBuf = 0; - p->iOff += LOG_READ_SIZE; - } - - nAvail = p->buf.n - p->iBuf; - if( ppBlob && nReq==nBlob && nBlob<=nAvail ){ - *ppBlob = (u8 *)&p->buf.z[p->iBuf]; - p->iBuf += nBlob; - nReq = 0; - }else{ - int nCopy = LSM_MIN(nAvail, nReq); - if( nBlob==nReq ){ - pBuf->n = 0; - } - rc = lsmStringBinAppend(pBuf, (u8 *)&p->buf.z[p->iBuf], nCopy); - nReq -= nCopy; - p->iBuf += nCopy; - if( nReq==0 && ppBlob ){ - *ppBlob = (u8*)pBuf->z; - } - } - } - - *pRc = rc; -} - -static void logReaderVarint( - LogReader *p, - LsmString *pBuf, - int *piVal, /* OUT: Value read from log */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==LSM_OK ){ - u8 *aVarint; - if( p->buf.n==p->iBuf ){ - logReaderBlob(p, 0, 10, &aVarint, pRc); - if( LSM_OK==*pRc ) p->iBuf -= (10 - lsmVarintGet32(aVarint, piVal)); - }else{ - logReaderBlob(p, pBuf, lsmVarintSize(p->buf.z[p->iBuf]), &aVarint, pRc); - if( LSM_OK==*pRc ) lsmVarintGet32(aVarint, piVal); - } - } -} - -static void logReaderByte(LogReader *p, u8 *pByte, int *pRc){ - u8 *pPtr = 0; - logReaderBlob(p, 0, 1, &pPtr, pRc); - if( pPtr ) *pByte = *pPtr; -} - -static void logReaderCksum(LogReader *p, LsmString *pBuf, int *pbEof, int *pRc){ - if( *pRc==LSM_OK ){ - u8 *pPtr = 0; - u32 cksum0, cksum1; - int nCksum = p->iBuf - p->iCksumBuf; - - /* Update in-memory (expected) checksums */ - assert( nCksum>=0 ); - logCksumUnaligned(&p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1); - p->iCksumBuf = p->iBuf + 8; - logReaderBlob(p, pBuf, 8, &pPtr, pRc); - assert( pPtr || *pRc ); - - /* Read the checksums from the log file. Set *pbEof if they do not match. */ - if( pPtr ){ - cksum0 = lsmGetU32(pPtr); - cksum1 = lsmGetU32(&pPtr[4]); - *pbEof = (cksum0!=p->cksum0 || cksum1!=p->cksum1); - p->iCksumBuf = p->iBuf; - } - } -} - -static void logReaderInit( - lsm_db *pDb, /* Database handle */ - DbLog *pLog, /* Log object associated with pDb */ - int bInitBuf, /* True if p->buf is uninitialized */ - LogReader *p /* Initialize this LogReader object */ -){ - p->pFS = pDb->pFS; - p->iOff = pLog->aRegion[2].iStart; - p->cksum0 = pLog->cksum0; - p->cksum1 = pLog->cksum1; - if( bInitBuf ){ lsmStringInit(&p->buf, pDb->pEnv); } - p->buf.n = 0; - p->iCksumBuf = 0; - p->iBuf = 0; -} - -/* -** This function is called after reading the header of a LOG_DELETE or -** LOG_WRITE record. Parameter nByte is the total size of the key and -** value that follow the header just read. Return true if the size and -** position of the record indicate that it should contain a checksum. -*/ -static int logRequireCksum(LogReader *p, int nByte){ - return ((p->iBuf + nByte - p->iCksumBuf) > LSM_CKSUM_MAXDATA); -} - -/* -** Recover the contents of the log file. -*/ -int lsmLogRecover(lsm_db *pDb){ - LsmString buf1; /* Key buffer */ - LsmString buf2; /* Value buffer */ - LogReader reader; /* Log reader object */ - int rc = LSM_OK; /* Return code */ - int nCommit = 0; /* Number of transactions to recover */ - int iPass; - int nJump = 0; /* Number of LSM_LOG_JUMP records in pass 0 */ - DbLog *pLog; - int bOpen; - - rc = lsmFsOpenLog(pDb, &bOpen); - if( rc!=LSM_OK ) return rc; - - rc = lsmTreeInit(pDb); - if( rc!=LSM_OK ) return rc; - - pLog = &pDb->treehdr.log; - lsmCheckpointLogoffset(pDb->pShmhdr->aSnap2, pLog); - - logReaderInit(pDb, pLog, 1, &reader); - lsmStringInit(&buf1, pDb->pEnv); - lsmStringInit(&buf2, pDb->pEnv); - - /* The outer for() loop runs at most twice. The first iteration is to - ** count the number of committed transactions in the log. The second - ** iterates through those transactions and updates the in-memory tree - ** structure with their contents. */ - if( bOpen ){ - for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ - int bEof = 0; - - while( rc==LSM_OK && !bEof ){ - u8 eType = 0; - logReaderByte(&reader, &eType, &rc); - - switch( eType ){ - case LSM_LOG_PAD1: - break; - - case LSM_LOG_PAD2: { - int nPad; - logReaderVarint(&reader, &buf1, &nPad, &rc); - logReaderBlob(&reader, &buf1, nPad, 0, &rc); - break; - } - - case LSM_LOG_DRANGE: - case LSM_LOG_DRANGE_CKSUM: - case LSM_LOG_WRITE: - case LSM_LOG_WRITE_CKSUM: { - int nKey; - int nVal; - u8 *aVal; - logReaderVarint(&reader, &buf1, &nKey, &rc); - logReaderVarint(&reader, &buf2, &nVal, &rc); - - if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey+nVal); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, 0, &rc); - logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); - if( iPass==1 && rc==LSM_OK ){ - if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){ - rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - }else{ - rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - } - } - break; - } - - case LSM_LOG_DELETE: - case LSM_LOG_DELETE_CKSUM: { - int nKey; u8 *aKey; - logReaderVarint(&reader, &buf1, &nKey, &rc); - - if( eType==LSM_LOG_DELETE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); - } - break; - } - - case LSM_LOG_COMMIT: - logReaderCksum(&reader, &buf1, &bEof, &rc); - if( bEof==0 ){ - nCommit++; - assert( nCommit>0 || iPass==1 ); - if( nCommit==0 ) bEof = 1; - } - break; - - case LSM_LOG_JUMP: { - int iOff = 0; - logReaderVarint(&reader, &buf1, &iOff, &rc); - if( rc==LSM_OK ){ - if( iPass==1 ){ - if( pLog->aRegion[2].iStart==0 ){ - assert( pLog->aRegion[1].iStart==0 ); - pLog->aRegion[1].iEnd = reader.iOff; - }else{ - assert( pLog->aRegion[0].iStart==0 ); - pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[0].iEnd = reader.iOff-reader.buf.n+reader.iBuf; - } - pLog->aRegion[2].iStart = iOff; - }else{ - if( (nJump++)==2 ){ - bEof = 1; - } - } - - reader.iOff = iOff; - reader.buf.n = reader.iBuf; - } - break; - } - - default: - /* Including LSM_LOG_EOF */ - bEof = 1; - break; - } - } - - if( rc==LSM_OK && iPass==0 ){ - if( nCommit==0 ){ - if( pLog->aRegion[2].iStart==0 ){ - iPass = 1; - }else{ - pLog->aRegion[2].iStart = 0; - iPass = -1; - lsmCheckpointZeroLogoffset(pDb); - } - } - logReaderInit(pDb, pLog, 0, &reader); - nCommit = nCommit * -1; - } - } - } - - /* Initialize DbLog object */ - if( rc==LSM_OK ){ - pLog->aRegion[2].iEnd = reader.iOff - reader.buf.n + reader.iBuf; - pLog->cksum0 = reader.cksum0; - pLog->cksum1 = reader.cksum1; - } - - if( rc==LSM_OK ){ - rc = lsmFinishRecovery(pDb); - }else{ - lsmFinishRecovery(pDb); - } - - if( pDb->bRoTrans ){ - lsmFsCloseLog(pDb); - } - - lsmStringClear(&buf1); - lsmStringClear(&buf2); - lsmStringClear(&reader.buf); - return rc; -} - -void lsmLogClose(lsm_db *db){ - if( db->pLogWriter ){ - lsmFree(db->pEnv, db->pLogWriter->buf.z); - lsmFree(db->pEnv, db->pLogWriter); - db->pLogWriter = 0; - } -} diff --git a/ext/lsm1/lsm_main.c b/ext/lsm1/lsm_main.c deleted file mode 100644 index f2b353105a..0000000000 --- a/ext/lsm1/lsm_main.c +++ /dev/null @@ -1,1008 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** The main interface to the LSM module. -*/ -#include "lsmInt.h" - - -#ifdef LSM_DEBUG -/* -** This function returns a copy of its only argument. -** -** When the library is built with LSM_DEBUG defined, this function is called -** whenever an error code is generated (not propagated - generated). So -** if the library is mysteriously returning (say) LSM_IOERR, a breakpoint -** may be set in this function to determine why. -*/ -int lsmErrorBkpt(int rc){ - /* Set breakpoint here! */ - return rc; -} - -/* -** This function contains various assert() statements that test that the -** lsm_db structure passed as an argument is internally consistent. -*/ -static void assert_db_state(lsm_db *pDb){ - - /* If there is at least one cursor or a write transaction open, the database - ** handle must be holding a pointer to a client snapshot. And the reverse - ** - if there are no open cursors and no write transactions then there must - ** not be a client snapshot. */ - - assert( (pDb->pCsr!=0||pDb->nTransOpen>0)==(pDb->iReader>=0||pDb->bRoTrans) ); - - assert( (pDb->iReader<0 && pDb->bRoTrans==0) || pDb->pClient!=0 ); - - assert( pDb->nTransOpen>=0 ); -} -#else -# define assert_db_state(x) -#endif - -/* -** The default key-compare function. -*/ -static int xCmp(void *p1, int n1, void *p2, int n2){ - int res; - res = memcmp(p1, p2, LSM_MIN(n1, n2)); - if( res==0 ) res = (n1-n2); - return res; -} - -static void xLog(void *pCtx, int rc, const char *z){ - (void)(rc); - (void)(pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -/* -** Allocate a new db handle. -*/ -int lsm_new(lsm_env *pEnv, lsm_db **ppDb){ - lsm_db *pDb; - - /* If the user did not provide an environment, use the default. */ - if( pEnv==0 ) pEnv = lsm_default_env(); - assert( pEnv ); - - /* Allocate the new database handle */ - *ppDb = pDb = (lsm_db *)lsmMallocZero(pEnv, sizeof(lsm_db)); - if( pDb==0 ) return LSM_NOMEM_BKPT; - - /* Initialize the new object */ - pDb->pEnv = pEnv; - pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; - pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; - pDb->bAutowork = LSM_DFLT_AUTOWORK; - pDb->eSafety = LSM_DFLT_SAFETY; - pDb->xCmp = xCmp; - pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; - pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; - pDb->nMerge = LSM_DFLT_AUTOMERGE; - pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; - pDb->bUseLog = LSM_DFLT_USE_LOG; - pDb->iReader = -1; - pDb->iRwclient = -1; - pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; - pDb->iMmap = LSM_DFLT_MMAP; - pDb->xLog = xLog; - pDb->compress.iId = LSM_COMPRESSION_NONE; - return LSM_OK; -} - -lsm_env *lsm_get_env(lsm_db *pDb){ - assert( pDb->pEnv ); - return pDb->pEnv; -} - -/* -** If database handle pDb is currently holding a client snapshot, but does -** not have any open cursors or write transactions, release it. -*/ -static void dbReleaseClientSnapshot(lsm_db *pDb){ - if( pDb->nTransOpen==0 && pDb->pCsr==0 ){ - lsmFinishReadTrans(pDb); - } -} - -static int getFullpathname( - lsm_env *pEnv, - const char *zRel, - char **pzAbs -){ - int nAlloc = 0; - char *zAlloc = 0; - int nReq = 0; - int rc; - - do{ - nAlloc = nReq; - rc = pEnv->xFullpath(pEnv, zRel, zAlloc, &nReq); - if( nReq>nAlloc ){ - zAlloc = lsmReallocOrFreeRc(pEnv, zAlloc, nReq, &rc); - } - }while( nReq>nAlloc && rc==LSM_OK ); - - if( rc!=LSM_OK ){ - lsmFree(pEnv, zAlloc); - zAlloc = 0; - } - *pzAbs = zAlloc; - return rc; -} - -/* -** Check that the bits in the db->mLock mask are consistent with the -** value stored in db->iRwclient. An assert shall fail otherwise. -*/ -static void assertRwclientLockValue(lsm_db *db){ -#ifndef NDEBUG - u64 msk; /* Mask of mLock bits for RWCLIENT locks */ - u64 rwclient = 0; /* Bit corresponding to db->iRwclient */ - - if( db->iRwclient>=0 ){ - rwclient = ((u64)1 << (LSM_LOCK_RWCLIENT(db->iRwclient)-1)); - } - msk = ((u64)1 << (LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT)-1)) - 1; - msk -= (((u64)1 << (LSM_LOCK_RWCLIENT(0)-1)) - 1); - - assert( (db->mLock & msk)==rwclient ); -#endif -} - -/* -** Open a new connection to database zFilename. -*/ -int lsm_open(lsm_db *pDb, const char *zFilename){ - int rc; - - if( pDb->pDatabase ){ - rc = LSM_MISUSE; - }else{ - char *zFull; - - /* Translate the possibly relative pathname supplied by the user into - ** an absolute pathname. This is required because the supplied path - ** is used (either directly or with "-log" appended to it) for more - ** than one purpose - to open both the database and log files, and - ** perhaps to unlink the log file during disconnection. An absolute - ** path is required to ensure that the correct files are operated - ** on even if the application changes the cwd. */ - rc = getFullpathname(pDb->pEnv, zFilename, &zFull); - assert( rc==LSM_OK || zFull==0 ); - - /* Connect to the database. */ - if( rc==LSM_OK ){ - rc = lsmDbDatabaseConnect(pDb, zFull); - } - - if( pDb->bReadonly==0 ){ - /* Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so - ** are guaranteed not to change during the lifetime of this connection. - */ - if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ - lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); - lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); - } - } - - lsmFree(pDb->pEnv, zFull); - assertRwclientLockValue(pDb); - } - - assert( pDb->bReadonly==0 || pDb->bReadonly==1 ); - assert( rc!=LSM_OK || (pDb->pShmhdr==0)==(pDb->bReadonly==1) ); - - return rc; -} - -int lsm_close(lsm_db *pDb){ - int rc = LSM_OK; - if( pDb ){ - assert_db_state(pDb); - if( pDb->pCsr || pDb->nTransOpen ){ - rc = LSM_MISUSE_BKPT; - }else{ - lsmMCursorFreeCache(pDb); - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - - assertRwclientLockValue(pDb); - - lsmDbDatabaseRelease(pDb); - lsmLogClose(pDb); - lsmFsClose(pDb->pFS); - /* assert( pDb->mLock==0 ); */ - - /* Invoke any destructors registered for the compression or - ** compression factory callbacks. */ - if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); - if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); - - lsmFree(pDb->pEnv, pDb->rollback.aArray); - lsmFree(pDb->pEnv, pDb->aTrans); - lsmFree(pDb->pEnv, pDb->apShm); - lsmFree(pDb->pEnv, pDb); - } - } - return rc; -} - -int lsm_config(lsm_db *pDb, int eParam, ...){ - int rc = LSM_OK; - va_list ap; - va_start(ap, eParam); - - switch( eParam ){ - case LSM_CONFIG_AUTOFLUSH: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ - int *piVal = va_arg(ap, int *); - int iVal = *piVal; - if( iVal>=0 && iVal<=(1024*1024) ){ - pDb->nTreeLimit = iVal*1024; - } - *piVal = (pDb->nTreeLimit / 1024); - break; - } - - case LSM_CONFIG_AUTOWORK: { - int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - pDb->bAutowork = *piVal; - } - *piVal = pDb->bAutowork; - break; - } - - case LSM_CONFIG_AUTOCHECKPOINT: { - /* This parameter is read and written in KB. But all internal processing - ** (including the lsm_db.nAutockpt variable) is done in bytes. */ - int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - int iVal = *piVal; - pDb->nAutockpt = (i64)iVal * 1024; - } - *piVal = (int)(pDb->nAutockpt / 1024); - break; - } - - case LSM_CONFIG_PAGE_SIZE: { - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the page-size according to the - ** FileSystem object. */ - *piVal = lsmFsPageSize(pDb->pFS); - }else{ - if( *piVal>=256 && *piVal<=65536 && ((*piVal-1) & *piVal)==0 ){ - pDb->nDfltPgsz = *piVal; - }else{ - *piVal = pDb->nDfltPgsz; - } - } - break; - } - - case LSM_CONFIG_BLOCK_SIZE: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the block-size in KB according to the - ** FileSystem object. */ - *piVal = lsmFsBlockSize(pDb->pFS) / 1024; - }else{ - int iVal = *piVal; - if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ - pDb->nDfltBlksz = iVal * 1024; - }else{ - *piVal = pDb->nDfltBlksz / 1024; - } - } - break; - } - - case LSM_CONFIG_SAFETY: { - int *piVal = va_arg(ap, int *); - if( *piVal>=0 && *piVal<=2 ){ - pDb->eSafety = *piVal; - } - *piVal = pDb->eSafety; - break; - } - - case LSM_CONFIG_MMAP: { - int *piVal = va_arg(ap, int *); - if( pDb->iReader<0 && *piVal>=0 ){ - pDb->iMmap = *piVal; - rc = lsmFsConfigure(pDb); - } - *piVal = pDb->iMmap; - break; - } - - case LSM_CONFIG_USE_LOG: { - int *piVal = va_arg(ap, int *); - if( pDb->nTransOpen==0 && (*piVal==0 || *piVal==1) ){ - pDb->bUseLog = *piVal; - } - *piVal = pDb->bUseLog; - break; - } - - case LSM_CONFIG_AUTOMERGE: { - int *piVal = va_arg(ap, int *); - if( *piVal>1 ) pDb->nMerge = *piVal; - *piVal = pDb->nMerge; - break; - } - - case LSM_CONFIG_MAX_FREELIST: { - int *piVal = va_arg(ap, int *); - if( *piVal>=2 && *piVal<=LSM_MAX_FREELIST_ENTRIES ){ - pDb->nMaxFreelist = *piVal; - } - *piVal = pDb->nMaxFreelist; - break; - } - - case LSM_CONFIG_MULTIPLE_PROCESSES: { - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to true if this connection is currently - ** in multi-process mode. */ - *piVal = lsmDbMultiProc(pDb); - }else{ - pDb->bMultiProc = *piVal = (*piVal!=0); - } - break; - } - - case LSM_CONFIG_READONLY: { - int *piVal = va_arg(ap, int *); - /* If lsm_open() has been called, this is a read-only parameter. */ - if( pDb->pDatabase==0 && *piVal>=0 ){ - pDb->bReadonly = *piVal = (*piVal!=0); - } - *piVal = pDb->bReadonly; - break; - } - - case LSM_CONFIG_SET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - if( pDb->iReader>=0 && pDb->bInFactory==0 ){ - /* May not change compression schemes with an open transaction */ - rc = LSM_MISUSE_BKPT; - }else{ - if( pDb->compress.xFree ){ - /* Invoke any destructor belonging to the current compression. */ - pDb->compress.xFree(pDb->compress.pCtx); - } - if( p->xBound==0 ){ - memset(&pDb->compress, 0, sizeof(lsm_compress)); - pDb->compress.iId = LSM_COMPRESSION_NONE; - }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); - } - rc = lsmFsConfigure(pDb); - } - break; - } - - case LSM_CONFIG_SET_COMPRESSION_FACTORY: { - lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); - if( pDb->factory.xFree ){ - /* Invoke any destructor belonging to the current factory. */ - pDb->factory.xFree(pDb->factory.pCtx); - } - memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); - break; - } - - case LSM_CONFIG_GET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - memcpy(p, &pDb->compress, sizeof(lsm_compress)); - break; - } - - default: - rc = LSM_MISUSE; - break; - } - - va_end(ap); - return rc; -} - -void lsmAppendSegmentList(LsmString *pStr, char *zPre, Segment *pSeg){ - lsmStringAppendf(pStr, "%s{%lld %lld %lld %lld}", zPre, - pSeg->iFirst, pSeg->iLastPg, pSeg->iRoot, pSeg->nSize - ); -} - -static int infoGetWorker(lsm_db *pDb, Snapshot **pp, int *pbUnlock){ - int rc = LSM_OK; - - assert( *pbUnlock==0 ); - if( !pDb->pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - *pbUnlock = 1; - } - if( pp ) *pp = pDb->pWorker; - return rc; -} - -static void infoFreeWorker(lsm_db *pDb, int bUnlock){ - if( bUnlock ){ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - } -} - -int lsmStructList( - lsm_db *pDb, /* Database handle */ - char **pzOut /* OUT: Nul-terminated string (tcl list) */ -){ - Level *pTopLevel = 0; /* Top level of snapshot to report on */ - int rc = LSM_OK; - Level *p; - LsmString s; - Snapshot *pWorker; /* Worker snapshot */ - int bUnlock = 0; - - /* Obtain the worker snapshot */ - rc = infoGetWorker(pDb, &pWorker, &bUnlock); - if( rc!=LSM_OK ) return rc; - - /* Format the contents of the snapshot as text */ - pTopLevel = lsmDbSnapshotLevel(pWorker); - lsmStringInit(&s, pDb->pEnv); - for(p=pTopLevel; rc==LSM_OK && p; p=p->pNext){ - int i; - lsmStringAppendf(&s, "%s{%d", (s.n ? " " : ""), (int)p->iAge); - lsmAppendSegmentList(&s, " ", &p->lhs); - for(i=0; rc==LSM_OK && inRight; i++){ - lsmAppendSegmentList(&s, " ", &p->aRhs[i]); - } - lsmStringAppend(&s, "}", 1); - } - rc = s.n>=0 ? LSM_OK : LSM_NOMEM; - - /* Release the snapshot and return */ - infoFreeWorker(pDb, bUnlock); - *pzOut = s.z; - return rc; -} - -static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - LsmString *pStr = (LsmString *)pCtx; - lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); - return 0; -} - -int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ - Snapshot *pWorker; /* Worker snapshot */ - int bUnlock = 0; - LsmString s; - int rc; - - /* Obtain the worker snapshot */ - rc = infoGetWorker(pDb, &pWorker, &bUnlock); - if( rc!=LSM_OK ) return rc; - - lsmStringInit(&s, pDb->pEnv); - rc = lsmWalkFreelist(pDb, 0, infoFreelistCb, &s); - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, s.z); - }else{ - *pzOut = s.z; - } - - /* Release the snapshot and return */ - infoFreeWorker(pDb, bUnlock); - return rc; -} - -static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ - ShmHeader *pShm = db->pShmhdr; - TreeHeader *p = &pShm->hdr1; - - /* The following code suffers from two race conditions, as it accesses and - ** trusts the contents of shared memory without verifying checksums: - ** - ** * The two values read - TreeHeader.root.nByte and oldroot.nByte - are - ** 32-bit fields. It is assumed that reading from one of these - ** is atomic - that it is not possible to read a partially written - ** garbage value. However the two values may be mutually inconsistent. - ** - ** * TreeHeader.iLogOff is a 64-bit value. And lsmCheckpointLogOffset() - ** reads a 64-bit value from a snapshot stored in shared memory. It - ** is assumed that in each case it is possible to read a partially - ** written garbage value. If this occurs, then the value returned - ** for the size of the "old" tree may reflect the size of an "old" - ** tree that was recently flushed to disk. - ** - ** Given the context in which this function is called (as a result of an - ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to - ** be problems. - */ - *pnNewKB = ((int)p->root.nByte + 1023) / 1024; - if( p->iOldShmid ){ - if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ - *pnOldKB = 0; - }else{ - *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; - } - }else{ - *pnOldKB = 0; - } - - return LSM_OK; -} - -int lsm_info(lsm_db *pDb, int eParam, ...){ - int rc = LSM_OK; - va_list ap; - va_start(ap, eParam); - - switch( eParam ){ - case LSM_INFO_NWRITE: { - int *piVal = va_arg(ap, int *); - *piVal = lsmFsNWrite(pDb->pFS); - break; - } - - case LSM_INFO_NREAD: { - int *piVal = va_arg(ap, int *); - *piVal = lsmFsNRead(pDb->pFS); - break; - } - - case LSM_INFO_DB_STRUCTURE: { - char **pzVal = va_arg(ap, char **); - rc = lsmStructList(pDb, pzVal); - break; - } - - case LSM_INFO_ARRAY_STRUCTURE: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); - break; - } - - case LSM_INFO_ARRAY_PAGES: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayPages(pDb, pgno, pzVal); - break; - } - - case LSM_INFO_PAGE_HEX_DUMP: - case LSM_INFO_PAGE_ASCII_DUMP: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - int bUnlock = 0; - rc = infoGetWorker(pDb, 0, &bUnlock); - if( rc==LSM_OK ){ - int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); - rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal); - } - infoFreeWorker(pDb, bUnlock); - break; - } - - case LSM_INFO_LOG_STRUCTURE: { - char **pzVal = va_arg(ap, char **); - rc = lsmInfoLogStructure(pDb, pzVal); - break; - } - - case LSM_INFO_FREELIST: { - char **pzVal = va_arg(ap, char **); - rc = lsmInfoFreelist(pDb, pzVal); - break; - } - - case LSM_INFO_CHECKPOINT_SIZE: { - int *pnKB = va_arg(ap, int *); - rc = lsmCheckpointSize(pDb, pnKB); - break; - } - - case LSM_INFO_TREE_SIZE: { - int *pnOld = va_arg(ap, int *); - int *pnNew = va_arg(ap, int *); - rc = infoTreeSize(pDb, pnOld, pnNew); - break; - } - - case LSM_INFO_COMPRESSION_ID: { - unsigned int *piOut = va_arg(ap, unsigned int *); - if( pDb->pClient ){ - *piOut = pDb->pClient->iCmpId; - }else{ - rc = lsmInfoCompressionId(pDb, piOut); - } - break; - } - - default: - rc = LSM_MISUSE; - break; - } - - va_end(ap); - return rc; -} - -static int doWriteOp( - lsm_db *pDb, - int bDeleteRange, - const void *pKey, int nKey, /* Key to write or delete */ - const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ -){ - int rc = LSM_OK; /* Return code */ - int bCommit = 0; /* True to commit before returning */ - - if( pDb->nTransOpen==0 ){ - bCommit = 1; - rc = lsm_begin(pDb, 1); - } - - if( rc==LSM_OK ){ - int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE)); - rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal); - } - - lsmSortedSaveTreeCursors(pDb); - - if( rc==LSM_OK ){ - int pgsz = lsmFsPageSize(pDb->pFS); - int nQuant = LSM_AUTOWORK_QUANT * pgsz; - int nBefore; - int nAfter; - int nDiff; - - if( nQuant>pDb->nTreeLimit ){ - nQuant = LSM_MAX(pDb->nTreeLimit, pgsz); - } - - nBefore = lsmTreeSize(pDb); - if( bDeleteRange ){ - rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal); - }else{ - rc = lsmTreeInsert(pDb, (void *)pKey, nKey, (void *)pVal, nVal); - } - - nAfter = lsmTreeSize(pDb); - nDiff = (nAfter/nQuant) - (nBefore/nQuant); - if( rc==LSM_OK && pDb->bAutowork && nDiff!=0 ){ - rc = lsmSortedAutoWork(pDb, nDiff * LSM_AUTOWORK_QUANT); - } - } - - /* If a transaction was opened at the start of this function, commit it. - ** Or, if an error has occurred, roll it back. */ - if( bCommit ){ - if( rc==LSM_OK ){ - rc = lsm_commit(pDb, 0); - }else{ - lsm_rollback(pDb, 0); - } - } - - return rc; -} - -/* -** Write a new value into the database. -*/ -int lsm_insert( - lsm_db *db, /* Database connection */ - const void *pKey, int nKey, /* Key to write or delete */ - const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ -){ - return doWriteOp(db, 0, pKey, nKey, pVal, nVal); -} - -/* -** Delete a value from the database. -*/ -int lsm_delete(lsm_db *db, const void *pKey, int nKey){ - return doWriteOp(db, 0, pKey, nKey, 0, -1); -} - -/* -** Delete a range of database keys. -*/ -int lsm_delete_range( - lsm_db *db, /* Database handle */ - const void *pKey1, int nKey1, /* Lower bound of range to delete */ - const void *pKey2, int nKey2 /* Upper bound of range to delete */ -){ - int rc = LSM_OK; - if( db->xCmp((void *)pKey1, nKey1, (void *)pKey2, nKey2)<0 ){ - rc = doWriteOp(db, 1, pKey1, nKey1, pKey2, nKey2); - } - return rc; -} - -/* -** Open a new cursor handle. -** -** If there are currently no other open cursor handles, and no open write -** transaction, open a read transaction here. -*/ -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr){ - int rc = LSM_OK; /* Return code */ - MultiCursor *pCsr = 0; /* New cursor object */ - - /* Open a read transaction if one is not already open. */ - assert_db_state(pDb); - - if( pDb->pShmhdr==0 ){ - assert( pDb->bReadonly ); - rc = lsmBeginRoTrans(pDb); - }else if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Allocate the multi-cursor. */ - if( rc==LSM_OK ){ - rc = lsmMCursorNew(pDb, &pCsr); - } - - /* If an error has occured, set the output to NULL and delete any partially - ** allocated cursor. If this means there are no open cursors, release the - ** client snapshot. */ - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - dbReleaseClientSnapshot(pDb); - } - - assert_db_state(pDb); - *ppCsr = (lsm_cursor *)pCsr; - return rc; -} - -/* -** Close a cursor opened using lsm_csr_open(). -*/ -int lsm_csr_close(lsm_cursor *p){ - if( p ){ - lsm_db *pDb = lsmMCursorDb((MultiCursor *)p); - assert_db_state(pDb); - lsmMCursorClose((MultiCursor *)p, 1); - dbReleaseClientSnapshot(pDb); - assert_db_state(pDb); - } - return LSM_OK; -} - -/* -** Attempt to seek the cursor to the database entry specified by pKey/nKey. -** If an error occurs (e.g. an OOM or IO error), return an LSM error code. -** Otherwise, return LSM_OK. -*/ -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek){ - return lsmMCursorSeek((MultiCursor *)pCsr, 0, (void *)pKey, nKey, eSeek); -} - -int lsm_csr_next(lsm_cursor *pCsr){ - return lsmMCursorNext((MultiCursor *)pCsr); -} - -int lsm_csr_prev(lsm_cursor *pCsr){ - return lsmMCursorPrev((MultiCursor *)pCsr); -} - -int lsm_csr_first(lsm_cursor *pCsr){ - return lsmMCursorFirst((MultiCursor *)pCsr); -} - -int lsm_csr_last(lsm_cursor *pCsr){ - return lsmMCursorLast((MultiCursor *)pCsr); -} - -int lsm_csr_valid(lsm_cursor *pCsr){ - return lsmMCursorValid((MultiCursor *)pCsr); -} - -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey){ - return lsmMCursorKey((MultiCursor *)pCsr, (void **)ppKey, pnKey); -} - -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal){ - return lsmMCursorValue((MultiCursor *)pCsr, (void **)ppVal, pnVal); -} - -void lsm_config_log( - lsm_db *pDb, - void (*xLog)(void *, int, const char *), - void *pCtx -){ - pDb->xLog = xLog; - pDb->pLogCtx = pCtx; -} - -void lsm_config_work_hook( - lsm_db *pDb, - void (*xWork)(lsm_db *, void *), - void *pCtx -){ - pDb->xWork = xWork; - pDb->pWorkCtx = pCtx; -} - -void lsmLogMessage(lsm_db *pDb, int rc, const char *zFormat, ...){ - if( pDb->xLog ){ - LsmString s; - va_list ap, ap2; - lsmStringInit(&s, pDb->pEnv); - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(&s, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); - pDb->xLog(pDb->pLogCtx, rc, s.z); - lsmStringClear(&s); - } -} - -int lsm_begin(lsm_db *pDb, int iLevel){ - int rc; - - assert_db_state( pDb ); - rc = (pDb->bReadonly ? LSM_READONLY : LSM_OK); - - /* A value less than zero means open one more transaction. */ - if( iLevel<0 ) iLevel = pDb->nTransOpen + 1; - if( iLevel>pDb->nTransOpen ){ - int i; - - /* Extend the pDb->aTrans[] array if required. */ - if( rc==LSM_OK && pDb->nTransAllocpEnv, pDb->aTrans, nByte); - if( !aNew ){ - rc = LSM_NOMEM; - }else{ - nByte = sizeof(TransMark) * (iLevel+1 - pDb->nTransAlloc); - memset(&aNew[pDb->nTransAlloc], 0, nByte); - pDb->nTransAlloc = iLevel+1; - pDb->aTrans = aNew; - } - } - - if( rc==LSM_OK && pDb->nTransOpen==0 ){ - rc = lsmBeginWriteTrans(pDb); - } - - if( rc==LSM_OK ){ - for(i=pDb->nTransOpen; iaTrans[i].tree); - lsmLogTell(pDb, &pDb->aTrans[i].log); - } - pDb->nTransOpen = iLevel; - } - } - - return rc; -} - -int lsm_commit(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; - - assert_db_state( pDb ); - - /* A value less than zero means close the innermost nested transaction. */ - if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); - - if( iLevelnTransOpen ){ - if( iLevel==0 ){ - int rc2; - /* Commit the transaction to disk. */ - if( rc==LSM_OK ) rc = lsmLogCommit(pDb); - if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ - rc = lsmFsSyncLog(pDb->pFS); - } - rc2 = lsmFinishWriteTrans(pDb, (rc==LSM_OK)); - if( rc==LSM_OK ) rc = rc2; - } - pDb->nTransOpen = iLevel; - } - dbReleaseClientSnapshot(pDb); - return rc; -} - -int lsm_rollback(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; - assert_db_state( pDb ); - - if( pDb->nTransOpen ){ - /* A value less than zero means close the innermost nested transaction. */ - if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); - - if( iLevel<=pDb->nTransOpen ){ - TransMark *pMark = &pDb->aTrans[(iLevel==0 ? 0 : iLevel-1)]; - lsmTreeRollback(pDb, &pMark->tree); - if( iLevel ) lsmLogSeek(pDb, &pMark->log); - pDb->nTransOpen = iLevel; - } - - if( pDb->nTransOpen==0 ){ - lsmFinishWriteTrans(pDb, 0); - } - dbReleaseClientSnapshot(pDb); - } - - return rc; -} - -int lsm_get_user_version(lsm_db *pDb, unsigned int *piUsr){ - int rc = LSM_OK; /* Return code */ - - /* Open a read transaction if one is not already open. */ - assert_db_state(pDb); - if( pDb->pShmhdr==0 ){ - assert( pDb->bReadonly ); - rc = lsmBeginRoTrans(pDb); - }else if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Allocate the multi-cursor. */ - if( rc==LSM_OK ){ - *piUsr = pDb->treehdr.iUsrVersion; - } - - dbReleaseClientSnapshot(pDb); - assert_db_state(pDb); - return rc; -} - -int lsm_set_user_version(lsm_db *pDb, unsigned int iUsr){ - int rc = LSM_OK; /* Return code */ - int bCommit = 0; /* True to commit before returning */ - - if( pDb->nTransOpen==0 ){ - bCommit = 1; - rc = lsm_begin(pDb, 1); - } - - if( rc==LSM_OK ){ - pDb->treehdr.iUsrVersion = iUsr; - } - - /* If a transaction was opened at the start of this function, commit it. - ** Or, if an error has occurred, roll it back. */ - if( bCommit ){ - if( rc==LSM_OK ){ - rc = lsm_commit(pDb, 0); - }else{ - lsm_rollback(pDb, 0); - } - } - - return rc; -} diff --git a/ext/lsm1/lsm_mem.c b/ext/lsm1/lsm_mem.c deleted file mode 100644 index 13dd9fe312..0000000000 --- a/ext/lsm1/lsm_mem.c +++ /dev/null @@ -1,104 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Helper routines for memory allocation. -*/ -#include "lsmInt.h" - -/* -** The following routines are called internally by LSM sub-routines. In -** this case a valid environment pointer must be supplied. -*/ -void *lsmMalloc(lsm_env *pEnv, size_t N){ - assert( pEnv ); - return pEnv->xMalloc(pEnv, N); -} -void lsmFree(lsm_env *pEnv, void *p){ - assert( pEnv ); - pEnv->xFree(pEnv, p); -} -void *lsmRealloc(lsm_env *pEnv, void *p, size_t N){ - assert( pEnv ); - return pEnv->xRealloc(pEnv, p, N); -} - -/* -** Core memory allocation routines for LSM. -*/ -void *lsm_malloc(lsm_env *pEnv, size_t N){ - return lsmMalloc(pEnv ? pEnv : lsm_default_env(), N); -} -void lsm_free(lsm_env *pEnv, void *p){ - lsmFree(pEnv ? pEnv : lsm_default_env(), p); -} -void *lsm_realloc(lsm_env *pEnv, void *p, size_t N){ - return lsmRealloc(pEnv ? pEnv : lsm_default_env(), p, N); -} - -void *lsmMallocZero(lsm_env *pEnv, size_t N){ - void *pRet; - assert( pEnv ); - pRet = lsmMalloc(pEnv, N); - if( pRet ) memset(pRet, 0, N); - return pRet; -} - -void *lsmMallocRc(lsm_env *pEnv, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc==LSM_OK ){ - pRet = lsmMalloc(pEnv, N); - if( pRet==0 ){ - *pRc = LSM_NOMEM_BKPT; - } - } - return pRet; -} - -void *lsmMallocZeroRc(lsm_env *pEnv, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc==LSM_OK ){ - pRet = lsmMallocZero(pEnv, N); - if( pRet==0 ){ - *pRc = LSM_NOMEM_BKPT; - } - } - return pRet; -} - -void *lsmReallocOrFree(lsm_env *pEnv, void *p, size_t N){ - void *pNew; - pNew = lsm_realloc(pEnv, p, N); - if( !pNew ) lsm_free(pEnv, p); - return pNew; -} - -void *lsmReallocOrFreeRc(lsm_env *pEnv, void *p, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc ){ - lsmFree(pEnv, p); - }else{ - pRet = lsmReallocOrFree(pEnv, p, N); - if( !pRet ) *pRc = LSM_NOMEM_BKPT; - } - return pRet; -} - -char *lsmMallocStrdup(lsm_env *pEnv, const char *zIn){ - int nByte; - char *zRet; - nByte = strlen(zIn); - zRet = lsmMalloc(pEnv, nByte+1); - if( zRet ){ - memcpy(zRet, zIn, nByte+1); - } - return zRet; -} diff --git a/ext/lsm1/lsm_mutex.c b/ext/lsm1/lsm_mutex.c deleted file mode 100644 index cb99b2a61e..0000000000 --- a/ext/lsm1/lsm_mutex.c +++ /dev/null @@ -1,88 +0,0 @@ -/* -** 2012-01-30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Mutex functions for LSM. -*/ -#include "lsmInt.h" - -/* -** Allocate a new mutex. -*/ -int lsmMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - return pEnv->xMutexNew(pEnv, ppNew); -} - -/* -** Return a handle for one of the static mutexes. -*/ -int lsmMutexStatic(lsm_env *pEnv, int iMutex, lsm_mutex **ppStatic){ - return pEnv->xMutexStatic(pEnv, iMutex, ppStatic); -} - -/* -** Free a mutex allocated by lsmMutexNew(). -*/ -void lsmMutexDel(lsm_env *pEnv, lsm_mutex *pMutex){ - if( pMutex ) pEnv->xMutexDel(pMutex); -} - -/* -** Enter a mutex. -*/ -void lsmMutexEnter(lsm_env *pEnv, lsm_mutex *pMutex){ - pEnv->xMutexEnter(pMutex); -} - -/* -** Attempt to enter a mutex, but do not block. If successful, return zero. -** Otherwise, if the mutex is already held by some other thread and is not -** entered, return non zero. -** -** Each successful call to this function must be matched by a call to -** lsmMutexLeave(). -*/ -int lsmMutexTry(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexTry(pMutex); -} - -/* -** Leave a mutex. -*/ -void lsmMutexLeave(lsm_env *pEnv, lsm_mutex *pMutex){ - pEnv->xMutexLeave(pMutex); -} - -#ifndef NDEBUG -/* -** Return non-zero if the mutex passed as the second argument is held -** by the calling thread, or zero otherwise. If the implementation is not -** able to tell if the mutex is held by the caller, it should return -** non-zero. -** -** This function is only used as part of assert() statements. -*/ -int lsmMutexHeld(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexHeld ? pEnv->xMutexHeld(pMutex) : 1; -} - -/* -** Return non-zero if the mutex passed as the second argument is not -** held by the calling thread, or zero otherwise. If the implementation -** is not able to tell if the mutex is held by the caller, it should -** return non-zero. -** -** This function is only used as part of assert() statements. -*/ -int lsmMutexNotHeld(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexNotHeld ? pEnv->xMutexNotHeld(pMutex) : 1; -} -#endif diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c deleted file mode 100644 index 09f9338488..0000000000 --- a/ext/lsm1/lsm_shared.c +++ /dev/null @@ -1,1994 +0,0 @@ -/* -** 2012-01-23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Utilities used to help multiple LSM clients to coexist within the -** same process space. -*/ -#include "lsmInt.h" - -/* -** Global data. All global variables used by code in this file are grouped -** into the following structure instance. -** -** pDatabase: -** Linked list of all Database objects allocated within this process. -** This list may not be traversed without holding the global mutex (see -** functions enterGlobalMutex() and leaveGlobalMutex()). -*/ -static struct SharedData { - Database *pDatabase; /* Linked list of all Database objects */ -} gShared; - -/* -** Database structure. There is one such structure for each distinct -** database accessed by this process. They are stored in the singly linked -** list starting at global variable gShared.pDatabase. Database objects are -** reference counted. Once the number of connections to the associated -** database drops to zero, they are removed from the linked list and deleted. -** -** pFile: -** In multi-process mode, this file descriptor is used to obtain locks -** and to access shared-memory. In single process mode, its only job is -** to hold the exclusive lock on the file. -** -*/ -struct Database { - /* Protected by the global mutex (enterGlobalMutex/leaveGlobalMutex): */ - char *zName; /* Canonical path to database file */ - int nName; /* strlen(zName) */ - int nDbRef; /* Number of associated lsm_db handles */ - Database *pDbNext; /* Next Database structure in global list */ - - /* Protected by the local mutex (pClientMutex) */ - int bReadonly; /* True if Database.pFile is read-only */ - int bMultiProc; /* True if running in multi-process mode */ - lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ - LsmFile *pLsmFile; /* List of deferred closes */ - lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ - int nShmChunk; /* Number of entries in apShmChunk[] array */ - void **apShmChunk; /* Array of "shared" memory regions */ - lsm_db *pConn; /* List of connections to this db. */ -}; - -/* -** Functions to enter and leave the global mutex. This mutex is used -** to protect the global linked-list headed at gShared.pDatabase. -*/ -static int enterGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - int rc = lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - if( rc==LSM_OK ) lsmMutexEnter(pEnv, p); - return rc; -} -static void leaveGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - lsmMutexLeave(pEnv, p); -} - -#ifdef LSM_DEBUG -static int holdingGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - return lsmMutexHeld(pEnv, p); -} -#endif - -#if 0 -static void assertNotInFreelist(Freelist *p, int iBlk){ - int i; - for(i=0; inEntry; i++){ - assert( p->aEntry[i].iBlk!=iBlk ); - } -} -#else -# define assertNotInFreelist(x,y) -#endif - -/* -** Append an entry to the free-list. If (iId==-1), this is a delete. -*/ -int freelistAppend(lsm_db *db, u32 iBlk, i64 iId){ - lsm_env *pEnv = db->pEnv; - Freelist *p; - int i; - - assert( iId==-1 || iId>=0 ); - p = db->bUseFreelist ? db->pFreelist : &db->pWorker->freelist; - - /* Extend the space allocated for the freelist, if required */ - assert( p->nAlloc>=p->nEntry ); - if( p->nAlloc==p->nEntry ){ - int nNew; - int nByte; - FreelistEntry *aNew; - - nNew = (p->nAlloc==0 ? 4 : p->nAlloc*2); - nByte = sizeof(FreelistEntry) * nNew; - aNew = (FreelistEntry *)lsmRealloc(pEnv, p->aEntry, nByte); - if( !aNew ) return LSM_NOMEM_BKPT; - p->nAlloc = nNew; - p->aEntry = aNew; - } - - for(i=0; inEntry; i++){ - assert( i==0 || p->aEntry[i].iBlk > p->aEntry[i-1].iBlk ); - if( p->aEntry[i].iBlk>=iBlk ) break; - } - - if( inEntry && p->aEntry[i].iBlk==iBlk ){ - /* Clobber an existing entry */ - p->aEntry[i].iId = iId; - }else{ - /* Insert a new entry into the list */ - int nByte = sizeof(FreelistEntry)*(p->nEntry-i); - memmove(&p->aEntry[i+1], &p->aEntry[i], nByte); - p->aEntry[i].iBlk = iBlk; - p->aEntry[i].iId = iId; - p->nEntry++; - } - - return LSM_OK; -} - -/* -** This function frees all resources held by the Database structure passed -** as the only argument. -*/ -static void freeDatabase(lsm_env *pEnv, Database *p){ - assert( holdingGlobalMutex(pEnv) ); - if( p ){ - /* Free the mutexes */ - lsmMutexDel(pEnv, p->pClientMutex); - - if( p->pFile ){ - lsmEnvClose(pEnv, p->pFile); - } - - /* Free the array of shm pointers */ - lsmFree(pEnv, p->apShmChunk); - - /* Free the memory allocated for the Database struct itself */ - lsmFree(pEnv, p); - } -} - -typedef struct DbTruncateCtx DbTruncateCtx; -struct DbTruncateCtx { - int nBlock; - i64 iInUse; -}; - -static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ - DbTruncateCtx *p = (DbTruncateCtx *)pCtx; - if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; - p->nBlock--; - return 0; -} - -static int dbTruncate(lsm_db *pDb, i64 iInUse){ - int rc = LSM_OK; -#if 0 - int i; - DbTruncateCtx ctx; - - assert( pDb->pWorker ); - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = iInUse; - - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ - rc = freelistAppend(pDb, i, -1); - } - - if( rc==LSM_OK ){ -#ifdef LSM_LOG_FREELIST - if( ctx.nBlock!=pDb->pWorker->nBlock ){ - lsmLogMessage(pDb, 0, - "dbTruncate(): truncated db to %d blocks",ctx.nBlock - ); - } -#endif - pDb->pWorker->nBlock = ctx.nBlock; - } -#endif - return rc; -} - - -/* -** This function is called during database shutdown (when the number of -** connections drops from one to zero). It truncates the database file -** to as small a size as possible without truncating away any blocks that -** contain data. -*/ -static int dbTruncateFile(lsm_db *pDb){ - int rc; - - assert( pDb->pWorker==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) ); - rc = lsmCheckpointLoadWorker(pDb); - - if( rc==LSM_OK ){ - DbTruncateCtx ctx; - - /* Walk the database free-block-list in reverse order. Set ctx.nBlock - ** to the block number of the last block in the database that actually - ** contains data. */ - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = -1; - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - - /* If the last block that contains data is not already the last block in - ** the database file, truncate the database file so that it is. */ - if( rc==LSM_OK ){ - rc = lsmFsTruncateDb( - pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) - ); - } - } - - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - return rc; -} - -static void doDbDisconnect(lsm_db *pDb){ - int rc; - - if( pDb->bReadonly ){ - lsmShmLock(pDb, LSM_LOCK_DMS3, LSM_LOCK_UNLOCK, 0); - }else{ - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc==LSM_OK ){ - - lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); - - /* Try an exclusive lock on DMS2. If successful, this is the last - ** connection to the database. In this case flush the contents of the - ** in-memory tree to disk and write a checkpoint. */ - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 1, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - rc = lsmShmTestLock(pDb, LSM_LOCK_CHECKPOINTER, 1, LSM_LOCK_EXCL); - } - if( rc==LSM_OK ){ - int bReadonly = 0; /* True if there exist read-only conns. */ - - /* Flush the in-memory tree, if required. If there is data to flush, - ** this will create a new client snapshot in Database.pClient. The - ** checkpoint (serialization) of this snapshot may be written to disk - ** by the following block. - ** - ** There is no need to take a WRITER lock here. That there are no - ** other locks on DMS2 guarantees that there are no other read-write - ** connections at this time (and the lock on DMS1 guarantees that - ** no new ones may appear). - */ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK && (lsmTreeHasOld(pDb) || lsmTreeSize(pDb)>0) ){ - rc = lsmFlushTreeToDisk(pDb); - } - - /* Now check if there are any read-only connections. If there are, - ** then do not truncate the db file or unlink the shared-memory - ** region. */ - if( rc==LSM_OK ){ - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS3, 1, LSM_LOCK_EXCL); - if( rc==LSM_BUSY ){ - bReadonly = 1; - rc = LSM_OK; - } - } - - /* Write a checkpoint to disk. */ - if( rc==LSM_OK ){ - rc = lsmCheckpointWrite(pDb, 0); - } - - /* If the checkpoint was written successfully, delete the log file - ** and, if possible, truncate the database file. */ - if( rc==LSM_OK ){ - int bRotrans = 0; - Database *p = pDb->pDatabase; - - /* The log file may only be deleted if there are no clients - ** read-only clients running rotrans transactions. */ - rc = lsmDetectRoTrans(pDb, &bRotrans); - if( rc==LSM_OK && bRotrans==0 ){ - lsmFsCloseAndDeleteLog(pDb->pFS); - } - - /* The database may only be truncated if there exist no read-only - ** clients - either connected or running rotrans transactions. */ - if( bReadonly==0 && bRotrans==0 ){ - lsmFsUnmap(pDb->pFS); - dbTruncateFile(pDb); - if( p->pFile && p->bMultiProc ){ - lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); - } - } - } - } - } - - if( pDb->iRwclient>=0 ){ - lsmShmLock(pDb, LSM_LOCK_RWCLIENT(pDb->iRwclient), LSM_LOCK_UNLOCK, 0); - pDb->iRwclient = -1; - } - - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - } - pDb->pShmhdr = 0; -} - -static int doDbConnect(lsm_db *pDb){ - const int nUsMax = 100000; /* Max value for nUs */ - int nUs = 1000; /* us to wait between DMS1 attempts */ - int rc; - - /* Obtain a pointer to the shared-memory header */ - assert( pDb->pShmhdr==0 ); - assert( pDb->bReadonly==0 ); - - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - while( 1 ){ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc!=LSM_BUSY ) break; - lsmEnvSleep(pDb->pEnv, nUs); - nUs = nUs * 2; - if( nUs>nUsMax ) nUs = nUsMax; - } - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, 1); - } - if( rc!=LSM_OK ) return rc; - pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; - - /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first - ** and only connection to the database. In this case initialize the - ** shared-memory and run log file recovery. */ - assert( LSM_LOCK_DMS3==1+LSM_LOCK_DMS2 ); - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 2, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - memset(pDb->pShmhdr, 0, sizeof(ShmHeader)); - rc = lsmCheckpointRecover(pDb); - if( rc==LSM_OK ){ - rc = lsmLogRecover(pDb); - } - if( rc==LSM_OK ){ - ShmHeader *pShm = pDb->pShmhdr; - pShm->aReader[0].iLsmId = lsmCheckpointId(pShm->aSnap1, 0); - pShm->aReader[0].iTreeId = pDb->treehdr.iUsedShmid; - } - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - - /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" - ** fail, as connections may only hold an exclusive lock on DMS2 if they - ** first hold an exclusive lock on DMS1. And this connection is currently - ** holding the exclusive lock on DSM1. - ** - ** However, if some other connection has the database open in single-process - ** mode, this operation will fail. In this case, return the error to the - ** caller - the attempt to connect to the db has failed. - */ - if( rc==LSM_OK ){ - rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); - } - - /* If anything went wrong, unlock DMS2. Otherwise, try to take an exclusive - ** lock on one of the LSM_LOCK_RWCLIENT() locks. Unlock DMS1 in any case. */ - if( rc!=LSM_OK ){ - pDb->pShmhdr = 0; - }else{ - int i; - for(i=0; iiRwclient = i; - if( rc2!=LSM_BUSY ){ - rc = rc2; - break; - } - } - } - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - - return rc; -} - -static int dbOpenSharedFd(lsm_env *pEnv, Database *p, int bRoOk){ - int rc; - - rc = lsmEnvOpen(pEnv, p->zName, 0, &p->pFile); - if( rc==LSM_IOERR && bRoOk ){ - rc = lsmEnvOpen(pEnv, p->zName, LSM_OPEN_READONLY, &p->pFile); - p->bReadonly = 1; - } - - return rc; -} - -/* -** Return a reference to the shared Database handle for the database -** identified by canonical path zName. If this is the first connection to -** the named database, a new Database object is allocated. Otherwise, a -** pointer to an existing object is returned. -** -** If successful, *ppDatabase is set to point to the shared Database -** structure and LSM_OK returned. Otherwise, *ppDatabase is set to NULL -** and and LSM error code returned. -** -** Each successful call to this function should be (eventually) matched -** by a call to lsmDbDatabaseRelease(). -*/ -int lsmDbDatabaseConnect( - lsm_db *pDb, /* Database handle */ - const char *zName /* Full-path to db file */ -){ - lsm_env *pEnv = pDb->pEnv; - int rc; /* Return code */ - Database *p = 0; /* Pointer returned via *ppDatabase */ - int nName = lsmStrlen(zName); - - assert( pDb->pDatabase==0 ); - rc = enterGlobalMutex(pEnv); - if( rc==LSM_OK ){ - - /* Search the global list for an existing object. TODO: Need something - ** better than the memcmp() below to figure out if a given Database - ** object represents the requested file. */ - for(p=gShared.pDatabase; p; p=p->pDbNext){ - if( nName==p->nName && 0==memcmp(zName, p->zName, nName) ) break; - } - - /* If no suitable Database object was found, allocate a new one. */ - if( p==0 ){ - p = (Database *)lsmMallocZeroRc(pEnv, sizeof(Database)+nName+1, &rc); - - /* If the allocation was successful, fill in other fields and - ** allocate the client mutex. */ - if( rc==LSM_OK ){ - p->bMultiProc = pDb->bMultiProc; - p->zName = (char *)&p[1]; - p->nName = nName; - memcpy((void *)p->zName, zName, nName+1); - rc = lsmMutexNew(pEnv, &p->pClientMutex); - } - - /* If nothing has gone wrong so far, open the shared fd. And if that - ** succeeds and this connection requested single-process mode, - ** attempt to take the exclusive lock on DMS2. */ - if( rc==LSM_OK ){ - int bReadonly = (pDb->bReadonly && pDb->bMultiProc); - rc = dbOpenSharedFd(pDb->pEnv, p, bReadonly); - } - - if( rc==LSM_OK && p->bMultiProc==0 ){ - /* Hold an exclusive lock DMS1 while grabbing DMS2. This ensures - ** that any ongoing call to doDbDisconnect() (even one in another - ** process) is finished before proceeding. */ - assert( p->bReadonly==0 ); - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); - lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK); - } - } - - if( rc==LSM_OK ){ - p->pDbNext = gShared.pDatabase; - gShared.pDatabase = p; - }else{ - freeDatabase(pEnv, p); - p = 0; - } - } - - if( p ){ - p->nDbRef++; - } - leaveGlobalMutex(pEnv); - - if( p ){ - lsmMutexEnter(pDb->pEnv, p->pClientMutex); - pDb->pNext = p->pConn; - p->pConn = pDb; - lsmMutexLeave(pDb->pEnv, p->pClientMutex); - } - } - - pDb->pDatabase = p; - if( rc==LSM_OK ){ - assert( p ); - rc = lsmFsOpen(pDb, zName, p->bReadonly); - } - - /* If the db handle is read-write, then connect to the system now. Run - ** recovery as necessary. Or, if this is a read-only database handle, - ** defer attempting to connect to the system until a read-transaction - ** is opened. */ - if( rc==LSM_OK ){ - rc = lsmFsConfigure(pDb); - } - if( rc==LSM_OK && pDb->bReadonly==0 ){ - rc = doDbConnect(pDb); - } - - return rc; -} - -static void dbDeferClose(lsm_db *pDb){ - if( pDb->pFS ){ - LsmFile *pLsmFile; - Database *p = pDb->pDatabase; - pLsmFile = lsmFsDeferClose(pDb->pFS); - pLsmFile->pNext = p->pLsmFile; - p->pLsmFile = pLsmFile; - } -} - -LsmFile *lsmDbRecycleFd(lsm_db *db){ - LsmFile *pRet; - Database *p = db->pDatabase; - lsmMutexEnter(db->pEnv, p->pClientMutex); - if( (pRet = p->pLsmFile)!=0 ){ - p->pLsmFile = pRet->pNext; - } - lsmMutexLeave(db->pEnv, p->pClientMutex); - return pRet; -} - -/* -** Release a reference to a Database object obtained from -** lsmDbDatabaseConnect(). There should be exactly one call to this function -** for each successful call to Find(). -*/ -void lsmDbDatabaseRelease(lsm_db *pDb){ - Database *p = pDb->pDatabase; - if( p ){ - lsm_db **ppDb; - - if( pDb->pShmhdr ){ - doDbDisconnect(pDb); - } - - lsmFsUnmap(pDb->pFS); - lsmMutexEnter(pDb->pEnv, p->pClientMutex); - for(ppDb=&p->pConn; *ppDb!=pDb; ppDb=&((*ppDb)->pNext)); - *ppDb = pDb->pNext; - dbDeferClose(pDb); - lsmMutexLeave(pDb->pEnv, p->pClientMutex); - - enterGlobalMutex(pDb->pEnv); - p->nDbRef--; - if( p->nDbRef==0 ){ - LsmFile *pIter; - LsmFile *pNext; - Database **pp; - - /* Remove the Database structure from the linked list. */ - for(pp=&gShared.pDatabase; *pp!=p; pp=&((*pp)->pDbNext)); - *pp = p->pDbNext; - - /* If they were allocated from the heap, free the shared memory chunks */ - if( p->bMultiProc==0 ){ - int i; - for(i=0; inShmChunk; i++){ - lsmFree(pDb->pEnv, p->apShmChunk[i]); - } - } - - /* Close any outstanding file descriptors */ - for(pIter=p->pLsmFile; pIter; pIter=pNext){ - pNext = pIter->pNext; - lsmEnvClose(pDb->pEnv, pIter->pFile); - lsmFree(pDb->pEnv, pIter); - } - freeDatabase(pDb->pEnv, p); - } - leaveGlobalMutex(pDb->pEnv); - } -} - -Level *lsmDbSnapshotLevel(Snapshot *pSnapshot){ - return pSnapshot->pLevel; -} - -void lsmDbSnapshotSetLevel(Snapshot *pSnap, Level *pLevel){ - pSnap->pLevel = pLevel; -} - -/* TODO: Shuffle things around to get rid of this */ -static int firstSnapshotInUse(lsm_db *, i64 *); - -/* -** Context object used by the lsmWalkFreelist() utility. -*/ -typedef struct WalkFreelistCtx WalkFreelistCtx; -struct WalkFreelistCtx { - lsm_db *pDb; - int bReverse; - Freelist *pFreelist; - int iFree; - int (*xUsr)(void *, int, i64); /* User callback function */ - void *pUsrctx; /* User callback context */ - int bDone; /* Set to true after xUsr() returns true */ -}; - -/* -** Callback used by lsmWalkFreelist(). -*/ -static int walkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - WalkFreelistCtx *p = (WalkFreelistCtx *)pCtx; - const int iDir = (p->bReverse ? -1 : 1); - Freelist *pFree = p->pFreelist; - - assert( p->bDone==0 ); - assert( iBlk>=0 ); - if( pFree ){ - while( (p->iFree < pFree->nEntry) && p->iFree>=0 ){ - FreelistEntry *pEntry = &pFree->aEntry[p->iFree]; - if( (p->bReverse==0 && pEntry->iBlk>(u32)iBlk) - || (p->bReverse!=0 && pEntry->iBlk<(u32)iBlk) - ){ - break; - }else{ - p->iFree += iDir; - if( pEntry->iId>=0 - && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) - ){ - p->bDone = 1; - return 1; - } - if( pEntry->iBlk==(u32)iBlk ) return 0; - } - } - } - - if( p->xUsr(p->pUsrctx, iBlk, iSnapshot) ){ - p->bDone = 1; - return 1; - } - return 0; -} - -/* -** The database handle passed as the first argument must be the worker -** connection. This function iterates through the contents of the current -** free block list, invoking the supplied callback once for each list -** element. -** -** The difference between this function and lsmSortedWalkFreelist() is -** that lsmSortedWalkFreelist() only considers those free-list elements -** stored within the LSM. This function also merges in any in-memory -** elements. -*/ -int lsmWalkFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - const int iDir = (bReverse ? -1 : 1); - int rc; - int iCtx; - - WalkFreelistCtx ctx[2]; - - ctx[0].pDb = pDb; - ctx[0].bReverse = bReverse; - ctx[0].pFreelist = &pDb->pWorker->freelist; - if( ctx[0].pFreelist && bReverse ){ - ctx[0].iFree = ctx[0].pFreelist->nEntry-1; - }else{ - ctx[0].iFree = 0; - } - ctx[0].xUsr = walkFreelistCb; - ctx[0].pUsrctx = (void *)&ctx[1]; - ctx[0].bDone = 0; - - ctx[1].pDb = pDb; - ctx[1].bReverse = bReverse; - ctx[1].pFreelist = pDb->pFreelist; - if( ctx[1].pFreelist && bReverse ){ - ctx[1].iFree = ctx[1].pFreelist->nEntry-1; - }else{ - ctx[1].iFree = 0; - } - ctx[1].xUsr = x; - ctx[1].pUsrctx = pCtx; - ctx[1].bDone = 0; - - rc = lsmSortedWalkFreelist(pDb, bReverse, walkFreelistCb, (void *)&ctx[0]); - - if( ctx[0].bDone==0 ){ - for(iCtx=0; iCtx<2; iCtx++){ - int i; - WalkFreelistCtx *p = &ctx[iCtx]; - for(i=p->iFree; - p->pFreelist && rc==LSM_OK && ipFreelist->nEntry && i>=0; - i += iDir - ){ - FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; - if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ - return LSM_OK; - } - } - } - } - - return rc; -} - - -typedef struct FindFreeblockCtx FindFreeblockCtx; -struct FindFreeblockCtx { - i64 iInUse; - int iRet; - int bNotOne; -}; - -static int findFreeblockCb(void *pCtx, int iBlk, i64 iSnapshot){ - FindFreeblockCtx *p = (FindFreeblockCtx *)pCtx; - if( iSnapshotiInUse && (iBlk!=1 || p->bNotOne==0) ){ - p->iRet = iBlk; - return 1; - } - return 0; -} - -static int findFreeblock(lsm_db *pDb, i64 iInUse, int bNotOne, int *piRet){ - int rc; /* Return code */ - FindFreeblockCtx ctx; /* Context object */ - - ctx.iInUse = iInUse; - ctx.iRet = 0; - ctx.bNotOne = bNotOne; - rc = lsmWalkFreelist(pDb, 0, findFreeblockCb, (void *)&ctx); - *piRet = ctx.iRet; - - return rc; -} - -/* -** Allocate a new database file block to write data to, either by extending -** the database file or by recycling a free-list entry. The worker snapshot -** must be held in order to call this function. -** -** If successful, *piBlk is set to the block number allocated and LSM_OK is -** returned. Otherwise, *piBlk is zeroed and an lsm error code returned. -*/ -int lsmBlockAllocate(lsm_db *pDb, int iBefore, int *piBlk){ - Snapshot *p = pDb->pWorker; - int iRet = 0; /* Block number of allocated block */ - int rc = LSM_OK; - i64 iInUse = 0; /* Snapshot id still in use */ - i64 iSynced = 0; /* Snapshot id synced to disk */ - - assert( p ); - -#ifdef LSM_LOG_FREELIST - { - static int nCall = 0; - char *zFree = 0; - nCall++; - rc = lsmInfoFreelist(pDb, &zFree); - if( rc!=LSM_OK ) return rc; - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): %d freelist: %s", nCall, zFree); - lsmFree(pDb->pEnv, zFree); - } -#endif - - /* Set iInUse to the smallest snapshot id that is either: - ** - ** * Currently in use by a database client, - ** * May be used by a database client in the future, or - ** * Is the most recently checkpointed snapshot (i.e. the one that will - ** be used following recovery if a failure occurs at this point). - */ - rc = lsmCheckpointSynced(pDb, &iSynced, 0, 0); - if( rc==LSM_OK && iSynced==0 ) iSynced = p->iId; - iInUse = iSynced; - if( rc==LSM_OK && pDb->iReader>=0 ){ - assert( pDb->pClient ); - iInUse = LSM_MIN(iInUse, pDb->pClient->iId); - } - if( rc==LSM_OK ) rc = firstSnapshotInUse(pDb, &iInUse); - -#ifdef LSM_LOG_FREELIST - { - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): " - "snapshot-in-use: %lld (iSynced=%lld) (client-id=%lld)", - iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) - ); - } -#endif - - - /* Unless there exists a read-only transaction (which prevents us from - ** recycling any blocks regardless, query the free block list for a - ** suitable block to reuse. - ** - ** It might seem more natural to check for a read-only transaction at - ** the start of this function. However, it is better do wait until after - ** the call to lsmCheckpointSynced() to do so. - */ - if( rc==LSM_OK ){ - int bRotrans; - rc = lsmDetectRoTrans(pDb, &bRotrans); - - if( rc==LSM_OK && bRotrans==0 ){ - rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); - } - } - - if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ - iRet = 0; - - }else if( rc==LSM_OK ){ - /* If a block was found in the free block list, use it and remove it from - ** the list. Otherwise, if no suitable block was found, allocate one from - ** the end of the file. */ - if( iRet>0 ){ -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, - "reusing block %d (snapshot-in-use=%lld)", iRet, iInUse); -#endif - rc = freelistAppend(pDb, iRet, -1); - if( rc==LSM_OK ){ - rc = dbTruncate(pDb, iInUse); - } - }else{ - iRet = ++(p->nBlock); -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); -#endif - } - } - - assert( iBefore>0 || iRet>0 || rc!=LSM_OK ); - *piBlk = iRet; - return rc; -} - -/* -** Free a database block. The worker snapshot must be held in order to call -** this function. -** -** If successful, LSM_OK is returned. Otherwise, an lsm error code (e.g. -** LSM_NOMEM). -*/ -int lsmBlockFree(lsm_db *pDb, int iBlk){ - Snapshot *p = pDb->pWorker; - assert( lsmShmAssertWorker(pDb) ); - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockFree(): Free block %d", iBlk); -#endif - - return freelistAppend(pDb, iBlk, p->iId); -} - -/* -** Refree a database block. The worker snapshot must be held in order to call -** this function. -** -** Refreeing is required when a block is allocated using lsmBlockAllocate() -** but then not used. This function is used to push the block back onto -** the freelist. Refreeing a block is different from freeing is, as a refreed -** block may be reused immediately. Whereas a freed block can not be reused -** until (at least) after the next checkpoint. -*/ -int lsmBlockRefree(lsm_db *pDb, int iBlk){ - int rc = LSM_OK; /* Return code */ - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); -#endif - - rc = freelistAppend(pDb, iBlk, 0); - return rc; -} - -/* -** If required, copy a database checkpoint from shared memory into the -** database itself. -** -** The WORKER lock must not be held when this is called. This is because -** this function may indirectly call fsync(). And the WORKER lock should -** not be held that long (in case it is required by a client flushing an -** in-memory tree to disk). -*/ -int lsmCheckpointWrite(lsm_db *pDb, u32 *pnWrite){ - int rc; /* Return Code */ - u32 nWrite = 0; - - assert( pDb->pWorker==0 ); - assert( 1 || pDb->pClient==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK) ); - - rc = lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_EXCL, 0); - if( rc!=LSM_OK ) return rc; - - rc = lsmCheckpointLoad(pDb, 0); - if( rc==LSM_OK ){ - int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); - ShmHeader *pShm = pDb->pShmhdr; - int bDone = 0; /* True if checkpoint is already stored */ - - /* Check if this checkpoint has already been written to the database - ** file. If so, set variable bDone to true. */ - if( pShm->iMetaPage ){ - MetaPage *pPg; /* Meta page */ - u8 *aData; /* Meta-page data buffer */ - int nData; /* Size of aData[] in bytes */ - i64 iCkpt; /* Id of checkpoint just loaded */ - i64 iDisk = 0; /* Id of checkpoint already stored in db */ - iCkpt = lsmCheckpointId(pDb->aSnapshot, 0); - rc = lsmFsMetaPageGet(pDb->pFS, 0, pShm->iMetaPage, &pPg); - if( rc==LSM_OK ){ - aData = lsmFsMetaPageData(pPg, &nData); - iDisk = lsmCheckpointId((u32 *)aData, 1); - nWrite = lsmCheckpointNWrite((u32 *)aData, 1); - lsmFsMetaPageRelease(pPg); - } - bDone = (iDisk>=iCkpt); - } - - if( rc==LSM_OK && bDone==0 ){ - int iMeta = (pShm->iMetaPage % 2) + 1; - if( pDb->eSafety!=LSM_SAFETY_OFF ){ - rc = lsmFsSyncDb(pDb->pFS, nBlock); - } - if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); - if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ - rc = lsmFsSyncDb(pDb->pFS, 0); - } - if( rc==LSM_OK ){ - pShm->iMetaPage = iMeta; - nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; - } -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, 0, "finish checkpoint %d", - (int)lsmCheckpointId(pDb->aSnapshot, 0) - ); -#endif - } - } - - lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); - if( pnWrite && rc==LSM_OK ) *pnWrite = nWrite; - return rc; -} - -int lsmBeginWork(lsm_db *pDb){ - int rc; - - /* Attempt to take the WORKER lock */ - rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); - - /* Deserialize the current worker snapshot */ - if( rc==LSM_OK ){ - rc = lsmCheckpointLoadWorker(pDb); - } - return rc; -} - -void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ - if( p ){ - lsmSortedFreeLevel(pEnv, p->pLevel); - lsmFree(pEnv, p->freelist.aEntry); - lsmFree(pEnv, p->redirect.a); - lsmFree(pEnv, p); - } -} - -/* -** Attempt to populate one of the read-lock slots to contain lock values -** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. -** -** It is not an error if no slot can be populated because the write-lock -** cannot be obtained. If any other error occurs, return an LSM error code. -** Otherwise, LSM_OK. -** -** This function is called at various points to try to ensure that there -** always exists at least one read-lock slot that can be used by a read-only -** client. And so that, in the usual case, there is an "exact match" available -** whenever a read transaction is opened by any client. At present this -** function is called when: -** -** * A write transaction that called lsmTreeDiscardOld() is committed, and -** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). -*/ -static int dbSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - /* Check if there is already a slot containing the required values. */ - for(i=0; iaReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; - } - - /* Iterate through all read-lock slots, attempting to take a write-lock - ** on each of them. If a write-lock succeeds, populate the locked slot - ** with the required values and break out of the loop. */ - for(i=0; rc==LSM_OK && iaReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShm; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - break; - } - } - - return rc; -} - -/* -** Release the read-lock currently held by connection db. -*/ -int dbReleaseReadlock(lsm_db *db){ - int rc = LSM_OK; - if( db->iReader>=0 ){ - rc = lsmShmLock(db, LSM_LOCK_READER(db->iReader), LSM_LOCK_UNLOCK, 0); - db->iReader = -1; - } - db->bRoTrans = 0; - return rc; -} - - -/* -** Argument bFlush is true if the contents of the in-memory tree has just -** been flushed to disk. The significance of this is that once the snapshot -** created to hold the updated state of the database is synced to disk, log -** file space can be recycled. -*/ -void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ - int rc = *pRc; - assert( rc!=0 || pDb->pWorker ); - if( pDb->pWorker ){ - /* If no error has occurred, serialize the worker snapshot and write - ** it to shared memory. */ - if( rc==LSM_OK ){ - rc = lsmSaveWorker(pDb, bFlush); - } - - /* Assuming no error has occurred, update a read lock slot with the - ** new snapshot id (see comments above function dbSetReadLock()). */ - if( rc==LSM_OK ){ - if( pDb->iReader<0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( rc==LSM_OK ){ - rc = dbSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); - } - } - - /* Free the snapshot object. */ - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - } - - lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); - *pRc = rc; -} - -/* -** Called when recovery is finished. -*/ -int lsmFinishRecovery(lsm_db *pDb){ - lsmTreeEndTransaction(pDb, 1); - return LSM_OK; -} - -/* -** Check if the currently configured compression functions -** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its -** compression id set to iReq. Compression routines are compatible if iReq -** is zero (indicating the database is empty), or if it is equal to the -** compression id of the configured compression routines. -** -** If the check shows that the current compression are incompatible and there -** is a compression factory registered, give it a chance to install new -** compression routines. -** -** If, after any registered factory is invoked, the compression functions -** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. -*/ -int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ - if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ - if( pDb->factory.xFactory ){ - pDb->bInFactory = 1; - pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); - pDb->bInFactory = 0; - } - if( pDb->compress.iId!=iReq ){ - /* Incompatible */ - return LSM_MISMATCH; - } - } - /* Compatible */ - return LSM_OK; -} - -/* -** Begin a read transaction. This function is a no-op if the connection -** passed as the only argument already has an open read transaction. -*/ -int lsmBeginReadTrans(lsm_db *pDb){ - const int MAX_READLOCK_ATTEMPTS = 10; - const int nMaxAttempt = (pDb->bRoTrans ? 1 : MAX_READLOCK_ATTEMPTS); - - int rc = LSM_OK; /* Return code */ - int iAttempt = 0; - - assert( pDb->pWorker==0 ); - - while( rc==LSM_OK && pDb->iReader<0 && (iAttempt++)pCsr==0 && pDb->nTransOpen==0 ); - - /* Load the in-memory tree header. */ - rc = lsmTreeLoadHeader(pDb, &iTreehdr); - - /* Load the database snapshot */ - if( rc==LSM_OK ){ - if( lsmCheckpointClientCacheOk(pDb)==0 ){ - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - lsmMCursorFreeCache(pDb); - lsmFsPurgeCache(pDb->pFS); - rc = lsmCheckpointLoad(pDb, &iSnap); - }else{ - iSnap = 1; - } - } - - /* Take a read-lock on the tree and snapshot just loaded. Then check - ** that the shared-memory still contains the same values. If so, proceed. - ** Otherwise, relinquish the read-lock and retry the whole procedure - ** (starting with loading the in-memory tree header). */ - if( rc==LSM_OK ){ - u32 iShmMax = pDb->treehdr.iUsedShmid; - u32 iShmMin = pDb->treehdr.iNextShmid+1-LSM_MAX_SHMCHUNKS; - rc = lsmReadlock( - pDb, lsmCheckpointId(pDb->aSnapshot, 0), iShmMin, iShmMax - ); - if( rc==LSM_OK ){ - if( lsmTreeLoadHeaderOk(pDb, iTreehdr) - && lsmCheckpointLoadOk(pDb, iSnap) - ){ - /* Read lock has been successfully obtained. Deserialize the - ** checkpoint just loaded. TODO: This will be removed after - ** lsm_sorted.c is changed to work directly from the serialized - ** version of the snapshot. */ - if( pDb->pClient==0 ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); - } - assert( (rc==LSM_OK)==(pDb->pClient!=0) ); - assert( pDb->iReader>=0 ); - - /* Check that the client has the right compression hooks loaded. - ** If not, set rc to LSM_MISMATCH. */ - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); - } - }else{ - rc = dbReleaseReadlock(pDb); - } - } - - if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } -#if 0 -if( rc==LSM_OK && pDb->pClient ){ - fprintf(stderr, - "reading %p: snapshot:%d used-shmid:%d trans-id:%d iOldShmid=%d\n", - (void *)pDb, - (int)pDb->pClient->iId, (int)pDb->treehdr.iUsedShmid, - (int)pDb->treehdr.root.iTransId, - (int)pDb->treehdr.iOldShmid - ); -} -#endif - } - - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - if( rc!=LSM_OK ){ - dbReleaseReadlock(pDb); - } - if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; - return rc; -} - -/* -** This function is used by a read-write connection to determine if there -** are currently one or more read-only transactions open on the database -** (in this context a read-only transaction is one opened by a read-only -** connection on a non-live database). -** -** If no error occurs, LSM_OK is returned and *pbExists is set to true if -** some other connection has a read-only transaction open, or false -** otherwise. If an error occurs an LSM error code is returned and the final -** value of *pbExist is undefined. -*/ -int lsmDetectRoTrans(lsm_db *db, int *pbExist){ - int rc; - - /* Only a read-write connection may use this function. */ - assert( db->bReadonly==0 ); - - rc = lsmShmTestLock(db, LSM_LOCK_ROTRANS, 1, LSM_LOCK_EXCL); - if( rc==LSM_BUSY ){ - *pbExist = 1; - rc = LSM_OK; - }else{ - *pbExist = 0; - } - - return rc; -} - -/* -** db is a read-only database handle in the disconnected state. This function -** attempts to open a read-transaction on the database. This may involve -** connecting to the database system (opening shared memory etc.). -*/ -int lsmBeginRoTrans(lsm_db *db){ - int rc = LSM_OK; - - assert( db->bReadonly && db->pShmhdr==0 ); - assert( db->iReader<0 ); - - if( db->bRoTrans==0 ){ - - /* Attempt a shared-lock on DMS1. */ - rc = lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_SHARED, 0); - if( rc!=LSM_OK ) return rc; - - rc = lsmShmTestLock( - db, LSM_LOCK_RWCLIENT(0), LSM_LOCK_NREADER, LSM_LOCK_SHARED - ); - if( rc==LSM_OK ){ - /* System is not live. Take a SHARED lock on the ROTRANS byte and - ** release DMS1. Locking ROTRANS tells all read-write clients that they - ** may not recycle any disk space from within the database or log files, - ** as a read-only client may be using it. */ - rc = lsmShmLock(db, LSM_LOCK_ROTRANS, LSM_LOCK_SHARED, 0); - lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - - if( rc==LSM_OK ){ - db->bRoTrans = 1; - rc = lsmShmCacheChunks(db, 1); - if( rc==LSM_OK ){ - db->pShmhdr = (ShmHeader *)db->apShm[0]; - memset(db->pShmhdr, 0, sizeof(ShmHeader)); - rc = lsmCheckpointRecover(db); - if( rc==LSM_OK ){ - rc = lsmLogRecover(db); - } - } - } - }else if( rc==LSM_BUSY ){ - /* System is live! */ - rc = lsmShmLock(db, LSM_LOCK_DMS3, LSM_LOCK_SHARED, 0); - lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(db, 1); - if( rc==LSM_OK ){ - db->pShmhdr = (ShmHeader *)db->apShm[0]; - } - } - } - - /* In 'lsm_open()' we don't update the page and block sizes in the - ** Filesystem for 'readonly' connection. Because member 'db->pShmhdr' is a - ** nullpointer, this prevents loading a checkpoint. Now that the system is - ** live this member should be set. So we can update both values in - ** the Filesystem. - ** - ** Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so - ** are guaranteed not to change during the lifetime of this connection. */ - if( LSM_OK==rc - && 0==lsmCheckpointClientCacheOk(db) - && LSM_OK==(rc=lsmCheckpointLoad(db, 0)) - ){ - lsmFsSetPageSize(db->pFS, lsmCheckpointPgsz(db->aSnapshot)); - lsmFsSetBlockSize(db->pFS, lsmCheckpointBlksz(db->aSnapshot)); - } - - if( rc==LSM_OK ){ - rc = lsmBeginReadTrans(db); - } - } - - return rc; -} - -/* -** Close the currently open read transaction. -*/ -void lsmFinishReadTrans(lsm_db *pDb){ - - /* Worker connections should not be closing read transactions. And - ** read transactions should only be closed after all cursors and write - ** transactions have been closed. Finally pClient should be non-NULL - ** only iff pDb->iReader>=0. */ - assert( pDb->pWorker==0 ); - assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); - - if( pDb->bRoTrans ){ - int i; - for(i=0; inShm; i++){ - lsmFree(pDb->pEnv, pDb->apShm[i]); - } - lsmFree(pDb->pEnv, pDb->apShm); - pDb->apShm = 0; - pDb->nShm = 0; - pDb->pShmhdr = 0; - - lsmShmLock(pDb, LSM_LOCK_ROTRANS, LSM_LOCK_UNLOCK, 0); - } - dbReleaseReadlock(pDb); -} - -/* -** Open a write transaction. -*/ -int lsmBeginWriteTrans(lsm_db *pDb){ - int rc = LSM_OK; /* Return code */ - ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ - - assert( pDb->nTransOpen==0 ); - assert( pDb->bDiscardOld==0 ); - assert( pDb->bReadonly==0 ); - - /* If there is no read-transaction open, open one now. */ - if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Attempt to take the WRITER lock */ - if( rc==LSM_OK ){ - rc = lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL, 0); - } - - /* If the previous writer failed mid-transaction, run emergency rollback. */ - if( rc==LSM_OK && pShm->bWriter ){ - rc = lsmTreeRepair(pDb); - if( rc==LSM_OK ) pShm->bWriter = 0; - } - - /* Check that this connection is currently reading from the most recent - ** version of the database. If not, return LSM_BUSY. */ - if( rc==LSM_OK && memcmp(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)) ){ - rc = LSM_BUSY; - } - - if( rc==LSM_OK ){ - rc = lsmLogBegin(pDb); - } - - /* If everything was successful, set the "transaction-in-progress" flag - ** and return LSM_OK. Otherwise, if some error occurred, relinquish the - ** WRITER lock and return an error code. */ - if( rc==LSM_OK ){ - TreeHeader *p = &pDb->treehdr; - pShm->bWriter = 1; - p->root.iTransId++; - if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ - lsmTreeDiscardOld(pDb); - pDb->bDiscardOld = 1; - } - }else{ - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); - } - return rc; -} - -/* -** End the current write transaction. The connection is left with an open -** read transaction. It is an error to call this if there is no open write -** transaction. -** -** If the transaction was committed, then a commit record has already been -** written into the log file when this function is called. Or, if the -** transaction was rolled back, both the log file and in-memory tree -** structure have already been restored. In either case, this function -** merely releases locks and other resources held by the write-transaction. -** -** LSM_OK is returned if successful, or an LSM error code otherwise. -*/ -int lsmFinishWriteTrans(lsm_db *pDb, int bCommit){ - int rc = LSM_OK; - int bFlush = 0; - - lsmLogEnd(pDb, bCommit); - if( rc==LSM_OK && bCommit && lsmTreeSize(pDb)>pDb->nTreeLimit ){ - bFlush = 1; - lsmTreeMakeOld(pDb); - } - lsmTreeEndTransaction(pDb, bCommit); - - if( rc==LSM_OK ){ - if( bFlush && pDb->bAutowork ){ - rc = lsmSortedAutoWork(pDb, 1); - }else if( bCommit && pDb->bDiscardOld ){ - rc = dbSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); - } - } - pDb->bDiscardOld = 0; - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - - if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ - pDb->xWork(pDb, pDb->pWorkCtx); - } - return rc; -} - - -/* -** Return non-zero if the caller is holding the client mutex. -*/ -#ifdef LSM_DEBUG -int lsmHoldingClientMutex(lsm_db *pDb){ - return lsmMutexHeld(pDb->pEnv, pDb->pDatabase->pClientMutex); -} -#endif - -static int slotIsUsable(ShmReader *p, i64 iLsm, u32 iShmMin, u32 iShmMax){ - return( - p->iLsmId && p->iLsmId<=iLsm - && shm_sequence_ge(iShmMax, p->iTreeId) - && shm_sequence_ge(p->iTreeId, iShmMin) - ); -} - -/* -** Obtain a read-lock on database version identified by the combination -** of snapshot iLsm and tree iTree. Return LSM_OK if successful, or -** an LSM error code otherwise. -*/ -int lsmReadlock(lsm_db *db, i64 iLsm, u32 iShmMin, u32 iShmMax){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - assert( db->iReader<0 ); - assert( shm_sequence_ge(iShmMax, iShmMin) ); - - /* This is a no-op if the read-only transaction flag is set. */ - if( db->bRoTrans ){ - db->iReader = 0; - return LSM_OK; - } - - /* Search for an exact match. */ - for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShmMax ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - if( rc==LSM_OK && p->iLsmId==iLsm && p->iTreeId==iShmMax ){ - db->iReader = i; - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } - } - - /* Try to obtain a write-lock on each slot, in order. If successful, set - ** the slot values to iLsm/iTree. */ - for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShmMax; - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - assert( rc!=LSM_BUSY ); - if( rc==LSM_OK ) db->iReader = i; - } - } - - /* Search for any usable slot */ - for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; - if( slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - if( rc==LSM_OK && slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ - db->iReader = i; - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } - } - - if( rc==LSM_OK && db->iReader<0 ){ - rc = LSM_BUSY; - } - return rc; -} - -/* -** This is used to check if there exists a read-lock locking a particular -** version of either the in-memory tree or database file. -** -** If iLsmId is non-zero, then it is a snapshot id. If there exists a -** read-lock using this snapshot or newer, set *pbInUse to true. Or, -** if there is no such read-lock, set it to false. -** -** Or, if iLsmId is zero, then iShmid is a shared-memory sequence id. -** Search for a read-lock using this sequence id or newer. etc. -*/ -static int isInUse(lsm_db *db, i64 iLsmId, u32 iShmid, int *pbInUse){ - ShmHeader *pShm = db->pShmhdr; - int i; - int rc = LSM_OK; - - for(i=0; rc==LSM_OK && iaReader[i]; - if( p->iLsmId ){ - if( (iLsmId!=0 && p->iLsmId!=0 && iLsmId>=p->iLsmId) - || (iLsmId==0 && shm_sequence_ge(p->iTreeId, iShmid)) - ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - } - } - } - } - - if( rc==LSM_BUSY ){ - *pbInUse = 1; - return LSM_OK; - } - *pbInUse = 0; - return rc; -} - -/* -** This function is called by worker connections to determine the smallest -** snapshot id that is currently in use by a database client. The worker -** connection uses this result to determine whether or not it is safe to -** recycle a database block. -*/ -static int firstSnapshotInUse( - lsm_db *db, /* Database handle */ - i64 *piInUse /* IN/OUT: Smallest snapshot id in use */ -){ - ShmHeader *pShm = db->pShmhdr; - i64 iInUse = *piInUse; - int i; - - assert( iInUse>0 ); - for(i=0; iaReader[i]; - if( p->iLsmId ){ - i64 iThis = p->iLsmId; - if( iThis!=0 && iInUse>iThis ){ - int rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - }else if( rc==LSM_BUSY ){ - iInUse = iThis; - }else{ - /* Some error other than LSM_BUSY. Return the error code to - ** the caller in this case. */ - return rc; - } - } - } - } - - *piInUse = iInUse; - return LSM_OK; -} - -int lsmTreeInUse(lsm_db *db, u32 iShmid, int *pbInUse){ - if( db->treehdr.iUsedShmid==iShmid ){ - *pbInUse = 1; - return LSM_OK; - } - return isInUse(db, 0, iShmid, pbInUse); -} - -int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse){ - if( db->pClient && db->pClient->iId<=iLsmId ){ - *pbInUse = 1; - return LSM_OK; - } - return isInUse(db, iLsmId, 0, pbInUse); -} - -/* -** This function may only be called after a successful call to -** lsmDbDatabaseConnect(). It returns true if the connection is in -** multi-process mode, or false otherwise. -*/ -int lsmDbMultiProc(lsm_db *pDb){ - return pDb->pDatabase && pDb->pDatabase->bMultiProc; -} - - -/************************************************************************* -************************************************************************** -************************************************************************** -************************************************************************** -************************************************************************** -*************************************************************************/ - -/* -** Ensure that database connection db has cached pointers to at least the -** first nChunk chunks of shared memory. -*/ -int lsmShmCacheChunks(lsm_db *db, int nChunk){ - int rc = LSM_OK; - if( nChunk>db->nShm ){ - static const int NINCR = 16; - Database *p = db->pDatabase; - lsm_env *pEnv = db->pEnv; - int nAlloc; - int i; - - /* Ensure that the db->apShm[] array is large enough. If an attempt to - ** allocate memory fails, return LSM_NOMEM immediately. The apShm[] array - ** is always extended in multiples of 16 entries - so the actual allocated - ** size can be inferred from nShm. */ - nAlloc = ((db->nShm + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); - if( !apShm ) return LSM_NOMEM_BKPT; - db->apShm = apShm; - } - - if( db->bRoTrans ){ - for(i=db->nShm; rc==LSM_OK && iapShm[i] = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - db->nShm++; - } - - }else{ - - /* Enter the client mutex */ - lsmMutexEnter(pEnv, p->pClientMutex); - - /* Extend the Database objects apShmChunk[] array if necessary. Using the - ** same pattern as for the lsm_db.apShm[] array above. */ - nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); - if( !apShm ){ - rc = LSM_NOMEM_BKPT; - break; - } - p->apShmChunk = apShm; - } - - for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ - void *pChunk = 0; - if( p->bMultiProc==0 ){ - /* Single process mode */ - pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - }else{ - /* Multi-process mode */ - rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); - } - if( rc==LSM_OK ){ - p->apShmChunk[i] = pChunk; - p->nShmChunk++; - } - } - if( rc==LSM_OK ){ - db->apShm[i] = p->apShmChunk[i]; - db->nShm++; - } - } - - /* Release the client mutex */ - lsmMutexLeave(pEnv, p->pClientMutex); - } - } - - return rc; -} - -static int lockSharedFile(lsm_env *pEnv, Database *p, int iLock, int eOp){ - int rc = LSM_OK; - if( p->bMultiProc ){ - rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); - } - return rc; -} - -/* -** Test if it would be possible for connection db to obtain a lock of type -** eType on the nLock locks starting at iLock. If so, return LSM_OK. If it -** would not be possible to obtain the lock due to a lock held by another -** connection, return LSM_BUSY. If an IO or other error occurs (i.e. in the -** lsm_env.xTestLock function), return some other LSM error code. -** -** Note that this function never actually locks the database - it merely -** queries the system to see if there exists a lock that would prevent -** it from doing so. -*/ -int lsmShmTestLock( - lsm_db *db, - int iLock, - int nLock, - int eOp -){ - int rc = LSM_OK; - lsm_db *pIter; - Database *p = db->pDatabase; - int i; - u64 mask = 0; - - for(i=iLock; i<(iLock+nLock); i++){ - mask |= ((u64)1 << (iLock-1)); - if( eOp==LSM_LOCK_EXCL ) mask |= ((u64)1 << (iLock+32-1)); - } - - lsmMutexEnter(db->pEnv, p->pClientMutex); - for(pIter=p->pConn; pIter; pIter=pIter->pNext){ - if( pIter!=db && (pIter->mLock & mask) ){ - assert( pIter!=db ); - break; - } - } - - if( pIter ){ - rc = LSM_BUSY; - }else if( p->bMultiProc ){ - rc = lsmEnvTestLock(db->pEnv, p->pFile, iLock, nLock, eOp); - } - - lsmMutexLeave(db->pEnv, p->pClientMutex); - return rc; -} - -/* -** Attempt to obtain the lock identified by the iLock and bExcl parameters. -** If successful, return LSM_OK. If the lock cannot be obtained because -** there exists some other conflicting lock, return LSM_BUSY. If some other -** error occurs, return an LSM error code. -** -** Parameter iLock must be one of LSM_LOCK_WRITER, WORKER or CHECKPOINTER, -** or else a value returned by the LSM_LOCK_READER macro. -*/ -int lsmShmLock( - lsm_db *db, - int iLock, - int eOp, /* One of LSM_LOCK_UNLOCK, SHARED or EXCL */ - int bBlock /* True for a blocking lock */ -){ - lsm_db *pIter; - const u64 me = ((u64)1 << (iLock-1)); - const u64 ms = ((u64)1 << (iLock+32-1)); - int rc = LSM_OK; - Database *p = db->pDatabase; - - assert( eOp!=LSM_LOCK_EXCL || p->bReadonly==0 ); - assert( iLock>=1 && iLock<=LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1) ); - assert( LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1)<=32 ); - assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); - - /* Check for a no-op. Proceed only if this is not one of those. */ - if( (eOp==LSM_LOCK_UNLOCK && (db->mLock & (me|ms))!=0) - || (eOp==LSM_LOCK_SHARED && (db->mLock & (me|ms))!=ms) - || (eOp==LSM_LOCK_EXCL && (db->mLock & me)==0) - ){ - int nExcl = 0; /* Number of connections holding EXCLUSIVE */ - int nShared = 0; /* Number of connections holding SHARED */ - lsmMutexEnter(db->pEnv, p->pClientMutex); - - /* Figure out the locks currently held by this process on iLock, not - ** including any held by connection db. */ - for(pIter=p->pConn; pIter; pIter=pIter->pNext){ - assert( (pIter->mLock & me)==0 || (pIter->mLock & ms)!=0 ); - if( pIter!=db ){ - if( pIter->mLock & me ){ - nExcl++; - }else if( pIter->mLock & ms ){ - nShared++; - } - } - } - assert( nExcl==0 || nExcl==1 ); - assert( nExcl==0 || nShared==0 ); - assert( nExcl==0 || (db->mLock & (me|ms))==0 ); - - switch( eOp ){ - case LSM_LOCK_UNLOCK: - if( nShared==0 ){ - lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_UNLOCK); - } - db->mLock &= ~(me|ms); - break; - - case LSM_LOCK_SHARED: - if( nExcl ){ - rc = LSM_BUSY; - }else{ - if( nShared==0 ){ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); - } - if( rc==LSM_OK ){ - db->mLock |= ms; - db->mLock &= ~me; - } - } - break; - - default: - assert( eOp==LSM_LOCK_EXCL ); - if( nExcl || nShared ){ - rc = LSM_BUSY; - }else{ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - db->mLock |= (me|ms); - } - } - break; - } - - lsmMutexLeave(db->pEnv, p->pClientMutex); - } - - return rc; -} - -#ifdef LSM_DEBUG - -int shmLockType(lsm_db *db, int iLock){ - const u64 me = ((u64)1 << (iLock-1)); - const u64 ms = ((u64)1 << (iLock+32-1)); - - if( db->mLock & me ) return LSM_LOCK_EXCL; - if( db->mLock & ms ) return LSM_LOCK_SHARED; - return LSM_LOCK_UNLOCK; -} - -/* -** The arguments passed to this function are similar to those passed to -** the lsmShmLock() function. However, instead of obtaining a new lock -** this function returns true if the specified connection already holds -** (or does not hold) such a lock, depending on the value of eOp. As -** follows: -** -** (eOp==LSM_LOCK_UNLOCK) -> true if db has no lock on iLock -** (eOp==LSM_LOCK_SHARED) -> true if db has at least a SHARED lock on iLock. -** (eOp==LSM_LOCK_EXCL) -> true if db has an EXCLUSIVE lock on iLock. -*/ -int lsmShmAssertLock(lsm_db *db, int iLock, int eOp){ - int ret = 0; - int eHave; - - assert( iLock>=1 && iLock<=LSM_LOCK_READER(LSM_LOCK_NREADER-1) ); - assert( iLock<=16 ); - assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); - - eHave = shmLockType(db, iLock); - - switch( eOp ){ - case LSM_LOCK_UNLOCK: - ret = (eHave==LSM_LOCK_UNLOCK); - break; - case LSM_LOCK_SHARED: - ret = (eHave!=LSM_LOCK_UNLOCK); - break; - case LSM_LOCK_EXCL: - ret = (eHave==LSM_LOCK_EXCL); - break; - default: - assert( !"bad eOp value passed to lsmShmAssertLock()" ); - break; - } - - return ret; -} - -int lsmShmAssertWorker(lsm_db *db){ - return lsmShmAssertLock(db, LSM_LOCK_WORKER, LSM_LOCK_EXCL) && db->pWorker; -} - -/* -** This function does not contribute to library functionality, and is not -** included in release builds. It is intended to be called from within -** an interactive debugger. -** -** When called, this function prints a single line of human readable output -** to stdout describing the locks currently held by the connection. For -** example: -** -** (gdb) call print_db_locks(pDb) -** (shared on dms2) (exclusive on writer) -*/ -void print_db_locks(lsm_db *db){ - int iLock; - for(iLock=0; iLock<16; iLock++){ - int bOne = 0; - const char *azLock[] = {0, "shared", "exclusive"}; - const char *azName[] = { - 0, "dms1", "dms2", "writer", "worker", "checkpointer", - "reader0", "reader1", "reader2", "reader3", "reader4", "reader5" - }; - int eHave = shmLockType(db, iLock); - if( azLock[eHave] ){ - printf("%s(%s on %s)", (bOne?" ":""), azLock[eHave], azName[iLock]); - bOne = 1; - } - } - printf("\n"); -} -void print_all_db_locks(lsm_db *db){ - lsm_db *p; - for(p=db->pDatabase->pConn; p; p=p->pNext){ - printf("%s connection %p ", ((p==db)?"*":""), p); - print_db_locks(p); - } -} -#endif - -void lsmShmBarrier(lsm_db *db){ - lsmEnvShmBarrier(db->pEnv); -} - -int lsm_checkpoint(lsm_db *pDb, int *pnKB){ - int rc; /* Return code */ - u32 nWrite = 0; /* Number of pages checkpointed */ - - /* Attempt the checkpoint. If successful, nWrite is set to the number of - ** pages written between this and the previous checkpoint. */ - rc = lsmCheckpointWrite(pDb, &nWrite); - - /* If required, calculate the output variable (KB of data checkpointed). - ** Set it to zero if an error occured. */ - if( pnKB ){ - int nKB = 0; - if( rc==LSM_OK && nWrite ){ - nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; - } - *pnKB = nKB; - } - - return rc; -} diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c deleted file mode 100644 index 6e5243e850..0000000000 --- a/ext/lsm1/lsm_sorted.c +++ /dev/null @@ -1,6195 +0,0 @@ -/* -** 2011-08-14 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** PAGE FORMAT: -** -** The maximum page size is 65536 bytes. -** -** Since all records are equal to or larger than 2 bytes in size, and -** some space within the page is consumed by the page footer, there must -** be less than 2^15 records on each page. -** -** Each page ends with a footer that describes the pages contents. This -** footer serves as similar purpose to the page header in an SQLite database. -** A footer is used instead of a header because it makes it easier to -** populate a new page based on a sorted list of key/value pairs. -** -** The footer consists of the following values (starting at the end of -** the page and continuing backwards towards the start). All values are -** stored as unsigned big-endian integers. -** -** * Number of records on page (2 bytes). -** * Flags field (2 bytes). -** * Left-hand pointer value (8 bytes). -** * The starting offset of each record (2 bytes per record). -** -** Records may span pages. Unless it happens to be an exact fit, the part -** of the final record that starts on page X that does not fit on page X -** is stored at the start of page (X+1). This means there may be pages where -** (N==0). And on most pages the first record that starts on the page will -** not start at byte offset 0. For example: -** -** aaaaa bbbbb ccc
    cc eeeee fffff g
    gggg.... -** -** RECORD FORMAT: -** -** The first byte of the record is a flags byte. It is a combination -** of the following flags (defined in lsmInt.h): -** -** LSM_START_DELETE -** LSM_END_DELETE -** LSM_POINT_DELETE -** LSM_INSERT -** LSM_SEPARATOR -** LSM_SYSTEMKEY -** -** Immediately following the type byte is a pointer to the smallest key -** in the next file that is larger than the key in the current record. The -** pointer is encoded as a varint. When added to the 32-bit page number -** stored in the footer, it is the page number of the page that contains the -** smallest key in the next sorted file that is larger than this key. -** -** Next is the number of bytes in the key, encoded as a varint. -** -** If the LSM_INSERT flag is set, the number of bytes in the value, as -** a varint, is next. -** -** Finally, the blob of data containing the key, and for LSM_INSERT -** records, the value as well. -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -#define LSM_LOG_STRUCTURE 0 -#define LSM_LOG_DATA 0 - -/* -** Macros to help decode record types. -*/ -#define rtTopic(eType) ((eType) & LSM_SYSTEMKEY) -#define rtIsDelete(eType) (((eType) & 0x0F)==LSM_POINT_DELETE) - -#define rtIsSeparator(eType) (((eType) & LSM_SEPARATOR)!=0) -#define rtIsWrite(eType) (((eType) & LSM_INSERT)!=0) -#define rtIsSystem(eType) (((eType) & LSM_SYSTEMKEY)!=0) - -/* -** The following macros are used to access a page footer. -*/ -#define SEGMENT_NRECORD_OFFSET(pgsz) ((pgsz) - 2) -#define SEGMENT_FLAGS_OFFSET(pgsz) ((pgsz) - 2 - 2) -#define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8) -#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2) - -#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1) - -#define SEGMENT_BTREE_FLAG 0x0001 -#define PGFTR_SKIP_NEXT_FLAG 0x0002 -#define PGFTR_SKIP_THIS_FLAG 0x0004 - - -#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD -# define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 -#endif - -typedef struct SegmentPtr SegmentPtr; -typedef struct LsmBlob LsmBlob; - -struct LsmBlob { - lsm_env *pEnv; - void *pData; - int nData; - int nAlloc; -}; - -/* -** A SegmentPtr object may be used for one of two purposes: -** -** * To iterate and/or seek within a single Segment (the combination of a -** main run and an optional sorted run). -** -** * To iterate through the separators array of a segment. -*/ -struct SegmentPtr { - Level *pLevel; /* Level object segment is part of */ - Segment *pSeg; /* Segment to access */ - - /* Current page. See segmentPtrLoadPage(). */ - Page *pPg; /* Current page */ - u16 flags; /* Copy of page flags field */ - int nCell; /* Number of cells on pPg */ - LsmPgno iPtr; /* Base cascade pointer */ - - /* Current cell. See segmentPtrLoadCell() */ - int iCell; /* Current record within page pPg */ - int eType; /* Type of current record */ - LsmPgno iPgPtr; /* Cascade pointer offset */ - void *pKey; int nKey; /* Key associated with current record */ - void *pVal; int nVal; /* Current record value (eType==WRITE only) */ - - /* Blobs used to allocate buffers for pKey and pVal as required */ - LsmBlob blob1; - LsmBlob blob2; -}; - -/* -** Used to iterate through the keys stored in a b-tree hierarchy from start -** to finish. Only First() and Next() operations are required. -** -** btreeCursorNew() -** btreeCursorFirst() -** btreeCursorNext() -** btreeCursorFree() -** btreeCursorPosition() -** btreeCursorRestore() -*/ -typedef struct BtreePg BtreePg; -typedef struct BtreeCursor BtreeCursor; -struct BtreePg { - Page *pPage; - int iCell; -}; -struct BtreeCursor { - Segment *pSeg; /* Iterate through this segments btree */ - FileSystem *pFS; /* File system to read pages from */ - int nDepth; /* Allocated size of aPg[] */ - int iPg; /* Current entry in aPg[]. -1 -> EOF. */ - BtreePg *aPg; /* Pages from root to current location */ - - /* Cache of current entry. pKey==0 for EOF. */ - void *pKey; - int nKey; - int eType; - LsmPgno iPtr; - - /* Storage for key, if not local */ - LsmBlob blob; -}; - - -/* -** A cursor used for merged searches or iterations through up to one -** Tree structure and any number of sorted files. -** -** lsmMCursorNew() -** lsmMCursorSeek() -** lsmMCursorNext() -** lsmMCursorPrev() -** lsmMCursorFirst() -** lsmMCursorLast() -** lsmMCursorKey() -** lsmMCursorValue() -** lsmMCursorValid() -** -** iFree: -** This variable is only used by cursors providing input data for a -** new top-level segment. Such cursors only ever iterate forwards, not -** backwards. -*/ -struct MultiCursor { - lsm_db *pDb; /* Connection that owns this cursor */ - MultiCursor *pNext; /* Next cursor owned by connection pDb */ - int flags; /* Mask of CURSOR_XXX flags */ - - int eType; /* Cache of current key type */ - LsmBlob key; /* Cache of current key (or NULL) */ - LsmBlob val; /* Cache of current value */ - - /* All the component cursors: */ - TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */ - int iFree; /* Next element of free-list (-ve for eof) */ - SegmentPtr *aPtr; /* Array of segment pointers */ - int nPtr; /* Size of array aPtr[] */ - BtreeCursor *pBtCsr; /* b-tree cursor (db writes only) */ - - /* Comparison results */ - int nTree; /* Size of aTree[] array */ - int *aTree; /* Array of comparison results */ - - /* Used by cursors flushing the in-memory tree only */ - void *pSystemVal; /* Pointer to buffer to free */ - - /* Used by worker cursors only */ - LsmPgno *pPrevMergePtr; -}; - -/* -** The following constants are used to assign integers to each component -** cursor of a multi-cursor. -*/ -#define CURSOR_DATA_TREE0 0 /* Current tree cursor (apTreeCsr[0]) */ -#define CURSOR_DATA_TREE1 1 /* The "old" tree, if any (apTreeCsr[1]) */ -#define CURSOR_DATA_SYSTEM 2 /* Free-list entries (new-toplevel only) */ -#define CURSOR_DATA_SEGMENT 3 /* First segment pointer (aPtr[0]) */ - -/* -** CURSOR_IGNORE_DELETE -** If set, this cursor will not visit SORTED_DELETE keys. -** -** CURSOR_FLUSH_FREELIST -** This cursor is being used to create a new toplevel. It should also -** iterate through the contents of the in-memory free block list. -** -** CURSOR_IGNORE_SYSTEM -** If set, this cursor ignores system keys. -** -** CURSOR_NEXT_OK -** Set if it is Ok to call lsm_csr_next(). -** -** CURSOR_PREV_OK -** Set if it is Ok to call lsm_csr_prev(). -** -** CURSOR_READ_SEPARATORS -** Set if this cursor should visit the separator keys in segment -** aPtr[nPtr-1]. -** -** CURSOR_SEEK_EQ -** Cursor has undergone a successful lsm_csr_seek(LSM_SEEK_EQ) operation. -** The key and value are stored in MultiCursor.key and MultiCursor.val -** respectively. -*/ -#define CURSOR_IGNORE_DELETE 0x00000001 -#define CURSOR_FLUSH_FREELIST 0x00000002 -#define CURSOR_IGNORE_SYSTEM 0x00000010 -#define CURSOR_NEXT_OK 0x00000020 -#define CURSOR_PREV_OK 0x00000040 -#define CURSOR_READ_SEPARATORS 0x00000080 -#define CURSOR_SEEK_EQ 0x00000100 - -typedef struct MergeWorker MergeWorker; -typedef struct Hierarchy Hierarchy; - -struct Hierarchy { - Page **apHier; - int nHier; -}; - -/* -** aSave: -** When mergeWorkerNextPage() is called to advance to the next page in -** the output segment, if the bStore flag for an element of aSave[] is -** true, it is cleared and the corresponding iPgno value is set to the -** page number of the page just completed. -** -** aSave[0] is used to record the pointer value to be pushed into the -** b-tree hierarchy. aSave[1] is used to save the page number of the -** page containing the indirect key most recently written to the b-tree. -** see mergeWorkerPushHierarchy() for details. -*/ -struct MergeWorker { - lsm_db *pDb; /* Database handle */ - Level *pLevel; /* Worker snapshot Level being merged */ - MultiCursor *pCsr; /* Cursor to read new segment contents from */ - int bFlush; /* True if this is an in-memory tree flush */ - Hierarchy hier; /* B-tree hierarchy under construction */ - Page *pPage; /* Current output page */ - int nWork; /* Number of calls to mergeWorkerNextPage() */ - LsmPgno *aGobble; /* Gobble point for each input segment */ - - LsmPgno iIndirect; - struct SavedPgno { - LsmPgno iPgno; - int bStore; - } aSave[2]; -}; - -#ifdef LSM_DEBUG_EXPENSIVE -static int assertPointersOk(lsm_db *, Segment *, Segment *, int); -static int assertBtreeOk(lsm_db *, Segment *); -static void assertRunInOrder(lsm_db *pDb, Segment *pSeg); -#else -#define assertRunInOrder(x,y) -#define assertBtreeOk(x,y) -#endif - - -struct FilePage { u8 *aData; int nData; }; -static u8 *fsPageData(Page *pPg, int *pnData){ - *pnData = ((struct FilePage *)(pPg))->nData; - return ((struct FilePage *)(pPg))->aData; -} -/*UNUSED static u8 *fsPageDataPtr(Page *pPg){ - return ((struct FilePage *)(pPg))->aData; -}*/ - -/* -** Write nVal as a 16-bit unsigned big-endian integer into buffer aOut. -*/ -void lsmPutU16(u8 *aOut, u16 nVal){ - aOut[0] = (u8)((nVal>>8) & 0xFF); - aOut[1] = (u8)(nVal & 0xFF); -} - -void lsmPutU32(u8 *aOut, u32 nVal){ - aOut[0] = (u8)((nVal>>24) & 0xFF); - aOut[1] = (u8)((nVal>>16) & 0xFF); - aOut[2] = (u8)((nVal>> 8) & 0xFF); - aOut[3] = (u8)((nVal ) & 0xFF); -} - -int lsmGetU16(u8 *aOut){ - return (aOut[0] << 8) + aOut[1]; -} - -u32 lsmGetU32(u8 *aOut){ - return ((u32)aOut[0] << 24) - + ((u32)aOut[1] << 16) - + ((u32)aOut[2] << 8) - + ((u32)aOut[3]); -} - -u64 lsmGetU64(u8 *aOut){ - return ((u64)aOut[0] << 56) - + ((u64)aOut[1] << 48) - + ((u64)aOut[2] << 40) - + ((u64)aOut[3] << 32) - + ((u64)aOut[4] << 24) - + ((u32)aOut[5] << 16) - + ((u32)aOut[6] << 8) - + ((u32)aOut[7]); -} - -void lsmPutU64(u8 *aOut, u64 nVal){ - aOut[0] = (u8)((nVal>>56) & 0xFF); - aOut[1] = (u8)((nVal>>48) & 0xFF); - aOut[2] = (u8)((nVal>>40) & 0xFF); - aOut[3] = (u8)((nVal>>32) & 0xFF); - aOut[4] = (u8)((nVal>>24) & 0xFF); - aOut[5] = (u8)((nVal>>16) & 0xFF); - aOut[6] = (u8)((nVal>> 8) & 0xFF); - aOut[7] = (u8)((nVal ) & 0xFF); -} - -static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){ - assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) ); - if( pBlob->nAllocpData = lsmReallocOrFree(pEnv, pBlob->pData, nData); - if( !pBlob->pData ) return LSM_NOMEM_BKPT; - pBlob->nAlloc = nData; - pBlob->pEnv = pEnv; - } - return LSM_OK; -} - -static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){ - if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM; - memcpy(pBlob->pData, pData, nData); - pBlob->nData = nData; - return LSM_OK; -} - -#if 0 -static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){ - return sortedBlobSet(pDest, pSrc->pData, pSrc->nData); -} -#endif - -static void sortedBlobFree(LsmBlob *pBlob){ - assert( pBlob->pEnv || pBlob->pData==0 ); - if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); - memset(pBlob, 0, sizeof(LsmBlob)); -} - -static int sortedReadData( - Segment *pSeg, - Page *pPg, - int iOff, - int nByte, - void **ppData, - LsmBlob *pBlob -){ - int rc = LSM_OK; - int iEnd; - int nData; - int nCell; - u8 *aData; - - aData = fsPageData(pPg, &nData); - nCell = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); - iEnd = SEGMENT_EOF(nData, nCell); - assert( iEnd>0 && iEndnData = nByte; - aDest = (u8 *)pBlob->pData; - *ppData = pBlob->pData; - - /* Increment the pointer pages ref-count. */ - lsmFsPageRef(pPg); - - while( rc==LSM_OK ){ - Page *pNext; - int flags; - - /* Copy data from pPg into the output buffer. */ - int nCopy = LSM_MIN(nRem, iEnd-i); - if( nCopy>0 ){ - memcpy(&aDest[nByte-nRem], &aData[i], nCopy); - nRem -= nCopy; - i += nCopy; - assert( nRem==0 || i==iEnd ); - } - assert( nRem>=0 ); - if( nRem==0 ) break; - i -= iEnd; - - /* Grab the next page in the segment */ - - do { - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - if( rc==LSM_OK && pNext==0 ){ - rc = LSM_CORRUPT_BKPT; - } - if( rc ) break; - lsmFsPageRelease(pPg); - pPg = pNext; - aData = fsPageData(pPg, &nData); - flags = lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]); - }while( flags&SEGMENT_BTREE_FLAG ); - - iEnd = SEGMENT_EOF(nData, lsmGetU16(&aData[nData-2])); - assert( iEnd>0 && iEnd=0 ); - aCell = pageGetCell(aData, nData, iCell); - lsmVarintGet64(&aCell[1], &iRet); - return iRet; -} - -static u8 *pageGetKey( - Segment *pSeg, /* Segment pPg belongs to */ - Page *pPg, /* Page to read from */ - int iCell, /* Index of cell on page to read */ - int *piTopic, /* OUT: Topic associated with this key */ - int *pnKey, /* OUT: Size of key in bytes */ - LsmBlob *pBlob /* If required, use this for dynamic memory */ -){ - u8 *pKey; - i64 nDummy; - int eType; - u8 *aData; - int nData; - - aData = fsPageData(pPg, &nData); - - assert( !(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); - assert( iCellpData || nKey==pBlob->nData ); - if( (void *)aKey!=pBlob->pData ){ - rc = sortedBlobSet(pEnv, pBlob, aKey, nKey); - } - - return rc; -} - -static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){ - LsmPgno iRef; - u8 *aData; - int nData; - u8 *aCell; - - aData = fsPageData(pPg, &nData); - aCell = pageGetCell(aData, nData, iKey); - assert( aCell[0]==0 ); - aCell++; - aCell += lsmVarintGet64(aCell, &iRef); - lsmVarintGet64(aCell, &iRef); - assert( iRef>0 ); - return iRef; -} - -#define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i))) -#define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i))) - -static int pageGetBtreeKey( - Segment *pSeg, /* Segment page pPg belongs to */ - Page *pPg, - int iKey, - LsmPgno *piPtr, - int *piTopic, - void **ppKey, - int *pnKey, - LsmBlob *pBlob -){ - u8 *aData; - int nData; - u8 *aCell; - int eType; - - aData = fsPageData(pPg, &nData); - assert( SEGMENT_BTREE_FLAG & pageGetFlags(aData, nData) ); - assert( iKey>=0 && iKeypData; - *pnKey = pBlob->nData; - }else{ - aCell += GETVARINT32(aCell, *pnKey); - *ppKey = aCell; - } - if( piTopic ) *piTopic = rtTopic(eType); - - return LSM_OK; -} - -static int btreeCursorLoadKey(BtreeCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->iPg<0 ){ - pCsr->pKey = 0; - pCsr->nKey = 0; - pCsr->eType = 0; - }else{ - LsmPgno dummy; - int iPg = pCsr->iPg; - int iCell = pCsr->aPg[iPg].iCell; - while( iCell<0 && (--iPg)>=0 ){ - iCell = pCsr->aPg[iPg].iCell-1; - } - if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT; - - rc = pageGetBtreeKey( - pCsr->pSeg, - pCsr->aPg[iPg].pPage, iCell, - &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob - ); - pCsr->eType |= LSM_SEPARATOR; - } - - return rc; -} - -static LsmPgno btreeCursorPtr(u8 *aData, int nData, int iCell){ - int nCell; - - nCell = pageGetNRec(aData, nData); - if( iCell>=nCell ){ - return pageGetPtr(aData, nData); - } - return pageGetRecordPtr(aData, nData, iCell); -} - -static int btreeCursorNext(BtreeCursor *pCsr){ - int rc = LSM_OK; - - BtreePg *pPg = &pCsr->aPg[pCsr->iPg]; - int nCell; - u8 *aData; - int nData; - - assert( pCsr->iPg>=0 ); - assert( pCsr->iPg==pCsr->nDepth-1 ); - - aData = fsPageData(pPg->pPage, &nData); - nCell = pageGetNRec(aData, nData); - assert( pPg->iCell<=nCell ); - pPg->iCell++; - if( pPg->iCell==nCell ){ - LsmPgno iLoad; - - /* Up to parent. */ - lsmFsPageRelease(pPg->pPage); - pPg->pPage = 0; - pCsr->iPg--; - while( pCsr->iPg>=0 ){ - pPg = &pCsr->aPg[pCsr->iPg]; - aData = fsPageData(pPg->pPage, &nData); - if( pPg->iCellpPage); - pCsr->iPg--; - } - - /* Read the key */ - rc = btreeCursorLoadKey(pCsr); - - /* Unless the cursor is at EOF, descend to cell -1 (yes, negative one) of - ** the left-most most descendent. */ - if( pCsr->iPg>=0 ){ - pCsr->aPg[pCsr->iPg].iCell++; - - iLoad = btreeCursorPtr(aData, nData, pPg->iCell); - do { - Page *pLoad; - pCsr->iPg++; - rc = lsmFsDbPageGet(pCsr->pFS, pCsr->pSeg, iLoad, &pLoad); - pCsr->aPg[pCsr->iPg].pPage = pLoad; - pCsr->aPg[pCsr->iPg].iCell = 0; - if( rc==LSM_OK ){ - if( pCsr->iPg==(pCsr->nDepth-1) ) break; - aData = fsPageData(pLoad, &nData); - iLoad = btreeCursorPtr(aData, nData, 0); - } - }while( rc==LSM_OK && pCsr->iPg<(pCsr->nDepth-1) ); - pCsr->aPg[pCsr->iPg].iCell = -1; - } - - }else{ - rc = btreeCursorLoadKey(pCsr); - } - - if( rc==LSM_OK && pCsr->iPg>=0 ){ - aData = fsPageData(pCsr->aPg[pCsr->iPg].pPage, &nData); - pCsr->iPtr = btreeCursorPtr(aData, nData, pCsr->aPg[pCsr->iPg].iCell+1); - } - - return rc; -} - -static void btreeCursorFree(BtreeCursor *pCsr){ - if( pCsr ){ - int i; - lsm_env *pEnv = lsmFsEnv(pCsr->pFS); - for(i=0; i<=pCsr->iPg; i++){ - lsmFsPageRelease(pCsr->aPg[i].pPage); - } - sortedBlobFree(&pCsr->blob); - lsmFree(pEnv, pCsr->aPg); - lsmFree(pEnv, pCsr); - } -} - -static int btreeCursorFirst(BtreeCursor *pCsr){ - int rc; - - Page *pPg = 0; - FileSystem *pFS = pCsr->pFS; - LsmPgno iPg = pCsr->pSeg->iRoot; - - do { - rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg); - assert( (rc==LSM_OK)==(pPg!=0) ); - if( rc==LSM_OK ){ - u8 *aData; - int nData; - int flags; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; - - if( (pCsr->nDepth % 8)==0 ){ - int nNew = pCsr->nDepth + 8; - pCsr->aPg = (BtreePg *)lsmReallocOrFreeRc( - lsmFsEnv(pFS), pCsr->aPg, sizeof(BtreePg) * nNew, &rc - ); - if( rc==LSM_OK ){ - memset(&pCsr->aPg[pCsr->nDepth], 0, sizeof(BtreePg) * 8); - } - } - - if( rc==LSM_OK ){ - assert( pCsr->aPg[pCsr->nDepth].iCell==0 ); - pCsr->aPg[pCsr->nDepth].pPage = pPg; - pCsr->nDepth++; - iPg = pageGetRecordPtr(aData, nData, 0); - } - } - }while( rc==LSM_OK ); - lsmFsPageRelease(pPg); - pCsr->iPg = pCsr->nDepth-1; - - if( rc==LSM_OK && pCsr->nDepth ){ - pCsr->aPg[pCsr->iPg].iCell = -1; - rc = btreeCursorNext(pCsr); - } - - return rc; -} - -static void btreeCursorPosition(BtreeCursor *pCsr, MergeInput *p){ - if( pCsr->iPg>=0 ){ - p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); - p->iCell = ((pCsr->aPg[pCsr->iPg].iCell + 1) << 8) + pCsr->nDepth; - }else{ - p->iPg = 0; - p->iCell = 0; - } -} - -static void btreeCursorSplitkey(BtreeCursor *pCsr, MergeInput *p){ - int iCell = pCsr->aPg[pCsr->iPg].iCell; - if( iCell>=0 ){ - p->iCell = iCell; - p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); - }else{ - int i; - for(i=pCsr->iPg-1; i>=0; i--){ - if( pCsr->aPg[i].iCell>0 ) break; - } - assert( i>=0 ); - p->iCell = pCsr->aPg[i].iCell-1; - p->iPg = lsmFsPageNumber(pCsr->aPg[i].pPage); - } -} - -static int sortedKeyCompare( - int (*xCmp)(void *, int, void *, int), - int iLhsTopic, void *pLhsKey, int nLhsKey, - int iRhsTopic, void *pRhsKey, int nRhsKey -){ - int res = iLhsTopic - iRhsTopic; - if( res==0 ){ - res = xCmp(pLhsKey, nLhsKey, pRhsKey, nRhsKey); - } - return res; -} - -static int btreeCursorRestore( - BtreeCursor *pCsr, - int (*xCmp)(void *, int, void *, int), - MergeInput *p -){ - int rc = LSM_OK; - - if( p->iPg ){ - lsm_env *pEnv = lsmFsEnv(pCsr->pFS); - int iCell; /* Current cell number on leaf page */ - LsmPgno iLeaf; /* Page number of current leaf page */ - int nDepth; /* Depth of b-tree structure */ - Segment *pSeg = pCsr->pSeg; - - /* Decode the MergeInput structure */ - iLeaf = p->iPg; - nDepth = (p->iCell & 0x00FF); - iCell = (p->iCell >> 8) - 1; - - /* Allocate the BtreeCursor.aPg[] array */ - assert( pCsr->aPg==0 ); - pCsr->aPg = (BtreePg *)lsmMallocZeroRc(pEnv, sizeof(BtreePg) * nDepth, &rc); - - /* Populate the last entry of the aPg[] array */ - if( rc==LSM_OK ){ - Page **pp = &pCsr->aPg[nDepth-1].pPage; - pCsr->iPg = nDepth-1; - pCsr->nDepth = nDepth; - pCsr->aPg[pCsr->iPg].iCell = iCell; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); - } - - /* Populate any other aPg[] array entries */ - if( rc==LSM_OK && nDepth>1 ){ - LsmBlob blob = {0,0,0}; - void *pSeek; - int nSeek; - int iTopicSeek; - int iPg = 0; - LsmPgno iLoad = pSeg->iRoot; - Page *pPg = pCsr->aPg[nDepth-1].pPage; - - if( pageObjGetNRec(pPg)==0 ){ - /* This can happen when pPg is the right-most leaf in the b-tree. - ** In this case, set the iTopicSeek/pSeek/nSeek key to a value - ** greater than any real key. */ - assert( iCell==-1 ); - iTopicSeek = 1000; - pSeek = 0; - nSeek = 0; - }else{ - LsmPgno dummy; - rc = pageGetBtreeKey(pSeg, pPg, - 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob - ); - } - - do { - Page *pPg2; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLoad, &pPg2); - assert( rc==LSM_OK || pPg2==0 ); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer containing page data */ - int nData; /* Size of aData[] in bytes */ - int iMin; - int iMax; - int iCell2; - - aData = fsPageData(pPg2, &nData); - assert( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); - - iLoad = pageGetPtr(aData, nData); - iCell2 = pageGetNRec(aData, nData); - iMax = iCell2-1; - iMin = 0; - - while( iMax>=iMin ){ - int iTry = (iMin+iMax)/2; - void *pKey; int nKey; /* Key for cell iTry */ - int iTopic; /* Topic for key pKeyT/nKeyT */ - LsmPgno iPtr; /* Pointer for cell iTry */ - int res; /* (pSeek - pKeyT) */ - - rc = pageGetBtreeKey( - pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob - ); - if( rc!=LSM_OK ) break; - - res = sortedKeyCompare( - xCmp, iTopicSeek, pSeek, nSeek, iTopic, pKey, nKey - ); - assert( res!=0 ); - - if( res<0 ){ - iLoad = iPtr; - iCell2 = iTry; - iMax = iTry-1; - }else{ - iMin = iTry+1; - } - } - - pCsr->aPg[iPg].pPage = pPg2; - pCsr->aPg[iPg].iCell = iCell2; - iPg++; - assert( iPg!=nDepth-1 - || lsmFsRedirectPage(pCsr->pFS, pSeg->pRedirect, iLoad)==iLeaf - ); - } - }while( rc==LSM_OK && iPg<(nDepth-1) ); - sortedBlobFree(&blob); - } - - /* Load the current key and pointer */ - if( rc==LSM_OK ){ - BtreePg *pBtreePg; - u8 *aData; - int nData; - - pBtreePg = &pCsr->aPg[pCsr->iPg]; - aData = fsPageData(pBtreePg->pPage, &nData); - pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1); - if( pBtreePg->iCell<0 ){ - LsmPgno dummy; - int i; - for(i=pCsr->iPg-1; i>=0; i--){ - if( pCsr->aPg[i].iCell>0 ) break; - } - assert( i>=0 ); - rc = pageGetBtreeKey(pSeg, - pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1, - &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob - ); - pCsr->eType |= LSM_SEPARATOR; - - }else{ - rc = btreeCursorLoadKey(pCsr); - } - } - } - return rc; -} - -static int btreeCursorNew( - lsm_db *pDb, - Segment *pSeg, - BtreeCursor **ppCsr -){ - int rc = LSM_OK; - BtreeCursor *pCsr; - - assert( pSeg->iRoot ); - pCsr = lsmMallocZeroRc(pDb->pEnv, sizeof(BtreeCursor), &rc); - if( pCsr ){ - pCsr->pFS = pDb->pFS; - pCsr->pSeg = pSeg; - pCsr->iPg = -1; - } - - *ppCsr = pCsr; - return rc; -} - -static void segmentPtrSetPage(SegmentPtr *pPtr, Page *pNext){ - lsmFsPageRelease(pPtr->pPg); - if( pNext ){ - int nData; - u8 *aData = fsPageData(pNext, &nData); - pPtr->nCell = pageGetNRec(aData, nData); - pPtr->flags = (u16)pageGetFlags(aData, nData); - pPtr->iPtr = pageGetPtr(aData, nData); - } - pPtr->pPg = pNext; -} - -/* -** Load a new page into the SegmentPtr object pPtr. -*/ -static int segmentPtrLoadPage( - FileSystem *pFS, - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - LsmPgno iNew /* Page number of new page */ -){ - Page *pPg = 0; /* The new page */ - int rc; /* Return Code */ - - rc = lsmFsDbPageGet(pFS, pPtr->pSeg, iNew, &pPg); - assert( rc==LSM_OK || pPg==0 ); - segmentPtrSetPage(pPtr, pPg); - - return rc; -} - -static int segmentPtrReadData( - SegmentPtr *pPtr, - int iOff, - int nByte, - void **ppData, - LsmBlob *pBlob -){ - return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); -} - -static int segmentPtrNextPage( - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - int eDir /* +1 for next(), -1 for prev() */ -){ - Page *pNext; /* New page to load */ - int rc; /* Return code */ - - assert( eDir==1 || eDir==-1 ); - assert( pPtr->pPg ); - assert( pPtr->pSeg || eDir>0 ); - - rc = lsmFsDbPageNext(pPtr->pSeg, pPtr->pPg, eDir, &pNext); - assert( rc==LSM_OK || pNext==0 ); - segmentPtrSetPage(pPtr, pNext); - return rc; -} - -static int segmentPtrLoadCell( - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - int iNew /* Cell number of new cell */ -){ - int rc = LSM_OK; - if( pPtr->pPg ){ - u8 *aData; /* Pointer to page data buffer */ - int iOff; /* Offset in aData[] to read from */ - int nPgsz; /* Size of page (aData[]) in bytes */ - - assert( iNewnCell ); - pPtr->iCell = iNew; - aData = fsPageData(pPtr->pPg, &nPgsz); - iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nPgsz, pPtr->iCell)]); - pPtr->eType = aData[iOff]; - iOff++; - iOff += GETVARINT64(&aData[iOff], pPtr->iPgPtr); - iOff += GETVARINT32(&aData[iOff], pPtr->nKey); - if( rtIsWrite(pPtr->eType) ){ - iOff += GETVARINT32(&aData[iOff], pPtr->nVal); - } - assert( pPtr->nKey>=0 ); - - rc = segmentPtrReadData( - pPtr, iOff, pPtr->nKey, &pPtr->pKey, &pPtr->blob1 - ); - if( rc==LSM_OK && rtIsWrite(pPtr->eType) ){ - rc = segmentPtrReadData( - pPtr, iOff+pPtr->nKey, pPtr->nVal, &pPtr->pVal, &pPtr->blob2 - ); - }else{ - pPtr->nVal = 0; - pPtr->pVal = 0; - } - } - - return rc; -} - - -static Segment *sortedSplitkeySegment(Level *pLevel){ - Merge *pMerge = pLevel->pMerge; - MergeInput *p = &pMerge->splitkey; - Segment *pSeg; - int i; - - for(i=0; inInput; i++){ - if( p->iPg==pMerge->aInput[i].iPg ) break; - } - if( pMerge->nInput==(pLevel->nRight+1) && i>=(pMerge->nInput-1) ){ - pSeg = &pLevel->pNext->lhs; - }else{ - pSeg = &pLevel->aRhs[i]; - } - - return pSeg; -} - -static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ - Segment *pSeg; - Page *pPg = 0; - lsm_env *pEnv = pDb->pEnv; /* Environment handle */ - int rc = *pRc; - Merge *pMerge = pLevel->pMerge; - - pSeg = sortedSplitkeySegment(pLevel); - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); - } - if( rc==LSM_OK ){ - int iTopic; - LsmBlob blob = {0, 0, 0, 0}; - u8 *aData; - int nData; - - aData = lsmFsPageData(pPg, &nData); - if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ - void *pKey; - int nKey; - LsmPgno dummy; - rc = pageGetBtreeKey(pSeg, - pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob - ); - if( rc==LSM_OK && blob.pData!=pKey ){ - rc = sortedBlobSet(pEnv, &blob, pKey, nKey); - } - }else{ - rc = pageGetKeyCopy( - pEnv, pSeg, pPg, pMerge->splitkey.iCell, &iTopic, &blob - ); - } - - pLevel->iSplitTopic = iTopic; - pLevel->pSplitKey = blob.pData; - pLevel->nSplitKey = blob.nData; - lsmFsPageRelease(pPg); - } - - *pRc = rc; -} - -/* -** Reset a segment cursor. Also free its buffers if they are nThreshold -** bytes or larger in size. -*/ -static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){ - lsmFsPageRelease(pPtr->pPg); - pPtr->pPg = 0; - pPtr->nCell = 0; - pPtr->pKey = 0; - pPtr->nKey = 0; - pPtr->pVal = 0; - pPtr->nVal = 0; - pPtr->eType = 0; - pPtr->iCell = 0; - if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1); - if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2); -} - -static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){ - return (pCsr->flags & CURSOR_READ_SEPARATORS)==0 - || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]); -} - -static int segmentPtrAdvance( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int bReverse -){ - int eDir = (bReverse ? -1 : 1); - Level *pLvl = pPtr->pLevel; - do { - int rc; - int iCell; /* Number of new cell in page */ - int svFlags = 0; /* SegmentPtr.eType before advance */ - - iCell = pPtr->iCell + eDir; - assert( pPtr->pPg ); - assert( iCell<=pPtr->nCell && iCell>=-1 ); - - if( bReverse && pPtr->pSeg!=&pPtr->pLevel->lhs ){ - svFlags = pPtr->eType; - assert( svFlags ); - } - - if( iCell>=pPtr->nCell || iCell<0 ){ - do { - rc = segmentPtrNextPage(pPtr, eDir); - }while( rc==LSM_OK - && pPtr->pPg - && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG) ) - ); - if( rc!=LSM_OK ) return rc; - iCell = bReverse ? (pPtr->nCell-1) : 0; - } - rc = segmentPtrLoadCell(pPtr, iCell); - if( rc!=LSM_OK ) return rc; - - if( svFlags && pPtr->pPg ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - - if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ - Segment *pSeg = pPtr->pSeg; - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); - if( rc!=LSM_OK ) return rc; - pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; - pPtr->eType |= (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - } - - }while( pCsr - && pPtr->pPg - && segmentPtrIgnoreSeparators(pCsr, pPtr) - && rtIsSeparator(pPtr->eType) - ); - - return LSM_OK; -} - -static void segmentPtrEndPage( - FileSystem *pFS, - SegmentPtr *pPtr, - int bLast, - int *pRc -){ - if( *pRc==LSM_OK ){ - Segment *pSeg = pPtr->pSeg; - Page *pNew = 0; - if( bLast ){ - *pRc = lsmFsDbPageLast(pFS, pSeg, &pNew); - }else{ - *pRc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pNew); - } - segmentPtrSetPage(pPtr, pNew); - } -} - - -/* -** Try to move the segment pointer passed as the second argument so that it -** points at either the first (bLast==0) or last (bLast==1) cell in the valid -** region of the segment defined by pPtr->iFirst and pPtr->iLast. -** -** Return LSM_OK if successful or an lsm error code if something goes -** wrong (IO error, OOM etc.). -*/ -static int segmentPtrEnd(MultiCursor *pCsr, SegmentPtr *pPtr, int bLast){ - Level *pLvl = pPtr->pLevel; - int rc = LSM_OK; - FileSystem *pFS = pCsr->pDb->pFS; - int bIgnore; - - segmentPtrEndPage(pFS, pPtr, bLast, &rc); - while( rc==LSM_OK && pPtr->pPg - && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG)) - ){ - rc = segmentPtrNextPage(pPtr, (bLast ? -1 : 1)); - } - - if( rc==LSM_OK && pPtr->pPg ){ - rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); - if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - - bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); - if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ - rc = segmentPtrAdvance(pCsr, pPtr, bLast); - } - -#if 0 - if( bLast && rc==LSM_OK && pPtr->pPg - && pPtr->pSeg==&pLvl->lhs - && pLvl->nRight && (pPtr->eType & LSM_START_DELETE) - ){ - pPtr->iCell++; - pPtr->eType = LSM_END_DELETE | (pLvl->iSplitTopic); - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - pPtr->pVal = 0; - pPtr->nVal = 0; - } -#endif - - return rc; -} - -static void segmentPtrKey(SegmentPtr *pPtr, void **ppKey, int *pnKey){ - assert( pPtr->pPg ); - *ppKey = pPtr->pKey; - *pnKey = pPtr->nKey; -} - -#if 0 /* NOT USED */ -static char *keyToString(lsm_env *pEnv, void *pKey, int nKey){ - int i; - u8 *aKey = (u8 *)pKey; - char *zRet = (char *)lsmMalloc(pEnv, nKey+1); - - for(i=0; ipDb->pFS); - LsmBlob blob = {0, 0, 0}; - int eDir; - int iTopic = 0; /* TODO: Fix me */ - - for(eDir=-1; eDir<=1; eDir+=2){ - Page *pTest = pPtr->pPg; - - lsmFsPageRef(pTest); - while( pTest ){ - Segment *pSeg = pPtr->pSeg; - Page *pNext; - - int rc = lsmFsDbPageNext(pSeg, pTest, eDir, &pNext); - lsmFsPageRelease(pTest); - if( rc ) return 1; - pTest = pNext; - - if( pTest ){ - int nData; - u8 *aData = fsPageData(pTest, &nData); - int nCell = pageGetNRec(aData, nData); - int flags = pageGetFlags(aData, nData); - if( nCell && 0==(flags&SEGMENT_BTREE_FLAG) ){ - int nPgKey; - int iPgTopic; - u8 *pPgKey; - int res; - int iCell; - - iCell = ((eDir < 0) ? (nCell-1) : 0); - pPgKey = pageGetKey(pSeg, pTest, iCell, &iPgTopic, &nPgKey, &blob); - res = iTopic - iPgTopic; - if( res==0 ) res = pCsr->pDb->xCmp(pKey, nKey, pPgKey, nPgKey); - if( (eDir==1 && res>0) || (eDir==-1 && res<0) ){ - /* Taking this branch means something has gone wrong. */ - char *zMsg = lsmMallocPrintf(pEnv, "Key \"%s\" is not on page %d", - keyToString(pEnv, pKey, nKey), lsmFsPageNumber(pPtr->pPg) - ); - fprintf(stderr, "%s\n", zMsg); - assert( !"assertKeyLocation() failed" ); - } - lsmFsPageRelease(pTest); - pTest = 0; - } - } - } - } - - sortedBlobFree(&blob); - return 1; -} -#endif - -#ifndef NDEBUG -static int assertSeekResult( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int iTopic, - void *pKey, - int nKey, - int eSeek -){ - if( pPtr->pPg ){ - int res; - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey - ); - - if( eSeek==LSM_SEEK_EQ ) return (res==0); - if( eSeek==LSM_SEEK_LE ) return (res>=0); - if( eSeek==LSM_SEEK_GE ) return (res<=0); - } - - return 1; -} -#endif - -static int segmentPtrSearchOversized( - MultiCursor *pCsr, /* Cursor context */ - SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Topic of key to search for */ - void *pKey, int nKey /* Key to seek to */ -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int rc = LSM_OK; - - /* If the OVERSIZED flag is set, then there is no pointer in the - ** upper level to the next page in the segment that contains at least - ** one key. So compare the largest key on the current page with the - ** key being sought (pKey/nKey). If (pKey/nKey) is larger, advance - ** to the next page in the segment that contains at least one key. - */ - while( rc==LSM_OK && (pPtr->flags & PGFTR_SKIP_NEXT_FLAG) ){ - u8 *pLastKey; - int nLastKey; - int iLastTopic; - int res; /* Result of comparison */ - Page *pNext; - - /* Load the last key on the current page. */ - pLastKey = pageGetKey(pPtr->pSeg, - pPtr->pPg, pPtr->nCell-1, &iLastTopic, &nLastKey, &pPtr->blob1 - ); - - /* If the loaded key is >= than (pKey/nKey), break out of the loop. - ** If (pKey/nKey) is present in this array, it must be on the current - ** page. */ - res = sortedKeyCompare( - xCmp, iLastTopic, pLastKey, nLastKey, iTopic, pKey, nKey - ); - if( res>=0 ) break; - - /* Advance to the next page that contains at least one key. */ - pNext = pPtr->pPg; - lsmFsPageRef(pNext); - while( 1 ){ - Page *pLoad; - u8 *aData; int nData; - - rc = lsmFsDbPageNext(pPtr->pSeg, pNext, 1, &pLoad); - lsmFsPageRelease(pNext); - pNext = pLoad; - if( pNext==0 ) break; - - assert( rc==LSM_OK ); - aData = lsmFsPageData(pNext, &nData); - if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 - && pageGetNRec(aData, nData)>0 - ){ - break; - } - } - if( pNext==0 ) break; - segmentPtrSetPage(pPtr, pNext); - - /* This should probably be an LSM_CORRUPT error. */ - assert( rc!=LSM_OK || (pPtr->flags & PGFTR_SKIP_THIS_FLAG) ); - } - - return rc; -} - -static int ptrFwdPointer( - Page *pPage, - int iCell, - Segment *pSeg, - LsmPgno *piPtr, - int *pbFound -){ - Page *pPg = pPage; - int iFirst = iCell; - int rc = LSM_OK; - - do { - Page *pNext = 0; - u8 *aData; - int nData; - - aData = lsmFsPageData(pPg, &nData); - if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 ){ - int i; - int nCell = pageGetNRec(aData, nData); - for(i=iFirst; ipPg && rc==LSM_OK ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey - ); - if( res<=0 ) break; - rc = segmentPtrAdvance(pCsr, pPtr, 0); - } - return rc; -} - - -/* -** This function is called as part of a SEEK_GE op on a multi-cursor if the -** FC pointer read from segment *pPtr comes from an entry with the -** LSM_START_DELETE flag set. In this case the pointer value cannot be -** trusted. Instead, the pointer that should be followed is that associated -** with the next entry in *pPtr that does not have LSM_START_DELETE set. -** -** Why the pointers can't be trusted: -** -** -** -** TODO: This is a stop-gap solution: -** -** At the moment, this function is called from within segmentPtrSeek(), -** as part of the initial lsmMCursorSeek() call. However, consider a -** database where the following has occurred: -** -** 1. A range delete removes keys 1..9999 using a range delete. -** 2. Keys 1 through 9999 are reinserted. -** 3. The levels containing the ops in 1. and 2. above are merged. Call -** this level N. Level N contains FC pointers to level N+1. -** -** Then, if the user attempts to query for (key>=2 LIMIT 10), the -** lsmMCursorSeek() call will iterate through 9998 entries searching for a -** pointer down to the level N+1 that is never actually used. It would be -** much better if the multi-cursor could do this lazily - only seek to the -** level (N+1) page after the user has moved the cursor on level N passed -** the big range-delete. -*/ -static int segmentPtrFwdPointer( - MultiCursor *pCsr, /* Multi-cursor pPtr belongs to */ - SegmentPtr *pPtr, /* Segment-pointer to extract FC ptr from */ - LsmPgno *piPtr /* OUT: FC pointer value */ -){ - Level *pLvl = pPtr->pLevel; - Level *pNext = pLvl->pNext; - Page *pPg = pPtr->pPg; - int rc; - int bFound; - LsmPgno iOut = 0; - - if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){ - if( pNext==0 - || (pNext->nRight==0 && pNext->lhs.iRoot) - || (pNext->nRight!=0 && pNext->aRhs[0].iRoot) - ){ - /* Do nothing. The pointer will not be used anyway. */ - return LSM_OK; - } - }else{ - if( pPtr[1].pSeg->iRoot ){ - return LSM_OK; - } - } - - /* Search for a pointer within the current segment. */ - lsmFsPageRef(pPg); - rc = ptrFwdPointer(pPg, pPtr->iCell, pPtr->pSeg, &iOut, &bFound); - - if( rc==LSM_OK && bFound==0 ){ - /* This case happens when pPtr points to the left-hand-side of a segment - ** currently undergoing an incremental merge. In this case, jump to the - ** oldest segment in the right-hand-side of the same level and continue - ** searching. But - do not consider any keys smaller than the levels - ** split-key. */ - SegmentPtr ptr; - - if( pPtr->pLevel->nRight==0 || pPtr->pSeg!=&pPtr->pLevel->lhs ){ - return LSM_CORRUPT_BKPT; - } - - memset(&ptr, 0, sizeof(SegmentPtr)); - ptr.pLevel = pPtr->pLevel; - ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; - rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); - if( rc==LSM_OK ){ - rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); - ptr.pPg = 0; - } - segmentPtrReset(&ptr, 0); - } - - *piPtr = iOut; - return rc; -} - -static int segmentPtrSeek( - MultiCursor *pCsr, /* Cursor context */ - SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Key topic to seek to */ - void *pKey, int nKey, /* Key to seek to */ - int eSeek, /* Search bias - see above */ - LsmPgno *piPtr, /* OUT: FC pointer */ - int *pbStop -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int res = 0; /* Result of comparison operation */ - int rc = LSM_OK; - int iMin; - int iMax; - LsmPgno iPtrOut = 0; - - /* If the current page contains an oversized entry, then there are no - ** pointers to one or more of the subsequent pages in the sorted run. - ** The following call ensures that the segment-ptr points to the correct - ** page in this case. */ - rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey); - iPtrOut = pPtr->iPtr; - - /* Assert that this page is the right page of this segment for the key - ** that we are searching for. Do this by loading page (iPg-1) and testing - ** that pKey/nKey is greater than all keys on that page, and then by - ** loading (iPg+1) and testing that pKey/nKey is smaller than all - ** the keys it houses. - ** - ** TODO: With range-deletes in the tree, the test described above may fail. - */ -#if 0 - assert( assertKeyLocation(pCsr, pPtr, pKey, nKey) ); -#endif - - assert( pPtr->nCell>0 - || pPtr->pSeg->nSize==1 - || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) - ); - if( pPtr->nCell==0 ){ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - }else{ - iMin = 0; - iMax = pPtr->nCell-1; - - while( 1 ){ - int iTry = (iMin+iMax)/2; - void *pKeyT; int nKeyT; /* Key for cell iTry */ - int iTopicT; - - assert( iTryeType); - - res = sortedKeyCompare(xCmp, iTopicT, pKeyT, nKeyT, iTopic, pKey, nKey); - if( res<=0 ){ - iPtrOut = pPtr->iPtr + pPtr->iPgPtr; - } - - if( res==0 || iMin==iMax ){ - break; - }else if( res>0 ){ - iMax = LSM_MAX(iTry-1, iMin); - }else{ - iMin = iTry+1; - } - } - - if( rc==LSM_OK ){ - assert( res==0 || (iMin==iMax && iMin>=0 && iMinnCell) ); - if( res ){ - rc = segmentPtrLoadCell(pPtr, iMin); - } - assert( rc!=LSM_OK || res>0 || iPtrOut==(pPtr->iPtr + pPtr->iPgPtr) ); - - if( rc==LSM_OK ){ - switch( eSeek ){ - case LSM_SEEK_EQ: { - int eType = pPtr->eType; - if( (res<0 && (eType & LSM_START_DELETE)) - || (res>0 && (eType & LSM_END_DELETE)) - || (res==0 && (eType & LSM_POINT_DELETE)) - ){ - *pbStop = 1; - }else if( res==0 && (eType & LSM_INSERT) ){ - lsm_env *pEnv = pCsr->pDb->pEnv; - *pbStop = 1; - pCsr->eType = pPtr->eType; - rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); - if( rc==LSM_OK ){ - rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); - } - pCsr->flags |= CURSOR_SEEK_EQ; - } - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - break; - } - case LSM_SEEK_LE: - if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1); - break; - case LSM_SEEK_GE: { - /* Figure out if we need to 'skip' the pointer forward or not */ - if( (res<=0 && (pPtr->eType & LSM_START_DELETE)) - || (res>0 && (pPtr->eType & LSM_END_DELETE)) - ){ - rc = segmentPtrFwdPointer(pCsr, pPtr, &iPtrOut); - } - if( res<0 && rc==LSM_OK ){ - rc = segmentPtrAdvance(pCsr, pPtr, 0); - } - break; - } - } - } - } - - /* If the cursor seek has found a separator key, and this cursor is - ** supposed to ignore separators keys, advance to the next entry. */ - if( rc==LSM_OK && pPtr->pPg - && segmentPtrIgnoreSeparators(pCsr, pPtr) - && rtIsSeparator(pPtr->eType) - ){ - assert( eSeek!=LSM_SEEK_EQ ); - rc = segmentPtrAdvance(pCsr, pPtr, eSeek==LSM_SEEK_LE); - } - } - - assert( rc!=LSM_OK || assertSeekResult(pCsr,pPtr,iTopic,pKey,nKey,eSeek) ); - *piPtr = iPtrOut; - return rc; -} - -static int seekInBtree( - MultiCursor *pCsr, /* Multi-cursor object */ - Segment *pSeg, /* Seek within this segment */ - int iTopic, - void *pKey, int nKey, /* Key to seek to */ - LsmPgno *aPg, /* OUT: Page numbers */ - Page **ppPg /* OUT: Leaf (sorted-run) page reference */ -){ - int i = 0; - int rc; - LsmPgno iPg; - Page *pPg = 0; - LsmBlob blob = {0, 0, 0}; - - iPg = pSeg->iRoot; - do { - LsmPgno *piFirst = 0; - if( aPg ){ - aPg[i++] = iPg; - piFirst = &aPg[i]; - } - - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg); - assert( rc==LSM_OK || pPg==0 ); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer containing page data */ - int nData; /* Size of aData[] in bytes */ - int iMin; - int iMax; - int nRec; - int flags; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; - - iPg = pageGetPtr(aData, nData); - nRec = pageGetNRec(aData, nData); - - iMin = 0; - iMax = nRec-1; - while( iMax>=iMin ){ - int iTry = (iMin+iMax)/2; - void *pKeyT; int nKeyT; /* Key for cell iTry */ - int iTopicT; /* Topic for key pKeyT/nKeyT */ - LsmPgno iPtr; /* Pointer associated with cell iTry */ - int res; /* (pKey - pKeyT) */ - - rc = pageGetBtreeKey( - pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob - ); - if( rc!=LSM_OK ) break; - if( piFirst && pKeyT==blob.pData ){ - *piFirst = pageGetBtreeRef(pPg, iTry); - piFirst = 0; - i++; - } - - res = sortedKeyCompare( - pCsr->pDb->xCmp, iTopic, pKey, nKey, iTopicT, pKeyT, nKeyT - ); - if( res<0 ){ - iPg = iPtr; - iMax = iTry-1; - }else{ - iMin = iTry+1; - } - } - lsmFsPageRelease(pPg); - pPg = 0; - } - }while( rc==LSM_OK ); - - sortedBlobFree(&blob); - assert( (rc==LSM_OK)==(pPg!=0) ); - if( ppPg ){ - *ppPg = pPg; - }else{ - lsmFsPageRelease(pPg); - } - return rc; -} - -static int seekInSegment( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int iTopic, - void *pKey, int nKey, - LsmPgno iPg, /* Page to search */ - int eSeek, /* Search bias - see above */ - LsmPgno *piPtr, /* OUT: FC pointer */ - int *pbStop /* OUT: Stop search flag */ -){ - LsmPgno iPtr = iPg; - int rc = LSM_OK; - - if( pPtr->pSeg->iRoot ){ - Page *pPg; - assert( pPtr->pSeg->iRoot!=0 ); - rc = seekInBtree(pCsr, pPtr->pSeg, iTopic, pKey, nKey, 0, &pPg); - if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg); - }else{ - if( iPtr==0 ){ - iPtr = pPtr->pSeg->iFirst; - } - if( rc==LSM_OK ){ - rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr); - } - } - - if( rc==LSM_OK ){ - rc = segmentPtrSeek(pCsr, pPtr, iTopic, pKey, nKey, eSeek, piPtr, pbStop); - } - return rc; -} - -/* -** Seek each segment pointer in the array of (pLvl->nRight+1) at aPtr[]. -** -** pbStop: -** This parameter is only significant if parameter eSeek is set to -** LSM_SEEK_EQ. In this case, it is set to true before returning if -** the seek operation is finished. This can happen in two ways: -** -** a) A key matching (pKey/nKey) is found, or -** b) A point-delete or range-delete deleting the key is found. -** -** In case (a), the multi-cursor CURSOR_SEEK_EQ flag is set and the pCsr->key -** and pCsr->val blobs populated before returning. -*/ -static int seekInLevel( - MultiCursor *pCsr, /* Sorted cursor object to seek */ - SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ - int eSeek, /* Search bias - see above */ - int iTopic, /* Key topic to search for */ - void *pKey, int nKey, /* Key to search for */ - LsmPgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ - int *pbStop /* OUT: See above */ -){ - Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ - int rc = LSM_OK; /* Return code */ - LsmPgno iOut = 0; /* Pointer to return to caller */ - int res = -1; /* Result of xCmp(pKey, split) */ - int nRhs = pLvl->nRight; /* Number of right-hand-side segments */ - int bStop = 0; - - /* If this is a composite level (one currently undergoing an incremental - ** merge), figure out if the search key is larger or smaller than the - ** levels split-key. */ - if( nRhs ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - } - - /* If (res<0), then key pKey/nKey is smaller than the split-key (or this - ** is not a composite level and there is no split-key). Search the - ** left-hand-side of the level in this case. */ - if( res<0 ){ - int i; - LsmPgno iPtr = 0; - if( nRhs==0 ) iPtr = *piPgno; - - rc = seekInSegment( - pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); - if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){ - res = 0; - } - for(i=1; i<=nRhs; i++){ - segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - - if( res>=0 ){ - int bHit = 0; /* True if at least one rhs is not EOF */ - LsmPgno iPtr = *piPgno; - int i; - segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD); - for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){ - SegmentPtr *pPtr = &aPtr[i]; - iOut = 0; - rc = seekInSegment( - pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); - iPtr = iOut; - - /* If the segment-pointer has settled on a key that is smaller than - ** the splitkey, invalidate the segment-pointer. */ - if( pPtr->pPg ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ){ - if( pPtr->eType & LSM_START_DELETE ){ - pPtr->eType &= ~LSM_INSERT; - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - pPtr->pVal = 0; - pPtr->nVal = 0; - }else{ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - } - - if( aPtr[i].pKey ) bHit = 1; - } - - if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ - rc = segmentPtrEnd(pCsr, &aPtr[0], 1); - } - } - - assert( eSeek==LSM_SEEK_EQ || bStop==0 ); - *piPgno = iOut; - *pbStop = bStop; - return rc; -} - -static void multiCursorGetKey( - MultiCursor *pCsr, - int iKey, - int *peType, /* OUT: Key type (SORTED_WRITE etc.) */ - void **ppKey, /* OUT: Pointer to buffer containing key */ - int *pnKey /* OUT: Size of *ppKey in bytes */ -){ - int nKey = 0; - void *pKey = 0; - int eType = 0; - - switch( iKey ){ - case CURSOR_DATA_TREE0: - case CURSOR_DATA_TREE1: { - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - if( lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorKey(pTreeCsr, &eType, &pKey, &nKey); - } - break; - } - - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker && (pCsr->flags & CURSOR_FLUSH_FREELIST) ){ - int nEntry = pWorker->freelist.nEntry; - if( pCsr->iFree < (nEntry*2) ){ - FreelistEntry *aEntry = pWorker->freelist.aEntry; - int i = nEntry - 1 - (pCsr->iFree / 2); - u32 iKey2 = 0; - - if( (pCsr->iFree % 2) ){ - eType = LSM_END_DELETE|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk-1; - }else if( aEntry[i].iId>=0 ){ - eType = LSM_INSERT|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk; - - /* If the in-memory entry immediately before this one was a - ** DELETE, and the block number is one greater than the current - ** block number, mark this entry as an "end-delete-range". */ - if( i<(nEntry-1) && aEntry[i+1].iBlk==iKey2+1 && aEntry[i+1].iId<0 ){ - eType |= LSM_END_DELETE; - } - - }else{ - eType = LSM_START_DELETE|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk + 1; - } - - /* If the in-memory entry immediately after this one is a - ** DELETE, and the block number is one less than the current - ** key, mark this entry as an "start-delete-range". */ - if( i>0 && aEntry[i-1].iBlk==iKey2-1 && aEntry[i-1].iId<0 ){ - eType |= LSM_START_DELETE; - } - - pKey = pCsr->pSystemVal; - nKey = 4; - lsmPutU32(pKey, ~iKey2); - } - } - break; - } - - default: { - int iPtr = iKey - CURSOR_DATA_SEGMENT; - assert( iPtr>=0 ); - if( iPtr==pCsr->nPtr ){ - if( pCsr->pBtCsr ){ - pKey = pCsr->pBtCsr->pKey; - nKey = pCsr->pBtCsr->nKey; - eType = pCsr->pBtCsr->eType; - } - }else if( iPtrnPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - if( pPtr->pPg ){ - pKey = pPtr->pKey; - nKey = pPtr->nKey; - eType = pPtr->eType; - } - } - break; - } - } - - if( peType ) *peType = eType; - if( pnKey ) *pnKey = nKey; - if( ppKey ) *ppKey = pKey; -} - -static int sortedDbKeyCompare( - MultiCursor *pCsr, - int iLhsFlags, void *pLhsKey, int nLhsKey, - int iRhsFlags, void *pRhsKey, int nRhsKey -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int res; - - /* Compare the keys, including the system flag. */ - res = sortedKeyCompare(xCmp, - rtTopic(iLhsFlags), pLhsKey, nLhsKey, - rtTopic(iRhsFlags), pRhsKey, nRhsKey - ); - - /* If a key has the LSM_START_DELETE flag set, but not the LSM_INSERT or - ** LSM_POINT_DELETE flags, it is considered a delta larger. This prevents - ** the beginning of an open-ended set from masking a database entry or - ** delete at a lower level. */ - if( res==0 && (pCsr->flags & CURSOR_IGNORE_DELETE) ){ - const int m = LSM_POINT_DELETE|LSM_INSERT|LSM_END_DELETE |LSM_START_DELETE; - int iDel1 = 0; - int iDel2 = 0; - - if( LSM_START_DELETE==(iLhsFlags & m) ) iDel1 = +1; - if( LSM_END_DELETE ==(iLhsFlags & m) ) iDel1 = -1; - if( LSM_START_DELETE==(iRhsFlags & m) ) iDel2 = +1; - if( LSM_END_DELETE ==(iRhsFlags & m) ) iDel2 = -1; - - res = (iDel1 - iDel2); - } - - return res; -} - -static void multiCursorDoCompare(MultiCursor *pCsr, int iOut, int bReverse){ - int i1; - int i2; - int iRes; - void *pKey1; int nKey1; int eType1; - void *pKey2; int nKey2; int eType2; - const int mul = (bReverse ? -1 : 1); - - assert( pCsr->aTree && iOutnTree ); - if( iOut>=(pCsr->nTree/2) ){ - i1 = (iOut - pCsr->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pCsr->aTree[iOut*2]; - i2 = pCsr->aTree[iOut*2+1]; - } - - multiCursorGetKey(pCsr, i1, &eType1, &pKey1, &nKey1); - multiCursorGetKey(pCsr, i2, &eType2, &pKey2, &nKey2); - - if( pKey1==0 ){ - iRes = i2; - }else if( pKey2==0 ){ - iRes = i1; - }else{ - int res; - - /* Compare the keys */ - res = sortedDbKeyCompare(pCsr, - eType1, pKey1, nKey1, eType2, pKey2, nKey2 - ); - - res = res * mul; - if( res==0 ){ - /* The two keys are identical. Normally, this means that the key from - ** the newer run clobbers the old. However, if the newer key is a - ** separator key, or a range-delete-boundary only, do not allow it - ** to clobber an older entry. */ - int nc1 = (eType1 & (LSM_INSERT|LSM_POINT_DELETE))==0; - int nc2 = (eType2 & (LSM_INSERT|LSM_POINT_DELETE))==0; - iRes = (nc1 > nc2) ? i2 : i1; - }else if( res<0 ){ - iRes = i1; - }else{ - iRes = i2; - } - } - - pCsr->aTree[iOut] = iRes; -} - -/* -** This function advances segment pointer iPtr belonging to multi-cursor -** pCsr forward (bReverse==0) or backward (bReverse!=0). -** -** If the segment pointer points to a segment that is part of a composite -** level, then the following special case is handled. -** -** * If iPtr is the lhs of a composite level, and the cursor is being -** advanced forwards, and segment iPtr is at EOF, move all pointers -** that correspond to rhs segments of the same level to the first -** key in their respective data. -*/ -static int segmentCursorAdvance( - MultiCursor *pCsr, - int iPtr, - int bReverse -){ - int rc; - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - Level *pLvl = pPtr->pLevel; - int bComposite; /* True if pPtr is part of composite level */ - - /* Advance the segment-pointer object. */ - rc = segmentPtrAdvance(pCsr, pPtr, bReverse); - if( rc!=LSM_OK ) return rc; - - bComposite = (pLvl->nRight>0 && pCsr->nPtr>pLvl->nRight); - if( bComposite && pPtr->pPg==0 ){ - int bFix = 0; - if( (bReverse==0)==(pPtr->pSeg==&pLvl->lhs) ){ - int i; - if( bReverse ){ - SegmentPtr *pLhs = &pCsr->aPtr[iPtr - 1 - (pPtr->pSeg - pLvl->aRhs)]; - for(i=0; inRight; i++){ - if( pLhs[i+1].pPg ) break; - } - if( i==pLvl->nRight ){ - bFix = 1; - rc = segmentPtrEnd(pCsr, pLhs, 1); - } - }else{ - bFix = 1; - for(i=0; rc==LSM_OK && inRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - } - } - - if( bFix ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bReverse); - } - } - } - -#if 0 - if( bComposite && pPtr->pSeg==&pLvl->lhs /* lhs of composite level */ - && bReverse==0 /* csr advanced forwards */ - && pPtr->pPg==0 /* segment at EOF */ - ){ - int i; - for(i=0; rc==LSM_OK && inRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, 0); - } - } -#endif - - return rc; -} - -static void mcursorFreeComponents(MultiCursor *pCsr){ - int i; - lsm_env *pEnv = pCsr->pDb->pEnv; - - /* Close the tree cursor, if any. */ - lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); - lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); - - /* Reset the segment pointers */ - for(i=0; inPtr; i++){ - segmentPtrReset(&pCsr->aPtr[i], 0); - } - - /* And the b-tree cursor, if any */ - btreeCursorFree(pCsr->pBtCsr); - - /* Free allocations */ - lsmFree(pEnv, pCsr->aPtr); - lsmFree(pEnv, pCsr->aTree); - lsmFree(pEnv, pCsr->pSystemVal); - - /* Zero fields */ - pCsr->nPtr = 0; - pCsr->aPtr = 0; - pCsr->nTree = 0; - pCsr->aTree = 0; - pCsr->pSystemVal = 0; - pCsr->apTreeCsr[0] = 0; - pCsr->apTreeCsr[1] = 0; - pCsr->pBtCsr = 0; -} - -void lsmMCursorFreeCache(lsm_db *pDb){ - MultiCursor *p; - MultiCursor *pNext; - for(p=pDb->pCsrCache; p; p=pNext){ - pNext = p->pNext; - lsmMCursorClose(p, 0); - } - pDb->pCsrCache = 0; -} - -/* -** Close the cursor passed as the first argument. -** -** If the bCache parameter is true, then shift the cursor to the pCsrCache -** list for possible reuse instead of actually deleting it. -*/ -void lsmMCursorClose(MultiCursor *pCsr, int bCache){ - if( pCsr ){ - lsm_db *pDb = pCsr->pDb; - MultiCursor **pp; /* Iterator variable */ - - /* The cursor may or may not be currently part of the linked list - ** starting at lsm_db.pCsr. If it is, extract it. */ - for(pp=&pDb->pCsr; *pp; pp=&((*pp)->pNext)){ - if( *pp==pCsr ){ - *pp = pCsr->pNext; - break; - } - } - - if( bCache ){ - int i; /* Used to iterate through segment-pointers */ - - /* Release any page references held by this cursor. */ - assert( !pCsr->pBtCsr ); - for(i=0; inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - lsmFsPageRelease(pPtr->pPg); - pPtr->pPg = 0; - } - - /* Reset the tree cursors */ - lsmTreeCursorReset(pCsr->apTreeCsr[0]); - lsmTreeCursorReset(pCsr->apTreeCsr[1]); - - /* Add the cursor to the pCsrCache list */ - pCsr->pNext = pDb->pCsrCache; - pDb->pCsrCache = pCsr; - }else{ - /* Free the allocation used to cache the current key, if any. */ - sortedBlobFree(&pCsr->key); - sortedBlobFree(&pCsr->val); - - /* Free the component cursors */ - mcursorFreeComponents(pCsr); - - /* Free the cursor structure itself */ - lsmFree(pDb->pEnv, pCsr); - } - } -} - -#define TREE_NONE 0 -#define TREE_OLD 1 -#define TREE_BOTH 2 - -/* -** Parameter eTree is one of TREE_OLD or TREE_BOTH. -*/ -static int multiCursorAddTree(MultiCursor *pCsr, Snapshot *pSnap, int eTree){ - int rc = LSM_OK; - lsm_db *db = pCsr->pDb; - - /* Add a tree cursor on the 'old' tree, if it exists. */ - if( eTree!=TREE_NONE - && lsmTreeHasOld(db) - && db->treehdr.iOldLog!=pSnap->iLogOff - ){ - rc = lsmTreeCursorNew(db, 1, &pCsr->apTreeCsr[1]); - } - - /* Add a tree cursor on the 'current' tree, if required. */ - if( rc==LSM_OK && eTree==TREE_BOTH ){ - rc = lsmTreeCursorNew(db, 0, &pCsr->apTreeCsr[0]); - } - - return rc; -} - -static int multiCursorAddRhs(MultiCursor *pCsr, Level *pLvl){ - int i; - int nRhs = pLvl->nRight; - - assert( pLvl->nRight>0 ); - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZero(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nRhs); - if( !pCsr->aPtr ) return LSM_NOMEM_BKPT; - pCsr->nPtr = nRhs; - - for(i=0; iaPtr[i].pSeg = &pLvl->aRhs[i]; - pCsr->aPtr[i].pLevel = pLvl; - } - - return LSM_OK; -} - -static void multiCursorAddOne(MultiCursor *pCsr, Level *pLvl, int *pRc){ - if( *pRc==LSM_OK ){ - int iPtr = pCsr->nPtr; - int i; - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; - iPtr++; - for(i=0; inRight; i++){ - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; - iPtr++; - } - - if( pLvl->nRight && pLvl->pSplitKey==0 ){ - sortedSplitkey(pCsr->pDb, pLvl, pRc); - } - pCsr->nPtr = iPtr; - } -} - -static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ - Level *pLvl; - int nPtr = 0; - int rc = LSM_OK; - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - /* If the LEVEL_INCOMPLETE flag is set, then this function is being - ** called (indirectly) from within a sortedNewToplevel() call to - ** construct pLvl. In this case ignore pLvl - this cursor is going to - ** be used to retrieve a freelist entry from the LSM, and the partially - ** complete level may confuse it. */ - if( pLvl->flags & LEVEL_INCOMPLETE ) continue; - nPtr += (1 + pLvl->nRight); - } - - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - if( (pLvl->flags & LEVEL_INCOMPLETE)==0 ){ - multiCursorAddOne(pCsr, pLvl, &rc); - } - } - - return rc; -} - -static int multiCursorInit(MultiCursor *pCsr, Snapshot *pSnap){ - int rc; - rc = multiCursorAddAll(pCsr, pSnap); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pSnap, TREE_BOTH); - } - pCsr->flags |= (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); - return rc; -} - -static MultiCursor *multiCursorNew(lsm_db *db, int *pRc){ - MultiCursor *pCsr; - pCsr = (MultiCursor *)lsmMallocZeroRc(db->pEnv, sizeof(MultiCursor), pRc); - if( pCsr ){ - pCsr->pNext = db->pCsr; - db->pCsr = pCsr; - pCsr->pDb = db; - } - return pCsr; -} - - -void lsmSortedRemap(lsm_db *pDb){ - MultiCursor *pCsr; - for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ - int iPtr; - if( pCsr->pBtCsr ){ - btreeCursorLoadKey(pCsr->pBtCsr); - } - for(iPtr=0; iPtrnPtr; iPtr++){ - segmentPtrLoadCell(&pCsr->aPtr[iPtr], pCsr->aPtr[iPtr].iCell); - } - } -} - -static void multiCursorReadSeparators(MultiCursor *pCsr){ - if( pCsr->nPtr>0 ){ - pCsr->flags |= CURSOR_READ_SEPARATORS; - } -} - -/* -** Have this cursor skip over SORTED_DELETE entries. -*/ -static void multiCursorIgnoreDelete(MultiCursor *pCsr){ - if( pCsr ) pCsr->flags |= CURSOR_IGNORE_DELETE; -} - -/* -** If the free-block list is not empty, then have this cursor visit a key -** with (a) the system bit set, and (b) the key "FREELIST" and (c) a value -** blob containing the serialized free-block list. -*/ -static int multiCursorVisitFreelist(MultiCursor *pCsr){ - int rc = LSM_OK; - pCsr->flags |= CURSOR_FLUSH_FREELIST; - pCsr->pSystemVal = lsmMallocRc(pCsr->pDb->pEnv, 4 + 8, &rc); - return rc; -} - -/* -** Allocate and return a new database cursor. -** -** This method should only be called to allocate user cursors. As it may -** recycle a cursor from lsm_db.pCsrCache. -*/ -int lsmMCursorNew( - lsm_db *pDb, /* Database handle */ - MultiCursor **ppCsr /* OUT: Allocated cursor */ -){ - MultiCursor *pCsr = 0; - int rc = LSM_OK; - - if( pDb->pCsrCache ){ - int bOld; /* True if there is an old in-memory tree */ - - /* Remove a cursor from the pCsrCache list and add it to the open list. */ - pCsr = pDb->pCsrCache; - pDb->pCsrCache = pCsr->pNext; - pCsr->pNext = pDb->pCsr; - pDb->pCsr = pCsr; - - /* The cursor can almost be used as is, except that the old in-memory - ** tree cursor may be present and not required, or required and not - ** present. Fix this if required. */ - bOld = (lsmTreeHasOld(pDb) && pDb->treehdr.iOldLog!=pDb->pClient->iLogOff); - if( !bOld && pCsr->apTreeCsr[1] ){ - lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); - pCsr->apTreeCsr[1] = 0; - }else if( bOld && !pCsr->apTreeCsr[1] ){ - rc = lsmTreeCursorNew(pDb, 1, &pCsr->apTreeCsr[1]); - } - - pCsr->flags = (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); - - }else{ - pCsr = multiCursorNew(pDb, &rc); - if( rc==LSM_OK ) rc = multiCursorInit(pCsr, pDb->pClient); - } - - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - pCsr = 0; - } - assert( (rc==LSM_OK)==(pCsr!=0) ); - *ppCsr = pCsr; - return rc; -} - -static int multiCursorGetVal( - MultiCursor *pCsr, - int iVal, - void **ppVal, - int *pnVal -){ - int rc = LSM_OK; - - *ppVal = 0; - *pnVal = 0; - - switch( iVal ){ - case CURSOR_DATA_TREE0: - case CURSOR_DATA_TREE1: { - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iVal-CURSOR_DATA_TREE0]; - if( lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorValue(pTreeCsr, ppVal, pnVal); - }else{ - *ppVal = 0; - *pnVal = 0; - } - break; - } - - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker - && (pCsr->iFree % 2)==0 - && pCsr->iFree < (pWorker->freelist.nEntry*2) - ){ - int iEntry = pWorker->freelist.nEntry - 1 - (pCsr->iFree / 2); - u8 *aVal = &((u8 *)(pCsr->pSystemVal))[4]; - lsmPutU64(aVal, pWorker->freelist.aEntry[iEntry].iId); - *ppVal = aVal; - *pnVal = 8; - } - break; - } - - default: { - int iPtr = iVal-CURSOR_DATA_SEGMENT; - if( iPtrnPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - if( pPtr->pPg ){ - *ppVal = pPtr->pVal; - *pnVal = pPtr->nVal; - } - } - } - } - - assert( rc==LSM_OK || (*ppVal==0 && *pnVal==0) ); - return rc; -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse); - -/* -** This function is called by worker connections to walk the part of the -** free-list stored within the LSM data structure. -*/ -int lsmSortedWalkFreelist( - lsm_db *pDb, /* Database handle */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - MultiCursor *pCsr; /* Cursor used to read db */ - int rc = LSM_OK; /* Return Code */ - Snapshot *pSnap = 0; - - assert( pDb->pWorker ); - if( pDb->bIncrMerge ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); - if( rc!=LSM_OK ) return rc; - }else{ - pSnap = pDb->pWorker; - } - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pSnap); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - if( bReverse==0 ){ - rc = lsmMCursorLast(pCsr); - }else{ - rc = lsmMCursorSeek(pCsr, 1, "", 0, LSM_SEEK_GE); - } - - while( rc==LSM_OK && lsmMCursorValid(pCsr) && rtIsSystem(pCsr->eType) ){ - void *pKey; int nKey; - void *pVal = 0; int nVal = 0; - - rc = lsmMCursorKey(pCsr, &pKey, &nKey); - if( rc==LSM_OK ) rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK && (nKey!=4 || nVal!=8) ) rc = LSM_CORRUPT_BKPT; - - if( rc==LSM_OK ){ - int iBlk; - i64 iSnap; - iBlk = (int)(~(lsmGetU32((u8 *)pKey))); - iSnap = (i64)lsmGetU64((u8 *)pVal); - if( x(pCtx, iBlk, iSnap) ) break; - rc = multiCursorAdvance(pCsr, !bReverse); - } - } - } - - lsmMCursorClose(pCsr, 0); - if( pSnap!=pDb->pWorker ){ - lsmFreeSnapshot(pDb->pEnv, pSnap); - } - - return rc; -} - -int lsmSortedLoadFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - void **ppVal, /* OUT: Blob containing LSM free-list */ - int *pnVal /* OUT: Size of *ppVal blob in bytes */ -){ - MultiCursor *pCsr; /* Cursor used to retreive free-list */ - int rc = LSM_OK; /* Return Code */ - - assert( pDb->pWorker ); - assert( *ppVal==0 && *pnVal==0 ); - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pDb->pWorker); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - rc = lsmMCursorLast(pCsr); - if( rc==LSM_OK - && rtIsWrite(pCsr->eType) && rtIsSystem(pCsr->eType) - && pCsr->key.nData==8 - && 0==memcmp(pCsr->key.pData, "FREELIST", 8) - ){ - void *pVal; int nVal; /* Value read from database */ - rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK ){ - *ppVal = lsmMallocRc(pDb->pEnv, nVal, &rc); - if( *ppVal ){ - memcpy(*ppVal, pVal, nVal); - *pnVal = nVal; - } - } - } - - lsmMCursorClose(pCsr, 0); - } - - return rc; -} - -static int multiCursorAllocTree(MultiCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->aTree==0 ){ - int nByte; /* Bytes of space to allocate */ - int nMin; /* Total number of cursors being merged */ - - nMin = CURSOR_DATA_SEGMENT + pCsr->nPtr + (pCsr->pBtCsr!=0); - pCsr->nTree = 2; - while( pCsr->nTreenTree = pCsr->nTree*2; - } - - nByte = sizeof(int)*pCsr->nTree*2; - pCsr->aTree = (int *)lsmMallocZeroRc(pCsr->pDb->pEnv, nByte, &rc); - } - return rc; -} - -static void multiCursorCacheKey(MultiCursor *pCsr, int *pRc){ - if( *pRc==LSM_OK ){ - void *pKey; - int nKey; - multiCursorGetKey(pCsr, pCsr->aTree[1], &pCsr->eType, &pKey, &nKey); - *pRc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->key, pKey, nKey); - } -} - -#ifdef LSM_DEBUG_EXPENSIVE -static void assertCursorTree(MultiCursor *pCsr){ - int bRev = !!(pCsr->flags & CURSOR_PREV_OK); - int *aSave = pCsr->aTree; - int nSave = pCsr->nTree; - int rc; - - pCsr->aTree = 0; - pCsr->nTree = 0; - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - - assert( nSave==pCsr->nTree - && 0==memcmp(aSave, pCsr->aTree, sizeof(int)*nSave) - ); - - lsmFree(pCsr->pDb->pEnv, pCsr->aTree); - } - - pCsr->aTree = aSave; - pCsr->nTree = nSave; -} -#else -# define assertCursorTree(x) -#endif - -static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){ - int eType = pCsr->eType; - int iKey; - int i; - int rdmask; - - assert( pCsr->flags & (CURSOR_NEXT_OK|CURSOR_PREV_OK) ); - assertCursorTree(pCsr); - - rdmask = (pCsr->flags & CURSOR_NEXT_OK) ? LSM_END_DELETE : LSM_START_DELETE; - - /* If the cursor does not currently point to an actual database key (i.e. - ** it points to a delete key, or the start or end of a range-delete), and - ** the CURSOR_IGNORE_DELETE flag is set, skip past this entry. */ - if( (pCsr->flags & CURSOR_IGNORE_DELETE) && bDeleteOk==0 ){ - if( (eType & LSM_INSERT)==0 ) return 0; - } - - /* If the cursor points to a system key (free-list entry), and the - ** CURSOR_IGNORE_SYSTEM flag is set, skip thie entry. */ - if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){ - return 0; - } - -#ifndef NDEBUG - /* This block fires assert() statements to check one of the assumptions - ** in the comment below - that if the lhs sub-cursor of a level undergoing - ** a merge is valid, then all the rhs sub-cursors must be at EOF. - ** - ** Also assert that all rhs sub-cursors are either at EOF or point to - ** a key that is not less than the level split-key. */ - for(i=0; inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - if( pLvl->nRight && pPtr->pPg ){ - if( pPtr->pSeg==&pLvl->lhs ){ - int j; - for(j=0; jnRight; j++) assert( pPtr[j+1].pPg==0 ); - }else{ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - assert( res>=0 ); - } - } - } -#endif - - /* Now check if this key has already been deleted by a range-delete. If - ** so, skip past it. - ** - ** Assume, for the moment, that the tree contains no levels currently - ** undergoing incremental merge, and that this cursor is iterating forwards - ** through the database keys. The cursor currently points to a key in - ** level L. This key has already been deleted if any of the sub-cursors - ** that point to levels newer than L (or to the in-memory tree) point to - ** a key greater than the current key with the LSM_END_DELETE flag set. - ** - ** Or, if the cursor is iterating backwards through data keys, if any - ** such sub-cursor points to a key smaller than the current key with the - ** LSM_START_DELETE flag set. - ** - ** Why it works with levels undergoing a merge too: - ** - ** When a cursor iterates forwards, the sub-cursors for the rhs of a - ** level are only activated once the lhs reaches EOF. So when iterating - ** forwards, the keys visited are the same as if the level was completely - ** merged. - ** - ** If the cursor is iterating backwards, then the lhs sub-cursor is not - ** initialized until the last of the rhs sub-cursors has reached EOF. - ** Additionally, if the START_DELETE flag is set on the last entry (in - ** reverse order - so the entry with the smallest key) of a rhs sub-cursor, - ** then a pseudo-key equal to the levels split-key with the END_DELETE - ** flag set is visited by the sub-cursor. - */ - iKey = pCsr->aTree[1]; - for(i=0; iflags & CURSOR_IGNORE_DELETE)==0 - ){ - void *pKey; int nKey; - multiCursorGetKey(pCsr, i, 0, &pKey, &nKey); - if( 0==sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(csrflags), pKey, nKey - )){ - continue; - } - } - return 0; - } - } - - /* The current cursor position is one this cursor should visit. Return 1. */ - return 1; -} - -static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ - int rc; - - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - } - - assertCursorTree(pCsr); - multiCursorCacheKey(pCsr, &rc); - - if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ - rc = multiCursorAdvance(pCsr, bRev); - } - return rc; -} - - -static int multiCursorEnd(MultiCursor *pCsr, int bLast){ - int rc = LSM_OK; - int i; - - pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); - pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK); - pCsr->iFree = 0; - - /* Position the two in-memory tree cursors */ - for(i=0; rc==LSM_OK && i<2; i++){ - if( pCsr->apTreeCsr[i] ){ - rc = lsmTreeCursorEnd(pCsr->apTreeCsr[i], bLast); - } - } - - for(i=0; rc==LSM_OK && inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - int iRhs; - int bHit = 0; - - if( bLast ){ - for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ - rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); - if( pPtr[iRhs+1].pPg ) bHit = 1; - } - if( bHit==0 && rc==LSM_OK ){ - rc = segmentPtrEnd(pCsr, pPtr, 1); - }else{ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - }else{ - int bLhs = (pPtr->pSeg==&pLvl->lhs); - assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); - - if( bLhs ){ - rc = segmentPtrEnd(pCsr, pPtr, 0); - if( pPtr->pKey ) bHit = 1; - } - for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ - if( bHit ){ - segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD); - }else{ - rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); - } - } - } - i += pLvl->nRight; - } - - /* And the b-tree cursor, if applicable */ - if( rc==LSM_OK && pCsr->pBtCsr ){ - assert( bLast==0 ); - rc = btreeCursorFirst(pCsr->pBtCsr); - } - - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, bLast); - } - - return rc; -} - - -int mcursorSave(MultiCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->aTree ){ - int iTree = pCsr->aTree[1]; - if( iTree==CURSOR_DATA_TREE0 || iTree==CURSOR_DATA_TREE1 ){ - multiCursorCacheKey(pCsr, &rc); - } - } - mcursorFreeComponents(pCsr); - return rc; -} - -int mcursorRestore(lsm_db *pDb, MultiCursor *pCsr){ - int rc; - rc = multiCursorInit(pCsr, pDb->pClient); - if( rc==LSM_OK && pCsr->key.pData ){ - rc = lsmMCursorSeek(pCsr, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, +1 - ); - } - return rc; -} - -int lsmSaveCursors(lsm_db *pDb){ - int rc = LSM_OK; - MultiCursor *pCsr; - - for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ - rc = mcursorSave(pCsr); - } - return rc; -} - -int lsmRestoreCursors(lsm_db *pDb){ - int rc = LSM_OK; - MultiCursor *pCsr; - - for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ - rc = mcursorRestore(pDb, pCsr); - } - return rc; -} - -int lsmMCursorFirst(MultiCursor *pCsr){ - return multiCursorEnd(pCsr, 0); -} - -int lsmMCursorLast(MultiCursor *pCsr){ - return multiCursorEnd(pCsr, 1); -} - -lsm_db *lsmMCursorDb(MultiCursor *pCsr){ - return pCsr->pDb; -} - -void lsmMCursorReset(MultiCursor *pCsr){ - int i; - lsmTreeCursorReset(pCsr->apTreeCsr[0]); - lsmTreeCursorReset(pCsr->apTreeCsr[1]); - for(i=0; inPtr; i++){ - segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); - } - pCsr->key.nData = 0; -} - -static int treeCursorSeek( - MultiCursor *pCsr, - TreeCursor *pTreeCsr, - void *pKey, int nKey, - int eSeek, - int *pbStop -){ - int rc = LSM_OK; - if( pTreeCsr ){ - int res = 0; - lsmTreeCursorSeek(pTreeCsr, pKey, nKey, &res); - switch( eSeek ){ - case LSM_SEEK_EQ: { - int eType = lsmTreeCursorFlags(pTreeCsr); - if( (res<0 && (eType & LSM_START_DELETE)) - || (res>0 && (eType & LSM_END_DELETE)) - || (res==0 && (eType & LSM_POINT_DELETE)) - ){ - *pbStop = 1; - }else if( res==0 && (eType & LSM_INSERT) ){ - lsm_env *pEnv = pCsr->pDb->pEnv; - void *p; int n; /* Key/value from tree-cursor */ - *pbStop = 1; - pCsr->flags |= CURSOR_SEEK_EQ; - rc = lsmTreeCursorKey(pTreeCsr, &pCsr->eType, &p, &n); - if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->key, p, n); - if( rc==LSM_OK ) rc = lsmTreeCursorValue(pTreeCsr, &p, &n); - if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->val, p, n); - } - lsmTreeCursorReset(pTreeCsr); - break; - } - case LSM_SEEK_GE: - if( res<0 && lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorNext(pTreeCsr); - } - break; - default: - if( res>0 ){ - assert( lsmTreeCursorValid(pTreeCsr) ); - lsmTreeCursorPrev(pTreeCsr); - } - break; - } - } - return rc; -} - - -/* -** Seek the cursor. -*/ -int lsmMCursorSeek( - MultiCursor *pCsr, - int iTopic, - void *pKey, int nKey, - int eSeek -){ - int eESeek = eSeek; /* Effective eSeek parameter */ - int bStop = 0; /* Set to true to halt search operation */ - int rc = LSM_OK; /* Return code */ - int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ - LsmPgno iPgno = 0; /* FC pointer value */ - - assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); - assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); - - if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; - - assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE ); - assert( (pCsr->flags & CURSOR_FLUSH_FREELIST)==0 ); - assert( pCsr->nPtr==0 || pCsr->aPtr[0].pLevel ); - - pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); - rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[0], pKey, nKey, eESeek, &bStop); - if( rc==LSM_OK && bStop==0 ){ - rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[1], pKey, nKey, eESeek, &bStop); - } - - /* Seek all segment pointers. */ - for(iPtr=0; iPtrnPtr && rc==LSM_OK && bStop==0; iPtr++){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - assert( pPtr->pSeg==&pPtr->pLevel->lhs ); - rc = seekInLevel(pCsr, pPtr, eESeek, iTopic, pKey, nKey, &iPgno, &bStop); - iPtr += pPtr->pLevel->nRight; - } - - if( eSeek!=LSM_SEEK_EQ ){ - if( rc==LSM_OK ){ - rc = multiCursorAllocTree(pCsr); - } - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, eESeek==LSM_SEEK_LE); - } - if( eSeek==LSM_SEEK_GE ) pCsr->flags |= CURSOR_NEXT_OK; - if( eSeek==LSM_SEEK_LE ) pCsr->flags |= CURSOR_PREV_OK; - } - - multiCursorCacheKey(pCsr, &rc); - if( rc==LSM_OK && eSeek!=LSM_SEEK_LEFAST && 0==mcursorLocationOk(pCsr, 0) ){ - switch( eESeek ){ - case LSM_SEEK_EQ: - lsmMCursorReset(pCsr); - break; - case LSM_SEEK_GE: - rc = lsmMCursorNext(pCsr); - break; - default: - rc = lsmMCursorPrev(pCsr); - break; - } - } - } - - return rc; -} - -int lsmMCursorValid(MultiCursor *pCsr){ - int res = 0; - if( pCsr->flags & CURSOR_SEEK_EQ ){ - res = 1; - }else if( pCsr->aTree ){ - int iKey = pCsr->aTree[1]; - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - res = lsmTreeCursorValid(pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]); - }else{ - void *pKey; - multiCursorGetKey(pCsr, iKey, 0, &pKey, 0); - res = pKey!=0; - } - } - return res; -} - -static int mcursorAdvanceOk( - MultiCursor *pCsr, - int bReverse, - int *pRc -){ - void *pNew; /* Pointer to buffer containing new key */ - int nNew; /* Size of buffer pNew in bytes */ - int eNewType; /* Type of new record */ - - if( *pRc ) return 1; - - /* Check the current key value. If it is not greater than (if bReverse==0) - ** or less than (if bReverse!=0) the key currently cached in pCsr->key, - ** then the cursor has not yet been successfully advanced. - */ - multiCursorGetKey(pCsr, pCsr->aTree[1], &eNewType, &pNew, &nNew); - if( pNew ){ - int typemask = (pCsr->flags & CURSOR_IGNORE_DELETE) ? ~(0) : LSM_SYSTEMKEY; - int res = sortedDbKeyCompare(pCsr, - eNewType & typemask, pNew, nNew, - pCsr->eType & typemask, pCsr->key.pData, pCsr->key.nData - ); - - if( (bReverse==0 && res<=0) || (bReverse!=0 && res>=0) ){ - return 0; - } - - multiCursorCacheKey(pCsr, pRc); - assert( pCsr->eType==eNewType ); - - /* If this cursor is configured to skip deleted keys, and the current - ** cursor points to a SORTED_DELETE entry, then the cursor has not been - ** successfully advanced. - ** - ** Similarly, if the cursor is configured to skip system keys and the - ** current cursor points to a system key, it has not yet been advanced. - */ - if( *pRc==LSM_OK && 0==mcursorLocationOk(pCsr, 0) ) return 0; - } - return 1; -} - -static void flCsrAdvance(MultiCursor *pCsr){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - if( pCsr->iFree % 2 ){ - pCsr->iFree++; - }else{ - int nEntry = pCsr->pDb->pWorker->freelist.nEntry; - FreelistEntry *aEntry = pCsr->pDb->pWorker->freelist.aEntry; - - int i = nEntry - 1 - (pCsr->iFree / 2); - - /* If the current entry is a delete and the "end-delete" key will not - ** be attached to the next entry, increment iFree by 1 only. */ - if( aEntry[i].iId<0 ){ - while( 1 ){ - if( i==0 || aEntry[i-1].iBlk!=aEntry[i].iBlk-1 ){ - pCsr->iFree--; - break; - } - if( aEntry[i-1].iId>=0 ) break; - pCsr->iFree += 2; - i--; - } - } - pCsr->iFree += 2; - } -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse){ - int rc = LSM_OK; /* Return Code */ - if( lsmMCursorValid(pCsr) ){ - do { - int iKey = pCsr->aTree[1]; - - assertCursorTree(pCsr); - - /* If this multi-cursor is advancing forwards, and the sub-cursor - ** being advanced is the one that separator keys may be being read - ** from, record the current absolute pointer value. */ - if( pCsr->pPrevMergePtr ){ - if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ - assert( pCsr->pBtCsr ); - *pCsr->pPrevMergePtr = pCsr->pBtCsr->iPtr; - }else if( pCsr->pBtCsr==0 && pCsr->nPtr>0 - && iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr-1) - ){ - SegmentPtr *pPtr = &pCsr->aPtr[iKey-CURSOR_DATA_SEGMENT]; - *pCsr->pPrevMergePtr = pPtr->iPtr+pPtr->iPgPtr; - } - } - - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - if( bReverse ){ - rc = lsmTreeCursorPrev(pTreeCsr); - }else{ - rc = lsmTreeCursorNext(pTreeCsr); - } - }else if( iKey==CURSOR_DATA_SYSTEM ){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - assert( bReverse==0 ); - flCsrAdvance(pCsr); - }else if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ - assert( bReverse==0 && pCsr->pBtCsr ); - rc = btreeCursorNext(pCsr->pBtCsr); - }else{ - rc = segmentCursorAdvance(pCsr, iKey-CURSOR_DATA_SEGMENT, bReverse); - } - if( rc==LSM_OK ){ - int i; - for(i=(iKey+pCsr->nTree)/2; i>0; i=i/2){ - multiCursorDoCompare(pCsr, i, bReverse); - } - assertCursorTree(pCsr); - } - }while( mcursorAdvanceOk(pCsr, bReverse, &rc)==0 ); - } - return rc; -} - -int lsmMCursorNext(MultiCursor *pCsr){ - if( (pCsr->flags & CURSOR_NEXT_OK)==0 ) return LSM_MISUSE_BKPT; - return multiCursorAdvance(pCsr, 0); -} - -int lsmMCursorPrev(MultiCursor *pCsr){ - if( (pCsr->flags & CURSOR_PREV_OK)==0 ) return LSM_MISUSE_BKPT; - return multiCursorAdvance(pCsr, 1); -} - -int lsmMCursorKey(MultiCursor *pCsr, void **ppKey, int *pnKey){ - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ - *pnKey = pCsr->key.nData; - *ppKey = pCsr->key.pData; - }else{ - int iKey = pCsr->aTree[1]; - - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - lsmTreeCursorKey(pTreeCsr, 0, ppKey, pnKey); - }else{ - int nKey; - -#ifndef NDEBUG - void *pKey; - int eType; - multiCursorGetKey(pCsr, iKey, &eType, &pKey, &nKey); - assert( eType==pCsr->eType ); - assert( nKey==pCsr->key.nData ); - assert( memcmp(pKey, pCsr->key.pData, nKey)==0 ); -#endif - - nKey = pCsr->key.nData; - if( nKey==0 ){ - *ppKey = 0; - }else{ - *ppKey = pCsr->key.pData; - } - *pnKey = nKey; - } - } - return LSM_OK; -} - -/* -** Compare the current key that cursor csr points to with pKey/nKey. Set -** *piRes to the result and return LSM_OK. -*/ -int lsm_csr_cmp(lsm_cursor *csr, const void *pKey, int nKey, int *piRes){ - MultiCursor *pCsr = (MultiCursor *)csr; - void *pCsrkey; int nCsrkey; - int rc; - rc = lsmMCursorKey(pCsr, &pCsrkey, &nCsrkey); - if( rc==LSM_OK ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - *piRes = sortedKeyCompare(xCmp, 0, pCsrkey, nCsrkey, 0, (void *)pKey, nKey); - } - return rc; -} - -int lsmMCursorValue(MultiCursor *pCsr, void **ppVal, int *pnVal){ - void *pVal; - int nVal; - int rc; - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ - rc = LSM_OK; - nVal = pCsr->val.nData; - pVal = pCsr->val.pData; - }else{ - - assert( pCsr->aTree ); - assert( mcursorLocationOk(pCsr, (pCsr->flags & CURSOR_IGNORE_DELETE)) ); - - rc = multiCursorGetVal(pCsr, pCsr->aTree[1], &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - rc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } - - if( rc!=LSM_OK ){ - pVal = 0; - nVal = 0; - } - } - *ppVal = pVal; - *pnVal = nVal; - return rc; -} - -int lsmMCursorType(MultiCursor *pCsr, int *peType){ - assert( pCsr->aTree ); - multiCursorGetKey(pCsr, pCsr->aTree[1], peType, 0, 0); - return LSM_OK; -} - -/* -** Buffer aData[], size nData, is assumed to contain a valid b-tree -** hierarchy page image. Return the offset in aData[] of the next free -** byte in the data area (where a new cell may be written if there is -** space). -*/ -static int mergeWorkerPageOffset(u8 *aData, int nData){ - int nRec; - int iOff; - int nKey; - int eType; - i64 nDummy; - - - nRec = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); - iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec-1)]); - eType = aData[iOff++]; - assert( eType==0 - || eType==(LSM_SYSTEMKEY|LSM_SEPARATOR) - || eType==(LSM_SEPARATOR) - ); - - iOff += lsmVarintGet64(&aData[iOff], &nDummy); - iOff += lsmVarintGet32(&aData[iOff], &nKey); - - return iOff + (eType ? nKey : 0); -} - -/* -** Following a checkpoint operation, database pages that are part of the -** checkpointed state of the LSM are deemed read-only. This includes the -** right-most page of the b-tree hierarchy of any separators array under -** construction, and all pages between it and the b-tree root, inclusive. -** This is a problem, as when further pages are appended to the separators -** array, entries must be added to the indicated b-tree hierarchy pages. -** -** This function copies all such b-tree pages to new locations, so that -** they can be modified as required. -** -** The complication is that not all database pages are the same size - due -** to the way the file.c module works some (the first and last in each block) -** are 4 bytes smaller than the others. -*/ -static int mergeWorkerMoveHierarchy( - MergeWorker *pMW, /* Merge worker */ - int bSep /* True for separators run */ -){ - lsm_db *pDb = pMW->pDb; /* Database handle */ - int rc = LSM_OK; /* Return code */ - int i; - Page **apHier = pMW->hier.apHier; - int nHier = pMW->hier.nHier; - - for(i=0; rc==LSM_OK && ipFS, pDb->pWorker, pMW->pLevel, 1, &pNew); - assert( rc==LSM_OK ); - - if( rc==LSM_OK ){ - u8 *a1; int n1; - u8 *a2; int n2; - - a1 = fsPageData(pNew, &n1); - a2 = fsPageData(apHier[i], &n2); - - assert( n1==n2 || n1+4==n2 ); - - if( n1==n2 ){ - memcpy(a1, a2, n2); - }else{ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - - memcpy(a1, a2, iEof2 - 4); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - } - - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - -#if 0 - assert( n1==n2 || n1+4==n2 || n2+4==n1 ); - if( n1>=n2 ){ - /* If n1 (size of the new page) is equal to or greater than n2 (the - ** size of the old page), then copy the data into the new page. If - ** n1==n2, this could be done with a single memcpy(). However, - ** since sometimes n1>n2, the page content and footer must be copied - ** separately. */ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - memcpy(a1, a2, iEof2); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - }else{ - lsmPutU16(&a1[SEGMENT_FLAGS_OFFSET(n1)], SEGMENT_BTREE_FLAG); - lsmPutU16(&a1[SEGMENT_NRECORD_OFFSET(n1)], 0); - lsmPutU64(&a1[SEGMENT_POINTER_OFFSET(n1)], 0); - i = i - 1; - lsmFsPageRelease(pNew); - } -#endif - } - } - -#ifdef LSM_DEBUG - if( rc==LSM_OK ){ - for(i=0; ipLevel->lhs; - p = &pMW->hier; - - if( p->apHier==0 && pSeg->iRoot!=0 ){ - FileSystem *pFS = pMW->pDb->pFS; - lsm_env *pEnv = pMW->pDb->pEnv; - Page **apHier = 0; - int nHier = 0; - LsmPgno iPg = pSeg->iRoot; - - do { - Page *pPg = 0; - u8 *aData; - int nData; - int flags; - - rc = lsmFsDbPageGet(pFS, pSeg, iPg, &pPg); - if( rc!=LSM_OK ) break; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( flags&SEGMENT_BTREE_FLAG ){ - Page **apNew = (Page **)lsmRealloc( - pEnv, apHier, sizeof(Page *)*(nHier+1) - ); - if( apNew==0 ){ - rc = LSM_NOMEM_BKPT; - break; - } - apHier = apNew; - memmove(&apHier[1], &apHier[0], sizeof(Page *) * nHier); - nHier++; - - apHier[0] = pPg; - iPg = pageGetPtr(aData, nData); - }else{ - lsmFsPageRelease(pPg); - break; - } - }while( 1 ); - - if( rc==LSM_OK ){ - u8 *aData; - int nData; - aData = fsPageData(apHier[0], &nData); - pMW->aSave[0].iPgno = pageGetPtr(aData, nData); - p->nHier = nHier; - p->apHier = apHier; - rc = mergeWorkerMoveHierarchy(pMW, 0); - }else{ - int i; - for(i=0; ihier; - lsm_db *pDb = pMW->pDb; /* Database handle */ - int rc = LSM_OK; /* Return Code */ - int iLevel; /* Level of b-tree hierachy to write to */ - int nData; /* Size of aData[] in bytes */ - u8 *aData; /* Page data for level iLevel */ - int iOff; /* Offset on b-tree page to write record to */ - int nRec; /* Initial number of records on b-tree page */ - - /* iKeyPg should be zero for an ordinary b-tree key, or non-zero for an - ** indirect key. The flags byte for an indirect key is 0x00. */ - assert( (eType==0)==(iKeyPg!=0) ); - - /* The MergeWorker.apHier[] array contains the right-most leaf of the b-tree - ** hierarchy, the root node, and all nodes that lie on the path between. - ** apHier[0] is the right-most leaf and apHier[pMW->nHier-1] is the current - ** root page. - ** - ** This loop searches for a node with enough space to store the key on, - ** starting with the leaf and iterating up towards the root. When the loop - ** exits, the key may be written to apHier[iLevel]. */ - for(iLevel=0; iLevel<=p->nHier; iLevel++){ - int nByte; /* Number of free bytes required */ - - if( iLevel==p->nHier ){ - /* Extend the array and allocate a new root page. */ - Page **aNew; - aNew = (Page **)lsmRealloc( - pMW->pDb->pEnv, p->apHier, sizeof(Page *)*(p->nHier+1) - ); - if( !aNew ){ - return LSM_NOMEM_BKPT; - } - p->apHier = aNew; - }else{ - Page *pOld; - int nFree; - - /* If the key will fit on this page, break out of the loop here. - ** The new entry will be written to page apHier[iLevel]. */ - pOld = p->apHier[iLevel]; - assert( lsmFsPageWritable(pOld) ); - aData = fsPageData(pOld, &nData); - if( eType==0 ){ - nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen64(iKeyPg); - }else{ - nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen32(nKey) + nKey; - } - - nRec = pageGetNRec(aData, nData); - nFree = SEGMENT_EOF(nData, nRec) - mergeWorkerPageOffset(aData, nData); - if( nByte<=nFree ) break; - - /* Otherwise, this page is full. Set the right-hand-child pointer - ** to iPtr and release it. */ - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - assert( lsmFsPageNumber(pOld)==0 ); - rc = lsmFsPagePersist(pOld); - if( rc==LSM_OK ){ - iPtr = lsmFsPageNumber(pOld); - lsmFsPageRelease(pOld); - } - } - - /* Allocate a new page for apHier[iLevel]. */ - p->apHier[iLevel] = 0; - if( rc==LSM_OK ){ - rc = lsmFsSortedAppend( - pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] - ); - } - if( rc!=LSM_OK ) return rc; - - aData = fsPageData(p->apHier[iLevel], &nData); - memset(aData, 0, nData); - lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], SEGMENT_BTREE_FLAG); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); - - if( iLevel==p->nHier ){ - p->nHier++; - break; - } - } - - /* Write the key into page apHier[iLevel]. */ - aData = fsPageData(p->apHier[iLevel], &nData); - iOff = mergeWorkerPageOffset(aData, nData); - nRec = pageGetNRec(aData, nData); - lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); - if( eType==0 ){ - aData[iOff++] = 0x00; - iOff += lsmVarintPut64(&aData[iOff], iPtr); - iOff += lsmVarintPut64(&aData[iOff], iKeyPg); - }else{ - aData[iOff++] = eType; - iOff += lsmVarintPut64(&aData[iOff], iPtr); - iOff += lsmVarintPut32(&aData[iOff], nKey); - memcpy(&aData[iOff], pKey, nKey); - } - - return rc; -} - -static int mergeWorkerBtreeIndirect(MergeWorker *pMW){ - int rc = LSM_OK; - if( pMW->iIndirect ){ - LsmPgno iKeyPg = pMW->aSave[1].iPgno; - rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0); - pMW->iIndirect = 0; - } - return rc; -} - -/* -** Append the database key (iTopic/pKey/nKey) to the b-tree under -** construction. This key has not yet been written to a segment page. -** The pointer that will accompany the new key in the b-tree - that -** points to the completed segment page that contains keys smaller than -** (pKey/nKey) is currently stored in pMW->aSave[0].iPgno. -*/ -static int mergeWorkerPushHierarchy( - MergeWorker *pMW, /* Merge worker object */ - int iTopic, /* Topic value for this key */ - void *pKey, /* Pointer to key buffer */ - int nKey /* Size of pKey buffer in bytes */ -){ - int rc = LSM_OK; /* Return Code */ - LsmPgno iPtr; /* Pointer value to accompany pKey/nKey */ - - assert( pMW->aSave[0].bStore==0 ); - assert( pMW->aSave[1].bStore==0 ); - rc = mergeWorkerBtreeIndirect(pMW); - - /* Obtain the absolute pointer value to store along with the key in the - ** page body. This pointer points to a page that contains keys that are - ** smaller than pKey/nKey. */ - iPtr = pMW->aSave[0].iPgno; - assert( iPtr!=0 ); - - /* Determine if the indirect format should be used. */ - if( (nKey*4 > lsmFsPageSize(pMW->pDb->pFS)) ){ - pMW->iIndirect = iPtr; - pMW->aSave[1].bStore = 1; - }else{ - rc = mergeWorkerBtreeWrite( - pMW, (u8)(iTopic | LSM_SEPARATOR), iPtr, 0, pKey, nKey - ); - } - - /* Ensure that the SortedRun.iRoot field is correct. */ - return rc; -} - -static int mergeWorkerFinishHierarchy( - MergeWorker *pMW /* Merge worker object */ -){ - int i; /* Used to loop through apHier[] */ - int rc = LSM_OK; /* Return code */ - LsmPgno iPtr; /* New right-hand-child pointer value */ - - iPtr = pMW->aSave[0].iPgno; - for(i=0; ihier.nHier && rc==LSM_OK; i++){ - Page *pPg = pMW->hier.apHier[i]; - int nData; /* Size of aData[] in bytes */ - u8 *aData; /* Page data for pPg */ - - aData = fsPageData(pPg, &nData); - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - - rc = lsmFsPagePersist(pPg); - iPtr = lsmFsPageNumber(pPg); - lsmFsPageRelease(pPg); - } - - if( pMW->hier.nHier ){ - pMW->pLevel->lhs.iRoot = iPtr; - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; - } - - return rc; -} - -static int mergeWorkerAddPadding( - MergeWorker *pMW /* Merge worker object */ -){ - FileSystem *pFS = pMW->pDb->pFS; - return lsmFsSortedPadding(pFS, pMW->pDb->pWorker, &pMW->pLevel->lhs); -} - -/* -** Release all page references currently held by the merge-worker passed -** as the only argument. Unless an error has occurred, all pages have -** already been released. -*/ -static void mergeWorkerReleaseAll(MergeWorker *pMW){ - int i; - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - - for(i=0; ihier.nHier; i++){ - lsmFsPageRelease(pMW->hier.apHier[i]); - pMW->hier.apHier[i] = 0; - } - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; -} - -static int keyszToSkip(FileSystem *pFS, int nKey){ - int nPgsz; /* Nominal database page size */ - nPgsz = lsmFsPageSize(pFS); - return LSM_MIN(((nKey * 4) / nPgsz), 3); -} - -/* -** Release the reference to the current output page of merge-worker *pMW -** (reference pMW->pPage). Set the page number values in aSave[] as -** required (see comments above struct MergeWorker for details). -*/ -static int mergeWorkerPersistAndRelease(MergeWorker *pMW){ - int rc; - int i; - - assert( pMW->pPage || (pMW->aSave[0].bStore==0 && pMW->aSave[1].bStore==0) ); - - /* Persist the page */ - rc = lsmFsPagePersist(pMW->pPage); - - /* If required, save the page number. */ - for(i=0; i<2; i++){ - if( pMW->aSave[i].bStore ){ - pMW->aSave[i].iPgno = lsmFsPageNumber(pMW->pPage); - pMW->aSave[i].bStore = 0; - } - } - - /* Release the completed output page. */ - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - return rc; -} - -/* -** Advance to the next page of an output run being populated by merge-worker -** pMW. The footer of the new page is initialized to indicate that it contains -** zero records. The flags field is cleared. The page footer pointer field -** is set to iFPtr. -** -** If successful, LSM_OK is returned. Otherwise, an error code. -*/ -static int mergeWorkerNextPage( - MergeWorker *pMW, /* Merge worker object to append page to */ - LsmPgno iFPtr /* Pointer value for footer of new page */ -){ - int rc = LSM_OK; /* Return code */ - Page *pNext = 0; /* New page appended to run */ - lsm_db *pDb = pMW->pDb; /* Database handle */ - - rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); - assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); - - if( rc==LSM_OK ){ - u8 *aData; /* Data buffer belonging to page pNext */ - int nData; /* Size of aData[] in bytes */ - - rc = mergeWorkerPersistAndRelease(pMW); - - pMW->pPage = pNext; - pMW->pLevel->pMerge->iOutputOff = 0; - aData = fsPageData(pNext, &nData); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); - lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], 0); - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iFPtr); - pMW->nWork++; - } - - return rc; -} - -/* -** Write a blob of data into an output segment being populated by a -** merge-worker object. If argument bSep is true, write into the separators -** array. Otherwise, the main array. -** -** This function is used to write the blobs of data for keys and values. -*/ -static int mergeWorkerData( - MergeWorker *pMW, /* Merge worker object */ - int bSep, /* True to write to separators run */ - LsmPgno iFPtr, /* Footer ptr for new pages */ - u8 *aWrite, /* Write data from this buffer */ - int nWrite /* Size of aWrite[] in bytes */ -){ - int rc = LSM_OK; /* Return code */ - int nRem = nWrite; /* Number of bytes still to write */ - - while( rc==LSM_OK && nRem>0 ){ - Merge *pMerge = pMW->pLevel->pMerge; - int nCopy; /* Number of bytes to copy */ - u8 *aData; /* Pointer to buffer of current output page */ - int nData; /* Size of aData[] in bytes */ - int nRec; /* Number of records on current output page */ - int iOff; /* Offset in aData[] to write to */ - - assert( lsmFsPageWritable(pMW->pPage) ); - - aData = fsPageData(pMW->pPage, &nData); - nRec = pageGetNRec(aData, nData); - iOff = pMerge->iOutputOff; - nCopy = LSM_MIN(nRem, SEGMENT_EOF(nData, nRec) - iOff); - - memcpy(&aData[iOff], &aWrite[nWrite-nRem], nCopy); - nRem -= nCopy; - - if( nRem>0 ){ - rc = mergeWorkerNextPage(pMW, iFPtr); - }else{ - pMerge->iOutputOff = iOff + nCopy; - } - } - - return rc; -} - - -/* -** The MergeWorker passed as the only argument is working to merge two or -** more existing segments together (not to flush an in-memory tree). It -** has not yet written the first key to the first page of the output. -*/ -static int mergeWorkerFirstPage(MergeWorker *pMW){ - int rc = LSM_OK; /* Return code */ - Page *pPg = 0; /* First page of run pSeg */ - LsmPgno iFPtr = 0; /* Pointer value read from footer of pPg */ - MultiCursor *pCsr = pMW->pCsr; - - assert( pMW->pPage==0 ); - - if( pCsr->pBtCsr ){ - rc = LSM_OK; - iFPtr = pMW->pLevel->pNext->lhs.iFirst; - }else if( pCsr->nPtr>0 ){ - Segment *pSeg; - pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg; - rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg, pSeg->iFirst, &pPg); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer for page pPg */ - int nData; /* Size of aData[] in bytes */ - aData = fsPageData(pPg, &nData); - iFPtr = pageGetPtr(aData, nData); - lsmFsPageRelease(pPg); - } - } - - if( rc==LSM_OK ){ - rc = mergeWorkerNextPage(pMW, iFPtr); - if( pCsr->pPrevMergePtr ) *pCsr->pPrevMergePtr = iFPtr; - pMW->aSave[0].bStore = 1; - } - - return rc; -} - -static int mergeWorkerWrite( - MergeWorker *pMW, /* Merge worker object to write into */ - int eType, /* One of SORTED_SEPARATOR, WRITE or DELETE */ - void *pKey, int nKey, /* Key value */ - void *pVal, int nVal, /* Value value */ - LsmPgno iPtr /* Absolute value of page pointer, or 0 */ -){ - int rc = LSM_OK; /* Return code */ - Merge *pMerge; /* Persistent part of level merge state */ - int nHdr; /* Space required for this record header */ - Page *pPg; /* Page to write to */ - u8 *aData; /* Data buffer for page pWriter->pPage */ - int nData = 0; /* Size of buffer aData[] in bytes */ - int nRec = 0; /* Number of records on page pPg */ - LsmPgno iFPtr = 0; /* Value of pointer in footer of pPg */ - LsmPgno iRPtr = 0; /* Value of pointer written into record */ - int iOff = 0; /* Current write offset within page pPg */ - Segment *pSeg; /* Segment being written */ - int flags = 0; /* If != 0, flags value for page footer */ - int bFirst = 0; /* True for first key of output run */ - - pMerge = pMW->pLevel->pMerge; - pSeg = &pMW->pLevel->lhs; - - if( pSeg->iFirst==0 && pMW->pPage==0 ){ - rc = mergeWorkerFirstPage(pMW); - bFirst = 1; - } - pPg = pMW->pPage; - if( pPg ){ - aData = fsPageData(pPg, &nData); - nRec = pageGetNRec(aData, nData); - iFPtr = pageGetPtr(aData, nData); - iRPtr = iPtr ? (iPtr - iFPtr) : 0; - } - - /* Figure out how much space is required by the new record. The space - ** required is divided into two sections: the header and the body. The - ** header consists of the intial varint fields. The body are the blobs - ** of data that correspond to the key and value data. The entire header - ** must be stored on the page. The body may overflow onto the next and - ** subsequent pages. - ** - ** The header space is: - ** - ** 1) record type - 1 byte. - ** 2) Page-pointer-offset - 1 varint - ** 3) Key size - 1 varint - ** 4) Value size - 1 varint (only if LSM_INSERT flag is set) - */ - if( rc==LSM_OK ){ - nHdr = 1 + lsmVarintLen64(iRPtr) + lsmVarintLen32(nKey); - if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal); - - /* If the entire header will not fit on page pPg, or if page pPg is - ** marked read-only, advance to the next page of the output run. */ - iOff = pMerge->iOutputOff; - if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ - if( iOff>=0 && pPg ){ - /* Zero any free space on the page */ - assert( aData ); - memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff); - } - iFPtr = *pMW->pCsr->pPrevMergePtr; - iRPtr = iPtr ? (iPtr - iFPtr) : 0; - iOff = 0; - nRec = 0; - rc = mergeWorkerNextPage(pMW, iFPtr); - pPg = pMW->pPage; - } - } - - /* If this record header will be the first on the page, and the page is - ** not the very first in the entire run, add a copy of the key to the - ** b-tree hierarchy. - */ - if( rc==LSM_OK && nRec==0 && bFirst==0 ){ - assert( pMerge->nSkip>=0 ); - - if( pMerge->nSkip==0 ){ - rc = mergeWorkerPushHierarchy(pMW, rtTopic(eType), pKey, nKey); - assert( pMW->aSave[0].bStore==0 ); - pMW->aSave[0].bStore = 1; - pMerge->nSkip = keyszToSkip(pMW->pDb->pFS, nKey); - }else{ - pMerge->nSkip--; - flags = PGFTR_SKIP_THIS_FLAG; - } - - if( pMerge->nSkip ) flags |= PGFTR_SKIP_NEXT_FLAG; - } - - /* Update the output segment */ - if( rc==LSM_OK ){ - aData = fsPageData(pPg, &nData); - - /* Update the page footer. */ - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); - lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); - if( flags ) lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], (u16)flags); - - /* Write the entry header into the current page. */ - aData[iOff++] = (u8)eType; /* 1 */ - iOff += lsmVarintPut64(&aData[iOff], iRPtr); /* 2 */ - iOff += lsmVarintPut32(&aData[iOff], nKey); /* 3 */ - if( rtIsWrite(eType) ) iOff += lsmVarintPut32(&aData[iOff], nVal); /* 4 */ - pMerge->iOutputOff = iOff; - - /* Write the key and data into the segment. */ - assert( iFPtr==pageGetPtr(aData, nData) ); - rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pKey, nKey); - if( rc==LSM_OK && rtIsWrite(eType) ){ - if( rc==LSM_OK ){ - rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pVal, nVal); - } - } - } - - return rc; -} - - -/* -** Free all resources allocated by mergeWorkerInit(). -*/ -static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ - int i; /* Iterator variable */ - int rc = *pRc; - MultiCursor *pCsr = pMW->pCsr; - - /* Unless the merge has finished, save the cursor position in the - ** Merge.aInput[] array. See function mergeWorkerInit() for the - ** code to restore a cursor position based on aInput[]. */ - if( rc==LSM_OK && pCsr ){ - Merge *pMerge = pMW->pLevel->pMerge; - if( lsmMCursorValid(pCsr) ){ - int bBtree = (pCsr->pBtCsr!=0); - int iPtr; - - /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ - assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); - assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); - - for(i=0; i<(pMerge->nInput-bBtree); i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - if( pPtr->pPg ){ - pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); - pMerge->aInput[i].iCell = pPtr->iCell; - }else{ - pMerge->aInput[i].iPg = 0; - pMerge->aInput[i].iCell = 0; - } - } - if( bBtree && pMerge->nInput ){ - assert( i==pCsr->nPtr ); - btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); - } - - /* Store the location of the split-key */ - iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iPtrnPtr ){ - pMerge->splitkey = pMerge->aInput[iPtr]; - }else{ - btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); - } - } - - /* Zero any free space left on the final page. This helps with - ** compression if using a compression hook. And prevents valgrind - ** from complaining about uninitialized byte passed to write(). */ - if( pMW->pPage ){ - int nData; - u8 *aData = fsPageData(pMW->pPage, &nData); - int iOff = pMerge->iOutputOff; - int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData)); - memset(&aData[iOff], 0, iEof - iOff); - } - - pMerge->iOutputOff = -1; - } - - lsmMCursorClose(pCsr, 0); - - /* Persist and release the output page. */ - if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); - if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); - if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); - if( rc==LSM_OK ) rc = mergeWorkerAddPadding(pMW); - lsmFsFlushWaiting(pMW->pDb->pFS, &rc); - mergeWorkerReleaseAll(pMW); - - lsmFree(pMW->pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - pMW->pCsr = 0; - - *pRc = rc; -} - -/* -** The cursor passed as the first argument is being used as the input for -** a merge operation. When this function is called, *piFlags contains the -** database entry flags for the current entry. The entry about to be written -** to the output. -** -** Note that this function only has to work for cursors configured to -** iterate forwards (not backwards). -*/ -static void mergeRangeDeletes(MultiCursor *pCsr, int *piVal, int *piFlags){ - int f = *piFlags; - int iKey = pCsr->aTree[1]; - int i; - - assert( pCsr->flags & CURSOR_NEXT_OK ); - if( pCsr->flags & CURSOR_IGNORE_DELETE ){ - /* The ignore-delete flag is set when the output of the merge will form - ** the oldest level in the database. In this case there is no point in - ** retaining any range-delete flags. */ - assert( (f & LSM_POINT_DELETE)==0 ); - f &= ~(LSM_START_DELETE|LSM_END_DELETE); - }else{ - for(i=0; i<(CURSOR_DATA_SEGMENT + pCsr->nPtr); i++){ - if( i!=iKey ){ - int eType; - void *pKey; - int nKey; - int res; - multiCursorGetKey(pCsr, i, &eType, &pKey, &nKey); - - if( pKey ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(eType), pKey, nKey - ); - assert( res<=0 ); - if( res==0 ){ - if( (f & (LSM_INSERT|LSM_POINT_DELETE))==0 ){ - if( eType & LSM_INSERT ){ - f |= LSM_INSERT; - *piVal = i; - } - else if( eType & LSM_POINT_DELETE ){ - f |= LSM_POINT_DELETE; - } - } - f |= (eType & (LSM_END_DELETE|LSM_START_DELETE)); - } - - if( i>iKey && (eType & LSM_END_DELETE) && res<0 ){ - if( f & (LSM_INSERT|LSM_POINT_DELETE) ){ - f |= (LSM_END_DELETE|LSM_START_DELETE); - }else{ - f = 0; - } - break; - } - } - } - } - - assert( (f & LSM_INSERT)==0 || (f & LSM_POINT_DELETE)==0 ); - if( (f & LSM_START_DELETE) - && (f & LSM_END_DELETE) - && (f & LSM_POINT_DELETE ) - ){ - f = 0; - } - } - - *piFlags = f; -} - -static int mergeWorkerStep(MergeWorker *pMW){ - lsm_db *pDb = pMW->pDb; /* Database handle */ - MultiCursor *pCsr; /* Cursor to read input data from */ - int rc = LSM_OK; /* Return code */ - int eType; /* SORTED_SEPARATOR, WRITE or DELETE */ - void *pKey; int nKey; /* Key */ - LsmPgno iPtr; - int iVal; - - pCsr = pMW->pCsr; - - /* Pull the next record out of the source cursor. */ - lsmMCursorKey(pCsr, &pKey, &nKey); - eType = pCsr->eType; - - /* Figure out if the output record may have a different pointer value - ** than the previous. This is the case if the current key is identical to - ** a key that appears in the lowest level run being merged. If so, set - ** iPtr to the absolute pointer value. If not, leave iPtr set to zero, - ** indicating that the output pointer value should be a copy of the pointer - ** value written with the previous key. */ - iPtr = (pCsr->pPrevMergePtr ? *pCsr->pPrevMergePtr : 0); - if( pCsr->pBtCsr ){ - BtreeCursor *pBtCsr = pCsr->pBtCsr; - if( pBtCsr->pKey ){ - int res = rtTopic(pBtCsr->eType) - rtTopic(eType); - if( res==0 ) res = pDb->xCmp(pBtCsr->pKey, pBtCsr->nKey, pKey, nKey); - if( 0==res ) iPtr = pBtCsr->iPtr; - assert( res>=0 ); - } - }else if( pCsr->nPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[pCsr->nPtr-1]; - if( pPtr->pPg - && 0==pDb->xCmp(pPtr->pKey, pPtr->nKey, pKey, nKey) - ){ - iPtr = pPtr->iPtr+pPtr->iPgPtr; - } - } - - iVal = pCsr->aTree[1]; - mergeRangeDeletes(pCsr, &iVal, &eType); - - if( eType!=0 ){ - if( pMW->aGobble ){ - int iGobble = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iGobblenPtr && iGobble>=0 ){ - SegmentPtr *pGobble = &pCsr->aPtr[iGobble]; - if( (pGobble->flags & PGFTR_SKIP_THIS_FLAG)==0 ){ - pMW->aGobble[iGobble] = lsmFsPageNumber(pGobble->pPg); - } - } - } - - /* If this is a separator key and we know that the output pointer has not - ** changed, there is no point in writing an output record. Otherwise, - ** proceed. */ - if( rc==LSM_OK && (rtIsSeparator(eType)==0 || iPtr!=0) ){ - /* Write the record into the main run. */ - void *pVal; int nVal; - rc = multiCursorGetVal(pCsr, iVal, &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - assert( nVal>=0 ); - rc = sortedBlobSet(pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } - if( rc==LSM_OK ){ - rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, iPtr); - } - } - } - - /* Advance the cursor to the next input record (assuming one exists). */ - assert( lsmMCursorValid(pMW->pCsr) ); - if( rc==LSM_OK ) rc = lsmMCursorNext(pMW->pCsr); - - return rc; -} - -static int mergeWorkerDone(MergeWorker *pMW){ - return pMW->pCsr==0 || !lsmMCursorValid(pMW->pCsr); -} - -static void sortedFreeLevel(lsm_env *pEnv, Level *p){ - if( p ){ - lsmFree(pEnv, p->pSplitKey); - lsmFree(pEnv, p->pMerge); - lsmFree(pEnv, p->aRhs); - lsmFree(pEnv, p); - } -} - -static void sortedInvokeWorkHook(lsm_db *pDb){ - if( pDb->xWork ){ - pDb->xWork(pDb, pDb->pWorkCtx); - } -} - -static int sortedNewToplevel( - lsm_db *pDb, /* Connection handle */ - int eTree, /* One of the TREE_XXX constants */ - int *pnWrite /* OUT: Number of database pages written */ -){ - int rc = LSM_OK; /* Return Code */ - MultiCursor *pCsr = 0; - Level *pNext = 0; /* The current top level */ - Level *pNew; /* The new level itself */ - Segment *pLinked = 0; /* Delete separators from this segment */ - Level *pDel = 0; /* Delete this entire level */ - int nWrite = 0; /* Number of database pages written */ - Freelist freelist; - - if( eTree!=TREE_NONE ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - - assert( pDb->bUseFreelist==0 ); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - memset(&freelist, 0, sizeof(freelist)); - - /* Allocate the new level structure to write to. */ - pNext = lsmDbSnapshotLevel(pDb->pWorker); - pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( pNew ){ - pNew->pNext = pNext; - lsmDbSnapshotSetLevel(pDb->pWorker, pNew); - } - - /* Create a cursor to gather the data required by the new segment. The new - ** segment contains everything in the tree and pointers to the next segment - ** in the database (if any). */ - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - pCsr->pDb = pDb; - rc = multiCursorVisitFreelist(pCsr); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); - } - if( rc==LSM_OK && pNext && pNext->pMerge==0 ){ - if( (pNext->flags & LEVEL_FREELIST_ONLY) ){ - pDel = pNext; - pCsr->aPtr = lsmMallocZeroRc(pDb->pEnv, sizeof(SegmentPtr), &rc); - multiCursorAddOne(pCsr, pNext, &rc); - }else if( eTree!=TREE_NONE && pNext->lhs.iRoot ){ - pLinked = &pNext->lhs; - rc = btreeCursorNew(pDb, pLinked, &pCsr->pBtCsr); - } - } - - /* If this will be the only segment in the database, discard any delete - ** markers present in the in-memory tree. */ - if( pNext==0 ){ - multiCursorIgnoreDelete(pCsr); - } - } - - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - }else{ - LsmPgno iLeftPtr = 0; - Merge merge; /* Merge object used to create new level */ - MergeWorker mergeworker; /* MergeWorker object for the same purpose */ - - memset(&merge, 0, sizeof(Merge)); - memset(&mergeworker, 0, sizeof(MergeWorker)); - - pNew->pMerge = &merge; - pNew->flags |= LEVEL_INCOMPLETE; - mergeworker.pDb = pDb; - mergeworker.pLevel = pNew; - mergeworker.pCsr = pCsr; - pCsr->pPrevMergePtr = &iLeftPtr; - - /* Mark the separators array for the new level as a "phantom". */ - mergeworker.bFlush = 1; - - /* Do the work to create the new merged segment on disk */ - if( rc==LSM_OK ) rc = lsmMCursorFirst(pCsr); - while( rc==LSM_OK && mergeWorkerDone(&mergeworker)==0 ){ - rc = mergeWorkerStep(&mergeworker); - } - mergeWorkerShutdown(&mergeworker, &rc); - assert( rc!=LSM_OK || mergeworker.nWork==0 || pNew->lhs.iFirst ); - if( rc==LSM_OK && pNew->lhs.iFirst ){ - rc = lsmFsSortedFinish(pDb->pFS, &pNew->lhs); - } - nWrite = mergeworker.nWork; - pNew->flags &= ~LEVEL_INCOMPLETE; - if( eTree==TREE_NONE ){ - pNew->flags |= LEVEL_FREELIST_ONLY; - } - pNew->pMerge = 0; - } - - if( rc!=LSM_OK || pNew->lhs.iFirst==0 ){ - assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); - lsmDbSnapshotSetLevel(pDb->pWorker, pNext); - sortedFreeLevel(pDb->pEnv, pNew); - }else{ - if( pLinked ){ - pLinked->iRoot = 0; - }else if( pDel ){ - assert( pNew->pNext==pDel ); - pNew->pNext = pDel->pNext; - lsmFsSortedDelete(pDb->pFS, pDb->pWorker, 1, &pDel->lhs); - sortedFreeLevel(pDb->pEnv, pDel); - } - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "new-toplevel"); -#endif - - if( freelist.nEntry ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - freelist.aEntry = 0; - }else{ - pDb->pWorker->freelist.nEntry = 0; - } - - assertBtreeOk(pDb, &pNew->lhs); - sortedInvokeWorkHook(pDb); - } - - if( pnWrite ) *pnWrite = nWrite; - pDb->pWorker->nWrite += nWrite; - pDb->pFreelist = 0; - pDb->bUseFreelist = 0; - lsmFree(pDb->pEnv, freelist.aEntry); - return rc; -} - -/* -** The nMerge levels in the LSM beginning with pLevel consist of a -** left-hand-side segment only. Replace these levels with a single new -** level consisting of a new empty segment on the left-hand-side and the -** nMerge segments from the replaced levels on the right-hand-side. -** -** Also, allocate and populate a Merge object and set Level.pMerge to -** point to it. -*/ -static int sortedMergeSetup( - lsm_db *pDb, /* Database handle */ - Level *pLevel, /* First level to merge */ - int nMerge, /* Merge this many levels together */ - Level **ppNew /* New, merged, level */ -){ - int rc = LSM_OK; /* Return Code */ - Level *pNew; /* New Level object */ - int bUseNext = 0; /* True to link in next separators */ - Merge *pMerge; /* New Merge object */ - int nByte; /* Bytes of space allocated at pMerge */ - -#ifdef LSM_DEBUG - int iLevel; - Level *pX = pLevel; - for(iLevel=0; iLevelnRight==0 ); - pX = pX->pNext; - } -#endif - - /* Allocate the new Level object */ - pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( pNew ){ - pNew->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, - nMerge * sizeof(Segment), &rc); - } - - /* Populate the new Level object */ - if( rc==LSM_OK ){ - Level *pNext = 0; /* Level following pNew */ - int i; - int bFreeOnly = 1; - Level *pTopLevel; - Level *p = pLevel; - Level **pp; - pNew->nRight = nMerge; - pNew->iAge = pLevel->iAge+1; - for(i=0; inRight==0 ); - pNext = p->pNext; - pNew->aRhs[i] = p->lhs; - if( (p->flags & LEVEL_FREELIST_ONLY)==0 ) bFreeOnly = 0; - sortedFreeLevel(pDb->pEnv, p); - p = pNext; - } - - if( bFreeOnly ) pNew->flags |= LEVEL_FREELIST_ONLY; - - /* Replace the old levels with the new. */ - pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - pNew->pNext = p; - for(pp=&pTopLevel; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pNew; - lsmDbSnapshotSetLevel(pDb->pWorker, pTopLevel); - - /* Determine whether or not the next separators will be linked in */ - if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot && pNext - && (bFreeOnly==0 || (pNext->flags & LEVEL_FREELIST_ONLY)) - ){ - bUseNext = 1; - } - } - - /* Allocate the merge object */ - nByte = sizeof(Merge) + sizeof(MergeInput) * (nMerge + bUseNext); - pMerge = (Merge *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - if( pMerge ){ - pMerge->aInput = (MergeInput *)&pMerge[1]; - pMerge->nInput = nMerge + bUseNext; - pNew->pMerge = pMerge; - } - - *ppNew = pNew; - return rc; -} - -static int mergeWorkerInit( - lsm_db *pDb, /* Db connection to do merge work */ - Level *pLevel, /* Level to work on merging */ - MergeWorker *pMW /* Object to initialize */ -){ - int rc = LSM_OK; /* Return code */ - Merge *pMerge = pLevel->pMerge; /* Persistent part of merge state */ - MultiCursor *pCsr = 0; /* Cursor opened for pMW */ - Level *pNext = pLevel->pNext; /* Next level in LSM */ - - assert( pDb->pWorker ); - assert( pLevel->pMerge ); - assert( pLevel->nRight>0 ); - - memset(pMW, 0, sizeof(MergeWorker)); - pMW->pDb = pDb; - pMW->pLevel = pLevel; - pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc); - - /* Create a multi-cursor to read the data to write to the new - ** segment. The new segment contains: - ** - ** 1. Records from LHS of each of the nMerge levels being merged. - ** 2. Separators from either the last level being merged, or the - ** separators attached to the LHS of the following level, or neither. - ** - ** If the new level is the lowest (oldest) in the db, discard any - ** delete keys. Key annihilation. - */ - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - pCsr->flags |= CURSOR_NEXT_OK; - rc = multiCursorAddRhs(pCsr, pLevel); - } - if( rc==LSM_OK && pMerge->nInput > pLevel->nRight ){ - rc = btreeCursorNew(pDb, &pNext->lhs, &pCsr->pBtCsr); - }else if( pNext ){ - multiCursorReadSeparators(pCsr); - }else{ - multiCursorIgnoreDelete(pCsr); - } - - assert( rc!=LSM_OK || pMerge->nInput==(pCsr->nPtr+(pCsr->pBtCsr!=0)) ); - pMW->pCsr = pCsr; - - /* Load the b-tree hierarchy into memory. */ - if( rc==LSM_OK ) rc = mergeWorkerLoadHierarchy(pMW); - if( rc==LSM_OK && pMW->hier.nHier==0 ){ - pMW->aSave[0].iPgno = pLevel->lhs.iFirst; - } - - /* Position the cursor. */ - if( rc==LSM_OK ){ - pCsr->pPrevMergePtr = &pMerge->iCurrentPtr; - if( pLevel->lhs.iFirst==0 ){ - /* The output array is still empty. So position the cursor at the very - ** start of the input. */ - rc = multiCursorEnd(pCsr, 0); - }else{ - /* The output array is non-empty. Position the cursor based on the - ** page/cell data saved in the Merge.aInput[] array. */ - int i; - for(i=0; rc==LSM_OK && inPtr; i++){ - MergeInput *pInput = &pMerge->aInput[i]; - if( pInput->iPg ){ - SegmentPtr *pPtr; - assert( pCsr->aPtr[i].pPg==0 ); - pPtr = &pCsr->aPtr[i]; - rc = segmentPtrLoadPage(pDb->pFS, pPtr, pInput->iPg); - if( rc==LSM_OK && pPtr->nCell>0 ){ - rc = segmentPtrLoadCell(pPtr, pInput->iCell); - } - } - } - - if( rc==LSM_OK && pCsr->pBtCsr ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - assert( i==pCsr->nPtr ); - rc = btreeCursorRestore(pCsr->pBtCsr, xCmp, &pMerge->aInput[i]); - } - - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, 0); - } - } - pCsr->flags |= CURSOR_NEXT_OK; - } - - return rc; -} - -static int sortedBtreeGobble( - lsm_db *pDb, /* Worker connection */ - MultiCursor *pCsr, /* Multi-cursor being used for a merge */ - int iGobble /* pCsr->aPtr[] entry to operate on */ -){ - int rc = LSM_OK; - if( rtTopic(pCsr->eType)==0 ){ - Segment *pSeg = pCsr->aPtr[iGobble].pSeg; - LsmPgno *aPg; - int nPg; - - /* Seek from the root of the b-tree to the segment leaf that may contain - ** a key equal to the one multi-cursor currently points to. Record the - ** page number of each b-tree page and the leaf. The segment may be - ** gobbled up to (but not including) the first of these page numbers. - */ - assert( pSeg->iRoot>0 ); - aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc); - if( rc==LSM_OK ){ - rc = seekInBtree(pCsr, pSeg, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 - ); - } - - if( rc==LSM_OK ){ - for(nPg=0; aPg[nPg]; nPg++); - lsmFsGobble(pDb, pSeg, aPg, nPg); - } - - lsmFree(pDb->pEnv, aPg); - } - return rc; -} - -/* -** Argument p points to a level of age N. Return the number of levels in -** the linked list starting at p that have age=N (always at least 1). -*/ -static int sortedCountLevels(Level *p){ - int iAge = p->iAge; - int nRet = 0; - do { - nRet++; - p = p->pNext; - }while( p && p->iAge==iAge ); - return nRet; -} - -static int sortedSelectLevel(lsm_db *pDb, int nMerge, Level **ppOut){ - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - int rc = LSM_OK; - Level *pLevel = 0; /* Output value */ - Level *pBest = 0; /* Best level to work on found so far */ - int nBest; /* Number of segments merged at pBest */ - Level *pThis = 0; /* First in run of levels with age=iAge */ - int nThis = 0; /* Number of levels starting at pThis */ - - assert( nMerge>=1 ); - nBest = LSM_MAX(1, nMerge-1); - - /* Find the longest contiguous run of levels not currently undergoing a - ** merge with the same age in the structure. Or the level being merged - ** with the largest number of right-hand segments. Work on it. */ - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - if( pLevel->nRight==0 && pThis && pLevel->iAge==pThis->iAge ){ - nThis++; - }else{ - if( nThis>nBest ){ - if( (pLevel->iAge!=pThis->iAge+1) - || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) - ){ - pBest = pThis; - nBest = nThis; - } - } - if( pLevel->nRight ){ - if( pLevel->nRight>nBest ){ - nBest = pLevel->nRight; - pBest = pLevel; - } - nThis = 0; - pThis = 0; - }else{ - pThis = pLevel; - nThis = 1; - } - } - } - if( nThis>nBest ){ - assert( pThis ); - pBest = pThis; - nBest = nThis; - } - - if( pBest==0 && nMerge==1 ){ - int nFree = 0; - int nUsr = 0; - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - assert( !pLevel->nRight ); - if( pLevel->flags & LEVEL_FREELIST_ONLY ){ - nFree++; - }else{ - nUsr++; - } - } - if( nUsr>1 ){ - pBest = pTopLevel; - nBest = nFree + nUsr; - } - } - - if( pBest ){ - if( pBest->nRight==0 ){ - rc = sortedMergeSetup(pDb, pBest, nBest, ppOut); - }else{ - *ppOut = pBest; - } - } - - return rc; -} - -static int sortedDbIsFull(lsm_db *pDb){ - Level *pTop = lsmDbSnapshotLevel(pDb->pWorker); - - if( lsmDatabaseFull(pDb) ) return 1; - if( pTop && pTop->iAge==0 - && (pTop->nRight || sortedCountLevels(pTop)>=pDb->nMerge) - ){ - return 1; - } - return 0; -} - -typedef struct MoveBlockCtx MoveBlockCtx; -struct MoveBlockCtx { - int iSeen; /* Previous free block on list */ - int iFrom; /* Total number of blocks in file */ -}; - -static int moveBlockCb(void *pCtx, int iBlk, i64 iSnapshot){ - MoveBlockCtx *p = (MoveBlockCtx *)pCtx; - assert( p->iFrom==0 ); - if( iBlk==(p->iSeen-1) ){ - p->iSeen = iBlk; - return 0; - } - p->iFrom = p->iSeen-1; - return 1; -} - -/* -** This function is called to further compact a database for which all -** of the content has already been merged into a single segment. If -** possible, it moves the contents of a single block from the end of the -** file to a free-block that lies closer to the start of the file (allowing -** the file to be eventually truncated). -*/ -static int sortedMoveBlock(lsm_db *pDb, int *pnWrite){ - Snapshot *p = pDb->pWorker; - Level *pLvl = lsmDbSnapshotLevel(p); - int iFrom; /* Block to move */ - int iTo; /* Destination to move block to */ - int rc; /* Return code */ - - MoveBlockCtx sCtx; - - assert( pLvl->pNext==0 && pLvl->nRight==0 ); - assert( p->redirect.n<=LSM_MAX_BLOCK_REDIRECTS ); - - *pnWrite = 0; - - /* Check that the redirect array is not already full. If it is, return - ** without moving any database content. */ - if( p->redirect.n>=LSM_MAX_BLOCK_REDIRECTS ) return LSM_OK; - - /* Find the last block of content in the database file. Do this by - ** traversing the free-list in reverse (descending block number) order. - ** The first block not on the free list is the one that will be moved. - ** Since the db consists of a single segment, there is no ambiguity as - ** to which segment the block belongs to. */ - sCtx.iSeen = p->nBlock+1; - sCtx.iFrom = 0; - rc = lsmWalkFreelist(pDb, 1, moveBlockCb, &sCtx); - if( rc!=LSM_OK || sCtx.iFrom==0 ) return rc; - iFrom = sCtx.iFrom; - - /* Find the first free block in the database, ignoring block 1. Block - ** 1 is tricky as it is smaller than the other blocks. */ - rc = lsmBlockAllocate(pDb, iFrom, &iTo); - if( rc!=LSM_OK || iTo==0 ) return rc; - assert( iTo!=1 && iTopFS, &pLvl->lhs, iTo, iFrom); - if( rc==LSM_OK ){ - if( p->redirect.a==0 ){ - int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; - p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - - /* Check if the block just moved was already redirected. */ - int i; - for(i=0; iredirect.n; i++){ - if( p->redirect.a[i].iTo==iFrom ) break; - } - - if( i==p->redirect.n ){ - /* Block iFrom was not already redirected. Add a new array entry. */ - memmove(&p->redirect.a[1], &p->redirect.a[0], - sizeof(struct RedirectEntry) * p->redirect.n - ); - p->redirect.a[0].iFrom = iFrom; - p->redirect.a[0].iTo = iTo; - p->redirect.n++; - }else{ - /* Block iFrom was already redirected. Overwrite existing entry. */ - p->redirect.a[i].iTo = iTo; - } - - rc = lsmBlockFree(pDb, iFrom); - - *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); - pLvl->lhs.pRedirect = &p->redirect; - } - } - -#if LSM_LOG_STRUCTURE - if( rc==LSM_OK ){ - char aBuf[64]; - sprintf(aBuf, "move-block %d/%d", p->redirect.n-1, LSM_MAX_BLOCK_REDIRECTS); - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, aBuf); - } -#endif - return rc; -} - -/* -*/ -static int mergeInsertFreelistSegments( - lsm_db *pDb, - int nFree, - MergeWorker *pMW -){ - int rc = LSM_OK; - if( nFree>0 ){ - MultiCursor *pCsr = pMW->pCsr; - Level *pLvl = pMW->pLevel; - SegmentPtr *aNew1; - Segment *aNew2; - - Level *pIter; - Level *pNext; - int i = 0; - - aNew1 = (SegmentPtr *)lsmMallocZeroRc( - pDb->pEnv, sizeof(SegmentPtr) * (pCsr->nPtr+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew1[nFree], pCsr->aPtr, sizeof(SegmentPtr)*pCsr->nPtr); - pCsr->nPtr += nFree; - lsmFree(pDb->pEnv, pCsr->aTree); - lsmFree(pDb->pEnv, pCsr->aPtr); - pCsr->aTree = 0; - pCsr->aPtr = aNew1; - - aNew2 = (Segment *)lsmMallocZeroRc( - pDb->pEnv, sizeof(Segment) * (pLvl->nRight+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew2[nFree], pLvl->aRhs, sizeof(Segment)*pLvl->nRight); - pLvl->nRight += nFree; - lsmFree(pDb->pEnv, pLvl->aRhs); - pLvl->aRhs = aNew2; - - for(pIter=pDb->pWorker->pLevel; rc==LSM_OK && pIter!=pLvl; pIter=pNext){ - Segment *pSeg = &pLvl->aRhs[i]; - memcpy(pSeg, &pIter->lhs, sizeof(Segment)); - - pCsr->aPtr[i].pSeg = pSeg; - pCsr->aPtr[i].pLevel = pLvl; - rc = segmentPtrEnd(pCsr, &pCsr->aPtr[i], 0); - - pDb->pWorker->pLevel = pNext = pIter->pNext; - sortedFreeLevel(pDb->pEnv, pIter); - i++; - } - assert( i==nFree ); - assert( rc!=LSM_OK || pDb->pWorker->pLevel==pLvl ); - - for(i=nFree; inPtr; i++){ - pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; - } - - lsmFree(pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - } - return rc; -} - -static int sortedWork( - lsm_db *pDb, /* Database handle. Must be worker. */ - int nWork, /* Number of pages of work to do */ - int nMerge, /* Try to merge this many levels at once */ - int bFlush, /* Set if call is to make room for a flush */ - int *pnWrite /* OUT: Actual number of pages written */ -){ - int rc = LSM_OK; /* Return Code */ - int nRemaining = nWork; /* Units of work to do before returning */ - Snapshot *pWorker = pDb->pWorker; - - assert( pWorker ); - if( lsmDbSnapshotLevel(pWorker)==0 ) return LSM_OK; - - while( nRemaining>0 ){ - Level *pLevel = 0; - - /* Find a level to work on. */ - rc = sortedSelectLevel(pDb, nMerge, &pLevel); - assert( rc==LSM_OK || pLevel==0 ); - - if( pLevel==0 ){ - int nDone = 0; - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - if( bFlush==0 && nMerge==1 && pTopLevel && pTopLevel->pNext==0 ){ - rc = sortedMoveBlock(pDb, &nDone); - } - nRemaining -= nDone; - - /* Could not find any work to do. Finished. */ - if( nDone==0 ) break; - }else{ - int bSave = 0; - Freelist freelist = {0, 0, 0}; - MergeWorker mergeworker; /* State used to work on the level merge */ - - assert( pDb->bIncrMerge==0 ); - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - pDb->bIncrMerge = 1; - rc = mergeWorkerInit(pDb, pLevel, &mergeworker); - assert( mergeworker.nWork==0 ); - - while( rc==LSM_OK - && 0==mergeWorkerDone(&mergeworker) - && (mergeworker.nWorkbUseFreelist) - ){ - int eType = rtTopic(mergeworker.pCsr->eType); - rc = mergeWorkerStep(&mergeworker); - - /* If the cursor now points at the first entry past the end of the - ** user data (i.e. either to EOF or to the first free-list entry - ** that will be added to the run), then check if it is possible to - ** merge in any free-list entries that are either in-memory or in - ** free-list-only blocks. */ - if( rc==LSM_OK && nMerge==1 && eType==0 - && (rtTopic(mergeworker.pCsr->eType) || mergeWorkerDone(&mergeworker)) - ){ - int nFree = 0; /* Number of free-list-only levels to merge */ - Level *pLvl; - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - /* Now check if all levels containing data newer than this one - ** are single-segment free-list only levels. If so, they will be - ** merged in now. */ - for(pLvl=pDb->pWorker->pLevel; - pLvl!=mergeworker.pLevel && (pLvl->flags & LEVEL_FREELIST_ONLY); - pLvl=pLvl->pNext - ){ - assert( pLvl->nRight==0 ); - nFree++; - } - if( pLvl==mergeworker.pLevel ){ - - rc = mergeInsertFreelistSegments(pDb, nFree, &mergeworker); - if( rc==LSM_OK ){ - rc = multiCursorVisitFreelist(mergeworker.pCsr); - } - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(mergeworker.pCsr, 0); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - } - } - } - } - nRemaining -= LSM_MAX(mergeworker.nWork, 1); - - if( rc==LSM_OK ){ - /* Check if the merge operation is completely finished. If not, - ** gobble up (declare eligible for recycling) any pages from rhs - ** segments for which the content has been completely merged into - ** the lhs of the level. */ - if( mergeWorkerDone(&mergeworker)==0 ){ - int i; - for(i=0; inRight; i++){ - SegmentPtr *pGobble = &mergeworker.pCsr->aPtr[i]; - if( pGobble->pSeg->iRoot ){ - rc = sortedBtreeGobble(pDb, mergeworker.pCsr, i); - }else if( mergeworker.aGobble[i] ){ - lsmFsGobble(pDb, pGobble->pSeg, &mergeworker.aGobble[i], 1); - } - } - }else{ - int i; - int bEmpty; - mergeWorkerShutdown(&mergeworker, &rc); - bEmpty = (pLevel->lhs.iFirst==0); - - if( bEmpty==0 && rc==LSM_OK ){ - rc = lsmFsSortedFinish(pDb->pFS, &pLevel->lhs); - } - - if( pDb->bUseFreelist ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - pDb->bUseFreelist = 0; - pDb->pFreelist = 0; - bSave = 1; - } - - for(i=0; inRight; i++){ - lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); - } - - if( bEmpty ){ - /* If the new level is completely empty, remove it from the - ** database snapshot. This can only happen if all input keys were - ** annihilated. Since keys are only annihilated if the new level - ** is the last in the linked list (contains the most ancient of - ** database content), this guarantees that pLevel->pNext==0. */ - Level *pTop; /* Top level of worker snapshot */ - Level **pp; /* Read/write iterator for Level.pNext list */ - - assert( pLevel->pNext==0 ); - - /* Remove the level from the worker snapshot. */ - pTop = lsmDbSnapshotLevel(pWorker); - for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pLevel->pNext; - lsmDbSnapshotSetLevel(pWorker, pTop); - - /* Free the Level structure. */ - sortedFreeLevel(pDb->pEnv, pLevel); - }else{ - - /* Free the separators of the next level, if required. */ - if( pLevel->pMerge->nInput > pLevel->nRight ){ - assert( pLevel->pNext->lhs.iRoot ); - pLevel->pNext->lhs.iRoot = 0; - } - - /* Zero the right-hand-side of pLevel */ - lsmFree(pDb->pEnv, pLevel->aRhs); - pLevel->nRight = 0; - pLevel->aRhs = 0; - - /* Free the Merge object */ - lsmFree(pDb->pEnv, pLevel->pMerge); - pLevel->pMerge = 0; - } - - if( bSave && rc==LSM_OK ){ - pDb->bIncrMerge = 0; - rc = lsmSaveWorker(pDb, 0); - } - } - } - - /* Clean up the MergeWorker object initialized above. If no error - ** has occurred, invoke the work-hook to inform the application that - ** the database structure has changed. */ - mergeWorkerShutdown(&mergeworker, &rc); - pDb->bIncrMerge = 0; - if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "work"); -#endif - assertBtreeOk(pDb, &pLevel->lhs); - assertRunInOrder(pDb, &pLevel->lhs); - - /* If bFlush is true and the database is no longer considered "full", - ** break out of the loop even if nRemaining is still greater than - ** zero. The caller has an in-memory tree to flush to disk. */ - if( bFlush && sortedDbIsFull(pDb)==0 ) break; - } - } - - if( pnWrite ) *pnWrite = (nWork - nRemaining); - pWorker->nWrite += (nWork - nRemaining); - -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, rc, "sortedWork(): %d pages", (nWork-nRemaining)); -#endif - return rc; -} - -/* -** The database connection passed as the first argument must be a worker -** connection. This function checks if there exists an "old" in-memory tree -** ready to be flushed to disk. If so, true is returned. Otherwise false. -** -** If an error occurs, *pRc is set to an LSM error code before returning. -** It is assumed that *pRc is set to LSM_OK when this function is called. -*/ -static int sortedTreeHasOld(lsm_db *pDb, int *pRc){ - int rc = LSM_OK; - int bRet = 0; - - assert( pDb->pWorker ); - if( *pRc==LSM_OK ){ - if( rc==LSM_OK - && pDb->treehdr.iOldShmid - && pDb->treehdr.iOldLog!=pDb->pWorker->iLogOff - ){ - bRet = 1; - }else{ - bRet = 0; - } - *pRc = rc; - } - assert( *pRc==LSM_OK || bRet==0 ); - return bRet; -} - -/* -** Create a new free-list only top-level segment. Return LSM_OK if successful -** or an LSM error code if some error occurs. -*/ -static int sortedNewFreelistOnly(lsm_db *pDb){ - return sortedNewToplevel(pDb, TREE_NONE, 0); -} - -int lsmSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *p = pDb->pWorker; - if( p->freelist.nEntry>pDb->nMaxFreelist ){ - int rc = sortedNewFreelistOnly(pDb); - if( rc!=LSM_OK ) return rc; - } - return lsmCheckpointSaveWorker(pDb, bFlush); -} - -static int doLsmSingleWork( - lsm_db *pDb, - int bShutdown, - int nMerge, /* Minimum segments to merge together */ - int nPage, /* Number of pages to write to disk */ - int *pnWrite, /* OUT: Pages actually written to disk */ - int *pbCkpt /* OUT: True if an auto-checkpoint is req. */ -){ - Snapshot *pWorker; /* Worker snapshot */ - int rc = LSM_OK; /* Return code */ - int bDirty = 0; - int nMax = nPage; /* Maximum pages to write to disk */ - int nRem = nPage; - int bCkpt = 0; - - assert( nPage>0 ); - - /* Open the worker 'transaction'. It will be closed before this function - ** returns. */ - assert( pDb->pWorker==0 ); - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - - /* If this connection is doing auto-checkpoints, set nMax (and nRem) so - ** that this call stops writing when the auto-checkpoint is due. The - ** caller will do the checkpoint, then possibly call this function again. */ - if( bShutdown==0 && pDb->nAutockpt ){ - u32 nSync; - u32 nUnsync; - int nPgsz; - - lsmCheckpointSynced(pDb, 0, 0, &nSync); - nUnsync = lsmCheckpointNWrite(pDb->pShmhdr->aSnap1, 0); - nPgsz = lsmCheckpointPgsz(pDb->pShmhdr->aSnap1); - - nMax = (int)LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (int)(nUnsync-nSync)); - if( nMaxnTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( sortedTreeHasOld(pDb, &rc) ){ - /* sortedDbIsFull() returns non-zero if either (a) there are too many - ** levels in total in the db, or (b) there are too many levels with the - ** the same age in the db. Either way, call sortedWork() to merge - ** existing segments together until this condition is cleared. */ - if( sortedDbIsFull(pDb) ){ - int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 1, &nPg); - nRem -= nPg; - assert( rc!=LSM_OK || nRem<=0 || !sortedDbIsFull(pDb) ); - bDirty = 1; - } - - if( rc==LSM_OK && nRem>0 ){ - int nPg = 0; - rc = sortedNewToplevel(pDb, TREE_OLD, &nPg); - nRem -= nPg; - if( rc==LSM_OK ){ - if( pDb->nTransOpen>0 ){ - lsmTreeDiscardOld(pDb); - } - rc = lsmSaveWorker(pDb, 1); - bDirty = 0; - } - } - } - - /* If nPage is still greater than zero, do some merging. */ - if( rc==LSM_OK && nRem>0 && bShutdown==0 ){ - int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 0, &nPg); - nRem -= nPg; - if( nPg ) bDirty = 1; - } - - /* If the in-memory part of the free-list is too large, write a new - ** top-level containing just the in-memory free-list entries to disk. */ - if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){ - while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ - int nPg = 0; - rc = sortedWork(pDb, 16, nMerge, 1, &nPg); - nRem -= nPg; - } - if( rc==LSM_OK ){ - rc = sortedNewFreelistOnly(pDb); - } - bDirty = 1; - } - - if( rc==LSM_OK ){ - *pnWrite = (nMax - nRem); - *pbCkpt = (bCkpt && nRem<=0); - if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0 - && pWorker->pLevel - && pWorker->pLevel->nRight==0 - && pWorker->pLevel->pNext==0 - ){ - *pbCkpt = 1; - } - } - - if( rc==LSM_OK && bDirty ){ - lsmFinishWork(pDb, 0, &rc); - }else{ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - *pnWrite = 0; - } - assert( pDb->pWorker==0 ); - return rc; -} - -static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){ - int rc = LSM_OK; /* Return code */ - int nWrite = 0; /* Number of pages written */ - - assert( nMerge>=1 ); - - if( nPage!=0 ){ - int bCkpt = 0; - do { - int nThis = 0; - int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); - - bCkpt = 0; - rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); - nWrite += nThis; - if( rc==LSM_OK && bCkpt ){ - rc = lsm_checkpoint(pDb, 0); - } - }while( rc==LSM_OK && bCkpt && (nWritenTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; - if( nMerge<=0 ) nMerge = pDb->nMerge; - - lsmFsPurgeCache(pDb->pFS); - - /* Convert from KB to pages */ - nPgsz = lsmFsPageSize(pDb->pFS); - if( nKB>=0 ){ - nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; - }else{ - nPage = -1; - } - - rc = doLsmWork(pDb, nMerge, nPage, &nWrite); - - if( pnWrite ){ - /* Convert back from pages to KB */ - *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); - } - return rc; -} - -int lsm_flush(lsm_db *db){ - int rc; - - if( db->nTransOpen>0 || db->pCsr ){ - rc = LSM_MISUSE_BKPT; - }else{ - rc = lsmBeginWriteTrans(db); - if( rc==LSM_OK ){ - lsmFlushTreeToDisk(db); - lsmTreeDiscardOld(db); - lsmTreeMakeOld(db); - lsmTreeDiscardOld(db); - } - - if( rc==LSM_OK ){ - rc = lsmFinishWriteTrans(db, 1); - }else{ - lsmFinishWriteTrans(db, 0); - } - lsmFinishReadTrans(db); - } - - return rc; -} - -/* -** This function is called in auto-work mode to perform merging work on -** the data structure. It performs enough merging work to prevent the -** height of the tree from growing indefinitely assuming that roughly -** nUnit database pages worth of data have been written to the database -** (i.e. the in-memory tree) since the last call. -*/ -int lsmSortedAutoWork( - lsm_db *pDb, /* Database handle */ - int nUnit /* Pages of data written to in-memory tree */ -){ - int rc = LSM_OK; /* Return code */ - int nDepth = 0; /* Current height of tree (longest path) */ - Level *pLevel; /* Used to iterate through levels */ - int bRestore = 0; - - assert( pDb->pWorker==0 ); - assert( pDb->nTransOpen>0 ); - - /* Determine how many units of work to do before returning. One unit of - ** work is achieved by writing one page (~4KB) of merged data. */ - for(pLevel=lsmDbSnapshotLevel(pDb->pClient); pLevel; pLevel=pLevel->pNext){ - /* nDepth += LSM_MAX(1, pLevel->nRight); */ - nDepth += 1; - } - if( lsmTreeHasOld(pDb) ){ - nDepth += 1; - bRestore = 1; - rc = lsmSaveCursors(pDb); - if( rc!=LSM_OK ) return rc; - } - - if( nDepth>0 ){ - int nRemaining; /* Units of work to do before returning */ - - nRemaining = nUnit * nDepth; -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", - nUnit, nDepth, nRemaining); -#endif - assert( nRemaining>=0 ); - rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); - if( rc==LSM_BUSY ) rc = LSM_OK; - - if( bRestore && pDb->pCsr ){ - lsmMCursorFreeCache(pDb); - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - if( rc==LSM_OK ){ - rc = lsmCheckpointLoad(pDb, 0); - } - if( rc==LSM_OK ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot, &pDb->pClient); - } - if( rc==LSM_OK ){ - rc = lsmRestoreCursors(pDb); - } - } - } - - return rc; -} - -/* -** This function is only called during system shutdown. The contents of -** any in-memory trees present (old or current) are written out to disk. -*/ -int lsmFlushTreeToDisk(lsm_db *pDb){ - int rc; - - rc = lsmBeginWork(pDb); - while( rc==LSM_OK && sortedDbIsFull(pDb) ){ - rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); - } - - if( rc==LSM_OK ){ - rc = sortedNewToplevel(pDb, TREE_BOTH, 0); - } - - lsmFinishWork(pDb, 1, &rc); - return rc; -} - -/* -** Return a string representation of the segment passed as the only argument. -** Space for the returned string is allocated using lsmMalloc(), and should -** be freed by the caller using lsmFree(). -*/ -static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){ - LsmPgno nSize = pSeg->nSize; - LsmPgno iRoot = pSeg->iRoot; - LsmPgno iFirst = pSeg->iFirst; - LsmPgno iLast = pSeg->iLastPg; - char *z; - - char *z1; - char *z2; - int nPad; - - z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast); - if( iRoot ){ - z2 = lsmMallocPrintf(pEnv, "root=%lld", iRoot); - }else{ - z2 = lsmMallocPrintf(pEnv, "size=%lld", nSize); - } - - nPad = nMin - 2 - strlen(z1) - 1 - strlen(z2); - nPad = LSM_MAX(0, nPad); - - if( iRoot ){ - z = lsmMallocPrintf(pEnv, "/%s %*s%s\\", z1, nPad, "", z2); - }else{ - z = lsmMallocPrintf(pEnv, "|%s %*s%s|", z1, nPad, "", z2); - } - lsmFree(pEnv, z1); - lsmFree(pEnv, z2); - - return z; -} - -static int fileToString( - lsm_db *pDb, /* For xMalloc() */ - char *aBuf, - int nBuf, - int nMin, - Segment *pSeg -){ - int i = 0; - if( pSeg ){ - char *zSeg; - - zSeg = segToString(pDb->pEnv, pSeg, nMin); - snprintf(&aBuf[i], nBuf-i, "%s", zSeg); - i += strlen(&aBuf[i]); - lsmFree(pDb->pEnv, zSeg); - -#ifdef LSM_LOG_FREELIST - lsmInfoArrayStructure(pDb, 1, pSeg->iFirst, &zSeg); - snprintf(&aBuf[i], nBuf-1, " (%s)", zSeg); - i += strlen(&aBuf[i]); - lsmFree(pDb->pEnv, zSeg); -#endif - aBuf[nBuf] = 0; - }else{ - aBuf[0] = '\0'; - } - - return i; -} - -void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ - LsmBlob blob = {0, 0, 0}; /* LsmBlob used for keys */ - LsmString s; - int i; - - int nRec; - LsmPgno iPtr; - int flags; - u8 *aData; - int nData; - - aData = fsPageData(pPg, &nData); - - nRec = pageGetNRec(aData, nData); - iPtr = pageGetPtr(aData, nData); - flags = pageGetFlags(aData, nData); - - lsmStringInit(&s, pDb->pEnv); - lsmStringAppendf(&s,"nCell=%d iPtr=%lld flags=%d {", nRec, iPtr, flags); - if( flags&SEGMENT_BTREE_FLAG ) iPtr = 0; - - for(i=0; ipFS, pRun, iRef, &pRef); - aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); - }else{ - aCell += lsmVarintGet32(aCell, &nKey); - if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); - aVal = &aKey[nKey]; - iTopic = eType; - } - - lsmStringAppendf(&s, "%s%2X:", (i==0?"":" "), iTopic); - for(iChar=0; iChar0 && bVals ){ - lsmStringAppendf(&s, "##"); - for(iChar=0; iCharpFS, pSeg, iRef, &pRef); - pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); - aKey = (u8 *)pBlob->pData; - nKey = pBlob->nData; - lsmFsPageRelease(pRef); - }else{ - aKey = (u8 *)""; - nKey = 11; - } - }else{ - aCell += lsmVarintGet32(aCell, &nKey); - if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(pSeg, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); - aVal = &aKey[nKey]; - } - - if( peType ) *peType = eType; - if( piPgPtr ) *piPgPtr = iPgPtr; - if( paKey ) *paKey = aKey; - if( paVal ) *paVal = aVal; - if( pnKey ) *pnKey = nKey; - if( pnVal ) *pnVal = nVal; -} - -static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){ - int iChar; - for(iChar=0; iCharpClient || pDb->pWorker ); - pSnap = pDb->pClient; - if( pSnap==0 ) pSnap = pDb->pWorker; - if( pSnap->redirect.n>0 ){ - Level *pLvl; - int bUse = 0; - for(pLvl=pSnap->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - pSeg = (pLvl->nRight==0 ? &pLvl->lhs : &pLvl->aRhs[pLvl->nRight-1]); - rc = lsmFsSegmentContainsPg(pDb->pFS, pSeg, iPg, &bUse); - if( bUse==0 ){ - pSeg = 0; - } - } - - /* iPg is a real page number (not subject to redirection). So it is safe - ** to pass a NULL in place of the segment pointer as the second argument - ** to lsmFsDbPageGet() here. */ - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); - } - - if( rc==LSM_OK ){ - LsmBlob blob = {0, 0, 0, 0}; - int nKeyWidth = 0; - LsmString str; - int nRec; - LsmPgno iPtr; - int flags2; - int iCell; - u8 *aData; int nData; /* Page data and size thereof */ - - aData = fsPageData(pPg, &nData); - nRec = pageGetNRec(aData, nData); - iPtr = pageGetPtr(aData, nData); - flags2 = pageGetFlags(aData, nData); - - lsmStringInit(&str, pDb->pEnv); - lsmStringAppendf(&str, "Page : %lld (%d bytes)\n", iPg, nData); - lsmStringAppendf(&str, "nRec : %d\n", nRec); - lsmStringAppendf(&str, "iPtr : %lld\n", iPtr); - lsmStringAppendf(&str, "flags: %04x\n", flags2); - lsmStringAppendf(&str, "\n"); - - for(iCell=0; iCellnKeyWidth ) nKeyWidth = nKey; - } - if( bHex ) nKeyWidth = nKeyWidth * 2; - - for(iCell=0; iCell0 && bValues ){ - lsmStringAppendf(&str, "%*s", nKeyWidth - (nKey*(1+bHex)), ""); - lsmStringAppendf(&str, " "); - infoAppendBlob(&str, bHex, aVal, nVal); - } - if( rtTopic(eType) ){ - int iBlk = (int)~lsmGetU32(aKey); - lsmStringAppendf(&str, " (block=%d", iBlk); - if( nVal>0 ){ - i64 iSnap = lsmGetU64(aVal); - lsmStringAppendf(&str, " snapshot=%lld", iSnap); - } - lsmStringAppendf(&str, ")"); - } - lsmStringAppendf(&str, "\n"); - } - - if( bData ){ - lsmStringAppendf(&str, "\n-------------------" - "-------------------------------------------------------------\n"); - lsmStringAppendf(&str, "Page %d\n", - iPg, (iPg-1)*nData, iPg*nData - 1); - for(i=0; inData ){ - lsmStringAppendf(&str, " "); - }else{ - lsmStringAppendf(&str, "%02x ", aData[i+j]); - } - } - lsmStringAppendf(&str, " "); - for(j=0; jnData ){ - lsmStringAppendf(&str, " "); - }else{ - lsmStringAppendf(&str,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); - } - } - lsmStringAppendf(&str,"\n"); - } - } - - *pzOut = str.z; - sortedBlobFree(&blob); - lsmFsPageRelease(pPg); - } - - return rc; -} - -int lsmInfoPageDump( - lsm_db *pDb, /* Database handle */ - LsmPgno iPg, /* Page number of page to dump */ - int bHex, /* True to output key/value in hex form */ - char **pzOut /* OUT: lsmMalloc'd string */ -){ - int flags = INFO_PAGE_DUMP_DATA | INFO_PAGE_DUMP_VALUES; - if( bHex ) flags |= INFO_PAGE_DUMP_HEX; - return infoPageDump(pDb, iPg, flags, pzOut); -} - -void sortedDumpSegment(lsm_db *pDb, Segment *pRun, int bVals){ - assert( pDb->xLog ); - if( pRun && pRun->iFirst ){ - int flags = (bVals ? INFO_PAGE_DUMP_VALUES : 0); - char *zSeg; - Page *pPg; - - zSeg = segToString(pDb->pEnv, pRun, 0); - lsmLogMessage(pDb, LSM_OK, "Segment: %s", zSeg); - lsmFree(pDb->pEnv, zSeg); - - lsmFsDbPageGet(pDb->pFS, pRun, pRun->iFirst, &pPg); - while( pPg ){ - Page *pNext; - char *z = 0; - infoPageDump(pDb, lsmFsPageNumber(pPg), flags, &z); - lsmLogMessage(pDb, LSM_OK, "%s", z); - lsmFree(pDb->pEnv, z); -#if 0 - sortedDumpPage(pDb, pRun, pPg, bVals); -#endif - lsmFsDbPageNext(pRun, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - } -} - -/* -** Invoke the log callback zero or more times with messages that describe -** the current database structure. -*/ -void lsmSortedDumpStructure( - lsm_db *pDb, /* Database handle (used for xLog callback) */ - Snapshot *pSnap, /* Snapshot to dump */ - int bKeys, /* Output the keys from each segment */ - int bVals, /* Output the values from each segment */ - const char *zWhy /* Caption to print near top of dump */ -){ - Snapshot *pDump = pSnap; - Level *pTopLevel; - char *zFree = 0; - - assert( pSnap ); - pTopLevel = lsmDbSnapshotLevel(pDump); - if( pDb->xLog && pTopLevel ){ - static int nCall = 0; - Level *pLevel; - int iLevel = 0; - - nCall++; - lsmLogMessage(pDb, LSM_OK, "Database structure %d (%s)", nCall, zWhy); - -#if 0 - if( nCall==1031 || nCall==1032 ) bKeys=1; -#endif - - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - char zLeft[1024]; - char zRight[1024]; - int i = 0; - - Segment *aLeft[24]; - Segment *aRight[24]; - - int nLeft = 0; - int nRight = 0; - - Segment *pSeg = &pLevel->lhs; - aLeft[nLeft++] = pSeg; - - for(i=0; inRight; i++){ - aRight[nRight++] = &pLevel->aRhs[i]; - } - -#ifdef LSM_LOG_FREELIST - if( nRight ){ - memmove(&aRight[1], aRight, sizeof(aRight[0])*nRight); - aRight[0] = 0; - nRight++; - } -#endif - - for(i=0; iiAge, (int)pLevel->flags - ); - }else{ - zLevel[0] = '\0'; - } - - if( nRight==0 ){ - iPad = 10; - } - - lsmLogMessage(pDb, LSM_OK, "% 25s % *s% -35s %s", - zLevel, iPad, "", zLeft, zRight - ); - } - - iLevel++; - } - - if( bKeys ){ - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - int i; - sortedDumpSegment(pDb, &pLevel->lhs, bVals); - for(i=0; inRight; i++){ - sortedDumpSegment(pDb, &pLevel->aRhs[i], bVals); - } - } - } - } - - lsmInfoFreelist(pDb, &zFree); - lsmLogMessage(pDb, LSM_OK, "Freelist: %s", zFree); - lsmFree(pDb->pEnv, zFree); - - assert( lsmFsIntegrityCheck(pDb) ); -} - -void lsmSortedFreeLevel(lsm_env *pEnv, Level *pLevel){ - Level *pNext; - Level *p; - - for(p=pLevel; p; p=pNext){ - pNext = p->pNext; - sortedFreeLevel(pEnv, p); - } -} - -void lsmSortedSaveTreeCursors(lsm_db *pDb){ - MultiCursor *pCsr; - for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ - lsmTreeCursorSave(pCsr->apTreeCsr[0]); - lsmTreeCursorSave(pCsr->apTreeCsr[1]); - } -} - -void lsmSortedExpandBtreePage(Page *pPg, int nOrig){ - u8 *aData; - int nData; - int nEntry; - int iHdr; - - aData = lsmFsPageData(pPg, &nData); - nEntry = pageGetNRec(aData, nOrig); - iHdr = SEGMENT_EOF(nOrig, nEntry); - memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr); -} - -#ifdef LSM_DEBUG_EXPENSIVE -static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){ - Page *pPg = 0; - LsmBlob blob1 = {0, 0, 0, 0}; - LsmBlob blob2 = {0, 0, 0, 0}; - - lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg); - while( pPg ){ - u8 *aData; int nData; - Page *pNext; - - aData = lsmFsPageData(pPg, &nData); - if( 0==(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ){ - int i; - int nRec = pageGetNRec(aData, nData); - for(i=0; ipEnv, pSeg, pPg, i, &iTopic1, &blob1); - - if( i==0 && blob2.nData ){ - assert( sortedKeyCompare( - pDb->xCmp, iTopic2, blob2.pData, blob2.nData, - iTopic1, blob1.pData, blob1.nData - )<0 ); - } - - if( i<(nRec-1) ){ - pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i+1, &iTopic2, &blob2); - assert( sortedKeyCompare( - pDb->xCmp, iTopic1, blob1.pData, blob1.nData, - iTopic2, blob2.pData, blob2.nData - )<0 ); - } - } - } - - lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - sortedBlobFree(&blob1); - sortedBlobFree(&blob2); -} -#endif - -#ifdef LSM_DEBUG_EXPENSIVE -/* -** This function is only included in the build if LSM_DEBUG_EXPENSIVE is -** defined. Its only purpose is to evaluate various assert() statements to -** verify that the database is well formed in certain respects. -** -** More specifically, it checks that the array pOne contains the required -** pointers to pTwo. Array pTwo must be a main array. pOne may be either a -** separators array or another main array. If pOne does not contain the -** correct set of pointers, an assert() statement fails. -*/ -static int assertPointersOk( - lsm_db *pDb, /* Database handle */ - Segment *pOne, /* Segment containing pointers */ - Segment *pTwo, /* Segment containing pointer targets */ - int bRhs /* True if pTwo may have been Gobble()d */ -){ - int rc = LSM_OK; /* Error code */ - SegmentPtr ptr1; /* Iterates through pOne */ - SegmentPtr ptr2; /* Iterates through pTwo */ - LsmPgno iPrev; - - assert( pOne && pTwo ); - - memset(&ptr1, 0, sizeof(ptr1)); - memset(&ptr2, 0, sizeof(ptr1)); - ptr1.pSeg = pOne; - ptr2.pSeg = pTwo; - segmentPtrEndPage(pDb->pFS, &ptr1, 0, &rc); - segmentPtrEndPage(pDb->pFS, &ptr2, 0, &rc); - - /* Check that the footer pointer of the first page of pOne points to - ** the first page of pTwo. */ - iPrev = pTwo->iFirst; - if( ptr1.iPtr!=iPrev && !bRhs ){ - assert( 0 ); - } - - if( rc==LSM_OK && ptr1.nCell>0 ){ - rc = segmentPtrLoadCell(&ptr1, 0); - } - - while( rc==LSM_OK && ptr2.pPg ){ - LsmPgno iThis; - - /* Advance to the next page of segment pTwo that contains at least - ** one cell. Break out of the loop if the iterator reaches EOF. */ - do{ - rc = segmentPtrNextPage(&ptr2, 1); - assert( rc==LSM_OK ); - }while( rc==LSM_OK && ptr2.pPg && ptr2.nCell==0 ); - if( rc!=LSM_OK || ptr2.pPg==0 ) break; - iThis = lsmFsPageNumber(ptr2.pPg); - - if( (ptr2.flags & (PGFTR_SKIP_THIS_FLAG|SEGMENT_BTREE_FLAG))==0 ){ - - /* Load the first cell in the array pTwo page. */ - rc = segmentPtrLoadCell(&ptr2, 0); - - /* Iterate forwards through pOne, searching for a key that matches the - ** key ptr2.pKey/nKey. This key should have a pointer to the page that - ** ptr2 currently points to. */ - while( rc==LSM_OK ){ - int res = rtTopic(ptr1.eType) - rtTopic(ptr2.eType); - if( res==0 ){ - res = pDb->xCmp(ptr1.pKey, ptr1.nKey, ptr2.pKey, ptr2.nKey); - } - - if( res<0 ){ - assert( bRhs || ptr1.iPtr+ptr1.iPgPtr==iPrev ); - }else if( res>0 ){ - assert( 0 ); - }else{ - assert( ptr1.iPtr+ptr1.iPgPtr==iThis ); - iPrev = iThis; - break; - } - - rc = segmentPtrAdvance(0, &ptr1, 0); - if( ptr1.pPg==0 ){ - assert( 0 ); - } - } - } - } - - segmentPtrReset(&ptr1, 0); - segmentPtrReset(&ptr2, 0); - return LSM_OK; -} - -/* -** This function is only included in the build if LSM_DEBUG_EXPENSIVE is -** defined. Its only purpose is to evaluate various assert() statements to -** verify that the database is well formed in certain respects. -** -** More specifically, it checks that the b-tree embedded in array pRun -** contains the correct keys. If not, an assert() fails. -*/ -static int assertBtreeOk( - lsm_db *pDb, - Segment *pSeg -){ - int rc = LSM_OK; /* Return code */ - if( pSeg->iRoot ){ - LsmBlob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */ - FileSystem *pFS = pDb->pFS; /* File system to read from */ - Page *pPg = 0; /* Main run page */ - BtreeCursor *pCsr = 0; /* Btree cursor */ - - rc = btreeCursorNew(pDb, pSeg, &pCsr); - if( rc==LSM_OK ){ - rc = btreeCursorFirst(pCsr); - } - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pPg); - } - - while( rc==LSM_OK ){ - Page *pNext; - u8 *aData; - int nData; - int flags; - - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - if( pPg==0 ) break; - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( rc==LSM_OK - && 0==((SEGMENT_BTREE_FLAG|PGFTR_SKIP_THIS_FLAG) & flags) - && 0!=pageGetNRec(aData, nData) - ){ - u8 *pKey; - int nKey; - int iTopic; - pKey = pageGetKey(pSeg, pPg, 0, &iTopic, &nKey, &blob); - assert( nKey==pCsr->nKey && 0==memcmp(pKey, pCsr->pKey, nKey) ); - assert( lsmFsPageNumber(pPg)==pCsr->iPtr ); - rc = btreeCursorNext(pCsr); - } - } - assert( rc!=LSM_OK || pCsr->pKey==0 ); - - if( pPg ) lsmFsPageRelease(pPg); - - btreeCursorFree(pCsr); - sortedBlobFree(&blob); - } - - return rc; -} -#endif /* ifdef LSM_DEBUG_EXPENSIVE */ diff --git a/ext/lsm1/lsm_str.c b/ext/lsm1/lsm_str.c deleted file mode 100644 index 9b1b63cee2..0000000000 --- a/ext/lsm1/lsm_str.c +++ /dev/null @@ -1,148 +0,0 @@ -/* -** 2012-04-27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Dynamic string functions. -*/ -#include "lsmInt.h" - -/* -** Turn bulk and uninitialized memory into an LsmString object -*/ -void lsmStringInit(LsmString *pStr, lsm_env *pEnv){ - memset(pStr, 0, sizeof(pStr[0])); - pStr->pEnv = pEnv; -} - -/* -** Increase the memory allocated for holding the string. Realloc as needed. -** -** If a memory allocation error occurs, set pStr->n to -1 and free the existing -** allocation. If a prior memory allocation has occurred, this routine is a -** no-op. -*/ -int lsmStringExtend(LsmString *pStr, int nNew){ - assert( nNew>0 ); - if( pStr->n<0 ) return LSM_NOMEM; - if( pStr->n + nNew >= pStr->nAlloc ){ - int nAlloc = pStr->n + nNew + 100; - char *zNew = lsmRealloc(pStr->pEnv, pStr->z, nAlloc); - if( zNew==0 ){ - lsmFree(pStr->pEnv, pStr->z); - nAlloc = 0; - pStr->n = -1; - } - pStr->nAlloc = nAlloc; - pStr->z = zNew; - } - return (pStr->z ? LSM_OK : LSM_NOMEM_BKPT); -} - -/* -** Clear an LsmString object, releasing any allocated memory that it holds. -** This also clears the error indication (if any). -*/ -void lsmStringClear(LsmString *pStr){ - lsmFree(pStr->pEnv, pStr->z); - lsmStringInit(pStr, pStr->pEnv); -} - -/* -** Append N bytes of text to the end of an LsmString object. If -** N is negative, append the entire string. -** -** If the string is in an error state, this routine is a no-op. -*/ -int lsmStringAppend(LsmString *pStr, const char *z, int N){ - int rc; - if( N<0 ) N = (int)strlen(z); - rc = lsmStringExtend(pStr, N+1); - if( pStr->nAlloc ){ - memcpy(pStr->z+pStr->n, z, N+1); - pStr->n += N; - } - return rc; -} - -int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n){ - int rc; - rc = lsmStringExtend(pStr, n); - if( pStr->nAlloc ){ - memcpy(pStr->z+pStr->n, a, n); - pStr->n += n; - } - return rc; -} - -/* -** Append printf-formatted content to an LsmString. -*/ -void lsmStringVAppendf( - LsmString *pStr, - const char *zFormat, - va_list ap1, - va_list ap2 -){ -#if (!defined(__STDC_VERSION__) || (__STDC_VERSION__<199901L)) && \ - !defined(__APPLE__) - extern int vsnprintf(char *str, size_t size, const char *format, va_list ap) - /* Compatibility crutch for C89 compilation mode. sqlite3_vsnprintf() - does not work identically and causes test failures if used here. - For the time being we are assuming that the target has vsnprintf(), - but that is not guaranteed to be the case for pure C89 platforms. - */; -#endif - int nWrite; - int nAvail; - - nAvail = pStr->nAlloc - pStr->n; - nWrite = vsnprintf(pStr->z + pStr->n, nAvail, zFormat, ap1); - - if( nWrite>=nAvail ){ - lsmStringExtend(pStr, nWrite+1); - if( pStr->nAlloc==0 ) return; - nWrite = vsnprintf(pStr->z + pStr->n, nWrite+1, zFormat, ap2); - } - - pStr->n += nWrite; - pStr->z[pStr->n] = 0; -} - -void lsmStringAppendf(LsmString *pStr, const char *zFormat, ...){ - va_list ap, ap2; - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(pStr, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); -} - -int lsmStrlen(const char *zName){ - int nRet = 0; - while( zName[nRet] ) nRet++; - return nRet; -} - -/* -** Write into memory obtained from lsm_malloc(). -*/ -char *lsmMallocPrintf(lsm_env *pEnv, const char *zFormat, ...){ - LsmString s; - va_list ap, ap2; - lsmStringInit(&s, pEnv); - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(&s, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); - if( s.n<0 ) return 0; - return (char *)lsmReallocOrFree(pEnv, s.z, s.n+1); -} diff --git a/ext/lsm1/lsm_tree.c b/ext/lsm1/lsm_tree.c deleted file mode 100644 index 1a199fc1ce..0000000000 --- a/ext/lsm1/lsm_tree.c +++ /dev/null @@ -1,2465 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of an in-memory tree structure. -** -** Technically the tree is a B-tree of order 4 (in the Knuth sense - each -** node may have up to 4 children). Keys are stored within B-tree nodes by -** reference. This may be slightly slower than a conventional red-black -** tree, but it is simpler. It is also an easier structure to modify to -** create a version that supports nested transaction rollback. -** -** This tree does not currently support a delete operation. One is not -** required. When LSM deletes a key from a database, it inserts a DELETE -** marker into the data structure. As a result, although the value associated -** with a key stored in the in-memory tree structure may be modified, no -** keys are ever removed. -*/ - -/* -** MVCC NOTES -** -** The in-memory tree structure supports SQLite-style MVCC. This means -** that while one client is writing to the tree structure, other clients -** may still be querying an older snapshot of the tree. -** -** One way to implement this is to use an append-only b-tree. In this -** case instead of modifying nodes in-place, a copy of the node is made -** and the required modifications made to the copy. The parent of the -** node is then modified (to update the pointer so that it points to -** the new copy), which causes a copy of the parent to be made, and so on. -** This means that each time the tree is written to a new root node is -** created. A snapshot is identified by the root node that it uses. -** -** The problem with the above is that each time the tree is written to, -** a copy of the node structure modified and all of its ancestor nodes -** is made. This may prove excessive with large tree structures. -** -** To reduce this overhead, the data structure used for a tree node is -** designed so that it may be edited in place exactly once without -** affecting existing users. In other words, the node structure is capable -** of storing two separate versions of the node at the same time. -** When a node is to be edited, if the node structure already contains -** two versions, a copy is made as in the append-only approach. Or, if -** it only contains a single version, it is edited in place. -** -** This reduces the overhead so that, roughly, one new node structure -** must be allocated for each write (on top of those allocations that -** would have been required by a non-MVCC tree). Logic: Assume that at -** any time, 50% of nodes in the tree already contain 2 versions. When -** a new entry is written to a node, there is a 50% chance that a copy -** of the node will be required. And a 25% chance that a copy of its -** parent is required. And so on. -** -** ROLLBACK -** -** The in-memory tree also supports transaction and sub-transaction -** rollback. In order to rollback to point in time X, the following is -** necessary: -** -** 1. All memory allocated since X must be freed, and -** 2. All "v2" data adding to nodes that existed at X should be zeroed. -** 3. The root node must be restored to its X value. -** -** The Mempool object used to allocate memory for the tree supports -** operation (1) - see the lsmPoolMark() and lsmPoolRevert() functions. -** -** To support (2), all nodes that have v2 data are part of a singly linked -** list, sorted by the age of the v2 data (nodes that have had data added -** most recently are at the end of the list). So to zero all v2 data added -** since X, the linked list is traversed from the first node added following -** X onwards. -** -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -#include - -#define MAX_DEPTH 32 - -typedef struct TreeKey TreeKey; -typedef struct TreeNode TreeNode; -typedef struct TreeLeaf TreeLeaf; -typedef struct NodeVersion NodeVersion; - -struct TreeOld { - u32 iShmid; /* Last shared-memory chunk in use by old */ - u32 iRoot; /* Offset of root node in shm file */ - u32 nHeight; /* Height of tree structure */ -}; - -#if 0 -/* -** assert() that a TreeKey.flags value is sane. Usage: -** -** assert( lsmAssertFlagsOk(pTreeKey->flags) ); -*/ -static int lsmAssertFlagsOk(u8 keyflags){ - /* At least one flag must be set. Otherwise, what is this key doing? */ - assert( keyflags!=0 ); - - /* The POINT_DELETE and INSERT flags cannot both be set. */ - assert( (keyflags & LSM_POINT_DELETE)==0 || (keyflags & LSM_INSERT)==0 ); - - /* If both the START_DELETE and END_DELETE flags are set, then the INSERT - ** flag must also be set. In other words - the three DELETE flags cannot - ** all be set */ - assert( (keyflags & LSM_END_DELETE)==0 - || (keyflags & LSM_START_DELETE)==0 - || (keyflags & LSM_POINT_DELETE)==0 - ); - - return 1; -} -#endif -static int assert_delete_ranges_match(lsm_db *); -static int treeCountEntries(lsm_db *db); - -/* -** Container for a key-value pair. Within the *-shm file, each key/value -** pair is stored in a single allocation (which may not actually be -** contiguous in memory). Layout is the TreeKey structure, followed by -** the nKey bytes of key blob, followed by the nValue bytes of value blob -** (if nValue is non-negative). -*/ -struct TreeKey { - int nKey; /* Size of pKey in bytes */ - int nValue; /* Size of pValue. Or negative. */ - u8 flags; /* Various LSM_XXX flags */ -}; - -#define TKV_KEY(p) ((void *)&(p)[1]) -#define TKV_VAL(p) ((void *)(((u8 *)&(p)[1]) + (p)->nKey)) - -/* -** A single tree node. A node structure may contain up to 3 key/value -** pairs. Internal (non-leaf) nodes have up to 4 children. -** -** TODO: Update the format of this to be more compact. Get it working -** first though... -*/ -struct TreeNode { - u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ - - /* The following fields are present for interior nodes only, not leaves. */ - u32 aiChildPtr[4]; /* Array of pointers to child nodes */ - - /* The extra child pointer slot. */ - u32 iV2; /* Transaction number of v2 */ - u8 iV2Child; /* apChild[] entry replaced by pV2Ptr */ - u32 iV2Ptr; /* Substitute pointer */ -}; - -struct TreeLeaf { - u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ -}; - -typedef struct TreeBlob TreeBlob; -struct TreeBlob { - int n; - u8 *a; -}; - -/* -** Cursor for searching a tree structure. -** -** If a cursor does not point to any element (a.k.a. EOF), then the -** TreeCursor.iNode variable is set to a negative value. Otherwise, the -** cursor currently points to key aiCell[iNode] on node apTreeNode[iNode]. -** -** Entries in the apTreeNode[] and aiCell[] arrays contain the node and -** index of the TreeNode.apChild[] pointer followed to descend to the -** current element. Hence apTreeNode[0] always contains the root node of -** the tree. -*/ -struct TreeCursor { - lsm_db *pDb; /* Database handle for this cursor */ - TreeRoot *pRoot; /* Root node and height of tree to access */ - int iNode; /* Cursor points at apTreeNode[iNode] */ - TreeNode *apTreeNode[MAX_DEPTH];/* Current position in tree */ - u8 aiCell[MAX_DEPTH]; /* Current position in tree */ - TreeKey *pSave; /* Saved key */ - TreeBlob blob; /* Dynamic storage for a key */ -}; - -/* -** A value guaranteed to be larger than the largest possible transaction -** id (TreeHeader.iTransId). -*/ -#define WORKING_VERSION (1<<30) - -static int tblobGrow(lsm_db *pDb, TreeBlob *p, int n, int *pRc){ - if( n>p->n ){ - lsmFree(pDb->pEnv, p->a); - p->a = lsmMallocRc(pDb->pEnv, n, pRc); - p->n = n; - } - return (p->a==0); -} -static void tblobFree(lsm_db *pDb, TreeBlob *p){ - lsmFree(pDb->pEnv, p->a); -} - - -/*********************************************************************** -** Start of IntArray methods. */ -/* -** Append value iVal to the contents of IntArray *p. Return LSM_OK if -** successful, or LSM_NOMEM if an OOM condition is encountered. -*/ -static int intArrayAppend(lsm_env *pEnv, IntArray *p, u32 iVal){ - assert( p->nArray<=p->nAlloc ); - if( p->nArray>=p->nAlloc ){ - u32 *aNew; - int nNew = p->nArray ? p->nArray*2 : 128; - aNew = lsmRealloc(pEnv, p->aArray, nNew*sizeof(u32)); - if( !aNew ) return LSM_NOMEM_BKPT; - p->aArray = aNew; - p->nAlloc = nNew; - } - - p->aArray[p->nArray++] = iVal; - return LSM_OK; -} - -/* -** Zero the IntArray object. -*/ -static void intArrayFree(lsm_env *pEnv, IntArray *p){ - p->nArray = 0; -} - -/* -** Return the number of entries currently in the int-array object. -*/ -static int intArraySize(IntArray *p){ - return p->nArray; -} - -/* -** Return a copy of the iIdx'th entry in the int-array. -*/ -static u32 intArrayEntry(IntArray *p, int iIdx){ - return p->aArray[iIdx]; -} - -/* -** Truncate the int-array so that all but the first nVal values are -** discarded. -*/ -static void intArrayTruncate(IntArray *p, int nVal){ - p->nArray = nVal; -} -/* End of IntArray methods. -***********************************************************************/ - -static int treeKeycmp(void *p1, int n1, void *p2, int n2){ - int res; - res = memcmp(p1, p2, LSM_MIN(n1, n2)); - if( res==0 ) res = (n1-n2); - return res; -} - -/* -** The pointer passed as the first argument points to an interior node, -** not a leaf. This function returns the offset of the iCell'th child -** sub-tree of the node. -*/ -static u32 getChildPtr(TreeNode *p, int iVersion, int iCell){ - assert( iVersion>=0 ); - assert( iCell>=0 && iCell<=array_size(p->aiChildPtr) ); - if( p->iV2 && p->iV2<=(u32)iVersion && iCell==p->iV2Child ) return p->iV2Ptr; - return p->aiChildPtr[iCell]; -} - -/* -** Given an offset within the *-shm file, return the associated chunk number. -*/ -static int treeOffsetToChunk(u32 iOff){ - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - return (int)(iOff>>15); -} - -#define treeShmptrUnsafe(pDb, iPtr) \ -(&((u8*)((pDb)->apShm[(iPtr)>>15]))[(iPtr) & (LSM_SHM_CHUNK_SIZE-1)]) - -/* -** Return a pointer to the mapped memory location associated with *-shm -** file offset iPtr. -*/ -static void *treeShmptr(lsm_db *pDb, u32 iPtr){ - - assert( (iPtr>>15)<(u32)pDb->nShm ); - assert( pDb->apShm[iPtr>>15] ); - - return iPtr ? treeShmptrUnsafe(pDb, iPtr) : 0; -} - -static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ - return (ShmChunk *)(pDb->apShm[iChunk]); -} - -static ShmChunk * treeShmChunkRc(lsm_db *pDb, int iChunk, int *pRc){ - assert( *pRc==LSM_OK ); - if( iChunknShm || LSM_OK==(*pRc = lsmShmCacheChunks(pDb, iChunk+1)) ){ - return (ShmChunk *)(pDb->apShm[iChunk]); - } - return 0; -} - - -#ifndef NDEBUG -static void assertIsWorkingChild( - lsm_db *db, - TreeNode *pNode, - TreeNode *pParent, - int iCell -){ - TreeNode *p; - u32 iPtr = getChildPtr(pParent, WORKING_VERSION, iCell); - p = treeShmptr(db, iPtr); - assert( p==pNode ); -} -#else -# define assertIsWorkingChild(w,x,y,z) -#endif - -/* Values for the third argument to treeShmkey(). */ -#define TKV_LOADKEY 1 -#define TKV_LOADVAL 2 - -static TreeKey *treeShmkey( - lsm_db *pDb, /* Database handle */ - u32 iPtr, /* Shmptr to TreeKey struct */ - int eLoad, /* Either zero or a TREEKEY_LOADXXX value */ - TreeBlob *pBlob, /* Used if dynamic memory is required */ - int *pRc /* IN/OUT: Error code */ -){ - TreeKey *pRet; - - assert( eLoad==TKV_LOADKEY || eLoad==TKV_LOADVAL ); - pRet = (TreeKey *)treeShmptr(pDb, iPtr); - if( pRet ){ - int nReq; /* Bytes of space required at pRet */ - int nAvail; /* Bytes of space available at pRet */ - - nReq = sizeof(TreeKey) + pRet->nKey; - if( eLoad==TKV_LOADVAL && pRet->nValue>0 ){ - nReq += pRet->nValue; - } - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - nAvail = LSM_SHM_CHUNK_SIZE - (iPtr & (LSM_SHM_CHUNK_SIZE-1)); - - if( nAvaila[nLoad], p, n); - nLoad += n; - if( nLoad==nReq ) break; - - pChunk = treeShmChunk(pDb, treeOffsetToChunk(iPtr)); - assert( pChunk ); - iPtr = (pChunk->iNext * LSM_SHM_CHUNK_SIZE) + LSM_SHM_CHUNK_HDR; - nAvail = LSM_SHM_CHUNK_SIZE - LSM_SHM_CHUNK_HDR; - } - } - pRet = (TreeKey *)(pBlob->a); - } - } - - return pRet; -} - -#if defined(LSM_DEBUG) && defined(LSM_EXPENSIVE_ASSERT) -void assert_leaf_looks_ok(TreeNode *pNode){ - assert( pNode->apKey[1] ); -} - -void assert_node_looks_ok(TreeNode *pNode, int nHeight){ - if( pNode ){ - assert( pNode->apKey[1] ); - if( nHeight>1 ){ - int i; - assert( getChildPtr(pNode, WORKING_VERSION, 1) ); - assert( getChildPtr(pNode, WORKING_VERSION, 2) ); - for(i=0; i<4; i++){ - assert_node_looks_ok(getChildPtr(pNode, WORKING_VERSION, i), nHeight-1); - } - } - } -} - -/* -** Run various assert() statements to check that the working-version of the -** tree is correct in the following respects: -** -** * todo... -*/ -void assert_tree_looks_ok(int rc, Tree *pTree){ -} -#else -# define assert_tree_looks_ok(x,y) -#endif - -void lsmFlagsToString(int flags, char *zFlags){ - - zFlags[0] = (flags & LSM_END_DELETE) ? ']' : '.'; - - /* Only one of LSM_POINT_DELETE, LSM_INSERT and LSM_SEPARATOR should ever - ** be set. If this is not true, write a '?' to the output. */ - switch( flags & (LSM_POINT_DELETE|LSM_INSERT|LSM_SEPARATOR) ){ - case 0: zFlags[1] = '.'; break; - case LSM_POINT_DELETE: zFlags[1] = '-'; break; - case LSM_INSERT: zFlags[1] = '+'; break; - case LSM_SEPARATOR: zFlags[1] = '^'; break; - default: zFlags[1] = '?'; break; - } - - zFlags[2] = (flags & LSM_SYSTEMKEY) ? '*' : '.'; - zFlags[3] = (flags & LSM_START_DELETE) ? '[' : '.'; - zFlags[4] = '\0'; -} - -#ifdef LSM_DEBUG - -/* -** Pointer pBlob points to a buffer containing a blob of binary data -** nBlob bytes long. Append the contents of this blob to *pStr, with -** each octet represented by a 2-digit hexadecimal number. For example, -** if the input blob is three bytes in size and contains {0x01, 0x44, 0xFF}, -** then "0144ff" is appended to *pStr. -*/ -static void lsmAppendStrBlob(LsmString *pStr, void *pBlob, int nBlob){ - int i; - lsmStringExtend(pStr, nBlob*2); - if( pStr->nAlloc==0 ) return; - for(i=0; i='a' && c<='z' ){ - pStr->z[pStr->n++] = c; - }else if( c!=0 || nBlob==1 || i!=(nBlob-1) ){ - pStr->z[pStr->n++] = "0123456789abcdef"[(c>>4)&0xf]; - pStr->z[pStr->n++] = "0123456789abcdef"[c&0xf]; - } - } - pStr->z[pStr->n] = 0; -} - -#if 0 /* NOT USED */ -/* -** Append nIndent space (0x20) characters to string *pStr. -*/ -static void lsmAppendIndent(LsmString *pStr, int nIndent){ - int i; - lsmStringExtend(pStr, nIndent); - for(i=0; ipEnv); - - /* Append each key to string s. */ - for(i=0; i<3; i++){ - u32 iPtr = pNode->aiKeyPtr[i]; - if( iPtr ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i],TKV_LOADKEY, &b,&rc); - strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); - lsmStringAppend(&s, " ", -1); - } - } - - printf("% 6d %.*sleaf%.*s: %s\n", - iNode, nPath, zPath, 20-nPath-4, zSpace, s.z - ); - lsmStringClear(&s); - }else{ - for(i=0; i<4 && nHeight>0; i++){ - u32 iPtr = getChildPtr(pNode, pDb->treehdr.root.iTransId, i); - zPath[nPath] = (char)(i+'0'); - zPath[nPath+1] = '/'; - - if( iPtr ){ - dump_node_contents(pDb, iPtr, zPath, nPath+2, nHeight-1); - } - if( i!=3 && pNode->aiKeyPtr[i] ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TKV_LOADKEY,&b,&rc); - lsmStringInit(&s, pDb->pEnv); - strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); - printf("% 6d %.*s%.*s: %s\n", - iNode, nPath+1, zPath, 20-nPath-1, zSpace, s.z); - lsmStringClear(&s); - } - } - } - - tblobFree(pDb, &b); -} - -void dump_tree_contents(lsm_db *pDb, const char *zCaption){ - char zPath[64]; - TreeRoot *p = &pDb->treehdr.root; - printf("\n%s\n", zCaption); - zPath[0] = '/'; - if( p->iRoot ){ - dump_node_contents(pDb, p->iRoot, zPath, 1, p->nHeight-1); - } - fflush(stdout); -} - -#endif - -/* -** Initialize a cursor object, the space for which has already been -** allocated. -*/ -static void treeCursorInit(lsm_db *pDb, int bOld, TreeCursor *pCsr){ - memset(pCsr, 0, sizeof(TreeCursor)); - pCsr->pDb = pDb; - if( bOld ){ - pCsr->pRoot = &pDb->treehdr.oldroot; - }else{ - pCsr->pRoot = &pDb->treehdr.root; - } - pCsr->iNode = -1; -} - -/* -** Return a pointer to the mapping of the TreeKey object that the cursor -** is pointing to. -*/ -static TreeKey *csrGetKey(TreeCursor *pCsr, TreeBlob *pBlob, int *pRc){ - TreeKey *pRet; - lsm_db *pDb = pCsr->pDb; - u32 iPtr = pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]]; - - assert( iPtr ); - pRet = (TreeKey*)treeShmptrUnsafe(pDb, iPtr); - if( !(pRet->flags & LSM_CONTIGUOUS) ){ - pRet = treeShmkey(pDb, iPtr, TKV_LOADVAL, pBlob, pRc); - } - - return pRet; -} - -/* -** Save the current position of tree cursor pCsr. -*/ -int lsmTreeCursorSave(TreeCursor *pCsr){ - int rc = LSM_OK; - if( pCsr && pCsr->pSave==0 ){ - int iNode = pCsr->iNode; - if( iNode>=0 ){ - pCsr->pSave = csrGetKey(pCsr, &pCsr->blob, &rc); - } - pCsr->iNode = -1; - } - return rc; -} - -/* -** Restore the position of a saved tree cursor. -*/ -static int treeCursorRestore(TreeCursor *pCsr, int *pRes){ - int rc = LSM_OK; - if( pCsr->pSave ){ - TreeKey *pKey = pCsr->pSave; - pCsr->pSave = 0; - if( pRes ){ - rc = lsmTreeCursorSeek(pCsr, TKV_KEY(pKey), pKey->nKey, pRes); - } - } - return rc; -} - -/* -** Allocate nByte bytes of space within the *-shm file. If successful, -** return LSM_OK and set *piPtr to the offset within the file at which -** the allocated space is located. -*/ -static u32 treeShmalloc(lsm_db *pDb, int bAlign, int nByte, int *pRc){ - u32 iRet = 0; - if( *pRc==LSM_OK ){ - const static int CHUNK_SIZE = LSM_SHM_CHUNK_SIZE; - const static int CHUNK_HDR = LSM_SHM_CHUNK_HDR; - u32 iWrite; /* Current write offset */ - u32 iEof; /* End of current chunk */ - int iChunk; /* Current chunk */ - - assert( nByte <= (CHUNK_SIZE-CHUNK_HDR) ); - - /* Check if there is enough space on the current chunk to fit the - ** new allocation. If not, link in a new chunk and put the new - ** allocation at the start of it. */ - iWrite = pDb->treehdr.iWrite; - if( bAlign ){ - iWrite = (iWrite + 3) & ~0x0003; - assert( (iWrite % 4)==0 ); - } - - assert( iWrite ); - iChunk = treeOffsetToChunk(iWrite-1); - iEof = (iChunk+1) * CHUNK_SIZE; - assert( iEof>=iWrite && (iEof-iWrite)<(u32)CHUNK_SIZE ); - if( (iWrite+nByte)>iEof ){ - ShmChunk *pHdr; /* Header of chunk just finished (iChunk) */ - ShmChunk *pFirst; /* Header of chunk treehdr.iFirst */ - ShmChunk *pNext; /* Header of new chunk */ - int iNext = 0; /* Next chunk */ - int rc = LSM_OK; - - pFirst = treeShmChunk(pDb, pDb->treehdr.iFirst); - - assert( shm_sequence_ge(pDb->treehdr.iUsedShmid, pFirst->iShmid) ); - assert( (pDb->treehdr.iNextShmid+1-pDb->treehdr.nChunk)==pFirst->iShmid ); - - /* Check if the chunk at the start of the linked list is still in - ** use. If not, reuse it. If so, allocate a new chunk by appending - ** to the *-shm file. */ - if( pDb->treehdr.iUsedShmid!=pFirst->iShmid ){ - int bInUse; - rc = lsmTreeInUse(pDb, pFirst->iShmid, &bInUse); - if( rc!=LSM_OK ){ - *pRc = rc; - return 0; - } - if( bInUse==0 ){ - iNext = pDb->treehdr.iFirst; - pDb->treehdr.iFirst = pFirst->iNext; - assert( pDb->treehdr.iFirst ); - } - } - if( iNext==0 ) iNext = pDb->treehdr.nChunk++; - - /* Set the header values for the new chunk */ - pNext = treeShmChunkRc(pDb, iNext, &rc); - if( pNext ){ - pNext->iNext = 0; - pNext->iShmid = (pDb->treehdr.iNextShmid++); - }else{ - *pRc = rc; - return 0; - } - - /* Set the header values for the chunk just finished */ - pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE); - pHdr->iNext = iNext; - - /* Advance to the next chunk */ - iWrite = iNext * CHUNK_SIZE + CHUNK_HDR; - } - - /* Allocate space at iWrite. */ - iRet = iWrite; - pDb->treehdr.iWrite = iWrite + nByte; - pDb->treehdr.root.nByte += nByte; - } - return iRet; -} - -/* -** Allocate and zero nByte bytes of space within the *-shm file. -*/ -static void *treeShmallocZero(lsm_db *pDb, int nByte, u32 *piPtr, int *pRc){ - u32 iPtr; - void *p; - iPtr = treeShmalloc(pDb, 1, nByte, pRc); - p = treeShmptr(pDb, iPtr); - if( p ){ - assert( *pRc==LSM_OK ); - memset(p, 0, nByte); - *piPtr = iPtr; - } - return p; -} - -static TreeNode *newTreeNode(lsm_db *pDb, u32 *piPtr, int *pRc){ - return treeShmallocZero(pDb, sizeof(TreeNode), piPtr, pRc); -} - -static TreeLeaf *newTreeLeaf(lsm_db *pDb, u32 *piPtr, int *pRc){ - return treeShmallocZero(pDb, sizeof(TreeLeaf), piPtr, pRc); -} - -static TreeKey *newTreeKey( - lsm_db *pDb, - u32 *piPtr, - void *pKey, int nKey, /* Key data */ - void *pVal, int nVal, /* Value data (or nVal<0 for delete) */ - int *pRc -){ - TreeKey *p; - u32 iPtr; - u32 iEnd; - int nRem; - u8 *a; - int n; - - /* Allocate space for the TreeKey structure itself */ - *piPtr = iPtr = treeShmalloc(pDb, 1, sizeof(TreeKey), pRc); - p = treeShmptr(pDb, iPtr); - if( *pRc ) return 0; - p->nKey = nKey; - p->nValue = nVal; - - /* Allocate and populate the space required for the key and value. */ - n = nRem = nKey; - a = (u8 *)pKey; - while( a ){ - while( nRem>0 ){ - u8 *aAlloc; - int nAlloc; - u32 iWrite; - - iWrite = (pDb->treehdr.iWrite & (LSM_SHM_CHUNK_SIZE-1)); - iWrite = LSM_MAX(iWrite, LSM_SHM_CHUNK_HDR); - nAlloc = LSM_MIN((LSM_SHM_CHUNK_SIZE-iWrite), (u32)nRem); - - aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc)); - if( aAlloc==0 ) break; - memcpy(aAlloc, &a[n-nRem], nAlloc); - nRem -= nAlloc; - } - a = pVal; - n = nRem = nVal; - pVal = 0; - } - - iEnd = iPtr + sizeof(TreeKey) + nKey + LSM_MAX(0, nVal); - if( (iPtr & ~(LSM_SHM_CHUNK_SIZE-1))!=(iEnd & ~(LSM_SHM_CHUNK_SIZE-1)) ){ - p->flags = 0; - }else{ - p->flags = LSM_CONTIGUOUS; - } - - if( *pRc ) return 0; -#if 0 - printf("store: %d %s\n", (int)iPtr, (char *)pKey); -#endif - return p; -} - -static TreeNode *copyTreeNode( - lsm_db *pDb, - TreeNode *pOld, - u32 *piNew, - int *pRc -){ - TreeNode *pNew; - - pNew = newTreeNode(pDb, piNew, pRc); - if( pNew ){ - memcpy(pNew->aiKeyPtr, pOld->aiKeyPtr, sizeof(pNew->aiKeyPtr)); - memcpy(pNew->aiChildPtr, pOld->aiChildPtr, sizeof(pNew->aiChildPtr)); - if( pOld->iV2 ) pNew->aiChildPtr[pOld->iV2Child] = pOld->iV2Ptr; - } - return pNew; -} - -static TreeNode *copyTreeLeaf( - lsm_db *pDb, - TreeLeaf *pOld, - u32 *piNew, - int *pRc -){ - TreeLeaf *pNew; - pNew = newTreeLeaf(pDb, piNew, pRc); - if( pNew ){ - memcpy(pNew, pOld, sizeof(TreeLeaf)); - } - return (TreeNode *)pNew; -} - -/* -** The tree cursor passed as the second argument currently points to an -** internal node (not a leaf). Specifically, to a sub-tree pointer. This -** function replaces the sub-tree that the cursor currently points to -** with sub-tree pNew. -** -** The sub-tree may be replaced either by writing the "v2 data" on the -** internal node, or by allocating a new TreeNode structure and then -** calling this function on the parent of the internal node. -*/ -static int treeUpdatePtr(lsm_db *pDb, TreeCursor *pCsr, u32 iNew){ - int rc = LSM_OK; - if( pCsr->iNode<0 ){ - /* iNew is the new root node */ - pDb->treehdr.root.iRoot = iNew; - }else{ - /* If this node already has version 2 content, allocate a copy and - ** update the copy with the new pointer value. Otherwise, store the - ** new pointer as v2 data within the current node structure. */ - - TreeNode *p; /* The node to be modified */ - int iChildPtr; /* apChild[] entry to modify */ - - p = pCsr->apTreeNode[pCsr->iNode]; - iChildPtr = pCsr->aiCell[pCsr->iNode]; - - if( p->iV2 ){ - /* The "allocate new TreeNode" option */ - u32 iCopy; - TreeNode *pCopy; - pCopy = copyTreeNode(pDb, p, &iCopy, &rc); - if( pCopy ){ - assert( rc==LSM_OK ); - pCopy->aiChildPtr[iChildPtr] = iNew; - pCsr->iNode--; - rc = treeUpdatePtr(pDb, pCsr, iCopy); - } - }else{ - /* The "v2 data" option */ - u32 iPtr; - assert( pDb->treehdr.root.iTransId>0 ); - - if( pCsr->iNode ){ - iPtr = getChildPtr( - pCsr->apTreeNode[pCsr->iNode-1], - pDb->treehdr.root.iTransId, pCsr->aiCell[pCsr->iNode-1] - ); - }else{ - iPtr = pDb->treehdr.root.iRoot; - } - rc = intArrayAppend(pDb->pEnv, &pDb->rollback, iPtr); - - if( rc==LSM_OK ){ - p->iV2 = pDb->treehdr.root.iTransId; - p->iV2Child = (u8)iChildPtr; - p->iV2Ptr = iNew; - } - } - } - - return rc; -} - -/* -** Cursor pCsr points at a node that is part of pTree. This function -** inserts a new key and optionally child node pointer into that node. -** -** The position into which the new key and pointer are inserted is -** determined by the iSlot parameter. The new key will be inserted to -** the left of the key currently stored in apKey[iSlot]. Or, if iSlot is -** greater than the index of the rightmost key in the node. -** -** Pointer pLeftPtr points to a child tree that contains keys that are -** smaller than pTreeKey. -*/ -static int treeInsert( - lsm_db *pDb, /* Database handle */ - TreeCursor *pCsr, /* Cursor indicating path to insert at */ - u32 iLeftPtr, /* Left child pointer */ - u32 iTreeKey, /* Location of key to insert */ - u32 iRightPtr, /* Right child pointer */ - int iSlot /* Position to insert key into */ -){ - int rc = LSM_OK; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - - /* Check if the node is currently full. If so, split pNode in two and - ** call this function recursively to add a key to the parent. Otherwise, - ** insert the new key directly into pNode. */ - assert( pNode->aiKeyPtr[1] ); - if( pNode->aiKeyPtr[0] && pNode->aiKeyPtr[2] ){ - u32 iLeft; TreeNode *pLeft; /* New left-hand sibling node */ - u32 iRight; TreeNode *pRight; /* New right-hand sibling node */ - - pLeft = newTreeNode(pDb, &iLeft, &rc); - pRight = newTreeNode(pDb, &iRight, &rc); - if( rc ) return rc; - - pLeft->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 0); - pLeft->aiKeyPtr[1] = pNode->aiKeyPtr[0]; - pLeft->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 1); - - pRight->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 2); - pRight->aiKeyPtr[1] = pNode->aiKeyPtr[2]; - pRight->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 3); - - if( pCsr->iNode==0 ){ - /* pNode is the root of the tree. Grow the tree by one level. */ - u32 iRoot; TreeNode *pRoot; /* New root node */ - - pRoot = newTreeNode(pDb, &iRoot, &rc); - pRoot->aiKeyPtr[1] = pNode->aiKeyPtr[1]; - pRoot->aiChildPtr[1] = iLeft; - pRoot->aiChildPtr[2] = iRight; - - pDb->treehdr.root.iRoot = iRoot; - pDb->treehdr.root.nHeight++; - }else{ - - pCsr->iNode--; - rc = treeInsert(pDb, pCsr, - iLeft, pNode->aiKeyPtr[1], iRight, pCsr->aiCell[pCsr->iNode] - ); - } - - assert( pLeft->iV2==0 ); - assert( pRight->iV2==0 ); - switch( iSlot ){ - case 0: - pLeft->aiKeyPtr[0] = iTreeKey; - pLeft->aiChildPtr[0] = iLeftPtr; - if( iRightPtr ) pLeft->aiChildPtr[1] = iRightPtr; - break; - case 1: - pLeft->aiChildPtr[3] = (iRightPtr ? iRightPtr : pLeft->aiChildPtr[2]); - pLeft->aiKeyPtr[2] = iTreeKey; - pLeft->aiChildPtr[2] = iLeftPtr; - break; - case 2: - pRight->aiKeyPtr[0] = iTreeKey; - pRight->aiChildPtr[0] = iLeftPtr; - if( iRightPtr ) pRight->aiChildPtr[1] = iRightPtr; - break; - case 3: - pRight->aiChildPtr[3] = (iRightPtr ? iRightPtr : pRight->aiChildPtr[2]); - pRight->aiKeyPtr[2] = iTreeKey; - pRight->aiChildPtr[2] = iLeftPtr; - break; - } - - }else{ - TreeNode *pNew; - u32 *piKey; - u32 *piChild; - u32 iStore = 0; - u32 iNew = 0; - int i; - - /* Allocate a new version of node pNode. */ - pNew = newTreeNode(pDb, &iNew, &rc); - if( rc ) return rc; - - piKey = pNew->aiKeyPtr; - piChild = pNew->aiChildPtr; - - for(i=0; iaiKeyPtr[i] ){ - *(piKey++) = pNode->aiKeyPtr[i]; - *(piChild++) = getChildPtr(pNode, WORKING_VERSION, i); - } - } - - *piKey++ = iTreeKey; - *piChild++ = iLeftPtr; - - iStore = iRightPtr; - for(i=iSlot; i<3; i++){ - if( pNode->aiKeyPtr[i] ){ - *(piKey++) = pNode->aiKeyPtr[i]; - *(piChild++) = iStore ? iStore : getChildPtr(pNode, WORKING_VERSION, i); - iStore = 0; - } - } - - if( iStore ){ - *piChild = iStore; - }else{ - *piChild = getChildPtr(pNode, WORKING_VERSION, - (pNode->aiKeyPtr[2] ? 3 : 2) - ); - } - pCsr->iNode--; - rc = treeUpdatePtr(pDb, pCsr, iNew); - } - - return rc; -} - -static int treeInsertLeaf( - lsm_db *pDb, /* Database handle */ - TreeCursor *pCsr, /* Cursor structure */ - u32 iTreeKey, /* Key pointer to insert */ - int iSlot /* Insert key to the left of this */ -){ - int rc = LSM_OK; /* Return code */ - TreeNode *pLeaf = pCsr->apTreeNode[pCsr->iNode]; - TreeLeaf *pNew; - u32 iNew; - - assert( iSlot>=0 && iSlot<=4 ); - assert( pCsr->iNode>0 ); - assert( pLeaf->aiKeyPtr[1] ); - - pCsr->iNode--; - - pNew = newTreeLeaf(pDb, &iNew, &rc); - if( pNew ){ - if( pLeaf->aiKeyPtr[0] && pLeaf->aiKeyPtr[2] ){ - /* The leaf is full. Split it in two. */ - TreeLeaf *pRight; - u32 iRight; - pRight = newTreeLeaf(pDb, &iRight, &rc); - if( pRight ){ - assert( rc==LSM_OK ); - pNew->aiKeyPtr[1] = pLeaf->aiKeyPtr[0]; - pRight->aiKeyPtr[1] = pLeaf->aiKeyPtr[2]; - switch( iSlot ){ - case 0: pNew->aiKeyPtr[0] = iTreeKey; break; - case 1: pNew->aiKeyPtr[2] = iTreeKey; break; - case 2: pRight->aiKeyPtr[0] = iTreeKey; break; - case 3: pRight->aiKeyPtr[2] = iTreeKey; break; - } - - rc = treeInsert(pDb, pCsr, iNew, pLeaf->aiKeyPtr[1], iRight, - pCsr->aiCell[pCsr->iNode] - ); - } - }else{ - int iOut = 0; - int i; - for(i=0; i<4; i++){ - if( i==iSlot ) pNew->aiKeyPtr[iOut++] = iTreeKey; - if( i<3 && pLeaf->aiKeyPtr[i] ){ - pNew->aiKeyPtr[iOut++] = pLeaf->aiKeyPtr[i]; - } - } - rc = treeUpdatePtr(pDb, pCsr, iNew); - } - } - - return rc; -} - -void lsmTreeMakeOld(lsm_db *pDb){ - - /* A write transaction must be open. Otherwise the code below that - ** assumes (pDb->pClient->iLogOff) is current may malfunction. - ** - ** Update: currently this assert fails due to lsm_flush(), which does - ** not set nTransOpen. - */ - assert( /* pDb->nTransOpen>0 && */ pDb->iReader>=0 ); - - if( pDb->treehdr.iOldShmid==0 ){ - pDb->treehdr.iOldLog = (pDb->treehdr.log.aRegion[2].iEnd << 1); - pDb->treehdr.iOldLog |= (~(pDb->pClient->iLogOff) & (i64)0x0001); - - pDb->treehdr.oldcksum0 = pDb->treehdr.log.cksum0; - pDb->treehdr.oldcksum1 = pDb->treehdr.log.cksum1; - pDb->treehdr.iOldShmid = pDb->treehdr.iNextShmid-1; - memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); - - pDb->treehdr.root.iTransId = 1; - pDb->treehdr.root.iRoot = 0; - pDb->treehdr.root.nHeight = 0; - pDb->treehdr.root.nByte = 0; - } -} - -void lsmTreeDiscardOld(lsm_db *pDb){ - assert( lsmShmAssertLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) - ); - pDb->treehdr.iUsedShmid = pDb->treehdr.iOldShmid; - pDb->treehdr.iOldShmid = 0; -} - -int lsmTreeHasOld(lsm_db *pDb){ - return pDb->treehdr.iOldShmid!=0; -} - -/* -** This function is called during recovery to initialize the -** tree header. Only the database connections private copy of the tree-header -** is initialized here - it will be copied into shared memory if log file -** recovery is successful. -*/ -int lsmTreeInit(lsm_db *pDb){ - ShmChunk *pOne; - int rc = LSM_OK; - - memset(&pDb->treehdr, 0, sizeof(TreeHeader)); - pDb->treehdr.root.iTransId = 1; - pDb->treehdr.iFirst = 1; - pDb->treehdr.nChunk = 2; - pDb->treehdr.iWrite = LSM_SHM_CHUNK_SIZE + LSM_SHM_CHUNK_HDR; - pDb->treehdr.iNextShmid = 2; - pDb->treehdr.iUsedShmid = 1; - - pOne = treeShmChunkRc(pDb, 1, &rc); - if( pOne ){ - pOne->iNext = 0; - pOne->iShmid = 1; - } - return rc; -} - -static void treeHeaderChecksum( - TreeHeader *pHdr, - u32 *aCksum -){ - u32 cksum1 = 0x12345678; - u32 cksum2 = 0x9ABCDEF0; - u32 *a = (u32 *)pHdr; - int i; - - assert( (offsetof(TreeHeader, aCksum) + sizeof(u32)*2)==sizeof(TreeHeader) ); - assert( (sizeof(TreeHeader) % (sizeof(u32)*2))==0 ); - - for(i=0; i<(offsetof(TreeHeader, aCksum) / sizeof(u32)); i+=2){ - cksum1 += a[i]; - cksum2 += (cksum1 + a[i+1]); - } - aCksum[0] = cksum1; - aCksum[1] = cksum2; -} - -/* -** Return true if the checksum stored in TreeHeader object *pHdr is -** consistent with the contents of its other fields. -*/ -static int treeHeaderChecksumOk(TreeHeader *pHdr){ - u32 aCksum[2]; - treeHeaderChecksum(pHdr, aCksum); - return (0==memcmp(aCksum, pHdr->aCksum, sizeof(aCksum))); -} - -/* -** This type is used by functions lsmTreeRepair() and treeSortByShmid() to -** make relinking the linked list of shared-memory chunks easier. -*/ -typedef struct ShmChunkLoc ShmChunkLoc; -struct ShmChunkLoc { - ShmChunk *pShm; - u32 iLoc; -}; - -/* -** This function checks that the linked list of shared memory chunks -** that starts at chunk db->treehdr.iFirst: -** -** 1) Includes all chunks in the shared-memory region, and -** 2) Links them together in order of ascending shm-id. -** -** If no error occurs and the conditions above are met, LSM_OK is returned. -** -** If either of the conditions are untrue, LSM_CORRUPT is returned. Or, if -** an error is encountered before the checks are completed, another LSM error -** code (i.e. LSM_IOERR or LSM_NOMEM) may be returned. -*/ -static int treeCheckLinkedList(lsm_db *db){ - int rc = LSM_OK; - int nVisit = 0; - ShmChunk *p; - - p = treeShmChunkRc(db, db->treehdr.iFirst, &rc); - while( rc==LSM_OK && p ){ - if( p->iNext ){ - if( p->iNext>=db->treehdr.nChunk ){ - rc = LSM_CORRUPT_BKPT; - }else{ - ShmChunk *pNext = treeShmChunkRc(db, p->iNext, &rc); - if( rc==LSM_OK ){ - if( pNext->iShmid!=p->iShmid+1 ){ - rc = LSM_CORRUPT_BKPT; - } - p = pNext; - } - } - }else{ - p = 0; - } - nVisit++; - } - - if( rc==LSM_OK && (u32)nVisit!=db->treehdr.nChunk-1 ){ - rc = LSM_CORRUPT_BKPT; - } - return rc; -} - -/* -** Iterate through the current in-memory tree. If there are any v2-pointers -** with transaction ids larger than db->treehdr.iTransId, zero them. -*/ -static int treeRepairPtrs(lsm_db *db){ - int rc = LSM_OK; - - if( db->treehdr.root.nHeight>1 ){ - TreeCursor csr; /* Cursor used to iterate through tree */ - u32 iTransId = db->treehdr.root.iTransId; - - /* Initialize the cursor structure. Also decrement the nHeight variable - ** in the tree-header. This will prevent the cursor from visiting any - ** leaf nodes. */ - db->treehdr.root.nHeight--; - treeCursorInit(db, 0, &csr); - - rc = lsmTreeCursorEnd(&csr, 0); - while( rc==LSM_OK && lsmTreeCursorValid(&csr) ){ - TreeNode *pNode = csr.apTreeNode[csr.iNode]; - if( pNode->iV2>iTransId ){ - pNode->iV2Child = 0; - pNode->iV2Ptr = 0; - pNode->iV2 = 0; - } - rc = lsmTreeCursorNext(&csr); - } - tblobFree(csr.pDb, &csr.blob); - - db->treehdr.root.nHeight++; - } - - return rc; -} - -static int treeRepairList(lsm_db *db){ - int rc = LSM_OK; - int i; - ShmChunk *p; - ShmChunk *pMin = 0; - u32 iMin = 0; - - /* Iterate through all shm chunks. Find the smallest shm-id present in - ** the shared-memory region. */ - for(i=1; rc==LSM_OK && (u32)itreehdr.nChunk; i++){ - p = treeShmChunkRc(db, i, &rc); - if( p && (pMin==0 || shm_sequence_ge(pMin->iShmid, p->iShmid)) ){ - pMin = p; - iMin = i; - } - } - - /* Fix the shm-id values on any chunks with a shm-id greater than or - ** equal to treehdr.iNextShmid. Then do a merge-sort of all chunks to - ** fix the ShmChunk.iNext pointers. - */ - if( rc==LSM_OK ){ - int nSort; - int nByte; - u32 iPrevShmid; - ShmChunkLoc *aSort; - - /* Allocate space for a merge sort. */ - nSort = 1; - while( (u32)nSort < (db->treehdr.nChunk-1) ) nSort = nSort * 2; - nByte = sizeof(ShmChunkLoc) * nSort * 2; - aSort = lsmMallocZeroRc(db->pEnv, nByte, &rc); - iPrevShmid = pMin->iShmid; - - /* Fix all shm-ids, if required. */ - if( rc==LSM_OK ){ - iPrevShmid = pMin->iShmid-1; - for(i=1; (u32)itreehdr.nChunk; i++){ - p = treeShmChunk(db, i); - aSort[i-1].pShm = p; - aSort[i-1].iLoc = i; - if( (u32)i!=db->treehdr.iFirst ){ - if( shm_sequence_ge(p->iShmid, db->treehdr.iNextShmid) ){ - p->iShmid = iPrevShmid--; - } - } - } - if( iMin!=db->treehdr.iFirst ){ - p = treeShmChunk(db, db->treehdr.iFirst); - p->iShmid = iPrevShmid; - } - } - - if( rc==LSM_OK ){ - ShmChunkLoc *aSpace = &aSort[nSort]; - for(i=0; iiShmid, iPrevShmid) ); - assert( aSpace[aSort[i].pShm->iShmid - iPrevShmid].pShm==0 ); - aSpace[aSort[i].pShm->iShmid - iPrevShmid] = aSort[i]; - } - } - - if( aSpace[nSort-1].pShm ) aSpace[nSort-1].pShm->iNext = 0; - for(i=0; iiNext = aSpace[i+1].iLoc; - } - } - - rc = treeCheckLinkedList(db); - lsmFree(db->pEnv, aSort); - } - } - - return rc; -} - -/* -** This function is called as part of opening a write-transaction if the -** writer-flag is already set - indicating that the previous writer -** failed before ending its transaction. -*/ -int lsmTreeRepair(lsm_db *db){ - int rc = LSM_OK; - TreeHeader hdr; - ShmHeader *pHdr = db->pShmhdr; - - /* Ensure that the two tree-headers are consistent. Copy one over the other - ** if necessary. Prefer the data from a tree-header for which the checksum - ** computes. Or, if they both compute, prefer tree-header-1. */ - if( memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)) ){ - if( treeHeaderChecksumOk(&pHdr->hdr1) ){ - memcpy(&pHdr->hdr2, &pHdr->hdr1, sizeof(TreeHeader)); - }else{ - memcpy(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)); - } - } - - /* Save the connections current copy of the tree-header. It will be - ** restored before returning. */ - memcpy(&hdr, &db->treehdr, sizeof(TreeHeader)); - - /* Walk the tree. Zero any v2 pointers with a transaction-id greater than - ** the transaction-id currently in the tree-headers. */ - rc = treeRepairPtrs(db); - - /* Repair the linked list of shared-memory chunks. */ - if( rc==LSM_OK ){ - rc = treeRepairList(db); - } - - memcpy(&db->treehdr, &hdr, sizeof(TreeHeader)); - return rc; -} - -static void treeOverwriteKey(lsm_db *db, TreeCursor *pCsr, u32 iKey, int *pRc){ - if( *pRc==LSM_OK ){ - TreeRoot *p = &db->treehdr.root; - TreeNode *pNew; - u32 iNew; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - int iCell = pCsr->aiCell[pCsr->iNode]; - - /* Create a copy of this node */ - if( (pCsr->iNode>0 && (u32)pCsr->iNode==(p->nHeight-1)) ){ - pNew = copyTreeLeaf(db, (TreeLeaf *)pNode, &iNew, pRc); - }else{ - pNew = copyTreeNode(db, pNode, &iNew, pRc); - } - - if( pNew ){ - /* Modify the value in the new version */ - pNew->aiKeyPtr[iCell] = iKey; - - /* Change the pointer in the parent (if any) to point at the new - ** TreeNode */ - pCsr->iNode--; - treeUpdatePtr(db, pCsr, iNew); - } - } -} - -static int treeNextIsEndDelete(lsm_db *db, TreeCursor *pCsr){ - int iNode = pCsr->iNode; - int iCell = pCsr->aiCell[iNode]+1; - - /* Cursor currently points to a leaf node. */ - assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); - - while( iNode>=0 ){ - TreeNode *pNode = pCsr->apTreeNode[iNode]; - if( iCell<3 && pNode->aiKeyPtr[iCell] ){ - int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); - assert( rc==LSM_OK ); - return ((pKey->flags & LSM_END_DELETE) ? 1 : 0); - } - iNode--; - iCell = pCsr->aiCell[iNode]; - } - - return 0; -} - -static int treePrevIsStartDelete(lsm_db *db, TreeCursor *pCsr){ - int iNode = pCsr->iNode; - - /* Cursor currently points to a leaf node. */ - assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); - - while( iNode>=0 ){ - TreeNode *pNode = pCsr->apTreeNode[iNode]; - int iCell = pCsr->aiCell[iNode]-1; - if( iCell>=0 && pNode->aiKeyPtr[iCell] ){ - int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); - assert( rc==LSM_OK ); - return ((pKey->flags & LSM_START_DELETE) ? 1 : 0); - } - iNode--; - } - - return 0; -} - - -static int treeInsertEntry( - lsm_db *pDb, /* Database handle */ - int flags, /* Flags associated with entry */ - void *pKey, /* Pointer to key data */ - int nKey, /* Size of key data in bytes */ - void *pVal, /* Pointer to value data (or NULL) */ - int nVal /* Bytes in value data (or -ve for delete) */ -){ - int rc = LSM_OK; /* Return Code */ - TreeKey *pTreeKey; /* New key-value being inserted */ - u32 iTreeKey; - TreeRoot *p = &pDb->treehdr.root; - TreeCursor csr; /* Cursor to seek to pKey/nKey */ - int res = 0; /* Result of seek operation on csr */ - - assert( nVal>=0 || pVal==0 ); - assert_tree_looks_ok(LSM_OK, pTree); - assert( flags==LSM_INSERT || flags==LSM_POINT_DELETE - || flags==LSM_START_DELETE || flags==LSM_END_DELETE - ); - assert( (flags & LSM_CONTIGUOUS)==0 ); -#if 0 - dump_tree_contents(pDb, "before"); -#endif - - if( p->iRoot ){ - TreeKey *pRes; /* Key at end of seek operation */ - treeCursorInit(pDb, 0, &csr); - - /* Seek to the leaf (or internal node) that the new key belongs on */ - rc = lsmTreeCursorSeek(&csr, pKey, nKey, &res); - pRes = csrGetKey(&csr, &csr.blob, &rc); - if( rc!=LSM_OK ) return rc; - assert( pRes ); - - if( flags==LSM_START_DELETE ){ - /* When inserting a start-delete-range entry, if the key that - ** occurs immediately before the new entry is already a START_DELETE, - ** then the new entry is not required. */ - if( (res<=0 && (pRes->flags & LSM_START_DELETE)) - || (res>0 && treePrevIsStartDelete(pDb, &csr)) - ){ - goto insert_entry_out; - } - }else if( flags==LSM_END_DELETE ){ - /* When inserting an start-delete-range entry, if the key that - ** occurs immediately after the new entry is already an END_DELETE, - ** then the new entry is not required. */ - if( (res<0 && treeNextIsEndDelete(pDb, &csr)) - || (res>=0 && (pRes->flags & LSM_END_DELETE)) - ){ - goto insert_entry_out; - } - } - - if( res==0 && (flags & (LSM_END_DELETE|LSM_START_DELETE)) ){ - if( pRes->flags & LSM_INSERT ){ - nVal = pRes->nValue; - pVal = TKV_VAL(pRes); - } - flags = flags | pRes->flags; - } - - if( flags & (LSM_INSERT|LSM_POINT_DELETE) ){ - if( (res<0 && (pRes->flags & LSM_START_DELETE)) - || (res>0 && (pRes->flags & LSM_END_DELETE)) - ){ - flags = flags | (LSM_END_DELETE|LSM_START_DELETE); - }else if( res==0 ){ - flags = flags | (pRes->flags & (LSM_END_DELETE|LSM_START_DELETE)); - } - } - }else{ - memset(&csr, 0, sizeof(TreeCursor)); - } - - /* Allocate and populate a new key-value pair structure */ - pTreeKey = newTreeKey(pDb, &iTreeKey, pKey, nKey, pVal, nVal, &rc); - if( rc!=LSM_OK ) return rc; - assert( pTreeKey->flags==0 || pTreeKey->flags==LSM_CONTIGUOUS ); - pTreeKey->flags |= flags; - - if( p->iRoot==0 ){ - /* The tree is completely empty. Add a new root node and install - ** (pKey/nKey) as the middle entry. Even though it is a leaf at the - ** moment, use newTreeNode() to allocate the node (i.e. allocate enough - ** space for the fields used by interior nodes). This is because the - ** treeInsert() routine may convert this node to an interior node. */ - TreeNode *pRoot = newTreeNode(pDb, &p->iRoot, &rc); - if( rc==LSM_OK ){ - assert( p->nHeight==0 ); - pRoot->aiKeyPtr[1] = iTreeKey; - p->nHeight = 1; - } - }else{ - if( res==0 ){ - /* The search found a match within the tree. */ - treeOverwriteKey(pDb, &csr, iTreeKey, &rc); - }else{ - /* The cursor now points to the leaf node into which the new entry should - ** be inserted. There may or may not be a free slot within the leaf for - ** the new key-value pair. - ** - ** iSlot is set to the index of the key within pLeaf that the new key - ** should be inserted to the left of (or to a value 1 greater than the - ** index of the rightmost key if the new key is larger than all keys - ** currently stored in the node). - */ - int iSlot = csr.aiCell[csr.iNode] + (res<0); - if( csr.iNode==0 ){ - rc = treeInsert(pDb, &csr, 0, iTreeKey, 0, iSlot); - }else{ - rc = treeInsertLeaf(pDb, &csr, iTreeKey, iSlot); - } - } - } - -#if 0 - dump_tree_contents(pDb, "after"); -#endif - insert_entry_out: - tblobFree(pDb, &csr.blob); - assert_tree_looks_ok(rc, pTree); - return rc; -} - -/* -** Insert a new entry into the in-memory tree. -** -** If the value of the 5th parameter, nVal, is negative, then a delete-marker -** is inserted into the tree. In this case the value pointer, pVal, must be -** NULL. -*/ -int lsmTreeInsert( - lsm_db *pDb, /* Database handle */ - void *pKey, /* Pointer to key data */ - int nKey, /* Size of key data in bytes */ - void *pVal, /* Pointer to value data (or NULL) */ - int nVal /* Bytes in value data (or -ve for delete) */ -){ - int flags; - if( nVal<0 ){ - flags = LSM_POINT_DELETE; - }else{ - flags = LSM_INSERT; - } - - return treeInsertEntry(pDb, flags, pKey, nKey, pVal, nVal); -} - -static int treeDeleteEntry(lsm_db *db, TreeCursor *pCsr, u32 iNewptr){ - TreeRoot *p = &db->treehdr.root; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - int iSlot = pCsr->aiCell[pCsr->iNode]; - int bLeaf; - int rc = LSM_OK; - - assert( pNode->aiKeyPtr[1] ); - assert( pNode->aiKeyPtr[iSlot] ); - assert( iSlot==0 || iSlot==1 || iSlot==2 ); - assert( ((u32)pCsr->iNode==(db->treehdr.root.nHeight-1))==(iNewptr==0) ); - - bLeaf = ((u32)pCsr->iNode==(p->nHeight-1) && p->nHeight>1); - - if( pNode->aiKeyPtr[0] || pNode->aiKeyPtr[2] ){ - /* There are currently at least 2 keys on this node. So just create - ** a new copy of the node with one of the keys removed. If the node - ** happens to be the root node of the tree, allocate an entire - ** TreeNode structure instead of just a TreeLeaf. */ - TreeNode *pNew; - u32 iNew; - - if( bLeaf ){ - pNew = (TreeNode *)newTreeLeaf(db, &iNew, &rc); - }else{ - pNew = newTreeNode(db, &iNew, &rc); - } - if( pNew ){ - int i; - int iOut = 1; - for(i=0; i<4; i++){ - if( i==iSlot ){ - i++; - if( bLeaf==0 ) pNew->aiChildPtr[iOut] = iNewptr; - if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; - iOut++; - }else if( bLeaf || p->nHeight==1 ){ - if( i<3 && pNode->aiKeyPtr[i] ){ - pNew->aiKeyPtr[iOut++] = pNode->aiKeyPtr[i]; - } - }else{ - if( getChildPtr(pNode, WORKING_VERSION, i) ){ - pNew->aiChildPtr[iOut] = getChildPtr(pNode, WORKING_VERSION, i); - if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; - iOut++; - } - } - } - assert( iOut<=4 ); - assert( bLeaf || pNew->aiChildPtr[0]==0 ); - pCsr->iNode--; - rc = treeUpdatePtr(db, pCsr, iNew); - } - - }else if( pCsr->iNode==0 ){ - /* Removing the only key in the root node. iNewptr is the new root. */ - assert( iSlot==1 ); - db->treehdr.root.iRoot = iNewptr; - db->treehdr.root.nHeight--; - - }else{ - /* There is only one key on this node and the node is not the root - ** node. Find a peer for this node. Then redistribute the contents of - ** the peer and the parent cell between the parent and either one or - ** two new nodes. */ - TreeNode *pParent; /* Parent tree node */ - int iPSlot; - u32 iPeer; /* Pointer to peer leaf node */ - int iDir; - TreeNode *pPeer; /* The peer leaf node */ - TreeNode *pNew1; u32 iNew1; /* First new leaf node */ - - assert( iSlot==1 ); - - pParent = pCsr->apTreeNode[pCsr->iNode-1]; - iPSlot = pCsr->aiCell[pCsr->iNode-1]; - - if( iPSlot>0 && getChildPtr(pParent, WORKING_VERSION, iPSlot-1) ){ - iDir = -1; - }else{ - iDir = +1; - } - iPeer = getChildPtr(pParent, WORKING_VERSION, iPSlot+iDir); - pPeer = (TreeNode *)treeShmptr(db, iPeer); - assertIsWorkingChild(db, pNode, pParent, iPSlot); - - /* Allocate the first new leaf node. This is always required. */ - if( bLeaf ){ - pNew1 = (TreeNode *)newTreeLeaf(db, &iNew1, &rc); - }else{ - pNew1 = (TreeNode *)newTreeNode(db, &iNew1, &rc); - } - - if( pPeer->aiKeyPtr[0] && pPeer->aiKeyPtr[2] ){ - /* Peer node is completely full. This means that two new leaf nodes - ** and a new parent node are required. */ - - TreeNode *pNew2; u32 iNew2; /* Second new leaf node */ - TreeNode *pNewP; u32 iNewP; /* New parent node */ - - if( bLeaf ){ - pNew2 = (TreeNode *)newTreeLeaf(db, &iNew2, &rc); - }else{ - pNew2 = (TreeNode *)newTreeNode(db, &iNew2, &rc); - } - pNewP = copyTreeNode(db, pParent, &iNewP, &rc); - - if( iDir==-1 ){ - pNew1->aiKeyPtr[1] = pPeer->aiKeyPtr[0]; - if( bLeaf==0 ){ - pNew1->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 0); - pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 1); - } - - pNewP->aiChildPtr[iPSlot-1] = iNew1; - pNewP->aiKeyPtr[iPSlot-1] = pPeer->aiKeyPtr[1]; - pNewP->aiChildPtr[iPSlot] = iNew2; - - pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[2]; - pNew2->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot-1]; - if( bLeaf==0 ){ - pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 2); - pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 3); - pNew2->aiChildPtr[2] = iNewptr; - } - }else{ - pNew1->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ){ - pNew1->aiChildPtr[1] = iNewptr; - pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 0); - } - - pNewP->aiChildPtr[iPSlot] = iNew1; - pNewP->aiKeyPtr[iPSlot] = pPeer->aiKeyPtr[0]; - pNewP->aiChildPtr[iPSlot+1] = iNew2; - - pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[1]; - pNew2->aiKeyPtr[1] = pPeer->aiKeyPtr[2]; - if( bLeaf==0 ){ - pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 1); - pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 2); - pNew2->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 3); - } - } - assert( pCsr->iNode>=1 ); - pCsr->iNode -= 2; - if( rc==LSM_OK ){ - assert( pNew1->aiKeyPtr[1] && pNew2->aiKeyPtr[1] ); - rc = treeUpdatePtr(db, pCsr, iNewP); - } - }else{ - int iKOut = 0; - int iPOut = 0; - int i; - - pCsr->iNode--; - - if( iDir==1 ){ - pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; - } - for(i=0; i<3; i++){ - if( pPeer->aiKeyPtr[i] ){ - pNew1->aiKeyPtr[iKOut++] = pPeer->aiKeyPtr[i]; - } - } - if( bLeaf==0 ){ - for(i=0; i<4; i++){ - if( getChildPtr(pPeer, WORKING_VERSION, i) ){ - pNew1->aiChildPtr[iPOut++] = getChildPtr(pPeer, WORKING_VERSION, i); - } - } - } - if( iDir==-1 ){ - iPSlot--; - pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; - pCsr->aiCell[pCsr->iNode] = (u8)iPSlot; - } - - rc = treeDeleteEntry(db, pCsr, iNew1); - } - } - - return rc; -} - -/* -** Delete a range of keys from the tree structure (i.e. the lsm_delete_range() -** function, not lsm_delete()). -** -** This is a two step process: -** -** 1) Remove all entries currently stored in the tree that have keys -** that fall into the deleted range. -** -** TODO: There are surely good ways to optimize this step - removing -** a range of keys from a b-tree. But for now, this function removes -** them one at a time using the usual approach. -** -** 2) Unless the largest key smaller than or equal to (pKey1/nKey1) is -** already marked as START_DELETE, insert a START_DELETE key. -** Similarly, unless the smallest key greater than or equal to -** (pKey2/nKey2) is already START_END, insert a START_END key. -*/ -int lsmTreeDelete( - lsm_db *db, - void *pKey1, int nKey1, /* Start of range */ - void *pKey2, int nKey2 /* End of range */ -){ - int rc = LSM_OK; - int bDone = 0; - TreeRoot *p = &db->treehdr.root; - TreeBlob blob = {0, 0}; - - /* The range must be sensible - that (key1 < key2). */ - assert( treeKeycmp(pKey1, nKey1, pKey2, nKey2)<0 ); - assert( assert_delete_ranges_match(db) ); - -#if 0 - static int nCall = 0; - printf("\n"); - nCall++; - printf("%d delete %s .. %s\n", nCall, (char *)pKey1, (char *)pKey2); - dump_tree_contents(db, "before delete"); -#endif - - /* Step 1. This loop runs until the tree contains no keys within the - ** range being deleted. Or until an error occurs. */ - while( bDone==0 && rc==LSM_OK ){ - int res; - TreeCursor csr; /* Cursor to seek to first key in range */ - void *pDel; int nDel; /* Key to (possibly) delete this iteration */ -#ifndef NDEBUG - int nEntry = treeCountEntries(db); -#endif - - /* Seek the cursor to the first entry in the tree greater than pKey1. */ - treeCursorInit(db, 0, &csr); - lsmTreeCursorSeek(&csr, pKey1, nKey1, &res); - if( res<=0 && lsmTreeCursorValid(&csr) ) lsmTreeCursorNext(&csr); - - /* If there is no such entry, or if it is greater than pKey2, then the - ** tree now contains no keys in the range being deleted. In this case - ** break out of the loop. */ - bDone = 1; - if( lsmTreeCursorValid(&csr) ){ - lsmTreeCursorKey(&csr, 0, &pDel, &nDel); - if( treeKeycmp(pDel, nDel, pKey2, nKey2)<0 ) bDone = 0; - } - - if( bDone==0 ){ - if( (u32)csr.iNode==(p->nHeight-1) ){ - /* The element to delete already lies on a leaf node */ - rc = treeDeleteEntry(db, &csr, 0); - }else{ - /* 1. Overwrite the current key with a copy of the next key in the - ** tree (key N). - ** - ** 2. Seek to key N (cursor will stop at the internal node copy of - ** N). Move to the next key (original copy of N). Delete - ** this entry. - */ - u32 iKey; - TreeKey *pKey; - int iNode = csr.iNode; - lsmTreeCursorNext(&csr); - assert( (u32)csr.iNode==(p->nHeight-1) ); - - iKey = csr.apTreeNode[csr.iNode]->aiKeyPtr[csr.aiCell[csr.iNode]]; - lsmTreeCursorPrev(&csr); - - treeOverwriteKey(db, &csr, iKey, &rc); - pKey = treeShmkey(db, iKey, TKV_LOADKEY, &blob, &rc); - if( pKey ){ - rc = lsmTreeCursorSeek(&csr, TKV_KEY(pKey), pKey->nKey, &res); - } - if( rc==LSM_OK ){ - assert( res==0 && csr.iNode==iNode ); - rc = lsmTreeCursorNext(&csr); - if( rc==LSM_OK ){ - rc = treeDeleteEntry(db, &csr, 0); - } - } - } - } - - /* Clean up any memory allocated by the cursor. */ - tblobFree(db, &csr.blob); -#if 0 - dump_tree_contents(db, "ddd delete"); -#endif - assert( bDone || treeCountEntries(db)==(nEntry-1) ); - } - -#if 0 - dump_tree_contents(db, "during delete"); -#endif - - /* Now insert the START_DELETE and END_DELETE keys. */ - if( rc==LSM_OK ){ - rc = treeInsertEntry(db, LSM_START_DELETE, pKey1, nKey1, 0, -1); - } -#if 0 - dump_tree_contents(db, "during delete 2"); -#endif - if( rc==LSM_OK ){ - rc = treeInsertEntry(db, LSM_END_DELETE, pKey2, nKey2, 0, -1); - } - -#if 0 - dump_tree_contents(db, "after delete"); -#endif - - tblobFree(db, &blob); - assert( assert_delete_ranges_match(db) ); - return rc; -} - -/* -** Return, in bytes, the amount of memory currently used by the tree -** structure. -*/ -int lsmTreeSize(lsm_db *pDb){ - return pDb->treehdr.root.nByte; -} - -/* -** Open a cursor on the in-memory tree pTree. -*/ -int lsmTreeCursorNew(lsm_db *pDb, int bOld, TreeCursor **ppCsr){ - TreeCursor *pCsr; - *ppCsr = pCsr = lsmMalloc(pDb->pEnv, sizeof(TreeCursor)); - if( pCsr ){ - treeCursorInit(pDb, bOld, pCsr); - return LSM_OK; - } - return LSM_NOMEM_BKPT; -} - -/* -** Close an in-memory tree cursor. -*/ -void lsmTreeCursorDestroy(TreeCursor *pCsr){ - if( pCsr ){ - tblobFree(pCsr->pDb, &pCsr->blob); - lsmFree(pCsr->pDb->pEnv, pCsr); - } -} - -void lsmTreeCursorReset(TreeCursor *pCsr){ - if( pCsr ){ - pCsr->iNode = -1; - pCsr->pSave = 0; - } -} - -#ifndef NDEBUG -static int treeCsrCompare(TreeCursor *pCsr, void *pKey, int nKey, int *pRc){ - TreeKey *p; - int cmp = 0; - assert( pCsr->iNode>=0 ); - p = csrGetKey(pCsr, &pCsr->blob, pRc); - if( p ){ - cmp = treeKeycmp(TKV_KEY(p), p->nKey, pKey, nKey); - } - return cmp; -} -#endif - - -/* -** Attempt to seek the cursor passed as the first argument to key (pKey/nKey) -** in the tree structure. If an exact match for the key is found, leave the -** cursor pointing to it and set *pRes to zero before returning. If an -** exact match cannot be found, do one of the following: -** -** * Leave the cursor pointing to the smallest element in the tree that -** is larger than the key and set *pRes to +1, or -** -** * Leave the cursor pointing to the largest element in the tree that -** is smaller than the key and set *pRes to -1, or -** -** * If the tree is empty, leave the cursor at EOF and set *pRes to -1. -*/ -int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes){ - int rc = LSM_OK; /* Return code */ - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - u32 iNodePtr; /* Location of current node in search */ - - /* Discard any saved position data */ - treeCursorRestore(pCsr, 0); - - iNodePtr = pRoot->iRoot; - if( iNodePtr==0 ){ - /* Either an error occurred or the tree is completely empty. */ - assert( rc!=LSM_OK || pRoot->iRoot==0 ); - *pRes = -1; - pCsr->iNode = -1; - }else{ - TreeBlob b = {0, 0}; - int res = 0; /* Result of comparison function */ - int iNode = -1; - while( iNodePtr ){ - TreeNode *pNode; /* Node at location iNodePtr */ - int iTest; /* Index of second key to test (0 or 2) */ - u32 iTreeKey; - TreeKey *pTreeKey; /* Key to compare against */ - - pNode = (TreeNode *)treeShmptrUnsafe(pDb, iNodePtr); - iNode++; - pCsr->apTreeNode[iNode] = pNode; - - /* Compare (pKey/nKey) with the key in the middle slot of B-tree node - ** pNode. The middle slot is never empty. If the comparison is a match, - ** then the search is finished. Break out of the loop. */ - pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, pNode->aiKeyPtr[1]); - if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); - if( rc!=LSM_OK ) break; - } - res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); - if( res==0 ){ - pCsr->aiCell[iNode] = 1; - break; - } - - /* Based on the results of the previous comparison, compare (pKey/nKey) - ** to either the left or right key of the B-tree node, if such a key - ** exists. */ - iTest = (res>0 ? 0 : 2); - iTreeKey = pNode->aiKeyPtr[iTest]; - if( iTreeKey ){ - pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, iTreeKey); - if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ - pTreeKey = treeShmkey(pDb, iTreeKey, TKV_LOADKEY, &b, &rc); - if( rc ) break; - } - res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); - if( res==0 ){ - pCsr->aiCell[iNode] = (u8)iTest; - break; - } - }else{ - iTest = 1; - } - - if( (u32)iNode<(pRoot->nHeight-1) ){ - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iTest + (res<0)); - }else{ - iNodePtr = 0; - } - pCsr->aiCell[iNode] = (u8)(iTest + (iNodePtr && (res<0))); - } - - *pRes = res; - pCsr->iNode = iNode; - tblobFree(pDb, &b); - } - - /* assert() that *pRes has been set properly */ -#ifndef NDEBUG - if( rc==LSM_OK && lsmTreeCursorValid(pCsr) ){ - int cmp = treeCsrCompare(pCsr, pKey, nKey, &rc); - assert( rc!=LSM_OK || *pRes==cmp || (*pRes ^ cmp)>0 ); - } -#endif - - return rc; -} - -int lsmTreeCursorNext(TreeCursor *pCsr){ -#ifndef NDEBUG - TreeKey *pK1; - TreeBlob key1 = {0, 0}; -#endif - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - const int iLeaf = pRoot->nHeight-1; - int iCell; - int rc = LSM_OK; - TreeNode *pNode; - - /* Restore the cursor position, if required */ - int iRestore = 0; - treeCursorRestore(pCsr, &iRestore); - if( iRestore>0 ) return LSM_OK; - - /* Save a pointer to the current key. This is used in an assert() at the - ** end of this function - to check that the 'next' key really is larger - ** than the current key. */ -#ifndef NDEBUG - pK1 = csrGetKey(pCsr, &key1, &rc); - if( rc!=LSM_OK ) return rc; -#endif - - assert( lsmTreeCursorValid(pCsr) ); - assert( pCsr->aiCell[pCsr->iNode]<3 ); - - pNode = pCsr->apTreeNode[pCsr->iNode]; - iCell = ++pCsr->aiCell[pCsr->iNode]; - - /* If the current node is not a leaf, and the current cell has sub-tree - ** associated with it, descend to the left-most key on the left-most - ** leaf of the sub-tree. */ - if( pCsr->iNodeiTransId, iCell) ){ - do { - u32 iNodePtr; - pCsr->iNode++; - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - pCsr->apTreeNode[pCsr->iNode] = pNode; - iCell = pCsr->aiCell[pCsr->iNode] = (pNode->aiKeyPtr[0]==0); - }while( pCsr->iNode < iLeaf ); - } - - /* Otherwise, the next key is found by following pointer up the tree - ** until there is a key immediately to the right of the pointer followed - ** to reach the sub-tree containing the current key. */ - else if( iCell>=3 || pNode->aiKeyPtr[iCell]==0 ){ - while( (--pCsr->iNode)>=0 ){ - iCell = pCsr->aiCell[pCsr->iNode]; - if( iCell<3 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; - } - } - -#ifndef NDEBUG - if( pCsr->iNode>=0 ){ - TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc||treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); - } - tblobFree(pDb, &key1); -#endif - - return rc; -} - -int lsmTreeCursorPrev(TreeCursor *pCsr){ -#ifndef NDEBUG - TreeKey *pK1; - TreeBlob key1 = {0, 0}; -#endif - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - const int iLeaf = pRoot->nHeight-1; - int iCell; - int rc = LSM_OK; - TreeNode *pNode; - - /* Restore the cursor position, if required */ - int iRestore = 0; - treeCursorRestore(pCsr, &iRestore); - if( iRestore<0 ) return LSM_OK; - - /* Save a pointer to the current key. This is used in an assert() at the - ** end of this function - to check that the 'next' key really is smaller - ** than the current key. */ -#ifndef NDEBUG - pK1 = csrGetKey(pCsr, &key1, &rc); - if( rc!=LSM_OK ) return rc; -#endif - - assert( lsmTreeCursorValid(pCsr) ); - pNode = pCsr->apTreeNode[pCsr->iNode]; - iCell = pCsr->aiCell[pCsr->iNode]; - assert( iCell>=0 && iCell<3 ); - - /* If the current node is not a leaf, and the current cell has sub-tree - ** associated with it, descend to the right-most key on the right-most - ** leaf of the sub-tree. */ - if( pCsr->iNodeiTransId, iCell) ){ - do { - u32 iNodePtr; - pCsr->iNode++; - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - if( rc!=LSM_OK ) break; - pCsr->apTreeNode[pCsr->iNode] = pNode; - iCell = 1 + (pNode->aiKeyPtr[2]!=0) + (pCsr->iNode < iLeaf); - pCsr->aiCell[pCsr->iNode] = (u8)iCell; - }while( pCsr->iNode < iLeaf ); - } - - /* Otherwise, the next key is found by following pointer up the tree until - ** there is a key immediately to the left of the pointer followed to reach - ** the sub-tree containing the current key. */ - else{ - do { - iCell = pCsr->aiCell[pCsr->iNode]-1; - if( iCell>=0 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; - }while( (--pCsr->iNode)>=0 ); - pCsr->aiCell[pCsr->iNode] = (u8)iCell; - } - -#ifndef NDEBUG - if( pCsr->iNode>=0 ){ - TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)<0 ); - } - tblobFree(pDb, &key1); -#endif - - return rc; -} - -/* -** Move the cursor to the first (bLast==0) or last (bLast!=0) entry in the -** in-memory tree. -*/ -int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast){ - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - int rc = LSM_OK; - - u32 iNodePtr; - pCsr->iNode = -1; - - /* Discard any saved position data */ - treeCursorRestore(pCsr, 0); - - iNodePtr = pRoot->iRoot; - while( iNodePtr ){ - int iCell; - TreeNode *pNode; - - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - if( rc ) break; - - if( bLast ){ - iCell = ((pNode->aiKeyPtr[2]==0) ? 2 : 3); - }else{ - iCell = ((pNode->aiKeyPtr[0]==0) ? 1 : 0); - } - pCsr->iNode++; - pCsr->apTreeNode[pCsr->iNode] = pNode; - - if( (u32)pCsr->iNodenHeight-1 ){ - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - }else{ - iNodePtr = 0; - } - pCsr->aiCell[pCsr->iNode] = (u8)(iCell - (iNodePtr==0 && bLast)); - } - - return rc; -} - -int lsmTreeCursorFlags(TreeCursor *pCsr){ - int flags = 0; - if( pCsr && pCsr->iNode>=0 ){ - int rc = LSM_OK; - TreeKey *pKey = (TreeKey *)treeShmptrUnsafe(pCsr->pDb, - pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] - ); - assert( rc==LSM_OK ); - flags = (pKey->flags & ~LSM_CONTIGUOUS); - } - return flags; -} - -int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey){ - TreeKey *pTreeKey; - int rc = LSM_OK; - - assert( lsmTreeCursorValid(pCsr) ); - - pTreeKey = pCsr->pSave; - if( !pTreeKey ){ - pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); - } - if( rc==LSM_OK ){ - *pnKey = pTreeKey->nKey; - if( pFlags ) *pFlags = pTreeKey->flags; - *ppKey = (void *)&pTreeKey[1]; - } - - return rc; -} - -int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal){ - int res = 0; - int rc; - - rc = treeCursorRestore(pCsr, &res); - if( res==0 ){ - TreeKey *pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); - if( rc==LSM_OK ){ - if( pTreeKey->flags & LSM_INSERT ){ - *pnVal = pTreeKey->nValue; - *ppVal = TKV_VAL(pTreeKey); - }else{ - *ppVal = 0; - *pnVal = -1; - } - } - }else{ - *ppVal = 0; - *pnVal = 0; - } - - return rc; -} - -/* -** Return true if the cursor currently points to a valid entry. -*/ -int lsmTreeCursorValid(TreeCursor *pCsr){ - return (pCsr && (pCsr->pSave || pCsr->iNode>=0)); -} - -/* -** Store a mark in *pMark. Later on, a call to lsmTreeRollback() with a -** pointer to the same TreeMark structure may be used to roll the tree -** contents back to their current state. -*/ -void lsmTreeMark(lsm_db *pDb, TreeMark *pMark){ - pMark->iRoot = pDb->treehdr.root.iRoot; - pMark->nHeight = pDb->treehdr.root.nHeight; - pMark->iWrite = pDb->treehdr.iWrite; - pMark->nChunk = pDb->treehdr.nChunk; - pMark->iNextShmid = pDb->treehdr.iNextShmid; - pMark->iRollback = intArraySize(&pDb->rollback); -} - -/* -** Roll back to mark pMark. Structure *pMark should have been previously -** populated by a call to lsmTreeMark(). -*/ -void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark){ - int iIdx; - int nIdx; - u32 iNext; - ShmChunk *pChunk; - u32 iChunk; - u32 iShmid; - - /* Revert all required v2 pointers. */ - nIdx = intArraySize(&pDb->rollback); - for(iIdx = pMark->iRollback; iIdxrollback, iIdx)); - assert( pNode ); - pNode->iV2 = 0; - pNode->iV2Child = 0; - pNode->iV2Ptr = 0; - } - intArrayTruncate(&pDb->rollback, pMark->iRollback); - - /* Restore the free-chunk list. */ - assert( pMark->iWrite!=0 ); - iChunk = treeOffsetToChunk(pMark->iWrite-1); - pChunk = treeShmChunk(pDb, iChunk); - iNext = pChunk->iNext; - pChunk->iNext = 0; - - pChunk = treeShmChunk(pDb, pDb->treehdr.iFirst); - iShmid = pChunk->iShmid-1; - - while( iNext ){ - u32 iFree = iNext; /* Current chunk being rollback-freed */ - ShmChunk *pFree; /* Pointer to chunk iFree */ - - pFree = treeShmChunk(pDb, iFree); - iNext = pFree->iNext; - - if( iFreenChunk ){ - pFree->iNext = pDb->treehdr.iFirst; - pFree->iShmid = iShmid--; - pDb->treehdr.iFirst = iFree; - } - } - - /* Restore the tree-header fields */ - pDb->treehdr.root.iRoot = pMark->iRoot; - pDb->treehdr.root.nHeight = pMark->nHeight; - pDb->treehdr.iWrite = pMark->iWrite; - pDb->treehdr.nChunk = pMark->nChunk; - pDb->treehdr.iNextShmid = pMark->iNextShmid; -} - -/* -** Load the in-memory tree header from shared-memory into pDb->treehdr. -** If the header cannot be loaded, return LSM_PROTOCOL. -** -** If the header is successfully loaded and parameter piRead is not NULL, -** is is set to 1 if the header was loaded from ShmHeader.hdr1, or 2 if -** the header was loaded from ShmHeader.hdr2. -*/ -int lsmTreeLoadHeader(lsm_db *pDb, int *piRead){ - int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; - while( (nRem--)>0 ){ - ShmHeader *pShm = pDb->pShmhdr; - - memcpy(&pDb->treehdr, &pShm->hdr1, sizeof(TreeHeader)); - if( treeHeaderChecksumOk(&pDb->treehdr) ){ - if( piRead ) *piRead = 1; - return LSM_OK; - } - memcpy(&pDb->treehdr, &pShm->hdr2, sizeof(TreeHeader)); - if( treeHeaderChecksumOk(&pDb->treehdr) ){ - if( piRead ) *piRead = 2; - return LSM_OK; - } - - lsmShmBarrier(pDb); - } - return LSM_PROTOCOL_BKPT; -} - -int lsmTreeLoadHeaderOk(lsm_db *pDb, int iRead){ - TreeHeader *p = (iRead==1) ? &pDb->pShmhdr->hdr1 : &pDb->pShmhdr->hdr2; - assert( iRead==1 || iRead==2 ); - return (0==memcmp(pDb->treehdr.aCksum, p->aCksum, sizeof(u32)*2)); -} - -/* -** This function is called to conclude a transaction. If argument bCommit -** is true, the transaction is committed. Otherwise it is rolled back. -*/ -int lsmTreeEndTransaction(lsm_db *pDb, int bCommit){ - ShmHeader *pShm = pDb->pShmhdr; - - treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); - memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); - lsmShmBarrier(pDb); - memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); - pShm->bWriter = 0; - intArrayFree(pDb->pEnv, &pDb->rollback); - - return LSM_OK; -} - -#ifndef NDEBUG -static int assert_delete_ranges_match(lsm_db *db){ - int prev = 0; - TreeBlob blob = {0, 0}; - TreeCursor csr; /* Cursor used to iterate through tree */ - int rc; - - treeCursorInit(db, 0, &csr); - for( rc = lsmTreeCursorEnd(&csr, 0); - rc==LSM_OK && lsmTreeCursorValid(&csr); - rc = lsmTreeCursorNext(&csr) - ){ - TreeKey *pKey = csrGetKey(&csr, &blob, &rc); - if( rc!=LSM_OK ) break; - assert( ((prev&LSM_START_DELETE)==0)==((pKey->flags&LSM_END_DELETE)==0) ); - prev = pKey->flags; - } - - tblobFree(csr.pDb, &csr.blob); - tblobFree(csr.pDb, &blob); - - return 1; -} - -static int treeCountEntries(lsm_db *db){ - TreeCursor csr; /* Cursor used to iterate through tree */ - int rc; - int nEntry = 0; - - treeCursorInit(db, 0, &csr); - for( rc = lsmTreeCursorEnd(&csr, 0); - rc==LSM_OK && lsmTreeCursorValid(&csr); - rc = lsmTreeCursorNext(&csr) - ){ - nEntry++; - } - - tblobFree(csr.pDb, &csr.blob); - - return nEntry; -} -#endif diff --git a/ext/lsm1/lsm_unix.c b/ext/lsm1/lsm_unix.c deleted file mode 100644 index 88952d15fc..0000000000 --- a/ext/lsm1/lsm_unix.c +++ /dev/null @@ -1,753 +0,0 @@ -/* -** 2011-12-03 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Unix-specific run-time environment implementation for LSM. -*/ - -#ifndef _WIN32 - -#if defined(__GNUC__) || defined(__TINYC__) -/* workaround for ftruncate() visibility on gcc. */ -# ifndef _XOPEN_SOURCE -# define _XOPEN_SOURCE 500 -# endif -#endif - -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include "lsmInt.h" - -/* There is no fdatasync() call on Android */ -#ifdef __ANDROID__ -# define fdatasync(x) fsync(x) -#endif - -/* -** An open file is an instance of the following object -*/ -typedef struct PosixFile PosixFile; -struct PosixFile { - lsm_env *pEnv; /* The run-time environment */ - const char *zName; /* Full path to file */ - int fd; /* The open file descriptor */ - int shmfd; /* Shared memory file-descriptor */ - void *pMap; /* Pointer to mapping of file fd */ - off_t nMap; /* Size of mapping at pMap in bytes */ - int nShm; /* Number of entries in array apShm[] */ - void **apShm; /* Array of 32K shared memory segments */ -}; - -static char *posixShmFile(PosixFile *p){ - char *zShm; - int nName = strlen(p->zName); - zShm = (char *)lsmMalloc(p->pEnv, nName+4+1); - if( zShm ){ - memcpy(zShm, p->zName, nName); - memcpy(&zShm[nName], "-shm", 5); - } - return zShm; -} - -static int lsmPosixOsOpen( - lsm_env *pEnv, - const char *zFile, - int flags, - lsm_file **ppFile -){ - int rc = LSM_OK; - PosixFile *p; - - p = lsm_malloc(pEnv, sizeof(PosixFile)); - if( p==0 ){ - rc = LSM_NOMEM; - }else{ - int bReadonly = (flags & LSM_OPEN_READONLY); - int oflags = (bReadonly ? O_RDONLY : (O_RDWR|O_CREAT)); - memset(p, 0, sizeof(PosixFile)); - p->zName = zFile; - p->pEnv = pEnv; - p->fd = open(zFile, oflags, 0644); - if( p->fd<0 ){ - lsm_free(pEnv, p); - p = 0; - if( errno==ENOENT ){ - rc = lsmErrorBkpt(LSM_IOERR_NOENT); - }else{ - rc = LSM_IOERR_BKPT; - } - } - } - - *ppFile = (lsm_file *)p; - return rc; -} - -static int lsmPosixOsWrite( - lsm_file *pFile, /* File to write to */ - lsm_i64 iOff, /* Offset to write to */ - void *pData, /* Write data from this buffer */ - int nData /* Bytes of data to write */ -){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - off_t offset; - - offset = lseek(p->fd, (off_t)iOff, SEEK_SET); - if( offset!=iOff ){ - rc = LSM_IOERR_BKPT; - }else{ - ssize_t prc = write(p->fd, pData, (size_t)nData); - if( prc<0 ) rc = LSM_IOERR_BKPT; - } - - return rc; -} - -static int lsmPosixOsTruncate( - lsm_file *pFile, /* File to write to */ - lsm_i64 nSize /* Size to truncate file to */ -){ - PosixFile *p = (PosixFile *)pFile; - int rc = LSM_OK; /* Return code */ - int prc; /* Posix Return Code */ - struct stat sStat; /* Result of fstat() invocation */ - - prc = fstat(p->fd, &sStat); - if( prc==0 && sStat.st_size>nSize ){ - prc = ftruncate(p->fd, (off_t)nSize); - } - if( prc<0 ) rc = LSM_IOERR_BKPT; - - return rc; -} - -static int lsmPosixOsRead( - lsm_file *pFile, /* File to read from */ - lsm_i64 iOff, /* Offset to read from */ - void *pData, /* Read data into this buffer */ - int nData /* Bytes of data to read */ -){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - off_t offset; - - offset = lseek(p->fd, (off_t)iOff, SEEK_SET); - if( offset!=iOff ){ - rc = LSM_IOERR_BKPT; - }else{ - ssize_t prc = read(p->fd, pData, (size_t)nData); - if( prc<0 ){ - rc = LSM_IOERR_BKPT; - }else if( prcpMap ){ - prc = msync(p->pMap, p->nMap, MS_SYNC); - } - if( prc==0 ) prc = fdatasync(p->fd); - if( prc<0 ) rc = LSM_IOERR_BKPT; -#else - (void)pFile; -#endif - - return rc; -} - -static int lsmPosixOsSectorSize(lsm_file *pFile){ - return 512; -} - -static int lsmPosixOsRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - off_t iSz; - int prc; - PosixFile *p = (PosixFile *)pFile; - struct stat buf; - - /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. - ** Thereafter, in chunks of 1MB at a time. */ - const int aIncrSz[] = {256*1024, 1024*1024}; - int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; - - if( p->pMap ){ - munmap(p->pMap, p->nMap); - *ppOut = p->pMap = 0; - *pnOut = p->nMap = 0; - } - - if( iMin>=0 ){ - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSzfd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - if( p->pMap==MAP_FAILED ){ - p->pMap = 0; - return LSM_IOERR_BKPT; - } - p->nMap = iSz; - } - - *ppOut = p->pMap; - *pnOut = p->nMap; - return LSM_OK; -} - -static int lsmPosixOsFullpath( - lsm_env *pEnv, - const char *zName, - char *zOut, - int *pnOut -){ - int nBuf = *pnOut; - int nReq; - - if( zName[0]!='/' ){ - char *z; - char *zTmp; - int nTmp = 512; - zTmp = lsmMalloc(pEnv, nTmp); - while( zTmp ){ - z = getcwd(zTmp, nTmp); - if( z || errno!=ERANGE ) break; - nTmp = nTmp*2; - zTmp = lsmReallocOrFree(pEnv, zTmp, nTmp); - } - if( zTmp==0 ) return LSM_NOMEM_BKPT; - if( z==0 ) return LSM_IOERR_BKPT; - assert( z==zTmp ); - - nTmp = strlen(zTmp); - nReq = nTmp + 1 + strlen(zName) + 1; - if( nReq<=nBuf ){ - memcpy(zOut, zTmp, nTmp); - zOut[nTmp] = '/'; - memcpy(&zOut[nTmp+1], zName, strlen(zName)+1); - } - lsmFree(pEnv, zTmp); - }else{ - nReq = strlen(zName)+1; - if( nReq<=nBuf ){ - memcpy(zOut, zName, strlen(zName)+1); - } - } - - *pnOut = nReq; - return LSM_OK; -} - -static int lsmPosixOsFileid( - lsm_file *pFile, - void *pBuf, - int *pnBuf -){ - int prc; - int nBuf; - int nReq; - PosixFile *p = (PosixFile *)pFile; - struct stat buf; - - nBuf = *pnBuf; - nReq = (sizeof(buf.st_dev) + sizeof(buf.st_ino)); - *pnBuf = nReq; - if( nReq>nBuf ) return LSM_OK; - - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - - memcpy(pBuf, &buf.st_dev, sizeof(buf.st_dev)); - memcpy(&(((u8 *)pBuf)[sizeof(buf.st_dev)]), &buf.st_ino, sizeof(buf.st_ino)); - return LSM_OK; -} - -static int lsmPosixOsUnlink(lsm_env *pEnv, const char *zFile){ - int prc = unlink(zFile); - return prc ? LSM_IOERR_BKPT : LSM_OK; -} - -static int lsmPosixOsLock(lsm_file *pFile, int iLock, int eType){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - static const short aType[3] = { F_UNLCK, F_RDLCK, F_WRLCK }; - struct flock lock; - - assert( aType[LSM_LOCK_UNLOCK]==F_UNLCK ); - assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); - assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); - assert( eType>=0 && eType0 && iLock<=32 ); - - memset(&lock, 0, sizeof(lock)); - lock.l_whence = SEEK_SET; - lock.l_len = 1; - lock.l_type = aType[eType]; - lock.l_start = (4096-iLock); - - if( fcntl(p->fd, F_SETLK, &lock) ){ - int e = errno; - if( e==EACCES || e==EAGAIN ){ - rc = LSM_BUSY; - }else{ - rc = LSM_IOERR_BKPT; - } - } - - return rc; -} - -static int lsmPosixOsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - static const short aType[3] = { 0, F_RDLCK, F_WRLCK }; - struct flock lock; - - assert( eType==LSM_LOCK_SHARED || eType==LSM_LOCK_EXCL ); - assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); - assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); - assert( eType>=0 && eType0 && iLock<=32 ); - - memset(&lock, 0, sizeof(lock)); - lock.l_whence = SEEK_SET; - lock.l_len = nLock; - lock.l_type = aType[eType]; - lock.l_start = (4096-iLock-nLock+1); - - if( fcntl(p->fd, F_GETLK, &lock) ){ - rc = LSM_IOERR_BKPT; - }else if( lock.l_type!=F_UNLCK ){ - rc = LSM_BUSY; - } - - return rc; -} - -static int lsmPosixOsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ - PosixFile *p = (PosixFile *)pFile; - - *ppShm = 0; - assert( sz==LSM_SHM_CHUNK_SIZE ); - if( iChunk>=p->nShm ){ - int i; - void **apNew; - int nNew = iChunk+1; - off_t nReq = nNew * LSM_SHM_CHUNK_SIZE; - struct stat sStat; - - /* If the shared-memory file has not been opened, open it now. */ - if( p->shmfd<=0 ){ - char *zShm = posixShmFile(p); - if( !zShm ) return LSM_NOMEM_BKPT; - p->shmfd = open(zShm, O_RDWR|O_CREAT, 0644); - lsmFree(p->pEnv, zShm); - if( p->shmfd<0 ){ - return LSM_IOERR_BKPT; - } - } - - /* If the shared-memory file is not large enough to contain the - ** requested chunk, cause it to grow. */ - if( fstat(p->shmfd, &sStat) ){ - return LSM_IOERR_BKPT; - } - if( sStat.st_sizeshmfd, nReq) ){ - return LSM_IOERR_BKPT; - } - } - - apNew = (void **)lsmRealloc(p->pEnv, p->apShm, sizeof(void *) * nNew); - if( !apNew ) return LSM_NOMEM_BKPT; - for(i=p->nShm; iapShm = apNew; - p->nShm = nNew; - } - - if( p->apShm[iChunk]==0 ){ - p->apShm[iChunk] = mmap(0, LSM_SHM_CHUNK_SIZE, - PROT_READ|PROT_WRITE, MAP_SHARED, p->shmfd, iChunk*LSM_SHM_CHUNK_SIZE - ); - if( p->apShm[iChunk]==MAP_FAILED ){ - p->apShm[iChunk] = 0; - return LSM_IOERR_BKPT; - } - } - - *ppShm = p->apShm[iChunk]; - return LSM_OK; -} - -static void lsmPosixOsShmBarrier(void){ -} - -static int lsmPosixOsShmUnmap(lsm_file *pFile, int bDelete){ - PosixFile *p = (PosixFile *)pFile; - if( p->shmfd>0 ){ - int i; - for(i=0; inShm; i++){ - if( p->apShm[i] ){ - munmap(p->apShm[i], LSM_SHM_CHUNK_SIZE); - p->apShm[i] = 0; - } - } - close(p->shmfd); - p->shmfd = 0; - if( bDelete ){ - char *zShm = posixShmFile(p); - if( zShm ) unlink(zShm); - lsmFree(p->pEnv, zShm); - } - } - return LSM_OK; -} - - -static int lsmPosixOsClose(lsm_file *pFile){ - PosixFile *p = (PosixFile *)pFile; - lsmPosixOsShmUnmap(pFile, 0); - if( p->pMap ) munmap(p->pMap, p->nMap); - close(p->fd); - lsm_free(p->pEnv, p->apShm); - lsm_free(p->pEnv, p); - return LSM_OK; -} - -static int lsmPosixOsSleep(lsm_env *pEnv, int us){ -#if 0 - /* Apparently on Android usleep() returns void */ - if( usleep(us) ) return LSM_IOERR; -#endif - usleep(us); - return LSM_OK; -} - -/**************************************************************************** -** Memory allocation routines. -*/ -#define BLOCK_HDR_SIZE ROUND8( sizeof(size_t) ) - -static void *lsmPosixOsMalloc(lsm_env *pEnv, size_t N){ - unsigned char * m; - N += BLOCK_HDR_SIZE; - m = (unsigned char *)malloc(N); - *((size_t*)m) = N; - return m + BLOCK_HDR_SIZE; -} - -static void lsmPosixOsFree(lsm_env *pEnv, void *p){ - if(p){ - free( ((unsigned char *)p) - BLOCK_HDR_SIZE ); - } -} - -static void *lsmPosixOsRealloc(lsm_env *pEnv, void *p, size_t N){ - unsigned char * m = (unsigned char *)p; - if(1>N){ - lsmPosixOsFree( pEnv, p ); - return NULL; - }else if(NULL==p){ - return lsmPosixOsMalloc(pEnv, N); - }else{ - void * re = NULL; - m -= BLOCK_HDR_SIZE; -#if 0 /* arguable: don't shrink */ - size_t * sz = (size_t*)m; - if(*sz >= (size_t)N){ - return p; - } -#endif - re = realloc( m, N + BLOCK_HDR_SIZE ); - if(re){ - m = (unsigned char *)re; - *((size_t*)m) = N; - return m + BLOCK_HDR_SIZE; - }else{ - return NULL; - } - } -} - -static size_t lsmPosixOsMSize(lsm_env *pEnv, void *p){ - unsigned char * m = (unsigned char *)p; - return *((size_t*)(m-BLOCK_HDR_SIZE)); -} -#undef BLOCK_HDR_SIZE - - -#ifdef LSM_MUTEX_PTHREADS -/************************************************************************* -** Mutex methods for pthreads based systems. If LSM_MUTEX_PTHREADS is -** missing then a no-op implementation of mutexes found in lsm_mutex.c -** will be used instead. -*/ -#include - -typedef struct PthreadMutex PthreadMutex; -struct PthreadMutex { - lsm_env *pEnv; - pthread_mutex_t mutex; -#ifdef LSM_DEBUG - pthread_t owner; -#endif -}; - -#ifdef LSM_DEBUG -# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER, 0 } -#else -# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER } -#endif - -static int lsmPosixOsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - static PthreadMutex sMutex[2] = { - LSM_PTHREAD_STATIC_MUTEX, - LSM_PTHREAD_STATIC_MUTEX - }; - - assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); - assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); - - *ppStatic = (lsm_mutex *)&sMutex[iMutex-1]; - return LSM_OK; -} - -static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - PthreadMutex *pMutex; /* Pointer to new mutex */ - pthread_mutexattr_t attr; /* Attributes object */ - - pMutex = (PthreadMutex *)lsmMallocZero(pEnv, sizeof(PthreadMutex)); - if( !pMutex ) return LSM_NOMEM_BKPT; - - pMutex->pEnv = pEnv; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&pMutex->mutex, &attr); - pthread_mutexattr_destroy(&attr); - - *ppNew = (lsm_mutex *)pMutex; - return LSM_OK; -} - -static void lsmPosixOsMutexDel(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - pthread_mutex_destroy(&pMutex->mutex); - lsmFree(pMutex->pEnv, pMutex); -} - -static void lsmPosixOsMutexEnter(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - pthread_mutex_lock(&pMutex->mutex); - -#ifdef LSM_DEBUG - assert( !pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = pthread_self(); - assert( pthread_equal(pMutex->owner, pthread_self()) ); -#endif -} - -static int lsmPosixOsMutexTry(lsm_mutex *p){ - int ret; - PthreadMutex *pMutex = (PthreadMutex *)p; - ret = pthread_mutex_trylock(&pMutex->mutex); -#ifdef LSM_DEBUG - if( ret==0 ){ - assert( !pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = pthread_self(); - assert( pthread_equal(pMutex->owner, pthread_self()) ); - } -#endif - return ret; -} - -static void lsmPosixOsMutexLeave(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; -#ifdef LSM_DEBUG - assert( pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = 0; - assert( !pthread_equal(pMutex->owner, pthread_self()) ); -#endif - pthread_mutex_unlock(&pMutex->mutex); -} - -#ifdef LSM_DEBUG -static int lsmPosixOsMutexHeld(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - return pMutex ? pthread_equal(pMutex->owner, pthread_self()) : 1; -} -static int lsmPosixOsMutexNotHeld(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - return pMutex ? !pthread_equal(pMutex->owner, pthread_self()) : 1; -} -#endif -/* -** End of pthreads mutex implementation. -*************************************************************************/ -#else -/************************************************************************* -** Noop mutex implementation -*/ -typedef struct NoopMutex NoopMutex; -struct NoopMutex { - lsm_env *pEnv; /* Environment handle (for xFree()) */ - int bHeld; /* True if mutex is held */ - int bStatic; /* True for a static mutex */ -}; -static NoopMutex aStaticNoopMutex[2] = { - {0, 0, 1}, - {0, 0, 1}, -}; - -static int lsmPosixOsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); - *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; - return LSM_OK; -} -static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - NoopMutex *p; - p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); - if( p ) p->pEnv = pEnv; - *ppNew = (lsm_mutex *)p; - return (p ? LSM_OK : LSM_NOMEM_BKPT); -} -static void lsmPosixOsMutexDel(lsm_mutex *pMutex) { - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bStatic==0 && p->pEnv ); - lsmFree(p->pEnv, p); -} -static void lsmPosixOsMutexEnter(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; -} -static int lsmPosixOsMutexTry(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; - return 0; -} -static void lsmPosixOsMutexLeave(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==1 ); - p->bHeld = 0; -} -#ifdef LSM_DEBUG -static int lsmPosixOsMutexHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? p->bHeld : 1; -} -static int lsmPosixOsMutexNotHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? !p->bHeld : 1; -} -#endif -/***************************************************************************/ -#endif /* else LSM_MUTEX_NONE */ - -/* Without LSM_DEBUG, the MutexHeld tests are never called */ -#ifndef LSM_DEBUG -# define lsmPosixOsMutexHeld 0 -# define lsmPosixOsMutexNotHeld 0 -#endif - -lsm_env *lsm_default_env(void){ - static lsm_env posix_env = { - sizeof(lsm_env), /* nByte */ - 1, /* iVersion */ - /***** file i/o ******************/ - 0, /* pVfsCtx */ - lsmPosixOsFullpath, /* xFullpath */ - lsmPosixOsOpen, /* xOpen */ - lsmPosixOsRead, /* xRead */ - lsmPosixOsWrite, /* xWrite */ - lsmPosixOsTruncate, /* xTruncate */ - lsmPosixOsSync, /* xSync */ - lsmPosixOsSectorSize, /* xSectorSize */ - lsmPosixOsRemap, /* xRemap */ - lsmPosixOsFileid, /* xFileid */ - lsmPosixOsClose, /* xClose */ - lsmPosixOsUnlink, /* xUnlink */ - lsmPosixOsLock, /* xLock */ - lsmPosixOsTestLock, /* xTestLock */ - lsmPosixOsShmMap, /* xShmMap */ - lsmPosixOsShmBarrier, /* xShmBarrier */ - lsmPosixOsShmUnmap, /* xShmUnmap */ - /***** memory allocation *********/ - 0, /* pMemCtx */ - lsmPosixOsMalloc, /* xMalloc */ - lsmPosixOsRealloc, /* xRealloc */ - lsmPosixOsFree, /* xFree */ - lsmPosixOsMSize, /* xSize */ - /***** mutexes *********************/ - 0, /* pMutexCtx */ - lsmPosixOsMutexStatic, /* xMutexStatic */ - lsmPosixOsMutexNew, /* xMutexNew */ - lsmPosixOsMutexDel, /* xMutexDel */ - lsmPosixOsMutexEnter, /* xMutexEnter */ - lsmPosixOsMutexTry, /* xMutexTry */ - lsmPosixOsMutexLeave, /* xMutexLeave */ - lsmPosixOsMutexHeld, /* xMutexHeld */ - lsmPosixOsMutexNotHeld, /* xMutexNotHeld */ - /***** other *********************/ - lsmPosixOsSleep, /* xSleep */ - }; - return &posix_env; -} - -#endif diff --git a/ext/lsm1/lsm_varint.c b/ext/lsm1/lsm_varint.c deleted file mode 100644 index f690e3b063..0000000000 --- a/ext/lsm1/lsm_varint.c +++ /dev/null @@ -1,201 +0,0 @@ - -/* -** 2012-02-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** SQLite4-compatible varint implementation. -*/ -#include "lsmInt.h" - -/************************************************************************* -** The following is a copy of the varint.c module from SQLite 4. -*/ - -/* -** Decode the varint in z[]. Write the integer value into *pResult and -** return the number of bytes in the varint. -*/ -static int lsmSqlite4GetVarint64(const unsigned char *z, u64 *pResult){ - unsigned int x; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( z[0]==249 ){ - *pResult = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *pResult = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - x = (z[1]<<24) + (z[2]<<16) + (z[3]<<8) + z[4]; - if( z[0]==251 ){ - *pResult = x; - return 5; - } - if( z[0]==252 ){ - *pResult = (((u64)x)<<8) + z[5]; - return 6; - } - if( z[0]==253 ){ - *pResult = (((u64)x)<<16) + (z[5]<<8) + z[6]; - return 7; - } - if( z[0]==254 ){ - *pResult = (((u64)x)<<24) + (z[5]<<16) + (z[6]<<8) + z[7]; - return 8; - } - *pResult = (((u64)x)<<32) + - (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); - return 9; -} - -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void lsmVarintWrite32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - -/* -** Write a varint into z[]. The buffer z[] must be at least 9 characters -** long to accommodate the largest possible varint. Return the number of -** bytes of z[] used. -*/ -static int lsmSqlite4PutVarint64(unsigned char *z, u64 x){ - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - lsmVarintWrite32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - lsmVarintWrite32(z+2, y); - return 6; - } - if( w<=32767 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - lsmVarintWrite32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - lsmVarintWrite32(z+4, y); - return 8; - } - z[0] = 255; - lsmVarintWrite32(z+1, w); - lsmVarintWrite32(z+5, y); - return 9; -} - -/* -** End of SQLite 4 code. -*************************************************************************/ - -int lsmVarintPut64(u8 *aData, i64 iVal){ - return lsmSqlite4PutVarint64(aData, (u64)iVal); -} - -int lsmVarintGet64(const u8 *aData, i64 *piVal){ - return lsmSqlite4GetVarint64(aData, (u64 *)piVal); -} - -int lsmVarintPut32(u8 *aData, int iVal){ - return lsmSqlite4PutVarint64(aData, (u64)iVal); -} - -int lsmVarintGet32(u8 *z, int *piVal){ - u64 i; - int ret; - - if( z[0]<=240 ){ - *piVal = z[0]; - return 1; - } - if( z[0]<=248 ){ - *piVal = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( z[0]==249 ){ - *piVal = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *piVal = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - - ret = lsmSqlite4GetVarint64(z, &i); - *piVal = (int)i; - return ret; -} - -int lsmVarintLen32(int n){ - u8 aData[9]; - return lsmVarintPut32(aData, n); -} - -int lsmVarintLen64(i64 n){ - u8 aData[9]; - return lsmVarintPut64(aData, n); -} - -/* -** The argument is the first byte of a varint. This function returns the -** total number of bytes in the entire varint (including the first byte). -*/ -int lsmVarintSize(u8 c){ - if( c<241 ) return 1; - if( c<249 ) return 2; - return (int)(c - 246); -} diff --git a/ext/lsm1/lsm_vtab.c b/ext/lsm1/lsm_vtab.c deleted file mode 100644 index bb1460297d..0000000000 --- a/ext/lsm1/lsm_vtab.c +++ /dev/null @@ -1,1079 +0,0 @@ -/* -** 2015-11-16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a virtual table for SQLite3 around the LSM -** storage engine from SQLite4. -** -** USAGE -** -** CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...); -** -** The filename parameter is the name of the LSM database file, which is -** separate and distinct from the SQLite3 database file. -** -** The keytype must be one of: UINT, TEXT, BLOB. All keys must be of that -** one type. "UINT" means unsigned integer. The values may be of any -** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL. -** -** The virtual table contains read-only hidden columns: -** -** lsm1_key A BLOB which is the raw LSM key. If the "keytype" -** is BLOB or TEXT then this column is exactly the -** same as the key. For the UINT keytype, this column -** will be a variable-length integer encoding of the key. -** -** lsm1_value A BLOB which is the raw LSM value. All of the value -** columns are packed into this BLOB using the encoding -** described below. -** -** Attempts to write values into the lsm1_key and lsm1_value columns are -** silently ignored. -** -** EXAMPLE -** -** The virtual table declared this way: -** -** CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d); -** -** Results in a new virtual table named "demo2" that acts as if it has -** the following schema: -** -** CREATE TABLE demo2( -** id UINT PRIMARY KEY ON CONFLICT REPLACE, -** a ANY, -** b ANY, -** c ANY, -** d ANY, -** lsm1_key BLOB HIDDEN, -** lsm1_value BLOB HIDDEN -** ) WITHOUT ROWID; -** -** -** -** INTERNALS -** -** The key encoding for BLOB and TEXT is just a copy of the blob or text. -** UTF-8 is used for text. The key encoding for UINT is the variable-length -** integer format at https://sqlite.org/src4/doc/trunk/www/varint.wiki. -** -** The values are encoded as a single blob (since that is what lsm stores as -** its content). There is a "type integer" followed by "content" for each -** value, alternating back and forth. The content might be empty. -** -** TYPE1 CONTENT1 TYPE2 CONTENT2 TYPE3 CONTENT3 .... -** -** Each "type integer" is encoded as a variable-length integer in the -** format of the link above. Let the type integer be T. The actual -** datatype is an integer 0-5 equal to T%6. Values 1 through 5 correspond -** to SQLITE_INTEGER through SQLITE_NULL. The size of the content in bytes -** is T/6. Type value 0 means that the value is an integer whose actual -** values is T/6 and there is no content. The type-value-0 integer format -** only works for integers in the range of 0 through 40. -** -** There is no content for NULL or type-0 integers. For BLOB and TEXT -** values, the content is the blob data or the UTF-8 text data. For -** non-negative integers X, the content is a variable-length integer X*2. -** For negative integers Y, the content is varaible-length integer (1-Y)*2+1. -** For FLOAT values, the content is the IEEE754 floating point value in -** native byte-order. This means that FLOAT values will be corrupted when -** database file is moved between big-endian and little-endian machines. -*/ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 -#include "lsm.h" -#include -#include - -/* Forward declaration of subclasses of virtual table objects */ -typedef struct lsm1_vtab lsm1_vtab; -typedef struct lsm1_cursor lsm1_cursor; -typedef struct lsm1_vblob lsm1_vblob; - -/* Primitive types */ -typedef unsigned char u8; -typedef unsigned int u32; -typedef sqlite3_uint64 u64; - -/* An open connection to an LSM table */ -struct lsm1_vtab { - sqlite3_vtab base; /* Base class - must be first */ - lsm_db *pDb; /* Open connection to the LSM table */ - u8 keyType; /* SQLITE_BLOB, _TEXT, or _INTEGER */ - u32 nVal; /* Number of value columns */ -}; - - -/* lsm1_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -struct lsm1_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - lsm_cursor *pLsmCur; /* The LSM cursor */ - u8 isDesc; /* 0: scan forward. 1: scan reverse */ - u8 atEof; /* True if the scan is complete */ - u8 bUnique; /* True if no more than one row of output */ - u8 *zData; /* Content of the current row */ - u32 nData; /* Number of bytes in the current row */ - u8 *aeType; /* Types for all column values */ - u32 *aiOfst; /* Offsets to the various fields */ - u32 *aiLen; /* Length of each field */ - u8 *pKey2; /* Loop termination key, or NULL */ - u32 nKey2; /* Length of the loop termination key */ -}; - -/* An extensible buffer object. -** -** Content can be appended. Space to hold new content is automatically -** allocated. -*/ -struct lsm1_vblob { - u8 *a; /* Space to hold content, from sqlite3_malloc64() */ - u64 n; /* Bytes of space used */ - u64 nAlloc; /* Bytes of space allocated */ - u8 errNoMem; /* True if a memory allocation error has been seen */ -}; - -#if defined(__GNUC__) -# define LSM1_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define LSM1_NOINLINE __declspec(noinline) -#else -# define LSM1_NOINLINE -#endif - - -/* Increase the available space in the vblob object so that it can hold -** at least N more bytes. Return the number of errors. -*/ -static int lsm1VblobEnlarge(lsm1_vblob *p, u32 N){ - if( p->n+N>p->nAlloc ){ - if( p->errNoMem ) return 1; - p->nAlloc += N + (p->nAlloc ? p->nAlloc : N); - p->a = sqlite3_realloc64(p->a, p->nAlloc); - if( p->a==0 ){ - p->n = 0; - p->nAlloc = 0; - p->errNoMem = 1; - return 1; - } - p->nAlloc = sqlite3_msize(p->a); - } - return 0; -} - -/* Append N bytes to a vblob after first enlarging it */ -static LSM1_NOINLINE void lsm1VblobEnlargeAndAppend( - lsm1_vblob *p, - const u8 *pData, - u32 N -){ - if( p->n+N>p->nAlloc && lsm1VblobEnlarge(p, N) ) return; - memcpy(p->a+p->n, pData, N); - p->n += N; -} - -/* Append N bytes to a vblob */ -static void lsm1VblobAppend(lsm1_vblob *p, const u8 *pData, u32 N){ - sqlite3_int64 n = p->n; - if( n+N>p->nAlloc ){ - lsm1VblobEnlargeAndAppend(p, pData, N); - }else{ - p->n += N; - memcpy(p->a+n, pData, N); - } -} - -/* append text to a vblob */ -static void lsm1VblobAppendText(lsm1_vblob *p, const char *z){ - lsm1VblobAppend(p, (u8*)z, (u32)strlen(z)); -} - -/* Dequote the string */ -static void lsm1Dequote(char *z){ - int j; - char cQuote = z[0]; - size_t i, n; - - if( cQuote!='\'' && cQuote!='"' ) return; - n = strlen(z); - if( n<2 || z[n-1]!=z[0] ) return; - for(i=1, j=0; ikeyType = keyType; - rc = lsm_new(0, &pNew->pDb); - if( rc ){ - *pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc); - rc = SQLITE_ERROR; - goto connect_failed; - } - zFilename = sqlite3_mprintf("%s", argv[3]); - lsm1Dequote(zFilename); - rc = lsm_open(pNew->pDb, zFilename); - sqlite3_free(zFilename); - if( rc ){ - *pzErr = sqlite3_mprintf("lsm_open failed with %d", rc); - rc = SQLITE_ERROR; - goto connect_failed; - } - - memset(&sql, 0, sizeof(sql)); - lsm1VblobAppendText(&sql, "CREATE TABLE x("); - lsm1VblobAppendText(&sql, argv[4]); - lsm1VblobAppendText(&sql, " "); - lsm1VblobAppendText(&sql, argv[5]); - lsm1VblobAppendText(&sql, " PRIMARY KEY"); - for(i=6; inVal++; - } - lsm1VblobAppendText(&sql, - ", lsm1_command HIDDEN" - ", lsm1_key HIDDEN" - ", lsm1_value HIDDEN) WITHOUT ROWID"); - lsm1VblobAppend(&sql, (u8*)"", 1); - if( sql.errNoMem ){ - rc = SQLITE_NOMEM; - goto connect_failed; - } - rc = sqlite3_declare_vtab(db, (const char*)sql.a); - sqlite3_free(sql.a); - -connect_failed: - if( rc!=SQLITE_OK ){ - if( pNew ){ - if( pNew->pDb ) lsm_close(pNew->pDb); - sqlite3_free(pNew); - } - *ppVtab = 0; - } - return rc; -} - -/* -** This method is the destructor for lsm1_cursor objects. -*/ -static int lsm1Disconnect(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - lsm_close(p->pDb); - sqlite3_free(p); - return SQLITE_OK; -} - -/* -** Constructor for a new lsm1_cursor object. -*/ -static int lsm1Open(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - lsm1_cursor *pCur; - int rc; - pCur = sqlite3_malloc64( sizeof(*pCur) - + p->nVal*(sizeof(pCur->aiOfst)+sizeof(pCur->aiLen)+1) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - pCur->aiOfst = (u32*)&pCur[1]; - pCur->aiLen = &pCur->aiOfst[p->nVal]; - pCur->aeType = (u8*)&pCur->aiLen[p->nVal]; - *ppCursor = &pCur->base; - rc = lsm_csr_open(p->pDb, &pCur->pLsmCur); - if( rc==LSM_OK ){ - rc = SQLITE_OK; - }else{ - sqlite3_free(pCur); - *ppCursor = 0; - rc = SQLITE_ERROR; - } - return rc; -} - -/* -** Destructor for a lsm1_cursor. -*/ -static int lsm1Close(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - sqlite3_free(pCur->pKey2); - lsm_csr_close(pCur->pLsmCur); - sqlite3_free(pCur); - return SQLITE_OK; -} - - -/* -** Advance a lsm1_cursor to its next row of output. -*/ -static int lsm1Next(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - int rc = LSM_OK; - if( pCur->bUnique ){ - pCur->atEof = 1; - }else{ - if( pCur->isDesc ){ - rc = lsm_csr_prev(pCur->pLsmCur); - }else{ - rc = lsm_csr_next(pCur->pLsmCur); - } - if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)==0 ){ - pCur->atEof = 1; - } - if( pCur->pKey2 && pCur->atEof==0 ){ - const u8 *pVal; - u32 nVal; - assert( pCur->isDesc==0 ); - rc = lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, (int*)&nVal); - if( rc==LSM_OK ){ - u32 len = pCur->nKey2; - int c; - if( len>nVal ) len = nVal; - c = memcmp(pVal, pCur->pKey2, len); - if( c==0 ) c = nVal - pCur->nKey2; - if( c>0 ) pCur->atEof = 1; - } - } - pCur->zData = 0; - } - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** Return TRUE if the cursor has been moved off of the last -** row of output. -*/ -static int lsm1Eof(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - return pCur->atEof; -} - -/* -** Rowids are not supported by the underlying virtual table. So always -** return 0 for the rowid. -*/ -static int lsm1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - *pRowid = 0; - return SQLITE_OK; -} - -/* -** Type prefixes on LSM keys -*/ -#define LSM1_TYPE_NEGATIVE 0 -#define LSM1_TYPE_POSITIVE 1 -#define LSM1_TYPE_TEXT 2 -#define LSM1_TYPE_BLOB 3 - -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void varintWrite32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - -/* -** Write a varint into z[]. The buffer z[] must be at least 9 characters -** long to accommodate the largest possible varint. Return the number of -** bytes of z[] used. -*/ -static int lsm1PutVarint64(unsigned char *z, sqlite3_uint64 x){ - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - varintWrite32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - varintWrite32(z+2, y); - return 6; - } - if( w<=65535 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - varintWrite32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - varintWrite32(z+4, y); - return 8; - } - z[0] = 255; - varintWrite32(z+1, w); - varintWrite32(z+5, y); - return 9; -} - -/* Append non-negative integer x as a variable-length integer. -*/ -static void lsm1VblobAppendVarint(lsm1_vblob *p, sqlite3_uint64 x){ - sqlite3_int64 n = p->n; - if( n+9>p->nAlloc && lsm1VblobEnlarge(p, 9) ) return; - p->n += lsm1PutVarint64(p->a+p->n, x); -} - -/* -** Decode the varint in the first n bytes z[]. Write the integer value -** into *pResult and return the number of bytes in the varint. -** -** If the decode fails because there are not enough bytes in z[] then -** return 0; -*/ -static int lsm1GetVarint64( - const unsigned char *z, - int n, - sqlite3_uint64 *pResult -){ - unsigned int x; - if( n<1 ) return 0; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - if( n<2 ) return 0; - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( n=0 ){ - u = (sqlite3_uint64)v; - return lsm1PutVarint64(z, u*2); - }else{ - u = (sqlite3_uint64)(-1-v); - return lsm1PutVarint64(z, u*2+1); - } -} - -/* Decoded a signed varint. */ -static int lsm1GetSignedVarint64( - const unsigned char *z, - int n, - sqlite3_int64 *pResult -){ - sqlite3_uint64 u = 0; - n = lsm1GetVarint64(z, n, &u); - if( u&1 ){ - *pResult = -1 - (sqlite3_int64)(u>>1); - }else{ - *pResult = (sqlite3_int64)(u>>1); - } - return n; -} - - -/* -** Read the value part of the key-value pair and decode it into columns. -*/ -static int lsm1DecodeValues(lsm1_cursor *pCur){ - lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); - int i, n; - int rc; - u8 eType; - sqlite3_uint64 v; - - if( pCur->zData ) return 1; - rc = lsm_csr_value(pCur->pLsmCur, (const void**)&pCur->zData, - (int*)&pCur->nData); - if( rc ) return 0; - for(i=n=0; inVal; i++){ - v = 0; - n += lsm1GetVarint64(pCur->zData+n, pCur->nData-n, &v); - pCur->aeType[i] = eType = (u8)(v%6); - if( eType==0 ){ - pCur->aiOfst[i] = (u32)(v/6); - pCur->aiLen[i] = 0; - }else{ - pCur->aiOfst[i] = n; - n += (pCur->aiLen[i] = (u32)(v/6)); - } - if( n>pCur->nData ) break; - } - if( inVal ){ - pCur->zData = 0; - return 0; - } - return 1; -} - -/* -** Return values of columns for the row at which the lsm1_cursor -** is currently pointing. -*/ -static int lsm1Column( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ -){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - lsm1_vtab *pTab = (lsm1_vtab*)(cur->pVtab); - if( i==0 ){ - /* The key column */ - const void *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - if( pTab->keyType==SQLITE_BLOB ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - }else if( pTab->keyType==SQLITE_TEXT ){ - sqlite3_result_text(ctx,(const char*)pVal, nVal, SQLITE_TRANSIENT); - }else{ - const unsigned char *z = (const unsigned char*)pVal; - sqlite3_uint64 v1; - lsm1GetVarint64(z, nVal, &v1); - sqlite3_result_int64(ctx, (sqlite3_int64)v1); - } - } - }else if( i>pTab->nVal ){ - if( i==pTab->nVal+2 ){ /* lsm1_key */ - const void *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - } - }else if( i==pTab->nVal+3 ){ /* lsm1_value */ - const void *pVal; - int nVal; - if( lsm_csr_value(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - } - } - }else if( lsm1DecodeValues(pCur) ){ - /* The i-th value column (where leftmost is 1) */ - const u8 *zData; - u32 nData; - i--; - zData = pCur->zData + pCur->aiOfst[i]; - nData = pCur->aiLen[i]; - switch( pCur->aeType[i] ){ - case 0: { /* in-line integer */ - sqlite3_result_int(ctx, pCur->aiOfst[i]); - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 v; - lsm1GetSignedVarint64(zData, nData, &v); - sqlite3_result_int64(ctx, v); - break; - } - case SQLITE_FLOAT: { - double v; - if( nData==sizeof(v) ){ - memcpy(&v, zData, sizeof(v)); - sqlite3_result_double(ctx, v); - } - break; - } - case SQLITE_TEXT: { - sqlite3_result_text(ctx, (const char*)zData, nData, SQLITE_TRANSIENT); - break; - } - case SQLITE_BLOB: { - sqlite3_result_blob(ctx, zData, nData, SQLITE_TRANSIENT); - break; - } - default: { - /* A NULL. Do nothing */ - } - } - } - return SQLITE_OK; -} - -/* Parameter "pValue" contains an SQL value that is to be used as -** a key in an LSM table. The type of the key is determined by -** "keyType". Extract the raw bytes used for the key in LSM1. -*/ -static void lsm1KeyFromValue( - int keyType, /* The key type */ - sqlite3_value *pValue, /* The key value */ - u8 *pBuf, /* Storage space for a generated key */ - const u8 **ppKey, /* OUT: the bytes of the key */ - int *pnKey /* OUT: size of the key */ -){ - if( keyType==SQLITE_BLOB ){ - *ppKey = (const u8*)sqlite3_value_blob(pValue); - *pnKey = sqlite3_value_bytes(pValue); - }else if( keyType==SQLITE_TEXT ){ - *ppKey = (const u8*)sqlite3_value_text(pValue); - *pnKey = sqlite3_value_bytes(pValue); - }else{ - sqlite3_int64 v = sqlite3_value_int64(pValue); - if( v<0 ) v = 0; - *pnKey = lsm1PutVarint64(pBuf, v); - *ppKey = pBuf; - } -} - -/* Move to the first row to return. -*/ -static int lsm1Filter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - lsm1_cursor *pCur = (lsm1_cursor *)pVtabCursor; - lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); - int rc = LSM_OK; - int seekType = -1; - const u8 *pVal = 0; - int nVal; - u8 keyType = pTab->keyType; - u8 aKey1[16]; - - pCur->atEof = 1; - sqlite3_free(pCur->pKey2); - pCur->pKey2 = 0; - if( idxNum<99 ){ - lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal); - } - switch( idxNum ){ - case 0: { /* key==argv[0] */ - assert( argc==1 ); - seekType = LSM_SEEK_EQ; - pCur->isDesc = 0; - pCur->bUnique = 1; - break; - } - case 1: { /* key>=argv[0] AND key<=argv[1] */ - u8 aKey[12]; - seekType = LSM_SEEK_GE; - pCur->isDesc = 0; - pCur->bUnique = 0; - if( keyType==SQLITE_INTEGER ){ - sqlite3_int64 v = sqlite3_value_int64(argv[1]); - if( v<0 ) v = 0; - pCur->nKey2 = lsm1PutVarint64(aKey, (sqlite3_uint64)v); - pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); - if( pCur->pKey2==0 ) return SQLITE_NOMEM; - memcpy(pCur->pKey2, aKey, pCur->nKey2); - }else{ - pCur->nKey2 = sqlite3_value_bytes(argv[1]); - pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); - if( pCur->pKey2==0 ) return SQLITE_NOMEM; - if( keyType==SQLITE_BLOB ){ - memcpy(pCur->pKey2, sqlite3_value_blob(argv[1]), pCur->nKey2); - }else{ - memcpy(pCur->pKey2, sqlite3_value_text(argv[1]), pCur->nKey2); - } - } - break; - } - case 2: { /* key>=argv[0] */ - seekType = LSM_SEEK_GE; - pCur->isDesc = 0; - pCur->bUnique = 0; - break; - } - case 3: { /* key<=argv[0] */ - seekType = LSM_SEEK_LE; - pCur->isDesc = 1; - pCur->bUnique = 0; - break; - } - default: { /* full table scan */ - pCur->isDesc = 0; - pCur->bUnique = 0; - break; - } - } - if( pVal ){ - rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, seekType); - }else{ - rc = lsm_csr_first(pCur->pLsmCur); - } - if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)!=0 ){ - pCur->atEof = 0; - } - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** Only comparisons against the key are allowed. The idxNum defines -** which comparisons are available: -** -** 0 key==?1 -** 1 key>=?1 AND key<=?2 -** 2 key>?1 or key>=?1 -** 3 keyaConstraint; - for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->iColumn!=0 ) continue; - switch( pConstraint->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: { - if( idxNum>0 ){ - argIdx = i; - iIdx2 = -1; - idxNum = 0; - omit1 = 1; - } - break; - } - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: { - if( idxNum==99 ){ - argIdx = i; - idxNum = 2; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; - }else if( idxNum==3 ){ - iIdx2 = idxNum; - omit2 = omit1; - argIdx = i; - idxNum = 1; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; - } - break; - } - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: { - if( idxNum==99 ){ - argIdx = i; - idxNum = 3; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; - }else if( idxNum==2 ){ - iIdx2 = i; - idxNum = 1; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; - } - break; - } - } - } - if( argIdx>=0 ){ - pIdxInfo->aConstraintUsage[argIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[argIdx].omit = omit1; - } - if( iIdx2>=0 ){ - pIdxInfo->aConstraintUsage[iIdx2].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[iIdx2].omit = omit2; - } - if( idxNum==0 ){ - pIdxInfo->estimatedCost = (double)1; - pIdxInfo->estimatedRows = 1; - pIdxInfo->orderByConsumed = 1; - }else if( idxNum==1 ){ - pIdxInfo->estimatedCost = (double)100; - pIdxInfo->estimatedRows = 100; - }else if( idxNum<99 ){ - pIdxInfo->estimatedCost = (double)5000; - pIdxInfo->estimatedRows = 5000; - }else{ - /* Full table scan */ - pIdxInfo->estimatedCost = (double)2147483647; - pIdxInfo->estimatedRows = 2147483647; - } - pIdxInfo->idxNum = idxNum; - return SQLITE_OK; -} - -/* -** The xUpdate method is normally used for INSERT, REPLACE, UPDATE, and -** DELETE. But this virtual table only supports INSERT and REPLACE. -** DELETE is accomplished by inserting a record with a value of NULL. -** UPDATE is achieved by using REPLACE. -*/ -int lsm1Update( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - lsm1_vtab *p = (lsm1_vtab*)pVTab; - int nKey, nKey2; - int i; - int rc = LSM_OK; - const u8 *pKey, *pKey2; - unsigned char aKey[16]; - unsigned char pSpace[16]; - lsm1_vblob val; - - if( argc==1 ){ - /* DELETE the record whose key is argv[0] */ - lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); - lsm_delete(p->pDb, pKey, nKey); - return SQLITE_OK; - } - - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - /* An UPDATE */ - lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); - lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2); - if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){ - /* The UPDATE changes the PRIMARY KEY value. DELETE the old key */ - lsm_delete(p->pDb, pKey, nKey); - } - /* Fall through into the INSERT case to complete the UPDATE */ - } - - /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement - ** special commands. - */ - if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){ - return SQLITE_OK; - } - lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey); - memset(&val, 0, sizeof(val)); - for(i=0; inVal; i++){ - sqlite3_value *pArg = argv[3+i]; - u8 eType = sqlite3_value_type(pArg); - switch( eType ){ - case SQLITE_NULL: { - lsm1VblobAppendVarint(&val, SQLITE_NULL); - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 v = sqlite3_value_int64(pArg); - if( v>=0 && v<=240/6 ){ - lsm1VblobAppendVarint(&val, v*6); - }else{ - int n = lsm1PutSignedVarint64(pSpace, v); - lsm1VblobAppendVarint(&val, SQLITE_INTEGER + n*6); - lsm1VblobAppend(&val, pSpace, n); - } - break; - } - case SQLITE_FLOAT: { - double r = sqlite3_value_double(pArg); - lsm1VblobAppendVarint(&val, SQLITE_FLOAT + 8*6); - lsm1VblobAppend(&val, (u8*)&r, sizeof(r)); - break; - } - case SQLITE_BLOB: { - int n = sqlite3_value_bytes(pArg); - lsm1VblobAppendVarint(&val, n*6 + SQLITE_BLOB); - lsm1VblobAppend(&val, sqlite3_value_blob(pArg), n); - break; - } - case SQLITE_TEXT: { - int n = sqlite3_value_bytes(pArg); - lsm1VblobAppendVarint(&val, n*6 + SQLITE_TEXT); - lsm1VblobAppend(&val, sqlite3_value_text(pArg), n); - break; - } - } - } - if( val.errNoMem ){ - return SQLITE_NOMEM; - } - rc = lsm_insert(p->pDb, pKey, nKey, val.a, val.n); - sqlite3_free(val.a); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Begin a transaction -*/ -static int lsm1Begin(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_begin(p->pDb, 1); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Phase 1 of a transaction commit. -*/ -static int lsm1Sync(sqlite3_vtab *pVtab){ - return SQLITE_OK; -} - -/* Commit a transaction -*/ -static int lsm1Commit(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_commit(p->pDb, 0); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Rollback a transaction -*/ -static int lsm1Rollback(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_rollback(p->pDb, 0); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** This following structure defines all the methods for the -** generate_lsm1 virtual table. -*/ -static sqlite3_module lsm1Module = { - 0, /* iVersion */ - lsm1Connect, /* xCreate */ - lsm1Connect, /* xConnect */ - lsm1BestIndex, /* xBestIndex */ - lsm1Disconnect, /* xDisconnect */ - lsm1Disconnect, /* xDestroy */ - lsm1Open, /* xOpen - open a cursor */ - lsm1Close, /* xClose - close a cursor */ - lsm1Filter, /* xFilter - configure scan constraints */ - lsm1Next, /* xNext - advance a cursor */ - lsm1Eof, /* xEof - check for end of scan */ - lsm1Column, /* xColumn - read data */ - lsm1Rowid, /* xRowid - read data */ - lsm1Update, /* xUpdate */ - lsm1Begin, /* xBegin */ - lsm1Sync, /* xSync */ - lsm1Commit, /* xCommit */ - lsm1Rollback, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ -}; - - -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_lsm_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - rc = sqlite3_create_module(db, "lsm1", &lsm1Module, 0); - return rc; -} diff --git a/ext/lsm1/lsm_win32.c b/ext/lsm1/lsm_win32.c deleted file mode 100644 index 6c5d06b4c8..0000000000 --- a/ext/lsm1/lsm_win32.c +++ /dev/null @@ -1,1063 +0,0 @@ -/* -** 2011-12-03 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Win32-specific run-time environment implementation for LSM. -*/ - -#ifdef _WIN32 - -#include -#include - -#include -#include -#include -#include - -#include "windows.h" - -#include "lsmInt.h" - -/* -** An open file is an instance of the following object -*/ -typedef struct Win32File Win32File; -struct Win32File { - lsm_env *pEnv; /* The run-time environment */ - const char *zName; /* Full path to file */ - - HANDLE hFile; /* Open file handle */ - HANDLE hShmFile; /* File handle for *-shm file */ - - SYSTEM_INFO sysInfo; /* Operating system information */ - HANDLE hMap; /* File handle for mapping */ - LPVOID pMap; /* Pointer to mapping of file fd */ - size_t nMap; /* Size of mapping at pMap in bytes */ - int nShm; /* Number of entries in ahShm[]/apShm[] */ - LPHANDLE ahShm; /* Array of handles for shared mappings */ - LPVOID *apShm; /* Array of 32K shared memory segments */ -}; - -static char *win32ShmFile(Win32File *pWin32File){ - char *zShm; - int nName = strlen(pWin32File->zName); - zShm = (char *)lsmMallocZero(pWin32File->pEnv, nName+4+1); - if( zShm ){ - memcpy(zShm, pWin32File->zName, nName); - memcpy(&zShm[nName], "-shm", 5); - } - return zShm; -} - -static int win32Sleep(int us){ - Sleep((us + 999) / 1000); - return LSM_OK; -} - -/* -** The number of times that an I/O operation will be retried following a -** locking error - probably caused by antivirus software. Also the initial -** delay before the first retry. The delay increases linearly with each -** retry. -*/ -#ifndef LSM_WIN32_IOERR_RETRY -# define LSM_WIN32_IOERR_RETRY 10 -#endif -#ifndef LSM_WIN32_IOERR_RETRY_DELAY -# define LSM_WIN32_IOERR_RETRY_DELAY 25000 -#endif -static int win32IoerrRetry = LSM_WIN32_IOERR_RETRY; -static int win32IoerrRetryDelay = LSM_WIN32_IOERR_RETRY_DELAY; - -/* -** The "win32IoerrCanRetry1" macro is used to determine if a particular -** I/O error code obtained via GetLastError() is eligible to be retried. -** It must accept the error code DWORD as its only argument and should -** return non-zero if the error code is transient in nature and the -** operation responsible for generating the original error might succeed -** upon being retried. The argument to this macro should be a variable. -** -** Additionally, a macro named "win32IoerrCanRetry2" may be defined. If -** it is defined, it will be consulted only when the macro -** "win32IoerrCanRetry1" returns zero. The "win32IoerrCanRetry2" macro -** is completely optional and may be used to include additional error -** codes in the set that should result in the failing I/O operation being -** retried by the caller. If defined, the "win32IoerrCanRetry2" macro -** must exhibit external semantics identical to those of the -** "win32IoerrCanRetry1" macro. -*/ -#if !defined(win32IoerrCanRetry1) -#define win32IoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ - ((a)==ERROR_SHARING_VIOLATION) || \ - ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_DEV_NOT_EXIST) || \ - ((a)==ERROR_NETNAME_DELETED) || \ - ((a)==ERROR_SEM_TIMEOUT) || \ - ((a)==ERROR_NETWORK_UNREACHABLE)) -#endif - -/* -** If an I/O error occurs, invoke this routine to see if it should be -** retried. Return TRUE to retry. Return FALSE to give up with an -** error. -*/ -static int win32RetryIoerr( - lsm_env *pEnv, - int *pnRetry -){ - DWORD lastErrno; - if( *pnRetry>=win32IoerrRetry ){ - return 0; - } - lastErrno = GetLastError(); - if( win32IoerrCanRetry1(lastErrno) ){ - win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); - ++*pnRetry; - return 1; - } -#if defined(win32IoerrCanRetry2) - else if( win32IoerrCanRetry2(lastErrno) ){ - win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); - ++*pnRetry; - return 1; - } -#endif - return 0; -} - -/* -** Convert a UTF-8 string to Microsoft Unicode. -** -** Space to hold the returned string is obtained from lsmMalloc(). -*/ -static LPWSTR win32Utf8ToUnicode(lsm_env *pEnv, const char *zText){ - int nChar; - LPWSTR zWideText; - - nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); - if( nChar==0 ){ - return 0; - } - zWideText = lsmMallocZero(pEnv, nChar * sizeof(WCHAR)); - if( zWideText==0 ){ - return 0; - } - nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); - if( nChar==0 ){ - lsmFree(pEnv, zWideText); - zWideText = 0; - } - return zWideText; -} - -/* -** Convert a Microsoft Unicode string to UTF-8. -** -** Space to hold the returned string is obtained from lsmMalloc(). -*/ -static char *win32UnicodeToUtf8(lsm_env *pEnv, LPCWSTR zWideText){ - int nByte; - char *zText; - - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); - if( nByte == 0 ){ - return 0; - } - zText = lsmMallocZero(pEnv, nByte); - if( zText==0 ){ - return 0; - } - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); - if( nByte == 0 ){ - lsmFree(pEnv, zText); - zText = 0; - } - return zText; -} - -#if !defined(win32IsNotFound) -#define win32IsNotFound(a) (((a)==ERROR_FILE_NOT_FOUND) || \ - ((a)==ERROR_PATH_NOT_FOUND)) -#endif - -static int win32Open( - lsm_env *pEnv, - const char *zFile, - int flags, - LPHANDLE phFile -){ - int rc; - LPWSTR zConverted; - - zConverted = win32Utf8ToUnicode(pEnv, zFile); - if( zConverted==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - int bReadonly = (flags & LSM_OPEN_READONLY); - DWORD dwDesiredAccess; - DWORD dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - DWORD dwCreationDisposition; - DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; - HANDLE hFile; - int nRetry = 0; - if( bReadonly ){ - dwDesiredAccess = GENERIC_READ; - dwCreationDisposition = OPEN_EXISTING; - }else{ - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - dwCreationDisposition = OPEN_ALWAYS; - } - while( (hFile = CreateFileW((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - win32RetryIoerr(pEnv, &nRetry) ){ - /* Noop */ - } - lsmFree(pEnv, zConverted); - if( hFile!=INVALID_HANDLE_VALUE ){ - *phFile = hFile; - rc = LSM_OK; - }else{ - if( win32IsNotFound(GetLastError()) ){ - rc = lsmErrorBkpt(LSM_IOERR_NOENT); - }else{ - rc = LSM_IOERR_BKPT; - } - } - } - return rc; -} - -static int lsmWin32OsOpen( - lsm_env *pEnv, - const char *zFile, - int flags, - lsm_file **ppFile -){ - int rc = LSM_OK; - Win32File *pWin32File; - - pWin32File = lsmMallocZero(pEnv, sizeof(Win32File)); - if( pWin32File==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - HANDLE hFile = NULL; - - rc = win32Open(pEnv, zFile, flags, &hFile); - if( rc==LSM_OK ){ - memset(&pWin32File->sysInfo, 0, sizeof(SYSTEM_INFO)); - GetSystemInfo(&pWin32File->sysInfo); - pWin32File->pEnv = pEnv; - pWin32File->zName = zFile; - pWin32File->hFile = hFile; - }else{ - lsmFree(pEnv, pWin32File); - pWin32File = 0; - } - } - *ppFile = (lsm_file *)pWin32File; - return rc; -} - -static int lsmWin32OsWrite( - lsm_file *pFile, /* File to write to */ - lsm_i64 iOff, /* Offset to write to */ - void *pData, /* Write data from this buffer */ - int nData /* Bytes of data to write */ -){ - Win32File *pWin32File = (Win32File *)pFile; - OVERLAPPED overlapped; /* The offset for WriteFile. */ - u8 *aRem = (u8 *)pData; /* Data yet to be written */ - int nRem = nData; /* Number of bytes yet to be written */ - int nRetry = 0; /* Number of retrys */ - - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); - while( nRem>0 ){ - DWORD nWrite = 0; /* Bytes written using WriteFile */ - if( !WriteFile(pWin32File->hFile, aRem, nRem, &nWrite, &overlapped) ){ - if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; - break; - } - assert( nWrite==0 || nWrite<=(DWORD)nRem ); - if( nWrite==0 || nWrite>(DWORD)nRem ){ - break; - } - iOff += nWrite; - overlapped.Offset = (LONG)(iOff & 0xFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); - aRem += nWrite; - nRem -= nWrite; - } - if( nRem!=0 ) return LSM_IOERR_BKPT; - return LSM_OK; -} - -static int win32Truncate( - HANDLE hFile, - lsm_i64 nSize -){ - LARGE_INTEGER offset; - offset.QuadPart = nSize; - if( !SetFilePointerEx(hFile, offset, 0, FILE_BEGIN) ){ - return LSM_IOERR_BKPT; - } - if (!SetEndOfFile(hFile) ){ - return LSM_IOERR_BKPT; - } - return LSM_OK; -} - -static int lsmWin32OsTruncate( - lsm_file *pFile, /* File to write to */ - lsm_i64 nSize /* Size to truncate file to */ -){ - Win32File *pWin32File = (Win32File *)pFile; - return win32Truncate(pWin32File->hFile, nSize); -} - -static int lsmWin32OsRead( - lsm_file *pFile, /* File to read from */ - lsm_i64 iOff, /* Offset to read from */ - void *pData, /* Read data into this buffer */ - int nData /* Bytes of data to read */ -){ - Win32File *pWin32File = (Win32File *)pFile; - OVERLAPPED overlapped; /* The offset for ReadFile */ - DWORD nRead = 0; /* Bytes read using ReadFile */ - int nRetry = 0; /* Number of retrys */ - - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0X7FFFFFFF); - while( !ReadFile(pWin32File->hFile, pData, nData, &nRead, &overlapped) && - GetLastError()!=ERROR_HANDLE_EOF ){ - if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; - return LSM_IOERR_BKPT; - } - if( nRead<(DWORD)nData ){ - /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pData)[nRead], 0, nData - nRead); - } - return LSM_OK; -} - -static int lsmWin32OsSync(lsm_file *pFile){ - int rc = LSM_OK; - -#ifndef LSM_NO_SYNC - Win32File *pWin32File = (Win32File *)pFile; - - if( pWin32File->pMap!=NULL ){ - if( !FlushViewOfFile(pWin32File->pMap, 0) ){ - rc = LSM_IOERR_BKPT; - } - } - if( rc==LSM_OK && !FlushFileBuffers(pWin32File->hFile) ){ - rc = LSM_IOERR_BKPT; - } -#else - unused_parameter(pFile); -#endif - - return rc; -} - -static int lsmWin32OsSectorSize(lsm_file *pFile){ - return 512; -} - -static void win32Unmap(Win32File *pWin32File){ - if( pWin32File->pMap!=NULL ){ - UnmapViewOfFile(pWin32File->pMap); - pWin32File->pMap = NULL; - pWin32File->nMap = 0; - } - if( pWin32File->hMap!=NULL ){ - CloseHandle(pWin32File->hMap); - pWin32File->hMap = NULL; - } -} - -static int lsmWin32OsRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - Win32File *pWin32File = (Win32File *)pFile; - - /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. - ** Thereafter, in chunks of 1MB at a time. */ - const int aIncrSz[] = {256*1024, 1024*1024}; - int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; - - *ppOut = NULL; - *pnOut = 0; - - win32Unmap(pWin32File); - if( iMin>=0 ){ - LARGE_INTEGER fileSize; - DWORD dwSizeHigh; - DWORD dwSizeLow; - HANDLE hMap; - LPVOID pMap; - memset(&fileSize, 0, sizeof(LARGE_INTEGER)); - if( !GetFileSizeEx(pWin32File->hFile, &fileSize) ){ - return LSM_IOERR_BKPT; - } - assert( fileSize.QuadPart>=0 ); - if( fileSize.QuadPart> 32); - hMap = CreateFileMappingW(pWin32File->hFile, NULL, PAGE_READWRITE, - dwSizeHigh, dwSizeLow, NULL); - if( hMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->hMap = hMap; - assert( fileSize.QuadPart<=0xFFFFFFFF ); - pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, 0, 0, - (SIZE_T)fileSize.QuadPart); - if( pMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->pMap = pMap; - pWin32File->nMap = (SIZE_T)fileSize.QuadPart; - } - *ppOut = pWin32File->pMap; - *pnOut = pWin32File->nMap; - return LSM_OK; -} - -static BOOL win32IsDriveLetterAndColon( - const char *zPathname -){ - return ( isalpha(zPathname[0]) && zPathname[1]==':' ); -} - -static int lsmWin32OsFullpath( - lsm_env *pEnv, - const char *zName, - char *zOut, - int *pnOut -){ - DWORD nByte; - void *zConverted; - LPWSTR zTempWide; - char *zTempUtf8; - - if( zName[0]=='/' && win32IsDriveLetterAndColon(zName+1) ){ - zName++; - } - zConverted = win32Utf8ToUnicode(pEnv, zName); - if( zConverted==0 ){ - return LSM_NOMEM_BKPT; - } - nByte = GetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); - if( nByte==0 ){ - lsmFree(pEnv, zConverted); - return LSM_IOERR_BKPT; - } - nByte += 3; - zTempWide = lsmMallocZero(pEnv, nByte * sizeof(zTempWide[0])); - if( zTempWide==0 ){ - lsmFree(pEnv, zConverted); - return LSM_NOMEM_BKPT; - } - nByte = GetFullPathNameW((LPCWSTR)zConverted, nByte, zTempWide, 0); - if( nByte==0 ){ - lsmFree(pEnv, zConverted); - lsmFree(pEnv, zTempWide); - return LSM_IOERR_BKPT; - } - lsmFree(pEnv, zConverted); - zTempUtf8 = win32UnicodeToUtf8(pEnv, zTempWide); - lsmFree(pEnv, zTempWide); - if( zTempUtf8 ){ - int nOut = *pnOut; - int nLen = strlen(zTempUtf8) + 1; - if( nLen<=nOut ){ - snprintf(zOut, nOut, "%s", zTempUtf8); - } - lsmFree(pEnv, zTempUtf8); - *pnOut = nLen; - return LSM_OK; - }else{ - return LSM_NOMEM_BKPT; - } -} - -static int lsmWin32OsFileid( - lsm_file *pFile, - void *pBuf, - int *pnBuf -){ - int nBuf; - int nReq; - u8 *pBuf2 = (u8 *)pBuf; - Win32File *pWin32File = (Win32File *)pFile; - BY_HANDLE_FILE_INFORMATION fileInfo; - - nBuf = *pnBuf; - nReq = (sizeof(fileInfo.dwVolumeSerialNumber) + - sizeof(fileInfo.nFileIndexHigh) + - sizeof(fileInfo.nFileIndexLow)); - *pnBuf = nReq; - if( nReq>nBuf ) return LSM_OK; - memset(&fileInfo, 0, sizeof(BY_HANDLE_FILE_INFORMATION)); - if( !GetFileInformationByHandle(pWin32File->hFile, &fileInfo) ){ - return LSM_IOERR_BKPT; - } - nReq = sizeof(fileInfo.dwVolumeSerialNumber); - memcpy(pBuf2, &fileInfo.dwVolumeSerialNumber, nReq); - pBuf2 += nReq; - nReq = sizeof(fileInfo.nFileIndexHigh); - memcpy(pBuf, &fileInfo.nFileIndexHigh, nReq); - pBuf2 += nReq; - nReq = sizeof(fileInfo.nFileIndexLow); - memcpy(pBuf2, &fileInfo.nFileIndexLow, nReq); - return LSM_OK; -} - -static int win32Delete( - lsm_env *pEnv, - const char *zFile -){ - int rc; - LPWSTR zConverted; - - zConverted = win32Utf8ToUnicode(pEnv, zFile); - if( zConverted==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - int nRetry = 0; - DWORD attr; - - do { - attr = GetFileAttributesW(zConverted); - if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = LSM_IOERR_BKPT; - break; - } - if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ - rc = LSM_IOERR_BKPT; /* Files only. */ - break; - } - if ( DeleteFileW(zConverted) ){ - rc = LSM_OK; /* Deleted OK. */ - break; - } - if ( !win32RetryIoerr(pEnv, &nRetry) ){ - rc = LSM_IOERR_BKPT; /* No more retries. */ - break; - } - }while( 1 ); - } - lsmFree(pEnv, zConverted); - return rc; -} - -static int lsmWin32OsUnlink(lsm_env *pEnv, const char *zFile){ - return win32Delete(pEnv, zFile); -} - -#if !defined(win32IsLockBusy) -#define win32IsLockBusy(a) (((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_IO_PENDING)) -#endif - -static int win32LockFile( - Win32File *pWin32File, - int iLock, - int nLock, - int eType -){ - OVERLAPPED ovlp; - - assert( LSM_LOCK_UNLOCK==0 ); - assert( LSM_LOCK_SHARED==1 ); - assert( LSM_LOCK_EXCL==2 ); - assert( eType>=LSM_LOCK_UNLOCK && eType<=LSM_LOCK_EXCL ); - assert( nLock>=0 ); - assert( iLock>0 && iLock<=32 ); - - memset(&ovlp, 0, sizeof(OVERLAPPED)); - ovlp.Offset = (4096-iLock-nLock+1); - if( eType>LSM_LOCK_UNLOCK ){ - DWORD flags = LOCKFILE_FAIL_IMMEDIATELY; - if( eType>=LSM_LOCK_EXCL ) flags |= LOCKFILE_EXCLUSIVE_LOCK; - if( !LockFileEx(pWin32File->hFile, flags, 0, (DWORD)nLock, 0, &ovlp) ){ - if( win32IsLockBusy(GetLastError()) ){ - return LSM_BUSY; - }else{ - return LSM_IOERR_BKPT; - } - } - }else{ - if( !UnlockFileEx(pWin32File->hFile, 0, (DWORD)nLock, 0, &ovlp) ){ - return LSM_IOERR_BKPT; - } - } - return LSM_OK; -} - -static int lsmWin32OsLock(lsm_file *pFile, int iLock, int eType){ - Win32File *pWin32File = (Win32File *)pFile; - return win32LockFile(pWin32File, iLock, 1, eType); -} - -static int lsmWin32OsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - int rc; - Win32File *pWin32File = (Win32File *)pFile; - rc = win32LockFile(pWin32File, iLock, nLock, eType); - if( rc!=LSM_OK ) return rc; - win32LockFile(pWin32File, iLock, nLock, LSM_LOCK_UNLOCK); - return LSM_OK; -} - -static int lsmWin32OsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ - int rc; - Win32File *pWin32File = (Win32File *)pFile; - int iOffset = iChunk * sz; - int iOffsetShift = iOffset % pWin32File->sysInfo.dwAllocationGranularity; - int nNew = iChunk + 1; - lsm_i64 nReq = nNew * sz; - - *ppShm = NULL; - assert( sz>=0 ); - assert( sz==LSM_SHM_CHUNK_SIZE ); - if( iChunk>=pWin32File->nShm ){ - LPHANDLE ahNew; - LPVOID *apNew; - LARGE_INTEGER fileSize; - - /* If the shared-memory file has not been opened, open it now. */ - if( pWin32File->hShmFile==NULL ){ - char *zShm = win32ShmFile(pWin32File); - if( !zShm ) return LSM_NOMEM_BKPT; - rc = win32Open(pWin32File->pEnv, zShm, 0, &pWin32File->hShmFile); - lsmFree(pWin32File->pEnv, zShm); - if( rc!=LSM_OK ){ - return rc; - } - } - - /* If the shared-memory file is not large enough to contain the - ** requested chunk, cause it to grow. */ - memset(&fileSize, 0, sizeof(LARGE_INTEGER)); - if( !GetFileSizeEx(pWin32File->hShmFile, &fileSize) ){ - return LSM_IOERR_BKPT; - } - assert( fileSize.QuadPart>=0 ); - if( fileSize.QuadParthShmFile, nReq); - if( rc!=LSM_OK ){ - return rc; - } - } - - ahNew = (LPHANDLE)lsmMallocZero(pWin32File->pEnv, sizeof(HANDLE) * nNew); - if( !ahNew ) return LSM_NOMEM_BKPT; - apNew = (LPVOID *)lsmMallocZero(pWin32File->pEnv, sizeof(LPVOID) * nNew); - if( !apNew ){ - lsmFree(pWin32File->pEnv, ahNew); - return LSM_NOMEM_BKPT; - } - memcpy(ahNew, pWin32File->ahShm, sizeof(HANDLE) * pWin32File->nShm); - memcpy(apNew, pWin32File->apShm, sizeof(LPVOID) * pWin32File->nShm); - lsmFree(pWin32File->pEnv, pWin32File->ahShm); - pWin32File->ahShm = ahNew; - lsmFree(pWin32File->pEnv, pWin32File->apShm); - pWin32File->apShm = apNew; - pWin32File->nShm = nNew; - } - - if( pWin32File->ahShm[iChunk]==NULL ){ - HANDLE hMap; - assert( nReq<=0xFFFFFFFF ); - hMap = CreateFileMappingW(pWin32File->hShmFile, NULL, PAGE_READWRITE, 0, - (DWORD)nReq, NULL); - if( hMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->ahShm[iChunk] = hMap; - } - if( pWin32File->apShm[iChunk]==NULL ){ - LPVOID pMap; - pMap = MapViewOfFile(pWin32File->ahShm[iChunk], - FILE_MAP_WRITE | FILE_MAP_READ, 0, - iOffset - iOffsetShift, sz + iOffsetShift); - if( pMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->apShm[iChunk] = pMap; - } - if( iOffsetShift!=0 ){ - char *p = (char *)pWin32File->apShm[iChunk]; - *ppShm = (void *)&p[iOffsetShift]; - }else{ - *ppShm = pWin32File->apShm[iChunk]; - } - return LSM_OK; -} - -static void lsmWin32OsShmBarrier(void){ - MemoryBarrier(); -} - -static int lsmWin32OsShmUnmap(lsm_file *pFile, int bDelete){ - Win32File *pWin32File = (Win32File *)pFile; - - if( pWin32File->hShmFile!=NULL ){ - int i; - for(i=0; inShm; i++){ - if( pWin32File->apShm[i]!=NULL ){ - UnmapViewOfFile(pWin32File->apShm[i]); - pWin32File->apShm[i] = NULL; - } - if( pWin32File->ahShm[i]!=NULL ){ - CloseHandle(pWin32File->ahShm[i]); - pWin32File->ahShm[i] = NULL; - } - } - CloseHandle(pWin32File->hShmFile); - pWin32File->hShmFile = NULL; - if( bDelete ){ - char *zShm = win32ShmFile(pWin32File); - if( zShm ){ win32Delete(pWin32File->pEnv, zShm); } - lsmFree(pWin32File->pEnv, zShm); - } - } - return LSM_OK; -} - -#define MX_CLOSE_ATTEMPT 3 -static int lsmWin32OsClose(lsm_file *pFile){ - int rc; - int nRetry = 0; - Win32File *pWin32File = (Win32File *)pFile; - lsmWin32OsShmUnmap(pFile, 0); - win32Unmap(pWin32File); - do{ - if( pWin32File->hFile==NULL ){ - rc = LSM_IOERR_BKPT; - break; - } - rc = CloseHandle(pWin32File->hFile); - if( rc ){ - pWin32File->hFile = NULL; - rc = LSM_OK; - break; - } - if( ++nRetry>=MX_CLOSE_ATTEMPT ){ - rc = LSM_IOERR_BKPT; - break; - } - }while( 1 ); - lsmFree(pWin32File->pEnv, pWin32File->ahShm); - lsmFree(pWin32File->pEnv, pWin32File->apShm); - lsmFree(pWin32File->pEnv, pWin32File); - return rc; -} - -static int lsmWin32OsSleep(lsm_env *pEnv, int us){ - unused_parameter(pEnv); - return win32Sleep(us); -} - -/**************************************************************************** -** Memory allocation routines. -*/ - -static void *lsmWin32OsMalloc(lsm_env *pEnv, size_t N){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - return HeapAlloc(GetProcessHeap(), 0, (SIZE_T)N); -} - -static void lsmWin32OsFree(lsm_env *pEnv, void *p){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - if( p ){ - HeapFree(GetProcessHeap(), 0, p); - } -} - -static void *lsmWin32OsRealloc(lsm_env *pEnv, void *p, size_t N){ - unsigned char *m = (unsigned char *)p; - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - if( 1>N ){ - lsmWin32OsFree(pEnv, p); - return NULL; - }else if( NULL==p ){ - return lsmWin32OsMalloc(pEnv, N); - }else{ -#if 0 /* arguable: don't shrink */ - SIZE_T sz = HeapSize(GetProcessHeap(), 0, m); - if( sz>=(SIZE_T)N ){ - return p; - } -#endif - return HeapReAlloc(GetProcessHeap(), 0, m, N); - } -} - -static size_t lsmWin32OsMSize(lsm_env *pEnv, void *p){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - return (size_t)HeapSize(GetProcessHeap(), 0, p); -} - - -#ifdef LSM_MUTEX_WIN32 -/************************************************************************* -** Mutex methods for Win32 based systems. If LSM_MUTEX_WIN32 is -** missing then a no-op implementation of mutexes found below will be -** used instead. -*/ -#include "windows.h" - -typedef struct Win32Mutex Win32Mutex; -struct Win32Mutex { - lsm_env *pEnv; - CRITICAL_SECTION mutex; -#ifdef LSM_DEBUG - DWORD owner; -#endif -}; - -#ifndef WIN32_MUTEX_INITIALIZER -# define WIN32_MUTEX_INITIALIZER { 0 } -#endif - -#ifdef LSM_DEBUG -# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER, 0 } -#else -# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER } -#endif - -static int lsmWin32OsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - static volatile LONG initialized = 0; - static Win32Mutex sMutex[2] = { - LSM_WIN32_STATIC_MUTEX, - LSM_WIN32_STATIC_MUTEX - }; - - assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); - assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); - - if( InterlockedCompareExchange(&initialized, 1, 0)==0 ){ - int i; - for(i=0; ipEnv = pEnv; - InitializeCriticalSection(&pMutex->mutex); - - *ppNew = (lsm_mutex *)pMutex; - return LSM_OK; -} - -static void lsmWin32OsMutexDel(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - DeleteCriticalSection(&pMutex->mutex); - lsmFree(pMutex->pEnv, pMutex); -} - -static void lsmWin32OsMutexEnter(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - EnterCriticalSection(&pMutex->mutex); - -#ifdef LSM_DEBUG - assert( pMutex->owner!=GetCurrentThreadId() ); - pMutex->owner = GetCurrentThreadId(); - assert( pMutex->owner==GetCurrentThreadId() ); -#endif -} - -static int lsmWin32OsMutexTry(lsm_mutex *p){ - BOOL bRet; - Win32Mutex *pMutex = (Win32Mutex *)p; - bRet = TryEnterCriticalSection(&pMutex->mutex); -#ifdef LSM_DEBUG - if( bRet ){ - assert( pMutex->owner!=GetCurrentThreadId() ); - pMutex->owner = GetCurrentThreadId(); - assert( pMutex->owner==GetCurrentThreadId() ); - } -#endif - return !bRet; -} - -static void lsmWin32OsMutexLeave(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; -#ifdef LSM_DEBUG - assert( pMutex->owner==GetCurrentThreadId() ); - pMutex->owner = 0; - assert( pMutex->owner!=GetCurrentThreadId() ); -#endif - LeaveCriticalSection(&pMutex->mutex); -} - -#ifdef LSM_DEBUG -static int lsmWin32OsMutexHeld(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - return pMutex ? pMutex->owner==GetCurrentThreadId() : 1; -} -static int lsmWin32OsMutexNotHeld(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - return pMutex ? pMutex->owner!=GetCurrentThreadId() : 1; -} -#endif -/* -** End of Win32 mutex implementation. -*************************************************************************/ -#else -/************************************************************************* -** Noop mutex implementation -*/ -typedef struct NoopMutex NoopMutex; -struct NoopMutex { - lsm_env *pEnv; /* Environment handle (for xFree()) */ - int bHeld; /* True if mutex is held */ - int bStatic; /* True for a static mutex */ -}; -static NoopMutex aStaticNoopMutex[2] = { - {0, 0, 1}, - {0, 0, 1}, -}; - -static int lsmWin32OsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); - *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; - return LSM_OK; -} -static int lsmWin32OsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - NoopMutex *p; - p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); - if( p ) p->pEnv = pEnv; - *ppNew = (lsm_mutex *)p; - return (p ? LSM_OK : LSM_NOMEM_BKPT); -} -static void lsmWin32OsMutexDel(lsm_mutex *pMutex) { - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bStatic==0 && p->pEnv ); - lsmFree(p->pEnv, p); -} -static void lsmWin32OsMutexEnter(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; -} -static int lsmWin32OsMutexTry(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; - return 0; -} -static void lsmWin32OsMutexLeave(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==1 ); - p->bHeld = 0; -} -#ifdef LSM_DEBUG -static int lsmWin32OsMutexHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? p->bHeld : 1; -} -static int lsmWin32OsMutexNotHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? !p->bHeld : 1; -} -#endif -/***************************************************************************/ -#endif /* else LSM_MUTEX_NONE */ - -/* Without LSM_DEBUG, the MutexHeld tests are never called */ -#ifndef LSM_DEBUG -# define lsmWin32OsMutexHeld 0 -# define lsmWin32OsMutexNotHeld 0 -#endif - -lsm_env *lsm_default_env(void){ - static lsm_env win32_env = { - sizeof(lsm_env), /* nByte */ - 1, /* iVersion */ - /***** file i/o ******************/ - 0, /* pVfsCtx */ - lsmWin32OsFullpath, /* xFullpath */ - lsmWin32OsOpen, /* xOpen */ - lsmWin32OsRead, /* xRead */ - lsmWin32OsWrite, /* xWrite */ - lsmWin32OsTruncate, /* xTruncate */ - lsmWin32OsSync, /* xSync */ - lsmWin32OsSectorSize, /* xSectorSize */ - lsmWin32OsRemap, /* xRemap */ - lsmWin32OsFileid, /* xFileid */ - lsmWin32OsClose, /* xClose */ - lsmWin32OsUnlink, /* xUnlink */ - lsmWin32OsLock, /* xLock */ - lsmWin32OsTestLock, /* xTestLock */ - lsmWin32OsShmMap, /* xShmMap */ - lsmWin32OsShmBarrier, /* xShmBarrier */ - lsmWin32OsShmUnmap, /* xShmUnmap */ - /***** memory allocation *********/ - 0, /* pMemCtx */ - lsmWin32OsMalloc, /* xMalloc */ - lsmWin32OsRealloc, /* xRealloc */ - lsmWin32OsFree, /* xFree */ - lsmWin32OsMSize, /* xSize */ - /***** mutexes *********************/ - 0, /* pMutexCtx */ - lsmWin32OsMutexStatic, /* xMutexStatic */ - lsmWin32OsMutexNew, /* xMutexNew */ - lsmWin32OsMutexDel, /* xMutexDel */ - lsmWin32OsMutexEnter, /* xMutexEnter */ - lsmWin32OsMutexTry, /* xMutexTry */ - lsmWin32OsMutexLeave, /* xMutexLeave */ - lsmWin32OsMutexHeld, /* xMutexHeld */ - lsmWin32OsMutexNotHeld, /* xMutexNotHeld */ - /***** other *********************/ - lsmWin32OsSleep, /* xSleep */ - }; - return &win32_env; -} - -#endif diff --git a/ext/lsm1/test/lsm1_common.tcl b/ext/lsm1/test/lsm1_common.tcl deleted file mode 100644 index 0e6cd84e31..0000000000 --- a/ext/lsm1/test/lsm1_common.tcl +++ /dev/null @@ -1,38 +0,0 @@ -# 2014 Dec 19 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. .. test] -} -source $testdir/tester.tcl - -# Check if the lsm1 extension has been compiled. -if {$::tcl_platform(platform) == "windows"} { - set lsm1 lsm.dll -} else { - set lsm1 lsm.so -} - -if {[file exists [file join .. $lsm1]]} { - proc return_if_no_lsm1 {} {} -} else { - proc return_if_no_lsm1 {} { - finish_test - return -code return - } - return -} - -proc load_lsm1_vtab {db} { - db enable_load_extension 1 - db eval {SELECT load_extension('../lsm')} -} diff --git a/ext/lsm1/test/lsm1_simple.test b/ext/lsm1/test/lsm1_simple.test deleted file mode 100644 index 2eab50a83a..0000000000 --- a/ext/lsm1/test/lsm1_simple.test +++ /dev/null @@ -1,152 +0,0 @@ -# 2017 July 14 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# This file implements regression tests for SQLite library. The -# focus of this script is testing the lsm1 virtual table module. -# - -source [file join [file dirname [info script]] lsm1_common.tcl] -set testprefix lsm1_simple -return_if_no_lsm1 -load_lsm1_vtab db - -forcedelete testlsm.db - -do_execsql_test 100 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d); - PRAGMA table_info(x1); -} { - 0 a UINT 1 {} 1 - 1 b {} 0 {} 0 - 2 c {} 0 {} 0 - 3 d {} 0 {} 0 -} - -do_execsql_test 110 { - INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), - (12,NULL,3.25,-559281390); - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33} -do_execsql_test 111 { - SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1; -} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'} - -do_execsql_test 120 { - UPDATE x1 SET d = d+1.0 WHERE a=15; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0} - -do_execsql_test 130 { - UPDATE x1 SET a=123456789 WHERE a=12; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390} -do_execsql_test 131 { - SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000; -} {X'FB075BCD15' 0x75bcd15} - -do_execsql_test 140 { - DELETE FROM x1 WHERE a=15; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390} - -do_test 150 { - lsort [glob testlsm.db*] -} {testlsm.db testlsm.db-log testlsm.db-shm} - -db close -do_test 160 { - lsort [glob testlsm.db*] -} {testlsm.db} - -forcedelete testlsm.db -forcedelete test.db -sqlite3 db test.db -load_lsm1_vtab db - - -do_execsql_test 200 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); - PRAGMA table_info(x1); -} { - 0 a TEXT 1 {} 1 - 1 b {} 0 {} 0 - 2 c {} 0 {} 0 - 3 d {} 0 {} 0 -} -do_execsql_test 210 { - INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), - (12,NULL,3.25,-559281390); - SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1; -} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |} -do_execsql_test 211 { - SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1; -} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |} -do_execsql_test 212 { - SELECT quote(a), quote(lsm1_key), quote(lsm1_value) FROM x1 WHERE a='12'; -} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB'} - -#------------------------------------------------------------------------- -reset_db -forcedelete testlsm.db -load_lsm1_vtab db -do_execsql_test 300 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); -} -do_eqp_test 310 { - SELECT * FROM x1 WHERE a=? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 0:} - -do_eqp_test 320 { - SELECT * FROM x1 WHERE a>? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 2:} - -do_eqp_test 330 { - SELECT * FROM x1 WHERE a 'five'; -} {4 1 3 2} -do_execsql_test 421 { - SELECT b FROM x1 WHERE a <= 'three'; -} {3 1 4 5} - -finish_test diff --git a/ext/lsm1/tool/mklsm1c.tcl b/ext/lsm1/tool/mklsm1c.tcl deleted file mode 100644 index d4a317b700..0000000000 --- a/ext/lsm1/tool/mklsm1c.tcl +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/sh -# restart with tclsh \ -exec tclsh "$0" "$@" - -set srcdir [file dirname [file dirname [info script]]] -set G(src) [string map [list %dir% $srcdir] { - %dir%/lsm.h - %dir%/lsmInt.h - %dir%/lsm_vtab.c - %dir%/lsm_ckpt.c - %dir%/lsm_file.c - %dir%/lsm_log.c - %dir%/lsm_main.c - %dir%/lsm_mem.c - %dir%/lsm_mutex.c - %dir%/lsm_shared.c - %dir%/lsm_sorted.c - %dir%/lsm_str.c - %dir%/lsm_tree.c - %dir%/lsm_unix.c - %dir%/lsm_varint.c - %dir%/lsm_win32.c -}] - -set G(hdr) { - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif -#if defined(NDEBUG) && defined(SQLITE_DEBUG) -# undef NDEBUG -#endif - -} - -set G(footer) { - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */ -} - -#------------------------------------------------------------------------- -# Read and return the entire contents of text file $zFile from disk. -# -proc readfile {zFile} { - set fd [open $zFile] - set data [read $fd] - close $fd - return $data -} - -proc lsm1c_init {zOut} { - global G - set G(fd) stdout - set G(fd) [open $zOut w] - - puts -nonewline $G(fd) $G(hdr) -} - -proc lsm1c_printfile {zIn} { - global G - set data [readfile $zIn] - set zTail [file tail $zIn] - puts $G(fd) "#line 1 \"$zTail\"" - - foreach line [split $data "\n"] { - if {[regexp {^# *include.*lsm} $line]} { - set line "/* $line */" - } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } { - set line "static $line" - } - puts $G(fd) $line - } -} - -proc lsm1c_close {} { - global G - puts -nonewline $G(fd) $G(footer) - if {$G(fd)!="stdout"} { - close $G(fd) - } -} - - -lsm1c_init lsm1.c -foreach f $G(src) { lsm1c_printfile $f } -lsm1c_close diff --git a/ext/misc/README.md b/ext/misc/README.md index 69cb230255..6d34ab9afb 100644 --- a/ext/misc/README.md +++ b/ext/misc/README.md @@ -1,7 +1,7 @@ ## Miscellaneous Extensions This folder contains a collection of smaller loadable extensions. -See for instructions on how +See for instructions on how to compile and use loadable extensions. Each extension in this folder is implemented in a single file of C code. @@ -9,11 +9,6 @@ Each source file contains a description in its header comment. See the header comments for details about each extension. Additional notes are as follows: - * **carray.c** — This module implements the - [carray](https://www.sqlite.org/carray.html) table-valued function. - It is a good example of how to go about implementing a custom - [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). - * **csv.c** — A [virtual table](https://sqlite.org/vtab.html) for reading [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values). @@ -21,24 +16,19 @@ as follows: * **dbdump.c** — This is not actually a loadable extension, but rather a library that implements an approximate equivalent to the ".dump" command of the - [command-line shell](https://www.sqlite.org/cli.html). + [command-line shell](https://sqlite.org/cli.html). * **json1.c** — Various SQL functions and table-valued functions for processing JSON. This extension is already built into the [SQLite amalgamation](https://sqlite.org/amalgamation.html). See for additional information. - * **memvfs.c** — This file implements a custom - [VFS](https://www.sqlite.org/vfs.html) that stores an entire database - file in a single block of RAM. It serves as a good example of how - to implement a simple custom VFS. - * **rot13.c** — This file implements the very simple rot13() substitution function. This file makes a good template for implementing new custom SQL functions for SQLite. * **series.c** — This is an implementation of the - "generate_series" [virtual table](https://www.sqlite.org/vtab.html). + "generate_series" [virtual table](https://sqlite.org/vtab.html). It can make a good template for new custom virtual table implementations. * **shathree.c** — An implementation of the sha3() and diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index bafa43283a..587c610b95 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -482,9 +482,9 @@ struct amatch_rule { amatch_rule *pNext; /* Next rule in order of increasing rCost */ char *zFrom; /* Transform from (a string from user input) */ amatch_cost rCost; /* Cost of this transformation */ - amatch_langid iLang; /* The langauge to which this rule belongs */ + amatch_langid iLang; /* The language to which this rule belongs */ amatch_len nFrom, nTo; /* Length of the zFrom and zTo strings */ - char zTo[4]; /* Tranform to V.W value (extra space appended) */ + char zTo[4]; /* Transform to V.W value (extra space appended) */ }; /* @@ -514,7 +514,7 @@ struct amatch_cursor { sqlite3_int64 iRowid; /* The rowid of the current word */ amatch_langid iLang; /* Use this language ID */ amatch_cost rLimit; /* Maximum cost of any term */ - int nBuf; /* Space allocated for zBuf */ + sqlite3_int64 nBuf; /* Space allocated for zBuf */ int oomErr; /* True following an OOM error */ int nWord; /* Number of amatch_word objects */ char *zBuf; /* Temp-use buffer space */ @@ -1039,7 +1039,7 @@ static void amatchAddWord( nTail = (int)strlen(zWordTail); if( nBase+nTail+3>pCur->nBuf ){ pCur->nBuf = nBase+nTail+100; - pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf); + pCur->zBuf = sqlite3_realloc64(pCur->zBuf, pCur->nBuf); if( pCur->zBuf==0 ){ pCur->nBuf = 0; return; @@ -1105,13 +1105,13 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ amatch_avl *pNode; int isMatch = 0; amatch_vtab *p = pCur->pVtab; - int nWord; + sqlite3_int64 nWord; int rc; int i; const char *zW; amatch_rule *pRule; char *zBuf = 0; - char nBuf = 0; + sqlite3_int64 nBuf = 0; char zNext[8]; char zNextIn[8]; int nNextIn; @@ -1158,7 +1158,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ nWord = (int)strlen(pWord->zWord+2); if( nWord+20>nBuf ){ nBuf = (char)(nWord+100); - zBuf = sqlite3_realloc(zBuf, nBuf); + zBuf = sqlite3_realloc64(zBuf, nBuf); if( zBuf==0 ) return SQLITE_NOMEM; } amatchStrcpy(zBuf, pWord->zWord+2); @@ -1475,7 +1475,8 @@ static sqlite3_module amatchModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/misc/base64.c b/ext/misc/base64.c index bc7a976abc..2da767bb0d 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -175,15 +175,15 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ case ND: /* Treat dark non-digits as pad, but they terminate decode too. */ ncIn = 0; - deliberate_fall_through; + deliberate_fall_through; /* FALLTHRU */ case WS: /* Treat whitespace as pad and terminate this group.*/ nti = nac; - deliberate_fall_through; + deliberate_fall_through; /* FALLTHRU */ case PC: bdp = 0; --nbo; - deliberate_fall_through; + deliberate_fall_through; /* FALLTHRU */ default: /* bdp is the digit value. */ qv = qv<<6 | bdp; break; @@ -192,10 +192,13 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ switch( nbo ){ case 3: pOut[2] = (qv) & 0xff; + deliberate_fall_through; /* FALLTHRU */ case 2: pOut[1] = (qv>>8) & 0xff; + deliberate_fall_through; /* FALLTHRU */ case 1: pOut[0] = (qv>>16) & 0xff; + break; } pOut += nbo; } @@ -204,7 +207,9 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ /* This function does the work for the SQLite base64(x) UDF. */ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ - int nb, nc, nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nb; + sqlite3_int64 nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nc; int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), SQLITE_LIMIT_LENGTH, -1); char *cBuf; @@ -213,7 +218,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ switch( sqlite3_value_type(av[0]) ){ case SQLITE_BLOB: nb = nv; - nc = 4*(nv+2/3); /* quads needed */ + nc = 4*((nv+2)/3); /* quads needed */ nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big", -1); diff --git a/ext/misc/base85.c b/ext/misc/base85.c index e7ef0a04c9..63245e2e4a 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -232,12 +232,16 @@ static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ switch( nbo ){ case 4: *pOut++ = (qv >> 24)&0xff; + /* FALLTHRU */ case 3: *pOut++ = (qv >> 16)&0xff; + /* FALLTHRU */ case 2: *pOut++ = (qv >> 8)&0xff; + /* FALLTHRU */ case 1: *pOut++ = qv&0xff; + /* FALLTHRU */ case 0: break; } @@ -282,7 +286,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ - int nb, nc, nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), SQLITE_LIMIT_LENGTH, -1); char *cBuf; diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c index 22f8268139..9c726f5f17 100644 --- a/ext/misc/btreeinfo.c +++ b/ext/misc/btreeinfo.c @@ -49,7 +49,7 @@ ** USAGE EXAMPLES: ** ** Show the table btrees in a schema order with the tables with the most -** rows occuring first: +** rows occurring first: ** ** SELECT name, nEntry ** FROM sqlite_btreeinfo @@ -411,7 +411,8 @@ int sqlite3BinfoRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0); } diff --git a/ext/misc/carray.h b/ext/misc/carray.h deleted file mode 100644 index ae0a9e8ab0..0000000000 --- a/ext/misc/carray.h +++ /dev/null @@ -1,50 +0,0 @@ -/* -** 2020-11-17 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Interface definitions for the CARRAY table-valued function -** extension. -*/ - -#ifndef _CARRAY_H -#define _CARRAY_H - -#include "sqlite3.h" /* Required for error code definitions */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Use this interface to bind an array to the single-argument version -** of CARRAY(). -*/ -SQLITE_API int sqlite3_carray_bind( - sqlite3_stmt *pStmt, /* Statement to be bound */ - int i, /* Parameter index */ - void *aData, /* Pointer to array data */ - int nData, /* Number of data elements */ - int mFlags, /* CARRAY flags */ - void (*xDel)(void*) /* Destructgor for aData*/ -); - -/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). -*/ -#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ -#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ -#define CARRAY_DOUBLE 2 /* Data is doubles */ -#define CARRAY_TEXT 3 /* Data is char* */ -#define CARRAY_BLOB 4 /* Data is struct iovec */ - -#ifdef __cplusplus -} /* end of the 'extern "C"' block */ -#endif - -#endif /* ifndef _CARRAY_H */ diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index e7c2c9d5c0..6491e85cde 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -197,8 +197,6 @@ struct CksmFile { char computeCksm; /* True to compute checksums. ** Always true if reserve size is 8. */ char verifyCksm; /* True to verify checksums */ - char isWal; /* True if processing a WAL file */ - char inCkpt; /* Currently doing a checkpoint */ CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ }; @@ -444,11 +442,9 @@ static int cksmRead( ** (1) the size indicates that we are dealing with a complete ** database page ** (2) checksum verification is enabled - ** (3) we are not in the middle of checkpoint */ - if( iAmt>=512 /* (1) */ - && p->verifyCksm /* (2) */ - && !p->inCkpt /* (3) */ + if( iAmt>=512 && (iAmt & (iAmt-1))==0 /* (1) */ + && p->verifyCksm /* (2) */ ){ u8 cksum[8]; cksmCompute((u8*)zBuf, iAmt-8, cksum); @@ -489,7 +485,6 @@ static int cksmWrite( */ if( iAmt>=512 && p->computeCksm - && !p->inCkpt ){ cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); } @@ -576,21 +571,6 @@ static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ /* Do not allow page size changes on a checksum database */ return SQLITE_OK; } - }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){ - p->inCkpt = op==SQLITE_FCNTL_CKPT_START; - if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt; - }else if( op==SQLITE_FCNTL_CKSM_FILE ){ - /* This VFS needs to obtain a pointer to the corresponding database - ** file handle from within xOpen() calls to open wal files. To do this, - ** it uses the sqlite3_database_file_object() API to obtain a pointer - ** to the file-handle used by SQLite to access the db file. This is - ** fine if cksmvfs happens to be the top-level VFS, but not if there - ** are one or more wrapper VFS. To handle this case, this file-control - ** is used to extract the cksmvfs file-handle from any wrapper file - ** handle. */ - sqlite3_file **ppFile = (sqlite3_file**)pArg; - *ppFile = (sqlite3_file*)p; - return SQLITE_OK; } rc = pFile->pMethods->xFileControl(pFile, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ @@ -611,8 +591,10 @@ static int cksmSectorSize(sqlite3_file *pFile){ ** Return the device characteristic flags supported by a cksm-file. */ static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + int devchar = 0; pFile = ORIGFILE(pFile); - return pFile->pMethods->xDeviceCharacteristics(pFile); + devchar = pFile->pMethods->xDeviceCharacteristics(pFile); + return (devchar & ~SQLITE_IOCAP_SUBPAGE_READ); } /* Create a shared memory file mapping */ @@ -689,7 +671,7 @@ static int cksmOpen( sqlite3_vfs *pSubVfs; int rc; pSubVfs = ORIGVFS(pVfs); - if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); } p = (CksmFile*)pFile; @@ -698,19 +680,6 @@ static int cksmOpen( pFile->pMethods = &cksm_io_methods; rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); if( rc ) goto cksm_open_done; - if( flags & SQLITE_OPEN_WAL ){ - sqlite3_file *pDb = sqlite3_database_file_object(zName); - rc = pDb->pMethods->xFileControl(pDb, SQLITE_FCNTL_CKSM_FILE, (void*)&pDb); - assert( rc==SQLITE_OK ); - p->pPartner = (CksmFile*)pDb; - assert( p->pPartner->pPartner==0 ); - p->pPartner->pPartner = p; - p->isWal = 1; - p->computeCksm = p->pPartner->computeCksm; - }else{ - p->isWal = 0; - p->computeCksm = 0; - } p->zFName = zName; cksm_open_done: if( rc ) pFile->pMethods = 0; @@ -820,9 +789,7 @@ static int cksmRegisterFunc( */ static int cksmRegisterVfs(void){ int rc = SQLITE_OK; - sqlite3_vfs *pOrig; - if( sqlite3_vfs_find("cksmvfs")!=0 ) return SQLITE_OK; - pOrig = sqlite3_vfs_find(0); + sqlite3_vfs *pOrig = sqlite3_vfs_find(0); if( pOrig==0 ) return SQLITE_ERROR; cksm_vfs.iVersion = pOrig->iVersion; cksm_vfs.pAppData = pOrig; diff --git a/ext/misc/closure.c b/ext/misc/closure.c index db9b2b7394..14caf271f9 100644 --- a/ext/misc/closure.c +++ b/ext/misc/closure.c @@ -137,7 +137,7 @@ ** AND closure.idname='groupId' ** AND closure.parentname='parentId'; ** -** See the documentation at http://www.sqlite.org/loadext.html for information +** See the documentation at http://sqlite.org/loadext.html for information ** on how to compile and use loadable extensions such as this one. */ #include "sqlite3ext.h" @@ -588,12 +588,17 @@ static int closureOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +/* +** Wrapper around sqlite3_free +*/ +static void closureMemFree(closure_avl *p){ sqlite3_free(p); } + /* ** Free up all the memory allocated by a cursor. Set it rLimit to 0 ** to indicate that it is at EOF. */ static void closureClearCursor(closure_cursor *pCur){ - closureAvlDestroy(pCur->pClosure, (void(*)(closure_avl*))sqlite3_free); + closureAvlDestroy(pCur->pClosure, closureMemFree); sqlite3_free(pCur->zTableName); sqlite3_free(pCur->zIdColumn); sqlite3_free(pCur->zParentColumn); @@ -939,7 +944,8 @@ static sqlite3_module closureModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/misc/completion.c b/ext/misc/completion.c index d9e7b85972..67b40d84d1 100644 --- a/ext/misc/completion.c +++ b/ext/misc/completion.c @@ -41,6 +41,11 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE +#ifndef IsAlnum +#define IsAlnum(X) isalnum((unsigned char)X) +#endif + + /* completion_vtab is a subclass of sqlite3_vtab which will ** serve as the underlying representation of a completion virtual table */ @@ -251,7 +256,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ zSql = sqlite3_mprintf( "%z%s" "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" - " JOIN pragma_table_info(sm.name,%Q) AS pti" + " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" " WHERE sm.type='table'", zSql, zSep, zDb, zDb ); @@ -365,6 +370,7 @@ static int completionFilter( if( pCur->nPrefix>0 ){ pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + pCur->nPrefix = (int)strlen(pCur->zPrefix); } iArg = 1; } @@ -373,17 +379,19 @@ static int completionFilter( if( pCur->nLine>0 ){ pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zLine==0 ) return SQLITE_NOMEM; + pCur->nLine = (int)strlen(pCur->zLine); } } if( pCur->zLine!=0 && pCur->zPrefix==0 ){ int i = pCur->nLine; - while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){ + while( i>0 && (IsAlnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){ i--; } pCur->nPrefix = pCur->nLine - i; if( pCur->nPrefix>0 ){ pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + pCur->nPrefix = (int)strlen(pCur->zPrefix); } } pCur->iRowid = 0; @@ -470,7 +478,8 @@ static sqlite3_module completionModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/misc/compress.c b/ext/misc/compress.c index 2e7a31636d..48ea5182d7 100644 --- a/ext/misc/compress.c +++ b/ext/misc/compress.c @@ -59,7 +59,7 @@ static void compressFunc( pIn = sqlite3_value_blob(argv[0]); nIn = sqlite3_value_bytes(argv[0]); nOut = 13 + nIn + (nIn+999)/1000; - pOut = sqlite3_malloc( nOut+5 ); + pOut = sqlite3_malloc64( nOut+5 ); for(i=4; i>=0; i--){ x[i] = (nIn >> (7*(4-i)))&0x7f; } @@ -89,7 +89,7 @@ static void uncompressFunc( unsigned int nIn; unsigned long int nOut; int rc; - int i; + unsigned int i; pIn = sqlite3_value_blob(argv[0]); nIn = sqlite3_value_bytes(argv[0]); @@ -98,7 +98,7 @@ static void uncompressFunc( nOut = (nOut<<7) | (pIn[i]&0x7f); if( (pIn[i]&0x80)!=0 ){ i++; break; } } - pOut = sqlite3_malloc( nOut+1 ); + pOut = sqlite3_malloc64( nOut+1 ); rc = uncompress(pOut, &nOut, &pIn[i], nIn-i); if( rc==Z_OK ){ sqlite3_result_blob(context, pOut, nOut, sqlite3_free); diff --git a/ext/misc/csv.c b/ext/misc/csv.c index 870a0cf60e..1caaaec876 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -62,6 +62,9 @@ SQLITE_EXTENSION_INIT1 # define CSV_NOINLINE #endif +#ifndef SQLITEINT_H +typedef sqlite3_int64 i64; +#endif /* Max size of the error message in a CsvReader */ #define CSV_MXERR 200 @@ -74,9 +77,9 @@ typedef struct CsvReader CsvReader; struct CsvReader { FILE *in; /* Read the CSV text from this input stream */ char *z; /* Accumulated text for a field */ - int n; /* Number of bytes in z */ - int nAlloc; /* Space allocated for z[] */ - int nLine; /* Current line number */ + i64 n; /* Number of bytes in z */ + i64 nAlloc; /* Space allocated for z[] */ + i64 nLine; /* Current line number */ int bNotFirst; /* True if prior text has been seen */ int cTerm; /* Character that terminated the most recent field */ size_t iIn; /* Next unread character in the input buffer */ @@ -174,7 +177,7 @@ static int csv_getc(CsvReader *p){ ** Return 0 on success and non-zero if there is an OOM error */ static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ char *zNew; - int nNew = p->nAlloc*2 + 100; + i64 nNew = p->nAlloc*2 + 100; zNew = sqlite3_realloc64(p->z, nNew); if( zNew ){ p->z = zNew; @@ -315,7 +318,7 @@ typedef struct CsvTable { } CsvTable; /* Allowed values for tstFlags */ -#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/ +#define CSVTEST_FIDX 0x0001 /* Pretend that constrained search cost less*/ /* A cursor for the CSV virtual table */ typedef struct CsvCursor { @@ -510,7 +513,6 @@ static int csvtabConnect( # define CSV_DATA (azPValue[1]) # define CSV_SCHEMA (azPValue[2]) - assert( sizeof(azPValue)==sizeof(azParam) ); memset(&sRdr, 0, sizeof(sRdr)); memset(azPValue, 0, sizeof(azPValue)); @@ -897,6 +899,11 @@ static sqlite3_module CsvModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #ifdef SQLITE_TEST @@ -929,6 +936,11 @@ static sqlite3_module CsvModuleFauxWrite = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_TEST */ diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c index ecf7d810d5..0b93d2b9be 100644 --- a/ext/misc/dbdump.c +++ b/ext/misc/dbdump.c @@ -67,9 +67,9 @@ struct DState { */ typedef struct DText DText; struct DText { - char *z; /* The text */ - int n; /* Number of bytes of content in z[] */ - int nAlloc; /* Number of bytes allocated to z[] */ + char *z; /* The text */ + sqlite3_int64 n; /* Number of bytes of content in z[] */ + sqlite3_int64 nAlloc; /* Number of bytes allocated to z[] */ }; /* @@ -107,7 +107,7 @@ static void appendText(DText *p, char const *zAppend, char quote){ if( p->n+len>=p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + len + 20; - zNew = sqlite3_realloc(p->z, p->nAlloc); + zNew = sqlite3_realloc64(p->z, p->nAlloc); if( zNew==0 ){ freeText(p); return; @@ -179,8 +179,8 @@ static char **tableColumnList(DState *p, const char *zTab){ char **azCol = 0; sqlite3_stmt *pStmt = 0; char *zSql; - int nCol = 0; - int nAlloc = 0; + sqlite3_int64 nCol = 0; + sqlite3_int64 nAlloc = 0; int nPK = 0; /* Number of PRIMARY KEY columns seen */ int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ int preserveRowid = 1; diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c index 4495cae469..f87699f96b 100644 --- a/ext/misc/decimal.c +++ b/ext/misc/decimal.c @@ -27,6 +27,9 @@ SQLITE_EXTENSION_INIT1 # define UNUSED_PARAMETER(X) (void)(X) #endif +#ifndef IsSpace +#define IsSpace(X) isspace((unsigned char)X) +#endif /* A decimal object */ typedef struct Decimal Decimal; @@ -58,42 +61,25 @@ static void decimal_free(Decimal *p){ } /* -** Allocate a new Decimal object. Initialize it to the number given -** by the input string. +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. */ -static Decimal *decimal_new( - sqlite3_context *pCtx, - sqlite3_value *pIn, - int nAlt, - const unsigned char *zAlt -){ - Decimal *p; - int n, i; - const unsigned char *zIn; +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; int iExp = 0; + p = sqlite3_malloc( sizeof(*p) ); - if( p==0 ) goto new_no_mem; + if( p==0 ) goto new_from_text_failed; p->sign = 0; p->oom = 0; p->isInit = 1; p->isNull = 0; p->nDigit = 0; p->nFrac = 0; - if( zAlt ){ - n = nAlt, - zIn = zAlt; - }else{ - if( sqlite3_value_type(pIn)==SQLITE_NULL ){ - p->a = 0; - p->isNull = 1; - return p; - } - n = sqlite3_value_bytes(pIn); - zIn = sqlite3_value_text(pIn); - } p->a = sqlite3_malloc64( n+1 ); - if( p->a==0 ) goto new_no_mem; - for(i=0; isspace(zIn[i]); i++){} + if( p->a==0 ) goto new_from_text_failed; + for(i=0; IsSpace(zIn[i]); i++){} if( zIn[i]=='-' ){ p->sign = 1; i++; @@ -142,8 +128,9 @@ static Decimal *decimal_new( } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_no_mem; + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; } @@ -161,17 +148,91 @@ static Decimal *decimal_new( } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); - if( p->a==0 ) goto new_no_mem; + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); p->nDigit += iExp; p->nFrac += iExp; } } + if( p->sign ){ + for(i=0; inDigit && p->a[i]==0; i++){} + if( i>=p->nDigit ) p->sign = 0; + } return p; -new_no_mem: +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; inDigit+4 ); + z = sqlite3_malloc64( (sqlite3_int64)p->nDigit+4 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -231,19 +292,64 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ } /* -** SQL Function: decimal(X) -** -** Convert input X into decimal and then back into text +** Make the given Decimal the result in an format similar to '%+#e'. +** In other words, show exponential notation with leading and trailing +** zeros omitted. */ -static void decimalFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Decimal *p = decimal_new(context, argv[0], 0, 0); - UNUSED_PARAMETER(argc); - decimal_result(context, p); - decimal_free(p); +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ + char *z; /* The output buffer */ + int i; /* Loop counter */ + int nZero; /* Number of leading zeros */ + int nDigit; /* Number of digits not counting trailing zeros */ + int nFrac; /* Digits to the right of the decimal point */ + int exp; /* Exponent value */ + signed char zero; /* Zero value */ + signed char *a; /* Array of digits */ + + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc64( (sqlite3_int64)nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull==0 */ -static int decimal_cmp(const Decimal *pA, const Decimal *pB){ +static int decimal_cmp(Decimal *pA, Decimal *pB){ int nASig, nBSig, rc, n; + while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ + pA->nDigit--; + pA->nFrac--; + } + while( pB->nFrac>0 && pB->a[pB->nDigit-1]==0 ){ + pB->nDigit--; + pB->nFrac--; + } if( pA->sign!=pB->sign ){ return pA->sign ? -1 : +1; } if( pA->sign ){ - const Decimal *pTemp = pA; + Decimal *pTemp = pA; pA = pB; pB = pTemp; } @@ -296,9 +410,9 @@ static void decimalCmpFunc( int rc; UNUSED_PARAMETER(argc); - pA = decimal_new(context, argv[0], 0, 0); + pA = decimal_new(context, argv[0], 1); if( pA==0 || pA->isNull ) goto cmp_done; - pB = decimal_new(context, argv[1], 0, 0); + pB = decimal_new(context, argv[1], 1); if( pB==0 || pB->isNull ) goto cmp_done; rc = decimal_cmp(pA, pB); if( rc<0 ) rc = -1; @@ -338,7 +452,7 @@ static void decimal_expand(Decimal *p, int nDigit, int nFrac){ } /* -** Add the value pB into pA. +** Add the value pB into pA. A := A + B. ** ** Both pA and pB might become denormalized by this routine. */ @@ -407,6 +521,173 @@ static void decimal_add(Decimal *pA, Decimal *pB){ } } +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + + (sqlite3_int64)pB->nDigit + 2 ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 || a==(sqlite3_int64)0x8000000000000000LL){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + UNUSED_PARAMETER(argc); + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + /* ** Compare text in decimal order. */ @@ -417,8 +698,8 @@ static int decimalCollFunc( ){ const unsigned char *zA = (const unsigned char*)pKey1; const unsigned char *zB = (const unsigned char*)pKey2; - Decimal *pA = decimal_new(0, 0, nKey1, zA); - Decimal *pB = decimal_new(0, 0, nKey2, zB); + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); int rc; UNUSED_PARAMETER(notUsed); if( pA==0 || pB==0 ){ @@ -443,8 +724,8 @@ static void decimalAddFunc( int argc, sqlite3_value **argv ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); decimal_add(pA, pB); decimal_result(context, pA); @@ -456,8 +737,8 @@ static void decimalSubFunc( int argc, sqlite3_value **argv ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); if( pB ){ pB->sign = !pB->sign; @@ -468,7 +749,7 @@ static void decimalSubFunc( decimal_free(pB); } -/* Aggregate funcion: decimal_sum(X) +/* Aggregate function: decimal_sum(X) ** ** Works like sum() except that it uses decimal arithmetic for unlimited ** precision. @@ -495,7 +776,7 @@ static void decimalSumStep( p->nFrac = 0; } if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pArg = decimal_new(context, argv[0], 0, 0); + pArg = decimal_new(context, argv[0], 1); decimal_add(p, pArg); decimal_free(pArg); } @@ -510,7 +791,7 @@ static void decimalSumInverse( p = sqlite3_aggregate_context(context, sizeof(*p)); if( p==0 ) return; if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pArg = decimal_new(context, argv[0], 0, 0); + pArg = decimal_new(context, argv[0], 1); if( pArg ) pArg->sign = !pArg->sign; decimal_add(p, pArg); decimal_free(pArg); @@ -531,66 +812,49 @@ static void decimalSumFinalize(sqlite3_context *context){ ** SQL Function: decimal_mul(X, Y) ** ** Return the product of X and Y. -** -** All significant digits after the decimal point are retained. -** Trailing zeros after the decimal point are omitted as long as -** the number of digits after the decimal point is no less than -** either the number of digits in either input. */ static void decimalMulFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ - Decimal *pA = decimal_new(context, argv[0], 0, 0); - Decimal *pB = decimal_new(context, argv[1], 0, 0); - signed char *acc = 0; - int i, j, k; - int minFrac; + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); UNUSED_PARAMETER(argc); if( pA==0 || pA->oom || pA->isNull || pB==0 || pB->oom || pB->isNull ){ goto mul_end; } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); - if( acc==0 ){ - sqlite3_result_error_nomem(context); + decimalMul(pA, pB); + if( pA->oom ){ goto mul_end; } - memset(acc, 0, pA->nDigit + pB->nDigit + 2); - minFrac = pA->nFrac; - if( pB->nFracnFrac; - for(i=pA->nDigit-1; i>=0; i--){ - signed char f = pA->a[i]; - int carry = 0, x; - for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ - x = acc[k] + f*pB->a[j] + carry; - acc[k] = x%10; - carry = x/10; - } - x = acc[k] + carry; - acc[k] = x%10; - acc[k-1] += x/10; - } - sqlite3_free(pA->a); - pA->a = acc; - acc = 0; - pA->nDigit += pB->nDigit + 2; - pA->nFrac += pB->nFrac; - pA->sign ^= pB->sign; - while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ - pA->nFrac--; - pA->nDigit--; - } decimal_result(context, pA); mul_end: - sqlite3_free(acc); decimal_free(pA); decimal_free(pB); } +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA); + decimal_free(pA); + } +} + #ifdef _WIN32 __declspec(dllexport) #endif @@ -603,13 +867,16 @@ int sqlite3_decimal_init( static const struct { const char *zFuncName; int nArg; + int iArg; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { - { "decimal", 1, decimalFunc }, - { "decimal_cmp", 2, decimalCmpFunc }, - { "decimal_add", 2, decimalAddFunc }, - { "decimal_sub", 2, decimalSubFunc }, - { "decimal_mul", 2, decimalMulFunc }, + { "decimal", 1, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, }; unsigned int i; (void)pzErrMsg; /* Unused parameter */ @@ -619,7 +886,7 @@ int sqlite3_decimal_init( for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, - 0, aFunc[i].xFunc, 0, 0); + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); } if( rc==SQLITE_OK ){ rc = sqlite3_create_window_function(db, "decimal_sum", 1, diff --git a/ext/misc/explain.c b/ext/misc/explain.c index 0095194570..726af76b96 100644 --- a/ext/misc/explain.c +++ b/ext/misc/explain.c @@ -293,6 +293,7 @@ static sqlite3_module explainModule = { 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 7cdbd5968f..6cc2ae0085 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -40,7 +40,7 @@ ** modification-time of the target file is set to this value before ** returning. ** -** If three or more arguments are passed to this function and an +** If five or more arguments are passed to this function and an ** error is encountered, an exception is raised. ** ** READFILE(FILE): @@ -67,6 +67,7 @@ ** data: For a regular file, a blob containing the file data. For a ** symlink, a text value containing the text of the link. For a ** directory, NULL. +** level: Directory hierarchy level. Topmost is 1. ** ** If a non-NULL value is specified for the optional $dir parameter and ** $path is a relative path, then $path is interpreted relative to $dir. @@ -92,36 +93,72 @@ SQLITE_EXTENSION_INIT1 # include # include # include +# define STRUCT_STAT struct stat #else -# include "windows.h" -# include +# include "windirent.h" # include -# include "test_windirent.h" -# define dirent DIRENT -# ifndef chmod -# define chmod _chmod -# endif -# ifndef stat -# define stat _stat -# endif -# define mkdir(path,mode) _mkdir(path) -# define lstat(path,buf) stat(path,buf) +# define STRUCT_STAT struct _stat +# define chmod(path,mode) fileio_chmod(path,mode) +# define mkdir(path,mode) fileio_mkdir(path) #endif #include #include +/* When used as part of the CLI, the sqlite3_stdio.h module will have +** been included before this one. In that case use the sqlite3_stdio.h +** #defines. If not, create our own for fopen(). +*/ +#ifndef _SQLITE3_STDIO_H_ +# define sqlite3_fopen fopen +#endif /* ** Structure of the fsdir() table-valued function */ - /* 0 1 2 3 4 5 */ -#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" + /* 0 1 2 3 4 5 6 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,level,path HIDDEN,dir HIDDEN)" + #define FSDIR_COLUMN_NAME 0 /* Name of the file */ #define FSDIR_COLUMN_MODE 1 /* Access mode */ #define FSDIR_COLUMN_MTIME 2 /* Last modification time */ #define FSDIR_COLUMN_DATA 3 /* File content */ -#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ -#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ +#define FSDIR_COLUMN_LEVEL 4 /* Level. Topmost is 1 */ +#define FSDIR_COLUMN_PATH 5 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 6 /* Path is relative to this directory */ + +/* +** UTF8 chmod() function for Windows +*/ +#if defined(_WIN32) || defined(WIN32) +static int fileio_chmod(const char *zPath, int pmode){ + sqlite3_int64 sz = strlen(zPath); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); + int rc; + if( b1==0 ) return -1; + sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); + b1[sz] = 0; + rc = _wchmod(b1, pmode); + sqlite3_free(b1); + return rc; +} +#endif + +/* +** UTF8 mkdir() function for Windows +*/ +#if defined(_WIN32) || defined(WIN32) +static int fileio_mkdir(const char *zPath){ + sqlite3_int64 sz = strlen(zPath); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); + int rc; + if( b1==0 ) return -1; + sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); + b1[sz] = 0; + rc = _wmkdir(b1); + sqlite3_free(b1); + return rc; +} +#endif /* @@ -142,7 +179,7 @@ static void readFileContents(sqlite3_context *ctx, const char *zName){ sqlite3 *db; int mxBlob; - in = fopen(zName, "rb"); + in = sqlite3_fopen(zName, "rb"); if( in==0 ){ /* File does not exist or is unreadable. Leave the result set to NULL. */ return; @@ -253,7 +290,7 @@ LPWSTR utf8_to_utf16(const char *z){ */ static void statTimesToUtc( const char *zPath, - struct stat *pStatBuf + STRUCT_STAT *pStatBuf ){ HANDLE hFindFile; WIN32_FIND_DATAW fd; @@ -281,11 +318,18 @@ static void statTimesToUtc( */ static int fileStat( const char *zPath, - struct stat *pStatBuf + STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) - int rc = stat(zPath, pStatBuf); + sqlite3_int64 sz = strlen(zPath); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); + int rc; + if( b1==0 ) return 1; + sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); + b1[sz] = 0; + rc = _wstat(b1, pStatBuf); if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + sqlite3_free(b1); return rc; #else return stat(zPath, pStatBuf); @@ -299,12 +343,10 @@ static int fileStat( */ static int fileLinkStat( const char *zPath, - struct stat *pStatBuf + STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) - int rc = lstat(zPath, pStatBuf); - if( rc==0 ) statTimesToUtc(zPath, pStatBuf); - return rc; + return fileStat(zPath, pStatBuf); #else return lstat(zPath, pStatBuf); #endif @@ -334,7 +376,7 @@ static int makeDirectory( int i = 1; while( rc==SQLITE_OK ){ - struct stat sStat; + STRUCT_STAT sStat; int rc2; for(; zCopy[i]!='/' && i> 32; zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile); @@ -458,13 +502,19 @@ static int writeFile( return 1; } #else - /* Legacy unix */ - struct timeval times[2]; - times[0].tv_usec = times[1].tv_usec = 0; - times[0].tv_sec = time(0); - times[1].tv_sec = mtime; - if( utimes(zFile, times) ){ - return 1; + /* Legacy unix. + ** + ** Do not use utimes() on a symbolic link - it sees through the link and + ** modifies the timestamps on the target. Or fails if the target does + ** not exist. */ + if( 0==S_ISLNK(mode) ){ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } } #endif } @@ -572,13 +622,14 @@ struct fsdir_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ int nLvl; /* Number of entries in aLvl[] array */ + int mxLvl; /* Maximum level */ int iLvl; /* Index of current entry */ FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ const char *zBase; int nBase; - struct stat sStat; /* Current lstat() results */ + STRUCT_STAT sStat; /* Current lstat() results */ char *zPath; /* Path to current entry */ sqlite3_int64 iRowid; /* Current rowid */ }; @@ -690,7 +741,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ mode_t m = pCur->sStat.st_mode; pCur->iRowid++; - if( S_ISDIR(m) ){ + if( S_ISDIR(m) && pCur->iLvl+3mxLvl ){ /* Descend into this directory */ int iNew = pCur->iLvl + 1; FsdirLevel *pLvl; @@ -710,7 +761,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ pCur->zPath = 0; pLvl->pDir = opendir(pLvl->zDir); if( pLvl->pDir==0 ){ - fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + fsdirSetErrmsg(pCur, "cannot read directory: %s", pLvl->zDir); return SQLITE_ERROR; } } @@ -798,7 +849,11 @@ static int fsdirColumn( }else{ readFileContents(ctx, pCur->zPath); } + break; } + case FSDIR_COLUMN_LEVEL: + sqlite3_result_int(ctx, pCur->iLvl+2); + break; case FSDIR_COLUMN_PATH: default: { /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. @@ -832,8 +887,11 @@ static int fsdirEof(sqlite3_vtab_cursor *cur){ /* ** xFilter callback. ** -** idxNum==1 PATH parameter only -** idxNum==2 Both PATH and DIR supplied +** idxNum bit Meaning +** 0x01 PATH=N +** 0x02 DIR=N +** 0x04 LEVEL0 ); zDir = (const char*)sqlite3_value_text(argv[0]); if( zDir==0 ){ fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); return SQLITE_ERROR; } - if( argc==2 ){ - pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + i = 1; + if( (idxNum & 0x02)!=0 ){ + assert( argc>i ); + pCur->zBase = (const char*)sqlite3_value_text(argv[i++]); + } + if( (idxNum & 0x0c)!=0 ){ + assert( argc>i ); + pCur->mxLvl = sqlite3_value_int(argv[i++]); + if( idxNum & 0x08 ) pCur->mxLvl++; + if( pCur->mxLvl<=0 ) pCur->mxLvl = 1000000000; + }else{ + pCur->mxLvl = 1000000000; } if( pCur->zBase ){ pCur->nBase = (int)strlen(pCur->zBase)+1; @@ -886,10 +955,11 @@ static int fsdirFilter( ** In this implementation idxNum is used to represent the ** query plan. idxStr is unused. ** -** The query plan is represented by values of idxNum: +** The query plan is represented by bits in idxNum: ** -** (1) The path value is supplied by argv[0] -** (2) Path is in argv[0] and dir is in argv[1] +** 0x01 The path value is supplied by argv[0] +** 0x02 dir is in argv[1] +** 0x04 maxdepth is in argv[1] or [2] */ static int fsdirBestIndex( sqlite3_vtab *tab, @@ -898,6 +968,9 @@ static int fsdirBestIndex( int i; /* Loop over constraints */ int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int idxLevel = -1; /* Index in pIdxInfo->aConstraint of LEVEL< or <= */ + int idxLevelEQ = 0; /* 0x08 for LEVEL<= or LEVEL=. 0x04 for LEVEL< */ + int omitLevel = 0; /* omit the LEVEL constraint */ int seenPath = 0; /* True if an unusable PATH= constraint is seen */ int seenDir = 0; /* True if an unusable DIR= constraint is seen */ const struct sqlite3_index_constraint *pConstraint; @@ -905,25 +978,48 @@ static int fsdirBestIndex( (void)tab; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - switch( pConstraint->iColumn ){ - case FSDIR_COLUMN_PATH: { - if( pConstraint->usable ){ - idxPath = i; - seenPath = 0; - }else if( idxPath<0 ){ - seenPath = 1; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; } - break; - } - case FSDIR_COLUMN_DIR: { - if( pConstraint->usable ){ - idxDir = i; - seenDir = 0; - }else if( idxDir<0 ){ - seenDir = 1; + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; } - break; + case FSDIR_COLUMN_LEVEL: { + if( pConstraint->usable && idxLevel<0 ){ + idxLevel = i; + idxLevelEQ = 0x08; + omitLevel = 0; + } + break; + } + } + }else + if( pConstraint->iColumn==FSDIR_COLUMN_LEVEL + && pConstraint->usable + && idxLevel<0 + ){ + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE ){ + idxLevel = i; + idxLevelEQ = 0x08; + omitLevel = 1; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ + idxLevel = i; + idxLevelEQ = 0x04; + omitLevel = 1; } } } @@ -940,14 +1036,20 @@ static int fsdirBestIndex( }else{ pIdxInfo->aConstraintUsage[idxPath].omit = 1; pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + pIdxInfo->idxNum = 0x01; + pIdxInfo->estimatedCost = 1.0e9; + i = 2; if( idxDir>=0 ){ pIdxInfo->aConstraintUsage[idxDir].omit = 1; - pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; - pIdxInfo->idxNum = 2; - pIdxInfo->estimatedCost = 10.0; - }else{ - pIdxInfo->idxNum = 1; - pIdxInfo->estimatedCost = 100.0; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = i++; + pIdxInfo->idxNum |= 0x02; + pIdxInfo->estimatedCost /= 1.0e4; + } + if( idxLevel>=0 ){ + pIdxInfo->aConstraintUsage[idxLevel].omit = omitLevel; + pIdxInfo->aConstraintUsage[idxLevel].argvIndex = i++; + pIdxInfo->idxNum |= idxLevelEQ; + pIdxInfo->estimatedCost /= 1.0e4; } } @@ -983,6 +1085,7 @@ static int fsdirRegister(sqlite3 *db){ 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c index 6a597e0d7d..d24a87700e 100644 --- a/ext/misc/fossildelta.c +++ b/ext/misc/fossildelta.c @@ -22,12 +22,12 @@ ** The delta format is the Fossil delta format, described in a comment ** on the delete_create() function implementation below, and also at ** -** https://www.fossil-scm.org/fossil/doc/trunk/www/delta_format.wiki +** https://fossil-scm.org/fossil/doc/trunk/www/delta_format.wiki ** ** This delta format is used by the RBU extension, which is the main ** reason that these routines are included in the extension library. ** RBU does not use this extension directly. Rather, this extension is -** provided as a convenience to developers who want to analyze RBU files +** provided as a convenience to developers who want to analyze RBU files ** that contain deltas. */ #include @@ -868,11 +868,21 @@ static int deltaparsevtabNext(sqlite3_vtab_cursor *cur){ int i = 0; pCur->iCursor = pCur->iNext; + if( pCur->iCursor >= pCur->nDelta ){ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->iNext = pCur->nDelta; + return SQLITE_OK; + } z = pCur->aDelta + pCur->iCursor; pCur->a1 = deltaGetInt(&z, &i); switch( z[0] ){ case '@': { z++; + if( pCur->iNext>=pCur->nDelta ){ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->iNext = pCur->nDelta; + break; + } pCur->a2 = deltaGetInt(&z, &i); pCur->eOp = DELTAPARSE_OP_COPY; pCur->iNext = (int)(&z[1] - pCur->aDelta); @@ -926,8 +936,12 @@ static int deltaparsevtabColumn( if( pCur->eOp==DELTAPARSE_OP_COPY ){ sqlite3_result_int(ctx, pCur->a2); }else if( pCur->eOp==DELTAPARSE_OP_INSERT ){ - sqlite3_result_blob(ctx, pCur->aDelta+pCur->a2, pCur->a1, - SQLITE_TRANSIENT); + if( pCur->a2 + pCur->a1 > pCur->nDelta ){ + sqlite3_result_zeroblob(ctx, pCur->a1); + }else{ + sqlite3_result_blob(ctx, pCur->aDelta+pCur->a2, pCur->a1, + SQLITE_TRANSIENT); + } } break; } @@ -955,17 +969,17 @@ static int deltaparsevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int deltaparsevtabEof(sqlite3_vtab_cursor *cur){ deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; - return pCur->eOp==DELTAPARSE_OP_EOF; + return pCur->eOp==DELTAPARSE_OP_EOF || pCur->iCursor>=pCur->nDelta; } /* ** This method is called to "rewind" the deltaparsevtab_cursor object back ** to the first row of output. This method is always called at least -** once prior to any call to deltaparsevtabColumn() or deltaparsevtabRowid() or +** once prior to any call to deltaparsevtabColumn() or deltaparsevtabRowid() or ** deltaparsevtabEof(). */ static int deltaparsevtabFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ @@ -1031,7 +1045,7 @@ static int deltaparsevtabBestIndex( } /* -** This following structure defines all the methods for the +** This following structure defines all the methods for the ** virtual table. */ static sqlite3_module deltaparsevtabModule = { @@ -1058,7 +1072,8 @@ static sqlite3_module deltaparsevtabModule = { /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, - /* xShadowName */ 0 + /* xShadowName */ 0, + /* xIntegrity */ 0 }; @@ -1067,8 +1082,8 @@ static sqlite3_module deltaparsevtabModule = { __declspec(dllexport) #endif int sqlite3_fossildelta_init( - sqlite3 *db, - char **pzErrMsg, + sqlite3 *db, + char **pzErrMsg, const sqlite3_api_routines *pApi ){ static const int enc = SQLITE_UTF8|SQLITE_INNOCUOUS; diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 65d9d8df69..e16d005d9c 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -68,7 +68,7 @@ ** AND distance<200; ** ** This first query outputs the string "abcdefg" and all strings that -** can be derived from that string by appling the specified transformations. +** can be derived from that string by applying the specified transformations. ** The strings are output together with their total transformation cost ** (called "distance") and appear in order of increasing cost. No string ** is output more than once. If there are multiple ways to transform the @@ -97,7 +97,7 @@ ** LIMIT 20 ** ** The query above gives the 20 closest words to the $word being tested. -** (Note that for good performance, the vocubulary.w column should be +** (Note that for good performance, the vocabulary.w column should be ** indexed.) ** ** A similar query can be used to find all words in the dictionary that @@ -207,7 +207,7 @@ struct fuzzer_rule { ** Every stem is added to a hash table as it is output. Generation of ** duplicate stems is suppressed. ** -** Active stems (those that might generate new outputs) are kepts on a linked +** Active stems (those that might generate new outputs) are kept on a linked ** list sorted by increasing cost. The cost is the sum of rBaseCost and ** pRule->rCost. */ @@ -1165,6 +1165,11 @@ static sqlite3_module fuzzerModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index ff5d2d333c..7f1c6d9e1f 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -79,7 +79,7 @@ ** WITH c(name,bin) AS (VALUES ** ('minimum positive value', x'0000000000000001'), ** ('maximum subnormal value', x'000fffffffffffff'), -** ('mininum positive nornal value', x'0010000000000000'), +** ('minimum positive normal value', x'0010000000000000'), ** ('maximum value', x'7fefffffffffffff')) ** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) ** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); @@ -134,6 +134,9 @@ static void ieee754func( if( a==0 ){ e = 0; m = 0; + }else if( a==(sqlite3_int64)0x8000000000000000LL ){ + e = -1996; + m = -1; }else{ e = a>>52; m = a & ((((sqlite3_int64)1)<<52)-1); @@ -256,6 +259,37 @@ static void ieee754func_to_blob( } } +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + #ifdef _WIN32 __declspec(dllexport) @@ -277,7 +311,7 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, - + { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; int rc = SQLITE_OK; diff --git a/ext/misc/memstat.c b/ext/misc/memstat.c index 800a86e7a4..8e69b46955 100644 --- a/ext/misc/memstat.c +++ b/ext/misc/memstat.c @@ -396,12 +396,19 @@ static sqlite3_module memstatModule = { 0, /* xRelease */ 0, /* xRollbackTo */ 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ -int sqlite3MemstatVtabInit(sqlite3 *db){ +int sqlite3MemstatVtabInit( + sqlite3 *db, + char **NotUsed1, + const sqlite3_api_routines *NotUsed2 +){ int rc = SQLITE_OK; + (void)NotUsed1; + (void)NotUsed2; #ifndef SQLITE_OMIT_VIRTUALTABLE rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0); #endif @@ -420,7 +427,7 @@ int sqlite3_memstat_init( int rc = SQLITE_OK; SQLITE_EXTENSION_INIT2(pApi); #ifndef SQLITE_OMIT_VIRTUALTABLE - rc = sqlite3MemstatVtabInit(db); + rc = sqlite3MemstatVtabInit(db, 0, 0); #endif return rc; } diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c deleted file mode 100644 index 83fc9468e6..0000000000 --- a/ext/misc/memvfs.c +++ /dev/null @@ -1,575 +0,0 @@ -/* -** 2016-09-07 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is an in-memory VFS implementation. The application supplies -** a chunk of memory to hold the database file. -** -** Because there is place to store a rollback or wal journal, the database -** must use one of journal_mode=MEMORY or journal_mode=NONE. -** -** USAGE: -** -** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db, -** SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI, -** "memvfs"); -** -** These are the query parameters: -** -** ptr= The address of the memory buffer that holds the database. -** -** sz= The current size the database file -** -** maxsz= The maximum size of the database. In other words, the -** amount of space allocated for the ptr= buffer. -** -** freeonclose= If true, then sqlite3_free() is called on the ptr= -** value when the connection closes. -** -** The ptr= and sz= query parameters are required. If maxsz= is omitted, -** then it defaults to the sz= value. Parameter values can be in either -** decimal or hexadecimal. The filename in the URI is ignored. -*/ -#include -SQLITE_EXTENSION_INIT1 -#include -#include - - -/* -** Forward declaration of objects used by this utility -*/ -typedef struct sqlite3_vfs MemVfs; -typedef struct MemFile MemFile; - -/* Access to a lower-level VFS that (might) implement dynamic loading, -** access to randomness, etc. -*/ -#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) - -/* An open file */ -struct MemFile { - sqlite3_file base; /* IO methods */ - sqlite3_int64 sz; /* Size of the file */ - sqlite3_int64 szMax; /* Space allocated to aData */ - unsigned char *aData; /* content of the file */ - int bFreeOnClose; /* Invoke sqlite3_free() on aData at close */ -}; - -/* -** Methods for MemFile -*/ -static int memClose(sqlite3_file*); -static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); -static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); -static int memTruncate(sqlite3_file*, sqlite3_int64 size); -static int memSync(sqlite3_file*, int flags); -static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); -static int memLock(sqlite3_file*, int); -static int memUnlock(sqlite3_file*, int); -static int memCheckReservedLock(sqlite3_file*, int *pResOut); -static int memFileControl(sqlite3_file*, int op, void *pArg); -static int memSectorSize(sqlite3_file*); -static int memDeviceCharacteristics(sqlite3_file*); -static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); -static int memShmLock(sqlite3_file*, int offset, int n, int flags); -static void memShmBarrier(sqlite3_file*); -static int memShmUnmap(sqlite3_file*, int deleteFlag); -static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); -static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); - -/* -** Methods for MemVfs -*/ -static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); -static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); -static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); -static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); -static void *memDlOpen(sqlite3_vfs*, const char *zFilename); -static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); -static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); -static void memDlClose(sqlite3_vfs*, void*); -static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); -static int memSleep(sqlite3_vfs*, int microseconds); -static int memCurrentTime(sqlite3_vfs*, double*); -static int memGetLastError(sqlite3_vfs*, int, char *); -static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); - -static sqlite3_vfs mem_vfs = { - 2, /* iVersion */ - 0, /* szOsFile (set when registered) */ - 1024, /* mxPathname */ - 0, /* pNext */ - "memvfs", /* zName */ - 0, /* pAppData (set when registered) */ - memOpen, /* xOpen */ - memDelete, /* xDelete */ - memAccess, /* xAccess */ - memFullPathname, /* xFullPathname */ - memDlOpen, /* xDlOpen */ - memDlError, /* xDlError */ - memDlSym, /* xDlSym */ - memDlClose, /* xDlClose */ - memRandomness, /* xRandomness */ - memSleep, /* xSleep */ - memCurrentTime, /* xCurrentTime */ - memGetLastError, /* xGetLastError */ - memCurrentTimeInt64 /* xCurrentTimeInt64 */ -}; - -static const sqlite3_io_methods mem_io_methods = { - 3, /* iVersion */ - memClose, /* xClose */ - memRead, /* xRead */ - memWrite, /* xWrite */ - memTruncate, /* xTruncate */ - memSync, /* xSync */ - memFileSize, /* xFileSize */ - memLock, /* xLock */ - memUnlock, /* xUnlock */ - memCheckReservedLock, /* xCheckReservedLock */ - memFileControl, /* xFileControl */ - memSectorSize, /* xSectorSize */ - memDeviceCharacteristics, /* xDeviceCharacteristics */ - memShmMap, /* xShmMap */ - memShmLock, /* xShmLock */ - memShmBarrier, /* xShmBarrier */ - memShmUnmap, /* xShmUnmap */ - memFetch, /* xFetch */ - memUnfetch /* xUnfetch */ -}; - - - -/* -** Close an mem-file. -** -** The pData pointer is owned by the application, so there is nothing -** to free. -*/ -static int memClose(sqlite3_file *pFile){ - MemFile *p = (MemFile *)pFile; - if( p->bFreeOnClose ) sqlite3_free(p->aData); - return SQLITE_OK; -} - -/* -** Read data from an mem-file. -*/ -static int memRead( - sqlite3_file *pFile, - void *zBuf, - int iAmt, - sqlite_int64 iOfst -){ - MemFile *p = (MemFile *)pFile; - memcpy(zBuf, p->aData+iOfst, iAmt); - return SQLITE_OK; -} - -/* -** Write data to an mem-file. -*/ -static int memWrite( - sqlite3_file *pFile, - const void *z, - int iAmt, - sqlite_int64 iOfst -){ - MemFile *p = (MemFile *)pFile; - if( iOfst+iAmt>p->sz ){ - if( iOfst+iAmt>p->szMax ) return SQLITE_FULL; - if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); - p->sz = iOfst+iAmt; - } - memcpy(p->aData+iOfst, z, iAmt); - return SQLITE_OK; -} - -/* -** Truncate an mem-file. -*/ -static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ - MemFile *p = (MemFile *)pFile; - if( size>p->sz ){ - if( size>p->szMax ) return SQLITE_FULL; - memset(p->aData+p->sz, 0, size-p->sz); - } - p->sz = size; - return SQLITE_OK; -} - -/* -** Sync an mem-file. -*/ -static int memSync(sqlite3_file *pFile, int flags){ - return SQLITE_OK; -} - -/* -** Return the current file-size of an mem-file. -*/ -static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ - MemFile *p = (MemFile *)pFile; - *pSize = p->sz; - return SQLITE_OK; -} - -/* -** Lock an mem-file. -*/ -static int memLock(sqlite3_file *pFile, int eLock){ - return SQLITE_OK; -} - -/* -** Unlock an mem-file. -*/ -static int memUnlock(sqlite3_file *pFile, int eLock){ - return SQLITE_OK; -} - -/* -** Check if another file-handle holds a RESERVED lock on an mem-file. -*/ -static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - *pResOut = 0; - return SQLITE_OK; -} - -/* -** File control method. For custom operations on an mem-file. -*/ -static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ - MemFile *p = (MemFile *)pFile; - int rc = SQLITE_NOTFOUND; - if( op==SQLITE_FCNTL_VFSNAME ){ - *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); - rc = SQLITE_OK; - } - return rc; -} - -/* -** Return the sector-size in bytes for an mem-file. -*/ -static int memSectorSize(sqlite3_file *pFile){ - return 1024; -} - -/* -** Return the device characteristic flags supported by an mem-file. -*/ -static int memDeviceCharacteristics(sqlite3_file *pFile){ - return SQLITE_IOCAP_ATOMIC | - SQLITE_IOCAP_POWERSAFE_OVERWRITE | - SQLITE_IOCAP_SAFE_APPEND | - SQLITE_IOCAP_SEQUENTIAL; -} - -/* Create a shared memory file mapping */ -static int memShmMap( - sqlite3_file *pFile, - int iPg, - int pgsz, - int bExtend, - void volatile **pp -){ - return SQLITE_IOERR_SHMMAP; -} - -/* Perform locking on a shared-memory segment */ -static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ - return SQLITE_IOERR_SHMLOCK; -} - -/* Memory barrier operation on shared memory */ -static void memShmBarrier(sqlite3_file *pFile){ - return; -} - -/* Unmap a shared memory segment */ -static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ - return SQLITE_OK; -} - -/* Fetch a page of a memory-mapped file */ -static int memFetch( - sqlite3_file *pFile, - sqlite3_int64 iOfst, - int iAmt, - void **pp -){ - MemFile *p = (MemFile *)pFile; - *pp = (void*)(p->aData + iOfst); - return SQLITE_OK; -} - -/* Release a memory-mapped page */ -static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ - return SQLITE_OK; -} - -/* -** Open an mem file handle. -*/ -static int memOpen( - sqlite3_vfs *pVfs, - const char *zName, - sqlite3_file *pFile, - int flags, - int *pOutFlags -){ - MemFile *p = (MemFile*)pFile; - memset(p, 0, sizeof(*p)); - if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; - p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); - if( p->aData==0 ) return SQLITE_CANTOPEN; - p->sz = sqlite3_uri_int64(zName,"sz",0); - if( p->sz<0 ) return SQLITE_CANTOPEN; - p->szMax = sqlite3_uri_int64(zName,"max",p->sz); - if( p->szMaxsz ) return SQLITE_CANTOPEN; - p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0); - pFile->pMethods = &mem_io_methods; - return SQLITE_OK; -} - -/* -** Delete the file located at zPath. If the dirSync argument is true, -** ensure the file-system modifications are synced to disk before -** returning. -*/ -static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - return SQLITE_IOERR_DELETE; -} - -/* -** Test for access permissions. Return true if the requested permission -** is available, or false otherwise. -*/ -static int memAccess( - sqlite3_vfs *pVfs, - const char *zPath, - int flags, - int *pResOut -){ - *pResOut = 0; - return SQLITE_OK; -} - -/* -** Populate buffer zOut with the full canonical pathname corresponding -** to the pathname in zPath. zOut is guaranteed to point to a buffer -** of at least (INST_MAX_PATHNAME+1) bytes. -*/ -static int memFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nOut, - char *zOut -){ - sqlite3_snprintf(nOut, zOut, "%s", zPath); - return SQLITE_OK; -} - -/* -** Open the dynamic library located at zPath and return a handle. -*/ -static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ - return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); -} - -/* -** Populate the buffer zErrMsg (size nByte bytes) with a human readable -** utf-8 string describing the most recent error encountered associated -** with dynamic libraries. -*/ -static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ - ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); -} - -/* -** Return a pointer to the symbol zSymbol in the dynamic library pHandle. -*/ -static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ - return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); -} - -/* -** Close the dynamic library handle pHandle. -*/ -static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ - ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); -} - -/* -** Populate the buffer pointed to by zBufOut with nByte bytes of -** random data. -*/ -static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ - return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); -} - -/* -** Sleep for nMicro microseconds. Return the number of microseconds -** actually slept. -*/ -static int memSleep(sqlite3_vfs *pVfs, int nMicro){ - return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); -} - -/* -** Return the current time as a Julian Day number in *pTimeOut. -*/ -static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ - return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); -} - -static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ - return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); -} -static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ - return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); -} - -#ifdef MEMVFS_TEST -/* -** memvfs_from_file(FILENAME, MAXSIZE) -** -** This an SQL function used to help in testing the memvfs VFS. The -** function reads the content of a file into memory and then returns -** a URI that can be handed to ATTACH to attach the memory buffer as -** a database. Example: -** -** ATTACH memvfs_from_file('test.db',1048576) AS inmem; -** -** The optional MAXSIZE argument gives the size of the memory allocation -** used to hold the database. If omitted, it defaults to the size of the -** file on disk. -*/ -#include -static void memvfsFromFileFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - unsigned char *p; - sqlite3_int64 sz; - sqlite3_int64 szMax; - FILE *in; - const char *zFilename = (const char*)sqlite3_value_text(argv[0]); - char *zUri; - - if( zFilename==0 ) return; - in = fopen(zFilename, "rb"); - if( in==0 ) return; - fseek(in, 0, SEEK_END); - szMax = sz = ftell(in); - rewind(in); - if( argc>=2 ){ - szMax = sqlite3_value_int64(argv[1]); - if( szMaxzName,"memvfs")!=0 ) return; - rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); - if( rc ) return; - fwrite(p->aData, 1, (size_t)p->sz, out); - fclose(out); -} -#endif /* MEMVFS_TEST */ - -#ifdef MEMVFS_TEST -/* Called for each new database connection */ -static int memvfsRegister( - sqlite3 *db, - char **pzErrMsg, - const struct sqlite3_api_routines *pThunk -){ - sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0, - memvfsFromFileFunc, 0, 0); - sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0, - memvfsFromFileFunc, 0, 0); - sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0, - memvfsToFileFunc, 0, 0); - return SQLITE_OK; -} -#endif /* MEMVFS_TEST */ - - -#ifdef _WIN32 -__declspec(dllexport) -#endif -/* -** This routine is called when the extension is loaded. -** Register the new VFS. -*/ -int sqlite3_memvfs_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - mem_vfs.pAppData = sqlite3_vfs_find(0); - if( mem_vfs.pAppData==0 ) return SQLITE_ERROR; - mem_vfs.szOsFile = sizeof(MemFile); - rc = sqlite3_vfs_register(&mem_vfs, 1); -#ifdef MEMVFS_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); - } - if( rc==SQLITE_OK ){ - rc = memvfsRegister(db, pzErrMsg, pApi); - } -#endif - if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; - return rc; -} diff --git a/ext/misc/mmapwarm.c b/ext/misc/mmapwarm.c index 5afa47bf7a..851f8b0eb7 100644 --- a/ext/misc/mmapwarm.c +++ b/ext/misc/mmapwarm.c @@ -38,7 +38,7 @@ int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; char *zSql = 0; int pgsz = 0; - int nTotal = 0; + unsigned int nTotal = 0; if( 0==sqlite3_get_autocommit(db) ) return SQLITE_MISUSE; @@ -86,8 +86,8 @@ int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){ rc = p->xFetch(pFd, pgsz*iPg, pgsz, (void**)&pMap); if( rc!=SQLITE_OK || pMap==0 ) break; - nTotal += pMap[0]; - nTotal += pMap[pgsz-1]; + nTotal += (unsigned int)pMap[0]; + nTotal += (unsigned int)pMap[pgsz-1]; rc = p->xUnfetch(pFd, pgsz*iPg, (void*)pMap); if( rc!=SQLITE_OK ) break; @@ -103,5 +103,6 @@ int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){ if( rc==SQLITE_OK ) rc = rc2; } + (void)nTotal; return rc; } diff --git a/ext/misc/noop.c b/ext/misc/noop.c index d3a58670c4..18c25e10f7 100644 --- a/ext/misc/noop.c +++ b/ext/misc/noop.c @@ -38,6 +38,24 @@ static void noopfunc( sqlite3_result_value(context, argv[0]); } +/* +** Implementation of the multitype_text() function. +** +** The function returns its argument. The result will always have a +** TEXT value. But if the original input is numeric, it will also +** have that numeric value. +*/ +static void multitypeTextFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + (void)argc; + (void)sqlite3_value_text(argv[0]); + sqlite3_result_value(context, argv[0]); +} + #ifdef _WIN32 __declspec(dllexport) #endif @@ -64,5 +82,9 @@ int sqlite3_noop_init( rc = sqlite3_create_function(db, "noop_nd", 1, SQLITE_UTF8, 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "multitype_text", 1, + SQLITE_UTF8, + 0, multitypeTextFunc, 0, 0); return rc; } diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c index 08d7733b96..44ddcd3882 100644 --- a/ext/misc/normalize.c +++ b/ext/misc/normalize.c @@ -286,7 +286,7 @@ static const unsigned char sqlite3CtypeMap[256] = { #define TK_VARIABLE TK_LITERAL #define TK_BLOB TK_LITERAL -/* Disable nuisence warnings about case fall-through */ +/* Disable nuisance warnings about case fall-through */ #if !defined(deliberate_fall_through) && defined(__GCC__) && __GCC__>=7 # define deliberate_fall_through __attribute__((fallthrough)); #else @@ -297,8 +297,9 @@ static const unsigned char sqlite3CtypeMap[256] = { ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ -static int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; +static sqlite3_int64 sqlite3GetToken(const unsigned char *z, int *tokenType){ + sqlite3_int64 i; + int c; switch( aiClass[*z] ){ /* Switch on the character-class of the first byte ** of the token. See the comment on the CC_ defines ** above. */ @@ -559,7 +560,7 @@ char *sqlite3_normalize(const char *zSql){ int i; /* Next character to read from zSql[] */ int j; /* Next slot to fill in on z[] */ int tokenType; /* Type of the next token */ - int n; /* Size of the next token */ + sqlite3_int64 n; /* Size of the next token */ int k; /* Loop counter */ nSql = strlen(zSql); diff --git a/ext/misc/pcachetrace.c b/ext/misc/pcachetrace.c new file mode 100644 index 0000000000..3757d8d4d5 --- /dev/null +++ b/ext/misc/pcachetrace.c @@ -0,0 +1,179 @@ +/* +** 2023-06-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_PCACHE2 +** mechanism to add a tracing layer on top of pluggable page cache of +** SQLite. If this extension is registered prior to sqlite3_initialize(), +** it will cause all page cache activities to be logged on standard output, +** or to some other FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --pcachetrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg +){ + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); + } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); + } + pcacheBase.xUnpin(p, pPg, bDiscard); +} +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } + } + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } + } + pcachetraceOut = 0; + return rc; +} diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index d83bc5b838..98e45cc3a8 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -11,7 +11,7 @@ ****************************************************************************** ** ** This file contains code to implement the percentile(Y,P) SQL function -** as described below: +** and similar as described below: ** ** (1) The percentile(Y,P) function is an aggregate function taking ** exactly two arguments. @@ -57,29 +57,108 @@ ** (12) The percentile(Y,P) is implemented as a single C99 source-code ** file that compiles into a shared-library or DLL that can be loaded ** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cont(Y,P) function is equivalent to +** percentile(Y,P/100.0). In other words, the fraction value in +** the second argument is in the range of 0 to 1 instead of 0 to 100. +** +** (15) A separate percentile_disc(Y,P) function is like +** percentile_cont(Y,P) except that instead of returning the weighted +** average of the nearest two input values, it returns the next lower +** value. So the percentile_disc(Y,P) will always return a value +** that was one of the inputs. +** +** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and +** percentile_disc(Y,P) can be used as window functions. +** +** Differences from standard SQL: +** +** * The percentile_cont(X,P) function is equivalent to the following in +** standard SQL: +** +** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) +** +** The SQLite syntax is much more compact. The standard SQL syntax +** is also supported if SQLite is compiled with the +** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. +** +** * No median(X) function exists in the SQL standard. App developers +** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". +** +** * No percentile(Y,P) function exists in the SQL standard. Instead of +** percential(Y,P), developers must write this: +** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that +** the fraction parameter to percentile() goes from 0 to 100 whereas +** the fraction parameter in SQL standard percentile_cont() goes from +** 0 to 1. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of these routines work +** by accumulating all values in an array of doubles, then sorting +** that array using quicksort before computing the answer. Thus +** the runtime is O(NlogN) where N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can imagine alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. */ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 +#if defined(SQLITE3_H) + /* no-op */ +#elif defined(SQLITE_STATIC_PERCENTILE) +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif #include #include #include -/* The following object is the session context for a single percentile() -** function. We have to remember all input Y values until the very end. +/* The following object is the group context for a single percentile() +** aggregate. Remember all input Y values until the very end. ** Those values are accumulated in the Percentile.a[] array. */ typedef struct Percentile Percentile; struct Percentile { unsigned nAlloc; /* Number of slots allocated for a[] */ unsigned nUsed; /* Number of slots actually used in a[] */ - double rPct; /* 1.0 more than the value for P */ + char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ + char bPctValid; /* True if rPct is valid */ + double rPct; /* Fraction. 0.0 to 1.0 */ double *a; /* Array of Y values */ }; +/* Details of each function in the percentile family */ +typedef struct PercentileFunc PercentileFunc; +struct PercentileFunc { + const char *zName; /* Function name */ + char nArg; /* Number of arguments */ + char mxFrac; /* Maximum value of the "fraction" input */ + char bDiscrete; /* True for percentile_disc() */ +}; +static const PercentileFunc aPercentFunc[] = { + { "median", 1, 1, 0 }, + { "percentile", 2, 100, 0 }, + { "percentile_cont", 2, 1, 0 }, + { "percentile_disc", 2, 1, 1 }, +}; + /* ** Return TRUE if the input floating-point number is an infinity. */ -static int isInfinity(double r){ +static int percentIsInfinity(double r){ sqlite3_uint64 u; assert( sizeof(u)==sizeof(r) ); memcpy(&u, &r, sizeof(u)); @@ -87,13 +166,64 @@ static int isInfinity(double r){ } /* -** Return TRUE if two doubles differ by 0.001 or less +** Return TRUE if two doubles differ by 0.001 or less. */ -static int sameValue(double a, double b){ +static int percentSameValue(double a, double b){ a -= b; return a>=-0.001 && a<=0.001; } +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static int percentBinarySearch(Percentile *p, double y, int bExact){ + int iFirst = 0; /* First element of search range */ + int iLast = p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + int iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( xy ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + +/* +** Generate an error for a percentile function. +** +** The error format string must have exactly one occurrence of "%%s()" +** (with two '%' characters). That substring will be replaced by the name +** of the function. +*/ +static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + char *zMsg1; + char *zMsg2; + va_list ap; + + va_start(ap, zFormat); + zMsg1 = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0; + sqlite3_result_error(pCtx, zMsg2, -1); + sqlite3_free(zMsg1); + sqlite3_free(zMsg2); +} + /* ** The "step" function for percentile(Y,P) is called once for each ** input row. @@ -103,16 +233,24 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ double rPct; int eType; double y; - assert( argc==2 ); - - /* Requirement 3: P must be a number between 0 and 100 */ - eType = sqlite3_value_numeric_type(argv[1]); - rPct = sqlite3_value_double(argv[1]); - if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) - || rPct<0.0 || rPct>100.0 ){ - sqlite3_result_error(pCtx, "2nd argument to percentile() is not " - "a number between 0.0 and 100.0", -1); - return; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 0.5; + }else{ + /* Requirement 3: P must be a number between 0 and 100 */ + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac; + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 + ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not between 0.0 and %.1f", + (double)pFunc->mxFrac); + return; + } } /* Allocate the session context. */ @@ -121,11 +259,12 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ /* Remember the P value. Throw an error if the P value is different ** from any prior row, per Requirement (2). */ - if( p->rPct==0.0 ){ - p->rPct = rPct+1.0; - }else if( !sameValue(p->rPct,rPct+1.0) ){ - sqlite3_result_error(pCtx, "2nd argument to percentile() is not the " - "same for all input rows", -1); + if( !p->bPctValid ){ + p->rPct = rPct; + p->bPctValid = 1; + }else if( !percentSameValue(p->rPct,rPct) ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not the same for all input rows"); return; } @@ -136,15 +275,14 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ /* If not NULL, then Y must be numeric. Otherwise throw an error. ** Requirement 4 */ if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ - sqlite3_result_error(pCtx, "1st argument to percentile() is not " - "numeric", -1); + percentError(pCtx, "input to %%s() is not numeric"); return; } /* Throw an error if the Y value is infinity or NaN */ y = sqlite3_value_double(argv[0]); - if( isInfinity(y) ){ - sqlite3_result_error(pCtx, "Inf input to percentile()", -1); + if( percentIsInfinity(y) ){ + percentError(pCtx, "Inf input to %%s()"); return; } @@ -161,26 +299,143 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ p->nAlloc = n; p->a = a; } - p->a[p->nUsed++] = y; + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; + p->bSorted = 1; + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + int i; + i = percentBinarySearch(p, y, 0); + if( i<(int)p->nUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; + p->bSorted = 0; + } } /* -** Compare to doubles for sorting using qsort() +** Interchange two doubles. */ -static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){ - double a = *(double*)pA; - double b = *(double*)pB; - if( a==b ) return 0; - if( a=2 ); + if( a[0]>a[n-1] ){ + SWAP_DOUBLE(a[0],a[n-1]) + } + if( n==2 ) return; + iGt = n-1; + i = n/2; + if( a[0]>a[i] ){ + SWAP_DOUBLE(a[0],a[i]) + }else if( a[i]>a[iGt] ){ + SWAP_DOUBLE(a[i],a[iGt]) + } + if( n==3 ) return; + rPivot = a[i]; + iLt = i = 1; + do{ + if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + SWAP_DOUBLE(a[i],a[iGt]) + }else{ + i++; + } + }while( i=2 ) percentSort(a, iLt); + if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; ibSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( i<(int)p->nUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. +*/ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ + Percentile *p; + PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx); unsigned i1, i2; double v1, v2; double ix, vx; @@ -188,21 +443,38 @@ static void percentFinal(sqlite3_context *pCtx){ if( p==0 ) return; if( p->a==0 ) return; if( p->nUsed ){ - qsort(p->a, p->nUsed, sizeof(double), doubleCmp); - ix = (p->rPct-1.0)*(p->nUsed-1)*0.01; + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + ix = p->rPct*(p->nUsed-1); i1 = (unsigned)ix; - i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; - v1 = p->a[i1]; - v2 = p->a[i2]; - vx = v1 + (v2-v1)*(ix-i1); + if( pFunc->bDiscrete ){ + vx = p->a[i1]; + }else{ + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + } sqlite3_result_double(pCtx, vx); } - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; + } +} +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); } - -#ifdef _WIN32 +#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE) __declspec(dllexport) #endif int sqlite3_percentile_init( @@ -211,10 +483,21 @@ int sqlite3_percentile_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; + unsigned int i; +#ifdef SQLITE3EXT_H SQLITE_EXTENSION_INIT2(pApi); +#else + (void)pApi; /* Unused parameter */ +#endif (void)pzErrMsg; /* Unused parameter */ - rc = sqlite3_create_function(db, "percentile", 2, - SQLITE_UTF8|SQLITE_INNOCUOUS, 0, - 0, percentStep, percentFinal); + for(i=0; i #include #include @@ -51,17 +56,18 @@ static unsigned int prngInt(Prng *p){ return p->x ^ p->y; } -static const char *azJsonAtoms[] = { - /* JSON /* JSON-5 */ +static char *azJsonAtoms[] = { + /* JSON JSON-5 */ "0", "0", "1", "1", "-1", "-1", "2", "+2", - "3", "3", - "2.5", "2.5", + "3DDDD", "3DDDD", + "2.5DD", "2.5DD", "0.75", ".75", "-4.0e2", "-4.e2", "5.0e-3", "+5e-3", + "6.DDe+0DD", "6.DDe+0DD", "0", "0x0", "512", "0x200", "256", "+0x100", @@ -73,12 +79,14 @@ static const char *azJsonAtoms[] = { "-9.0e999", "-Infinity", "9.0e999", "+Infinity", "null", "NaN", - "-0.0005123", "-0.0005123", + "-0.0005DD", "-0.0005DD", "4.35e-3", "+4.35e-3", "\"gem\\\"hay\"", "\"gem\\\"hay\"", "\"icy'joy\"", "'icy\\'joy\'", "\"keylog\"", "\"key\\\nlog\"", "\"mix\\\\\\tnet\"", "\"mix\\\\\\tnet\"", + "\"oat\\r\\n\"", "\"oat\\r\\n\"", + "\"\\fpan\\b\"", "\"\\fpan\\b\"", "{}", "{}", "[]", "[]", "[]", "[/*empty*/]", @@ -89,19 +97,20 @@ static const char *azJsonAtoms[] = { "\"day\"", "\"day\"", "\"end\"", "'end'", "\"fly\"", "\"fly\"", + "\"\\u00XX\\u00XX\"", "\"\\xXX\\xXX\"", + "\"y\\uXXXXz\"", "\"y\\uXXXXz\"", "\"\"", "\"\"", }; -static const char *azJsonTemplate[] = { +static char *azJsonTemplate[] = { /* JSON JSON-5 */ - "{\"a\":%,\"b\":%,\"c\":%}", "{a:%,b:%,c:%}", + "{\"a\":%,\"b\":%,\"cDD\":%}", "{a:%,b:%,cDD:%}", "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}", - "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,\"\":%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,'':%}", "{\"d\":%}", "{d:%}", "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", - "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", + "{\"$g\":%,\"_h_\":%,\"a b c d\":%}", "{$g:%,_h_:%,\"a b c d\":%}", "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", - "{\"a b c d\":%,\"e\":%,\"f\":%,\"x\":%,\"y\":%}", - "{\"a b c d\":%,e:%,f:%,x:%,y:%}", + "{\"\\u00XX\":%,\"\\uXXXX\":%}", "{\"\\xXX\":%,\"\\uXXXX\":%}", "{\"Z\":%}", "{Z:%,}", "[%]", "[%,]", "[%,%]", "[%,%]", @@ -122,15 +131,13 @@ static void jsonExpand( unsigned int r /* Growth probability 0..1000. 0 means no growth */ ){ unsigned int i, j, k; - const char *z; + char *z; + char *zX; size_t n; + char zBuf[200]; j = 0; - if( zSrc==0 ){ - k = prngInt(p)%(count(azJsonTemplate)/2); - k = k*2 + eType; - zSrc = azJsonTemplate[k]; - } + if( zSrc==0 ) zSrc = "%"; if( strlen(zSrc)>=STRSZ/10 ) r = 0; for(i=0; zSrc[i]; i++){ if( zSrc[i]!='%' ){ @@ -149,9 +156,36 @@ static void jsonExpand( z = azJsonTemplate[k]; } n = strlen(z); + if( (zX = strstr(z,"XX"))!=0 ){ + unsigned int y = prngInt(p); + if( (y&0xff)==((y>>8)&0xff) ) y += 0x100; + while( (y&0xff)==((y>>16)&0xff) || ((y>>8)&0xff)==((y>>16)&0xff) ){ + y += 0x10000; + } + memcpy(zBuf, z, n+1); + z = zBuf; + zX = strstr(z,"XX"); + while( zX!=0 ){ + zX[0] = "0123456789abcdef"[y%16]; y /= 16; + zX[1] = "0123456789abcdef"[y%16]; y /= 16; + zX = strstr(zX, "XX"); + } + }else if( (zX = strstr(z,"DD"))!=0 ){ + unsigned int y = prngInt(p); + memcpy(zBuf, z, n+1); + z = zBuf; + zX = strstr(z,"DD"); + while( zX!=0 ){ + zX[0] = "0123456789"[y%10]; y /= 10; + zX[1] = "0123456789"[y%10]; y /= 10; + zX = strstr(zX, "DD"); + } + } + assert( strstr(z, "XX")==0 ); + assert( strstr(z, "DD")==0 ); if( j+n #include @@ -96,32 +98,6 @@ SQLITE_EXTENSION_INIT1 #define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ #define RE_OP_ATSTART 18 /* Currently at the start of the string */ -#if defined(SQLITE_DEBUG) -/* Opcode names used for symbolic debugging */ -static const char *ReOpName[] = { - "EOF", - "MATCH", - "ANY", - "ANYSTAR", - "FORK", - "GOTO", - "ACCEPT", - "CC_INC", - "CC_EXC", - "CC_VALUE", - "CC_RANGE", - "WORD", - "NOTWORD", - "DIGIT", - "NOTDIGIT", - "SPACE", - "NOTSPACE", - "BOUNDARY", - "ATSTART", -}; -#endif /* SQLITE_DEBUG */ - - /* Each opcode is a "state" in the NFA */ typedef unsigned short ReStateNumber; @@ -158,6 +134,7 @@ struct ReCompiled { int nInit; /* Number of bytes in zInit */ unsigned nState; /* Number of entries in aOp[] and aArg[] */ unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ + unsigned mxAlloc; /* Complexity limit */ }; /* Add a state to the given state set if it is not already there */ @@ -372,14 +349,15 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ /* Resize the opcode and argument arrays for an RE under construction. */ -static int re_resize(ReCompiled *p, int N){ +static int re_resize(ReCompiled *p, unsigned int N){ char *aOp; int *aArg; + if( N>p->mxAlloc ){ p->zErr = "REGEXP pattern too big"; return 1; } aOp = sqlite3_realloc64(p->aOp, N*sizeof(p->aOp[0])); - if( aOp==0 ) return 1; + if( aOp==0 ){ p->zErr = "out of memory"; return 1; } p->aOp = aOp; aArg = sqlite3_realloc64(p->aArg, N*sizeof(p->aArg[0])); - if( aArg==0 ) return 1; + if( aArg==0 ){ p->zErr = "out of memory"; return 1; } p->aArg = aArg; p->nAlloc = N; return 0; @@ -410,7 +388,7 @@ static int re_append(ReCompiled *p, int op, int arg){ /* Make a copy of N opcodes starting at iStart onto the end of the RE ** under construction. */ -static void re_copy(ReCompiled *p, int iStart, int N){ +static void re_copy(ReCompiled *p, int iStart, unsigned int N){ if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); @@ -563,18 +541,26 @@ static const char *re_subcompile_string(ReCompiled *p){ break; } case '{': { - int m = 0, n = 0; - int sz, j; + unsigned int m = 0, n = 0; + unsigned int sz, j; if( iPrev<0 ) return "'{m,n}' without operand"; - while( (c=rePeek(p))>='0' && c<='9' ){ m = m*10 + c - '0'; p->sIn.i++; } + while( (c=rePeek(p))>='0' && c<='9' ){ + m = m*10 + c - '0'; + if( m*2>p->mxAlloc ) return "REGEXP pattern too big"; + p->sIn.i++; + } n = m; if( c==',' ){ p->sIn.i++; n = 0; - while( (c=rePeek(p))>='0' && c<='9' ){ n = n*10 + c-'0'; p->sIn.i++; } + while( (c=rePeek(p))>='0' && c<='9' ){ + n = n*10 + c-'0'; + if( n*2>p->mxAlloc ) return "REGEXP pattern too big"; + p->sIn.i++; + } } if( c!='}' ) return "unmatched '{'"; - if( n>0 && nsIn.i++; sz = p->nState - iPrev; if( m==0 ){ @@ -590,7 +576,7 @@ static const char *re_subcompile_string(ReCompiled *p){ re_copy(p, iPrev, sz); } if( n==0 && m>0 ){ - re_append(p, RE_OP_FORK, -sz); + re_append(p, RE_OP_FORK, -(int)sz); } break; } @@ -664,13 +650,27 @@ static void re_free(ReCompiled *pRe){ } } +/* +** Version of re_free() that accepts a pointer of type (void*). Required +** to satisfy sanitizers when the re_free() function is called via a +** function pointer. +*/ +static void re_free_voidptr(void *p){ + re_free((ReCompiled*)p); +} + /* ** Compile a textual regular expression in zIn[] into a compiled regular ** expression suitable for us by re_match() and return a pointer to the ** compiled regular expression in *ppRe. Return NULL on success or an ** error message if something goes wrong. */ -static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ +static const char *re_compile( + ReCompiled **ppRe, /* OUT: write compiled NFA here */ + const char *zIn, /* Input regular expression */ + int mxRe, /* Complexity limit */ + int noCase /* True for caseless comparisons */ +){ ReCompiled *pRe; const char *zErr; int i, j; @@ -682,9 +682,11 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ } memset(pRe, 0, sizeof(*pRe)); pRe->xNextChar = noCase ? re_next_char_nocase : re_next_char; + pRe->mxAlloc = mxRe; if( re_resize(pRe, 30) ){ + zErr = pRe->zErr; re_free(pRe); - return "out of memory"; + return zErr; } if( zIn[0]=='^' ){ zIn++; @@ -737,6 +739,14 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ return pRe->zErr; } +/* +** Compute a reasonable limit on the length of the REGEXP NFA. +*/ +static int re_maxlen(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + return 75 + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1)/2; +} + /* ** Implementation of the regexp() SQL function. This function implements ** the build-in REGEXP operator. The first argument to the function is the @@ -762,7 +772,8 @@ static void re_sql_func( if( pRe==0 ){ zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + zErr = re_compile(&pRe, zPattern, re_maxlen(context), + sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -779,7 +790,7 @@ static void re_sql_func( sqlite3_result_int(context, re_match(pRe, zStr, -1)); } if( setAux ){ - sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + sqlite3_set_auxdata(context, 0, pRe, re_free_voidptr); } } @@ -804,10 +815,32 @@ static void re_bytecode_func( int n; char *z; (void)argc; + static const char *ReOpName[] = { + "EOF", + "MATCH", + "ANY", + "ANYSTAR", + "FORK", + "GOTO", + "ACCEPT", + "CC_INC", + "CC_EXC", + "CC_VALUE", + "CC_RANGE", + "WORD", + "NOTWORD", + "DIGIT", + "NOTDIGIT", + "SPACE", + "NOTSPACE", + "BOUNDARY", + "ATSTART", + }; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + zErr = re_compile(&pRe, zPattern, re_maxlen(context), + sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); diff --git a/ext/misc/series.c b/ext/misc/series.c index 0deabf95a6..ffdb23c1a0 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -30,19 +30,20 @@ ** SELECT * FROM generate_series(0,100,5); ** ** The query above returns integers from 0 through 100 counting by steps -** of 5. +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. ** ** SELECT * FROM generate_series(0,100); ** -** Integers from 0 through 100 with a step size of 1. +** Integers from 0 through 100 with a step size of 1. 101 rows. ** ** SELECT * FROM generate_series(20) LIMIT 10; ** -** Integers 20 through 29. +** Integers 20 through 29. 10 rows. ** ** SELECT * FROM generate_series(0,-100,-5); ** -** Integers 0 -5 -10 ... -100. +** Integers 0 -5 -10 ... -100. 21 rows. ** ** SELECT * FROM generate_series(0,-1); ** @@ -60,8 +61,7 @@ ** step HIDDEN ** ); ** -** The virtual table also has a rowid, logically equivalent to n+1 where -** "n" is the ascending integer in the aforesaid production definition. +** The virtual table also has a rowid which is an alias for the value. ** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other @@ -90,141 +90,117 @@ ** and a very large cost if either start or stop are unavailable. This ** encourages the query planner to order joins such that the bounds of the ** series are well-defined. +** +** Update on 2024-08-22: +** xBestIndex now also looks for equality and inequality constraints against +** the value column and uses those constraints as additional bounds against +** the sequence range. Thus, a query like this: +** +** SELECT value FROM generate_series($SA,$EA) +** WHERE value BETWEEN $SB AND $EB; +** +** Is logically the same as: +** +** SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB)); +** +** Constraints on the value column can server as substitutes for constraints +** on the hidden start and stop columns. So, the following two queries +** are equivalent: +** +** SELECT value FROM generate_series($S,$E); +** SELECT value FROM generate_series WHERE value BETWEEN $S and $E; +** */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Return that member of a generate_series(...) sequence whose 0-based -** index is ix. The 0th member is given by smBase. The sequence members -** progress per ix increment by smStep. -*/ -static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix){ - if( ix>=(sqlite3_uint64)LLONG_MAX ){ - /* Get ix into signed i64 range. */ - ix -= (sqlite3_uint64)LLONG_MAX; - /* With 2's complement ALU, this next can be 1 step, but is split into - * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (LLONG_MAX/2) * smStep; - smBase += (LLONG_MAX - LLONG_MAX/2) * smStep; - } - /* Under UBSAN (or on 1's complement machines), must do this last term - * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ - if( ix>=2 ){ - sqlite3_int64 ix2 = (sqlite3_int64)ix/2; - smBase += ix2*smStep; - ix -= ix2; - } - return smBase + ((sqlite3_int64)ix)*smStep; -} +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 +*/ typedef unsigned char u8; - -typedef struct SequenceSpec { - sqlite3_int64 iBase; /* Starting value ("start") */ - sqlite3_int64 iTerm; /* Given terminal value ("stop") */ - sqlite3_int64 iStep; /* Increment ("step") */ - sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ - sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ - sqlite3_int64 iValueNow; /* Current value during generation */ - u8 isNotEOF; /* Sequence generation not exhausted */ - u8 isReversing; /* Sequence is being reverse generated */ -} SequenceSpec; +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ +}; /* -** Prepare a SequenceSpec for use in generating an integer series -** given initialized iBase, iTerm and iStep values. Sequence is -** initialized per given isReversing. Other members are computed. +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. */ -static void setupSequence( SequenceSpec *pss ){ - int bSameSigns; - pss->uSeqIndexMax = 0; - pss->isNotEOF = 0; - bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); - if( pss->iTerm < pss->iBase ){ - sqlite3_uint64 nuspan = 0; - if( bSameSigns ){ - nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iBase>=0 and iTerm<0 . */ - nuspan = 1; - nuspan += pss->iBase; - nuspan += -(pss->iTerm+1); - } - if( pss->iStep<0 ){ - pss->isNotEOF = 1; - if( nuspan==ULONG_MAX ){ - pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; - }else if( pss->iStep>LLONG_MIN ){ - pss->uSeqIndexMax = nuspan/-pss->iStep; - } - } - }else if( pss->iTerm > pss->iBase ){ - sqlite3_uint64 puspan = 0; - if( bSameSigns ){ - puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iTerm>=0 and iBase<0 . */ - puspan = 1; - puspan += pss->iTerm; - puspan += -(pss->iBase+1); - } - if( pss->iStep>0 ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = puspan/pss->iStep; - } - }else if( pss->iTerm == pss->iBase ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = 0; - } - pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; - pss->iValueNow = (pss->isReversing) - ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) - : pss->iBase; -} +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} /* -** Progress sequence generator to yield next value, if any. -** Leave its state to either yield next value or be at EOF. -** Return whether there is a next value, or 0 at EOF. +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. */ -static int progressSequence( SequenceSpec *pss ){ - if( !pss->isNotEOF ) return 0; - if( pss->isReversing ){ - if( pss->uSeqIndexNow > 0 ){ - pss->uSeqIndexNow--; - pss->iValueNow -= pss->iStep; - }else{ - pss->isNotEOF = 0; - } - }else{ - if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ - pss->uSeqIndexNow++; - pss->iValueNow += pss->iStep; - }else{ - pss->isNotEOF = 0; - } - } - return pss->isNotEOF; +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; } - -/* series_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -typedef struct series_cursor series_cursor; -struct series_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - SequenceSpec ss; /* (this) Derived class data */ -}; /* ** The seriesConnect() method is invoked to create a new @@ -250,6 +226,7 @@ static int seriesConnect( int rc; /* Column numbers */ +#define SERIES_COLUMN_ROWID (-1) #define SERIES_COLUMN_VALUE 0 #define SERIES_COLUMN_START 1 #define SERIES_COLUMN_STOP 2 @@ -305,7 +282,15 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - progressSequence( & pCur->ss ); + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } return SQLITE_OK; } @@ -321,23 +306,27 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; - default: x = pCur->ss.iValueNow; break; + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; } +#ifndef LARGEST_UINT64 +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) +#endif + /* -** Return the rowid for the current row, logically equivalent to n+1 where -** "n" is the ascending integer in the aforesaid production definition. +** The rowid is the same as the value. */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - sqlite3_uint64 n = pCur->ss.uSeqIndexNow; - *pRowid = (sqlite3_int64)((n<0xffffffffffffffff)? n+1 : 0); + *pRowid = pCur->iValue; return SQLITE_OK; } @@ -347,7 +336,7 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - return !pCur->ss.isNotEOF; + return pCur->bDone; } /* True to cause run-time checking of the start=, stop=, and/or step= @@ -358,6 +347,59 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ # define SQLITE_SERIES_CONSTRAINT_VERIFY 0 #endif +/* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least @@ -368,13 +410,18 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** parameter. (idxStr is not used in this implementation.) idxNum ** is a bitmask showing which constraints are available: ** -** 1: start=VALUE -** 2: stop=VALUE -** 4: step=VALUE -** -** Also, if bit 8 is set, that means that the series should be output -** in descending order rather than in ascending order. If bit 16 is -** set, then output must appear in ascending order. +** 0x0001: start=VALUE +** 0x0002: stop=VALUE +** 0x0004: step=VALUE +** 0x0008: descending order +** 0x0010: ascending order +** 0x0020: LIMIT VALUE +** 0x0040: OFFSET VALUE +** 0x0080: value=VALUE +** 0x0100: value>=VALUE +** 0x0200: value>VALUE +** 0x1000: value<=VALUE +** 0x2000: value0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ + (void)idxStrUnused; - if( idxNum & 1 ){ - pCur->ss.iBase = sqlite3_value_int64(argv[i++]); + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i
  • demo-worker1-promiser: a demo of the Promise-based wrapper of the Worker1 API.
    • +
  • module-symbols gives + a high-level overview of the symbols exposed by the JS + module.
    • diff --git a/ext/wasm/index.html b/ext/wasm/index.html index a9872a9eaf..55e4cdb75c 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -31,6 +31,9 @@ file:// URLs.
  • Any OPFS-related pages or tests require:
      +
    • An OPFS-capable browser released after February + 2023. Some tests will work with Chromium-based browsers + going back to around v102.
    • That the web server emit the so-called COOP and @@ -38,12 +41,6 @@ headers. althttpd requires the -enable-sab flag for that.
      • -
    • A very recent version of a - Chromium-based browser (v102 at least, possibly newer). OPFS - support in the other major browsers is pending. Development - and testing is currently done against a dev-channel release - of Chrome (v111 as of 2023-02-10). -
    • @@ -52,20 +49,23 @@
    • Core-most tests
        -
      • tester1: Core unit and +
      • tester1 (32-bit + 64-bit): Core unit and regression tests for the various APIs and surrounding utility code.
        • -
      • tester1-worker: same thing - but running in a Worker.
        • -
      • tester1-esm: same as +
      • tester1-esm (32-bit + 64-bit): same as tester1 but loads sqlite3 in the main thread via an ES6 module.
        • -
      • tester1-worker?esm: - same as tester1-esm but loads a Worker Module which - then loads the sqlite3 API via an ES6 module. Note that - not all browsers permit loading modules in Worker - threads. +
      • tester1-worker (32-bit + 64-bit): same thing + but running in a Worker.
        • +
      • tester1-worker ESM (32-bit + 64-bit): same as + tester1-worker but loads a Worker Module which then + loads the sqlite3 API via an ES6 module. Not all + browsers permit loading modules in Worker threads.
      • @@ -87,15 +87,35 @@ wrapper is significantly easier to use, however.
    • demo-worker1-promiser: a demo of the Promise-based wrapper of the Worker1 API.
      • +
    • demo-worker1-promiser-esm: + same as the previous demo except loads the promiser from an ESM module.
  • speedtest1 ports (sqlite3's primary benchmarking tool)...
      -
    • speedtest1: a main-thread WASM build of speedtest1.
      • -
    • speedtest1?vfs=kvvfs: speedtest1 with the kvvfs.
      • -
    • speedtest1-worker: an interactive Worker-thread variant of speedtest1.
      • -
    • speedtest1-worker?vfs=opfs: speedtest1-worker with the +
    • speedtest1 + (32-bit, + 64-bit): + a main-thread WASM build of speedtest1.
      • +
    • speedtest1?vfs=kvvfs + (32-bit, + 64-bit): + speedtest1 with the kvvfs.
      • +
    • speedtest1-worker + (32-bit, + 64-bit): + an interactive Worker-thread variant of speedtest1.
      • +
    • speedtest1-worker?vfs=opfs + (32-bit, + 64-bit): + speedtest1-worker with the OPFS VFS preselected and configured for a moderate workload.
      • +
    • speedtest1-worker?vfs=opfs-sahpool + (32-bit, + 64-bit): + speedtest1-worker with the OPFS-SAHPOOL VFS preselected + and configured for a moderate workload. +
  • The obligatory "misc." category... @@ -103,7 +123,8 @@
  • module-symbols gives a high-level overview of the symbols exposed by the JS module.
    • -
  • batch-runner: runs batches of SQL exported from speedtest1.
    • +
  • test-opfs-vfs (same with verbose output and sanity-checking tests) is an @@ -115,20 +136,35 @@
  • OPFS concurrency tests using multiple workers.
    • +
  • OPFS SAHPool cooperative semi-concurrency + demonstrates usage of the OPFS SAHPool VFS's "pause" feature to coordinate + access to a database. +
    • +
  • SQLTester is + another testing tool for the library but making use of + it requires tests from an external, proprietary + project. Despite not being useful to the general public, + a link to it is in this list so that its developer has + easy access to it. +
    • - - + diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index bde7d051e8..8ea08e2136 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -16,33 +16,20 @@ Project homes: - https://fossil.wanderinghorse.net/r/jaccwabyt - https://sqlite.org/src/dir/ext/wasm/jaccwabyt - */ 'use strict'; globalThis.Jaccwabyt = function StructBinderFactory(config){ /* ^^^^ it is recommended that clients move that object into wherever - they'd like to have it and delete the self-held copy ("self" being - the global window or worker object). This API does not require the - global reference - it is simply installed as a convenience for - connecting these bits to other co-developed code before it gets - removed from the global namespace. + they'd like to have it and delete the globalThis-held copy. This + API does not require the global reference - it is simply installed + as a convenience for connecting these bits to other co-developed + code before it gets removed from the global namespace. */ /** Throws a new Error, the message of which is the concatenation all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; - /** - Implementing function bindings revealed significant - shortcomings in Emscripten's addFunction()/removeFunction() - interfaces: - - https://github.com/emscripten-core/emscripten/issues/17323 - - Until those are resolved, or a suitable replacement can be - implemented, our function-binding API will be more limited - and/or clumsier to use than initially hoped. - */ if(!(config.heap instanceof WebAssembly.Memory) && !(config.heap instanceof Function)){ toss("config.heap must be WebAssembly.Memory instance or a function."); @@ -51,23 +38,54 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ (config[k] instanceof Function) || toss("Config option '"+k+"' must be a function."); }); + const __heap = config.heap; const SBF = StructBinderFactory; - const heap = (config.heap instanceof Function) - ? config.heap : (()=>new Uint8Array(config.heap.buffer)), + const heap = __heap ? __heap : ()=>new Uint8Array(__heap.buffer), alloc = config.alloc, dealloc = config.dealloc, - log = config.log || console.log.bind(console), + log = config.log || console.debug.bind(console), memberPrefix = (config.memberPrefix || ""), memberSuffix = (config.memberSuffix || ""), - bigIntEnabled = (undefined===config.bigIntEnabled - ? !!self['BigInt64Array'] : !!config.bigIntEnabled), - BigInt = self['BigInt'], - BigInt64Array = self['BigInt64Array'], + BigInt = globalThis['BigInt'], + BigInt64Array = globalThis['BigInt64Array'], + bigIntEnabled = config.bigIntEnabled ?? !!BigInt64Array, + ptrIR = config.pointerIR + || config.ptrIR/*deprecated*/ + || 'i32', /* Undocumented (on purpose) config options: */ - ptrSizeof = config.ptrSizeof || 4, - ptrIR = config.ptrIR || 'i32' + ptrSize = config.ptrSize/*deprecated*/ + || ('i32'===ptrIR ? 4 : 8) ; + if(ptrIR!=='i32' && ptrIR!=='i64') toss("Invalid pointer representation:",ptrIR); + if(ptrSize!==4 && ptrSize!==8) toss("Invalid pointer size:",ptrSize); + + /** Either BigInt or, if !bigIntEnabled, a function which + throws complaining that BigInt is not enabled. */ + const __BigInt = (bigIntEnabled && BigInt) + ? (v)=>BigInt(v || 0) + : (v)=>toss("BigInt support is disabled in this build."); + const __asPtrType = ('i32'==ptrIR) ? Number : __BigInt; + const __NullPtr = __asPtrType(0); + + /** + Expects any number of numeric arguments, each one of either type + Number or BigInt. It sums them up (from an implicit starting + point of 0 or 0n) and returns them as a number of the same type + which target.asPtrType() uses. + + This is a workaround for not being able to mix Number/BigInt in + addition/subtraction expressions (which we frequently need for + calculating pointer offsets). + */ + const __ptrAdd = function(...args){ + let rc = __NullPtr; + for( let i = 0; i < args.length; ++i ){ + rc += __asPtrType(args[i]); + } + return rc; + }; + if(!SBF.debugFlags){ SBF.__makeDebugFlags = function(deriveFrom=null){ /* This is disgustingly overengineered. :/ */ @@ -103,12 +121,15 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ // Int16Array uses the platform's endianness. return new Int16Array(buffer)[0] === 256; })(); + /** Some terms used in the internal docs: StructType: a struct-wrapping class generated by this framework. + DEF: struct description object. + SIG: struct member signature string. */ @@ -118,6 +139,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ /** True if SIG s is-a pointer signature. */ const isPtrSig = (s)=>'p'===s || 'P'===s; const isAutoPtrSig = (s)=>'P'===s /*EXPERIMENTAL*/; + /** Returns p if SIG s is a function SIG, else returns s[0]. */ const sigLetter = (s)=>isFuncSig(s) ? 'p' : s[0]; /** Returns the WASM IR form of the Emscripten-conventional letter at SIG s[0]. Throws for an unknown SIG. */ @@ -140,7 +162,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const sigDVGetter = function(s){ switch(sigLetter(s)) { case 'p': case 'P': case 's': { - switch(ptrSizeof){ + switch(ptrSize){ case 4: return 'getInt32'; case 8: return affirmBigIntArray() && 'getBigInt64'; } @@ -160,7 +182,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const sigDVSetter = function(s){ switch(sigLetter(s)){ case 'p': case 'P': case 's': { - switch(ptrSizeof){ + switch(ptrSize){ case 4: return 'setInt32'; case 8: return affirmBigIntArray() && 'setBigInt64'; } @@ -175,19 +197,20 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ } toss("Unhandled DataView setter for signature:",s); }; + /** - Returns either Number of BigInt, depending on the given - SIG. This constructor is used in property setters to coerce - the being-set value to the correct size. + Returns a factory for either Number or BigInt, depending on the + given SIG. This constructor is used in property setters to coerce + the being-set value to the correct pointer size. */ const sigDVSetWrapper = function(s){ switch(sigLetter(s)) { case 'i': case 'f': case 'c': case 'C': case 'd': return Number; - case 'j': return affirmBigIntArray() && BigInt; + case 'j': return __BigInt; case 'p': case 'P': case 's': - switch(ptrSizeof){ + switch(ptrSize){ case 4: return Number; - case 8: return affirmBigIntArray() && BigInt; + case 8: return __BigInt; } break; } @@ -203,20 +226,29 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ }; /** - In order to completely hide StructBinder-bound struct - pointers from JS code, we store them in a scope-local - WeakMap which maps the struct-bound objects to their WASM - pointers. The pointers are accessible via - boundObject.pointer, which is gated behind an accessor - function, but are not exposed anywhere else in the - object. The main intention of that is to make it impossible - for stale copies to be made. + In order to completely hide StructBinder-bound struct pointers + from JS code, we store them in a scope-local WeakMap which maps + the struct-bound objects to their WASM pointers. The pointers are + accessible via boundObject.pointer, which is gated behind a + property interceptor, but are not exposed anywhere else in the + object. */ const __instancePointerMap = new WeakMap(); /** Property name for the pointer-is-external marker. */ const xPtrPropName = '(pointer-is-external)'; + const __isPtr32 = (ptr)=>('number'===typeof ptr && (ptr===(ptr|0)) && ptr>=0); + const __isPtr64 = (ptr)=>( + ('bigint'===typeof ptr && ptr >= 0) || __isPtr32(ptr) + ); + + /** + isPtr() is an alias for isPtr32() or isPtr64(), depending on + ptrSize. + */ + const __isPtr = (4===ptrSize) ? __isPtr32 : __isPtr64; + /** Frees the obj.pointer memory and clears the pointer property. */ const __freeStruct = function(ctor, obj, m){ @@ -229,7 +261,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ try{ if(x instanceof Function) x.call(obj); else if(x instanceof StructType) x.dispose(); - else if('number' === typeof x) dealloc(x); + else if(__isPtr(x)) dealloc(x); // else ignore. Strings are permitted to annotate entries // to assist in debugging. }catch(e){ @@ -275,7 +307,9 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ ctor.structName,"instance:", ctor.structInfo.sizeof,"bytes @"+m); } - if(fill) heap().fill(0, m, m + ctor.structInfo.sizeof); + if(fill){ + heap().fill(0, Number(m), Number(m) + ctor.structInfo.sizeof); + } __instancePointerMap.set(obj, m); }catch(e){ __freeStruct(ctor, obj, m); @@ -286,7 +320,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const __memoryDump = function(){ const p = this.pointer; return p - ? new Uint8Array(heap().slice(p, p+this.structInfo.sizeof)) + ? new Uint8Array(heap().slice(Number(p), Number(p) + this.structInfo.sizeof)) : null; }; @@ -350,12 +384,17 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const __SAB = ('undefined'===typeof SharedArrayBuffer) ? function(){} : SharedArrayBuffer; const __utf8Decode = function(arrayBuffer, begin, end){ + if( 8===ptrSize ){ + begin = Number(begin); + end = Number(end); + } return __utf8Decoder.decode( (arrayBuffer.buffer instanceof __SAB) ? arrayBuffer.slice(begin, end) : arrayBuffer.subarray(begin, end) ); }; + /** Uses __lookupMember() to find the given obj.structInfo key. Returns that member if it is a string, else returns false. If the @@ -430,8 +469,8 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const h = heap(); //let i = 0; //for( ; i < u.length; ++i ) h[mem + i] = u[i]; - h.set(u, mem); - h[mem + u.length] = 0; + h.set(u, Number(mem)); + h[__ptrAdd(mem, u.length)] = 0; //log("allocCString @",mem," =",u); return mem; }; @@ -500,7 +539,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ return __setMemberCString(this, memberName, str); }) }); - // Function-type non-Property inherited members + // Function-type non-Property inherited members Object.assign(StructType.prototype,{ addOnDispose: function(...v){ __addOnDispose(this,...v); @@ -518,8 +557,6 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ memberKey: __memberKeyProp }); - const isNumericValue = (v)=>Number.isFinite(v) || (v instanceof (BigInt || Number)); - /** Pass this a StructBinder-generated prototype, and the struct member description object. It will define property accessors for @@ -561,7 +598,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const dbg = ctor.prototype.debugFlags.__flags; /* TODO?: set prototype of descr to an object which can set/fetch - its prefered representation, e.g. conversion to string or mapped + its preferred representation, e.g. conversion to string or mapped function. Advantage: we can avoid doing that via if/else if/else in the get/set methods. */ @@ -574,7 +611,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof); } let rc = ( - new DataView(heap().buffer, this.pointer + descr.offset, descr.sizeof) + new DataView(heap().buffer, Number(this.pointer) + descr.offset, descr.sizeof) )[f._.getters[sigGlyph]](0, isLittleEndian); if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); return rc; @@ -590,24 +627,25 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ if(!this.pointer){ toss("Cannot set struct property on disposed instance."); } - if(null===v) v = 0; - else while(!isNumericValue(v)){ + if(null===v) v = __NullPtr; + else while(!__isPtr(v)){ if(isAutoPtrSig(descr.signature) && (v instanceof StructType)){ // It's a struct instance: let's store its pointer value! - v = v.pointer || 0; + v = v.pointer || __NullPtr; if(dbg.setter) log("debug.setter:",xPropName,"resolved to",v); break; } toss("Invalid value for pointer-type",xPropName+'.'); } ( - new DataView(heap().buffer, this.pointer + descr.offset, descr.sizeof) + new DataView(heap().buffer, Number(this.pointer) + descr.offset, + descr.sizeof) )[f._.setters[sigGlyph]](0, f._.sw[sigGlyph](v), isLittleEndian); }; } Object.defineProperty(ctor.prototype, key, prop); }/*makeMemberWrapper*/; - + /** The main factory function which will be returned to the caller. @@ -652,12 +690,15 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ } const debugFlags = rop(SBF.__makeDebugFlags(StructBinder.debugFlags)); /** Constructor for the StructCtor. */ + const zeroAsPtr = __asPtrType(0); const StructCtor = function StructCtor(externalMemory){ + externalMemory = __asPtrType(externalMemory); + //console.warn("externalMemory",externalMemory,arguments[0]); if(!(this instanceof StructCtor)){ toss("The",structName,"constructor may only be called via 'new'."); }else if(arguments.length){ - if(externalMemory!==(externalMemory|0) || externalMemory<=0){ - toss("Invalid pointer value for",structName,"constructor."); + if(Number.isNaN(externalMemory) || externalMemory<=zeroAsPtr){ + toss("Invalid pointer value",arguments[0],"for",structName,"constructor."); } __allocStruct(StructCtor, this, externalMemory); }else{ diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index dd80ed1c68..5ec3151d50 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -23,6 +23,14 @@ Firefox and Chrome on Linux and all claims of Safari compatibility are based solely on feature compatibility tables provided at [MDN][]. +**Non-browser compatibility**: this code does not target non-browser +JS engines and is completely untested on them. That said, it "might +work". + +**64-bit WASM:** as of 2025-09-21 this API supports 64-bit WASM builds +but it has to be configured for it (see [](#api-binderfactory) for +details). + **Formalities:** - Author: [Stephan Beal][sgb] @@ -43,9 +51,9 @@ project was spawned: > The author disclaims copyright to this source code. In place of a > legal notice, here is a blessing: > -> May you do good and not evil. -> May you find forgiveness for yourself and forgive others. -> May you share freely, never taking more than you give. +> - May you do good and not evil. +> - May you find forgiveness for yourself and forgive others. +> - May you share freely, never taking more than you give. ----- @@ -176,7 +184,7 @@ essentially boils down to: 3. [Feed (2) to the function generated by (1)](#step-3) to create JS constuctor functions for each struct. This is done at runtime, as opposed to during a build-process step, and can be set up in such a - way that it does not require any maintenace after its initial + way that it does not require any maintenance after its initial setup. 4. [Create and use instances of those structs](#step-4). @@ -199,21 +207,23 @@ const MyBinder = StructBinderFactory({ heap: WebAssembly.Memory instance or a function which returns a Uint8Array or Int8Array view of the WASM memory, alloc: function(howMuchMemory){...}, - dealloc: function(pointerToFree){...} + dealloc: function(pointerToFree){...}, + pointerIR: 'i32' or 'i64' // WASM pointer type - default = 'i32' }); ``` +See [](#api-binderfactory) for full details about the config options. + It also offers a number of other settings, but all are optional except for the ones shown above. Those three config options abstract away details which are specific to a given WASM environment. They provide -the WASM "heap" memory (a byte array), the memory allocator, and the -deallocator. In a conventional Emscripten setup, that config might -simply look like: +the WASM "heap" memory, the memory allocator, and the deallocator. In +a conventional Emscripten setup, that config might simply look like: > ```javascript { - heap: Module['asm']['memory'], + heap: Module?.asm?.memory || Module['wasmMemory'], //Or: // heap: ()=>Module['HEAP8'], alloc: (n)=>Module['_malloc'](n), @@ -316,12 +326,13 @@ Noting that: - **All of these types are numeric**. Attempting to set any struct-bound property to a non-numeric value will trigger an exception except in cases explicitly noted otherwise. -- **"Char" types**: WASM does not define an `int8` type, nor does it - distinguish between signed and unsigned. This API treats `c` as - `int8` and `C` as `uint8` for purposes of getting and setting values - when using the `DataView` class. It is _not_ recommended that client - code use these types in new WASM-capable code, but they were added - for the sake of binding some immutable legacy code to WASM. +- **"Char" types**: WASM does not define an `int8` type, nor does its + JS representation distinguish between signed and unsigned. This API + treats `c` as `int8` and `C` as `uint8` for purposes of getting and + setting values when using the `DataView` class. It is _not_ + recommended that client code use these types in new WASM-capable + code, but they were added for the sake of binding some immutable + legacy code to WASM. > Sidebar: Emscripten's public docs do not mention `p`, but their generated code includes `p` as an alias for `i`, presumably to mean @@ -329,11 +340,9 @@ generated code includes `p` as an alias for `i`, presumably to mean is more descriptive, so this framework encourages the use of `p` for pointer-type members. Using `p` for pointers also helps future-proof the signatures against the eventuality that WASM eventually supports -64-bit pointers. Note that sometimes `p` really means -pointer-to-pointer, but the Emscripten JS/WASM glue does not offer -that level of expressiveness in these signatures. We simply have to be -aware of when we need to deal with pointers and pointers-to-pointers -in JS code. +64-bit pointers. Note that sometimes `p` _really_ means a +pointer-to-pointer. We simply have to be aware of when we need to deal +with pointers and pointers-to-pointers in JS code. > Trivia: this API treates `p` as distinctly different from `i` in some contexts, so its use is encouraged for pointer types. @@ -357,17 +366,17 @@ signature.replace(/[^vipPsjfdcC]/g,'').replace(/[pPscC]/g,'i'); *This support is experimental and subject to change.* The method signature letter `p` means "pointer," which, in WASM, means -"integer." `p` is treated as an integer for most contexts, while still -also being a separate type (analog to how pointers in C are just a -special use of unsigned numbers). A capital `P` changes the semantics -of plain member pointers (but not, as of this writing, function -pointer members) as follows: +either int32 or int64. `p` is treated as a point for most contexts, +while still also being a separate type for function signature purposes +(analog to how pointers in C are just a special use of unsigned +numbers). A capital `P` changes the semantics of plain member pointers +(but not, as of this writing, function pointer members) as follows: -- When a `P`-type member is **set** via `myStruct.x=y`, if - [`(y instanceof StructType)`][StructType] then the value of `y.pointer` is - stored in `myStruct.x`. If `y` is neither a number nor - a [StructType][], an exception is triggered (regardless of whether - `p` or `P` is used). +When a `P`-type member is **set** via `myStruct.x=y`, if +[`(y instanceof StructType)`][StructType] then the value of `y.pointer` is +stored in `myStruct.x`. If `y` is neither a pointer nor a +[StructType][], an exception is triggered (regardless of whether `p` +or `P` is used). @@ -474,15 +483,19 @@ It returns a function which these docs refer to as a [StructBinder][] The binder factory supports the following options in its configuration object argument: +- `pointerIR` (Added 2025-09-21) + Optionally specify the WASM pointer size with the string `'i32'` or + `'i64'`, defaulting to the former. When using with 64-bit WASM + builds, this must be set to `'i64'` by the client. Any other value + triggers an exception. - `heap` Must be either a `WebAssembly.Memory` instance representing the WASM heap memory OR a function which returns an Int8Array or Uint8Array view of the WASM heap. In the latter case the function should, if appropriate for the environment, account for the heap being able to - grow. Jaccwabyt uses this property in such a way that it "should" be - okay for the WASM heap to grow at runtime (that case is, however, - untested). + grow. Jaccwabyt uses this property in such a way that it is legal + for the WASM heap to grow at runtime. - `alloc` Must be a function semantically compatible with Emscripten's @@ -524,12 +537,11 @@ configuration object argument: - `log` Optional function used for debugging output. By default - `console.log` is used but by default no debug output is generated. + `console.debug` is used but by default no debug output is generated. This API assumes that the function will space-separate each argument - (like `console.log` does). See [Appendix D](#appendix-d) for info + (like `console.debug` does). See [Appendix D](#appendix-d) for info about enabling debugging output. - API: Struct Binder ------------------------------------------------------------ @@ -689,9 +701,12 @@ legally be called on concrete struct instances unless noted otherwise: - `memoryDump()` Returns a Uint8Array which contains the current state of this object's raw memory buffer. Potentially useful for debugging, but - not much else. Note that the memory is necessarily, for - compatibility with C, written in the host platform's endianness and - is thus not useful as a persistent/portable serialization format. + not much else. The memory is necessarily, for compatibility with C, + written in the host platform's endianness. Since all WASM is + little-endian, it's the same everywhere. Even so: it should not be + used as a persistent serialization format because (A) any changes to + the struct will invalidate older serialized data and (B) serializing + member pointers is useless. - `setMemberCString(memberName,str)` Uses `StructType.allocCString()` to allocate a new C-style string, @@ -951,7 +966,7 @@ const char * wasm__ctype_json(void){ assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck //////////////////////////////////////////////////////////////////// - // Macros for emiting StructBinders... + // Macros for emitting StructBinders... #define StructBinder__(TYPE) \ n = 0; \ outf("%s{", (structCount++ ? ", " : "")); \ diff --git a/ext/wasm/mkdist.sh b/ext/wasm/mkdist.sh new file mode 100755 index 0000000000..84780668b8 --- /dev/null +++ b/ext/wasm/mkdist.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +### +# +# Builds the SQLite JS/WASM distribution zip file. +# +# Usage: $0 build-name ?flags? +# +# build-name is the dir/archive name prefix for the +# build and defaults to sqlite-wasm. +# +# -?|--help = Show this text +# +# -0 = Use -O0 instead of ${optFlag} +# +# -1 = Use make -j1 instead of ${makeFlag} +# +# -64 = Include 64-bit builds +# +# --noclean = do not run 'make clean' first +# +# --snapshot = gives the archive name a distinctive suffix +# +### + +function die(){ + local rc=$1 + shift + echo "Error: $@" 1>&2 + exit $rc +} + +dirTop=../.. +buildName= +b64=0 +optFlag=-Oz +clean=1 +makeFlag=-j4 +snapshotSuffix= +for arg in $@; do + case $arg in + + -64) b64=1 + ;; + + -0) optFlag=-O0 + ;; + + -1) makeFlag= + ;; + + --noclean) clean=0 + ;; + + --snapshot) + snapshotSuffix=$(date +%Y%m%d) + ;; + + -?|--help) + sed -n -e '2,/^###/p' $0 + exit + ;; + + *) if [[ x != x${buildName} ]]; then + die 1 "Unhandled argument: $arg" + fi + buildName=$arg + ;; + esac +done + +make= +for i in gmake make; do + make=$(which $i 2>/dev/null) + [[ x != x${make} ]] && break +done +[[ x = x$make ]] && die 127 "Cannot find make" + + +[[ x = x${buildName} ]] && buildName=sqlite-wasm + +buildName=${buildName}${snapshotSuffix} + +echo "Creating the SQLite wasm dist bundle..." + +# +# Generates files which, when built, will also build all of the pieces +# neaded for the dist bundle. +# +tgtFiles=( + demo-worker1-promiser.html + demo-worker1-promiser.js + demo-worker1-promiser-esm.html + demo-worker1-promiser.mjs + + tester1.html + tester1-esm.html + tester1-worker.html + tester1.js + tester1.mjs +) + +if [[ 1 = $b64 ]]; then + tgtFiles+=( + tester1-64bit.html + tester1-esm-64bit.html + tester1-worker-64bit.html + tester1-64bit.js + tester1-64bit.mjs + ) +fi + +[[ 1 = $clean ]] && $make clean +$make $makeFlag \ + t-version-info t-stripccomments \ + ${tgtFiles[@]} \ + "emcc_opt=${optFlag}" || die $? + +dirTmp=d.dist +rm -fr $dirTmp +mkdir -p $dirTmp/jswasm || die $? +mkdir -p $dirTmp/common || die $? + +# Static files for the top-most dir: +fTop=( + demo-123.html + demo-123-worker.html + demo-123.js + + demo-worker1.html + demo-worker1.js + + demo-jsstorage.html + demo-jsstorage.js + + module-symbols.html +) + +# Files for the jswasm subdir sans jswasm prefix: +# +# fJ1 = JS files to stripccomments -k on +# fJ2 = JS files to stripccomments -k -k on +fJ1=( + sqlite3-opfs-async-proxy.js + sqlite3-worker1.js + sqlite3-worker1.mjs + sqlite3-worker1-bundler-friendly.mjs + sqlite3-worker1-promiser.js + sqlite3-worker1-promiser.mjs + sqlite3-worker1-promiser-bundler-friendly.mjs +) +fJ2=( + sqlite3.js + sqlite3.mjs +) + +# fW = list of wasm files to copy from jswasm/. +fW=(sqlite3.wasm) +if [[ 1 = $b64 ]]; then + fW+=(sqlite3-64bit.wasm) +fi + +function fcp() { + cp -p $@ || die $? + chmod +w ${@: -1} +} + +function scc(){ + ${dirTop}/tool/stripccomments $@ || die $? +} + +jw=jswasm +fcp ${tgtFiles[@]} $dirTmp/. +fcp README-dist.txt $dirTmp/README.txt +fcp index-dist.html $dirTmp/index.html +fcp common/*.css common/SqliteTestUtil.js $dirTmp/common/. + +for i in ${fTop[@]}; do + fcp $i $dirTmp/. +done + +for i in ${fW[@]}; do + fcp $jw/$i $dirTmp/$jw/. +done + +for i in ${fJ1[@]}; do + scc -k < $jw/$i > $dirTmp/$jw/$i || die $? +done + +for i in ${fJ2[@]}; do + scc -k -k < $jw/$i > $dirTmp/$jw/$i || die $? +done + +# +# Done copying files. Now zip it up... +# +svi=./version-info +vnum=$($svi --download-version) +[ "" = "$vnum" ] && die "version number is empty!" +vdir=${buildName}-${vnum} +fzip=${vdir}.zip +rm -fr ${vdir} ${fzip} +mv $dirTmp $vdir || die $? +zip -rq9 $fzip $(find $vdir -type f | sort) || die $? +ls -la $fzip +unzip -lv $fzip || die $? +cat < +#include +#include + +#define pf printf +#define ps puts + +/* Separator to help eyeballs find the different output sections */ +#define zBanner \ + "\n########################################################################\n" + +/* +** Flags for use with BuildDef::flags. +** +** Maintenance reminder: do not combine flags within this enum, +** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead +** to breakage in some of the flag checks. +*/ +enum BuildDefFlags { + /* Indicates an ESM module build. */ + F_ESM = 0x01, + /* Indicates a "bundler-friendly" build mode. */ + F_BUNDLER_FRIENDLY = 1<<1, + /* Indicates that this build is unsupported. Such builds are not + ** added to the 'all' target. The unsupported builds exist primarily + ** for experimentation's sake. */ + F_UNSUPPORTED = 1<<2, + /* Elide this build from the 'all' target. */ + F_NOT_IN_ALL = 1<<3, + /* If it's a 64-bit build. */ + F_64BIT = 1<<4, + /* Indicates a node.js-for-node.js build (untested and + ** unsupported). */ + F_NODEJS = 1<<5, + /* Indicates a wasmfs build (untested and unsupported). */ + F_WASMFS = 1<<6, + + /* + ** Which compiled files from $(dir.dout)/buildName/*.{js,mjs,wasm} + ** to copy to $(dir.dout) after creating them. This should only be + ** applied to builds which result in end-user deliverables. Some + ** builds, like the bundler-friendly ones, are a hybrid: we keep + ** only their JS file and patch their JS to use the WASM file from a + ** canonical build which uses that same WASM file. Reusing X.wasm + ** that way can only work for builds which are processed identically + ** by Emscripten. For a given set of C flags (as opposed to + ** JS-influencing flags), all builds of X.js and Y.js will produce + ** identical X.wasm and Y.wasm files. Their JS files may well + ** differ, however. + */ + CP_JS = 1 << 30, + CP_WASM = 1 << 31, + CP_ALL = CP_JS | CP_WASM +}; + +/* +** Info needed for building one concrete JS/WASM combination.. +** +** Notes about Emscripten builds... +** +** When emcc processes X.js it also generates X.wasm and hard-codes +** the name "X.wasm" into the JS file (it has to - there's no reliable +** way to derive that name at runtime for certain modes of loading the +** WASM file). Because we only need two sqlite3.wasm files (one each +** for 32- and 64-bit), the build then copies just those into the +** final build directory $(dir.dout). +** +** To keep parallel builds from stepping on each other, each distinct +** build goes into its own subdir $(dir.dout.BuildName)[^1], i.e. +** $(dir.dout)/BuildName. Builds which produce deliverables we'd like +** to keep/distribute copy their final results into the build dir +** $(dir.dout). See the notes for the CP_JS enum entry for more +** details on that. +** +** The final result of each build is a pair of JS/WASM files, but +** getting there requires generation of several files, primarily as +** inputs for specific Emscripten flags: +** +** --pre-js = file gets injected after Emscripten's earliest starting +** point, enabling limited customization of Emscripten's +** behavior. This code lives/runs within the generated sqlite3InitModule(). +** +** --post-js = gets injected after Emscripten's main work, but still +** within the body of sqlite3InitModule(). +** +** --extern-pre-js = gets injected before sqlite3InitModule(), in the +** global scope. We inject the license and version info here. +** +** --extern-post-js = gets injected immediately after +** sqlite3InitModule(), in the global scope. In this step we replace +** sqlite3InitModule() with a slightly customized, the main purpose of +** which is to (A) give us (not Emscripten) control over the arguments +** it accepts and (B) to run the library bootstrap step. +** +** Then there's sqlite3-api.BuildName.js, which is the entire SQLite3 +** JS API (generated from the list defined in $(sqlite3-api.jses)). It +** gets sandwitched inside --post-js. +** +** Each of those inputs has to be generated before passing them on to +** Emscripten so that any build-specific capabilities can get filtered +** in or out (using ./c-pp.c). +** +** [^1]: The legal BuildNames are in this file's BuildDef_map macro. +*/ +struct BuildDef { + /* + ** Base name of output JS and WASM files. The X part of X.js and + ** X.wasm. + */ + const char *zBaseName; + /* + ** A glyph to use in log messages for this build, intended to help + ** the eyes distinguish the build lines more easily in parallel + ** builds. + ** + ** The convention for 32- vs 64-bit pairs is to give them similar + ** emoji, e.g. a cookie for 32-bit and a donut or cake for 64. + ** Alternately, the same emoji a "64" suffix, excep that that throws + ** off the output alignment in parallel builds ;). + */ + const char *zEmo; + /* + ** If the build needs its x.wasm renamed in its x.{js,mjs} then this + ** must hold the base name to rename it to. Typically "sqlite3" or + ** "sqlite3-64bit". This is the case for builds which are named + ** something like sqlite3-foo-bar but can use the vanilla + ** sqlite3.wasm file. In such cases we don't need the extra + ** sqlite3-foo-bar.wasm which Emscripten (necessarily) creates when + ** compiling the module, so we patch (at build-time) the JS file to + ** use this name instead sqlite3-foo-bar. + */ + const char *zDotWasm; + const char *zCmppD; /* Extra -D... flags for c-pp */ + const char *zEmcc; /* Full flags for emcc. Normally NULL for default. */ + const char *zEmccExtra; /* Extra flags for emcc */ + const char *zDeps; /* Extra deps */ + const char *zEnv; /* emcc -sENVIRONMENT=... value */ + /* + ** Makefile code "ifeq (...)". If set, this build is enclosed in a + ** $zIfCond/endif block. + */ + const char *zIfCond; /* makefile "ifeq (...)" or similar */ + int flags; /* Flags from BuildDefFlags */ +}; +typedef struct BuildDef BuildDef; + +/* +** List of distinct library builds. Each one has to be set up in +** oBuildDefs. See the next comment block. +** +** Many makefile vars use these identifiers for naming stuff, e.g.: +** +** out.NAME.js = output JS file for the build named NAME +** out.NAME.wasm = output WASM file for the build named NAME +** logtag.NAME = Used for decorating log output +** +** etc. +***/ +#define BuildDefs_map(E) \ + E(vanilla) E(vanilla64) \ + E(esm) E(esm64) \ + E(bundler) E(bundler64) \ + E(speedtest1) E(speedtest164) \ + E(node) E(node64) \ + E(wasmfs) + +/* +** The set of WASM builds for the library (as opposed to the apps +** (fiddle, speedtest1)). Their order in BuildDefs_map is mostly +** insignificant, but some makefile vars used by some builds are set +** up by prior builds. Because of that, the (sqlite3, vanilla), +** (sqlite3, esm), and (sqlite3, bundler-friendly) builds should be +** defined first (in that order). +*/ +struct BuildDefs { +#define E(N) BuildDef N; + BuildDefs_map(E) +#undef E +}; +typedef struct BuildDefs BuildDefs; + +const BuildDefs oBuildDefs = { + /* + ** The canonical build, against which all others are compared and + ** contrasted. This is the one we post downloads for. + ** + ** This one's zBaseName and zEnv MUST be non-NULL so it can be used + ** as a default for all others + */ + .vanilla = { + .zEmo = "🍦", + .zBaseName = "sqlite3", + .zDotWasm = 0, + .zCmppD = 0, + .zEmcc = 0, + .zEmccExtra = 0, + .zEnv = "web,worker", + .zDeps = 0, + .zIfCond = 0, + .flags = CP_ALL + }, + + /* The canonical build in 64-bit. */ + .vanilla64 = { + .zEmo = "🍨", + .zBaseName = "sqlite3-64bit", + .zDotWasm = 0, + .zCmppD = 0, + .zEmcc = 0, + .zEmccExtra = "-sMEMORY64=1 -sWASM_BIGINT=1", + .zEnv = 0, + .zDeps = 0, + .zIfCond = 0, + .flags = CP_ALL | F_64BIT + }, + + /* The canonical esm build. */ + .esm = { + .zEmo = "🍬", + .zBaseName = "sqlite3", + .zDotWasm = 0, + .zCmppD = "-Dtarget:es6-module", + .zEmcc = 0, + .zEmccExtra = 0, + .zEnv = 0, + .zDeps = 0, + .zIfCond = 0, + .flags = CP_JS | F_ESM + }, + + /* The canonical esm build in 64-bit. */ + .esm64 = { + .zEmo = "🍫", + .zBaseName = "sqlite3-64bit", + .zDotWasm = 0, + .zCmppD = "-Dtarget:es6-module", + .zEmcc = 0, + .zEmccExtra = "-sMEMORY64=1 -sWASM_BIGINT=1", + .zEnv = 0, + .zDeps = 0, + .zIfCond = 0, + .flags = CP_JS | F_ESM | F_64BIT + }, + + /* speedtest1, our primary benchmarking tool */ + .speedtest1 = { + .zEmo = "🛼", + .zBaseName = "speedtest1", + .zDotWasm = 0, + .zCmppD = 0, + .zEmcc = + "$(emcc.speedtest1)" + " $(emcc.speedtest1.common)" + " $(pre-post.speedtest1.flags)" + " $(cflags.common)" + " -DSQLITE_SPEEDTEST1_WASM" + " $(SQLITE_OPT)" + " -USQLITE_WASM_BARE_BONES" + " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" + " $(speedtest1.exit-runtime0)" + " $(speedtest1.c.in)" + " -lm", + .zEmccExtra = 0, + .zEnv = 0, + .zDeps = + "$(speedtest1.c.in)" + " $(EXPORTED_FUNCTIONS.speedtest1)", + .zIfCond = 0, + .flags = CP_ALL + }, + + /* speedtest1 64-bit */ + .speedtest164 = { + .zEmo = "🛼64", + .zBaseName = "speedtest1-64bit", + .zDotWasm = 0, + .zCmppD = 0, + .zEmcc = + "$(emcc.speedtest1)" + " $(emcc.speedtest1.common)" + " -sMEMORY64=1 -sWASM_BIGINT=1" + " $(pre-post.speedtest164.flags)" + " $(cflags.common)" + " -DSQLITE_SPEEDTEST1_WASM" + " $(SQLITE_OPT)" + " -USQLITE_WASM_BARE_BONES" + " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" + " $(speedtest1.exit-runtime0)" + " $(speedtest1.c.in)" + " -lm", + .zEmccExtra = 0, + .zEmccExtra = 0, + .zEnv = 0, + .zDeps = + "$(speedtest1.c.in)" + " $(EXPORTED_FUNCTIONS.speedtest1)", + .zIfCond = 0, + .flags = CP_ALL | F_NOT_IN_ALL + }, + + /* + ** Core bundler-friendly build. Untested and "not really" supported, + ** but required by the downstream npm subproject. + ** + ** Testing these requires special-purpose node-based tools and + ** custom test apps, none of which we have/use. So we can pass them + ** off as-is to the npm subproject and they spot failures pretty + ** quickly ;). + */ + .bundler = { + .zEmo = "👛", + .zBaseName = "sqlite3-bundler-friendly", + .zDotWasm = "sqlite3", + .zCmppD = "$(c-pp.D.esm) -Dtarget:es6-bundler-friendly", + .zEmcc = 0, + .zEmccExtra = 0, + .zEnv = 0, + .zDeps = 0, + .zIfCond = 0, + .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM + //| F_NOT_IN_ALL + }, + + /* 64-bit bundler-friendly. */ + .bundler64 = { + .zEmo = "📦", + .zBaseName = "sqlite3-bundler-friendly-64bit", + .zDotWasm = "sqlite3-64bit", + .zCmppD = "$(c-pp.D.bundler)", + .zEmcc = 0, + .zEmccExtra = "-sMEMORY64=1", + .zEnv = 0, + .zDeps = 0, + .zIfCond = 0, + .flags = CP_JS | F_ESM | F_BUNDLER_FRIENDLY | F_64BIT | F_NOT_IN_ALL + }, + + /* + ** We neither build node builds on a regular basis nor test them at + ** all. They are fully unsupported. Also, our JS targets only + ** browsers. + */ + .node = { + .zEmo = "🍟", + .zBaseName = "sqlite3-node", + .zDotWasm = 0, + .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", + .zEmcc = 0, + .zEmccExtra = 0, + .zEnv = "node" + /* Adding ",node" to the zEnv list for the other builds causes + ** Emscripten to generate code which confuses node: it cannot + ** reliably determine whether the build is for a browser or for + ** node. */, + .zDeps = 0, + .zIfCond = 0, + .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS + }, + + /* 64-bit node. */ + .node64 = { + .zEmo = "🍔", + .zBaseName = "sqlite3-node-64bit", + .zDotWasm = 0, + .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", + .zEmcc = 0, + .zEmccExtra = 0, + .zEnv = "node", + .zDeps = 0, + .zIfCond = 0, + .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS | F_64BIT + }, + + /* Entirely unsupported. */ + .wasmfs = { + .zEmo = "💿", + .zBaseName = "sqlite3-wasmfs", + .zDotWasm = 0, + .zCmppD = "$(c-pp.D.bundler) -Dwasmfs", + .zEmcc = 0, + .zEmccExtra = + "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META" + " -sUSE_CLOSURE_COMPILER=0" + " -pthread -sWASMFS -sPTHREAD_POOL_SIZE=1" + " -sERROR_ON_UNDEFINED_SYMBOLS=0 -sLLD_REPORT_UNDEFINED" + " -DSQLITE_ENABLE_WASMFS" + , + .zEnv = 0, + .zDeps = 0, + .zIfCond = "ifeq (1,$(wasmfs.enable))", + .flags = CP_ALL | F_UNSUPPORTED | F_WASMFS | F_ESM + } +}; + +/* +** Emits common vars needed by the rest of the emitted code (but not +** needed by makefile code outside of these generated pieces). +*/ +static void mk_prologue(void){ + /* A 0-terminated list of makefile vars which we expect to have been + ** set up by this point in the build process. */ + char const * aRequiredVars[] = { + "dir.top", + "dir.api", "dir.dout", "dir.tmp", + "dir.fiddle", "dir.fiddle.debug", + /*"just-testing",*/ + 0 + }; + char const * zVar; + int i; + ps(zBanner "# Build setup sanity checks..."); + for( i = 0; (zVar = aRequiredVars[i]); ++i ){ + pf("ifeq (,$(%s))\n", zVar); + pf(" $(error build process error: expecting make var $$(%s) to " + "have been set up by now)\n", zVar); + ps("endif"); + } + + ps("define label.unsupported-build\n" + "$(emo.fire)$(emo.fire)$(emo.fire)Unsupported build:" + " use at your own risk!\n" + "endef"); + + ps(zBanner + /** $1 = build name */ + "b.call.wasm-strip = " + "echo '[$(emo.b.$(1)) $(out.$(1).wasm)] $(emo.strip) wasm-strip'; " + "$(bin.wasm-strip) $(out.$(1).wasm)\n" + ); + + pf(zBanner + "define b.do.emcc\n" + /* $1 = build name */ + "$(bin.emcc) -o $@ $(emcc_opt_full) $(emcc.flags) " + "$(emcc.jsflags) -sENVIRONMENT=$(emcc.environment.$(1)) " + " $(pre-post.$(1).flags) " + " $(emcc.flags.$(1)) " + " $(cflags.common) $(cflags.$(1)) " + " $(SQLITE_OPT) " + " $(cflags.wasm_extra_init) $(sqlite3-wasm.c.in)\n" + "endef\n" + ); + + { + /* b.do.wasm-opt + ** + ** $1 = build name + ** + ** Runs $(out.$(1).wasm) through $(bin.wasm-opt) + */ + const char * zOptFlags = + /* + ** Flags for wasm-opt. It has many, many, MANY "passes" options + ** and the ones which appear here were selected solely on the + ** basis of trial and error. + ** + ** All wasm file size savings/costs mentioned below are based on + ** the vanilla build of sqlite3.wasm with -Oz (our shipping + ** configuration). Comments like "saves nothing" may not be + ** technically correct: "nothing" means "some neglible amount." + ** + ** Note that performance gains/losses are _not_ taken into + ** account here: only wasm file size. + */ + "--enable-bulk-memory-opt " /* required */ + "--all-features " /* required */ + "--post-emscripten " /* Saves roughly 12kb */ + "--strip-debug " /* We already wasm-strip, but in + ** case this environment has no + ** wasm-strip... */ + /* + ** The rest are trial-and-error. See wasm-opt --help and search + ** for "Optimization passes" to find the full list. + ** + ** With many flags this gets unusuably slow. + */ + /*"--converge " saves nothing for the options we're using */ + /*"--dce " saves nothing */ + /*"--directize " saves nothing */ + /*"--gsi " no: requires --closed-world flag, which does not + ** sound like something we want. */ + /*"--gufa --gufa-cast-all --gufa-optimizing " costs roughly 2kb */ + /*"--heap-store-optimization " saves nothing */ + /*"--heap2local " saves nothing */ + //"--inlining --inlining-optimizing " costs roughly 3kb */ + "--local-cse " /* saves roughly 1kb */ + /*"--once-reduction " saves nothing */ + /*"--remove-memory-init " presumably a performance tweak */ + /*"--remove-unused-names " saves nothing */ + /*"--safe-heap "*/ + /*"--vacuum " saves nothing */ + ; + ps(zBanner + "# post-compilation WASM file optimization"); + + /* b.do.wasm-opt $1 = build name*/ + ps("ifeq (,$(bin.wasm-opt))"); { + ps("b.do.wasm-opt = echo '$(logtag.$(1)) wasm-opt not available'"); + } + ps("else"); { + ps("define b.do.wasm-opt"); + pf( + "echo '[$(emo.b.$(1)) $(out.$(1).wasm)] $(emo.wasm-opt) $(bin.wasm-opt)';\\\n" + "\ttmpfile=$(dir.dout.$(1))/wasm-opt-tmp.$(1).wasm; \\\n" + "\trm -f $$tmpfile; \\\n" + "\tif $(bin.wasm-opt) $(out.$(1).wasm) " + "-o $$tmpfile \\\n" + "\t\t%s; then \\\n" + "\t\tmv $$tmpfile $(out.$(1).wasm); \\\n" +#if 0 + "\t\techo -n 'After wasm-opt: '; \\\n" + "\t\tls -l $(1); \\\n" +#endif + /* It's very likely that the set of wasm-opt flags varies from + ** version to version, so we'll ignore any errors here. */ + "\telse \\\n" + "\t\trm -f $$tmpfile; \\\n" + "\t\techo '$(logtag.$(1)) $(emo.fire) ignoring wasm-opt failure'; \\\n" + "\tfi\n", + zOptFlags + ); + ps("endef"); + } + ps("endif"); + } + + ps("more: all"); +} + +/* +** WASM_CUSTOM_INSTANTIATE changes how the JS pieces load the .wasm +** file from the .js file. When set, our JS takes over that step from +** Emscripten. Both modes are functionally equivalent but +** customization gives us access to wasm module state which we don't +** otherwise have. That said, the library also does not _need_ that +** state, so we don't _need_ to customize that step. +*/ +#if !defined(WASM_CUSTOM_INSTANTIATE) +# define WASM_CUSTOM_INSTANTIATE 0 +#elif (WASM_CUSTOM_INSTANTIATE+0)==0 +# undef WASM_CUSTOM_INSTANTIATE +# define WASM_CUSTOM_INSTANTIATE 0 +#endif + +#if WASM_CUSTOM_INSTANTIATE +/* c-pp -D... flags for the custom instantiateWasm(). */ +#define C_PP_D_CUSTOM_INSTANTIATE " -DModule.instantiateWasm " +#else +#define C_PP_D_CUSTOM_INSTANTIATE +#endif + +static char const * BuildDef_jsext(const BuildDef * pB){ + return (F_ESM & pB->flags) ? ".mjs" : ".js"; +} + +static char const * BuildDef_basename(const BuildDef * pB){ + return pB->zBaseName ? pB->zBaseName : oBuildDefs.vanilla.zBaseName; +} + +/* +** Emits makefile code for setting up values for the --pre-js=FILE, +** --post-js=FILE, and --extern-post-js=FILE emcc flags, as well as +** populating those files. This is necessary for any builds which +** embed the library's JS parts of this build (as opposed to parts +** which do not use the library-level code). +** +** pB may be NULL. +*/ +static void mk_pre_post(char const *zBuildName, BuildDef const * pB){ + char const * const zBaseName = pB + ? BuildDef_basename(pB) : 0; + + assert( zBuildName ); + pf("%s# Begin --pre/--post flags for %s\n", zBanner, zBuildName); + + ps("# --pre-js=..."); + pf("pre-js.%s.js = $(dir.tmp)/pre-js.%s.js\n", + zBuildName, zBuildName); + + if( 0==WASM_CUSTOM_INSTANTIATE || !pB ){ + pf("$(eval $(call b.c-pp.target," + "%s," + "$(pre-js.in.js)," + "$(pre-js.%s.js)," + "$(c-pp.D.%s)" + "))", + zBuildName, zBuildName, zBuildName); + }else{ + char const *zWasmFile = pB->zDotWasm + ? pB->zDotWasm + : pB->zBaseName; + /* + ** See BuildDef::zDotWasm for _why_ we do this. _What_ we're doing + ** is generate $(pre-js.BUILDNAME.js) as above, but: + ** + ** 1) Add an extra -D... flag to activate the custom + ** Module.intantiateWasm() in the JS code. + ** + ** 2) Amend the generated pre-js.js with the name of the WASM + ** file which should be loaded. That tells the custom + ** Module.instantiateWasm() to use that file instead of + ** the default. + */ + pf("$(pre-js.%s.js): $(pre-js.in.js) $(bin.c-pp) $(MAKEFILE_LIST)", + zBuildName); + if( pB->zDotWasm ){ + pf(" $(dir.dout)/%s.wasm" /* This .wasm is from some other + build, so this may trigger a full + build of the reference copy. */, + pB->zDotWasm); + } + ps(""); + pf("\t@$(call b.c-pp.shcmd," + "%s," + "$(pre-js.in.js)," + "$(pre-js.%s.js)," + "$(c-pp.D.%s)" C_PP_D_CUSTOM_INSTANTIATE + ")\n", + zBuildName, zBuildName, zBuildName); + } + + ps("\n# --post-js=..."); + pf("post-js.%s.js = $(dir.tmp)/post-js.%s.js\n", + zBuildName, zBuildName); + pf("post-js.%s.in =" + " $(dir.api)/post-js-header.js" + " $(sqlite3-api.%s.js)" + " $(dir.api)/post-js-footer.js\n", + zBuildName, zBuildName); + + pf("$(eval $(call b.c-pp.target," + "%s," + "$(post-js.%s.in)," + "$(post-js.%s.js)," + "$(c-pp.D.%s)" + "))\n", + zBuildName, zBuildName, zBuildName, zBuildName); + + pf("$(post-js.%s.js): $(post-js.%s.in) $(bin.c-pp)\n", + zBuildName, zBuildName); + + ps("\n# --extern-post-js=..."); + pf("extern-post-js.%s.js = $(dir.tmp)/extern-post-js.%s.js\n", + zBuildName, zBuildName); + if( 0!=WASM_CUSTOM_INSTANTIATE && zBaseName ){ + pf("$(eval $(call b.c-pp.target," + "%s," + "$(extern-post-js.in.js)," + "$(extern-post-js.%s.js)," + "$(c-pp.D.%s) --@policy=error -Dsqlite3.wasm=%s.wasm" + "))", + zBuildName, zBuildName, zBuildName, + zBaseName); + }else{ + pf("$(eval $(call b.c-pp.target," + "%s," + "$(extern-post-js.in.js)," + "$(extern-post-js.%s.js)," + "$(c-pp.D.%s)" + "))", + zBuildName, zBuildName, zBuildName); + } + + ps("\n# --pre/post misc..."); + /* Combined flags for use with emcc... */ + pf("pre-post.%s.flags = " + "--extern-pre-js=$(sqlite3-license-version.js) " + "--pre-js=$(pre-js.%s.js) " + "--post-js=$(post-js.%s.js) " + "--extern-post-js=$(extern-post-js.%s.js)\n", + zBuildName, zBuildName, zBuildName, zBuildName); + + + /* Set up deps... */ + pf("pre-post.%s.deps = " + "$(pre-post-jses.common.deps) " + "$(post-js.%s.js) $(extern-post-js.%s.js) " + "$(dir.tmp)/pre-js.%s.js\n", + zBuildName, zBuildName, zBuildName, zBuildName); + pf("# End --pre/--post flags for %s%s", zBuildName, zBanner); +} + +static void emit_compile_start(char const *zBuildName){ + pf("\t@$(call b.mkdir@);" + " $(call b.echo,%s,$(emo.compile) building ...)\n", + zBuildName); +} + +static void emit_logtag(char const *zBuildName){ +#if 1 + pf("logtag.%s ?= [$(emo.b.%s)$(if $@, $@,)]:\n", + zBuildName, zBuildName); +#else + pf("logtag.%s ?= [$(emo.b.%s) [%s] $@]:\n", + zBuildName, zBuildName, zBuildName); +#endif + pf("$(info $(logtag.%s) Setting up target b-%s)\n", + zBuildName, zBuildName ); +} + +/** + Emit rules for sqlite3-api.${zBuildName}.js. +*/ +static void emit_api_js(char const *zBuildName){ + pf("sqlite3-api.%s.js = $(dir.tmp)/sqlite3-api.%s.js\n", + zBuildName, zBuildName); + pf("$(eval $(call b.c-pp.target," + "%s," + "$(sqlite3-api.jses)," + "$(sqlite3-api.%s.js)," + "$(c-pp.D.%s)" + "))\n", + zBuildName, zBuildName, zBuildName); + pf("$(out.%s.js): $(sqlite3-api.%s.js)\n", + zBuildName, zBuildName); +} + +/* +** Emits makefile code for one build of the library. +*/ +static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ + const char * zJsExt = BuildDef_jsext(pB); + char const * const zBaseName = BuildDef_basename(pB); + + assert( oBuildDefs.vanilla.zEnv ); + assert( zBaseName ); + + pf("%s# Begin build [%s%s]. flags=0x%02x\n", zBanner, + pB->zEmo, zBuildName, pB->flags); + pf("# zCmppD=%s\n# zBaseName=%s\n", + pB->zCmppD ? pB->zCmppD : "", zBaseName); + pf("b.names += %s\n" + "emo.b.%s = %s\n", + zBuildName, zBuildName, pB->zEmo); + emit_logtag(zBuildName); + + if( pB->zIfCond ){ + pf("%s\n", pB->zIfCond ); + } + + pf("dir.dout.%s ?= $(dir.dout)/%s\n", zBuildName, zBuildName); + + pf("out.%s.base ?= $(dir.dout.%s)/%s\n", + zBuildName, zBuildName, zBaseName); + pf("out.%s.js ?= $(dir.dout.%s)/%s%s\n", + zBuildName, zBuildName, zBaseName, zJsExt); + pf("out.%s.wasm ?= $(dir.dout.%s)/%s.wasm\n", + //"$(basename $@).wasm" + zBuildName, zBuildName, zBaseName); + + pf("dir.dout.%s ?= $(dir.dout)/%s\n", zBuildName, zBuildName); + pf("out.%s.base ?= $(dir.dout.%s)/%s\n", + zBuildName, zBuildName, zBaseName); + + pf("c-pp.D.%s ?= %s\n", zBuildName, pB->zCmppD ? pB->zCmppD : ""); + if( pB->flags & F_64BIT ){ + pf("c-pp.D.%s += $(c-pp.D.64bit)\n", zBuildName); + } + + pf("emcc.environment.%s ?= %s\n", zBuildName, + pB->zEnv ? pB->zEnv : oBuildDefs.vanilla.zEnv); + if( pB->zEmccExtra ){ + pf("emcc.flags.%s = %s\n", zBuildName, pB->zEmccExtra); + } + + if( pB->zDeps ){ + pf("deps.%s += %s\n", zBuildName, pB->zDeps); + } + + emit_api_js(zBuildName); + mk_pre_post(zBuildName, pB); + + { /* build it... */ + pf(zBanner + "$(out.%s.js): $(MAKEFILE_LIST) $(sqlite3-wasm.c.in)" + " $(EXPORTED_FUNCTIONS.api) $(deps.%s)" + " $(bin.mkwb) $(pre-post.%s.deps)" + "\n", + zBuildName, zBuildName, zBuildName); + + emit_compile_start(zBuildName); + + if( F_UNSUPPORTED & pB->flags ){ + pf("\t@echo '$(logtag.%s) $(label.unsupported-build)'\n", + zBuildName); + } + + /* emcc ... */ + { + pf("\t$(b.cmd@)$(bin.emcc) -o $@ "); + if( pB->zEmcc ){ + pf("%s $(emcc.flags.%s)\n", + pB->zEmcc, zBuildName); + }else{ + pf("$(emcc_opt_full) $(emcc.flags)" + " $(emcc.jsflags)" + " -sENVIRONMENT=$(emcc.environment.%s)" + " $(pre-post.%s.flags)" + " $(emcc.flags.%s)" + " $(cflags.common)" + " $(cflags.%s)" + " $(SQLITE_OPT)" + " $(cflags.wasm_extra_init) $(sqlite3-wasm.c.in)\n", + zBuildName, zBuildName, zBuildName, zBuildName + ); + } + } + + { /* Post-compilation transformations and copying to + $(dir.dout)... */ + + /* Avoid a 3rd occurrence of the bug fixed by 65798c09a00662a3, + ** which was (in two cases) caused by makefile refactoring and + ** not recognized until after a release was made with the broken + ** sqlite3-bundler-friendly.mjs (which is used by the npm + ** subproject but is otherwise untested/unsupported): */ + pf("\t@if grep -e '^ *importScripts(' $@; " + "then echo '$(logtag.%s) $(emo.bug)$(emo.fire): " + "bug fixed in 65798c09a00662a3 has re-appeared'; " + "exit 1; fi;\n", zBuildName); + + if( (F_ESM & pB->flags) || (F_NODEJS & pB->flags) ){ + pf("\t@$(call b.call.patch-export-default,1,%d,$(logtag.%s))\n", + (F_WASMFS & pB->flags) ? 1 : 0, + zBuildName + ); + } + + pf("\t@chmod -x $(out.%s.wasm)\n", zBuildName + /* althttpd will automatically try to execute wasm files + if they have the +x bit set. Why that bit is set + at all is a mystery. */); + pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName); + + pf("\t@$(call b.do.wasm-opt,%s)\n", zBuildName); + pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n", zBuildName); + + if( CP_JS & pB->flags ){ + /* + ** $(bin.emcc) will write out $@ and will create a like-named + ** .wasm file. The resulting .wasm and .js/.mjs files are + ** identical across all builds which have the same pB->zEmmc + ** and/or pB->zEmccExtra. + ** + ** For the final deliverables we copy one or both of those + ** js/wasm files to $(dir.dout) (the top-most build target + ** dir). We only copy the wasm file for the "base-most" builds + ** and recycle those for the rest of the builds. The catch is: + ** that .wasm file name gets hard-coded into $@ so we need, + ** for cases in which we "recycle" a .wasm file from another + ** build, to patch the name to pB->zDotWasm when copying to + ** $(dir.dout). + */ + if( pB->zDotWasm ){ + pf("\t@echo '$(logtag.%s) $(emo.disk) " + "s/\"%s.wasm\"/\"%s.wasm\"/g " + "in $(dir.dout)/$(notdir $@)'; \\\n" + "sed" + " -e 's/\"%s.wasm\"/\"%s.wasm\"/g'" + " -e \"s/'%s.wasm'/'%s.wasm'/g\"" + " $@ > $(dir.dout)/$(notdir $@);\n", + zBuildName, + zBaseName, pB->zDotWasm, + zBaseName, pB->zDotWasm, + zBaseName, pB->zDotWasm); + }else{ + pf("\t@$(call b.cp,%s,$@,$(dir.dout))\n", + zBuildName); + } + } + if( CP_WASM & pB->flags ){ + pf("\t@$(call b.cp,%s,$(basename $@).wasm,$(dir.dout))\n", + zBuildName + //"\techo '[%s $(out.%s.wasm)] $(emo.disk) $(dir.dout)/$(notdir $(out.%s.wasm))' + //pB->zEmo, zBuildName + ); + } + + } + } + pf("\t@$(call b.echo,%s,$(emo.done) done!%s)\n", + zBuildName, + (F_UNSUPPORTED & pB->flags) + ? " $(label.unsupported-build)" + : ""); + + pf("\n%dbit: $(out.%s.js)\n" + "$(out.%s.wasm): $(out.%s.js)\n" + "b-%s: $(out.%s.js) $(out.%s.wasm)\n", + (F_64BIT & pB->flags) ? 64 : 32, zBuildName, + zBuildName, zBuildName, + zBuildName, zBuildName, zBuildName); + + if( CP_JS & pB->flags ){ + pf("$(dir.dout)/%s%s: $(out.%s.js)\n", + zBaseName, zJsExt, zBuildName + ); + } + + if( CP_WASM & pB->flags ){ + pf("$(dir.dout)/%s.wasm: $(out.%s.wasm)\n", + zBaseName, zBuildName + ); + } + + pf("%s: $(out.%s.js)\n", + 0==((F_UNSUPPORTED | F_NOT_IN_ALL) & pB->flags) + ? "all" : "more", zBuildName); + + if( pB->zIfCond ){ + pf("else\n" + "$(info $(logtag.%s) $(emo.stop) disabled by condition: %s)\n" + "endif\n", + zBuildName, pB->zIfCond); + } + pf("# End build [%s]%s", zBuildName, zBanner); +} + + +static void emit_gz(char const *zBuildName, + char const *zFileExt){ + pf("\n$(out.%s.%s).gz: $(out.%s.%s)\n" + "\t@$(call b.echo,%s,$(emo.disk))\n" + "\t@gzip < $< > $@\n", + zBuildName, zFileExt, + zBuildName, zFileExt, + zBuildName); +} + +/* +** Emits rules for the fiddle builds. +*/ +static void mk_fiddle(void){ + for(int i = 0; i < 2; ++i ){ + /* 0==normal, 1==debug */ + int const isDebug = i>0; + const char * const zBuildName = i ? "fiddle.debug" : "fiddle"; + + pf(zBanner "# Begin build %s\n", zBuildName); + if( isDebug ){ + pf("emo.b.%s = $(emo.b.fiddle)$(emo.bug)\n", zBuildName); + }else{ + pf("emo.b.fiddle = 🎻\n"); + } + emit_logtag(zBuildName); + + pf("dir.%s = %s\n" + "out.%s.js = $(dir.%s)/fiddle-module.js\n" + "out.%s.wasm = $(dir.%s)/fiddle-module.wasm\n" + "$(out.%s.wasm): $(out.%s.js)\n", + zBuildName, zBuildName, + zBuildName, zBuildName, + zBuildName, zBuildName, + zBuildName, zBuildName); + + emit_api_js(zBuildName); + mk_pre_post(zBuildName, 0); + + {/* emcc */ + pf("$(out.%s.js): $(MAKEFILE_LIST) " + "$(EXPORTED_FUNCTIONS.fiddle) " + "$(fiddle.c.in) " + "$(pre-post.%s.deps)\n", + zBuildName, zBuildName); + emit_compile_start(zBuildName); + pf("\t$(b.cmd@)$(bin.emcc) -o $@" + " $(emcc.flags.%s)" /* set in fiddle.make */ + " $(pre-post.%s.flags)" + " $(fiddle.c.in)" + "\n", + zBuildName, zBuildName); + pf("\t@chmod -x $(out.%s.wasm)\n", zBuildName); + pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName); + pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n", + zBuildName); + pf("\t@$(call b.cp," + "%s," + "$(dir.api)/sqlite3-opfs-async-proxy.js," + "$(dir $@))\n", zBuildName); + if( isDebug ){ + pf("\t@$(call b.cp,%s," + "$(dir.fiddle)/index.html " + "$(dir.fiddle)/fiddle.js " + "$(dir.fiddle)/fiddle-worker.js," + "$(dir $@)" + ")\n", + zBuildName); + } + pf("\t@$(call b.echo,%s,$(emo.done) done!)\n", zBuildName); + } + + pf("\n%s: $(out.%s.wasm)\n", isDebug ? "more" : "all", zBuildName); + + /* Compress fiddle files. We handle each file separately, rather + than compressing them in a loop in the previous target, to help + avoid that hand-edited files, like fiddle-worker.js, do not end + up with stale .gz files (which althttpd will then serve instead + of the up-to-date uncompressed one). */ + emit_gz(zBuildName, "js"); + emit_gz(zBuildName, "wasm"); + + pf("\n%s: $(out.%s.js).gz $(out.%s.wasm).gz\n" + "b-%s: %s\n", + zBuildName, zBuildName, zBuildName, + zBuildName, zBuildName); + if( isDebug ){ + ps("fiddle-debug: fiddle.debug"); /* older name */ + }else{ + ps("all: b-fiddle"); + } + pf("# End %s" zBanner, zBuildName); + } +} + +int main(int argc, char const ** argv){ + int rc = 0; + const BuildDef *pB; + pf("# What follows was GENERATED by %s. Edit at your own risk.\n", __FILE__); + + if(argc>1){ + /* + ** Only emit the rules for the given list of builds, sans prologue + ** (unless the arg "prologue" is given). Intended only for + ** debugging, not actual makefile generation. + */ + for( int i = 1; i < argc; ++i ){ + char const * const zArg = argv[i]; +#define E(N) if(0==strcmp(#N, zArg)) {mk_lib_mode(# N, &oBuildDefs.N);} else /**/ + BuildDefs_map(E) if( 0==strcmp("prologue",zArg) ){ + mk_prologue(); + }else { + fprintf(stderr,"Unkown build name: %s\n", zArg); + rc = 1; + break; + } +#undef E + } + }else{ + /* + ** Emit the whole shebang... + */ + mk_prologue(); +#define E(N) mk_lib_mode(# N, &oBuildDefs.N); + BuildDefs_map(E) +#undef E + mk_fiddle(); + } + return rc; +} diff --git a/ext/wasm/module-symbols.html b/ext/wasm/module-symbols.html index 865f1b3338..09a8a41e13 100644 --- a/ext/wasm/module-symbols.html +++ b/ext/wasm/module-symbols.html @@ -58,9 +58,7 @@ border: 0 !important; padding: 0 !important; } - .func-wasm { - - } + .func-wasm {} .func-wasm::after { content: "WASM"; color: saddlebrown; @@ -117,6 +115,7 @@
  • sqlite3.wasm namespace
  • sqlite3.wasm.pstack namespace
    • +
  • sqlite3.wasm.ptr namespace
  • Compilation options used in this module build
    • @@ -179,6 +178,14 @@

    sqlite3.wasm.pstack Namespace

    + +

    sqlite3.wasm.ptr Namespace

    +

    + The sqlite3.wasm.ptr namespace exposes the + following... +

    +
    +

    Compilation Options

    @@ -355,6 +362,7 @@

    Compilation Options

    sqlite3_js_db_vfs: 'wasm:/api-c-style.md#sqlite3_js_db_vfs', sqlite3_js_kvvfs_clear: 'wasm:/api-c-style.md#sqlite3_js_kvvfs', sqlite3_js_kvvfs_size: 'wasm:/api-c-style.md#sqlite3_js_kvvfs', + sqlite3_js_posix_create_file: 'wasm:/api-c-style.md#sqlite3_js_posix_create_file', sqlite3_js_rc_str: 'wasm:/api-c-style.md#sqlite3_js_rc_str', sqlite3_js_vfs_create_file: 'wasm:/api-c-style.md#sqlite3_js_vfs_create_file', sqlite3_js_vfs_list: 'wasm:/api-c-style.md#sqlite3_js_vfs_list', @@ -484,6 +492,9 @@

    Compilation Options

    psKeys.push('pointer','quota','remaining'); renderX(E('#list-wasm-pstack'), wasm.pstack, psKeys); + const pwKeys = Object.keys(wasm.ptr); + renderX(E('#list-wasm-ptr'), wasm.ptr, pwKeys); + const cou = wasm.compileOptionUsed(); //const cou2 = Object.create(null); //Object.entries(cou).forEach((e)=>cou2['SQLITE_'+e[0]] = e[1]); diff --git a/ext/wasm/scratchpad-wasmfs-main.js b/ext/wasm/scratchpad-wasmfs-main.js deleted file mode 100644 index 56f9325de5..0000000000 --- a/ext/wasm/scratchpad-wasmfs-main.js +++ /dev/null @@ -1,70 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - A basic test script for sqlite3-api.js. This file must be run in - main JS thread and sqlite3.js must have been loaded before it. -*/ -'use strict'; -(function(){ - const toss = function(...args){throw new Error(args.join(' '))}; - const log = console.log.bind(console), - warn = console.warn.bind(console), - error = console.error.bind(console); - - const stdout = log; - const stderr = error; - - const test1 = function(db){ - db.exec("create table if not exists t(a);") - .transaction(function(db){ - db.prepare("insert into t(a) values(?)") - .bind(new Date().getTime()) - .stepFinalize(); - stdout("Number of values in table t:", - db.selectValue("select count(*) from t")); - }); - }; - - const runTests = function(sqlite3){ - const capi = sqlite3.capi, - oo = sqlite3.oo1, - wasm = sqlite3.wasm; - stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); - const persistentDir = capi.sqlite3_wasmfs_opfs_dir(); - if(persistentDir){ - stdout("Persistent storage dir:",persistentDir); - }else{ - stderr("No persistent storage available."); - } - const startTime = performance.now(); - let db; - try { - db = new oo.DB(persistentDir+'/foo.db'); - stdout("DB filename:",db.filename); - const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', - banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; - [ - test1 - ].forEach((f)=>{ - const n = performance.now(); - stdout(banner1,"Running",f.name+"()..."); - f(db, sqlite3); - stdout(banner2,f.name+"() took ",(performance.now() - n),"ms"); - }); - }finally{ - if(db) db.close(); - } - stdout("Total test time:",(performance.now() - startTime),"ms"); - }; - - sqlite3InitModule(self.sqlite3TestModule).then(runTests); -})(); diff --git a/ext/wasm/scratchpad-wasmfs-main.html b/ext/wasm/scratchpad-wasmfs.html similarity index 54% rename from ext/wasm/scratchpad-wasmfs-main.html rename to ext/wasm/scratchpad-wasmfs.html index 91f61526cd..c37febff18 100644 --- a/ext/wasm/scratchpad-wasmfs-main.html +++ b/ext/wasm/scratchpad-wasmfs.html @@ -10,20 +10,6 @@
    sqlite3 WASMFS/OPFS Main-thread Scratchpad
    - -
    - -
    Initializing app...
    -
    - On a slow internet connection this may take a moment. If this - message displays for "a long time", intialization may have - failed and the JavaScript console may contain clues as to why. -
    -
    -
    Downloading...
    -
    - -

    Scratchpad/test app for the WASMF/OPFS integration in the main window thread. This page requires that the sqlite3 API have been built with WASMFS support. If OPFS support is available then @@ -33,8 +19,12 @@

    All stuff on this page happens in the dev console.

    - - - + diff --git a/ext/wasm/scratchpad-wasmfs.mjs b/ext/wasm/scratchpad-wasmfs.mjs new file mode 100644 index 0000000000..d6b69a1d6e --- /dev/null +++ b/ext/wasm/scratchpad-wasmfs.mjs @@ -0,0 +1,70 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +import sqlite3InitModule from './jswasm/sqlite3-wasmfs.mjs'; +//console.log('sqlite3InitModule =',sqlite3InitModule); +const toss = function(...args){throw new Error(args.join(' '))}; +const log = console.log.bind(console), + warn = console.warn.bind(console), + error = console.error.bind(console); + +const stdout = log; +const stderr = error; + +const test1 = function(db){ + db.exec("create table if not exists t(a);") + .transaction(function(db){ + db.prepare("insert into t(a) values(?)") + .bind(new Date().getTime()) + .stepFinalize(); + stdout("Number of values in table t:", + db.selectValue("select count(*) from t")); + }); +}; + +const runTests = function(sqlite3){ + const capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = sqlite3.wasm; + stdout("Loaded module:",sqlite3); + stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + const persistentDir = capi.sqlite3_wasmfs_opfs_dir(); + if(persistentDir){ + stdout("Persistent storage dir:",persistentDir); + }else{ + stderr("No persistent storage available."); + } + const startTime = performance.now(); + let db; + try { + db = new oo.DB(persistentDir+'/foo.db'); + stdout("DB filename:",db.filename); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + test1 + ].forEach((f)=>{ + const n = performance.now(); + stdout(banner1,"Running",f.name+"()..."); + f(db, sqlite3); + stdout(banner2,f.name+"() took ",(performance.now() - n),"ms"); + }); + }finally{ + if(db) db.close(); + } + stdout("Total test time:",(performance.now() - startTime),"ms"); +}; + +sqlite3InitModule().then(runTests); diff --git a/ext/wasm/speedtest1-wasmfs.html b/ext/wasm/speedtest1-wasmfs.html index 4c4db32bca..c018583d0f 100644 --- a/ext/wasm/speedtest1-wasmfs.html +++ b/ext/wasm/speedtest1-wasmfs.html @@ -10,141 +10,46 @@
    speedtest1-wasmfs.wasm
    - - -
    - -
    Initializing app...
    -
    - On a slow internet connection this may take a moment. If this - message displays for "a long time", intialization may have - failed and the JavaScript console may contain clues as to why. -
    -
    -
    Downloading...
    -
    - -
    -
    This page starts running the main exe when it loads, which will - block the UI until it finishes! Adding UI controls to manually configure and start it - are TODO.
    - +
    Achtung: running it with the dev tools open may drastically slow it down. For faster results, keep the dev tools closed when running it!
    -
    Output is delayed/buffered because we cannot update the UI while the - speedtest is running. Output will appear below when ready...
    - - - + diff --git a/ext/wasm/speedtest1-wasmfs.mjs b/ext/wasm/speedtest1-wasmfs.mjs new file mode 100644 index 0000000000..0e46806678 --- /dev/null +++ b/ext/wasm/speedtest1-wasmfs.mjs @@ -0,0 +1,90 @@ +import sqlite3InitModule from './jswasm/sqlite3-wasmfs.mjs'; +const wMsg = (type,...args)=>{ + postMessage({type, args}); +}; +wMsg('log',"speedtest1-wasmfs starting..."); +/** + If this environment contains OPFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it returns + an empty string. +*/ +const wasmfsDir = function f(wasmUtil,dirName="/opfs"){ + if(undefined !== f._) return f._; + if( !globalThis.FileSystemHandle + || !globalThis.FileSystemDirectoryHandle + || !globalThis.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3__wasm_init_wasmfs', 'i32', ['string'], dirName + )){ + return f._ = dirName; + }else{ + return f._ = ""; + } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return f._ = ""; + } +}; +wasmfsDir._ = undefined; + +const log = (...args)=>wMsg('log',...args); +const logErr = (...args)=>wMsg('logErr',...args); + +const runTests = function(sqlite3){ + console.log("Module inited.",sqlite3); + const wasm = sqlite3.wasm; + const __unlink = wasm.xWrap("sqlite3__wasm_vfs_unlink", "int", ["*","string"]); + const unlink = (fn)=>__unlink(0,fn); + const pDir = wasmfsDir(wasm); + if(pDir) log("Persistent storage:",pDir); + else{ + logErr("Expecting persistent storage in this build."); + return; + } + const scope = wasm.scopedAllocPush(); + const dbFile = pDir+"/speedtest1.db"; + const urlParams = new URL(globalThis.location.href).searchParams; + const argv = ["speedtest1"]; + if(urlParams.has('flags')){ + argv.push(...(urlParams.get('flags').split(','))); + let i = argv.indexOf('--vfs'); + if(i>=0) argv.splice(i,2); + }else{ + argv.push( + "--singlethread", + "--nomutex", + //"--nosync", + "--nomemstat", + "--size", "10" + ); + } + + if(argv.indexOf('--memdb')>=0){ + logErr("WARNING: --memdb flag trumps db filename."); + } + argv.push("--big-transactions"/*important for tests 410 and 510!*/, + dbFile); + //log("argv =",argv); + // These log messages are not emitted to the UI until after main() returns. Fixing that + // requires moving the main() call and related cleanup into a timeout handler. + if(pDir) unlink(dbFile); + log("Starting native app:\n ",argv.join(' ')); + log("This will take a while and the browser might warn about the runaway JS.", + "Give it time..."); + setTimeout(function(){ + if(pDir) unlink(dbFile); + wasm.xCall('wasm_main', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + if(pDir) unlink(dbFile); + log("Done running native main()"); + }, 25); +}/*runTests()*/; + +sqlite3InitModule({ + print: log, + printErr: logErr +}).then(runTests); diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 0d88049b95..c207e51824 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -170,8 +170,10 @@ const urlParams = new URL(self.location.href).searchParams; const W = new Worker( - "speedtest1-worker.js?sqlite3.dir=jswasm"+ - (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '') + "speedtest1-worker.js?sqlite3.dir=jswasm"+ + '&sqlite3.debugModule'+ + (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')+ + (urlParams.has('opfs-disable') ? '&opfs-disable' : '') ); const mPost = function(msgType,payload){ W.postMessage({type: msgType, data: payload}); @@ -259,7 +261,7 @@ flags["--utf16be"] = "Set text encoding to UTF-16BE"; flags["--utf16le"] = "Set text encoding to UTF-16LE"; flags["--verify"] = "Run additional verification steps."; - flags["--without"] = "rowid Use WITHOUT ROWID where appropriate"; + flags["--without-rowid"] = "Use WITHOUT ROWID where appropriate"; const preselectedFlags = [ '--big-transactions', '--singlethread' @@ -267,7 +269,7 @@ if(urlParams.has('flags')){ preselectedFlags.push(...urlParams.get('flags').split(',')); } - if('opfs'!==urlParams.get('vfs')){ + if(!urlParams.get('vfs')){ preselectedFlags.push('--memdb'); } Object.keys(flags).sort().forEach(function(f){ @@ -278,7 +280,7 @@ opt.innerHTML = lbl; opt.value = f; if(preselectedFlags.indexOf(f) >= 0) opt.selected = true; - }); + }); const cbReverseLog = E('#cb-reverse-log-order'); const lblReverseLog = E('#lbl-reverse-log-order'); if(cbReverseLog.checked){ @@ -349,7 +351,7 @@ eControls.classList.remove('hidden'); break; case 'stdout': log(msg.data); break; - case 'stdout': logErr(msg.data); break; + case 'stderr': logErr(msg.data); break; case 'run-start': eControls.disabled = true; log("Running speedtest1 with argv =",msg.data.join(' ')); diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index c61cab9190..ba11fd163d 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -5,11 +5,11 @@ if(urlParams.has('sqlite3.dir')){ speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs; } - importScripts('common/whwasmutil.js', speedtestJs); + importScripts(speedtestJs); /** - If this environment contains OPFS, this function initializes it and - returns the name of the dir on which OPFS is mounted, else it returns - an empty string. + If this build includes WASMFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it + returns an empty string. */ const wasmfsDir = function f(wasmUtil){ if(undefined !== f._) return f._; @@ -47,8 +47,9 @@ }; const log = (...args)=>logMsg('stdout',args); const logErr = (...args)=>logMsg('stderr',args); + const realSahName = 'opfs-sahpool-speedtest1'; - const runSpeedtest = function(cliFlagsArray){ + const runSpeedtest = async function(cliFlagsArray){ const scope = App.wasm.scopedAllocPush(); const dbFile = App.pDir+"/speedtest1.sqlite3"; try{ @@ -56,9 +57,32 @@ "speedtest1.wasm", ...cliFlagsArray, dbFile ]; App.logBuffer.length = 0; + const ndxSahPool = argv.indexOf('opfs-sahpool'); + if(ndxSahPool>0){ + argv[ndxSahPool] = realSahName; + log("Updated argv for opfs-sahpool: --vfs",realSahName); + } mPost('run-start', [...argv]); + if(App.sqlite3.installOpfsSAHPoolVfs + && !App.sqlite3.$SAHPoolUtil + && ndxSahPool>0){ + log("Installing opfs-sahpool as",realSahName,"..."); + await App.sqlite3.installOpfsSAHPoolVfs({ + name: realSahName, + initialCapacity: 3, + clearOnInit: true, + verbosity: 2 + }).then(PoolUtil=>{ + log("opfs-sahpool successfully installed as",PoolUtil.vfsName); + App.sqlite3.$SAHPoolUtil = PoolUtil; + //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb); + }); + } App.wasm.xCall('wasm_main', argv.length, App.wasm.scopedAllocMainArgv(argv)); + log("WASM heap size:",App.wasm.heap8().byteLength,"bytes"); + log("WASM pointer size:",App.wasm.ptr.size); + }catch(e){ mPost('error',e.message); }finally{ @@ -71,7 +95,10 @@ self.onmessage = function(msg){ msg = msg.data; switch(msg.type){ - case 'run': runSpeedtest(msg.data || []); break; + case 'run': + runSpeedtest(msg.data || []) + .catch(e=>mPost('error',e)); + break; default: logErr("Unhandled worker message type:",msg.type); break; @@ -83,17 +110,19 @@ printErr: logErr, setStatus: (text)=>mPost('load-status',text) }; - self.sqlite3InitModule(EmscriptenModule).then((sqlite3)=>{ - const S = sqlite3; - App.vfsUnlink = function(pDb, fname){ - const pVfs = S.wasm.sqlite3_wasm_db_vfs(pDb, 0); - if(pVfs) S.wasm.sqlite3_wasm_vfs_unlink(pVfs, fname||0); - }; + log("Initializing speedtest1 module..."); + self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ + const S = globalThis.S = App.sqlite3 = sqlite3; + log("Loaded speedtest1 module. Setting up..."); App.pDir = wasmfsDir(S.wasm); App.wasm = S.wasm; + log("WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); + log("WASM pointer size:",sqlite3.wasm.ptr.size); //if(App.pDir) log("Persistent storage:",pDir); //else log("Using transient storage."); mPost('ready',true); log("Registered VFSes:", ...S.capi.sqlite3_js_vfs_list()); + }).catch(e=>{ + logErr(e); }); })(); diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index 9cc20924e9..cce6171850 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -23,11 +23,10 @@
    Downloading...
    - +
    This page starts running the main exe when it loads, which will - block the UI until it finishes! Adding UI controls to manually configure and start it - are TODO.
    + block the UI until it finishes!
    Achtung: running it with the dev tools open may drastically slow it down. For faster results, keep the dev @@ -80,7 +79,7 @@ logList.forEach((v)=>log2('',v)); logList.length = 0; }; - /* can't update DOM while speedtest is running unless we run + /* we cannot update DOM while speedtest is running unless we run speedtest in a worker thread. */; const log = (...args)=>{ console.log(...args); @@ -118,7 +117,7 @@ argv.push("--vfs", vfs); log2('',"Using VFS:",vfs); if('kvvfs' === vfs){ - forceSize = 4 /* 5 uses approx. 4.96mb */; + forceSize = 2 /* >2 is too big as of mid-2025 */; dbFile = 'session'; log2('warning',"kvvfs VFS: forcing --size",forceSize, "and filename '"+dbFile+"'."); @@ -148,6 +147,8 @@ // These log messages are not emitted to the UI until after main() returns. Fixing that // requires moving the main() call and related cleanup into a timeout handler. if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); + log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); log2('',"Starting native app:\n ",argv.join(' ')); log2('',"This will take a while and the browser might warn about the runaway JS.", "Give it time..."); @@ -163,6 +164,7 @@ if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); logList.unshift("Done running native main(). Output:"); dumpLogList(); + log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); }, 50); }/*runTests()*/; diff --git a/ext/wasm/test-opfs-vfs.js b/ext/wasm/test-opfs-vfs.js index 292d77af19..96d0eacfc9 100644 --- a/ext/wasm/test-opfs-vfs.js +++ b/ext/wasm/test-opfs-vfs.js @@ -22,7 +22,7 @@ const tryOpfsVfs = async function(sqlite3){ const opfs = sqlite3.opfs; log("tryOpfsVfs()"); if(!sqlite3.opfs){ - const e = toss("OPFS is not available."); + const e = new Error("OPFS is not available."); error(e); throw e; } diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.c-pp.html similarity index 73% rename from ext/wasm/tester1-worker.html rename to ext/wasm/tester1-worker.c-pp.html index 03e1f02b02..e461b6cbff 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.c-pp.html @@ -1,16 +1,17 @@ +//#@policy error - - - sqlite3 tester #1: Worker thread + + + sqlite3 tester #1: Worker thread (@bitness@-bit WASM) -

    sqlite3 tester #1: Worker thread

    +

    sqlite3 tester #1: Worker thread (@bitness@-bit WASM)

    Variants: conventional UI thread, conventional worker, @@ -35,23 +36,29 @@

    sqlite3 tester #1: Worker thread

    logTarget.append(ln); }; const cbReverse = document.querySelector('#cb-log-reverse'); + const cbReverseKey = 'tester1:cb-log-reverse'; const cbReverseIt = ()=>{ logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); }; cbReverse.addEventListener('change',cbReverseIt,true); + if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + } cbReverseIt(); const urlParams = new URL(self.location.href).searchParams; const workerArgs = []; + const baseName = "64"==="@bitness@" ? 'tester1-64bit' : 'tester1'; if(urlParams.has('esm')){ - logHtml('warning',"Attempting to run an ES6 Worker Module, "+ - "which is not supported by all browsers! "+ - "e.g. Firefox (as of 2022-12) cannot do this."); - workerArgs.push("tester1.mjs",{type:"module"}); + logHtml('warning',"Attempting to run an ES6 Worker Module, "+ + "which is not supported by all browsers!."); + workerArgs.push(baseName+'.mjs', {type:"module"}); document.querySelectorAll('title,#color-target').forEach((e)=>{ - e.innerText = "sqlite3 tester #1: ES6 Worker Module"; + e.innerText = + "sqlite3 tester #1: ES6 Worker Module (@bitness@-bit WASM)"; }); }else{ - workerArgs.push("tester1.js?sqlite3.dir=jswasm"); + workerArgs.push(baseName+'.js?sqlite3.dir=jswasm'); } const w = new Worker(...workerArgs); w.onmessage = function({data}){ diff --git a/ext/wasm/tester1.c-pp.html b/ext/wasm/tester1.c-pp.html index bbdd8b2233..95fe52219e 100644 --- a/ext/wasm/tester1.c-pp.html +++ b/ext/wasm/tester1.c-pp.html @@ -1,4 +1,5 @@ +//#@policy error @@ -6,13 +7,7 @@ - sqlite3 tester #1: -//#if target=es6-module -ES6 Module in UI thread -//#else -UI thread -//#endif - + sqlite3 tester #1: @title@ (@bitness@-bit WASM)

    @@ -31,11 +26,11 @@ document.querySelector('h1').innerHTML = document.querySelector('title').innerHTML; })(); -//#if target=es6-module - +//#if target:es6-module + //#else - - + + //#endif diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 02eeba7a49..f72e0803fc 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -41,11 +41,12 @@ ES6 worker module build: - ./c-pp -f tester1.c-pp.js -o tester1-esm.js -Dtarget=es6-module + ./c-pp -f tester1.c-pp.js -o tester1-esm.mjs -Dtarget:es6-module */ -//#if target=es6-module -import {default as sqlite3InitModule} from './jswasm/sqlite3.mjs'; -self.sqlite3InitModule = sqlite3InitModule; +//#@policy error +//#if target:es6-module +import {default as sqlite3InitModule} from "@sqlite3.js@"; +globalThis.sqlite3InitModule = sqlite3InitModule; //#else 'use strict'; //#endif @@ -57,14 +58,24 @@ self.sqlite3InitModule = sqlite3InitModule; */ let logClass; /* Predicate for tests/groups. */ - const isUIThread = ()=>(self.window===self && self.document); + const isUIThread = ()=>(globalThis.window===self && globalThis.document); /* Predicate for tests/groups. */ const isWorker = ()=>!isUIThread(); /* Predicate for tests/groups. */ const testIsTodo = ()=>false; const haveWasmCTests = ()=>{ - return !!wasm.exports.sqlite3_wasm_test_intptr; + return !!wasm.exports.sqlite3__wasm_test_intptr; }; + const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; + }; + + let SQLite3 /* populated after module load */; + { const mapToString = (v)=>{ switch(typeof v){ @@ -101,12 +112,12 @@ self.sqlite3InitModule = sqlite3InitModule; const cbReverseKey = 'tester1:cb-log-reverse'; const cbReverseIt = ()=>{ logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); - //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); }; cbReverse.addEventListener('change', cbReverseIt, true); - /*if(localStorage.getItem(cbReverseKey)){ + if(localStorage.getItem(cbReverseKey)){ cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); - }*/ + } cbReverseIt(); }else{ /* Worker thread */ console.log("Running in a Worker thread."); @@ -127,6 +138,7 @@ self.sqlite3InitModule = sqlite3InitModule; }else{ postMessage({type:'test-result', payload:{pass}}); } + TestUtil.checkHeapSize(true); }; const log = (...args)=>{ //console.log(...args); @@ -151,6 +163,8 @@ self.sqlite3InitModule = sqlite3InitModule; const roundMs = (ms)=>Math.round(ms*100)/100; + const looksLikePtr = (v,positive=true)=> positive ? v>0 : v>=0; + /** Helpers for writing sqlite3-specific tests. */ @@ -158,8 +172,6 @@ self.sqlite3InitModule = sqlite3InitModule; /** Running total of the number of tests run via this API. */ counter: 0, - /* Separator line for log messages. */ - separator: '------------------------------------------------------------', /** If expr is a function, it is called and its result is returned, coerced to a bool, else expr, coerced to @@ -214,7 +226,7 @@ self.sqlite3InitModule = sqlite3InitModule; else if(filter instanceof Function) pass = filter(err); else if('string' === typeof filter) pass = (err.message === filter); if(!pass){ - throw new Error(msg || ("Filter rejected this exception: "+err.message)); + throw new Error(msg || ("Filter rejected this exception: <<"+err.message+">>")); } return this; }, @@ -247,13 +259,11 @@ self.sqlite3InitModule = sqlite3InitModule; return this; }, run: async function(sqlite3){ - log(TestUtil.separator); logClass('group-start',"Group #"+this.number+':',this.name); - const indent = ' '; if(this.predicate){ const p = this.predicate(sqlite3); if(!p || 'string'===typeof p){ - logClass('warning',indent, + logClass(['warning','skipping-group'], "SKIPPING group:", p ? p : "predicate says to" ); return; } @@ -265,26 +275,35 @@ self.sqlite3InitModule = sqlite3InitModule; for(const t of this.tests){ ++i; const n = this.number+"."+i; - log(indent, n+":", t.name); + logClass('one-test-line', n+":", t.name); if(t.predicate){ const p = t.predicate(sqlite3); if(!p || 'string'===typeof p){ - logClass('warning',indent, + logClass(['warning','skipping-test'], "SKIPPING:", p ? p : "predicate says to" ); skipped.push( n+': '+t.name ); continue; } } const tc = TestUtil.counter, now = performance.now(); - await t.test.call(groupState, sqlite3); + let rc = t.test.call(groupState, sqlite3); + /*if(rc instanceof Promise){ + rc = rc.catch((e)=>{ + error("Test failure:",e); + throw e; + }); + }*/ + await rc; const then = performance.now(); runtime += then - now; - logClass('faded',indent, indent, + logClass(['faded','one-test-summary'], TestUtil.counter - tc, 'assertion(s) in', roundMs(then-now),'ms'); + TestUtil.checkHeapSize(); } - logClass('green', - "Group #"+this.number+":",(TestUtil.counter - assertCount), + logClass(['green','group-end'], + "#"+this.number+":", + (TestUtil.counter - assertCount), "assertion(s) in",roundMs(runtime),"ms"); if(0 && skipped.length){ logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped); @@ -320,8 +339,7 @@ self.sqlite3InitModule = sqlite3InitModule; await g.run(sqlite3); runtime += performance.now() - now; } - log(TestUtil.separator); - logClass(['strong','green'], + logClass(['strong','green','full-test-summary'], "Done running tests.",TestUtil.counter,"assertions in", roundMs(runtime),'ms'); pok(); @@ -332,12 +350,25 @@ self.sqlite3InitModule = sqlite3InitModule; reportFinalTestStatus(false); } }.bind(this)); + }, + + checkHeapSize: function(force=false){ + const heapSize = SQLite3.wasm.heap8().byteLength; + if( force || heapSize !== TestUtil.lastHeapSize ){ + TestUtil.lastHeapSize = heapSize; + log('WASM heap size:', heapSize,'bytes'); + } } }/*TestUtil*/; const T = TestUtil; T.g = T.addGroup; T.t = T.addTest; let capi, wasm/*assigned after module init*/; + const sahPoolConfig = { + name: 'opfs-sahpool-tester1', + clearOnInit: true, + initialCapacity: 6 + }; //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -388,7 +419,7 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(wasm.realloc.impl === wasm.exports.realloc); }else{ T.assert(wasm.alloc.impl === wasm.exports.sqlite3_malloc) - .assert(wasm.dealloc === wasm.exports.sqlite3_free) + .assert(wasm.dealloc.impl === wasm.exports.sqlite3_free) .assert(wasm.realloc.impl === wasm.exports.sqlite3_realloc); } } @@ -444,7 +475,7 @@ self.sqlite3InitModule = sqlite3InitModule; try{ sqlite3.SQLite3Error.toss("resultCode check") } catch(e){ T.assert(capi.SQLITE_ERROR === e.resultCode) - .assert('resultCode check' === e.message); + .assert('resultCode check' === e.message); } }) //////////////////////////////////////////////////////////////////// @@ -485,7 +516,8 @@ self.sqlite3InitModule = sqlite3InitModule; let m = w.alloc(14); let m2 = w.realloc(m, 16); T.assert(m === m2/* because of alignment */); - T.assert(0 === w.realloc(m, 0)); + let x = w.realloc(m, 0); + T.assert(w.ptr.null === x); m = m2 = 0; // Check allocation limits and allocator's responses... @@ -494,24 +526,26 @@ self.sqlite3InitModule = sqlite3InitModule; const tooMuch = sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE + 1, isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; T.mustThrowMatching(()=>w.alloc(tooMuch), isAllocErr) - .assert(0 === w.alloc.impl(tooMuch)) + .assert(w.ptr.null === w.alloc.impl(tooMuch)) .mustThrowMatching(()=>w.realloc(0, tooMuch), isAllocErr) - .assert(0 === w.realloc.impl(0, tooMuch)); + .assert(w.ptr.null === w.realloc.impl(wasm.ptr.null, tooMuch)); } // Check allocFromTypedArray()... const byteList = [11,22,33] const u = new Uint8Array(byteList); m = w.allocFromTypedArray(u); + let mAsNumber = Number(m); for(let i = 0; i < u.length; ++i){ T.assert(u[i] === byteList[i]) - .assert(u[i] === w.peek8(m + i)); + .assert(u[i] === w.peek8(mAsNumber + i)); } w.dealloc(m); m = w.allocFromTypedArray(u.buffer); + mAsNumber = Number(m); for(let i = 0; i < u.length; ++i){ T.assert(u[i] === byteList[i]) - .assert(u[i] === w.peek8(m + i)); + .assert(u[i] === w.peek8(mAsNumber + i)); } w.dealloc(m); @@ -606,18 +640,19 @@ self.sqlite3InitModule = sqlite3InitModule; try { let cStr = w.scopedAllocCString("hello"); const n = w.cstrlen(cStr); + const nPtr = w.ptr.coerce(n); let cpy = w.scopedAlloc(n+10); let rc = w.cstrncpy(cpy, cStr, n+10); T.assert(n+1 === rc). assert("hello" === w.cstrToJs(cpy)). - assert(chr('o') === w.peek8(cpy+n-1)). - assert(0 === w.peek8(cpy+n)); + assert(chr('o') === w.peek8( w.ptr.add(cpy,nPtr, -1))). + assert(0 === w.peek8( w.ptr.add(cpy,nPtr) ) ); let cStr2 = w.scopedAllocCString("HI!!!"); rc = w.cstrncpy(cpy, cStr2, 3); T.assert(3===rc). assert("HI!lo" === w.cstrToJs(cpy)). - assert(chr('!') === w.peek8(cpy+2)). - assert(chr('l') === w.peek8(cpy+3)); + assert(chr('!') === w.peek8( w.ptr.add(cpy, 2) )). + assert(chr('l') === w.peek8( w.ptr.add(cpy, 3) ) ); }finally{ w.scopedAllocPop(scope); } @@ -640,8 +675,8 @@ self.sqlite3InitModule = sqlite3InitModule; const jstr = "hällo, world!"; const [cstr, n] = w.allocCString(jstr, true); T.assert(14 === n) - .assert(0===w.peek8(cstr+n)) - .assert(chr('!')===w.peek8(cstr+n-1)); + .assert(0===w.peek8(w.ptr.add(cstr,n))) + .assert(chr('!')===w.peek8(w.ptr.add(cstr,n,-1))); w.dealloc(cstr); } @@ -664,21 +699,21 @@ self.sqlite3InitModule = sqlite3InitModule; const p1 = w.scopedAlloc(16), p2 = w.scopedAlloc(16); T.assert(1===w.scopedAlloc.level) - .assert(Number.isFinite(p1)) - .assert(Number.isFinite(p2)) + .assert(looksLikePtr(p1)) + .assert(looksLikePtr(p2)) .assert(asc[0] === p1) .assert(asc[1]===p2); asc2 = w.scopedAllocPush(); const p3 = w.scopedAlloc(16); T.assert(2===w.scopedAlloc.level) - .assert(Number.isFinite(p3)) + .assert(looksLikePtr(p3)) .assert(2===asc.length) .assert(p3===asc2[0]); const [z1, z2, z3] = w.scopedAllocPtr(3); - T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) - .assert(0===w.peek32(z1), 'allocPtr() must zero the targets') - .assert(0===w.peek32(z3)); + T.assert(typeof w.ptr.null===typeof z1).assert(z2>z1).assert(z3>z2) + .assert(w.ptr.null===w.peekPtr(z1), 'allocPtr() must zero the targets') + .assert(w.ptr.null===w.peekPtr(z3)); }finally{ // Pop them in "incorrect" order to make sure they behave: w.scopedAllocPop(asc); @@ -698,15 +733,15 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(1===w.scopedAlloc.level); const [cstr, n] = w.scopedAllocCString("hello, world", true); T.assert(12 === n) - .assert(0===w.peek8(cstr+n)) - .assert(chr('d')===w.peek8(cstr+n-1)); + .assert(0===w.peek8( w.ptr.add(cstr,n) )) + .assert(chr('d')===w.peek8( w.ptr.add(cstr, n, -1) )); }); }/*scopedAlloc()*/ //log("xCall()..."); { - const pJson = w.xCall('sqlite3_wasm_enum_json'); - T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); + const pJson = w.xCall('sqlite3__wasm_enum_json'); + T.assert(looksLikePtr(pJson)).assert(w.cstrlen(pJson)>300); } //log("xWrap()..."); @@ -719,9 +754,9 @@ self.sqlite3InitModule = sqlite3InitModule; T.mustThrowMatching(()=>fw(1), /requires 0 arg/); let rc = fw(); T.assert('string'===typeof rc).assert(rc.length>5); - rc = w.xCallWrapped('sqlite3_wasm_enum_json','*'); - T.assert(rc>0 && Number.isFinite(rc)); - rc = w.xCallWrapped('sqlite3_wasm_enum_json','utf8'); + rc = w.xCallWrapped('sqlite3__wasm_enum_json','*'); + T.assert(rc>0 && looksLikePtr(rc)); + rc = w.xCallWrapped('sqlite3__wasm_enum_json','utf8'); T.assert('string'===typeof rc).assert(rc.length>300); @@ -736,8 +771,9 @@ self.sqlite3InitModule = sqlite3InitModule; // 'string:flexible' argAdapter() sanity checks... w.scopedAllocCall(()=>{ - const argAd = w.xWrap.argAdapter('string:flexible'); - const cj = (v)=>w.cstrToJs(argAd(v)); + const toFlexStr = w.xWrap.argAdapter('string:flexible'); + const cj = (v)=>w.cstrToJs(toFlexStr(v)); + //console.debug("toFlexStr(new Uint8Array([72, 73]))",toFlexStr(new Uint8Array([72, 73]))); T.assert('Hi' === cj('Hi')) .assert('hi' === cj(['h','i'])) .assert('HI' === cj(new Uint8Array([72, 73]))); @@ -804,7 +840,7 @@ self.sqlite3InitModule = sqlite3InitModule; if(haveWasmCTests()){ if(!sqlite3.config.useStdAlloc){ - fw = w.xWrap('sqlite3_wasm_test_str_hello', 'utf8:dealloc',['i32']); + fw = w.xWrap('sqlite3__wasm_test_str_hello', 'utf8:dealloc',['i32']); rc = fw(0); T.assert('hello'===rc); rc = fw(1); @@ -814,14 +850,14 @@ self.sqlite3InitModule = sqlite3InitModule; if(w.bigIntEnabled){ w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); - fw = w.xWrap('sqlite3_wasm_test_int64_times2','thrice','twice'); + fw = w.xWrap('sqlite3__wasm_test_int64_times2','thrice','twice'); rc = fw(1); T.assert(12n===rc); w.scopedAllocCall(function(){ - const pI1 = w.scopedAlloc(8), pI2 = pI1+4; - w.pokePtr([pI1, pI2], 0); - const f = w.xWrap('sqlite3_wasm_test_int64_minmax',undefined,['i64*','i64*']); + const pI1 = w.scopedAlloc(w.ptr.size), pI2 = w.ptr.add(pI1, w.ptr.size); + w.pokePtr([pI1, pI2], w.ptr.null); + const f = w.xWrap('sqlite3__wasm_test_int64_minmax',undefined,['i64*','i64*']); const [r1, r2] = w.peek64([pI1, pI2]); T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); }); @@ -831,171 +867,196 @@ self.sqlite3InitModule = sqlite3InitModule; }/*WhWasmUtil*/) //////////////////////////////////////////////////////////////////// - .t('sqlite3.StructBinder (jaccwabyt🐇)', function(sqlite3){ - const S = sqlite3, W = S.wasm; - const MyStructDef = { - sizeof: 16, - members: { - p4: {offset: 0, sizeof: 4, signature: "i"}, - pP: {offset: 4, sizeof: 4, signature: "P"}, - ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, - cstr: {offset: 12, sizeof: 4, signature: "s"} - } - }; - if(W.bigIntEnabled){ - const m = MyStructDef; - m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; - m.sizeof += m.members.p8.sizeof; - } - const StructType = S.StructBinder.StructType; - const K = S.StructBinder('my_struct',MyStructDef); - T.mustThrowMatching(()=>K(), /via 'new'/). - mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); - const k1 = new K(), k2 = new K(); - try { - T.assert(k1.constructor === K). - assert(K.isA(k1)). - assert(k1 instanceof K). - assert(K.prototype.lookupMember('p4').key === '$p4'). - assert(K.prototype.lookupMember('$p4').name === 'p4'). - mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). - assert(undefined === K.prototype.lookupMember('nope',false)). - assert(k1 instanceof StructType). - assert(StructType.isA(k1)). - mustThrowMatching(()=>k1.$ro = 1, /read-only/); - Object.keys(MyStructDef.members).forEach(function(key){ - key = K.memberKey(key); - T.assert(0 == k1[key], - "Expecting allocation to zero the memory "+ - "for "+key+" but got: "+k1[key]+ - " from "+k1.memoryDump()); + .t({ + name: 'sqlite3.StructBinder (jaccwabyt🐇)', + predicate: (sqlite3)=>!!sqlite3.wasm.exports.sqlite3__wasm_test_struct + || "Built without SQLITE_WASM_ENABLE_C_TESTS", + test: function(sqlite3){ + const S = sqlite3, W = S.wasm; + const MyStructDef = { + sizeof: 0, members: {} + }; + const addMember = function(tgt, name, member){ + member.offset = tgt.sizeof; + tgt.sizeof += member.sizeof; + tgt.members[name] = member; + }; + const msd = MyStructDef; + addMember(msd, 'p4', {sizeof: 4, signature: "i"}); + addMember(msd, 'pP', {sizeof: wasm.ptr.size, signature: "P"}); + addMember(msd, 'ro', { + sizeof: 4, + signature: "i", + readOnly: true + }); + addMember(msd, 'cstr', { + sizeof: wasm.ptr.size, + signature: "s" }); - T.assert('number' === typeof k1.pointer). - mustThrowMatching(()=>k1.pointer = 1, /pointer/); - k1.$p4 = 1; k1.$pP = 2; - T.assert(1 === k1.$p4).assert(2 === k1.$pP); - if(MyStructDef.members.$p8){ - k1.$p8 = 1/*must not throw despite not being a BigInt*/; - k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); - T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + if(W.bigIntEnabled){ + addMember(msd, 'p8', {sizeof: 8, signature: "j"}); + } + const StructType = S.StructBinder.StructType; + const K = S.StructBinder('my_struct',MyStructDef); + //K.debugFlags(0x03); + T.mustThrowMatching(()=>K(), /via 'new'/). + mustThrowMatching(()=>new K('hi'), (err)=>{ + return /^Invalid pointer/.test(err.message)/*32-bit*/ + || /.*bigint.*/i.test(err.message)/*64-bit*/; + }); + const k1 = new K(), k2 = new K(); + try { + T.assert(k1.constructor === K). + assert(K.isA(k1)). + assert(k1 instanceof K). + assert(K.prototype.lookupMember('p4').key === '$p4'). + assert(K.prototype.lookupMember('$p4').name === 'p4'). + mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). + assert(undefined === K.prototype.lookupMember('nope',false)). + assert(k1 instanceof StructType). + assert(StructType.isA(k1)). + mustThrowMatching(()=>k1.$ro = 1, /read-only/); + Object.keys(MyStructDef.members).forEach(function(key){ + key = K.memberKey(key); + T.assert(0 == k1[key], + "Expecting allocation to zero the memory "+ + "for "+key+" but got: "+k1[key]+ + " from "+k1.memoryDump()); + }); + T.assert(looksLikePtr(k1.pointer)). + mustThrowMatching(()=>k1.pointer = 1, /pointer/); + k1.$p4 = 1; k1.$pP = 2; + T.assert(1 == k1.$p4).assert(2 == k1.$pP); + if(MyStructDef.members.$p8){ + k1.$p8 = 1/*must not throw despite not being a BigInt*/; + k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + } + T.assert(!k1.ondispose); + k1.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(k1.ondispose)). + assert(k1.ondispose[0] === k1.$cstr). + assert(looksLikePtr(k1.$cstr)). + assert('A C-string.' === k1.memberToJsString('cstr')); + k1.$pP = k2; + T.assert(k1.$pP === k2.pointer); + k1.$pP = null/*null is special-cased to 0.*/; + T.assert(0==k1.$pP); + let ptr = k1.pointer; + k1.dispose(); + T.assert(undefined === k1.pointer). + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + }finally{ + k1.dispose(); + k2.dispose(); } - T.assert(!k1.ondispose); - k1.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(k1.ondispose)). - assert(k1.ondispose[0] === k1.$cstr). - assert('number' === typeof k1.$cstr). - assert('A C-string.' === k1.memberToJsString('cstr')); - k1.$pP = k2; - T.assert(k1.$pP === k2.pointer); - k1.$pP = null/*null is special-cased to 0.*/; - T.assert(0===k1.$pP); - let ptr = k1.pointer; - k1.dispose(); - T.assert(undefined === k1.pointer). - mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); - }finally{ - k1.dispose(); - k2.dispose(); - } - if(!W.bigIntEnabled){ - log("Skipping WasmTestStruct tests: BigInt not enabled."); - return; - } + if(!W.bigIntEnabled){ + log("Skipping WasmTestStruct tests: BigInt not enabled."); + return; + } - const WTStructDesc = - W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; - const autoResolvePtr = true /* EXPERIMENTAL */; - if(autoResolvePtr){ - WTStructDesc.members.ppV.signature = 'P'; - } - const WTStruct = S.StructBinder(WTStructDesc); - //log(WTStruct.structName, WTStruct.structInfo); - const wts = new WTStruct(); - //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); - try{ - T.assert(wts.constructor === WTStruct). - assert(WTStruct.memberKeys().indexOf('$ppV')>=0). - assert(wts.memberKeys().indexOf('$v8')>=0). - assert(!K.isA(wts)). - assert(WTStruct.isA(wts)). - assert(wts instanceof WTStruct). - assert(wts instanceof StructType). - assert(StructType.isA(wts)). - assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). - assert(0===wts.$ppV).assert(0===wts.$xFunc); - const testFunc = - W.xGet('sqlite3_wasm_test_struct'/*name gets mangled in -O3 builds!*/); - let counter = 0; - //log("wts.pointer =",wts.pointer); - const wtsFunc = function(arg){ - /*log("This from a JS function called from C, "+ + const WTStructDesc = + W.ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; + const autoResolvePtr = true /* EXPERIMENTAL */; + if(autoResolvePtr){ + WTStructDesc.members.ppV.signature = 'P'; + } + const WTStruct = S.StructBinder(WTStructDesc); + //log(WTStruct.structName, WTStruct.structInfo); + const wts = new WTStruct(); + //log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); + try{ + T.assert(wts.constructor === WTStruct). + assert(WTStruct.memberKeys().indexOf('$ppV')>=0). + assert(wts.memberKeys().indexOf('$v8')>=0). + assert(!K.isA(wts)). + assert(WTStruct.isA(wts)). + assert(wts instanceof WTStruct). + assert(wts instanceof StructType). + assert(StructType.isA(wts)). + assert(looksLikePtr(wts.pointer)).assert(0==wts.$v4).assert(0n===wts.$v8). + assert(0==wts.$ppV).assert(0==wts.$xFunc); + const testFunc = 1 + ? W.xGet('sqlite3__wasm_test_struct'/*name gets mangled in -O3 builds!*/) + : W.xWrap('sqlite3__wasm_test_struct', undefined, '*'); + let counter = 0; + //log("wts.pointer =",wts.pointer); + const wtsFunc = function(arg){ + /*log("This from a JS function called from C, "+ "which itself was called from JS. arg =",arg);*/ - ++counter; - if(3===counter){ - tossQuietly("Testing exception propagation."); + ++counter; + if(3===counter){ + tossQuietly("Testing exception propagation."); + } + } + wts.$v4 = 10; wts.$v8 = 20; + wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) + //console.debug("wts.memberSignature('xFunc')",wts.memberSignature('xFunc')); + //console.debug("wts.$xFunc",wts.$xFunc, W.functionEntry(wts.$xFunc)); + T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) + .assert(0 == wts.$ppV).assert(looksLikePtr(wts.$xFunc)) + .assert(0 == wts.$cstr) + .assert(wts.memberIsString('$cstr')) + .assert(!wts.memberIsString('$v4')) + .assert(null === wts.memberToJsString('$cstr')) + .assert(W.functionEntry(wts.$xFunc) instanceof Function); + /* It might seem silly to assert that the values match + what we just set, but recall that all of those property + reads and writes are, via property interceptors, + actually marshaling their data to/from a raw memory + buffer, so merely reading them back is actually part of + testing the struct-wrapping API. */ + + if( 0 ){ + console.debug("wts",wts,"wts.pointer",wts.pointer, + "testFunc",testFunc/*FF v142 emits the wrong function here!*/); } + testFunc(wts.pointer); + //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); + T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) + .assert(wts.$ppV === wts.pointer) + .assert('string' === typeof wts.memberToJsString('cstr')) + .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) + .mustThrowMatching(()=>wts.memberToJsString('xFunc'), + /Invalid member type signature for C-string/) + ; + testFunc(wts.pointer); + T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) + .assert(wts.$ppV === wts.pointer); + /** The 3rd call to wtsFunc throw from JS, which is called + from C, which is called from JS. Let's ensure that + that exception propagates back here... */ + T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); + W.uninstallFunction(wts.$xFunc); + wts.$xFunc = 0; + wts.$ppV = 0; + T.assert(!wts.$ppV); + //WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts.pointer === wts.$ppV) + wts.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(wts.ondispose)). + assert(wts.ondispose[0] === wts.$cstr). + assert('A C-string.' === wts.memberToJsString('cstr')); + const ptr = wts.pointer; + wts.dispose(); + T.assert(ptr).assert(undefined === wts.pointer); + }finally{ + wts.dispose(); } - wts.$v4 = 10; wts.$v8 = 20; - wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) - T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) - .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) - .assert(0 === wts.$cstr) - .assert(wts.memberIsString('$cstr')) - .assert(!wts.memberIsString('$v4')) - .assert(null === wts.memberToJsString('$cstr')) - .assert(W.functionEntry(wts.$xFunc) instanceof Function); - /* It might seem silly to assert that the values match - what we just set, but recall that all of those property - reads and writes are, via property interceptors, - actually marshaling their data to/from a raw memory - buffer, so merely reading them back is actually part of - testing the struct-wrapping API. */ - - testFunc(wts.pointer); - //log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); - T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) - .assert(wts.$ppV === wts.pointer) - .assert('string' === typeof wts.memberToJsString('cstr')) - .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) - .mustThrowMatching(()=>wts.memberToJsString('xFunc'), - /Invalid member type signature for C-string/) - ; - testFunc(wts.pointer); - T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) - .assert(wts.$ppV === wts.pointer); - /** The 3rd call to wtsFunc throw from JS, which is called - from C, which is called from JS. Let's ensure that - that exception propagates back here... */ - T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); - W.uninstallFunction(wts.$xFunc); - wts.$xFunc = 0; - wts.$ppV = 0; - T.assert(!wts.$ppV); - //WTStruct.debugFlags(0x03); - wts.$ppV = wts; - T.assert(wts.pointer === wts.$ppV) - wts.setMemberCString('cstr', "A C-string."); - T.assert(Array.isArray(wts.ondispose)). - assert(wts.ondispose[0] === wts.$cstr). - assert('A C-string.' === wts.memberToJsString('cstr')); - const ptr = wts.pointer; - wts.dispose(); - T.assert(ptr).assert(undefined === wts.pointer); - }finally{ - wts.dispose(); - } - if(1){ // ondispose of other struct instances - const s1 = new WTStruct, s2 = new WTStruct, s3 = new WTStruct; - T.assert(s1.lookupMember instanceof Function) - .assert(s1.addOnDispose instanceof Function); - s1.addOnDispose(s2,"testing variadic args"); - T.assert(2===s1.ondispose.length); - s2.addOnDispose(s3); - s1.dispose(); - T.assert(!s2.pointer,"Expecting s2 to be ondispose'd by s1."); - T.assert(!s3.pointer,"Expecting s3 to be ondispose'd by s2."); + if(1){ // ondispose of other struct instances + const s1 = new WTStruct, s2 = new WTStruct, s3 = new WTStruct; + T.assert(s1.lookupMember instanceof Function) + .assert(s1.addOnDispose instanceof Function); + s1.addOnDispose(s2,"testing variadic args"); + T.assert(2===s1.ondispose.length); + s2.addOnDispose(s3); + s1.dispose(); + T.assert(!s2.pointer,"Expecting s2 to be ondispose'd by s1."); + T.assert(!s3.pointer,"Expecting s3 to be ondispose'd by s2."); + } } }/*StructBinder*/) @@ -1004,7 +1065,7 @@ self.sqlite3InitModule = sqlite3InitModule; const P = wasm.pstack; const isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; const stack = P.pointer; - T.assert(0===stack % 8 /* must be 8-byte aligned */); + T.assert(0===Number(stack) % 8 /* must be 8-byte aligned */); try{ const remaining = P.remaining; T.assert(P.quota >= 4096) @@ -1017,16 +1078,16 @@ self.sqlite3InitModule = sqlite3InitModule; ); ; let p1 = P.alloc(12); - T.assert(p1 === stack - 16/*8-byte aligned*/) + T.assert(p1 == Number(stack) - 16/*8-byte aligned*/) .assert(P.pointer === p1); let p2 = P.alloc(7); - T.assert(p2 === p1-8/*8-byte aligned, stack grows downwards*/) + T.assert(p2 == Number(p1)-8/*8-byte aligned, stack grows downwards*/) .mustThrowMatching(()=>P.alloc(remaining), isAllocErr) - .assert(24 === stack - p2) + .assert(24 == Number(stack) - Number(p2)) .assert(P.pointer === p2); - let n = remaining - (stack - p2); + let n = remaining - (Number(stack) - Number(p2)); let p3 = P.alloc(n); - T.assert(p3 === stack-remaining) + T.assert(p3 == Number(stack)-Number(remaining)) .mustThrowMatching(()=>P.alloc(1), isAllocErr); }finally{ P.restore(stack); @@ -1035,9 +1096,11 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(P.pointer === stack); try { const [p1, p2, p3] = P.allocChunks(3,'i32'); - T.assert(P.pointer === stack-16/*always rounded to multiple of 8*/) - .assert(p2 === p1 + 4) - .assert(p3 === p2 + 4); + let sPos = wasm.ptr.add(stack,-16)/*pstack alloc always rounds to multiple of 8*/; + T.assert(P.pointer === sPos) + .assert(p1 === sPos) + .assert(p2 == Number(p1) + 4) + .assert(p3 == Number(p2) + 4); T.mustThrowMatching(()=>P.allocChunks(1024, 1024 * 16), (e)=>e instanceof sqlite3.WasmAllocError) }finally{ @@ -1047,16 +1110,20 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(P.pointer === stack); try { let [p1, p2, p3] = P.allocPtr(3,false); - let sPos = stack-16/*always rounded to multiple of 8*/; - T.assert(P.pointer === sPos) - .assert(p2 === p1 + 4) - .assert(p3 === p2 + 4); + let sPos = wasm.ptr.add(stack, + -(4===wasm.ptr.size + ? 16/*pstack alloc always rounds to multiple of 8*/ + : 24)); + T.assert(P.pointer === p1) + .assert(p1 === sPos) + .assert(p2 == Number(p1) + wasm.ptr.size) + .assert(p3 == Number(p2) + wasm.ptr.size); [p1, p2, p3] = P.allocPtr(3); - T.assert(P.pointer === sPos-24/*3 x 8 bytes*/) - .assert(p2 === p1 + 8) - .assert(p3 === p2 + 8); + T.assert(P.pointer === wasm.ptr.add(sPos, -24)/*3 x 8 bytes*/) + .assert(p2 == Number(p1) + 8) + .assert(p3 == Number(p2) + 8); p1 = P.allocPtr(); - T.assert('number'===typeof p1); + T.assert(looksLikePtr(p1)); }finally{ P.restore(stack); } @@ -1070,19 +1137,19 @@ self.sqlite3InitModule = sqlite3InitModule; try{ const n = 520; const p = wasm.pstack.alloc(n); - T.assert(0===wasm.peek8(p)) - .assert(0===wasm.peek8(p+n-1)); + T.assert(0==wasm.peek8(p)) + .assert(0==wasm.peek8(wasm.ptr.add(p,n,-1))); T.assert(undefined === capi.sqlite3_randomness(n - 10, p)); let j, check = 0; const heap = wasm.heap8u(); for(j = 0; j < 10 && 0===check; ++j){ - check += heap[p + j]; + check += heap[wasm.ptr.add(p, j)]; } T.assert(check > 0); check = 0; // Ensure that the trailing bytes were not modified... for(j = n - 10; j < n && 0===check; ++j){ - check += heap[p + j]; + check += heap[wasm.ptr.add(p, j)]; } T.assert(0===check); }finally{ @@ -1109,77 +1176,184 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('sqlite3.oo1') - .t('Create db', function(sqlite3){ - const dbFile = '/tester1.db'; - wasm.sqlite3_wasm_vfs_unlink(0, dbFile); - const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); - db.onclose = { - disposeAfter: [], - disposeBefore: [ - (db)=>{ - //console.debug("db.onclose.before dropping modules"); - //sqlite3.capi.sqlite3_drop_modules(db.pointer, 0); - } - ], - before: function(db){ - while(this.disposeBefore.length){ - const v = this.disposeBefore.shift(); - console.debug("db.onclose.before cleaning up:",v); - if(wasm.isPtr(v)) wasm.dealloc(v); - else if(v instanceof sqlite3.StructBinder.StructType){ - v.dispose(); - }else if(v instanceof Function){ - try{ v(db) } catch(e){ - console.warn("beforeDispose() callback threw:",e); + .t({ + name:'Create db', + //predicate: (sqlite3)=> + test: function(sqlite3){ + const dbFile = '/tester1.db'; + sqlite3.util.sqlite3__wasm_vfs_unlink(0, dbFile); + const db = this.db = new sqlite3.oo1.DB(dbFile, 0 ? 'ct' : 'c'); + db.onclose = { + disposeAfter: [], + disposeBefore: [ + (db)=>{ + //console.debug("db.onclose.before dropping modules"); + //sqlite3.capi.sqlite3_drop_modules(db.pointer, 0); + } + ], + before: function(db){ + while(this.disposeBefore.length){ + const v = this.disposeBefore.shift(); + console.debug("db.onclose.before cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + }else if(v instanceof Function){ + try{ v(db) } catch(e){ + console.warn("beforeDispose() callback threw:",e); + } } } - } - }, - after: function(){ - while(this.disposeAfter.length){ - const v = this.disposeAfter.shift(); - console.debug("db.onclose.after cleaning up:",v); - if(wasm.isPtr(v)) wasm.dealloc(v); - else if(v instanceof sqlite3.StructBinder.StructType){ - v.dispose(); - }else if(v instanceof Function){ - try{v()} catch(e){/*ignored*/} + }, + after: function(){ + while(this.disposeAfter.length){ + const v = this.disposeAfter.shift(); + console.debug("db.onclose.after cleaning up:",v); + if(wasm.isPtr(v)) wasm.dealloc(v); + else if(v instanceof sqlite3.StructBinder.StructType){ + v.dispose(); + }else if(v instanceof Function){ + try{v()} catch(e){/*ignored*/} + } } } + }; + + T.assert(wasm.isPtr(db.pointer)) + .mustThrowMatching(()=>db.pointer=1, /read-only/) + .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) + .assert('main'===db.dbName(0)) + .assert('string' === typeof db.dbVfsName()) + .assert(db.pointer === wasm.xWrap.testConvertArg('sqlite3*',db)); + // Custom db error message handling via sqlite3_prepare_v2/v3() + let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); + T.assert(capi.SQLITE_MISUSE === rc) + .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) + .assert(dbFile === db.dbFilename()) + .assert(!db.dbFilename('nope')); + //Sanity check DB.checkRc()... + let ex; + try{db.checkRc(rc)} + catch(e){ex = e} + T.assert(ex instanceof sqlite3.SQLite3Error) + .assert(capi.SQLITE_MISUSE===ex.resultCode) + .assert(0===ex.message.indexOf("SQLITE_MISUSE: sqlite3 result code")) + .assert(ex.message.indexOf("Invalid SQL")>0); + T.assert(db === db.checkRc(0)) + .assert(db === sqlite3.oo1.DB.checkRc(db,0)) + .assert(null === sqlite3.oo1.DB.checkRc(null,0)); + this.progressHandlerCount = 0; + if( wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK') ){ + T.assert( !capi.sqlite3_progress_handler ); + }else{ + T.assert( !!capi.sqlite3_progress_handler ); + capi.sqlite3_progress_handler(db, 5, (p)=>{ + ++this.progressHandlerCount; + return 0; + }, 0); } - }; + } + }) - T.assert(wasm.isPtr(db.pointer)) - .mustThrowMatching(()=>db.pointer=1, /read-only/) - .assert(0===sqlite3.capi.sqlite3_extended_result_codes(db.pointer,1)) - .assert('main'===db.dbName(0)) - .assert('string' === typeof db.dbVfsName()) - .assert(db.pointer === wasm.xWrap.testConvertArg('sqlite3*',db)); - // Custom db error message handling via sqlite3_prepare_v2/v3() - let rc = capi.sqlite3_prepare_v3(db.pointer, {/*invalid*/}, -1, 0, null, null); - T.assert(capi.SQLITE_MISUSE === rc) - .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")) - .assert(dbFile === db.dbFilename()) - .assert(!db.dbFilename('nope')); - //Sanity check DB.checkRc()... - let ex; - try{db.checkRc(rc)} - catch(e){ex = e} - T.assert(ex instanceof sqlite3.SQLite3Error) - .assert(0===ex.message.indexOf("sqlite3 result code")) - .assert(ex.message.indexOf("Invalid SQL")>0); - T.assert(db === db.checkRc(0)) - .assert(db === sqlite3.oo1.DB.checkRc(db,0)) - .assert(null === sqlite3.oo1.DB.checkRc(null,0)); + //////////////////////////////////////////////////////////////////// + .t({ + name: "oo1.DB/Stmt.wrapDbHandle()", + test: function(sqlite3){ + /* Maintenance reminder: this function is early in the list to + demonstrate that the wrappers for this.db created by this + function do not interfere with downstream tests, e.g. by + closing this.db.pointer. */ + //sqlite3.config.debug("Proxying",this.db); + const misuseMsg = "SQLITE_MISUSE: Argument must be a WASM sqlite3 pointer"; + T.mustThrowMatching(()=>sqlite3.oo1.DB.wrapHandle(this.db), misuseMsg) + .mustThrowMatching(()=>sqlite3.oo1.DB.wrapHandle(0), misuseMsg); + let dw = sqlite3.oo1.DB.wrapHandle(this.db.pointer); + //sqlite3.config.debug('dw',dw); + T.assert( dw, '!!dw' ) + .assert( dw instanceof sqlite3.oo1.DB, 'dw is-a oo1.DB' ) + .assert( dw.pointer, 'dw.pointer' ) + .assert( dw.pointer === this.db.pointer, 'dw.pointer===db.pointer' ) + .assert( dw.filename === this.db.filename, 'dw.filename===db.filename' ); + + T.assert( dw === dw.exec("select 1") ); + let q; + try { + q = dw.prepare("select 1"); + T.assert( q.step() ) + .assert( !q.step() ); + }finally{ + if( q ) q.finalize(); + } + dw.close(); + T.assert( !dw.pointer ) + .assert( this.db === this.db.exec("select 1") ); + dw = undefined; + + let pDb = 0, pStmt = 0; + const stack = wasm.pstack.pointer; + try { + const ppOut = wasm.pstack.allocPtr(); + T.assert( 0==wasm.peekPtr(ppOut) ); + let rc = capi.sqlite3_open_v2( ":memory:", ppOut, + capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE, + 0); + T.assert( 0===rc, 'open_v2()' ); + pDb = wasm.peekPtr(ppOut); + wasm.pokePtr(ppOut, 0); + T.assert( pDb>0, 'pDb>0' ); + const pTmp = pDb; + dw = sqlite3.oo1.DB.wrapHandle(pDb, true); + pDb = 0; + //sqlite3.config.debug("dw",dw); + T.assert( pTmp===dw.pointer, 'pTmp===dw.pointer' ); + T.assert( dw.filename === "", "dw.filename == "+dw.filename ); + let q = dw.prepare("select 1"); + try { + T.assert( q.step(), "step()" ); + T.assert( !q.step(), "!step()" ); + }finally{ + q.finalize(); + q = undefined; + } + T.assert( dw===dw.exec("select 1") ); + dw.affirmOpen(); + const select1 = "select 1"; + rc = capi.sqlite3_prepare_v2( dw, select1, -1, ppOut, 0 ); + T.assert( 0===rc, 'prepare_v2() rc='+rc ); + pStmt = wasm.peekPtr(ppOut); + T.assert( pStmt && wasm.isPtr(pStmt), 'pStmt is valid?' ); + try { + //log( "capi.sqlite3_sql() =",capi.sqlite3_sql(pStmt)); + T.assert( select1 === capi.sqlite3_sql(pStmt), 'SQL mismatch' ); + q = sqlite3.oo1.Stmt.wrapHandle(dw, pStmt, false); + //log("q@"+pStmt+" does not own handle"); + T.assert( q.step(), "step()" ) + .assert( !q.step(), "!step()" ); + q.finalize(); + q = undefined; + T.assert( select1 === capi.sqlite3_sql(pStmt), 'SQL mismatch' + /* This will fail if we've mismanaged pStmt's lifetime */); + q = sqlite3.oo1.Stmt.wrapHandle(dw, pStmt, true); + pStmt = 0; + q.reset(); + T.assert( q.step(), "step()" ) + .assert( !q.step(), "!step()" ); + }finally{ + if( pStmt ) capi.sqlite3_finalize(pStmt) + if( q ) q.finalize(); + } + + }finally{ + wasm.pstack.restore(stack); + if( pDb ){ capi.sqlite3_close_v2(pDb); } + else if( dw ){ dw.close(); } + } + } + })/*oo1.DB/Stmt.wrapHandle()*/ - this.progressHandlerCount = 0; - capi.sqlite3_progress_handler(db, 5, (p)=>{ - ++this.progressHandlerCount; - return 0; - }, 0); - }) //////////////////////////////////////////////////////////////////// - .t('sqlite3_db_config() and sqlite3_db_status()', function(sqlite3){ + .t('sqlite3_db_config() and sqlite3_status()', function(sqlite3){ let rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, 0, 0); T.assert(0===rc); rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_MAX+1, 0); @@ -1210,6 +1384,12 @@ self.sqlite3InitModule = sqlite3InitModule; }finally{ wasm.pstack.restore(stack); } + + capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_ENABLE_COMMENTS, 0, null); + T.mustThrow(()=>this.db.exec("select 1 /* with comments */"), "SQL comments are disallowed"); + capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_ENABLE_COMMENTS, 1, null); + this.db.exec("select 1 /* with comments */"); + /* SQLITE_DBCONFIG_ENABLE_ATTACH_... are in the ATTACH-specific tests */ }) //////////////////////////////////////////////////////////////////// @@ -1217,8 +1397,7 @@ self.sqlite3InitModule = sqlite3InitModule; let st = this.db.prepare( new TextEncoder('utf-8').encode("select 3 as a") ); - //debug("statement =",st); - this.progressHandlerCount = 0; + let rc; try { T.assert(wasm.isPtr(st.pointer)) .mustThrowMatching(()=>st.pointer=1, /read-only/) @@ -1227,10 +1406,12 @@ self.sqlite3InitModule = sqlite3InitModule; capi.sqlite3_stmt_status( st, capi.SQLITE_STMTSTATUS_RUN, 0 ) === 0) - .assert(!st._mayGet) .assert('a' === st.getColumnName(0)) + .mustThrowMatching(()=>st.columnCount=2, + /columnCount property is read-only/) .assert(1===st.columnCount) .assert(0===st.parameterCount) + .assert(0===capi.sqlite3_bind_parameter_count(st)) .mustThrow(()=>st.bind(1,null)) .assert(true===st.step()) .assert(3 === st.get(0)) @@ -1249,25 +1430,23 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(1===st.get(0,capi.SQLITE_BLOB).length) .assert(st.getBlob(0) instanceof Uint8Array) .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) - .assert(st._mayGet) .assert(false===st.step()) - .assert(!st._mayGet) + .mustThrowMatching(()=>st.get(0), + "Stmt.step() has not (recently) returned true.") .assert( capi.sqlite3_stmt_status( st, capi.SQLITE_STMTSTATUS_RUN, 0 ) > 0); - T.assert(this.progressHandlerCount > 0, - "Expecting progress callback."). - assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). - assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). - assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). - assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + T.assert(this.progressHandlerCount>0 + || wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK'), + "Expecting progress callback."); }finally{ - st.finalize(); + rc = st.finalize(); } T.assert(!st.pointer) - .assert(0===this.db.openStatementCount()); + .assert(0===this.db.openStatementCount()) + .assert(0===rc); T.mustThrowMatching(()=>new sqlite3.oo1.Stmt("hi"), function(err){ return (err instanceof sqlite3.SQLite3Error) @@ -1282,7 +1461,6 @@ self.sqlite3InitModule = sqlite3InitModule; if(1){ const vfsList = capi.sqlite3_js_vfs_list(); T.assert(vfsList.length>1); - //log("vfsList =",vfsList); wasm.scopedAllocCall(()=>{ const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); for(const v of vfsList){ @@ -1311,7 +1489,7 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(pVfsDb > 0) .assert(pVfsMem !== pVfsDflt /* memdb lives on top of the default vfs */) - .assert(pVfsDb === pVfsDflt || pVfsdb === pVfsMem) + .assert(pVfsDb === pVfsDflt || pVfsDb === pVfsMem) ; /*const vMem = new capi.sqlite3_vfs(pVfsMem), vDflt = new capi.sqlite3_vfs(pVfsDflt), @@ -1328,12 +1506,13 @@ self.sqlite3InitModule = sqlite3InitModule; const db = this.db; let list = []; this.progressHandlerCount = 0; + //wasm.xWrap.debug = true; let rc = db.exec({ sql:['CREATE TABLE t(a,b);', // ^^^ using TEMP TABLE breaks the db export test "INSERT INTO t(a,b) VALUES(1,2),(3,4),", - "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for - off-by-one bug in string-to-WASM conversion*/], + "(?,?)"/*intentionally missing semicolon to test for + off-by-one bug in string-to-WASM conversion*/], saveSql: list, bind: [5,6] }); @@ -1341,12 +1520,21 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(rc === db) .assert(2 === list.length) .assert('string'===typeof list[1]) - .assert(4===db.changes()) - .assert(this.progressHandlerCount > 0, + .assert(3===db.changes()) + .assert(this.progressHandlerCount > 0 + || wasm.compileOptionUsed('OMIT_PROGRESS_CALLBACK'), "Expecting progress callback.") if(wasm.bigIntEnabled){ - T.assert(4n===db.changes(false,true)); + T.assert(3n===db.changes(false,true)); } + rc = db.exec({ + sql: "INSERT INTO t(a,b) values('blob',X'6869') RETURNING 13", + rowMode: 0 + }); + T.assert(Array.isArray(rc)) + .assert(1===rc.length) + .assert(13 === rc[0]) + .assert(1===db.changes()); let vals = db.selectValues('select a from t order by a limit 2'); T.assert( 2 === vals.length ) @@ -1373,10 +1561,10 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert( 3 === this._myState /* Recall that "this" is the options object. */ + ).assert( + this.columnNames===colNames ).assert( this.columnNames[0]==='a' && this.columnNames[1]==='b' - /* options.columnNames is filled out before the first - Stmt.step(). */ ).assert( (row.a%2 && row.a<6) || 'blob'===row.a ); @@ -1386,6 +1574,14 @@ self.sqlite3InitModule = sqlite3InitModule; .assert('a' === colNames[0]) .assert(4 === counter) .assert(4 === list.length); + colNames = []; + db.exec({ + /* Ensure that columnNames is populated for empty result sets. */ + sql: "SELECT a a, b B FROM t WHERE 0", + columnNames: colNames + }); + T.assert(2===colNames.length) + .assert('a'===colNames[0] && 'B'===colNames[1]); list.length = 0; db.exec("SELECT a a, b b FROM t",{ rowMode: 'array', @@ -1422,7 +1618,7 @@ self.sqlite3InitModule = sqlite3InitModule; rv = db.exec("SELECT 1 WHERE 0",{rowMode: 0}); T.assert(Array.isArray(rv)).assert(0===rv.length); if(wasm.bigIntEnabled && haveWasmCTests()){ - const mI = wasm.xCall('sqlite3_wasm_test_int64_max'); + const mI = wasm.xCall('sqlite3__wasm_test_int64_max'); const b = BigInt(Number.MAX_SAFE_INTEGER * 2); T.assert(b === db.selectValue("SELECT "+b)). assert(b === db.selectValue("SELECT ?", b)). @@ -1438,11 +1634,17 @@ self.sqlite3InitModule = sqlite3InitModule; let st = db.prepare("update t set b=:b where a='blob'"); try { + T.assert(0===st.columnCount) + .assert(1===st.parameterCount) + .assert(1===capi.sqlite3_bind_parameter_count(st)) + .assert( false===st.isReadOnly() ); const ndx = st.getParamIndex(':b'); T.assert(1===ndx); - st.bindAsBlob(ndx, "ima blob").reset(true); + st.bindAsBlob(ndx, "ima blob") + /*step() skipped intentionally*/.reset(true); } finally { - st.finalize(); + T.assert(0===st.finalize()) + .assert(undefined===st.finalize()); } try { @@ -1469,6 +1671,7 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// .t({ name: "sqlite3_set_authorizer()", + predicate: ()=>!!wasm.exports.sqlite3_set_authorizer || "Missing sqlite3_set_authorizer()", test:function(sqlite3){ T.assert(capi.SQLITE_IGNORE>0) .assert(capi.SQLITE_DENY>0); @@ -1604,8 +1807,20 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(3===rc[1].a) .assert(4===rc[1].b); }) - //////////////////////////////////////////////////////////////////////// + .t('selectArray/Object/Values() via INSERT/UPDATE...RETURNING', function(sqlite3){ + let rc = this.db.selectObject("INSERT INTO t(a,b) VALUES(83,84) RETURNING a as AA"); + T.assert(83===rc.AA); + rc = this.db.selectArray("UPDATE T set a=85 WHERE a=83 RETURNING b as BB"); + T.assert(Array.isArray(rc)).assert(84===rc[0]); + //log("select * from t:",this.db.selectObjects("select * from t order by a")); + rc = this.db.selectValues("UPDATE T set a=a*1 RETURNING a"); + T.assert(Array.isArray(rc)) + .assert(5 === rc.length) + .assert('number'===typeof rc[0]) + .assert(rc[0]|0 === rc[0] /* is small integer */); + }) + //////////////////////////////////////////////////////////////////////// .t({ name: 'sqlite3_js_db_export()', predicate: ()=>true, @@ -1619,13 +1834,12 @@ self.sqlite3InitModule = sqlite3InitModule; } }/*sqlite3_js_db_export()*/) .t({ - name: 'sqlite3_js_vfs_create_file() with db in default VFS', + name: 'sqlite3_js_posix_create_file()', predicate: ()=>true, test: function(sqlite3){ const db = this.db; - const pVfs = capi.sqlite3_js_db_vfs(db); - const filename = "sqlite3_js_vfs_create_file().db"; - capi.sqlite3_js_vfs_create_file(pVfs, filename, this.dbExport); + const filename = "sqlite3_js_posix_create_file.db"; + capi.sqlite3_js_posix_create_file(filename, this.dbExport); delete this.dbExport; const db2 = new sqlite3.oo1.DB(filename,'r'); try { @@ -1634,17 +1848,27 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(n>0 && db2.selectValue(sql) === n); }finally{ db2.close(); - wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); + sqlite3.util.sqlite3__wasm_vfs_unlink(0, filename); } } - }/*sqlite3_js_vfs_create_file()*/) + }/*sqlite3_js_posix_create_file()*/) //////////////////////////////////////////////////////////////////// .t({ name:'Scalar UDFs', test: function(sqlite3){ const db = this.db; - db.createFunction("foo",(pCx,a,b)=>a+b); + db.createFunction( + "foo", + 1 ? (pCx,a,b)=>a+b + : (pCx,a,b)=>{ + /*return sqlite3.capi.sqlite3_result_error_js( + db, sqlite3.capi.SQLITE_ERROR, "foo???" + );*/ + console.debug("foo UDF", pCx, a, b); + return Number(a)+Number(b); + } + ); T.assert(7===db.selectValue("select foo(3,4)")). assert(5===db.selectValue("select foo(3,?)",2)). assert(5===db.selectValue("select foo(?,?2)",[1,4])). @@ -1851,7 +2075,9 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////// .t({ name: 'Window UDFs', - //predicate: ()=>false, + predicate: (sqlite3)=>!!sqlite3.wasm.exports.sqlite3_create_window_function + /*!sqlite3.wasm.compileOptionUsed('OMIT_WINDOWFUNC')*/ + || "Missing window functions", test: function(){ /* Example window function, table, and results taken from: https://sqlite.org/windowfunctions.html#udfwinfunc */ @@ -1931,7 +2157,7 @@ self.sqlite3InitModule = sqlite3InitModule; }/*window UDFs*/) //////////////////////////////////////////////////////////////////// - .t("ATTACH", function(){ + .t("ATTACH", function(sqlite3){ const db = this.db; const resultRows = []; db.exec({ @@ -1982,7 +2208,7 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(wasm.isPtr(pVoid)) .assert(wasm.isPtr(aVals)) .assert(wasm.isPtr(aCols)) - .assert(+wasm.cstrToJs(wasm.peekPtr(aVals + wasm.ptrSizeof)) + .assert(+wasm.cstrToJs(wasm.peekPtr(wasm.ptr.add(aVals, wasm.ptr.size))) === 2 * +wasm.cstrToJs(wasm.peekPtr(aVals))); return 0; }); @@ -2010,6 +2236,44 @@ self.sqlite3InitModule = sqlite3InitModule; db.exec("detach foo"); T.mustThrow(()=>db.exec("select * from foo.bar"), "Because foo is no longer attached."); + + /* SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE/WRITE... */ + const db2 = new sqlite3.oo1.DB(); + try{ + capi.sqlite3_db_config(db2, capi.SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE, 0, null); + T.mustThrow(()=>db2.exec("attach 'attached.db' as foo"), + "Cannot create a new db via ATTACH"); + capi.sqlite3_db_config(db2, capi.SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE, 1, null); + db2.exec([ + "attach 'attached.db' as foo;", + "create table foo.t(a);", + "insert into foo.t(a) values(1);", + "detach foo;" + ]); + capi.sqlite3_db_config(db2, capi.SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE, 0, null); + db2.exec("attach 'attached.db' as foo"); + T.mustThrow(()=>db2.exec("insert into foo.t(a) values(2)"), + "ATTACH_WRITE is false"); + capi.sqlite3_db_config(db2, capi.SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE, 1, null); + db2.exec([ + "detach foo;", + "attach 'attached.db' as foo;", + "insert into foo.t(a) values(2);", + "drop table foo.t;", + "detach foo" + ]); + }finally{ + db2.close(); + } + })/*ATTACH tests*/ + + //////////////////////////////////////////////////////////////////// + .t("Read-only", function(sqlite3){ + T.assert( 0===capi.sqlite3_db_readonly(this.db, "main") ); + const db = new sqlite3.oo1.DB('file://'+this.db.filename+'?mode=ro'); + T.assert( 1===capi.sqlite3_db_readonly(db, "main") ); + T.assert( -1===capi.sqlite3_db_readonly(db, "nope") ); + db.close(); }) //////////////////////////////////////////////////////////////////// @@ -2024,7 +2288,7 @@ self.sqlite3InitModule = sqlite3InitModule; try{ ptrInt = w.scopedAlloc(4); w.poke32(ptrInt,origValue); - const cf = w.xGet('sqlite3_wasm_test_intptr'); + const cf = w.xGet('sqlite3__wasm_test_intptr'); const oldPtrInt = ptrInt; T.assert(origValue === w.peek32(ptrInt)); const rc = cf(ptrInt); @@ -2039,29 +2303,29 @@ self.sqlite3InitModule = sqlite3InitModule; const v64 = ()=>w.peek64(pi64) T.assert(v64() == o64); //T.assert(o64 === w.peek64(pi64)); - const cf64w = w.xGet('sqlite3_wasm_test_int64ptr'); + const cf64w = w.xGet('sqlite3__wasm_test_int64ptr'); cf64w(pi64); T.assert(v64() == BigInt(2 * o64)); cf64w(pi64); T.assert(v64() == BigInt(4 * o64)); - const biTimes2 = w.xGet('sqlite3_wasm_test_int64_times2'); + const biTimes2 = w.xGet('sqlite3__wasm_test_int64_times2'); T.assert(BigInt(2 * o64) === biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError in the call :/ */)); const pMin = w.scopedAlloc(16); - const pMax = pMin + 8; + const pMax = w.ptr.add(pMin, 8); const g64 = (p)=>w.peek64(p); w.poke64([pMin, pMax], 0); const minMaxI64 = [ - w.xCall('sqlite3_wasm_test_int64_min'), - w.xCall('sqlite3_wasm_test_int64_max') + w.xCall('sqlite3__wasm_test_int64_min'), + w.xCall('sqlite3__wasm_test_int64_max') ]; T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); - w.xCall('sqlite3_wasm_test_int64_minmax', pMin, pMax); + w.xCall('sqlite3__wasm_test_int64_minmax', pMin, pMax); T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). assert(g64(pMax) === minMaxI64[1], "int64 mismatch"); //log("pMin",g64(pMin), "pMax",g64(pMax)); @@ -2081,7 +2345,7 @@ self.sqlite3InitModule = sqlite3InitModule; "back into JS because of the lack of 64-bit integer support."); } }finally{ - const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); + //const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); //log("x=",x,"y=",y,"z=",z); // just looking at the alignment w.scopedAllocPop(stack); } @@ -2091,7 +2355,7 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t({ name: 'virtual table #1: eponymous w/ manual exception handling', - predicate: ()=>!!capi.sqlite3_index_info, + predicate: (sqlite3)=>(!!sqlite3.capi.sqlite3_vtab || "Missing vtab support"), test: function(sqlite3){ const VT = sqlite3.vtab; const tmplCols = Object.assign(Object.create(null),{ @@ -2102,18 +2366,20 @@ self.sqlite3InitModule = sqlite3InitModule; ext/misc/templatevtab.c. */ const tmplMod = new sqlite3.capi.sqlite3_module(); - T.assert(0===tmplMod.$xUpdate); + T.assert(!tmplMod.$xUpdate); + const dbg = 1 ? ()=>{} : sqlite3.config.debug; + //tmplMod.debugFlags(0x03); tmplMod.setupModule({ catchExceptions: false, methods: { xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + dbg("xConnect",...arguments); try{ const args = wasm.cArgvToJs(argc, argv); T.assert(args.length>=3) .assert(args[0] === 'testvtab') .assert(args[1] === 'main') .assert(args[2] === 'testvtab'); - //console.debug("xConnect() args =",args); const rc = capi.sqlite3_declare_vtab( pDb, "CREATE TABLE ignored(a,b)" ); @@ -2132,6 +2398,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, xCreate: true /* just for testing. Will be removed afterwards. */, xDisconnect: function(pVtab){ + dbg("xDisconnect",...arguments); try { VT.xVtab.unget(pVtab).dispose(); return 0; @@ -2140,6 +2407,7 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xOpen: function(pVtab, ppCursor){ + dbg("xOpen",...arguments); try{ const t = VT.xVtab.get(pVtab), c = VT.xCursor.create(ppCursor); @@ -2152,6 +2420,7 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xClose: function(pCursor){ + dbg("xClose",...arguments); try{ const c = VT.xCursor.unget(pCursor); T.assert(c instanceof capi.sqlite3_vtab_cursor) @@ -2163,6 +2432,7 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xNext: function(pCursor){ + dbg("xNext",...arguments); try{ const c = VT.xCursor.get(pCursor); ++c._rowId; @@ -2172,6 +2442,7 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xColumn: function(pCursor, pCtx, iCol){ + dbg("xColumn",...arguments); try{ const c = VT.xCursor.get(pCursor); switch(iCol){ @@ -2189,6 +2460,7 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xRowid: function(pCursor, ppRowid64){ + dbg("xRowid",...arguments); try{ const c = VT.xCursor.get(pCursor); VT.xRowid(ppRowid64, c._rowId); @@ -2198,12 +2470,14 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xEof: function(pCursor){ + dbg("xEof",...arguments); const c = VT.xCursor.get(pCursor), rc = c._rowId>=10; return rc; }, xFilter: function(pCursor, idxNum, idxCStr, argc, argv/* [sqlite3_value* ...] */){ + dbg("xFilter",...arguments); try{ const c = VT.xCursor.get(pCursor); c._rowId = 0; @@ -2216,6 +2490,7 @@ self.sqlite3InitModule = sqlite3InitModule; } }, xBestIndex: function(pVtab, pIdxInfo){ + dbg("xBestIndex",...arguments); try{ //const t = VT.xVtab.get(pVtab); const sii = capi.sqlite3_index_info; @@ -2264,12 +2539,13 @@ self.sqlite3InitModule = sqlite3InitModule; } }); this.db.onclose.disposeAfter.push(tmplMod); - T.assert(0===tmplMod.$xUpdate) - .assert(tmplMod.$xCreate) + T.assert(!tmplMod.$xUpdate) + .assert(wasm.isPtr(tmplMod.$xRowid)) + .assert(wasm.isPtr(tmplMod.$xCreate)) .assert(tmplMod.$xCreate === tmplMod.$xConnect, "setup() must make these equivalent and "+ "installMethods() must avoid re-compiling identical functions"); - tmplMod.$xCreate = 0 /* make tmplMod eponymous-only */; + tmplMod.$xCreate = wasm.ptr.null /* make tmplMod eponymous-only */; let rc = capi.sqlite3_create_module( this.db, "testvtab", tmplMod, 0 ); @@ -2288,7 +2564,7 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t({ name: 'virtual table #2: non-eponymous w/ automated exception wrapping', - predicate: ()=>!!capi.sqlite3_index_info, + predicate: (sqlite3)=>!!sqlite3.capi.sqlite3_vtab || "Missing vtab support", test: function(sqlite3){ const VT = sqlite3.vtab; const tmplCols = Object.assign(Object.create(null),{ @@ -2509,7 +2785,7 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// .t('Close db', function(){ T.assert(this.db).assert(wasm.isPtr(this.db.pointer)); - //wasm.sqlite3_wasm_db_reset(this.db); // will leak virtual tables! + //wasm.sqlite3__wasm_db_reset(this.db); // will leak virtual tables! this.db.close(); T.assert(!this.db.pointer); }) @@ -2534,9 +2810,9 @@ self.sqlite3InitModule = sqlite3InitModule; test: function(sqlite3){ const filename = this.kvvfsDbFile = 'session'; const pVfs = capi.sqlite3_vfs_find('kvvfs'); - T.assert(pVfs); + T.assert(looksLikePtr(pVfs)); const JDb = this.JDb = sqlite3.oo1.JsStorageDb; - const unlink = this.kvvfsUnlink = ()=>{JDb.clearStorage(filename)}; + const unlink = this.kvvfsUnlink = ()=>JDb.clearStorage(this.kvvfsDbFile); unlink(); let db = new JDb(filename); try { @@ -2554,152 +2830,82 @@ self.sqlite3InitModule = sqlite3InitModule; } } }/*kvvfs sanity checks*/) +//#if enable-see .t({ - name: 'kvvfs sqlite3_js_vfs_create_file()', - predicate: ()=>"kvvfs does not currently support this", + name: 'kvvfs with SEE encryption', + predicate: ()=>(isUIThread() + || "Only available in main thread."), test: function(sqlite3){ - let db; - try { - db = new this.JDb(this.kvvfsDbFile); - const exp = capi.sqlite3_js_db_export(db); - db.close(); - this.kvvfsUnlink(); - capi.sqlite3_js_vfs_create_file("kvvfs", this.kvvfsDbFile, exp); - db = new this.JDb(filename); - T.assert(6 === db.selectValue('select count(*) from kvvfs')); - }finally{ - db.close(); - this.kvvfsUnlink(); - } - delete this.kvvfsDbFile; - delete this.kvvfsUnlink; - delete this.JDb; + this.kvvfsUnlink(); + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: this.kvvfsDbFile + //vfs: 'kvvfs' + //,flags: 'ct' + }; + try { + if (initDb) { + initDb = false; + db = new this.JDb({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new this.JDb(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("(should not be reached) sessionStorage =", sessionStorage); + } catch (e) { + err = e; + } finally { + db.close() + } + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + //console.debug('tryKey()',arguments); + db = new sqlite3.oo1.DB({ + ...ctoropt, + vfs: 'kvvfs', + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }.bind(this); + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + this.kvvfsUnlink(); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + this.kvvfsUnlink(); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + this.kvvfsUnlink(); } - }/*kvvfs sqlite3_js_vfs_create_file()*/) + })/*kvvfs with SEE*/ +//#endif enable-see ;/* end kvvfs tests */ - //////////////////////////////////////////////////////////////////////// - T.g('OPFS: Origin-Private File System', - (sqlite3)=>(sqlite3.opfs - ? true : "requires Worker thread in a compatible browser")) - .t({ - name: 'OPFS db sanity checks', - test: async function(sqlite3){ - const filename = this.opfsDbFile = 'sqlite3-tester1.db'; - const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs'); - T.assert(pVfs); - const unlink = this.opfsUnlink = - (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)}; - unlink(); - let db = new sqlite3.oo1.OpfsDb(filename); - try { - db.exec([ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]); - T.assert(3 === db.selectValue('select count(*) from p')); - db.close(); - db = new sqlite3.oo1.OpfsDb(filename); - db.exec('insert into p(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from p')); - this.opfsDbExport = capi.sqlite3_js_db_export(db); - T.assert(this.opfsDbExport instanceof Uint8Array) - .assert(this.opfsDbExport.byteLength>0 - && 0===this.opfsDbExport.byteLength % 512); - }finally{ - db.close(); - unlink(); - } - } - }/*OPFS db sanity checks*/) - .t({ - name: 'OPFS export/import', - test: async function(sqlite3){ - let db; - try { - const exp = this.opfsDbExport; - delete this.opfsDbExport; - capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp); - const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')); - }finally{ - if(db) db.close(); - } - } - }/*OPFS export/import*/) - .t({ - name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()', - test: async function(sqlite3){ - const filename = this.opfsDbFile; - const pVfs = this.opfsVfs; - const unlink = this.opfsUnlink; - T.assert(filename && pVfs && !!unlink); - delete this.opfsDbFile; - delete this.opfsVfs; - delete this.opfsUnlink; - unlink(); - // Sanity-test sqlite3_js_vfs_create_file()... - /************************************************************** - ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended - for client-side use. It is only for this project's own - internal use. Its APIs are subject to change or removal at - any time. - ***************************************************************/ - const opfs = sqlite3.opfs; - const fSize = 1379; - let sh; - try{ - T.assert(!(await opfs.entryExists(filename))); - capi.sqlite3_js_vfs_create_file( - pVfs, filename, null, fSize - ); - T.assert(await opfs.entryExists(filename)); - let fh = await opfs.rootDirectory.getFileHandle(filename); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - T.assert(!(await opfs.entryExists(filename))); - - const ba = new Uint8Array([1,2,3,4,5]); - capi.sqlite3_js_vfs_create_file( - "opfs", filename, ba - ); - T.assert(await opfs.entryExists(filename)); - fh = await opfs.rootDirectory.getFileHandle(filename); - sh = await fh.createSyncAccessHandle(); - T.assert(ba.byteLength === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - - T.mustThrowMatching(()=>{ - capi.sqlite3_js_vfs_create_file( - "no-such-vfs", filename, ba - ); - }, "SQLITE_NOTFOUND: Unknown sqlite3_vfs name: no-such-vfs"); - }finally{ - if(sh) await sh.close(); - unlink(); - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); - } - }/*OPFS util sanity checks*/) - ;/* end OPFS tests */ - //////////////////////////////////////////////////////////////////////// T.g('Hook APIs') .t({ @@ -2709,27 +2915,33 @@ self.sqlite3InitModule = sqlite3InitModule; let countCommit = 0, countRollback = 0;; const db = new sqlite3.oo1.DB(':memory:',1 ? 'c' : 'ct'); let rc = capi.sqlite3_commit_hook(db, (p)=>{ + //console.debug("commit hook",arguments); ++countCommit; - return (1 === p) ? 0 : capi.SQLITE_ERROR; - }, 1); - T.assert( 0 === rc /*void pointer*/ ); + return (17 == p) ? 0 : capi.SQLITE_ERROR; + }, 17); + T.assert( wasm.ptr.null === rc ); // Commit hook... + T.assert( 0!=capi.sqlite3_get_autocommit(db) ); db.exec("BEGIN; SELECT 1; COMMIT"); T.assert(0 === countCommit, "No-op transactions (mostly) do not trigger commit hook."); db.exec("BEGIN EXCLUSIVE; SELECT 1; COMMIT"); T.assert(1 === countCommit, "But EXCLUSIVE transactions do."); - db.transaction((d)=>{d.exec("create table t(a)");}); + db.transaction((d)=>{ + T.assert( 0==capi.sqlite3_get_autocommit(db) ); + d.exec("create table t(a)"); + }); T.assert(2 === countCommit); + T.assert(17 == capi.sqlite3_commit_hook(db, 0, 0)); // Rollback hook: rc = capi.sqlite3_rollback_hook(db, (p)=>{ ++countRollback; - T.assert( 2 === p ); - }, 2); - T.assert( 0 === rc /*void pointer*/ ); + T.assert( 21 == p ); + }, 21); + T.assert( wasm.ptr.null===rc ); T.mustThrowMatching(()=>{ db.transaction('drop table t',()=>{}) }, (e)=>{ @@ -2742,16 +2954,18 @@ self.sqlite3InitModule = sqlite3InitModule; sqlite3.SQLite3Error.toss(capi.SQLITE_FULL,'testing rollback hook'); }); }, (e)=>{ + //console.error("transaction error:",e); return capi.SQLITE_FULL === e.resultCode }); T.assert(1 === countRollback); + T.assert(21 == capi.sqlite3_rollback_hook(db, 0, 0)); // Update hook... const countUpdate = Object.create(null); capi.sqlite3_update_hook(db, (p,op,dbName,tbl,rowid)=>{ T.assert('main' === dbName.toLowerCase()) .assert('t' === tbl.toLowerCase()) - .assert(3===p) + .assert(33==p) .assert('bigint' === typeof rowid); switch(op){ case capi.SQLITE_INSERT: @@ -2761,38 +2975,36 @@ self.sqlite3InitModule = sqlite3InitModule; break; default: toss("Unexpected hook operator:",op); } - }, 3); + }, 33); db.transaction((d)=>{ - d.exec([ + db.exec([ "insert into t(a) values(1);", "update t set a=2;", "update t set a=3;", - "delete from t where a=3" + "delete from t where a=3;" // update hook is not called for an unqualified DELETE ]); }); T.assert(1 === countRollback) - .assert(3 === countCommit) + .assert(2 === countCommit) .assert(1 === countUpdate[capi.SQLITE_INSERT]) .assert(2 === countUpdate[capi.SQLITE_UPDATE]) .assert(1 === countUpdate[capi.SQLITE_DELETE]); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; - T.assert(1 === capi.sqlite3_commit_hook(db, 0, 0)); - T.assert(2 === capi.sqlite3_rollback_hook(db, 0, 0)); - T.assert(3 === capi.sqlite3_update_hook(db, 0, 0)); + T.assert(33 == capi.sqlite3_update_hook(db, 0, 0)); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = false; db.close(); } })/* commit/rollback/update hooks */ .t({ name: "sqlite3_preupdate_hook()", - predicate: ()=>wasm.bigIntEnabled || "Pre-update hook requires int64", + predicate: ()=>capi.sqlite3_preupdate_hook || "Missing pre-update hook API", test: function(sqlite3){ const db = new sqlite3.oo1.DB(':memory:', 1 ? 'c' : 'ct'); const countHook = Object.create(null); let rc = capi.sqlite3_preupdate_hook( db, function(p, pDb, op, zDb, zTbl, iKey1, iKey2){ - T.assert(9 === p) + T.assert(9 == p) .assert(db.pointer === pDb) .assert(1 === capi.sqlite3_preupdate_count(pDb)) .assert( 0 > capi.sqlite3_preupdate_blobwrite(pDb) ); @@ -2810,6 +3022,7 @@ self.sqlite3InitModule = sqlite3InitModule; }, 9 ); + T.assert( 0==rc ); db.transaction((d)=>{ d.exec([ "create table t(a);", @@ -2823,8 +3036,10 @@ self.sqlite3InitModule = sqlite3InitModule; .assert(2 === countHook[capi.SQLITE_UPDATE]) .assert(1 === countHook[capi.SQLITE_DELETE]); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = true; - db.close(); + T.assert( !!capi.sqlite3_preupdate_hook(db, 0, 0) ); //wasm.xWrap.FuncPtrAdapter.debugFuncInstall = false; + T.assert( !capi.sqlite3_preupdate_hook(db, 0, 0) ); + db.close(); } })/*pre-update hooks*/ ;/*end hook API tests*/ @@ -2857,9 +3072,9 @@ self.sqlite3InitModule = sqlite3InitModule; T.g('Session API') .t({ name: 'Session API sanity checks', - predicate: ()=>!!capi.sqlite3changegroup_add, + predicate: ()=>!!capi.sqlite3changegroup_add || "Missing session API", test: function(sqlite3){ - warn("The session API tests could use some expansion."); + //warn("The session API tests could use some expansion."); const db1 = new sqlite3.oo1.DB(), db2 = new sqlite3.oo1.DB(); const sqlInit = [ "create table t(rowid INTEGER PRIMARY KEY,a,b); ", @@ -2894,7 +3109,9 @@ self.sqlite3InitModule = sqlite3InitModule; .assert('b4' === db1.selectValue('select b from t where rowid=4')) .assert(3 === db1.selectValue('select count(*) from t')); - const testSessionEnable = false; + const testSessionEnable = + false /* it's not yet clear whether these test failures are + broken tests or broken bindings. */; if(testSessionEnable){ rc = capi.sqlite3session_enable(pSession, 0); T.assert( 0 === rc ) @@ -2905,8 +3122,7 @@ self.sqlite3InitModule = sqlite3InitModule; .assert( capi.sqlite3session_enable(pSession, -1) > 0 ) .assert(undefined === db1.selectValue('select a from t where rowid=2')); }else{ - warn("sqlite3session_enable() tests disabled due to unexpected results.", - "(Possibly a tester misunderstanding, as opposed to a bug.)"); + //warn("sqlite3session_enable() tests are currently disabled."); } let db1Count = db1.selectValue("select count(*) from t"); T.assert( db1Count === (testSessionEnable ? 2 : 3) ); @@ -2971,17 +3187,780 @@ self.sqlite3InitModule = sqlite3InitModule; })/*session API sanity tests*/ ;/*end of session API group*/; + //////////////////////////////////////////////////////////////////////// + T.g('OPFS: Origin-Private File System', + (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") + || 'requires "opfs" VFS')) + .t({ + name: 'OPFS db sanity checks', + test: async function(sqlite3){ + T.assert(capi.sqlite3_vfs_find('opfs')); + const opfs = sqlite3.opfs; + const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; + const fileUri = 'file://'+filename+'?delete-before-open=1'; + const initSql = [ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]; + let db = new sqlite3.oo1.OpfsDb(fileUri); + try { + db.exec(initSql); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new sqlite3.oo1.OpfsDb(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); + }finally{ + db.close(); + } + T.assert(await opfs.entryExists(filename)); + try { + db = new sqlite3.oo1.OpfsDb(fileUri); + db.exec(initSql) /* will throw if delete-before-open did not work */; + T.assert(3 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + } + }/*OPFS db sanity checks*/) + .t({ + name: 'OPFS import', + test: async function(sqlite3){ + let db; + const filename = this.opfsDbFile; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp); + db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + db.close(); + this.opfsUnlink = + (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn); + this.opfsUnlink(filename); + T.assert(!(await sqlite3.opfs.entryExists(filename))); + // Try again with a function as an input source: + let cursor = 0; + const blockSize = 512, end = exp.byteLength; + const reader = async function(){ + if(cursor >= exp.byteLength){ + return undefined; + } + const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); + cursor += blockSize; + return rv; + }; + this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader); + db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + }finally{ + if(db) db.close(); + } + } + }/*OPFS export/import*/) + .t({ + name: '(Internal-use) OPFS utility APIs', + test: async function(sqlite3){ + const filename = this.opfsDbFile; + const unlink = this.opfsUnlink; + T.assert(filename && !!unlink); + delete this.opfsDbFile; + delete this.opfsUnlink; + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = this.opfsImportSize; + delete this.opfsImportSize; + let sh; + try{ + T.assert(await opfs.entryExists(filename)); + const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); + const fh = await dirHandle.getFileHandle(filenamePart); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*OPFS util sanity checks*/) +//#if enable-see + .t({ + name: 'OPFS with SEE encryption', + test: function(sqlite3){ + const dbFile = 'file:///sqlite3-see.edb'; + const dbCtor = sqlite3.oo1.OpfsDb; + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: dbFile, + flags: 'c' + }; + try { + if (initDb) { + initDb = false; + const opt = { + ...ctoropt, + [keyKey]: key + }; + opt.filename += '?delete-before-open=1'; + db = new dbCtor(opt); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new dbCtor(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + const rv = db.exec({ + sql:"select count(*) from sqlite_schema", + returnValue: 'resultRows' + }); + console.warn("(should not be reached) rv =",rv); + } catch (e) { + err = e; + } finally { + db.close() + } + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + db = new dbCtor({ + ...ctoropt, + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }; + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + } + })/*OPFS with SEE*/ +//#endif enable-see + ;/* end OPFS tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS SyncAccessHandle Pool VFS', + (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) + .t({ + name: 'SAH sanity checks', + test: async function(sqlite3){ + T.assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) < 0) + const inst = sqlite3.installOpfsSAHPoolVfs, + catcher = (e)=>{ + error("Cannot load SAH pool VFS.", + "This might not be a problem,", + "depending on the environment."); + return false; + }; + let u1, u2; + // Ensure that two immediately-consecutive installations + // resolve to the same Promise instead of triggering + // a locking error. + const P1 = inst(sahPoolConfig).then(u=>u1 = u).catch(catcher), + P2 = inst(sahPoolConfig).then(u=>u2 = u).catch(catcher); + await Promise.all([P1, P2]); + if(!(await P1)) return; + T.assert(u1 === u2) + .assert(sahPoolConfig.name === u1.vfsName) + .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) + .assert(u1.getCapacity() >= sahPoolConfig.initialCapacity + /* If a test fails before we get to nuke the VFS, we + can have more than the initial capacity on the next + run. */) + .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2))) + .assert(2 === (await u2.reduceCapacity(2))) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0); + + T.assert(0 === u1.getFileCount()); + const dbName = '/foo.db'; + //wasm.xWrap.debug = true; + let db = new u1.OpfsSAHPoolDb(dbName); + T.assert(db instanceof sqlite3.oo1.DB) + .assert(1 === u1.getFileCount()); + db.exec([ + 'pragma locking_mode=exclusive;', + 'pragma journal_mode=wal;' + /* WAL mode only works in this VFS if locking_mode=exclusive + is invoked prior to the first db access, as this build + does not have the shared-memory APIs needed for WAL without + exclusive-mode locking. See: + + https://sqlite.org/wal.html#use_of_wal_without_shared_memory + + Note that WAL mode here DOES NOT add any concurrency capabilities + to this VFS, but it MAY provide slightly improved performance + over the other journaling modes. + */, + 'create table t(a);', + 'insert into t(a) values(1),(2),(3)' + ]); + T.assert(2 === u1.getFileCount() /* one is the journal file */) + .assert(3 === db.selectValue('select count(*) from t')) + .assert( + 'wal'===db.selectValue('pragma journal_mode') + || wasm.compileOptionUsed('OMIT_WAL') + ); + db.close(); + T.assert(1 === u1.getFileCount()); + db = new u2.OpfsSAHPoolDb(dbName); + T.assert(1 === u1.getFileCount()) + .mustThrowMatching( + ()=>u1.pauseVfs(), + (err)=>{ + return capi.SQLITE_MISUSE===err.resultCode + && /^SQLITE_MISUSE: Cannot pause VFS /.test(err.message); + }, + "Cannot pause VFS with opened db." + ); + db.close(); + T.assert( u2===u2.pauseVfs() ) + .assert( u2.isPaused() ) + .assert( !capi.sqlite3_vfs_find(u2.vfsName) ) + .mustThrowMatching(()=>new u2.OpfsSAHPoolDb(dbName), + /.+no such vfs: .+/, + "VFS is not available") + .assert( u2===await u2.unpauseVfs() ) + .assert( u2===await u1.unpauseVfs(), "unpause is a no-op if the VFS is not paused" ) + .assert( !!capi.sqlite3_vfs_find(u2.vfsName) ); + const fileNames = u1.getFileNames(); + T.assert(1 === fileNames.length) + .assert(dbName === fileNames[0]) + .assert(1 === u1.getFileCount()) + + if(1){ // test exportFile() and importDb() + const dbytes = u1.exportFile(dbName); + T.assert(dbytes.length >= 4096); + const dbName2 = '/exported.db'; + let nWrote = u1.importDb(dbName2, dbytes); + T.assert( 2 == u1.getFileCount() ) + .assert( dbytes.byteLength == nWrote ); + let db2 = new u1.OpfsSAHPoolDb(dbName2); + T.assert(db2 instanceof sqlite3.oo1.DB) + .assert('wal' !== db2.selectValue("pragma journal_mode") + /* importDb() unsets the WAL-mode header for + historical reasons. Because clients must + explicitly enable pragma locking_mode=exclusive + before using WAL, that behavior is retained. */) + .assert(3 === db2.selectValue('select count(*) from t')); + db2.close(); + T.assert(true === u1.unlink(dbName2)) + .assert(false === u1.unlink(dbName2)) + .assert(1 === u1.getFileCount()) + .assert(1 === u1.getFileNames().length); + // Try again with a function as an input source: + let cursor = 0; + const blockSize = 1024, end = dbytes.byteLength; + const reader = async function(){ + if(cursor >= dbytes.byteLength){ + return undefined; + } + const rv = dbytes.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); + cursor += blockSize; + return rv; + }; + nWrote = await u1.importDb(dbName2, reader); + T.assert( 2 == u1.getFileCount() ); + db2 = new u1.OpfsSAHPoolDb(dbName2); + T.assert(db2 instanceof sqlite3.oo1.DB) + .assert(3 === db2.selectValue('select count(*) from t')); + db2.close(); + T.assert(true === u1.unlink(dbName2)) + .assert(dbytes.byteLength == nWrote); + } + + T.assert(true === u1.unlink(dbName)) + .assert(false === u1.unlink(dbName)) + .assert(0 === u1.getFileCount()) + .assert(0 === u1.getFileNames().length); + + // Demonstrate that two SAH pools can coexist so long as + // they have different names. + const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); + conf2.name += '-test2'; + const POther = await inst(conf2); + //log("Installed second SAH instance as",conf2.name); + T.assert(0 === POther.getFileCount()) + .assert(true === await POther.removeVfs()); + + if(0){ + /* Enable this block to inspect vfs's contents via the dev + console or OPFS Explorer browser extension. The + following bits will remove them. */ + return; + } + T.assert(true === await u2.removeVfs()) + .assert(false === await u1.removeVfs()) + .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); + + let cErr, u3; + conf2.$testThrowPhase2 = new Error("Testing throwing during init."); + conf2.name = sahPoolConfig.name+'-err'; + const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); + T.assert(P3 === conf2.$testThrowPhase2) + .assert(cErr === P3) + .assert(undefined === u3) + .assert(!sqlite3.capi.sqlite3_vfs_find(conf2.name)); + delete conf2.$testThrowPhase2; + T.assert(cErr === await inst(conf2).catch(e=>e), + "Init result is cached even if it failed"); + + /* Ensure that the forceReinitIfPreviouslyFailed fallback bypasses + the VFS init cache... */ + cErr = u3 = undefined; + conf2.forceReinitIfPreviouslyFailed = true; + conf2.verbosity = 3; + const P3b = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); + T.assert(undefined === cErr) + .assert(P3b === u3) + .assert(P3b === await inst(conf2)) + .assert(true === await u3.removeVfs()) + .assert(false === await P3b.removeVfs()); + } + }/*OPFS SAH Pool sanity checks*/) +//#if enable-see + .t({ + name: 'OPFS SAHPool with SEE encryption', + test: async function(sqlite3){ + const inst = sqlite3.installOpfsSAHPoolVfs, + catcher = (e)=>{ + error("Cannot load SAH pool VFS.", + "This might not be a problem,", + "depending on the environment."); + return false; + }; + const poolConfig = { + name: 'opfs-sahpool-see', + clearOnInit: true, + initialCapacity: 6 + } + let poolUtil; + const P1 = await inst(poolConfig).then(u=>poolUtil = u).catch(catcher); + const dbFile = '/sqlite3-see.edb'; + const dbCtor = poolUtil.OpfsSAHPoolDb; + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: dbFile, + flags: 'c' + }; + try { + if (initDb) { + initDb = false; + poolUtil.unlink(dbFile); + db = new dbCtor({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new dbCtor(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + const rv = db.exec({ + sql:"select count(*) from sqlite_schema", + returnValue: 'resultRows' + }); + console.warn("(should not be reached) rv =",rv); + } catch (e) { + err = e; + } finally { + db.close() + } + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + db = new dbCtor({ + ...ctoropt, + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }; + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + poolUtil.removeVfs(); + } + })/*opfs-sahpool with SEE*/ +//#endif enable-see + ; + + //////////////////////////////////////////////////////////////////////// + T.g('Misc. APIs') + .t('bind_parameter_...', function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec("create table t(a)"); + const stmt = db.prepare("insert into t(a) values($a)"); + T.assert( 1===capi.sqlite3_bind_parameter_count(stmt) ) + .assert( 1===stmt.parameterCount ) + .assert( 1===capi.sqlite3_bind_parameter_index(stmt, "$a") ) + .assert( 0===capi.sqlite3_bind_parameter_index(stmt, ":a") ) + .assert( 1===stmt.getParamIndex("$a") ) + .assert( 0===stmt.getParamIndex(":a") ) + .assert( "$a"===capi.sqlite3_bind_parameter_name(stmt, 1) ) + .assert( null===capi.sqlite3_bind_parameter_name(stmt, 0) ) + .assert( "$a"===stmt.getParamName(1) ) + .assert( null===stmt.getParamName(0) ); + stmt.finalize(); + db.close(); + }) + + /** + Ensure that certain Stmt members throw when called + via DB.exec(). + */ + .t('locked-by-exec() APIs', function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec("create table t(a);insert into t(a) values(1);"); + let checkCount = 0; + const checkOp = function(op){ + ++checkCount; + T.mustThrowMatching(() => { + db.exec({ + sql: "select ?1", + bind: op, + callback: (row, stmt) => { + switch (row[0]) { + case 'bind': stmt.bind(1); break; + case 'finalize': + case 'clearBindings': + case 'reset': + case 'step': stmt[op](); break; + } + } + }); + }, /^Operation is illegal when statement is locked.*/) + }; + try{ + checkOp('bind'); + checkOp('finalize'); + checkOp('clearBindings'); + checkOp('reset'); + checkOp('step'); + T.assert(5===checkCount); + }finally{ + db.close(); + } + }) + + //////////////////////////////////////////////////////////////////// + .t("Misc. stmt_...", function(sqlite3){ + const db = new sqlite3.oo1.DB(); + db.exec("create table t(a doggiebiscuits); insert into t(a) values(123)"); + const stmt = db.prepare("select a, a+1 from t"); + T.assert( stmt.isReadOnly() ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); + let n = 0; + while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ + ++n; + T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), + "Because stmt is busy" ) + .assert( capi.sqlite3_stmt_busy(stmt) ) + .assert( stmt.isBusy() ) + .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) + .assert( true===stmt.isReadOnly() ); + const sv = capi.sqlite3_column_value(stmt, 0); + T.assert( 123===capi.sqlite3_value_int(sv) ) + .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) + .assert( null===capi.sqlite3_column_decltype(stmt,1) ); + } + T.assert( 1===n ) + .assert( 0===capi.sqlite3_stmt_busy(stmt) ) + .assert( !stmt.isBusy() ); + + if( wasm.exports.sqlite3_column_origin_name ){ + log("Column metadata APIs enabled"); + T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) + .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) + .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) + }else{ + log("Column metadata APIs not enabled"); + } // column metadata APIs + + stmt.finalize(); + db.close(); + }) + + //////////////////////////////////////////////////////////////////// + .t("interrupt", function(sqlite3){ + const db = new sqlite3.oo1.DB(); + T.assert( 0===capi.sqlite3_is_interrupted(db) ); + capi.sqlite3_interrupt(db); + T.assert( 0!==capi.sqlite3_is_interrupted(db) ); + db.close(); + }) + + //////////////////////////////////////////////////////////////////// + .t("sqlite3_set_errmsg()", function(sqlite3){ + /* Added in 3.51.0 */ + const db = new sqlite3.oo1.DB();//(':memory:','wt'); + try{ + const capi = sqlite3.capi; + const sse = capi.sqlite3_set_errmsg, + sec = capi.sqlite3_errcode, + sem = capi.sqlite3_errmsg; + T.assert( 0===sec(db) ) + .assert( "not an error"===sem(db) ); + let rc = sse(db, capi.SQLITE_RANGE, "nope"); + T.assert( 0==rc ) + .assert( capi.SQLITE_RANGE===sec(db) ) + .assert( "nope"===sem(db) ); + rc = sse(0, 0, 0); + T.assert( capi.SQLITE_MISUSE===rc ); + rc = sse(db, 0, 0); + T.assert( 0===rc ) + .assert( 0===sec(db) ) + .assert( "not an error"===sem(db) ); + }finally{ + db.close(); + } + }); + ; + + //////////////////////////////////////////////////////////////////// + T.g('Bug Reports') + .t({ + name: 'Delete via bound parameter in subquery', + predicate: ()=>wasm.compileOptionUsed('ENABLE_FTS5') || "Missing FTS5", + test: function(sqlite3){ + /** + Testing https://sqlite.org/forum/forumpost/40ce55bdf5 with + the exception that that post uses "external content" for + the FTS index. This isn't testing a fix, just confirming + that the bug report is not really a bug. + */ + const db = new sqlite3.oo1.DB();//(':memory:','wt'); + db.exec([ + "create virtual table f using fts5 (path);", + "insert into f(path) values('abc'),('def'),('ghi');" + ]); + const fetchEm = ()=> db.exec({ + sql: "SELECT * FROM f order by path", + rowMode: 'array' + }); + /*const dump = function(lbl){ + let rc = fetchEm(); + log((lbl ? (lbl+' results') : ''),rc); + };*/ + //dump('Full fts table'); + let rc = fetchEm(); + T.assert(3===rc.length); + db.exec( + ["delete from f where rowid in (", + "select rowid from f where path = :path", + ")"], + {bind: {":path": "def"}} + ); + //dump('After deleting one entry via subquery'); + rc = fetchEm(); + T.assert(2===rc.length) + .assert('abcghi'===rc.join('')); + //log('rc =',rc); + db.close(); + } + }) + .t({ + name: 'r/o connection recovery from write op error', + predicate: ()=>hasOpfs() || "Requires OPFS to reproduce", + //predicate: ()=>false, + test: async function(sqlite3){ + /* https://sqlite.org/forum/forumpost/cf37d5ff1182c31081 + + The "opfs" VFS (but not SAHPool) was formerly misbehaving + after a write attempt was made on a db opened with + mode=ro. This test ensures that that behavior is fixed and + compares that behavior with other VFSes. */ + const tryOne = function(vfsName,descr){ + const uri = 'file:///foo.db'; + let db = new sqlite3.oo1.DB(uri + (vfsName ? '?vfs='+vfsName : '')); + db.exec([ + "drop table if exists t;", + "create table t(a);", + "insert into t(a) values('abc'),('def'),('ghi');" + ]); + db.close(); + db = new sqlite3.oo1.DB(uri+'?mode=ro'+ + (vfsName ? '&vfs='+vfsName : '')); + let err; + try { + db.exec('insert into t(a) values(1)'); + }catch(e){ + err = e; + } + T.assert(err && (err.message.indexOf('SQLITE_READONLY')===0)); + try{ + db.exec('select a from t'); + }finally{ + db.close(); + } + }; + const poolConfig = JSON.parse(JSON.stringify(sahPoolConfig)); + poolConfig.name = 'opfs-sahpool-cf37d5ff11'; + let poolUtil; + await sqlite3.installOpfsSAHPoolVfs(poolConfig).then(p=>poolUtil=p); + T.assert(!!sqlite3.capi.sqlite3_vfs_find(poolConfig.name), "Expecting to find just-registered VFS"); + try{ + tryOne(false, "Emscripten filesystem"); + tryOne(poolConfig.name); + tryOne('opfs'); + }finally{ + await poolUtil.removeVfs(); + } + } + }) + .t({ + /* https://github.com/sqlite/sqlite-wasm/issues/92 */ + name: 'sqlite3_set_auxdata() binding signature', + test: function(sqlite3){ + const db = new sqlite3.oo1.DB(); + const stack = wasm.pstack.pointer; + const pAux = wasm.pstack.alloc(4); + let pAuxDestructed = 0; + const pAuxDtor = wasm.installFunction('v(p)', function(ptr){ + //log("freeing auxdata"); + ++pAuxDestructed; + }); + let pAuxDtorDestructed = false; + db.onclose = { + after: ()=>{ + pAuxDtorDestructed = true; + wasm.uninstallFunction(pAuxDtor); + } + }; + let nAuxSet = 0 /* how many times we set aux data */; + let nAuxReused = 0 /* how many times we reused aux data */; + try{ + db.createFunction("auxtest",{ + xFunc: function(pCx, x, y){ + T.assert(wasm.isPtr(pCx)); + const localAux = capi.sqlite3_get_auxdata(pCx, 0); + if( !localAux ){ + //log("setting auxdata"); + /** + We do not currently an automated way to clean up + auxdata finalizer functions (the 4th argument to + sqlite3_set_auxdata()) which get automatically + converted from JS to WASM. Because of that, enabling + automated conversions here would lead to leaks more + often than not. Instead, follow the pattern show in + this function: use wasm.installFunction() to create + the function, then pass the resulting function + pointer this function, and cleanup (at some point) + using wasm.uninstallFunction(). + */ + ++nAuxSet; + capi.sqlite3_set_auxdata(pCx, 0, pAux, pAuxDtor); + }else{ + //log("reusing auxdata",localAux); + T.assert(pAux===localAux); + ++nAuxReused; + } + return x; + } + }); + db.exec([ + "create table t(a);", + "insert into t(a) values(1),(2),(1);", + "select auxtest(1,a), auxtest(1,a) from t order by a" + ]); + }finally{ + db.close(); + wasm.pstack.restore(stack); + } + T.assert(nAuxSet>0).assert(nAuxReused>0) + .assert(6===nAuxReused+nAuxSet); + T.assert(pAuxDestructed>0); + T.assert(pAuxDtorDestructed); + } + }) + ;/*end of Bug Reports group*/; + //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); if(0){ - self.sqlite3ApiConfig = { + globalThis.sqlite3ApiConfig = { debug: ()=>{}, log: ()=>{}, warn: ()=>{}, error: ()=>{} } } - if(!self.sqlite3InitModule && !isUIThread()){ +//#if not target:es6-module + if(!globalThis.sqlite3InitModule && !isUIThread()){ /* Vanilla worker, as opposed to an ES6 module worker */ /* If sqlite3.js is in a directory other than this script, in order @@ -2994,27 +3973,37 @@ self.sqlite3InitModule = sqlite3InitModule; that's not needed. URL arguments passed as part of the filename via importScripts() - are simply lost, and such scripts see the self.location of + are simply lost, and such scripts see the globalThis.location of _this_ script. */ - let sqlite3Js = 'sqlite3.js'; - const urlParams = new URL(self.location.href).searchParams; + let sqlite3Js = '@sqlite3.js@' + .split('/').pop()/*the build-injected name has a dir part and + we specifically want to test the following + support for locating the wasm, so remove + that dir part. */; + const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; } importScripts(sqlite3Js); } - self.sqlite3InitModule.__isUnderTest = +//#endif + globalThis.sqlite3InitModule.__isUnderTest = true /* disables certain API-internal cleanup so that we can test internal APIs from here */; - self.sqlite3InitModule({ + globalThis.sqlite3InitModule({ print: log, printErr: error - }).then(function(sqlite3){ - //console.log('sqlite3 =',sqlite3); + }).then(async function(sqlite3){ + TestUtil.assert(!!sqlite3.util); log("Done initializing WASM/JS bits. Running tests..."); sqlite3.config.warn("Installing sqlite3 bits as global S for local dev/test purposes."); - self.S = sqlite3; + globalThis.S = sqlite3; + /*await sqlite3.installOpfsSAHPoolVfs(sahPoolConfig) + .then((u)=>log("Loaded",u.vfsName,"VFS")) + .catch(e=>{ + log("Cannot install OpfsSAHPool.",e); + });*/ capi = sqlite3.capi; wasm = sqlite3.wasm; log("sqlite3 version:",capi.sqlite3_libversion(), @@ -3025,11 +4014,14 @@ self.sqlite3InitModule = sqlite3InitModule; logClass('warning',"BigInt/int64 support is disabled."); } if(haveWasmCTests()){ - log("sqlite3_wasm_test_...() APIs are available."); + log("sqlite3__wasm_test_...() APIs are available."); }else{ - logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); + logClass('warning',"sqlite3__wasm_test_...() APIs unavailable."); } + log("registered vfs list =",capi.sqlite3_js_vfs_list().join(', ')); + SQLite3 = sqlite3; + log("WASM pointer size:",wasm.ptr.size,"bytes."); + TestUtil.checkHeapSize(); TestUtil.runTests(sqlite3); }); })(self); - diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 595ab24529..54ed04a4f6 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -35,6 +35,9 @@

    re-opening the tab usually resolves it, but sometimes restarting the browser is required.

    +

    + Links for various testing options:

    +

    diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 14cd6f514f..1848901afe 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -113,6 +113,31 @@ } }; + /* Set up links to launch this tool with various combinations of + flags... */ + const eTestLinks = document.querySelector('#testlinks'); + const optArgs = function(obj){ + const li = []; + for(const k of ['interval','iterations','workers','verbose','unlock-asap']){ + if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); + } + return li.join('&'); + }; + for(const opt of [ + {interval: 1000, workers: 5, iterations: 30}, + {interval: 500, workers: 5, iterations: 30}, + {interval: 250, workers: 3, iterations: 30}, + {interval: 600, workers: 5, iterations: 100} + ]){ + const li = document.createElement('li'); + eTestLinks.appendChild(li); + const a = document.createElement('a'); + li.appendChild(a); + const args = optArgs(opt); + a.setAttribute('href', '?'+args); + a.innerText = args; + } + stdout("Launching",options.workerCount,"workers. Options:",options); workers.uri = ( 'worker.js?' diff --git a/ext/wasm/tests/opfs/sahpool/digest-worker.js b/ext/wasm/tests/opfs/sahpool/digest-worker.js new file mode 100644 index 0000000000..28b3c1673f --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/digest-worker.js @@ -0,0 +1,94 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is part of testing the OPFS SAHPool VFS's computeDigest() + fix. See ./digest.html for the details. +*/ +const clog = console.log.bind(console); +const wPost = (type,...args)=>postMessage({type, payload:args}); +const log = (...args)=>{ + clog("Worker:",...args); + wPost('log',...args); +} + +const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; +}; +if( !hasOpfs() ){ + wPost('error',"OPFS not detected"); + throw new Error("OPFS not detected"); +} + +clog("Importing sqlite3..."); +const searchParams = new URL(self.location.href).searchParams; +importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js'); + +const runTests = function(sqlite3, poolUtil){ + const fname = '/my.db'; + let db = new poolUtil.OpfsSAHPoolDb(fname); + let n = (new Date()).valueOf(); + try { + db.exec([ + "create table if not exists t(a);" + ]); + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log(fname,"record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } + + db = new poolUtil.OpfsSAHPoolDb(fname); + try { + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log(fname,"record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } + + const fname2 = '/my2.db'; + db = new poolUtil.OpfsSAHPoolDb(fname2); + try { + db.exec([ + "create table if not exists t(a);" + ]); + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log(fname2,"record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } +}; + +globalThis.sqlite3InitModule().then(async function(sqlite3){ + log("sqlite3 version:",sqlite3.version); + const sahPoolConfig = { + name: 'opfs-sahpool-digest', + clearOnInit: false, + initialCapacity: 6 + }; + return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(poolUtil=>{ + log('vfs acquired'); + runTests(sqlite3, poolUtil); + }); +}); diff --git a/ext/wasm/tests/opfs/sahpool/digest.html b/ext/wasm/tests/opfs/sahpool/digest.html new file mode 100644 index 0000000000..daa1f77287 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/digest.html @@ -0,0 +1,151 @@ + + + + + + + + + sqlite3 tester: OpfsSAHPool Digest + + +

    + +

    + This is a test app for the digest calculation of the OPFS + SAHPool VFS. It requires running it with a new database created using + v3.49.0 or older, then running it again with a newer version, then + again with 3.49.0 or older. +

    +
    + + +
    +
    + + + diff --git a/ext/wasm/tests/opfs/sahpool/index.html b/ext/wasm/tests/opfs/sahpool/index.html new file mode 100644 index 0000000000..f3d07f456a --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/index.html @@ -0,0 +1,31 @@ + + + + + + + + + sqlite3 tester: OpfsSAHPool Pausing + + +

    + +

    + This page provides a very basic demonstration of + "pausing" and "unpausing" the OPFS SAHPool VFS such that + multiple pages or workers can use it by coordinating which + handler may have it open at any given time. +

    +
    + + +
    +
    + + + + diff --git a/ext/wasm/tests/opfs/sahpool/sahpool-pausing.js b/ext/wasm/tests/opfs/sahpool/sahpool-pausing.js new file mode 100644 index 0000000000..1aa98d3cb3 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/sahpool-pausing.js @@ -0,0 +1,183 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + These tests are specific to the opfs-sahpool VFS and are limited to + demonstrating its pause/unpause capabilities. + + Most of this file is infrastructure for displaying results to the + user. Search for runTests() to find where the work actually starts. +*/ +'use strict'; +(function(){ + let logClass; + + const mapToString = (v)=>{ + switch(typeof v){ + case 'number': case 'string': case 'boolean': + case 'undefined': case 'bigint': + return ''+v; + default: break; + } + if(null===v) return 'null'; + if(v instanceof Error){ + v = { + message: v.message, + stack: v.stack, + errorClass: v.name + }; + } + return JSON.stringify(v,undefined,2); + }; + const normalizeArgs = (args)=>args.map(mapToString); + const logTarget = document.querySelector('#test-output'); + logClass = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass){ + for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ + ln.classList.add(c); + } + } + ln.append(document.createTextNode(normalizeArgs(args).join(' '))); + logTarget.append(ln); + }; + const cbReverse = document.querySelector('#cb-log-reverse'); + //cbReverse.setAttribute('checked','checked'); + const cbReverseKey = 'tester1:cb-log-reverse'; + const cbReverseIt = ()=>{ + logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + }; + cbReverse.addEventListener('change', cbReverseIt, true); + /*if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + }*/ + cbReverseIt(); + + const log = (...args)=>{ + //console.log(...args); + logClass('',...args); + } + const warn = (...args)=>{ + console.warn(...args); + logClass('warning',...args); + } + const error = (...args)=>{ + console.error(...args); + logClass('error',...args); + }; + + const toss = (...args)=>{ + error(...args); + throw new Error(args.join(' ')); + }; + + const endOfWork = (passed=true)=>{ + const eH = document.querySelector('#color-target'); + const eT = document.querySelector('title'); + if(passed){ + log("End of work chain. If you made it this far, you win."); + eH.innerText = 'PASS: '+eH.innerText; + eH.classList.add('tests-pass'); + eT.innerText = 'PASS: '+eT.innerText; + }else{ + eH.innerText = 'FAIL: '+eH.innerText; + eH.classList.add('tests-fail'); + eT.innerText = 'FAIL: '+eT.innerText; + } + }; + + const nextHandlerQueue = []; + + const nextHandler = function(workerId,...msg){ + log(workerId,...msg); + (nextHandlerQueue.shift())(); + }; + + const postThen = function(W, msgType, callback){ + nextHandlerQueue.push(callback); + W.postMessage({type:msgType}); + }; + + /** + Run a series of operations on an sahpool db spanning two workers. + This would arguably be more legible with Promises, but creating a + Promise-based communication channel for this purpose is left as + an exercise for the reader. An example of such a proxy can be + found in the SQLite source tree: + + https://sqlite.org/src/file/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js + */ + const runPyramidOfDoom = function(W1, W2){ + postThen(W1, 'vfs-acquire', function(){ + postThen(W1, 'db-init', function(){ + postThen(W1, 'db-query', function(){ + postThen(W1, 'vfs-pause', function(){ + postThen(W2, 'vfs-acquire', function(){ + postThen(W2, 'db-query', function(){ + postThen(W2, 'vfs-remove', endOfWork); + }); + }); + }); + }); + }); + }); + }; + + const runTests = function(){ + log("Running opfs-sahpool pausing tests..."); + const wjs = 'sahpool-worker.js?sqlite3.dir=../../../jswasm'; + const W1 = new Worker(wjs+'&workerId=w1'), + W2 = new Worker(wjs+'&workerId=w2'); + W1.workerId = 'w1'; + W2.workerId = 'w2'; + let initCount = 0; + const onmessage = function({data}){ + //log("onmessage:",data); + switch(data.type){ + case 'vfs-acquired': + nextHandler(data.workerId, "VFS acquired"); + break; + case 'vfs-paused': + nextHandler(data.workerId, "VFS paused"); + break; + case 'vfs-unpaused': + nextHandler(data.workerId, 'VFS unpaused'); + break; + case 'vfs-removed': + nextHandler(data.workerId, 'VFS removed'); + break; + case 'db-inited': + nextHandler(data.workerId, 'db initialized'); + break; + case 'query-result': + nextHandler(data.workerId, 'query result', data.payload); + break; + case 'log': + log(data.workerId, ':', ...data.payload); + break; + case 'error': + error(data.workerId, ':', ...data.payload); + endOfWork(false); + break; + case 'initialized': + log(data.workerId, ': Worker initialized',...data.payload); + if( 2===++initCount ){ + runPyramidOfDoom(W1, W2); + } + break; + } + }; + W1.onmessage = W2.onmessage = onmessage; + }; + + runTests(); +})(); diff --git a/ext/wasm/tests/opfs/sahpool/sahpool-worker.js b/ext/wasm/tests/opfs/sahpool/sahpool-worker.js new file mode 100644 index 0000000000..592f159551 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/sahpool-worker.js @@ -0,0 +1,104 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is part of sahpool-pausing.js's demonstration of the + pause/unpause feature of the opfs-sahpool VFS. +*/ +const searchParams = new URL(self.location.href).searchParams; +const workerId = searchParams.get('workerId'); +const wPost = (type,...args)=>postMessage({type, workerId, payload:args}); +const log = (...args)=>wPost('log',...args); +let capi, wasm, S, poolUtil; + +const sahPoolConfig = { + name: 'opfs-sahpool-pausable', + clearOnInit: false, + initialCapacity: 3 +}; + +importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js'); + +const sqlExec = function(sql){ + const db = new poolUtil.OpfsSAHPoolDb('/my.db'); + try{ + return db.exec(sql); + }finally{ + db.close(); + } +}; + +const clog = console.log.bind(console); +globalThis.onmessage = function({data}){ + clog(workerId+": onmessage:",data); + switch(data.type){ + case 'vfs-acquire': + if( poolUtil ){ + poolUtil.unpauseVfs().then(()=>wPost('vfs-unpaused')); + }else{ + S.installOpfsSAHPoolVfs(sahPoolConfig).then(pu=>{ + poolUtil = pu; + wPost('vfs-acquired'); + }); + } + break; + case 'db-init': + try{ + sqlExec([ + "DROP TABLE IF EXISTS mytable;", + "CREATE TABLE mytable(a);", + "INSERT INTO mytable(a) VALUES(11),(22),(33)" + ]); + wPost('db-inited'); + }catch(e){ + wPost('error',e.message); + } + break; + case 'db-query': { + const rc = sqlExec({ + sql: 'select * from mytable order by a', + rowMode: 'array', + returnValue: 'resultRows' + }); + wPost('query-result',rc); + break; + } + case 'vfs-remove': + poolUtil.removeVfs().then(()=>wPost('vfs-removed')); + break; + case 'vfs-pause': + poolUtil.pauseVfs(); + wPost('vfs-paused'); + break; + } +}; + +const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; +}; +if( !hasOpfs() ){ + wPost('error',"OPFS not detected"); +}else{ + globalThis.sqlite3InitModule().then(async function(sqlite3){ + S = sqlite3; + capi = S.capi; + wasm = S.wasm; + log("sqlite3 version:",capi.sqlite3_libversion(), + capi.sqlite3_sourceid()); + //return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(pu=>poolUtil=pu); + }).then(()=>{ + wPost('initialized'); + }); +} diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make deleted file mode 100644 index 020014f3e8..0000000000 --- a/ext/wasm/wasmfs.make +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/make -#^^^^ help emacs select makefile mode -# -# This is a sub-make for building a standalone wasmfs-based -# sqlite3.wasm. It is intended to be "include"d from the main -# GNUMakefile. -######################################################################## -MAKEFILE.wasmfs := $(lastword $(MAKEFILE_LIST)) - -# Maintenance reminder: these particular files cannot be built into a -# subdirectory because loading of the auxiliary -# sqlite3-wasmfs.worker.js file it creates fails if sqlite3-wasmfs.js -# is loaded from any directory other than the one in which the -# containing HTML lives. Similarly, they cannot be loaded from a -# Worker to an Emscripten quirk regarding loading nested Workers. -dir.wasmfs := $(dir.wasm) -sqlite3-wasmfs.js := $(dir.wasmfs)/sqlite3-wasmfs.js -sqlite3-wasmfs.mjs := $(dir.wasmfs)/sqlite3-wasmfs.mjs -sqlite3-wasmfs.wasm := $(dir.wasmfs)/sqlite3-wasmfs.wasm - -CLEAN_FILES += $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.wasm) \ - $(subst .js,.worker.js,$(sqlite3-wasmfs.js)) \ - $(sqlite3-wasmfs.mjs) \ - $(subst .mjs,.worker.mjs,$(sqlite3-wasmfs.mjs)) - -######################################################################## -# emcc flags for .c/.o. -cflags.sqlite3-wasmfs := -cflags.sqlite3-wasmfs += -std=c99 -fPIC -cflags.sqlite3-wasmfs += -pthread -cflags.sqlite3-wasmfs += $(cflags.speedtest1) -cflags.sqlite3-wasmfs += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS - -######################################################################## -# emcc flags specific to building the final .js/.wasm file... -emcc.flags.sqlite3-wasmfs := -fPIC -emcc.flags.sqlite3-wasmfs += --no-entry -emcc.flags.sqlite3-wasmfs += --minify 0 -emcc.flags.sqlite3-wasmfs += -sMODULARIZE -emcc.flags.sqlite3-wasmfs += -sEXPORT_NAME=$(sqlite3.js.init-func) -emcc.flags.sqlite3-wasmfs += -sSTRICT_JS -emcc.flags.sqlite3-wasmfs += -sDYNAMIC_EXECUTION=0 -emcc.flags.sqlite3-wasmfs += -sNO_POLYFILL -emcc.flags.sqlite3-wasmfs += -sWASM_BIGINT=$(emcc.WASM_BIGINT) -emcc.flags.sqlite3-wasmfs += -sEXPORTED_FUNCTIONS=@$(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) -emcc.flags.sqlite3-wasmfs += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack - # wasmMemory ==> for -sIMPORTED_MEMORY - # allocateUTF8OnStack ==> wasmfs internals -emcc.flags.sqlite3-wasmfs += -sUSE_CLOSURE_COMPILER=0 -emcc.flags.sqlite3-wasmfs += -Wno-limited-postlink-optimizations -# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. -emcc.flags.sqlite3-wasmfs += -sALLOW_TABLE_GROWTH -emcc.flags.sqlite3-wasmfs += -sSTACK_SIZE=512KB -emcc.flags.sqlite3-wasmfs += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. -emcc.flags.sqlite3-wasmfs += -sMEMORY64=0 -emcc.flags.sqlite3-wasmfs += -sIMPORTED_MEMORY -emcc.flags.sqlite3-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) -# ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js -sqlite3-wasmfs.fsflags := -pthread -sWASMFS \ - -sPTHREAD_POOL_SIZE=2 -sENVIRONMENT=web,worker \ - -sERROR_ON_UNDEFINED_SYMBOLS=0 -sLLD_REPORT_UNDEFINED -# ^^^^^ why undefined symbols are necessary for the wasmfs build is anyone's guess. -emcc.flags.sqlite3-wasmfs += $(sqlite3-wasmfs.fsflags) -#emcc.flags.sqlite3-wasmfs += -sALLOW_MEMORY_GROWTH -#^^^ using ALLOW_MEMORY_GROWTH produces a warning from emcc: -# USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, -# see https://github.com/WebAssembly/design/issues/1271 [-Wpthreads-mem-growth] -# And, indeed, it runs slowly if memory is permitted to grow. -emcc.flags.sqlite3-wasmfs.vanilla := -emcc.flags.sqlite3-wasmfs.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META -$(eval $(call call-make-pre-js,sqlite3-wasmfs,vanilla)) -$(eval $(call call-make-pre-js,sqlite3-wasmfs,esm)) -Xemcc.flags.sqlite3-wasmfs.vanilla += \ - $(pre-post-common.flags.vanilla) \ - $(pre-post-sqlite3-wasmfs.flags.vanilla) -Xemcc.flags.sqlite3-wasmfs.esm += \ - $(pre-post-common.flags.esm) \ - $(pre-post-sqlite3-wasmfs.flags.esm) -$(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(sqlite3-wasm.c) \ - $(EXPORTED_FUNCTIONS.api) $(MAKEFILE) $(MAKEFILE.wasmfs) -$(sqlite3-wasmfs.js): $(pre-post-sqlite3-wasmfs.deps.vanilla) -$(sqlite3-wasmfs.mjs): $(pre-post-sqlite3-wasmfs.deps.esm) -# SQLITE3-WASMFS.xJS.RECIPE is the wasmfs-specific counterpart -# of SQLITE3.xJS.RECIPE from the main makefile. -define SQLITE3-WASMFS.xJS.RECIPE - @echo "Building $@ ..." - $(emcc.bin) -o $@ $(emcc_opt_full) $(emcc.flags) \ - $(cflags.sqlite3-wasmfs) \ - $(emcc.flags.sqlite3-wasmfs) $(emcc.flags.sqlite3-wasmfs.$(1)) \ - $(pre-post-sqlite3-wasmfs.flags.$(1)) \ - $(sqlite3-wasm.c) - @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(1)) - chmod -x $(sqlite3-wasmfs.wasm) - $(maybe-wasm-strip) $(sqlite3-wasmfs.wasm) - @ls -la $(sqlite3-wasmfs.wasm) sqlite3-wasmfs*js -endef -$(sqlite3-wasmfs.js): - $(call SQLITE3-WASMFS.xJS.RECIPE,vanilla) -$(sqlite3-wasmfs.mjs): $(sqlite3-wasmfs.js) - $(call SQLITE3-WASMFS.xJS.RECIPE,esm) -$(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) -wasmfs: $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs) -#all: wasmfs - -######################################################################## -# speedtest1 for wasmfs. -speedtest1-wasmfs.js := $(dir.wasmfs)/speedtest1-wasmfs.js -speedtest1-wasmfs.wasm := $(subst .js,.wasm,$(speedtest1-wasmfs.js)) -emcc.flags.speedtest1-wasmfs := $(sqlite3-wasmfs.fsflags) -emcc.flags.speedtest1-wasmfs += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS -emcc.flags.speedtest1-wasmfs += -sALLOW_MEMORY_GROWTH=0 -emcc.flags.speedtest1-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) -#$(eval $(call call-make-pre-js,speedtest1-wasmfs,vanilla)) -$(speedtest1-wasmfs.js): $(speedtest1.cses) $(sqlite3-wasmfs.js) \ - $(MAKEFILE) $(MAKEFILE.wasmfs) \ - $(pre-post-sqlite3-wasmfs.deps) \ - $(EXPORTED_FUNCTIONS.speedtest1) - @echo "Building $@ ..." - $(emcc.bin) \ - $(emcc.speedtest1.common) $(emcc.flags.speedtest1-wasmfs) \ - $(pre-post-sqlite3-wasmfs.flags.vanilla) \ - $(cflags.sqlite3-wasmfs) \ - -o $@ $(speedtest1.cses) -lm - $(maybe-wasm-strip) $(speedtest1-wasmfs.wasm) - ls -la $@ $(speedtest1-wasmfs.wasm) - -#speedtest1: $(speedtest1-wasmfs.js) -wasmfs: $(speedtest1-wasmfs.js) -CLEAN_FILES += $(speedtest1-wasmfs.js) $(speedtest1-wasmfs.wasm) \ - $(subst .js,.worker.js,$(speedtest1-wasmfs.js)) -# end speedtest1.js -######################################################################## diff --git a/install-sh b/install-sh deleted file mode 100755 index e9de23842d..0000000000 --- a/install-sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/bin/sh -# -# install - install a program, script, or datafile -# This comes from X11R5 (mit/util/scripts/install.sh). -# -# Copyright 1991 by the Massachusetts Institute of Technology -# -# Permission to use, copy, modify, distribute, and sell this software and its -# documentation for any purpose is hereby granted without fee, provided that -# the above copyright notice appear in all copies and that both that -# copyright notice and this permission notice appear in supporting -# documentation, and that the name of M.I.T. not be used in advertising or -# publicity pertaining to distribution of the software without specific, -# written prior permission. M.I.T. makes no representations about the -# suitability of this software for any purpose. It is provided "as is" -# without express or implied warranty. -# -# Calling this script install-sh is preferred over install.sh, to prevent -# `make' implicit rules from creating a file called install from it -# when there is no Makefile. -# -# This script is compatible with the BSD install script, but was written -# from scratch. It can only install one file at a time, a restriction -# shared with many OS's install programs. - - -# set DOITPROG to echo to test this script - -# Don't use :- since 4.3BSD and earlier shells don't like it. -doit="${DOITPROG-}" - - -# put in absolute paths if you don't have them in your path; or use env. vars. - -mvprog="${MVPROG-mv}" -cpprog="${CPPROG-cp}" -chmodprog="${CHMODPROG-chmod}" -chownprog="${CHOWNPROG-chown}" -chgrpprog="${CHGRPPROG-chgrp}" -stripprog="${STRIPPROG-strip}" -rmprog="${RMPROG-rm}" -mkdirprog="${MKDIRPROG-mkdir}" - -transformbasename="" -transform_arg="" -instcmd="$mvprog" -chmodcmd="$chmodprog 0755" -chowncmd="" -chgrpcmd="" -stripcmd="" -rmcmd="$rmprog -f" -mvcmd="$mvprog" -src="" -dst="" -dir_arg="" - -while [ x"$1" != x ]; do - case $1 in - -c) instcmd="$cpprog" - shift - continue;; - - -d) dir_arg=true - shift - continue;; - - -m) chmodcmd="$chmodprog $2" - shift - shift - continue;; - - -o) chowncmd="$chownprog $2" - shift - shift - continue;; - - -g) chgrpcmd="$chgrpprog $2" - shift - shift - continue;; - - -s) stripcmd="$stripprog" - shift - continue;; - - -t=*) transformarg=`echo $1 | sed 's/-t=//'` - shift - continue;; - - -b=*) transformbasename=`echo $1 | sed 's/-b=//'` - shift - continue;; - - *) if [ x"$src" = x ] - then - src=$1 - else - # this colon is to work around a 386BSD /bin/sh bug - : - dst=$1 - fi - shift - continue;; - esac -done - -if [ x"$src" = x ] -then - echo "install: no input file specified" - exit 1 -else - true -fi - -if [ x"$dir_arg" != x ]; then - dst=$src - src="" - - if [ -d $dst ]; then - instcmd=: - chmodcmd="" - else - instcmd=mkdir - fi -else - -# Waiting for this to be detected by the "$instcmd $src $dsttmp" command -# might cause directories to be created, which would be especially bad -# if $src (and thus $dsttmp) contains '*'. - - if [ -f $src -o -d $src ] - then - true - else - echo "install: $src does not exist" - exit 1 - fi - - if [ x"$dst" = x ] - then - echo "install: no destination specified" - exit 1 - else - true - fi - -# If destination is a directory, append the input filename; if your system -# does not like double slashes in filenames, you may need to add some logic - - if [ -d $dst ] - then - dst="$dst"/`basename $src` - else - true - fi -fi - -## this sed command emulates the dirname command -dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'` - -# Make sure that the destination directory exists. -# this part is taken from Noah Friedman's mkinstalldirs script - -# Skip lots of stat calls in the usual case. -if [ ! -d "$dstdir" ]; then -defaultIFS=' -' -IFS="${IFS-${defaultIFS}}" - -oIFS="${IFS}" -# Some sh's can't handle IFS=/ for some reason. -IFS='%' -set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'` -IFS="${oIFS}" - -pathcomp='' - -while [ $# -ne 0 ] ; do - pathcomp="${pathcomp}${1}" - shift - - if [ ! -d "${pathcomp}" ] ; - then - $mkdirprog "${pathcomp}" - else - true - fi - - pathcomp="${pathcomp}/" -done -fi - -if [ x"$dir_arg" != x ] -then - $doit $instcmd $dst && - - if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi && - if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi && - if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi && - if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi -else - -# If we're going to rename the final executable, determine the name now. - - if [ x"$transformarg" = x ] - then - dstfile=`basename $dst` - else - dstfile=`basename $dst $transformbasename | - sed $transformarg`$transformbasename - fi - -# don't allow the sed command to completely eliminate the filename - - if [ x"$dstfile" = x ] - then - dstfile=`basename $dst` - else - true - fi - -# Make a temp file name in the proper directory. - - dsttmp=$dstdir/#inst.$$# - -# Move or copy the file name to the temp name - - $doit $instcmd $src $dsttmp && - - trap "rm -f ${dsttmp}" 0 && - -# and set any options; do chmod last to preserve setuid bits - -# If any of these fail, we abort the whole thing. If we want to -# ignore errors from any of these, just make sure not to ignore -# errors from the above "$doit $instcmd $src $dsttmp" command. - - if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi && - if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi && - if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi && - if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi && - -# Now rename the file to the real destination. - - $doit $rmcmd -f $dstdir/$dstfile && - $doit $mvcmd $dsttmp $dstdir/$dstfile - -fi && - - -exit 0 diff --git a/ltmain.sh b/ltmain.sh deleted file mode 100644 index 0f0a2da3f9..0000000000 --- a/ltmain.sh +++ /dev/null @@ -1,11147 +0,0 @@ -#! /bin/sh -## DO NOT EDIT - This file generated from ./build-aux/ltmain.in -## by inline-source v2014-01-03.01 - -# libtool (GNU libtool) 2.4.6 -# Provide generalized library-building support services. -# Written by Gordon Matzigkeit , 1996 - -# Copyright (C) 1996-2015 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# GNU Libtool is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# As a special exception to the GNU General Public License, -# if you distribute this file as part of a program or library that -# is built using GNU Libtool, you may include this file under the -# same distribution terms that you use for the rest of that program. -# -# GNU Libtool is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -PROGRAM=libtool -PACKAGE=libtool -VERSION=2.4.6 -package_revision=2.4.6 - - -## ------ ## -## Usage. ## -## ------ ## - -# Run './libtool --help' for help with using this script from the -# command line. - - -## ------------------------------- ## -## User overridable command paths. ## -## ------------------------------- ## - -# After configure completes, it has a better idea of some of the -# shell tools we need than the defaults used by the functions shared -# with bootstrap, so set those here where they can still be over- -# ridden by the user, but otherwise take precedence. - -: ${AUTOCONF="autoconf"} -: ${AUTOMAKE="automake"} - - -## -------------------------- ## -## Source external libraries. ## -## -------------------------- ## - -# Much of our low-level functionality needs to be sourced from external -# libraries, which are installed to $pkgauxdir. - -# Set a version string for this script. -scriptversion=2015-01-20.17; # UTC - -# General shell script boiler plate, and helper functions. -# Written by Gary V. Vaughan, 2004 - -# Copyright (C) 2004-2015 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. - -# As a special exception to the GNU General Public License, if you distribute -# this file as part of a program or library that is built using GNU Libtool, -# you may include this file under the same distribution terms that you use -# for the rest of that program. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNES FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Please report bugs or propose patches to gary@gnu.org. - - -## ------ ## -## Usage. ## -## ------ ## - -# Evaluate this file near the top of your script to gain access to -# the functions and variables defined here: -# -# . `echo "$0" | ${SED-sed} 's|[^/]*$||'`/build-aux/funclib.sh -# -# If you need to override any of the default environment variable -# settings, do that before evaluating this file. - - -## -------------------- ## -## Shell normalisation. ## -## -------------------- ## - -# Some shells need a little help to be as Bourne compatible as possible. -# Before doing anything else, make sure all that help has been provided! - -DUALCASE=1; export DUALCASE # for MKS sh -if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then : - emulate sh - NULLCMD=: - # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which - # is contrary to our usage. Disable this feature. - alias -g '${1+"$@"}'='"$@"' - setopt NO_GLOB_SUBST -else - case `(set -o) 2>/dev/null` in *posix*) set -o posix ;; esac -fi - -# NLS nuisances: We save the old values in case they are required later. -_G_user_locale= -_G_safe_locale= -for _G_var in LANG LANGUAGE LC_ALL LC_CTYPE LC_COLLATE LC_MESSAGES -do - eval "if test set = \"\${$_G_var+set}\"; then - save_$_G_var=\$$_G_var - $_G_var=C - export $_G_var - _G_user_locale=\"$_G_var=\\\$save_\$_G_var; \$_G_user_locale\" - _G_safe_locale=\"$_G_var=C; \$_G_safe_locale\" - fi" -done - -# CDPATH. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH - -# Make sure IFS has a sensible default -sp=' ' -nl=' -' -IFS="$sp $nl" - -# There are apparently some retarded systems that use ';' as a PATH separator! -if test "${PATH_SEPARATOR+set}" != set; then - PATH_SEPARATOR=: - (PATH='/bin;/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 && { - (PATH='/bin:/bin'; FPATH=$PATH; sh -c :) >/dev/null 2>&1 || - PATH_SEPARATOR=';' - } -fi - - - -## ------------------------- ## -## Locate command utilities. ## -## ------------------------- ## - - -# func_executable_p FILE -# ---------------------- -# Check that FILE is an executable regular file. -func_executable_p () -{ - test -f "$1" && test -x "$1" -} - - -# func_path_progs PROGS_LIST CHECK_FUNC [PATH] -# -------------------------------------------- -# Search for either a program that responds to --version with output -# containing "GNU", or else returned by CHECK_FUNC otherwise, by -# trying all the directories in PATH with each of the elements of -# PROGS_LIST. -# -# CHECK_FUNC should accept the path to a candidate program, and -# set $func_check_prog_result if it truncates its output less than -# $_G_path_prog_max characters. -func_path_progs () -{ - _G_progs_list=$1 - _G_check_func=$2 - _G_PATH=${3-"$PATH"} - - _G_path_prog_max=0 - _G_path_prog_found=false - _G_save_IFS=$IFS; IFS=${PATH_SEPARATOR-:} - for _G_dir in $_G_PATH; do - IFS=$_G_save_IFS - test -z "$_G_dir" && _G_dir=. - for _G_prog_name in $_G_progs_list; do - for _exeext in '' .EXE; do - _G_path_prog=$_G_dir/$_G_prog_name$_exeext - func_executable_p "$_G_path_prog" || continue - case `"$_G_path_prog" --version 2>&1` in - *GNU*) func_path_progs_result=$_G_path_prog _G_path_prog_found=: ;; - *) $_G_check_func $_G_path_prog - func_path_progs_result=$func_check_prog_result - ;; - esac - $_G_path_prog_found && break 3 - done - done - done - IFS=$_G_save_IFS - test -z "$func_path_progs_result" && { - echo "no acceptable sed could be found in \$PATH" >&2 - exit 1 - } -} - - -# We want to be able to use the functions in this file before configure -# has figured out where the best binaries are kept, which means we have -# to search for them ourselves - except when the results are already set -# where we skip the searches. - -# Unless the user overrides by setting SED, search the path for either GNU -# sed, or the sed that truncates its output the least. -test -z "$SED" && { - _G_sed_script=s/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ - for _G_i in 1 2 3 4 5 6 7; do - _G_sed_script=$_G_sed_script$nl$_G_sed_script - done - echo "$_G_sed_script" 2>/dev/null | sed 99q >conftest.sed - _G_sed_script= - - func_check_prog_sed () - { - _G_path_prog=$1 - - _G_count=0 - printf 0123456789 >conftest.in - while : - do - cat conftest.in conftest.in >conftest.tmp - mv conftest.tmp conftest.in - cp conftest.in conftest.nl - echo '' >> conftest.nl - "$_G_path_prog" -f conftest.sed conftest.out 2>/dev/null || break - diff conftest.out conftest.nl >/dev/null 2>&1 || break - _G_count=`expr $_G_count + 1` - if test "$_G_count" -gt "$_G_path_prog_max"; then - # Best one so far, save it but keep looking for a better one - func_check_prog_result=$_G_path_prog - _G_path_prog_max=$_G_count - fi - # 10*(2^10) chars as input seems more than enough - test 10 -lt "$_G_count" && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out - } - - func_path_progs "sed gsed" func_check_prog_sed $PATH:/usr/xpg4/bin - rm -f conftest.sed - SED=$func_path_progs_result -} - - -# Unless the user overrides by setting GREP, search the path for either GNU -# grep, or the grep that truncates its output the least. -test -z "$GREP" && { - func_check_prog_grep () - { - _G_path_prog=$1 - - _G_count=0 - _G_path_prog_max=0 - printf 0123456789 >conftest.in - while : - do - cat conftest.in conftest.in >conftest.tmp - mv conftest.tmp conftest.in - cp conftest.in conftest.nl - echo 'GREP' >> conftest.nl - "$_G_path_prog" -e 'GREP$' -e '-(cannot match)-' conftest.out 2>/dev/null || break - diff conftest.out conftest.nl >/dev/null 2>&1 || break - _G_count=`expr $_G_count + 1` - if test "$_G_count" -gt "$_G_path_prog_max"; then - # Best one so far, save it but keep looking for a better one - func_check_prog_result=$_G_path_prog - _G_path_prog_max=$_G_count - fi - # 10*(2^10) chars as input seems more than enough - test 10 -lt "$_G_count" && break - done - rm -f conftest.in conftest.tmp conftest.nl conftest.out - } - - func_path_progs "grep ggrep" func_check_prog_grep $PATH:/usr/xpg4/bin - GREP=$func_path_progs_result -} - - -## ------------------------------- ## -## User overridable command paths. ## -## ------------------------------- ## - -# All uppercase variable names are used for environment variables. These -# variables can be overridden by the user before calling a script that -# uses them if a suitable command of that name is not already available -# in the command search PATH. - -: ${CP="cp -f"} -: ${ECHO="printf %s\n"} -: ${EGREP="$GREP -E"} -: ${FGREP="$GREP -F"} -: ${LN_S="ln -s"} -: ${MAKE="make"} -: ${MKDIR="mkdir"} -: ${MV="mv -f"} -: ${RM="rm -f"} -: ${SHELL="${CONFIG_SHELL-/bin/sh}"} - - -## -------------------- ## -## Useful sed snippets. ## -## -------------------- ## - -sed_dirname='s|/[^/]*$||' -sed_basename='s|^.*/||' - -# Sed substitution that helps us do robust quoting. It backslashifies -# metacharacters that are still active within double-quoted strings. -sed_quote_subst='s|\([`"$\\]\)|\\\1|g' - -# Same as above, but do not quote variable references. -sed_double_quote_subst='s/\(["`\\]\)/\\\1/g' - -# Sed substitution that turns a string into a regex matching for the -# string literally. -sed_make_literal_regex='s|[].[^$\\*\/]|\\&|g' - -# Sed substitution that converts a w32 file name or path -# that contains forward slashes, into one that contains -# (escaped) backslashes. A very naive implementation. -sed_naive_backslashify='s|\\\\*|\\|g;s|/|\\|g;s|\\|\\\\|g' - -# Re-'\' parameter expansions in output of sed_double_quote_subst that -# were '\'-ed in input to the same. If an odd number of '\' preceded a -# '$' in input to sed_double_quote_subst, that '$' was protected from -# expansion. Since each input '\' is now two '\'s, look for any number -# of runs of four '\'s followed by two '\'s and then a '$'. '\' that '$'. -_G_bs='\\' -_G_bs2='\\\\' -_G_bs4='\\\\\\\\' -_G_dollar='\$' -sed_double_backslash="\ - s/$_G_bs4/&\\ -/g - s/^$_G_bs2$_G_dollar/$_G_bs&/ - s/\\([^$_G_bs]\\)$_G_bs2$_G_dollar/\\1$_G_bs2$_G_bs$_G_dollar/g - s/\n//g" - - -## ----------------- ## -## Global variables. ## -## ----------------- ## - -# Except for the global variables explicitly listed below, the following -# functions in the '^func_' namespace, and the '^require_' namespace -# variables initialised in the 'Resource management' section, sourcing -# this file will not pollute your global namespace with anything -# else. There's no portable way to scope variables in Bourne shell -# though, so actually running these functions will sometimes place -# results into a variable named after the function, and often use -# temporary variables in the '^_G_' namespace. If you are careful to -# avoid using those namespaces casually in your sourcing script, things -# should continue to work as you expect. And, of course, you can freely -# overwrite any of the functions or variables defined here before -# calling anything to customize them. - -EXIT_SUCCESS=0 -EXIT_FAILURE=1 -EXIT_MISMATCH=63 # $? = 63 is used to indicate version mismatch to missing. -EXIT_SKIP=77 # $? = 77 is used to indicate a skipped test to automake. - -# Allow overriding, eg assuming that you follow the convention of -# putting '$debug_cmd' at the start of all your functions, you can get -# bash to show function call trace with: -# -# debug_cmd='eval echo "${FUNCNAME[0]} $*" >&2' bash your-script-name -debug_cmd=${debug_cmd-":"} -exit_cmd=: - -# By convention, finish your script with: -# -# exit $exit_status -# -# so that you can set exit_status to non-zero if you want to indicate -# something went wrong during execution without actually bailing out at -# the point of failure. -exit_status=$EXIT_SUCCESS - -# Work around backward compatibility issue on IRIX 6.5. On IRIX 6.4+, sh -# is ksh but when the shell is invoked as "sh" and the current value of -# the _XPG environment variable is not equal to 1 (one), the special -# positional parameter $0, within a function call, is the name of the -# function. -progpath=$0 - -# The name of this program. -progname=`$ECHO "$progpath" |$SED "$sed_basename"` - -# Make sure we have an absolute progpath for reexecution: -case $progpath in - [\\/]*|[A-Za-z]:\\*) ;; - *[\\/]*) - progdir=`$ECHO "$progpath" |$SED "$sed_dirname"` - progdir=`cd "$progdir" && pwd` - progpath=$progdir/$progname - ;; - *) - _G_IFS=$IFS - IFS=${PATH_SEPARATOR-:} - for progdir in $PATH; do - IFS=$_G_IFS - test -x "$progdir/$progname" && break - done - IFS=$_G_IFS - test -n "$progdir" || progdir=`pwd` - progpath=$progdir/$progname - ;; -esac - - -## ----------------- ## -## Standard options. ## -## ----------------- ## - -# The following options affect the operation of the functions defined -# below, and should be set appropriately depending on run-time para- -# meters passed on the command line. - -opt_dry_run=false -opt_quiet=false -opt_verbose=false - -# Categories 'all' and 'none' are always available. Append any others -# you will pass as the first argument to func_warning from your own -# code. -warning_categories= - -# By default, display warnings according to 'opt_warning_types'. Set -# 'warning_func' to ':' to elide all warnings, or func_fatal_error to -# treat the next displayed warning as a fatal error. -warning_func=func_warn_and_continue - -# Set to 'all' to display all warnings, 'none' to suppress all -# warnings, or a space delimited list of some subset of -# 'warning_categories' to display only the listed warnings. -opt_warning_types=all - - -## -------------------- ## -## Resource management. ## -## -------------------- ## - -# This section contains definitions for functions that each ensure a -# particular resource (a file, or a non-empty configuration variable for -# example) is available, and if appropriate to extract default values -# from pertinent package files. Call them using their associated -# 'require_*' variable to ensure that they are executed, at most, once. -# -# It's entirely deliberate that calling these functions can set -# variables that don't obey the namespace limitations obeyed by the rest -# of this file, in order that that they be as useful as possible to -# callers. - - -# require_term_colors -# ------------------- -# Allow display of bold text on terminals that support it. -require_term_colors=func_require_term_colors -func_require_term_colors () -{ - $debug_cmd - - test -t 1 && { - # COLORTERM and USE_ANSI_COLORS environment variables take - # precedence, because most terminfo databases neglect to describe - # whether color sequences are supported. - test -n "${COLORTERM+set}" && : ${USE_ANSI_COLORS="1"} - - if test 1 = "$USE_ANSI_COLORS"; then - # Standard ANSI escape sequences - tc_reset='' - tc_bold=''; tc_standout='' - tc_red=''; tc_green='' - tc_blue=''; tc_cyan='' - else - # Otherwise trust the terminfo database after all. - test -n "`tput sgr0 2>/dev/null`" && { - tc_reset=`tput sgr0` - test -n "`tput bold 2>/dev/null`" && tc_bold=`tput bold` - tc_standout=$tc_bold - test -n "`tput smso 2>/dev/null`" && tc_standout=`tput smso` - test -n "`tput setaf 1 2>/dev/null`" && tc_red=`tput setaf 1` - test -n "`tput setaf 2 2>/dev/null`" && tc_green=`tput setaf 2` - test -n "`tput setaf 4 2>/dev/null`" && tc_blue=`tput setaf 4` - test -n "`tput setaf 5 2>/dev/null`" && tc_cyan=`tput setaf 5` - } - fi - } - - require_term_colors=: -} - - -## ----------------- ## -## Function library. ## -## ----------------- ## - -# This section contains a variety of useful functions to call in your -# scripts. Take note of the portable wrappers for features provided by -# some modern shells, which will fall back to slower equivalents on -# less featureful shells. - - -# func_append VAR VALUE -# --------------------- -# Append VALUE onto the existing contents of VAR. - - # We should try to minimise forks, especially on Windows where they are - # unreasonably slow, so skip the feature probes when bash or zsh are - # being used: - if test set = "${BASH_VERSION+set}${ZSH_VERSION+set}"; then - : ${_G_HAVE_ARITH_OP="yes"} - : ${_G_HAVE_XSI_OPS="yes"} - # The += operator was introduced in bash 3.1 - case $BASH_VERSION in - [12].* | 3.0 | 3.0*) ;; - *) - : ${_G_HAVE_PLUSEQ_OP="yes"} - ;; - esac - fi - - # _G_HAVE_PLUSEQ_OP - # Can be empty, in which case the shell is probed, "yes" if += is - # useable or anything else if it does not work. - test -z "$_G_HAVE_PLUSEQ_OP" \ - && (eval 'x=a; x+=" b"; test "a b" = "$x"') 2>/dev/null \ - && _G_HAVE_PLUSEQ_OP=yes - -if test yes = "$_G_HAVE_PLUSEQ_OP" -then - # This is an XSI compatible shell, allowing a faster implementation... - eval 'func_append () - { - $debug_cmd - - eval "$1+=\$2" - }' -else - # ...otherwise fall back to using expr, which is often a shell builtin. - func_append () - { - $debug_cmd - - eval "$1=\$$1\$2" - } -fi - - -# func_append_quoted VAR VALUE -# ---------------------------- -# Quote VALUE and append to the end of shell variable VAR, separated -# by a space. -if test yes = "$_G_HAVE_PLUSEQ_OP"; then - eval 'func_append_quoted () - { - $debug_cmd - - func_quote_for_eval "$2" - eval "$1+=\\ \$func_quote_for_eval_result" - }' -else - func_append_quoted () - { - $debug_cmd - - func_quote_for_eval "$2" - eval "$1=\$$1\\ \$func_quote_for_eval_result" - } -fi - - -# func_append_uniq VAR VALUE -# -------------------------- -# Append unique VALUE onto the existing contents of VAR, assuming -# entries are delimited by the first character of VALUE. For example: -# -# func_append_uniq options " --another-option option-argument" -# -# will only append to $options if " --another-option option-argument " -# is not already present somewhere in $options already (note spaces at -# each end implied by leading space in second argument). -func_append_uniq () -{ - $debug_cmd - - eval _G_current_value='`$ECHO $'$1'`' - _G_delim=`expr "$2" : '\(.\)'` - - case $_G_delim$_G_current_value$_G_delim in - *"$2$_G_delim"*) ;; - *) func_append "$@" ;; - esac -} - - -# func_arith TERM... -# ------------------ -# Set func_arith_result to the result of evaluating TERMs. - test -z "$_G_HAVE_ARITH_OP" \ - && (eval 'test 2 = $(( 1 + 1 ))') 2>/dev/null \ - && _G_HAVE_ARITH_OP=yes - -if test yes = "$_G_HAVE_ARITH_OP"; then - eval 'func_arith () - { - $debug_cmd - - func_arith_result=$(( $* )) - }' -else - func_arith () - { - $debug_cmd - - func_arith_result=`expr "$@"` - } -fi - - -# func_basename FILE -# ------------------ -# Set func_basename_result to FILE with everything up to and including -# the last / stripped. -if test yes = "$_G_HAVE_XSI_OPS"; then - # If this shell supports suffix pattern removal, then use it to avoid - # forking. Hide the definitions single quotes in case the shell chokes - # on unsupported syntax... - _b='func_basename_result=${1##*/}' - _d='case $1 in - */*) func_dirname_result=${1%/*}$2 ;; - * ) func_dirname_result=$3 ;; - esac' - -else - # ...otherwise fall back to using sed. - _b='func_basename_result=`$ECHO "$1" |$SED "$sed_basename"`' - _d='func_dirname_result=`$ECHO "$1" |$SED "$sed_dirname"` - if test "X$func_dirname_result" = "X$1"; then - func_dirname_result=$3 - else - func_append func_dirname_result "$2" - fi' -fi - -eval 'func_basename () -{ - $debug_cmd - - '"$_b"' -}' - - -# func_dirname FILE APPEND NONDIR_REPLACEMENT -# ------------------------------------------- -# Compute the dirname of FILE. If nonempty, add APPEND to the result, -# otherwise set result to NONDIR_REPLACEMENT. -eval 'func_dirname () -{ - $debug_cmd - - '"$_d"' -}' - - -# func_dirname_and_basename FILE APPEND NONDIR_REPLACEMENT -# -------------------------------------------------------- -# Perform func_basename and func_dirname in a single function -# call: -# dirname: Compute the dirname of FILE. If nonempty, -# add APPEND to the result, otherwise set result -# to NONDIR_REPLACEMENT. -# value returned in "$func_dirname_result" -# basename: Compute filename of FILE. -# value retuned in "$func_basename_result" -# For efficiency, we do not delegate to the functions above but instead -# duplicate the functionality here. -eval 'func_dirname_and_basename () -{ - $debug_cmd - - '"$_b"' - '"$_d"' -}' - - -# func_echo ARG... -# ---------------- -# Echo program name prefixed message. -func_echo () -{ - $debug_cmd - - _G_message=$* - - func_echo_IFS=$IFS - IFS=$nl - for _G_line in $_G_message; do - IFS=$func_echo_IFS - $ECHO "$progname: $_G_line" - done - IFS=$func_echo_IFS -} - - -# func_echo_all ARG... -# -------------------- -# Invoke $ECHO with all args, space-separated. -func_echo_all () -{ - $ECHO "$*" -} - - -# func_echo_infix_1 INFIX ARG... -# ------------------------------ -# Echo program name, followed by INFIX on the first line, with any -# additional lines not showing INFIX. -func_echo_infix_1 () -{ - $debug_cmd - - $require_term_colors - - _G_infix=$1; shift - _G_indent=$_G_infix - _G_prefix="$progname: $_G_infix: " - _G_message=$* - - # Strip color escape sequences before counting printable length - for _G_tc in "$tc_reset" "$tc_bold" "$tc_standout" "$tc_red" "$tc_green" "$tc_blue" "$tc_cyan" - do - test -n "$_G_tc" && { - _G_esc_tc=`$ECHO "$_G_tc" | $SED "$sed_make_literal_regex"` - _G_indent=`$ECHO "$_G_indent" | $SED "s|$_G_esc_tc||g"` - } - done - _G_indent="$progname: "`echo "$_G_indent" | $SED 's|.| |g'`" " ## exclude from sc_prohibit_nested_quotes - - func_echo_infix_1_IFS=$IFS - IFS=$nl - for _G_line in $_G_message; do - IFS=$func_echo_infix_1_IFS - $ECHO "$_G_prefix$tc_bold$_G_line$tc_reset" >&2 - _G_prefix=$_G_indent - done - IFS=$func_echo_infix_1_IFS -} - - -# func_error ARG... -# ----------------- -# Echo program name prefixed message to standard error. -func_error () -{ - $debug_cmd - - $require_term_colors - - func_echo_infix_1 " $tc_standout${tc_red}error$tc_reset" "$*" >&2 -} - - -# func_fatal_error ARG... -# ----------------------- -# Echo program name prefixed message to standard error, and exit. -func_fatal_error () -{ - $debug_cmd - - func_error "$*" - exit $EXIT_FAILURE -} - - -# func_grep EXPRESSION FILENAME -# ----------------------------- -# Check whether EXPRESSION matches any line of FILENAME, without output. -func_grep () -{ - $debug_cmd - - $GREP "$1" "$2" >/dev/null 2>&1 -} - - -# func_len STRING -# --------------- -# Set func_len_result to the length of STRING. STRING may not -# start with a hyphen. - test -z "$_G_HAVE_XSI_OPS" \ - && (eval 'x=a/b/c; - test 5aa/bb/cc = "${#x}${x%%/*}${x%/*}${x#*/}${x##*/}"') 2>/dev/null \ - && _G_HAVE_XSI_OPS=yes - -if test yes = "$_G_HAVE_XSI_OPS"; then - eval 'func_len () - { - $debug_cmd - - func_len_result=${#1} - }' -else - func_len () - { - $debug_cmd - - func_len_result=`expr "$1" : ".*" 2>/dev/null || echo $max_cmd_len` - } -fi - - -# func_mkdir_p DIRECTORY-PATH -# --------------------------- -# Make sure the entire path to DIRECTORY-PATH is available. -func_mkdir_p () -{ - $debug_cmd - - _G_directory_path=$1 - _G_dir_list= - - if test -n "$_G_directory_path" && test : != "$opt_dry_run"; then - - # Protect directory names starting with '-' - case $_G_directory_path in - -*) _G_directory_path=./$_G_directory_path ;; - esac - - # While some portion of DIR does not yet exist... - while test ! -d "$_G_directory_path"; do - # ...make a list in topmost first order. Use a colon delimited - # list incase some portion of path contains whitespace. - _G_dir_list=$_G_directory_path:$_G_dir_list - - # If the last portion added has no slash in it, the list is done - case $_G_directory_path in */*) ;; *) break ;; esac - - # ...otherwise throw away the child directory and loop - _G_directory_path=`$ECHO "$_G_directory_path" | $SED -e "$sed_dirname"` - done - _G_dir_list=`$ECHO "$_G_dir_list" | $SED 's|:*$||'` - - func_mkdir_p_IFS=$IFS; IFS=: - for _G_dir in $_G_dir_list; do - IFS=$func_mkdir_p_IFS - # mkdir can fail with a 'File exist' error if two processes - # try to create one of the directories concurrently. Don't - # stop in that case! - $MKDIR "$_G_dir" 2>/dev/null || : - done - IFS=$func_mkdir_p_IFS - - # Bail out if we (or some other process) failed to create a directory. - test -d "$_G_directory_path" || \ - func_fatal_error "Failed to create '$1'" - fi -} - - -# func_mktempdir [BASENAME] -# ------------------------- -# Make a temporary directory that won't clash with other running -# libtool processes, and avoids race conditions if possible. If -# given, BASENAME is the basename for that directory. -func_mktempdir () -{ - $debug_cmd - - _G_template=${TMPDIR-/tmp}/${1-$progname} - - if test : = "$opt_dry_run"; then - # Return a directory name, but don't create it in dry-run mode - _G_tmpdir=$_G_template-$$ - else - - # If mktemp works, use that first and foremost - _G_tmpdir=`mktemp -d "$_G_template-XXXXXXXX" 2>/dev/null` - - if test ! -d "$_G_tmpdir"; then - # Failing that, at least try and use $RANDOM to avoid a race - _G_tmpdir=$_G_template-${RANDOM-0}$$ - - func_mktempdir_umask=`umask` - umask 0077 - $MKDIR "$_G_tmpdir" - umask $func_mktempdir_umask - fi - - # If we're not in dry-run mode, bomb out on failure - test -d "$_G_tmpdir" || \ - func_fatal_error "cannot create temporary directory '$_G_tmpdir'" - fi - - $ECHO "$_G_tmpdir" -} - - -# func_normal_abspath PATH -# ------------------------ -# Remove doubled-up and trailing slashes, "." path components, -# and cancel out any ".." path components in PATH after making -# it an absolute path. -func_normal_abspath () -{ - $debug_cmd - - # These SED scripts presuppose an absolute path with a trailing slash. - _G_pathcar='s|^/\([^/]*\).*$|\1|' - _G_pathcdr='s|^/[^/]*||' - _G_removedotparts=':dotsl - s|/\./|/|g - t dotsl - s|/\.$|/|' - _G_collapseslashes='s|/\{1,\}|/|g' - _G_finalslash='s|/*$|/|' - - # Start from root dir and reassemble the path. - func_normal_abspath_result= - func_normal_abspath_tpath=$1 - func_normal_abspath_altnamespace= - case $func_normal_abspath_tpath in - "") - # Empty path, that just means $cwd. - func_stripname '' '/' "`pwd`" - func_normal_abspath_result=$func_stripname_result - return - ;; - # The next three entries are used to spot a run of precisely - # two leading slashes without using negated character classes; - # we take advantage of case's first-match behaviour. - ///*) - # Unusual form of absolute path, do nothing. - ;; - //*) - # Not necessarily an ordinary path; POSIX reserves leading '//' - # and for example Cygwin uses it to access remote file shares - # over CIFS/SMB, so we conserve a leading double slash if found. - func_normal_abspath_altnamespace=/ - ;; - /*) - # Absolute path, do nothing. - ;; - *) - # Relative path, prepend $cwd. - func_normal_abspath_tpath=`pwd`/$func_normal_abspath_tpath - ;; - esac - - # Cancel out all the simple stuff to save iterations. We also want - # the path to end with a slash for ease of parsing, so make sure - # there is one (and only one) here. - func_normal_abspath_tpath=`$ECHO "$func_normal_abspath_tpath" | $SED \ - -e "$_G_removedotparts" -e "$_G_collapseslashes" -e "$_G_finalslash"` - while :; do - # Processed it all yet? - if test / = "$func_normal_abspath_tpath"; then - # If we ascended to the root using ".." the result may be empty now. - if test -z "$func_normal_abspath_result"; then - func_normal_abspath_result=/ - fi - break - fi - func_normal_abspath_tcomponent=`$ECHO "$func_normal_abspath_tpath" | $SED \ - -e "$_G_pathcar"` - func_normal_abspath_tpath=`$ECHO "$func_normal_abspath_tpath" | $SED \ - -e "$_G_pathcdr"` - # Figure out what to do with it - case $func_normal_abspath_tcomponent in - "") - # Trailing empty path component, ignore it. - ;; - ..) - # Parent dir; strip last assembled component from result. - func_dirname "$func_normal_abspath_result" - func_normal_abspath_result=$func_dirname_result - ;; - *) - # Actual path component, append it. - func_append func_normal_abspath_result "/$func_normal_abspath_tcomponent" - ;; - esac - done - # Restore leading double-slash if one was found on entry. - func_normal_abspath_result=$func_normal_abspath_altnamespace$func_normal_abspath_result -} - - -# func_notquiet ARG... -# -------------------- -# Echo program name prefixed message only when not in quiet mode. -func_notquiet () -{ - $debug_cmd - - $opt_quiet || func_echo ${1+"$@"} - - # A bug in bash halts the script if the last line of a function - # fails when set -e is in force, so we need another command to - # work around that: - : -} - - -# func_relative_path SRCDIR DSTDIR -# -------------------------------- -# Set func_relative_path_result to the relative path from SRCDIR to DSTDIR. -func_relative_path () -{ - $debug_cmd - - func_relative_path_result= - func_normal_abspath "$1" - func_relative_path_tlibdir=$func_normal_abspath_result - func_normal_abspath "$2" - func_relative_path_tbindir=$func_normal_abspath_result - - # Ascend the tree starting from libdir - while :; do - # check if we have found a prefix of bindir - case $func_relative_path_tbindir in - $func_relative_path_tlibdir) - # found an exact match - func_relative_path_tcancelled= - break - ;; - $func_relative_path_tlibdir*) - # found a matching prefix - func_stripname "$func_relative_path_tlibdir" '' "$func_relative_path_tbindir" - func_relative_path_tcancelled=$func_stripname_result - if test -z "$func_relative_path_result"; then - func_relative_path_result=. - fi - break - ;; - *) - func_dirname $func_relative_path_tlibdir - func_relative_path_tlibdir=$func_dirname_result - if test -z "$func_relative_path_tlibdir"; then - # Have to descend all the way to the root! - func_relative_path_result=../$func_relative_path_result - func_relative_path_tcancelled=$func_relative_path_tbindir - break - fi - func_relative_path_result=../$func_relative_path_result - ;; - esac - done - - # Now calculate path; take care to avoid doubling-up slashes. - func_stripname '' '/' "$func_relative_path_result" - func_relative_path_result=$func_stripname_result - func_stripname '/' '/' "$func_relative_path_tcancelled" - if test -n "$func_stripname_result"; then - func_append func_relative_path_result "/$func_stripname_result" - fi - - # Normalisation. If bindir is libdir, return '.' else relative path. - if test -n "$func_relative_path_result"; then - func_stripname './' '' "$func_relative_path_result" - func_relative_path_result=$func_stripname_result - fi - - test -n "$func_relative_path_result" || func_relative_path_result=. - - : -} - - -# func_quote_for_eval ARG... -# -------------------------- -# Aesthetically quote ARGs to be evaled later. -# This function returns two values: -# i) func_quote_for_eval_result -# double-quoted, suitable for a subsequent eval -# ii) func_quote_for_eval_unquoted_result -# has all characters that are still active within double -# quotes backslashified. -func_quote_for_eval () -{ - $debug_cmd - - func_quote_for_eval_unquoted_result= - func_quote_for_eval_result= - while test 0 -lt $#; do - case $1 in - *[\\\`\"\$]*) - _G_unquoted_arg=`printf '%s\n' "$1" |$SED "$sed_quote_subst"` ;; - *) - _G_unquoted_arg=$1 ;; - esac - if test -n "$func_quote_for_eval_unquoted_result"; then - func_append func_quote_for_eval_unquoted_result " $_G_unquoted_arg" - else - func_append func_quote_for_eval_unquoted_result "$_G_unquoted_arg" - fi - - case $_G_unquoted_arg in - # Double-quote args containing shell metacharacters to delay - # word splitting, command substitution and variable expansion - # for a subsequent eval. - # Many Bourne shells cannot handle close brackets correctly - # in scan sets, so we specify it separately. - *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") - _G_quoted_arg=\"$_G_unquoted_arg\" - ;; - *) - _G_quoted_arg=$_G_unquoted_arg - ;; - esac - - if test -n "$func_quote_for_eval_result"; then - func_append func_quote_for_eval_result " $_G_quoted_arg" - else - func_append func_quote_for_eval_result "$_G_quoted_arg" - fi - shift - done -} - - -# func_quote_for_expand ARG -# ------------------------- -# Aesthetically quote ARG to be evaled later; same as above, -# but do not quote variable references. -func_quote_for_expand () -{ - $debug_cmd - - case $1 in - *[\\\`\"]*) - _G_arg=`$ECHO "$1" | $SED \ - -e "$sed_double_quote_subst" -e "$sed_double_backslash"` ;; - *) - _G_arg=$1 ;; - esac - - case $_G_arg in - # Double-quote args containing shell metacharacters to delay - # word splitting and command substitution for a subsequent eval. - # Many Bourne shells cannot handle close brackets correctly - # in scan sets, so we specify it separately. - *[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \ ]*|*]*|"") - _G_arg=\"$_G_arg\" - ;; - esac - - func_quote_for_expand_result=$_G_arg -} - - -# func_stripname PREFIX SUFFIX NAME -# --------------------------------- -# strip PREFIX and SUFFIX from NAME, and store in func_stripname_result. -# PREFIX and SUFFIX must not contain globbing or regex special -# characters, hashes, percent signs, but SUFFIX may contain a leading -# dot (in which case that matches only a dot). -if test yes = "$_G_HAVE_XSI_OPS"; then - eval 'func_stripname () - { - $debug_cmd - - # pdksh 5.2.14 does not do ${X%$Y} correctly if both X and Y are - # positional parameters, so assign one to ordinary variable first. - func_stripname_result=$3 - func_stripname_result=${func_stripname_result#"$1"} - func_stripname_result=${func_stripname_result%"$2"} - }' -else - func_stripname () - { - $debug_cmd - - case $2 in - .*) func_stripname_result=`$ECHO "$3" | $SED -e "s%^$1%%" -e "s%\\\\$2\$%%"`;; - *) func_stripname_result=`$ECHO "$3" | $SED -e "s%^$1%%" -e "s%$2\$%%"`;; - esac - } -fi - - -# func_show_eval CMD [FAIL_EXP] -# ----------------------------- -# Unless opt_quiet is true, then output CMD. Then, if opt_dryrun is -# not true, evaluate CMD. If the evaluation of CMD fails, and FAIL_EXP -# is given, then evaluate it. -func_show_eval () -{ - $debug_cmd - - _G_cmd=$1 - _G_fail_exp=${2-':'} - - func_quote_for_expand "$_G_cmd" - eval "func_notquiet $func_quote_for_expand_result" - - $opt_dry_run || { - eval "$_G_cmd" - _G_status=$? - if test 0 -ne "$_G_status"; then - eval "(exit $_G_status); $_G_fail_exp" - fi - } -} - - -# func_show_eval_locale CMD [FAIL_EXP] -# ------------------------------------ -# Unless opt_quiet is true, then output CMD. Then, if opt_dryrun is -# not true, evaluate CMD. If the evaluation of CMD fails, and FAIL_EXP -# is given, then evaluate it. Use the saved locale for evaluation. -func_show_eval_locale () -{ - $debug_cmd - - _G_cmd=$1 - _G_fail_exp=${2-':'} - - $opt_quiet || { - func_quote_for_expand "$_G_cmd" - eval "func_echo $func_quote_for_expand_result" - } - - $opt_dry_run || { - eval "$_G_user_locale - $_G_cmd" - _G_status=$? - eval "$_G_safe_locale" - if test 0 -ne "$_G_status"; then - eval "(exit $_G_status); $_G_fail_exp" - fi - } -} - - -# func_tr_sh -# ---------- -# Turn $1 into a string suitable for a shell variable name. -# Result is stored in $func_tr_sh_result. All characters -# not in the set a-zA-Z0-9_ are replaced with '_'. Further, -# if $1 begins with a digit, a '_' is prepended as well. -func_tr_sh () -{ - $debug_cmd - - case $1 in - [0-9]* | *[!a-zA-Z0-9_]*) - func_tr_sh_result=`$ECHO "$1" | $SED -e 's/^\([0-9]\)/_\1/' -e 's/[^a-zA-Z0-9_]/_/g'` - ;; - * ) - func_tr_sh_result=$1 - ;; - esac -} - - -# func_verbose ARG... -# ------------------- -# Echo program name prefixed message in verbose mode only. -func_verbose () -{ - $debug_cmd - - $opt_verbose && func_echo "$*" - - : -} - - -# func_warn_and_continue ARG... -# ----------------------------- -# Echo program name prefixed warning message to standard error. -func_warn_and_continue () -{ - $debug_cmd - - $require_term_colors - - func_echo_infix_1 "${tc_red}warning$tc_reset" "$*" >&2 -} - - -# func_warning CATEGORY ARG... -# ---------------------------- -# Echo program name prefixed warning message to standard error. Warning -# messages can be filtered according to CATEGORY, where this function -# elides messages where CATEGORY is not listed in the global variable -# 'opt_warning_types'. -func_warning () -{ - $debug_cmd - - # CATEGORY must be in the warning_categories list! - case " $warning_categories " in - *" $1 "*) ;; - *) func_internal_error "invalid warning category '$1'" ;; - esac - - _G_category=$1 - shift - - case " $opt_warning_types " in - *" $_G_category "*) $warning_func ${1+"$@"} ;; - esac -} - - -# func_sort_ver VER1 VER2 -# ----------------------- -# 'sort -V' is not generally available. -# Note this deviates from the version comparison in automake -# in that it treats 1.5 < 1.5.0, and treats 1.4.4a < 1.4-p3a -# but this should suffice as we won't be specifying old -# version formats or redundant trailing .0 in bootstrap.conf. -# If we did want full compatibility then we should probably -# use m4_version_compare from autoconf. -func_sort_ver () -{ - $debug_cmd - - printf '%s\n%s\n' "$1" "$2" \ - | sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4n -k 5,5n -k 6,6n -k 7,7n -k 8,8n -k 9,9n -} - -# func_lt_ver PREV CURR -# --------------------- -# Return true if PREV and CURR are in the correct order according to -# func_sort_ver, otherwise false. Use it like this: -# -# func_lt_ver "$prev_ver" "$proposed_ver" || func_fatal_error "..." -func_lt_ver () -{ - $debug_cmd - - test "x$1" = x`func_sort_ver "$1" "$2" | $SED 1q` -} - - -# Local variables: -# mode: shell-script -# sh-indentation: 2 -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-pattern: "10/scriptversion=%:y-%02m-%02d.%02H; # UTC" -# time-stamp-time-zone: "UTC" -# End: -#! /bin/sh - -# Set a version string for this script. -scriptversion=2014-01-07.03; # UTC - -# A portable, pluggable option parser for Bourne shell. -# Written by Gary V. Vaughan, 2010 - -# Copyright (C) 2010-2015 Free Software Foundation, Inc. -# This is free software; see the source for copying conditions. There is NO -# warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Please report bugs or propose patches to gary@gnu.org. - - -## ------ ## -## Usage. ## -## ------ ## - -# This file is a library for parsing options in your shell scripts along -# with assorted other useful supporting features that you can make use -# of too. -# -# For the simplest scripts you might need only: -# -# #!/bin/sh -# . relative/path/to/funclib.sh -# . relative/path/to/options-parser -# scriptversion=1.0 -# func_options ${1+"$@"} -# eval set dummy "$func_options_result"; shift -# ...rest of your script... -# -# In order for the '--version' option to work, you will need to have a -# suitably formatted comment like the one at the top of this file -# starting with '# Written by ' and ending with '# warranty; '. -# -# For '-h' and '--help' to work, you will also need a one line -# description of your script's purpose in a comment directly above the -# '# Written by ' line, like the one at the top of this file. -# -# The default options also support '--debug', which will turn on shell -# execution tracing (see the comment above debug_cmd below for another -# use), and '--verbose' and the func_verbose function to allow your script -# to display verbose messages only when your user has specified -# '--verbose'. -# -# After sourcing this file, you can plug processing for additional -# options by amending the variables from the 'Configuration' section -# below, and following the instructions in the 'Option parsing' -# section further down. - -## -------------- ## -## Configuration. ## -## -------------- ## - -# You should override these variables in your script after sourcing this -# file so that they reflect the customisations you have added to the -# option parser. - -# The usage line for option parsing errors and the start of '-h' and -# '--help' output messages. You can embed shell variables for delayed -# expansion at the time the message is displayed, but you will need to -# quote other shell meta-characters carefully to prevent them being -# expanded when the contents are evaled. -usage='$progpath [OPTION]...' - -# Short help message in response to '-h' and '--help'. Add to this or -# override it after sourcing this library to reflect the full set of -# options your script accepts. -usage_message="\ - --debug enable verbose shell tracing - -W, --warnings=CATEGORY - report the warnings falling in CATEGORY [all] - -v, --verbose verbosely report processing - --version print version information and exit - -h, --help print short or long help message and exit -" - -# Additional text appended to 'usage_message' in response to '--help'. -long_help_message=" -Warning categories include: - 'all' show all warnings - 'none' turn off all the warnings - 'error' warnings are treated as fatal errors" - -# Help message printed before fatal option parsing errors. -fatal_help="Try '\$progname --help' for more information." - - - -## ------------------------- ## -## Hook function management. ## -## ------------------------- ## - -# This section contains functions for adding, removing, and running hooks -# to the main code. A hook is just a named list of of function, that can -# be run in order later on. - -# func_hookable FUNC_NAME -# ----------------------- -# Declare that FUNC_NAME will run hooks added with -# 'func_add_hook FUNC_NAME ...'. -func_hookable () -{ - $debug_cmd - - func_append hookable_fns " $1" -} - - -# func_add_hook FUNC_NAME HOOK_FUNC -# --------------------------------- -# Request that FUNC_NAME call HOOK_FUNC before it returns. FUNC_NAME must -# first have been declared "hookable" by a call to 'func_hookable'. -func_add_hook () -{ - $debug_cmd - - case " $hookable_fns " in - *" $1 "*) ;; - *) func_fatal_error "'$1' does not accept hook functions." ;; - esac - - eval func_append ${1}_hooks '" $2"' -} - - -# func_remove_hook FUNC_NAME HOOK_FUNC -# ------------------------------------ -# Remove HOOK_FUNC from the list of functions called by FUNC_NAME. -func_remove_hook () -{ - $debug_cmd - - eval ${1}_hooks='`$ECHO "\$'$1'_hooks" |$SED "s| '$2'||"`' -} - - -# func_run_hooks FUNC_NAME [ARG]... -# --------------------------------- -# Run all hook functions registered to FUNC_NAME. -# It is assumed that the list of hook functions contains nothing more -# than a whitespace-delimited list of legal shell function names, and -# no effort is wasted trying to catch shell meta-characters or preserve -# whitespace. -func_run_hooks () -{ - $debug_cmd - - case " $hookable_fns " in - *" $1 "*) ;; - *) func_fatal_error "'$1' does not support hook funcions.n" ;; - esac - - eval _G_hook_fns=\$$1_hooks; shift - - for _G_hook in $_G_hook_fns; do - eval $_G_hook '"$@"' - - # store returned options list back into positional - # parameters for next 'cmd' execution. - eval _G_hook_result=\$${_G_hook}_result - eval set dummy "$_G_hook_result"; shift - done - - func_quote_for_eval ${1+"$@"} - func_run_hooks_result=$func_quote_for_eval_result -} - - - -## --------------- ## -## Option parsing. ## -## --------------- ## - -# In order to add your own option parsing hooks, you must accept the -# full positional parameter list in your hook function, remove any -# options that you action, and then pass back the remaining unprocessed -# options in '_result', escaped suitably for -# 'eval'. Like this: -# -# my_options_prep () -# { -# $debug_cmd -# -# # Extend the existing usage message. -# usage_message=$usage_message' -# -s, --silent don'\''t print informational messages -# ' -# -# func_quote_for_eval ${1+"$@"} -# my_options_prep_result=$func_quote_for_eval_result -# } -# func_add_hook func_options_prep my_options_prep -# -# -# my_silent_option () -# { -# $debug_cmd -# -# # Note that for efficiency, we parse as many options as we can -# # recognise in a loop before passing the remainder back to the -# # caller on the first unrecognised argument we encounter. -# while test $# -gt 0; do -# opt=$1; shift -# case $opt in -# --silent|-s) opt_silent=: ;; -# # Separate non-argument short options: -# -s*) func_split_short_opt "$_G_opt" -# set dummy "$func_split_short_opt_name" \ -# "-$func_split_short_opt_arg" ${1+"$@"} -# shift -# ;; -# *) set dummy "$_G_opt" "$*"; shift; break ;; -# esac -# done -# -# func_quote_for_eval ${1+"$@"} -# my_silent_option_result=$func_quote_for_eval_result -# } -# func_add_hook func_parse_options my_silent_option -# -# -# my_option_validation () -# { -# $debug_cmd -# -# $opt_silent && $opt_verbose && func_fatal_help "\ -# '--silent' and '--verbose' options are mutually exclusive." -# -# func_quote_for_eval ${1+"$@"} -# my_option_validation_result=$func_quote_for_eval_result -# } -# func_add_hook func_validate_options my_option_validation -# -# You'll alse need to manually amend $usage_message to reflect the extra -# options you parse. It's preferable to append if you can, so that -# multiple option parsing hooks can be added safely. - - -# func_options [ARG]... -# --------------------- -# All the functions called inside func_options are hookable. See the -# individual implementations for details. -func_hookable func_options -func_options () -{ - $debug_cmd - - func_options_prep ${1+"$@"} - eval func_parse_options \ - ${func_options_prep_result+"$func_options_prep_result"} - eval func_validate_options \ - ${func_parse_options_result+"$func_parse_options_result"} - - eval func_run_hooks func_options \ - ${func_validate_options_result+"$func_validate_options_result"} - - # save modified positional parameters for caller - func_options_result=$func_run_hooks_result -} - - -# func_options_prep [ARG]... -# -------------------------- -# All initialisations required before starting the option parse loop. -# Note that when calling hook functions, we pass through the list of -# positional parameters. If a hook function modifies that list, and -# needs to propogate that back to rest of this script, then the complete -# modified list must be put in 'func_run_hooks_result' before -# returning. -func_hookable func_options_prep -func_options_prep () -{ - $debug_cmd - - # Option defaults: - opt_verbose=false - opt_warning_types= - - func_run_hooks func_options_prep ${1+"$@"} - - # save modified positional parameters for caller - func_options_prep_result=$func_run_hooks_result -} - - -# func_parse_options [ARG]... -# --------------------------- -# The main option parsing loop. -func_hookable func_parse_options -func_parse_options () -{ - $debug_cmd - - func_parse_options_result= - - # this just eases exit handling - while test $# -gt 0; do - # Defer to hook functions for initial option parsing, so they - # get priority in the event of reusing an option name. - func_run_hooks func_parse_options ${1+"$@"} - - # Adjust func_parse_options positional parameters to match - eval set dummy "$func_run_hooks_result"; shift - - # Break out of the loop if we already parsed every option. - test $# -gt 0 || break - - _G_opt=$1 - shift - case $_G_opt in - --debug|-x) debug_cmd='set -x' - func_echo "enabling shell trace mode" - $debug_cmd - ;; - - --no-warnings|--no-warning|--no-warn) - set dummy --warnings none ${1+"$@"} - shift - ;; - - --warnings|--warning|-W) - test $# = 0 && func_missing_arg $_G_opt && break - case " $warning_categories $1" in - *" $1 "*) - # trailing space prevents matching last $1 above - func_append_uniq opt_warning_types " $1" - ;; - *all) - opt_warning_types=$warning_categories - ;; - *none) - opt_warning_types=none - warning_func=: - ;; - *error) - opt_warning_types=$warning_categories - warning_func=func_fatal_error - ;; - *) - func_fatal_error \ - "unsupported warning category: '$1'" - ;; - esac - shift - ;; - - --verbose|-v) opt_verbose=: ;; - --version) func_version ;; - -\?|-h) func_usage ;; - --help) func_help ;; - - # Separate optargs to long options (plugins may need this): - --*=*) func_split_equals "$_G_opt" - set dummy "$func_split_equals_lhs" \ - "$func_split_equals_rhs" ${1+"$@"} - shift - ;; - - # Separate optargs to short options: - -W*) - func_split_short_opt "$_G_opt" - set dummy "$func_split_short_opt_name" \ - "$func_split_short_opt_arg" ${1+"$@"} - shift - ;; - - # Separate non-argument short options: - -\?*|-h*|-v*|-x*) - func_split_short_opt "$_G_opt" - set dummy "$func_split_short_opt_name" \ - "-$func_split_short_opt_arg" ${1+"$@"} - shift - ;; - - --) break ;; - -*) func_fatal_help "unrecognised option: '$_G_opt'" ;; - *) set dummy "$_G_opt" ${1+"$@"}; shift; break ;; - esac - done - - # save modified positional parameters for caller - func_quote_for_eval ${1+"$@"} - func_parse_options_result=$func_quote_for_eval_result -} - - -# func_validate_options [ARG]... -# ------------------------------ -# Perform any sanity checks on option settings and/or unconsumed -# arguments. -func_hookable func_validate_options -func_validate_options () -{ - $debug_cmd - - # Display all warnings if -W was not given. - test -n "$opt_warning_types" || opt_warning_types=" $warning_categories" - - func_run_hooks func_validate_options ${1+"$@"} - - # Bail if the options were screwed! - $exit_cmd $EXIT_FAILURE - - # save modified positional parameters for caller - func_validate_options_result=$func_run_hooks_result -} - - - -## ----------------- ## -## Helper functions. ## -## ----------------- ## - -# This section contains the helper functions used by the rest of the -# hookable option parser framework in ascii-betical order. - - -# func_fatal_help ARG... -# ---------------------- -# Echo program name prefixed message to standard error, followed by -# a help hint, and exit. -func_fatal_help () -{ - $debug_cmd - - eval \$ECHO \""Usage: $usage"\" - eval \$ECHO \""$fatal_help"\" - func_error ${1+"$@"} - exit $EXIT_FAILURE -} - - -# func_help -# --------- -# Echo long help message to standard output and exit. -func_help () -{ - $debug_cmd - - func_usage_message - $ECHO "$long_help_message" - exit 0 -} - - -# func_missing_arg ARGNAME -# ------------------------ -# Echo program name prefixed message to standard error and set global -# exit_cmd. -func_missing_arg () -{ - $debug_cmd - - func_error "Missing argument for '$1'." - exit_cmd=exit -} - - -# func_split_equals STRING -# ------------------------ -# Set func_split_equals_lhs and func_split_equals_rhs shell variables after -# splitting STRING at the '=' sign. -test -z "$_G_HAVE_XSI_OPS" \ - && (eval 'x=a/b/c; - test 5aa/bb/cc = "${#x}${x%%/*}${x%/*}${x#*/}${x##*/}"') 2>/dev/null \ - && _G_HAVE_XSI_OPS=yes - -if test yes = "$_G_HAVE_XSI_OPS" -then - # This is an XSI compatible shell, allowing a faster implementation... - eval 'func_split_equals () - { - $debug_cmd - - func_split_equals_lhs=${1%%=*} - func_split_equals_rhs=${1#*=} - test "x$func_split_equals_lhs" = "x$1" \ - && func_split_equals_rhs= - }' -else - # ...otherwise fall back to using expr, which is often a shell builtin. - func_split_equals () - { - $debug_cmd - - func_split_equals_lhs=`expr "x$1" : 'x\([^=]*\)'` - func_split_equals_rhs= - test "x$func_split_equals_lhs" = "x$1" \ - || func_split_equals_rhs=`expr "x$1" : 'x[^=]*=\(.*\)$'` - } -fi #func_split_equals - - -# func_split_short_opt SHORTOPT -# ----------------------------- -# Set func_split_short_opt_name and func_split_short_opt_arg shell -# variables after splitting SHORTOPT after the 2nd character. -if test yes = "$_G_HAVE_XSI_OPS" -then - # This is an XSI compatible shell, allowing a faster implementation... - eval 'func_split_short_opt () - { - $debug_cmd - - func_split_short_opt_arg=${1#??} - func_split_short_opt_name=${1%"$func_split_short_opt_arg"} - }' -else - # ...otherwise fall back to using expr, which is often a shell builtin. - func_split_short_opt () - { - $debug_cmd - - func_split_short_opt_name=`expr "x$1" : 'x-\(.\)'` - func_split_short_opt_arg=`expr "x$1" : 'x-.\(.*\)$'` - } -fi #func_split_short_opt - - -# func_usage -# ---------- -# Echo short help message to standard output and exit. -func_usage () -{ - $debug_cmd - - func_usage_message - $ECHO "Run '$progname --help |${PAGER-more}' for full usage" - exit 0 -} - - -# func_usage_message -# ------------------ -# Echo short help message to standard output. -func_usage_message () -{ - $debug_cmd - - eval \$ECHO \""Usage: $usage"\" - echo - $SED -n 's|^# || - /^Written by/{ - x;p;x - } - h - /^Written by/q' < "$progpath" - echo - eval \$ECHO \""$usage_message"\" -} - - -# func_version -# ------------ -# Echo version message to standard output and exit. -func_version () -{ - $debug_cmd - - printf '%s\n' "$progname $scriptversion" - $SED -n ' - /(C)/!b go - :more - /\./!{ - N - s|\n# | | - b more - } - :go - /^# Written by /,/# warranty; / { - s|^# || - s|^# *$|| - s|\((C)\)[ 0-9,-]*[ ,-]\([1-9][0-9]* \)|\1 \2| - p - } - /^# Written by / { - s|^# || - p - } - /^warranty; /q' < "$progpath" - - exit $? -} - - -# Local variables: -# mode: shell-script -# sh-indentation: 2 -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-pattern: "10/scriptversion=%:y-%02m-%02d.%02H; # UTC" -# time-stamp-time-zone: "UTC" -# End: - -# Set a version string. -scriptversion='(GNU libtool) 2.4.6' - - -# func_echo ARG... -# ---------------- -# Libtool also displays the current mode in messages, so override -# funclib.sh func_echo with this custom definition. -func_echo () -{ - $debug_cmd - - _G_message=$* - - func_echo_IFS=$IFS - IFS=$nl - for _G_line in $_G_message; do - IFS=$func_echo_IFS - $ECHO "$progname${opt_mode+: $opt_mode}: $_G_line" - done - IFS=$func_echo_IFS -} - - -# func_warning ARG... -# ------------------- -# Libtool warnings are not categorized, so override funclib.sh -# func_warning with this simpler definition. -func_warning () -{ - $debug_cmd - - $warning_func ${1+"$@"} -} - - -## ---------------- ## -## Options parsing. ## -## ---------------- ## - -# Hook in the functions to make sure our own options are parsed during -# the option parsing loop. - -usage='$progpath [OPTION]... [MODE-ARG]...' - -# Short help message in response to '-h'. -usage_message="Options: - --config show all configuration variables - --debug enable verbose shell tracing - -n, --dry-run display commands without modifying any files - --features display basic configuration information and exit - --mode=MODE use operation mode MODE - --no-warnings equivalent to '-Wnone' - --preserve-dup-deps don't remove duplicate dependency libraries - --quiet, --silent don't print informational messages - --tag=TAG use configuration variables from tag TAG - -v, --verbose print more informational messages than default - --version print version information - -W, --warnings=CATEGORY report the warnings falling in CATEGORY [all] - -h, --help, --help-all print short, long, or detailed help message -" - -# Additional text appended to 'usage_message' in response to '--help'. -func_help () -{ - $debug_cmd - - func_usage_message - $ECHO "$long_help_message - -MODE must be one of the following: - - clean remove files from the build directory - compile compile a source file into a libtool object - execute automatically set library path, then run a program - finish complete the installation of libtool libraries - install install libraries or executables - link create a library or an executable - uninstall remove libraries from an installed directory - -MODE-ARGS vary depending on the MODE. When passed as first option, -'--mode=MODE' may be abbreviated as 'MODE' or a unique abbreviation of that. -Try '$progname --help --mode=MODE' for a more detailed description of MODE. - -When reporting a bug, please describe a test case to reproduce it and -include the following information: - - host-triplet: $host - shell: $SHELL - compiler: $LTCC - compiler flags: $LTCFLAGS - linker: $LD (gnu? $with_gnu_ld) - version: $progname (GNU libtool) 2.4.6 - automake: `($AUTOMAKE --version) 2>/dev/null |$SED 1q` - autoconf: `($AUTOCONF --version) 2>/dev/null |$SED 1q` - -Report bugs to . -GNU libtool home page: . -General help using GNU software: ." - exit 0 -} - - -# func_lo2o OBJECT-NAME -# --------------------- -# Transform OBJECT-NAME from a '.lo' suffix to the platform specific -# object suffix. - -lo2o=s/\\.lo\$/.$objext/ -o2lo=s/\\.$objext\$/.lo/ - -if test yes = "$_G_HAVE_XSI_OPS"; then - eval 'func_lo2o () - { - case $1 in - *.lo) func_lo2o_result=${1%.lo}.$objext ;; - * ) func_lo2o_result=$1 ;; - esac - }' - - # func_xform LIBOBJ-OR-SOURCE - # --------------------------- - # Transform LIBOBJ-OR-SOURCE from a '.o' or '.c' (or otherwise) - # suffix to a '.lo' libtool-object suffix. - eval 'func_xform () - { - func_xform_result=${1%.*}.lo - }' -else - # ...otherwise fall back to using sed. - func_lo2o () - { - func_lo2o_result=`$ECHO "$1" | $SED "$lo2o"` - } - - func_xform () - { - func_xform_result=`$ECHO "$1" | $SED 's|\.[^.]*$|.lo|'` - } -fi - - -# func_fatal_configuration ARG... -# ------------------------------- -# Echo program name prefixed message to standard error, followed by -# a configuration failure hint, and exit. -func_fatal_configuration () -{ - func__fatal_error ${1+"$@"} \ - "See the $PACKAGE documentation for more information." \ - "Fatal configuration error." -} - - -# func_config -# ----------- -# Display the configuration for all the tags in this script. -func_config () -{ - re_begincf='^# ### BEGIN LIBTOOL' - re_endcf='^# ### END LIBTOOL' - - # Default configuration. - $SED "1,/$re_begincf CONFIG/d;/$re_endcf CONFIG/,\$d" < "$progpath" - - # Now print the configurations for the tags. - for tagname in $taglist; do - $SED -n "/$re_begincf TAG CONFIG: $tagname\$/,/$re_endcf TAG CONFIG: $tagname\$/p" < "$progpath" - done - - exit $? -} - - -# func_features -# ------------- -# Display the features supported by this script. -func_features () -{ - echo "host: $host" - if test yes = "$build_libtool_libs"; then - echo "enable shared libraries" - else - echo "disable shared libraries" - fi - if test yes = "$build_old_libs"; then - echo "enable static libraries" - else - echo "disable static libraries" - fi - - exit $? -} - - -# func_enable_tag TAGNAME -# ----------------------- -# Verify that TAGNAME is valid, and either flag an error and exit, or -# enable the TAGNAME tag. We also add TAGNAME to the global $taglist -# variable here. -func_enable_tag () -{ - # Global variable: - tagname=$1 - - re_begincf="^# ### BEGIN LIBTOOL TAG CONFIG: $tagname\$" - re_endcf="^# ### END LIBTOOL TAG CONFIG: $tagname\$" - sed_extractcf=/$re_begincf/,/$re_endcf/p - - # Validate tagname. - case $tagname in - *[!-_A-Za-z0-9,/]*) - func_fatal_error "invalid tag name: $tagname" - ;; - esac - - # Don't test for the "default" C tag, as we know it's - # there but not specially marked. - case $tagname in - CC) ;; - *) - if $GREP "$re_begincf" "$progpath" >/dev/null 2>&1; then - taglist="$taglist $tagname" - - # Evaluate the configuration. Be careful to quote the path - # and the sed script, to avoid splitting on whitespace, but - # also don't use non-portable quotes within backquotes within - # quotes we have to do it in 2 steps: - extractedcf=`$SED -n -e "$sed_extractcf" < "$progpath"` - eval "$extractedcf" - else - func_error "ignoring unknown tag $tagname" - fi - ;; - esac -} - - -# func_check_version_match -# ------------------------ -# Ensure that we are using m4 macros, and libtool script from the same -# release of libtool. -func_check_version_match () -{ - if test "$package_revision" != "$macro_revision"; then - if test "$VERSION" != "$macro_version"; then - if test -z "$macro_version"; then - cat >&2 <<_LT_EOF -$progname: Version mismatch error. This is $PACKAGE $VERSION, but the -$progname: definition of this LT_INIT comes from an older release. -$progname: You should recreate aclocal.m4 with macros from $PACKAGE $VERSION -$progname: and run autoconf again. -_LT_EOF - else - cat >&2 <<_LT_EOF -$progname: Version mismatch error. This is $PACKAGE $VERSION, but the -$progname: definition of this LT_INIT comes from $PACKAGE $macro_version. -$progname: You should recreate aclocal.m4 with macros from $PACKAGE $VERSION -$progname: and run autoconf again. -_LT_EOF - fi - else - cat >&2 <<_LT_EOF -$progname: Version mismatch error. This is $PACKAGE $VERSION, revision $package_revision, -$progname: but the definition of this LT_INIT comes from revision $macro_revision. -$progname: You should recreate aclocal.m4 with macros from revision $package_revision -$progname: of $PACKAGE $VERSION and run autoconf again. -_LT_EOF - fi - - exit $EXIT_MISMATCH - fi -} - - -# libtool_options_prep [ARG]... -# ----------------------------- -# Preparation for options parsed by libtool. -libtool_options_prep () -{ - $debug_mode - - # Option defaults: - opt_config=false - opt_dlopen= - opt_dry_run=false - opt_help=false - opt_mode= - opt_preserve_dup_deps=false - opt_quiet=false - - nonopt= - preserve_args= - - # Shorthand for --mode=foo, only valid as the first argument - case $1 in - clean|clea|cle|cl) - shift; set dummy --mode clean ${1+"$@"}; shift - ;; - compile|compil|compi|comp|com|co|c) - shift; set dummy --mode compile ${1+"$@"}; shift - ;; - execute|execut|execu|exec|exe|ex|e) - shift; set dummy --mode execute ${1+"$@"}; shift - ;; - finish|finis|fini|fin|fi|f) - shift; set dummy --mode finish ${1+"$@"}; shift - ;; - install|instal|insta|inst|ins|in|i) - shift; set dummy --mode install ${1+"$@"}; shift - ;; - link|lin|li|l) - shift; set dummy --mode link ${1+"$@"}; shift - ;; - uninstall|uninstal|uninsta|uninst|unins|unin|uni|un|u) - shift; set dummy --mode uninstall ${1+"$@"}; shift - ;; - esac - - # Pass back the list of options. - func_quote_for_eval ${1+"$@"} - libtool_options_prep_result=$func_quote_for_eval_result -} -func_add_hook func_options_prep libtool_options_prep - - -# libtool_parse_options [ARG]... -# --------------------------------- -# Provide handling for libtool specific options. -libtool_parse_options () -{ - $debug_cmd - - # Perform our own loop to consume as many options as possible in - # each iteration. - while test $# -gt 0; do - _G_opt=$1 - shift - case $_G_opt in - --dry-run|--dryrun|-n) - opt_dry_run=: - ;; - - --config) func_config ;; - - --dlopen|-dlopen) - opt_dlopen="${opt_dlopen+$opt_dlopen -}$1" - shift - ;; - - --preserve-dup-deps) - opt_preserve_dup_deps=: ;; - - --features) func_features ;; - - --finish) set dummy --mode finish ${1+"$@"}; shift ;; - - --help) opt_help=: ;; - - --help-all) opt_help=': help-all' ;; - - --mode) test $# = 0 && func_missing_arg $_G_opt && break - opt_mode=$1 - case $1 in - # Valid mode arguments: - clean|compile|execute|finish|install|link|relink|uninstall) ;; - - # Catch anything else as an error - *) func_error "invalid argument for $_G_opt" - exit_cmd=exit - break - ;; - esac - shift - ;; - - --no-silent|--no-quiet) - opt_quiet=false - func_append preserve_args " $_G_opt" - ;; - - --no-warnings|--no-warning|--no-warn) - opt_warning=false - func_append preserve_args " $_G_opt" - ;; - - --no-verbose) - opt_verbose=false - func_append preserve_args " $_G_opt" - ;; - - --silent|--quiet) - opt_quiet=: - opt_verbose=false - func_append preserve_args " $_G_opt" - ;; - - --tag) test $# = 0 && func_missing_arg $_G_opt && break - opt_tag=$1 - func_append preserve_args " $_G_opt $1" - func_enable_tag "$1" - shift - ;; - - --verbose|-v) opt_quiet=false - opt_verbose=: - func_append preserve_args " $_G_opt" - ;; - - # An option not handled by this hook function: - *) set dummy "$_G_opt" ${1+"$@"}; shift; break ;; - esac - done - - - # save modified positional parameters for caller - func_quote_for_eval ${1+"$@"} - libtool_parse_options_result=$func_quote_for_eval_result -} -func_add_hook func_parse_options libtool_parse_options - - - -# libtool_validate_options [ARG]... -# --------------------------------- -# Perform any sanity checks on option settings and/or unconsumed -# arguments. -libtool_validate_options () -{ - # save first non-option argument - if test 0 -lt $#; then - nonopt=$1 - shift - fi - - # preserve --debug - test : = "$debug_cmd" || func_append preserve_args " --debug" - - case $host in - # Solaris2 added to fix http://debbugs.gnu.org/cgi/bugreport.cgi?bug=16452 - # see also: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=59788 - *cygwin* | *mingw* | *pw32* | *cegcc* | *solaris2* | *os2*) - # don't eliminate duplications in $postdeps and $predeps - opt_duplicate_compiler_generated_deps=: - ;; - *) - opt_duplicate_compiler_generated_deps=$opt_preserve_dup_deps - ;; - esac - - $opt_help || { - # Sanity checks first: - func_check_version_match - - test yes != "$build_libtool_libs" \ - && test yes != "$build_old_libs" \ - && func_fatal_configuration "not configured to build any kind of library" - - # Darwin sucks - eval std_shrext=\"$shrext_cmds\" - - # Only execute mode is allowed to have -dlopen flags. - if test -n "$opt_dlopen" && test execute != "$opt_mode"; then - func_error "unrecognized option '-dlopen'" - $ECHO "$help" 1>&2 - exit $EXIT_FAILURE - fi - - # Change the help message to a mode-specific one. - generic_help=$help - help="Try '$progname --help --mode=$opt_mode' for more information." - } - - # Pass back the unparsed argument list - func_quote_for_eval ${1+"$@"} - libtool_validate_options_result=$func_quote_for_eval_result -} -func_add_hook func_validate_options libtool_validate_options - - -# Process options as early as possible so that --help and --version -# can return quickly. -func_options ${1+"$@"} -eval set dummy "$func_options_result"; shift - - - -## ----------- ## -## Main. ## -## ----------- ## - -magic='%%%MAGIC variable%%%' -magic_exe='%%%MAGIC EXE variable%%%' - -# Global variables. -extracted_archives= -extracted_serial=0 - -# If this variable is set in any of the actions, the command in it -# will be execed at the end. This prevents here-documents from being -# left over by shells. -exec_cmd= - - -# A function that is used when there is no print builtin or printf. -func_fallback_echo () -{ - eval 'cat <<_LTECHO_EOF -$1 -_LTECHO_EOF' -} - -# func_generated_by_libtool -# True iff stdin has been generated by Libtool. This function is only -# a basic sanity check; it will hardly flush out determined imposters. -func_generated_by_libtool_p () -{ - $GREP "^# Generated by .*$PACKAGE" > /dev/null 2>&1 -} - -# func_lalib_p file -# True iff FILE is a libtool '.la' library or '.lo' object file. -# This function is only a basic sanity check; it will hardly flush out -# determined imposters. -func_lalib_p () -{ - test -f "$1" && - $SED -e 4q "$1" 2>/dev/null | func_generated_by_libtool_p -} - -# func_lalib_unsafe_p file -# True iff FILE is a libtool '.la' library or '.lo' object file. -# This function implements the same check as func_lalib_p without -# resorting to external programs. To this end, it redirects stdin and -# closes it afterwards, without saving the original file descriptor. -# As a safety measure, use it only where a negative result would be -# fatal anyway. Works if 'file' does not exist. -func_lalib_unsafe_p () -{ - lalib_p=no - if test -f "$1" && test -r "$1" && exec 5<&0 <"$1"; then - for lalib_p_l in 1 2 3 4 - do - read lalib_p_line - case $lalib_p_line in - \#\ Generated\ by\ *$PACKAGE* ) lalib_p=yes; break;; - esac - done - exec 0<&5 5<&- - fi - test yes = "$lalib_p" -} - -# func_ltwrapper_script_p file -# True iff FILE is a libtool wrapper script -# This function is only a basic sanity check; it will hardly flush out -# determined imposters. -func_ltwrapper_script_p () -{ - test -f "$1" && - $lt_truncate_bin < "$1" 2>/dev/null | func_generated_by_libtool_p -} - -# func_ltwrapper_executable_p file -# True iff FILE is a libtool wrapper executable -# This function is only a basic sanity check; it will hardly flush out -# determined imposters. -func_ltwrapper_executable_p () -{ - func_ltwrapper_exec_suffix= - case $1 in - *.exe) ;; - *) func_ltwrapper_exec_suffix=.exe ;; - esac - $GREP "$magic_exe" "$1$func_ltwrapper_exec_suffix" >/dev/null 2>&1 -} - -# func_ltwrapper_scriptname file -# Assumes file is an ltwrapper_executable -# uses $file to determine the appropriate filename for a -# temporary ltwrapper_script. -func_ltwrapper_scriptname () -{ - func_dirname_and_basename "$1" "" "." - func_stripname '' '.exe' "$func_basename_result" - func_ltwrapper_scriptname_result=$func_dirname_result/$objdir/${func_stripname_result}_ltshwrapper -} - -# func_ltwrapper_p file -# True iff FILE is a libtool wrapper script or wrapper executable -# This function is only a basic sanity check; it will hardly flush out -# determined imposters. -func_ltwrapper_p () -{ - func_ltwrapper_script_p "$1" || func_ltwrapper_executable_p "$1" -} - - -# func_execute_cmds commands fail_cmd -# Execute tilde-delimited COMMANDS. -# If FAIL_CMD is given, eval that upon failure. -# FAIL_CMD may read-access the current command in variable CMD! -func_execute_cmds () -{ - $debug_cmd - - save_ifs=$IFS; IFS='~' - for cmd in $1; do - IFS=$sp$nl - eval cmd=\"$cmd\" - IFS=$save_ifs - func_show_eval "$cmd" "${2-:}" - done - IFS=$save_ifs -} - - -# func_source file -# Source FILE, adding directory component if necessary. -# Note that it is not necessary on cygwin/mingw to append a dot to -# FILE even if both FILE and FILE.exe exist: automatic-append-.exe -# behavior happens only for exec(3), not for open(2)! Also, sourcing -# 'FILE.' does not work on cygwin managed mounts. -func_source () -{ - $debug_cmd - - case $1 in - */* | *\\*) . "$1" ;; - *) . "./$1" ;; - esac -} - - -# func_resolve_sysroot PATH -# Replace a leading = in PATH with a sysroot. Store the result into -# func_resolve_sysroot_result -func_resolve_sysroot () -{ - func_resolve_sysroot_result=$1 - case $func_resolve_sysroot_result in - =*) - func_stripname '=' '' "$func_resolve_sysroot_result" - func_resolve_sysroot_result=$lt_sysroot$func_stripname_result - ;; - esac -} - -# func_replace_sysroot PATH -# If PATH begins with the sysroot, replace it with = and -# store the result into func_replace_sysroot_result. -func_replace_sysroot () -{ - case $lt_sysroot:$1 in - ?*:"$lt_sysroot"*) - func_stripname "$lt_sysroot" '' "$1" - func_replace_sysroot_result='='$func_stripname_result - ;; - *) - # Including no sysroot. - func_replace_sysroot_result=$1 - ;; - esac -} - -# func_infer_tag arg -# Infer tagged configuration to use if any are available and -# if one wasn't chosen via the "--tag" command line option. -# Only attempt this if the compiler in the base compile -# command doesn't match the default compiler. -# arg is usually of the form 'gcc ...' -func_infer_tag () -{ - $debug_cmd - - if test -n "$available_tags" && test -z "$tagname"; then - CC_quoted= - for arg in $CC; do - func_append_quoted CC_quoted "$arg" - done - CC_expanded=`func_echo_all $CC` - CC_quoted_expanded=`func_echo_all $CC_quoted` - case $@ in - # Blanks in the command may have been stripped by the calling shell, - # but not from the CC environment variable when configure was run. - " $CC "* | "$CC "* | " $CC_expanded "* | "$CC_expanded "* | \ - " $CC_quoted"* | "$CC_quoted "* | " $CC_quoted_expanded "* | "$CC_quoted_expanded "*) ;; - # Blanks at the start of $base_compile will cause this to fail - # if we don't check for them as well. - *) - for z in $available_tags; do - if $GREP "^# ### BEGIN LIBTOOL TAG CONFIG: $z$" < "$progpath" > /dev/null; then - # Evaluate the configuration. - eval "`$SED -n -e '/^# ### BEGIN LIBTOOL TAG CONFIG: '$z'$/,/^# ### END LIBTOOL TAG CONFIG: '$z'$/p' < $progpath`" - CC_quoted= - for arg in $CC; do - # Double-quote args containing other shell metacharacters. - func_append_quoted CC_quoted "$arg" - done - CC_expanded=`func_echo_all $CC` - CC_quoted_expanded=`func_echo_all $CC_quoted` - case "$@ " in - " $CC "* | "$CC "* | " $CC_expanded "* | "$CC_expanded "* | \ - " $CC_quoted"* | "$CC_quoted "* | " $CC_quoted_expanded "* | "$CC_quoted_expanded "*) - # The compiler in the base compile command matches - # the one in the tagged configuration. - # Assume this is the tagged configuration we want. - tagname=$z - break - ;; - esac - fi - done - # If $tagname still isn't set, then no tagged configuration - # was found and let the user know that the "--tag" command - # line option must be used. - if test -z "$tagname"; then - func_echo "unable to infer tagged configuration" - func_fatal_error "specify a tag with '--tag'" -# else -# func_verbose "using $tagname tagged configuration" - fi - ;; - esac - fi -} - - - -# func_write_libtool_object output_name pic_name nonpic_name -# Create a libtool object file (analogous to a ".la" file), -# but don't create it if we're doing a dry run. -func_write_libtool_object () -{ - write_libobj=$1 - if test yes = "$build_libtool_libs"; then - write_lobj=\'$2\' - else - write_lobj=none - fi - - if test yes = "$build_old_libs"; then - write_oldobj=\'$3\' - else - write_oldobj=none - fi - - $opt_dry_run || { - cat >${write_libobj}T </dev/null` - if test "$?" -eq 0 && test -n "$func_convert_core_file_wine_to_w32_tmp"; then - func_convert_core_file_wine_to_w32_result=`$ECHO "$func_convert_core_file_wine_to_w32_tmp" | - $SED -e "$sed_naive_backslashify"` - else - func_convert_core_file_wine_to_w32_result= - fi - fi -} -# end: func_convert_core_file_wine_to_w32 - - -# func_convert_core_path_wine_to_w32 ARG -# Helper function used by path conversion functions when $build is *nix, and -# $host is mingw, cygwin, or some other w32 environment. Relies on a correctly -# configured wine environment available, with the winepath program in $build's -# $PATH. Assumes ARG has no leading or trailing path separator characters. -# -# ARG is path to be converted from $build format to win32. -# Result is available in $func_convert_core_path_wine_to_w32_result. -# Unconvertible file (directory) names in ARG are skipped; if no directory names -# are convertible, then the result may be empty. -func_convert_core_path_wine_to_w32 () -{ - $debug_cmd - - # unfortunately, winepath doesn't convert paths, only file names - func_convert_core_path_wine_to_w32_result= - if test -n "$1"; then - oldIFS=$IFS - IFS=: - for func_convert_core_path_wine_to_w32_f in $1; do - IFS=$oldIFS - func_convert_core_file_wine_to_w32 "$func_convert_core_path_wine_to_w32_f" - if test -n "$func_convert_core_file_wine_to_w32_result"; then - if test -z "$func_convert_core_path_wine_to_w32_result"; then - func_convert_core_path_wine_to_w32_result=$func_convert_core_file_wine_to_w32_result - else - func_append func_convert_core_path_wine_to_w32_result ";$func_convert_core_file_wine_to_w32_result" - fi - fi - done - IFS=$oldIFS - fi -} -# end: func_convert_core_path_wine_to_w32 - - -# func_cygpath ARGS... -# Wrapper around calling the cygpath program via LT_CYGPATH. This is used when -# when (1) $build is *nix and Cygwin is hosted via a wine environment; or (2) -# $build is MSYS and $host is Cygwin, or (3) $build is Cygwin. In case (1) or -# (2), returns the Cygwin file name or path in func_cygpath_result (input -# file name or path is assumed to be in w32 format, as previously converted -# from $build's *nix or MSYS format). In case (3), returns the w32 file name -# or path in func_cygpath_result (input file name or path is assumed to be in -# Cygwin format). Returns an empty string on error. -# -# ARGS are passed to cygpath, with the last one being the file name or path to -# be converted. -# -# Specify the absolute *nix (or w32) name to cygpath in the LT_CYGPATH -# environment variable; do not put it in $PATH. -func_cygpath () -{ - $debug_cmd - - if test -n "$LT_CYGPATH" && test -f "$LT_CYGPATH"; then - func_cygpath_result=`$LT_CYGPATH "$@" 2>/dev/null` - if test "$?" -ne 0; then - # on failure, ensure result is empty - func_cygpath_result= - fi - else - func_cygpath_result= - func_error "LT_CYGPATH is empty or specifies non-existent file: '$LT_CYGPATH'" - fi -} -#end: func_cygpath - - -# func_convert_core_msys_to_w32 ARG -# Convert file name or path ARG from MSYS format to w32 format. Return -# result in func_convert_core_msys_to_w32_result. -func_convert_core_msys_to_w32 () -{ - $debug_cmd - - # awkward: cmd appends spaces to result - func_convert_core_msys_to_w32_result=`( cmd //c echo "$1" ) 2>/dev/null | - $SED -e 's/[ ]*$//' -e "$sed_naive_backslashify"` -} -#end: func_convert_core_msys_to_w32 - - -# func_convert_file_check ARG1 ARG2 -# Verify that ARG1 (a file name in $build format) was converted to $host -# format in ARG2. Otherwise, emit an error message, but continue (resetting -# func_to_host_file_result to ARG1). -func_convert_file_check () -{ - $debug_cmd - - if test -z "$2" && test -n "$1"; then - func_error "Could not determine host file name corresponding to" - func_error " '$1'" - func_error "Continuing, but uninstalled executables may not work." - # Fallback: - func_to_host_file_result=$1 - fi -} -# end func_convert_file_check - - -# func_convert_path_check FROM_PATHSEP TO_PATHSEP FROM_PATH TO_PATH -# Verify that FROM_PATH (a path in $build format) was converted to $host -# format in TO_PATH. Otherwise, emit an error message, but continue, resetting -# func_to_host_file_result to a simplistic fallback value (see below). -func_convert_path_check () -{ - $debug_cmd - - if test -z "$4" && test -n "$3"; then - func_error "Could not determine the host path corresponding to" - func_error " '$3'" - func_error "Continuing, but uninstalled executables may not work." - # Fallback. This is a deliberately simplistic "conversion" and - # should not be "improved". See libtool.info. - if test "x$1" != "x$2"; then - lt_replace_pathsep_chars="s|$1|$2|g" - func_to_host_path_result=`echo "$3" | - $SED -e "$lt_replace_pathsep_chars"` - else - func_to_host_path_result=$3 - fi - fi -} -# end func_convert_path_check - - -# func_convert_path_front_back_pathsep FRONTPAT BACKPAT REPL ORIG -# Modifies func_to_host_path_result by prepending REPL if ORIG matches FRONTPAT -# and appending REPL if ORIG matches BACKPAT. -func_convert_path_front_back_pathsep () -{ - $debug_cmd - - case $4 in - $1 ) func_to_host_path_result=$3$func_to_host_path_result - ;; - esac - case $4 in - $2 ) func_append func_to_host_path_result "$3" - ;; - esac -} -# end func_convert_path_front_back_pathsep - - -################################################## -# $build to $host FILE NAME CONVERSION FUNCTIONS # -################################################## -# invoked via '$to_host_file_cmd ARG' -# -# In each case, ARG is the path to be converted from $build to $host format. -# Result will be available in $func_to_host_file_result. - - -# func_to_host_file ARG -# Converts the file name ARG from $build format to $host format. Return result -# in func_to_host_file_result. -func_to_host_file () -{ - $debug_cmd - - $to_host_file_cmd "$1" -} -# end func_to_host_file - - -# func_to_tool_file ARG LAZY -# converts the file name ARG from $build format to toolchain format. Return -# result in func_to_tool_file_result. If the conversion in use is listed -# in (the comma separated) LAZY, no conversion takes place. -func_to_tool_file () -{ - $debug_cmd - - case ,$2, in - *,"$to_tool_file_cmd",*) - func_to_tool_file_result=$1 - ;; - *) - $to_tool_file_cmd "$1" - func_to_tool_file_result=$func_to_host_file_result - ;; - esac -} -# end func_to_tool_file - - -# func_convert_file_noop ARG -# Copy ARG to func_to_host_file_result. -func_convert_file_noop () -{ - func_to_host_file_result=$1 -} -# end func_convert_file_noop - - -# func_convert_file_msys_to_w32 ARG -# Convert file name ARG from (mingw) MSYS to (mingw) w32 format; automatic -# conversion to w32 is not available inside the cwrapper. Returns result in -# func_to_host_file_result. -func_convert_file_msys_to_w32 () -{ - $debug_cmd - - func_to_host_file_result=$1 - if test -n "$1"; then - func_convert_core_msys_to_w32 "$1" - func_to_host_file_result=$func_convert_core_msys_to_w32_result - fi - func_convert_file_check "$1" "$func_to_host_file_result" -} -# end func_convert_file_msys_to_w32 - - -# func_convert_file_cygwin_to_w32 ARG -# Convert file name ARG from Cygwin to w32 format. Returns result in -# func_to_host_file_result. -func_convert_file_cygwin_to_w32 () -{ - $debug_cmd - - func_to_host_file_result=$1 - if test -n "$1"; then - # because $build is cygwin, we call "the" cygpath in $PATH; no need to use - # LT_CYGPATH in this case. - func_to_host_file_result=`cygpath -m "$1"` - fi - func_convert_file_check "$1" "$func_to_host_file_result" -} -# end func_convert_file_cygwin_to_w32 - - -# func_convert_file_nix_to_w32 ARG -# Convert file name ARG from *nix to w32 format. Requires a wine environment -# and a working winepath. Returns result in func_to_host_file_result. -func_convert_file_nix_to_w32 () -{ - $debug_cmd - - func_to_host_file_result=$1 - if test -n "$1"; then - func_convert_core_file_wine_to_w32 "$1" - func_to_host_file_result=$func_convert_core_file_wine_to_w32_result - fi - func_convert_file_check "$1" "$func_to_host_file_result" -} -# end func_convert_file_nix_to_w32 - - -# func_convert_file_msys_to_cygwin ARG -# Convert file name ARG from MSYS to Cygwin format. Requires LT_CYGPATH set. -# Returns result in func_to_host_file_result. -func_convert_file_msys_to_cygwin () -{ - $debug_cmd - - func_to_host_file_result=$1 - if test -n "$1"; then - func_convert_core_msys_to_w32 "$1" - func_cygpath -u "$func_convert_core_msys_to_w32_result" - func_to_host_file_result=$func_cygpath_result - fi - func_convert_file_check "$1" "$func_to_host_file_result" -} -# end func_convert_file_msys_to_cygwin - - -# func_convert_file_nix_to_cygwin ARG -# Convert file name ARG from *nix to Cygwin format. Requires Cygwin installed -# in a wine environment, working winepath, and LT_CYGPATH set. Returns result -# in func_to_host_file_result. -func_convert_file_nix_to_cygwin () -{ - $debug_cmd - - func_to_host_file_result=$1 - if test -n "$1"; then - # convert from *nix to w32, then use cygpath to convert from w32 to cygwin. - func_convert_core_file_wine_to_w32 "$1" - func_cygpath -u "$func_convert_core_file_wine_to_w32_result" - func_to_host_file_result=$func_cygpath_result - fi - func_convert_file_check "$1" "$func_to_host_file_result" -} -# end func_convert_file_nix_to_cygwin - - -############################################# -# $build to $host PATH CONVERSION FUNCTIONS # -############################################# -# invoked via '$to_host_path_cmd ARG' -# -# In each case, ARG is the path to be converted from $build to $host format. -# The result will be available in $func_to_host_path_result. -# -# Path separators are also converted from $build format to $host format. If -# ARG begins or ends with a path separator character, it is preserved (but -# converted to $host format) on output. -# -# All path conversion functions are named using the following convention: -# file name conversion function : func_convert_file_X_to_Y () -# path conversion function : func_convert_path_X_to_Y () -# where, for any given $build/$host combination the 'X_to_Y' value is the -# same. If conversion functions are added for new $build/$host combinations, -# the two new functions must follow this pattern, or func_init_to_host_path_cmd -# will break. - - -# func_init_to_host_path_cmd -# Ensures that function "pointer" variable $to_host_path_cmd is set to the -# appropriate value, based on the value of $to_host_file_cmd. -to_host_path_cmd= -func_init_to_host_path_cmd () -{ - $debug_cmd - - if test -z "$to_host_path_cmd"; then - func_stripname 'func_convert_file_' '' "$to_host_file_cmd" - to_host_path_cmd=func_convert_path_$func_stripname_result - fi -} - - -# func_to_host_path ARG -# Converts the path ARG from $build format to $host format. Return result -# in func_to_host_path_result. -func_to_host_path () -{ - $debug_cmd - - func_init_to_host_path_cmd - $to_host_path_cmd "$1" -} -# end func_to_host_path - - -# func_convert_path_noop ARG -# Copy ARG to func_to_host_path_result. -func_convert_path_noop () -{ - func_to_host_path_result=$1 -} -# end func_convert_path_noop - - -# func_convert_path_msys_to_w32 ARG -# Convert path ARG from (mingw) MSYS to (mingw) w32 format; automatic -# conversion to w32 is not available inside the cwrapper. Returns result in -# func_to_host_path_result. -func_convert_path_msys_to_w32 () -{ - $debug_cmd - - func_to_host_path_result=$1 - if test -n "$1"; then - # Remove leading and trailing path separator characters from ARG. MSYS - # behavior is inconsistent here; cygpath turns them into '.;' and ';.'; - # and winepath ignores them completely. - func_stripname : : "$1" - func_to_host_path_tmp1=$func_stripname_result - func_convert_core_msys_to_w32 "$func_to_host_path_tmp1" - func_to_host_path_result=$func_convert_core_msys_to_w32_result - func_convert_path_check : ";" \ - "$func_to_host_path_tmp1" "$func_to_host_path_result" - func_convert_path_front_back_pathsep ":*" "*:" ";" "$1" - fi -} -# end func_convert_path_msys_to_w32 - - -# func_convert_path_cygwin_to_w32 ARG -# Convert path ARG from Cygwin to w32 format. Returns result in -# func_to_host_file_result. -func_convert_path_cygwin_to_w32 () -{ - $debug_cmd - - func_to_host_path_result=$1 - if test -n "$1"; then - # See func_convert_path_msys_to_w32: - func_stripname : : "$1" - func_to_host_path_tmp1=$func_stripname_result - func_to_host_path_result=`cygpath -m -p "$func_to_host_path_tmp1"` - func_convert_path_check : ";" \ - "$func_to_host_path_tmp1" "$func_to_host_path_result" - func_convert_path_front_back_pathsep ":*" "*:" ";" "$1" - fi -} -# end func_convert_path_cygwin_to_w32 - - -# func_convert_path_nix_to_w32 ARG -# Convert path ARG from *nix to w32 format. Requires a wine environment and -# a working winepath. Returns result in func_to_host_file_result. -func_convert_path_nix_to_w32 () -{ - $debug_cmd - - func_to_host_path_result=$1 - if test -n "$1"; then - # See func_convert_path_msys_to_w32: - func_stripname : : "$1" - func_to_host_path_tmp1=$func_stripname_result - func_convert_core_path_wine_to_w32 "$func_to_host_path_tmp1" - func_to_host_path_result=$func_convert_core_path_wine_to_w32_result - func_convert_path_check : ";" \ - "$func_to_host_path_tmp1" "$func_to_host_path_result" - func_convert_path_front_back_pathsep ":*" "*:" ";" "$1" - fi -} -# end func_convert_path_nix_to_w32 - - -# func_convert_path_msys_to_cygwin ARG -# Convert path ARG from MSYS to Cygwin format. Requires LT_CYGPATH set. -# Returns result in func_to_host_file_result. -func_convert_path_msys_to_cygwin () -{ - $debug_cmd - - func_to_host_path_result=$1 - if test -n "$1"; then - # See func_convert_path_msys_to_w32: - func_stripname : : "$1" - func_to_host_path_tmp1=$func_stripname_result - func_convert_core_msys_to_w32 "$func_to_host_path_tmp1" - func_cygpath -u -p "$func_convert_core_msys_to_w32_result" - func_to_host_path_result=$func_cygpath_result - func_convert_path_check : : \ - "$func_to_host_path_tmp1" "$func_to_host_path_result" - func_convert_path_front_back_pathsep ":*" "*:" : "$1" - fi -} -# end func_convert_path_msys_to_cygwin - - -# func_convert_path_nix_to_cygwin ARG -# Convert path ARG from *nix to Cygwin format. Requires Cygwin installed in a -# a wine environment, working winepath, and LT_CYGPATH set. Returns result in -# func_to_host_file_result. -func_convert_path_nix_to_cygwin () -{ - $debug_cmd - - func_to_host_path_result=$1 - if test -n "$1"; then - # Remove leading and trailing path separator characters from - # ARG. msys behavior is inconsistent here, cygpath turns them - # into '.;' and ';.', and winepath ignores them completely. - func_stripname : : "$1" - func_to_host_path_tmp1=$func_stripname_result - func_convert_core_path_wine_to_w32 "$func_to_host_path_tmp1" - func_cygpath -u -p "$func_convert_core_path_wine_to_w32_result" - func_to_host_path_result=$func_cygpath_result - func_convert_path_check : : \ - "$func_to_host_path_tmp1" "$func_to_host_path_result" - func_convert_path_front_back_pathsep ":*" "*:" : "$1" - fi -} -# end func_convert_path_nix_to_cygwin - - -# func_dll_def_p FILE -# True iff FILE is a Windows DLL '.def' file. -# Keep in sync with _LT_DLL_DEF_P in libtool.m4 -func_dll_def_p () -{ - $debug_cmd - - func_dll_def_p_tmp=`$SED -n \ - -e 's/^[ ]*//' \ - -e '/^\(;.*\)*$/d' \ - -e 's/^\(EXPORTS\|LIBRARY\)\([ ].*\)*$/DEF/p' \ - -e q \ - "$1"` - test DEF = "$func_dll_def_p_tmp" -} - - -# func_mode_compile arg... -func_mode_compile () -{ - $debug_cmd - - # Get the compilation command and the source file. - base_compile= - srcfile=$nonopt # always keep a non-empty value in "srcfile" - suppress_opt=yes - suppress_output= - arg_mode=normal - libobj= - later= - pie_flag= - - for arg - do - case $arg_mode in - arg ) - # do not "continue". Instead, add this to base_compile - lastarg=$arg - arg_mode=normal - ;; - - target ) - libobj=$arg - arg_mode=normal - continue - ;; - - normal ) - # Accept any command-line options. - case $arg in - -o) - test -n "$libobj" && \ - func_fatal_error "you cannot specify '-o' more than once" - arg_mode=target - continue - ;; - - -pie | -fpie | -fPIE) - func_append pie_flag " $arg" - continue - ;; - - -shared | -static | -prefer-pic | -prefer-non-pic) - func_append later " $arg" - continue - ;; - - -no-suppress) - suppress_opt=no - continue - ;; - - -Xcompiler) - arg_mode=arg # the next one goes into the "base_compile" arg list - continue # The current "srcfile" will either be retained or - ;; # replaced later. I would guess that would be a bug. - - -Wc,*) - func_stripname '-Wc,' '' "$arg" - args=$func_stripname_result - lastarg= - save_ifs=$IFS; IFS=, - for arg in $args; do - IFS=$save_ifs - func_append_quoted lastarg "$arg" - done - IFS=$save_ifs - func_stripname ' ' '' "$lastarg" - lastarg=$func_stripname_result - - # Add the arguments to base_compile. - func_append base_compile " $lastarg" - continue - ;; - - *) - # Accept the current argument as the source file. - # The previous "srcfile" becomes the current argument. - # - lastarg=$srcfile - srcfile=$arg - ;; - esac # case $arg - ;; - esac # case $arg_mode - - # Aesthetically quote the previous argument. - func_append_quoted base_compile "$lastarg" - done # for arg - - case $arg_mode in - arg) - func_fatal_error "you must specify an argument for -Xcompile" - ;; - target) - func_fatal_error "you must specify a target with '-o'" - ;; - *) - # Get the name of the library object. - test -z "$libobj" && { - func_basename "$srcfile" - libobj=$func_basename_result - } - ;; - esac - - # Recognize several different file suffixes. - # If the user specifies -o file.o, it is replaced with file.lo - case $libobj in - *.[cCFSifmso] | \ - *.ada | *.adb | *.ads | *.asm | \ - *.c++ | *.cc | *.ii | *.class | *.cpp | *.cxx | \ - *.[fF][09]? | *.for | *.java | *.go | *.obj | *.sx | *.cu | *.cup) - func_xform "$libobj" - libobj=$func_xform_result - ;; - esac - - case $libobj in - *.lo) func_lo2o "$libobj"; obj=$func_lo2o_result ;; - *) - func_fatal_error "cannot determine name of library object from '$libobj'" - ;; - esac - - func_infer_tag $base_compile - - for arg in $later; do - case $arg in - -shared) - test yes = "$build_libtool_libs" \ - || func_fatal_configuration "cannot build a shared library" - build_old_libs=no - continue - ;; - - -static) - build_libtool_libs=no - build_old_libs=yes - continue - ;; - - -prefer-pic) - pic_mode=yes - continue - ;; - - -prefer-non-pic) - pic_mode=no - continue - ;; - esac - done - - func_quote_for_eval "$libobj" - test "X$libobj" != "X$func_quote_for_eval_result" \ - && $ECHO "X$libobj" | $GREP '[]~#^*{};<>?"'"'"' &()|`$[]' \ - && func_warning "libobj name '$libobj' may not contain shell special characters." - func_dirname_and_basename "$obj" "/" "" - objname=$func_basename_result - xdir=$func_dirname_result - lobj=$xdir$objdir/$objname - - test -z "$base_compile" && \ - func_fatal_help "you must specify a compilation command" - - # Delete any leftover library objects. - if test yes = "$build_old_libs"; then - removelist="$obj $lobj $libobj ${libobj}T" - else - removelist="$lobj $libobj ${libobj}T" - fi - - # On Cygwin there's no "real" PIC flag so we must build both object types - case $host_os in - cygwin* | mingw* | pw32* | os2* | cegcc*) - pic_mode=default - ;; - esac - if test no = "$pic_mode" && test pass_all != "$deplibs_check_method"; then - # non-PIC code in shared libraries is not supported - pic_mode=default - fi - - # Calculate the filename of the output object if compiler does - # not support -o with -c - if test no = "$compiler_c_o"; then - output_obj=`$ECHO "$srcfile" | $SED 's%^.*/%%; s%\.[^.]*$%%'`.$objext - lockfile=$output_obj.lock - else - output_obj= - need_locks=no - lockfile= - fi - - # Lock this critical section if it is needed - # We use this script file to make the link, it avoids creating a new file - if test yes = "$need_locks"; then - until $opt_dry_run || ln "$progpath" "$lockfile" 2>/dev/null; do - func_echo "Waiting for $lockfile to be removed" - sleep 2 - done - elif test warn = "$need_locks"; then - if test -f "$lockfile"; then - $ECHO "\ -*** ERROR, $lockfile exists and contains: -`cat $lockfile 2>/dev/null` - -This indicates that another process is trying to use the same -temporary object file, and libtool could not work around it because -your compiler does not support '-c' and '-o' together. If you -repeat this compilation, it may succeed, by chance, but you had better -avoid parallel builds (make -j) in this platform, or get a better -compiler." - - $opt_dry_run || $RM $removelist - exit $EXIT_FAILURE - fi - func_append removelist " $output_obj" - $ECHO "$srcfile" > "$lockfile" - fi - - $opt_dry_run || $RM $removelist - func_append removelist " $lockfile" - trap '$opt_dry_run || $RM $removelist; exit $EXIT_FAILURE' 1 2 15 - - func_to_tool_file "$srcfile" func_convert_file_msys_to_w32 - srcfile=$func_to_tool_file_result - func_quote_for_eval "$srcfile" - qsrcfile=$func_quote_for_eval_result - - # Only build a PIC object if we are building libtool libraries. - if test yes = "$build_libtool_libs"; then - # Without this assignment, base_compile gets emptied. - fbsd_hideous_sh_bug=$base_compile - - if test no != "$pic_mode"; then - command="$base_compile $qsrcfile $pic_flag" - else - # Don't build PIC code - command="$base_compile $qsrcfile" - fi - - func_mkdir_p "$xdir$objdir" - - if test -z "$output_obj"; then - # Place PIC objects in $objdir - func_append command " -o $lobj" - fi - - func_show_eval_locale "$command" \ - 'test -n "$output_obj" && $RM $removelist; exit $EXIT_FAILURE' - - if test warn = "$need_locks" && - test "X`cat $lockfile 2>/dev/null`" != "X$srcfile"; then - $ECHO "\ -*** ERROR, $lockfile contains: -`cat $lockfile 2>/dev/null` - -but it should contain: -$srcfile - -This indicates that another process is trying to use the same -temporary object file, and libtool could not work around it because -your compiler does not support '-c' and '-o' together. If you -repeat this compilation, it may succeed, by chance, but you had better -avoid parallel builds (make -j) in this platform, or get a better -compiler." - - $opt_dry_run || $RM $removelist - exit $EXIT_FAILURE - fi - - # Just move the object if needed, then go on to compile the next one - if test -n "$output_obj" && test "X$output_obj" != "X$lobj"; then - func_show_eval '$MV "$output_obj" "$lobj"' \ - 'error=$?; $opt_dry_run || $RM $removelist; exit $error' - fi - - # Allow error messages only from the first compilation. - if test yes = "$suppress_opt"; then - suppress_output=' >/dev/null 2>&1' - fi - fi - - # Only build a position-dependent object if we build old libraries. - if test yes = "$build_old_libs"; then - if test yes != "$pic_mode"; then - # Don't build PIC code - command="$base_compile $qsrcfile$pie_flag" - else - command="$base_compile $qsrcfile $pic_flag" - fi - if test yes = "$compiler_c_o"; then - func_append command " -o $obj" - fi - - # Suppress compiler output if we already did a PIC compilation. - func_append command "$suppress_output" - func_show_eval_locale "$command" \ - '$opt_dry_run || $RM $removelist; exit $EXIT_FAILURE' - - if test warn = "$need_locks" && - test "X`cat $lockfile 2>/dev/null`" != "X$srcfile"; then - $ECHO "\ -*** ERROR, $lockfile contains: -`cat $lockfile 2>/dev/null` - -but it should contain: -$srcfile - -This indicates that another process is trying to use the same -temporary object file, and libtool could not work around it because -your compiler does not support '-c' and '-o' together. If you -repeat this compilation, it may succeed, by chance, but you had better -avoid parallel builds (make -j) in this platform, or get a better -compiler." - - $opt_dry_run || $RM $removelist - exit $EXIT_FAILURE - fi - - # Just move the object if needed - if test -n "$output_obj" && test "X$output_obj" != "X$obj"; then - func_show_eval '$MV "$output_obj" "$obj"' \ - 'error=$?; $opt_dry_run || $RM $removelist; exit $error' - fi - fi - - $opt_dry_run || { - func_write_libtool_object "$libobj" "$objdir/$objname" "$objname" - - # Unlock the critical section if it was locked - if test no != "$need_locks"; then - removelist=$lockfile - $RM "$lockfile" - fi - } - - exit $EXIT_SUCCESS -} - -$opt_help || { - test compile = "$opt_mode" && func_mode_compile ${1+"$@"} -} - -func_mode_help () -{ - # We need to display help for each of the modes. - case $opt_mode in - "") - # Generic help is extracted from the usage comments - # at the start of this file. - func_help - ;; - - clean) - $ECHO \ -"Usage: $progname [OPTION]... --mode=clean RM [RM-OPTION]... FILE... - -Remove files from the build directory. - -RM is the name of the program to use to delete files associated with each FILE -(typically '/bin/rm'). RM-OPTIONS are options (such as '-f') to be passed -to RM. - -If FILE is a libtool library, object or program, all the files associated -with it are deleted. Otherwise, only FILE itself is deleted using RM." - ;; - - compile) - $ECHO \ -"Usage: $progname [OPTION]... --mode=compile COMPILE-COMMAND... SOURCEFILE - -Compile a source file into a libtool library object. - -This mode accepts the following additional options: - - -o OUTPUT-FILE set the output file name to OUTPUT-FILE - -no-suppress do not suppress compiler output for multiple passes - -prefer-pic try to build PIC objects only - -prefer-non-pic try to build non-PIC objects only - -shared do not build a '.o' file suitable for static linking - -static only build a '.o' file suitable for static linking - -Wc,FLAG pass FLAG directly to the compiler - -COMPILE-COMMAND is a command to be used in creating a 'standard' object file -from the given SOURCEFILE. - -The output file name is determined by removing the directory component from -SOURCEFILE, then substituting the C source code suffix '.c' with the -library object suffix, '.lo'." - ;; - - execute) - $ECHO \ -"Usage: $progname [OPTION]... --mode=execute COMMAND [ARGS]... - -Automatically set library path, then run a program. - -This mode accepts the following additional options: - - -dlopen FILE add the directory containing FILE to the library path - -This mode sets the library path environment variable according to '-dlopen' -flags. - -If any of the ARGS are libtool executable wrappers, then they are translated -into their corresponding uninstalled binary, and any of their required library -directories are added to the library path. - -Then, COMMAND is executed, with ARGS as arguments." - ;; - - finish) - $ECHO \ -"Usage: $progname [OPTION]... --mode=finish [LIBDIR]... - -Complete the installation of libtool libraries. - -Each LIBDIR is a directory that contains libtool libraries. - -The commands that this mode executes may require superuser privileges. Use -the '--dry-run' option if you just want to see what would be executed." - ;; - - install) - $ECHO \ -"Usage: $progname [OPTION]... --mode=install INSTALL-COMMAND... - -Install executables or libraries. - -INSTALL-COMMAND is the installation command. The first component should be -either the 'install' or 'cp' program. - -The following components of INSTALL-COMMAND are treated specially: - - -inst-prefix-dir PREFIX-DIR Use PREFIX-DIR as a staging area for installation - -The rest of the components are interpreted as arguments to that command (only -BSD-compatible install options are recognized)." - ;; - - link) - $ECHO \ -"Usage: $progname [OPTION]... --mode=link LINK-COMMAND... - -Link object files or libraries together to form another library, or to -create an executable program. - -LINK-COMMAND is a command using the C compiler that you would use to create -a program from several object files. - -The following components of LINK-COMMAND are treated specially: - - -all-static do not do any dynamic linking at all - -avoid-version do not add a version suffix if possible - -bindir BINDIR specify path to binaries directory (for systems where - libraries must be found in the PATH setting at runtime) - -dlopen FILE '-dlpreopen' FILE if it cannot be dlopened at runtime - -dlpreopen FILE link in FILE and add its symbols to lt_preloaded_symbols - -export-dynamic allow symbols from OUTPUT-FILE to be resolved with dlsym(3) - -export-symbols SYMFILE - try to export only the symbols listed in SYMFILE - -export-symbols-regex REGEX - try to export only the symbols matching REGEX - -LLIBDIR search LIBDIR for required installed libraries - -lNAME OUTPUT-FILE requires the installed library libNAME - -module build a library that can dlopened - -no-fast-install disable the fast-install mode - -no-install link a not-installable executable - -no-undefined declare that a library does not refer to external symbols - -o OUTPUT-FILE create OUTPUT-FILE from the specified objects - -objectlist FILE use a list of object files found in FILE to specify objects - -os2dllname NAME force a short DLL name on OS/2 (no effect on other OSes) - -precious-files-regex REGEX - don't remove output files matching REGEX - -release RELEASE specify package release information - -rpath LIBDIR the created library will eventually be installed in LIBDIR - -R[ ]LIBDIR add LIBDIR to the runtime path of programs and libraries - -shared only do dynamic linking of libtool libraries - -shrext SUFFIX override the standard shared library file extension - -static do not do any dynamic linking of uninstalled libtool libraries - -static-libtool-libs - do not do any dynamic linking of libtool libraries - -version-info CURRENT[:REVISION[:AGE]] - specify library version info [each variable defaults to 0] - -weak LIBNAME declare that the target provides the LIBNAME interface - -Wc,FLAG - -Xcompiler FLAG pass linker-specific FLAG directly to the compiler - -Wl,FLAG - -Xlinker FLAG pass linker-specific FLAG directly to the linker - -XCClinker FLAG pass link-specific FLAG to the compiler driver (CC) - -All other options (arguments beginning with '-') are ignored. - -Every other argument is treated as a filename. Files ending in '.la' are -treated as uninstalled libtool libraries, other files are standard or library -object files. - -If the OUTPUT-FILE ends in '.la', then a libtool library is created, -only library objects ('.lo' files) may be specified, and '-rpath' is -required, except when creating a convenience library. - -If OUTPUT-FILE ends in '.a' or '.lib', then a standard library is created -using 'ar' and 'ranlib', or on Windows using 'lib'. - -If OUTPUT-FILE ends in '.lo' or '.$objext', then a reloadable object file -is created, otherwise an executable program is created." - ;; - - uninstall) - $ECHO \ -"Usage: $progname [OPTION]... --mode=uninstall RM [RM-OPTION]... FILE... - -Remove libraries from an installation directory. - -RM is the name of the program to use to delete files associated with each FILE -(typically '/bin/rm'). RM-OPTIONS are options (such as '-f') to be passed -to RM. - -If FILE is a libtool library, all the files associated with it are deleted. -Otherwise, only FILE itself is deleted using RM." - ;; - - *) - func_fatal_help "invalid operation mode '$opt_mode'" - ;; - esac - - echo - $ECHO "Try '$progname --help' for more information about other modes." -} - -# Now that we've collected a possible --mode arg, show help if necessary -if $opt_help; then - if test : = "$opt_help"; then - func_mode_help - else - { - func_help noexit - for opt_mode in compile link execute install finish uninstall clean; do - func_mode_help - done - } | $SED -n '1p; 2,$s/^Usage:/ or: /p' - { - func_help noexit - for opt_mode in compile link execute install finish uninstall clean; do - echo - func_mode_help - done - } | - $SED '1d - /^When reporting/,/^Report/{ - H - d - } - $x - /information about other modes/d - /more detailed .*MODE/d - s/^Usage:.*--mode=\([^ ]*\) .*/Description of \1 mode:/' - fi - exit $? -fi - - -# func_mode_execute arg... -func_mode_execute () -{ - $debug_cmd - - # The first argument is the command name. - cmd=$nonopt - test -z "$cmd" && \ - func_fatal_help "you must specify a COMMAND" - - # Handle -dlopen flags immediately. - for file in $opt_dlopen; do - test -f "$file" \ - || func_fatal_help "'$file' is not a file" - - dir= - case $file in - *.la) - func_resolve_sysroot "$file" - file=$func_resolve_sysroot_result - - # Check to see that this really is a libtool archive. - func_lalib_unsafe_p "$file" \ - || func_fatal_help "'$lib' is not a valid libtool archive" - - # Read the libtool library. - dlname= - library_names= - func_source "$file" - - # Skip this library if it cannot be dlopened. - if test -z "$dlname"; then - # Warn if it was a shared library. - test -n "$library_names" && \ - func_warning "'$file' was not linked with '-export-dynamic'" - continue - fi - - func_dirname "$file" "" "." - dir=$func_dirname_result - - if test -f "$dir/$objdir/$dlname"; then - func_append dir "/$objdir" - else - if test ! -f "$dir/$dlname"; then - func_fatal_error "cannot find '$dlname' in '$dir' or '$dir/$objdir'" - fi - fi - ;; - - *.lo) - # Just add the directory containing the .lo file. - func_dirname "$file" "" "." - dir=$func_dirname_result - ;; - - *) - func_warning "'-dlopen' is ignored for non-libtool libraries and objects" - continue - ;; - esac - - # Get the absolute pathname. - absdir=`cd "$dir" && pwd` - test -n "$absdir" && dir=$absdir - - # Now add the directory to shlibpath_var. - if eval "test -z \"\$$shlibpath_var\""; then - eval "$shlibpath_var=\"\$dir\"" - else - eval "$shlibpath_var=\"\$dir:\$$shlibpath_var\"" - fi - done - - # This variable tells wrapper scripts just to set shlibpath_var - # rather than running their programs. - libtool_execute_magic=$magic - - # Check if any of the arguments is a wrapper script. - args= - for file - do - case $file in - -* | *.la | *.lo ) ;; - *) - # Do a test to see if this is really a libtool program. - if func_ltwrapper_script_p "$file"; then - func_source "$file" - # Transform arg to wrapped name. - file=$progdir/$program - elif func_ltwrapper_executable_p "$file"; then - func_ltwrapper_scriptname "$file" - func_source "$func_ltwrapper_scriptname_result" - # Transform arg to wrapped name. - file=$progdir/$program - fi - ;; - esac - # Quote arguments (to preserve shell metacharacters). - func_append_quoted args "$file" - done - - if $opt_dry_run; then - # Display what would be done. - if test -n "$shlibpath_var"; then - eval "\$ECHO \"\$shlibpath_var=\$$shlibpath_var\"" - echo "export $shlibpath_var" - fi - $ECHO "$cmd$args" - exit $EXIT_SUCCESS - else - if test -n "$shlibpath_var"; then - # Export the shlibpath_var. - eval "export $shlibpath_var" - fi - - # Restore saved environment variables - for lt_var in LANG LANGUAGE LC_ALL LC_CTYPE LC_COLLATE LC_MESSAGES - do - eval "if test \"\${save_$lt_var+set}\" = set; then - $lt_var=\$save_$lt_var; export $lt_var - else - $lt_unset $lt_var - fi" - done - - # Now prepare to actually exec the command. - exec_cmd=\$cmd$args - fi -} - -test execute = "$opt_mode" && func_mode_execute ${1+"$@"} - - -# func_mode_finish arg... -func_mode_finish () -{ - $debug_cmd - - libs= - libdirs= - admincmds= - - for opt in "$nonopt" ${1+"$@"} - do - if test -d "$opt"; then - func_append libdirs " $opt" - - elif test -f "$opt"; then - if func_lalib_unsafe_p "$opt"; then - func_append libs " $opt" - else - func_warning "'$opt' is not a valid libtool archive" - fi - - else - func_fatal_error "invalid argument '$opt'" - fi - done - - if test -n "$libs"; then - if test -n "$lt_sysroot"; then - sysroot_regex=`$ECHO "$lt_sysroot" | $SED "$sed_make_literal_regex"` - sysroot_cmd="s/\([ ']\)$sysroot_regex/\1/g;" - else - sysroot_cmd= - fi - - # Remove sysroot references - if $opt_dry_run; then - for lib in $libs; do - echo "removing references to $lt_sysroot and '=' prefixes from $lib" - done - else - tmpdir=`func_mktempdir` - for lib in $libs; do - $SED -e "$sysroot_cmd s/\([ ']-[LR]\)=/\1/g; s/\([ ']\)=/\1/g" $lib \ - > $tmpdir/tmp-la - mv -f $tmpdir/tmp-la $lib - done - ${RM}r "$tmpdir" - fi - fi - - if test -n "$finish_cmds$finish_eval" && test -n "$libdirs"; then - for libdir in $libdirs; do - if test -n "$finish_cmds"; then - # Do each command in the finish commands. - func_execute_cmds "$finish_cmds" 'admincmds="$admincmds -'"$cmd"'"' - fi - if test -n "$finish_eval"; then - # Do the single finish_eval. - eval cmds=\"$finish_eval\" - $opt_dry_run || eval "$cmds" || func_append admincmds " - $cmds" - fi - done - fi - - # Exit here if they wanted silent mode. - $opt_quiet && exit $EXIT_SUCCESS - - if test -n "$finish_cmds$finish_eval" && test -n "$libdirs"; then - echo "----------------------------------------------------------------------" - echo "Libraries have been installed in:" - for libdir in $libdirs; do - $ECHO " $libdir" - done - echo - echo "If you ever happen to want to link against installed libraries" - echo "in a given directory, LIBDIR, you must either use libtool, and" - echo "specify the full pathname of the library, or use the '-LLIBDIR'" - echo "flag during linking and do at least one of the following:" - if test -n "$shlibpath_var"; then - echo " - add LIBDIR to the '$shlibpath_var' environment variable" - echo " during execution" - fi - if test -n "$runpath_var"; then - echo " - add LIBDIR to the '$runpath_var' environment variable" - echo " during linking" - fi - if test -n "$hardcode_libdir_flag_spec"; then - libdir=LIBDIR - eval flag=\"$hardcode_libdir_flag_spec\" - - $ECHO " - use the '$flag' linker flag" - fi - if test -n "$admincmds"; then - $ECHO " - have your system administrator run these commands:$admincmds" - fi - if test -f /etc/ld.so.conf; then - echo " - have your system administrator add LIBDIR to '/etc/ld.so.conf'" - fi - echo - - echo "See any operating system documentation about shared libraries for" - case $host in - solaris2.[6789]|solaris2.1[0-9]) - echo "more information, such as the ld(1), crle(1) and ld.so(8) manual" - echo "pages." - ;; - *) - echo "more information, such as the ld(1) and ld.so(8) manual pages." - ;; - esac - echo "----------------------------------------------------------------------" - fi - exit $EXIT_SUCCESS -} - -test finish = "$opt_mode" && func_mode_finish ${1+"$@"} - - -# func_mode_install arg... -func_mode_install () -{ - $debug_cmd - - # There may be an optional sh(1) argument at the beginning of - # install_prog (especially on Windows NT). - if test "$SHELL" = "$nonopt" || test /bin/sh = "$nonopt" || - # Allow the use of GNU shtool's install command. - case $nonopt in *shtool*) :;; *) false;; esac - then - # Aesthetically quote it. - func_quote_for_eval "$nonopt" - install_prog="$func_quote_for_eval_result " - arg=$1 - shift - else - install_prog= - arg=$nonopt - fi - - # The real first argument should be the name of the installation program. - # Aesthetically quote it. - func_quote_for_eval "$arg" - func_append install_prog "$func_quote_for_eval_result" - install_shared_prog=$install_prog - case " $install_prog " in - *[\\\ /]cp\ *) install_cp=: ;; - *) install_cp=false ;; - esac - - # We need to accept at least all the BSD install flags. - dest= - files= - opts= - prev= - install_type= - isdir=false - stripme= - no_mode=: - for arg - do - arg2= - if test -n "$dest"; then - func_append files " $dest" - dest=$arg - continue - fi - - case $arg in - -d) isdir=: ;; - -f) - if $install_cp; then :; else - prev=$arg - fi - ;; - -g | -m | -o) - prev=$arg - ;; - -s) - stripme=" -s" - continue - ;; - -*) - ;; - *) - # If the previous option needed an argument, then skip it. - if test -n "$prev"; then - if test X-m = "X$prev" && test -n "$install_override_mode"; then - arg2=$install_override_mode - no_mode=false - fi - prev= - else - dest=$arg - continue - fi - ;; - esac - - # Aesthetically quote the argument. - func_quote_for_eval "$arg" - func_append install_prog " $func_quote_for_eval_result" - if test -n "$arg2"; then - func_quote_for_eval "$arg2" - fi - func_append install_shared_prog " $func_quote_for_eval_result" - done - - test -z "$install_prog" && \ - func_fatal_help "you must specify an install program" - - test -n "$prev" && \ - func_fatal_help "the '$prev' option requires an argument" - - if test -n "$install_override_mode" && $no_mode; then - if $install_cp; then :; else - func_quote_for_eval "$install_override_mode" - func_append install_shared_prog " -m $func_quote_for_eval_result" - fi - fi - - if test -z "$files"; then - if test -z "$dest"; then - func_fatal_help "no file or destination specified" - else - func_fatal_help "you must specify a destination" - fi - fi - - # Strip any trailing slash from the destination. - func_stripname '' '/' "$dest" - dest=$func_stripname_result - - # Check to see that the destination is a directory. - test -d "$dest" && isdir=: - if $isdir; then - destdir=$dest - destname= - else - func_dirname_and_basename "$dest" "" "." - destdir=$func_dirname_result - destname=$func_basename_result - - # Not a directory, so check to see that there is only one file specified. - set dummy $files; shift - test "$#" -gt 1 && \ - func_fatal_help "'$dest' is not a directory" - fi - case $destdir in - [\\/]* | [A-Za-z]:[\\/]*) ;; - *) - for file in $files; do - case $file in - *.lo) ;; - *) - func_fatal_help "'$destdir' must be an absolute directory name" - ;; - esac - done - ;; - esac - - # This variable tells wrapper scripts just to set variables rather - # than running their programs. - libtool_install_magic=$magic - - staticlibs= - future_libdirs= - current_libdirs= - for file in $files; do - - # Do each installation. - case $file in - *.$libext) - # Do the static libraries later. - func_append staticlibs " $file" - ;; - - *.la) - func_resolve_sysroot "$file" - file=$func_resolve_sysroot_result - - # Check to see that this really is a libtool archive. - func_lalib_unsafe_p "$file" \ - || func_fatal_help "'$file' is not a valid libtool archive" - - library_names= - old_library= - relink_command= - func_source "$file" - - # Add the libdir to current_libdirs if it is the destination. - if test "X$destdir" = "X$libdir"; then - case "$current_libdirs " in - *" $libdir "*) ;; - *) func_append current_libdirs " $libdir" ;; - esac - else - # Note the libdir as a future libdir. - case "$future_libdirs " in - *" $libdir "*) ;; - *) func_append future_libdirs " $libdir" ;; - esac - fi - - func_dirname "$file" "/" "" - dir=$func_dirname_result - func_append dir "$objdir" - - if test -n "$relink_command"; then - # Determine the prefix the user has applied to our future dir. - inst_prefix_dir=`$ECHO "$destdir" | $SED -e "s%$libdir\$%%"` - - # Don't allow the user to place us outside of our expected - # location b/c this prevents finding dependent libraries that - # are installed to the same prefix. - # At present, this check doesn't affect windows .dll's that - # are installed into $libdir/../bin (currently, that works fine) - # but it's something to keep an eye on. - test "$inst_prefix_dir" = "$destdir" && \ - func_fatal_error "error: cannot install '$file' to a directory not ending in $libdir" - - if test -n "$inst_prefix_dir"; then - # Stick the inst_prefix_dir data into the link command. - relink_command=`$ECHO "$relink_command" | $SED "s%@inst_prefix_dir@%-inst-prefix-dir $inst_prefix_dir%"` - else - relink_command=`$ECHO "$relink_command" | $SED "s%@inst_prefix_dir@%%"` - fi - - func_warning "relinking '$file'" - func_show_eval "$relink_command" \ - 'func_fatal_error "error: relink '\''$file'\'' with the above command before installing it"' - fi - - # See the names of the shared library. - set dummy $library_names; shift - if test -n "$1"; then - realname=$1 - shift - - srcname=$realname - test -n "$relink_command" && srcname=${realname}T - - # Install the shared library and build the symlinks. - func_show_eval "$install_shared_prog $dir/$srcname $destdir/$realname" \ - 'exit $?' - tstripme=$stripme - case $host_os in - cygwin* | mingw* | pw32* | cegcc*) - case $realname in - *.dll.a) - tstripme= - ;; - esac - ;; - os2*) - case $realname in - *_dll.a) - tstripme= - ;; - esac - ;; - esac - if test -n "$tstripme" && test -n "$striplib"; then - func_show_eval "$striplib $destdir/$realname" 'exit $?' - fi - - if test "$#" -gt 0; then - # Delete the old symlinks, and create new ones. - # Try 'ln -sf' first, because the 'ln' binary might depend on - # the symlink we replace! Solaris /bin/ln does not understand -f, - # so we also need to try rm && ln -s. - for linkname - do - test "$linkname" != "$realname" \ - && func_show_eval "(cd $destdir && { $LN_S -f $realname $linkname || { $RM $linkname && $LN_S $realname $linkname; }; })" - done - fi - - # Do each command in the postinstall commands. - lib=$destdir/$realname - func_execute_cmds "$postinstall_cmds" 'exit $?' - fi - - # Install the pseudo-library for information purposes. - func_basename "$file" - name=$func_basename_result - instname=$dir/${name}i - func_show_eval "$install_prog $instname $destdir/$name" 'exit $?' - - # Maybe install the static library, too. - test -n "$old_library" && func_append staticlibs " $dir/$old_library" - ;; - - *.lo) - # Install (i.e. copy) a libtool object. - - # Figure out destination file name, if it wasn't already specified. - if test -n "$destname"; then - destfile=$destdir/$destname - else - func_basename "$file" - destfile=$func_basename_result - destfile=$destdir/$destfile - fi - - # Deduce the name of the destination old-style object file. - case $destfile in - *.lo) - func_lo2o "$destfile" - staticdest=$func_lo2o_result - ;; - *.$objext) - staticdest=$destfile - destfile= - ;; - *) - func_fatal_help "cannot copy a libtool object to '$destfile'" - ;; - esac - - # Install the libtool object if requested. - test -n "$destfile" && \ - func_show_eval "$install_prog $file $destfile" 'exit $?' - - # Install the old object if enabled. - if test yes = "$build_old_libs"; then - # Deduce the name of the old-style object file. - func_lo2o "$file" - staticobj=$func_lo2o_result - func_show_eval "$install_prog \$staticobj \$staticdest" 'exit $?' - fi - exit $EXIT_SUCCESS - ;; - - *) - # Figure out destination file name, if it wasn't already specified. - if test -n "$destname"; then - destfile=$destdir/$destname - else - func_basename "$file" - destfile=$func_basename_result - destfile=$destdir/$destfile - fi - - # If the file is missing, and there is a .exe on the end, strip it - # because it is most likely a libtool script we actually want to - # install - stripped_ext= - case $file in - *.exe) - if test ! -f "$file"; then - func_stripname '' '.exe' "$file" - file=$func_stripname_result - stripped_ext=.exe - fi - ;; - esac - - # Do a test to see if this is really a libtool program. - case $host in - *cygwin* | *mingw*) - if func_ltwrapper_executable_p "$file"; then - func_ltwrapper_scriptname "$file" - wrapper=$func_ltwrapper_scriptname_result - else - func_stripname '' '.exe' "$file" - wrapper=$func_stripname_result - fi - ;; - *) - wrapper=$file - ;; - esac - if func_ltwrapper_script_p "$wrapper"; then - notinst_deplibs= - relink_command= - - func_source "$wrapper" - - # Check the variables that should have been set. - test -z "$generated_by_libtool_version" && \ - func_fatal_error "invalid libtool wrapper script '$wrapper'" - - finalize=: - for lib in $notinst_deplibs; do - # Check to see that each library is installed. - libdir= - if test -f "$lib"; then - func_source "$lib" - fi - libfile=$libdir/`$ECHO "$lib" | $SED 's%^.*/%%g'` - if test -n "$libdir" && test ! -f "$libfile"; then - func_warning "'$lib' has not been installed in '$libdir'" - finalize=false - fi - done - - relink_command= - func_source "$wrapper" - - outputname= - if test no = "$fast_install" && test -n "$relink_command"; then - $opt_dry_run || { - if $finalize; then - tmpdir=`func_mktempdir` - func_basename "$file$stripped_ext" - file=$func_basename_result - outputname=$tmpdir/$file - # Replace the output file specification. - relink_command=`$ECHO "$relink_command" | $SED 's%@OUTPUT@%'"$outputname"'%g'` - - $opt_quiet || { - func_quote_for_expand "$relink_command" - eval "func_echo $func_quote_for_expand_result" - } - if eval "$relink_command"; then : - else - func_error "error: relink '$file' with the above command before installing it" - $opt_dry_run || ${RM}r "$tmpdir" - continue - fi - file=$outputname - else - func_warning "cannot relink '$file'" - fi - } - else - # Install the binary that we compiled earlier. - file=`$ECHO "$file$stripped_ext" | $SED "s%\([^/]*\)$%$objdir/\1%"` - fi - fi - - # remove .exe since cygwin /usr/bin/install will append another - # one anyway - case $install_prog,$host in - */usr/bin/install*,*cygwin*) - case $file:$destfile in - *.exe:*.exe) - # this is ok - ;; - *.exe:*) - destfile=$destfile.exe - ;; - *:*.exe) - func_stripname '' '.exe' "$destfile" - destfile=$func_stripname_result - ;; - esac - ;; - esac - func_show_eval "$install_prog\$stripme \$file \$destfile" 'exit $?' - $opt_dry_run || if test -n "$outputname"; then - ${RM}r "$tmpdir" - fi - ;; - esac - done - - for file in $staticlibs; do - func_basename "$file" - name=$func_basename_result - - # Set up the ranlib parameters. - oldlib=$destdir/$name - func_to_tool_file "$oldlib" func_convert_file_msys_to_w32 - tool_oldlib=$func_to_tool_file_result - - func_show_eval "$install_prog \$file \$oldlib" 'exit $?' - - if test -n "$stripme" && test -n "$old_striplib"; then - func_show_eval "$old_striplib $tool_oldlib" 'exit $?' - fi - - # Do each command in the postinstall commands. - func_execute_cmds "$old_postinstall_cmds" 'exit $?' - done - - test -n "$future_libdirs" && \ - func_warning "remember to run '$progname --finish$future_libdirs'" - - if test -n "$current_libdirs"; then - # Maybe just do a dry run. - $opt_dry_run && current_libdirs=" -n$current_libdirs" - exec_cmd='$SHELL "$progpath" $preserve_args --finish$current_libdirs' - else - exit $EXIT_SUCCESS - fi -} - -test install = "$opt_mode" && func_mode_install ${1+"$@"} - - -# func_generate_dlsyms outputname originator pic_p -# Extract symbols from dlprefiles and create ${outputname}S.o with -# a dlpreopen symbol table. -func_generate_dlsyms () -{ - $debug_cmd - - my_outputname=$1 - my_originator=$2 - my_pic_p=${3-false} - my_prefix=`$ECHO "$my_originator" | $SED 's%[^a-zA-Z0-9]%_%g'` - my_dlsyms= - - if test -n "$dlfiles$dlprefiles" || test no != "$dlself"; then - if test -n "$NM" && test -n "$global_symbol_pipe"; then - my_dlsyms=${my_outputname}S.c - else - func_error "not configured to extract global symbols from dlpreopened files" - fi - fi - - if test -n "$my_dlsyms"; then - case $my_dlsyms in - "") ;; - *.c) - # Discover the nlist of each of the dlfiles. - nlist=$output_objdir/$my_outputname.nm - - func_show_eval "$RM $nlist ${nlist}S ${nlist}T" - - # Parse the name list into a source file. - func_verbose "creating $output_objdir/$my_dlsyms" - - $opt_dry_run || $ECHO > "$output_objdir/$my_dlsyms" "\ -/* $my_dlsyms - symbol resolution table for '$my_outputname' dlsym emulation. */ -/* Generated by $PROGRAM (GNU $PACKAGE) $VERSION */ - -#ifdef __cplusplus -extern \"C\" { -#endif - -#if defined __GNUC__ && (((__GNUC__ == 4) && (__GNUC_MINOR__ >= 4)) || (__GNUC__ > 4)) -#pragma GCC diagnostic ignored \"-Wstrict-prototypes\" -#endif - -/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ -#if defined _WIN32 || defined __CYGWIN__ || defined _WIN32_WCE -/* DATA imports from DLLs on WIN32 can't be const, because runtime - relocations are performed -- see ld's documentation on pseudo-relocs. */ -# define LT_DLSYM_CONST -#elif defined __osf__ -/* This system does not cope well with relocations in const data. */ -# define LT_DLSYM_CONST -#else -# define LT_DLSYM_CONST const -#endif - -#define STREQ(s1, s2) (strcmp ((s1), (s2)) == 0) - -/* External symbol declarations for the compiler. */\ -" - - if test yes = "$dlself"; then - func_verbose "generating symbol list for '$output'" - - $opt_dry_run || echo ': @PROGRAM@ ' > "$nlist" - - # Add our own program objects to the symbol list. - progfiles=`$ECHO "$objs$old_deplibs" | $SP2NL | $SED "$lo2o" | $NL2SP` - for progfile in $progfiles; do - func_to_tool_file "$progfile" func_convert_file_msys_to_w32 - func_verbose "extracting global C symbols from '$func_to_tool_file_result'" - $opt_dry_run || eval "$NM $func_to_tool_file_result | $global_symbol_pipe >> '$nlist'" - done - - if test -n "$exclude_expsyms"; then - $opt_dry_run || { - eval '$EGREP -v " ($exclude_expsyms)$" "$nlist" > "$nlist"T' - eval '$MV "$nlist"T "$nlist"' - } - fi - - if test -n "$export_symbols_regex"; then - $opt_dry_run || { - eval '$EGREP -e "$export_symbols_regex" "$nlist" > "$nlist"T' - eval '$MV "$nlist"T "$nlist"' - } - fi - - # Prepare the list of exported symbols - if test -z "$export_symbols"; then - export_symbols=$output_objdir/$outputname.exp - $opt_dry_run || { - $RM $export_symbols - eval "$SED -n -e '/^: @PROGRAM@ $/d' -e 's/^.* \(.*\)$/\1/p' "'< "$nlist" > "$export_symbols"' - case $host in - *cygwin* | *mingw* | *cegcc* ) - eval "echo EXPORTS "'> "$output_objdir/$outputname.def"' - eval 'cat "$export_symbols" >> "$output_objdir/$outputname.def"' - ;; - esac - } - else - $opt_dry_run || { - eval "$SED -e 's/\([].[*^$]\)/\\\\\1/g' -e 's/^/ /' -e 's/$/$/'"' < "$export_symbols" > "$output_objdir/$outputname.exp"' - eval '$GREP -f "$output_objdir/$outputname.exp" < "$nlist" > "$nlist"T' - eval '$MV "$nlist"T "$nlist"' - case $host in - *cygwin* | *mingw* | *cegcc* ) - eval "echo EXPORTS "'> "$output_objdir/$outputname.def"' - eval 'cat "$nlist" >> "$output_objdir/$outputname.def"' - ;; - esac - } - fi - fi - - for dlprefile in $dlprefiles; do - func_verbose "extracting global C symbols from '$dlprefile'" - func_basename "$dlprefile" - name=$func_basename_result - case $host in - *cygwin* | *mingw* | *cegcc* ) - # if an import library, we need to obtain dlname - if func_win32_import_lib_p "$dlprefile"; then - func_tr_sh "$dlprefile" - eval "curr_lafile=\$libfile_$func_tr_sh_result" - dlprefile_dlbasename= - if test -n "$curr_lafile" && func_lalib_p "$curr_lafile"; then - # Use subshell, to avoid clobbering current variable values - dlprefile_dlname=`source "$curr_lafile" && echo "$dlname"` - if test -n "$dlprefile_dlname"; then - func_basename "$dlprefile_dlname" - dlprefile_dlbasename=$func_basename_result - else - # no lafile. user explicitly requested -dlpreopen . - $sharedlib_from_linklib_cmd "$dlprefile" - dlprefile_dlbasename=$sharedlib_from_linklib_result - fi - fi - $opt_dry_run || { - if test -n "$dlprefile_dlbasename"; then - eval '$ECHO ": $dlprefile_dlbasename" >> "$nlist"' - else - func_warning "Could not compute DLL name from $name" - eval '$ECHO ": $name " >> "$nlist"' - fi - func_to_tool_file "$dlprefile" func_convert_file_msys_to_w32 - eval "$NM \"$func_to_tool_file_result\" 2>/dev/null | $global_symbol_pipe | - $SED -e '/I __imp/d' -e 's/I __nm_/D /;s/_nm__//' >> '$nlist'" - } - else # not an import lib - $opt_dry_run || { - eval '$ECHO ": $name " >> "$nlist"' - func_to_tool_file "$dlprefile" func_convert_file_msys_to_w32 - eval "$NM \"$func_to_tool_file_result\" 2>/dev/null | $global_symbol_pipe >> '$nlist'" - } - fi - ;; - *) - $opt_dry_run || { - eval '$ECHO ": $name " >> "$nlist"' - func_to_tool_file "$dlprefile" func_convert_file_msys_to_w32 - eval "$NM \"$func_to_tool_file_result\" 2>/dev/null | $global_symbol_pipe >> '$nlist'" - } - ;; - esac - done - - $opt_dry_run || { - # Make sure we have at least an empty file. - test -f "$nlist" || : > "$nlist" - - if test -n "$exclude_expsyms"; then - $EGREP -v " ($exclude_expsyms)$" "$nlist" > "$nlist"T - $MV "$nlist"T "$nlist" - fi - - # Try sorting and uniquifying the output. - if $GREP -v "^: " < "$nlist" | - if sort -k 3 /dev/null 2>&1; then - sort -k 3 - else - sort +2 - fi | - uniq > "$nlist"S; then - : - else - $GREP -v "^: " < "$nlist" > "$nlist"S - fi - - if test -f "$nlist"S; then - eval "$global_symbol_to_cdecl"' < "$nlist"S >> "$output_objdir/$my_dlsyms"' - else - echo '/* NONE */' >> "$output_objdir/$my_dlsyms" - fi - - func_show_eval '$RM "${nlist}I"' - if test -n "$global_symbol_to_import"; then - eval "$global_symbol_to_import"' < "$nlist"S > "$nlist"I' - fi - - echo >> "$output_objdir/$my_dlsyms" "\ - -/* The mapping between symbol names and symbols. */ -typedef struct { - const char *name; - void *address; -} lt_dlsymlist; -extern LT_DLSYM_CONST lt_dlsymlist -lt_${my_prefix}_LTX_preloaded_symbols[];\ -" - - if test -s "$nlist"I; then - echo >> "$output_objdir/$my_dlsyms" "\ -static void lt_syminit(void) -{ - LT_DLSYM_CONST lt_dlsymlist *symbol = lt_${my_prefix}_LTX_preloaded_symbols; - for (; symbol->name; ++symbol) - {" - $SED 's/.*/ if (STREQ (symbol->name, \"&\")) symbol->address = (void *) \&&;/' < "$nlist"I >> "$output_objdir/$my_dlsyms" - echo >> "$output_objdir/$my_dlsyms" "\ - } -}" - fi - echo >> "$output_objdir/$my_dlsyms" "\ -LT_DLSYM_CONST lt_dlsymlist -lt_${my_prefix}_LTX_preloaded_symbols[] = -{ {\"$my_originator\", (void *) 0}," - - if test -s "$nlist"I; then - echo >> "$output_objdir/$my_dlsyms" "\ - {\"@INIT@\", (void *) <_syminit}," - fi - - case $need_lib_prefix in - no) - eval "$global_symbol_to_c_name_address" < "$nlist" >> "$output_objdir/$my_dlsyms" - ;; - *) - eval "$global_symbol_to_c_name_address_lib_prefix" < "$nlist" >> "$output_objdir/$my_dlsyms" - ;; - esac - echo >> "$output_objdir/$my_dlsyms" "\ - {0, (void *) 0} -}; - -/* This works around a problem in FreeBSD linker */ -#ifdef FREEBSD_WORKAROUND -static const void *lt_preloaded_setup() { - return lt_${my_prefix}_LTX_preloaded_symbols; -} -#endif - -#ifdef __cplusplus -} -#endif\ -" - } # !$opt_dry_run - - pic_flag_for_symtable= - case "$compile_command " in - *" -static "*) ;; - *) - case $host in - # compiling the symbol table file with pic_flag works around - # a FreeBSD bug that causes programs to crash when -lm is - # linked before any other PIC object. But we must not use - # pic_flag when linking with -static. The problem exists in - # FreeBSD 2.2.6 and is fixed in FreeBSD 3.1. - *-*-freebsd2.*|*-*-freebsd3.0*|*-*-freebsdelf3.0*) - pic_flag_for_symtable=" $pic_flag -DFREEBSD_WORKAROUND" ;; - *-*-hpux*) - pic_flag_for_symtable=" $pic_flag" ;; - *) - $my_pic_p && pic_flag_for_symtable=" $pic_flag" - ;; - esac - ;; - esac - symtab_cflags= - for arg in $LTCFLAGS; do - case $arg in - -pie | -fpie | -fPIE) ;; - *) func_append symtab_cflags " $arg" ;; - esac - done - - # Now compile the dynamic symbol file. - func_show_eval '(cd $output_objdir && $LTCC$symtab_cflags -c$no_builtin_flag$pic_flag_for_symtable "$my_dlsyms")' 'exit $?' - - # Clean up the generated files. - func_show_eval '$RM "$output_objdir/$my_dlsyms" "$nlist" "${nlist}S" "${nlist}T" "${nlist}I"' - - # Transform the symbol file into the correct name. - symfileobj=$output_objdir/${my_outputname}S.$objext - case $host in - *cygwin* | *mingw* | *cegcc* ) - if test -f "$output_objdir/$my_outputname.def"; then - compile_command=`$ECHO "$compile_command" | $SED "s%@SYMFILE@%$output_objdir/$my_outputname.def $symfileobj%"` - finalize_command=`$ECHO "$finalize_command" | $SED "s%@SYMFILE@%$output_objdir/$my_outputname.def $symfileobj%"` - else - compile_command=`$ECHO "$compile_command" | $SED "s%@SYMFILE@%$symfileobj%"` - finalize_command=`$ECHO "$finalize_command" | $SED "s%@SYMFILE@%$symfileobj%"` - fi - ;; - *) - compile_command=`$ECHO "$compile_command" | $SED "s%@SYMFILE@%$symfileobj%"` - finalize_command=`$ECHO "$finalize_command" | $SED "s%@SYMFILE@%$symfileobj%"` - ;; - esac - ;; - *) - func_fatal_error "unknown suffix for '$my_dlsyms'" - ;; - esac - else - # We keep going just in case the user didn't refer to - # lt_preloaded_symbols. The linker will fail if global_symbol_pipe - # really was required. - - # Nullify the symbol file. - compile_command=`$ECHO "$compile_command" | $SED "s% @SYMFILE@%%"` - finalize_command=`$ECHO "$finalize_command" | $SED "s% @SYMFILE@%%"` - fi -} - -# func_cygming_gnu_implib_p ARG -# This predicate returns with zero status (TRUE) if -# ARG is a GNU/binutils-style import library. Returns -# with nonzero status (FALSE) otherwise. -func_cygming_gnu_implib_p () -{ - $debug_cmd - - func_to_tool_file "$1" func_convert_file_msys_to_w32 - func_cygming_gnu_implib_tmp=`$NM "$func_to_tool_file_result" | eval "$global_symbol_pipe" | $EGREP ' (_head_[A-Za-z0-9_]+_[ad]l*|[A-Za-z0-9_]+_[ad]l*_iname)$'` - test -n "$func_cygming_gnu_implib_tmp" -} - -# func_cygming_ms_implib_p ARG -# This predicate returns with zero status (TRUE) if -# ARG is an MS-style import library. Returns -# with nonzero status (FALSE) otherwise. -func_cygming_ms_implib_p () -{ - $debug_cmd - - func_to_tool_file "$1" func_convert_file_msys_to_w32 - func_cygming_ms_implib_tmp=`$NM "$func_to_tool_file_result" | eval "$global_symbol_pipe" | $GREP '_NULL_IMPORT_DESCRIPTOR'` - test -n "$func_cygming_ms_implib_tmp" -} - -# func_win32_libid arg -# return the library type of file 'arg' -# -# Need a lot of goo to handle *both* DLLs and import libs -# Has to be a shell function in order to 'eat' the argument -# that is supplied when $file_magic_command is called. -# Despite the name, also deal with 64 bit binaries. -func_win32_libid () -{ - $debug_cmd - - win32_libid_type=unknown - win32_fileres=`file -L $1 2>/dev/null` - case $win32_fileres in - *ar\ archive\ import\ library*) # definitely import - win32_libid_type="x86 archive import" - ;; - *ar\ archive*) # could be an import, or static - # Keep the egrep pattern in sync with the one in _LT_CHECK_MAGIC_METHOD. - if eval $OBJDUMP -f $1 | $SED -e '10q' 2>/dev/null | - $EGREP 'file format (pei*-i386(.*architecture: i386)?|pe-arm-wince|pe-x86-64)' >/dev/null; then - case $nm_interface in - "MS dumpbin") - if func_cygming_ms_implib_p "$1" || - func_cygming_gnu_implib_p "$1" - then - win32_nmres=import - else - win32_nmres= - fi - ;; - *) - func_to_tool_file "$1" func_convert_file_msys_to_w32 - win32_nmres=`eval $NM -f posix -A \"$func_to_tool_file_result\" | - $SED -n -e ' - 1,100{ - / I /{ - s|.*|import| - p - q - } - }'` - ;; - esac - case $win32_nmres in - import*) win32_libid_type="x86 archive import";; - *) win32_libid_type="x86 archive static";; - esac - fi - ;; - *DLL*) - win32_libid_type="x86 DLL" - ;; - *executable*) # but shell scripts are "executable" too... - case $win32_fileres in - *MS\ Windows\ PE\ Intel*) - win32_libid_type="x86 DLL" - ;; - esac - ;; - esac - $ECHO "$win32_libid_type" -} - -# func_cygming_dll_for_implib ARG -# -# Platform-specific function to extract the -# name of the DLL associated with the specified -# import library ARG. -# Invoked by eval'ing the libtool variable -# $sharedlib_from_linklib_cmd -# Result is available in the variable -# $sharedlib_from_linklib_result -func_cygming_dll_for_implib () -{ - $debug_cmd - - sharedlib_from_linklib_result=`$DLLTOOL --identify-strict --identify "$1"` -} - -# func_cygming_dll_for_implib_fallback_core SECTION_NAME LIBNAMEs -# -# The is the core of a fallback implementation of a -# platform-specific function to extract the name of the -# DLL associated with the specified import library LIBNAME. -# -# SECTION_NAME is either .idata$6 or .idata$7, depending -# on the platform and compiler that created the implib. -# -# Echos the name of the DLL associated with the -# specified import library. -func_cygming_dll_for_implib_fallback_core () -{ - $debug_cmd - - match_literal=`$ECHO "$1" | $SED "$sed_make_literal_regex"` - $OBJDUMP -s --section "$1" "$2" 2>/dev/null | - $SED '/^Contents of section '"$match_literal"':/{ - # Place marker at beginning of archive member dllname section - s/.*/====MARK====/ - p - d - } - # These lines can sometimes be longer than 43 characters, but - # are always uninteresting - /:[ ]*file format pe[i]\{,1\}-/d - /^In archive [^:]*:/d - # Ensure marker is printed - /^====MARK====/p - # Remove all lines with less than 43 characters - /^.\{43\}/!d - # From remaining lines, remove first 43 characters - s/^.\{43\}//' | - $SED -n ' - # Join marker and all lines until next marker into a single line - /^====MARK====/ b para - H - $ b para - b - :para - x - s/\n//g - # Remove the marker - s/^====MARK====// - # Remove trailing dots and whitespace - s/[\. \t]*$// - # Print - /./p' | - # we now have a list, one entry per line, of the stringified - # contents of the appropriate section of all members of the - # archive that possess that section. Heuristic: eliminate - # all those that have a first or second character that is - # a '.' (that is, objdump's representation of an unprintable - # character.) This should work for all archives with less than - # 0x302f exports -- but will fail for DLLs whose name actually - # begins with a literal '.' or a single character followed by - # a '.'. - # - # Of those that remain, print the first one. - $SED -e '/^\./d;/^.\./d;q' -} - -# func_cygming_dll_for_implib_fallback ARG -# Platform-specific function to extract the -# name of the DLL associated with the specified -# import library ARG. -# -# This fallback implementation is for use when $DLLTOOL -# does not support the --identify-strict option. -# Invoked by eval'ing the libtool variable -# $sharedlib_from_linklib_cmd -# Result is available in the variable -# $sharedlib_from_linklib_result -func_cygming_dll_for_implib_fallback () -{ - $debug_cmd - - if func_cygming_gnu_implib_p "$1"; then - # binutils import library - sharedlib_from_linklib_result=`func_cygming_dll_for_implib_fallback_core '.idata$7' "$1"` - elif func_cygming_ms_implib_p "$1"; then - # ms-generated import library - sharedlib_from_linklib_result=`func_cygming_dll_for_implib_fallback_core '.idata$6' "$1"` - else - # unknown - sharedlib_from_linklib_result= - fi -} - - -# func_extract_an_archive dir oldlib -func_extract_an_archive () -{ - $debug_cmd - - f_ex_an_ar_dir=$1; shift - f_ex_an_ar_oldlib=$1 - if test yes = "$lock_old_archive_extraction"; then - lockfile=$f_ex_an_ar_oldlib.lock - until $opt_dry_run || ln "$progpath" "$lockfile" 2>/dev/null; do - func_echo "Waiting for $lockfile to be removed" - sleep 2 - done - fi - func_show_eval "(cd \$f_ex_an_ar_dir && $AR x \"\$f_ex_an_ar_oldlib\")" \ - 'stat=$?; rm -f "$lockfile"; exit $stat' - if test yes = "$lock_old_archive_extraction"; then - $opt_dry_run || rm -f "$lockfile" - fi - if ($AR t "$f_ex_an_ar_oldlib" | sort | sort -uc >/dev/null 2>&1); then - : - else - func_fatal_error "object name conflicts in archive: $f_ex_an_ar_dir/$f_ex_an_ar_oldlib" - fi -} - - -# func_extract_archives gentop oldlib ... -func_extract_archives () -{ - $debug_cmd - - my_gentop=$1; shift - my_oldlibs=${1+"$@"} - my_oldobjs= - my_xlib= - my_xabs= - my_xdir= - - for my_xlib in $my_oldlibs; do - # Extract the objects. - case $my_xlib in - [\\/]* | [A-Za-z]:[\\/]*) my_xabs=$my_xlib ;; - *) my_xabs=`pwd`"/$my_xlib" ;; - esac - func_basename "$my_xlib" - my_xlib=$func_basename_result - my_xlib_u=$my_xlib - while :; do - case " $extracted_archives " in - *" $my_xlib_u "*) - func_arith $extracted_serial + 1 - extracted_serial=$func_arith_result - my_xlib_u=lt$extracted_serial-$my_xlib ;; - *) break ;; - esac - done - extracted_archives="$extracted_archives $my_xlib_u" - my_xdir=$my_gentop/$my_xlib_u - - func_mkdir_p "$my_xdir" - - case $host in - *-darwin*) - func_verbose "Extracting $my_xabs" - # Do not bother doing anything if just a dry run - $opt_dry_run || { - darwin_orig_dir=`pwd` - cd $my_xdir || exit $? - darwin_archive=$my_xabs - darwin_curdir=`pwd` - func_basename "$darwin_archive" - darwin_base_archive=$func_basename_result - darwin_arches=`$LIPO -info "$darwin_archive" 2>/dev/null | $GREP Architectures 2>/dev/null || true` - if test -n "$darwin_arches"; then - darwin_arches=`$ECHO "$darwin_arches" | $SED -e 's/.*are://'` - darwin_arch= - func_verbose "$darwin_base_archive has multiple architectures $darwin_arches" - for darwin_arch in $darwin_arches; do - func_mkdir_p "unfat-$$/$darwin_base_archive-$darwin_arch" - $LIPO -thin $darwin_arch -output "unfat-$$/$darwin_base_archive-$darwin_arch/$darwin_base_archive" "$darwin_archive" - cd "unfat-$$/$darwin_base_archive-$darwin_arch" - func_extract_an_archive "`pwd`" "$darwin_base_archive" - cd "$darwin_curdir" - $RM "unfat-$$/$darwin_base_archive-$darwin_arch/$darwin_base_archive" - done # $darwin_arches - ## Okay now we've a bunch of thin objects, gotta fatten them up :) - darwin_filelist=`find unfat-$$ -type f -name \*.o -print -o -name \*.lo -print | $SED -e "$sed_basename" | sort -u` - darwin_file= - darwin_files= - for darwin_file in $darwin_filelist; do - darwin_files=`find unfat-$$ -name $darwin_file -print | sort | $NL2SP` - $LIPO -create -output "$darwin_file" $darwin_files - done # $darwin_filelist - $RM -rf unfat-$$ - cd "$darwin_orig_dir" - else - cd $darwin_orig_dir - func_extract_an_archive "$my_xdir" "$my_xabs" - fi # $darwin_arches - } # !$opt_dry_run - ;; - *) - func_extract_an_archive "$my_xdir" "$my_xabs" - ;; - esac - my_oldobjs="$my_oldobjs "`find $my_xdir -name \*.$objext -print -o -name \*.lo -print | sort | $NL2SP` - done - - func_extract_archives_result=$my_oldobjs -} - - -# func_emit_wrapper [arg=no] -# -# Emit a libtool wrapper script on stdout. -# Don't directly open a file because we may want to -# incorporate the script contents within a cygwin/mingw -# wrapper executable. Must ONLY be called from within -# func_mode_link because it depends on a number of variables -# set therein. -# -# ARG is the value that the WRAPPER_SCRIPT_BELONGS_IN_OBJDIR -# variable will take. If 'yes', then the emitted script -# will assume that the directory where it is stored is -# the $objdir directory. This is a cygwin/mingw-specific -# behavior. -func_emit_wrapper () -{ - func_emit_wrapper_arg1=${1-no} - - $ECHO "\ -#! $SHELL - -# $output - temporary wrapper script for $objdir/$outputname -# Generated by $PROGRAM (GNU $PACKAGE) $VERSION -# -# The $output program cannot be directly executed until all the libtool -# libraries that it depends on are installed. -# -# This wrapper script should never be moved out of the build directory. -# If it is, it will not operate correctly. - -# Sed substitution that helps us do robust quoting. It backslashifies -# metacharacters that are still active within double-quoted strings. -sed_quote_subst='$sed_quote_subst' - -# Be Bourne compatible -if test -n \"\${ZSH_VERSION+set}\" && (emulate sh) >/dev/null 2>&1; then - emulate sh - NULLCMD=: - # Zsh 3.x and 4.x performs word splitting on \${1+\"\$@\"}, which - # is contrary to our usage. Disable this feature. - alias -g '\${1+\"\$@\"}'='\"\$@\"' - setopt NO_GLOB_SUBST -else - case \`(set -o) 2>/dev/null\` in *posix*) set -o posix;; esac -fi -BIN_SH=xpg4; export BIN_SH # for Tru64 -DUALCASE=1; export DUALCASE # for MKS sh - -# The HP-UX ksh and POSIX shell print the target directory to stdout -# if CDPATH is set. -(unset CDPATH) >/dev/null 2>&1 && unset CDPATH - -relink_command=\"$relink_command\" - -# This environment variable determines our operation mode. -if test \"\$libtool_install_magic\" = \"$magic\"; then - # install mode needs the following variables: - generated_by_libtool_version='$macro_version' - notinst_deplibs='$notinst_deplibs' -else - # When we are sourced in execute mode, \$file and \$ECHO are already set. - if test \"\$libtool_execute_magic\" != \"$magic\"; then - file=\"\$0\"" - - qECHO=`$ECHO "$ECHO" | $SED "$sed_quote_subst"` - $ECHO "\ - -# A function that is used when there is no print builtin or printf. -func_fallback_echo () -{ - eval 'cat <<_LTECHO_EOF -\$1 -_LTECHO_EOF' -} - ECHO=\"$qECHO\" - fi - -# Very basic option parsing. These options are (a) specific to -# the libtool wrapper, (b) are identical between the wrapper -# /script/ and the wrapper /executable/ that is used only on -# windows platforms, and (c) all begin with the string "--lt-" -# (application programs are unlikely to have options that match -# this pattern). -# -# There are only two supported options: --lt-debug and -# --lt-dump-script. There is, deliberately, no --lt-help. -# -# The first argument to this parsing function should be the -# script's $0 value, followed by "$@". -lt_option_debug= -func_parse_lt_options () -{ - lt_script_arg0=\$0 - shift - for lt_opt - do - case \"\$lt_opt\" in - --lt-debug) lt_option_debug=1 ;; - --lt-dump-script) - lt_dump_D=\`\$ECHO \"X\$lt_script_arg0\" | $SED -e 's/^X//' -e 's%/[^/]*$%%'\` - test \"X\$lt_dump_D\" = \"X\$lt_script_arg0\" && lt_dump_D=. - lt_dump_F=\`\$ECHO \"X\$lt_script_arg0\" | $SED -e 's/^X//' -e 's%^.*/%%'\` - cat \"\$lt_dump_D/\$lt_dump_F\" - exit 0 - ;; - --lt-*) - \$ECHO \"Unrecognized --lt- option: '\$lt_opt'\" 1>&2 - exit 1 - ;; - esac - done - - # Print the debug banner immediately: - if test -n \"\$lt_option_debug\"; then - echo \"$outputname:$output:\$LINENO: libtool wrapper (GNU $PACKAGE) $VERSION\" 1>&2 - fi -} - -# Used when --lt-debug. Prints its arguments to stdout -# (redirection is the responsibility of the caller) -func_lt_dump_args () -{ - lt_dump_args_N=1; - for lt_arg - do - \$ECHO \"$outputname:$output:\$LINENO: newargv[\$lt_dump_args_N]: \$lt_arg\" - lt_dump_args_N=\`expr \$lt_dump_args_N + 1\` - done -} - -# Core function for launching the target application -func_exec_program_core () -{ -" - case $host in - # Backslashes separate directories on plain windows - *-*-mingw | *-*-os2* | *-cegcc*) - $ECHO "\ - if test -n \"\$lt_option_debug\"; then - \$ECHO \"$outputname:$output:\$LINENO: newargv[0]: \$progdir\\\\\$program\" 1>&2 - func_lt_dump_args \${1+\"\$@\"} 1>&2 - fi - exec \"\$progdir\\\\\$program\" \${1+\"\$@\"} -" - ;; - - *) - $ECHO "\ - if test -n \"\$lt_option_debug\"; then - \$ECHO \"$outputname:$output:\$LINENO: newargv[0]: \$progdir/\$program\" 1>&2 - func_lt_dump_args \${1+\"\$@\"} 1>&2 - fi - exec \"\$progdir/\$program\" \${1+\"\$@\"} -" - ;; - esac - $ECHO "\ - \$ECHO \"\$0: cannot exec \$program \$*\" 1>&2 - exit 1 -} - -# A function to encapsulate launching the target application -# Strips options in the --lt-* namespace from \$@ and -# launches target application with the remaining arguments. -func_exec_program () -{ - case \" \$* \" in - *\\ --lt-*) - for lt_wr_arg - do - case \$lt_wr_arg in - --lt-*) ;; - *) set x \"\$@\" \"\$lt_wr_arg\"; shift;; - esac - shift - done ;; - esac - func_exec_program_core \${1+\"\$@\"} -} - - # Parse options - func_parse_lt_options \"\$0\" \${1+\"\$@\"} - - # Find the directory that this script lives in. - thisdir=\`\$ECHO \"\$file\" | $SED 's%/[^/]*$%%'\` - test \"x\$thisdir\" = \"x\$file\" && thisdir=. - - # Follow symbolic links until we get to the real thisdir. - file=\`ls -ld \"\$file\" | $SED -n 's/.*-> //p'\` - while test -n \"\$file\"; do - destdir=\`\$ECHO \"\$file\" | $SED 's%/[^/]*\$%%'\` - - # If there was a directory component, then change thisdir. - if test \"x\$destdir\" != \"x\$file\"; then - case \"\$destdir\" in - [\\\\/]* | [A-Za-z]:[\\\\/]*) thisdir=\"\$destdir\" ;; - *) thisdir=\"\$thisdir/\$destdir\" ;; - esac - fi - - file=\`\$ECHO \"\$file\" | $SED 's%^.*/%%'\` - file=\`ls -ld \"\$thisdir/\$file\" | $SED -n 's/.*-> //p'\` - done - - # Usually 'no', except on cygwin/mingw when embedded into - # the cwrapper. - WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=$func_emit_wrapper_arg1 - if test \"\$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR\" = \"yes\"; then - # special case for '.' - if test \"\$thisdir\" = \".\"; then - thisdir=\`pwd\` - fi - # remove .libs from thisdir - case \"\$thisdir\" in - *[\\\\/]$objdir ) thisdir=\`\$ECHO \"\$thisdir\" | $SED 's%[\\\\/][^\\\\/]*$%%'\` ;; - $objdir ) thisdir=. ;; - esac - fi - - # Try to get the absolute directory name. - absdir=\`cd \"\$thisdir\" && pwd\` - test -n \"\$absdir\" && thisdir=\"\$absdir\" -" - - if test yes = "$fast_install"; then - $ECHO "\ - program=lt-'$outputname'$exeext - progdir=\"\$thisdir/$objdir\" - - if test ! -f \"\$progdir/\$program\" || - { file=\`ls -1dt \"\$progdir/\$program\" \"\$progdir/../\$program\" 2>/dev/null | $SED 1q\`; \\ - test \"X\$file\" != \"X\$progdir/\$program\"; }; then - - file=\"\$\$-\$program\" - - if test ! -d \"\$progdir\"; then - $MKDIR \"\$progdir\" - else - $RM \"\$progdir/\$file\" - fi" - - $ECHO "\ - - # relink executable if necessary - if test -n \"\$relink_command\"; then - if relink_command_output=\`eval \$relink_command 2>&1\`; then : - else - \$ECHO \"\$relink_command_output\" >&2 - $RM \"\$progdir/\$file\" - exit 1 - fi - fi - - $MV \"\$progdir/\$file\" \"\$progdir/\$program\" 2>/dev/null || - { $RM \"\$progdir/\$program\"; - $MV \"\$progdir/\$file\" \"\$progdir/\$program\"; } - $RM \"\$progdir/\$file\" - fi" - else - $ECHO "\ - program='$outputname' - progdir=\"\$thisdir/$objdir\" -" - fi - - $ECHO "\ - - if test -f \"\$progdir/\$program\"; then" - - # fixup the dll searchpath if we need to. - # - # Fix the DLL searchpath if we need to. Do this before prepending - # to shlibpath, because on Windows, both are PATH and uninstalled - # libraries must come first. - if test -n "$dllsearchpath"; then - $ECHO "\ - # Add the dll search path components to the executable PATH - PATH=$dllsearchpath:\$PATH -" - fi - - # Export our shlibpath_var if we have one. - if test yes = "$shlibpath_overrides_runpath" && test -n "$shlibpath_var" && test -n "$temp_rpath"; then - $ECHO "\ - # Add our own library path to $shlibpath_var - $shlibpath_var=\"$temp_rpath\$$shlibpath_var\" - - # Some systems cannot cope with colon-terminated $shlibpath_var - # The second colon is a workaround for a bug in BeOS R4 sed - $shlibpath_var=\`\$ECHO \"\$$shlibpath_var\" | $SED 's/::*\$//'\` - - export $shlibpath_var -" - fi - - $ECHO "\ - if test \"\$libtool_execute_magic\" != \"$magic\"; then - # Run the actual program with our arguments. - func_exec_program \${1+\"\$@\"} - fi - else - # The program doesn't exist. - \$ECHO \"\$0: error: '\$progdir/\$program' does not exist\" 1>&2 - \$ECHO \"This script is just a wrapper for \$program.\" 1>&2 - \$ECHO \"See the $PACKAGE documentation for more information.\" 1>&2 - exit 1 - fi -fi\ -" -} - - -# func_emit_cwrapperexe_src -# emit the source code for a wrapper executable on stdout -# Must ONLY be called from within func_mode_link because -# it depends on a number of variable set therein. -func_emit_cwrapperexe_src () -{ - cat < -#include -#ifdef _MSC_VER -# include -# include -# include -#else -# include -# include -# ifdef __CYGWIN__ -# include -# endif -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#define STREQ(s1, s2) (strcmp ((s1), (s2)) == 0) - -/* declarations of non-ANSI functions */ -#if defined __MINGW32__ -# ifdef __STRICT_ANSI__ -int _putenv (const char *); -# endif -#elif defined __CYGWIN__ -# ifdef __STRICT_ANSI__ -char *realpath (const char *, char *); -int putenv (char *); -int setenv (const char *, const char *, int); -# endif -/* #elif defined other_platform || defined ... */ -#endif - -/* portability defines, excluding path handling macros */ -#if defined _MSC_VER -# define setmode _setmode -# define stat _stat -# define chmod _chmod -# define getcwd _getcwd -# define putenv _putenv -# define S_IXUSR _S_IEXEC -#elif defined __MINGW32__ -# define setmode _setmode -# define stat _stat -# define chmod _chmod -# define getcwd _getcwd -# define putenv _putenv -#elif defined __CYGWIN__ -# define HAVE_SETENV -# define FOPEN_WB "wb" -/* #elif defined other platforms ... */ -#endif - -#if defined PATH_MAX -# define LT_PATHMAX PATH_MAX -#elif defined MAXPATHLEN -# define LT_PATHMAX MAXPATHLEN -#else -# define LT_PATHMAX 1024 -#endif - -#ifndef S_IXOTH -# define S_IXOTH 0 -#endif -#ifndef S_IXGRP -# define S_IXGRP 0 -#endif - -/* path handling portability macros */ -#ifndef DIR_SEPARATOR -# define DIR_SEPARATOR '/' -# define PATH_SEPARATOR ':' -#endif - -#if defined _WIN32 || defined __MSDOS__ || defined __DJGPP__ || \ - defined __OS2__ -# define HAVE_DOS_BASED_FILE_SYSTEM -# define FOPEN_WB "wb" -# ifndef DIR_SEPARATOR_2 -# define DIR_SEPARATOR_2 '\\' -# endif -# ifndef PATH_SEPARATOR_2 -# define PATH_SEPARATOR_2 ';' -# endif -#endif - -#ifndef DIR_SEPARATOR_2 -# define IS_DIR_SEPARATOR(ch) ((ch) == DIR_SEPARATOR) -#else /* DIR_SEPARATOR_2 */ -# define IS_DIR_SEPARATOR(ch) \ - (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2)) -#endif /* DIR_SEPARATOR_2 */ - -#ifndef PATH_SEPARATOR_2 -# define IS_PATH_SEPARATOR(ch) ((ch) == PATH_SEPARATOR) -#else /* PATH_SEPARATOR_2 */ -# define IS_PATH_SEPARATOR(ch) ((ch) == PATH_SEPARATOR_2) -#endif /* PATH_SEPARATOR_2 */ - -#ifndef FOPEN_WB -# define FOPEN_WB "w" -#endif -#ifndef _O_BINARY -# define _O_BINARY 0 -#endif - -#define XMALLOC(type, num) ((type *) xmalloc ((num) * sizeof(type))) -#define XFREE(stale) do { \ - if (stale) { free (stale); stale = 0; } \ -} while (0) - -#if defined LT_DEBUGWRAPPER -static int lt_debug = 1; -#else -static int lt_debug = 0; -#endif - -const char *program_name = "libtool-wrapper"; /* in case xstrdup fails */ - -void *xmalloc (size_t num); -char *xstrdup (const char *string); -const char *base_name (const char *name); -char *find_executable (const char *wrapper); -char *chase_symlinks (const char *pathspec); -int make_executable (const char *path); -int check_executable (const char *path); -char *strendzap (char *str, const char *pat); -void lt_debugprintf (const char *file, int line, const char *fmt, ...); -void lt_fatal (const char *file, int line, const char *message, ...); -static const char *nonnull (const char *s); -static const char *nonempty (const char *s); -void lt_setenv (const char *name, const char *value); -char *lt_extend_str (const char *orig_value, const char *add, int to_end); -void lt_update_exe_path (const char *name, const char *value); -void lt_update_lib_path (const char *name, const char *value); -char **prepare_spawn (char **argv); -void lt_dump_script (FILE *f); -EOF - - cat <= 0) - && (st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) - return 1; - else - return 0; -} - -int -make_executable (const char *path) -{ - int rval = 0; - struct stat st; - - lt_debugprintf (__FILE__, __LINE__, "(make_executable): %s\n", - nonempty (path)); - if ((!path) || (!*path)) - return 0; - - if (stat (path, &st) >= 0) - { - rval = chmod (path, st.st_mode | S_IXOTH | S_IXGRP | S_IXUSR); - } - return rval; -} - -/* Searches for the full path of the wrapper. Returns - newly allocated full path name if found, NULL otherwise - Does not chase symlinks, even on platforms that support them. -*/ -char * -find_executable (const char *wrapper) -{ - int has_slash = 0; - const char *p; - const char *p_next; - /* static buffer for getcwd */ - char tmp[LT_PATHMAX + 1]; - size_t tmp_len; - char *concat_name; - - lt_debugprintf (__FILE__, __LINE__, "(find_executable): %s\n", - nonempty (wrapper)); - - if ((wrapper == NULL) || (*wrapper == '\0')) - return NULL; - - /* Absolute path? */ -#if defined HAVE_DOS_BASED_FILE_SYSTEM - if (isalpha ((unsigned char) wrapper[0]) && wrapper[1] == ':') - { - concat_name = xstrdup (wrapper); - if (check_executable (concat_name)) - return concat_name; - XFREE (concat_name); - } - else - { -#endif - if (IS_DIR_SEPARATOR (wrapper[0])) - { - concat_name = xstrdup (wrapper); - if (check_executable (concat_name)) - return concat_name; - XFREE (concat_name); - } -#if defined HAVE_DOS_BASED_FILE_SYSTEM - } -#endif - - for (p = wrapper; *p; p++) - if (*p == '/') - { - has_slash = 1; - break; - } - if (!has_slash) - { - /* no slashes; search PATH */ - const char *path = getenv ("PATH"); - if (path != NULL) - { - for (p = path; *p; p = p_next) - { - const char *q; - size_t p_len; - for (q = p; *q; q++) - if (IS_PATH_SEPARATOR (*q)) - break; - p_len = (size_t) (q - p); - p_next = (*q == '\0' ? q : q + 1); - if (p_len == 0) - { - /* empty path: current directory */ - if (getcwd (tmp, LT_PATHMAX) == NULL) - lt_fatal (__FILE__, __LINE__, "getcwd failed: %s", - nonnull (strerror (errno))); - tmp_len = strlen (tmp); - concat_name = - XMALLOC (char, tmp_len + 1 + strlen (wrapper) + 1); - memcpy (concat_name, tmp, tmp_len); - concat_name[tmp_len] = '/'; - strcpy (concat_name + tmp_len + 1, wrapper); - } - else - { - concat_name = - XMALLOC (char, p_len + 1 + strlen (wrapper) + 1); - memcpy (concat_name, p, p_len); - concat_name[p_len] = '/'; - strcpy (concat_name + p_len + 1, wrapper); - } - if (check_executable (concat_name)) - return concat_name; - XFREE (concat_name); - } - } - /* not found in PATH; assume curdir */ - } - /* Relative path | not found in path: prepend cwd */ - if (getcwd (tmp, LT_PATHMAX) == NULL) - lt_fatal (__FILE__, __LINE__, "getcwd failed: %s", - nonnull (strerror (errno))); - tmp_len = strlen (tmp); - concat_name = XMALLOC (char, tmp_len + 1 + strlen (wrapper) + 1); - memcpy (concat_name, tmp, tmp_len); - concat_name[tmp_len] = '/'; - strcpy (concat_name + tmp_len + 1, wrapper); - - if (check_executable (concat_name)) - return concat_name; - XFREE (concat_name); - return NULL; -} - -char * -chase_symlinks (const char *pathspec) -{ -#ifndef S_ISLNK - return xstrdup (pathspec); -#else - char buf[LT_PATHMAX]; - struct stat s; - char *tmp_pathspec = xstrdup (pathspec); - char *p; - int has_symlinks = 0; - while (strlen (tmp_pathspec) && !has_symlinks) - { - lt_debugprintf (__FILE__, __LINE__, - "checking path component for symlinks: %s\n", - tmp_pathspec); - if (lstat (tmp_pathspec, &s) == 0) - { - if (S_ISLNK (s.st_mode) != 0) - { - has_symlinks = 1; - break; - } - - /* search backwards for last DIR_SEPARATOR */ - p = tmp_pathspec + strlen (tmp_pathspec) - 1; - while ((p > tmp_pathspec) && (!IS_DIR_SEPARATOR (*p))) - p--; - if ((p == tmp_pathspec) && (!IS_DIR_SEPARATOR (*p))) - { - /* no more DIR_SEPARATORS left */ - break; - } - *p = '\0'; - } - else - { - lt_fatal (__FILE__, __LINE__, - "error accessing file \"%s\": %s", - tmp_pathspec, nonnull (strerror (errno))); - } - } - XFREE (tmp_pathspec); - - if (!has_symlinks) - { - return xstrdup (pathspec); - } - - tmp_pathspec = realpath (pathspec, buf); - if (tmp_pathspec == 0) - { - lt_fatal (__FILE__, __LINE__, - "could not follow symlinks for %s", pathspec); - } - return xstrdup (tmp_pathspec); -#endif -} - -char * -strendzap (char *str, const char *pat) -{ - size_t len, patlen; - - assert (str != NULL); - assert (pat != NULL); - - len = strlen (str); - patlen = strlen (pat); - - if (patlen <= len) - { - str += len - patlen; - if (STREQ (str, pat)) - *str = '\0'; - } - return str; -} - -void -lt_debugprintf (const char *file, int line, const char *fmt, ...) -{ - va_list args; - if (lt_debug) - { - (void) fprintf (stderr, "%s:%s:%d: ", program_name, file, line); - va_start (args, fmt); - (void) vfprintf (stderr, fmt, args); - va_end (args); - } -} - -static void -lt_error_core (int exit_status, const char *file, - int line, const char *mode, - const char *message, va_list ap) -{ - fprintf (stderr, "%s:%s:%d: %s: ", program_name, file, line, mode); - vfprintf (stderr, message, ap); - fprintf (stderr, ".\n"); - - if (exit_status >= 0) - exit (exit_status); -} - -void -lt_fatal (const char *file, int line, const char *message, ...) -{ - va_list ap; - va_start (ap, message); - lt_error_core (EXIT_FAILURE, file, line, "FATAL", message, ap); - va_end (ap); -} - -static const char * -nonnull (const char *s) -{ - return s ? s : "(null)"; -} - -static const char * -nonempty (const char *s) -{ - return (s && !*s) ? "(empty)" : nonnull (s); -} - -void -lt_setenv (const char *name, const char *value) -{ - lt_debugprintf (__FILE__, __LINE__, - "(lt_setenv) setting '%s' to '%s'\n", - nonnull (name), nonnull (value)); - { -#ifdef HAVE_SETENV - /* always make a copy, for consistency with !HAVE_SETENV */ - char *str = xstrdup (value); - setenv (name, str, 1); -#else - size_t len = strlen (name) + 1 + strlen (value) + 1; - char *str = XMALLOC (char, len); - sprintf (str, "%s=%s", name, value); - if (putenv (str) != EXIT_SUCCESS) - { - XFREE (str); - } -#endif - } -} - -char * -lt_extend_str (const char *orig_value, const char *add, int to_end) -{ - char *new_value; - if (orig_value && *orig_value) - { - size_t orig_value_len = strlen (orig_value); - size_t add_len = strlen (add); - new_value = XMALLOC (char, add_len + orig_value_len + 1); - if (to_end) - { - strcpy (new_value, orig_value); - strcpy (new_value + orig_value_len, add); - } - else - { - strcpy (new_value, add); - strcpy (new_value + add_len, orig_value); - } - } - else - { - new_value = xstrdup (add); - } - return new_value; -} - -void -lt_update_exe_path (const char *name, const char *value) -{ - lt_debugprintf (__FILE__, __LINE__, - "(lt_update_exe_path) modifying '%s' by prepending '%s'\n", - nonnull (name), nonnull (value)); - - if (name && *name && value && *value) - { - char *new_value = lt_extend_str (getenv (name), value, 0); - /* some systems can't cope with a ':'-terminated path #' */ - size_t len = strlen (new_value); - while ((len > 0) && IS_PATH_SEPARATOR (new_value[len-1])) - { - new_value[--len] = '\0'; - } - lt_setenv (name, new_value); - XFREE (new_value); - } -} - -void -lt_update_lib_path (const char *name, const char *value) -{ - lt_debugprintf (__FILE__, __LINE__, - "(lt_update_lib_path) modifying '%s' by prepending '%s'\n", - nonnull (name), nonnull (value)); - - if (name && *name && value && *value) - { - char *new_value = lt_extend_str (getenv (name), value, 0); - lt_setenv (name, new_value); - XFREE (new_value); - } -} - -EOF - case $host_os in - mingw*) - cat <<"EOF" - -/* Prepares an argument vector before calling spawn(). - Note that spawn() does not by itself call the command interpreter - (getenv ("COMSPEC") != NULL ? getenv ("COMSPEC") : - ({ OSVERSIONINFO v; v.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - GetVersionEx(&v); - v.dwPlatformId == VER_PLATFORM_WIN32_NT; - }) ? "cmd.exe" : "command.com"). - Instead it simply concatenates the arguments, separated by ' ', and calls - CreateProcess(). We must quote the arguments since Win32 CreateProcess() - interprets characters like ' ', '\t', '\\', '"' (but not '<' and '>') in a - special way: - - Space and tab are interpreted as delimiters. They are not treated as - delimiters if they are surrounded by double quotes: "...". - - Unescaped double quotes are removed from the input. Their only effect is - that within double quotes, space and tab are treated like normal - characters. - - Backslashes not followed by double quotes are not special. - - But 2*n+1 backslashes followed by a double quote become - n backslashes followed by a double quote (n >= 0): - \" -> " - \\\" -> \" - \\\\\" -> \\" - */ -#define SHELL_SPECIAL_CHARS "\"\\ \001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" -#define SHELL_SPACE_CHARS " \001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" -char ** -prepare_spawn (char **argv) -{ - size_t argc; - char **new_argv; - size_t i; - - /* Count number of arguments. */ - for (argc = 0; argv[argc] != NULL; argc++) - ; - - /* Allocate new argument vector. */ - new_argv = XMALLOC (char *, argc + 1); - - /* Put quoted arguments into the new argument vector. */ - for (i = 0; i < argc; i++) - { - const char *string = argv[i]; - - if (string[0] == '\0') - new_argv[i] = xstrdup ("\"\""); - else if (strpbrk (string, SHELL_SPECIAL_CHARS) != NULL) - { - int quote_around = (strpbrk (string, SHELL_SPACE_CHARS) != NULL); - size_t length; - unsigned int backslashes; - const char *s; - char *quoted_string; - char *p; - - length = 0; - backslashes = 0; - if (quote_around) - length++; - for (s = string; *s != '\0'; s++) - { - char c = *s; - if (c == '"') - length += backslashes + 1; - length++; - if (c == '\\') - backslashes++; - else - backslashes = 0; - } - if (quote_around) - length += backslashes + 1; - - quoted_string = XMALLOC (char, length + 1); - - p = quoted_string; - backslashes = 0; - if (quote_around) - *p++ = '"'; - for (s = string; *s != '\0'; s++) - { - char c = *s; - if (c == '"') - { - unsigned int j; - for (j = backslashes + 1; j > 0; j--) - *p++ = '\\'; - } - *p++ = c; - if (c == '\\') - backslashes++; - else - backslashes = 0; - } - if (quote_around) - { - unsigned int j; - for (j = backslashes; j > 0; j--) - *p++ = '\\'; - *p++ = '"'; - } - *p = '\0'; - - new_argv[i] = quoted_string; - } - else - new_argv[i] = (char *) string; - } - new_argv[argc] = NULL; - - return new_argv; -} -EOF - ;; - esac - - cat <<"EOF" -void lt_dump_script (FILE* f) -{ -EOF - func_emit_wrapper yes | - $SED -n -e ' -s/^\(.\{79\}\)\(..*\)/\1\ -\2/ -h -s/\([\\"]\)/\\\1/g -s/$/\\n/ -s/\([^\n]*\).*/ fputs ("\1", f);/p -g -D' - cat <<"EOF" -} -EOF -} -# end: func_emit_cwrapperexe_src - -# func_win32_import_lib_p ARG -# True if ARG is an import lib, as indicated by $file_magic_cmd -func_win32_import_lib_p () -{ - $debug_cmd - - case `eval $file_magic_cmd \"\$1\" 2>/dev/null | $SED -e 10q` in - *import*) : ;; - *) false ;; - esac -} - -# func_suncc_cstd_abi -# !!ONLY CALL THIS FOR SUN CC AFTER $compile_command IS FULLY EXPANDED!! -# Several compiler flags select an ABI that is incompatible with the -# Cstd library. Avoid specifying it if any are in CXXFLAGS. -func_suncc_cstd_abi () -{ - $debug_cmd - - case " $compile_command " in - *" -compat=g "*|*\ -std=c++[0-9][0-9]\ *|*" -library=stdcxx4 "*|*" -library=stlport4 "*) - suncc_use_cstd_abi=no - ;; - *) - suncc_use_cstd_abi=yes - ;; - esac -} - -# func_mode_link arg... -func_mode_link () -{ - $debug_cmd - - case $host in - *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*) - # It is impossible to link a dll without this setting, and - # we shouldn't force the makefile maintainer to figure out - # what system we are compiling for in order to pass an extra - # flag for every libtool invocation. - # allow_undefined=no - - # FIXME: Unfortunately, there are problems with the above when trying - # to make a dll that has undefined symbols, in which case not - # even a static library is built. For now, we need to specify - # -no-undefined on the libtool link line when we can be certain - # that all symbols are satisfied, otherwise we get a static library. - allow_undefined=yes - ;; - *) - allow_undefined=yes - ;; - esac - libtool_args=$nonopt - base_compile="$nonopt $@" - compile_command=$nonopt - finalize_command=$nonopt - - compile_rpath= - finalize_rpath= - compile_shlibpath= - finalize_shlibpath= - convenience= - old_convenience= - deplibs= - old_deplibs= - compiler_flags= - linker_flags= - dllsearchpath= - lib_search_path=`pwd` - inst_prefix_dir= - new_inherited_linker_flags= - - avoid_version=no - bindir= - dlfiles= - dlprefiles= - dlself=no - export_dynamic=no - export_symbols= - export_symbols_regex= - generated= - libobjs= - ltlibs= - module=no - no_install=no - objs= - os2dllname= - non_pic_objects= - precious_files_regex= - prefer_static_libs=no - preload=false - prev= - prevarg= - release= - rpath= - xrpath= - perm_rpath= - temp_rpath= - thread_safe=no - vinfo= - vinfo_number=no - weak_libs= - single_module=$wl-single_module - func_infer_tag $base_compile - - # We need to know -static, to get the right output filenames. - for arg - do - case $arg in - -shared) - test yes != "$build_libtool_libs" \ - && func_fatal_configuration "cannot build a shared library" - build_old_libs=no - break - ;; - -all-static | -static | -static-libtool-libs) - case $arg in - -all-static) - if test yes = "$build_libtool_libs" && test -z "$link_static_flag"; then - func_warning "complete static linking is impossible in this configuration" - fi - if test -n "$link_static_flag"; then - dlopen_self=$dlopen_self_static - fi - prefer_static_libs=yes - ;; - -static) - if test -z "$pic_flag" && test -n "$link_static_flag"; then - dlopen_self=$dlopen_self_static - fi - prefer_static_libs=built - ;; - -static-libtool-libs) - if test -z "$pic_flag" && test -n "$link_static_flag"; then - dlopen_self=$dlopen_self_static - fi - prefer_static_libs=yes - ;; - esac - build_libtool_libs=no - build_old_libs=yes - break - ;; - esac - done - - # See if our shared archives depend on static archives. - test -n "$old_archive_from_new_cmds" && build_old_libs=yes - - # Go through the arguments, transforming them on the way. - while test "$#" -gt 0; do - arg=$1 - shift - func_quote_for_eval "$arg" - qarg=$func_quote_for_eval_unquoted_result - func_append libtool_args " $func_quote_for_eval_result" - - # If the previous option needs an argument, assign it. - if test -n "$prev"; then - case $prev in - output) - func_append compile_command " @OUTPUT@" - func_append finalize_command " @OUTPUT@" - ;; - esac - - case $prev in - bindir) - bindir=$arg - prev= - continue - ;; - dlfiles|dlprefiles) - $preload || { - # Add the symbol object into the linking commands. - func_append compile_command " @SYMFILE@" - func_append finalize_command " @SYMFILE@" - preload=: - } - case $arg in - *.la | *.lo) ;; # We handle these cases below. - force) - if test no = "$dlself"; then - dlself=needless - export_dynamic=yes - fi - prev= - continue - ;; - self) - if test dlprefiles = "$prev"; then - dlself=yes - elif test dlfiles = "$prev" && test yes != "$dlopen_self"; then - dlself=yes - else - dlself=needless - export_dynamic=yes - fi - prev= - continue - ;; - *) - if test dlfiles = "$prev"; then - func_append dlfiles " $arg" - else - func_append dlprefiles " $arg" - fi - prev= - continue - ;; - esac - ;; - expsyms) - export_symbols=$arg - test -f "$arg" \ - || func_fatal_error "symbol file '$arg' does not exist" - prev= - continue - ;; - expsyms_regex) - export_symbols_regex=$arg - prev= - continue - ;; - framework) - case $host in - *-*-darwin*) - case "$deplibs " in - *" $qarg.ltframework "*) ;; - *) func_append deplibs " $qarg.ltframework" # this is fixed later - ;; - esac - ;; - esac - prev= - continue - ;; - inst_prefix) - inst_prefix_dir=$arg - prev= - continue - ;; - mllvm) - # Clang does not use LLVM to link, so we can simply discard any - # '-mllvm $arg' options when doing the link step. - prev= - continue - ;; - objectlist) - if test -f "$arg"; then - save_arg=$arg - moreargs= - for fil in `cat "$save_arg"` - do -# func_append moreargs " $fil" - arg=$fil - # A libtool-controlled object. - - # Check to see that this really is a libtool object. - if func_lalib_unsafe_p "$arg"; then - pic_object= - non_pic_object= - - # Read the .lo file - func_source "$arg" - - if test -z "$pic_object" || - test -z "$non_pic_object" || - test none = "$pic_object" && - test none = "$non_pic_object"; then - func_fatal_error "cannot find name of object for '$arg'" - fi - - # Extract subdirectory from the argument. - func_dirname "$arg" "/" "" - xdir=$func_dirname_result - - if test none != "$pic_object"; then - # Prepend the subdirectory the object is found in. - pic_object=$xdir$pic_object - - if test dlfiles = "$prev"; then - if test yes = "$build_libtool_libs" && test yes = "$dlopen_support"; then - func_append dlfiles " $pic_object" - prev= - continue - else - # If libtool objects are unsupported, then we need to preload. - prev=dlprefiles - fi - fi - - # CHECK ME: I think I busted this. -Ossama - if test dlprefiles = "$prev"; then - # Preload the old-style object. - func_append dlprefiles " $pic_object" - prev= - fi - - # A PIC object. - func_append libobjs " $pic_object" - arg=$pic_object - fi - - # Non-PIC object. - if test none != "$non_pic_object"; then - # Prepend the subdirectory the object is found in. - non_pic_object=$xdir$non_pic_object - - # A standard non-PIC object - func_append non_pic_objects " $non_pic_object" - if test -z "$pic_object" || test none = "$pic_object"; then - arg=$non_pic_object - fi - else - # If the PIC object exists, use it instead. - # $xdir was prepended to $pic_object above. - non_pic_object=$pic_object - func_append non_pic_objects " $non_pic_object" - fi - else - # Only an error if not doing a dry-run. - if $opt_dry_run; then - # Extract subdirectory from the argument. - func_dirname "$arg" "/" "" - xdir=$func_dirname_result - - func_lo2o "$arg" - pic_object=$xdir$objdir/$func_lo2o_result - non_pic_object=$xdir$func_lo2o_result - func_append libobjs " $pic_object" - func_append non_pic_objects " $non_pic_object" - else - func_fatal_error "'$arg' is not a valid libtool object" - fi - fi - done - else - func_fatal_error "link input file '$arg' does not exist" - fi - arg=$save_arg - prev= - continue - ;; - os2dllname) - os2dllname=$arg - prev= - continue - ;; - precious_regex) - precious_files_regex=$arg - prev= - continue - ;; - release) - release=-$arg - prev= - continue - ;; - rpath | xrpath) - # We need an absolute path. - case $arg in - [\\/]* | [A-Za-z]:[\\/]*) ;; - *) - func_fatal_error "only absolute run-paths are allowed" - ;; - esac - if test rpath = "$prev"; then - case "$rpath " in - *" $arg "*) ;; - *) func_append rpath " $arg" ;; - esac - else - case "$xrpath " in - *" $arg "*) ;; - *) func_append xrpath " $arg" ;; - esac - fi - prev= - continue - ;; - shrext) - shrext_cmds=$arg - prev= - continue - ;; - weak) - func_append weak_libs " $arg" - prev= - continue - ;; - xcclinker) - func_append linker_flags " $qarg" - func_append compiler_flags " $qarg" - prev= - func_append compile_command " $qarg" - func_append finalize_command " $qarg" - continue - ;; - xcompiler) - func_append compiler_flags " $qarg" - prev= - func_append compile_command " $qarg" - func_append finalize_command " $qarg" - continue - ;; - xlinker) - func_append linker_flags " $qarg" - func_append compiler_flags " $wl$qarg" - prev= - func_append compile_command " $wl$qarg" - func_append finalize_command " $wl$qarg" - continue - ;; - *) - eval "$prev=\"\$arg\"" - prev= - continue - ;; - esac - fi # test -n "$prev" - - prevarg=$arg - - case $arg in - -all-static) - if test -n "$link_static_flag"; then - # See comment for -static flag below, for more details. - func_append compile_command " $link_static_flag" - func_append finalize_command " $link_static_flag" - fi - continue - ;; - - -allow-undefined) - # FIXME: remove this flag sometime in the future. - func_fatal_error "'-allow-undefined' must not be used because it is the default" - ;; - - -avoid-version) - avoid_version=yes - continue - ;; - - -bindir) - prev=bindir - continue - ;; - - -dlopen) - prev=dlfiles - continue - ;; - - -dlpreopen) - prev=dlprefiles - continue - ;; - - -export-dynamic) - export_dynamic=yes - continue - ;; - - -export-symbols | -export-symbols-regex) - if test -n "$export_symbols" || test -n "$export_symbols_regex"; then - func_fatal_error "more than one -exported-symbols argument is not allowed" - fi - if test X-export-symbols = "X$arg"; then - prev=expsyms - else - prev=expsyms_regex - fi - continue - ;; - - -framework) - prev=framework - continue - ;; - - -inst-prefix-dir) - prev=inst_prefix - continue - ;; - - # The native IRIX linker understands -LANG:*, -LIST:* and -LNO:* - # so, if we see these flags be careful not to treat them like -L - -L[A-Z][A-Z]*:*) - case $with_gcc/$host in - no/*-*-irix* | /*-*-irix*) - func_append compile_command " $arg" - func_append finalize_command " $arg" - ;; - esac - continue - ;; - - -L*) - func_stripname "-L" '' "$arg" - if test -z "$func_stripname_result"; then - if test "$#" -gt 0; then - func_fatal_error "require no space between '-L' and '$1'" - else - func_fatal_error "need path for '-L' option" - fi - fi - func_resolve_sysroot "$func_stripname_result" - dir=$func_resolve_sysroot_result - # We need an absolute path. - case $dir in - [\\/]* | [A-Za-z]:[\\/]*) ;; - *) - absdir=`cd "$dir" && pwd` - test -z "$absdir" && \ - func_fatal_error "cannot determine absolute directory name of '$dir'" - dir=$absdir - ;; - esac - case "$deplibs " in - *" -L$dir "* | *" $arg "*) - # Will only happen for absolute or sysroot arguments - ;; - *) - # Preserve sysroot, but never include relative directories - case $dir in - [\\/]* | [A-Za-z]:[\\/]* | =*) func_append deplibs " $arg" ;; - *) func_append deplibs " -L$dir" ;; - esac - func_append lib_search_path " $dir" - ;; - esac - case $host in - *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*) - testbindir=`$ECHO "$dir" | $SED 's*/lib$*/bin*'` - case :$dllsearchpath: in - *":$dir:"*) ;; - ::) dllsearchpath=$dir;; - *) func_append dllsearchpath ":$dir";; - esac - case :$dllsearchpath: in - *":$testbindir:"*) ;; - ::) dllsearchpath=$testbindir;; - *) func_append dllsearchpath ":$testbindir";; - esac - ;; - esac - continue - ;; - - -l*) - if test X-lc = "X$arg" || test X-lm = "X$arg"; then - case $host in - *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-beos* | *-cegcc* | *-*-haiku*) - # These systems don't actually have a C or math library (as such) - continue - ;; - *-*-os2*) - # These systems don't actually have a C library (as such) - test X-lc = "X$arg" && continue - ;; - *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-bitrig*) - # Do not include libc due to us having libc/libc_r. - test X-lc = "X$arg" && continue - ;; - *-*-rhapsody* | *-*-darwin1.[012]) - # Rhapsody C and math libraries are in the System framework - func_append deplibs " System.ltframework" - continue - ;; - *-*-sco3.2v5* | *-*-sco5v6*) - # Causes problems with __ctype - test X-lc = "X$arg" && continue - ;; - *-*-sysv4.2uw2* | *-*-sysv5* | *-*-unixware* | *-*-OpenUNIX*) - # Compiler inserts libc in the correct place for threads to work - test X-lc = "X$arg" && continue - ;; - esac - elif test X-lc_r = "X$arg"; then - case $host in - *-*-openbsd* | *-*-freebsd* | *-*-dragonfly* | *-*-bitrig*) - # Do not include libc_r directly, use -pthread flag. - continue - ;; - esac - fi - func_append deplibs " $arg" - continue - ;; - - -mllvm) - prev=mllvm - continue - ;; - - -module) - module=yes - continue - ;; - - # Tru64 UNIX uses -model [arg] to determine the layout of C++ - # classes, name mangling, and exception handling. - # Darwin uses the -arch flag to determine output architecture. - -model|-arch|-isysroot|--sysroot) - func_append compiler_flags " $arg" - func_append compile_command " $arg" - func_append finalize_command " $arg" - prev=xcompiler - continue - ;; - - -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe \ - |-threads|-fopenmp|-openmp|-mp|-xopenmp|-omp|-qsmp=*) - func_append compiler_flags " $arg" - func_append compile_command " $arg" - func_append finalize_command " $arg" - case "$new_inherited_linker_flags " in - *" $arg "*) ;; - * ) func_append new_inherited_linker_flags " $arg" ;; - esac - continue - ;; - - -multi_module) - single_module=$wl-multi_module - continue - ;; - - -no-fast-install) - fast_install=no - continue - ;; - - -no-install) - case $host in - *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-*-darwin* | *-cegcc*) - # The PATH hackery in wrapper scripts is required on Windows - # and Darwin in order for the loader to find any dlls it needs. - func_warning "'-no-install' is ignored for $host" - func_warning "assuming '-no-fast-install' instead" - fast_install=no - ;; - *) no_install=yes ;; - esac - continue - ;; - - -no-undefined) - allow_undefined=no - continue - ;; - - -objectlist) - prev=objectlist - continue - ;; - - -os2dllname) - prev=os2dllname - continue - ;; - - -o) prev=output ;; - - -precious-files-regex) - prev=precious_regex - continue - ;; - - -release) - prev=release - continue - ;; - - -rpath) - prev=rpath - continue - ;; - - -R) - prev=xrpath - continue - ;; - - -R*) - func_stripname '-R' '' "$arg" - dir=$func_stripname_result - # We need an absolute path. - case $dir in - [\\/]* | [A-Za-z]:[\\/]*) ;; - =*) - func_stripname '=' '' "$dir" - dir=$lt_sysroot$func_stripname_result - ;; - *) - func_fatal_error "only absolute run-paths are allowed" - ;; - esac - case "$xrpath " in - *" $dir "*) ;; - *) func_append xrpath " $dir" ;; - esac - continue - ;; - - -shared) - # The effects of -shared are defined in a previous loop. - continue - ;; - - -shrext) - prev=shrext - continue - ;; - - -static | -static-libtool-libs) - # The effects of -static are defined in a previous loop. - # We used to do the same as -all-static on platforms that - # didn't have a PIC flag, but the assumption that the effects - # would be equivalent was wrong. It would break on at least - # Digital Unix and AIX. - continue - ;; - - -thread-safe) - thread_safe=yes - continue - ;; - - -version-info) - prev=vinfo - continue - ;; - - -version-number) - prev=vinfo - vinfo_number=yes - continue - ;; - - -weak) - prev=weak - continue - ;; - - -Wc,*) - func_stripname '-Wc,' '' "$arg" - args=$func_stripname_result - arg= - save_ifs=$IFS; IFS=, - for flag in $args; do - IFS=$save_ifs - func_quote_for_eval "$flag" - func_append arg " $func_quote_for_eval_result" - func_append compiler_flags " $func_quote_for_eval_result" - done - IFS=$save_ifs - func_stripname ' ' '' "$arg" - arg=$func_stripname_result - ;; - - -Wl,*) - func_stripname '-Wl,' '' "$arg" - args=$func_stripname_result - arg= - save_ifs=$IFS; IFS=, - for flag in $args; do - IFS=$save_ifs - func_quote_for_eval "$flag" - func_append arg " $wl$func_quote_for_eval_result" - func_append compiler_flags " $wl$func_quote_for_eval_result" - func_append linker_flags " $func_quote_for_eval_result" - done - IFS=$save_ifs - func_stripname ' ' '' "$arg" - arg=$func_stripname_result - ;; - - -Xcompiler) - prev=xcompiler - continue - ;; - - -Xlinker) - prev=xlinker - continue - ;; - - -XCClinker) - prev=xcclinker - continue - ;; - - # -msg_* for osf cc - -msg_*) - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result - ;; - - # Flags to be passed through unchanged, with rationale: - # -64, -mips[0-9] enable 64-bit mode for the SGI compiler - # -r[0-9][0-9]* specify processor for the SGI compiler - # -xarch=*, -xtarget=* enable 64-bit mode for the Sun compiler - # +DA*, +DD* enable 64-bit mode for the HP compiler - # -q* compiler args for the IBM compiler - # -m*, -t[45]*, -txscale* architecture-specific flags for GCC - # -F/path path to uninstalled frameworks, gcc on darwin - # -p, -pg, --coverage, -fprofile-* profiling flags for GCC - # -fstack-protector* stack protector flags for GCC - # @file GCC response files - # -tp=* Portland pgcc target processor selection - # --sysroot=* for sysroot support - # -O*, -g*, -flto*, -fwhopr*, -fuse-linker-plugin GCC link-time optimization - # -stdlib=* select c++ std lib with clang - -64|-mips[0-9]|-r[0-9][0-9]*|-xarch=*|-xtarget=*|+DA*|+DD*|-q*|-m*| \ - -t[45]*|-txscale*|-p|-pg|--coverage|-fprofile-*|-F*|@*|-tp=*|--sysroot=*| \ - -O*|-g*|-flto*|-fwhopr*|-fuse-linker-plugin|-fstack-protector*|-stdlib=*) - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result - func_append compile_command " $arg" - func_append finalize_command " $arg" - func_append compiler_flags " $arg" - continue - ;; - - -Z*) - if test os2 = "`expr $host : '.*\(os2\)'`"; then - # OS/2 uses -Zxxx to specify OS/2-specific options - compiler_flags="$compiler_flags $arg" - func_append compile_command " $arg" - func_append finalize_command " $arg" - case $arg in - -Zlinker | -Zstack) - prev=xcompiler - ;; - esac - continue - else - # Otherwise treat like 'Some other compiler flag' below - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result - fi - ;; - - # Some other compiler flag. - -* | +*) - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result - ;; - - *.$objext) - # A standard object. - func_append objs " $arg" - ;; - - *.lo) - # A libtool-controlled object. - - # Check to see that this really is a libtool object. - if func_lalib_unsafe_p "$arg"; then - pic_object= - non_pic_object= - - # Read the .lo file - func_source "$arg" - - if test -z "$pic_object" || - test -z "$non_pic_object" || - test none = "$pic_object" && - test none = "$non_pic_object"; then - func_fatal_error "cannot find name of object for '$arg'" - fi - - # Extract subdirectory from the argument. - func_dirname "$arg" "/" "" - xdir=$func_dirname_result - - test none = "$pic_object" || { - # Prepend the subdirectory the object is found in. - pic_object=$xdir$pic_object - - if test dlfiles = "$prev"; then - if test yes = "$build_libtool_libs" && test yes = "$dlopen_support"; then - func_append dlfiles " $pic_object" - prev= - continue - else - # If libtool objects are unsupported, then we need to preload. - prev=dlprefiles - fi - fi - - # CHECK ME: I think I busted this. -Ossama - if test dlprefiles = "$prev"; then - # Preload the old-style object. - func_append dlprefiles " $pic_object" - prev= - fi - - # A PIC object. - func_append libobjs " $pic_object" - arg=$pic_object - } - - # Non-PIC object. - if test none != "$non_pic_object"; then - # Prepend the subdirectory the object is found in. - non_pic_object=$xdir$non_pic_object - - # A standard non-PIC object - func_append non_pic_objects " $non_pic_object" - if test -z "$pic_object" || test none = "$pic_object"; then - arg=$non_pic_object - fi - else - # If the PIC object exists, use it instead. - # $xdir was prepended to $pic_object above. - non_pic_object=$pic_object - func_append non_pic_objects " $non_pic_object" - fi - else - # Only an error if not doing a dry-run. - if $opt_dry_run; then - # Extract subdirectory from the argument. - func_dirname "$arg" "/" "" - xdir=$func_dirname_result - - func_lo2o "$arg" - pic_object=$xdir$objdir/$func_lo2o_result - non_pic_object=$xdir$func_lo2o_result - func_append libobjs " $pic_object" - func_append non_pic_objects " $non_pic_object" - else - func_fatal_error "'$arg' is not a valid libtool object" - fi - fi - ;; - - *.$libext) - # An archive. - func_append deplibs " $arg" - func_append old_deplibs " $arg" - continue - ;; - - *.la) - # A libtool-controlled library. - - func_resolve_sysroot "$arg" - if test dlfiles = "$prev"; then - # This library was specified with -dlopen. - func_append dlfiles " $func_resolve_sysroot_result" - prev= - elif test dlprefiles = "$prev"; then - # The library was specified with -dlpreopen. - func_append dlprefiles " $func_resolve_sysroot_result" - prev= - else - func_append deplibs " $func_resolve_sysroot_result" - fi - continue - ;; - - # Some other compiler argument. - *) - # Unknown arguments in both finalize_command and compile_command need - # to be aesthetically quoted because they are evaled later. - func_quote_for_eval "$arg" - arg=$func_quote_for_eval_result - ;; - esac # arg - - # Now actually substitute the argument into the commands. - if test -n "$arg"; then - func_append compile_command " $arg" - func_append finalize_command " $arg" - fi - done # argument parsing loop - - test -n "$prev" && \ - func_fatal_help "the '$prevarg' option requires an argument" - - if test yes = "$export_dynamic" && test -n "$export_dynamic_flag_spec"; then - eval arg=\"$export_dynamic_flag_spec\" - func_append compile_command " $arg" - func_append finalize_command " $arg" - fi - - oldlibs= - # calculate the name of the file, without its directory - func_basename "$output" - outputname=$func_basename_result - libobjs_save=$libobjs - - if test -n "$shlibpath_var"; then - # get the directories listed in $shlibpath_var - eval shlib_search_path=\`\$ECHO \"\$$shlibpath_var\" \| \$SED \'s/:/ /g\'\` - else - shlib_search_path= - fi - eval sys_lib_search_path=\"$sys_lib_search_path_spec\" - eval sys_lib_dlsearch_path=\"$sys_lib_dlsearch_path_spec\" - - # Definition is injected by LT_CONFIG during libtool generation. - func_munge_path_list sys_lib_dlsearch_path "$LT_SYS_LIBRARY_PATH" - - func_dirname "$output" "/" "" - output_objdir=$func_dirname_result$objdir - func_to_tool_file "$output_objdir/" - tool_output_objdir=$func_to_tool_file_result - # Create the object directory. - func_mkdir_p "$output_objdir" - - # Determine the type of output - case $output in - "") - func_fatal_help "you must specify an output file" - ;; - *.$libext) linkmode=oldlib ;; - *.lo | *.$objext) linkmode=obj ;; - *.la) linkmode=lib ;; - *) linkmode=prog ;; # Anything else should be a program. - esac - - specialdeplibs= - - libs= - # Find all interdependent deplibs by searching for libraries - # that are linked more than once (e.g. -la -lb -la) - for deplib in $deplibs; do - if $opt_preserve_dup_deps; then - case "$libs " in - *" $deplib "*) func_append specialdeplibs " $deplib" ;; - esac - fi - func_append libs " $deplib" - done - - if test lib = "$linkmode"; then - libs="$predeps $libs $compiler_lib_search_path $postdeps" - - # Compute libraries that are listed more than once in $predeps - # $postdeps and mark them as special (i.e., whose duplicates are - # not to be eliminated). - pre_post_deps= - if $opt_duplicate_compiler_generated_deps; then - for pre_post_dep in $predeps $postdeps; do - case "$pre_post_deps " in - *" $pre_post_dep "*) func_append specialdeplibs " $pre_post_deps" ;; - esac - func_append pre_post_deps " $pre_post_dep" - done - fi - pre_post_deps= - fi - - deplibs= - newdependency_libs= - newlib_search_path= - need_relink=no # whether we're linking any uninstalled libtool libraries - notinst_deplibs= # not-installed libtool libraries - notinst_path= # paths that contain not-installed libtool libraries - - case $linkmode in - lib) - passes="conv dlpreopen link" - for file in $dlfiles $dlprefiles; do - case $file in - *.la) ;; - *) - func_fatal_help "libraries can '-dlopen' only libtool libraries: $file" - ;; - esac - done - ;; - prog) - compile_deplibs= - finalize_deplibs= - alldeplibs=false - newdlfiles= - newdlprefiles= - passes="conv scan dlopen dlpreopen link" - ;; - *) passes="conv" - ;; - esac - - for pass in $passes; do - # The preopen pass in lib mode reverses $deplibs; put it back here - # so that -L comes before libs that need it for instance... - if test lib,link = "$linkmode,$pass"; then - ## FIXME: Find the place where the list is rebuilt in the wrong - ## order, and fix it there properly - tmp_deplibs= - for deplib in $deplibs; do - tmp_deplibs="$deplib $tmp_deplibs" - done - deplibs=$tmp_deplibs - fi - - if test lib,link = "$linkmode,$pass" || - test prog,scan = "$linkmode,$pass"; then - libs=$deplibs - deplibs= - fi - if test prog = "$linkmode"; then - case $pass in - dlopen) libs=$dlfiles ;; - dlpreopen) libs=$dlprefiles ;; - link) libs="$deplibs %DEPLIBS% $dependency_libs" ;; - esac - fi - if test lib,dlpreopen = "$linkmode,$pass"; then - # Collect and forward deplibs of preopened libtool libs - for lib in $dlprefiles; do - # Ignore non-libtool-libs - dependency_libs= - func_resolve_sysroot "$lib" - case $lib in - *.la) func_source "$func_resolve_sysroot_result" ;; - esac - - # Collect preopened libtool deplibs, except any this library - # has declared as weak libs - for deplib in $dependency_libs; do - func_basename "$deplib" - deplib_base=$func_basename_result - case " $weak_libs " in - *" $deplib_base "*) ;; - *) func_append deplibs " $deplib" ;; - esac - done - done - libs=$dlprefiles - fi - if test dlopen = "$pass"; then - # Collect dlpreopened libraries - save_deplibs=$deplibs - deplibs= - fi - - for deplib in $libs; do - lib= - found=false - case $deplib in - -mt|-mthreads|-kthread|-Kthread|-pthread|-pthreads|--thread-safe \ - |-threads|-fopenmp|-openmp|-mp|-xopenmp|-omp|-qsmp=*) - if test prog,link = "$linkmode,$pass"; then - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - else - func_append compiler_flags " $deplib" - if test lib = "$linkmode"; then - case "$new_inherited_linker_flags " in - *" $deplib "*) ;; - * ) func_append new_inherited_linker_flags " $deplib" ;; - esac - fi - fi - continue - ;; - -l*) - if test lib != "$linkmode" && test prog != "$linkmode"; then - func_warning "'-l' is ignored for archives/objects" - continue - fi - func_stripname '-l' '' "$deplib" - name=$func_stripname_result - if test lib = "$linkmode"; then - searchdirs="$newlib_search_path $lib_search_path $compiler_lib_search_dirs $sys_lib_search_path $shlib_search_path" - else - searchdirs="$newlib_search_path $lib_search_path $sys_lib_search_path $shlib_search_path" - fi - for searchdir in $searchdirs; do - for search_ext in .la $std_shrext .so .a; do - # Search the libtool library - lib=$searchdir/lib$name$search_ext - if test -f "$lib"; then - if test .la = "$search_ext"; then - found=: - else - found=false - fi - break 2 - fi - done - done - if $found; then - # deplib is a libtool library - # If $allow_libtool_libs_with_static_runtimes && $deplib is a stdlib, - # We need to do some special things here, and not later. - if test yes = "$allow_libtool_libs_with_static_runtimes"; then - case " $predeps $postdeps " in - *" $deplib "*) - if func_lalib_p "$lib"; then - library_names= - old_library= - func_source "$lib" - for l in $old_library $library_names; do - ll=$l - done - if test "X$ll" = "X$old_library"; then # only static version available - found=false - func_dirname "$lib" "" "." - ladir=$func_dirname_result - lib=$ladir/$old_library - if test prog,link = "$linkmode,$pass"; then - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - else - deplibs="$deplib $deplibs" - test lib = "$linkmode" && newdependency_libs="$deplib $newdependency_libs" - fi - continue - fi - fi - ;; - *) ;; - esac - fi - else - # deplib doesn't seem to be a libtool library - if test prog,link = "$linkmode,$pass"; then - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - else - deplibs="$deplib $deplibs" - test lib = "$linkmode" && newdependency_libs="$deplib $newdependency_libs" - fi - continue - fi - ;; # -l - *.ltframework) - if test prog,link = "$linkmode,$pass"; then - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - else - deplibs="$deplib $deplibs" - if test lib = "$linkmode"; then - case "$new_inherited_linker_flags " in - *" $deplib "*) ;; - * ) func_append new_inherited_linker_flags " $deplib" ;; - esac - fi - fi - continue - ;; - -L*) - case $linkmode in - lib) - deplibs="$deplib $deplibs" - test conv = "$pass" && continue - newdependency_libs="$deplib $newdependency_libs" - func_stripname '-L' '' "$deplib" - func_resolve_sysroot "$func_stripname_result" - func_append newlib_search_path " $func_resolve_sysroot_result" - ;; - prog) - if test conv = "$pass"; then - deplibs="$deplib $deplibs" - continue - fi - if test scan = "$pass"; then - deplibs="$deplib $deplibs" - else - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - fi - func_stripname '-L' '' "$deplib" - func_resolve_sysroot "$func_stripname_result" - func_append newlib_search_path " $func_resolve_sysroot_result" - ;; - *) - func_warning "'-L' is ignored for archives/objects" - ;; - esac # linkmode - continue - ;; # -L - -R*) - if test link = "$pass"; then - func_stripname '-R' '' "$deplib" - func_resolve_sysroot "$func_stripname_result" - dir=$func_resolve_sysroot_result - # Make sure the xrpath contains only unique directories. - case "$xrpath " in - *" $dir "*) ;; - *) func_append xrpath " $dir" ;; - esac - fi - deplibs="$deplib $deplibs" - continue - ;; - *.la) - func_resolve_sysroot "$deplib" - lib=$func_resolve_sysroot_result - ;; - *.$libext) - if test conv = "$pass"; then - deplibs="$deplib $deplibs" - continue - fi - case $linkmode in - lib) - # Linking convenience modules into shared libraries is allowed, - # but linking other static libraries is non-portable. - case " $dlpreconveniencelibs " in - *" $deplib "*) ;; - *) - valid_a_lib=false - case $deplibs_check_method in - match_pattern*) - set dummy $deplibs_check_method; shift - match_pattern_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"` - if eval "\$ECHO \"$deplib\"" 2>/dev/null | $SED 10q \ - | $EGREP "$match_pattern_regex" > /dev/null; then - valid_a_lib=: - fi - ;; - pass_all) - valid_a_lib=: - ;; - esac - if $valid_a_lib; then - echo - $ECHO "*** Warning: Linking the shared library $output against the" - $ECHO "*** static library $deplib is not portable!" - deplibs="$deplib $deplibs" - else - echo - $ECHO "*** Warning: Trying to link with static lib archive $deplib." - echo "*** I have the capability to make that library automatically link in when" - echo "*** you link to this library. But I can only do this if you have a" - echo "*** shared version of the library, which you do not appear to have" - echo "*** because the file extensions .$libext of this argument makes me believe" - echo "*** that it is just a static archive that I should not use here." - fi - ;; - esac - continue - ;; - prog) - if test link != "$pass"; then - deplibs="$deplib $deplibs" - else - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - fi - continue - ;; - esac # linkmode - ;; # *.$libext - *.lo | *.$objext) - if test conv = "$pass"; then - deplibs="$deplib $deplibs" - elif test prog = "$linkmode"; then - if test dlpreopen = "$pass" || test yes != "$dlopen_support" || test no = "$build_libtool_libs"; then - # If there is no dlopen support or we're linking statically, - # we need to preload. - func_append newdlprefiles " $deplib" - compile_deplibs="$deplib $compile_deplibs" - finalize_deplibs="$deplib $finalize_deplibs" - else - func_append newdlfiles " $deplib" - fi - fi - continue - ;; - %DEPLIBS%) - alldeplibs=: - continue - ;; - esac # case $deplib - - $found || test -f "$lib" \ - || func_fatal_error "cannot find the library '$lib' or unhandled argument '$deplib'" - - # Check to see that this really is a libtool archive. - func_lalib_unsafe_p "$lib" \ - || func_fatal_error "'$lib' is not a valid libtool archive" - - func_dirname "$lib" "" "." - ladir=$func_dirname_result - - dlname= - dlopen= - dlpreopen= - libdir= - library_names= - old_library= - inherited_linker_flags= - # If the library was installed with an old release of libtool, - # it will not redefine variables installed, or shouldnotlink - installed=yes - shouldnotlink=no - avoidtemprpath= - - - # Read the .la file - func_source "$lib" - - # Convert "-framework foo" to "foo.ltframework" - if test -n "$inherited_linker_flags"; then - tmp_inherited_linker_flags=`$ECHO "$inherited_linker_flags" | $SED 's/-framework \([^ $]*\)/\1.ltframework/g'` - for tmp_inherited_linker_flag in $tmp_inherited_linker_flags; do - case " $new_inherited_linker_flags " in - *" $tmp_inherited_linker_flag "*) ;; - *) func_append new_inherited_linker_flags " $tmp_inherited_linker_flag";; - esac - done - fi - dependency_libs=`$ECHO " $dependency_libs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - if test lib,link = "$linkmode,$pass" || - test prog,scan = "$linkmode,$pass" || - { test prog != "$linkmode" && test lib != "$linkmode"; }; then - test -n "$dlopen" && func_append dlfiles " $dlopen" - test -n "$dlpreopen" && func_append dlprefiles " $dlpreopen" - fi - - if test conv = "$pass"; then - # Only check for convenience libraries - deplibs="$lib $deplibs" - if test -z "$libdir"; then - if test -z "$old_library"; then - func_fatal_error "cannot find name of link library for '$lib'" - fi - # It is a libtool convenience library, so add in its objects. - func_append convenience " $ladir/$objdir/$old_library" - func_append old_convenience " $ladir/$objdir/$old_library" - elif test prog != "$linkmode" && test lib != "$linkmode"; then - func_fatal_error "'$lib' is not a convenience library" - fi - tmp_libs= - for deplib in $dependency_libs; do - deplibs="$deplib $deplibs" - if $opt_preserve_dup_deps; then - case "$tmp_libs " in - *" $deplib "*) func_append specialdeplibs " $deplib" ;; - esac - fi - func_append tmp_libs " $deplib" - done - continue - fi # $pass = conv - - - # Get the name of the library we link against. - linklib= - if test -n "$old_library" && - { test yes = "$prefer_static_libs" || - test built,no = "$prefer_static_libs,$installed"; }; then - linklib=$old_library - else - for l in $old_library $library_names; do - linklib=$l - done - fi - if test -z "$linklib"; then - func_fatal_error "cannot find name of link library for '$lib'" - fi - - # This library was specified with -dlopen. - if test dlopen = "$pass"; then - test -z "$libdir" \ - && func_fatal_error "cannot -dlopen a convenience library: '$lib'" - if test -z "$dlname" || - test yes != "$dlopen_support" || - test no = "$build_libtool_libs" - then - # If there is no dlname, no dlopen support or we're linking - # statically, we need to preload. We also need to preload any - # dependent libraries so libltdl's deplib preloader doesn't - # bomb out in the load deplibs phase. - func_append dlprefiles " $lib $dependency_libs" - else - func_append newdlfiles " $lib" - fi - continue - fi # $pass = dlopen - - # We need an absolute path. - case $ladir in - [\\/]* | [A-Za-z]:[\\/]*) abs_ladir=$ladir ;; - *) - abs_ladir=`cd "$ladir" && pwd` - if test -z "$abs_ladir"; then - func_warning "cannot determine absolute directory name of '$ladir'" - func_warning "passing it literally to the linker, although it might fail" - abs_ladir=$ladir - fi - ;; - esac - func_basename "$lib" - laname=$func_basename_result - - # Find the relevant object directory and library name. - if test yes = "$installed"; then - if test ! -f "$lt_sysroot$libdir/$linklib" && test -f "$abs_ladir/$linklib"; then - func_warning "library '$lib' was moved." - dir=$ladir - absdir=$abs_ladir - libdir=$abs_ladir - else - dir=$lt_sysroot$libdir - absdir=$lt_sysroot$libdir - fi - test yes = "$hardcode_automatic" && avoidtemprpath=yes - else - if test ! -f "$ladir/$objdir/$linklib" && test -f "$abs_ladir/$linklib"; then - dir=$ladir - absdir=$abs_ladir - # Remove this search path later - func_append notinst_path " $abs_ladir" - else - dir=$ladir/$objdir - absdir=$abs_ladir/$objdir - # Remove this search path later - func_append notinst_path " $abs_ladir" - fi - fi # $installed = yes - func_stripname 'lib' '.la' "$laname" - name=$func_stripname_result - - # This library was specified with -dlpreopen. - if test dlpreopen = "$pass"; then - if test -z "$libdir" && test prog = "$linkmode"; then - func_fatal_error "only libraries may -dlpreopen a convenience library: '$lib'" - fi - case $host in - # special handling for platforms with PE-DLLs. - *cygwin* | *mingw* | *cegcc* ) - # Linker will automatically link against shared library if both - # static and shared are present. Therefore, ensure we extract - # symbols from the import library if a shared library is present - # (otherwise, the dlopen module name will be incorrect). We do - # this by putting the import library name into $newdlprefiles. - # We recover the dlopen module name by 'saving' the la file - # name in a special purpose variable, and (later) extracting the - # dlname from the la file. - if test -n "$dlname"; then - func_tr_sh "$dir/$linklib" - eval "libfile_$func_tr_sh_result=\$abs_ladir/\$laname" - func_append newdlprefiles " $dir/$linklib" - else - func_append newdlprefiles " $dir/$old_library" - # Keep a list of preopened convenience libraries to check - # that they are being used correctly in the link pass. - test -z "$libdir" && \ - func_append dlpreconveniencelibs " $dir/$old_library" - fi - ;; - * ) - # Prefer using a static library (so that no silly _DYNAMIC symbols - # are required to link). - if test -n "$old_library"; then - func_append newdlprefiles " $dir/$old_library" - # Keep a list of preopened convenience libraries to check - # that they are being used correctly in the link pass. - test -z "$libdir" && \ - func_append dlpreconveniencelibs " $dir/$old_library" - # Otherwise, use the dlname, so that lt_dlopen finds it. - elif test -n "$dlname"; then - func_append newdlprefiles " $dir/$dlname" - else - func_append newdlprefiles " $dir/$linklib" - fi - ;; - esac - fi # $pass = dlpreopen - - if test -z "$libdir"; then - # Link the convenience library - if test lib = "$linkmode"; then - deplibs="$dir/$old_library $deplibs" - elif test prog,link = "$linkmode,$pass"; then - compile_deplibs="$dir/$old_library $compile_deplibs" - finalize_deplibs="$dir/$old_library $finalize_deplibs" - else - deplibs="$lib $deplibs" # used for prog,scan pass - fi - continue - fi - - - if test prog = "$linkmode" && test link != "$pass"; then - func_append newlib_search_path " $ladir" - deplibs="$lib $deplibs" - - linkalldeplibs=false - if test no != "$link_all_deplibs" || test -z "$library_names" || - test no = "$build_libtool_libs"; then - linkalldeplibs=: - fi - - tmp_libs= - for deplib in $dependency_libs; do - case $deplib in - -L*) func_stripname '-L' '' "$deplib" - func_resolve_sysroot "$func_stripname_result" - func_append newlib_search_path " $func_resolve_sysroot_result" - ;; - esac - # Need to link against all dependency_libs? - if $linkalldeplibs; then - deplibs="$deplib $deplibs" - else - # Need to hardcode shared library paths - # or/and link against static libraries - newdependency_libs="$deplib $newdependency_libs" - fi - if $opt_preserve_dup_deps; then - case "$tmp_libs " in - *" $deplib "*) func_append specialdeplibs " $deplib" ;; - esac - fi - func_append tmp_libs " $deplib" - done # for deplib - continue - fi # $linkmode = prog... - - if test prog,link = "$linkmode,$pass"; then - if test -n "$library_names" && - { { test no = "$prefer_static_libs" || - test built,yes = "$prefer_static_libs,$installed"; } || - test -z "$old_library"; }; then - # We need to hardcode the library path - if test -n "$shlibpath_var" && test -z "$avoidtemprpath"; then - # Make sure the rpath contains only unique directories. - case $temp_rpath: in - *"$absdir:"*) ;; - *) func_append temp_rpath "$absdir:" ;; - esac - fi - - # Hardcode the library path. - # Skip directories that are in the system default run-time - # search path. - case " $sys_lib_dlsearch_path " in - *" $absdir "*) ;; - *) - case "$compile_rpath " in - *" $absdir "*) ;; - *) func_append compile_rpath " $absdir" ;; - esac - ;; - esac - case " $sys_lib_dlsearch_path " in - *" $libdir "*) ;; - *) - case "$finalize_rpath " in - *" $libdir "*) ;; - *) func_append finalize_rpath " $libdir" ;; - esac - ;; - esac - fi # $linkmode,$pass = prog,link... - - if $alldeplibs && - { test pass_all = "$deplibs_check_method" || - { test yes = "$build_libtool_libs" && - test -n "$library_names"; }; }; then - # We only need to search for static libraries - continue - fi - fi - - link_static=no # Whether the deplib will be linked statically - use_static_libs=$prefer_static_libs - if test built = "$use_static_libs" && test yes = "$installed"; then - use_static_libs=no - fi - if test -n "$library_names" && - { test no = "$use_static_libs" || test -z "$old_library"; }; then - case $host in - *cygwin* | *mingw* | *cegcc* | *os2*) - # No point in relinking DLLs because paths are not encoded - func_append notinst_deplibs " $lib" - need_relink=no - ;; - *) - if test no = "$installed"; then - func_append notinst_deplibs " $lib" - need_relink=yes - fi - ;; - esac - # This is a shared library - - # Warn about portability, can't link against -module's on some - # systems (darwin). Don't bleat about dlopened modules though! - dlopenmodule= - for dlpremoduletest in $dlprefiles; do - if test "X$dlpremoduletest" = "X$lib"; then - dlopenmodule=$dlpremoduletest - break - fi - done - if test -z "$dlopenmodule" && test yes = "$shouldnotlink" && test link = "$pass"; then - echo - if test prog = "$linkmode"; then - $ECHO "*** Warning: Linking the executable $output against the loadable module" - else - $ECHO "*** Warning: Linking the shared library $output against the loadable module" - fi - $ECHO "*** $linklib is not portable!" - fi - if test lib = "$linkmode" && - test yes = "$hardcode_into_libs"; then - # Hardcode the library path. - # Skip directories that are in the system default run-time - # search path. - case " $sys_lib_dlsearch_path " in - *" $absdir "*) ;; - *) - case "$compile_rpath " in - *" $absdir "*) ;; - *) func_append compile_rpath " $absdir" ;; - esac - ;; - esac - case " $sys_lib_dlsearch_path " in - *" $libdir "*) ;; - *) - case "$finalize_rpath " in - *" $libdir "*) ;; - *) func_append finalize_rpath " $libdir" ;; - esac - ;; - esac - fi - - if test -n "$old_archive_from_expsyms_cmds"; then - # figure out the soname - set dummy $library_names - shift - realname=$1 - shift - libname=`eval "\\$ECHO \"$libname_spec\""` - # use dlname if we got it. it's perfectly good, no? - if test -n "$dlname"; then - soname=$dlname - elif test -n "$soname_spec"; then - # bleh windows - case $host in - *cygwin* | mingw* | *cegcc* | *os2*) - func_arith $current - $age - major=$func_arith_result - versuffix=-$major - ;; - esac - eval soname=\"$soname_spec\" - else - soname=$realname - fi - - # Make a new name for the extract_expsyms_cmds to use - soroot=$soname - func_basename "$soroot" - soname=$func_basename_result - func_stripname 'lib' '.dll' "$soname" - newlib=libimp-$func_stripname_result.a - - # If the library has no export list, then create one now - if test -f "$output_objdir/$soname-def"; then : - else - func_verbose "extracting exported symbol list from '$soname'" - func_execute_cmds "$extract_expsyms_cmds" 'exit $?' - fi - - # Create $newlib - if test -f "$output_objdir/$newlib"; then :; else - func_verbose "generating import library for '$soname'" - func_execute_cmds "$old_archive_from_expsyms_cmds" 'exit $?' - fi - # make sure the library variables are pointing to the new library - dir=$output_objdir - linklib=$newlib - fi # test -n "$old_archive_from_expsyms_cmds" - - if test prog = "$linkmode" || test relink != "$opt_mode"; then - add_shlibpath= - add_dir= - add= - lib_linked=yes - case $hardcode_action in - immediate | unsupported) - if test no = "$hardcode_direct"; then - add=$dir/$linklib - case $host in - *-*-sco3.2v5.0.[024]*) add_dir=-L$dir ;; - *-*-sysv4*uw2*) add_dir=-L$dir ;; - *-*-sysv5OpenUNIX* | *-*-sysv5UnixWare7.[01].[10]* | \ - *-*-unixware7*) add_dir=-L$dir ;; - *-*-darwin* ) - # if the lib is a (non-dlopened) module then we cannot - # link against it, someone is ignoring the earlier warnings - if /usr/bin/file -L $add 2> /dev/null | - $GREP ": [^:]* bundle" >/dev/null; then - if test "X$dlopenmodule" != "X$lib"; then - $ECHO "*** Warning: lib $linklib is a module, not a shared library" - if test -z "$old_library"; then - echo - echo "*** And there doesn't seem to be a static archive available" - echo "*** The link will probably fail, sorry" - else - add=$dir/$old_library - fi - elif test -n "$old_library"; then - add=$dir/$old_library - fi - fi - esac - elif test no = "$hardcode_minus_L"; then - case $host in - *-*-sunos*) add_shlibpath=$dir ;; - esac - add_dir=-L$dir - add=-l$name - elif test no = "$hardcode_shlibpath_var"; then - add_shlibpath=$dir - add=-l$name - else - lib_linked=no - fi - ;; - relink) - if test yes = "$hardcode_direct" && - test no = "$hardcode_direct_absolute"; then - add=$dir/$linklib - elif test yes = "$hardcode_minus_L"; then - add_dir=-L$absdir - # Try looking first in the location we're being installed to. - if test -n "$inst_prefix_dir"; then - case $libdir in - [\\/]*) - func_append add_dir " -L$inst_prefix_dir$libdir" - ;; - esac - fi - add=-l$name - elif test yes = "$hardcode_shlibpath_var"; then - add_shlibpath=$dir - add=-l$name - else - lib_linked=no - fi - ;; - *) lib_linked=no ;; - esac - - if test yes != "$lib_linked"; then - func_fatal_configuration "unsupported hardcode properties" - fi - - if test -n "$add_shlibpath"; then - case :$compile_shlibpath: in - *":$add_shlibpath:"*) ;; - *) func_append compile_shlibpath "$add_shlibpath:" ;; - esac - fi - if test prog = "$linkmode"; then - test -n "$add_dir" && compile_deplibs="$add_dir $compile_deplibs" - test -n "$add" && compile_deplibs="$add $compile_deplibs" - else - test -n "$add_dir" && deplibs="$add_dir $deplibs" - test -n "$add" && deplibs="$add $deplibs" - if test yes != "$hardcode_direct" && - test yes != "$hardcode_minus_L" && - test yes = "$hardcode_shlibpath_var"; then - case :$finalize_shlibpath: in - *":$libdir:"*) ;; - *) func_append finalize_shlibpath "$libdir:" ;; - esac - fi - fi - fi - - if test prog = "$linkmode" || test relink = "$opt_mode"; then - add_shlibpath= - add_dir= - add= - # Finalize command for both is simple: just hardcode it. - if test yes = "$hardcode_direct" && - test no = "$hardcode_direct_absolute"; then - add=$libdir/$linklib - elif test yes = "$hardcode_minus_L"; then - add_dir=-L$libdir - add=-l$name - elif test yes = "$hardcode_shlibpath_var"; then - case :$finalize_shlibpath: in - *":$libdir:"*) ;; - *) func_append finalize_shlibpath "$libdir:" ;; - esac - add=-l$name - elif test yes = "$hardcode_automatic"; then - if test -n "$inst_prefix_dir" && - test -f "$inst_prefix_dir$libdir/$linklib"; then - add=$inst_prefix_dir$libdir/$linklib - else - add=$libdir/$linklib - fi - else - # We cannot seem to hardcode it, guess we'll fake it. - add_dir=-L$libdir - # Try looking first in the location we're being installed to. - if test -n "$inst_prefix_dir"; then - case $libdir in - [\\/]*) - func_append add_dir " -L$inst_prefix_dir$libdir" - ;; - esac - fi - add=-l$name - fi - - if test prog = "$linkmode"; then - test -n "$add_dir" && finalize_deplibs="$add_dir $finalize_deplibs" - test -n "$add" && finalize_deplibs="$add $finalize_deplibs" - else - test -n "$add_dir" && deplibs="$add_dir $deplibs" - test -n "$add" && deplibs="$add $deplibs" - fi - fi - elif test prog = "$linkmode"; then - # Here we assume that one of hardcode_direct or hardcode_minus_L - # is not unsupported. This is valid on all known static and - # shared platforms. - if test unsupported != "$hardcode_direct"; then - test -n "$old_library" && linklib=$old_library - compile_deplibs="$dir/$linklib $compile_deplibs" - finalize_deplibs="$dir/$linklib $finalize_deplibs" - else - compile_deplibs="-l$name -L$dir $compile_deplibs" - finalize_deplibs="-l$name -L$dir $finalize_deplibs" - fi - elif test yes = "$build_libtool_libs"; then - # Not a shared library - if test pass_all != "$deplibs_check_method"; then - # We're trying link a shared library against a static one - # but the system doesn't support it. - - # Just print a warning and add the library to dependency_libs so - # that the program can be linked against the static library. - echo - $ECHO "*** Warning: This system cannot link to static lib archive $lib." - echo "*** I have the capability to make that library automatically link in when" - echo "*** you link to this library. But I can only do this if you have a" - echo "*** shared version of the library, which you do not appear to have." - if test yes = "$module"; then - echo "*** But as you try to build a module library, libtool will still create " - echo "*** a static module, that should work as long as the dlopening application" - echo "*** is linked with the -dlopen flag to resolve symbols at runtime." - if test -z "$global_symbol_pipe"; then - echo - echo "*** However, this would only work if libtool was able to extract symbol" - echo "*** lists from a program, using 'nm' or equivalent, but libtool could" - echo "*** not find such a program. So, this module is probably useless." - echo "*** 'nm' from GNU binutils and a full rebuild may help." - fi - if test no = "$build_old_libs"; then - build_libtool_libs=module - build_old_libs=yes - else - build_libtool_libs=no - fi - fi - else - deplibs="$dir/$old_library $deplibs" - link_static=yes - fi - fi # link shared/static library? - - if test lib = "$linkmode"; then - if test -n "$dependency_libs" && - { test yes != "$hardcode_into_libs" || - test yes = "$build_old_libs" || - test yes = "$link_static"; }; then - # Extract -R from dependency_libs - temp_deplibs= - for libdir in $dependency_libs; do - case $libdir in - -R*) func_stripname '-R' '' "$libdir" - temp_xrpath=$func_stripname_result - case " $xrpath " in - *" $temp_xrpath "*) ;; - *) func_append xrpath " $temp_xrpath";; - esac;; - *) func_append temp_deplibs " $libdir";; - esac - done - dependency_libs=$temp_deplibs - fi - - func_append newlib_search_path " $absdir" - # Link against this library - test no = "$link_static" && newdependency_libs="$abs_ladir/$laname $newdependency_libs" - # ... and its dependency_libs - tmp_libs= - for deplib in $dependency_libs; do - newdependency_libs="$deplib $newdependency_libs" - case $deplib in - -L*) func_stripname '-L' '' "$deplib" - func_resolve_sysroot "$func_stripname_result";; - *) func_resolve_sysroot "$deplib" ;; - esac - if $opt_preserve_dup_deps; then - case "$tmp_libs " in - *" $func_resolve_sysroot_result "*) - func_append specialdeplibs " $func_resolve_sysroot_result" ;; - esac - fi - func_append tmp_libs " $func_resolve_sysroot_result" - done - - if test no != "$link_all_deplibs"; then - # Add the search paths of all dependency libraries - for deplib in $dependency_libs; do - path= - case $deplib in - -L*) path=$deplib ;; - *.la) - func_resolve_sysroot "$deplib" - deplib=$func_resolve_sysroot_result - func_dirname "$deplib" "" "." - dir=$func_dirname_result - # We need an absolute path. - case $dir in - [\\/]* | [A-Za-z]:[\\/]*) absdir=$dir ;; - *) - absdir=`cd "$dir" && pwd` - if test -z "$absdir"; then - func_warning "cannot determine absolute directory name of '$dir'" - absdir=$dir - fi - ;; - esac - if $GREP "^installed=no" $deplib > /dev/null; then - case $host in - *-*-darwin*) - depdepl= - eval deplibrary_names=`$SED -n -e 's/^library_names=\(.*\)$/\1/p' $deplib` - if test -n "$deplibrary_names"; then - for tmp in $deplibrary_names; do - depdepl=$tmp - done - if test -f "$absdir/$objdir/$depdepl"; then - depdepl=$absdir/$objdir/$depdepl - darwin_install_name=`$OTOOL -L $depdepl | awk '{if (NR == 2) {print $1;exit}}'` - if test -z "$darwin_install_name"; then - darwin_install_name=`$OTOOL64 -L $depdepl | awk '{if (NR == 2) {print $1;exit}}'` - fi - func_append compiler_flags " $wl-dylib_file $wl$darwin_install_name:$depdepl" - func_append linker_flags " -dylib_file $darwin_install_name:$depdepl" - path= - fi - fi - ;; - *) - path=-L$absdir/$objdir - ;; - esac - else - eval libdir=`$SED -n -e 's/^libdir=\(.*\)$/\1/p' $deplib` - test -z "$libdir" && \ - func_fatal_error "'$deplib' is not a valid libtool archive" - test "$absdir" != "$libdir" && \ - func_warning "'$deplib' seems to be moved" - - path=-L$absdir - fi - ;; - esac - case " $deplibs " in - *" $path "*) ;; - *) deplibs="$path $deplibs" ;; - esac - done - fi # link_all_deplibs != no - fi # linkmode = lib - done # for deplib in $libs - if test link = "$pass"; then - if test prog = "$linkmode"; then - compile_deplibs="$new_inherited_linker_flags $compile_deplibs" - finalize_deplibs="$new_inherited_linker_flags $finalize_deplibs" - else - compiler_flags="$compiler_flags "`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - fi - fi - dependency_libs=$newdependency_libs - if test dlpreopen = "$pass"; then - # Link the dlpreopened libraries before other libraries - for deplib in $save_deplibs; do - deplibs="$deplib $deplibs" - done - fi - if test dlopen != "$pass"; then - test conv = "$pass" || { - # Make sure lib_search_path contains only unique directories. - lib_search_path= - for dir in $newlib_search_path; do - case "$lib_search_path " in - *" $dir "*) ;; - *) func_append lib_search_path " $dir" ;; - esac - done - newlib_search_path= - } - - if test prog,link = "$linkmode,$pass"; then - vars="compile_deplibs finalize_deplibs" - else - vars=deplibs - fi - for var in $vars dependency_libs; do - # Add libraries to $var in reverse order - eval tmp_libs=\"\$$var\" - new_libs= - for deplib in $tmp_libs; do - # FIXME: Pedantically, this is the right thing to do, so - # that some nasty dependency loop isn't accidentally - # broken: - #new_libs="$deplib $new_libs" - # Pragmatically, this seems to cause very few problems in - # practice: - case $deplib in - -L*) new_libs="$deplib $new_libs" ;; - -R*) ;; - *) - # And here is the reason: when a library appears more - # than once as an explicit dependence of a library, or - # is implicitly linked in more than once by the - # compiler, it is considered special, and multiple - # occurrences thereof are not removed. Compare this - # with having the same library being listed as a - # dependency of multiple other libraries: in this case, - # we know (pedantically, we assume) the library does not - # need to be listed more than once, so we keep only the - # last copy. This is not always right, but it is rare - # enough that we require users that really mean to play - # such unportable linking tricks to link the library - # using -Wl,-lname, so that libtool does not consider it - # for duplicate removal. - case " $specialdeplibs " in - *" $deplib "*) new_libs="$deplib $new_libs" ;; - *) - case " $new_libs " in - *" $deplib "*) ;; - *) new_libs="$deplib $new_libs" ;; - esac - ;; - esac - ;; - esac - done - tmp_libs= - for deplib in $new_libs; do - case $deplib in - -L*) - case " $tmp_libs " in - *" $deplib "*) ;; - *) func_append tmp_libs " $deplib" ;; - esac - ;; - *) func_append tmp_libs " $deplib" ;; - esac - done - eval $var=\"$tmp_libs\" - done # for var - fi - - # Add Sun CC postdeps if required: - test CXX = "$tagname" && { - case $host_os in - linux*) - case `$CC -V 2>&1 | sed 5q` in - *Sun\ C*) # Sun C++ 5.9 - func_suncc_cstd_abi - - if test no != "$suncc_use_cstd_abi"; then - func_append postdeps ' -library=Cstd -library=Crun' - fi - ;; - esac - ;; - - solaris*) - func_cc_basename "$CC" - case $func_cc_basename_result in - CC* | sunCC*) - func_suncc_cstd_abi - - if test no != "$suncc_use_cstd_abi"; then - func_append postdeps ' -library=Cstd -library=Crun' - fi - ;; - esac - ;; - esac - } - - # Last step: remove runtime libs from dependency_libs - # (they stay in deplibs) - tmp_libs= - for i in $dependency_libs; do - case " $predeps $postdeps $compiler_lib_search_path " in - *" $i "*) - i= - ;; - esac - if test -n "$i"; then - func_append tmp_libs " $i" - fi - done - dependency_libs=$tmp_libs - done # for pass - if test prog = "$linkmode"; then - dlfiles=$newdlfiles - fi - if test prog = "$linkmode" || test lib = "$linkmode"; then - dlprefiles=$newdlprefiles - fi - - case $linkmode in - oldlib) - if test -n "$dlfiles$dlprefiles" || test no != "$dlself"; then - func_warning "'-dlopen' is ignored for archives" - fi - - case " $deplibs" in - *\ -l* | *\ -L*) - func_warning "'-l' and '-L' are ignored for archives" ;; - esac - - test -n "$rpath" && \ - func_warning "'-rpath' is ignored for archives" - - test -n "$xrpath" && \ - func_warning "'-R' is ignored for archives" - - test -n "$vinfo" && \ - func_warning "'-version-info/-version-number' is ignored for archives" - - test -n "$release" && \ - func_warning "'-release' is ignored for archives" - - test -n "$export_symbols$export_symbols_regex" && \ - func_warning "'-export-symbols' is ignored for archives" - - # Now set the variables for building old libraries. - build_libtool_libs=no - oldlibs=$output - func_append objs "$old_deplibs" - ;; - - lib) - # Make sure we only generate libraries of the form 'libNAME.la'. - case $outputname in - lib*) - func_stripname 'lib' '.la' "$outputname" - name=$func_stripname_result - eval shared_ext=\"$shrext_cmds\" - eval libname=\"$libname_spec\" - ;; - *) - test no = "$module" \ - && func_fatal_help "libtool library '$output' must begin with 'lib'" - - if test no != "$need_lib_prefix"; then - # Add the "lib" prefix for modules if required - func_stripname '' '.la' "$outputname" - name=$func_stripname_result - eval shared_ext=\"$shrext_cmds\" - eval libname=\"$libname_spec\" - else - func_stripname '' '.la' "$outputname" - libname=$func_stripname_result - fi - ;; - esac - - if test -n "$objs"; then - if test pass_all != "$deplibs_check_method"; then - func_fatal_error "cannot build libtool library '$output' from non-libtool objects on this host:$objs" - else - echo - $ECHO "*** Warning: Linking the shared library $output against the non-libtool" - $ECHO "*** objects $objs is not portable!" - func_append libobjs " $objs" - fi - fi - - test no = "$dlself" \ - || func_warning "'-dlopen self' is ignored for libtool libraries" - - set dummy $rpath - shift - test 1 -lt "$#" \ - && func_warning "ignoring multiple '-rpath's for a libtool library" - - install_libdir=$1 - - oldlibs= - if test -z "$rpath"; then - if test yes = "$build_libtool_libs"; then - # Building a libtool convenience library. - # Some compilers have problems with a '.al' extension so - # convenience libraries should have the same extension an - # archive normally would. - oldlibs="$output_objdir/$libname.$libext $oldlibs" - build_libtool_libs=convenience - build_old_libs=yes - fi - - test -n "$vinfo" && \ - func_warning "'-version-info/-version-number' is ignored for convenience libraries" - - test -n "$release" && \ - func_warning "'-release' is ignored for convenience libraries" - else - - # Parse the version information argument. - save_ifs=$IFS; IFS=: - set dummy $vinfo 0 0 0 - shift - IFS=$save_ifs - - test -n "$7" && \ - func_fatal_help "too many parameters to '-version-info'" - - # convert absolute version numbers to libtool ages - # this retains compatibility with .la files and attempts - # to make the code below a bit more comprehensible - - case $vinfo_number in - yes) - number_major=$1 - number_minor=$2 - number_revision=$3 - # - # There are really only two kinds -- those that - # use the current revision as the major version - # and those that subtract age and use age as - # a minor version. But, then there is irix - # that has an extra 1 added just for fun - # - case $version_type in - # correct linux to gnu/linux during the next big refactor - darwin|freebsd-elf|linux|osf|windows|none) - func_arith $number_major + $number_minor - current=$func_arith_result - age=$number_minor - revision=$number_revision - ;; - freebsd-aout|qnx|sunos) - current=$number_major - revision=$number_minor - age=0 - ;; - irix|nonstopux) - func_arith $number_major + $number_minor - current=$func_arith_result - age=$number_minor - revision=$number_minor - lt_irix_increment=no - ;; - esac - ;; - no) - current=$1 - revision=$2 - age=$3 - ;; - esac - - # Check that each of the things are valid numbers. - case $current in - 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;; - *) - func_error "CURRENT '$current' must be a nonnegative integer" - func_fatal_error "'$vinfo' is not valid version information" - ;; - esac - - case $revision in - 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;; - *) - func_error "REVISION '$revision' must be a nonnegative integer" - func_fatal_error "'$vinfo' is not valid version information" - ;; - esac - - case $age in - 0|[1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-9][0-9][0-9][0-9]|[1-9][0-9][0-9][0-9][0-9]) ;; - *) - func_error "AGE '$age' must be a nonnegative integer" - func_fatal_error "'$vinfo' is not valid version information" - ;; - esac - - if test "$age" -gt "$current"; then - func_error "AGE '$age' is greater than the current interface number '$current'" - func_fatal_error "'$vinfo' is not valid version information" - fi - - # Calculate the version variables. - major= - versuffix= - verstring= - case $version_type in - none) ;; - - darwin) - # Like Linux, but with the current version available in - # verstring for coding it into the library header - func_arith $current - $age - major=.$func_arith_result - versuffix=$major.$age.$revision - # Darwin ld doesn't like 0 for these options... - func_arith $current + 1 - minor_current=$func_arith_result - xlcverstring="$wl-compatibility_version $wl$minor_current $wl-current_version $wl$minor_current.$revision" - verstring="-compatibility_version $minor_current -current_version $minor_current.$revision" - # On Darwin other compilers - case $CC in - nagfor*) - verstring="$wl-compatibility_version $wl$minor_current $wl-current_version $wl$minor_current.$revision" - ;; - *) - verstring="-compatibility_version $minor_current -current_version $minor_current.$revision" - ;; - esac - ;; - - freebsd-aout) - major=.$current - versuffix=.$current.$revision - ;; - - freebsd-elf) - func_arith $current - $age - major=.$func_arith_result - versuffix=$major.$age.$revision - ;; - - irix | nonstopux) - if test no = "$lt_irix_increment"; then - func_arith $current - $age - else - func_arith $current - $age + 1 - fi - major=$func_arith_result - - case $version_type in - nonstopux) verstring_prefix=nonstopux ;; - *) verstring_prefix=sgi ;; - esac - verstring=$verstring_prefix$major.$revision - - # Add in all the interfaces that we are compatible with. - loop=$revision - while test 0 -ne "$loop"; do - func_arith $revision - $loop - iface=$func_arith_result - func_arith $loop - 1 - loop=$func_arith_result - verstring=$verstring_prefix$major.$iface:$verstring - done - - # Before this point, $major must not contain '.'. - major=.$major - versuffix=$major.$revision - ;; - - linux) # correct to gnu/linux during the next big refactor - func_arith $current - $age - major=.$func_arith_result - versuffix=$major.$age.$revision - ;; - - osf) - func_arith $current - $age - major=.$func_arith_result - versuffix=.$current.$age.$revision - verstring=$current.$age.$revision - - # Add in all the interfaces that we are compatible with. - loop=$age - while test 0 -ne "$loop"; do - func_arith $current - $loop - iface=$func_arith_result - func_arith $loop - 1 - loop=$func_arith_result - verstring=$verstring:$iface.0 - done - - # Make executables depend on our current version. - func_append verstring ":$current.0" - ;; - - qnx) - major=.$current - versuffix=.$current - ;; - - sco) - major=.$current - versuffix=.$current - ;; - - sunos) - major=.$current - versuffix=.$current.$revision - ;; - - windows) - # Use '-' rather than '.', since we only want one - # extension on DOS 8.3 file systems. - func_arith $current - $age - major=$func_arith_result - versuffix=-$major - ;; - - *) - func_fatal_configuration "unknown library version type '$version_type'" - ;; - esac - - # Clear the version info if we defaulted, and they specified a release. - if test -z "$vinfo" && test -n "$release"; then - major= - case $version_type in - darwin) - # we can't check for "0.0" in archive_cmds due to quoting - # problems, so we reset it completely - verstring= - ;; - *) - verstring=0.0 - ;; - esac - if test no = "$need_version"; then - versuffix= - else - versuffix=.0.0 - fi - fi - - # Remove version info from name if versioning should be avoided - if test yes,no = "$avoid_version,$need_version"; then - major= - versuffix= - verstring= - fi - - # Check to see if the archive will have undefined symbols. - if test yes = "$allow_undefined"; then - if test unsupported = "$allow_undefined_flag"; then - if test yes = "$build_old_libs"; then - func_warning "undefined symbols not allowed in $host shared libraries; building static only" - build_libtool_libs=no - else - func_fatal_error "can't build $host shared library unless -no-undefined is specified" - fi - fi - else - # Don't allow undefined symbols. - allow_undefined_flag=$no_undefined_flag - fi - - fi - - func_generate_dlsyms "$libname" "$libname" : - func_append libobjs " $symfileobj" - test " " = "$libobjs" && libobjs= - - if test relink != "$opt_mode"; then - # Remove our outputs, but don't remove object files since they - # may have been created when compiling PIC objects. - removelist= - tempremovelist=`$ECHO "$output_objdir/*"` - for p in $tempremovelist; do - case $p in - *.$objext | *.gcno) - ;; - $output_objdir/$outputname | $output_objdir/$libname.* | $output_objdir/$libname$release.*) - if test -n "$precious_files_regex"; then - if $ECHO "$p" | $EGREP -e "$precious_files_regex" >/dev/null 2>&1 - then - continue - fi - fi - func_append removelist " $p" - ;; - *) ;; - esac - done - test -n "$removelist" && \ - func_show_eval "${RM}r \$removelist" - fi - - # Now set the variables for building old libraries. - if test yes = "$build_old_libs" && test convenience != "$build_libtool_libs"; then - func_append oldlibs " $output_objdir/$libname.$libext" - - # Transform .lo files to .o files. - oldobjs="$objs "`$ECHO "$libobjs" | $SP2NL | $SED "/\.$libext$/d; $lo2o" | $NL2SP` - fi - - # Eliminate all temporary directories. - #for path in $notinst_path; do - # lib_search_path=`$ECHO "$lib_search_path " | $SED "s% $path % %g"` - # deplibs=`$ECHO "$deplibs " | $SED "s% -L$path % %g"` - # dependency_libs=`$ECHO "$dependency_libs " | $SED "s% -L$path % %g"` - #done - - if test -n "$xrpath"; then - # If the user specified any rpath flags, then add them. - temp_xrpath= - for libdir in $xrpath; do - func_replace_sysroot "$libdir" - func_append temp_xrpath " -R$func_replace_sysroot_result" - case "$finalize_rpath " in - *" $libdir "*) ;; - *) func_append finalize_rpath " $libdir" ;; - esac - done - if test yes != "$hardcode_into_libs" || test yes = "$build_old_libs"; then - dependency_libs="$temp_xrpath $dependency_libs" - fi - fi - - # Make sure dlfiles contains only unique files that won't be dlpreopened - old_dlfiles=$dlfiles - dlfiles= - for lib in $old_dlfiles; do - case " $dlprefiles $dlfiles " in - *" $lib "*) ;; - *) func_append dlfiles " $lib" ;; - esac - done - - # Make sure dlprefiles contains only unique files - old_dlprefiles=$dlprefiles - dlprefiles= - for lib in $old_dlprefiles; do - case "$dlprefiles " in - *" $lib "*) ;; - *) func_append dlprefiles " $lib" ;; - esac - done - - if test yes = "$build_libtool_libs"; then - if test -n "$rpath"; then - case $host in - *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-*-beos* | *-cegcc* | *-*-haiku*) - # these systems don't actually have a c library (as such)! - ;; - *-*-rhapsody* | *-*-darwin1.[012]) - # Rhapsody C library is in the System framework - func_append deplibs " System.ltframework" - ;; - *-*-netbsd*) - # Don't link with libc until the a.out ld.so is fixed. - ;; - *-*-openbsd* | *-*-freebsd* | *-*-dragonfly*) - # Do not include libc due to us having libc/libc_r. - ;; - *-*-sco3.2v5* | *-*-sco5v6*) - # Causes problems with __ctype - ;; - *-*-sysv4.2uw2* | *-*-sysv5* | *-*-unixware* | *-*-OpenUNIX*) - # Compiler inserts libc in the correct place for threads to work - ;; - *) - # Add libc to deplibs on all other systems if necessary. - if test yes = "$build_libtool_need_lc"; then - func_append deplibs " -lc" - fi - ;; - esac - fi - - # Transform deplibs into only deplibs that can be linked in shared. - name_save=$name - libname_save=$libname - release_save=$release - versuffix_save=$versuffix - major_save=$major - # I'm not sure if I'm treating the release correctly. I think - # release should show up in the -l (ie -lgmp5) so we don't want to - # add it in twice. Is that correct? - release= - versuffix= - major= - newdeplibs= - droppeddeps=no - case $deplibs_check_method in - pass_all) - # Don't check for shared/static. Everything works. - # This might be a little naive. We might want to check - # whether the library exists or not. But this is on - # osf3 & osf4 and I'm not really sure... Just - # implementing what was already the behavior. - newdeplibs=$deplibs - ;; - test_compile) - # This code stresses the "libraries are programs" paradigm to its - # limits. Maybe even breaks it. We compile a program, linking it - # against the deplibs as a proxy for the library. Then we can check - # whether they linked in statically or dynamically with ldd. - $opt_dry_run || $RM conftest.c - cat > conftest.c </dev/null` - $nocaseglob - else - potential_libs=`ls $i/$libnameglob[.-]* 2>/dev/null` - fi - for potent_lib in $potential_libs; do - # Follow soft links. - if ls -lLd "$potent_lib" 2>/dev/null | - $GREP " -> " >/dev/null; then - continue - fi - # The statement above tries to avoid entering an - # endless loop below, in case of cyclic links. - # We might still enter an endless loop, since a link - # loop can be closed while we follow links, - # but so what? - potlib=$potent_lib - while test -h "$potlib" 2>/dev/null; do - potliblink=`ls -ld $potlib | $SED 's/.* -> //'` - case $potliblink in - [\\/]* | [A-Za-z]:[\\/]*) potlib=$potliblink;; - *) potlib=`$ECHO "$potlib" | $SED 's|[^/]*$||'`"$potliblink";; - esac - done - if eval $file_magic_cmd \"\$potlib\" 2>/dev/null | - $SED -e 10q | - $EGREP "$file_magic_regex" > /dev/null; then - func_append newdeplibs " $a_deplib" - a_deplib= - break 2 - fi - done - done - fi - if test -n "$a_deplib"; then - droppeddeps=yes - echo - $ECHO "*** Warning: linker path does not have real file for library $a_deplib." - echo "*** I have the capability to make that library automatically link in when" - echo "*** you link to this library. But I can only do this if you have a" - echo "*** shared version of the library, which you do not appear to have" - echo "*** because I did check the linker path looking for a file starting" - if test -z "$potlib"; then - $ECHO "*** with $libname but no candidates were found. (...for file magic test)" - else - $ECHO "*** with $libname and none of the candidates passed a file format test" - $ECHO "*** using a file magic. Last file checked: $potlib" - fi - fi - ;; - *) - # Add a -L argument. - func_append newdeplibs " $a_deplib" - ;; - esac - done # Gone through all deplibs. - ;; - match_pattern*) - set dummy $deplibs_check_method; shift - match_pattern_regex=`expr "$deplibs_check_method" : "$1 \(.*\)"` - for a_deplib in $deplibs; do - case $a_deplib in - -l*) - func_stripname -l '' "$a_deplib" - name=$func_stripname_result - if test yes = "$allow_libtool_libs_with_static_runtimes"; then - case " $predeps $postdeps " in - *" $a_deplib "*) - func_append newdeplibs " $a_deplib" - a_deplib= - ;; - esac - fi - if test -n "$a_deplib"; then - libname=`eval "\\$ECHO \"$libname_spec\""` - for i in $lib_search_path $sys_lib_search_path $shlib_search_path; do - potential_libs=`ls $i/$libname[.-]* 2>/dev/null` - for potent_lib in $potential_libs; do - potlib=$potent_lib # see symlink-check above in file_magic test - if eval "\$ECHO \"$potent_lib\"" 2>/dev/null | $SED 10q | \ - $EGREP "$match_pattern_regex" > /dev/null; then - func_append newdeplibs " $a_deplib" - a_deplib= - break 2 - fi - done - done - fi - if test -n "$a_deplib"; then - droppeddeps=yes - echo - $ECHO "*** Warning: linker path does not have real file for library $a_deplib." - echo "*** I have the capability to make that library automatically link in when" - echo "*** you link to this library. But I can only do this if you have a" - echo "*** shared version of the library, which you do not appear to have" - echo "*** because I did check the linker path looking for a file starting" - if test -z "$potlib"; then - $ECHO "*** with $libname but no candidates were found. (...for regex pattern test)" - else - $ECHO "*** with $libname and none of the candidates passed a file format test" - $ECHO "*** using a regex pattern. Last file checked: $potlib" - fi - fi - ;; - *) - # Add a -L argument. - func_append newdeplibs " $a_deplib" - ;; - esac - done # Gone through all deplibs. - ;; - none | unknown | *) - newdeplibs= - tmp_deplibs=`$ECHO " $deplibs" | $SED 's/ -lc$//; s/ -[LR][^ ]*//g'` - if test yes = "$allow_libtool_libs_with_static_runtimes"; then - for i in $predeps $postdeps; do - # can't use Xsed below, because $i might contain '/' - tmp_deplibs=`$ECHO " $tmp_deplibs" | $SED "s|$i||"` - done - fi - case $tmp_deplibs in - *[!\ \ ]*) - echo - if test none = "$deplibs_check_method"; then - echo "*** Warning: inter-library dependencies are not supported in this platform." - else - echo "*** Warning: inter-library dependencies are not known to be supported." - fi - echo "*** All declared inter-library dependencies are being dropped." - droppeddeps=yes - ;; - esac - ;; - esac - versuffix=$versuffix_save - major=$major_save - release=$release_save - libname=$libname_save - name=$name_save - - case $host in - *-*-rhapsody* | *-*-darwin1.[012]) - # On Rhapsody replace the C library with the System framework - newdeplibs=`$ECHO " $newdeplibs" | $SED 's/ -lc / System.ltframework /'` - ;; - esac - - if test yes = "$droppeddeps"; then - if test yes = "$module"; then - echo - echo "*** Warning: libtool could not satisfy all declared inter-library" - $ECHO "*** dependencies of module $libname. Therefore, libtool will create" - echo "*** a static module, that should work as long as the dlopening" - echo "*** application is linked with the -dlopen flag." - if test -z "$global_symbol_pipe"; then - echo - echo "*** However, this would only work if libtool was able to extract symbol" - echo "*** lists from a program, using 'nm' or equivalent, but libtool could" - echo "*** not find such a program. So, this module is probably useless." - echo "*** 'nm' from GNU binutils and a full rebuild may help." - fi - if test no = "$build_old_libs"; then - oldlibs=$output_objdir/$libname.$libext - build_libtool_libs=module - build_old_libs=yes - else - build_libtool_libs=no - fi - else - echo "*** The inter-library dependencies that have been dropped here will be" - echo "*** automatically added whenever a program is linked with this library" - echo "*** or is declared to -dlopen it." - - if test no = "$allow_undefined"; then - echo - echo "*** Since this library must not contain undefined symbols," - echo "*** because either the platform does not support them or" - echo "*** it was explicitly requested with -no-undefined," - echo "*** libtool will only create a static version of it." - if test no = "$build_old_libs"; then - oldlibs=$output_objdir/$libname.$libext - build_libtool_libs=module - build_old_libs=yes - else - build_libtool_libs=no - fi - fi - fi - fi - # Done checking deplibs! - deplibs=$newdeplibs - fi - # Time to change all our "foo.ltframework" stuff back to "-framework foo" - case $host in - *-*-darwin*) - newdeplibs=`$ECHO " $newdeplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - new_inherited_linker_flags=`$ECHO " $new_inherited_linker_flags" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - deplibs=`$ECHO " $deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - ;; - esac - - # move library search paths that coincide with paths to not yet - # installed libraries to the beginning of the library search list - new_libs= - for path in $notinst_path; do - case " $new_libs " in - *" -L$path/$objdir "*) ;; - *) - case " $deplibs " in - *" -L$path/$objdir "*) - func_append new_libs " -L$path/$objdir" ;; - esac - ;; - esac - done - for deplib in $deplibs; do - case $deplib in - -L*) - case " $new_libs " in - *" $deplib "*) ;; - *) func_append new_libs " $deplib" ;; - esac - ;; - *) func_append new_libs " $deplib" ;; - esac - done - deplibs=$new_libs - - # All the library-specific variables (install_libdir is set above). - library_names= - old_library= - dlname= - - # Test again, we may have decided not to build it any more - if test yes = "$build_libtool_libs"; then - # Remove $wl instances when linking with ld. - # FIXME: should test the right _cmds variable. - case $archive_cmds in - *\$LD\ *) wl= ;; - esac - if test yes = "$hardcode_into_libs"; then - # Hardcode the library paths - hardcode_libdirs= - dep_rpath= - rpath=$finalize_rpath - test relink = "$opt_mode" || rpath=$compile_rpath$rpath - for libdir in $rpath; do - if test -n "$hardcode_libdir_flag_spec"; then - if test -n "$hardcode_libdir_separator"; then - func_replace_sysroot "$libdir" - libdir=$func_replace_sysroot_result - if test -z "$hardcode_libdirs"; then - hardcode_libdirs=$libdir - else - # Just accumulate the unique libdirs. - case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in - *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*) - ;; - *) - func_append hardcode_libdirs "$hardcode_libdir_separator$libdir" - ;; - esac - fi - else - eval flag=\"$hardcode_libdir_flag_spec\" - func_append dep_rpath " $flag" - fi - elif test -n "$runpath_var"; then - case "$perm_rpath " in - *" $libdir "*) ;; - *) func_append perm_rpath " $libdir" ;; - esac - fi - done - # Substitute the hardcoded libdirs into the rpath. - if test -n "$hardcode_libdir_separator" && - test -n "$hardcode_libdirs"; then - libdir=$hardcode_libdirs - eval "dep_rpath=\"$hardcode_libdir_flag_spec\"" - fi - if test -n "$runpath_var" && test -n "$perm_rpath"; then - # We should set the runpath_var. - rpath= - for dir in $perm_rpath; do - func_append rpath "$dir:" - done - eval "$runpath_var='$rpath\$$runpath_var'; export $runpath_var" - fi - test -n "$dep_rpath" && deplibs="$dep_rpath $deplibs" - fi - - shlibpath=$finalize_shlibpath - test relink = "$opt_mode" || shlibpath=$compile_shlibpath$shlibpath - if test -n "$shlibpath"; then - eval "$shlibpath_var='$shlibpath\$$shlibpath_var'; export $shlibpath_var" - fi - - # Get the real and link names of the library. - eval shared_ext=\"$shrext_cmds\" - eval library_names=\"$library_names_spec\" - set dummy $library_names - shift - realname=$1 - shift - - if test -n "$soname_spec"; then - eval soname=\"$soname_spec\" - else - soname=$realname - fi - if test -z "$dlname"; then - dlname=$soname - fi - - lib=$output_objdir/$realname - linknames= - for link - do - func_append linknames " $link" - done - - # Use standard objects if they are pic - test -z "$pic_flag" && libobjs=`$ECHO "$libobjs" | $SP2NL | $SED "$lo2o" | $NL2SP` - test "X$libobjs" = "X " && libobjs= - - delfiles= - if test -n "$export_symbols" && test -n "$include_expsyms"; then - $opt_dry_run || cp "$export_symbols" "$output_objdir/$libname.uexp" - export_symbols=$output_objdir/$libname.uexp - func_append delfiles " $export_symbols" - fi - - orig_export_symbols= - case $host_os in - cygwin* | mingw* | cegcc*) - if test -n "$export_symbols" && test -z "$export_symbols_regex"; then - # exporting using user supplied symfile - func_dll_def_p "$export_symbols" || { - # and it's NOT already a .def file. Must figure out - # which of the given symbols are data symbols and tag - # them as such. So, trigger use of export_symbols_cmds. - # export_symbols gets reassigned inside the "prepare - # the list of exported symbols" if statement, so the - # include_expsyms logic still works. - orig_export_symbols=$export_symbols - export_symbols= - always_export_symbols=yes - } - fi - ;; - esac - - # Prepare the list of exported symbols - if test -z "$export_symbols"; then - if test yes = "$always_export_symbols" || test -n "$export_symbols_regex"; then - func_verbose "generating symbol list for '$libname.la'" - export_symbols=$output_objdir/$libname.exp - $opt_dry_run || $RM $export_symbols - cmds=$export_symbols_cmds - save_ifs=$IFS; IFS='~' - for cmd1 in $cmds; do - IFS=$save_ifs - # Take the normal branch if the nm_file_list_spec branch - # doesn't work or if tool conversion is not needed. - case $nm_file_list_spec~$to_tool_file_cmd in - *~func_convert_file_noop | *~func_convert_file_msys_to_w32 | ~*) - try_normal_branch=yes - eval cmd=\"$cmd1\" - func_len " $cmd" - len=$func_len_result - ;; - *) - try_normal_branch=no - ;; - esac - if test yes = "$try_normal_branch" \ - && { test "$len" -lt "$max_cmd_len" \ - || test "$max_cmd_len" -le -1; } - then - func_show_eval "$cmd" 'exit $?' - skipped_export=false - elif test -n "$nm_file_list_spec"; then - func_basename "$output" - output_la=$func_basename_result - save_libobjs=$libobjs - save_output=$output - output=$output_objdir/$output_la.nm - func_to_tool_file "$output" - libobjs=$nm_file_list_spec$func_to_tool_file_result - func_append delfiles " $output" - func_verbose "creating $NM input file list: $output" - for obj in $save_libobjs; do - func_to_tool_file "$obj" - $ECHO "$func_to_tool_file_result" - done > "$output" - eval cmd=\"$cmd1\" - func_show_eval "$cmd" 'exit $?' - output=$save_output - libobjs=$save_libobjs - skipped_export=false - else - # The command line is too long to execute in one step. - func_verbose "using reloadable object file for export list..." - skipped_export=: - # Break out early, otherwise skipped_export may be - # set to false by a later but shorter cmd. - break - fi - done - IFS=$save_ifs - if test -n "$export_symbols_regex" && test : != "$skipped_export"; then - func_show_eval '$EGREP -e "$export_symbols_regex" "$export_symbols" > "${export_symbols}T"' - func_show_eval '$MV "${export_symbols}T" "$export_symbols"' - fi - fi - fi - - if test -n "$export_symbols" && test -n "$include_expsyms"; then - tmp_export_symbols=$export_symbols - test -n "$orig_export_symbols" && tmp_export_symbols=$orig_export_symbols - $opt_dry_run || eval '$ECHO "$include_expsyms" | $SP2NL >> "$tmp_export_symbols"' - fi - - if test : != "$skipped_export" && test -n "$orig_export_symbols"; then - # The given exports_symbols file has to be filtered, so filter it. - func_verbose "filter symbol list for '$libname.la' to tag DATA exports" - # FIXME: $output_objdir/$libname.filter potentially contains lots of - # 's' commands, which not all seds can handle. GNU sed should be fine - # though. Also, the filter scales superlinearly with the number of - # global variables. join(1) would be nice here, but unfortunately - # isn't a blessed tool. - $opt_dry_run || $SED -e '/[ ,]DATA/!d;s,\(.*\)\([ \,].*\),s|^\1$|\1\2|,' < $export_symbols > $output_objdir/$libname.filter - func_append delfiles " $export_symbols $output_objdir/$libname.filter" - export_symbols=$output_objdir/$libname.def - $opt_dry_run || $SED -f $output_objdir/$libname.filter < $orig_export_symbols > $export_symbols - fi - - tmp_deplibs= - for test_deplib in $deplibs; do - case " $convenience " in - *" $test_deplib "*) ;; - *) - func_append tmp_deplibs " $test_deplib" - ;; - esac - done - deplibs=$tmp_deplibs - - if test -n "$convenience"; then - if test -n "$whole_archive_flag_spec" && - test yes = "$compiler_needs_object" && - test -z "$libobjs"; then - # extract the archives, so we have objects to list. - # TODO: could optimize this to just extract one archive. - whole_archive_flag_spec= - fi - if test -n "$whole_archive_flag_spec"; then - save_libobjs=$libobjs - eval libobjs=\"\$libobjs $whole_archive_flag_spec\" - test "X$libobjs" = "X " && libobjs= - else - gentop=$output_objdir/${outputname}x - func_append generated " $gentop" - - func_extract_archives $gentop $convenience - func_append libobjs " $func_extract_archives_result" - test "X$libobjs" = "X " && libobjs= - fi - fi - - if test yes = "$thread_safe" && test -n "$thread_safe_flag_spec"; then - eval flag=\"$thread_safe_flag_spec\" - func_append linker_flags " $flag" - fi - - # Make a backup of the uninstalled library when relinking - if test relink = "$opt_mode"; then - $opt_dry_run || eval '(cd $output_objdir && $RM ${realname}U && $MV $realname ${realname}U)' || exit $? - fi - - # Do each of the archive commands. - if test yes = "$module" && test -n "$module_cmds"; then - if test -n "$export_symbols" && test -n "$module_expsym_cmds"; then - eval test_cmds=\"$module_expsym_cmds\" - cmds=$module_expsym_cmds - else - eval test_cmds=\"$module_cmds\" - cmds=$module_cmds - fi - else - if test -n "$export_symbols" && test -n "$archive_expsym_cmds"; then - eval test_cmds=\"$archive_expsym_cmds\" - cmds=$archive_expsym_cmds - else - eval test_cmds=\"$archive_cmds\" - cmds=$archive_cmds - fi - fi - - if test : != "$skipped_export" && - func_len " $test_cmds" && - len=$func_len_result && - test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then - : - else - # The command line is too long to link in one step, link piecewise - # or, if using GNU ld and skipped_export is not :, use a linker - # script. - - # Save the value of $output and $libobjs because we want to - # use them later. If we have whole_archive_flag_spec, we - # want to use save_libobjs as it was before - # whole_archive_flag_spec was expanded, because we can't - # assume the linker understands whole_archive_flag_spec. - # This may have to be revisited, in case too many - # convenience libraries get linked in and end up exceeding - # the spec. - if test -z "$convenience" || test -z "$whole_archive_flag_spec"; then - save_libobjs=$libobjs - fi - save_output=$output - func_basename "$output" - output_la=$func_basename_result - - # Clear the reloadable object creation command queue and - # initialize k to one. - test_cmds= - concat_cmds= - objlist= - last_robj= - k=1 - - if test -n "$save_libobjs" && test : != "$skipped_export" && test yes = "$with_gnu_ld"; then - output=$output_objdir/$output_la.lnkscript - func_verbose "creating GNU ld script: $output" - echo 'INPUT (' > $output - for obj in $save_libobjs - do - func_to_tool_file "$obj" - $ECHO "$func_to_tool_file_result" >> $output - done - echo ')' >> $output - func_append delfiles " $output" - func_to_tool_file "$output" - output=$func_to_tool_file_result - elif test -n "$save_libobjs" && test : != "$skipped_export" && test -n "$file_list_spec"; then - output=$output_objdir/$output_la.lnk - func_verbose "creating linker input file list: $output" - : > $output - set x $save_libobjs - shift - firstobj= - if test yes = "$compiler_needs_object"; then - firstobj="$1 " - shift - fi - for obj - do - func_to_tool_file "$obj" - $ECHO "$func_to_tool_file_result" >> $output - done - func_append delfiles " $output" - func_to_tool_file "$output" - output=$firstobj\"$file_list_spec$func_to_tool_file_result\" - else - if test -n "$save_libobjs"; then - func_verbose "creating reloadable object files..." - output=$output_objdir/$output_la-$k.$objext - eval test_cmds=\"$reload_cmds\" - func_len " $test_cmds" - len0=$func_len_result - len=$len0 - - # Loop over the list of objects to be linked. - for obj in $save_libobjs - do - func_len " $obj" - func_arith $len + $func_len_result - len=$func_arith_result - if test -z "$objlist" || - test "$len" -lt "$max_cmd_len"; then - func_append objlist " $obj" - else - # The command $test_cmds is almost too long, add a - # command to the queue. - if test 1 -eq "$k"; then - # The first file doesn't have a previous command to add. - reload_objs=$objlist - eval concat_cmds=\"$reload_cmds\" - else - # All subsequent reloadable object files will link in - # the last one created. - reload_objs="$objlist $last_robj" - eval concat_cmds=\"\$concat_cmds~$reload_cmds~\$RM $last_robj\" - fi - last_robj=$output_objdir/$output_la-$k.$objext - func_arith $k + 1 - k=$func_arith_result - output=$output_objdir/$output_la-$k.$objext - objlist=" $obj" - func_len " $last_robj" - func_arith $len0 + $func_len_result - len=$func_arith_result - fi - done - # Handle the remaining objects by creating one last - # reloadable object file. All subsequent reloadable object - # files will link in the last one created. - test -z "$concat_cmds" || concat_cmds=$concat_cmds~ - reload_objs="$objlist $last_robj" - eval concat_cmds=\"\$concat_cmds$reload_cmds\" - if test -n "$last_robj"; then - eval concat_cmds=\"\$concat_cmds~\$RM $last_robj\" - fi - func_append delfiles " $output" - - else - output= - fi - - ${skipped_export-false} && { - func_verbose "generating symbol list for '$libname.la'" - export_symbols=$output_objdir/$libname.exp - $opt_dry_run || $RM $export_symbols - libobjs=$output - # Append the command to create the export file. - test -z "$concat_cmds" || concat_cmds=$concat_cmds~ - eval concat_cmds=\"\$concat_cmds$export_symbols_cmds\" - if test -n "$last_robj"; then - eval concat_cmds=\"\$concat_cmds~\$RM $last_robj\" - fi - } - - test -n "$save_libobjs" && - func_verbose "creating a temporary reloadable object file: $output" - - # Loop through the commands generated above and execute them. - save_ifs=$IFS; IFS='~' - for cmd in $concat_cmds; do - IFS=$save_ifs - $opt_quiet || { - func_quote_for_expand "$cmd" - eval "func_echo $func_quote_for_expand_result" - } - $opt_dry_run || eval "$cmd" || { - lt_exit=$? - - # Restore the uninstalled library and exit - if test relink = "$opt_mode"; then - ( cd "$output_objdir" && \ - $RM "${realname}T" && \ - $MV "${realname}U" "$realname" ) - fi - - exit $lt_exit - } - done - IFS=$save_ifs - - if test -n "$export_symbols_regex" && ${skipped_export-false}; then - func_show_eval '$EGREP -e "$export_symbols_regex" "$export_symbols" > "${export_symbols}T"' - func_show_eval '$MV "${export_symbols}T" "$export_symbols"' - fi - fi - - ${skipped_export-false} && { - if test -n "$export_symbols" && test -n "$include_expsyms"; then - tmp_export_symbols=$export_symbols - test -n "$orig_export_symbols" && tmp_export_symbols=$orig_export_symbols - $opt_dry_run || eval '$ECHO "$include_expsyms" | $SP2NL >> "$tmp_export_symbols"' - fi - - if test -n "$orig_export_symbols"; then - # The given exports_symbols file has to be filtered, so filter it. - func_verbose "filter symbol list for '$libname.la' to tag DATA exports" - # FIXME: $output_objdir/$libname.filter potentially contains lots of - # 's' commands, which not all seds can handle. GNU sed should be fine - # though. Also, the filter scales superlinearly with the number of - # global variables. join(1) would be nice here, but unfortunately - # isn't a blessed tool. - $opt_dry_run || $SED -e '/[ ,]DATA/!d;s,\(.*\)\([ \,].*\),s|^\1$|\1\2|,' < $export_symbols > $output_objdir/$libname.filter - func_append delfiles " $export_symbols $output_objdir/$libname.filter" - export_symbols=$output_objdir/$libname.def - $opt_dry_run || $SED -f $output_objdir/$libname.filter < $orig_export_symbols > $export_symbols - fi - } - - libobjs=$output - # Restore the value of output. - output=$save_output - - if test -n "$convenience" && test -n "$whole_archive_flag_spec"; then - eval libobjs=\"\$libobjs $whole_archive_flag_spec\" - test "X$libobjs" = "X " && libobjs= - fi - # Expand the library linking commands again to reset the - # value of $libobjs for piecewise linking. - - # Do each of the archive commands. - if test yes = "$module" && test -n "$module_cmds"; then - if test -n "$export_symbols" && test -n "$module_expsym_cmds"; then - cmds=$module_expsym_cmds - else - cmds=$module_cmds - fi - else - if test -n "$export_symbols" && test -n "$archive_expsym_cmds"; then - cmds=$archive_expsym_cmds - else - cmds=$archive_cmds - fi - fi - fi - - if test -n "$delfiles"; then - # Append the command to remove temporary files to $cmds. - eval cmds=\"\$cmds~\$RM $delfiles\" - fi - - # Add any objects from preloaded convenience libraries - if test -n "$dlprefiles"; then - gentop=$output_objdir/${outputname}x - func_append generated " $gentop" - - func_extract_archives $gentop $dlprefiles - func_append libobjs " $func_extract_archives_result" - test "X$libobjs" = "X " && libobjs= - fi - - save_ifs=$IFS; IFS='~' - for cmd in $cmds; do - IFS=$sp$nl - eval cmd=\"$cmd\" - IFS=$save_ifs - $opt_quiet || { - func_quote_for_expand "$cmd" - eval "func_echo $func_quote_for_expand_result" - } - $opt_dry_run || eval "$cmd" || { - lt_exit=$? - - # Restore the uninstalled library and exit - if test relink = "$opt_mode"; then - ( cd "$output_objdir" && \ - $RM "${realname}T" && \ - $MV "${realname}U" "$realname" ) - fi - - exit $lt_exit - } - done - IFS=$save_ifs - - # Restore the uninstalled library and exit - if test relink = "$opt_mode"; then - $opt_dry_run || eval '(cd $output_objdir && $RM ${realname}T && $MV $realname ${realname}T && $MV ${realname}U $realname)' || exit $? - - if test -n "$convenience"; then - if test -z "$whole_archive_flag_spec"; then - func_show_eval '${RM}r "$gentop"' - fi - fi - - exit $EXIT_SUCCESS - fi - - # Create links to the real library. - for linkname in $linknames; do - if test "$realname" != "$linkname"; then - func_show_eval '(cd "$output_objdir" && $RM "$linkname" && $LN_S "$realname" "$linkname")' 'exit $?' - fi - done - - # If -module or -export-dynamic was specified, set the dlname. - if test yes = "$module" || test yes = "$export_dynamic"; then - # On all known operating systems, these are identical. - dlname=$soname - fi - fi - ;; - - obj) - if test -n "$dlfiles$dlprefiles" || test no != "$dlself"; then - func_warning "'-dlopen' is ignored for objects" - fi - - case " $deplibs" in - *\ -l* | *\ -L*) - func_warning "'-l' and '-L' are ignored for objects" ;; - esac - - test -n "$rpath" && \ - func_warning "'-rpath' is ignored for objects" - - test -n "$xrpath" && \ - func_warning "'-R' is ignored for objects" - - test -n "$vinfo" && \ - func_warning "'-version-info' is ignored for objects" - - test -n "$release" && \ - func_warning "'-release' is ignored for objects" - - case $output in - *.lo) - test -n "$objs$old_deplibs" && \ - func_fatal_error "cannot build library object '$output' from non-libtool objects" - - libobj=$output - func_lo2o "$libobj" - obj=$func_lo2o_result - ;; - *) - libobj= - obj=$output - ;; - esac - - # Delete the old objects. - $opt_dry_run || $RM $obj $libobj - - # Objects from convenience libraries. This assumes - # single-version convenience libraries. Whenever we create - # different ones for PIC/non-PIC, this we'll have to duplicate - # the extraction. - reload_conv_objs= - gentop= - # if reload_cmds runs $LD directly, get rid of -Wl from - # whole_archive_flag_spec and hope we can get by with turning comma - # into space. - case $reload_cmds in - *\$LD[\ \$]*) wl= ;; - esac - if test -n "$convenience"; then - if test -n "$whole_archive_flag_spec"; then - eval tmp_whole_archive_flags=\"$whole_archive_flag_spec\" - test -n "$wl" || tmp_whole_archive_flags=`$ECHO "$tmp_whole_archive_flags" | $SED 's|,| |g'` - reload_conv_objs=$reload_objs\ $tmp_whole_archive_flags - else - gentop=$output_objdir/${obj}x - func_append generated " $gentop" - - func_extract_archives $gentop $convenience - reload_conv_objs="$reload_objs $func_extract_archives_result" - fi - fi - - # If we're not building shared, we need to use non_pic_objs - test yes = "$build_libtool_libs" || libobjs=$non_pic_objects - - # Create the old-style object. - reload_objs=$objs$old_deplibs' '`$ECHO "$libobjs" | $SP2NL | $SED "/\.$libext$/d; /\.lib$/d; $lo2o" | $NL2SP`' '$reload_conv_objs - - output=$obj - func_execute_cmds "$reload_cmds" 'exit $?' - - # Exit if we aren't doing a library object file. - if test -z "$libobj"; then - if test -n "$gentop"; then - func_show_eval '${RM}r "$gentop"' - fi - - exit $EXIT_SUCCESS - fi - - test yes = "$build_libtool_libs" || { - if test -n "$gentop"; then - func_show_eval '${RM}r "$gentop"' - fi - - # Create an invalid libtool object if no PIC, so that we don't - # accidentally link it into a program. - # $show "echo timestamp > $libobj" - # $opt_dry_run || eval "echo timestamp > $libobj" || exit $? - exit $EXIT_SUCCESS - } - - if test -n "$pic_flag" || test default != "$pic_mode"; then - # Only do commands if we really have different PIC objects. - reload_objs="$libobjs $reload_conv_objs" - output=$libobj - func_execute_cmds "$reload_cmds" 'exit $?' - fi - - if test -n "$gentop"; then - func_show_eval '${RM}r "$gentop"' - fi - - exit $EXIT_SUCCESS - ;; - - prog) - case $host in - *cygwin*) func_stripname '' '.exe' "$output" - output=$func_stripname_result.exe;; - esac - test -n "$vinfo" && \ - func_warning "'-version-info' is ignored for programs" - - test -n "$release" && \ - func_warning "'-release' is ignored for programs" - - $preload \ - && test unknown,unknown,unknown = "$dlopen_support,$dlopen_self,$dlopen_self_static" \ - && func_warning "'LT_INIT([dlopen])' not used. Assuming no dlopen support." - - case $host in - *-*-rhapsody* | *-*-darwin1.[012]) - # On Rhapsody replace the C library is the System framework - compile_deplibs=`$ECHO " $compile_deplibs" | $SED 's/ -lc / System.ltframework /'` - finalize_deplibs=`$ECHO " $finalize_deplibs" | $SED 's/ -lc / System.ltframework /'` - ;; - esac - - case $host in - *-*-darwin*) - # Don't allow lazy linking, it breaks C++ global constructors - # But is supposedly fixed on 10.4 or later (yay!). - if test CXX = "$tagname"; then - case ${MACOSX_DEPLOYMENT_TARGET-10.0} in - 10.[0123]) - func_append compile_command " $wl-bind_at_load" - func_append finalize_command " $wl-bind_at_load" - ;; - esac - fi - # Time to change all our "foo.ltframework" stuff back to "-framework foo" - compile_deplibs=`$ECHO " $compile_deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - finalize_deplibs=`$ECHO " $finalize_deplibs" | $SED 's% \([^ $]*\).ltframework% -framework \1%g'` - ;; - esac - - - # move library search paths that coincide with paths to not yet - # installed libraries to the beginning of the library search list - new_libs= - for path in $notinst_path; do - case " $new_libs " in - *" -L$path/$objdir "*) ;; - *) - case " $compile_deplibs " in - *" -L$path/$objdir "*) - func_append new_libs " -L$path/$objdir" ;; - esac - ;; - esac - done - for deplib in $compile_deplibs; do - case $deplib in - -L*) - case " $new_libs " in - *" $deplib "*) ;; - *) func_append new_libs " $deplib" ;; - esac - ;; - *) func_append new_libs " $deplib" ;; - esac - done - compile_deplibs=$new_libs - - - func_append compile_command " $compile_deplibs" - func_append finalize_command " $finalize_deplibs" - - if test -n "$rpath$xrpath"; then - # If the user specified any rpath flags, then add them. - for libdir in $rpath $xrpath; do - # This is the magic to use -rpath. - case "$finalize_rpath " in - *" $libdir "*) ;; - *) func_append finalize_rpath " $libdir" ;; - esac - done - fi - - # Now hardcode the library paths - rpath= - hardcode_libdirs= - for libdir in $compile_rpath $finalize_rpath; do - if test -n "$hardcode_libdir_flag_spec"; then - if test -n "$hardcode_libdir_separator"; then - if test -z "$hardcode_libdirs"; then - hardcode_libdirs=$libdir - else - # Just accumulate the unique libdirs. - case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in - *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*) - ;; - *) - func_append hardcode_libdirs "$hardcode_libdir_separator$libdir" - ;; - esac - fi - else - eval flag=\"$hardcode_libdir_flag_spec\" - func_append rpath " $flag" - fi - elif test -n "$runpath_var"; then - case "$perm_rpath " in - *" $libdir "*) ;; - *) func_append perm_rpath " $libdir" ;; - esac - fi - case $host in - *-*-cygwin* | *-*-mingw* | *-*-pw32* | *-*-os2* | *-cegcc*) - testbindir=`$ECHO "$libdir" | $SED -e 's*/lib$*/bin*'` - case :$dllsearchpath: in - *":$libdir:"*) ;; - ::) dllsearchpath=$libdir;; - *) func_append dllsearchpath ":$libdir";; - esac - case :$dllsearchpath: in - *":$testbindir:"*) ;; - ::) dllsearchpath=$testbindir;; - *) func_append dllsearchpath ":$testbindir";; - esac - ;; - esac - done - # Substitute the hardcoded libdirs into the rpath. - if test -n "$hardcode_libdir_separator" && - test -n "$hardcode_libdirs"; then - libdir=$hardcode_libdirs - eval rpath=\" $hardcode_libdir_flag_spec\" - fi - compile_rpath=$rpath - - rpath= - hardcode_libdirs= - for libdir in $finalize_rpath; do - if test -n "$hardcode_libdir_flag_spec"; then - if test -n "$hardcode_libdir_separator"; then - if test -z "$hardcode_libdirs"; then - hardcode_libdirs=$libdir - else - # Just accumulate the unique libdirs. - case $hardcode_libdir_separator$hardcode_libdirs$hardcode_libdir_separator in - *"$hardcode_libdir_separator$libdir$hardcode_libdir_separator"*) - ;; - *) - func_append hardcode_libdirs "$hardcode_libdir_separator$libdir" - ;; - esac - fi - else - eval flag=\"$hardcode_libdir_flag_spec\" - func_append rpath " $flag" - fi - elif test -n "$runpath_var"; then - case "$finalize_perm_rpath " in - *" $libdir "*) ;; - *) func_append finalize_perm_rpath " $libdir" ;; - esac - fi - done - # Substitute the hardcoded libdirs into the rpath. - if test -n "$hardcode_libdir_separator" && - test -n "$hardcode_libdirs"; then - libdir=$hardcode_libdirs - eval rpath=\" $hardcode_libdir_flag_spec\" - fi - finalize_rpath=$rpath - - if test -n "$libobjs" && test yes = "$build_old_libs"; then - # Transform all the library objects into standard objects. - compile_command=`$ECHO "$compile_command" | $SP2NL | $SED "$lo2o" | $NL2SP` - finalize_command=`$ECHO "$finalize_command" | $SP2NL | $SED "$lo2o" | $NL2SP` - fi - - func_generate_dlsyms "$outputname" "@PROGRAM@" false - - # template prelinking step - if test -n "$prelink_cmds"; then - func_execute_cmds "$prelink_cmds" 'exit $?' - fi - - wrappers_required=: - case $host in - *cegcc* | *mingw32ce*) - # Disable wrappers for cegcc and mingw32ce hosts, we are cross compiling anyway. - wrappers_required=false - ;; - *cygwin* | *mingw* ) - test yes = "$build_libtool_libs" || wrappers_required=false - ;; - *) - if test no = "$need_relink" || test yes != "$build_libtool_libs"; then - wrappers_required=false - fi - ;; - esac - $wrappers_required || { - # Replace the output file specification. - compile_command=`$ECHO "$compile_command" | $SED 's%@OUTPUT@%'"$output"'%g'` - link_command=$compile_command$compile_rpath - - # We have no uninstalled library dependencies, so finalize right now. - exit_status=0 - func_show_eval "$link_command" 'exit_status=$?' - - if test -n "$postlink_cmds"; then - func_to_tool_file "$output" - postlink_cmds=`func_echo_all "$postlink_cmds" | $SED -e 's%@OUTPUT@%'"$output"'%g' -e 's%@TOOL_OUTPUT@%'"$func_to_tool_file_result"'%g'` - func_execute_cmds "$postlink_cmds" 'exit $?' - fi - - # Delete the generated files. - if test -f "$output_objdir/${outputname}S.$objext"; then - func_show_eval '$RM "$output_objdir/${outputname}S.$objext"' - fi - - exit $exit_status - } - - if test -n "$compile_shlibpath$finalize_shlibpath"; then - compile_command="$shlibpath_var=\"$compile_shlibpath$finalize_shlibpath\$$shlibpath_var\" $compile_command" - fi - if test -n "$finalize_shlibpath"; then - finalize_command="$shlibpath_var=\"$finalize_shlibpath\$$shlibpath_var\" $finalize_command" - fi - - compile_var= - finalize_var= - if test -n "$runpath_var"; then - if test -n "$perm_rpath"; then - # We should set the runpath_var. - rpath= - for dir in $perm_rpath; do - func_append rpath "$dir:" - done - compile_var="$runpath_var=\"$rpath\$$runpath_var\" " - fi - if test -n "$finalize_perm_rpath"; then - # We should set the runpath_var. - rpath= - for dir in $finalize_perm_rpath; do - func_append rpath "$dir:" - done - finalize_var="$runpath_var=\"$rpath\$$runpath_var\" " - fi - fi - - if test yes = "$no_install"; then - # We don't need to create a wrapper script. - link_command=$compile_var$compile_command$compile_rpath - # Replace the output file specification. - link_command=`$ECHO "$link_command" | $SED 's%@OUTPUT@%'"$output"'%g'` - # Delete the old output file. - $opt_dry_run || $RM $output - # Link the executable and exit - func_show_eval "$link_command" 'exit $?' - - if test -n "$postlink_cmds"; then - func_to_tool_file "$output" - postlink_cmds=`func_echo_all "$postlink_cmds" | $SED -e 's%@OUTPUT@%'"$output"'%g' -e 's%@TOOL_OUTPUT@%'"$func_to_tool_file_result"'%g'` - func_execute_cmds "$postlink_cmds" 'exit $?' - fi - - exit $EXIT_SUCCESS - fi - - case $hardcode_action,$fast_install in - relink,*) - # Fast installation is not supported - link_command=$compile_var$compile_command$compile_rpath - relink_command=$finalize_var$finalize_command$finalize_rpath - - func_warning "this platform does not like uninstalled shared libraries" - func_warning "'$output' will be relinked during installation" - ;; - *,yes) - link_command=$finalize_var$compile_command$finalize_rpath - relink_command=`$ECHO "$compile_var$compile_command$compile_rpath" | $SED 's%@OUTPUT@%\$progdir/\$file%g'` - ;; - *,no) - link_command=$compile_var$compile_command$compile_rpath - relink_command=$finalize_var$finalize_command$finalize_rpath - ;; - *,needless) - link_command=$finalize_var$compile_command$finalize_rpath - relink_command= - ;; - esac - - # Replace the output file specification. - link_command=`$ECHO "$link_command" | $SED 's%@OUTPUT@%'"$output_objdir/$outputname"'%g'` - - # Delete the old output files. - $opt_dry_run || $RM $output $output_objdir/$outputname $output_objdir/lt-$outputname - - func_show_eval "$link_command" 'exit $?' - - if test -n "$postlink_cmds"; then - func_to_tool_file "$output_objdir/$outputname" - postlink_cmds=`func_echo_all "$postlink_cmds" | $SED -e 's%@OUTPUT@%'"$output_objdir/$outputname"'%g' -e 's%@TOOL_OUTPUT@%'"$func_to_tool_file_result"'%g'` - func_execute_cmds "$postlink_cmds" 'exit $?' - fi - - # Now create the wrapper script. - func_verbose "creating $output" - - # Quote the relink command for shipping. - if test -n "$relink_command"; then - # Preserve any variables that may affect compiler behavior - for var in $variables_saved_for_relink; do - if eval test -z \"\${$var+set}\"; then - relink_command="{ test -z \"\${$var+set}\" || $lt_unset $var || { $var=; export $var; }; }; $relink_command" - elif eval var_value=\$$var; test -z "$var_value"; then - relink_command="$var=; export $var; $relink_command" - else - func_quote_for_eval "$var_value" - relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command" - fi - done - relink_command="(cd `pwd`; $relink_command)" - relink_command=`$ECHO "$relink_command" | $SED "$sed_quote_subst"` - fi - - # Only actually do things if not in dry run mode. - $opt_dry_run || { - # win32 will think the script is a binary if it has - # a .exe suffix, so we strip it off here. - case $output in - *.exe) func_stripname '' '.exe' "$output" - output=$func_stripname_result ;; - esac - # test for cygwin because mv fails w/o .exe extensions - case $host in - *cygwin*) - exeext=.exe - func_stripname '' '.exe' "$outputname" - outputname=$func_stripname_result ;; - *) exeext= ;; - esac - case $host in - *cygwin* | *mingw* ) - func_dirname_and_basename "$output" "" "." - output_name=$func_basename_result - output_path=$func_dirname_result - cwrappersource=$output_path/$objdir/lt-$output_name.c - cwrapper=$output_path/$output_name.exe - $RM $cwrappersource $cwrapper - trap "$RM $cwrappersource $cwrapper; exit $EXIT_FAILURE" 1 2 15 - - func_emit_cwrapperexe_src > $cwrappersource - - # The wrapper executable is built using the $host compiler, - # because it contains $host paths and files. If cross- - # compiling, it, like the target executable, must be - # executed on the $host or under an emulation environment. - $opt_dry_run || { - $LTCC $LTCFLAGS -o $cwrapper $cwrappersource - $STRIP $cwrapper - } - - # Now, create the wrapper script for func_source use: - func_ltwrapper_scriptname $cwrapper - $RM $func_ltwrapper_scriptname_result - trap "$RM $func_ltwrapper_scriptname_result; exit $EXIT_FAILURE" 1 2 15 - $opt_dry_run || { - # note: this script will not be executed, so do not chmod. - if test "x$build" = "x$host"; then - $cwrapper --lt-dump-script > $func_ltwrapper_scriptname_result - else - func_emit_wrapper no > $func_ltwrapper_scriptname_result - fi - } - ;; - * ) - $RM $output - trap "$RM $output; exit $EXIT_FAILURE" 1 2 15 - - func_emit_wrapper no > $output - chmod +x $output - ;; - esac - } - exit $EXIT_SUCCESS - ;; - esac - - # See if we need to build an old-fashioned archive. - for oldlib in $oldlibs; do - - case $build_libtool_libs in - convenience) - oldobjs="$libobjs_save $symfileobj" - addlibs=$convenience - build_libtool_libs=no - ;; - module) - oldobjs=$libobjs_save - addlibs=$old_convenience - build_libtool_libs=no - ;; - *) - oldobjs="$old_deplibs $non_pic_objects" - $preload && test -f "$symfileobj" \ - && func_append oldobjs " $symfileobj" - addlibs=$old_convenience - ;; - esac - - if test -n "$addlibs"; then - gentop=$output_objdir/${outputname}x - func_append generated " $gentop" - - func_extract_archives $gentop $addlibs - func_append oldobjs " $func_extract_archives_result" - fi - - # Do each command in the archive commands. - if test -n "$old_archive_from_new_cmds" && test yes = "$build_libtool_libs"; then - cmds=$old_archive_from_new_cmds - else - - # Add any objects from preloaded convenience libraries - if test -n "$dlprefiles"; then - gentop=$output_objdir/${outputname}x - func_append generated " $gentop" - - func_extract_archives $gentop $dlprefiles - func_append oldobjs " $func_extract_archives_result" - fi - - # POSIX demands no paths to be encoded in archives. We have - # to avoid creating archives with duplicate basenames if we - # might have to extract them afterwards, e.g., when creating a - # static archive out of a convenience library, or when linking - # the entirety of a libtool archive into another (currently - # not supported by libtool). - if (for obj in $oldobjs - do - func_basename "$obj" - $ECHO "$func_basename_result" - done | sort | sort -uc >/dev/null 2>&1); then - : - else - echo "copying selected object files to avoid basename conflicts..." - gentop=$output_objdir/${outputname}x - func_append generated " $gentop" - func_mkdir_p "$gentop" - save_oldobjs=$oldobjs - oldobjs= - counter=1 - for obj in $save_oldobjs - do - func_basename "$obj" - objbase=$func_basename_result - case " $oldobjs " in - " ") oldobjs=$obj ;; - *[\ /]"$objbase "*) - while :; do - # Make sure we don't pick an alternate name that also - # overlaps. - newobj=lt$counter-$objbase - func_arith $counter + 1 - counter=$func_arith_result - case " $oldobjs " in - *[\ /]"$newobj "*) ;; - *) if test ! -f "$gentop/$newobj"; then break; fi ;; - esac - done - func_show_eval "ln $obj $gentop/$newobj || cp $obj $gentop/$newobj" - func_append oldobjs " $gentop/$newobj" - ;; - *) func_append oldobjs " $obj" ;; - esac - done - fi - func_to_tool_file "$oldlib" func_convert_file_msys_to_w32 - tool_oldlib=$func_to_tool_file_result - eval cmds=\"$old_archive_cmds\" - - func_len " $cmds" - len=$func_len_result - if test "$len" -lt "$max_cmd_len" || test "$max_cmd_len" -le -1; then - cmds=$old_archive_cmds - elif test -n "$archiver_list_spec"; then - func_verbose "using command file archive linking..." - for obj in $oldobjs - do - func_to_tool_file "$obj" - $ECHO "$func_to_tool_file_result" - done > $output_objdir/$libname.libcmd - func_to_tool_file "$output_objdir/$libname.libcmd" - oldobjs=" $archiver_list_spec$func_to_tool_file_result" - cmds=$old_archive_cmds - else - # the command line is too long to link in one step, link in parts - func_verbose "using piecewise archive linking..." - save_RANLIB=$RANLIB - RANLIB=: - objlist= - concat_cmds= - save_oldobjs=$oldobjs - oldobjs= - # Is there a better way of finding the last object in the list? - for obj in $save_oldobjs - do - last_oldobj=$obj - done - eval test_cmds=\"$old_archive_cmds\" - func_len " $test_cmds" - len0=$func_len_result - len=$len0 - for obj in $save_oldobjs - do - func_len " $obj" - func_arith $len + $func_len_result - len=$func_arith_result - func_append objlist " $obj" - if test "$len" -lt "$max_cmd_len"; then - : - else - # the above command should be used before it gets too long - oldobjs=$objlist - if test "$obj" = "$last_oldobj"; then - RANLIB=$save_RANLIB - fi - test -z "$concat_cmds" || concat_cmds=$concat_cmds~ - eval concat_cmds=\"\$concat_cmds$old_archive_cmds\" - objlist= - len=$len0 - fi - done - RANLIB=$save_RANLIB - oldobjs=$objlist - if test -z "$oldobjs"; then - eval cmds=\"\$concat_cmds\" - else - eval cmds=\"\$concat_cmds~\$old_archive_cmds\" - fi - fi - fi - func_execute_cmds "$cmds" 'exit $?' - done - - test -n "$generated" && \ - func_show_eval "${RM}r$generated" - - # Now create the libtool archive. - case $output in - *.la) - old_library= - test yes = "$build_old_libs" && old_library=$libname.$libext - func_verbose "creating $output" - - # Preserve any variables that may affect compiler behavior - for var in $variables_saved_for_relink; do - if eval test -z \"\${$var+set}\"; then - relink_command="{ test -z \"\${$var+set}\" || $lt_unset $var || { $var=; export $var; }; }; $relink_command" - elif eval var_value=\$$var; test -z "$var_value"; then - relink_command="$var=; export $var; $relink_command" - else - func_quote_for_eval "$var_value" - relink_command="$var=$func_quote_for_eval_result; export $var; $relink_command" - fi - done - # Quote the link command for shipping. - relink_command="(cd `pwd`; $SHELL \"$progpath\" $preserve_args --mode=relink $libtool_args @inst_prefix_dir@)" - relink_command=`$ECHO "$relink_command" | $SED "$sed_quote_subst"` - if test yes = "$hardcode_automatic"; then - relink_command= - fi - - # Only create the output if not a dry run. - $opt_dry_run || { - for installed in no yes; do - if test yes = "$installed"; then - if test -z "$install_libdir"; then - break - fi - output=$output_objdir/${outputname}i - # Replace all uninstalled libtool libraries with the installed ones - newdependency_libs= - for deplib in $dependency_libs; do - case $deplib in - *.la) - func_basename "$deplib" - name=$func_basename_result - func_resolve_sysroot "$deplib" - eval libdir=`$SED -n -e 's/^libdir=\(.*\)$/\1/p' $func_resolve_sysroot_result` - test -z "$libdir" && \ - func_fatal_error "'$deplib' is not a valid libtool archive" - func_append newdependency_libs " ${lt_sysroot:+=}$libdir/$name" - ;; - -L*) - func_stripname -L '' "$deplib" - func_replace_sysroot "$func_stripname_result" - func_append newdependency_libs " -L$func_replace_sysroot_result" - ;; - -R*) - func_stripname -R '' "$deplib" - func_replace_sysroot "$func_stripname_result" - func_append newdependency_libs " -R$func_replace_sysroot_result" - ;; - *) func_append newdependency_libs " $deplib" ;; - esac - done - dependency_libs=$newdependency_libs - newdlfiles= - - for lib in $dlfiles; do - case $lib in - *.la) - func_basename "$lib" - name=$func_basename_result - eval libdir=`$SED -n -e 's/^libdir=\(.*\)$/\1/p' $lib` - test -z "$libdir" && \ - func_fatal_error "'$lib' is not a valid libtool archive" - func_append newdlfiles " ${lt_sysroot:+=}$libdir/$name" - ;; - *) func_append newdlfiles " $lib" ;; - esac - done - dlfiles=$newdlfiles - newdlprefiles= - for lib in $dlprefiles; do - case $lib in - *.la) - # Only pass preopened files to the pseudo-archive (for - # eventual linking with the app. that links it) if we - # didn't already link the preopened objects directly into - # the library: - func_basename "$lib" - name=$func_basename_result - eval libdir=`$SED -n -e 's/^libdir=\(.*\)$/\1/p' $lib` - test -z "$libdir" && \ - func_fatal_error "'$lib' is not a valid libtool archive" - func_append newdlprefiles " ${lt_sysroot:+=}$libdir/$name" - ;; - esac - done - dlprefiles=$newdlprefiles - else - newdlfiles= - for lib in $dlfiles; do - case $lib in - [\\/]* | [A-Za-z]:[\\/]*) abs=$lib ;; - *) abs=`pwd`"/$lib" ;; - esac - func_append newdlfiles " $abs" - done - dlfiles=$newdlfiles - newdlprefiles= - for lib in $dlprefiles; do - case $lib in - [\\/]* | [A-Za-z]:[\\/]*) abs=$lib ;; - *) abs=`pwd`"/$lib" ;; - esac - func_append newdlprefiles " $abs" - done - dlprefiles=$newdlprefiles - fi - $RM $output - # place dlname in correct position for cygwin - # In fact, it would be nice if we could use this code for all target - # systems that can't hard-code library paths into their executables - # and that have no shared library path variable independent of PATH, - # but it turns out we can't easily determine that from inspecting - # libtool variables, so we have to hard-code the OSs to which it - # applies here; at the moment, that means platforms that use the PE - # object format with DLL files. See the long comment at the top of - # tests/bindir.at for full details. - tdlname=$dlname - case $host,$output,$installed,$module,$dlname in - *cygwin*,*lai,yes,no,*.dll | *mingw*,*lai,yes,no,*.dll | *cegcc*,*lai,yes,no,*.dll) - # If a -bindir argument was supplied, place the dll there. - if test -n "$bindir"; then - func_relative_path "$install_libdir" "$bindir" - tdlname=$func_relative_path_result/$dlname - else - # Otherwise fall back on heuristic. - tdlname=../bin/$dlname - fi - ;; - esac - $ECHO > $output "\ -# $outputname - a libtool library file -# Generated by $PROGRAM (GNU $PACKAGE) $VERSION -# -# Please DO NOT delete this file! -# It is necessary for linking the library. - -# The name that we can dlopen(3). -dlname='$tdlname' - -# Names of this library. -library_names='$library_names' - -# The name of the static archive. -old_library='$old_library' - -# Linker flags that cannot go in dependency_libs. -inherited_linker_flags='$new_inherited_linker_flags' - -# Libraries that this one depends upon. -dependency_libs='$dependency_libs' - -# Names of additional weak libraries provided by this library -weak_library_names='$weak_libs' - -# Version information for $libname. -current=$current -age=$age -revision=$revision - -# Is this an already installed library? -installed=$installed - -# Should we warn about portability when linking against -modules? -shouldnotlink=$module - -# Files to dlopen/dlpreopen -dlopen='$dlfiles' -dlpreopen='$dlprefiles' - -# Directory that this library needs to be installed in: -libdir='$install_libdir'" - if test no,yes = "$installed,$need_relink"; then - $ECHO >> $output "\ -relink_command=\"$relink_command\"" - fi - done - } - - # Do a symbolic link so that the libtool archive can be found in - # LD_LIBRARY_PATH before the program is installed. - func_show_eval '( cd "$output_objdir" && $RM "$outputname" && $LN_S "../$outputname" "$outputname" )' 'exit $?' - ;; - esac - exit $EXIT_SUCCESS -} - -if test link = "$opt_mode" || test relink = "$opt_mode"; then - func_mode_link ${1+"$@"} -fi - - -# func_mode_uninstall arg... -func_mode_uninstall () -{ - $debug_cmd - - RM=$nonopt - files= - rmforce=false - exit_status=0 - - # This variable tells wrapper scripts just to set variables rather - # than running their programs. - libtool_install_magic=$magic - - for arg - do - case $arg in - -f) func_append RM " $arg"; rmforce=: ;; - -*) func_append RM " $arg" ;; - *) func_append files " $arg" ;; - esac - done - - test -z "$RM" && \ - func_fatal_help "you must specify an RM program" - - rmdirs= - - for file in $files; do - func_dirname "$file" "" "." - dir=$func_dirname_result - if test . = "$dir"; then - odir=$objdir - else - odir=$dir/$objdir - fi - func_basename "$file" - name=$func_basename_result - test uninstall = "$opt_mode" && odir=$dir - - # Remember odir for removal later, being careful to avoid duplicates - if test clean = "$opt_mode"; then - case " $rmdirs " in - *" $odir "*) ;; - *) func_append rmdirs " $odir" ;; - esac - fi - - # Don't error if the file doesn't exist and rm -f was used. - if { test -L "$file"; } >/dev/null 2>&1 || - { test -h "$file"; } >/dev/null 2>&1 || - test -f "$file"; then - : - elif test -d "$file"; then - exit_status=1 - continue - elif $rmforce; then - continue - fi - - rmfiles=$file - - case $name in - *.la) - # Possibly a libtool archive, so verify it. - if func_lalib_p "$file"; then - func_source $dir/$name - - # Delete the libtool libraries and symlinks. - for n in $library_names; do - func_append rmfiles " $odir/$n" - done - test -n "$old_library" && func_append rmfiles " $odir/$old_library" - - case $opt_mode in - clean) - case " $library_names " in - *" $dlname "*) ;; - *) test -n "$dlname" && func_append rmfiles " $odir/$dlname" ;; - esac - test -n "$libdir" && func_append rmfiles " $odir/$name $odir/${name}i" - ;; - uninstall) - if test -n "$library_names"; then - # Do each command in the postuninstall commands. - func_execute_cmds "$postuninstall_cmds" '$rmforce || exit_status=1' - fi - - if test -n "$old_library"; then - # Do each command in the old_postuninstall commands. - func_execute_cmds "$old_postuninstall_cmds" '$rmforce || exit_status=1' - fi - # FIXME: should reinstall the best remaining shared library. - ;; - esac - fi - ;; - - *.lo) - # Possibly a libtool object, so verify it. - if func_lalib_p "$file"; then - - # Read the .lo file - func_source $dir/$name - - # Add PIC object to the list of files to remove. - if test -n "$pic_object" && test none != "$pic_object"; then - func_append rmfiles " $dir/$pic_object" - fi - - # Add non-PIC object to the list of files to remove. - if test -n "$non_pic_object" && test none != "$non_pic_object"; then - func_append rmfiles " $dir/$non_pic_object" - fi - fi - ;; - - *) - if test clean = "$opt_mode"; then - noexename=$name - case $file in - *.exe) - func_stripname '' '.exe' "$file" - file=$func_stripname_result - func_stripname '' '.exe' "$name" - noexename=$func_stripname_result - # $file with .exe has already been added to rmfiles, - # add $file without .exe - func_append rmfiles " $file" - ;; - esac - # Do a test to see if this is a libtool program. - if func_ltwrapper_p "$file"; then - if func_ltwrapper_executable_p "$file"; then - func_ltwrapper_scriptname "$file" - relink_command= - func_source $func_ltwrapper_scriptname_result - func_append rmfiles " $func_ltwrapper_scriptname_result" - else - relink_command= - func_source $dir/$noexename - fi - - # note $name still contains .exe if it was in $file originally - # as does the version of $file that was added into $rmfiles - func_append rmfiles " $odir/$name $odir/${name}S.$objext" - if test yes = "$fast_install" && test -n "$relink_command"; then - func_append rmfiles " $odir/lt-$name" - fi - if test "X$noexename" != "X$name"; then - func_append rmfiles " $odir/lt-$noexename.c" - fi - fi - fi - ;; - esac - func_show_eval "$RM $rmfiles" 'exit_status=1' - done - - # Try to remove the $objdir's in the directories where we deleted files - for dir in $rmdirs; do - if test -d "$dir"; then - func_show_eval "rmdir $dir >/dev/null 2>&1" - fi - done - - exit $exit_status -} - -if test uninstall = "$opt_mode" || test clean = "$opt_mode"; then - func_mode_uninstall ${1+"$@"} -fi - -test -z "$opt_mode" && { - help=$generic_help - func_fatal_help "you must specify a MODE" -} - -test -z "$exec_cmd" && \ - func_fatal_help "invalid operation mode '$opt_mode'" - -if test -n "$exec_cmd"; then - eval exec "$exec_cmd" - exit $EXIT_FAILURE -fi - -exit $exit_status - - -# The TAGs below are defined such that we never get into a situation -# where we disable both kinds of libraries. Given conflicting -# choices, we go for a static library, that is the most portable, -# since we can't tell whether shared libraries were disabled because -# the user asked for that or because the platform doesn't support -# them. This is particularly important on AIX, because we don't -# support having both static and shared libraries enabled at the same -# time on that platform, so we default to a shared-only configuration. -# If a disable-shared tag is given, we'll fallback to a static-only -# configuration. But we'll never go from static-only to shared-only. - -# ### BEGIN LIBTOOL TAG CONFIG: disable-shared -build_libtool_libs=no -build_old_libs=yes -# ### END LIBTOOL TAG CONFIG: disable-shared - -# ### BEGIN LIBTOOL TAG CONFIG: disable-static -build_old_libs=`case $build_libtool_libs in yes) echo no;; *) echo yes;; esac` -# ### END LIBTOOL TAG CONFIG: disable-static - -# Local Variables: -# mode:shell-script -# sh-indentation:2 -# End: diff --git a/main.mk b/main.mk index 628f3d1621..c6bdeafaa3 100644 --- a/main.mk +++ b/main.mk @@ -1,89 +1,602 @@ - +#!/do/not/make +# ^^^^ help out editors which guess this file's type. ############################################################################### -# The following macros should be defined before this script is -# invoked: +# This is the main makefile for sqlite. It expects to be included from +# a higher-level makefile which configures any dynamic state needed by +# this one (as documented below). +# +# Maintenance reminders: +# +# - This file must remain devoid of GNU Make-isms. i.e. it must be +# POSIX Make compatible. "bmake" (BSD make) is available on most +# Linux systems, so compatibility is relatively easy to test. As a +# harmless exception, this file sometimes uses $(MAKEFILE_LIST) as a +# dependency. That var, in GNU Make, holds a list of all of the +# makefiles currently loaded. +# +# The variables listed below must be defined before this script is +# invoked. This file will use defaults, very possibly invalid, for any +# which are not defined. +######################################################################## +all: +# +# $(TOP) = +# +# The top-level directory of the source tree. For canonical builds +# this is the directory that contains this "Makefile.in" and the +# "auto.def" script. For out-of-tree builds, this will differ from +# $(PWD). +# +TOP ?= $(PWD) +# +# $(PACKAGE_VERSION) = +# +# The MAJOR.MINOR.PATCH version number of this build. +# +PACKAGE_VERSION ?= +# +# $(B.cc) = +# +# C Compiler and options for use in building executables that will run +# on the platform that is doing the build. +# +B.cc ?= $(CC) +# +# $(T.cc) = +# +# C Compiler and options for use in building executables that will run +# on the target platform. This is usually the same as B.cc, unless you +# are cross-compiling. Note that it should only contain flags which +# are used by _all_ build targets. Flags needed only by specific +# targets are defined elsewhere and applied on a per-target basis. +# +T.cc ?= $(B.cc) +# +# $(AR) = +# +# Tool used to build a static library from object files, without its +# arguments. $(AR.flags) are its flags for creating a lib. +# +AR ?= ar +AR.flags ?= cr +# +# $(B.exe) = +# +# File extension for executables on the build platform:. .exe for +# Windows and empty everywhere else. +# +B.exe ?= +# +# $(B.dll) and $(B.lib) = +# +# The DLL resp. static library counterparts of $(B.exe). +# +B.dll ?= .so +B.lib ?= .a +# +# $(T.exe) = +# +# File extension for executables on the target platform: .exe for +# Windows and empty everywhere else. +# +T.exe ?= $(B.exe) +# +# $(T.dll) and $(T.lib) = +# +# The DLL resp. static library counterparts of $(T.exe). +# +T.dll ?= $(B.dll) +T.lib ?= $(B.lib) +# +# HAVE_TCL = 1 to enable full tcl support, else 0. +# +HAVE_TCL ?= 0 +# +# $(TCLSH_CMD) = +# +# The canonical tclsh. +# +TCLSH_CMD ?= tclsh +# +# JimTCL is part of the autosetup suite and is suitable for all +# current in-tree code-generation TCL jobs, but it requires that we +# build it with non-default flags. The canonical build tree will, if +# no system-level tclsh is found, also have a ./jimsh0 binary. That +# one is a bare-bones build for the configure process, whereas we need +# to build it with another option enabled for use with the various +# code generators. +# +# JIMSH requires a leading path component, even if it's ./, so that it +# can be used as a shell command. +# +# On Windows platforms, if -DHAVE_REALPATH does not work then try +# -DHAVE__FULLPATH (note the double-underscore). +# +CFLAGS.jimsh ?= -DHAVE_REALPATH +JIMSH ?= ./jimsh$(T.exe) +# +# $(B.tclsh) = +# +# The TCL interpreter for in-tree code generation. May be either the +# in-tree JimTCL ($(JIMSH)) or the canonical TCL ($(TCLSH_CMD)). If +# it's JimTCL, it must be compiled with -DHAVE_REALPATH (Unix) or +# -DHAVE__FULLPATH (Windows) so that the Tcl function [file normalize] +# can work. +# +B.tclsh ?= $(JIMSH) + +# +# Autotools-conventional vars which are (in this tree) used only by +# package installation rules and for generating sqlite3.pc (pkg-config +# data file). +# +# The following ${XYZdir} vars are provided for the sake of clients +# who expect to be able to override these using autotools-conventional +# dir name vars. +# +prefix ?= /usr/local +datadir ?= $(prefix)/share +mandir ?= $(datadir)/man +includedir ?= $(prefix)/include +exec_prefix ?= $(prefix) +bindir ?= $(exec_prefix)/bin +libdir ?= $(exec_prefix)/lib +# This makefile does not use any of: +# sbindir ?= $(exec_prefix)/sbin +# sysconfdir ?= /etc +# sharedstatedir ?= $(prefix)/com +# localstatedir ?= /var +# runstatedir ?= /run +# infodir ?= $(datadir)/info +# libexecdir ?= $(exec_prefix)/libexec +### end of autotools-compatible install dir vars + + +# +# $(LDFLAGS.{feature}) and $(CFLAGS.{feature}) = +# +# Linker resp. C/CPP flags required by a specific feature, e.g. +# $(LDFLAGS.pthread) or $(CFLAGS.readline). +# +# Rather that stuffing all CFLAGS and LDFLAGS into a single set, we +# break them down on a per-feature basis and expect the build targets +# to use the one(s) it needs. +# +LDFLAGS.zlib ?= -lz +LDFLAGS.math ?= -lm +LDFLAGS.rpath ?= -Wl,-rpath -Wl,$(prefix)/lib +LDFLAGS.pthread ?= -lpthread +LDFLAGS.dlopen ?= -ldl +LDFLAGS.shlib ?= -shared +LDFLAGS.rt ?= # nanosleep on some platforms +LDFLAGS.icu ?= # -licui18n -licuuc -licudata +CFLAGS.icu ?= +LDFLAGS.libsqlite3.soname ?= # see https://sqlite.org/src/forumpost/5a3b44f510df8ded +LDFLAGS.libsqlite3.os-specific ?= # see https://sqlite.org/forum/forumpost/9dfd5b8fd525a5d7 +# libreadline (or a workalike): +# To activate readline in the shell: SHELL_OPT = -DHAVE_READLINE=1 +LDFLAGS.readline ?= -lreadline # these vary across platforms +CFLAGS.readline ?= -I$(prefix)/include +# ^^^ When using linenoise instead of readline, do something like: +# SHELL_OPT += -DHAVE_LINENOISE=1 +# CFLAGS.readline = -I$(HOME)/linenoise $(HOME)/linenoise/linenoise.c +# LDFLAGS.readline = # empty + +# +# +# $(INSTALL) = +# +# Tool for installing files and directories. It must be compatible +# with conventional Unix /usr/bin/install. Note that libtool's +# install-sh is _not_ compatible with this because it _moves_ targets +# during installation, which may break the build of targets which are +# built after others are installed. +# +# Maintenance reminder: we specifically do not strip binaries, as +# discussed in https://sqlite.org/forum/forumpost/9a67df63eda9925c. +# +INSTALL ?= install +# +# $(ENABLE_LIB_SHARED) = +# +# 1 if libsqlite3$(T.dll) should be built. +# +ENABLE_LIB_SHARED ?= 1 +# +# $(ENABLE_LIB_STATIC) = +# +# 1 if libsqlite3$(T.lib) should be built. Some components, +# e.g. libtclsqlite3 and some test apps, implicitly require the static +# library and will ignore this preference. # -# TOP The toplevel directory of the source tree. This is the -# directory that contains this "Makefile.in" and the -# "configure.in" script. +ENABLE_LIB_STATIC ?= 1 # -# BCC C Compiler and options for use in building executables that -# will run on the platform that is doing the build. +# $(USE_AMALGAMATION) # -# THREADLIB Specify any extra linker options needed to make the library -# thread safe +# 1 if the amalgamation (sqlite3.c/h) should be built/used, otherwise +# the library is built from all of its original source files. +# Certain tools, like sqlite3$(T.exe), require the amalgamation and +# will ignore this preference. # -# LIBS Extra libraries options +USE_AMALGAMATION ?= 1 # -# OPTS Extra compiler command-line options. +# $(LINK_TOOLS_DYNAMICALLY) # -# EXE The suffix to add to executable files. ".exe" for windows -# and "" for Unix. +# If 1, certain binaries which typically statically link against +# libsqlite3 or its component object files will instead link against +# the DLL. The caveat is that running such builds from the source tree +# may require that the user specifically prepend "." to their +# $LD_LIBRARY_PATH so that the dynamic linker does not pick up a +# libsqlite3.so from outside the source tree. Alternately, symlinking +# the in-build-tree $(libsqlite3.DLL) to some dir in the system's +# library path will work for giving the apps access to the in-tree +# DLL. # -# TCC C Compiler and options for use in building executables that -# will run on the target platform. This is usually the same -# as BCC, unless you are cross-compiling. +LINK_TOOLS_DYNAMICALLY ?= 0 # -# AR Tools used to build a static library. -# RANLIB +# $(AMALGAMATION_GEN_FLAGS) = # -# TCL_FLAGS Extra compiler options needed for programs that use the -# TCL library. +# Optional flags for the amalgamation generator. # -# LIBTCL Linker options needed to link against the TCL library. +AMALGAMATION_GEN_FLAGS ?= --linemacros=0 # -# READLINE_FLAGS Compiler options needed for programs that use the -# readline() library. +# EXTRA_SRC = list of C files to append as-is to the generated +# amalgamation. It should arguably be called AMALGAMATION_EXTRA_SRC +# but this older name is already in use by clients. # -# LIBREADLINE Linker options needed by programs using readline() must -# link against. +EXTRA_SRC ?= + +# +# $(OPT_FEATURE_FLAGS) = +# +# Preprocessor flags for enabling and disabling specific libsqlite3 +# features (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). The same set of OMIT +# and ENABLE flags must be passed to the LEMON parser generator and +# the mkkeywordhash tool as well. This is normally set by the +# configure process, and passing a custom value to a +# configure-filtered Makefile may not work. +# +# When using the canonical makefile, add $(OPTIONS)=... on the make +# command line to append additional options to the +# $(OPT_FEATURE_FLAGS). Some flags, because they influence generation +# of the SQL parser, only work if the build is specifically configured +# to account for them. Adding them later, when compiling the +# amalgamation separately, may or may not work. +# +# $(OPTS)=... is another way of influencing C compilation. It is +# distinctly separate from $(OPTIONS) and $(OPT_FEATURE_FLAGS) but, +# like those, $(OPTS) applies to all invocations of $(T.cc) (and some +# invocations of $(B.cc)). The configure process does not set either +# of $(OPTIONS) or $(OPTS). +# +OPT_FEATURE_FLAGS ?= +# +# $(SHELL_OPT) = +# +# CFLAGS specific to the sqlite3 CLI shell app and its close cousins. # -# Once the macros above are defined, the rest of this make script will -# build the SQLite library and testing tools. +SHELL_OPT ?= +# +# TCL_CONFIG_SH must, for some of the build targets, refer to a valid +# tclConfig.sh. That script will be used to populate most of the other +# TCL-related vars the build needs. The core library does not require +# TCL, but TCL is needed for running tests and certain tools, e.g. +# sqlite3_analyzer. +# +TCL_CONFIG_SH ?= +# +# $(HAVE_WASI_SDK) = +# +# Set to 1 when building with the WASI SDK. This disables certain +# build targets. It is expected that the invoker sets $(CC), $(LD), +# and $(AR) to their counterparts from the wasi-sdk. +# +HAVE_WASI_SDK ?= 0 +# +# ... and many, many more. Sane defaults are selected where possible. +# +# With the above-described defined, the rest of this make script will +# build the project's deliverables and testing tools. ################################################################################ +all: sqlite3.h sqlite3.c + +######################################################################## +######################################################################## +# Modifying anything after this point should not be necessary for most +# builds. +######################################################################## +######################################################################## + +# +# $(CFLAGS.env) holds the any $(CFLAGS) provided at configure- or +# make-time (the latter overriding the former). +# +# $(CFLAGS) should ideally only contain flags which are relevant for +# all binaries built for the target platform. However, many people +# like to pass it to "make" without realizing that it applies to +# dozens of deliverables, and they override core flags (like -fPIC) +# when doing so. To help work around that, we expect all core-most +# CFLAGS, e.g. -fPIC, to be set in $(CFLAGS.core). That enables people +# to pass their other CFLAGS without triggering, e.g., "recompile with +# -fPIC" errors. +# +# Historical note: the pre-3.48 build does not honor CPPFLAGS passed +# to make, so we do not do so here. Both the legacy and 3.48+ builds +# support CPPFLAGS passed at configure-time, and combines them with +# the configure-time CFLAGS. +# +CFLAGS.core ?= +CFLAGS.env = $(CFLAGS) +T.cc += $(CFLAGS.core) $(CFLAGS.env) + +# +# $(LDFLAGS.configure) represents any LDFLAGS=... the client passes to +# the configure process. The historical build enabled passing-on of +# user-provided LDFLAGS at configure-time but not make-time. That +# behavior is not possible to fully emulate here because this makefile +# is not filtered by the configure script, so we instead +# "soft-enforce" it by using a level of indirection, which clients who +# read this can (but are not advised to!) bypass by passing +# LDFLAGS.configure=... to this makefile. (We do not guaranty this +# variable name to be stable, so do not rely on that capability!) +# +# A significant difference from the legacy build: +# +# The legacy build applied such LDFLAGS to all link operations for all +# deliverables. The 3.48+ build applies them (as of this writing) more +# selectively: search this file LDFLAGS.configure to see where they're +# set. +# +LDFLAGS.configure ?= + +# +# The difference between $(OPT_FEATURE_FLAGS) and $(OPTS) is that the +# former is historically provided by the configure script, whereas +# $(OPTS) is intended to be provided as arguments to the make +# invocation. +# +T.cc += $(OPT_FEATURE_FLAGS) + +# +# Add in any optional global compilation flags on the make command +# line i.e. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". +# +T.cc += $(OPTS) + +# +# $(INSTALL) invocation for use with non-executable files. +# +INSTALL.noexec = $(INSTALL) -m 0644 +# ^^^ do not use GNU-specific flags to $(INSTALL), e.g. --mode=... + +# +# T.compile.gcov = gcov-specific compilation flags for the target +# platform. +# +T.compile.gcov ?= +# +# T.link.gcov = gcov-specific link flags for the target platform. +# +T.link.gcov ?= + +# +# $(T.compile) = generic target platform compiler invocation, +# differing only from $(T.cc) in that it appends $(T.compile.gcov), +# which is intended for use with gcov-related flags. +# +T.compile = $(T.cc) $(T.compile.gcov) + +# +# Optionally set by the configure script to include -DSQLITE_DEBUG=1 +# and other debug-related flags. +# +T.cc.TARGET_DEBUG ?= + +# +# Extra CFLAGS for both the core sqlite3 components and extensions. +# +# Define -D_HAVE_SQLITE_CONFIG_H so that the code knows it +# can include the generated sqlite_cfg.h. +# +T.cc.sqlite.extras = -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite $(T.cc.TARGET_DEBUG) + +# +# $(T.cc.sqlite) is $(T.cc) plus any flags which are desired for the +# library as a whole, but not necessarily needed for every binary. It +# will normally get initially populated with flags by the +# configure-generated makefile. +# +T.cc.sqlite ?= $(T.compile) $(T.cc.sqlite.extras) + +# +# $(CFLAGS.intree_includes) = -I... flags relevant specifically to +# this tree, including any subdirectories commonly needed for building +# various tools. +# +CFLAGS.intree_includes = \ + -I. -I$(TOP)/src -I$(TOP)/ext/rtree -I$(TOP)/ext/icu \ + -I$(TOP)/ext/fts3 -I$(TOP)/ext/session \ + -I$(TOP)/ext/misc +T.cc.sqlite += $(CFLAGS.intree_includes) + +# +# $(T.cc.extension) = compiler invocation for loadable extensions. +# +T.cc.extension = $(T.compile) -I. -I$(TOP)/src $(T.cc.sqlite.extras) -DSQLITE_CORE + +# +# $(T.link) = compiler invocation for when the target will be an +# executable. +# +# $(T.link.gcov) = optional config-specific flags for $(T.link), +# intended for use with gcov-related flags. +# +T.link = $(T.cc.sqlite) $(T.link.gcov) +# +# $(T.link.shared) = $(T.link) invocation specifically for shared libraries +# +T.link.shared = $(T.link) $(LDFLAGS.shlib) + +# +# $(LDFLAGS.libsqlite3) should be used with any deliverable for which +# any of the following apply: +# +# - Results in building libsqlite3.so +# - Compiles sqlite3.c in to an application +# - Links with libsqlite3.a +# - Links in either of $(LIBOBJSO) or $(LIBOBJS1) +# +# Note that these flags are for the target build platform, not +# necessarily localhost. i.e. it should be used with $(T.cc.sqlite) +# or $(T.link) but not $(B.cc). +# +LDFLAGS.libsqlite3 = \ + $(LDFLAGS.rpath) $(LDFLAGS.pthread) \ + $(LDFLAGS.math) $(LDFLAGS.dlopen) \ + $(LDFLAGS.zlib) $(LDFLAGS.icu) \ + $(LDFLAGS.rt) $(LDFLAGS.configure) + +# +# $(install-dir.XYZ) = dirs for installation. +# +# Design note: these should arguably all be defined with surrounding +# double-quotes so that targets which have spaces in their paths will +# work, but that leads to Make treating the quotes as part of the dir +# name, which in turn leads to it never finding a matching name in the +# filesystem and always invoking ($(INSTALL) -d ...) for them. The +# moral of this story is that spaces in installation paths will break +# the install process. +# +install-dir.bin = $(DESTDIR)$(bindir) +install-dir.lib = $(DESTDIR)$(libdir) +install-dir.include = $(DESTDIR)$(includedir) +install-dir.pkgconfig = $(DESTDIR)$(libdir)/pkgconfig +install-dir.man1 = $(DESTDIR)$(mandir)/man1 +install-dir.all = $(install-dir.bin) $(install-dir.include) \ + $(install-dir.lib) $(install-dir.man1) \ + $(install-dir.pkgconfig) +$(install-dir.all): + @if [ ! -d "$@" ]; then set -x; $(INSTALL) -d "$@"; fi +# ^^^^ on some platforms, install -d fails if the target already exists. + +# +# After jimsh is compiled, we run some sanity checks to ensure that +# it was built in a way compatible with this project's scripts: +# +# 1) Ensure that it was built with realpath() or _fullpath() support. +# Without that flag the [file normalize] command will always resolve +# to an empty string. +# +# 2) Ensure that it is built with -DJIM_COMPAT (which may be +# hard-coded into jimsh0.c). Without this, the [expr] command accepts +# only a single argument. (That said: the real fix for that is to +# update any scripts which still pass multiple arguments to [expr].) +# +$(JIMSH): $(TOP)/autosetup/jimsh0.c + $(B.cc) -o $@ $(CFLAGS.jimsh) $(TOP)/autosetup/jimsh0.c + @if [ x = "x$$($(JIMSH) -e 'file normalize $(JIMSH)' 2>/dev/null)" ]; then \ + echo "$(JIMSH) was built without -DHAVE_REALPATH or -DHAVE__FULLPATH." 1>&2; \ + exit 1; \ + fi + @if [ x3 != "x$$($(JIMSH) -e 'expr 1 + 2' 2>/dev/null)" ]; then \ + echo "$(JIMSH) was built without -DJIM_COMPAT." 1>&2; \ + exit 1; \ + fi +distclean-jimsh: + rm -f $(JIMSH) +distclean: distclean-jimsh -# This is how we compile # -TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) -TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 -TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth -TCCX += -I$(TOP)/ext/session -TCCX += -I$(TOP)/ext/fts5 -THREADLIB += $(LIBS) +# $(MAKE_SANITY_CHECK) = a set of checks for various make vars which +# must be provided to this file before including it. If any are +# missing, this target fails. It does (almost) no semantic validation, +# only checks to see that appropriate vars are not empty. +# +# Note that $(MAKEFILE_LIST) is a GNU-make-ism but its use is harmless +# in other flavors of Make. +# +MAKE_SANITY_CHECK = .main.mk.checks +$(MAKE_SANITY_CHECK): $(MAKEFILE_LIST) $(TOP)/auto.def + @if [ x = "x$(TOP)" ]; then echo "Missing TOP var" 1>&2; exit 1; fi + @if [ ! -d "$(TOP)" ]; then echo "$(TOP) is not a directory" 1>&2; exit 1; fi + @if [ ! -f "$(TOP)/auto.def" ]; then echo "$(TOP) does not appear to be the top-most source dir" 1>&2; exit 1; fi + @if [ x = "x$(PACKAGE_VERSION)" ]; then echo "PACKAGE_VERSION must be set to the library's X.Y.Z-format version number" 1>&2; exit 1; fi + @if [ x = "x$(B.cc)" ]; then echo "Missing B.cc var" 1>&2; exit 1; fi + @if [ x = "x$(T.cc)" ]; then echo "Missing T.cc var" 1>&2; exit 1; fi + @if [ x = "x$(B.tclsh)" ]; then echo "Missing B.tclsh var" 1>&2; exit 1; fi + @if [ x = "x$(AR)" ]; then echo "Missing AR var" 1>&2; exit 1; fi + touch $@ +clean-sanity-check: + rm -f $(MAKE_SANITY_CHECK) +clean: clean-sanity-check + +# BEGIN SQLCIPHER +SQLCIPHER_OBJ = \ + sqlcipher.o \ + crypto_openssl.o \ + crypto_cc.o + +SQLCIPHER_SRC = \ + $(TOP)/src/sqlcipher.h \ + $(TOP)/src/sqlcipher.c \ + $(TOP)/src/crypto_openssl.c \ + $(TOP)/src/crypto_cc.c + +sqlcipher.o: $(TOP)/src/sqlcipher.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/sqlcipher.c +crypto_openssl.o: $(TOP)/src/crypto_openssl.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_openssl.c +crypto_cc.o: $(TOP)/src/crypto_cc.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_cc.c + +# END SQLCIPHER -# Object files for the SQLite library. # -LIBOBJ+= vdbe.o parse.o \ - alter.o analyze.o attach.o auth.o \ +# Object files for the SQLite library (non-amalgamation). +# +LIBOBJS0 = alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ - callback.o complete.o ctime.o \ - date.o dbpage.o dbstat.o delete.o expr.o \ - fault.o fkey.o \ - fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ - fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ + callback.o carray.o complete.o ctime.o \ + date.o dbpage.o dbstat.o delete.o \ + expr.o fault.o fkey.o \ + fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o \ + fts3_porter.o fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ fts3_tokenize_vtab.o \ - fts3_unicode.o fts3_unicode2.o \ - fts3_write.o fts5.o func.o global.o hash.o \ + fts3_unicode.o fts3_unicode2.o fts3_write.o \ + fts5.o \ + func.o global.o hash.o \ icu.o insert.o json.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memdb.o memjournal.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_kv.o os_unix.o os_win.o \ - pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \ + pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o \ - select.o sqlite3rbu.o status.o stmt.o \ + sqlite3session.o select.o sqlite3rbu.o status.o stmt.o \ table.o threads.o tokenize.o treeview.o trigger.o \ - update.o upsert.o userauth.o util.o vacuum.o \ - vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \ - vdbetrace.o vdbevtab.o \ + update.o upsert.o utf.o util.o vacuum.o \ + vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \ + vdbetrace.o vdbevtab.o vtab.o \ wal.o walker.o where.o wherecode.o whereexpr.o \ - utf.o vtab.o window.o + window.o $(SQLCIPHER_OBJ) +LIBOBJS = $(LIBOBJS0) + +# +# Object files for the amalgamation. +# +LIBOBJS1 = sqlite3.o -LIBOBJ += sqlite3session.o +# +# Determine the real value of LIBOBJ based on whether the amalgamation +# is enabled or not. +# +LIBOBJ = $(LIBOBJS$(USE_AMALGAMATION)) +$(LIBOBJ): $(MAKE_SANITY_CHECK) +# # All of the source code files. # -SRC = \ +SRC = $(SQLCIPHER_SRC) \ $(TOP)/src/alter.c \ $(TOP)/src/analyze.c \ $(TOP)/src/attach.c \ @@ -96,8 +609,9 @@ SRC = \ $(TOP)/src/btreeInt.h \ $(TOP)/src/build.c \ $(TOP)/src/callback.c \ + $(TOP)/src/carray.c \ $(TOP)/src/complete.c \ - $(TOP)/src/ctime.c \ + ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ @@ -145,7 +659,7 @@ SRC = \ $(TOP)/src/pcache.h \ $(TOP)/src/pcache1.c \ $(TOP)/src/pragma.c \ - $(TOP)/src/pragma.h \ + pragma.h \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ @@ -214,7 +728,6 @@ SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ - $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c \ $(TOP)/ext/rtree/geopoly.c @@ -222,56 +735,11 @@ SRC += \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/sqlite3session.h SRC += \ - $(TOP)/ext/userauth/userauth.c \ - $(TOP)/ext/userauth/sqlite3userauth.h -SRC += \ - $(TOP)/ext/rbu/sqlite3rbu.c \ - $(TOP)/ext/rbu/sqlite3rbu.h + $(TOP)/ext/rbu/sqlite3rbu.h \ + $(TOP)/ext/rbu/sqlite3rbu.c SRC += \ $(TOP)/ext/misc/stmt.c - -# FTS5 things -# -FTS5_HDR = \ - $(TOP)/ext/fts5/fts5.h \ - $(TOP)/ext/fts5/fts5Int.h \ - fts5parse.h - -FTS5_SRC = \ - $(TOP)/ext/fts5/fts5_aux.c \ - $(TOP)/ext/fts5/fts5_buffer.c \ - $(TOP)/ext/fts5/fts5_main.c \ - $(TOP)/ext/fts5/fts5_config.c \ - $(TOP)/ext/fts5/fts5_expr.c \ - $(TOP)/ext/fts5/fts5_hash.c \ - $(TOP)/ext/fts5/fts5_index.c \ - fts5parse.c \ - $(TOP)/ext/fts5/fts5_storage.c \ - $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c \ - $(TOP)/ext/fts5/fts5_varint.c \ - $(TOP)/ext/fts5/fts5_vocab.c \ - -LSM1_SRC = \ - $(TOP)/ext/lsm1/lsm.h \ - $(TOP)/ext/lsm1/lsmInt.h \ - $(TOP)/ext/lsm1/lsm_ckpt.c \ - $(TOP)/ext/lsm1/lsm_file.c \ - $(TOP)/ext/lsm1/lsm_log.c \ - $(TOP)/ext/lsm1/lsm_main.c \ - $(TOP)/ext/lsm1/lsm_mem.c \ - $(TOP)/ext/lsm1/lsm_mutex.c \ - $(TOP)/ext/lsm1/lsm_shared.c \ - $(TOP)/ext/lsm1/lsm_sorted.c \ - $(TOP)/ext/lsm1/lsm_str.c \ - $(TOP)/ext/lsm1/lsm_tree.c \ - $(TOP)/ext/lsm1/lsm_unix.c \ - $(TOP)/ext/lsm1/lsm_varint.c \ - $(TOP)/ext/lsm1/lsm_vtab.c \ - $(TOP)/ext/lsm1/lsm_win32.c - - # Generated source code files # SRC += \ @@ -280,18 +748,13 @@ SRC += \ opcodes.h \ parse.c \ parse.h \ + sqlite_cfg.h \ shell.c \ sqlite3.h - # Source code to the test files. # TESTSRC = \ - $(TOP)/ext/expert/sqlite3expert.c \ - $(TOP)/ext/expert/test_expert.c \ - $(TOP)/ext/fts3/fts3_term.c \ - $(TOP)/ext/fts3/fts3_test.c \ - $(TOP)/ext/rbu/test_rbu.c \ $(TOP)/src/test1.c \ $(TOP)/src/test2.c \ $(TOP)/src/test3.c \ @@ -301,7 +764,6 @@ TESTSRC = \ $(TOP)/src/test8.c \ $(TOP)/src/test9.c \ $(TOP)/src/test_autoext.c \ - $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ $(TOP)/src/test_bestindex.c \ $(TOP)/src/test_blob.c \ @@ -326,7 +788,6 @@ TESTSRC = \ $(TOP)/src/test_quota.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ - $(TOP)/src/test_sqllog.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ $(TOP)/src/test_tclsh.c \ @@ -334,17 +795,26 @@ TESTSRC = \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vdbecov.c \ $(TOP)/src/test_vfs.c \ - $(TOP)/src/test_windirent.c \ $(TOP)/src/test_window.c \ - $(TOP)/src/test_wsd.c + $(TOP)/src/test_wsd.c \ + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c \ + $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/test_recover.c \ + $(TOP)/ext/intck/test_intck.c \ + $(TOP)/ext/intck/sqlite3intck.c \ + $(TOP)/ext/rbu/test_rbu.c -# Extensions to be statically loaded. +# Statically linked extensions # TESTSRC += \ + $(TOP)/ext/expert/sqlite3expert.c \ + $(TOP)/ext/expert/test_expert.c \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/appendvfs.c \ $(TOP)/ext/misc/basexx.c \ - $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/cksumvfs.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/csv.c \ @@ -353,37 +823,37 @@ TESTSRC += \ $(TOP)/ext/misc/explain.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/fts5/fts5_tcl.c \ + $(TOP)/ext/fts5/fts5_test_mi.c \ + $(TOP)/ext/fts5/fts5_test_tok.c \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/mmapwarm.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/normalize.c \ - $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/prefixes.c \ $(TOP)/ext/misc/qpvtab.c \ + $(TOP)/ext/misc/randomjson.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ + $(TOP)/ext/misc/stmtrand.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/unionvtab.c \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/zipfile.c \ - $(TOP)/ext/fts5/fts5_tcl.c \ - $(TOP)/ext/fts5/fts5_test_mi.c \ - $(TOP)/ext/fts5/fts5_test_tok.c \ - $(TOP)/ext/rtree/test_rtreedoc.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/test_recover.c - - -#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c + $(TOP)/ext/rtree/test_rtreedoc.c +# Source code to the library files needed by the test fixture +# TESTSRC2 = \ $(TOP)/src/attach.c \ $(TOP)/src/backup.c \ + $(TOP)/src/bitvec.c \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ + $(TOP)/src/carray.c \ + ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ @@ -406,7 +876,6 @@ TESTSRC2 = \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ - $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/treeview.c \ $(TOP)/src/utf.c \ @@ -415,20 +884,20 @@ TESTSRC2 = \ $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ + $(TOP)/src/vdbetrace.c \ $(TOP)/src/vdbevtab.c \ $(TOP)/src/where.c \ $(TOP)/src/wherecode.c \ $(TOP)/src/whereexpr.c \ + $(TOP)/src/window.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c \ - $(TOP)/ext/misc/stmt.c \ $(TOP)/ext/session/sqlite3session.c \ - $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/misc/stmt.c \ fts5.c # Header files used by all library source files. @@ -449,7 +918,7 @@ HDR = \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ - $(TOP)/src/pragma.h \ + pragma.h \ sqlite3.h \ $(TOP)/src/sqlite3ext.h \ $(TOP)/src/sqliteInt.h \ @@ -457,7 +926,9 @@ HDR = \ $(TOP)/src/vdbe.h \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vxworks.h \ - $(TOP)/src/whereInt.h + $(TOP)/src/whereInt.h \ + sqlite_cfg.h +# Reminder: sqlite_cfg.h is typically created by the configure script # Header files used by extensions # @@ -472,22 +943,18 @@ EXTHDR += \ EXTHDR += \ $(TOP)/ext/icu/sqliteicu.h EXTHDR += \ - $(TOP)/ext/fts5/fts5Int.h \ - fts5parse.h \ - $(TOP)/ext/fts5/fts5.h -EXTHDR += \ - $(TOP)/ext/userauth/sqlite3userauth.h + $(TOP)/ext/rtree/sqlite3rtree.h -# executables needed for testing +# +# Executables needed for testing # TESTPROGS = \ - testfixture$(EXE) \ - sqlite3$(EXE) \ - sqlite3_analyzer$(EXE) \ - sqlite3_checker$(EXE) \ - sqldiff$(EXE) \ - dbhash$(EXE) \ - sqltclsh$(EXE) + testfixture$(T.exe) \ + sqlite3$(T.exe) \ + sqlite3_analyzer$(T.exe) \ + sqldiff$(T.exe) \ + dbhash$(T.exe) \ + sqltclsh$(T.exe) # Databases containing fuzzer test cases # @@ -501,13 +968,20 @@ FUZZDATA = \ $(TOP)/test/fuzzdata7.db \ $(TOP)/test/fuzzdata8.db +# # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt +# # Extra compiler options for various shell tools # -SHELL_OPT += -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +# Note that some of these will only apply when embedding sqlite3.c +# into the shell, as these flags are not otherwise passed on to the +# library. +SHELL_OPT += -DSQLITE_DQS=0 +SHELL_OPT += -DSQLITE_ENABLE_FTS4 +#SHELL_OPT += -DSQLITE_ENABLE_FTS5 SHELL_OPT += -DSQLITE_ENABLE_RTREE SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION @@ -516,161 +990,200 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 +FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover -FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 -FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 -FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE -FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY -FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -FUZZSRC += $(TOP)/test/fuzzcheck.c -FUZZSRC += $(TOP)/test/ossfuzz.c -FUZZSRC += $(TOP)/test/vt02.c -FUZZSRC += $(TOP)/test/fuzzinvariants.c -FUZZSRC += $(TOP)/ext/recover/dbdata.c -FUZZSRC += $(TOP)/ext/recover/sqlite3recover.c -DBFUZZ_OPT = -KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ -ST_OPT = -DSQLITE_THREADSAFE=0 - -# This is the default Makefile target. The objects listed here -# are what get build when you type just "make" with no arguments. -# -all: sqlite3.h sqlite3ext.h libsqlite3.a sqlite3$(EXE) - -libsqlite3.a: sqlite3.h $(LIBOBJ) - $(AR) libsqlite3.a $(LIBOBJ) - $(RANLIB) libsqlite3.a - -sqlite3$(EXE): sqlite3.h libsqlite3.a shell.c - $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \ - shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) - -sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h - $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ - $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) - -dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h - $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ - $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) - -scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o - $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) - -srcck1$(EXE): $(TOP)/tool/srcck1.c - $(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c - -sourcetest: srcck1$(EXE) sqlite3.c - ./srcck1 sqlite3.c - -fuzzershell$(EXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h - $(TCCX) -o fuzzershell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - $(FUZZERSHELL_OPT) $(TOP)/tool/fuzzershell.c sqlite3.c \ - $(TLIBS) $(THREADLIB) - -dbfuzz$(EXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h - $(TCCX) -o dbfuzz$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c \ - $(TLIBS) $(THREADLIB) - -DBFUZZ2_OPTS = \ - -DSQLITE_THREADSAFE=0 \ - -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_DEBUG \ - -DSQLITE_ENABLE_DBSTAT_VTAB \ +FUZZCHECK_OPT += \ + -DSQLITE_OSS_FUZZ \ -DSQLITE_ENABLE_BYTECODE_VTAB \ - -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_CARRAY \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS3_PARENTHESIS \ -DSQLITE_ENABLE_FTS4 \ - -DSQLITE_ENABLE_FTS5 - -dbfuzz2$(EXE): $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h - $(TCCX) -I. -g -O0 -DSTANDALONE -o dbfuzz2$(EXE) \ - $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS) $(THREADLIB) - -fuzzcheck$(EXE): $(FUZZSRC) sqlite3.c sqlite3.h $(FUZZDEP) - $(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) -DSQLITE_OSS_FUZZ \ - $(FUZZSRC) sqlite3.c $(TLIBS) $(THREADLIB) + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_MEMSYS5 \ + -DSQLITE_ENABLE_NORMALIZE \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PERCENTILE \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_STAT4 \ + -DSQLITE_ENABLE_STMT_SCANSTATUS \ + -DSQLITE_JSON_MAX_DEPTH=500 \ + -DSQLITE_MAX_MEMORY=50000000 \ + -DSQLITE_MAX_MMAP_SIZE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ + -DSQLITE_PRIVATE="" \ + -DSQLITE_STRICT_SUBTYPE=1 \ + -DSQLITE_STATIC_RANDOMJSON + +FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c +FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c +FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c +FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c +FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c +FUZZCHECK_SRC += $(TOP)/test/vt02.c +FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c +DBFUZZ_OPT = +ST_OPT = -DSQLITE_OS_KV_OPTIONAL -ossshell$(EXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h - $(TCCX) -o ossshell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) \ - $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c $(TLIBS) $(THREADLIB) +$(TCLSH_CMD): +has_tclsh84: + sh $(TOP)/tool/cktclsh.sh 8.4 $(TCLSH_CMD) + touch has_tclsh84 -sessionfuzz$(EXE): $(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h - $(TCC) -o sessionfuzz$(EXE) $(TOP)/test/sessionfuzz.c -lz $(TLIBS) $(THREADLIB) +has_tclsh85: + sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD) + touch has_tclsh85 -mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c - $(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ - $(TLIBS) $(THREADLIB) +# +# $(T.tcl.env.sh) is a shell script intended for source'ing to set +# various TCL config info in the current shell context: +# +# - All info exported by tclConfig.sh +# +# - TCLLIBDIR = the first entry from TCL's $auto_path which refers to +# an existing dir, then append /sqlite3 to it. If TCLLIBDIR is +# provided via the environment, that value is used instead. +# +# Maintenance reminder: the ./ at the start of the name is required or /bin/sh +# refuses to source it: +# +# . .tclenv.sh ==> .tclenv.sh: not found +# . ./.tclenv.sh ==> fine +# +# It took half an hour to figure that out. +# +T.tcl.env.sh = ./.tclenv.sh +$(T.tcl.env.sh): $(TCLSH_CMD) $(TCL_CONFIG_SH) $(MAKEFILE_LIST) + @if [ x = "x$(TCL_CONFIG_SH)" ]; then \ + echo 'TCL_CONFIG_SH must be set to point to a "tclConfig.sh"' 1>&2; exit 1; \ + fi; \ + if [ x != "x$(TCLLIBDIR)" ]; then \ + echo "# generated by main.mk"; \ + echo TCLLIBDIR="$(TCLLIBDIR)"; \ + else \ + ld= ; \ + for d in `echo "puts stdout \\$$auto_path" | $(TCLSH_CMD)`; do \ + if [ -d "$$d" ]; then ld=$$d; break; fi; \ + done; \ + if [ x = "x$$ld" ]; then echo "Cannot determine TCLLIBDIR" 1>&2; exit 1; fi; \ + echo "# generated by main.mk"; \ + echo "TCLLIBDIR=$$ld/sqlite3$(PACKAGE_VERSION)"; \ + fi > $@; \ + echo ". \"$(TCL_CONFIG_SH)\" || exit \$$?" >> $@; \ + echo "Created $@" -MPTEST1=./mptester$(EXE) mptest1.db $(TOP)/mptest/crash01.test --repeat 20 -MPTEST2=./mptester$(EXE) mptest2.db $(TOP)/mptest/multiwrite01.test --repeat 20 -mptest: mptester$(EXE) - $(MPTEST1) --journalmode DELETE - $(MPTEST2) --journalmode WAL - $(MPTEST1) --journalmode WAL - $(MPTEST2) --journalmode PERSIST - $(MPTEST1) --journalmode PERSIST - $(MPTEST2) --journalmode TRUNCATE - $(MPTEST1) --journalmode TRUNCATE - $(MPTEST2) --journalmode DELETE +# +# $(T.tcl.env.source) is shell code to be run as part of any +# compilation or link step which requires vars from +# $(TCL_CONFIG_SH). All targets which use this should also have a +# dependency on $(T.tcl.env.sh). +# +T.tcl.env.source = . $(T.tcl.env.sh) || exit $$? -sqlite3.o: sqlite3.c - $(TCCX) -I. -c sqlite3.c +# +# $(T.compile.tcl) and $(T.link.tcl) are TCL-specific counterparts for $(T.compile) +# and $(T.link) which first invoke $(T.tcl.env.source). Any targets which used them +# must have a dependency on $(T.tcl.env.sh) +# +T.compile.tcl = $(T.tcl.env.source); $(T.compile) $(CFLAGS.intree_includes) +T.link.tcl = $(T.tcl.env.source); $(T.link) +# # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # -target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c +.target_source: $(MAKE_SANITY_CHECK) $(SRC) $(TOP)/tool/vdbe-compress.tcl \ + fts5.c $(B.tclsh) rm -rf tsrc mkdir tsrc cp -f $(SRC) tsrc - rm tsrc/sqlite.h.in tsrc/parse.y - tclsh $(TOP)/tool/vdbe-compress.tcl $(OPTS) vdbe.new - mv vdbe.new tsrc/vdbe.c + rm -f tsrc/sqlite.h.in tsrc/parse.y + $(B.tclsh) $(TOP)/tool/vdbe-compress.tcl $(OPTS) vdbe.new + mv -f vdbe.new tsrc/vdbe.c cp fts5.c fts5.h tsrc - touch target_source + touch .target_source -sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl - tclsh $(TOP)/tool/mksqlite3c.tcl - cp tsrc/sqlite3ext.h . - cp $(TOP)/ext/session/sqlite3session.h . - echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c - cat sqlite3.c >>tclsqlite3.c - echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c - cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c - -sqlite3ext.h: target_source - cp tsrc/sqlite3ext.h . - -sqlite3.c-debug: target_source $(TOP)/tool/mksqlite3c.tcl - tclsh $(TOP)/tool/mksqlite3c.tcl --linemacros=1 - echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c - cat sqlite3.c >>tclsqlite3.c - echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c - echo '#line 1 "tclsqlite.c"' >>tclsqlite3.c - cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c +# +# libsqlite3.DLL.basename = the base name of the resulting DLL. This +# is typically libsqlite3 but varies wildly on Unix-like Windows +# environments (msys, cygwin, and friends). Conversely, the base name +# of the static library ($(libsqlite3.LIB)) is constant on all tested +# platforms. +# +libsqlite3.DLL.basename ?= libsqlite3 +# +# libsqlite3.DLL => the DLL library +# +libsqlite3.DLL = $(libsqlite3.DLL.basename)$(T.dll) +# +# libsqlite3.out.implib => "import library" file generated by the +# --out-implib linker flag. Not commonly used on Unix systems but is +# on the Windows-side Unix-esque environments and typically as a value +# of "libsqlite3.dll.a". It is expected to match the filename, if any, +# provided by the -Wl,--out-implib,FILENAME flag. +# +libsqlite3.out.implib ?= +# +# libsqlite3.LIB => the static library +# +libsqlite3.LIB = libsqlite3$(T.lib) -sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl - tclsh $(TOP)/tool/split-sqlite3c.tcl +# +# libsqlite3.DLL.install-rules => the suffix of the symoblic name of +# the makefile rules for installing the DLL. Must be one of +# (unix-generic, msys, mingw, cygwin, darwin) and a target named +# install-dll-THATNAME is responsible for DLL installation. +libsqlite3.DLL.install-rules ?= unix-generic # Rules to build the LEMON compiler generator # -lemon: $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c - $(BCC) -o lemon $(TOP)/tool/lemon.c +lemon$(B.exe): $(MAKE_SANITY_CHECK) $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c + $(B.cc) -o $@ $(TOP)/tool/lemon.c cp $(TOP)/tool/lempar.c . -# A tool to generate the source-id +# Rules to build the program that generates the source-id # -mksourceid: $(TOP)/tool/mksourceid.c - $(BCC) -o mksourceid $(TOP)/tool/mksourceid.c +mksourceid$(B.exe): $(MAKE_SANITY_CHECK) $(TOP)/tool/mksourceid.c + $(B.cc) -o $@ $(TOP)/tool/mksourceid.c + +sqlite3.h: $(MAKE_SANITY_CHECK) $(TOP)/src/sqlite.h.in \ + $(TOP)/manifest mksourceid$(B.exe) \ + $(TOP)/VERSION $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mksqlite3h.tcl $(TOP) -o sqlite3.h + +sqlite3.c: .target_source sqlite3.h $(TOP)/tool/mksqlite3c.tcl src-verify$(B.exe) \ + $(B.tclsh) $(EXTRA_SRC) + $(B.tclsh) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_GEN_FLAGS) $(EXTRA_SRC) + cp tsrc/sqlite3ext.h . + cp $(TOP)/ext/session/sqlite3session.h . + +sqlite3r.h: sqlite3.h $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover -o sqlite3r.h + +sqlite3r.c: sqlite3.c sqlite3r.h $(B.tclsh) + cp $(TOP)/ext/recover/sqlite3recover.c tsrc/ + cp $(TOP)/ext/recover/sqlite3recover.h tsrc/ + cp $(TOP)/ext/recover/dbdata.c tsrc/ + $(B.tclsh) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_GEN_FLAGS) $(EXTRA_SRC) + +sqlite3ext.h: .target_source + cp tsrc/sqlite3ext.h . # Rules to build individual *.o files from generated *.c files. This # applies to: @@ -678,290 +1191,968 @@ mksourceid: $(TOP)/tool/mksourceid.c # parse.o # opcodes.o # -%.o: %.c $(HDR) - $(TCCX) -c $< +DEPS_OBJ_COMMON = $(MAKE_SANITY_CHECK) $(HDR) +parse.o: parse.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c parse.c + +opcodes.o: opcodes.c + $(T.cc.sqlite) -c opcodes.c # Rules to build individual *.o files from files in the src directory. # -%.o: $(TOP)/src/%.c $(HDR) - $(TCCX) -c $< +alter.o: $(TOP)/src/alter.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/alter.c -tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) - $(TCCX) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c +analyze.o: $(TOP)/src/analyze.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/analyze.c +attach.o: $(TOP)/src/attach.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/attach.c +auth.o: $(TOP)/src/auth.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/auth.c -# Rules to build opcodes.c and opcodes.h -# -opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl - tclsh $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c +backup.o: $(TOP)/src/backup.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/backup.c -opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl - cat parse.h $(TOP)/src/vdbe.c | \ - tclsh $(TOP)/tool/mkopcodeh.tcl >opcodes.h +bitvec.o: $(TOP)/src/bitvec.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/bitvec.c -# Rules to build parse.c and parse.h - the outputs of lemon. -# -parse.h: parse.c +btmutex.o: $(TOP)/src/btmutex.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/btmutex.c -parse.c: $(TOP)/src/parse.y lemon - cp $(TOP)/src/parse.y . - ./lemon -s $(OPTS) parse.y +btree.o: $(TOP)/src/btree.c $(DEPS_OBJ_COMMON) $(TOP)/src/pager.h + $(T.cc.sqlite) -c $(TOP)/src/btree.c -sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h - tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h +build.o: $(TOP)/src/build.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/build.c -sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION - echo '#ifndef SQLITE_RESOURCE_VERSION' >$@ - echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@ - cat $(TOP)/VERSION | tclsh $(TOP)/tool/replace.tcl exact . , >>$@ - echo '#endif' >>sqlite3rc.h +callback.o: $(TOP)/src/callback.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/callback.c -keywordhash.h: $(TOP)/tool/mkkeywordhash.c - $(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c - ./mkkeywordhash >keywordhash.h - -# Source files that go into making shell.c -SHELL_SRC = \ - $(TOP)/src/shell.c.in \ - $(TOP)/ext/misc/appendvfs.c \ - $(TOP)/ext/misc/completion.c \ - $(TOP)/ext/misc/base64.c \ - $(TOP)/ext/misc/base85.c \ - $(TOP)/ext/misc/decimal.c \ - $(TOP)/ext/misc/fileio.c \ - $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/regexp.c \ - $(TOP)/ext/misc/series.c \ - $(TOP)/ext/misc/shathree.c \ - $(TOP)/ext/misc/sqlar.c \ - $(TOP)/ext/misc/uint.c \ - $(TOP)/ext/expert/sqlite3expert.c \ - $(TOP)/ext/expert/sqlite3expert.h \ - $(TOP)/ext/misc/zipfile.c \ - $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/recover/sqlite3recover.h \ - $(TOP)/src/test_windirent.c - -shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl - tclsh $(TOP)/tool/mkshellc.tcl >shell.c +carray.o: $(TOP)/src/carray.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/carray.c +complete.o: $(TOP)/src/complete.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/complete.c +ctime.c: $(TOP)/tool/mkctimec.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkctimec.tcl -# Rules to build the extension objects. -# -icu.o: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c +ctime.o: ctime.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c ctime.c -fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c +date.o: $(TOP)/src/date.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/date.c -fts3_aux.o: $(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c +dbpage.o: $(TOP)/src/dbpage.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/dbpage.c -fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c +dbstat.o: $(TOP)/src/dbstat.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/dbstat.c -fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c +delete.o: $(TOP)/src/delete.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/delete.c -fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c +expr.o: $(TOP)/src/expr.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/expr.c -fts3_snippet.o: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c +fault.o: $(TOP)/src/fault.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/fault.c -fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c +fkey.o: $(TOP)/src/fkey.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/fkey.c -fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c +func.o: $(TOP)/src/func.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/func.c -fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c +global.o: $(TOP)/src/global.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/global.c -fts3_tokenize_vtab.o: $(TOP)/ext/fts3/fts3_tokenize_vtab.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenize_vtab.c +hash.o: $(TOP)/src/hash.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/hash.c -fts3_unicode.o: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c +insert.o: $(TOP)/src/insert.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/insert.c -fts3_unicode2.o: $(TOP)/ext/fts3/fts3_unicode2.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode2.c +json.o: $(TOP)/src/json.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/json.c -fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c +legacy.o: $(TOP)/src/legacy.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/legacy.c -fts5.o: fts5.c sqlite3ext.h sqlite3.h - $(TCCX) -DSQLITE_CORE -c fts5.c +loadext.o: $(TOP)/src/loadext.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/loadext.c -stmt.o: $(TOP)/ext/misc/stmt.c sqlite3ext.h sqlite3.h - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/misc/stmt.c +main.o: $(TOP)/src/main.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/main.c -rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c +malloc.o: $(TOP)/src/malloc.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/malloc.c +mem0.o: $(TOP)/src/mem0.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mem0.c +mem1.o: $(TOP)/src/mem1.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mem1.c -fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon - cp $(TOP)/ext/fts5/fts5parse.y . - rm -f fts5parse.h - ./lemon $(OPTS) fts5parse.y +mem2.o: $(TOP)/src/mem2.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mem2.c -fts5parse.h: fts5parse.c +mem3.o: $(TOP)/src/mem3.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mem3.c -fts5.c: $(FTS5_SRC) $(FTS5_HDR) - tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl - cp $(TOP)/ext/fts5/fts5.h . +mem5.o: $(TOP)/src/mem5.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mem5.c + +memdb.o: $(TOP)/src/memdb.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/memdb.c + +memjournal.o: $(TOP)/src/memjournal.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/memjournal.c + +mutex.o: $(TOP)/src/mutex.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mutex.c + +mutex_noop.o: $(TOP)/src/mutex_noop.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mutex_noop.c + +mutex_unix.o: $(TOP)/src/mutex_unix.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mutex_unix.c + +mutex_w32.o: $(TOP)/src/mutex_w32.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/mutex_w32.c + +notify.o: $(TOP)/src/notify.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/notify.c + +pager.o: $(TOP)/src/pager.c $(DEPS_OBJ_COMMON) $(TOP)/src/pager.h + $(T.cc.sqlite) -c $(TOP)/src/pager.c + +pcache.o: $(TOP)/src/pcache.c $(DEPS_OBJ_COMMON) $(TOP)/src/pcache.h + $(T.cc.sqlite) -c $(TOP)/src/pcache.c + +pcache1.o: $(TOP)/src/pcache1.c $(DEPS_OBJ_COMMON) $(TOP)/src/pcache.h + $(T.cc.sqlite) -c $(TOP)/src/pcache1.c + +os.o: $(TOP)/src/os.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/os.c + +os_kv.o: $(TOP)/src/os_kv.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/os_kv.c + +os_unix.o: $(TOP)/src/os_unix.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/os_unix.c + +os_win.o: $(TOP)/src/os_win.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/os_win.c + +pragma.o: $(TOP)/src/pragma.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/pragma.c + +prepare.o: $(TOP)/src/prepare.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/prepare.c + +printf.o: $(TOP)/src/printf.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/printf.c + +random.o: $(TOP)/src/random.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/random.c + +resolve.o: $(TOP)/src/resolve.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/resolve.c + +rowset.o: $(TOP)/src/rowset.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/rowset.c + +select.o: $(TOP)/src/select.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/select.c + +status.o: $(TOP)/src/status.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/status.c + +sqlite3.o: sqlite3.h sqlite3.c + $(T.cc.sqlite) -c sqlite3.c + +table.o: $(TOP)/src/table.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/table.c + +threads.o: $(TOP)/src/threads.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/threads.c + +tokenize.o: $(TOP)/src/tokenize.c keywordhash.h $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/tokenize.c + +treeview.o: $(TOP)/src/treeview.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/treeview.c + +trigger.o: $(TOP)/src/trigger.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/trigger.c + +update.o: $(TOP)/src/update.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/update.c + +upsert.o: $(TOP)/src/upsert.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/upsert.c + +utf.o: $(TOP)/src/utf.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/utf.c + +util.o: $(TOP)/src/util.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/util.c + +vacuum.o: $(TOP)/src/vacuum.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vacuum.c + +vdbe.o: $(TOP)/src/vdbe.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbe.c + +vdbeapi.o: $(TOP)/src/vdbeapi.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbeapi.c + +vdbeaux.o: $(TOP)/src/vdbeaux.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbeaux.c + +vdbeblob.o: $(TOP)/src/vdbeblob.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbeblob.c + +vdbemem.o: $(TOP)/src/vdbemem.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbemem.c + +vdbesort.o: $(TOP)/src/vdbesort.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbesort.c + +vdbetrace.o: $(TOP)/src/vdbetrace.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbetrace.c + +vdbevtab.o: $(TOP)/src/vdbevtab.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vdbevtab.c -lsm1.c: $(LSM1_SRC) - tclsh $(TOP)/ext/lsm1/tool/mklsm1c.tcl - cp $(TOP)/ext/lsm1/lsm.h . +vtab.o: $(TOP)/src/vtab.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/vtab.c -userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c +wal.o: $(TOP)/src/wal.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/wal.c -sqlite3session.o: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c +walker.o: $(TOP)/src/walker.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/walker.c -sqlite3rbu.o: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) - $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c +where.o: $(TOP)/src/where.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/where.c + +wherecode.o: $(TOP)/src/wherecode.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/wherecode.c + +whereexpr.o: $(TOP)/src/whereexpr.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/whereexpr.c + +window.o: $(TOP)/src/window.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c $(TOP)/src/window.c + +tclsqlite.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DUSE_TCL_STUBS=1 $$TCL_INCLUDE_SPEC \ + -c $(TOP)/src/tclsqlite.c + +tclsqlite-shell.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC + +tclsqlite-stubs.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC -# Rules for building test programs and for running tests # -tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a - $(TCCX) $(TCL_FLAGS) -DTCLSH -o tclsqlite3 \ - $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) +# STATIC_TCLSQLITE3 = 1 to statically link tclsqlite3, else +# 0. Requires static versions of all requisite libraries. Primarily +# intended for use with static-friendly environments like Alpine +# Linux. It won't work on glibc-based systems. +# +STATIC_TCLSQLITE3 ?= 0 +# +# tclsqlite3.(deps|flags).N = N is $(STATIC_TCLSQLITE3) +# +tclsqlite3.deps.1 = sqlite3.o +tclsqlite3.flags.1 = -static $(tclsqlite3.deps.1) +tclsqlite3.deps.0 = $(libsqlite3.DLL) +tclsqlite3.flags.0 = $(tclsqlite3.deps.0) +tclsqlite3$(T.exe): $(T.tcl.env.sh) tclsqlite-shell.o $(tclsqlite3.deps.$(STATIC_TCLSQLITE3)) + $(T.link.tcl) -o $@ tclsqlite-shell.o \ + $(tclsqlite3.flags.$(STATIC_TCLSQLITE3)) $$TCL_INCLUDE_SPEC $$TCL_LIB_SPEC \ + $(LDFLAGS.libsqlite3) +tclsqlite3$(T.exe)-1: tclsqlite3$(T.exe) +tclsqlite3$(T.exe)-0: +tcl: tclsqlite3$(T.exe)-$(HAVE_TCL) -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(TOP)/tool/mkccode.tcl - tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c +# Rules to build opcodes.c and opcodes.h +# +opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c -sqlite3_analyzer$(EXE): sqlite3_analyzer.c - $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +opcodes.h: parse.h $(TOP)/src/vdbe.c \ + $(TOP)/tool/mkopcodeh.tcl $(B.tclsh) + cat parse.h $(TOP)/src/vdbe.c | $(B.tclsh) $(TOP)/tool/mkopcodeh.tcl >opcodes.h -sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl - tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c +# Rules to build parse.c and parse.h - the outputs of lemon. +# +parse.h: parse.c -sqltclsh$(EXE): sqltclsh.c - $(TCCX) $(TCL_FLAGS) sqltclsh.c -o $@ $(LIBTCL) $(THREADLIB) +parse.c: $(TOP)/src/parse.y lemon$(B.exe) + cp $(TOP)/src/parse.y . + ./lemon$(B.exe) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y -sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c - $(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert$(EXE) $(THREADLIB) +pragma.h: $(TOP)/tool/mkpragmatab.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkpragmatab.tcl -CHECKER_DEPS =\ - $(TOP)/tool/mkccode.tcl \ - sqlite3.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/ext/repair/sqlite3_checker.tcl \ - $(TOP)/ext/repair/checkindex.c \ - $(TOP)/ext/repair/checkfreelist.c \ - $(TOP)/ext/misc/btreeinfo.c \ - $(TOP)/ext/repair/sqlite3_checker.c.in +sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION $(B.tclsh) + echo '#ifndef SQLITE_RESOURCE_VERSION' >$@ + echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@ + cat $(TOP)/VERSION | $(B.tclsh) $(TOP)/tool/replace.tcl exact . , >>$@ + echo '#endif' >>sqlite3rc.h -sqlite3_checker.c: $(CHECKER_DEPS) - tclsh $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ +mkkeywordhash$(B.exe): $(TOP)/tool/mkkeywordhash.c + $(B.cc) -o $@ $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c +keywordhash.h: mkkeywordhash$(B.exe) + ./mkkeywordhash$(B.exe) > $@ + +# +# sqlite3.c split into many smaller files. +# +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/split-sqlite3c.tcl + +# +# Static libsqlite3 +# +$(libsqlite3.LIB): $(LIBOBJ) + $(AR) $(AR.flags) $@ $(LIBOBJ) +$(libsqlite3.LIB)-1: $(libsqlite3.LIB) +$(libsqlite3.LIB)-0 $(libsqlite3.LIB)-: +lib: $(libsqlite3.LIB)-$(ENABLE_LIB_STATIC) +all: lib + +# +# Dynamic libsqlite3 +# +$(libsqlite3.DLL): $(LIBOBJ) + $(T.link.shared) -o $@ $(LIBOBJ) $(LDFLAGS.libsqlite3) \ + $(LDFLAGS.libsqlite3.os-specific) $(LDFLAGS.libsqlite3.soname) +$(libsqlite3.DLL)-1: $(libsqlite3.DLL) +$(libsqlite3.DLL)-0 $(libsqlite3.DLL)-: +so: $(libsqlite3.DLL)-$(ENABLE_LIB_SHARED) +all: so + +# +# DLL installation... +# +# On most Unix-like platforms, install the $(libsqlite3.DLL) as +# $(libsqlite3.DLL).$(PACKAGE_VERSION) and create symlinks which point +# to it: +# +# - libsqlite3.so.$(PACKAGE_VERSION) +# - libsqlite3.so.0 =symlink-> libsqlite3.so.$(PACKAGE_VERSION) (see below) +# - libsqlite3.so =symlink-> libsqlite3.so.3 +# +# Different rules apply for platforms where $(T.dll)==.dylib and for +# the "Unix on Windows" environments. +# +# The link named libsqlite3.so.0 is provided in an attempt to reduce +# downstream disruption when performing upgrades from pre-3.48 to a +# version 3.48 or higher. That name is considered a legacy remnant +# and may eventually be removed from this installation process. +# +# Historically libtool installed the library like so: +# +# libsqlite3.so -> libsqlite3.so.0.8.6 +# libsqlite3.so.0 -> libsqlite3.so.0.8.6 +# libsqlite3.so.0.8.6 +# +# The historical SQLite build always used a version number of 0.8.6 +# for reasons lost to history but having something to do with libtool +# (which is no longer used in this tree). In order to retain filename +# compatibility for systems which have libraries installed using those +# conventions: +# +# 1) If libsqlite3.so.0.8.6 is found in the target installation +# directory then it is re-linked to point to the newer-style +# names. We cannot retain both the old and new installation because +# they both share the high-level name $(libsqlite3.DLL). The +# down-side of this is that it may upset packaging tools when we +# replace libsqlite3.so (from a legacy package) with a new symlink. +# +# 2) If INSTALL_SO_086_LINK=1 and point (1) does not apply then links +# to the legacy-style names are created. The primary intent of this +# is to enable chains of operations such as the hypothetical (apt +# remove sqlite3-3.47.0 && apt install sqlite3-3.48.0). In such +# cases, condition (1) would never trigger but applications might +# still expect to see the legacy file names. +# +# In either case, libsqlite3.la, if found, is deleted because it would +# contain stale state, referring to non-libtool-generated libraries. +# -sqlite3_checker$(TEXE): sqlite3_checker.c - $(TCCX) $(TCL_FLAGS) sqlite3_checker.c -o $@ $(LIBTCL) $(THREADLIB) +install-dll-out-implib: $(install-dir.lib) $(libsqlite3.DLL) + if [ x != "x$(libsqlite3.out.implib)" ] && [ -f "$(libsqlite3.out.implib)" ]; then \ + $(INSTALL) $(libsqlite3.out.implib) "$(install-dir.lib)"; \ + fi + +install-dll-unix-generic: install-dll-out-implib + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f $(libsqlite3.DLL).0 $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + mv $(libsqlite3.DLL) $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0 || exit $$?; \ + ls -la $(libsqlite3.DLL) $(libsqlite3.DLL).[a03]*; \ + if [ -e $(libsqlite3.DLL).0.8.6 ]; then \ + echo "ACHTUNG: legacy libtool-compatible install found. Re-linking it..."; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ + elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \ + echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ + fi + +install-dll-msys: install-dll-out-implib $(install-dir.bin) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.bin)" +# ----------------------------------------------^^^ yes, bin +# Each of {msys,mingw,cygwin} uses a different name for the DLL, but +# that is already accounted for via $(libsqlite3.DLL). +install-dll-mingw: install-dll-msys +install-dll-cygwin: install-dll-msys + +install-dll-darwin: $(install-dir.lib) $(libsqlite3.DLL) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f libsqlite3.0$(T.dll) libsqlite3.$(PACKAGE_VERSION)$(T.dll) || exit $$?; \ + dllname=libsqlite3.$(PACKAGE_VERSION)$(T.dll); \ + mv $(libsqlite3.DLL) $$dllname || exit $$?; \ + ln -s $$dllname $(libsqlite3.DLL) || exit $$?; \ + ln -s $$dllname libsqlite3.0$(T.dll) || exit $$?; \ + ls -la $$dllname $(libsqlite3.DLL) libsqlite3.0$(T.dll) + +install-dll-1: install-dll-$(libsqlite3.DLL.install-rules) +install-dll-0 install-dll-: +install-dll: install-dll-$(ENABLE_LIB_SHARED) +install: install-dll -dbdump$(EXE): $(TOP)/ext/misc/dbdump.c sqlite3.o - $(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \ - $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB) +# +# Install $(libsqlite3.LIB) +# +install-lib-1: $(install-dir.lib) $(libsqlite3.LIB) + $(INSTALL.noexec) $(libsqlite3.LIB) "$(install-dir.lib)" +install-lib-0 install-lib-: +install-lib: install-lib-$(ENABLE_LIB_STATIC) +install: install-lib +# +# Install C header files +# +install-headers: sqlite3.h $(install-dir.include) + $(INSTALL.noexec) sqlite3.h "$(TOP)/src/sqlite3ext.h" "$(install-dir.include)" +install: install-headers + +# +# If TCL_EXT_DLL_BASENAME is not set then guess the Tcl extension's +# DLL name depending on the Tcl version. This does not account for +# Cygwin's naming - the canonical build will usually set it, but +# static makefiles importing this one will need to account for that on +# their own. They can do that by setting libtclsqlite3.basename-[89] +# to appropriate names (cygsqlite resp. cygtcl9sqlite). +# +TCL_MAJOR_VERSION ?= 0 +libtclsqlite3.basename-8 ?= libsqlite +libtclsqlite3.basename-9 ?= libtcl9sqlite +TCL_EXT_DLL_BASENAME ?= $(libtclsqlite3.basename-$(TCL_MAJOR_VERSION)) +libtclsqlite3.DLL ?= $(TCL_EXT_DLL_BASENAME)$(PACKAGE_VERSION)$(T.dll) + +# +# libtclsqlite3... +# +pkgIndex.tcl: $(TOP)/main.mk + echo 'package ifneeded sqlite3 $(PACKAGE_VERSION) [list load [file join $$dir $(libtclsqlite3.DLL)] Sqlite3]' > $@ +pkgIndex.tcl-1: pkgIndex.tcl +pkgIndex.tcl-0 pkgIndex.tcl-: +tcl: pkgIndex.tcl-$(HAVE_TCL) + +$(libtclsqlite3.DLL): $(T.tcl.env.sh) tclsqlite.o $(LIBOBJ) + $(T.tcl.env.source); \ + $(T.link.shared) -o $@ tclsqlite.o \ + $$TCL_INCLUDE_SPEC $$TCL_STUB_LIB_SPEC $(LDFLAGS.libsqlite3) \ + $(LIBOBJ) -Wl,-rpath,$$TCLLIBDIR +# ^^^ that rpath bit is defined as TCL_LD_SEARCH_FLAGS in +# tclConfig.sh, but it's defined in such a way as to be useless for a +# _static_ makefile. +$(libtclsqlite3.DLL)-1: $(libtclsqlite3.DLL) +$(libtclsqlite3.DLL)-0 $(libtclsqlite3.DLL)-: +libtcl: $(libtclsqlite3.DLL)-$(HAVE_TCL) +tcl: libtcl +all: tcl + +install-tcl-1: $(libtclsqlite3.DLL) pkgIndex.tcl + $(T.tcl.env.source); \ + $(INSTALL) -d "$(DESTDIR)$$TCLLIBDIR"; \ + $(INSTALL) $(libtclsqlite3.DLL) "$(DESTDIR)$$TCLLIBDIR"; \ + $(INSTALL.noexec) pkgIndex.tcl "$(DESTDIR)$$TCLLIBDIR" +install-tcl-0 install-tcl-: + @echo "TCL support disabled, so not installing $(libtclsqlite3.DLL)" +install-tcl: install-tcl-$(HAVE_TCL) +install: install-tcl + +tclsqlite3.c: sqlite3.c + echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c + cat sqlite3.c >>tclsqlite3.c + echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c + cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c + +# +# $(CFLAGS.tclextension) = CFLAGS for the tclextension* targets. +# +CFLAGS.tclextension = $(CFLAGS.intree_includes) $(CFLAGS.env) $(OPT_FEATURE_FLAGS) $(OPTS) +# +# Build the SQLite TCL extension in a way that make it compatible +# with whatever version of TCL is running as $TCLSH_CMD, possibly defined +# by --with-tclsh=/path/to/tclsh. +# +tclextension: tclsqlite3.c + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --build-only \ + --tclConfig.sh $(TCL_CONFIG_SH) --cc "$(T.cc)" $(CFLAGS.tclextension) + +# +# Install the SQLite TCL extension in a way that is appropriate for $TCLSH_CMD +# to find it. +# +tclextension-install: tclsqlite3.c + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --destdir "$(DESTDIR)" \ + --tclConfig.sh $(TCL_CONFIG_SH) --cc "$(T.cc)" $(CFLAGS.tclextension) + +# +# Uninstall the SQLite TCL extension that is used by $TCLSH_CMD. +# +tclextension-uninstall: + $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --uninstall \ + --tclConfig.sh $(TCL_CONFIG_SH) + +# +# List all installed the SQLite TCL extensions that is are accessible +# by $TCLSH_CMD, including prior versions. +# +tclextension-list: + @ $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --info \ + --tclConfig.sh $(TCL_CONFIG_SH) + +# Verify that the SQLite TCL extension that is loaded by default +# in $(TCLSH_CMD) is the same as the version of SQLite for the +# current source tree +# +tclextension-verify: sqlite3.h + @ $(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --version-check \ + --tclConfig.sh $(TCL_CONFIG_SH) + +# Run all of the tclextension targets in order, ending with uninstall. +tclextension-all: + $(MAKE) tclextension + $(MAKE) tclextension-install + $(MAKE) tclextension-list + $(MAKE) tclextension-verify + $(MAKE) tclextension-uninstall + +# +# FTS5 things +# +FTS5_SRC = \ + $(TOP)/ext/fts5/fts5.h \ + $(TOP)/ext/fts5/fts5Int.h \ + $(TOP)/ext/fts5/fts5_aux.c \ + $(TOP)/ext/fts5/fts5_buffer.c \ + $(TOP)/ext/fts5/fts5_main.c \ + $(TOP)/ext/fts5/fts5_config.c \ + $(TOP)/ext/fts5/fts5_expr.c \ + $(TOP)/ext/fts5/fts5_hash.c \ + $(TOP)/ext/fts5/fts5_index.c \ + fts5parse.c fts5parse.h \ + $(TOP)/ext/fts5/fts5_storage.c \ + $(TOP)/ext/fts5/fts5_tokenize.c \ + $(TOP)/ext/fts5/fts5_unicode2.c \ + $(TOP)/ext/fts5/fts5_varint.c \ + $(TOP)/ext/fts5/fts5_vocab.c \ + +fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon$(B.exe) + cp $(TOP)/ext/fts5/fts5parse.y . + rm -f fts5parse.h + ./lemon$(B.exe) $(OPTS) -S fts5parse.y + +fts5parse.h: fts5parse.c + +fts5.c: $(FTS5_SRC) $(B.tclsh) + $(B.tclsh) $(TOP)/ext/fts5/tool/mkfts5c.tcl + cp $(TOP)/ext/fts5/fts5.h . + +fts5.o: fts5.c $(DEPS_OBJ_COMMON) $(EXTHDR) + $(T.cc.extension) -c fts5.c + +sqlite3rbu.o: $(TOP)/ext/rbu/sqlite3rbu.c $(DEPS_OBJ_COMMON) $(EXTHDR) + $(T.cc.extension) -c $(TOP)/ext/rbu/sqlite3rbu.c + + +# # Rules to build the 'testfixture' application. # +# If using the amalgamation, use sqlite3.c directly to build the test +# fixture. Otherwise link against libsqlite3.a. (This distinction is +# necessary because the test fixture requires non-API symbols which are +# hidden when the library is built via the amalgamation). +# TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE +TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB -TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_CARRAY +TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_PERCENTILE TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC +TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON +TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 + +TESTFIXTURE_SRC0 = $(TESTSRC2) $(libsqlite3.LIB) +TESTFIXTURE_SRC1 = sqlite3.c +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c +TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) -testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ - $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ - -o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB) +testfixture$(T.exe): $(T.tcl.env.sh) has_tclsh85 $(TESTFIXTURE_SRC) + $(T.link.tcl) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ + -o $@ $(TESTFIXTURE_SRC) \ + $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \ + $(LDFLAGS.libsqlite3) -amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c \ - $(TOP)/ext/session/test_session.c - $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ - $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ - $(TOP)/ext/session/test_session.c \ - -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) +coretestprogs: testfixture$(B.exe) sqlite3$(B.exe) -coretestprogs: $(TESTPROGS) +testprogs: $(TESTPROGS) srcck1$(B.exe) fuzzcheck$(T.exe) sessionfuzz$(T.exe) -testprogs: coretestprogs srcck1$(EXE) fuzzcheck$(EXE) sessionfuzz$(EXE) +# A very detailed test running most or all test cases +fulltest: alltest fuzztest -fulltest: $(TESTPROGS) fuzztest - ./testfixture$(EXE) $(TOP)/test/all.test $(TESTOPTS) +# Run most or all tcl test cases +alltest: $(TESTPROGS) + ./testfixture$(T.exe) $(TOP)/test/all.test $(TESTOPTS) +# Really really long testing soaktest: $(TESTPROGS) - ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 $(TESTOPTS) + ./testfixture$(T.exe) $(TOP)/test/all.test -soak=1 $(TESTOPTS) +# Do extra testing but not everything. fulltestonly: $(TESTPROGS) fuzztest - ./testfixture$(EXE) $(TOP)/test/full.test $(TESTOPTS) - -queryplantest: testfixture$(EXE) sqlite3$(EXE) - ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner $(TESTOPTS) + ./testfixture$(T.exe) $(TOP)/test/full.test -fuzztest: fuzzcheck$(EXE) $(FUZZDATA) sessionfuzz$(EXE) $(TOP)/test/sessionfuzz-data1.db - ./fuzzcheck$(EXE) $(FUZZDATA) - ./sessionfuzz run $(TOP)/test/sessionfuzz-data1.db +# +# Fuzz testing +# +# WARNING: When the "fuzztest" target is run by the testrunner.tcl script, +# it does not actually run this code. Instead, it schedules equivalent +# commands. Therefore, if this target is updated, then code in +# testrunner_data.tcl (search for "trd_fuzztest_data") must also be updated. +# +fuzztest: fuzzcheck$(T.exe) $(FUZZDATA) sessionfuzz$(T.exe) + ./fuzzcheck$(T.exe) $(FUZZDATA) + ./sessionfuzz$(T.exe) run $(TOP)/test/sessionfuzz-data1.db -valgrindfuzz: fuzzcheck$(EXE) $(FUZZDATA) sessionfuzz$(EXE) $(TOP)/test/sessionfuzz-data1.db - valgrind ./fuzzcheck$(EXE) --cell-size-check --limit-mem 10M $(FUZZDATA) - valgrind ./sessionfuzz run $(TOP)/test/sessionfuzz-data1.db +valgrindfuzz: fuzzcheck$(T.exe) $(FUZZDATA) sessionfuzz$(T.exe) + valgrind ./fuzzcheck$(T.exe) --cell-size-check --limit-mem 10M $(FUZZDATA) + valgrind ./sessionfuzz$(T.exe) run $(TOP)/test/sessionfuzz-data1.db +# # The veryquick.test TCL tests. # -tcltest: ./testfixture$(EXE) - ./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS) +tcltest: ./testfixture$(T.exe) + ./testfixture$(T.exe) $(TOP)/test/veryquick.test $(TESTOPTS) +# # Runs all the same tests cases as the "tcltest" target but uses # the testrunner.tcl script to run them in multiple cores # concurrently. -testrunner: testfixture$(EXE) - ./testfixture$(EXE) $(TOP)/test/testrunner.tcl +testrunner: testfixture$(T.exe) + ./testfixture$(T.exe) $(TOP)/test/testrunner.tcl + +# +# This is the testing target preferred by the core SQLite developers. +# It runs tests under a standard configuration, regardless of how +# ./configure was run. The devs run "make devtest" prior to each +# check-in, at a minimum. Probably other tests too, but at least this +# one. +# +devtest: srctree-check sourcetest + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) + +mdevtest: srctree-check has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS) + +sdevtest: has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) -# Runs both fuzztest and testrunner, consecutively. +# Like releasetest, except it omits srctree-check and verify-source so +# that it can be used on a modified source tree. # -devtest: testfixture$(EXE) fuzztest testrunner +xdevtest: has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) -# A very quick test using only testfixture and omitting all the slower -# tests. Designed to run in under 3 minutes on a workstation. # -quicktest: ./testfixture$(EXE) - ./testfixture$(EXE) $(TOP)/test/extraquick.test $(TESTOPTS) +# Validate that various generated files in the source tree +# are up-to-date. +# +srctree-check: $(TOP)/tool/srctree-check.tcl + $(TCLSH_CMD) $(TOP)/tool/srctree-check.tcl -# The default test case. Runs most of the faster standard TCL tests, -# and fuzz tests, and sqlite3_analyzer and sqldiff tests. -test: fuzztest sourcetest $(TESTPROGS) tcltest +# +# Testing for a release +# +releasetest: srctree-check has_tclsh85 verify-source + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) +# +# Minimal testing that runs in less than 3 minutes +# +quicktest: ./testfixture$(T.exe) + ./testfixture$(T.exe) $(TOP)/test/extraquick.test $(TESTOPTS) + +# +# Try to run tests on whatever options are specified by the +# ./configure. The developers seldom use this target. Instead +# they use "make devtest" which runs tests on a standard set of +# options regardless of how SQLite is configured. This "test" +# target is provided for legacy only. +# +test: srctree-check fuzztest sourcetest $(TESTPROGS) tcltest +# # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # valgrindtest: $(TESTPROGS) valgrindfuzz - OMIT_MISUSE=1 valgrind -v \ - ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS) + OMIT_MISUSE=1 valgrind -v ./testfixture$(T.exe) $(TOP)/test/permutations.test valgrind $(TESTOPTS) +# # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # -smoketest: $(TESTPROGS) fuzzcheck$(EXE) - ./testfixture$(EXE) $(TOP)/test/main.test $(TESTOPTS) +smoketest: $(TESTPROGS) fuzzcheck$(T.exe) + ./testfixture$(T.exe) $(TOP)/test/main.test $(TESTOPTS) + +shelltest: + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell + +# +# sqlite3_analyzer.c build depends on $(LINK_TOOLS_DYNAMICALLY). +# +sqlite3_analyzer.c.flags.0 = -DINCLUDE_SQLITE3_C=1 +sqlite3_analyzer.c.flags.1 = +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \ + $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in + $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in \ + $(sqlite3_analyzer.c.flags.$(LINK_TOOLS_DYNAMICALLY)) \ + $(OPT_FEATURE_FLAGS) \ + > $@ + +# +# sqlite3_analyzer's build mode depends on $(LINK_TOOLS_DYNAMICALLY). +# +sqlite3_analyzer.flags.1 = -L. -lsqlite3 +sqlite3_analyzer.flags.0 = $(LDFLAGS.libsqlite3) +sqlite3_analyzer.deps.1 = $(libsqlite3.DLL) +sqlite3_analyzer.deps.0 = +sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \ + $(sqlite3_analyzer.deps.$(LINK_TOOLS_DYNAMICALLY)) + $(T.link.tcl) sqlite3_analyzer.c -o $@ \ + $(sqlite3_analyzer.flags.$(LINK_TOOLS_DYNAMICALLY)) \ + $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC $$TCL_LIBS +# ^^^^ the order of those flags is relevant for +# $(sqlite3_analyzer.flags.1): if the $$TCL_... flags come first they +# can cause the $@ to link to an out-of-tree libsqlite3.so, which may +# or may not fail or otherwise cause confusion. + +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl \ + $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl \ + $(TOP)/tool/sqltclsh.c.in + $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c + +sqltclsh$(T.exe): $(T.tcl.env.sh) sqltclsh.c + $(T.link.tcl) sqltclsh.c -o $@ $$TCL_INCLUDE_SPEC \ + $(LDFLAGS.libsqlite3) $$TCL_LIB_SPEC $$TCL_LIBS +# xbin: target for generic binaries which aren't usually built. It is +# used primarily for testing the build process. +xbin: sqltclsh$(T.exe) sqlite3_analyzer$(T.exe) + +sqlite3_expert$(T.exe): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c \ + $(TOP)/ext/expert/expert.c sqlite3.c + $(T.link) $(TOP)/ext/expert/sqlite3expert.c \ + $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(LDFLAGS.libsqlite3) +xbin: sqlite3_expert$(T.exe) + +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ + +sqlite3_checker$(T.exe): $(T.tcl.env.sh) sqlite3_checker.c + $(T.link.tcl) sqlite3_checker.c -o $@ $$TCL_INCLUDE_SPEC \ + $$TCL_LIB_SPEC $(LDFLAGS.libsqlite3) +xbin: sqlite3_checker$(T.exe) + +dbdump$(T.exe): $(TOP)/ext/misc/dbdump.c sqlite3.o + $(T.link) -DDBDUMP_STANDALONE -o $@ \ + $(TOP)/ext/misc/dbdump.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: dbdump$(T.exe) + +dbtotxt$(T.exe): $(TOP)/tool/dbtotxt.c + $(T.link)-o $@ $(TOP)/tool/dbtotxt.c $(LDFLAGS.configure) +xbin: dbtotxt$(T.exe) + +showdb$(T.exe): $(TOP)/tool/showdb.c sqlite3.o + $(T.link) -o $@ $(TOP)/tool/showdb.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: showdb$(T.exe) + +showstat4$(T.exe): $(TOP)/tool/showstat4.c sqlite3.o + $(T.link) -o $@ $(TOP)/tool/showstat4.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: showstat4$(T.exe) + +showjournal$(T.exe): $(TOP)/tool/showjournal.c sqlite3.o + $(T.link) -o $@ $(TOP)/tool/showjournal.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: showjournal$(T.exe) + +showwal$(T.exe): $(TOP)/tool/showwal.c sqlite3.o + $(T.link) -o $@ $(TOP)/tool/showwal.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: showwal$(T.exe) + +showshm$(T.exe): $(TOP)/tool/showshm.c + $(T.link) -o $@ $(TOP)/tool/showshm.c $(LDFLAGS.configure) +xbin: showshm$(T.exe) + +index_usage$(T.exe): $(TOP)/tool/index_usage.c sqlite3.o + $(T.link) $(SHELL_OPT) -o $@ $(TOP)/tool/index_usage.c sqlite3.o \ + $(LDFLAGS.libsqlite3) +xbin: index_usage$(T.exe) + +# Reminder: changeset does not build without -DSQLITE_ENABLE_SESSION +changeset$(T.exe): $(TOP)/ext/session/changeset.c sqlite3.o + $(T.link) -o $@ $(TOP)/ext/session/changeset.c sqlite3.o \ + $(LDFLAGS.libsqlite3) +xbin: changeset$(T.exe) + +changesetfuzz$(T.exe): $(TOP)/ext/session/changesetfuzz.c sqlite3.o + $(T.link) -o $@ $(TOP)/ext/session/changesetfuzz.c sqlite3.o \ + $(LDFLAGS.libsqlite3) +xbin: changesetfuzz$(T.exe) + +rollback-test$(T.exe): $(TOP)/tool/rollback-test.c sqlite3.o + $(T.link) -o $@ $(TOP)/tool/rollback-test.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: rollback-test$(T.exe) + +atrc$(T.exe): $(TOP)/test/atrc.c sqlite3.o + $(T.link) -o $@ $(TOP)/test/atrc.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: atrc$(T.exe) + +LogEst$(T.exe): $(TOP)/tool/logest.c sqlite3.h + $(T.link) -I. -o $@ $(TOP)/tool/logest.c $(LDFLAGS.configure) +xbin: LogEst$(T.exe) + +wordcount$(T.exe): $(TOP)/test/wordcount.c sqlite3.o + $(T.link) -o $@ $(TOP)/test/wordcount.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: wordcount$(T.exe) + +speedtest1$(T.exe): $(TOP)/test/speedtest1.c sqlite3.c Makefile + $(T.link) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c \ + $(LDFLAGS.libsqlite3) +xbin: speedtest1$(T.exe) + +startup$(T.exe): $(TOP)/test/startup.c sqlite3.c + $(T.link) -Os -g -USQLITE_THREADSAFE -DSQLITE_THREADSAFE=0 \ + -o $@ $(TOP)/test/startup.c sqlite3.c $(LDFLAGS.libsqlite3) +xbin: startup$(T.exe) + +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ + +kvtest$(T.exe): $(TOP)/test/kvtest.c sqlite3.c + $(T.link) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c \ + $(LDFLAGS.libsqlite3) +xbin: kvtest$(T.exe) + +# +# rbu$(T.exe) requires building with -DSQLITE_ENABLE_RBU, which +# specifically does not have an --enable-rbu flag in the configure +# script. +rbu$(T.exe): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o + $(T.link) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.o $(LDFLAGS.libsqlite3) + +loadfts$(T.exe): $(TOP)/tool/loadfts.c $(libsqlite3.LIB) + $(T.link) $(TOP)/tool/loadfts.c $(libsqlite3.LIB) \ + -o $@ $(LDFLAGS.libsqlite3) +xbin: loadfts$(T.exe) + +# This target will fail if the SQLite amalgamation contains any exported +# symbols that do not begin with "sqlite3_". It is run as part of the +# releasetest.tcl script. +# +VALIDIDS=' sqlite3(changeset|changegroup|session|rebaser)?_' +checksymbols: sqlite3.o + nm -g --defined-only sqlite3.o + nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0 + echo '0 errors out of 1 tests' + +# Build the amalgamation-autoconf package. The amalgamation-tarball target builds +# a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. +# The snapshot-tarball target builds a tarball named by the SHA3 hash +# +amalgamation-tarball: sqlite3.c sqlite3rc.h + TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal -shelltest: $(TESTPROGS) - ./testfixture$(EXT) $(TOP)/test/permutations.test shell +snapshot-tarball: sqlite3.c sqlite3rc.h + TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot + +# Build a ZIP archive snapshot of the latest check-in. +# +sqlite-src.zip: $(TOP)/tool/mksrczip.tcl + $(TCLSH_CMD) $(TOP)/tool/mksrczip.tcl + +# Build a ZIP archive of the amalgamation +# +sqlite-amalgamation.zip: $(TOP)/tool/mkamalzip.tcl sqlite3.c sqlite3.h shell.c sqlite3ext.h + $(TCLSH_CMD) $(TOP)/tool/mkamalzip.tcl -# The next two rules are used to support the "threadtest" target. Building +# Build all the source code deliverables +# +src-archives: sqlite-amalgamation.zip amalgamation-tarball sqlite-src.zip + ls -ltr *.zip *.tar.gz | tail -3 + +# Build a ZIP archive containing various command-line tools. +# +tool-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ + sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl + strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) + ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl +snapshot-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ + sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl + strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) + ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl --snapshot +clean-tool-zip: + rm -f sqlite-tools-*.zip +clean: clean-tool-zip + +# +# The next few rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # @@ -972,148 +2163,401 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ $(TOP)/test/tt3_stress.c \ $(TOP)/test/tt3_lookaside1.c -threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) $(TOP)/src/test_multiplex.c - $(TCCX) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o -o $@ $(THREADLIB) +threadtest3$(T.exe): sqlite3.o $(THREADTEST3_SRC) + $(T.link) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o \ + -o $@ $(LDFLAGS.libsqlite3) +xbin: threadtest3$(T.exe) + +threadtest: threadtest3$(T.exe) + ./threadtest3$(T.exe) + +threadtest5: sqlite3.c $(TOP)/test/threadtest5.c + $(T.link) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(LDFLAGS.libsqlite3) +xbin: threadtest5 + +# +# STATIC_CLI_SHELL = 1 to statically link sqlite3$(T.exe), else +# 0. Requires static versions of all requisite libraries. Primarily +# intended for use with static-friendly environments like Alpine +# Linux. +# +STATIC_CLI_SHELL ?= 0 +# +# sqlite3-shell-static.flags.N = N is $(STATIC_CLI_SHELL) +# +sqlite3-shell-static.flags.1 = -static +sqlite3-shell-static.flags.0 = +# +# When building sqlite3$(T.exe) we specifically embed a copy of +# sqlite3.c, and not link to libsqlite3.so or libsqlite3.a, because +# the shell needs to be able to enable arbitrary library features, +# some of which have significant performance impacts. For example,, +# SQLITE_ENABLE_EXPLAIN_COMMENTS has been measured as having a 5.2% +# runtime performance hit, which is fine for use in the shell but is +# not appropriate for the canonical library build. +# +sqlite3$(T.exe): shell.c sqlite3.c + $(T.link) -o $@ \ + shell.c sqlite3.c \ + $(sqlite3-shell-static.flags.$(STATIC_CLI_SHELL)) \ + $(CFLAGS.readline) $(SHELL_OPT) $(CFLAGS.icu) \ + $(LDFLAGS.libsqlite3) $(LDFLAGS.readline) +# +# Build sqlite3$(T.exe) by default except in wasi-sdk builds. Yes, the +# semantics of 0 and 1 are confusingly swapped here. +# +sqlite3$(T.exe)-1: +sqlite3$(T.exe)-0: sqlite3$(T.exe) +all: sqlite3$(T.exe)-$(HAVE_WASI_SDK) + +# The "sqlite3d" CLI is build using separate source files. This +# is useful during development and debugging. +# +sqlite3d$(T.exe): shell.c $(LIBOBJS0) + $(T.link) -o $@ \ + shell.c $(LIBOBJS0) \ + $(CFLAGS.readline) $(SHELL_OPT) \ + $(LDFLAGS.libsqlite3) $(LDFLAGS.readline) + +install-shell-0: sqlite3$(T.exe) $(install-dir.bin) + $(INSTALL) sqlite3$(T.exe) "$(install-dir.bin)" +install-shell-1: +install: install-shell-$(HAVE_WASI_SDK) + +# How to build sqldiff$(T.exe) depends on $(LINK_TOOLS_DYNAMICALLY) +# +sqldiff.0.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.o sqlite3.h +sqldiff.0.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c sqlite3.o $(LDFLAGS.libsqlite3) +sqldiff.1.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h $(libsqlite3.DLL) +sqldiff.1.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c -L. -lsqlite3 $(LDFLAGS.configure) +sqldiff$(T.exe): $(sqldiff.$(LINK_TOOLS_DYNAMICALLY).deps) + $(sqldiff.$(LINK_TOOLS_DYNAMICALLY).rules) + +install-diff: sqldiff$(T.exe) $(install-dir.bin) + $(INSTALL) sqldiff$(T.exe) "$(install-dir.bin)" +#install: install-diff + +dbhash$(T.exe): $(TOP)/tool/dbhash.c sqlite3.o sqlite3.h + $(T.link) -o $@ $(TOP)/tool/dbhash.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: dbhash$(T.exe) + +RSYNC_SRC = \ + $(TOP)/tool/sqlite3_rsync.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -USQLITE_THREADSAFE \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3_rsync$(T.exe): $(RSYNC_SRC) + $(T.cc.sqlite) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(LDFLAGS.libsqlite3) +xbin: sqlite3_rsync$(T.exe) + +install-rsync: sqlite3_rsync$(T.exe) $(install-dir.bin) + $(INSTALL) sqlite3_rsync$(T.exe) "$(install-dir.bin)" +#install: install-rsync + +install-man1: $(install-dir.man1) + $(INSTALL.noexec) "$(TOP)/sqlite3.1" "$(install-dir.man1)" +install: install-man1 + +# +# sqlite3.pc is typically generated by the configure script but could +# conceivably be generated by hand. +install-pc: sqlite3.pc $(install-dir.pkgconfig) + $(INSTALL.noexec) sqlite3.pc "$(install-dir.pkgconfig)" + +scrub$(T.exe): $(TOP)/ext/misc/scrub.c sqlite3.o + $(T.link) -o $@ -I. -DSCRUB_STANDALONE \ + $(TOP)/ext/misc/scrub.c sqlite3.o $(LDFLAGS.libsqlite3) +xbin: scrub$(T.exe) + +srcck1$(B.exe): $(TOP)/tool/srcck1.c + $(B.cc) -o srcck1$(B.exe) $(TOP)/tool/srcck1.c +xbin: srcck1$(B.exe) + +sourcetest: srcck1$(B.exe) sqlite3.c + ./srcck1$(B.exe) sqlite3.c + +src-verify$(B.exe): $(TOP)/tool/src-verify.c + $(B.cc) -o src-verify$(B.exe) $(TOP)/tool/src-verify.c +xbin: src-verify$(B.exe) + +verify-source: ./src-verify$(B.exe) + ./src-verify$(B.exe) $(TOP) + +fuzzershell$(T.exe): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h + $(T.link) -o $@ $(FUZZERSHELL_OPT) \ + $(TOP)/tool/fuzzershell.c sqlite3.c $(LDFLAGS.libsqlite3) +fuzzy: fuzzershell$(T.exe) +xbin: fuzzershell$(T.exe) + +fuzzcheck$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) + $(T.link) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(LDFLAGS.libsqlite3) +fuzzy: fuzzcheck$(T.exe) +xbin: fuzzcheck$(T.exe) + +# -fsanitize=... flags for fuzzcheck-asan. +CFLAGS.fuzzcheck-asan.fsanitize ?= -fsanitize=address + +fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) + $(T.link) -o $@ $(CFLAGS.fuzzcheck-asan.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + sqlite3.c $(LDFLAGS.libsqlite3) +fuzzy: fuzzcheck-asan$(T.exe) +xbin: fuzzcheck-asan$(T.exe) + +fuzzcheck-ubsan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) + $(T.link) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + sqlite3.c $(LDFLAGS.libsqlite3) +fuzzy: fuzzcheck-ubsan$(T.exe) +xbin: fuzzcheck-ubsan$(T.exe) + + +ossshell$(T.exe): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h + $(T.link) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ + $(TOP)/test/ossfuzz.c sqlite3.c $(LDFLAGS.libsqlite3) +fuzzy: ossshell$(T.exe) +xbin: ossshell$(T.exe) + +sessionfuzz$(T.exe): $(TOP)/test/sessionfuzz.c sqlite3.c sqlite3.h + $(T.link) -o $@ $(TOP)/test/sessionfuzz.c $(LDFLAGS.libsqlite3) +fuzzy: sessionfuzz$(T.exe) + +dbfuzz$(T.exe): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h + $(T.link) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c \ + $(LDFLAGS.libsqlite3) +fuzzy: dbfuzz$(T.exe) +xbin: dbfuzz$(T.exe) + +DBFUZZ2_OPTS = \ + -USQLITE_THREADSAFE \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_DEBUG \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 + +dbfuzz2$(T.exe): $(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h + $(T.cc) -I. -g -O0 \ + -DSTANDALONE -o dbfuzz2 \ + $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(LDFLAGS.libsqlite3) + mkdir -p dbfuzz2-dir + cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir +fuzzy: dbfuzz2$(T.exe) +xbin: dbfuzz2$(T.exe) + +mptester$(T.exe): $(libsqlite3.LIB) $(TOP)/mptest/mptest.c + $(T.link) -o $@ -I. $(TOP)/mptest/mptest.c $(libsqlite3.LIB) \ + $(LDFLAGS.libsqlite3) +xbin: mptester$(T.exe) + +MPTEST1=./mptester$(T.exe) mptest.db $(TOP)/mptest/crash01.test --repeat 20 +MPTEST2=./mptester$(T.exe) mptest.db $(TOP)/mptest/multiwrite01.test --repeat 20 +mptest: mptester$(T.exe) + rm -f mptest.db + $(MPTEST1) --journalmode DELETE + $(MPTEST2) --journalmode WAL + $(MPTEST1) --journalmode WAL + $(MPTEST2) --journalmode PERSIST + $(MPTEST1) --journalmode PERSIST + $(MPTEST2) --journalmode TRUNCATE + $(MPTEST1) --journalmode TRUNCATE + $(MPTEST2) --journalmode DELETE -threadtest: threadtest3$(EXE) - ./threadtest3$(EXE) +# Source and header files that shell.c depends on +SHELL_DEP = \ + $(TOP)/src/shell.c.in \ + $(TOP)/ext/expert/sqlite3expert.c \ + $(TOP)/ext/expert/sqlite3expert.h \ + $(TOP)/ext/intck/sqlite3intck.c \ + $(TOP)/ext/intck/sqlite3intck.h \ + $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ + $(TOP)/ext/misc/completion.c \ + $(TOP)/ext/misc/decimal.c \ + $(TOP)/ext/misc/fileio.c \ + $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/pcachetrace.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ + $(TOP)/ext/misc/shathree.c \ + $(TOP)/ext/misc/sqlar.c \ + $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/vfstrace.c \ + $(TOP)/ext/misc/windirent.h \ + $(TOP)/ext/misc/zipfile.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h + + +shell.c: $(SHELL_DEP) $(TOP)/tool/mkshellc.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkshellc.tcl shell.c -TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO) -$(TEST_EXTENSION): $(TOP)/src/test_loadext.c - $(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION) +# +# Rules to build the extension objects. +# +DEPS_EXT_COMMON = $(DEPS_OBJ_COMMON) $(EXTHDR) +icu.o: $(TOP)/ext/icu/icu.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/icu/icu.c $(CFLAGS.icu) -extensiontest: testfixture$(EXE) $(TEST_EXTENSION) - ./testfixture$(EXE) $(TOP)/test/loadext.test +fts3.o: $(TOP)/ext/fts3/fts3.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3.c -dbtotxt$(EXE): $(TOP)/tool/dbtotxt.c - $(TCC) -o dbtotxt$(EXE) $(TOP)/tool/dbtotxt.c +fts3_aux.o: $(TOP)/ext/fts3/fts3_aux.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_aux.c -showdb$(EXE): $(TOP)/tool/showdb.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \ - $(TOP)/tool/showdb.c sqlite3.o $(THREADLIB) +fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_expr.c -showstat4$(EXE): $(TOP)/tool/showstat4.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \ - $(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB) +fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_hash.c -showjournal$(EXE): $(TOP)/tool/showjournal.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showjournal$(EXE) \ - $(TOP)/tool/showjournal.c sqlite3.o $(THREADLIB) +fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_icu.c $(CFLAGS.icu) -showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ - $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) +fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_porter.c -showshm$(EXE): $(TOP)/tool/showshm.c - $(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c +fts3_snippet.o: $(TOP)/ext/fts3/fts3_snippet.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_snippet.c -index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED $(SHELL_OPTS) -o index_usage$(EXE) \ - $(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB) +fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_tokenizer.c -changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ - $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) +fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_tokenizer1.c -changesetfuzz$(EXE): $(TOP)/ext/session/changesetfuzz.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changesetfuzz$(EXE) \ - $(TOP)/ext/session/changesetfuzz.c sqlite3.o $(THREADLIB) +fts3_tokenize_vtab.o: $(TOP)/ext/fts3/fts3_tokenize_vtab.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_tokenize_vtab.c -fts3view$(EXE): $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o fts3view$(EXE) \ - $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(THREADLIB) +fts3_unicode.o: $(TOP)/ext/fts3/fts3_unicode.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_unicode.c -rollback-test$(EXE): $(TOP)/tool/rollback-test.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o rollback-test$(EXE) \ - $(TOP)/tool/rollback-test.c sqlite3.o $(THREADLIB) +fts3_unicode2.o: $(TOP)/ext/fts3/fts3_unicode2.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_unicode2.c -atrc$(EXE): $(TOP)/test/atrc.c sqlite3.o - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o atrc$(EXE) \ - $(TOP)/test/atrc.c sqlite3.o $(THREADLIB) +fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/fts3/fts3_write.c -LogEst$(EXE): $(TOP)/tool/logest.c sqlite3.h - $(TCC) -o LogEst$(EXE) $(TOP)/tool/logest.c +rtree.o: $(TOP)/ext/rtree/rtree.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/rtree/rtree.c -wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c - $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o wordcount$(EXE) \ - $(TOP)/test/wordcount.c sqlite3.c +sqlite3session.o: $(TOP)/ext/session/sqlite3session.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/session/sqlite3session.c -speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.c - $(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB) +stmt.o: $(TOP)/ext/misc/stmt.c $(DEPS_EXT_COMMON) + $(T.cc.extension) -c $(TOP)/ext/misc/stmt.c -kvtest$(EXE): $(TOP)/test/kvtest.c sqlite3.c - $(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) +# +# Windows section +# +# 2025-03-03: sqlite3.def and sqlite3.dll might no longer be relevant +# in this particular build, but that's difficult to verify. +# +dll: sqlite3.dll +sqlite3.def: $(LIBOBJ) + echo 'EXPORTS' >sqlite3.def + nm $(LIBOBJ) | grep ' T ' | grep ' _sqlite3_' \ + | sed 's/^.* _//' >>sqlite3.def -rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o - $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ - $(THREADLIB) +sqlite3.dll: $(LIBOBJ) sqlite3.def + $(T.cc.sqlite) $(LDFLAGS.shlib) -o $@ sqlite3.def \ + -Wl,"--strip-all" $(LIBOBJ) $(LDFLAGS.configure) -loadfts: $(TOP)/tool/loadfts.c libsqlite3.a - $(TCC) $(TOP)/tool/loadfts.c libsqlite3.a -o loadfts $(THREADLIB) +# +# Emit a list of commonly-used targets +# +help: + @echo; echo "Frequently-used high-level make targets:"; echo; \ + echo " - all [default] = builds most components"; \ + echo " - clean = cleans up most build products"; \ + echo " - distclean = cleans up all build products"; \ + echo " - install = installs activated components"; \ + echo; echo "Testing-related targets:"; echo; \ + echo " - test = a number of sanity checks"; \ + echo " - quicktest = minimal tests"; \ + echo " - releasetest = pre-release tests"; \ + echo " - devtest = Minimum tests required before code check-ins"; \ + echo " - mdevtest = A variant of devtest"; \ + echo " - sdevtest = A variant of devtest"; \ + echo " - tcltest = Runs test/veryquick.test"; \ + echo " - testrunner = Like tcltest but spread across multiple cores"; \ + echo " - fuzztest = The core fuzz tester (see target docs for important info)"; \ + echo " - valgrindfuzz = Runs fuzztest under valgrind"; \ + echo " - soaktest = Really, really long tests"; \ + echo " - alltest = Runs most or all TCL tests"; \ + echo -threadtest5: $(TOP)/test/threadtest5.c libsqlite3.a - $(TCC) $(TOP)/test/threadtest5.c libsqlite3.a -o threadtest5 $(THREADLIB) -# This target will fail if the SQLite amalgamation contains any exported -# symbols that do not begin with "sqlite3_". It is run as part of the -# releasetest.tcl script. # -checksymbols: sqlite3.o - nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 +# Remove build products sufficient so that subsequent makes will recompile +# everything from scratch. Do not remove: +# +# * test results and test logs +# * output from ./configure +# +# +tidy: + rm -f *.o *.obj *.c *.da *.bb *.bbg gmon.* *.rws sqlite3$(T.exe) + rm -f fts5.h keywordhash.h opcodes.h sqlite3.h sqlite3ext.h sqlite3session.h + rm -rf .libs .deps tsrc .target_source + rm -f lemon$(B.exe) sqlite*.tar.gz + rm -f mkkeywordhash$(B.exe) mksourceid$(B.exe) + rm -f parse.* fts5parse.* + rm -f $(libsqlite3.DLL) $(libsqlite3.LIB) $(libtclsqlite3.DLL) libsqlite3$(T.dll).a + rm -f tclsqlite3$(T.exe) $(TESTPROGS) + rm -f LogEst$(T.exe) fts3view$(T.exe) rollback-test$(T.exe) showdb$(T.exe) + rm -f showjournal$(T.exe) showstat4$(T.exe) showwal$(T.exe) speedtest1$(T.exe) + rm -f wordcount$(T.exe) changeset$(T.exe) version-info$(T.exe) + rm -f *.exp *.vsix pkgIndex.tcl + rm -f sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) sqlite3_expert$(T.exe) + rm -f mptester$(T.exe) rbu$(T.exe) srcck1$(T.exe) + rm -f fuzzershell$(T.exe) fuzzcheck$(T.exe) sqldiff$(T.exe) dbhash$(T.exe) + rm -f dbfuzz$(T.exe) dbfuzz2$(T.exe) + rm -fr dbfuzz2-dir + rm -f fuzzcheck-asan$(T.exe) fuzzcheck-ubsan$(T.exe) ossshell$(T.exe) + rm -f scrub$(T.exe) showshm$(T.exe) sqlite3_checker$(T.exe) loadfts$(T.exe) + rm -f index_usage$(T.exe) kvtest$(T.exe) startup$(T.exe) threadtest3$(T.exe) + rm -f sessionfuzz$(T.exe) changesetfuzz$(T.exe) + rm -f dbdump$(T.exe) dbtotxt$(T.exe) atrc$(T.exe) + rm -f threadtest5$(T.exe) + rm -f src-verify$(B.exe) + rm -f tclsqlite3.c has_tclsh* $(T.tcl.env.sh) + rm -f sqlite3rc.h sqlite3.def + rm -f ctime.c pragma.h -# Build the amalgamation-autoconf package. The amalamgation-tarball target builds -# a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. -# The snapshot-tarball target builds a tarball named by the SHA1 hash # -amalgamation-tarball: sqlite3.c sqlite3rc.h - TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal +# Removes build products and test logs. Retains ./configure outputs. +# +clean: tidy + rm -rf omittest* testrunner* testdir* -snapshot-tarball: sqlite3.c sqlite3rc.h - TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot +# +# Clean up everything. No exceptions. From an out-of-tree build which +# starts in an empty directory, this should result in an empty +# directory (assuming the user does not create new files in this +# directory). +# +# The main distclean rules are in Makefile.in. +# +distclean: clean -# Standard install and cleanup targets -# -install: sqlite3 libsqlite3.a sqlite3.h - mv sqlite3 /usr/bin - mv libsqlite3.a /usr/lib - mv sqlite3.h /usr/include - -clean: - rm -f *.o sqlite3 sqlite3.exe libsqlite3.a sqlite3.h opcodes.* - rm -f lemon lemon.exe lempar.c parse.* sqlite*.tar.gz - rm -f mkkeywordhash mkkeywordhash.exe keywordhash.h - rm -f $(PUBLISH) - rm -f *.da *.bb *.bbg gmon.out - rm -rf tsrc target_source - rm -f testloadext.dll libtestloadext.so - rm -f amalgamation-testfixture amalgamation-testfixture.exe - rm -f fts3-testfixture fts3-testfixture.exe - rm -f testfixture testfixture.exe - rm -f threadtest3 threadtest3.exe - rm -f LogEst LogEst.exe - rm -f fts3view fts3view.exe - rm -f rollback-test rollback-test.exe - rm -f showdb showdb.exe - rm -f showjournal showjournal.exe - rm -f showstat4 showstat4.exe - rm -f showwal showwal.exe - rm -f changeset changeset.exe - rm -f speedtest1 speedtest1.exe - rm -f wordcount wordcount.exe - rm -f rbu rbu.exe - rm -f srcck1 srcck1.exe - rm -f sqlite3.c sqlite3-*.c fts?amal.c tclsqlite3.c - rm -f sqlite3rc.h - rm -f shell.c sqlite3ext.h - rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c - rm -f sqlite3_expert sqlite3_expert.exe - rm -f sqlite-*-output.vsix - rm -f mptester mptester.exe - rm -f fuzzershell fuzzershell.exe - rm -f fuzzcheck fuzzcheck.exe - rm -f sessionfuzz - rm -f sqldiff sqldiff.exe - rm -f fts5.* fts5parse.* - rm -f lsm.h lsm1.c - rm -f threadtest5 +# +# Show important variable settings. +# +show-variables: + @echo "CC = $(CC)" + @echo "B.cc = $(B.cc)" + @echo "T.cc = $(T.cc)" + @echo "T.cc.sqlite = $(T.cc.sqlite)" diff --git a/manifest b/manifest index 515248a699..7be9b18d30 100644 --- a/manifest +++ b/manifest @@ -1,333 +1,421 @@ -C Version\s3.42.0 -D 2023-05-16T12:36:15.536 +C Version\s3.51.2 +D 2026-01-09T17:27:48.405 +F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 764f2e3e8fb4ae1c8dfe03e65b2b3b01bd1fc57edf78ec2cab3a1301e90e1905 -F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc ce7bea7931e1ef96b0f44c0362074a8bb62e61a0e7cfdb05afebd928adaacc2b -F README.md e05bd8fcb45da04ab045c37f79a98654e8aa3b3b8f302cfbba80a0d510df75f7 -F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf -F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 +F LICENSE.md 6bc480fc673fb4acbc4094e77edb326267dd460162d7723c7f30bee2d3d9e97d +F Makefile.in 3ce07126d7e87c7464301482e161fdae6a51d0a2aa06b200b8f0000ef4d6163b +F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 +F Makefile.msc d4459fad28b388063698cbb7a73bfce8684da998a844a04b21d4b9b10291196a +F README.md dae499194b75deed76a13a4a83c82493f2530331882d7dfe5754d63287d3f8f7 +F VERSION 53fb08d314af314f884da9b33cabad229928aac28b53984a2c38fd4d7dc608ab +F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 +F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 -F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am a8d1d24affe52ebf8d7ddcf91aa973fa0316618ab95bb68c87cabf8faf527dc8 +F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 +F auto.def 44a0d1bf09d78355fc88251ccbf8e64e6341fd89c11de68a01c3645e53a2bade F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.msc 20366d19fbfc3fceecce95344a2114069eaab4fc22e4f02430da167a1e2ddf04 -F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt 42cfd21d0b19dc7d5d85fb5c405c5f3c6a4c923021c39128f6ba685355d8fd56 -F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d5893b277 -F autoconf/tea/Makefile.in 106a96f2f745d41a0f6193f1de98d7355830b65d45032c18cd7c90295ec24196 -F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 -F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac e6a7f2abf7a94b5dae00713ad6b3bebc59b878e2705607589830792c61d2ca38 -F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb +F autoconf/Makefile.in 118aa2c4d49173672d065fdda19eb8a28642e2c684212d7a626d6db5e6762521 +F autoconf/Makefile.msc 9c1ca648062fd5a4a83ba7590c4422090cccd6399002af0346b7572f086c7483 +F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 +F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 +F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac +F autoconf/tea/Makefile.in 00e60cf3bf5580f31bfdcf3c914e9ba1831d676948363962de92ce65e5be4431 +F autoconf/tea/README.txt 23475876343498ef2b514cc7510e8f1559a17e8e03fbc7a41c1c8a3b89e7b7e3 +F autoconf/tea/_teaish.tester.tcl.in 8253b44be88e2e3f21de95a65d3a90c2be8e70b7bdd08a5b80e337ba7402f8f1 +F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 +F autoconf/tea/configure 993eb27dafb35253965f9c0eb0eeefd113cae0508361c8fd90a4b58c3caf14ec x F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 -F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce -F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 -F autoconf/tea/tclconfig/tcl.m4 debe13280bd5a9d76dc34e7919cd9ed3a1408c7320400900357128c2d1abb723 -F autoconf/tea/win/makefile.vc 2c478a9a962e48b2bf9062734e04d7c63c556e217095419173f9d7938d7d78f7 -F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb -F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 -F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 -F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 2f2c090e0a1d051bb53741f0c961f5ebb24507a4f599d686ad6d7ff236144609 x -F configure.ac 4654d32ac0a0d0b48f1e1e79bdc3d777b723cf2f63c33eb1d7c4ed8b435938e8 -F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad +F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 +F autoconf/tea/teaish.tcl 81feb417e718ed75cdd7e2fdf6771f3da80dae97377a90c4d5b62b3754abbf1d +F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 +F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 +F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e +F autosetup/README.md ce0f95980a687bb861bd830b76bc4b48513567be5cf5ee7004f4f3439ffe3841 +F autosetup/autosetup b16e44924c197783df67366762dda985b45d49ebc4af15f4054e3ee0e3b65169 x +F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x +F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x +F autosetup/autosetup-find-tclsh b08f883f5753cfff1ecb8581f98b314e190b7e3f3059798e274ae5f5aad571af x +F autosetup/autosetup-test-tclsh 749d20defee533a3842139df47d700fc7a334a5da7bdbd444ae5331744b06c5f +F autosetup/cc-db.tcl 6e0ed90146197a5a05b245e649975c07c548e30926b218ca3e1d4dc034b10a7b +F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795facf7360 +F autosetup/cc-shared.tcl 163eda58c14cd662fd8a504bd2ad8a716ef4db7015dc1de0095d5de8dd601a4b +F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f +F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e +F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 +F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba +F autosetup/proj.tcl 6fc14ef82b19b77a95788ffbcfad7989b4e3cb4ce96a21dcb5cf7312f362fba9 +F autosetup/sqlite-config.tcl 5d779fce20c11fde3fe99d157dcd5b5569d729b301141b8dfb8d5aacf9d48cba +F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 +F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca +F autosetup/teaish/core.tcl e014dd95900c7f9a34e8e0f460f47e94841059827bce8b4c49668b0c7ae3f1a0 +F autosetup/teaish/feature.tcl 18194fb79a24d30e5bbdeab40999616f39278b53a27525349ded033af2fd73be +F autosetup/teaish/tester.tcl 1799514c2652db49561b3386c5242b94534d1663f2cfac861a955e071895fdd0 +F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x +F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd +F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8af4a0f4adbc +F doc/compile-for-windows.md f9e74d74da88f384edd5809f825035e071608f00f7f39c0e448df7b3982f979c F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f -F doc/lemon.html d2862dbef72496e87f7996f37e814b146848190a742c12161d13fd15346051b0 -F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 +F doc/jsonb.md acd77fc3a709f51242655ad7803510c886aa8304202fa9cf2abc5f5c4e9d7ae5 +F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 +F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 +F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 +F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 -F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a -F doc/wal-lock.md 781726aaba20bafeceb7ba9f91d5c98c6731691b30c954e37cf0b49a053d461d -F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd -F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 -F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 -F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a +F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 +F doc/wal-lock.md 7db0cd61e2000b545b78ce89b0c2a9a8dd8d64c097839258ac10d7c5c4156ec1 +F ext/README.md 6eb1ac267d917767952ed0ef63f55de003b6a5da433ce1fa389e1a9532e73132 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test 95b00567ce0775126a1b788af2d055255014714ecfddc97913864d2f9266e583 -F ext/expert/sqlite3expert.c a912efbad597eafdb0ce934ebc11039f3190b2d479685d89184e107f65d856e1 +F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c +F ext/expert/sqlite3expert.c 546010043fbec93544f762de5161b3d553165859e6bd853c4b85c05f93484260 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b -F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 +F ext/expert/test_expert.c c395134bd6d4efa594a7d26578a1cb624c4027b79b4b5fcd44736c5ef1f5f725 F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c7cc3bf59ee -F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a +F ext/fts3/README.syntax b72477722e9b4fe43f8403227d790a1c94221bfad15c27863a4b36d1052e892b F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 66d2ced306ac88c39c00d81184f2c60f338696af6ae8cc26ed7c361b157b09f7 +F ext/fts3/fts3.c 4f02858ab845a97bedf06e6cc1fba49a81fe5e00a26df68d0ad0f00a5814fa70 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h e573c6d881f7238d77cc3fd2396cbb9b2fe13efef7d2ad295a155151c4e7efbd -F ext/fts3/fts3_aux.c f0dc9bd98582615b7750218899bd0c729879b6bbf94d1be57ca1833ff49afc6f -F ext/fts3/fts3_expr.c 903bfb9433109fffb10e910d7066c49cbf8eeae316adc93f0499c4da7dfc932a -F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7 +F ext/fts3/fts3Int.h ed9b8bc5ed5be402069651e49d4855cb849af706cf3fe68548f58a2c21eefc7f +F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 +F ext/fts3/fts3_expr.c 5c13796638d8192c388777166075cdc8bc4b6712024cd5b72c31acdbefce5984 +F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641814672a F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 -F ext/fts3/fts3_porter.c e19807ce0ae31c1c6e9898e89ecc93183d7ec224ea101af039722a4f49e5f2b8 -F ext/fts3/fts3_snippet.c 4d6523e3eddeb7b46e7a82b3476a0a86a0c04821e0e2b8dd40f45ee28057cb13 -F ext/fts3/fts3_term.c f45a1e7c6ef464abb1231245d123dae12266b69e05cc56e14045b76591ae92d1 -F ext/fts3/fts3_test.c d8d7b2734f894e8a489987447658e374cdd3a3bc8575c401decf1911cb7c6454 -F ext/fts3/fts3_tokenize_vtab.c a95feda3590f3c3e17672fe35b67ea6112471aeea4c07ef7744a6606b66549aa -F ext/fts3/fts3_tokenizer.c 6d8fc150c48238955d5182bf661498db0dd473c8a2a80e00c16994a646fa96e7 +F ext/fts3/fts3_porter.c 024417020c57dd1ab39816f5fe6cf45222a857b78a1f6412f040ada1ceabd4ff +F ext/fts3/fts3_snippet.c abe3b2998e7cb6d1ab6019f87f021758a0df3ee4010fe144a174a524cff96fe6 +F ext/fts3/fts3_term.c 6a96027ad364001432545fe43322b6af04ed28bb5619ec51af1f59d0710d6d69 +F ext/fts3/fts3_test.c cc329471e573f95a6ea9fbca87e89dcfa1d355591c80172ffcd759ac521d25d8 +F ext/fts3/fts3_tokenize_vtab.c 66eba6c2baa04b2b15e80d68341b8fd0b4d3831f6b2edb33916a2906ff2d4389 +F ext/fts3/fts3_tokenizer.c defede96b5dd5d658edfae77355b9c31ea65236eedc7bbe1adbc50d645cca5bc F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 33d2d0db4dd4e7a7a7e9a7f790414293277f9e7682a2fd9d61c713bfc37cd8b6 +F ext/fts3/fts3_write.c 2bee1c5828f6401adffd07ca64260aeb79d64138958273a56de8fa5e8759a0c1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 +F ext/fts3/unicode/mkunicode.tcl cbf5f7b5c8ce8014bad731f246f2e520eece908465de4778c951ca17003381f1 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb -F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a -F ext/fts5/fts5Int.h ed48a096418ff4a7c02ac9bd1e8d40c46de21b79a132b8b08d3f32233703de7d -F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 -F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 -F ext/fts5/fts5_config.c 051056a9052f5d3a4d1c695f996fd364f920e341f136c60ab2c04aa7e267113f -F ext/fts5/fts5_expr.c 58fb8ceddfb1cefcd54510f9f2f33c220ef9d1b3fa77462111f5ae2a825ab7b1 -F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c de3cdae2e0056594aad97a728be5c43b6d7a6cdc7e9cc16f197892b2d8689c21 -F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d -F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 -F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae -F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee -F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff -F ext/fts5/fts5_tokenize.c 5e251efb0f1af99a25ed50010ba6b1ad1250aca5921af1988fdcabe5ebc3cb43 -F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00 +F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 +F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2 +F ext/fts5/fts5Int.h 8d98f8e180fe28d6067e240ed45b9011735d29d5cfb5bac194e1e376baa7c708 +F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb +F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 +F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 +F ext/fts5/fts5_expr.c b8c32da1127bafaf10d6b4768b0dcb92285798524bed2d87a8686f99a8e8d259 +F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 +F ext/fts5/fts5_index.c 4e94cec64da9a61f8763f033fee310d3ce22805e1452fd4190e3f972ec60dfb0 +F ext/fts5/fts5_main.c 42025174a556257287071e90516d3ab8115daf1dd525a301883544469a260014 +F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 +F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 +F ext/fts5/fts5_test_mi.c 4308d5658cb1f5eee5998dcbaac7d5bdf7a2ef43c8192ca6e0c843f856ccee26 +F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b +F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b +F ext/fts5/fts5_unicode2.c 536a6dae41d16edadd6a6b58c56e2ebbb133f0dfe757562a2edbcdc9b8362e50 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520424dc9d6 +F ext/fts5/fts5_vocab.c 23e263ad94ac357cfffd19bd7e001c3f15c4420fb10fa35b5993142127e780e6 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b -F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9 -F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390 -F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d -F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de -F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7 -F ext/fts5/test/fts5af.test 2329b6c6e6e9243371022dd9439892ff1850b590ae9ce073e3a83eefaf993a81 -F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd -F ext/fts5/test/fts5ah.test 2f047dfe89dc8611fa53e3d8bfc453b79cff037aa423c8d171e91e645745aa2c -F ext/fts5/test/fts5ai.test bc97e4758cc93e06bf851d61c98fdf4e8b8f8315ee28a84fb15f916360856414 -F ext/fts5/test/fts5aj.test 745020852d85f5dd49d11cb7ad11d3cc6dafc4fe6d6d24bc0875ac8f43ee4149 -F ext/fts5/test/fts5ak.test f459a64c9d38698af72a7c657ab6349bca96150241dd69fcce752634b2742d41 -F ext/fts5/test/fts5al.test 00c4c1c6a1366b73aa48ce2068c634520867c3cf7f5d1676ebbb775ee1f35734 -F ext/fts5/test/fts5alter.test 5565f7e4605512b69171ac18ca84398603f9f6456dbe377beeca97e83cc242cd -F ext/fts5/test/fts5auto.test 78989e6527ce69c9eddbef7392fea5c10b0010cd2b2ae68eec7bc869c471e691 -F ext/fts5/test/fts5aux.test ebf6f2ff7cb556e83f66991b7f12bff016d3c83d4eab36704b649dd6b1437318 -F ext/fts5/test/fts5auxdata.test eacc97ff04892f1a5f3d4df5a73f8bcbc3955ea1d12c9f24137eb1fc079e7611 -F ext/fts5/test/fts5bigpl.test 6466c89b38439f0aba26ac09e232a6b963f29b1cbe1304f6a664fe1e7a8f5fd3 +F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe +F ext/fts5/test/fts5aa.test cf4ff6180873bbc131666ba846ddd90148fcb61c20aad089711d3511cce24300 +F ext/fts5/test/fts5ab.test c7e5c1519afb20366cb40d0179897a3c39d9fc06ba6b9c286d79df0ccd97e2ee +F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b +F ext/fts5/test/fts5ad.test 058e616612964e61d19f70295f0e6eaedceb4b29b1fbf4f859615ef7e779dc22 +F ext/fts5/test/fts5ae.test 3d49edbd50bb0684199a2e7568aeb30d1d29718f5c0f61751983740fa836d15f +F ext/fts5/test/fts5af.test ae81f08b8da4c5f9b3ec1ef538a4ab6b7c278e92fa9058d6dc5d842c5d9771b9 +F ext/fts5/test/fts5ag.test 6667807b5d3fbf460892e756763fbe3d87a2fffe345a06514ba010ca6f6641f7 +F ext/fts5/test/fts5ah.test e1f01314b35745a30e1b494b46045b82005d71cae74f1ebd9f1338566b77f9fc +F ext/fts5/test/fts5ai.test cbe26d78030998f535bc103f37915350b137a822c71a9db439a077d7666a3539 +F ext/fts5/test/fts5aj.test 53c8508dab4acca3e691a4c51eca4b3b018319ab8635e540103d5bbdc91543c9 +F ext/fts5/test/fts5ak.test 25e2f8afdcff30d98ca9dee8c5cacca2f26db17501c9401f16d99ee036f70e8d +F ext/fts5/test/fts5al.test f0e655606771b2b5dbaf70e7f0044d560257cf3531d5eea40df58d0d7add8c39 +F ext/fts5/test/fts5alter.test ebbee06419c2d3cee5ef7ebb5ba6a9996f1aa374035361c0acd37368cc5f64f3 +F ext/fts5/test/fts5auto.test 2278de323172ced485d2844cb1357d00036ac1665f27e70fa1a48ce57bf31c2c +F ext/fts5/test/fts5aux.test 27210687338133b1e9bc0dd669322fca59fd432439f40b126895e2d7c2f899d6 +F ext/fts5/test/fts5aux2.test 4f59ac5e7c06c430a9f4890877e10f7b4708e46897422ee6743d27b0a8d01497 +F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06cffec0b37ce4624161 +F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd +F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a -F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 -F ext/fts5/test/fts5colset.test 7031ce84fb4d312df5a99fc4e7b324e660ccb513c97eccdef469bfd52d3d0f8f -F ext/fts5/test/fts5columnsize.test 45459ce4dd9fd853b6044cdc9674921bff89e3d840f348ca8c1630f9edbf5482 -F ext/fts5/test/fts5config.test 60094712debc59286c59aef0e6cf511c37d866802776a825ce437d26afe0817f -F ext/fts5/test/fts5conflict.test 655925678e630d3cdf145d18725a558971806416f453ac8410ca8c04d934238d +F ext/fts5/test/fts5blob.test 9644a5f917306690e08c5f89a470a3f2489376eaa52026eeca3209d149d6af74 +F ext/fts5/test/fts5cat.test bf67dd335f964482ee658287521b81e2b88697b45eb7f73933e15f198ed447cb +F ext/fts5/test/fts5circref.test 0918c69440a73fff429bc9797b07086fc74d018eb3abb1cf9738980390bb2713 +F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 +F ext/fts5/test/fts5columnsize.test 0af91d63985afdf663455d4b572b935238380140d74079eac362760866d3297b +F ext/fts5/test/fts5config.test 017daf10d2642496e97402baa0134de8b5b46b9c37e53c229cd9ab711d21522c +F ext/fts5/test/fts5conflict.test bf6030a77dbb1bedfcc42e589ed7980846c995765d77460551e448b56d741244 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4 -F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283 -F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe -F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f -F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78 -F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3 -F ext/fts5/test/fts5corrupt5.test 550d0884c14424f9acad051a741f1dd99ec9342277d938e91ff3daf9123d1209 -F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4 -F ext/fts5/test/fts5corrupt7.test f3e68673af2514e31dd67a2ed163f7f597252ab683dec155b8db0cdc0b668342 -F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 +F ext/fts5/test/fts5content.test d5c0c2142e64cb305f0968de70c01f8e59dbc3ecc56520c22e739e5dd99ea3bb +F ext/fts5/test/fts5contentless.test 606f063b29ba0f46d4b79aa36cdd1ef4dab5de53eae8c881d731af75a4894aca +F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e2739b358439e9bdcf56ced35f +F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a +F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8 +F ext/fts5/test/fts5contentless5.test 38cd0392c730dc7090c550321ce3c24ba4c392bc97308b51a4180e9959dca7b5 +F ext/fts5/test/fts5corrupt.test 237fce1c3261bb3a5bec333b0f0dbf5b105ec32627ef14cccbda3cfe13833193 +F ext/fts5/test/fts5corrupt2.test 4a03a158c2cb617c9f76d26b35c1ef2534124bc0bbddcea38dfd5b170ebea27b +F ext/fts5/test/fts5corrupt3.test e489b51b6c4ded2a808e0f78bdbe126f6d369de41a59ac2717878e37fc3db0e8 +F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 +F ext/fts5/test/fts5corrupt5.test 73985d4fe6d8f0d5d5c7bcf79ae7c6522c376cd6ad710a0ff2f26e0c2e222abe +F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 +F ext/fts5/test/fts5corrupt7.test 814aab492d7a09abb5bfdd81cc66fc206d7f3868f9a3bae91876e02efc466fb3 +F ext/fts5/test/fts5corrupt8.test 0b10750caf8aa23fa1c379ca4caf6130d41454505e4d5315590f4061eedcbe44 +F ext/fts5/test/fts5corruptbig.test 9f95b40fa36e292feceab02b2ef06e21878bfa1ac7afefa138aae05518b51774 +F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 -F ext/fts5/test/fts5dlidx.test b90852c55881b29dbac6380b274de27beae623ac4b6d567c6c8fb9cdc315a86e -F ext/fts5/test/fts5doclist.test faa9e9cc3c0645fa6203667cb5f007c359447c6ee66753f71a58175c2497cacd -F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 -F ext/fts5/test/fts5eb.test a973baadac524dbbb4ad9b0e99030e12cabde2c6b28e0ac437298007b642cd12 +F ext/fts5/test/fts5dlidx.test a7c42b0a74dc7c8aa1a46d586e0aadda4b6cc42c24450f8d3774b21166e93159 +F ext/fts5/test/fts5doclist.test b7cb84758504519746957802db9cd31187bb4e0028b89d9087ba06e26cc4155f +F ext/fts5/test/fts5ea.test cefdf66024550fa7920c03395c71ce5046235ed1a1a7a469d79b19e7aad5afb5 +F ext/fts5/test/fts5eb.test 401f756fdb77083aeba8b696c1e0ad4d834c39dbd6f17e492bb55a2ad64b4296 +F ext/fts5/test/fts5expr.test c7e208813df7a90badc856fde3796da79569b39382e0fdb43042127f3b8e06a7 F ext/fts5/test/fts5fault1.test d28a65caee75db6897c3cf1358c5230d3bb2a3bf7fb31062c19c7e5382b3d2bd F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421c20a0756251fe344 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522 -F ext/fts5/test/fts5fault4.test 87a10d0caee57da587c7588b0c8d25d2930197399b4812ad1e4d574c75324cee +F ext/fts5/test/fts5fault4.test a5c0e849127c24e1751bc453a817f09a1b8d460e75f9ae4764017e216a870db3 F ext/fts5/test/fts5fault5.test a336e4e11847de24c9497f80cce18e00bb3fab7fb11f97d04eb9af898900a762 -F ext/fts5/test/fts5fault6.test a0fc0a8f99e4b16500c31dfc7e38e1defe0f1693ac47650517ac7b723b1956f8 +F ext/fts5/test/fts5fault6.test 40f49976c6ca8927bf7d65d0b8df46009d7ea172e1d4050b294610e7ea0a2979 F ext/fts5/test/fts5fault7.test 0acbec416edb24b8881f154e99c31e9ccf73f539cfcd164090be139e9e97ed4c -F ext/fts5/test/fts5fault8.test 318238659d35f82ad215ecb57ca4c87486ea85d45dbeedaee42f148ff5105ee2 +F ext/fts5/test/fts5fault8.test 9353fe6a2a993c3231e09c49b0f4a12c8d306319555ff2ca6672b5b86fe9b0dd F ext/fts5/test/fts5fault9.test 098e6b894bbdf9b2192f994a30f4043673fb3f338b6b8ab1624c704422f39119 F ext/fts5/test/fts5faultA.test be4487576bff8c22cee6597d1893b312f306504a8c6ccd3c53ca85af12290c8c F ext/fts5/test/fts5faultB.test d606bdb8e81aaeb6f41de3fc9fc7ae315733f0903fbff05cf54f5b045b729ab5 F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860bbf9e504b9647996 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075 -F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079 -F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717 +F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 +F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 +F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d +F ext/fts5/test/fts5faultI.test 4e3d5a9d3e3b3f17d5e5087ee069414632667719dcfccafd715bc87c72838c72 +F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 +F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e -F ext/fts5/test/fts5hash.test dc7bc7e0cdeb42cfce31294ad2f8fcf43192bfd0145bb7f3ecc5465d8c72696f -F ext/fts5/test/fts5integrity.test 62147a1e85405b986691177e0312be5a64ec9e67b17994e83892d9afa6247600 -F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fdde7ea00e78ea283d227 -F ext/fts5/test/fts5lastrowid.test be98fe3e03235296585b72daad7aed5717ba0062bae5e5c18dd6e04e194c6b28 -F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad +F ext/fts5/test/fts5hash.test fd3e0367fbf0b0944d6936fdb22696350f57b9871069c6766251578a103e8a14 +F ext/fts5/test/fts5integrity.test c423ce16fd1ccadcac7fc22f794226b2bb00f5a187c0ab1d9f8502521b1bae05 +F ext/fts5/test/fts5integrity2.test 4c3636615c0201232c44a8105d5cb14fd5499fd0ee3014d7ffd7e83aac76ece8 +F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba1044c168db0e106f9ac92c +F ext/fts5/test/fts5join.test 48b7ed36956948c5b8456c8bcaa5b087808d99000675918a43c4f51a925f1514 +F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 +F ext/fts5/test/fts5leftjoin.test 1c14b51f4d1344a89e488160882f05a2246dd7e70c5cf077c8fb473e03c66338 F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a -F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6 +F ext/fts5/test/fts5locale.test 83ba7ee12628b540d3098f39c39c1de0c0440eddff8f7512c8c698d0c4a3ae3c +F ext/fts5/test/fts5matchinfo.test bc9e74157773db7f00aec1e85587f1145956ebdf1672c136f0f04323b2752aa0 +F ext/fts5/test/fts5merge.test 088133e135ef7dcd6701753c95b8b10be3c52fa1a99507933e00756d6437489e F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test 416ec0ffbc79320a0760ec32d6684866e3ccd3fbce09f9bcd62d9aee4c666b43 +F ext/fts5/test/fts5misc.test 83d6c5101a092c5db8fb631cfdd69a6482e20528b2750427641ac9050d9d0381 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 -F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd -F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618834bf1fcc3e7b84da -F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 -F ext/fts5/test/fts5optimize2.test c7c97693abe8a2cb572acfb1f252d78f03d3984094cfc5eb2285a76d8a702a92 -F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b -F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a -F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 -F ext/fts5/test/fts5porter2.test 0d251a673f02fa13ca7f011654873b3add20745f7402f108600a23e52d8c7457 -F ext/fts5/test/fts5prefix.test a0fa67b06650f2deaa7bf27745899d94e0fb547ad9ecbd08bfad98c04912c056 -F ext/fts5/test/fts5prefix2.test 3847ce46f70b82d61c6095103a9d7c53f2952c40a4704157bc079c04d9c8b18b -F ext/fts5/test/fts5query.test ac363b17a442620bb0780e93c24f16a5f963dfe2f23dc85647b869efcfada728 -F ext/fts5/test/fts5rank.test c9fd4a1e36b4fa92d572ec13d846469b97da249d1c2f7fd3ee7e017ce46f2416 -F ext/fts5/test/fts5rebuild.test 55d6f17715cddbf825680dd6551efbc72ed916d8cf1cde40a46fc5d785b451e7 -F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415 -F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff171013003d8297203a20ad6 -F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e332d654375690f8e6e43f -F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 +F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74 +F ext/fts5/test/fts5onepass.test b56d4109e841c2bc83555c162515748780ea6e0c455c54cf4afd4bd940d14b84 +F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47 +F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed +F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 +F ext/fts5/test/fts5origintext.test 3b73aa036ce5244bb7c5782c5441b979585bdca026accf75d16026a2a8119c09 +F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba +F ext/fts5/test/fts5origintext3.test 4988b6375acc3bbb0515667765f57e389caf449814af9c1095c053f7de2b4223 +F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 +F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b +F ext/fts5/test/fts5origintext6.test 09eb1347cb0dceaebbebf3d3e6bd5d24c7c1006efddc2984540450324bbdafa4 +F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c +F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626 +F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0 +F ext/fts5/test/fts5porter2.test 94f0e4351e2c99b4e74f1fae05a4ddf1cb5b926620a8c14554160d075ddc7a59 +F ext/fts5/test/fts5prefix.test c0b7842f1a2d830c0b146cd438a95ea4c5a25635719ed0d973ffe41907338b83 +F ext/fts5/test/fts5prefix2.test a5bb43b8a2687efafa7ac4e5ccff6812015cf8cf18e3086bb0eb3126f30fbbf6 +F ext/fts5/test/fts5query.test 0320a7a4b58a6e3e50ec8910b301649da90ace675001f9e0bf6392750ad4591d +F ext/fts5/test/fts5rank.test 47c1e8e5d84754ff18e012fdd629776088b5a15de41bdd24957581cf084d8a00 +F ext/fts5/test/fts5rebuild.test dc09779fbbe151ab68206a0931c10a611912a7a12c7a85d71c5e48453f2375a5 +F ext/fts5/test/fts5restart.test 9af2084b8e065130037b95f05f3f220bb7973903a7701e2c5fb916dff7cf80c5 +F ext/fts5/test/fts5rowid.test 8632829fec04996832a4cfb4f0bd89721ba65b7e398c1731741bdb63f070e1a3 +F ext/fts5/test/fts5savepoint.test 1447758d7900afe903cef08b4524c5331fb60c1126ae6fba7f4d8704268013c5 +F ext/fts5/test/fts5secure.test a02f771742fb2b1b9bdcb4bf523bcf2d0aa1ff597831d40fe3e72aaa6d0ec40f F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc -F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 +F ext/fts5/test/fts5secure3.test 6d066828d225b0dbe5db818d4d6165df7bb70210e68a577e858e8762400d5a23 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a -F ext/fts5/test/fts5secure6.test 7a959d834be6725c641b3c3b38ef86570ea671216ad803e054e4fdff33a72ce2 -F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b -F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b -F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 -F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f -F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb -F ext/fts5/test/fts5synonym2.test b54cce5c34ec08ed616f646635538ae82e34a0e28f947ec60b6fadbc4b3fb17a +F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd +F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a +F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0 +F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 +F ext/fts5/test/fts5simple.test 302cdb4f8a3350b091f4f1bccd82d05610428657f6f9e81c17703ba48267ec40 +F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc +F ext/fts5/test/fts5simple3.test 4e03b82e669dc07bf9c9a04c306fa493764bc49c93e539896d87d88bd374fece +F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 +F ext/fts5/test/fts5synonym2.test 58f357b997cf2fedeeb9d0de4db9f880fa96fa2fe27a743bfe7d7b96895bdd87 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 -F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43 -F ext/fts5/test/fts5trigram.test c76acc1913a06182e791a0dfdae285b9cdd67327a1a35b34cabf0a6aa09cf05e -F ext/fts5/test/fts5ubsan.test 783d5a8d13ebfa169e634940228db54540780e3ba7a87ad1e4510e61440bf64b +F ext/fts5/test/fts5tokendata.test 7cad79af82e8e81b7a36b450087233d2fca051bb0d584421afc375d6dd26d6f6 +F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 +F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6 +F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 +F ext/fts5/test/fts5trigram.test a55fde7065ae69a0f82c5a7a5bf5286a97de11ae4bff6537fd3e27ca9a01416f +F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61 +F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 -F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9 -F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e -F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c75ae10f4afc6faa0744 -F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687 -F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d +F ext/fts5/test/fts5unicode.test 41898f7e476e6515cd4b737c02a442cda5a580a74509788aa9072a2074948e0e +F ext/fts5/test/fts5unicode2.test 3bbd30152f9f760bf13886e5b1e5ec23ff62f56758ddda5d9c775a6082fb4c7c +F ext/fts5/test/fts5unicode3.test f4891a3dac3b49c3d7c0fdb29566e9eb0ecff35263370c89f9661b1952b20818 +F ext/fts5/test/fts5unicode4.test 6d70dbe56e5179bb1990cfb22e62fdf2aae9458e443ade856e598ce95832fe9b +F ext/fts5/test/fts5unindexed.test 168838d2c385e131120bbf5b516d2432a5fabc4caa2259c932e1d49ae209a4ae +F ext/fts5/test/fts5unindexed2.test 516236eceaac05ace322290a0d3705b4c4ffe4760d8eb9d014d9d27d56dfcc02 F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc -F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67 -F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760 -F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68 +F ext/fts5/test/fts5update2.test c5baa76799ac605ebb8e5e21035db2014b396cef25c903eb96ba39b1d6f9f046 +F ext/fts5/test/fts5version.test 44ab35566267b7618c090443de2d9ad84f633df5d20bf72e9bad199ae5fced84 +F ext/fts5/test/fts5vocab.test 2a2bdb60d0998fa3124d541b6d30b019504918dc43a6584645b63a24be72f992 +F ext/fts5/test/fts5vocab2.test 4265137a3747b27deb1e2e2bde5654120c6de72bfed3238e67806d85af60fc4c F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 -F ext/fts5/tool/mkfts5c.tcl 3eba8e9bee4221ed165f3304b51b2a74a705f4ec5df3d044573a2be539534af8 +F ext/fts5/tool/mkfts5c.tcl 135b9e160f8e10211c10c5873d5e8c3eaebd3da9ec56a12ae4db157d4738ffe4 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c -F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 -F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 +F ext/icu/README.txt 1f8d76e10d2385fc77914a14ccd99acfbaf68111dfcf26a360ad9063787f57fb +F ext/icu/icu.c 9837f4611915baad1edbe38222f3ee7d1b5e118ab16fec9ba603720f72c78b2a F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/lsm1/Makefile a553b728bba6c11201b795188c5708915cc4290f02b7df6ba7e8c4c943fd5cd9 -F ext/lsm1/Makefile.msc f8c878b467232226de288da320e1ac71c131f5ec91e08b21f502303347260013 -F ext/lsm1/lsm-test/README 87ea529d2abe615e856d4714bfe8bb185e6c2771b8612aa6298588b7b43e6f86 -F ext/lsm1/lsm-test/lsmtest.h cf58528ffe0cfe535e91b44584e2ec5fb1caacdabecef0d8dcf83bf83168bf28 -F ext/lsm1/lsm-test/lsmtest1.c 54374fe88cee888c52c31160013c26184288f47a45b23d4d85390aa539733aab -F ext/lsm1/lsm-test/lsmtest2.c 188b09aec776516aeedcfd13b9c6faf85ba16b3671a0897a2c740ee00a5dc4f8 -F ext/lsm1/lsm-test/lsmtest3.c 9ab87528a36dbf4a61d7c8ad954f5ee368c0878c127b84b942b2e2abe522de26 -F ext/lsm1/lsm-test/lsmtest4.c d258d6a245db5d8eaede096e2368d23f859c5e92c80ab9122463f708514fe10c -F ext/lsm1/lsm-test/lsmtest5.c 8d5242a0f870d65eeada191c8945781fed9cb8ece3886573790ebd373b62dac5 -F ext/lsm1/lsm-test/lsmtest6.c 869cb4a172cd07d1a75b3aeaecd61d0a477787b3b8668bad0d3ff0f43b642b7c -F ext/lsm1/lsm-test/lsmtest7.c 7a917455a0f956a8ed3f44f5c9387ec0ea6627714874464cc3fa5c5a9cabb2f2 -F ext/lsm1/lsm-test/lsmtest8.c 773f226163d0f0d62701e3764d0c35fd4365faca74098bd63648bc57d6f14402 -F ext/lsm1/lsm-test/lsmtest9.c 0a168757b757b106191acf43143dbbb5b2d76e57a3c8fd3018cecbaee1080aba -F ext/lsm1/lsm-test/lsmtest_bt.c 79b24bfd37e05fd626c35ec23bc5bb62d8a403afd66c710335384884dc1366d7 -F ext/lsm1/lsm-test/lsmtest_datasource.c 5d770be191d0ca51315926723009b2c25c0b4b8136840494ef710ac324aa916c -F ext/lsm1/lsm-test/lsmtest_func.c 159aa401bc8032bfa3d8cf2977bd687abebab880255895a5eb45770d626fa38d -F ext/lsm1/lsm-test/lsmtest_io.c cf11b27b129c6bd5818fa1d440176502dc27229f0db892b4479118d61993ea20 -F ext/lsm1/lsm-test/lsmtest_main.c a9bc647738c0dcaebf205d6d194b3ce4a6ef3925801cd2d919f0a4ea33a15aeb -F ext/lsm1/lsm-test/lsmtest_mem.c 4e63c764345ab1df59d4f13a77980c6f3643798210b10d6cdbd785b4b888fda5 -F ext/lsm1/lsm-test/lsmtest_tdb.c 754b1ca8e1cfa7b29cbe2e4ab500f7eee0059033741b8d83267afe6f495a536d -F ext/lsm1/lsm-test/lsmtest_tdb.h 8733eee249b12956a9df8322994b43d19bd8c02ad2e8b0bb5164db4d6ccc1735 -F ext/lsm1/lsm-test/lsmtest_tdb2.cc aebe50f2cb7a759214241938046fe5f00da66e4217637f946f436ca209776af9 -F ext/lsm1/lsm-test/lsmtest_tdb3.c 7a7ccae189f5bb25bcd1ec3bbd740529706eded7f6729a5a0a9eeaeb57785320 -F ext/lsm1/lsm-test/lsmtest_tdb4.c cbe230727b9413d244062943371af1421ace472ccb023b75af6540e0fa52b1bb -F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806 -F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f5261e1b4cc19068641da2d -F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001 -F ext/lsm1/lsmInt.h 3bcc280347196e4ed14925b64a07685415238bf41317db0598c8d3f6aaceb9c1 -F ext/lsm1/lsm_ckpt.c ad9a8028d401be9e76f20c4d86d49f33f4fc27785577b452ca955094314a72b4 -F ext/lsm1/lsm_file.c 5486f4a63b19e4d7d972ee2482f29ebdf06c29544f31845f713cccb5199f9ad1 -F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c -F ext/lsm1/lsm_main.c 87770a9c7e73859fce7620cb79623776ba4b30369086229ad82c3e6eeaf45457 -F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a -F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea -F ext/lsm1/lsm_shared.c c67282a4f2c91e2a3362bdd40a81f9041cd587973ffc4bca8b8fbdab5470dee1 -F ext/lsm1/lsm_sorted.c bc276055afc21e7f23538d39d7cf2722379b56c79778ab7232f710e3374d501c -F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 -F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb -F ext/lsm1/lsm_unix.c 11e0a5c19d754a4e1d93dfad06de8cc201f10f886b8e61a4c599ed34e334fc24 -F ext/lsm1/lsm_varint.c fe134ad7b2db1ecd99b6a155d2f3625cfd497730e227ae18892452e457b73327 -F ext/lsm1/lsm_vtab.c e57aa3eb456bf2b98064014027e097c9402d6dec7b59564ddbfa1c0ead8f96c5 -F ext/lsm1/lsm_win32.c 0a4acbd7e8d136dd3a5753f0a9e7a9802263a9d96cef3278cf120bcaa724db7c -F ext/lsm1/test/lsm1_common.tcl 5ed4bab07c93be2e4f300ebe46007ecf4b3e20bc5fbe1dedaf04a8774a6d8d82 -F ext/lsm1/test/lsm1_simple.test a04d08e8661ae6fc53786c67f0bd102c6692f003e859dde03ed9ac3f12e066e5 -F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f6078e07335398b0 -F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 -F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 +F ext/intck/intck1.test 53d885075abeb45aeb1eeffeaa8560b329060835ade4af5c44cf5fcb581c1e63 +F ext/intck/intck2.test a29343a8e65c5c3400e10747f394924f3df95a5b2de94f46e9b5c9b97f5e7339 +F ext/intck/intck_common.tcl a61fd2697ae55b0a3d89847ca0b590c6e0d8ff64bebb70920d93724799894159 +F ext/intck/intckbusy.test d5ed4ef85a4b1dc1dee2484bd14a4bb68529659cca743327df0c775f005fa387 +F ext/intck/intckcorrupt.test f6c302792326fb3db9dcfc70b554c55369bc4b52882eaaf039cfe0b74c821029 +F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028f051e985efab +F ext/intck/sqlite3intck.c b1c8a86f90fc00741d13314db9c58f7e2f92d1d19c5ad1c6904ec83a6bbd5c96 +F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 +F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 +F ext/jni/GNUmakefile 8a94e3a1953b88cf117fb2a5380480feada8b4f5316f02572cab425030a720b4 +F ext/jni/README.md e3fbd47c774683539b7fdc95a667eb9cd6e64d8510f3ee327e7fa0c61c8aa787 +F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa +F ext/jni/src/c/sqlite3-jni.c 3d84a0176af779737ae977ba1c90d2ffe2537b8299c5d9f6552620493f12ac4b +F ext/jni/src/c/sqlite3-jni.h ac180ba9b21978727006c790d3006a95a2402a4c3ec7a0def92707ed89b79945 +F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b +F ext/jni/src/org/sqlite/jni/annotation/NotNull.java be6cc3e8e114485822331630097cc0f816377e8503af2fc02f9305ff2b353917 +F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 56e3dee1f3f703a545dfdeddc1c3d64d1581172b1ad01ffcae95c18547fafd90 +F ext/jni/src/org/sqlite/jni/annotation/package-info.java 977b374aed9d5853cbf3438ba3b0940abfa2ea4574f702a2448ee143b98ac3ca +F ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java 1afa90d3f236f79cc7fcd2497e111992644f7596fbc8e8bcf7f1908ae00acd6c +F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0e28a0df51368c7127e505f1e9acd92a7e66e035bcd6288463aa691cb300c9af +F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a +F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759 +F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca +F ext/jni/src/org/sqlite/jni/capi/CApi.java 3d275f5f4fbdbe4fff15f4d42cf5ff559f9a4897e7373fa99f3b1dc9cf7f849c +F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 1b3baf5b772f266e8baf8f35f0ddc5bd87fc8c4927ec69115c46fd6fba6701c3 +F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a +F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab +F ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java 482f53dfec9e3ac2a9070d3fceebd56250932aaaf7c4f5bc8de29fc011416e0c +F ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b995ca412f59b631803b93aa5b3684fce62e335d1e123207084c054abfd488d4 +F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7 +F ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b7036dcb1ef1b39f1f36ac605dde0ff1a24a9a01ade6aa1a605039443e089a61 +F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 418a82e705ec80b0dabffacfe11b9fab3cb5f2215dcafcfec083eebf5bce9d20 +F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java 621cd9d887956fb2c99c9be59ee831c00a0b538dbae7313c525cd2936b5d5647 +F ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java efcf57545c5e282d1dd332fa63329b3b218d98f356ef107a9dbe3979be82213a +F ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java 01bc0c238eed2d5f93c73522cb7849a445cc9098c2ed1e78248fa20ed1cfde5b +F ext/jni/src/org/sqlite/jni/capi/ResultCode.java 8141171f1bcf9f46eef303b9d3c5dc2537a25ad1628f3638398d8a60cacefa7f +F ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java e172210a2080e851ebb694c70e9f0bf89284237795e38710a7f5f1b61e3f6787 +F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385155fa3b8011a5cca0bb3c28468c7131c1a5 +F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 3c0babc067d8560627a9ed1b07979f9d4393464e2282c2fca4832052e982c7bc +F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615 +F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java 9133bb7685901d2edf07801191284975e33b5583ce09dce1c05202ff91e7bb99 +F ext/jni/src/org/sqlite/jni/capi/Tester1.java 4c3d16fdf6e979f839b2ecdb14d0a0c04bd3d0e41500fc9e8110b588883b140b +F ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java 0a25e117a0daae3394a77f24713e36d7b44c67d6e6d30e9e1d56a63442eef723 +F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java c8bdf7848e6599115d601bcc9427ff902cb33129b9be32870ac6808e04b6ae56 +F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 2ce069f3e007fdbbe1f4e507a5a407fc9679da31a0aa40985e6317ed4d5ec7b5 +F ext/jni/src/org/sqlite/jni/capi/WindowFunction.java caf4396f91b2567904cf94bc538a069fd62260d975bd037d15a02a890ed1ef9e +F ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java 9c8cc33995a3df385feaf476a8306d29dbab611ed4f55da736597357bde68620 +F ext/jni/src/org/sqlite/jni/capi/package-info.java 08ff986a65d2be9162442c82d28a65ce431d826f188520717c2ecb1484d0a50e +F ext/jni/src/org/sqlite/jni/capi/sqlite3.java c6a5c555d163d76663534f2b2cce7cab15325b9852d0f58c6688a85e73ae52f0 +F ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java 6742b431cd4d77e8000c1f92ec66265a58414c86bf3b0b5fbcb1164e08477227 +F ext/jni/src/org/sqlite/jni/capi/sqlite3_blob.java 59e26ca5254cd4771f467237bcfe2d8deed30a77152fabcd4574fd406c301d63 +F ext/jni/src/org/sqlite/jni/capi/sqlite3_context.java f0ef982009c335c4393ffcb68051809ca1711e4f47bcb8d1d46952f22c01bc22 +F ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java 293b5fa7d5b5724c87de544654aca1103d76f3092bc2c8f4360102a65ba25dff +F ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java e1d62a257c13504b46d39d5c21c49cf157ad73fda00cc5f34c931aa008c37049 +F ext/jni/src/org/sqlite/jni/fts5/Fts5.java e94681023785f1eff5399f0ddc82f46b035977d350f14838db659236ebdf6b41 +F ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java 338637e6e5a2cc385d962b220f3c1f475cc371d12ae43d18ef27327b6e6225f7 +F ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java 2de029b3a12b16f779f4df6381c5f0cd358dd82bdaf99ec837504a1110b829f3 +F ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java 28045042d593a1f1b9b80d54ec77cbf1d8a1bc95e442eceefa9a3a6f56600b0e +F ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java 3c8f677ffb85b8782f865d6fcbc16200b3375d0e3c29ed541a494fde3011bf49 +F ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java 793a5f6f2f581034dc3b503d5023b05e7e38558a80542148b82527dc2a7bf209 +F ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java 1efd1220ea328a32f2d2a1b16c735864159e929480f71daad4de9d5944839167 +F ext/jni/src/org/sqlite/jni/fts5/fts5_api.java a8e88c3783d21cec51b0748568a96653fead88f8f4953376178d9c7385b197ea +F ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java 1e721873c62df2eb79f45bbf55b8662625365885b02d1d51915f773de16a90e3 +F ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java 92bdaa3893bd684533004d64ade23d329843f809cd0d0f4f1a2856da6e6b4d90 +F ext/jni/src/org/sqlite/jni/test-script-interpreter.md d7987b432870d23f7c72a7804d099fe5ccfb945f519ac90a33e189297fbbfa1c +F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483 +F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 326ffba29aab836a6ea189703c3d7fb573305fd93da2d14b0f9e9dcf314c8290 +F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java e920f7a031e04975579240d4a07ac5e4a9d0f8de31b0aa7a4be753c98ae596c9 +F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java c82bc00c1988f86246a89f721d3c41f0d952f33f934aa6677ec87f7ca42519a0 +F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 982538ddb4c0719ef87dfa664cd137b09890b546029a7477810bd64d4c47ee35 +F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java 08f92d52be2cec28a7b4555479cc54b7ebeeb94985256144eeb727154ec3f85b +F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java a84e90c43724a69c2ecebd601bc8e5139f869b7d08cb705c77ef757dacdd0593 +F ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java c7d1452f9ff26175b3c19bbf273116cc2846610af68e01756d755f037fe7319f +F ext/jni/src/tests/000-000-sanity.test c3427a0e0ac84d7cbe4c95fdc1cd4b61f9ddcf43443408f3000139478c4dc745 +F ext/jni/src/tests/000-001-ignored.test e17e874c6ab3c437f1293d88093cf06286083b65bf162317f91bbfd92f961b70 +F ext/jni/src/tests/900-001-fts.test bf0ce17a8d082773450e91f2388f5bbb2dfa316d0b676c313c637a91198090f0 +F ext/misc/README.md 6243cdc4d7eb791c41ef0716f3980b8b5f6aa8c61ff76a3958cbf0031c6ebfa7 +F ext/misc/amatch.c 0e0124c1e03ee4cb99b25969f6b7b39c53a847b8bf12279efbcb896b0df1059a F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c a71b131e50300c654a66c469a25b62874481f3d1cb3beb56aca9a68edd812e0d -F ext/misc/base85.c 073054111988db593ef5fdb87ab8c459df1ea0c3aaaddf0f5bfa3d72b7e6280a +F ext/misc/base64.c 8dc0a08cee11722822858a62625f1b63e5d5f1adac1cf4492d5732b571e37aa0 +F ext/misc/base85.c ff54cc676c6ec86231f75ecc86ea45416fcb69751dfb79690d5f5da5f7d39867 F ext/misc/basexx.c 89ad6b76558efbceb627afd5e2ef1d84b2e96d9aaf9b7ecb20e3d00b51be6fcf F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a -F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 -F ext/misc/carray.c 0ba03f1e6647785d4e05b51be567f5652f06941314ff9d3d3763900aa353b6b5 -F ext/misc/carray.h 503209952ccf2431c7fd899ebb92bf46bf7635b38aace42ec8aa1b8d7b6e98a5 -F ext/misc/cksumvfs.c 9224e33cc0cb6aa61ff1d7d7b8fd6fe56beca9f9c47954fa4ae0a69bef608f69 -F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c85c243 -F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 -F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 -F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 -F ext/misc/decimal.c 57d85fa20a5a74d3b0dfc78ab7934ae6c9f5aa8eed915faa2b5246bec87ddc6d +F ext/misc/btreeinfo.c 8f5e6da2c82ec2f06ee0216e922370a436dafdbb06ffa7a552203515ff9e7ddf +F ext/misc/cksumvfs.c 9d7d0cf1a8893ac5d48922bfe9f3f217b4a61a6265f559263a02bb2001259913 +F ext/misc/closure.c 5559daf1daf742228431db929d1aa86dd535a4224cc634a81d2fd0d1e6ad7839 +F ext/misc/completion.c c27b64fdd0943c1b7f152376599814cee2641f7d67a7bb9bd2b957c2a64a5591 +F ext/misc/compress.c 8191118b9b73e7796c961790db62d35d9b0fb724b045e005a5713dc9e0795565 +F ext/misc/csv.c d9dab032420d81cdf0abc4b8523711d20a355704f82d958f963c86be48387dd9 +F ext/misc/dbdump.c 678f1b9ae2317b4473f65d03132a2482c3f4b08920799ed80feedd2941a06680 +F ext/misc/decimal.c d4883de142f6dcd36eda23da40b55e2b51374e7b01eb54a7173940191389fc5e F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 -F ext/misc/explain.c 0086fab288d4352ea638cf40ac382aad3b0dc5e845a1ea829a694c015fd970fe -F ext/misc/fileio.c 4e7f7cd30de8df4820c552f14af3c9ca451c5ffe1f2e7bef34d598a12ebfb720 -F ext/misc/fossildelta.c 1240b2d3e52eab1d50c160c7fe1902a9bd210e052dc209200a750bbf885402d5 -F ext/misc/fuzzer.c eae560134f66333e9e1ca4c8ffea75df42056e2ce8456734565dbe1c2a92bf3d -F ext/misc/ieee754.c 984d51fe23e956484ec1049df6f5257002e3ab338cabceb39761c2e80ad10bf4 -F ext/misc/memstat.c 3017a0832c645c0f8c773435620d663855f04690172316bd127270d1a7523d4d +F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b +F ext/misc/fileio.c d80268a5328fe839062a9d3103ea0fc7cacc6d42605959275675cb37867c84f7 +F ext/misc/fossildelta.c 2fc2dd4f34f478df674887db62586b1071c4cd3c9e73ee40f9ee669670e482d1 +F ext/misc/fuzzer.c 6b231352815304ba60d8e9ec2ee73d4918e74d9b76bda8940ba2b64e8777515e +F ext/misc/ieee754.c 176c061c94857b543313959289cb60cf777c999fd002f82b53d194b95e9f347a +F ext/misc/memstat.c 43705d795090efb78c85c736b89251e743c291e23daaa8382fe7a0df2c6a283d F ext/misc/memtrace.c 7c0d115d2ef716ad0ba632c91e05bd119cb16c1aedf3bec9f06196ead2d5537b -F ext/misc/memvfs.c 7dffa8cc89c7f2d73da4bd4ccea1bcbd2bd283e3bb4cea398df7c372a197291b -F ext/misc/mmapwarm.c 347caa99915fb254e8949ec131667b7fae99e2a9ce91bd468efb6dc372d9b7a9 +F ext/misc/mmapwarm.c a81af4aaec00f24f308e2f4c19bf1d88f3ac3ce848c36daa7a4cd38145c4080d F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58f34edd -F ext/misc/noop.c 81efe4cad9ec740e64388b14281cb983e6e2c223fed43eb77ab3e34946e0c1ab -F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f -F ext/misc/percentile.c b9086e223d583bdaf8cb73c98a6539d501a2fc4282654adbfea576453d82e691 -F ext/misc/prefixes.c 0f4f8cff5aebc00a7e3ac4021fd59cfe1a8e17c800ceaf592859ecb9cbc38196 -F ext/misc/qpvtab.c 09738419e25f603a35c0ac8bd0a04daab794f48d08a9bc07a6085b9057b99009 -F ext/misc/randomjson.c 7dd13664155319d47b9facc0d8dbf45e13062966a47168e54e3f26d48240d7ea -F ext/misc/regexp.c 4bdd0045912f81c84908bd535ec5ad3b1c8540b4287c70ab84070963624047db +F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5a0 +F ext/misc/normalize.c fbb144a861809686ff2b5b6eee8bb2e1207f9bf13ce7376e5273c700a1eafbd5 +F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 +F ext/misc/percentile.c 72e05a21db20a2fa85264b99515941f00ae698824c9db82d7edfbb16cea8ec80 +F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 +F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c +F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed +F ext/misc/regexp.c f1f7cfe90fc027b33d2b5ae7d6235eecce69c3aca71c9afce56fec62342c8b44 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c d4001360b2b05a831bbad378e131bd585b29499482e3f2557e86dbd3e2367f25 -F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac +F ext/misc/series.c 22c6d8f00cc1b5089b1b37392e9097e9df9a5db53be86daf9a7669d95bb179f4 +F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 +F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f -F ext/misc/sqlar.c 53e7d48f68d699a24f1a92e68e71eca8b3a9ff991fe9588c2a05bde103c6e7b7 -F ext/misc/stmt.c bc30d60d55e70d0133f10ac6103fe9336543f673740b73946f98758a2bb16dd7 -F ext/misc/templatevtab.c 8a16a91a5ceaccfcbd6aaaa56d46828806e460dd194965b3f77bf38f14b942c4 -F ext/misc/totype.c fa4aedeb07f66169005dffa8de3b0a2b621779fd44f85c103228a42afa71853b -F ext/misc/uint.c 053fed3bce2e89583afcd4bf804d75d659879bbcedac74d0fa9ed548839a030b -F ext/misc/unionvtab.c 36237f0607ca954ac13a4a0e2d2ac40c33bc6e032a5f55f431713061ef1625f9 +F ext/misc/spellfix.c 693c8fd3293087fa821322967a97e59dfa24051e5d2ca7fa85790a4034db6fa4 +F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 +F ext/misc/sqlite3_stdio.c 0fe5a45bd332b30aef2b68c64edbe69e31e9c42365b0fa79ce95a034bca6fbb0 +F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176 +F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 +F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc +F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b +F ext/misc/totype.c ba11aac3c0b52c685bd25aa4e0f80c41c624fb1cc5ab763250e09ddc762bc3a8 +F ext/misc/uint.c 327afc166058acf566f33a15bf47c869d2d3564612644d9ff81a23efc8b36039 +F ext/misc/unionvtab.c 716d385256d5fb4beea31b0efede640807e423e85c9784d21d22f0cce010a785 F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917b9c751 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 -F ext/misc/vfsstat.c 474d08efc697b8eba300082cb1eb74a5f0f3df31ed257db1cb07e72ab0e53dfb -F ext/misc/vtablog.c 5538acd0c8ddaae372331bee11608d76973436b77d6a91e8635cfc9432fba5ae -F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd -F ext/misc/wholenumber.c a838d1bea913c514ff316c69695efbb49ea3b8cb37d22afc57f73b6b010b4546 -F ext/misc/zipfile.c b1f36004c19fb5f949fb166fc4ab88e96a86f66629e9ddb4736a45b63fc3d553 -F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 +F ext/misc/vfsstat.c 0b23c0a69a2b63dc0ef0af44f9c1fc977300c480a1f7a9814500369d8211f56e +F ext/misc/vfstrace.c 0e4b8b17ac0675ea90f6d168d8214687e06ca3efbc0060aad4814994d82b41fb +F ext/misc/vtablog.c 2d04386c2f5a3bb93bc9ae978f0b7dcd5a264e126abd640dd6d82aa9067cbd48 +F ext/misc/vtshim.c e5bce24ab8c532f4fdc600148718fe1802cb6ed57417f1c1032d8961f72b0e8f +F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 +F ext/misc/windirent.h 02211ce51f3034c675f2dbf4d228194d51b3ee05734678bad5106fff6292e60c +F ext/misc/zipfile.c 71d3fd3155ed5e738473e286e550cf0bcf346cc2fd63646eaf944e7b40531a1b +F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363 @@ -369,378 +457,387 @@ F ext/rbu/rbusave.test 588b618dad9d65c4b13d03a79931de82213503fedc26bdf5789c996ec F ext/rbu/rbusplit.test a6dedd23cf37bcf2e8646d9d7139846e96d60d92f9bc6d6ba6ca8c24c0bd1f72 F ext/rbu/rbutemplimit.test 4980df2d4b74f4dd982add8f78809106154ef5a3c4bdce747422ab0b0481e029 F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f323783b15311e -F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b +F ext/rbu/rbuvacuum2.test 1a9bd41f127be2826de2a65204df9118525a8af8d16e61e6bc63ba3ac0010a23 F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9 -F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 -F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c 31d580785cf14eb3c20ed6fbb421a10a66569858f837928e6b326088c38d4c72 -F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf +F ext/rbu/sqlite3rbu.c c208f72f20784bf2f39244b6cdf8019724a706e1246be289e7621c42aad54695 +F ext/rbu/sqlite3rbu.h e3a5bf21e09ca93ce4e8740e00d6a853e90a697968ec0ea98f40826938bdb68e +F ext/rbu/test_rbu.c 8b6e64e486c28c41ef29f6f4ea6be7b3091958987812784904f5e903f6b56418 +F ext/recover/dbdata.c 10d3c56968a9af6853722a47280805ad1564714d79ea45ac6f7da14bb57fd137 +F ext/recover/recover1.test e16d78e94183562abff569967b18b7c77451d7044365516cd0fe14713a284851 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 -F ext/recover/recovercorrupt2.test 74bef7dd2d7dd4856f3da21be6e213d27da44827e0f5f0946ca0325b46d163ed +F ext/recover/recovercorrupt2.test 7347ccc9c36a925d99b56689c791423b45294834198f17575183fd500f52d85d +F ext/recover/recovercorrupt3.test 2e7b9a1b528ca23ed382cec6f64e3fcbbd0f8e852add7562397fd8df83f335d5 +F ext/recover/recovercorrupt4.test 3e2794145dad2517c018cb68b96f59d4d55b18b3d6271e1d37852cfd7a30b50c F ext/recover/recoverfault.test 9d9f88eeb222615a25e7514f234c950d46bee20d24cd8db49d8fff8d650dcfe1 F ext/recover/recoverfault2.test 730e7371bcda769554d15460cb23126abba1be8eca9539ccabf63623e7bb7e09 F ext/recover/recoverold.test 68db3d6f85dd2b98e785b6c4da4f5eea4bbe52ccf6674d9a94c7506dc92596aa -F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1be847182410ffc58 +F ext/recover/recoverpgsz.test 88766fcb810e52ee05335c456d4e5fb06d02b73d3ccb48c52bf293434305e2b1 F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f -F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 +F ext/recover/recoverslowidx.test c90d59c46bb8924a973ac6fbc38f3163cee38cc240256addcab1cf1a322c37dc F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c b3813090c97df34858bab8ece4e8c49d9e2f56ba2b9bb019f57f44bc234b3c6d +F ext/recover/sqlite3recover.c 56c216332ea91233d6d820d429f3384adbec9ecedda67aa98186b691d427cc57 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 -F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade +F ext/recover/test_recover.c 3d0fb1df7823f5bc22a0b93955034d16a2dfa2eb1e443e9a0123a77f120599a3 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 -F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 +F ext/repair/checkindex.c 7639b4f8928f82c10b950169e60cc45a7f6798df0b299771d17bef025736f657 F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 F ext/repair/sqlite3_checker.tcl a9a2caa9660567257c177a91124d8c0dccdfa341e25c51e6da7f1fd9e601eafa F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c -F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/geopoly.c 971e0b5bd9adaf0811feb8c0842a310811159da10319eb0e74fdb42bf26b99ca -F ext/rtree/rtree.c 925888f7672b326fc2a29b3a212ec3ae4aa2331507ecccfaf7f0ac0335020330 +F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 +F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb +F ext/rtree/rtree.c 9331997a76b88a9bc04e156bdfd6e2fe35c0aa93bc338ebc6aa0ae470fe4a852 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 -F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11 +F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d F ext/rtree/rtree3.test 272594f88c344e973864008bbe4c71fd3a41a264c097d568593ee7886d83d409 F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b F ext/rtree/rtree5.test 49c9041d713d54560b315c2c7ef7207ee287eba1b20f8266968a06f2e55d3142 F ext/rtree/rtree6.test 2f5ffc69670395c1a84fad7924e2d49e82a25460c5293fb1e54e1aa906f04945 F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5 -F ext/rtree/rtree8.test 2d99006a1386663978c9e1df167554671e4f711c419175b39f332719deb1ce0e +F ext/rtree/rtree8.test 4da84c7f328bbdca15052fa13da6e8b8d426433347bf75fc85574c2f5a411a02 F ext/rtree/rtree9.test fd3c9384ef8aabbc127b3878764070398f136eebc551cd20484b570f2cc1956a -F ext/rtree/rtreeA.test a7fd235d8194115fa2e14d300337931eb2e960fe8a46cdfb66add2206412ea41 -F ext/rtree/rtreeB.test 4cec297f8e5c588654bbf3c6ed0903f10612be8a2878055dd25faf8c71758bc9 +F ext/rtree/rtreeA.test 14e67fccc5b41efbad7ea99d21d11aaa66d2067da7d5b296ee86e4de64391d82 +F ext/rtree/rtreeB.test ab93136c45cf25af78d22665c2a6d75068eef6bf3a710356e4ba8d5f37bed364 F ext/rtree/rtreeC.test 2978b194d09b13e106bdb0e1c5b408b9d42eb338c1082bf43c87ef43bd626147 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9 F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331 F ext/rtree/rtreeG.test 1b9ca6e3effb48f4161edaa463ddeaa8fca4b2526d084f9cbf5dbe4e0184939c -F ext/rtree/rtreeH.test 0885151ee8429242625600ae47142cca935332c70a06737f35af53a7bd7aaf90 +F ext/rtree/rtreeH.test c304651ee87dbb60296366d2c802f3043c3d00506c79a5c85bd20f2c9274498b F ext/rtree/rtreeI.test 608e77f7fde9be5a12eae316baef640fffaafcfa90a3d67443e78123e19c4ca4 +F ext/rtree/rtreeJ.test 93227ccd4d6c328f5ac46a902b8880041509dd2d68f6ce71560f0d8ab5bb507a F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 -F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed -F ext/rtree/rtreecheck.test 4e859a9cd49d2353ff10c122f72183ec37b400e35d2b0349b2e9696649b6a00e +F ext/rtree/rtree_util.tcl 202ca70df1f0645ef9d5a2170e62d378a28098d9407f0569e85c9c1cf1bd020a +F ext/rtree/rtreecheck.test 84eedb43b25b3edf591125266d0bb1cebdfcdcc9c4a56b27d85bcb63c7dd7558 F ext/rtree/rtreecirc.test aec664eb21ae943aeb344191407afff5d392d3ae9d12b9a112ced0d9c5de298e F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d -F ext/rtree/rtreedoc.test 27a5703cb1200f6f69051de68da546cef3dfdcf59be73afadfc50b9f9c9960d9 -F ext/rtree/rtreedoc2.test 194ebb7d561452dcdc10bf03f44e30c082c2f0c14efeb07f5e02c7daf8284d93 +F ext/rtree/rtreedoc.test d633982d61542f3bc0a0a2df0382a02cc699ac56cbda01130cde6da44a228490 +F ext/rtree/rtreedoc2.test 0102b4e60f1baf0039bef7c1c4b39f1d3d77a68a5741513a0c190db28b884835 F ext/rtree/rtreedoc3.test 555a878c4d79c4e37fa439a1c3b02ee65d3ebaf75d9e8d96a9c55d66db3efbf8 -F ext/rtree/rtreefuzz001.test 0fc793f67897c250c5fde96cefee455a5e2fb92f4feeabde5b85ea02040790ee +F ext/rtree/rtreefuzz001.test 44f680a23dbe00d1061dbde381d711119099846d166580c4381e402b9d62cb74 F ext/rtree/sqlite3rtree.h 03c8db3261e435fbddcfc961471795cbf12b24e03001d0015b2636b0f3881373 -F ext/rtree/test_rtreedoc.c de76b3472bc74b788d079342fdede22ff598796dd3d97acffe46e09228af83a3 +F ext/rtree/test_rtreedoc.c d20f51d1ad69c72947a4ac72194e5a12e70b3464e7492538fcef66fa871c5081 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/util/randomshape.tcl 54ee03d0d4a1c621806f7f44d5b78d2db8fac26e0e8687c36c4bd0203b27dbff F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/rtree/visual01.txt e9c2564083bcd30ec51b07f881bffbf0e12b50a3f6fced0c222c5c1d2f94ac66 -F ext/session/changeset.c 7a1e6a14c7e92d36ca177e92e88b5281acd709f3b726298dc34ec0fb58869cb5 +F ext/session/changeset.c 06b585d977391d498746f002b2d5f9315d0d37888ce9551bd0cb30bfe9a4cf47 F ext/session/changesetfuzz.c 227076ab0ae4447d742c01ee88a564da6478bbf26b65108bf8fac9cd8b0b24aa -F ext/session/changesetfuzz1.test 2e1b90d888fbf0eea5e1bd2f1e527a48cc85f8e0ff75df1ec4e320b21f580b3a -F ext/session/session1.test e94f764fbfb672147c0ef7026b195988133b371dc8cf9e52423eba6cad69717e +F ext/session/changesetfuzz1.test 15b629004e58d5ffcc852e6842a603775bb64b1ce51254831f3d12b113b616cd +F ext/session/session1.test 5d2502922d38a1579076863827342379a1609ca6bae78c40691a2be1ed1be2aa x F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f -F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 -F ext/session/session4.test 6778997065b44d99c51ff9cece047ff9244a32856b328735ae27ddef68979c40 +F ext/session/session3.test 2cc1629cfb880243aec1a7251145e07b78411d851b39b2aa1390704550db8e6a +F ext/session/session4.test 823f6f018fcbb8dacf61e2960f8b3b848d492b094f8b495eae1d9407d9ab7219 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 35279f2ec45448cd2e24a61688219dc6cf7871757716063acf4a8b5455e1e926 F ext/session/session8.test 326f3273abf9d5d2d7d559eee8f5994c4ea74a5d935562454605e6607ee29904 -F ext/session/session9.test 5409d90d8141881d08285ed1c2c0d8d10fb92069 +F ext/session/session9.test 0c4a8fbe7a5031f50855f020f3408e1f07fd7859f1daa1629eadcec3422072d6 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf -F ext/session/sessionD.test f5c6a762d00bc6ca9d561695c322ba8ecca2bed370486707ef37cf565d2f6c73 +F ext/session/sessionD.test 470ff917dc849e2eb78142ade63aaabd729d773833cff0ff01bca0eda68a21ce F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d085eb8efdad0a F ext/session/sessionH.test 71bbff6b1abb2c4ac62b84dee53273c37e0b21e5fde3aed80929403e091ef859 -F ext/session/session_common.tcl e5598096425486b363718e2cda48ee85d660c96b4f8ea9d9d7a4c3ef514769da +F ext/session/sessionI.test 11e7b6729fc942982a5104a40132f70a2e964d64d60dc5809b8206465af74822 +F ext/session/session_common.tcl a31f537a929a695a852d241c9434f2847cadf329856401921139fbb03a5a7697 +F ext/session/session_gen.test 942a0002df10da53c45b40b581cc3ed25e7ff42bda1e7ba497273dc2887aa8e6 F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3 +F ext/session/sessionalter.test e852acb3d2357aac7d0b920a2109da758c4331bfdf85b41d39aa3a8c18914f65 F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf -F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec -F ext/session/sessionfault.test 573bf027fb870d57bd4e7cf50822a3e4b17b2b923407438747aaa918dec57a09 +F ext/session/sessionblob.test 87faf667870b72f08e91969abd9f52a383ab7b514506ee194d64a39d8faff00a +F ext/session/sessionchange.test 6618cb1c1338a4b6df173b6ac42d09623fb71269962abf23ebb7617fe9f45a50 +F ext/session/sessionconflict.test 19e4a53795c4c930bfec49e809311e09b2a9e202d9446e56d7a8b139046a0c07 x +F ext/session/sessiondiff.test e89f7aedcdd89e5ebac3a455224eb553a171e9586fc3e1e6a7b3388d2648ba8d +F ext/session/sessionfault.test c2b43d01213b389a3f518e90775fca2120812ba51e50444c4066962263e45c11 F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c -F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 +F ext/session/sessionfault3.test ce0b5d182133935c224d72507dbf1c5be1a1febf7e85d0b0fbd6d2f724b32b96 +F ext/session/sessioninvert.test 9018f6a7387ac745084b6374c5e1aa14d648b372e6e1181cfab3df632b662d26 x F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 +F ext/session/sessionnoact.test 4c7ae5c7d351cb5323bca62b6b095592ad24bd90a6713c178b62ab0063d23e19 F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7 -F ext/session/sessionnoop2.test de4672dce88464396ec9f30ed08c6c01643a69c53ae540fadbbf6d30642d64e8 +F ext/session/sessionnoop2.test 2d8146321785bdc7cee8966d984560184cfca83fd46da2ff81ea883d6e977b36 F ext/session/sessionrebase.test 702378bdcb5062f1106e74457beca8797d09c113a81768734a58b197b5b334e2 F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a80600a44396f7363 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 -F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c +F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c e50a9218ee360db0a25298adc6614162d80ebe65d3f6a5b0a021e0902f6536a1 -F ext/session/sqlite3session.h 653e9d49c4edae231df8a4c8d69c2145195aedb32462d4b44229dbee7d2680fb -F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb -F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 -F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 -F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb +F ext/session/sqlite3session.c b3de195ce668cace9b324599bf6255a70290cbfb5451e826e946f3aee6e64c54 +F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a +F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 38700d5074af690f004e4e5f3533164ab49693b9d0832929c4ecf97a0bc09494 -F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 -F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab +F ext/wasm/GNUmakefile 3dc01e673c456d3b752674c9407276e8fef35dec1d304b3cc1de362f019b2a09 +F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a +F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 +F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff +F ext/wasm/SQLTester/SQLTester.mjs 6b3c52ed36a5573ca4883176f326332a8d4c0cecf5efd80489528267fa5d9ed4 +F ext/wasm/SQLTester/SQLTester.run.mjs 57f2adb33f43f2784abbf8026c1bfd049d8013af1998e7dcb8b50c89ffc332e0 +F ext/wasm/SQLTester/index.html 64f3435084c7d6139b08d1f2a713828a73f68de2ae6a3112cbb5980d991ba06f +F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core ef34398a903d0a2425fbbfbd4ed2cd596daea55b8515e2617c8dc7ad7c0767dd +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras 9eae68943ce91ab145892b31370819c2103525240eb72e0fce53c498b8d8275a F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 -F ext/wasm/api/extern-post-js.c-pp.js 393ab78b807da94096eae1a68bfddb999a2299936a185d910162fe87a57a9a3a +F ext/wasm/api/README.md f4c0d67caaee21a77b8938c30b5f79667bfc9d0c95d01b51df77ea35ee773884 +F ext/wasm/api/extern-post-js.c-pp.js 205f55aacfc62c580985db5c790300779de3876a76a5c7e1bfb13e71c8b4506b F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 -F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 -F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 -F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 -F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 -F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f -F ext/wasm/api/sqlite3-api-prologue.js 17f4ec398ba34c5c666fea8e8c4eb82064a35b302f2f2eb355283cd8d3f68ed5 -F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf +F ext/wasm/api/post-js-footer.js 5bd7170b5e8ce7b62102702bbcf47ef7b3b49cd56ed40c043fd990aa715b74ee +F ext/wasm/api/post-js-header.js 79d078aec33d93b640a19c574b504d88bb2446432f38e2fbb3bb8e36da436e70 +F ext/wasm/api/pre-js.c-pp.js a876c6399dff29b6fe9e434036beb89889164cc872334e184291723ecc7cb072 +F ext/wasm/api/sqlite3-api-cleanup.js a3d6b9e449aefbb8bba283c2ba9477e2333a0eeb94a7a26b5bf952736f65a6dd +F ext/wasm/api/sqlite3-api-glue.c-pp.js d2b8263b3ce0cefc6c5a68d0a4d448a9770eda4bf9d9ded9d7eb0198e4ce4da1 +F ext/wasm/api/sqlite3-api-oo1.c-pp.js c4260f3fdc553c56ee530c20cc1119029067b503f0d6d7b472705536cb45aa1d +F ext/wasm/api/sqlite3-api-prologue.js 307583ff39a978c897c4ef4ce53fe231dce5c73dc84785969c81c1ab5960a293 +F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f -F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 89640e4874a60cb2d973306b272384ffb45c7915375c7bb0355c7586f88dc39c -F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc -F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 -F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 -F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e -F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 -F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b +F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 +F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 418c33fe284739564daab3c7a7a88882fdd3c99137497900f98eddec1e409af5 +F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 +F ext/wasm/api/sqlite3-wasm.c dd7fc1d535281f0d5d2732bb1b662d1d403a762f07b63c2ea5663053377b2804 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a +F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7 +F ext/wasm/c-pp-lite.c 8fa0148e73782a86274db688c4730e2962cd675af329490493adddaf3322f16f +F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 -F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f -F ext/wasm/common/whwasmutil.js 749a1f81f85835e9a384e9706f2a955a7158f2b8cc9da33f41105cac7775a305 +F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f +F ext/wasm/common/whwasmutil.js 0d539324097fc83b953e9844267359ba0fd02286caa784ea2f597ced279ea640 +F ext/wasm/config.make.in c424ae1cc3c89274520ad312509d36c4daa34a3fce5d0c688e5f8f4365e1049a F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 -F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 +F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e -F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 -F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f -F ext/wasm/demo-worker1-promiser.js 51b02509a109e82f623fb4c900c8b48b9a77cc13fbd038396f9a083b86593ae3 +F ext/wasm/demo-jsstorage.js 42131ddfa18e817d0e39ac63745e9ea31553980a5ebd2222e04d4fac60c19837 +F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 +F ext/wasm/demo-worker1-promiser.c-pp.js f40ec65810048e368896be71461028bd10de01e24277208c59266edf23bb9f52 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d -F ext/wasm/demo-worker1.js 2c7794d8bc4ab9ecf9cdc2c15de940b11a006942226e441ea41edd458dfc0a26 -F ext/wasm/dist.make 451fb1b732257849f6e898d2a862512a0401500ed369ef53bdfeddf9c77bc3b9 +F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94 -F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10 -F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 -F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 -F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e -F ext/wasm/index.html dd900891844caebd9cadbddd704f66bd841d7c12fd69ce5af490e2c10fb49f45 -F ext/wasm/jaccwabyt/jaccwabyt.js 8287c0537fa0750414edbe75ce64668a81c8716df5ec4c3e6bb4f11bd1c36031 -F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb -F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 -F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 -F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 -F ext/wasm/speedtest1-wasmfs.html 7a301f4f5b6ad4f5d37fd6e7ca03a2f5d5547fd289da60a39075a93d7646d354 -F ext/wasm/speedtest1-worker.html 82869822e641c1bef3ec0cd2d7d2b6a42d0b4f68a7b160fb2e1dd0b523940a9b -F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 -F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da +F ext/wasm/fiddle/fiddle-worker.js 7798af02e672e088ff192716f80626c8895e19301a65b8af6d5d12b2d13d2451 +F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 +F ext/wasm/fiddle/index.html a27b8127ef9ecf19612da93b2a6a73bdb3777b5c56b5450bb7200a94bc108ff9 +F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 +F ext/wasm/index.html 54e27db740695ab2cb296e02d42c4c66b3f11b65797340d19fa6590f5b287da1 +F ext/wasm/jaccwabyt/jaccwabyt.js bbac67bc7a79dca34afe6215fd16b27768d84e22273507206f888c117e2ede7d +F ext/wasm/jaccwabyt/jaccwabyt.md 167fc0b624c9bc2c477846e336de9403842d81b1a24fc4d3b24317cb9eba734f +F ext/wasm/mkdist.sh 64d53f469c823ed311f6696f69cec9093f745e467334b34f5ceabdf9de3c5b28 x +F ext/wasm/mkwasmbuilds.c 1b53c4d2a1350c19a96a8cdfbda6a39baea9d2142bfe0cbef0ccb0e898787f47 +F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 +F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 +F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 +F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d +F ext/wasm/speedtest1-wasmfs.mjs 60dd5842f6d2a70a6d0bef12633a11491bde6984aff75a37c2040980d8cbf36a +F ext/wasm/speedtest1-worker.html 068d4190f304fa1c34e6501a1b3a4c32fe8d8dac93c2d0f53d667a1cb386eedc +F ext/wasm/speedtest1-worker.js 958a2d3c710bf8e82567277f656193a0248216db99a3c2c86966124b84309efb +F ext/wasm/speedtest1.html c90d63dfa795f0cb1ad188de587be9024b1ff73b4adc5fdf7efc0d781be94d03 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f -F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b -F ext/wasm/tester1-worker.html 258d08f1ba9cc2d455958751e26be833893cf9ff7853e9436e593e1f778a386b -F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 587a18db0f794c594eb0fade37c92af3e2370501576bb08dafd8ed7d009b6516 -F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 -F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d +F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c +F ext/wasm/tester1-worker.c-pp.html 883881eeac14eeeecc8ff22acf9fe0f18a97cacb48be08ebb0bae891ceded584 +F ext/wasm/tester1.c-pp.html 949920126dcf477925d8d540093d9cc374d3ab4c4ddee920c1dcadcf37917306 +F ext/wasm/tester1.c-pp.js 2b014884dadf28928fabcb688746ca87145673eef75e154486505a266203fc15 +F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e +F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 -F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make cf9a68162d92ca2bcb0b9528b244cb36d5cc2d84ccc9c2d398461927d6e75aea -F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x -F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 +F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 +F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 +F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 +F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2 +F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk ac6b13f8ecc43f377e9912380ea4cf366051d7f784cf61c8886e03e1cf0fbefa -F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 +F main.mk 00dd631c66c1f7922b2d691631163899eff1c3d1da780ff37a62f8d997b368e1 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c aa41ace6dbc5050d76b02548d3521e6bbccae4f0 +F mptest/mptest.c 7ecf39040c1187d9cbc487409bc052f194e93c165f667c55b052690a741d9d71 F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d -F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b -F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 -F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c 482c534877fbb543f8295992cde925df55443febac5db5438d5aaba6f78c4940 -F src/analyze.c a1f3061af16c99f73aed0362160176c31a6452de1b02ada1d68f6839f2a37df0 -F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 -F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf +F sqlite3.1 1b9c24374a85dfc7eb8fa7c4266ee0db4f9609cceecfc5481cd8307e5af04366 +F sqlite3.pc.in e6dee284fba59ef500092fdc1843df3be8433323a3733c91da96690a50a5b398 +F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da +F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d +F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d +F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 -F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d -F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c ecaaf8d57cd8b5f4e3167bd59cf61cef031b4b2ee606e6afa11b96a60a14f9ef -F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc -F src/btreeInt.h b900603c8956bdeb313841f9b67bdeceef32c64d962d35477c07ec25e8cf0f9b -F src/build.c 7a7217f75f202eff03617ca447bb9c3bc07d5af49da1d3cff2b1a88e8e394686 -F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 +F src/bitvec.c e242d4496774dfc88fa278177dd23b607dce369ccafb3f61b41638eea2c9b399 +F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea +F src/btree.c cb5b8ceb9baa02a63a2f83dec09c4153e1cfbdf9c2adef5c62c26d2160eeb067 +F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0 +F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 +F src/build.c 611e07299d72ff04bbcb9e7109183467e30925d203c3e121ef9bb3cf6876289b +F src/callback.c 3605bbf02bd7ed46c79cd48346db4a32fc51d67624400539c0532f4eead804ad +F src/carray.c ff6081a31878fc34df8fa1052a9cbf17ddc22652544dcb3e2326886ed1053b55 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d -F src/date.c aca9e0c08b400b21238b609aea7c09585396cd770985cf8f475560f69222dad3 -F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 -F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef -F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 941fe758212c6cf0007c6d7daf5368e11c199376ace9b3018494296e18a27eac +F src/date.c e19e0cfff9a41bfdd884c655755f6f00bca4c1a22272b56e0dd6667b7ea893a2 +F src/dbpage.c c9ea81c11727f27e02874611e92773e68e2a90a875ef2404b084564c235fd91f +F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c +F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 +F src/expr.c 28b1cc3d2f147cc888703d5482f9581f17656d02abfa331c34370cb3350776be F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 03c134cc8bffe54835f742ddea0b72ebfc8f6b32773d175c71b8afeea6cb5c83 -F src/func.c 03e6b501f3056d0ba398bda17df938b2b566aa0b3ca7e1942a3cd1925d04ec36 -F src/global.c bd0892ade7289f6e20bff44c07d06371f2ff9b53cea359e7854b9b72f65adc30 -F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 -F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 -F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 +F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f +F src/func.c 0b802107498048d3dcac0b757720bcb8506507ce02159e213ab8161458eb293b +F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b +F src/hash.c 03c8c0f4be9e8bcb6de65aa26d34a61d48a9430747084a69f9469fbb00ea52ca +F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf +F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 39b1c7527f3111923e65f168a87b03b591f12a41400a63d05c119794bee36620 +F src/insert.c dfd311b0ac2d4f6359e62013db67799757f4d2cc56cca5c10f4888acfbbfa3fd +F src/json.c fb031340edee159c07ad37dbe668ffe945ed86f525b0eb3822e4a67cbc498a72 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 -F src/main.c 035be2e9ba2a0fc1701a8ab1880af3001a968a24556433538a6c073558ee4341 -F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 +F src/loadext.c a3bc9a2522dc3b960e38b7582d1818f6245a49289387c2c7b19f27bfeabf1e81 +F src/main.c 65d11c17890966d271c925c6cc55e3ba50fa08374633cb99c0dee4719a20915a +F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de +F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff -F src/memdb.c 559c42e61eb70cd6d4bc692b042497133c6d96c09a3d514d92f3dac72268e223 +F src/memdb.c a3feb427cdd4036ea2db0ba56d152f14c8212ca760ccb05fb7aa49ff6b897df3 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 -F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 +F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c dd2b3f1cc1863079bc1349ac0fec395a500090c4fe4e11ab775310a49f2f956d -F src/mutex_w32.c caa50e1c0258ac4443f52e00fe8aaea73b6d0728bd8856bedfff822cae418541 -F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 -F src/os.c 81c9c1c52eab711e27e33fd51fe5788488d3a02bc1a71439857abbee5d0d2c97 +F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1 +F src/mutex_w32.c 28f8d480387db5b2ef5248705dd4e19db0cfc12c3ba426695a7d2c45c48e6885 +F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878 +F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a -F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d -F src/os_win.c 2b2411279f7b24f927591561303fc5871845732df42641cbf695c23640b16975 -F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 45e2ef5e9eb5cc0138bcc32b5d2299479be80b206f621873f59149dfb517f496 -F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 -F src/parse.y 146f9a1db7db5ef4299c6897d335e5abed348c2626190d2877d45ffa210fd4ca -F src/pcache.c 8ee13acccfd9accbf0af94910b7323dd7f7d55300d92ddafcf40e34fcc8e21be +F src/os_kv.c fb7ba8d6204197357f1eb7e1c7450d09c10043bf7e99aba602f4aa46b8fb11a3 +F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2 +F src/os_unix.c dcf7988ddbdd68619b821c9a722f9377abb46f1d26c9279eb5a50402fd43d749 +F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f +F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 +F src/pager.c cd562b878ea1b44d021ba199abc9d3b54f6b3347500a9fed03f66d6000620945 +F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 +F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250 +F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 -F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1 -F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 -F src/prepare.c 6350675966bd0e7ac3a464af9dbfe26db6f0d4237f4e1f1acdb17b12ad371e6e -F src/printf.c b9320cdbeca0b336c3f139fd36dd121e4167dd62b35fbe9ccaa9bab44c0af38d +F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd +F src/pragma.c ecec75795c1821520266e4f93fa8840cce48979af532db06f085e36a7813860f +F src/prepare.c 2af0b5c1ec787c8eebd21baa9d79caf4a4dc3a18e76ce2edbf2027d706bca37a +F src/printf.c 7297c2aeed4d90d80c5ba82920d9e57b7bfad04b3466be1d7e042db382fe296e F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 -F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 738c3a3d6929f8be66c319bad17f6b297bd60a4eb14006075c48a28487dc7786 -F src/shell.c.in 52836b4002a2cad8095b451f0c39a6542c23a231eb0ed5e39387bc8b1f7aaa9e -F src/sqlite.h.in c14a4471fcd897a03631ac7ad3d05505e895e7b6419ec5b96cae9bc4df7a9fc6 -F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h a3ced8b9ebc573189c87b69f24bf10d2b9cd3cefefaae52623a2fa79e6fdd408 -F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 -F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 +F src/resolve.c 5616fbcf3b833c7c705b24371828215ad0925d0c0073216c4f153348d5753f0a +F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 +F src/select.c fa67ce1dd60744c315791085897c1571ffe5554bf3e0410942b5fc9d7e0a4b56 +F src/shell.c.in 2c7e751795f38bb1855c35b556419cab5b8ba22e0f6758f5a629338065d6b79f +F src/sqlite.h.in c0979f9ac1f5be887397dd2a0bb485636893a81b34d64df85123aae9650c42f2 +F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 +F src/sqlite3ext.h 7f236ca1b175ffe03316d974ef57df79b3938466c28d2f95caef5e08c57f3a52 +F src/sqliteInt.h 88f7fc9ce1630d9a5f7e0a8e1f3287cdc63882fba985c18e7eee1b9f457f59aa +F src/sqliteLimit.h fe70bd8983e5d317a264f2ea97473b359faf3ebb0827877a76813f5cf0cdc364 +F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd -F src/test1.c 8eab61fb2813aa212d97ab188e85fc9ca7b89d9ff5ff05d59d9aa0c491a6c721 -F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef -F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 -F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 -F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d -F src/test6.c ae73a3a42bbc982fb9e301b84d30bda65a307be48c6dff20aba1461e17a9b0ce -F src/test8.c 0c856d6ff6b0d2ff6696addc467a15ed17c6910f14475302cd5b3b4e54406161 -F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 -F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a -F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 -F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 -F src/test_bestindex.c 68c62586d2ae9f032903fe53be743657d0c2aac0a850b880938b668e1161d516 -F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce -F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c 8264637b06a3c1f0727c88d1ea32dcf7986b9e7e358a970cae87cdac8a5b2708 -F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f -F src/test_demovfs.c 38a459d1c78fd9afa770445b224c485e079018d6ac07332ff9bd07b54d2b8ce9 -F src/test_devsym.c aff2255ea290d7718da08af30cdf18e470ff7325a5eff63e0057b1496ed66593 -F src/test_fs.c ba1e1dc18fd3159fdba0b9c4256f14032159785320dfbd6776eb9973cb75d480 -F src/test_func.c 24df3a346c012b1fc9e1001d346db6054deb426db0a7437e92490630e71c9b0a -F src/test_hexio.c 9478e56a0f08e07841a014a93b20e4ba2709ab56d039d1ca8020e26846aa19bd -F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 -F src/test_intarray.c 39b4181662a0f33a427748d87218e7578d913e683dc27eab7098bb41617cac71 -F src/test_intarray.h d57ae92f420cda25e22790dac474d60961bd0c500cbaa3338a05152d4a669ef7 +F src/tclsqlite.c 3c604c49e6cf4211960a9ddb9505280fd22cde32175f40884c641c0f5a286036 +F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a +F src/test1.c 5d061afe479c7364842e0170be7220dea13389575fa6030d30b3e20bec4e1f75 +F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff +F src/test3.c 432646f581d8af1bb495e58fc98234380250954f5d5535e507fc785eccc3987a +F src/test4.c 0ac87fc13cdb334ab3a71823f99b6c32a6bebe5d603cd6a71d84c823d43a25a0 +F src/test5.c 38fa635a70a94f2aa8b47ecbab15d821386205d27ad4159c3551ab3ba45efa11 +F src/test6.c 9722054d37257459f1b8988e59e7db1dd630bfb291f16b2759764e778a9d1899 +F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 +F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af +F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f +F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72 +F src/test_bestindex.c a9428931bec06de830b2630f57a7b1f2711761269f04df62b7aa1affcbce15bb +F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c +F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 +F src/test_config.c 18aa596d37de1d5968c439fd58ebf38bc4d9c9d1db63621504e241fde375cecd +F src/test_delete.c d0e8f6dc55cfc98a7c27c057fb88d512260564bf0b611482656c68b8f7f401ed +F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c942383 +F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 +F src/test_fs.c a946408c81231feff4b6a2c18068f20aefe254dc82288687128af8b7520d32cb +F src/test_func.c 858d4dddb7acf88222ebcba7cffb585f6dde83e4a15b838c0d05ccdf8d5219b9 +F src/test_hexio.c a90baa0a8ab5e7cfe2216a61c9a31cfd1f8378353a3d23e25fa94c09aa755bb0 +F src/test_init.c 1649e02448f536e53172f6b1ff873254fe9a0c6c8a4502a2d25c0cc7b11945ea +F src/test_intarray.c 3fcf8ca7bb5c8776ea83f6aa9b66f8df0d1f37a99207b0097c8486f9c15cedbf +F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293 F src/test_journal.c a0b9709b2f12b1ec819eea8a1176f283bca6d688a6d4a502bd6fd79786f4e287 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd -F src/test_malloc.c 21121ea85b49ec0bdb69995847cef9036ef9beca3ce63bbb776e4ea2ecc44b97 -F src/test_md5.c 7268e1e8c399d4a5e181b64ac20e1e6f3bc4dd9fc87abac02db145a3d951fa8c -F src/test_multiplex.c d8bc260a57c2028946a9bce9918d24e69e44cebd71ac6be240aa5b6b75e2b369 -F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635 -F src/test_mutex.c abf486e91bd65e2448027d4bb505e7cce6ba110e1afb9bd348d1996961cadf0d +F src/test_malloc.c 55e2b5398f6e184300d91278b8a84768f6a740735adc030411181945d5276da0 +F src/test_md5.c 811a45330c9391933360f998156a8907ee29909c828ab83ac05d329942cbea8f +F src/test_multiplex.c 82f0aa8eee629b6949782cfab8782ed35a9b56dc80d12877af52147f304d22b8 +F src/test_multiplex.h f0ff5b6f4462bfd46dac165d6375b9530d08089b7bcbe75e88e0926110db5363 +F src/test_mutex.c dacae6790956c0d4e705aaed2090227792e291b0496cccd688e9994c1e21f740 F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 -F src/test_osinst.c d341f9d7613e007c8c3f7eba6cd307230047506aa8f97858c1fd21f5069616bd -F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 6cb9297115b551f433a9ad1741817a9831abed99 +F src/test_osinst.c 269039d9c0820a02ee928014c30860d57ee757ecda54df42e463d0ca1377b835 +F src/test_pcache.c 496da3f7e2ca66aefbc36bbf22138b1eff43ba0dff175c228b760fa020a37bd0 +F src/test_quota.c 180e87437250bed7e17e4e61c106730939e39fec9be73d28961f27f579a92078 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d -F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b -F src/test_schema.c f5d6067dfc2f2845c4dd56df63e66ee826fb23877855c785f75cc2ca83fd0c1b -F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a -F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e -F src/test_syscall.c 9fdb13b1df05e639808d44fcb8f6064aaded32b6565c00b215cfd05a060d1aca -F src/test_tclsh.c 3ff5d188a72f00807425954ea3b493dfd3a4b890ecc6700ea83bad2fd1332ecf -F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc -F src/test_thread.c 7ddcf0c8b79fa3c1d172f82f322302c963d923cdb503c6171f3c8081586d0b01 -F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43 -F src/test_vfs.c 193c18da3dbf62a0e33ae7a240bbef938a50846672ee947664512b77d853fe81 -F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 -F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 -F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215 -F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f +F src/test_rtree.c d844d746a3cc027247318b970025a927f14772339c991f40e7911583ea5ed0d9 +F src/test_schema.c b06d3ddc3edc173c143878f3edb869dd200d57d918ae2f38820534f9a5e3d7d9 +F src/test_sqllog.c 5abf04865758c0a3915b4ec2b2ee5ab75f74c00e2f05bf503b9083e0ab6829d7 +F src/test_superlock.c 3387fc794a68d8c6b6ed059aabacbfe870dc502c5cf65562f36aac78b4a4d629 +F src/test_syscall.c c5bf039261973135068aa68f4d185a6147333dcf266977989f8245b3a1968f1b +F src/test_tclsh.c c01706ac60bd3176754d3ccd37da74c6ad97c2e14489f8ed71b497c1c0ac0dd4 +F src/test_tclvar.c ae873248a0188459b1c16ca7cc431265dacce524399e8b46725c2b3b7e048424 +F src/test_thread.c 3edb4a5b5aeb1a6e9a275dccc848ac95acab7f496b3e9230f6d2d04953a2b862 +F src/test_vdbecov.c 5c426d9cd2b351f5f9ceb30cabf8c64a63bfcad644c507e0bd9ce2f6ae1a3bf3 +F src/test_vfs.c b4135c1308516adf0dfd494e6d6c33114e03732be899eace0502919b674586b5 +F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c fccf3b8c517c1f55cb380c1522febe6921fcb2bd800c16c78cab571d0eb0ccbd -F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 -F src/update.c 3f4fb5ad7c9b48d7911974d6579192bb3a6c27f46140b6cbb9139cc8a77b8691 -F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 -F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c d4bcb560471cd94e6e17d448311f8d5bf81a7e5276295a53501058ef1b95dd1a -F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c fedd2dfa5165256c8e372f2ae9454c4a82cf60ce79a04dff80a86ab2116ea15a -F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 -F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb -F src/vdbeapi.c b4982cde547054c4f7341198db3c3008a48e1eb028f757601bf5bf2fc026cbcf -F src/vdbeaux.c 6ee48db408d4c297a363f1e31145c09793a580e7c508bb36063dd017d67117a2 -F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 1cac4028c0dabbf1f3259f107440e2780e05ac9fe419e9709e6eb4e166ba714b -F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 -F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 -F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac -F src/vtab.c 4758a96d36c9a120848386ae603b1ab32a4876e0a1faf81bfcfb524455e583dc -F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 7a65f64bfe4a783c5e2df73ffb0efc383dec934dee9e3ac706b2eeb3631d17ac -F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a -F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c b74a83b4c8f65b218c5c1c8d9122433f85ee1300fd9263ba1697d0e1040eeb36 -F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c b300db0bcd84ad6c2642bf3f509f92fad7b7d697b9856b64dd66d692d184d054 -F src/whereexpr.c 22cf19b0ececeaf838daed1039c5231a8778784eba5ad67b991442a23473fd3f -F src/window.c e075ea85bea322e30e361fa6e69eddba74f461e99e2a564dc09973f8a1fb27d9 +F src/tokenize.c cb3294cf23c11106b50d9af6998a6c1bf389b52e15b17698c9fab97bbaa9b37f +F src/treeview.c 3ce7ac9835d2d70cc1c868b01b747ae8a062322e155701e58e3d62ca79aada7a +F src/trigger.c d5cf2541ff048f30b6a0507eb3d1ec4e695c53584e3b2298a5bf248714fe185e +F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf +F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 +F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 +F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 +F src/vacuum.c 1bacdd0a81d2b5dc1c508fbf0d938c89fa78dd8d5b46ec92686d44030d4f4789 +F src/vdbe.c b44c366e83412d3b8c190feb1f029b7d02e1bd69252a57b32f195107f0d03964 +F src/vdbe.h be33bd7b17f2ec92939642416030491508c51071f6c14e27cd195983fec56b63 +F src/vdbeInt.h 2aaeb6df2938b181b4700a9328688a3986f2bba71e8b96f6a80671316618fa49 +F src/vdbeapi.c 869a0da5d855495055f4d35c6ada582f64ce995ce14b26ff9d336274d497266c +F src/vdbeaux.c 908d8a191aed444b2e4c920159249127f3ff67b94c56a16fad1dfdf9c7488f20 +F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 +F src/vdbemem.c 48e562ff27e6386eb8613207ac27d3d98c1f67fdc4775a1ab13759d2c2a1c021 +F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 +F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b41 +F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 +F src/vtab.c 5437ce986db2f70e639ce8a3fe68dcdfe64b0f1abb14eaebecdabd5e0766cc68 +F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab +F src/wal.c 505a98fbc599a971d92cb90371cf54546c404cd61e04fd093e7b0c8ff978f9b6 +F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 +F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 +F src/where.c 287324fe73a0ae8e55b3be89bb2fe4148e3a8394e1e2f10ed2113713a037d8a3 +F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da +F src/wherecode.c 71c5c6804b7f882dec8ec858758accae02fcfca13df3cc720f1f258e663ec7c5 +F src/whereexpr.c 403a44eeec1a0f0914fccc6a59376b6924bc00ef6728fe6ffce4cf3051b320fc +F src/window.c 538195bbc75bb924e18e368fbd4ed731a3fe3f901351b44f6466ec486f53affe F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 -F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 -F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 +F test/affinity2.test 4d7a34d328e58ca2a2d78fd76c27614a41ca7ddf4312ded9c68c04f430b3b47d +F test/affinity3.test 9b7d1133e11d5edd7805573c4ab6f3ba73b0b74a1f280d5b130d4bf3506a93ff F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggfault.test 777f269d0da5b0c2524c7ff6d99ae9a93db4f1b1839a914dd2a12e3035c29829 -F test/aggnested.test 7269d07ac879fce161cb26c8fabe65cba5715742fac8a1fccac570dcdaf28f00 +F test/aggnested.test 610b0ce2c3e8f3daee25f9752800ee8d785db10da4aa1fbeea0ea1aabaf1d704 +F test/aggorderby.test 7be65e743f82ee49ba62da1c799e59341d23884a99edfe093df0cdfaac94cbbb F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 -F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 -F test/alter.test 313073774ab5c3f2ef1d3f0d03757c9d3a81284ae7e1b4a6ca34db088f886896 -F test/alter2.test a966ccfcddf9ce0a4e0e6ff1aca9e6e7948e0e242cd7e43fc091948521807687 -F test/alter3.test ffc4ab29ce78a3517a66afd69b2730667e3471622509c283b2bd4c46f680fba3 -F test/alter4.test 716caa071dd8a3c6d57225778d15d3c3cbf5e34b2e84ae44199aeb2bbf50a707 +F test/all.test cf929f721e20960ca9db89471fa44f9176322ba8f25e97193f91881c223643b3 +F test/alter.test 3c00eff1e2036b9f93e9cd0f3d3e63750ac87ecb5bc71b9d7bd07cbf2ac4c494 +F test/alter2.test 7e3d26ab409df52df887b366a63902c3429b935c41cb962fd58ffc25784f2f19 +F test/alter3.test dcdd5f850f30656a45a0f05e41abfb52b74bbf6ccba165d0f7adf6b0116e4fd6 +F test/alter4.test 37cafe164067a6590a0ee4cec780bddbbaa33dc50b11542dcfbe0e65626494fd F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959 F test/alterauth2.test 48967abae0494d9a300d1c92473d99fcb66edfcc23579c89322f033f49410adc -F test/altercol.test 8465ca659c2c55a359cf16cc261df4fcb5c45a5f104a50827c337ae66c09dc15 +F test/altercol.test b43fb5725332f4cf8cff0280605202c1672e808281accea60a066d2ccc5129e5 F test/altercorrupt.test 2e1d705342cf9d7de884518ddbb053fd52d7e60d2b8869b7b63b2fda68435c12 F test/alterdropcol.test a653a3945f964d26845ec0cd0a8e74189f46de3119a984c5bc45457da392612e F test/alterdropcol2.test 527fce683b200d620f560f666c44ae33e22728e990a10a48a543280dfd4b4d41 @@ -748,37 +845,32 @@ F test/alterfault.test 289067108947bedca27534edd4ff251bcd298cf84402d7b24eaa37493 F test/alterlegacy.test f38c6d06cda39e1f7b955bbce57f2e3ef5b7cb566d3d1234502093e228c15811 F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74a3fb80c9 F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584f73747c81 -F test/altermalloc3.test 8531b3086f0a7889f43971a579a8c81832d5123f4703d8c86b89ba1136c63b9a -F test/alterqf.test ff6c6f881485c29ed699b8ef4774864ca1b0c01a6c08f5cdd624a008e4b40fca +F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd436f076e81 +F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 -F test/altertab2.test 62597b6fd08feaba1b6bfe7d31dac6117c67e06dc9ce9c478a3abe75b5926de0 -F test/altertab3.test 6c432fbb9963e0bd6549bf1422f6861d744ee5a80cb3298564e81e556481df16 -F test/altertrig.test fb5951d21a2c954be3b8a8cf8e10b5c0fa20687c53fd67d63cea88d08dd058d5 +F test/altertab2.test 0889ba0700cc1cdb7bc7d25975aa61fece34f621de963d0886e2395716b38576 +F test/altertab3.test 471b8898d10bbc6488db9c23dc76811f405de6707d2d342b1b8b6fd1f13cd3c8 +F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f -F test/analyze.test 547bb700f903107b38611b014ca645d6b5bb819f5210d7bf39c40802aafeb7d7 -F test/analyze3.test 03f4b3d794760cf15da2d85a52df9bae300e51c8fefe9c36cfae1f86dc10d23f +F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b +F test/analyze3.test c5156cef33f04b90a6b9e9d5d0bbc273a0fb44147d4508407bf1080811e2c6c8 F test/analyze4.test 68bd069f3ac7ac1e652ddd9f04f57d5606ddb4208450f5297005db7aa0dd707d F test/analyze5.test fa5131952303ac4146aba101b116b9c8cb89e2637531c334a6df7f7d19dddc0d F test/analyze6.test 028f5bdfc9e5b5294768fa9a7185b8cd1d019aa7aab5b2f8ee42d7271d9a3b28 F test/analyze7.test 079d17c495e396bdbd6cc6a083112788a6fbfb3b95c42e760e4270a53c9ead8f F test/analyze8.test 29ef237d8a59b39cc31c3310134fefe96a690b195e3deed5ecb652839089f15c F test/analyze9.test 30e1cb99336045a384a11d97900720184333c88174b3b89bc07444ea39e7df19 -F test/analyzeC.test 1111830ad355d29a294a5dda654dd5f6a8622c6a223a4f7b7b3d091df7a7a42b +F test/analyzeC.test ca20c103809836d15ab7bacb78d3dd9842c7d8743f492e5e5d058b2ffe4cd849 F test/analyzeD.test 485f621cfd2ef0a8f8ac79672586651bfa495bd899db50461bb4b558400ab3c1 F test/analyzeE.test d2ec7921c162cdc33ac8e7eb01f9ebf78100610af7c94c8552bbf551de1fb397 F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae1068f949 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 -F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 -F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b -F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b -F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 -F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a -F test/async5.test 383ab533fdb9f7ad228cc99ee66e1acb34cc0dc0 -F test/atof1.test 1f24a6f16e44892848298bd8e488159d4a60cf47be003d32747a136103c9bbac +F test/analyzer1.test b6a624ec0af92eec209e1328465b66937c8fdf2fb442a3fa45321ddb3700f4aa +F test/atof1.test bd21c4a0e718ab1470de07a2a79f2544d7903be34feebcc80de04beee4807b00 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061 F test/atomic2.test b6863b4aa552543874f80b42fb3063f1c8c2e3d8e56b6562f00a3cc347b5c1da F test/atrc.c c388fac43dbba05c804432a7135ae688b32e8f25818e9994ffba4b64cf60c27c -F test/attach.test 54f8e49e88d0de48f6428267a678465863d2b8f72320612f35bd5c02e240bc2f +F test/attach.test 12b2a9872c1ce20edf40289f00d82d13faae59ced522d98496ab06ad5c5e0a1c F test/attach2.test 6d1e3a457ce260d6fc8e5945c07fba6c76dc2aa90e1c701f067b50ee88f7315a F test/attach3.test c59d92791070c59272e00183b7353eeb94915976 F test/attach4.test 00e754484859998d124d144de6d114d920f2ed6ca2f961e6a7f4183c714f885e @@ -787,28 +879,28 @@ F test/auth.test 5b8558a40571ebc55c1581cb7cec3b2348a699542a0a51b83ef21c6a953d95e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec -F test/autoinc.test 997d6f185f138229dc4251583a1d04816423dddc2fc034871a01aeb1d728cb39 -F test/autoindex1.test d34caffb0384003ee28eae87679214c029e9be4b332d9649a79e0b94ab70502c +F test/autoinc.test 9df9930966dbe92c55ef37a4d89112cfd537be0d0596d397177c12db9e581be0 +F test/autoindex1.test 65931519206bbec71948b11e125af0656435a0937973fe5fed70d776a712911f F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df -F test/autoindex3.test dcd6b2f8bed2be67b131e2e671f892e971d934e24fd00988952d0e0a67e24aa7 -F test/autoindex4.test 5df39313526b6f22a26bd119bbd97ca69f28386ab3c671fc10568d921c41eb08 -F test/autoindex5.test 2ee94f033b87ca0160e08d81034c507aff8e230df2627f0304fa309b2fee19a3 +F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6 +F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 +F test/autoindex5.test 3fb938cbf4e7f3896563ce04e2a24b0bc653fc6245b4bf3268cd7b20f441d87f F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df F test/autovacuum2.test 76f7eb4fe6a6bf6d33a196a7141dba98886d2fb53a268d7feca285d5da4759d7 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 0c3a38e03cccb0fc3127838462dc05dc3f4c1480d770c084b388304c25de3652 -F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e -F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d +F test/avfs.test 76f59743dc1f5fa533840d1818b420fe1ee45e21c0fd6bbac7942ba677903128 +F test/avtrans.test 7a6eae44763293024b137b53ff824d8500d754dbae060a8d940afbacfc1d4a15 +F test/backcompat.test f2431465ed668f09fc3f6998e56e893a1506ccea6e8b6f409f085f759f431b48 F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f -F test/backup2.test 8facb54df1388419d34b362ab1f7e233310ff3a3af64e8ad5ec47ba3c2bbe5cf +F test/backup2.test b153553ee5667b0748b43346b0725fbf80ce1f5544613bf087d669778b60ec56 F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f310fd3b24a491b6b77bccdf14923b85d6ebcce751068c180d93a6b8ff854399 -F test/basexx1.test d8a50f0744b93dca656625597bcd3499ff4b9a4ea2a82432b119b7d46e3e0c08 -F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c +F test/basexx1.test 4ae6ddbd92a7ebcabb5d844664c3e755d29fb69c8ddcf0c8d59bbe4e07c23919 +F test/bc_common.tcl c70b896d1d4ce72f769d2c7c1fc15b2cb07559eb2093f2736c8ca51664b29ff5 F test/bestindex1.test 856a453dff8c68b4568601eed5a8b5e20b4763af9229f3947c215729ed878db0 F test/bestindex2.test 394ff8fbf34703391247116d6a44e1c50ee7282236ee77909044573cefc37bc0 F test/bestindex3.test 34bea272b0e0f835651b16a3931dbe7ac927039be6b2e1cb617bbe1d584b492b @@ -816,21 +908,25 @@ F test/bestindex4.test 3039894f2dad50f3a68443dffad1b44c9b067ac03870102df1ce3d9a4 F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb75dc98d F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e -F test/bestindex8.test 333ad8c6a554b885a49b68c019166eda92b05f493a92b36b0acdf7f766d04dad -F test/bestindex9.test bf2eb8556e8d5c00ef3ee18c521751cd03c1b55454b6e7683b4c6742e3131b23 -F test/bestindexA.test dd7b7439a46169b45d0305c4cbbb14fc20c7044acc2055c767d2f838b3479c3f -F test/between.test b9a65fb065391980119e8a781a7409d3fcf059d89968279c750e190a9a1d5263 +F test/bestindex8.test b63a4f171a2c83d481bb14c431a8b72e85d27b2ffdaa0435a95d58ca941678f9 +F test/bestindex9.test 1a4b93db117fd8abe74ae9be982f86aa72f01e60cd4ac541e6ede39673a451a0 +F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572eef44c7f +F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce +F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc +F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 +F test/bestindexE.test 297f3ea8500a8f3c17d6f78e55bdfee089064c6144ee84a110bd005a03338f49 +F test/between.test e7587149796101cbe8d5f8abae8d2a7b87f04d8226610aa1091615005dcf4d54 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc F test/bigmmap.test 6021e205487347c6d7e5a541aa472a4b8efc4e9f4a3799a823b61a8e6616105d -F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 +F test/bigrow.test 3e6b189dea465a78306eac67fc6754c1cf90eb1694a25c88c338900c216c1e6b F test/bigsort.test 997e172009905873c06426145e4b3794c7dfe2d563724cb2fd39d45f319cf3d2 F test/bind.test 1e136709b306f7ed3192d349c2930d89df6ab621654ad6f1a72381d3fe76f483 F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aab7 F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test cf613a27054bbaf61c5bfc440a5cfd3ff76798d0695f3fc5e5d1bbc819b8dab1 +F test/bloom1.test 3b6277a647ac503b5d5df331037b0c01c40e88cc9537b94eaf2d8aa334ed4c8f F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -839,32 +935,35 @@ F test/boundary3.tcl 23361e108a125dca9c4080c2feb884fe54d69243 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b -F test/btree01.test fef17d9e999ac4f04095948e3438fbe674f4e07bb2c63bb1cad41d87baee077f +F test/btree01.test 7207a813400798c7403dbfdd0b1eb04e204804257de384dd6f221f3b189cb7fc F test/btree02.test 7555a5440453d900410160a52554fe6478af4faf53098f7235f1f443d5a1d6cc F test/btreefault.test a82a23b0578bc587afbf9a622c8f54a54f63762f062ba8a35613cfee38ab42f9 -F test/busy.test 510dc6daaad18bcbbc085bcc6217d6dc418def5e73f72ce1475eea0cb7834727 +F test/busy.test caff7164c16ce06a53af51f9e4c2753d4cc64250e00790a5e48b9c4f4be37597 F test/busy2.test 20823a5d7c42fb257d9f108c66312d90b1bb4ec3d80ba6b4e371073727560f98 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9 F test/capi2.test 4ee545824adc3eb33bf57ef89f77440b28188ec3da72e5425ff0fcdba32e8d5a -F test/capi3.test 3910a73c38ac76d69778dd9eb481ab7cd6ed59117fc047b4f6056a5c72529de1 +F test/capi3.test ab90c548969613315605c555a8623f6b56e00e28d451c46a17ef73683c422c70 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a490f7 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe -F test/carray01.test 23ed7074307c4a829ba5ff2970993a9d87db7c5cdbbe1a2cbef672d0df6d6e31 -F test/cast.test af2286fdd28f3470b7dcad23977282b8cc117747ad55acff74a770dad3b19398 +F test/carray01.test 49e2aedfdf2c715bc002d2773cdc1217166679639542c79c8aa4115f06421407 +F test/carray02.test 9d070b54f24a34d1f3b3c552ba34db0375a9d1c4219067416fb07d1595987c9d +F test/carrayfault.test 108a7d83904fc267c448e27c13b2a857c700bd6ddaa2f1e2518be718b159cb6b +F test/cast.test a2a3b32df86e3c0601ffa2e9f028a18796305d251801efea807092dbf374a040 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef -F test/changes.test 9dd8e597d84072122fc8a4fcdea837f4a54a461e6e536053ea984303e8ca937b -F test/changes2.test d222c0cbf5ab0ac4d7c180594e486c1bf20b2098d33e56ce33b8e12eba6823b9 -F test/check.test 56e4ed457e9f8683b9fc56f5b964f461f6e8a8dd5a13f3d495408215d66419ed +F test/changes.test 4377d202a487f66fc2822c1bf57c46798c8b2caf7446f4f701723b1dbb6b86f6 +F test/changes2.test 07949edcc732af28cb54276bfb7d99723bccc1e905a423648bf57ac5cb0dc792 +F test/check.test 3a7972ccbaad80d496833da8714d69d9d5d4ce9e7211af1cd2a06ae488a7de12 F test/checkfault.test da6cb3d50247169efcb20bdf57863a3ccfa1d27d9e55cd324f0680096970f014 -F test/chunksize.test 427d87791743486cbf0c3b8c625002f3255cb3a89c6eba655a98923b1387b760 +F test/chunksize.test faea11c5d6df9d392252a8dd879e1b1d68c9d3e8b7909cbed8bcec3b60c706f1 +F test/cksumvfs.test 38748166f486c59e8d94f3ee705040031ed3790a4dd69c4a9eef64f623a30ad4 F test/close.test eccbad8ecd611d974cbf47278c3d4e5874faf02d811338d5d348af42d56d647c F test/closure01.test 9905883f1b171a4638f98fc764879f154e214a306d3d8daf412a15e7f3a9b1e0 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 -F test/collate1.test 71a6f27fdc93a92f14d8ab80c05e1937656a5a03197e1a10157314554d630ce8 +F test/collate1.test 0890fa372753b59eba53832d37328af815f6b8e4b16761823180eeb62c8e8f64 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd @@ -874,18 +973,18 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 -F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test 87ad5458bb8709312dac0d6755fd30e8e4ca83298d0a9ef6e5c24277a3c3390e +F test/collateB.test 9c840e21f9aead6fc533cea310f0bd202d5c11511088811b7e93ae7b47fccb24 +F test/colmeta.test 248a644cec4c7c371cf1e107fd8fdba708dc290866c572672b6260e3466cce79 +F test/colname.test 0e32125de701c6bd86247194c6f5639b740b4f71a0d88ee1e67ff3bda9ae99ca F test/columncount.test 6fe99c2f35738b0129357a1cf3fa483f76140f4cd8a89014c88c33c876d2638f -F test/conflict.test b705cddf025a675d3c13d62fa78ab1e2696fb8e07a3d7cccce1596ff8b301492 +F test/conflict.test 3307ffdf988e04b01c4e942d8aa369a977f085bf629f43a627c9a77f39d65926 F test/conflict2.test 5557909ce683b1073982f5d1b61dfb1d41e369533bfdaf003180c5bc87282dd1 F test/conflict3.test 81865d9599609aca394fb3b9cd5f561d4729ea5b176bece3644f6ecb540f88ac F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 -F test/corrupt.test d7cb0300e4a297147b6a05e92a1684bc8973635c3bcaa3d66e983c9cbdbf47a3 -F test/corrupt2.test 6e0c1e1c2ff4bedde4bc73f16250d74ae5b3d9ece086640ce88b9a94620ba993 +F test/corrupt.test 54509b182b1927663e0a425b681b0935a08a01b11d8153a4a9545ed36760ebe2 +F test/corrupt2.test 4ce5eadd51baa1aedb48e141dd885d155946f5c3677bb032547e350ce91b17f4 F test/corrupt3.test 6a982535d52c8165654cbc79a043cfd0bf02495a5efbf4754295e056fc548539 -F test/corrupt4.test b5ae41607e8d17d9c1f3e94fdb572ce061ed3beeebdb46fb3a348181b8c8a097 +F test/corrupt4.test 5fa4559bcfd14afbb99670d463546ba75fb4975c710b7f6dfa592ae90471cce7 F test/corrupt5.test 387be3250795e2a86e6234745558b80efb248a357d0cd8e53bce75c7463f545d F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568 F test/corrupt7.test ffa86896fe63a3d00b0a131e1e64f402e4da9f7e5d89609d6501c851e511d73a @@ -893,30 +992,30 @@ F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec -F test/corruptC.test 9cf32275dae3ca33f645afe5d1d3f5ba5ac2af2b0833dfb5282f9dccb6fb81bb -F test/corruptD.test a828c788535946a372a56a750b242cd96287cd823657abe5a73c5e51b91bdd28 +F test/corruptC.test 7d6d9e907334ea3ccb7111a0656cafa30a28f8a5f2aaf1c45ad712236302856a +F test/corruptD.test 614320aa519f6bf6c7dd2f581f9513ff7b6826954180cca1a606d0e25ea084a3 F test/corruptE.test 4143791f2dfb443aec5b7fabfa5821e6063eccc3b49b06f212c2f014715fd476 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 9d8cbf6214e492abe9e822e759b9751ae336cec0a6fe3ff3b37bfbd8ff9c22ca F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 -F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af -F test/corruptL.test b42978028afc5eefc8b51d8d7cd6a9344ba7362d7ed4511ee2070f56e06d5a1c +F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc23dcb +F test/corruptL.test 652fc8ac0763a6fd3eb28b951d481924167b2d9936083bcc68253b2274a0c8fe F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 -F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 -F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 +F test/corruptN.test a034bb217bebd8d007625dfb078e76ec3d53515052dbceb68bd47b2c27674d5c +F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b -F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f +F test/crash.test f699152b8ae759bdf1c19c278b135f5d43fa4b6466e63489cd02edbc94aebad0 F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 F test/crash3.test 8f5de9d32ab9ab95475a9efe7f47a940aa889418 F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc F test/crash5.test 4aa55e7ac3c4bc511873e457aa65d2827d52da9b51e061511899dadcfe22b1e8 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df -F test/crash8.test 64366e459c28dd62edfb7ad87253a409c7533b92d16fcc479a6a8131bdcc3100 +F test/crash8.test 3a0c39c079b441a9b1da22e3edb58817f9dd330c4c3a7f9dd1f9f7c8368ea352 F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f54140699f5165 @@ -926,32 +1025,35 @@ F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c47 F test/cursorhint.test 05cf0febe5c5f8a31f199401fd1c9322249e753950d55f26f9d5aca61408a270 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 -F test/date.test 1d44557f668298b10d3335b22ab8feb133267b67ec4d85538908fe4dfebd2611 +F test/date.test 328ed63091d34c8a6e8dcec4f999a031ce310f24395b8f0f67f07eaf36cbfd1f F test/date2.test 7e12ec14aaf4d5e6294b4ba140445b0eca06ea50062a9c3a69c4ee13d0b6f8b1 F test/date3.test a1b77abf05c6772fe5ca2337cac1398892f2a41e62bce7e6be0f4a08a0e64ae5 +F test/date4.test b5ad22baf7394e008ac59383840159daedd45be31dcf74a3b2450ec0e28955ce +F test/date5.test 14ba189bc4d03efc371dd5302e035764f6633355a3e13acb4a45e7b33530231e F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603 -F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e -F test/dbfuzz001.test 55e1a3504f8dea84155e09912fe3b1c3ad77e0b1a938ec42ca03b8e51b321e30 +F test/dbfuzz.c fc566102f72c8af84ae8077b4faf7f056c571e6fa7a32e98b66e42b7505f47b6 +F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 -F test/dbpage.test fce29035c7566fd7835ec0f19422cb4b9c6944ce0e1b936ff8452443f92e887d -F test/dbpagefault.test d9111a62f3601d3efc6841ace3940181937342d245f92a1cca6cba8206d4f58a +F test/dblwidth-a.sql eb4141518610e52f931a55a984310075e98dc31eee5a28ae806b1e35377be85a +F test/dbpage.test 63fab1eb026bada121107e53436fa749bbf83281dc9dea17af422f7a5c0f289f +F test/dbpagefault.test ea39de2ca86041a9c6df1135645180a76d0a8da93ac159e2fafe38e39636530b F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 -F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef -F test/decimal.test fcf403fd5585f47342234e153c4a4338cd737b8e0884ac66fc484df47dbcf1a7 -F test/default.test 9687cfb16717e4b8238c191697c98be88c0b16e568dd5368cd9284154097ef50 +F test/dbstatus2.test a36518c0f0951d8fd5a3dc36f99948ad1af93fb7fc0d2e03e5bb5a643186cf52 +F test/decimal.test a11b87a2c3294eb4c11b55f3168aeac63d683fa8ada35921a6ec1d511eff7648 +F test/default.test c7124864cded213a3f118bc7e2e26f34b7c36dfa26cf6945cc8b7f5db1191277 F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab -F test/delete4.test 51fafebe9503a40796d1aae1565c60524cada720e50eecac01b7fd0419d9ea0b +F test/delete4.test 2bcf66fe9978206e8fad1bacb69c00fe05dd1a3f2036c421420f6360f7d18b53 F test/delete_db.test 096d828493c7907f9ea11a7098ea6a0f73edba89406487d5d6cc2228dc4ab8b0 F test/descidx1.test edc8adee58d491b06c7157c50364eaf1c3605c9c19f8093cb1ea2b6184f3ac13 F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad78298 F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 -F test/distinct2.test bb71cc7b5e58e895787f9910a788c254f679928d324732d063fe9bc202ecbe71 -F test/distinctagg.test 14ec5026e684eddd414c61c08692b43773e224ac92efbed6ec08c6994bc39723 +F test/distinct2.test 4d6316b6487a0aa5a90bee111575c957e2a5ba5a9be9156febe9533ce78876e8 +F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50 @@ -961,13 +1063,13 @@ F test/e_createtable.test 31b9bcb6ac8876bc7ec342d86d9c231a84c62b442093a6651dfd0f F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 235c610f8bf8ec44513e222b9085c7e49fad65ad0c1975ac2577109dd06fd8fa F test/e_dropview.test 74e405df7fa0f762e0c9445b166fe03955856532e2bb234c372f7c51228d75e7 -F test/e_expr.test 27e905ed17266c745bffe65f56b809c13ae6e225e56aeda1aaec926b32439286 +F test/e_expr.test 0a1e175caddc78b27306647cb4ce2362c55790190f8cdd178b75fd6262eb8f76 F test/e_fkey.test feeba6238aeff9d809fb6236b351da8df4ae9bda89e088e54526b31a0cbfeec5 F test/e_fts3.test 17ba7c373aba4d4f5696ba147ee23fd1a1ef70782af050e03e262ca187c5ee07 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e -F test/e_reindex.test 2b0e29344497d9a8a999453a003cb476b6b1d2eef2d6c120f83c2d3a429f3164 +F test/e_reindex.test 027bb13d2c7e9e865886eed6349f126a273f8037899b636bf5fb53c7fc815921 F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 -F test/e_select.test 89fa483f68d868f1be3d6f56180ed42d979979c266e051bf8b5e66a251e6b44a +F test/e_select.test 327a15f14068bbd6f647cedc67210f8680fcb2f05e481a0a855fccd2abfa1292 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f3b30e9b41e4 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 @@ -978,76 +1080,51 @@ F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 -F test/enc.test 9a7be5479da985381d740b15f432800f65e2c87029ee57a318f42cb2eb43763a -F test/enc2.test 848bf05f15b011719f478dddb7b5e9aea35e39e457493cba4c4eef75d849a5ec +F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb +F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test f3f7548d2f3df03e2f23ecaf35c7c2cc7b89848bd7c3606d94a010521b7ea4f6 +F test/eqp.test 746db9fe11629a0d00328e1721cc2a2e4726d574b677ab14de35fd914f54cc82 +F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 +F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53 F test/eval.test 73969a2d43a511bf44080c44485a8c4d796b6a4f038d19e491867081155692c0 F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650bf0747 -F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 +F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/expr.test 5c06696478212e5a04e04b043f993373f6f8e5ce5a80f5548a84703b123b6caa +F test/existsexpr.test 75c1c13cda18b53f68c135fe644afc034d72a6de5b0774b0219ca4a6dc4b96b0 +F test/existsexpr2.test dc23e76389eff3d29f6488ff733012a3560cd67ec8cfaecbecd52cced5d5af11 +F test/existsfault.test ff41c11f3052c1bbd4f8dd557802310026253d67d7c4e3a180c16d2f0862973e +F test/expr.test db981f8a85520e99ae20aab7ad2e9b5b0437ed09159b57ced434c672075d2e61 F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/exprfault.test da33606d799718e2f8e34efd0e5858884a1ad87f608774c552a7f5517cc27181 -F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 -F test/external_reader.test c7d34694f1b25c32d866f56ac80c1e29edddc42b4ef90cad589263ffac2cde0c +F test/exprfault2.test c49e84273898969af5dbc4fe6a3f4335f14639799f343590336c9ddf84425965 +F test/extension01.test 5de412c66276105901c370770175003381fdcb0c4da7054fa43cf4a31e0bfa3a +F test/external_reader.test 6fdec43eeca23eb32faad1e95a4d1abc402bc8b3db70df12d6fc08a637f4a2b5 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 37a62e396a68eeede8f8d2ecf23573a80faceb630788d314d0a073d862616717 -F test/filectrl.test 6e871c2d35dead1d9a88e176e8d2ca094fec6bb3 +F test/filectrl.test 4b720117388cf6766d0b798e2dddd785953f8f371633b0c0084d2f34cf72336a F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4 F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f3727 F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8 -F test/filter2.test 485cf95d1f6d6ceee5632201ca52a71868599836f430cdee42e5f7f14666e30a +F test/filter2.test 3cc20eaea2ea1ab245197cc4a62468deb460b78f5aa9bd7d5d3353c2fe569bae F test/filterfault.test c08fb491d698e8df6c122c98f7db1c65ffcfcad2c1ab0e07fa8a5be1b34eaa8b F test/fkey1.test e563bcb4cb108ce3f40363cda4f84009dc89a39e2973076e5057ba99fca35378 F test/fkey2.test 1063d65e5923c054cfb8f0555a92a3ae0fa8c067275a33ee1715bd856cdb304c F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 6727452e163a427147e84e739da18713da553d79f9783559b04fdcd36d5c7421 -F test/fkey6.test d078a1e323a740062bed38df32b8a736fd320dc0 +F test/fkey6.test 668a7299e75899b0a3342c36df655be57f76a05aca3544bda939a6e676e2f000 F test/fkey7.test 64fb28da03da5dfe3cdef5967aa7e832c2507bf7fb8f0780cacbca1f2338d031 F test/fkey8.test 51deda7f1a1448bca95875e4a6e1a3a75b4bd7215e924e845bd60de60e4d84bf F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/fordelete.test ba98f14446b310f9c9d935b97ec748753d0144a28b356ba30d1f4f6958fdde5c +F test/fork-test.c 9ac2e6423a1d38df3d6be0e8ac15608b545de21e2b19d9d876254c5931b63edb F test/format4.test eeae341953db8b6bda7f549044797c3278a6cc345d11ada81471671b654f8ef4 +F test/fp-speed-1.c b37de94eba034e1703668816225f54510ec60fb0685406608cc707afe6b8234d +F test/fpconv1.test d5d8aa0c427533006c112fb1957cdd1ea68c1d0709470dabb9ca02c2e4c06ad8 F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c -F test/fts1a.test 46090311f85da51bb33bd5ce84f7948359c6d8d7 -F test/fts1b.test 5d8a01aefbecc8b7442b36c94c05eb7a845462d5 -F test/fts1c.test 85a525ce7428907469b4cce13d5563ce542ce64c -F test/fts1d.test a73deace5c18df4a549b12908bade4f05dcf1a2f -F test/fts1e.test 77244843e925560b5a0b70069c3e7ab62f181ed2 -F test/fts1f.test 2d6cb10d8b7a4e6edc321bbdb3982f1f48774714 -F test/fts1i.test 6bfe08cdfdced063a39a50c8601da65e6274d879 -F test/fts1j.test e3797475796043a161e348c46a309664cac83f7f -F test/fts1k.test 65d3b41487b9f738d11b0f00eca375c0ca6bd970 -F test/fts1l.test 15c119ed2362b2b28d5300c0540a6a43eab66c36 -F test/fts1m.test 2d9ca67b095d49f037a914087cc0a61e89da4f0c -F test/fts1n.test a2317dcd27b1d087ee3878b30e0a59c593c98b7a -F test/fts1o.test d1554caede42bba2c82fe613bcc921856c196b752449ead0470fac52a20fd3b8 -F test/fts1porter.test d86e9c3e0c7f8ff95add6582b4b585fb4e02b96d -F test/fts2.test e3fb95f96a650411574efc136f3fb10eef479ed7 -F test/fts2a.test 473a5c8b473a4e21a8e3fddaed1e59666e0c6ab7 -F test/fts2b.test 964abc0236c849c07ca1ae496bb25c268ae94816 -F test/fts2c.test ffb5a35230ac72c4354535c547965ce6824537c0 -F test/fts2d.test b7eaa671ca9a16997f3e5b158ee777ae21052b0b -F test/fts2e.test 2da13dbc2d009105f42196845c1e1ce136c03d38 -F test/fts2f.test cf84096235991709c1e61caa389632aa0a4f976d -F test/fts2g.test 3d26fe171bda6133ebf5a380731d70eaa2ef2f6f73d79769cf8946e622b6d597 -F test/fts2h.test 223af921323b409d4b5b18ff4e51619541b174bb -F test/fts2i.test 1b22451d1f13f7c509baec620dc3a4a754885dd6 -F test/fts2j.test 298fa1670aa21cd445b282d139b70c72e7ade12b -F test/fts2k.test c7ebf4a4937594aa07459e3e1bca1251c1be8659 -F test/fts2l.test 3333336621524cf7d60bb62d6ef6ab69647866ed -F test/fts2m.test 4b30142ead6f3ed076e880a2a464064c5ad58c51 -F test/fts2n.test 12b9c5352128cebd1c6b8395e43788d4b09087c2 -F test/fts2o.test 4054ac7433eb5440f1b1d200cfa449342dc4aabd991759139813e17c73e5bf9a -F test/fts2p.test 4b48c35c91e6a7dbf5ac8d1e5691823cc999aafb -F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682 -F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e -F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 F test/fts3_common.tcl dffad248f9ce090800e272017d2898005c28ee6314fc1dd5550643a02666907a F test/fts3aa.test 814d60a1ba30b4a71d8f9306a6564bc7b636dd6efacd2ad80306f9b23ef3ebee @@ -1066,20 +1143,22 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test 266989148fec6d9f1bb6c5382f7aa3dcea0e9cd444576e28dd2b9287ac7dd220 F test/fts3atoken.test dc2078ce464914efe3a8dfc545dd034a0fc14f2ab425c240471d5a5f1c721400 +F test/fts3atoken2.test aa6664d26277064844ead962f5988c28b091c51b112f36d30fa33bd83fecf906 F test/fts3auto.test 649aa4c198d7acc5cd6355e19ee073d051c40d9e88a43fc3d88af46bdf3e99d5 F test/fts3aux1.test 1880eaa75c586cd10f53080479a2b819b3915ae7ce55c4e0ba8f1fe05ac0a6a7 F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f F test/fts3b.test c15c4a9d04e210d0be67e54ce6a87b927168fbf9c1e3faec8c1a732c366fd491 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c -F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f5a76b +F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d31f29 F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test 589e043d1114ea02c83530e459ef5c2d6adce63094e72826ed3bceb02e795116 +F test/fts3corrupt4.test c7f414fe29b97a478d15c90382c4ae077a2bbd2283bf8c63bf66dadaaed3edb8 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 -F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf +F test/fts3corrupt7.test 1da31776e24bb91d3c028e663456b61280b121a74496ccf2fef3fe33790ad2b0 +F test/fts3cov.test 1e5ecea0e4c1394cea97adcfb9fd3d2d5998fd563dacf465f413e6c7fa5cffb3 F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f F test/fts3defer.test f4c20e4c7153d20a98ee49ee5f3faef624fefc9a067f8d8d629db380c4d9f1de F test/fts3defer2.test 3bbe54a7fca7d548bb7ac4f59447ee591070bfbe0c9f3e279defa0b898e9afbb @@ -1088,16 +1167,18 @@ F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3dropmod.test 7de242ea1c8a713a8b143ea54468f4b1c4953fa068349e23ac178e2c90c59889 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test ebae205a7a89446c32583bcd492dcb817b9f6b31819bb4dde2583bb99c77e526 -F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a +F test/fts3expr2.test ef381978b9bdcfc6d4b47c86270ba356a75dbff164ce52ff5d18a5924a788614 F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 F test/fts3expr4.test 6c7675bbdbffe6ffc95e9e861500b8ac3f739c4d004ffda812f138eeb1b45529 F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e F test/fts3f.test 8c438d5e1cab526b0021988fb1dc70cf3597b006a33ffd6c955ee89929077fe3 -F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45c5df45 +F test/fts3fault.test 9228f00cd69e2a5d2ed0f06c181981f4f90bd36da9f86b73f3a58b4b23451fd4 F test/fts3fault2.test 7b2741e5095367238380b0fcdb837f36c24484c7a5f353659b387df63cf039ec +F test/fts3fault3.test ccdd2292dd2d4e21e30fc5f4c8e064f79e516087eec5ff57ab6bc4f6a7714097 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 -F test/fts3fuzz001.test e3c7b0ce9b04cc02281dcc96812a277f02df03cd7dc082055d87e11eb18aaf56 -F test/fts3join.test 1a4d786539b2b79a41c28ef2ac22cacd92a8ee830249b68a7dee4a020848e3bb +F test/fts3fuzz001.test c78afcd8ad712ea0b8d2ed50851a8aab3bc9dc52c64a536291e07112f519357c +F test/fts3integrity.test 0c6fe7353d7b24d78862f4272ee9df4da2f32b3ff30fa3396945cda8119580a8 +F test/fts3join.test de31d304ba479043a7d33d2f201c514b3e1da809da6797d7a58704d00e8da2e6 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test aa66cc50615578b30f6df9984819ae5b702511cf8a94251ec7c594096a703a4a F test/fts3matchinfo2.test 00144e841704b8debfcdf6097969cd9f2a1cf759e2203cda42583648f2e6bf58 @@ -1110,22 +1191,23 @@ F test/fts3query.test 45806a302921b245a9dba5d85c9d51fb98b3f137eea6e6bf6eae4883e0 F test/fts3rank.test cd99bc83a3c923c8d52afd90d86979cf05fc41849f892faeac3988055ef37b99 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 57e26a801f21027b7530da77db54286a6fe4997e -F test/fts3snippet.test 0887196d67cffbe365edde535b95ecc642a532ce8551ccd9a73aab5999c3ffae -F test/fts3snippet2.test e79afeb1f673713f96d7fc5655726081975399d11e659d15553207be43301dc4 +F test/fts3snippet.test 560c7f38c5fa591d88e367eac1313b64e503625616708ff61da9d5f52cbf75e5 +F test/fts3snippet2.test 03f6738ab3897bea2ba6be424a0613872e167acbf37a66200d655d737b470f65 F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test a663f4cac22a9505400bc22aacb818d7055240409c28729669ea7d4cc2120d15 F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 0b84a3fd4eba8a39f3687523804d18f3b322e6d4539a55bf342079c3614f2ada F test/fts4aa.test 0e6bfd6a81695a39b23e448dda25d864e63dda75bde6949c45ddc95426c6c3f5 -F test/fts4check.test 6259f856604445d7b684c9b306b2efb6346834c3f50e8fc4a59a2ca6d5319ad0 +F test/fts4check.test f0ea5e5581951d8ef7a341eea14486daf6c5f516a2f3273b0d5e8cb8a6cd3bd2 F test/fts4content.test 73bbb123420d2c46ef2fb3b24761e9acdb78b0877179d3a5d7d57aada08066f6 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 289833c34ad45a5e6e6133b53b6a71647231fb89d36ddcb8d9c87211b6721d7f F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d -F test/fts4langid.test 89e623218935507bca69d076ca254a7a8969dfc681c282b6374feaea8c7de784 +F test/fts4intck1.test 54e7f28e34b72fb0c614d414bb1f568154d463c5a00b20944e893df858372ed4 +F test/fts4langid.test 1cc6fe045f094f1695d0b20e6b817a2ce22ec470cb7c6577470cd71ed0256e90 F test/fts4lastrowid.test 185835895948d5325c7710649824042373b2203149abe8024a9319d25234dfd7 -F test/fts4merge.test e2b2ec21e287d54ec09824ccfb41e66896eeca568fc818ba0e0eb2efd94c35d2 +F test/fts4merge.test 57d093660a5093ae6e9fbd2d17592a88b45bbd66db2703c4b640b28828dbe38b F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test 66fce89934cd9508cbdc67de486558c34912ffb2e8ffe5c9a1bbb9b8a4408ba7 @@ -1140,154 +1222,167 @@ F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f75 F test/fts4unicode.test 82a9c16b68ba2f358a856226bb2ee02f81583797bc4744061c54401bf1a0f4c9 F test/fts4upfrom.test f25835162c989dffd5e2ef91ec24c4848cc9973093e2d492d1c7b32afac1b49d F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test 4be8bed4be235e333f1e0ea31e32f5be3c9f456c30780363e7fcb15e3ff3e6bc -F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f -F test/func3.test 600a632c305a88f3946d38f9a51efe145c989b2e13bd2b2a488db47fe76bab6a -F test/func4.test 2285fb5792d593fef442358763f0fd9de806eda47dbc7a5934df57ffdc484c31 +F test/func.test b6b6942b29e1b00120cb4ed00ba8ed8ef561e7ba6ae1c41cc005143c7e6f5bf6 +F test/func2.test 69f6ae3751b4ec765bdc3b803c0a255aa0f693f28f44805bef03e6b4a3fd242f +F test/func3.test 6b22dfada95b778d01dba75aa00fa8e6a1fb90d30b859241638e7540edcfb3ca +F test/func4.test a02e695f62beb31cb092dccf6873ff97543407fff97a5f3ec4da70b5b337bc84 F test/func5.test 863e6d1bd0013d09c17236f8a13ea34008dd857d87d85a13a673960e4c25d82a -F test/func6.test 9cc9b1f43b435af34fe1416eb1e318c8920448ea7a6962f2121972f5215cb9b0 -F test/func7.test adbfc910385a6ffd15dc47be3c619ef070c542fcb7488964badb17b2d9a4d080 +F test/func6.test 3bc89ec0f2605736d3a118f43d25ef58115a7db4dba8ae939a363917d815c0bb +F test/func7.test 7e009275f52c52954c8c028fdb62f8bc16cc47276fcc8753c1d2b22c6e074598 F test/func8.test c4e2ecacf9f16e47a245e7a25fbabcc7e78f9c7c41a80f158527cdfdc6dd299d +F test/func9.test 32c7d53b9bc99ddd45fad29457add438f2aca43d24920793773a96b13610f552 F test/fuzz-oss1.test 514dcabb24687818ea949fa6760229eaacad74ca70157743ef36d35bbe01ffb0 -F test/fuzz.test 4608c1310cff4c3014a84bcced6278139743e080046e5f6784b0de7b069371d8 +F test/fuzz.test 819ea7e483bcee91209aacbe6f9eaf3287baa1841479ee5f639f57c5e7c42b86 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 -F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c31 +F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce313 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 0d90cee9fd7ebecdfbdbe5bdc6fad92fa32410c48ccad747cd9e0ac603b9130c +F test/fuzzcheck.c 34a025386f84d818cd3343e69e9d9083091af83153e226d71d4e1c126b5f1dd0 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e42ed2 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5 -F test/fuzzdata6.db 92a80e4afc172c24f662a10a612d188fb272de4a9bd19e017927c95f737de6d7 +F test/fuzzdata6.db b8725a5f5cf7a3b7241a9038e57ca7e7cc8c3f4d86b44bd770617bda245ab2b0 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2 -F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c3493f80 +F test/fuzzdata8.db 8f34ae00d8d5d4747dd80983cf46161065e4f78324dcff3c893506ff8db3a4a6 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c b34530e8431f2cf3591eff588fc7684d6fdef466916fb46141c8c5374a3d8099 -F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c -F test/gencol1.test aef8b0670abd4b1ae4cae786b15a43758d86f6cd9f12b381d45d96bb51e597c9 +F test/fuzzinvariants.c 3ddfec7f5b970b018f1a982532de905cf180e0c1e48cd653be9365d3e6177625 +F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d +F test/gencol1.test ceb3163b59cb77f4ad57ae4f01a143ce36b06fdd6a8dab1149235db89979ffd8 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/having.test a89236dd8d55aa50c4805f82ac9daf64d477a44d712d8209c118978d0ca21ec9 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 18cae9140fa7f9a6f346e892a3fe3e31b2ca0be1494cd01b918adb74281016a6 +F test/hook.test 2d89bf9480646feb8093be3a58ea502d6521906779ed960de31dd9c4502c0541 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 -F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1 -F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e -F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test d1cad4ededd425568b2e39fb0c31fa9a3772311dd595801ff13ba3912b69bba6 +F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 +F test/ieee754.test 0d3ab84ab2069c9994c833a7cd820ee6037f0cf888e206a4a7fc05f735d5790a +F test/imposter1.test 5a20b2cdeb53e65fc57cdb10a33750bd4ef6259909eaf1972253b9e79f7a3fb2 +F test/in.test edf979bff3244b9e47849e2b43886631354c8213791f42da92216f08012141af F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test fdd1d8134da8376985c2edba6035a2de1f6c731524d2ffa651419e8fe2cd1c5a -F test/in5.test b32ce7f4a93f44c5dee94af16886d922cc16ebe33c8e1765c73d4049d0f4b40f +F test/in4.test e7b1456d423453884aeb9d68fafe9c8e7be7abddc8f028c04205e67820f10772 +F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b +F test/in7.test d9efdee00b074a60c6343993b2eda78bc369ab080dad864513c73f8aca89d566 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f F test/incrblob3.test 67621a04b3084113bf38ce03797d70eca012d9d8f948193b8f655df577b0da6f -F test/incrblob4.test 21a52a6843a56cdcce968c6a86b72a7066d0e6ba +F test/incrblob4.test a8d6b5ff04055fcec747a50b78485ebf4fcd80074e0b3cedbe952bde346da54a F test/incrblob_err.test 89372a28f1d98254f03fed705f9efcd34ef61a674df16d2dbb4726944a2de5e9 F test/incrblobfault.test de274b1e329169c2c3438f9528994807ea8201ebf38ae9f157d34bf3ec0cc549 F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test 3fa6145f5e71f603554fd7b8ec3da4290b1341029682313285cb5f9e1893d6ba F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d -F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a +F test/incrvacuum3.test 0bf0ffe7f2cbc87ba1d471e4bbadabbf10dacf8d4ee26b3a072708d575d637a9 F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test d866054c88b394fd42cbf2825628f127ca24dfac525fa019069a936674d92cbe +F test/index.test 9a060cbb12b70dd0e6bca0404106242703c0827ba58712f46b0d5a3b3968b341 F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test b376a648e85aa71c50074382784e6cb0c126ec46e43d1ad15af9a4d234c52e65 +F test/index6.test 656502465b30c3f8de4eb956928aaded02cf36128e8fc8be2b95390fb23e74f5 F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6eb3 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 -F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test 62558b1cfd7ccbe7bc015849cc6d1a13ef124e80cbd5b3a98dc66c3c9cce0cf4 +F test/indexA.test 11d84f6995e6e5b9d8315953fb1b6d29772ee7c7803ee9112715e7e4dd3e4974 +F test/indexedby.test 444fb04ce0b21a3daf79f84e6735b49e5a5b3396623b37df5431eb09c8b8f557 +F test/indexexpr1.test d32dba192b8a566a81bb437ee8b6219931458e010c2d94610da775fa9d318f17 F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a +F test/indexexpr3.test 47b91bc7999805c9a34d356f672259bc49295ecc797448511cae554a309b47cd F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 -F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6 +F test/insert.test 97cfb30b83ca1622b9422a1e4c4831b4cb767cf5d654660945036d1e72067e70 F test/insert2.test 4d14b8f1b810a41995f6286b64a6943215d52208 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 2bf81535a990c969665d66db51fcf76c23499b39893b5109f413d1de4ad34cd3 F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 F test/insertfault.test ac63d14ea3b49c573673a572f4014b9117383a03e497c58f308b5c776e4a7f74 -F test/instr.test 107df2b9b74a4b59315916b575590a08f2a714de0754abe541f10a0971d0a2a4 +F test/instr.test 67ba309e9697c24a304e98a7c8f372456177dd4e32237d2a305e1e05f7bb79c2 F test/instrfault.test 95e28efade652e6d51ae11b377088fe523a581a07ec428009e152a4dd0e0f44c F test/intarray.test bb976b0b3df0ebb6a2eddfb61768280440e672beba5460ed49679ea984ccf440 F test/interrupt.test ac1ef50ec9ab8e4f0e17c47629f82539d4b22558904e321ed5abea2e6187da7a F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d -F test/intpkey.test aee694afed1a65c86c4e69ad030224b3fc268113d00685234d40079fca16bad3 +F test/intpkey.test 7d54711acf553cdd641a40e9c6cfc2bf1a76070074940c1b126442517054320f F test/intreal.test 68829a8bb073ee1610ca3f8f9e0f99b0371fb36e0fa64862dd5ced4ef03c2343 -F test/io.test f138f3fe696d1ed8c51dfea5b325910d319a1b29e1d25ea57231a02092f02cca -F test/ioerr.test 530d05801ff1b6327018b2e140da34a74effa2773a844ddb8dc79c32e9567318 +F test/io.test d267fdc8915444a45e19841489033ebe70bb69f6db605b00df70be16b2a80f59 +F test/ioerr.test 78552a95d53b9674d85f63bdf3d76a8df70b4d5dba5a6a1b8a1d60f166cd2c6b F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c -F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 +F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d520 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test f7abfef3faeaf2800308872e33a57e5b6e4a2b44fb8c6b90c6068412e71a6cf4 -F test/join2.test 8561fe82ce434ac96de91544072e578dc2cadddf2d9bc9cd802f866a9b92502e +F test/join.test 2fcfd84640cfd9ff48f31b4b0d370c4d5498c355ae4384544668ca54d37ae186 +F test/join2.test f59d63264fb24784ae9c3bc9d867eb569cd6d442da5660f8852effe5c1938c27 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 91f1f4c7d81fd87b58e9ba7cf4a2b5d39e3583b4f8e498a162722a60259c5208 +F test/join5.test 76f99f69a410241de73fa5cd3f4a6ef469e64bb501a48de2437881792111c589 F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c F test/join7.test 2268dcbb54b724391dda3748ea95c60d960607ffeed67885675998e7117697f6 F test/join8.test d384d63985e3991c404afccadaf3efd1cdf9cd72680167f80e3cb80b95c18c68 F test/join9.test 9056ddd3b0c0f4f9d658f4521038d9a37dc23ead8ca9a505d0b0db2b6a471e05 -F test/joinA.test 7eab225dc1c1ab258a5e62513a4ed7cabbd3db971d59d5d92f4fb6fa14c12f6a +F test/joinA.test 6ac4efdbb1eb9ca398162c5bc5623a757803b04bb4d76453c8563a0bdc2f73bd F test/joinB.test 1b2ba3fc8568b49411787fccbf540570c148e9b6a53a30f80691cb6268098ded F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f207 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test 705157cf9b9b7c207caf960812a7d0e4dc1dd45aa5fb2b563f12df59088645f3 -F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 +F test/joinH.test 1d2fc3190be68525fd9ce749b9468c40ba2930181e52fb5ee6f836051b38effb +F test/joinI.test fc7d24a2b1e444979b83bd92c30ebb975cebb5b9eae4442ce94969bd8d083053 +F test/journal1.test bc61a4228db11bffca118bd358ba4b868524bf080f3532749de6c539656e20fa F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 -F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e +F test/journal3.test e5aeff93a7776cf644dbc48dec277655cff80a1cd24689036abc87869b120ea6 F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json/README.md 506af1f54574b524106acb50d1a341ab5ddfa6d83fe25095007892b07e663e85 +F test/json/README.md de59d5ba0bd2796d797115688630a6405bbf43a2891bad445ac6b9f38b83f236 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f -F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d -F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x -F test/json101.test 94126d4291d4a00e45f6988ce885c410de69243490e46e70e9946cb6e6f9ea02 -F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8 -F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe +F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 +F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x +F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 +F test/json101.test cf53254f0f0c1399a01b21fc58fee0e63a12a556be91b9ee9faccdb8b82c083c +F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1 +F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 -F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d -F test/json501.test f71710f60fa45b19dc336fbaac9e8362f70f80cf81badefdb845ed3f7c7c2ccc -F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b +F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 +F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8abef +F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 +F test/json108.test 0a5f1e2d4b35a1bc33052563d2a5ede03052e2099e58cb424547656c898e0f49 +F test/json501.test b95e2d14988b682a5cadf079dd6162f0f85fb74cd59c6b1f1624110104a974eb +F test/json502.test 4ef68e4f272dfb083d4cbceb4e9e51d67ec1186a185e0c13637c50a4dc2f9796 +F test/jsonb01.test f4cdfb4cf5a0c940091b17675ed9583f45add0c938f07d65b0de0e19d3a9a101 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff -F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b +F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 -F test/like.test 5fe0bc37f307aef0a453ce2de4632bdfc0759448f0421c39f6d53caefe905fac +F test/lemon-test01.y 70110eff607ab137ccc851edb2bc7e14a6d4f246b5d2d25f82a60b69d87a9ff2 +F test/like.test 0036f8fe548fceabd1496361bfa262f35cf5ed17c794bd15e22e9f3de12e0eb0 F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3 -F test/like3.test a76e5938fadbe6d32807284c796bafd869974a961057bc5fc5a28e06de98745c +F test/like3.test 3c9be7a0122908d8ead6aa25e47c2b9787eea88e8062926078ea7b3e95c71d99 F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf -F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e -F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 +F test/limit2.test 621188fc3e5c3b8d2ef9827e05fa8313792ae563579073136efd25cb65325f1b +F test/literal.test a65dca9fef86e51b8e45544268e37abbd4bb94ba35fd65f6fdcab2f288cd8f79 +F test/literal2.tcl 1499037beaf661aeecdbe48801220a181d805372a64c6128d5f26bb6a4a8f0ce +F test/literal2.test b149e16b5fc9ee6249069a8858ed41052f222014fe0ba7ad43c2fb989c2dada2 +F test/loadext.test 878db71cf74b48250dbe9033bbfe6088ff869db3353ffd4febc68c0cd459959e F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db +F test/lock.test 675b4367ec58b21009e46982d071c8259255e69072296b7756ea75fac5425d1a F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37b8 -F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 +F test/lock5.test 583cae05992af0f66607286917f7d5f8aed3b6053c52df5994efb98f2a8fdbaf F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 -F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413 +F test/lock_common.tcl f0a1f7b8f3fbb8629dc6231613a02841736f86ef72151429d5ffc12c7f613fb3 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083 -F test/main.test 6bbb3999fd461eb8fb335cbab97409a3d7f91bbb8da60635e8be3e4a04a77772 +F test/main.test e8752d76233b1c8906cd2c98ad920dba868bd63c87d51d8a2ea5e9cba55dd496 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test 18dd1c4188c81ca79cf123527c71b19ee0c31feb9947fdffb0dc6ceb1436816a +F test/malloc.test f7781c8151179fe4b7f743044a737ac2dfd87bf9cc18dd01398caf28d34e09c5 F test/malloc3.test 6e88bae6312854a4adb4ecc2a6a5ea8c59b4db778b724ba718e1c43fc8c3c136 F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test 2e4ad7684a13389a44a840499cd47173a8d05f22f082d7d083eece433a7a64eb @@ -1314,49 +1409,50 @@ F test/malloctraceviewer.tcl 3e3ddf11e30d2b20f53aa16aa6615082fb24a100bea61cca721 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 -F test/memdb1.test 2c4e9cc10d21c6bf4e217d72b7f6b8ba9b2605971bb2c5e6df76018e189f98f5 -F test/memdb2.test 7789975b96b0726032a22c1afc026592c7ff175bf05be11f1d640791541a77ea +F test/memdb1.test c737ac9aa5895092332b1dde24fae7ae494b7fcbcd346d22d600891096a3836d +F test/memdb2.test 4ba1fc09e2f51df80d148a540e4a3fa66d0462e91167b27497084de4d1f6b5b4 F test/memjournal.test 70f3a00c7f84ee2978ad14e831231caa1e7f23915a2c54b4f775a021d5740c6c -F test/memjournal2.test 6b9083cfaab9a3281ec545c3da2487999e8025fb7501bbae10f713f80c56454c -F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 +F test/memjournal2.test dbc2c5cb5f7b38950f4f6dc3e73fcecf0fcbed3fc32c7ce913bba164d288da1e +F test/memleak.test c7478f1195d64887dd1c677edc39fa03b5bf29024e6dcc5b5cc554d7ed00b01f F test/memsubsys1.test 86b8158752af9188ed5b32a30674a1ef71183e6bc4e6808e815cd658ca9058a6 F test/memsubsys2.test 774b93cb09ca50d1b759bb7c645baa2a9ce172edc3a3da67d5150a26a9fc2a08 -F test/merge1.test 2de6d6ef8d25402764b1aab49d8f9d7f89208c89a6674e437f76de4c812157b8 -F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a +F test/merge1.test 7dd9dc6838bcd0623a069485fe3a8dd498a051c16e1877cf84f506c0d6a29b43 +F test/minmax.test 885d89737b955905b52bd5b203f27b9b56d1e399b8155c2a95ab0f3264e0f582 F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 272ca395257f05937dc96441c9dde4bc9fbf116a8d4fa02baeb0d13d50e36c87 -F test/misc1.test 8d138a4926ab90617c1aa29ce26e7785ae2b83a4d3a195d543b7374e05589dd1 -F test/misc2.test 71e746af479119386ac2ed7ab7d81d99970e75b49ffd3e8efffee100b4b5f350 -F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d +F test/misc1.test e3e36262aff1bd9b8b9bf1eeb3af04adb3fc1e23f0a92dbff708bba9e939ace1 +F test/misc2.test a1a3573cc02662becd967766021d6f16c54684d56df5f227481c7ef0d9df0bd0 +F test/misc3.test 651b88bca19b8ff6a7b6af73dae00c3fd5b3ea5bee0c0d1d91abd4c4b4748718 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e -F test/misc5.test c4aeaa0fa28faa08f2485309c38db4719e6cd1364215d5687a5b96d340a3fa58 +F test/misc5.test 02fcaf4d42405be02ec975e946270a50b0282dac98c78303ade0d1392839d2b8 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 -F test/misc7.test d912f3d45c2989191b797504a220ca225d6be80b21acad22ba0d35f4a9ee4579 -F test/misc8.test 4db9f8be59834cea08c87e9658014080efa02678ef54a088f84fa5647e81fee0 -F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7 +F test/misc7.test d595599972ec0b436985f0f02f243b68500ffc977b9b3194ec66c0866cfddcab +F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd +F test/misuse.test 859f37014d9824ca66bd90c36372c08c80c51c9593a7cfa8a31d4f92cd4d5b7f F test/mjournal.test 28a08d5cb5fb5b5702a46e19176e45e964e0800d1f894677169e79f34030e152 -F test/mmap1.test 5c1f768828094b0dd94e55ae7f10489a1ded74772682be2c4c78679d0acaf7ef -F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 +F test/mmap1.test 18de3fd7b70a777af6004ca2feecfcdd3d0be17fa04058e808baf530c94b1a1d +F test/mmap2.test dba452dc7db91e9df10f70bdd73dc4190c7b8ee7b5133b4684f04277ada0b9ac F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 +F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test 4d7ecb155ae5ba9ca6f1817c1c74d51b5bb284d7f599de1859a9f2c9a1ca0d38 +F test/mutex1.test 42cb5e244c3a77bb0ef2b967e06fa5e7ba7d32d90a9b20bed98f6f5ede185a25 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 -F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 -F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a +F test/nan.test 73ea63ab43668313e2f8cc9ef9e9a966672c7934f3ce76926fbe991235d07d91 +F test/nockpt.test 3db354270fc63b6871eebd40285d4c55324fb27be629c958adbff6d7fcaa8e14 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/normalize.test f23b6c5926c59548635fcf39678ac613e726121e073dd902a3062fbb83903b72 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test 1ee4acbd614d3cf5f1c4a52f5af7fc771b82352f1a51a86afeaa02c9df1d82ef +F test/notnull2.test 5b7dd6e82c409b2d011ad6acf19ae4bf0816a9c69ccf600b529d7405d7c49874 F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 @@ -1367,85 +1463,88 @@ F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 F test/optfuzz.c 690430a0bf0ad047d5a168bf52b05b2ee97aedaad8c14337e9eb5050faa64994 -F test/orderby1.test 02cfd870127a7342170b829175c5c53e9e7405744451ac1aeb2f7e2b0c18ca76 +F test/orderby1.test 64d6aac62e68c289d378a18cea9b9941d812b6a26f141a18c9fbde79bd7f28db F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 -F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 -F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 +F test/orderby3.test 64465d29ab6dabb38731ff24ead2aa0bb262bb06a7b6aa17c6f874a5f24fee94 +F test/orderby4.test daff719d0eead9ae92863a0b5a2e367bfdc7efcc9dfe16d83d667f84df597eff F test/orderby5.test bd7d9e3380e87e5dcf6ea817ebaab6d15da213c7804b38767e1b3e695e85650b F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/orderbyA.test df608e59efc2ef50c1eddf1a773b272de3252e9401bfec86d04b52fd973866d5 -F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 -F test/ossfuzz.c 9636dad2092a05a32110df0ca06713038dd0c43dd89a77dabe4b8b0d71096715 +F test/orderbyB.test 32576c7b138105bc72f7fbf33bd320ca3a7d303641fc939e0e56af6cba884b3d +F test/oserror.test ee3fad06ec8671c4d047c2c92a567fc2e0e8161caaec7edd6d48325c5ac97f30 +F test/ossfuzz.c b5d232d9717fc999a121c82c4880ae5b9d7fb3ae55d2d87a8da906bc80020906 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f -F test/pager1.test ffd885cdc98b986c9f746496508c0c4810ed0eaade3575ddf53c222e85880552 +F test/pager1.test b083c2d5d89df8e979658d9320bfc0b9d50b4ef8ae1d9e115a692ff0b9768393 F test/pager2.test c0ede15952b607f9a38f653acdfa73c19e657958e9104aab1a71950ea7b71831 F test/pager3.test 4e9a83d6ca0838d7c602c9eb93d1357562d9059c1e02ffb138a8271020838370 -F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test 63c5da625562c66345ab4528790327ca63db2f6f9cbae2aba8cb7c51de3d1628 +F test/pager4.test b995066c699472614eb5949db5a2e2c51fd463863518afe68675d7fac09216bd +F test/pagerfault.test 43692e660fe480812dc5d44171fdcb8da1a65a644428def1ee9de79edace4028 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035ce4b3 +F test/parser1.test 131f4733472252d53d8ed681115257866f55740ab697fa05900d766049348f27 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 -F test/pendingrace.test cbdf0f74bc939fb43cebad64dda7a0b5a3941a10b7e9cc2b596ff3e423a18156 -F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 8bd6b6db541e2a7f9bb894be99ef5c00526b23762c4a00c574e1cba697495125 +F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 +F test/percentile.test eaee1ff3e35d5fe933ac98d927c9a7d2f4f1a1a19ee22e7d45e97a6d9ee32077 +F test/permutations.test e6de4f5777f7785737ac3d1d964b8656e5477a134665b2fe8a91884ab9b685b3 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f -F test/pragma.test 57a36226218c03cfb381019fe43234b2cefbd8a1f12825514f906a17ccf7991e +F test/pragma.test 7d07b7bb76e273215d6a20c4f83c3062cc28976c737ccb70a686025801e86c8f F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 92a46bbea12322dd94a404f49edcfbfc913a2c98115f0d030a7459bb4712ef31 -F test/pragma4.test ca5e4dfc46adfe490f75d73734f70349d95a199e6510973899e502eef2c8b1f8 +F test/pragma4.test 396ef9bff1fb966d41721545ad4b12bfc26aae315f5fe51d9b917828d49e6f8e F test/pragma5.test 7b33fc43e2e41abf17f35fb73f71b49671a380ea92a6c94b6ce530a25f8d9102 +F test/pragma6.test c5ec577ba087954b4dfa619a3cbe97b155b60a0af487527abe89b10fc17e6512 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/prefixes.test b524a1c44bffec225b9aec98bd728480352aa8532ac4c15771fb85e8beef65d9 -F test/printf.test 512152dca7f2f578f045a5a732e7bee08e4f47a8a212f83ce46791b518eba70f +F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b05333de F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 1495a09837a1cedfc0adf07ba42dc6b83be05a2c15de331b67c39a0e22078238 +F test/pushdown.test 46a626ef1c0ca79b85296ff2e078b9da20a50e9b804b38f441590c3987580ddd F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 -F test/quickcheck.test f86b25b33455af0189b4d3fe7bd6e553115e80b2d7ec9bbe9a6b37fce0881bfe +F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 -F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a +F test/quota.test 2747bfdf50d01155c06feb391286bb585f66977feaf5e26e5a37bc00ff7dee7c F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 -F test/quote.test ffb40f0eb7a25c1d8cfe11ee2fe67f8e85fbf3fed348810834114be1fdada142 +F test/quote.test 7b01b2a261bc26d9821aea9f4941ce1e08191d62fc55ba8862440fb3a59197a4 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test fd5199f928757cb308661b5fdca1abc19398a798ff7f24b57c3071e9f8e0471e +F test/readonly.test 0d307c335b3421898cfe64a783a376138aa003849b6bff61ee2d21e805bc0051 +F test/recover.test c76d05f33f0271fba0f0752170e03b0ab5952dc61dcea7ab3ba40df03c4c42de F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 -F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 +F test/regexp2.test 64f9726b2ddc71aea06725fcad53231833d038d58b936d49083ace658b370a13 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d -F test/releasetest_data.tcl b550dd1b122a9c969df794d05ea272df535f10ff1a245062e7ba080822378016 +F test/reservebytes.test 6163640b5a5120c0dee6591481e673a0fa0bf0d12d4da7513bad692c1a49a162 F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test db532cde29d6aebbc48c6ddc3149b30476f8e69ca7a2c4b53986c7635e6fd8ec -F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 -F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa +F test/returning1.test cd32517148948859db214dd814354597dd40e7489259590fac1a4f7bf44deb97 +F test/returningfault.test 5f9649d05680357ab077fa6bad574a3eb3f6e4858a6880b25171be516a4efcda +F test/rollback.test 952c4d805bca96adc2be76f621ea22115fe40b330015af36fcc8028c8547fcee F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a -F test/round1.test 1bb32cf3fc505eed9e86b5e523d07e15d4428189665524587512fbcc85d114bb +F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190cfd4 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test e29025be95baf6b32f0d5edef59a7633028325896a98f1caa8019559ca910350 -F test/rowvalue.test baf4fa3ec1a8c1c920c3faa5fd25959cb454bbd99ac8960397c34549d9fc4abe +F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e +F test/rowvalue.test 93474d8e1c496e970bdcc3a7f54ac835adda667d2fd971957b4bce0c0b11707b F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b -F test/rowvalue3.test 1347e25ca11c547c5a6ff0cc5626f95aa9740e9275bfaec096029f57cb2130ce -F test/rowvalue4.test 441e7e366ac6d939a3a95a574031c56ec2a854077a91d66eee5ff1d86cb5be58 +F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c +F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 -F test/rowvalue7.test c1cbdbf407029db01f87764097c6ac02a1c5a37efd2776eff32a9cdfdf6f2dba +F test/rowvalue7.test 06ec0aca725bf683313d03793aa2943bc7f45a901848c7056a9665b769c8fc38 F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 -F test/rowvalue9.test 138252b53b835208a5712e01595403a0ae32b4bc58284d9fe6bea10e58203fe4 -F test/rowvalueA.test 51f79b6098c193f838168752c9640f4eae6c63346bf64b5bed4f4e22fe2c71d0 +F test/rowvalue9.test 7499a8fd7ca3a3f0e19d94e135355439aa2b596f86b775ca8de79672da2ca378 +F test/rowvalueA.test be8d6ad8b476eb24c151bb20bfd487e0d50c5e99618b7b0e656035069d2fc2cf F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972d511c54fff F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1454,11 +1553,11 @@ F test/savepoint.test 6e9804a17767f08432c7a5e738b9a8f4b891d243110b63d3a41d270d3d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd -F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa +F test/savepoint6.test 48a645a7bb3a59a6fcf06a7364cfe5b655c336760de39068f7c241b0fc80d963 +F test/savepoint7.test 24c69af86d750c80d51cf6500fde9270717f2b6e5658f055b5e75af75d5af179 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e -F test/scanstatus2.test 9a00becb1d60d168944f8e2f3585d44d46123aa95c5c7c25563309e1f15f924d +F test/scanstatus2.test d85d17f2b0b4c013dde95232f7beab749f11f0ef847f5ecffb9486d2f5ecf9f9 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -1471,29 +1570,29 @@ F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/seekscan1.test 31af16e3bb3203d153aea320939c5da97ec44705c2710d153c06a01397d45b09 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 -F test/select3.test 8d04b66df7475275a65f7e4a786d6a724c30bd9929f8ae5bd59c8d3d6e75e6cd -F test/select4.test f0684d3da3bccacbe2a1ebadf6fb49d9df6f53acb4c6ebc228a88d0d6054cc7b +F test/select3.test 152ce3978c8600c70413e921e39e4c5aeb8ff1e96dd722442a9981e1b8afc702 +F test/select4.test 21941409ac6b65bfca83de020afb5d976d802d3d8ad216dc774a3fbbf55fe277 F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f390ae -F test/select6.test 9b2fb4ffedf52e1b5703cfcae1212e7a4a063f014c0458d78d29aca3db766d1f -F test/select7.test f659f231489349e8c5734e610803d7654207318f +F test/select6.test da91e61d26b8dea4b61e4a862088dd6ab19998f7be22a16a5b0cfe806e597639 +F test/select7.test b825420da8a0b5722fdb77f3369f6396a3d198c46e8787eb26ff9425d4ac9d27 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 -F test/selectA.test 6aef8b2136a4ac7a3e2e4161d2b8ca7bc6ebe2779de084f9bb66ca9e2323a937 +F test/selectA.test 1da8ce3884c326e11d2855baffb76436b0d7e044404af8a2a70d1399a4ff7e29 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d32189bfb F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae8840 -F test/selectH.test 88237ded5925adfb3f27fdafb5428c2ffc4d61e313bceb854e225ffc3ef0d206 +F test/selectH.test 0b54599f1917d99568c9b929df22ec6261ed7b6d2f02a46b5945ef81b7871aac F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb -F test/sessionfuzz.c 5eef09af01eeff6f20250ae4c0112c2e576e4d2f2026cc9a49dc5be6886fa6ee -F test/shared.test f022874d9d299fe913529dc10f52ad5a386e4e7ff709270b9b1111b3a0f3420a +F test/sessionfuzz.c f693b8827034a3bed7616d89c65fb4fe8b7ff3c0f000c6ea6beda69b7f1aced3 +F test/shared.test 50bd8091735b272732125928c363476a17b5fb264835de7d19e90c72055c888b F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 -F test/shared3.test f8cd07c1a2b7cdb315c01671a0b2f8e3830b11ef31da6baa9a9cd8da88965403 +F test/shared3.test cb92d083003ddf0f313166e494ec2fcafa55fdebf648628923ded3169dba8850 F test/shared4.test c75f476804e76e26bf6fa0e7b421fb0ca7d07558 -F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 +F test/shared6.test 104e1e25b4c4f47aaccca7dba75b3d87bb505b46b009af03ae49bf55b7c4976c F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 933ed7d71f598bb6c7a8c192a3cd30f2562fdccf514df383798599c34ffa672f F test/shared9.test 600a257fe9d8b0272746b230e761aa1bd8802ca4cf3ba5b2136b9204f3d51efa @@ -1501,36 +1600,38 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 300b77328aaafb9f3e7a53a26e4162fbf92181d92251d259ff105a2275ff998d -F test/shell2.test 09a202f57e7cd99788537f763e0845796a173fcea06a0d199a08d69446fe1daf -F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a -F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f -F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b -F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 -F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f -F test/shell8.test 3fd093d481aaa94dc77fb73f1044c1f19c7efe3477a395cc4f7450133bc54915 -F test/shmlock.test 3dbf017d34ab0c60abe6a44e447d3552154bd0c87b41eaf5ceacd408dd13fda5 +F test/shell1.test ebe953d64c937ad42a0f33170ac0d2d2568faae26813fc7a95203756446d54aa +F test/shell2.test ab23f01ea2347e4b72bb2399af7ee82aa00f9c059141749f7c4064abca5ad728 +F test/shell3.test 603b448e917537cf77be0f265c05c6f63bc677c63a533c8e96aae923b56f4a0e +F test/shell4.test 03593fa7908a55f255916ffeda707cdf55680c777736e3da62b1d78cde0d684d +F test/shell5.test d17e7927ab8b7f720efbdd9b5d05fceb6c3c56c25917901b315400214bf24ef4 +F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 +F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 +F test/shell8.test 641cf21a99c59404c24e3062923734951c4099a6b6b6520de00cf7a1249ee871 +F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209 +F test/shellA.test 4ecff8b7b2c0122ba8174abfbcc4b0f59e44d80f2a911068f8cd4cfc6661032d +F test/shmlock.test 9f1f729a7fe2c46c88b156af819ac9b72c0714ac6f7246638a73c5752b5fd13c F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 2668e607dcdfa19c52828c09b69685b38da793856582ae31debf79d90c7bbbdc F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test e03ba5b977da6fd71662a4b0a668f04053bda4b187ff3214db7533e28c732279 +F test/skipscan1.test 9cbbb6575517b15292bd87ee85b853bbd3cd4b4735d69b0f083020cec16ff304 F test/skipscan2.test b032ed3e0ba5caa4df6c43ef22c31566aac67783bc031869155989a7ccdb5bd5 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 0672103fd2c8f96bd114133f356192b35ece45c794fe3677e1d9e5e3104a608e -F test/skipscan6.test bddbb35dd335e2d21b7791a61e3b2e1f3255dc307ce80aa6fe19cc298e6feb13 +F test/skipscan6.test e2b256cf5d538a605beb97dc97ca5e2836dfc24c5e1d9b7a09e13c069a3b8b49 F test/snapshot.test a504f2e7009f512ef66c719f0ea1c55a556bdaf1e1312c80a04d46fc1a3e9632 F test/snapshot2.test 8d6ff5dd9cc503f6e12d408a30409c3f9c653507b24408d9cd7195931c89bc54 -F test/snapshot3.test 8744313270c55f6e18574283553d3c5c5fe4c5970585663613a0e75c151e599b +F test/snapshot3.test 2e0328ba019aa981848e10aded4d7dcd6094ec1f9c6290a34ab18415be0c44eb F test/snapshot4.test d4e9347ef2fcabc491fc893506c7bbaf334da3be111d6eb4f3a97cc623b78322 -F test/snapshot_fault.test f6c5ef7cb93bf92fbb4e864ecc5c87df7d3a250064838822db5b4d3a5563ede4 -F test/snapshot_up.test a0a29c4cf33475fcef07c3f8e64af795e24ab91b4cc68295863402a393cdd41c +F test/snapshot_fault.test 129234ceb9b26a0e1000e8563a16e790f5c1412354e70749cbd78c3d5d07d60a +F test/snapshot_up.test 77dc7853bfb2b4fa249f76e1714cfa1e596826165d9ef22c06ac3a0b7b778d9a F test/soak.test 18944cf21b94a7fe0df02016a6ee1e9632bc4e8d095a0cb49d95e15d5cca2d5c F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 F test/sort.test f86751134159abb5e5fd4381a0d7038c91013638cd1e3fa1d7850901f6df6196 -F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 +F test/sort2.test 2f8c66402a03adebe77ce7aafca129fbf32df27d6c9b8f7a9f1b958e39f56da8 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5 -F test/sort4.test cca6f4b0b5255882645bbbe346a6a9f4a5c7b6a18513a6a7bf4ac1c4761ddc19 +F test/sort4.test c7a88629aecc8eec3c919eda54b221da5cf7a1b48f0cd372e7e832188d6737d8 F test/sort5.test 6b43ae0e2169b5ceed441844492e55ba7f1ae0790528395ddf7888ab3094525d F test/sorterref.test 9a606c86a4c682db5eeaaefa0565b52102778db53e48ca7101cd4f9ebcc0ad94 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66 @@ -1542,43 +1643,47 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c 9703465daf778dd90bad7adadc1d2ee205ec90a79437cababab0a4111190f0c8 +F test/speedtest.md ee958457ae1b729d9715ae33c0320600000bf1d9ddea1a88dcf79f56729d6fad +F test/speedtest.tcl 6b66974d833d35a63d0e9ec344e0ffa92fbbfac83e173556f700a61cb3be96fc x +F test/speedtest1.c 6c01252e66f46de0b6b8d5316e03521e2151782104f3608c10262aa5dce85721 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb -F test/sqldiff1.test 182058e09c7082de5c6a470ff9c291337bbeb650052c2cc68fbb3d7e25861d91 -F test/sqllimits1.test b28e5cc8d337aaf290614d96a47e8fbfb720bb7ad35620c9d5432996fd413ac4 +F test/sqldiff1.test 1b7ab4f312442c5cc6b3a5f299fa8ca051416d1dd173cb1126fd51bf64f2c3fb +F test/sqllimits1.test 408131e4975d61868711c83f101a56d4602313cc5cae88d3eee81c1da364fd89 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a +F test/starschema1.test f5388cd32527ab18d3f98f9e3402ec780f6a186e04e0d9c8531d7568ee734e11 F test/startup.c 1beb5ca66fcc0fce95c3444db9d1674f90fc605499a574ae2434dcfc10d22805 F test/stat.test 123212a20ceb496893d5254a5f6c76442ce549fdc08d1702d8288a2bbaac8408 -F test/statfault.test 55f86055f9cd7b2d962a621b8a04215c1cebd4eaaecde92d279442327fe648a0 +F test/statfault.test 064f43379e4992b5221b7d9ac887c313b3191f85cce605d78e416fc4045da64e F test/stmt.test 54ed2cc0764bf3e48a058331813c3dbd19fc1d0827c3d8369914a5d8f564ec75 +F test/stmtrand.test 340e2ea4841c5cdc02d36e33739769c5d907ab529b12bb535407def0e413ca17 F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b359468e2d5 -F test/strict1.test 4d2b492152b984fd7e8196d23eb88e2ccb0ef9e46ca2f96c2ce7147ceef9d168 +F test/strict1.test a7f9091603fe71cdc62baab0766684cba12a97ec69bfbb70be965532669cd77a F test/strict2.test b22c7a98b5000aef937f1990776497f0e979b1a23bc4f63e2d53b00e59b20070 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49 -F test/subquery.test 3a1a5b600b8d4f504d2a2c61f33db820983dba94a0ef3e4aedca8f0165eaecb8 -F test/subquery2.test 90cf944b9de8204569cf656028391e4af1ccc8c0cc02d4ef38ee3be8de1ffb12 +F test/subquery.test 23087f9b1c15ab9cc5231d04946bdebc51db527c95eb9d7434a2222127e17a84 +F test/subquery2.test ab96ff3fa9c4e3dce0d699f74e61c50250ed4335bc8f400e127707d552a8999e F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test a673e3763e247e9b5e497a6cacbaf3da2bd8ec8921c0677145c109f2e633f36b -F test/subtype1.test 7a9c55ed84d4ce551203d18046f925e293d75f69da81bff71aaf2696e4a2a748 -F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 +F test/subtype1.test 96fd2a59bfc845c955b5f339d23b37ef4d50de5f8a04acd1450a68605fa2e3e7 +F test/superlock.test 85256830339a6871ce36a2ef591c3f67716a701b5497788fb2068b90159c2442 F test/swarmvtab.test 250231404fcac88f61a6c147bb0e3a118ed879278cd3ccb0ae2d3a729e1e8e26 F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c F test/swarmvtab3.test 41a3ab47cb7a834d4e5336425103b617410a67bb95d335ef536f887587ece073 F test/swarmvtabfault.test 8a67a9f27c61073a47990829e92bc0c64420a807cb642b15a25f6c788210ed95 -F test/symlink.test 4368af0e213dd6e726a6240a16f2bb96a5a58f83f2d5d60652f27547b28cbf06 -F test/symlink2.test 9531f475a53d8781c4f81373f87faf2e2aff4f5fb2102ec6386e0c827916a670 -F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d4333092 -F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 -F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d +F test/symlink.test 60e16915cd0ee068244563f354ae012149cf7541e922025e31ac613e3fa3e389 +F test/symlink2.test 3cf7f09dcf3cf74541f5fdb1ede3c731e4e35d2018d85efc61e32ac114435ce3 +F test/sync.test a619e407ede58a7b6e3e44375328628559fc9695a9c24c47cb5690a866b0031b +F test/sync2.test 06152269ed73128782c450c355988fe8dd794d305833af75e1a5e79edd4dae47 +F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 54f27eacd054aa528a8b6e3331192c484104f30aaee351ad035f2b39a00f87c4 -F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132 -F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 +F test/tabfunc01.test 56eeae736217204bb1d9f9ef38340d48058f809b64249217cf77ff4ba600cc21 +F test/table.test e87294bf1c80bfd7792142b84ab32ea5beb4f3f71e535d7fb263a6b2068377bf +F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 -F test/tclsqlite.test ad0bbd92edabe64cc91d990a0748142fe5ab962d74ac71fa3bfa94d50d2f4c87 +F test/tclsqlite.test 3f697424cfc1cdc9c076ec0cadb0e700f059400a3e3ce134b7d856fc9f880e1c F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cbd08 F test/tempdb2.test 353864e96fd3ae2f70773d0ffbf8b1fe48589b02c2ec05013b540879410c3440 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 @@ -1586,9 +1691,11 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede -F test/testrunner.tcl 59490f189cac99b16b0376d0cc0a7ecfb753a84b89c9f4c361af337d88db53ac -F test/testrunner_data.tcl 8169c68654ac8906833b8a6aadca973358a441ebf88270dd05c153e5f96f76b8 +F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a +F test/testloadext.c 862b848783eaed9985fbce46c65cd214664376b549fae252b364d5d1ef350a27 +F test/testrunner.tcl 60d7efa1816c5dfc37df3e3454b94b9042c0c8c50b27ae296d4a797cd309ace6 x +F test/testrunner_data.tcl c507a9afa911c03446ed90442ffd4a98aca02882c3d51bd1177c24795674def8 +F test/testrunner_estwork.tcl 7927a84327259a32854926f68a75292e33a61e7e052fdbfcb01f18696c99c724 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1596,19 +1703,20 @@ F test/thread004.test f51dfc3936184aaf73ee85f315224baad272a87f F test/thread005.test 50d10b5684399676174bd96c94ad4250b1a2c8b6 F test/thread1.test df115faa10a4ba1d456e9d4d9ec165016903eae4 F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 -F test/thread3.test 5f53b6a8e7391d8653116fd0bee4f9774efee4410e039990821de39c6b4375a9 -F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd +F test/thread3.test a12656a56cdf67acb6a2ff7638826c6d6a645f79909d86df521045ad31cf547d +F test/thread_common.tcl b3b19a769fe30ef5537cdfa60acd49b78f771301627720d1add2d3bac77d9039 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8 F test/threadtest3.c 655bff6c0895ec03f014126aa65e808fac9aae8c5a7a7da58a510cbe8b43b781 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/threadtest5.c 9b4d782c58d8915d7e955ff8051f3d03628bda0d33b82971ea8c0f2f2808c421 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90 +F test/timediff1.test d982b2b5f1b22f58380c5db94ea5b17518d50ad0c55583cf0ecfa0b176e20888 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-18458b1a.test 6a62cb1ee50fa3c620da59e3a6f531eb38fceaf7e2166203816b724524e6f1d6 F test/tkt-26ff0c2d1e.test c15bec890c4d226c0da2f35ff30f9e84c169cfef90e73a8cb5cec11d723dfa96 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 -F test/tkt-2d1a5c67d.test f143872a2102c62e777be3486b38ac2744c18ece31585ed3d0afcb573ca3b4f5 +F test/tkt-2d1a5c67d.test 92bf2a2de5757d2d24ef554f8a6a38476a6735074e32dc28c775b5b9a23f96a3 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 F test/tkt-313723c356.test 4b306ad45c736cedf2f5221f6155b92143244b6d @@ -1632,11 +1740,10 @@ F test/tkt-7a31705a7e6.test 9e9c057b6a9497c8f7ba7b16871029414ccf6550e7345d9085d6 F test/tkt-7bbfb7d442.test e87b59e620700b5a52ecd92f05d56686c1cad9e1aa17456eada55e0bb821b698 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 F test/tkt-80e031a00f.test 7c93af53f43527f50020983a4bcc39b077e77c7362d7af8e04a5fd155fe5e829 -F test/tkt-8454a207b9.test aff2e76143cfa443ddce6f7d85968a2e9b57a3deb0b881b730120740555f9e2f +F test/tkt-8454a207b9.test ead80b7a01438ca1436cee029694a96c821346cf1e24f06de12f8172e139ddbb F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 -F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-99378177930f87bd.test 9d6cff39b50d062c813ae1cb0ebbd1b7acf81ecc23ae5d5215e5bb05667dc137 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 @@ -1649,11 +1756,11 @@ F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-b75a9ca6b0.test dc6a853c242f7d0326564ae32e9e5eb462b5e8d2bc5b01ea3b18fd24f8e5894b F test/tkt-ba7cbfaedc.test b4c0deccc12aeb55cfdb57935b16b5d67c5a9877 -F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 +F test/tkt-bd484a090c.test e6af3e3a4242cd8f1c91c736364f09075d8e33e3b86f6492a1ee36278ea71b61 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 F test/tkt-c694113d5.test 82c461924ada5c14866c47e85535b0b0923ba16a2e907e370061a5ca77f65d77 -F test/tkt-cbd054fa6b.test 708475ef4d730a6853512c8ce363bcbd3becf0e26826e1f4cd46e2f52ff38edf +F test/tkt-cbd054fa6b.test 6ec9f1a5721fba74a83397683c50f472df68a0a749d193a537264eda3ad6d113 F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 @@ -1714,7 +1821,7 @@ F test/tkt3357.test 77c37c6482b526fe89941ce951c22d011f5922ed F test/tkt3419.test 1bbf36d7ea03b638c15804251287c2391f5c1f6b F test/tkt3424.test 61f831bd2b071bd128fa5d00fbda57e656ca5812 F test/tkt3442.test c9d95b4c8f4f35a51b523f35d2afd0ce124937812af296545ad551ff763504fd -F test/tkt3457.test 5651e2cbb94645b677ec663160b9e192b87b7d365aecdfb24e19f749575a6fc2 +F test/tkt3457.test adf048188761581124a2f0f91f9d23a5b76fb425270ecbfd73c9c7949fa10786 F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d @@ -1752,27 +1859,27 @@ F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 4479008f85f6f8f25f7ab2cb305d665752b4727fa28a8df3d8e0ad46520c62ff F test/trace.test a659a9862957f4789e37a92b3bf6d2caf5c86b02cdeefc41e850ae53acf6992a F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983 -F test/trace3.test ae2004df24b585fed9046cc0bae4601762bc6fc4aa321d475f1350bba5047f31 +F test/trace3.test 2deeac66359c9f007f0fc9fb6336994a5d68fc1a65129f322a9e9546fd537d0a F test/trans.test 45f6f9ab6f66a7b5744f1caac06b558f95da62501916906cf55586a896f9f439 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test f8ee983600b33d167da1885657f064aec404e1c0d0bc8765fdf163f4c749237a -F test/trigger1.test 02cc64dc98278816c1c1ed8e472e18db8edbad88f37018bf46223e9614831963 -F test/trigger2.test 6e35bd7321c49e63d540aee980eb95dec63e1d1caca175224101045bcc80871f +F test/trigger1.test 141bd2bbfa82fa91aed7391f50517373c9823ff8f55af35e56313dbc75112ca1 +F test/trigger2.test 30fcb3a6aa6782020d47968735ee6086ed795f73a7affa9406c8d5a36e7b5265 F test/trigger3.test aa640bb2bbb03edd5ff69c055117ea088f121945 F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test e7ce54bfda67a88d778aea42544e151c465547a7e617127b6914c2221a6d53c1 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test fd49aff8b724ae1a6238989ecf84320d88c0e8f8d9142a891b93f826dfe37b59 +F test/trigger9.test 1724b595661da3dd3c8d79f0ebae818132a39e65c241bad2049f66952b1dc29d F test/triggerA.test 837be862d8721f903dba3f3ceff05b32e0bee5214cf6ea3da5fadf12d3650e9d F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 29f5a28d0fe39e6e2c01f6e1f53f08c0955170ae10a63ad023e33cb0a1682a51 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 612969cb57a4ef792059ad6d01af0117e1ae862c283753ffcc9a6428642b22ee F test/triggerF.test 5d76f0a8c428ff87a4d5ed52da06f6096a2c787a1e21b846111dfac4123de3ad -F test/triggerG.test 2b816093c91ba73c733cfa8aedcc210ad819d72a98b1da30768a3c56505233e9 +F test/triggerG.test b4e3fbccde6cf8995177cd6cad880256c8c00e407e07d8c67149f46106292a2c F test/triggerupfrom.test d1f9e56090408115c522bee626cc33a2f3370f627a5e341d832589d72e3aa271 F test/trustschema1.test d2996bb284859c99956ac706160eab9f086919da738d19bfef3ac431cce8fd47 F test/tt3_checkpoint.c ac7ca661d739280c89d9c253897df64a59a49369bd1247207ac0f655b622579d @@ -1783,49 +1890,51 @@ F test/tt3_stress.c f9a769ca8b026ecc76ee93ca8c9700a5619f8e51c581107c4053ba6ac97f F test/tt3_vacuum.c 71b254cde1fc49d6c8c44efd54f4668f3e57d7b3a8f4601ade069f75a999ba39 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac -F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a -F test/unhex.test 47b547f4b35e4f6525ecac7c7839bd3ae4eb4613d4e8932592eff55da83308f1 -F test/unionall.test eb9afa030897af75fd2f0dd28354ef63c8a5897b6c76aa1f15acae61a12eabcf +F test/types3.test c60e89c4d6babe44b23a2ea0090f3044e549403b20648b1c6bb65a69fea5f1ed +F test/unhex.test b7f1b806207cb77fa31c3e434fe92fba524464e3e9356809bfcc28f15af1a8b7 +F test/unionall.test 04d30726c5056f84f92b3a12bf8d8a1dbbe807d1ddc8af95def09e6ef2dd91e3 F test/unionall2.test 71e8fa08d5699d50dc9f9dc0c9799c2e7a6bb7931a330d369307a4df7f157fa1 F test/unionallfault.test 652bfbb630e6c43135965dc1e8f0a9a791da83aec885d626a632fe1909c56f73 F test/unionvtab.test e1704ab1b4c1bb3ffc9da4681f8e85a0b909fd80b937984fc94b27415ac8e5a4 F test/unionvtabfault.test e8759f3d14fb938ce9657e2342db34aeac0fb9bc1692b0d1ebb0069630151d06 F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 -F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 +F test/unixexcl.test d2366ef2d3d95249314307861d748924d9ab4f24305541159a08be61ccd4a9ee F test/unordered.test 0edaf3411d300693bca595897c5201421c6c5ec787990a1dfe2f7f60ae93f1e2 -F test/update.test 90772ede84cfc779fc3d31884a84ec4c74deb501eeb09a1c6c91c03d8e94c0d8 +F test/update.test 258dcf26d401177d3cb7fdf0beab14d671dbe72e8ff6e0435fd85a08fcd57bd9 F test/update2.test 67455bc61fcbcf96923c45b3bc4f87bc72be7d67575ad35f134906148c7b06d3 F test/upfrom1.tcl 8859d9d437f03b44174c4524a7a734a391fd4526fcff65be08285dafc9dc9041 F test/upfrom1.test 8cb06689e99cd707d884faa16da0e8eb26ff658bb01c47ddf72fadade666e6e1 F test/upfrom2.test 66f3ebf721b3cebd922faee5c386bf244f816d416b57c000753ff51af62328a1 F test/upfrom3.test 6130f24ebf97f5ea865e5d2a14a2d543fe5428a62e87cc60f62d875e45c1f5f0 -F test/upfrom4.test 1cd82e9423e02b1f63d069e8665c6c3932ec424fd0043d033cc0ba99abf33236 +F test/upfrom4.test 78f742a6577c91a7a55c64edb8811004e7c6aa99b8d57b2320f70a918c357807 F test/upfromfault.test 3a10075a0043f0c4fad6614b2c371f88a8ba5a4acab68b907438413865d6a8d6 -F test/upsert1.test b0ae2f58680c5205b4bc1cdeed3c3d444057c506f6c44494fa3eac60731d68a2 +F test/upsert1.test 77e3cbabd6b5c773056aca39bda7a690901f1dbd08b0a26ee3b5aec9e9e26198 F test/upsert2.test 720e94d09f7362a282bc69b3c6b83d51daeaaf0440eb4920a08b86518b8c7496 F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3179c F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5 -F test/upsert5.test fff0dcfce73c649204543088d8e5bde01172676063ec9b8f8fc7f195abc386fe +F test/upsert5.test 9953b180d02d1369cdbb6c73c900834e5fef8cb78e98e07511c8762ec21cc176 F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35 -F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384 +F test/uri.test 1250724af9beeed2d6c3716f5b990c483200c54f408d3c0ec9543a3c7961f8fc F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 -F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da -F test/vacuum-into.test e0e3406845be4cf1b44db354179e5d9437e38bc267e4ac8e8dc617f9c3c903ab -F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d +F test/vacuum-into.test 5a489714feecfdabfc7b293be4111564a173dee92c0d6818dd0207f3ade65783 +F test/vacuum.test f3b2257a4fcd659513c866a5d9e5f70999cc58fd5d74e979290385fa350b79ee F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520 F test/vacuum3.test d9d9a04ee58c485b94694fd4f68cffaba49c32234fdefe1ac1a622c5e17d4ce3 F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb +F test/values.test 0eda08a6ce6545f1ab012dff4cc72a7dd0fee2510f42444136bb2b2b5ed84bc0 +F test/valuesfault.test 2ef23ed965e3bd08e268cdc38a0d11653390ddbbe1e8e2e98d16f55edd30f6e8 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test d4c4281e1679245829db35597817282f60dc513fc39cc5439078f009bd118487 +F test/view.test 3c23d7a068e9e4a0c4e6907498042772adea725f0630c3d9638ffd4e5a08b92b F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 -F test/vt02.c f4a357c8180d71120ca2b466a8df48d9c40fc50873694840327d9647450485f3 +F test/vt02.c 5b44ac67b1a283fedecf2d6e2ceda61e7a157f01d44dcb4490dcb1e87d057060 +F test/vt100-a.sql 631eeab18c5adb531bab79aecf64eee3934b42c75a309ee395c814717a6a7651 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1841,61 +1950,66 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test 2a143fe75a11275781d1fd1988d86b66a3f69cb98f4add62e3da8fd0f637b45f F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test 2efb5a24b0bb50796b21eca23032cfb77abfa4b0c03938e38ce5897abac404ca +F test/vtabH.test 3fe4c1cbc93a85971990f379116255e2d4f14b583be4d0b9cca67e90226041f1 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtabJ.test a6aef49d558af90fae10565b29501f82a95781cb4f797f2d13e2d19f9b6bc77b F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3afbf +F test/vtabL.test c7b7f537978005d063fa2f53a3cd5a46ecf651ecd19970cb9ed4203698398deb F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395804dfe072 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c -F test/wal.test b7cc6984709f54afbf8441747ced1f646af120bf0c1b1d847bfa39306fbea089 -F test/wal2.test 31f6e2c404b9f2cdf9ca19b105a1742fdc19653c2c936da39e3658c617524046 +F test/wal.test 519c550255c78f55959e9159b93ebbfad2b4e9f36f5b76284da41f572f9d27da +F test/wal2.test f058016abe4627d2664db4b4b87990298d925e66a4c5a2c8e674a0ff6f4c841d F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 -F test/wal6.test b602704e4b066199bc89d91ca9000f335dcf4572 -F test/wal64k.test 2a525c0f45d709bae3765c71045ccec5df7d100ccbd3a7860fdba46c9addb965 +F test/wal6.test 6a773eff47b989c5142d17f2a7778c02d8260149a648d44ef8345aa080e428e3 +F test/wal64k.test bb8c52f0140aae1de877ffed86e2a97d903f98cf9ac263f185d51c58cde92327 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 -F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe +F test/wal_common.tcl 204d1721ac13c5e0c7fae6380315b5ab7f4e8423f580d826c5e9df1995cb018d F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 -F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d -F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278 +F test/walblock.test 6bb472e82730e7e4e81395e907a01d8cfc2bd9e1f01f8a9184ca572e2955a4bf +F test/walckptnoop.test b13a2c3140f2c913cfd422d9a224544757d04b8b14ab4c267ab9910467c0b9be +F test/walcksum.test 50e204500eed9c691b6045e467bb2923f49aa93d8adf315e2be135fdb202c1c2 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af -F test/walcrash4.test e7b6e7639a950a0cca8e210e248c8dad4d63bf20 +F test/walcrash4.test 93d8825e9d0b1b183e3a73ee67e5c8fc9d86cdaf1f3a03bcc960c3aeede28001 F test/walfault.test 09b8ad7e52d2f54bce50e31aa7ea51412bb9f70ac13c74e669ddcd8b48b0d98d F test/walfault2.test e039ac66c78d5561683cacde04097213cdad3b58e2b3f3fe1112862217bfd915 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 -F test/walmode.test cd6e7cff618eaaa5910ce57c3657aa50110397f86213886a2400afb9bfec7b7b -F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 +F test/walmode.test 2a5530972948948a17211e070263fcf25ef1ca4e06d742a32d81a470b91441dc +F test/walnoshm.test 844b3eb7d8e8ee76c834ef723babec57b0be51fa52ef7e321c289ed0fe3cddc2 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8d78a1ec91299163451417b451a2bac3481f8eb9f455b1ca507a6625c927ca6e F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878 F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 -F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 +F test/walro.test 78a84bc0fdae1385c06b017215c426b6845734d6a5a3ac75c918dd9b801b1b9d F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 -F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a +F test/walseh1.test bae700eb99519b6d5cd3f893c04759accc5a59c391d4189fe4dd6995a533442b +F test/walsetlk.test 9079cd8ef82570b8cf0067f31e049a72bec353fb2d5f0cc88f1736dc42ba9704 +F test/walsetlk2.test 4a67823b1e759ac5a4fe78a83c1f857c3c5761bf8d755421c8b55907957f23dd +F test/walsetlk3.test 1b82bd92dea7e58f498b4399b0b3d26773dd8ac5c74205ce4a23c207cb8e85fe +F test/walsetlk_recover.test adccbffc59e365063a4efd2da6b661ae2fcf15d775b6719fe46acd87face08ff +F test/walsetlk_snapshot.test 86d5588380f9927d8fcbbd75133b0a34fddf959378d6823c6f164a390123f70a F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370dbc7a3 -F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f -F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 +F test/walslow.test 0c51843836c9dcf40a5ac05aa781bfb977b396ee2c872d92bd48b79d5dd9aa23 +F test/walthread.test d562f51a61191ccfab64940df7aa1cef87c902fa5ab742590ef7f859dfe6a44b F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 -F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl 1bea58a6a8e68a73f542ee4fca28b771b84ed803bd0c9e385087070b3d747b3c x -F test/where.test 59abb854eee24f166b5f7ba9d17eb250abc59ce0a66c48912ffb10763648196d -F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 -F test/where3.test 5b4ffc0ac2ea0fe92f02b1244b7531522fe4d7bccf6fa8741d54e82c10e67753 +F test/where.test 5087c72d26fd075a1644c8512be9fe18de9bf2d2b0754f7fd9b74a1c6540c4fc +F test/where2.test 52237a8cb27ebbf6583469429bb5733d1b94ac37d09c3dbd0f487952ed0ab3f8 +F test/where3.test 4ccb156ae33de86414a52775a6f590a9d60ba2cbc7a93a24fa331b7bcf5b6030 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b -F test/where7.test 1c1bf436bf31b913d4764a2b62ac6e98b9681e5c7ae2b562605592a56b7e946b +F test/where7.test 15041c7a5838f3bac98f3fb933709674a0b59367664e88fafaf105ff7416eb07 F test/where8.test 461ca40265ed996a6305da99bb024b0e41602bb586acf544c08f95922358e49f F test/where9.test 2db942671a002621eff4f713e347bb25243295f79d8990297cd160bebcfde3f7 F test/whereA.test 9d1077b117f1b68d5f739d94f36956c36cf995eb87bb19b77b2e81af020edd20 @@ -1904,176 +2018,163 @@ F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 F test/whereD.test c1c335e914e28b122e000e9310f02d2be83e1c9dbca2e29f46bd732703944d1b F test/whereE.test 7a727b5d5b6bc8fa4cef5206e90cc0363e55ca7f0566f6fbad0206e43170f59e F test/whereF.test 926b65519608e3f2aa28720822b9154fb5c7b13519dd78194f434a511ab3dac5 -F test/whereG.test b2a479f425f7d0a432df7e842e8484560908ef703fe0fd407888ff85e7097238 +F test/whereG.test 875d020ac0a47828b31e36c54f1bf0cf81c9ea43b257bc21286eca1fe9a4880b F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a -F test/whereL.test 9d7c8a9f4e5e82d6859e61cf8758c3856c7e0a7fd8be11c92cac8c3ec39228fd +F test/whereL.test cb115604cc9bd61acbc99a1f1df0eb1ea7a7875a77fef25ba9282f01d10283e1 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 +F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb0e837a7 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31 -F test/wherelimit2.test 657a3f24aadee62d058c5091ea682dc4af4b95ffe32f137155be49799a58e721 +F test/wherelimit2.test b9e4bfe7b4d7c2f85f99cf2bd2c51369378d04b1f3d1b60557423752003bfd90 +F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b3667c1fae6 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 -F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 -F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c -F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 -F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 5ba48e9d33231e6ef16f21426bade9ccc52abf65a10587bff90a6c14fe174594 +F test/win32heap.test 1ec2ce646aee491ec23bfcdfd005b33c79f13bf91467966f374a76ffe7c7e85f +F test/win32lock.test e56d7a9b6cf9d5f3867c2dd19ff36c5326881e4038c6867610ecb3a9868ea4eb +F test/win32longpath.test 0f9837039b306735c13521c5f25b6ed42937b600dace58e28a3d2f8baf429b6a +F test/win32nolock.test 95854dc0206b8a95e4aee15a76acc082767b38f079b2e24676aed6cbb0f32798 +F test/window1.test b46d28b9698559e66aa4adafd8074b940faee498bf0c4fbdb62548bfcccc67e7 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 -F test/window3.test e9959a993c8a71e96433be8daaa1827d78b8921e4f12debd7bdbeb3c856ef3cb +F test/window3.test 330733bcca73aba4ddae7a1011f2a2120ef7a0c68d8155854e08677417b8dbd0 F test/window4.tcl 6f85307eb67242b654d051f7da32a996a66aee039a09c5ae358541aa61720742 F test/window4.test fbead87f681400ac07ef3555e0488b544a47d35491f8bf09a7474b6f76ce9b4e F test/window5.test d328dd18221217c49c144181975eea17339eaeaf0e9aa558cee3afb84652821e F test/window6.test 311de885bd7e453134fa6747680bfb4a1be87c91720bf58703db945891e7d30b F test/window7.tcl 6a1210f05d40ec89c22960213a22cd3f98d4e2f2eb20646c83c8c30d4d76108f F test/window7.test 1d31276961ae7801edc72173edaf7593e3cbc79c06d1f1f09e20d8418af403cd -F test/window8.tcl 5e02e41d9d9a80f597063aed1a381eb19d1d0ef677a4f0df352c5365cf23f79c -F test/window8.test 4ab16817414af0c904abe2ebdf88eb6c2b00058b84f9748c6174ff11fc45f1ed -F test/window9.test 349c71eab4288a1ffc19e2f65872ec2c37e6cf8a1dda2ad300364b7450ae4836 +F test/window8.tcl c57364e64d816f6e26df60437e1202e2c1031c7b818a1a67535d1006862a026a +F test/window8.test 3d931e58802b8ab8063da00f0cf30aa3351640238a952c0efb5a129e2349a4bb +F test/window9.test 7b98a7916dd87763ea35f56ea023e3b29e99744582204ccf2937a3bac411cd4d F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af23be F test/windowB.test aad7c31739999f68a98a813cfd78390918fc70f56d2d925317a1523cab548ecf F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b -F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316684c +F test/windowE.test d045a5fbaaf50ecac9483e1249dd317ba4f9d189c405a730ba6effdefb87b94f F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c -F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df -F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d -F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df -F test/with3.test fe15975c0b53c9098a757902a908e3f8d6d80ce47c5363ac600f28a79ef8c0ca +F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc +F test/with1.test 1ee171d7c306ab8b0771f3511d870f56c735607729836585bbceb1fc2f47e0b1 +F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 +F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 -F test/with6.test e097a03e5c898a8cd8f3a2d6a994ec510ea4376b5d484c2b669a41001e7758c8 +F test/with6.test 281e4861b5e517f6c3c2f08517a520c1e2ee7c11966545d3901f258a4fe8ef76 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 -F test/without_rowid1.test a5210b8770dc4736bca4e74bc96588f43025ad03ad6a80f885afd36d9890e217 +F test/without_rowid1.test f6e75e32821eb423ac3812434d12bdd8098f17e3b2206da61575e1db77f82428 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test 39ab0dd773eaa62e59b17093f875327630f54c4145458f6d2b053d68d4b2f67b F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a F test/without_rowid5.test f14298eb5ac8013894b75141c3f4f5f325a6ad0eded55516eef72c49e60a67cb F test/without_rowid6.test efbd7add62c59bf5ca97bf8da674e734e6a70ef979234e816166824b4d258f68 -F test/without_rowid7.test d7c59a93d726b55812d620f8f284e01904a5b85f9ee9eea8f2f68571a5e8c40e +F test/without_rowid7.test 747d3fbef7108d416d6416694b03d8ff046dc9cc82b3d911715feb472444b95a F test/wordcount.c d721a4b6fae93e6e33449700bce1686bc23257c27425bc3ef1599dc912adec66 -F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc +F test/writecrash.test 13520af28f376bfc8c0bcd130efc1fff20bb165198e8b94cf153f1f754154bb9 F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b1d99e F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 416adb0ca9bb54f978fe2e77978b1b964ce5e1c0199f45e381caa771e9f8cfc1 -F test/zipfile2.test 9903388a602a3834189857a985106ff95c3bba6a3969e0134127df991889db5d +F test/zipfile.test c52db63e31a66ae4245affa3e4e65e302442a87e5fd5f2ad29060bc849a83480 +F test/zipfile2.test a577e0775e32ef8972e7d5e9a45bc071a5ae061b5b965a08c9c4b709ad036a25 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x -F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 +F tool/build-all-msvc.bat 1960a7a3e5d8176c4329e31476f6e3dfa9543675355fa9020a569f4452628458 x +F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f +F tool/buildtclext.tcl d09b753d7858314104eeaf5f4def85d35784470279809e47a633f142226f2b3f F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 +F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629cca +F tool/cp.tcl 9a0d663ad45828de13763ee7ca0200f31f56c6d742cf104a56ae80e027c242d8 +F tool/custom.txt 24ed55e71c5edae0067ba159bbf09240d58b160331f7716e95816cd3aa0ba5c4 F tool/dbhash.c 5da0c61032d23d74f2ab84ffc5740f0e8abec94f2c45c0b4306be7eb3ae96df0 F tool/dbtotxt.c ca48d34eaca6d6b6e4bd6a7be2b72caf34475869054240244c60fa7e69a518d6 F tool/dbtotxt.md c9a57af8739957ef36d2cfad5c4b1443ff3688ed33e4901ee200c8b651f43f3c +F tool/emcc.sh.in 41a049468c8155433e37e656ba5bae063a000768b1d627025f277732c4e7c4a4 F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651ee21 F tool/extract-sqlite3h.tcl 069ceab0cee26cba99952bfa08c0b23e35941c837acabe143f0c355d96c9e2eb x F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c c129ae2924a48310c7b766810391da9e8fda532b9f6bd3f9a9e3a799a1b42af9 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 -F tool/fuzzershell.c e1d90a03ca790d7c331c2aae08ca46ff435f1ae1faa6cb9cc48f4687c18fdc6e -F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 +F tool/fuzzershell.c 41480c8a1e4749351f381431ecfdfceba645396c5d836f8d26b51a33c4a21b33 +F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf836eb1 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 -F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c ea5c8589c7749e9bd32ba10432aeeed3c16e215de72a12ada2bc707884837149 -F tool/lempar.c 57478ea48420da05faa873c6d1616321caa5464644588c97fbe8e0ea04450748 +F tool/lemon.c 8f6c122e5727cb0e5f302b8efc91489b1947a8d98206d7a1b1cfc0ed685b6e7c +F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 -F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 +F tool/loadfts.c 63412f9790e5e8538fbde0b4f6db154aaaf80f7a10a01e3c94d14b773a8dd5a6 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 -F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 -F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x -F tool/mkctimec.tcl 38e3db33210a200aae791635125052a643a27aa0619a0debf19aa9c55e1b2dde x -F tool/mkkeywordhash.c 9822bd1f58a70e5f84179df3045bba4791df69180e991bec88662703f2e93761 -F tool/mkmsvcmin.tcl 8897d515ef7f94772322db95a3b6fce6c614d84fe0bdd06ba5a1c786351d5a1d +F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a +F tool/mkautoconfamal.sh 647dada5e34c466bef62a4408e1c99a7e5e1922805479dd57944f33f9803f2f8 +F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x +F tool/mkctimec.tcl 3fb5cad05922f5da61262cb6bcd5868a34e94a49ca8833ae2d7796e7df075576 x +F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 +F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef -F tool/mkopcodeh.tcl 769d9e6a8b462323150dc13a8539d6064664b72974f7894befe2491cc73e05cd +F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa -F tool/mkpragmatab.tcl bd07bd59d45d0f3448e123d6937e9811195f9908a51e09d774609883055bfd3d -F tool/mkshellc.tcl b7adf08b82de60811d2cb6af05ff59fc17e5cd6f3e98743c14eaaa3f8971fed0 +F tool/mkpragmatab.tcl 3801ce32f8c55fe63a3b279f231fb26c2c1a2ea9a09d2dd599239d87a609acec +F tool/mkshellc.tcl bab0a72a68384181a5706712dfdf6815f6526446d4e8aacace2de5e80cda91b2 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 -F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f -F tool/mksqlite3c.tcl eb47021591b1ad4a6862e2cb5625f1ac67ec1e0c6db5ba3953c069c635284cf5 -F tool/mksqlite3h.tcl d391cff7cad0a372ee1406faee9ccc7dad9cb80a0c95cae0f73d10dd26e06762 -F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b -F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 +F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 +F tool/mksqlite3c.tcl 7a268139158e5deef27a370bc2f8db6ccf100c1ad7ac5e5b23743c0fd354f609 +F tool/mksqlite3h.tcl ef6831c97e6e638d2324863e8125306baea239b23defd75da77edffa3b620e81 +F tool/mksqlite3internalh.tcl 46ef6ed6ccd3c36e23051109dd25085d8edef3887635cea25afa81c4adf4d4db +F tool/mksrczip.tcl 81efd9974dbb36005383f2cd655520057a2ae5aa85ac2441a80c7c28f803ac52 +F tool/mktoolzip.tcl c9f388b2b0751982aef29a0c1d80f7959dbe38065ebb4421c39844e92622b086 +F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e540a +F tool/mkwinarm64ec.tcl 171f79234fa53552a129b360356df5599fdab15239caffb3d29c571292728033 F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 -F tool/omittest.tcl e99c9fecc3f7a8ca2fa75d8ec8bdbb5acce33dc69f0c280aae53064693387f65 +F tool/omittest.tcl bec70ef0e16255c8d9eb06ecd7edf823c07a60a836186cdbce3528fb34b67995 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a -F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b -F tool/replace.tcl 937c931ad560688e85bdd6258bdc754371bb1e2732e1fb28ef441e44c9228fce -F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a +F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafeb +F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a +F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c 495a43b759ae37a0c4561a557a70090cb79b8c1601204e5a77e8b5360e65a954 +F tool/showdb.c 3956d71e5193162609a60e8c9edfcf09274c00cfea2b1d221261427adb2b5cca F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 -F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 -F tool/showwal.c 65ecabae3a2dcff4116301d5a8dbb8c4964814da1b2aff6d85c806a88b71fa4e -F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl 1b5be34c6223cb1af06da2a10fb77863eb869b1962d055820b0a11cf2336ab45 -F tool/speed-check.sh c24c30cddd0ecb6d1d0775411d22f7619f55fa696a305487645e27b1b8fc05a2 -F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 -F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e -F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff -F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 -F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd -F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 -F tool/sqldiff.c 4f967c199c5f93eec64978e3a625d6c07fb1162212b1d48f65740d9eb4607eee -F tool/sqlite3_analyzer.c.in f88615bf33098945e0a42f17733f472083d150b58bdaaa5555a7129d0a51621c +F tool/showstat4.c b706fcbc4cd1a6e4a73ac32549afc4b460479d650402d64b23e8d813516e8de4 +F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d +F tool/soak1.tcl a3892082ed1079671565c044e93b55c3c7f38829aedf53cc597c65d23ffdaddf +F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 +F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x +F tool/split-sqlite3c.tcl 4969fd642dad0ea483e4e104163021d92baf98f6a8eac981fe48525f9b873430 +F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 +F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b +F tool/sqlite3_rsync.c d0e58a1e49fe2192c3ee0b697aed182d502bebfe5b4b406ba6b2baa52a04ecbe F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 -F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f +F tool/src-verify.c 6c655d9a8d6b30f3648fc78a79bf3838ed68f8543869d380c43ea9f17b3b8501 +F tool/srcck1.c 559e703c6cca1d70398bdba1d7f91036c1a71adf718a1aaa6401a562ccaed154 +F tool/srctree-check.tcl fa4d82dd3e8a38d5cbce7d6ade8abef2f42b9eca0394484d521dc8d086739460 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 -F tool/stripccomments.c 20b8aabc4694d0d4af5566e42da1f1a03aff057689370326e9269a9ddcffdc37 +F tool/stripccomments.c 68d2aa8cb504439f541ce66b8f128067612bdd16f5fb7bfe540f3fcb67c9c197 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh 1612bd947750e21e7b47befad5f6b3825b06cce0705441f903bf35ced65ae9b9 +F tool/tclConfigShToMake.sh 7c065d81c2d178e15e45a77372c6e5a38b5a1b08755301cd6f20a3a862db7312 x F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 -F tool/vdbe-compress.tcl 1dcb7632e57cf57105248029e6e162fddaf6c0fccb3bb9e6215603752c5a2d4a +F tool/vdbe-compress.tcl fa2f37ab39b2a0087fafb6a7f3ce19503e25e624ffa8ed9951717ab72920c088 F tool/vdbe_profile.tcl 3ac5a4a9449f4baf77059358ea050db3e34395ccf59c5464d29b91746d5b961e +F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6ddf2700c F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 -F tool/warnings.sh ab651bb82586c43ff8b560beceac959735bf917b44c5e0f67ba3426e474f29f8 +F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -F vsixtest/App.xaml b76d3b48860e7454775c47ea38ffea9c4abe3e85 -F vsixtest/App.xaml.cpp 41158ee43269820136fa3bba00c0bd91b26cc38b650ee392aec2a8d823e54318 -F vsixtest/App.xaml.h 4a9768e2983d05600ad1e1c2f1b00a132967da9f -F vsixtest/Assets/LockScreenLogo.scale-200.png e820c9a3deb909197081b0bf3216c06e13905f0a -F vsixtest/Assets/SplashScreen.scale-200.png cab70988ca71bebec7bfeb3b6dbafe17b9ab0b4a -F vsixtest/Assets/Square150x150Logo.scale-200.png e17b40817db7a239fc239d83efcc951fb824e3ff -F vsixtest/Assets/Square44x44Logo.scale-200.png 2f166237094dea94d952d10b9eeae81806844f1c -F vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png 5f6a6d391b95a3061ccca6e6fdd6955ede63b4ed -F vsixtest/Assets/StoreLogo.png 0828b7257db74a4ecd5eeb6b7b4971f0fdc4d9d1 -F vsixtest/Assets/Wide310x150Logo.scale-200.png 04ddefe5bc5f43ae12a7433f6f236ddab101ac42 -F vsixtest/MainPage.xaml 34f49897e3ca533a7e74506ba0759b66eebce151 -F vsixtest/MainPage.xaml.cpp 7f31fc6de751b64676c0924c97a5485d950a91d7 -F vsixtest/MainPage.xaml.h cc05cca10d50a003f6c6e4448b701cdd07f52f29 -F vsixtest/Package.appxmanifest 6b6db1eb7df3a315c5d681059754d5f0e0c47a93 -F vsixtest/pch.cpp cb823cfac36f1a39a7eb0acbd7e9a0b0de8f23af -F vsixtest/pch.h 9cab7980f2ac4baa40807d8b5e52af32a21cf78c -F vsixtest/vsixtest.sln 77cadbe4e96c1fe1bf51cd77de9e9b0a12ada547 -F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 -F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc -F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e -F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4254f086419892634c80b0b915e14edea365adc6a45bcc75eecba889da274ac3 -R 5363a07f3e1462ec2f4570a33edb7167 +P 10e11b9c539a8be50fd93bdf7cf5afe97d9757ce8577cac58426a1b218063e47 +R ec4161915fdee7c9c786931b31f64669 T +sym-release * -T +sym-version-3.42.0 * +T +sym-version-3.51.2 * U drh -Z 0ee8e0816fa6ac728bed7565aed4ec35 +Z 6e3a819551f499011e2275a586e8d608 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags new file mode 100644 index 0000000000..c644e06f90 --- /dev/null +++ b/manifest.tags @@ -0,0 +1,4 @@ +branch branch-3.51 +tag release +tag branch-3.51 +tag version-3.51.2 diff --git a/manifest.uuid b/manifest.uuid index f9f720edbb..75ccd2e605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0 +b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075 diff --git a/mkso.sh b/mkso.sh deleted file mode 100644 index 6f2e8e25ed..0000000000 --- a/mkso.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# This script is used to compile SQLite into a shared library on Linux. -# -# Two separate shared libraries are generated. "sqlite3.so" is the core -# library. "tclsqlite3.so" contains the TCL bindings and is the -# library that is loaded into TCL in order to run SQLite. -# -make target_source -cd tsrc -rm shell.c -TCLDIR=/home/drh/tcltk/846/linux/846linux -TCLSTUBLIB=$TCLDIR/libtclstub8.4g.a -OPTS='-DUSE_TCL_STUBS=1 -DNDEBUG=1 -DHAVE_DLOPEN=1' -OPTS="$OPTS -DSQLITE_THREADSAFE=1" -OPTS="$OPTS -DSQLITE_ENABLE_FTS3=1" -OPTS="$OPTS -DSQLITE_ENABLE_COLUMN_METADATA=1" -for i in *.c; do - if test $i != 'keywordhash.c'; then - CMD="cc -fPIC $OPTS -O2 -I. -I$TCLDIR -c $i" - echo $CMD - $CMD - fi -done -echo gcc -shared *.o $TCLSTUBLIB -o tclsqlite3.so -gcc -shared *.o $TCLSTUBLIB -o tclsqlite3.so -strip tclsqlite3.so -rm tclsqlite.c tclsqlite.o -echo gcc -shared *.o -o sqlite3.so -gcc -shared *.o -o sqlite3.so -strip sqlite3.so -cd .. diff --git a/mptest/mptest.c b/mptest/mptest.c index 5022b009e6..25a1bcc821 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -1037,6 +1037,10 @@ static void runScript( for(k=(int)strlen(zFilename)-1; k>=0 && !isDirSep(zFilename[k]); k--){} if( k>0 ){ zNewFile = zToDel = sqlite3_mprintf("%.*s/%s", k,zFilename,zNewFile); + if( zNewFile==0 ){ + fprintf(stderr, "out of memory\n"); + abort(); + } } } zNewScript = readFile(zNewFile); diff --git a/spec.template b/spec.template deleted file mode 100644 index 6cc7ab2eab..0000000000 --- a/spec.template +++ /dev/null @@ -1,67 +0,0 @@ -%define name sqlite -%define version SQLITE_VERSION -%define release 1 - -Name: %{name} -Summary: SQLite is a C library that implements an embeddable SQL database engine -Version: %{version} -Release: %{release} -Source: %{name}-%{version}.tar.gz -Group: System/Libraries -URL: http://www.sqlite.org/ -License: Public Domain -BuildRoot: %{_tmppath}/%{name}-%{version}-root - -%description -SQLite is a software library that implements a self-contained, serverless, -zero-configuration, transactional SQL database engine. -Programs that link with the SQLite library can have SQL database access -without running a separate RDBMS process. The distribution comes with a -standalone command-line access program (sqlite) that can be used to -administer an SQLite database and which serves as an example of how to -use the SQLite library. - -%package -n %{name}-devel -Summary: Header files and libraries for developing apps which will use sqlite -Group: Development/C -Requires: %{name} = %{version}-%{release} - -%description -n %{name}-devel -The sqlite-devel package contains the header files and libraries needed -to develop programs that use the SQLite database library. - -%prep -%setup -q -n %{name} - -%build -CFLAGS="%optflags -DNDEBUG=1" CXXFLAGS="%optflags -DNDEBUG=1" ./configure --prefix=%{_prefix} - -make -make doc - -%install -install -d $RPM_BUILD_ROOT/%{_prefix} -install -d $RPM_BUILD_ROOT/%{_prefix}/bin -install -d $RPM_BUILD_ROOT/%{_prefix}/include -install -d $RPM_BUILD_ROOT/%{_prefix}/lib -make install prefix=$RPM_BUILD_ROOT/%{_prefix} - -%post -p /sbin/ldconfig - -%postun -p /sbin/ldconfig - -%clean -rm -fr $RPM_BUILD_ROOT - -%files -%defattr(-, root, root) -%{_libdir}/*.so* -%{_bindir}/* - -%files -n %{name}-devel -%defattr(-, root, root) -%{_libdir}/pkgconfig/sqlite3.pc -%{_libdir}/*.a -%{_libdir}/*.la -%{_includedir}/* -%doc doc/* diff --git a/sqlcipher-resources/PrivacyInfo.xcprivacy b/sqlcipher-resources/PrivacyInfo.xcprivacy new file mode 100644 index 0000000000..60b27a68ca --- /dev/null +++ b/sqlcipher-resources/PrivacyInfo.xcprivacy @@ -0,0 +1,32 @@ + + + + + NSPrivacyCollectedDataTypes + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryDiskSpace + NSPrivacyAccessedAPITypeReasons + + E174.1 + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + + C617.1 + 3B52.1 + + + + NSPrivacyTrackingDomains + + NSPrivacyTracking + + + diff --git a/sqlcipher-1.1.8-testkey.db b/sqlcipher-resources/sqlcipher-1.1.8-testkey.db similarity index 100% rename from sqlcipher-1.1.8-testkey.db rename to sqlcipher-resources/sqlcipher-1.1.8-testkey.db diff --git a/sqlcipher-2.0-be-testkey.db b/sqlcipher-resources/sqlcipher-2.0-be-testkey.db similarity index 100% rename from sqlcipher-2.0-be-testkey.db rename to sqlcipher-resources/sqlcipher-2.0-be-testkey.db diff --git a/sqlcipher-2.0-beta-testkey.db b/sqlcipher-resources/sqlcipher-2.0-beta-testkey.db similarity index 100% rename from sqlcipher-2.0-beta-testkey.db rename to sqlcipher-resources/sqlcipher-2.0-beta-testkey.db diff --git a/sqlcipher-2.0-le-testkey.db b/sqlcipher-resources/sqlcipher-2.0-le-testkey.db similarity index 100% rename from sqlcipher-2.0-le-testkey.db rename to sqlcipher-resources/sqlcipher-2.0-le-testkey.db diff --git a/sqlcipher-3.0-testkey.db b/sqlcipher-resources/sqlcipher-3.0-testkey.db similarity index 100% rename from sqlcipher-3.0-testkey.db rename to sqlcipher-resources/sqlcipher-3.0-testkey.db diff --git a/sqlcipher-4.0-testkey.db b/sqlcipher-resources/sqlcipher-4.0-testkey.db similarity index 100% rename from sqlcipher-4.0-testkey.db rename to sqlcipher-resources/sqlcipher-4.0-testkey.db diff --git a/sqlcipher.1 b/sqlcipher.1 index 6a08486033..c9a310057d 100644 --- a/sqlcipher.1 +++ b/sqlcipher.1 @@ -2,7 +2,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH SQLCIPHER 1 "Fri Oct 31 10:41:31 EDT 2014" +.TH SQLCIPHER 1 "Fri Aug 11 23:50:12 CET 2023" .\" Please adjust this date whenever revising the manpage. .\" .\" Some roff macros, for reference: @@ -17,7 +17,7 @@ .\" for manpage-specific macros, see man(7) .SH NAME .B sqlcipher -\- A command line interface for SQLCipher version 2 +\- A command line interface for SQLCipher version 3 .SH SYNOPSIS .B sqlcipher @@ -49,9 +49,9 @@ a table named "memos" and insert a couple of records into that table: $ .B sqlcipher mydata.db .br -SQLite version 3.8.8 +SQLCipher version 3.43.0 2023-08-11 17:45:23 .br -Enter ".help" for instructions +Enter ".help" for usage hints. .br sqlite> .B create table memos(text, priority INTEGER); @@ -108,141 +108,13 @@ sqlite> .B .help .nf .tr %. -%backup ?DB? FILE Backup DB (default "main") to FILE -%bail on|off Stop after hitting an error. Default OFF -%clone NEWDB Clone data into NEWDB from the existing database -%databases List names and files of attached databases -%dump ?TABLE? ... Dump the database in an SQL text format - If TABLE specified, only dump tables matching - LIKE pattern TABLE. -%echo on|off Turn command echo on or off -%eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN -%exit Exit this program -%explain ?on|off? Turn output mode suitable for EXPLAIN on or off. - With no args, it turns EXPLAIN on. -%fullschema Show schema and the content of sqlite_stat tables -%headers on|off Turn display of headers on or off -%help Show this message -%import FILE TABLE Import data from FILE into TABLE -%indices ?TABLE? Show names of all indices - If TABLE specified, only show indices for tables - matching LIKE pattern TABLE. -%load FILE ?ENTRY? Load an extension library -%log FILE|off Turn logging on or off. FILE can be stderr/stdout -%mode MODE ?TABLE? Set output mode where MODE is one of: - csv Comma-separated values - column Left-aligned columns. (See .width) - html HTML code - insert SQL insert statements for TABLE - line One value per line - list Values delimited by .separator string - tabs Tab-separated values - tcl TCL list elements -%nullvalue STRING Use STRING in place of NULL values -%once FILENAME Output for the next SQL command only to FILENAME -%open ?FILENAME? Close existing database and reopen FILENAME -%output ?FILENAME? Send output to FILENAME or stdout -%print STRING... Print literal STRING -%prompt MAIN CONTINUE Replace the standard prompts -%quit Exit this program -%read FILENAME Execute SQL in FILENAME -%restore ?DB? FILE Restore content of DB (default "main") from FILE -%save FILE Write in-memory database into FILE -%schema ?TABLE? Show the CREATE statements - If TABLE specified, only show tables matching - LIKE pattern TABLE. -%separator STRING ?NL? Change separator used by output mode and .import - NL is the end-of-line mark for CSV -%shell CMD ARGS... Run CMD ARGS... in a system shell -%show Show the current values for various settings -%stats on|off Turn stats on or off -%system CMD ARGS... Run CMD ARGS... in a system shell -%tables ?TABLE? List names of tables - If TABLE specified, only list tables matching - LIKE pattern TABLE. -%timeout MS Try opening locked tables for MS milliseconds -%timer on|off Turn SQL timer on or off -%trace FILE|off Output each SQL statement as it is run -%vfsname ?AUX? Print the name of the VFS stack -%width NUM1 NUM2 ... Set column widths for "column" mode - Negative values right-justify -sqlite> +... .sp .fi -.SH OPTIONS -.B sqlcipher -has the following options: -.TP -.B \-bail -Stop after hitting an error. -.TP -.B \-batch -Force batch I/O. -.TP -.B \-column -Query results will be displayed in a table like form, using -whitespace characters to separate the columns and align the -output. -.TP -.BI \-cmd\ command -run -.I command -before reading stdin -.TP -.B \-csv -Set output mode to CSV (comma separated values). -.TP -.B \-echo -Print commands before execution. -.TP -.BI \-init\ file -Read and execute commands from -.I file -, which can contain a mix of SQL statements and meta-commands. -.TP -.B \-[no]header -Turn headers on or off. -.TP -.B \-help -Show help on options and exit. -.TP -.B \-html -Query results will be output as simple HTML tables. -.TP -.B \-interactive -Force interactive I/O. -.TP -.B \-line -Query results will be displayed with one value per line, rows -separated by a blank line. Designed to be easily parsed by -scripts or other programs -.TP -.B \-list -Query results will be displayed with the separator (|, by default) -character between each field value. The default. -.TP -.BI \-mmap\ N -Set default mmap size to -.I N -\. -.TP -.BI \-nullvalue\ string -Set string used to represent NULL values. Default is '' -(empty string). -.TP -.BI \-separator\ separator -Set output field separator. Default is '|'. -.TP -.B \-stats -Print memory stats before each finalize. -.TP -.B \-version -Show SQLite version. -.TP -.BI \-vfs\ name -Use -.I name -as the default VFS. + +The available commands differ by version and build options, so they +are not listed here. Please refer to your local copy for all available +options. .SH INIT FILE @@ -265,22 +137,24 @@ continue prompt = " ...> " .sp .fi -o If the file +o If the file +.B ${XDG_CONFIG_HOME}/sqlcipher/sqliterc +or .B ~/.sqliterc -exists, it is processed first. -can be found in the user's home directory, it is -read and processed. It should generally only contain meta-commands. +exists, the first of those to be found is processed during startup. +It should generally only contain meta-commands. o If the -init option is present, the specified file is processed. o All other command line options are processed. .SH SEE ALSO -https://www.zetetic.net/sqlcipher +https://zetetic.net/sqlcipher + .br -The sqlite3-doc package. +The sqlcipher-doc package. .SH AUTHOR This manual page was originally written by Andreas Rottmann , for the Debian GNU/Linux system (but may be used -by others). It was subsequently revised by Bill Bumgarner and -further updated by Laszlo Boszormenyi . +by others). It was subsequently revised by Bill Bumgarner , +Laszlo Boszormenyi , and the sqlcipher developers. diff --git a/sqlite3.1 b/sqlite3.1 new file mode 100644 index 0000000000..e4ba5a0cb8 --- /dev/null +++ b/sqlite3.1 @@ -0,0 +1,181 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH SQLITE3 1 "Fri Aug 11 23:50:12 CET 2023" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) +.SH NAME +.B sqlite3 +\- A command line interface for SQLite version 3 + +.SH SYNOPSIS +.B sqlite3 +.RI [ options ] +.RI [ databasefile ] +.RI [ SQL ] + +.SH SUMMARY +.PP +.B sqlite3 +is a terminal-based front-end to the SQLite library that can evaluate +queries interactively and display the results in multiple formats. +.B sqlite3 +can also be used within shell scripts and other applications to provide +batch processing features. + +.SH DESCRIPTION +To start a +.B sqlite3 +interactive session, invoke the +.B sqlite3 +command and optionally provide the name of a database file. If the +database file does not exist, it will be created. If the database file +does exist, it will be opened. + +For example, to create a new database file named "mydata.db", create +a table named "memos" and insert a couple of records into that table: +.sp +$ +.B sqlite3 mydata.db +.br +SQLite version 3.43.0 2023-08-11 17:45:23 +.br +Enter ".help" for usage hints. +.br +sqlite> +.B create table memos(text, priority INTEGER); +.br +sqlite> +.B insert into memos values('deliver project description', 10); +.br +sqlite> +.B insert into memos values('lunch with Christine', 100); +.br +sqlite> +.B select * from memos; +.br +deliver project description|10 +.br +lunch with Christine|100 +.br +sqlite> +.sp + +If no database name is supplied, the ATTACH sql command can be used +to attach to existing or create new database files. ATTACH can also +be used to attach to multiple databases within the same interactive +session. This is useful for migrating data between databases, +possibly changing the schema along the way. + +Optionally, a SQL statement or set of SQL statements can be supplied as +a single argument. Multiple statements should be separated by +semi-colons. + +For example: +.sp +$ +.B sqlite3 -line mydata.db 'select * from memos where priority > 20;' +.br + text = lunch with Christine +.br +priority = 100 +.br +.sp + +.SS SQLITE META-COMMANDS +.PP +The interactive interpreter offers a set of meta-commands that can be +used to control the output format, examine the currently attached +database files, or perform administrative operations upon the +attached databases (such as rebuilding indices). Meta-commands are +always prefixed with a dot (.). + +A list of available meta-commands can be viewed at any time by issuing +the '.help' command. For example: +.sp +sqlite> +.B .help +.nf +.tr %. +... +.sp +.fi + +The available commands differ by version and build options, so they +are not listed here. Please refer to your local copy for all available +options. + + +.SH INIT FILE +.B sqlite3 +reads an initialization file to set the configuration of the +interactive environment. Throughout initialization, any previously +specified setting can be overridden. The sequence of initialization is +as follows: + +o The default configuration is established as follows: + +.sp +.nf +.cc | +mode = LIST +separator = "|" +main prompt = "sqlite> " +continue prompt = " ...> " +|cc . +.sp +.fi + +o If the environment variable XDG_CONFIG_HOME is set then +.B ${XDG_CONFIG_HOME}/sqlite3/sqliterc +is checked, else +.B ~/.config/sqlite3/sqliterc +is checked. If the selected file does not exist then the fallback of +.B ~/.sqliterc +is used. It should generally only contain meta-commands. + +o If the -init option is present, the specified file is processed. + +o All other command line options are processed. + +.SH HISTORY FILE +.B sqlite3 +may be configured to use a history file to save SQL statements and +meta-commands entered interactively. These statements and commands can be +retrieved, edited and, reused at the main and continue prompts. If the +environment variable +.B SQLITE_HISTORY +is set, it will be used as the name of the history file, whether it +already exists or not. If it is not set but the XDG_STATE_HOME +environment variable is then +.B ${XDG_STATE_HOME}/sqlite_history +is used. If XDG_STATE_HOME is not set then +.B ~/.local/state/sqlite_history +is used. If the selected file does not exist then +.B ~/.sqlite_history +will be used as the history file. If any history file is found, it +will be written if the shell exits interactive mode normally, +regardless of whether it existed previously, though saving will +silently fail if the history file's directory does not exist. +.SH SEE ALSO +https://sqlite.org/cli.html +.br +https://sqlite.org/fiddle (a WebAssembly build of the CLI app) +.br +The sqlite3-doc package. +.SH AUTHOR +This manual page was originally written by Andreas Rottmann +, for the Debian GNU/Linux system (but may be used +by others). It was subsequently revised by Bill Bumgarner , +Laszlo Boszormenyi , and the sqlite3 developers. diff --git a/sqlcipher.pc.in b/sqlite3.pc.in similarity index 53% rename from sqlcipher.pc.in rename to sqlite3.pc.in index a8789bd39e..723dd51563 100644 --- a/sqlcipher.pc.in +++ b/sqlite3.pc.in @@ -5,9 +5,9 @@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ -Name: SQLCipher +Name: SQLite Description: SQL database engine Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lsqlcipher -Libs.private: @LIBS@ -Cflags: -I${includedir}/sqlcipher +Libs: -L${libdir} -lsqlite3 +Libs.private: @LDFLAGS_MATH@ @LDFLAGS_ZLIB@ @LDFLAGS_DLOPEN@ @LDFLAGS_PTHREAD@ @LDFLAGS_ICU@ +Cflags: -I${includedir} diff --git a/sqlite_cfg.h.in b/sqlite_cfg.h.in deleted file mode 100644 index 5e68659664..0000000000 --- a/sqlite_cfg.h.in +++ /dev/null @@ -1,150 +0,0 @@ -/* sqlite_cfg.h.in. Generated from configure.ac by autoheader. */ - -/* Define to 1 if you have the header file. */ -#undef HAVE_DLFCN_H - -/* Define to 1 if you have the `fdatasync' function. */ -#undef HAVE_FDATASYNC - -/* Define to 1 if you have the `gmtime_r' function. */ -#undef HAVE_GMTIME_R - -/* Define to 1 if the system has the type `int16_t'. */ -#undef HAVE_INT16_T - -/* Define to 1 if the system has the type `int32_t'. */ -#undef HAVE_INT32_T - -/* Define to 1 if the system has the type `int64_t'. */ -#undef HAVE_INT64_T - -/* Define to 1 if the system has the type `int8_t'. */ -#undef HAVE_INT8_T - -/* Define to 1 if the system has the type `intptr_t'. */ -#undef HAVE_INTPTR_T - -/* Define to 1 if you have the header file. */ -#undef HAVE_INTTYPES_H - -/* Define to 1 if you have the `isnan' function. */ -#undef HAVE_ISNAN - -/* Define to 1 if you have the `crypto' library (-lcrypto). */ -#undef HAVE_LIBCRYPTO - -/* Define to 1 if you have the `nss3' library (-lnss3). */ -#undef HAVE_LIBNSS3 - -/* Define to 1 if you have the `tomcrypt' library (-ltomcrypt). */ -#undef HAVE_LIBTOMCRYPT - -/* Define to 1 if you have the `localtime_r' function. */ -#undef HAVE_LOCALTIME_R - -/* Define to 1 if you have the `localtime_s' function. */ -#undef HAVE_LOCALTIME_S - -/* Define to 1 if you have the header file. */ -#undef HAVE_MALLOC_H - -/* Define to 1 if you have the `malloc_usable_size' function. */ -#undef HAVE_MALLOC_USABLE_SIZE - -/* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H - -/* Define to 1 if you have the `pread' function. */ -#undef HAVE_PREAD - -/* Define to 1 if you have the `pread64' function. */ -#undef HAVE_PREAD64 - -/* Define to 1 if you have the `pwrite' function. */ -#undef HAVE_PWRITE - -/* Define to 1 if you have the `pwrite64' function. */ -#undef HAVE_PWRITE64 - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDINT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDLIB_H - -/* Define to 1 if you have the `strchrnul' function. */ -#undef HAVE_STRCHRNUL - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRING_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STAT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TYPES_H - -/* Define to 1 if the system has the type `uint16_t'. */ -#undef HAVE_UINT16_T - -/* Define to 1 if the system has the type `uint32_t'. */ -#undef HAVE_UINT32_T - -/* Define to 1 if the system has the type `uint64_t'. */ -#undef HAVE_UINT64_T - -/* Define to 1 if the system has the type `uint8_t'. */ -#undef HAVE_UINT8_T - -/* Define to 1 if the system has the type `uintptr_t'. */ -#undef HAVE_UINTPTR_T - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* Define to 1 if you have the `usleep' function. */ -#undef HAVE_USLEEP - -/* Define to 1 if you have the `utime' function. */ -#undef HAVE_UTIME - -/* Define to 1 if you have the header file. */ -#undef HAVE_ZLIB_H - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#undef LT_OBJDIR - -/* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT - -/* Define to the full name of this package. */ -#undef PACKAGE_NAME - -/* Define to the full name and version of this package. */ -#undef PACKAGE_STRING - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the home page for this package. */ -#undef PACKAGE_URL - -/* Define to the version of this package. */ -#undef PACKAGE_VERSION - -/* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS - -/* Enable large inode numbers on Mac OS X 10.5. */ -#ifndef _DARWIN_USE_64_BIT_INODE -# define _DARWIN_USE_64_BIT_INODE 1 -#endif - -/* Number of bits in a file offset, on hosts where this is settable. */ -#undef _FILE_OFFSET_BITS - -/* Define for large files, on AIX-style hosts. */ -#undef _LARGE_FILES diff --git a/src/alter.c b/src/alter.c index 121b617117..a7255e75ef 100644 --- a/src/alter.c +++ b/src/alter.c @@ -58,7 +58,7 @@ static void renameTestSchema( int bNoDQS /* Do not allow DQS in the schema */ ){ pParse->colNamesSet = 1; - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "SELECT 1 " "FROM \"%w\"." LEGACY_SCHEMA_TABLE " " "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" @@ -69,7 +69,7 @@ static void renameTestSchema( ); if( bTemp==0 ){ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "SELECT 1 " "FROM temp." LEGACY_SCHEMA_TABLE " " "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" @@ -82,20 +82,20 @@ static void renameTestSchema( /* ** Generate VM code to replace any double-quoted strings (but not double-quoted -** identifiers) within the "sql" column of the sqlite_schema table in +** identifiers) within the "sql" column of the sqlite_schema table in ** database zDb with their single-quoted equivalents. If argument bTemp is ** not true, similarly update all SQL statements in the sqlite_schema table ** of the temp db. */ static void renameFixQuotes(Parse *pParse, const char *zDb, int bTemp){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET sql = sqlite_rename_quotefix(%Q, sql)" "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" " AND sql NOT LIKE 'create virtual%%'" , zDb, zDb ); if( bTemp==0 ){ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE temp." LEGACY_SCHEMA_TABLE " SET sql = sqlite_rename_quotefix('temp', sql)" "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" @@ -118,8 +118,8 @@ static void renameReloadSchema(Parse *pParse, int iDb, u16 p5){ } /* -** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" -** command. +** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" +** command. */ void sqlite3AlterRenameTable( Parse *pParse, /* Parser context. */ @@ -129,7 +129,7 @@ void sqlite3AlterRenameTable( int iDb; /* Database that contains the table */ char *zDb; /* Name of database iDb */ Table *pTab; /* Table being renamed */ - char *zName = 0; /* NULL-terminated version of pName */ + char *zName = 0; /* NULL-terminated version of pName */ sqlite3 *db = pParse->db; /* Database connection */ int nTabName; /* Number of UTF-8 characters in zTabName */ const char *zTabName; /* Original name of the table */ @@ -156,7 +156,7 @@ void sqlite3AlterRenameTable( || sqlite3FindIndex(db, zName, zDb) || sqlite3IsShadowTableOf(db, pTab, zName) ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "there is already another table or index with this name: %s", zName); goto exit_rename_table; } @@ -199,7 +199,7 @@ void sqlite3AlterRenameTable( /* Begin a transaction for database iDb. Then modify the schema cookie ** (since the ALTER TABLE modifies the schema). Call sqlite3MayAbort(), - ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the + ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the ** nested SQL may raise an exception. */ v = sqlite3GetVdbe(pParse); if( v==0 ){ @@ -213,7 +213,7 @@ void sqlite3AlterRenameTable( /* Rewrite all CREATE TABLE, INDEX, TRIGGER or VIEW statements in ** the schema to use the new table name. */ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" @@ -233,14 +233,14 @@ void sqlite3AlterRenameTable( "'sqlite_autoindex_' || %Q || substr(name,%d+18) " "ELSE name END " "WHERE tbl_name=%Q COLLATE nocase AND " - "(type='table' OR type='index' OR type='trigger');", + "(type='table' OR type='index' OR type='trigger');", zDb, - zName, zName, zName, + zName, zName, zName, nTabName, zTabName ); #ifndef SQLITE_OMIT_AUTOINCREMENT - /* If the sqlite_sequence table exists in this database, then update + /* If the sqlite_sequence table exists in this database, then update ** it with the new table name. */ if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ @@ -251,10 +251,10 @@ void sqlite3AlterRenameTable( #endif /* If the table being renamed is not itself part of the temp database, - ** edit view and trigger definitions within the temp database + ** edit view and trigger definitions within the temp database ** as required. */ if( iDb!=1 ){ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE sqlite_temp_schema SET " "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " "tbl_name = " @@ -361,7 +361,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ return; } if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ - /* If the default value for the new column was specified with a + /* If the default value for the new column was specified with a ** literal NULL, then set pDflt to 0. This simplifies checking ** for an SQL NULL default below. */ @@ -414,11 +414,11 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** have to use printf() to translate between these units: */ assert( IsOrdinaryTable(pTab) ); assert( IsOrdinaryTable(pNew) ); - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " "sql = printf('%%.%ds, ',sql) || %Q" " || substr(sql,1+length(printf('%%.%ds',sql))) " - "WHERE type = 'table' AND name = %Q", + "WHERE type = 'table' AND name = %Q", zDb, pNew->u.tab.addColOffset, zCol, pNew->u.tab.addColOffset, zTab ); @@ -446,14 +446,19 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ /* Verify that constraints are still satisfied */ if( pNew->pCheck!=0 || (pCol->notNull && (pCol->colFlags & COLFLAG_GENERATED)!=0) + || (pTab->tabFlags & TF_Strict)!=0 ){ sqlite3NestedParse(pParse, "SELECT CASE WHEN quick_check GLOB 'CHECK*'" " THEN raise(ABORT,'CHECK constraint failed')" + " WHEN quick_check GLOB 'non-* value in*'" + " THEN raise(ABORT,'type mismatch on DEFAULT')" " ELSE raise(ABORT,'NOT NULL constraint failed')" " END" " FROM pragma_quick_check(%Q,%Q)" - " WHERE quick_check GLOB 'CHECK*' OR quick_check GLOB 'NULL*'", + " WHERE quick_check GLOB 'CHECK*'" + " OR quick_check GLOB 'NULL*'" + " OR quick_check GLOB 'non-* value in*'", zTab, zDb ); } @@ -462,14 +467,14 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ /* ** This function is called by the parser after the table-name in -** an "ALTER TABLE ADD" statement is parsed. Argument +** an "ALTER TABLE ADD" statement is parsed. Argument ** pSrc is the full-name of the table being altered. ** ** This routine makes a (partial) copy of the Table structure ** for the table being altered and sets Parse.pNewTable to point ** to it. Routines called by the parser as the column definition -** is parsed (i.e. sqlite3AddColumn()) add the new Column data to -** the copy. The copy of the Table structure is deleted by tokenize.c +** is parsed (i.e. sqlite3AddColumn()) add the new Column data to +** the copy. The copy of the Table structure is deleted by tokenize.c ** after parsing is finished. ** ** Routine sqlite3AlterFinishAddColumn() will be called to complete @@ -526,13 +531,13 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); - pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*(u32)nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ assert( db->mallocFailed ); goto exit_begin_add_column; } - memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*(size_t)pNew->nCol); for(i=0; inCol; i++){ Column *pCol = &pNew->aCol[i]; pCol->zCnName = sqlite3DbStrDup(db, pCol->zCnName); @@ -542,7 +547,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->u.tab.pDfltList = sqlite3ExprListDup(db, pTab->u.tab.pDfltList, 0); pNew->pSchema = db->aDb[iDb].pSchema; pNew->u.tab.addColOffset = pTab->u.tab.addColOffset; - pNew->nTabRef = 1; + assert( pNew->nTabRef==1 ); exit_begin_add_column: sqlite3SrcListDelete(db, pSrc); @@ -571,7 +576,7 @@ static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ } #endif if( zType ){ - sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", + sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", (bDrop ? "drop column from" : "rename columns of"), zType, pTab->zName ); @@ -611,7 +616,7 @@ void sqlite3AlterRenameColumn( if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_rename_column; if( SQLITE_OK!=isRealTable(pParse, pTab, 0) ) goto exit_rename_column; - /* Which schema holds the table to be altered */ + /* Which schema holds the table to be altered */ iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iSchema>=0 ); zDb = db->aDb[iSchema].zDbSName; @@ -627,10 +632,8 @@ void sqlite3AlterRenameColumn( ** altered. Set iCol to be the index of the column being renamed */ zOld = sqlite3NameFromToken(db, pOld); if( !zOld ) goto exit_rename_column; - for(iCol=0; iColnCol; iCol++){ - if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break; - } - if( iCol==pTab->nCol ){ + iCol = sqlite3ColumnIndex(pTab, zOld); + if( iCol<0 ){ sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pOld); goto exit_rename_column; } @@ -648,7 +651,7 @@ void sqlite3AlterRenameColumn( if( !zNew ) goto exit_rename_column; assert( pNew->n>0 ); bQuote = sqlite3Isquote(pNew->z[0]); - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X' " @@ -658,7 +661,7 @@ void sqlite3AlterRenameColumn( pTab->zName ); - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE temp." LEGACY_SCHEMA_TABLE " SET " "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " "WHERE type IN ('trigger', 'view')", @@ -708,7 +711,7 @@ struct RenameCtx { RenameToken *pList; /* List of tokens to overwrite */ int nList; /* Number of tokens in pList */ int iCol; /* Index of column being renamed */ - Table *pTab; /* Table being ALTERed */ + Table *pTab; /* Table being ALTERed */ const char *zOld; /* Old column name */ }; @@ -716,14 +719,14 @@ struct RenameCtx { /* ** This function is only for debugging. It performs two tasks: ** -** 1. Checks that pointer pPtr does not already appear in the +** 1. Checks that pointer pPtr does not already appear in the ** rename-token list. ** ** 2. Dereferences each pointer in the rename-token list. ** ** The second is most effective when debugging under valgrind or -** address-sanitizer or similar. If any of these pointers no longer -** point to valid objects, an exception is raised by the memory-checking +** address-sanitizer or similar. If any of these pointers no longer +** point to valid objects, an exception is raised by the memory-checking ** tool. ** ** The point of this is to prevent comparisons of invalid pointer values. @@ -918,7 +921,7 @@ void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ } /* -** Remove all nodes that are part of expression-list pEList from the +** Remove all nodes that are part of expression-list pEList from the ** rename list. */ void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ @@ -959,8 +962,8 @@ static void renameTokenFree(sqlite3 *db, RenameToken *pToken){ ** the list maintained by the RenameCtx object. */ static RenameToken *renameTokenFind( - Parse *pParse, - struct RenameCtx *pCtx, + Parse *pParse, + struct RenameCtx *pCtx, const void *pPtr ){ RenameToken **pp; @@ -1008,12 +1011,12 @@ static int renameColumnSelectCb(Walker *pWalker, Select *p){ */ static int renameColumnExprCb(Walker *pWalker, Expr *pExpr){ RenameCtx *p = pWalker->u.pRename; - if( pExpr->op==TK_TRIGGER - && pExpr->iColumn==p->iCol + if( pExpr->op==TK_TRIGGER + && pExpr->iColumn==p->iCol && pWalker->pParse->pTriggerTab==p->pTab ){ renameTokenFind(pWalker->pParse, p, (void*)pExpr); - }else if( pExpr->op==TK_COLUMN + }else if( pExpr->op==TK_COLUMN && pExpr->iColumn==p->iCol && ALWAYS(ExprUseYTab(pExpr)) && p->pTab==pExpr->y.pTab @@ -1047,14 +1050,14 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ } /* -** An error occured while parsing or otherwise processing a database +** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an ** ALTER TABLE RENAME COLUMN program. The error message emitted by the ** sub-routine is currently stored in pParse->zErrMsg. This function ** adds context to the error message and then stores it in pCtx. */ static void renameColumnParseError( - sqlite3_context *pCtx, + sqlite3_context *pCtx, const char *zWhen, sqlite3_value *pType, sqlite3_value *pObject, @@ -1064,7 +1067,7 @@ static void renameColumnParseError( const char *zN = (const char*)sqlite3_value_text(pObject); char *zErr; - zErr = sqlite3MPrintf(pParse->db, "error in %s %s%s%s: %s", + zErr = sqlite3MPrintf(pParse->db, "error in %s %s%s%s: %s", zT, zN, (zWhen[0] ? " " : ""), zWhen, pParse->zErrMsg ); @@ -1074,14 +1077,14 @@ static void renameColumnParseError( /* ** For each name in the the expression-list pEList (i.e. each -** pEList->a[i].zName) that matches the string in zOld, extract the +** pEList->a[i].zName) that matches the string in zOld, extract the ** corresponding rename-token from Parse object pParse and add it ** to the RenameCtx pCtx. */ static void renameColumnElistNames( - Parse *pParse, - RenameCtx *pCtx, - const ExprList *pEList, + Parse *pParse, + RenameCtx *pCtx, + const ExprList *pEList, const char *zOld ){ if( pEList ){ @@ -1099,14 +1102,14 @@ static void renameColumnElistNames( } /* -** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) -** that matches the string in zOld, extract the corresponding rename-token +** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) +** that matches the string in zOld, extract the corresponding rename-token ** from Parse object pParse and add it to the RenameCtx pCtx. */ static void renameColumnIdlistNames( - Parse *pParse, - RenameCtx *pCtx, - const IdList *pIdList, + Parse *pParse, + RenameCtx *pCtx, + const IdList *pIdList, const char *zOld ){ if( pIdList ){ @@ -1133,6 +1136,7 @@ static int renameParseSql( int bTemp /* True if SQL is from temp schema */ ){ int rc; + u64 flags; sqlite3ParseObjectInit(p, db); if( zSql==0 ){ @@ -1141,13 +1145,23 @@ static int renameParseSql( if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){ return SQLITE_CORRUPT_BKPT; } - db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + if( bTemp ){ + db->init.iDb = 1; + }else{ + int iDb = sqlite3FindDbName(db, zDb); + assert( iDb>=0 && iDb<=0xff ); + db->init.iDb = (u8)iDb; + } p->eParseMode = PARSE_MODE_RENAME; p->db = db; p->nQueryLoop = 1; + flags = db->flags; + testcase( (db->flags & SQLITE_Comments)==0 && strstr(zSql," /* ")!=0 ); + db->flags |= SQLITE_Comments; rc = sqlite3RunParser(p, zSql); + db->flags = flags; if( db->mallocFailed ) rc = SQLITE_NOMEM; - if( rc==SQLITE_OK + if( rc==SQLITE_OK && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0) ){ rc = SQLITE_CORRUPT_BKPT; @@ -1196,8 +1210,8 @@ static int renameEditSql( char *zBuf2 = 0; if( zNew ){ - /* Set zQuot to point to a buffer containing a quoted copy of the - ** identifier zNew. If the corresponding identifier in the original + /* Set zQuot to point to a buffer containing a quoted copy of the + ** identifier zNew. If the corresponding identifier in the original ** ALTER TABLE statement was quoted (bQuote==1), then set zNew to ** point to zQuot so that all substitutions are made using the ** quoted version of the new column name. */ @@ -1208,10 +1222,11 @@ static int renameEditSql( nQuot = sqlite3Strlen30(zQuot)-1; } - assert( nQuot>=nNew ); - zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + assert( nQuot>=nNew && nSql>=0 && nNew>=0 ); + zOut = sqlite3DbMallocZero(db, (u64)nSql + pRename->nList*(u64)nQuot + 1); }else{ - zOut = (char*)sqlite3DbMallocZero(db, (nSql*2+1) * 3); + assert( nSql>0 ); + zOut = (char*)sqlite3DbMallocZero(db, (2*(u64)nSql + 1) * 3); if( zOut ){ zBuf1 = &zOut[nSql*2+1]; zBuf2 = &zOut[nSql*4+2]; @@ -1220,19 +1235,20 @@ static int renameEditSql( /* At this point pRename->pList contains a list of RenameToken objects ** corresponding to all tokens in the input SQL that must be replaced - ** with the new column name, or with single-quoted versions of themselves. + ** with the new column name, or with single-quoted versions of themselves. ** All that remains is to construct and return the edited SQL string. */ if( zOut ){ - int nOut = nSql; - memcpy(zOut, zSql, nSql); + i64 nOut = nSql; + assert( nSql>0 ); + memcpy(zOut, zSql, (size_t)nSql); while( pRename->pList ){ int iOff; /* Offset of token to replace in zOut */ - u32 nReplace; + i64 nReplace; const char *zReplace; RenameToken *pBest = renameColumnTokenNext(pRename); if( zNew ){ - if( bQuote==0 && sqlite3IsIdChar(*pBest->t.z) ){ + if( bQuote==0 && sqlite3IsIdChar(*(u8*)pBest->t.z) ){ nReplace = nNew; zReplace = zNew; }else{ @@ -1250,16 +1266,17 @@ static int renameEditSql( memcpy(zBuf1, pBest->t.z, pBest->t.n); zBuf1[pBest->t.n] = 0; sqlite3Dequote(zBuf1); - sqlite3_snprintf(nSql*2, zBuf2, "%Q%s", zBuf1, + assert( nSql < 0x15555554 /* otherwise malloc would have failed */ ); + sqlite3_snprintf((int)(nSql*2), zBuf2, "%Q%s", zBuf1, pBest->t.z[pBest->t.n]=='\'' ? " " : "" ); zReplace = zBuf2; nReplace = sqlite3Strlen30(zReplace); } - iOff = pBest->t.z - zSql; + iOff = (int)(pBest->t.z - zSql); if( pBest->t.n!=nReplace ){ - memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], + memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], nOut - (iOff + pBest->t.n) ); nOut += nReplace - pBest->t.n; @@ -1283,18 +1300,19 @@ static int renameEditSql( ** Set all pEList->a[].fg.eEName fields in the expression-list to val. */ static void renameSetENames(ExprList *pEList, int val){ + assert( val==ENAME_NAME || val==ENAME_TAB || val==ENAME_SPAN ); if( pEList ){ int i; for(i=0; inExpr; i++){ assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); - pEList->a[i].fg.eEName = val; + pEList->a[i].fg.eEName = val&0x3; } } } /* ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming -** it was read from the schema of database zDb. Return SQLITE_OK if +** it was read from the schema of database zDb. Return SQLITE_OK if ** successful. Otherwise, return an SQLite error code and leave an error ** message in the Parse object. */ @@ -1308,14 +1326,14 @@ static int renameResolveTrigger(Parse *pParse){ memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; assert( pNew->pTabSchema ); - pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, + pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, db->aDb[sqlite3SchemaToIndex(db, pNew->pTabSchema)].zDbSName ); pParse->eTriggerOp = pNew->op; /* ALWAYS() because if the table of the trigger does not exist, the ** error would have been hit before this point */ if( ALWAYS(pParse->pTriggerTab) ){ - rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; } /* Resolve symbols in WHEN clause */ @@ -1361,8 +1379,9 @@ static int renameResolveTrigger(Parse *pParse){ int i; for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ SrcItem *p = &pStep->pFrom->a[i]; - if( p->pSelect ){ - sqlite3SelectPrep(pParse, p->pSelect, 0); + if( p->fg.isSubquery ){ + assert( p->u4.pSubq!=0 ); + sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); } } } @@ -1430,8 +1449,12 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ } if( pStep->pFrom ){ int i; - for(i=0; ipFrom->nSrc; i++){ - sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect); + SrcList *pFrom = pStep->pFrom; + for(i=0; inSrc; i++){ + if( pFrom->a[i].fg.isSubquery ){ + assert( pFrom->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + } } } } @@ -1539,7 +1562,7 @@ static void renameColumnFunc( if( sParse.pNewTable ){ if( IsView(sParse.pNewTable) ){ Select *pSelect = sParse.pNewTable->u.view.pSelect; - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sParse.rc = SQLITE_OK; sqlite3SelectPrep(&sParse, pSelect, 0); rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); @@ -1601,7 +1624,7 @@ static void renameColumnFunc( if( rc!=SQLITE_OK ) goto renameColumnFunc_done; for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget ){ + if( pStep->zTarget ){ Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); if( pTarget==pTab ){ if( pStep->pUpsert ){ @@ -1647,7 +1670,7 @@ static void renameColumnFunc( } /* -** Walker expression callback used by "RENAME TABLE". +** Walker expression callback used by "RENAME TABLE". */ static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ RenameCtx *p = pWalker->u.pRename; @@ -1661,7 +1684,7 @@ static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ } /* -** Walker select callback used by "RENAME TABLE". +** Walker select callback used by "RENAME TABLE". */ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ int i; @@ -1678,7 +1701,7 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ } for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->pTab==p->pTab ){ + if( pItem->pSTab==p->pTab ){ renameTokenFind(pWalker->pParse, p, pItem->zName); } } @@ -1691,7 +1714,7 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ /* ** This C function implements an SQL user function that is used by SQL code ** generated by the ALTER TABLE ... RENAME command to modify the definition -** of any foreign key constraints that use the table being renamed as the +** of any foreign key constraints that use the table being renamed as the ** parent table. It is passed three arguments: ** ** 0: The database containing the table being renamed. @@ -1757,7 +1780,7 @@ static void renameTableFunc( sNC.pParse = &sParse; assert( pSelect->selFlags & SF_View ); - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sqlite3SelectPrep(&sParse, pTab->u.view.pSelect, &sNC); if( sParse.nErr ){ rc = sParse.rc; @@ -1805,7 +1828,7 @@ static void renameTableFunc( else{ Trigger *pTrigger = sParse.pNewTrigger; TriggerStep *pStep; - if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) + if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) && sCtx.pTab->pSchema==pTrigger->pTabSchema ){ renameTokenFind(&sParse, &sCtx, sParse.pNewTrigger->table); @@ -1881,12 +1904,12 @@ static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){ ** ** CREATE TABLE t1(a, b, c); ** -** SELECT sqlite_rename_quotefix('main', +** SELECT sqlite_rename_quotefix('main', ** 'CREATE VIEW v1 AS SELECT "a", "string" FROM t1' ** ); ** ** returns the string: -** +** ** CREATE VIEW v1 AS SELECT "a", 'string' FROM t1 ** ** If there is a error in the input SQL, then raise an error, except @@ -1930,7 +1953,7 @@ static void renameQuotefixFunc( if( sParse.pNewTable ){ if( IsView(sParse.pNewTable) ){ Select *pSelect = sParse.pNewTable->u.view.pSelect; - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sParse.rc = SQLITE_OK; sqlite3SelectPrep(&sParse, pSelect, 0); rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); @@ -1942,8 +1965,8 @@ static void renameQuotefixFunc( sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); #ifndef SQLITE_OMIT_GENERATED_COLUMNS for(i=0; inCol; i++){ - sqlite3WalkExpr(&sWalker, - sqlite3ColumnExpr(sParse.pNewTable, + sqlite3WalkExpr(&sWalker, + sqlite3ColumnExpr(sParse.pNewTable, &sParse.pNewTable->aCol[i])); } #endif /* SQLITE_OMIT_GENERATED_COLUMNS */ @@ -1960,7 +1983,7 @@ static void renameQuotefixFunc( #endif /* SQLITE_OMIT_TRIGGER */ } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ rc = renameEditSql(context, &sCtx, zInput, 0, 0); } renameTokenFree(db, sCtx.pList); @@ -2029,10 +2052,10 @@ static void renameTableTest( if( zDb && zInput ){ int rc; Parse sParse; - int flags = db->flags; + u64 flags = db->flags; if( bNoDQS ) db->flags &= ~(SQLITE_DqsDML|SQLITE_DqsDDL); rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); - db->flags |= (flags & (SQLITE_DqsDML|SQLITE_DqsDDL)); + db->flags = flags; if( rc==SQLITE_OK ){ if( isLegacy==0 && sParse.pNewTable && IsView(sParse.pNewTable) ){ NameContext sNC; @@ -2071,7 +2094,7 @@ static void renameTableTest( /* ** The implementation of internal UDF sqlite_drop_column(). -** +** ** Arguments: ** ** argv[0]: An integer - the index of the schema containing the table @@ -2107,7 +2130,7 @@ static void dropColumnFunc( rc = renameParseSql(&sParse, zDb, db, zSql, iSchema==1); if( rc!=SQLITE_OK ) goto drop_column_done; pTab = sParse.pNewTable; - if( pTab==0 || pTab->nCol==1 || iCol>=pTab->nCol ){ + if( pTab==0 || pTab->nCol==1 || iCol>=pTab->nCol ){ /* This can happen if the sqlite_schema table is corrupt */ rc = SQLITE_CORRUPT_BKPT; goto drop_column_done; @@ -2139,7 +2162,7 @@ static void dropColumnFunc( } /* -** This function is called by the parser upon parsing an +** This function is called by the parser upon parsing an ** ** ALTER TABLE pSrc DROP COLUMN pName ** @@ -2161,7 +2184,7 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_drop_column; - /* Make sure this is not an attempt to ALTER a view, virtual table or + /* Make sure this is not an attempt to ALTER a view, virtual table or ** system table. */ if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_drop_column; if( SQLITE_OK!=isRealTable(pParse, pTab, 1) ) goto exit_drop_column; @@ -2178,10 +2201,10 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ goto exit_drop_column; } - /* Do not allow the user to drop a PRIMARY KEY column or a column + /* Do not allow the user to drop a PRIMARY KEY column or a column ** constrained by a UNIQUE constraint. */ if( pTab->aCol[iCol].colFlags & (COLFLAG_PRIMKEY|COLFLAG_UNIQUE) ){ - sqlite3ErrorMsg(pParse, "cannot drop %s column: \"%s\"", + sqlite3ErrorMsg(pParse, "cannot drop %s column: \"%s\"", (pTab->aCol[iCol].colFlags&COLFLAG_PRIMKEY) ? "PRIMARY KEY" : "UNIQUE", zCol ); @@ -2206,7 +2229,7 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ #endif renameTestSchema(pParse, zDb, iDb==1, "", 0); renameFixQuotes(pParse, zDb, iDb==1); - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " "sql = sqlite_drop_column(%d, sql, %d) " "WHERE (type=='table' AND tbl_name=%Q COLLATE nocase)" @@ -2257,7 +2280,12 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } diff --git a/src/analyze.c b/src/analyze.c index 0823bcaefc..2721f25234 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -215,7 +215,8 @@ static void openStatTable( sqlite3NestedParse(pParse, "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); - aRoot[i] = (u32)pParse->regRoot; + assert( pParse->isCreate || pParse->nErr ); + aRoot[i] = (u32)pParse->u1.cr.regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; } }else{ @@ -264,9 +265,9 @@ static void openStatTable( typedef struct StatAccum StatAccum; typedef struct StatSample StatSample; struct StatSample { - tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anDLt; /* sqlite_stat4.nDLt */ #ifdef SQLITE_ENABLE_STAT4 + tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anLt; /* sqlite_stat4.nLt */ union { i64 iRowid; /* Rowid in main table of the key */ @@ -406,7 +407,7 @@ static void statInit( int nCol; /* Number of columns in index being sampled */ int nKeyCol; /* Number of key columns */ int nColUp; /* nCol rounded up for alignment */ - int n; /* Bytes of space to allocate */ + i64 n; /* Bytes of space to allocate */ sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ #ifdef SQLITE_ENABLE_STAT4 /* Maximum number of samples. 0 if STAT4 data is not collected */ @@ -424,9 +425,9 @@ static void statInit( /* Allocate the space required for the StatAccum object */ n = sizeof(*p) - + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ - + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ #ifdef SQLITE_ENABLE_STAT4 + n += sizeof(tRowcnt)*nColUp; /* StatAccum.anEq */ if( mxSample ){ n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ @@ -442,14 +443,14 @@ static void statInit( p->db = db; p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; - p->nLimit = sqlite3_value_int64(argv[3]); + p->nLimit = sqlite3_value_int(argv[3]); p->nCol = nCol; p->nKeyCol = nKeyCol; p->nSkipAhead = 0; p->current.anDLt = (tRowcnt*)&p[1]; - p->current.anEq = &p->current.anDLt[nColUp]; #ifdef SQLITE_ENABLE_STAT4 + p->current.anEq = &p->current.anDLt[nColUp]; p->mxSample = p->nLimit==0 ? mxSample : 0; if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ @@ -716,7 +717,9 @@ static void statPush( if( p->nRow==0 ){ /* This is the first call to this function. Do initialization. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; inCol; i++) p->current.anEq[i] = 1; +#endif }else{ /* Second and subsequent calls get processed here */ #ifdef SQLITE_ENABLE_STAT4 @@ -725,15 +728,17 @@ static void statPush( /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply ** to the current row of the index. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; icurrent.anEq[i]++; } +#endif for(i=iChng; inCol; i++){ p->current.anDLt[i]++; #ifdef SQLITE_ENABLE_STAT4 if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; -#endif p->current.anEq[i] = 1; +#endif } } @@ -867,7 +872,9 @@ static void statGet( u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; sqlite3_str_appendf(&sStat, " %llu", iVal); - assert( p->current.anEq[i] ); +#ifdef SQLITE_ENABLE_STAT4 + assert( p->current.anEq[i] || p->nRow==0 ); +#endif } sqlite3ResultStrAccum(context, &sStat); } @@ -1051,7 +1058,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -1075,9 +1082,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -1116,41 +1128,36 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: - ** + /* Implementation of the following: + ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) estimated number of rows in the index, - */ + ** (3) estimated number of rows in the index. */ sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); assert( regRowid==regStat+2 ); sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); -#ifdef SQLITE_ENABLE_STAT4 - if( OptimizationEnabled(db, SQLITE_Stat4) ){ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - }else -#endif - { - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); - } - assert( regTemp2==regStat+4 ); - sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); @@ -1257,6 +1264,12 @@ static void analyzeOneTable( } /* Add the entry to the stat1 table. */ + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); @@ -1280,6 +1293,13 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + if( doOnce ){ int mxCol = nCol; Index *pX; @@ -1332,7 +1352,7 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* End of analysis */ - sqlite3VdbeJumpHere(v, addrRewind); + if( addrGotoEnd ) sqlite3VdbeJumpHere(v, addrGotoEnd); } @@ -1768,12 +1788,13 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int nSample; /* Number of samples */ - int nByte; /* Bytes of space required */ - int i; /* Bytes of space required */ - tRowcnt *pSpace; + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + i64 nByte; /* Bytes of space required */ + i64 i; /* Bytes of space required */ + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -1793,7 +1814,7 @@ static int loadStatTbl( } pIdx->nSampleCol = nIdxCol; pIdx->mxSample = nSample; - nByte = sizeof(IndexSample) * nSample; + nByte = ROUND8(sizeof(IndexSample) * nSample); nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -1802,7 +1823,10 @@ static int loadStatTbl( sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; } - pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pPtr = (u8*)pIdx->aSample; + pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); pIdx->aAvgEq = pSpace; pSpace += nIdxCol; pIdx->pTable->tabFlags |= TF_HasStat4; for(i=0; ianLt,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); - /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + /* Take a copy of the sample. Add 8 extra 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the ** sqlite3VdbeRecordCompare() may read up to two varints past the ** end of the allocated buffer before it realizes it is dealing with - ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a corrupt record. Or it might try to read a large integer from the + ** buffer. In any case, eight 0x00 bytes prevents this from causing ** a buffer overread. */ pSample->n = sqlite3_column_bytes(pStmt, 4); - pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 8); if( pSample->p==0 ){ sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; diff --git a/src/attach.c b/src/attach.c index 81db50d9ff..b442e54512 100644 --- a/src/attach.c +++ b/src/attach.c @@ -156,7 +156,7 @@ static void attachFunc( if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ - aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(1+(i64)db->nDb)); if( aNew==0 ) return; } db->aDb = aNew; @@ -175,6 +175,12 @@ static void attachFunc( sqlite3_free(zErr); return; } + if( (db->flags & SQLITE_AttachWrite)==0 ){ + flags &= ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE); + flags |= SQLITE_OPEN_READONLY; + }else if( (db->flags & SQLITE_AttachCreate)==0 ){ + flags &= ~SQLITE_OPEN_CREATE; + } assert( pVfs ); flags |= SQLITE_OPEN_MAIN_DB; rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags); @@ -216,6 +222,7 @@ static void attachFunc( if( rc==SQLITE_OK ){ extern int sqlcipherCodecAttach(sqlite3*, int, const void*, int); extern void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); + extern void sqlcipher_free(void*, sqlite3_uint64); int nKey; char *zKey; int t = sqlite3_value_type(argv[2]); @@ -230,7 +237,14 @@ static void attachFunc( case SQLITE_BLOB: nKey = sqlite3_value_bytes(argv[2]); zKey = (char *)sqlite3_value_blob(argv[2]); - rc = sqlcipherCodecAttach(db, db->nDb-1, zKey, nKey); + /* SQLCipher allows a special case to attach a plaintext database + * to an encrypted database by passing key as an empty string, eg. + * ATTACH DATABASE 'plain.db' AS plain KEY ''; + * In this case, do not attempt to attach a codec to the attached + * database */ + if(nKey && zKey) { + rc = sqlcipherCodecAttach(db, db->nDb-1, zKey, nKey); + } break; case SQLITE_NULL: @@ -241,6 +255,7 @@ static void attachFunc( if( nKey || sqlite3BtreeGetRequestedReserve(db->aDb[0].pBt)>0 ){ rc = sqlcipherCodecAttach(db, db->nDb-1, zKey, nKey); } + if(nKey) sqlcipher_free(zKey, nKey); } break; } @@ -258,21 +273,19 @@ static void attachFunc( sqlite3BtreeEnterAll(db); db->init.iDb = 0; db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( db->setlkFlags & SQLITE_SETLK_BLOCK_ON_CONNECT ){ + int val = 1; + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(pNew->pBt)); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_BLOCK_ON_CONNECT, &val); + } +#endif if( !REOPEN_AS_MEMDB(db) ){ rc = sqlite3Init(db, &zErrDyn); } sqlite3BtreeLeaveAll(db); assert( zErrDyn==0 || rc!=SQLITE_OK ); } -#ifdef SQLITE_USER_AUTHENTICATION - if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ - u8 newAuth = 0; - rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); - if( newAuthauth.authLevel ){ - rc = SQLITE_AUTH_USER; - } - } -#endif if( rc ){ if( ALWAYS(!REOPEN_AS_MEMDB(db)) ){ int iDb = db->nDb - 1; @@ -516,20 +529,21 @@ static int fixSelectCb(Walker *p, Select *pSelect){ if( NEVER(pList==0) ) return WRC_Continue; for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pFix->bTemp==0 ){ - if( pItem->zDatabase ){ - if( iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + if( pFix->bTemp==0 && pItem->fg.isSubquery==0 ){ + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( iDb!=sqlite3FindDbName(db, pItem->u4.zDatabase) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); + pFix->zType, pFix->pName, pItem->u4.zDatabase); return WRC_Abort; } - sqlite3DbFree(db, pItem->zDatabase); - pItem->zDatabase = 0; + sqlite3DbFree(db, pItem->u4.zDatabase); pItem->fg.notCte = 1; + pItem->fg.hadSchema = 1; } - pItem->pSchema = pFix->pSchema; + pItem->u4.pSchema = pFix->pSchema; pItem->fg.fromDDL = 1; + pItem->fg.fixedSchema = 1; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( pList->a[i].fg.isUsing==0 diff --git a/src/auth.c b/src/auth.c index 3a4f73a234..9ec2e7d046 100644 --- a/src/auth.c +++ b/src/auth.c @@ -112,11 +112,7 @@ int sqlite3AuthReadCol( int rc; /* Auth callback return code */ if( db->init.busy ) return SQLITE_OK; - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext -#ifdef SQLITE_USER_AUTHENTICATION - ,db->auth.zAuthUser -#endif - ); + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); if( rc==SQLITE_DENY ){ char *z = sqlite3_mprintf("%s.%s", zTab, zCol); if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z); @@ -165,7 +161,7 @@ void sqlite3AuthRead( assert( pTabList ); for(iSrc=0; iSrcnSrc; iSrc++){ if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ - pTab = pTabList->a[iSrc].pTab; + pTab = pTabList->a[iSrc].pSTab; break; } } @@ -204,7 +200,7 @@ int sqlite3AuthCheck( sqlite3 *db = pParse->db; int rc; - /* Don't do any authorization checks if the database is initialising + /* Don't do any authorization checks if the database is initializing ** or if the parser is being invoked from within sqlite3_declare_vtab. */ assert( !IN_RENAME_OBJECT || db->xAuth==0 ); @@ -223,11 +219,7 @@ int sqlite3AuthCheck( testcase( zArg3==0 ); testcase( pParse->zAuthContext==0 ); - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext -#ifdef SQLITE_USER_AUTHENTICATION - ,db->auth.zAuthUser -#endif - ); + rc = db->xAuth(db->pAuthArg,code,zArg1,zArg2,zArg3,pParse->zAuthContext); if( rc==SQLITE_DENY ){ sqlite3ErrorMsg(pParse, "not authorized"); pParse->rc = SQLITE_AUTH; diff --git a/src/backup.c b/src/backup.c index b73ce76bda..cf3cb445e8 100644 --- a/src/backup.c +++ b/src/backup.c @@ -157,12 +157,14 @@ sqlite3_backup *sqlite3_backup_init( { extern int sqlcipher_find_db_index(sqlite3*, const char*); extern void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); + extern void sqlcipher_free(void*, sqlite3_uint64); int srcNKey, destNKey; void *zKey; sqlcipherCodecGetKey(pSrcDb, sqlcipher_find_db_index(pSrcDb, zSrcDb), &zKey, &srcNKey); + if(srcNKey) sqlcipher_free(zKey, srcNKey); sqlcipherCodecGetKey(pDestDb, sqlcipher_find_db_index(pDestDb, zDestDb), &zKey, &destNKey); - zKey = NULL; + if(destNKey) sqlcipher_free(zKey, destNKey); /* either both databases must be plaintext, or both must be encrypted */ if((srcNKey == 0 && destNKey > 0) || (srcNKey > 0 && destNKey == 0)) { diff --git a/src/bitvec.c b/src/bitvec.c index 9393428d9d..7c5fa71d9b 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -17,8 +17,8 @@ ** property. Usually only a few pages are meet either condition. ** So the bitmap is usually sparse and has low cardinality. ** But sometimes (for example when during a DROP of a large table) most -** or all of the pages in a database can get journalled. In those cases, -** the bitmap becomes dense with high cardinality. The algorithm needs +** or all of the pages in a database can get journalled. In those cases, +** the bitmap becomes dense with high cardinality. The algorithm needs ** to handle both cases well. ** ** The size of the bitmap is fixed when the object is created. @@ -39,13 +39,13 @@ /* Size of the Bitvec structure in bytes. */ #define BITVEC_SZ 512 -/* Round the union size down to the nearest pointer boundary, since that's how +/* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ #define BITVEC_USIZE \ (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) -/* Type of the array "element" for the bitmap representation. -** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. +/* Type of the array "element" for the bitmap representation. +** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. ** Setting this to the "natural word" size of your CPU may improve ** performance. */ #define BITVEC_TELEM u8 @@ -58,16 +58,16 @@ /* Number of u32 values in hash table. */ #define BITVEC_NINT (BITVEC_USIZE/sizeof(u32)) -/* Maximum number of entries in hash table before +/* Maximum number of entries in hash table before ** sub-dividing and re-hashing. */ #define BITVEC_MXHASH (BITVEC_NINT/2) /* Hashing function for the aHash representation. -** Empirical testing showed that the *37 multiplier -** (an arbitrary prime)in the hash function provided +** Empirical testing showed that the *37 multiplier +** (an arbitrary prime)in the hash function provided ** no fewer collisions than the no-op *1. */ #define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) -#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) +#define BITVEC_NPTR ((u32)(BITVEC_USIZE/sizeof(Bitvec *))) /* @@ -107,9 +107,10 @@ struct Bitvec { } u; }; + /* ** Create a new bitmap object able to handle bits between 0 and iSize, -** inclusive. Return a pointer to the new object. Return NULL if +** inclusive. Return a pointer to the new object. Return NULL if ** malloc fails. */ Bitvec *sqlite3BitvecCreate(u32 iSize){ @@ -188,7 +189,7 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ h = BITVEC_HASH(i++); /* if there wasn't a hash collision, and this doesn't */ /* completely fill the hash, then just add it without */ - /* worring about sub-dividing and re-hashing. */ + /* worrying about sub-dividing and re-hashing. */ if( !p->u.aHash[h] ){ if (p->nSet<(BITVEC_NINT-1)) { goto bitvec_set_end; @@ -216,7 +217,9 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ }else{ memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); - p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + p->iDivisor = p->iSize/BITVEC_NPTR; + if( (p->iSize%BITVEC_NPTR)!=0 ) p->iDivisor++; + if( p->iDivisoriDivisor = BITVEC_NBIT; rc = sqlite3BitvecSet(p, i); for(j=0; jiSize<=BITVEC_NBIT ){ - p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(BITVEC_TELEM)(1<<(i&(BITVEC_SZELEM-1))); }else{ unsigned int j; u32 *aiValues = pBuf; @@ -293,6 +296,52 @@ u32 sqlite3BitvecSize(Bitvec *p){ return p->iSize; } +#ifdef SQLITE_DEBUG +/* +** Show the content of a Bitvec option and its children. Indent +** everything by n spaces. Add x to each bitvec value. +** +** From a debugger such as gdb, one can type: +** +** call sqlite3ShowBitvec(p) +** +** For some Bitvec p and see a recursive view of the Bitvec's content. +*/ +static void showBitvec(Bitvec *p, int n, unsigned x){ + int i; + if( p==0 ){ + printf("NULL\n"); + return; + } + printf("Bitvec 0x%p iSize=%u", p, p->iSize); + if( p->iSize<=BITVEC_NBIT ){ + printf(" bitmap\n"); + printf("%*s bits:", n, ""); + for(i=1; i<=BITVEC_NBIT; i++){ + if( sqlite3BitvecTest(p,i) ) printf(" %u", x+(unsigned)i); + } + printf("\n"); + }else if( p->iDivisor==0 ){ + printf(" hash with %u entries\n", p->nSet); + printf("%*s bits:", n, ""); + for(i=0; iu.aHash[i] ) printf(" %u", x+(unsigned)p->u.aHash[i]); + } + printf("\n"); + }else{ + printf(" sub-bitvec with iDivisor=%u\n", p->iDivisor); + for(i=0; iu.apSub[i]==0 ) continue; + printf("%*s apSub[%d]=", n, "", i); + showBitvec(p->u.apSub[i], n+4, i*p->iDivisor); + } + } +} +void sqlite3ShowBitvec(Bitvec *p){ + showBitvec(p, 0, 0); +} +#endif + #ifndef SQLITE_UNTESTABLE /* ** Let V[] be an array of unsigned characters sufficient to hold @@ -301,9 +350,10 @@ u32 sqlite3BitvecSize(Bitvec *p){ ** individual bits within V. */ #define SETBIT(V,I) V[I>>3] |= (1<<(I&7)) -#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(BITVEC_TELEM)(1<<(I&7)) #define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + /* ** This routine runs an extensive test of the Bitvec code. ** @@ -312,7 +362,7 @@ u32 sqlite3BitvecSize(Bitvec *p){ ** by 0, 1, or 3 operands, depending on the opcode. Another ** opcode follows immediately after the last operand. ** -** There are 6 opcodes numbered from 0 through 5. 0 is the +** There are opcodes numbered starting with 0. 0 is the ** "halt" opcode and causes the test to end. ** ** 0 Halt and return the number of errors @@ -321,18 +371,25 @@ u32 sqlite3BitvecSize(Bitvec *p){ ** 3 N Set N randomly chosen bits ** 4 N Clear N randomly chosen bits ** 5 N S X Set N bits from S increment X in array only, not in bitvec +** 6 Invoice sqlite3ShowBitvec() on the Bitvec object so far +** 7 X Show compile-time parameters and the hash of X ** ** The opcodes 1 through 4 perform set and clear operations are performed ** on both a Bitvec object and on a linear array of bits obtained from malloc. ** Opcode 5 works on the linear array only, not on the Bitvec. ** Opcode 5 is used to deliberately induce a fault in order to -** confirm that error detection works. +** confirm that error detection works. Opcodes 6 and greater are +** state output opcodes. Opcodes 6 and greater are no-ops unless +** SQLite has been compiled with SQLITE_DEBUG. ** ** At the conclusion of the test the linear array is compared ** against the Bitvec object. If there are any differences, ** an error is returned. If they are the same, zero is returned. ** ** If a memory allocation error occurs, return -1. +** +** sz is the size of the Bitvec. Or if sz is negative, make the size +** 2*(unsigned)(-sz) and disabled the linear vector check. */ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ Bitvec *pBitvec = 0; @@ -343,10 +400,15 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Allocate the Bitvec to be tested and a linear array of ** bits to act as the reference */ - pBitvec = sqlite3BitvecCreate( sz ); - pV = sqlite3MallocZero( (sz+7)/8 + 1 ); + if( sz<=0 ){ + pBitvec = sqlite3BitvecCreate( 2*(unsigned)(-sz) ); + pV = 0; + }else{ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3MallocZero( (7+(i64)sz)/8 + 1 ); + } pTmpSpace = sqlite3_malloc64(BITVEC_SZ); - if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; + if( pBitvec==0 || pTmpSpace==0 || (pV==0 && sz>0) ) goto bitvec_end; /* NULL pBitvec tests */ sqlite3BitvecSet(0, 1); @@ -355,6 +417,24 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Run the program */ pc = i = 0; while( (op = aOp[pc])!=0 ){ + if( op>=6 ){ +#ifdef SQLITE_DEBUG + if( op==6 ){ + sqlite3ShowBitvec(pBitvec); + }else if( op==7 ){ + printf("BITVEC_SZ = %d (%d by sizeof)\n", + BITVEC_SZ, (int)sizeof(Bitvec)); + printf("BITVEC_USIZE = %d\n", (int)BITVEC_USIZE); + printf("BITVEC_NELEM = %d\n", (int)BITVEC_NELEM); + printf("BITVEC_NBIT = %d\n", (int)BITVEC_NBIT); + printf("BITVEC_NINT = %d\n", (int)BITVEC_NINT); + printf("BITVEC_MXHASH = %d\n", (int)BITVEC_MXHASH); + printf("BITVEC_NPTR = %d\n", (int)BITVEC_NPTR); + } +#endif + pc++; + continue; + } switch( op ){ case 1: case 2: @@ -365,7 +445,7 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ break; } case 3: - case 4: + case 4: default: { nx = 2; sqlite3_randomness(sizeof(i), &i); @@ -376,12 +456,12 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ pc += nx; i = (i & 0x7fffffff)%sz; if( (op & 1)!=0 ){ - SETBIT(pV, (i+1)); + if( pV ) SETBIT(pV, (i+1)); if( op!=5 ){ if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; } }else{ - CLEARBIT(pV, (i+1)); + if( pV ) CLEARBIT(pV, (i+1)); sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); } } @@ -391,14 +471,18 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ ** match (rc==0). Change rc to non-zero if a discrepancy ** is found. */ - rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) - + sqlite3BitvecTest(pBitvec, 0) - + (sqlite3BitvecSize(pBitvec) - sz); - for(i=1; i<=sz; i++){ - if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ - rc = i; - break; + if( pV ){ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } } + }else{ + rc = 0; } /* Free allocated structure */ diff --git a/src/btmutex.c b/src/btmutex.c index 45745b001d..620047c151 100644 --- a/src/btmutex.c +++ b/src/btmutex.c @@ -178,14 +178,14 @@ int sqlite3BtreeHoldsMutex(Btree *p){ ** ** There is a corresponding leave-all procedures. ** -** Enter the mutexes in accending order by BtShared pointer address +** Enter the mutexes in ascending order by BtShared pointer address ** to avoid the possibility of deadlock when two threads with ** two or more btrees in common both try to lock all their btrees ** at the same instant. */ static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){ int i; - int skipOk = 1; + u8 skipOk = 1; Btree *p; assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb; i++){ diff --git a/src/btree.c b/src/btree.c index 87bc0058bb..a931b0d127 100644 --- a/src/btree.c +++ b/src/btree.c @@ -51,7 +51,7 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ #define BTALLOC_LE 2 /* Allocate any page <= the parameter */ /* -** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not ** defined, or 0 if it is. For example: ** ** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); @@ -66,7 +66,7 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ /* ** A list of BtShared objects that are eligible for participation ** in shared cache. This variable has file scope during normal builds, -** but the test harness needs to access it so we make it global for +** but the test harness needs to access it so we make it global for ** test builds. ** ** Access to this variable is protected by SQLITE_MUTEX_STATIC_MAIN. @@ -101,7 +101,7 @@ int sqlite3_enable_shared_cache(int enable){ ** manipulate entries in the BtShared.pLock linked list used to store ** shared-cache table level locks. If the library is compiled with the ** shared-cache feature disabled, then there is only ever one user - ** of each BtShared structure and so this locking is not necessary. + ** of each BtShared structure and so this locking is not necessary. ** So define the lock related functions as no-ops. */ #define querySharedCacheTableLock(a,b,c) SQLITE_OK @@ -151,21 +151,60 @@ int corruptPageError(int lineno, MemPage *p){ # define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) #endif +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** ** -** Check to see if pBtree holds the required locks to read or write to the +** Check to see if pBtree holds the required locks to read or write to the ** table with root page iRoot. Return 1 if it does and 0 if not. ** -** For example, when writing to a table with root-page iRoot via +** For example, when writing to a table with root-page iRoot via ** Btree connection pBtree: ** ** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); ** -** When writing to an index that resides in a sharable database, the +** When writing to an index that resides in a sharable database, the ** caller should have first obtained a lock specifying the root page of ** the corresponding table. This makes things a bit more complicated, ** as this module treats each table as a separate structure. To determine @@ -187,7 +226,7 @@ static int hasSharedCacheTableLock( BtLock *pLock; /* If this database is not shareable, or if the client is reading - ** and has the read-uncommitted flag set, then no lock is required. + ** and has the read-uncommitted flag set, then no lock is required. ** Return true immediately. */ if( (pBtree->sharable==0) @@ -229,13 +268,15 @@ static int hasSharedCacheTableLock( iTab = iRoot; } - /* Search for the required lock. Either a write-lock on root-page iTab, a + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ - if( pLock->pBtree==pBtree + if( pLock->pBtree==pBtree && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) - && pLock->eLock>=eLockType + && pLock->eLock>=eLockType ){ return 1; } @@ -268,7 +309,7 @@ static int hasSharedCacheTableLock( static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ BtCursor *p; for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - if( p->pgnoRoot==iRoot + if( p->pgnoRoot==iRoot && p->pBtree!=pBtree && 0==(p->pBtree->db->flags & SQLITE_ReadUncommit) ){ @@ -280,7 +321,7 @@ static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ #endif /* #ifdef SQLITE_DEBUG */ /* -** Query to see if Btree handle p may obtain a lock of type eLock +** Query to see if Btree handle p may obtain a lock of type eLock ** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return ** SQLITE_OK if the lock may be obtained (by calling ** setSharedCacheTableLock()), or SQLITE_LOCKED if not. @@ -293,14 +334,14 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); assert( !(p->db->flags&SQLITE_ReadUncommit)||eLock==WRITE_LOCK||iTab==1 ); - + /* If requesting a write-lock, then the Btree must have an open write - ** transaction on this file. And, obviously, for this to be so there + ** transaction on this file. And, obviously, for this to be so there ** must be an open write transaction on the file itself. */ assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); - + /* This routine is a no-op if the shared-cache is not enabled */ if( !p->sharable ){ return SQLITE_OK; @@ -315,7 +356,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ } for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - /* The condition (pIter->eLock!=eLock) in the following if(...) + /* The condition (pIter->eLock!=eLock) in the following if(...) ** statement is a simplification of: ** ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) @@ -342,7 +383,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ #ifndef SQLITE_OMIT_SHARED_CACHE /* ** Add a lock on the table with root-page iTable to the shared-btree used -** by Btree handle p. Parameter eLock must be either READ_LOCK or +** by Btree handle p. Parameter eLock must be either READ_LOCK or ** WRITE_LOCK. ** ** This function assumes the following: @@ -354,7 +395,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ ** with the requested lock (i.e. querySharedCacheTableLock() has ** already been called and returned SQLITE_OK). ** -** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM +** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM ** is returned if a malloc attempt fails. */ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ @@ -362,17 +403,19 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); /* A connection with the read-uncommitted flag set will never try to ** obtain a read-lock using this function. The only read-lock obtained - ** by a connection in read-uncommitted mode is on the sqlite_schema + ** by a connection in read-uncommitted mode is on the sqlite_schema ** table, and that lock is obtained in BtreeBeginTrans(). */ assert( 0==(p->db->flags&SQLITE_ReadUncommit) || eLock==WRITE_LOCK ); - /* This function should only be called on a sharable b-tree after it + /* This function should only be called on a sharable b-tree after it ** has been determined that no other b-tree holds a conflicting lock. */ assert( p->sharable ); assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); @@ -417,7 +460,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ ** Release all the table locks (locks obtained via calls to ** the setSharedCacheTableLock() procedure) held by Btree object p. ** -** This function assumes that Btree p has an open read or write +** This function assumes that Btree p has an open read or write ** transaction. If it does not, then the BTS_PENDING flag ** may be incorrectly cleared. */ @@ -429,6 +472,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -449,7 +494,7 @@ static void clearAllSharedCacheTableLocks(Btree *p){ pBt->pWriter = 0; pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); }else if( pBt->nTransaction==2 ){ - /* This function is called when Btree p is concluding its + /* This function is called when Btree p is concluding its ** transaction. If there currently exists a writer, and p is not ** that writer, then the number of locks held by connections other ** than the writer must be about to drop to zero. In this case @@ -467,6 +512,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -495,7 +543,7 @@ static int cursorHoldsMutex(BtCursor *p){ } /* Verify that the cursor and the BtShared agree about what is the current -** database connetion. This is important in shared-cache mode. If the database +** database connetion. This is important in shared-cache mode. If the database ** connection pointers get out-of-sync, it is possible for routines like ** btreeInitPage() to reference an stale connection pointer that references a ** a connection that has already closed. This routine is used inside assert() @@ -566,8 +614,8 @@ static void invalidateIncrblobCursors( #endif /* SQLITE_OMIT_INCRBLOB */ /* -** Set bit pgno of the BtShared.pHasContent bitvec. This is called -** when a page that previously contained data becomes a free-list leaf +** Set bit pgno of the BtShared.pHasContent bitvec. This is called +** when a page that previously contained data becomes a free-list leaf ** page. ** ** The BtShared.pHasContent bitvec exists to work around an obscure @@ -593,7 +641,7 @@ static void invalidateIncrblobCursors( ** may be lost. In the event of a rollback, it may not be possible ** to restore the database to its original configuration. ** -** The solution is the BtShared.pHasContent bitvec. Whenever a page is +** The solution is the BtShared.pHasContent bitvec. Whenever a page is ** moved to become a free-list leaf page, the corresponding bit is ** set in the bitvec. Whenever a leaf page is extracted from the free-list, ** optimization 2 above is omitted if the corresponding bit is already @@ -654,13 +702,13 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ ** The cursor passed as the only argument must point to a valid entry ** when this function is called (i.e. have eState==CURSOR_VALID). This ** function saves the current cursor key in variables pCur->nKey and -** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error ** code otherwise. ** ** If the cursor is open on an intkey table, then the integer key ** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to -** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is -** set to point to a malloced buffer pCur->nKey bytes in size containing +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing ** the key. */ static int saveCursorKey(BtCursor *pCur){ @@ -676,12 +724,12 @@ static int saveCursorKey(BtCursor *pCur){ /* For an index btree, save the complete key content. It is possible ** that the current key is corrupt. In that case, it is possible that ** the sqlite3VdbeRecordUnpack() function may overread the buffer by - ** up to the size of 1 varint plus 1 8-byte value when the cursor - ** position is restored. Hence the 17 bytes of padding allocated + ** up to the size of 1 varint plus 1 8-byte value when the cursor + ** position is restored. Hence the 17 bytes of padding allocated ** below. */ void *pKey; pCur->nKey = sqlite3BtreePayloadSize(pCur); - pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); + pKey = sqlite3Malloc( ((i64)pCur->nKey) + 9 + 8 ); if( pKey ){ rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ @@ -699,11 +747,11 @@ static int saveCursorKey(BtCursor *pCur){ } /* -** Save the current cursor position in the variables BtCursor.nKey +** Save the current cursor position in the variables BtCursor.nKey ** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. ** ** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) -** prior to calling this routine. +** prior to calling this routine. */ static int saveCursorPosition(BtCursor *pCur){ int rc; @@ -742,7 +790,7 @@ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); ** routine is called just before cursor pExcept is used to modify the ** table, for example in BtreeDelete() or BtreeInsert(). ** -** If there are two or more cursors on the same btree, then all such +** If there are two or more cursors on the same btree, then all such ** cursors should have their BTCF_Multiple flag set. The btreeCursor() ** routine enforces that rule. This routine only needs to be called in ** the uncommon case when pExpect has the BTCF_Multiple flag set. @@ -824,7 +872,7 @@ static int btreeMoveto( assert( nKey==(i64)(int)nKey ); pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + sqlite3VdbeRecordUnpack((int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ rc = SQLITE_CORRUPT_BKPT; }else{ @@ -840,9 +888,9 @@ static int btreeMoveto( /* ** Restore the cursor to the position it was in (or as close to as possible) -** when saveCursorPosition() was called. Note that this call deletes the +** when saveCursorPosition() was called. Note that this call deletes the ** saved position info stored by saveCursorPosition(), so there can be -** at most one effective restoreCursorPosition() call after each +** at most one effective restoreCursorPosition() call after each ** saveCursorPosition(). */ static int btreeRestoreCursorPosition(BtCursor *pCur){ @@ -910,7 +958,7 @@ BtCursor *sqlite3BtreeFakeValidCursor(void){ /* ** This routine restores a cursor back to its original position after it ** has been moved by some outside activity (such as a btree rebalance or -** a row having been deleted out from under the cursor). +** a row having been deleted out from under the cursor). ** ** On success, the *pDifferentRow parameter is false if the cursor is left ** pointing at exactly the same row. *pDifferntRow is the row the cursor @@ -971,7 +1019,7 @@ void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ */ void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); - pCur->hints = x; + pCur->hints = (u8)x; } @@ -992,7 +1040,7 @@ static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ if( pgno<2 ) return 0; nPagesPerMapPage = (pBt->usableSize/5)+1; iPtrMap = (pgno-2)/nPagesPerMapPage; - ret = (iPtrMap*nPagesPerMapPage) + 2; + ret = (iPtrMap*nPagesPerMapPage) + 2; if( ret==PENDING_BYTE_PAGE(pBt) ){ ret++; } @@ -1165,14 +1213,15 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( static int btreePayloadToLocal(MemPage *pPage, i64 nPayload){ int maxLocal; /* Maximum amount of payload held locally */ maxLocal = pPage->maxLocal; + assert( nPayload>=0 ); if( nPayload<=maxLocal ){ - return nPayload; + return (int)nPayload; }else{ int minLocal; /* Minimum amount of payload held locally */ int surplus; /* Overflow payload available for local storage */ minLocal = pPage->minLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); - return ( surplus <= maxLocal ) ? surplus : minLocal; + surplus = (int)(minLocal +(nPayload - minLocal)%(pPage->pBt->usableSize-4)); + return (surplus <= maxLocal) ? surplus : minLocal; } } @@ -1282,11 +1331,13 @@ static void btreeParseCellPtr( pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); + assert( nPayload>=0 ); + assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); + pInfo->nSize = (u16)nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; }else{ @@ -1319,11 +1370,13 @@ static void btreeParseCellPtrIndex( pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); + assert( nPayload>=0 ); + assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); + pInfo->nSize = (u16)nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; }else{ @@ -1480,7 +1533,7 @@ static u16 cellSizePtrTableLeaf(MemPage *pPage, u8 *pCell){ }while( *(pIter)>=0x80 && pIterxParseCell(pPage, pCell, &info); if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + if( SQLITE_OVERFLOW(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ testcase( pSrc!=pPage ); *pRC = SQLITE_CORRUPT_BKPT; return; @@ -1589,7 +1642,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ /* This block handles pages with two or fewer free blocks and nMaxFrag ** or fewer fragmented bytes. In this case it is faster to move the ** two (or one) blocks of cells using memmove() and add the required - ** offsets to each pointer in the cell-pointer array than it is to + ** offsets to each pointer in the cell-pointer array than it is to ** reconstruct the entire page. */ if( (int)data[hdr+7]<=nMaxFrag ){ int iFree = get2byte(&data[hdr+1]); @@ -1634,7 +1687,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ iCellStart = get2byte(&data[hdr+5]); if( nCell>0 ){ temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); + memcpy(temp, data, usableSize); src = temp; for(i=0; ipDbPage) ); assert( pPage->pBt ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -1858,30 +1911,30 @@ static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** ** Even though the freeblock list was checked by btreeComputeFreeSpace(), ** that routine will not detect overlap between cells or freeblocks. Nor -** does it detect cells or freeblocks that encrouch into the reserved bytes +** does it detect cells or freeblocks that encroach into the reserved bytes ** at the end of the page. So do additional corruption checks inside this ** routine and return SQLITE_CORRUPT if any problems are found. */ -static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ - u16 iPtr; /* Address of ptr to next freeblock */ - u16 iFreeBlk; /* Address of the next freeblock */ +static int freeSpace(MemPage *pPage, int iStart, int iSize){ + int iPtr; /* Address of ptr to next freeblock */ + int iFreeBlk; /* Address of the next freeblock */ u8 hdr; /* Page header size. 0 or 100 */ - u8 nFrag = 0; /* Reduction in fragmentation */ - u16 iOrigSize = iSize; /* Original value of iSize */ - u16 x; /* Offset to cell content area */ - u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + int nFrag = 0; /* Reduction in fragmentation */ + int iOrigSize = iSize; /* Original value of iSize */ + int x; /* Offset to cell content area */ + int iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ u8 *pTmp; /* Temporary ptr into data[] */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= (int)pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=(int)pPage->pBt->usableSize-4 ); - /* The list of freeblocks must be in ascending order. Find the + /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. */ hdr = pPage->hdrOffset; @@ -1896,11 +1949,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } iPtr = iFreeBlk; } - if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + if( iFreeBlk>(int)pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ return SQLITE_CORRUPT_PAGE(pPage); } assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB ); - + /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none ** iPtr: The address of a pointer to iFreeBlk @@ -1911,13 +1964,13 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - if( iEnd > pPage->pBt->usableSize ){ + if( iEnd > (int)pPage->pBt->usableSize ){ return SQLITE_CORRUPT_PAGE(pPage); } iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); } - + /* If iPtr is another freeblock (that is, if iPtr is not the freelist ** pointer in the page header) then check to see if iStart should be ** coalesced onto the end of iPtr. @@ -1932,7 +1985,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } } if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); - data[hdr+7] -= nFrag; + data[hdr+7] -= (u8)nFrag; } pTmp = &data[hdr+5]; x = get2byte(pTmp); @@ -1953,7 +2006,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); + assert( iSize>=0 && iSize<=0xffff ); + put2byte(&data[iStart+2], (u16)iSize); } pPage->nFree += iOrigSize; return SQLITE_OK; @@ -2075,7 +2129,7 @@ static int btreeComputeFreeSpace(MemPage *pPage){ /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will ** always be at least one cell before the first freeblock. */ - return SQLITE_CORRUPT_PAGE(pPage); + return SQLITE_CORRUPT_PAGE(pPage); } while( 1 ){ if( pc>iCellLast ){ @@ -2114,7 +2168,7 @@ static int btreeComputeFreeSpace(MemPage *pPage){ /* ** Do additional sanity check after btreeInitPage() if -** PRAGMA cell_size_check=ON +** PRAGMA cell_size_check=ON */ static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ int iCellFirst; /* First allowable cell or freeblock offset */ @@ -2152,7 +2206,7 @@ static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ ** Initialize the auxiliary information for a disk block. ** ** Return SQLITE_OK on success. If we see that the page does -** not contain a well-formed database page, then return +** not contain a well-formed database page, then return ** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not ** guarantee that the page is well-formed. It only shows that ** we failed to detect any corruption. @@ -2179,7 +2233,7 @@ static int btreeInitPage(MemPage *pPage){ assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; - pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->cellOffset = (u16)(pPage->hdrOffset + 8 + pPage->childPtrSize); pPage->aCellIdx = data + pPage->childPtrSize + 8; pPage->aDataEnd = pPage->aData + pBt->pageSize; pPage->aDataOfst = pPage->aData + pPage->childPtrSize; @@ -2213,8 +2267,8 @@ static int btreeInitPage(MemPage *pPage){ static void zeroPage(MemPage *pPage, int flags){ unsigned char *data = pPage->aData; BtShared *pBt = pPage->pBt; - u8 hdr = pPage->hdrOffset; - u16 first; + int hdr = pPage->hdrOffset; + int first; assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); @@ -2231,7 +2285,7 @@ static void zeroPage(MemPage *pPage, int flags){ put2byte(&data[hdr+5], pBt->usableSize); pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); - pPage->cellOffset = first; + pPage->cellOffset = (u16)first; pPage->aDataEnd = &data[pBt->pageSize]; pPage->aCellIdx = &data[first]; pPage->aDataOfst = &data[pPage->childPtrSize]; @@ -2257,7 +2311,7 @@ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ pPage->hdrOffset = pgno==1 ? 100 : 0; } assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); - return pPage; + return pPage; } /* @@ -2317,68 +2371,41 @@ Pgno sqlite3BtreeLastPage(Btree *p){ /* ** Get a page from the pager and initialize it. -** -** If pCur!=0 then the page is being fetched as part of a moveToChild() -** call. Do additional sanity checking on the page in this case. -** And if the fetch fails, this routine must decrement pCur->iPage. -** -** The page is fetched as read-write unless pCur is not NULL and is -** a read-only cursor. -** -** If an error occurs, then *ppPage is undefined. It -** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - BtCursor *pCur, /* Cursor to receive the page, or NULL */ int bReadOnly /* True for a read-only page */ ){ int rc; DbPage *pDbPage; + MemPage *pPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pCur==0 || ppPage==&pCur->pPage ); - assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); - assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ - rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error1; + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error1; + *ppPage = 0; + return rc; } - *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - if( (*ppPage)->isInit==0 ){ + pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( pPage->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); - rc = btreeInitPage(*ppPage); + rc = btreeInitPage(pPage); if( rc!=SQLITE_OK ){ - goto getAndInitPage_error2; + releasePage(pPage); + *ppPage = 0; + return rc; } } - assert( (*ppPage)->pgno==pgno || CORRUPT_DB ); - assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); - - /* If obtaining a child page for a cursor, we must verify that the page is - ** compatible with the root page. */ - if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ - rc = SQLITE_CORRUPT_PGNO(pgno); - goto getAndInitPage_error2; - } + assert( pPage->pgno==pgno || CORRUPT_DB ); + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + *ppPage = pPage; return SQLITE_OK; - -getAndInitPage_error2: - releasePage(*ppPage); -getAndInitPage_error1: - if( pCur ){ - pCur->iPage--; - pCur->pPage = pCur->apPage[pCur->iPage]; - } - testcase( pgno==0 ); - assert( pgno!=0 || rc!=SQLITE_OK ); - return rc; } /* @@ -2461,7 +2488,7 @@ static void pageReinit(DbPage *pData){ ** call to btreeInitPage() will likely return SQLITE_CORRUPT. ** But no harm is done by this. And it is very important that ** btreeInitPage() be called on every btree page so we make - ** the call for every page that comes in for re-initing. */ + ** the call for every page that comes in for re-initializing. */ btreeInitPage(pPage); } } @@ -2479,11 +2506,11 @@ static int btreeInvokeBusyHandler(void *pArg){ /* ** Open a database file. -** +** ** zFilename is the name of the database file. If zFilename is NULL ** then an ephemeral database is created. The ephemeral database might ** be exclusively in memory, or it might use a disk-based memory cache. -** Either way, the ephemeral database will be automatically deleted +** Either way, the ephemeral database will be automatically deleted ** when sqlite3BtreeClose() is called. ** ** If zFilename is ":memory:" then an in-memory database is created @@ -2516,7 +2543,7 @@ int sqlite3BtreeOpen( /* True if opening an ephemeral, temporary database */ const int isTempDb = zFilename==0 || zFilename[0]==0; - /* Set the variable isMemdb to true for an in-memory database, or + /* Set the variable isMemdb to true for an in-memory database, or ** false for a file-based database. */ #ifdef SQLITE_OMIT_MEMORYDB @@ -2639,7 +2666,10 @@ int sqlite3BtreeOpen( assert( sizeof(u32)==4 ); assert( sizeof(u16)==2 ); assert( sizeof(Pgno)==4 ); - + + /* Suppress false-positive compiler warning from PVS-Studio */ + memset(&zDbHeader[16], 0, 8); + pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -2658,7 +2688,7 @@ int sqlite3BtreeOpen( pBt->db = db; sqlite3PagerSetBusyHandler(pBt->pPager, btreeInvokeBusyHandler, pBt); p->pBt = pBt; - + pBt->pCursor = 0; pBt->pPage1 = 0; if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY; @@ -2702,7 +2732,7 @@ int sqlite3BtreeOpen( if( rc ) goto btree_open_out; pBt->usableSize = pBt->pageSize - nReserve; assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ - + #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* Add the new BtShared object to the linked list sharable BtShareds. */ @@ -2826,12 +2856,13 @@ static int removeFromSharingList(BtShared *pBt){ sqlite3_mutex_leave(pMainMtx); return removed; #else + UNUSED_PARAMETER( pBt ); return 1; #endif } /* -** Make sure pBt->pTmpSpace points to an allocation of +** Make sure pBt->pTmpSpace points to an allocation of ** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child ** pointer. */ @@ -2856,7 +2887,7 @@ static SQLITE_NOINLINE int allocateTempSpace(BtShared *pBt){ ** can mean that fillInCell() only initializes the first 2 or 3 ** bytes of pTmpSpace, but that the first 4 bytes are copied from ** it into a database page. This is not actually a problem, but it - ** does cause a valgrind error when the 1 or 2 bytes of unitialized + ** does cause a valgrind error when the 1 or 2 bytes of uninitialized ** data is passed to system call write(). So to avoid this error, ** zero the first 4 bytes of temp space here. ** @@ -2911,7 +2942,7 @@ int sqlite3BtreeClose(Btree *p){ sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree - ** structure, return now. The remainder of this procedure cleans + ** structure, return now. The remainder of this procedure cleans ** up the shared-btree. */ assert( p->wantToLock==0 && p->locked==0 ); @@ -3017,7 +3048,7 @@ int sqlite3BtreeSetPagerFlags( /* ** Change the default pages size and the number of reserved bytes per page. -** Or, if the page size has already been fixed, return SQLITE_READONLY +** Or, if the page size has already been fixed, return SQLITE_READONLY ** without changing anything. ** ** The page size must be a power of 2 between 512 and 65536. If the page @@ -3041,8 +3072,12 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ BtShared *pBt = p->pBt; assert( nReserve>=0 && nReserve<=255 ); sqlite3BtreeEnter(p); - pBt->nReserveWanted = nReserve; + pBt->nReserveWanted = (u8)nReserve; x = pBt->pageSize - pBt->usableSize; + if( x==nReserve && (pageSize==0 || (u32)pageSize==pBt->pageSize) ){ + sqlite3BtreeLeave(p); + return SQLITE_OK; + } if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); @@ -3077,7 +3112,7 @@ int sqlite3BtreeGetPageSize(Btree *p){ ** held. ** ** This is useful in one special case in the backup API code where it is -** known that the shared b-tree mutex is held, but the mutex on the +** known that the shared b-tree mutex is held, but the mutex on the ** database handle that owns *p is not. In this case if sqlite3BtreeEnter() ** were to be called, it might collide with some other operation on the ** database handle that owns *p, causing undefined behavior. @@ -3091,7 +3126,7 @@ int sqlite3BtreeGetReserveNoMutex(Btree *p){ /* ** Return the number of bytes of space at the end of every page that -** are intentually left unused. This is the "reserved" space that is +** are intentionally left unused. This is the "reserved" space that is ** sometimes used by extensions. ** ** The value returned is the larger of the current reserve size and @@ -3147,7 +3182,7 @@ int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ assert( BTS_FAST_SECURE==(BTS_OVERWRITE|BTS_SECURE_DELETE) ); if( newFlag>=0 ){ p->pBt->btsFlags &= ~BTS_FAST_SECURE; - p->pBt->btsFlags |= BTS_SECURE_DELETE*newFlag; + p->pBt->btsFlags |= (u16)(BTS_SECURE_DELETE*newFlag); } b = (p->pBt->btsFlags & BTS_FAST_SECURE)/BTS_SECURE_DELETE; sqlite3BtreeLeave(p); @@ -3157,7 +3192,7 @@ int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ /* ** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' ** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it -** is disabled. The default value for the auto-vacuum property is +** is disabled. The default value for the auto-vacuum property is ** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. */ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ @@ -3181,7 +3216,7 @@ int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ } /* -** Return the value of the 'auto-vacuum' property. If auto-vacuum is +** Return the value of the 'auto-vacuum' property. If auto-vacuum is ** enabled 1 is returned. Otherwise 0. */ int sqlite3BtreeGetAutoVacuum(Btree *p){ @@ -3213,9 +3248,9 @@ static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ Db *pDb; if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } - if( pDb->bSyncSet==0 - && pDb->safety_level!=safety_level - && pDb!=&db->aDb[1] + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] ){ pDb->safety_level = safety_level; sqlite3PagerSetFlags(pBt->pPager, @@ -3238,7 +3273,7 @@ static int newDatabase(BtShared*); ** SQLITE_OK is returned on success. If the file is not a ** well-formed database file, then SQLITE_CORRUPT is returned. ** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM -** is returned if we run out of memory. +** is returned if we run out of memory. */ static int lockBtree(BtShared *pBt){ int rc; /* Result code from subfunctions */ @@ -3254,7 +3289,7 @@ static int lockBtree(BtShared *pBt){ if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is - ** a valid database file. + ** a valid database file. */ nPage = get4byte(28+(u8*)pPage1->aData); sqlite3PagerPagecount(pBt->pPager, (int*)&nPageFile); @@ -3292,7 +3327,7 @@ static int lockBtree(BtShared *pBt){ } /* If the read version is set to 2, this database should be accessed - ** in WAL mode. If the log is not already open, open it now. Then + ** in WAL mode. If the log is not already open, open it now. Then ** return SQLITE_OK and return without populating BtShared.pPage1. ** The caller detects this and calls this function again. This is ** required as the version of page 1 currently in the page1 buffer @@ -3333,16 +3368,15 @@ static int lockBtree(BtShared *pBt){ /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 - || pageSize>SQLITE_MAX_PAGE_SIZE - || pageSize<=256 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 ){ goto page1_init_failed; } - pBt->btsFlags |= BTS_PAGESIZE_FIXED; assert( (pageSize & 7)==0 ); /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte ** integer at offset 20 is the number of bytes of space at the end of - ** each page to reserve for extensions. + ** each page to reserve for extensions. ** ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is ** determined by the one-byte unsigned integer found at an offset of 20 @@ -3358,6 +3392,7 @@ static int lockBtree(BtShared *pBt){ releasePageOne(pPage1); pBt->usableSize = usableSize; pBt->pageSize = pageSize; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); @@ -3377,6 +3412,7 @@ static int lockBtree(BtShared *pBt){ if( usableSize<480 ){ goto page1_init_failed; } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; pBt->pageSize = pageSize; pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -3436,7 +3472,7 @@ static int countValidCursors(BtShared *pBt, int wrOnly){ int r = 0; for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) - && pCur->eState!=CURSOR_FAULT ) r++; + && pCur->eState!=CURSOR_FAULT ) r++; } return r; } @@ -3445,7 +3481,7 @@ static int countValidCursors(BtShared *pBt, int wrOnly){ /* ** If there are no outstanding cursors and we are not in the middle ** of a transaction but there is a read lock on the database, then -** this routine unrefs the first page of the database file which +** this routine unrefs the first page of the database file which ** has the effect of releasing the read lock. ** ** If there is a transaction in progress, this routine is a no-op. @@ -3529,8 +3565,8 @@ int sqlite3BtreeNewDb(Btree *p){ ** upgraded to exclusive by calling this routine a second time - the ** exclusivity flag only works for a new transaction. ** -** A write-transaction must be started before attempting any -** changes to the database. None of the following routines +** A write-transaction must be started before attempting any +** changes to the database. None of the following routines ** will work unless a transaction is started first: ** ** sqlite3BtreeCreateTable() @@ -3544,7 +3580,7 @@ int sqlite3BtreeNewDb(Btree *p){ ** If an initial attempt to acquire the lock fails because of lock contention ** and the database was previously unlocked, then invoke the busy handler ** if there is one. But if there was previously a read-lock, do not -** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is +** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is ** returned when there is already a read-lock in order to avoid a deadlock. ** ** Suppose there are two processes A and B. A has a read lock and B has @@ -3555,7 +3591,11 @@ int sqlite3BtreeNewDb(Btree *p){ ** when A already has a read lock, we encourage A to give up and let B ** proceed. */ -int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ +static SQLITE_NOINLINE int btreeBeginTrans( + Btree *p, /* The btree in which to start the transaction */ + int wrflag, /* True to start a write transaction */ + int *pSchemaVersion /* Put schema version number here, if not NULL */ +){ BtShared *pBt = p->pBt; Pager *pPager = pBt->pPager; int rc = SQLITE_OK; @@ -3572,8 +3612,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ } assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); - if( (p->db->flags & SQLITE_ResetDatabase) - && sqlite3PagerIsreadonly(pPager)==0 + if( (p->db->flags & SQLITE_ResetDatabase) + && sqlite3PagerIsreadonly(pPager)==0 ){ pBt->btsFlags &= ~BTS_READ_ONLY; } @@ -3587,7 +3627,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ #ifndef SQLITE_OMIT_SHARED_CACHE { sqlite3 *pBlock = 0; - /* If another database handle has already opened a write transaction + /* If another database handle has already opened a write transaction ** on this shared-btree structure and a second write transaction is ** requested, return SQLITE_LOCKED. */ @@ -3612,8 +3652,8 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ } #endif - /* Any read-only or read-write transaction implies a read-lock on - ** page 1. So if some other shared-cache client already has a write-lock + /* Any read-only or read-write transaction implies a read-lock on + ** page 1. So if some other shared-cache client already has a write-lock ** on page 1, the transaction cannot be opened. */ rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); if( SQLITE_OK!=rc ) goto trans_begun; @@ -3636,7 +3676,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after - ** reading page 1 it discovers that the page-size of the database + ** reading page 1 it discovers that the page-size of the database ** file is not pBt->pageSize. In this case lockBtree() will update ** pBt->pageSize to the page-size of the file on disk. */ @@ -3657,11 +3697,18 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ } } } - + if( rc!=SQLITE_OK ){ (void)sqlite3PagerWalWriteLock(pPager, 0); unlockBtreeIfUnused(pBt); } +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) + if( rc==SQLITE_BUSY_TIMEOUT ){ + /* If a blocking lock timed out, break out of the loop here so that + ** the busy-handler is not invoked. */ + break; + } +#endif }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && btreeInvokeBusyHandler(pBt) ); sqlite3PagerWalDb(pPager, 0); @@ -3696,7 +3743,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ /* If the db-size header field is incorrect (as it may be if an old ** client has been writing the database file), update it now. Doing - ** this sooner rather than later means the database size can safely + ** this sooner rather than later means the database size can safely ** re-read the database size from page 1 if a savepoint or transaction ** rollback occurs within the transaction. */ @@ -3727,6 +3774,28 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ sqlite3BtreeLeave(p); return rc; } +int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt; + if( p->sharable + || p->inTrans==TRANS_NONE + || (p->inTrans==TRANS_READ && wrflag!=0) + ){ + return btreeBeginTrans(p,wrflag,pSchemaVersion); + } + pBt = p->pBt; + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + return sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + }else{ + return SQLITE_OK; + } +} #ifndef SQLITE_OMIT_AUTOVACUUM @@ -3771,7 +3840,7 @@ static int setChildPtrmaps(MemPage *pPage){ ** that it points to iTo. Parameter eType describes the type of pointer to ** be modified, as follows: ** -** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child ** page of pPage. ** ** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow @@ -3822,9 +3891,9 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } } } - + if( i==nCell ){ - if( eType!=PTRMAP_BTREE || + if( eType!=PTRMAP_BTREE || get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ return SQLITE_CORRUPT_PAGE(pPage); } @@ -3836,11 +3905,11 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ /* -** Move the open database page pDbPage to location iFreePage in the +** Move the open database page pDbPage to location iFreePage in the ** database. The pDbPage reference remains valid. ** ** The isCommit flag indicates that there is no need to remember that -** the journal needs to be sync()ed before database page pDbPage->pgno +** the journal needs to be sync()ed before database page pDbPage->pgno ** can be written to. The caller has already promised not to write to that ** page. */ @@ -3857,14 +3926,14 @@ static int relocatePage( Pager *pPager = pBt->pPager; int rc; - assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || + assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); assert( sqlite3_mutex_held(pBt->mutex) ); assert( pDbPage->pBt==pBt ); if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", + TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ @@ -3923,19 +3992,19 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); /* ** Perform a single step of an incremental-vacuum. If successful, return -** SQLITE_OK. If there is no work to do (and therefore no point in -** calling this function again), return SQLITE_DONE. Or, if an error +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error ** occurs, return some other error code. ** -** More specifically, this function attempts to re-organize the database so +** More specifically, this function attempts to re-organize the database so ** that the last page of the file currently in use is no longer in use. ** ** Parameter nFin is the number of pages that this database would contain ** were this function called until it returns SQLITE_DONE. ** -** If the bCommit parameter is non-zero, this function assumes that the -** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE -** or an error. bCommit is passed true for an auto-vacuum-on-commit +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commit ** operation, or false for an incremental vacuum. */ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ @@ -3966,7 +4035,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ if( bCommit==0 ){ /* Remove the page from the files free-list. This is not required ** if bCommit is non-zero. In that case, the free-list will be - ** truncated to zero after this function returns, so it doesn't + ** truncated to zero after this function returns, so it doesn't ** matter if it still contains some garbage entries. */ Pgno iFreePg; @@ -4015,7 +4084,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ } }while( bCommit && iFreePg>nFin ); assert( iFreePgpBt; + pBt = p->pBt; pPager = pBt->pPager; VVA_ONLY( nRef = sqlite3PagerRefcount(pPager); ) @@ -4204,7 +4273,7 @@ static int autoVacuumCommit(Btree *p){ ** ** Otherwise, sync the database file for the btree pBt. zSuperJrnl points to ** the name of a super-journal file that should be written into the -** individual journal file, or is NULL, indicating no super-journal file +** individual journal file, or is NULL, indicating no super-journal file ** (single database transaction). ** ** When this is called, the super-journal should already have been @@ -4255,8 +4324,8 @@ static void btreeEndTransaction(Btree *p){ downgradeAllSharedCacheTableLocks(p); p->inTrans = TRANS_READ; }else{ - /* If the handle had any kind of transaction open, decrement the - ** transaction count of the shared btree. If the transaction count + /* If the handle had any kind of transaction open, decrement the + ** transaction count of the shared btree. If the transaction count ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() ** call below will unlock the pager. */ if( p->inTrans!=TRANS_NONE ){ @@ -4267,7 +4336,7 @@ static void btreeEndTransaction(Btree *p){ } } - /* Set the current transaction state to TRANS_NONE and unlock the + /* Set the current transaction state to TRANS_NONE and unlock the ** pager if this call closed the only read or write transaction. */ p->inTrans = TRANS_NONE; unlockBtreeIfUnused(pBt); @@ -4288,12 +4357,12 @@ static void btreeEndTransaction(Btree *p){ ** the rollback journal (which causes the transaction to commit) and ** drop locks. ** -** Normally, if an error occurs while the pager layer is attempting to +** Normally, if an error occurs while the pager layer is attempting to ** finalize the underlying journal file, this function returns an error and ** the upper layer will attempt a rollback. However, if the second argument -** is non-zero then this b-tree transaction is part of a multi-file -** transaction. In this case, the transaction has already been committed -** (by deleting a super-journal file) and the caller will ignore this +** is non-zero then this b-tree transaction is part of a multi-file +** transaction. In this case, the transaction has already been committed +** (by deleting a super-journal file) and the caller will ignore this ** functions return code. So, even if an error occurs in the pager layer, ** reset the b-tree objects internal state to indicate that the write ** transaction has been closed. This is quite safe, as the pager will have @@ -4308,7 +4377,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeEnter(p); btreeIntegrity(p); - /* If the handle has a write-transaction open, commit the shared-btrees + /* If the handle has a write-transaction open, commit the shared-btrees ** transaction and set the shared state to TRANS_READ. */ if( p->inTrans==TRANS_WRITE ){ @@ -4357,15 +4426,15 @@ int sqlite3BtreeCommit(Btree *p){ ** ** This routine gets called when a rollback occurs. If the writeOnly ** flag is true, then only write-cursors need be tripped - read-only -** cursors save their current positions so that they may continue -** following the rollback. Or, if writeOnly is false, all cursors are +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are ** tripped. In general, writeOnly is false if the transaction being ** rolled back modified the database schema. In this case b-tree root ** pages may be moved or deleted from the database altogether, making ** it unsafe for read cursors to continue. ** -** If the writeOnly flag is true and an error is encountered while -** saving the current position of a read-only cursor, all cursors, +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, ** including all read-cursors are tripped. ** ** SQLITE_OK is returned if successful, or if an error occurs while @@ -4471,8 +4540,8 @@ int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ /* ** Start a statement subtransaction. The subtransaction can be rolled -** back independently of the main transaction. You must start a transaction -** before starting a subtransaction. The subtransaction is ended automatically +** back independently of the main transaction. You must start a transaction +** before starting a subtransaction. The subtransaction is ended automatically ** if the main transaction commits or rolls back. ** ** Statement subtransactions are used around individual SQL statements @@ -4509,11 +4578,11 @@ int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ /* ** The second argument to this function, op, is always SAVEPOINT_ROLLBACK ** or SAVEPOINT_RELEASE. This function either releases or rolls back the -** savepoint identified by parameter iSavepoint, depending on the value +** savepoint identified by parameter iSavepoint, depending on the value ** of op. ** ** Normally, iSavepoint is greater than or equal to zero. However, if op is -** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the ** contents of the entire transaction are rolled back. This is different ** from a normal transaction rollback, as no locks are released and the ** transaction remains open. @@ -4538,7 +4607,7 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ rc = newDatabase(pBt); btreeSetNPage(pBt, pBt->pPage1); - /* pBt->nPage might be zero if the database was corrupt when + /* pBt->nPage might be zero if the database was corrupt when ** the transaction was started. Otherwise, it must be at least 1. */ assert( CORRUPT_DB || pBt->nPage>0 ); } @@ -4576,10 +4645,10 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** is set. If FORDELETE is set, that is a hint to the implementation that ** this cursor will only be used to seek to and delete entries of an index ** as part of a larger DELETE statement. The FORDELETE hint is not used by -** this implementation. But in a hypothetical alternative storage engine +** this implementation. But in a hypothetical alternative storage engine ** in which index entries are automatically deleted when corresponding table ** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE -** operations on this cursor can be no-ops and all READ operations can +** operations on this cursor can be no-ops and all READ operations can ** return a null row (2-bytes: 0x01 0x00). ** ** No checking is done to make sure that page iTable really is the @@ -4600,14 +4669,14 @@ static int btreeCursor( BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 - || wrFlag==BTREE_WRCSR - || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) ); - /* The following assert statements verify that if this is a sharable - ** b-tree database, the connection is holding the required table locks, - ** and that no other connection has any open cursor that conflicts with + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with ** this lock. The iTable<1 term disables the check for corrupt schemas. */ assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) || iTable<1 ); @@ -4695,6 +4764,25 @@ int sqlite3BtreeCursorSize(void){ return ROUND8(sizeof(BtCursor)); } +#ifdef SQLITE_DEBUG +/* +** Return true if and only if the Btree object will be automatically +** closed with the BtCursor closes. This is used within assert() statements +** only. +*/ +int sqlite3BtreeClosesWithCursor( + Btree *pBtree, /* the btree object */ + BtCursor *pCur /* Corresponding cursor */ +){ + BtShared *pBt = pBtree->pBt; + if( (pBt->openFlags & BTREE_SINGLE)==0 ) return 0; + if( pBt->pCursor!=pCur ) return 0; + if( pCur->pNext!=0 ) return 0; + if( pCur->pBtree!=pBtree ) return 0; + return 1; +} +#endif + /* ** Initialize memory that will be converted into a BtCursor object. ** @@ -4822,7 +4910,6 @@ void sqlite3BtreeCursorUnpin(BtCursor *pCur){ pCur->curFlags &= ~BTCF_Pinned; } -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC /* ** Return the offset into the database file for the start of the ** payload to which the cursor is pointing. @@ -4834,7 +4921,6 @@ i64 sqlite3BtreeOffset(BtCursor *pCur){ return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + (i64)(pCur->info.pPayload - pCur->pPage->aData); } -#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ /* ** Return the number of bytes of payload for the entry that pCur is @@ -4860,7 +4946,7 @@ u32 sqlite3BtreePayloadSize(BtCursor *pCur){ ** routine always returns 2147483647 (which is the largest record ** that SQLite can handle) or more. But returning a smaller value might ** prevent large memory allocations when trying to interpret a -** corrupt datrabase. +** corrupt database. ** ** The current implementation merely returns the size of the underlying ** database file. @@ -4873,15 +4959,15 @@ sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor *pCur){ /* ** Given the page number of an overflow page in the database (parameter -** ovfl), this function finds the page number of the next page in the +** ovfl), this function finds the page number of the next page in the ** linked list of overflow pages. If possible, it uses the auto-vacuum -** pointer-map data instead of reading the content of page ovfl to do so. +** pointer-map data instead of reading the content of page ovfl to do so. ** ** If an error occurs an SQLite error code is returned. Otherwise: ** -** The page number of the next overflow page in the linked list is -** written to *pPgnoNext. If page ovfl is the last page in its linked -** list, *pPgnoNext is set to zero. +** The page number of the next overflow page in the linked list is +** written to *pPgnoNext. If page ovfl is the last page in its linked +** list, *pPgnoNext is set to zero. ** ** If ppPage is not NULL, and a reference to the MemPage object corresponding ** to page number pOvfl was obtained, then *ppPage is set to point to that @@ -4905,9 +4991,9 @@ static int getOverflowPage( #ifndef SQLITE_OMIT_AUTOVACUUM /* Try to find the next page in the overflow list using the - ** autovacuum pointer-map pages. Guess that the next page in - ** the overflow list is page number (ovfl+1). If that guess turns - ** out to be wrong, fall back to loading the data of page + ** autovacuum pointer-map pages. Guess that the next page in + ** the overflow list is page number (ovfl+1). If that guess turns + ** out to be wrong, fall back to loading the data of page ** number ovfl to determine the next page number. */ if( pBt->autoVacuum ){ @@ -4995,8 +5081,8 @@ static int copyPayload( ** ** If the current cursor entry uses one or more overflow pages ** this function may allocate space for and lazily populate -** the overflow page-list cache array (BtCursor.aOverflow). -** Subsequent calls use this cache to make seeking to the supplied offset +** the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** ** Once an overflow page-list cache has been allocated, it must be @@ -5012,7 +5098,7 @@ static int accessPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 offset, /* Begin reading this far into payload */ u32 amt, /* Read this many bytes */ - unsigned char *pBuf, /* Write the bytes into this buffer */ + unsigned char *pBuf, /* Write the bytes into this buffer */ int eOp /* zero to read. non-zero to write. */ ){ unsigned char *aPayload; @@ -5066,7 +5152,7 @@ static int accessPayload( Pgno nextPage; nextPage = get4byte(&aPayload[pCur->info.nLocal]); - + /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. ** ** The aOverflow[] array is sized at one entry for each overflow page @@ -5079,9 +5165,12 @@ static int accessPayload( if( pCur->aOverflow==0 || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -5091,6 +5180,12 @@ static int accessPayload( memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); pCur->curFlags |= BTCF_ValidOvfl; }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + /* If the overflow page-list cache has been allocated and the ** entry for the first required overflow page is valid, skip ** directly to it. @@ -5138,12 +5233,12 @@ static int accessPayload( #ifdef SQLITE_DIRECT_OVERFLOW_READ /* If all the following are true: ** - ** 1) this is a read operation, and + ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and ** 3) there are no dirty pages in the page-cache ** 4) the database is file-backed, and ** 5) the page is not in the WAL file - ** 6) at least 4 bytes have already been read into the output buffer + ** 6) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds @@ -5160,7 +5255,6 @@ static int accessPayload( assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); - if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else @@ -5250,7 +5344,7 @@ int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ #endif /* SQLITE_OMIT_INCRBLOB */ /* -** Return a pointer to payload information from the entry that the +** Return a pointer to payload information from the entry that the ** pCur cursor is pointing to. The pointer is to the beginning of ** the key if index btrees (pPage->intKey==0) and is the data for ** table btrees (pPage->intKey==1). The number of bytes of available @@ -5322,6 +5416,7 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ + int rc; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageapPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur, - pCur->curPagerFlags); + rc = getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur->curPagerFlags); + assert( pCur->pPage!=0 || rc!=SQLITE_OK ); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(newPgno); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + } + return rc; } #ifdef SQLITE_DEBUG /* -** Page pParent is an internal (non-leaf) tree page. This function +** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th ** cell in page pParent. Or, if iIdx is equal to the total number of ** cells in pParent, that page number iChild is the right-child of @@ -5358,7 +5463,7 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ } } #else -# define assertParentIndex(x,y,z) +# define assertParentIndex(x,y,z) #endif /* @@ -5376,8 +5481,8 @@ static void moveToParent(BtCursor *pCur){ assert( pCur->iPage>0 ); assert( pCur->pPage ); assertParentIndex( - pCur->apPage[pCur->iPage-1], - pCur->aiIdx[pCur->iPage-1], + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], pCur->pPage->pgno ); testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); @@ -5394,19 +5499,19 @@ static void moveToParent(BtCursor *pCur){ ** ** If the table has a virtual root page, then the cursor is moved to point ** to the virtual root page instead of the actual root page. A table has a -** virtual root page when the actual root page contains no cells and a +** virtual root page when the actual root page contains no cells and a ** single child page. This can only happen with the table rooted at page 1. ** -** If the b-tree structure is empty, the cursor state is set to +** If the b-tree structure is empty, the cursor state is set to ** CURSOR_INVALID and this routine returns SQLITE_EMPTY. Otherwise, ** the cursor is set to point to the first cell located on the root ** (or virtual root) page and the cursor state is set to CURSOR_VALID. ** ** If this function returns successfully, it may be assumed that the -** page-header flags indicate that the [virtual] root-page is the expected +** page-header flags indicate that the [virtual] root-page is the expected ** kind of b-tree page (i.e. if when opening the cursor the caller did not ** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, -** indicating a table b-tree, or if the caller did specify a KeyInfo +** indicating a table b-tree, or if the caller did specify a KeyInfo ** structure the flags byte is set to 0x02 or 0x0A, indicating an index ** b-tree). */ @@ -5443,7 +5548,7 @@ static int moveToRoot(BtCursor *pCur){ sqlite3BtreeClearCursor(pCur); } rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, - 0, pCur->curPagerFlags); + pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -5457,19 +5562,19 @@ static int moveToRoot(BtCursor *pCur){ /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is ** NULL, the caller expects a table b-tree. If this is not the case, - ** return an SQLITE_CORRUPT error. + ** return an SQLITE_CORRUPT error. ** ** Earlier versions of SQLite assumed that this test could not fail ** if the root page was already loaded when this function was called (i.e. - ** if pCur->iPage>=0). But this is not so if the database is corrupted - ** in such a way that page pRoot is linked into a second b-tree table + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table ** (or the freelist). */ assert( pRoot->intKey==1 || pRoot->intKey==0 ); if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ return SQLITE_CORRUPT_PAGE(pCur->pPage); } -skip_init: +skip_init: pCur->ix = 0; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); @@ -5555,13 +5660,54 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ *pRes = 0; rc = moveToLeftmost(pCur); }else if( rc==SQLITE_EMPTY ){ - assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + assert( pCur->pgnoRoot==0 || (pCur->pPage!=0 && pCur->pPage->nCell==0) ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} + +/* Set *pRes to 1 (true) if the BTree pointed to by cursor pCur contains zero +** rows of content. Set *pRes to 0 (false) if the table contains content. +** Return SQLITE_OK on success or some error code (ex: SQLITE_NOMEM) if +** something goes wrong. +*/ +int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + if( pCur->eState==CURSOR_VALID ){ + *pRes = 0; + return SQLITE_OK; + } + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ *pRes = 1; rc = SQLITE_OK; + }else{ + *pRes = 0; } return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -5590,18 +5736,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); - testcase( pCur->ix!=pCur->pPage->nCell-1 ); - /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ - assert( pCur->pPage->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = 0; return SQLITE_OK; } @@ -5617,7 +5752,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ ** before or after the key. ** ** An integer is written into *pRes which is the result of -** comparing the key with the entry to which the cursor is +** comparing the key with the entry to which the cursor is ** pointing. The meaning of the integer written into ** *pRes is as follows: ** @@ -5654,13 +5789,14 @@ int sqlite3BtreeTableMoveto( } if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); *pRes = -1; return SQLITE_OK; } /* If the requested key is one more than the previous key, then ** try to get there using sqlite3BtreeNext() rather than a full ** binary search. This is an optimization only. The correct answer - ** is still obtained without this case, only a little more slowely */ + ** is still obtained without this case, only a little more slowly. */ if( pCur->info.nKey+1==intKey ){ *pRes = 0; rc = sqlite3BtreeNext(pCur, 0); @@ -5774,8 +5910,8 @@ int sqlite3BtreeTableMoveto( } /* -** Compare the "idx"-th cell on the page the cursor pCur is currently -** pointing to to pIdxKey using xRecordCompare. Return negative or +** Compare the "idx"-th cell on the page pPage against the key +** pointing to by pIdxKey using xRecordCompare. Return negative or ** zero if the cell is less than or equal pIdxKey. Return positive ** if unknown. ** @@ -5790,12 +5926,11 @@ int sqlite3BtreeTableMoveto( ** a positive value as that will cause the optimization to be skipped. */ static int indexCellCompare( - BtCursor *pCur, + MemPage *pPage, int idx, UnpackedRecord *pIdxKey, RecordCompare xRecordCompare ){ - MemPage *pPage = pCur->pPage; int c; int nCell; /* Size of the pCell cell in bytes */ u8 *pCell = findCellPastPtr(pPage, idx); @@ -5807,10 +5942,10 @@ static int indexCellCompare( ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); - }else if( !(pCell[1] & 0x80) + }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ - /* The record-size field is a 2 byte varint and the record + /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); @@ -5845,7 +5980,7 @@ static int cursorOnLastPage(BtCursor *pCur){ ** before or after the key. ** ** An integer is written into *pRes which is the result of -** comparing the key with the entry to which the cursor is +** comparing the key with the entry to which the cursor is ** pointing. The meaning of the integer written into ** *pRes is as follows: ** @@ -5860,7 +5995,7 @@ static int cursorOnLastPage(BtCursor *pCur){ ** is larger than pIdxKey. ** ** The pIdxKey->eqSeen field is set to 1 if there -** exists an entry in the table that exactly matches pIdxKey. +** exists an entry in the table that exactly matches pIdxKey. */ int sqlite3BtreeIndexMoveto( BtCursor *pCur, /* The cursor to be moved */ @@ -5881,8 +6016,8 @@ int sqlite3BtreeIndexMoveto( xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); pIdxKey->errCode = 0; - assert( pIdxKey->default_rc==1 - || pIdxKey->default_rc==0 + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 ); @@ -5904,17 +6039,17 @@ int sqlite3BtreeIndexMoveto( ){ int c; if( pCur->ix==pCur->pPage->nCell-1 - && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0 + && (c = indexCellCompare(pCur->pPage,pCur->ix,pIdxKey,xRecordCompare))<=0 && pIdxKey->errCode==SQLITE_OK ){ *pRes = c; return SQLITE_OK; /* Cursor already pointing at the correct spot */ } - if( pCur->iPage>0 - && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 + if( pCur->iPage>0 + && indexCellCompare(pCur->pPage, 0, pIdxKey, xRecordCompare)<=0 && pIdxKey->errCode==SQLITE_OK ){ - pCur->curFlags &= ~BTCF_ValidOvfl; + pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); if( !pCur->pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } @@ -5964,9 +6099,9 @@ int sqlite3BtreeIndexMoveto( /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree ** page is less than 16384 bytes and may be stored as a 2-byte - ** varint. This information is used to attempt to avoid parsing - ** the entire cell by checking for the cases where the record is - ** stored entirely within the b-tree page by inspecting the first + ** varint. This information is used to attempt to avoid parsing + ** the entire cell by checking for the cases where the record is + ** stored entirely within the b-tree page by inspecting the first ** 2 bytes of the cell. */ nCell = pCell[0]; @@ -5976,10 +6111,10 @@ int sqlite3BtreeIndexMoveto( ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); - }else if( !(pCell[1] & 0x80) + }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ - /* The record-size field is a 2 byte varint and the record + /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); @@ -5987,10 +6122,10 @@ int sqlite3BtreeIndexMoveto( /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. + ** buffer before VdbeRecordCompare() can be called. ** ** If the record is corrupt, the xRecordCompare routine may read - ** up to two varints past the end of the buffer. An extra 18 + ** up to two varints past the end of the buffer. An extra 18 ** bytes of padding is allocated at the end of the buffer in ** case this happens. */ void *pCellKey; @@ -6006,7 +6141,7 @@ int sqlite3BtreeIndexMoveto( rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_index_finish; } - pCellKey = sqlite3Malloc( nCell+nOverrun ); + pCellKey = sqlite3Malloc( (u64)nCell+(u64)nOverrun ); if( pCellKey==0 ){ rc = SQLITE_NOMEM_BKPT; goto moveto_index_finish; @@ -6022,7 +6157,7 @@ int sqlite3BtreeIndexMoveto( c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } - assert( + assert( (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) ); @@ -6056,10 +6191,36 @@ int sqlite3BtreeIndexMoveto( }else{ chldPg = get4byte(findCell(pPage, lwr)); } - pCur->ix = (u16)lwr; - rc = moveToChild(pCur, chldPg); - if( rc ) break; - } + + /* This block is similar to an in-lined version of: + ** + ** pCur->ix = (u16)lwr; + ** rc = moveToChild(pCur, chldPg); + ** if( rc ) break; + */ + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + pCur->aiIdx[pCur->iPage] = (u16)lwr; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; + pCur->iPage++; + rc = getAndInitPage(pCur->pBt, chldPg, &pCur->pPage, pCur->curPagerFlags); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(chldPg); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + break; + } + /* + ***** End of in-lined moveToChild() call */ + } moveto_index_finish: pCur->info.nSize = 0; assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); @@ -6084,7 +6245,7 @@ int sqlite3BtreeEof(BtCursor *pCur){ /* ** Return an estimate for the number of rows in the table that pCur is -** pointing to. Return a negative number if no estimate is currently +** pointing to. Return a negative number if no estimate is currently ** available. */ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ @@ -6094,21 +6255,21 @@ i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - /* Currently this interface is only called by the OP_IfSmaller - ** opcode, and it that case the cursor will always be valid and - ** will always point to a leaf node. */ - if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; if( NEVER(pCur->pPage->leaf==0) ) return -1; n = pCur->pPage->nCell; for(i=0; iiPage; i++){ - n *= pCur->apPage[i]->nCell; + n *= pCur->apPage[i]->nCell+1; } return n; } /* -** Advance the cursor to the next entry in the database. +** Advance the cursor to the next entry in the database. ** Return value: ** ** SQLITE_OK success @@ -6243,7 +6404,10 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ } pPage = pCur->pPage; - assert( pPage->isInit ); + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } if( !pPage->leaf ){ int idx = pCur->ix; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); @@ -6297,7 +6461,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int flags){ ** SQLITE_OK is returned on success. Any other return value indicates ** an error. *ppPage is set to NULL in the event of an error. ** -** If the "nearby" parameter is not 0, then an effort is made to +** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an ** attempt to keep related pages close to each other in the database file, ** which in turn can make database access faster. @@ -6339,7 +6503,7 @@ static int allocateBtreePage( Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ u32 nSearch = 0; /* Count of the number of search attempts */ - + /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. @@ -6402,8 +6566,8 @@ static int allocateBtreePage( ** is the number of leaf page pointers to follow. */ k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ - /* The trunk has no leaves and the list is not being searched. - ** So extract the trunk page itself and use it as the newly + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly ** allocated page */ assert( pPrevTrunk==0 ); rc = sqlite3PagerWrite(pTrunk->pDbPage); @@ -6420,8 +6584,8 @@ static int allocateBtreePage( rc = SQLITE_CORRUPT_PGNO(iTrunk); goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList - && (nearby==iTrunk || (iTrunkaData[0], &pTrunk->aData[0], 4); } }else{ - /* The trunk page is required by the caller but it contains + /* The trunk page is required by the caller but it contains ** pointers to free-list leaves. The first leaf becomes a trunk ** page in this case. */ MemPage *pNewTrunk; Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); - if( iNewTrunk>mxPage ){ + if( iNewTrunk>mxPage ){ rc = SQLITE_CORRUPT_PGNO(iTrunk); goto end_allocate_page; } @@ -6520,8 +6684,8 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iPage==mxPage ); - if( !searchList - || (iPage==nearby || (iPagepPage1; /* Local reference to page 1 */ MemPage *pPage; /* Page being freed. May be NULL. */ int rc; /* Return Code */ @@ -6740,7 +6904,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If control flows to this point, then it was not possible to add the ** the page being freed as a leaf page of the first trunk in the free-list. - ** Possibly because the free-list is empty, or possibly because the + ** Possibly because the free-list is empty, or possibly because the ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ @@ -6797,15 +6961,15 @@ static SQLITE_NOINLINE int clearCellOverflow( assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; - assert( nOvfl>0 || + assert( nOvfl>0 || (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)btreePagecount(pBt) ){ - /* 0 is not a legal page number and page 1 cannot be an - ** overflow page. Therefore if ovflPgno<2 or past the end of the + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the ** file the database must be corrupt. */ return SQLITE_CORRUPT_BKPT; } @@ -6817,11 +6981,11 @@ static SQLITE_NOINLINE int clearCellOverflow( if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 ){ - /* There is no reason any cursor should have an outstanding reference + /* There is no reason any cursor should have an outstanding reference ** to an overflow page belonging to a cell that is being deleted/updated. - ** So if there exists more than one reference to this page, then it - ** must not really be an overflow page and the database must be corrupt. - ** It is helpful to detect this before calling freePage2(), as + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as ** freePage2() may zero the page contents if secure-delete mode is ** enabled. If this 'overflow' page happens to be a page that the ** caller is iterating through or using in some other way, this @@ -6843,7 +7007,7 @@ static SQLITE_NOINLINE int clearCellOverflow( /* Call xParseCell to compute the size of a cell. If the cell contains ** overflow, then invoke cellClearOverflow to clear out that overflow. -** STore the result code (SQLITE_OK or some error code) in rc. +** Store the result code (SQLITE_OK or some error code) in rc. ** ** Implemented as macro to force inlining for performance. */ @@ -6907,7 +7071,7 @@ static int fillInCell( pSrc = pX->pKey; nHeader += putVarint32(&pCell[nHeader], nPayload); } - + /* Fill in the payload */ pPayload = &pCell[nHeader]; if( nPayload<=pPage->maxLocal ){ @@ -6916,7 +7080,10 @@ static int fillInCell( n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; assert( nSrc<=nPayload ); testcase( nSrcautoVacuum ){ do{ pgnoOvfl++; - } while( - PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) + } while( + PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) ); } #endif @@ -7007,9 +7174,9 @@ static int fillInCell( #ifndef SQLITE_OMIT_AUTOVACUUM /* If the database supports auto-vacuum, and the second or subsequent ** overflow page is being allocated, add an entry to the pointer-map - ** for that page now. + ** for that page now. ** - ** If this is the first overflow page, then write a partial entry + ** If this is the first overflow page, then write a partial entry ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() ** may misinterpret the uninitialized values and delete the @@ -7110,8 +7277,8 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** will not fit, then make a copy of the cell content into pTemp if ** pTemp is not null. Regardless of pTemp, allocate a new entry ** in pPage->apOvfl[] and make it point to the cell content (either -** in pTemp or the original pCell) and also record its index. -** Allocating a new entry in pPage->aCell[] implies that +** in pTemp or the original pCell) and also record its index. +** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. ** ** The insertCellFast() routine below works exactly the same as @@ -7423,16 +7590,16 @@ static u16 cachedCellSize(CellArray *p, int N){ } /* -** Array apCell[] contains pointers to nCell b-tree page cells. The +** Array apCell[] contains pointers to nCell b-tree page cells. The ** szCell[] array contains the size in bytes of each cell. This function ** replaces the current contents of page pPg with the contents of the cell ** array. ** ** Some of the cells in apCell[] may currently be stored in pPg. This -** function works around problems caused by this by making a copy of any +** function works around problems caused by this by making a copy of any ** such cells before overwriting the page data. ** -** The MemPage.nFree field is invalidated by this function. It is the +** The MemPage.nFree field is invalidated by this function. It is the ** responsibility of the caller to set it correctly. */ static int rebuildPage( @@ -7454,12 +7621,14 @@ static int rebuildPage( int k; /* Current slot in pCArray->apEnd[] */ u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ + assert( nCell>0 ); assert( i(u32)usableSize) ){ j = 0; } + if( j>(u32)usableSize ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); - for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i; k++){} pSrcEnd = pCArray->apEnd[k]; pData = pEnd; @@ -7491,7 +7660,8 @@ static int rebuildPage( } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ - pPg->nCell = nCell; + assert( nCell < 10922 ); + pPg->nCell = (u16)nCell; pPg->nOverflow = 0; put2byte(&aData[hdr+1], 0); @@ -7514,7 +7684,7 @@ static int rebuildPage( ** cell in the array. It is the responsibility of the caller to ensure ** that it is safe to overwrite this part of the cell-pointer array. ** -** When this function is called, *ppData points to the start of the +** When this function is called, *ppData points to the start of the ** content area on page pPg. If the size of the content area is extended, ** *ppData is updated to point to the new start of the content area ** before returning. @@ -7522,7 +7692,7 @@ static int rebuildPage( ** Finally, argument pBegin points to the byte immediately following the ** end of the space required by this page for the cell-pointer area (for ** all cells - not just those inserted by the current call). If the content -** area must be extended to before this point in order to accomodate all +** area must be extended to before this point in order to accommodate all ** cells in apCell[], then the cells do not fit and non-zero is returned. */ static int pageInsertArray( @@ -7542,7 +7712,8 @@ static int pageInsertArray( u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ if( iEnd<=iFirst ) return 0; - for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i ; k++){} pEnd = pCArray->apEnd[k]; while( 1 /*Exit by break*/ ){ int sz, rc; @@ -7737,9 +7908,13 @@ static int editPage( if( pageInsertArray( pPg, pBegin, &pData, pCellptr, iNew+nCell, nNew-nCell, pCArray - ) ) goto editpage_fail; + ) + ){ + goto editpage_fail; + } - pPg->nCell = nNew; + assert( nNew < 10922 ); + pPg->nCell = (u16)nNew; pPg->nOverflow = 0; put2byte(&aData[hdr+3], pPg->nCell); @@ -7760,6 +7935,7 @@ static int editPage( return SQLITE_OK; editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ + if( nNew<1 ) return SQLITE_CORRUPT_BKPT; populateCellCache(pCArray, iNew, nNew); return rebuildPage(pCArray, iNew, nNew, pPg); } @@ -7798,12 +7974,12 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); - + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ assert( pPage->nFree>=0 ); assert( pParent->nFree>=0 ); - /* Allocate a new page. This page will become the right-sibling of + /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell ** may be inserted. If both these operations are successful, proceed. */ @@ -7826,6 +8002,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ b.szCell = &szCell; b.apEnd[0] = pPage->aDataEnd; b.ixNx[0] = 2; + b.ixNx[NB*2-1] = 0x7fffffff; rc = rebuildPage(&b, 0, 1, pNew); if( NEVER(rc) ){ releasePage(pNew); @@ -7834,10 +8011,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map - ** with entries for the new page, and any pointer from the + ** with entries for the new page, and any pointer from the ** cell on the page to an overflow page. If either of these ** operations fails, the return code is set, but the contents - ** of the parent page are still manipulated by thh code below. + ** of the parent page are still manipulated by the code below. ** That is Ok, at this point the parent page is guaranteed to ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. @@ -7848,14 +8025,14 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); } } - + /* Create a divider cell to insert into pParent. The divider cell ** consists of a 4-byte page number (the page number of pPage) and ** a variable length key value (which must be the same value as the ** largest key on pPage). ** - ** To find the largest key value on pPage, first find the right-most - ** cell on pPage. The first two fields of this cell are the + ** To find the largest key value on pPage, first find the right-most + ** cell on pPage. The first two fields of this cell are the ** record-length (a variable length integer at most 32-bits in size) ** and the key value (a variable length integer, may have any value). ** The first of the while(...) loops below skips over the record-length @@ -7876,7 +8053,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); - + /* Release the reference to the new page. */ releasePage(pNew); } @@ -7888,7 +8065,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ #if 0 /* ** This function does not contribute anything to the operation of SQLite. -** it is sometimes activated temporarily while debugging code responsible +** it is sometimes activated temporarily while debugging code responsible ** for setting pointer-map entries. */ static int ptrmapCheckPages(MemPage **apPage, int nPage){ @@ -7903,7 +8080,7 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ for(j=0; jnCell; j++){ CellInfo info; u8 *z; - + z = findCell(pPage, j); pPage->xParseCell(pPage, z, &info); if( info.nLocalpgno==1) ? 100 : 0); int rc; int iData; - - + + assert( pFrom->isInit ); assert( pFrom->nFree>=iToHdr ); assert( get2byte(&aFrom[iFromHdr+5]) <= (int)pBt->usableSize ); - + /* Copy the b-tree node content from page pFrom to page pTo. */ iData = get2byte(&aFrom[iFromHdr+5]); memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); - + /* Reinitialize page pTo so that the contents of the MemPage structure ** match the new data. The initialization of pTo can actually fail under - ** fairly obscure circumstances, even though it is a copy of initialized + ** fairly obscure circumstances, even though it is a copy of initialized ** page pFrom. */ pTo->isInit = 0; @@ -7976,7 +8153,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ *pRC = rc; return; } - + /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ @@ -7991,13 +8168,13 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** (hereafter "the page") and up to 2 siblings so that all pages have about the ** same amount of free space. Usually a single sibling on either side of the ** page are used in the balancing, though both siblings might come from one -** side if the page is the first or last child of its parent. If the page +** side if the page is the first or last child of its parent. If the page ** has fewer than 2 siblings (something which can only happen if the page ** is a root page or a child of a root page) then all available siblings ** participate in the balancing. ** -** The number of siblings of the page might be increased or decreased by -** one or two in an effort to keep pages nearly full but not over full. +** The number of siblings of the page might be increased or decreased by +** one or two in an effort to keep pages nearly full but not over full. ** ** Note that when this routine is called, some of the cells on the page ** might not actually be stored in MemPage.aData[]. This can happen @@ -8008,7 +8185,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** inserted into or removed from the parent page (pParent). Doing so ** may cause the parent page to become overfull or underfull. If this ** happens, it is the responsibility of the caller to invoke the correct -** balancing routine to fix this problem (see the balance() routine). +** balancing routine to fix this problem (see the balance() routine). ** ** If this routine fails for any reason, it might leave the database ** in a corrupted state. So if this routine fails, the database should @@ -8023,7 +8200,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** of the page-size, the aOvflSpace[] buffer is guaranteed to be large ** enough for all overflow cells. ** -** If aOvflSpace is set to a null pointer, this function returns +** If aOvflSpace is set to a null pointer, this function returns ** SQLITE_NOMEM. */ static int balance_nonroot( @@ -8046,7 +8223,7 @@ static int balance_nonroot( int pageFlags; /* Value of pPage->aData[0] */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ - int szScratch; /* Size of scratch memory requested */ + u64 szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ @@ -8061,13 +8238,15 @@ static int balance_nonroot( CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); - memset(&b, 0, sizeof(b)); + assert( sizeof(b) - sizeof(b.ixNx) == offsetof(CellArray,ixNx) ); + memset(&b, 0, sizeof(b)-sizeof(b.ixNx[0])); + b.ixNx[NB*2-1] = 0x7fffffff; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); /* At this point pParent may have at most one overflow cell. And if - ** this overflow cell is present, it must be the cell with + ** this overflow cell is present, it must be the cell with ** index iParentIdx. This scenario comes about when this function ** is called (indirectly) from sqlite3BtreeDelete(). */ @@ -8079,11 +8258,11 @@ static int balance_nonroot( } assert( pParent->nFree>=0 ); - /* Find the sibling pages to balance. Also locate the cells in pParent - ** that divide the siblings. An attempt is made to find NN siblings on - ** either side of pPage. More siblings are taken from one side, however, + /* Find the sibling pages to balance. Also locate the cells in pParent + ** that divide the siblings. An attempt is made to find NN siblings on + ** either side of pPage. More siblings are taken from one side, however, ** if there are fewer than NN siblings on the other side. If pParent - ** has NB or fewer children then all children of pParent are taken. + ** has NB or fewer children then all children of pParent are taken. ** ** This loop also drops the divider cells from the parent page. This ** way, the remainder of the function does not have to deal with any @@ -8095,7 +8274,7 @@ static int balance_nonroot( nxDiv = 0; }else{ assert( bBulk==0 || bBulk==1 ); - if( iParentIdx==0 ){ + if( iParentIdx==0 ){ nxDiv = 0; }else if( iParentIdx==i ){ nxDiv = i-2+bBulk; @@ -8113,7 +8292,7 @@ static int balance_nonroot( pgno = get4byte(pRight); while( 1 ){ if( rc==SQLITE_OK ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); } if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); @@ -8144,7 +8323,7 @@ static int balance_nonroot( ** This is safe because dropping a cell only overwrites the first ** four bytes of it, and this function does not need the first ** four bytes of the divider cell. So the pointer is safe to use - ** later on. + ** later on. ** ** But not if we are in secure-delete mode. In secure-delete mode, ** the dropCell() routine will overwrite the entire cell with zeroes. @@ -8220,7 +8399,7 @@ static int balance_nonroot( ** table-interior, index-leaf, or index-interior). */ if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } @@ -8244,7 +8423,7 @@ static int balance_nonroot( memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ if( NEVER(limitaiOvfl[0]) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } limit = pOld->aiOvfl[0]; @@ -8307,7 +8486,7 @@ static int balance_nonroot( ** Figure out the number of pages needed to hold all b.nCell cells. ** Store this number in "k". Also compute szNew[] which is the total ** size of all cells on the i-th page and cntNew[] which is the index - ** in b.apCell[] of the cell that divides page i from page i+1. + ** in b.apCell[] of the cell that divides page i from page i+1. ** cntNew[k] should equal b.nCell. ** ** Values computed by this block: @@ -8317,7 +8496,7 @@ static int balance_nonroot( ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to ** the right of the i-th sibling page. ** usableSpace: Number of bytes of space available on each sibling. - ** + ** */ usableSpace = pBt->usableSize - 12 + leafCorrection; for(i=k=0; i0), or ** (2) pPage is a virtual root page. A virtual root page is when @@ -8478,15 +8657,15 @@ static int balance_nonroot( } /* - ** Reassign page numbers so that the new pages are in ascending order. + ** Reassign page numbers so that the new pages are in ascending order. ** This helps to keep entries in the disk file in order so that a scan - ** of the table is closer to a linear scan through the file. That in turn + ** of the table is closer to a linear scan through the file. That in turn ** helps the operating system to deliver pages from the disk more rapidly. ** ** An O(N*N) sort algorithm is used, but since N is never more than NB+2 ** (5), that is not a performance concern. ** - ** When NB==3, this one optimization makes the database about 25% faster + ** When NB==3, this one optimization makes the database about 25% faster ** for large insertions and deletions. */ for(i=0; ipgno); /* If the sibling pages are not leaves, ensure that the right-child pointer - ** of the right-most new sibling page is set to the value that was + ** of the right-most new sibling page is set to the value that was ** originally in the same field of the right-most old sibling page. */ if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ - MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + MemPage *pOld; + if( nNew>nOld ){ + pOld = apNew[nOld-1]; + }else{ + pOld = apOld[nOld-1]; + } memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); } - /* Make any required updates to pointer map entries associated with + /* Make any required updates to pointer map entries associated with ** cells stored on sibling pages following the balance operation. Pointer ** map entries associated with divider cells are set by the insertCell() ** routine. The associated pointer map entries are: @@ -8556,9 +8740,9 @@ static int balance_nonroot( ** b) if the sibling pages are not leaves, the child page associated ** with the cell. ** - ** If the sibling pages are not leaves, then the pointer map entry - ** associated with the right-child of each sibling may also need to be - ** updated. This happens below, after the sibling pages have been + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been ** populated, not here. */ if( ISAUTOVACUUM(pBt) ){ @@ -8583,7 +8767,7 @@ static int balance_nonroot( } /* Cell pCell is destined for new sibling page pNew. Originally, it - ** was either part of sibling page iOld (possibly an overflow cell), + ** was either part of sibling page iOld (possibly an overflow cell), ** or else the divider cell to the left of sibling page iOld. So, ** if sibling page iOld had the same page number as pNew, and if ** pCell really was a part of sibling page iOld (not a divider or @@ -8620,9 +8804,9 @@ static int balance_nonroot( if( !pNew->leaf ){ memcpy(&pNew->aData[8], pCell, 4); }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in b.apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in b.apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of ** the sibling-page assembled above only. */ CellInfo info; @@ -8635,9 +8819,9 @@ static int balance_nonroot( pCell -= 4; /* Obscure case for non-leaf-data trees: If the cell at pCell was ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this + ** bytes, then it may actually be smaller than this ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to + ** any cell). But it is important to pass the correct size to ** insertCell(), so reparse the cell now. ** ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" @@ -8652,9 +8836,10 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - for(k=0; b.ixNx[k]<=j && ALWAYS(kj ); + for(k=0; b.ixNx[k]<=j; k++){} pSrcEnd = b.apEnd[k]; - if( SQLITE_WITHIN(pSrcEnd, pCell, pCell+sz) ){ + if( SQLITE_OVERFLOW(pSrcEnd, pCell, pCell+sz) ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } @@ -8688,6 +8873,8 @@ static int balance_nonroot( for(i=1-nNew; i=0 && iPg=1 || i>=0 ); + assert( iPg=0 /* On the upwards pass, or... */ || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ @@ -8735,8 +8922,8 @@ static int balance_nonroot( ** b-tree structure by one. This is described as the "balance-shallower" ** sub-algorithm in some documentation. ** - ** If this is an auto-vacuum database, the call to copyNodeContent() - ** sets all pointer-map entries corresponding to database image pages + ** If this is an auto-vacuum database, the call to copyNodeContent() + ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** ** It is critical that the child page be defragmented before being @@ -8747,7 +8934,7 @@ static int balance_nonroot( assert( nNew==1 || CORRUPT_DB ); rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); - assert( apNew[0]->nFree == + assert( apNew[0]->nFree == (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset - apNew[0]->nCell*2) || rc!=SQLITE_OK @@ -8777,7 +8964,7 @@ static int balance_nonroot( #if 0 if( ISAUTOVACUUM(pBt) && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that - ** all pointer map pages are set correctly. This is helpful while + ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); @@ -8807,15 +8994,15 @@ static int balance_nonroot( ** ** A new child page is allocated and the contents of the current root ** page, including overflow cells, are copied into the child. The root -** page is then overwritten to make it an empty page with the right-child +** page is then overwritten to make it an empty page with the right-child ** pointer pointing to the new page. ** -** Before returning, all pointer-map entries corresponding to pages +** Before returning, all pointer-map entries corresponding to pages ** that the new child-page now contains pointers to are updated. The ** entry corresponding to the new right-child pointer of the root ** page is also updated. ** -** If successful, *ppChild is set to contain a reference to the child +** If successful, *ppChild is set to contain a reference to the child ** page and SQLITE_OK is returned. In this case the caller is required ** to call releasePage() on *ppChild exactly once. If an error occurs, ** an error code is returned and *ppChild is set to 0. @@ -8829,7 +9016,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ assert( pRoot->nOverflow>0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - /* Make pRoot, the root page of the b-tree, writable. Allocate a new + /* Make pRoot, the root page of the b-tree, writable. Allocate a new ** page that will become the new right-child of pPage. Copy the contents ** of the node stored on pRoot into the new child page. */ @@ -8885,7 +9072,7 @@ static int anotherValidCursor(BtCursor *pCur){ && pOther->eState==CURSOR_VALID && pOther->pPage==pCur->pPage ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pCur->pPage); } } return SQLITE_OK; @@ -8894,7 +9081,7 @@ static int anotherValidCursor(BtCursor *pCur){ /* ** The page that pCur currently points to has just been modified in ** some way. This function figures out if this modification means the -** tree needs to be balanced, and if so calls the appropriate balancing +** tree needs to be balanced, and if so calls the appropriate balancing ** routine. Balancing routines are: ** ** balance_quick() @@ -8926,7 +9113,7 @@ static int balance(BtCursor *pCur){ ** balance_deeper() function to create a new child for the root-page ** and copy the current contents of the root-page to it. The ** next iteration of the do-loop will balance the child page. - */ + */ assert( balance_deeper_called==0 ); VVA_ONLY( balance_deeper_called++ ); rc = balance_deeper(pPage, &pCur->apPage[1]); @@ -8943,9 +9130,9 @@ static int balance(BtCursor *pCur){ } }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){ /* The page being written is not a root page, and there is currently - ** more than one reference to it. This only happens if the page is one + ** more than one reference to it. This only happens if the page is one ** of its own ancestor pages. Corruption. */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; @@ -8965,17 +9152,17 @@ static int balance(BtCursor *pCur){ /* Call balance_quick() to create a new sibling of pPage on which ** to store the overflow cell. balance_quick() inserts a new cell ** into pParent, which may cause pParent overflow. If this - ** happens, the next iteration of the do-loop will balance pParent + ** happens, the next iteration of the do-loop will balance pParent ** use either balance_nonroot() or balance_deeper(). Until this ** happens, the overflow cell is stored in the aBalanceQuickSpace[] - ** buffer. + ** buffer. ** ** The purpose of the following assert() is to check that only a ** single call to balance_quick() is made for each call to this ** function. If this were not verified, a subtle bug involving reuse ** of the aBalanceQuickSpace[] might sneak in. */ - assert( balance_quick_called==0 ); + assert( balance_quick_called==0 ); VVA_ONLY( balance_quick_called++ ); rc = balance_quick(pParent, pPage, aBalanceQuickSpace); }else @@ -8986,15 +9173,15 @@ static int balance(BtCursor *pCur){ ** modifying the contents of pParent, which may cause pParent to ** become overfull or underfull. The next iteration of the do-loop ** will balance the parent page to correct this. - ** + ** ** If the parent page becomes overfull, the overflow cell or cells - ** are stored in the pSpace buffer allocated immediately below. + ** are stored in the pSpace buffer allocated immediately below. ** A subsequent iteration of the do-loop will deal with this by ** calling balance_nonroot() (balance_deeper() may be called first, ** but it doesn't deal with overflow cells - just moves them to a - ** different page). Once this subsequent call to balance_nonroot() + ** different page). Once this subsequent call to balance_nonroot() ** has completed, it is safe to release the pSpace buffer used by - ** the previous call, as the overflow cell data will have been + ** the previous call, as the overflow cell data will have been ** copied either into the body of a database page or into the new ** pSpace buffer passed to the latter call to balance_nonroot(). */ @@ -9002,9 +9189,9 @@ static int balance(BtCursor *pCur){ rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, pCur->hints&BTREE_BULKLOAD); if( pFree ){ - /* If pFree is not NULL, it points to the pSpace buffer used + /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are - ** now stored either on real database pages or within the + ** now stored either on real database pages or within the ** new pSpace buffer, so it may be safely freed here. */ sqlite3PageFree(pFree); } @@ -9044,7 +9231,7 @@ static int btreeOverwriteContent( ){ int nData = pX->nData - iOffset; if( nData<=0 ){ - /* Overwritting with zeros */ + /* Overwriting with zeros */ int i; for(i=0; ipData to write */ @@ -9109,7 +9296,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -9123,7 +9310,7 @@ static SQLITE_NOINLINE int btreeOverwriteOverflowCell( if( rc ) return rc; iOffset += ovflPageSize; }while( iOffsetinfo.pPayload + pCur->info.nLocal > pPage->aDataEnd || pCur->info.pPayload < pPage->aData + pPage->cellOffset ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCur->info.nLocal==nTotal ){ /* The entire cell is local */ @@ -9162,7 +9349,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ ** hold the content of the row. ** ** For an index btree (used for indexes and WITHOUT ROWID tables), the -** key is an arbitrary byte sequence stored in pX.pKey,nKey. The +** key is an arbitrary byte sequence stored in pX.pKey,nKey. The ** pX.pData,nData,nZero fields must be zero. ** ** If the seekResult parameter is non-zero, then a successful call to @@ -9202,8 +9389,8 @@ int sqlite3BtreeInsert( ** ** In some cases, the call to btreeMoveto() below is a no-op. For ** example, when inserting data into a table with auto-generated integer - ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the - ** integer key to use. It then calls this function to actually insert the + ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the + ** integer key to use. It then calls this function to actually insert the ** data into the intkey B-Tree. In this case btreeMoveto() recognizes ** that the cursor is already where it needs to be and returns without ** doing any work. To avoid thwarting these optimizations, it is important @@ -9218,7 +9405,7 @@ int sqlite3BtreeInsert( ** Which can only happen if the SQLITE_NoSchemaError flag was set when ** the schema was loaded. This cannot be asserted though, as a user might ** set the flag, load the schema, and then unset the flag. */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } @@ -9247,13 +9434,13 @@ int sqlite3BtreeInsert( if( pCur->pKeyInfo==0 ){ assert( pX->pKey==0 ); - /* If this is an insert into a table b-tree, invalidate any incrblob + /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ if( p->hasIncrblobCur ){ invalidateIncrblobCursors(p, pCur->pgnoRoot, pX->nKey, 0); } - /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing ** to a row with the same key as the new entry being inserted. */ #ifdef SQLITE_DEBUG @@ -9284,14 +9471,14 @@ int sqlite3BtreeInsert( ** to an adjacent cell. Move the cursor so that it is pointing either ** to the cell to be overwritten or an adjacent cell. */ - rc = sqlite3BtreeTableMoveto(pCur, pX->nKey, + rc = sqlite3BtreeTableMoveto(pCur, pX->nKey, (flags & BTREE_APPEND)!=0, &loc); if( rc ) return rc; } }else{ /* This is an index or a WITHOUT ROWID table */ - /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing ** to a row with the same key as the new entry being inserted. */ assert( (flags & BTREE_SAVEPOSITION)==0 || loc==0 ); @@ -9311,7 +9498,7 @@ int sqlite3BtreeInsert( r.eqSeen = 0; rc = sqlite3BtreeIndexMoveto(pCur, &r, &loc); }else{ - rc = btreeMoveto(pCur, pX->pKey, pX->nKey, + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, (flags & BTREE_APPEND)!=0, &loc); } if( rc ) return rc; @@ -9326,13 +9513,13 @@ int sqlite3BtreeInsert( if( pCur->info.nKey==pX->nKey ){ BtreePayload x2; x2.pData = pX->pKey; - x2.nData = pX->nKey; + x2.nData = (int)pX->nKey; assert( pX->nKey<=0x7fffffff ); x2.nZero = 0; return btreeOverwriteCell(pCur, &x2); } } } - assert( pCur->eState==CURSOR_VALID + assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB ); pPage = pCur->pPage; @@ -9341,7 +9528,7 @@ int sqlite3BtreeInsert( if( pPage->nFree<0 ){ if( NEVER(pCur->eState>CURSOR_INVALID) ){ /* ^^^^^--- due to the moveToRoot() call above */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ rc = btreeComputeFreeSpace(pPage); } @@ -9358,7 +9545,10 @@ int sqlite3BtreeInsert( if( flags & BTREE_PREFORMAT ){ rc = SQLITE_OK; szNew = p->pBt->nPreformatSize; - if( szNew<4 ) szNew = 4; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ CellInfo info; pPage->xParseCell(pPage, newCell, &info); @@ -9380,7 +9570,7 @@ int sqlite3BtreeInsert( CellInfo info; assert( idx>=0 ); if( idx>=pPage->nCell ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -9393,24 +9583,24 @@ int sqlite3BtreeInsert( BTREE_CLEAR_CELL(rc, pPage, oldCell, info); testcase( pCur->curFlags & BTCF_ValidOvfl ); invalidateOverflowCache(pCur); - if( info.nSize==szNew && info.nLocal==info.nPayload + if( info.nSize==szNew && info.nLocal==info.nPayload && (!ISAUTOVACUUM(p->pBt) || szNewminLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add ** the leftover space to the free list. But experiments show that ** doing that is no faster then skipping this optimization and just - ** calling dropCell() and insertCell(). + ** calling dropCell() and insertCell(). ** ** This optimization cannot be used on an autovacuum database if the ** new entry uses overflow pages, as the insertCell() call below is ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( oldCell+szNew > pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } memcpy(oldCell, newCell, szNew); return SQLITE_OK; @@ -9420,7 +9610,7 @@ int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; - pCur->curFlags &= ~BTCF_ValidNKey; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } @@ -9428,7 +9618,7 @@ int sqlite3BtreeInsert( assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); - /* If no error has occurred and pPage has an overflow cell, call balance() + /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey ** variables. @@ -9450,11 +9640,11 @@ int sqlite3BtreeInsert( */ if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); - pCur->curFlags &= ~(BTCF_ValidNKey); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() - ** fails. Internal data structure corruption will result otherwise. + ** fails. Internal data structure corruption will result otherwise. ** Also, set the cursor state to invalid. This stops saveCursorPosition() ** from trying to save the current position of the cursor. */ pCur->pPage->nOverflow = 0; @@ -9504,7 +9694,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ getCellInfo(pSrc); if( pSrc->info.nPayload<0x80 ){ - *(aOut++) = pSrc->info.nPayload; + *(aOut++) = (u8)pSrc->info.nPayload; }else{ aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload); } @@ -9512,12 +9702,12 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ nIn = pSrc->info.nLocal; aIn = pSrc->info.pPayload; if( aIn+nIn>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ memcpy(aOut, aIn, nIn); - pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + pBt->nPreformatSize = nIn + (int)(aOut - pBt->pTmpSpace); return SQLITE_OK; }else{ int rc = SQLITE_OK; @@ -9529,19 +9719,19 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ u32 nOut; /* Size of output buffer aOut[] */ nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); - pBt->nPreformatSize = nOut + (aOut - pBt->pTmpSpace); + pBt->nPreformatSize = (int)nOut + (int)(aOut - pBt->pTmpSpace); if( nOutinfo.nPayload ){ pPgnoOut = &aOut[nOut]; pBt->nPreformatSize += 4; } - + if( nRem>nIn ){ if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pSrc->pPage); } ovflIn = get4byte(&pSrc->info.pPayload[nIn]); } - + do { nRem -= nOut; do{ @@ -9566,7 +9756,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ } } }while( rc==SQLITE_OK && nOut>0 ); - + if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ Pgno pgnoNew; MemPage *pNew = 0; @@ -9585,7 +9775,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ } } }while( nRem>0 && rc==SQLITE_OK ); - + releasePage(pPageOut); sqlite3PagerUnref(pPageIn); return rc; @@ -9593,7 +9783,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ } /* -** Delete the entry that the cursor is pointing to. +** Delete the entry that the cursor is pointing to. ** ** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then ** the cursor is left pointing at an arbitrary location after the delete. @@ -9611,12 +9801,12 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ */ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; + BtShared *pBt = p->pBt; int rc; /* Return code */ MemPage *pPage; /* Page to delete cell from */ unsigned char *pCell; /* Pointer to cell to delete */ int iCellIdx; /* Index of cell to delete */ - int iCellDepth; /* Depth of node containing pCell */ + int iCellDepth; /* Depth of node containing pCell */ CellInfo info; /* Size of the cell being deleted */ u8 bPreserve; /* Keep cursor valid. 2 for CURSOR_SKIPNEXT */ @@ -9633,7 +9823,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); if( rc || pCur->eState!=CURSOR_VALID ) return rc; }else{ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); } } assert( pCur->eState==CURSOR_VALID ); @@ -9642,21 +9832,21 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ iCellIdx = pCur->ix; pPage = pCur->pPage; if( pPage->nCell<=iCellIdx ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } pCell = findCell(pPage, iCellIdx); if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } if( pCell<&pPage->aCellIdx[pPage->nCell] ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must ** be preserved following this delete operation. If the current delete ** will cause a b-tree rebalance, then this is done by saving the cursor - ** key and leaving the cursor in CURSOR_REQUIRESEEK state before - ** returning. + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. ** ** If the current delete will not cause a rebalance, then the cursor ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately @@ -9670,7 +9860,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ */ bPreserve = (flags & BTREE_SAVEPOSITION)!=0; if( bPreserve ){ - if( !pPage->leaf + if( !pPage->leaf || (pPage->nFree+pPage->xCellSize(pPage,pCell)+2) > (int)(pBt->usableSize*2/3) || pPage->nCell==1 /* See dbfuzz001.test for a test case */ @@ -9740,7 +9930,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ n = pCur->pPage->pgno; } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; @@ -9766,7 +9956,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** on the leaf node first. If the balance proceeds far enough up the ** tree that we can be sure that any problem in the internal node has ** been corrected, so be it. Otherwise, after balancing the leaf node, - ** walk the cursor up the tree to the internal node and balance it as + ** walk the cursor up the tree to the internal node and balance it as ** well. */ assert( pCur->pPage->nOverflow==0 ); assert( pCur->pPage->nFree>=0 ); @@ -9827,7 +10017,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ MemPage *pRoot; Pgno pgnoRoot; int rc; - int ptfFlags; /* Page-type flage for the root page of new table */ + int ptfFlags; /* Page-type flags for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -9856,7 +10046,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); if( pgnoRoot>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgnoRoot); } pgnoRoot++; @@ -9904,7 +10094,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -9930,7 +10120,7 @@ static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ } }else{ pRoot = pPageMove; - } + } /* Update the pointer-map and meta-data with the new root-page number. */ ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); @@ -9994,14 +10184,14 @@ static int clearDatabasePage( assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno); } - rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; - if( (pBt->openFlags & BTREE_SINGLE)==0 + if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; } hdr = pPage->hdrOffset; @@ -10085,12 +10275,12 @@ int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ ** cursors on the table. ** ** If AUTOVACUUM is enabled and the page at iTable is not the last -** root page in the database file, then the last root page +** root page in the database file, then the last root page ** in the database file is moved into the slot formerly occupied by ** iTable and that last slot formerly occupied by the last root page ** is added to the freelist instead of iTable. In this say, all ** root pages are kept at the beginning of the database file, which -** is necessary for AUTOVACUUM to work right. *piMoved is set to the +** is necessary for AUTOVACUUM to work right. *piMoved is set to the ** page number that used to be the last root page in the file before ** the move. If no page gets moved, *piMoved is set to 0. ** The last root page is recorded in meta[3] and the value of @@ -10105,7 +10295,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); if( iTable>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(iTable); } rc = sqlite3BtreeClearTable(p, iTable, 0); @@ -10128,7 +10318,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ if( iTable==maxRootPgno ){ /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. + ** number in the database, put the root page on the free list. */ freePage(pPage, &rc); releasePage(pPage); @@ -10137,7 +10327,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ } }else{ /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the + ** number in the database. So move the page that does into the ** gap left by the deleted root-page. */ MemPage *pMove; @@ -10179,7 +10369,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ releasePage(pPage); } #endif - return rc; + return rc; } int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ int rc; @@ -10198,7 +10388,7 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** is the number of free pages currently in the database. Meta[1] ** through meta[15] are available for use by higher layers. Meta[0] ** is read-only, the others are read/write. -** +** ** The schema layer numbers meta values differently. At the schema ** layer (and the SetCookie and ReadCookie opcodes) the number of ** free pages is not visible. So Cookie[0] is the same as Meta[1]. @@ -10268,7 +10458,7 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ ** The first argument, pCur, is a cursor opened on some b-tree. Count the ** number of entries in the b-tree and write the result to *pnEntry. ** -** SQLITE_OK is returned if the operation is successfully executed. +** SQLITE_OK is returned if the operation is successfully executed. ** Otherwise, if an error is encountered (i.e. an IO error or database ** corruption) an SQLite error code is returned. */ @@ -10283,13 +10473,13 @@ int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ } /* Unless an error occurs, the following loop runs one iteration for each - ** page in the B-Tree structure (not including overflow pages). + ** page in the B-Tree structure (not including overflow pages). */ while( rc==SQLITE_OK && !AtomicLoad(&db->u1.isInterrupted) ){ int iIdx; /* Index of child node in parent */ MemPage *pPage; /* Current page of the b-tree */ - /* If this is a leaf page or the tree is not an int-key tree, then + /* If this is a leaf page or the tree is not an int-key tree, then ** this page contains countable entries. Increment the entry counter ** accordingly. */ @@ -10298,7 +10488,7 @@ int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ nEntry += pPage->nCell; } - /* pPage is a leaf node. This loop navigates the cursor so that it + /* pPage is a leaf node. This loop navigates the cursor so that it ** points to the first interior cell that it points to the parent of ** the next page in the tree that has not yet been visited. The ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell @@ -10322,7 +10512,7 @@ int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ pPage = pCur->pPage; } - /* Descend to the child node of the cell that the cursor currently + /* Descend to the child node of the cell that the cursor currently ** points at. This is the right-child if (iIdx==pPage->nCell). */ iIdx = pCur->ix; @@ -10417,7 +10607,8 @@ static void checkAppendMsg( ** corresponds to page iPg is already set. */ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); } @@ -10425,7 +10616,8 @@ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. */ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); } @@ -10439,7 +10631,7 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Also check that the page number is in bounds. */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ - if( iPage>pCheck->nPage || iPage==0 ){ + if( iPage>pCheck->nCkPage || iPage==0 ){ checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } @@ -10453,7 +10645,7 @@ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ #ifndef SQLITE_OMIT_AUTOVACUUM /* -** Check that the entry in the pointer-map for page iChild maps to +** Check that the entry in the pointer-map for page iChild maps to ** page iParent, pointer type ptrType. If not, append an error message ** to pCheck. */ @@ -10476,7 +10668,7 @@ static void checkPtrmap( if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, - "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", + "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } } @@ -10571,7 +10763,7 @@ static void checkList( ** property. ** ** This heap is used for cell overlap and coverage testing. Each u32 -** entry represents the span of a cell or freeblock on a btree page. +** entry represents the span of a cell or freeblock on a btree page. ** The upper 16 bits are the index of the first byte of a range and the ** lower 16 bits are the index of the last byte of that range. */ @@ -10603,7 +10795,7 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ aHeap[j] = x; i = j; } - return 1; + return 1; } #ifndef SQLITE_OMIT_INTEGRITY_CHECK @@ -10611,7 +10803,7 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ ** Do various sanity checks on a single page of a tree. Return ** the tree depth. Root pages return 0. Parents of root pages ** return 1, and so forth. -** +** ** These checks are done: ** ** 1. Make sure that cells and freeblocks do not overlap @@ -10662,10 +10854,11 @@ static int checkTreePage( if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; pCheck->zPfx = "Tree %u page %u: "; - pCheck->v0 = pCheck->v1 = iPage; + pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); + if( rc==SQLITE_IOERR_NOMEM ) pCheck->rc = SQLITE_NOMEM; goto end_of_check; } @@ -10696,6 +10889,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -10807,11 +11003,12 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header ** is the offset of the first freeblock, or zero if there are no - ** freeblocks on the page. + ** freeblocks on the page. */ i = get2byte(&data[hdr+1]); while( i>0 ){ @@ -10831,13 +11028,13 @@ static int checkTreePage( assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ i = j; } - /* Analyze the min-heap looking for overlap between cells and/or + /* Analyze the min-heap looking for overlap between cells and/or ** freeblocks, and counting the number of untracked bytes in nFrag. - ** + ** ** Each min-heap entry is of the form: (start_address<<16)|end_address. ** There is an implied first entry the covers the page header, the cell ** pointer index, and the gap between the cell pointer index and the start - ** of cell content. + ** of cell content. ** ** The loop below pulls entries from the min-heap in order and compares ** the start_address against the previous end_address. If there is an @@ -10906,6 +11103,7 @@ int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ @@ -10919,7 +11117,9 @@ int sqlite3BtreeIntegrityCheck( int bPartial = 0; /* True if not checking all btrees */ int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); /* aRoot[0]==0 means this is a partial check */ if( aRoot[0]==0 ){ @@ -10936,15 +11136,15 @@ int sqlite3BtreeIntegrityCheck( sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; - sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.nCkPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; - if( sCheck.nPage==0 ){ + if( sCheck.nCkPage==0 ){ goto integrity_ck_cleanup; } - sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + sCheck.aPgRef = sqlite3MallocZero((sCheck.nCkPage / 8)+ 1); if( !sCheck.aPgRef ){ checkOom(&sCheck); goto integrity_ck_cleanup; @@ -10956,7 +11156,7 @@ int sqlite3BtreeIntegrityCheck( } i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + if( i<=sCheck.nCkPage ) setPageReferenced(&sCheck, i); /* Check the integrity of the freelist */ @@ -10992,21 +11192,25 @@ int sqlite3BtreeIntegrityCheck( testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); - } + if( pBt->autoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } #endif - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ if( !bPartial ){ - for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ + for(i=1; i<=sCheck.nCkPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ checkAppendMsg(&sCheck, "Page %u: never used", i); @@ -11015,11 +11219,11 @@ int sqlite3BtreeIntegrityCheck( /* If the database supports auto-vacuum, make sure no tables contain ** references to pointer-map pages. */ - if( getPageReferenced(&sCheck, i)==0 && + if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ checkAppendMsg(&sCheck, "Page %u: never used", i); } - if( getPageReferenced(&sCheck, i)!=0 && + if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i); } @@ -11084,7 +11288,7 @@ int sqlite3BtreeTxnState(Btree *p){ /* ** Run a checkpoint on the Btree passed as the first argument. ** -** Return SQLITE_LOCKED if this or any other connection has an open +** Return SQLITE_LOCKED if this or any other connection has an open ** transaction on the shared-cache the argument Btree is connected to. ** ** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. @@ -11117,25 +11321,26 @@ int sqlite3BtreeIsInBackup(Btree *p){ /* ** This function returns a pointer to a blob of memory associated with ** a single shared-btree. The memory is used by client code for its own -** purposes (for example, to store a high-level schema associated with +** purposes (for example, to store a high-level schema associated with ** the shared-btree). The btree layer manages reference counting issues. ** ** The first time this is called on a shared-btree, nBytes bytes of memory -** are allocated, zeroed, and returned to the caller. For each subsequent +** are allocated, zeroed, and returned to the caller. For each subsequent ** call the nBytes parameter is ignored and a pointer to the same blob -** of memory returned. +** of memory returned. ** ** If the nBytes parameter is 0 and the blob of memory has not yet been ** allocated, a null pointer is returned. If the blob has already been ** allocated, it is returned as normal. ** -** Just before the shared-btree is closed, the function passed as the -** xFree argument when the memory allocation was made is invoked on the +** Just before the shared-btree is closed, the function passed as the +** xFree argument when the memory allocation was made is invoked on the ** blob of allocated memory. The xFree function should not call sqlite3_free() ** on the memory, the btree layer does that. */ void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ BtShared *pBt = p->pBt; + assert( nBytes==0 || nBytes==sizeof(Schema) ); sqlite3BtreeEnter(p); if( !pBt->pSchema && nBytes ){ pBt->pSchema = sqlite3DbMallocZero(0, nBytes); @@ -11146,12 +11351,13 @@ void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ } /* -** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared -** btree as the argument handle holds an exclusive lock on the +** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared +** btree as the argument handle holds an exclusive lock on the ** sqlite_schema table. Otherwise SQLITE_OK. */ int sqlite3BtreeSchemaLocked(Btree *p){ int rc; + UNUSED_PARAMETER(p); /* only used in DEBUG builds */ assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); @@ -11188,11 +11394,11 @@ int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ #ifndef SQLITE_OMIT_INCRBLOB /* -** Argument pCsr must be a cursor opened for writing on an -** INTKEY table currently pointing at a valid table entry. +** Argument pCsr must be a cursor opened for writing on an +** INTKEY table currently pointing at a valid table entry. ** This function modifies the data stored as part of that entry. ** -** Only the data content may only be modified, it is not possible to +** Only the data content may only be modified, it is not possible to ** change the length of the data stored. If this function is called with ** parameters that attempt to write past the end of the existing data, ** no modifications are made and SQLITE_CORRUPT is returned. @@ -11223,7 +11429,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); assert( rc==SQLITE_OK ); - /* Check some assumptions: + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, ** (c) the connection holds a write-lock on the table (if required), @@ -11242,7 +11448,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); } -/* +/* ** Mark this cursor as an incremental blob cursor. */ void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ @@ -11252,14 +11458,14 @@ void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ #endif /* -** Set both the "read version" (single byte at byte offset 18) and +** Set both the "read version" (single byte at byte offset 18) and ** "write version" (single byte at byte offset 19) fields in the database ** header to iVersion. */ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ BtShared *pBt = pBtree->pBt; int rc; /* Return code */ - + assert( iVersion==1 || iVersion==2 ); /* If setting the version fields to 1, do not automatically open the @@ -11328,7 +11534,7 @@ int sqlite3BtreeSharable(Btree *p){ /* ** Return the number of connections to the BtShared object accessed by -** the Btree handle passed as the only argument. For private caches +** the Btree handle passed as the only argument. For private caches ** this is always 1. For shared caches it may be 1 or greater. */ int sqlite3BtreeConnectionCount(Btree *p){ diff --git a/src/btree.h b/src/btree.h index b9078f9010..96f4c4c607 100644 --- a/src/btree.h +++ b/src/btree.h @@ -240,6 +240,9 @@ int sqlite3BtreeCursor( ); BtCursor *sqlite3BtreeFakeValidCursor(void); int sqlite3BtreeCursorSize(void); +#ifdef SQLITE_DEBUG +int sqlite3BtreeClosesWithCursor(Btree*,BtCursor*); +#endif void sqlite3BtreeCursorZero(BtCursor*); void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -314,6 +317,7 @@ struct BtreePayload { int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); int sqlite3BtreeFirst(BtCursor*, int *pRes); +int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes); int sqlite3BtreeLast(BtCursor*, int *pRes); int sqlite3BtreeNext(BtCursor*, int flags); int sqlite3BtreeEof(BtCursor*); @@ -321,9 +325,7 @@ int sqlite3BtreePrevious(BtCursor*, int flags); i64 sqlite3BtreeIntegerKey(BtCursor*); void sqlite3BtreeCursorPin(BtCursor*); void sqlite3BtreeCursorUnpin(BtCursor*); -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC i64 sqlite3BtreeOffset(BtCursor*); -#endif int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); u32 sqlite3BtreePayloadSize(BtCursor*); @@ -333,6 +335,7 @@ int sqlite3BtreeIntegrityCheck( sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ int *pnErr, /* OUT: Write number of errors seen to this variable */ diff --git a/src/btreeInt.h b/src/btreeInt.h index 6d3578422f..17e3a1add5 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -64,7 +64,7 @@ ** 22 1 Min embedded payload fraction (must be 32) ** 23 1 Min leaf payload fraction (must be 32) ** 24 4 File change counter -** 28 4 Reserved for future use +** 28 4 The size of the database in pages ** 32 4 First freelist page ** 36 4 Number of freelist pages in the file ** 40 60 15 4-byte meta values passed to higher layers @@ -180,7 +180,7 @@ ** 0x81 0x00 becomes 0x00000080 ** 0x82 0x00 becomes 0x00000100 ** 0x80 0x7f becomes 0x0000007f -** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x91 0xd1 0xac 0x78 becomes 0x12345678 ** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 ** ** Variable length integers are used for rowids and to hold the number of @@ -263,7 +263,7 @@ typedef struct CellInfo CellInfo; ** page that has been loaded into memory. The information in this object ** is derived from the raw on-disk page content. ** -** As each database page is loaded into memory, the pager allocats an +** As each database page is loaded into memory, the pager allocates an ** instance of this object and zeros the first 8 bytes. (This is the ** "extra" information associated with each page of the pager.) ** @@ -496,6 +496,12 @@ struct CellInfo { */ #define BTCURSOR_MAX_DEPTH 20 +/* +** Maximum amount of storage local to a database page, regardless of +** page size. +*/ +#define BT_MAX_LOCAL 65501 /* 65536 - 35 */ + /* ** A cursor is a pointer to a particular entry within a particular ** b-tree within a database file. @@ -695,7 +701,7 @@ struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ u8 *aPgRef; /* 1 bit per page in the db (see above) */ - Pgno nPage; /* Number of pages in the database */ + Pgno nCkPage; /* Pages in the database. 0 for partial check */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ @@ -707,6 +713,7 @@ struct IntegrityCk { StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* @@ -719,7 +726,7 @@ struct IntegrityCk { /* ** get2byteAligned(), unlike get2byte(), requires that its argument point to a -** two-byte aligned address. get2bytea() is only used for accessing the +** two-byte aligned address. get2byteAligned() is only used for accessing the ** cell addresses in a btree header. */ #if SQLITE_BYTEORDER==4321 diff --git a/src/build.c b/src/build.c index 9be444c3c3..de890c2e91 100644 --- a/src/build.c +++ b/src/build.c @@ -37,7 +37,7 @@ struct TableLock { }; /* -** Record the fact that we want to lock a table at run-time. +** Record the fact that we want to lock a table at run-time. ** ** The table to be locked has root page iTab and is found in database iDb. ** A read or a write lock can be taken depending on isWritelock. @@ -68,6 +68,7 @@ static SQLITE_NOINLINE void lockTable( } } + assert( pToplevel->nTableLock < 0x7fff0000 ); nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); pToplevel->aTableLock = sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); @@ -100,7 +101,7 @@ void sqlite3TableLock( */ static void codeTableLocks(Parse *pParse){ int i; - Vdbe *pVdbe = pParse->pVdbe; + Vdbe *pVdbe = pParse->pVdbe; assert( pVdbe!=0 ); for(i=0; inTableLock; i++){ @@ -164,14 +165,16 @@ void sqlite3FinishCoding(Parse *pParse){ v = sqlite3GetVdbe(pParse); if( v==0 ) pParse->rc = SQLITE_ERROR; } - assert( !pParse->isMultiWrite + assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ if( pParse->bReturning ){ - Returning *pReturning = pParse->u1.pReturning; + Returning *pReturning; int addrRewind; int reg; + assert( !pParse->isCreate ); + pReturning = pParse->u1.d.pReturning; if( pReturning->nRetCol ){ sqlite3VdbeAddOp0(v, OP_FkCheck); addrRewind = @@ -189,17 +192,6 @@ void sqlite3FinishCoding(Parse *pParse){ } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION - if( pParse->nTableLock>0 && db->init.busy==0 ){ - sqlite3UserAuthInit(db); - if( db->auth.authLevelrc = SQLITE_AUTH_USER; - return; - } - } -#endif - /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a @@ -234,34 +226,33 @@ void sqlite3FinishCoding(Parse *pParse){ pParse->nVtabLock = 0; #endif - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the +#ifndef SQLITE_OMIT_SHARED_CACHE + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the ** shared-cache feature is enabled. */ - codeTableLocks(pParse); + if( pParse->nTableLock ) codeTableLocks(pParse); +#endif /* Initialize any AUTOINCREMENT data structures required. */ - sqlite3AutoincrementBegin(pParse); + if( pParse->pAinc ) sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops. - ** - ** The pConstExpr list might also contain expressions that we simply - ** want to keep around until the Parse object is deleted. Such - ** expressions have iConstExprReg==0. Do not generate code for - ** those expressions, of course. + /* Code constant expressions that were factored out of inner loops. */ if( pParse->pConstExpr ){ ExprList *pEL = pParse->pConstExpr; pParse->okConstFactor = 0; for(i=0; inExpr; i++){ - int iReg = pEL->a[i].u.iConstExprReg; - sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); + assert( pEL->a[i].u.iConstExprReg>0 ); + sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); } } if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; + Returning *pRet; + assert( !pParse->isCreate ); + pRet = pParse->u1.d.pReturning; if( pRet->nRetCol ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); } @@ -331,16 +322,6 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ pParse->nested--; } -#if SQLITE_USER_AUTHENTICATION -/* -** Return TRUE if zTable is the name of the system table that stores the -** list of users and their access credentials. -*/ -int sqlite3UserAuthTable(const char *zTable){ - return sqlite3_stricmp(zTable, "sqlite_user")==0; -} -#endif - /* ** Locate the in-memory structure that describes a particular database ** table given the name of that table and (optionally) the name of the @@ -359,13 +340,6 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); -#if SQLITE_USER_AUTHENTICATION - /* Only the admin user is allowed to know that the sqlite_user table - ** exists */ - if( db->auth.authLevelnDb; i++){ if( sqlite3StrICmp(zDatabase, db->aDb[i].zDbSName)==0 ) break; @@ -386,7 +360,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ || sqlite3StrICmp(zName+7, &PREFERRED_SCHEMA_TABLE[7])==0 || sqlite3StrICmp(zName+7, &LEGACY_SCHEMA_TABLE[7])==0 ){ - p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, LEGACY_TEMP_SCHEMA_TABLE); } }else{ @@ -413,7 +387,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ if( sqlite3StrICmp(zName+7, &PREFERRED_SCHEMA_TABLE[7])==0 ){ p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, LEGACY_SCHEMA_TABLE); }else if( sqlite3StrICmp(zName+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ - p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, LEGACY_TEMP_SCHEMA_TABLE); } } @@ -442,7 +416,7 @@ Table *sqlite3LocateTable( /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ - if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 + if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 && SQLITE_OK!=sqlite3ReadSchema(pParse) ){ return 0; @@ -459,6 +433,16 @@ Table *sqlite3LocateTable( if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); } +#ifndef SQLITE_OMIT_JSON + if( pMod==0 && sqlite3_strnicmp(zName, "json", 4)==0 ){ + pMod = sqlite3JsonVtabRegister(db, zName); + } +#endif +#ifdef SQLITE_ENABLE_CARRAY + if( pMod==0 && sqlite3_stricmp(zName, "carray")==0 ){ + pMod = sqlite3CarrayRegister(db); + } +#endif if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ testcase( pMod->pEpoTab==0 ); return pMod->pEpoTab; @@ -495,17 +479,17 @@ Table *sqlite3LocateTable( ** sqlite3FixSrcList() for details. */ Table *sqlite3LocateTableItem( - Parse *pParse, + Parse *pParse, u32 flags, SrcItem *p ){ const char *zDb; - assert( p->pSchema==0 || p->zDatabase==0 ); - if( p->pSchema ){ - int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + if( p->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); zDb = pParse->db->aDb[iDb].zDbSName; }else{ - zDb = p->zDatabase; + assert( !p->fg.isSubquery ); + zDb = p->u4.zDatabase; } return sqlite3LocateTable(pParse, flags, p->zName, zDb); } @@ -527,7 +511,7 @@ const char *sqlite3PreferredTableName(const char *zName){ } /* -** Locate the in-memory structure that describes +** Locate the in-memory structure that describes ** a particular index given the name of that index ** and the name of the database that contains the index. ** Return NULL if not found. @@ -723,7 +707,7 @@ void sqlite3ColumnSetExpr( */ Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ if( pCol->iDflt==0 ) return 0; - if( NEVER(!IsOrdinaryTable(pTab)) ) return 0; + if( !IsOrdinaryTable(pTab) ) return 0; if( NEVER(pTab->u.tab.pDfltList==0) ) return 0; if( NEVER(pTab->u.tab.pDfltList->nExpriDflt) ) return 0; return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; @@ -755,7 +739,7 @@ void sqlite3ColumnSetColl( } /* -** Return the collating squence name for a column +** Return the collating sequence name for a column */ const char *sqlite3ColumnColl(Column *pCol){ const char *z; @@ -802,10 +786,10 @@ void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ ** ** This routine just deletes the data structure. It does not unlink ** the table data structure from the hash table. But it does destroy -** memory structures of the indices and foreign keys associated with +** memory structures of the indices and foreign keys associated with ** the table. ** -** The db parameter is optional. It is needed if the Table object +** The db parameter is optional. It is needed if the Table object ** contains lookaside memory. (Table objects in the schema do not use ** lookaside memory, but some ephemeral Table objects do.) Or the ** db parameter can be used with db->pnBytesFreed to measure the memory @@ -817,7 +801,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ #ifdef SQLITE_DEBUG /* Record the number of outstanding lookaside allocations in schema Tables ** prior to doing any free() operations. Since schema Tables do not use - ** lookaside, this number should not change. + ** lookaside, this number should not change. ** ** If malloc has already failed, it may be that it failed while allocating ** a Table object that was going to be marked ephemeral. So do not check @@ -835,7 +819,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ assert( pIndex->pSchema==pTable->pSchema || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){ - char *zName = pIndex->zName; + char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 ); @@ -876,6 +860,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } +void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){ + sqlite3DeleteTable(db, (Table*)pTable); +} /* @@ -957,7 +944,7 @@ int sqlite3FindDbName(sqlite3 *db, const char *zName){ /* ** The token *pName contains the name of a database (either "main" or ** "temp" or the name of an attached db). This routine returns the -** index of the named database in db->aDb[], or -1 if the named db +** index of the named database in db->aDb[], or -1 if the named db ** does not exist. */ int sqlite3FindDb(sqlite3 *db, Token *pName){ @@ -973,7 +960,7 @@ int sqlite3FindDb(sqlite3 *db, Token *pName){ ** pName1 and pName2. If the table name was fully qualified, for example: ** ** CREATE TABLE xxx.yyy (...); -** +** ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if ** the table name is not fully qualified, i.e.: ** @@ -1090,10 +1077,16 @@ Index *sqlite3PrimaryKeyIndex(Table *pTab){ ** find the (first) offset of that column in index pIdx. Or return -1 ** if column iCol is not used in index pIdx. */ -i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ +int sqlite3TableColumnToIndex(Index *pIdx, int iCol){ int i; + i16 iCol16; + assert( iCol>=(-1) && iCol<=SQLITE_MAX_COLUMN ); + assert( pIdx->nColumn<=SQLITE_MAX_COLUMN*2 ); + iCol16 = iCol; for(i=0; inColumn; i++){ - if( iCol==pIdx->aiColumn[i] ) return i; + if( iCol16==pIdx->aiColumn[i] ){ + return i; + } } return -1; } @@ -1127,7 +1120,7 @@ i16 sqlite3StorageColumnToTable(Table *pTab, i16 iCol){ ** The storage column number (0,1,2,....) is the index of the value ** as it appears in the record on disk. Or, if the input column is ** the N-th virtual column (zero-based) then the storage number is -** the number of non-virtual columns in the table plus N. +** the number of non-virtual columns in the table plus N. ** ** The true column number is the index (0,1,2,...) of the column in ** the CREATE TABLE statement. @@ -1235,7 +1228,7 @@ void sqlite3StartTable( iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) return; if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless + /* If creating a temp table, the name may not be qualified. Unless ** the database name is "temp" anyway. */ sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); return; @@ -1326,7 +1319,7 @@ void sqlite3StartTable( ** the schema table. Note in particular that we must go ahead ** and allocate the record number for the table entry now. Before any ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause - ** indices to be created and the table record must come before the + ** indices to be created and the table record must come before the ** indices. Hence, the record number for the table must be allocated ** now. */ @@ -1344,11 +1337,12 @@ void sqlite3StartTable( } #endif - /* If the file format and encoding in the database have not been set, + /* If the file format and encoding in the database have not been set, ** set them now. */ - reg1 = pParse->regRowid = ++pParse->nMem; - reg2 = pParse->regRoot = ++pParse->nMem; + assert( pParse->isCreate ); + reg1 = pParse->u1.cr.regRowid = ++pParse->nMem; + reg2 = pParse->u1.cr.regRoot = ++pParse->nMem; reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); @@ -1363,8 +1357,8 @@ void sqlite3StartTable( ** The record created does not contain anything yet. It will be replaced ** by the real entry in code generated at sqlite3EndTable(). ** - ** The rowid for the new entry is left in register pParse->regRowid. - ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid for the new entry is left in register pParse->u1.cr.regRowid. + ** The root page of the new table is left in reg pParse->u1.cr.regRoot. ** The rowid and root page number values are needed by the code that ** sqlite3EndTable will generate. */ @@ -1375,7 +1369,7 @@ void sqlite3StartTable( #endif { assert( !pParse->bReturning ); - pParse->u1.addrCrTab = + pParse->u1.cr.addrCrTab = sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); } sqlite3OpenSchemaTable(pParse, iDb); @@ -1384,6 +1378,9 @@ void sqlite3StartTable( sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); + }else if( db->init.imposterTable ){ + pTable->tabFlags |= TF_Imposter; + if( db->init.imposterTable>=2 ) pTable->tabFlags |= TF_Readonly; } /* Normal (non-error) return. */ @@ -1410,20 +1407,14 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ } #endif -/* -** Name of the special TEMP trigger used to implement RETURNING. The -** name begins with "sqlite_" so that it is guaranteed not to collide -** with any application-generated triggers. -*/ -#define RETURNING_TRIGGER_NAME "sqlite_returning" - /* ** Clean up the data structures associated with the RETURNING clause. */ -static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){ +static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){ + Returning *pRet = (Returning*)pArg; Hash *pHash; pHash = &(db->aDb[1].pSchema->trigHash); - sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, 0); + sqlite3HashInsert(pHash, pRet->zName, 0); sqlite3ExprListDelete(db, pRet->pReturnEL); sqlite3DbFree(db, pRet); } @@ -1459,14 +1450,16 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ sqlite3ExprListDelete(db, pList); return; } - pParse->u1.pReturning = pRet; + assert( !pParse->isCreate ); + pParse->u1.d.pReturning = pRet; pRet->pParse = pParse; pRet->pReturnEL = pList; - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet); + sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet); testcase( pParse->earlyCleanup ); if( db->mallocFailed ) return; - pRet->retTrig.zName = RETURNING_TRIGGER_NAME; + sqlite3_snprintf(sizeof(pRet->zName), pRet->zName, + "sqlite_returning_%p", pParse); + pRet->retTrig.zName = pRet->zName; pRet->retTrig.op = TK_RETURNING; pRet->retTrig.tr_tm = TRIGGER_AFTER; pRet->retTrig.bReturning = 1; @@ -1477,9 +1470,9 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTStep.pTrig = &pRet->retTrig; pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); - assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 + assert( sqlite3HashFind(pHash, pRet->zName)==0 || pParse->nErr || pParse->ifNotExists ); - if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) + if( sqlite3HashInsert(pHash, pRet->zName, &pRet->retTrig) ==&pRet->retTrig ){ sqlite3OomFault(db); } @@ -1500,7 +1493,6 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ char *zType; Column *pCol; sqlite3 *db = pParse->db; - u8 hName; Column *aNew; u8 eType = COLTYPE_CUSTOM; u8 szEst = 1; @@ -1513,7 +1505,7 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ } if( !IN_RENAME_OBJECT ) sqlite3DequoteToken(&sName); - /* Because keywords GENERATE ALWAYS can be converted into indentifiers + /* Because keywords GENERATE ALWAYS can be converted into identifiers ** by the parser, we can sometimes end up with a typename that ends ** with "generated always". Check for this case and omit the surplus ** text. */ @@ -1554,13 +1546,10 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ memcpy(z, sName.z, sName.n); z[sName.n] = 0; sqlite3Dequote(z); - hName = sqlite3StrIHash(z); - for(i=0; inCol; i++){ - if( p->aCol[i].hName==hName && sqlite3StrICmp(z, p->aCol[i].zCnName)==0 ){ - sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); - sqlite3DbFree(db, z); - return; - } + if( p->nCol && sqlite3ColumnIndex(p, z)>=0 ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; } aNew = sqlite3DbRealloc(db,p->aCol,((i64)p->nCol+1)*sizeof(p->aCol[0])); if( aNew==0 ){ @@ -1571,9 +1560,9 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zCnName = z; - pCol->hName = hName; + pCol->hName = sqlite3StrIHash(z); sqlite3ColumnPropertiesFromName(p, pCol); - + if( sType.n==0 ){ /* If there is no type specified, columns have the default affinity ** 'BLOB' with a default size of 4 bytes. */ @@ -1595,9 +1584,14 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } + if( p->nCol<=0xff ){ + u8 h = pCol->hName % sizeof(p->aHx); + p->aHx[h] = p->nCol; + } p->nCol++; p->nNVCol++; - pParse->constraintName.n = 0; + assert( pParse->isCreate ); + pParse->u1.cr.constraintName.n = 0; } /* @@ -1632,11 +1626,11 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ ** Scan the column type name zType (length nType) and return the ** associated affinity type. ** -** This routine does a case-independent search of zType for the +** This routine does a case-independent search of zType for the ** substrings in the following table. If one of the substrings is ** found, the corresponding affinity is returned. If zType contains -** more than one of the substrings, entries toward the top of -** the table take priority. For example, if zType is 'BLOBINT', +** more than one of the substrings, entries toward the top of +** the table take priority. For example, if zType is 'BLOBINT', ** SQLITE_AFF_INTEGER is returned. ** ** Substring | Affinity @@ -1660,7 +1654,8 @@ char sqlite3AffinityType(const char *zIn, Column *pCol){ assert( zIn!=0 ); while( zIn[0] ){ - h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + u8 x = *(u8*)zIn; + h = (h<<8) + sqlite3UpperToLower[x]; zIn++; if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ aff = SQLITE_AFF_TEXT; @@ -1734,7 +1729,7 @@ void sqlite3AddDefaultValue( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The parsed expression of the default value */ const char *zStart, /* Start of the default value text */ - const char *zEnd /* First character past end of defaut value text */ + const char *zEnd /* First character past end of default value text */ ){ Table *p; Column *pCol; @@ -1775,7 +1770,7 @@ void sqlite3AddDefaultValue( /* ** Backwards Compatibility Hack: -** +** ** Historical versions of SQLite accepted strings as column names in ** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: ** @@ -1809,11 +1804,11 @@ static void makeColumnPartOfPrimaryKey(Parse *pParse, Column *pCol){ sqlite3ErrorMsg(pParse, "generated columns cannot be part of the PRIMARY KEY"); } -#endif +#endif } /* -** Designate the PRIMARY KEY for the table. pList is a list of names +** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the ** most recently added column of the table is the primary key. ** @@ -1843,7 +1838,7 @@ void sqlite3AddPrimaryKey( int nTerm; if( pTab==0 ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); goto primary_key_exit; } @@ -1860,15 +1855,11 @@ void sqlite3AddPrimaryKey( assert( pCExpr!=0 ); sqlite3StringToId(pCExpr); if( pCExpr->op==TK_ID ){ - const char *zCName; assert( !ExprHasProperty(pCExpr, EP_IntValue) ); - zCName = pCExpr->u.zToken; - for(iCol=0; iColnCol; iCol++){ - if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zCnName)==0 ){ - pCol = &pTab->aCol[iCol]; - makeColumnPartOfPrimaryKey(pParse, pCol); - break; - } + iCol = sqlite3ColumnIndex(pTab, pCExpr->u.zToken); + if( iCol>=0 ){ + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); } } } @@ -1920,15 +1911,17 @@ void sqlite3AddCheckConstraint( && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) ){ pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); - if( pParse->constraintName.n ){ - sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + assert( pParse->isCreate ); + if( pParse->u1.cr.constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, + &pParse->u1.cr.constraintName, 1); }else{ Token t; for(zStart++; sqlite3Isspace(zStart[0]); zStart++){} while( sqlite3Isspace(zEnd[-1]) ){ zEnd--; } t.z = zStart; t.n = (int)(zEnd - t.z); - sqlite3ExprListSetName(pParse, pTab->pCheck, &t, 1); + sqlite3ExprListSetName(pParse, pTab->pCheck, &t, 1); } }else #endif @@ -1956,7 +1949,7 @@ void sqlite3AddCollateType(Parse *pParse, Token *pToken){ if( sqlite3LocateCollSeq(pParse, zColl) ){ Index *pIdx; sqlite3ColumnSetColl(db, &p->aCol[i], zColl); - + /* If the column is declared as " PRIMARY KEY COLLATE ", ** then an index may have been created on this column before the ** collation type was added. Correct this if it is the case. @@ -2054,7 +2047,7 @@ void sqlite3ChangeCookie(Parse *pParse, int iDb){ sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, (int)(1+(unsigned)db->aDb[iDb].pSchema->schema_cookie)); } @@ -2075,14 +2068,14 @@ static int identLength(const char *z){ } /* -** The first parameter is a pointer to an output buffer. The second +** The first parameter is a pointer to an output buffer. The second ** parameter is a pointer to an integer that contains the offset at ** which to write into the output buffer. This function copies the ** nul-terminated string pointed to by the third parameter, zSignedIdent, ** to the specified offset in the buffer and updates *pIdx to refer ** to the first byte after the last byte written before returning. -** -** If the string zSignedIdent consists entirely of alpha-numeric +** +** If the string zSignedIdent consists entirely of alphanumeric ** characters, does not begin with a digit and is not an SQL keyword, ** then it is copied to the output buffer exactly as it is. Otherwise, ** it is quoted using double-quotes. @@ -2116,7 +2109,8 @@ static void identPut(char *z, int *pIdx, char *zSignedIdent){ ** from sqliteMalloc() and must be freed by the calling function. */ static char *createTableStmt(sqlite3 *db, Table *p){ - int i, k, n; + int i, k, len; + i64 n; char *zStmt; char *zSep, *zSep2, *zEnd; Column *pCol; @@ -2125,7 +2119,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ n += identLength(pCol->zCnName) + 5; } n += identLength(p->zName); - if( n<50 ){ + if( n<50 ){ zSep = ""; zSep2 = ","; zEnd = ")"; @@ -2140,8 +2134,9 @@ static char *createTableStmt(sqlite3 *db, Table *p){ sqlite3OomFault(db); return 0; } - sqlite3_snprintf(n, zStmt, "CREATE TABLE "); - k = sqlite3Strlen30(zStmt); + assert( n>14 && n<=0x7fffffff ); + memcpy(zStmt, "CREATE TABLE ", 13); + k = 13; identPut(zStmt, &k, p->zName); zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ @@ -2153,13 +2148,15 @@ static char *createTableStmt(sqlite3 *db, Table *p){ /* SQLITE_AFF_REAL */ " REAL", /* SQLITE_AFF_FLEXNUM */ " NUM", }; - int len; const char *zType; - sqlite3_snprintf(n-k, &zStmt[k], zSep); - k += sqlite3Strlen30(&zStmt[k]); + len = sqlite3Strlen30(zSep); + assert( k+lenzCnName); + assert( kaffinity-SQLITE_AFF_BLOB >= 0 ); assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_BLOB ); @@ -2168,17 +2165,20 @@ static char *createTableStmt(sqlite3 *db, Table *p){ testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); testcase( pCol->affinity==SQLITE_AFF_FLEXNUM ); - + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_BLOB || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); + assert( k+lennColumn>=N ) return SQLITE_OK; + db = pParse->db; + assert( N>0 ); + assert( N <= SQLITE_MAX_COLUMN*2 /* tag-20250221-1 */ ); + testcase( N==2*pParse->db->aLimit[SQLITE_LIMIT_COLUMN] ); assert( pIdx->isResized==0 ); - nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N; + nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*(u64)N; zExtra = sqlite3DbMallocZero(db, nByte); if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); @@ -2205,7 +2210,7 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ zExtra += sizeof(i16)*N; memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); pIdx->aSortOrder = (u8*)zExtra; - pIdx->nColumn = N; + pIdx->nColumn = (u16)N; /* See tag-20250221-1 above for proof of safety */ pIdx->isResized = 1; return SQLITE_OK; } @@ -2234,7 +2239,7 @@ static void estimateIndexWidth(Index *pIdx){ for(i=0; inColumn; i++){ i16 x = pIdx->aiColumn[i]; assert( xpTable->nCol ); - wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + wIndex += x<0 ? 1 : aCol[x].szEst; } pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } @@ -2277,7 +2282,7 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ assert( j!=XN_ROWID && j!=XN_EXPR ); for(i=0; iaiColumn[i]>=0 || j>=0 ); - if( pIdx->aiColumn[i]==j + if( pIdx->aiColumn[i]==j && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 ){ return 1; @@ -2329,7 +2334,7 @@ static void recomputeColumnsNotIndexed(Index *pIdx){ ** Changes include: ** ** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. -** (2) Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY +** (2) Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY ** into BTREE_BLOBKEY. ** (3) Bypass the creation of the sqlite_schema table entry ** for the PRIMARY KEY as the primary key index is now @@ -2371,19 +2376,19 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** into BTREE_BLOBKEY. */ assert( !pParse->bReturning ); - if( pParse->u1.addrCrTab ){ + if( pParse->u1.cr.addrCrTab ){ assert( v ); - sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); + sqlite3VdbeChangeP3(v, pParse->u1.cr.addrCrTab, BTREE_BLOBKEY); } /* Locate the PRIMARY KEY index. Or, if this table was originally - ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. + ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. */ if( pTab->iPKey>=0 ){ ExprList *pList; Token ipkToken; sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zCnName); - pList = sqlite3ExprListAppend(pParse, 0, + pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ){ pTab->tabFlags &= ~TF_WithoutRowid; @@ -2459,14 +2464,14 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pIdx->nColumn = pIdx->nKeyCol; continue; } - if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + if( resizeIndexObject(pParse, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; inKeyCol, pPk, i) ){ testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; if( pPk->aSortOrder[i] ){ - /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + /* See ticket https://sqlite.org/src/info/bba7b69f9849b5bf */ pIdx->bAscKeyBug = 1; } j++; @@ -2483,7 +2488,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, nPk, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; } - if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + if( resizeIndexObject(pParse, pPk, nPk+nExtra) ) return; for(i=0, j=nPk; inCol; i++){ if( !hasColumn(pPk->aiColumn, j, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 @@ -2622,7 +2627,7 @@ static void markExprListImmutable(ExprList *pList){ ** the sqlite_schema table. We do not want to create it again. ** ** If the pSelect argument is not NULL, it means that this routine -** was called to create a table generated from a +** was called to create a table generated from a ** "CREATE TABLE ... AS SELECT ..." statement. The column names of ** the new table will match the result set of the SELECT. */ @@ -2701,7 +2706,7 @@ void sqlite3EndTable( pCol->notNull = OE_Abort; p->tabFlags |= TF_HasNotNull; } - } + } } assert( (p->tabFlags & TF_HasPrimaryKey)==0 @@ -2758,7 +2763,7 @@ void sqlite3EndTable( ** tree that have been allocated from lookaside memory, which is ** illegal in a schema and will lead to errors or heap corruption ** when the database connection closes. */ - sqlite3ColumnSetExpr(pParse, p, &p->aCol[ii], + sqlite3ColumnSetExpr(pParse, p, &p->aCol[ii], sqlite3ExprAlloc(db, TK_NULL, 0, 0)); } }else{ @@ -2796,7 +2801,7 @@ void sqlite3EndTable( sqlite3VdbeAddOp1(v, OP_Close, 0); - /* + /* ** Initialize zType for the new view or table. */ if( IsOrdinaryTable(p) ){ @@ -2813,7 +2818,7 @@ void sqlite3EndTable( /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT ** statement to populate the new table. The root-page number for the - ** new table is in register pParse->regRoot. + ** new table is in register pParse->u1.cr.regRoot. ** ** Once the SELECT has been coded by sqlite3Select(), it is in a ** suitable state to query for the column names and types to be used @@ -2832,20 +2837,21 @@ void sqlite3EndTable( int regRowid; /* Rowid of the next row to insert */ int addrInsLoop; /* Top of the loop for inserting rows */ Table *pSelTab; /* A table that describes the SELECT results */ + int iCsr; /* Write cursor on the new table */ if( IN_SPECIAL_PARSE ){ pParse->rc = SQLITE_ERROR; pParse->nErr++; return; } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + assert( pParse->isCreate ); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->u1.cr.regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; @@ -2866,11 +2872,11 @@ void sqlite3EndTable( VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -2880,15 +2886,16 @@ void sqlite3EndTable( Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd; n = (int)(pEnd2->z - pParse->sNameToken.z); if( pEnd2->z[0]!=';' ) n += pEnd2->n; - zStmt = sqlite3MPrintf(db, + zStmt = sqlite3MPrintf(db, "CREATE %s %.*s", zType2, n, pParse->sNameToken.z ); } - /* A slot for the record has already been allocated in the + /* A slot for the record has already been allocated in the ** schema table. We just need to update that slot with all ** the information we've collected. */ + assert( pParse->isCreate ); sqlite3NestedParse(pParse, "UPDATE %Q." LEGACY_SCHEMA_TABLE " SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q" @@ -2897,9 +2904,9 @@ void sqlite3EndTable( zType, p->zName, p->zName, - pParse->regRoot, + pParse->u1.cr.regRoot, zStmt, - pParse->regRowid + pParse->u1.cr.regRowid ); sqlite3DbFree(db, zStmt); sqlite3ChangeCookie(pParse, iDb); @@ -2923,6 +2930,14 @@ void sqlite3EndTable( /* Reparse everything to update our internal data structures */ sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); + + /* Test for cycles in generated columns and illegal expressions + ** in CHECK constraints and in DEFAULT clauses. */ + if( p->tabFlags & TF_HasGenerated ){ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, + sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", + db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); + } } /* Add the table to the in-memory representation of the database. @@ -2999,9 +3014,12 @@ void sqlite3CreateView( ** on a view, even though views do not have rowids. The following flag ** setting fixes this problem. But the fix can be disabled by compiling ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that - ** depend upon the old buggy behavior. */ -#ifndef SQLITE_ALLOW_ROWID_IN_VIEW - p->tabFlags |= TF_NoVisibleRowid; + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ #endif sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -3057,8 +3075,9 @@ void sqlite3CreateView( #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ @@ -3100,7 +3119,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ ** Actually, the error above is now caught prior to reaching this point. ** But the following test is still important as it does come up ** in the following: - ** + ** ** CREATE TABLE main.ex1(a); ** CREATE TEMP VIEW ex1 AS SELECT a FROM ex1; ** SELECT * FROM temp.ex1; @@ -3148,7 +3167,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ ** normally holds CHECK constraints on an ordinary table, but for ** a VIEW it holds the list of column names. */ - sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, &pTable->nCol, &pTable->aCol); if( pParse->nErr==0 && pTable->nCol==pSel->pEList->nExpr @@ -3181,7 +3200,7 @@ static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteColumnNames(db, pTable); } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; } int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable!=0 ); @@ -3220,7 +3239,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ ** on tables and/or indices that are the process of being deleted. ** If you are unlucky, one of those deleted indices or tables might ** have the same rootpage number as the real table or index that is -** being moved. So we cannot stop searching after the first match +** being moved. So we cannot stop searching after the first match ** because the first match might be for one of the deleted indices ** or tables and not the table/index that is actually being moved. ** We must continue looping until all tables and indices with @@ -3257,7 +3276,7 @@ void sqlite3RootPageMoved(sqlite3 *db, int iDb, Pgno iFrom, Pgno iTo){ ** Also write code to modify the sqlite_schema table and internal schema ** if a root-page of another table is moved by the btree-layer whilst ** erasing iTable (this can happen with an auto-vacuum database). -*/ +*/ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); @@ -3274,7 +3293,7 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ ** is in register NNN. See grammar rules associated with the TK_REGISTER ** token for additional information. */ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "UPDATE %Q." LEGACY_SCHEMA_TABLE " SET rootpage=%d WHERE #%d AND rootpage=#%d", pParse->db->aDb[iDb].zDbSName, iTable, r1, r1); @@ -3291,7 +3310,7 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ static void destroyTable(Parse *pParse, Table *pTab){ /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM ** is not defined), then it is important to call OP_Destroy on the - ** table and index root-pages in order, starting with the numerically + ** table and index root-pages in order, starting with the numerically ** largest root-page number. This guarantees that none of the root-pages ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the ** following were coded: @@ -3301,7 +3320,7 @@ static void destroyTable(Parse *pParse, Table *pTab){ ** OP_Destroy 5 0 ** ** and root page 5 happened to be the largest root-page number in the - ** database, then root page 5 would be moved to page 4 by the + ** database, then root page 5 would be moved to page 4 by the ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit ** a free-list page. */ @@ -3382,7 +3401,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ */ pTrigger = sqlite3TriggerList(pParse, pTab); while( pTrigger ){ - assert( pTrigger->pSchema==pTab->pSchema || + assert( pTrigger->pSchema==pTab->pSchema || pTrigger->pSchema==db->aDb[1].pSchema ); sqlite3DropTriggerPtr(pParse, pTrigger); pTrigger = pTrigger->pNext; @@ -3409,7 +3428,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ ** created in the temp database that refers to a table in another ** database. */ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "DELETE FROM %Q." LEGACY_SCHEMA_TABLE " WHERE tbl_name=%Q and type!='trigger'", pDb->zDbSName, pTab->zName); @@ -3479,6 +3498,8 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; assert( isView==0 || isView==LOCATE_VIEW ); @@ -3487,7 +3508,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ if( pTab==0 ){ if( noErr ){ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } goto exit_drop_table; @@ -3625,7 +3646,7 @@ void sqlite3CreateForeignKey( }else{ nCol = pFromCol->nExpr; } - nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + nByte = SZ_FKEY(nCol) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; @@ -3660,8 +3681,8 @@ void sqlite3CreateForeignKey( } } if( j>=p->nCol ){ - sqlite3ErrorMsg(pParse, - "unknown column \"%s\" in foreign key definition", + sqlite3ErrorMsg(pParse, + "unknown column \"%s\" in foreign key definition", pFromCol->a[i].zEName); goto fk_end; } @@ -3687,7 +3708,7 @@ void sqlite3CreateForeignKey( pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); - pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, + pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ @@ -3796,7 +3817,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, (int)tnum, iDb, + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, (int)tnum, iDb, (char *)pKey, P4_KEYINFO); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); @@ -3815,7 +3836,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ ** user function that throws an exception when it is evaluated. But the ** overhead of adding a statement journal to a CREATE INDEX statement is ** very small (since most of the pages written do not contain content that - ** needs to be restored if the statement aborts), so we call + ** needs to be restored if the statement aborts), so we call ** sqlite3MayAbort() for all CREATE INDEX statements. */ sqlite3MayAbort(pParse); addr2 = sqlite3VdbeCurrentAddr(v); @@ -3827,7 +3848,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables ** with DESC primary keys, since those indexes have there keys in ** a different order from the main table. - ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + ** See ticket: https://sqlite.org/src/info/bba7b69f9849b5bf */ sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); } @@ -3851,13 +3872,14 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ */ Index *sqlite3AllocateIndexObject( sqlite3 *db, /* Database connection */ - i16 nCol, /* Total number of columns in the index */ + int nCol, /* Total number of columns in the index */ int nExtra, /* Number of bytes of extra space to alloc */ char **ppExtra /* Pointer to the "extra" space */ ){ Index *p; /* Allocated index object */ - int nByte; /* Bytes of space for Index object + arrays */ + i64 nByte; /* Bytes of space for Index object + arrays */ + assert( nCol <= 2*db->aLimit[SQLITE_LIMIT_COLUMN] ); nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ @@ -3870,8 +3892,9 @@ Index *sqlite3AllocateIndexObject( p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; - p->nColumn = nCol; - p->nKeyCol = nCol - 1; + assert( nCol>0 ); + p->nColumn = (u16)nCol; + p->nKeyCol = (u16)(nCol - 1); *ppExtra = ((char*)p) + nByte; } return p; @@ -3888,7 +3911,7 @@ int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ for(i=0; inExpr; i++){ if( pList->a[i].fg.bNulls ){ u8 sf = pList->a[i].fg.sortFlags; - sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", + sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", (sf==0 || sf==3) ? "FIRST" : "LAST" ); return 1; @@ -3899,8 +3922,8 @@ int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ } /* -** Create a new index for an SQL table. pName1.pName2 is the name of the index -** and pTblList is the name of the table that is to be indexed. Both will +** Create a new index for an SQL table. pName1.pName2 is the name of the index +** and pTblList is the name of the table that is to be indexed. Both will ** be NULL for a primary key or an index that is created to satisfy a ** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable ** as the table to be indexed. pParse->pNewTable is a table that is @@ -3908,7 +3931,7 @@ int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ ** ** pList is a list of columns to be indexed. pList will be NULL if this ** is a primary key or unique-constraint on the most recent column added -** to the table currently under construction. +** to the table currently under construction. */ void sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ @@ -3960,7 +3983,7 @@ void sqlite3CreateIndex( */ if( pTblName!=0 ){ - /* Use the two-part index name to determine the database + /* Use the two-part index name to determine the database ** to search for the table. 'Fix' the table name to this db ** before looking up the table. */ @@ -3972,7 +3995,7 @@ void sqlite3CreateIndex( #ifndef SQLITE_OMIT_TEMPDB /* If the index name was unqualified, check if the table ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. + ** if initializing a database schema. */ if( !db->init.busy ){ pTab = sqlite3SrcListLookup(pParse, pTblName); @@ -3992,7 +4015,7 @@ void sqlite3CreateIndex( assert( db->mallocFailed==0 || pTab==0 ); if( pTab==0 ) goto exit_create_index; if( iDb==1 && db->aDb[iDb].pSchema!=pTab->pSchema ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "cannot create a TEMP index on non-TEMP table \"%s\"", pTab->zName); goto exit_create_index; @@ -4008,12 +4031,9 @@ void sqlite3CreateIndex( pDb = &db->aDb[iDb]; assert( pTab!=0 ); - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 && db->init.busy==0 && pTblName!=0 -#if SQLITE_USER_AUTHENTICATION - && sqlite3UserAuthTable(pTab->zName)==0 -#endif ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; @@ -4033,7 +4053,7 @@ void sqlite3CreateIndex( /* ** Find the name of the index. Make sure there is not already another - ** index or table with the same name. + ** index or table with the same name. ** ** Exception: If we are reading the names of permanent indices from the ** sqlite_schema table (because some other process changed the schema) and @@ -4133,8 +4153,8 @@ void sqlite3CreateIndex( } } - /* - ** Allocate the index structure. + /* + ** Allocate the index structure. */ nName = sqlite3Strlen30(zName); nExtraCol = pPk ? pPk->nKeyCol : 1; @@ -4255,7 +4275,7 @@ void sqlite3CreateIndex( int x = pPk->aiColumn[j]; assert( x>=0 ); if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ - pIndex->nColumn--; + pIndex->nColumn--; }else{ testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); pIndex->aiColumn[i] = x; @@ -4274,7 +4294,7 @@ void sqlite3CreateIndex( /* If this index contains every column of its table, then mark ** it as a covering index */ - assert( HasRowid(pTab) + assert( HasRowid(pTab) || pTab->iPKey<0 || sqlite3TableColumnToIndex(pIndex, pTab->iPKey)>=0 ); recomputeColumnsNotIndexed(pIndex); if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ @@ -4330,13 +4350,13 @@ void sqlite3CreateIndex( if( pIdx->onError!=pIndex->onError ){ /* This constraint creates the same index as a previous ** constraint specified somewhere in the CREATE TABLE statement. - ** However the ON CONFLICT clauses are different. If both this + ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "conflicting ON CONFLICT clauses specified", 0); } if( pIdx->onError==OE_Default ){ @@ -4357,7 +4377,7 @@ void sqlite3CreateIndex( if( !IN_RENAME_OBJECT ){ /* Link the new Index structure to its table and to the other - ** in-memory database structures. + ** in-memory database structures. */ assert( pParse->nErr==0 ); if( db->init.busy ){ @@ -4372,7 +4392,7 @@ void sqlite3CreateIndex( goto exit_create_index; } } - p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ @@ -4406,9 +4426,9 @@ void sqlite3CreateIndex( sqlite3BeginWriteOperation(pParse, 1, iDb); /* Create the rootpage for the index using CreateIndex. But before - ** doing so, code a Noop instruction and store its address in - ** Index.tnum. This is required in case this index is actually a - ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In ** that case the convertToWithoutRowidTable() routine will replace ** the Noop with a Goto to jump over the VDBE code generated below. */ pIndex->tnum = (Pgno)sqlite3VdbeAddOp0(v, OP_Noop); @@ -4432,7 +4452,7 @@ void sqlite3CreateIndex( /* Add an entry in sqlite_schema for this index */ - sqlite3NestedParse(pParse, + sqlite3NestedParse(pParse, "INSERT INTO %Q." LEGACY_SCHEMA_TABLE " VALUES('index',%Q,%Q,#%d,%Q);", db->aDb[iDb].zDbSName, pIndex->zName, @@ -4534,7 +4554,7 @@ void sqlite3DefaultRowEst(Index *pIdx){ /* Indexes with default row estimates should not have stat1 data */ assert( !pIdx->hasStat1 ); - /* Set the first entry (number of rows in the index) to the estimated + /* Set the first entry (number of rows in the index) to the estimated ** number of rows in the table, or half the number of rows in the table ** for a partial index. ** @@ -4578,15 +4598,17 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ } assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; } - pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].u4.zDatabase); if( pIndex==0 ){ if( !ifExists ){ sqlite3ErrorMsg(pParse, "no such index: %S", pName->a); }else{ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); sqlite3ForceNotReadOnly(pParse); } pParse->checkSchema = 1; @@ -4683,12 +4705,11 @@ IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ sqlite3 *db = pParse->db; int i; if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + pList = sqlite3DbMallocZero(db, SZ_IDLIST(1)); if( pList==0 ) return 0; }else{ IdList *pNew; - pNew = sqlite3DbRealloc(db, pList, - sizeof(IdList) + pList->nId*sizeof(pList->a)); + pNew = sqlite3DbRealloc(db, pList, SZ_IDLIST(pList->nId+1)); if( pNew==0 ){ sqlite3IdListDelete(db, pList); return 0; @@ -4710,7 +4731,6 @@ void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; assert( db!=0 ); if( pList==0 ) return; - assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; inId; i++){ sqlite3DbFree(db, pList->a[i].zName); } @@ -4788,8 +4808,7 @@ SrcList *sqlite3SrcListEnlarge( return 0; } if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; - pNew = sqlite3DbRealloc(db, pSrc, - sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + pNew = sqlite3DbRealloc(db, pSrc, SZ_SRCLIST(nAlloc)); if( pNew==0 ){ assert( db->mallocFailed ); return 0; @@ -4830,7 +4849,7 @@ SrcList *sqlite3SrcListEnlarge( ** database name prefix. Like this: "database.table". The pDatabase ** points to the table name and the pTable points to the database name. ** The SrcList.a[].zName field is filled with the table name which might -** come from pTable (if pDatabase is NULL) or from pDatabase. +** come from pTable (if pDatabase is NULL) or from pDatabase. ** SrcList.a[].zDatabase is filled with the database name from pTable, ** or with NULL if no database is specified. ** @@ -4864,7 +4883,7 @@ SrcList *sqlite3SrcListAppend( assert( pParse->db!=0 ); db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(pParse->db, SZ_SRCLIST(1)); if( pList==0 ) return 0; pList->nAlloc = 1; pList->nSrc = 1; @@ -4883,12 +4902,14 @@ SrcList *sqlite3SrcListAppend( if( pDatabase && pDatabase->z==0 ){ pDatabase = 0; } + assert( pItem->fg.fixedSchema==0 ); + assert( pItem->fg.isSubquery==0 ); if( pDatabase ){ pItem->zName = sqlite3NameFromToken(db, pDatabase); - pItem->zDatabase = sqlite3NameFromToken(db, pTable); + pItem->u4.zDatabase = sqlite3NameFromToken(db, pTable); }else{ pItem->zName = sqlite3NameFromToken(db, pTable); - pItem->zDatabase = 0; + pItem->u4.zDatabase = 0; } return pList; } @@ -4904,13 +4925,40 @@ void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; - if( pItem->pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + if( pItem->fg.isSubquery ){ + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + assert( pItem->u4.pSubq->pSelect->pSrc!=0 ); + sqlite3SrcListAssignCursors(pParse, pItem->u4.pSubq->pSelect->pSrc); } } } } +/* +** Delete a Subquery object and its substructure. +*/ +void sqlite3SubqueryDelete(sqlite3 *db, Subquery *pSubq){ + assert( pSubq!=0 && pSubq->pSelect!=0 ); + sqlite3SelectDelete(db, pSubq->pSelect); + sqlite3DbFree(db, pSubq); +} + +/* +** Remove a Subquery from a SrcItem. Return the associated Select object. +** The returned Select becomes the responsibility of the caller. +*/ +Select *sqlite3SubqueryDetach(sqlite3 *db, SrcItem *pItem){ + Select *pSel; + assert( pItem!=0 ); + assert( pItem->fg.isSubquery ); + pSel = pItem->u4.pSubq->pSelect; + sqlite3DbFree(db, pItem->u4.pSubq); + pItem->u4.pSubq = 0; + pItem->fg.isSubquery = 0; + return pSel; +} + /* ** Delete an entire SrcList including all its substructure. */ @@ -4920,13 +4968,24 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - if( pItem->zDatabase ) sqlite3DbNNFreeNN(db, pItem->zDatabase); + + /* Check invariants on SrcItem */ + assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc ); + assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy ); + assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery ); + assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && + pItem->u4.pSubq->pSelect!=0) ); + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); + if( pItem->fg.isSubquery ){ + sqlite3SubqueryDelete(db, pItem->u4.pSubq); + }else if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3DbNNFreeNN(db, pItem->u4.zDatabase); + } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); - sqlite3DeleteTable(db, pItem->pTab); - if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); + sqlite3DeleteTable(db, pItem->pSTab); if( pItem->fg.isUsing ){ sqlite3IdListDelete(db, pItem->u3.pUsing); }else if( pItem->u3.pOn ){ @@ -4936,6 +4995,54 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ sqlite3DbNNFreeNN(db, pList); } +/* +** Attach a Subquery object to pItem->uv.pSubq. Set the +** pSelect value but leave all the other values initialized +** to zero. +** +** A copy of the Select object is made if dupSelect is true, and the +** SrcItem takes responsibility for deleting the copy. If dupSelect is +** false, ownership of the Select passes to the SrcItem. Either way, +** the SrcItem will take responsibility for deleting the Select. +** +** When dupSelect is zero, that means the Select might get deleted right +** away if there is an OOM error. Beware. +** +** Return non-zero on success. Return zero on an OOM error. +*/ +int sqlite3SrcItemAttachSubquery( + Parse *pParse, /* Parsing context */ + SrcItem *pItem, /* Item to which the subquery is to be attached */ + Select *pSelect, /* The subquery SELECT. Must be non-NULL */ + int dupSelect /* If true, attach a copy of pSelect, not pSelect itself.*/ +){ + Subquery *p; + assert( pSelect!=0 ); + assert( pItem->fg.isSubquery==0 ); + if( pItem->fg.fixedSchema ){ + pItem->u4.pSchema = 0; + pItem->fg.fixedSchema = 0; + }else if( pItem->u4.zDatabase!=0 ){ + sqlite3DbFree(pParse->db, pItem->u4.zDatabase); + pItem->u4.zDatabase = 0; + } + if( dupSelect ){ + pSelect = sqlite3SelectDup(pParse->db, pSelect, 0); + if( pSelect==0 ) return 0; + } + p = pItem->u4.pSubq = sqlite3DbMallocRawNN(pParse->db, sizeof(Subquery)); + if( p==0 ){ + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + } + pItem->fg.isSubquery = 1; + p->pSelect = pSelect; + assert( offsetof(Subquery, pSelect)==0 ); + memset(((char*)p)+sizeof(p->pSelect), 0, sizeof(*p)-sizeof(p->pSelect)); + return 1; +} + + /* ** This routine is called by the parser to add a new term to the ** end of a growing FROM clause. The "p" parameter is the part of @@ -4964,7 +5071,7 @@ SrcList *sqlite3SrcListAppendFromTerm( SrcItem *pItem; sqlite3 *db = pParse->db; if( !p && pOnUsing!=0 && (pOnUsing->pOn || pOnUsing->pUsing) ){ - sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", + sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", (pOnUsing->pOn ? "ON" : "USING") ); goto append_from_error; @@ -4985,10 +5092,12 @@ SrcList *sqlite3SrcListAppendFromTerm( if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); } + assert( pSubquery==0 || pDatabase==0 ); if( pSubquery ){ - pItem->pSelect = pSubquery; - if( pSubquery->selFlags & SF_NestedFrom ){ - pItem->fg.isNestedFrom = 1; + if( sqlite3SrcItemAttachSubquery(pParse, pItem, pSubquery, 0) ){ + if( pSubquery->selFlags & SF_NestedFrom ){ + pItem->fg.isNestedFrom = 1; + } } } assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 ); @@ -5011,7 +5120,7 @@ SrcList *sqlite3SrcListAppendFromTerm( } /* -** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** Add an INDEXED BY or NOT INDEXED clause to the most recently added ** element of the source-list passed as the second argument. */ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ @@ -5024,7 +5133,7 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ assert( pItem->fg.isIndexedBy==0 ); assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ - /* A "NOT INDEXED" clause was supplied. See parse.y + /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ pItem->fg.notIndexed = 1; }else{ @@ -5039,18 +5148,24 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ ** Append the contents of SrcList p2 to SrcList p1 and return the resulting ** SrcList. Or, if an error occurs, return NULL. In all cases, p1 and p2 ** are deleted by this function. -*/ +*/ SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ - assert( p1 && p1->nSrc==1 ); + assert( p1 ); + assert( p2 || pParse->nErr ); + assert( p2==0 || p2->nSrc>=1 ); + testcase( p1->nSrc==0 ); if( p2 ){ - SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + int nOld = p1->nSrc; + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, nOld); if( pNew==0 ){ sqlite3SrcListDelete(pParse->db, p2); }else{ p1 = pNew; - memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + memcpy(&p1->a[nOld], p2->a, p2->nSrc*sizeof(SrcItem)); + assert( nOld==1 || (p2->a[0].fg.jointype & JT_LTORJ)==0 ); + assert( p1->nSrc>=1 ); + p1->a[0].fg.jointype |= (JT_LTORJ & p2->a[0].fg.jointype); sqlite3DbFree(pParse->db, p2); - p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype); } } return p1; @@ -5165,7 +5280,7 @@ void sqlite3EndTransaction(Parse *pParse, int eType){ assert( pParse->db!=0 ); assert( eType==TK_COMMIT || eType==TK_END || eType==TK_ROLLBACK ); isRollback = eType==TK_ROLLBACK; - if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, isRollback ? "ROLLBACK" : "COMMIT", 0, 0) ){ return; } @@ -5177,7 +5292,7 @@ void sqlite3EndTransaction(Parse *pParse, int eType){ /* ** This function is called by the parser when it parses a command to create, -** release or rollback an SQL savepoint. +** release or rollback an SQL savepoint. */ void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ char *zName = sqlite3NameFromToken(pParse->db, pName); @@ -5204,7 +5319,7 @@ int sqlite3OpenTempDatabase(Parse *pParse){ if( db->aDb[1].pBt==0 && !pParse->explain ){ int rc; Btree *pBt; - static const int flags = + static const int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | @@ -5252,7 +5367,7 @@ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ /* -** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each +** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each ** attached database. Otherwise, invoke it for the database named zDb only. */ void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ @@ -5298,9 +5413,9 @@ void sqlite3MultiWrite(Parse *pParse){ pToplevel->isMultiWrite = 1; } -/* +/* ** The code generator calls this routine if is discovers that it is -** possible to abort a statement prior to completion. In order to +** possible to abort a statement prior to completion. In order to ** perform this abort without corrupting the database, we need to make ** sure that the statement is protected by a statement transaction. ** @@ -5309,7 +5424,7 @@ void sqlite3MultiWrite(Parse *pParse){ ** such that the abort must occur after the multiwrite. This makes ** some statements involving the REPLACE conflict resolution algorithm ** go a little faster. But taking advantage of this time dependency -** makes it more difficult to prove that the code is correct (in +** makes it more difficult to prove that the code is correct (in ** particular, it prevents us from writing an effective ** implementation of sqlite3AssertMayAbort()) and so we have chosen ** to take the safe route and skip the optimization. @@ -5356,7 +5471,7 @@ void sqlite3UniqueConstraint( StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); if( pIdx->aColExpr ){ sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); @@ -5372,8 +5487,8 @@ void sqlite3UniqueConstraint( } } zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, - IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + sqlite3HaltConstraint(pParse, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY : SQLITE_CONSTRAINT_UNIQUE, onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); } @@ -5385,7 +5500,7 @@ void sqlite3UniqueConstraint( void sqlite3RowidConstraint( Parse *pParse, /* Parsing context */ int onError, /* Conflict resolution algorithm */ - Table *pTab /* The table with the non-unique rowid */ + Table *pTab /* The table with the non-unique rowid */ ){ char *zMsg; int rc; @@ -5514,7 +5629,7 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zDbSName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -5524,6 +5639,7 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ pIndex = sqlite3FindIndex(db, z, zDb); sqlite3DbFree(db, z); if( pIndex ){ + iDb = sqlite3SchemaToIndex(db, pIndex->pTable->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3RefillIndex(pParse, pIndex, -1); return; @@ -5560,14 +5676,19 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ } if( pParse->nErr ){ assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); - if( pIdx->bNoQuery==0 ){ + if( pIdx->bNoQuery==0 + && sqlite3HashFind(&pIdx->pSchema->idxHash, pIdx->zName) + ){ /* Deactivate the index because it contains an unknown collating ** sequence. The only way to reactive the index is to reload the ** schema. Adding the missing collating sequence later does not ** reactive the index. The application had the chance to register ** the missing index using the collation-needed callback. For ** simplicity, SQLite will not give the application a second chance. - */ + ** + ** Except, do not do this if the index is not in the schema hash + ** table. In this case the index is currently being constructed + ** by a CREATE INDEX statement, and retrying will not help. */ pIdx->bNoQuery = 1; pParse->rc = SQLITE_ERROR_RETRY; } @@ -5627,9 +5748,9 @@ void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ sqlite3DbFree(db, pCte); } -/* -** This routine is invoked once per CTE by the parser while parsing a -** WITH clause. The CTE described by teh third argument is added to +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. The CTE described by the third argument is added to ** the WITH clause of the second argument. If the second argument is ** NULL, then a new WITH argument is created. */ @@ -5659,10 +5780,9 @@ With *sqlite3WithAdd( } if( pWith ){ - sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); - pNew = sqlite3DbRealloc(db, pWith, nByte); + pNew = sqlite3DbRealloc(db, pWith, SZ_WITH(pWith->nCte+1)); }else{ - pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + pNew = sqlite3DbMallocZero(db, SZ_WITH(1)); } assert( (pNew!=0 && zName!=0) || db->mallocFailed ); @@ -5689,4 +5809,7 @@ void sqlite3WithDelete(sqlite3 *db, With *pWith){ sqlite3DbFree(db, pWith); } } +void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){ + sqlite3WithDelete(db, (With*)pWith); +} #endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/src/callback.c b/src/callback.c index c36d51a4ec..e6418097f6 100644 --- a/src/callback.c +++ b/src/callback.c @@ -302,12 +302,18 @@ static int matchQuality( u8 enc /* Desired text encoding */ ){ int match; - assert( p->nArg>=-1 ); + assert( p->nArg>=(-4) && p->nArg!=(-2) ); + assert( nArg>=(-2) ); /* Wrong number of arguments means "no match" */ if( p->nArg!=nArg ){ - if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( nArg==(-2) ) return p->xSFunc==0 ? 0 : FUNC_PERFECT_MATCH; if( p->nArg>=0 ) return 0; + /* Special p->nArg values available to built-in functions only: + ** -3 1 or more arguments required + ** -4 2 or more arguments required + */ + if( p->nArg<(-2) && nArg<(-2-p->nArg) ) return 0; } /* Give a better score to a function with a specific number of arguments @@ -501,6 +507,7 @@ void sqlite3SchemaClear(void *p){ for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } + sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ diff --git a/ext/misc/carray.c b/src/carray.c similarity index 81% rename from ext/misc/carray.c rename to src/carray.c index 709c894f27..154d107ddf 100644 --- a/ext/misc/carray.c +++ b/src/carray.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** -** This file demonstrates how to create a table-valued-function that +** This file implements a table-valued-function that ** returns the values in a C-language array. ** Examples: ** @@ -52,11 +52,9 @@ ** as the number of elements in the array. The virtual table steps through ** the array, element by element. */ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 -#include -#include -#ifdef _WIN32 +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) +#include "sqliteInt.h" +#if defined(_WIN32) || defined(__RTP__) || defined(_WRS_KERNEL) struct iovec { void *iov_base; size_t iov_len; @@ -64,33 +62,13 @@ SQLITE_EXTENSION_INIT1 #else # include #endif - -/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). -** Must exactly match the definitions in carray.h. -*/ -#ifndef CARRAY_INT32 -# define CARRAY_INT32 0 /* Data is 32-bit signed integers */ -# define CARRAY_INT64 1 /* Data is 64-bit signed integers */ -# define CARRAY_DOUBLE 2 /* Data is doubles */ -# define CARRAY_TEXT 3 /* Data is char* */ -# define CARRAY_BLOB 4 /* Data is struct iovec* */ -#endif - -#ifndef SQLITE_API -# ifdef _WIN32 -# define SQLITE_API __declspec(dllexport) -# else -# define SQLITE_API -# endif -#endif - -#ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Names of allowed datatypes */ -static const char *azType[] = { "int32", "int64", "double", "char*", - "struct iovec" }; +static const char *azCarrayType[] = { + "int32", "int64", "double", "char*", "struct iovec" +}; /* ** Structure used to hold the sqlite3_carray_bind() information @@ -209,7 +187,7 @@ static int carrayColumn( case CARRAY_COLUMN_POINTER: return SQLITE_OK; case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; case CARRAY_COLUMN_CTYPE: { - sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC); + sqlite3_result_text(ctx, azCarrayType[pCur->eType], -1, SQLITE_STATIC); return SQLITE_OK; } default: { @@ -234,10 +212,11 @@ static int carrayColumn( sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); return SQLITE_OK; } - case CARRAY_BLOB: { + default: { const struct iovec *p = (struct iovec*)pCur->pPtr; + assert( pCur->eType==CARRAY_BLOB ); sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base, - (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); + (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); return SQLITE_OK; } } @@ -296,10 +275,10 @@ static int carrayFilter( }else{ unsigned char i; const char *zType = (const char*)sqlite3_value_text(argv[2]); - for(i=0; i=sizeof(azType)/sizeof(azType[0]) ){ + if( i>=sizeof(azCarrayType)/sizeof(azCarrayType[0]) ){ pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( "unknown datatype: %Q", zType); return SQLITE_ERROR; @@ -342,12 +321,14 @@ static int carrayBestIndex( int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + unsigned seen = 0; /* Bitmask of == constrainted columns */ const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn>=0 ) seen |= 1 << pConstraint->iColumn; + if( pConstraint->usable==0 ) continue; switch( pConstraint->iColumn ){ case CARRAY_COLUMN_POINTER: ptrIdx = i; @@ -374,7 +355,15 @@ static int carrayBestIndex( pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; pIdxInfo->idxNum = 3; + }else if( seen & (1<estimatedCost = (double)2147483647; @@ -409,6 +398,11 @@ static sqlite3_module carrayModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadow */ + 0 /* xIntegrity */ }; /* @@ -434,42 +428,53 @@ SQLITE_API int sqlite3_carray_bind( int mFlags, void (*xDestroy)(void*) ){ - carray_bind *pNew; + carray_bind *pNew = 0; int i; + int rc = SQLITE_OK; + + /* Ensure that the mFlags value is acceptable. */ + assert( CARRAY_INT32==0 && CARRAY_INT64==1 && CARRAY_DOUBLE==2 ); + assert( CARRAY_TEXT==3 && CARRAY_BLOB==4 ); + if( mFlagsCARRAY_BLOB ){ + rc = SQLITE_ERROR; + goto carray_bind_error; + } + pNew = sqlite3_malloc64(sizeof(*pNew)); if( pNew==0 ){ - if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ - xDestroy(aData); - } - return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + goto carray_bind_error; } + pNew->nData = nData; pNew->mFlags = mFlags; if( xDestroy==SQLITE_TRANSIENT ){ sqlite3_int64 sz = nData; - switch( mFlags & 0x07 ){ + switch( mFlags ){ case CARRAY_INT32: sz *= 4; break; case CARRAY_INT64: sz *= 8; break; case CARRAY_DOUBLE: sz *= 8; break; case CARRAY_TEXT: sz *= sizeof(char*); break; - case CARRAY_BLOB: sz *= sizeof(struct iovec); break; + default: sz *= sizeof(struct iovec); break; } - if( (mFlags & 0x07)==CARRAY_TEXT ){ + if( mFlags==CARRAY_TEXT ){ for(i=0; iaData = sqlite3_malloc64( sz ); if( pNew->aData==0 ){ - sqlite3_free(pNew); - return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + goto carray_bind_error; } - if( (mFlags & 0x07)==CARRAY_TEXT ){ + + if( mFlags==CARRAY_TEXT ){ char **az = (char**)pNew->aData; char *z = (char*)&az[nData]; for(i=0; iaData; unsigned char *z = (unsigned char*)&p[nData]; for(i=0; ixDel = xDestroy; } return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); + + carray_bind_error: + if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ + xDestroy(aData); + } + sqlite3_free(pNew); + return rc; } - /* -** For testing purpose in the TCL test harness, we need a method for -** setting the pointer value. The inttoptr(X) SQL function accomplishes -** this. Tcl script will bind an integer to X and the inttoptr() SQL -** function will use sqlite3_result_pointer() to convert that integer into -** a pointer. -** -** This is for testing on TCL only. +** Invoke this routine to register the carray() function. */ -#ifdef SQLITE_TEST -static void inttoptrFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - void *p; - sqlite3_int64 i64; - i64 = sqlite3_value_int64(argv[0]); - if( sizeof(i64)==sizeof(p) ){ - memcpy(&p, &i64, sizeof(p)); - }else{ - int i32 = i64 & 0xffffffff; - memcpy(&p, &i32, sizeof(p)); - } - sqlite3_result_pointer(context, p, "carray", 0); +Module *sqlite3CarrayRegister(sqlite3 *db){ + return sqlite3VtabCreateModule(db, "carray", &carrayModule, 0, 0); } -#endif /* SQLITE_TEST */ -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -SQLITE_API int sqlite3_carray_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); -#ifndef SQLITE_OMIT_VIRTUALTABLE - rc = sqlite3_create_module(db, "carray", &carrayModule, 0); -#ifdef SQLITE_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0, - inttoptrFunc, 0, 0); - } -#endif /* SQLITE_TEST */ -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - return rc; -} +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) */ diff --git a/src/crypto.c b/src/crypto.c deleted file mode 100644 index 9c722fd42e..0000000000 --- a/src/crypto.c +++ /dev/null @@ -1,1270 +0,0 @@ -/* -** SQLCipher -** http://sqlcipher.net -** -** Copyright (c) 2008 - 2013, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC - -#include -#include "sqlcipher.h" -#include "crypto.h" - -#ifdef SQLCIPHER_EXT -#include "sqlcipher_ext.h" -#endif - -void sqlcipher_vdbe_return_string(Parse *pParse, const char *zLabel, const char *value, int value_type){ - Vdbe *v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, value, value_type); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); -} - -static int codec_set_btree_to_codec_pagesize(sqlite3 *db, Db *pDb, codec_ctx *ctx) { - int rc, page_sz, reserve_sz; - - page_sz = sqlcipher_codec_ctx_get_pagesize(ctx); - reserve_sz = sqlcipher_codec_ctx_get_reservesize(ctx); - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "codec_set_btree_to_codec_pagesize: sqlite3BtreeSetPageSize() size=%d reserve=%d", page_sz, reserve_sz); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "codec_set_btree_to_codec_pagesize: entering database mutex %p", db->mutex); - sqlite3_mutex_enter(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "codec_set_btree_to_codec_pagesize: entered database mutex %p", db->mutex); - db->nextPagesize = page_sz; - - /* before forcing the page size we need to unset the BTS_PAGESIZE_FIXED flag, else - sqliteBtreeSetPageSize will block the change */ - pDb->pBt->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; - rc = sqlite3BtreeSetPageSize(pDb->pBt, page_sz, reserve_sz, 0); - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "codec_set_btree_to_codec_pagesize: sqlite3BtreeSetPageSize returned %d", rc); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "codec_set_btree_to_codec_pagesize: leaving database mutex %p", db->mutex); - sqlite3_mutex_leave(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "codec_set_btree_to_codec_pagesize: left database mutex %p", db->mutex); - - return rc; -} - -static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) { - struct Db *pDb = &db->aDb[nDb]; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "codec_set_pass_key: db=%p nDb=%d for_ctx=%d", db, nDb, for_ctx); - if(pDb->pBt) { - codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); - - if(ctx) { - return sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, for_ctx); - } else { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "codec_set_pass_key: error ocurred fetching codec from pager on db %d", nDb); - return SQLITE_ERROR; - } - } - sqlcipher_log(SQLCIPHER_LOG_ERROR, "codec_set_pass_key: no btree present on db %d", nDb); - return SQLITE_ERROR; -} - -int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const char *zRight) { - struct Db *pDb = &db->aDb[iDb]; - codec_ctx *ctx = NULL; - int rc; - - if(pDb->pBt) { - ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); - } - - if(sqlite3_stricmp(zLeft, "key") !=0 && sqlite3_stricmp(zLeft, "rekey") != 0) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_pragma: db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p", db, iDb, pParse, zLeft, zRight, ctx); - } - -#ifdef SQLCIPHER_EXT - if(sqlcipher_ext_pragma(db, iDb, pParse, zLeft, zRight)) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_pragma: PRAGMA handled by sqlcipher_ext_pragma"); - } else -#endif -#ifdef SQLCIPHER_TEST - if( sqlite3_stricmp(zLeft,"cipher_test_on")==0 ){ - if( zRight ) { - unsigned int flags = sqlcipher_get_test_flags(); - if(sqlite3_stricmp(zRight, "fail_encrypt")==0) { - SQLCIPHER_FLAG_SET(flags,TEST_FAIL_ENCRYPT); - } else - if(sqlite3_stricmp(zRight, "fail_decrypt")==0) { - SQLCIPHER_FLAG_SET(flags,TEST_FAIL_DECRYPT); - } else - if(sqlite3_stricmp(zRight, "fail_migrate")==0) { - SQLCIPHER_FLAG_SET(flags,TEST_FAIL_MIGRATE); - } - sqlcipher_set_test_flags(flags); - } - } else - if( sqlite3_stricmp(zLeft,"cipher_test_off")==0 ){ - if( zRight ) { - unsigned int flags = sqlcipher_get_test_flags(); - if(sqlite3_stricmp(zRight, "fail_encrypt")==0) { - SQLCIPHER_FLAG_UNSET(flags,TEST_FAIL_ENCRYPT); - } else - if(sqlite3_stricmp(zRight, "fail_decrypt")==0) { - SQLCIPHER_FLAG_UNSET(flags,TEST_FAIL_DECRYPT); - } else - if(sqlite3_stricmp(zRight, "fail_migrate")==0) { - SQLCIPHER_FLAG_UNSET(flags,TEST_FAIL_MIGRATE); - } - sqlcipher_set_test_flags(flags); - } - } else - if( sqlite3_stricmp(zLeft,"cipher_test")==0 ){ - char *flags = sqlite3_mprintf("%u", sqlcipher_get_test_flags()); - sqlcipher_vdbe_return_string(pParse, "cipher_test", flags, P4_DYNAMIC); - }else - if( sqlite3_stricmp(zLeft,"cipher_test_rand")==0 ){ - if( zRight ) { - int rand = atoi(zRight); - sqlcipher_set_test_rand(rand); - } else { - char *rand = sqlite3_mprintf("%d", sqlcipher_get_test_rand()); - sqlcipher_vdbe_return_string(pParse, "cipher_test_rand", rand, P4_DYNAMIC); - } - } else -#endif - if( sqlite3_stricmp(zLeft, "cipher_fips_status")== 0 && !zRight ){ - if(ctx) { - char *fips_mode_status = sqlite3_mprintf("%d", sqlcipher_codec_fips_status(ctx)); - sqlcipher_vdbe_return_string(pParse, "cipher_fips_status", fips_mode_status, P4_DYNAMIC); - } - } else - if( sqlite3_stricmp(zLeft, "cipher_store_pass")==0 && zRight ) { - if(ctx) { - char *deprecation = "PRAGMA cipher_store_pass is deprecated, please remove from use"; - sqlcipher_codec_set_store_pass(ctx, sqlite3GetBoolean(zRight, 1)); - sqlcipher_vdbe_return_string(pParse, "cipher_store_pass", deprecation, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, deprecation); - } - } else - if( sqlite3_stricmp(zLeft, "cipher_store_pass")==0 && !zRight ) { - if(ctx){ - char *store_pass_value = sqlite3_mprintf("%d", sqlcipher_codec_get_store_pass(ctx)); - sqlcipher_vdbe_return_string(pParse, "cipher_store_pass", store_pass_value, P4_DYNAMIC); - } - } - if( sqlite3_stricmp(zLeft, "cipher_profile")== 0 && zRight ){ - char *profile_status = sqlite3_mprintf("%d", sqlcipher_cipher_profile(db, zRight)); - sqlcipher_vdbe_return_string(pParse, "cipher_profile", profile_status, P4_DYNAMIC); - } else - if( sqlite3_stricmp(zLeft, "cipher_add_random")==0 && zRight ){ - if(ctx) { - char *add_random_status = sqlite3_mprintf("%d", sqlcipher_codec_add_random(ctx, zRight, sqlite3Strlen30(zRight))); - sqlcipher_vdbe_return_string(pParse, "cipher_add_random", add_random_status, P4_DYNAMIC); - } - } else - if( sqlite3_stricmp(zLeft, "cipher_migrate")==0 && !zRight ){ - if(ctx){ - int status = sqlcipher_codec_ctx_migrate(ctx); - char *migrate_status = sqlite3_mprintf("%d", status); - sqlcipher_vdbe_return_string(pParse, "cipher_migrate", migrate_status, P4_DYNAMIC); - if(status != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_pragma: error occurred during cipher_migrate: %d", status); - sqlcipher_codec_ctx_set_error(ctx, status); - } - } - } else - if( sqlite3_stricmp(zLeft, "cipher_provider")==0 && !zRight ){ - if(ctx) { sqlcipher_vdbe_return_string(pParse, "cipher_provider", - sqlcipher_codec_get_cipher_provider(ctx), P4_TRANSIENT); - } - } else - if( sqlite3_stricmp(zLeft, "cipher_provider_version")==0 && !zRight){ - if(ctx) { sqlcipher_vdbe_return_string(pParse, "cipher_provider_version", - sqlcipher_codec_get_provider_version(ctx), P4_TRANSIENT); - } - } else - if( sqlite3_stricmp(zLeft, "cipher_version")==0 && !zRight ){ - sqlcipher_vdbe_return_string(pParse, "cipher_version", sqlcipher_version(), P4_DYNAMIC); - }else - if( sqlite3_stricmp(zLeft, "cipher")==0 ){ - if(ctx) { - if( zRight ) { - const char* message = "PRAGMA cipher is no longer supported."; - sqlcipher_vdbe_return_string(pParse, "cipher", message, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, message); - }else { - sqlcipher_vdbe_return_string(pParse, "cipher", sqlcipher_codec_ctx_get_cipher(ctx), P4_TRANSIENT); - } - } - }else - if( sqlite3_stricmp(zLeft, "rekey_cipher")==0 && zRight ){ - const char* message = "PRAGMA rekey_cipher is no longer supported."; - sqlcipher_vdbe_return_string(pParse, "rekey_cipher", message, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, message); - }else - if( sqlite3_stricmp(zLeft,"cipher_default_kdf_iter")==0 ){ - if( zRight ) { - sqlcipher_set_default_kdf_iter(atoi(zRight)); /* change default KDF iterations */ - } else { - char *kdf_iter = sqlite3_mprintf("%d", sqlcipher_get_default_kdf_iter()); - sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_iter", kdf_iter, P4_DYNAMIC); - } - }else - if( sqlite3_stricmp(zLeft, "kdf_iter")==0 ){ - if(ctx) { - if( zRight ) { - sqlcipher_codec_ctx_set_kdf_iter(ctx, atoi(zRight)); /* change of RW PBKDF2 iteration */ - } else { - char *kdf_iter = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_kdf_iter(ctx)); - sqlcipher_vdbe_return_string(pParse, "kdf_iter", kdf_iter, P4_DYNAMIC); - } - } - }else - if( sqlite3_stricmp(zLeft, "fast_kdf_iter")==0){ - if(ctx) { - if( zRight ) { - char *deprecation = "PRAGMA fast_kdf_iter is deprecated, please remove from use"; - sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, atoi(zRight)); /* change of RW PBKDF2 iteration */ - sqlcipher_vdbe_return_string(pParse, "fast_kdf_iter", deprecation, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, deprecation); - } else { - char *fast_kdf_iter = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_fast_kdf_iter(ctx)); - sqlcipher_vdbe_return_string(pParse, "fast_kdf_iter", fast_kdf_iter, P4_DYNAMIC); - } - } - }else - if( sqlite3_stricmp(zLeft, "rekey_kdf_iter")==0 && zRight ){ - const char* message = "PRAGMA rekey_kdf_iter is no longer supported."; - sqlcipher_vdbe_return_string(pParse, "rekey_kdf_iter", message, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, message); - }else - if( sqlite3_stricmp(zLeft,"cipher_page_size")==0 ){ - if(ctx) { - if( zRight ) { - int size = atoi(zRight); - rc = sqlcipher_codec_ctx_set_pagesize(ctx, size); - if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); - rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx); - if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); - } else { - char * page_size = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_pagesize(ctx)); - sqlcipher_vdbe_return_string(pParse, "cipher_page_size", page_size, P4_DYNAMIC); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_page_size")==0 ){ - if( zRight ) { - sqlcipher_set_default_pagesize(atoi(zRight)); - } else { - char *default_page_size = sqlite3_mprintf("%d", sqlcipher_get_default_pagesize()); - sqlcipher_vdbe_return_string(pParse, "cipher_default_page_size", default_page_size, P4_DYNAMIC); - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_use_hmac")==0 ){ - if( zRight ) { - sqlcipher_set_default_use_hmac(sqlite3GetBoolean(zRight,1)); - } else { - char *default_use_hmac = sqlite3_mprintf("%d", sqlcipher_get_default_use_hmac()); - sqlcipher_vdbe_return_string(pParse, "cipher_default_use_hmac", default_use_hmac, P4_DYNAMIC); - } - }else - if( sqlite3_stricmp(zLeft,"cipher_use_hmac")==0 ){ - if(ctx) { - if( zRight ) { - rc = sqlcipher_codec_ctx_set_use_hmac(ctx, sqlite3GetBoolean(zRight,1)); - if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); - /* since the use of hmac has changed, the page size may also change */ - rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx); - if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc); - } else { - char *hmac_flag = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_use_hmac(ctx)); - sqlcipher_vdbe_return_string(pParse, "cipher_use_hmac", hmac_flag, P4_DYNAMIC); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_hmac_pgno")==0 ){ - if(ctx) { - if(zRight) { - char *deprecation = "PRAGMA cipher_hmac_pgno is deprecated, please remove from use"; - /* clear both pgno endian flags */ - if(sqlite3_stricmp(zRight, "le") == 0) { - SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_BE_PGNO); - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_LE_PGNO); - } else if(sqlite3_stricmp(zRight, "be") == 0) { - SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_LE_PGNO); - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_BE_PGNO); - } else if(sqlite3_stricmp(zRight, "native") == 0) { - SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_LE_PGNO); - SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_BE_PGNO); - } - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", deprecation, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, deprecation); - - } else { - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_LE_PGNO)) { - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", "le", P4_TRANSIENT); - } else if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_BE_PGNO)) { - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", "be", P4_TRANSIENT); - } else { - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", "native", P4_TRANSIENT); - } - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_hmac_salt_mask")==0 ){ - if(ctx) { - if(zRight) { - char *deprecation = "PRAGMA cipher_hmac_salt_mask is deprecated, please remove from use"; - if (sqlite3StrNICmp(zRight ,"x'", 2) == 0 && sqlite3Strlen30(zRight) == 5) { - unsigned char mask = 0; - const unsigned char *hex = (const unsigned char *)zRight+2; - cipher_hex2bin(hex,2,&mask); - sqlcipher_set_hmac_salt_mask(mask); - } - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_salt_mask", deprecation, P4_TRANSIENT); - sqlite3_log(SQLITE_WARNING, deprecation); - } else { - char *hmac_salt_mask = sqlite3_mprintf("%02x", sqlcipher_get_hmac_salt_mask()); - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_salt_mask", hmac_salt_mask, P4_DYNAMIC); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_plaintext_header_size")==0 ){ - if(ctx) { - if( zRight ) { - int size = atoi(zRight); - /* deliberately ignore result code, if size is invalid it will be set to -1 - and trip the error later in the codec */ - sqlcipher_codec_ctx_set_plaintext_header_size(ctx, size); - } else { - char *size = sqlite3_mprintf("%d", sqlcipher_codec_ctx_get_plaintext_header_size(ctx)); - sqlcipher_vdbe_return_string(pParse, "cipher_plaintext_header_size", size, P4_DYNAMIC); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_plaintext_header_size")==0 ){ - if( zRight ) { - sqlcipher_set_default_plaintext_header_size(atoi(zRight)); - } else { - char *size = sqlite3_mprintf("%d", sqlcipher_get_default_plaintext_header_size()); - sqlcipher_vdbe_return_string(pParse, "cipher_default_plaintext_header_size", size, P4_DYNAMIC); - } - }else - if( sqlite3_stricmp(zLeft,"cipher_salt")==0 ){ - if(ctx) { - if(zRight) { - if (sqlite3StrNICmp(zRight ,"x'", 2) == 0 && sqlite3Strlen30(zRight) == (FILE_HEADER_SZ*2)+3) { - unsigned char *salt = (unsigned char*) sqlite3_malloc(FILE_HEADER_SZ); - const unsigned char *hex = (const unsigned char *)zRight+2; - cipher_hex2bin(hex,FILE_HEADER_SZ*2,salt); - sqlcipher_codec_ctx_set_kdf_salt(ctx, salt, FILE_HEADER_SZ); - sqlite3_free(salt); - } - } else { - void *salt; - char *hexsalt = (char*) sqlite3_malloc((FILE_HEADER_SZ*2)+1); - if((rc = sqlcipher_codec_ctx_get_kdf_salt(ctx, &salt)) == SQLITE_OK) { - cipher_bin2hex(salt, FILE_HEADER_SZ, hexsalt); - sqlcipher_vdbe_return_string(pParse, "cipher_salt", hexsalt, P4_DYNAMIC); - } else { - sqlite3_free(hexsalt); - sqlcipher_codec_ctx_set_error(ctx, rc); - } - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_hmac_algorithm")==0 ){ - if(ctx) { - if(zRight) { - rc = SQLITE_ERROR; - if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA1_LABEL) == 0) { - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA256_LABEL) == 0) { - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA256); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA512_LABEL) == 0) { - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA512); - } - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - } else { - int algorithm = sqlcipher_codec_ctx_get_hmac_algorithm(ctx); - if(algorithm == SQLCIPHER_HMAC_SHA1) { - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_algorithm", SQLCIPHER_HMAC_SHA1_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_HMAC_SHA256) { - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_algorithm", SQLCIPHER_HMAC_SHA256_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_HMAC_SHA512) { - sqlcipher_vdbe_return_string(pParse, "cipher_hmac_algorithm", SQLCIPHER_HMAC_SHA512_LABEL, P4_TRANSIENT); - } - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_hmac_algorithm")==0 ){ - if(zRight) { - rc = SQLITE_ERROR; - if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA1_LABEL) == 0) { - rc = sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA1); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA256_LABEL) == 0) { - rc = sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA256); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA512_LABEL) == 0) { - rc = sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA512); - } - } else { - int algorithm = sqlcipher_get_default_hmac_algorithm(); - if(algorithm == SQLCIPHER_HMAC_SHA1) { - sqlcipher_vdbe_return_string(pParse, "cipher_default_hmac_algorithm", SQLCIPHER_HMAC_SHA1_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_HMAC_SHA256) { - sqlcipher_vdbe_return_string(pParse, "cipher_default_hmac_algorithm", SQLCIPHER_HMAC_SHA256_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_HMAC_SHA512) { - sqlcipher_vdbe_return_string(pParse, "cipher_default_hmac_algorithm", SQLCIPHER_HMAC_SHA512_LABEL, P4_TRANSIENT); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_kdf_algorithm")==0 ){ - if(ctx) { - if(zRight) { - rc = SQLITE_ERROR; - if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL) == 0) { - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL) == 0) { - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA256); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL) == 0) { - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA512); - } - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - } else { - int algorithm = sqlcipher_codec_ctx_get_kdf_algorithm(ctx); - if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) { - sqlcipher_vdbe_return_string(pParse, "cipher_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) { - sqlcipher_vdbe_return_string(pParse, "cipher_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) { - sqlcipher_vdbe_return_string(pParse, "cipher_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL, P4_TRANSIENT); - } - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_kdf_algorithm")==0 ){ - if(zRight) { - rc = SQLITE_ERROR; - if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL) == 0) { - rc = sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA1); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL) == 0) { - rc = sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA256); - } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL) == 0) { - rc = sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA512); - } - } else { - int algorithm = sqlcipher_get_default_kdf_algorithm(); - if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) { - sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) { - sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL, P4_TRANSIENT); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) { - sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL, P4_TRANSIENT); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_compatibility")==0 ){ - if(ctx) { - if(zRight) { - int version = atoi(zRight); - - switch(version) { - case 1: - rc = sqlcipher_codec_ctx_set_pagesize(ctx, 1024); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 4000); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 0); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - break; - - case 2: - rc = sqlcipher_codec_ctx_set_pagesize(ctx, 1024); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 4000); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - break; - - case 3: - rc = sqlcipher_codec_ctx_set_pagesize(ctx, 1024); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 64000); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - break; - - default: - rc = sqlcipher_codec_ctx_set_pagesize(ctx, 4096); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA512); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA512); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 256000); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 1); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - break; - } - - rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx); - if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_compatibility")==0 ){ - if(zRight) { - int version = atoi(zRight); - switch(version) { - case 1: - sqlcipher_set_default_pagesize(1024); - sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA1); - sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA1); - sqlcipher_set_default_kdf_iter(4000); - sqlcipher_set_default_use_hmac(0); - break; - - case 2: - sqlcipher_set_default_pagesize(1024); - sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA1); - sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA1); - sqlcipher_set_default_kdf_iter(4000); - sqlcipher_set_default_use_hmac(1); - break; - - case 3: - sqlcipher_set_default_pagesize(1024); - sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA1); - sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA1); - sqlcipher_set_default_kdf_iter(64000); - sqlcipher_set_default_use_hmac(1); - break; - - default: - sqlcipher_set_default_pagesize(4096); - sqlcipher_set_default_hmac_algorithm(SQLCIPHER_HMAC_SHA512); - sqlcipher_set_default_kdf_algorithm(SQLCIPHER_PBKDF2_HMAC_SHA512); - sqlcipher_set_default_kdf_iter(256000); - sqlcipher_set_default_use_hmac(1); - break; - } - } - }else - if( sqlite3_stricmp(zLeft,"cipher_memory_security")==0 ){ - if( zRight ) { - sqlcipher_set_mem_security(sqlite3GetBoolean(zRight,1)); - } else { - char *on = sqlite3_mprintf("%d", sqlcipher_get_mem_security()); - sqlcipher_vdbe_return_string(pParse, "cipher_memory_security", on, P4_DYNAMIC); - } - }else - if( sqlite3_stricmp(zLeft,"cipher_settings")==0 ){ - if(ctx) { - int algorithm; - char *pragma; - - pragma = sqlite3_mprintf("PRAGMA kdf_iter = %d;", sqlcipher_codec_ctx_get_kdf_iter(ctx)); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - pragma = sqlite3_mprintf("PRAGMA cipher_page_size = %d;", sqlcipher_codec_ctx_get_pagesize(ctx)); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - pragma = sqlite3_mprintf("PRAGMA cipher_use_hmac = %d;", sqlcipher_codec_ctx_get_use_hmac(ctx)); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - pragma = sqlite3_mprintf("PRAGMA cipher_plaintext_header_size = %d;", sqlcipher_codec_ctx_get_plaintext_header_size(ctx)); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - algorithm = sqlcipher_codec_ctx_get_hmac_algorithm(ctx); - pragma = NULL; - if(algorithm == SQLCIPHER_HMAC_SHA1) { - pragma = sqlite3_mprintf("PRAGMA cipher_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA1_LABEL); - } else if(algorithm == SQLCIPHER_HMAC_SHA256) { - pragma = sqlite3_mprintf("PRAGMA cipher_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA256_LABEL); - } else if(algorithm == SQLCIPHER_HMAC_SHA512) { - pragma = sqlite3_mprintf("PRAGMA cipher_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA512_LABEL); - } - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - algorithm = sqlcipher_codec_ctx_get_kdf_algorithm(ctx); - pragma = NULL; - if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) { - pragma = sqlite3_mprintf("PRAGMA cipher_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) { - pragma = sqlite3_mprintf("PRAGMA cipher_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) { - pragma = sqlite3_mprintf("PRAGMA cipher_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL); - } - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - } - }else - if( sqlite3_stricmp(zLeft,"cipher_default_settings")==0 ){ - int algorithm; - char *pragma; - - pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_iter = %d;", sqlcipher_get_default_kdf_iter()); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - pragma = sqlite3_mprintf("PRAGMA cipher_default_page_size = %d;", sqlcipher_get_default_pagesize()); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - pragma = sqlite3_mprintf("PRAGMA cipher_default_use_hmac = %d;", sqlcipher_get_default_use_hmac()); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - pragma = sqlite3_mprintf("PRAGMA cipher_default_plaintext_header_size = %d;", sqlcipher_get_default_plaintext_header_size()); - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - algorithm = sqlcipher_get_default_hmac_algorithm(); - pragma = NULL; - if(algorithm == SQLCIPHER_HMAC_SHA1) { - pragma = sqlite3_mprintf("PRAGMA cipher_default_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA1_LABEL); - } else if(algorithm == SQLCIPHER_HMAC_SHA256) { - pragma = sqlite3_mprintf("PRAGMA cipher_default_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA256_LABEL); - } else if(algorithm == SQLCIPHER_HMAC_SHA512) { - pragma = sqlite3_mprintf("PRAGMA cipher_default_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA512_LABEL); - } - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - - algorithm = sqlcipher_get_default_kdf_algorithm(); - pragma = NULL; - if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) { - pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) { - pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL); - } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) { - pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL); - } - sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC); - }else - if( sqlite3_stricmp(zLeft,"cipher_integrity_check")==0 ){ - if(ctx) { - sqlcipher_codec_ctx_integrity_check(ctx, pParse, "cipher_integrity_check"); - } - } else - if( sqlite3_stricmp(zLeft, "cipher_log_level")==0 && zRight){ - unsigned int level = SQLCIPHER_LOG_NONE; - if(sqlite3_stricmp(zRight, "ERROR")==0) level = SQLCIPHER_LOG_ERROR; - else if(sqlite3_stricmp(zRight, "WARN" )==0) level = SQLCIPHER_LOG_WARN; - else if(sqlite3_stricmp(zRight, "INFO" )==0) level = SQLCIPHER_LOG_INFO; - else if(sqlite3_stricmp(zRight, "DEBUG")==0) level = SQLCIPHER_LOG_DEBUG; - else if(sqlite3_stricmp(zRight, "TRACE")==0) level = SQLCIPHER_LOG_TRACE; - sqlcipher_set_log_level(level); - sqlcipher_vdbe_return_string(pParse, "cipher_log_level", sqlite3_mprintf("%u", level), P4_DYNAMIC); - } else - if( sqlite3_stricmp(zLeft, "cipher_log")== 0 && zRight ){ - char *status = sqlite3_mprintf("%d", sqlcipher_set_log(zRight)); - sqlcipher_vdbe_return_string(pParse, "cipher_log", status, P4_DYNAMIC); - }else { - return 0; - } - return 1; -} - -/* these constants are used internally within SQLite's pager.c to differentiate between - operations on the main database or journal pages. This is important in the context - of a rekey operations, where the journal must be written using the original key - material (to allow a transactional rollback), while the new database pages are being - written with the new key material*/ -#define CODEC_READ_OP 3 -#define CODEC_WRITE_OP 6 -#define CODEC_JOURNAL_OP 7 - -/* - * sqlite3Codec can be called in multiple modes. - * encrypt mode - expected to return a pointer to the - * encrypted data without altering pData. - * decrypt mode - expected to return a pointer to pData, with - * the data decrypted in the input buffer - */ -static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { - codec_ctx *ctx = (codec_ctx *) iCtx; - int offset = 0, rc = 0; - int page_sz = sqlcipher_codec_ctx_get_pagesize(ctx); - unsigned char *pData = (unsigned char *) data; - void *buffer = sqlcipher_codec_ctx_get_data(ctx); - int plaintext_header_sz = sqlcipher_codec_ctx_get_plaintext_header_size(ctx); - int cctx = CIPHER_READ_CTX; - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3Codec: pgno=%d, mode=%d, page_sz=%d", pgno, mode, page_sz); - -#ifdef SQLCIPHER_EXT - if(sqlcipher_license_check(ctx) != SQLITE_OK) return NULL; -#endif - - /* call to derive keys if not present yet */ - if((rc = sqlcipher_codec_key_derive(ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3Codec: error occurred during key derivation: %d", rc); - sqlcipher_codec_ctx_set_error(ctx, rc); - return NULL; - } - - /* if the plaintext_header_size is negative that means an invalid size was set via - PRAGMA. We can't set the error state on the pager at that point because the pager - may not be open yet. However, this is a fatal error state, so abort the codec */ - if(plaintext_header_sz < 0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3Codec: error invalid plaintext_header_sz: %d", plaintext_header_sz); - sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - return NULL; - } - - if(pgno == 1) /* adjust starting pointers in data page for header offset on first page*/ - offset = plaintext_header_sz ? plaintext_header_sz : FILE_HEADER_SZ; - - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3Codec: switch mode=%d offset=%d", mode, offset); - switch(mode) { - case CODEC_READ_OP: /* decrypt */ - if(pgno == 1) /* copy initial part of file header or SQLite magic to buffer */ - memcpy(buffer, plaintext_header_sz ? pData : (void *) SQLITE_FILE_HEADER, offset); - - rc = sqlcipher_page_cipher(ctx, cctx, pgno, CIPHER_DECRYPT, page_sz - offset, pData + offset, (unsigned char*)buffer + offset); -#ifdef SQLCIPHER_TEST - if((sqlcipher_get_test_flags() & TEST_FAIL_DECRYPT) > 0 && sqlcipher_get_test_fail()) { - rc = SQLITE_ERROR; - sqlcipher_log(SQLCIPHER_LOG_ERROR, "simulating decryption failure for pgno=%d, mode=%d, page_sz=%d\n", pgno, mode, page_sz); - } -#endif - if(rc != SQLITE_OK) { - /* failure to decrypt a page is considered a permanent error and will render the pager unusable - in order to prevent inconsistent data being loaded into page cache */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3Codec: error decrypting page %d data: %d", pgno, rc); - sqlcipher_memset((unsigned char*) buffer+offset, 0, page_sz-offset); - sqlcipher_codec_ctx_set_error(ctx, rc); - } else { - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_KEY_USED); - } - memcpy(pData, buffer, page_sz); /* copy buffer data back to pData and return */ - return pData; - break; - - case CODEC_WRITE_OP: /* encrypt database page, operate on write context and fall through to case 7, so the write context is used*/ - cctx = CIPHER_WRITE_CTX; - - case CODEC_JOURNAL_OP: /* encrypt journal page, operate on read context use to get the original page data from the database */ - if(pgno == 1) { /* copy initial part of file header or salt to buffer */ - void *kdf_salt = NULL; - /* retrieve the kdf salt */ - if((rc = sqlcipher_codec_ctx_get_kdf_salt(ctx, &kdf_salt)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3Codec: error retrieving salt: %d", rc); - sqlcipher_codec_ctx_set_error(ctx, rc); - return NULL; - } - memcpy(buffer, plaintext_header_sz ? pData : kdf_salt, offset); - } - rc = sqlcipher_page_cipher(ctx, cctx, pgno, CIPHER_ENCRYPT, page_sz - offset, pData + offset, (unsigned char*)buffer + offset); -#ifdef SQLCIPHER_TEST - if((sqlcipher_get_test_flags() & TEST_FAIL_ENCRYPT) > 0 && sqlcipher_get_test_fail()) { - rc = SQLITE_ERROR; - sqlcipher_log(SQLCIPHER_LOG_ERROR, "simulating encryption failure for pgno=%d, mode=%d, page_sz=%d\n", pgno, mode, page_sz); - } -#endif - if(rc != SQLITE_OK) { - /* failure to encrypt a page is considered a permanent error and will render the pager unusable - in order to prevent corrupted pages from being written to the main databased when using WAL */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3Codec: error encrypting page %d data: %d", pgno, rc); - sqlcipher_memset((unsigned char*)buffer+offset, 0, page_sz-offset); - sqlcipher_codec_ctx_set_error(ctx, rc); - return NULL; - } - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_KEY_USED); - return buffer; /* return persistent buffer data, pData remains intact */ - break; - - default: - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3Codec: error unsupported codec mode %d", mode); - sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); /* unsupported mode, set error */ - return pData; - break; - } -} - -static void sqlite3FreeCodecArg(void *pCodecArg) { - codec_ctx *ctx = (codec_ctx *) pCodecArg; - if(pCodecArg == NULL) return; - sqlcipher_codec_ctx_free(&ctx); /* wipe and free allocated memory for the context */ - sqlcipher_deactivate(); /* cleanup related structures, OpenSSL etc, when codec is detatched */ -} - -int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { - struct Db *pDb = &db->aDb[nDb]; - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: db=%p, nDb=%d", db, nDb); - - if(nKey && zKey && pDb->pBt) { - int rc; - Pager *pPager = pDb->pBt->pBt->pPager; - sqlite3_file *fd; - codec_ctx *ctx; - - ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); - - if(ctx != NULL && SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) { - /* there is already a codec attached to this database, so we should not proceed */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipherCodecAttach: no codec attached to db, exiting"); - return SQLITE_OK; - } - - /* check if the sqlite3_file is open, and if not force handle to NULL */ - if((fd = sqlite3PagerFile(pPager))->pMethods == 0) fd = NULL; - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlcipher_activate()"); - sqlcipher_activate(); /* perform internal initialization for sqlcipher */ - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: entering database mutex %p", db->mutex); - sqlite3_mutex_enter(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: entered database mutex %p", db->mutex); - -#ifdef SQLCIPHER_EXT - if((rc = sqlite3_set_authorizer(db, sqlcipher_license_authorizer, db)) != SQLITE_OK) { - sqlite3_mutex_leave(db->mutex); - return rc; - } -#endif - - /* point the internal codec argument against the contet to be prepared */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlcipher_codec_ctx_init()"); - rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, zKey, nKey); - - if(rc != SQLITE_OK) { - /* initialization failed, do not attach potentially corrupted context */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipherCodecAttach: context initialization failed forcing error state with rc=%d", rc); - /* force an error at the pager level, such that even the upstream caller ignores the return code - the pager will be in an error state and will process no further operations */ - sqlite3pager_error(pPager, rc); - pDb->pBt->pBt->db->errCode = rc; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: leaving database mutex %p (early return on rc=%d)", db->mutex, rc); - sqlite3_mutex_leave(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: left database mutex %p (early return on rc=%d)", db->mutex, rc); - return rc; - } - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlcipherPagerSetCodec()"); - sqlcipherPagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling codec_set_btree_to_codec_pagesize()"); - codec_set_btree_to_codec_pagesize(db, pDb, ctx); - - /* force secure delete. This has the benefit of wiping internal data when deleted - and also ensures that all pages are written to disk (i.e. not skipped by - sqlite3PagerDontWrite optimizations) */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlite3BtreeSecureDelete()"); - sqlite3BtreeSecureDelete(pDb->pBt, 1); - - /* if fd is null, then this is an in-memory database and - we dont' want to overwrite the AutoVacuum settings - if not null, then set to the default */ - if(fd != NULL) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecAttach: calling sqlite3BtreeSetAutoVacuum()"); - sqlite3BtreeSetAutoVacuum(pDb->pBt, SQLITE_DEFAULT_AUTOVACUUM); - } - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: leaving database mutex %p", db->mutex); - sqlite3_mutex_leave(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipherCodecAttach: left database mutex %p", db->mutex); - } - return SQLITE_OK; -} - -int sqlcipher_find_db_index(sqlite3 *db, const char *zDb) { - int db_index; - if(zDb == NULL){ - return 0; - } - for(db_index = 0; db_index < db->nDb; db_index++) { - struct Db *pDb = &db->aDb[db_index]; - if(strcmp(pDb->zDbSName, zDb) == 0) { - return db_index; - } - } - return 0; -} - -void sqlite3_activate_see(const char* in) { - /* do nothing, security enhancements are always active */ -} - -int sqlite3_key(sqlite3 *db, const void *pKey, int nKey) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_key: db=%p", db); - return sqlite3_key_v2(db, "main", pKey, nKey); -} - -int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_key_v2: db=%p zDb=%s", db, zDb); - /* attach key if db and pKey are not null and nKey is > 0 */ - if(db && pKey && nKey) { - int db_index = sqlcipher_find_db_index(db, zDb); - return sqlcipherCodecAttach(db, db_index, pKey, nKey); - } - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3_key_v2: no key provided"); - return SQLITE_ERROR; -} - -int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_rekey: db=%p", db); - return sqlite3_rekey_v2(db, "main", pKey, nKey); -} - -/* sqlite3_rekey_v2 -** Given a database, this will reencrypt the database using a new key. -** There is only one possible modes of operation - to encrypt a database -** that is already encrpyted. If the database is not already encrypted -** this should do nothing -** The proposed logic for this function follows: -** 1. Determine if the database is already encryptped -** 2. If there is NOT already a key present do nothing -** 3. If there is a key present, re-encrypt the database with the new key -*/ -int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_rekey_v2: db=%p zDb=%s", db, zDb); - if(db && pKey && nKey) { - int db_index = sqlcipher_find_db_index(db, zDb); - struct Db *pDb = &db->aDb[db_index]; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_rekey_v2: database zDb=%p db_index:%d", zDb, db_index); - if(pDb->pBt) { - codec_ctx *ctx; - int rc, page_count; - Pgno pgno; - PgHdr *page; - Pager *pPager = pDb->pBt->pBt->pPager; - - ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); - - if(ctx == NULL) { - /* there was no codec attached to this database, so this should do nothing! */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3_rekey_v2: no codec attached to db, exiting"); - return SQLITE_MISUSE; - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3_rekey_v2: entering database mutex %p", db->mutex); - sqlite3_mutex_enter(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3_rekey_v2: entered database mutex %p", db->mutex); - - codec_set_pass_key(db, db_index, pKey, nKey, CIPHER_WRITE_CTX); - - /* do stuff here to rewrite the database - ** 1. Create a transaction on the database - ** 2. Iterate through each page, reading it and then writing it. - ** 3. If that goes ok then commit and put ctx->rekey into ctx->key - ** note: don't deallocate rekey since it may be used in a subsequent iteration - */ - rc = sqlite3BtreeBeginTrans(pDb->pBt, 1, 0); /* begin write transaction */ - sqlite3PagerPagecount(pPager, &page_count); - for(pgno = 1; rc == SQLITE_OK && pgno <= (unsigned int)page_count; pgno++) { /* pgno's start at 1 see pager.c:pagerAcquire */ - if(!sqlite3pager_is_sj_pgno(pPager, pgno)) { /* skip this page (see pager.c:pagerAcquire for reasoning) */ - rc = sqlite3PagerGet(pPager, pgno, &page, 0); - if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */ - rc = sqlite3PagerWrite(page); - if(rc == SQLITE_OK) { - sqlite3PagerUnref(page); - } else { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3_rekey_v2: error %d occurred writing page %d", rc, pgno); - } - } else { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3_rekey_v2: error %d occurred getting page %d", rc, pgno); - } - } - } - - /* if commit was successful commit and copy the rekey data to current key, else rollback to release locks */ - if(rc == SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_rekey_v2: committing"); - rc = sqlite3BtreeCommit(pDb->pBt); - sqlcipher_codec_key_copy(ctx, CIPHER_WRITE_CTX); - } else { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlite3_rekey_v2: rollback"); - sqlite3BtreeRollback(pDb->pBt, SQLITE_ABORT_ROLLBACK, 0); - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3_rekey_v2: leaving database mutex %p", db->mutex); - sqlite3_mutex_leave(db->mutex); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlite3_rekey_v2: left database mutex %p", db->mutex); - } - return SQLITE_OK; - } - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlite3_rekey_v2: no key provided"); - return SQLITE_ERROR; -} - -void sqlcipherCodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { - struct Db *pDb = &db->aDb[nDb]; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipherCodecGetKey:db=%p, nDb=%d", db, nDb); - if( pDb->pBt ) { - codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); - - if(ctx) { - /* pass back the keyspec from the codec, unless PRAGMA cipher_store_pass - is set or keyspec has not yet been derived, in which case pass - back the password key material */ - sqlcipher_codec_get_keyspec(ctx, zKey, nKey); - if(sqlcipher_codec_get_store_pass(ctx) == 1 || *zKey == NULL) { - sqlcipher_codec_get_pass(ctx, zKey, nKey); - } - } else { - *zKey = NULL; - *nKey = 0; - } - } -} - -/* - * Implementation of an "export" function that allows a caller - * to duplicate the main database to an attached database. This is intended - * as a conveneince for users who need to: - * - * 1. migrate from an non-encrypted database to an encrypted database - * 2. move from an encrypted database to a non-encrypted database - * 3. convert beween the various flavors of encrypted databases. - * - * This implementation is based heavily on the procedure and code used - * in vacuum.c, but is exposed as a function that allows export to any - * named attached database. - */ - -/* -** Finalize a prepared statement. If there was an error, store the -** text of the error message in *pzErrMsg. Return the result code. -** -** Based on vacuumFinalize from vacuum.c -*/ -static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ - int rc; - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - if( rc ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - } - return rc; -} - -/* -** Execute zSql on database db. Return an error code. -** -** Based on execSql from vacuum.c -*/ -static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - VVA_ONLY( int rc; ) - if( !zSql ){ - return SQLITE_NOMEM; - } - if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - return sqlite3_errcode(db); - } - VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW ); - return sqlcipher_finalize(db, pStmt, pzErrMsg); -} - -/* -** Execute zSql on database db. The statement returns exactly -** one column. Execute this as SQL on the same database. -** -** Based on execExecSql from vacuum.c -*/ -static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - int rc; - - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); - if( rc!=SQLITE_OK ){ - sqlcipher_finalize(db, pStmt, pzErrMsg); - return rc; - } - } - - return sqlcipher_finalize(db, pStmt, pzErrMsg); -} - -/* - * copy database and schema from the main database to an attached database - * - * Based on sqlite3RunVacuum from vacuum.c -*/ -void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { - sqlite3 *db = sqlite3_context_db_handle(context); - const char* targetDb, *sourceDb; - int targetDb_idx = 0; - u64 saved_flags = db->flags; /* Saved value of the db->flags */ - u32 saved_mDbFlags = db->mDbFlags; /* Saved value of the db->mDbFlags */ - int saved_nChange = db->nChange; /* Saved value of db->nChange */ - int saved_nTotalChange = db->nTotalChange; /* Saved value of db->nTotalChange */ - u8 saved_mTrace = db->mTrace; /* Saved value of db->mTrace */ - int rc = SQLITE_OK; /* Return code from service routines */ - char *zSql = NULL; /* SQL statements */ - char *pzErrMsg = NULL; - - if(argc != 1 && argc != 2) { - rc = SQLITE_ERROR; - pzErrMsg = sqlite3_mprintf("invalid number of arguments (%d) passed to sqlcipher_export", argc); - goto end_of_export; - } - - if(sqlite3_value_type(argv[0]) == SQLITE_NULL) { - rc = SQLITE_ERROR; - pzErrMsg = sqlite3_mprintf("target database can't be NULL"); - goto end_of_export; - } - - targetDb = (const char*) sqlite3_value_text(argv[0]); - sourceDb = "main"; - - if(argc == 2) { - if(sqlite3_value_type(argv[1]) == SQLITE_NULL) { - rc = SQLITE_ERROR; - pzErrMsg = sqlite3_mprintf("target database can't be NULL"); - goto end_of_export; - } - sourceDb = (char *) sqlite3_value_text(argv[1]); - } - - - /* if the name of the target is not main, but the index returned is zero - there is a mismatch and we should not proceed */ - targetDb_idx = sqlcipher_find_db_index(db, targetDb); - if(targetDb_idx == 0 && targetDb != NULL && sqlite3_stricmp("main", targetDb) != 0) { - rc = SQLITE_ERROR; - pzErrMsg = sqlite3_mprintf("unknown database %s", targetDb); - goto end_of_export; - } - db->init.iDb = targetDb_idx; - - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; - db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; - db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); - db->mTrace = 0; - - /* Query the schema of the main database. Create a mirror schema - ** in the temporary database. - */ - zSql = sqlite3_mprintf( - "SELECT sql " - " FROM %s.sqlite_schema WHERE type='table' AND name!='sqlite_sequence'" - " AND rootpage>0" - , sourceDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT sql " - " FROM %s.sqlite_schema WHERE sql LIKE 'CREATE INDEX %%' " - , sourceDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = sqlite3_mprintf( - "SELECT sql " - " FROM %s.sqlite_schema WHERE sql LIKE 'CREATE UNIQUE INDEX %%'" - , sourceDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Loop through the tables in the main database. For each, do - ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy - ** the contents to the temporary database. - */ - zSql = sqlite3_mprintf( - "SELECT 'INSERT INTO %s.' || quote(name) " - "|| ' SELECT * FROM %s.' || quote(name) || ';'" - "FROM %s.sqlite_schema " - "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND rootpage>0" - , targetDb, sourceDb, sourceDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Copy over the contents of the sequence table - */ - zSql = sqlite3_mprintf( - "SELECT 'INSERT INTO %s.' || quote(name) " - "|| ' SELECT * FROM %s.' || quote(name) || ';' " - "FROM %s.sqlite_schema WHERE name=='sqlite_sequence';" - , targetDb, sourceDb, targetDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - /* Copy the triggers, views, and virtual tables from the main database - ** over to the temporary database. None of these objects has any - ** associated storage, so all we have to do is copy their entries - ** from the SQLITE_MASTER table. - */ - zSql = sqlite3_mprintf( - "INSERT INTO %s.sqlite_schema " - " SELECT type, name, tbl_name, rootpage, sql" - " FROM %s.sqlite_schema" - " WHERE type='view' OR type='trigger'" - " OR (type='table' AND rootpage=0)" - , targetDb, sourceDb); - rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); - if( rc!=SQLITE_OK ) goto end_of_export; - sqlite3_free(zSql); - - zSql = NULL; -end_of_export: - db->init.iDb = 0; - db->flags = saved_flags; - db->mDbFlags = saved_mDbFlags; - db->nChange = saved_nChange; - db->nTotalChange = saved_nTotalChange; - db->mTrace = saved_mTrace; - - if(zSql) sqlite3_free(zSql); - - if(rc) { - if(pzErrMsg != NULL) { - sqlite3_result_error(context, pzErrMsg, -1); - sqlite3DbFree(db, pzErrMsg); - } else { - sqlite3_result_error(context, sqlite3ErrStr(rc), -1); - } - } -} -#endif -/* END SQLCIPHER */ diff --git a/src/crypto.h b/src/crypto.h deleted file mode 100644 index 804306d838..0000000000 --- a/src/crypto.h +++ /dev/null @@ -1,360 +0,0 @@ -/* -** SQLCipher -** crypto.h developed by Stephen Lombardo (Zetetic LLC) -** sjlombardo at zetetic dot net -** http://zetetic.net -** -** Copyright (c) 2008, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC -#ifndef CRYPTO_H -#define CRYPTO_H - -#include "sqliteInt.h" -#include "btreeInt.h" -#include "pager.h" -#include "vdbeInt.h" - -#ifdef __ANDROID__ -#include -#endif - -#include - -#if defined(_WIN32) || defined(SQLITE_OS_WINRT) -#include /* amalgamator: dontcache */ -#else -#include /* amalgamator: dontcache */ -#endif - -#ifndef OMIT_MEMLOCK -#if defined(__unix__) || defined(__APPLE__) || defined(_AIX) -#include /* amalgamator: dontcache */ -#include /* amalgamator: dontcache */ -#include /* amalgamator: dontcache */ -#include /* amalgamator: dontcache */ -#endif -#endif - -#include "sqlcipher.h" - -/* extensions defined in pager.c */ -void *sqlcipherPagerGetCodec(Pager*); -void sqlcipherPagerSetCodec(Pager*, void *(*)(void*,void*,Pgno,int), void (*)(void*,int,int), void (*)(void*), void *); -int sqlite3pager_is_sj_pgno(Pager*, Pgno); -void sqlite3pager_error(Pager*, int); -void sqlite3pager_reset(Pager *pPager); -/* end extensions defined in pager.c */ - -#if !defined (SQLCIPHER_CRYPTO_CC) \ - && !defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) \ - && !defined (SQLCIPHER_CRYPTO_NSS) \ - && !defined (SQLCIPHER_CRYPTO_OPENSSL) -#define SQLCIPHER_CRYPTO_OPENSSL -#endif - -#define FILE_HEADER_SZ 16 - -#define CIPHER_XSTR(s) CIPHER_STR(s) -#define CIPHER_STR(s) #s - -#ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.5.5 -#endif - -#ifndef CIPHER_VERSION_BUILD -#define CIPHER_VERSION_BUILD community -#endif - -#define CIPHER_DECRYPT 0 -#define CIPHER_ENCRYPT 1 - -#define CIPHER_READ_CTX 0 -#define CIPHER_WRITE_CTX 1 -#define CIPHER_READWRITE_CTX 2 - -#ifndef PBKDF2_ITER -#define PBKDF2_ITER 256000 -#endif - -#define SQLCIPHER_FLAG_GET(FLAG,BIT) ((FLAG & BIT) != 0) -#define SQLCIPHER_FLAG_SET(FLAG,BIT) FLAG |= BIT -#define SQLCIPHER_FLAG_UNSET(FLAG,BIT) FLAG &= ~BIT - -/* possible flags for codec_ctx->flags */ -#define CIPHER_FLAG_HMAC (1 << 0) -#define CIPHER_FLAG_LE_PGNO (1 << 1) -#define CIPHER_FLAG_BE_PGNO (1 << 2) -#define CIPHER_FLAG_KEY_USED (1 << 3) -#define CIPHER_FLAG_HAS_KDF_SALT (1 << 4) - - -#ifndef DEFAULT_CIPHER_FLAGS -#define DEFAULT_CIPHER_FLAGS CIPHER_FLAG_HMAC | CIPHER_FLAG_LE_PGNO -#endif - - -/* by default, sqlcipher will use a reduced number of iterations to generate - the HMAC key / or transform a raw cipher key - */ -#ifndef FAST_PBKDF2_ITER -#define FAST_PBKDF2_ITER 2 -#endif - -/* this if a fixed random array that will be xor'd with the database salt to ensure that the - salt passed to the HMAC key derivation function is not the same as that used to derive - the encryption key. This can be overridden at compile time but it will make the resulting - binary incompatible with the default builds when using HMAC. A future version of SQLcipher - will likely allow this to be defined at runtime via pragma */ -#ifndef HMAC_SALT_MASK -#define HMAC_SALT_MASK 0x3a -#endif - -#ifndef CIPHER_MAX_IV_SZ -#define CIPHER_MAX_IV_SZ 16 -#endif - -#ifndef CIPHER_MAX_KEY_SZ -#define CIPHER_MAX_KEY_SZ 64 -#endif - - -/* -** Simple shared routines for converting hex char strings to binary data - */ -static int cipher_hex2int(char c) { - return (c>='0' && c<='9') ? (c)-'0' : - (c>='A' && c<='F') ? (c)-'A'+10 : - (c>='a' && c<='f') ? (c)-'a'+10 : 0; -} - -static void cipher_hex2bin(const unsigned char *hex, int sz, unsigned char *out){ - int i; - for(i = 0; i < sz; i += 2){ - out[i/2] = (cipher_hex2int(hex[i])<<4) | cipher_hex2int(hex[i+1]); - } -} - -static void cipher_bin2hex(const unsigned char* in, int sz, char *out) { - int i; - for(i=0; i < sz; i++) { - sqlite3_snprintf(3, out + (i*2), "%02x ", in[i]); - } -} - -static int cipher_isHex(const unsigned char *hex, int sz){ - int i; - for(i = 0; i < sz; i++) { - unsigned char c = hex[i]; - if ((c < '0' || c > '9') && - (c < 'A' || c > 'F') && - (c < 'a' || c > 'f')) { - return 0; - } - } - return 1; -} - -/* possible flags for simulating specific test conditions */ -#ifdef SQLCIPHER_TEST -#define TEST_FAIL_ENCRYPT 0x01 -#define TEST_FAIL_DECRYPT 0x02 -#define TEST_FAIL_MIGRATE 0x04 -unsigned int sqlcipher_get_test_flags(void); -void sqlcipher_set_test_flags(unsigned int); -int sqlcipher_get_test_rand(void); -void sqlcipher_set_test_rand(int); -int sqlcipher_get_test_fail(void); -#endif - -/* extensions defined in crypto_impl.c */ -/* the default implementation of SQLCipher uses a cipher_ctx - to keep track of read / write state separately. The following - struct and associated functions are defined here */ -typedef struct { - int derive_key; - int pass_sz; - unsigned char *key; - unsigned char *hmac_key; - unsigned char *pass; - char *keyspec; -} cipher_ctx; - - -typedef struct { - int store_pass; - int kdf_iter; - int fast_kdf_iter; - int kdf_salt_sz; - int key_sz; - int iv_sz; - int block_sz; - int page_sz; - int keyspec_sz; - int reserve_sz; - int hmac_sz; - int plaintext_header_sz; - int hmac_algorithm; - int kdf_algorithm; - unsigned int flags; - unsigned char *kdf_salt; - unsigned char *hmac_kdf_salt; - unsigned char *buffer; - Btree *pBt; - cipher_ctx *read_ctx; - cipher_ctx *write_ctx; - sqlcipher_provider *provider; - void *provider_ctx; -} codec_ctx ; - -/* crypto.c functions */ -int sqlcipher_codec_pragma(sqlite3*, int, Parse*, const char *, const char*); -int sqlcipherCodecAttach(sqlite3*, int, const void *, int); -void sqlcipherCodecGetKey(sqlite3*, int, void**, int*); -void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); - -/* crypto_impl.c functions */ - -void sqlcipher_init_memmethods(void); - -/* activation and initialization */ -void sqlcipher_activate(void); -void sqlcipher_deactivate(void); - -int sqlcipher_codec_ctx_init(codec_ctx **, Db *, Pager *, const void *, int); -void sqlcipher_codec_ctx_free(codec_ctx **); -int sqlcipher_codec_key_derive(codec_ctx *); -int sqlcipher_codec_key_copy(codec_ctx *, int); - -/* page cipher implementation */ -int sqlcipher_page_cipher(codec_ctx *, int, Pgno, int, int, unsigned char *, unsigned char *); - -/* context setters & getters */ -void sqlcipher_codec_ctx_set_error(codec_ctx *, int); - -void sqlcipher_codec_get_pass(codec_ctx *, void **, int *); -int sqlcipher_codec_ctx_set_pass(codec_ctx *, const void *, int, int); -void sqlcipher_codec_get_keyspec(codec_ctx *, void **zKey, int *nKey); - -int sqlcipher_codec_ctx_set_pagesize(codec_ctx *, int); -int sqlcipher_codec_ctx_get_pagesize(codec_ctx *); -int sqlcipher_codec_ctx_get_reservesize(codec_ctx *); - -void sqlcipher_set_default_pagesize(int page_size); -int sqlcipher_get_default_pagesize(void); - -void sqlcipher_set_default_kdf_iter(int iter); -int sqlcipher_get_default_kdf_iter(void); -int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *, int); -int sqlcipher_codec_ctx_get_kdf_iter(codec_ctx *ctx); - -int sqlcipher_codec_ctx_set_kdf_salt(codec_ctx *ctx, unsigned char *salt, int sz); -int sqlcipher_codec_ctx_get_kdf_salt(codec_ctx *ctx, void **salt); - -int sqlcipher_codec_ctx_set_fast_kdf_iter(codec_ctx *, int); -int sqlcipher_codec_ctx_get_fast_kdf_iter(codec_ctx *); - -const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx); - -void* sqlcipher_codec_ctx_get_data(codec_ctx *); - -void sqlcipher_set_default_use_hmac(int use); -int sqlcipher_get_default_use_hmac(void); - -void sqlcipher_set_hmac_salt_mask(unsigned char mask); -unsigned char sqlcipher_get_hmac_salt_mask(void); - -int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use); -int sqlcipher_codec_ctx_get_use_hmac(codec_ctx *ctx); - -const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx); -int sqlcipher_codec_ctx_migrate(codec_ctx *ctx); -int sqlcipher_codec_add_random(codec_ctx *ctx, const char *data, int random_sz); -int sqlcipher_cipher_profile(sqlite3 *db, const char *destination); -int sqlcipher_codec_get_store_pass(codec_ctx *ctx); -void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey); -void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value); -int sqlcipher_codec_fips_status(codec_ctx *ctx); -const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx); - -int sqlcipher_set_default_plaintext_header_size(int size); -int sqlcipher_get_default_plaintext_header_size(void); -int sqlcipher_codec_ctx_set_plaintext_header_size(codec_ctx *ctx, int size); -int sqlcipher_codec_ctx_get_plaintext_header_size(codec_ctx *ctx); - -int sqlcipher_set_default_hmac_algorithm(int algorithm); -int sqlcipher_get_default_hmac_algorithm(void); -int sqlcipher_codec_ctx_set_hmac_algorithm(codec_ctx *ctx, int algorithm); -int sqlcipher_codec_ctx_get_hmac_algorithm(codec_ctx *ctx); - -int sqlcipher_set_default_kdf_algorithm(int algorithm); -int sqlcipher_get_default_kdf_algorithm(void); -int sqlcipher_codec_ctx_set_kdf_algorithm(codec_ctx *ctx, int algorithm); -int sqlcipher_codec_ctx_get_kdf_algorithm(codec_ctx *ctx); - -void sqlcipher_set_mem_security(int); -int sqlcipher_get_mem_security(void); - -int sqlcipher_find_db_index(sqlite3 *db, const char *zDb); - -int sqlcipher_codec_ctx_integrity_check(codec_ctx *, Parse *, char *); - -int sqlcipher_set_log(const char *destination); -void sqlcipher_set_log_level(unsigned int level); -void sqlcipher_log(unsigned int tag, const char *message, ...); - -#define SQLCIPHER_LOG_NONE 0x00 -#define SQLCIPHER_LOG_ERROR 0x01 -#define SQLCIPHER_LOG_WARN 0x02 -#define SQLCIPHER_LOG_INFO 0x04 -#define SQLCIPHER_LOG_DEBUG 0x08 -#define SQLCIPHER_LOG_TRACE 0x10 -#define SQLCIPHER_LOG_ALL 0xffffffff - -void sqlcipher_vdbe_return_string(Parse*, const char*, const char*, int); - -#ifdef CODEC_DEBUG_PAGEDATA -#define CODEC_HEXDUMP(DESC,BUFFER,LEN) \ - { \ - int __pctr; \ - printf(DESC); \ - for(__pctr=0; __pctr < LEN; __pctr++) { \ - if(__pctr % 16 == 0) printf("\n%05x: ",__pctr); \ - printf("%02x ",((unsigned char*) BUFFER)[__pctr]); \ - } \ - printf("\n"); \ - fflush(stdout); \ - } -#else -#define CODEC_HEXDUMP(DESC,BUFFER,LEN) -#endif - -#endif -#endif -/* END SQLCIPHER */ - diff --git a/src/crypto_cc.c b/src/crypto_cc.c index a800830715..37669cb46b 100644 --- a/src/crypto_cc.c +++ b/src/crypto_cc.c @@ -31,7 +31,6 @@ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_CC -#include "crypto.h" #include "sqlcipher.h" #include #include @@ -39,7 +38,7 @@ int sqlcipher_cc_setup(sqlcipher_provider *p); -static int sqlcipher_cc_add_random(void *ctx, void *buffer, int length) { +static int sqlcipher_cc_add_random(void *ctx, const void *buffer, int length) { return SQLITE_OK; } @@ -54,19 +53,39 @@ static const char* sqlcipher_cc_get_provider_name(void *ctx) { static const char* sqlcipher_cc_get_provider_version(void *ctx) { #if TARGET_OS_MAC - CFTypeRef version; + /* macOS uses com.apple.security with a lowercase s */ CFBundleRef bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.security")); - if(bundle == NULL) { - return "unknown"; + CFTypeRef bundle_ver; + const char *ver; + + /* if the bundle wasn't identified, try secrurity with a capial S (works for iOS) */ + if(!bundle) { + bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.Security")); } - version = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleShortVersionString")); - return CFStringGetCStringPtr(version, kCFStringEncodingUTF8); -#else - return "unknown"; + + /* If the bundle was resolved, retrieve the bundle version key then attempt to convert it to a C string. + * Note that it is possible for CFStringGetCString to return NULL (this is warned against extensively in the + * header), so only return a value if the conversion was successful. */ + if( + bundle + && (bundle_ver = CFBundleGetValueForInfoDictionaryKey(bundle, kCFBundleVersionKey)) + && (ver = CFStringGetCStringPtr(bundle_ver, kCFStringEncodingUTF8)) + ) { + return ver; + } + #endif + /* unable to detect the CoreCrypto version, return a fixed string "unknown" */ + return "unknown"; } -static int sqlcipher_cc_hmac(void *ctx, int algorithm, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { +static int sqlcipher_cc_hmac( + void *ctx, int algorithm, + const unsigned char *hmac_key, int key_sz, + const unsigned char *in, int in_sz, + const unsigned char *in2, int in2_sz, + unsigned char *out +) { CCHmacContext hmac_context; if(in == NULL) return SQLITE_ERROR; switch(algorithm) { @@ -88,7 +107,13 @@ static int sqlcipher_cc_hmac(void *ctx, int algorithm, unsigned char *hmac_key, return SQLITE_OK; } -static int sqlcipher_cc_kdf(void *ctx, int algorithm, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { +static int sqlcipher_cc_kdf( + void *ctx, int algorithm, + const unsigned char *pass, int pass_sz, + const unsigned char* salt, int salt_sz, + int workfactor, + int key_sz, unsigned char *key +) { switch(algorithm) { case SQLCIPHER_HMAC_SHA1: if(CCKeyDerivationPBKDF(kCCPBKDF2, (const char *)pass, pass_sz, salt, salt_sz, kCCPRFHmacAlgSHA1, workfactor, key, key_sz) != kCCSuccess) return SQLITE_ERROR; @@ -105,10 +130,16 @@ static int sqlcipher_cc_kdf(void *ctx, int algorithm, const unsigned char *pass, return SQLITE_OK; } -static int sqlcipher_cc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { +static int sqlcipher_cc_cipher( + void *ctx, int mode, + const unsigned char *key, int key_sz, + const unsigned char *iv, + const unsigned char *in, int in_sz, + unsigned char *out +) { CCCryptorRef cryptor; size_t tmp_csz, csz; - CCOperation op = mode == CIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; + CCOperation op = mode == SQLCIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; if(CCCryptorCreate(op, kCCAlgorithmAES128, 0, key, kCCKeySizeAES256, iv, &cryptor) != kCCSuccess) return SQLITE_ERROR; if(CCCryptorUpdate(cryptor, in, in_sz, out, in_sz, &tmp_csz) != kCCSuccess) return SQLITE_ERROR; @@ -167,6 +198,8 @@ static int sqlcipher_cc_fips_status(void *ctx) { } int sqlcipher_cc_setup(sqlcipher_provider *p) { + p->init = NULL; + p->shutdown = NULL; p->random = sqlcipher_cc_random; p->get_provider_name = sqlcipher_cc_get_provider_name; p->hmac = sqlcipher_cc_hmac; diff --git a/src/crypto_impl.c b/src/crypto_impl.c deleted file mode 100644 index 318c48b03d..0000000000 --- a/src/crypto_impl.c +++ /dev/null @@ -1,1774 +0,0 @@ -/* -** SQLCipher -** http://sqlcipher.net -** -** Copyright (c) 2008 - 2013, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC - -#include "sqlcipher.h" -#include "crypto.h" - -#ifdef SQLCIPHER_TEST -static volatile unsigned int cipher_test_flags = 0; -unsigned int sqlcipher_get_test_flags() { - return cipher_test_flags; -} -void sqlcipher_set_test_flags(unsigned int flags) { - cipher_test_flags = flags; -} - -static volatile int cipher_test_rand = 0; -int sqlcipher_get_test_rand() { - return cipher_test_rand; -} -void sqlcipher_set_test_rand(int rand) { - cipher_test_rand = rand; -} -int sqlcipher_get_test_fail() { - int x; - - /* if cipher_test_rand is not set to a non-zero value always fail (return true) */ - if (cipher_test_rand == 0) return 1; - - sqlite3_randomness(sizeof(x), &x); - return ((x % cipher_test_rand) == 0); -} -#endif - -/* Generate code to return a string value */ - -static volatile unsigned int default_flags = DEFAULT_CIPHER_FLAGS; -static volatile unsigned char hmac_salt_mask = HMAC_SALT_MASK; -static volatile int default_kdf_iter = PBKDF2_ITER; -static volatile int default_page_size = 4096; -static volatile int default_plaintext_header_sz = 0; -static volatile int default_hmac_algorithm = SQLCIPHER_HMAC_SHA512; -static volatile int default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA512; -static volatile int sqlcipher_mem_security_on = 0; -static volatile int sqlcipher_mem_executed = 0; -static volatile int sqlcipher_mem_initialized = 0; -static volatile unsigned int sqlcipher_activate_count = 0; -static volatile sqlite3_mem_methods default_mem_methods; -static sqlcipher_provider *default_provider = NULL; - -static sqlite3_mutex* sqlcipher_static_mutex[SQLCIPHER_MUTEX_COUNT]; -static FILE* sqlcipher_log_file = NULL; -static volatile int sqlcipher_log_logcat = 0; -static volatile unsigned int sqlcipher_log_level = SQLCIPHER_LOG_NONE; - -sqlite3_mutex* sqlcipher_mutex(int mutex) { - if(mutex < 0 || mutex >= SQLCIPHER_MUTEX_COUNT) return NULL; - return sqlcipher_static_mutex[mutex]; -} - -static int sqlcipher_mem_init(void *pAppData) { - return default_mem_methods.xInit(pAppData); -} -static void sqlcipher_mem_shutdown(void *pAppData) { - default_mem_methods.xShutdown(pAppData); -} -static void *sqlcipher_mem_malloc(int n) { - void *ptr = default_mem_methods.xMalloc(n); - if(!sqlcipher_mem_executed) sqlcipher_mem_executed = 1; - if(sqlcipher_mem_security_on) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_malloc: calling sqlcipher_mlock(%p,%d)", ptr, n); - sqlcipher_mlock(ptr, n); - } - return ptr; -} -static int sqlcipher_mem_size(void *p) { - return default_mem_methods.xSize(p); -} -static void sqlcipher_mem_free(void *p) { - int sz; - if(!sqlcipher_mem_executed) sqlcipher_mem_executed = 1; - if(sqlcipher_mem_security_on) { - sz = sqlcipher_mem_size(p); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_free: calling sqlcipher_memset(%p,0,%d) and sqlcipher_munlock(%p, %d)", p, sz, p, sz); - sqlcipher_memset(p, 0, sz); - sqlcipher_munlock(p, sz); - } - default_mem_methods.xFree(p); -} -static void *sqlcipher_mem_realloc(void *p, int n) { - void *new = NULL; - int orig_sz = 0; - if(sqlcipher_mem_security_on) { - orig_sz = sqlcipher_mem_size(p); - if (n==0) { - sqlcipher_mem_free(p); - return NULL; - } else if (!p) { - return sqlcipher_mem_malloc(n); - } else if(n <= orig_sz) { - return p; - } else { - new = sqlcipher_mem_malloc(n); - if(new) { - memcpy(new, p, orig_sz); - sqlcipher_mem_free(p); - } - return new; - } - } else { - return default_mem_methods.xRealloc(p, n); - } -} - -static int sqlcipher_mem_roundup(int n) { - return default_mem_methods.xRoundup(n); -} - -static sqlite3_mem_methods sqlcipher_mem_methods = { - sqlcipher_mem_malloc, - sqlcipher_mem_free, - sqlcipher_mem_realloc, - sqlcipher_mem_size, - sqlcipher_mem_roundup, - sqlcipher_mem_init, - sqlcipher_mem_shutdown, - 0 -}; - -void sqlcipher_init_memmethods() { - if(sqlcipher_mem_initialized) return; - if(sqlite3_config(SQLITE_CONFIG_GETMALLOC, &default_mem_methods) != SQLITE_OK || - sqlite3_config(SQLITE_CONFIG_MALLOC, &sqlcipher_mem_methods) != SQLITE_OK) { - sqlcipher_mem_security_on = sqlcipher_mem_executed = sqlcipher_mem_initialized = 0; - } else { - sqlcipher_mem_initialized = 1; - } -} - -int sqlcipher_register_provider(sqlcipher_provider *p) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_register_provider: entering SQLCIPHER_MUTEX_PROVIDER"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_register_provider: entered SQLCIPHER_MUTEX_PROVIDER"); - - if(default_provider != NULL && default_provider != p) { - /* only free the current registerd provider if it has been initialized - and it isn't a pointer to the same provider passed to the function - (i.e. protect against a caller calling register twice for the same provider) */ - sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); - } - default_provider = p; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_register_provider: leaving SQLCIPHER_MUTEX_PROVIDER"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_register_provider: left SQLCIPHER_MUTEX_PROVIDER"); - - return SQLITE_OK; -} - -/* return a pointer to the currently registered provider. This will - allow an application to fetch the current registered provider and - make minor changes to it */ -sqlcipher_provider* sqlcipher_get_provider() { - return default_provider; -} - -void sqlcipher_activate() { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_activate: entering static master mutex"); - sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_activate: entered static master mutex"); - - /* allocate new mutexes */ - if(sqlcipher_activate_count == 0) { - int i; - for(i = 0; i < SQLCIPHER_MUTEX_COUNT; i++) { - sqlcipher_static_mutex[i] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - } - } - - /* check to see if there is a provider registered at this point - if there no provider registered at this point, register the - default provider */ - if(sqlcipher_get_provider() == NULL) { - sqlcipher_provider *p = sqlcipher_malloc(sizeof(sqlcipher_provider)); -#if defined (SQLCIPHER_CRYPTO_CC) - extern int sqlcipher_cc_setup(sqlcipher_provider *p); - sqlcipher_cc_setup(p); -#elif defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) - extern int sqlcipher_ltc_setup(sqlcipher_provider *p); - sqlcipher_ltc_setup(p); -#elif defined (SQLCIPHER_CRYPTO_NSS) - extern int sqlcipher_nss_setup(sqlcipher_provider *p); - sqlcipher_nss_setup(p); -#elif defined (SQLCIPHER_CRYPTO_OPENSSL) - extern int sqlcipher_openssl_setup(sqlcipher_provider *p); - sqlcipher_openssl_setup(p); -#else -#error "NO DEFAULT SQLCIPHER CRYPTO PROVIDER DEFINED" -#endif - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_activate: calling sqlcipher_register_provider(%p)", p); -#ifdef SQLCIPHER_EXT - sqlcipher_ext_provider_setup(p); -#endif - sqlcipher_register_provider(p); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_activate: called sqlcipher_register_provider(%p)",p); - } - - sqlcipher_activate_count++; /* increment activation count */ - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_activate: leaving static master mutex"); - sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_activate: left static master mutex"); -} - -void sqlcipher_deactivate() { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: entering static master mutex"); - sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: entered static master mutex"); - - sqlcipher_activate_count--; - /* if no connections are using sqlcipher, cleanup globals */ - if(sqlcipher_activate_count < 1) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: entering SQLCIPHER_MUTEX_PROVIDER"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: entered SQLCIPHER_MUTEX_PROVIDER"); - - if(default_provider != NULL) { - sqlcipher_free(default_provider, sizeof(sqlcipher_provider)); - default_provider = NULL; - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: leaving SQLCIPHER_MUTEX_PROVIDER"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: left SQLCIPHER_MUTEX_PROVIDER"); - -#ifdef SQLCIPHER_EXT - sqlcipher_ext_provider_destroy(); -#endif - - /* last connection closed, free mutexes */ - if(sqlcipher_activate_count == 0) { - int i; - for(i = 0; i < SQLCIPHER_MUTEX_COUNT; i++) { - sqlite3_mutex_free(sqlcipher_static_mutex[i]); - } - } - sqlcipher_activate_count = 0; /* reset activation count */ - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: leaving static master mutex"); - sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_deactivate: left static master mutex"); -} - -/* constant time memset using volitile to avoid having the memset - optimized out by the compiler. - Note: As suggested by Joachim Schipper (joachim.schipper@fox-it.com) -*/ -void* sqlcipher_memset(void *v, unsigned char value, sqlite_uint64 len) { - volatile sqlite_uint64 i = 0; - volatile unsigned char *a = v; - - if (v == NULL) return v; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_memset: setting %p[0-%llu]=%d)", a, len, value); - for(i = 0; i < len; i++) { - a[i] = value; - } - - return v; -} - -/* constant time memory check tests every position of a memory segement - matches a single value (i.e. the memory is all zeros) - returns 0 if match, 1 of no match */ -int sqlcipher_ismemset(const void *v, unsigned char value, sqlite_uint64 len) { - const volatile unsigned char *a = v; - volatile sqlite_uint64 i = 0, result = 0; - - for(i = 0; i < len; i++) { - result |= a[i] ^ value; - } - - return (result != 0); -} - -/* constant time memory comparison routine. - returns 0 if match, 1 if no match */ -int sqlcipher_memcmp(const void *v0, const void *v1, int len) { - const volatile unsigned char *a0 = v0, *a1 = v1; - volatile int i = 0, result = 0; - - for(i = 0; i < len; i++) { - result |= a0[i] ^ a1[i]; - } - - return (result != 0); -} - -void sqlcipher_mlock(void *ptr, sqlite_uint64 sz) { -#ifndef OMIT_MEMLOCK -#if defined(__unix__) || defined(__APPLE__) - int rc; - unsigned long pagesize = sysconf(_SC_PAGESIZE); - unsigned long offset = (unsigned long) ptr % pagesize; - - if(ptr == NULL || sz == 0) return; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_lock: calling mlock(%p,%lu); _SC_PAGESIZE=%lu", ptr - offset, sz + offset, pagesize); - rc = mlock(ptr - offset, sz + offset); - if(rc!=0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_mem_lock: mlock(%p,%lu) returned %d errno=%d", ptr - offset, sz + offset, rc, errno); - } -#elif defined(_WIN32) -#if !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_APP)) - int rc; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_lock: calling VirtualLock(%p,%d)", ptr, sz); - rc = VirtualLock(ptr, sz); - if(rc==0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_mem_lock: VirtualLock(%p,%d) returned %d LastError=%d", ptr, sz, rc, GetLastError()); - } -#endif -#endif -#endif -} - -void sqlcipher_munlock(void *ptr, sqlite_uint64 sz) { -#ifndef OMIT_MEMLOCK -#if defined(__unix__) || defined(__APPLE__) - int rc; - unsigned long pagesize = sysconf(_SC_PAGESIZE); - unsigned long offset = (unsigned long) ptr % pagesize; - - if(ptr == NULL || sz == 0) return; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_unlock: calling munlock(%p,%lu)", ptr - offset, sz + offset); - rc = munlock(ptr - offset, sz + offset); - if(rc!=0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_mem_unlock: munlock(%p,%lu) returned %d errno=%d", ptr - offset, sz + offset, rc, errno); - } -#elif defined(_WIN32) -#if !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_APP)) - int rc; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_mem_lock: calling VirtualUnlock(%p,%d)", ptr, sz); - rc = VirtualUnlock(ptr, sz); - if(!rc) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_mem_unlock: VirtualUnlock(%p,%d) returned %d LastError=%d", ptr, sz, rc, GetLastError()); - } -#endif -#endif -#endif -} - -/** - * Free and wipe memory. Uses SQLites internal sqlite3_free so that memory - * can be countend and memory leak detection works in the test suite. - * If ptr is not null memory will be freed. - * If sz is greater than zero, the memory will be overwritten with zero before it is freed - * If sz is > 0, and not compiled with OMIT_MEMLOCK, system will attempt to unlock the - * memory segment so it can be paged - */ -void sqlcipher_free(void *ptr, sqlite_uint64 sz) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_free: calling sqlcipher_memset(%p,0,%llu)", ptr, sz); - sqlcipher_memset(ptr, 0, sz); - sqlcipher_munlock(ptr, sz); - sqlite3_free(ptr); -} - -/** - * allocate memory. Uses sqlite's internall malloc wrapper so memory can be - * reference counted and leak detection works. Unless compiled with OMIT_MEMLOCK - * attempts to lock the memory pages so sensitive information won't be swapped - */ -void* sqlcipher_malloc(sqlite_uint64 sz) { - void *ptr; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_malloc: calling sqlite3Malloc(%llu)", sz); - ptr = sqlite3Malloc(sz); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_malloc: calling sqlcipher_memset(%p,0,%llu)", ptr, sz); - sqlcipher_memset(ptr, 0, sz); - sqlcipher_mlock(ptr, sz); - return ptr; -} - -char* sqlcipher_version() { -#ifdef CIPHER_VERSION_QUALIFIER - char *version = sqlite3_mprintf("%s %s %s", CIPHER_XSTR(CIPHER_VERSION_NUMBER), CIPHER_XSTR(CIPHER_VERSION_QUALIFIER), CIPHER_XSTR(CIPHER_VERSION_BUILD)); -#else - char *version = sqlite3_mprintf("%s %s", CIPHER_XSTR(CIPHER_VERSION_NUMBER), CIPHER_XSTR(CIPHER_VERSION_BUILD)); -#endif - return version; -} - -/** - * Initialize new cipher_ctx struct. This function will allocate memory - * for the cipher context and for the key - * - * returns SQLITE_OK if initialization was successful - * returns SQLITE_NOMEM if an error occured allocating memory - */ -static int sqlcipher_cipher_ctx_init(codec_ctx *ctx, cipher_ctx **iCtx) { - cipher_ctx *c_ctx; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_cipher_ctx_init: allocating context"); - *iCtx = (cipher_ctx *) sqlcipher_malloc(sizeof(cipher_ctx)); - c_ctx = *iCtx; - if(c_ctx == NULL) return SQLITE_NOMEM; - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_cipher_ctx_init: allocating key"); - c_ctx->key = (unsigned char *) sqlcipher_malloc(ctx->key_sz); - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_cipher_ctx_init: allocating hmac_key"); - c_ctx->hmac_key = (unsigned char *) sqlcipher_malloc(ctx->key_sz); - - if(c_ctx->key == NULL) return SQLITE_NOMEM; - if(c_ctx->hmac_key == NULL) return SQLITE_NOMEM; - - return SQLITE_OK; -} - -/** - * Free and wipe memory associated with a cipher_ctx - */ -static void sqlcipher_cipher_ctx_free(codec_ctx* ctx, cipher_ctx **iCtx) { - cipher_ctx *c_ctx = *iCtx; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "cipher_ctx_free: iCtx=%p", iCtx); - sqlcipher_free(c_ctx->key, ctx->key_sz); - sqlcipher_free(c_ctx->hmac_key, ctx->key_sz); - sqlcipher_free(c_ctx->pass, c_ctx->pass_sz); - sqlcipher_free(c_ctx->keyspec, ctx->keyspec_sz); - sqlcipher_free(c_ctx, sizeof(cipher_ctx)); -} - -static int sqlcipher_codec_ctx_reserve_setup(codec_ctx *ctx) { - int base_reserve = ctx->iv_sz; /* base reserve size will be IV only */ - int reserve = base_reserve; - - ctx->hmac_sz = ctx->provider->get_hmac_sz(ctx->provider_ctx, ctx->hmac_algorithm); - - if(sqlcipher_codec_ctx_get_use_hmac(ctx)) - reserve += ctx->hmac_sz; /* if reserve will include hmac, update that size */ - - /* calculate the amount of reserve needed in even increments of the cipher block size */ - if(ctx->block_sz > 0) { - reserve = ((reserve % ctx->block_sz) == 0) ? reserve : - ((reserve / ctx->block_sz) + 1) * ctx->block_sz; - } - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_reserve_setup: base_reserve=%d block_sz=%d md_size=%d reserve=%d", - base_reserve, ctx->block_sz, ctx->hmac_sz, reserve); - - ctx->reserve_sz = reserve; - - return SQLITE_OK; -} - -/** - * Compare one cipher_ctx to another. - * - * returns 0 if all the parameters (except the derived key data) are the same - * returns 1 otherwise - */ -static int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) { - int are_equal = ( - c1->pass_sz == c2->pass_sz - && ( - c1->pass == c2->pass - || !sqlcipher_memcmp((const unsigned char*)c1->pass, - (const unsigned char*)c2->pass, - c1->pass_sz) - )); - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_cipher_ctx_cmp: c1=%p c2=%p sqlcipher_memcmp(c1->pass, c2_pass)=%d are_equal=%d", - c1, c2, - (c1->pass == NULL || c2->pass == NULL) ? - -1 : - sqlcipher_memcmp( - (const unsigned char*)c1->pass, - (const unsigned char*)c2->pass, - c1->pass_sz - ), - are_equal - ); - - return !are_equal; /* return 0 if they are the same, 1 otherwise */ -} - -/** - * Copy one cipher_ctx to another. For instance, assuming that read_ctx is a - * fully initialized context, you could copy it to write_ctx and all yet data - * and pass information across - * - * returns SQLITE_OK if initialization was successful - * returns SQLITE_NOMEM if an error occured allocating memory - */ -static int sqlcipher_cipher_ctx_copy(codec_ctx *ctx, cipher_ctx *target, cipher_ctx *source) { - void *key = target->key; - void *hmac_key = target->hmac_key; - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_cipher_ctx_copy: target=%p, source=%p", target, source); - sqlcipher_free(target->pass, target->pass_sz); - sqlcipher_free(target->keyspec, ctx->keyspec_sz); - memcpy(target, source, sizeof(cipher_ctx)); - - target->key = key; /* restore pointer to previously allocated key data */ - memcpy(target->key, source->key, ctx->key_sz); - - target->hmac_key = hmac_key; /* restore pointer to previously allocated hmac key data */ - memcpy(target->hmac_key, source->hmac_key, ctx->key_sz); - - if(source->pass && source->pass_sz) { - target->pass = sqlcipher_malloc(source->pass_sz); - if(target->pass == NULL) return SQLITE_NOMEM; - memcpy(target->pass, source->pass, source->pass_sz); - } - if(source->keyspec) { - target->keyspec = sqlcipher_malloc(ctx->keyspec_sz); - if(target->keyspec == NULL) return SQLITE_NOMEM; - memcpy(target->keyspec, source->keyspec, ctx->keyspec_sz); - } - return SQLITE_OK; -} - -/** - * Set the keyspec for the cipher_ctx - * - * returns SQLITE_OK if assignment was successfull - * returns SQLITE_NOMEM if an error occured allocating memory - */ -static int sqlcipher_cipher_ctx_set_keyspec(codec_ctx *ctx, cipher_ctx *c_ctx, const unsigned char *key) { - /* free, zero existing pointers and size */ - sqlcipher_free(c_ctx->keyspec, ctx->keyspec_sz); - c_ctx->keyspec = NULL; - - c_ctx->keyspec = sqlcipher_malloc(ctx->keyspec_sz); - if(c_ctx->keyspec == NULL) return SQLITE_NOMEM; - - c_ctx->keyspec[0] = 'x'; - c_ctx->keyspec[1] = '\''; - cipher_bin2hex(key, ctx->key_sz, c_ctx->keyspec + 2); - cipher_bin2hex(ctx->kdf_salt, ctx->kdf_salt_sz, c_ctx->keyspec + (ctx->key_sz * 2) + 2); - c_ctx->keyspec[ctx->keyspec_sz - 1] = '\''; - return SQLITE_OK; -} - -int sqlcipher_codec_get_store_pass(codec_ctx *ctx) { - return ctx->store_pass; -} - -void sqlcipher_codec_set_store_pass(codec_ctx *ctx, int value) { - ctx->store_pass = value; -} - -void sqlcipher_codec_get_pass(codec_ctx *ctx, void **zKey, int *nKey) { - *zKey = ctx->read_ctx->pass; - *nKey = ctx->read_ctx->pass_sz; -} - -static void sqlcipher_set_derive_key(codec_ctx *ctx, int derive) { - if(ctx->read_ctx != NULL) ctx->read_ctx->derive_key = derive; - if(ctx->write_ctx != NULL) ctx->write_ctx->derive_key = derive; -} - -/** - * Set the passphrase for the cipher_ctx - * - * returns SQLITE_OK if assignment was successfull - * returns SQLITE_NOMEM if an error occured allocating memory - */ -static int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) { - /* free, zero existing pointers and size */ - sqlcipher_free(ctx->pass, ctx->pass_sz); - ctx->pass = NULL; - ctx->pass_sz = 0; - - if(zKey && nKey) { /* if new password is provided, copy it */ - ctx->pass_sz = nKey; - ctx->pass = sqlcipher_malloc(nKey); - if(ctx->pass == NULL) return SQLITE_NOMEM; - memcpy(ctx->pass, zKey, nKey); - } - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_set_pass(codec_ctx *ctx, const void *zKey, int nKey, int for_ctx) { - cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; - int rc; - - if((rc = sqlcipher_cipher_ctx_set_pass(c_ctx, zKey, nKey)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_set_pass: error %d from sqlcipher_cipher_ctx_set_pass", rc); - return rc; - } - - c_ctx->derive_key = 1; - - if(for_ctx == 2) { - if((rc = sqlcipher_cipher_ctx_copy(ctx, for_ctx ? ctx->read_ctx : ctx->write_ctx, c_ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_set_pass: error %d from sqlcipher_cipher_ctx_copy", rc); - return rc; - } - } - - return SQLITE_OK; -} - -const char* sqlcipher_codec_ctx_get_cipher(codec_ctx *ctx) { - return ctx->provider->get_cipher(ctx->provider_ctx); -} - -/* set the global default KDF iteration */ -void sqlcipher_set_default_kdf_iter(int iter) { - default_kdf_iter = iter; -} - -int sqlcipher_get_default_kdf_iter() { - return default_kdf_iter; -} - -int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *ctx, int kdf_iter) { - ctx->kdf_iter = kdf_iter; - sqlcipher_set_derive_key(ctx, 1); - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_get_kdf_iter(codec_ctx *ctx) { - return ctx->kdf_iter; -} - -int sqlcipher_codec_ctx_set_fast_kdf_iter(codec_ctx *ctx, int fast_kdf_iter) { - ctx->fast_kdf_iter = fast_kdf_iter; - sqlcipher_set_derive_key(ctx, 1); - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_get_fast_kdf_iter(codec_ctx *ctx) { - return ctx->fast_kdf_iter; -} - -/* set the global default flag for HMAC */ -void sqlcipher_set_default_use_hmac(int use) { - if(use) SQLCIPHER_FLAG_SET(default_flags, CIPHER_FLAG_HMAC); - else SQLCIPHER_FLAG_UNSET(default_flags,CIPHER_FLAG_HMAC); -} - -int sqlcipher_get_default_use_hmac() { - return SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC); -} - -void sqlcipher_set_hmac_salt_mask(unsigned char mask) { - hmac_salt_mask = mask; -} - -unsigned char sqlcipher_get_hmac_salt_mask() { - return hmac_salt_mask; -} - -/* set the codec flag for whether this individual database should be using hmac */ -int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) { - if(use) { - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_HMAC); - } else { - SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_HMAC); - } - - return sqlcipher_codec_ctx_reserve_setup(ctx); -} - -int sqlcipher_codec_ctx_get_use_hmac(codec_ctx *ctx) { - return SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC); -} - -/* the length of plaintext header size must be: - * 1. greater than or equal to zero - * 2. a multiple of the cipher block size - * 3. less than the usable size of the first database page - */ -int sqlcipher_set_default_plaintext_header_size(int size) { - default_plaintext_header_sz = size; - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_set_plaintext_header_size(codec_ctx *ctx, int size) { - if(size >= 0 && ctx->block_sz > 0 && (size % ctx->block_sz) == 0 && size < (ctx->page_sz - ctx->reserve_sz)) { - ctx->plaintext_header_sz = size; - return SQLITE_OK; - } - ctx->plaintext_header_sz = -1; - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_set_plaintext_header_size: attempt to set invalid plantext_header_size %d", size); - return SQLITE_ERROR; -} - -int sqlcipher_get_default_plaintext_header_size() { - return default_plaintext_header_sz; -} - -int sqlcipher_codec_ctx_get_plaintext_header_size(codec_ctx *ctx) { - return ctx->plaintext_header_sz; -} - -/* manipulate HMAC algorithm */ -int sqlcipher_set_default_hmac_algorithm(int algorithm) { - default_hmac_algorithm = algorithm; - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_set_hmac_algorithm(codec_ctx *ctx, int algorithm) { - ctx->hmac_algorithm = algorithm; - return sqlcipher_codec_ctx_reserve_setup(ctx); -} - -int sqlcipher_get_default_hmac_algorithm() { - return default_hmac_algorithm; -} - -int sqlcipher_codec_ctx_get_hmac_algorithm(codec_ctx *ctx) { - return ctx->hmac_algorithm; -} - -/* manipulate KDF algorithm */ -int sqlcipher_set_default_kdf_algorithm(int algorithm) { - default_kdf_algorithm = algorithm; - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_set_kdf_algorithm(codec_ctx *ctx, int algorithm) { - ctx->kdf_algorithm = algorithm; - return SQLITE_OK; -} - -int sqlcipher_get_default_kdf_algorithm() { - return default_kdf_algorithm; -} - -int sqlcipher_codec_ctx_get_kdf_algorithm(codec_ctx *ctx) { - return ctx->kdf_algorithm; -} - -void sqlcipher_codec_ctx_set_error(codec_ctx *ctx, int error) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_set_error: ctx=%p, error=%d", ctx, error); - sqlite3pager_error(ctx->pBt->pBt->pPager, error); - ctx->pBt->pBt->db->errCode = error; -} - -int sqlcipher_codec_ctx_get_reservesize(codec_ctx *ctx) { - return ctx->reserve_sz; -} - -void* sqlcipher_codec_ctx_get_data(codec_ctx *ctx) { - return ctx->buffer; -} - -static int sqlcipher_codec_ctx_init_kdf_salt(codec_ctx *ctx) { - sqlite3_file *fd = sqlite3PagerFile(ctx->pBt->pBt->pPager); - - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) { - return SQLITE_OK; /* don't reload salt when not needed */ - } - - /* read salt from header, if present, otherwise generate a new random salt */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_init_kdf_salt: obtaining salt"); - if(fd == NULL || fd->pMethods == 0 || sqlite3OsRead(fd, ctx->kdf_salt, ctx->kdf_salt_sz, 0) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_init_kdf_salt: unable to read salt from file header, generating random"); - if(ctx->provider->random(ctx->provider_ctx, ctx->kdf_salt, ctx->kdf_salt_sz) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init_kdf_salt: error retrieving random bytes from provider"); - return SQLITE_ERROR; - } - } - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT); - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_set_kdf_salt(codec_ctx *ctx, unsigned char *salt, int size) { - if(size >= ctx->kdf_salt_sz) { - memcpy(ctx->kdf_salt, salt, ctx->kdf_salt_sz); - SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT); - return SQLITE_OK; - } - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_set_kdf_salt: attempt to set salt of incorrect size %d", size); - return SQLITE_ERROR; -} - -int sqlcipher_codec_ctx_get_kdf_salt(codec_ctx *ctx, void** salt) { - int rc = SQLITE_OK; - if(!SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) { - if((rc = sqlcipher_codec_ctx_init_kdf_salt(ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_get_kdf_salt: error %d from sqlcipher_codec_ctx_init_kdf_salt", rc); - } - } - *salt = ctx->kdf_salt; - - return rc; -} - -void sqlcipher_codec_get_keyspec(codec_ctx *ctx, void **zKey, int *nKey) { - *zKey = ctx->read_ctx->keyspec; - *nKey = ctx->keyspec_sz; -} - -int sqlcipher_codec_ctx_set_pagesize(codec_ctx *ctx, int size) { - if(!((size != 0) && ((size & (size - 1)) == 0)) || size < 512 || size > 65536) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "cipher_page_size not a power of 2 and between 512 and 65536 inclusive"); - return SQLITE_ERROR; - } - /* attempt to free the existing page buffer */ - sqlcipher_free(ctx->buffer,ctx->page_sz); - ctx->page_sz = size; - - /* pre-allocate a page buffer of PageSize bytes. This will - be used as a persistent buffer for encryption and decryption - operations to avoid overhead of multiple memory allocations*/ - ctx->buffer = sqlcipher_malloc(size); - if(ctx->buffer == NULL) return SQLITE_NOMEM; - - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_get_pagesize(codec_ctx *ctx) { - return ctx->page_sz; -} - -void sqlcipher_set_default_pagesize(int page_size) { - default_page_size = page_size; -} - -int sqlcipher_get_default_pagesize() { - return default_page_size; -} - -void sqlcipher_set_mem_security(int on) { - /* memory security can only be enabled, not disabled */ - if(on) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_set_mem_security: on"); - sqlcipher_mem_security_on = on; - } -} - -int sqlcipher_get_mem_security() { - /* only report that memory security is enabled if pragma cipher_memory_security is ON and - SQLCipher's allocator/deallocator was run at least one timecurrently used */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_get_mem_security: sqlcipher_mem_security_on = %d, sqlcipher_mem_executed = %d", sqlcipher_mem_security_on, sqlcipher_mem_executed); - return sqlcipher_mem_security_on && sqlcipher_mem_executed; -} - - -int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, const void *zKey, int nKey) { - int rc; - codec_ctx *ctx; - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_init: allocating context"); - - *iCtx = sqlcipher_malloc(sizeof(codec_ctx)); - ctx = *iCtx; - - if(ctx == NULL) return SQLITE_NOMEM; - - ctx->pBt = pDb->pBt; /* assign pointer to database btree structure */ - - /* allocate space for salt data. Then read the first 16 bytes - directly off the database file. This is the salt for the - key derivation function. If we get a short read allocate - a new random salt value */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_init: allocating kdf_salt"); - ctx->kdf_salt_sz = FILE_HEADER_SZ; - ctx->kdf_salt = sqlcipher_malloc(ctx->kdf_salt_sz); - if(ctx->kdf_salt == NULL) return SQLITE_NOMEM; - - /* allocate space for separate hmac salt data. We want the - HMAC derivation salt to be different than the encryption - key derivation salt */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_init: allocating hmac_kdf_salt"); - ctx->hmac_kdf_salt = sqlcipher_malloc(ctx->kdf_salt_sz); - if(ctx->hmac_kdf_salt == NULL) return SQLITE_NOMEM; - - /* setup default flags */ - ctx->flags = default_flags; - - /* setup the crypto provider */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_ctx_init: allocating provider"); - ctx->provider = (sqlcipher_provider *) sqlcipher_malloc(sizeof(sqlcipher_provider)); - if(ctx->provider == NULL) return SQLITE_NOMEM; - - /* make a copy of the provider to be used for the duration of the context */ - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_codec_ctx_init: entering SQLCIPHER_MUTEX_PROVIDER"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_codec_ctx_init: entered SQLCIPHER_MUTEX_PROVIDER"); - - memcpy(ctx->provider, default_provider, sizeof(sqlcipher_provider)); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_codec_ctx_init: leaving SQLCIPHER_MUTEX_PROVIDER"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_codec_ctx_init: left SQLCIPHER_MUTEX_PROVIDER"); - - if((rc = ctx->provider->ctx_init(&ctx->provider_ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d returned from ctx_init", rc); - return rc; - } - - ctx->key_sz = ctx->provider->get_key_sz(ctx->provider_ctx); - ctx->iv_sz = ctx->provider->get_iv_sz(ctx->provider_ctx); - ctx->block_sz = ctx->provider->get_block_sz(ctx->provider_ctx); - - /* establic the size for a hex-formated key specification, containing the - raw encryption key and the salt used to generate it format. will be x'hexkey...hexsalt' - so oversize by 3 bytes */ - ctx->keyspec_sz = ((ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3; - - /* - Always overwrite page size and set to the default because the first page of the database - in encrypted and thus sqlite can't effectively determine the pagesize. this causes an issue in - cases where bytes 16 & 17 of the page header are a power of 2 as reported by John Lehman - */ - if((rc = sqlcipher_codec_ctx_set_pagesize(ctx, default_page_size)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d returned from sqlcipher_codec_ctx_set_pagesize with %d", rc, default_page_size); - return rc; - } - - /* establish settings for the KDF iterations and fast (HMAC) KDF iterations */ - if((rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, default_kdf_iter)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting default_kdf_iter %d", rc, default_kdf_iter); - return rc; - } - - if((rc = sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, FAST_PBKDF2_ITER)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting fast_kdf_iter to %d", rc, FAST_PBKDF2_ITER); - return rc; - } - - /* set the default HMAC and KDF algorithms which will determine the reserve size */ - if((rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, default_hmac_algorithm)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting sqlcipher_codec_ctx_set_hmac_algorithm with %d", rc, default_hmac_algorithm); - return rc; - } - - /* Note that use_hmac is a special case that requires recalculation of page size - so we call set_use_hmac to perform setup */ - if((rc = sqlcipher_codec_ctx_set_use_hmac(ctx, SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC))) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting use_hmac %d", rc, SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC)); - return rc; - } - - if((rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, default_kdf_algorithm)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting sqlcipher_codec_ctx_set_kdf_algorithm with %d", rc, default_kdf_algorithm); - return rc; - } - - /* setup the default plaintext header size */ - if((rc = sqlcipher_codec_ctx_set_plaintext_header_size(ctx, default_plaintext_header_sz)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting sqlcipher_codec_ctx_set_plaintext_header_size with %d", rc, default_plaintext_header_sz); - return rc; - } - - /* initialize the read and write sub-contexts. this must happen after key_sz is established */ - if((rc = sqlcipher_cipher_ctx_init(ctx, &ctx->read_ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d initializing read_ctx", rc); - return rc; - } - - if((rc = sqlcipher_cipher_ctx_init(ctx, &ctx->write_ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d initializing write_ctx", rc); - return rc; - } - - /* set the key material on one of the sub cipher contexts and sync them up */ - if((rc = sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, 0)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d setting pass key", rc); - return rc; - } - - if((rc = sqlcipher_cipher_ctx_copy(ctx, ctx->write_ctx, ctx->read_ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_ctx_init: error %d copying write_ctx to read_ctx", rc); - return rc; - } - - return SQLITE_OK; -} - -/** - * Free and wipe memory associated with a cipher_ctx, including the allocated - * read_ctx and write_ctx. - */ -void sqlcipher_codec_ctx_free(codec_ctx **iCtx) { - codec_ctx *ctx = *iCtx; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "codec_ctx_free: iCtx=%p", iCtx); - sqlcipher_free(ctx->kdf_salt, ctx->kdf_salt_sz); - sqlcipher_free(ctx->hmac_kdf_salt, ctx->kdf_salt_sz); - sqlcipher_free(ctx->buffer, ctx->page_sz); - - ctx->provider->ctx_free(&ctx->provider_ctx); - sqlcipher_free(ctx->provider, sizeof(sqlcipher_provider)); - - sqlcipher_cipher_ctx_free(ctx, &ctx->read_ctx); - sqlcipher_cipher_ctx_free(ctx, &ctx->write_ctx); - sqlcipher_free(ctx, sizeof(codec_ctx)); -} - -/** convert a 32bit unsigned integer to little endian byte ordering */ -static void sqlcipher_put4byte_le(unsigned char *p, u32 v) { - p[0] = (u8)v; - p[1] = (u8)(v>>8); - p[2] = (u8)(v>>16); - p[3] = (u8)(v>>24); -} - -static int sqlcipher_page_hmac(codec_ctx *ctx, cipher_ctx *c_ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) { - unsigned char pgno_raw[sizeof(pgno)]; - /* we may convert page number to consistent representation before calculating MAC for - compatibility across big-endian and little-endian platforms. - - Note: The public release of sqlcipher 2.0.0 to 2.0.6 had a bug where the bytes of pgno - were used directly in the MAC. SQLCipher convert's to little endian by default to preserve - backwards compatibility on the most popular platforms, but can optionally be configured - to use either big endian or native byte ordering via pragma. */ - - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_LE_PGNO)) { /* compute hmac using little endian pgno*/ - sqlcipher_put4byte_le(pgno_raw, pgno); - } else if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_BE_PGNO)) { /* compute hmac using big endian pgno */ - sqlite3Put4byte(pgno_raw, pgno); /* sqlite3Put4byte converts 32bit uint to big endian */ - } else { /* use native byte ordering */ - memcpy(pgno_raw, &pgno, sizeof(pgno)); - } - - /* include the encrypted page data, initialization vector, and page number in HMAC. This will - prevent both tampering with the ciphertext, manipulation of the IV, or resequencing otherwise - valid pages out of order in a database */ - return ctx->provider->hmac( - ctx->provider_ctx, ctx->hmac_algorithm, c_ctx->hmac_key, - ctx->key_sz, in, - in_sz, (unsigned char*) &pgno_raw, - sizeof(pgno), out); -} - -/* - * ctx - codec context - * pgno - page number in database - * size - size in bytes of input and output buffers - * mode - 1 to encrypt, 0 to decrypt - * in - pointer to input bytes - * out - pouter to output bytes - */ -int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int page_sz, unsigned char *in, unsigned char *out) { - cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx; - unsigned char *iv_in, *iv_out, *hmac_in, *hmac_out, *out_start; - int size; - - /* calculate some required positions into various buffers */ - size = page_sz - ctx->reserve_sz; /* adjust size to useable size and memset reserve at end of page */ - iv_out = out + size; - iv_in = in + size; - - /* hmac will be written immediately after the initialization vector. the remainder of the page reserve will contain - random bytes. note, these pointers are only valid when using hmac */ - hmac_in = in + size + ctx->iv_sz; - hmac_out = out + size + ctx->iv_sz; - out_start = out; /* note the original position of the output buffer pointer, as out will be rewritten during encryption */ - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_page_cipher: pgno=%d, mode=%d, size=%d", pgno, mode, size); - CODEC_HEXDUMP("sqlcipher_page_cipher: input page data", in, page_sz); - - /* the key size should never be zero. If it is, error out. */ - if(ctx->key_sz == 0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_page_cipher: error possible context corruption, key_sz is zero for pgno=%d", pgno); - goto error; - } - - if(mode == CIPHER_ENCRYPT) { - /* start at front of the reserve block, write random data to the end */ - if(ctx->provider->random(ctx->provider_ctx, iv_out, ctx->reserve_sz) != SQLITE_OK) goto error; - } else { /* CIPHER_DECRYPT */ - memcpy(iv_out, iv_in, ctx->iv_sz); /* copy the iv from the input to output buffer */ - } - - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == CIPHER_DECRYPT)) { - if(sqlcipher_page_hmac(ctx, c_ctx, pgno, in, size + ctx->iv_sz, hmac_out) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_page_cipher: hmac operation on decrypt failed for pgno=%d", pgno); - goto error; - } - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_page_cipher: comparing hmac on in=%p out=%p hmac_sz=%d", hmac_in, hmac_out, ctx->hmac_sz); - if(sqlcipher_memcmp(hmac_in, hmac_out, ctx->hmac_sz) != 0) { /* the hmac check failed */ - if(sqlite3BtreeGetAutoVacuum(ctx->pBt) != BTREE_AUTOVACUUM_NONE && sqlcipher_ismemset(in, 0, page_sz) == 0) { - /* first check if the entire contents of the page is zeros. If so, this page - resulted from a short read (i.e. sqlite attempted to pull a page after the end of the file. these - short read failures must be ignored for autovaccum mode to work so wipe the output buffer - and return SQLITE_OK to skip the decryption step. */ - sqlcipher_log(SQLCIPHER_LOG_WARN, "sqlcipher_page_cipher: zeroed page (short read) for pgno %d, encryption but returning SQLITE_OK", pgno); - sqlcipher_memset(out, 0, page_sz); - return SQLITE_OK; - } else { - /* if the page memory is not all zeros, it means the there was data and a hmac on the page. - since the check failed, the page was either tampered with or corrupted. wipe the output buffer, - and return SQLITE_ERROR to the caller */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_page_cipher: hmac check failed for pgno=%d returning SQLITE_ERROR", pgno); - goto error; - } - } - } - - if(ctx->provider->cipher(ctx->provider_ctx, mode, c_ctx->key, ctx->key_sz, iv_out, in, size, out) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_page_cipher: cipher operation mode=%d failed for pgno=%d returning SQLITE_ERROR", mode, pgno); - goto error; - }; - - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == CIPHER_ENCRYPT)) { - if(sqlcipher_page_hmac(ctx, c_ctx, pgno, out_start, size + ctx->iv_sz, hmac_out) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_page_cipher: hmac operation on encrypt failed for pgno=%d", pgno); - goto error; - }; - } - - CODEC_HEXDUMP("sqlcipher_page_cipher: output page data", out_start, page_sz); - - return SQLITE_OK; -error: - sqlcipher_memset(out, 0, page_sz); - return SQLITE_ERROR; -} - -/** - * Derive an encryption key for a cipher contex key based on the raw password. - * - * If the raw key data is formated as x'hex' and there are exactly enough hex chars to fill - * the key (i.e 64 hex chars for a 256 bit key) then the key data will be used directly. - - * Else, if the raw key data is formated as x'hex' and there are exactly enough hex chars to fill - * the key and the salt (i.e 92 hex chars for a 256 bit key and 16 byte salt) then it will be unpacked - * as the key followed by the salt. - * - * Otherwise, a key data will be derived using PBKDF2 - * - * returns SQLITE_OK if initialization was successful - * returns SQLITE_ERROR if the key could't be derived (for instance if pass is NULL or pass_sz is 0) - */ -static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) { - int rc; - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_cipher_ctx_key_derive: ctx->kdf_salt_sz=%d ctx->kdf_iter=%d ctx->fast_kdf_iter=%d ctx->key_sz=%d", - ctx->kdf_salt_sz, ctx->kdf_iter, ctx->fast_kdf_iter, ctx->key_sz); - - if(c_ctx->pass && c_ctx->pass_sz) { /* if key material is present on the context for derivation */ - - /* if necessary, initialize the salt from the header or random source */ - if(!SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) { - if((rc = sqlcipher_codec_ctx_init_kdf_salt(ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_cipher_ctx_key_derive: error %d from sqlcipher_codec_ctx_init_kdf_salt", rc); - return rc; - } - } - - if (c_ctx->pass_sz == ((ctx->key_sz * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0 && cipher_isHex(c_ctx->pass + 2, ctx->key_sz * 2)) { - int n = c_ctx->pass_sz - 3; /* adjust for leading x' and tailing ' */ - const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "cipher_ctx_key_derive: using raw key from hex"); - cipher_hex2bin(z, n, c_ctx->key); - } else if (c_ctx->pass_sz == (((ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3) && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0 && cipher_isHex(c_ctx->pass + 2, (ctx->key_sz + ctx->kdf_salt_sz) * 2)) { - const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "cipher_ctx_key_derive: using raw key from hex"); - cipher_hex2bin(z, (ctx->key_sz * 2), c_ctx->key); - cipher_hex2bin(z + (ctx->key_sz * 2), (ctx->kdf_salt_sz * 2), ctx->kdf_salt); - } else { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "cipher_ctx_key_derive: deriving key using full PBKDF2 with %d iterations", ctx->kdf_iter); - if(ctx->provider->kdf(ctx->provider_ctx, ctx->kdf_algorithm, c_ctx->pass, c_ctx->pass_sz, - ctx->kdf_salt, ctx->kdf_salt_sz, ctx->kdf_iter, - ctx->key_sz, c_ctx->key) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "cipher_ctx_key_derive: error occurred from provider kdf generating encryption key"); - return SQLITE_ERROR; - } - } - - /* set the context "keyspec" containing the hex-formatted key and salt to be used when attaching databases */ - if((rc = sqlcipher_cipher_ctx_set_keyspec(ctx, c_ctx, c_ctx->key)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_cipher_ctx_key_derive: error %d from sqlcipher_cipher_ctx_set_keyspec", rc); - return rc; - } - - /* if this context is setup to use hmac checks, generate a seperate and different - key for HMAC. In this case, we use the output of the previous KDF as the input to - this KDF run. This ensures a distinct but predictable HMAC key. */ - if(ctx->flags & CIPHER_FLAG_HMAC) { - int i; - - /* start by copying the kdf key into the hmac salt slot - then XOR it with the fixed hmac salt defined at compile time - this ensures that the salt passed in to derive the hmac key, while - easy to derive and publically known, is not the same as the salt used - to generate the encryption key */ - memcpy(ctx->hmac_kdf_salt, ctx->kdf_salt, ctx->kdf_salt_sz); - for(i = 0; i < ctx->kdf_salt_sz; i++) { - ctx->hmac_kdf_salt[i] ^= hmac_salt_mask; - } - - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "cipher_ctx_key_derive: deriving hmac key from encryption key using PBKDF2 with %d iterations", - ctx->fast_kdf_iter); - - - if(ctx->provider->kdf(ctx->provider_ctx, ctx->kdf_algorithm, c_ctx->key, ctx->key_sz, - ctx->hmac_kdf_salt, ctx->kdf_salt_sz, ctx->fast_kdf_iter, - ctx->key_sz, c_ctx->hmac_key) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "cipher_ctx_key_derive: error occurred from provider kdf generating HMAC key"); - return SQLITE_ERROR; - } - } - - c_ctx->derive_key = 0; - return SQLITE_OK; - } - sqlcipher_log(SQLCIPHER_LOG_ERROR, "cipher_ctx_key_derive: key material is not present on the context for key derivation"); - return SQLITE_ERROR; -} - -int sqlcipher_codec_key_derive(codec_ctx *ctx) { - /* derive key on first use if necessary */ - if(ctx->read_ctx->derive_key) { - if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->read_ctx) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_key_derive: error occurred deriving read_ctx key"); - return SQLITE_ERROR; - } - } - - if(ctx->write_ctx->derive_key) { - if(sqlcipher_cipher_ctx_cmp(ctx->write_ctx, ctx->read_ctx) == 0) { - /* the relevant parameters are the same, just copy read key */ - if(sqlcipher_cipher_ctx_copy(ctx, ctx->write_ctx, ctx->read_ctx) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_key_derive: error occurred copying read_ctx to write_ctx"); - return SQLITE_ERROR; - } - } else { - if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->write_ctx) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_key_derive: error occurred deriving write_ctx key"); - return SQLITE_ERROR; - } - } - } - - /* TODO: wipe and free passphrase after key derivation */ - if(ctx->store_pass != 1) { - sqlcipher_cipher_ctx_set_pass(ctx->read_ctx, NULL, 0); - sqlcipher_cipher_ctx_set_pass(ctx->write_ctx, NULL, 0); - } - - return SQLITE_OK; -} - -int sqlcipher_codec_key_copy(codec_ctx *ctx, int source) { - if(source == CIPHER_READ_CTX) { - return sqlcipher_cipher_ctx_copy(ctx, ctx->write_ctx, ctx->read_ctx); - } else { - return sqlcipher_cipher_ctx_copy(ctx, ctx->read_ctx, ctx->write_ctx); - } -} - -const char* sqlcipher_codec_get_cipher_provider(codec_ctx *ctx) { - return ctx->provider->get_provider_name(ctx->provider_ctx); -} - - -static int sqlcipher_check_connection(const char *filename, char *key, int key_sz, char *sql, int *user_version, char** journal_mode) { - int rc; - sqlite3 *db = NULL; - sqlite3_stmt *statement = NULL; - char *query_journal_mode = "PRAGMA journal_mode;"; - char *query_user_version = "PRAGMA user_version;"; - - rc = sqlite3_open(filename, &db); - if(rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_key(db, key, key_sz); - if(rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_exec(db, sql, NULL, NULL, NULL); - if(rc != SQLITE_OK) goto cleanup; - - /* start by querying the user version. - this will fail if the key is incorrect */ - rc = sqlite3_prepare(db, query_user_version, -1, &statement, NULL); - if(rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_step(statement); - if(rc == SQLITE_ROW) { - *user_version = sqlite3_column_int(statement, 0); - } else { - goto cleanup; - } - sqlite3_finalize(statement); - - rc = sqlite3_prepare(db, query_journal_mode, -1, &statement, NULL); - if(rc != SQLITE_OK) goto cleanup; - - rc = sqlite3_step(statement); - if(rc == SQLITE_ROW) { - *journal_mode = sqlite3_mprintf("%s", sqlite3_column_text(statement, 0)); - } else { - goto cleanup; - } - rc = SQLITE_OK; - /* cleanup will finalize open statement */ - -cleanup: - if(statement) sqlite3_finalize(statement); - if(db) sqlite3_close(db); - return rc; -} - -int sqlcipher_codec_ctx_integrity_check(codec_ctx *ctx, Parse *pParse, char *column) { - Pgno page = 1; - int rc = 0; - char *result; - unsigned char *hmac_out = NULL; - sqlite3_file *fd = sqlite3PagerFile(ctx->pBt->pBt->pPager); - i64 file_sz; - - Vdbe *v = sqlite3GetVdbe(pParse); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, column, SQLITE_STATIC); - - if(fd == NULL || fd->pMethods == 0) { - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, "database file is undefined", P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - goto cleanup; - } - - if(!(ctx->flags & CIPHER_FLAG_HMAC)) { - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, "HMAC is not enabled, unable to integrity check", P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - goto cleanup; - } - - if((rc = sqlcipher_codec_key_derive(ctx)) != SQLITE_OK) { - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, "unable to derive keys", P4_TRANSIENT); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - goto cleanup; - } - - sqlite3OsFileSize(fd, &file_sz); - hmac_out = sqlcipher_malloc(ctx->hmac_sz); - - for(page = 1; page <= file_sz / ctx->page_sz; page++) { - i64 offset = (page - 1) * ctx->page_sz; - int payload_sz = ctx->page_sz - ctx->reserve_sz + ctx->iv_sz; - int read_sz = ctx->page_sz; - - /* skip integrity check on PAGER_SJ_PGNO since it will have no valid content */ - if(sqlite3pager_is_sj_pgno(ctx->pBt->pBt->pPager, page)) continue; - - if(page==1) { - int page1_offset = ctx->plaintext_header_sz ? ctx->plaintext_header_sz : FILE_HEADER_SZ; - read_sz = read_sz - page1_offset; - payload_sz = payload_sz - page1_offset; - offset += page1_offset; - } - - sqlcipher_memset(ctx->buffer, 0, ctx->page_sz); - sqlcipher_memset(hmac_out, 0, ctx->hmac_sz); - if(sqlite3OsRead(fd, ctx->buffer, read_sz, offset) != SQLITE_OK) { - result = sqlite3_mprintf("error reading %d bytes from file page %d at offset %d", read_sz, page, offset); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } else if(sqlcipher_page_hmac(ctx, ctx->read_ctx, page, ctx->buffer, payload_sz, hmac_out) != SQLITE_OK) { - result = sqlite3_mprintf("HMAC operation failed for page %d", page); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } else if(sqlcipher_memcmp(ctx->buffer + payload_sz, hmac_out, ctx->hmac_sz) != 0) { - result = sqlite3_mprintf("HMAC verification failed for page %d", page); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } - } - - if(file_sz % ctx->page_sz != 0) { - result = sqlite3_mprintf("page %d has an invalid size of %lld bytes", page, file_sz - ((file_sz / ctx->page_sz) * ctx->page_sz)); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); - } - -cleanup: - if(hmac_out != NULL) sqlcipher_free(hmac_out, ctx->hmac_sz); - return SQLITE_OK; -} - -int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { - int i, pass_sz, keyspec_sz, nRes, user_version, rc, oflags; - Db *pDb = 0; - sqlite3 *db = ctx->pBt->db; - const char *db_filename = sqlite3_db_filename(db, "main"); - char *set_user_version = NULL, *pass = NULL, *attach_command = NULL, *migrated_db_filename = NULL, *keyspec = NULL, *temp = NULL, *journal_mode = NULL, *set_journal_mode = NULL, *pragma_compat = NULL; - Btree *pDest = NULL, *pSrc = NULL; - sqlite3_file *srcfile, *destfile; -#if defined(_WIN32) || defined(SQLITE_OS_WINRT) - LPWSTR w_db_filename = NULL, w_migrated_db_filename = NULL; - int w_db_filename_sz = 0, w_migrated_db_filename_sz = 0; -#endif - pass_sz = keyspec_sz = rc = user_version = 0; - - if(!db_filename || sqlite3Strlen30(db_filename) < 1) - goto cleanup; /* exit immediately if this is an in memory database */ - - /* pull the provided password / key material off the current codec context */ - pass_sz = ctx->read_ctx->pass_sz; - pass = sqlcipher_malloc(pass_sz+1); - memset(pass, 0, pass_sz+1); - memcpy(pass, ctx->read_ctx->pass, pass_sz); - - /* Version 4 - current, no upgrade required, so exit immediately */ - rc = sqlcipher_check_connection(db_filename, pass, pass_sz, "", &user_version, &journal_mode); - if(rc == SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "No upgrade required - exiting"); - goto cleanup; - } - - for(i = 3; i > 0; i--) { - pragma_compat = sqlite3_mprintf("PRAGMA cipher_compatibility = %d;", i); - rc = sqlcipher_check_connection(db_filename, pass, pass_sz, pragma_compat, &user_version, &journal_mode); - if(rc == SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "Version %d format found", i); - goto migrate; - } - if(pragma_compat) sqlcipher_free(pragma_compat, sqlite3Strlen30(pragma_compat)); - pragma_compat = NULL; - } - - /* if we exit the loop normally we failed to determine the version, this is an error */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "Upgrade format not determined"); - goto handle_error; - -migrate: - - temp = sqlite3_mprintf("%s-migrated", db_filename); - /* overallocate migrated_db_filename, because sqlite3OsOpen will read past the null terminator - * to determine whether the filename was URI formatted */ - migrated_db_filename = sqlcipher_malloc(sqlite3Strlen30(temp)+2); - memcpy(migrated_db_filename, temp, sqlite3Strlen30(temp)); - sqlcipher_free(temp, sqlite3Strlen30(temp)); - - attach_command = sqlite3_mprintf("ATTACH DATABASE '%s' as migrate;", migrated_db_filename, pass); - set_user_version = sqlite3_mprintf("PRAGMA migrate.user_version = %d;", user_version); - - rc = sqlite3_exec(db, pragma_compat, NULL, NULL, NULL); - if(rc != SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "set compatibility mode failed, error code %d", rc); - goto handle_error; - } - - /* force journal mode to DELETE, we will set it back later if different */ - rc = sqlite3_exec(db, "PRAGMA journal_mode = delete;", NULL, NULL, NULL); - if(rc != SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "force journal mode DELETE failed, error code %d", rc); - goto handle_error; - } - - rc = sqlite3_exec(db, attach_command, NULL, NULL, NULL); - if(rc != SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "attach failed, error code %d", rc); - goto handle_error; - } - - rc = sqlite3_key_v2(db, "migrate", pass, pass_sz); - if(rc != SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "keying attached database failed, error code %d", rc); - goto handle_error; - } - - rc = sqlite3_exec(db, "SELECT sqlcipher_export('migrate');", NULL, NULL, NULL); - if(rc != SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_export failed, error code %d", rc); - goto handle_error; - } - -#ifdef SQLCIPHER_TEST - if((sqlcipher_get_test_flags() & TEST_FAIL_MIGRATE) > 0) { - rc = SQLITE_ERROR; - sqlcipher_log(SQLCIPHER_LOG_ERROR, "simulated migrate failure, error code %d", rc); - goto handle_error; - } -#endif - - rc = sqlite3_exec(db, set_user_version, NULL, NULL, NULL); - if(rc != SQLITE_OK){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "set user version failed, error code %d", rc); - goto handle_error; - } - - if( !db->autoCommit ){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "cannot migrate from within a transaction"); - goto handle_error; - } - if( db->nVdbeActive>1 ){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "cannot migrate - SQL statements in progress"); - goto handle_error; - } - - pDest = db->aDb[0].pBt; - pDb = &(db->aDb[db->nDb-1]); - pSrc = pDb->pBt; - - nRes = sqlite3BtreeGetRequestedReserve(pSrc); - /* unset the BTS_PAGESIZE_FIXED flag to avoid SQLITE_READONLY */ - pDest->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; - rc = sqlite3BtreeSetPageSize(pDest, default_page_size, nRes, 0); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "set btree page size to %d res %d rc %d", default_page_size, nRes, rc); - if( rc!=SQLITE_OK ) goto handle_error; - - sqlcipherCodecGetKey(db, db->nDb - 1, (void**)&keyspec, &keyspec_sz); - SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_KEY_USED); - sqlcipherCodecAttach(db, 0, keyspec, keyspec_sz); - - srcfile = sqlite3PagerFile(pSrc->pBt->pPager); - destfile = sqlite3PagerFile(pDest->pBt->pPager); - - sqlite3OsClose(srcfile); - sqlite3OsClose(destfile); - -#if defined(_WIN32) || defined(SQLITE_OS_WINRT) - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "performing windows MoveFileExA"); - - w_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) db_filename, -1, NULL, 0); - w_db_filename = sqlcipher_malloc(w_db_filename_sz * sizeof(wchar_t)); - w_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) db_filename, -1, (const LPWSTR) w_db_filename, w_db_filename_sz); - - w_migrated_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) migrated_db_filename, -1, NULL, 0); - w_migrated_db_filename = sqlcipher_malloc(w_migrated_db_filename_sz * sizeof(wchar_t)); - w_migrated_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) migrated_db_filename, -1, (const LPWSTR) w_migrated_db_filename, w_migrated_db_filename_sz); - - if(!MoveFileExW(w_migrated_db_filename, w_db_filename, MOVEFILE_REPLACE_EXISTING)) { - rc = SQLITE_ERROR; - sqlcipher_log(SQLCIPHER_LOG_ERROR, "error occurred while renaming %d", rc); - goto handle_error; - } -#else - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "performing POSIX rename"); - if ((rc = rename(migrated_db_filename, db_filename)) != 0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "error occurred while renaming %d", rc); - goto handle_error; - } -#endif - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "renamed migration database %s to main database %s: %d", migrated_db_filename, db_filename, rc); - - rc = sqlite3OsOpen(db->pVfs, migrated_db_filename, srcfile, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB, &oflags); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "reopened migration database: %d", rc); - if( rc!=SQLITE_OK ) goto handle_error; - - rc = sqlite3OsOpen(db->pVfs, db_filename, destfile, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB, &oflags); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "reopened main database: %d", rc); - if( rc!=SQLITE_OK ) goto handle_error; - - sqlite3pager_reset(pDest->pBt->pPager); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "reset pager"); - - rc = sqlite3_exec(db, "DETACH DATABASE migrate;", NULL, NULL, NULL); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "DETACH DATABASE called %d", rc); - if(rc != SQLITE_OK) goto cleanup; - - sqlite3ResetAllSchemasOfConnection(db); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "reset all schemas"); - - set_journal_mode = sqlite3_mprintf("PRAGMA journal_mode = %s;", journal_mode); - rc = sqlite3_exec(db, set_journal_mode, NULL, NULL, NULL); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "%s: %d", set_journal_mode, rc); - if( rc!=SQLITE_OK ) goto handle_error; - - goto cleanup; - -handle_error: - sqlcipher_log(SQLCIPHER_LOG_ERROR, "An error occurred attempting to migrate the database - last error %d", rc); - -cleanup: - if(migrated_db_filename) { - int del_rc = sqlite3OsDelete(db->pVfs, migrated_db_filename, 0); - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "deleted migration database: %d", del_rc); - } - - if(pass) sqlcipher_free(pass, pass_sz); - if(attach_command) sqlcipher_free(attach_command, sqlite3Strlen30(attach_command)); - if(migrated_db_filename) sqlcipher_free(migrated_db_filename, sqlite3Strlen30(migrated_db_filename)); - if(set_user_version) sqlcipher_free(set_user_version, sqlite3Strlen30(set_user_version)); - if(set_journal_mode) sqlcipher_free(set_journal_mode, sqlite3Strlen30(set_journal_mode)); - if(journal_mode) sqlcipher_free(journal_mode, sqlite3Strlen30(journal_mode)); - if(pragma_compat) sqlcipher_free(pragma_compat, sqlite3Strlen30(pragma_compat)); -#if defined(_WIN32) || defined(SQLITE_OS_WINRT) - if(w_db_filename) sqlcipher_free(w_db_filename, w_db_filename_sz); - if(w_migrated_db_filename) sqlcipher_free(w_migrated_db_filename, w_migrated_db_filename_sz); -#endif - return rc; -} - -int sqlcipher_codec_add_random(codec_ctx *ctx, const char *zRight, int random_sz){ - const char *suffix = &zRight[random_sz-1]; - int n = random_sz - 3; /* adjust for leading x' and tailing ' */ - if (n > 0 && - sqlite3StrNICmp((const char *)zRight ,"x'", 2) == 0 && - sqlite3StrNICmp(suffix, "'", 1) == 0 && - n % 2 == 0) { - int rc = 0; - int buffer_sz = n / 2; - unsigned char *random; - const unsigned char *z = (const unsigned char *)zRight + 2; /* adjust lead offset of x' */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, "sqlcipher_codec_add_random: using raw random blob from hex"); - random = sqlcipher_malloc(buffer_sz); - memset(random, 0, buffer_sz); - cipher_hex2bin(z, n, random); - rc = ctx->provider->add_random(ctx->provider_ctx, random, buffer_sz); - sqlcipher_free(random, buffer_sz); - return rc; - } - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_codec_add_random: attemt to add random with invalid format"); - return SQLITE_ERROR; -} - -#if !defined(SQLITE_OMIT_TRACE) -static int sqlcipher_profile_callback(unsigned int trace, void *file, void *stmt, void *run_time){ - FILE *f = (FILE*) file; - char *fmt = "Elapsed time:%.3f ms - %s\n"; - double elapsed = (*((sqlite3_uint64*)run_time))/1000000.0; -#ifdef __ANDROID__ - if(f == NULL) { - __android_log_print(ANDROID_LOG_DEBUG, "sqlcipher", fmt, elapsed, sqlite3_sql((sqlite3_stmt*)stmt)); - } -#endif - if(f) fprintf(f, fmt, elapsed, sqlite3_sql((sqlite3_stmt*)stmt)); - return SQLITE_OK; -} -#endif - -int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){ -#if defined(SQLITE_OMIT_TRACE) - return SQLITE_ERROR; -#else - FILE *f = NULL; - if(sqlite3_stricmp(destination, "off") == 0){ - sqlite3_trace_v2(db, 0, NULL, NULL); /* disable tracing */ - } else { - if(sqlite3_stricmp(destination, "stdout") == 0){ - f = stdout; - }else if(sqlite3_stricmp(destination, "stderr") == 0){ - f = stderr; - }else if(sqlite3_stricmp(destination, "logcat") == 0){ - f = NULL; /* file pointer will be NULL indicating logcat on android */ - }else{ -#if !defined(SQLCIPHER_PROFILE_USE_FOPEN) && (defined(_WIN32) && (__STDC_VERSION__ > 199901L) || defined(SQLITE_OS_WINRT)) - if(fopen_s(&f, destination, "a") != 0) return SQLITE_ERROR; -#else - if((f = fopen(destination, "a")) == 0) return SQLITE_ERROR; -#endif - } - sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, sqlcipher_profile_callback, f); - } - return SQLITE_OK; -#endif -} - -int sqlcipher_codec_fips_status(codec_ctx *ctx) { - return ctx->provider->fips_status(ctx->provider_ctx); -} - -const char* sqlcipher_codec_get_provider_version(codec_ctx *ctx) { - return ctx->provider->get_provider_version(ctx->provider_ctx); -} - -#ifndef SQLCIPHER_OMIT_LOG -/* constants from https://github.com/Alexpux/mingw-w64/blob/master/mingw-w64-crt/misc/gettimeofday.c */ -#define FILETIME_1970 116444736000000000ull /* seconds between 1/1/1601 and 1/1/1970 */ -#define HECTONANOSEC_PER_SEC 10000000ull -void sqlcipher_log(unsigned int level, const char *message, ...) { - va_list params; - va_start(params, message); - -#ifdef CODEC_DEBUG -#ifdef __ANDROID__ - __android_log_vprint(ANDROID_LOG_DEBUG, "sqlcipher", message, params); -#else - vfprintf(stderr, message, params); - fprintf(stderr, "\n"); -#endif -#endif - - if(level > sqlcipher_log_level || (sqlcipher_log_logcat == 0 && sqlcipher_log_file == NULL)) { - /* no log target or tag not in included filters */ - goto end; - } - if(sqlcipher_log_file != NULL){ - char buffer[24]; - struct tm tt; - int ms; - time_t sec; -#ifdef _WIN32 - SYSTEMTIME st; - FILETIME ft; - GetSystemTime(&st); - SystemTimeToFileTime(&st, &ft); - sec = (time_t) ((*((sqlite_int64*)&ft) - FILETIME_1970) / HECTONANOSEC_PER_SEC); - ms = st.wMilliseconds; - localtime_s(&tt, &sec); -#else - struct timeval tv; - gettimeofday(&tv, NULL); - sec = tv.tv_sec; - ms = tv.tv_usec/1000.0; - localtime_r(&sec, &tt); -#endif - if(strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tt)) { - fprintf((FILE*)sqlcipher_log_file, "%s.%03d: ", buffer, ms); - vfprintf((FILE*)sqlcipher_log_file, message, params); - fprintf((FILE*)sqlcipher_log_file, "\n"); - } - } -#ifdef __ANDROID__ - if(sqlcipher_log_logcat) { - __android_log_vprint(ANDROID_LOG_DEBUG, "sqlcipher", message, params); - } -#endif -end: - va_end(params); -} -#endif - -void sqlcipher_set_log_level(unsigned int level) { - sqlcipher_log_level = level; -} - -int sqlcipher_set_log(const char *destination){ -#ifdef SQLCIPHER_OMIT_LOG - return SQLITE_ERROR; -#else - /* close open trace file if it is not stdout or stderr, then - reset trace settings */ - if(sqlcipher_log_file != NULL && sqlcipher_log_file != stdout && sqlcipher_log_file != stderr) { - fclose((FILE*)sqlcipher_log_file); - } - sqlcipher_log_file = NULL; - sqlcipher_log_logcat = 0; - - if(sqlite3_stricmp(destination, "logcat") == 0){ - sqlcipher_log_logcat = 1; - } else if(sqlite3_stricmp(destination, "stdout") == 0){ - sqlcipher_log_file = stdout; - }else if(sqlite3_stricmp(destination, "stderr") == 0){ - sqlcipher_log_file = stderr; - }else if(sqlite3_stricmp(destination, "off") != 0){ -#if !defined(SQLCIPHER_PROFILE_USE_FOPEN) && (defined(_WIN32) && (__STDC_VERSION__ > 199901L) || defined(SQLITE_OS_WINRT)) - if(fopen_s(&sqlcipher_log_file, destination, "a") != 0) return SQLITE_ERROR; -#else - if((sqlcipher_log_file = fopen(destination, "a")) == 0) return SQLITE_ERROR; -#endif - } - sqlcipher_log(SQLCIPHER_LOG_INFO, "sqlcipher_set_log: set log to %s", destination); - return SQLITE_OK; -#endif -} - -#endif -/* END SQLCIPHER */ diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c deleted file mode 100644 index 03a0689798..0000000000 --- a/src/crypto_libtomcrypt.c +++ /dev/null @@ -1,296 +0,0 @@ -/* -** SQLCipher -** http://sqlcipher.net -** -** Copyright (c) 2008 - 2013, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC -#ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT -#include "sqliteInt.h" -#include "sqlcipher.h" -#include - -#define FORTUNA_MAX_SZ 32 -static prng_state prng; -static volatile unsigned int ltc_init = 0; -static volatile unsigned int ltc_ref_count = 0; - -#define LTC_CIPHER "rijndael" - -static int sqlcipher_ltc_add_random(void *ctx, void *buffer, int length) { - int rc = 0; - int data_to_read = length; - int block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; - const unsigned char * data = (const unsigned char *)buffer; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_add_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_add_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); - - while(data_to_read > 0){ - rc = fortuna_add_entropy(data, block_sz, &prng); - rc = rc != CRYPT_OK ? SQLITE_ERROR : SQLITE_OK; - if(rc != SQLITE_OK){ - break; - } - data_to_read -= block_sz; - data += block_sz; - block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; - } - fortuna_ready(&prng); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_add_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_add_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); - - return rc; -} - -static int sqlcipher_ltc_activate(void *ctx) { - unsigned char random_buffer[FORTUNA_MAX_SZ]; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); - if(ltc_init == 0) { - if(register_prng(&fortuna_desc) < 0) return SQLITE_ERROR; - if(register_cipher(&rijndael_desc) < 0) return SQLITE_ERROR; - if(register_hash(&sha512_desc) < 0) return SQLITE_ERROR; - if(register_hash(&sha256_desc) < 0) return SQLITE_ERROR; - if(register_hash(&sha1_desc) < 0) return SQLITE_ERROR; - if(fortuna_start(&prng) != CRYPT_OK) { - return SQLITE_ERROR; - } - - ltc_init = 1; - } - ltc_ref_count++; - -#ifndef SQLCIPHER_TEST - sqlite3_randomness(FORTUNA_MAX_SZ, random_buffer); -#endif - - if(sqlcipher_ltc_add_random(ctx, random_buffer, FORTUNA_MAX_SZ) != SQLITE_OK) { - return SQLITE_ERROR; - } - sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - return SQLITE_OK; -} - -static int sqlcipher_ltc_deactivate(void *ctx) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_deactivate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_deactivate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - ltc_ref_count--; - if(ltc_ref_count == 0){ - fortuna_done(&prng); - sqlcipher_memset((void *)&prng, 0, sizeof(prng)); - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_deactivate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_deactivate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - return SQLITE_OK; -} - -static const char* sqlcipher_ltc_get_provider_name(void *ctx) { - return "libtomcrypt"; -} - -static const char* sqlcipher_ltc_get_provider_version(void *ctx) { - return SCRYPT; -} - -static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); - - fortuna_read(buffer, length, &prng); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_ltc_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); - - return SQLITE_OK; -} - -static int sqlcipher_ltc_hmac(void *ctx, int algorithm, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { - int rc, hash_idx; - hmac_state hmac; - unsigned long outlen; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - hash_idx = find_hash("sha1"); - break; - case SQLCIPHER_HMAC_SHA256: - hash_idx = find_hash("sha256"); - break; - case SQLCIPHER_HMAC_SHA512: - hash_idx = find_hash("sha512"); - break; - default: - return SQLITE_ERROR; - } - - if(hash_idx < 0) return SQLITE_ERROR; - outlen = hash_descriptor[hash_idx].hashsize; - - if(in == NULL) return SQLITE_ERROR; - if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; - if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; - if(in2 != NULL && (rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; - if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; - return SQLITE_OK; -} - -static int sqlcipher_ltc_kdf(void *ctx, int algorithm, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { - int rc, hash_idx; - unsigned long outlen = key_sz; - - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - hash_idx = find_hash("sha1"); - break; - case SQLCIPHER_HMAC_SHA256: - hash_idx = find_hash("sha256"); - break; - case SQLCIPHER_HMAC_SHA512: - hash_idx = find_hash("sha512"); - break; - default: - return SQLITE_ERROR; - } - if(hash_idx < 0) return SQLITE_ERROR; - - if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, - workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { - return SQLITE_ERROR; - } - return SQLITE_OK; -} - -static const char* sqlcipher_ltc_get_cipher(void *ctx) { - return "aes-256-cbc"; -} - -static int sqlcipher_ltc_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { - int rc, cipher_idx; - symmetric_CBC cbc; - - if((cipher_idx = find_cipher(LTC_CIPHER)) == -1) return SQLITE_ERROR; - if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; - rc = mode == 1 ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); - if(rc != CRYPT_OK) return SQLITE_ERROR; - cbc_done(&cbc); - return SQLITE_OK; -} - -static int sqlcipher_ltc_get_key_sz(void *ctx) { - int cipher_idx = find_cipher(LTC_CIPHER); - return cipher_descriptor[cipher_idx].max_key_length; -} - -static int sqlcipher_ltc_get_iv_sz(void *ctx) { - int cipher_idx = find_cipher(LTC_CIPHER); - return cipher_descriptor[cipher_idx].block_length; -} - -static int sqlcipher_ltc_get_block_sz(void *ctx) { - int cipher_idx = find_cipher(LTC_CIPHER); - return cipher_descriptor[cipher_idx].block_length; -} - -static int sqlcipher_ltc_get_hmac_sz(void *ctx, int algorithm) { - int hash_idx; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - hash_idx = find_hash("sha1"); - break; - case SQLCIPHER_HMAC_SHA256: - hash_idx = find_hash("sha256"); - break; - case SQLCIPHER_HMAC_SHA512: - hash_idx = find_hash("sha512"); - break; - default: - return 0; - } - - if(hash_idx < 0) return 0; - - return hash_descriptor[hash_idx].hashsize; -} - -static int sqlcipher_ltc_ctx_init(void **ctx) { - sqlcipher_ltc_activate(NULL); - return SQLITE_OK; -} - -static int sqlcipher_ltc_ctx_free(void **ctx) { - sqlcipher_ltc_deactivate(&ctx); - return SQLITE_OK; -} - -static int sqlcipher_ltc_fips_status(void *ctx) { - return 0; -} - -int sqlcipher_ltc_setup(sqlcipher_provider *p) { - p->activate = sqlcipher_ltc_activate; - p->deactivate = sqlcipher_ltc_deactivate; - p->get_provider_name = sqlcipher_ltc_get_provider_name; - p->random = sqlcipher_ltc_random; - p->hmac = sqlcipher_ltc_hmac; - p->kdf = sqlcipher_ltc_kdf; - p->cipher = sqlcipher_ltc_cipher; - p->get_cipher = sqlcipher_ltc_get_cipher; - p->get_key_sz = sqlcipher_ltc_get_key_sz; - p->get_iv_sz = sqlcipher_ltc_get_iv_sz; - p->get_block_sz = sqlcipher_ltc_get_block_sz; - p->get_hmac_sz = sqlcipher_ltc_get_hmac_sz; - p->ctx_init = sqlcipher_ltc_ctx_init; - p->ctx_free = sqlcipher_ltc_ctx_free; - p->add_random = sqlcipher_ltc_add_random; - p->fips_status = sqlcipher_ltc_fips_status; - p->get_provider_version = sqlcipher_ltc_get_provider_version; - return SQLITE_OK; -} - -#endif -#endif -/* END SQLCIPHER */ diff --git a/src/crypto_nss.c b/src/crypto_nss.c deleted file mode 100644 index 5de302ca7b..0000000000 --- a/src/crypto_nss.c +++ /dev/null @@ -1,306 +0,0 @@ -/* -** SQLCipher -** http://sqlcipher.net -** -** Copyright (c) 2008 - 2013, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC -#ifdef SQLCIPHER_CRYPTO_NSS -#include "crypto.h" -#include "sqlcipher.h" -#include -#include -#include - -static NSSInitContext* nss_init_context = NULL; -static unsigned int nss_init_count = 0; - -int sqlcipher_nss_setup(sqlcipher_provider *p); - -static int sqlcipher_nss_activate(void *ctx) { - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - if (nss_init_context == NULL) { - nss_init_context = NSS_InitContext("", "", "", "", NULL, - NSS_INIT_READONLY | NSS_INIT_NOCERTDB | NSS_INIT_NOMODDB | - NSS_INIT_FORCEOPEN | NSS_INIT_OPTIMIZESPACE | NSS_INIT_NOROOTINIT); - } - nss_init_count++; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - return SQLITE_OK; -} - -static int sqlcipher_nss_deactivate(void *ctx) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - nss_init_count--; - if (nss_init_count == 0 && nss_init_context != NULL) { - NSS_ShutdownContext(nss_init_context); - nss_init_context = NULL; - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_nss_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - return SQLITE_OK; -} - -static int sqlcipher_nss_add_random(void *ctx, void *buffer, int length) { - return SQLITE_OK; -} - -/* generate a defined number of random bytes */ -static int sqlcipher_nss_random (void *ctx, void *buffer, int length) { - // PK11_GenerateRandom should be thread-safe. - return (PK11_GenerateRandom((unsigned char *)buffer, length) == SECSuccess) ? SQLITE_OK : SQLITE_ERROR; -} - -static const char* sqlcipher_nss_get_provider_name(void *ctx) { - return "nss"; -} - -static const char* sqlcipher_nss_get_provider_version(void *ctx) { - return NSS_GetVersion(); -} - -static const char* sqlcipher_nss_get_cipher(void *ctx) { - return "aes-256-cbc"; -} - -static int sqlcipher_nss_get_key_sz(void *ctx) { - return AES_256_KEY_LENGTH; -} - -static int sqlcipher_nss_get_iv_sz(void *ctx) { - return AES_BLOCK_SIZE; -} - -static int sqlcipher_nss_get_block_sz(void *ctx) { - return AES_BLOCK_SIZE; -} - -static int sqlcipher_nss_get_hmac_sz(void *ctx, int algorithm) { - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - return SHA1_LENGTH; - break; - case SQLCIPHER_HMAC_SHA256: - return SHA256_LENGTH; - break; - case SQLCIPHER_HMAC_SHA512: - return SHA512_LENGTH; - break; - default: - return 0; - } -} - -static int sqlcipher_nss_hmac(void *ctx, int algorithm, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { - int rc = SQLITE_OK; - unsigned int length; - unsigned int outLen; - PK11Context* context = NULL; - PK11SlotInfo * slot = NULL; - PK11SymKey* symKey = NULL; - if(in == NULL) goto error; - CK_MECHANISM_TYPE mech; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - mech = CKM_SHA_1_HMAC; - break; - case SQLCIPHER_HMAC_SHA256: - mech = CKM_SHA256_HMAC; - break; - case SQLCIPHER_HMAC_SHA512: - mech = CKM_SHA512_HMAC; - break; - default: - goto error; - } - length = sqlcipher_nss_get_hmac_sz(ctx, algorithm); - slot = PK11_GetInternalSlot(); - if (slot == NULL) goto error; - SECItem keyItem; - keyItem.data = hmac_key; - keyItem.len = key_sz; - symKey = PK11_ImportSymKey(slot, mech, PK11_OriginUnwrap, - CKA_SIGN, &keyItem, NULL); - if (symKey == NULL) goto error; - SECItem noParams; - noParams.data = 0; - noParams.len = 0; - context = PK11_CreateContextBySymKey(mech, CKA_SIGN, symKey, &noParams); - if (context == NULL) goto error; - if (PK11_DigestBegin(context) != SECSuccess) goto error; - if (PK11_DigestOp(context, in, in_sz) != SECSuccess) goto error; - if (in2 != NULL) { - if (PK11_DigestOp(context, in2, in2_sz) != SECSuccess) goto error; - } - if (PK11_DigestFinal(context, out, &outLen, length) != SECSuccess) goto error; - - goto cleanup; - error: - rc = SQLITE_ERROR; - cleanup: - if (context) PK11_DestroyContext(context, PR_TRUE); - if (symKey) PK11_FreeSymKey(symKey); - if (slot) PK11_FreeSlot(slot); - return rc; -} - -static int sqlcipher_nss_kdf(void *ctx, int algorithm, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { - int rc = SQLITE_OK; - PK11SlotInfo * slot = NULL; - SECAlgorithmID * algid = NULL; - PK11SymKey* symKey = NULL; - SECOidTag oidtag; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - oidtag = SEC_OID_HMAC_SHA1; - break; - case SQLCIPHER_HMAC_SHA256: - oidtag = SEC_OID_HMAC_SHA256; - break; - case SQLCIPHER_HMAC_SHA512: - oidtag = SEC_OID_HMAC_SHA512; - break; - default: - goto error; - } - SECItem secSalt; - secSalt.data = salt; - secSalt.len = salt_sz; - // Always pass SEC_OID_HMAC_SHA1 (i.e. PBMAC1) as this parameter - // is unused for key generation. It is currently only used - // for PBKDF2 authentication or key (un)wrapping when specifying an - // encryption algorithm (PBES2). - algid = PK11_CreatePBEV2AlgorithmID(SEC_OID_PKCS5_PBKDF2, SEC_OID_HMAC_SHA1, - oidtag, key_sz, workfactor, &secSalt); - if (algid == NULL) goto error; - slot = PK11_GetInternalSlot(); - if (slot == NULL) goto error; - SECItem pwItem; - pwItem.data = (unsigned char *) pass; // PK11_PBEKeyGen doesn't modify the key. - pwItem.len = pass_sz; - symKey = PK11_PBEKeyGen(slot, algid, &pwItem, PR_FALSE, NULL); - if (symKey == NULL) goto error; - if (PK11_ExtractKeyValue(symKey) != SECSuccess) goto error; - // No need to free keyData as it is a buffer managed by symKey. - SECItem* keyData = PK11_GetKeyData(symKey); - if (keyData == NULL) goto error; - memcpy(key, keyData->data, key_sz); - - goto cleanup; - error: - rc = SQLITE_ERROR; - cleanup: - if (slot) PK11_FreeSlot(slot); - if (algid) SECOID_DestroyAlgorithmID(algid, PR_TRUE); - if (symKey) PK11_FreeSymKey(symKey); - return rc; -} - -static int sqlcipher_nss_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { - int rc = SQLITE_OK; - PK11SlotInfo * slot = NULL; - PK11SymKey* symKey = NULL; - unsigned int outLen; - SECItem params; - params.data = iv; - params.len = sqlcipher_nss_get_iv_sz(ctx); - slot = PK11_GetInternalSlot(); - if (slot == NULL) goto error; - SECItem keyItem; - keyItem.data = key; - keyItem.len = key_sz; - symKey = PK11_ImportSymKey(slot, CKM_AES_CBC, PK11_OriginUnwrap, - CKA_ENCRYPT, &keyItem, NULL); - if (symKey == NULL) goto error; - SECStatus rv; - if (mode == CIPHER_ENCRYPT) { - rv = PK11_Encrypt(symKey, CKM_AES_CBC, ¶ms, out, &outLen, - in_sz + 16, in, in_sz); - } else { - rv = PK11_Decrypt(symKey, CKM_AES_CBC, ¶ms, out, &outLen, - in_sz + 16, in, in_sz); - } - if (rv != SECSuccess) goto error; - - goto cleanup; - error: - rc = SQLITE_ERROR; - cleanup: - if (slot) PK11_FreeSlot(slot); - if (symKey) PK11_FreeSymKey(symKey); - return rc; -} - -static int sqlcipher_nss_ctx_init(void **ctx) { - sqlcipher_nss_activate(NULL); - return SQLITE_OK; -} - -static int sqlcipher_nss_ctx_free(void **ctx) { - sqlcipher_nss_deactivate(NULL); - return SQLITE_OK; -} - -static int sqlcipher_nss_fips_status(void *ctx) { - return 0; -} - -int sqlcipher_nss_setup(sqlcipher_provider *p) { - p->activate = sqlcipher_nss_activate; - p->deactivate = sqlcipher_nss_deactivate; - p->random = sqlcipher_nss_random; - p->get_provider_name = sqlcipher_nss_get_provider_name; - p->hmac = sqlcipher_nss_hmac; - p->kdf = sqlcipher_nss_kdf; - p->cipher = sqlcipher_nss_cipher; - p->get_cipher = sqlcipher_nss_get_cipher; - p->get_key_sz = sqlcipher_nss_get_key_sz; - p->get_iv_sz = sqlcipher_nss_get_iv_sz; - p->get_block_sz = sqlcipher_nss_get_block_sz; - p->get_hmac_sz = sqlcipher_nss_get_hmac_sz; - p->ctx_init = sqlcipher_nss_ctx_init; - p->ctx_free = sqlcipher_nss_ctx_free; - p->add_random = sqlcipher_nss_add_random; - p->fips_status = sqlcipher_nss_fips_status; - p->get_provider_version = sqlcipher_nss_get_provider_version; - return SQLITE_OK; -} - -#endif -#endif -/* END SQLCIPHER */ diff --git a/src/crypto_openssl.c b/src/crypto_openssl.c index 8ab1ad3abd..828588b2d1 100644 --- a/src/crypto_openssl.c +++ b/src/crypto_openssl.c @@ -32,58 +32,34 @@ #ifdef SQLITE_HAS_CODEC #ifdef SQLCIPHER_CRYPTO_OPENSSL #include "sqliteInt.h" -#include "crypto.h" #include "sqlcipher.h" -#include -#include -#include -#include -#include -#include +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ +#include /* amalgamator: dontcache */ static unsigned int openssl_init_count = 0; static void sqlcipher_openssl_log_errors() { unsigned long err = 0; while((err = ERR_get_error()) != 0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_log_errors: ERR_get_error() returned %lx: %s", err, ERR_error_string(err, NULL)); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_log_errors: ERR_get_error() returned %lx: %s", err, ERR_error_string(err, NULL)); } } -#if (defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L) || (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x20700000L) -static HMAC_CTX *HMAC_CTX_new(void) -{ - HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); - if (ctx != NULL) { - HMAC_CTX_init(ctx); - } - return ctx; -} - -/* Per 1.1.0 (https://wiki.openssl.org/index.php/1.1_API_Changes) - HMAC_CTX_free should call HMAC_CTX_cleanup, then EVP_MD_CTX_Cleanup. - HMAC_CTX_cleanup internally calls EVP_MD_CTX_cleanup so these - calls are not needed. */ -static void HMAC_CTX_free(HMAC_CTX *ctx) -{ - if (ctx != NULL) { - HMAC_CTX_cleanup(ctx); - OPENSSL_free(ctx); - } -} -#endif - -static int sqlcipher_openssl_add_random(void *ctx, void *buffer, int length) { +static int sqlcipher_openssl_add_random(void *ctx, const void *buffer, int length) { #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_add_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_add_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_add_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_add_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); #endif RAND_add(buffer, length, 0); #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_add_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_add_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_add_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_add_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); #endif return SQLITE_OK; } @@ -100,29 +76,18 @@ static int sqlcipher_openssl_activate(void *ctx) { /* initialize openssl and increment the internal init counter but only if it hasn't been initalized outside of SQLCipher by this program e.g. on startup */ - int rc = 0; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); #if (defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L) ERR_load_crypto_strings(); #endif -#ifdef SQLCIPHER_FIPS - if(!FIPS_mode()){ - if(!(rc = FIPS_mode_set(1))){ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_activate: FIPS_mode_set() returned %d", rc); - sqlcipher_openssl_log_errors(); - } - } -#endif - openssl_init_count++; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); return SQLITE_OK; } @@ -130,15 +95,15 @@ static int sqlcipher_openssl_activate(void *ctx) { freeing the EVP structures on the final deactivation to ensure that OpenSSL memory is cleaned up */ static int sqlcipher_openssl_deactivate(void *ctx) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_deactivate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_deactivate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_deactivate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_deactivate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); openssl_init_count--; - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_deactivate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_deactivate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_deactivate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_deactivate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); return SQLITE_OK; } @@ -164,97 +129,33 @@ static int sqlcipher_openssl_random (void *ctx, void *buffer, int length) { but a more proper solution is that applications setup platform-appropriate thread saftey in openssl externally */ #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); #endif rc = RAND_bytes((unsigned char *)buffer, length); #ifndef SQLCIPHER_OPENSSL_NO_MUTEX_RAND - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, "sqlcipher_openssl_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_openssl_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); #endif if(!rc) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_random: RAND_bytes() returned %d", rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_random: RAND_bytes() returned %d", rc); sqlcipher_openssl_log_errors(); return SQLITE_ERROR; } return SQLITE_OK; } -static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out) { +static int sqlcipher_openssl_hmac( + void *ctx, int algorithm, + const unsigned char *hmac_key, int key_sz, + const unsigned char *in, int in_sz, + const unsigned char *in2, int in2_sz, + unsigned char *out +) { int rc = 0; -#if (defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x30000000L) - unsigned int outlen; - HMAC_CTX* hctx = NULL; - - if(in == NULL) goto error; - - hctx = HMAC_CTX_new(); - if(hctx == NULL) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_CTX_new() failed"); - sqlcipher_openssl_log_errors(); - goto error; - } - - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - if(!(rc = HMAC_Init_ex(hctx, hmac_key, key_sz, EVP_sha1(), NULL))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_Init_ex() with key size %d and EVP_sha1() returned %d", key_sz, rc); - sqlcipher_openssl_log_errors(); - goto error; - } - break; - case SQLCIPHER_HMAC_SHA256: - if(!(rc = HMAC_Init_ex(hctx, hmac_key, key_sz, EVP_sha256(), NULL))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_Init_ex() with key size %d and EVP_sha256() returned %d", key_sz, rc); - sqlcipher_openssl_log_errors(); - goto error; - } - break; - case SQLCIPHER_HMAC_SHA512: - if(!(rc = HMAC_Init_ex(hctx, hmac_key, key_sz, EVP_sha512(), NULL))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_Init_ex() with key size %d and EVP_sha512() returned %d", key_sz, rc); - sqlcipher_openssl_log_errors(); - goto error; - } - break; - default: - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: invalid algorithm %d", algorithm); - goto error; - } - - if(!(rc = HMAC_Update(hctx, in, in_sz))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_Update() on 1st input buffer of %d bytes using algorithm %d returned %d", in_sz, algorithm, rc); - sqlcipher_openssl_log_errors(); - goto error; - } - - if(in2 != NULL) { - if(!(rc = HMAC_Update(hctx, in2, in2_sz))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_Update() on 2nd input buffer of %d bytes using algorithm %d returned %d", in2_sz, algorithm, rc); - sqlcipher_openssl_log_errors(); - goto error; - } - } - - if(!(rc = HMAC_Final(hctx, out, &outlen))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: HMAC_Final() using algorithm %d returned %d", algorithm, rc); - sqlcipher_openssl_log_errors(); - goto error; - } - - rc = SQLITE_OK; - goto cleanup; - -error: - rc = SQLITE_ERROR; - -cleanup: - if(hctx) HMAC_CTX_free(hctx); - -#else size_t outlen; EVP_MAC *mac = NULL; EVP_MAC_CTX *hctx = NULL; @@ -266,14 +167,14 @@ static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_ mac = EVP_MAC_fetch(NULL, "HMAC", NULL); if(mac == NULL) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_fetch for HMAC failed"); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_fetch for HMAC failed"); sqlcipher_openssl_log_errors(); goto error; } hctx = EVP_MAC_CTX_new(mac); if(hctx == NULL) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_CTX_new() failed"); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_CTX_new() failed"); sqlcipher_openssl_log_errors(); goto error; } @@ -281,52 +182,52 @@ static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_ switch(algorithm) { case SQLCIPHER_HMAC_SHA1: if(!(rc = EVP_MAC_init(hctx, hmac_key, key_sz, sha1))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_init() with key size %d and sha1 returned %d", key_sz, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_init() with key size %d and sha1 returned %d", key_sz, rc); sqlcipher_openssl_log_errors(); goto error; } break; case SQLCIPHER_HMAC_SHA256: if(!(rc = EVP_MAC_init(hctx, hmac_key, key_sz, sha256))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_init() with key size %d and sha256 returned %d", key_sz, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_init() with key size %d and sha256 returned %d", key_sz, rc); sqlcipher_openssl_log_errors(); goto error; } break; case SQLCIPHER_HMAC_SHA512: if(!(rc = EVP_MAC_init(hctx, hmac_key, key_sz, sha512))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_init() with key size %d and sha512 returned %d", key_sz, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_init() with key size %d and sha512 returned %d", key_sz, rc); sqlcipher_openssl_log_errors(); goto error; } break; default: - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: invalid algorithm %d", algorithm); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: invalid algorithm %d", algorithm); goto error; } if(!(rc = EVP_MAC_update(hctx, in, in_sz))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_update() on 1st input buffer of %d bytes using algorithm %d returned %d", in_sz, algorithm, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_update() on 1st input buffer of %d bytes using algorithm %d returned %d", in_sz, algorithm, rc); sqlcipher_openssl_log_errors(); goto error; } if(in2 != NULL) { if(!(rc = EVP_MAC_update(hctx, in2, in2_sz))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: EVP_MAC_update() on 2nd input buffer of %d bytes using algorithm %d returned %d", in_sz, algorithm, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: EVP_MAC_update() on 2nd input buffer of %d bytes using algorithm %d returned %d", in_sz, algorithm, rc); sqlcipher_openssl_log_errors(); goto error; } } if(!(rc = EVP_MAC_final(hctx, NULL, &outlen, 0))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: 1st EVP_MAC_final() for output length calculation using algorithm %d returned %d", algorithm, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: 1st EVP_MAC_final() for output length calculation using algorithm %d returned %d", algorithm, rc); sqlcipher_openssl_log_errors(); goto error; } if(!(rc = EVP_MAC_final(hctx, out, &outlen, outlen))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_hmac: 2nd EVP_MAC_final() using algorithm %d returned %d", algorithm, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_hmac: 2nd EVP_MAC_final() using algorithm %d returned %d", algorithm, rc); sqlcipher_openssl_log_errors(); goto error; } @@ -341,32 +242,36 @@ static int sqlcipher_openssl_hmac(void *ctx, int algorithm, unsigned char *hmac_ if(hctx) EVP_MAC_CTX_free(hctx); if(mac) EVP_MAC_free(mac); -#endif - return rc; } -static int sqlcipher_openssl_kdf(void *ctx, int algorithm, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key) { +static int sqlcipher_openssl_kdf( + void *ctx, int algorithm, + const unsigned char *pass, int pass_sz, + const unsigned char* salt, int salt_sz, + int workfactor, + int key_sz, unsigned char *key +) { int rc = 0; switch(algorithm) { case SQLCIPHER_HMAC_SHA1: if(!(rc = PKCS5_PBKDF2_HMAC((const char *)pass, pass_sz, salt, salt_sz, workfactor, EVP_sha1(), key_sz, key))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_kdf: PKCS5_PBKDF2_HMAC() for EVP_sha1() workfactor %d and key size %d returned %d", workfactor, key_sz, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_kdf: PKCS5_PBKDF2_HMAC() for EVP_sha1() workfactor %d and key size %d returned %d", workfactor, key_sz, rc); sqlcipher_openssl_log_errors(); goto error; } break; case SQLCIPHER_HMAC_SHA256: if(!(rc = PKCS5_PBKDF2_HMAC((const char *)pass, pass_sz, salt, salt_sz, workfactor, EVP_sha256(), key_sz, key))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_kdf: PKCS5_PBKDF2_HMAC() for EVP_sha256() workfactor %d and key size %d returned %d", workfactor, key_sz, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_kdf: PKCS5_PBKDF2_HMAC() for EVP_sha256() workfactor %d and key size %d returned %d", workfactor, key_sz, rc); sqlcipher_openssl_log_errors(); goto error; } break; case SQLCIPHER_HMAC_SHA512: if(!(rc = PKCS5_PBKDF2_HMAC((const char *)pass, pass_sz, salt, salt_sz, workfactor, EVP_sha512(), key_sz, key))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_kdf: PKCS5_PBKDF2_HMAC() for EVP_sha512() workfactor %d and key size %d returned %d", workfactor, key_sz, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_kdf: PKCS5_PBKDF2_HMAC() for EVP_sha512() workfactor %d and key size %d returned %d", workfactor, key_sz, rc); sqlcipher_openssl_log_errors(); goto error; } @@ -383,35 +288,41 @@ static int sqlcipher_openssl_kdf(void *ctx, int algorithm, const unsigned char * return rc; } -static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out) { +static int sqlcipher_openssl_cipher( + void *ctx, int mode, + const unsigned char *key, int key_sz, + const unsigned char *iv, + const unsigned char *in, int in_sz, + unsigned char *out +) { int tmp_csz, csz, rc = 0; EVP_CIPHER_CTX* ectx = EVP_CIPHER_CTX_new(); if(ectx == NULL) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_cipher: EVP_CIPHER_CTX_new failed"); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_cipher: EVP_CIPHER_CTX_new failed"); sqlcipher_openssl_log_errors(); goto error; } if(!(rc = EVP_CipherInit_ex(ectx, OPENSSL_CIPHER, NULL, NULL, NULL, mode))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_cipher: EVP_CipherInit_ex for mode %d returned %d", mode, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_cipher: EVP_CipherInit_ex for mode %d returned %d", mode, rc); sqlcipher_openssl_log_errors(); goto error; } if(!(rc = EVP_CIPHER_CTX_set_padding(ectx, 0))) { /* no padding */ - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_cipher: EVP_CIPHER_CTX_set_padding 0 returned %d", rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_cipher: EVP_CIPHER_CTX_set_padding 0 returned %d", rc); sqlcipher_openssl_log_errors(); goto error; } if(!(rc = EVP_CipherInit_ex(ectx, NULL, NULL, key, iv, mode))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_cipher: EVP_CipherInit_ex for mode %d returned %d", mode, rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_cipher: EVP_CipherInit_ex for mode %d returned %d", mode, rc); sqlcipher_openssl_log_errors(); goto error; } if(!(rc = EVP_CipherUpdate(ectx, out, &tmp_csz, in, in_sz))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_cipher: EVP_CipherUpdate returned %d", rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_cipher: EVP_CipherUpdate returned %d", rc); sqlcipher_openssl_log_errors(); goto error; } @@ -419,7 +330,7 @@ static int sqlcipher_openssl_cipher(void *ctx, int mode, unsigned char *key, int csz = tmp_csz; out += tmp_csz; if(!(rc = EVP_CipherFinal_ex(ectx, out, &tmp_csz))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, "sqlcipher_openssl_cipher: EVP_CipherFinal_ex returned %d", rc); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "sqlcipher_openssl_cipher: EVP_CipherFinal_ex returned %d", rc); sqlcipher_openssl_log_errors(); goto error; } @@ -477,16 +388,12 @@ static int sqlcipher_openssl_ctx_free(void **ctx) { } static int sqlcipher_openssl_fips_status(void *ctx) { -#ifdef SQLCIPHER_FIPS - return FIPS_mode(); -#else return 0; -#endif } int sqlcipher_openssl_setup(sqlcipher_provider *p) { - p->activate = sqlcipher_openssl_activate; - p->deactivate = sqlcipher_openssl_deactivate; + p->init = NULL; + p->shutdown = NULL; p->get_provider_name = sqlcipher_openssl_get_provider_name; p->random = sqlcipher_openssl_random; p->hmac = sqlcipher_openssl_hmac; diff --git a/src/ctime.c b/src/ctime.c deleted file mode 100644 index 4d384a9dca..0000000000 --- a/src/ctime.c +++ /dev/null @@ -1,792 +0,0 @@ -/* DO NOT EDIT! -** This file is automatically generated by the script in the canonical -** SQLite source tree at tool/mkctimec.tcl. -** -** To modify this header, edit any of the various lists in that script -** which specify categories of generated conditionals in this file. -*/ - -/* -** 2010 February 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements routines used to report what compile-time options -** SQLite was built with. -*/ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ - -/* -** Include the configuration header output by 'configure' if we're using the -** autoconf-based build -*/ -#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "sqlite_cfg.h" -#define SQLITECONFIG_H 1 -#endif - -/* These macros are provided to "stringify" the value of the define -** for those options in which the value is meaningful. */ -#define CTIMEOPT_VAL_(opt) #opt -#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) - -/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This -** option requires a separate macro because legal values contain a single -** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ -#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 -#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) -#include "sqliteInt.h" - -/* -** An array of names of all compile-time options. This array should -** be sorted A-Z. -** -** This array looks large, but in a typical installation actually uses -** only a handful of compile-time options, so most times this array is usually -** rather short and uses little memory space. -*/ -static const char * const sqlite3azCompileOpt[] = { - -#ifdef SQLITE_32BIT_ROWID - "32BIT_ROWID", -#endif -#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC - "4_BYTE_ALIGNED_MALLOC", -#endif -#ifdef SQLITE_64BIT_STATS - "64BIT_STATS", -#endif -#ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN -# if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 - "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), -# endif -#endif -#ifdef SQLITE_ALLOW_URI_AUTHORITY - "ALLOW_URI_AUTHORITY", -#endif -#ifdef SQLITE_ATOMIC_INTRINSICS - "ATOMIC_INTRINSICS=" CTIMEOPT_VAL(SQLITE_ATOMIC_INTRINSICS), -#endif -#ifdef SQLITE_BITMASK_TYPE - "BITMASK_TYPE=" CTIMEOPT_VAL(SQLITE_BITMASK_TYPE), -#endif -#ifdef SQLITE_BUG_COMPATIBLE_20160819 - "BUG_COMPATIBLE_20160819", -#endif -#ifdef SQLITE_CASE_SENSITIVE_LIKE - "CASE_SENSITIVE_LIKE", -#endif -#ifdef SQLITE_CHECK_PAGES - "CHECK_PAGES", -#endif -#if defined(__clang__) && defined(__clang_major__) - "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." - CTIMEOPT_VAL(__clang_minor__) "." - CTIMEOPT_VAL(__clang_patchlevel__), -#elif defined(_MSC_VER) - "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), -#elif defined(__GNUC__) && defined(__VERSION__) - "COMPILER=gcc-" __VERSION__, -#endif -#ifdef SQLITE_COVERAGE_TEST - "COVERAGE_TEST", -#endif -#ifdef SQLITE_DEBUG - "DEBUG", -#endif -#ifdef SQLITE_DEFAULT_AUTOMATIC_INDEX - "DEFAULT_AUTOMATIC_INDEX", -#endif -#ifdef SQLITE_DEFAULT_AUTOVACUUM - "DEFAULT_AUTOVACUUM", -#endif -#ifdef SQLITE_DEFAULT_CACHE_SIZE - "DEFAULT_CACHE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_CACHE_SIZE), -#endif -#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC - "DEFAULT_CKPTFULLFSYNC", -#endif -#ifdef SQLITE_DEFAULT_FILE_FORMAT - "DEFAULT_FILE_FORMAT=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_FORMAT), -#endif -#ifdef SQLITE_DEFAULT_FILE_PERMISSIONS - "DEFAULT_FILE_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_PERMISSIONS), -#endif -#ifdef SQLITE_DEFAULT_FOREIGN_KEYS - "DEFAULT_FOREIGN_KEYS", -#endif -#ifdef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT - "DEFAULT_JOURNAL_SIZE_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT), -#endif -#ifdef SQLITE_DEFAULT_LOCKING_MODE - "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), -#endif -#ifdef SQLITE_DEFAULT_LOOKASIDE - "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), -#endif -#ifdef SQLITE_DEFAULT_MEMSTATUS -# if SQLITE_DEFAULT_MEMSTATUS != 1 - "DEFAULT_MEMSTATUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_MEMSTATUS), -# endif -#endif -#ifdef SQLITE_DEFAULT_MMAP_SIZE - "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), -#endif -#ifdef SQLITE_DEFAULT_PAGE_SIZE - "DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_PAGE_SIZE), -#endif -#ifdef SQLITE_DEFAULT_PCACHE_INITSZ - "DEFAULT_PCACHE_INITSZ=" CTIMEOPT_VAL(SQLITE_DEFAULT_PCACHE_INITSZ), -#endif -#ifdef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS - "DEFAULT_PROXYDIR_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_PROXYDIR_PERMISSIONS), -#endif -#ifdef SQLITE_DEFAULT_RECURSIVE_TRIGGERS - "DEFAULT_RECURSIVE_TRIGGERS", -#endif -#ifdef SQLITE_DEFAULT_ROWEST - "DEFAULT_ROWEST=" CTIMEOPT_VAL(SQLITE_DEFAULT_ROWEST), -#endif -#ifdef SQLITE_DEFAULT_SECTOR_SIZE - "DEFAULT_SECTOR_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_SECTOR_SIZE), -#endif -#ifdef SQLITE_DEFAULT_SYNCHRONOUS - "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), -#endif -#ifdef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT - "DEFAULT_WAL_AUTOCHECKPOINT=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_AUTOCHECKPOINT), -#endif -#ifdef SQLITE_DEFAULT_WAL_SYNCHRONOUS - "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), -#endif -#ifdef SQLITE_DEFAULT_WORKER_THREADS - "DEFAULT_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WORKER_THREADS), -#endif -#ifdef SQLITE_DIRECT_OVERFLOW_READ - "DIRECT_OVERFLOW_READ", -#endif -#ifdef SQLITE_DISABLE_DIRSYNC - "DISABLE_DIRSYNC", -#endif -#ifdef SQLITE_DISABLE_FTS3_UNICODE - "DISABLE_FTS3_UNICODE", -#endif -#ifdef SQLITE_DISABLE_FTS4_DEFERRED - "DISABLE_FTS4_DEFERRED", -#endif -#ifdef SQLITE_DISABLE_INTRINSIC - "DISABLE_INTRINSIC", -#endif -#ifdef SQLITE_DISABLE_LFS - "DISABLE_LFS", -#endif -#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - "DISABLE_PAGECACHE_OVERFLOW_STATS", -#endif -#ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT - "DISABLE_SKIPAHEAD_DISTINCT", -#endif -#ifdef SQLITE_DQS - "DQS=" CTIMEOPT_VAL(SQLITE_DQS), -#endif -#ifdef SQLITE_ENABLE_8_3_NAMES - "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), -#endif -#ifdef SQLITE_ENABLE_API_ARMOR - "ENABLE_API_ARMOR", -#endif -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - "ENABLE_ATOMIC_WRITE", -#endif -#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE - "ENABLE_BATCH_ATOMIC_WRITE", -#endif -#ifdef SQLITE_ENABLE_BYTECODE_VTAB - "ENABLE_BYTECODE_VTAB", -#endif -#ifdef SQLITE_ENABLE_CEROD - "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), -#endif -#ifdef SQLITE_ENABLE_COLUMN_METADATA - "ENABLE_COLUMN_METADATA", -#endif -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - "ENABLE_COLUMN_USED_MASK", -#endif -#ifdef SQLITE_ENABLE_COSTMULT - "ENABLE_COSTMULT", -#endif -#ifdef SQLITE_ENABLE_CURSOR_HINTS - "ENABLE_CURSOR_HINTS", -#endif -#ifdef SQLITE_ENABLE_DBPAGE_VTAB - "ENABLE_DBPAGE_VTAB", -#endif -#ifdef SQLITE_ENABLE_DBSTAT_VTAB - "ENABLE_DBSTAT_VTAB", -#endif -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT - "ENABLE_EXPENSIVE_ASSERT", -#endif -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - "ENABLE_EXPLAIN_COMMENTS", -#endif -#ifdef SQLITE_ENABLE_FTS3 - "ENABLE_FTS3", -#endif -#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS - "ENABLE_FTS3_PARENTHESIS", -#endif -#ifdef SQLITE_ENABLE_FTS3_TOKENIZER - "ENABLE_FTS3_TOKENIZER", -#endif -#ifdef SQLITE_ENABLE_FTS4 - "ENABLE_FTS4", -#endif -#ifdef SQLITE_ENABLE_FTS5 - "ENABLE_FTS5", -#endif -#ifdef SQLITE_ENABLE_GEOPOLY - "ENABLE_GEOPOLY", -#endif -#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS - "ENABLE_HIDDEN_COLUMNS", -#endif -#ifdef SQLITE_ENABLE_ICU - "ENABLE_ICU", -#endif -#ifdef SQLITE_ENABLE_IOTRACE - "ENABLE_IOTRACE", -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - "ENABLE_LOAD_EXTENSION", -#endif -#ifdef SQLITE_ENABLE_LOCKING_STYLE - "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), -#endif -#ifdef SQLITE_ENABLE_MATH_FUNCTIONS - "ENABLE_MATH_FUNCTIONS", -#endif -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - "ENABLE_MEMORY_MANAGEMENT", -#endif -#ifdef SQLITE_ENABLE_MEMSYS3 - "ENABLE_MEMSYS3", -#endif -#ifdef SQLITE_ENABLE_MEMSYS5 - "ENABLE_MEMSYS5", -#endif -#ifdef SQLITE_ENABLE_MULTIPLEX - "ENABLE_MULTIPLEX", -#endif -#ifdef SQLITE_ENABLE_NORMALIZE - "ENABLE_NORMALIZE", -#endif -#ifdef SQLITE_ENABLE_NULL_TRIM - "ENABLE_NULL_TRIM", -#endif -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - "ENABLE_OFFSET_SQL_FUNC", -#endif -#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK - "ENABLE_OVERSIZE_CELL_CHECK", -#endif -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - "ENABLE_PREUPDATE_HOOK", -#endif -#ifdef SQLITE_ENABLE_QPSG - "ENABLE_QPSG", -#endif -#ifdef SQLITE_ENABLE_RBU - "ENABLE_RBU", -#endif -#ifdef SQLITE_ENABLE_RTREE - "ENABLE_RTREE", -#endif -#ifdef SQLITE_ENABLE_SESSION - "ENABLE_SESSION", -#endif -#ifdef SQLITE_ENABLE_SNAPSHOT - "ENABLE_SNAPSHOT", -#endif -#ifdef SQLITE_ENABLE_SORTER_REFERENCES - "ENABLE_SORTER_REFERENCES", -#endif -#ifdef SQLITE_ENABLE_SQLLOG - "ENABLE_SQLLOG", -#endif -#ifdef SQLITE_ENABLE_STAT4 - "ENABLE_STAT4", -#endif -#ifdef SQLITE_ENABLE_STMTVTAB - "ENABLE_STMTVTAB", -#endif -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - "ENABLE_STMT_SCANSTATUS", -#endif -#ifdef SQLITE_ENABLE_TREETRACE - "ENABLE_TREETRACE", -#endif -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - "ENABLE_UNKNOWN_SQL_FUNCTION", -#endif -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - "ENABLE_UNLOCK_NOTIFY", -#endif -#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT - "ENABLE_UPDATE_DELETE_LIMIT", -#endif -#ifdef SQLITE_ENABLE_URI_00_ERROR - "ENABLE_URI_00_ERROR", -#endif -#ifdef SQLITE_ENABLE_VFSTRACE - "ENABLE_VFSTRACE", -#endif -#ifdef SQLITE_ENABLE_WHERETRACE - "ENABLE_WHERETRACE", -#endif -#ifdef SQLITE_ENABLE_ZIPVFS - "ENABLE_ZIPVFS", -#endif -#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - "EXPLAIN_ESTIMATED_ROWS", -#endif -#ifdef SQLITE_EXTRA_IFNULLROW - "EXTRA_IFNULLROW", -#endif -#ifdef SQLITE_EXTRA_INIT - "EXTRA_INIT=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT), -#endif -#ifdef SQLITE_EXTRA_SHUTDOWN - "EXTRA_SHUTDOWN=" CTIMEOPT_VAL(SQLITE_EXTRA_SHUTDOWN), -#endif -#ifdef SQLITE_FTS3_MAX_EXPR_DEPTH - "FTS3_MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_FTS3_MAX_EXPR_DEPTH), -#endif -#ifdef SQLITE_FTS5_ENABLE_TEST_MI - "FTS5_ENABLE_TEST_MI", -#endif -#ifdef SQLITE_FTS5_NO_WITHOUT_ROWID - "FTS5_NO_WITHOUT_ROWID", -#endif -/* BEGIN SQLCIPHER */ -#if SQLITE_HAS_CODEC - "HAS_CODEC", -#endif -/* END SQLCIPHER */ -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN - "HAVE_ISNAN", -#endif -#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX -# if SQLITE_HOMEGROWN_RECURSIVE_MUTEX != 1 - "HOMEGROWN_RECURSIVE_MUTEX=" CTIMEOPT_VAL(SQLITE_HOMEGROWN_RECURSIVE_MUTEX), -# endif -#endif -#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS - "IGNORE_AFP_LOCK_ERRORS", -#endif -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - "IGNORE_FLOCK_LOCK_ERRORS", -#endif -#ifdef SQLITE_INLINE_MEMCPY - "INLINE_MEMCPY", -#endif -#ifdef SQLITE_INT64_TYPE - "INT64_TYPE", -#endif -#ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX - "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), -#endif -#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - "LIKE_DOESNT_MATCH_BLOBS", -#endif -#ifdef SQLITE_LOCK_TRACE - "LOCK_TRACE", -#endif -#ifdef SQLITE_LOG_CACHE_SPILL - "LOG_CACHE_SPILL", -#endif -#ifdef SQLITE_MALLOC_SOFT_LIMIT - "MALLOC_SOFT_LIMIT=" CTIMEOPT_VAL(SQLITE_MALLOC_SOFT_LIMIT), -#endif -#ifdef SQLITE_MAX_ATTACHED - "MAX_ATTACHED=" CTIMEOPT_VAL(SQLITE_MAX_ATTACHED), -#endif -#ifdef SQLITE_MAX_COLUMN - "MAX_COLUMN=" CTIMEOPT_VAL(SQLITE_MAX_COLUMN), -#endif -#ifdef SQLITE_MAX_COMPOUND_SELECT - "MAX_COMPOUND_SELECT=" CTIMEOPT_VAL(SQLITE_MAX_COMPOUND_SELECT), -#endif -#ifdef SQLITE_MAX_DEFAULT_PAGE_SIZE - "MAX_DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_DEFAULT_PAGE_SIZE), -#endif -#ifdef SQLITE_MAX_EXPR_DEPTH - "MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_EXPR_DEPTH), -#endif -#ifdef SQLITE_MAX_FUNCTION_ARG - "MAX_FUNCTION_ARG=" CTIMEOPT_VAL(SQLITE_MAX_FUNCTION_ARG), -#endif -#ifdef SQLITE_MAX_LENGTH - "MAX_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LENGTH), -#endif -#ifdef SQLITE_MAX_LIKE_PATTERN_LENGTH - "MAX_LIKE_PATTERN_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LIKE_PATTERN_LENGTH), -#endif -#ifdef SQLITE_MAX_MEMORY - "MAX_MEMORY=" CTIMEOPT_VAL(SQLITE_MAX_MEMORY), -#endif -#ifdef SQLITE_MAX_MMAP_SIZE - "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), -#endif -#ifdef SQLITE_MAX_MMAP_SIZE_ - "MAX_MMAP_SIZE_=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE_), -#endif -#ifdef SQLITE_MAX_PAGE_COUNT - "MAX_PAGE_COUNT=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_COUNT), -#endif -#ifdef SQLITE_MAX_PAGE_SIZE - "MAX_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_SIZE), -#endif -#ifdef SQLITE_MAX_SCHEMA_RETRY - "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), -#endif -#ifdef SQLITE_MAX_SQL_LENGTH - "MAX_SQL_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_SQL_LENGTH), -#endif -#ifdef SQLITE_MAX_TRIGGER_DEPTH - "MAX_TRIGGER_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_TRIGGER_DEPTH), -#endif -#ifdef SQLITE_MAX_VARIABLE_NUMBER - "MAX_VARIABLE_NUMBER=" CTIMEOPT_VAL(SQLITE_MAX_VARIABLE_NUMBER), -#endif -#ifdef SQLITE_MAX_VDBE_OP - "MAX_VDBE_OP=" CTIMEOPT_VAL(SQLITE_MAX_VDBE_OP), -#endif -#ifdef SQLITE_MAX_WORKER_THREADS - "MAX_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_MAX_WORKER_THREADS), -#endif -#ifdef SQLITE_MEMDEBUG - "MEMDEBUG", -#endif -#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT - "MIXED_ENDIAN_64BIT_FLOAT", -#endif -#ifdef SQLITE_MMAP_READWRITE - "MMAP_READWRITE", -#endif -#ifdef SQLITE_MUTEX_NOOP - "MUTEX_NOOP", -#endif -#ifdef SQLITE_MUTEX_OMIT - "MUTEX_OMIT", -#endif -#ifdef SQLITE_MUTEX_PTHREADS - "MUTEX_PTHREADS", -#endif -#ifdef SQLITE_MUTEX_W32 - "MUTEX_W32", -#endif -#ifdef SQLITE_NEED_ERR_NAME - "NEED_ERR_NAME", -#endif -#ifdef SQLITE_NO_SYNC - "NO_SYNC", -#endif -#ifdef SQLITE_OMIT_ALTERTABLE - "OMIT_ALTERTABLE", -#endif -#ifdef SQLITE_OMIT_ANALYZE - "OMIT_ANALYZE", -#endif -#ifdef SQLITE_OMIT_ATTACH - "OMIT_ATTACH", -#endif -#ifdef SQLITE_OMIT_AUTHORIZATION - "OMIT_AUTHORIZATION", -#endif -#ifdef SQLITE_OMIT_AUTOINCREMENT - "OMIT_AUTOINCREMENT", -#endif -#ifdef SQLITE_OMIT_AUTOINIT - "OMIT_AUTOINIT", -#endif -#ifdef SQLITE_OMIT_AUTOMATIC_INDEX - "OMIT_AUTOMATIC_INDEX", -#endif -#ifdef SQLITE_OMIT_AUTORESET - "OMIT_AUTORESET", -#endif -#ifdef SQLITE_OMIT_AUTOVACUUM - "OMIT_AUTOVACUUM", -#endif -#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION - "OMIT_BETWEEN_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_BLOB_LITERAL - "OMIT_BLOB_LITERAL", -#endif -#ifdef SQLITE_OMIT_CAST - "OMIT_CAST", -#endif -#ifdef SQLITE_OMIT_CHECK - "OMIT_CHECK", -#endif -#ifdef SQLITE_OMIT_COMPLETE - "OMIT_COMPLETE", -#endif -#ifdef SQLITE_OMIT_COMPOUND_SELECT - "OMIT_COMPOUND_SELECT", -#endif -#ifdef SQLITE_OMIT_CONFLICT_CLAUSE - "OMIT_CONFLICT_CLAUSE", -#endif -#ifdef SQLITE_OMIT_CTE - "OMIT_CTE", -#endif -#if defined(SQLITE_OMIT_DATETIME_FUNCS) || defined(SQLITE_OMIT_FLOATING_POINT) - "OMIT_DATETIME_FUNCS", -#endif -#ifdef SQLITE_OMIT_DECLTYPE - "OMIT_DECLTYPE", -#endif -#ifdef SQLITE_OMIT_DEPRECATED - "OMIT_DEPRECATED", -#endif -#ifdef SQLITE_OMIT_DESERIALIZE - "OMIT_DESERIALIZE", -#endif -#ifdef SQLITE_OMIT_DISKIO - "OMIT_DISKIO", -#endif -#ifdef SQLITE_OMIT_EXPLAIN - "OMIT_EXPLAIN", -#endif -#ifdef SQLITE_OMIT_FLAG_PRAGMAS - "OMIT_FLAG_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_FLOATING_POINT - "OMIT_FLOATING_POINT", -#endif -#ifdef SQLITE_OMIT_FOREIGN_KEY - "OMIT_FOREIGN_KEY", -#endif -#ifdef SQLITE_OMIT_GET_TABLE - "OMIT_GET_TABLE", -#endif -#ifdef SQLITE_OMIT_HEX_INTEGER - "OMIT_HEX_INTEGER", -#endif -#ifdef SQLITE_OMIT_INCRBLOB - "OMIT_INCRBLOB", -#endif -#ifdef SQLITE_OMIT_INTEGRITY_CHECK - "OMIT_INTEGRITY_CHECK", -#endif -#ifdef SQLITE_OMIT_INTROSPECTION_PRAGMAS - "OMIT_INTROSPECTION_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_JSON - "OMIT_JSON", -#endif -#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION - "OMIT_LIKE_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_LOAD_EXTENSION - "OMIT_LOAD_EXTENSION", -#endif -#ifdef SQLITE_OMIT_LOCALTIME - "OMIT_LOCALTIME", -#endif -#ifdef SQLITE_OMIT_LOOKASIDE - "OMIT_LOOKASIDE", -#endif -#ifdef SQLITE_OMIT_MEMORYDB - "OMIT_MEMORYDB", -#endif -#ifdef SQLITE_OMIT_OR_OPTIMIZATION - "OMIT_OR_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_PAGER_PRAGMAS - "OMIT_PAGER_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_PARSER_TRACE - "OMIT_PARSER_TRACE", -#endif -#ifdef SQLITE_OMIT_POPEN - "OMIT_POPEN", -#endif -#ifdef SQLITE_OMIT_PRAGMA - "OMIT_PRAGMA", -#endif -#ifdef SQLITE_OMIT_PROGRESS_CALLBACK - "OMIT_PROGRESS_CALLBACK", -#endif -#ifdef SQLITE_OMIT_QUICKBALANCE - "OMIT_QUICKBALANCE", -#endif -#ifdef SQLITE_OMIT_REINDEX - "OMIT_REINDEX", -#endif -#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS - "OMIT_SCHEMA_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS - "OMIT_SCHEMA_VERSION_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_SHARED_CACHE - "OMIT_SHARED_CACHE", -#endif -#ifdef SQLITE_OMIT_SHUTDOWN_DIRECTORIES - "OMIT_SHUTDOWN_DIRECTORIES", -#endif -#ifdef SQLITE_OMIT_SUBQUERY - "OMIT_SUBQUERY", -#endif -#ifdef SQLITE_OMIT_TCL_VARIABLE - "OMIT_TCL_VARIABLE", -#endif -#ifdef SQLITE_OMIT_TEMPDB - "OMIT_TEMPDB", -#endif -#ifdef SQLITE_OMIT_TEST_CONTROL - "OMIT_TEST_CONTROL", -#endif -#ifdef SQLITE_OMIT_TRACE -# if SQLITE_OMIT_TRACE != 1 - "OMIT_TRACE=" CTIMEOPT_VAL(SQLITE_OMIT_TRACE), -# endif -#endif -#ifdef SQLITE_OMIT_TRIGGER - "OMIT_TRIGGER", -#endif -#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION - "OMIT_TRUNCATE_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_UTF16 - "OMIT_UTF16", -#endif -#ifdef SQLITE_OMIT_VACUUM - "OMIT_VACUUM", -#endif -#ifdef SQLITE_OMIT_VIEW - "OMIT_VIEW", -#endif -#ifdef SQLITE_OMIT_VIRTUALTABLE - "OMIT_VIRTUALTABLE", -#endif -#ifdef SQLITE_OMIT_WAL - "OMIT_WAL", -#endif -#ifdef SQLITE_OMIT_WSD - "OMIT_WSD", -#endif -#ifdef SQLITE_OMIT_XFER_OPT - "OMIT_XFER_OPT", -#endif -#ifdef SQLITE_PERFORMANCE_TRACE - "PERFORMANCE_TRACE", -#endif -#ifdef SQLITE_POWERSAFE_OVERWRITE -# if SQLITE_POWERSAFE_OVERWRITE != 1 - "POWERSAFE_OVERWRITE=" CTIMEOPT_VAL(SQLITE_POWERSAFE_OVERWRITE), -# endif -#endif -#ifdef SQLITE_PREFER_PROXY_LOCKING - "PREFER_PROXY_LOCKING", -#endif -#ifdef SQLITE_PROXY_DEBUG - "PROXY_DEBUG", -#endif -#ifdef SQLITE_REVERSE_UNORDERED_SELECTS - "REVERSE_UNORDERED_SELECTS", -#endif -#ifdef SQLITE_RTREE_INT_ONLY - "RTREE_INT_ONLY", -#endif -#ifdef SQLITE_SECURE_DELETE - "SECURE_DELETE", -#endif -#ifdef SQLITE_SMALL_STACK - "SMALL_STACK", -#endif -#ifdef SQLITE_SORTER_PMASZ - "SORTER_PMASZ=" CTIMEOPT_VAL(SQLITE_SORTER_PMASZ), -#endif -#ifdef SQLITE_SOUNDEX - "SOUNDEX", -#endif -#ifdef SQLITE_STAT4_SAMPLES - "STAT4_SAMPLES=" CTIMEOPT_VAL(SQLITE_STAT4_SAMPLES), -#endif -#ifdef SQLITE_STMTJRNL_SPILL - "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), -#endif -#ifdef SQLITE_SUBSTR_COMPATIBILITY - "SUBSTR_COMPATIBILITY", -#endif -#if (!defined(SQLITE_WIN32_MALLOC) \ - && !defined(SQLITE_ZERO_MALLOC) \ - && !defined(SQLITE_MEMDEBUG) \ - ) || defined(SQLITE_SYSTEM_MALLOC) - "SYSTEM_MALLOC", -#endif -#ifdef SQLITE_TCL - "TCL", -#endif -#ifdef SQLITE_TEMP_STORE - "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), -#endif -#ifdef SQLITE_TEST - "TEST", -#endif -#if defined(SQLITE_THREADSAFE) - "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), -#elif defined(THREADSAFE) - "THREADSAFE=" CTIMEOPT_VAL(THREADSAFE), -#else - "THREADSAFE=1", -#endif -#ifdef SQLITE_UNLINK_AFTER_CLOSE - "UNLINK_AFTER_CLOSE", -#endif -#ifdef SQLITE_UNTESTABLE - "UNTESTABLE", -#endif -#ifdef SQLITE_USER_AUTHENTICATION - "USER_AUTHENTICATION", -#endif -#ifdef SQLITE_USE_ALLOCA - "USE_ALLOCA", -#endif -#ifdef SQLITE_USE_FCNTL_TRACE - "USE_FCNTL_TRACE", -#endif -#ifdef SQLITE_USE_URI - "USE_URI", -#endif -#ifdef SQLITE_VDBE_COVERAGE - "VDBE_COVERAGE", -#endif -#ifdef SQLITE_WIN32_MALLOC - "WIN32_MALLOC", -#endif -#ifdef SQLITE_ZERO_MALLOC - "ZERO_MALLOC", -#endif - -} ; - -const char **sqlite3CompileOptions(int *pnOpt){ - *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt[0]); - return (const char**)sqlite3azCompileOpt; -} - -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ diff --git a/src/date.c b/src/date.c index 9b7957bbf0..5e7ae6f1fc 100644 --- a/src/date.c +++ b/src/date.c @@ -71,13 +71,14 @@ struct DateTime { int tz; /* Timezone offset in minutes */ double s; /* Seconds */ char validJD; /* True (1) if iJD is valid */ - char rawS; /* Raw numeric value stored in s */ char validYMD; /* True (1) if Y,M,D are valid */ char validHMS; /* True (1) if h,m,s are valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ - char isError; /* An overflow has occurred */ - char useSubsec; /* Display subsecond precision */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -110,8 +111,8 @@ struct DateTime { */ static int getDigits(const char *zDate, const char *zFormat, ...){ /* The aMx[] array translates the 3rd character of each format - ** spec into a max size: a b c d e f */ - static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 14712 }; va_list ap; int cnt = 0; char nextC; @@ -175,6 +176,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -185,9 +188,12 @@ static int parseTimezone(const char *zDate, DateTime *p){ } zDate += 5; p->tz = sgn*(nMn + nHr*60); + if( p->tz==0 ){ /* Forum post 2025-09-17T10:12:14z */ + p->isLocal = 0; + p->isUtc = 1; + } zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -220,6 +226,9 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ zDate++; } ms /= rScale; + /* Truncate to avoid problems with sub-milliseconds + ** rounding. https://sqlite.org/forum/forumpost/766a2c9231 */ + if( ms>0.999 ) ms = 0.999; } }else{ s = 0; @@ -231,7 +240,6 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } @@ -270,23 +278,48 @@ static void computeJD(DateTime *p){ Y--; M += 12; } - A = Y/100; - B = 2 - A + (A/4); + A = (Y+4800)/100; + B = 38 - A + (A/4); X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); - if( p->validTZ ){ + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } +/* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + /* ** Parse dates of the form ** @@ -325,12 +358,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -340,6 +377,9 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; @@ -423,7 +463,7 @@ static int validJulianDay(sqlite3_int64 iJD){ ** Compute the Year, Month, and Day from the julian day number. */ static void computeYMD(DateTime *p){ - int Z, A, B, C, D, E, X1; + int Z, alpha, A, B, C, D, E, X1; if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; @@ -434,8 +474,8 @@ static void computeYMD(DateTime *p){ return; }else{ Z = (int)((p->iJD + 43200000)/86400000); - A = (int)((Z - 1867216.25)/36524.25); - A = Z + 1 + A - (A/4); + alpha = (int)((Z + 32044.75)/36524.25) - 52; + A = Z + 1 + alpha - ((alpha+100)/4) + 25; B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; @@ -452,17 +492,14 @@ static void computeYMD(DateTime *p){ ** Compute the Hour, Minute, and Seconds from the julian day number. */ static void computeHMS(DateTime *p){ - int s; + int day_ms, day_min; /* milliseconds, minutes into the day */ if( p->validHMS ) return; computeJD(p); - s = (int)((p->iJD + 43200000) % 86400000); - p->s = s/1000.0; - s = (int)p->s; - p->s -= s; - p->h = s/3600; - s -= p->h*3600; - p->m = s/60; - p->s += s - p->m*60; + day_ms = (int)((p->iJD + 43200000) % 86400000); + p->s = (day_ms % 60000)/1000.0; + day_min = day_ms/60000; + p->m = day_min % 60; + p->h = day_min / 60; p->rawS = 0; p->validHMS = 1; } @@ -481,7 +518,7 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME @@ -613,7 +650,7 @@ static int toLocaltime( p->validHMS = 1; p->validJD = 0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->isError = 0; return SQLITE_OK; } @@ -633,14 +670,33 @@ static const struct { float rLimit; /* Maximum NNN value for this transform */ float rXform; /* Constant used for this transform */ } aXformType[] = { - { 6, "second", 4.6427e+14, 1.0 }, - { 6, "minute", 7.7379e+12, 60.0 }, - { 4, "hour", 1.2897e+11, 3600.0 }, - { 3, "day", 5373485.0, 86400.0 }, - { 5, "month", 176546.0, 2592000.0 }, - { 4, "year", 14713.0, 31536000.0 }, + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 2592000.0 }, + /* 5 */ { 4, "year", 14713.0, 31536000.0 }, }; +/* +** If the DateTime p is raw number, try to figure out if it is +** a julian day number of a unix timestamp. Set the p value +** appropriately. +*/ +static void autoAdjustDate(DateTime *p){ + if( !p->rawS || p->validJD ){ + p->rawS = 0; + }else if( p->s>=-21086676*(i64)10000 /* -4713-11-24 12:00:00 */ + && p->s<=(25340230*(i64)10000)+799 /* 9999-12-31 23:59:59 */ + ){ + double r = p->s*1000.0 + 210866760000000.0; + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + } +} + /* ** Process a modifier to a date-time stamp. The modifiers are ** as follows: @@ -651,14 +707,20 @@ static const struct { ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written @@ -684,19 +746,39 @@ static int parseModifier( */ if( sqlite3_stricmp(z, "auto")==0 ){ if( idx>1 ) return 1; /* IMP: R-33611-57934 */ - if( !p->rawS || p->validJD ){ - rc = 0; - p->rawS = 0; - }else if( p->s>=-21086676*(i64)10000 /* -4713-11-24 12:00:00 */ - && p->s<=(25340230*(i64)10000)+799 /* 9999-12-31 23:59:59 */ - ){ - r = p->s*1000.0 + 210866760000000.0; - clearYMD_HMS_TZ(p); - p->iJD = (sqlite3_int64)(r + 0.5); - p->validJD = 1; - p->rawS = 0; - rc = 0; - } + autoAdjustDate(p); + rc = 0; + } + break; + } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; } break; } @@ -726,7 +808,9 @@ static int parseModifier( ** show local time. */ if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ - rc = toLocaltime(p, pCtx); + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -751,7 +835,7 @@ static int parseModifier( } #ifndef SQLITE_OMIT_LOCALTIME else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ - if( p->tzSet==0 ){ + if( p->isUtc==0 ){ i64 iOrigJD; /* Original localtime */ i64 iGuess; /* Guess at the corresponding utc time */ int cnt = 0; /* Safety to prevent infinite loop */ @@ -774,7 +858,8 @@ static int parseModifier( memset(p, 0, sizeof(*p)); p->iJD = iGuess; p->validJD = 1; - p->tzSet = 1; + p->isUtc = 1; + p->isLocal = 0; } rc = SQLITE_OK; } @@ -794,7 +879,7 @@ static int parseModifier( && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -834,7 +919,7 @@ static int parseModifier( p->h = p->m = 0; p->s = 0.0; p->rawS = 0; - p->validTZ = 0; + p->tz = 0; p->validJD = 0; if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; @@ -862,18 +947,74 @@ static int parseModifier( case '9': { double rRounder; int i; - for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + int Y,M,D,h,m,x; + const char *z2 = z; + char z0 = z[0]; + for(n=1; z[n]; n++){ + if( z[n]==':' ) break; + if( sqlite3Isspace(z[n]) ) break; + if( z[n]=='-' ){ + if( n==5 && getDigits(&z[1], "40f", &Y)==1 ) break; + if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; + } + } if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ - rc = 1; + assert( rc==1 ); break; } - if( z[n]==':' ){ + if( z[n]=='-' ){ + /* A modifier of the form (+|-)YYYY-MM-DD adds or subtracts the + ** specified number of years, months, and days. MM is limited to + ** the range 0-11 and DD is limited to 0-30. + */ + if( z0!='+' && z0!='-' ) break; /* Must start with +/- */ + if( n==5 ){ + if( getDigits(&z[1], "40f-20a-20d", &Y, &M, &D)!=3 ) break; + }else{ + assert( n==6 ); + if( getDigits(&z[1], "50f-20a-20d", &Y, &M, &D)!=3 ) break; + z++; + } + if( M>=12 ) break; /* M range 0..11 */ + if( D>=31 ) break; /* D range 0..30 */ + computeYMD_HMS(p); + p->validJD = 0; + if( z0=='-' ){ + p->Y -= Y; + p->M -= M; + D = -D; + }else{ + p->Y += Y; + p->M += M; + } + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + computeFloor(p); + computeJD(p); + p->validHMS = 0; + p->validYMD = 0; + p->iJD += (i64)D*86400000; + if( z[11]==0 ){ + rc = 0; + break; + } + if( sqlite3Isspace(z[11]) + && getDigits(&z[12], "20c:20e", &h, &m)==2 + ){ + z2 = &z[12]; + n = 2; + }else{ + break; + } + } + if( z2[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be ** omitted. */ - const char *z2 = z; + DateTime tx; sqlite3_int64 day; if( !sqlite3Isdigit(*z2) ) z2++; @@ -883,7 +1024,7 @@ static int parseModifier( tx.iJD -= 43200000; day = tx.iJD/86400000; tx.iJD -= day*86400000; - if( z[0]=='-' ) tx.iJD = -tx.iJD; + if( z0=='-' ) tx.iJD = -tx.iJD; computeJD(p); clearYMD_HMS_TZ(p); p->iJD += tx.iJD; @@ -896,11 +1037,12 @@ static int parseModifier( z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; + if( n<3 || n>10 ) break; if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); - rc = 1; + assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; + p->nFloor = 0; for(i=0; iM += (int)r; x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; p->Y += x; p->M -= x*12; + computeFloor(p); p->validJD = 0; r -= (int)r; break; } case 5: { /* Special processing to add years */ int y = (int)r; - assert( strcmp(aXformType[i].zName,"year")==0 ); + assert( strcmp(aXformType[5].zName,"year")==0 ); computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); p->Y += y; + computeFloor(p); p->validJD = 0; r -= (int)r; break; @@ -984,6 +1128,12 @@ static int isDate( } computeJD(p); if( p->isError || !validJulianDay(p->iJD) ) return 1; + if( argc==1 && p->validYMD && p->D>28 ){ + /* Make sure a YYYY-MM-DD is normalized. + ** Example: 2023-02-31 -> 2023-03-03 */ + assert( p->validJD ); + p->validYMD = 0; + } return 0; } @@ -1067,7 +1217,7 @@ static void datetimeFunc( zBuf[16] = '0' + (x.m)%10; zBuf[17] = ':'; if( x.useSubsec ){ - s = (int)1000.0*x.s; + s = (int)(1000.0*x.s + 0.5); zBuf[18] = '0' + (s/10000)%10; zBuf[19] = '0' + (s/1000)%10; zBuf[20] = '.'; @@ -1114,7 +1264,7 @@ static void timeFunc( zBuf[4] = '0' + (x.m)%10; zBuf[5] = ':'; if( x.useSubsec ){ - s = (int)1000.0*x.s; + s = (int)(1000.0*x.s + 0.5); zBuf[6] = '0' + (s/10000)%10; zBuf[7] = '0' + (s/1000)%10; zBuf[8] = '.'; @@ -1171,22 +1321,83 @@ static void dateFunc( } } +/* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tuesday, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "AM" or "PM" +** %P "am" or "pm" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -1211,44 +1422,61 @@ static void strftimeFunc( computeJD(&x); computeYMD_HMS(&x); for(i=j=0; zFmt[i]; i++){ + char cf; if( zFmt[i]!='%' ) continue; if( j59.999 ) s = 59.999; + if( NEVER(s>59.999) ) s = 59.999; sqlite3_str_appendf(&sRes, "%06.3f", s); break; } - case 'H': { - sqlite3_str_appendf(&sRes, "%02d", x.h); + case 'F': { + sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); break; } - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ + case 'G': /* Fall thru */ + case 'g': { DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( zFmt[i]=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); }else{ - sqlite3_str_appendf(&sRes,"%03d",nDay+1); + sqlite3_str_appendf(&sRes, "%04d", y.Y); } break; } - case 'J': { + case 'H': + case 'k': { + sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); + break; + } + case 'I': /* Fall thru */ + case 'l': { + int h = x.h; + if( h>12 ) h -= 12; + if( h==0 ) h = 12; + sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); + break; + } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); + break; + } + case 'J': { /* Julian day number. (Non-standard) */ sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); break; } @@ -1260,6 +1488,19 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d",x.m); break; } + case 'p': /* Fall thru */ + case 'P': { + if( x.h>=12 ){ + sqlite3_str_append(&sRes, cf=='p' ? "PM" : "pm", 2); + }else{ + sqlite3_str_append(&sRes, cf=='p' ? "AM" : "am", 2); + } + break; + } + case 'R': { + sqlite3_str_appendf(&sRes, "%02d:%02d", x.h, x.m); + break; + } case 's': { if( x.useSubsec ){ sqlite3_str_appendf(&sRes,"%.3f", @@ -1274,9 +1515,35 @@ static void strftimeFunc( sqlite3_str_appendf(&sRes,"%02d",(int)x.s); break; } - case 'w': { - sqlite3_str_appendchar(&sRes, 1, - (char)(((x.iJD+129600000)/86400000) % 7) + '0'); + case 'T': { + sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); + break; + } + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; + if( c=='0' && cf=='u' ) c = '7'; + sqlite3_str_appendchar(&sRes, 1, c); + break; + } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); break; } case 'Y': { @@ -1325,6 +1592,115 @@ static void cdateFunc( dateFunc(context, 0, 0); } +/* +** timediff(DATE1, DATE2) +** +** Return the amount of time that must be added to DATE2 in order to +** convert it into DATE2. The time difference format is: +** +** +YYYY-MM-DD HH:MM:SS.SSS +** +** The initial "+" becomes "-" if DATE1 occurs before DATE2. For +** date/time values A and B, the following invariant should hold: +** +** datetime(A) == (datetime(B, timediff(A,B)) +** +** Both DATE arguments must be either a julian day number, or an +** ISO-8601 string. The unix timestamps are not supported by this +** routine. +*/ +static void timediffFunc( + sqlite3_context *context, + int NotUsed1, + sqlite3_value **argv +){ + char sign; + int Y, M; + DateTime d1, d2; + sqlite3_str sRes; + UNUSED_PARAMETER(NotUsed1); + if( isDate(context, 1, &argv[0], &d1) ) return; + if( isDate(context, 1, &argv[1], &d2) ) return; + computeYMD_HMS(&d1); + computeYMD_HMS(&d2); + if( d1.iJD>=d2.iJD ){ + sign = '+'; + Y = d1.Y - d2.Y; + if( Y ){ + d2.Y = d1.Y; + d2.validJD = 0; + computeJD(&d2); + } + M = d1.M - d2.M; + if( M<0 ){ + Y--; + M += 12; + } + if( M!=0 ){ + d2.M = d1.M; + d2.validJD = 0; + computeJD(&d2); + } + while( d1.iJDd2.iJD ){ + M--; + if( M<0 ){ + M = 11; + Y--; + } + d2.M++; + if( d2.M>12 ){ + d2.M = 1; + d2.Y++; + } + d2.validJD = 0; + computeJD(&d2); + } + d1.iJD = d2.iJD - d1.iJD; + d1.iJD += (u64)1486995408 * (u64)100000; + } + clearYMD_HMS_TZ(&d1); + computeYMD_HMS(&d1); + sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); + sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", + sign, Y, M, d1.D-1, d1.h, d1.m, d1.s); + sqlite3ResultStrAccum(context, &sRes); +} + + /* ** current_timestamp() ** @@ -1385,6 +1761,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -1399,6 +1805,10 @@ void sqlite3RegisterDateTimeFunctions(void){ PURE_DATE(time, -1, 0, 0, timeFunc ), PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), + PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), diff --git a/src/dbpage.c b/src/dbpage.c index 32a9ce55bf..71cf305d19 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -28,7 +28,13 @@ ** ** The data field of sqlite_dbpage table can be updated. The new ** value must be a BLOB which is the correct page size, otherwise the -** update fails. Rows may not be deleted or inserted. +** update fails. INSERT operations also work, and operate as if they +** where REPLACE. The size of the database can be extended by INSERT-ing +** new pages on the end. +** +** Rows may not be deleted. However, doing an INSERT to page number N +** with NULL page data causes the N-th page and all subsequent pages to be +** deleted and the database to be truncated. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -40,8 +46,8 @@ typedef struct DbpageCursor DbpageCursor; struct DbpageCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ - int pgno; /* Current page number */ - int mxPgno; /* Last page to visit on this scan */ + Pgno pgno; /* Current page number */ + Pgno mxPgno; /* Last page to visit on this scan */ Pager *pPager; /* Pager being read/written */ DbPage *pPage1; /* Page 1 of the database */ int iDb; /* Index of database to analyze */ @@ -51,6 +57,8 @@ struct DbpageCursor { struct DbpageTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database */ + int iDbTrunc; /* Database to truncate */ + Pgno pgnoTrunc; /* Size to truncate to */ }; /* Columns */ @@ -59,7 +67,6 @@ struct DbpageTable { #define DBPAGE_COLUMN_SCHEMA 2 - /* ** Connect to or create a dbpagevfs virtual table. */ @@ -177,7 +184,7 @@ static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ }else{ memset(pCsr, 0, sizeof(DbpageCursor)); pCsr->base.pVtab = pVTab; - pCsr->pgno = -1; + pCsr->pgno = 0; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -220,7 +227,7 @@ static int dbpageEof(sqlite3_vtab_cursor *pCursor){ ** idxStr is not used */ static int dbpageFilter( - sqlite3_vtab_cursor *pCursor, + sqlite3_vtab_cursor *pCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ @@ -230,10 +237,11 @@ static int dbpageFilter( sqlite3 *db = pTab->db; Btree *pBt; - (void)idxStr; - + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); + /* Default setting is no rows of result */ - pCsr->pgno = 1; + pCsr->pgno = 1; pCsr->mxPgno = 0; if( idxNum & 2 ){ @@ -268,20 +276,20 @@ static int dbpageFilter( } static int dbpageColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, int i ){ DbpageCursor *pCsr = (DbpageCursor *)pCursor; int rc = SQLITE_OK; switch( i ){ case 0: { /* pgno */ - sqlite3_result_int(ctx, pCsr->pgno); + sqlite3_result_int64(ctx, (sqlite3_int64)pCsr->pgno); break; } case 1: { /* data */ DbPage *pDbPage = 0; - if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + if( pCsr->pgno==(Pgno)((PENDING_BYTE/pCsr->szPage)+1) ){ /* The pending byte page. Assume it is zeroed out. Attempting to ** request this page from the page is an SQLITE_CORRUPT error. */ sqlite3_result_zeroblob(ctx, pCsr->szPage); @@ -310,6 +318,24 @@ static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ return SQLITE_OK; } +/* +** Open write transactions. Since we do not know in advance which database +** files will be written by the sqlite_dbpage virtual table, start a write +** transaction on them all. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int dbpageBeginTrans(DbpageTable *pTab){ + sqlite3 *db = pTab->db; + int rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ) rc = sqlite3BtreeBeginTrans(pBt, 1, 0); + } + return rc; +} + static int dbpageUpdate( sqlite3_vtab *pVtab, int argc, @@ -321,11 +347,11 @@ static int dbpageUpdate( DbPage *pDbPage = 0; int rc = SQLITE_OK; char *zErr = 0; - const char *zSchema; int iDb; Btree *pBt; Pager *pPager; int szPage; + int isInsert; (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ @@ -336,21 +362,29 @@ static int dbpageUpdate( zErr = "cannot delete"; goto update_fail; } - pgno = sqlite3_value_int(argv[0]); - if( sqlite3_value_type(argv[0])==SQLITE_NULL - || (Pgno)sqlite3_value_int(argv[1])!=pgno - ){ - zErr = "cannot insert"; - goto update_fail; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + pgno = (Pgno)sqlite3_value_int64(argv[2]); + isInsert = 1; + }else{ + pgno = (Pgno)sqlite3_value_int64(argv[0]); + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + isInsert = 0; } - zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = ALWAYS(zSchema) ? sqlite3FindDbName(pTab->db, zSchema) : -1; - if( NEVER(iDb<0) ){ - zErr = "no such schema"; - goto update_fail; + if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ + iDb = 0; + }else{ + const char *zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = sqlite3FindDbName(pTab->db, zSchema); + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } } pBt = pTab->db->aDb[iDb].pBt; - if( NEVER(pgno<1) || NEVER(pBt==0) || NEVER(pgno>sqlite3BtreeLastPage(pBt)) ){ + if( pgno<1 || NEVER(pBt==0) ){ zErr = "bad page number"; goto update_fail; } @@ -358,51 +392,84 @@ static int dbpageUpdate( if( sqlite3_value_type(argv[3])!=SQLITE_BLOB || sqlite3_value_bytes(argv[3])!=szPage ){ - zErr = "bad page value"; + if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert && pgno>1 ){ + /* "INSERT INTO dbpage($PGNO,NULL)" causes page number $PGNO and + ** all subsequent pages to be deleted. */ + pTab->iDbTrunc = iDb; + pTab->pgnoTrunc = pgno-1; + pgno = 1; + }else{ + zErr = "bad page value"; + goto update_fail; + } + } + + if( dbpageBeginTrans(pTab)!=SQLITE_OK ){ + zErr = "failed to open transaction"; goto update_fail; } + pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ const void *pData = sqlite3_value_blob(argv[3]); - assert( pData!=0 || pTab->db->mallocFailed ); - if( pData - && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK - ){ - memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); + if( (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK && pData ){ + unsigned char *aPage = sqlite3PagerGetData(pDbPage); + memcpy(aPage, pData, szPage); + pTab->pgnoTrunc = 0; } } + if( rc!=SQLITE_OK ){ + pTab->pgnoTrunc = 0; + } sqlite3PagerUnref(pDbPage); return rc; update_fail: + pTab->pgnoTrunc = 0; sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = sqlite3_mprintf("%s", zErr); return SQLITE_ERROR; } -/* Since we do not know in advance which database files will be -** written by the sqlite_dbpage virtual table, start a write transaction -** on them all. -*/ static int dbpageBegin(sqlite3_vtab *pVtab){ DbpageTable *pTab = (DbpageTable *)pVtab; - sqlite3 *db = pTab->db; - int i; - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); + pTab->pgnoTrunc = 0; + return SQLITE_OK; +} + +/* Invoke sqlite3PagerTruncate() as necessary, just prior to COMMIT +*/ +static int dbpageSync(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + if( pTab->pgnoTrunc>0 ){ + Btree *pBt = pTab->db->aDb[pTab->iDbTrunc].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3BtreeEnter(pBt); + if( pTab->pgnoTruncpgnoTrunc); + } + sqlite3BtreeLeave(pBt); } + pTab->pgnoTrunc = 0; return SQLITE_OK; } +/* Cancel any pending truncate. +*/ +static int dbpageRollbackTo(sqlite3_vtab *pVtab, int notUsed1){ + DbpageTable *pTab = (DbpageTable *)pVtab; + pTab->pgnoTrunc = 0; + (void)notUsed1; + return SQLITE_OK; +} /* ** Invoke this routine to register the "dbpage" virtual table module */ int sqlite3DbpageRegister(sqlite3 *db){ static sqlite3_module dbpage_module = { - 0, /* iVersion */ + 2, /* iVersion */ dbpageConnect, /* xCreate */ dbpageConnect, /* xConnect */ dbpageBestIndex, /* xBestIndex */ @@ -417,15 +484,16 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageRowid, /* xRowid - read data */ dbpageUpdate, /* xUpdate */ dbpageBegin, /* xBegin */ - 0, /* xSync */ + dbpageSync, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ - 0, /* xRollbackTo */ - 0 /* xShadowName */ + dbpageRollbackTo, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); } diff --git a/src/dbstat.c b/src/dbstat.c index 0a89d05249..d635a82975 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -279,6 +279,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; pIdxInfo->idxNum |= 0x08; } + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_HEX; return SQLITE_OK; } @@ -895,7 +896,8 @@ int sqlite3DbstatRegister(sqlite3 *db){ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "dbstat", &dbstat_module, 0); } diff --git a/src/delete.c b/src/delete.c index 27a554916a..8fac7c2f32 100644 --- a/src/delete.c +++ b/src/delete.c @@ -19,13 +19,13 @@ ** (as in the FROM clause of a SELECT statement) in this case it contains ** the name of a single table, as one might find in an INSERT, DELETE, ** or UPDATE statement. Look up that table in the symbol table and -** return a pointer. Set an error message and return NULL if the table +** return a pointer. Set an error message and return NULL if the table ** name is not found or if any other error occurs. ** ** The following fields are initialized appropriate in pSrc: ** -** pSrc->a[0].pTab Pointer to the Table object -** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** pSrc->a[0].spTab Pointer to the Table object +** pSrc->a[0].u2.pIBIndex Pointer to the INDEXED BY index, if there is one ** */ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ @@ -33,8 +33,9 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ Table *pTab; assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - sqlite3DeleteTable(pParse->db, pItem->pTab); - pItem->pTab = pTab; + if( pItem->pSTab ) sqlite3DeleteTable(pParse->db, pItem->pSTab); + pItem->pSTab = pTab; + pItem->fg.notCte = 1; if( pTab ){ pTab->nTabRef++; if( pItem->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pItem) ){ @@ -44,7 +45,7 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ return pTab; } -/* Generate byte-code that will report the number of rows modified +/* Generate byte-code that will report the number of rows modified ** by a DELETE, INSERT, or UPDATE statement. */ void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ @@ -66,7 +67,7 @@ void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ ** the table is not SQLITE_VTAB_INNOCUOUS. ** ** 3) It is a system table (i.e. sqlite_schema), this call is not -** part of a nested parse and writable_schema pragma has not +** part of a nested parse and writable_schema pragma has not ** been specified ** ** 4) The table is a shadow table, the database connection is in @@ -74,6 +75,7 @@ void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ ** is for a top-level SQL statement. */ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + assert( IsVirtual(pTab) ); if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ return 1; } @@ -85,7 +87,7 @@ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ ** virtual tables if PRAGMA trusted_schema=ON. */ if( pParse->pToplevel!=0 - && pTab->u.vtab.p->eVtabRisk > + && pTab->u.vtab.p->eVtabRisk > ((pParse->db->flags & SQLITE_TrustedSchema)!=0) ){ sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", @@ -108,7 +110,7 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ } /* -** Check to make sure the given table is writable. +** Check to make sure the given table is writable. ** ** If pTab is not writable -> generate an error message and return 1. ** If pTab is writable but other errors have occurred -> return 1. @@ -120,7 +122,7 @@ int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){ return 1; } #ifndef SQLITE_OMIT_VIEW - if( IsView(pTab) + if( IsView(pTab) && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0)) ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); @@ -155,11 +157,12 @@ void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].fg.fixedSchema==0 && pFrom->a[0].fg.isSubquery==0 ); + pFrom->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].fg.isUsing==0 ); assert( pFrom->a[0].u3.pOn==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, SF_IncludeHidden, pLimit); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); @@ -187,7 +190,7 @@ Expr *sqlite3LimitWhere( sqlite3 *db = pParse->db; Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + ExprList *pEList = NULL; /* Expression list containing only pSelectRowid*/ SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ Select *pSelect = NULL; /* Complete SELECT tree */ Table *pTab; @@ -208,16 +211,16 @@ Expr *sqlite3LimitWhere( return pWhere; } - /* Generate a select expression tree to enforce the limit/offset + /* Generate a select expression tree to enforce the limit/offset ** term for the DELETE or UPDATE statement. For example: ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 ** becomes: - ** DELETE FROM table_a WHERE rowid IN ( + ** DELETE FROM table_a WHERE rowid IN ( ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 ** ); */ - pTab = pSrc->a[0].pTab; + pTab = pSrc->a[0].pSTab; if( HasRowid(pTab) ){ pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); pEList = sqlite3ExprListAppend( @@ -225,14 +228,20 @@ Expr *sqlite3LimitWhere( ); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol>=1 ); if( pPk->nKeyCol==1 ){ - const char *zName = pTab->aCol[pPk->aiColumn[0]].zCnName; + const char *zName; + assert( pPk->aiColumn[0]>=0 && pPk->aiColumn[0]nCol ); + zName = pTab->aCol[pPk->aiColumn[0]].zCnName; pLhs = sqlite3Expr(db, TK_ID, zName); pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); }else{ int i; for(i=0; inKeyCol; i++){ - Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); + Expr *p; + assert( pPk->aiColumn[i]>=0 && pPk->aiColumn[i]nCol ); + p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); pEList = sqlite3ExprListAppend(pParse, pEList, p); } pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); @@ -244,9 +253,9 @@ Expr *sqlite3LimitWhere( /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ - pSrc->a[0].pTab = 0; + pSrc->a[0].pSTab = 0; pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); - pSrc->a[0].pTab = pTab; + pSrc->a[0].pSTab = pTab; if( pSrc->a[0].fg.isIndexedBy ){ assert( pSrc->a[0].fg.isCte==0 ); pSrc->a[0].u2.pIBIndex = 0; @@ -257,11 +266,11 @@ Expr *sqlite3LimitWhere( } /* generate the SELECT expression tree. */ - pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, + pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, pOrderBy,0,pLimit ); - /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + /* now generate the new WHERE rowid IN clause for the DELETE/UPDATE */ pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; @@ -313,7 +322,7 @@ void sqlite3DeleteFrom( int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ int bComplex; /* True if there are triggers or FKs or ** subqueries in the WHERE clause */ - + #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ @@ -381,7 +390,7 @@ void sqlite3DeleteFrom( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, db->aDb[iDb].zDbSName); assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); if( rcauth==SQLITE_DENY ){ @@ -417,7 +426,7 @@ void sqlite3DeleteFrom( */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - sqlite3MaterializeView(pParse, pTab, + sqlite3MaterializeView(pParse, pTab, pWhere, pOrderBy, pLimit, iTabCur ); iDataCur = iIdxCur = iTabCur; @@ -450,7 +459,7 @@ void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION /* Special case: A DELETE without a WHERE clause deletes everything. ** It is easier just to erase the whole table. Prior to version 3.6.5, - ** this optimization caused the row change count (the value returned by + ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. ** ** The "rcauth==SQLITE_OK" terms is the @@ -490,7 +499,7 @@ void sqlite3DeleteFrom( if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; - nPk = 1; + assert( nPk==1 ); iRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ @@ -505,7 +514,7 @@ void sqlite3DeleteFrom( addrEphOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEphCur, nPk); sqlite3VdbeSetP4KeyInfo(pParse, pPk); } - + /* Construct a query to find the rowid or primary key for every row ** to be deleted, based on the WHERE clause. Set variable eOnePass ** to indicate the strategy used to implement this delete: @@ -518,17 +527,18 @@ void sqlite3DeleteFrom( if( pWInfo==0 ) goto delete_from_cleanup; eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); - assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF + || OptimizationDisabled(db, SQLITE_OnePass) ); if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); } - + /* Keep track of the number of rows to be deleted */ if( memCnt ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); } - + /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; inMem; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, -1, iKey); } - + if( eOnePass!=ONEPASS_OFF ){ /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the @@ -573,10 +583,10 @@ void sqlite3DeleteFrom( } sqlite3WhereEnd(pWInfo); } - - /* Unless this is a view, open cursors for the table we are + + /* Unless this is a view, open cursors for the table we are ** deleting from and all its indices. If this is a view, then the - ** only effect this statement has is to fire the INSTEAD OF + ** only effect this statement has is to fire the INSTEAD OF ** triggers. */ if( !isView ){ @@ -593,7 +603,7 @@ void sqlite3DeleteFrom( sqlite3VdbeJumpHereOrPopInst(v, iAddrOnce); } } - + /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ @@ -616,8 +626,8 @@ void sqlite3DeleteFrom( addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); VdbeCoverage(v); assert( nKey==1 ); - } - + } + /* Delete the row */ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ @@ -640,7 +650,7 @@ void sqlite3DeleteFrom( sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); } - + /* End of the loop over all rowids/primary-keys. */ if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); @@ -651,7 +661,7 @@ void sqlite3DeleteFrom( }else{ sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); - } + } } /* End non-truncate path */ /* Update the sqlite_sequence table by storing the content of the @@ -662,7 +672,7 @@ void sqlite3DeleteFrom( sqlite3AutoincrementEnd(pParse); } - /* Return the number of rows that were deleted. If this routine is + /* Return the number of rows that were deleted. If this routine is ** generating code because of a call to sqlite3NestedParse(), do not ** invoke the callback function. */ @@ -674,7 +684,7 @@ void sqlite3DeleteFrom( sqlite3AuthContextPop(&sContext); sqlite3SrcListDelete(db, pTabList); sqlite3ExprDelete(db, pWhere); -#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) sqlite3ExprListDelete(db, pOrderBy); sqlite3ExprDelete(db, pLimit); #endif @@ -719,7 +729,7 @@ void sqlite3DeleteFrom( ** and nPk before reading from it. ** ** If eMode is ONEPASS_MULTI, then this call is being made as part -** of a ONEPASS delete that affects multiple rows. In this case, if +** of a ONEPASS delete that affects multiple rows. In this case, if ** iIdxNoSeek is a valid cursor number (>=0) and is not the same as ** iDataCur, then its position should be preserved following the delete ** operation. Or, if iIdxNoSeek is not a valid cursor number, the @@ -755,7 +765,7 @@ void sqlite3GenerateRowDelete( VdbeModuleComment((v, "BEGIN: GenRowDel(%d,%d,%d,%d)", iDataCur, iIdxCur, iPk, (int)nPk)); - /* Seek cursor iCur to the row to delete. If this row no longer exists + /* Seek cursor iCur to the row to delete. If this row no longer exists ** (this can happen if a trigger program has already deleted it), do ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(pParse); @@ -765,7 +775,7 @@ void sqlite3GenerateRowDelete( VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); } - + /* If there are any triggers to fire, allocate a range of registers to ** use for the old.* references in the triggers. */ if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ @@ -782,7 +792,7 @@ void sqlite3GenerateRowDelete( iOld = pParse->nMem+1; pParse->nMem += (1 + pTab->nCol); - /* Populate the OLD.* pseudo-table register array. These values will be + /* Populate the OLD.* pseudo-table register array. These values will be ** used by any BEFORE and AFTER triggers that exist. */ sqlite3VdbeAddOp2(v, OP_Copy, iPk, iOld); for(iCol=0; iColnCol; iCol++){ @@ -796,11 +806,11 @@ void sqlite3GenerateRowDelete( /* Invoke BEFORE DELETE trigger programs. */ addrStart = sqlite3VdbeCurrentAddr(v); - sqlite3CodeRowTrigger(pParse, pTrigger, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel ); - /* If any BEFORE triggers were coded, then seek the cursor to the + /* If any BEFORE triggers were coded, then seek the cursor to the ** row to be deleted again. It may be that the BEFORE triggers moved ** the cursor or already deleted the row that the cursor was ** pointing to. @@ -817,21 +827,21 @@ void sqlite3GenerateRowDelete( } /* Do FK processing. This call checks that any FK constraints that - ** refer to this table (i.e. constraints attached to other tables) + ** refer to this table (i.e. constraints attached to other tables) ** are not violated by deleting this row. */ sqlite3FkCheck(pParse, pTab, iOld, 0, 0, 0); } /* Delete the index and table entries. Skip this step if pTab is really ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). + ** fire the INSTEAD OF triggers). ** ** If variable 'count' is non-zero, then this OP_Delete instruction should ** invoke the update-hook. The pre-update-hook, on the other hand should ** be invoked unless table pTab is a system table. The difference is that - ** the update-hook is not invoked for rows removed by REPLACE, but the + ** the update-hook is not invoked for rows removed by REPLACE, but the ** pre-update-hook is. - */ + */ if( !IsView(pTab) ){ u8 p5 = 0; sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); @@ -851,16 +861,18 @@ void sqlite3GenerateRowDelete( /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key - ** to the row just deleted. */ + ** to the row just deleted. */ sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); /* Invoke AFTER DELETE trigger programs. */ - sqlite3CodeRowTrigger(pParse, pTrigger, - TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel - ); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + } /* Jump here if the row had already been deleted before any BEFORE - ** trigger programs were invoked. Or if a trigger program throws a + ** trigger programs were invoked. Or if a trigger program throws a ** RAISE(IGNORE) exception. */ sqlite3VdbeResolveLabel(v, iLabel); VdbeModuleComment((v, "END: GenRowDel()")); @@ -945,7 +957,7 @@ void sqlite3GenerateRowIndexDelete( ** its key into the same sequence of registers and if pPrior and pIdx share ** a column in common, then the register corresponding to that column already ** holds the correct value and the loading of that register is skipped. -** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK ** on a table with multiple indices, and especially with the ROWID or ** PRIMARY KEY columns of the index. */ @@ -968,7 +980,7 @@ int sqlite3GenerateIndexKey( if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(pParse); pParse->iSelfTab = iDataCur + 1; - sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); pParse->iSelfTab = 0; pPrior = 0; /* Ticket a9efb42811fa41ee 2019-11-02; diff --git a/src/expr.c b/src/expr.c index a81b4595bb..fdc05366cf 100644 --- a/src/expr.c +++ b/src/expr.c @@ -30,7 +30,7 @@ char sqlite3TableColumnAffinity(const Table *pTab, int iCol){ ** Return the 'affinity' of the expression pExpr if any. ** ** If pExpr is a column, a reference to a column via an 'AS' alias, -** or a sub-select with a column as the return value, then the +** or a sub-select with a column as the return value, then the ** affinity of that column is returned. Otherwise, 0x00 is returned, ** indicating no affinity for the expression. ** @@ -67,12 +67,15 @@ char sqlite3ExprAffinity(const Expr *pExpr){ if( op==TK_SELECT_COLUMN ){ assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iColumn >= 0 ); assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); return sqlite3ExprAffinity( pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr ); } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && pExpr->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(pExpr) ); return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); } @@ -84,7 +87,9 @@ char sqlite3ExprAffinity(const Expr *pExpr){ op = pExpr->op; continue; } - if( op!=TK_REGISTER || (op = pExpr->op2)==TK_REGISTER ) break; + if( op!=TK_REGISTER ) break; + op = pExpr->op2; + if( NEVER( op==TK_REGISTER ) ) break; } return pExpr->affExpr; } @@ -201,7 +206,7 @@ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; - } + } return pExpr; } @@ -217,11 +222,12 @@ Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } - } + } return pExpr; } @@ -246,7 +252,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ while( p ){ int op = p->op; if( op==TK_REGISTER ) op = p->op2; - if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) || op==TK_COLUMN || op==TK_TRIGGER ){ int j; @@ -262,7 +268,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ p = p->pLeft; continue; } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && p->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(p) ); p = p->x.pList->a[0].pExpr; continue; @@ -294,7 +302,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ break; } } - if( sqlite3CheckCollSeq(pParse, pColl) ){ + if( sqlite3CheckCollSeq(pParse, pColl) ){ pColl = 0; } return pColl; @@ -303,7 +311,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ /* ** Return the collation sequence for the expression pExpr. If ** there is no defined collating sequence, return a pointer to the -** defautl collation sequence. +** default collation sequence. ** ** See also: sqlite3ExprCollSeq() ** @@ -414,8 +422,8 @@ static u8 binaryCompareP5( ** it is not considered. */ CollSeq *sqlite3BinaryCompareCollSeq( - Parse *pParse, - const Expr *pLeft, + Parse *pParse, + const Expr *pLeft, const Expr *pRight ){ CollSeq *pColl; @@ -433,7 +441,7 @@ CollSeq *sqlite3BinaryCompareCollSeq( return pColl; } -/* Expresssion p is a comparison operator. Return a collation sequence +/* Expression p is a comparison operator. Return a collation sequence ** appropriate for the comparison operator. ** ** This is normally just a wrapper around sqlite3BinaryCompareCollSeq(). @@ -475,7 +483,7 @@ static int codeCompare( p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, (void*)p4, P4_COLLSEQ); - sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + sqlite3VdbeChangeP5(pParse->pVdbe, (u16)p5); return addr; } @@ -493,7 +501,7 @@ int sqlite3ExprIsVector(const Expr *pExpr){ } /* -** If the expression passed as the only argument is of type TK_VECTOR +** If the expression passed as the only argument is of type TK_VECTOR ** return the number of expressions in the vector. Or, if the expression ** is a sub-select, return the number of columns in the sub-select. For ** any other type of expression, return 1. @@ -547,7 +555,7 @@ Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ ** sqlite3ExprCode() will generate all necessary code to compute ** the iField-th column of the vector expression pVector. ** -** It is ok for pVector to be a scalar (as long as iField==0). +** It is ok for pVector to be a scalar (as long as iField==0). ** In that case, this routine works like sqlite3ExprDup(). ** ** The caller owns the returned Expr object and is responsible for @@ -590,6 +598,7 @@ Expr *sqlite3ExprForVectorField( */ pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); if( pRet ){ + ExprSetProperty(pRet, EP_FullSize); pRet->iTable = nField; pRet->iColumn = iField; pRet->pLeft = pVector; @@ -613,7 +622,7 @@ Expr *sqlite3ExprForVectorField( /* ** If expression pExpr is of type TK_SELECT, generate code to evaluate -** it. Return the register in which the result is stored (or, if the +** it. Return the register in which the result is stored (or, if the ** sub-select returns more than one column, the first in an array ** of registers in which the result is stored). ** @@ -635,10 +644,10 @@ static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ ** the register number of a register that contains the value of ** element iField of the vector. ** -** If pVector is a TK_SELECT expression, then code for it must have +** If pVector is a TK_SELECT expression, then code for it must have ** already been generated using the exprCodeSubselect() routine. In this ** case parameter regSelect should be the first in an array of registers -** containing the results of the sub-select. +** containing the results of the sub-select. ** ** If pVector is of type TK_VECTOR, then code for the requested field ** is generated. In this case (*pRegFree) may be set to the number of @@ -710,10 +719,10 @@ static void codeVectorCompare( sqlite3ErrorMsg(pParse, "row value misused"); return; } - assert( pExpr->op==TK_EQ || pExpr->op==TK_NE - || pExpr->op==TK_IS || pExpr->op==TK_ISNOT - || pExpr->op==TK_LT || pExpr->op==TK_GT - || pExpr->op==TK_LE || pExpr->op==TK_GE + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE ); assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) || (pExpr->op==TK_ISNOT && op==TK_NE) ); @@ -730,7 +739,7 @@ static void codeVectorCompare( sqlite3VdbeAddOp2(v, OP_Integer, 1, dest); for(i=0; 1 /*Loop exits by "break"*/; i++){ int regFree1 = 0, regFree2 = 0; - Expr *pL = 0, *pR = 0; + Expr *pL = 0, *pR = 0; int r1, r2; assert( i>=0 && idb->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; if( nHeight>mxHeight ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "Expression tree is too large (maximum depth %d)", mxHeight ); rc = SQLITE_ERROR; @@ -829,10 +838,10 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){ } /* -** Set the Expr.nHeight variable in the structure passed as an -** argument. An expression with no children, Expr.pList or +** Set the Expr.nHeight variable in the structure passed as an +** argument. An expression with no children, Expr.pList or ** Expr.pSelect member has a height of 1. Any other expression -** has a height equal to the maximum height of any other +** has a height equal to the maximum height of any other ** referenced Expr plus one. ** ** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, @@ -858,7 +867,7 @@ static void exprSetHeight(Expr *p){ ** leave an error in pParse. ** ** Also propagate all EP_Propagate flags from the Expr.x.pList into -** Expr.flags. +** Expr.flags. */ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ if( pParse->nErr ) return; @@ -878,7 +887,7 @@ int sqlite3SelectExprHeight(const Select *p){ #else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ /* ** Propagate all EP_Propagate flags from the Expr.x.pList into -** Expr.flags. +** Expr.flags. */ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ if( pParse->nErr ) return; @@ -889,6 +898,15 @@ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ #define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ +/* +** Set the error offset for an Expr node, if possible. +*/ +void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ + if( pExpr==0 ) return; + if( NEVER(ExprUseWJoin(pExpr)) ) return; + pExpr->w.iOfst = iOfst; +} + /* ** This routine is the core allocator for Expr nodes. ** @@ -903,11 +921,12 @@ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ Expr *sqlite3ExprAlloc( sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ @@ -923,7 +942,7 @@ Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -948,7 +967,7 @@ Expr *sqlite3ExprAlloc( } #if SQLITE_MAX_EXPR_DEPTH>0 pNew->nHeight = 1; -#endif +#endif } return pNew; } @@ -1054,7 +1073,7 @@ void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ /* ** Expression list pEList is a list of vector values. This function ** converts the contents of pEList to a VALUES(...) Select statement -** returning 1 row for each element of the list. For example, the +** returning 1 row for each element of the list. For example, the ** expression list: ** ** ( (1,2), (3,4) (5,6) ) @@ -1085,7 +1104,7 @@ Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprList *pEList){ nExprElem = 1; } if( nExprElem!=nElem ){ - sqlite3ErrorMsg(pParse, "IN(...) element has %d term%s - expected %d", + sqlite3ErrorMsg(pParse, "IN(...) element has %d term%s - expected %d", nExprElem, nExprElem>1?"s":"", nElem ); break; @@ -1125,7 +1144,7 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pLeft; }else{ u32 f = pLeft->flags | pRight->flags; - if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse|EP_HasFunc))==EP_IsFalse && !IN_RENAME_OBJECT ){ sqlite3ExprDeferredDelete(pParse, pLeft); @@ -1157,7 +1176,7 @@ Expr *sqlite3ExprFunction( } assert( !ExprHasProperty(pNew, EP_InnerON|EP_OuterON) ); pNew->w.iOfst = (int)(pToken->z - pParse->zTail); - if( pList + if( pList && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] && !pParse->nested ){ @@ -1171,6 +1190,72 @@ Expr *sqlite3ExprFunction( return pNew; } +/* +** Report an error when attempting to use an ORDER BY clause within +** the arguments of a non-aggregate function. +*/ +void sqlite3ExprOrderByAggregateError(Parse *pParse, Expr *p){ + sqlite3ErrorMsg(pParse, + "ORDER BY may not be used with non-aggregate %#T()", p + ); +} + +/* +** Attach an ORDER BY clause to a function call. +** +** functionname( arguments ORDER BY sortlist ) +** \_____________________/ \______/ +** pExpr pOrderBy +** +** The ORDER BY clause is inserted into a new Expr node of type TK_ORDER +** and added to the Expr.pLeft field of the parent TK_FUNCTION node. +*/ +void sqlite3ExprAddFunctionOrderBy( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The function call to which ORDER BY is to be added */ + ExprList *pOrderBy /* The ORDER BY clause to add */ +){ + Expr *pOB; + sqlite3 *db = pParse->db; + if( NEVER(pOrderBy==0) ){ + assert( db->mallocFailed ); + return; + } + if( pExpr==0 ){ + assert( db->mallocFailed ); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + assert( pExpr->op==TK_FUNCTION ); + assert( pExpr->pLeft==0 ); + assert( ExprUseXList(pExpr) ); + if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){ + /* Ignore ORDER BY on zero-argument aggregates */ + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy); + return; + } + if( IsWindowFunc(pExpr) ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + + pOB = sqlite3ExprAlloc(db, TK_ORDER, 0, 0); + if( pOB==0 ){ + sqlite3ExprListDelete(db, pOrderBy); + return; + } + pOB->x.pList = pOrderBy; + assert( ExprUseXList(pOB) ); + pExpr->pLeft = pOB; + ExprSetProperty(pOB, EP_FullSize); +} + /* ** Check to see if a function is usable according to current access ** rules: @@ -1196,7 +1281,7 @@ void sqlite3ExprFunctionUsable( /* Functions prohibited in triggers and views if: ** (1) tagged with SQLITE_DIRECTONLY ** (2) not tagged with SQLITE_INNOCUOUS (which means it - ** is tagged with SQLITE_FUNC_UNSAFE) and + ** is tagged with SQLITE_FUNC_UNSAFE) and ** SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning ** that the schema is possibly tainted). */ @@ -1207,7 +1292,7 @@ void sqlite3ExprFunctionUsable( /* ** Assign a variable number to an expression that encodes a wildcard -** in the original SQL statement. +** in the original SQL statement. ** ** Wildcards consisting of a single "?" are assigned the next sequential ** variable number. @@ -1294,6 +1379,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); assert( db!=0 ); +exprDeleteRestart: assert( !ExprUseUValue(p) || p->u.iValue>=0 ); assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); @@ -1309,7 +1395,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); - if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); @@ -1324,6 +1409,19 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ } #endif } + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } + } } if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbNNFreeNN(db, p); @@ -1332,6 +1430,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } +void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){ + if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p); +} /* ** Clear both elements of an OnOrUsing object @@ -1349,17 +1450,15 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ /* ** Arrange to cause pExpr to be deleted when the pParse is deleted. ** This is similar to sqlite3ExprDelete() except that the delete is -** deferred untilthe pParse is deleted. +** deferred until the pParse is deleted. ** ** The pExpr might be deleted immediately on an OOM error. ** -** The deferred delete is (currently) implemented by adding the -** pExpr to the pParse->pConstExpr list with a register number of 0. +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. */ -void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprDelete, - pExpr); +int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -1375,7 +1474,7 @@ void sqlite3ExprUnmapAndDelete(Parse *pParse, Expr *p){ } /* -** Return the number of bytes allocated for the expression structure +** Return the number of bytes allocated for the expression structure ** passed as the first argument. This is always one of EXPR_FULLSIZE, ** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. */ @@ -1390,14 +1489,14 @@ static int exprStructSize(const Expr *p){ ** to store a copy of an expression or expression tree. They differ in ** how much of the tree is measured. ** -** dupedExprStructSize() Size of only the Expr structure +** dupedExprStructSize() Size of only the Expr structure ** dupedExprNodeSize() Size of Expr + space for token ** dupedExprSize() Expr + token + subtree components ** *************************************************************************** ** -** The dupedExprStructSize() function returns two values OR-ed together: -** (1) the space required for a copy of the Expr structure only and +** The dupedExprStructSize() function returns two values OR-ed together: +** (1) the space required for a copy of the Expr structure only and ** (2) the EP_xxx flags that indicate what the structure size should be. ** The return values is always one of: ** @@ -1424,15 +1523,11 @@ static int dupedExprStructSize(const Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN -#ifndef SQLITE_OMIT_WINDOWFUNC - || ExprHasProperty(p, EP_WinFunc) -#endif - ){ + if( 0==flags || ExprHasProperty(p, EP_FullSize) ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); - assert( !ExprHasProperty(p, EP_OuterON) ); + assert( !ExprHasProperty(p, EP_OuterON) ); assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; @@ -1445,7 +1540,7 @@ static int dupedExprStructSize(const Expr *p, int flags){ } /* -** This function returns the space in bytes required to store the copy +** This function returns the space in bytes required to store the copy ** of the Expr structure and a copy of the Expr.u.zToken string (if that ** string is defined.) */ @@ -1458,57 +1553,94 @@ static int dupedExprNodeSize(const Expr *p, int flags){ } /* -** Return the number of bytes required to create a duplicate of the -** expression passed as the first argument. The second argument is a -** mask containing EXPRDUP_XXX flags. +** Return the number of bytes required to create a duplicate of the +** expression passed as the first argument. ** ** The value returned includes space to create a copy of the Expr struct ** itself and the buffer referred to by Expr.u.zToken, if any. ** -** If the EXPRDUP_REDUCE flag is set, then the return value includes -** space to duplicate all Expr nodes in the tree formed by Expr.pLeft -** and Expr.pRight variables (but not for any structures pointed to or -** descended from the Expr.x.pList or Expr.x.pSelect variables). +** The return value includes space to duplicate all Expr nodes in the +** tree formed by Expr.pLeft and Expr.pRight, but not any other +** substructure such as Expr.x.pList, Expr.x.pSelect, and Expr.y.pWin. */ -static int dupedExprSize(const Expr *p, int flags){ - int nByte = 0; - if( p ){ - nByte = dupedExprNodeSize(p, flags); - if( flags&EXPRDUP_REDUCE ){ - nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); - } - } +static int dupedExprSize(const Expr *p){ + int nByte; + assert( p!=0 ); + nByte = dupedExprNodeSize(p, EXPRDUP_REDUCE); + if( p->pLeft ) nByte += dupedExprSize(p->pLeft); + if( p->pRight ) nByte += dupedExprSize(p->pRight); + assert( nByte==ROUND8(nByte) ); return nByte; } /* -** This function is similar to sqlite3ExprDup(), except that if pzBuffer -** is not NULL then *pzBuffer is assumed to point to a buffer large enough -** to store the copy of expression p, the copies of p->u.zToken -** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte past the -** portion of the buffer copied into by this function. +** An EdupBuf is a memory allocation used to stored multiple Expr objects +** together with their Expr.zToken content. This is used to help implement +** compression while doing sqlite3ExprDup(). The top-level Expr does the +** allocation for itself and many of its decendents, then passes an instance +** of the structure down into exprDup() so that they decendents can have +** access to that memory. */ -static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ +typedef struct EdupBuf EdupBuf; +struct EdupBuf { + u8 *zAlloc; /* Memory space available for storage */ +#ifdef SQLITE_DEBUG + u8 *zEnd; /* First byte past the end of memory */ +#endif +}; + +/* +** This function is similar to sqlite3ExprDup(), except that if pEdupBuf +** is not NULL then it points to memory that can be used to store a copy +** of the input Expr p together with its p->u.zToken (if any). pEdupBuf +** is updated with the new buffer tail prior to returning. +*/ +static Expr *exprDup( + sqlite3 *db, /* Database connection (for memory allocation) */ + const Expr *p, /* Expr tree to be duplicated */ + int dupFlags, /* EXPRDUP_REDUCE for compression. 0 if not */ + EdupBuf *pEdupBuf /* Preallocated storage space, or NULL */ +){ Expr *pNew; /* Value to return */ - u8 *zAlloc; /* Memory space from which to build Expr object */ + EdupBuf sEdupBuf; /* Memory space from which to build Expr object */ u32 staticFlag; /* EP_Static if space not obtained from malloc */ + int nToken = -1; /* Space needed for p->u.zToken. -1 means unknown */ assert( db!=0 ); assert( p ); assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pEdupBuf==0 || dupFlags==EXPRDUP_REDUCE ); /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; + if( pEdupBuf ){ + sEdupBuf.zAlloc = pEdupBuf->zAlloc; +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = pEdupBuf->zEnd; +#endif staticFlag = EP_Static; - assert( zAlloc!=0 ); + assert( sEdupBuf.zAlloc!=0 ); + assert( dupFlags==EXPRDUP_REDUCE ); }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + int nAlloc; + if( dupFlags ){ + nAlloc = dupedExprSize(p); + }else if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30NN(p->u.zToken)+1; + nAlloc = ROUND8(EXPR_FULLSIZE + nToken); + }else{ + nToken = 0; + nAlloc = ROUND8(EXPR_FULLSIZE); + } + assert( nAlloc==ROUND8(nAlloc) ); + sEdupBuf.zAlloc = sqlite3DbMallocRawNN(db, nAlloc); +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = sEdupBuf.zAlloc ? sEdupBuf.zAlloc+nAlloc : 0; +#endif + staticFlag = 0; } - pNew = (Expr *)zAlloc; + pNew = (Expr *)sEdupBuf.zAlloc; + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); if( pNew ){ /* Set nNewSize to the size allocated for the structure pointed to @@ -1517,22 +1649,27 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ ** by the copy of the p->u.zToken string (if any). */ const unsigned nStructSize = dupedExprStructSize(p, dupFlags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; - }else{ - nToken = 0; + int nNewSize = nStructSize & 0xfff; + if( nToken<0 ){ + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } } if( dupFlags ){ + assert( (int)(sEdupBuf.zEnd - sEdupBuf.zAlloc) >= nNewSize+nToken ); assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); + memcpy(sEdupBuf.zAlloc, p, nNewSize); }else{ u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); - if( nSize= + (int)EXPR_FULLSIZE+nToken ); + memcpy(sEdupBuf.zAlloc, p, nSize); + if( nSizeu.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + assert( nToken>=0 ); + if( nToken>0 ){ + char *zToken = pNew->u.zToken = (char*)&sEdupBuf.zAlloc[nNewSize]; memcpy(zToken, p->u.zToken, nToken); + nNewSize += nToken; } + sEdupBuf.zAlloc += ROUND8(nNewSize); + + if( ((p->flags|pNew->flags)&(EP_TokenOnly|EP_Leaf))==0 ){ - if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ if( ExprUseXSelect(p) ){ pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, + p->op!=TK_ORDER ? dupFlags : 0); } - } - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly|EP_WinFunc) ){ - zAlloc += dupedExprNodeSize(p, dupFlags); - if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ - pNew->pLeft = p->pLeft ? - exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; - pNew->pRight = p->pRight ? - exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; - } #ifndef SQLITE_OMIT_WINDOWFUNC if( ExprHasProperty(p, EP_WinFunc) ){ pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); assert( ExprHasProperty(pNew, EP_WinFunc) ); } #endif /* SQLITE_OMIT_WINDOWFUNC */ - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ - if( pNew->op==TK_SELECT_COLUMN ){ + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( dupFlags ){ + if( p->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); + }else{ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &sEdupBuf) : 0; + } + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &sEdupBuf) : 0; + }else{ + if( p->op==TK_SELECT_COLUMN ){ pNew->pLeft = p->pLeft; - assert( p->pRight==0 || p->pRight==p->pLeft - || ExprHasProperty(p->pLeft, EP_Subquery) ); + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); }else{ pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); } @@ -1590,11 +1733,13 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ } } } + if( pEdupBuf ) memcpy(pEdupBuf, &sEdupBuf, sizeof(sEdupBuf)); + assert( sEdupBuf.zAlloc <= sEdupBuf.zEnd ); return pNew; } /* -** Create and return a deep copy of the object passed as the second +** Create and return a deep copy of the object passed as the second ** argument. If an OOM condition is encountered, NULL is returned ** and the db->mallocFailed flag set. */ @@ -1602,7 +1747,7 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ With *sqlite3WithDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = SZ_WITH(p->nCte); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -1661,7 +1806,7 @@ static void gatherSelectWindows(Select *p){ ** without effecting the originals. ** ** The expression list, ID, and source lists return by sqlite3ExprListDup(), -** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded ** by subsequent calls to sqlite*ListAppend() routines. ** ** Any tables that the SrcList might point to are not duplicated. @@ -1694,9 +1839,9 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ Expr *pOldExpr = pOldItem->pExpr; Expr *pNewExpr; pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); - if( pOldExpr + if( pOldExpr && pOldExpr->op==TK_SELECT_COLUMN - && (pNewExpr = pItem->pExpr)!=0 + && (pNewExpr = pItem->pExpr)!=0 ){ if( pNewExpr->pRight ){ pPriorSelectColOld = pOldExpr->pRight; @@ -1713,7 +1858,6 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ } pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); pItem->fg = pOldItem->fg; - pItem->fg.done = 0; pItem->u = pOldItem->u; } return pNew; @@ -1721,7 +1865,7 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ /* ** If cursors, triggers, views and subqueries are all omitted from -** the build, then none of the following routines, except for +** the build, then none of the following routines, except for ** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes ** called with a NULL argument. */ @@ -1730,41 +1874,55 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcList *pNew; int i; - int nByte; assert( db!=0 ); if( p==0 ) return 0; - nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRawNN(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, SZ_SRCLIST(p->nSrc) ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ SrcItem *pNewItem = &pNew->a[i]; const SrcItem *pOldItem = &p->a[i]; Table *pTab; - pNewItem->pSchema = pOldItem->pSchema; - pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->fg = pOldItem->fg; + if( pOldItem->fg.isSubquery ){ + Subquery *pNewSubq = sqlite3DbMallocRaw(db, sizeof(Subquery)); + if( pNewSubq==0 ){ + assert( db->mallocFailed ); + pNewItem->fg.isSubquery = 0; + }else{ + memcpy(pNewSubq, pOldItem->u4.pSubq, sizeof(*pNewSubq)); + pNewSubq->pSelect = sqlite3SelectDup(db, pNewSubq->pSelect, flags); + if( pNewSubq->pSelect==0 ){ + sqlite3DbFree(db, pNewSubq); + pNewSubq = 0; + pNewItem->fg.isSubquery = 0; + } + } + pNewItem->u4.pSubq = pNewSubq; + }else if( pOldItem->fg.fixedSchema ){ + pNewItem->u4.pSchema = pOldItem->u4.pSchema; + }else{ + pNewItem->u4.zDatabase = sqlite3DbStrDup(db, pOldItem->u4.zDatabase); + } pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; - pNewItem->addrFillSub = pOldItem->addrFillSub; - pNewItem->regReturn = pOldItem->regReturn; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } pNewItem->u2 = pOldItem->u2; if( pNewItem->fg.isCte ){ pNewItem->u2.pCteUse->nUse++; } - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = - sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); - } - pTab = pNewItem->pTab = pOldItem->pTab; + pTab = pNewItem->pSTab = pOldItem->pSTab; if( pTab ){ pTab->nTabRef++; } - pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); if( pOldItem->fg.isUsing ){ assert( pNewItem->fg.isUsing ); pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); @@ -1780,16 +1938,13 @@ IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ int i; assert( db!=0 ); if( p==0 ) return 0; - assert( p->eU4!=EU4_EXPR ); - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) ); + pNew = sqlite3DbMallocRawNN(db, SZ_IDLIST(p->nId)); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->eU4 = p->eU4; for(i=0; inId; i++){ struct IdList_item *pNewItem = &pNew->a[i]; const struct IdList_item *pOldItem = &p->a[i]; pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pNewItem->u4 = pOldItem->u4; } return pNew; } @@ -1815,7 +1970,7 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; @@ -1838,7 +1993,6 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pp = &pNew->pPrior; pNext = pNew; } - return pRet; } #else @@ -1854,11 +2008,7 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *p, int flags){ ** initially NULL, then create a new expression list. ** ** The pList argument must be either NULL or a pointer to an ExprList -** obtained from a prior call to sqlite3ExprListAppend(). This routine -** may not be used with an ExprList obtained from sqlite3ExprListDup(). -** Reason: This routine assumes that the number of slots in pList->a[] -** is a power of two. That is true for sqlite3ExprListAppend() returns -** but is not necessarily true from the return value of sqlite3ExprListDup(). +** obtained from a prior call to sqlite3ExprListAppend(). ** ** If a memory allocation error occurs, the entire list is freed and ** NULL is returned. If non-NULL is returned, then it is guaranteed @@ -1872,7 +2022,7 @@ SQLITE_NOINLINE ExprList *sqlite3ExprListAppendNew( struct ExprList_item *pItem; ExprList *pList; - pList = sqlite3DbMallocRawNN(db, sizeof(ExprList)+sizeof(pList->a[0])*4 ); + pList = sqlite3DbMallocRawNN(db, SZ_EXPRLIST(4)); if( pList==0 ){ sqlite3ExprDelete(db, pExpr); return 0; @@ -1892,8 +2042,7 @@ SQLITE_NOINLINE ExprList *sqlite3ExprListAppendGrow( struct ExprList_item *pItem; ExprList *pNew; pList->nAlloc *= 2; - pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(pList->nAlloc-1)*sizeof(pList->a[0])); + pNew = sqlite3DbRealloc(db, pList, SZ_EXPRLIST(pList->nAlloc)); if( pNew==0 ){ sqlite3ExprListDelete(db, pList); sqlite3ExprDelete(db, pExpr); @@ -1950,8 +2099,8 @@ ExprList *sqlite3ExprListAppendVector( if( NEVER(pColumns==0) ) goto vector_append_error; if( pExpr==0 ) goto vector_append_error; - /* If the RHS is a vector, then we can immediately check to see that - ** the size of the RHS and LHS match. But if the RHS is a SELECT, + /* If the RHS is a vector, then we can immediately check to see that + ** the size of the RHS and LHS match. But if the RHS is a SELECT, ** wildcards ("*") in the result set of the SELECT must be expanded before ** we can do the size check, so defer the size check until code generation. */ @@ -1977,7 +2126,7 @@ ExprList *sqlite3ExprListAppendVector( Expr *pFirst = pList->a[iFirst].pExpr; assert( pFirst!=0 ); assert( pFirst->op==TK_SELECT_COLUMN ); - + /* Store the SELECT statement in pRight so it will be deleted when ** sqlite3ExprListDelete() is called */ pFirst->pRight = pExpr; @@ -2003,13 +2152,13 @@ void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){ assert( p->nExpr>0 ); assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC==0 && SQLITE_SO_DESC>0 ); - assert( iSortOrder==SQLITE_SO_UNDEFINED - || iSortOrder==SQLITE_SO_ASC - || iSortOrder==SQLITE_SO_DESC + assert( iSortOrder==SQLITE_SO_UNDEFINED + || iSortOrder==SQLITE_SO_ASC + || iSortOrder==SQLITE_SO_DESC ); - assert( eNulls==SQLITE_SO_UNDEFINED - || eNulls==SQLITE_SO_ASC - || eNulls==SQLITE_SO_DESC + assert( eNulls==SQLITE_SO_UNDEFINED + || eNulls==SQLITE_SO_ASC + || eNulls==SQLITE_SO_DESC ); pItem = &p->a[p->nExpr-1]; @@ -2123,6 +2272,9 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); } +void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){ + if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList); +} /* ** Return the bitwise-OR of all Expr.flags fields in the given @@ -2191,7 +2343,7 @@ int sqlite3ExprIdToTrueFalse(Expr *pExpr){ ** and 0 if it is FALSE. */ int sqlite3ExprTruthValue(const Expr *pExpr){ - pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); + pExpr = sqlite3ExprSkipCollateAndLikely((Expr*)pExpr); assert( pExpr->op==TK_TRUEFALSE ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 @@ -2226,6 +2378,133 @@ Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ return pExpr; } +/* +** Return true if it might be advantageous to compute the right operand +** of expression pExpr first, before the left operand. +** +** Normally the left operand is computed before the right operand. But if +** the left operand contains a subquery and the right does not, then it +** might be more efficient to compute the right operand first. +*/ +static int exprEvalRhsFirst(Expr *pExpr){ + if( ExprHasProperty(pExpr->pLeft, EP_Subquery) + && !ExprHasProperty(pExpr->pRight, EP_Subquery) + ){ + return 1; + }else{ + return 0; + } +} + +/* +** Compute the two operands of a binary operator. +** +** If either operand contains a subquery, then the code strives to +** compute the operand containing the subquery second. If the other +** operand evalutes to NULL, then a jump is made. The address of the +** IsNull operand that does this jump is returned. The caller can use +** this to optimize the computation so as to avoid doing the potentially +** expensive subquery. +** +** If no optimization opportunities exist, return 0. +*/ +static int exprComputeOperands( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The comparison expression */ + int *pR1, /* OUT: Register holding the left operand */ + int *pR2, /* OUT: Register holding the right operand */ + int *pFree1, /* OUT: Temp register to free if not zero */ + int *pFree2 /* OUT: Another temp register to free if not zero */ +){ + int addrIsNull; + int r1, r2; + Vdbe *v = pParse->pVdbe; + + assert( v!=0 ); + /* + ** If the left operand contains a (possibly expensive) subquery and the + ** right operand does not and the right operation might be NULL, + ** then compute the right operand first and do an IsNull jump if the + ** right operand evalutes to NULL. + */ + if( exprEvalRhsFirst(pExpr) && sqlite3ExprCanBeNull(pExpr->pRight) ){ + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + }else{ + r2 = 0; /* Silence a false-positive uninit-var warning in MSVC */ + addrIsNull = 0; + } + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pFree1); + if( addrIsNull==0 ){ + /* + ** If the right operand contains a subquery and the left operand does not + ** and the left operand might be NULL, then do an IsNull check + ** check on the left operand before computing the right operand. + */ + if( ExprHasProperty(pExpr->pRight, EP_Subquery) + && sqlite3ExprCanBeNull(pExpr->pLeft) + ){ + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pFree2); + } + *pR1 = r1; + *pR2 = r2; + return addrIsNull; +} + +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -2248,12 +2527,13 @@ Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ ** when parsing an existing schema out of the sqlite_schema table and 4 ** when processing a new CREATE TABLE statement. A bound parameter raises ** an error for new statements, but is silently converted -** to NULL for existing schemas. This allows sqlite_schema tables that +** to NULL for existing schemas. This allows sqlite_schema tables that ** contain a bound parameter because they were generated by older versions ** of SQLite to be parsed by newer versions of SQLite without raising a ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from ** the ON or USING clauses of an outer join disqualifies the expression @@ -2273,6 +2553,8 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ){ if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; @@ -2301,9 +2583,11 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_IF_NULL_ROW: case TK_REGISTER: case TK_DOT: + case TK_RAISE: testcase( pExpr->op==TK_REGISTER ); testcase( pExpr->op==TK_IF_NULL_ROW ); testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); pWalker->eCode = 0; return WRC_Abort; case TK_VARIABLE: @@ -2325,15 +2609,15 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Continue; } } -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = sqlite3SelectWalkFail; #ifdef SQLITE_DEBUG w.xSelectCallback2 = sqlite3SelectWalkAssert2; #endif - w.u.iCur = iCur; sqlite3WalkExpr(&w, p); return w.eCode; } @@ -2345,9 +2629,15 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* @@ -2363,8 +2653,24 @@ int sqlite3ExprIsConstant(Expr *p){ ** can be added to the pParse->pConstExpr list and evaluated once when ** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -2372,9 +2678,26 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* @@ -2392,7 +2715,10 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** -** (2) pExpr cannot use subqueries or non-deterministic functions. +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. ** ** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. ** (Is there some way to relax this constraint?) @@ -2401,7 +2727,7 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** (4a) pExpr must come from an ON clause.. ** (4b) and specifically the ON clause associated with the LEFT JOIN. ** -** (5) If pSrc is not the right operand of a LEFT JOIN or the left +** (5) If pSrc is the right operand of a LEFT JOIN or the left ** operand of a RIGHT JOIN, then pExpr must be from the WHERE ** clause, not an ON clause. ** @@ -2421,7 +2747,8 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){ int sqlite3ExprIsSingleTableConstraint( Expr *pExpr, /* The constraint */ const SrcList *pSrcList, /* Complete FROM clause */ - int iSrc /* Which element of pSrcList to use */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ ){ const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ @@ -2446,7 +2773,8 @@ int sqlite3ExprIsSingleTableConstraint( } } } - return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); } @@ -2480,7 +2808,7 @@ static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ /* ** Walk the expression tree passed as the first argument. Return non-zero -** if the expression consists entirely of constants or copies of terms +** if the expression consists entirely of constants or copies of terms ** in pGroupBy that sort with the BINARY collation sequence. ** ** This routine is used to determine if a term of the HAVING clause can @@ -2510,7 +2838,7 @@ int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ /* ** Walk an expression tree for the DEFAULT field of a column definition -** in a CREATE TABLE statement. Return non-zero if the expression is +** in a CREATE TABLE statement. Return non-zero if the expression is ** acceptable for use as a DEFAULT. That is to say, return non-zero if ** the expression is constant or a function call with constant arguments. ** Return and 0 if there are any variables. @@ -2531,7 +2859,7 @@ int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -2557,8 +2885,12 @@ int sqlite3ExprContainsSubquery(Expr *p){ ** to fit in a 32-bit integer, return 1 and put the value of the integer ** in *pValue. If the expression is not an integer or if it is too big ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** +** If the pParse pointer is provided, then allow the expression p to be +** a parameter (TK_VARIABLE) that is bound to an integer. +** But if pParse is NULL, then p must be a pure integer literal. */ -int sqlite3ExprIsInteger(const Expr *p, int *pValue){ +int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){ int rc = 0; if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ @@ -2573,18 +2905,38 @@ int sqlite3ExprIsInteger(const Expr *p, int *pValue){ } switch( p->op ){ case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue); + rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0); break; } case TK_UMINUS: { int v = 0; - if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){ assert( ((unsigned int)v)!=0x80000000 ); *pValue = -v; rc = 1; } break; } + case TK_VARIABLE: { + sqlite3_value *pVal; + if( pParse==0 ) break; + if( NEVER(pParse->pVdbe==0) ) break; + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break; + sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn); + pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn, + SQLITE_AFF_BLOB); + if( pVal ){ + if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){ + sqlite3_int64 vv = sqlite3_value_int64(pVal); + if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */ + *pValue = (int)vv; + rc = 1; + } + } + sqlite3ValueFree(pVal); + } + break; + } default: break; } return rc; @@ -2594,7 +2946,7 @@ int sqlite3ExprIsInteger(const Expr *p, int *pValue){ ** Return FALSE if there is no chance that the expression can be NULL. ** ** If the expression might be NULL or if the expression is too complex -** to tell return TRUE. +** to tell return TRUE. ** ** This routine is used as an optimization, to skip OP_IsNull opcodes ** when we know that a value cannot be NULL. Hence, a false positive @@ -2621,10 +2973,14 @@ int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( ExprUseYTab(p) ); - return ExprHasProperty(p, EP_CanBeNull) || - p->y.pTab==0 || /* Reference to column of index on expression */ - (p->iColumn>=0 + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 && p->y.pTab->aCol!=0 /* Possible due to prior error */ + && ALWAYS(p->iColumny.pTab->nCol) && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; @@ -2685,7 +3041,22 @@ int sqlite3IsRowid(const char *z){ } /* -** pX is the RHS of an IN operator. If pX is a SELECT statement +** Return a pointer to a buffer containing a usable rowid alias for table +** pTab. An alias is usable if there is not an explicit user-defined column +** of the same name. +*/ +const char *sqlite3RowidAlias(Table *pTab){ + const char *azOpt[] = {"_ROWID_", "ROWID", "OID"}; + int ii; + assert( VisibleRowid(pTab) ); + for(ii=0; iipSrc; assert( pSrc!=0 ); if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ - pTab = pSrc->a[0].pTab; + if( pSrc->a[0].fg.isSubquery) return 0;/* FROM is not a subquery or view */ + pTab = pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ @@ -2751,16 +3122,16 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ #ifndef SQLITE_OMIT_SUBQUERY /* -** The argument is an IN operator with a list (not a subquery) on the +** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -2784,7 +3155,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** populated ephemeral table. ** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be ** implemented as a sequence of comparisons. ** @@ -2797,7 +3168,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** an ephemeral table might need to be generated from the RHS and then ** pX->iTable made to point to the ephemeral table instead of an ** existing table. In this case, the creation and initialization of the -** ephmeral table might be put inside of a subroutine, the EP_Subrtn flag +** ephemeral table might be put inside of a subroutine, the EP_Subrtn flag ** will be set on pX and the pX->y.sub fields will be set to show where ** the subroutine is coded. ** @@ -2809,13 +3180,13 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table will be created unless the selected columns are guaranteed +** An ephemeral table will be created unless the selected columns are guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or due to ** a UNIQUE constraint or index. ** -** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used -** for fast set membership tests) then an epheremal table must -** be used unless is a single INTEGER PRIMARY KEY column or an +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an ephemeral table must +** be used unless is a single INTEGER PRIMARY KEY column or an ** index can be found with the specified as its left-most. ** ** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and @@ -2827,7 +3198,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When the b-tree is being used for membership tests, the calling function ** might need to know whether or not the RHS side of the IN operator -** contains a NULL. If prRhsHasNull is not a NULL pointer and +** contains a NULL. If prRhsHasNull is not a NULL pointer and ** if there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written ** to *prRhsHasNull. If there is no chance that the (...) contains a @@ -2868,9 +3239,9 @@ int sqlite3FindInIndex( mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; iTab = pParse->nTab++; - /* If the RHS of this IN(...) operator is a SELECT, and if it matters + /* If the RHS of this IN(...) operator is a SELECT, and if it matters ** whether or not the SELECT result contains NULL values, check whether - ** or not NULL is actually possible (it may not be, for example, due + ** or not NULL is actually possible (it may not be, for example, due ** to NOT NULL constraints in the schema). If no NULL values are possible, ** set prRhsHasNull to 0 before continuing. */ if( prRhsHasNull && ExprUseXSelect(pX) ){ @@ -2885,7 +3256,7 @@ int sqlite3FindInIndex( } /* Check to see if an existing table or index can be used to - ** satisfy the query. This is preferable to generating a new + ** satisfy the query. This is preferable to generating a new ** ephemeral table. */ if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ sqlite3 *db = pParse->db; /* Database connection */ @@ -2897,7 +3268,7 @@ int sqlite3FindInIndex( assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; /* Code an OP_Transaction and OP_TableLock for
    . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -2921,7 +3292,7 @@ int sqlite3FindInIndex( int affinity_ok = 1; int i; - /* Check that the affinity that will be used to perform each + /* Check that the affinity that will be used to perform each ** comparison is the same as the affinity of each column in table ** on the RHS of the IN operator. If it not, it is not possible to ** use any index of the RHS table. */ @@ -2966,14 +3337,14 @@ int sqlite3FindInIndex( continue; /* This index is not unique over the IN RHS columns */ } } - + colUsed = 0; /* Columns of index used so far */ for(i=0; ipLeft, i); Expr *pRhs = pEList->a[i].pExpr; CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); int j; - + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; assert( pIdx->azColl[j] ); @@ -2988,7 +3359,8 @@ int sqlite3FindInIndex( colUsed |= mCol; if( aiMap ) aiMap[i] = j; } - + + assert( nExpr>0 && nExprzName)); assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - + if( prRhsHasNull ){ #ifdef SQLITE_ENABLE_COLUMN_USED_MASK i64 mask = (1<nMem; @@ -3029,7 +3401,7 @@ int sqlite3FindInIndex( if( eType==0 && (inFlags & IN_INDEX_NOOP_OK) && ExprUseXList(pX) - && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ pParse->nTab--; /* Back out the allocation of the unused cursor */ iTab = -1; /* Cursor is not allocated */ @@ -3068,8 +3440,8 @@ int sqlite3FindInIndex( #ifndef SQLITE_OMIT_SUBQUERY /* -** Argument pExpr is an (?, ?...) IN(...) expression. This -** function allocates and returns a nul-terminated string containing +** Argument pExpr is an (?, ?...) IN(...) expression. This +** function allocates and returns a nul-terminated string containing ** the affinities to be used for each column of the comparison. ** ** It is the responsibility of the caller to ensure that the returned @@ -3082,7 +3454,7 @@ static char *exprINAffinity(Parse *pParse, const Expr *pExpr){ char *zRet; assert( pExpr->op==TK_IN ); - zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + zRet = sqlite3DbMallocRaw(pParse->db, 1+(i64)nVal); if( zRet ){ int i; for(i=0; inErr==0 ){ const char *zFmt = "sub-select returns %d columns - expected %d"; @@ -3117,7 +3489,7 @@ void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ /* ** Expression pExpr is a vector that has been used in a context where -** it is not permitted. If pExpr is a sub-select vector, this routine +** it is not permitted. If pExpr is a sub-select vector, this routine ** loads the Parse object with a message of the form: ** ** "sub-select returns N columns - expected 1" @@ -3125,7 +3497,7 @@ void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ ** Or, if it is a regular scalar vector: ** ** "row value misused" -*/ +*/ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ #ifndef SQLITE_OMIT_SUBQUERY if( ExprUseXSelect(pExpr) ){ @@ -3137,6 +3509,50 @@ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ } } +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Scan all previously generated bytecode looking for an OP_BeginSubrtn +** that is compatible with pExpr. If found, add the y.sub values +** to pExpr and return true. If not found, return false. +*/ +static int findCompatibleInRhsSubrtn( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* IN operator with RHS that we want to reuse */ + SubrtnSig *pNewSig /* Signature for the IN operator */ +){ + VdbeOp *pOp, *pEnd; + SubrtnSig *pSig; + Vdbe *v; + + if( pNewSig==0 ) return 0; + if( (pParse->mSubrtnSig & (1<<(pNewSig->selId&7)))==0 ) return 0; + assert( pExpr->op==TK_IN ); + assert( !ExprUseYSub(pExpr) ); + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( (pExpr->x.pSelect->selFlags & SF_All)==0 ); + v = pParse->pVdbe; + assert( v!=0 ); + pOp = sqlite3VdbeGetOp(v, 1); + pEnd = sqlite3VdbeGetLastOp(v); + for(; pOpp4type!=P4_SUBRTNSIG ) continue; + assert( pOp->opcode==OP_BeginSubrtn ); + pSig = pOp->p4.pSubrtnSig; + assert( pSig!=0 ); + if( !pSig->bComplete ) continue; + if( pNewSig->selId!=pSig->selId ) continue; + if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue; + pExpr->y.sub.iAddr = pSig->iAddr; + pExpr->y.sub.regReturn = pSig->regReturn; + pExpr->iTable = pSig->iTable; + ExprSetProperty(pExpr, EP_Subrtn); + return 1; + } + return 0; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that will construct an ephemeral table containing all terms @@ -3147,7 +3563,7 @@ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ ** x IN (SELECT a FROM b) -- IN operator with subquery on the right ** ** The pExpr parameter is the IN operator. The cursor number for the -** constructed ephermeral table is returned. The first time the ephemeral +** constructed ephemeral table is returned. The first time the ephemeral ** table is computed, the cursor number is also stored in pExpr->iTable, ** however the cursor number returned might not be the same, as it might ** have been duplicated using OP_OpenDup. @@ -3171,6 +3587,7 @@ void sqlite3CodeRhsOfIN( KeyInfo *pKeyInfo = 0; /* Key information */ int nVal; /* Size of vector pLeft */ Vdbe *v; /* The prepared statement under construction */ + SubrtnSig *pSig = 0; /* Signature for this subroutine */ v = pParse->pVdbe; assert( v!=0 ); @@ -3186,11 +3603,27 @@ void sqlite3CodeRhsOfIN( ** and reuse it many names. */ if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ - /* Reuse of the RHS is allowed */ - /* If this routine has already been coded, but the previous code - ** might not have been invoked yet, so invoke it now as a subroutine. + /* Reuse of the RHS is allowed + ** + ** Compute a signature for the RHS of the IN operator to facility + ** finding and reusing prior instances of the same IN operator. */ - if( ExprHasProperty(pExpr, EP_Subrtn) ){ + assert( !ExprUseXSelect(pExpr) || pExpr->x.pSelect!=0 ); + if( ExprUseXSelect(pExpr) && (pExpr->x.pSelect->selFlags & SF_All)==0 ){ + pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0])); + if( pSig ){ + pSig->selId = pExpr->x.pSelect->selId; + pSig->zAff = exprINAffinity(pParse, pExpr); + } + } + + /* Check to see if there is a prior materialization of the RHS of + ** this IN operator. If there is, then make use of that prior + ** materialization rather than recomputing it. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) + || findCompatibleInRhsSubrtn(pParse, pExpr, pSig) + ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); if( ExprUseXSelect(pExpr) ){ ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", @@ -3202,6 +3635,10 @@ void sqlite3CodeRhsOfIN( assert( iTab!=pExpr->iTable ); sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); sqlite3VdbeJumpHere(v, addrOnce); + if( pSig ){ + sqlite3DbFree(pParse->db, pSig->zAff); + sqlite3DbFree(pParse->db, pSig); + } return; } @@ -3212,7 +3649,14 @@ void sqlite3CodeRhsOfIN( pExpr->y.sub.regReturn = ++pParse->nMem; pExpr->y.sub.iAddr = sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; - + if( pSig ){ + pSig->bComplete = 0; + pSig->iAddr = pExpr->y.sub.iAddr; + pSig->regReturn = pExpr->y.sub.regReturn; + pSig->iTable = iTab; + pParse->mSubrtnSig = 1 << (pSig->selId&7); + sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG); + } addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } @@ -3253,19 +3697,35 @@ void sqlite3CodeRhsOfIN( SelectDest dest; int i; int rc; + int addrBloom = 0; sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; + if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + int regBloom = ++pParse->nMem; + addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); + VdbeComment((v, "Bloom filter")); + dest.iSDParm2 = regBloom; + } testcase( pSelect->selFlags & SF_Distinct ); testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ pCopy = sqlite3SelectDup(pParse->db, pSelect, 0); rc = pParse->db->mallocFailed ? 1 :sqlite3Select(pParse, pCopy, &dest); sqlite3SelectDelete(pParse->db, pCopy); sqlite3DbFree(pParse->db, dest.zAffSdst); + if( addrBloom ){ + /* Remember that location of the Bloom filter in the P3 operand + ** of the OP_Once that began this subroutine. tag-202407032019 */ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + if( dest.iSDParm2==0 ){ + /* If the Bloom filter won't actually be used, keep it small */ + sqlite3VdbeGetOp(v, addrBloom)->p1 = 10; + } + } if( rc ){ sqlite3KeyInfoUnref(pKeyInfo); return; - } + } assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ assert( pEList!=0 ); assert( pEList->nExpr>0 ); @@ -3312,7 +3772,7 @@ void sqlite3CodeRhsOfIN( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ sqlite3VdbeChangeToNoop(v, addrOnce-1); sqlite3VdbeChangeToNoop(v, addrOnce); ExprClearProperty(pExpr, EP_Subrtn); @@ -3327,6 +3787,7 @@ void sqlite3CodeRhsOfIN( sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempReg(pParse, r2); } + if( pSig ) pSig->bComplete = 1; if( pKeyInfo ){ sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } @@ -3354,7 +3815,7 @@ void sqlite3CodeRhsOfIN( ** ** The pExpr parameter is the SELECT or EXISTS operator to be coded. ** -** Return the register that holds the result. For a multi-column SELECT, +** Return the register that holds the result. For a multi-column SELECT, ** the result is stored in a contiguous array of registers and the ** return value is the register of the left-most result column. ** Return 0 if an error occurs. @@ -3411,7 +3872,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } - + /* For a SELECT, generate code to put the values for all columns of ** the first row into an array of registers and return the index of ** the first register. @@ -3419,7 +3880,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) ** into a register and return that register number. ** - ** In both cases, the query is augmented with "LIMIT 1". Any + ** In both cases, the query is augmented with "LIMIT 1". Any ** preexisting limit is discarded in place of the new LIMIT 1. */ ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", @@ -3430,9 +3891,22 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ pParse->nMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; - dest.iSdst = dest.iSDParm; + if( (pSel->selFlags&SF_Distinct) && pSel->pLimit && pSel->pLimit->pRight ){ + /* If there is both a DISTINCT and an OFFSET clause, then allocate + ** a separate dest.iSdst array for sqlite3Select() and other + ** routines to populate. In this case results will be copied over + ** into the dest.iSDParm array only after OFFSET processing. This + ** ensures that in the case where OFFSET excludes all rows, the + ** dest.iSDParm array is not left populated with the contents of the + ** last row visited - it should be all NULLs if all rows were + ** excluded by OFFSET. */ + dest.iSdst = pParse->nMem+1; + pParse->nMem += nReg; + }else{ + dest.iSdst = dest.iSDParm; + } dest.nSdst = nReg; - sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, pParse->nMem); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; @@ -3440,17 +3914,23 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ VdbeComment((v, "Init EXISTS result")); } if( pSel->pLimit ){ - /* The subquery already has a limit. If the pre-existing limit is X - ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ - sqlite3 *db = pParse->db; - pLimit = sqlite3Expr(db, TK_INTEGER, "0"); - if( pLimit ){ - pLimit->affExpr = SQLITE_AFF_NUMERIC; - pLimit = sqlite3PExpr(pParse, TK_NE, - sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); - } - sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); - pSel->pLimit->pLeft = pLimit; + /* The subquery already has a limit. If the pre-existing limit X is + ** not already integer value 1 or 0, then make the new limit X<>0 so that + ** the new limit is either 1 or 0 */ + Expr *pLeft = pSel->pLimit->pLeft; + if( ExprHasProperty(pLeft, EP_IntValue)==0 + || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) + ){ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pLeft, 0), pLimit); + } + sqlite3ExprDeferredDelete(pParse, pLeft); + pSel->pLimit->pLeft = pLimit; + } }else{ /* If there is no pre-existing limit add a limit of 1 */ pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); @@ -3483,9 +3963,9 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ #ifndef SQLITE_OMIT_SUBQUERY /* -** Expr pIn is an IN(...) expression. This function checks that the -** sub-select on the RHS of the IN() operator has the same number of -** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not ** a sub-query, that the LHS is a vector of size 1. */ int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ @@ -3510,18 +3990,18 @@ int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ ** x IN (SELECT ...) ** x IN (value, value, ...) ** -** The left-hand side (LHS) is a scalar or vector expression. The +** The left-hand side (LHS) is a scalar or vector expression. The ** right-hand side (RHS) is an array of zero or more scalar values, or a ** subquery. If the RHS is a subquery, the number of result columns must ** match the number of columns in the vector on the LHS. If the RHS is -** a list of values, the LHS must be a scalar. +** a list of values, the LHS must be a scalar. ** ** The IN operator is true if the LHS value is contained within the RHS. -** The result is false if the LHS is definitely not in the RHS. The -** result is NULL if the presence of the LHS in the RHS cannot be +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be ** determined due to NULLs. ** -** This routine generates code that jumps to destIfFalse if the LHS is not +** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. @@ -3538,7 +4018,6 @@ static void sqlite3ExprCodeIN( int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ int eType; /* Type of the RHS */ int rLhs; /* Register(s) holding the LHS values */ - int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ int *aiMap = 0; /* Map from vector field to index column */ char *zAff = 0; /* Affinity string for comparisons */ @@ -3550,7 +4029,7 @@ static void sqlite3ExprCodeIN( int destStep6 = 0; /* Start of code for Step 6 */ int addrTruthOp; /* Address of opcode that determines the IN is true */ int destNotNull; /* Jump here if a comparison is not true in step 6 */ - int addrTop; /* Top of the step-6 loop */ + int addrTop; /* Top of the step-6 loop */ int iTab = 0; /* Index to use */ u8 okConstFactor = pParse->okConstFactor; @@ -3559,9 +4038,7 @@ static void sqlite3ExprCodeIN( if( sqlite3ExprCheckIN(pParse, pExpr) ) return; zAff = exprINAffinity(pParse, pExpr); nVector = sqlite3ExprVectorSize(pExpr->pLeft); - aiMap = (int*)sqlite3DbMallocZero( - pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 - ); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, nVector*sizeof(int)); if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; /* Attempt to compute the RHS. After this step, if anything other than @@ -3577,7 +4054,7 @@ static void sqlite3ExprCodeIN( aiMap, &iTab); assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH - || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC ); #ifdef SQLITE_DEBUG /* Confirm that aiMap[] contains nVector integer values between 0 and @@ -3589,8 +4066,8 @@ static void sqlite3ExprCodeIN( } #endif - /* Code the LHS, the from " IN (...)". If the LHS is a - ** vector, then it is stored in an array of nVector registers starting + /* Code the LHS, the from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting ** at r1. ** ** sqlite3FindInIndex() might have reordered the fields of the LHS vector @@ -3603,19 +4080,8 @@ static void sqlite3ExprCodeIN( ** by code generated below. */ assert( pParse->okConstFactor==okConstFactor ); pParse->okConstFactor = 0; - rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + rLhs = exprCodeVector(pParse, pLeft, &iDummy); pParse->okConstFactor = okConstFactor; - for(i=0; ix.pList; pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); @@ -3671,6 +4138,26 @@ static void sqlite3ExprCodeIN( goto sqlite3ExprCodeIN_finished; } + if( eType!=IN_INDEX_ROWID ){ + /* If this IN operator will use an index, then the order of columns in the + ** vector might be different from the order in the index. In that case, + ** we need to reorder the LHS values to be in index order. Run Affinity + ** before reordering the columns, so that the affinity is correct. + */ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + for(i=0; iy.sub.iAddr); + assert( pOp->opcode==OP_Once || pParse->nErr ); + if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + } + } sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, rLhs, nVector); VdbeCoverage(v); goto sqlite3ExprCodeIN_finished; @@ -3722,7 +4218,7 @@ static void sqlite3ExprCodeIN( } /* Step 5. If we do not care about the difference between NULL and - ** FALSE, then just return false. + ** FALSE, then just return false. */ if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); @@ -3770,7 +4266,6 @@ static void sqlite3ExprCodeIN( sqlite3VdbeJumpHere(v, addrTruthOp); sqlite3ExprCodeIN_finished: - if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); VdbeComment((v, "end IN expr")); sqlite3ExprCodeIN_oom_error: sqlite3DbFree(pParse->db, aiMap); @@ -3783,7 +4278,7 @@ static void sqlite3ExprCodeIN( ** Generate an instruction that will put the floating point ** value described by z[0..n-1] into register iMem. ** -** The z[] string will probably not be zero-terminated. But the +** The z[] string will probably not be zero-terminated. But the ** z[n] character is guaranteed to be something that does not look ** like the continuation of the number. */ @@ -3885,7 +4380,12 @@ void sqlite3ExprCodeGeneratedColumn( iAddr = 0; } sqlite3ExprCodeCopy(pParse, sqlite3ColumnExpr(pTab,pCol), regOut); - if( pCol->affinity>=SQLITE_AFF_TEXT ){ + if( (pCol->colFlags & COLFLAG_VIRTUAL)!=0 + && (pTab->tabFlags & TF_Strict)!=0 + ){ + int p3 = 2+(int)(pCol - pTab->aCol); + sqlite3VdbeAddOp4(v, OP_TypeCheck, regOut, 1, p3, (char*)pTab, P4_TABLE); + }else if( pCol->affinity>=SQLITE_AFF_TEXT ){ sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); @@ -3948,7 +4448,7 @@ void sqlite3ExprCodeGetColumnOfTable( /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in register iReg. +** table pTab and store the column value in register iReg. ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -3962,10 +4462,13 @@ int sqlite3ExprCodeGetColumn( u8 p5 /* P5 value for OP_Column + FLAGS */ ){ assert( pParse->pVdbe!=0 ); + assert( (p5 & (OPFLAG_NOCHNG|OPFLAG_TYPEOFARG|OPFLAG_LENGTHARG))==p5 ); + assert( IsVirtual(pTab) || (p5 & OPFLAG_NOCHNG)==0 ); sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); if( pOp->opcode==OP_Column ) pOp->p5 = p5; + if( pOp->opcode==OP_VColumn ) pOp->p5 = (p5 & OPFLAG_NOCHNG); } return iReg; } @@ -3983,18 +4486,22 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ ** register iReg. The caller must ensure that iReg already contains ** the correct value for the expression. */ -static void exprToRegister(Expr *pExpr, int iReg){ +void sqlite3ExprToRegister(Expr *pExpr, int iReg){ Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); if( NEVER(p==0) ) return; - p->op2 = p->op; - p->op = TK_REGISTER; - p->iTable = iReg; - ExprClearProperty(p, EP_Skip); + if( p->op==TK_REGISTER ){ + assert( p->iTable==iReg ); + }else{ + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); + } } /* ** Evaluate an expression (either a vector or a scalar expression) and store -** the result in continguous temporary registers. Return the index of +** the result in contiguous temporary registers. Return the index of ** the first register used to store the result. ** ** If the returned result register is a temporary scalar, then also write @@ -4034,7 +4541,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ */ static void setDoNotMergeFlagOnCopy(Vdbe *v){ if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ - sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergeable */ } } @@ -4091,7 +4598,7 @@ static int exprCodeInlineFunction( break; } #endif - default: { + default: { /* The UNLIKELY() function is a no-op. The result is the value ** of the first argument. */ @@ -4108,7 +4615,7 @@ static int exprCodeInlineFunction( case INLINEFUNC_expr_compare: { /* Compare two expressions using sqlite3ExprCompare() */ assert( nFarg==2 ); - sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprCompare(0,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), target); break; @@ -4117,20 +4624,20 @@ static int exprCodeInlineFunction( case INLINEFUNC_expr_implies_expr: { /* Compare two expressions using sqlite3ExprImpliesExpr() */ assert( nFarg==2 ); - sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprImpliesExpr(pParse,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), target); break; } case INLINEFUNC_implies_nonnull_row: { - /* REsult of sqlite3ExprImpliesNonNullRow() */ + /* Result of sqlite3ExprImpliesNonNullRow() */ Expr *pA1; assert( nFarg==2 ); pA1 = pFarg->a[1].pExpr; if( pA1->op==TK_COLUMN ){ - sqlite3VdbeAddOp2(v, OP_Integer, - sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable), + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable,1), target); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); @@ -4159,6 +4666,59 @@ static int exprCodeInlineFunction( return target; } +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = ALWAYS(pExpr->x.pList) ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + + /* ** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. ** If it is, then resolve the expression by reading from the index and @@ -4191,6 +4751,17 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( continue; } + + /* Functions that might set a subtype should not be replaced by the + ** value taken from an expression index if they are themselves an + ** argument to another scalar function or aggregate. + ** https://sqlite.org/forum/forumpost/68d284c86b082c3e */ + if( ExprHasProperty(pExpr, EP_SubtArg) + && sqlite3ExprCanReturnSubtype(pParse, pExpr) + ){ + continue; + } + v = pParse->pVdbe; assert( v!=0 ); if( p->bMaybeNullRow ){ @@ -4218,6 +4789,115 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( } +/* +** Expression pExpr is guaranteed to be a TK_COLUMN or equivalent. This +** function checks the Parse.pIdxPartExpr list to see if this column +** can be replaced with a constant value. If so, it generates code to +** put the constant value in a register (ideally, but not necessarily, +** register iTarget) and returns the register number. +** +** Or, if the TK_COLUMN cannot be replaced by a constant, zero is +** returned. +*/ +static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ + IndexedExpr *p; + for(p=pParse->pIdxPartExpr; p; p=p->pIENext){ + if( pExpr->iColumn==p->iIdxCol && pExpr->iTable==p->iDataCur ){ + Vdbe *v = pParse->pVdbe; + int addr = 0; + int ret; + + if( p->bMaybeNullRow ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNullRow, p->iIdxCur); + } + ret = sqlite3ExprCodeTarget(pParse, p->pExpr, iTarget); + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Affinity, ret, 1, 0, + (const char*)&p->aff, 1); + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP3(v, addr, ret); + } + return ret; + } + } + return 0; +} + +/* +** Generate code that evaluates an AND or OR operator leaving a +** boolean result in a register. pExpr is the AND/OR expression. +** Store the result in the "target" register. Use short-circuit +** evaluation to avoid computing both operands, if possible. +** +** The code generated might require the use of a temporary register. +** If it does, then write the number of that temporary register +** into *pTmpReg. If not, leave *pTmpReg unchanged. +*/ +static SQLITE_NOINLINE int exprCodeTargetAndOr( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* AND or OR expression to be coded */ + int target, /* Put result in this register, guaranteed */ + int *pTmpReg /* Write a temporary register here */ +){ + int op; /* The opcode. TK_AND or TK_OR */ + int skipOp; /* Opcode for the branch that skips one operand */ + int addrSkip; /* Branch instruction that skips one of the operands */ + int regSS = 0; /* Register holding computed operand when other omitted */ + int r1, r2; /* Registers for left and right operands, respectively */ + Expr *pAlt; /* Alternative, simplified expression */ + Vdbe *v; /* statement being coded */ + + assert( pExpr!=0 ); + op = pExpr->op; + assert( op==TK_AND || op==TK_OR ); + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( pParse->pVdbe!=0 ); + v = pParse->pVdbe; + pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + r1 = sqlite3ExprCodeTarget(pParse, pAlt, target); + sqlite3VdbeAddOp3(v, OP_And, r1, r1, target); + return target; + } + skipOp = op==TK_AND ? OP_IfNot : OP_If; + if( exprEvalRhsFirst(pExpr) ){ + /* Compute the right operand first. Skip the computation of the left + ** operand if the right operand fully determines the result */ + r2 = regSS = sqlite3ExprCodeTarget(pParse, pExpr->pRight, target); + addrSkip = sqlite3VdbeAddOp1(v, skipOp, r2); + VdbeComment((v, "skip left operand")); + VdbeCoverage(v); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, pTmpReg); + }else{ + /* Compute the left operand first */ + r1 = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + if( ExprHasProperty(pExpr->pRight, EP_Subquery) ){ + /* Skip over the computation of the right operand if the right + ** operand is a subquery and the left operand completely determines + ** the result */ + regSS = r1; + addrSkip = sqlite3VdbeAddOp1(v, skipOp, r1); + VdbeComment((v, "skip right operand")); + VdbeCoverage(v); + }else{ + addrSkip = regSS = 0; + } + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, pTmpReg); + } + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( (*pTmpReg)==0 ); + if( addrSkip ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrSkip); + sqlite3VdbeAddOp3(v, OP_Or, regSS, regSS, target); + VdbeComment((v, "short-circut value")); + } + return target; +} + + + /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -4245,7 +4925,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ expr_code_doover: if( pExpr==0 ){ op = TK_NULL; - }else if( pParse->pIdxEpr!=0 + }else if( pParse->pIdxEpr!=0 && !ExprHasProperty(pExpr, EP_Leaf) && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 ){ @@ -4254,6 +4934,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; } + assert( op!=TK_ORDER ); switch( op ){ case TK_AGG_COLUMN: { AggInfo *pAggInfo = pExpr->pAggInfo; @@ -4267,7 +4948,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ #ifdef SQLITE_VDBE_COVERAGE /* Verify that the OP_Null above is exercised by tests ** tag-20230325-2 */ - sqlite3VdbeAddOp2(v, OP_NotNull, target, 1); + sqlite3VdbeAddOp3(v, OP_NotNull, target, 1, 20230325); VdbeCoverageNeverTaken(v); #endif break; @@ -4284,7 +4965,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ }else if( pCol->iColumn<0 ){ VdbeComment((v,"%s.rowid",pTab->zName)); }else{ - VdbeComment((v,"%s.%s", + VdbeComment((v,"%s.%s", pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, target); @@ -4306,7 +4987,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( ExprHasProperty(pExpr, EP_FixedCol) ){ /* This COLUMN expression is really a constant due to WHERE clause ** constraints, and that constant is coded by the pExpr->pLeft - ** expresssion. However, make sure the constant has the correct + ** expression. However, make sure the constant has the correct ** datatype by applying the Affinity of the table column to the ** constant. */ @@ -4375,6 +5056,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ iTab = pParse->iSelfTab - 1; } } + else if( pParse->pIdxPartExpr + && 0!=(r1 = exprPartidxExprLookup(pParse, pExpr, target)) + ){ + return r1; + } assert( ExprUseYTab(pExpr) ); assert( pExpr->y.pTab!=0 ); iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, @@ -4402,6 +5088,12 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeLoadString(v, target, pExpr->u.zToken); return target; } + case TK_NULLS: { + /* Set a range of registers to NULL. pExpr->y.nReg registers starting + ** with target */ + sqlite3VdbeAddOp3(v, OP_Null, 0, target, target + pExpr->y.nReg - 1); + return target; + } default: { /* Make NULL the default case so that if a bug causes an illegal ** Expr node to be passed into this function, it will be handled @@ -4432,12 +5124,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); - pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ - sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); - } return target; } case TK_REGISTER: { @@ -4466,11 +5152,17 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ case TK_NE: case TK_EQ: { Expr *pLeft = pExpr->pLeft; + int addrIsNull = 0; if( sqlite3ExprIsVector(pLeft) ){ codeVectorCompare(pParse, pExpr, target, op, p5); }else{ - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && p5!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + } sqlite3VdbeAddOp2(v, OP_Integer, 1, inReg); codeCompare(pParse, pLeft, pExpr->pRight, op, r1, r2, sqlite3VdbeCurrentAddr(v)+2, p5, @@ -4485,6 +5177,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(v, OP_Integer, 0, inReg); }else{ sqlite3VdbeAddOp3(v, OP_ZeroOrNull, r1, inReg, r2); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, inReg); + } } testcase( regFree1==0 ); testcase( regFree2==0 ); @@ -4492,7 +5189,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } case TK_AND: - case TK_OR: + case TK_OR: { + inReg = exprCodeTargetAndOr(pParse, pExpr, target, ®Free1); + break; + } case TK_PLUS: case TK_STAR: case TK_MINUS: @@ -4501,10 +5201,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ case TK_BITOR: case TK_SLASH: case TK_LSHIFT: - case TK_RSHIFT: + case TK_RSHIFT: case TK_CONCAT: { - assert( TK_AND==OP_And ); testcase( op==TK_AND ); - assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + int addrIsNull; assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); @@ -4514,11 +5213,23 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } sqlite3VdbeAddOp3(v, op, r2, r1, target); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeJumpHere(v, addrIsNull); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + VdbeComment((v, "short-circut value")); + } break; } case TK_UMINUS: { @@ -4611,7 +5322,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } #endif - if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ /* SQL functions can be expensive. So try to avoid running them ** multiple times if we know they always give the same result */ return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -4632,7 +5345,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ErrorMsg(pParse, "unknown function: %#T()", pExpr); break; } - if( pDef->funcFlags & SQLITE_FUNC_INLINE ){ + if( (pDef->funcFlags & SQLITE_FUNC_INLINE)!=0 && ALWAYS(pFarg!=0) ){ assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); return exprCodeInlineFunction(pParse, pFarg, @@ -4642,7 +5355,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } for(i=0; ia[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -4658,10 +5371,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ r1 = sqlite3GetTempRange(pParse, nFarg); } - /* For length() and typeof() functions with a column argument, + /* For length() and typeof() and octet_length() functions, ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG - ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data - ** loading. + ** or OPFLAG_TYPEOFARG or OPFLAG_BYTELENARG respectively, to avoid + ** unnecessary data loading. */ if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ u8 exprOp; @@ -4671,14 +5384,16 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); - testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); - pFarg->a[0].pExpr->op2 = - pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + assert( SQLITE_FUNC_BYTELEN==OPFLAG_BYTELENARG ); + assert( (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG)==OPFLAG_BYTELENARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG); + pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG; } } - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); }else{ r1 = 0; } @@ -4691,7 +5406,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** see if it is a column in a virtual table. This is done because ** the left operand of infix functions (the operand we want to ** control overloading) ends up as the second argument to the - ** function. The expression "A glob B" is equivalent to + ** function. The expression "A glob B" is equivalent to ** "glob(B,A). We want to use the A in "A glob B" to test ** for function overloading. But we use the B term in "glob(B,A)". */ @@ -4702,7 +5417,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } #endif if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - if( !pColl ) pColl = db->pDfltColl; + if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, @@ -4782,8 +5497,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( !ExprHasProperty(pExpr, EP_Collate) ){ /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called ** "SOFT-COLLATE" that is added to constraints that are pushed down - ** from outer queries into sub-queries by the push-down optimization. - ** Clear subtypes as subtypes may not cross a subquery boundary. + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. */ assert( pExpr->pLeft ); sqlite3ExprCode(pParse, pExpr->pLeft, target); @@ -4810,7 +5526,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** ** The expression is implemented using an OP_Param opcode. The p1 ** parameter is set to 0 for an old.rowid reference, or to (i+1) - ** to reference another column of the old.* pseudo-table, where + ** to reference another column of the old.* pseudo-table, where ** i is the index of the column. For a new.rowid reference, p1 is ** set to (n+1), where n is the number of columns in each pseudo-table. ** For a reference to any other column in the new.* pseudo-table, p1 @@ -4824,7 +5540,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** ** p1==0 -> old.rowid p1==3 -> new.rowid ** p1==1 -> old.a p1==4 -> new.a - ** p1==2 -> old.b p1==5 -> new.b + ** p1==2 -> old.b p1==5 -> new.b */ Table *pTab; int iCol; @@ -4833,7 +5549,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( ExprUseYTab(pExpr) ); pTab = pExpr->y.pTab; iCol = pExpr->iColumn; - p1 = pExpr->iTable * (pTab->nCol+1) + 1 + p1 = pExpr->iTable * (pTab->nCol+1) + 1 + sqlite3TableColumnToStorage(pTab, iCol); assert( pExpr->iTable==0 || pExpr->iTable==1 ); @@ -4952,7 +5668,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } testcase( pX->op==TK_COLUMN ); - exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); testcase( regFree1==0 ); memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; @@ -4991,7 +5707,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { - assert( pExpr->affExpr==OE_Rollback + assert( pExpr->affExpr==OE_Rollback || pExpr->affExpr==OE_Abort || pExpr->affExpr==OE_Fail || pExpr->affExpr==OE_Ignore @@ -5006,15 +5722,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } assert( !ExprHasProperty(pExpr, EP_IntValue) ); if( pExpr->affExpr==OE_Ignore ){ - sqlite3VdbeAddOp4( - v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, OE_Ignore); VdbeCoverage(v); }else{ - sqlite3HaltConstraint(pParse, + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp3(v, OP_Halt, pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, - pExpr->affExpr, pExpr->u.zToken, 0, 0); + pExpr->affExpr, r1); } - break; } #endif @@ -5033,9 +5748,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** once. If no functions are involved, then factor the code out and put it at ** the end of the prepared statement in the initialization section. ** -** If regDest>=0 then the result is always stored in that register and the -** result is not reusable. If regDest<0 then this routine is free to -** store the value whereever it wants. The register where the expression +** If regDest>0 then the result is always stored in that register and the +** result is not reusable. If regDest<0 then this routine is free to +** store the value wherever it wants. The register where the expression ** is stored is returned. When regDest<0, two identical expressions might ** code to the same register, if they do not contain function calls and hence ** are factored out into the initialization section at the end of the @@ -5048,6 +5763,7 @@ int sqlite3ExprCodeRunJustOnce( ){ ExprList *p; assert( ConstFactorOk(pParse) ); + assert( regDest!=0 ); p = pParse->pConstExpr; if( regDest<0 && p ){ struct ExprList_item *pItem; @@ -5087,6 +5803,25 @@ int sqlite3ExprCodeRunJustOnce( return regDest; } +/* +** Make arrangements to invoke OP_Null on a range of registers +** during initialization. +*/ +SQLITE_NOINLINE void sqlite3ExprNullRegisterRange( + Parse *pParse, /* Parsing context */ + int iReg, /* First register to set to NULL */ + int nReg /* Number of sequential registers to NULL out */ +){ + u8 okConstFactor = pParse->okConstFactor; + Expr t; + memset(&t, 0, sizeof(t)); + t.op = TK_NULLS; + t.y.nReg = nReg; + pParse->okConstFactor = 1; + sqlite3ExprCodeRunJustOnce(pParse, &t, iReg); + pParse->okConstFactor = okConstFactor; +} + /* ** Generate code to evaluate an expression and store the results ** into a register. Return the register number where the results @@ -5106,7 +5841,7 @@ int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ if( ConstFactorOk(pParse) && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ *pReg = 0; r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); @@ -5138,8 +5873,10 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) - && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER) + Expr *pX = sqlite3ExprSkipCollateAndLikely(pExpr); + testcase( pX!=pExpr ); + if( ALWAYS(pX) + && (ExprHasProperty(pX,EP_Subquery) || pX->op==TK_REGISTER) ){ op = OP_Copy; }else{ @@ -5168,7 +5905,7 @@ void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ sqlite3ExprCodeCopy(pParse, pExpr, target); @@ -5227,7 +5964,7 @@ int sqlite3ExprCodeExprList( sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ @@ -5255,7 +5992,7 @@ int sqlite3ExprCodeExprList( ** ** x BETWEEN y AND z ** -** The above is equivalent to +** The above is equivalent to ** ** x>=y AND x<=z ** @@ -5300,7 +6037,7 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = pDel; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); if( xJump ){ xJump(pParse, &exprAnd, dest, jumpIfNull); }else{ @@ -5360,17 +6097,27 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); - }else if( op==TK_AND ){ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); }else{ - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pSecond, dest, jumpIfNull); + } } break; } @@ -5409,10 +6156,16 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -5427,6 +6180,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: @@ -5434,11 +6194,11 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { @@ -5473,7 +6233,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } } sqlite3ReleaseTempReg(pParse, regFree1); - sqlite3ReleaseTempReg(pParse, regFree2); + sqlite3ReleaseTempReg(pParse, regFree2); } /* @@ -5534,17 +6294,27 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); if( pAlt!=pExpr ){ sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); - }else if( pExpr->op==TK_AND ){ - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); }else{ - int d2 = sqlite3VdbeMakeLabel(pParse); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, - jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); + Expr *pFirst, *pSecond; + if( exprEvalRhsFirst(pExpr) ){ + pFirst = pExpr->pRight; + pSecond = pExpr->pLeft; + }else{ + pFirst = pExpr->pLeft; + pSecond = pExpr->pRight; + } + if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pFirst, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pFirst, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pSecond, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } } break; } @@ -5586,10 +6356,16 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_GE: case TK_NE: case TK_EQ: { + int addrIsNull; if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; - testcase( jumpIfNull==0 ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + if( ExprHasProperty(pExpr, EP_Subquery) && jumpIfNull!=SQLITE_NULLEQ ){ + addrIsNull = exprComputeOperands(pParse, pExpr, + &r1, &r2, ®Free1, ®Free2); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + addrIsNull = 0; + } codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); @@ -5604,16 +6380,23 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); + if( addrIsNull ){ + if( jumpIfNull ){ + sqlite3VdbeChangeP2(v, addrIsNull, dest); + }else{ + sqlite3VdbeJumpHere(v, addrIsNull); + } + } break; } case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { @@ -5634,7 +6417,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { - default_expr: + default_expr: if( ExprAlwaysFalse(pExpr) ){ sqlite3VdbeGoto(v, dest); }else if( ExprAlwaysTrue(pExpr) ){ @@ -5679,16 +6462,23 @@ void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ ** same as that currently bound to variable pVar, non-zero is returned. ** Otherwise, if the values are not the same or if pExpr is not a simple ** SQL value, zero is returned. +** +** If the SQLITE_EnableQPSG flag is set on the database connection, then +** this routine always returns false. */ -static int exprCompareVariable( +static SQLITE_NOINLINE int exprCompareVariable( const Parse *pParse, const Expr *pVar, const Expr *pExpr ){ - int res = 0; + int res = 2; int iVar; sqlite3_value *pL, *pR = 0; - + + if( pExpr->op==TK_VARIABLE && pVar->iColumn==pExpr->iColumn ){ + return 0; + } + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) return 2; sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR); if( pR ){ iVar = pVar->iColumn; @@ -5698,12 +6488,11 @@ static int exprCompareVariable( if( sqlite3_value_type(pL)==SQLITE_TEXT ){ sqlite3_value_text(pL); /* Make sure the encoding is UTF-8 */ } - res = 0==sqlite3MemCompare(pL, pR, 0); + res = sqlite3MemCompare(pL, pR, 0) ? 2 : 0; } sqlite3ValueFree(pR); sqlite3ValueFree(pL); } - return res; } @@ -5729,12 +6518,10 @@ static int exprCompareVariable( ** just might result in some slightly slower code. But returning ** an incorrect 0 or 1 could lead to a malfunction. ** -** If pParse is not NULL then TK_VARIABLE terms in pA with bindings in -** pParse->pReprepare can be matched against literals in pB. The -** pParse->pVdbe->expmask bitmask is updated for each variable referenced. -** If pParse is NULL (the normal case) then any TK_VARIABLE term in -** Argument pParse should normally be NULL. If it is not NULL and pA or -** pB causes a return value of 2. +** If pParse is not NULL and SQLITE_EnableQPSG is off then TK_VARIABLE +** terms in pA with bindings in pParse->pReprepare can be matched against +** literals in pB. The pParse->pVdbe->expmask bitmask is updated for +** each variable referenced. */ int sqlite3ExprCompare( const Parse *pParse, @@ -5746,8 +6533,8 @@ int sqlite3ExprCompare( if( pA==0 || pB==0 ){ return pB==pA ? 0 : 2; } - if( pParse && pA->op==TK_VARIABLE && exprCompareVariable(pParse, pA, pB) ){ - return 0; + if( pParse && pA->op==TK_VARIABLE ){ + return exprCompareVariable(pParse, pA, pB); } combinedFlags = pA->flags | pB->flags; if( combinedFlags & EP_IntValue ){ @@ -5763,7 +6550,7 @@ int sqlite3ExprCompare( if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN && pB->iTable<0 && pA->iTable==iTab ){ /* fall through */ @@ -5791,7 +6578,7 @@ int sqlite3ExprCompare( return 0; }else if( pA->op==TK_COLLATE ){ if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; - }else + }else if( pB->u.zToken!=0 && pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN @@ -5824,7 +6611,7 @@ int sqlite3ExprCompare( /* ** Compare two ExprList objects. Return 0 if they are identical, 1 -** if they are certainly different, or 2 if it is not possible to +** if they are certainly different, or 2 if it is not possible to ** determine if they are identical or not. ** ** If any subelement of pB has Expr.iTable==(-1) then it is allowed @@ -5859,8 +6646,8 @@ int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB, int iTab){ */ int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){ return sqlite3ExprCompare(0, - sqlite3ExprSkipCollateAndLikely(pA), - sqlite3ExprSkipCollateAndLikely(pB), + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), iTab); } @@ -5912,8 +6699,8 @@ static int exprImpliesNotNull( case TK_MINUS: case TK_BITOR: case TK_LSHIFT: - case TK_RSHIFT: - case TK_CONCAT: + case TK_RSHIFT: + case TK_CONCAT: seenNot = 1; /* no break */ deliberate_fall_through case TK_STAR: @@ -5942,25 +6729,77 @@ static int exprImpliesNotNull( return 0; } +/* +** Return true if the boolean value of the expression is always either +** FALSE or NULL. +*/ +static int sqlite3ExprIsNotTrue(Expr *pExpr){ + int v; + if( pExpr->op==TK_NULL ) return 1; + if( pExpr->op==TK_TRUEFALSE && sqlite3ExprTruthValue(pExpr)==0 ) return 1; + v = 1; + if( sqlite3ExprIsInteger(pExpr, &v, 0) && v==0 ) return 1; + return 0; +} + +/* +** Return true if the expression is one of the following: +** +** CASE WHEN x THEN y END +** CASE WHEN x THEN y ELSE NULL END +** CASE WHEN x THEN y ELSE false END +** iif(x,y) +** iif(x,y,NULL) +** iif(x,y,false) +*/ +static int sqlite3ExprIsIIF(sqlite3 *db, const Expr *pExpr){ + ExprList *pList; + if( pExpr->op==TK_FUNCTION ){ + const char *z = pExpr->u.zToken; + FuncDef *pDef; + if( (z[0]!='i' && z[0]!='I') ) return 0; + if( pExpr->x.pList==0 ) return 0; + pDef = sqlite3FindFunction(db, z, pExpr->x.pList->nExpr, ENC(db), 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return 0; +#else + if( NEVER(pDef==0) ) return 0; +#endif + if( (pDef->funcFlags & SQLITE_FUNC_INLINE)==0 ) return 0; + if( SQLITE_PTR_TO_INT(pDef->pUserData)!=INLINEFUNC_iif ) return 0; + }else if( pExpr->op==TK_CASE ){ + if( pExpr->pLeft!=0 ) return 0; + }else{ + return 0; + } + pList = pExpr->x.pList; + assert( pList!=0 ); + if( pList->nExpr==2 ) return 1; + if( pList->nExpr==3 && sqlite3ExprIsNotTrue(pList->a[2].pExpr) ) return 1; + return 0; +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might ** be false. Examples: ** -** pE1: x==5 pE2: x==5 Result: true -** pE1: x>0 pE2: x==5 Result: false -** pE1: x=21 pE2: x=21 OR y=43 Result: true -** pE1: x!=123 pE2: x IS NOT NULL Result: true -** pE1: x!=?1 pE2: x IS NOT NULL Result: true -** pE1: x IS NULL pE2: x IS NOT NULL Result: false -** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** pE1: x==5 pE2: x==5 Result: true +** pE1: x>0 pE2: x==5 Result: false +** pE1: x=21 pE2: x=21 OR y=43 Result: true +** pE1: x!=123 pE2: x IS NOT NULL Result: true +** pE1: x!=?1 pE2: x IS NOT NULL Result: true +** pE1: x IS NULL pE2: x IS NOT NULL Result: false +** pE1: x IS ?2 pE2: x IS NOT NULL Result: false +** pE1: iif(x,y) pE2: x Result: true +** PE1: iif(x,y,0) pE2: x Result: true ** ** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has ** Expr.iTable<0 then assume a table number given by iTab. ** -** If pParse is not NULL, then the values of bound variables in pE1 are +** If pParse is not NULL, then the values of bound variables in pE1 are ** compared against literal values in pE2 and pParse->pVdbe->expmask is -** modified to record which bound variables are referenced. If pParse +** modified to record which bound variables are referenced. If pParse ** is NULL, then false will be returned if pE1 contains any bound variables. ** ** When in doubt, return false. Returning true might give a performance @@ -5987,14 +6826,35 @@ int sqlite3ExprImpliesExpr( ){ return 1; } + if( sqlite3ExprIsIIF(pParse->db, pE1) ){ + return sqlite3ExprImpliesExpr(pParse,pE1->x.pList->a[0].pExpr,pE2,iTab); + } return 0; } +/* This is a helper function to impliesNotNullRow(). In this routine, +** set pWalker->eCode to one only if *both* of the input expressions +** separately have the implies-not-null-row property. +*/ +static void bothImplyNotNullRow(Walker *pWalker, Expr *pE1, Expr *pE2){ + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pE1); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pE2); + } + } +} + /* ** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). ** If the expression node requires that the table at pWalker->iCur ** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. ** +** pWalker->mWFlags is non-zero if this inquiry is being undertaking on +** behalf of a RIGHT JOIN (or FULL JOIN). That makes a difference when +** evaluating terms in the ON clause of an inner join. +** ** This routine controls an optimization. False positives (setting ** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives ** (never setting pWalker->eCode) is a harmless missed optimization. @@ -6003,28 +6863,33 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); if( ExprHasProperty(pExpr, EP_OuterON) ) return WRC_Prune; + if( ExprHasProperty(pExpr, EP_InnerON) && pWalker->mWFlags ){ + /* If iCur is used in an inner-join ON clause to the left of a + ** RIGHT JOIN, that does *not* mean that the table must be non-null. + ** But it is difficult to check for that condition precisely. + ** To keep things simple, any use of iCur from any inner-join is + ** ignored while attempting to simplify a RIGHT JOIN. */ + return WRC_Prune; + } switch( pExpr->op ){ case TK_ISNOT: case TK_ISNULL: case TK_NOTNULL: case TK_IS: - case TK_OR: case TK_VECTOR: - case TK_CASE: - case TK_IN: case TK_FUNCTION: case TK_TRUTH: + case TK_CASE: testcase( pExpr->op==TK_ISNOT ); testcase( pExpr->op==TK_ISNULL ); testcase( pExpr->op==TK_NOTNULL ); testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_OR ); testcase( pExpr->op==TK_VECTOR ); - testcase( pExpr->op==TK_CASE ); - testcase( pExpr->op==TK_IN ); testcase( pExpr->op==TK_FUNCTION ); testcase( pExpr->op==TK_TRUTH ); + testcase( pExpr->op==TK_CASE ); return WRC_Prune; + case TK_COLUMN: if( pWalker->u.iCur==pExpr->iTable ){ pWalker->eCode = 1; @@ -6032,21 +6897,38 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ } return WRC_Prune; + case TK_OR: case TK_AND: - if( pWalker->eCode==0 ){ + /* Both sides of an AND or OR must separately imply non-null-row. + ** Consider these cases: + ** 1. NOT (x AND y) + ** 2. x OR y + ** If only one of x or y is non-null-row, then the overall expression + ** can be true if the other arm is false (case 1) or true (case 2). + */ + testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_AND ); + bothImplyNotNullRow(pWalker, pExpr->pLeft, pExpr->pRight); + return WRC_Prune; + + case TK_IN: + /* Beware of "x NOT IN ()" and "x NOT IN (SELECT 1 WHERE false)", + ** both of which can be true. But apart from these cases, if + ** the left-hand side of the IN is NULL then the IN itself will be + ** NULL. */ + if( ExprUseXList(pExpr) && ALWAYS(pExpr->x.pList->nExpr>0) ){ sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( pWalker->eCode ){ - pWalker->eCode = 0; - sqlite3WalkExpr(pWalker, pExpr->pRight); - } } return WRC_Prune; case TK_BETWEEN: - if( sqlite3WalkExpr(pWalker, pExpr->pLeft)==WRC_Abort ){ - assert( pWalker->eCode ); - return WRC_Abort; - } + /* In "x NOT BETWEEN y AND z" either x must be non-null-row or else + ** both y and z must be non-null row */ + assert( ExprUseXList(pExpr) ); + assert( pExpr->x.pList->nExpr==2 ); + sqlite3WalkExpr(pWalker, pExpr->pLeft); + bothImplyNotNullRow(pWalker, pExpr->x.pList->a[0].pExpr, + pExpr->x.pList->a[1].pExpr); return WRC_Prune; /* Virtual tables are allowed to use constraints like x=NULL. So @@ -6108,7 +6990,7 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ ** be non-NULL, then the LEFT JOIN can be safely converted into an ** ordinary join. */ -int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ +int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab, int isRJ){ Walker w; p = sqlite3ExprSkipCollateAndLikely(p); if( p==0 ) return 0; @@ -6116,7 +6998,7 @@ int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ p = p->pLeft; }else{ while( p->op==TK_AND ){ - if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab, isRJ) ) return 1; p = p->pRight; } } @@ -6124,6 +7006,7 @@ int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ w.xSelectCallback = 0; w.xSelectCallback2 = 0; w.eCode = 0; + w.mWFlags = isRJ!=0; w.u.iCur = iTab; sqlite3WalkExpr(&w, p); return w.eCode; @@ -6142,7 +7025,7 @@ struct IdxCover { }; /* -** Check to see if there are references to columns in table +** Check to see if there are references to columns in table ** pWalker->u.pIdxCover->iCur can be satisfied using the index ** pWalker->u.pIdxCover->pIdx. */ @@ -6184,7 +7067,7 @@ int sqlite3ExprCoveredByIndex( } -/* Structure used to pass information throught the Walker in order to +/* Structure used to pass information throughout the Walker in order to ** implement sqlite3ReferencesSrcList(). */ struct RefSrcList { @@ -6232,7 +7115,7 @@ static void selectRefLeave(Walker *pWalker, Select *pSelect){ } /* This is the Walker EXPR callback for sqlite3ReferencesSrcList(). -** +** ** Set the 0x01 bit of pWalker->eCode if there is a reference to any ** of the tables shown in RefSrcList.pRef. ** @@ -6291,6 +7174,12 @@ int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ assert( pExpr->op==TK_AGG_FUNCTION ); assert( ExprUseXList(pExpr) ); sqlite3WalkExprList(&w, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + assert( pExpr->pLeft->x.pList!=0 ); + sqlite3WalkExprList(&w, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC if( ExprHasProperty(pExpr, EP_WinFunc) ){ sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); @@ -6331,9 +7220,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } }else{ @@ -6342,9 +7230,8 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ && pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); - if( pExpr ){ + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; - sqlite3ExprDeferredDelete(pParse, pExpr); } } } @@ -6377,7 +7264,7 @@ static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ &i ); return i; -} +} /* ** Add a new element to the pAggInfo->aFunc[] array. Return the index of @@ -6386,7 +7273,7 @@ static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ int i; pInfo->aFunc = sqlite3ArrayAllocate( - db, + db, pInfo->aFunc, sizeof(pInfo->aFunc[0]), &pInfo->nFunc, @@ -6397,10 +7284,10 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ /* ** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. -** Return the index in aCol[] of the entry that describes that column. +** Return the index in aCol[] of the entry that describes that column. ** ** If no prior entry is found, create a new one and return -1. The -** new column will have an idex of pAggInfo->nColumn-1. +** new column will have an index of pAggInfo->nColumn-1. */ static void findOrCreateAggInfoColumn( Parse *pParse, /* Parsing context */ @@ -6409,10 +7296,13 @@ static void findOrCreateAggInfoColumn( ){ struct AggInfo_col *pCol; int k; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ + if( pCol->pCExpr==pExpr ) return; if( pCol->iTable==pExpr->iTable && pCol->iColumn==pExpr->iColumn && pExpr->op!=TK_IF_NULL_ROW @@ -6426,6 +7316,10 @@ static void findOrCreateAggInfoColumn( assert( pParse->db->mallocFailed ); return; } + if( k>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + k = mxTerm; + } pCol = &pAggInfo->aCol[k]; assert( ExprUseYTab(pExpr) ); pCol->pTab = pExpr->y.pTab; @@ -6459,6 +7353,7 @@ static void findOrCreateAggInfoColumn( if( pExpr->op==TK_COLUMN ){ pExpr->op = TK_AGG_COLUMN; } + assert( k <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)k; } @@ -6537,31 +7432,68 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ case TK_AGG_FUNCTION: { if( (pNC->ncFlags & NC_InAggFunc)==0 && pWalker->walkerDepth==pExpr->op2 + && pExpr->pAggInfo==0 ){ - /* Check to see if pExpr is a duplicate of another aggregate + /* Check to see if pExpr is a duplicate of another aggregate ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); for(i=0; inFunc; i++, pItem++){ - if( pItem->pFExpr==pExpr ) break; + if( NEVER(pItem->pFExpr==pExpr) ) break; if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ break; } } - if( i>=pAggInfo->nFunc ){ + if( i>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + i = mxTerm; + assert( inFunc ); + }else if( i>=pAggInfo->nFunc ){ /* pExpr is original. Make a new entry in pAggInfo->aFunc[] */ u8 enc = ENC(pParse->db); i = addAggInfoFunc(pParse->db, pAggInfo); if( i>=0 ){ + int nArg; assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; pItem->pFExpr = pExpr; assert( ExprUseUToken(pExpr) ); + nArg = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, - pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); - if( pExpr->flags & EP_Distinct ){ + pExpr->u.zToken, nArg, enc, 0); + assert( pItem->bOBUnique==0 ); + if( pExpr->pLeft + && (pItem->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)==0 + ){ + /* The NEEDCOLL test above causes any ORDER BY clause on + ** aggregate min() or max() to be ignored. */ + ExprList *pOBList; + assert( nArg>0 ); + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + pItem->iOBTab = pParse->nTab++; + pOBList = pExpr->pLeft->x.pList; + assert( pOBList->nExpr>0 ); + assert( pItem->bOBUnique==0 ); + if( pOBList->nExpr==1 + && nArg==1 + && sqlite3ExprCompare(0,pOBList->a[0].pExpr, + pExpr->x.pList->a[0].pExpr,0)==0 + ){ + pItem->bOBPayload = 0; + pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); + }else{ + pItem->bOBPayload = 1; + } + pItem->bUseSubtype = + (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0; + }else{ + pItem->iOBTab = -1; + } + if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){ pItem->iDistinct = pParse->nTab++; }else{ pItem->iDistinct = -1; @@ -6572,6 +7504,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( i <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; return WRC_Prune; diff --git a/src/fkey.c b/src/fkey.c index 29609916b6..f1117a8845 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -858,6 +858,7 @@ static int isSetNullAction(Parse *pParse, FKey *pFKey){ if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) ){ + assert( (pTop->db->flags & SQLITE_FkNoAction)==0 ); return 1; } } @@ -1042,9 +1043,9 @@ void sqlite3FkCheck( pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ SrcItem *pItem = pSrc->a; - pItem->pTab = pFKey->pFrom; + pItem->pSTab = pFKey->pFrom; pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nTabRef++; + pItem->pSTab->nTabRef++; pItem->iCursor = pParse->nTab++; if( regNew!=0 ){ @@ -1052,6 +1053,8 @@ void sqlite3FkCheck( } if( regOld!=0 ){ int eAction = pFKey->aAction[aChange!=0]; + if( (db->flags & SQLITE_FkNoAction) ) eAction = OE_None; + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); /* If this is a deferred FK constraint, or a CASCADE or SET NULL ** action applies, then any foreign key violations caused by @@ -1167,7 +1170,11 @@ int sqlite3FkRequired( /* Check if any parent key columns are being modified. */ for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ - if( p->aAction[1]!=OE_None ) return 2; + if( (pParse->db->flags & SQLITE_FkNoAction)==0 + && p->aAction[1]!=OE_None + ){ + return 2; + } bHaveFK = 1; } } @@ -1217,6 +1224,7 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( (db->flags & SQLITE_FkNoAction) ) action = OE_None; if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ return 0; } @@ -1320,7 +1328,8 @@ static Trigger *fkActionTrigger( SrcList *pSrc; Expr *pRaise; - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + pRaise = sqlite3Expr(db, TK_STRING, "FOREIGN KEY constraint failed"), + pRaise = sqlite3PExpr(pParse, TK_RAISE, pRaise, 0); if( pRaise ){ pRaise->affExpr = OE_Abort; } @@ -1328,7 +1337,8 @@ static Trigger *fkActionTrigger( if( pSrc ){ assert( pSrc->nSrc==1 ); pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); + pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), @@ -1448,9 +1458,8 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ - void *p = (void *)pFKey->pNextTo; - const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); + const char *z = (pFKey->pNextTo ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, pFKey->pNextTo); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; diff --git a/src/func.c b/src/func.c index 93c9a8f8a4..6dac7195a4 100644 --- a/src/func.c +++ b/src/func.c @@ -149,11 +149,47 @@ static void lengthFunc( } } +/* +** Implementation of the octet_length() function +*/ +static void bytelengthFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_BLOB: { + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + i64 m = sqlite3_context_db_handle(context)->enc<=SQLITE_UTF8 ? 1 : 2; + sqlite3_result_int64(context, sqlite3_value_bytes(argv[0])*m); + break; + } + case SQLITE_TEXT: { + if( sqlite3_value_encoding(argv[0])<=SQLITE_UTF8 ){ + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + }else{ + sqlite3_result_int(context, sqlite3_value_bytes16(argv[0])); + } + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + /* ** Implementation of the abs() function. ** ** IMP: R-23979-26855 The abs(X) function returns the absolute value of -** the numeric argument X. +** the numeric argument X. */ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); @@ -170,7 +206,7 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ return; } iVal = -iVal; - } + } sqlite3_result_int64(context, iVal); break; } @@ -318,16 +354,10 @@ static void substrFunc( int len; int p0type; i64 p1, p2; - int negP2 = 0; assert( argc==3 || argc==2 ); - if( sqlite3_value_type(argv[1])==SQLITE_NULL - || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) - ){ - return; - } p0type = sqlite3_value_type(argv[0]); - p1 = sqlite3_value_int(argv[1]); + p1 = sqlite3_value_int64(argv[1]); if( p0type==SQLITE_BLOB ){ len = sqlite3_value_bytes(argv[0]); z = sqlite3_value_blob(argv[0]); @@ -343,28 +373,31 @@ static void substrFunc( } } } -#ifdef SQLITE_SUBSTR_COMPATIBILITY - /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as - ** as substr(X,1,N) - it returns the first N characters of X. This - ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] - ** from 2009-02-02 for compatibility of applications that exploited the - ** old buggy behavior. */ - if( p1==0 ) p1 = 1; /* */ -#endif if( argc==3 ){ - p2 = sqlite3_value_int(argv[2]); - if( p2<0 ){ - p2 = -p2; - negP2 = 1; - } + p2 = sqlite3_value_int64(argv[2]); + if( p2==0 && sqlite3_value_type(argv[2])==SQLITE_NULL ) return; }else{ p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; } + if( p1==0 ){ +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + p1 = 1; /* */ +#endif + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + } if( p1<0 ){ p1 += len; if( p1<0 ){ - p2 += p1; - if( p2<0 ) p2 = 0; + if( p2<0 ){ + p2 = 0; + }else{ + p2 += p1; + } p1 = 0; } }else if( p1>0 ){ @@ -372,12 +405,13 @@ static void substrFunc( }else if( p2>0 ){ p2--; } - if( negP2 ){ - p1 -= p2; - if( p1<0 ){ - p2 += p1; - p1 = 0; + if( p2<0 ){ + if( p2<-p1 ){ + p2 = p1; + }else{ + p2 = -p2; } + p1 -= p2; } assert( p1>=0 && p2>=0 ); if( p0type!=SQLITE_BLOB ){ @@ -391,9 +425,11 @@ static void substrFunc( sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, SQLITE_UTF8); }else{ - if( p1+p2>len ){ + if( p1>=len ){ + p1 = p2 = 0; + }else if( p2>len-p1 ){ p2 = len-p1; - if( p2<0 ) p2 = 0; + assert( p2>0 ); } sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); } @@ -404,13 +440,13 @@ static void substrFunc( */ #ifndef SQLITE_OMIT_FLOATING_POINT static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - int n = 0; + i64 n = 0; double r; char *zBuf; assert( argc==1 || argc==2 ); if( argc==2 ){ if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; - n = sqlite3_value_int(argv[1]); + n = sqlite3_value_int64(argv[1]); if( n>30 ) n = 30; if( n<0 ) n = 0; } @@ -422,10 +458,10 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ */ if( r<-4503599627370496.0 || r>+4503599627370496.0 ){ /* The value has no fractional part so there is nothing to round */ - }else if( n==0 ){ + }else if( n==0 ){ r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ - zBuf = sqlite3_mprintf("%.*f",n,r); + zBuf = sqlite3_mprintf("%!.*f",(int)n,r); if( zBuf==0 ){ sqlite3_result_error_nomem(context); return; @@ -449,7 +485,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3 *db = sqlite3_context_db_handle(context); assert( nByte>0 ); testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + testcase( nByte==(i64)db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); z = 0; @@ -515,7 +551,7 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ #define noopFunc versionFunc /* Substitute function - never called */ /* -** Implementation of random(). Return a random integer. +** Implementation of random(). Return a random integer. */ static void randomFunc( sqlite3_context *context, @@ -526,11 +562,11 @@ static void randomFunc( UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_randomness(sizeof(r), &r); if( r<0 ){ - /* We need to prevent a random number of 0x8000000000000000 + /* We need to prevent a random number of 0x8000000000000000 ** (or -9223372036854775808) since when you do abs() of that ** number of you get the same value back again. To do this ** in a way that is testable, mask the sign bit off of negative - ** values, resulting in a positive value. Then take the + ** values, resulting in a positive value. Then take the ** 2s complement of that positive value. The end result can ** therefore be no less than -9223372036854775807. */ @@ -568,8 +604,8 @@ static void randomBlob( ** value is the same as the sqlite3_last_insert_rowid() API function. */ static void last_insert_rowid( - sqlite3_context *context, - int NotUsed, + sqlite3_context *context, + int NotUsed, sqlite3_value **NotUsed2 ){ sqlite3 *db = sqlite3_context_db_handle(context); @@ -625,7 +661,7 @@ struct compareInfo { /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, provde the Utf8Read() +** character is exactly one byte in size. Also, provide the Utf8Read() ** macro for fast reading of the next character in the common case where ** the next character is ASCII. */ @@ -677,7 +713,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** it the last character in the list. ** ** Like matching rules: -** +** ** '%' Matches any sequence of zero or more characters ** *** '_' Matches any one character @@ -700,13 +736,13 @@ static int patternCompare( u32 matchAll = pInfo->matchAll; /* "*" or "%" */ u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ const u8 *zEscaped = 0; /* One past the last escaped input char */ - + while( (c = Utf8Read(zPattern))!=0 ){ if( c==matchAll ){ /* Match "*" */ /* Skip over multiple "*" characters in the pattern. If there ** are also "?" characters, skip those as well, but consume a ** single character of the input string for each "?" skipped */ - while( (c=Utf8Read(zPattern)) == matchAll + while( (c=Utf8Read(zPattern)) == matchAll || (c == matchOne && matchOne!=0) ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ return SQLITE_NOWILDCARDMATCH; @@ -858,7 +894,7 @@ int sqlite3_like_count = 0; /* ** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the +** the built-in LIKE operator. The first argument to the function is the ** pattern and the second argument is the string. So, the SQL statements: ** ** A LIKE B @@ -869,8 +905,8 @@ int sqlite3_like_count = 0; ** the GLOB operator. */ static void likeFunc( - sqlite3_context *context, - int argc, + sqlite3_context *context, + int argc, sqlite3_value **argv ){ const unsigned char *zA, *zB; @@ -909,7 +945,7 @@ static void likeFunc( const unsigned char *zEsc = sqlite3_value_text(argv[2]); if( zEsc==0 ) return; if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ - sqlite3_result_error(context, + sqlite3_result_error(context, "ESCAPE expression must be a single character", -1); return; } @@ -1022,8 +1058,8 @@ static void compileoptionusedFunc( #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ /* -** Implementation of the sqlite_compileoption_get() function. -** The result is a string that identifies the compiler options +** Implementation of the sqlite_compileoption_get() function. +** The result is a string that identifies the compiler options ** used to build SQLite. */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS @@ -1047,14 +1083,14 @@ static void compileoptiongetFunc( ** digits. */ static const char hexdigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; /* ** Append to pStr text that is the SQL literal representation of the ** value contained in pValue. */ -void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ +void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int bEscape){ /* As currently implemented, the string must be initially empty. ** we might relax this requirement in the future, but that will ** require enhancements to the implementation. */ @@ -1065,13 +1101,13 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ double r1, r2; const char *zVal; r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!.15g", r1); + sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); if( r1!=r2 ){ sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!.20e", r1); + sqlite3_str_appendf(pStr, "%!0.20e", r1); } } break; @@ -1102,7 +1138,7 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } case SQLITE_TEXT: { const unsigned char *zArg = sqlite3_value_text(pValue); - sqlite3_str_appendf(pStr, "%Q", zArg); + sqlite3_str_appendf(pStr, bEscape ? "%#Q" : "%Q", zArg); break; } default: { @@ -1114,7 +1150,106 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } /* -** Implementation of the QUOTE() function. +** Return true if z[] begins with N hexadecimal digits, and write +** a decoding of those digits into *pVal. Or return false if any +** one of the first N characters in z[] is not a hexadecimal digit. +*/ +static int isNHex(const char *z, int N, u32 *pVal){ + int i; + u32 v = 0; + for(i=0; i0 ){ + memmove(&zOut[j], &zIn[i], n); + j += n; + i += n; + } + if( zIn[i+1]=='\\' ){ + i += 2; + zOut[j++] = '\\'; + }else if( sqlite3Isxdigit(zIn[i+1]) ){ + if( !isNHex(&zIn[i+1], 4, &v) ) goto unistr_error; + i += 5; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='+' ){ + if( !isNHex(&zIn[i+2], 6, &v) ) goto unistr_error; + i += 8; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='u' ){ + if( !isNHex(&zIn[i+2], 4, &v) ) goto unistr_error; + i += 6; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='U' ){ + if( !isNHex(&zIn[i+2], 8, &v) ) goto unistr_error; + i += 10; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else{ + goto unistr_error; + } + } + zOut[j] = 0; + sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8); + return; + +unistr_error: + sqlite3_free(zOut); + sqlite3_result_error(context, "invalid Unicode escape", -1); + return; +} + + +/* +** Implementation of the QUOTE() function. ** ** The quote(X) function returns the text of an SQL literal which is the ** value of its argument suitable for inclusion into an SQL statement. @@ -1122,6 +1257,10 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ ** as needed. BLOBs are encoded as hexadecimal literals. Strings with ** embedded NUL characters cannot be represented as string literals in SQL ** and hence the returned string literal is truncated prior to the first NUL. +** +** If sqlite3_user_data() is non-zero, then the UNISTR_QUOTE() function is +** implemented instead. The difference is that UNISTR_QUOTE() uses the +** UNISTR() function to escape control characters. */ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_str str; @@ -1129,7 +1268,7 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); UNUSED_PARAMETER(argc); sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); - sqlite3QuoteValue(&str,argv[0]); + sqlite3QuoteValue(&str,argv[0],SQLITE_PTR_TO_INT(sqlite3_user_data(context))); sqlite3_result_text(context, sqlite3StrAccumFinish(&str), str.nChar, SQLITE_DYNAMIC); if( str.accError!=SQLITE_OK ){ @@ -1140,7 +1279,7 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ /* ** The unicode() function. Return the integer unicode code-point value -** for the first character of the input string. +** for the first character of the input string. */ static void unicodeFunc( sqlite3_context *context, @@ -1191,6 +1330,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } + *zOut = 0; sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } @@ -1219,7 +1359,8 @@ static void hexFunc( *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite3_result_text(context, zHex, n*2, sqlite3_free); + sqlite3_result_text64(context, zHex, (u64)(z-zHex), + sqlite3_free, SQLITE_UTF8); } } @@ -1244,12 +1385,12 @@ static int strContainsChar(const u8 *zStr, int nStr, u32 ch){ ** decoded and returned as a blob. ** ** If there is only a single argument, then it must consist only of an -** even number of hexadeximal digits. Otherwise, return NULL. +** even number of hexadecimal digits. Otherwise, return NULL. ** ** Or, if there is a second argument, then any character that appears in ** the second argument is also allowed to appear between pairs of hexadecimal ** digits in the first argument. If any other character appears in the -** first argument, or if one of the allowed characters appears between +** first argument, or if one of the allowed characters appears between ** two hexadecimal digits that make up a single byte, NULL is returned. ** ** The following expressions are all true: @@ -1371,7 +1512,7 @@ static void replaceFunc( } if( zPattern[0]==0 ){ assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); - sqlite3_result_value(context, argv[0]); + sqlite3_result_text(context, (const char*)zStr, nStr, SQLITE_TRANSIENT); return; } nPattern = sqlite3_value_bytes(argv[1]); @@ -1382,11 +1523,11 @@ static void replaceFunc( assert( zRep==sqlite3_value_text(argv[2]) ); nOut = nStr + 1; assert( nOut0 ){ - azChar = contextMalloc(context, + azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+sizeof(unsigned))); if( azChar==0 ){ return; @@ -1513,6 +1654,83 @@ static void trimFunc( sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); } +/* The core implementation of the CONCAT(...) and CONCAT_WS(SEP,...) +** functions. +** +** Return a string value that is the concatenation of all non-null +** entries in argv[]. Use zSep as the separator. +*/ +static void concatFuncCore( + sqlite3_context *context, + int argc, + sqlite3_value **argv, + int nSep, + const char *zSep +){ + i64 j, n = 0; + int i; + int bNotNull = 0; /* True after at least NOT NULL argument seen */ + char *z; + for(i=0; i0 ){ + memcpy(&z[j], zSep, nSep); + j += nSep; + } + memcpy(&z[j], v, k); + j += k; + bNotNull = 1; + } + } + } + z[j] = 0; + assert( j<=n ); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); +} + +/* +** The CONCAT(...) function. Generate a string result that is the +** concatentation of all non-null arguments. +*/ +static void concatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + concatFuncCore(context, argc, argv, 0, ""); +} + +/* +** The CONCAT_WS(separator, ...) function. +** +** Generate a string that is the concatenation of 2nd through the Nth +** argument. Use the first argument (which must be non-NULL) as the +** separator. +*/ +static void concatwsFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nSep = sqlite3_value_bytes(argv[0]); + const char *zSep = (const char*)sqlite3_value_text(argv[0]); + if( zSep==0 ) return; + concatFuncCore(context, argc-1, argv+1, nSep, zSep); +} + #ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION /* @@ -1546,7 +1764,7 @@ static void unknownFunc( ** Compute the soundex encoding of a word. ** ** IMP: R-59782-00072 The soundex(X) function returns a string that is the -** soundex encoding of the string X. +** soundex encoding of the string X. */ static void soundexFunc( sqlite3_context *context, @@ -1634,13 +1852,68 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ */ typedef struct SumCtx SumCtx; struct SumCtx { - double rSum; /* Floating point sum */ - i64 iSum; /* Integer sum */ + double rSum; /* Running sum as as a double */ + double rErr; /* Error term for Kahan-Babushka-Neumaier summation */ + i64 iSum; /* Running sum as a signed integer */ i64 cnt; /* Number of elements summed */ - u8 overflow; /* True if integer overflow seen */ - u8 approx; /* True if non-integer value was input to the sum */ + u8 approx; /* True if any non-integer value was input to the sum */ + u8 ovrfl; /* Integer overflow seen */ }; +/* +** Do one step of the Kahan-Babushka-Neumaier summation. +** +** https://en.wikipedia.org/wiki/Kahan_summation_algorithm +** +** Variables are marked "volatile" to defeat c89 x86 floating point +** optimizations can mess up this algorithm. +*/ +static void kahanBabuskaNeumaierStep( + volatile SumCtx *pSum, + volatile double r +){ + volatile double s = pSum->rSum; + volatile double t = s + r; + if( fabs(s) > fabs(r) ){ + pSum->rErr += (s - t) + r; + }else{ + pSum->rErr += (r - t) + s; + } + pSum->rSum = t; +} + +/* +** Add a (possibly large) integer to the running sum. +*/ +static void kahanBabuskaNeumaierStepInt64(volatile SumCtx *pSum, i64 iVal){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iBig, iSm; + iSm = iVal % 16384; + iBig = iVal - iSm; + kahanBabuskaNeumaierStep(pSum, iBig); + kahanBabuskaNeumaierStep(pSum, iSm); + }else{ + kahanBabuskaNeumaierStep(pSum, (double)iVal); + } +} + +/* +** Initialize the Kahan-Babaska-Neumaier sum from a 64-bit integer +*/ +static void kahanBabuskaNeumaierInit( + volatile SumCtx *p, + i64 iVal +){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iSm = iVal % 16384; + p->rSum = (double)(iVal - iSm); + p->rErr = (double)iSm; + }else{ + p->rSum = (double)iVal; + p->rErr = 0.0; + } +} + /* ** Routines used to compute the sum, average, and total. ** @@ -1648,7 +1921,7 @@ struct SumCtx { ** that it returns NULL if it sums over no inputs. TOTAL returns ** 0.0 in that case. In addition, TOTAL always returns a float where ** SUM might return an integer if it never encounters a floating point -** value. TOTAL never fails, but SUM might through an exception if +** value. TOTAL never fails, but SUM might throw an exception if ** it overflows an integer. */ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ @@ -1660,15 +1933,29 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ type = sqlite3_value_numeric_type(argv[0]); if( p && type!=SQLITE_NULL ){ p->cnt++; - if( type==SQLITE_INTEGER ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum += v; - if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ - p->approx = p->overflow = 1; + if( p->approx==0 ){ + if( type!=SQLITE_INTEGER ){ + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + }else{ + i64 x = p->iSum; + if( sqlite3AddInt64(&x, sqlite3_value_int64(argv[0]))==0 ){ + p->iSum = x; + }else{ + p->ovrfl = 1; + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + } } }else{ - p->rSum += sqlite3_value_double(argv[0]); - p->approx = 1; + if( type==SQLITE_INTEGER ){ + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + }else{ + p->ovrfl = 0; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + } } } } @@ -1685,13 +1972,21 @@ static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ if( ALWAYS(p) && type!=SQLITE_NULL ){ assert( p->cnt>0 ); p->cnt--; - assert( type==SQLITE_INTEGER || p->approx ); - if( type==SQLITE_INTEGER && p->approx==0 ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum -= v; - p->iSum -= v; + if( !p->approx ){ + if( sqlite3SubInt64(&p->iSum, sqlite3_value_int64(argv[0])) ){ + p->ovrfl = 1; + p->approx = 1; + } + }else if( type==SQLITE_INTEGER ){ + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal!=SMALLEST_INT64 ){ + kahanBabuskaNeumaierStepInt64(p, -iVal); + }else{ + kahanBabuskaNeumaierStepInt64(p, LARGEST_INT64); + kahanBabuskaNeumaierStepInt64(p, 1); + } }else{ - p->rSum -= sqlite3_value_double(argv[0]); + kahanBabuskaNeumaierStep(p, -sqlite3_value_double(argv[0])); } } } @@ -1702,10 +1997,14 @@ static void sumFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - if( p->overflow ){ - sqlite3_result_error(context,"integer overflow",-1); - }else if( p->approx ){ - sqlite3_result_double(context, p->rSum); + if( p->approx ){ + if( p->ovrfl ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( !sqlite3IsOverflow(p->rErr) ){ + sqlite3_result_double(context, p->rSum+p->rErr); + }else{ + sqlite3_result_double(context, p->rSum); + } }else{ sqlite3_result_int64(context, p->iSum); } @@ -1715,14 +2014,29 @@ static void avgFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - sqlite3_result_double(context, p->rSum/(double)p->cnt); + double r; + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + sqlite3_result_double(context, r/(double)p->cnt); } } static void totalFinalize(sqlite3_context *context){ SumCtx *p; + double r = 0.0; p = sqlite3_aggregate_context(context, 0); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - sqlite3_result_double(context, p ? p->rSum : (double)0); + if( p ){ + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + } + sqlite3_result_double(context, r); } /* @@ -1749,13 +2063,13 @@ static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ #ifndef SQLITE_OMIT_DEPRECATED /* The sqlite3_aggregate_count() function is deprecated. But just to make - ** sure it still operates correctly, verify that its count agrees with our + ** sure it still operates correctly, verify that its count agrees with our ** internal count when using count(*) and when the total count can be ** expressed as a 32-bit integer. */ assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse || p->n==sqlite3_aggregate_count(context) ); #endif -} +} static void countFinalize(sqlite3_context *context){ CountCtx *p; p = sqlite3_aggregate_context(context, 0); @@ -1772,7 +2086,7 @@ static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ p->bInverse = 1; #endif } -} +} #else # define countInverse 0 #endif /* SQLITE_OMIT_WINDOWFUNC */ @@ -1781,8 +2095,8 @@ static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ ** Routines to implement min() and max() aggregate functions. */ static void minmaxStep( - sqlite3_context *context, - int NotUsed, + sqlite3_context *context, + int NotUsed, sqlite3_value **argv ){ Mem *pArg = (Mem *)argv[0]; @@ -1841,8 +2155,13 @@ static void minMaxFinalize(sqlite3_context *context){ /* ** group_concat(EXPR, ?SEPARATOR?) +** string_agg(EXPR, SEPARATOR) +** +** Content is accumulated in GroupConcatCtx.str with the SEPARATOR +** coming before the EXPR value, except for the first entry which +** omits the SEPARATOR. ** -** The SEPARATOR goes before the EXPR string. This is tragic. The +** It is tragic that the SEPARATOR goes before the EXPR string. The ** groupConcatInverse() implementation would have been easier if the ** SEPARATOR were appended after EXPR. And the order is undocumented, ** so we could change it, in theory. But the old behavior has been @@ -1944,9 +2263,9 @@ static void groupConcatInverse( if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); /* pGCC is always non-NULL since groupConcatStep() will have always - ** run frist to initialize it */ + ** run first to initialize it */ if( ALWAYS(pGCC) ){ - int nVS; + int nVS; /* Number of characters to remove */ /* Must call sqlite3_value_text() to convert the argument into text prior ** to invoking sqlite3_value_bytes(), in case the text encoding is UTF16 */ (void)sqlite3_value_text(argv[0]); @@ -1999,7 +2318,9 @@ static void groupConcatValue(sqlite3_context *context){ sqlite3_result_error_toobig(context); }else if( pAccum->accError==SQLITE_NOMEM ){ sqlite3_result_error_nomem(context); - }else{ + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); + }else{ const char *zText = sqlite3_str_value(pAccum); sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); } @@ -2020,17 +2341,6 @@ void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); } -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC - { - extern void sqlcipher_exportFunc(sqlite3_context *, int, sqlite3_value **); - sqlite3CreateFunc(db, "sqlcipher_export", -1, SQLITE_TEXT, 0, sqlcipher_exportFunc, 0, 0, 0, 0, 0); - } -#ifdef SQLCIPHER_EXT -#include "sqlcipher_funcs_init.h" -#endif -#endif -/* END SQLCIPHER */ } /* @@ -2039,8 +2349,10 @@ void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ ** sensitive. */ void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + FuncDef *pDef; struct compareInfo *pInfo; int flags; + int nArg; if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; @@ -2048,17 +2360,20 @@ void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ pInfo = (struct compareInfo*)&likeInfoNorm; flags = SQLITE_FUNC_LIKE; } - sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); - sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; - sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; + for(nArg=2; nArg<=3; nArg++){ + sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, + 0, 0, 0, 0, 0); + pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); + pDef->funcFlags |= flags; + pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; + } } /* ** pExpr points to an expression which implements a function. If ** it is appropriate to apply the LIKE optimization to that function ** then set aWc[0] through aWc[2] to the wildcard characters and the -** escape character and then return TRUE. If the function is not a +** escape character and then return TRUE. If the function is not a ** LIKE-style function then return FALSE. ** ** The expression "a LIKE b ESCAPE c" is only considered a valid LIKE @@ -2323,6 +2638,616 @@ static void signFunc( sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); } +#if defined(SQLITE_ENABLE_PERCENTILE) +/*********************************************************************** +** This section implements the percentile(Y,P) SQL function and similar. +** Requirements: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +** +** (13) A separate median(Y) function is the equivalent percentile(Y,50). +** +** (14) A separate percentile_cont(Y,P) function is equivalent to +** percentile(Y,P/100.0). In other words, the fraction value in +** the second argument is in the range of 0 to 1 instead of 0 to 100. +** +** (15) A separate percentile_disc(Y,P) function is like +** percentile_cont(Y,P) except that instead of returning the weighted +** average of the nearest two input values, it returns the next lower +** value. So the percentile_disc(Y,P) will always return a value +** that was one of the inputs. +** +** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and +** percentile_disc(Y,P) can be used as window functions. +** +** Differences from standard SQL: +** +** * The percentile_cont(X,P) function is equivalent to the following in +** standard SQL: +** +** (percentile_cont(P) WITHIN GROUP (ORDER BY X)) +** +** The SQLite syntax is much more compact. The standard SQL syntax +** is also supported if SQLite is compiled with the +** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option. +** +** * No median(X) function exists in the SQL standard. App developers +** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)". +** +** * No percentile(Y,P) function exists in the SQL standard. Instead of +** percential(Y,P), developers must write this: +** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that +** the fraction parameter to percentile() goes from 0 to 100 whereas +** the fraction parameter in SQL standard percentile_cont() goes from +** 0 to 1. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of these routines work +** by accumulating all values in an array of doubles, then sorting +** that array using quicksort before computing the answer. Thus +** the runtime is O(NlogN) where N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can imagine alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. +*/ + +/* The following object is the group context for a single percentile() +** aggregate. Remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + u64 nAlloc; /* Number of slots allocated for a[] */ + u64 nUsed; /* Number of slots actually used in a[] */ + char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ + char bPctValid; /* True if rPct is valid */ + double rPct; /* Fraction. 0.0 to 1.0 */ + double *a; /* Array of Y values */ +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int percentIsInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less. +*/ +static int percentSameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static i64 percentBinarySearch(Percentile *p, double y, int bExact){ + i64 iFirst = 0; /* First element of search range */ + i64 iLast = (i64)p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + i64 iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( xy ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + +/* +** Generate an error for a percentile function. +** +** The error format string must have exactly one occurrence of "%%s()" +** (with two '%' characters). That substring will be replaced by the name +** of the function. +*/ +static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){ + char *zMsg1; + char *zMsg2; + va_list ap; + + va_start(ap, zFormat); + zMsg1 = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, sqlite3VdbeFuncName(pCtx)) : 0; + sqlite3_result_error(pCtx, zMsg2, -1); + sqlite3_free(zMsg1); + sqlite3_free(zMsg2); +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 || argc==1 ); + + if( argc==1 ){ + /* Requirement 13: median(Y) is the same as percentile(Y,50). */ + rPct = 0.5; + }else{ + /* P must be a number between 0 and 100 for percentile() or between + ** 0.0 and 1.0 for percentile_cont() and percentile_disc(). + ** + ** The user-data is an integer which is 10 times the upper bound. + */ + double mxFrac = (SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx))&2)? 100.0 : 1.0; + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1])/mxFrac; + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>1.0 + ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not between 0.0 and %.1f", + (double)mxFrac); + return; + } + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( !p->bPctValid ){ + p->rPct = rPct; + p->bPctValid = 1; + }else if( !percentSameValue(p->rPct,rPct) ){ + percentError(pCtx, "the fraction argument to %%s()" + " is not the same for all input rows"); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + percentError(pCtx, "input to %%s() is not numeric"); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + percentError(pCtx, "Inf input to %%s()"); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + u64 n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc64(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; + p->bSorted = 1; + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + i64 i; + i = percentBinarySearch(p, y, 0); + if( i<(int)p->nUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; + p->bSorted = 0; + } +} + +/* +** Interchange two doubles. +*/ +#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;} + +/* +** Sort an array of doubles. +** +** Algorithm: quicksort +** +** This is implemented separately rather than using the qsort() routine +** from the standard library because: +** +** (1) To avoid a dependency on qsort() +** (2) To avoid the function call to the comparison routine for each +** comparison. +*/ +static void percentSort(double *a, unsigned int n){ + int iLt; /* Entries before a[iLt] are less than rPivot */ + int iGt; /* Entries at or after a[iGt] are greater than rPivot */ + int i; /* Loop counter */ + double rPivot; /* The pivot value */ + + assert( n>=2 ); + if( a[0]>a[n-1] ){ + SWAP_DOUBLE(a[0],a[n-1]) + } + if( n==2 ) return; + iGt = n-1; + i = n/2; + if( a[0]>a[i] ){ + SWAP_DOUBLE(a[0],a[i]) + }else if( a[i]>a[iGt] ){ + SWAP_DOUBLE(a[i],a[iGt]) + } + if( n==3 ) return; + rPivot = a[i]; + iLt = i = 1; + do{ + if( a[i]iLt ) SWAP_DOUBLE(a[i],a[iLt]) + iLt++; + i++; + }else if( a[i]>rPivot ){ + do{ + iGt--; + }while( iGt>i && a[iGt]>rPivot ); + SWAP_DOUBLE(a[i],a[iGt]) + }else{ + i++; + } + }while( i=2 ) percentSort(a, iLt); + if( n-iGt>=2 ) percentSort(a+iGt, n-iGt); + +/* Uncomment for testing */ +#if 0 + for(i=0; ibSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( i<(int)p->nUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. +*/ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ + Percentile *p; + int settings = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx))&1; /* Discrete? */ + unsigned i1, i2; + double v1, v2; + double ix, vx; + p = (Percentile*)sqlite3_aggregate_context(pCtx, 0); + if( p==0 ) return; + if( p->a==0 ) return; + if( p->nUsed ){ + if( p->bSorted==0 ){ + assert( p->nUsed>1 ); + percentSort(p->a, p->nUsed); + p->bSorted = 1; + } + ix = p->rPct*(p->nUsed-1); + i1 = (unsigned)ix; + if( settings & 1 ){ + vx = p->a[i1]; + }else{ + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + } + sqlite3_result_double(pCtx, vx); + } + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; + } +} +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); +} +/****** End of percentile family of functions ******/ +#endif /* SQLITE_ENABLE_PERCENTILE */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Implementation of sqlite_filestat(SCHEMA). +** +** Return JSON text that describes low-level debug/diagnostic information +** about the sqlite3_file object associated with SCHEMA. +*/ +static void filestatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDbName; + sqlite3_str *pStr; + Btree *pBtree; + + zDbName = (const char*)sqlite3_value_text(argv[0]); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + int rc; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + pStr = sqlite3_str_new(db); + if( pStr==0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_str_append(pStr, "{\"db\":", 6); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_FILESTAT, pStr); + if( rc ) sqlite3_str_append(pStr, "null", 4); + fd = sqlite3PagerJrnlFile(pPager); + if( fd && fd->pMethods!=0 ){ + sqlite3_str_appendall(pStr, ",\"journal\":"); + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_FILESTAT, pStr); + if( rc ) sqlite3_str_append(pStr, "null", 4); + } + sqlite3_str_append(pStr, "}", 1); + sqlite3_result_text(context, sqlite3_str_finish(pStr), -1, + sqlite3_free); + } + sqlite3BtreeLeave(pBtree); + }else{ + sqlite3_result_text(context, "{}", 2, SQLITE_STATIC); + } +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + +#ifdef SQLITE_DEBUG +/* +** Implementation of fpdecode(x,y,z) function. +** +** x is a real number that is to be decoded. y is the precision. +** z is the maximum real precision. Return a string that shows the +** results of the sqlite3FpDecode() function. +** +** Used for testing and debugging only, specifically testing and debugging +** of the sqlite3FpDecode() function. This SQL function does not appear +** in production builds. This function is not an API and is subject to +** modification or removal in future versions of SQLite. +*/ +static void fpdecodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FpDecode s; + double x; + int y, z; + char zBuf[100]; + UNUSED_PARAMETER(argc); + assert( argc==3 ); + x = sqlite3_value_double(argv[0]); + y = sqlite3_value_int(argv[1]); + z = sqlite3_value_int(argv[2]); + if( z<=0 ) z = 1; + sqlite3FpDecode(&s, x, y, z); + if( s.isSpecial==2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%c%.*s/%d", s.sign, s.n, s.z, s.iDP); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Implementation of parseuri(uri,flags) function. +** +** Required Arguments: +** "uri" The URI to parse. +** "flags" Bitmask of flags, as if to sqlite3_open_v2(). +** +** Additional arguments beyond the first two make calls to +** sqlite3_uri_key() for integers and sqlite3_uri_parameter for +** anything else. +** +** The result is a string showing the results of calling sqlite3ParseUri(). +** +** Used for testing and debugging only, specifically testing and debugging +** of the sqlite3ParseUri() function. This SQL function does not appear +** in production builds. This function is not an API and is subject to +** modification or removal in future versions of SQLite. +*/ +static void parseuriFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + sqlite3_str *pResult; + const char *zVfs; + const char *zUri; + unsigned int flgs; + int rc; + sqlite3_vfs *pVfs = 0; + char *zFile = 0; + char *zErr = 0; + + if( argc<2 ) return; + pVfs = sqlite3_vfs_find(0); + assert( pVfs ); + zVfs = pVfs->zName; + zUri = (const char*)sqlite3_value_text(argv[0]); + if( zUri==0 ) return; + flgs = (unsigned int)sqlite3_value_int(argv[1]); + rc = sqlite3ParseUri(zVfs, zUri, &flgs, &pVfs, &zFile, &zErr); + pResult = sqlite3_str_new(0); + if( pResult ){ + int i; + sqlite3_str_appendf(pResult, "rc=%d", rc); + sqlite3_str_appendf(pResult, ", flags=0x%x", flgs); + sqlite3_str_appendf(pResult, ", vfs=%Q", pVfs ? pVfs->zName: 0); + sqlite3_str_appendf(pResult, ", err=%Q", zErr); + sqlite3_str_appendf(pResult, ", file=%Q", zFile); + if( zFile ){ + const char *z = zFile; + z += sqlite3Strlen30(z)+1; + while( z[0] ){ + sqlite3_str_appendf(pResult, ", %Q", z); + z += sqlite3Strlen30(z)+1; + } + for(i=2; ihtsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey) % new_size; next_elem = elem->next; - insertElement(pH, &new_ht[h], elem); + insertElement(pH, &new_ht[elem->h % new_size], elem); } return 1; } @@ -152,23 +158,22 @@ static HashElem *findElementWithHash( HashElem *elem; /* Used to loop thru the element list */ unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - static HashElem nullElement = { 0, 0, 0, 0 }; + static HashElem nullElement = { 0, 0, 0, 0, 0 }; + h = strHash(pKey); if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; - h = strHash(pKey) % pH->htsize; - pEntry = &pH->ht[h]; + pEntry = &pH->ht[h % pH->htsize]; elem = pEntry->chain; count = pEntry->count; }else{ - h = 0; elem = pH->first; count = pH->count; } if( pHash ) *pHash = h; while( count ){ assert( elem!=0 ); - if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ + if( h==elem->h && sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; @@ -180,10 +185,9 @@ static HashElem *findElementWithHash( /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ -static void removeElementGivenHash( +static void removeElement( Hash *pH, /* The pH containing "elem" */ - HashElem* elem, /* The element to be removed from the pH */ - unsigned int h /* Hash value for the element */ + HashElem *elem /* The element to be removed from the pH */ ){ struct _ht *pEntry; if( elem->prev ){ @@ -195,7 +199,7 @@ static void removeElementGivenHash( elem->next->prev = elem->prev; } if( pH->ht ){ - pEntry = &pH->ht[h]; + pEntry = &pH->ht[elem->h % pH->htsize]; if( pEntry->chain==elem ){ pEntry->chain = elem->next; } @@ -246,7 +250,7 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ if( elem->data ){ void *old_data = elem->data; if( data==0 ){ - removeElementGivenHash(pH,elem,h); + removeElement(pH,elem); }else{ elem->data = data; elem->pKey = pKey; @@ -257,14 +261,13 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; new_elem->pKey = pKey; + new_elem->h = h; new_elem->data = data; pH->count++; - if( pH->count>=10 && pH->count > 2*pH->htsize ){ - if( rehash(pH, pH->count*2) ){ - assert( pH->htsize>0 ); - h = strHash(pKey) % pH->htsize; - } + if( pH->count>=5 && pH->count > 2*pH->htsize ){ + rehash(pH, pH->count*3); } - insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); + insertElement(pH, pH->ht ? &pH->ht[new_elem->h % pH->htsize] : 0, new_elem); return 0; } + diff --git a/src/hash.h b/src/hash.h index 3f491e45c0..cff65d6e50 100644 --- a/src/hash.h +++ b/src/hash.h @@ -60,6 +60,7 @@ struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ const char *pKey; /* Key associated with this element */ + unsigned int h; /* hash for pKey */ }; /* diff --git a/src/hwtime.h b/src/hwtime.h index d27204a69c..f808fa40eb 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -17,7 +17,7 @@ #define SQLITE_HWTIME_H /* -** The following routine only works on pentium-class (or newer) processors. +** The following routine only works on Pentium-class (or newer) processors. ** It uses the RDTSC opcode to read the cycle count value out of the ** processor and returns that value. This can be used for high-res ** profiling. diff --git a/src/insert.c b/src/insert.c index 423af77234..f0c56a7a8f 100644 --- a/src/insert.c +++ b/src/insert.c @@ -15,7 +15,7 @@ #include "sqliteInt.h" /* -** Generate code that will +** Generate code that will ** ** (1) acquire a lock for table pTab then ** (2) open pTab as cursor iCur. @@ -35,8 +35,10 @@ void sqlite3OpenTable( assert( pParse->pVdbe!=0 ); v = pParse->pVdbe; assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(pParse, iDb, pTab->tnum, - (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( !pParse->db->noSharedCache ){ + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + } if( HasRowid(pTab) ){ sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); VdbeComment((v, "%s", pTab->zName)); @@ -52,7 +54,7 @@ void sqlite3OpenTable( /* ** Return a pointer to the column affinity string associated with index -** pIdx. A column affinity string has one character for each column in +** pIdx. A column affinity string has one character for each column in ** the table, according to the affinity of the column: ** ** Character Column affinity @@ -131,7 +133,7 @@ char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ zColAff[j--] = 0; }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); } - return zColAff; + return zColAff; } /* @@ -165,7 +167,7 @@ char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ ** For STRICT tables: ** ------------------ ** -** Generate an appropropriate OP_TypeCheck opcode that will verify the +** Generate an appropriate OP_TypeCheck opcode that will verify the ** datatypes against the column definitions in pTab. If iReg==0, that ** means an OP_MakeRecord opcode has already been generated and should be ** the last opcode generated. The new OP_TypeCheck needs to be inserted @@ -183,12 +185,15 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ ** by one slot and insert a new OP_TypeCheck where the current ** OP_MakeRecord is found */ VdbeOp *pPrev; + int p3; sqlite3VdbeAppendP4(v, pTab, P4_TABLE); pPrev = sqlite3VdbeGetLastOp(v); assert( pPrev!=0 ); assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); pPrev->opcode = OP_TypeCheck; - sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, pPrev->p3); + p3 = pPrev->p3; + pPrev->p3 = 0; + sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, p3); }else{ /* Insert an isolated OP_Typecheck */ sqlite3VdbeAddOp2(v, OP_TypeCheck, iReg, pTab->nNVCol); @@ -220,9 +225,9 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ /* ** Return non-zero if the table pTab in database iDb or any of its indices -** have been opened at any point in the VDBE program. This is used to see if -** a statement of the form "INSERT INTO SELECT ..." can -** run without using a temporary table for the results of the SELECT. +** have been opened at any point in the VDBE program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using a temporary table for the results of the SELECT. */ static int readsTable(Parse *p, int iDb, Table *pTab){ Vdbe *v = sqlite3GetVdbe(p); @@ -450,7 +455,7 @@ static int autoIncBegin( /* ** This routine generates code that will initialize all of the -** register used by the autoincrement tracker. +** register used by the autoincrement tracker. */ void sqlite3AutoincrementBegin(Parse *pParse){ AutoincInfo *p; /* Information about an AUTOINCREMENT */ @@ -479,7 +484,7 @@ void sqlite3AutoincrementBegin(Parse *pParse){ /* 8 */ {OP_Goto, 0, 11, 0}, /* 9 */ {OP_Next, 0, 2, 0}, /* 10 */ {OP_Integer, 0, 0, 0}, - /* 11 */ {OP_Close, 0, 0, 0} + /* 11 */ {OP_Close, 0, 0, 0} }; VdbeOp *aOp; pDb = &db->aDb[p->iDb]; @@ -575,6 +580,210 @@ void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + assert( (pItem->fg.isSubquery && pItem->u4.pSubq!=0) || pParse->nErr ); + if( pItem->fg.isSubquery ){ + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); + } + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~(u32)SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + Subquery *pSubq; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->fg.viaCoroutine = 1; + p->iCursor = -1; + assert( !p->fg.isIndexedBy && !p->fg.isTabFunc ); + p->u1.nRow = 2; + if( sqlite3SrcItemAttachSubquery(pParse, p, pLeft, 0) ){ + pSubq = p->u4.pSubq; + pSubq->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, + pSubq->regReturn, 0, pSubq->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + pSubq->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + } + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + Subquery *pSubq; + assert( p!=0 ); + assert( p->fg.isSubquery ); + pSubq = p->u4.pSubq; + assert( pSubq!=0 ); + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + if( pSubq->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, pSubq->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, pSubq->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, pSubq->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -721,6 +930,7 @@ void sqlite3Insert( int regRowid; /* registers holding insert rowid */ int regData; /* register holding first column to insert */ int *aRegIdx = 0; /* One register allocated to each index */ + int *aTabColMap = 0; /* Mapping from pTab columns to pCol entries */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to insert into a view */ @@ -817,7 +1027,7 @@ void sqlite3Insert( ** ** This is the 2nd template. */ - if( pColumn==0 + if( pColumn==0 && pSelect!=0 && pTrigger==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) @@ -845,7 +1055,7 @@ void sqlite3Insert( regData = regRowid+1; /* If the INSERT statement included an IDLIST term, then make sure - ** all elements of the IDLIST really are columns of the table and + ** all elements of the IDLIST really are columns of the table and ** remember the column indices. ** ** If the table has an INTEGER PRIMARY KEY column and that column @@ -865,31 +1075,25 @@ void sqlite3Insert( */ bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; if( pColumn ){ - assert( pColumn->eU4!=EU4_EXPR ); - pColumn->eU4 = EU4_IDX; + aTabColMap = sqlite3DbMallocZero(db, pTab->nCol*sizeof(int)); + if( aTabColMap==0 ) goto insert_cleanup; for(i=0; inId; i++){ - pColumn->a[i].u4.idx = -1; - } - for(i=0; inId; i++){ - for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zCnName)==0 ){ - pColumn->a[i].u4.idx = j; - if( i!=j ) bIdListInOrder = 0; - if( j==pTab->iPKey ){ - ipkColumn = i; assert( !withoutRowid ); - } + j = sqlite3ColumnIndex(pTab, pColumn->a[i].zName); + if( j>=0 ){ + if( aTabColMap[j]==0 ) aTabColMap[j] = i+1; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } #ifndef SQLITE_OMIT_GENERATED_COLUMNS - if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ - sqlite3ErrorMsg(pParse, - "cannot INSERT into generated column \"%s\"", - pTab->aCol[j].zCnName); - goto insert_cleanup; - } -#endif - break; + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zCnName); + goto insert_cleanup; } - } - if( j>=pTab->nCol ){ +#endif + }else{ if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ ipkColumn = i; bIdListInOrder = 0; @@ -911,25 +1115,45 @@ void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - assert( db->pParse==pParse ); - if( rc || pParse->nErr ) goto insert_cleanup; - assert( db->mallocFailed==0 ); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + Subquery *pSubq; + assert( pItem->fg.isSubquery ); + pSubq = pItem->u4.pSubq; + dest.iSDParm = pSubq->regReturn; + regFromSelect = pSubq->regResult; + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + nColumn = pSubq->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -937,7 +1161,7 @@ void sqlite3Insert( ** the destination table (template 3). ** ** A temp table must be used if the table being updated is also one - ** of the tables being read by the SELECT statement. Also use a + ** of the tables being read by the SELECT statement. Also use a ** temp table in the case of row triggers. */ if( pTrigger || readsTable(pParse, iDb, pTab) ){ @@ -973,7 +1197,7 @@ void sqlite3Insert( sqlite3ReleaseTempReg(pParse, regTempRowid); } }else{ - /* This is the case if the data for the INSERT is coming from a + /* This is the case if the data for the INSERT is coming from a ** single-row VALUES clause */ NameContext sNC; @@ -992,7 +1216,7 @@ void sqlite3Insert( } /* If there is no IDLIST term but the table has an integer primary - ** key, the set the ipkColumn variable to the integer primary key + ** key, the set the ipkColumn variable to the integer primary key ** column index in the original table definition. */ if( pColumn==0 && nColumn>0 ){ @@ -1023,7 +1247,7 @@ void sqlite3Insert( } } if( nColumn!=(pTab->nCol-nHidden) ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "table %S has %d columns but %d values were supplied", pTabList->a, pTab->nCol-nHidden, nColumn); goto insert_cleanup; @@ -1033,7 +1257,7 @@ void sqlite3Insert( sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); goto insert_cleanup; } - + /* Initialize the count of rows to be inserted */ if( (db->flags & SQLITE_CountRows)!=0 @@ -1084,7 +1308,7 @@ void sqlite3Insert( pNx->iDataCur = iDataCur; pNx->iIdxCur = iIdxCur; if( pNx->pUpsertTarget ){ - if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ goto insert_cleanup; } } @@ -1167,28 +1391,28 @@ void sqlite3Insert( continue; }else if( pColumn==0 ){ /* Hidden columns that are not explicitly named in the INSERT - ** get there default value */ - sqlite3ExprCodeFactorable(pParse, + ** get their default value */ + sqlite3ExprCodeFactorable(pParse, sqlite3ColumnExpr(pTab, &pTab->aCol[i]), iRegStore); continue; } } if( pColumn ){ - assert( pColumn->eU4==EU4_IDX ); - for(j=0; jnId && pColumn->a[j].u4.idx!=i; j++){} - if( j>=pColumn->nId ){ + j = aTabColMap[i]; + assert( j>=0 && j<=pColumn->nId ); + if( j==0 ){ /* A column not named in the insert column list gets its ** default value */ - sqlite3ExprCodeFactorable(pParse, + sqlite3ExprCodeFactorable(pParse, sqlite3ColumnExpr(pTab, &pTab->aCol[i]), iRegStore); continue; } - k = j; + k = j - 1; }else if( nColumn==0 ){ /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ - sqlite3ExprCodeFactorable(pParse, + sqlite3ExprCodeFactorable(pParse, sqlite3ColumnExpr(pTab, &pTab->aCol[i]), iRegStore); continue; @@ -1197,7 +1421,7 @@ void sqlite3Insert( } if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); + sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); }else if( pSelect ){ if( regFromSelect!=regData ){ sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); @@ -1268,7 +1492,7 @@ void sqlite3Insert( } /* Fire BEFORE or INSTEAD OF triggers */ - sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, pTab, regCols-pTab->nCol-1, onError, endOfLoop); sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); @@ -1353,7 +1577,7 @@ void sqlite3Insert( ** constraints or (b) there are no triggers and this table is not a ** parent table in a foreign key constraint. It is safe to set the ** flag in the second case as if any REPLACE constraint is hit, an - ** OP_Delete or OP_IdxDelete instruction will be executed on each + ** OP_Delete or OP_IdxDelete instruction will be executed on each ** cursor that is disturbed. And these instructions both clear the ** VdbeCursor.seekResult variable, disabling the OPFLAG_USESEEKRESULT ** functionality. */ @@ -1379,7 +1603,7 @@ void sqlite3Insert( if( pTrigger ){ /* Code AFTER triggers */ - sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, pTab, regData-2-pTab->nCol, onError, endOfLoop); } @@ -1417,7 +1641,7 @@ void sqlite3Insert( } /* - ** Return the number of rows inserted. If this routine is + ** Return the number of rows inserted. If this routine is ** generating code because of a call to sqlite3NestedParse(), do not ** invoke the callback function. */ @@ -1430,7 +1654,10 @@ void sqlite3Insert( sqlite3ExprListDelete(db, pList); sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); - sqlite3IdListDelete(db, pColumn); + if( pColumn ){ + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aTabColMap); + } if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } @@ -1448,7 +1675,7 @@ void sqlite3Insert( #endif /* -** Meanings of bits in of pWalker->eCode for +** Meanings of bits in of pWalker->eCode for ** sqlite3ExprReferencesUpdatedColumn() */ #define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ @@ -1457,7 +1684,7 @@ void sqlite3Insert( /* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). * Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this ** expression node references any of the -** columns that are being modifed by an UPDATE statement. +** columns that are being modified by an UPDATE statement. */ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_COLUMN ){ @@ -1529,7 +1756,7 @@ struct IndexIterator { union { struct { /* Use this object for eType==0: A Index.pNext list */ Index *pIdx; /* The current Index */ - } lx; + } lx; struct { /* Use this object for eType==1; Array of IndexListTerm */ int nIdx; /* Size of the array */ IndexListTerm *aIdx; /* Array of IndexListTerms */ @@ -1573,7 +1800,7 @@ static Index *indexIteratorNext(IndexIterator *pIter, int *pIx){ return pIter->u.lx.pIdx; } } - + /* ** Generate code to do constraint checks prior to an INSERT or an UPDATE ** on table pTab. @@ -1680,7 +1907,7 @@ void sqlite3GenerateConstraintChecks( int *aiChng, /* column i is unchanged if aiChng[i]<0 */ Upsert *pUpsert /* ON CONFLICT clauses, if any. NULL otherwise */ ){ - Vdbe *v; /* VDBE under constrution */ + Vdbe *v; /* VDBE under construction */ Index *pIdx; /* Pointer to one of the indices */ Index *pPk = 0; /* The PRIMARY KEY index for WITHOUT ROWID tables */ sqlite3 *db; /* Database connection */ @@ -1712,9 +1939,9 @@ void sqlite3GenerateConstraintChecks( assert( v!=0 ); assert( !IsView(pTab) ); /* This table is not a VIEW */ nCol = pTab->nCol; - + /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for - ** normal rowid tables. nPkField is the number of key fields in the + ** normal rowid tables. nPkField is the number of key fields in the ** pPk index or 1 for a rowid table. In other words, nPkField is the ** number of fields in the true primary key of the table. */ if( HasRowid(pTab) ){ @@ -1889,7 +2116,7 @@ void sqlite3GenerateConstraintChecks( ** could happen in any order, but they are grouped up front for ** convenience. ** - ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** 2018-08-14: Ticket https://sqlite.org/src/info/908f001483982c43 ** The order of constraints used to have OE_Update as (2) and OE_Abort ** and so forth as (1). But apparently PostgreSQL checks the OE_Update ** constraint before any others, so it had to be moved. @@ -2091,10 +2318,10 @@ void sqlite3GenerateConstraintChecks( ** the triggers and remove both the table and index b-tree entries. ** ** Otherwise, if there are no triggers or the recursive-triggers - ** flag is not set, but the table has one or more indexes, call - ** GenerateRowIndexDelete(). This removes the index b-tree entries - ** only. The table b-tree entry will be replaced by the new entry - ** when it is inserted. + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. ** ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, ** also invoke MultiWrite() to indicate that this VDBE may require @@ -2163,7 +2390,7 @@ void sqlite3GenerateConstraintChecks( pIdx; pIdx = indexIteratorNext(&sIdxIter, &ix) ){ - int regIdx; /* Range of registers hold conent for pIdx */ + int regIdx; /* Range of registers holding content for pIdx */ int regR; /* Range of registers holding conflicting PK */ int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ @@ -2226,7 +2453,7 @@ void sqlite3GenerateConstraintChecks( #endif sqlite3VdbeReleaseRegisters(pParse, regIdx, pIdx->nColumn, 0, 0); - /* In an UPDATE operation, if this index is the PRIMARY KEY index + /* In an UPDATE operation, if this index is the PRIMARY KEY index ** of a WITHOUT ROWID table and there has been no change the ** primary key, then no collision is possible. The collision detection ** logic below can all be skipped. */ @@ -2237,7 +2464,7 @@ void sqlite3GenerateConstraintChecks( /* Find out what action to take in case there is a uniqueness conflict */ onError = pIdx->onError; - if( onError==OE_None ){ + if( onError==OE_None ){ sqlite3VdbeResolveLabel(v, addrUniqueOk); continue; /* pIdx is not a UNIQUE index */ } @@ -2283,7 +2510,7 @@ void sqlite3GenerateConstraintChecks( /* Check to see if the new index entry will be unique */ sqlite3VdbeVerifyAbortable(v, onError); - addrConflictCk = + addrConflictCk = sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, regIdx, pIdx->nKeyCol); VdbeCoverage(v); @@ -2313,7 +2540,7 @@ void sqlite3GenerateConstraintChecks( } } if( isUpdate ){ - /* If currently processing the PRIMARY KEY of a WITHOUT ROWID + /* If currently processing the PRIMARY KEY of a WITHOUT ROWID ** table, only conflict if the new PRIMARY KEY values are actually ** different from the old. See TH3 withoutrowid04.test. ** @@ -2323,7 +2550,7 @@ void sqlite3GenerateConstraintChecks( int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; int op = OP_Ne; int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); - + for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; @@ -2333,7 +2560,7 @@ void sqlite3GenerateConstraintChecks( op = OP_Eq; } x = sqlite3TableColumnToStorage(pTab, x); - sqlite3VdbeAddOp4(v, op, + sqlite3VdbeAddOp4(v, op, regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ ); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); @@ -2443,7 +2670,7 @@ void sqlite3GenerateConstraintChecks( } sqlite3VdbeResolveLabel(v, addrUniqueOk); if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); - if( pUpsertClause + if( pUpsertClause && upsertIpkReturn && sqlite3UpsertNextIsIPK(pUpsertClause) ){ @@ -2658,6 +2885,8 @@ int sqlite3OpenTableAndIndices( assert( op==OP_OpenRead || op==OP_OpenWrite ); assert( op==OP_OpenWrite || p5==0 ); + assert( piDataCur!=0 ); + assert( piIdxCur!=0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output ** variables *piDataCur and *piIdxCur set to illegal cursor numbers @@ -2670,18 +2899,18 @@ int sqlite3OpenTableAndIndices( assert( v!=0 ); if( iBase<0 ) iBase = pParse->nTab; iDataCur = iBase++; - if( piDataCur ) *piDataCur = iDataCur; + *piDataCur = iDataCur; if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); - }else{ + }else if( pParse->db->noSharedCache==0 ){ sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); } - if( piIdxCur ) *piIdxCur = iBase; + *piIdxCur = iBase; for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; + *piDataCur = iIdxCur; p5 = 0; } if( aToOpen==0 || aToOpen[i+1] ){ @@ -2760,7 +2989,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** ** INSERT INTO tab1 SELECT * FROM tab2; ** -** The xfer optimization transfers raw records from tab2 over to tab1. +** The xfer optimization transfers raw records from tab2 over to tab1. ** Columns are not decoded and reassembled, which greatly improves ** performance. Raw index records are transferred in the same way. ** @@ -2823,7 +3052,7 @@ static int xferOptimization( if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } - if( pSelect->pSrc->a[0].pSelect ){ + if( pSelect->pSrc->a[0].fg.isSubquery ){ return 0; /* FROM clause cannot contain a subquery */ } if( pSelect->pWhere ){ @@ -2888,8 +3117,8 @@ static int xferOptimization( Column *pDestCol = &pDest->aCol[i]; Column *pSrcCol = &pSrc->aCol[i]; #ifdef SQLITE_ENABLE_HIDDEN_COLUMNS - if( (db->mDbFlags & DBFLAG_Vacuum)==0 - && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN ){ return 0; /* Neither table may have __hidden__ columns */ } @@ -2906,7 +3135,7 @@ static int xferOptimization( ** ** Nevertheless, this is a useful notational shorthand to tell SQLite ** to do a bulk transfer all of the content from t1 over to t2. - ** + ** ** We could, in theory, disable this (except for internal use by the ** VACUUM command where it is actually needed). But why do that? It ** seems harmless enough, and provides a useful service. @@ -2932,7 +3161,7 @@ static int xferOptimization( if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( sqlite3_stricmp(sqlite3ColumnColl(pDestCol), + if( sqlite3_stricmp(sqlite3ColumnColl(pDestCol), sqlite3ColumnColl(pSrcCol))!=0 ){ return 0; /* Collating sequence must be the same on all columns */ } @@ -2947,7 +3176,7 @@ static int xferOptimization( assert( pDestExpr==0 || !ExprHasProperty(pDestExpr, EP_IntValue) ); assert( pSrcExpr==0 || pSrcExpr->op==TK_SPAN ); assert( pSrcExpr==0 || !ExprHasProperty(pSrcExpr, EP_IntValue) ); - if( (pDestExpr==0)!=(pSrcExpr==0) + if( (pDestExpr==0)!=(pSrcExpr==0) || (pDestExpr!=0 && strcmp(pDestExpr->u.zToken, pSrcExpr->u.zToken)!=0) ){ @@ -2974,14 +3203,17 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif #ifndef SQLITE_OMIT_FOREIGN_KEY - /* Disallow the transfer optimization if the destination table constains + /* Disallow the transfer optimization if the destination table contains ** any foreign key constraints. This is more restrictive than necessary. - ** But the main beneficiary of the transfer optimization is the VACUUM + ** But the main beneficiary of the transfer optimization is the VACUUM ** command, and the VACUUM command disables foreign key constraints. So ** the extra complication to make this rule less restrictive is probably ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] @@ -3030,7 +3262,7 @@ static int xferOptimization( ** (If the destination is not initially empty, the rowid fields ** of index entries might need to change.) ** - ** (2) The destination has a unique index. (The xfer optimization + ** (2) The destination has a unique index. (The xfer optimization ** is unable to test uniqueness.) ** ** (3) onError is something other than OE_Abort and OE_Rollback. @@ -3106,14 +3338,14 @@ static int xferOptimization( /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use ** collation sequence BINARY, then it can also be assumed that the - ** index will be populated by inserting keys in strictly sorted + ** index will be populated by inserting keys in strictly sorted ** order. In this case, instead of seeking within the b-tree as part ** of every OP_IdxInsert opcode, an OP_SeekEnd is added before the - ** OP_IdxInsert to seek to the point within the b-tree where each key + ** OP_IdxInsert to seek to the point within the b-tree where each key ** should be inserted. This is faster. ** ** If any of the indexed columns use a collation sequence other than - ** BINARY, this optimization is disabled. This is because the user + ** BINARY, this optimization is disabled. This is because the user ** might change the definition of a collation sequence and then run ** a VACUUM command. In that case keys may not be written in strictly ** sorted order. */ @@ -3131,9 +3363,9 @@ static int xferOptimization( } if( idxInsFlags!=(OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT) ){ sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); - if( (db->mDbFlags & DBFLAG_Vacuum)==0 - && !HasRowid(pDest) - && IsPrimaryKeyIndex(pDestIdx) + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && !HasRowid(pDest) + && IsPrimaryKeyIndex(pDestIdx) ){ codeWithoutRowidPreupdate(pParse, pDest, iDest, regData); } diff --git a/src/json.c b/src/json.c index 8735634c84..d0d3c53a23 100644 --- a/src/json.c +++ b/src/json.c @@ -10,59 +10,295 @@ ** ****************************************************************************** ** -** This SQLite JSON functions. +** SQLite JSON functions. ** ** This file began as an extension in ext/misc/json1.c in 2015. That ** extension proved so useful that it has now been moved into the core. ** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) +** The original design stored all JSON as pure text, canonical RFC-8259. +** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16). +** All generated JSON text still conforms strictly to RFC-8259, but text +** with JSON-5 extensions is accepted as input. +** +** Beginning with version 3.45.0 (circa 2024-01-01), these routines also +** accept BLOB values that have JSON encoded using a binary representation +** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk +** format for SQLite-JSONB is completely different and incompatible with +** PostgreSQL-JSONB. +** +** Decoding and interpreting JSONB is still O(N) where N is the size of +** the input, the same as text JSON. However, the constant of proportionality +** for JSONB is much smaller due to faster parsing. The size of each +** element in JSONB is encoded in its header, so there is no need to search +** for delimiters using persnickety syntax rules. JSONB seems to be about +** 3x faster than text JSON as a result. JSONB is also tends to be slightly +** smaller than text JSON, by 5% or 10%, but there are corner cases where +** JSONB can be slightly larger. So you are not far mistaken to say that +** a JSONB blob is the same size as the equivalent RFC-8259 text. +** +** +** THE JSONB ENCODING: +** +** Every JSON element is encoded in JSONB as a header and a payload. +** The header is between 1 and 9 bytes in size. The payload is zero +** or more bytes. +** +** The lower 4 bits of the first byte of the header determines the +** element type: +** +** 0: NULL +** 1: TRUE +** 2: FALSE +** 3: INT -- RFC-8259 integer literal +** 4: INT5 -- JSON5 integer literal +** 5: FLOAT -- RFC-8259 floating point literal +** 6: FLOAT5 -- JSON5 floating point literal +** 7: TEXT -- Text literal acceptable to both SQL and JSON +** 8: TEXTJ -- Text containing RFC-8259 escapes +** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes +** 10: TEXTRAW -- Text containing unescaped syntax characters +** 11: ARRAY +** 12: OBJECT +** +** The other three possible values (13-15) are reserved for future +** enhancements. +** +** The upper 4 bits of the first byte determine the size of the header +** and sometimes also the size of the payload. If X is the first byte +** of the element and if X>>4 is between 0 and 11, then the payload +** will be that many bytes in size and the header is exactly one byte +** in size. Other four values for X>>4 (12-15) indicate that the header +** is more than one byte in size and that the payload size is determined +** by the remainder of the header, interpreted as a unsigned big-endian +** integer. +** +** Value of X>>4 Size integer Total header size +** ------------- -------------------- ----------------- +** 12 1 byte (0-255) 2 +** 13 2 byte (0-65535) 3 +** 14 4 byte (0-4294967295) 5 +** 15 8 byte (0-1.8e19) 9 +** +** The payload size need not be expressed in its minimal form. For example, +** if the payload size is 10, the size can be expressed in any of 5 different +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by one 0x0a byte, +** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by +** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and +** a single byte of 0x0a. The shorter forms are preferred, of course, but +** sometimes when generating JSONB, the payload size is not known in advance +** and it is convenient to reserve sufficient header space to cover the +** largest possible payload size and then come back later and patch up +** the size when it becomes known, resulting in a non-minimal encoding. +** +** The value (X>>4)==15 is not actually used in the current implementation +** (as SQLite is currently unable to handle BLOBs larger than about 2GB) +** but is included in the design to allow for future enhancements. +** +** The payload follows the header. NULL, TRUE, and FALSE have no payload and +** their payload size must always be zero. The payload for INT, INT5, +** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the +** "..." or '...' delimiters are omitted from the various text encodings. +** The payload for ARRAY and OBJECT is a list of additional elements that +** are the content for the array or object. The payload for an OBJECT +** must be an even number of elements. The first element of each pair is +** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW. +** +** A valid JSONB blob consists of a single element, as described above. +** Usually this will be an ARRAY or OBJECT element which has many more +** elements as its content. But the overall blob is just a single element. +** +** Input validation for JSONB blobs simply checks that the element type +** code is between 0 and 12 and that the total size of the element +** (header plus payload) is the same as the size of the BLOB. If those +** checks are true, the BLOB is assumed to be JSONB and processing continues. +** Errors are only raised if some other miscoding is discovered during +** processing. +** +** Additional information can be found in the doc/jsonb.md file of the +** canonical SQLite source tree. */ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" +/* JSONB element types +*/ +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ + +/* Human-readable names for the JSONB values. The index for each +** string must correspond to the JSONB_* integer above. +*/ +static const char * const jsonbType[] = { + "null", "true", "false", "integer", "integer", + "real", "real", "text", "text", "text", + "text", "array", "object", "", "", "", "" +}; + /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +#ifdef SQLITE_ASCII +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */ +#endif +#ifdef SQLITE_EBCDIC +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */ +#endif + }; -#define fast_isspace(x) (jsonIsSpace[(unsigned char)x]) +#define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) -# define VVA(X) -#else -# define VVA(X) X +/* +** The set of all space characters recognized by jsonIsspace(). +** Useful as the second argument to strspn(). +*/ +#ifdef SQLITE_ASCII +static const char jsonSpaces[] = "\011\012\015\040"; +#endif +#ifdef SQLITE_EBCDIC +static const char jsonSpaces[] = "\005\045\015\100"; +#endif + + +/* +** Characters that are special to JSON. Control characters, +** '"' and '\\' and '\''. Actually, '\'' is not special to +** canonical JSON, but it is special in JSON-5, so we include +** it in the set of special characters. +*/ +static const char jsonIsOk[256] = { +#ifdef SQLITE_ASCII +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, /* 2 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 3 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, /* 5 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 7 */ + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */ +#endif +#ifdef SQLITE_EBCDIC +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, /* 3 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 5 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, /* 7 */ + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */ #endif +}; /* Objects */ +typedef struct JsonCache JsonCache; typedef struct JsonString JsonString; -typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* Cache entry */ +#define JSON_CACHE_SIZE 4 /* Max number of cache entries */ + +/* +** jsonUnescapeOneChar() returns this invalid code point if it encounters +** a syntax error. +*/ +#define JSON_INVALID_CHAR 0x99999 + +/* A cache mapping JSON text into JSONB blobs. +** +** Each cache entry is a JsonParse object with the following restrictions: +** +** * The bReadOnly flag must be set +** +** * The aBlob[] array must be owned by the JsonParse object. In other +** words, nBlobAlloc must be non-zero. +** +** * eEdit and delta must be zero. +** +** * zJson must be an RCStr. In other words bJsonIsRCStr must be true. +*/ +struct JsonCache { + sqlite3 *db; /* Database connection */ + int nUsed; /* Number of active entries in the cache */ + JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */ +}; + /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. +** +** If the generated string is longer than will fit into the zSpace[] buffer, +** then it will be an RCStr string. This aids with caching of large +** JSON strings. */ struct JsonString { sqlite3_context *pCtx; /* Function context - put error messages here */ @@ -70,76 +306,75 @@ struct JsonString { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 bErr; /* True if an error has been encountered */ + u8 eErr; /* True if an error has been encountered */ char zSpace[100]; /* Initial static space */ }; -/* JSON type values -*/ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 +/* Allowed values for JsonString.eErr */ +#define JSTRING_OOM 0x01 /* Out of memory */ +#define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ +#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ -/* The "subtype" set for JSON values */ +/* The "subtype" set for text JSON values passed through using +** sqlite3_result_subtype() and sqlite3_value_subtype(). +*/ #define JSON_SUBTYPE 74 /* Ascii for "J" */ /* -** Names of the various JSON types: +** Bit values for the flags passed into various SQL function implementations +** via the sqlite3_user_data() value. */ -static const char * const jsonType[] = { - "null", "true", "false", "integer", "real", "text", "array", "object" -}; +#define JSON_JSON 0x01 /* Result is always JSON */ +#define JSON_SQL 0x02 /* Result is always SQL */ +#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +#define JSON_BLOB 0x08 /* Use the BLOB output format */ -/* Bit values for the JsonNode.jnFlag field -*/ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ -#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ - - -/* A single node of parsed JSON -*/ -struct JsonNode { - u8 eType; /* One of the JSON_ type values */ - u8 jnFlags; /* JNODE flags */ - u8 eU; /* Which union element to use */ - u32 n; /* Bytes of content, or number of sub-nodes */ - union { - const char *zJContent; /* 1: Content for INT, REAL, and STRING */ - u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ - u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - u32 iReplace; /* 4: Replacement content for JNODE_REPLACE */ - JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ - } u; -}; -/* A completely parsed JSON string +/* A parsed JSON value. Lifecycle: +** +** 1. JSON comes in and is parsed into a JSONB value in aBlob. The +** original text is stored in zJson. This step is skipped if the +** input is JSONB instead of text JSON. +** +** 2. The aBlob[] array is searched using the JSON path notation, if needed. +** +** 3. Zero or more changes are made to aBlob[] (via json_remove() or +** json_replace() or json_patch() or similar). +** +** 4. New JSON text is generated from the aBlob[] for output. This step +** is skipped if the function is one of the jsonb_* functions that +** returns JSONB instead of text JSON. */ struct JsonParse { - u32 nNode; /* Number of slots of aNode[] used */ - u32 nAlloc; /* Number of slots of aNode[] allocated */ - JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ - u32 *aUp; /* Index of parent of each node */ + u8 *aBlob; /* JSONB representation of JSON value */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ + char *zJson; /* Json text used for parsing */ + sqlite3 *db; /* The database connection to which this object belongs */ + int nJson; /* Length of the zJson string in bytes */ + u32 nJPRef; /* Number of references to this object */ + u32 iErr; /* Error location in zJson[] */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ - int nJson; /* Length of the zJson string in bytes */ - u32 iErr; /* Error location in zJson[] */ - u32 iHold; /* Replace cache line with the lowest iHold value */ + u8 bReadOnly; /* Do not modify. */ + /* Search and edit information. See jsonLookupStep() */ + u8 eEdit; /* Edit operation to apply */ + int delta; /* Size change due to the edit */ + u32 nIns; /* Number of bytes to insert */ + u32 iLabel; /* Location of label if search landed on an object value */ + u8 *aIns; /* Content to be inserted */ }; +/* Allowed values for JsonParse.eEdit */ +#define JEDIT_DEL 1 /* Delete if exists */ +#define JEDIT_REPL 2 /* Overwrite if exists */ +#define JEDIT_INS 3 /* Insert if not exists */ +#define JEDIT_SET 4 /* Insert or overwrite */ + /* ** Maximum nesting depth of JSON for this implementation. ** @@ -147,15 +382,151 @@ struct JsonParse { ** descent parser. A depth of 1000 is far deeper than any sane JSON ** should go. Historical note: This limit was 2000 prior to version 3.42.0 */ -#define JSON_MAX_DEPTH 1000 +#ifndef SQLITE_JSON_MAX_DEPTH +# define JSON_MAX_DEPTH 1000 +#else +# define JSON_MAX_DEPTH SQLITE_JSON_MAX_DEPTH +#endif + +/* +** Allowed values for the flgs argument to jsonParseFuncArg(); +*/ +#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ +#define JSON_KEEPERROR 0x02 /* Return non-NULL even if there is an error */ + +/************************************************************************** +** Forward references +**************************************************************************/ +static void jsonReturnStringAsBlob(JsonString*); +static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); +static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); +static void jsonReturnParse(sqlite3_context*,JsonParse*); +static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); +static void jsonParseFree(JsonParse*); +static u32 jsonbPayloadSize(const JsonParse*, u32, u32*); +static u32 jsonUnescapeOneChar(const char*, u32, u32*); + +/************************************************************************** +** Utility routines for dealing with JsonCache objects +**************************************************************************/ + +/* +** Free a JsonCache object. +*/ +static void jsonCacheDelete(JsonCache *p){ + int i; + for(i=0; inUsed; i++){ + jsonParseFree(p->a[i]); + } + sqlite3DbFree(p->db, p); +} +static void jsonCacheDeleteGeneric(void *p){ + jsonCacheDelete((JsonCache*)p); +} + +/* +** Insert a new entry into the cache. If the cache is full, expel +** the least recently used entry. Return SQLITE_OK on success or a +** result code otherwise. +** +** Cache entries are stored in age order, oldest first. +*/ +static int jsonCacheInsert( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + JsonParse *pParse /* The parse object to be added to the cache */ +){ + JsonCache *p; + + assert( pParse->zJson!=0 ); + assert( pParse->bJsonIsRCStr ); + assert( pParse->delta==0 ); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + p->db = db; + sqlite3_set_auxdata(ctx, JSON_CACHE_ID, p, jsonCacheDeleteGeneric); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ) return SQLITE_NOMEM; + } + if( p->nUsed >= JSON_CACHE_SIZE ){ + jsonParseFree(p->a[0]); + memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); + p->nUsed = JSON_CACHE_SIZE-1; + } + assert( pParse->nBlobAlloc>0 ); + pParse->eEdit = 0; + pParse->nJPRef++; + pParse->bReadOnly = 1; + p->a[p->nUsed] = pParse; + p->nUsed++; + return SQLITE_OK; +} + +/* +** Search for a cached translation the json text supplied by pArg. Return +** the JsonParse object if found. Return NULL if not found. +** +** When a match if found, the matching entry is moved to become the +** most-recently used entry if it isn't so already. +** +** The JsonParse object returned still belongs to the Cache and might +** be deleted at any moment. If the caller wants the JsonParse to +** linger, it needs to increment the nPJRef reference counter. +*/ +static JsonParse *jsonCacheSearch( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + sqlite3_value *pArg /* Function argument containing SQL text */ +){ + JsonCache *p; + int i; + const char *zJson; + int nJson; + + if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){ + return 0; + } + zJson = (const char*)sqlite3_value_text(pArg); + if( zJson==0 ) return 0; + nJson = sqlite3_value_bytes(pArg); + + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + return 0; + } + for(i=0; inUsed; i++){ + if( p->a[i]->zJson==zJson ) break; + } + if( i>=p->nUsed ){ + for(i=0; inUsed; i++){ + if( p->a[i]->nJson!=nJson ) continue; + if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break; + } + } + if( inUsed ){ + if( inUsed-1 ){ + /* Make the matching entry the most recently used entry */ + JsonParse *tmp = p->a[i]; + memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); + p->a[p->nUsed-1] = tmp; + i = p->nUsed - 1; + } + assert( p->a[i]->delta==0 ); + return p->a[i]; + }else{ + return 0; + } +} /************************************************************************** ** Utility routines for dealing with JsonString objects **************************************************************************/ -/* Set the JsonString object to an empty string +/* Turn uninitialized bulk memory into a valid JsonString object +** holding a zero-length string. */ -static void jsonZero(JsonString *p){ +static void jsonStringZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; @@ -164,53 +535,51 @@ static void jsonZero(JsonString *p){ /* Initialize the JsonString object */ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ +static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->bErr = 0; - jsonZero(p); + p->eErr = 0; + jsonStringZero(p); } - /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); - jsonZero(p); +static void jsonStringReset(JsonString *p){ + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); + jsonStringZero(p); } - -/* Report an out-of-memory (OOM) condition +/* Report an out-of-memory (OOM) condition */ -static void jsonOom(JsonString *p){ - p->bErr = 1; - sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); +static void jsonStringOom(JsonString *p){ + p->eErr |= JSTRING_OOM; + if( p->pCtx ) sqlite3_result_error_nomem(p->pCtx); + jsonStringReset(p); } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(JsonString *p, u32 N){ +static int jsonStringGrow(JsonString *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ - if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); + if( p->eErr ) return 1; + zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ - jsonOom(p); + jsonStringOom(p); return SQLITE_NOMEM; } memcpy(zNew, p->zBuf, (size_t)p->nUsed); p->zBuf = zNew; p->bStatic = 0; }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); + if( p->zBuf==0 ){ + p->eErr |= JSTRING_OOM; + jsonStringZero(p); return SQLITE_NOMEM; } - p->zBuf = zNew; } p->nAlloc = nTotal; return SQLITE_OK; @@ -218,18 +587,40 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ - if( N==0 ) return; - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; +static SQLITE_NOINLINE void jsonStringExpandAndAppend( + JsonString *p, + const char *zIn, + u32 N +){ + assert( N>0 ); + if( jsonStringGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( N+p->nUsed >= p->nAlloc ){ + jsonStringExpandAndAppend(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} +static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ + assert( N>0 ); + if( N+p->nUsed >= p->nAlloc ){ + jsonStringExpandAndAppend(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} /* Append formatted text (not to exceed N bytes) to the JsonString. */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return; va_start(ap, zFormat); sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); va_end(ap); @@ -238,10 +629,38 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ -static void jsonAppendChar(JsonString *p, char c){ - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; +static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ + if( jsonStringGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc ){ + jsonAppendCharExpand(p,c); + }else{ + p->zBuf[p->nUsed++] = c; + } +} + +/* Remove a single character from the end of the string +*/ +static void jsonStringTrimOneChar(JsonString *p){ + if( p->eErr==0 ){ + assert( p->nUsed>0 ); + p->nUsed--; + } +} + + +/* Make sure there is a zero terminator on p->zBuf[] +** +** Return true on success. Return false if an OOM prevents this +** from happening. +*/ +static int jsonStringTerminate(JsonString *p){ + jsonAppendChar(p, 0); + jsonStringTrimOneChar(p); + return p->eErr==0; +} /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. @@ -250,187 +669,130 @@ static void jsonAppendSeparator(JsonString *p){ char c; if( p->nUsed==0 ) return; c = p->zBuf[p->nUsed-1]; - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); + if( c=='[' || c=='{' ) return; + jsonAppendChar(p, ','); } -/* Append the N-byte string in zIn to the end of the JsonString string -** under construction. Enclose the string in "..." and escape -** any double-quotes or backslash characters contained within the -** string. +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. */ -static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; - p->zBuf[p->nUsed++] = '"'; - for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - }else if( c<=0x1f ){ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - if( aSpecial[c] ){ - c = aSpecial[c]; - goto json_simple_escape; - } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0' + (c>>4); - c = "0123456789abcdef"[c&0xf]; - } - p->zBuf[p->nUsed++] = c; +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && cnUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; } - p->zBuf[p->nUsed++] = '"'; - assert( p->nUsednAlloc ); } -/* -** The zIn[0..N] string is a JSON5 string literal. Append to p a translation -** of the string literal that standard JSON and that omits all JSON5 -** features. +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in double-quotes ("...") and +** escape any double-quotes or backslash characters contained within the +** string. +** +** This routine is a high-runner. There is a measurable performance +** increase associated with unwinding the jsonIsOk[] loop. */ -static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ - u32 i; - jsonAppendChar(p, '"'); - zIn++; - N -= 2; - while( N>0 ){ - for(i=0; i0 ){ - jsonAppendRaw(p, zIn, i); - zIn += i; - N -= i; - if( N==0 ) break; - } - assert( zIn[0]=='\\' ); - switch( (u8)zIn[1] ){ - case '\'': - jsonAppendChar(p, '\''); - break; - case 'v': - jsonAppendRaw(p, "\\u0009", 6); - break; - case 'x': - jsonAppendRaw(p, "\\u00", 4); - jsonAppendRaw(p, &zIn[2], 2); - zIn += 2; - N -= 2; - break; - case '0': - jsonAppendRaw(p, "\\u0000", 6); +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ + u32 k; + u8 c; + const u8 *z = (const u8*)zIn; + if( z==0 ) return; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + while( 1 /*exit-by-break*/ ){ + k = 0; + /* The following while() is the 4-way unwound equivalent of + ** + ** while( k=N ){ + while( k=4 ); - assert( 0x80==(u8)zIn[2] ); - assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); - zIn += 2; - N -= 2; + } + if( !jsonIsOk[z[k+2]] ){ + k += 2; break; - default: - jsonAppendRaw(p, zIn, 2); + } + if( !jsonIsOk[z[k+3]] ){ + k += 3; break; + }else{ + k += 4; + } } - zIn += 2; - N -= 2; - } - jsonAppendChar(p, '"'); -} - -/* -** The zIn[0..N] string is a JSON5 integer literal. Append to p a translation -** of the string literal that standard JSON and that omits all JSON5 -** features. -*/ -static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ - if( zIn[0]=='+' ){ - zIn++; - N--; - }else if( zIn[0]=='-' ){ - jsonAppendChar(p, '-'); - zIn++; - N--; - } - if( zIn[0]=='0' && (zIn[1]=='x' || zIn[1]=='X') ){ - sqlite3_int64 i = 0; - int rc = sqlite3DecOrHexToI64(zIn, &i); - if( rc<=1 ){ - jsonPrintf(100,p,"%lld",i); + if( k>=N ){ + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + } + break; + } + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + z += k; + N -= k; + } + c = z[0]; + if( c=='"' || c=='\\' ){ + if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = c; + }else if( c=='\'' ){ + p->zBuf[p->nUsed++] = c; }else{ - assert( rc==2 ); - jsonAppendRaw(p, "9.0e999", 7); + if( (p->nUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; + jsonAppendControlChar(p, c); } - return; - } - jsonAppendRaw(p, zIn, N); -} - -/* -** The zIn[0..N] string is a JSON5 real literal. Append to p a translation -** of the string literal that standard JSON and that omits all JSON5 -** features. -*/ -static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( zIn[0]=='+' ){ - zIn++; - N--; - }else if( zIn[0]=='-' ){ - jsonAppendChar(p, '-'); - zIn++; + z++; N--; } - if( zIn[0]=='.' ){ - jsonAppendChar(p, '0'); - } - for(i=0; i0 ){ - jsonAppendRaw(p, zIn, N); - } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsednAlloc ); } - - /* -** Append a function parameter value to the JSON string under -** construction. +** Append an sqlite3_value (such as a function parameter) to the JSON +** string under construction in p. */ -static void jsonAppendValue( +static void jsonAppendSqlValue( JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { - jsonAppendRaw(p, "null", 4); + jsonAppendRawNZ(p, "null", 4); break; } case SQLITE_FLOAT: { @@ -454,205 +816,125 @@ static void jsonAppendValue( break; } default: { - if( p->bErr==0 ){ + JsonParse px; + memset(&px, 0, sizeof(px)); + if( jsonArgIsJsonb(pValue, &px) ){ + jsonTranslateBlobToText(&px, 0, p); + }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 2; - jsonReset(p); + p->eErr = JSTRING_ERR; + jsonStringReset(p); } break; } } } - -/* Make the JSON in p the result of the SQL function. +/* Make the text in p (which is probably a generated JSON text string) +** the result of the SQL function. +** +** The JsonString is reset. +** +** If pParse and ctx are both non-NULL, then the SQL string in p is +** loaded into the zJson field of the pParse object as a RCStr and the +** pParse is added to the cache. */ -static void jsonResult(JsonString *p){ - if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); +static void jsonReturnString( + JsonString *p, /* String to return */ + JsonParse *pParse, /* JSONB source or NULL */ + sqlite3_context *ctx /* Where to cache */ +){ + assert( (pParse!=0)==(ctx!=0) ); + assert( ctx==0 || ctx==p->pCtx ); + if( p->eErr==0 ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); + if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(p); + }else if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + }else if( jsonStringTerminate(p) ){ + if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ + int rc; + pParse->zJson = sqlite3RCStrRef(p->zBuf); + pParse->nJson = p->nUsed; + pParse->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, pParse); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(ctx); + jsonStringReset(p); + return; + } + } + sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, + sqlite3RCStrUnref, + SQLITE_UTF8); + }else{ + sqlite3_result_error_nomem(p->pCtx); + } + }else if( p->eErr & JSTRING_OOM ){ + sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_MALFORMED ){ + sqlite3_result_error(p->pCtx, "malformed JSON", -1); } - assert( p->bStatic ); + jsonStringReset(p); } /************************************************************************** -** Utility routines for dealing with JsonNode and JsonParse objects +** Utility routines for dealing with JsonParse objects **************************************************************************/ -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonNodeSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} - /* ** Reclaim all memory allocated by a JsonParse object. But do not ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - pParse->nNode = 0; - pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; + assert( pParse->nJPRef<=1 ); + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->nJson = 0; + pParse->bJsonIsRCStr = 0; + } + if( pParse->nBlobAlloc ){ + sqlite3DbFree(pParse->db, pParse->aBlob); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } } /* -** Free a JsonParse object that was obtained from sqlite3_malloc(). +** Decrement the reference count on the JsonParse object. When the +** count reaches zero, free the object. */ static void jsonParseFree(JsonParse *pParse){ - jsonParseReset(pParse); - sqlite3_free(pParse); + if( pParse ){ + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + }else{ + jsonParseReset(pParse); + sqlite3DbFree(pParse->db, pParse); + } + } } +/************************************************************************** +** Utility routines for the JSON text parser +**************************************************************************/ + /* -** Convert the JsonNode pNode into a pure JSON string and -** append to pOut. Subsubstructure is also included. Return -** the number of JsonNode objects that are encoded. -*/ -static void jsonRenderNode( - JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ -){ - assert( pNode!=0 ); - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ - if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ - assert( pNode->eU==4 ); - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); - return; - } - assert( pNode->eU==5 ); - pNode = pNode->u.pPatch; - } - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - jsonAppendRaw(pOut, "null", 4); - break; - } - case JSON_TRUE: { - jsonAppendRaw(pOut, "true", 4); - break; - } - case JSON_FALSE: { - jsonAppendRaw(pOut, "false", 5); - break; - } - case JSON_STRING: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->jnFlags & JNODE_LABEL ){ - jsonAppendChar(pOut, '"'); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - jsonAppendChar(pOut, '"'); - }else{ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - } - }else if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); - }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_REAL: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); - }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_INT: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); - }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_ARRAY: { - u32 j = 1; - jsonAppendChar(pOut, '['); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, ']'); - break; - } - case JSON_OBJECT: { - u32 j = 1; - jsonAppendChar(pOut, '{'); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - jsonAppendChar(pOut, ':'); - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - assert( pNode->eU==2 ); - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, '}'); - break; - } - } -} - -/* -** Return a JsonNode and all its descendents as a JSON string. -*/ -static void jsonReturnJson( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); -} - -/* -** Translate a single byte of Hex into an integer. -** This routine only works if h really is a valid hexadecimal -** character: 0..9a..fA..F +** Translate a single byte of Hex into an integer. +** This routine only gives a correct answer if h really is a valid hexadecimal +** character: 0..9a..fA..F. But unlike sqlite3HexToInt(), it does not +** assert() if the digit is not hex. */ static u8 jsonHexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif #ifdef SQLITE_EBCDIC h += 9*(1&~(h>>4)); -#else - h += 9*(1&(h>>6)); #endif return (u8)(h & 0xf); } @@ -662,10 +944,6 @@ static u8 jsonHexToInt(int h){ */ static u32 jsonHexToInt4(const char *z){ u32 v; - assert( sqlite3Isxdigit(z[0]) ); - assert( sqlite3Isxdigit(z[1]) ); - assert( sqlite3Isxdigit(z[2]) ); - assert( sqlite3Isxdigit(z[3]) ); v = (jsonHexToInt(z[0])<<12) + (jsonHexToInt(z[1])<<8) + (jsonHexToInt(z[2])<<4) @@ -673,227 +951,6 @@ static u32 jsonHexToInt4(const char *z){ return v; } -/* -** Make the JsonNode the return value of the function. -*/ -static void jsonReturn( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - sqlite3_result_null(pCtx); - break; - } - case JSON_TRUE: { - sqlite3_result_int(pCtx, 1); - break; - } - case JSON_FALSE: { - sqlite3_result_int(pCtx, 0); - break; - } - case JSON_INT: { - sqlite3_int64 i = 0; - int rc; - int bNeg = 0; - const char *z; - - - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; bNeg = 1; } - else if( z[0]=='+' ){ z++; } - rc = sqlite3DecOrHexToI64(z, &i); - if( rc<=1 ){ - sqlite3_result_int64(pCtx, bNeg ? -i : i); - }else if( rc==3 && bNeg ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - }else{ - goto to_double; - } - break; - } - case JSON_REAL: { - double r; - const char *z; - assert( pNode->eU==1 ); - to_double: - z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); - sqlite3_result_double(pCtx, r); - break; - } - case JSON_STRING: { - if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ - /* JSON formatted without any backslash-escapes */ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, - SQLITE_TRANSIENT); - }else{ - /* Translate JSON formatted string into raw text */ - u32 i; - u32 n = pNode->n; - const char *z; - char *zOut; - u32 j; - u32 nOut = n; - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - zOut = sqlite3_malloc( nOut+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } - for(i=1, j=0; i>6)); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - u32 vlo; - if( (v&0xfc00)==0xd800 - && i>18); - zOut[j++] = 0x80 | ((v>>12)&0x3f); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - zOut[j++] = 0xe0 | (v>>12); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - } - } - continue; - }else if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - }else if( c=='v' ){ - c = '\v'; - }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ - /* pass through unchanged */ - }else if( c=='0' ){ - c = 0; - }else if( c=='x' ){ - c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]); - i += 2; - }else if( c=='\r' && z[i+1]=='\n' ){ - i++; - continue; - }else if( 0xe2==(u8)c ){ - assert( 0x80==(u8)z[i+1] ); - assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); - i += 2; - continue; - }else{ - continue; - } - } /* end if( c=='\\' ) */ - zOut[j++] = c; - } /* end for() */ - zOut[j] = 0; - sqlite3_result_text(pCtx, zOut, j, sqlite3_free); - } - break; - } - case JSON_ARRAY: - case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); - break; - } - } -} - -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - -/* -** A macro to hint to the compiler that a function should not be -** inlined. -*/ -#if defined(__GNUC__) -# define JSON_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define JSON_NOINLINE __declspec(noinline) -#else -# define JSON_NOINLINE -#endif - - -static JSON_NOINLINE int jsonParseAddNodeExpand( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - u32 nNew; - JsonNode *pNew; - assert( pParse->nNode>=pParse->nAlloc ); - if( pParse->oom ) return -1; - nNew = pParse->nAlloc*2 + 10; - pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); - if( pNew==0 ){ - pParse->oom = 1; - return -1; - } - pParse->nAlloc = nNew; - pParse->aNode = pNew; - assert( pParse->nNodenAlloc ); - return jsonParseAddNode(pParse, eType, n, zContent); -} - -/* -** Create a new JsonNode instance based on the arguments and append that -** instance to the JsonParse. Return the index in pParse->aNode[] of the -** new node, or -1 if a memory allocation fails. -*/ -static int jsonParseAddNode( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - JsonNode *p; - if( pParse->aNode==0 || pParse->nNode>=pParse->nAlloc ){ - return jsonParseAddNodeExpand(pParse, eType, n, zContent); - } - p = &pParse->aNode[pParse->nNode]; - p->eType = (u8)(eType & 0xff); - p->jnFlags = (u8)(eType >> 8); - VVA( p->eU = zContent ? 1 : 0 ); - p->n = n; - p->u.zJContent = zContent; - return pParse->nNode++; -} - /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -1047,63 +1104,506 @@ static const struct NanInfName { char *zMatch; char *zRepl; } aNanInfName[] = { - { 'i', 'I', 3, JSON_REAL, 7, "inf", "9.0e999" }, - { 'i', 'I', 8, JSON_REAL, 7, "infinity", "9.0e999" }, - { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" }, - { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, - { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, + { 'i', 'I', 3, JSONB_FLOAT, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSONB_FLOAT, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSONB_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSONB_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" }, }; + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/**************************************************************************** +** Utility routines for dealing with the binary BLOB representation of JSON +****************************************************************************/ + +/* +** Expand pParse->aBlob so that it holds at least N bytes. +** +** Return the number of errors. +*/ +static int jsonBlobExpand(JsonParse *pParse, u32 N){ + u8 *aNew; + u64 t; + assert( N>pParse->nBlobAlloc ); + if( pParse->nBlobAlloc==0 ){ + t = 100; + }else{ + t = pParse->nBlobAlloc*2; + } + if( tdb, pParse->aBlob, t); + if( aNew==0 ){ pParse->oom = 1; return 1; } + assert( t<0x7fffffff ); + pParse->aBlob = aNew; + pParse->nBlobAlloc = (u32)t; + return 0; +} + +/* +** If pParse->aBlob is not previously editable (because it is taken +** from sqlite3_value_blob(), as indicated by the fact that +** pParse->nBlobAlloc==0 and pParse->nBlob>0) then make it editable +** by making a copy into space obtained from malloc. +** +** Return true on success. Return false on OOM. +*/ +static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ + u8 *aOld; + u32 nSize; + assert( !pParse->bReadOnly ); + if( pParse->oom ) return 0; + if( pParse->nBlobAlloc>0 ) return 1; + aOld = pParse->aBlob; + nSize = pParse->nBlob + nExtra; + pParse->aBlob = 0; + if( jsonBlobExpand(pParse, nSize) ){ + return 0; + } + assert( pParse->nBlobAlloc >= pParse->nBlob + nExtra ); + memcpy(pParse->aBlob, aOld, pParse->nBlob); + return 1; +} + +/* Expand pParse->aBlob and append one bytes. +*/ +static SQLITE_NOINLINE void jsonBlobExpandAndAppendOneByte( + JsonParse *pParse, + u8 c +){ + jsonBlobExpand(pParse, pParse->nBlob+1); + if( pParse->oom==0 ){ + assert( pParse->nBlob+1<=pParse->nBlobAlloc ); + pParse->aBlob[pParse->nBlob++] = c; + } +} + +/* Append a single character. +*/ +static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ + if( pParse->nBlob >= pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendOneByte(pParse, c); + }else{ + pParse->aBlob[pParse->nBlob++] = c; + } +} + +/* Slow version of jsonBlobAppendNode() that first resizes the +** pParse->aBlob structure. +*/ +static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( + JsonParse *pParse, + u8 eType, + u32 szPayload, + const void *aPayload +){ + if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; + jsonBlobAppendNode(pParse, eType, szPayload, aPayload); +} + + +/* Append a node type byte together with the payload size and +** possibly also the payload. +** +** If aPayload is not NULL, then it is a pointer to the payload which +** is also appended. If aPayload is NULL, the pParse->aBlob[] array +** is resized (if necessary) so that it is big enough to hold the +** payload, but the payload is not appended and pParse->nBlob is left +** pointing to where the first byte of payload will eventually be. +*/ +static void jsonBlobAppendNode( + JsonParse *pParse, /* The JsonParse object under construction */ + u8 eType, /* Node type. One of JSONB_* */ + u32 szPayload, /* Number of bytes of payload */ + const void *aPayload /* The payload. Might be NULL */ +){ + u8 *a; + if( pParse->nBlob+szPayload+9 > pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendNode(pParse,eType,szPayload,aPayload); + return; + } + assert( pParse->aBlob!=0 ); + a = &pParse->aBlob[pParse->nBlob]; + if( szPayload<=11 ){ + a[0] = eType | (szPayload<<4); + pParse->nBlob += 1; + }else if( szPayload<=0xff ){ + a[0] = eType | 0xc0; + a[1] = szPayload & 0xff; + pParse->nBlob += 2; + }else if( szPayload<=0xffff ){ + a[0] = eType | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + pParse->nBlob += 3; + }else{ + a[0] = eType | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + pParse->nBlob += 5; + } + if( aPayload ){ + pParse->nBlob += szPayload; + memcpy(&pParse->aBlob[pParse->nBlob-szPayload], aPayload, szPayload); + } +} + +/* Change the payload size for the node at index i to be szPayload. +*/ +static int jsonBlobChangePayloadSize( + JsonParse *pParse, + u32 i, + u32 szPayload +){ + u8 *a; + u8 szType; + u8 nExtra; + u8 nNeeded; + int delta; + if( pParse->oom ) return 0; + a = &pParse->aBlob[i]; + szType = a[0]>>4; + if( szType<=11 ){ + nExtra = 0; + }else if( szType==12 ){ + nExtra = 1; + }else if( szType==13 ){ + nExtra = 2; + }else if( szType==14 ){ + nExtra = 4; + }else{ + nExtra = 8; + } + if( szPayload<=11 ){ + nNeeded = 0; + }else if( szPayload<=0xff ){ + nNeeded = 1; + }else if( szPayload<=0xffff ){ + nNeeded = 2; + }else{ + nNeeded = 4; + } + delta = nNeeded - nExtra; + if( delta ){ + u32 newSize = pParse->nBlob + delta; + if( delta>0 ){ + if( newSize>pParse->nBlobAlloc && jsonBlobExpand(pParse, newSize) ){ + return 0; /* OOM error. Error state recorded in pParse->oom. */ + } + a = &pParse->aBlob[i]; + memmove(&a[1+delta], &a[1], pParse->nBlob - (i+1)); + }else{ + memmove(&a[1], &a[1-delta], pParse->nBlob - (i+1-delta)); + } + pParse->nBlob = newSize; + } + if( nNeeded==0 ){ + a[0] = (a[0] & 0x0f) | (szPayload<<4); + }else if( nNeeded==1 ){ + a[0] = (a[0] & 0x0f) | 0xc0; + a[1] = szPayload & 0xff; + }else if( nNeeded==2 ){ + a[0] = (a[0] & 0x0f) | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + }else{ + a[0] = (a[0] & 0x0f) | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + } + return delta; +} + +/* +** If z[0] is 'u' and is followed by exactly 4 hexadecimal character, +** then set *pOp to JSONB_TEXTJ and return true. If not, do not make +** any changes to *pOp and return false. +*/ +static int jsonIs4HexB(const char *z, int *pOp){ + if( z[0]!='u' ) return 0; + if( !jsonIs4Hex(&z[1]) ) return 0; + *pOp = JSONB_TEXTJ; + return 1; +} + +/* +** Check a single element of the JSONB in pParse for validity. +** +** The element to be checked starts at offset i and must end at on the +** last byte before iEnd. +** +** Return 0 if everything is correct. Return the 1-based byte offset of the +** error if a problem is detected. (In other words, if the error is at offset +** 0, return 1). +*/ +static u32 jsonbValidityCheck( + const JsonParse *pParse, /* Input JSONB. Only aBlob and nBlob are used */ + u32 i, /* Start of element as pParse->aBlob[i] */ + u32 iEnd, /* One more than the last byte of the element */ + u32 iDepth /* Current nesting depth */ +){ + u32 n, sz, j, k; + const u8 *z; + u8 x; + if( iDepth>JSON_MAX_DEPTH ) return i+1; + sz = 0; + n = jsonbPayloadSize(pParse, i, &sz); + if( NEVER(n==0) ) return i+1; /* Checked by caller */ + if( NEVER(i+n+sz!=iEnd) ) return i+1; /* Checked by caller */ + z = pParse->aBlob; + x = z[i] & 0x0f; + switch( x ){ + case JSONB_NULL: + case JSONB_TRUE: + case JSONB_FALSE: { + return n+sz==1 ? 0 : i+1; + } + case JSONB_INT: { + if( sz<1 ) return i+1; + j = i+n; + if( z[j]=='-' ){ + j++; + if( sz<2 ) return i+1; + } + k = i+n+sz; + while( jk ) return j+1; + if( z[j+1]!='.' && z[j+1]!='e' && z[j+1]!='E' ) return j+1; + j++; + } + for(; j0 ) return j+1; + if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ + return j+1; + } + seen = 1; + continue; + } + if( z[j]=='e' || z[j]=='E' ){ + if( seen==2 ) return j+1; + if( j==k-1 ) return j+1; + if( z[j+1]=='+' || z[j+1]=='-' ){ + j++; + if( j==k-1 ) return j+1; + } + seen = 2; + continue; + } + return j+1; + } + if( seen==0 ) return i+1; + return 0; + } + case JSONB_TEXT: { + j = i+n; + k = j+sz; + while( j=k ){ + return j+1; + }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ + j++; + }else if( z[j+1]=='u' ){ + if( j+5>=k ) return j+1; + if( !jsonIs4Hex((const char*)&z[j+2]) ) return j+1; + j++; + }else if( x!=JSONB_TEXT5 ){ + return j+1; + }else{ + u32 c = 0; + u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c); + if( c==JSON_INVALID_CHAR ) return j+1; + j += szC - 1; + } + } + j++; + } + return 0; + } + case JSONB_TEXTRAW: { + return 0; + } + case JSONB_ARRAY: { + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + j += n + sz; + } + assert( j==k ); + return 0; + } + case JSONB_OBJECT: { + u32 cnt = 0; + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + if( (cnt & 1)==0 ){ + x = z[j] & 0x0f; + if( xJSONB_TEXTRAW ) return j+1; + } + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + cnt++; + j += n + sz; + } + assert( j==k ); + if( (cnt & 1)!=0 ) return j+1; + return 0; + } + default: { + return i+1; + } + } +} + /* -** Parse a single JSON value which begins at pParse->zJson[i]. Return the -** index of the first character past the end of the value parsed. +** Translate a single element of JSON text at pParse->zJson[i] into +** its equivalent binary JSONB representation. Append the translation into +** pParse->aBlob[] beginning at pParse->nBlob. The size of +** pParse->aBlob[] is increased as necessary. ** -** Special return values: +** Return the index of the first character past the end of the element parsed, +** or one of the following special result codes: ** -** 0 End if input -** -1 Syntax error -** -2 '}' seen -** -3 ']' seen -** -4 ',' seen -** -5 ':' seen +** 0 End of input +** -1 Syntax error or OOM +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / */ -static int jsonParseValue(JsonParse *pParse, u32 i){ +static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ char c; u32 j; - int iThis; + u32 iThis, iStart; int x; - JsonNode *pNode; + u8 t; const char *z = pParse->zJson; json_parse_restart: switch( (u8)z[i] ){ case '{': { /* Parse object */ - iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; + iThis = pParse->nBlob; + jsonBlobAppendNode(pParse, JSONB_OBJECT, pParse->nJson-i, 0); if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; return -1; } + iStart = pParse->nBlob; for(j=i+1;;j++){ - u32 nNode = pParse->nNode; - x = jsonParseValue(pParse, j); + u32 iBlob = pParse->nBlob; + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ + int op; if( x==(-2) ){ j = pParse->iErr; - if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; + if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; break; } j += json5Whitespace(&z[j]); - if( sqlite3JsonId1(z[j]) - || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) + op = JSONB_TEXT; + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && jsonIs4HexB(&z[j+1], &op)) ){ int k = j+1; while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) - || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) + || (z[k]=='\\' && jsonIs4HexB(&z[k+1], &op)) ){ k++; } - jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); + assert( iBlob==pParse->nBlob ); + jsonBlobAppendNode(pParse, op, k-j, &z[j]); pParse->hasNonstd = 1; x = k; }else{ @@ -1112,24 +1612,24 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ } } if( pParse->oom ) return -1; - pNode = &pParse->aNode[nNode]; - if( pNode->eType!=JSON_STRING ){ + t = pParse->aBlob[iBlob] & 0x0f; + if( tJSONB_TEXTRAW ){ pParse->iErr = j; return -1; } - pNode->jnFlags |= JNODE_LABEL; j = x; if( z[j]==':' ){ j++; }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); + if( jsonIsspace(z[j]) ){ + /* strspn() is not helpful here */ + do{ j++; }while( jsonIsspace(z[j]) ); if( z[j]==':' ){ j++; goto parse_object_value; } } - x = jsonParseValue(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1137,7 +1637,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ j = pParse->iErr+1; } parse_object_value: - x = jsonParseValue(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1148,15 +1648,15 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ }else if( z[j]=='}' ){ break; }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); + if( jsonIsspace(z[j]) ){ + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]=='}' ){ break; } } - x = jsonParseValue(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1169,25 +1669,27 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ pParse->iErr = j; return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); pParse->iDepth--; return j+1; } case '[': { /* Parse array */ - iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; + iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); + jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); + iStart = pParse->nBlob; + if( pParse->oom ) return -1; if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; return -1; } - memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - x = jsonParseValue(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; - if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; + if( pParse->nBlob!=iStart ) pParse->hasNonstd = 1; break; } if( x!=(-1) ) pParse->iErr = j; @@ -1199,15 +1701,15 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ }else if( z[j]==']' ){ break; }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); + if( jsonIsspace(z[j]) ){ + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]==']' ){ break; } } - x = jsonParseValue(pParse, j); + x = jsonTranslateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1220,60 +1722,81 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ pParse->iErr = j; return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); pParse->iDepth--; return j+1; } case '\'': { - u8 jnFlags; + u8 opcode; char cDelim; pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; + opcode = JSONB_TEXT; goto parse_string; case '"': /* Parse string */ - jnFlags = 0; + opcode = JSONB_TEXT; parse_string: cDelim = z[i]; j = i+1; - for(;;){ - c = z[j]; - if( (c & ~0x1f)==0 ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; + while( 1 /*exit-by-break*/ ){ + if( jsonIsOk[(u8)z[j]] ){ + if( !jsonIsOk[(u8)z[j+1]] ){ + j += 1; + }else if( !jsonIsOk[(u8)z[j+2]] ){ + j += 2; + }else{ + j += 3; + continue; + } } - if( c=='\\' ){ + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' || (c=='u' && jsonIs4Hex(&z[j+1])) ){ - jnFlags |= JNODE_ESCAPE; - }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; + }else if( c=='\'' || c=='v' || c=='\n' +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + || (c=='0') /* Legacy bug compatible */ +#else + || (c=='0' && !sqlite3Isdigit(z[j+1])) /* Correct implementation */ +#endif || (0xe2==(u8)c && 0x80==(u8)z[j+1] && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) || (c=='x' && jsonIs2Hex(&z[j+1])) ){ - jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + opcode = JSONB_TEXT5; pParse->hasNonstd = 1; }else if( c=='\r' ){ if( z[j+1]=='\n' ) j++; - jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + opcode = JSONB_TEXT5; pParse->hasNonstd = 1; }else{ pParse->iErr = j; return -1; } - }else if( c==cDelim ){ - break; + }else if( c<=0x1f ){ + if( c==0 ){ + pParse->iErr = j; + return -1; + } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='"' ){ + opcode = JSONB_TEXT5; } j++; } - jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); + jsonBlobAppendNode(pParse, opcode, j-1-i, &z[i+1]); return j+1; } case 't': { if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + jsonBlobAppendOneByte(pParse, JSONB_TRUE); return i+4; } pParse->iErr = i; @@ -1281,23 +1804,22 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ } case 'f': { if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + jsonBlobAppendOneByte(pParse, JSONB_FALSE); return i+5; } pParse->iErr = i; return -1; } case '+': { - u8 seenDP, seenE, jnFlags; + u8 seenE; pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ goto parse_number; case '.': if( sqlite3Isdigit(z[i+1]) ){ pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; + t = 0x03; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ seenE = 0; - seenDP = JSON_REAL; goto parse_number_2; } pParse->iErr = i; @@ -1314,9 +1836,8 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ case '8': case '9': /* Parse number */ - jnFlags = 0; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ parse_number: - seenDP = JSON_INT; seenE = 0; assert( '-' < '0' ); assert( '+' < '0' ); @@ -1326,9 +1847,9 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c<='0' ){ if( c=='0' ){ if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ - assert( seenDP==JSON_INT ); + assert( t==0x00 ); pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t = 0x01; for(j=i+3; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; }else if( sqlite3Isdigit(z[i+1]) ){ @@ -1345,15 +1866,15 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ ){ pParse->hasNonstd = 1; if( z[i]=='-' ){ - jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); }else{ - jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); } return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); } if( z[i+1]=='.' ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; goto parse_number_2; } pParse->iErr = i; @@ -1365,30 +1886,31 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return -1; }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; for(j=i+4; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; } } } } + parse_number_2: for(j=i+1;; j++){ c = z[j]; if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( seenDP==JSON_REAL ){ + if( (t & 0x02)!=0 ){ pParse->iErr = j; return -1; } - seenDP = JSON_REAL; + t |= 0x02; continue; } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; }else{ pParse->iErr = j; return -1; @@ -1398,7 +1920,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ pParse->iErr = j; return -1; } - seenDP = JSON_REAL; + t |= 0x02; seenE = 1; c = z[j+1]; if( c=='+' || c=='-' ){ @@ -1416,14 +1938,18 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( z[j-1]<'0' ){ if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; }else{ pParse->iErr = j; return -1; } } parse_number_finish: - jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]); + assert( JSONB_INT+0x01==JSONB_INT5 ); + assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); + assert( JSONB_INT+0x02==JSONB_FLOAT ); + if( z[i]=='+' ) i++; + jsonBlobAppendNode(pParse, JSONB_INT+t, j-i, &z[i]); return j; } case '}': { @@ -1449,9 +1975,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ case 0x0a: case 0x0d: case 0x20: { - do{ - i++; - }while( fast_isspace(z[i]) ); + i += 1 + (u32)strspn(&z[i+1], jsonSpaces); goto json_parse_restart; } case 0x0b: @@ -1473,10 +1997,11 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ } case 'n': { if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); + jsonBlobAppendOneByte(pParse, JSONB_NULL); return i+4; } /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through } default: { u32 k; @@ -1489,8 +2014,11 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ continue; } if( sqlite3Isalnum(z[i+nn]) ) continue; - jsonParseAddNode(pParse, aNanInfName[k].eType, - aNanInfName[k].nRepl, aNanInfName[k].zRepl); + if( aNanInfName[k].eType==JSONB_FLOAT ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + }else{ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + } pParse->hasNonstd = 1; return i + nn; } @@ -1500,30 +2028,35 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ } /* End switch(z[i]) */ } + /* ** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. ** -** pParse is uninitialized when this routine is called. +** pParse must be initialized to an empty parse object prior to calling +** this routine. */ -static int jsonParse( +static int jsonConvertTextToBlob( JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ + sqlite3_context *pCtx /* Report errors here */ ){ int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; - i = jsonParseValue(pParse, 0); + const char *zJson = pParse->zJson; + i = jsonTranslateTextToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ +#ifdef SQLITE_DEBUG assert( pParse->iDepth==0 ); - while( fast_isspace(zJson[i]) ) i++; + if( sqlite3Config.bJsonSelfcheck ){ + assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 ); + } +#endif + while( jsonIsspace(zJson[i]) ) i++; if( zJson[i] ){ i += json5Whitespace(&zJson[i]); if( zJson[i] ){ + if( pCtx ) sqlite3_result_error(pCtx, "malformed JSON", -1); jsonParseReset(pParse); return 1; } @@ -1544,488 +2077,1799 @@ static int jsonParse( return 0; } -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. +/* +** The input string pStr is a well-formed JSON text string. Convert +** this into the JSONB format and make it the return value of the +** SQL function. */ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); +static void jsonReturnStringAsBlob(JsonString *pStr){ + JsonParse px; + memset(&px, 0, sizeof(px)); + jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } + px.zJson = pStr->zBuf; + px.nJson = pStr->nUsed; + px.db = sqlite3_context_db_handle(pStr->pCtx); + (void)jsonTranslateTextToBlob(&px, 0); + if( px.oom ){ + sqlite3DbFree(px.db, px.aBlob); + sqlite3_result_error_nomem(pStr->pCtx); + }else{ + assert( px.nBlobAlloc>0 ); + assert( !px.bReadOnly ); + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, SQLITE_DYNAMIC); + } +} + +/* The byte at index i is a node type-code. This routine +** determines the payload size for that node and writes that +** payload size in to *pSz. It returns the offset from i to the +** beginning of the payload. Return 0 on error. +*/ +static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ + u8 x; + u32 sz; + u32 n; + assert( i<=pParse->nBlob ); + x = pParse->aBlob[i]>>4; + if( x<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else if( x==13 ){ + if( i+2>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 3; + }else if( x==14 ){ + if( i+4>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = ((u32)pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 5; + }else{ + if( i+8>=pParse->nBlob + || pParse->aBlob[i+1]!=0 + || pParse->aBlob[i+2]!=0 + || pParse->aBlob[i+3]!=0 + || pParse->aBlob[i+4]!=0 + ){ + *pSz = 0; + return 0; + } + sz = ((u32)pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; + n = 9; + } + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta + ){ + *pSz = 0; + return 0; + } + *pSz = sz; + return n; +} + + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToText( + const JsonParse *pParse, /* the complete parse of the JSON */ + u32 i, /* Start rendering at this index */ + JsonString *pOut /* Write JSON here */ +){ + u32 sz, n, j, iEnd; + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + jsonAppendRawNZ(pOut, "null", 4); + return i+1; + } + case JSONB_TRUE: { + jsonAppendRawNZ(pOut, "true", 4); + return i+1; + } + case JSONB_FALSE: { + jsonAppendRawNZ(pOut, "false", 5); + return i+1; + } + case JSONB_INT: + case JSONB_FLOAT: { + if( sz==0 ) goto malformed_jsonb; + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_INT5: { /* Integer literal in hexadecimal notation */ + u32 k = 2; + sqlite3_uint64 u = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + int bOverflow = 0; + if( sz==0 ) goto malformed_jsonb; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + }else if( zIn[0]=='+' ){ + k++; + } + for(; keErr |= JSTRING_MALFORMED; + break; + }else if( (u>>60)!=0 ){ + bOverflow = 1; + }else{ + u = u*16 + sqlite3HexToInt(zIn[k]); + } + } + jsonPrintf(100,pOut,bOverflow?"9.0e999":"%llu", u); + break; + } + case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ + u32 k = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( sz==0 ) goto malformed_jsonb; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + } + if( zIn[k]=='.' ){ + jsonAppendChar(pOut, '0'); + } + for(; knUsed+sz+2<=pOut->nAlloc || jsonStringGrow(pOut, sz+2)==0 ){ + pOut->zBuf[pOut->nUsed] = '"'; + memcpy(pOut->zBuf+pOut->nUsed+1,(const char*)&pParse->aBlob[i+n],sz); + pOut->zBuf[pOut->nUsed+sz+1] = '"'; + pOut->nUsed += sz+2; + } + break; + } + case JSONB_TEXT5: { + const char *zIn; + u32 k; + u32 sz2 = sz; + zIn = (const char*)&pParse->aBlob[i+n]; + jsonAppendChar(pOut, '"'); + while( sz2>0 ){ + for(k=0; k0 ){ + jsonAppendRawNZ(pOut, zIn, k); + if( k>=sz2 ){ + break; + } + zIn += k; + sz2 -= k; + } + if( zIn[0]=='"' ){ + jsonAppendRawNZ(pOut, "\\\"", 2); + zIn++; + sz2--; + continue; + } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } + assert( zIn[0]=='\\' ); + assert( sz2>=1 ); + if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(pOut, '\''); + break; + case 'v': + jsonAppendRawNZ(pOut, "\\u000b", 6); + break; + case 'x': + if( sz2<4 ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 2; + break; + } + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz2 -= 2; + break; + case '0': + jsonAppendRawNZ(pOut, "\\u0000", 6); + break; + case '\r': + if( sz2>2 && zIn[2]=='\n' ){ + zIn++; + sz2--; + } + break; + case '\n': + break; + case 0xe2: + /* '\' followed by either U+2028 or U+2029 is ignored as + ** whitespace. Not that in UTF8, U+2028 is 0xe2 0x80 0x29. + ** U+2029 is the same except for the last byte */ + if( sz2<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 2; + break; + } + zIn += 2; + sz2 -= 2; + break; + default: + jsonAppendRawNZ(pOut, zIn, 2); + break; + } + assert( sz2>=2 ); + zIn += 2; + sz2 -= 2; + } + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXTRAW: { + jsonAppendString(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_ARRAY: { + jsonAppendChar(pOut, '['); + j = i+n; + iEnd = j+sz; + while( jeErr==0 ){ + j = jsonTranslateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, ','); + } + if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); + jsonAppendChar(pOut, ']'); + break; + } + case JSONB_OBJECT: { + int x = 0; + jsonAppendChar(pOut, '{'); + j = i+n; + iEnd = j+sz; + while( jeErr==0 ){ + j = jsonTranslateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); + } + if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); + jsonAppendChar(pOut, '}'); + break; + } + + default: { + malformed_jsonb: + pOut->eErr |= JSTRING_MALFORMED; + break; + } + } + return i+n+sz; +} + +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jjnIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } +} + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); } + jsonAppendChar(pOut, ']'); + i = iEnd; break; } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); } + jsonAppendChar(pOut, '}'); + i = iEnd; break; } default: { + i = jsonTranslateBlobToText(pParse, i, pOut); break; } } + return i; } /* -** Compute the parentage of all nodes in a completed parse. +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. */ -static int jsonParseFindParents(JsonParse *pParse){ - u32 *aUp; - assert( pParse->aUp==0 ); - aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); - if( aUp==0 ){ - pParse->oom = 1; - return SQLITE_NOMEM; +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; n>0 && idelta. */ -#define JSON_CACHE_ID (-429938) /* First cache entry */ -#define JSON_CACHE_SZ 4 /* Max number of cache entries */ +static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ + u32 sz = 0; + u32 nBlob; + assert( pParse->delta!=0 ); + assert( pParse->nBlobAlloc >= pParse->nBlob ); + nBlob = pParse->nBlob; + pParse->nBlob = pParse->nBlobAlloc; + (void)jsonbPayloadSize(pParse, iRoot, &sz); + pParse->nBlob = nBlob; + sz += pParse->delta; + pParse->delta += jsonBlobChangePayloadSize(pParse, iRoot, sz); +} /* -** Obtain a complete parse of the JSON found in the first argument -** of the argv array. Use the sqlite3_get_auxdata() cache for this -** parse if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse, -** and also register the new parse so that it will be available for -** future sqlite3_get_auxdata() calls. +** If the JSONB at aIns[0..nIns-1] can be expanded (by denormalizing the +** size field) by d bytes, then write the expansion into aOut[] and +** return true. In this way, an overwrite happens without changing the +** size of the JSONB, which reduces memcpy() operations and also make it +** faster and easier to update the B-Tree entry that contains the JSONB +** in the database. +** +** If the expansion of aIns[] by d bytes cannot be (easily) accomplished +** then return false. ** -** If an error occurs and pErrCtx!=0 then report the error on pErrCtx -** and return NULL. +** The d parameter is guaranteed to be between 1 and 8. ** -** If an error occurs and pErrCtx==0 then return the Parse object with -** JsonParse.nErr non-zero. If the caller invokes this routine with -** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller -** is responsible for invoking jsonParseFree() on the returned value. -** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. +** This routine is an optimization. A correct answer is obtained if it +** always leaves the output unchanged and returns false. */ -static JsonParse *jsonParseCached( - sqlite3_context *pCtx, - sqlite3_value **argv, - sqlite3_context *pErrCtx +static int jsonBlobOverwrite( + u8 *aOut, /* Overwrite here */ + const u8 *aIns, /* New content */ + u32 nIns, /* Bytes of new content */ + u32 d /* Need to expand new content by this much */ ){ - const char *zJson = (const char*)sqlite3_value_text(argv[0]); - int nJson = sqlite3_value_bytes(argv[0]); - JsonParse *p; - JsonParse *pMatch = 0; - int iKey; - int iMinKey = 0; - u32 iMinHold = 0xffffffff; - u32 iMaxHold = 0; - if( zJson==0 ) return 0; - for(iKey=0; iKey>4 ){ + default: { /* aIns[] header size 1 */ + if( ((1<nJson==nJson - && memcmp(p->zJson,zJson,nJson)==0 - ){ - p->nErr = 0; - pMatch = p; - }else if( p->iHoldiHold; - iMinKey = iKey; + case 12: { /* aIns[] header size is 2 */ + if( ((1<iHold>iMaxHold ){ - iMaxHold = p->iHold; + case 14: { /* aIns[] header size is 5 */ + if( d!=4 ) return 0; /* d must be 4 */ + i = 9; /* New hdr sz: 9 */ + szHdr = 5; + break; + } + case 15: { /* aIns[] header size is 9 */ + return 0; /* No solution */ } } - if( pMatch ){ - pMatch->nErr = 0; - pMatch->iHold = iMaxHold+1; - return pMatch; + assert( i>=2 && i<=9 && aType[i-2]!=0 ); + aOut[0] = (aIns[0] & 0x0f) | aType[i-2]; + memcpy(&aOut[i], &aIns[szHdr], nIns-szHdr); + szPayload = nIns - szHdr; + while( 1/*edit-by-break*/ ){ + i--; + aOut[i] = szPayload & 0xff; + if( i==1 ) break; + szPayload >>= 8; } - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); - if( p==0 ){ - sqlite3_result_error_nomem(pCtx); - return 0; + assert( (szPayload>>8)==0 ); + return 1; +} + +/* +** Modify the JSONB blob at pParse->aBlob by removing nDel bytes of +** content beginning at iDel, and replacing them with nIns bytes of +** content given by aIns. +** +** nDel may be zero, in which case no bytes are removed. But iDel is +** still important as new bytes will be insert beginning at iDel. +** +** aIns may be zero, in which case space is created to hold nIns bytes +** beginning at iDel, but that space is uninitialized. +** +** Set pParse->oom if an OOM occurs. +*/ +static void jsonBlobEdit( + JsonParse *pParse, /* The JSONB to be modified is in pParse->aBlob */ + u32 iDel, /* First byte to be removed */ + u32 nDel, /* Number of bytes to remove */ + const u8 *aIns, /* Content to insert */ + u32 nIns /* Bytes of content to insert */ +){ + i64 d = (i64)nIns - (i64)nDel; + if( d<0 && d>=(-8) && aIns!=0 + && jsonBlobOverwrite(&pParse->aBlob[iDel], aIns, nIns, (int)-d) + ){ + return; } - memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pErrCtx, p->zJson) ){ - if( pErrCtx==0 ){ - p->nErr = 1; - return p; + if( d!=0 ){ + if( pParse->nBlob + d > pParse->nBlobAlloc ){ + jsonBlobExpand(pParse, pParse->nBlob+d); + if( pParse->oom ) return; } - sqlite3_free(p); - return 0; + memmove(&pParse->aBlob[iDel+nIns], + &pParse->aBlob[iDel+nDel], + pParse->nBlob - (iDel+nDel)); + pParse->nBlob += d; + pParse->delta += d; + } + if( nIns && aIns ){ + memcpy(&pParse->aBlob[iDel], aIns, nIns); } - p->nJson = nJson; - p->iHold = iMaxHold+1; - sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, - (void(*)(void*))jsonParseFree); - return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); } /* -** Compare the OBJECT label at pNode against zKey,nKey. Return true on -** a match. +** Return the number of escaped newlines to be ignored. +** An escaped newline is a one of the following byte sequences: +** +** 0x5c 0x0a +** 0x5c 0x0d +** 0x5c 0x0d 0x0a +** 0x5c 0xe2 0x80 0xa8 +** 0x5c 0xe2 0x80 0xa9 */ -static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; +static u32 jsonBytesToBypass(const char *z, u32 n){ + u32 i = 0; + while( i+10 ); + assert( z[0]=='\\' ); + if( n<2 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + switch( (u8)z[1] ){ + case 'u': { + u32 v, vlo; + if( n<6 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + v = jsonHexToInt4(&z[2]); + if( (v & 0xfc00)==0xd800 + && n>=12 + && z[6]=='\\' + && z[7]=='u' + && ((vlo = jsonHexToInt4(&z[8]))&0xfc00)==0xdc00 + ){ + *piOut = ((v&0x3ff)<<10) + (vlo&0x3ff) + 0x10000; + return 12; + }else{ + *piOut = v; + return 6; + } + } + case 'b': { *piOut = '\b'; return 2; } + case 'f': { *piOut = '\f'; return 2; } + case 'n': { *piOut = '\n'; return 2; } + case 'r': { *piOut = '\r'; return 2; } + case 't': { *piOut = '\t'; return 2; } + case 'v': { *piOut = '\v'; return 2; } + case '0': { + /* JSON5 requires that the \0 escape not be followed by a digit. + ** But SQLite did not enforce this restriction in versions 3.42.0 + ** through 3.49.2. That was a bug. But some applications might have + ** come to depend on that bug. Use the SQLITE_BUG_COMPATIBLE_20250510 + ** option to restore the old buggy behavior. */ +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + /* Legacy bug-compatible behavior */ + *piOut = 0; +#else + /* Correct behavior */ + *piOut = (n>2 && sqlite3Isdigit(z[2])) ? JSON_INVALID_CHAR : 0; +#endif + return 2; + } + case '\'': + case '"': + case '/': + case '\\':{ *piOut = z[1]; return 2; } + case 'x': { + if( n<4 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + *piOut = (jsonHexToInt(z[2])<<4) | jsonHexToInt(z[3]); + return 4; + } + case 0xe2: + case '\r': + case '\n': { + u32 nSkip = jsonBytesToBypass(z, n); + if( nSkip==0 ){ + *piOut = JSON_INVALID_CHAR; + return n; + }else if( nSkip==n ){ + *piOut = 0; + return n; + }else if( z[nSkip]=='\\' ){ + return nSkip + jsonUnescapeOneChar(&z[nSkip], n-nSkip, piOut); + }else{ + int sz = sqlite3Utf8ReadLimited((u8*)&z[nSkip], n-nSkip, piOut); + return nSkip + sz; + } + } + default: { + *piOut = JSON_INVALID_CHAR; + return 2; + } + } +} + + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. +** +** In this version, we know that one or the other or both of the +** two comparands contains an escape sequence. +*/ +static SQLITE_NOINLINE int jsonLabelCompareEscaped( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + u32 cLeft, cRight; + assert( rawLeft==0 || rawRight==0 ); + while( 1 /*exit-by-return*/ ){ + if( nLeft==0 ){ + cLeft = 0; + }else if( rawLeft || zLeft[0]!='\\' ){ + cLeft = ((u8*)zLeft)[0]; + if( cLeft>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zLeft, nLeft, &cLeft); + zLeft += sz; + nLeft -= sz; + }else{ + zLeft++; + nLeft--; + } + }else{ + u32 n = jsonUnescapeOneChar(zLeft, nLeft, &cLeft); + zLeft += n; + assert( n<=nLeft ); + nLeft -= n; + } + if( nRight==0 ){ + cRight = 0; + }else if( rawRight || zRight[0]!='\\' ){ + cRight = ((u8*)zRight)[0]; + if( cRight>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zRight, nRight, &cRight); + zRight += sz; + nRight -= sz; + }else{ + zRight++; + nRight--; + } + }else{ + u32 n = jsonUnescapeOneChar(zRight, nRight, &cRight); + zRight += n; + assert( n<=nRight ); + nRight -= n; + } + if( cLeft!=cRight ) return 0; + if( cLeft==0 ) return 1; } } -static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ - if( p1->jnFlags & JNODE_RAW ){ - return jsonLabelCompare(p2, p1->u.zJContent, p1->n); - }else if( p2->jnFlags & JNODE_RAW ){ - return jsonLabelCompare(p1, p2->u.zJContent, p2->n); + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. Return -1 if an OOM occurs. +*/ +static int jsonLabelCompare( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + if( rawLeft && rawRight ){ + /* Simpliest case: Neither label contains escapes. A simple + ** memcmp() is sufficient. */ + if( nLeft!=nRight ) return 0; + return memcmp(zLeft, zRight, nLeft)==0; }else{ - return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; + return jsonLabelCompareEscaped(zLeft, nLeft, rawLeft, + zRight, nRight, rawRight); } } -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); +/* +** Error returns from jsonLookupStep() +*/ +#define JSON_LOOKUP_ERROR 0xffffffff +#define JSON_LOOKUP_NOTFOUND 0xfffffffe +#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) + +/* Forward declaration */ +static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); + + +/* This helper routine for jsonLookupStep() populates pIns with +** binary data that is to be inserted into pParse. +** +** In the common case, pIns just points to pParse->aIns and pParse->nIns. +** But if the zPath of the original edit operation includes path elements +** that go deeper, additional substructure must be created. +** +** For example: +** +** json_insert('{}', '$.a.b.c', 123); +** +** The search stops at '$.a' But additional substructure must be +** created for the ".b.c" part of the patch so that the final result +** is: {"a":{"b":{"c"::123}}}. This routine populates pIns with +** the binary equivalent of {"b":{"c":123}} so that it can be inserted. +** +** The caller is responsible for resetting pIns when it has finished +** using the substructure. +*/ +static u32 jsonCreateEditSubstructure( + JsonParse *pParse, /* The original JSONB that is being edited */ + JsonParse *pIns, /* Populate this with the blob data to insert */ + const char *zTail /* Tail of the path that determins substructure */ +){ + static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; + int rc; + memset(pIns, 0, sizeof(*pIns)); + pIns->db = pParse->db; + if( zTail[0]==0 ){ + /* No substructure. Just insert what is given in pParse. */ + pIns->aBlob = pParse->aIns; + pIns->nBlob = pParse->nIns; + rc = 0; + }else{ + /* Construct the binary substructure */ + pIns->nBlob = 1; + pIns->aBlob = (u8*)&emptyObject[zTail[0]=='.']; + pIns->eEdit = pParse->eEdit; + pIns->nIns = pParse->nIns; + pIns->aIns = pParse->aIns; + rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->oom |= pIns->oom; + } + return rc; /* Error code only */ +} /* -** Search along zPath to find the node specified. Return a pointer -** to that node, or NULL if zPath is malformed or if there is no such -** node. +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. ** -** If pApnd!=0, then try to append new nodes to complete zPath if it is -** possible to do so and if no existing node corresponds to zPath. If -** new nodes are appended *pApnd is set to 1. +** If the value found by this routine is the value half of label/value pair +** within an object, then set pPath->iLabel to the start of the corresponding +** label, before returning. +** +** Return one of the JSON_LOOKUP error codes if problems are seen. +** +** This routine will also modify the blob. If pParse->eEdit is one of +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be +** made to the selected value. If an edit is performed, then the return +** value does not necessarily point to the select element. If an edit +** is performed, the return value is only useful for detecting error +** conditions. */ -static JsonNode *jsonLookupStep( +static u32 jsonLookupStep( JsonParse *pParse, /* The JSON to search */ - u32 iRoot, /* Begin the search at this node */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ + u32 iLabel /* Label if iRoot is a value of in an object */ ){ - u32 i, j, nKey; + u32 i, j, k, nKey, sz, n, iEnd, rc; const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; - if( zPath[0]==0 ) return pRoot; - if( pRoot->jnFlags & JNODE_REPLACE ) return 0; + u8 x; + + if( zPath[0]==0 ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse, pParse->nIns) ){ + n = jsonbPayloadSize(pParse, iRoot, &sz); + sz += n; + if( pParse->eEdit==JEDIT_DEL ){ + if( iLabel>0 ){ + sz += iRoot - iLabel; + iRoot = iLabel; + } + jsonBlobEdit(pParse, iRoot, sz, 0, 0); + }else if( pParse->eEdit==JEDIT_INS ){ + /* Already exists, so json_insert() is a no-op */ + }else{ + /* json_set() or json_replace() */ + jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); + } + } + pParse->iLabel = iLabel; + return iRoot; + } if( zPath[0]=='.' ){ - if( pRoot->eType!=JSON_OBJECT ) return 0; + int rawKey = 1; + x = pParse->aBlob[iRoot]; zPath++; if( zPath[0]=='"' ){ zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} + for(i=1; zPath[i] && zPath[i]!='"'; i++){ + if( zPath[i]=='\\' && zPath[i+1]!=0 ) i++; + } nKey = i-1; if( zPath[i] ){ i++; }else{ - *pzErr = zPath; - return 0; + return JSON_LOOKUP_PATHERROR; } testcase( nKey==0 ); + rawKey = memchr(zKey, '\\', nKey)==0; }else{ zKey = zPath; for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} nKey = i; if( nKey==0 ){ - *pzErr = zPath; - return 0; + return JSON_LOOKUP_PATHERROR; + } + } + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_LOOKUP_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; /* j is the index of a label */ + iEnd = j+sz; + while( jaBlob[j] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + k = j+n; /* k is the index of the label text */ + if( k+sz>=iEnd ) return JSON_LOOKUP_ERROR; + zLabel = (const char*)&pParse->aBlob[k]; + rawLabel = x==JSONB_TEXT || x==JSONB_TEXTRAW; + if( jsonLabelCompare(zKey, nKey, rawKey, zLabel, sz, rawLabel) ){ + u32 v = k+sz; /* v is the index of the value */ + if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, v, &sz); + if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; + assert( j>0 ); + rc = jsonLookupStep(pParse, v, &zPath[i], j); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; } + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + j += n+sz; } - j = 1; - for(;;){ - while( j<=pRoot->n ){ - if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ - return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( pParse->eEdit>=JEDIT_INS ){ + u32 nIns; /* Total bytes to insert (label+value) */ + JsonParse v; /* BLOB encoding of the value to be inserted */ + JsonParse ix; /* Header of the label to be inserted */ + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + memset(&ix, 0, sizeof(ix)); + ix.db = pParse->db; + jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); + pParse->oom |= ix.oom; + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) + ){ + assert( !pParse->oom ); + nIns = ix.nBlob + nKey + v.nBlob; + jsonBlobEdit(pParse, j, 0, 0, nIns); + if( !pParse->oom ){ + assert( pParse->aBlob!=0 ); /* Because pParse->oom!=0 */ + assert( ix.aBlob!=0 ); /* Because pPasre->oom!=0 */ + memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); + k = j + ix.nBlob; + memcpy(&pParse->aBlob[k], zKey, nKey); + k += nKey; + memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); + if( ALWAYS(pParse->delta) ) jsonAfterEditSizeAdjust(pParse, iRoot); } - j++; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( pApnd ){ - u32 iStart, iLabel; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); - pParse->aNode[iLabel].jnFlags |= JNODE_RAW; - } - return pNode; + } + jsonParseReset(&v); + jsonParseReset(&ix); + return rc; } }else if( zPath[0]=='[' ){ - i = 0; - j = 1; - while( sqlite3Isdigit(zPath[j]) ){ - i = i*10 + zPath[j] - '0'; - j++; + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; } - if( j<2 || zPath[j]!=']' ){ + if( i<2 || zPath[i]!=']' ){ if( zPath[1]=='#' ){ - JsonNode *pBase = pRoot; - int iBase = iRoot; - if( pRoot->eType!=JSON_ARRAY ) return 0; - for(;;){ - while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; - j += jsonNodeSize(&pBase[j]); - } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; - assert( pBase->eU==2 ); - iBase += pBase->u.iAppend; - pBase = &pParse->aNode[iBase]; - j = 1; - } - j = 2; + k = jsonbArrayCount(pParse, iRoot); + i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int x = 0; - j = 3; + unsigned int nn = 0; + i = 3; do{ - x = x*10 + zPath[j] - '0'; - j++; - }while( sqlite3Isdigit(zPath[j]) ); - if( x>i ) return 0; - i -= x; + nn = nn*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( nn>k ) return JSON_LOOKUP_NOTFOUND; + k -= nn; } - if( zPath[j]!=']' ){ - *pzErr = zPath; - return 0; + if( zPath[i]!=']' ){ + return JSON_LOOKUP_PATHERROR; } }else{ - *pzErr = zPath; - return 0; - } - } - if( pRoot->eType!=JSON_ARRAY ) return 0; - zPath += j + 1; - j = 1; - for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - assert( pRoot->eU==2 ); - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( j<=pRoot->n ){ - return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); - } - if( i==0 && pApnd ){ - u32 iStart; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); - } - return pNode; + return JSON_LOOKUP_PATHERROR; + } + } + j = iRoot+n; + iEnd = j+sz; + while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + k--; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( k>0 ) return JSON_LOOKUP_NOTFOUND; + if( pParse->eEdit>=JEDIT_INS ){ + JsonParse v; + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, v.nBlob) + ){ + assert( !pParse->oom ); + jsonBlobEdit(pParse, j, 0, v.aBlob, v.nBlob); + } + jsonParseReset(&v); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; } }else{ - *pzErr = zPath; + return JSON_LOOKUP_PATHERROR; } - return 0; + return JSON_LOOKUP_NOTFOUND; } /* -** Append content to pParse that will complete zPath. Return a pointer -** to the inserted node, or return NULL if the append fails. +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. */ -static JsonNode *jsonLookupAppend( - JsonParse *pParse, /* Append content to the JSON parse */ - const char *zPath, /* Description of content to append */ - int *pApnd, /* Set this flag to 1 */ - const char **pzErr /* Make this point to any syntax error */ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob ){ - *pApnd = 1; - if( zPath[0]==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + JsonParse x; + JsonString s; + + if( NEVER(aBlob==0) ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonStringInit(&s, ctx); + jsonTranslateBlobToText(&x, 0, &s); + jsonReturnString(&s, 0, 0); +} + + +/* +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the eMode flag +** as follows: +** +** eMode==0 JSONB if the JSON_B flag is set in userdata or +** text if the JSON_B flag is omitted from userdata. +** +** eMode==1 Text +** +** eMode==2 JSONB +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx, /* Return value for this function */ + int eMode /* Format of return: text of JSONB */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( eMode>=0 && eMode<=2 ); + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; } - if( zPath[0]=='.' ){ - jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - }else if( strncmp(zPath,"[0]",3)==0 ){ - jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - }else{ - return 0; + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x; + if( sz==0 ) goto returnfromblob_malformed; + x = (char)pParse->aBlob[i+n]; + if( x=='-' ){ + if( sz<2 ) goto returnfromblob_malformed; + n++; + sz--; + bNeg = 1; + } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc==0 ){ + if( iRes<0 ){ + /* A hexadecimal literal with 16 significant digits and with the + ** high-order bit set is a negative integer in SQLite (and hence + ** iRes comes back as negative) but should be interpreted as a + ** positive value if it occurs within JSON. The value is too + ** large to appear as an SQLite integer so it must be converted + ** into floating point. */ + double r; + r = (double)*(sqlite3_uint64*)&iRes; + sqlite3_result_double(pCtx, bNeg ? -r : r); + }else{ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + } + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else if( rc==1 ){ + goto returnfromblob_malformed; + }else{ + if( bNeg ){ n--; sz++; } + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + if( sz==0 ) goto returnfromblob_malformed; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + if( rc<=0 ) goto returnfromblob_malformed; + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3DbMallocRaw(db, ((u64)nOut)+1); + if( zOut==0 ) goto returnfromblob_oom; + for(iIn=iOut=0; iIn=2 ); + zOut[iOut++] = (char)(0xc0 | (v>>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v<0x10000 ){ + assert( szEscape>=3 ); + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v==JSON_INVALID_CHAR ){ + /* Silently ignore illegal unicode */ + }else{ + assert( szEscape>=4 ); + zOut[iOut++] = 0xf0 | (v>>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + iIn += szEscape - 1; + }else{ + zOut[iOut++] = c; + } + } /* end for() */ + assert( iOut<=nOut ); + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, SQLITE_DYNAMIC); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + if( eMode==0 ){ + if( (SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)) & JSON_BLOB)!=0 ){ + eMode = 2; + }else{ + eMode = 1; + } + } + if( eMode==2 ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } + break; + } + default: { + goto returnfromblob_malformed; + } } - if( pParse->oom ) return 0; - return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); + return; + +returnfromblob_oom: + sqlite3_result_error_nomem(pCtx); + return; + +returnfromblob_malformed: + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; } /* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). +** pArg is a function argument that might be an SQL value or a JSON +** value. Figure out what it is and encode it as a JSONB blob. +** Return the results in pParse. +** +** pParse is uninitialized upon entry. This routine will handle the +** initialization of pParse. The result will be contained in +** pParse->aBlob and pParse->nBlob. pParse->aBlob might be dynamically +** allocated (if pParse->nBlobAlloc is greater than zero) in which case +** the caller is responsible for freeing the space allocated to pParse->aBlob +** when it has finished with it. Or pParse->aBlob might be a static string +** or a value obtained from sqlite3_value_blob(pArg). +** +** If the argument is a BLOB that is clearly not a JSONB, then this +** function might set an error message in ctx and return non-zero. +** It might also set an error message and return non-zero on an OOM error. */ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); +static int jsonFunctionArgToBlob( + sqlite3_context *ctx, + sqlite3_value *pArg, + JsonParse *pParse +){ + int eType = sqlite3_value_type(pArg); + static u8 aNull[] = { 0x00 }; + memset(pParse, 0, sizeof(pParse[0])); + pParse->db = sqlite3_context_db_handle(ctx); + switch( eType ){ + default: { + pParse->aBlob = aNull; + pParse->nBlob = 1; + return 0; + } + case SQLITE_BLOB: { + if( !jsonArgIsJsonb(pArg, pParse) ){ + sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); + return 1; + } + break; + } + case SQLITE_TEXT: { + const char *zJson = (const char*)sqlite3_value_text(pArg); + int nJson = sqlite3_value_bytes(pArg); + if( zJson==0 ) return 1; + if( sqlite3_value_subtype(pArg)==JSON_SUBTYPE ){ + pParse->zJson = (char*)zJson; + pParse->nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3DbFree(pParse->db, pParse->aBlob); + memset(pParse, 0, sizeof(pParse[0])); + return 1; + } + }else{ + jsonBlobAppendNode(pParse, JSONB_TEXTRAW, nJson, zJson); + } + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + if( NEVER(sqlite3IsNaN(r)) ){ + jsonBlobAppendNode(pParse, JSONB_NULL, 0, 0); + }else{ + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + if( z[0]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + }else if( z[0]=='-' && z[1]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, n, z); + } + } + break; + } + case SQLITE_INTEGER: { + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + jsonBlobAppendNode(pParse, JSONB_INT, n, z); + break; + } + } + if( pParse->oom ){ + sqlite3_result_error_nomem(ctx); + return 1; + }else{ + return 0; + } } /* -** Do a node lookup using zPath. Return a pointer to the node on success. -** Return NULL if not found or if there is an error. +** Generate a bad path error. ** -** On an error, write an error message into pCtx and increment the -** pParse->nErr counter. -** -** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if -** nodes are appended. +** If ctx is not NULL then push the error message into ctx and return NULL. +** If ctx is NULL, then return the text of the error message. */ -static JsonNode *jsonLookup( - JsonParse *pParse, /* The JSON to search */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx /* Report errors here, if not NULL */ +static char *jsonBadPathError( + sqlite3_context *ctx, /* The function call containing the error */ + const char *zPath /* The path with the problem */ ){ - const char *zErr = 0; - JsonNode *pNode = 0; - char *zMsg; - - if( zPath==0 ) return 0; - if( zPath[0]!='$' ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); - if( zErr==0 ) return pNode; - -lookup_err: - pParse->nErr++; - assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); + char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + if( ctx==0 ) return zMsg; if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_result_error(ctx, zMsg, -1); sqlite3_free(zMsg); }else{ - sqlite3_result_error_nomem(pCtx); + sqlite3_result_error_nomem(ctx); } return 0; } +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments come in pairs where each pair contains a JSON path and +** content to insert or set at that patch. Do the updates +** and return the result. +** +** The specific operation is determined by eEdit, which can be one +** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +*/ +static void jsonInsertIntoBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv, + int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ +){ + int i; + u32 rc = 0; + const char *zPath = 0; + int flgs; + JsonParse *p; + JsonParse ax; + + assert( (argc&1)==1 ); + flgs = argc==1 ? 0 : JSON_EDITABLE; + p = jsonParseFuncArg(ctx, argv[0], flgs); + if( p==0 ) return; + for(i=1; inBlob, ax.aBlob, ax.nBlob); + } + rc = 0; + }else{ + p->eEdit = eEdit; + p->nIns = ax.nBlob; + p->aIns = ax.aBlob; + p->delta = 0; + rc = jsonLookupStep(p, 0, zPath+1, 0); + } + jsonParseReset(&ax); + if( rc==JSON_LOOKUP_NOTFOUND ) continue; + if( JSON_LOOKUP_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; + } + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +jsonInsertIntoBlob_patherror: + jsonParseFree(p); + if( rc==JSON_LOOKUP_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + jsonBadPathError(ctx, zPath); + } + return; +} /* -** Report the wrong number of arguments for json_insert(), json_replace() -** or json_set(). +** If pArg is a blob that seems like a JSONB blob, then initialize +** p to point to that JSONB and return TRUE. If pArg does not seem like +** a JSONB blob, then return FALSE. +** +** For small BLOBs (having no more than 7 bytes of payload) a full +** validity check is done. So for small BLOBs this routine only returns +** true if the value is guaranteed to be a valid JSONB. For larger BLOBs +** (8 byte or more of payload) only the size of the outermost element is +** checked to verify that the BLOB is superficially valid JSONB. +** +** A full JSONB validation is done on smaller BLOBs because those BLOBs might +** also be text JSON that has been incorrectly cast into a BLOB. +** (See tag-20240123-a and https://sqlite.org/forum/forumpost/012136abd5) +** If the BLOB is 9 bytes are larger, then it is not possible for the +** superficial size check done here to pass if the input is really text +** JSON so we do not need to look deeper in that case. +** +** Why we only need to do full JSONB validation for smaller BLOBs: +** +** The first byte of valid JSON text must be one of: '{', '[', '"', ' ', '\n', +** '\r', '\t', '-', or a digit '0' through '9'. Of these, only a subset +** can also be the first byte of JSONB: '{', '[', and digits '3' +** through '9'. In every one of those cases, the payload size is 7 bytes +** or less. So if we do full JSONB validation for every BLOB where the +** payload is less than 7 bytes, we will never get a false positive for +** JSONB on an input that is really text JSON. */ -static void jsonWrongNumArgs( - sqlite3_context *pCtx, - const char *zFuncName -){ - char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", - zFuncName); - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); +static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ + u32 n, sz = 0; + u8 c; + if( sqlite3_value_type(pArg)!=SQLITE_BLOB ) return 0; + p->aBlob = (u8*)sqlite3_value_blob(pArg); + p->nBlob = (u32)sqlite3_value_bytes(pArg); + if( p->nBlob>0 + && ALWAYS(p->aBlob!=0) + && ((c = p->aBlob[0]) & 0x0f)<=JSONB_OBJECT + && (n = jsonbPayloadSize(p, 0, &sz))>0 + && sz+n==p->nBlob + && ((c & 0x0f)>JSONB_FALSE || sz==0) + && (sz>7 + || (c!=0x7b && c!=0x5b && !sqlite3Isdigit(c)) + || jsonbValidityCheck(p, 0, p->nBlob, 1)==0) + ){ + return 1; + } + p->aBlob = 0; + p->nBlob = 0; + return 0; } /* -** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, +** from the SQL function argument pArg. Return a pointer to the new +** JsonParse object. +** +** Ownership of the new JsonParse object is passed to the caller. The +** caller should invoke jsonParseFree() on the return value when it +** has finished using it. +** +** If any errors are detected, an appropriate error messages is set +** using sqlite3_result_error() or the equivalent and this routine +** returns NULL. This routine also returns NULL if the pArg argument +** is an SQL NULL value, but no error message is set in that case. This +** is so that SQL functions that are given NULL arguments will return +** a NULL value. */ -static void jsonRemoveAllNulls(JsonNode *pNode){ - int i, n; - assert( pNode->eType==JSON_OBJECT ); - n = pNode->n; - for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ - switch( pNode[i].eType ){ - case JSON_NULL: - pNode[i].jnFlags |= JNODE_REMOVE; - break; - case JSON_OBJECT: - jsonRemoveAllNulls(&pNode[i]); - break; +static JsonParse *jsonParseFuncArg( + sqlite3_context *ctx, + sqlite3_value *pArg, + u32 flgs +){ + int eType; /* Datatype of pArg */ + JsonParse *p = 0; /* Value to be returned */ + JsonParse *pFromCache = 0; /* Value taken from cache */ + sqlite3 *db; /* The database connection */ + + assert( ctx!=0 ); + eType = sqlite3_value_type(pArg); + if( eType==SQLITE_NULL ){ + return 0; + } + pFromCache = jsonCacheSearch(ctx, pArg); + if( pFromCache ){ + pFromCache->nJPRef++; + if( (flgs & JSON_EDITABLE)==0 ){ + return pFromCache; + } + } + db = sqlite3_context_db_handle(ctx); +rebuild_from_cache: + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) goto json_pfa_oom; + memset(p, 0, sizeof(*p)); + p->db = db; + p->nJPRef = 1; + if( pFromCache!=0 ){ + u32 nBlob = pFromCache->nBlob; + p->aBlob = sqlite3DbMallocRaw(db, nBlob); + if( p->aBlob==0 ) goto json_pfa_oom; + memcpy(p->aBlob, pFromCache->aBlob, nBlob); + p->nBlobAlloc = p->nBlob = nBlob; + p->hasNonstd = pFromCache->hasNonstd; + jsonParseFree(pFromCache); + return p; + } + if( eType==SQLITE_BLOB ){ + if( jsonArgIsJsonb(pArg,p) ){ + if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ + goto json_pfa_oom; + } + return p; + } + /* If the blob is not valid JSONB, fall through into trying to cast + ** the blob into text which is then interpreted as JSON. (tag-20240123-a) + ** + ** This goes against all historical documentation about how the SQLite + ** JSON functions were suppose to work. From the beginning, blob was + ** reserved for expansion and a blob value should have raised an error. + ** But it did not, due to a bug. And many applications came to depend + ** upon this buggy behavior, especially when using the CLI and reading + ** JSON text using readfile(), which returns a blob. For this reason + ** we will continue to support the bug moving forward. + ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d + */ + } + p->zJson = (char*)sqlite3_value_text(pArg); + p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; + if( p->nJson==0 ) goto json_pfa_malformed; + assert( p->zJson!=0 ); + if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + return 0; } + }else{ + int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); + int rc; + if( !isRCStr ){ + char *zNew = sqlite3RCStrNew( p->nJson ); + if( zNew==0 ) goto json_pfa_oom; + memcpy(zNew, p->zJson, p->nJson); + p->zJson = zNew; + p->zJson[p->nJson] = 0; + }else{ + sqlite3RCStrRef(p->zJson); + } + p->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, p); + if( rc==SQLITE_NOMEM ) goto json_pfa_oom; + if( flgs & JSON_EDITABLE ){ + pFromCache = p; + p = 0; + goto rebuild_from_cache; + } + } + return p; + +json_pfa_malformed: + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + sqlite3_result_error(ctx, "malformed JSON", -1); + return 0; } + +json_pfa_oom: + jsonParseFree(pFromCache); + jsonParseFree(p); + sqlite3_result_error_nomem(ctx); + return 0; } +/* +** Make the return value of a JSON function either the raw JSONB blob +** or make it JSON text, depending on whether the JSON_BLOB flag is +** set on the function. +*/ +static void jsonReturnParse( + sqlite3_context *ctx, + JsonParse *p +){ + int flgs; + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + return; + } + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + if( p->nBlobAlloc>0 && !p->bReadOnly ){ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_DYNAMIC); + p->nBlobAlloc = 0; + }else{ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_TRANSIENT); + } + }else{ + JsonString s; + jsonStringInit(&s, ctx); + p->delta = 0; + jsonTranslateBlobToText(p, 0, &s); + jsonReturnString(&s, p, ctx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } +} /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ -#ifdef SQLITE_DEBUG +#if SQLITE_DEBUG /* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. +** Decode JSONB bytes in aBlob[] starting at iStart through but not +** including iEnd. Indent the +** content by nIndent spaces. */ -static void jsonParseFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv +static void jsonDebugPrintBlob( + JsonParse *pParse, /* JSON content */ + u32 iStart, /* Start rendering here */ + u32 iEnd, /* Do not render this byte or any byte after this one */ + int nIndent, /* Indent by this many spaces */ + sqlite3_str *pOut /* Generate output into this sqlite3_str object */ ){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; - - assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; iaBlob[iStart] & 0x0f; + u32 savedNBlob = pParse->nBlob; + sqlite3_str_appendf(pOut, "%5d:%*s", iStart, nIndent, ""); + if( pParse->nBlobAlloc>pParse->nBlob ){ + pParse->nBlob = pParse->nBlobAlloc; + } + nn = n = jsonbPayloadSize(pParse, iStart, &sz); + if( nn==0 ) nn = 1; + if( sz>0 && xaBlob[iStart+i]); } - jsonAppendRaw(&s, "\n", 1); + if( n==0 ){ + sqlite3_str_appendf(pOut, " ERROR invalid node size\n"); + iStart = n==0 ? iStart+1 : iEnd; + continue; + } + pParse->nBlob = savedNBlob; + if( iStart+n+sz>iEnd ){ + iEnd = iStart+n+sz; + if( iEnd>pParse->nBlob ){ + if( pParse->nBlobAlloc>0 && iEnd>pParse->nBlobAlloc ){ + iEnd = pParse->nBlobAlloc; + }else{ + iEnd = pParse->nBlob; + } + } + } + sqlite3_str_appendall(pOut," <-- "); + switch( x ){ + case JSONB_NULL: sqlite3_str_appendall(pOut,"null"); break; + case JSONB_TRUE: sqlite3_str_appendall(pOut,"true"); break; + case JSONB_FALSE: sqlite3_str_appendall(pOut,"false"); break; + case JSONB_INT: sqlite3_str_appendall(pOut,"int"); break; + case JSONB_INT5: sqlite3_str_appendall(pOut,"int5"); break; + case JSONB_FLOAT: sqlite3_str_appendall(pOut,"float"); break; + case JSONB_FLOAT5: sqlite3_str_appendall(pOut,"float5"); break; + case JSONB_TEXT: sqlite3_str_appendall(pOut,"text"); break; + case JSONB_TEXTJ: sqlite3_str_appendall(pOut,"textj"); break; + case JSONB_TEXT5: sqlite3_str_appendall(pOut,"text5"); break; + case JSONB_TEXTRAW: sqlite3_str_appendall(pOut,"textraw"); break; + case JSONB_ARRAY: { + sqlite3_str_appendf(pOut,"array, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); + showContent = 0; + break; + } + case JSONB_OBJECT: { + sqlite3_str_appendf(pOut, "object, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); + showContent = 0; + break; + } + default: { + sqlite3_str_appendall(pOut, "ERROR: unknown node type\n"); + showContent = 0; + break; + } + } + if( showContent ){ + if( sz==0 && x<=JSONB_FALSE ){ + sqlite3_str_append(pOut, "\n", 1); + }else{ + u32 j; + sqlite3_str_appendall(pOut, ": \""); + for(j=iStart+n; jaBlob[j]; + if( c<0x20 || c>=0x7f ) c = '.'; + sqlite3_str_append(pOut, (char*)&c, 1); + } + sqlite3_str_append(pOut, "\"\n", 2); + } + } + iStart += n + sz; + } +} +static void jsonShowParse(JsonParse *pParse){ + sqlite3_str out; + char zBuf[1000]; + if( pParse==0 ){ + printf("NULL pointer\n"); + return; + }else{ + printf("nBlobAlloc = %u\n", pParse->nBlobAlloc); + printf("nBlob = %u\n", pParse->nBlob); + printf("delta = %d\n", pParse->delta); + if( pParse->nBlob==0 ) return; + printf("content (bytes 0..%u):\n", pParse->nBlob-1); } - jsonParseReset(&x); - jsonResult(&s); + sqlite3StrAccumInit(&out, 0, zBuf, sizeof(zBuf), 1000000); + jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0, &out); + printf("%s", sqlite3_str_value(&out)); + sqlite3_str_reset(&out); } +#endif /* SQLITE_DEBUG */ +#ifdef SQLITE_DEBUG /* -** The json_test1(JSON) function return true (1) if the input is JSON -** text generated by another json function. It returns (0) if the input -** is not known to be JSON. +** SQL function: json_parse(JSON) +** +** Parse JSON using jsonParseFuncArg(). Return text that is a +** human-readable dump of the binary JSONB for the input parameter. */ -static void jsonTest1Func( +static void jsonParseFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - UNUSED_PARAMETER(argc); - sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); + JsonParse *p; /* The parse */ + sqlite3_str out; + + assert( argc>=1 ); + sqlite3StrAccumInit(&out, 0, 0, 0, 1000000); + p = jsonParseFuncArg(ctx, argv[0], 0); + if( p==0 ) return; + if( argc==1 ){ + jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); + }else{ + jsonShowParse(p); + } + jsonParseFree(p); + sqlite3_str_reset(&out); } #endif /* SQLITE_DEBUG */ @@ -2034,8 +3878,8 @@ static void jsonTest1Func( ****************************************************************************/ /* -** Implementation of the json_QUOTE(VALUE) function. Return a JSON value -** corresponding to the SQL value input. Mostly this means putting +** Implementation of the json_quote(VALUE) function. Return a JSON value +** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. */ @@ -2047,9 +3891,9 @@ static void jsonQuoteFunc( JsonString jx; UNUSED_PARAMETER(argc); - jsonInit(&jx, ctx); - jsonAppendValue(&jx, argv[0]); - jsonResult(&jx); + jsonStringInit(&jx, ctx); + jsonAppendSqlValue(&jx, argv[0]); + jsonReturnString(&jx, 0, 0); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2066,23 +3910,22 @@ static void jsonArrayFunc( int i; JsonString jx; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=0; inNode ); if( argc==2 ){ const char *zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); + if( zPath==0 ){ + jsonParseFree(p); + return; + } + i = jsonLookupStep(p, 0, zPath[0]=='$' ? zPath+1 : "@", 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + eErr = 1; + i = 0; + } }else{ - pNode = p->aNode; - } - if( pNode==0 ){ - return; + i = 0; } - if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); - } + if( (p->aBlob[i] & 0x0f)==JSONB_ARRAY ){ + cnt = jsonbArrayCount(p, i); } - sqlite3_result_int64(ctx, n); + if( !eErr ) sqlite3_result_int64(ctx, cnt); + jsonParseFree(p); } -/* -** Bit values for the flags passed into jsonExtractFunc() or -** jsonSetFunc() via the user-data value. -*/ -#define JSON_JSON 0x01 /* Result is always JSON */ -#define JSON_SQL 0x02 /* Result is always SQL */ -#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ -#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +/* True if the string is all alphanumerics and underscores */ +static int jsonAllAlphanum(const char *z, int n){ + int i; + for(i=0; i2 ){ + jsonAppendChar(&jx, '['); + } + for(i=1; i and ->> operators accept abbreviated PATH arguments. This - ** is mostly for compatibility with PostgreSQL, but also for - ** convenience. - ** - ** NUMBER ==> $[NUMBER] // PG compatible - ** LABEL ==> $.LABEL // PG compatible - ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience - */ - jsonInit(&jx, ctx); - if( sqlite3Isdigit(zPath[0]) ){ - jsonAppendRaw(&jx, "$[", 2); - jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendRaw(&jx, "]", 2); - }else{ - jsonAppendRaw(&jx, "$.", 1 + (zPath[0]!='[')); - jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendChar(&jx, 0); - } - pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); - jsonReset(&jx); + const char *zPath = (const char*)sqlite3_value_text(argv[i]); + int nPath; + u32 j; + if( zPath==0 ) goto json_extract_error; + nPath = sqlite3Strlen30(zPath); + if( zPath[0]=='$' ){ + j = jsonLookupStep(p, 0, zPath+1, 0); + }else if( (flags & JSON_ABPATH) ){ + /* The -> and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for + ** convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + ** + ** Updated 2024-05-27: If the NUMBER is negative, then PG counts from + ** the right of the array. Hence for negative NUMBER: + ** + ** NUMBER ==> $[#NUMBER] // PG compatible + */ + jsonStringInit(&jx, ctx); + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ + jsonAppendRawNZ(&jx, "[", 1); + if( zPath[0]=='-' ) jsonAppendRawNZ(&jx,"#",1); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "]", 2); + }else if( jsonAllAlphanum(zPath, nPath) ){ + jsonAppendRawNZ(&jx, ".", 1); + jsonAppendRaw(&jx, zPath, nPath); + }else if( zPath[0]=='[' && nPath>=3 && zPath[nPath-1]==']' ){ + jsonAppendRaw(&jx, zPath, nPath); }else{ - pNode = jsonLookup(p, zPath, 0, ctx); + jsonAppendRawNZ(&jx, ".\"", 2); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "\"", 1); } - if( pNode ){ + jsonStringTerminate(&jx); + j = jsonLookupStep(p, 0, jx.zBuf, 0); + jsonStringReset(&jx); + }else{ + jsonBadPathError(ctx, zPath); + goto json_extract_error; + } + if( jnBlob ){ + if( argc==2 ){ if( flags & JSON_JSON ){ - jsonReturnJson(pNode, ctx, 0); + jsonStringInit(&jx, ctx); + jsonTranslateBlobToText(p, j, &jx); + jsonReturnString(&jx, 0, 0); + jsonStringReset(&jx); + assert( (flags & JSON_BLOB)==0 ); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); }else{ - jsonReturn(pNode, ctx, 0); - sqlite3_result_subtype(ctx, 0); + jsonReturnFromBlob(p, j, ctx, 0); + if( (flags & (JSON_SQL|JSON_BLOB))==0 + && (p->aBlob[j]&0x0f)>=JSONB_ARRAY + ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } } - } - }else{ - pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0); - } - }else{ - /* Two or more PATH arguments results in a JSON array with each - ** element of the array being the value selected by one of the PATHs */ - int i; - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '['); - for(i=1; inErr ) break; - jsonAppendSeparator(&jx); - if( pNode ){ - jsonRenderNode(pNode, &jx, 0); }else{ - jsonAppendRaw(&jx, "null", 4); + jsonAppendSeparator(&jx); + jsonTranslateBlobToText(p, j, &jx); } + }else if( j==JSON_LOOKUP_NOTFOUND ){ + if( argc==2 ){ + goto json_extract_error; /* Return NULL if not found */ + }else{ + jsonAppendSeparator(&jx); + jsonAppendRawNZ(&jx, "null", 4); + } + }else if( j==JSON_LOOKUP_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + goto json_extract_error; + }else{ + jsonBadPathError(ctx, zPath); + goto json_extract_error; } - if( i==argc ){ - jsonAppendChar(&jx, ']'); - jsonResult(&jx); + } + if( argc>2 ){ + jsonAppendChar(&jx, ']'); + jsonReturnString(&jx, 0, 0); + if( (flags & JSON_BLOB)==0 ){ sqlite3_result_subtype(ctx, JSON_SUBTYPE); } - jsonReset(&jx); } +json_extract_error: + jsonStringReset(&jx); + jsonParseFree(p); + return; } -/* This is the RFC 7396 MergePatch algorithm. +/* +** Return codes for jsonMergePatch() +*/ +#define JSON_MERGE_OK 0 /* Success */ +#define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ +#define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ +#define JSON_MERGE_OOM 3 /* Out-of-memory condition */ + +/* +** RFC-7396 MergePatch for two JSONB blobs. +** +** pTarget is the target. pPatch is the patch. The target is updated +** in place. The patch is read-only. +** +** The original RFC-7396 algorithm is this: +** +** define MergePatch(Target, Patch): +** if Patch is an Object: +** if Target is not an Object: +** Target = {} # Ignore the contents and set it to an empty Object +** for each Name/Value pair in Patch: +** if Value is null: +** if Name exists in Target: +** remove the Name/Value pair from Target +** else: +** Target[Name] = MergePatch(Target[Name], Value) +** return Target +** else: +** return Patch +** +** Here is an equivalent algorithm restructured to show the actual +** implementation: +** +** 01 define MergePatch(Target, Patch): +** 02 if Patch is not an Object: +** 03 return Patch +** 04 else: // if Patch is an Object +** 05 if Target is not an Object: +** 06 Target = {} +** 07 for each Name/Value pair in Patch: +** 08 if Name exists in Target: +** 09 if Value is null: +** 10 remove the Name/Value pair from Target +** 11 else +** 12 Target[name] = MergePatch(Target[Name], Value) +** 13 else if Value is not NULL: +** 14 if Value is not an Object: +** 15 Target[name] = Value +** 16 else: +** 17 Target[name] = MergePatch('{}',value) +** 18 return Target +** | +** ^---- Line numbers referenced in comments in the implementation */ -static JsonNode *jsonMergePatch( - JsonParse *pParse, /* The JSON parser that contains the TARGET */ - u32 iTarget, /* Node of the TARGET in pParse */ - JsonNode *pPatch /* The PATCH */ +static int jsonMergePatch( + JsonParse *pTarget, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ + const JsonParse *pPatch, /* The PATCH */ + u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ ){ - u32 i, j; - u32 iRoot; - JsonNode *pTarget; - if( pPatch->eType!=JSON_OBJECT ){ - return pPatch; - } - assert( iTargetnNode ); - pTarget = &pParse->aNode[iTarget]; - assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); - if( pTarget->eType!=JSON_OBJECT ){ - jsonRemoveAllNulls(pPatch); - return pPatch; - } - iRoot = iTarget; - for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ - u32 nKey; - const char *zKey; - assert( pPatch[i].eType==JSON_STRING ); - assert( pPatch[i].jnFlags & JNODE_LABEL ); - assert( pPatch[i].eU==1 ); - nKey = pPatch[i].n; - zKey = pPatch[i].u.zJContent; - for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ - assert( pTarget[j].eType==JSON_STRING ); - assert( pTarget[j].jnFlags & JNODE_LABEL ); - if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; - if( pPatch[i+1].eType==JSON_NULL ){ - pTarget[j+1].jnFlags |= JNODE_REMOVE; - }else{ - JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); - if( pNew==0 ) return 0; - pTarget = &pParse->aNode[iTarget]; - if( pNew!=&pTarget[j+1] ){ - assert( pTarget[j+1].eU==0 - || pTarget[j+1].eU==1 - || pTarget[j+1].eU==2 ); - testcase( pTarget[j+1].eU==1 ); - testcase( pTarget[j+1].eU==2 ); - VVA( pTarget[j+1].eU = 5 ); - pTarget[j+1].u.pPatch = pNew; - pTarget[j+1].jnFlags |= JNODE_PATCH; - } - } - break; + u8 x; /* Type of a single node */ + u32 n, sz=0; /* Return values from jsonbPayloadSize() */ + u32 iTCursor; /* Cursor position while scanning the target object */ + u32 iTStart; /* First label in the target object */ + u32 iTEndBE; /* Original first byte past end of target, before edit */ + u32 iTEnd; /* Current first byte past end of target */ + u8 eTLabel; /* Node type of the target label */ + u32 iTLabel = 0; /* Index of the label */ + u32 nTLabel = 0; /* Header size in bytes for the target label */ + u32 szTLabel = 0; /* Size of the target label payload */ + u32 iTValue = 0; /* Index of the target value */ + u32 nTValue = 0; /* Header size of the target value */ + u32 szTValue = 0; /* Payload size for the target value */ + + u32 iPCursor; /* Cursor position while scanning the patch */ + u32 iPEnd; /* First byte past the end of the patch */ + u8 ePLabel; /* Node type of the patch label */ + u32 iPLabel; /* Start of patch label */ + u32 nPLabel; /* Size of header on the patch label */ + u32 szPLabel; /* Payload size of the patch label */ + u32 iPValue; /* Start of patch value */ + u32 nPValue; /* Header size for the patch value */ + u32 szPValue; /* Payload size of the patch value */ + + assert( iTarget>=0 && iTargetnBlob ); + assert( iPatch>=0 && iPatchnBlob ); + x = pPatch->aBlob[iPatch] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 02 */ + u32 szPatch; /* Total size of the patch, header+payload */ + u32 szTarget; /* Total size of the target, header+payload */ + n = jsonbPayloadSize(pPatch, iPatch, &sz); + szPatch = n+sz; + sz = 0; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + szTarget = n+sz; + jsonBlobEdit(pTarget, iTarget, szTarget, pPatch->aBlob+iPatch, szPatch); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; /* Line 03 */ + } + x = pTarget->aBlob[iTarget] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 05 */ + n = jsonbPayloadSize(pTarget, iTarget, &sz); + jsonBlobEdit(pTarget, iTarget+n, sz, 0, 0); + x = pTarget->aBlob[iTarget]; + pTarget->aBlob[iTarget] = (x & 0xf0) | JSONB_OBJECT; + } + n = jsonbPayloadSize(pPatch, iPatch, &sz); + if( NEVER(n==0) ) return JSON_MERGE_BADPATCH; + iPCursor = iPatch+n; + iPEnd = iPCursor+sz; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + if( NEVER(n==0) ) return JSON_MERGE_BADTARGET; + iTStart = iTarget+n; + iTEndBE = iTStart+sz; + + while( iPCursoraBlob[iPCursor] & 0x0f; + if( ePLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADPATCH; + } + nPLabel = jsonbPayloadSize(pPatch, iPCursor, &szPLabel); + if( nPLabel==0 ) return JSON_MERGE_BADPATCH; + iPValue = iPCursor + nPLabel + szPLabel; + if( iPValue>=iPEnd ) return JSON_MERGE_BADPATCH; + nPValue = jsonbPayloadSize(pPatch, iPValue, &szPValue); + if( nPValue==0 ) return JSON_MERGE_BADPATCH; + iPCursor = iPValue + nPValue + szPValue; + if( iPCursor>iPEnd ) return JSON_MERGE_BADPATCH; + + iTCursor = iTStart; + iTEnd = iTEndBE + pTarget->delta; + while( iTCursoraBlob[iTCursor] & 0x0f; + if( eTLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADTARGET; } + nTLabel = jsonbPayloadSize(pTarget, iTCursor, &szTLabel); + if( nTLabel==0 ) return JSON_MERGE_BADTARGET; + iTValue = iTLabel + nTLabel + szTLabel; + if( iTValue>=iTEnd ) return JSON_MERGE_BADTARGET; + nTValue = jsonbPayloadSize(pTarget, iTValue, &szTValue); + if( nTValue==0 ) return JSON_MERGE_BADTARGET; + if( iTValue + nTValue + szTValue > iTEnd ) return JSON_MERGE_BADTARGET; + isEqual = jsonLabelCompare( + (const char*)&pPatch->aBlob[iPLabel+nPLabel], + szPLabel, + (ePLabel==JSONB_TEXT || ePLabel==JSONB_TEXTRAW), + (const char*)&pTarget->aBlob[iTLabel+nTLabel], + szTLabel, + (eTLabel==JSONB_TEXT || eTLabel==JSONB_TEXTRAW)); + if( isEqual ) break; + iTCursor = iTValue + nTValue + szTValue; } - if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart, iPatch; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - if( pParse->oom ) return 0; - jsonRemoveAllNulls(pPatch); - pTarget = &pParse->aNode[iTarget]; - assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 ); - testcase( pParse->aNode[iRoot].eU==2 ); - pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; - VVA( pParse->aNode[iRoot].eU = 2 ); - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; - iRoot = iStart; - assert( pParse->aNode[iPatch].eU==0 ); - VVA( pParse->aNode[iPatch].eU = 5 ); - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + x = pPatch->aBlob[iPValue] & 0x0f; + if( iTCursoroom) ) return JSON_MERGE_OOM; + }else{ + /* Algorithm line 12 */ + int rc, savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); + if( rc ) return rc; + pTarget->delta += savedDelta; + } + }else if( x>0 ){ /* Algorithm line 13 */ + /* No match and patch value is not NULL */ + u32 szNew = szPLabel+nPLabel; + if( (pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ /* Line 14 */ + jsonBlobEdit(pTarget, iTEnd, 0, 0, szPValue+nPValue+szNew); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + memcpy(&pTarget->aBlob[iTEnd+szNew], + &pPatch->aBlob[iPValue], szPValue+nPValue); + }else{ + int rc, savedDelta; + jsonBlobEdit(pTarget, iTEnd, 0, 0, szNew+1); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + pTarget->aBlob[iTEnd+szNew] = 0x00; + savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); + if( rc ) return rc; + pTarget->delta += savedDelta; + } } } - return pTarget; + if( pTarget->delta ) jsonAfterEditSizeAdjust(pTarget, iTarget); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; } + /* ** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON ** object that is the result of running the RFC 7396 MergePatch() algorithm @@ -2313,25 +4317,27 @@ static void jsonPatchFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The JSON that is being patched */ - JsonParse y; /* The patch */ - JsonNode *pResult; /* The result of the merge */ + JsonParse *pTarget; /* The TARGET */ + JsonParse *pPatch; /* The PATCH */ + int rc; /* Result code */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ - jsonParseReset(&x); - return; - } - pResult = jsonMergePatch(&x, 0, y.aNode); - assert( pResult!=0 || x.oom ); - if( pResult ){ - jsonReturnJson(pResult, ctx, 0); - }else{ - sqlite3_result_error_nomem(ctx); + assert( argc==2 ); + pTarget = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); + if( pTarget==0 ) return; + pPatch = jsonParseFuncArg(ctx, argv[1], 0); + if( pPatch ){ + rc = jsonMergePatch(pTarget, 0, pPatch, 0); + if( rc==JSON_MERGE_OK ){ + jsonReturnParse(ctx, pTarget); + }else if( rc==JSON_MERGE_OOM ){ + sqlite3_result_error_nomem(ctx); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + jsonParseFree(pPatch); } - jsonParseReset(&x); - jsonParseReset(&y); + jsonParseFree(pTarget); } @@ -2355,23 +4361,23 @@ static void jsonObjectFunc( "of arguments", -1); return; } - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '{'); for(i=0; i1 ? JSON_EDITABLE : 0); + if( p==0 ) return; + for(i=1; ijnFlags |= JNODE_REMOVE; - } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); + if( zPath==0 ){ + goto json_remove_done; + } + if( zPath[0]!='$' ){ + goto json_remove_patherror; + } + if( zPath[1]==0 ){ + /* json_remove(j,'$') returns NULL */ + goto json_remove_done; + } + p->eEdit = JEDIT_DEL; + p->delta = 0; + rc = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(rc) ){ + if( rc==JSON_LOOKUP_NOTFOUND ){ + continue; /* No-op */ + }else if( rc==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_remove_done; + } } -remove_done: - jsonParseReset(&x); + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +json_remove_patherror: + jsonBadPathError(ctx, zPath); + +json_remove_done: + jsonParseFree(p); + return; } /* @@ -2420,38 +4450,12 @@ static void jsonReplaceFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - if( argc<1 ) return; if( (argc&1)==0 ) { jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; - if( pNode ){ - assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 ); - testcase( pNode->eU!=0 && pNode->eU!=1 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - VVA( pNode->eU = 4 ); - pNode->u.iReplace = i + 1; - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -replace_err: - jsonParseReset(&x); + jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); } @@ -2472,45 +4476,16 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - int bApnd; - int bIsSet = sqlite3_user_data(ctx)!=0; + + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + int bIsSet = (flags&JSON_ISSET)!=0; if( argc<1 ) return; if( (argc&1)==0 ) { jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ - sqlite3_result_error_nomem(ctx); - goto jsonSetDone; - }else if( x.nErr ){ - goto jsonSetDone; - }else if( pNode && (bApnd || bIsSet) ){ - testcase( pNode->eU!=0 && pNode->eU!=1 ); - assert( pNode->eU!=3 && pNode->eU!=5 ); - VVA( pNode->eU = 4 ); - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - assert( x.aNode[0].eU==4 ); - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -jsonSetDone: - jsonParseReset(&x); + jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); } /* @@ -2526,101 +4501,250 @@ static void jsonTypeFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ - const char *zPath; - JsonNode *pNode; + const char *zPath = 0; + u32 i; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); + if( zPath==0 ) goto json_type_done; + if( zPath[0]!='$' ){ + jsonBadPathError(ctx, zPath); + goto json_type_done; + } + i = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_type_done; + } }else{ - pNode = p->aNode; - } - if( pNode ){ - sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + i = 0; } + sqlite3_result_text(ctx, jsonbType[p->aBlob[i]&0x0f], -1, SQLITE_STATIC); +json_type_done: + jsonParseFree(p); } /* -** json_valid(JSON) +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. ** -** Return 1 if JSON is a well-formed canonical JSON string according -** to RFC-7159. Return 0 otherwise. +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). */ -static void jsonValidFunc( +static void jsonPrettyFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonParse *p; /* The parse */ - UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(ctx); - sqlite3_free(p); + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; }else{ - sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); - if( p->nErr ) jsonParseFree(p); + x.szIndent = (u32)strlen(x.zIndent); } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); } /* -** json_error_position(JSON) +** json_valid(JSON) +** json_valid(JSON, FLAGS) +** +** Check the JSON argument to see if it is well-formed. The FLAGS argument +** encodes the various constraints on what is meant by "well-formed": +** +** 0x01 Canonical RFC-8259 JSON text +** 0x02 JSON text with optional JSON-5 extensions +** 0x04 Superficially appears to be JSONB +** 0x08 Strictly well-formed JSONB ** -** If the argument is not an interpretable JSON string, then return the 1-based -** character position at which the parser first recognized that the input -** was in error. The left-most character is 1. If the string is valid -** JSON, then return 0. +** If the FLAGS argument is omitted, it defaults to 1. Useful values for +** FLAGS include: ** -** Note that json_valid() is only true for strictly conforming canonical JSON. -** But this routine returns zero if the input contains extension. Thus: +** 1 Strict canonical JSON text +** 2 JSON text perhaps with JSON-5 extensions +** 4 Superficially appears to be JSONB +** 5 Canonical JSON text or superficial JSONB +** 6 JSON-5 text or superficial JSONB +** 8 Strict JSONB +** 9 Canonical JSON text or strict JSONB +** 10 JSON-5 text or strict JSONB ** -** (1) If the input X is strictly conforming canonical JSON: +** Other flag combinations are redundant. For example, every canonical +** JSON text is also well-formed JSON-5 text, so FLAG values 2 and 3 +** are the same. Similarly, any input that passes a strict JSONB validation +** will also pass the superficial validation so 12 through 15 are the same +** as 8 through 11 respectively. ** -** json_valid(X) returns true -** json_error_position(X) returns 0 +** This routine runs in linear time to validate text and when doing strict +** JSONB validation. Superficial JSONB validation is constant time, +** assuming the BLOB is already in memory. The performance advantage +** of superficial JSONB validation is why that option is provided. +** Application developers can choose to do fast superficial validation or +** slower strict validation, according to their specific needs. ** -** (2) If the input X is JSON but it includes extension (such as JSON5) that -** are not part of RFC-8259: +** Only the lower four bits of the FLAGS argument are currently used. +** Higher bits are reserved for future expansion. To facilitate +** compatibility, the current implementation raises an error if any bit +** in FLAGS is set other than the lower four bits. ** -** json_valid(X) returns false -** json_error_position(X) return 0 +** The original circa 2015 implementation of the JSON routines in +** SQLite only supported canonical RFC-8259 JSON text and the json_valid() +** function only accepted one argument. That is why the default value +** for the FLAGS argument is 1, since FLAGS=1 causes this routine to only +** recognize canonical RFC-8259 JSON text as valid. The extra FLAGS +** argument was added when the JSON routines were extended to support +** JSON5-like extensions and binary JSONB stored in BLOBs. ** -** (3) If the input X cannot be interpreted as JSON even taking extensions -** into account: +** Return Values: ** -** json_valid(X) return false -** json_error_position(X) returns 1 or more +** * Raise an error if FLAGS is outside the range of 1 to 15. +** * Return NULL if the input is NULL +** * Return 1 if the input is well-formed. +** * Return 0 if the input is not well-formed. */ -static void jsonErrorFunc( +static void jsonValidFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ JsonParse *p; /* The parse */ + u8 flags = 1; + u8 res = 0; + if( argc==2 ){ + i64 f = sqlite3_value_int64(argv[1]); + if( f<1 || f>15 ){ + sqlite3_result_error(ctx, "FLAGS parameter to json_valid() must be" + " between 1 and 15", -1); + return; + } + flags = f & 0x0f; + } + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_NULL: { +#ifdef SQLITE_LEGACY_JSON_VALID + /* Incorrect legacy behavior was to return FALSE for a NULL input */ + sqlite3_result_int(ctx, 0); +#endif + return; + } + case SQLITE_BLOB: { + JsonParse py; + memset(&py, 0, sizeof(py)); + if( jsonArgIsJsonb(argv[0], &py) ){ + if( flags & 0x04 ){ + /* Superficial checking only - accomplished by the + ** jsonArgIsJsonb() call above. */ + res = 1; + }else if( flags & 0x08 ){ + /* Strict checking. Check by translating BLOB->TEXT->BLOB. If + ** no errors occur, call that a "strict check". */ + res = 0==jsonbValidityCheck(&py, 0, py.nBlob, 1); + } + break; + } + /* Fall through into interpreting the input as text. See note + ** above at tag-20240123-a. */ + /* no break */ deliberate_fall_through + } + default: { + JsonParse px; + if( (flags & 0x3)==0 ) break; + memset(&px, 0, sizeof(px)); + + p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); + if( p ){ + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + }else if( p->nErr ){ + /* no-op */ + }else if( (flags & 0x02)!=0 || p->hasNonstd==0 ){ + res = 1; + } + jsonParseFree(p); + }else{ + sqlite3_result_error_nomem(ctx); + } + break; + } + } + sqlite3_result_int(ctx, res); +} + +/* +** json_error_position(JSON) +** +** If the argument is NULL, return NULL +** +** If the argument is BLOB, do a full validity check and return non-zero +** if the check fails. The return value is the approximate 1-based offset +** to the byte of the element that contains the first error. +** +** Otherwise interpret the argument is TEXT (even if it is numeric) and +** return the 1-based character position for where the parser first recognized +** that the input was not valid JSON, or return 0 if the input text looks +** ok. JSON-5 extensions are accepted. +*/ +static void jsonErrorFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + i64 iErrPos = 0; /* Error position to be returned */ + JsonParse s; + + assert( argc==1 ); UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv, 0); - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(ctx); - sqlite3_free(p); - }else if( p->nErr==0 ){ - sqlite3_result_int(ctx, 0); + memset(&s, 0, sizeof(s)); + s.db = sqlite3_context_db_handle(ctx); + if( jsonArgIsJsonb(argv[0], &s) ){ + iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); }else{ - int n = 1; - u32 i; - const char *z = p->zJson; - for(i=0; iiErr && ALWAYS(z[i]); i++){ - if( (z[i]&0xc0)!=0x80 ) n++; + s.zJson = (char*)sqlite3_value_text(argv[0]); + if( s.zJson==0 ) return; /* NULL input or OOM */ + s.nJson = sqlite3_value_bytes(argv[0]); + if( jsonConvertTextToBlob(&s,0) ){ + if( s.oom ){ + iErrPos = -1; + }else{ + /* Convert byte-offset s.iErr into a character offset */ + u32 k; + assert( s.zJson!=0 ); /* Because s.oom is false */ + for(k=0; kzBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '['); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - jsonAppendValue(pStr, argv[0]); + jsonAppendSqlValue(pStr, argv[0]); } } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr, 0, 0); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); + }else{ + jsonStringTrimOneChar(pStr); + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); @@ -2700,7 +4835,7 @@ static void jsonGroupInverse( pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); #ifdef NEVER /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will - ** always have been called to initalize it */ + ** always have been called to initialize it */ if( NEVER(!pStr) ) return; #endif z = pStr->zBuf; @@ -2743,35 +4878,49 @@ static void jsonObjectStep( UNUSED_PARAMETER(argc); pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ + z = (const char*)sqlite3_value_text(argv[0]); + n = sqlite3Strlen30(z); if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); - }else if( pStr->nUsed>1 ){ + }else if( pStr->nUsed>1 && z!=0 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - z = (const char*)sqlite3_value_text(argv[0]); - n = (u32)sqlite3_value_bytes(argv[0]); - jsonAppendString(pStr, z, n); - jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); + if( z!=0 ){ + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendSqlValue(pStr, argv[1]); + } } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; jsonAppendChar(pStr, '}'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr, 0, 0); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); + }else{ + jsonStringTrimOneChar(pStr); + } + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); pStr->bStatic = 1; }else{ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); - pStr->nUsed--; + jsonStringTrimOneChar(pStr); } }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); @@ -2791,19 +4940,40 @@ static void jsonObjectFinal(sqlite3_context *ctx){ /**************************************************************************** ** The json_each virtual table ****************************************************************************/ +typedef struct JsonParent JsonParent; +struct JsonParent { + u32 iHead; /* Start of object or array */ + u32 iValue; /* Start of the value */ + u32 iEnd; /* First byte past the end */ + u32 nPath; /* Length of path */ + i64 iKey; /* Key for JSONB_ARRAY */ +}; + typedef struct JsonEachCursor JsonEachCursor; struct JsonEachCursor { sqlite3_vtab_cursor base; /* Base class - must be first */ u32 iRowid; /* The rowid */ - u32 iBegin; /* The first node of the scan */ - u32 i; /* Index in sParse.aNode[] of current row */ + u32 i; /* Index in sParse.aBlob[] of current row */ u32 iEnd; /* EOF when i equals or exceeds this value */ - u8 eType; /* Type of top-level element */ + u32 nRoot; /* Size of the root path in bytes */ + u8 eType; /* Type of the container for element i */ u8 bRecursive; /* True for json_tree(). False for json_each() */ - char *zJson; /* Input JSON */ - char *zRoot; /* Path by which to filter zJson */ + u8 eMode; /* 1 for json_each(). 2 for jsonb_each() */ + u32 nParent; /* Current nesting depth */ + u32 nParentAlloc; /* Space allocated for aParent[] */ + JsonParent *aParent; /* Parent elements of i */ + sqlite3 *db; /* Database connection */ + JsonString path; /* Current path */ JsonParse sParse; /* Parse of the input JSON */ }; +typedef struct JsonEachConnection JsonEachConnection; +struct JsonEachConnection { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ + u8 eMode; /* 1 for json_each(). 2 for jsonb_each() */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ +}; + /* Constructor for the json_each virtual table */ static int jsonEachConnect( @@ -2813,7 +4983,7 @@ static int jsonEachConnect( sqlite3_vtab **ppVtab, char **pzErr ){ - sqlite3_vtab *pNew; + JsonEachConnection *pNew; int rc; /* Column numbers */ @@ -2835,65 +5005,65 @@ static int jsonEachConnect( UNUSED_PARAMETER(argv); UNUSED_PARAMETER(argc); UNUSED_PARAMETER(pAux); - rc = sqlite3_declare_vtab(db, + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = (JsonEachConnection*)sqlite3DbMallocZero(db, sizeof(*pNew)); + *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + pNew->db = db; + pNew->eMode = argv[0][4]=='b' ? 2 : 1; + pNew->bRecursive = argv[0][4+pNew->eMode]=='t'; } return rc; } /* destructor for json_each virtual table */ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); + JsonEachConnection *p = (JsonEachConnection*)pVtab; + sqlite3DbFree(p->db, pVtab); return SQLITE_OK; } -/* constructor for a JsonEachCursor object for json_each(). */ -static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ +/* constructor for a JsonEachCursor object for json_each()/json_tree(). */ +static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachConnection *pVtab = (JsonEachConnection*)p; JsonEachCursor *pCur; UNUSED_PARAMETER(p); - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3DbMallocZero(pVtab->db, sizeof(*pCur)); if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); + pCur->db = pVtab->db; + pCur->eMode = pVtab->eMode; + pCur->bRecursive = pVtab->bRecursive; + jsonStringZero(&pCur->path); *ppCursor = &pCur->base; return SQLITE_OK; } -/* constructor for a JsonEachCursor object for json_tree(). */ -static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - int rc = jsonEachOpenEach(p, ppCursor); - if( rc==SQLITE_OK ){ - JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; - pCur->bRecursive = 1; - } - return rc; -} - /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); - sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); + jsonStringReset(&p->path); + sqlite3DbFree(p->db, p->aParent); p->iRowid = 0; p->i = 0; + p->aParent = 0; + p->nParent = 0; + p->nParentAlloc = 0; p->iEnd = 0; p->eType = 0; - p->zJson = 0; - p->zRoot = 0; } /* Destructor for a jsonEachCursor object */ static int jsonEachClose(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; jsonEachCursorReset(p); - sqlite3_free(cur); + + sqlite3DbFree(p->db, cur); return SQLITE_OK; } @@ -2904,200 +5074,233 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ return p->i >= p->iEnd; } +/* +** If the cursor is currently pointing at the label of a object entry, +** then return the index of the value. For all other cases, return the +** current pointer position, which is the value. +*/ +static int jsonSkipLabel(JsonEachCursor *p){ + if( p->eType==JSONB_OBJECT ){ + u32 sz = 0; + u32 n = jsonbPayloadSize(&p->sParse, p->i, &sz); + return p->i + n + sz; + }else{ + return p->i; + } +} + +/* +** Append the path name for the current element. +*/ +static void jsonAppendPathName(JsonEachCursor *p){ + assert( p->nParent>0 ); + assert( p->eType==JSONB_ARRAY || p->eType==JSONB_OBJECT ); + if( p->eType==JSONB_ARRAY ){ + jsonPrintf(30, &p->path, "[%lld]", p->aParent[p->nParent-1].iKey); + }else{ + u32 n, sz = 0, k, i; + const char *z; + int needQuote = 0; + n = jsonbPayloadSize(&p->sParse, p->i, &sz); + k = p->i + n; + z = (const char*)&p->sParse.aBlob[k]; + if( sz==0 || !sqlite3Isalpha(z[0]) ){ + needQuote = 1; + }else{ + for(i=0; ipath,".\"%.*s\"", sz, z); + }else{ + jsonPrintf(sz+2,&p->path,".%.*s", sz, z); + } + } +} + /* Advance the cursor to the next element for json_tree() */ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; + int rc = SQLITE_OK; if( p->bRecursive ){ - if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; - p->i++; - p->iRowid++; - if( p->iiEnd ){ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==0 || pUp->eU==3 ); - testcase( pUp->eU==3 ); - VVA( pUp->eU = 3 ); - if( iUp==p->i-1 ){ - pUp->u.iKey = 0; - }else{ - pUp->u.iKey++; - } - } - } - }else{ - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; + u8 x; + u8 levelChange = 0; + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + x = p->sParse.aBlob[i] & 0x0f; + n = jsonbPayloadSize(&p->sParse, i, &sz); + if( x==JSONB_OBJECT || x==JSONB_ARRAY ){ + JsonParent *pParent; + if( p->nParent>=p->nParentAlloc ){ + JsonParent *pNew; + u64 nNew; + nNew = p->nParentAlloc*2 + 3; + pNew = sqlite3DbRealloc(p->db, p->aParent, sizeof(JsonParent)*nNew); + if( pNew==0 ) return SQLITE_NOMEM; + p->nParentAlloc = (u32)nNew; + p->aParent = pNew; } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; + levelChange = 1; + pParent = &p->aParent[p->nParent]; + pParent->iHead = p->i; + pParent->iValue = i; + pParent->iEnd = i + n + sz; + pParent->iKey = -1; + pParent->nPath = (u32)p->path.nUsed; + if( p->eType && p->nParent ){ + jsonAppendPathName(p); + if( p->path.eErr ) rc = SQLITE_NOMEM; } - default: { - p->i = p->iEnd; - break; + p->nParent++; + p->i = i + n; + }else{ + p->i = i + n + sz; + } + while( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ + p->nParent--; + p->path.nUsed = p->aParent[p->nParent].nPath; + levelChange = 1; + } + if( levelChange ){ + if( p->nParent>0 ){ + JsonParent *pParent = &p->aParent[p->nParent-1]; + u32 iVal = pParent->iValue; + p->eType = p->sParse.aBlob[iVal] & 0x0f; + }else{ + p->eType = 0; } } + }else{ + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->i = i + n + sz; } - return SQLITE_OK; + if( p->eType==JSONB_ARRAY && p->nParent ){ + p->aParent[p->nParent-1].iKey++; + } + p->iRowid++; + return rc; } -/* Append an object label to the JSON Path being constructed -** in pStr. +/* Length of the path for rowid==0 in bRecursive mode. */ -static void jsonAppendObjectPathElement( - JsonString *pStr, - JsonNode *pNode -){ - int jj, nn; - const char *z; - assert( pNode->eType==JSON_STRING ); - assert( pNode->jnFlags & JNODE_LABEL ); - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - nn = pNode->n; - if( (pNode->jnFlags & JNODE_RAW)==0 ){ - assert( nn>=2 ); - assert( z[0]=='"' || z[0]=='\'' ); - assert( z[nn-1]=='"' || z[0]=='\'' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jjpath.nUsed; + char *z = p->path.zBuf; + if( p->iRowid==0 && p->bRecursive && n>=2 ){ + while( n>1 ){ + n--; + if( z[n]=='[' || z[n]=='.' ){ + u32 x, sz = 0; + char cSaved = z[n]; + z[n] = 0; + assert( p->sParse.eEdit==0 ); + x = jsonLookupStep(&p->sParse, 0, z+1, 0); + z[n] = cSaved; + if( JSON_LOOKUP_ISERROR(x) ) continue; + if( x + jsonbPayloadSize(&p->sParse, x, &sz) == p->i ) break; } } } - jsonPrintf(nn+2, pStr, ".%.*s", nn, z); -} - -/* Append the name of the path for element i to pStr -*/ -static void jsonEachComputePath( - JsonEachCursor *p, /* The cursor */ - JsonString *pStr, /* Write the path here */ - u32 i /* Path to this element */ -){ - JsonNode *pNode, *pUp; - u32 iUp; - if( i==0 ){ - jsonAppendChar(pStr, '$'); - return; - } - iUp = p->sParse.aUp[i]; - jsonEachComputePath(p, pStr, iUp); - pNode = &p->sParse.aNode[i]; - pUp = &p->sParse.aNode[iUp]; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==3 || (pUp->eU==0 && pUp->u.iKey==0) ); - testcase( pUp->eU==0 ); - jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); - }else{ - assert( pUp->eType==JSON_OBJECT ); - if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - jsonAppendObjectPathElement(pStr, pNode); - } + return n; } /* Return the value of a column */ static int jsonEachColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ + int iColumn /* Which column to return */ ){ JsonEachCursor *p = (JsonEachCursor*)cur; - JsonNode *pThis = &p->sParse.aNode[p->i]; - switch( i ){ + switch( iColumn ){ case JEACH_KEY: { - if( p->i==0 ) break; - if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); - }else if( p->eType==JSON_ARRAY ){ - u32 iKey; - if( p->bRecursive ){ - if( p->iRowid==0 ) break; - assert( p->sParse.aNode[p->sParse.aUp[p->i]].eU==3 ); - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + if( p->nParent==0 ){ + u32 n, j; + if( p->nRoot==1 ) break; + j = jsonEachPathLength(p); + n = p->nRoot - j; + if( n==0 ){ + break; + }else if( p->path.zBuf[j]=='[' ){ + i64 x; + sqlite3Atoi64(&p->path.zBuf[j+1], &x, n-1, SQLITE_UTF8); + sqlite3_result_int64(ctx, x); + }else if( p->path.zBuf[j+1]=='"' ){ + sqlite3_result_text(ctx, &p->path.zBuf[j+2], n-3, SQLITE_TRANSIENT); }else{ - iKey = p->iRowid; + sqlite3_result_text(ctx, &p->path.zBuf[j+1], n-1, SQLITE_TRANSIENT); } - sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + break; + } + if( p->eType==JSONB_OBJECT ){ + jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); + }else{ + assert( p->eType==JSONB_ARRAY ); + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iKey); } break; } case JEACH_VALUE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); + u32 i = jsonSkipLabel(p); + jsonReturnFromBlob(&p->sParse, i, ctx, p->eMode); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } break; } case JEACH_TYPE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + u32 i = jsonSkipLabel(p); + u8 eType = p->sParse.aBlob[i] & 0x0f; + sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); break; } case JEACH_ATOM: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); + u32 i = jsonSkipLabel(p); + if( (p->sParse.aBlob[i] & 0x0f)sParse, i, ctx, 1); + } break; } case JEACH_ID: { - sqlite3_result_int64(ctx, - (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + sqlite3_result_int64(ctx, (sqlite3_int64)p->i); break; } case JEACH_PARENT: { - if( p->i>p->iBegin && p->bRecursive ){ - sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + if( p->nParent>0 && p->bRecursive ){ + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); } break; } case JEACH_FULLKEY: { - JsonString x; - jsonInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else if( p->eType==JSON_OBJECT ){ - jsonAppendObjectPathElement(&x, pThis); - } - } - jsonResult(&x); + u64 nBase = p->path.nUsed; + if( p->nParent ) jsonAppendPathName(p); + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + p->path.nUsed = nBase; break; } case JEACH_PATH: { - if( p->bRecursive ){ - JsonString x; - jsonInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); - break; - } - /* For json_each() path and root are the same so fall through - ** into the root case */ - /* no break */ deliberate_fall_through + u32 n = jsonEachPathLength(p); + sqlite3_result_text64(ctx, p->path.zBuf, n, + SQLITE_TRANSIENT, SQLITE_UTF8); + break; } default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->path.zBuf, p->nRoot, SQLITE_STATIC); break; } case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + if( p->sParse.zJson==0 ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); + } break; } } @@ -3147,8 +5350,8 @@ static int jsonEachBestIndex( idxMask |= iMask; } } - if( pIdxInfo->nOrderBy>0 - && pIdxInfo->aOrderBy[0].iColumn<0 + if( pIdxInfo->nOrderBy>0 + && pIdxInfo->aOrderBy[0].iColumn<0 && pIdxInfo->aOrderBy[0].desc==0 ){ pIdxInfo->orderByConsumed = 1; @@ -3188,78 +5391,96 @@ static int jsonEachFilter( int argc, sqlite3_value **argv ){ JsonEachCursor *p = (JsonEachCursor*)cur; - const char *z; const char *zRoot = 0; - sqlite3_int64 n; + u32 i, n, sz; UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); - if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ - int rc = SQLITE_NOMEM; - if( p->sParse.oom==0 ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); - if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; - } - jsonEachCursorReset(p); - return rc; - }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ - jsonEachCursorReset(p); - return SQLITE_NOMEM; + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; + p->sParse.db = p->db; + if( jsonArgIsJsonb(argv[0], &p->sParse) ){ + /* We have JSONB */ }else{ - JsonNode *pNode = 0; - if( idxNum==3 ){ - const char *zErr = 0; - zRoot = (const char*)sqlite3_value_text(argv[1]); - if( zRoot==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[1]); - p->zRoot = sqlite3_malloc64( n+1 ); - if( p->zRoot==0 ) return SQLITE_NOMEM; - memcpy(p->zRoot, zRoot, (size_t)n+1); - if( zRoot[0]!='$' ){ - zErr = zRoot; - }else{ - pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); + p->sParse.nJson = sqlite3_value_bytes(argv[0]); + if( p->sParse.zJson==0 ){ + p->i = p->iEnd = 0; + return SQLITE_OK; + } + if( jsonConvertTextToBlob(&p->sParse, 0) ){ + if( p->sParse.oom ){ + return SQLITE_NOMEM; } - if( zErr ){ + goto json_each_malformed_input; + } + } + if( idxNum==3 ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + if( zRoot[0]!='$' ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + p->nRoot = sqlite3Strlen30(zRoot); + if( zRoot[1]==0 ){ + i = p->i = 0; + p->eType = 0; + }else{ + i = jsonLookupStep(&p->sParse, 0, zRoot+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + p->i = 0; + p->eType = 0; + p->iEnd = 0; + return SQLITE_OK; + } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; - }else if( pNode==0 ){ - return SQLITE_OK; } - }else{ - pNode = p->sParse.aNode; - } - p->iBegin = p->i = (int)(pNode - p->sParse.aNode); - p->eType = pNode->eType; - if( p->eType>=JSON_ARRAY ){ - assert( pNode->eU==0 ); - VVA( pNode->eU = 3 ); - pNode->u.iKey = 0; - p->iEnd = p->i + pNode->n + 1; - if( p->bRecursive ){ - p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; - if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ - p->i--; - } + if( p->sParse.iLabel ){ + p->i = p->sParse.iLabel; + p->eType = JSONB_OBJECT; }else{ - p->i++; + p->i = i; + p->eType = JSONB_ARRAY; } - }else{ - p->iEnd = p->i+1; } + jsonAppendRaw(&p->path, zRoot, p->nRoot); + }else{ + i = p->i = 0; + p->eType = 0; + p->nRoot = 1; + jsonAppendRaw(&p->path, "$", 1); + } + p->nParent = 0; + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->iEnd = i+n+sz; + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ + p->i = i + n; + p->eType = p->sParse.aBlob[i] & 0x0f; + p->aParent = sqlite3DbMallocZero(p->db, sizeof(JsonParent)); + if( p->aParent==0 ) return SQLITE_NOMEM; + p->nParent = 1; + p->nParentAlloc = 1; + p->aParent[0].iKey = 0; + p->aParent[0].iEnd = p->iEnd; + p->aParent[0].iHead = p->i; + p->aParent[0].iValue = i; } return SQLITE_OK; + +json_each_malformed_input: + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } /* The methods of the json_each virtual table */ @@ -3270,7 +5491,7 @@ static sqlite3_module jsonEachModule = { jsonEachBestIndex, /* xBestIndex */ jsonEachDisconnect, /* xDisconnect */ 0, /* xDestroy */ - jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachOpen, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ jsonEachNext, /* xNext - advance a cursor */ @@ -3287,35 +5508,8 @@ static sqlite3_module jsonEachModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ -}; - -/* The methods of the json_tree virtual table. */ -static sqlite3_module jsonTreeModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenTree, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ #endif /* !defined(SQLITE_OMIT_JSON) */ @@ -3326,34 +5520,59 @@ static sqlite3_module jsonTreeModule = { void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - JFUNCTION(json, 1, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_error_position,1, 0, jsonErrorFunc), - JFUNCTION(json_extract, -1, 0, jsonExtractFunc), - JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), - JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), - JFUNCTION(json_insert, -1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), - JFUNCTION(json_type, 1, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 0, jsonValidFunc), + /* sqlite3_result_subtype() ----, ,--- sqlite3_value_subtype() */ + /* | | */ + /* Uses cache ------, | | ,---- Returns JSONB */ + /* | | | | */ + /* Number of arguments ---, | | | | ,--- Flags */ + /* | | | | | | */ + JFUNCTION(json, 1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), + JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), + JFUNCTION(json_extract, -1,1,1, 0,0,0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1,1,0, 0,1,0, jsonExtractFunc), + JFUNCTION(->, 2,1,1, 0,0,JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2,1,0, 0,0,JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1,1,1, 1,0,0, jsonSetFunc), + JFUNCTION(jsonb_insert, -1,1,0, 1,1,0, jsonSetFunc), + JFUNCTION(json_object, -1,0,1, 1,0,0, jsonObjectFunc), + JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), + JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), + JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), + JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_replace, -1,1,1, 1,0,0, jsonReplaceFunc), + JFUNCTION(jsonb_replace, -1,1,0, 1,1,0, jsonReplaceFunc), + JFUNCTION(json_set, -1,1,1, 1,0,JSON_ISSET, jsonSetFunc), + JFUNCTION(jsonb_set, -1,1,0, 1,1,JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_type, 2,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_valid, 1,1,0, 0,0,0, jsonValidFunc), + JFUNCTION(json_valid, 2,1,0, 0,0,0, jsonValidFunc), #if SQLITE_DEBUG - JFUNCTION(json_parse, 1, 0, jsonParseFunc), - JFUNCTION(json_test1, 1, 0, jsonTest1Func), + JFUNCTION(json_parse, 1,1,0, 0,0,0, jsonParseFunc), #endif - WAGGREGATE(json_group_array, 1, 0, 0, + WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, - SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), - WAGGREGATE(json_group_object, 2, 0, 0, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_array, 1, JSON_BLOB, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(json_group_object, 2, 0, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_object,2, JSON_BLOB, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, - SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC) }; sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc)); #endif @@ -3361,21 +5580,20 @@ void sqlite3RegisterJsonFunctions(void){ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) /* -** Register the JSON table-valued functions +** Register the JSON table-valued function named zName and return a +** pointer to its Module object. Return NULL if something goes wrong. */ -int sqlite3JsonTableFunctions(sqlite3 *db){ - int rc = SQLITE_OK; - static const struct { - const char *zName; - sqlite3_module *pModule; - } aMod[] = { - { "json_each", &jsonEachModule }, - { "json_tree", &jsonTreeModule }, - }; +Module *sqlite3JsonVtabRegister(sqlite3 *db, const char *zName){ unsigned int i; - for(i=0; iaModule, zName)==0 ); + for(i=0; iSQLITE_MAX_PATHLEN ) goto extension_not_found; + + /* Do not allow sqlite3_load_extension() to link to a copy of the + ** running application, by passing in an empty filename. */ + if( nMsg==0 ) goto extension_not_found; handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN @@ -724,6 +738,9 @@ void sqlite3CloseExtensions(sqlite3 *db){ ** default so as not to open security holes in older applications. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); if( onoff ){ db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; @@ -745,7 +762,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ */ typedef struct sqlite3AutoExtList sqlite3AutoExtList; static SQLITE_WSD struct sqlite3AutoExtList { - u32 nExt; /* Number of entries in aExt[] */ + u32 nExt; /* Number of entries in aExt[] */ void (**aExt)(void); /* Pointers to the extension init functions */ } sqlite3Autoext = { 0, 0 }; @@ -773,6 +790,9 @@ int sqlite3_auto_extension( void (*xInit)(void) ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return SQLITE_MISUSE_BKPT; +#endif #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ){ @@ -825,6 +845,9 @@ int sqlite3_cancel_auto_extension( int i; int n = 0; wsdAutoextInit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return 0; +#endif sqlite3_mutex_enter(mutex); for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ diff --git a/src/main.c b/src/main.c index 0c58988184..371ad1f0bd 100644 --- a/src/main.c +++ b/src/main.c @@ -41,30 +41,20 @@ static int sqlite3TestExtInit(sqlite3 *db){ ** Forward declarations of external module initializer functions ** for modules that need them. */ -#ifdef SQLITE_ENABLE_FTS1 -int sqlite3Fts1Init(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_FTS2 -int sqlite3Fts2Init(sqlite3*); -#endif #ifdef SQLITE_ENABLE_FTS5 int sqlite3Fts5Init(sqlite3*); #endif #ifdef SQLITE_ENABLE_STMTVTAB int sqlite3StmtVtabInit(sqlite3*); #endif - +#ifdef SQLITE_EXTRA_AUTOEXT +int SQLITE_EXTRA_AUTOEXT(sqlite3*); +#endif /* ** An array of pointers to extension initializer functions for ** built-in extensions. */ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { -#ifdef SQLITE_ENABLE_FTS1 - sqlite3Fts1Init, -#endif -#ifdef SQLITE_ENABLE_FTS2 - sqlite3Fts2Init, -#endif #ifdef SQLITE_ENABLE_FTS3 sqlite3Fts3Init, #endif @@ -84,15 +74,15 @@ static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { sqlite3DbstatRegister, #endif sqlite3TestExtInit, -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) - sqlite3JsonTableFunctions, -#endif #ifdef SQLITE_ENABLE_STMTVTAB sqlite3StmtVtabInit, #endif #ifdef SQLITE_ENABLE_BYTECODE_VTAB sqlite3VdbeBytecodeVtabInit, #endif +#ifdef SQLITE_EXTRA_AUTOEXT + SQLITE_EXTRA_AUTOEXT, +#endif }; #ifndef SQLITE_AMALGAMATION @@ -103,7 +93,7 @@ const char sqlite3_version[] = SQLITE_VERSION; #endif /* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. +** a pointer to the to the sqlite3_version[] string constant. */ const char *sqlite3_libversion(void){ return sqlite3_version; } @@ -167,13 +157,13 @@ char *sqlite3_temp_directory = 0; char *sqlite3_data_directory = 0; /* -** Initialize SQLite. +** Initialize SQLite. ** ** This routine must be called to initialize the memory allocation, ** VFS, and mutex subsystems prior to doing any serious work with ** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT ** this routine will be called automatically by key routines such as -** sqlite3_open(). +** sqlite3_open(). ** ** This routine is a no-op except on its very first call for the process, ** or for the first call after a call to sqlite3_shutdown. @@ -226,7 +216,7 @@ int sqlite3_initialize(void){ return SQLITE_OK; } - /* Make sure the mutex subsystem is initialized. If unable to + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, ** there is not much SQLite is going to be able to do. @@ -308,8 +298,16 @@ int sqlite3_initialize(void){ } #endif if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); +#ifdef SQLITE_EXTRA_INIT_MUTEXED + { + int SQLITE_EXTRA_INIT_MUTEXED(const char*); + rc = SQLITE_EXTRA_INIT_MUTEXED(0); + } +#endif + } + if( rc==SQLITE_OK ){ sqlite3MemoryBarrier(); sqlite3GlobalConfig.isInit = 1; #ifdef SQLITE_EXTRA_INIT @@ -360,7 +358,6 @@ int sqlite3_initialize(void){ rc = SQLITE_EXTRA_INIT(0); } #endif - return rc; } @@ -539,7 +536,7 @@ int sqlite3_config(int op, ...){ ** a single parameter which is a pointer to an integer and writes into ** that integer the number of extra bytes per page required for each page ** in SQLITE_CONFIG_PAGECACHE. */ - *va_arg(ap, int*) = + *va_arg(ap, int*) = sqlite3HeaderSizeBtree() + sqlite3HeaderSizePcache() + sqlite3HeaderSizePcache1(); @@ -626,7 +623,7 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.nLookaside = va_arg(ap, int); break; } - + /* Record a pointer to the logger function and its first argument. ** The default is NULL. Logging is disabled if the function pointer is ** NULL. @@ -740,6 +737,18 @@ int sqlite3_config(int op, ...){ } #endif /* SQLITE_OMIT_DESERIALIZE */ + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + default: { rc = SQLITE_ERROR; break; @@ -751,44 +760,54 @@ int sqlite3_config(int op, ...){ /* ** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. +** Return SQLITE_OK on success. ** If lookaside is already active, return SQLITE_BUSY. ** ** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. +** The cnt parameter is the number of slots. If pBuf is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc() +** or similar. If pBuf is not NULL then it is sz*cnt bytes of memory +** to use for the lookaside memory. */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +static int setupLookaside( + sqlite3 *db, /* Database connection being configured */ + void *pBuf, /* Memory to use for lookaside. May be NULL */ + int sz, /* Desired size of each lookaside memory slot */ + int cnt /* Number of slots to allocate */ +){ #ifndef SQLITE_OMIT_LOOKASIDE - void *pStart; - sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; - int nBig; /* Number of full-size slots */ - int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ - + void *pStart; /* Start of the lookaside buffer */ + sqlite3_int64 szAlloc; /* Total space set aside for lookaside memory */ + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + if( sqlite3LookasideUsed(db,0)>0 ){ return SQLITE_BUSY; } /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for + ** allocating a new one so we don't have to have space for ** both at the same time. */ if( db->lookaside.bMalloced ){ sqlite3_free(db->lookaside.pStart); } /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. + ** than a pointer and small enough to fit in a u16. */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + sz = ROUNDDOWN8(sz); if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ + if( sz>65528 ) sz = 65528; + /* Count must be at least 1 to be useful, but not so large as to use + ** more than 0x7fff0000 total bytes for lookaside. */ + if( cnt<1 ) cnt = 0; + if( sz>0 && cnt>(0x7fff0000/sz) ) cnt = 0x7fff0000/sz; + szAlloc = (i64)sz*(i64)cnt; + if( szAlloc==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ + pStart = sqlite3Malloc( szAlloc ); sqlite3EndBenignMalloc(); if( pStart ) szAlloc = sqlite3MallocSize(pStart); }else{ @@ -797,10 +816,10 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE if( sz>=LOOKASIDE_SMALL*3 ){ nBig = szAlloc/(3*LOOKASIDE_SMALL+sz); - nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + nSm = (szAlloc - (i64)sz*(i64)nBig)/LOOKASIDE_SMALL; }else if( sz>=LOOKASIDE_SMALL*2 ){ nBig = szAlloc/(LOOKASIDE_SMALL+sz); - nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + nSm = (szAlloc - (i64)sz*(i64)nBig)/LOOKASIDE_SMALL; }else #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( sz>0 ){ @@ -931,6 +950,10 @@ int sqlite3_db_cacheflush(sqlite3 *db){ int sqlite3_db_config(sqlite3 *db, int op, ...){ va_list ap; int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); va_start(ap, op); switch( op ){ @@ -951,7 +974,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ default: { static const struct { int op; /* The opcode */ - u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + u64 mask; /* Mask of the bit in sqlite3.flags to set/clear */ } aFlagOp[] = { { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, @@ -972,6 +995,9 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, + { SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE, SQLITE_AttachCreate }, + { SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE, SQLITE_AttachWrite }, + { SQLITE_DBCONFIG_ENABLE_COMMENTS, SQLITE_Comments }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -1051,7 +1077,7 @@ int sqlite3IsBinary(const CollSeq *p){ } /* -** Another built-in collating sequence: NOCASE. +** Another built-in collating sequence: NOCASE. ** ** This collating sequence is intended to be used for "case independent ** comparison". SQLite's knowledge of upper and lower case equivalents @@ -1201,7 +1227,7 @@ static void disconnectAllVtab(sqlite3 *db){ /* ** Return TRUE if database connection db has unfinalized prepared -** statements or unfinished sqlite3_backup objects. +** statements or unfinished sqlite3_backup objects. */ static int connectionIsBusy(sqlite3 *db){ int j; @@ -1260,6 +1286,14 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ } #endif + while( db->pDbData ){ + DbClientData *p = db->pDbData; + db->pDbData = p->pNext; + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + sqlite3_free(p); + } + /* Convert the connection into a zombie and then close it. */ db->eOpenState = SQLITE_STATE_ZOMBIE; @@ -1360,6 +1394,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* Clear the TEMP schema separately and last */ if( db->aDb[1].pSchema ){ sqlite3SchemaClear(db->aDb[1].pSchema); + assert( db->aDb[1].pSchema->trigHash.count==0 ); } sqlite3VtabUnlockList(db); @@ -1407,17 +1442,13 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); -#if SQLITE_USER_AUTHENTICATION - sqlite3_free(db->auth.zAuthUser); - sqlite3_free(db->auth.zAuthPW); -#endif db->eOpenState = SQLITE_STATE_ERROR; /* The temp-database schema is allocated differently from the other schema ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database + ** the same sqliteMalloc() as the one that allocates the database ** structure? */ sqlite3DbFree(db, db->aDb[1].pSchema); @@ -1448,7 +1479,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); - /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). ** This is important in case the transaction being rolled back has ** modified the database schema. If the b-tree mutexes are not taken ** here, then another shared-cache connection might sneak in between @@ -1499,6 +1530,9 @@ const char *sqlite3ErrName(int rc){ case SQLITE_OK: zName = "SQLITE_OK"; break; case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_ERROR_RETRY: zName = "SQLITE_ERROR_RETRY"; break; + case SQLITE_ERROR_MISSING_COLLSEQ: + zName = "SQLITE_ERROR_MISSING_COLLSEQ"; break; case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; case SQLITE_PERM: zName = "SQLITE_PERM"; break; case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; @@ -1677,9 +1711,9 @@ static int sqliteDefaultBusyCallback( void *ptr, /* Database connection */ int count /* Number of times table has been busy */ ){ -#if SQLITE_OS_WIN || HAVE_USLEEP +#if SQLITE_OS_WIN || !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP /* This case is for systems that have support for sleeping for fractions of - ** a second. Examples: All windows systems, unix systems with usleep() */ + ** a second. Examples: All windows systems, unix systems with nanosleep() */ static const u8 delays[] = { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; static const u8 totals[] = @@ -1734,7 +1768,7 @@ int sqlite3InvokeBusyHandler(BusyHandler *p){ }else{ p->nBusy++; } - return rc; + return rc; } /* @@ -1754,6 +1788,9 @@ int sqlite3_busy_handler( db->busyHandler.pBusyArg = pArg; db->busyHandler.nBusy = 0; db->busyTimeout = 0; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = 0; +#endif sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -1765,9 +1802,9 @@ int sqlite3_busy_handler( ** be invoked every nOps opcodes. */ void sqlite3_progress_handler( - sqlite3 *db, + sqlite3 *db, int nOps, - int (*xProgress)(void*), + int (*xProgress)(void*), void *pArg ){ #ifdef SQLITE_ENABLE_API_ARMOR @@ -1803,12 +1840,49 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = ms; +#endif }else{ sqlite3_busy_handler(db, 0, 0); } return SQLITE_OK; } +/* +** Set the setlk timeout value. +*/ +int sqlite3_setlk_timeout(sqlite3 *db, int ms, int flags){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int iDb; + int bBOC = ((flags & SQLITE_SETLK_BLOCK_ON_CONNECT) ? 1 : 0); +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms<-1 ) return SQLITE_RANGE; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3_mutex_enter(db->mutex); + db->setlkTimeout = ms; + db->setlkFlags = flags; + sqlite3BtreeEnterAll(db); + for(iDb=0; iDbnDb; iDb++){ + Btree *pBt = db->aDb[iDb].pBt; + if( pBt ){ + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(pBt)); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_BLOCK_ON_CONNECT, (void*)&bBOC); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); +#endif +#if !defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_ENABLE_SETLK_TIMEOUT) + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(flags); +#endif + return SQLITE_OK; +} + /* ** Cause any pending operation to stop at its earliest opportunity. */ @@ -1844,7 +1918,7 @@ int sqlite3_is_interrupted(sqlite3 *db){ ** This function is exactly the same as sqlite3_create_function(), except ** that it is designed to be called by internal code. The difference is ** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. +** is returned and the mallocFailed flag cleared. */ int sqlite3CreateFunc( sqlite3 *db, @@ -1877,7 +1951,8 @@ int sqlite3CreateFunc( assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| - SQLITE_SUBTYPE|SQLITE_INNOCUOUS); + SQLITE_SUBTYPE|SQLITE_INNOCUOUS| + SQLITE_RESULT_SUBTYPE|SQLITE_SELFORDER1); enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But @@ -1885,7 +1960,7 @@ int sqlite3CreateFunc( assert( SQLITE_FUNC_UNSAFE==SQLITE_INNOCUOUS ); extraFlags ^= SQLITE_FUNC_UNSAFE; /* tag-20230109-1 */ - + #ifndef SQLITE_OMIT_UTF16 /* If SQLITE_UTF16 is specified as the encoding type, transform this ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the @@ -1925,7 +2000,7 @@ int sqlite3CreateFunc( #else enc = SQLITE_UTF8; #endif - + /* Check if an existing function is being overridden or deleted. If so, ** and there are active VMs, then return SQLITE_BUSY. If a function ** is being overridden/deleted but there are no active VMs, allow the @@ -1934,7 +2009,7 @@ int sqlite3CreateFunc( p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==(u32)enc && p->nArg==nArg ){ if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; @@ -2013,7 +2088,7 @@ static int createFunctionApi( pArg->xDestroy = xDestroy; pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, xValue, xInverse, pArg ); if( pArg && pArg->nRef==0 ){ @@ -2130,7 +2205,7 @@ static void sqlite3InvalidFunction( ** ** If the function already exists as a regular global function, then ** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. +** a new one that always throws a run-time error. ** ** When virtual tables intend to provide an overloaded function, they ** should call this routine to make sure the global function exists. @@ -2163,7 +2238,7 @@ int sqlite3_overload_function( #ifndef SQLITE_OMIT_TRACE /* ** Register a trace function. The pArg from the previously registered trace -** is returned. +** is returned. ** ** A NULL trace function means that no tracing is executes. A non-NULL ** trace is a pointer to a function that is invoked at the start of each @@ -2214,8 +2289,8 @@ int sqlite3_trace_v2( #ifndef SQLITE_OMIT_DEPRECATED /* -** Register a profile function. The pArg from the previously registered -** profile function is returned. +** Register a profile function. The pArg from the previously registered +** profile function is returned. ** ** A NULL profile function means that no profiling is executes. A non-NULL ** profile is a pointer to a function that is invoked at the conclusion of @@ -2334,6 +2409,12 @@ void *sqlite3_preupdate_hook( void *pArg /* First callback argument */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 ){ + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pPreUpdateArg; db->xPreUpdateCallback = xCallback; @@ -2349,7 +2430,7 @@ void *sqlite3_preupdate_hook( */ int sqlite3_autovacuum_pages( sqlite3 *db, /* Attach the hook to this database */ - unsigned int (*xCallback)(void*,const char*,u32,u32,u32), + unsigned int (*xCallback)(void*,const char*,u32,u32,u32), void *pArg, /* Argument to the function */ void (*xDestructor)(void*) /* Destructor for pArg */ ){ @@ -2377,7 +2458,7 @@ int sqlite3_autovacuum_pages( ** Invoke sqlite3_wal_checkpoint if the number of frames in the log file ** is greater than sqlite3.pWalArg cast to an integer (the value configured by ** wal_autocheckpoint()). -*/ +*/ int sqlite3WalDefaultHook( void *pClientData, /* Argument */ sqlite3 *db, /* Connection */ @@ -2480,7 +2561,7 @@ int sqlite3_wal_checkpoint_v2( if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -2513,7 +2594,7 @@ int sqlite3_wal_checkpoint_v2( /* ** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are +** to contains a zero-length string, all attached databases are ** checkpointed. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ @@ -2527,9 +2608,9 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ ** Run a checkpoint on database iDb. This is a no-op if database iDb is ** not currently open in WAL mode. ** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is ** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. ** ** The mutex on database handle db should be held by the caller. The mutex @@ -2633,6 +2714,29 @@ const char *sqlite3_errmsg(sqlite3 *db){ return z; } +/* +** Set the error code and error message associated with the database handle. +** +** This routine is intended to be called by outside extensions (ex: the +** Session extension). Internal logic should invoke sqlite3Error() or +** sqlite3ErrorWithMsg() directly. +*/ +int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zMsg){ + int rc = SQLITE_OK; + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + if( zMsg ){ + sqlite3ErrorWithMsg(db, errcode, "%s", zMsg); + }else{ + sqlite3Error(db, errcode); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + /* ** Return the byte offset of the most recent error */ @@ -2713,7 +2817,7 @@ int sqlite3_extended_errcode(sqlite3 *db){ } int sqlite3_system_errno(sqlite3 *db){ return db ? db->iSysErrno : 0; -} +} /* ** Return a string that describes the kind of error specified in the @@ -2730,7 +2834,7 @@ const char *sqlite3_errstr(int rc){ */ static int createCollation( sqlite3* db, - const char *zName, + const char *zName, u8 enc, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*), @@ -2738,7 +2842,7 @@ static int createCollation( ){ CollSeq *pColl; int enc2; - + assert( sqlite3_mutex_held(db->mutex) ); /* If SQLITE_UTF16 is specified as the encoding type, transform this @@ -2755,14 +2859,14 @@ static int createCollation( return SQLITE_MISUSE_BKPT; } - /* Check if this call is removing or replacing an existing collation + /* Check if this call is removing or replacing an existing collation ** sequence. If so, and there are active VMs, return busy. If there ** are no active VMs, invalidate any pre-compiled statements. */ pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); if( pColl && pColl->xCmp ){ if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } @@ -2773,7 +2877,7 @@ static int createCollation( ** then any copies made by synthCollSeq() need to be invalidated. ** Also, collation destructor - CollSeq.xDel() - function may need ** to be called. - */ + */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); int j; @@ -2838,8 +2942,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_VDBE_OP<40 # error SQLITE_MAX_VDBE_OP must be at least 40 #endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>32767 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 32767 #endif #if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 # error SQLITE_MAX_ATTACHED must be between 0 and 125 @@ -2906,8 +3010,8 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ if( newLimit>=0 ){ /* IMP: R-52476-28732 */ if( newLimit>aHardLimit[limitId] ){ newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ - }else if( newLimit<1 && limitId==SQLITE_LIMIT_LENGTH ){ - newLimit = 1; + }else if( newLimitaLimit[limitId] = newLimit; } @@ -2924,7 +3028,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ ** query parameter. The second argument contains the URI (or non-URI filename) ** itself. When this function is called the *pFlags variable should contain ** the default flags to open the database handle with. The value stored in -** *pFlags may be updated before returning if the URI filename contains +** *pFlags may be updated before returning if the URI filename contains ** "cache=xxx" or "mode=xxx" query parameters. ** ** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to @@ -2936,7 +3040,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ ** the value returned in *pzFile to avoid a memory leak. ** ** If an error occurs, then an SQLite error code is returned and *pzErrMsg -** may be set to point to a buffer containing an English language error +** may be set to point to a buffer containing an English language error ** message. It is the responsibility of the caller to eventually release ** this buffer by calling sqlite3_free(). */ @@ -2944,7 +3048,7 @@ int sqlite3ParseUri( const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ const char *zUri, /* Nul-terminated URI to parse */ unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ char **pzFile, /* OUT: Filename component of URI */ char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ ){ @@ -2967,7 +3071,7 @@ int sqlite3ParseUri( int iOut = 0; /* Output character index */ u64 nByte = nUri+8; /* Bytes of space to allocate */ - /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen ** method that there may be extra parameters following the file-name. */ flags |= SQLITE_OPEN_URI; @@ -2985,7 +3089,7 @@ int sqlite3ParseUri( /* The following condition causes URIs with five leading / characters ** like file://///host/path to be converted into UNCs like //host/path. ** The correct URI for that UNC has only two or four leading / characters - ** file://host/path or file:////host/path. But 5 leading slashes is a + ** file://host/path or file:////host/path. But 5 leading slashes is a ** common error, we are told, so we handle it as a special case. */ if( strncmp(zUri+7, "///", 3)==0 ){ iIn++; } }else if( strncmp(zUri+5, "//localhost/", 12)==0 ){ @@ -2997,7 +3101,7 @@ int sqlite3ParseUri( iIn = 7; while( zUri[iIn] && zUri[iIn]!='/' ) iIn++; if( iIn!=7 && (iIn!=16 || memcmp("localhost", &zUri[7], 9)) ){ - *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", + *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", iIn-7, &zUri[7]); rc = SQLITE_ERROR; goto parse_uri_out; @@ -3005,8 +3109,8 @@ int sqlite3ParseUri( } #endif - /* Copy the filename and any query parameters into the zFile buffer. - ** Decode %HH escape codes along the way. + /* Copy the filename and any query parameters into the zFile buffer. + ** Decode %HH escape codes along the way. ** ** Within this loop, variable eState may be set to 0, 1 or 2, depending ** on the parsing context. As follows: @@ -3018,9 +3122,9 @@ int sqlite3ParseUri( eState = 0; while( (c = zUri[iIn])!=0 && c!='#' ){ iIn++; - if( c=='%' - && sqlite3Isxdigit(zUri[iIn]) - && sqlite3Isxdigit(zUri[iIn+1]) + if( c=='%' + && sqlite3Isxdigit(zUri[iIn]) + && sqlite3Isxdigit(zUri[iIn+1]) ){ int octet = (sqlite3HexToInt(zUri[iIn++]) << 4); octet += sqlite3HexToInt(zUri[iIn++]); @@ -3032,7 +3136,7 @@ int sqlite3ParseUri( ** case we ignore all text in the remainder of the path, name or ** value currently being parsed. So ignore the current character ** and skip to the next "?", "=" or "&", as appropriate. */ - while( (c = zUri[iIn])!=0 && c!='#' + while( (c = zUri[iIn])!=0 && c!='#' && (eState!=0 || c!='?') && (eState!=1 || (c!='=' && c!='&')) && (eState!=2 || c!='&') @@ -3069,7 +3173,7 @@ int sqlite3ParseUri( if( eState==1 ) zFile[iOut++] = '\0'; memset(zFile+iOut, 0, 4); /* end-of-options + empty journal filenames */ - /* Check if there were any options specified that should be interpreted + /* Check if there were any options specified that should be interpreted ** here. Options that are interpreted here include "vfs" and those that ** correspond to flags that may be passed to the sqlite3_open_v2() ** method. */ @@ -3105,7 +3209,7 @@ int sqlite3ParseUri( if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ static struct OpenMode aOpenMode[] = { { "ro", SQLITE_OPEN_READONLY }, - { "rw", SQLITE_OPEN_READWRITE }, + { "rw", SQLITE_OPEN_READWRITE }, { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, { "memory", SQLITE_OPEN_MEMORY }, { 0, 0 } @@ -3229,7 +3333,7 @@ int sqlite3CodecQueryParameters( /* ** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" ** is UTF-8 encoded. */ static int openDatabase( @@ -3282,11 +3386,11 @@ static int openDatabase( flags &= ~( SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_SUPER_JOURNAL | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_FULLMUTEX | @@ -3296,7 +3400,7 @@ static int openDatabase( /* Allocate the sqlite data structure */ db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; - if( isThreadsafe + if( isThreadsafe #ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS || sqlite3GlobalConfig.bCoreMutex #endif @@ -3328,7 +3432,7 @@ static int openDatabase( db->nextPagesize = 0; db->init.azInit = sqlite3StdType; /* Any array of string ptrs will do */ #ifdef SQLITE_ENABLE_SORTER_MMAP - /* Beginning with version 3.37.0, using the VFS xFetch() API to memory-map + /* Beginning with version 3.37.0, using the VFS xFetch() API to memory-map ** the temporary files used to do external sorts (see code in vdbesort.c) ** is disabled. It can still be used either by defining ** SQLITE_ENABLE_SORTER_MMAP at compile time or by using the @@ -3339,6 +3443,9 @@ static int openDatabase( | SQLITE_EnableTrigger | SQLITE_EnableView | SQLITE_CacheSpill + | SQLITE_AttachCreate + | SQLITE_AttachWrite + | SQLITE_Comments #if !defined(SQLITE_TRUSTED_SCHEMA) || SQLITE_TRUSTED_SCHEMA+0!=0 | SQLITE_TrustedSchema #endif @@ -3347,14 +3454,14 @@ static int openDatabase( ** ** SQLITE_DQS SQLITE_DBCONFIG_DQS_DDL SQLITE_DBCONFIG_DQS_DML ** ---------- ----------------------- ----------------------- -** undefined on on +** undefined on on ** 3 on on ** 2 on off ** 1 off on ** 0 off off ** ** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) -** and so that is the default. But developers are encouranged to use +** and so that is the default. But developers are encouraged to use ** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. */ #if !defined(SQLITE_DQS) @@ -3443,7 +3550,7 @@ static int openDatabase( /* Parse the filename/URI argument ** - ** Only allow sensible combinations of bits in the flags argument. + ** Only allow sensible combinations of bits in the flags argument. ** Throw an error if any non-sense combination is used. If we ** do not block illegal combinations here, it could trigger ** assert() statements in deeper layers. Sensible combinations @@ -3463,6 +3570,7 @@ static int openDatabase( if( ((1<<(flags&7)) & 0x46)==0 ){ rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ }else{ + if( zFilename==0 ) zFilename = ":memory:"; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); } if( rc!=SQLITE_OK ){ @@ -3497,7 +3605,7 @@ static int openDatabase( db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); /* The default safety_level for the main database is FULL; for the temp - ** database it is OFF. This matches the pager layer defaults. + ** database it is OFF. This matches the pager layer defaults. */ db->aDb[0].zDbSName = "main"; db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; @@ -3534,16 +3642,9 @@ static int openDatabase( } } -#ifdef SQLCIPHER_EXT - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlcipherVtabInit(sqlite3 *); - rc = sqlcipherVtabInit(db); - } -#endif - #ifdef SQLITE_ENABLE_INTERNAL_FUNCTIONS /* Testing use only!!! The -DSQLITE_ENABLE_INTERNAL_FUNCTIONS=1 compile-time - ** option gives access to internal functions by default. + ** option gives access to internal functions by default. ** Testing use only!!! */ db->mDbFlags |= DBFLAG_InternalFunc; #endif @@ -3602,8 +3703,8 @@ static int openDatabase( ** Open a new database handle. */ int sqlite3_open( - const char *zFilename, - sqlite3 **ppDb + const char *zFilename, + sqlite3 **ppDb ){ return openDatabase(zFilename, ppDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); @@ -3622,7 +3723,7 @@ int sqlite3_open_v2( ** Open a new database handle. */ int sqlite3_open16( - const void *zFilename, + const void *zFilename, sqlite3 **ppDb ){ char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ @@ -3661,9 +3762,9 @@ int sqlite3_open16( ** Register a new collation sequence with the database handle db. */ int sqlite3_create_collation( - sqlite3* db, - const char *zName, - int enc, + sqlite3* db, + const char *zName, + int enc, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ @@ -3674,9 +3775,9 @@ int sqlite3_create_collation( ** Register a new collation sequence with the database handle db. */ int sqlite3_create_collation_v2( - sqlite3* db, - const char *zName, - int enc, + sqlite3* db, + const char *zName, + int enc, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDel)(void*) @@ -3699,9 +3800,9 @@ int sqlite3_create_collation_v2( ** Register a new collation sequence with the database handle db. */ int sqlite3_create_collation16( - sqlite3* db, + sqlite3* db, const void *zName, - int enc, + int enc, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ @@ -3729,8 +3830,8 @@ int sqlite3_create_collation16( ** db. Replace any previously installed collation sequence factory. */ int sqlite3_collation_needed( - sqlite3 *db, - void *pCollNeededArg, + sqlite3 *db, + void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ #ifdef SQLITE_ENABLE_API_ARMOR @@ -3750,8 +3851,8 @@ int sqlite3_collation_needed( ** db. Replace any previously installed collation sequence factory. */ int sqlite3_collation_needed16( - sqlite3 *db, - void *pCollNeededArg, + sqlite3 *db, + void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ #ifdef SQLITE_ENABLE_API_ARMOR @@ -3766,6 +3867,69 @@ int sqlite3_collation_needed16( } #endif /* SQLITE_OMIT_UTF16 */ +/* +** Find existing client data. +*/ +void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ + DbClientData *p; + sqlite3_mutex_enter(db->mutex); + for(p=db->pDbData; p; p=p->pNext){ + if( strcmp(p->zName, zName)==0 ){ + void *pResult = p->pData; + sqlite3_mutex_leave(db->mutex); + return pResult; + } + } + sqlite3_mutex_leave(db->mutex); + return 0; +} + +/* +** Add new client data to a database connection. +*/ +int sqlite3_set_clientdata( + sqlite3 *db, /* Attach client data to this connection */ + const char *zName, /* Name of the client data */ + void *pData, /* The client data itself */ + void (*xDestructor)(void*) /* Destructor */ +){ + DbClientData *p, **pp; + sqlite3_mutex_enter(db->mutex); + pp = &db->pDbData; + for(p=db->pDbData; p && strcmp(p->zName,zName); p=p->pNext){ + pp = &p->pNext; + } + if( p ){ + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + if( pData==0 ){ + *pp = p->pNext; + sqlite3_free(p); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + } + }else if( pData==0 ){ + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + }else{ + size_t n = strlen(zName); + p = sqlite3_malloc64( SZ_DBCLIENTDATA(n+1) ); + if( p==0 ){ + if( xDestructor ) xDestructor(pData); + sqlite3_mutex_leave(db->mutex); + return SQLITE_NOMEM; + } + memcpy(p->zName, zName, n+1); + p->pNext = db->pDbData; + db->pDbData = p; + } + p->pData = pData; + p->xDestructor = xDestructor; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + + #ifndef SQLITE_OMIT_DEPRECATED /* ** This function is now an anachronism. It used to be used to recover from a @@ -3901,15 +4065,12 @@ int sqlite3_table_column_metadata( /* Find the column for which info is requested */ if( zColumnName==0 ){ - /* Query for existance of table only */ + /* Query for existence of table only */ }else{ - for(iCol=0; iColnCol; iCol++){ + iCol = sqlite3ColumnIndex(pTab, zColumnName); + if( iCol>=0 ){ pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zCnName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ + }else{ if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ iCol = pTab->iPKey; pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; @@ -3923,13 +4084,13 @@ int sqlite3_table_column_metadata( /* The following block stores the meta information that will be returned ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. ** - ** 2. The table is not a view and the column name identified an + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an ** explicitly declared column. Copy meta information from *pCol. - */ + */ if( pCol ){ zDataType = sqlite3ColumnType(pCol,0); zCollSeq = sqlite3ColumnColl(pCol); @@ -3979,7 +4140,7 @@ int sqlite3_sleep(int ms){ pVfs = sqlite3_vfs_find(0); if( pVfs==0 ) return 0; - /* This function works in milliseconds, but the underlying OsSleep() + /* This function works in milliseconds, but the underlying OsSleep() ** API uses microseconds. Hence the 1000's. */ rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000); @@ -4115,6 +4276,28 @@ int sqlite3_test_control(int op, ...){ } #endif + /* sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, sqlite3 *db, int b); + ** + ** If b is true, then activate the SQLITE_FkNoAction setting. If b is + ** false then clear that setting. If the SQLITE_FkNoAction setting is + ** enabled, all foreign key ON DELETE and ON UPDATE actions behave as if + ** they were NO ACTION, regardless of how they are defined. + ** + ** NB: One must usually run "PRAGMA writable_schema=RESET" after + ** using this test-control, before it will take full effect. failing + ** to reset the schema can result in some unexpected behavior. + */ + case SQLITE_TESTCTRL_FK_NO_ACTION: { + sqlite3 *db = va_arg(ap, sqlite3*); + int b = va_arg(ap, int); + if( b ){ + db->flags |= SQLITE_FkNoAction; + }else{ + db->flags &= ~SQLITE_FkNoAction; + } + break; + } + /* ** sqlite3_test_control(BITVEC_TEST, size, program) ** @@ -4159,7 +4342,7 @@ int sqlite3_test_control(int op, ...){ /* ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) ** - ** Register hooks to call to indicate which malloc() failures + ** Register hooks to call to indicate which malloc() failures ** are benign. */ case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { @@ -4214,17 +4397,18 @@ int sqlite3_test_control(int op, ...){ /* Invoke these debugging routines so that the compiler does not ** issue "defined but not used" warnings. */ if( x==9999 ){ - sqlite3ShowExpr(0); sqlite3ShowExpr(0); sqlite3ShowExprList(0); sqlite3ShowIdList(0); sqlite3ShowSrcList(0); sqlite3ShowWith(0); sqlite3ShowUpsert(0); +#ifndef SQLITE_OMIT_TRIGGER sqlite3ShowTriggerStep(0); sqlite3ShowTriggerStepList(0); sqlite3ShowTrigger(0); sqlite3ShowTriggerList(0); +#endif #ifndef SQLITE_OMIT_WINDOWFUNC sqlite3ShowWindow(0); sqlite3ShowWinFunc(0); @@ -4279,7 +4463,7 @@ int sqlite3_test_control(int op, ...){ ** 10 little-endian, determined at run-time ** 432101 big-endian, determined at compile-time ** 123410 little-endian, determined at compile-time - */ + */ case SQLITE_TESTCTRL_BYTEORDER: { rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; break; @@ -4287,7 +4471,7 @@ int sqlite3_test_control(int op, ...){ /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) ** - ** Enable or disable various optimizations for testing purposes. The + ** Enable or disable various optimizations for testing purposes. The ** argument N is a bitmask of optimizations to be disabled. For normal ** operation N should be 0. The idea is that a test program (like the ** SQL Logic Test or SLT test module) can run the same SQL multiple times @@ -4300,6 +4484,18 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, sqlite3 *db, int *N) + ** + ** Write the current optimization settings into *N. A zero bit means that + ** the optimization is on, and a 1 bit means that the optimization is off. + */ + case SQLITE_TESTCTRL_GETOPT: { + sqlite3 *db = va_arg(ap, sqlite3*); + int *pN = va_arg(ap, int*); + *pN = db->dbOptFlags; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt); ** ** If parameter onoff is 1, subsequent calls to localtime() fail. @@ -4341,7 +4537,7 @@ int sqlite3_test_control(int op, ...){ ** formed and never corrupt. This flag is clear by default, indicating that ** database files might have arbitrary corruption. Setting the flag during ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. + ** that demonstrate invariants on well-formed database files. */ case SQLITE_TESTCTRL_NEVER_CORRUPT: { sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); @@ -4377,7 +4573,7 @@ int sqlite3_test_control(int op, ...){ /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); ** - ** Set the VDBE coverage callback function to xCallback with context + ** Set the VDBE coverage callback function to xCallback with context ** pointer ptr. */ case SQLITE_TESTCTRL_VDBE_COVERAGE: { @@ -4407,13 +4603,15 @@ int sqlite3_test_control(int op, ...){ break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, mode, tnum); ** ** This test control is used to create imposter tables. "db" is a pointer ** to the database connection. dbName is the database name (ex: "main" or - ** "temp") which will receive the imposter. "onOff" turns imposter mode on - ** or off. "tnum" is the root page of the b-tree to which the imposter - ** table should connect. + ** "temp") which will receive the imposter. "mode" turns imposter mode on + ** or off. mode==0 means imposter mode is off. mode==1 means imposter mode + ** is on. mode==2 means imposter mode is on but results in an imposter + ** table that is read-only unless writable_schema is on. "tnum" is the + ** root page of the b-tree to which the imposter table should connect. ** ** Enable imposter mode only when the schema has already been parsed. Then ** run a single CREATE TABLE statement to construct the imposter table in @@ -4491,11 +4689,11 @@ int sqlite3_test_control(int op, ...){ /* sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, op, ptr) ** - ** "ptr" is a pointer to a u32. + ** "ptr" is a pointer to a u32. ** ** op==0 Store the current sqlite3TreeTrace in *ptr ** op==1 Set sqlite3TreeTrace to the value *ptr - ** op==3 Store the current sqlite3WhereTrace in *ptr + ** op==2 Store the current sqlite3WhereTrace in *ptr ** op==3 Set sqlite3WhereTrace to the value *ptr */ case SQLITE_TESTCTRL_TRACEFLAGS: { @@ -4530,7 +4728,6 @@ int sqlite3_test_control(int op, ...){ *pI2 = sqlite3LogEst(*pU64); break; } - #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) @@ -4561,6 +4758,28 @@ int sqlite3_test_control(int op, ...){ break; } #endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff); + ** + ** Activate or deactivate validation of JSONB that is generated from + ** text. Off by default, as the validation is slow. Validation is + ** only available if compiled using SQLITE_DEBUG. + ** + ** If onOff is initially 1, then turn it on. If onOff is initially + ** off, turn it off. If onOff is initially -1, then change onOff + ** to be the current setting. + */ + case SQLITE_TESTCTRL_JSON_SELFCHECK: { +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) + int *pOnOff = va_arg(ap, int*); + if( *pOnOff<0 ){ + *pOnOff = sqlite3Config.bJsonSelfcheck; + }else{ + sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff); + } +#endif + break; + } } va_end(ap); #endif /* SQLITE_UNTESTABLE */ @@ -4646,7 +4865,7 @@ void sqlite3_free_filename(const char *p){ /* ** This is a utility routine, useful to VFS implementations, that checks -** to see if a database file was a URI that contained a specific query +** to see if a database file was a URI that contained a specific query ** parameter, and if so obtains the value of the query parameter. ** ** The zFilename argument is the filename pointer passed into the xOpen() @@ -4794,11 +5013,11 @@ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ #ifdef SQLITE_ENABLE_SNAPSHOT /* -** Obtain a snapshot handle for the snapshot of database zDb currently +** Obtain a snapshot handle for the snapshot of database zDb currently ** being read by handle db. */ int sqlite3_snapshot_get( - sqlite3 *db, + sqlite3 *db, const char *zDb, sqlite3_snapshot **ppSnapshot ){ @@ -4817,7 +5036,11 @@ int sqlite3_snapshot_get( if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; if( SQLITE_TXN_WRITE!=sqlite3BtreeTxnState(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + i64 dummy = 0; + sqlite3PagerSnapshotOpen(pPager, (sqlite3_snapshot*)&dummy); rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); } @@ -4831,11 +5054,11 @@ int sqlite3_snapshot_get( } /* -** Open a read-transaction on the snapshot idendified by pSnapshot. +** Open a read-transaction on the snapshot identified by pSnapshot. */ int sqlite3_snapshot_open( - sqlite3 *db, - const char *zDb, + sqlite3 *db, + const char *zDb, sqlite3_snapshot *pSnapshot ){ int rc = SQLITE_ERROR; @@ -4937,8 +5160,8 @@ int sqlite3_compileoption_used(const char *zOptName){ int i, n; int nOpt; const char **azCompileOpt; - -#if SQLITE_ENABLE_API_ARMOR + +#ifdef SQLITE_ENABLE_API_ARMOR if( zOptName==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; @@ -4950,7 +5173,7 @@ int sqlite3_compileoption_used(const char *zOptName){ if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); - /* Since nOpt is normally in single digits, a linear search is + /* Since nOpt is normally in single digits, a linear search is ** adequate. No need for a binary search. */ for(i=0; i>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -258,6 +276,7 @@ static void mallocWithAlarm(int n, void **pp){ if( mem0.hardLimit ){ nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); *pp = 0; return; } @@ -546,6 +565,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); return 0; } } @@ -907,5 +927,5 @@ int sqlite3ApiExit(sqlite3* db, int rc){ if( db->mallocFailed || rc ){ return apiHandleError(db, rc); } - return rc & db->errMask; + return 0; } diff --git a/src/mem1.c b/src/mem1.c index 512ab3747f..12f96beaec 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -156,7 +156,7 @@ static void *sqlite3MemMalloc(int nByte){ ** or sqlite3MemRealloc(). ** ** For this low-level routine, we already know that pPrior!=0 since -** cases where pPrior==0 will have been intecepted and dealt with +** cases where pPrior==0 will have been intercepted and dealt with ** by higher-level routines. */ static void sqlite3MemFree(void *pPrior){ @@ -244,13 +244,13 @@ static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } len = sizeof(cpuCount); - /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + /* One usually wants to use hw.activecpu for MT decisions, but not here */ sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); if( cpuCount>1 ){ /* defer MT decisions to system malloc */ _sqliteZone_ = malloc_default_zone(); }else{ - /* only 1 core, use our own zone to contention over global locks, + /* only 1 core, use our own zone to contention over global locks, ** e.g. we have our own dedicated locks */ _sqliteZone_ = malloc_create_zone(4096, 0); malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); diff --git a/src/memdb.c b/src/memdb.c index 657cb9ca65..61a378bb3d 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -567,13 +567,13 @@ static int memdbOpen( } if( p==0 ){ MemStore **apNew; - p = sqlite3Malloc( sizeof(*p) + szName + 3 ); + p = sqlite3Malloc( sizeof(*p) + (i64)szName + 3 ); if( p==0 ){ sqlite3_mutex_leave(pVfsMutex); return SQLITE_NOMEM; } apNew = sqlite3Realloc(memdb_g.apMemStore, - sizeof(apNew[0])*(memdb_g.nMemStore+1) ); + sizeof(apNew[0])*(1+(i64)memdb_g.nMemStore) ); if( apNew==0 ){ sqlite3_free(p); sqlite3_mutex_leave(pVfsMutex); @@ -799,6 +799,14 @@ unsigned char *sqlite3_serialize( pOut = 0; }else{ sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } if( piSize ) *piSize = sz; if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ pOut = 0; diff --git a/src/mutex.c b/src/mutex.c index 13a9fca15b..62e09cb4fa 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -133,7 +133,7 @@ static void checkMutexFree(sqlite3_mutex *p){ assert( SQLITE_MUTEX_FAST<2 ); assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( ((CheckMutex*)p)->iType<2 ) #endif { @@ -347,15 +347,28 @@ void sqlite3_mutex_leave(sqlite3_mutex *p){ /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. +** +** Because these routines raise false-positive alerts in TSAN, disable +** them (make them always return 1) when compiling with TSAN. */ int sqlite3_mutex_held(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } int sqlite3_mutex_notheld(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } -#endif +#endif /* NDEBUG */ #endif /* !defined(SQLITE_MUTEX_OMIT) */ diff --git a/src/mutex_unix.c b/src/mutex_unix.c index 2afaddec69..beae877f98 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -26,7 +26,7 @@ /* ** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields -** are necessary under two condidtions: (1) Debug builds and (2) using +** are necessary under two conditions: (1) Debug builds and (2) using ** home-grown mutexes. Encapsulate these conditions into a single #define. */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) @@ -64,7 +64,7 @@ struct sqlite3_mutex { ** there might be race conditions that can cause these routines to ** deliver incorrect results. In particular, if pthread_equal() is ** not an atomic operation, then these routines might delivery -** incorrect results. On most platforms, pthread_equal() is a +** incorrect results. On most platforms, pthread_equal() is a ** comparison of two integers and is therefore atomic. But we are ** told that HPUX is not such a platform. If so, then these routines ** will not always work correctly on HPUX. @@ -146,7 +146,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } ** ** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. But for the static +** returns a different mutex on every call. But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. */ @@ -223,7 +223,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ */ static void pthreadMutexFree(sqlite3_mutex *p){ assert( p->nRef==0 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) #endif { @@ -257,7 +257,7 @@ static void pthreadMutexEnter(sqlite3_mutex *p){ ** is atomic - that it cannot be deceived into thinking self ** and p->owner are equal if p->owner changes between two values ** that are not equal to self while the comparison is taking place. - ** This implementation also assumes a coherent cache - that + ** This implementation also assumes a coherent cache - that ** separate processes cannot read different values from the same ** address at the same time. If either of these two conditions ** are not met, then the mutexes will fail and problems will result. @@ -300,7 +300,7 @@ static int pthreadMutexTry(sqlite3_mutex *p){ ** is atomic - that it cannot be deceived into thinking self ** and p->owner are equal if p->owner changes between two values ** that are not equal to self while the comparison is taking place. - ** This implementation also assumes a coherent cache - that + ** This implementation also assumes a coherent cache - that ** separate processes cannot read different values from the same ** address at the same time. If either of these two conditions ** are not met, then the mutexes will fail and problems will result. diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 09deda4091..7eb5b50be1 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -38,7 +38,7 @@ struct sqlite3_mutex { CRITICAL_SECTION mutex; /* Mutex controlling the lock */ int id; /* Mutex type */ #ifdef SQLITE_DEBUG - volatile int nRef; /* Number of enterances */ + volatile int nRef; /* Number of entrances */ volatile DWORD owner; /* Thread holding this mutex */ volatile LONG trace; /* True to trace changes */ #endif @@ -87,7 +87,7 @@ void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif MSVC_VERSION>=1300 +#elif MSVC_VERSION>=1400 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); diff --git a/src/notify.c b/src/notify.c index 4960ab76b1..6a4cab8755 100644 --- a/src/notify.c +++ b/src/notify.c @@ -152,6 +152,9 @@ int sqlite3_unlock_notify( ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); enterMutex(); diff --git a/src/os.c b/src/os.c index e2914e03c0..a9fc732e7e 100644 --- a/src/os.c +++ b/src/os.c @@ -137,7 +137,7 @@ int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite ** is using a regular VFS, it is called after the corresponding ** transaction has been committed. Injecting a fault at this point - ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** confuses the test scripts - the COMMIT command returns SQLITE_NOMEM ** but the transaction is committed anyway. ** ** The core must call OsFileControl() though, not OsFileControlHint(), @@ -279,7 +279,7 @@ int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ }else{ return pVfs->xRandomness(pVfs, nByte, zBufOut); } - + } int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); diff --git a/src/os_kv.c b/src/os_kv.c index 5e0ea49f16..c2d1f9b7ad 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -174,7 +174,7 @@ static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); #define KVSTORAGE_KEY_SZ 32 /* Expand the key name with an appropriate prefix and put the result -** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** in zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least ** KVSTORAGE_KEY_SZ bytes. */ static void kvstorageMakeKey( @@ -233,10 +233,12 @@ static int kvstorageDelete(const char *zClass, const char *zKey){ ** ** Return the total number of bytes in the data, without truncation, and ** not counting the final zero terminator. Return -1 if the key does -** not exist. +** not exist or its key cannot be read. ** -** If nBuf<=0 then this routine simply returns the size of the data without -** actually reading it. +** If nBuf<=0 then this routine simply returns the size of the data +** without actually reading it. Similarly, if nBuf==1 then it +** zero-terminates zBuf at zBuf[0] and returns the size of the data +** without reading it. */ static int kvstorageRead( const char *zClass, @@ -285,11 +287,9 @@ static int kvstorageRead( ** kvvfs i/o methods with JavaScript implementations in WASM builds. ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in -** sqlite3_wasm_enum_json(). There are no binary compatibility -** concerns, so it does not need an iVersion member. This file is -** necessarily always compiled together with sqlite3_wasm_enum_json(), -** and JS code dynamically creates the mapping of members based on -** that JSON description. +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion +** member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { @@ -306,8 +306,8 @@ struct sqlite3_kvvfs_methods { ** the compiler can hopefully optimize this level of indirection out. ** That said, kvvfs is intended primarily for use in WASM builds. ** -** Note that this is not explicitly flagged as static because the -** amalgamation build will tag it with SQLITE_PRIVATE. +** This is not explicitly flagged as static because the amalgamation +** build will tag it with SQLITE_PRIVATE. */ #ifndef SQLITE_WASM const diff --git a/src/os_setup.h b/src/os_setup.h index a82f86fd9f..2aa4b26b33 100644 --- a/src/os_setup.h +++ b/src/os_setup.h @@ -35,8 +35,8 @@ ** must provide its own VFS implementation together with sqlite3_os_init() ** and sqlite3_os_end() routines. */ -#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ - !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +#if SQLITE_OS_KV+1<=1 && SQLITE_OS_OTHER+1<=1 && \ + SQLITE_OS_WIN+1<=1 && SQLITE_OS_UNIX+1<=1 # if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ defined(__MINGW32__) || defined(__BORLANDC__) # define SQLITE_OS_WIN 1 diff --git a/src/os_unix.c b/src/os_unix.c index d0ebb86b95..d73d899241 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -22,7 +22,7 @@ ** This source file is organized into divisions where the logic for various ** subfunctions is contained within the appropriate division. PLEASE ** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed -** in the correct division and should be clearly labeled. +** in the correct division and should be clearly labelled. ** ** The layout of divisions is as follows: ** @@ -61,7 +61,7 @@ ** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE ** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic ** selection of the appropriate locking style based on the filesystem -** where the database is located. +** where the database is located. */ #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -213,7 +213,7 @@ # endif #else /* !SQLITE_WASI */ # ifndef HAVE_FCHMOD -# define HAVE_FCHMOD +# define HAVE_FCHMOD 1 # endif #endif /* SQLITE_WASI */ @@ -226,7 +226,7 @@ #endif /* -** Only set the lastErrno if the error code is a real error and not +** Only set the lastErrno if the error code is a real error and not ** a normal expected return code of SQLITE_BUSY or SQLITE_OK */ #define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) @@ -284,6 +284,7 @@ struct unixFile { #endif #ifdef SQLITE_ENABLE_SETLK_TIMEOUT unsigned iBusyTimeout; /* Wait this many millisec on locks */ + int bBlockOnConnect; /* True to block for SHARED locks */ #endif #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID */ @@ -294,7 +295,7 @@ struct unixFile { ** whenever any part of the database changes. An assertion fault will ** occur if a file is updated without also updating the transaction ** counter. This test is made to avoid new problems similar to the - ** one described by ticket #3584. + ** one described by ticket #3584. */ unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ @@ -303,7 +304,7 @@ struct unixFile { #endif #ifdef SQLITE_TEST - /* In test mode, increase the size of this structure a bit so that + /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. */ char aPadding[32]; @@ -322,7 +323,7 @@ static pid_t randomnessPid = 0; #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ #define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ -#ifndef SQLITE_DISABLE_DIRSYNC +#if !defined(SQLITE_DISABLE_DIRSYNC) && !defined(_AIX) # define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ #else # define UNIXFILE_DIRSYNC 0x00 @@ -448,7 +449,7 @@ static struct unix_syscall { #ifdef __DJGPP__ { "fstat", 0, 0 }, #define osFstat(a,b,c) 0 -#else +#else { "fstat", (sqlite3_syscall_ptr)fstat, 0 }, #define osFstat ((int(*)(int,struct stat*))aSyscall[5].pCurrent) #endif @@ -497,10 +498,11 @@ static struct unix_syscall { #if defined(HAVE_FCHMOD) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #else { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#define osFchmod(FID,MODE) 0 #endif -#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, @@ -594,6 +596,119 @@ static struct unix_syscall { }; /* End of the overrideable system calls */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Extract Posix Advisory Locking information about file description fd +** from the /proc/PID/fdinfo/FD pseudo-file. Fill the string buffer a[16] +** with characters to indicate which SQLite-relevant locks are held. +** a[16] will be a 15-character zero-terminated string with the following +** schema: +** +** AAA/B.DDD.DDDDD +** +** Each of character A-D will be "w" or "r" or "-" to indicate either a +** write-lock, a read-lock, or no-lock, respectively. The "." and "/" +** characters are delimiters intended to make the string more easily +** readable by humans. Here are the meaning of the specific letters: +** +** AAA -> The main database locks. PENDING_BYTE, RESERVED_BYTE, +** and SHARED_FIRST, respectively. +** +** B -> The deadman switch lock. Offset 128 of the -shm file. +** +** CCC -> WAL locks: WRITE, CKPT, RECOVER +** +** DDDDD -> WAL read-locks 0 through 5 +** +** Note that elements before the "/" apply to the main database file and +** elements after the "/" apply to the -shm file in WAL mode. +** +** Here is another way of thinking about the meaning of the result string: +** +** AAA/B.CCC.DDDDD +** ||| | ||| \___/ +** PENDING--'|| | ||| `----- READ 0-5 +** RESERVED--'| | ||`---- RECOVER +** SHARED ----' | |`----- CKPT +** DMS ------' `------ WRITE +** +** Return SQLITE_OK on success and SQLITE_ERROR_UNABLE if the /proc +** pseudo-filesystem is unavailable. +*/ +static int unixPosixAdvisoryLocks( + int fd, /* The file descriptor to analyze */ + char a[16] /* Write a text description of PALs here */ +){ + int in; + ssize_t n; + char *p, *pNext, *x; + char z[2000]; + + /* 1 */ + /* 012 4 678 01234 */ + memcpy(a, "---/-.---.-----", 16); + sqlite3_snprintf(sizeof(z), z, "/proc/%d/fdinfo/%d", getpid(), fd); + in = osOpen(z, O_RDONLY, 0); + if( in<0 ){ + return SQLITE_ERROR_UNABLE; + } + n = osRead(in, z, sizeof(z)-1); + osClose(in); + if( n<=0 ) return SQLITE_ERROR_UNABLE; + z[n] = 0; + + /* We are looking for lines that begin with "lock:\t". Examples: + ** + ** lock: 1: POSIX ADVISORY READ 494716 08:02:5277597 1073741826 1073742335 + ** lock: 1: POSIX ADVISORY WRITE 494716 08:02:5282282 120 120 + ** lock: 2: POSIX ADVISORY READ 494716 08:02:5282282 123 123 + ** lock: 3: POSIX ADVISORY READ 494716 08:02:5282282 128 128 + */ + pNext = strstr(z, "lock:\t"); + while( pNext ){ + char cType = 0; + sqlite3_int64 iFirst, iLast; + p = pNext+6; + pNext = strstr(p, "lock:\t"); + if( pNext ) pNext[-1] = 0; + if( (x = strstr(p, " READ "))!=0 ){ + cType = 'r'; + x += 6; + }else if( (x = strstr(p, " WRITE "))!=0 ){ + cType = 'w'; + x += 7; + }else{ + continue; + } + x = strrchr(x, ' '); + if( x==0 ) continue; + iLast = strtoll(x+1, 0, 10); + *x = 0; + x = strrchr(p, ' '); + if( x==0 ) continue; + iFirst = strtoll(x+1, 0, 10); + if( iLast>=PENDING_BYTE ){ + if( iFirst<=PENDING_BYTE && iLast>=PENDING_BYTE ) a[0] = cType; + if( iFirst<=PENDING_BYTE+1 && iLast>=PENDING_BYTE+1 ) a[1] = cType; + if( iFirst<=PENDING_BYTE+2 && iLast>=PENDING_BYTE+510 ) a[2] = cType; + }else if( iLast<=128 ){ + if( iFirst<=128 && iLast>=128 ) a[4] = cType; + if( iFirst<=120 && iLast>=120 ) a[6] = cType; + if( iFirst<=121 && iLast>=121 ) a[7] = cType; + if( iFirst<=122 && iLast>=122 ) a[8] = cType; + if( iFirst<=123 && iLast>=123 ) a[10] = cType; + if( iFirst<=124 && iLast>=124 ) a[11] = cType; + if( iFirst<=125 && iLast>=125 ) a[12] = cType; + if( iFirst<=126 && iLast>=126 ) a[13] = cType; + if( iFirst<=127 && iLast>=127 ) a[14] = cType; + } + } + return SQLITE_OK; +} +#else +# define unixPosixAdvisoryLocks(A,B) SQLITE_ERROR_UNABLE +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + /* ** On some systems, calls to fchown() will trigger a message in a security ** log if they come from non-root processes. So avoid calling fchown() if @@ -609,7 +724,7 @@ static int robustFchown(int fd, uid_t uid, gid_t gid){ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** "unix" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -692,7 +807,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ /* ** Do not accept any file descriptor less than this value, in order to avoid -** opening database file using file descriptors that are commonly used for +** opening database file using file descriptors that are commonly used for ** standard input, output, and error. */ #ifndef SQLITE_MINIMUM_FILE_DESCRIPTOR @@ -734,7 +849,7 @@ static int robust_open(const char *z, int f, mode_t m){ (void)osUnlink(z); } osClose(fd); - sqlite3_log(SQLITE_WARNING, + sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); fd = -1; if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; @@ -742,9 +857,9 @@ static int robust_open(const char *z, int f, mode_t m){ if( fd>=0 ){ if( m!=0 ){ struct stat statbuf; - if( osFstat(fd, &statbuf)==0 + if( osFstat(fd, &statbuf)==0 && statbuf.st_size==0 - && (statbuf.st_mode&0777)!=m + && (statbuf.st_mode&0777)!=m ){ osFchmod(fd, m); } @@ -758,12 +873,11 @@ static int robust_open(const char *z, int f, mode_t m){ /* ** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the unixInodeInfo and -** vxworksFileId objects used by this file, all of which may be -** shared by multiple threads. +** global mutex is used to protect the unixInodeInfo objects used by +** this file, all of which may be shared by multiple threads. ** -** Function unixMutexHeld() is used to assert() that the global mutex -** is held when required. This function is only used as part of assert() +** Function unixMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() ** statements. e.g. ** ** unixEnterMutex() @@ -885,7 +999,7 @@ static int lockTrace(int fd, int op, struct flock *p){ static int robust_ftruncate(int h, sqlite3_int64 sz){ int rc; #ifdef __ANDROID__ - /* On Android, ftruncate() always uses 32-bit offsets, even if + /* On Android, ftruncate() always uses 32-bit offsets, even if ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to ** truncate a file to any size larger than 2GiB. Silently ignore any ** such attempts. */ @@ -901,32 +1015,32 @@ static int robust_ftruncate(int h, sqlite3_int64 sz){ ** This routine translates a standard POSIX errno code into something ** useful to the clients of the sqlite3 functions. Specifically, it is ** intended to translate a variety of "try again" errors into SQLITE_BUSY -** and a variety of "please close the file descriptor NOW" errors into +** and a variety of "please close the file descriptor NOW" errors into ** SQLITE_IOERR -** +** ** Errors during initialization of locks, or file system support for locks, ** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { - assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || (sqliteIOErr == SQLITE_IOERR_RDLOCK) || (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ); switch (posixError) { - case EACCES: + case EACCES: case EAGAIN: case ETIMEDOUT: case EBUSY: case EINTR: - case ENOLCK: - /* random NFS retry error, unless during file system support + case ENOLCK: + /* random NFS retry error, unless during file system support * introspection, in which it actually means what it says */ return SQLITE_BUSY; - - case EPERM: + + case EPERM: return SQLITE_PERM; - - default: + + default: return sqliteIOErr; } } @@ -941,7 +1055,7 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { ** ** A pointer to an instance of the following structure can be used as a ** unique file ID in VxWorks. Each instance of this structure contains -** a copy of the canonical filename. There is also a reference count. +** a copy of the canonical filename. There is also a reference count. ** The structure is reclaimed when the number of pointers to it drops to ** zero. ** @@ -957,11 +1071,12 @@ struct vxworksFileId { }; #if OS_VXWORKS -/* +/* ** All unique filenames are held on a linked list headed by this ** variable: */ static struct vxworksFileId *vxworksFileList = 0; +static sqlite3_mutex *vxworksMutex = 0; /* ** Simplify a filename into its canonical form @@ -1027,14 +1142,14 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ ** If found, increment the reference count and return a pointer to ** the existing file ID. */ - unixEnterMutex(); + sqlite3_mutex_enter(vxworksMutex); for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ - if( pCandidate->nName==n + if( pCandidate->nName==n && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 ){ sqlite3_free(pNew); pCandidate->nRef++; - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); return pCandidate; } } @@ -1044,7 +1159,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ pNew->nName = n; pNew->pNext = vxworksFileList; vxworksFileList = pNew; - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); return pNew; } @@ -1053,7 +1168,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ ** the object when the reference count reaches zero. */ static void vxworksReleaseFileId(struct vxworksFileId *pId){ - unixEnterMutex(); + sqlite3_mutex_enter(vxworksMutex); assert( pId->nRef>0 ); pId->nRef--; if( pId->nRef==0 ){ @@ -1063,7 +1178,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ *pp = pId->pNext; sqlite3_free(pId); } - unixLeaveMutex(); + sqlite3_mutex_leave(vxworksMutex); } #endif /* OS_VXWORKS */ /*************** End of Unique File ID Utility Used By VxWorks **************** @@ -1122,7 +1237,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** cnt>0 means there are cnt shared locks on the file. ** ** Any attempt to lock or unlock a file first checks the locking -** structure. The fcntl() system call is only invoked to set a +** structure. The fcntl() system call is only invoked to set a ** POSIX lock if the internal lock structure transitions between ** a locked and an unlocked state. ** @@ -1131,7 +1246,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** If you close a file descriptor that points to a file that has locks, ** all locks on that file that are owned by the current process are ** released. To work around this problem, each unixInodeInfo object -** maintains a count of the number of pending locks on tha inode. +** maintains a count of the number of pending locks on the inode. ** When an attempt is made to close an unixFile, if there are ** other unixFile open on the same inode that are holding locks, the call ** to close() the file descriptor is deferred until all of the locks clear. @@ -1145,7 +1260,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** not posix compliant. Under LinuxThreads, a lock created by thread ** A cannot be modified or overridden by a different thread B. ** Only thread A can modify the lock. Locking behavior is correct -** if the appliation uses the newer Native Posix Thread Library (NPTL) +** if the application uses the newer Native Posix Thread Library (NPTL) ** on linux - with NPTL a lock created by thread A can override locks ** in thread B. But there is no way to know at compile-time which ** threading library is being used. So there is no way to know at @@ -1155,7 +1270,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** ** SQLite used to support LinuxThreads. But support for LinuxThreads ** was dropped beginning with version 3.7.0. SQLite will still work with -** LinuxThreads provided that (1) there is no more than one connection +** LinuxThreads provided that (1) there is no more than one connection ** per database file in the same process and (2) database connections ** do not move across threads. */ @@ -1172,7 +1287,7 @@ struct unixFileId { /* We are told that some versions of Android contain a bug that ** sizes ino_t at only 32-bits instead of 64-bits. (See ** https://android-review.googlesource.com/#/c/115351/3/dist/sqlite3.c) - ** To work around this, always allocate 64-bits for the inode number. + ** To work around this, always allocate 64-bits for the inode number. ** On small machines that only have 32-bit inodes, this wastes 4 bytes, ** but that should not be a big deal. */ /* WAS: ino_t ino; */ @@ -1260,7 +1375,7 @@ int unixFileMutexNotheld(unixFile *pFile){ ** strerror_r(). ** ** The first argument passed to the macro should be the error code that -** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). ** The two subsequent arguments should be the name of the OS function that ** failed (e.g. "unlink", "open") and the associated file-system path, ** if any. @@ -1278,7 +1393,7 @@ static int unixLogErrorAtLine( /* If this is not a threadsafe build (SQLITE_THREADSAFE==0), then use ** the strerror() function to obtain the human-readable error message ** equivalent to errno. Otherwise, use strerror_r(). - */ + */ #if SQLITE_THREADSAFE && defined(HAVE_STRERROR_R) char aErr[80]; memset(aErr, 0, sizeof(aErr)); @@ -1286,18 +1401,22 @@ static int unixLogErrorAtLine( /* If STRERROR_R_CHAR_P (set by autoconf scripts) or __USE_GNU is defined, ** assume that the system provides the GNU version of strerror_r() that - ** returns a pointer to a buffer containing the error message. That pointer - ** may point to aErr[], or it may point to some static storage somewhere. - ** Otherwise, assume that the system provides the POSIX version of + ** returns a pointer to a buffer containing the error message. That pointer + ** may point to aErr[], or it may point to some static storage somewhere. + ** Otherwise, assume that the system provides the POSIX version of ** strerror_r(), which always writes an error message into aErr[]. ** ** If the code incorrectly assumes that it is the POSIX version that is ** available, the error message will often be an empty string. Not a - ** huge problem. Incorrectly concluding that the GNU version is available + ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) - zErr = +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) + zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -1347,8 +1466,8 @@ static void storeLastErrno(unixFile *pFile, int error){ } /* -** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. -*/ +** Close all file descriptors accumulated in the unixInodeInfo->pUnused list. +*/ static void closePendingFds(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; UnixUnusedFd *p; @@ -1447,6 +1566,10 @@ static int findInodeInfo( storeLastErrno(pFile, errno); return SQLITE_IOERR; } + if( fsync(fd) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_FSYNC; + } rc = osFstat(fd, &statbuf); if( rc!=0 ){ storeLastErrno(pFile, errno); @@ -1503,7 +1626,7 @@ static int fileHasMoved(unixFile *pFile){ #else struct stat buf; return pFile->pInode!=0 && - (osStat(pFile->zPath, &buf)!=0 + (osStat(pFile->zPath, &buf)!=0 || (u64)buf.st_ino!=pFile->pInode->fileId.ino); #endif } @@ -1584,7 +1707,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ } } #endif - + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); @@ -1616,25 +1739,49 @@ static int osSetPosixAdvisoryLock( struct flock *pLock, /* The description of the lock */ unixFile *pFile /* Structure holding timeout value */ ){ - int tm = pFile->iBusyTimeout; - int rc = osFcntl(h,F_SETLK,pLock); - while( rc<0 && tm>0 ){ - /* On systems that support some kind of blocking file lock with a timeout, - ** make appropriate changes here to invoke that blocking file lock. On - ** generic posix, however, there is no such API. So we simply try the - ** lock once every millisecond until either the timeout expires, or until - ** the lock is obtained. */ - unixSleep(0,1000); + int rc = 0; + + if( pFile->iBusyTimeout==0 ){ + /* unixFile->iBusyTimeout is set to 0. In this case, attempt a + ** non-blocking lock. */ + rc = osFcntl(h,F_SETLK,pLock); + }else{ + /* unixFile->iBusyTimeout is set to greater than zero. In this case, + ** attempt a blocking-lock with a unixFile->iBusyTimeout ms timeout. + ** + ** On systems that support some kind of blocking file lock operation, + ** this block should be replaced by code to attempt a blocking lock + ** with a timeout of unixFile->iBusyTimeout ms. The code below is + ** placeholder code. If SQLITE_TEST is defined, the placeholder code + ** retries the lock once every 1ms until it succeeds or the timeout + ** is reached. Or, if SQLITE_TEST is not defined, the placeholder + ** code attempts a non-blocking lock and sets unixFile->iBusyTimeout + ** to 0. This causes the caller to return SQLITE_BUSY, instead of + ** SQLITE_BUSY_TIMEOUT to SQLite - as required by a VFS that does not + ** support blocking locks. + */ +#ifdef SQLITE_TEST + int tm = pFile->iBusyTimeout; + while( tm>0 ){ + rc = osFcntl(h,F_SETLK,pLock); + if( rc==0 ) break; + unixSleep(0,1000); + tm--; + } +#else rc = osFcntl(h,F_SETLK,pLock); - tm--; + pFile->iBusyTimeout = 0; +#endif + /* End of code to replace with real blocking-locks code. */ } + return rc; } #endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ /* -** Attempt to set a system-lock on the file pFile. The lock is +** Attempt to set a system-lock on the file pFile. The lock is ** described by pLock. ** ** If the pFile was opened read/write from unix-excl, then the only lock @@ -1660,7 +1807,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; - assert( pInode->nLock==0 ); + /* assert( pInode->nLock==0 ); <-- Not true if unix-excl READONLY used */ lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; @@ -1673,11 +1820,25 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ rc = 0; } }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( pFile->bBlockOnConnect && pLock->l_type==F_RDLCK + && pLock->l_start==SHARED_FIRST && pLock->l_len==SHARED_SIZE + ){ + rc = osFcntl(pFile->h, F_SETLKW, pLock); + }else +#endif rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile); } return rc; } +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) +/* Forward reference */ +static int unixIsSharingShmNode(unixFile*); +#else +#define unixIsSharingShmNode(pFile) (0) +#endif + /* ** Lock the file with the lock specified by parameter eFileLock - one ** of the following: @@ -1710,7 +1871,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** - ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** Symbols defined in os.h identify the 'pending byte' and the 'reserved ** byte', each single bytes at well known offsets, and the 'shared byte ** range', a range of 510 bytes at a well known offset. ** @@ -1718,7 +1879,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** byte'. If this is successful, 'shared byte range' is read-locked ** and the lock on the 'pending byte' released. (Legacy note: When ** SQLite was first developed, Windows95 systems were still very common, - ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** and Windows95 lacks a shared-lock capability. So on Windows95, a ** single randomly selected by from the 'shared byte range' is locked. ** Windows95 is now pretty much extinct, but this work-around for the ** lack of shared-locks on Windows95 lives on, for backwards @@ -1726,20 +1887,20 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** ** A process may only obtain a RESERVED lock after it has a SHARED lock. ** A RESERVED lock is implemented by grabbing a write-lock on the - ** 'reserved byte'. + ** 'reserved byte'. ** ** An EXCLUSIVE lock may only be requested after either a SHARED or - ** RESERVED lock is held. An EXCLUSIVE lock is implemented by obtaining - ** a write-lock on the entire 'shared byte range'. Since all other locks - ** require a read-lock on one of the bytes within this range, this ensures - ** that no other locks are held on the database. + ** RESERVED lock is held. An EXCLUSIVE lock is implemented by obtaining + ** a write-lock on the entire 'shared byte range'. Since all other locks + ** require a read-lock on one of the bytes within this range, this ensures + ** that no other locks are held on the database. ** ** If a process that holds a RESERVED lock requests an EXCLUSIVE, then - ** a PENDING lock is obtained first. A PENDING lock is implemented by - ** obtaining a write-lock on the 'pending byte'. This ensures that no new - ** SHARED locks can be obtained, but existing SHARED locks are allowed to + ** a PENDING lock is obtained first. A PENDING lock is implemented by + ** obtaining a write-lock on the 'pending byte'. This ensures that no new + ** SHARED locks can be obtained, but existing SHARED locks are allowed to ** persist. If the call to this function fails to obtain the EXCLUSIVE - ** lock in this case, it holds the PENDING lock intead. The client may + ** lock in this case, it holds the PENDING lock instead. The client may ** then re-attempt the EXCLUSIVE lock later on, after existing SHARED ** locks have cleared. */ @@ -1767,7 +1928,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct. ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); @@ -1782,7 +1943,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. */ - if( (pFile->eFileLock!=pInode->eFileLock && + if( (pFile->eFileLock!=pInode->eFileLock && (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; @@ -1793,7 +1954,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** has a SHARED or RESERVED lock, then increment reference counts and ** return SQLITE_OK. */ - if( eFileLock==SHARED_LOCK && + if( eFileLock==SHARED_LOCK && (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ assert( eFileLock==SHARED_LOCK ); assert( pFile->eFileLock==0 ); @@ -1811,7 +1972,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ */ lock.l_len = 1L; lock.l_whence = SEEK_SET; - if( eFileLock==SHARED_LOCK + if( eFileLock==SHARED_LOCK || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLock==RESERVED_LOCK) ){ lock.l_type = (eFileLock==SHARED_LOCK?F_RDLCK:F_WRLCK); @@ -1853,7 +2014,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ /* This could happen with a network mount */ tErrno = errno; - rc = SQLITE_IOERR_UNLOCK; + rc = SQLITE_IOERR_UNLOCK; } if( rc ){ @@ -1870,6 +2031,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; + }else if( unixIsSharingShmNode(pFile) ){ + /* We are in WAL mode and attempting to delete the SHM and WAL + ** files due to closing the connection or changing out of WAL mode, + ** but another process still holds locks on the SHM file, thus + ** indicating that database locks have been broken, perhaps due + ** to a rogue close(open(dbFile)) or similar. + */ + rc = SQLITE_BUSY; }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file @@ -1895,7 +2064,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } } } - + #ifdef SQLITE_DEBUG /* Set up the transaction-counter change checking flags when @@ -1920,7 +2089,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ end_lock: sqlite3_mutex_leave(pInode->pLockMutex); - OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), + OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; } @@ -1945,11 +2114,11 @@ static void setPendingFd(unixFile *pFile){ ** ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. -** +** ** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED ** the byte range is divided into 2 parts and the first part is unlocked then -** set to a read lock, then the other part is simply unlocked. This works -** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to ** remove the write lock on a region when a read lock is set. */ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ @@ -1987,7 +2156,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ /* downgrading to a shared lock on NFS involves clearing the write lock ** before establishing the readlock - to avoid a race condition we downgrade - ** the lock in 2 blocks, so that part of the range will be covered by a + ** the lock in 2 blocks, so that part of the range will be covered by a ** write lock until the rest is covered by a read lock: ** 1: [WWWWW] ** 2: [....W] @@ -2003,7 +2172,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( handleNFSUnlock ){ int tErrno; /* Error code from system call errors */ off_t divSize = SHARED_SIZE - 1; - + lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; @@ -2045,11 +2214,11 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_len = SHARED_SIZE; if( unixFileLock(pFile, &lock) ){ /* In theory, the call to unixFileLock() cannot fail because another - ** process is holding an incompatible lock. If it does, this + ** process is holding an incompatible lock. If it does, this ** indicates that the other process is not following the locking ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning - ** SQLITE_BUSY would confuse the upper layer (in practice it causes - ** an assert to fail). */ + ** SQLITE_BUSY would confuse the upper layer (in practice it causes + ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; storeLastErrno(pFile, errno); goto end_unlock; @@ -2125,7 +2294,7 @@ static void unixUnmapfile(unixFile *pFd); #endif /* -** This function performs the parts of the "close file" operation +** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file ** handles, if they are valid, and sets all fields of the unixFile ** structure to 0. @@ -2188,7 +2357,7 @@ static int unixClose(sqlite3_file *id){ if( pInode->nLock ){ /* If there are outstanding locks, do not actually close the file just ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->pUnused list. It will be automatically closed + ** descriptor to pInode->pUnused list. It will be automatically closed ** when the last lock is cleared. */ setPendingFd(pFile); @@ -2275,26 +2444,22 @@ static int nolockClose(sqlite3_file *id) { /* ** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. -** -** In dotfile locking, either a lock exists or it does not. So in this -** variation of CheckReservedLock(), *pResOut is set to true if any lock -** is held on the file and false if the file is unlocked. +** file by this or any other process. If the caller holds a SHARED +** or greater lock when it is called, then it is assumed that no other +** client may hold RESERVED. Or, if the caller holds no lock, then it +** is assumed another client holds RESERVED if the lock-file exists. */ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { - int rc = SQLITE_OK; - int reserved = 0; unixFile *pFile = (unixFile*)id; - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - - assert( pFile ); - reserved = osAccess((const char*)pFile->lockingContext, 0)==0; - OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); - *pResOut = reserved; - return rc; + + if( pFile->eFileLock>=SHARED_LOCK ){ + *pResOut = 0; + }else{ + *pResOut = osAccess((const char*)pFile->lockingContext, 0)==0; + } + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, 0, *pResOut)); + return SQLITE_OK; } /* @@ -2343,7 +2508,7 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { #endif return SQLITE_OK; } - + /* grab an exclusive lock */ rc = osMkdir(zLockFile, 0777); if( rc<0 ){ @@ -2358,8 +2523,8 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { } } return rc; - } - + } + /* got it, set the type and return ok */ pFile->eFileLock = eFileLock; return rc; @@ -2383,7 +2548,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); - + /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; @@ -2396,7 +2561,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { pFile->eFileLock = SHARED_LOCK; return SQLITE_OK; } - + /* To fully unlock the database, delete the lock file */ assert( eFileLock==NO_LOCK ); rc = osRmdir(zLockFile); @@ -2408,7 +2573,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { rc = SQLITE_IOERR_UNLOCK; storeLastErrno(pFile, tErrno); } - return rc; + return rc; } pFile->eFileLock = NO_LOCK; return SQLITE_OK; @@ -2455,7 +2620,7 @@ static int robust_flock(int fd, int op){ #else # define robust_flock(a,b) flock(a,b) #endif - + /* ** This routine checks if there is a RESERVED lock held on the specified @@ -2464,54 +2629,33 @@ static int robust_flock(int fd, int op){ ** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc = SQLITE_OK; - int reserved = 0; +#ifdef SQLITE_DEBUG unixFile *pFile = (unixFile*)id; - +#else + UNUSED_PARAMETER(id); +#endif + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - + assert( pFile ); - - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - reserved = 1; - } - - /* Otherwise see if some other process holds it. */ - if( !reserved ){ - /* attempt to get the lock */ - int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); - if( !lrc ){ - /* got the lock, unlock it */ - lrc = robust_flock(pFile->h, LOCK_UN); - if ( lrc ) { - int tErrno = errno; - /* unlock failed with an error */ - lrc = SQLITE_IOERR_UNLOCK; - storeLastErrno(pFile, tErrno); - rc = lrc; - } - } else { - int tErrno = errno; - reserved = 1; - /* someone else might have it reserved */ - lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(lrc) ){ - storeLastErrno(pFile, tErrno); - rc = lrc; - } - } - } - OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); + assert( pFile->eFileLock<=SHARED_LOCK ); -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & 0xff) == SQLITE_IOERR ){ - rc = SQLITE_OK; - reserved=1; - } -#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ - *pResOut = reserved; - return rc; + /* The flock VFS only ever takes exclusive locks (see function flockLock). + ** Therefore, if this connection is holding any lock at all, no other + ** connection may be holding a RESERVED lock. So set *pResOut to 0 + ** in this case. + ** + ** Or, this connection may be holding no lock. In that case, set *pResOut to + ** 0 as well. The caller will then attempt to take an EXCLUSIVE lock on the + ** db in order to roll the hot journal back. If there is another connection + ** holding a lock, that attempt will fail and an SQLITE_BUSY returned to + ** the user. With other VFS, we try to avoid this, in order to allow a reader + ** to proceed while a writer is preparing its transaction. But that won't + ** work with the flock VFS - as it always takes EXCLUSIVE locks - so it is + ** not a problem in this case. */ + *pResOut = 0; + + return SQLITE_OK; } /* @@ -2549,15 +2693,15 @@ static int flockLock(sqlite3_file *id, int eFileLock) { assert( pFile ); - /* if we already have a lock, it is exclusive. + /* if we already have a lock, it is exclusive. ** Just adjust level and punt on outta here. */ if (pFile->eFileLock > NO_LOCK) { pFile->eFileLock = eFileLock; return SQLITE_OK; } - + /* grab an exclusive lock */ - + if (robust_flock(pFile->h, LOCK_EX | LOCK_NB)) { int tErrno = errno; /* didn't get, must be busy */ @@ -2569,7 +2713,7 @@ static int flockLock(sqlite3_file *id, int eFileLock) { /* got it, set the type and return ok */ pFile->eFileLock = eFileLock; } - OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), + OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS if( (rc & 0xff) == SQLITE_IOERR ){ @@ -2589,23 +2733,23 @@ static int flockLock(sqlite3_file *id, int eFileLock) { */ static int flockUnlock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - + assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); - + /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; } - + /* shared can just be set because we always have an exclusive */ if (eFileLock==SHARED_LOCK) { pFile->eFileLock = eFileLock; return SQLITE_OK; } - + /* no, really, unlock. */ if( robust_flock(pFile->h, LOCK_UN) ){ #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS @@ -2656,14 +2800,14 @@ static int semXCheckReservedLock(sqlite3_file *id, int *pResOut) { unixFile *pFile = (unixFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - + assert( pFile ); /* Check if a thread in this process holds such a lock */ if( pFile->eFileLock>SHARED_LOCK ){ reserved = 1; } - + /* Otherwise see if some other process holds it. */ if( !reserved ){ sem_t *pSem = pFile->pInode->pSem; @@ -2722,14 +2866,14 @@ static int semXLock(sqlite3_file *id, int eFileLock) { sem_t *pSem = pFile->pInode->pSem; int rc = SQLITE_OK; - /* if we already have a lock, it is exclusive. + /* if we already have a lock, it is exclusive. ** Just adjust level and punt on outta here. */ if (pFile->eFileLock > NO_LOCK) { pFile->eFileLock = eFileLock; rc = SQLITE_OK; goto sem_end_lock; } - + /* lock semaphore now but bail out when already locked. */ if( sem_trywait(pSem)==-1 ){ rc = SQLITE_BUSY; @@ -2759,18 +2903,18 @@ static int semXUnlock(sqlite3_file *id, int eFileLock) { OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); - + /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; } - + /* shared can just be set because we always have an exclusive */ if (eFileLock==SHARED_LOCK) { pFile->eFileLock = eFileLock; return SQLITE_OK; } - + /* no, really unlock. */ if ( sem_post(pSem)==-1 ) { int rc, tErrno = errno; @@ -2778,7 +2922,7 @@ static int semXUnlock(sqlite3_file *id, int eFileLock) { if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } - return rc; + return rc; } pFile->eFileLock = NO_LOCK; return SQLITE_OK; @@ -2844,7 +2988,7 @@ struct ByteRangeLockPB2 /* ** This is a utility for setting or clearing a bit-range lock on an ** AFP filesystem. -** +** ** Return SQLITE_OK on success, SQLITE_BUSY on failure. */ static int afpSetLock( @@ -2856,14 +3000,14 @@ static int afpSetLock( ){ struct ByteRangeLockPB2 pb; int err; - + pb.unLockFlag = setLockFlag ? 0 : 1; pb.startEndFlag = 0; pb.offset = offset; - pb.length = length; + pb.length = length; pb.fd = pFile->h; - - OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + + OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), offset, length)); err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); @@ -2898,9 +3042,9 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ int reserved = 0; unixFile *pFile = (unixFile*)id; afpLockingContext *context; - + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - + assert( pFile ); context = (afpLockingContext *) pFile->lockingContext; if( context->reserved ){ @@ -2912,12 +3056,12 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ if( pFile->pInode->eFileLock>SHARED_LOCK ){ reserved = 1; } - + /* Otherwise see if some other process holds it. */ if( !reserved ){ /* lock the RESERVED byte */ - int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); if( SQLITE_OK==lrc ){ /* if we succeeded in taking the reserved lock, unlock it to restore ** the original state */ @@ -2930,10 +3074,10 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ rc=lrc; } } - + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); - + *pResOut = reserved; return rc; } @@ -2967,7 +3111,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ unixFile *pFile = (unixFile*)id; unixInodeInfo *pInode = pFile->pInode; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - + assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), @@ -2985,13 +3129,13 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); assert( eFileLock!=PENDING_LOCK ); assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); - + /* This mutex is needed because pFile->pInode is shared across threads */ pInode = pFile->pInode; @@ -3000,18 +3144,18 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. */ - if( (pFile->eFileLock!=pInode->eFileLock && + if( (pFile->eFileLock!=pInode->eFileLock && (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; goto afp_end_lock; } - + /* If a SHARED lock is requested, and some thread using this PID already ** has a SHARED or RESERVED lock, then increment reference counts and ** return SQLITE_OK. */ - if( eFileLock==SHARED_LOCK && + if( eFileLock==SHARED_LOCK && (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ assert( eFileLock==SHARED_LOCK ); assert( pFile->eFileLock==0 ); @@ -3021,12 +3165,12 @@ static int afpLock(sqlite3_file *id, int eFileLock){ pInode->nLock++; goto afp_end_lock; } - + /* A PENDING lock is needed before acquiring a SHARED lock and before ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will ** be released. */ - if( eFileLock==SHARED_LOCK + if( eFileLock==SHARED_LOCK || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLocknShared==0 ); assert( pInode->eFileLock==0 ); - + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; /* Now get the read-lock SHARED_LOCK */ /* note that the quality of the randomness doesn't matter that much */ - lk = random(); + lk = random(); pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); - lrc1 = afpSetLock(context->dbPath, pFile, + lrc1 = afpSetLock(context->dbPath, pFile, SHARED_FIRST+pInode->sharedByte, 1, 1); if( IS_LOCK_ERROR(lrc1) ){ lrc1Errno = pFile->lastErrno; } /* Drop the temporary PENDING lock */ lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); - + if( IS_LOCK_ERROR(lrc1) ) { storeLastErrno(pFile, lrc1Errno); rc = lrc1; @@ -3094,34 +3238,34 @@ static int afpLock(sqlite3_file *id, int eFileLock){ } if (!failed && eFileLock == EXCLUSIVE_LOCK) { /* Acquire an EXCLUSIVE lock */ - - /* Remove the shared lock before trying the range. we'll need to + + /* Remove the shared lock before trying the range. we'll need to ** reestablish the shared lock if we can't get the afpUnlock */ if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 0)) ){ int failed2 = SQLITE_OK; - /* now attemmpt to get the exclusive lock range */ - failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + /* now attempt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 1); - if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ /* Can't reestablish the shared lock. Sqlite can't deal, this is ** a critical I/O error */ - rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : + rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : SQLITE_IOERR_LOCK; goto afp_end_lock; - } + } }else{ - rc = failed; + rc = failed; } } if( failed ){ rc = failed; } } - + if( rc==SQLITE_OK ){ pFile->eFileLock = eFileLock; pInode->eFileLock = eFileLock; @@ -3129,10 +3273,10 @@ static int afpLock(sqlite3_file *id, int eFileLock){ pFile->eFileLock = PENDING_LOCK; pInode->eFileLock = PENDING_LOCK; } - + afp_end_lock: sqlite3_mutex_leave(pInode->pLockMutex); - OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), + OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; } @@ -3150,9 +3294,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unixInodeInfo *pInode; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; int skipShared = 0; -#ifdef SQLITE_TEST - int h = pFile->h; -#endif assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, @@ -3168,10 +3309,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); - + #ifdef SQLITE_DEBUG /* When reducing a lock such that other processes can start ** reading the database file again, make sure that the @@ -3186,7 +3324,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { || pFile->transCntrChng==1 ); pFile->inNormalWrite = 0; #endif - + if( pFile->eFileLock==EXCLUSIVE_LOCK ){ rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ @@ -3199,11 +3337,11 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { } if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); - } + } if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); - if( !rc ){ - context->reserved = 0; + if( !rc ){ + context->reserved = 0; } } if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1)){ @@ -3219,9 +3357,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; pInode->nShared--; if( pInode->nShared==0 ){ - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); if( !skipShared ){ rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); } @@ -3236,7 +3371,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( pInode->nLock==0 ) closePendingFds(pFile); } } - + sqlite3_mutex_leave(pInode->pLockMutex); if( rc==SQLITE_OK ){ pFile->eFileLock = eFileLock; @@ -3245,7 +3380,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { } /* -** Close a file & cleanup AFP specific locking context +** Close a file & cleanup AFP specific locking context */ static int afpClose(sqlite3_file *id) { int rc = SQLITE_OK; @@ -3303,7 +3438,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ /* ** The code above is the NFS lock implementation. The code is specific ** to MacOSX and does not work on other unix platforms. No alternative -** is available. +** is available. ** ********************* End of the NFS lock implementation ********************** ******************************************************************************/ @@ -3311,7 +3446,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ /****************************************************************************** **************** Non-locking sqlite3_file methods ***************************** ** -** The next division contains implementations for all methods of the +** The next division contains implementations for all methods of the ** sqlite3_file object other than the locking methods. The locking ** methods were defined in divisions above (one locking method per ** division). Those methods that are common to all locking modes @@ -3319,7 +3454,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ */ /* -** Seek to the offset passed as the second argument, then read cnt +** Seek to the offset passed as the second argument, then read cnt ** bytes into pBuf. Return the number of bytes actually read. ** ** To avoid stomping the errno value on a failed read the lastErrno value @@ -3375,8 +3510,8 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ ** wrong. */ static int unixRead( - sqlite3_file *id, - void *pBuf, + sqlite3_file *id, + void *pBuf, int amt, sqlite3_int64 offset ){ @@ -3391,12 +3526,12 @@ static int unixRead( #if 0 assert( pFile->pPreallocatedUnused==0 || offset>=PENDING_BYTE+512 - || offset+amt<=PENDING_BYTE + || offset+amt<=PENDING_BYTE ); #endif #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -3446,7 +3581,7 @@ static int unixRead( /* ** Attempt to seek the file-descriptor passed as the first argument to ** absolute offset iOff, then attempt to write nBuf bytes of data from -** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, ** return the actual number of bytes written (which may be less than ** nBuf). */ @@ -3506,10 +3641,10 @@ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ ** or some other error code on failure. */ static int unixWrite( - sqlite3_file *id, - const void *pBuf, + sqlite3_file *id, + const void *pBuf, int amt, - sqlite3_int64 offset + sqlite3_int64 offset ){ unixFile *pFile = (unixFile*)id; int wrote = 0; @@ -3521,7 +3656,7 @@ static int unixWrite( #if 0 assert( pFile->pPreallocatedUnused==0 || offset>=PENDING_BYTE+512 - || offset+amt<=PENDING_BYTE + || offset+amt<=PENDING_BYTE ); #endif @@ -3548,7 +3683,7 @@ static int unixWrite( #endif #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -3563,7 +3698,7 @@ static int unixWrite( } } #endif - + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ amt -= wrote; offset += wrote; @@ -3629,8 +3764,8 @@ int sqlite3_fullsync_count = 0; ** ** SQLite sets the dataOnly flag if the size of the file is unchanged. ** The idea behind dataOnly is that it should only write the file content -** to disk, not the inode. We only set dataOnly if the file size is -** unchanged since the file size is part of the inode. However, +** to disk, not the inode. We only set dataOnly if the file size is +** unchanged since the file size is part of the inode. However, ** Ted Ts'o tells us that fdatasync() will also write the inode if the ** file size has changed. The only real difference between fdatasync() ** and fsync(), Ted tells us, is that fdatasync() will not flush the @@ -3644,7 +3779,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ int rc; /* The following "ifdef/elif/else/" block has the same structure as - ** the one below. It is replicated here solely to avoid cluttering + ** the one below. It is replicated here solely to avoid cluttering ** up the real code with the UNUSED_PARAMETER() macros. */ #ifdef SQLITE_NO_SYNC @@ -3658,7 +3793,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ UNUSED_PARAMETER(dataOnly); #endif - /* Record the number of times that we do a normal fsync() and + /* Record the number of times that we do a normal fsync() and ** FULLSYNC. This is used during testing to verify that this procedure ** gets called with the correct arguments. */ @@ -3670,7 +3805,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a ** no-op. But go ahead and call fstat() to validate the file ** descriptor as we need a method to provoke a failure during - ** coverate testing. + ** coverage testing. */ #ifdef SQLITE_NO_SYNC { @@ -3684,11 +3819,11 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ rc = 1; } /* If the FULLFSYNC failed, fall back to attempting an fsync(). - ** It shouldn't be possible for fullfsync to fail on the local + ** It shouldn't be possible for fullfsync to fail on the local ** file system (on OSX), so failure indicates that FULLFSYNC - ** isn't supported for this file system. So, attempt an fsync - ** and (for now) ignore the overhead of a superfluous fcntl call. - ** It'd be better to detect fullfsync support once and avoid + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid ** the fcntl call every time sync is called. */ if( rc ) rc = fsync(fd); @@ -3698,7 +3833,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ ** so currently we default to the macro that redefines fdatasync to fsync */ rc = fsync(fd); -#else +#else rc = fdatasync(fd); #if OS_VXWORKS if( rc==-1 && errno==ENOTSUP ){ @@ -3859,7 +3994,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ #if SQLITE_MAX_MMAP_SIZE>0 /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. + ** use read() and write() to access data beyond this point from now on. */ if( nBytemmapSize ){ pFile->mmapSize = nByte; @@ -3905,8 +4040,8 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ static int proxyFileControl(sqlite3_file*,int,void*); #endif -/* -** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT ** file-control operation. Enlarge the database to nBytes in size ** (rounded up to the next chunk-size). If the database is already ** nBytes or larger, this routine is a no-op. @@ -3915,7 +4050,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ if( pFile->szChunk>0 ){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ - + if( osFstat(pFile->h, &buf) ){ return SQLITE_IOERR_FSTAT; } @@ -3924,8 +4059,8 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ if( nSize>(i64)buf.st_size ){ #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE - /* The code below is handling the return value of osFallocate() - ** correctly. posix_fallocate() is defined to "returns zero on success, + /* The code below is handling the return value of osFallocate() + ** correctly. posix_fallocate() is defined to "returns zero on success, ** or an error number on failure". See the manpage for details. */ int err; do{ @@ -3933,7 +4068,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ }while( err==EINTR ); if( err && err!=EINVAL ) return SQLITE_IOERR_WRITE; #else - /* If the OS does not have posix_fallocate(), fake it. Write a + /* If the OS does not have posix_fallocate(), fake it. Write a ** single byte to the last byte in each block that falls entirely ** within the extended region. Then, if required, a single byte ** at offset (nSize-1), to set the size of the file correctly. @@ -3992,9 +4127,13 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ /* Forward declaration */ static int unixGetTempname(int nBuf, char *zBuf); -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) static int unixFcntlExternalReader(unixFile*, int*); #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + static void unixDescribeShm(sqlite3_str*,unixShm*); +#endif + /* ** Information and control of an open file handle. @@ -4017,6 +4156,11 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } #endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + case SQLITE_FCNTL_NULL_IO: { + osClose(pFile->h); + pFile->h = -1; + return SQLITE_OK; + } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; @@ -4063,11 +4207,23 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT case SQLITE_FCNTL_LOCK_TIMEOUT: { int iOld = pFile->iBusyTimeout; - pFile->iBusyTimeout = *(int*)pArg; + int iNew = *(int*)pArg; +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + pFile->iBusyTimeout = iNew<0 ? 0x7FFFFFFF : (unsigned)iNew; +#elif SQLITE_ENABLE_SETLK_TIMEOUT==2 + pFile->iBusyTimeout = !!(*(int*)pArg); +#else +# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2" +#endif *(int*)pArg = iOld; return SQLITE_OK; } -#endif + case SQLITE_FCNTL_BLOCK_ON_CONNECT: { + int iNew = *(int*)pArg; + pFile->bBlockOnConnect = iNew; + return SQLITE_OK; + } +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ #if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; @@ -4113,22 +4269,82 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ case SQLITE_FCNTL_EXTERNAL_READER: { -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) return unixFcntlExternalReader((unixFile*)id, (int*)pArg); #else *(int*)pArg = 0; return SQLITE_OK; #endif } + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + case SQLITE_FCNTL_FILESTAT: { + sqlite3_str *pStr = (sqlite3_str*)pArg; + char aLck[16]; + unixInodeInfo *pInode; + static const char *azLock[] = { "SHARED", "RESERVED", + "PENDING", "EXCLUSIVE" }; + sqlite3_str_appendf(pStr, "{\"h\":%d", pFile->h); + sqlite3_str_appendf(pStr, ",\"vfs\":\"%s\"", pFile->pVfs->zName); + if( pFile->eFileLock ){ + sqlite3_str_appendf(pStr, ",\"eFileLock\":\"%s\"", + azLock[pFile->eFileLock-1]); + if( unixPosixAdvisoryLocks(pFile->h, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + } + unixEnterMutex(); + if( pFile->pShm ){ + sqlite3_str_appendall(pStr, ",\"shm\":"); + unixDescribeShm(pStr, pFile->pShm); + } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSize ){ + sqlite3_str_appendf(pStr, ",\"mmapSize\":%lld", pFile->mmapSize); + sqlite3_str_appendf(pStr, ",\"nFetchOut\":%d", pFile->nFetchOut); + } +#endif + if( (pInode = pFile->pInode)!=0 ){ + sqlite3_str_appendf(pStr, ",\"inode\":{\"nRef\":%d",pInode->nRef); + sqlite3_mutex_enter(pInode->pLockMutex); + sqlite3_str_appendf(pStr, ",\"nShared\":%d", pInode->nShared); + if( pInode->eFileLock ){ + sqlite3_str_appendf(pStr, ",\"eFileLock\":\"%s\"", + azLock[pInode->eFileLock-1]); + } + if( pInode->pUnused ){ + char cSep = '['; + UnixUnusedFd *pUFd = pFile->pInode->pUnused; + sqlite3_str_appendall(pStr, ",\"unusedFd\":"); + while( pUFd ){ + sqlite3_str_appendf(pStr, "%c{\"fd\":%d,\"flags\":%d", + cSep, pUFd->fd, pUFd->flags); + cSep = ','; + if( unixPosixAdvisoryLocks(pUFd->fd, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + sqlite3_str_append(pStr, "}", 1); + pUFd = pUFd->pNext; + } + sqlite3_str_append(pStr, "]", 1); + } + sqlite3_mutex_leave(pInode->pLockMutex); + sqlite3_str_append(pStr, "}", 1); + } + unixLeaveMutex(); + sqlite3_str_append(pStr, "}", 1); + return SQLITE_OK; + } +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ } return SQLITE_NOTFOUND; } /* ** If pFd->sectorSize is non-zero when this function is called, it is a -** no-op. Otherwise, the values of pFd->sectorSize and -** pFd->deviceCharacteristics are set according to the file-system -** characteristics. +** no-op. Otherwise, the values of pFd->sectorSize and +** pFd->deviceCharacteristics are set according to the file-system +** characteristics. ** ** There are two versions of this function. One for QNX and one for all ** other systems. @@ -4152,6 +4368,7 @@ static void setDeviceCharacteristics(unixFile *pFd){ if( pFd->ctrlFlags & UNIXFILE_PSOW ){ pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; } + pFd->deviceCharacteristics |= SQLITE_IOCAP_SUBPAGE_READ; pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; } @@ -4162,7 +4379,7 @@ static void setDeviceCharacteristics(unixFile *pFd){ static void setDeviceCharacteristics(unixFile *pFile){ if( pFile->sectorSize == 0 ){ struct statvfs fsInfo; - + /* Set defaults for non-supported filesystems */ pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; pFile->deviceCharacteristics = 0; @@ -4202,7 +4419,7 @@ static void setDeviceCharacteristics(unixFile *pFile){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -4210,7 +4427,7 @@ static void setDeviceCharacteristics(unixFile *pFile){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -4271,7 +4488,7 @@ static int unixDeviceCharacteristics(sqlite3_file *id){ /* ** Return the system page size. ** -** This function should not be called directly by other code in this file. +** This function should not be called directly by other code in this file. ** Instead, it should be called via macro osGetpagesize(). */ static int unixGetpagesize(void){ @@ -4286,10 +4503,10 @@ static int unixGetpagesize(void){ #endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) /* -** Object used to represent an shared memory buffer. +** Object used to represent an shared memory buffer. ** ** When multiple threads all reference the same wal-index, each thread ** has its own unixShm object, but they all point to a single instance @@ -4309,13 +4526,32 @@ static int unixGetpagesize(void){ ** nRef ** ** The following fields are read-only after the object is created: -** +** ** hShm ** zFilename ** ** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. +** +** aLock[SQLITE_SHM_NLOCK]: +** This array records the various locks held by clients on each of the +** SQLITE_SHM_NLOCK slots. If the aLock[] entry is set to 0, then no +** locks are held by the process on this slot. If it is set to -1, then +** some client holds an EXCLUSIVE lock on the locking slot. If the aLock[] +** value is set to a positive value, then it is the number of shared +** locks currently held on the slot. +** +** aMutex[SQLITE_SHM_NLOCK]: +** Normally, when SQLITE_ENABLE_SETLK_TIMEOUT is not defined, mutex +** pShmMutex is used to protect the aLock[] array and the right to +** call fcntl() on unixShmNode.hShm to obtain or release locks. +** +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined though, we use an array +** of mutexes - one for each locking slot. To read or write locking +** slot aLock[iSlot], the caller must hold the corresponding mutex +** aMutex[iSlot]. Similarly, to call fcntl() to obtain or release a +** lock corresponding to slot iSlot, mutex aMutex[iSlot] must be held. */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ @@ -4329,10 +4565,11 @@ struct unixShmNode { char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK]; +#endif int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ #ifdef SQLITE_DEBUG - u8 exclMask; /* Mask of exclusive locks held */ - u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif }; @@ -4365,10 +4602,30 @@ struct unixShm { #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) +/* +** Describe the pShm object using JSON. Used for diagnostics only. +*/ +static void unixDescribeShm(sqlite3_str *pStr, unixShm *pShm){ + unixShmNode *pNode = pShm->pShmNode; + char aLck[16]; + sqlite3_str_appendf(pStr, "{\"h\":%d", pNode->hShm); + assert( unixMutexHeld() ); + sqlite3_str_appendf(pStr, ",\"nRef\":%d", pNode->nRef); + sqlite3_str_appendf(pStr, ",\"id\":%d", pShm->id); + sqlite3_str_appendf(pStr, ",\"sharedMask\":%d", pShm->sharedMask); + sqlite3_str_appendf(pStr, ",\"exclMask\":%d", pShm->exclMask); + if( unixPosixAdvisoryLocks(pNode->hShm, aLck)==SQLITE_OK ){ + sqlite3_str_appendf(pStr, ",\"pal\":\"%s\"", aLck); + } + sqlite3_str_append(pStr, "}", 1); +} +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + /* ** Use F_GETLK to check whether or not there are any readers with open ** wal-mode transactions in other processes on database file pFile. If -** no error occurs, return SQLITE_OK and set (*piOut) to 1 if there are +** no error occurs, return SQLITE_OK and set (*piOut) to 1 if there are ** such transactions, or 0 otherwise. If an error occurs, return an ** SQLite error code. The final value of *piOut is undefined in this ** case. @@ -4398,6 +4655,44 @@ static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ return rc; } +/* +** If pFile has a -shm file open and it is sharing that file with some +** other connection, either in the same process or in a separate process, +** then return true. Return false if either pFile does not have a -shm +** file open or if it is the only connection to that -shm file across the +** entire system. +** +** This routine is not required for correct operation. It can always return +** false and SQLite will continue to operate according to spec. However, +** when this routine does its job, it adds extra robustness in cases +** where database file locks have been erroneously deleted in a WAL-mode +** database by doing close(open(DATABASE_PATHNAME)) or similar. +** +** With false negatives, SQLite still operates to spec, though with less +** robustness. With false positives, the last database connection on a +** WAL-mode database will fail to unlink the -wal and -shm files, which +** is annoying but harmless. False positives will also prevent a database +** connection from running "PRAGMA journal_mode=DELETE" in order to take +** the database out of WAL mode, which is perhaps more serious, but is +** still not a disaster. +*/ +static int unixIsSharingShmNode(unixFile *pFile){ + unixShmNode *pShmNode; + struct flock lock; + if( pFile->pShm==0 ) return 0; + if( pFile->ctrlFlags & UNIXFILE_EXCL ) return 0; + pShmNode = pFile->pShm->pShmNode; +#if SQLITE_ATOMIC_INTRINSICS + assert( AtomicLoad(&pShmNode->nRef)==1 ); +#endif + memset(&lock, 0, sizeof(lock)); + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + osFcntl(pShmNode->hShm, F_GETLK, &lock); + return (lock.l_type!=F_UNLCK); +} /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. @@ -4415,16 +4710,36 @@ static int unixShmSystemLock( struct flock f; /* The posix advisory locking structure */ int rc = SQLITE_OK; /* Result code form fcntl() */ - /* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; - assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); - assert( pShmNode->nRef>0 || unixMutexHeld() ); + + /* Assert that the parameters are within expected range and that the + ** correct mutex or mutexes are held. */ + assert( pShmNode->nRef>=0 ); + assert( (ofst==UNIX_SHM_DMS && n==1) + || (ofst>=UNIX_SHM_BASE && ofst+n<=(UNIX_SHM_BASE+SQLITE_SHM_NLOCK)) + ); + if( ofst==UNIX_SHM_DMS ){ + assert( pShmNode->nRef>0 || unixMutexHeld() ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int ii; + for(ii=ofst-UNIX_SHM_BASE; iiaMutex[ii]) ); + } +#else + assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 ); +#endif + } /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + assert( ofst>=UNIX_SHM_BASE && ofst<=UNIX_SHM_DMS ); + assert( ofst+n-1<=UNIX_SHM_DMS ); if( pShmNode->hShm>=0 ){ int res; @@ -4435,7 +4750,7 @@ static int unixShmSystemLock( f.l_len = n; res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); if( res==-1 ){ -#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) && SQLITE_ENABLE_SETLK_TIMEOUT==1 rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); #else rc = SQLITE_BUSY; @@ -4443,42 +4758,31 @@ static int unixShmSystemLock( } } - /* Update the global lock state and do debug tracing */ + /* Do debug tracing */ #ifdef SQLITE_DEBUG - { u16 mask; OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; + OSTRACE(("unlock %d..%d ok\n", ofst, ofst+n-1)); }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; + OSTRACE(("read-lock %d..%d ok\n", ofst, ofst+n-1)); }else{ assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; + OSTRACE(("write-lock %d..%d ok\n", ofst, ofst+n-1)); } }else{ if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); + OSTRACE(("unlock %d..%d failed\n", ofst, ofst+n-1)); }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); + OSTRACE(("read-lock %d..%d failed\n", ofst, ofst+n-1)); }else{ assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); + OSTRACE(("write-lock %d..%d failed\n", ofst, ofst+n-1)); } } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); - } #endif - return rc; + return rc; } /* @@ -4512,6 +4816,11 @@ static void unixShmPurge(unixFile *pFd){ int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->pShmMutex); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + for(i=0; iaMutex[i]); + } +#endif for(i=0; inRegion; i+=nShmPerMap){ if( p->hShm>=0 ){ osMunmap(p->apRegion[i], p->szRegion); @@ -4534,7 +4843,7 @@ static void unixShmPurge(unixFile *pFd){ ** take it now. Return SQLITE_OK if successful, or an SQLite error ** code otherwise. ** -** If the DMS cannot be locked because this is a readonly_shm=1 +** If the DMS cannot be locked because this is a readonly_shm=1 ** connection and no other process already holds a lock, return ** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. */ @@ -4545,7 +4854,7 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ /* Use F_GETLK to determine the locks other processes are holding ** on the DMS byte. If it indicates that another process is holding ** a SHARED lock, then this process may also take a SHARED lock - ** and proceed with opening the *-shm file. + ** and proceed with opening the *-shm file. ** ** Or, if no other process is holding any lock, then this process ** is the first to open it. In this case take an EXCLUSIVE lock on the @@ -4571,7 +4880,20 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ pShmNode->isUnlocked = 1; rc = SQLITE_READONLY_CANTINIT; }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* Do not use a blocking lock here. If the lock cannot be obtained + ** immediately, it means some other connection is truncating the + ** *-shm file. And after it has done so, it will not release its + ** lock, but only downgrade it to a shared lock. So no point in + ** blocking here. The call below to obtain the shared DMS lock may + ** use a blocking lock. */ + int iSaveTimeout = pDbFd->iBusyTimeout; + pDbFd->iBusyTimeout = 0; +#endif rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pDbFd->iBusyTimeout = iSaveTimeout; +#endif /* The first connection to attach must truncate the -shm file. We ** truncate to 3 bytes (an arbitrary small number, less than the ** -shm header size) rather than 0 as a system debugging aid, to @@ -4593,20 +4915,20 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ } /* -** Open a shared-memory area associated with open database file pDbFd. +** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. ** ** The file used to implement shared-memory is in the same directory ** as the open database file and has the same name as the open database ** file with the "-shm" suffix added. For example, if the database file ** is "/home/user1/config.db" then the file that is created and mmapped -** for shared memory will be called "/home/user1/config.db-shm". +** for shared memory will be called "/home/user1/config.db-shm". ** ** Another approach to is to use files in /dev/shm or /dev/tmp or an ** some other tmpfs mount. But if a file in a different directory ** from the database file is used, then differing access permissions ** or a chroot() might cause two different processes on the same -** database to end up using different files for shared memory - +** database to end up using different files for shared memory - ** meaning that their memory would not really be shared - resulting ** in database corruption. Nevertheless, this tmpfs file usage ** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" @@ -4676,7 +4998,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); zShm = pShmNode->zFilename = (char*)&pShmNode[1]; #ifdef SQLITE_SHM_DIRECTORY - sqlite3_snprintf(nShmFilename, zShm, + sqlite3_snprintf(nShmFilename, zShm, SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else @@ -4692,6 +5014,18 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { + int ii; + for(ii=0; iiaMutex[ii] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->aMutex[ii]==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + } + } +#endif } if( pInode->bProcessLock==0 ){ @@ -4751,22 +5085,22 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion ** bytes in size. ** ** If an error occurs, an error code is returned and *pp is set to NULL. ** ** Otherwise, if the bExtend parameter is 0 and the requested shared-memory ** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** bExtend is non-zero and the requested shared-memory region has not yet +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet ** been allocated, it is allocated by this function. ** ** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped ** memory and SQLITE_OK returned. */ static int unixShmMap( @@ -4821,7 +5155,7 @@ static int unixShmMap( rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; } - + if( sStat.st_sizehShm>=0 ){ pMem = osMmap(0, nMap, - pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->hShm, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ @@ -4913,9 +5247,11 @@ static int unixShmMap( */ #ifdef SQLITE_DEBUG static int assertLockingArrayOk(unixShmNode *pShmNode){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + return 1; +#else unixShm *pX; int aLock[SQLITE_SHM_NLOCK]; - assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); memset(aLock, 0, sizeof(aLock)); for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ @@ -4933,13 +5269,14 @@ static int assertLockingArrayOk(unixShmNode *pShmNode){ assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) ); return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); -} #endif +} +#endif /* !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) */ /* ** Change the lock state for a shared-memory segment. ** -** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** Note that the relationship between SHARED and EXCLUSIVE locks is a little ** different here than in posix. In xShmLock(), one can go from unlocked ** to shared and back or from unlocked to exclusive and back. But one may ** not go from shared to exclusive or from exclusive to shared. @@ -4954,7 +5291,7 @@ static int unixShmLock( unixShm *p; /* The shared memory being locked */ unixShmNode *pShmNode; /* The underlying file iNode */ int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ + u16 mask = (1<<(ofst+n)) - (1<pShm; @@ -4978,106 +5315,168 @@ static int unixShmLock( /* Check that, if this to be a blocking lock, no locks that occur later ** in the following list than the lock being obtained are already held: ** - ** 1. Checkpointer lock (ofst==1). - ** 2. Write lock (ofst==0). - ** 3. Read locks (ofst>=3 && ofst=3 && ofstexclMask|p->sharedMask); + assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( + (ofst!=2 || lockMask==0) + && (ofst!=1 || lockMask==0 || lockMask==2) + && (ofst!=0 || lockMask<3) + && (ofst<3 || lockMask<(1<exclMask & mask) + ); + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) + ){ + + /* Take the required mutexes. In SETLK_TIMEOUT mode (blocking locks), if + ** this is an attempt on an exclusive lock use sqlite3_mutex_try(). If any + ** other thread is holding this mutex, then it is either holding or about + ** to hold a lock exclusive to the one being requested, and we may + ** therefore return SQLITE_BUSY to the caller. + ** + ** Doing this prevents some deadlock scenarios. For example, thread 1 may + ** be a checkpointer blocked waiting on the WRITER lock. And thread 2 + ** may be a normal SQL client upgrading to a write transaction. In this + ** case thread 2 does a non-blocking request for the WRITER lock. But - + ** if it were to use sqlite3_mutex_enter() then it would effectively + ** become a (doomed) blocking request, as thread 2 would block until thread + ** 1 obtained WRITER and released the mutex. Since thread 2 already holds + ** a lock on a read-locking slot at this point, this breaks the + ** anti-deadlock rules (see above). */ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT - assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( - (ofst!=2) /* not RECOVER */ - && (ofst!=1 || (p->exclMask|p->sharedMask)==0) - && (ofst!=0 || (p->exclMask|p->sharedMask)<3) - && (ofst<3 || (p->exclMask|p->sharedMask)<(1<aMutex[iMutex]); + if( rc!=SQLITE_OK ) goto leave_shmnode_mutexes; + }else{ + sqlite3_mutex_enter(pShmNode->aMutex[iMutex]); + } + } +#else + sqlite3_mutex_enter(pShmNode->pShmMutex); #endif - mask = (1<<(ofst+n)) - (1<1 || mask==(1<pShmMutex); - assert( assertLockingArrayOk(pShmNode) ); - if( flags & SQLITE_SHM_UNLOCK ){ - if( (p->exclMask|p->sharedMask) & mask ){ - int ii; - int bUnlock = 1; - - for(ii=ofst; ii((p->sharedMask & (1<exclMask & p->sharedMask)==0 ); + assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); + assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); + + /* If this is a SHARED lock being unlocked, it is possible that other + ** clients within this process are holding the same SHARED lock. In + ** this case, set bUnlock to 0 so that the posix lock is not removed + ** from the file-descriptor below. */ + if( flags & SQLITE_SHM_SHARED ){ + assert( n==1 ); + assert( aLock[ofst]>=1 ); + if( aLock[ofst]>1 ){ + bUnlock = 0; + aLock[ofst]--; + p->sharedMask &= ~mask; + } } - } - - if( bUnlock ){ - rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); + + if( bUnlock ){ + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); + if( rc==SQLITE_OK ){ + memset(&aLock[ofst], 0, sizeof(int)*n); + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + } + } + }else if( flags & SQLITE_SHM_SHARED ){ + /* Case (b) - a shared lock. */ + + if( aLock[ofst]<0 ){ + /* An exclusive lock is held by some other connection. BUSY. */ + rc = SQLITE_BUSY; + }else if( aLock[ofst]==0 ){ + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); + } + + /* Get the local shared locks */ if( rc==SQLITE_OK ){ - memset(&aLock[ofst], 0, sizeof(int)*n); + p->sharedMask |= mask; + aLock[ofst]++; } - }else if( ALWAYS(p->sharedMask & (1<1 ); - aLock[ofst]--; - } - - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - } - }else if( flags & SQLITE_SHM_SHARED ){ - assert( n==1 ); - assert( (p->exclMask & (1<sharedMask & mask)==0 ){ - if( aLock[ofst]<0 ){ - rc = SQLITE_BUSY; - }else if( aLock[ofst]==0 ){ - rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); - } - - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - aLock[ofst]++; - } - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. */ - int ii; - for(ii=ofst; iisharedMask & mask)==0 ); - if( ALWAYS((p->exclMask & (1<sharedMask & mask)==0 ); - p->exclMask |= mask; + assert( (p->exclMask & mask)==0 ); + + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. */ for(ii=ofst; iiexclMask |= mask; + for(ii=ofst; ii=ofst; iMutex--){ + sqlite3_mutex_leave(pShmNode->aMutex[iMutex]); } +#else + sqlite3_mutex_leave(pShmNode->pShmMutex); +#endif } - assert( assertLockingArrayOk(pShmNode) ); - sqlite3_mutex_leave(pShmNode->pShmMutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; } /* -** Implement a memory barrier or memory fence on shared memory. +** Implement a memory barrier or memory fence on shared memory. ** ** All loads and stores begun before the barrier must complete before ** any load or store begun after the barrier. @@ -5087,15 +5486,15 @@ static void unixShmBarrier( ){ UNUSED_PARAMETER(fd); sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ - assert( fd->pMethods->xLock==nolockLock - || unixFileMutexNotheld((unixFile*)fd) + assert( fd->pMethods->xLock==nolockLock + || unixFileMutexNotheld((unixFile*)fd) ); unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } /* -** Close a connection to shared-memory. Delete the underlying +** Close a connection to shared-memory. Delete the underlying ** storage if deleteFlag is true. ** ** If there is no shared memory associated with the connection then this @@ -5169,7 +5568,7 @@ static void unixUnmapfile(unixFile *pFd){ } /* -** Attempt to set the size of the memory mapping maintained by file +** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. ** ** If successful, this function sets the following variables: @@ -5261,14 +5660,14 @@ static void unixRemapfile( /* ** Memory map or remap the file opened by file-descriptor pFd (if the file -** is already mapped, the existing mapping is replaced by the new). Or, if -** there already exists a mapping for this file, and there are still +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still ** outstanding xFetch() references to it, this function is a no-op. ** -** If parameter nByte is non-negative, then it is the requested size of -** the mapping to create. Otherwise, if nByte is less than zero, then the +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the -** created mapping is either the requested size or the value configured +** created mapping is either the requested size or the value configured ** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not @@ -5309,7 +5708,7 @@ static int unixMapfile(unixFile *pFd, i64 nMap){ ** Finally, if an error does occur, return an SQLite error code. The final ** value of *pp is undefined in this case. ** -** If this function does return a pointer, the caller must eventually +** If this function does return a pointer, the caller must eventually ** release the reference by calling unixUnfetch(). */ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ @@ -5320,11 +5719,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -5334,13 +5738,13 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the third argument is non-NULL, then this function releases a +** If the third argument is non-NULL, then this function releases a ** reference obtained by an earlier call to unixFetch(). The second ** argument passed to this function must be the same as the corresponding -** argument that was passed to the unixFetch() invocation. +** argument that was passed to the unixFetch() invocation. ** -** Or, if the third argument is NULL, then this function is being called -** to inform the VFS layer that, according to POSIX, any existing mapping +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping ** may now be invalid and should be unmapped. */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ @@ -5348,7 +5752,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ UNUSED_PARAMETER(iOff); - /* If p==0 (unmap the entire file) then there must be no outstanding + /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); @@ -5556,8 +5960,8 @@ IOMETHODS( #endif #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -/* -** This "finder" function attempts to determine the best locking strategy +/* +** This "finder" function attempts to determine the best locking strategy ** for the database file "filePath". It then returns the sqlite3_io_methods ** object that implements that strategy. ** @@ -5599,8 +6003,8 @@ static const sqlite3_io_methods *autolockIoFinderImpl( } /* Default case. Handles, amongst others, "nfs". - ** Test byte-range lock using fcntl(). If the call succeeds, - ** assume that the file-system supports POSIX style locks. + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. */ lockInfo.l_len = 1; lockInfo.l_start = 0; @@ -5616,7 +6020,7 @@ static const sqlite3_io_methods *autolockIoFinderImpl( return &dotlockIoMethods; } } -static const sqlite3_io_methods +static const sqlite3_io_methods *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; #endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ @@ -5652,7 +6056,7 @@ static const sqlite3_io_methods *vxworksIoFinderImpl( return &semIoMethods; } } -static const sqlite3_io_methods +static const sqlite3_io_methods *(*const vxworksIoFinder)(const char*,unixFile*) = vxworksIoFinderImpl; #endif /* OS_VXWORKS */ @@ -5780,14 +6184,14 @@ static int fillInUnixFile( robust_close(pNew, h, __LINE__); h = -1; } - unixLeaveMutex(); + unixLeaveMutex(); } } #endif else if( pLockingStyle == &dotlockIoMethods ){ /* Dotfile locking uses the file path so it needs to be included in - ** the dotlockLockingContext + ** the dotlockLockingContext */ char *zLockFile; int nFilename; @@ -5825,14 +6229,21 @@ static int fillInUnixFile( unixLeaveMutex(); } #endif - + storeLastErrno(pNew, 0); #if OS_VXWORKS if( rc!=SQLITE_OK ){ - if( h>=0 ) robust_close(pNew, h, __LINE__); - h = -1; - osUnlink(zFilename); - pNew->ctrlFlags |= UNIXFILE_DELETE; + if( h>=0 ){ + robust_close(pNew, h, __LINE__); + h = -1; + } + if( pNew->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(zFilename); + } + if( pNew->pId ){ + vxworksReleaseFileId(pNew->pId); + pNew->pId = 0; + } } #endif if( rc!=SQLITE_OK ){ @@ -5876,6 +6287,9 @@ static const char *unixTempFileDir(void){ while(1){ if( zDir!=0 +#if OS_VXWORKS + && zDir[0]=='/' +#endif && osStat(zDir, &buf)==0 && S_ISDIR(buf.st_mode) && osAccess(zDir, 03)==0 @@ -5900,7 +6314,7 @@ static int unixGetTempname(int nBuf, char *zBuf){ /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this - ** function failing. + ** function failing. */ zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); @@ -5937,7 +6351,7 @@ static int proxyTransformUnixFile(unixFile*, const char*); #endif /* -** Search for an unused file descriptor that was opened on the database +** Search for an unused file descriptor that was opened on the database ** file (not a journal or super-journal file) identified by pathname ** zPath with SQLITE_OPEN_XXX flags matching those passed as the second ** argument to this function. @@ -5946,7 +6360,7 @@ static int proxyTransformUnixFile(unixFile*, const char*); ** but the associated file descriptor could not be closed because some ** other file descriptor open on the same file is holding a file-lock. ** Refer to comments in the unixClose() function and the lengthy comment -** describing "Posix Advisory Locking" at the start of this file for +** describing "Posix Advisory Locking" at the start of this file for ** further details. Also, ticket #4018. ** ** If a suitable file descriptor is found, then it is returned. If no @@ -5957,8 +6371,8 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ /* Do not search for an unused file descriptor on vxworks. Not because ** vxworks would not benefit from the change (it might, we're not sure), - ** but because no way to test it is currently available. It is better - ** not to risk breaking vxworks support for the sake of such an obscure + ** but because no way to test it is currently available. It is better + ** not to risk breaking vxworks support for the sake of such an obscure ** feature. */ #if !OS_VXWORKS struct stat sStat; /* Results of stat() call */ @@ -6000,7 +6414,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ } /* -** Find the mode, uid and gid of file zFile. +** Find the mode, uid and gid of file zFile. */ static int getFileMode( const char *zFile, /* File name */ @@ -6024,16 +6438,16 @@ static int getFileMode( ** This function is called by unixOpen() to determine the unix permissions ** to create new files with. If no error occurs, then SQLITE_OK is returned ** and a value suitable for passing as the third argument to open(2) is -** written to *pMode. If an IO error occurs, an SQLite error code is +** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** ** In most cases, this routine sets *pMode to 0, which will become ** an indication to robust_open() to create the file using ** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. -** But if the file being opened is a WAL or regular journal file, then -** this function queries the file-system for the permissions on the -** corresponding database file and sets *pMode to this value. Whenever -** possible, WAL and journal files are created using the same permissions +** But if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions ** as the associated database file. ** ** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the @@ -6065,7 +6479,7 @@ static int findCreateFileMode( ** "-journalNN" ** "-walNN" ** - ** where NN is a decimal number. The NN naming schemes are + ** where NN is a decimal number. The NN naming schemes are ** used by the test_multiplex.c module. ** ** In normal operation, the journal file name will always contain @@ -6101,7 +6515,7 @@ static int findCreateFileMode( /* ** Open the file zPath. -** +** ** Previously, the SQLite OS layer used three functions in place of this ** one: ** @@ -6112,13 +6526,13 @@ static int findCreateFileMode( ** These calls correspond to the following combinations of flags: ** ** ReadWrite() -> (READWRITE | CREATE) -** ReadOnly() -> (READONLY) +** ReadOnly() -> (READONLY) ** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) ** ** The old OpenExclusive() accepted a boolean argument - "delFlag". If ** true, the file was configured to be automatically deleted when the -** file handle closed. To achieve the same effect using this new -** interface, add the DELETEONCLOSE flag to those specified above for +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for ** OpenExclusive(). */ static int unixOpen( @@ -6153,8 +6567,8 @@ static int unixOpen( ** is called the directory file descriptor will be fsync()ed and close()d. */ int isNewJrnl = (isCreate && ( - eType==SQLITE_OPEN_SUPER_JOURNAL - || eType==SQLITE_OPEN_MAIN_JOURNAL + eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_WAL )); @@ -6164,9 +6578,9 @@ static int unixOpen( char zTmpname[MAX_PATHNAME+2]; const char *zName = zPath; - /* Check the following statements are true: + /* Check the following statements are true: ** - ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and ** (b) if CREATE is set, then READWRITE must also be set, and ** (c) if EXCLUSIVE is set, then CREATE must also be set. ** (d) if DELETEONCLOSE is set, then CREATE must also be set. @@ -6176,7 +6590,7 @@ static int unixOpen( assert(isExclusive==0 || isCreate); assert(isDelete==0 || isCreate); - /* The main DB, main journal, WAL file and super-journal are never + /* The main DB, main journal, WAL file and super-journal are never ** automatically deleted. Nor are they ever temporary files. */ assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); @@ -6184,12 +6598,18 @@ static int unixOpen( assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); /* Assert that the upper layer has set one of the "file-type" flags. */ - assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB - || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); +#if OS_VXWORKS + /* The file-ID mechanism used in Vxworks requires that all pathnames + ** provided to unixOpen must be absolute pathnames. */ + if( zPath!=0 && zPath[0]!='/' ){ return SQLITE_CANTOPEN; } +#endif + /* Detect a pid change and reset the PRNG. There is a race condition ** here such that two or more threads all trying to open databases at ** the same instant might all reset the PRNG. But multiple resets @@ -6240,7 +6660,7 @@ static int unixOpen( /* Determine the value of the flags parameter passed to POSIX function ** open(). These must be calculated even if open() is not called, as - ** they may be stored as part of the file handle and used by the + ** they may be stored as part of the file handle and used by the ** 'conch file' locking functions later on. */ if( isReadonly ) openFlags |= O_RDONLY; if( isReadWrite ) openFlags |= O_RDWR; @@ -6268,12 +6688,19 @@ static int unixOpen( rc = SQLITE_READONLY_DIRECTORY; }else if( errno!=EISDIR && isReadWrite ){ /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } } } if( fd<0 ){ @@ -6305,7 +6732,7 @@ static int unixOpen( if( p->pPreallocatedUnused ){ p->pPreallocatedUnused->fd = fd; - p->pPreallocatedUnused->flags = + p->pPreallocatedUnused->flags = flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); } @@ -6327,7 +6754,7 @@ static int unixOpen( p->openFlags = openFlags; } #endif - + #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ storeLastErrno(p, errno); @@ -6358,7 +6785,7 @@ static int unixOpen( char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); int useProxy = 0; - /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means ** never use proxy, NULL means use proxy for non-local files only. */ if( envforce!=NULL ){ useProxy = atoi(envforce)>0; @@ -6370,9 +6797,9 @@ static int unixOpen( if( rc==SQLITE_OK ){ rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); if( rc!=SQLITE_OK ){ - /* Use unixClose to clean up the resources added in fillInUnixFile - ** and clear all the structure's references. Specifically, - ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op */ unixClose(pFile); return rc; @@ -6382,9 +6809,12 @@ static int unixOpen( } } #endif - - assert( zPath==0 || zPath[0]=='/' - || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL + + assert( zPath==0 + || zPath[0]=='/' + || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_TEMP_JOURNAL ); rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); @@ -6569,9 +6999,9 @@ static void appendAllPathElements( /* ** Turn a relative pathname into a full pathname. The relative path ** is stored as a nul-terminated string in the buffer pointed to by -** zPath. +** zPath. ** -** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes ** (in this case, MAX_PATHNAME bytes). The full-path is written to ** this buffer before returning. */ @@ -6631,7 +7061,7 @@ static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ unixLeaveMutex(); } static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ - /* + /* ** GCC with -pedantic-errors says that C90 does not allow a void* to be ** cast into a pointer to a function. And yet the library dlsym() routine ** returns a void* which is really a pointer to a function. So how do we @@ -6641,7 +7071,7 @@ static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ ** parameters void* and const char* and returning a pointer to a function. ** We initialize x by assigning it a pointer to the dlsym() function. ** (That assignment requires a cast.) Then we call the function that - ** x points to. + ** x points to. ** ** This work-around is unlikely to work correctly on any system where ** you really cannot cast a function pointer into void*. But then, on the @@ -6684,7 +7114,7 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** tests repeatable. */ memset(zBuf, 0, nBuf); - randomnessPid = osGetpid(0); + randomnessPid = osGetpid(0); #if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) { int fd, got; @@ -6715,12 +7145,17 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS || _POSIX_C_SOURCE >= 199309L +#if !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP+0 struct timespec sp; - sp.tv_sec = microseconds / 1000000; sp.tv_nsec = (microseconds % 1000000) * 1000; + + /* Almost all modern unix systems support nanosleep(). But if you are + ** compiling for one of the rare exceptions, you can use + ** -DHAVE_NANOSLEEP=0 (perhaps in conjunction with -DHAVE_USLEEP if + ** usleep() is available) in order to bypass the use of nanosleep() */ nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); return microseconds; #elif defined(HAVE_USLEEP) && HAVE_USLEEP @@ -6752,7 +7187,7 @@ int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ ** epoch of noon in Greenwich on November 24, 4714 B.C according to the ** proleptic Gregorian calendar. ** -** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date ** cannot be found. */ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ @@ -6859,7 +7294,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** To address the performance and cache coherency issues, proxy file locking ** changes the way database access is controlled by limiting access to a ** single host at a time and moving file locks off of the database file -** and onto a proxy file on the local file system. +** and onto a proxy file on the local file system. ** ** ** Using proxy locks @@ -6885,19 +7320,19 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** actual proxy file name is generated from the name and path of the ** database file. For example: ** -** For database path "/Users/me/foo.db" +** For database path "/Users/me/foo.db" ** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") ** ** Once a lock proxy is configured for a database connection, it can not ** be removed, however it may be switched to a different proxy path via ** the above APIs (assuming the conch file is not being held by another -** connection or process). +** connection or process). ** ** ** How proxy locking works ** ----------------------- ** -** Proxy file locking relies primarily on two new supporting files: +** Proxy file locking relies primarily on two new supporting files: ** ** * conch file to limit access to the database file to a single host ** at a time @@ -6924,11 +7359,11 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** host (the conch ensures that they all use the same local lock file). ** ** Requesting the lock proxy does not immediately take the conch, it is -** only taken when the first request to lock database file is made. +** only taken when the first request to lock database file is made. ** This matches the semantics of the traditional locking behavior, where ** opening a connection to a database file does not take a lock on it. -** The shared lock and an open file descriptor are maintained until -** the connection to the database is closed. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. ** ** The proxy file and the lock file are never deleted so they only need ** to be created the first time they are used. @@ -6942,7 +7377,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** automatically configured for proxy locking, lock files are ** named automatically using the same logic as ** PRAGMA lock_proxy_file=":auto:" -** +** ** SQLITE_PROXY_DEBUG ** ** Enables the logging of error messages during host id file @@ -6957,8 +7392,8 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** ** Permissions to use when creating a directory for storing the ** lock proxy files, only used when LOCKPROXYDIR is not set. -** -** +** +** ** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, ** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will ** force proxy locking to be used for every database file opened, and 0 @@ -6968,12 +7403,12 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ */ /* -** Proxy locking is only available on MacOSX +** Proxy locking is only available on MacOSX */ #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE /* -** The proxyLockingContext has the path and file structures for the remote +** The proxyLockingContext has the path and file structures for the remote ** and local proxy files in it */ typedef struct proxyLockingContext proxyLockingContext; @@ -6989,10 +7424,10 @@ struct proxyLockingContext { sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ }; -/* -** The proxy lock file path for the database at dbPath is written into lPath, +/* +** The proxy lock file path for the database at dbPath is written into lPath, ** which must point to valid, writable memory large enough for a maxLen length -** file path. +** file path. */ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ int len; @@ -7009,7 +7444,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ lPath, errno, osGetpid(0))); return SQLITE_IOERR_LOCK; } - len = strlcat(lPath, "sqliteplocks", maxLen); + len = strlcat(lPath, "sqliteplocks", maxLen); } # else len = strlcpy(lPath, "/tmp/", maxLen); @@ -7019,7 +7454,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ if( lPath[len-1]!='/' ){ len = strlcat(lPath, "/", maxLen); } - + /* transform the db path to a unique cache name */ dbLen = (int)strlen(dbPath); for( i=0; i 0) ){ /* only mkdir if leaf dir != "." or "/" or ".." */ - if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ buf[i]='\0'; if( osMkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ @@ -7124,13 +7559,13 @@ static int proxyCreateUnixFile( switch (terrno) { case EACCES: return SQLITE_PERM; - case EIO: + case EIO: return SQLITE_IOERR_LOCK; /* even though it is the conch */ default: return SQLITE_CANTOPEN_BKPT; } } - + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ rc = SQLITE_NOMEM_BKPT; @@ -7144,13 +7579,13 @@ static int proxyCreateUnixFile( pUnused->fd = fd; pUnused->flags = openFlags; pNew->pPreallocatedUnused = pUnused; - + rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); if( rc==SQLITE_OK ){ *ppFile = pNew; return SQLITE_OK; } -end_create_proxy: +end_create_proxy: robust_close(pNew, fd, __LINE__); sqlite3_free(pNew); sqlite3_free(pUnused); @@ -7169,7 +7604,7 @@ int sqlite3_hostid_num = 0; extern int gethostuuid(uuid_t id, const struct timespec *wait); #endif -/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN ** bytes of writable memory. */ static int proxyGetHostID(unsigned char *pHostID, int *pError){ @@ -7195,7 +7630,7 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); } #endif - + return SQLITE_OK; } @@ -7206,14 +7641,14 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ #define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) #define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) -/* -** Takes an open conch file, copies the contents to a new path and then moves +/* +** Takes an open conch file, copies the contents to a new path and then moves ** it back. The newly created file's file descriptor is assigned to the -** conch file structure and finally the original conch file descriptor is +** conch file structure and finally the original conch file descriptor is ** closed. Returns zero if successful. */ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *conchFile = pCtx->conchFile; char tPath[MAXPATHLEN]; char buf[PROXY_MAXCONCHLEN]; @@ -7227,7 +7662,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ /* create a new path by replace the trailing '-conch' with '-break' */ pathLen = strlcpy(tPath, cPath, MAXPATHLEN); - if( pathLen>MAXPATHLEN || pathLen<6 || + if( pathLen>MAXPATHLEN || pathLen<6 || (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); goto end_breaklock; @@ -7269,24 +7704,24 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ return rc; } -/* Take the requested lock on the conch file and break a stale lock if the +/* Take the requested lock on the conch file and break a stale lock if the ** host id matches. */ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *conchFile = pCtx->conchFile; int rc = SQLITE_OK; int nTries = 0; struct timespec conchModTime; - + memset(&conchModTime, 0, sizeof(conchModTime)); do { rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); nTries ++; if( rc==SQLITE_BUSY ){ /* If the lock failed (busy): - * 1st try: get the mod time of the conch, wait 0.5s and try again. - * 2nd try: fail if the mod time changed or host id is different, wait + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait * 10 sec and try again * 3rd try: break the lock unless the mod time has changed. */ @@ -7295,20 +7730,20 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } - + if( nTries==1 ){ conchModTime = buf.st_mtimespec; unixSleep(0,500000); /* wait 0.5 sec and try the lock again*/ - continue; + continue; } assert( nTries>1 ); - if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ return SQLITE_BUSY; } - - if( nTries==2 ){ + + if( nTries==2 ){ char tBuf[PROXY_MAXCONCHLEN]; int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ @@ -7325,9 +7760,9 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ return SQLITE_BUSY; } unixSleep(0,10000000); /* wait 10 sec and try the lock again */ - continue; + continue; } - + assert( nTries==3 ); if( 0==proxyBreakConchLock(pFile, myHostID) ){ rc = SQLITE_OK; @@ -7340,19 +7775,19 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ } } } while( rc==SQLITE_BUSY && nTries<3 ); - + return rc; } -/* Takes the conch by taking a shared lock and read the contents conch, if -** lockPath is non-NULL, the host ID and lock file path must match. A NULL -** lockPath means that the lockPath in the conch file will be used if the -** host IDs match, or a new lock path will be generated automatically +/* Takes the conch by taking a shared lock and read the contents conch, if +** lockPath is non-NULL, the host ID and lock file path must match. A NULL +** lockPath means that the lockPath in the conch file will be used if the +** host IDs match, or a new lock path will be generated automatically ** and written to the conch file. */ static int proxyTakeConch(unixFile *pFile){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld!=0 ){ return SQLITE_OK; }else{ @@ -7368,7 +7803,7 @@ static int proxyTakeConch(unixFile *pFile){ int readLen = 0; int tryOldLockPath = 0; int forceNewLockPath = 0; - + OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), osGetpid(0))); @@ -7389,21 +7824,21 @@ static int proxyTakeConch(unixFile *pFile){ storeLastErrno(pFile, conchFile->lastErrno); rc = SQLITE_IOERR_READ; goto end_takeconch; - }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || readBuf[0]!=(char)PROXY_CONCHVERSION ){ - /* a short read or version format mismatch means we need to create a new - ** conch file. + /* a short read or version format mismatch means we need to create a new + ** conch file. */ createConch = 1; } /* if the host id matches and the lock path already exists in the conch - ** we'll try to use the path there, if we can't open that path, we'll - ** retry with a new auto-generated path + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path */ do { /* in case we need to try again for an :auto: named lock file */ if( !createConch && !forceNewLockPath ){ - hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); /* if the conch has data compare the contents */ if( !pCtx->lockProxyPath ){ @@ -7412,7 +7847,7 @@ static int proxyTakeConch(unixFile *pFile){ */ if( hostIdMatch ){ size_t pathLen = (readLen - PROXY_PATHINDEX); - + if( pathLen>=MAXPATHLEN ){ pathLen=MAXPATHLEN-1; } @@ -7428,23 +7863,23 @@ static int proxyTakeConch(unixFile *pFile){ readLen-PROXY_PATHINDEX) ){ /* conch host and lock path match */ - goto end_takeconch; + goto end_takeconch; } } - + /* if the conch isn't writable and doesn't match, we can't take it */ if( (conchFile->openFlags&O_RDWR) == 0 ){ rc = SQLITE_BUSY; goto end_takeconch; } - + /* either the conch didn't match or we need to create a new one */ if( !pCtx->lockProxyPath ){ proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); tempLockPath = lockPath; /* create a copy of the lock path _only_ if the conch is taken */ } - + /* update conch with host and path (this will fail if other process ** has a shared lock already), if the host id matches, use the big ** stick. @@ -7455,7 +7890,7 @@ static int proxyTakeConch(unixFile *pFile){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; - } else { + } else { rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } }else{ @@ -7464,7 +7899,7 @@ static int proxyTakeConch(unixFile *pFile){ if( rc==SQLITE_OK ){ char writeBuffer[PROXY_MAXCONCHLEN]; int writeSize = 0; - + writeBuffer[0] = (char)PROXY_CONCHVERSION; memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); if( pCtx->lockProxyPath!=NULL ){ @@ -7477,8 +7912,8 @@ static int proxyTakeConch(unixFile *pFile){ robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); full_fsync(conchFile->h,0,0); - /* If we created a new conch file (not just updated the contents of a - ** valid conch file), try to match the permissions of the database + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database */ if( rc==SQLITE_OK && createConch ){ struct stat buf; @@ -7502,14 +7937,14 @@ static int proxyTakeConch(unixFile *pFile){ } }else{ int code = errno; - fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", err, code, strerror(code)); #endif } } } conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); - + end_takeconch: OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); if( rc==SQLITE_OK && pFile->openFlags ){ @@ -7532,7 +7967,7 @@ static int proxyTakeConch(unixFile *pFile){ rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ /* we couldn't create the proxy lock file with the old lock file path - ** so try again via auto-naming + ** so try again via auto-naming */ forceNewLockPath = 1; tryOldLockPath = 0; @@ -7552,7 +7987,7 @@ static int proxyTakeConch(unixFile *pFile){ } if( rc==SQLITE_OK ){ pCtx->conchHeld = 1; - + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ afpLockingContext *afpCtx; afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; @@ -7564,7 +7999,7 @@ static int proxyTakeConch(unixFile *pFile){ OSTRACE(("TAKECONCH %d %s\n", conchFile->h, rc==SQLITE_OK?"ok":"failed")); return rc; - } while (1); /* in case we need to retry the :auto: lock file - + } while (1); /* in case we need to retry the :auto: lock file - ** we should never get here except via the 'continue' call. */ } } @@ -7580,7 +8015,7 @@ static int proxyReleaseConch(unixFile *pFile){ pCtx = (proxyLockingContext *)pFile->lockingContext; conchFile = pCtx->conchFile; OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, - (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), osGetpid(0))); if( pCtx->conchHeld>0 ){ rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); @@ -7608,13 +8043,13 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ char *conchPath; /* buffer in which to construct conch name */ /* Allocate space for the conch filename and initialize the name to - ** the name of the original database file. */ + ** the name of the original database file. */ *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ return SQLITE_NOMEM_BKPT; } memcpy(conchPath, dbPath, len+1); - + /* now insert a "." before the last / character */ for( i=(len-1); i>=0; i-- ){ if( conchPath[i]=='/' ){ @@ -7637,7 +8072,7 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ /* Takes a fully configured proxy locking-style unix file and switches -** the local lock file path +** the local lock file path */ static int switchLockProxyPath(unixFile *pFile, const char *path) { proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; @@ -7646,7 +8081,7 @@ static int switchLockProxyPath(unixFile *pFile, const char *path) { if( pFile->eFileLock!=NO_LOCK ){ return SQLITE_BUSY; - } + } /* nothing to do if the path is NULL, :auto: or matches the existing path */ if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || @@ -7664,7 +8099,7 @@ static int switchLockProxyPath(unixFile *pFile, const char *path) { sqlite3_free(oldPath); pCtx->lockProxyPath = sqlite3DbStrDup(0, path); } - + return rc; } @@ -7678,7 +8113,7 @@ static int switchLockProxyPath(unixFile *pFile, const char *path) { static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ #if defined(__APPLE__) if( pFile->pMethod == &afpIoMethods ){ - /* afp style keeps a reference to the db path in the filePath field + /* afp style keeps a reference to the db path in the filePath field ** of the struct */ assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, @@ -7699,9 +8134,9 @@ static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ } /* -** Takes an already filled in unix file and alters it so all file locking +** Takes an already filled in unix file and alters it so all file locking ** will be performed on the local proxy lock file. The following fields -** are preserved in the locking context so that they can be restored and +** are preserved in the locking context so that they can be restored and ** the unix structure properly cleaned up at close time: ** ->lockingContext ** ->pMethod @@ -7711,7 +8146,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { char dbPath[MAXPATHLEN+1]; /* Name of the database file */ char *lockPath=NULL; int rc = SQLITE_OK; - + if( pFile->eFileLock!=NO_LOCK ){ return SQLITE_BUSY; } @@ -7721,7 +8156,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { }else{ lockPath=(char *)path; } - + OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, (lockPath ? lockPath : ":auto:"), osGetpid(0))); @@ -7755,7 +8190,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { rc = SQLITE_OK; } } - } + } if( rc==SQLITE_OK && lockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); } @@ -7767,7 +8202,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { } } if( rc==SQLITE_OK ){ - /* all memory is allocated, proxys are created and assigned, + /* all memory is allocated, proxys are created and assigned, ** switch the locking context and pMethod then return. */ pCtx->oldLockingContext = pFile->lockingContext; @@ -7775,12 +8210,12 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx->pOldMethod = pFile->pMethod; pFile->pMethod = &proxyIoMethods; }else{ - if( pCtx->conchFile ){ + if( pCtx->conchFile ){ pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); sqlite3_free(pCtx->conchFile); } sqlite3DbFree(0, pCtx->lockProxyPath); - sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx->conchFilePath); sqlite3_free(pCtx); } OSTRACE(("TRANSPROXY %d %s\n", pFile->h, @@ -7818,7 +8253,7 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ if( isProxyStyle ){ /* turn off proxy locking - not supported. If support is added for ** switching proxy locking mode off then it will need to fail if - ** the journal mode is WAL mode. + ** the journal mode is WAL mode. */ rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; }else{ @@ -7828,9 +8263,9 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ }else{ const char *proxyPath = (const char *)pArg; if( isProxyStyle ){ - proxyLockingContext *pCtx = + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; - if( !strcmp(pArg, ":auto:") + if( !strcmp(pArg, ":auto:") || (pCtx->lockProxyPath && !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) ){ @@ -7955,7 +8390,7 @@ static int proxyClose(sqlite3_file *id) { unixFile *lockProxy = pCtx->lockProxy; unixFile *conchFile = pCtx->conchFile; int rc = SQLITE_OK; - + if( lockProxy ){ rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); if( rc ) return rc; @@ -7992,7 +8427,7 @@ static int proxyClose(sqlite3_file *id) { ** The proxy locking style is intended for use with AFP filesystems. ** And since AFP is only supported on MacOSX, the proxy locking is also ** restricted to MacOSX. -** +** ** ******************* End of the proxy lock implementation ********************** ******************************************************************************/ @@ -8010,8 +8445,8 @@ static int proxyClose(sqlite3_file *id) { ** necessarily been initialized when this routine is called, and so they ** should not be used. */ -int sqlite3_os_init(void){ - /* +int sqlite3_os_init(void){ + /* ** The following macro defines an initializer for an sqlite3_vfs object. ** The name of the VFS is NAME. The pAppData is a pointer to a pointer ** to the "finder" function. (pAppData is a pointer to a pointer because @@ -8027,7 +8462,7 @@ int sqlite3_os_init(void){ ** ** Most finders simply return a pointer to a fixed sqlite3_io_methods ** object. But the "autolockIoFinder" available on MacOSX does a little - ** more than that; it looks at the filesystem type that hosts the + ** more than that; it looks at the filesystem type that hosts the ** database file and tries to choose an locking method appropriate for ** that filesystem time. */ @@ -8108,6 +8543,9 @@ int sqlite3_os_init(void){ sqlite3KvvfsInit(); #endif unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#if OS_VXWORKS + vxworksMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS2); +#endif #ifndef SQLITE_OMIT_WAL /* Validate lock assumptions */ @@ -8130,7 +8568,7 @@ int sqlite3_os_init(void){ /* Initialize temp file dir array. */ unixTempFileInit(); - return SQLITE_OK; + return SQLITE_OK; } /* @@ -8140,9 +8578,12 @@ int sqlite3_os_init(void){ ** to release dynamically allocated objects. But not on unix. ** This routine is a no-op for unix. */ -int sqlite3_os_end(void){ +int sqlite3_os_end(void){ unixBigLock = 0; - return SQLITE_OK; +#if OS_VXWORKS + vxworksMutex = 0; +#endif + return SQLITE_OK; } - + #endif /* SQLITE_OS_UNIX */ diff --git a/src/os_win.c b/src/os_win.c index b7b6897452..a6b25f2e86 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -287,8 +287,18 @@ struct winFile { sqlite3_int64 mmapSize; /* Size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + DWORD iBusyTimeout; /* Wait this many millisec on locks */ + int bBlockOnConnect; +#endif }; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +# define winFileBusyTimeout(pDbFd) pDbFd->iBusyTimeout +#else +# define winFileBusyTimeout(pDbFd) 0 +#endif + /* ** The winVfsAppData structure is used for the pAppData member for all of the ** Win32 VFS variants. @@ -607,7 +617,7 @@ static struct win_syscall { { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, #endif -#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(const FILETIME*, \ LPFILETIME))aSyscall[11].pCurrent) #if SQLITE_OS_WINCE @@ -616,7 +626,7 @@ static struct win_syscall { { "FileTimeToSystemTime", (SYSCALL)0, 0 }, #endif -#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(const FILETIME*, \ LPSYSTEMTIME))aSyscall[12].pCurrent) { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, @@ -722,6 +732,12 @@ static struct win_syscall { #define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ LPWSTR*))aSyscall[25].pCurrent) +/* +** For GetLastError(), MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ { "GetLastError", (SYSCALL)GetLastError, 0 }, #define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) @@ -890,7 +906,7 @@ static struct win_syscall { { "LockFile", (SYSCALL)0, 0 }, #endif -#ifndef osLockFile +#if !defined(osLockFile) && defined(SQLITE_WIN32_HAS_ANSI) #define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ DWORD))aSyscall[47].pCurrent) #endif @@ -954,7 +970,7 @@ static struct win_syscall { { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, -#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT @@ -963,7 +979,7 @@ static struct win_syscall { { "UnlockFile", (SYSCALL)0, 0 }, #endif -#ifndef osUnlockFile +#if !defined(osUnlockFile) && defined(SQLITE_WIN32_HAS_ANSI) #define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ DWORD))aSyscall[57].pCurrent) #endif @@ -1004,11 +1020,13 @@ static struct win_syscall { #define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ DWORD,DWORD))aSyscall[62].pCurrent) -#if !SQLITE_OS_WINRT +/* +** For WaitForSingleObject(), MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, -#else - { "WaitForSingleObject", (SYSCALL)0, 0 }, -#endif #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) @@ -1155,11 +1173,102 @@ static struct win_syscall { #define osFlushViewOfFile \ ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) +/* +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() +** to implement blocking locks with timeouts. MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { "CreateEvent", (SYSCALL)CreateEvent, 0 }, +#else + { "CreateEvent", (SYSCALL)0, 0 }, +#endif + +#define osCreateEvent ( \ + (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ + aSyscall[80].pCurrent \ +) + +/* +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CancelIo() +** for the case where a timeout expires and a lock request must be +** cancelled. +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { "CancelIo", (SYSCALL)CancelIo, 0 }, +#else + { "CancelIo", (SYSCALL)0, 0 }, +#endif + +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) + { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, +#else + { "GetModuleHandleW", (SYSCALL)0, 0 }, +#endif + +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) + +#ifndef _WIN32 + { "getenv", (SYSCALL)getenv, 0 }, +#else + { "getenv", (SYSCALL)0, 0 }, +#endif + +#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) + +#ifndef _WIN32 + { "getcwd", (SYSCALL)getcwd, 0 }, +#else + { "getcwd", (SYSCALL)0, 0 }, +#endif + +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) + +#ifndef _WIN32 + { "readlink", (SYSCALL)readlink, 0 }, +#else + { "readlink", (SYSCALL)0, 0 }, +#endif + +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) + +#ifndef _WIN32 + { "lstat", (SYSCALL)lstat, 0 }, +#else + { "lstat", (SYSCALL)0, 0 }, +#endif + +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) + +#ifndef _WIN32 + { "__errno", (SYSCALL)__errno, 0 }, +#else + { "__errno", (SYSCALL)0, 0 }, +#endif + +#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) + +#ifndef _WIN32 + { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, +#else + { "cygwin_conv_path", (SYSCALL)0, 0 }, +#endif + +#define osCygwin_conv_path ((size_t(*)(unsigned int, \ + const void *, void *, size_t))aSyscall[88].pCurrent) + }; /* End of the overrideable system calls */ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** "win32" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -1328,6 +1437,7 @@ int sqlite3_win32_reset_heap(){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** This function outputs the specified (ANSI) string to the Win32 debugger ** (if available). @@ -1370,6 +1480,7 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif } +#endif /* _WIN32 */ /* ** The following routine suspends the current thread for at least ms @@ -1453,7 +1564,9 @@ int sqlite3_win32_is_nt(void){ } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #elif SQLITE_TEST - return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2 + || osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 + ; #else /* ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are @@ -1668,6 +1781,7 @@ void sqlite3MemSetDefault(void){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** Convert a UTF-8 string to Microsoft Unicode. ** @@ -1693,6 +1807,7 @@ static LPWSTR winUtf8ToUnicode(const char *zText){ } return zWideText; } +#endif /* _WIN32 */ /* ** Convert a Microsoft Unicode string to UTF-8. @@ -1727,28 +1842,29 @@ static char *winUnicodeToUtf8(LPCWSTR zWideText){ ** Space to hold the returned string is obtained from sqlite3_malloc(). */ static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ - int nByte; + int nWideChar; LPWSTR zMbcsText; int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, - 0)*sizeof(WCHAR); - if( nByte==0 ){ + nWideChar = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, + 0); + if( nWideChar==0 ){ return 0; } - zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + zMbcsText = sqlite3MallocZero( nWideChar*sizeof(WCHAR) ); if( zMbcsText==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, - nByte); - if( nByte==0 ){ + nWideChar = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, + nWideChar); + if( nWideChar==0 ){ sqlite3_free(zMbcsText); zMbcsText = 0; } return zMbcsText; } +#ifdef _WIN32 /* ** Convert a Microsoft Unicode string to a multi-byte character string, ** using the ANSI or OEM code page. @@ -1776,6 +1892,7 @@ static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ } return zText; } +#endif /* _WIN32 */ /* ** Convert a multi-byte character string to UTF-8. @@ -1795,6 +1912,7 @@ static char *winMbcsToUtf8(const char *zText, int useAnsi){ return zTextUtf8; } +#ifdef _WIN32 /* ** Convert a UTF-8 string to a multi-byte character string. ** @@ -1844,6 +1962,7 @@ char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ #endif return winUnicodeToUtf8(zWideText); } +#endif /* _WIN32 */ /* ** This is a public wrapper for the winMbcsToUtf8() function. @@ -1861,6 +1980,7 @@ char *sqlite3_win32_mbcs_to_utf8(const char *zText){ return winMbcsToUtf8(zText, osAreFileApisANSI()); } +#ifdef _WIN32 /* ** This is a public wrapper for the winMbcsToUtf8() function. */ @@ -1985,6 +2105,7 @@ int sqlite3_win32_set_directory( ){ return sqlite3_win32_set_directory16(type, zValue); } +#endif /* _WIN32 */ /* ** The return value of winGetLastErrorMsg @@ -2533,13 +2654,100 @@ static BOOL winLockFile( ovlp.Offset = offsetLow; ovlp.OffsetHigh = offsetHigh; return osLockFileEx(*phFile, flags, 0, numBytesLow, numBytesHigh, &ovlp); +#ifdef SQLITE_WIN32_HAS_ANSI }else{ return osLockFile(*phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); +#endif } #endif } +#ifndef SQLITE_OMIT_WAL +/* +** Lock a region of nByte bytes starting at offset offset of file hFile. +** Take an EXCLUSIVE lock if parameter bExclusive is true, or a SHARED lock +** otherwise. If nMs is greater than zero and the lock cannot be obtained +** immediately, block for that many ms before giving up. +** +** This function returns SQLITE_OK if the lock is obtained successfully. If +** some other process holds the lock, SQLITE_BUSY is returned if nMs==0, or +** SQLITE_BUSY_TIMEOUT otherwise. Or, if an error occurs, SQLITE_IOERR. +*/ +static int winHandleLockTimeout( + HANDLE hFile, + DWORD offset, + DWORD nByte, + int bExcl, + DWORD nMs +){ + DWORD flags = LOCKFILE_FAIL_IMMEDIATELY | (bExcl?LOCKFILE_EXCLUSIVE_LOCK:0); + int rc = SQLITE_OK; + BOOL ret; + + if( !osIsNT() ){ + ret = winLockFile(&hFile, flags, offset, 0, nByte, 0); + }else{ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offset; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( nMs!=0 ){ + flags &= ~LOCKFILE_FAIL_IMMEDIATELY; + } + ovlp.hEvent = osCreateEvent(NULL, TRUE, FALSE, NULL); + if( ovlp.hEvent==NULL ){ + return SQLITE_IOERR_LOCK; + } +#endif + + ret = osLockFileEx(hFile, flags, 0, nByte, 0, &ovlp); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If SQLITE_ENABLE_SETLK_TIMEOUT is defined, then the file-handle was + ** opened with FILE_FLAG_OVERHEAD specified. In this case, the call to + ** LockFileEx() may fail because the request is still pending. This can + ** happen even if LOCKFILE_FAIL_IMMEDIATELY was specified. + ** + ** If nMs is 0, then LOCKFILE_FAIL_IMMEDIATELY was set in the flags + ** passed to LockFileEx(). In this case, if the operation is pending, + ** block indefinitely until it is finished. + ** + ** Otherwise, wait for up to nMs ms for the operation to finish. nMs + ** may be set to INFINITE. + */ + if( !ret && GetLastError()==ERROR_IO_PENDING ){ + DWORD nDelay = (nMs==0 ? INFINITE : nMs); + DWORD res = osWaitForSingleObject(ovlp.hEvent, nDelay); + if( res==WAIT_OBJECT_0 ){ + ret = TRUE; + }else if( res==WAIT_TIMEOUT ){ +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + rc = SQLITE_BUSY_TIMEOUT; +#else + rc = SQLITE_BUSY; +#endif + }else{ + /* Some other error has occurred */ + rc = SQLITE_IOERR_LOCK; + } + + /* If it is still pending, cancel the LockFileEx() call. */ + osCancelIo(hFile); + } + + osCloseHandle(ovlp.hEvent); +#endif + } + + if( rc==SQLITE_OK && !ret ){ + rc = SQLITE_BUSY; + } + return rc; +} +#endif /* #ifndef SQLITE_OMIT_WAL */ + /* ** Unlock a file region. */ @@ -2564,13 +2772,25 @@ static BOOL winUnlockFile( ovlp.Offset = offsetLow; ovlp.OffsetHigh = offsetHigh; return osUnlockFileEx(*phFile, 0, numBytesLow, numBytesHigh, &ovlp); +#ifdef SQLITE_WIN32_HAS_ANSI }else{ return osUnlockFile(*phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); +#endif } #endif } +#ifndef SQLITE_OMIT_WAL +/* +** Remove an nByte lock starting at offset iOff from HANDLE h. +*/ +static int winHandleUnlock(HANDLE h, int iOff, int nByte){ + BOOL ret = winUnlockFile(&h, iOff, 0, nByte, 0); + return (ret ? SQLITE_OK : SQLITE_IOERR_UNLOCK); +} +#endif + /***************************************************************************** ** The next group of routines implement the I/O methods specified ** by the sqlite3_io_methods object. @@ -2584,66 +2804,70 @@ static BOOL winUnlockFile( #endif /* -** Move the current position of the file handle passed as the first -** argument to offset iOffset within the file. If successful, return 0. -** Otherwise, set pFile->lastErrno and return non-zero. +** Seek the file handle h to offset nByte of the file. +** +** If successful, return SQLITE_OK. Or, if an error occurs, return an SQLite +** error code. */ -static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ +static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ + int rc = SQLITE_OK; /* Return value */ + #if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ - DWORD lastErrno; /* Value returned by GetLastError() */ - - OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); + dwRet = osSetFilePointer(h, lowerBits, &upperBits, FILE_BEGIN); + /* API oddity: If successful, SetFilePointer() returns a dword ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine ** whether an error has actually occurred, it is also necessary to call - ** GetLastError(). - */ - dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - - if( (dwRet==INVALID_SET_FILE_POINTER - && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ - pFile->lastErrno = lastErrno; - winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "winSeekFile", pFile->zPath); - OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); - return 1; + ** GetLastError(). */ + if( dwRet==INVALID_SET_FILE_POINTER ){ + DWORD lastErrno = osGetLastError(); + if( lastErrno!=NO_ERROR ){ + rc = SQLITE_IOERR_SEEK; + } } - - OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); - return 0; #else - /* - ** Same as above, except that this implementation works for WinRT. - */ - + /* This implementation works for WinRT. */ LARGE_INTEGER x; /* The new offset */ BOOL bRet; /* Value returned by SetFilePointerEx() */ x.QuadPart = iOffset; - bRet = osSetFilePointerEx(pFile->h, x, 0, FILE_BEGIN); + bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); if(!bRet){ - pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "winSeekFile", pFile->zPath); - OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); - return 1; + rc = SQLITE_IOERR_SEEK; } - - OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); - return 0; #endif + + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); + return rc; } +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ + int rc; + + rc = winHandleSeek(pFile->h, iOffset); + if( rc!=SQLITE_OK ){ + pFile->lastErrno = osGetLastError(); + winLogError(rc, pFile->lastErrno, "winSeekFile", pFile->zPath); + } + return rc; +} + + #if SQLITE_MAX_MMAP_SIZE>0 /* Forward references to VFS helper methods used for memory mapped files */ static int winMapfile(winFile*, sqlite3_int64); @@ -2739,7 +2963,7 @@ static int winRead( pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -2817,7 +3041,7 @@ static int winWrite( pFile->h, pBuf, amt, offset, pFile->locktype)); #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -2903,6 +3127,62 @@ static int winWrite( return SQLITE_OK; } +#ifndef SQLITE_OMIT_WAL +/* +** Truncate the file opened by handle h to nByte bytes in size. +*/ +static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ + int rc = SQLITE_OK; /* Return code */ + rc = winHandleSeek(h, nByte); + if( rc==SQLITE_OK ){ + if( 0==osSetEndOfFile(h) ){ + rc = SQLITE_IOERR_TRUNCATE; + } + } + return rc; +} + +/* +** Determine the size in bytes of the file opened by the handle passed as +** the first argument. +*/ +static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ + int rc = SQLITE_OK; + +#if SQLITE_OS_WINRT + FILE_STANDARD_INFO info; + BOOL b; + b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); + if( b ){ + *pnByte = info.EndOfFile.QuadPart; + }else{ + rc = SQLITE_IOERR_FSTAT; + } +#else + DWORD upperBits = 0; + DWORD lowerBits = 0; + + assert( pnByte ); + lowerBits = osGetFileSize(h, &upperBits); + *pnByte = (((sqlite3_int64)upperBits)<<32) + lowerBits; + if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ + rc = SQLITE_IOERR_FSTAT; + } +#endif + + return rc; +} + +/* +** Close the handle passed as the only argument. +*/ +static void winHandleClose(HANDLE h){ + if( h!=INVALID_HANDLE_VALUE ){ + osCloseHandle(h); + } +} +#endif /* #ifndef SQLITE_OMIT_WAL */ + /* ** Truncate an open file to a specified size */ @@ -2927,7 +3207,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ ** all references to memory-mapped content are closed. That is doable, ** but involves adding a few branches in the common write code path which ** could slow down normal operations slightly. Hence, we have decided for - ** now to simply make trancations a no-op if there are pending reads. We + ** now to simply make transactions a no-op if there are pending reads. We ** can maybe revisit this decision in the future. */ return SQLITE_OK; @@ -2986,7 +3266,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ #ifdef SQLITE_TEST /* ** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occuring at the right times. +** that syncs and fullsyncs are occurring at the right times. */ int sqlite3_sync_count = 0; int sqlite3_fullsync_count = 0; @@ -3158,8 +3438,9 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ ** Different API routines are called depending on whether or not this ** is Win9x or WinNT. */ -static int winGetReadLock(winFile *pFile){ +static int winGetReadLock(winFile *pFile, int bBlock){ int res; + DWORD mask = ~(bBlock ? LOCKFILE_FAIL_IMMEDIATELY : 0); OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( osIsNT() ){ #if SQLITE_OS_WINCE @@ -3169,7 +3450,7 @@ static int winGetReadLock(winFile *pFile){ */ res = winceLockFile(&pFile->h, SHARED_FIRST, 0, 1, 0); #else - res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS, SHARED_FIRST, 0, + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS&mask, SHARED_FIRST, 0, SHARED_SIZE, 0); #endif } @@ -3178,7 +3459,7 @@ static int winGetReadLock(winFile *pFile){ int lk; sqlite3_randomness(sizeof(lk), &lk); pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); - res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS&mask, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); } #endif @@ -3273,46 +3554,62 @@ static int winLock(sqlite3_file *id, int locktype){ assert( locktype!=PENDING_LOCK ); assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); - /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + /* Lock the PENDING_LOCK byte if we need to acquire an EXCLUSIVE lock or ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( pFile->locktype==NO_LOCK - || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) + if( locktype==SHARED_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype==RESERVED_LOCK) ){ int cnt = 3; - while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, - PENDING_BYTE, 0, 1, 0))==0 ){ + + /* Flags for the LockFileEx() call. This should be an exclusive lock if + ** this call is to obtain EXCLUSIVE, or a shared lock if this call is to + ** obtain SHARED. */ + int flags = LOCKFILE_FAIL_IMMEDIATELY; + if( locktype==EXCLUSIVE_LOCK ){ + flags |= LOCKFILE_EXCLUSIVE_LOCK; + } + while( cnt>0 ){ /* Try 3 times to get the pending lock. This is needed to work ** around problems caused by indexing and/or anti-virus software on ** Windows systems. + ** ** If you are using this code as a model for alternative VFSes, do not - ** copy this retry logic. It is a hack intended for Windows only. - */ + ** copy this retry logic. It is a hack intended for Windows only. */ + res = winLockFile(&pFile->h, flags, PENDING_BYTE, 0, 1, 0); + if( res ) break; + lastErrno = osGetLastError(); - OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", - pFile->h, cnt, res)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", + pFile->h, cnt, res + )); + if( lastErrno==ERROR_INVALID_HANDLE ){ pFile->lastErrno = lastErrno; rc = SQLITE_IOERR_LOCK; - OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", - pFile->h, cnt, sqlite3ErrName(rc))); + OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(rc) + )); return rc; } - if( cnt ) sqlite3_win32_sleep(1); + + cnt--; + if( cnt>0 ) sqlite3_win32_sleep(1); } gotPendingLock = res; - if( !res ){ - lastErrno = osGetLastError(); - } } /* Acquire a shared lock */ if( locktype==SHARED_LOCK && res ){ assert( pFile->locktype==NO_LOCK ); - res = winGetReadLock(pFile); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + res = winGetReadLock(pFile, pFile->bBlockOnConnect); +#else + res = winGetReadLock(pFile, 0); +#endif if( res ){ newLocktype = SHARED_LOCK; }else{ @@ -3343,14 +3640,14 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); - res = winUnlockReadLock(pFile); + (void)winUnlockReadLock(pFile); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - winGetReadLock(pFile); + winGetReadLock(pFile, 0); } } @@ -3430,7 +3727,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); - if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ + if( locktype==SHARED_LOCK && !winGetReadLock(pFile, 0) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), @@ -3599,6 +3896,11 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } #endif + case SQLITE_FCNTL_NULL_IO: { + (void)osCloseHandle(pFile->h); + pFile->h = NULL; + return SQLITE_OK; + } case SQLITE_FCNTL_TEMPFILENAME: { char *zTFile = 0; int rc = winGetTempname(pFile->pVfs, &zTFile); @@ -3635,6 +3937,50 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return rc; } #endif + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; + int iNew = *(int*)pArg; +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + pFile->iBusyTimeout = (iNew < 0) ? INFINITE : (DWORD)iNew; +#elif SQLITE_ENABLE_SETLK_TIMEOUT==2 + pFile->iBusyTimeout = (DWORD)(!!iNew); +#else +# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2" +#endif + *(int*)pArg = iOld; + return SQLITE_OK; + } + case SQLITE_FCNTL_BLOCK_ON_CONNECT: { + int iNew = *(int*)pArg; + pFile->bBlockOnConnect = iNew; + return SQLITE_OK; + } +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_FILESTAT) + case SQLITE_FCNTL_FILESTAT: { + sqlite3_str *pStr = (sqlite3_str*)pArg; + sqlite3_str_appendf(pStr, "{\"h\":%llu", (sqlite3_uint64)pFile->h); + sqlite3_str_appendf(pStr, ",\"vfs\":\"%s\"", pFile->pVfs->zName); + if( pFile->locktype ){ + static const char *azLock[] = { "SHARED", "RESERVED", + "PENDING", "EXCLUSIVE" }; + sqlite3_str_appendf(pStr, ",\"locktype\":\"%s\"", + azLock[pFile->locktype-1]); + } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSize ){ + sqlite3_str_appendf(pStr, ",\"mmapSize\":%lld", pFile->mmapSize); + sqlite3_str_appendf(pStr, ",\"nFetchOut\":%d", pFile->nFetchOut); + } +#endif + sqlite3_str_append(pStr, "}", 1); + return SQLITE_OK; + } +#endif /* SQLITE_DEBUG || SQLITE_ENABLE_FILESTAT */ + } OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; @@ -3660,7 +4006,7 @@ static int winSectorSize(sqlite3_file *id){ */ static int winDeviceCharacteristics(sqlite3_file *id){ winFile *p = (winFile*)id; - return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | SQLITE_IOCAP_SUBPAGE_READ | ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } @@ -3672,6 +4018,103 @@ static int winDeviceCharacteristics(sqlite3_file *id){ */ static SYSTEM_INFO winSysInfo; +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function +** +** On Cygwin, 3 possible input forms are accepted: +** - If the filename starts with ":/" or ":\", +** it is converted to UTF-16 as-is. +** - If the filename contains '/', it is assumed to be a +** Cygwin absolute path, it is converted to a win32 +** absolute path in UTF-16. +** - Otherwise it must be a filename only, the win32 filename +** is returned in UTF-16. +** Note: If the function cygwin_conv_path() fails, only +** UTF-8 -> UTF-16 conversion will be done. This can only +** happen when the file path >32k, in which case winUtf8ToUnicode() +** will fail too. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ +#ifdef __CYGWIN__ + int nChar; + LPWSTR zWideFilename; + + if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2])) ){ + i64 nByte; + int convertflag = CCP_POSIX_TO_WIN_W; + if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; + nByte = (i64)osCygwin_conv_path(convertflag, + zFilename, 0, 0); + if( nByte>0 ){ + zConverted = sqlite3MallocZero(12+(u64)nByte); + if ( zConverted==0 ){ + return zConverted; + } + zWideFilename = zConverted; + /* Filenames should be prefixed, except when converted + * full path already starts with "\\?\". */ + if( osCygwin_conv_path(convertflag, zFilename, + zWideFilename+4, nByte)==0 ){ + if( (convertflag&CCP_RELATIVE) ){ + memmove(zWideFilename, zWideFilename+4, nByte); + }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( zWideFilename[6]!='?' ){ + memmove(zWideFilename+6, zWideFilename+4, nByte); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + }else{ + memmove(zWideFilename, zWideFilename+4, nByte); + } + return zConverted; + } + sqlite3_free(zConverted); + } + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); + if( zWideFilename==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, + zWideFilename, nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + }else if( nChar>MAX_PATH + && winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2]) ){ + memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); + zWideFilename[2] = '\\'; + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( nChar>MAX_PATH + && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) + && zFilename[2] != '?' ){ + memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + } + zConverted = zWideFilename; +#else + zConverted = winUtf8ToUnicode(zFilename); +#endif /* __CYGWIN__ */ + } +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + #ifndef SQLITE_OMIT_WAL /* @@ -3708,30 +4151,40 @@ static int winShmMutexHeld(void) { ** log-summary is opened only once per process. ** ** winShmMutexHeld() must be true when creating or destroying -** this object or while reading or writing the following fields: +** this object, or while editing the global linked list that starts +** at winShmNodeList. ** -** nRef -** pNext +** When reading or writing the linked list starting at winShmNode.pWinShmList, +** pShmNode->mutex must be held. ** -** The following fields are read-only after the object is created: +** The following fields are constant after the object is created: ** -** fid ** zFilename +** hSharedShm +** mutex +** bUseSharedLockHandle ** -** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** Either winShmNode.mutex must be held or winShmNode.pWinShmList==0 and ** winShmMutexHeld() is true when reading or writing any other field ** in this structure. ** +** File-handle hSharedShm is always used to (a) take the DMS lock, (b) +** truncate the *-shm file if the DMS-locking protocol demands it, and +** (c) map regions of the *-shm file into memory using MapViewOfFile() +** or similar. If bUseSharedLockHandle is true, then other locks are also +** taken on hSharedShm. Or, if bUseSharedLockHandle is false, then other +** locks are taken using each connection's winShm.hShm handles. */ struct winShmNode { sqlite3_mutex *mutex; /* Mutex to access this object */ char *zFilename; /* Name of the file */ - winFile hFile; /* File handle from winOpen */ + HANDLE hSharedShm; /* File handle open on zFilename */ + int bUseSharedLockHandle; /* True to use hSharedShm for everything */ + int isUnlocked; /* DMS lock has not yet been obtained */ + int isReadonly; /* True if read-only */ int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ - u8 isReadonly; /* True if read-only */ - u8 isUnlocked; /* True if no DMS lock held */ struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ @@ -3739,8 +4192,8 @@ struct winShmNode { } *aRegion; DWORD lastErrno; /* The Windows errno from the last I/O error */ - int nRef; /* Number of winShm objects pointing to this */ - winShm *pFirst; /* All winShm objects pointing to this */ + winShm *pWinShmList; /* List of winShm objects with ptrs to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ @@ -3756,26 +4209,19 @@ static winShmNode *winShmNodeList = 0; /* ** Structure used internally by this VFS to record the state of an -** open shared memory connection. -** -** The following fields are initialized when this object is created and -** are read-only thereafter: -** -** winShm.pShmNode -** winShm.id -** -** All other fields are read/write. The winShm.pShmNode->mutex must be held -** while accessing any read/write fields. +** open shared memory connection. There is one such structure for each +** winFile open on a wal mode database. */ struct winShm { winShmNode *pShmNode; /* The underlying winShmNode object */ - winShm *pNext; /* Next winShm with the same winShmNode */ - u8 hasMutex; /* True if holding the winShmNode mutex */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ + HANDLE hShm; /* File-handle on *-shm file. For locking. */ + int bReadonly; /* True if hShm is opened read-only */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif + winShm *pWinShmNext; /* Next winShm object on same winShmNode */ }; /* @@ -3784,56 +4230,12 @@ struct winShm { #define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ -/* -** Apply advisory locks for all n bytes beginning at ofst. -*/ -#define WINSHM_UNLCK 1 -#define WINSHM_RDLCK 2 -#define WINSHM_WRLCK 3 -static int winShmSystemLock( - winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ - int ofst, /* Offset to first byte to be locked/unlocked */ - int nByte /* Number of bytes to lock or unlock */ -){ - int rc = 0; /* Result code form Lock/UnlockFileEx() */ - - /* Access to the winShmNode object is serialized by the caller */ - assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); - - OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", - pFile->hFile.h, lockType, ofst, nByte)); - - /* Release/Acquire the system-level lock */ - if( lockType==WINSHM_UNLCK ){ - rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); - }else{ - /* Initialize the locking parameters */ - DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; - rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); - } - - if( rc!= 0 ){ - rc = SQLITE_OK; - }else{ - pFile->lastErrno = osGetLastError(); - rc = SQLITE_BUSY; - } - - OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : - "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); - - return rc; -} - /* Forward references to VFS methods */ static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); static int winDelete(sqlite3_vfs *,const char*,int); /* -** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** Purge the winShmNodeList list of all entries with winShmNode.pWinShmList==0. ** ** This is not a VFS shared-memory method; it is a utility function called ** by VFS shared-memory methods. @@ -3846,7 +4248,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ - if( p->nRef==0 ){ + if( p->pWinShmList==0 ){ int i; if( p->mutex ){ sqlite3_mutex_free(p->mutex); } for(i=0; inRegion; i++){ @@ -3859,11 +4261,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); UNUSED_VARIABLE_VALUE(bRc); } - if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ - SimulateIOErrorBenign(1); - winClose((sqlite3_file *)&p->hFile); - SimulateIOErrorBenign(0); - } + winHandleClose(p->hSharedShm); if( deleteFlag ){ SimulateIOErrorBenign(1); sqlite3BeginBenignMalloc(); @@ -3881,42 +4279,196 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } /* -** The DMS lock has not yet been taken on shm file pShmNode. Attempt to -** take it now. Return SQLITE_OK if successful, or an SQLite error -** code otherwise. -** -** If the DMS cannot be locked because this is a readonly_shm=1 -** connection and no other process already holds a lock, return -** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +** The DMS lock has not yet been taken on the shm file associated with +** pShmNode. Take the lock. Truncate the *-shm file if required. +** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ -static int winLockSharedMemory(winShmNode *pShmNode){ - int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); +static int winLockSharedMemory(winShmNode *pShmNode, DWORD nMs){ + HANDLE h = pShmNode->hSharedShm; + int rc = SQLITE_OK; + assert( sqlite3_mutex_held(pShmNode->mutex) ); + rc = winHandleLockTimeout(h, WIN_SHM_DMS, 1, 1, 0); if( rc==SQLITE_OK ){ + /* We have an EXCLUSIVE lock on the DMS byte. This means that this + ** is the first process to open the file. Truncate it to zero bytes + ** in this case. */ if( pShmNode->isReadonly ){ - pShmNode->isUnlocked = 1; - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - return SQLITE_READONLY_CANTINIT; - }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winLockSharedMemory", pShmNode->zFilename); + rc = SQLITE_READONLY_CANTINIT; + }else{ + rc = winHandleTruncate(h, 0); } + + /* Release the EXCLUSIVE lock acquired above. */ + winUnlockFile(&h, WIN_SHM_DMS, 0, 1, 0); + }else if( (rc & 0xFF)==SQLITE_BUSY ){ + rc = SQLITE_OK; } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + /* Take a SHARED lock on the DMS byte. */ + rc = winHandleLockTimeout(h, WIN_SHM_DMS, 1, 0, nMs); + if( rc==SQLITE_OK ){ + pShmNode->isUnlocked = 0; + } } - return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); + return rc; } + /* -** Open the shared-memory area associated with database file pDbFd. +** This function is used to open a handle on a *-shm file. ** -** When opening a new shared-memory file, if no other instances of that -** file are currently open, in this process or in other processes, then -** the file must be truncated to zero length or have its header cleared. +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined at build time, then the file +** is opened with FILE_FLAG_OVERLAPPED specified. If not, it is not. +*/ +static int winHandleOpen( + const char *zUtf8, /* File to open */ + int *pbReadonly, /* IN/OUT: True for readonly handle */ + HANDLE *ph /* OUT: New HANDLE for file */ +){ + int rc = SQLITE_OK; + void *zConverted = 0; + int bReadonly = *pbReadonly; + HANDLE h = INVALID_HANDLE_VALUE; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + const DWORD flag_overlapped = FILE_FLAG_OVERLAPPED; +#else + const DWORD flag_overlapped = 0; +#endif + + /* Convert the filename to the system encoding. */ + zConverted = winConvertFromUtf8Filename(zUtf8); + if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8)); + rc = SQLITE_IOERR_NOMEM_BKPT; + goto winopenfile_out; + } + + /* Ensure the file we are trying to open is not actually a directory. */ + if( winIsDir(zConverted) ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8)); + rc = SQLITE_CANTOPEN_ISDIR; + goto winopenfile_out; + } + + /* TODO: platforms. + ** TODO: retry-on-ioerr. + */ + if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + memset(&extendedParameters, 0, sizeof(extendedParameters)); + extendedParameters.dwSize = sizeof(extendedParameters); + extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; + extendedParameters.dwFileFlags = flag_overlapped; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + h = osCreateFile2((LPCWSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + OPEN_ALWAYS, /* dwCreationDisposition */ + &extendedParameters + ); +#else + h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + NULL, /* lpSecurityAttributes */ + OPEN_ALWAYS, /* dwCreationDisposition */ + FILE_ATTRIBUTE_NORMAL|flag_overlapped, + NULL + ); +#endif + }else{ + /* Due to pre-processor directives earlier in this file, + ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ +#ifdef SQLITE_WIN32_HAS_ANSI + h = osCreateFileA((LPCSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + NULL, /* lpSecurityAttributes */ + OPEN_ALWAYS, /* dwCreationDisposition */ + FILE_ATTRIBUTE_NORMAL|flag_overlapped, + NULL + ); +#endif + } + + if( h==INVALID_HANDLE_VALUE ){ + if( bReadonly==0 ){ + bReadonly = 1; + rc = winHandleOpen(zUtf8, &bReadonly, &h); + }else{ + rc = SQLITE_CANTOPEN_BKPT; + } + } + + winopenfile_out: + sqlite3_free(zConverted); + *pbReadonly = bReadonly; + *ph = h; + return rc; +} + +/* +** Close pDbFd's connection to shared-memory. Delete the underlying +** *-shm file if deleteFlag is true. +*/ +static int winCloseSharedMemory(winFile *pDbFd, int deleteFlag){ + winShm *p; /* The connection to be closed */ + winShm **pp; /* Iterator for pShmNode->pWinShmList */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + if( p->hShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(p->hShm); + } + + winShmEnterMutex(); + pShmNode = p->pShmNode; + + /* Remove this connection from the winShmNode.pWinShmList list */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pWinShmList; *pp!=p; pp=&(*pp)->pWinShmNext){} + *pp = p->pWinShmNext; + sqlite3_mutex_leave(pShmNode->mutex); + + winShmPurge(pDbFd->pVfs, deleteFlag); + winShmLeaveMutex(); + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + return SQLITE_OK; +} + +/* +** testfixture builds may set this global variable to true via a +** Tcl interface. This forces the VFS to use the locking normally +** only used for UNC paths for all files. +*/ +#ifdef SQLITE_TEST +int sqlite3_win_test_unc_locking = 0; +#else +# define sqlite3_win_test_unc_locking 0 +#endif + +/* +** Return true if the string passed as the only argument is likely +** to be a UNC path. In other words, if it starts with "\\". +*/ +static int winIsUNCPath(const char *zFile){ + if( zFile[0]=='\\' && zFile[1]=='\\' ){ + return 1; + } + return sqlite3_win_test_unc_locking; +} + +/* +** Open the shared-memory area associated with database file pDbFd. */ static int winOpenSharedMemory(winFile *pDbFd){ struct winShm *p; /* The connection to be opened */ @@ -3928,98 +4480,93 @@ static int winOpenSharedMemory(winFile *pDbFd){ assert( pDbFd->pShm==0 ); /* Not previously opened */ /* Allocate space for the new sqlite3_shm object. Also speculatively - ** allocate space for a new winShmNode and filename. - */ + ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + (i64)nName + 17 ); if( pNew==0 ){ sqlite3_free(p); return SQLITE_IOERR_NOMEM_BKPT; } pNew->zFilename = (char*)&pNew[1]; + pNew->hSharedShm = INVALID_HANDLE_VALUE; + pNew->isUnlocked = 1; + pNew->bUseSharedLockHandle = winIsUNCPath(pDbFd->zPath); sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); /* Look to see if there is an existing winShmNode that can be used. - ** If no matching winShmNode currently exists, create a new one. - */ + ** If no matching winShmNode currently exists, then create a new one. */ winShmEnterMutex(); for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ /* TBD need to come up with better match here. Perhaps - ** use FILE_ID_BOTH_DIR_INFO Structure. - */ + ** use FILE_ID_BOTH_DIR_INFO Structure. */ if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; } - if( pShmNode ){ - sqlite3_free(pNew); - }else{ - int inFlags = SQLITE_OPEN_WAL; - int outFlags = 0; - + if( pShmNode==0 ){ pShmNode = pNew; - pNew = 0; - ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; - pShmNode->pNext = winShmNodeList; - winShmNodeList = pShmNode; + /* Allocate a mutex for this winShmNode object, if one is required. */ if( sqlite3GlobalConfig.bCoreMutex ){ pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; - } + if( pShmNode->mutex==0 ) rc = SQLITE_IOERR_NOMEM_BKPT; } - if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - }else{ - inFlags |= SQLITE_OPEN_READONLY; - } - rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, - (sqlite3_file*)&pShmNode->hFile, - inFlags, &outFlags); - if( rc!=SQLITE_OK ){ - rc = winLogError(rc, osGetLastError(), "winOpenShm", - pShmNode->zFilename); - goto shm_open_err; + /* Open a file-handle to use for mappings, and for the DMS lock. */ + if( rc==SQLITE_OK ){ + HANDLE h = INVALID_HANDLE_VALUE; + pShmNode->isReadonly = sqlite3_uri_boolean(pDbFd->zPath,"readonly_shm",0); + rc = winHandleOpen(pNew->zFilename, &pShmNode->isReadonly, &h); + pShmNode->hSharedShm = h; } - if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1; - rc = winLockSharedMemory(pShmNode); - if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + /* If successful, link the new winShmNode into the global list. If an + ** error occurred, free the object. */ + if( rc==SQLITE_OK ){ + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; + pNew = 0; + }else{ + sqlite3_mutex_free(pShmNode->mutex); + if( pShmNode->hSharedShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(pShmNode->hSharedShm); + } + } } - /* Make the new connection a child of the winShmNode */ - p->pShmNode = pShmNode; + /* If no error has occurred, link the winShm object to the winShmNode and + ** the winShm to pDbFd. */ + if( rc==SQLITE_OK ){ + sqlite3_mutex_enter(pShmNode->mutex); + p->pShmNode = pShmNode; + p->pWinShmNext = pShmNode->pWinShmList; + pShmNode->pWinShmList = p; #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) - p->id = pShmNode->nextShmId++; + p->id = pShmNode->nextShmId++; #endif - pShmNode->nRef++; - pDbFd->pShm = p; + pDbFd->pShm = p; + sqlite3_mutex_leave(pShmNode->mutex); + }else if( p ){ + sqlite3_free(p); + } + + assert( rc!=SQLITE_OK || pShmNode->isUnlocked==0 || pShmNode->nRegion==0 ); winShmLeaveMutex(); + sqlite3_free(pNew); - /* The reference count on pShmNode has already been incremented under - ** the cover of the winShmEnterMutex() mutex and the pointer from the - ** new (struct winShm) object to the pShmNode has been set. All that is - ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. - */ - sqlite3_mutex_enter(pShmNode->mutex); - p->pNext = pShmNode->pFirst; - pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); - return rc; + /* Open a file-handle on the *-shm file for this connection. This file-handle + ** is only used for locking. The mapping of the *-shm file is created using + ** the shared file handle in winShmNode.hSharedShm. */ + if( rc==SQLITE_OK && pShmNode->bUseSharedLockHandle==0 ){ + p->bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0); + rc = winHandleOpen(pShmNode->zFilename, &p->bReadonly, &p->hShm); + if( rc!=SQLITE_OK ){ + assert( p->hShm==INVALID_HANDLE_VALUE ); + winCloseSharedMemory(pDbFd, 0); + } + } - /* Jump here on any error */ -shm_open_err: - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ - sqlite3_free(p); - sqlite3_free(pNew); - winShmLeaveMutex(); return rc; } @@ -4031,38 +4578,7 @@ static int winShmUnmap( sqlite3_file *fd, /* Database holding shared memory */ int deleteFlag /* Delete after closing if true */ ){ - winFile *pDbFd; /* Database holding shared-memory */ - winShm *p; /* The connection to be closed */ - winShmNode *pShmNode; /* The underlying shared-memory file */ - winShm **pp; /* For looping over sibling connections */ - - pDbFd = (winFile*)fd; - p = pDbFd->pShm; - if( p==0 ) return SQLITE_OK; - pShmNode = p->pShmNode; - - /* Remove connection p from the set of connections associated - ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); - for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} - *pp = p->pNext; - - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); - - /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too */ - winShmEnterMutex(); - assert( pShmNode->nRef>0 ); - pShmNode->nRef--; - if( pShmNode->nRef==0 ){ - winShmPurge(pDbFd->pVfs, deleteFlag); - } - winShmLeaveMutex(); - - return SQLITE_OK; + return winCloseSharedMemory((winFile*)fd, deleteFlag); } /* @@ -4076,10 +4592,9 @@ static int winShmLock( ){ winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ winShm *p = pDbFd->pShm; /* The shared memory being locked */ - winShm *pX; /* For looping over all siblings */ winShmNode *pShmNode; int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ + u16 mask = (u16)((1U<<(ofst+n)) - (1U<pShmNode; @@ -4093,85 +4608,127 @@ static int winShmLock( || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - u16 allMask = 0; /* Mask of locks held by siblings */ - - /* See if any siblings hold this same lock */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( pX==p ) continue; - assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); - allMask |= pX->sharedMask; - } - - /* Unlock the system-level locks */ - if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } - - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - }else if( flags & SQLITE_SHM_SHARED ){ - u16 allShared = 0; /* Union of locks held by connections other than "p" */ + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Recovery lock (ofst==2). + ** 2. Checkpointer lock (ofst==1). + ** 3. Write lock (ofst==0). + ** 4. Read locks (ofst>=3 && ofstexclMask|p->sharedMask); + assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( + (ofst!=2 || lockMask==0) + && (ofst!=1 || lockMask==0 || lockMask==2) + && (ofst!=0 || lockMask<3) + && (ofst<3 || lockMask<(1<pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; + /* Check if there is any work to do. There are three cases: + ** + ** a) An unlock operation where there are locks to unlock, + ** b) An shared lock where the requested lock is not already held + ** c) An exclusive lock where the requested lock is not already held + ** + ** The SQLite core never requests an exclusive lock that it already holds. + ** This is assert()ed immediately below. */ + assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) + || 0==(p->exclMask & mask) + ); + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) + ){ + HANDLE h = p->hShm; + + if( flags & SQLITE_SHM_UNLOCK ){ + /* Case (a) - unlock. */ + + assert( (p->exclMask & p->sharedMask)==0 ); + assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); + assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); + + assert( !(flags & SQLITE_SHM_SHARED) || n==1 ); + if( pShmNode->bUseSharedLockHandle ){ + h = pShmNode->hSharedShm; + if( flags & SQLITE_SHM_SHARED ){ + winShm *pShm; + sqlite3_mutex_enter(pShmNode->mutex); + for(pShm=pShmNode->pWinShmList; pShm; pShm=pShm->pWinShmNext){ + if( pShm!=p && (pShm->sharedMask & mask) ){ + /* Another connection within this process is also holding this + ** SHARED lock. So do not actually release the OS lock. */ + h = INVALID_HANDLE_VALUE; + break; + } + } + sqlite3_mutex_leave(pShmNode->mutex); + } } - allShared |= pX->sharedMask; - } - /* Get shared locks at the system level, if necessary */ - if( rc==SQLITE_OK ){ - if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; + if( h!=INVALID_HANDLE_VALUE ){ + rc = winHandleUnlock(h, ofst+WIN_SHM_BASE, n); } - } - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; + /* If successful, also clear the bits in sharedMask/exclMask */ + if( rc==SQLITE_OK ){ + p->exclMask = (p->exclMask & ~mask); + p->sharedMask = (p->sharedMask & ~mask); + } + }else{ + int bExcl = ((flags & SQLITE_SHM_EXCLUSIVE) ? 1 : 0); + DWORD nMs = winFileBusyTimeout(pDbFd); + + if( pShmNode->bUseSharedLockHandle ){ + winShm *pShm; + h = pShmNode->hSharedShm; + sqlite3_mutex_enter(pShmNode->mutex); + for(pShm=pShmNode->pWinShmList; pShm; pShm=pShm->pWinShmNext){ + if( bExcl ){ + if( (pShm->sharedMask|pShm->exclMask) & mask ){ + rc = SQLITE_BUSY; + h = INVALID_HANDLE_VALUE; + } + }else{ + if( pShm->sharedMask & mask ){ + h = INVALID_HANDLE_VALUE; + }else if( pShm->exclMask & mask ){ + rc = SQLITE_BUSY; + h = INVALID_HANDLE_VALUE; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); } - } - /* Get the exclusive locks at the system level. Then if successful - ** also mark the local connection as being locked. - */ - if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( h!=INVALID_HANDLE_VALUE ){ + rc = winHandleLockTimeout(h, ofst+WIN_SHM_BASE, n, bExcl, nMs); + } if( rc==SQLITE_OK ){ - assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; + if( bExcl ){ + p->exclMask = (p->exclMask | mask); + }else{ + p->sharedMask = (p->sharedMask | mask); + } } } } - sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", - osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, - sqlite3ErrName(rc))); + + OSTRACE(( + "SHM-LOCK(%d,%d,%d) pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x," + " rc=%s\n", + ofst, n, flags, + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc)) + ); return rc; } @@ -4233,13 +4790,15 @@ static int winShmMap( sqlite3_mutex_enter(pShmNode->mutex); if( pShmNode->isUnlocked ){ - rc = winLockSharedMemory(pShmNode); + /* Take the DMS lock. */ + assert( pShmNode->nRegion==0 ); + rc = winLockSharedMemory(pShmNode, winFileBusyTimeout(pDbFd)); if( rc!=SQLITE_OK ) goto shmpage_out; - pShmNode->isUnlocked = 0; } - assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ + HANDLE hShared = pShmNode->hSharedShm; struct ShmRegion *apNew; /* New aRegion[] array */ int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ sqlite3_int64 sz; /* Current size of wal-index file */ @@ -4250,10 +4809,9 @@ static int winShmMap( ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ - rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + rc = winHandleSize(hShared, &sz); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), - "winShmMap1", pDbFd->zPath); + rc = winLogError(rc, osGetLastError(), "winShmMap1", pDbFd->zPath); goto shmpage_out; } @@ -4262,19 +4820,17 @@ static int winShmMap( ** zero, exit early. *pp will be set to NULL and SQLITE_OK returned. ** ** Alternatively, if isWrite is non-zero, use ftruncate() to allocate - ** the requested memory region. - */ + ** the requested memory region. */ if( !isWrite ) goto shmpage_out; - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte); + rc = winHandleTruncate(hShared, nByte); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), - "winShmMap2", pDbFd->zPath); + rc = winLogError(rc, osGetLastError(), "winShmMap2", pDbFd->zPath); goto shmpage_out; } } /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc64( + apNew = (struct ShmRegion*)sqlite3_realloc64( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ @@ -4293,18 +4849,13 @@ static int winShmMap( void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, protect, nByte, NULL - ); + hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, protect, 0, nByte, NULL - ); + hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA - hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, protect, 0, nByte, NULL - ); + hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); #endif + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); @@ -4347,7 +4898,9 @@ static int winShmMap( }else{ *pp = 0; } - if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + if( pShmNode->isReadonly && rc==SQLITE_OK ){ + rc = SQLITE_READONLY; + } sqlite3_mutex_leave(pShmNode->mutex); return rc; } @@ -4521,6 +5074,11 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ){ @@ -4529,7 +5087,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return rc; } } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; @@ -4662,47 +5220,6 @@ static winVfsAppData winNolockAppData = { ** sqlite3_vfs object. */ -#if defined(__CYGWIN__) -/* -** Convert a filename from whatever the underlying operating system -** supports for filenames into UTF-8. Space to hold the result is -** obtained from malloc and must be freed by the calling function. -*/ -static char *winConvertToUtf8Filename(const void *zFilename){ - char *zConverted = 0; - if( osIsNT() ){ - zConverted = winUnicodeToUtf8(zFilename); - } -#ifdef SQLITE_WIN32_HAS_ANSI - else{ - zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} -#endif - -/* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from malloc and must be freed by the calling -** function. -*/ -static void *winConvertFromUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( osIsNT() ){ - zConverted = winUtf8ToUnicode(zFilename); - } -#ifdef SQLITE_WIN32_HAS_ANSI - else{ - zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} - /* ** This function returns non-zero if the specified UTF-8 string buffer ** ends with a directory separator character or one was successfully @@ -4715,7 +5232,14 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ if( winIsDirSep(zBuf[nLen-1]) ){ return 1; }else if( nLen+1mxPathname; nBuf = nMax + 2; + nMax = pVfs->mxPathname; + nBuf = 2 + (i64)nMax; zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); @@ -4791,7 +5317,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #if defined(__CYGWIN__) - else{ + else if( osGetenv!=NULL ){ static const char *azDirs[] = { 0, /* getenv("SQLITE_TMPDIR") */ 0, /* getenv("TMPDIR") */ @@ -4807,11 +5333,11 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ unsigned int i; const char *zDir = 0; - if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); - if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - if( !azDirs[2] ) azDirs[2] = getenv("TMP"); - if( !azDirs[3] ) azDirs[3] = getenv("TEMP"); - if( !azDirs[4] ) azDirs[4] = getenv("USERPROFILE"); + if( !azDirs[0] ) azDirs[0] = osGetenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = osGetenv("TMPDIR"); + if( !azDirs[2] ) azDirs[2] = osGetenv("TMP"); + if( !azDirs[3] ) azDirs[3] = osGetenv("TEMP"); + if( !azDirs[4] ) azDirs[4] = osGetenv("USERPROFILE"); for(i=0; i>= 8; zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } zBuf[j] = 0; @@ -4991,7 +5488,7 @@ static int winIsDir(const void *zConverted){ return 0; /* Invalid name? */ } attr = sAttrData.dwFileAttributes; -#if SQLITE_OS_WINCE==0 +#if SQLITE_OS_WINCE==0 && defined(SQLITE_WIN32_HAS_ANSI) }else{ attr = osGetFileAttributesA((char*)zConverted); #endif @@ -5007,6 +5504,12 @@ static int winAccess( int *pResOut /* OUT: Result */ ); +/* +** The Windows version of xAccess() accepts an extra bit in the flags +** parameter that prevents an anti-virus retry loop. +*/ +#define NORETRY 0x4000 + /* ** Open a file. */ @@ -5031,6 +5534,7 @@ static int winOpen( void *zConverted; /* Filename in OS encoding */ const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ int cnt = 0; + int isRO = 0; /* file is known to be accessible readonly */ /* If argument zPath is a NULL pointer, this function is required to open ** a temporary file. Use this buffer to store the file name in. @@ -5039,7 +5543,7 @@ static int winOpen( int rc = SQLITE_OK; /* Function Return Code */ #if !defined(NDEBUG) || SQLITE_OS_WINCE - int eType = flags&0xFFFFFF00; /* Type of file to open */ + int eType = flags&0x0FFF00; /* Type of file to open */ #endif int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); @@ -5195,9 +5699,9 @@ static int winOpen( &extendedParameters); if( h!=INVALID_HANDLE_VALUE ) break; if( isReadWrite ){ - int rc2, isRO = 0; + int rc2; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -5212,9 +5716,9 @@ static int winOpen( NULL); if( h!=INVALID_HANDLE_VALUE ) break; if( isReadWrite ){ - int rc2, isRO = 0; + int rc2; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -5232,9 +5736,9 @@ static int winOpen( NULL); if( h!=INVALID_HANDLE_VALUE ) break; if( isReadWrite ){ - int rc2, isRO = 0; + int rc2; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -5249,7 +5753,7 @@ static int winOpen( if( h==INVALID_HANDLE_VALUE ){ sqlite3_free(zConverted); sqlite3_free(zTmpname); - if( isReadWrite && !isExclusive ){ + if( isReadWrite && isRO && !isExclusive ){ return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY) & ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), @@ -5451,8 +5955,14 @@ static int winAccess( int rc = 0; DWORD lastErrno = 0; void *zConverted; + int noRetry = 0; /* Do not use winRetryIoerr() */ UNUSED_PARAMETER(pVfs); + if( (flags & NORETRY)!=0 ){ + noRetry = 1; + flags &= ~NORETRY; + } + SimulateIOError( return SQLITE_IOERR_ACCESS; ); OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); @@ -5475,7 +5985,10 @@ static int winAccess( memset(&sAttrData, 0, sizeof(sAttrData)); while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, GetFileExInfoStandard, - &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + &sAttrData)) + && !noRetry + && winRetryIoerr(&cnt, &lastErrno) + ){ /* Loop until true */} if( rc ){ /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file ** as if it does not exist. @@ -5543,6 +6056,7 @@ static BOOL winIsDriveLetterAndColon( return ( sqlite3Isalpha(zPathname[0]) && zPathname[1]==':' ); } +#ifdef _WIN32 /* ** Returns non-zero if the specified path name should be used verbatim. If ** non-zero is returned from this function, the calling function must simply @@ -5579,6 +6093,70 @@ static BOOL winIsVerbatimPathname( */ return FALSE; } +#endif /* _WIN32 */ + +#ifdef __CYGWIN__ +/* +** Simplify a filename into its canonical form +** by making the following changes: +** +** * convert any '/' to '\' (win32) or reverse (Cygwin) +** * removing any trailing and duplicate / (except for UNC paths) +** * convert /./ into just / +** +** Changes are made in-place. Return the new name length. +** +** The original filename is in z[0..]. If the path is shortened, +** no-longer used bytes will be written by '\0'. +*/ +static void winSimplifyName(char *z){ + int i, j; + for(i=j=0; z[i]; ++i){ + if( winIsDirSep(z[i]) ){ +#if !defined(SQLITE_TEST) + /* Some test-cases assume that "./foo" and "foo" are different */ + if( z[i+1]=='.' && winIsDirSep(z[i+2]) ){ + ++i; + continue; + } +#endif + if( !z[i+1] || (winIsDirSep(z[i+1]) && (i!=0)) ){ + continue; + } + z[j++] = osGetenv?'/':'\\'; + }else{ + z[j++] = z[i]; + } + } + while(jnOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + return SQLITE_OK; +} +#endif /* __CYGWIN__ */ /* ** Turn a relative pathname into a full pathname. Write the full @@ -5591,8 +6169,8 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + int nByte; void *zConverted; char *zOut; #endif @@ -5605,64 +6183,82 @@ static int winFullPathnameNoMutex( zRelative++; } -#if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); - UNUSED_PARAMETER(nFull); - assert( nFull>=pVfs->mxPathname ); - if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ - /* - ** NOTE: We are dealing with a relative path name and the data - ** directory has been set. Therefore, use it as the basis - ** for converting the relative path name to an absolute - ** one by prepending the data directory and a slash. - */ - char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); - if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - if( cygwin_conv_path( - (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | - CCP_RELATIVE, zRelative, zOut, pVfs->mxPathname+1)<0 ){ - sqlite3_free(zOut); - return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, - "winFullPathname1", zRelative); - }else{ - char *zUtf8 = winConvertToUtf8Filename(zOut); - if( !zUtf8 ){ - sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; - } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", - sqlite3_data_directory, winGetDirSep(), zUtf8); - sqlite3_free(zUtf8); - sqlite3_free(zOut); - } - }else{ - char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); - if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - if( cygwin_conv_path( - (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), - zRelative, zOut, pVfs->mxPathname+1)<0 ){ - sqlite3_free(zOut); - return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, - "winFullPathname2", zRelative); - }else{ - char *zUtf8 = winConvertToUtf8Filename(zOut); - if( !zUtf8 ){ - sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; - } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); - sqlite3_free(zUtf8); - sqlite3_free(zOut); + +#ifdef __CYGWIN__ + if( osGetcwd ){ + zFull[nFull-1] = '\0'; + if( !winIsDriveLetterAndColon(zRelative) || !winIsDirSep(zRelative[2]) ){ + int rc = SQLITE_OK; + int nLink = 1; /* Number of symbolic links followed so far */ + const char *zIn = zRelative; /* Input path for each iteration of loop */ + char *zDel = 0; + struct stat buf; + + UNUSED_PARAMETER(pVfs); + + do { + /* Call lstat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + if( osLstat && osReadlink ) { + if( osLstat(zIn, &buf)!=0 ){ + int myErrno = osErrno; + if( myErrno!=ENOENT ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, (DWORD)myErrno, "lstat", zIn); + } + }else{ + bLink = ((buf.st_mode & 0170000) == 0120000); + } + + if( bLink ){ + if( zDel==0 ){ + zDel = sqlite3MallocZero(nFull); + if( zDel==0 ) rc = SQLITE_NOMEM; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nFull-1); + if( nByte ==(DWORD)-1 ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, (DWORD)osErrno, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nFull ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + } + + assert( rc!=SQLITE_OK || zIn!=zFull || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zFull ){ + rc = mkFullPathname(zIn, zFull, nFull); + } + if( bLink==0 ) break; + zIn = zFull; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + winSimplifyName(zFull); + return rc; } } - return SQLITE_OK; -#endif +#endif /* __CYGWIN__ */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && !defined(__CYGWIN__) +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -5681,7 +6277,8 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -5699,6 +6296,7 @@ static int winFullPathnameNoMutex( sqlite3_data_directory, winGetDirSep(), zRelative); return SQLITE_OK; } +#endif zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM_BKPT; @@ -5737,13 +6335,12 @@ static int winFullPathnameNoMutex( return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), "winFullPathname3", zRelative); } - nByte += 3; - zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) + 3*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM_BKPT; } - nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameA((char*)zConverted, nByte+3, zTemp, 0); if( nByte==0 ){ sqlite3_free(zConverted); sqlite3_free(zTemp); @@ -5756,7 +6353,26 @@ static int winFullPathnameNoMutex( } #endif if( zOut ){ +#ifdef __CYGWIN__ + if( memcmp(zOut, "\\\\?\\", 4) ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); + }else if( memcmp(zOut+4, "UNC\\", 4) ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut+4); + }else{ + char *p = zOut+6; + *p = '\\'; + if( osGetcwd ){ + /* On Cygwin, UNC paths use forward slashes */ + while( *p ){ + if( *p=='\\' ) *p = '/'; + ++p; + } + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut+6); + } +#else sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); +#endif /* __CYGWIN__ */ sqlite3_free(zOut); return SQLITE_OK; }else{ @@ -5786,25 +6402,8 @@ static int winFullPathname( */ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; -#if defined(__CYGWIN__) - int nFull = pVfs->mxPathname+1; - char *zFull = sqlite3MallocZero( nFull ); - void *zConverted = 0; - if( zFull==0 ){ - OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); - return 0; - } - if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ - sqlite3_free(zFull); - OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); - return 0; - } - zConverted = winConvertFromUtf8Filename(zFull); - sqlite3_free(zFull); -#else void *zConverted = winConvertFromUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); -#endif if( zConverted==0 ){ OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); return 0; @@ -6153,7 +6752,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==80 ); + assert( ArraySize(aSyscall)==89 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); diff --git a/src/os_win.h b/src/os_win.h index 27714ed079..a0845f0038 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -22,6 +22,8 @@ #ifdef __CYGWIN__ # include +# include /* amalgamator: dontcache */ +# include /* amalgamator: dontcache */ # include /* amalgamator: dontcache */ #endif diff --git a/src/pager.c b/src/pager.c index 0a5ddd1c5e..49318fc712 100644 --- a/src/pager.c +++ b/src/pager.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** This is the implementation of the page cache subsystem or "pager". -** +** ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file @@ -36,60 +36,60 @@ ** ** Definition: A page of the database file is said to be "overwriteable" if ** one or more of the following are true about the page: -** +** ** (a) The original content of the page as it was at the beginning of ** the transaction has been written into the rollback journal and ** synced. -** +** ** (b) The page was a freelist leaf page at the start of the transaction. -** +** ** (c) The page number is greater than the largest page that existed in ** the database file at the start of the transaction. -** +** ** (1) A page of the database file is never overwritten unless one of the ** following are true: -** +** ** (a) The page and all other pages on the same sector are overwriteable. -** +** ** (b) The atomic page write optimization is enabled, and the entire ** transaction other than the update of the transaction sequence ** number consists of a single page change. -** +** ** (2) The content of a page written into the rollback journal exactly matches ** both the content in the database when the rollback journal was written ** and the content in the database at the beginning of the current ** transaction. -** +** ** (3) Writes to the database file are an integer multiple of the page size ** in length and are aligned on a page boundary. -** +** ** (4) Reads from the database file are either aligned on a page boundary and ** an integer multiple of the page size in length or are taken from the ** first 100 bytes of the database file. -** +** ** (5) All writes to the database file are synced prior to the rollback journal ** being deleted, truncated, or zeroed. -** +** ** (6) If a super-journal file is used, then all writes to the database file ** are synced prior to the super-journal being deleted. -** +** ** Definition: Two databases (or the same database at two points it time) ** are said to be "logically equivalent" if they give the same answer to ** all queries. Note in particular the content of freelist leaf ** pages can be changed arbitrarily without affecting the logical equivalence ** of the database. -** +** ** (7) At any time, if any subset, including the empty set and the total set, -** of the unsynced changes to a rollback journal are removed and the +** of the unsynced changes to a rollback journal are removed and the ** journal is rolled back, the resulting database file will be logically ** equivalent to the database file at the beginning of the transaction. -** +** ** (8) When a transaction is rolled back, the xTruncate method of the VFS ** is called to restore the database file to the same size it was at ** the beginning of the transaction. (In some VFSes, the xTruncate ** method is a no-op, but that does not change the fact the SQLite will ** invoke it.) -** +** ** (9) Whenever the database file is modified, at least one bit in the range ** of bytes from 24 through 39 inclusive will be changed prior to releasing ** the EXCLUSIVE lock, thus signaling other connections on the same @@ -122,7 +122,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ /* ** The following two macros are used within the PAGERTRACE() macros above -** to print out file-descriptors. +** to print out file-descriptors. ** ** PAGERID() takes a pointer to a Pager struct as its argument. The ** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file @@ -143,7 +143,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** | | | ** | V | ** |<-------WRITER_LOCKED------> ERROR -** | | ^ +** | | ^ ** | V | ** |<------WRITER_CACHEMOD-------->| ** | | | @@ -155,7 +155,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** ** List of state transitions and the C [function] that performs each: -** +** ** OPEN -> READER [sqlite3PagerSharedLock] ** READER -> OPEN [pager_unlock] ** @@ -167,7 +167,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** WRITER_*** -> ERROR [pager_error] ** ERROR -> OPEN [pager_unlock] -** +** ** ** OPEN: ** @@ -181,9 +181,9 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** READER: ** -** In this state all the requirements for reading the database in +** In this state all the requirements for reading the database in ** rollback (non-WAL) mode are met. Unless the pager is (or recently -** was) in exclusive-locking mode, a user-level read transaction is +** was) in exclusive-locking mode, a user-level read transaction is ** open. The database size is known in this state. ** ** A connection running with locking_mode=normal enters this state when @@ -193,28 +193,28 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** this state even after the read-transaction is closed. The only way ** a locking_mode=exclusive connection can transition from READER to OPEN ** is via the ERROR state (see below). -** +** ** * A read transaction may be active (but a write-transaction cannot). ** * A SHARED or greater lock is held on the database file. -** * The dbSize variable may be trusted (even if a user-level read +** * The dbSize variable may be trusted (even if a user-level read ** transaction is not active). The dbOrigSize and dbFileSize variables ** may not be trusted at this point. ** * If the database is a WAL database, then the WAL connection is open. -** * Even if a read-transaction is not open, it is guaranteed that +** * Even if a read-transaction is not open, it is guaranteed that ** there is no hot-journal in the file-system. ** ** WRITER_LOCKED: ** ** The pager moves to this state from READER when a write-transaction -** is first opened on the database. In WRITER_LOCKED state, all locks -** required to start a write-transaction are held, but no actual +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual ** modifications to the cache or database have taken place. ** -** In rollback mode, a RESERVED or (if the transaction was opened with +** In rollback mode, a RESERVED or (if the transaction was opened with ** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when -** moving to this state, but the journal file is not written to or opened -** to in this state. If the transaction is committed or rolled back while -** in WRITER_LOCKED state, all that is required is to unlock the database +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database ** file. ** ** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. @@ -222,7 +222,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** is made to obtain an EXCLUSIVE lock on the database file. ** ** * A write transaction is active. -** * If the connection is open in rollback-mode, a RESERVED or greater +** * If the connection is open in rollback-mode, a RESERVED or greater ** lock is held on the database file. ** * If the connection is open in WAL-mode, a WAL write transaction ** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully @@ -241,7 +241,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** * A write transaction is active. ** * A RESERVED or greater lock is held on the database file. -** * The journal file is open and the first header has been written +** * The journal file is open and the first header has been written ** to it, but the header has not been synced to disk. ** * The contents of the page cache have been modified. ** @@ -254,7 +254,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. -** * The journal file is open and the first header has been written +** * The journal file is open and the first header has been written ** and synced to disk. ** * The contents of the page cache have been modified (and possibly ** written to disk). @@ -266,8 +266,8 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD ** state after the entire transaction has been successfully written into the ** database file. In this state the transaction may be committed simply -** by finalizing the journal file. Once in WRITER_FINISHED state, it is -** not possible to modify the database further. At this point, the upper +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper ** layer must either commit or rollback the transaction. ** ** * A write transaction is active. @@ -275,19 +275,19 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * All writing and syncing of journal and database data has finished. ** If no error occurred, all that remains is to finalize the journal to ** commit the transaction. If an error did occur, the caller will need -** to rollback the transaction. +** to rollback the transaction. ** ** ERROR: ** ** The ERROR state is entered when an IO or disk-full error (including -** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it -** difficult to be sure that the in-memory pager state (cache contents, +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, ** db size etc.) are consistent with the contents of the file-system. ** ** Temporary pager files may enter the ERROR state, but in-memory pagers ** cannot. ** -** For example, if an IO error occurs while performing a rollback, +** For example, if an IO error occurs while performing a rollback, ** the contents of the page-cache may be left in an inconsistent state. ** At this point it would be dangerous to change back to READER state ** (as usually happens after a rollback). Any subsequent readers might @@ -297,13 +297,13 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** instead of READER following such an error. ** ** Once it has entered the ERROR state, any attempt to use the pager -** to read or write data returns an error. Eventually, once all +** to read or write data returns an error. Eventually, once all ** outstanding transactions have been abandoned, the pager is able to -** transition back to OPEN state, discarding the contents of the +** transition back to OPEN state, discarding the contents of the ** page-cache and any other in-memory state at the same time. Everything -** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** is reloaded from disk (and, if necessary, hot-journal rollback performed) ** when a read-transaction is next opened on the pager (transitioning -** the pager into READER state). At that point the system has recovered +** the pager into READER state). At that point the system has recovered ** from the error. ** ** Specifically, the pager jumps into the ERROR state if: @@ -319,21 +319,21 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** memory. ** ** In other cases, the error is returned to the b-tree layer. The b-tree -** layer then attempts a rollback operation. If the error condition +** layer then attempts a rollback operation. If the error condition ** persists, the pager enters the ERROR state via condition (1) above. ** ** Condition (3) is necessary because it can be triggered by a read-only ** statement executed within a transaction. In this case, if the error ** code were simply returned to the user, the b-tree layer would not ** automatically attempt a rollback, as it assumes that an error in a -** read-only statement cannot leave the pager in an internally inconsistent +** read-only statement cannot leave the pager in an internally inconsistent ** state. ** ** * The Pager.errCode variable is set to something other than SQLITE_OK. ** * There are one or more outstanding references to pages (after the ** last reference is dropped the pager should move back to OPEN state). ** * The pager is not an in-memory pager. -** +** ** ** Notes: ** @@ -343,7 +343,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** * Normally, a connection open in exclusive mode is never in PAGER_OPEN ** state. There are two exceptions: immediately after exclusive-mode has -** been turned on (and before any read or write transactions are +** been turned on (and before any read or write transactions are ** executed), and when the pager is leaving the "error state". ** ** * See also: assert_pager_state(). @@ -357,7 +357,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define PAGER_ERROR 6 /* -** The Pager.eLock variable is almost always set to one of the +** The Pager.eLock variable is almost always set to one of the ** following locking-states, according to the lock currently held on ** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. ** This variable is kept up to date as locks are taken and released by @@ -372,20 +372,20 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** to a less exclusive (lower) value than the lock that is actually held ** at the system level, but it is never set to a more exclusive value. ** -** This is usually safe. If an xUnlock fails or appears to fail, there may +** This is usually safe. If an xUnlock fails or appears to fail, there may ** be a few redundant xLock() calls or a lock may be held for longer than ** required, but nothing really goes wrong. ** ** The exception is when the database file is unlocked as the pager moves -** from ERROR to OPEN state. At this point there may be a hot-journal file +** from ERROR to OPEN state. At this point there may be a hot-journal file ** in the file-system that needs to be rolled back (as part of an OPEN->SHARED ** transition, by the same pager or any other). If the call to xUnlock() ** fails at this point and the pager is left holding an EXCLUSIVE lock, this ** can confuse the call to xCheckReservedLock() call made later as part ** of hot-journal detection. ** -** xCheckReservedLock() is defined as returning true "if there is a RESERVED -** lock held by this process or any others". So xCheckReservedLock may +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may ** return true because the caller itself is holding an EXCLUSIVE lock (but ** doesn't know it because of a previous error in xUnlock). If this happens ** a hot-journal may be mistaken for a journal being created by an active @@ -396,12 +396,12 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It ** is only changed back to a real locking state after a successful call ** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition -** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK ** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE ** lock on the database file before attempting to roll it back. See function ** PagerSharedLock() for more detail. ** -** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in ** PAGER_OPEN state. */ #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) @@ -423,7 +423,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ /* END SQLCIPHER */ /* -** The maximum allowed sector size. 64KiB. If the xSectorsize() method +** The maximum allowed sector size. 64KiB. If the xSectorsize() method ** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. ** This could conceivably cause corruption following a power failure on ** such a system. This is currently an undocumented limit. @@ -439,7 +439,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** When a savepoint is created, the PagerSavepoint.iHdrOffset field is ** set to 0. If a journal-header is written into the main journal while -** the savepoint is active, then iHdrOffset is set to the byte offset +** the savepoint is active, then iHdrOffset is set to the byte offset ** immediately following the last journal record written into the main ** journal before the journal-header. This is required during savepoint ** rollback (see pagerPlaybackSavepoint()). @@ -490,34 +490,34 @@ struct PagerSavepoint { ** ** changeCountDone ** -** This boolean variable is used to make sure that the change-counter -** (the 4-byte header field at byte offset 24 of the database file) is -** not updated more often than necessary. +** This boolean variable is used to make sure that the change-counter +** (the 4-byte header field at byte offset 24 of the database file) is +** not updated more often than necessary. ** -** It is set to true when the change-counter field is updated, which +** It is set to true when the change-counter field is updated, which ** can only happen if an exclusive lock is held on the database file. -** It is cleared (set to false) whenever an exclusive lock is +** It is cleared (set to false) whenever an exclusive lock is ** relinquished on the database file. Each time a transaction is committed, ** The changeCountDone flag is inspected. If it is true, the work of ** updating the change-counter is omitted for the current transaction. ** -** This mechanism means that when running in exclusive mode, a connection +** This mechanism means that when running in exclusive mode, a connection ** need only update the change-counter once, for the first transaction ** committed. ** ** setSuper ** ** When PagerCommitPhaseOne() is called to commit a transaction, it may -** (or may not) specify a super-journal name to be written into the +** (or may not) specify a super-journal name to be written into the ** journal file before it is synced to disk. ** -** Whether or not a journal file contains a super-journal pointer affects -** the way in which the journal file is finalized after the transaction is +** Whether or not a journal file contains a super-journal pointer affects +** the way in which the journal file is finalized after the transaction is ** committed or rolled back when running in "journal_mode=PERSIST" mode. ** If a journal file does not contain a super-journal pointer, it is ** finalized by overwriting the first journal header with zeroes. If -** it does contain a super-journal pointer the journal file is finalized -** by truncating it to zero bytes, just as if the connection were +** it does contain a super-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were ** running in "journal_mode=truncate" mode. ** ** Journal files that contain super-journal pointers cannot be finalized @@ -543,12 +543,12 @@ struct PagerSavepoint { ** to allocate a new page to prevent the journal file from being written ** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF ** case is a user preference. -** +** ** If the SPILLFLAG_NOSYNC bit is set, writing to the database from ** pagerStress() is permitted, but syncing the journal file is not. ** This flag is set by sqlite3PagerWrite() when the file-system sector-size ** is larger than the database page-size in order to prevent a journal sync -** from happening in between the journalling of two pages on the same sector. +** from happening in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -556,16 +556,16 @@ struct PagerSavepoint { ** is opened as an in-memory journal file. If false, then in-memory ** sub-journals are only used for in-memory pager files. ** -** This variable is updated by the upper layer each time a new +** This variable is updated by the upper layer each time a new ** write-transaction is opened. ** ** dbSize, dbOrigSize, dbFileSize ** ** Variable dbSize is set to the number of pages in the database file. ** It is valid in PAGER_READER and higher states (all states except for -** OPEN and ERROR). +** OPEN and ERROR). ** -** dbSize is set based on the size of the database file, which may be +** dbSize is set based on the size of the database file, which may be ** larger than the size of the database (the value stored at offset ** 28 of the database header by the btree). If the size of the file ** is not an integer multiple of the page-size, the value stored in @@ -576,10 +576,10 @@ struct PagerSavepoint { ** ** During a write-transaction, if pages with page-numbers greater than ** dbSize are modified in the cache, dbSize is updated accordingly. -** Similarly, if the database is truncated using PagerTruncateImage(), +** Similarly, if the database is truncated using PagerTruncateImage(), ** dbSize is updated. ** -** Variables dbOrigSize and dbFileSize are valid in states +** Variables dbOrigSize and dbFileSize are valid in states ** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize ** variable at the start of the transaction. It is used during rollback, ** and to determine whether or not pages need to be journalled before @@ -588,12 +588,12 @@ struct PagerSavepoint { ** Throughout a write-transaction, dbFileSize contains the size of ** the file on disk in pages. It is set to a copy of dbSize when the ** write-transaction is first opened, and updated when VFS calls are made -** to write or truncate the database file on disk. +** to write or truncate the database file on disk. ** -** The only reason the dbFileSize variable is required is to suppress -** unnecessary calls to xTruncate() after committing a transaction. If, -** when a transaction is committed, the dbFileSize variable indicates -** that the database file is larger than the database image (Pager.dbSize), +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), ** pager_truncate() is called. The pager_truncate() call uses xFilesize() ** to measure the database file on disk, and then truncates it if required. ** dbFileSize is not used when rolling back a transaction. In this case @@ -604,20 +604,20 @@ struct PagerSavepoint { ** dbHintSize ** ** The dbHintSize variable is used to limit the number of calls made to -** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** the VFS xFileControl(FCNTL_SIZE_HINT) method. ** ** dbHintSize is set to a copy of the dbSize variable when a ** write-transaction is opened (at the same time as dbFileSize and ** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, ** dbHintSize is increased to the number of pages that correspond to the -** size-hint passed to the method call. See pager_write_pagelist() for +** size-hint passed to the method call. See pager_write_pagelist() for ** details. ** ** errCode ** ** The Pager.errCode variable is only ever used in PAGER_ERROR state. It -** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode -** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. ** ** syncFlags, walSyncFlags @@ -704,7 +704,7 @@ struct Pager { char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int aStat[4]; /* Total cache hits, misses, writes, spills */ + u32 aStat[4]; /* Total cache hits, misses, writes, spills */ #ifdef SQLITE_TEST int nRead; /* Database pages read */ #endif @@ -724,11 +724,14 @@ struct Pager { Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3 *dbWal; +#endif }; /* ** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains -** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS ** or CACHE_WRITE to sqlite3_db_status(). */ #define PAGER_STAT_HIT 0 @@ -786,7 +789,7 @@ static const unsigned char aJournalMagic[] = { #define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) /* -** The journal header size for this pager. This is usually the same +** The journal header size for this pager. This is usually the same ** size as a single disk sector. See also setSectorSize(). */ #define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) @@ -813,32 +816,21 @@ static const unsigned char aJournalMagic[] = { # define USEFETCH(x) 0 #endif -/* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. -** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... -*/ -#define isOpen(pFd) ((pFd)->pMethods!=0) - #ifdef SQLITE_DIRECT_OVERFLOW_READ /* ** Return true if page pgno can be read directly from the database file ** by the b-tree layer. This is the case if: ** -** * the database file is open, -** * there are no dirty pages in the cache, and -** * the desired page is not currently in the wal file. +** (1) the database file is open +** (2) the VFS for the database is able to do unaligned sub-page reads +** (3) there are no dirty pages in the cache, and +** (4) the desired page is not currently in the wal file. */ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ - if( pPager->fd->pMethods==0 ) return 0; - if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; + assert( pPager!=0 ); + assert( pPager->fd!=0 ); + if( pPager->fd->pMethods==0 ) return 0; /* Case (1) */ + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; /* Failed (3) */ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC if( pPager->xCodec!=0 ) return 0; @@ -847,11 +839,15 @@ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ #ifndef SQLITE_OMIT_WAL if( pPager->pWal ){ u32 iRead = 0; - int rc; - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); - return (rc==SQLITE_OK && iRead==0); + (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + if( iRead ) return 0; /* Case (4) */ } #endif + assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); + if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) + & SQLITE_IOCAP_SUBPAGE_READ)==0 ){ + return 0; /* Case (2) */ + } return 1; } #endif @@ -866,7 +862,7 @@ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ # define pagerBeginReadTransaction(z) SQLITE_OK #endif -#ifndef NDEBUG +#ifndef NDEBUG /* ** Usage: ** @@ -895,25 +891,25 @@ static int assert_pager_state(Pager *p){ assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); assert( p->tempFile==0 || pPager->changeCountDone ); - /* If the useJournal flag is clear, the journal-mode must be "OFF". + /* If the useJournal flag is clear, the journal-mode must be "OFF". ** And if the journal-mode is "OFF", the journal file must not be open. */ assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); - /* Check that MEMDB implies noSync. And an in-memory journal. Since - ** this means an in-memory pager performs no IO at all, it cannot encounter - ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing - ** a journal file. (although the in-memory journal implementation may - ** return SQLITE_IOERR_NOMEM while the journal file is being written). It - ** is therefore not possible for an in-memory pager to enter the ERROR + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR ** state. */ if( MEMDB ){ assert( !isOpen(p->fd) ); assert( p->noSync ); - assert( p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_MEMORY + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY ); assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); assert( pagerUseWal(p)==0 ); @@ -960,9 +956,9 @@ static int assert_pager_state(Pager *p){ ** to journal_mode=wal. */ assert( p->eLock>=RESERVED_LOCK ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL ); } assert( pPager->dbOrigSize==pPager->dbFileSize ); @@ -974,9 +970,9 @@ static int assert_pager_state(Pager *p){ assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); assert( p->eLock>=EXCLUSIVE_LOCK ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); assert( pPager->dbOrigSize<=pPager->dbHintSize ); @@ -986,9 +982,9 @@ static int assert_pager_state(Pager *p){ assert( p->eLock==EXCLUSIVE_LOCK ); assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); break; @@ -1007,7 +1003,7 @@ static int assert_pager_state(Pager *p){ } #endif /* ifndef NDEBUG */ -#ifdef SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Return a pointer to a human readable string in a static buffer ** containing the state of the Pager object passed as an argument. This @@ -1164,7 +1160,7 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ ** succeeds, set the Pager.eLock variable to match the (attempted) new lock. ** ** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is -** called, do not modify it. See the comment above the #define of +** called, do not modify it. See the comment above the #define of ** UNKNOWN_LOCK for an explanation of this. */ static int pagerUnlockDb(Pager *pPager, int eLock){ @@ -1188,11 +1184,11 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ /* ** Lock the database file to level eLock, which must be either SHARED_LOCK, ** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the -** Pager.eLock variable to the new locking state. +** Pager.eLock variable to the new locking state. ** -** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is -** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. -** See the comment above the #define of UNKNOWN_LOCK for an explanation +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation ** of this. */ static int pagerLockDb(Pager *pPager, int eLock){ @@ -1219,7 +1215,7 @@ static int pagerLockDb(Pager *pPager, int eLock){ ** (b) the value returned by OsSectorSize() is less than or equal ** to the page size. ** -** If it can be used, then the value returned is the size of the journal +** If it can be used, then the value returned is the size of the journal ** file when it contains rollback data for exactly one page. ** ** The atomic-batch-write optimization can be used if OsDeviceCharacteristics() @@ -1310,8 +1306,8 @@ static void checkPage(PgHdr *pPg){ /* ** When this is called the journal file for pager pPager must be open. -** This function attempts to read a super-journal file name from the -** end of the file and, if successful, copies it into memory supplied +** This function attempts to read a super-journal file name from the +** end of the file and, if successful, copies it into memory supplied ** by the caller. See comments above writeSuperJournal() for the format ** used to store a super-journal file name at the end of a journal file. ** @@ -1327,13 +1323,13 @@ static void checkPage(PgHdr *pPg){ ** nul-terminator byte is appended to the buffer following the ** super-journal file name. ** -** If it is determined that no super-journal file name is present +** If it is determined that no super-journal file name is present ** zSuper[0] is set to 0 and SQLITE_OK returned. ** ** If an error occurs while reading from the journal file, an SQLite ** error code is returned. */ -static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ +static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u64 nSuper){ int rc; /* Return code */ u32 len; /* Length in bytes of super-journal name */ i64 szJ; /* Total size in bytes of journal file pJrnl */ @@ -1345,9 +1341,9 @@ static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) || szJ<16 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) - || len>=nSuper + || len>=nSuper || len>szJ-16 - || len==0 + || len==0 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) || memcmp(aMagic, aJournalMagic, 8) @@ -1370,13 +1366,13 @@ static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ } zSuper[len] = '\0'; zSuper[len+1] = '\0'; - + return SQLITE_OK; } /* -** Return the offset of the sector boundary at or immediately -** following the value in pPager->journalOff, assuming a sector +** Return the offset of the sector boundary at or immediately +** following the value in pPager->journalOff, assuming a sector ** size of pPager->sectorSize bytes. ** ** i.e for a sector size of 512: @@ -1387,7 +1383,7 @@ static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ ** 512 512 ** 100 512 ** 2000 2048 -** +** */ static i64 journalHdrOffset(Pager *pPager){ i64 offset = 0; @@ -1409,12 +1405,12 @@ static i64 journalHdrOffset(Pager *pPager){ ** ** If doTruncate is non-zero or the Pager.journalSizeLimit variable is ** set to 0, then truncate the journal file to zero bytes in size. Otherwise, -** zero the 28-byte header at the start of the journal file. In either case, -** if the pager is not in no-sync mode, sync the journal file immediately +** zero the 28-byte header at the start of the journal file. In either case, +** if the pager is not in no-sync mode, sync the journal file immediately ** after writing or truncating it. ** ** If Pager.journalSizeLimit is set to a positive, non-zero value, and -** following the truncation or zeroing described above the size of the +** following the truncation or zeroing described above the size of the ** journal file in bytes is larger than this value, then truncate the ** journal file to Pager.journalSizeLimit bytes. The journal file does ** not need to be synced following this operation. @@ -1440,8 +1436,8 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); } - /* At this point the transaction is committed but the write lock - ** is still held on the file. If there is a size limit configured for + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for ** the persistent journal and the journal file currently consumes more ** space than that limit allows for, truncate it now. There is no need ** to sync the file following this operation. @@ -1469,7 +1465,7 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ ** - 4 bytes: Initial database page count. ** - 4 bytes: Sector size used by the process that wrote this journal. ** - 4 bytes: Database page size. -** +** ** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. */ static int writeJournalHdr(Pager *pPager){ @@ -1485,8 +1481,8 @@ static int writeJournalHdr(Pager *pPager){ nHeader = JOURNAL_HDR_SZ(pPager); } - /* If there are active savepoints and any of them were created - ** since the most recent journal header was written, update the + /* If there are active savepoints and any of them were created + ** since the most recent journal header was written, update the ** PagerSavepoint.iHdrOffset fields now. */ for(ii=0; iinSavepoint; ii++){ @@ -1497,10 +1493,10 @@ static int writeJournalHdr(Pager *pPager){ pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); - /* + /* ** Write the nRec Field - the number of page records that follow this ** journal header. Normally, zero is written to this value at this time. - ** After the records are added to the journal (and the journal synced, + ** After the records are added to the journal (and the journal synced, ** if in full-sync mode), the zero is overwritten with the true number ** of records (see syncJournal()). ** @@ -1519,7 +1515,7 @@ static int writeJournalHdr(Pager *pPager){ */ assert( isOpen(pPager->fd) || pPager->noSync ); if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) - || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); @@ -1527,9 +1523,32 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* The random check-hash initializer */ - sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + + + /* The random check-hash initializer */ + if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + } +#ifdef SQLITE_DEBUG + else{ + /* The Pager.cksumInit variable is usually randomized above to protect + ** against there being existing records in the journal file. This is + ** dangerous, as following a crash they may be mistaken for records + ** written by the current transaction and rolled back into the database + ** file, causing corruption. The following assert statements verify + ** that this is not required in "journal_mode=memory" mode, as in that + ** case the journal file is always 0 bytes in size at this point. + ** It is advantageous to avoid the sqlite3_randomness() call if possible + ** as it takes the global PRNG mutex. */ + i64 sz = 0; + sqlite3OsFileSize(pPager->jfd, &sz); + assert( sz==0 ); + assert( pPager->journalOff==journalHdrOffset(pPager) ); + assert( sqlite3JournalIsInMemory(pPager->jfd) ); + } +#endif put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); /* The assumed sector size for this process */ @@ -1546,23 +1565,23 @@ static int writeJournalHdr(Pager *pPager){ memset(&zHeader[sizeof(aJournalMagic)+20], 0, nHeader-(sizeof(aJournalMagic)+20)); - /* In theory, it is only necessary to write the 28 bytes that the - ** journal header consumes to the journal file here. Then increment the - ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next + /* In theory, it is only necessary to write the 28 bytes that the + ** journal header consumes to the journal file here. Then increment the + ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next ** record is written to the following sector (leaving a gap in the file ** that will be implicitly filled in by the OS). ** - ** However it has been discovered that on some systems this pattern can + ** However it has been discovered that on some systems this pattern can ** be significantly slower than contiguously writing data to the file, - ** even if that means explicitly writing data to the block of + ** even if that means explicitly writing data to the block of ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what - ** is done. + ** is done. ** - ** The loop is required here in case the sector-size is larger than the + ** The loop is required here in case the sector-size is larger than the ** database page size. Since the zHeader buffer is only Pager.pageSize ** bytes in size, more than one call to sqlite3OsWrite() may be required ** to populate the entire journal header sector. - */ + */ for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); @@ -1660,29 +1679,29 @@ static int readJournalHdr( /* Check that the values read from the page-size and sector-size fields ** are within range. To be 'in range', both values need to be a power - ** of two greater than or equal to 512 or 32, and not greater than their + ** of two greater than or equal to 512 or 32, and not greater than their ** respective compile time maximum limits. */ if( iPageSize<512 || iSectorSize<32 || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE - || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 + || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 ){ - /* If the either the page-size or sector-size in the journal-header is - ** invalid, then the process that wrote the journal-header must have - ** crashed before the header was synced. In this case stop reading + /* If the either the page-size or sector-size in the journal-header is + ** invalid, then the process that wrote the journal-header must have + ** crashed before the header was synced. In this case stop reading ** the journal file here. */ return SQLITE_DONE; } - /* Update the page-size to match the value read from the journal. - ** Use a testcase() macro to make sure that malloc failure within + /* Update the page-size to match the value read from the journal. + ** Use a testcase() macro to make sure that malloc failure within ** PagerSetPagesize() is tested. */ rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); testcase( rc!=SQLITE_OK ); - /* Update the assumed sector-size to match the value used by + /* Update the assumed sector-size to match the value used by ** the process that created this journal. If this journal was ** created by a process other than this one, then this routine ** is being called from within pager_playback(). The local value @@ -1709,10 +1728,10 @@ static int readJournalHdr( ** + 4 bytes: super-journal name checksum. ** + 8 bytes: aJournalMagic[]. ** -** The super-journal page checksum is the sum of the bytes in thesuper-journal +** The super-journal page checksum is the sum of the bytes in the super-journal ** name, where each byte is interpreted as a signed 8-bit integer. ** -** If zSuper is a NULL pointer (occurs for a single database transaction), +** If zSuper is a NULL pointer (occurs for a single database transaction), ** this call is a no-op. */ static int writeSuperJournal(Pager *pPager, const char *zSuper){ @@ -1725,8 +1744,8 @@ static int writeSuperJournal(Pager *pPager, const char *zSuper){ assert( pPager->setSuper==0 ); assert( !pagerUseWal(pPager) ); - if( !zSuper - || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + if( !zSuper + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || !isOpen(pPager->jfd) ){ return SQLITE_OK; @@ -1762,16 +1781,16 @@ static int writeSuperJournal(Pager *pPager, const char *zSuper){ } pPager->journalOff += (nSuper+20); - /* If the pager is in peristent-journal mode, then the physical + /* If the pager is in persistent-journal mode, then the physical ** journal-file may extend past the end of the super-journal name - ** and 8 bytes of magic data just written to the file. This is + ** and 8 bytes of magic data just written to the file. This is ** dangerous because the code to rollback a hot-journal file - ** will not be able to find the super-journal name to determine - ** whether or not the journal is hot. + ** will not be able to find the super-journal name to determine + ** whether or not the journal is hot. ** - ** Easiest thing to do in this scenario is to truncate the journal + ** Easiest thing to do in this scenario is to truncate the journal ** file to the required size. - */ + */ if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) && jrnlSize>pPager->journalOff ){ @@ -1816,7 +1835,7 @@ static void releaseAllSavepoints(Pager *pPager){ } /* -** Set the bit number pgno in the PagerSavepoint.pInSavepoint +** Set the bit number pgno in the PagerSavepoint.pInSavepoint ** bitvecs of all open savepoints. Return SQLITE_OK if successful ** or SQLITE_NOMEM if a malloc failure occurs. */ @@ -1845,8 +1864,8 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ ** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is ** closed (if it is open). ** -** If the pager is in ERROR state when this function is called, the -** contents of the pager cache are discarded before switching back to +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to ** the OPEN state. Regardless of whether the pager is in exclusive-mode ** or not, any journal file left in the file-system will be treated ** as a hot-journal and rolled back the next time a read-transaction @@ -1854,9 +1873,9 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ */ static void pager_unlock(Pager *pPager){ - assert( pPager->eState==PAGER_READER - || pPager->eState==PAGER_OPEN - || pPager->eState==PAGER_ERROR + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR ); sqlite3BitvecDestroy(pPager->pInJournal); @@ -1865,6 +1884,15 @@ static void pager_unlock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( !isOpen(pPager->jfd) ); + if( pPager->eState==PAGER_ERROR ){ + /* If an IO error occurs in wal.c while attempting to wrap the wal file, + ** then the Wal object may be holding a write-lock but no read-lock. + ** This call ensures that the write-lock is dropped as well. We cannot + ** have sqlite3WalEndReadTransaction() drop the write-lock, as it once + ** did, because this would break "BEGIN EXCLUSIVE" handling for + ** SQLITE_ENABLE_SETLK_TIMEOUT builds. */ + (void)sqlite3WalEndWriteTransaction(pPager->pWal); + } sqlite3WalEndReadTransaction(pPager->pWal); pPager->eState = PAGER_OPEN; }else if( !pPager->exclusiveMode ){ @@ -1932,18 +1960,18 @@ static void pager_unlock(Pager *pPager){ /* ** This function is called whenever an IOERR or FULL error that requires -** the pager to transition into the ERROR state may ahve occurred. -** The first argument is a pointer to the pager structure, the second -** the error-code about to be returned by a pager API function. The -** value returned is a copy of the second argument to this function. +** the pager to transition into the ERROR state may have occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. ** ** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the ** IOERR sub-codes, the pager enters the ERROR state and the error code ** is stored in Pager.errCode. While the pager remains in the ERROR state, ** all major API calls on the Pager will immediately return Pager.errCode. ** -** The ERROR state indicates that the contents of the pager-cache -** cannot be trusted. This state can be cleared by completely discarding +** The ERROR state indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding ** the contents of the pager-cache. If a transaction was active when ** the persistent error occurred, then the rollback journal may need ** to be replayed to restore the contents of the database file (as if @@ -1991,27 +2019,27 @@ static int pagerFlushOnCommit(Pager *pPager, int bCommit){ } /* -** This routine ends a transaction. A transaction is usually ended by -** either a COMMIT or a ROLLBACK operation. This routine may be called +** This routine ends a transaction. A transaction is usually ended by +** either a COMMIT or a ROLLBACK operation. This routine may be called ** after rollback of a hot-journal, or if an error occurs while opening ** the journal file or writing the very first journal-header of a ** database transaction. -** +** ** This routine is never called in PAGER_ERROR state. If it is called ** in PAGER_NONE or PAGER_SHARED state and the lock held is less ** exclusive than a RESERVED lock, it is a no-op. ** ** Otherwise, any active savepoints are released. ** -** If the journal file is open, then it is "finalized". Once a journal -** file has been finalized it is not possible to use it to roll back a +** If the journal file is open, then it is "finalized". Once a journal +** file has been finalized it is not possible to use it to roll back a ** transaction. Nor will it be considered to be a hot-journal by this ** or any other database connection. Exactly how a journal is finalized ** depends on whether or not the pager is running in exclusive mode and ** the current journal-mode (Pager.journalMode value), as follows: ** ** journalMode==MEMORY -** Journal file descriptor is simply closed. This destroys an +** Journal file descriptor is simply closed. This destroys an ** in-memory journal. ** ** journalMode==TRUNCATE @@ -2031,12 +2059,12 @@ static int pagerFlushOnCommit(Pager *pPager, int bCommit){ ** journalMode==PERSIST is used instead. ** ** After the journal is finalized, the pager moves to PAGER_READER state. -** If running in non-exclusive rollback mode, the lock on the file is +** If running in non-exclusive rollback mode, the lock on the file is ** downgraded to a SHARED_LOCK. ** ** SQLITE_OK is returned if no error occurs. If an error occurs during ** any of the IO operations to finalize the journal file or unlock the -** database then the IO error code is returned to the user. If the +** database then the IO error code is returned to the user. If the ** operation to finalize the journal file fails, then the code still ** tries to unlock the database file if not in exclusive mode. If the ** unlock operation fails as well, then the first error code related @@ -2055,9 +2083,9 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ ** 1. After a successful hot-journal rollback, it is called with ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. ** - ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE ** lock switches back to locking_mode=normal and then executes a - ** read-transaction, this function is called with eState==PAGER_READER + ** read-transaction, this function is called with eState==PAGER_READER ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. */ assert( assert_pager_state(pPager) ); @@ -2067,7 +2095,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ } releaseAllSavepoints(pPager); - assert( isOpen(pPager->jfd) || pPager->pInJournal==0 + assert( isOpen(pPager->jfd) || pPager->pInJournal==0 || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); if( isOpen(pPager->jfd) ){ @@ -2093,7 +2121,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST - || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + || (pPager->exclusiveMode && pPager->journalModetempFile); pPager->journalOff = 0; @@ -2105,9 +2133,9 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ */ int bDelete = !pPager->tempFile; assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); - assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE - || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_WAL + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( bDelete ){ @@ -2140,8 +2168,8 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ } if( pagerUseWal(pPager) ){ - /* Drop the WAL write-lock, if any. Also, if the connection was in - ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE ** lock held on the database file. */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); @@ -2149,7 +2177,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ /* This branch is taken when committing a transaction in rollback-journal ** mode if the database file on disk is larger than the database image. - ** At this point the journal has been finalized and the transaction + ** At this point the journal has been finalized and the transaction ** successfully committed, but the EXCLUSIVE lock is still held on the ** file. So it is safe to truncate the database file to its minimum ** required size. */ @@ -2162,7 +2190,7 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } - if( !pPager->exclusiveMode + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); @@ -2173,20 +2201,23 @@ static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ return (rc==SQLITE_OK?rc2:rc); } +/* Forward reference */ +static int pager_playback(Pager *pPager, int isHot); + /* -** Execute a rollback if a transaction is active and unlock the -** database file. +** Execute a rollback if a transaction is active and unlock the +** database file. ** -** If the pager has already entered the ERROR state, do not attempt +** If the pager has already entered the ERROR state, do not attempt ** the rollback at this time. Instead, pager_unlock() is called. The ** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and move the pager back to OPEN state. If this -** means that there is a hot-journal left in the file-system, the next -** connection to obtain a shared lock on the pager (which may be this one) +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) ** will roll it back. ** ** If the pager has not already entered the ERROR state, but an IO or -** malloc error occurs during a rollback, then this will itself cause +** malloc error occurs during a rollback, then this will itself cause ** the pager to enter the ERROR state. Which will be cleared by the ** call to pager_unlock(), as described above. */ @@ -2201,16 +2232,31 @@ static void pagerUnlockAndRollback(Pager *pPager){ assert( pPager->eState==PAGER_READER ); pager_end_transaction(pPager, 0, 0); } + }else if( pPager->eState==PAGER_ERROR + && pPager->journalMode==PAGER_JOURNALMODE_MEMORY + && isOpen(pPager->jfd) + ){ + /* Special case for a ROLLBACK due to I/O error with an in-memory + ** journal: We have to rollback immediately, before the journal is + ** closed, because once it is closed, all content is forgotten. */ + int errCode = pPager->errCode; + u8 eLock = pPager->eLock; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + pPager->eLock = EXCLUSIVE_LOCK; + pager_playback(pPager, 1); + pPager->errCode = errCode; + pPager->eLock = eLock; } pager_unlock(pPager); } /* ** Parameter aData must point to a buffer of pPager->pageSize bytes -** of data. Compute and return a checksum based ont the contents of the +** of data. Compute and return a checksum based on the contents of the ** page of data and the current value of pPager->cksumInit. ** -** This is not a real checksum. It is really just the sum of the +** This is not a real checksum. It is really just the sum of the ** random initial value (pPager->cksumInit) and every 200th byte ** of the page data, starting with byte offset (pPager->pageSize%200). ** Each byte is interpreted as an 8-bit unsigned integer. @@ -2218,8 +2264,8 @@ static void pagerUnlockAndRollback(Pager *pPager){ ** Changing the formula used to compute this checksum results in an ** incompatible journal file format. ** -** If journal corruption occurs due to a power failure, the most likely -** scenario is that one end or the other of the record will be changed. +** If journal corruption occurs due to a power failure, the most likely +** scenario is that one end or the other of the record will be changed. ** It is much less likely that the two ends of the journal record will be ** correct and the middle be corrupt. Thus, this "checksum" scheme, ** though fast and simple, catches the mostly likely kind of corruption. @@ -2273,7 +2319,7 @@ void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ ** The page begins at offset *pOffset into the file. The *pOffset ** value is increased to the start of the next page in the journal. ** -** The main rollback journal uses checksums - the statement journal does +** The main rollback journal uses checksums - the statement journal does ** not. ** ** If the page number of the page record read from the (sub-)journal file @@ -2293,7 +2339,7 @@ void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ ** is successfully read from the (sub-)journal file but appears to be ** corrupted, SQLITE_DONE is returned. Data is considered corrupted in ** two circumstances: -** +** ** * If the record page-number is illegal (0 or PAGER_SJ_PGNO), or ** * If the record is being rolled back from the main journal file ** and the checksum field does not match the record content. @@ -2335,7 +2381,7 @@ static int pager_playback_one_page( assert( aData ); /* Temp storage must have already been allocated */ assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); - /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction ** or savepoint rollback done at the request of the caller) or this is ** a hot-journal rollback. If it is a hot-journal rollback, the pager ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback @@ -2402,7 +2448,7 @@ static int pager_playback_one_page( ** assert()able. ** ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the - ** pager cache if it exists and the main file. The page is then marked + ** pager cache if it exists and the main file. The page is then marked ** not dirty. Since this code is only executed in PAGER_OPEN state for ** a hot-journal rollback, it is guaranteed that the page-cache is empty ** if the pager is in OPEN state. @@ -2484,18 +2530,18 @@ static int pager_playback_one_page( }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to ** the database and the page is not in-memory, there is a potential - ** problem. When the page is next fetched by the b-tree layer, it - ** will be read from the database file, which may or may not be - ** current. + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. ** ** There are a couple of different ways this can happen. All are quite - ** obscure. When running in synchronous mode, this can only happen + ** obscure. When running in synchronous mode, this can only happen ** if the page is on the free-list at the start of the transaction, then ** populated, then moved using sqlite3PagerMovepage(). ** ** The solution is to add an in-memory page to the cache containing - ** the data just read from the sub-journal. Mark the page as dirty - ** and if the pager requires a journal-sync, then mark the page as + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as ** requiring a journal-sync before it is written. */ assert( isSavepnt ); @@ -2547,26 +2593,26 @@ static int pager_playback_one_page( ** This routine checks if it is possible to delete the super-journal file, ** and does so if it is. ** -** Argument zSuper may point to Pager.pTmpSpace. So that buffer is not +** Argument zSuper may point to Pager.pTmpSpace. So that buffer is not ** available for use within this function. ** -** When a super-journal file is created, it is populated with the names -** of all of its child journals, one after another, formatted as utf-8 -** encoded text. The end of each child journal file is marked with a +** When a super-journal file is created, it is populated with the names +** of all of its child journals, one after another, formatted as utf-8 +** encoded text. The end of each child journal file is marked with a ** nul-terminator byte (0x00). i.e. the entire contents of a super-journal ** file for a transaction involving two databases might be: ** ** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" ** -** A super-journal file may only be deleted once all of its child +** A super-journal file may only be deleted once all of its child ** journals have been rolled back. ** -** This function reads the contents of the super-journal file into +** This function reads the contents of the super-journal file into ** memory and loops through each of the child journal names. For ** each child journal, it checks if: ** ** * if the child journal exists, and if so -** * if the child journal contains a reference to super-journal +** * if the child journal contains a reference to super-journal ** file zSuper ** ** If a child journal can be found that matches both of the criteria @@ -2576,12 +2622,12 @@ static int pager_playback_one_page( ** ** If an IO error within this function, an error code is returned. This ** function allocates memory by calling sqlite3Malloc(). If an allocation -** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors +** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors ** occur, SQLITE_OK is returned. ** ** TODO: This function allocates a single block of memory to load ** the entire contents of the super-journal file. This could be -** a couple of kilobytes or so - potentially larger than the page +** a couple of kilobytes or so - potentially larger than the page ** size. */ static int pager_delsuper(Pager *pPager, const char *zSuper){ @@ -2594,12 +2640,12 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ char *zJournal; /* Pointer to one journal within MJ file */ char *zSuperPtr; /* Space to hold super-journal filename */ char *zFree = 0; /* Free this buffer */ - int nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ + i64 nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ /* Allocate space for both the pJournal and pSuper file descriptors. ** If successful, open the super-journal file for reading. */ - pSuper = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + pSuper = (sqlite3_file *)sqlite3MallocZero(2 * (i64)pVfs->szOsFile); if( !pSuper ){ rc = SQLITE_NOMEM_BKPT; pJournal = 0; @@ -2617,11 +2663,14 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ */ rc = sqlite3OsFileSize(pSuper, &nSuperJournal); if( rc!=SQLITE_OK ) goto delsuper_out; - nSuperPtr = pVfs->mxPathname+1; + nSuperPtr = 1 + (i64)pVfs->mxPathname; + assert( nSuperJournal>=0 && nSuperPtr>0 ); zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2); if( !zFree ){ rc = SQLITE_NOMEM_BKPT; goto delsuper_out; + }else{ + assert( nSuperJournal<=0x7fffffff ); } zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0; zSuperJournal = &zFree[4]; @@ -2642,7 +2691,7 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ /* One of the journals pointed to by the super-journal exists. ** Open it and check if it points at the super-journal. If ** so, return without deleting the super-journal file. - ** NB: zJournal is really a MAIN_JOURNAL. But call it a + ** NB: zJournal is really a MAIN_JOURNAL. But call it a ** SUPER_JOURNAL here so that the VFS will not send the zJournal ** name into sqlite3_database_file_object(). */ @@ -2667,7 +2716,7 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ } zJournal += (sqlite3Strlen30(zJournal)+1); } - + sqlite3OsClose(pSuper); rc = sqlite3OsDelete(pVfs, zSuper, 0); @@ -2683,20 +2732,20 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ /* -** This function is used to change the actual size of the database +** This function is used to change the actual size of the database ** file in the file-system. This only happens when committing a transaction, ** or rolling back a transaction (including rolling back a hot-journal). ** ** If the main database file is not open, or the pager is not in either -** DBMOD or OPEN state, this function is a no-op. Otherwise, the size -** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). ** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** -** Or, it might be the case that the file on disk is smaller than -** nPage pages. Some operating system implementations can get confused if -** you try to truncate a file to some size that is larger than it -** currently is, so detect this case and write a single zero byte to +** Or, it might be the case that the file on disk is smaller than +** nPage pages. Some operating system implementations can get confused if +** you try to truncate a file to some size that is larger than it +** currently is, so detect this case and write a single zero byte to ** the end of the new file instead. ** ** If successful, return SQLITE_OK. If an IO error occurs while modifying @@ -2708,9 +2757,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ assert( pPager->eState!=PAGER_READER ); PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage)); - - if( isOpen(pPager->fd) - && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ i64 currentSize, newSize; int szPage = pPager->pageSize; @@ -2755,8 +2804,8 @@ int sqlite3SectorSize(sqlite3_file *pFile){ /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method -** of the open database file. The sector size will be used -** to determine the size and alignment of journal header and +** of the open database file. The sector size will be used +** to determine the size and alignment of journal header and ** super-journal pointers within created journal files. ** ** For temporary files the effective sector size is always 512 bytes. @@ -2779,7 +2828,7 @@ static void setSectorSize(Pager *pPager){ assert( isOpen(pPager->fd) || pPager->tempFile ); if( pPager->tempFile - || (sqlite3OsDeviceCharacteristics(pPager->fd) & + || (sqlite3OsDeviceCharacteristics(pPager->fd) & SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0 ){ /* Sector size doesn't matter for temporary files. Also, the file @@ -2793,15 +2842,15 @@ static void setSectorSize(Pager *pPager){ /* ** Playback the journal and thus restore the database file to -** the state it was in before we started making changes. +** the state it was in before we started making changes. ** -** The journal file format is as follows: +** The journal file format is as follows: ** ** (1) 8 byte prefix. A copy of aJournalMagic[]. ** (2) 4 byte big-endian integer which is the number of valid page records ** in the journal. If this value is 0xffffffff, then compute the ** number of page records from the journal size. -** (3) 4 byte big-endian integer which is the initial value for the +** (3) 4 byte big-endian integer which is the initial value for the ** sanity checksum. ** (4) 4 byte integer which is the number of pages to truncate the ** database to during a rollback. @@ -2830,7 +2879,7 @@ static void setSectorSize(Pager *pPager){ ** from the file size. This value is used when the user selects the ** no-sync option for the journal. A power failure could lead to corruption ** in this case. But for things like temporary table (which will be -** deleted when the power is restored) we don't care. +** deleted when the power is restored) we don't care. ** ** If the file opened as the journal file is not a well-formed ** journal file then all pages up to the first corrupted page are rolled @@ -2842,7 +2891,7 @@ static void setSectorSize(Pager *pPager){ ** and an error code is returned. ** ** The isHot parameter indicates that we are trying to rollback a journal -** that might be a hot journal. Or, it could be that the journal is +** that might be a hot journal. Or, it could be that the journal is ** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. ** If the journal really is hot, reset the pager cache prior rolling ** back any content. If the journal is merely persistent, no reset is @@ -2882,7 +2931,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** for pageSize. */ zSuper = pPager->pTmpSpace; - rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + rc = readSuperJournal(pPager->jfd, zSuper, 1+(i64)pPager->pVfs->mxPathname); if( rc==SQLITE_OK && zSuper[0] ){ rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); } @@ -2893,9 +2942,9 @@ static int pager_playback(Pager *pPager, int isHot){ pPager->journalOff = 0; needPagerReset = isHot; - /* This loop terminates either when a readJournalHdr() or - ** pager_playback_one_page() call returns SQLITE_DONE or an IO error - ** occurs. + /* This loop terminates either when a readJournalHdr() or + ** pager_playback_one_page() call returns SQLITE_DONE or an IO error + ** occurs. */ while( 1 ){ /* Read the next journal header from the journal file. If there are @@ -2904,7 +2953,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** This indicates nothing more needs to be rolled back. */ rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } @@ -2932,7 +2981,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** chunk of the journal contains zero pages to be rolled back. But ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in ** the journal, it means that the journal might contain additional - ** pages that need to be rolled back and that the number of pages + ** pages that need to be rolled back and that the number of pages ** should be computed based on the journal file size. */ if( nRec==0 && !isHot && @@ -2954,7 +3003,7 @@ static int pager_playback(Pager *pPager, int isHot){ } } - /* Copy original pages out of the journal and back into the + /* Copy original pages out of the journal and back into the ** database file and/or page cache. */ for(u=0; ufd,SQLITE_FCNTL_DB_UNCHANGED,0); #endif - /* If this playback is happening automatically as a result of an IO or - ** malloc error that occurred after the change-counter was updated but - ** before the transaction was committed, then the change-counter - ** modification may just have been reverted. If this happens in exclusive + /* If this playback is happening automatically as a result of an IO or + ** malloc error that occurred after the change-counter was updated but + ** before the transaction was committed, then the change-counter + ** modification may just have been reverted. If this happens in exclusive ** mode, then subsequent transactions performed by the connection will not ** update the change-counter at all. This may lead to cache inconsistency ** problems for other processes at some point in the future. So, just @@ -3021,7 +3070,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** which case it requires 4 0x00 bytes in memory immediately before ** the filename. */ zSuper = &pPager->pTmpSpace[4]; - rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + rc = readSuperJournal(pPager->jfd, zSuper, 1+(i64)pPager->pVfs->mxPathname); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK @@ -3058,7 +3107,7 @@ static int pager_playback(Pager *pPager, int isHot){ /* ** Read the content for page pPg out of the database file (or out of -** the WAL if that is where the most recent copy if found) into +** the WAL if that is where the most recent copy if found) into ** pPg->pData. A shared lock or greater must be held on the database ** file before this function is called. ** @@ -3150,15 +3199,15 @@ static void pager_write_changecounter(PgHdr *pPg){ #ifndef SQLITE_OMIT_WAL /* -** This function is invoked once for each page that has already been +** This function is invoked once for each page that has already been ** written into the log file when a WAL transaction is rolled back. -** Parameter iPg is the page number of said page. The pCtx argument +** Parameter iPg is the page number of said page. The pCtx argument ** is actually a pointer to the Pager structure. ** ** If page iPg is present in the cache, and has no outstanding references, ** it is discarded. Otherwise, if there are one or more outstanding ** references, the page content is reloaded from the database. If the -** attempt to reload content from the database is required and fails, +** attempt to reload content from the database is required and fails, ** return an SQLite error code. Otherwise, SQLITE_OK. */ static int pagerUndoCallback(void *pCtx, Pgno iPg){ @@ -3184,7 +3233,7 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ ** updated as data is copied out of the rollback journal and into the ** database. This is not generally possible with a WAL database, as ** rollback involves simply truncating the log file. Therefore, if one - ** or more frames have already been written to the log (and therefore + ** or more frames have already been written to the log (and therefore ** also copied into the backup databases) as part of this transaction, ** the backups must be restarted. */ @@ -3201,7 +3250,7 @@ static int pagerRollbackWal(Pager *pPager){ PgHdr *pList; /* List of dirty pages to revert */ /* For all pages in the cache that are currently dirty or have already - ** been written (but not committed) to the log file, do one of the + ** been written (but not committed) to the log file, do one of the ** following: ** ** + Discard the cached page (if refcount==0), or @@ -3223,11 +3272,11 @@ static int pagerRollbackWal(Pager *pPager){ ** This function is a wrapper around sqlite3WalFrames(). As well as logging ** the contents of the list of pages headed by pList (connected by pDirty), ** this function notifies any active backup processes that the pages have -** changed. +** changed. ** ** The list of pages passed into this routine is always sorted by page number. ** Hence, if page 1 appears anywhere on the list, it will be the first page. -*/ +*/ static int pagerWalFrames( Pager *pPager, /* Pager object */ PgHdr *pList, /* List of frames to log */ @@ -3241,7 +3290,7 @@ static int pagerWalFrames( assert( pPager->pWal ); assert( pList ); #ifdef SQLITE_DEBUG - /* Verify that the page list is in accending order */ + /* Verify that the page list is in ascending order */ for(p=pList; p && p->pDirty; p=p->pDirty){ assert( p->pgno < p->pDirty->pgno ); } @@ -3268,7 +3317,7 @@ static int pagerWalFrames( pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); - rc = sqlite3WalFrames(pPager->pWal, + rc = sqlite3WalFrames(pPager->pWal, pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ @@ -3372,7 +3421,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ #ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager -** exists if the database is not empy, or verify that the *-wal file does +** exists if the database is not empty, or verify that the *-wal file does ** not exist (by deleting it) if the database file is empty. ** ** If the database is not empty and the *-wal file exists, open the pager @@ -3383,9 +3432,9 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** Return SQLITE_OK or an error code. ** ** The caller must hold a SHARED lock on the database file to call this -** function. Because an EXCLUSIVE lock on the db file is required to delete -** a WAL on a none-empty database, this ensures there is no race condition -** between the xAccess() below and an xDelete() being executed by some +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL on a none-empty database, this ensures there is no race condition +** between the xAccess() below and an xDelete() being executed by some ** other connection. */ static int pagerOpenWalIfPresent(Pager *pPager){ @@ -3421,21 +3470,21 @@ static int pagerOpenWalIfPresent(Pager *pPager){ /* ** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback -** the entire super-journal file. The case pSavepoint==NULL occurs when -** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction +** the entire super-journal file. The case pSavepoint==NULL occurs when +** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction ** savepoint. ** -** When pSavepoint is not NULL (meaning a non-transaction savepoint is +** When pSavepoint is not NULL (meaning a non-transaction savepoint is ** being rolled back), then the rollback consists of up to three stages, ** performed in the order specified: ** ** * Pages are played back from the main journal starting at byte -** offset PagerSavepoint.iOffset and continuing to +** offset PagerSavepoint.iOffset and continuing to ** PagerSavepoint.iHdrOffset, or to the end of the main journal ** file if PagerSavepoint.iHdrOffset is zero. ** ** * If PagerSavepoint.iHdrOffset is not zero, then pages are played -** back starting from the journal header immediately following +** back starting from the journal header immediately following ** PagerSavepoint.iHdrOffset to the end of the main journal file. ** ** * Pages are then played back from the sub-journal file, starting @@ -3451,7 +3500,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ ** journal file. There is no need for a bitvec in this case. ** ** In either case, before playback commences the Pager.dbSize variable -** is reset to the value that it held at the start of the savepoint +** is reset to the value that it held at the start of the savepoint ** (or transaction). No page with a page-number greater than this value ** is played back. If one is encountered it is simply skipped. */ @@ -3472,7 +3521,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ } } - /* Set the database size back to the value it was before the savepoint + /* Set the database size back to the value it was before the savepoint ** being reverted was opened. */ pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; @@ -3525,7 +3574,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ ** test is related to ticket #2565. See the discussion in the ** pager_playback() function for additional information. */ - if( nJRec==0 + if( nJRec==0 && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); @@ -3666,14 +3715,27 @@ void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - if( pPager->tempFile ){ + if( pPager->tempFile || level==PAGER_SYNCHRONOUS_OFF ){ pPager->noSync = 1; pPager->fullSync = 0; pPager->extraSync = 0; }else{ - pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->noSync = 0; pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; - pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + + /* Set Pager.extraSync if "PRAGMA synchronous=EXTRA" is requested, or + ** if the file-system supports F2FS style atomic writes. If this flag + ** is set, SQLite syncs the directory to disk immediately after deleting + ** a journal file in "PRAGMA journal_mode=DELETE" mode. */ + if( level==PAGER_SYNCHRONOUS_EXTRA +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (sqlite3OsDeviceCharacteristics(pPager->fd) & SQLITE_IOCAP_BATCH_ATOMIC) +#endif + ){ + pPager->extraSync = 1; + }else{ + pPager->extraSync = 0; + } } if( pPager->noSync ){ pPager->syncFlags = 0; @@ -3699,7 +3761,7 @@ void sqlite3PagerSetFlags( /* ** The following global variable is incremented whenever the library ** attempts to open a temporary file. This information is used for -** testing and analysis only. +** testing and analysis only. */ #ifdef SQLITE_TEST int sqlite3_opentemp_count = 0; @@ -3708,8 +3770,8 @@ int sqlite3_opentemp_count = 0; /* ** Open a temporary file. ** -** Write the file descriptor into *pFile. Return SQLITE_OK on success -** or some other error code if we fail. The OS will automatically +** Write the file descriptor into *pFile. Return SQLITE_OK on success +** or some other error code if we fail. The OS will automatically ** delete the temporary file when it is closed. ** ** The flags passed to the VFS layer xOpen() call are those specified @@ -3741,9 +3803,9 @@ static int pagerOpentemp( /* ** Set the busy handler function. ** -** The pager invokes the busy-handler if sqlite3OsLock() returns +** The pager invokes the busy-handler if sqlite3OsLock() returns ** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, -** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE +** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE ** lock. It does *not* invoke the busy handler when upgrading from ** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE ** (which occurs during hot-journal rollback). Summary: @@ -3755,7 +3817,7 @@ static int pagerOpentemp( ** SHARED_LOCK -> EXCLUSIVE_LOCK | No ** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes ** -** If the busy-handler callback returns non-zero, the lock is +** If the busy-handler callback returns non-zero, the lock is ** retried. If it returns zero, then the SQLITE_BUSY error is ** returned to the caller of the pager API function. */ @@ -3774,16 +3836,16 @@ void sqlite3PagerSetBusyHandler( } /* -** Change the page size used by the Pager object. The new page size +** Change the page size used by the Pager object. The new page size ** is passed in *pPageSize. ** ** If the pager is in the error state when this function is called, it -** is a no-op. The value returned is the error state error code (i.e. +** is a no-op. The value returned is the error state error code (i.e. ** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). ** ** Otherwise, if all of the following are true: ** -** * the new page size (value of *pPageSize) is valid (a power +** * the new page size (value of *pPageSize) is valid (a power ** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and ** ** * there are no outstanding page references, and @@ -3793,14 +3855,14 @@ void sqlite3PagerSetBusyHandler( ** ** then the pager object page size is set to *pPageSize. ** -** If the page size is changed, then this function uses sqlite3PagerMalloc() -** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt -** fails, SQLITE_NOMEM is returned and the page size remains unchanged. +** If the page size is changed, then this function uses sqlite3PagerMalloc() +** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt +** fails, SQLITE_NOMEM is returned and the page size remains unchanged. ** In all other cases, SQLITE_OK is returned. ** ** If the page size is not changed, either because one of the enumerated ** conditions above is not true, the pager was in error state when this -** function was called, or because the memory allocation attempt failed, +** function was called, or because the memory allocation attempt failed, ** then *pPageSize is set to the old, retained page size before returning. */ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ @@ -3810,7 +3872,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ ** function may be called from within PagerOpen(), before the state ** of the Pager object is internally consistent. ** - ** At one point this function returned an error if the pager was in + ** At one point this function returned an error if the pager was in ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that ** there is at least one outstanding page reference, this function ** is a no-op for that case anyhow. @@ -3819,8 +3881,8 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ u32 pageSize = *pPageSize; assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); if( (pPager->memDb==0 || pPager->dbSize==0) - && sqlite3PcacheRefCount(pPager->pPCache)==0 - && pageSize && pageSize!=(u32)pPager->pageSize + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize ){ char *pNew = NULL; /* New temp space */ i64 nByte = 0; @@ -3878,7 +3940,7 @@ void *sqlite3PagerTempSpace(Pager *pPager){ } /* -** Attempt to set the maximum database page count if mxPage is positive. +** Attempt to set the maximum database page count if mxPage is positive. ** Make no changes if mxPage is zero or negative. And never reduce the ** maximum page count below the current size of the database. ** @@ -3922,11 +3984,11 @@ void enable_simulated_io_errors(void){ /* ** Read the first N bytes from the beginning of the file into memory -** that pDest points to. +** that pDest points to. ** ** If the pager was opened on a transient file (zFilename==""), or ** opened on a file less than N bytes in size, the output buffer is -** zeroed and SQLITE_OK returned. The rationale for this is that this +** zeroed and SQLITE_OK returned. The rationale for this is that this ** function is used to read database headers, and a new transient or ** zero sized database has a header than consists entirely of zeroes. ** @@ -3959,7 +4021,7 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ ** This function may only be called when a read-transaction is open on ** the pager. It returns the total number of pages in the database. ** -** However, if the file is between 1 and bytes in size, then +** However, if the file is between 1 and bytes in size, then ** this is considered a 1 page file. */ void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ @@ -3974,19 +4036,19 @@ void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ ** a similar or greater lock is already held, this function is a no-op ** (returning SQLITE_OK immediately). ** -** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke -** the busy callback if the lock is currently not available. Repeat -** until the busy callback returns false or until the attempt to +** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke +** the busy callback if the lock is currently not available. Repeat +** until the busy callback returns false or until the attempt to ** obtain the lock succeeds. ** ** Return SQLITE_OK on success and an error code if we cannot obtain -** the lock. If the lock is obtained successfully, set the Pager.state +** the lock. If the lock is obtained successfully, set the Pager.state ** variable to locktype before returning. */ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ - /* Check that this is either a no-op (because the requested lock is + /* Check that this is either a no-op (because the requested lock is ** already held), or one of the transitions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). @@ -4003,10 +4065,10 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ } /* -** Function assertTruncateConstraint(pPager) checks that one of the +** Function assertTruncateConstraint(pPager) checks that one of the ** following is true for all dirty pages currently in the page-cache: ** -** a) The page number is less than or equal to the size of the +** a) The page number is less than or equal to the size of the ** current database image, in pages, OR ** ** b) if the page content were written at this time, it would not @@ -4018,9 +4080,9 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** the database file. If a savepoint transaction were rolled back after ** this happened, the correct behavior would be to restore the current ** content of the page. However, since this content is not present in either -** the database file or the portion of the rollback journal and +** the database file or the portion of the rollback journal and ** sub-journal rolled back the content could not be restored and the -** database image would become corrupt. It is therefore fortunate that +** database image would become corrupt. It is therefore fortunate that ** this circumstance cannot arise. */ #if defined(SQLITE_DEBUG) @@ -4044,9 +4106,9 @@ static void assertTruncateConstraint(Pager *pPager){ #endif /* -** Truncate the in-memory database file image to nPage pages. This -** function does not actually modify the database file on disk. It -** just sets the internal state of the pager object so that the +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. ** ** This function is only called right before committing a transaction. @@ -4061,11 +4123,11 @@ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ /* At one point the code here called assertTruncateConstraint() to ** ensure that all pages being truncated away by this operation are, - ** if one or more savepoints are open, present in the savepoint + ** if one or more savepoints are open, present in the savepoint ** journal so that they can be restored if the savepoint is rolled ** back. This is no longer necessary as this function is now only - ** called right before committing a transaction. So although the - ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), ** they cannot be rolled back. So the assertTruncateConstraint() call ** is no longer correct. */ } @@ -4077,12 +4139,12 @@ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ ** size of the journal file so that the pager_playback() routine knows ** that the entire journal file has been synced. ** -** Syncing a hot-journal to disk before attempting to roll it back ensures +** Syncing a hot-journal to disk before attempting to roll it back ensures ** that if a power-failure occurs during the rollback, the process that ** attempts rollback following system recovery sees the same journal ** content as this process. ** -** If everything goes as planned, SQLITE_OK is returned. Otherwise, +** If everything goes as planned, SQLITE_OK is returned. Otherwise, ** an SQLite error code. */ static int pagerSyncHotJournal(Pager *pPager){ @@ -4098,7 +4160,7 @@ static int pagerSyncHotJournal(Pager *pPager){ #if SQLITE_MAX_MMAP_SIZE>0 /* -** Obtain a reference to a memory mapped page object for page number pgno. +** Obtain a reference to a memory mapped page object for page number pgno. ** The new object will use the pointer pData, obtained from xFetch(). ** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set @@ -4114,7 +4176,7 @@ static int pagerAcquireMapPage( PgHdr **ppPage /* OUT: Acquired page object */ ){ PgHdr *p; /* Memory mapped page to return */ - + if( pPager->pMmapFreelist ){ *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; @@ -4128,6 +4190,7 @@ static int pagerAcquireMapPage( return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; + assert( EIGHT_BYTE_ALIGNMENT( p->pExtra ) ); p->flags = PGHDR_MMAP; p->nRef = 1; p->pPager = pPager; @@ -4148,7 +4211,7 @@ static int pagerAcquireMapPage( #endif /* -** Release a reference to page pPg. pPg must have been returned by an +** Release a reference to page pPg. pPg must have been returned by an ** earlier call to pagerAcquireMapPage(). */ static void pagerReleaseMapPage(PgHdr *pPg){ @@ -4208,7 +4271,7 @@ static int databaseIsUnmoved(Pager *pPager){ ** result in a coredump. ** ** This function always succeeds. If a transaction is active an attempt -** is made to roll it back. If an error occurs during the rollback +** is made to roll it back. If an error occurs during the rollback ** a hot journal may be left in the filesystem but no error is returned ** to the caller. */ @@ -4225,7 +4288,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ { u8 *a = 0; assert( db || pPager->pWal==0 ); - if( db && 0==(db->flags & SQLITE_NoCkptOnClose) + if( db && 0==(db->flags & SQLITE_NoCkptOnClose) && SQLITE_OK==databaseIsUnmoved(pPager) ){ a = pTmp; @@ -4239,8 +4302,8 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ pager_unlock(pPager); }else{ /* If it is open, sync the journal file before calling UnlockAndRollback. - ** If this is not done, then an unsynced portion of the open journal - ** file may be played back into the database. If a power failure occurs + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs ** while this is happening, the database could become corrupt. ** ** If an error occurs while trying to sync the journal, shift the pager @@ -4298,7 +4361,7 @@ void sqlite3PagerRef(DbPage *pPg){ ** disk and can be restored in the event of a hot-journal rollback. ** ** If the Pager.noSync flag is set, then this function is a no-op. -** Otherwise, the actions required depend on the journal-mode and the +** Otherwise, the actions required depend on the journal-mode and the ** device characteristics of the file-system, as follows: ** ** * If the journal file is an in-memory journal file, no action need @@ -4310,7 +4373,7 @@ void sqlite3PagerRef(DbPage *pPg){ ** been written following it. If the pager is operating in full-sync ** mode, then the journal file is synced before this field is updated. ** -** * If the device does not support the SEQUENTIAL property, then +** * If the device does not support the SEQUENTIAL property, then ** journal file is synced. ** ** Or, in pseudo-code: @@ -4319,11 +4382,11 @@ void sqlite3PagerRef(DbPage *pPg){ ** if( NOT SAFE_APPEND ){ ** if( ) xSync(); ** -** } +** } ** if( NOT SEQUENTIAL ) xSync(); ** } ** -** If successful, this routine clears the PGHDR_NEED_SYNC flag of every +** If successful, this routine clears the PGHDR_NEED_SYNC flag of every ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ @@ -4351,10 +4414,10 @@ static int syncJournal(Pager *pPager, int newHdr){ ** mode, then the journal file may at this point actually be larger ** than Pager.journalOff bytes. If the next thing in the journal ** file happens to be a journal-header (written as part of the - ** previous connection's transaction), and a crash or power-failure - ** occurs after nRec is updated but before this connection writes - ** anything else to the journal file (or commits/rolls back its - ** transaction), then SQLite may become confused when doing the + ** previous connection's transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the ** hot-journal rollback following recovery. It may roll back all ** of this connections data, then proceed to rolling back the old, ** out-of-date data that follows it. Database corruption. @@ -4364,7 +4427,7 @@ static int syncJournal(Pager *pPager, int newHdr){ ** byte to the start of it to prevent it from being recognized. ** ** Variable iNextHdrOffset is set to the offset at which this - ** problematic header will occur, if it exists. aMagic is used + ** problematic header will occur, if it exists. aMagic is used ** as a temporary buffer to inspect the first couple of bytes of ** the potential journal header. */ @@ -4391,7 +4454,7 @@ static int syncJournal(Pager *pPager, int newHdr){ ** it as a candidate for rollback. ** ** This is not required if the persistent media supports the - ** SAFE_APPEND property. Because in this case it is not possible + ** SAFE_APPEND property. Because in this case it is not possible ** for garbage data to be appended to the file, the nRec field ** is populated with 0xFFFFFFFF when the journal header is written ** and never needs to be updated. @@ -4411,7 +4474,7 @@ static int syncJournal(Pager *pPager, int newHdr){ if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) ); if( rc!=SQLITE_OK ) return rc; @@ -4428,8 +4491,8 @@ static int syncJournal(Pager *pPager, int newHdr){ } } - /* Unless the pager is in noSync mode, the journal file was just - ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on ** all pages. */ sqlite3PcacheClearSyncFlags(pPager->pPCache); @@ -4449,9 +4512,9 @@ static int syncJournal(Pager *pPager, int newHdr){ ** is called. Before writing anything to the database file, this lock ** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, ** SQLITE_BUSY is returned and no data is written to the database file. -** +** ** If the pager is a temp-file pager and the actual file-system file -** is not yet open, it is created and opened before any data is +** is not yet open, it is created and opened before any data is ** written out. ** ** Once the lock has been upgraded and, if necessary, the file opened, @@ -4466,7 +4529,7 @@ static int syncJournal(Pager *pPager, int newHdr){ ** in Pager.dbFileVers[] is updated to match the new value stored in ** the database file. ** -** If everything is successful, SQLITE_OK is returned. If an IO error +** If everything is successful, SQLITE_OK is returned. If an IO error ** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot ** be obtained, SQLITE_BUSY is returned. */ @@ -4492,7 +4555,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK + if( rc==SQLITE_OK && pPager->dbHintSizedbSize && (pList->pDirty || pList->pgno>pPager->dbHintSize) ){ @@ -4514,7 +4577,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ */ if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ - char *pData; /* Data to write */ + char *pData; /* Data to write */ assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); if( pList->pgno==1 ) pager_write_changecounter(pList); @@ -4526,8 +4589,8 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match - ** the value now stored in the database file. If writing this - ** page caused the database file to grow, update dbFileSize. + ** the value now stored in the database file. If writing this + ** page caused the database file to grow, update dbFileSize. */ if( pgno==1 ){ memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); @@ -4555,18 +4618,18 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ } /* -** Ensure that the sub-journal file is open. If it is already open, this +** Ensure that the sub-journal file is open. If it is already open, this ** function is a no-op. ** -** SQLITE_OK is returned if everything goes according to plan. An -** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() ** fails. */ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ - const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; int nStmtSpill = sqlite3Config.nStmtSpill; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ @@ -4578,13 +4641,13 @@ static int openSubJournal(Pager *pPager){ } /* -** Append a record of the current state of page pPg to the sub-journal. +** Append a record of the current state of page pPg to the sub-journal. ** ** If successful, set the bit corresponding to pPg->pgno in the bitvecs ** for all open savepoints before returning. ** ** This function returns SQLITE_OK if everything is successful, an IO -** error code if the attempt to write to the sub-journal fails, or +** error code if the attempt to write to the sub-journal fails, or ** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint ** bitvec. */ @@ -4597,9 +4660,9 @@ static int subjournalPage(PgHdr *pPg){ assert( pPager->useJournal ); assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); - assert( pagerUseWal(pPager) - || pageInJournal(pPager, pPg) - || pPg->pgno>pPager->dbOrigSize + assert( pagerUseWal(pPager) + || pageInJournal(pPager, pPg) + || pPg->pgno>pPager->dbOrigSize ); rc = openSubJournal(pPager); @@ -4644,14 +4707,14 @@ static int subjournalPageIfRequired(PgHdr *pPg){ ** This function is called by the pcache layer when it has reached some ** soft memory limit. The first argument is a pointer to a Pager object ** (cast as a void*). The pager is always 'purgeable' (not an in-memory -** database). The second argument is a reference to a page that is +** database). The second argument is a reference to a page that is ** currently dirty but has no outstanding references. The page -** is always associated with the Pager object passed as the first +** is always associated with the Pager object passed as the first ** argument. ** ** The job of this function is to make pPg clean by writing its contents ** out to the database file, if possible. This may involve syncing the -** journal file. +** journal file. ** ** If successful, sqlite3PcacheMakeClean() is called on the page and ** SQLITE_OK returned. If an IO error occurs while trying to make the @@ -4676,7 +4739,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** a rollback or by user request, respectively. ** ** Spilling is also prohibited when in an error state since that could - ** lead to database corruption. In the current implementation it + ** lead to database corruption. In the current implementation it ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() @@ -4697,26 +4760,26 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ - rc = subjournalPageIfRequired(pPg); + rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } }else{ - + #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE if( pPager->tempFile==0 ){ rc = sqlite3JournalCreate(pPager->jfd); if( rc!=SQLITE_OK ) return pager_error(pPager, rc); } #endif - + /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC + if( pPg->flags&PGHDR_NEED_SYNC || pPager->eState==PAGER_WRITER_CACHEMOD ){ rc = syncJournal(pPager, 1); } - + /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); @@ -4730,7 +4793,7 @@ static int pagerStress(void *p, PgHdr *pPg){ sqlite3PcacheMakeClean(pPg); } - return pager_error(pPager, rc); + return pager_error(pPager, rc); } /* @@ -4761,8 +4824,8 @@ int sqlite3PagerFlush(Pager *pPager){ ** The zFilename argument is the path to the database file to open. ** If zFilename is NULL then a randomly-named temporary file is created ** and used as the file to be cached. Temporary files are be deleted -** automatically when they are closed. If zFilename is ":memory:" then -** all information is held in cache. It is never written to disk. +** automatically when they are closed. If zFilename is ":memory:" then +** all information is held in cache. It is never written to disk. ** This can be used to implement an in-memory database. ** ** The nExtra parameter specifies the number of bytes of space allocated @@ -4776,13 +4839,13 @@ int sqlite3PagerFlush(Pager *pPager){ ** of the PAGER_* flags. ** ** The vfsFlags parameter is a bitmask to pass to the flags parameter -** of the xOpen() method of the supplied VFS when opening files. +** of the xOpen() method of the supplied VFS when opening files. ** -** If the pager object is allocated and the specified file opened +** If the pager object is allocated and the specified file opened ** successfully, SQLITE_OK is returned and *ppPager set to point to ** the new pager object. If an error occurs, *ppPager is set to NULL ** and error code returned. This function may return SQLITE_NOMEM -** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or +** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or ** various SQLITE_IO_XXX errors. */ int sqlite3PagerOpen( @@ -4799,11 +4862,7 @@ int sqlite3PagerOpen( int rc = SQLITE_OK; /* Return code */ int tempFile = 0; /* True for temp files (incl. in-memory files) */ int memDb = 0; /* True if this is an in-memory file */ -#ifndef SQLITE_OMIT_DESERIALIZE int memJM = 0; /* Memory journal mode */ -#else -# define memJM 0 -#endif int readOnly = 0; /* True if this is a read-only file */ int journalFileSize; /* Bytes to allocate for each journal fd */ char *zPathname = 0; /* Full path to database file */ @@ -4813,6 +4872,7 @@ int sqlite3PagerOpen( u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ + /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ @@ -4839,8 +4899,8 @@ int sqlite3PagerOpen( */ if( zFilename && zFilename[0] ){ const char *z; - nPathname = pVfs->mxPathname+1; - zPathname = sqlite3DbMallocRaw(0, nPathname*2); + nPathname = pVfs->mxPathname + 1; + zPathname = sqlite3DbMallocRaw(0, 2*(i64)nPathname); if( zPathname==0 ){ return SQLITE_NOMEM_BKPT; } @@ -4879,7 +4939,7 @@ int sqlite3PagerOpen( } /* Allocate memory for the Pager structure, PCache object, the - ** three file descriptors, the database file name and the journal + ** three file descriptors, the database file name and the journal ** file name. The layout in memory is as follows: ** ** Pager object (sizeof(Pager) bytes) @@ -4922,18 +4982,19 @@ int sqlite3PagerOpen( ** specific formatting and order of the various filenames, so if the format ** changes here, be sure to change it there as well. */ + assert( SQLITE_PTRSIZE==sizeof(Pager*) ); pPtr = (u8 *)sqlite3MallocZero( ROUND8(sizeof(*pPager)) + /* Pager structure */ ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ - journalFileSize * 2 + /* The two journal files */ - sizeof(pPager) + /* Space to hold a pointer */ + (u64)journalFileSize * 2 + /* The two journal files */ + SQLITE_PTRSIZE + /* Space to hold a pointer */ 4 + /* Database prefix */ - nPathname + 1 + /* database filename */ - nUriByte + /* query parameters */ - nPathname + 8 + 1 + /* Journal filename */ + (u64)nPathname + 1 + /* database filename */ + (u64)nUriByte + /* query parameters */ + (u64)nPathname + 8 + 1 + /* Journal filename */ #ifndef SQLITE_OMIT_WAL - nPathname + 4 + 1 + /* WAL filename */ + (u64)nPathname + 4 + 1 + /* WAL filename */ #endif 3 /* Terminator */ ); @@ -4948,7 +5009,7 @@ int sqlite3PagerOpen( pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); - memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); + memcpy(pPtr, &pPager, SQLITE_PTRSIZE); pPtr += SQLITE_PTRSIZE; /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ pPtr += 4; /* Skip zero prefix */ @@ -5002,9 +5063,7 @@ int sqlite3PagerOpen( int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); assert( !memDb ); -#ifndef SQLITE_OMIT_DESERIALIZE pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0; -#endif readOnly = (fout&SQLITE_OPEN_READONLY)!=0; /* If the file was successfully opened for read/write access, @@ -5058,7 +5117,7 @@ int sqlite3PagerOpen( ** disk and uses an in-memory rollback journal. ** ** This branch also runs for files marked as immutable. - */ + */ act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; /* Pretend we already have a lock */ @@ -5067,7 +5126,7 @@ int sqlite3PagerOpen( readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } - /* The following call to PagerSetPagesize() serves to set the value of + /* The following call to PagerSetPagesize() serves to set the value of ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. */ if( rc==SQLITE_OK ){ @@ -5107,10 +5166,10 @@ int sqlite3PagerOpen( /* pPager->state = PAGER_UNLOCK; */ /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; - assert( tempFile==PAGER_LOCKINGMODE_NORMAL + assert( tempFile==PAGER_LOCKINGMODE_NORMAL || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); - pPager->exclusiveMode = (u8)tempFile; + pPager->exclusiveMode = (u8)tempFile; pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; @@ -5141,15 +5200,18 @@ int sqlite3PagerOpen( /* ** Return the sqlite3_file for the main database given the name -** of the corresonding WAL or Journal name as passed into +** of the corresponding WAL or Journal name as passed into ** xOpen. */ sqlite3_file *sqlite3_database_file_object(const char *zName){ Pager *pPager; + const char *p; while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ zName--; } - pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + p = zName - 4 - sizeof(Pager*); + assert( EIGHT_BYTE_ALIGNMENT(p) ); + pPager = *(Pager**)p; return pPager->fd; } @@ -5157,7 +5219,7 @@ sqlite3_file *sqlite3_database_file_object(const char *zName){ /* ** This function is called after transitioning from PAGER_UNLOCK to ** PAGER_SHARED state. It tests if there is a hot journal present in -** the file-system for the given pager. A hot journal is one that +** the file-system for the given pager. A hot journal is one that ** needs to be played back. According to this function, a hot-journal ** file exists if the following criteria are met: ** @@ -5176,10 +5238,10 @@ sqlite3_file *sqlite3_database_file_object(const char *zName){ ** at the end of the file. If there is, and that super-journal file ** does not exist, then the journal file is not really hot. In this ** case this routine will return a false-positive. The pager_playback() -** routine will discover that the journal file is not really hot and -** will not roll it back. +** routine will discover that the journal file is not really hot and +** will not roll it back. ** -** If a hot-journal file is found to exist, *pExists is set to 1 and +** If a hot-journal file is found to exist, *pExists is set to 1 and ** SQLITE_OK returned. If no hot-journal file is present, *pExists is ** set to 0 and SQLITE_OK returned. If an IO error occurs while trying ** to determine whether or not a hot-journal file exists, the IO error @@ -5207,7 +5269,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ int locked = 0; /* True if some process holds a RESERVED lock */ /* Race condition here: Another process might have been holding the - ** the RESERVED lock and have a journal open at the sqlite3OsAccess() + ** the RESERVED lock and have a journal open at the sqlite3OsAccess() ** call above, but then delete the journal and drop the lock before ** we get to the following sqlite3OsCheckReservedLock() call. If that ** is the case, this routine might think there is a hot journal when @@ -5240,7 +5302,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ /* The journal file exists and no other connection has a reserved ** or greater lock on the database file. Now check that there is ** at least one non-zero bytes at the start of the journal file. - ** If there is, then we consider this journal to be hot. If not, + ** If there is, then we consider this journal to be hot. If not, ** it can be ignored. */ if( !jrnlOpen ){ @@ -5290,7 +5352,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** on the database file), then an attempt is made to obtain a ** SHARED lock on the database file. Immediately after obtaining ** the SHARED lock, the file-system is checked for a hot-journal, -** which is played back if present. Following any hot-journal +** which is played back if present. Following any hot-journal ** rollback, the contents of the cache are validated by checking ** the 'change-counter' field of the database file header and ** discarded if they are found to be invalid. @@ -5301,8 +5363,8 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** the contents of the page cache and rolling back any open journal ** file. ** -** If everything is successful, SQLITE_OK is returned. If an IO error -** occurs while locking the database, checking for a hot-journal file or +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or ** rolling back a journal file, the IO error code is returned. */ int sqlite3PagerSharedLock(Pager *pPager){ @@ -5310,7 +5372,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either - ** be OPEN or READER. READER is only possible if the pager is or was in + ** be OPEN or READER. READER is only possible if the pager is or was in ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); @@ -5348,12 +5410,12 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the ** database file, detect the RESERVED lock, and conclude that the - ** database is safe to read while this process is still rolling the + ** database is safe to read while this process is still rolling the ** hot-journal back. - ** + ** ** Because the intermediate RESERVED lock is not requested, any - ** other process attempting to access the database file will get to - ** this point in the code and fail to obtain its own EXCLUSIVE lock + ** other process attempting to access the database file will get to + ** this point in the code and fail to obtain its own EXCLUSIVE lock ** on the database file. ** ** Unless the pager is in locking_mode=exclusive mode, the lock is @@ -5363,17 +5425,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( rc!=SQLITE_OK ){ goto failed; } - - /* If it is not already open and the file exists on disk, open the - ** journal for read/write access. Write access is required because - ** in exclusive-access mode the file descriptor will be kept open - ** and possibly used for a transaction later on. Also, write-access - ** is usually required to finalize the journal in journal_mode=persist + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist ** mode (and also for journal_mode=truncate on some systems). ** - ** If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it ** may mean that the pager was in the error-state when this ** function was called and the journal file does not exist. */ @@ -5394,7 +5456,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ } } } - + /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. Purge the cache before ** playing back the hot-journal so that we don't end up with @@ -5419,8 +5481,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** or roll back a hot-journal while holding an EXCLUSIVE lock. The ** pager_unlock() routine will be called before returning to unlock ** the file. If the unlock attempt fails, then Pager.eLock must be - ** set to UNKNOWN_LOCK (see the comment above the #define for - ** UNKNOWN_LOCK above for an explanation). + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). ** ** In order to get pager_unlock() to do this, set Pager.eState to ** PAGER_ERROR now. This is not actually counted as a transition @@ -5428,7 +5490,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** since we know that the same call to pager_unlock() will very ** shortly transition the pager object to the OPEN state. Calling ** assert_pager_state() would fail now, as it should not be possible - ** to be in ERROR state when there are zero outstanding page + ** to be in ERROR state when there are zero outstanding page ** references. */ pager_error(pPager, rc); @@ -5453,8 +5515,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** a 32-bit counter that is incremented with each change. The ** other bytes change randomly with each file change when ** a codec is in use. - ** - ** There is a vanishingly small chance that a change will not be + ** + ** There is a vanishingly small chance that a change will not be ** detected. The chance of an undetected change is so small that ** it can be neglected. */ @@ -5521,7 +5583,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** Except, in locking_mode=EXCLUSIVE when there is nothing to in ** the rollback journal, the unlock is not performed and there is ** nothing to rollback, so this routine is a no-op. -*/ +*/ static void pagerUnlockIfUnused(Pager *pPager){ if( sqlite3PcacheRefCount(pPager->pPCache)==0 ){ assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */ @@ -5531,7 +5593,7 @@ static void pagerUnlockIfUnused(Pager *pPager){ /* ** The page getter methods each try to acquire a reference to a -** page with page number pgno. If the requested reference is +** page with page number pgno. If the requested reference is ** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. ** ** There are different implementations of the getter method depending @@ -5541,22 +5603,22 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** getPageError() -- Used if the pager is in an error state ** getPageMmap() -- Used if memory-mapped I/O is enabled ** -** If the requested page is already in the cache, it is returned. +** If the requested page is already in the cache, it is returned. ** Otherwise, a new page object is allocated and populated with data ** read from the database file. In some cases, the pcache module may ** choose not to allocate a new page object and may reuse an existing ** object with no outstanding references. ** -** The extra data appended to a page is always initialized to zeros the -** first time a page is loaded into memory. If the page requested is +** The extra data appended to a page is always initialized to zeros the +** first time a page is loaded into memory. If the page requested is ** already in the cache when this function is called, then the extra ** data is left as it was when the page object was last used. ** -** If the database image is smaller than the requested page or if -** the flags parameter contains the PAGER_GET_NOCONTENT bit and the -** requested page is not already stored in the cache, then no -** actual disk read occurs. In this case the memory image of the -** page is initialized to all zeros. +** If the database image is smaller than the requested page or if +** the flags parameter contains the PAGER_GET_NOCONTENT bit and the +** requested page is not already stored in the cache, then no +** actual disk read occurs. In this case the memory image of the +** page is initialized to all zeros. ** ** If PAGER_GET_NOCONTENT is true, it means that we do not care about ** the contents of the page. This occurs in two scenarios: @@ -5627,7 +5689,7 @@ static int getPageNormal( return SQLITE_OK; }else{ - /* The pager cache has created a new page. Its content needs to + /* The pager cache has created a new page. Its content needs to ** be initialized. But first some error checks: ** ** (*) obsolete. Was: maximum page number is 2^31 @@ -5652,9 +5714,9 @@ static int getPageNormal( } if( noContent ){ /* Failure to set the bits in the InJournal bit-vectors is benign. - ** It merely means that we might do some extra work to journal a - ** page that does not need to be journaled. Nevertheless, be sure - ** to test the case where a malloc error occurs while trying to set + ** It merely means that we might do some extra work to journal a + ** page that does not need to be journaled. Nevertheless, be sure + ** to test the case where a malloc error occurs while trying to set ** a bit in a bit vector. */ sqlite3BeginBenignMalloc(); @@ -5704,7 +5766,7 @@ static int getPageMMap( /* It is acceptable to use a read-only (mmap) page for any page except ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY - ** flag was specified by the caller. And so long as the db is not a + ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ const int bMmapOk = (pgno>1 && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) @@ -5738,7 +5800,7 @@ static int getPageMMap( } if( bMmapOk && iFrame==0 ){ void *pData = 0; - rc = sqlite3OsFetch(pPager->fd, + rc = sqlite3OsFetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData ); if( rc==SQLITE_OK && pData ){ @@ -5788,19 +5850,31 @@ int sqlite3PagerGet( DbPage **ppPage, /* Write a pointer to the page here */ int flags /* PAGER_GET_XXX flags */ ){ - /* printf("PAGE %u\n", pgno); fflush(stdout); */ +#if 0 /* Trace page fetch by setting to 1 */ + int rc; + printf("PAGE %u\n", pgno); + fflush(stdout); + rc = pPager->xGet(pPager, pgno, ppPage, flags); + if( rc ){ + printf("PAGE %u failed with 0x%02x\n", pgno, rc); + fflush(stdout); + } + return rc; +#else + /* Normal, high-speed version of sqlite3PagerGet() */ return pPager->xGet(pPager, pgno, ppPage, flags); +#endif } /* ** Acquire a page if it is already in the in-memory cache. Do ** not read the page from disk. Return a pointer to the page, -** or 0 if the page is not in cache. +** or 0 if the page is not in cache. ** ** See also sqlite3PagerGet(). The difference between this routine ** and sqlite3PagerGet() is that _get() will go to the disk and read ** in the page if the page is not already in cache. This routine -** returns NULL if the page is not in cache or if a disk I/O error +** returns NULL if the page is not in cache or if a disk I/O error ** has ever happened. */ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ @@ -5855,24 +5929,24 @@ void sqlite3PagerUnrefPageOne(DbPage *pPg){ /* ** This function is called at the start of every write transaction. -** There must already be a RESERVED or EXCLUSIVE lock on the database +** There must already be a RESERVED or EXCLUSIVE lock on the database ** file when this routine is called. ** ** Open the journal file for pager pPager and write a journal header ** to the start of it. If there are active savepoints, open the sub-journal -** as well. This function is only used when the journal file is being -** opened to write a rollback log for a transaction. It is not used +** as well. This function is only used when the journal file is being +** opened to write a rollback log for a transaction. It is not used ** when opening a hot journal file to roll it back. ** ** If the journal file is already open (as it may be in exclusive mode), ** then this function just writes a journal header to the start of the -** already open file. +** already open file. ** ** Whether or not the journal file is opened by this function, the ** Pager.pInJournal bitvec structure is allocated. ** -** Return SQLITE_OK if everything is successful. Otherwise, return -** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or +** Return SQLITE_OK if everything is successful. Otherwise, return +** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or ** an IO error code if opening or writing the journal file fails. */ static int pager_open_journal(Pager *pPager){ @@ -5882,7 +5956,7 @@ static int pager_open_journal(Pager *pPager){ assert( pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); - + /* If already in the error state, this function is a no-op. But on ** the other hand, this routine is never called if we are already in ** an error state. */ @@ -5893,7 +5967,7 @@ static int pager_open_journal(Pager *pPager){ if( pPager->pInJournal==0 ){ return SQLITE_NOMEM_BKPT; } - + /* Open the journal file if it is not already open. */ if( !isOpen(pPager->jfd) ){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ @@ -5910,7 +5984,7 @@ static int pager_open_journal(Pager *pPager){ flags |= SQLITE_OPEN_MAIN_JOURNAL; nSpill = jrnlBufferSize(pPager); } - + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); @@ -5922,9 +5996,9 @@ static int pager_open_journal(Pager *pPager){ } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); } - - - /* Write the first journal header to the journal file and open + + + /* Write the first journal header to the journal file and open ** the sub-journal if necessary. */ if( rc==SQLITE_OK ){ @@ -5950,12 +6024,12 @@ static int pager_open_journal(Pager *pPager){ } /* -** Begin a write-transaction on the specified pager object. If a +** Begin a write-transaction on the specified pager object. If a ** write-transaction has already been opened, this function is a no-op. ** ** If the exFlag argument is false, then acquire at least a RESERVED ** lock on the database file. If exFlag is true, then acquire at least -** an EXCLUSIVE lock. If such a lock is already held, no locking +** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** ** If the subjInMemory argument is non-zero, then any sub-journal opened @@ -5963,7 +6037,7 @@ static int pager_open_journal(Pager *pPager){ ** has no effect if the sub-journal is already opened (as it may be when ** running in exclusive mode) or if the transaction does not require a ** sub-journal. If the subjInMemory argument is zero, then any required -** sub-journal is implemented in-memory if pPager is an in-memory database, +** sub-journal is implemented in-memory if pPager is an in-memory database, ** or using a temporary file otherwise. */ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ @@ -6011,9 +6085,9 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD ** when it has an open transaction, but never to DBMOD or FINISHED. - ** This is because in those states the code to roll back savepoint - ** transactions may copy data from the sub-journal into the database - ** file as well as into the page cache. Which would be incorrect in + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in ** WAL mode. */ pPager->eState = PAGER_WRITER_LOCKED; @@ -6067,11 +6141,11 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); if( rc!=SQLITE_OK ) return rc; - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, pPager->journalOff, pPager->pageSize)); PAGER_INCR(sqlite3_pager_writej_count); PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, + PAGERID(pPager), pPg->pgno, ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); pPager->journalOff += 8 + pPager->pageSize; @@ -6086,9 +6160,9 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ } /* -** Mark a single data page as writeable. The page is written into the +** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into -** one of the journals, the corresponding bit is set in the +** one of the journals, the corresponding bit is set in the ** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs ** of any open savepoints as appropriate. */ @@ -6096,7 +6170,7 @@ static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - /* This routine is not called unless a write-transaction has already + /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. ** It is never called in the ERROR state. */ @@ -6113,7 +6187,7 @@ static int pager_write(PgHdr *pPg){ ** obtained the necessary locks to begin the write-transaction, but the ** rollback journal might not yet be open. Open it now if this is the case. ** - ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** This is done before calling sqlite3PcacheMakeDirty() on the page. ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then ** an error might occur and the pager would end up in WRITER_LOCKED state ** with pages marked as dirty in the cache. @@ -6158,7 +6232,7 @@ static int pager_write(PgHdr *pPg){ ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. */ pPg->flags |= PGHDR_WRITEABLE; - + /* If the statement journal is open and the page is not in it, ** then write the page into the statement journal. */ @@ -6242,7 +6316,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ } } - /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages ** starting at pg1, then it needs to be set for all of them. Because ** writing to any of these nPage pages may damage the others, the ** journal file must contain sync()ed copies of all of them @@ -6265,9 +6339,9 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ } /* -** Mark a data page as writeable. This routine must be called before -** making changes to a page. The caller must check the return value -** of this function and be careful not to change any page data unless +** Mark a data page as writeable. This routine must be called before +** making changes to a page. The caller must check the return value +** of this function and be careful not to change any page data unless ** this routine returns SQLITE_OK. ** ** The difference between this function and pager_write() is that this @@ -6318,13 +6392,13 @@ int sqlite3PagerIswriteable(DbPage *pPg){ ** on the given page is unused. The pager marks the page as clean so ** that it does not get written to disk. ** -** Tests show that this optimization can quadruple the speed of large +** Tests show that this optimization can quadruple the speed of large ** DELETE operations. ** ** This optimization cannot be used with a temp-file, as the page may ** have been dirty at the start of the transaction. In that case, if -** memory pressure forces page pPg out of the cache, the data does need -** to be written out to disk so that it may be read back in if the +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the ** current transaction is rolled back. */ void sqlite3PagerDontWrite(PgHdr *pPg){ @@ -6340,17 +6414,17 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ } /* -** This routine is called to increment the value of the database file -** change-counter, stored as a 4-byte big-endian integer starting at +** This routine is called to increment the value of the database file +** change-counter, stored as a 4-byte big-endian integer starting at ** byte offset 24 of the pager file. The secondary change counter at ** 92 is also updated, as is the SQLite version number at offset 96. ** ** But this only happens if the pPager->changeCountDone flag is false. ** To avoid excess churning of page 1, the update only happens once. -** See also the pager_write_changecounter() routine that does an +** See also the pager_write_changecounter() routine that does an ** unconditional update of the change counters. ** -** If the isDirectMode flag is zero, then this is done by calling +** If the isDirectMode flag is zero, then this is done by calling ** sqlite3PagerWrite() on page 1, then modifying the contents of the ** page data. In this case the file will be updated when the current ** transaction is committed. @@ -6358,7 +6432,7 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ ** The isDirectMode flag may only be non-zero if the library was compiled ** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, ** if isDirect is non-zero, then the database file is updated directly -** by writing an updated version of page 1 using a call to the +** by writing an updated version of page 1 using a call to the ** sqlite3OsWrite() function. */ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ @@ -6397,7 +6471,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ assert( pPgHdr==0 || rc==SQLITE_OK ); /* If page one was fetched successfully, and this function is not - ** operating in direct-mode, make page 1 writable. When not in + ** operating in direct-mode, make page 1 writable. When not in ** direct mode, page 1 is always held in cache and hence the PagerGet() ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. */ @@ -6458,22 +6532,22 @@ int sqlite3PagerSync(Pager *pPager, const char *zSuper){ /* ** This function may only be called while a write-transaction is active in -** rollback. If the connection is in WAL mode, this call is a no-op. -** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on ** the database file, an attempt is made to obtain one. ** ** If the EXCLUSIVE lock is already held or the attempt to obtain it is ** successful, or the connection is in WAL mode, SQLITE_OK is returned. -** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is ** returned. */ int sqlite3PagerExclusiveLock(Pager *pPager){ int rc = pPager->errCode; assert( assert_pager_state(pPager) ); if( rc==SQLITE_OK ){ - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ @@ -6486,19 +6560,19 @@ int sqlite3PagerExclusiveLock(Pager *pPager){ /* ** Sync the database file for the pager pPager. zSuper points to the name ** of a super-journal file that should be written into the individual -** journal file. zSuper may be NULL, which is interpreted as no +** journal file. zSuper may be NULL, which is interpreted as no ** super-journal (a single database transaction). ** ** This routine ensures that: ** ** * The database file change-counter is updated, ** * the journal is synced (unless the atomic-write optimization is used), -** * all dirty pages are written to the database file, +** * all dirty pages are written to the database file, ** * the database file is truncated (if required), and -** * the database file synced. +** * the database file synced. ** -** The only thing that remains to commit the transaction is to finalize -** (delete, truncate or zero the first part of) the journal file (or +** The only thing that remains to commit the transaction is to finalize +** (delete, truncate or zero the first part of) the journal file (or ** delete the super-journal file if specified). ** ** Note that if zSuper==NULL, this does not overwrite a previous value @@ -6529,7 +6603,7 @@ int sqlite3PagerCommitPhaseOne( /* Provide the ability to easily simulate an I/O error during testing */ if( sqlite3FaultSim(400) ) return SQLITE_IOERR; - PAGERTRACE(("DATABASE SYNC: File=%s zSuper=%s nSize=%d\n", + PAGERTRACE(("DATABASE SYNC: File=%s zSuper=%s nSize=%d\n", pPager->zFilename, zSuper, pPager->dbSize)); /* If no database changes have been made, return early. */ @@ -6580,11 +6654,11 @@ int sqlite3PagerCommitPhaseOne( #ifdef SQLITE_ENABLE_ATOMIC_WRITE /* The following block updates the change-counter. Exactly how it ** does this depends on whether or not the atomic-update optimization - ** was enabled at compile time, and if this transaction meets the - ** runtime criteria to use the operation: + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: ** ** * The file-system supports the atomic-write property for - ** blocks of size page-size, and + ** blocks of size page-size, and ** * This commit is not part of a multi-file transaction, and ** * Exactly one page has been modified and store in the journal file. ** @@ -6594,7 +6668,7 @@ int sqlite3PagerCommitPhaseOne( ** is not applicable to this transaction, call sqlite3JournalCreate() ** to make sure the journal file has actually been created, then call ** pager_incr_changecounter() to update the change-counter in indirect - ** mode. + ** mode. ** ** Otherwise, if the optimization is both enabled and applicable, ** then call pager_incr_changecounter() to update the change-counter @@ -6603,19 +6677,19 @@ int sqlite3PagerCommitPhaseOne( */ if( bBatch==0 ){ PgHdr *pPg; - assert( isOpen(pPager->jfd) - || pPager->journalMode==PAGER_JOURNALMODE_OFF - || pPager->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL ); - if( !zSuper && isOpen(pPager->jfd) - && pPager->journalOff==jrnlBufferSize(pPager) + if( !zSuper && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) && pPager->dbSize>=pPager->dbOrigSize && (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) ){ - /* Update the db file change counter via the direct-write method. The - ** following call will modify the in-memory representation of page 1 - ** to include the updated change counter and then write page 1 - ** directly to the database file. Because of the atomic-write + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write ** property of the host file-system, this is safe. */ rc = pager_incr_changecounter(pPager, 1); @@ -6637,24 +6711,24 @@ int sqlite3PagerCommitPhaseOne( rc = pager_incr_changecounter(pPager, 0); #endif /* !SQLITE_ENABLE_ATOMIC_WRITE */ if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - + /* Write the super-journal name into the journal file. If a - ** super-journal file name has already been written to the journal file, + ** super-journal file name has already been written to the journal file, ** or if zSuper is NULL (no super-journal), then this call is a no-op. */ rc = writeSuperJournal(pPager, zSuper); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - + /* Sync the journal file and write all dirty pages to the database. - ** If the atomic-update optimization is being used, this sync will not + ** If the atomic-update optimization is being used, this sync will not ** create the journal file or perform any real IO. ** ** Because the change-counter page was just modified, unless the ** atomic-update optimization is used it is almost certain that the ** journal requires a sync here. However, in locking_mode=exclusive - ** on a system under memory pressure it is just possible that this is + ** on a system under memory pressure it is just possible that this is ** not the case. In this case it is likely enough that the redundant - ** xSync() call will be changed to a no-op by the OS anyhow. + ** xSync() call will be changed to a no-op by the OS anyhow. */ rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; @@ -6665,6 +6739,13 @@ int sqlite3PagerCommitPhaseOne( rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); if( rc==SQLITE_OK ){ rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbFileSize ){ + char *pTmp = pPager->pTmpSpace; + int szPage = (int)pPager->pageSize; + memset(pTmp, 0, szPage); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, + ((i64)pPager->dbSize*pPager->pageSize)-szPage); + } if( rc==SQLITE_OK ){ rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); } @@ -6695,7 +6776,7 @@ int sqlite3PagerCommitPhaseOne( } sqlite3PcacheCleanAll(pPager->pPCache); - /* If the file on disk is smaller than the database image, use + /* If the file on disk is smaller than the database image, use ** pager_truncate to grow the file here. This can happen if the database ** image was extended as part of the current transaction and then the ** last page in the db image moved to the free-list. In this case the @@ -6707,7 +6788,7 @@ int sqlite3PagerCommitPhaseOne( rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } - + /* Finally, sync the database file. */ if( !noSync ){ rc = sqlite3PagerSync(pPager, zSuper); @@ -6727,12 +6808,12 @@ int sqlite3PagerCommitPhaseOne( /* ** When this function is called, the database file has been completely ** updated to reflect the changes made by the current transaction and -** synced to disk. The journal file still exists in the file-system +** synced to disk. The journal file still exists in the file-system ** though, and if a failure occurs at this point it will eventually ** be used as a hot-journal and the current transaction rolled back. ** -** This function finalizes the journal file, either by deleting, -** truncating or partially zeroing it, so that it cannot be used +** This function finalizes the journal file, either by deleting, +** truncating or partially zeroing it, so that it cannot be used ** for hot-journal rollback. Once this is done the transaction is ** irrevocably committed. ** @@ -6758,15 +6839,15 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** this transaction, the pager is running in exclusive-mode and is ** using persistent journals, then this function is a no-op. ** - ** The start of the journal file currently contains a single journal + ** The start of the journal file currently contains a single journal ** header with the nRec field set to 0. If such a journal is used as ** a hot-journal during hot-journal rollback, 0 changes will be made - ** to the database file. So there is no need to zero the journal + ** to the database file. So there is no need to zero the journal ** header. Since the pager is in exclusive mode, there is no need ** to drop any locks either. */ - if( pPager->eState==PAGER_WRITER_LOCKED - && pPager->exclusiveMode + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); @@ -6780,7 +6861,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } /* -** If a write transaction is open, then all changes made within the +** If a write transaction is open, then all changes made within the ** transaction are reverted and the current write-transaction is closed. ** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR ** state if an error occurs. @@ -6790,14 +6871,14 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** ** Otherwise, in rollback mode, this function performs two functions: ** -** 1) It rolls back the journal file, restoring all database file and +** 1) It rolls back the journal file, restoring all database file and ** in-memory cache pages to the state they were in when the transaction ** was opened, and ** ** 2) It finalizes the journal file, so that it is not used for hot ** rollback at any point in the future. ** -** Finalization of the journal file (task 2) is only performed if the +** Finalization of the journal file (task 2) is only performed if the ** rollback is successful. ** ** In WAL mode, all cache-entries containing data modified within the @@ -6810,7 +6891,7 @@ int sqlite3PagerRollback(Pager *pPager){ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); /* PagerRollback() is a no-op if called in READER or OPEN state. If - ** the pager is already in the ERROR state, the rollback is not + ** the pager is already in the ERROR state, the rollback is not ** attempted here. Instead, the error code is returned to the caller. */ assert( assert_pager_state(pPager) ); @@ -6826,7 +6907,7 @@ int sqlite3PagerRollback(Pager *pPager){ int eState = pPager->eState; rc = pager_end_transaction(pPager, 0, 0); if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ - /* This can happen using journal_mode=off. Move the pager to the error + /* This can happen using journal_mode=off. Move the pager to the error ** state to indicate that the contents of the cache may not be trusted. ** Any active readers will get SQLITE_ABORT. */ @@ -6841,7 +6922,7 @@ int sqlite3PagerRollback(Pager *pPager){ assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT - || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR + || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR || rc==SQLITE_CANTOPEN ); @@ -6899,11 +6980,11 @@ int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->aStat[PAGER_STAT_HIT]; - a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[6] = (int)pPager->aStat[PAGER_STAT_HIT] & 0x7fffffff; + a[7] = (int)pPager->aStat[PAGER_STAT_MISS] & 0x7fffffff; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->aStat[PAGER_STAT_WRITE]; + a[10] = (int)pPager->aStat[PAGER_STAT_WRITE] & 0x7fffffff; return a; } #endif @@ -6915,11 +6996,11 @@ int *sqlite3PagerStats(Pager *pPager){ ** it was added later. ** ** Before returning, *pnVal is incremented by the -** current cache hit or miss count, according to the value of eStat. If the -** reset parameter is non-zero, the cache hit or miss count is zeroed before +** current cache hit or miss count, according to the value of eStat. If the +** reset parameter is non-zero, the cache hit or miss count is zeroed before ** returning. */ -void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ +void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, u64 *pnVal){ assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS @@ -6952,7 +7033,7 @@ int sqlite3PagerIsMemdb(Pager *pPager){ ** to make up the difference. If the number of savepoints is already ** equal to nSavepoint, then this function is a no-op. ** -** If a memory allocation fails, SQLITE_NOMEM is returned. If an error +** If a memory allocation fails, SQLITE_NOMEM is returned. If an error ** occurs while opening the sub-journal file, then an IO error code is ** returned. Otherwise, SQLITE_OK. */ @@ -6967,7 +7048,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ assert( nSavepoint>nCurrent && pPager->useJournal ); /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a + ** if the allocation fails. Otherwise, zero the new portion in case a ** malloc failure occurs while populating it in the for(...) loop below. */ aNew = (PagerSavepoint *)sqlite3Realloc( @@ -7016,7 +7097,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ /* ** This function is called to rollback or release (commit) a savepoint. -** The savepoint to release or rollback need not be the most recently +** The savepoint to release or rollback need not be the most recently ** created savepoint. ** ** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. @@ -7024,29 +7105,29 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes ** that have occurred since the specified savepoint was created. ** -** The savepoint to rollback or release is identified by parameter +** The savepoint to rollback or release is identified by parameter ** iSavepoint. A value of 0 means to operate on the outermost savepoint ** (the first created). A value of (Pager.nSavepoint-1) means operate ** on the most recently created savepoint. If iSavepoint is greater than ** (Pager.nSavepoint-1), then this function is a no-op. ** ** If a negative value is passed to this function, then the current -** transaction is rolled back. This is different to calling +** transaction is rolled back. This is different to calling ** sqlite3PagerRollback() because this function does not terminate -** the transaction or unlock the database, it just restores the -** contents of the database to its original state. +** the transaction or unlock the database, it just restores the +** contents of the database to its original state. ** -** In any case, all savepoints with an index greater than iSavepoint +** In any case, all savepoints with an index greater than iSavepoint ** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), ** then savepoint iSavepoint is also destroyed. ** ** This function may return SQLITE_NOMEM if a memory allocation fails, -** or an IO error code if an IO error occurs while rolling back a +** or an IO error code if an IO error occurs while rolling back a ** savepoint. If no errors occur, SQLITE_OK is returned. -*/ +*/ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ int rc = pPager->errCode; - + #ifdef SQLITE_ENABLE_ZIPVFS if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; #endif @@ -7059,7 +7140,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ int nNew; /* Number of remaining savepoints after this op. */ /* Figure out how many savepoints will still be active after this - ** operation. Store this value in nNew. Then free resources associated + ** operation. Store this value in nNew. Then free resources associated ** with any savepoints that are destroyed by this operation. */ nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); @@ -7092,14 +7173,14 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - + #ifdef SQLITE_ENABLE_ZIPVFS - /* If the cache has been modified but the savepoint cannot be rolled + /* If the cache has been modified but the savepoint cannot be rolled ** back journal_mode=off, put the pager in the error state. This way, ** if the VFS used by this pager includes ZipVFS, the entire transaction ** can be rolled back at the ZipVFS level. */ - else if( - pPager->journalMode==PAGER_JOURNALMODE_OFF + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF && pPager->eState>=PAGER_WRITER_CACHEMOD ){ pPager->errCode = SQLITE_ABORT; @@ -7155,7 +7236,7 @@ sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; @@ -7172,7 +7253,10 @@ const char *sqlite3PagerJournalname(Pager *pPager){ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC /* -** Set or retrieve the codec for this pager +** Set (or overwrite) the codec for this pager. If there is +** already a codec context (pCodec) attached, it WILL NOT be freed. +** The caller is responsible for freeing the old codec context prior +** setting a new one. */ void sqlcipherPagerSetCodec( Pager *pPager, @@ -7181,11 +7265,7 @@ void sqlcipherPagerSetCodec( void (*xCodecFree)(void*), void *pCodec ){ - if( pPager->xCodecFree ){ - pPager->xCodecFree(pPager->pCodec); - }else{ - pager_reset(pPager); - } + pager_reset(pPager); pPager->xCodec = pPager->memDb ? 0 : xCodec; pPager->xCodecSizeChng = xCodecSizeChng; pPager->xCodecFree = xCodecFree; @@ -7193,6 +7273,7 @@ void sqlcipherPagerSetCodec( setGetterMethod(pPager); pagerReportSize(pPager); } +/* Retrieve the codec for this pager */ void *sqlcipherPagerGetCodec(Pager *pPager){ return pPager->pCodec; } @@ -7231,8 +7312,8 @@ void *sqlcipherPagerCodec(PgHdr *pPg){ ** transaction is active). ** ** If the fourth argument, isCommit, is non-zero, then this page is being -** moved as part of a database reorganization just before the transaction -** is being committed. In this case, it is guaranteed that the database page +** moved as part of a database reorganization just before the transaction +** is being committed. In this case, it is guaranteed that the database page ** pPg refers to will not be written to again within this transaction. ** ** This function may return SQLITE_NOMEM or an IO error code if an error @@ -7260,7 +7341,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } /* If the page being moved is dirty and has not been saved by the latest - ** savepoint, then save the current contents of the page into the + ** savepoint, then save the current contents of the page into the ** sub-journal now. This is required to handle the following scenario: ** ** BEGIN; @@ -7283,7 +7364,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return rc; } - PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) @@ -7291,7 +7372,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** be written to, store pPg->pgno in local variable needSyncPgno. ** ** If the isCommit flag is set, there is no need to remember that - ** the journal needs to be sync()ed before database page pPg->pgno + ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ @@ -7302,8 +7383,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for - ** page pgno before the 'move' operation, it needs to be retained + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for + ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ pPg->flags &= ~PGHDR_NEED_SYNC; @@ -7338,9 +7419,9 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ } if( needSyncPgno ){ - /* If needSyncPgno is non-zero, then the journal file needs to be + /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. - ** Currently, no such page exists in the page-cache and the + ** Currently, no such page exists in the page-cache and the ** "is journaled" bitvec flag has been set. This needs to be remedied by ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC ** flag. @@ -7371,9 +7452,9 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ #endif /* -** The page handle passed as the first argument refers to a dirty page -** with a page number other than iNew. This function changes the page's -** page number to iNew and sets the value of the PgHdr.flags field to +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to ** the value passed as the third parameter. */ void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ @@ -7391,7 +7472,7 @@ void *sqlite3PagerGetData(DbPage *pPg){ } /* -** Return a pointer to the Pager.nExtra bytes of "extra" space +** Return a pointer to the Pager.nExtra bytes of "extra" space ** allocated along with the specified page. */ void *sqlite3PagerGetExtra(DbPage *pPg){ @@ -7400,7 +7481,7 @@ void *sqlite3PagerGetExtra(DbPage *pPg){ /* ** Get/set the locking-mode for this pager. Parameter eMode must be one -** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or +** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or ** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then ** the locking-mode is set to the value specified. ** @@ -7474,7 +7555,7 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; - /* When transistioning from TRUNCATE or PERSIST to any other journal + /* When transitioning from TRUNCATE or PERSIST to any other journal ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ @@ -7519,7 +7600,7 @@ int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } assert( state==pPager->eState ); } - }else if( eMode==PAGER_JOURNALMODE_OFF ){ + }else if( eMode==PAGER_JOURNALMODE_OFF || eMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3OsClose(pPager->jfd); } } @@ -7612,7 +7693,7 @@ int sqlite3PagerCheckpoint( } if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, - (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + (eMode<=SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt @@ -7647,7 +7728,7 @@ static int pagerExclusiveLock(Pager *pPager){ eOrigLock = pPager->eLock; rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ - /* If the attempt to grab the exclusive lock failed, release the + /* If the attempt to grab the exclusive lock failed, release the ** pending lock that may have been obtained instead. */ pagerUnlockDb(pPager, eOrigLock); } @@ -7656,7 +7737,7 @@ static int pagerExclusiveLock(Pager *pPager){ } /* -** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in ** exclusive-locking mode when this function is called, take an EXCLUSIVE ** lock on the database file and use heap-memory to store the wal-index ** in. Otherwise, use the normal shared-memory. @@ -7667,8 +7748,8 @@ static int pagerOpenWal(Pager *pPager){ assert( pPager->pWal==0 && pPager->tempFile==0 ); assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); - /* If the pager is already in exclusive-mode, the WAL module will use - ** heap-memory for the wal-index instead of the VFS shared-memory + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory ** implementation. Take the exclusive lock now, before opening the WAL ** file, to make sure this is safe. */ @@ -7676,7 +7757,7 @@ static int pagerOpenWal(Pager *pPager){ rc = pagerExclusiveLock(pPager); } - /* Open the connection to the log file. If this operation fails, + /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ @@ -7684,6 +7765,11 @@ static int pagerOpenWal(Pager *pPager){ pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_OK ){ + sqlite3WalDb(pPager->pWal, pPager->dbWal); + } +#endif } pagerFixMaplimit(pPager); @@ -7698,7 +7784,7 @@ static int pagerOpenWal(Pager *pPager){ ** If the pager passed as the first argument is open on a real database ** file (not a temp file or an in-memory database), and the WAL file ** is not already open, make an attempt to open it now. If successful, -** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** return SQLITE_OK. If an error occurs or the VFS used by the pager does ** not support the xShmXXX() methods, return an error code. *pbOpen is ** not modified in either case. ** @@ -7740,7 +7826,7 @@ int sqlite3PagerOpenWal( ** This function is called to close the connection to the log file prior ** to switching from WAL to rollback mode. ** -** Before closing the log file, this function attempts to take an +** Before closing the log file, this function attempts to take an ** EXCLUSIVE lock on the database file. If this cannot be obtained, an ** error (SQLITE_BUSY) is returned and the log connection is not closed. ** If successful, the EXCLUSIVE lock is not released before returning. @@ -7766,7 +7852,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ rc = pagerOpenWal(pPager); } } - + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on ** the database file, the log and log-summary files will be deleted. */ @@ -7786,7 +7872,7 @@ int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT /* ** If pager pPager is a wal-mode database not in exclusive locking mode, -** invoke the sqlite3WalWriteLock() function on the associated Wal object +** invoke the sqlite3WalWriteLock() function on the associated Wal object ** with the same db and bLock parameters as were passed to this function. ** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. */ @@ -7799,10 +7885,11 @@ int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ } /* -** Set the database handle used by the wal layer to determine if +** Set the database handle used by the wal layer to determine if ** blocking locks are required. */ void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + pPager->dbWal = db; if( pagerUseWal(pPager) ){ sqlite3WalDb(pPager->pWal, db); } @@ -7824,11 +7911,11 @@ int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot){ /* ** If this is a WAL database, store a pointer to pSnapshot. Next time a -** read transaction is opened, attempt to read from the snapshot it +** read transaction is opened, attempt to read from the snapshot it ** identifies. If this is not a WAL database, return an error. */ int sqlite3PagerSnapshotOpen( - Pager *pPager, + Pager *pPager, sqlite3_snapshot *pSnapshot ){ int rc = SQLITE_OK; @@ -7841,7 +7928,7 @@ int sqlite3PagerSnapshotOpen( } /* -** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this +** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this ** is not a WAL database, return an error. */ int sqlite3PagerSnapshotRecover(Pager *pPager){ @@ -7858,7 +7945,7 @@ int sqlite3PagerSnapshotRecover(Pager *pPager){ ** The caller currently has a read transaction open on the database. ** If this is not a WAL database, SQLITE_ERROR is returned. Otherwise, ** this function takes a SHARED lock on the CHECKPOINTER slot and then -** checks if the snapshot passed as the second argument is still +** checks if the snapshot passed as the second argument is still ** available. If so, SQLITE_OK is returned. ** ** If the snapshot is not available, SQLITE_ERROR is returned. Or, if @@ -7902,6 +7989,12 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +int sqlite3PagerWalSystemErrno(Pager *pPager){ + return sqlite3WalSystemErrno(pPager->pWal); +} +#endif + #endif /* SQLITE_OMIT_DISKIO */ /* BEGIN SQLCIPHER */ diff --git a/src/pager.h b/src/pager.h index 102bea1411..b30775a15f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -83,6 +83,22 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +#define isWalMode(x) ((x)==PAGER_JOURNALMODE_WAL) + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods!=0) + /* ** Flags that make up the mask passed to sqlite3PagerGet(). */ @@ -221,7 +237,7 @@ sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); -void sqlite3PagerCacheStat(Pager *, int, int, int *); +void sqlite3PagerCacheStat(Pager *, int, int, u64*); void sqlite3PagerClearCache(Pager*); int sqlite3SectorSize(sqlite3_file *); @@ -251,4 +267,8 @@ void *sqlcipherPagerCodec(DbPage *); # define enable_simulated_io_errors() #endif +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +int sqlite3PagerWalSystemErrno(Pager*); +#endif + #endif /* SQLITE_PAGER_H */ diff --git a/src/parse.y b/src/parse.y index 541b9ffe58..617eb7303b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -15,12 +15,16 @@ ** The canonical source code to this file ("parse.y") is a Lemon grammar ** file that specifies the input grammar and actions to take while parsing. ** That input file is processed by Lemon to generate a C-language -** implementation of a parser for the given grammer. You might be reading +** implementation of a parser for the given grammar. You might be reading ** this comment as part of the translated C-code. Edits should be made ** to the original parse.y sources. */ } +// Function used to enlarge the parser stack, if needed +%realloc parserStackRealloc +%free sqlite3_free + // All token codes are small integers with #defines that begin with "TK_" %token_prefix TK_ @@ -39,13 +43,13 @@ %syntax_error { UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ if( TOKEN.z[0] ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + parserSyntaxError(pParse, &TOKEN); }else{ sqlite3ErrorMsg(pParse, "incomplete input"); } } %stack_overflow { - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); } // The name of the generated procedure that implements the parser @@ -58,6 +62,11 @@ %include { #include "sqliteInt.h" +/* +** Verify that the pParse->isCreate field is set +*/ +#define ASSERT_IS_CREATE assert(pParse->isCreate) + /* ** Disable all error recovery processing in the parser push-down ** automaton. @@ -107,6 +116,13 @@ struct TrigEvent { int a; IdList * b; }; struct FrameBound { int eType; Expr *pExpr; }; +/* +** Generate a syntax error +*/ +static void parserSyntaxError(Parse *pParse, Token *p){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p); +} + /* ** Disable lookaside memory allocation for objects that might be ** shared across database connections. @@ -114,6 +130,10 @@ struct FrameBound { int eType; Expr *pExpr; }; static void disableLookaside(Parse *pParse){ sqlite3 *db = pParse->db; pParse->disableLookaside++; +#ifdef SQLITE_DEBUG + pParse->isCreate = 1; +#endif + memset(&pParse->u1.cr, 0, sizeof(pParse->u1.cr)); DisableLookaside; } @@ -148,8 +168,8 @@ ecmd ::= SEMI. ecmd ::= cmdx SEMI. %ifndef SQLITE_OMIT_EXPLAIN ecmd ::= explain cmdx SEMI. {NEVER-REDUCE} -explain ::= EXPLAIN. { pParse->explain = 1; } -explain ::= EXPLAIN QUERY PLAN. { pParse->explain = 2; } +explain ::= EXPLAIN. { if( pParse->pReprepare==0 ) pParse->explain = 1; } +explain ::= EXPLAIN QUERY PLAN. { if( pParse->pReprepare==0 ) pParse->explain = 2; } %endif SQLITE_OMIT_EXPLAIN cmdx ::= cmd. { sqlite3FinishCoding(pParse); } @@ -186,7 +206,9 @@ cmd ::= create_table create_table_args. create_table ::= createkw temp(T) TABLE ifnotexists(E) nm(Y) dbnm(Z). { sqlite3StartTable(pParse,&Y,&Z,T,0,0,E); } -createkw(A) ::= CREATE(A). {disableLookaside(pParse);} +createkw(A) ::= CREATE(A). { + disableLookaside(pParse); +} %type ifnotexists {int} ifnotexists(A) ::= . {A = 0;} @@ -232,11 +254,13 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} // improve performance and reduce the executable size. The goal here is // to get the "jump" operations in ISNULL through ESCAPE to have numeric // values that are early enough so that all jump operations are clustered -// at the beginning. +// at the beginning. Also, operators like NE and EQ need to be adjacent, +// and all of the comparison operators need to be clustered together. +// Various assert() statements throughout the code enforce these restrictions. // %token ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST. %token CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL. -%token OR AND NOT IS MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. +%token OR AND NOT IS ISNOT MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. %token GT LE LT GE ESCAPE. // The following directive causes tokens ABORT, AFTER, ASC, etc. to @@ -258,6 +282,9 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} CURRENT FOLLOWING PARTITION PRECEDING RANGE UNBOUNDED EXCLUDE GROUPS OTHERS TIES %endif SQLITE_OMIT_WINDOWFUNC +%ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES + WITHIN +%endif SQLITE_ENABLE_ORDERED_SET_AGGREGATES %ifndef SQLITE_OMIT_GENERATED_COLUMNS GENERATED ALWAYS %endif @@ -267,7 +294,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} %wildcard ANY. // Define operator precedence early so that this is the first occurrence -// of the operator tokens in the grammer. Keeping the operators together +// of the operator tokens in the grammar. Keeping the operators together // causes them to be assigned integer values that are close together, // which keeps parser tables smaller. // @@ -296,7 +323,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} // %token_class id ID|INDEXED. -// And "ids" is an identifer-or-string. +// And "ids" is an identifier-or-string. // %token_class ids ID|STRING. @@ -357,7 +384,7 @@ scantok(A) ::= . { // carglist ::= carglist ccons. carglist ::= . -ccons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} +ccons ::= CONSTRAINT nm(X). {ASSERT_IS_CREATE; pParse->u1.cr.constraintName = X;} ccons ::= DEFAULT scantok(A) term(X). {sqlite3AddDefaultValue(pParse,X,A.z,&A.z[A.n]);} ccons ::= DEFAULT LP(A) expr(X) RP(Z). @@ -432,9 +459,9 @@ conslist_opt(A) ::= . {A.n = 0; A.z = 0;} conslist_opt(A) ::= COMMA(A) conslist. conslist ::= conslist tconscomma tcons. conslist ::= tcons. -tconscomma ::= COMMA. {pParse->constraintName.n = 0;} +tconscomma ::= COMMA. {ASSERT_IS_CREATE; pParse->u1.cr.constraintName.n = 0;} tconscomma ::= . -tcons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} +tcons ::= CONSTRAINT nm(X). {ASSERT_IS_CREATE; pParse->u1.cr.constraintName = X;} tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} tcons ::= UNIQUE LP sortlist(X) RP onconf(R). @@ -490,7 +517,11 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { // cmd ::= select(X). { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, X, &dest); + if( (pParse->db->mDbFlags & DBFLAG_EncodingFixed)!=0 + || sqlite3ReadSchema(pParse)==SQLITE_OK + ){ + sqlite3Select(pParse, X, &dest); + } sqlite3SelectDelete(pParse->db, X); } @@ -526,9 +557,9 @@ cmd ::= select(X). { break; } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } @@ -547,19 +578,27 @@ cmd ::= select(X). { } return pSelect; } + + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. + */ + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } } %ifndef SQLITE_OMIT_CTE select(A) ::= WITH wqlist(W) selectnowith(X). {A = attachWithToSelect(pParse,X,W);} select(A) ::= WITH RECURSIVE wqlist(W) selectnowith(X). {A = attachWithToSelect(pParse,X,W);} + %endif /* SQLITE_OMIT_CTE */ -select(A) ::= selectnowith(X). { - Select *p = X; +select(A) ::= selectnowith(A). { + Select *p = A; if( p ){ parserDoubleLinkSelect(pParse, p); } - A = p; /*A-overwrites-X*/ } selectnowith(A) ::= oneselect(A). @@ -578,8 +617,8 @@ selectnowith(A) ::= selectnowith(A) multiselect_op(Y) oneselect(Z). { if( pRhs ){ pRhs->op = (u8)Y; pRhs->pPrior = pLhs; - if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; - pRhs->selFlags &= ~SF_MultiValue; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~(u32)SF_MultiValue; + pRhs->selFlags &= ~(u32)SF_MultiValue; if( Y!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); @@ -611,24 +650,27 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) %endif -oneselect(A) ::= values(A). - +// Single row VALUES clause. +// %type values {Select*} +oneselect(A) ::= values(A). %destructor values {sqlite3SelectDelete(pParse->db, $$);} values(A) ::= VALUES LP nexprlist(X) RP. { A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0); } -values(A) ::= values(A) COMMA LP nexprlist(Y) RP. { - Select *pRight, *pLeft = A; - pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - A = pRight; - }else{ - A = pLeft; - } + +// Multiple row VALUES clause. +// +%type mvalues {Select*} +oneselect(A) ::= mvalues(A). { + sqlite3MultiValuesEnd(pParse, A); +} +%destructor mvalues {sqlite3SelectDelete(pParse->db, $$);} +mvalues(A) ::= values(A) COMMA LP nexprlist(Y) RP. { + A = sqlite3MultiValues(pParse, A, Y); +} +mvalues(A) ::= mvalues(A) COMMA LP nexprlist(Y) RP. { + A = sqlite3MultiValues(pParse, A, Y); } // The "distinct" nonterminal is true (1) if the DISTINCT keyword is @@ -655,14 +697,17 @@ selcollist(A) ::= sclp(A) scanpt(B) expr(X) scanpt(Z) as(Y). { if( Y.n>0 ) sqlite3ExprListSetName(pParse, A, &Y, 1); sqlite3ExprListSetSpan(pParse,A,B,Z); } -selcollist(A) ::= sclp(A) scanpt STAR. { +selcollist(A) ::= sclp(A) scanpt STAR(X). { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + sqlite3ExprSetErrorOffset(p, (int)(X.z - pParse->zTail)); A = sqlite3ExprListAppend(pParse, A, p); } -selcollist(A) ::= sclp(A) scanpt nm(X) DOT STAR. { - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); - Expr *pLeft = tokenExpr(pParse, TK_ID, X); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); +selcollist(A) ::= sclp(A) scanpt nm(X) DOT STAR(Y). { + Expr *pRight, *pLeft, *pDot; + pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + sqlite3ExprSetErrorOffset(pRight, (int)(Y.z - pParse->zTail)); + pLeft = tokenExpr(pParse, TK_ID, X); + pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); A = sqlite3ExprListAppend(pParse,A, pDot); } @@ -720,11 +765,21 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N if( A ){ SrcItem *pNew = &A->a[A->nSrc-1]; SrcItem *pOld = F->a; + assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; - pNew->pSelect = pOld->pSelect; - if( pNew->pSelect && (pNew->pSelect->selFlags & SF_NestedFrom)!=0 ){ - pNew->fg.isNestedFrom = 1; + assert( pOld->fg.fixedSchema==0 ); + if( pOld->fg.isSubquery ){ + pNew->fg.isSubquery = 1; + pNew->u4.pSubq = pOld->u4.pSubq; + pOld->u4.pSubq = 0; + pOld->fg.isSubquery = 0; + assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 ); + if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){ + pNew->fg.isNestedFrom = 1; + } + }else{ + pNew->u4.zDatabase = pOld->u4.zDatabase; + pOld->u4.zDatabase = 0; } if( pOld->fg.isTabFunc ){ pNew->u1.pFuncArg = pOld->u1.pFuncArg; @@ -732,8 +787,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N pOld->fg.isTabFunc = 0; pNew->fg.isTabFunc = 1; } - pOld->zName = pOld->zDatabase = 0; - pOld->pSelect = 0; + pOld->zName = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ @@ -784,7 +838,7 @@ joinop(X) ::= JOIN_KW(A) nm(B) JOIN. joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN. {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/} -// There is a parsing abiguity in an upsert statement that uses a +// There is a parsing ambiguity in an upsert statement that uses a // SELECT on the RHS of a the INSERT: // // INSERT INTO tab SELECT * FROM aaa JOIN bbb ON CONFLICT ... @@ -1120,7 +1174,7 @@ expr(A) ::= VARIABLE(X). { Token t = X; /*A-overwrites-X*/ assert( t.n>=2 ); if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + parserSyntaxError(pParse, &t); A = 0; }else{ A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); @@ -1142,20 +1196,96 @@ expr(A) ::= CAST LP expr(E) AS typetoken(T) RP. { expr(A) ::= idj(X) LP distinct(D) exprlist(Y) RP. { A = sqlite3ExprFunction(pParse, Y, &X, D); } +expr(A) ::= idj(X) LP distinct(D) exprlist(Y) ORDER BY sortlist(O) RP. { + A = sqlite3ExprFunction(pParse, Y, &X, D); + sqlite3ExprAddFunctionOrderBy(pParse, A, O); +} expr(A) ::= idj(X) LP STAR RP. { A = sqlite3ExprFunction(pParse, 0, &X, 0); } +%ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES +%include { + /* Generate an expression node that represents an ordered-set aggregate function. + ** + ** SQLite does not do anything special to evaluate ordered-set aggregates. The + ** aggregate function itself is expected to do any required ordering on its own. + ** This is just syntactic sugar. + ** + ** This syntax: percentile(f) WITHIN GROUP ( ORDER BY y ) + ** + ** Is equivalent to: percentile(y,f) + ** + ** The purpose of this function is to generate an Expr node from the first syntax + ** into a TK_FUNCTION node that looks like it came from the second syntax. + ** + ** Only functions that have the SQLITE_SELFORDER1 property are allowed to do this + ** transformation. Because DISTINCT is not allowed in the ordered-set aggregate + ** syntax, an error is raised if DISTINCT is used. + */ + static Expr *sqlite3ExprAddOrderedsetFunction( + Parse *pParse, /* Parsing context */ + Token *pFuncname, /* Name of the function */ + int isDistinct, /* DISTINCT or ALL qualifier */ + ExprList *pOrig, /* Arguments to the function */ + Expr *pOrderby /* Expression in the ORDER BY clause */ + ){ + ExprList *p; /* Modified argument list */ + Expr *pExpr; /* Final result */ + p = sqlite3ExprListAppend(pParse, 0, pOrderby); + if( pOrig ){ + int i; + for(i=0; inExpr; i++){ + p = sqlite3ExprListAppend(pParse, p, pOrig->a[i].pExpr); + pOrig->a[i].pExpr = 0; + } + sqlite3ExprListDelete(pParse->db, pOrig); + } + pExpr = sqlite3ExprFunction(pParse, p, pFuncname, 0); + if( pParse->nErr==0 ){ + FuncDef *pDef; + u8 enc = ENC(pParse->db); + assert( pExpr!=0 ); /* Because otherwise pParse->nErr would not be zero */ + assert( p!=0 ); /* Because otherwise pParse->nErr would not be zero */ + pDef = sqlite3FindFunction(pParse->db, pExpr->u.zToken, -2, enc, 0); + if( pDef==0 || (pDef->funcFlags & SQLITE_SELFORDER1)==0 ){ + sqlite3ErrorMsg(pParse, "%#T() is not an ordered-set aggregate", pExpr); + }else if( isDistinct==SF_Distinct ){ + sqlite3ErrorMsg(pParse, "DISTINCT not allowed on ordered-set aggregate %T()", + pFuncname); + } + } + return pExpr; + } +} +expr(A) ::= idj(X) LP distinct(D) exprlist(Y) RP WITHIN GROUP LP ORDER BY expr(E) RP. { + A = sqlite3ExprAddOrderedsetFunction(pParse, &X, D, Y, E); +} +%endif SQLITE_ENABLE_ORDERED_SET_AGGREGATES + %ifndef SQLITE_OMIT_WINDOWFUNC expr(A) ::= idj(X) LP distinct(D) exprlist(Y) RP filter_over(Z). { A = sqlite3ExprFunction(pParse, Y, &X, D); sqlite3WindowAttach(pParse, A, Z); } +expr(A) ::= idj(X) LP distinct(D) exprlist(Y) ORDER BY sortlist(O) RP filter_over(Z). { + A = sqlite3ExprFunction(pParse, Y, &X, D); + sqlite3WindowAttach(pParse, A, Z); + sqlite3ExprAddFunctionOrderBy(pParse, A, O); +} expr(A) ::= idj(X) LP STAR RP filter_over(Z). { A = sqlite3ExprFunction(pParse, 0, &X, 0); sqlite3WindowAttach(pParse, A, Z); } -%endif +%ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES +expr(A) ::= idj(X) LP distinct(D) exprlist(Y) RP WITHIN GROUP LP ORDER BY expr(E) RP + filter_over(Z). { + A = sqlite3ExprAddOrderedsetFunction(pParse, &X, D, Y, E); + sqlite3WindowAttach(pParse, A, Z); +} +%endif SQLITE_ENABLE_ORDERED_SET_AGGREGATES + +%endif SQLITE_OMIT_WINDOWFUNC term(A) ::= CTIME_KW(OP). { A = sqlite3ExprFunction(pParse, 0, &OP, 0); @@ -1255,8 +1385,17 @@ expr(A) ::= NOT(B) expr(X). expr(A) ::= BITNOT(B) expr(X). {A = sqlite3PExpr(pParse, @B, X, 0);/*A-overwrites-B*/} expr(A) ::= PLUS|MINUS(B) expr(X). [BITNOT] { - A = sqlite3PExpr(pParse, @B==TK_PLUS ? TK_UPLUS : TK_UMINUS, X, 0); - /*A-overwrites-B*/ + Expr *p = X; + u8 op = @B + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + A = p; + }else{ + A = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } } expr(A) ::= expr(B) PTR(C) expr(D). { @@ -1290,15 +1429,24 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 IN () ** expr1 NOT IN () ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. + ** simplify to constants 0 (false) and 1 (true), respectively. + ** + ** Except, do not apply this optimization if expr1 contains a function + ** because that function might be an aggregate (we don't know yet whether + ** it is or not) and if it is an aggregate, that could change the meaning + ** of the whole query. */ - sqlite3ExprUnmapAndDelete(pParse, A); - A = sqlite3Expr(pParse->db, TK_STRING, N ? "true" : "false"); - if( A ) sqlite3ExprIdToTrueFalse(A); + Expr *pB = sqlite3Expr(pParse->db, TK_STRING, N ? "true" : "false"); + if( pB ) sqlite3ExprIdToTrueFalse(pB); + if( !ExprHasProperty(A, EP_HasFunc) ){ + sqlite3ExprUnmapAndDelete(pParse, A); + A = pB; + }else{ + A = sqlite3PExpr(pParse, N ? TK_OR : TK_AND, pB, A); + } }else{ Expr *pRHS = Y->a[0].pExpr; - if( Y->nExpr==1 && sqlite3ExprIsConstant(pRHS) && A->op!=TK_VECTOR ){ + if( Y->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && A->op!=TK_VECTOR ){ Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); @@ -1531,6 +1679,10 @@ trigger_decl(A) ::= temp(T) TRIGGER ifnotexists(NOERR) nm(B) dbnm(Z) ON fullname(E) foreach_clause when_clause(G). { sqlite3BeginTrigger(pParse, &B, &Z, C, D.a, D.b, E, G, T, NOERR); A = (Z.n==0?B:Z); /*A-overwrites-T*/ +#ifdef SQLITE_DEBUG + assert( pParse->isCreate ); /* Set by createkw reduce action */ + pParse->isCreate = 0; /* But, should not be set for CREATE TRIGGER */ +#endif } %type trigger_time {int} @@ -1622,8 +1774,8 @@ expr(A) ::= RAISE LP IGNORE RP. { A->affExpr = OE_Ignore; } } -expr(A) ::= RAISE LP raisetype(T) COMMA nm(Z) RP. { - A = sqlite3ExprAlloc(pParse->db, TK_RAISE, &Z, 1); +expr(A) ::= RAISE LP raisetype(T) COMMA expr(Z) RP. { + A = sqlite3PExpr(pParse, TK_RAISE, Z, 0); if( A ) { A->affExpr = (char)T; } @@ -1738,9 +1890,10 @@ with ::= WITH RECURSIVE wqlist(W). { sqlite3WithPush(pParse, W, 1); } wqas(A) ::= AS. {A = M10d_Any;} wqas(A) ::= AS MATERIALIZED. {A = M10d_Yes;} wqas(A) ::= AS NOT MATERIALIZED. {A = M10d_No;} -wqitem(A) ::= nm(X) eidlist_opt(Y) wqas(M) LP select(Z) RP. { +wqitem(A) ::= withnm(X) eidlist_opt(Y) wqas(M) LP select(Z) RP. { A = sqlite3CteNew(pParse, &X, Y, Z, M); /*A-overwrites-X*/ } +withnm(A) ::= nm(A). {pParse->bHasWith = 1;} wqlist(A) ::= wqitem(X). { A = sqlite3WithAdd(pParse, 0, X); /*A-overwrites-X*/ } @@ -1753,12 +1906,13 @@ wqlist(A) ::= wqlist(A) COMMA wqitem(X). { // These must be at the end of this file. Specifically, the rules that // introduce tokens WINDOW, OVER and FILTER must appear last. This causes // the integer values assigned to these tokens to be larger than all other -// tokens that may be output by the tokenizer except TK_SPACE and TK_ILLEGAL. +// tokens that may be output by the tokenizer except TK_SPACE, TK_COMMENT, +// and TK_ILLEGAL. // %ifndef SQLITE_OMIT_WINDOWFUNC %type windowdefn_list {Window*} %destructor windowdefn_list {sqlite3WindowListDelete(pParse->db, $$);} -windowdefn_list(A) ::= windowdefn(Z). { A = Z; } +windowdefn_list(A) ::= windowdefn(A). windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). { assert( Z!=0 ); sqlite3WindowChain(pParse, Z, Y); @@ -1814,9 +1968,7 @@ window(A) ::= ORDER BY sortlist(Y) frame_opt(Z). { window(A) ::= nm(W) ORDER BY sortlist(Y) frame_opt(Z). { A = sqlite3WindowAssemble(pParse, Z, 0, Y, &W); } -window(A) ::= frame_opt(Z). { - A = Z; -} +window(A) ::= frame_opt(A). window(A) ::= nm(W) frame_opt(Z). { A = sqlite3WindowAssemble(pParse, Z, 0, 0, &W); } @@ -1903,8 +2055,8 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } TRUEFALSE /* True or false keyword */ ISNOT /* Combination of IS and NOT */ FUNCTION /* A function invocation */ - UMINUS /* Unary minus */ UPLUS /* Unary plus */ + UMINUS /* Unary minus */ TRUTH /* IS TRUE or IS FALSE or IS NOT TRUE or IS NOT FALSE */ REGISTER /* Reference to a VDBE register */ VECTOR /* Vector */ @@ -1914,6 +2066,12 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } SPAN /* The span operator */ ERROR /* An expression containing an error */ . + +term(A) ::= QNUMBER(X). { + A=tokenExpr(pParse,@X,X); + sqlite3DequoteNumber(pParse, A); +} + /* There must be no more than 255 tokens defined above. If this grammar ** is extended with new rules and tokens, they must either be so few in ** number that TK_SPAN is no more than 255, or else the new tokens must @@ -1926,9 +2084,9 @@ filter_clause(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } } /* -** The TK_SPACE and TK_ILLEGAL tokens must be the last two tokens. The -** parser depends on this. Those tokens are not used in any grammar rule. -** They are only used by the tokenizer. Declare them last so that they -** are guaranteed to be the last two tokens +** The TK_SPACE, TK_COMMENT, and TK_ILLEGAL tokens must be the last three +** tokens. The parser depends on this. Those tokens are not used in any +** grammar rule. They are only used by the tokenizer. Declare them last +** so that they are guaranteed to be the last three. */ -%token SPACE ILLEGAL. +%token SPACE COMMENT ILLEGAL. diff --git a/src/pcache.c b/src/pcache.c index 8a96cbeaee..3429284dc9 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -107,7 +107,7 @@ struct PCache { ** Return 1 if pPg is on the dirty list for pCache. Return 0 if not. ** This routine runs inside of assert() statements only. */ -#ifdef SQLITE_DEBUG +#if defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ PgHdr *p; for(p=pCache->pDirty; p; p=p->pDirtyNext){ @@ -115,6 +115,16 @@ static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ } return 0; } +static int pageNotOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 0; + } + return 1; +} +#else +# define pageOnDirtyList(A,B) 1 +# define pageNotOnDirtyList(A,B) 1 #endif /* @@ -135,7 +145,7 @@ int sqlite3PcachePageSanity(PgHdr *pPg){ assert( pCache!=0 ); /* Every page has an associated PCache */ if( pPg->flags & PGHDR_CLEAN ){ assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( !pageOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirty list */ + assert( pageNotOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirtylist */ }else{ assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */ assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg ); @@ -271,7 +281,7 @@ static int numberOfCachePages(PCache *p){ return p->szCache; }else{ i64 n; - /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + /* IMPLEMENTATION-OF: R-59858-46238 If the argument N is negative, then the ** number of cache pages is adjusted to be a number of pages that would ** use approximately abs(N*1024) bytes of memory based on the current ** page size. */ @@ -502,6 +512,7 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; memset(pPgHdr->pExtra, 0, 8); + assert( EIGHT_BYTE_ALIGNMENT( pPgHdr->pExtra ) ); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; pPgHdr->flags = PGHDR_CLEAN; @@ -759,7 +770,7 @@ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ } /* -** Sort the list of pages in accending order by pgno. Pages are +** Sort the list of pages in ascending order by pgno. Pages are ** connected by pDirty pointers. The pDirtyPrev pointers are ** corrupted by this sort. ** diff --git a/src/pcache1.c b/src/pcache1.c index adbe953959..39607328f3 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -64,14 +64,14 @@ ** ** The third case is a chunk of heap memory (defaulting to 100 pages worth) ** that is allocated when the page cache is created. The size of the local -** bulk allocation can be adjusted using +** bulk allocation can be adjusted using ** ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). ** ** If N is positive, then N pages worth of memory are allocated using a single ** sqlite3Malloc() call and that memory is used for the first N pages allocated. ** Or if N is negative, then -1024*N bytes of memory are allocated and used -** for as many pages as can be accomodated. +** for as many pages as can be accommodated. ** ** Only one of (2) or (3) can be used. Once the memory available to (2) or ** (3) is exhausted, subsequent allocations fail over to the general-purpose @@ -89,7 +89,7 @@ typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; /* -** Each cache entry is represented by an instance of the following +** Each cache entry is represented by an instance of the following ** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated ** directly before this structure and is used to cache the page content. ** @@ -100,12 +100,12 @@ typedef struct PGroup PGroup; ** overrun area, so that overreads are harmless. ** ** Variables isBulkLocal and isAnchor were once type "u8". That works, -** but causes a 2-byte gap in the structure for most architectures (since +** but causes a 2-byte gap in the structure for most architectures (since ** pointers must be either 4 or 8-byte aligned). As this structure is located ** in memory directly after the associated page data, if the database is -** corrupt, code at the b-tree layer may overread the page buffer and +** corrupt, code at the b-tree layer may overread the page buffer and ** read part of this structure before the corruption is detected. This -** can cause a valgrind error if the unitialized gap is accessed. Using u16 +** can cause a valgrind error if the uninitialized gap is accessed. Using u16 ** ensures there is no such gap, and therefore no bytes of uninitialized ** memory in the structure. ** @@ -133,7 +133,7 @@ struct PgHdr1 { #define PAGE_IS_PINNED(p) ((p)->pLruNext==0) #define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0) -/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set +/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of ** the following object. @@ -169,13 +169,13 @@ struct PGroup { ** temporary or transient database) has a single page cache which ** is an instance of this object. ** -** Pointers to structures of this type are cast and returned as +** Pointers to structures of this type are cast and returned as ** opaque sqlite3_pcache* handles. */ struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable ** flag (bPurgeable) and the pnPurgeable pointer are all set when the - ** cache is created and are never changed thereafter. nMax may be + ** cache is created and are never changed thereafter. nMax may be ** modified at any time by a call to the pcache1Cachesize() method. ** The PGroup mutex must be held when accessing nMax. */ @@ -223,7 +223,7 @@ static SQLITE_WSD struct PCacheGlobal { */ int isInit; /* True if initialized */ int separateCache; /* Use a new PGroup for each PCache */ - int nInitPage; /* Initial bulk allocation size */ + int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ @@ -232,10 +232,6 @@ static SQLITE_WSD struct PCacheGlobal { sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ int nFreeSlot; /* Number of unused pcache slots */ - /* The following value requires a mutex to change. We skip the mutex on - ** reading because (1) most platforms read a 32-bit integer atomically and - ** (2) even if an incorrect value is read, no great harm is done since this - ** is really just an optimization. */ int bUnderPressure; /* True if low on PAGECACHE memory */ } pcache1_g; @@ -264,7 +260,7 @@ static SQLITE_WSD struct PCacheGlobal { /* -** This function is called during initialization if a static buffer is +** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE ** verb to sqlite3_config(). Parameter pBuf points to an allocation large ** enough to contain 'n' buffers of 'sz' bytes each. @@ -283,7 +279,7 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; pcache1.pFree = 0; - pcache1.bUnderPressure = 0; + AtomicStore(&pcache1.bUnderPressure,0); while( n-- ){ p = (PgFreeslot*)pBuf; p->pNext = pcache1.pFree; @@ -320,7 +316,8 @@ static int pcache1InitBulk(PCache1 *pCache){ do{ PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage]; pX->page.pBuf = zBulk; - pX->page.pExtra = &pX[1]; + pX->page.pExtra = (u8*)pX + ROUND8(sizeof(*pX)); + assert( EIGHT_BYTE_ALIGNMENT( pX->page.pExtra ) ); pX->isBulkLocal = 1; pX->isAnchor = 0; pX->pNext = pCache->pFree; @@ -334,8 +331,8 @@ static int pcache1InitBulk(PCache1 *pCache){ /* ** Malloc function used within this file to allocate space from the buffer -** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no -** such buffer exists or there is no space left in it, this function falls +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls ** back to sqlite3Malloc(). ** ** Multiple threads can run this routine at the same time. Global variables @@ -350,7 +347,7 @@ static void *pcache1Alloc(int nByte){ if( p ){ pcache1.pFree = pcache1.pFree->pNext; pcache1.nFreeSlot--; - pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); @@ -389,7 +386,7 @@ static void pcache1Free(void *p){ pSlot->pNext = pcache1.pFree; pcache1.pFree = pSlot; pcache1.nFreeSlot++; - pcache1.bUnderPressure = pcache1.nFreeSlotpGroup==&pcache1.grp ); @@ -457,7 +454,8 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ if( pPg==0 ) return 0; p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; p->page.pBuf = pPg; - p->page.pExtra = &p[1]; + p->page.pExtra = (u8*)p + ROUND8(sizeof(*p)); + assert( EIGHT_BYTE_ALIGNMENT( p->page.pExtra ) ); p->isBulkLocal = 0; p->isAnchor = 0; p->pLruPrev = 0; /* Initializing this saves a valgrind error */ @@ -519,7 +517,7 @@ void sqlite3PageFree(void *p){ */ static int pcache1UnderMemoryPressure(PCache1 *pCache){ if( pcache1.nSlot && (pCache->szPage+pCache->szExtra)<=pcache1.szSlot ){ - return pcache1.bUnderPressure; + return AtomicLoad(&pcache1.bUnderPressure); }else{ return sqlite3HeapNearlyFull(); } @@ -536,12 +534,12 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){ */ static void pcache1ResizeHash(PCache1 *p){ PgHdr1 **apNew; - unsigned int nNew; - unsigned int i; + u64 nNew; + u32 i; assert( sqlite3_mutex_held(p->pGroup->mutex) ); - nNew = p->nHash*2; + nNew = 2*(u64)p->nHash; if( nNew<256 ){ nNew = 256; } @@ -569,7 +567,7 @@ static void pcache1ResizeHash(PCache1 *p){ } /* -** This function is used internally to remove the page pPage from the +** This function is used internally to remove the page pPage from the ** PGroup LRU list, if is part of it. If pPage is not part of the PGroup ** LRU list, then this function is a no-op. ** @@ -594,7 +592,7 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ /* -** Remove the page supplied as an argument from the hash table +** Remove the page supplied as an argument from the hash table ** (PCache1.apHash structure) that it is currently stored in. ** Also free the page if freePage is true. ** @@ -637,8 +635,8 @@ static void pcache1EnforceMaxPage(PCache1 *pCache){ } /* -** Discard all pages from cache pCache with a page number (key value) -** greater than or equal to iLimit. Any pinned pages that meet this +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this ** criteria are unpinned before they are discarded. ** ** The PCache mutex must be held when this function is called. @@ -670,7 +668,7 @@ static void pcache1TruncateUnsafe( PgHdr1 **pp; PgHdr1 *pPage; assert( hnHash ); - pp = &pCache->apHash[h]; + pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; @@ -709,7 +707,7 @@ static int pcache1Init(void *NotUsed){ ** ** * Use a unified cache in single-threaded applications that have ** configured a start-time buffer for use as page-cache memory using - ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL ** pBuf argument. ** ** * Otherwise use separate caches (mode-1) @@ -744,7 +742,7 @@ static int pcache1Init(void *NotUsed){ /* ** Implementation of the sqlite3_pcache.xShutdown method. -** Note that the static mutex allocated in xInit does +** Note that the static mutex allocated in xInit does ** not need to be freed. */ static void pcache1Shutdown(void *NotUsed){ @@ -764,7 +762,7 @@ static void pcache1Destroy(sqlite3_pcache *p); static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PCache1 *pCache; /* The newly created page cache */ PGroup *pGroup; /* The group the new page cache will belong to */ - int sz; /* Bytes of memory required to allocate the new cache */ + i64 sz; /* Bytes of memory required to allocate the new cache */ assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); @@ -807,7 +805,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ } /* -** Implementation of the sqlite3_pcache.xCachesize method. +** Implementation of the sqlite3_pcache.xCachesize method. ** ** Configure the cache_size limit for a cache. */ @@ -832,7 +830,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ } /* -** Implementation of the sqlite3_pcache.xShrink method. +** Implementation of the sqlite3_pcache.xShrink method. ** ** Free up as much memory as possible. */ @@ -851,7 +849,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ } /* -** Implementation of the sqlite3_pcache.xPagecount method. +** Implementation of the sqlite3_pcache.xPagecount method. */ static int pcache1Pagecount(sqlite3_pcache *p){ int n; @@ -872,8 +870,8 @@ static int pcache1Pagecount(sqlite3_pcache *p){ ** for these steps, the main pcache1Fetch() procedure can run faster. */ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( - PCache1 *pCache, - unsigned int iKey, + PCache1 *pCache, + unsigned int iKey, int createFlag ){ unsigned int nPinned; @@ -915,8 +913,8 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( } } - /* Step 5. If a usable page buffer has still not been found, - ** attempt to allocate a new one. + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. */ if( !pPage ){ pPage = pcache1AllocPage(pCache, createFlag==1); @@ -941,13 +939,13 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( } /* -** Implementation of the sqlite3_pcache.xFetch method. +** Implementation of the sqlite3_pcache.xFetch method. ** ** Fetch a page by key value. ** ** Whether or not a new page may be allocated by this function depends on ** the value of the createFlag argument. 0 means do not allocate a new -** page. 1 means allocate a new page if space is easily available. 2 +** page. 1 means allocate a new page if space is easily available. 2 ** means to try really hard to allocate a new page. ** ** For a non-purgeable cache (a cache used as the storage for an in-memory @@ -958,7 +956,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** There are three different approaches to obtaining space for a page, ** depending on the value of parameter createFlag (which may be 0, 1 or 2). ** -** 1. Regardless of the value of createFlag, the cache is searched for a +** 1. Regardless of the value of createFlag, the cache is searched for a ** copy of the requested page. If one is found, it is returned. ** ** 2. If createFlag==0 and the page is not already in the cache, NULL is @@ -972,13 +970,13 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** PCache1.nMax, or ** ** (b) the number of pages pinned by the cache is greater than -** the sum of nMax for all purgeable caches, less the sum of +** the sum of nMax for all purgeable caches, less the sum of ** nMin for all other purgeable caches, or ** ** 4. If none of the first three conditions apply and the cache is marked ** as purgeable, and if one of the following is true: ** -** (a) The number of pages allocated for the cache is already +** (a) The number of pages allocated for the cache is already ** PCache1.nMax, or ** ** (b) The number of pages allocated for all purgeable caches is @@ -990,7 +988,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** ** then attempt to recycle a page from the LRU list. If it is the right ** size, return the recycled buffer. Otherwise, free the buffer and -** proceed to step 5. +** proceed to step 5. ** ** 5. Otherwise, allocate and return a new page buffer. ** @@ -1000,8 +998,8 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** invokes the appropriate routine. */ static PgHdr1 *pcache1FetchNoMutex( - sqlite3_pcache *p, - unsigned int iKey, + sqlite3_pcache *p, + unsigned int iKey, int createFlag ){ PCache1 *pCache = (PCache1 *)p; @@ -1030,8 +1028,8 @@ static PgHdr1 *pcache1FetchNoMutex( } #if PCACHE1_MIGHT_USE_GROUP_MUTEX static PgHdr1 *pcache1FetchWithMutex( - sqlite3_pcache *p, - unsigned int iKey, + sqlite3_pcache *p, + unsigned int iKey, int createFlag ){ PCache1 *pCache = (PCache1 *)p; @@ -1045,8 +1043,8 @@ static PgHdr1 *pcache1FetchWithMutex( } #endif static sqlite3_pcache_page *pcache1Fetch( - sqlite3_pcache *p, - unsigned int iKey, + sqlite3_pcache *p, + unsigned int iKey, int createFlag ){ #if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) @@ -1076,18 +1074,18 @@ static sqlite3_pcache_page *pcache1Fetch( ** Mark a page as unpinned (eligible for asynchronous recycling). */ static void pcache1Unpin( - sqlite3_pcache *p, - sqlite3_pcache_page *pPg, + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, int reuseUnlikely ){ PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PGroup *pGroup = pCache->pGroup; - + assert( pPage->pCache==pCache ); pcache1EnterMutex(pGroup); - /* It is an error to call this function if the page is already + /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ assert( pPage->pLruNext==0 ); @@ -1108,7 +1106,7 @@ static void pcache1Unpin( } /* -** Implementation of the sqlite3_pcache.xRekey method. +** Implementation of the sqlite3_pcache.xRekey method. */ static void pcache1Rekey( sqlite3_pcache *p, @@ -1119,7 +1117,7 @@ static void pcache1Rekey( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; - unsigned int hOld, hNew; + unsigned int hOld, hNew; assert( pPage->iKey==iOld ); assert( pPage->pCache==pCache ); assert( iOld!=iNew ); /* The page number really is changing */ @@ -1147,7 +1145,7 @@ static void pcache1Rekey( } /* -** Implementation of the sqlite3_pcache.xTruncate method. +** Implementation of the sqlite3_pcache.xTruncate method. ** ** Discard all unpinned pages in the cache with a page number equal to ** or greater than parameter iLimit. Any pinned pages with a page number @@ -1164,7 +1162,7 @@ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ } /* -** Implementation of the sqlite3_pcache.xDestroy method. +** Implementation of the sqlite3_pcache.xDestroy method. ** ** Destroy a cache allocated using pcache1Create(). */ @@ -1230,7 +1228,7 @@ sqlite3_mutex *sqlite3Pcache1Mutex(void){ ** by the current thread may be sqlite3_free()ed. ** ** nReq is the number of bytes of memory required. Once this much has -** been released, the function returns. The return value is the total number +** been released, the function returns. The return value is the total number ** of bytes of memory released. */ int sqlite3PcacheReleaseMemory(int nReq){ diff --git a/src/pragma.c b/src/pragma.c index 769bdd0fca..84b95fefb3 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -26,13 +26,41 @@ ** that includes the PragType_XXXX macro definitions and the aPragmaName[] ** object. This ensures that the aPragmaName[] table is arranged in ** lexicographical order to facility a binary search of the pragma name. -** Do not edit pragma.h directly. Edit and rerun the script in at +** Do not edit pragma.h directly. Edit and rerun the script in at ** ../tool/mkpragmatab.tcl. */ #include "pragma.h" +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen empirically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or ** unrecognized string argument. The FULL and EXTRA option is disallowed ** if the omitFull parameter it 1. ** @@ -91,7 +119,7 @@ static int getLockingMode(const char *z){ /* ** Interpret the given string as an auto-vacuum mode value. ** -** The following strings, "none", "full" and "incremental" are +** The following strings, "none", "full" and "incremental" are ** acceptable, as are their numeric equivalents: 0, 1 and 2 respectively. */ static int getAutoVacuum(const char *z){ @@ -245,7 +273,7 @@ static const char *actionName(u8 action){ case OE_SetDflt: zName = "SET DEFAULT"; break; case OE_Cascade: zName = "CASCADE"; break; case OE_Restrict: zName = "RESTRICT"; break; - default: zName = "NO ACTION"; + default: zName = "NO ACTION"; assert( action==OE_None ); break; } return zName; @@ -307,7 +335,7 @@ static void pragmaFunclistLine( int isBuiltin, /* True if this is a built-in function */ int showInternFuncs /* True if showing internal functions */ ){ - u32 mask = + u32 mask = SQLITE_DETERMINISTIC | SQLITE_DIRECTONLY | SQLITE_SUBTYPE | @@ -329,7 +357,7 @@ static void pragmaFunclistLine( && showInternFuncs==0 ){ continue; - } + } if( p->xValue!=0 ){ zType = "w"; }else if( p->xFinalize!=0 ){ @@ -364,7 +392,23 @@ static int integrityCheckResultRow(Vdbe *v){ } /* -** Process a pragma statement. +** Should table pTab be skipped when doing an integrity_check? +** Return true or false. +** +** If pObjTab is not null, the return true if pTab matches pObjTab. +** +** If pObjTab is null, then return true only if pTab is an imposter table. +*/ +static int tableSkipIntegrityCheck(const Table *pTab, const Table *pObjTab){ + if( pObjTab ){ + return pTab!=pObjTab; + }else{ + return (pTab->tabFlags & TF_Imposter)!=0; + } +} + +/* +** Process a pragma statement. ** ** Pragmas are of this form: ** @@ -379,7 +423,7 @@ static int integrityCheckResultRow(Vdbe *v){ ** id and pId2 is any empty string. */ void sqlite3Pragma( - Parse *pParse, + Parse *pParse, Token *pId1, /* First part of [schema.]id field */ Token *pId2, /* Second part of [schema.]id field, or NULL */ Token *pValue, /* Token for , or NULL */ @@ -412,8 +456,8 @@ void sqlite3Pragma( if( iDb<0 ) return; pDb = &db->aDb[iDb]; - /* If the temp database has been explicitly named as part of the - ** pragma, make sure it is open. + /* If the temp database has been explicitly named as part of the + ** pragma, make sure it is open. */ if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ return; @@ -495,7 +539,7 @@ void sqlite3Pragma( } /* Register the result column names for pragmas that return results */ - if( (pPragma->mPragFlg & PragFlg_NoColumns)==0 + if( (pPragma->mPragFlg & PragFlg_NoColumns)==0 && ((pPragma->mPragFlg & PragFlg_NoColumns1)==0 || zRight==0) ){ setPragmaResultColumnNames(v, pPragma); @@ -503,7 +547,7 @@ void sqlite3Pragma( /* Jump to the appropriate pragma handler */ switch( pPragma->ePragTyp ){ - + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* ** PRAGMA [schema.]default_cache_size @@ -619,7 +663,7 @@ void sqlite3Pragma( ** PRAGMA [schema.]max_page_count=N ** ** The first form reports the current setting for the - ** maximum number of pages in the database file. The + ** maximum number of pages in the database file. The ** second form attempts to change this setting. Both ** forms return the current setting. ** @@ -786,7 +830,7 @@ void sqlite3Pragma( */ rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ - /* When setting the auto_vacuum mode to either "full" or + /* When setting the auto_vacuum mode to either "full" or ** "incremental", write the value of meta[6] in the database ** file. Before writing to meta[6], check that meta[3] indicates ** that this really is an auto-vacuum capable database. @@ -869,7 +913,7 @@ void sqlite3Pragma( ** ** The first form reports the current local setting for the ** page cache spill size. The second form turns cache spill on - ** or off. When turnning cache spill on, the size is set to the + ** or off. When turning cache spill on, the size is set to the ** current cache_size. The third form sets a spill size that ** may be different form the cache size. ** If N is positive then that is the @@ -888,7 +932,7 @@ void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ returnSingleInt(v, - (db->flags & SQLITE_CacheSpill)==0 ? 0 : + (db->flags & SQLITE_CacheSpill)==0 ? 0 : sqlite3BtreeSetSpillSize(pDb->pBt,0)); }else{ int size = 1; @@ -1068,7 +1112,7 @@ void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); char *proxy_file_path = NULL; sqlite3_file *pFile = sqlite3PagerFile(pPager); - sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); returnSingleText(v, proxy_file_path); }else{ @@ -1076,10 +1120,10 @@ void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); int res; if( zRight[0] ){ - res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, zRight); } else { - res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, NULL); } if( res!=SQLITE_OK ){ @@ -1089,8 +1133,8 @@ void sqlite3Pragma( } break; } -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + /* ** PRAGMA [schema.]synchronous ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA @@ -1105,7 +1149,7 @@ void sqlite3Pragma( returnSingleInt(v, pDb->safety_level-1); }else{ if( !db->autoCommit ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); }else if( iDb!=1 ){ int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; @@ -1131,18 +1175,19 @@ void sqlite3Pragma( ** in auto-commit mode. */ mask &= ~(SQLITE_ForeignKeys); } -#if SQLITE_USER_AUTHENTICATION - if( db->auth.authLevel==UAUTH_User ){ - /* Do not allow non-admin users to modify the schema arbitrarily */ - mask &= ~(SQLITE_WriteSchema); - } -#endif if( sqlite3GetBoolean(zRight, 0) ){ - db->flags |= mask; + if( (mask & SQLITE_WriteSchema)==0 + || (db->flags & SQLITE_Defensive)==0 + ){ + db->flags |= mask; + } }else{ db->flags &= ~mask; - if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + if( mask==SQLITE_DeferFKs ){ + db->nDeferredImmCons = 0; + db->nDeferredCons = 0; + } if( (mask & SQLITE_WriteSchema)!=0 && sqlite3_stricmp(zRight, "reset")==0 ){ @@ -1153,7 +1198,7 @@ void sqlite3Pragma( } } - /* Many of the flag-pragmas modify the code generated by the SQL + /* Many of the flag-pragmas modify the code generated by the SQL ** compiler (eg. count_changes). So add an opcode to expire all ** compiled SQL statements after modifying a pragma value. */ @@ -1268,7 +1313,8 @@ void sqlite3Pragma( char *zSql = sqlite3MPrintf(db, "SELECT*FROM\"%w\"", pTab->zName); if( zSql ){ sqlite3_stmt *pDummy = 0; - (void)sqlite3_prepare(db, zSql, -1, &pDummy, 0); + (void)sqlite3_prepare_v3(db, zSql, -1, SQLITE_PREPARE_DONT_LOG, + &pDummy, 0); (void)sqlite3_finalize(pDummy); sqlite3DbFree(db, zSql); } @@ -1480,7 +1526,7 @@ void sqlite3Pragma( pFK = pTab->u.tab.pFKey; if( pFK ){ int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); - int i = 0; + int i = 0; pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iTabDb); while(pFK){ @@ -1577,8 +1623,8 @@ void sqlite3Pragma( addrOk = sqlite3VdbeMakeLabel(pParse); /* Generate code to read the child key values into registers - ** regRow..regRow+n. If any of the child key values are NULL, this - ** row cannot cause an FK violation. Jump directly to addrOk in + ** regRow..regRow+n. If any of the child key values are NULL, this + ** row cannot cause an FK violation. Jump directly to addrOk in ** this case. */ sqlite3TouchRegister(pParse, regRow + pFK->nCol); for(j=0; jnCol; j++){ @@ -1644,12 +1690,12 @@ void sqlite3Pragma( ** ** Verify the integrity of the database. ** - ** The "quick_check" is reduced version of + ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption ** without the overhead of cross-checking indexes. Quick_check - ** is linear time wherease integrity_check is O(NlogN). + ** is linear time whereas integrity_check is O(NlogN). ** - ** The maximum nubmer of errors is 100 by default. A different default + ** The maximum number of errors is 100 by default. A different default ** can be specified using a numeric parameter N. ** ** Or, the parameter N can be the name of a table. In that case, only @@ -1686,7 +1732,7 @@ void sqlite3Pragma( /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - if( sqlite3GetInt32(zRight, &mxErr) ){ + if( sqlite3GetInt32(pValue->z, &mxErr) ){ if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } @@ -1703,7 +1749,6 @@ void sqlite3Pragma( Hash *pTbls; /* Set of all tables in the schema */ int *aRoot; /* Array of root page numbers of all btrees */ int cnt = 0; /* Number of entries in aRoot[] */ - int mxIdx = 0; /* Maximum number of indexes for any table */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -1722,10 +1767,9 @@ void sqlite3Pragma( Table *pTab = sqliteHashData(x); /* Current table */ Index *pIdx; /* An index on pTab */ int nIdx; /* Number of indexes on pTab */ - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } if( cnt==0 ) continue; if( pObjTab ) cnt++; @@ -1736,7 +1780,7 @@ void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( pObjTab && pObjTab!=pTab ) continue; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ aRoot[++cnt] = pIdx->tnum; @@ -1745,12 +1789,13 @@ void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - sqlite3TouchRegister(pParse, 8+mxIdx); + sqlite3TouchRegister(pParse, 8+cnt); + sqlite3VdbeAddOp3(v, OP_Null, 0, 8, 8+cnt); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); - sqlite3VdbeChangeP5(v, (u8)i); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeChangeP5(v, (u16)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), @@ -1759,6 +1804,36 @@ void sqlite3Pragma( integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ @@ -1772,8 +1847,8 @@ void sqlite3Pragma( int r2; /* Previous key for WITHOUT ROWID tables */ int mxCol; /* Maximum non-virtual column number */ + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; if( !IsOrdinaryTable(pTab) ) continue; - if( pObjTab && pObjTab!=pTab ) continue; if( isQuick || HasRowid(pTab) ){ pPk = 0; r2 = 0; @@ -1785,7 +1860,7 @@ void sqlite3Pragma( sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. - ** reg[8+i] counts the number of entries in the i-th index + ** reg[8+i] counts the number of entries in the i-th index */ sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ @@ -1810,7 +1885,7 @@ void sqlite3Pragma( if( mxCol==pTab->iPKey ) mxCol--; }else{ /* COLFLAG_VIRTUAL columns are not included in the WITHOUT ROWID - ** PK index column-count, so there is no need to account for them + ** PK index column-count, so there is no need to account for them ** in this case. */ mxCol = sqlite3PrimaryKeyIndex(pTab)->nColumn-1; } @@ -1908,9 +1983,10 @@ void sqlite3Pragma( ** is REAL, we have to load the actual data using OP_Column ** to reliably determine if the value is a NULL. */ sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); VdbeCoverage(v); - } + } zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); @@ -1982,7 +2058,7 @@ void sqlite3Pragma( for(k=pCheck->nExpr-1; k>0; k--){ sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); } - sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, SQLITE_JUMPIFNULL); sqlite3VdbeResolveLabel(v, addrCkFault); pParse->iSelfTab = 0; @@ -2052,7 +2128,7 @@ void sqlite3Pragma( sqlite3VdbeGoto(v, jmp5-1); sqlite3VdbeJumpHere(v, jmp6); } - + /* For UNIQUE indexes, verify that only one entry exists with the ** current key. The entry is unique if (1) any column is NULL ** or (2) the next entry has a different key */ @@ -2081,23 +2157,43 @@ void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); - if( !isQuick ){ - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); - integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, addr); - } - if( pPk ){ - sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); - } + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } - } + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( tableSkipIntegrityCheck(pTab,pObjTab) ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; + } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); @@ -2139,7 +2235,7 @@ void sqlite3Pragma( ** encoding that will be used for the main database file if a new file ** is created. If an existing main database file is opened, then the ** default text encoding for the existing database is used. - ** + ** ** In all cases new databases created using the ATTACH command are ** created to use the same default text encoding as the main database. If ** the main database has not been initialized and/or created when ATTACH @@ -2307,6 +2403,8 @@ void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_RESTART; }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ eMode = SQLITE_CHECKPOINT_TRUNCATE; + }else if( sqlite3StrICmp(zRight, "noop")==0 ){ + eMode = SQLITE_CHECKPOINT_NOOP; } } pParse->nMem = 3; @@ -2327,8 +2425,8 @@ void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(v, - db->xWalCallback==sqlite3WalDefaultHook ? + returnSingleInt(v, + db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } break; @@ -2361,44 +2459,63 @@ void sqlite3Pragma( ** ** The optional argument is a bitmask of optimizations to perform: ** - ** 0x0001 Debugging mode. Do not actually perform any optimizations - ** but instead return one line of text for each optimization - ** that would have been done. Off by default. + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. ** - ** 0x0002 Run ANALYZE on tables that might benefit. On by default. - ** See below for additional information. + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. ** - ** 0x0004 (Not yet implemented) Record usage and performance - ** information from the current session in the - ** database file so that it will be available to "optimize" - ** pragmas run by future database connections. + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. ** - ** 0x0008 (Not yet implemented) Create indexes that might have - ** been helpful to recent queries + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. ** - ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all - ** of the optimizations listed above except Debug Mode, including new - ** optimizations that have not yet been invented. If new optimizations are - ** ever added that should be off by default, those off-by-default - ** optimizations will have bitmasks of 0x10000 or larger. + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. ** ** DETERMINATION OF WHEN TO RUN ANALYZE ** ** In the current implementation, a table is analyzed if only if all of ** the following are true: ** - ** (1) MASK bit 0x02 is set. + ** (1) MASK bit 0x00002 is set. + ** + ** (2) The table is an ordinary table, not a virtual table or view. ** - ** (2) The query planner used sqlite_stat1-style statistics for one or - ** more indexes of the table at some point during the lifetime of - ** the current connection. + ** (3) The table name does not begin with "sqlite_". ** - ** (3) One or more indexes of the table are currently unanalyzed OR - ** the number of rows in the table has increased by 25 times or more - ** since the last time ANALYZE was run. + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. + ** + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. ** ** The rules for when tables are analyzed are likely to change in - ** future releases. + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. */ case PragTyp_OPTIMIZE: { int iDbLast; /* Loop termination point for the schema loop */ @@ -2407,9 +2524,13 @@ void sqlite3Pragma( Schema *pSchema; /* The current schema */ Table *pTab; /* A table in the schema */ Index *pIdx; /* An index of the table */ - LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + LogEst szThreshold; /* Size threshold above which reanalysis needed */ char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ if( zRight ){ opMask = (u32)sqlite3Atoi(zRight); @@ -2417,6 +2538,14 @@ void sqlite3Pragma( }else{ opMask = 0xfffe; } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ if( iDb==1 ) continue; @@ -2425,23 +2554,61 @@ void sqlite3Pragma( for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ pTab = (Table*)sqliteHashData(k); - /* If table pTab has not been used in a way that would benefit from - ** having analysis statistics during the current session, then skip it. - ** This also has the effect of skipping virtual tables and views */ - if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; - /* Reanalyze if the table is 25 times larger than the last analysis */ - szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; if( !pIdx->hasStat1 ){ - szThreshold = 0; /* Always analyze if any index lacks statistics */ - break; + szThreshold = -1; /* Always analyze if any index lacks statistics */ } } - if( szThreshold ){ - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, - sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); VdbeCoverage(v); } zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", @@ -2451,11 +2618,27 @@ void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); }else{ - sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); } } } sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddraDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } @@ -2647,7 +2830,7 @@ void sqlite3Pragma( /* The following block is a no-op unless SQLITE_DEBUG is defined. Its only ** purpose is to execute assert() statements to verify that if the ** PragFlg_NoColumns1 flag is set and the caller specified an argument - ** to the PRAGMA, the implementation has not added any OP_ResultRow + ** to the PRAGMA, the implementation has not added any OP_ResultRow ** instructions to the VM. */ if( (pPragma->mPragFlg & PragFlg_NoColumns1) && zRight ){ sqlite3VdbeVerifyNoResultRow(v); @@ -2678,7 +2861,7 @@ struct PragmaVtabCursor { char *azArg[2]; /* Value of the argument and schema */ }; -/* +/* ** Pragma virtual table module xConnect method. */ static int pragmaVtabConnect( @@ -2740,7 +2923,7 @@ static int pragmaVtabConnect( return rc; } -/* +/* ** Pragma virtual table module xDisconnect method. */ static int pragmaVtabDisconnect(sqlite3_vtab *pVtab){ @@ -2768,9 +2951,9 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ seen[0] = 0; seen[1] = 0; for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; j = pConstraint->iColumn - pTab->iHidden; assert( j < 2 ); seen[j] = i+1; @@ -2783,12 +2966,13 @@ static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ j = seen[0]-1; pIdxInfo->aConstraintUsage[j].argvIndex = 1; pIdxInfo->aConstraintUsage[j].omit = 1; - if( seen[1]==0 ) return SQLITE_OK; pIdxInfo->estimatedCost = (double)20; pIdxInfo->estimatedRows = 20; - j = seen[1]-1; - pIdxInfo->aConstraintUsage[j].argvIndex = 2; - pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } return SQLITE_OK; } @@ -2808,6 +2992,7 @@ static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ int i; sqlite3_finalize(pCsr->pPragma); pCsr->pPragma = 0; + pCsr->iRowid = 0; for(i=0; iazArg); i++){ sqlite3_free(pCsr->azArg[i]); pCsr->azArg[i] = 0; @@ -2838,11 +3023,11 @@ static int pragmaVtabNext(sqlite3_vtab_cursor *pVtabCursor){ return rc; } -/* +/* ** Pragma virtual table module xFilter method. */ static int pragmaVtabFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ @@ -2897,11 +3082,11 @@ static int pragmaVtabEof(sqlite3_vtab_cursor *pVtabCursor){ } /* The xColumn method simply returns the corresponding column from -** the PRAGMA. +** the PRAGMA. */ static int pragmaVtabColumn( - sqlite3_vtab_cursor *pVtabCursor, - sqlite3_context *ctx, + sqlite3_vtab_cursor *pVtabCursor, + sqlite3_context *ctx, int i ){ PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; @@ -2914,7 +3099,7 @@ static int pragmaVtabColumn( return SQLITE_OK; } -/* +/* ** Pragma virtual table module xRowid method. */ static int pragmaVtabRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *p){ @@ -2948,7 +3133,8 @@ static const sqlite3_module pragmaVtabModule = { 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ - 0 /* xShadowName */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; /* diff --git a/src/pragma.h b/src/pragma.h deleted file mode 100644 index 35f8e666b9..0000000000 --- a/src/pragma.h +++ /dev/null @@ -1,709 +0,0 @@ -/* DO NOT EDIT! -** This file is automatically generated by the script at -** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit -** that script and rerun it. -*/ - -/* The various pragma types */ -#define PragTyp_KEY 255 -#define PragTyp_ACTIVATE_EXTENSIONS 0 -#define PragTyp_ANALYSIS_LIMIT 1 -#define PragTyp_HEADER_VALUE 2 -#define PragTyp_AUTO_VACUUM 3 -#define PragTyp_FLAG 4 -#define PragTyp_BUSY_TIMEOUT 5 -#define PragTyp_CACHE_SIZE 6 -#define PragTyp_CACHE_SPILL 7 -#define PragTyp_CASE_SENSITIVE_LIKE 8 -#define PragTyp_COLLATION_LIST 9 -#define PragTyp_COMPILE_OPTIONS 10 -#define PragTyp_DATA_STORE_DIRECTORY 11 -#define PragTyp_DATABASE_LIST 12 -#define PragTyp_DEFAULT_CACHE_SIZE 13 -#define PragTyp_ENCODING 14 -#define PragTyp_FOREIGN_KEY_CHECK 15 -#define PragTyp_FOREIGN_KEY_LIST 16 -#define PragTyp_FUNCTION_LIST 17 -#define PragTyp_HARD_HEAP_LIMIT 18 -#define PragTyp_INCREMENTAL_VACUUM 19 -#define PragTyp_INDEX_INFO 20 -#define PragTyp_INDEX_LIST 21 -#define PragTyp_INTEGRITY_CHECK 22 -#define PragTyp_JOURNAL_MODE 23 -#define PragTyp_JOURNAL_SIZE_LIMIT 24 -#define PragTyp_LOCK_PROXY_FILE 25 -#define PragTyp_LOCKING_MODE 26 -#define PragTyp_PAGE_COUNT 27 -#define PragTyp_MMAP_SIZE 28 -#define PragTyp_MODULE_LIST 29 -#define PragTyp_OPTIMIZE 30 -#define PragTyp_PAGE_SIZE 31 -#define PragTyp_PRAGMA_LIST 32 -#define PragTyp_SECURE_DELETE 33 -#define PragTyp_SHRINK_MEMORY 34 -#define PragTyp_SOFT_HEAP_LIMIT 35 -#define PragTyp_SYNCHRONOUS 36 -#define PragTyp_TABLE_INFO 37 -#define PragTyp_TABLE_LIST 38 -#define PragTyp_TEMP_STORE 39 -#define PragTyp_TEMP_STORE_DIRECTORY 40 -#define PragTyp_THREADS 41 -#define PragTyp_WAL_AUTOCHECKPOINT 42 -#define PragTyp_WAL_CHECKPOINT 43 -#define PragTyp_LOCK_STATUS 44 -#define PragTyp_STATS 45 - -/* Property flags associated with various pragma. */ -#define PragFlg_NeedSchema 0x01 /* Force schema load before running */ -#define PragFlg_NoColumns 0x02 /* OP_ResultRow called with zero columns */ -#define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */ -#define PragFlg_ReadOnly 0x08 /* Read-only HEADER_VALUE */ -#define PragFlg_Result0 0x10 /* Acts as query when no argument */ -#define PragFlg_Result1 0x20 /* Acts as query when has one argument */ -#define PragFlg_SchemaOpt 0x40 /* Schema restricts name search if present */ -#define PragFlg_SchemaReq 0x80 /* Schema required - "main" is default */ - -/* Names of columns for pragmas that return multi-column result -** or that return single-column results where the name of the -** result column is different from the name of the pragma -*/ -static const char *const pragCName[] = { - /* 0 */ "id", /* Used by: foreign_key_list */ - /* 1 */ "seq", - /* 2 */ "table", - /* 3 */ "from", - /* 4 */ "to", - /* 5 */ "on_update", - /* 6 */ "on_delete", - /* 7 */ "match", - /* 8 */ "cid", /* Used by: table_xinfo */ - /* 9 */ "name", - /* 10 */ "type", - /* 11 */ "notnull", - /* 12 */ "dflt_value", - /* 13 */ "pk", - /* 14 */ "hidden", - /* table_info reuses 8 */ - /* 15 */ "schema", /* Used by: table_list */ - /* 16 */ "name", - /* 17 */ "type", - /* 18 */ "ncol", - /* 19 */ "wr", - /* 20 */ "strict", - /* 21 */ "seqno", /* Used by: index_xinfo */ - /* 22 */ "cid", - /* 23 */ "name", - /* 24 */ "desc", - /* 25 */ "coll", - /* 26 */ "key", - /* 27 */ "name", /* Used by: function_list */ - /* 28 */ "builtin", - /* 29 */ "type", - /* 30 */ "enc", - /* 31 */ "narg", - /* 32 */ "flags", - /* 33 */ "tbl", /* Used by: stats */ - /* 34 */ "idx", - /* 35 */ "wdth", - /* 36 */ "hght", - /* 37 */ "flgs", - /* 38 */ "seq", /* Used by: index_list */ - /* 39 */ "name", - /* 40 */ "unique", - /* 41 */ "origin", - /* 42 */ "partial", - /* 43 */ "table", /* Used by: foreign_key_check */ - /* 44 */ "rowid", - /* 45 */ "parent", - /* 46 */ "fkid", - /* index_info reuses 21 */ - /* 47 */ "seq", /* Used by: database_list */ - /* 48 */ "name", - /* 49 */ "file", - /* 50 */ "busy", /* Used by: wal_checkpoint */ - /* 51 */ "log", - /* 52 */ "checkpointed", - /* collation_list reuses 38 */ - /* 53 */ "database", /* Used by: lock_status */ - /* 54 */ "status", - /* 55 */ "cache_size", /* Used by: default_cache_size */ - /* module_list pragma_list reuses 9 */ - /* 56 */ "timeout", /* Used by: busy_timeout */ -}; - -/* Definitions of all built-in pragmas */ -typedef struct PragmaName { - const char *const zName; /* Name of pragma */ - u8 ePragTyp; /* PragTyp_XXX value */ - u8 mPragFlg; /* Zero or more PragFlg_XXX values */ - u8 iPragCName; /* Start of column names in pragCName[] */ - u8 nPragCName; /* Num of col names. 0 means use pragma name */ - u64 iArg; /* Extra argument */ -} PragmaName; -static const PragmaName aPragmaName[] = { -#if defined(SQLITE_ENABLE_CEROD) - {/* zName: */ "activate_extensions", - /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif - {/* zName: */ "analysis_limit", - /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "application_id", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_APPLICATION_ID }, -#endif -#if !defined(SQLITE_OMIT_AUTOVACUUM) - {/* zName: */ "auto_vacuum", - /* ePragTyp: */ PragTyp_AUTO_VACUUM, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) - {/* zName: */ "automatic_index", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_AutoIndex }, -#endif -#endif - {/* zName: */ "busy_timeout", - /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 56, 1, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "cache_size", - /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_CACHE_SPILL, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) - {/* zName: */ "case_sensitive_like", - /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, - /* ePragFlg: */ PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif - {/* zName: */ "cell_size_check", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_CellSizeCk }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "checkpoint_fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_CkptFullFSync }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "collation_list", - /* ePragTyp: */ PragTyp_COLLATION_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 38, 2, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) - {/* zName: */ "compile_options", - /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "count_changes", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_CountRows }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN - {/* zName: */ "data_store_directory", - /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, - /* ePragFlg: */ PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "data_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_DATA_VERSION }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "database_list", - /* ePragTyp: */ PragTyp_DATABASE_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 47, 3, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) - {/* zName: */ "default_cache_size", - /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 55, 1, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - {/* zName: */ "defer_foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_DeferFKs }, -#endif -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "empty_result_callbacks", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_NullCallback }, -#endif -#if !defined(SQLITE_OMIT_UTF16) - {/* zName: */ "encoding", - /* ePragTyp: */ PragTyp_ENCODING, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - {/* zName: */ "foreign_key_check", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 43, 4, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FOREIGN_KEY) - {/* zName: */ "foreign_key_list", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 0, 8, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - {/* zName: */ "foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ForeignKeys }, -#endif -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "freelist_count", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_FREE_PAGE_COUNT }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "full_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_FullColNames }, - {/* zName: */ "fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_FullFSync }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) - {/* zName: */ "function_list", - /* ePragTyp: */ PragTyp_FUNCTION_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 27, 6, - /* iArg: */ 0 }, -#endif -#endif - {/* zName: */ "hard_heap_limit", - /* ePragTyp: */ PragTyp_HARD_HEAP_LIMIT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -/* BEGIN SQLCIPHER */ -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "hexkey", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 2 }, - {/* zName: */ "hexrekey", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 3 }, -#endif -/* END SQLCIPHER */ -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_CHECK) - {/* zName: */ "ignore_check_constraints", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_IgnoreChecks }, -#endif -#endif -#if !defined(SQLITE_OMIT_AUTOVACUUM) - {/* zName: */ "incremental_vacuum", - /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "index_info", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 3, - /* iArg: */ 0 }, - {/* zName: */ "index_list", - /* ePragTyp: */ PragTyp_INDEX_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 38, 5, - /* iArg: */ 0 }, - {/* zName: */ "index_xinfo", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 6, - /* iArg: */ 1 }, -#endif -#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - {/* zName: */ "integrity_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "journal_mode", - /* ePragTyp: */ PragTyp_JOURNAL_MODE, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "journal_size_limit", - /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -/* BEGIN SQLCIPHER */ -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "key", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -/* END SQLCIPHER */ -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "legacy_alter_table", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_LegacyAlter }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE - {/* zName: */ "lock_proxy_file", - /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, - /* ePragFlg: */ PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - {/* zName: */ "lock_status", - /* ePragTyp: */ PragTyp_LOCK_STATUS, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 53, 2, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "locking_mode", - /* ePragTyp: */ PragTyp_LOCKING_MODE, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "max_page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "mmap_size", - /* ePragTyp: */ PragTyp_MMAP_SIZE, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) -#if !defined(SQLITE_OMIT_VIRTUALTABLE) -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) - {/* zName: */ "module_list", - /* ePragTyp: */ PragTyp_MODULE_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 9, 1, - /* iArg: */ 0 }, -#endif -#endif -#endif - {/* zName: */ "optimize", - /* ePragTyp: */ PragTyp_OPTIMIZE, - /* ePragFlg: */ PragFlg_Result1|PragFlg_NeedSchema, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "page_size", - /* ePragTyp: */ PragTyp_PAGE_SIZE, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) - {/* zName: */ "parser_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ParserTrace }, -#endif -#endif -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) - {/* zName: */ "pragma_list", - /* ePragTyp: */ PragTyp_PRAGMA_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 9, 1, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "query_only", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_QueryOnly }, -#endif -#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - {/* zName: */ "quick_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "read_uncommitted", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ReadUncommit }, - {/* zName: */ "recursive_triggers", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_RecTriggers }, -#endif -/* BEGIN SQLCIPHER */ -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "rekey", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 1 }, -#endif -/* END SQLCIPHER */ -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "reverse_unordered_selects", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ReverseOrder }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "schema_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_SCHEMA_VERSION }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "secure_delete", - /* ePragTyp: */ PragTyp_SECURE_DELETE, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "short_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ShortColNames }, -#endif - {/* zName: */ "shrink_memory", - /* ePragTyp: */ PragTyp_SHRINK_MEMORY, - /* ePragFlg: */ PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "soft_heap_limit", - /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) - {/* zName: */ "sql_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_SqlTrace }, -#endif -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) - {/* zName: */ "stats", - /* ePragTyp: */ PragTyp_STATS, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 33, 5, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "synchronous", - /* ePragTyp: */ PragTyp_SYNCHRONOUS, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "table_info", - /* ePragTyp: */ PragTyp_TABLE_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 8, 6, - /* iArg: */ 0 }, - {/* zName: */ "table_list", - /* ePragTyp: */ PragTyp_TABLE_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1, - /* ColNames: */ 15, 6, - /* iArg: */ 0 }, - {/* zName: */ "table_xinfo", - /* ePragTyp: */ PragTyp_TABLE_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 8, 7, - /* iArg: */ 1 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "temp_store", - /* ePragTyp: */ PragTyp_TEMP_STORE, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "temp_store_directory", - /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, - /* ePragFlg: */ PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -/* BEGIN SQLCIPHER */ -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "textkey", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 4 }, - {/* zName: */ "textrekey", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 5 }, -#endif -/* END SQLCIPHER */ - {/* zName: */ "threads", - /* ePragTyp: */ PragTyp_THREADS, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "trusted_schema", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_TrustedSchema }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "user_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_USER_VERSION }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) - {/* zName: */ "vdbe_addoptrace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeAddopTrace }, - {/* zName: */ "vdbe_debug", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, - {/* zName: */ "vdbe_eqp", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeEQP }, - {/* zName: */ "vdbe_listing", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeListing }, - {/* zName: */ "vdbe_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeTrace }, -#endif -#endif -#if !defined(SQLITE_OMIT_WAL) - {/* zName: */ "wal_autocheckpoint", - /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "wal_checkpoint", - /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, - /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 50, 3, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "writable_schema", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, -#endif -}; -/* Number of pragmas: 68 on by default, 78 total. */ diff --git a/src/prepare.c b/src/prepare.c index ff48265023..539360b745 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -306,14 +306,7 @@ int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ #else encoding = SQLITE_UTF8; #endif - if( db->nVdbeActive>0 && encoding!=ENC(db) - && (db->mDbFlags & DBFLAG_Vacuum)==0 - ){ - rc = SQLITE_LOCKED; - goto initone_error_out; - }else{ - sqlite3SetTextEncoding(db, encoding); - } + sqlite3SetTextEncoding(db, encoding); }else{ /* If opening an attached database, the encoding much match ENC(db) */ if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ @@ -598,8 +591,6 @@ void sqlite3ParseObjectReset(Parse *pParse){ db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; assert( pParse->db->pParse==pParse ); db->pParse = pParse->pOuterParse; - pParse->db = 0; - pParse->disableLookaside = 0; } /* @@ -608,7 +599,7 @@ void sqlite3ParseObjectReset(Parse *pParse){ ** immediately. ** ** Use this mechanism for uncommon cleanups. There is a higher setup -** cost for this mechansim (an extra malloc), so it should not be used +** cost for this mechanism (an extra malloc), so it should not be used ** for common cleanups that happen on most calls. But for less ** common cleanups, we save a single NULL-pointer comparison in ** sqlite3ParseObjectReset(), which reduces the total CPU cycle count. @@ -635,7 +626,13 @@ void *sqlite3ParserAddCleanup( void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ void *pPtr /* Pointer to object to be cleaned up */ ){ - ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } if( pCleanup ){ pCleanup->pNext = pParse->pCleanup; pParse->pCleanup = pCleanup; @@ -700,7 +697,12 @@ static int sqlite3Prepare( sParse.pOuterParse = db->pParse; db->pParse = &sParse; sParse.db = db; - sParse.pReprepare = pReprepare; + if( pReprepare ){ + sParse.pReprepare = pReprepare; + sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare); + }else{ + assert( sParse.pReprepare==0 ); + } assert( ppStmt && *ppStmt==0 ); if( db->mallocFailed ){ sqlite3ErrorMsg(&sParse, "out of memory"); @@ -858,13 +860,16 @@ static int sqlite3LockAndPrepare( rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); assert( rc==SQLITE_OK || *ppStmt==0 ); if( rc==SQLITE_OK || db->mallocFailed ) break; - }while( (rc==SQLITE_ERROR_RETRY && (cnt++)errMask)==rc ); db->busyHandler.nBusy = 0; sqlite3_mutex_leave(db->mutex); + assert( rc==SQLITE_OK || (*ppStmt)==0 ); return rc; } @@ -997,12 +1002,24 @@ static int sqlite3Prepare16( if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } + + /* Make sure nBytes is non-negative and correct. It should be the + ** number of bytes until the end of the input buffer or until the first + ** U+0000 character. If the input nBytes is odd, convert it into + ** an even number. If the input nBytes is negative, then the input + ** must be terminated by at least one U+0000 character */ if( nBytes>=0 ){ int sz; const char *z = (const char*)zSql; for(sz=0; szmutex); zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ @@ -1016,7 +1033,7 @@ static int sqlite3Prepare16( ** the same number of characters into the UTF-16 string. */ int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); - *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, nBytes, chars_parsed); } sqlite3DbFree(db, zSql8); rc = sqlite3ApiExit(db, rc); diff --git a/src/printf.c b/src/printf.c index 3e1782d466..f75ed3b8a2 100644 --- a/src/printf.c +++ b/src/printf.c @@ -25,17 +25,17 @@ #define etPERCENT 7 /* Percent symbol. %% */ #define etCHARX 8 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', - NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 11 /* a pointer to a Token structure */ -#define etSRCITEM 12 /* a pointer to a SrcItem */ -#define etPOINTER 13 /* The %p conversion */ -#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ -#define etDECIMAL 16 /* %d or %u, but not %x, %o */ +#define etESCAPE_q 9 /* Strings with '\'' doubled. %q */ +#define etESCAPE_Q 10 /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCITEM 12 /* a pointer to a SrcItem */ +#define etPOINTER 13 /* The %p conversion */ +#define etESCAPE_w 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 17 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -54,6 +54,7 @@ typedef struct et_info { /* Information about each format field */ etByte type; /* Conversion paradigm */ etByte charset; /* Offset into aDigits[] of the digits string */ etByte prefix; /* Offset into aPrefix[] of the prefix string */ + char iNxt; /* Next with same hash, or 0 for end of chain */ } et_info; /* @@ -62,100 +63,66 @@ typedef struct et_info { /* Information about each format field */ #define FLAG_SIGNED 1 /* True if the value to convert is signed */ #define FLAG_STRING 4 /* Allow infinite precision */ - /* -** The following table is searched linearly, so it is good to put the -** most frequently used conversion types first. +** The table is searched by hash. In the case of %C where C is the character +** and that character has ASCII value j, then the hash is j%23. +** +** The order of the entries in fmtinfo[] and the hash chain was entered +** manually, but based on the output of the following TCL script: */ +#if 0 /***** Beginning of script ******/ +foreach c {d s g z q Q w c o u x X f e E G i n % p T S r} { + scan $c %c x + set n($c) $x +} +set mx [llength [array names n]] +puts "count: $mx" + +set mx 27 +puts "*********** mx=$mx ************" +for {set r 0} {$r<$mx} {incr r} { + puts -nonewline [format %2d: $r] + foreach c [array names n] { + if {($n($c))%$mx==$r} {puts -nonewline " $c"} + } + puts "" +} +#endif /***** End of script ********/ + static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; -static const et_info fmtinfo[] = { - { 'd', 10, 1, etDECIMAL, 0, 0 }, - { 's', 0, 4, etSTRING, 0, 0 }, - { 'g', 0, 1, etGENERIC, 30, 0 }, - { 'z', 0, 4, etDYNSTRING, 0, 0 }, - { 'q', 0, 4, etSQLESCAPE, 0, 0 }, - { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, - { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, - { 'c', 0, 0, etCHARX, 0, 0 }, - { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etDECIMAL, 0, 0 }, - { 'x', 16, 0, etRADIX, 16, 1 }, - { 'X', 16, 0, etRADIX, 0, 4 }, -#ifndef SQLITE_OMIT_FLOATING_POINT - { 'f', 0, 1, etFLOAT, 0, 0 }, - { 'e', 0, 1, etEXP, 30, 0 }, - { 'E', 0, 1, etEXP, 14, 0 }, - { 'G', 0, 1, etGENERIC, 14, 0 }, -#endif - { 'i', 10, 1, etDECIMAL, 0, 0 }, - { 'n', 0, 0, etSIZE, 0, 0 }, - { '%', 0, 0, etPERCENT, 0, 0 }, - { 'p', 16, 0, etPOINTER, 0, 1 }, - - /* All the rest are undocumented and are for internal use only */ - { 'T', 0, 0, etTOKEN, 0, 0 }, - { 'S', 0, 0, etSRCITEM, 0, 0 }, - { 'r', 10, 1, etORDINAL, 0, 0 }, +static const et_info fmtinfo[23] = { + /* 0 */ { 's', 0, 4, etSTRING, 0, 0, 1 }, + /* 1 */ { 'E', 0, 1, etEXP, 14, 0, 0 }, /* Hash: 0 */ + /* 2 */ { 'u', 10, 0, etDECIMAL, 0, 0, 3 }, + /* 3 */ { 'G', 0, 1, etGENERIC, 14, 0, 0 }, /* Hash: 2 */ + /* 4 */ { 'w', 0, 4, etESCAPE_w, 0, 0, 0 }, + /* 5 */ { 'x', 16, 0, etRADIX, 16, 1, 0 }, + /* 6 */ { 'c', 0, 0, etCHARX, 0, 0, 0 }, /* Hash: 7 */ + /* 7 */ { 'z', 0, 4, etDYNSTRING, 0, 0, 6 }, + /* 8 */ { 'd', 10, 1, etDECIMAL, 0, 0, 0 }, + /* 9 */ { 'e', 0, 1, etEXP, 30, 0, 0 }, + /* 10 */ { 'f', 0, 1, etFLOAT, 0, 0, 0 }, + /* 11 */ { 'g', 0, 1, etGENERIC, 30, 0, 0 }, + /* 12 */ { 'Q', 0, 4, etESCAPE_Q, 0, 0, 0 }, + /* 13 */ { 'i', 10, 1, etDECIMAL, 0, 0, 0 }, + /* 14 */ { '%', 0, 0, etPERCENT, 0, 0, 16 }, + /* 15 */ { 'T', 0, 0, etTOKEN, 0, 0, 0 }, + /* 16 */ { 'S', 0, 0, etSRCITEM, 0, 0, 0 }, /* Hash: 14 */ + /* 17 */ { 'X', 16, 0, etRADIX, 0, 4, 0 }, /* Hash: 19 */ + /* 18 */ { 'n', 0, 0, etSIZE, 0, 0, 0 }, + /* 19 */ { 'o', 8, 0, etRADIX, 0, 2, 17 }, + /* 20 */ { 'p', 16, 0, etPOINTER, 0, 1, 0 }, + /* 21 */ { 'q', 0, 4, etESCAPE_q, 0, 0, 0 }, + /* 22 */ { 'r', 10, 1, etORDINAL, 0, 0, 0 } }; -/* Notes: +/* Additional Notes: ** ** %S Takes a pointer to SrcItem. Shows name or database.name ** %!S Like %S but prefer the zName over the zAlias */ -/* Floating point constants used for rounding */ -static const double arRound[] = { - 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, - 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, -}; - -/* -** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point -** conversions will work. -*/ -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** "*val" is a double such that 0.1 <= *val < 10.0 -** Return the ascii code for the leading digit of *val, then -** multiply "*val" by 10.0 to renormalize. -** -** Example: -** input: *val = 3.14159 -** output: *val = 1.4159 function return = '3' -** -** The counter *cnt is incremented each time. After counter exceeds -** 16 (the number of significant digits in a 64-bit float) '0' is -** always returned. -*/ -static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ - int digit; - LONGDOUBLE_TYPE d; - if( (*cnt)<=0 ) return '0'; - (*cnt)--; - digit = (int)*val; - d = digit; - digit += '0'; - *val = (*val - d)*10.0; - return (char)digit; -} -#endif /* SQLITE_OMIT_FLOATING_POINT */ - -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** "*val" is a u64. *msd is a divisor used to extract the -** most significant digit of *val. Extract that most significant -** digit and return it. -*/ -static char et_getdigit_int(u64 *val, u64 *msd){ - u64 x = (*val)/(*msd); - *val -= x*(*msd); - if( *msd>=10 ) *msd /= 10; - return '0' + (char)(x & 15); -} -#endif /* SQLITE_OMIT_FLOATING_POINT */ - /* ** Set the StrAccum object to an error mode. */ @@ -247,20 +214,15 @@ void sqlite3_str_vappendf( u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ - LONGDOUBLE_TYPE realvalue; /* Value for real types */ - sqlite_uint64 msd; /* Divisor to get most-significant-digit - ** of longvalue */ + double realvalue; /* Value for real types */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ char *zExtra = 0; /* Malloced memory used by some conversion */ -#ifndef SQLITE_OMIT_FLOATING_POINT - int exp, e2; /* exponent of real numbers */ - int nsd; /* Number of significant digits returned */ - double rounder; /* Used for rounding floating point values */ + int exp, e2; /* exponent of real numbers */ etByte flag_dp; /* True if decimal point should be shown */ etByte flag_rtz; /* True if trailing zeros should be removed */ -#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ @@ -282,7 +244,10 @@ void sqlite3_str_vappendf( #if HAVE_STRCHRNUL fmt = strchrnul(fmt, '%'); #else - do{ fmt++; }while( *fmt && *fmt != '%' ); + fmt = strchr(fmt, '%'); + if( fmt==0 ){ + fmt = bufpt + strlen(bufpt); + } #endif sqlite3_str_append(pAccum, bufpt, (int)(fmt - bufpt)); if( *fmt==0 ) break; @@ -396,6 +361,9 @@ void sqlite3_str_vappendf( }while( !done && (c=(*++fmt))!=0 ); /* Fetch the info entry for the field */ +#ifdef SQLITE_EBCDIC + /* The hash table only works for ASCII. For EBCDIC, we need to do + ** a linear search of the table */ infop = &fmtinfo[0]; xtype = etINVALID; for(idx=0; idxtype; + }else{ + infop = &fmtinfo[0]; + xtype = etINVALID; + } +#endif /* ** At this point, variables are initialized as follows: @@ -472,6 +454,14 @@ void sqlite3_str_vappendf( } prefix = 0; } + +#if WHERETRACE_ENABLED + if( xtype==etPOINTER && sqlite3WhereTrace & 0x100000 ) longvalue = 0; +#endif +#if TREETRACE_ENABLED + if( xtype==etPOINTER && sqlite3TreeTrace & 0x100000 ) longvalue = 0; +#endif + if( longvalue==0 ) flag_alternateform = 0; if( flag_zeropad && precisionSQLITE_FP_PRECISION_LIMIT ){ precision = SQLITE_FP_PRECISION_LIMIT; } #endif - if( realvalue<0.0 ){ - realvalue = -realvalue; - prefix = '-'; + if( xtype==etFLOAT ){ + iRound = -precision; + }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; + iRound = precision; }else{ - prefix = flag_prefix; + iRound = precision+1; } - exp = 0; - if( xtype==etGENERIC && precision>0 ) precision--; - testcase( precision>0xfff ); - if( realvalue<1.0e+16 - && realvalue==(LONGDOUBLE_TYPE)(longvalue = (u64)realvalue) - ){ - /* Number is a pure integer that can be represented as u64 */ - for(msd=1; msd*10<=longvalue; msd *= 10, exp++){} - if( exp>precision && xtype!=etFLOAT ){ - u64 rnd = msd/2; - int kk = precision; - while( kk-- > 0 ){ rnd /= 10; } - longvalue += rnd; - } - }else{ - msd = 0; - longvalue = 0; /* To prevent a compiler warning */ - idx = precision & 0xfff; - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } - if( sqlite3IsNaN((double)realvalue) ){ - if( flag_zeropad ){ - bufpt = "null"; - length = 4; + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + if( s.isSpecial ){ + if( s.isSpecial==2 ){ + bufpt = flag_zeropad ? "null" : "NaN"; + length = sqlite3Strlen30(bufpt); + break; + }else if( flag_zeropad ){ + s.z[0] = '9'; + s.iDP = 1000; + s.n = 1; + }else{ + memcpy(buf, "-Inf", 5); + bufpt = buf; + if( s.sign=='-' ){ + /* no-op */ + }else if( flag_prefix ){ + buf[0] = flag_prefix; }else{ - bufpt = "NaN"; - length = 3; + bufpt++; } + length = sqlite3Strlen30(bufpt); break; } - - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - if( ALWAYS(realvalue>0.0) ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ - if( flag_zeropad ){ - realvalue = 9.0; - exp = 999; - }else{ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); - break; - } - } - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } + } + if( s.sign=='-' ){ + if( flag_alternateform + && !flag_prefix + && xtype==etFLOAT + && s.iDP<=iRound + ){ + /* Suppress the minus sign if all of the following are true: + ** * The value displayed is zero + ** * The '#' flag is used + ** * The '+' flag is not used, and + ** * The format is %f + */ + prefix = 0; + }else{ + prefix = '-'; } + }else{ + prefix = flag_prefix; } + exp = s.iDP-1; + /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -642,9 +615,8 @@ void sqlite3_str_vappendf( if( xtype==etEXP ){ e2 = 0; }else{ - e2 = exp; + e2 = s.iDP - 1; } - nsd = 16 + flag_altform2*10; bufpt = buf; { i64 szBufNeeded; /* Size of a temporary buffer needed */ @@ -662,16 +634,12 @@ void sqlite3_str_vappendf( *(bufpt++) = prefix; } /* Digits prior to the decimal point */ + j = 0; if( e2<0 ){ *(bufpt++) = '0'; - }else if( msd>0 ){ - for(; e2>=0; e2--){ - *(bufpt++) = et_getdigit_int(&longvalue,&msd); - if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; - } }else{ for(; e2>=0; e2--){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + *(bufpt++) = j1 ) *(bufpt++) = ','; } } @@ -681,19 +649,12 @@ void sqlite3_str_vappendf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0; precision--, e2++){ - assert( precision>0 ); + for(e2++; e2<0 && precision>0; precision--, e2++){ *(bufpt++) = '0'; } /* Significant digits after the decimal point */ - if( msd>0 ){ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit_int(&longvalue,&msd); - } - }else{ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); - } + while( (precision--)>0 ){ + *(bufpt++) = jcharset]; if( exp<0 ){ *(bufpt++) = '-'; exp = -exp; @@ -742,8 +704,8 @@ void sqlite3_str_vappendf( while( nPad-- ) bufpt[i++] = '0'; length = width; } -#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ break; + } case etSIZE: if( !bArgList ){ *(va_arg(ap,int*)) = pAccum->nChar; @@ -771,25 +733,7 @@ void sqlite3_str_vappendf( } }else{ unsigned int ch = va_arg(ap,unsigned int); - if( ch<0x00080 ){ - buf[0] = ch & 0xff; - length = 1; - }else if( ch<0x00800 ){ - buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); - buf[1] = 0x80 + (u8)(ch & 0x3f); - length = 2; - }else if( ch<0x10000 ){ - buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); - buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); - buf[2] = 0x80 + (u8)(ch & 0x3f); - length = 3; - }else{ - buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); - buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); - buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); - buf[3] = 0x80 + (u8)(ch & 0x3f); - length = 4; - } + length = sqlite3AppendOneUtf8Character(buf, ch); } if( precision>1 ){ i64 nPrior = 1; @@ -869,22 +813,31 @@ void sqlite3_str_vappendf( while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; } break; - case etSQLESCAPE: /* %q: Escape ' characters */ - case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* %w: Escape " characters */ + case etESCAPE_q: /* %q: Escape ' characters */ + case etESCAPE_Q: /* %Q: Escape ' and enclose in '...' */ + case etESCAPE_w: { /* %w: Escape " characters */ i64 i, j, k, n; - int needQuote, isnull; + int needQuote = 0; char ch; - char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; + char q; if( bArgList ){ escarg = getTextArg(pArgList); }else{ escarg = va_arg(ap,char*); } - isnull = escarg==0; - if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + if( escarg==0 ){ + escarg = (xtype==etESCAPE_Q ? "NULL" : "(NULL)"); + }else if( xtype==etESCAPE_Q ){ + needQuote = 1; + } + if( xtype==etESCAPE_w ){ + q = '"'; + flag_alternateform = 0; + }else{ + q = '\''; + } /* For %q, %Q, and %w, the precision is the number of bytes (or ** characters if the ! flags is present) to use from the input. ** Because of the extra quoting characters inserted, the number @@ -897,7 +850,30 @@ void sqlite3_str_vappendf( while( (escarg[i+1]&0xc0)==0x80 ){ i++; } } } - needQuote = !isnull && xtype==etSQLESCAPE2; + if( flag_alternateform ){ + /* For %#q, do unistr()-style backslash escapes for + ** all control characters, and for backslash itself. + ** For %#Q, do the same but only if there is at least + ** one control character. */ + u32 nBack = 0; + u32 nCtrl = 0; + for(k=0; ketBUFSIZE ){ bufpt = zExtra = printfTempBuf(pAccum, n); @@ -906,13 +882,41 @@ void sqlite3_str_vappendf( bufpt = buf; } j = 0; - if( needQuote ) bufpt[j++] = q; + if( needQuote ){ + if( needQuote==2 ){ + memcpy(&bufpt[j], "unistr('", 8); + j += 8; + }else{ + bufpt[j++] = '\''; + } + } k = i; - for(i=0; i=0x10 ? '1' : '0'; + bufpt[j++] = "0123456789abcdef"[ch&0xf]; + } + } + }else{ + for(i=0; izAlias && !flag_altform2 ){ sqlite3_str_appendall(pAccum, pItem->zAlias); }else if( pItem->zName ){ - if( pItem->zDatabase ){ - sqlite3_str_appendall(pAccum, pItem->zDatabase); + if( pItem->fg.fixedSchema==0 + && pItem->fg.isSubquery==0 + && pItem->u4.zDatabase!=0 + ){ + sqlite3_str_appendall(pAccum, pItem->u4.zDatabase); sqlite3_str_append(pAccum, ".", 1); } sqlite3_str_appendall(pAccum, pItem->zName); }else if( pItem->zAlias ){ sqlite3_str_appendall(pAccum, pItem->zAlias); - }else{ - Select *pSel = pItem->pSelect; - assert( pSel!=0 ); + }else if( ALWAYS(pItem->fg.isSubquery) ){/* Because of tag-20240424-1 */ + Select *pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); if( pSel->selFlags & SF_NestedFrom ){ sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); }else{ sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); } @@ -1029,6 +1040,7 @@ void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExpr){ pExpr = pExpr->pLeft; } if( pExpr==0 ) return; + if( ExprHasProperty(pExpr, EP_FromDDL) ) return; db->errByteOffset = pExpr->w.iOfst; } @@ -1147,7 +1159,7 @@ void sqlite3_str_appendall(sqlite3_str *p, const char *z){ static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ char *zText; assert( p->mxAlloc>0 && !isMalloced(p) ); - zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + zText = sqlite3DbMallocRaw(p->db, 1+(u64)p->nChar ); if( zText ){ memcpy(zText, p->zText, p->nChar+1); p->printfFlags |= SQLITE_PRINTF_MALLOCED; @@ -1392,6 +1404,15 @@ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ return zBuf; } +/* Maximum size of an sqlite3_log() message. */ +#if defined(SQLITE_MAX_LOG_MESSAGE) + /* Leave the definition as supplied */ +#elif SQLITE_PRINT_BUF_SIZE*10>10000 +# define SQLITE_MAX_LOG_MESSAGE 10000 +#else +# define SQLITE_MAX_LOG_MESSAGE (SQLITE_PRINT_BUF_SIZE*10) +#endif + /* ** This is the routine that actually formats the sqlite3_log() message. ** We house it in a separate routine from sqlite3_log() to avoid using @@ -1408,7 +1429,7 @@ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ - char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + char zMsg[SQLITE_MAX_LOG_MESSAGE]; /* Complete log message */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); sqlite3_str_vappendf(&acc, zFormat, ap); @@ -1466,3 +1487,72 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ sqlite3_str_vappendf(p, zFormat, ap); va_end(ap); } + + +/***************************************************************************** +** Reference counted string/blob storage +*****************************************************************************/ + +/* +** Increase the reference count of the string by one. +** +** The input parameter is returned. +*/ +char *sqlite3RCStrRef(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + p->nRCRef++; + return z; +} + +/* +** Decrease the reference count by one. Free the string when the +** reference count reaches zero. +*/ +void sqlite3RCStrUnref(void *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + if( p->nRCRef>=2 ){ + p->nRCRef--; + }else{ + sqlite3_free(p); + } +} + +/* +** Create a new string that is capable of holding N bytes of text, not counting +** the zero byte at the end. The string is uninitialized. +** +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the +** newly allocated string. +** +** This routine returns 0 on an OOM. +*/ +char *sqlite3RCStrNew(u64 N){ + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); + if( p==0 ) return 0; + p->nRCRef = 1; + return (char*)&p[1]; +} + +/* +** Change the size of the string so that it is able to hold N bytes. +** The string might be reallocated, so return the new allocation. +*/ +char *sqlite3RCStrResize(char *z, u64 N){ + RCStr *p = (RCStr*)z; + RCStr *pNew; + assert( p!=0 ); + p--; + assert( p->nRCRef==1 ); + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); + if( pNew==0 ){ + sqlite3_free(p); + return 0; + }else{ + return (char*)&pNew[1]; + } +} diff --git a/src/resolve.c b/src/resolve.c index adfcc8dbe9..16c193ca23 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -79,6 +79,8 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( db->mallocFailed ){ @@ -104,21 +106,36 @@ static void resolveAlias( } /* -** Subqueries stores the original database, table and column names for their -** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". -** Check to see if the zSpan given to this routine matches the zDb, zTab, -** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will -** match anything. +** Subqueries store the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN", +** and mark the expression-list item by setting ExprList.a[].fg.eEName +** to ENAME_TAB. +** +** Check to see if the zSpan/eEName of the expression-list item passed to this +** routine matches the zDb, zTab, and zCol. If any of zDb, zTab, and zCol are +** NULL then those fields will match anything. Return true if there is a match, +** or false otherwise. +** +** SF_NestedFrom subqueries also store an entry for the implicit rowid (or +** _rowid_, or oid) column by setting ExprList.a[].fg.eEName to ENAME_ROWID, +** and setting zSpan to "DATABASE.TABLE.". This type of pItem +** argument matches if zCol is a rowid alias. If it is not NULL, (*pbRowid) +** is set to 1 if there is this kind of match. */ int sqlite3MatchEName( const struct ExprList_item *pItem, const char *zCol, const char *zTab, - const char *zDb + const char *zDb, + int *pbRowid ){ int n; const char *zSpan; - if( pItem->fg.eEName!=ENAME_TAB ) return 0; + int eEName = pItem->fg.eEName; + if( eEName!=ENAME_TAB && (eEName!=ENAME_ROWID || NEVER(pbRowid==0)) ){ + return 0; + } + assert( pbRowid==0 || *pbRowid==0 ); zSpan = pItem->zEName; for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ @@ -130,9 +147,11 @@ int sqlite3MatchEName( return 0; } zSpan += n+1; - if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ - return 0; + if( zCol ){ + if( eEName==ENAME_TAB && sqlite3StrICmp(zSpan, zCol)!=0 ) return 0; + if( eEName==ENAME_ROWID && sqlite3IsRowid(zCol)==0 ) return 0; } + if( eEName==ENAME_ROWID ) *pbRowid = 1; return 1; } @@ -165,8 +184,9 @@ Bitmask sqlite3ExprColUsed(Expr *pExpr){ assert( ExprUseYTab(pExpr) ); pExTab = pExpr->y.pTab; assert( pExTab!=0 ); + assert( n < pExTab->nCol ); if( (pExTab->tabFlags & TF_HasGenerated)!=0 - && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 + && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 ){ testcase( pExTab->nCol==BMS-1 ); testcase( pExTab->nCol==BMS ); @@ -195,7 +215,7 @@ static void extendFJMatch( if( pNew ){ pNew->iTable = pMatch->iCursor; pNew->iColumn = iColumn; - pNew->y.pTab = pMatch->pTab; + pNew->y.pTab = pMatch->pSTab; assert( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ); ExprSetProperty(pNew, EP_CanBeNull); *ppList = sqlite3ExprListAppend(pParse, *ppList, pNew); @@ -208,7 +228,7 @@ static void extendFJMatch( static SQLITE_NOINLINE int isValidSchemaTableName( const char *zTab, /* Name as it appears in the SQL */ Table *pTab, /* The schema table we are trying to match */ - Schema *pSchema /* non-NULL if a database qualifier is present */ + const char *zDb /* non-NULL if a database qualifier is present */ ){ const char *zLegacy; assert( pTab!=0 ); @@ -219,7 +239,7 @@ static SQLITE_NOINLINE int isValidSchemaTableName( if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ return 1; } - if( pSchema==0 ) return 0; + if( zDb==0 ) return 0; if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; }else{ @@ -230,7 +250,7 @@ static SQLITE_NOINLINE int isValidSchemaTableName( /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up -** that name in the set of source tables in pSrcList and make the pExpr +** that name in the set of source tables in pSrcList and make the pExpr ** expression node refer back to that source column. The following changes ** are made to pExpr: ** @@ -259,13 +279,13 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ int i, j; /* Loop counters */ int cnt = 0; /* Number of matching column names */ - int cntTab = 0; /* Number of matching table names */ + int cntTab = 0; /* Number of potential "rowid" matches */ int nSubquery = 0; /* How many levels of subquery */ sqlite3 *db = pParse->db; /* The database connection */ SrcItem *pItem; /* Use for looping over pSrcList items */ @@ -274,8 +294,8 @@ static int lookupName( Schema *pSchema = 0; /* Schema of the expression */ int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ Table *pTab = 0; /* Table holding the row */ - Column *pCol; /* A column of pTab */ ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ @@ -324,11 +344,10 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ - u8 hCol; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem)); if( pItem->fg.isNestedFrom ){ /* In this case, pItem is a subquery that has been formed from a ** parenthesized subset of the FROM clause terms. Example: @@ -337,44 +356,61 @@ static int lookupName( ** This pItem -------------^ */ int hit = 0; - assert( pItem->pSelect!=0 ); - pEList = pItem->pSelect->pEList; + Select *pSel; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); + pEList = pSel->pEList; assert( pEList!=0 ); assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ - if( !sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb) ){ + int bRowid = 0; /* True if possible rowid match */ + if( !sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb, &bRowid) ){ continue; } - if( cnt>0 ){ - if( pItem->fg.isUsing==0 - || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 - ){ - /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else - if( (pItem->fg.jointype & JT_RIGHT)==0 ){ - /* An INNER or LEFT JOIN. Use the left-most table */ - continue; - }else - if( (pItem->fg.jointype & JT_LEFT)==0 ){ - /* A RIGHT JOIN. Use the right-most table */ - cnt = 0; - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else{ - /* For a FULL JOIN, we must construct a coalesce() func */ - extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + if( bRowid==0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + || pMatch==pItem + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. Or, a single table has two columns + ** that match a USING term (if pMatch==pItem). These are both + ** "ambiguous column name" errors. Signal as much by clearing + ** pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + } } + cnt++; + hit = 1; + }else if( cnt>0 ){ + /* This is a potential rowid match, but there has already been + ** a real match found. So this can be ignored. */ + continue; } - cnt++; - cntTab = 2; + cntTab++; pMatch = pItem; pExpr->iColumn = j; pEList->a[j].fg.bUsed = 1; - hit = 1; + + /* rowid cannot be part of a USING clause - assert() this. */ + assert( bRowid==0 || pEList->a[j].fg.bUsingTerm==0 ); if( pEList->a[j].fg.bUsingTerm ) break; } if( hit || zTab==0 ) continue; @@ -391,61 +427,85 @@ static int lookupName( } }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ if( pTab->tnum!=1 ) continue; - if( !isValidSchemaTableName(zTab, pTab, pSchema) ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; } assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT && pItem->zAlias ){ sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); } } - hCol = sqlite3StrIHash(zCol); - for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( pCol->hName==hCol - && sqlite3StrICmp(pCol->zCnName, zCol)==0 - ){ - if( cnt>0 ){ - if( pItem->fg.isUsing==0 - || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 - ){ - /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else - if( (pItem->fg.jointype & JT_RIGHT)==0 ){ - /* An INNER or LEFT JOIN. Use the left-most table */ - continue; - }else - if( (pItem->fg.jointype & JT_LEFT)==0 ){ - /* A RIGHT JOIN. Use the right-most table */ - cnt = 0; - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else{ - /* For a FULL JOIN, we must construct a coalesce() func */ - extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); - } - } - cnt++; - pMatch = pItem; - /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ - pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; - if( pItem->fg.isNestedFrom ){ - sqlite3SrcItemColumnUsed(pItem, j); + j = sqlite3ColumnIndex(pTab, zCol); + if( j>=0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); } - break; + } + cnt++; + pMatch = pItem; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + if( pItem->fg.isNestedFrom ){ + sqlite3SrcItemColumnUsed(pItem, j); } } if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && pMatch!=0 + && ALWAYS(pMatch->pSTab!=0) + && (pMatch->pSTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ cntTab++; pMatch = pItem; +#endif } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pMatch->pTab; + pExpr->y.pTab = pMatch->pSTab; if( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } @@ -454,7 +514,7 @@ static int lookupName( } /* if( pSrcList ) */ #if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) - /* If we have not already resolved the name, then maybe + /* If we have not already resolved the name, then maybe ** it is a new.* or old.* trigger argument reference. Or ** maybe it is an excluded.* from an upsert. Or maybe it is ** a reference in the RETURNING clause to a table being modified. @@ -468,7 +528,8 @@ static int lookupName( if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 && ALWAYS(zTab==0 - || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -486,31 +547,26 @@ static int lookupName( if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ Upsert *pUpsert = pNC->uNC.pUpsert; if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ - pTab = pUpsert->pUpsertSrc->a[0].pTab; + pTab = pUpsert->pUpsertSrc->a[0].pSTab; pExpr->iTable = EXCLUDED_TABLE_NUMBER; } } #endif /* SQLITE_OMIT_UPSERT */ - if( pTab ){ + if( pTab ){ int iCol; - u8 hCol = sqlite3StrIHash(zCol); pSchema = pTab->pSchema; cntTab++; - for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ - if( pCol->hName==hCol - && sqlite3StrICmp(pCol->zCnName, zCol)==0 - ){ - if( iCol==pTab->iPKey ){ - iCol = -1; - } - break; + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol>=0 ){ + if( pTab->iPKey==iCol ) iCol = -1; + }else{ + if( sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ + iCol = -1; + }else{ + iCol = pTab->nCol; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ - /* IMP: R-51414-32910 */ - iCol = -1; - } if( iColnCol ){ cnt++; pMatch = 0; @@ -565,14 +621,19 @@ static int lookupName( ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && ALWAYS(VisibleRowid(pMatch->pTab)) + && ALWAYS(VisibleRowid(pMatch->pSTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; - pExpr->iColumn = -1; + cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pSTab!=0 && IsView(pMatch->pSTab) ){ + eNewExprOp = TK_NULL; + } +#endif + if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; pExpr->affExpr = SQLITE_AFF_INTEGER; } @@ -633,7 +694,7 @@ static int lookupName( } goto lookupname_end; } - } + } } /* Advance to the next name context. The loop will exit when either @@ -725,12 +786,17 @@ static int lookupName( sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); pParse->checkSchema = 1; pTopNC->nNcErr++; + eNewExprOp = TK_NULL; } assert( pFJMatch==0 ); @@ -757,8 +823,12 @@ static int lookupName( ** If a generated column is referenced, set bits for every column ** of the table. */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ - pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; + } } pExpr->op = eNewExprOp; @@ -796,7 +866,7 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ SrcItem *pItem = &pSrc->a[iSrc]; Table *pTab; assert( ExprUseYTab(p) ); - pTab = p->y.pTab = pItem->pTab; + pTab = p->y.pTab = pItem->pSTab; p->iTable = pItem->iCursor; if( p->y.pTab->iPKey==iCol ){ p->iColumn = -1; @@ -824,7 +894,7 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ ** ** static void notValid( ** Parse *pParse, // Leave error message here -** NameContext *pNC, // The name context +** NameContext *pNC, // The name context ** const char *zMsg, // Type of error ** int validMask, // Set of contexts for which prohibited ** Expr *pExpr // Invalidate this expression on error @@ -859,8 +929,8 @@ static void notValidImpl( /* ** Expression p should encode a floating point value between 1.0 and 0.0. -** Return 1024 times this value. Or return -1 if p is not a floating point -** value between 1.0 and 0.0. +** Return 134,217,728 (2^27) times this value. Or return -1 if p is not +** a floating point value between 1.0 and 0.0. */ static int exprProbability(Expr *p){ double r = -1.0; @@ -905,7 +975,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* The special operator TK_ROW means use the rowid for the first ** column in the FROM clause. This is used by the LIMIT and ORDER BY - ** clause processing on UPDATE and DELETE statements, and by + ** clause processing on UPDATE and DELETE statements, and by ** UPDATE ... FROM statement processing. */ case TK_ROW: { @@ -915,7 +985,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pItem = pSrcList->a; pExpr->op = TK_COLUMN; assert( ExprUseYTab(pExpr) ); - pExpr->y.pTab = pItem->pTab; + pExpr->y.pTab = pItem->pSTab; pExpr->iTable = pItem->iCursor; pExpr->iColumn--; pExpr->affExpr = SQLITE_AFF_INTEGER; @@ -935,29 +1005,59 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** resolved. This prevents "column" from being counted as having been ** referenced, which might prevent a SELECT from being erroneously ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ case TK_NOTNULL: case TK_ISNULL: { int anRef[8]; NameContext *p; int i; - for(i=0, p=pNC; p && ipNext, i++){ + for(i=0, p=pNC; p && ipNext, i++){ anRef[i] = p->nRef; } sqlite3WalkExpr(pWalker, pExpr->pLeft); - if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ - testcase( ExprHasProperty(pExpr, EP_OuterON) ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->u.iValue = (pExpr->op==TK_NOTNULL); - pExpr->flags |= EP_IntValue; - pExpr->op = TK_INTEGER; + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; + } - for(i=0, p=pNC; p && ipNext, i++){ - p->nRef = anRef[i]; + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ } - sqlite3ExprDelete(pParse->db, pExpr->pLeft); - pExpr->pLeft = 0; } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; return WRC_Prune; } @@ -966,12 +1066,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** Or a database, table and column: ID.ID.ID ** ** The TK_ID and TK_OUT cases are combined so that there will only - ** be one call to lookupName(). Then the compiler will in-line + ** be one call to lookupName(). Then the compiler will in-line ** lookupName() for a size reduction and performance increase. */ case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; @@ -980,7 +1079,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zDb = 0; zTable = 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); - zColumn = pExpr->u.zToken; + pRight = pExpr; }else{ Expr *pLeft = pExpr->pLeft; testcase( pNC->ncFlags & NC_IdxExpr ); @@ -999,21 +1098,20 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); zTable = pLeft->u.zToken; - zColumn = pRight->u.zToken; assert( ExprUseYTab(pExpr) ); if( IN_RENAME_OBJECT ){ sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names */ case TK_FUNCTION: { - ExprList *pList = pExpr->x.pList; /* The argument list */ - int n = pList ? pList->nExpr : 0; /* Number of arguments */ + ExprList *pList; /* The argument list */ + int n; /* Number of arguments */ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ @@ -1025,6 +1123,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); #endif assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); + assert( pExpr->pLeft==0 || pExpr->pLeft->op==TK_ORDER ); + pList = pExpr->x.pList; + n = pList ? pList->nExpr : 0; zId = pExpr->u.zToken; pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ @@ -1057,7 +1158,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** to likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; - } + } } #ifndef SQLITE_OMIT_AUTHORIZATION { @@ -1073,6 +1174,24 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } } #endif + + /* If the function may call sqlite3_value_subtype(), then set the + ** EP_SubtArg flag on all of its argument expressions. This prevents + ** where.c from replacing the expression with a value read from an + ** index on the same expression, which will not have the correct + ** subtype. Also set the flag if the function expression itself is + ** an EP_SubtArg expression. In this case subtypes are required as + ** the function may return a value with a subtype back to its + ** caller using sqlite3_result_value(). */ + if( (pDef->funcFlags & SQLITE_SUBTYPE) + || ExprHasProperty(pExpr, EP_SubtArg) + ){ + int ii; + for(ii=0; iia[ii].pExpr, EP_SubtArg); + } + } + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ /* For the purposes of the EP_ConstFunc flag, date and time ** functions and other functions that change slowly are considered @@ -1086,13 +1205,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** sqlite_version() that might change over time cannot be used ** in an index or generated column. Curiously, they can be used ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all - ** all this. */ + ** allow this. */ sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr|NC_PartIdx|NC_GenCol, 0, pExpr); }else{ assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ pExpr->op2 = pNC->ncFlags & NC_SelfRef; - if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); } if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 && pParse->nested==0 @@ -1108,6 +1226,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 && !IN_RENAME_OBJECT ){ + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } } @@ -1119,11 +1238,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) ); if( pDef && pDef->xValue==0 && pWin ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "%#T() may not be used as a window function", pExpr ); pNC->nNcErr++; - }else if( + }else if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pWin) || (is_agg && pWin && (pNC->ncFlags & NC_AllowWin)==0) @@ -1159,13 +1278,17 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } #ifndef SQLITE_OMIT_WINDOWFUNC else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "FILTER may not be used with non-aggregate %#T()", pExpr ); pNC->nNcErr++; } #endif + else if( is_agg==0 && pExpr->pLeft ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + pNC->nNcErr++; + } if( is_agg ){ /* Window functions may not be arguments of aggregate functions. ** Or arguments of other window functions. But aggregate functions @@ -1177,17 +1300,20 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif } } -#ifndef SQLITE_OMIT_WINDOWFUNC - else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ is_agg = 1; } -#endif sqlite3WalkExprList(pWalker, pList); if( is_agg ){ + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC - if( pWin ){ + if( pWin && pParse->nErr==0 ){ Select *pSel = pNC->pWinSelect; - assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) ); + assert( ExprUseYWin(pExpr) && pWin==pExpr->y.pWin ); if( IN_RENAME_OBJECT==0 ){ sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); if( pParse->db->mallocFailed ) break; @@ -1209,19 +1335,20 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } #endif pNC2 = pNC; - while( pNC2 + while( pNC2 && sqlite3ReferencesSrcList(pParse, pExpr, pNC2->pSrcList)==0 ){ - pExpr->op2++; + pExpr->op2 += (1 + pNC2->nNestedSelect); pNC2 = pNC2->pNext; } assert( pDef!=0 || IN_RENAME_OBJECT ); if( pNC2 && pDef ){ + pExpr->op2 += pNC2->nNestedSelect; assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); assert( SQLITE_FUNC_ANYORDER==NC_OrderAgg ); testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); testcase( (pDef->funcFlags & SQLITE_FUNC_ANYORDER)!=0 ); - pNC2->ncFlags |= NC_HasAgg + pNC2->ncFlags |= NC_HasAgg | ((pDef->funcFlags^SQLITE_FUNC_ANYORDER) & (SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER)); } @@ -1229,22 +1356,26 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pNC->ncFlags |= savedAllowFlags; } /* FIX ME: Compute pExpr->affinity based on the expected return - ** type of the function + ** type of the function */ return WRC_Prune; } #ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: case TK_SELECT: - case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); #endif case TK_IN: { testcase( pExpr->op==TK_IN ); + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); if( ExprUseXSelect(pExpr) ){ int nRef = pNC->nRef; testcase( pNC->ncFlags & NC_IsCheck ); testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IdxExpr ); testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); + if( pExpr->op==TK_EXISTS ) pParse->bHasExists = 1; if( pNC->ncFlags & NC_SelfRef ){ notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); }else{ @@ -1253,6 +1384,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pExpr->x.pSelect->selFlags |= SF_Correlated; } pNC->ncFlags |= NC_Subquery; } @@ -1318,7 +1450,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ sqlite3ErrorMsg(pParse, "row value misused"); sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); } - break; + break; } } assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); @@ -1391,7 +1523,7 @@ static int resolveOrderByTermToExprList( int rc; /* Return code from subprocedures */ u8 savedSuppErr; /* Saved value of db->suppressErr */ - assert( sqlite3ExprIsInteger(pE, &i)==0 ); + assert( sqlite3ExprIsInteger(pE, &i, 0)==0 ); pEList = pSelect->pEList; /* Resolve all names in the ORDER BY term expression @@ -1433,7 +1565,7 @@ static void resolveOutOfRangeError( int mx, /* Largest permissible value of i */ Expr *pError /* Associate the error with the expression */ ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "%r %s BY term out of range - should be " "between 1 and %d", i, zType, mx); sqlite3RecordErrorOffsetOfExpr(pParse->db, pError); @@ -1490,7 +1622,7 @@ static int resolveCompoundOrderBy( if( pItem->fg.done ) continue; pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( NEVER(pE==0) ) continue; - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( sqlite3ExprIsInteger(pE, &iCol, 0) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr, pE); return 1; @@ -1499,7 +1631,7 @@ static int resolveCompoundOrderBy( iCol = resolveAsName(pParse, pEList, pE); if( iCol==0 ){ /* Now test if expression pE matches one of the values returned - ** by pSelect. In the usual case this is done by duplicating the + ** by pSelect. In the usual case this is done by duplicating the ** expression, resolving any symbols in it, and then comparing ** it against each expression returned by the SELECT statement. ** Once the comparisons are finished, the duplicate expression @@ -1675,7 +1807,7 @@ static int resolveOrderGroupBy( continue; } } - if( sqlite3ExprIsInteger(pE2, &iCol) ){ + if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ @@ -1694,7 +1826,7 @@ static int resolveOrderGroupBy( } for(j=0; jpEList->nExpr; j++){ if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ - /* Since this expresion is being changed into a reference + /* Since this expression is being changed into a reference ** to an identical expression in the result set, remove all Window ** objects belonging to the expression from the Select.pWin list. */ windowRemoveExprFromSelect(pSelect, pE); @@ -1718,7 +1850,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ sqlite3 *db; /* Database connection */ - + assert( p!=0 ); if( p->selFlags & SF_Resolved ){ @@ -1747,10 +1879,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ while( p ){ assert( (p->selFlags & SF_Expanded)!=0 ); assert( (p->selFlags & SF_Resolved)==0 ); - assert( db->suppressErr==0 ); /* SF_Resolved not set if errors suppressed */ p->selFlags |= SF_Resolved; - /* Resolve the expressions in the LIMIT and OFFSET clauses. These ** are not allowed to refer to any names, so pass an empty NameContext. */ @@ -1768,23 +1898,32 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** moves the pOrderBy down to the sub-query. It will be moved back ** after the names have been resolved. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + assert( p->pSrc->a[0].u4.pSubq!=0 ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); assert( p->pSrc->nSrc==1 && p->pOrderBy ); assert( pSub->pPrior && pSub->pOrderBy==0 ); pSub->pOrderBy = p->pOrderBy; p->pOrderBy = 0; } - + /* Recursively resolve names in all subqueries in the FROM clause */ + if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; - if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ + assert( pItem->zName!=0 + || pItem->fg.isSubquery ); /* Test of tag-20240424-1*/ + if( pItem->fg.isSubquery + && (pItem->u4.pSubq->pSelect->selFlags & SF_Resolved)==0 + ){ int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + sqlite3ResolveSelectNames(pParse, pItem->u4.pSubq->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr ) return WRC_Abort; assert( db->mallocFailed==0 ); @@ -1801,19 +1940,22 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } } - + if( pOuterNC && ALWAYS(pOuterNC->nNestedSelect>0) ){ + pOuterNC->nNestedSelect--; + } + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to ** resolve the result-set expression list. */ sNC.ncFlags = NC_AllowAgg|NC_AllowWin; sNC.pSrcList = p->pSrc; sNC.pNext = pOuterNC; - + /* Resolve names in the result set. */ if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; sNC.ncFlags &= ~NC_AllowWin; - - /* If there are no aggregate functions in the result-set, and no GROUP BY + + /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. */ assert( (p->selFlags & SF_Aggregate)==0 ); @@ -1825,7 +1967,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ }else{ sNC.ncFlags &= ~NC_AllowAgg; } - + /* Add the output column list to the name-context before parsing the ** other expressions in the SELECT statement. This is so that ** expressions in the WHERE clause (etc.) can refer to expressions by @@ -1844,13 +1986,15 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; ipSrc->nSrc; i++){ SrcItem *pItem = &p->pSrc->a[i]; if( pItem->fg.isTabFunc - && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) ){ return WRC_Abort; } @@ -1870,18 +2014,21 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ #endif /* The ORDER BY and GROUP BY clauses may not refer to terms in - ** outer queries + ** outer queries */ sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; - /* If this is a converted compound query, move the ORDER BY clause from + /* If this is a converted compound query, move the ORDER BY clause from ** the sub-query back to the parent query. At this point each term ** within the ORDER BY clause has been transformed to an integer value. ** These integers will be replaced by copies of the corresponding result ** set expressions by the call to resolveOrderGroupBy() below. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); p->pOrderBy = pSub->pOrderBy; pSub->pOrderBy = 0; } @@ -1906,13 +2053,13 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ return WRC_Abort; } sNC.ncFlags &= ~NC_AllowWin; - - /* Resolve the GROUP BY clause. At the same time, make sure + + /* Resolve the GROUP BY clause. At the same time, make sure ** the GROUP BY clause does not contain aggregate functions. */ if( pGroupBy ){ struct ExprList_item *pItem; - + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ return WRC_Abort; } @@ -1954,7 +2101,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** checking on function usage and set a flag if any aggregate functions ** are seen. ** -** To resolve table columns references we look for nodes (or subtrees) of the +** To resolve table columns references we look for nodes (or subtrees) of the ** form X.Y.Z or Y.Z or just Z where ** ** X: The name of a database. Ex: "main" or "temp" or @@ -1986,7 +2133,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** ** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; ** -** Function calls are checked to make sure that the function is +** Function calls are checked to make sure that the function is ** defined and that the correct number of arguments are specified. ** If the function is an aggregate function, then the NC_HasAgg flag is ** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. @@ -1996,7 +2143,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** An error message is left in pParse if anything is amiss. The number ** if errors is returned. */ -int sqlite3ResolveExprNames( +int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ @@ -2017,7 +2164,8 @@ int sqlite3ResolveExprNames( return SQLITE_ERROR; } #endif - sqlite3WalkExpr(&w, pExpr); + assert( pExpr!=0 ); + sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif @@ -2034,15 +2182,18 @@ int sqlite3ResolveExprNames( ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ -int sqlite3ResolveExprListNames( +int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ ExprList *pList /* The expression list to be analyzed. */ ){ int i; int savedHasAgg = 0; Walker w; - if( pList==0 ) return WRC_Continue; + if( pList==0 ) return SQLITE_OK; w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -2056,10 +2207,10 @@ int sqlite3ResolveExprListNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight += pExpr->nHeight; if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ - return WRC_Abort; + return SQLITE_ERROR; } #endif - sqlite3WalkExpr(&w, pExpr); + sqlite3WalkExprNN(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif @@ -2073,15 +2224,15 @@ int sqlite3ResolveExprListNames( (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); } - if( w.pParse->nErr>0 ) return WRC_Abort; + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } pNC->ncFlags |= savedHasAgg; - return WRC_Continue; + return SQLITE_OK; } /* ** Resolve all names in all expressions of a SELECT and in all -** decendents of the SELECT, including compounds off of p->pPrior, +** descendants of the SELECT, including compounds off of p->pPrior, ** subqueries in expressions, and subqueries used as FROM clause ** terms. ** @@ -2132,20 +2283,25 @@ int sqlite3ResolveSelfReference( Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NULL. */ ){ - SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + SrcList *pSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; + union { + SrcList sSrc; + u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ + } uSrc; assert( type==0 || pTab!=0 ); assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); - memset(&sSrc, 0, sizeof(sSrc)); + memset(&uSrc, 0, sizeof(uSrc)); + pSrc = &uSrc.sSrc; if( pTab ){ - sSrc.nSrc = 1; - sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pTab = pTab; - sSrc.a[0].iCursor = -1; + pSrc->nSrc = 1; + pSrc->a[0].zName = pTab->zName; + pSrc->a[0].pSTab = pTab; + pSrc->a[0].iCursor = -1; if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP ** schema elements */ @@ -2153,7 +2309,7 @@ int sqlite3ResolveSelfReference( } } sNC.pParse = pParse; - sNC.pSrcList = &sSrc; + sNC.pSrcList = pSrc; sNC.ncFlags = type | NC_IsDDL; if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); diff --git a/src/rowset.c b/src/rowset.c index 0562320ab4..5956cb2ad8 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -35,14 +35,14 @@ ** extracts the least value from the RowSet. ** ** The INSERT primitive might allocate additional memory. Memory is -** allocated in chunks so most INSERTs do no allocation. There is an +** allocated in chunks so most INSERTs do no allocation. There is an ** upper bound on the size of allocated memory. No memory is freed ** until DESTROY. ** ** The TEST primitive includes a "batch" number. The TEST primitive ** will only see elements that were inserted before the last change ** in the batch number. In other words, if an INSERT occurs between -** two TESTs where the TESTs have the same batch nubmer, then the +** two TESTs where the TESTs have the same batch number, then the ** value added by the INSERT will not be visible to the second TEST. ** The initial batch number is zero, so if the very first TEST contains ** a non-zero batch number, it will see all prior INSERTs. @@ -83,7 +83,7 @@ ** in the list, pLeft points to the tree, and v is unused. The ** RowSet.pForest value points to the head of this forest list. */ -struct RowSetEntry { +struct RowSetEntry { i64 v; /* ROWID value for this entry */ struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ @@ -235,7 +235,7 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ /* ** Merge two lists of RowSetEntry objects. Remove duplicates. ** -** The input lists are connected via pRight pointers and are +** The input lists are connected via pRight pointers and are ** assumed to each already be in sorted order. */ static struct RowSetEntry *rowSetEntryMerge( @@ -272,7 +272,7 @@ static struct RowSetEntry *rowSetEntryMerge( /* ** Sort all elements on the list of RowSetEntry objects into order of ** increasing v. -*/ +*/ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ unsigned int i; struct RowSetEntry *pNext, *aBucket[40]; @@ -345,7 +345,7 @@ static struct RowSetEntry *rowSetNDeepTree( struct RowSetEntry *pLeft; /* Left subtree */ if( *ppList==0 ){ /*OPTIMIZATION-IF-TRUE*/ /* Prevent unnecessary deep recursion when we run out of entries */ - return 0; + return 0; } if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ /* This branch causes a *balanced* tree to be generated. A valid tree diff --git a/src/select.c b/src/select.c index f32db2c2a0..bc17ecf846 100644 --- a/src/select.c +++ b/src/select.c @@ -154,7 +154,7 @@ Select *sqlite3SelectNew( pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; - if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; @@ -184,6 +184,9 @@ Select *sqlite3SelectNew( void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); } +void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){ + if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1); +} /* ** Return a pointer to the right-most SELECT statement in a compound. @@ -231,8 +234,8 @@ static Select *findRightmost(Select *p){ ** NATURAL FULL - JT_NATURAL|JT_LEFT|JT_RIGHT ** NATURAL FULL OUTER JT_NATRUAL|JT_LEFT|JT_RIGHT ** -** To preserve historical compatibly, SQLite also accepts a variety -** of other non-standard and in many cases non-sensical join types. +** To preserve historical compatibly, SQLite also accepts a variety +** of other non-standard and in many cases nonsensical join types. ** This routine makes as much sense at it can from the nonsense join ** type and returns a result. Examples of accepted nonsense join types ** include but are not limited to: @@ -282,7 +285,7 @@ int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ for(i=0; i<3 && apAll[i]; i++){ p = apAll[i]; for(j=0; jn==aKeyword[j].nChar + if( p->n==aKeyword[j].nChar && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ jointype |= aKeyword[j].code; break; @@ -316,10 +319,33 @@ int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ */ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ int i; - u8 h = sqlite3StrIHash(zCol); - Column *pCol; - for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ - if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i; + u8 h; + const Column *aCol; + int nCol; + + h = sqlite3StrIHash(zCol); + aCol = pTab->aCol; + nCol = pTab->nCol; + + /* See if the aHx gives us a lucky match */ + i = pTab->aHx[h % sizeof(pTab->aHx)]; + assert( i=nCol ) break; } return -1; } @@ -329,11 +355,13 @@ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ */ void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ assert( pItem!=0 ); - assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); if( pItem->fg.isNestedFrom ){ ExprList *pResults; - assert( pItem->pSelect!=0 ); - pResults = pItem->pSelect->pEList; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + pResults = pItem->u4.pSubq->pSelect->pEList; assert( pResults!=0 ); assert( iCol>=0 && iColnExpr ); pResults->a[iCol].fg.bUsed = 1; @@ -356,7 +384,7 @@ static int tableAndColumnIndex( int iEnd, /* Last member of pSrc->a[] to check */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ - int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pSTab->aCol[] here */ int bIgnoreHidden /* Ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ @@ -367,9 +395,9 @@ static int tableAndColumnIndex( assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ for(i=iStart; i<=iEnd; i++){ - iCol = sqlite3ColumnIndex(pSrc->a[i].pTab, zCol); - if( iCol>=0 - && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + iCol = sqlite3ColumnIndex(pSrc->a[i].pSTab, zCol); + if( iCol>=0 + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pSTab->aCol[iCol])==0) ){ if( piTab ){ sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol); @@ -415,8 +443,7 @@ void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); p->w.iJoin = iTable; - if( p->op==TK_FUNCTION ){ - assert( ExprUseXList(p) ); + if( ExprUseXList(p) ){ if( p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ @@ -426,7 +453,7 @@ void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){ } sqlite3SetJoinExpr(p->pLeft, iTable, joinFlag); p = p->pRight; - } + } } /* Undo the work of sqlite3SetJoinExpr(). This is used when a LEFT JOIN @@ -454,6 +481,7 @@ static void unsetJoinExpr(Expr *p, int iTable, int nullable){ } if( p->op==TK_FUNCTION ){ assert( ExprUseXList(p) ); + assert( p->pLeft==0 ); if( p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ @@ -463,7 +491,7 @@ static void unsetJoinExpr(Expr *p, int iTable, int nullable){ } unsetJoinExpr(p->pLeft, iTable, nullable); p = p->pRight; - } + } } /* @@ -497,13 +525,13 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pLeft = &pSrc->a[0]; pRight = &pLeft[1]; for(i=0; inSrc-1; i++, pRight++, pLeft++){ - Table *pRightTab = pRight->pTab; + Table *pRightTab = pRight->pSTab; u32 joinType; - if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; + if( NEVER(pLeft->pSTab==0 || pRightTab==0) ) continue; joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; - /* If this is a NATURAL join, synthesize an approprate USING clause + /* If this is a NATURAL join, synthesize an appropriate USING clause ** to specify which columns should be joined. */ if( pRight->fg.jointype & JT_NATURAL ){ @@ -536,7 +564,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ } /* Create extra terms on the WHERE clause for each column named - ** in the USING clause. Example: If the two tables to be joined are + ** in the USING clause. Example: If the two tables to be joined are ** A and B and the USING clause names X, Y, and Z, then add this ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z ** Report an error if any column mentioned in the USING clause is @@ -567,7 +595,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ } pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); - if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 && pParse->nErr==0 ){ /* This branch runs if the query contains one or more RIGHT or FULL ** JOINs. If only a single table on the left side of this join ** contains the zName column, then this branch is a no-op. @@ -583,6 +611,8 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ */ ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */ static const Token tkCoalesce = { "coalesce", 8 }; + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol, pRight->fg.isSynthUsing)!=0 ){ if( pSrc->a[iLeft].fg.isUsing==0 @@ -599,7 +629,13 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ if( pFuncArgs ){ pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0); + if( pE1 ){ + pE1->affExpr = SQLITE_AFF_DEFER; + } } + }else if( (pSrc->a[i+1].fg.jointype & JT_LEFT)!=0 && pParse->nErr==0 ){ + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); } pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol); sqlite3SrcItemColumnUsed(pRight, iRightCol); @@ -623,6 +659,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn); pRight->u3.pOn = 0; pRight->fg.isOn = 1; + p->selFlags |= SF_OnToWhere; } } return 0; @@ -718,8 +755,8 @@ static void pushOntoSorter( ** case regData==regOrigData. ** (3) Some output columns are omitted from the sort record due to ** the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the - ** SQLITE_ECEL_OMITREF optimization, or due to the - ** SortCtx.pDeferredRowLoad optimiation. In any of these cases + ** SQLITE_ECEL_OMITREF optimization, or due to the + ** SortCtx.pDeferredRowLoad optimization. In any of these cases ** regOrigData is 0 to prevent this routine from trying to copy ** values that might not yet exist. */ @@ -760,7 +797,7 @@ static void pushOntoSorter( pParse->nMem += pSort->nOBSat; nKey = nExpr - pSort->nOBSat + bSeq; if( bSeq ){ - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); }else{ addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); } @@ -775,7 +812,7 @@ static void pushOntoSorter( testcase( pKI->nAllField > pKI->nKeyField+2 ); pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, pKI->nAllField-pKI->nKeyField-1); - pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */ + pOp = 0; /* Ensure pOp not used after sqlite3VdbeAddOp3() */ addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); @@ -794,10 +831,10 @@ static void pushOntoSorter( /* At this point the values for the new sorter entry are stored ** in an array of registers. They need to be composed into a record ** and inserted into the sorter if either (a) there are currently - ** less than LIMIT+OFFSET items or (b) the new record is smaller than + ** less than LIMIT+OFFSET items or (b) the new record is smaller than ** the largest record currently in the sorter. If (b) is true and there ** are already LIMIT+OFFSET items in the sorter, delete the largest - ** entry before inserting the new one. This way there are never more + ** entry before inserting the new one. This way there are never more ** than LIMIT+OFFSET items in the sorter. ** ** If the new record does not need to be inserted into the sorter, @@ -869,7 +906,7 @@ static void codeOffset( ** The returned value in this case is a copy of parameter iTab. ** ** WHERE_DISTINCT_ORDERED: -** In this case rows are being delivered sorted order. The ephermal +** In this case rows are being delivered sorted order. The ephemeral ** table is not required. Instead, the current set of values ** is compared against previous row. If they match, the new row ** is not distinct and control jumps to VM address addrRepeat. Otherwise, @@ -885,10 +922,10 @@ static void codeOffset( ** In this case it has already been determined that the rows are distinct. ** No special action is required. The return value is zero. ** -** Parameter pEList is the list of expressions used to generated the -** contents of each row. It is used by this routine to determine (a) -** how many elements there are in the array of registers and (b) the -** collation sequences that should be used for the comparisons if +** Parameter pEList is the list of expressions used to generated the +** contents of each row. It is used by this routine to determine (a) +** how many elements there are in the array of registers and (b) the +** collation sequences that should be used for the comparisons if ** eTnctType is WHERE_DISTINCT_ORDERED. */ static int codeDistinct( @@ -990,7 +1027,7 @@ static void fixDistinctOpenEph( sqlite3VdbeChangeToNoop(v, iOpenEphAddr+1); } if( eTnctType==WHERE_DISTINCT_ORDERED ){ - /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared + /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared ** bit on the first register of the previous value. This will cause the ** OP_Ne added in codeDistinct() to always fail on the first iteration of ** the loop even if the first row is all NULLs. */ @@ -1005,8 +1042,8 @@ static void fixDistinctOpenEph( #ifdef SQLITE_ENABLE_SORTER_REFERENCES /* ** This function is called as part of inner-loop generation for a SELECT -** statement with an ORDER BY that is not optimized by an index. It -** determines the expressions, if any, that the sorter-reference +** statement with an ORDER BY that is not optimized by an index. It +** determines the expressions, if any, that the sorter-reference ** optimization should be used for. The sorter-reference optimization ** is used for SELECT queries like: ** @@ -1016,11 +1053,11 @@ static void fixDistinctOpenEph( ** storing values read from that column in the sorter records, the PK of ** the row from table t1 is stored instead. Then, as records are extracted from ** the sorter to return to the user, the required value of bigblob is -** retrieved directly from table t1. If the values are very large, this +** retrieved directly from table t1. If the values are very large, this ** can be more efficient than storing them directly in the sorter records. ** -** The ExprList_item.fg.bSorterRef flag is set for each expression in pEList -** for which the sorter-reference optimization should be enabled. +** The ExprList_item.fg.bSorterRef flag is set for each expression in pEList +** for which the sorter-reference optimization should be enabled. ** Additionally, the pSort->aDefer[] array is populated with entries ** for all cursors required to evaluate all selected expressions. Finally. ** output variable (*ppExtra) is set to an expression list containing @@ -1094,7 +1131,7 @@ static void selectExprDefer( ** ** If srcTab is negative, then the p->pEList expressions ** are evaluated in order to get the data for this row. If srcTab is -** zero or more, then data is pulled from srcTab and p->pEList is used only +** zero or more, then data is pulled from srcTab and p->pEList is used only ** to get the number of columns and the collation sequence for each column. */ static void selectInnerLoop( @@ -1176,8 +1213,8 @@ static void selectInnerLoop( } if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){ /* For each expression in p->pEList that is a copy of an expression in - ** the ORDER BY clause (pSort->pOrderBy), set the associated - ** iOrderByCol value to one more than the index of the ORDER BY + ** the ORDER BY clause (pSort->pOrderBy), set the associated + ** iOrderByCol value to one more than the index of the ORDER BY ** expression within the sort-key that pushOntoSorter() will generate. ** This allows the p->pEList field to be omitted from the sorted record, ** saving space and CPU cycles. */ @@ -1193,7 +1230,7 @@ static void selectInnerLoop( selectExprDefer(pParse, pSort, p->pEList, &pExtra); if( pExtra && pParse->db->mallocFailed==0 ){ /* If there are any extra PK columns to add to the sorter records, - ** allocate extra memory cells and adjust the OpenEphemeral + ** allocate extra memory cells and adjust the OpenEphemeral ** instruction to account for the larger records. This is only ** required if there are one or more WITHOUT ROWID tables with ** composite primary keys in the SortCtx.aDefer[] array. */ @@ -1223,7 +1260,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Mem ); testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); - assert( eDest==SRT_Set || eDest==SRT_Mem + assert( eDest==SRT_Set || eDest==SRT_Mem || eDest==SRT_Coroutine || eDest==SRT_Output || eDest==SRT_Upfrom ); } @@ -1235,7 +1272,7 @@ static void selectInnerLoop( if( pExtra ) nResultCol += pExtra->nExpr; #endif if( p->iLimit - && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 + && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 && nPrefixReg>0 ){ assert( pSort!=0 ); @@ -1298,6 +1335,16 @@ static void selectInnerLoop( testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -1363,12 +1410,18 @@ static void selectInnerLoop( ** case the order does matter */ pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm2 = 0; /* Signal that any Bloom filter is unpopulated */ }else{ int r1 = sqlite3GetTempReg(pParse); assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, r1, pDest->zAffSdst, nResultCol); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + if( pDest->iSDParm2 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + regResult, nResultCol); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); } break; @@ -1384,7 +1437,7 @@ static void selectInnerLoop( } /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell or array of + ** store the results in the appropriate memory cell or array of ** memory cells and break out of the scan loop. */ case SRT_Mem: { @@ -1392,9 +1445,14 @@ static void selectInnerLoop( assert( nResultCol<=pDest->nSdst ); pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm = regResult; }else{ assert( nResultCol==pDest->nSdst ); - assert( regResult==iParm ); + if( regResult!=iParm ){ + /* This occurs in cases where the SELECT had both a DISTINCT and + ** an OFFSET clause. */ + sqlite3VdbeAddOp3(v, OP_Copy, regResult, iParm, nResultCol-1); + } /* The LIMIT clause will jump out of the loop for us */ } break; @@ -1439,7 +1497,7 @@ static void selectInnerLoop( /* If the destination is DistQueue, then cursor (iParm+1) is open ** on a second ephemeral index that holds all values every previously ** added to the queue. */ - addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, regResult, nResultCol); VdbeCoverage(v); } @@ -1492,8 +1550,11 @@ static void selectInnerLoop( ** X extra columns. */ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); - KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p; + assert( X>=0 ); + if( NEVER(N+X>0xffff) ) return (KeyInfo*)sqlite3OomFault(db); + p = sqlite3DbMallocRawNN(db, SZ_KEYINFO(0) + nExtra); if( p ){ p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; @@ -1501,7 +1562,7 @@ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; - memset(&p[1], 0, nExtra); + memset(p->aColl, 0, nExtra); }else{ return (KeyInfo*)sqlite3OomFault(db); } @@ -1659,9 +1720,16 @@ static void generateSortTail( int addrExplain; /* Address of OP_Explain instruction */ #endif - ExplainQueryPlan2(addrExplain, (pParse, 0, - "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat>0?"RIGHT PART OF ":"") - ); + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); @@ -1699,14 +1767,13 @@ static void generateSortTail( regRow = sqlite3GetTempRange(pParse, nColumn); } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; if( pSort->labelBkOut ){ addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } - sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nColumn+nRefKey); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); @@ -1742,7 +1809,7 @@ static void generateSortTail( sqlite3VdbeAddOp1(v, OP_NullRow, iCsr); if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey); - sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, sqlite3VdbeCurrentAddr(v)+1, regKey); }else{ int k; @@ -1812,7 +1879,7 @@ static void generateSortTail( break; } default: { - assert( eDest==SRT_Output || eDest==SRT_Coroutine ); + assert( eDest==SRT_Output || eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); testcase( eDest==SRT_Coroutine ); if( eDest==SRT_Output ){ @@ -1852,14 +1919,14 @@ static void generateSortTail( ** original CREATE TABLE statement if the expression is a column. The ** declaration type for a ROWID field is INTEGER. Exactly when an expression ** is considered a column can be complex in the presence of subqueries. The -** result-set expression in all of the following SELECT statements is +** result-set expression in all of the following SELECT statements is ** considered a column by this function. ** ** SELECT col FROM tbl; ** SELECT (SELECT col FROM tbl; ** SELECT (SELECT col FROM tbl); ** SELECT abc FROM (SELECT col AS abc FROM tbl); -** +** ** The declaration type for any expression other than a column is NULL. ** ** This routine has either 3 or 6 parameters depending on whether or not @@ -1871,7 +1938,7 @@ static void generateSortTail( # define columnType(A,B,C,D,E) columnTypeImpl(A,B) #endif static const char *columnTypeImpl( - NameContext *pNC, + NameContext *pNC, #ifndef SQLITE_ENABLE_COLUMN_METADATA Expr *pExpr #else @@ -1904,8 +1971,12 @@ static const char *columnTypeImpl( SrcList *pTabList = pNC->pSrcList; for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( jnSrc ){ - pTab = pTabList->a[j].pTab; - pS = pTabList->a[j].pSelect; + pTab = pTabList->a[j].pSTab; + if( pTabList->a[j].fg.isSubquery ){ + pS = pTabList->a[j].u4.pSubq->pSelect; + }else{ + pS = 0; + } }else{ pNC = pNC->pNext; } @@ -1914,19 +1985,19 @@ static const char *columnTypeImpl( if( pTab==0 ){ /* At one time, code such as "SELECT new.x" within a trigger would ** cause this condition to run. Since then, we have restructured how - ** trigger code is generated and so this condition is no longer + ** trigger code is generated and so this condition is no longer ** possible. However, it can still be true for statements like ** the following: ** ** CREATE TABLE t1(col INTEGER); ** SELECT (SELECT t1.col) FROM FROM t1; ** - ** when columnType() is called on the expression "t1.col" in the + ** when columnType() is called on the expression "t1.col" in the ** sub-select. In this case, set the column type to NULL, even ** though it should really be "INTEGER". ** ** This is not a problem, as the column type of "t1.col" is never - ** used. When columnType() is called on the expression + ** used. When columnType() is called on the expression ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT ** branch below. */ break; @@ -1939,14 +2010,10 @@ static const char *columnTypeImpl( ** data for the result-set column of the sub-select. */ if( iColpEList->nExpr -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - && iCol>=0 -#else - && ALWAYS(iCol>=0) -#endif - ){ + && (!ViewCanHaveRowid || iCol>=0) + ){ /* If iCol is less than zero, then the expression requests the - ** rowid of the sub-select or view. This expression is legal (see + ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. */ NameContext sNC; @@ -1954,7 +2021,7 @@ static const char *columnTypeImpl( sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); + zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); } }else{ /* A real table or a CTE table */ @@ -2000,13 +2067,13 @@ static const char *columnTypeImpl( sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); break; } #endif } -#ifdef SQLITE_ENABLE_COLUMN_METADATA +#ifdef SQLITE_ENABLE_COLUMN_METADATA if( pzOrigDb ){ assert( pzOrigTab && pzOrigCol ); *pzOrigDb = zOrigDb; @@ -2042,7 +2109,7 @@ static void generateColumnTypes( const char *zOrigCol = 0; zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); - /* The vdbe must make its own copy of the column-type and other + /* The vdbe must make its own copy of the column-type and other ** column specific strings, in case the schema is reset before this ** virtual machine is deleted. */ @@ -2054,6 +2121,10 @@ static void generateColumnTypes( #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } +#else + UNUSED_PARAMETER(pParse); + UNUSED_PARAMETER(pTabList); + UNUSED_PARAMETER(pEList); #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } @@ -2101,13 +2172,6 @@ void sqlite3GenerateColumnNames( int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ -#ifndef SQLITE_OMIT_EXPLAIN - /* If this is an EXPLAIN, skip this step */ - if( pParse->explain ){ - return; - } -#endif - if( pParse->colNamesSet ) return; /* Column names are determined by the left-most term of a compound select */ while( pSelect->pPrior ) pSelect = pSelect->pPrior; @@ -2294,7 +2358,7 @@ int sqlite3ColumnsFromExprList( ** kind (maybe a parenthesized subquery in the FROM clause of a larger ** query, or a VIEW, or a CTE). This routine computes type information ** for that Table object based on the Select object that implements the -** subquery. For the purposes of this routine, "type infomation" means: +** subquery. For the purposes of this routine, "type information" means: ** ** * The datatype name, as it might appear in a CREATE TABLE statement ** * Which collating sequence to use for the column @@ -2326,17 +2390,22 @@ void sqlite3SubqueryColumnTypes( for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; i64 n; + int m = 0; + Select *pS2 = pSelect; pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } if( pCol->affinity<=SQLITE_AFF_NONE ){ pCol->affinity = aff; } - if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ - int m = 0; - Select *pS2; - for(m=0, pS2=pSelect->pNext; pS2; pS2=pS2->pNext){ + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); } if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ @@ -2366,12 +2435,12 @@ void sqlite3SubqueryColumnTypes( } } if( zType ){ - i64 m = sqlite3Strlen30(zType); + const i64 k = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; } } @@ -2438,9 +2507,9 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){ ** Compute the iLimit and iOffset fields of the SELECT based on the ** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions ** that appear in the original SQL statement after the LIMIT and OFFSET -** keywords. Or NULL if those keywords are omitted. iLimit and iOffset -** are the integer memory register numbers for counters used to compute -** the limit and offset. If there is no limit and/or offset, then +** keywords. Or NULL if those keywords are omitted. iLimit and iOffset +** are the integer memory register numbers for counters used to compute +** the limit and offset. If there is no limit and/or offset, then ** iLimit and iOffset are negative. ** ** This routine changes the values of iLimit and iOffset only if @@ -2466,7 +2535,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ if( p->iLimit ) return; - /* + /* ** "LIMIT -1" always shows all rows. There is some ** controversy about what the correct behavior should be. ** The current implementation interprets "LIMIT 0" to mean @@ -2478,7 +2547,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -2594,7 +2663,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** inserted into the Queue table. The iDistinct table keeps a copy of all rows ** that have ever been inserted into Queue and causes duplicates to be ** discarded. If the operator is UNION ALL, then duplicates are allowed. -** +** ** If the query has an ORDER BY, then entries in the Queue table are kept in ** ORDER BY order and the first entry is extracted for each cycle. Without ** an ORDER BY, the Queue table is just a FIFO. @@ -2623,7 +2692,7 @@ static void generateWithRecursiveQuery( int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ int eDest = SRT_Fifo; /* How to write to Queue */ - SelectDest destQueue; /* SelectDest targetting the Queue table */ + SelectDest destQueue; /* SelectDest targeting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ ExprList *pOrderBy; /* The ORDER BY clause */ @@ -2831,7 +2900,7 @@ static int hasAnchor(Select *p){ ** ** "p" points to the right-most of the two queries. the query on the ** left is p->pPrior. The left query could also be a compound query -** in which case this routine will be called recursively. +** in which case this routine will be called recursively. ** ** The results of the total query are to be written into a destination ** of type eDest with parameter iParm. @@ -2958,8 +3027,8 @@ static int multiSelect( p->pPrior = pPrior; p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ p->nSelectRow = sqlite3LogEst((u64)nLimit); } @@ -2975,8 +3044,10 @@ static int multiSelect( int priorOp; /* The SRT_ operation to apply to prior selects */ Expr *pLimit; /* Saved values of p->nLimit */ int addr; + int emptyBypass = 0; /* IfEmpty opcode to bypass RHS */ SelectDest uniondest; - + + testcase( p->op==TK_EXCEPT ); testcase( p->op==TK_UNION ); priorOp = SRT_Union; @@ -2998,8 +3069,8 @@ static int multiSelect( findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); } - - + + /* Code the SELECT statements to our left */ assert( !pPrior->pOrderBy ); @@ -3009,11 +3080,13 @@ static int multiSelect( if( rc ){ goto multi_select_end; } - + /* Code the current SELECT statement */ if( p->op==TK_EXCEPT ){ op = SRT_Except; + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, unionTab); + VdbeCoverage(v); }else{ assert( p->op==TK_UNION ); op = SRT_Union; @@ -3034,11 +3107,12 @@ static int multiSelect( if( p->op==TK_UNION ){ p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } + if( emptyBypass ) sqlite3VdbeJumpHere(v, emptyBypass); sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->iLimit = 0; p->iOffset = 0; - + /* Convert the data in the temporary table into whatever form ** it is that we currently need. */ @@ -3064,10 +3138,11 @@ static int multiSelect( int tab1, tab2; int iCont, iBreak, iStart; Expr *pLimit; - int addr; + int addr, iLimit, iOffset; SelectDest intersectdest; int r1; - + int emptyBypass; + /* INTERSECT is different from the others since it requires ** two temporary tables. Hence it has its own case. Begin ** by allocating the tables we will need. @@ -3075,13 +3150,13 @@ static int multiSelect( tab1 = pParse->nTab++; tab2 = pParse->nTab++; assert( p->pOrderBy==0 ); - + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); assert( p->addrOpenEphm[0] == -1 ); p->addrOpenEphm[0] = addr; findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); - + /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); @@ -3090,15 +3165,29 @@ static int multiSelect( if( rc ){ goto multi_select_end; } - + + /* Initialize LIMIT counters before checking to see if the LHS + ** is empty, in case the jump is taken */ + iBreak = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, tab1); VdbeCoverage(v); + /* Code the current SELECT into temporary table "tab2" */ addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); assert( p->addrOpenEphm[1] == -1 ); p->addrOpenEphm[1] = addr; - p->pPrior = 0; + + /* Disable prior SELECTs and the LIMIT counters during the computation + ** of the RHS select */ pLimit = p->pLimit; + iLimit = p->iLimit; + iOffset = p->iOffset; + p->pPrior = 0; p->pLimit = 0; + p->iLimit = 0; + p->iOffset = 0; + intersectdest.iSDParm = tab2; ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", sqlite3SelectOpName(p->op))); @@ -3111,19 +3200,21 @@ static int multiSelect( p->nSelectRow = pPrior->nSelectRow; } sqlite3ExprDelete(db, p->pLimit); + + /* Reinstate the LIMIT counters prior to running the final intersect */ p->pLimit = pLimit; - + p->iLimit = iLimit; + p->iOffset = iOffset; + /* Generate code to take the intersection of the two temporary ** tables. */ if( rc ) break; assert( p->pEList ); - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Rewind, tab1); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + iCont = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); @@ -3133,11 +3224,12 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeJumpHere(v, emptyBypass); sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); break; } } - + #ifndef SQLITE_OMIT_EXPLAIN if( p->pNext==0 ){ ExplainQueryPlanPop(pParse); @@ -3145,8 +3237,8 @@ static int multiSelect( #endif } if( pParse->nErr ) goto multi_select_end; - - /* Compute collating sequences used by + + /* Compute collating sequences used by ** temporary tables needed to implement the compound select. ** Attach the KeyInfo structure to all temporary tables. ** @@ -3198,10 +3290,9 @@ static int multiSelect( multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; + pDest->iSDParm2 = dest.iSDParm2; if( pDelete ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3SelectDelete, - pDelete); + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); } return rc; } @@ -3223,7 +3314,7 @@ void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ /* ** Code an output subroutine for a coroutine implementation of a -** SELECT statment. +** SELECT statement. ** ** The data to be output is contained in pIn->iSdst. There are ** pIn->nSdst columns to be output. pDest is where the output should @@ -3258,7 +3349,7 @@ static int generateOutputSubroutine( addr = sqlite3VdbeCurrentAddr(v); iContinue = sqlite3VdbeMakeLabel(pParse); - /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ int addr1, addr2; @@ -3300,10 +3391,15 @@ static int generateOutputSubroutine( int r1; testcase( pIn->nSdst>1 ); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1, pDest->zAffSdst, pIn->nSdst); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, pIn->iSdst, pIn->nSdst); + if( pDest->iSDParm2>0 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + pIn->iSdst, pIn->nSdst); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); break; } @@ -3338,7 +3434,7 @@ static int generateOutputSubroutine( ** SRT_Output. This routine is never called with any other ** destination other than the ones handled above or SRT_Output. ** - ** For SRT_Output, results are stored in a sequence of registers. + ** For SRT_Output, results are stored in a sequence of registers. ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to ** return the next row of result. */ @@ -3395,7 +3491,7 @@ static int generateOutputSubroutine( ** ** EofB: Called when data is exhausted from selectB. ** -** The implementation of the latter five subroutines depend on which +** The implementation of the latter five subroutines depend on which ** is used: ** ** @@ -3445,7 +3541,7 @@ static int generateOutputSubroutine( ** ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop -** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** until all data is exhausted then jump to the "end" label. AltB, AeqB, ** and AgtB jump to either L2 or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -3482,7 +3578,7 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int addr1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargeted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ @@ -3502,7 +3598,7 @@ static int multiSelectOrderBy( /* Patch up the ORDER BY clause */ - op = p->op; + op = p->op; assert( p->pPrior->pOrderBy==0 ); pOrderBy = p->pOrderBy; assert( pOrderBy ); @@ -3574,7 +3670,7 @@ static int multiSelectOrderBy( } } } - + /* Separate the left and the right query from one another */ nSelect = 1; @@ -3637,7 +3733,7 @@ static int multiSelectOrderBy( sqlite3VdbeEndCoroutine(v, regAddrA); sqlite3VdbeJumpHere(v, addr1); - /* Generate a coroutine to evaluate the SELECT statement on + /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; @@ -3646,7 +3742,7 @@ static int multiSelectOrderBy( savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; - p->iOffset = 0; + p->iOffset = 0; ExplainQueryPlan((pParse, 1, "RIGHT")); sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; @@ -3660,7 +3756,7 @@ static int multiSelectOrderBy( addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, regPrev, pKeyDup, labelEnd); - + /* Generate a subroutine that outputs the current row of the B ** select as the next output row of the compound select. */ @@ -3677,7 +3773,7 @@ static int multiSelectOrderBy( */ if( op==TK_EXCEPT || op==TK_INTERSECT ){ addrEofA_noB = addrEofA = labelEnd; - }else{ + }else{ VdbeNoopComment((v, "eof-A subroutine")); addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); @@ -3692,7 +3788,7 @@ static int multiSelectOrderBy( if( op==TK_INTERSECT ){ addrEofB = addrEofA; if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; - }else{ + }else{ VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); @@ -3752,8 +3848,7 @@ static int multiSelectOrderBy( /* Make arrangements to free the 2nd and subsequent arms of the compound ** after the parse has finished */ if( pSplit->pPrior ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior); + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior); } pSplit->pPrior = pPrior; pPrior->pNext = pSplit; @@ -3778,7 +3873,7 @@ static int multiSelectOrderBy( ** ## About "isOuterJoin": ** ** The isOuterJoin column indicates that the replacement will occur into a -** position in the parent that NULL-able due to an OUTER JOIN. Either the +** position in the parent that is NULL-able due to an OUTER JOIN. Either the ** target slot in the parent is the right operand of a LEFT JOIN, or one of ** the left operands of a RIGHT JOIN. In either case, we need to potentially ** bypass the substituted expression with OP_IfNullRow. @@ -3808,6 +3903,7 @@ typedef struct SubstContext { int iTable; /* Replace references to this table */ int iNewTable; /* New table number */ int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + int nSelDepth; /* Depth of sub-query recursion. Top==1 */ ExprList *pEList; /* Replacement expressions */ ExprList *pCList; /* Collation sequences for replacement expr */ } SubstContext; @@ -3819,13 +3915,13 @@ static void substSelect(SubstContext*, Select*, int); /* ** Scan through the expression pExpr. Replace every reference to ** a column in table number iTable with a copy of the iColumn-th -** entry in pEList. (But leave references to the ROWID column +** entry in pEList. (But leave references to the ROWID column ** unchanged.) ** ** This routine is part of the flattening procedure. A subquery ** whose result set is defined by pEList appears as entry in the ** FROM clause of a SELECT such that the VDBE cursor assigned to that -** FORM clause entry is iTable. This routine makes the necessary +** FORM clause entry is iTable. This routine makes the necessary ** changes to pExpr so that it refers directly to the source table ** of the subquery rather the result set of the subquery. */ @@ -3851,11 +3947,14 @@ static Expr *substExpr( #endif { Expr *pNew; - int iColumn = pExpr->iColumn; - Expr *pCopy = pSubst->pEList->a[iColumn].pExpr; + int iColumn; + Expr *pCopy; Expr ifNullRow; + iColumn = pExpr->iColumn; + assert( iColumn>=0 ); assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); assert( pExpr->pRight==0 ); + pCopy = pSubst->pEList->a[iColumn].pExpr; if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ @@ -3880,38 +3979,41 @@ static Expr *substExpr( if( pSubst->isOuterJoin ){ ExprSetProperty(pNew, EP_CanBeNull); } - if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){ - sqlite3SetJoinExpr(pNew, pExpr->w.iJoin, - pExpr->flags & (EP_OuterON|EP_InnerON)); - } - sqlite3ExprDelete(db, pExpr); - pExpr = pNew; - if( pExpr->op==TK_TRUEFALSE ){ - pExpr->u.iValue = sqlite3ExprTruthValue(pExpr); - pExpr->op = TK_INTEGER; - ExprSetProperty(pExpr, EP_IntValue); + if( pNew->op==TK_TRUEFALSE ){ + pNew->u.iValue = sqlite3ExprTruthValue(pNew); + pNew->op = TK_INTEGER; + ExprSetProperty(pNew, EP_IntValue); } /* Ensure that the expression now has an implicit collation sequence, ** just as it did when it was a column of a view or sub-query. */ { - CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pNew); CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pSubst->pCList->a[iColumn].pExpr ); - if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ - pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + if( pNat!=pColl || (pNew->op!=TK_COLUMN && pNew->op!=TK_COLLATE) ){ + pNew = sqlite3ExprAddCollateString(pSubst->pParse, pNew, (pColl ? pColl->zName : "BINARY") ); } } - ExprClearProperty(pExpr, EP_Collate); + ExprClearProperty(pNew, EP_Collate); + if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){ + sqlite3SetJoinExpr(pNew, pExpr->w.iJoin, + pExpr->flags & (EP_OuterON|EP_InnerON)); + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; } } }else{ if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ pExpr->iTable = pSubst->iNewTable; } + if( pExpr->op==TK_AGG_FUNCTION && pExpr->op2>=pSubst->nSelDepth ){ + pExpr->op2--; + } pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); pExpr->pRight = substExpr(pSubst, pExpr->pRight); if( ExprUseXSelect(pExpr) ){ @@ -3949,6 +4051,7 @@ static void substSelect( SrcItem *pItem; int i; if( !p ) return; + pSubst->nSelDepth++; do{ substExprList(pSubst, p->pEList); substExprList(pSubst, p->pGroupBy); @@ -3958,12 +4061,15 @@ static void substSelect( pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(pSubst, pItem->pSelect, 1); + if( pItem->fg.isSubquery ){ + substSelect(pSubst, pItem->u4.pSubq->pSelect, 1); + } if( pItem->fg.isTabFunc ){ substExprList(pSubst, pItem->u1.pFuncArg); } } }while( doPrior && (p = p->pPrior)!=0 ); + pSubst->nSelDepth--; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -3989,7 +4095,7 @@ static void recomputeColumnsUsed( SrcItem *pSrcItem /* Which FROM clause item to recompute */ ){ Walker w; - if( NEVER(pSrcItem->pTab==0) ) return; + if( NEVER(pSrcItem->pSTab==0) ) return; memset(&w, 0, sizeof(w)); w.xExprCallback = recomputeColumnsUsedExpr; w.xSelectCallback = sqlite3SelectWalkNoop; @@ -4002,7 +4108,7 @@ static void recomputeColumnsUsed( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** Assign new cursor numbers to each of the items in pSrc. For each -** new cursor number assigned, set an entry in the aCsrMap[] array +** new cursor number assigned, set an entry in the aCsrMap[] array ** to map the old cursor number to the new: ** ** aCsrMap[iOld+1] = iNew; @@ -4029,8 +4135,10 @@ static void srclistRenumberCursors( aCsrMap[pItem->iCursor+1] = pParse->nTab++; } pItem->iCursor = aCsrMap[pItem->iCursor+1]; - for(p=pItem->pSelect; p; p=p->pPrior){ - srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + if( pItem->fg.isSubquery ){ + for(p=pItem->u4.pSubq->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } } } } @@ -4064,10 +4172,10 @@ static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ /* ** Assign a new cursor number to each cursor in the FROM clause (Select.pSrc) -** of the SELECT statement passed as the second argument, and to each +** of the SELECT statement passed as the second argument, and to each ** cursor in the FROM clause of any FROM clause sub-selects, recursively. ** Except, do not assign a new cursor number to the iExcept'th element in -** the FROM clause of (*p). Update all expressions and other references +** the FROM clause of (*p). Update all expressions and other references ** to refer to the new cursor numbers. ** ** Argument aCsrMap is an array that may be used for temporary working @@ -4076,8 +4184,8 @@ static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ ** * the array is larger than the largest cursor number used within the ** select statement passed as an argument, and ** -** * the array entries for all cursor numbers that do *not* appear in -** FROM clauses of the select statement as described above are +** * the array entries for all cursor numbers that do *not* appear in +** FROM clauses of the select statement as described above are ** initialized to zero. */ static void renumberCursors( @@ -4159,7 +4267,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 ** ** The code generated for this simplification gives the same result -** but only has to scan the data once. And because indices might +** but only has to scan the data once. And because indices might ** exist on the table t1, a complete scan of the data might be ** avoided. ** @@ -4177,9 +4285,9 @@ static int compoundHasDifferentAffinities(Select *p){ ** from 2015-02-09.) ** ** (3) If the subquery is the right operand of a LEFT JOIN then -** (3a) the subquery may not be a join and -** (3b) the FROM clause of the subquery may not contain a virtual -** table and +** (3a) the subquery may not be a join +** (**) Was (3b): "the FROM clause of the subquery may not contain +** a virtual table" ** (**) Was: "The outer query may not have a GROUP BY." This case ** is now managed correctly ** (3d) the outer query may not be DISTINCT. @@ -4188,7 +4296,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** (4) The subquery can not be DISTINCT. ** ** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT -** sub-queries that were excluded from this optimization. Restriction +** sub-queries that were excluded from this optimization. Restriction ** (4) has since been expanded to exclude all DISTINCT subqueries. ** ** (**) We no longer attempt to flatten aggregate subqueries. Was: @@ -4204,8 +4312,8 @@ static int compoundHasDifferentAffinities(Select *p){ ** (9) If the subquery uses LIMIT then the outer query may not be aggregate. ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we -** accidently carried the comment forward until 2014-09-15. Original -** constraint: "If the subquery is aggregate then the outer query +** accidentally carried the comment forward until 2014-09-15. Original +** constraint: "If the subquery is aggregate then the outer query ** may not use LIMIT." ** ** (11) The subquery and the outer query may not both have ORDER BY clauses. @@ -4223,7 +4331,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** ** (16) If the outer query is aggregate, then the subquery may not ** use ORDER BY. (Ticket #2942) This used to not matter -** until we introduced the group_concat() function. +** until we introduced the group_concat() function. ** ** (17) If the subquery is a compound select, then ** (17a) all compound operators must be a UNION ALL, and @@ -4278,7 +4386,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** recursive queries in multiSelect(). ** ** (**) We no longer attempt to flatten aggregate subqueries. Was: -** The subquery may not be an aggregate that uses the built-in min() or +** The subquery may not be an aggregate that uses the built-in min() or ** or max() functions. (Without this restriction, a query like: ** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily ** return the value X for which Y was maximal.) @@ -4296,7 +4404,8 @@ static int compoundHasDifferentAffinities(Select *p){ ** (27b) the subquery is a compound query and the RIGHT JOIN occurs ** in any arm of the compound query. (See also (17g).) ** -** (28) The subquery is not a MATERIALIZED CTE. +** (28) The subquery is not a MATERIALIZED CTE. (This is handled +** in the caller before ever reaching this routine.) ** ** ** In this routine, the "p" parameter is a pointer to the outer query. @@ -4323,7 +4432,7 @@ static int flattenSubquery( SrcList *pSubSrc; /* The FROM clause of the subquery */ int iParent; /* VDBE cursor number of the pSub result set temp table */ int iNewParent = -1;/* Replacement table for iParent */ - int isOuterJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ + int isOuterJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ int i; /* Loop counter */ Expr *pWhere; /* The WHERE clause */ SrcItem *pSubitem; /* The subquery */ @@ -4340,7 +4449,8 @@ static int flattenSubquery( assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; - pSub = pSubitem->pSelect; + assert( pSubitem->fg.isSubquery ); + pSub = pSubitem->u4.pSubq->pSelect; assert( pSub!=0 ); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -4393,7 +4503,7 @@ static int flattenSubquery( */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || IsVirtual(pSubSrc->a[0].pTab) /* (3b) */ + /**** || IsVirtual(pSubSrc->a[0].pSTab) (3b)-omitted */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ @@ -4406,9 +4516,9 @@ static int flattenSubquery( if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ return 0; /* Restriction (27a) */ } - if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){ - return 0; /* (28) */ - } + + /* Condition (28) is blocked by the caller */ + assert( !pSubitem->fg.isCte || pSubitem->u2.pCteUse->eM10d!=M10d_Yes ); /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries @@ -4478,25 +4588,29 @@ static int flattenSubquery( testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; - /* Delete the transient structures associated with thesubquery */ - pSub1 = pSubitem->pSelect; - sqlite3DbFree(db, pSubitem->zDatabase); + /* Delete the transient structures associated with the subquery */ + + if( ALWAYS(pSubitem->fg.isSubquery) ){ + pSub1 = sqlite3SubqueryDetach(db, pSubitem); + }else{ + pSub1 = 0; + } + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->fg.fixedSchema==0 ); sqlite3DbFree(db, pSubitem->zName); sqlite3DbFree(db, pSubitem->zAlias); - pSubitem->zDatabase = 0; pSubitem->zName = 0; pSubitem->zAlias = 0; - pSubitem->pSelect = 0; assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); /* If the sub-query is a compound SELECT statement, then (by restrictions - ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** 17 and 18 above) it must be a UNION ALL and the parent query must ** be of the form: ** - ** SELECT FROM () + ** SELECT FROM () ** ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block - ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or ** OFFSET clauses and joins them to the left-hand-side of the original ** using UNION ALL operators. In this case N is the number of simple ** select statements in the compound sub-query. @@ -4527,8 +4641,8 @@ static int flattenSubquery( ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; Select *pPrior = p->pPrior; - Table *pItemTab = pSubitem->pTab; - pSubitem->pTab = 0; + Table *pItemTab = pSubitem->pSTab; + pSubitem->pSTab = 0; p->pOrderBy = 0; p->pPrior = 0; p->pLimit = 0; @@ -4536,7 +4650,7 @@ static int flattenSubquery( p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->op = TK_ALL; - pSubitem->pTab = pItemTab; + pSubitem->pSTab = pItemTab; if( pNew==0 ){ p->pPrior = pPrior; }else{ @@ -4551,11 +4665,14 @@ static int flattenSubquery( TREETRACE(0x4,pParse,p,("compound-subquery flattener" " creates %u as peer\n",pNew->selId)); } - assert( pSubitem->pSelect==0 ); + assert( pSubitem->fg.isSubquery==0 ); } sqlite3DbFree(db, aCsrMap); if( db->mallocFailed ){ - pSubitem->pSelect = pSub1; + assert( pSubitem->fg.fixedSchema==0 ); + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->u4.zDatabase==0 ); + sqlite3SrcItemAttachSubquery(pParse, pSubitem, pSub1, 0); return 1; } @@ -4564,20 +4681,18 @@ static int flattenSubquery( ** complete, since there may still exist Expr.pTab entries that ** refer to the subquery even after flattening. Ticket #3346. ** - ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + ** pSubitem->pSTab is always non-NULL by test restrictions and tests above. */ - if( ALWAYS(pSubitem->pTab!=0) ){ - Table *pTabToDel = pSubitem->pTab; + if( ALWAYS(pSubitem->pSTab!=0) ){ + Table *pTabToDel = pSubitem->pSTab; if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3ParserAddCleanup(pToplevel, - (void(*)(sqlite3*,void*))sqlite3DeleteTable, - pTabToDel); + sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); testcase( pToplevel->earlyCleanup ); }else{ pTabToDel->nTabRef--; } - pSubitem->pTab = 0; + pSubitem->pSTab = 0; } /* The following loop runs once for each term in a compound-subquery @@ -4596,17 +4711,12 @@ static int flattenSubquery( pSub = pSub1; for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; - u8 jointype = 0; - u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ; + u8 jointype = pSubitem->fg.jointype; assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ pSrc = pParent->pSrc; /* FROM clause of the outer query */ - if( pParent==p ){ - jointype = pSubitem->fg.jointype; /* First time through the loop */ - } - /* The subquery uses a single slot of the FROM clause of the outer ** query. If the subquery has more than one element in its FROM clause, ** then expand the outer query to make space for it to hold all elements @@ -4626,26 +4736,29 @@ static int flattenSubquery( pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); if( pSrc==0 ) break; pParent->pSrc = pSrc; + pSubitem = &pSrc->a[iFrom]; } /* Transfer the FROM clause terms from the subquery into the ** outer query. */ + iNewParent = pSubSrc->a[0].iCursor; for(i=0; ia[i+iFrom]; - if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); assert( pItem->fg.isTabFunc==0 ); + assert( pItem->fg.isSubquery + || pItem->fg.fixedSchema + || pItem->u4.zDatabase==0 ); + if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); *pItem = pSubSrc->a[i]; - pItem->fg.jointype |= ltorj; - iNewParent = pSubSrc->a[i].iCursor; + pItem->fg.jointype |= (jointype & JT_LTORJ); memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].fg.jointype &= JT_LTORJ; - pSrc->a[iFrom].fg.jointype |= jointype | ltorj; - - /* Now begin substituting subquery result set expressions for + pSubitem->fg.jointype |= jointype; + + /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. - ** + ** ** Example: ** ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; @@ -4660,7 +4773,7 @@ static int flattenSubquery( ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, - ** zero them before transfering the ORDER BY clause. + ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this ** function attempts to flatten a compound sub-query into pParent @@ -4677,6 +4790,7 @@ static int flattenSubquery( pWhere = pSub->pWhere; pSub->pWhere = 0; if( isOuterJoin>0 ){ + assert( pSubSrc->nSrc==1 ); sqlite3SetJoinExpr(pWhere, iNewParent, EP_OuterON); } if( pWhere ){ @@ -4692,16 +4806,17 @@ static int flattenSubquery( x.iTable = iParent; x.iNewTable = iNewParent; x.isOuterJoin = isOuterJoin; + x.nSelDepth = 0; x.pEList = pSub->pEList; x.pCList = findLeftmostExprlist(pSub); substSelect(&x, pParent, 0); } - + /* The flattened query is a compound if either the inner or the ** outer query is a compound. */ pParent->selFlags |= pSub->selFlags & SF_Compound; assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */ - + /* ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; ** @@ -4720,8 +4835,7 @@ static int flattenSubquery( } } - /* Finially, delete what is left of the subquery and return - ** success. + /* Finally, delete what is left of the subquery and return success. */ sqlite3AggInfoPersistWalkerInit(&w, pParse); sqlite3WalkSelect(&w,pSub1); @@ -4756,7 +4870,7 @@ struct WhereConst { /* ** Add a new entry to the pConst object. Except, do not add duplicate -** pColumn entires. Also, do not add if doing so would not be appropriate. +** pColumn entries. Also, do not add if doing so would not be appropriate. ** ** The caller guarantees the pColumn is a column and pValue is a constant. ** This routine has to do some additional checks before completing the @@ -4770,7 +4884,7 @@ static void constInsert( ){ int i; assert( pColumn->op==TK_COLUMN ); - assert( sqlite3ExprIsConstant(pValue) ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); if( ExprHasProperty(pColumn, EP_FixedCol) ) return; if( sqlite3ExprAffinity(pValue)!=0 ) return; @@ -4789,7 +4903,8 @@ static void constInsert( return; /* Already present. Return without doing anything. */ } } - if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONEbHasAffBlob = 1; } @@ -4828,10 +4943,10 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ pLeft = pExpr->pLeft; assert( pRight!=0 ); assert( pLeft!=0 ); - if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ constInsert(pConst,pRight,pLeft,pExpr); } - if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ constInsert(pConst,pLeft,pRight,pExpr); } } @@ -4839,15 +4954,15 @@ static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ /* ** This is a helper function for Walker callback propagateConstantExprRewrite(). ** -** Argument pExpr is a candidate expression to be replaced by a value. If -** pExpr is equivalent to one of the columns named in pWalker->u.pConst, +** Argument pExpr is a candidate expression to be replaced by a value. If +** pExpr is equivalent to one of the columns named in pWalker->u.pConst, ** then overwrite it with the corresponding value. Except, do not do so ** if argument bIgnoreAffBlob is non-zero and the affinity of pExpr ** is SQLITE_AFF_BLOB. */ static int propagateConstantExprRewriteOne( WhereConst *pConst, - Expr *pExpr, + Expr *pExpr, int bIgnoreAffBlob ){ int i; @@ -4864,7 +4979,8 @@ static int propagateConstantExprRewriteOne( if( pColumn==pExpr ) continue; if( pColumn->iTable!=pExpr->iTable ) continue; if( pColumn->iColumn!=pExpr->iColumn ) continue; - if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONE=, <, >) that ** uses an affinity other than TEXT and one of its immediate -** children is a column that matches one of the columns in +** children is a column that matches one of the columns in ** pWalker->u.pConst. */ static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ @@ -4942,7 +5058,7 @@ static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ ** SELECT * FROM t1 WHERE a=123 AND b=123; ** ** The two SELECT statements above should return different answers. b=a -** is alway true because the comparison uses numeric affinity, but b=123 +** is always true because the comparison uses numeric affinity, but b=123 ** is false because it uses text affinity and '0123' is not the same as '123'. ** To work around this, the expression tree is not actually changed from ** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol @@ -5004,7 +5120,7 @@ static int propagateConstants( sqlite3DbFree(x.pParse->db, x.apExpr); nChng += x.nChng; } - }while( x.nChng ); + }while( x.nChng ); return nChng; } @@ -5014,19 +5130,19 @@ static int propagateConstants( ** This function is called to determine whether or not it is safe to ** push WHERE clause expression pExpr down to FROM clause sub-query ** pSubq, which contains at least one window function. Return 1 -** if it is safe and the expression should be pushed down, or 0 +** if it is safe and the expression should be pushed down, or 0 ** otherwise. ** -** It is only safe to push the expression down if it consists only +** It is only safe to push the expression down if it consists only ** of constants and copies of expressions that appear in the PARTITION ** BY clause of all window function used by the sub-query. It is safe ** to filter out entire partitions, but not rows within partitions, as ** this may change the results of the window functions. ** -** At the time this function is called it is guaranteed that +** At the time this function is called it is guaranteed that ** -** * the sub-query uses only one distinct window frame, and -** * that the window frame has a PARTITION BY clase. +** * the sub-query uses only one distinct window frame, and +** * that the window frame has a PARTITION BY clause. */ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ assert( pSubq->pWin->pPartition ); @@ -5052,6 +5168,19 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization" +** or sometimes the "predicate push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** ** (1) (** This restriction was removed on 2017-09-29. We used to @@ -5084,19 +5213,19 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** But if the (b2=2) term were to be pushed down into the bb subquery, ** then the (1,1,NULL) row would be suppressed. ** -** (6) Window functions make things tricky as changes to the WHERE clause -** of the inner query could change the window over which window +** (6) Window functions make things tricky as changes to the WHERE clause +** of the inner query could change the window over which window ** functions are calculated. Therefore, do not attempt the optimization ** if: ** ** (6a) The inner query uses multiple incompatible window partitions. ** -** (6b) The inner query is a compound and uses window-functions. +** (6b) The inner query is a compound and uses window-functions. ** ** (6c) The WHERE clause does not consist entirely of constants and ** copies of expressions found in the PARTITION BY clause of ** all window-functions used by the sub-query. It is safe to -** filter out entire partitions, as this does not change the +** filter out entire partitions, as this does not change the ** window over which any window-function is calculated. ** ** (7) The inner query is a Common Table Expression (CTE) that should @@ -5117,15 +5246,19 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING ** clause and the subquery. ** -** Without this restriction, the push-down optimization might move -** the ON/USING filter expression from the left side of a RIGHT JOIN -** over to the right side, which leads to incorrect answers. See -** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). ** ** (10) The inner query is not the right-hand table of a RIGHT JOIN. ** ** (11) The subquery is not a VALUES clause ** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -5153,7 +5286,7 @@ static int pushDownWhereTerms( int notUnionAll = 0; for(pSel=pSubq; pSel; pSel=pSel->pPrior){ u8 op = pSel->op; - assert( op==TK_ALL || op==TK_SELECT + assert( op==TK_ALL || op==TK_SELECT || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); if( op!=TK_ALL && op!=TK_SELECT ){ notUnionAll = 1; @@ -5190,7 +5323,7 @@ static int pushDownWhereTerms( ** in the future. */ { - Select *pX; + Select *pX; for(pX=pSubq; pX; pX=pX->pPrior){ assert( (pX->selFlags & (SF_Recursive))==0 ); } @@ -5230,13 +5363,25 @@ static int pushDownWhereTerms( return 0; /* restriction (4) */ } if( ExprHasProperty(pWhere,EP_OuterON) - && pWhere->w.iJoin!=iCursor + && pWhere->w.iJoin!=iCursor ){ return 0; /* restriction (5) */ } #endif - if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -5247,6 +5392,7 @@ static int pushDownWhereTerms( x.iTable = pSrc->iCursor; x.iNewTable = pSrc->iCursor; x.isOuterJoin = 0; + x.nSelDepth = 0; x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); @@ -5290,17 +5436,17 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ if( pItem->fg.isCorrelated || pItem->fg.isCte ){ return 0; } - assert( pItem->pTab!=0 ); - pTab = pItem->pTab; - assert( pItem->pSelect!=0 ); - pSub = pItem->pSelect; + assert( pItem->pSTab!=0 ); + pTab = pItem->pSTab; + assert( pItem->fg.isSubquery ); + pSub = pItem->u4.pSubq->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); - if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ - testcase( pSub->selFlags & SF_Distinct ); - testcase( pSub->selFlags & SF_Aggregate ); - return 0; - } for(pX=pSub; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pX->selFlags & SF_Distinct ); + testcase( pX->selFlags & SF_Aggregate ); + return 0; + } if( pX->pPrior && pX->op!=TK_ALL ){ /* This optimization does not work for compound subqueries that ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ @@ -5344,7 +5490,7 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ /* ** The pFunc is the only aggregate function in the query. Check to see -** if the query is a candidate for the min/max optimization. +** if the query is a candidate for the min/max optimization. ** ** If the query is a candidate for the min/max optimization, then set ** *ppMinMax to be an ORDER BY clause to be used for the optimization @@ -5370,7 +5516,7 @@ static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ assert( !IsWindowFunc(pFunc) ); assert( ExprUseXList(pFunc) ); pEList = pFunc->x.pList; - if( pEList==0 + if( pEList==0 || pEList->nExpr!=1 || ExprHasProperty(pFunc, EP_WinFunc) || OptimizationDisabled(db, SQLITE_MinMaxOpt) @@ -5398,7 +5544,7 @@ static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ /* ** The select statement passed as the first argument is an aggregate query. -** The second argument is the associated aggregate-info object. This +** The second argument is the associated aggregate-info object. This ** function tests if the SELECT is of the form: ** ** SELECT count(*) FROM @@ -5419,16 +5565,16 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ assert( !p->pGroupBy ); - if( p->pWhere - || p->pEList->nExpr!=1 + if( p->pWhere + || p->pEList->nExpr!=1 || p->pSrc->nSrc!=1 - || p->pSrc->a[0].pSelect + || p->pSrc->a[0].fg.isSubquery || pAggInfo->nFunc!=1 || p->pHaving ){ return 0; } - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; assert( pTab!=0 ); assert( !IsView(pTab) ); if( !IsOrdinaryTable(pTab) ) return 0; @@ -5448,19 +5594,19 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ /* ** If the source-list item passed as an argument was augmented with an ** INDEXED BY clause, then try to locate the specified index. If there -** was such a clause and the named index cannot be found, return -** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate ** pFrom->pIndex and return SQLITE_OK. */ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; char *zIndexedBy = pFrom->u1.zIndexedBy; Index *pIdx; assert( pTab!=0 ); assert( pFrom->fg.isIndexedBy!=0 ); - for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); pIdx=pIdx->pNext ); if( !pIdx ){ @@ -5474,7 +5620,7 @@ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ } /* -** Detect compound SELECT statements that use an ORDER BY clause with +** Detect compound SELECT statements that use an ORDER BY clause with ** an alternative collating sequence. ** ** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... @@ -5488,7 +5634,7 @@ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ ** above that generates the code for a compound SELECT with an ORDER BY clause ** uses a merge algorithm that requires the same collating sequence on the ** result columns as on the ORDER BY clause. See ticket -** http://www.sqlite.org/src/info/6709574d2a +** http://sqlite.org/src/info/6709574d2a ** ** This transformation is only needed for EXCEPT, INTERSECT, and UNION. ** The UNION ALL operator works fine with multiSelectOrderBy() even when @@ -5530,7 +5676,11 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ if( pNew==0 ) return WRC_Abort; memset(&dummy, 0, sizeof(dummy)); pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0); - if( pNewSrc==0 ) return WRC_Abort; + assert( pNewSrc!=0 || pParse->nErr ); + if( pParse->nErr ){ + sqlite3SrcListDelete(db, pNewSrc); + return WRC_Abort; + } *pNew = *p; p->pSrc = pNewSrc; p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); @@ -5545,7 +5695,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ #ifndef SQLITE_OMIT_WINDOWFUNC p->pWinDefn = 0; #endif - p->selFlags &= ~SF_Compound; + p->selFlags &= ~(u32)SF_Compound; assert( (p->selFlags & SF_Converted)==0 ); p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); @@ -5569,9 +5719,9 @@ static int cannotBeFunction(Parse *pParse, SrcItem *pFrom){ #ifndef SQLITE_OMIT_CTE /* -** Argument pWith (which may be NULL) points to a linked list of nested -** WITH contexts, from inner to outermost. If the table identified by -** FROM clause element pItem is really a common-table-expression (CTE) +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) ** then return a pointer to the CTE definition for that table. Otherwise ** return NULL. ** @@ -5585,7 +5735,7 @@ static struct Cte *searchWith( ){ const char *zName = pItem->zName; With *p; - assert( pItem->zDatabase==0 ); + assert( pItem->fg.fixedSchema || pItem->u4.zDatabase==0 ); assert( zName!=0 ); for(p=pWith; p; p=p->pOuter){ int i; @@ -5607,7 +5757,7 @@ static struct Cte *searchWith( ** onto the top of the stack. If argument bFree is true, then this ** WITH clause will never be popped from the stack but should instead ** be freed along with the Parse object. In other cases, when -** bFree==0, the With object will be freed along with the SELECT +** bFree==0, the With object will be freed along with the SELECT ** statement with which it is associated. ** ** This routine returns a copy of pWith. Or, if bFree is true and @@ -5620,8 +5770,7 @@ static struct Cte *searchWith( With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ if( pWith ){ if( bFree ){ - pWith = (With*)sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3WithDelete, + pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric, pWith); if( pWith==0 ) return 0; } @@ -5635,16 +5784,16 @@ With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ } /* -** This function checks if argument pFrom refers to a CTE declared by +** This function checks if argument pFrom refers to a CTE declared by ** a WITH clause on the stack currently maintained by the parser (on the ** pParse->pWith linked list). And if currently processing a CTE ** CTE expression, through routine checks to see if the reference is ** a recursive reference to the CTE. ** -** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** If pFrom matches a CTE according to either of these two above, pFrom->pSTab ** and other fields are populated accordingly. ** -** Return 0 if no match is found. +** Return 0 if no match is found. ** Return 1 if a match is found. ** Return 2 if an error condition is detected. */ @@ -5656,7 +5805,7 @@ static int resolveFromTermToCte( Cte *pCte; /* Matched CTE (or NULL if no match) */ With *pWith; /* The matching WITH */ - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( pParse->pWith==0 ){ /* There are no WITH clauses in the stack. No match is possible */ return 0; @@ -5666,7 +5815,8 @@ static int resolveFromTermToCte( ** go no further. */ return 0; } - if( pFrom->zDatabase!=0 ){ + assert( pFrom->fg.hadSchema==0 || pFrom->fg.notCte!=0 ); + if( pFrom->fg.fixedSchema==0 && pFrom->u4.zDatabase!=0 ){ /* The FROM term contains a schema qualifier (ex: main.t1) and so ** it cannot possibly be a CTE reference. */ return 0; @@ -5702,7 +5852,7 @@ static int resolveFromTermToCte( } if( cannotBeFunction(pParse, pFrom) ) return 2; - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return 2; pCteUse = pCte->pUse; @@ -5716,26 +5866,29 @@ static int resolveFromTermToCte( } pCteUse->eM10d = pCte->eM10d; } - pFrom->pTab = pTab; + pFrom->pSTab = pTab; pTab->nTabRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; - pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pCte->pSelect, 1); if( db->mallocFailed ) return 2; - pFrom->pSelect->selFlags |= SF_CopyCte; - assert( pFrom->pSelect ); + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + pSel = pFrom->u4.pSubq->pSelect; + assert( pSel!=0 ); + pSel->selFlags |= SF_CopyCte; if( pFrom->fg.isIndexedBy ){ sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); return 2; } + assert( !pFrom->fg.isIndexedBy ); pFrom->fg.isCte = 1; pFrom->u2.pCteUse = pCteUse; pCteUse->nUse++; /* Check if this is a recursive CTE. */ - pRecTerm = pSel = pFrom->pSelect; + pRecTerm = pSel; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; @@ -5743,11 +5896,13 @@ static int resolveFromTermToCte( assert( pRecTerm->pPrior!=0 ); for(i=0; inSrc; i++){ SrcItem *pItem = &pSrc->a[i]; - if( pItem->zDatabase==0 - && pItem->zName!=0 + if( pItem->zName!=0 + && !pItem->fg.hadSchema + && ALWAYS( !pItem->fg.isSubquery ) + && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0) && 0==sqlite3StrICmp(pItem->zName, pCte->zName) ){ - pItem->pTab = pTab; + pItem->pSTab = pTab; pTab->nTabRef++; pItem->fg.isRecursive = 1; if( pRecTerm->selFlags & SF_Recursive ){ @@ -5822,12 +5977,12 @@ static int resolveFromTermToCte( #ifndef SQLITE_OMIT_CTE /* -** If the SELECT passed as the second argument has an associated WITH +** If the SELECT passed as the second argument has an associated WITH ** clause, pop it from the stack stored as part of the Parse object. ** ** This function is used as the xSelectCallback2() callback by ** sqlite3SelectExpand() when walking a SELECT tree to resolve table -** names and other FROM clause elements. +** names and other FROM clause elements. */ void sqlite3SelectPopWith(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; @@ -5849,11 +6004,14 @@ void sqlite3SelectPopWith(Walker *pWalker, Select *p){ ** SQLITE_NOMEM. */ int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ - Select *pSel = pFrom->pSelect; + Select *pSel; Table *pTab; + assert( pFrom->fg.isSubquery ); + assert( pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; assert( pSel ); - pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + pFrom->pSTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); if( pTab==0 ) return SQLITE_NOMEM; pTab->nTabRef = 1; if( pFrom->zAlias ){ @@ -5864,12 +6022,14 @@ int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); #ifndef SQLITE_ALLOW_ROWID_IN_VIEW /* The usual case - do not allow ROWID on a subquery */ pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; #else - pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; #endif return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; } @@ -5907,7 +6067,7 @@ static int inAnyUsingClause( ** (1) Make sure VDBE cursor numbers have been assigned to every ** element of the FROM clause. ** -** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that ** defines FROM clause. When views appear in the FROM clause, ** fill pTabList->a[].pSelect with a copy of the SELECT statement ** that implements the view. A copy is made of the view's SELECT @@ -5951,7 +6111,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pEList = p->pEList; if( pParse->pWith && (p->selFlags & SF_View) ){ if( p->pWith==0 ){ - p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With)); + p->pWith = (With*)sqlite3DbMallocZero(db, SZ_WITH(1) ); if( p->pWith==0 ){ return WRC_Abort; } @@ -5971,33 +6131,35 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); - if( pFrom->pTab ) continue; + assert( pFrom->fg.isRecursive==0 || pFrom->pSTab!=0 ); + if( pFrom->pSTab ) continue; assert( pFrom->fg.isRecursive==0 ); if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->pSelect; + Select *pSel; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; #endif #ifndef SQLITE_OMIT_CTE }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ if( rc>1 ) return WRC_Abort; - pTab = pFrom->pTab; + pTab = pFrom->pSTab; assert( pTab!=0 ); #endif }else{ /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + assert( pFrom->pSTab==0 ); + pFrom->pSTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; if( pTab->nTabRef>=0xffff ){ sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", pTab->zName); - pFrom->pTab = 0; + pFrom->pSTab = 0; return WRC_Abort; } pTab->nTabRef++; @@ -6009,7 +6171,7 @@ static int selectExpander(Walker *pWalker, Select *p){ i16 nCol; u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->pSelect==0 ); + assert( pFrom->fg.isSubquery==0 ); if( IsView(pTab) ){ if( (db->flags & SQLITE_EnableView)==0 && pTab->pSchema!=db->aDb[1].pSchema @@ -6017,7 +6179,7 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", pTab->zName); } - pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pTab->u.view.pSelect, 1); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) @@ -6033,7 +6195,9 @@ static int selectExpander(Walker *pWalker, Select *p){ nCol = pTab->nCol; pTab->nCol = -1; pWalker->eCode = 1; /* Turn on Select.selId renumbering */ - sqlite3WalkSelect(pWalker, pFrom->pSelect); + if( pFrom->fg.isSubquery ){ + sqlite3WalkSelect(pWalker, pFrom->u4.pSubq->pSelect); + } pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } @@ -6106,13 +6270,21 @@ static int selectExpander(Walker *pWalker, Select *p){ ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ char *zTName = 0; /* text of name of TABLE */ + int iErrOfst; if( pE->op==TK_DOT ){ + assert( (selFlags & SF_NestedFrom)==0 ); assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; + assert( ExprUseWOfst(pE->pLeft) ); + iErrOfst = pE->pRight->w.iOfst; + }else{ + assert( ExprUseWOfst(pE) ); + iErrOfst = pE->w.iOfst; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; /* Table for this data source */ + int nAdd; /* Number of cols including rowid */ + Table *pTab = pFrom->pSTab; /* Table for this data source */ ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ char *zTabName; /* AS name for this data source */ const char *zSchemaName = 0; /* Schema name for this data source */ @@ -6123,12 +6295,14 @@ static int selectExpander(Walker *pWalker, Select *p){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom->pSelect) ); + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom) ); if( pFrom->fg.isNestedFrom ){ - assert( pFrom->pSelect!=0 ); - pNestedFrom = pFrom->pSelect->pEList; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + assert( pFrom->u4.pSubq->pSelect!=0 ); + pNestedFrom = pFrom->u4.pSubq->pSelect->pEList; assert( pNestedFrom!=0 ); assert( pNestedFrom->nExpr==pTab->nCol ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; @@ -6146,6 +6320,7 @@ static int selectExpander(Walker *pWalker, Select *p){ for(ii=0; iinId; ii++){ const char *zUName = pUsing->a[ii].zName; pRight = sqlite3Expr(db, TK_ID, zUName); + sqlite3ExprSetErrorOffset(pRight, iErrOfst); pNew = sqlite3ExprListAppend(pParse, pNew, pRight); if( pNew ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; @@ -6158,33 +6333,49 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ pUsing = 0; } - for(j=0; jnCol; j++){ - char *zName = pTab->aCol[j].zCnName; + + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; + for(j=0; ja[j], 0, zTName, 0)==0 - ){ - continue; - } + if( j==pTab->nCol ){ + zName = sqlite3RowidAlias(pTab); + if( zName==0 ) continue; + }else{ + zName = pTab->aCol[j].zCnName; - /* If a column is marked as 'hidden', omit it from the expanded - ** result-set list unless the SELECT has the SF_IncludeHidden - ** bit set. - */ - if( (p->selFlags & SF_IncludeHidden)==0 - && IsHiddenColumn(&pTab->aCol[j]) - ){ - continue; - } - if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 - && zTName==0 - && (selFlags & (SF_NestedFrom))==0 - ){ - continue; + /* If pTab is actually an SF_NestedFrom sub-select, do not + ** expand any ENAME_ROWID columns. */ + if( pNestedFrom && pNestedFrom->a[j].fg.eEName==ENAME_ROWID ){ + continue; + } + + if( zTName + && pNestedFrom + && sqlite3MatchEName(&pNestedFrom->a[j], 0, zTName, 0, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. + */ + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ + continue; + } + if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 + && zTName==0 + && (selFlags & (SF_NestedFrom))==0 + ){ + continue; + } } + assert( zName ); tableSeen = 1; if( i>0 && zTName==0 && (selFlags & SF_NestedFrom)==0 ){ @@ -6218,6 +6409,7 @@ static int selectExpander(Walker *pWalker, Select *p){ }else{ pExpr = pRight; } + sqlite3ExprSetErrorOffset(pExpr, iErrOfst); pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); if( pNew==0 ){ break; /* OOM */ @@ -6225,7 +6417,8 @@ static int selectExpander(Walker *pWalker, Select *p){ pX = &pNew->a[pNew->nExpr-1]; assert( pX->zEName==0 ); if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ - if( pNestedFrom ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); testcase( pX->zEName==0 ); }else{ @@ -6233,11 +6426,11 @@ static int selectExpander(Walker *pWalker, Select *p){ zSchemaName, zTabName, zName); testcase( pX->zEName==0 ); } - pX->fg.eEName = ENAME_TAB; + pX->fg.eEName = (j==pTab->nCol ? ENAME_ROWID : ENAME_TAB); if( (pFrom->fg.isUsing && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0) || (pUsing && sqlite3IdListIndex(pUsing, zName)>=0) - || (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 + || (jnCol && (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)) ){ pX->fg.bNoExpand = 1; } @@ -6339,20 +6532,18 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ SrcList *pTabList; SrcItem *pFrom; - assert( p->selFlags & SF_Resolved ); if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + if( (pTab->tabFlags & TF_Ephemeral)!=0 && pFrom->fg.isSubquery ){ /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); - } + Select *pSel = pFrom->u4.pSubq->pSelect; + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -6412,14 +6603,16 @@ void sqlite3SelectPrep( */ static void printAggInfo(AggInfo *pAggInfo){ int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); for(ii=0; iinColumn; ii++){ struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; sqlite3DebugPrintf( "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" " iSorterColumn=%d %s\n", - ii, pCol->pTab ? pCol->pTab->zName : "NULL", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, - pCol->iSorterColumn, + pCol->iSorterColumn, ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); } @@ -6458,8 +6651,14 @@ static void analyzeAggFuncArgs( pNC->ncFlags |= NC_InAggFunc; for(i=0; inFunc; i++){ Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr->op==TK_FUNCTION || pExpr->op==TK_AGG_FUNCTION ); assert( ExprUseXList(pExpr) ); sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->pLeft->x.pList); + } #ifndef SQLITE_OMIT_WINDOWFUNC assert( !IsWindowFunc(pExpr) ); if( ExprHasProperty(pExpr, EP_WinFunc) ){ @@ -6534,7 +6733,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ pExpr->op = TK_AGG_COLUMN; pExpr->iTable = pCol->iTable; pExpr->iColumn = pCol->iColumn; - ExprClearProperty(pExpr, EP_Skip|EP_Collate); + ExprClearProperty(pExpr, EP_Skip|EP_Collate|EP_Unlikely); return WRC_Prune; } @@ -6565,7 +6764,7 @@ static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ ** * The aCol[] and aFunc[] arrays may be modified ** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used ** -** After clling this routine: +** After calling this routine: ** ** * The aCol[] and aFunc[] arrays are fixed ** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used @@ -6608,12 +6807,43 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ pFunc->iDistinct = -1; }else{ KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); - pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(DISTINCT)", pFunc->pFunc->zName)); } } + if( pFunc->iOBTab>=0 ){ + ExprList *pOBList; + KeyInfo *pKeyInfo; + int nExtra = 0; + assert( pFunc->pFExpr->pLeft!=0 ); + assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pFunc->pFExpr->pLeft) ); + assert( pFunc->pFunc!=0 ); + pOBList = pFunc->pFExpr->pLeft->x.pList; + if( !pFunc->bOBUnique ){ + nExtra++; /* One extra column for the OP_Sequence */ + } + if( pFunc->bOBPayload ){ + /* extra columns for the function arguments */ + assert( ExprUseXList(pFunc->pFExpr) ); + assert( pFunc->pFExpr->x.pList!=0 ); + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + if( pFunc->bUseSubtype ){ + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); + if( !pFunc->bOBUnique && pParse->nErr==0 ){ + pKeyInfo->nKeyField++; + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iOBTab, pOBList->nExpr+nExtra, 0, + (char*)pKeyInfo, P4_KEYINFO); + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(ORDER BY)", + pFunc->pFunc->zName)); + } } } @@ -6628,14 +6858,58 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); + if( pParse->nErr ) return; pList = pF->pFExpr->x.pList; + if( pF->iOBTab>=0 ){ + /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs + ** were stored in emphermal table pF->iOBTab. Here, we extract those + ** inputs (in ORDER BY order) and make all calls to OP_AggStep + ** before doing the OP_AggFinal call. */ + int iTop; /* Start of loop for extracting columns */ + int nArg; /* Number of columns to extract */ + int nKey; /* Key columns to be skipped */ + int regAgg; /* Extract into this array */ + int j; /* Loop counter */ + + assert( pF->pFunc!=0 ); + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + + if( pF->bOBPayload==0 ){ + nKey = 0; + }else{ + assert( pF->pFExpr->pLeft!=0 ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + assert( pF->pFExpr->pLeft->x.pList!=0 ); + nKey = pF->pFExpr->pLeft->x.pList->nExpr; + if( ALWAYS(!pF->bOBUnique) ) nKey++; + } + iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); + } + if( pF->bUseSubtype ){ + int regSubtype = sqlite3GetTempReg(pParse); + int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype); + sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j); + } + sqlite3ReleaseTempReg(pParse, regSubtype); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u16)nArg); + sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, iTop); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + } sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), pList ? pList->nExpr : 0); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } - /* ** Generate code that will update the accumulator memory cells for an ** aggregate based on the current cursor position. @@ -6644,10 +6918,17 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ ** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator ** registers if register regAcc contains 0. The caller will take care ** of setting and clearing regAcc. +** +** For an ORDER BY aggregate, the actual accumulator memory cell update +** is deferred until after all input rows have been received, so that they +** can be run in the requested order. In that case, instead of invoking +** OP_AggStep to update the accumulator, just add the arguments that would +** have been passed into OP_AggStep into the sorting ephemeral table +** (along with the appropriate sort key). */ static void updateAccumulator( - Parse *pParse, - int regAcc, + Parse *pParse, + int regAcc, AggInfo *pAggInfo, int eDistinctType ){ @@ -6665,14 +6946,17 @@ static void updateAccumulator( int nArg; int addrNext = 0; int regAgg; + int regAggSz = 0; + int regDistinct = 0; ExprList *pList; assert( ExprUseXList(pF->pFExpr) ); assert( !IsWindowFunc(pF->pFExpr) ); + assert( pF->pFunc!=0 ); pList = pF->pFExpr->x.pList; if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ Expr *pFilter = pF->pFExpr->y.pWin->pFilter; - if( pAggInfo->nAccumulator - && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + if( pAggInfo->nAccumulator + && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) && regAcc ){ /* If regAcc==0, there there exists some min() or max() function @@ -6681,7 +6965,7 @@ static void updateAccumulator( if( regHit==0 ) regHit = ++pParse->nMem; /* If this is the first row of the group (regAcc contains 0), clear the ** "magnet" register regHit so that the accumulator registers - ** are populated if the FILTER clause jumps over the the + ** are populated if the FILTER clause jumps over the the ** invocation of min() or max() altogether. Or, if this is not ** the first row (regAcc contains 1), set the magnet register so that ** the accumulators are not populated unless the min()/max() is invoked @@ -6691,42 +6975,100 @@ static void updateAccumulator( addrNext = sqlite3VdbeMakeLabel(pParse); sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); } - if( pList ){ + if( pF->iOBTab>=0 ){ + /* Instead of invoking AggStep, we must push the arguments that would + ** have been passed to AggStep onto the sorting table. */ + int jj; /* Registered used so far in building the record */ + ExprList *pOBList; /* The ORDER BY clause */ + assert( pList!=0 ); + nArg = pList->nExpr; + assert( nArg>0 ); + assert( pF->pFExpr->pLeft!=0 ); + assert( pF->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + pOBList = pF->pFExpr->pLeft->x.pList; + assert( pOBList!=0 ); + assert( pOBList->nExpr>0 ); + regAggSz = pOBList->nExpr; + if( !pF->bOBUnique ){ + regAggSz++; /* One register for OP_Sequence */ + } + if( pF->bOBPayload ){ + regAggSz += nArg; + } + if( pF->bUseSubtype ){ + regAggSz += nArg; + } + regAggSz++; /* One extra register to hold result of MakeRecord */ + regAgg = sqlite3GetTempRange(pParse, regAggSz); + regDistinct = regAgg; + sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP); + jj = pOBList->nExpr; + if( !pF->bOBUnique ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj); + jj++; + } + if( pF->bOBPayload ){ + regDistinct = regAgg+jj; + sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); + jj += nArg; + } + if( pF->bUseSubtype ){ + int kk; + int regBase = pF->bOBPayload ? regDistinct : regAgg; + for(kk=0; kknExpr; regAgg = sqlite3GetTempRange(pParse, nArg); + regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; } if( pF->iDistinct>=0 && pList ){ - if( addrNext==0 ){ + if( addrNext==0 ){ addrNext = sqlite3VdbeMakeLabel(pParse); } - pF->iDistinct = codeDistinct(pParse, eDistinctType, - pF->iDistinct, addrNext, pList, regAgg); - } - if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - CollSeq *pColl = 0; - struct ExprList_item *pItem; - int j; - assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ - for(j=0, pItem=pList->a; !pColl && jpExpr); - } - if( !pColl ){ - pColl = pParse->db->pDfltColl; + pF->iDistinct = codeDistinct(pParse, eDistinctType, + pF->iDistinct, addrNext, pList, regDistinct); + } + if( pF->iOBTab>=0 ){ + /* Insert a new record into the ORDER BY table */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regAgg, regAggSz-1, + regAgg+regAggSz-1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pF->iOBTab, regAgg+regAggSz-1, + regAgg, regAggSz-1); + sqlite3ReleaseTempRange(pParse, regAgg, regAggSz); + }else{ + /* Invoke the AggStep function */ + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, + (char *)pColl, P4_COLLSEQ); } - if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u16)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); } - sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); - sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); } + if( pParse->nErr ) return; } if( regHit==0 && pAggInfo->nAccumulator ){ regHit = regAcc; @@ -6736,6 +7078,7 @@ static void updateAccumulator( } for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); + if( pParse->nErr ) return; } pAggInfo->directMode = 0; @@ -6770,10 +7113,10 @@ static void explainSimpleCount( /* ** sqlite3WalkExpr() callback used by havingToWhere(). ** -** If the node passed to the callback is a TK_AND node, return +** If the node passed to the callback is a TK_AND node, return ** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes. ** -** Otherwise, return WRC_Prune. In this case, also check if the +** Otherwise, return WRC_Prune. In this case, also check if the ** sub-expression matches the criteria for being moved to the WHERE ** clause. If so, add it to the WHERE clause and replace the sub-expression ** within the HAVING expression with a constant "1". @@ -6788,7 +7131,7 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ ** belongs to an outer query. Do not move the expression to the WHERE ** clause in this obscure case, as doing so may corrupt the outer Select ** statements AggInfo structure. */ - if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) + if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) && ExprAlwaysFalse(pExpr)==0 && pExpr->pAggInfo==0 ){ @@ -6851,25 +7194,28 @@ static SrcItem *isSelfJoinView( int iFirst, int iEnd /* Range of FROM-clause entries to search. */ ){ SrcItem *pItem; - assert( pThis->pSelect!=0 ); - if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + Select *pSel; + assert( pThis->fg.isSubquery ); + pSel = pThis->u4.pSubq->pSelect; + assert( pSel!=0 ); + if( pSel->selFlags & SF_PushDown ) return 0; while( iFirsta[iFirst++]; - if( pItem->pSelect==0 ) continue; + if( !pItem->fg.isSubquery ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; - assert( pItem->pTab!=0 ); - assert( pThis->pTab!=0 ); - if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; + assert( pItem->pSTab!=0 ); + assert( pThis->pSTab!=0 ); + if( pItem->pSTab->pSchema!=pThis->pSTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; - pS1 = pItem->pSelect; - if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + pS1 = pItem->u4.pSubq->pSelect; + if( pItem->pSTab->pSchema==0 && pSel->selId!=pS1->selId ){ /* The query flattener left two different CTE tables with identical ** names in the same FROM clause. */ continue; } - if( pItem->pSelect->selFlags & SF_PushDown ){ + if( pS1->selFlags & SF_PushDown ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ continue; @@ -6882,7 +7228,8 @@ static SrcItem *isSelfJoinView( /* ** Deallocate a single AggInfo object */ -static void agginfoFree(sqlite3 *db, AggInfo *p){ +static void agginfoFree(sqlite3 *db, void *pArg){ + AggInfo *p = (AggInfo*)pArg; sqlite3DbFree(db, p->aCol); sqlite3DbFree(db, p->aFunc); sqlite3DbFreeNN(db, p); @@ -6904,6 +7251,7 @@ static void agginfoFree(sqlite3 *db, AggInfo *p){ ** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries ** * The outer query is a simple count(*) with no WHERE clause or other ** extraneous syntax. +** * None of the subqueries are DISTINCT (forumpost/a860f5fb2e 2025-03-10) ** ** Return TRUE if the optimization is undertaken. */ @@ -6912,6 +7260,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ Expr *pExpr; Expr *pCount; sqlite3 *db; + SrcItem *pFrom; if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; @@ -6926,17 +7275,22 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ - pSub = p->pSrc->a[0].pSelect; - if( pSub==0 ) return 0; /* The FROM is a subquery */ + pFrom = p->pSrc->a; + if( pFrom->fg.isSubquery==0 ) return 0; /* FROM is a subquery */ + pSub = pFrom->u4.pSubq->pSelect; if( pSub->pPrior==0 ) return 0; /* Must be a compound */ if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ - if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ + if( pSub->selFlags & (SF_Aggregate|SF_Distinct) ){ + testcase( pSub->selFlags & SF_Aggregate ); + testcase( pSub->selFlags & SF_Distinct ); + return 0; /* Not an aggregate nor DISTINCT */ + } assert( pSub->pHaving==0 ); /* Due to the previous */ - pSub = pSub->pPrior; /* Repeat over compound */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -6944,19 +7298,18 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ db = pParse->db; pCount = pExpr; pExpr = 0; - pSub = p->pSrc->a[0].pSelect; - p->pSrc->a[0].pSelect = 0; + pSub = sqlite3SubqueryDetach(db, pFrom); sqlite3SrcListDelete(db, p->pSrc); - p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + p->pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); while( pSub ){ Expr *pTerm; pPrior = pSub->pPrior; pSub->pPrior = 0; pSub->pNext = 0; pSub->selFlags |= SF_Aggregate; - pSub->selFlags &= ~SF_Compound; + pSub->selFlags &= ~(u32)SF_Compound; pSub->nSelectRow = 0; - sqlite3ExprListDelete(db, pSub->pEList); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); @@ -6969,7 +7322,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pSub = pPrior; } p->pEList->a[0].pExpr = pExpr; - p->selFlags &= ~SF_Aggregate; + p->selFlags &= ~(u32)SF_Aggregate; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x200 ){ @@ -6990,12 +7343,12 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ for(i=0; inSrc; i++){ SrcItem *p1 = &pSrc->a[i]; if( p1==p0 ) continue; - if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + if( p0->pSTab==p1->pSTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ return 1; } - if( p1->pSelect - && (p1->pSelect->selFlags & SF_NestedFrom)!=0 - && sameSrcAlias(p0, p1->pSelect->pSrc) + if( p1->fg.isSubquery + && (p1->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->u4.pSubq->pSelect->pSrc) ){ return 1; } @@ -7060,13 +7413,212 @@ static int fromClauseTermCanBeCoroutine( if( i==0 ) break; i--; pItem--; - if( pItem->pSelect!=0 ) return 0; /* (1c-i) */ + if( pItem->fg.isSubquery ) return 0; /* (1c-i) */ } return 1; } /* -** Generate code for the SELECT statement given in the p argument. +** Argument pWhere is the WHERE clause belonging to SELECT statement p. This +** function attempts to transform expressions of the form: +** +** EXISTS (SELECT ...) +** +** into joins. For example, given +** +** CREATE TABLE sailors(sid INTEGER PRIMARY KEY, name TEXT); +** CREATE TABLE reserves(sid INT, day DATE, PRIMARY KEY(sid, day)); +** +** SELECT name FROM sailors AS S WHERE EXISTS ( +** SELECT * FROM reserves AS R WHERE S.sid = R.sid AND R.day = '2022-10-25' +** ); +** +** the SELECT statement may be transformed as follows: +** +** SELECT name FROM sailors AS S, reserves AS R +** WHERE S.sid = R.sid AND R.day = '2022-10-25'; +** +** **Approximately**. Really, we have to ensure that the FROM-clause term +** that was formerly inside the EXISTS is only executed once. This is handled +** by setting the SrcItem.fg.fromExists flag, which then causes code in +** the where.c file to exit the corresponding loop after the first successful +** match (if any). +*/ +static SQLITE_NOINLINE void existsToJoin( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement being optimized */ + Expr *pWhere /* part of the WHERE clause currently being examined */ +){ + if( pParse->nErr==0 + && pWhere!=0 + && !ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) + && ALWAYS(p->pSrc!=0) + && p->pSrc->nSrcop==TK_AND ){ + Expr *pRight = pWhere->pRight; + existsToJoin(pParse, p, pWhere->pLeft); + existsToJoin(pParse, p, pRight); + } + else if( pWhere->op==TK_EXISTS ){ + Select *pSub = pWhere->x.pSelect; + Expr *pSubWhere = pSub->pWhere; + if( pSub->pSrc->nSrc==1 + && (pSub->selFlags & SF_Aggregate)==0 + && !pSub->pSrc->a[0].fg.isSubquery + && pSub->pLimit==0 + && pSub->pPrior==0 + ){ + /* Before combining the sub-select with the parent, renumber the + ** cursor used by the subselect. This is because the EXISTS expression + ** might be a copy of another EXISTS expression from somewhere + ** else in the tree, and in this case it is important that it use + ** a unique cursor number. */ + sqlite3 *db = pParse->db; + int *aCsrMap = sqlite3DbMallocZero(db, (pParse->nTab+2)*sizeof(int)); + if( aCsrMap==0 ) return; + aCsrMap[0] = (pParse->nTab+1); + renumberCursors(pParse, pSub, -1, aCsrMap); + sqlite3DbFree(db, aCsrMap); + + memset(pWhere, 0, sizeof(*pWhere)); + pWhere->op = TK_INTEGER; + pWhere->u.iValue = 1; + ExprSetProperty(pWhere, EP_IntValue); + assert( p->pWhere!=0 ); + pSub->pSrc->a[0].fg.fromExists = 1; + pSub->pSrc->a[0].fg.jointype |= JT_CROSS; + p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); + if( pSubWhere ){ + p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); + pSub->pWhere = 0; + } + pSub->pSrc = 0; + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSub); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x100000 ){ + TREETRACE(0x100000,pParse,p, + ("After EXISTS-to-JOIN optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + existsToJoin(pParse, p, pSubWhere); + } + } + } +} + +/* +** Type used for Walker callbacks by selectCheckOnClauses(). +*/ +typedef struct CheckOnCtx CheckOnCtx; +struct CheckOnCtx { + SrcList *pSrc; /* SrcList for this context */ + int iJoin; /* Cursor numbers must be =< than this */ + CheckOnCtx *pParent; /* Parent context */ +}; + +/* +** True if the SrcList passed as the only argument contains at least +** one RIGHT or FULL JOIN. False otherwise. +*/ +#define hasRightJoin(pSrc) (((pSrc)->a[0].fg.jointype & JT_LTORJ)!=0) + +/* +** The xExpr callback for the search of invalid ON clause terms. +*/ +static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ + CheckOnCtx *pCtx = pWalker->u.pCheckOnCtx; + + /* Check if pExpr is root or near-root of an ON clause constraint that needs + ** to be checked to ensure that it does not refer to tables in its FROM + ** clause to the right of itself. i.e. it is either: + ** + ** + an ON clause on an OUTER join, or + ** + an ON clause on an INNER join within a FROM that features at + ** least one RIGHT or FULL join. + */ + if( (ExprHasProperty(pExpr, EP_OuterON)) + || (ExprHasProperty(pExpr, EP_InnerON) && hasRightJoin(pCtx->pSrc)) + ){ + /* If CheckOnCtx.iJoin is already set, then fall through and process + ** this expression node as normal. Or, if CheckOnCtx.iJoin is still 0, + ** set it to the cursor number of the RHS of the join to which this + ** ON expression was attached and then iterate through the entire + ** expression. */ + assert( pCtx->iJoin==0 || pCtx->iJoin==pExpr->w.iJoin ); + if( pCtx->iJoin==0 ){ + pCtx->iJoin = pExpr->w.iJoin; + sqlite3WalkExprNN(pWalker, pExpr); + pCtx->iJoin = 0; + return WRC_Prune; + } + } + + if( pExpr->op==TK_COLUMN ){ + /* A column expression. Find the SrcList (if any) to which it refers. + ** Then, if CheckOnCtx.iJoin indicates that this expression is part of an + ** ON clause from that SrcList (i.e. if iJoin is non-zero), check that it + ** does not refer to a table to the right of CheckOnCtx.iJoin. */ + do { + SrcList *pSrc = pCtx->pSrc; + int iTab = pExpr->iTable; + if( iTab>=pSrc->a[0].iCursor && iTab<=pSrc->a[pSrc->nSrc-1].iCursor ){ + if( pCtx->iJoin && iTab>pCtx->iJoin ){ + sqlite3ErrorMsg(pWalker->pParse, + "ON clause references tables to its right"); + return WRC_Abort; + } + break; + } + pCtx = pCtx->pParent; + }while( pCtx ); + } + return WRC_Continue; +} + +/* +** The xSelect callback for the search of invalid ON clause terms. +*/ +static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ + CheckOnCtx *pCtx = pWalker->u.pCheckOnCtx; + if( pSelect->pSrc==pCtx->pSrc || pSelect->pSrc->nSrc==0 ){ + return WRC_Continue; + }else{ + CheckOnCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pSrc = pSelect->pSrc; + sCtx.pParent = pCtx; + pWalker->u.pCheckOnCtx = &sCtx; + sqlite3WalkSelect(pWalker, pSelect); + pWalker->u.pCheckOnCtx = pCtx; + pSelect->selFlags &= ~SF_OnToWhere; + return WRC_Prune; + } +} + +/* +** Check all ON clauses in pSelect to verify that they do not reference +** columns to the right. +*/ +static void selectCheckOnClauses(Parse *pParse, Select *pSelect){ + Walker w; + CheckOnCtx sCtx; + assert( pSelect->selFlags & SF_OnToWhere ); + assert( pSelect->pSrc!=0 && pSelect->pSrc->nSrc>=2 ); + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = selectCheckOnClausesExpr; + w.xSelectCallback = selectCheckOnClausesSelect; + w.u.pCheckOnCtx = &sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pSrc = pSelect->pSrc; + sqlite3WalkExprNN(&w, pSelect->pWhere); + pSelect->selFlags &= ~SF_OnToWhere; +} + +/* +** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. ** See comments in sqliteInt.h for further information. @@ -7077,6 +7629,40 @@ static int fromClauseTermCanBeCoroutine( ** ** This routine does NOT free the Select structure passed in. The ** calling function needs to do that. +** +** This is a long function. The following is an outline of the processing +** steps, with tags referencing various milestones: +** +** * Resolve names and similar preparation tag-select-0100 +** * Scan of the FROM clause tag-select-0200 +** + OUTER JOIN strength reduction tag-select-0220 +** + Sub-query ORDER BY removal tag-select-0230 +** + Query flattening tag-select-0240 +** * Separate subroutine for compound-SELECT tag-select-0300 +** * WHERE-clause constant propagation tag-select-0330 +** * Count()-of-VIEW optimization tag-select-0350 +** * Scan of the FROM clause again tag-select-0400 +** + Authorize unreferenced tables tag-select-0410 +** + Predicate push-down optimization tag-select-0420 +** + Omit unused subquery columns optimization tag-select-0440 +** + Generate code to implement subqueries tag-select-0480 +** - Co-routines tag-select-0482 +** - Reuse previously computed CTE tag-select-0484 +** - REuse previously computed VIEW tag-select-0486 +** - Materialize a VIEW or CTE tag-select-0488 +** * DISTINCT ORDER BY -> GROUP BY optimization tag-select-0500 +** * Set up for ORDER BY tag-select-0600 +** * Create output table tag-select-0630 +** * Prepare registers for LIMIT tag-select-0650 +** * Setup for DISTINCT tag-select-0680 +** * Generate code for non-aggregate and non-GROUP BY tag-select-0700 +** * Generate code for aggregate and/or GROUP BY tag-select-0800 +** + GROUP BY queries tag-select-0810 +** + non-GROUP BY queries tag-select-0820 +** - Special case of count() w/o GROUP BY tag-select-0821 +** - General case of non-GROUP BY aggregates tag-select-0822 +** * Sort results, as needed tag-select-0900 +** * Internal self-checks tag-select-1000 */ int sqlite3Select( Parse *pParse, /* The parser context */ @@ -7120,12 +7706,13 @@ int sqlite3Select( } #endif + /* tag-select-0100 */ assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableDistinct(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ @@ -7135,14 +7722,13 @@ int sqlite3Select( if( sqlite3TreeTrace & 0x800 ){ sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } -#endif - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprListDelete, - p->pOrderBy); +#endif + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + p->pOrderBy); testcase( pParse->earlyCleanup ); p->pOrderBy = 0; } - p->selFlags &= ~SF_Distinct; + p->selFlags &= ~(u32)SF_Distinct; p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); @@ -7158,21 +7744,33 @@ int sqlite3Select( } #endif + /* If the SELECT statement contains ON clauses that were moved into + ** the WHERE clause, go through and verify that none of the terms + ** in the ON clauses reference tables to the right of the ON clause. + ** Do this now, after name resolution, but before query flattening + */ + if( p->selFlags & SF_OnToWhere ){ + selectCheckOnClauses(pParse, p); + if( pParse->nErr ){ + goto select_end; + } + } + /* If the SF_UFSrcCheck flag is set, then this function is being called ** as part of populating the temp table for an UPDATE...FROM statement. - ** In this case, it is an error if the target object (pSrc->a[0]) name - ** or alias is duplicated within FROM clause (pSrc->a[1..n]). + ** In this case, it is an error if the target object (pSrc->a[0]) name + ** or alias is duplicated within FROM clause (pSrc->a[1..n]). ** - ** Postgres disallows this case too. The reason is that some other - ** systems handle this case differently, and not all the same way, + ** Postgres disallows this case too. The reason is that some other + ** systems handle this case differently, and not all the same way, ** which is just confusing. To avoid this, we follow PG's lead and ** disallow it altogether. */ if( p->selFlags & SF_UFSrcCheck ){ SrcItem *p0 = &p->pSrc->a[0]; if( sameSrcAlias(p0, p->pSrc) ){ - sqlite3ErrorMsg(pParse, - "target object/alias may not appear in FROM clause: %s", - p0->zAlias ? p0->zAlias : p0->pTab->zName + sqlite3ErrorMsg(pParse, + "target object/alias may not appear in FROM clause: %s", + p0->zAlias ? p0->zAlias : p0->pSTab->zName ); goto select_end; } @@ -7181,7 +7779,7 @@ int sqlite3Select( ** and leaving this flag set can cause errors if a compound sub-query ** in p->pSrc is flattened into this query and this function called ** again as part of compound SELECT processing. */ - p->selFlags &= ~SF_UFSrcCheck; + p->selFlags &= ~(u32)SF_UFSrcCheck; } if( pDest->eDest==SRT_Output ){ @@ -7207,34 +7805,72 @@ int sqlite3Select( /* Try to do various optimizations (flattening subqueries, and strength ** reduction of join operators) in the FROM clause up into the main query + ** tag-select-0200 */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; - Select *pSub = pItem->pSelect; - Table *pTab = pItem->pTab; + Select *pSub = pItem->fg.isSubquery ? pItem->u4.pSubq->pSelect : 0; + Table *pTab = pItem->pSTab; /* The expander should have already created transient Table objects ** even for FROM clause elements such as subqueries that do not correspond ** to a real table */ assert( pTab!=0 ); - /* Convert LEFT JOIN into JOIN if there are terms of the right table - ** of the LEFT JOIN used in the WHERE clause. + /* Try to simplify joins: + ** + ** LEFT JOIN -> JOIN + ** RIGHT JOIN -> JOIN + ** FULL JOIN -> RIGHT JOIN + ** + ** If terms of the i-th table are used in the WHERE clause in such a + ** way that the i-th table cannot be the NULL row of a join, then + ** perform the appropriate simplification. This is called + ** "OUTER JOIN strength reduction" in the SQLite documentation. + ** tag-select-0220 */ - if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==JT_LEFT - && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) + if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, + pItem->fg.jointype & JT_LTORJ) && OptimizationEnabled(db, SQLITE_SimplifyJoin) ){ - TREETRACE(0x1000,pParse,p, - ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); - pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); - assert( pItem->iCursor>=0 ); - unsetJoinExpr(p->pWhere, pItem->iCursor, - pTabList->a[0].fg.jointype & JT_LTORJ); + if( pItem->fg.jointype & JT_LEFT ){ + if( pItem->fg.jointype & JT_RIGHT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to RIGHT-JOIN on term %d\n",i)); + pItem->fg.jointype &= ~JT_LEFT; + }else{ + TREETRACE(0x1000,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor, 0); + } + } + if( pItem->fg.jointype & JT_LTORJ ){ + for(j=i+1; jnSrc; j++){ + SrcItem *pI2 = &pTabList->a[j]; + if( pI2->fg.jointype & JT_RIGHT ){ + if( pI2->fg.jointype & JT_LEFT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to LEFT-JOIN on term %d\n",j)); + pI2->fg.jointype &= ~JT_RIGHT; + }else{ + TREETRACE(0x1000,pParse,p, + ("RIGHT-JOIN simplifies to JOIN on term %d\n",j)); + pI2->fg.jointype &= ~(JT_RIGHT|JT_OUTER); + unsetJoinExpr(p->pWhere, pI2->iCursor, 1); + } + } + } + for(j=pTabList->nSrc-1; j>=0; j--){ + pTabList->a[j].fg.jointype &= ~JT_LTORJ; + if( pTabList->a[j].fg.jointype & JT_RIGHT ) break; + } + } } - /* No futher action if this term of the FROM clause is not a subquery */ + /* No further action if this term of the FROM clause is not a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -7245,6 +7881,14 @@ int sqlite3Select( goto select_end; } + /* Do not attempt the usual optimizations (flattening and ORDER BY + ** elimination) on a MATERIALIZED common table expression because + ** a MATERIALIZED common table expression is an optimization fence. + */ + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ){ + continue; + } + /* Do not try to flatten an aggregate subquery. ** ** Flattening an aggregate subquery is only possible if the outer query @@ -7255,7 +7899,8 @@ int sqlite3Select( if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; assert( pSub->pGroupBy==0 ); - /* If a FROM-clause subquery has an ORDER BY clause that is not + /* tag-select-0230: + ** If a FROM-clause subquery has an ORDER BY clause that is not ** really doing anything, then delete it now so that it does not ** interfere with query flattening. See the discussion at ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a @@ -7274,19 +7919,23 @@ int sqlite3Select( ** (a) The outer query has a different ORDER BY clause ** (b) The subquery is part of a join ** See forum post 062d576715d277c8 + ** (6) The subquery is not a recursive CTE. ORDER BY has a different + ** meaning for recursive CTEs and this optimization does not + ** apply. + ** + ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. */ if( pSub->pOrderBy!=0 && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ && pSub->pLimit==0 /* Condition (1) */ - && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */ + && (pSub->selFlags & (SF_OrderByReqd|SF_Recursive))==0 /* (2) and (6) */ && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ && OptimizationEnabled(db, SQLITE_OmitOrderBy) ){ TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprListDelete, - pSub->pOrderBy); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + pSub->pOrderBy); pSub->pOrderBy = 0; } @@ -7317,6 +7966,7 @@ int sqlite3Select( continue; } + /* tag-select-0240 */ if( flattenSubquery(pParse, p, i, isAgg) ){ if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ @@ -7332,7 +7982,7 @@ int sqlite3Select( #ifndef SQLITE_OMIT_COMPOUND_SELECT /* Handle compound SELECT statements using the separate multiSelect() - ** procedure. + ** procedure. tag-select-0300 */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); @@ -7347,10 +7997,17 @@ int sqlite3Select( } #endif + /* If there may be an "EXISTS (SELECT ...)" in the WHERE clause, attempt + ** to change it into a join. */ + if( pParse->bHasExists && OptimizationEnabled(db,SQLITE_ExistsToJoin) ){ + existsToJoin(pParse, p, p->pWhere); + pTabList = p->pSrc; + } + /* Do the WHERE-clause constant propagation optimization if this is - ** a join. No need to speed time on this operation for non-join queries + ** a join. No need to spend time on this operation for non-join queries ** as the equivalent optimization will be handled by query planner in - ** sqlite3WhereBegin(). + ** sqlite3WhereBegin(). tag-select-0330 */ if( p->pWhere!=0 && p->pWhere->op==TK_AND @@ -7367,6 +8024,7 @@ int sqlite3Select( TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } + /* tag-select-0350 */ if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ @@ -7374,20 +8032,26 @@ int sqlite3Select( pTabList = p->pSrc; } - /* For each term in the FROM clause, do two things: - ** (1) Authorized unreferenced tables - ** (2) Generate code for all sub-queries + /* Loop over all terms in the FROM clause and do two things for each term: + ** + ** (1) Authorize unreferenced tables + ** (2) Generate code for all sub-queries + ** + ** tag-select-0400 */ for(i=0; inSrc; i++){ SrcItem *pItem = &pTabList->a[i]; SrcItem *pPrior; SelectDest dest; + Subquery *pSubq; Select *pSub; #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) const char *zSavedAuthContext; #endif - /* Issue SQLITE_READ authorizations with a fake column name for any + /* Authorized unreferenced tables. tag-select-0410 + ** + ** Issue SQLITE_READ authorizations with a fake column name for any ** tables that are referenced but from which no values are extracted. ** Examples of where these kinds of null SQLITE_READ authorizations ** would occur: @@ -7404,17 +8068,28 @@ int sqlite3Select( ** string for the fake column name seems safer. */ if( pItem->colUsed==0 && pItem->zName!=0 ){ - sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); + const char *zDb; + if( pItem->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pItem->u4.pSchema); + zDb = db->aDb[iDb].zDbSName; + }else if( pItem->fg.isSubquery ){ + zDb = 0; + }else{ + zDb = pItem->u4.zDatabase; + } + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", zDb); } #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Generate code for all sub-queries in the FROM clause */ - pSub = pItem->pSelect; - if( pSub==0 ) continue; + if( pItem->fg.isSubquery==0 ) continue; + pSubq = pItem->u4.pSubq; + assert( pSubq!=0 ); + pSub = pSubq->pSelect; /* The code for a subquery should only be generated once. */ - assert( pItem->addrFillSub==0 ); + if( pSubq->addrFillSub!=0 ) continue; /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -7427,9 +8102,10 @@ int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. + ** This is the "predicate push-down optimization". tag-select-0420 */ if( OptimizationEnabled(db, SQLITE_PushDown) - && (pItem->fg.isCte==0 + && (pItem->fg.isCte==0 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) && pushDownWhereTerms(pParse, pSub, p->pWhere, pTabList, i) ){ @@ -7440,13 +8116,14 @@ int sqlite3Select( sqlite3TreeViewSelect(0, p, 0); } #endif - assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); + assert( pSubq->pSelect && (pSub->selFlags & SF_PushDown)!=0 ); }else{ - TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); + TREETRACE(0x4000,pParse,p,("WHERE-clause push-down not possible\n")); } /* Convert unused result columns of the subquery into simple NULL ** expressions, to avoid unneeded searching and computation. + ** tag-select-0440 */ if( OptimizationEnabled(db, SQLITE_NullUnusedCols) && disableUnusedSubqueryResultColumns(pItem) @@ -7464,32 +8141,33 @@ int sqlite3Select( zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; - /* Generate code to implement the subquery + /* Generate byte-code to implement the subquery tag-select-0480 */ if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result - ** set on each invocation. + ** set on each invocation. tag-select-0482 */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; - - pItem->regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pSubq->regReturn, 0, addrTop); VdbeComment((v, "%!S", pItem)); - pItem->addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + pSubq->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; - pItem->regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->regReturn); + pSubq->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pSubq->regReturn); + VdbeComment((v, "end %!S", pItem)); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ /* This is a CTE for which materialization code has already been ** generated. Invoke the subroutine to compute the materialization, - ** the make the pItem->iCursor be a copy of the ephemerial table that - ** holds the result of the materialization. */ + ** then make the pItem->iCursor be a copy of the ephemeral table that + ** holds the result of the materialization. tag-select-0484 */ CteUse *pCteUse = pItem->u2.pCteUse; sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); if( pItem->iCursor!=pCteUse->iCur ){ @@ -7499,25 +8177,30 @@ int sqlite3Select( pSub->nSelectRow = pCteUse->nRowEst; }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ /* This view has already been materialized by a prior entry in - ** this same FROM clause. Reuse it. */ - if( pPrior->addrFillSub ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + ** this same FROM clause. Reuse it. tag-select-0486 */ + Subquery *pPriorSubq; + assert( pPrior->fg.isSubquery ); + pPriorSubq = pPrior->u4.pSubq; + assert( pPriorSubq!=0 ); + if( pPriorSubq->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPriorSubq->regReturn, + pPriorSubq->addrFillSub); } sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); - pSub->nSelectRow = pPrior->pSelect->nSelectRow; + pSub->nSelectRow = pPriorSubq->pSelect->nSelectRow; }else{ /* Materialize the view. If the view is not correlated, generate a ** subroutine to do the materialization so that subsequent uses of - ** the same view can reuse the materialization. */ + ** the same view can reuse the materialization. tag-select-0488 */ int topAddr; int onceAddr = 0; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int addrExplain; #endif - pItem->regReturn = ++pParse->nMem; + pSubq->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp0(v, OP_Goto); - pItem->addrFillSub = topAddr+1; + pSubq->addrFillSub = topAddr+1; pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of @@ -7532,17 +8215,17 @@ int sqlite3Select( ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - sqlite3VdbeAddOp2(v, OP_Return, pItem->regReturn, topAddr+1); + sqlite3VdbeAddOp2(v, OP_Return, pSubq->regReturn, topAddr+1); VdbeComment((v, "end %!S", pItem)); sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ CteUse *pCteUse = pItem->u2.pCteUse; - pCteUse->addrM9e = pItem->addrFillSub; - pCteUse->regRtn = pItem->regReturn; + pCteUse->addrM9e = pSubq->addrFillSub; + pCteUse->regRtn = pSubq->regReturn; pCteUse->iCur = pItem->iCursor; pCteUse->nRowEst = pSub->nSelectRow; } @@ -7568,7 +8251,9 @@ int sqlite3Select( } #endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + /* tag-select-0500 + ** + ** If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: ** @@ -7578,19 +8263,25 @@ int sqlite3Select( ** ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** - ** The second form is preferred as a single index (or temp-table) may be - ** used for both the ORDER BY and DISTINCT processing. As originally - ** written the query must use a temp-table for at least one of the ORDER + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER ** BY and DISTINCT, and an index or separate temp-table for the other. */ - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 #endif ){ - p->selFlags &= ~SF_Distinct; + p->selFlags &= ~(u32)SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + if( pGroupBy ){ + for(i=0; inExpr; i++){ + pGroupBy->a[i].u.x.iOrderByCol = i+1; + } + } p->selFlags |= SF_Aggregate; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the @@ -7612,7 +8303,7 @@ int sqlite3Select( ** If that is the case, then the OP_OpenEphemeral instruction will be ** changed to an OP_Noop once we figure out that the sorting index is ** not needed. The sSort.addrSortIndex variable is used to facilitate - ** that change. + ** that change. tag-select-0600 */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; @@ -7629,6 +8320,7 @@ int sqlite3Select( } /* If the output is destined for a temporary table, open that table. + ** tag-select-0630 */ if( pDest->eDest==SRT_EphemTab ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); @@ -7646,7 +8338,7 @@ int sqlite3Select( } } - /* Set the limiter. + /* Set the limiter. tag-select-0650 */ iEnd = sqlite3VdbeMakeLabel(pParse); if( (p->selFlags & SF_FixedLimit)==0 ){ @@ -7658,7 +8350,7 @@ int sqlite3Select( sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open an ephemeral index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. tag-select-0680 */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; @@ -7673,7 +8365,7 @@ int sqlite3Select( } if( !isAgg && pGroupBy==0 ){ - /* No aggregate functions and no GROUP BY clause */ + /* No aggregate functions and no GROUP BY clause. tag-select-0700 */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) | (p->selFlags & SF_FixedLimit); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -7692,6 +8384,12 @@ int sqlite3Select( if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + if( pDest->eDest<=SRT_DistQueue && pDest->eDest>=SRT_DistFifo ){ + /* TUNING: For a UNION CTE, because UNION is implies DISTINCT, + ** reduce the estimated output row count by 8 (LogEst 30). + ** Search for tag-20250414a to see other cases */ + p->nSelectRow -= 30; + } } if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); @@ -7705,7 +8403,7 @@ int sqlite3Select( } TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); - /* If sorting index that was created by a prior OP_OpenEphemeral + /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ @@ -7746,8 +8444,8 @@ int sqlite3Select( sqlite3WhereEnd(pWInfo); } }else{ - /* This case when there exist aggregate functions or a GROUP BY clause - ** or both */ + /* This case is for when there exist aggregate functions or a GROUP BY + ** clause or both. tag-select-0800 */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -7778,8 +8476,8 @@ int sqlite3Select( if( p->nSelectRow>66 ) p->nSelectRow = 66; /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then it may be possible to disable the ORDER BY clause - ** on the grounds that the GROUP BY will cause elements to come out + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out ** in the correct order. It also may not - the GROUP BY might use a ** database index that causes rows to be grouped together as required ** but not actually sorted. Either way, record the fact that the @@ -7789,8 +8487,8 @@ int sqlite3Select( int ii; /* The GROUP BY processing doesn't care whether rows are delivered in ** ASC or DESC order - only that each group is returned contiguously. - ** So set the ASC/DESC flags in the GROUP BY to match those in the - ** ORDER BY to maximize the chances of rows being delivered in an + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an ** order that makes the ORDER BY redundant. */ for(ii=0; iinExpr; ii++){ u8 sortFlags; @@ -7815,8 +8513,7 @@ int sqlite3Select( */ pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); if( pAggInfo ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))agginfoFree, pAggInfo); + sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo); testcase( pParse->earlyCleanup ); } if( db->mallocFailed ){ @@ -7867,11 +8564,11 @@ int sqlite3Select( /* Processing for aggregates with GROUP BY is very different and - ** much more complex than aggregates without a GROUP BY. + ** much more complex than aggregates without a GROUP BY. tag-select-0810 */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int addr1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparison jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -7883,7 +8580,7 @@ int sqlite3Select( u16 distFlag = 0; int eDist = WHERE_DISTINCT_NOOP; - if( pAggInfo->nFunc==1 + if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 && ALWAYS(pAggInfo->aFunc[0].pFExpr!=0) && ALWAYS(ExprUseXList(pAggInfo->aFunc[0].pFExpr)) @@ -7899,13 +8596,13 @@ int sqlite3Select( /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out ** that we do not need it after all, the OP_SorterOpen instruction - ** will be converted into a Noop. + ** will be converted into a Noop. */ pAggInfo->sortingIdx = pParse->nTab++; pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy, 0, pAggInfo->nColumn); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, - pAggInfo->sortingIdx, pAggInfo->nSortingColumn, + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, + pAggInfo->sortingIdx, pAggInfo->nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); /* Initialize memory locations used by GROUP BY aggregate processing @@ -7923,6 +8620,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + sqlite3ExprNullRegisterRange(pParse, iAMem, pGroupBy->nExpr); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or @@ -7932,7 +8630,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, - p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); if( pWInfo==0 ){ @@ -7962,9 +8660,13 @@ int sqlite3Select( int nCol; int nGroupBy; - explainTempTable(pParse, +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp; /* Address of OP_Explain instruction */ +#endif + ExplainQueryPlan2(addrExp, (pParse, 0, "USE TEMP B-TREE FOR %s", (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? - "DISTINCT" : "GROUP BY"); + "DISTINCT" : "GROUP BY" + )); groupBySort = 1; nGroupBy = pGroupBy->nExpr; @@ -7989,18 +8691,23 @@ int sqlite3Select( } pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, 0, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); + sqlite3VdbeScanStatusRange(v, addrExp, sqlite3VdbeCurrentAddr(v)-2, -1); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, sqlite3VdbeCurrentAddr(v), 0); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); pAggInfo->useSortingIdx = 1; + sqlite3VdbeScanStatusRange(v, addrExp, -1, sortPTab); + sqlite3VdbeScanStatusRange(v, addrExp, -1, pAggInfo->sortingIdx); } /* If there are entries in pAgggInfo->aFunc[] that contain subexpressions @@ -8025,9 +8732,9 @@ int sqlite3Select( ** clause, cancel the ephemeral table open coded earlier. ** ** This is an optimization - the correct answer should result regardless. - ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to ** disable this optimization for testing purposes. */ - if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) && (groupBySort || sqlite3WhereIsSorted(pWInfo)) ){ sSort.pOrderBy = 0; @@ -8045,12 +8752,29 @@ int sqlite3Select( sortOut, sortPTab); } for(j=0; jnExpr; j++){ + int iOrderByCol = pGroupBy->a[j].u.x.iOrderByCol; + if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); }else{ pAggInfo->directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } + + if( iOrderByCol ){ + Expr *pX = p->pEList->a[iOrderByCol-1].pExpr; + Expr *pBase = sqlite3ExprSkipCollateAndLikely(pX); + while( ALWAYS(pBase!=0) && pBase->op==TK_IF_NULL_ROW ){ + pX = pBase->pLeft; + pBase = sqlite3ExprSkipCollateAndLikely(pX); + } + if( ALWAYS(pBase!=0) + && pBase->op!=TK_AGG_COLUMN + && pBase->op!=TK_REGISTER + ){ + sqlite3ExprToRegister(pX, iAMem+j); + } + } } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); @@ -8066,13 +8790,13 @@ int sqlite3Select( ** and resets the aggregate accumulator registers in preparation ** for the next GROUP BY batch. */ - sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output one row")); + VdbeComment((v, "output one row of %d", p->selId)); + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - VdbeComment((v, "reset accumulator")); + VdbeComment((v, "reset accumulator %d", p->selId)); /* Update the aggregate accumulators based on the content of ** the current row @@ -8080,7 +8804,7 @@ int sqlite3Select( sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); - VdbeComment((v, "indicate data in accumulator")); + VdbeComment((v, "indicate data in accumulator %d", p->selId)); /* End of the loop */ @@ -8097,7 +8821,7 @@ int sqlite3Select( /* Output the final row of result */ sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); - VdbeComment((v, "output final row")); + VdbeComment((v, "output final row of %d", p->selId)); /* Jump over the subroutines */ @@ -8118,7 +8842,7 @@ int sqlite3Select( addrOutputRow = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeCoverage(v); - VdbeComment((v, "Groupby result generator entry point")); + VdbeComment((v, "Groupby result generator entry point %d", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, pAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); @@ -8126,14 +8850,14 @@ int sqlite3Select( &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - VdbeComment((v, "end groupby result generator")); + VdbeComment((v, "end groupby result generator %d", p->selId)); /* Generate a subroutine that will reset the group-by accumulator */ sqlite3VdbeResolveLabel(v, addrReset); resetAccumulator(pParse, pAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); + VdbeComment((v, "indicate accumulator %d empty", p->selId)); sqlite3VdbeAddOp1(v, OP_Return, regReset); if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ @@ -8142,9 +8866,12 @@ int sqlite3Select( } } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { + /* Aggregate functions without GROUP BY. tag-select-0820 */ Table *pTab; if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ - /* If isSimpleCount() returns a pointer to a Table structure, then + /* tag-select-0821 + ** + ** If isSimpleCount() returns a pointer to a Table structure, then ** the SQL statement is of the form: ** ** SELECT count(*) FROM @@ -8173,7 +8900,7 @@ int sqlite3Select( ** ** (2013-10-03) Do not count the entries in a partial index. ** - ** In practice the KeyInfo structure will not be used. It is only + ** In practice the KeyInfo structure will not be used. It is only ** passed to keep OP_OpenRead happy. */ if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); @@ -8203,6 +8930,8 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); }else{ + /* The general case of an aggregate query without GROUP BY + ** tag-select-0822 */ int regAcc = 0; /* "populate accumulators" flag */ ExprList *pDistinct = 0; u16 distFlag = 0; @@ -8215,7 +8944,7 @@ int sqlite3Select( ** that the accumulator registers are (a) updated only once if ** there are no min() or max functions or (b) always updated for the ** first row visited by the aggregate, so that they are updated at - ** least once even if the FILTER clause means the min() or max() + ** least once even if the FILTER clause means the min() or max() ** function visits zero rows. */ if( pAggInfo->nAccumulator ){ for(i=0; inFunc; i++){ @@ -8279,11 +9008,11 @@ int sqlite3Select( sSort.pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, -1, 0, 0, + selectInnerLoop(pParse, p, -1, 0, 0, pDest, addrEnd, addrEnd); } sqlite3VdbeResolveLabel(v, addrEnd); - + } /* endif aggregate query */ if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ @@ -8291,7 +9020,7 @@ int sqlite3Select( } /* If there is an ORDER BY clause, then we need to sort the results - ** and send them to the callback one by one. + ** and send them to the callback one by one. tag-select-0900 */ if( sSort.pOrderBy ){ assert( p->pEList==pEList ); @@ -8314,7 +9043,14 @@ int sqlite3Select( assert( db->mallocFailed==0 || pParse->nErr!=0 ); sqlite3ExprListDelete(db, pMinMaxOrderBy); #ifdef SQLITE_DEBUG + /* Internal self-checks. tag-select-1000 */ if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } +#endif for(i=0; inColumn; i++){ Expr *pExpr = pAggInfo->aCol[i].pCExpr; if( pExpr==0 ) continue; diff --git a/src/shell.c.in b/src/shell.c.in index fe80749ae4..042190c175 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -104,11 +104,11 @@ typedef unsigned short int u16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; typedef unsigned char u8; -#if SQLITE_USER_AUTHENTICATION -# include "sqlite3userauth.h" -#endif #include #include +#ifndef _WIN32 +# include +#endif #if !defined(_WIN32) && !defined(WIN32) # include @@ -192,8 +192,6 @@ typedef unsigned char u8; # ifndef strdup # define strdup _strdup # endif -# undef popen -# define popen _popen # undef pclose # define pclose _pclose # endif @@ -223,41 +221,40 @@ typedef unsigned char u8; #define IsSpace(X) isspace((unsigned char)X) #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) +#define IsAlnum(X) isalnum((unsigned char)X) +#define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) #if SQLITE_OS_WINRT #include #endif +#undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include /* string conversion routines only needed on Win32 */ extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); -extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); -extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); #endif -/* On Windows, we normally run with output mode of TEXT so that \n characters -** are automatically translated into \r\n. However, this behavior needs -** to be disabled in some cases (ex: when generating CSV output and when -** rendering quoted strings that contain \n characters). The following -** routines take care of that. -*/ -#if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT -static void setBinaryMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_BINARY); -} -static void setTextMode(FILE *file, int isOutput){ - if( isOutput ) fflush(file); - _setmode(_fileno(file), _O_TEXT); -} -#else -# define setBinaryMode(X,Y) -# define setTextMode(X,Y) +INCLUDE ../ext/misc/sqlite3_stdio.h +INCLUDE ../ext/misc/sqlite3_stdio.c + +/* Use console I/O package as a direct INCLUDE. */ +#define SQLITE_INTERNAL_LINKAGE static + +#ifdef SQLITE_SHELL_FIDDLE +/* Deselect most features from the console I/O package for Fiddle. */ +# define SQLITE_CIO_NO_REDIRECT +# define SQLITE_CIO_NO_CLASSIFY +# define SQLITE_CIO_NO_TRANSLATE +# define SQLITE_CIO_NO_SETMODE +# define SQLITE_CIO_NO_FLUSH #endif +#define eputz(z) sqlite3_fputs(z,stderr) +#define sputz(fp,z) sqlite3_fputs(z,fp) + /* True if the timer is enabled */ static int enableTimer = 0; @@ -273,8 +270,19 @@ static int cli_strncmp(const char *a, const char *b, size_t n){ return strncmp(a,b,n); } -/* Return the current wall-clock time */ +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) static sqlite3_vfs *clockVfs = 0; sqlite3_int64 t; if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); @@ -286,7 +294,12 @@ static sqlite3_int64 timeOfDay(void){ clockVfs->xCurrentTime(clockVfs, &r); t = (sqlite3_int64)(r*86400000.0); } - return t; + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif } #if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) @@ -302,6 +315,7 @@ struct rusage { #define getrusage(A,B) memset(B,0,sizeof(*B)) #endif + /* Saved resource information for the beginning of an operation */ static struct rusage sBegin; /* CPU time at start */ static sqlite3_int64 iBegin; /* Wall-clock time at start */ @@ -325,20 +339,20 @@ static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ /* ** Print the timing results. */ -static void endTimer(void){ +static void endTimer(FILE *out){ if( enableTimer ){ sqlite3_int64 iEnd = timeOfDay(); struct rusage sEnd; getrusage(RUSAGE_SELF, &sEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (iEnd - iBegin)*0.001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + sqlite3_fprintf(out, "Run Time: real %.6f user %.6f sys %.6f\n", + (iEnd - iBegin)*0.000001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); } } #define BEGIN_TIMER beginTimer() -#define END_TIMER endTimer() +#define END_TIMER(X) endTimer(X) #define HAS_TIMER 1 #elif (defined(_WIN32) || defined(WIN32)) @@ -404,25 +418,34 @@ static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ /* ** Print the timing results. */ -static void endTimer(void){ +static void endTimer(FILE *out){ if( enableTimer && getProcessTimesAddr){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - printf("Run Time: real %.3f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.000001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.3f user %.3f sys %.3f\n", + (ftWallEnd - ftWallBegin)*0.000001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif } } #define BEGIN_TIMER beginTimer() -#define END_TIMER endTimer() +#define END_TIMER(X) endTimer(X) #define HAS_TIMER hasTimer() #else #define BEGIN_TIMER -#define END_TIMER +#define END_TIMER(X) /*no-op*/ #define HAS_TIMER 0 #endif @@ -448,24 +471,10 @@ static int bail_on_error = 0; */ static int stdin_is_interactive = 1; -#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \ - && !defined(SHELL_OMIT_WIN_UTF8) -# define SHELL_WIN_UTF8_OPT 1 -#else -# define SHELL_WIN_UTF8_OPT 0 -#endif - -#if SHELL_WIN_UTF8_OPT -/* -** Setup console for UTF-8 input/output when following variable true. -*/ -static int console_utf8 = 0; -#endif - /* -** On Windows systems we have to know if standard output is a console -** in order to translate UTF-8 into MBCS. The following variable is -** true if translation is required. +** On Windows systems we need to know if standard output is a console +** in order to show that UTF-16 translation is done in the sign-on +** banner. The following variable is true if it is the console. */ static int stdout_is_console = 1; @@ -491,7 +500,7 @@ static char *Argv0; ** Prompt strings. Initialized in main. Settable with ** .prompt main continue */ -#define PROMPT_LEN_MAX 20 +#define PROMPT_LEN_MAX 128 /* First line prompt. default: "sqlite> " */ static char mainPrompt[PROMPT_LEN_MAX]; /* Continuation prompt. default: " ...> " */ @@ -509,6 +518,14 @@ static char *shell_strncpy(char *dest, const char *src, size_t n){ return dest; } +/* +** strcpy() workalike to squelch an unwarranted link-time warning +** from OpenBSD. +*/ +static void shell_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} + /* ** Optionally disable dynamic continuation prompt. ** Unless disabled, the continuation prompt shows open SQL lexemes if any, @@ -574,7 +591,7 @@ static char *dynamicContinuePrompt(void){ size_t ncp = strlen(continuePrompt); size_t ndp = strlen(dynPrompt.zScannerAwaits); if( ndp > ncp-3 ) return continuePrompt; - strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); + shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits); while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' '; shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); @@ -587,182 +604,17 @@ static char *dynamicContinuePrompt(void){ shell_strncpy(dynPrompt.dynamicPrompt, "(x.", 4); dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel); } - shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4); + shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, + PROMPT_LEN_MAX-4); } } return dynPrompt.dynamicPrompt; } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ -#if SHELL_WIN_UTF8_OPT -/* Following struct is used for -utf8 operation. */ -static struct ConsoleState { - int stdinEof; /* EOF has been seen on console input */ - int infsMode; /* Input file stream mode upon shell start */ - UINT inCodePage; /* Input code page upon shell start */ - UINT outCodePage; /* Output code page upon shell start */ - HANDLE hConsoleIn; /* Console input handle */ - DWORD consoleMode; /* Console mode upon shell start */ -} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; - -#ifndef _O_U16TEXT /* For build environments lacking this constant: */ -# define _O_U16TEXT 0x20000 -#endif - -/* -** Prepare console, (if known to be a WIN32 console), for UTF-8 -** input (from either typing or suitable paste operations) and for -** UTF-8 rendering. This may "fail" with a message to stderr, where -** the preparation is not done and common "code page" issues occur. -*/ -static void console_prepare(void){ - HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE); - DWORD consoleMode = 0; - if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR - && GetConsoleMode( hCI, &consoleMode) ){ - if( !IsValidCodePage(CP_UTF8) ){ - fprintf(stderr, "Cannot use UTF-8 code page.\n"); - console_utf8 = 0; - return; - } - conState.hConsoleIn = hCI; - conState.consoleMode = consoleMode; - conState.inCodePage = GetConsoleCP(); - conState.outCodePage = GetConsoleOutputCP(); - SetConsoleCP(CP_UTF8); - SetConsoleOutputCP(CP_UTF8); - consoleMode |= ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; - SetConsoleMode(conState.hConsoleIn, consoleMode); - conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); - console_utf8 = 1; - }else{ - console_utf8 = 0; - } -} - -/* -** Undo the effects of console_prepare(), if any. -*/ -static void SQLITE_CDECL console_restore(void){ - if( console_utf8 && conState.inCodePage!=0 - && conState.hConsoleIn!=INVALID_HANDLE_VALUE ){ - _setmode(_fileno(stdin), conState.infsMode); - SetConsoleCP(conState.inCodePage); - SetConsoleOutputCP(conState.outCodePage); - SetConsoleMode(conState.hConsoleIn, conState.consoleMode); - /* Avoid multiple calls. */ - conState.hConsoleIn = INVALID_HANDLE_VALUE; - conState.consoleMode = 0; - console_utf8 = 0; - } -} - -/* -** Collect input like fgets(...) with special provisions for input -** from the Windows console to get around its strange coding issues. -** Defers to plain fgets() when input is not interactive or when the -** startup option, -utf8, has not been provided or taken effect. -*/ -static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ - if( fin==0 ) fin = stdin; - if( fin==stdin && stdin_is_interactive && console_utf8 ){ -# define SQLITE_IALIM 150 - wchar_t wbuf[SQLITE_IALIM]; - int lend = 0; - int noc = 0; - if( ncmax==0 || conState.stdinEof ) return 0; - buf[0] = 0; - while( noc SQLITE_IALIM*4+1 + noc) - ? SQLITE_IALIM : (ncmax-1 - noc)/4; -# undef SQLITE_IALIM - DWORD nbr = 0; - BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); - if( !bRC || (noc==0 && nbr==0) ) return 0; - if( nbr > 0 ){ - int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, - wbuf,nbr,0,0,0,0); - if( nmb !=0 && noc+nmb <= ncmax ){ - int iseg = noc; - nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, - wbuf,nbr,buf+noc,nmb,0,0); - noc += nmb; - /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/ - if( noc > 0 ){ - if( buf[noc-1]=='\n' ){ - lend = 1; - if( noc > 1 && buf[noc-2]=='\r' ){ - buf[noc-2] = '\n'; - --noc; - } - } - } - /* Check for ^Z (anywhere in line) too. */ - while( iseg < noc ){ - if( buf[iseg]==0x1a ){ - conState.stdinEof = 1; - noc = iseg; /* Chop ^Z and anything following. */ - break; - } - ++iseg; - } - }else break; /* Drop apparent garbage in. (Could assert.) */ - }else break; - } - /* If got nothing, (after ^Z chop), must be at end-of-file. */ - if( noc == 0 ) return 0; - buf[noc] = 0; - return buf; - }else{ - return fgets(buf, ncmax, fin); - } -} - -# define fgets(b,n,f) utf8_fgets(b,n,f) -#endif /* SHELL_WIN_UTF8_OPT */ - -/* -** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, and if the -utf8 -** option is unavailable or (available and inactive), translate the -** output from UTF-8 into MBCS for output through 8-bit stdout stream. -** (With -utf8 active, no translation is needed and must not be done.) -*/ -#if defined(_WIN32) || defined(WIN32) -void utf8_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) -# if SHELL_WIN_UTF8_OPT - && !console_utf8 -# endif - ){ - char *z1 = sqlite3_vmprintf(zFormat, ap); - char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); - sqlite3_free(z1); - fputs(z2, out); - sqlite3_free(z2); - }else{ - vfprintf(out, zFormat, ap); - } - va_end(ap); -} -#elif !defined(utf8_printf) -# define utf8_printf fprintf -#endif - -/* -** Render output like fprintf(). This should not be used on anything that -** includes string formatting (e.g. "%s"). -*/ -#if !defined(raw_printf) -# define raw_printf fprintf -#endif - /* Indicate out-of-memory and exit. */ static void shell_out_of_memory(void){ - raw_printf(stderr,"Error: out of memory\n"); + eputz("Error: out of memory\n"); exit(1); } @@ -794,37 +646,242 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - utf8_printf(iotrace, "%s", z); + sqlite3_fprintf(iotrace, "%s", z); sqlite3_free(z); } #endif +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aUWidth[] = { + /* {1, 0x00000}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int cli_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<=0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; + while( iFirst c ){ + iLast = iMid - 1; + }else{ + return aUWidth[iMid].w; + } + } + if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; + return aUWidth[iLast].w; +} + +/* +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. +*/ +static int decodeUtf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; + } + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; + } + if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 + && (z[3] & 0xc0)==0x80 + ){ + *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 + | (z[3] & 0x3f); + return 4; + } + *pU = 0; + return 1; +} + + +#if 0 /* NOT USED */ +/* +** Return the width, in display columns, of a UTF-8 string. +** +** Each normal character counts as 1. Zero-width characters count +** as zero, and double-width characters count as 2. +*/ +int cli_wcswidth(const char *z){ + const unsigned char *a = (const unsigned char*)z; + int n = 0; + int i = 0; + unsigned char c; + while( (c = a[i])!=0 ){ + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&a[i], &u); + i += len; + n += cli_wcwidth(u); + }else if( c>=' ' ){ + n++; + i++; + }else{ + i++; + } + } + return n; +} +#endif + +/* +** Check to see if z[] is a valid VT100 escape. If it is, then +** return the number of bytes in the escape sequence. Return 0 if +** z[] is not a VT100 escape. +** +** This routine assumes that z[0] is \033 (ESC). +*/ +static int isVt100(const unsigned char *z){ + int i; + if( z[1]!='[' ) return 0; + i = 2; + while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} + /* -** Output string zUtf to stream pOut as w characters. If w is negative, +** Output string zUtf to stdout as w characters. If w is negative, ** then right-justify the text. W is the width in UTF-8 characters, not ** in bytes. This is different from the %*.*s specification in printf ** since with %*.*s the width is measured in bytes, not characters. +** +** Take into account zero-width and double-width Unicode characters. +** In other words, a zero-width character does not count toward the +** the w limit. A double-width character counts as two. +** +** w should normally be a small number. A couple hundred at most. This +** routine caps w at 100 million to avoid integer overflow issues. */ -static void utf8_width_print(FILE *pOut, int w, const char *zUtf){ - int i; - int n; - int aw = w<0 ? -w : w; +static void utf8_width_print(FILE *out, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; + unsigned char c; + int i = 0; + int n = 0; + int k; + int aw; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; if( zUtf==0 ) zUtf = ""; - for(i=n=0; zUtf[i]; i++){ - if( (zUtf[i]&0xc0)!=0x80 ){ - n++; - if( n==aw ){ - do{ i++; }while( (zUtf[i]&0xc0)==0x80 ); + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = decodeUtf8(a+i, &u); + int x = cli_wcwidth(u); + if( x+n>aw ){ break; } + i += len; + n += x; + }else if( c==0x1b && (k = isVt100(&a[i]))>0 ){ + i += k; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; } } if( n>=aw ){ - utf8_printf(pOut, "%.*s", i, zUtf); + sqlite3_fprintf(out, "%.*s", i, zUtf); }else if( w<0 ){ - utf8_printf(pOut, "%*s%s", aw-n, "", zUtf); + sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); }else{ - utf8_printf(pOut, "%s%*s", zUtf, aw-n, ""); + sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); } } @@ -868,12 +925,21 @@ static int strlen30(const char *z){ /* ** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character. +** count as a single character for single-width characters, or as two +** characters for double-width characters. */ static int strlenChar(const char *z){ int n = 0; while( *z ){ - if( (0xc0&*(z++))!=0x80 ) n++; + if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = decodeUtf8((const u8*)z, &u); + z += len; + n += cli_wcwidth(u); + } } return n; } @@ -884,15 +950,15 @@ static int strlenChar(const char *z){ ** Otherwise return 0. */ static FILE * openChrSource(const char *zFile){ -#ifdef _WIN32 - struct _stat x = {0}; +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; # define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) /* On Windows, open first, then check the stream nature. This order ** is necessary because _stat() and sibs, when checking a named pipe, ** effectively break the pipe as its supplier sees it. */ - FILE *rv = fopen(zFile, "rb"); + FILE *rv = sqlite3_fopen(zFile, "rb"); if( rv==0 ) return 0; - if( _fstat(_fileno(rv), &x) != 0 + if( _fstat64(_fileno(rv), &x) != 0 || !STAT_CHR_SRC(x.st_mode)){ fclose(rv); rv = 0; @@ -904,7 +970,7 @@ static FILE * openChrSource(const char *zFile){ # define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) if( rc!=0 ) return 0; if( STAT_CHR_SRC(x.st_mode) ){ - return fopen(zFile, "rb"); + return sqlite3_fopen(zFile, "rb"); }else{ return 0; } @@ -916,7 +982,7 @@ static FILE * openChrSource(const char *zFile){ ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer ** to the text. NULL is returned at end of file, or if malloc() -** fails. +** fails, or if the length of the line is longer than about a gigabyte. ** ** If zLine is not NULL then it is a malloced buffer returned from ** a previous call to this routine that may be reused. @@ -927,11 +993,15 @@ static char *local_getline(char *zLine, FILE *in){ while( 1 ){ if( n+100>nLine ){ + if( nLine>=1073741773 ){ + free(zLine); + return 0; + } nLine = nLine*2 + 100; zLine = realloc(zLine, nLine); shell_check_oom(zLine); } - if( fgets(&zLine[n], nLine - n, in)==0 ){ + if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ if( n==0 ){ free(zLine); return 0; @@ -947,27 +1017,6 @@ static char *local_getline(char *zLine, FILE *in){ break; } } -#if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, without -utf8, - ** translate the multi-byte characterset characters into UTF-8. - ** This is the translation that predates the -utf8 option. */ - if( stdin_is_interactive && in==stdin -# if SHELL_WIN_UTF8_OPT - && !console_utf8 -# endif /* SHELL_WIN_UTF8_OPT */ - ){ - char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); - if( zTrans ){ - i64 nTrans = strlen(zTrans)+1; - if( nTrans>nLine ){ - zLine = realloc(zLine, nTrans); - shell_check_oom(zLine); - } - memcpy(zLine, zTrans, nTrans); - sqlite3_free(zTrans); - } - } -#endif /* defined(_WIN32) || defined(WIN32) */ return zLine; } @@ -994,7 +1043,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ }else{ zPrompt = isContinuation ? CONTINUATION_PROMPT : mainPrompt; #if SHELL_USE_LOCAL_GETLINE - printf("%s", zPrompt); + sputz(stdout, zPrompt); fflush(stdout); do{ zResult = local_getline(zPrior, stdin); @@ -1044,10 +1093,14 @@ static int hexDigitValue(char c){ /* ** Interpret zArg as an integer value, possibly with suffixes. +** +** If the value specified by zArg is outside the range of values that +** can be represented using a 64-bit twos-complement integer, then return +** the nearest representable value. */ static sqlite3_int64 integerValue(const char *zArg){ - sqlite3_int64 v = 0; - static const struct { char *zSuffix; int iMult; } aMult[] = { + sqlite3_uint64 v = 0; + static const struct { char *zSuffix; unsigned int iMult; } aMult[] = { { "KiB", 1024 }, { "MiB", 1024*1024 }, { "GiB", 1024*1024*1024 }, @@ -1070,22 +1123,30 @@ static sqlite3_int64 integerValue(const char *zArg){ int x; zArg += 2; while( (x = hexDigitValue(zArg[0]))>=0 ){ + if( v > 0x0fffffffffffffffULL ) goto integer_overflow; v = (v<<4) + x; zArg++; } }else{ while( IsDigit(zArg[0]) ){ - v = v*10 + zArg[0] - '0'; + if( v>=922337203685477580LL ){ + if( v>922337203685477580LL || zArg[0]>='8' ) goto integer_overflow; + } + v = v*10 + (zArg[0] - '0'); zArg++; } } for(i=0; i0x7fffffffffffffffULL ) goto integer_overflow; + return isNeg? -(sqlite3_int64)v : (sqlite3_int64)v; +integer_overflow: + return isNeg ? (i64)0x8000000000000000LL : 0x7fffffffffffffffLL; } /* @@ -1093,9 +1154,9 @@ static sqlite3_int64 integerValue(const char *zArg){ */ typedef struct ShellText ShellText; struct ShellText { - char *z; - int n; - int nAlloc; + char *zTxt; /* The text */ + i64 n; /* Number of bytes of zTxt[] actually used */ + i64 nAlloc; /* Number of bytes allocated for zTxt[] */ }; /* @@ -1105,7 +1166,7 @@ static void initText(ShellText *p){ memset(p, 0, sizeof(*p)); } static void freeText(ShellText *p){ - free(p->z); + sqlite3_free(p->zTxt); initText(p); } @@ -1130,26 +1191,26 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ } } - if( p->z==0 || p->n+len>=p->nAlloc ){ + if( p->zTxt==0 || p->n+len>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + len + 20; - p->z = realloc(p->z, p->nAlloc); - shell_check_oom(p->z); + p->zTxt = sqlite3_realloc64(p->zTxt, p->nAlloc); + shell_check_oom(p->zTxt); } if( quote ){ - char *zCsr = p->z+p->n; + char *zCsr = p->zTxt+p->n; *zCsr++ = quote; for(i=0; in = (int)(zCsr - p->z); + p->n = (i64)(zCsr - p->zTxt); *zCsr = '\0'; }else{ - memcpy(p->z+p->n, zAppend, nAppend); + memcpy(p->zTxt+p->n, zAppend, nAppend); p->n += nAppend; - p->z[p->n] = '\0'; + p->zTxt[p->n] = '\0'; } } @@ -1164,9 +1225,9 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ static char quoteChar(const char *zName){ int i; if( zName==0 ) return '"'; - if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + if( !IsAlpha(zName[0]) && zName[0]!='_' ) return '"'; for(i=0; zName[i]; i++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + if( !IsAlnum(zName[i]) && zName[i]!='_' ) return '"'; } return sqlite3_keyword_check(zName, i) ? '"' : 0; } @@ -1174,6 +1235,9 @@ static char quoteChar(const char *zName){ /* ** Construct a fake object name and column list to describe the structure ** of the view, virtual table, or table valued function zSchema.zName. +** +** The returned string comes from sqlite3_mprintf() and should be freed +** by the caller using sqlite3_free(). */ static char *shellFakeSchema( sqlite3 *db, /* The database connection containing the vtab */ @@ -1214,32 +1278,48 @@ static char *shellFakeSchema( sqlite3_finalize(pStmt); if( nRow==0 ){ freeText(&s); - s.z = 0; + s.zTxt = 0; } - return s.z; + return s.zTxt; } /* -** SQL function: shell_module_schema(X) +** SQL function: strtod(X) ** -** Return a fake schema for the table-valued function or eponymous virtual -** table X. +** Use the C-library strtod() function to convert string X into a double. +** Used for comparing the accuracy of SQLite's internal text-to-float conversion +** routines against the C-library. */ -static void shellModuleSchema( +static void shellStrtod( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ - const char *zName; - char *zFake; + char *z = (char*)sqlite3_value_text(apVal[0]); UNUSED_PARAMETER(nVal); - zName = (const char*)sqlite3_value_text(apVal[0]); - zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; - if( zFake ){ - sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), - -1, sqlite3_free); - free(zFake); - } + if( z==0 ) return; + sqlite3_result_double(pCtx, strtod(z,0)); +} + +/* +** SQL function: dtostr(X) +** +** Use the C-library printf() function to convert real value X into a string. +** Used for comparing the accuracy of SQLite's internal float-to-text conversion +** routines against the C-library. +*/ +static void shellDtostr( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + double r = sqlite3_value_double(apVal[0]); + int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26; + char z[400]; + if( n<1 ) n = 1; + if( n>350 ) n = 350; + sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } /* @@ -1303,7 +1383,7 @@ static void shellAddSchemaName( }else{ z = sqlite3_mprintf("%z\n/* %s */", z, zFake); } - free(zFake); + sqlite3_free(zFake); } if( z ){ sqlite3_result_text(pCtx, z, -1, sqlite3_free); @@ -1324,13 +1404,11 @@ static void shellAddSchemaName( #define SQLITE_EXTENSION_INIT1 #define SQLITE_EXTENSION_INIT2(X) (void)(X) -#if defined(_WIN32) && defined(_MSC_VER) -INCLUDE test_windirent.h -INCLUDE test_windirent.c -#define dirent DIRENT -#endif +INCLUDE ../ext/misc/windirent.h INCLUDE ../ext/misc/memtrace.c +INCLUDE ../ext/misc/pcachetrace.c INCLUDE ../ext/misc/shathree.c +INCLUDE ../ext/misc/sha1.c INCLUDE ../ext/misc/uint.c INCLUDE ../ext/misc/decimal.c #undef sqlite3_base_init @@ -1352,8 +1430,14 @@ INCLUDE ../ext/misc/appendvfs.c INCLUDE ../ext/misc/zipfile.c INCLUDE ../ext/misc/sqlar.c #endif +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +#endif +INCLUDE ../ext/intck/sqlite3intck.h +INCLUDE ../ext/intck/sqlite3intck.c +INCLUDE ../ext/misc/stmtrand.c +INCLUDE ../ext/misc/vfstrace.c #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 @@ -1384,11 +1468,13 @@ struct OpenSession { }; #endif +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) typedef struct ExpertInfo ExpertInfo; struct ExpertInfo { sqlite3expert *pExpert; int bVerbose; }; +#endif /* A single line in the EQP output */ typedef struct EQPGraphRow EQPGraphRow; @@ -1424,7 +1510,7 @@ typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ u8 autoEQPtest; /* autoEQP is in test mode */ u8 autoEQPtrace; /* autoEQP is in trace mode */ u8 scanstatsOn; /* True to display scan stats before each finalize */ @@ -1434,13 +1520,16 @@ struct ShellState { u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ u8 bSafeMode; /* True to prohibit unsafe operations */ u8 bSafeModePersist; /* The long-term value of bSafeMode */ + u8 eRestoreState; /* See comments above doAutoDetectRestore() */ + u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ + u8 eEscMode; /* Escape mode for text output */ ColModeOpts cmOpts; /* Option values affecting columnar mode output */ unsigned statsOn; /* True to display memory stats before each finalize */ - unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ + unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ int inputNesting; /* Track nesting level of .read and other redirects */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ - int lineno; /* Line number of last line read from in */ + i64 lineno; /* Line number of last line read from in */ int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ FILE *in; /* Read commands from this stream */ FILE *out; /* Write results here */ @@ -1487,9 +1576,11 @@ struct ShellState { int *aiIndent; /* Array of indents used in MODE_Explain */ int nIndent; /* Size of array aiIndent[] */ int iIndent; /* Index of current op in aiIndent[] */ - char *zNonce; /* Nonce for temporary safe-mode excapes */ + char *zNonce; /* Nonce for temporary safe-mode escapes */ EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +#endif #ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ @@ -1517,9 +1608,8 @@ static ShellState shellState; #define SHELL_OPEN_NORMAL 1 /* Normal database file */ #define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ #define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ -#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ -#define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ -#define SHELL_OPEN_HEXDB 6 /* Use "dbtotxt" output as data source */ +#define SHELL_OPEN_DESERIALIZE 4 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 5 /* Use "dbtotxt" output as data source */ /* Allowed values for ShellState.eTraceType */ @@ -1529,11 +1619,20 @@ static ShellState shellState; /* Bits in the ShellState.flgProgress variable */ #define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ -#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progres +#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progress ** callback limit is reached, and for each ** top-level SQL statement */ #define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ +/* Allowed values for ShellState.eEscMode. The default value should +** be 0, so to change the default, reorder the names. +*/ +#define SHELL_ESC_ASCII 0 /* Substitute ^Y for X where Y=X+0x40 */ +#define SHELL_ESC_SYMBOL 1 /* Substitute U+2400 graphics */ +#define SHELL_ESC_OFF 2 /* Send characters verbatim */ + +static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; + /* ** These are the allowed shellFlgs values */ @@ -1578,6 +1677,8 @@ static ShellState shellState; #define MODE_Box 16 /* Unicode box-drawing characters */ #define MODE_Count 17 /* Output only a count of the rows of output */ #define MODE_Off 18 /* No query output shown */ +#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ +#define MODE_Www 20 /* Full web-page output */ static const char *modeDescr[] = { "line", @@ -1598,7 +1699,9 @@ static const char *modeDescr[] = { "table", "box", "count", - "off" + "off", + "scanexp", + "www", }; /* @@ -1626,7 +1729,7 @@ static const char *modeDescr[] = { static void shellLog(void *pArg, int iErrCode, const char *zMsg){ ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; - utf8_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); } @@ -1643,7 +1746,7 @@ static void shellPutsFunc( ){ ShellState *p = (ShellState*)sqlite3_user_data(pCtx); (void)nVal; - utf8_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); sqlite3_result_value(pCtx, apVal[0]); } @@ -1662,8 +1765,7 @@ static void failIfSafeMode( va_start(ap, zErrMsg); zMsg = sqlite3_vmprintf(zErrMsg, ap); va_end(ap); - raw_printf(stderr, "line %d: ", p->lineno); - utf8_printf(stderr, "%s\n", zMsg); + sqlite3_fprintf(stderr, "line %lld: %s\n", p->lineno, zMsg); exit(1); } } @@ -1696,7 +1798,7 @@ static void editFunc( char *zCmd = 0; int bBin; int rc; - int hasCRNL = 0; + int hasCRLF = 0; FILE *f = 0; sqlite3_int64 sz; sqlite3_int64 x; @@ -1730,7 +1832,7 @@ static void editFunc( bBin = sqlite3_value_type(argv[0])==SQLITE_BLOB; /* When writing the file to be edited, do \n to \r\n conversions on systems ** that want \r\n line endings */ - f = fopen(zTempFile, bBin ? "wb" : "w"); + f = sqlite3_fopen(zTempFile, bBin ? "wb" : "w"); if( f==0 ){ sqlite3_result_error(context, "edit() cannot open temp file", -1); goto edit_func_end; @@ -1741,7 +1843,7 @@ static void editFunc( }else{ const char *z = (const char*)sqlite3_value_text(argv[0]); /* Remember whether or not the value originally contained \r\n */ - if( z && strstr(z,"\r\n")!=0 ) hasCRNL = 1; + if( z && strstr(z,"\r\n")!=0 ) hasCRLF = 1; x = fwrite(sqlite3_value_text(argv[0]), 1, (size_t)sz, f); } fclose(f); @@ -1761,7 +1863,7 @@ static void editFunc( sqlite3_result_error(context, "EDITOR returned non-zero", -1); goto edit_func_end; } - f = fopen(zTempFile, "rb"); + f = sqlite3_fopen(zTempFile, "rb"); if( f==0 ){ sqlite3_result_error(context, "edit() cannot reopen temp file after edit", -1); @@ -1786,7 +1888,7 @@ static void editFunc( sqlite3_result_blob64(context, p, sz, sqlite3_free); }else{ sqlite3_int64 i, j; - if( hasCRNL ){ + if( hasCRLF ){ /* If the original contains \r\n then do no conversions back to \n */ }else{ /* If the file did not originally contain \r\n then convert any new @@ -1828,6 +1930,21 @@ static void outputModePop(ShellState *p){ memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); } +/* +** Set output mode to text or binary for Windows. +*/ +static void setCrlfMode(ShellState *p){ +#ifdef _WIN32 + if( p->crlfMode ){ + sqlite3_fsetmode(p->out, _O_TEXT); + }else{ + sqlite3_fsetmode(p->out, _O_BINARY); + } +#else + UNUSED_PARAMETER(p); +#endif +} + /* ** Output the given string as a hex-encoded blob (eg. X'1234' ) */ @@ -1835,7 +1952,7 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ int i; unsigned char *aBlob = (unsigned char*)pBlob; - char *zStr = sqlite3_malloc(nBlob*2 + 1); + char *zStr = sqlite3_malloc64((i64)nBlob*2 + 1); shell_check_oom(zStr); for(i=0; iout; + sqlite3_fsetmode(out, _O_BINARY); if( z==0 ) return; - for(i=0; (c = z[i])!=0 && c!='\''; i++){} - if( c==0 ){ - utf8_printf(out,"'%s'",z); + for(i=0; (c = z[i])!=0; i++){ + if( c=='\'' ){ needDblQuote = 1; } + if( c>0x1f ) continue; + if( c=='\t' || c=='\n' ) continue; + if( c=='\r' && z[i+1]=='\n' ) continue; + needUnistr = 1; + break; + } + if( (needDblQuote==0 && needUnistr==0) + || (needDblQuote==0 && p->eEscMode==SHELL_ESC_OFF) + ){ + sqlite3_fprintf(out, "'%s'",z); + }else if( p->eEscMode==SHELL_ESC_OFF ){ + char *zEncoded = sqlite3_mprintf("%Q", z); + sqlite3_fputs(zEncoded, out); + sqlite3_free(zEncoded); }else{ - raw_printf(out, "'"); + if( needUnistr ){ + sqlite3_fputs("unistr('", out); + }else{ + sqlite3_fputs("'", out); + } while( *z ){ - for(i=0; (c = z[i])!=0 && c!='\''; i++){} - if( c=='\'' ) i++; + for(i=0; (c = z[i])!=0; i++){ + if( c=='\'' ) break; + if( c>0x1f ) continue; + if( c=='\t' || c=='\n' ) continue; + if( c=='\r' && z[i+1]=='\n' ) continue; + break; + } if( i ){ - utf8_printf(out, "%.*s", i, z); + sqlite3_fprintf(out, "%.*s", i, z); z += i; } + if( c==0 ) break; if( c=='\'' ){ - raw_printf(out, "'"); - continue; - } - if( c==0 ){ - break; + sqlite3_fputs("''", out); + }else{ + sqlite3_fprintf(out, "\\u%04x", c); } z++; } - raw_printf(out, "'"); + if( needUnistr ){ + sqlite3_fputs("')", out); + }else{ + sqlite3_fputs("'", out); + } } - setTextMode(out, 1); + setCrlfMode(p); } /* @@ -1918,128 +2052,250 @@ static void output_quoted_string(FILE *out, const char *z){ ** This is like output_quoted_string() but with the addition of the \r\n ** escape mechanism. */ -static void output_quoted_escaped_string(FILE *out, const char *z){ - int i; - char c; - setBinaryMode(out, 1); - for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} - if( c==0 ){ - utf8_printf(out,"'%s'",z); +static void output_quoted_escaped_string(ShellState *p, const char *z){ + char *zEscaped; + sqlite3_fsetmode(p->out, _O_BINARY); + if( p->eEscMode==SHELL_ESC_OFF ){ + zEscaped = sqlite3_mprintf("%Q", z); }else{ - const char *zNL = 0; - const char *zCR = 0; - int nNL = 0; - int nCR = 0; - char zBuf1[20], zBuf2[20]; - for(i=0; z[i]; i++){ - if( z[i]=='\n' ) nNL++; - if( z[i]=='\r' ) nCR++; - } - if( nNL ){ - raw_printf(out, "replace("); - zNL = unused_string(z, "\\n", "\\012", zBuf1); - } - if( nCR ){ - raw_printf(out, "replace("); - zCR = unused_string(z, "\\r", "\\015", zBuf2); - } - raw_printf(out, "'"); - while( *z ){ - for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} - if( c=='\'' ) i++; - if( i ){ - utf8_printf(out, "%.*s", i, z); - z += i; - } - if( c=='\'' ){ - raw_printf(out, "'"); - continue; - } - if( c==0 ){ - break; - } - z++; - if( c=='\n' ){ - raw_printf(out, "%s", zNL); - continue; - } - raw_printf(out, "%s", zCR); - } - raw_printf(out, "'"); - if( nCR ){ - raw_printf(out, ",'%s',char(13))", zCR); + zEscaped = sqlite3_mprintf("%#Q", z); + } + sqlite3_fputs(zEscaped, p->out); + sqlite3_free(zEscaped); + setCrlfMode(p); +} + +/* +** Find earliest of chars within s specified in zAny. +** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. +*/ +static const char *anyOfInStr(const char *s, const char *zAny, size_t ns){ + const char *pcFirst = 0; + if( ns == ~(size_t)0 ) ns = strlen(s); + while(*zAny){ + const char *pc = (const char*)memchr(s, *zAny&0xff, ns); + if( pc ){ + pcFirst = pc; + ns = pcFirst - s; } - if( nNL ){ - raw_printf(out, ",'%s',char(10))", zNL); + ++zAny; + } + return pcFirst; +} + +/* Skip over as much z[] input char sequence as is valid UTF-8, +** limited per nAccept char's or whole characters and containing +** no char cn such that ((1<=0 => char count, nAccept<0 => character + */ +const char *zSkipValidUtf8(const char *z, int nAccept, long ccm){ + int ng = (nAccept<0)? -nAccept : 0; + const char *pcLimit = (nAccept>=0)? z+nAccept : 0; + assert(z!=0); + while( (pcLimit)? (z= pcLimit ) return z; + else{ + char ct = *zt++; + if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){ + /* Trailing bytes are too few, too many, or invalid. */ + return z; + } + } + } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */ + z = zt; } } - setTextMode(out, 1); + return z; } + /* ** Output the given string as a quoted according to C or TCL quoting rules. */ static void output_c_string(FILE *out, const char *z){ - unsigned int c; - fputc('"', out); - while( (c = *(z++))!=0 ){ - if( c=='\\' ){ - fputc(c, out); - fputc(c, out); - }else if( c=='"' ){ - fputc('\\', out); - fputc('"', out); - }else if( c=='\t' ){ - fputc('\\', out); - fputc('t', out); - }else if( c=='\n' ){ - fputc('\\', out); - fputc('n', out); - }else if( c=='\r' ){ - fputc('\\', out); - fputc('r', out); + char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ + char ace[3] = "\\?"; + char cbsSay; + sqlite3_fputs(zq, out); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; + if( pcEnd > z ){ + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + } + if( (c = *pcEnd)==0 ) break; + ++pcEnd; + switch( c ){ + case '\\': case '"': + cbsSay = (char)c; + break; + case '\t': cbsSay = 't'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\f': cbsSay = 'f'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + sqlite3_fputs(ace, out); }else if( !isprint(c&0xff) ){ - raw_printf(out, "\\%03o", c&0xff); + sqlite3_fprintf(out, "\\%03o", c&0xff); }else{ - fputc(c, out); + ace[1] = (char)c; + sqlite3_fputs(ace+1, out); } + z = pcEnd; } - fputc('"', out); + sqlite3_fputs(zq, out); } /* -** Output the given string as a quoted according to JSON quoting rules. +** Output the given string as quoted according to JSON quoting rules. */ static void output_json_string(FILE *out, const char *z, i64 n){ - unsigned int c; + unsigned char c; + static const char *zq = "\""; + static long ctrlMask = ~0L; + static const char *zDQBS = "\"\\"; + const char *pcLimit; + char ace[3] = "\\?"; + char cbsSay; + if( z==0 ) z = ""; - if( n<0 ) n = strlen(z); - fputc('"', out); - while( n-- ){ - c = *(z++); - if( c=='\\' || c=='"' ){ - fputc('\\', out); - fputc(c, out); - }else if( c<=0x1f ){ - fputc('\\', out); - if( c=='\b' ){ - fputc('b', out); - }else if( c=='\f' ){ - fputc('f', out); - }else if( c=='\n' ){ - fputc('n', out); - }else if( c=='\r' ){ - fputc('r', out); - }else if( c=='\t' ){ - fputc('t', out); - }else{ - raw_printf(out, "u%04x",c); - } + pcLimit = z + ((n<0)? strlen(z) : (size_t)n); + sqlite3_fputs(zq, out); + while( z < pcLimit ){ + const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); + const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); + const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + if( pcEnd > z ){ + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + z = pcEnd; + } + if( z >= pcLimit ) break; + c = (unsigned char)*(z++); + switch( c ){ + case '"': case '\\': + cbsSay = (char)c; + break; + case '\b': cbsSay = 'b'; break; + case '\f': cbsSay = 'f'; break; + case '\n': cbsSay = 'n'; break; + case '\r': cbsSay = 'r'; break; + case '\t': cbsSay = 't'; break; + default: cbsSay = 0; break; + } + if( cbsSay ){ + ace[1] = cbsSay; + sqlite3_fputs(ace, out); + }else if( c<=0x1f || c>=0x7f ){ + sqlite3_fprintf(out, "\\u%04x", c); }else{ - fputc(c, out); + ace[1] = (char)c; + sqlite3_fputs(ace+1, out); + } + } + sqlite3_fputs(zq, out); +} + +/* +** Escape the input string if it is needed and in accordance with +** eEscMode. +** +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppFree to NULL +** and return the original string. If escaping is needed, write the +** escaped string into memory obtained from sqlite3_malloc64() or the +** equivalent, and return the new string and set *ppFree to the new string +** as well. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. +*/ +static const char *escapeOutput( + ShellState *p, + const char *zInX, + char **ppFree +){ + i64 i, j; + i64 nCtrl = 0; + unsigned char *zIn; + unsigned char c; + unsigned char *zOut; + + + /* No escaping if disabled */ + if( p->eEscMode==SHELL_ESC_OFF ){ + *ppFree = 0; + return zInX; + } + + /* Count the number of control characters in the string. */ + zIn = (unsigned char*)zInX; + for(i=0; (c = zIn[i])!=0; i++){ + if( c<=0x1f + && c!='\t' + && c!='\n' + && (c!='\r' || zIn[i+1]!='\n') + ){ + nCtrl++; + } + } + if( nCtrl==0 ){ + *ppFree = 0; + return zInX; + } + if( p->eEscMode==SHELL_ESC_SYMBOL ) nCtrl *= 2; + zOut = sqlite3_malloc64( i + nCtrl + 1 ); + shell_check_oom(zOut); + for(i=j=0; (c = zIn[i])!=0; i++){ + if( c>0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') + ){ + continue; + } + if( i>0 ){ + memcpy(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + switch( p->eEscMode ){ + case SHELL_ESC_SYMBOL: + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + break; + case SHELL_ESC_ASCII: + zOut[j++] = '^'; + zOut[j++] = 0x40+c; + break; } } - fputc('"', out); + if( i>0 ){ + memcpy(&zOut[j], zIn, i); + j += i; + } + zOut[j] = 0; + *ppFree = (char*)zOut; + return (char*)zOut; } /* @@ -2058,18 +2314,18 @@ static void output_html_string(FILE *out, const char *z){ && z[i]!='\''; i++){} if( i>0 ){ - utf8_printf(out,"%.*s",i,z); + sqlite3_fprintf(out, "%.*s",i,z); } if( z[i]=='<' ){ - raw_printf(out,"<"); + sqlite3_fputs("<", out); }else if( z[i]=='&' ){ - raw_printf(out,"&"); + sqlite3_fputs("&", out); }else if( z[i]=='>' ){ - raw_printf(out,">"); + sqlite3_fputs(">", out); }else if( z[i]=='\"' ){ - raw_printf(out,"""); + sqlite3_fputs(""", out); }else if( z[i]=='\'' ){ - raw_printf(out,"'"); + sqlite3_fputs("'", out); }else{ break; } @@ -2107,9 +2363,8 @@ static const char needCsvQuote[] = { ** is only issued if bSep is true. */ static void output_csv(ShellState *p, const char *z, int bSep){ - FILE *out = p->out; if( z==0 ){ - utf8_printf(out,"%s",p->nullValue); + sqlite3_fprintf(p->out, "%s",p->nullValue); }else{ unsigned i; for(i=0; z[i]; i++){ @@ -2121,14 +2376,14 @@ static void output_csv(ShellState *p, const char *z, int bSep){ if( i==0 || strstr(z, p->colSeparator)!=0 ){ char *zQuoted = sqlite3_mprintf("\"%w\"", z); shell_check_oom(zQuoted); - utf8_printf(out, "%s", zQuoted); + sqlite3_fputs(zQuoted, p->out); sqlite3_free(zQuoted); }else{ - utf8_printf(out, "%s", z); + sqlite3_fputs(z, p->out); } } if( bSep ){ - utf8_printf(p->out, "%s", p->colSeparator); + sqlite3_fputs(p->colSeparator, p->out); } } @@ -2236,16 +2491,16 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - utf8_printf(p->out, "authorizer: %s", azAction[op]); + sqlite3_fprintf(p->out, "authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - raw_printf(p->out, " "); + sqlite3_fputs(" ", p->out); if( az[i] ){ output_c_string(p->out, az[i]); }else{ - raw_printf(p->out, "NULL"); + sqlite3_fputs("NULL", p->out); } } - raw_printf(p->out, "\n"); + sqlite3_fputs("\n", p->out); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } @@ -2283,9 +2538,9 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - utf8_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - utf8_printf(out, "%s%s", z, zTail); + sqlite3_fprintf(out, "%s%s", z, zTail); } sqlite3_free(zToFree); } @@ -2320,7 +2575,7 @@ static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ if( zText==0 ) return; nText = strlen(zText); if( p->autoEQPtest ){ - utf8_printf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); + sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); } pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); shell_check_oom(pNew); @@ -2368,8 +2623,8 @@ static void eqp_render_level(ShellState *p, int iEqpId){ for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ pNext = eqp_next_row(p, iEqpId, pRow); z = pRow->zText; - utf8_printf(p->out, "%s%s%s\n", p->sGraph.zPrefix, - pNext ? "|--" : "`--", z); + sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, + pNext ? "|--" : "`--", z); if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); eqp_render_level(p, pRow->iEqpId); @@ -2389,13 +2644,13 @@ static void eqp_render(ShellState *p, i64 nCycle){ eqp_reset(p); return; } - utf8_printf(p->out, "%s\n", pRow->zText+3); + sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); p->sGraph.pRow = pRow->pNext; sqlite3_free(pRow); }else if( nCycle>0 ){ - utf8_printf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); + sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); }else{ - utf8_printf(p->out, "QUERY PLAN\n"); + sqlite3_fputs("QUERY PLAN\n", p->out); } p->sGraph.zPrefix[0] = 0; eqp_render_level(p, 0); @@ -2411,13 +2666,13 @@ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - raw_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); + sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - raw_printf(p->out, "Progress %u\n", p->nProgress); + sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); } return 0; } @@ -2430,10 +2685,10 @@ static void print_dashes(FILE *out, int N){ const char zDash[] = "--------------------------------------------------"; const int nDash = sizeof(zDash) - 1; while( N>nDash ){ - fputs(zDash, out); + sqlite3_fputs(zDash, out); N -= nDash; } - raw_printf(out, "%.*s", N, zDash); + sqlite3_fprintf(out, "%.*s", N, zDash); } /* @@ -2446,15 +2701,15 @@ static void print_row_separator( ){ int i; if( nArg>0 ){ - fputs(zSep, p->out); + sqlite3_fputs(zSep, p->out); print_dashes(p->out, p->actualWidth[0]+2); for(i=1; iout); + sqlite3_fputs(zSep, p->out); print_dashes(p->out, p->actualWidth[i]+2); } - fputs(zSep, p->out); + sqlite3_fputs(zSep, p->out); } - fputs("\n", p->out); + sqlite3_fputs("\n", p->out); } /* @@ -2484,45 +2739,69 @@ static int shell_callback( int len = strlen30(azCol[i] ? azCol[i] : ""); if( len>w ) w = len; } - if( p->cnt++>0 ) utf8_printf(p->out, "%s", p->rowSeparator); + if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); for(i=0; iout,"%*s = %s%s", w, azCol[i], - azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); + char *pFree = 0; + const char *pDisplay; + pDisplay = escapeOutput(p, azArg[i] ? azArg[i] : p->nullValue, &pFree); + sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i], + pDisplay, p->rowSeparator); + if( pFree ) sqlite3_free(pFree); } break; } + case MODE_ScanExp: case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - if( nArg>ArraySize(aExplainWidth) ){ - nArg = ArraySize(aExplainWidth); + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, w, azCol[i]); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); } for(i=0; iout, w); - fputs(i==nArg-1 ? "\n" : " ", p->out); + print_dashes(p->out, aWidth[i]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); } } + + /* If there is no data, exit early. */ if( azArg==0 ) break; + for(i=0; iw ){ - w = strlenChar(azArg[i]); + if( zVal && strlenChar(zVal)>w ){ + w = strlenChar(zVal); + zSep = " "; } - if( i==1 && p->aiIndent && p->pStmt ){ + if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ - utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); + sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); } break; } @@ -2537,14 +2816,18 @@ static int shell_callback( char cEnd = 0; char c; int nLine = 0; + int isIndex; + int isWhere = 0; assert( nArg==1 ); if( azArg[0]==0 ) break; if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 ){ - utf8_printf(p->out, "%s;\n", azArg[0]); + sqlite3_fprintf(p->out, "%s;\n", azArg[0]); break; } + isIndex = sqlite3_strlike("CREATE INDEX%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE UNIQUE INDEX%", azArg[0], 0)==0; z = sqlite3_mprintf("%s", azArg[0]); shell_check_oom(z); j = 0; @@ -2574,14 +2857,26 @@ static int shell_callback( nParen++; }else if( c==')' ){ nParen--; - if( nLine>0 && nParen==0 && j>0 ){ + if( nLine>0 && nParen==0 && j>0 && !isWhere ){ printSchemaLineN(p->out, z, j, "\n"); j = 0; } + }else if( (c=='w' || c=='W') + && nParen==0 && isIndex + && sqlite3_strnicmp("WHERE",&z[i],5)==0 + && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ + isWhere = 1; + }else if( isWhere && (c=='A' || c=='a') + && nParen==0 + && sqlite3_strnicmp("AND",&z[i],3)==0 + && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ + printSchemaLineN(p->out, z, j, "\n "); + j = 0; } z[j++] = c; if( nParen==1 && cEnd==0 && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + && !isWhere ){ if( c=='\n' ) j--; printSchemaLineN(p->out, z, j, "\n "); @@ -2599,116 +2894,129 @@ static int shell_callback( case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,"%s%s",azCol[i], - i==nArg-1 ? p->rowSeparator : p->colSeparator); + char *z = azCol[i]; + char *pFree; + const char *zOut = escapeOutput(p, z, &pFree); + sqlite3_fprintf(p->out, "%s%s", zOut, + i==nArg-1 ? p->rowSeparator : p->colSeparator); + if( pFree ) sqlite3_free(pFree); } } if( azArg==0 ) break; for(i=0; inullValue; - utf8_printf(p->out, "%s", z); - if( iout, "%s", p->colSeparator); - }else{ - utf8_printf(p->out, "%s", p->rowSeparator); - } + zOut = escapeOutput(p, z, &pFree); + sqlite3_fputs(zOut, p->out); + if( pFree ) sqlite3_free(pFree); + sqlite3_fputs((icolSeparator : p->rowSeparator, p->out); } break; } + case MODE_Www: case MODE_Html: { - if( p->cnt++==0 && p->showHeader ){ - raw_printf(p->out,""); + if( p->cnt==0 && p->cMode==MODE_Www ){ + sqlite3_fputs( + "\n" + "
    \n" + ,p->out + ); + } + if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ + sqlite3_fputs("", p->out); for(i=0; iout,"\n"); + sqlite3_fputs("\n", p->out); } - raw_printf(p->out,"\n"); + sqlite3_fputs("\n", p->out); } + p->cnt++; if( azArg==0 ) break; - raw_printf(p->out,""); + sqlite3_fputs("", p->out); for(i=0; iout,"\n"); + sqlite3_fputs("\n", p->out); } - raw_printf(p->out,"\n"); + sqlite3_fputs("\n", p->out); break; } case MODE_Tcl: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,azCol[i] ? azCol[i] : ""); - if(iout, "%s", p->colSeparator); + output_c_string(p->out, azCol[i] ? azCol[i] : ""); + if(icolSeparator, p->out); } - utf8_printf(p->out, "%s", p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } if( azArg==0 ) break; for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); - if(iout, "%s", p->colSeparator); + if(icolSeparator, p->out); } - utf8_printf(p->out, "%s", p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); break; } case MODE_Csv: { - setBinaryMode(p->out, 1); + sqlite3_fsetmode(p->out, _O_BINARY); if( p->cnt++==0 && p->showHeader ){ for(i=0; iout, "%s", p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } if( nArg>0 ){ for(i=0; iout, "%s", p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } - setTextMode(p->out, 1); + setCrlfMode(p); break; } case MODE_Insert: { if( azArg==0 ) break; - utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); + sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); if( p->showHeader ){ - raw_printf(p->out,"("); + sqlite3_fputs("(", p->out); for(i=0; i0 ) raw_printf(p->out, ","); + if( i>0 ) sqlite3_fputs(",", p->out); if( quoteChar(azCol[i]) ){ char *z = sqlite3_mprintf("\"%w\"", azCol[i]); shell_check_oom(z); - utf8_printf(p->out, "%s", z); + sqlite3_fputs(z, p->out); sqlite3_free(z); }else{ - raw_printf(p->out, "%s", azCol[i]); + sqlite3_fprintf(p->out, "%s", azCol[i]); } } - raw_printf(p->out,")"); + sqlite3_fputs(")", p->out); } p->cnt++; for(i=0; iout, i>0 ? "," : " VALUES("); + sqlite3_fputs(i>0 ? "," : " VALUES(", p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - utf8_printf(p->out,"NULL"); + sqlite3_fputs("NULL", p->out); }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(p, azArg[i]); }else{ - output_quoted_escaped_string(p->out, azArg[i]); + output_quoted_escaped_string(p, azArg[i]); } }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - utf8_printf(p->out,"%s", azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "9.0e+999"); + sqlite3_fputs("9.0e+999", p->out); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-9.0e+999"); + sqlite3_fputs("-9.0e+999", p->out); }else{ sqlite3_int64 ir = (sqlite3_int64)r; if( r==(double)ir ){ @@ -2716,48 +3024,48 @@ static int shell_callback( }else{ sqlite3_snprintf(50,z,"%!.20g", r); } - raw_printf(p->out, "%s", z); + sqlite3_fputs(z, p->out); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); output_hex_blob(p->out, pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - utf8_printf(p->out,"%s", azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(p, azArg[i]); }else{ - output_quoted_escaped_string(p->out, azArg[i]); + output_quoted_escaped_string(p, azArg[i]); } } - raw_printf(p->out,");\n"); + sqlite3_fputs(");\n", p->out); break; } case MODE_Json: { if( azArg==0 ) break; if( p->cnt==0 ){ - fputs("[{", p->out); + sqlite3_fputs("[{", p->out); }else{ - fputs(",\n{", p->out); + sqlite3_fputs(",\n{", p->out); } p->cnt++; for(i=0; iout, azCol[i], -1); - putc(':', p->out); + sqlite3_fputs(":", p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - fputs("null",p->out); + sqlite3_fputs("null", p->out); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "9.0e+999"); + sqlite3_fputs("9.0e+999", p->out); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-9.0e+999"); + sqlite3_fputs("-9.0e+999", p->out); }else{ sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + sqlite3_fputs(z, p->out); } }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); @@ -2766,65 +3074,65 @@ static int shell_callback( }else if( aiType && aiType[i]==SQLITE_TEXT ){ output_json_string(p->out, azArg[i], -1); }else{ - utf8_printf(p->out,"%s", azArg[i]); + sqlite3_fputs(azArg[i], p->out); } if( iout); + sqlite3_fputs(",", p->out); } } - putc('}', p->out); + sqlite3_fputs("}", p->out); break; } case MODE_Quote: { if( azArg==0 ) break; if( p->cnt==0 && p->showHeader ){ for(i=0; i0 ) fputs(p->colSeparator, p->out); - output_quoted_string(p->out, azCol[i]); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + output_quoted_string(p, azCol[i]); } - fputs(p->rowSeparator, p->out); + sqlite3_fputs(p->rowSeparator, p->out); } p->cnt++; for(i=0; i0 ) fputs(p->colSeparator, p->out); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - utf8_printf(p->out,"NULL"); + sqlite3_fputs("NULL", p->out); }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(p, azArg[i]); }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - utf8_printf(p->out,"%s", azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else if( aiType && aiType[i]==SQLITE_FLOAT ){ char z[50]; double r = sqlite3_column_double(p->pStmt, i); sqlite3_snprintf(50,z,"%!.20g", r); - raw_printf(p->out, "%s", z); + sqlite3_fputs(z, p->out); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); output_hex_blob(p->out, pBlob, nBlob); }else if( isNumber(azArg[i], 0) ){ - utf8_printf(p->out,"%s", azArg[i]); + sqlite3_fputs(azArg[i], p->out); }else{ - output_quoted_string(p->out, azArg[i]); + output_quoted_string(p, azArg[i]); } } - fputs(p->rowSeparator, p->out); + sqlite3_fputs(p->rowSeparator, p->out); break; } case MODE_Ascii: { if( p->cnt++==0 && p->showHeader ){ for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); - utf8_printf(p->out,"%s",azCol[i] ? azCol[i] : ""); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); } - utf8_printf(p->out, "%s", p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); } if( azArg==0 ) break; for(i=0; i0 ) utf8_printf(p->out, "%s", p->colSeparator); - utf8_printf(p->out,"%s",azArg[i] ? azArg[i] : p->nullValue); + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); } - utf8_printf(p->out, "%s", p->rowSeparator); + sqlite3_fputs(p->rowSeparator, p->out); break; } case MODE_EQP: { @@ -2903,7 +3211,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -2916,28 +3224,17 @@ static void createSelftestTable(ShellState *p){ ** table name. */ static void set_table_name(ShellState *p, const char *zName){ - int i, n; - char cQuote; - char *z; - if( p->zDestTable ){ - free(p->zDestTable); + sqlite3_free(p->zDestTable); p->zDestTable = 0; } if( zName==0 ) return; - cQuote = quoteChar(zName); - n = strlen30(zName); - if( cQuote ) n += n+2; - z = p->zDestTable = malloc( n+1 ); - shell_check_oom(z); - n = 0; - if( cQuote ) z[n++] = cQuote; - for(i=0; zName[i]; i++){ - z[n++] = zName[i]; - if( zName[i]==cQuote ) z[n++] = cQuote; + if( quoteChar(zName) ){ + p->zDestTable = sqlite3_mprintf("\"%w\"", zName); + }else{ + p->zDestTable = sqlite3_mprintf("%s", zName); } - if( cQuote ) z[n++] = cQuote; - z[n] = 0; + shell_check_oom(p->zDestTable); } /* @@ -3006,8 +3303,8 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, - sqlite3_errmsg(p->db), zContext); + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", + rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; return rc; @@ -3016,23 +3313,23 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - utf8_printf(p->out, "%s", z); + sqlite3_fprintf(p->out, "%s", z); for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + sqlite3_fprintf(p->out, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - raw_printf(p->out, "\n;\n"); + sqlite3_fputs("\n;\n", p->out); }else{ - raw_printf(p->out, ";\n"); + sqlite3_fputs(";\n", p->out); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - utf8_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, - sqlite3_errmsg(p->db)); + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", + rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } return rc; @@ -3043,7 +3340,7 @@ static int run_table_dump_query( */ static char *save_err_msg( sqlite3 *db, /* Database to query */ - const char *zPhase, /* When the error occcurs */ + const char *zPhase, /* When the error occurs */ int rc, /* Error code returned from API */ const char *zSql /* SQL string, or NULL */ ){ @@ -3072,9 +3369,9 @@ static void displayLinuxIoStats(FILE *out){ FILE *in; char z[200]; sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); - in = fopen(z, "rb"); + in = sqlite3_fopen(z, "rb"); if( in==0 ) return; - while( fgets(z, sizeof(z), in)!=0 ){ + while( sqlite3_fgets(z, sizeof(z), in)!=0 ){ static const struct { const char *zPattern; const char *zDesc; @@ -3091,7 +3388,7 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; iout, "%-36s %s\n", zLabel, zLine); + sqlite3_fprintf(out, "%-36s %s\n", zLabel, zLine); } /* @@ -3134,8 +3431,8 @@ static int display_stats( ShellState *pArg, /* Pointer to ShellState */ int bReset /* True to reset the stats */ ){ - int iCur; - int iHiwtr; + int iCur, iHiwtr; + sqlite3_int64 iCur64, iHiwtr64; FILE *out; if( pArg==0 || pArg->out==0 ) return 0; out = pArg->out; @@ -3145,21 +3442,22 @@ static int display_stats( sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - raw_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); for(i=0; istatsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - raw_printf(pArg->out, "VM-steps: %d\n", iCur); + sqlite3_fprintf(out, "VM-steps: %d\n", iCur); } return 0; } - displayStatLine(pArg, "Memory Used:", + displayStatLine(out, "Memory Used:", "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); - displayStatLine(pArg, "Number of Outstanding Allocations:", + displayStatLine(out, "Number of Outstanding Allocations:", "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - displayStatLine(pArg, "Number of Pcache Pages Used:", + displayStatLine(out, "Number of Pcache Pages Used:", "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + displayStatLine(out, "Number of Pcache Overflow Bytes:", "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); - displayStatLine(pArg, "Largest Allocation:", + displayStatLine(out, "Largest Allocation:", "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); - displayStatLine(pArg, "Largest Pcache Allocation:", + displayStatLine(out, "Largest Pcache Allocation:", "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - displayStatLine(pArg, "Deepest Parser Stack:", + displayStatLine(out, "Deepest Parser Stack:", "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif @@ -3196,73 +3494,90 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Lookaside Slots Used: %d (max %d)\n", - iCur, iHiwtr); + sqlite3_fprintf(out, + "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Successful lookaside attempts: %d\n", - iHiwtr); + sqlite3_fprintf(out, + "Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Lookaside failures due to size: %d\n", - iHiwtr); + sqlite3_fprintf(out, + "Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Lookaside failures due to OOM: %d\n", - iHiwtr); + sqlite3_fprintf(out, + "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Pager Heap Usage: %d bytes\n", - iCur); + sqlite3_fprintf(out, + "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache hits: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache misses: %d\n", iCur); + iHiwtr64 = iCur64 = -1; + sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, + 0); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache spills: %d\n", iCur); + sqlite3_fprintf(out, + "Page cache spills: %d\n", iCur); + sqlite3_fprintf(out, + "Temporary data spilled to disk: %lld\n", iCur64); + sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, + 1); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", - iCur); + sqlite3_fprintf(out, + "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", - iCur); + sqlite3_fprintf(out, + "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } if( pArg->pStmt ){ int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - raw_printf(pArg->out, "Fullscan Steps: %d\n", iCur); + sqlite3_fprintf(out, + "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - raw_printf(pArg->out, "Sort Operations: %d\n", iCur); + sqlite3_fprintf(out, + "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - raw_printf(pArg->out, "Autoindex Inserts: %d\n", iCur); + sqlite3_fprintf(out, + "Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - raw_printf(pArg->out, "Bloom filter bypass taken: %d/%d\n", - iHit, iHit+iMiss); + sqlite3_fprintf(out, + "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); + sqlite3_fprintf(out, + "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + sqlite3_fprintf(out, + "Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - raw_printf(pArg->out, "Number of times run: %d\n", iCur); + sqlite3_fprintf(out, + "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); + sqlite3_fprintf(out, + "Memory used by prepared stmt: %d\n", iCur); } #ifdef __linux__ @@ -3303,17 +3618,11 @@ static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ } #endif -/* -** Display scan stats. -*/ -static void display_scanstats( +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else static const int f = SQLITE_SCANSTAT_COMPLEX; sqlite3_stmt *p = pArg->pStmt; int ii = 0; @@ -3327,7 +3636,7 @@ static void display_scanstats( if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ break; } - n = strlen(z) + scanStatsHeight(p, ii)*3; + n = (int)strlen(z) + scanStatsHeight(p, ii)*3; if( n>nWidth ) nWidth = n; } nWidth += 4; @@ -3339,12 +3648,12 @@ static void display_scanstats( i64 nCycle = 0; int iId = 0; int iPid = 0; - const char *z = 0; + const char *zo = 0; const char *zName = 0; char *zText = 0; double rEst = 0.0; - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ break; } sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); @@ -3355,7 +3664,7 @@ static void display_scanstats( sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); - zText = sqlite3_mprintf("%s", z); + zText = sqlite3_mprintf("%s", zo); if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ char *z = 0; if( nCycle>=0 && nTotal>0 ){ @@ -3385,8 +3694,9 @@ static void display_scanstats( } eqp_render(pArg, nTotal); -#endif } +#endif + /* ** Parameter azArray points to a zero-terminated array of strings. zStr @@ -3424,8 +3734,6 @@ static int str_in_array(const char *zStr, const char **azArray){ ** and "Goto" by 2 spaces. */ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - const char *zSql; /* The text of the SQL statement */ - const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ @@ -3436,65 +3744,45 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; - /* Try to figure out if this is really an EXPLAIN statement. If this - ** cannot be verified, return early. */ - if( sqlite3_column_count(pSql)!=8 ){ - p->cMode = p->mode; - return; - } - zSql = sqlite3_sql(pSql); - if( zSql==0 ) return; - for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); - if( sqlite3_strnicmp(z, "explain", 7) ){ - p->cMode = p->mode; - return; - } + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ int i; int iAddr = sqlite3_column_int(pSql, 0); const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - - /* Set p2 to the P2 field of the current opcode. Then, assuming that - ** p2 is an instruction address, set variable p2op to the index of that - ** instruction in the aiIndent[] array. p2 and p2op may be different if - ** the current instruction is part of a sub-program generated by an - ** SQL trigger or foreign key. */ + int p1 = sqlite3_column_int(pSql, 2); int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ int p2op = (p2 + (iOp-iAddr)); /* Grow the p->aiIndent array as required */ if( iOp>=nAlloc ){ - if( iOp==0 ){ - /* Do further verfication that this is explain output. Abort if - ** it is not */ - static const char *explainCols[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; - int jj; - for(jj=0; jjcMode = p->mode; - sqlite3_reset(pSql); - return; - } - } - } nAlloc += 100; p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); shell_check_oom(p->aiIndent); abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); shell_check_oom(abYield); } + abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ for(i=p2op; iaiIndent[i] += 2; } - if( str_in_array(zOp, azGoto) && p2opnIndent - && (abYield[p2op] || sqlite3_column_int(pSql, 2)) - ){ + if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; } } @@ -3514,6 +3802,54 @@ static void explain_data_delete(ShellState *p){ p->iIndent = 0; } +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + /* ** Disable and restore .wheretrace and .treetrace/.selecttrace settings. */ @@ -3596,6 +3932,45 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ sqlite3_bind_double(pStmt, i, INFINITY); #endif + }else if( strncmp(zVar, "$int_", 5)==0 ){ + sqlite3_bind_int(pStmt, i, atoi(&zVar[5])); + }else if( strncmp(zVar, "$text_", 6)==0 ){ + size_t szVar = strlen(zVar); + char *zBuf = sqlite3_malloc64( szVar-5 ); + if( zBuf ){ + memcpy(zBuf, &zVar[6], szVar-5); + sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); + } +#ifdef SQLITE_ENABLE_CARRAY + }else if( strncmp(zVar, "$carray_", 8)==0 ){ + static char *azColorNames[] = { + "azure", "black", "blue", "brown", "cyan", "fuchsia", "gold", + "gray", "green", "indigo", "khaki", "lime", "magenta", "maroon", + "navy", "olive", "orange", "pink", "purple", "red", "silver", + "tan", "teal", "violet", "white", "yellow" + }; + static int aPrimes[] = { + 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, + 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 + }; + /* Special bindings: carray($carray_clr), carray($carray_primes) + ** with --unsafe-testing: carray($carray_clr_p,26,'char*'), + ** carray($carray_primes_p,26,'int32') + */ + if( strcmp(zVar+8,"clr")==0 ){ + sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0); + }else if( strcmp(zVar+8,"primes")==0 ){ + sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0); + }else if( strcmp(zVar+8,"clr_p")==0 + && ShellHasFlag(pArg,SHFLG_TestingMode) ){ + sqlite3_bind_pointer(pStmt,i,azColorNames,"carray",0); + }else if( strcmp(zVar+8,"primes_p")==0 + && ShellHasFlag(pArg,SHFLG_TestingMode) ){ + sqlite3_bind_pointer(pStmt,i,aPrimes,"carray",0); + }else{ + sqlite3_bind_null(pStmt, i); + } +#endif }else{ sqlite3_bind_null(pStmt, i); } @@ -3639,10 +4014,10 @@ static void print_box_line(FILE *out, int N){ const int nDash = sizeof(zDash) - 1; N *= 3; while( N>nDash ){ - utf8_printf(out, zDash); + sqlite3_fputs(zDash, out); N -= nDash; } - utf8_printf(out, "%.*s", N, zDash); + sqlite3_fprintf(out, "%.*s", N, zDash); } /* @@ -3657,15 +4032,15 @@ static void print_box_row_separator( ){ int i; if( nArg>0 ){ - utf8_printf(p->out, "%s", zSep1); + sqlite3_fputs(zSep1, p->out); print_box_line(p->out, p->actualWidth[0]+2); for(i=1; iout, "%s", zSep2); + sqlite3_fputs(zSep2, p->out); print_box_line(p->out, p->actualWidth[i]+2); } - utf8_printf(p->out, "%s", zSep3); + sqlite3_fputs(zSep3, p->out); } - fputs("\n", p->out); + sqlite3_fputs("\n", p->out); } /* @@ -3680,6 +4055,7 @@ static void print_box_row_separator( ** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) */ static char *translateForDisplayAndDup( + ShellState *p, /* To access current settings */ const unsigned char *z, /* Input text to be transformed */ const unsigned char **pzTail, /* OUT: Tail of the input for next line */ int mxWidth, /* Max width. 0 means no limit */ @@ -3699,12 +4075,23 @@ static char *translateForDisplayAndDup( if( mxWidth==0 ) mxWidth = 1000000; i = j = n = 0; while( n=' ' ){ + unsigned char c = z[i]; + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + i += len; + j += len; + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ n++; - do{ i++; j++; }while( (z[i]&0xc0)==0x80 ); + i++; + j++; continue; } - if( z[i]=='\t' ){ + if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break; + if( c=='\t' ){ do{ n++; j++; @@ -3712,16 +4099,23 @@ static char *translateForDisplayAndDup( i++; continue; } - break; + if( c==0x1b && p->eEscMode==SHELL_ESC_OFF && (k = isVt100(&z[i]))>0 ){ + i += k; + j += k; + }else{ + n++; + j += 3; + i++; + } } if( n>=mxWidth && bWordWrap ){ /* Perhaps try to back up to a better place to break the line */ for(k=i; k>i/2; k--){ - if( isspace(z[k-1]) ) break; + if( IsSpace(z[k-1]) ) break; } if( k<=i/2 ){ for(k=i; k>i/2; k--){ - if( isalnum(z[k-1])!=isalnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + if( IsAlnum(z[k-1])!=IsAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; } } if( k<=i/2 ){ @@ -3746,11 +4140,20 @@ static char *translateForDisplayAndDup( shell_check_oom(zOut); i = j = n = 0; while( i=' ' ){ + unsigned char c = z[i]; + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + do{ zOut[j++] = z[i++]; }while( (--len)>0 ); + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ n++; - do{ zOut[j++] = z[i++]; }while( (z[i]&0xc0)==0x80 ); + zOut[j++] = z[i++]; continue; } + if( c==0 ) break; if( z[i]=='\t' ){ do{ n++; @@ -3759,12 +4162,44 @@ static char *translateForDisplayAndDup( i++; continue; } - break; + switch( p->eEscMode ){ + case SHELL_ESC_SYMBOL: + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80 + c; + break; + case SHELL_ESC_ASCII: + zOut[j++] = '^'; + zOut[j++] = 0x40 + c; + break; + case SHELL_ESC_OFF: { + int nn; + if( c==0x1b && (nn = isVt100(&z[i]))>0 ){ + memcpy(&zOut[j], &z[i], nn); + j += nn; + i += nn - 1; + }else{ + zOut[j++] = c; + } + break; + } + } + i++; } zOut[j] = 0; return (char*)zOut; } +/* Return true if the text string z[] contains characters that need +** unistr() escaping. +*/ +static int needUnistr(const unsigned char *z){ + unsigned char c; + if( z==0 ) return 0; + while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; } + return c!=0; +} + /* Extract the value of the i-th current column for pStmt as an SQL literal ** value. Memory is obtained from sqlite3_malloc64() and must be freed by ** the caller. @@ -3779,7 +4214,8 @@ static char *quoted_column(sqlite3_stmt *pStmt, int i){ return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); } case SQLITE_TEXT: { - return sqlite3_mprintf("%Q",sqlite3_column_text(pStmt,i)); + const unsigned char *zText = sqlite3_column_text(pStmt,i); + return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText); } case SQLITE_BLOB: { int j; @@ -3809,7 +4245,7 @@ static char *quoted_column(sqlite3_stmt *pStmt, int i){ */ static void exec_prepared_stmt_columnar( ShellState *p, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statment to run */ + sqlite3_stmt *pStmt /* Statement to run */ ){ sqlite3_int64 nRow = 0; int nColumn = 0; @@ -3834,6 +4270,7 @@ static void exec_prepared_stmt_columnar( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ) return; nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; nAlloc = nColumn*4; if( nAlloc<=0 ) nAlloc = 1; azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); @@ -3870,7 +4307,7 @@ static void exec_prepared_stmt_columnar( if( wx<0 ) wx = -wx; uz = (const unsigned char*)sqlite3_column_name(pStmt,i); if( uz==0 ) uz = (u8*)""; - azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); + azData[i] = translateForDisplayAndDup(p, uz, &zNotUsed, wx, bw); } do{ int useNextLine = bNextLine; @@ -3894,6 +4331,7 @@ static void exec_prepared_stmt_columnar( uz = azNextLine[i]; if( uz==0 ) uz = (u8*)zEmpty; }else if( p->cmOpts.bQuote ){ + assert( azQuoted!=0 ); sqlite3_free(azQuoted[i]); azQuoted[i] = quoted_column(pStmt,i); uz = (const unsigned char*)azQuoted[i]; @@ -3902,7 +4340,7 @@ static void exec_prepared_stmt_columnar( if( uz==0 ) uz = (u8*)zShowNull; } azData[nRow*nColumn + i] - = translateForDisplayAndDup(uz, &azNextLine[i], wx, bw); + = translateForDisplayAndDup(p, uz, &azNextLine[i], wx, bw); if( azNextLine[i] ){ bNextLine = 1; abRowDiv[nRow-1] = 0; @@ -3919,7 +4357,6 @@ static void exec_prepared_stmt_columnar( if( n>p->actualWidth[j] ) p->actualWidth[j] = n; } if( seenInterrupt ) goto columnar_end; - if( nColumn==0 ) goto columnar_end; switch( p->cMode ){ case MODE_Column: { colSep = " "; @@ -3929,11 +4366,11 @@ static void exec_prepared_stmt_columnar( w = p->actualWidth[i]; if( p->colWidth[i]<0 ) w = -w; utf8_width_print(p->out, w, azData[i]); - fputs(i==nColumn-1?"\n":" ", p->out); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); } for(i=0; iout, p->actualWidth[i]); - fputs(i==nColumn-1?"\n":" ", p->out); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); } } break; @@ -3942,12 +4379,13 @@ static void exec_prepared_stmt_columnar( colSep = " | "; rowSep = " |\n"; print_row_separator(p, nColumn, "+"); - fputs("| ", p->out); + sqlite3_fputs("| ", p->out); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - fputs(i==nColumn-1?" |\n":" | ", p->out); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); } print_row_separator(p, nColumn, "+"); break; @@ -3955,12 +4393,13 @@ static void exec_prepared_stmt_columnar( case MODE_Markdown: { colSep = " | "; rowSep = " |\n"; - fputs("| ", p->out); + sqlite3_fputs("| ", p->out); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s", (w-n)/2, "", azData[i], (w-n+1)/2, ""); - fputs(i==nColumn-1?" |\n":" | ", p->out); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); } print_row_separator(p, nColumn, "|"); break; @@ -3969,13 +4408,13 @@ static void exec_prepared_stmt_columnar( colSep = " " BOX_13 " "; rowSep = " " BOX_13 "\n"; print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - utf8_printf(p->out, BOX_13 " "); + sqlite3_fputs(BOX_13 " ", p->out); for(i=0; iactualWidth[i]; n = strlenChar(azData[i]); - utf8_printf(p->out, "%*s%s%*s%s", - (w-n)/2, "", azData[i], (w-n+1)/2, "", - i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + sqlite3_fprintf(p->out, "%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); } print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); break; @@ -3983,7 +4422,7 @@ static void exec_prepared_stmt_columnar( } for(i=nColumn, j=0; icMode!=MODE_Column ){ - utf8_printf(p->out, "%s", p->cMode==MODE_Box?BOX_13" ":"| "); + sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); } z = azData[i]; if( z==0 ) z = p->nullValue; @@ -3991,20 +4430,20 @@ static void exec_prepared_stmt_columnar( if( p->colWidth[j]<0 ) w = -w; utf8_width_print(p->out, w, z); if( j==nColumn-1 ){ - utf8_printf(p->out, "%s", rowSep); + sqlite3_fputs(rowSep, p->out); if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1cMode==MODE_Table ){ print_row_separator(p, nColumn, "+"); }else if( p->cMode==MODE_Box ){ print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); }else if( p->cMode==MODE_Column ){ - raw_printf(p->out, "\n"); + sqlite3_fputs("\n", p->out); } } j = -1; if( seenInterrupt ) goto columnar_end; }else{ - utf8_printf(p->out, "%s", colSep); + sqlite3_fputs(colSep, p->out); } } if( p->cMode==MODE_Table ){ @@ -4014,7 +4453,7 @@ static void exec_prepared_stmt_columnar( } columnar_end: if( seenInterrupt ){ - utf8_printf(p->out, "Interrupt\n"); + sqlite3_fputs("Interrupt\n", p->out); } nData = (nRow+1)*nColumn; for(i=0; icMode==MODE_Json ){ - fputs("]\n", pArg->out); + sqlite3_fputs("]\n", pArg->out); + }else if( pArg->cMode==MODE_Www ){ + sqlite3_fputs("
    "); + sqlite3_fputs("", p->out); output_html_string(p->out, azCol[i]); - raw_printf(p->out,"
    "); + sqlite3_fputs("", p->out); output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - raw_printf(p->out,"
    \n
    \n", pArg->out);
       }else if( pArg->cMode==MODE_Count ){
         char zBuf[200];
         sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n",
@@ -4112,7 +4553,7 @@ static void exec_prepared_stmt(
   }
 }
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
 /*
 ** This function is called to process SQL if the previous shell command
 ** was ".expert". It passes the SQL in the second argument directly to
@@ -4150,10 +4591,10 @@ static int expertFinish(
 ){
   int rc = SQLITE_OK;
   sqlite3expert *p = pState->expert.pExpert;
+  FILE *out = pState->out;
   assert( p );
   assert( bCancel || pzErr==0 || *pzErr==0 );
   if( bCancel==0 ){
-    FILE *out = pState->out;
     int bVerbose = pState->expert.bVerbose;
 
     rc = sqlite3_expert_analyze(p, pzErr);
@@ -4163,8 +4604,8 @@ static int expertFinish(
 
       if( bVerbose ){
         const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
-        raw_printf(out, "-- Candidates -----------------------------\n");
-        raw_printf(out, "%s\n", zCand);
+        sqlite3_fputs("-- Candidates -----------------------------\n", out);
+        sqlite3_fprintf(out, "%s\n", zCand);
       }
       for(i=0; i=2 && 0==cli_strncmp(z, "-sample", n) ){
       if( i==(nArg-1) ){
-        raw_printf(stderr, "option requires an argument: %s\n", z);
+        sqlite3_fprintf(stderr, "option requires an argument: %s\n", z);
         rc = SQLITE_ERROR;
       }else{
         iSample = (int)integerValue(azArg[++i]);
         if( iSample<0 || iSample>100 ){
-          raw_printf(stderr, "value out of range: %s\n", azArg[i]);
+          sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]);
           rc = SQLITE_ERROR;
         }
       }
     }
     else{
-      raw_printf(stderr, "unknown option: %s\n", z);
+      sqlite3_fprintf(stderr,"unknown option: %s\n", z);
       rc = SQLITE_ERROR;
     }
   }
@@ -4230,8 +4672,8 @@ static int expertDotCommand(
   if( rc==SQLITE_OK ){
     pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
     if( pState->expert.pExpert==0 ){
-      raw_printf(stderr, "sqlite3_expert_new: %s\n",
-                 zErr ? zErr : "out of memory");
+      sqlite3_fprintf(stderr,
+          "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory");
       rc = SQLITE_ERROR;
     }else{
       sqlite3_expert_config(
@@ -4243,7 +4685,7 @@ static int expertDotCommand(
 
   return rc;
 }
-#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+#endif /* !SQLITE_OMIT_VIRTUALTABLE && !SQLITE_OMIT_AUTHORIZATION */
 
 /*
 ** Execute a statement or set of statements.  Print
@@ -4269,7 +4711,7 @@ static int shell_exec(
     *pzErrMsg = NULL;
   }
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
   if( pArg->expert.pExpert ){
     rc = expertHandleSQL(pArg, zSql, pzErrMsg);
     return expertFinish(pArg, (rc!=SQLITE_OK), pzErrMsg);
@@ -4294,7 +4736,7 @@ static int shell_exec(
       if( zStmtSql==0 ) zStmtSql = "";
       while( IsSpace(zStmtSql[0]) ) zStmtSql++;
 
-      /* save off the prepared statment handle and reset row count */
+      /* save off the prepared statement handle and reset row count */
       if( pArg ){
         pArg->pStmt = pStmt;
         pArg->cnt = 0;
@@ -4303,17 +4745,17 @@ static int shell_exec(
       /* Show the EXPLAIN QUERY PLAN if .eqp is on */
       if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){
         sqlite3_stmt *pExplain;
-        char *zEQP;
         int triggerEQP = 0;
         disable_debug_trace_modes();
         sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
         if( pArg->autoEQP>=AUTOEQP_trigger ){
           sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
         }
-        zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
-        shell_check_oom(zEQP);
-        rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+        pExplain = pStmt;
+        sqlite3_reset(pExplain);
+        rc = sqlite3_stmt_explain(pExplain, 2);
         if( rc==SQLITE_OK ){
+          bind_prepared_stmt(pArg, pExplain);
           while( sqlite3_step(pExplain)==SQLITE_ROW ){
             const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
             int iEqpId = sqlite3_column_int(pExplain, 0);
@@ -4324,36 +4766,32 @@ static int shell_exec(
           }
           eqp_render(pArg, 0);
         }
-        sqlite3_finalize(pExplain);
-        sqlite3_free(zEQP);
         if( pArg->autoEQP>=AUTOEQP_full ){
           /* Also do an EXPLAIN for ".eqp full" mode */
-          zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql);
-          shell_check_oom(zEQP);
-          rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
+          sqlite3_reset(pExplain);
+          rc = sqlite3_stmt_explain(pExplain, 1);
           if( rc==SQLITE_OK ){
             pArg->cMode = MODE_Explain;
+            assert( sqlite3_stmt_isexplain(pExplain)==1 );
+            bind_prepared_stmt(pArg, pExplain);
             explain_data_prepare(pArg, pExplain);
             exec_prepared_stmt(pArg, pExplain);
             explain_data_delete(pArg);
           }
-          sqlite3_finalize(pExplain);
-          sqlite3_free(zEQP);
         }
         if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){
           sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0);
-          /* Reprepare pStmt before reactiving trace modes */
-          sqlite3_finalize(pStmt);
-          sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-          if( pArg ) pArg->pStmt = pStmt;
         }
+        sqlite3_reset(pStmt);
+        sqlite3_stmt_explain(pStmt, 0);
         restore_debug_trace_modes();
       }
 
       if( pArg ){
+        int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1);
         pArg->cMode = pArg->mode;
         if( pArg->autoExplain ){
-          if( sqlite3_stmt_isexplain(pStmt)==1 ){
+          if( bIsExplain ){
             pArg->cMode = MODE_Explain;
           }
           if( sqlite3_stmt_isexplain(pStmt)==2 ){
@@ -4363,7 +4801,7 @@ static int shell_exec(
 
         /* If the shell is currently in ".explain" mode, gather the extra
         ** data required to add indents to the output.*/
-        if( pArg->cMode==MODE_Explain ){
+        if( pArg->cMode==MODE_Explain && bIsExplain ){
           explain_data_prepare(pArg, pStmt);
         }
       }
@@ -4435,7 +4873,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
   sqlite3_stmt *pStmt;
   char *zSql;
   int nCol = 0;
-  int nAlloc = 0;
+  i64 nAlloc = 0;
   int nPK = 0;       /* Number of PRIMARY KEY columns seen */
   int isIPK = 0;     /* True if one PRIMARY KEY column of type INTEGER */
   int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid);
@@ -4449,7 +4887,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
   while( sqlite3_step(pStmt)==SQLITE_ROW ){
     if( nCol>=nAlloc-2 ){
       nAlloc = nAlloc*2 + nCol + 10;
-      azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0]));
+      azCol = sqlite3_realloc64(azCol, nAlloc*sizeof(azCol[0]));
       shell_check_oom(azCol);
     }
     azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
@@ -4479,7 +4917,7 @@ static char **tableColumnList(ShellState *p, const char *zTab){
   */
   if( preserveRowid && isIPK ){
     /* If a single PRIMARY KEY column with type INTEGER was seen, then it
-    ** might be an alise for the ROWID.  But it might also be a WITHOUT ROWID
+    ** might be an alias for the ROWID.  But it might also be a WITHOUT ROWID
     ** table or a INTEGER PRIMARY KEY DESC column, neither of which are
     ** ROWID aliases.  To distinguish these cases, check to see if
     ** there is a "pk" entry in "PRAGMA index_list".  There will be
@@ -4538,6 +4976,9 @@ static void toggleSelectOrder(sqlite3 *db){
   sqlite3_exec(db, zStmt, 0, 0, 0);
 }
 
+/* Forward reference */
+static int db_int(sqlite3 *db, const char *zSql, ...);
+
 /*
 ** This is a different callback routine used for dumping the database.
 ** Each row received by this callback consists of a table name,
@@ -4564,9 +5005,23 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
   noSys    = (p->shellFlgs & SHFLG_DumpNoSys)!=0;
 
   if( cli_strcmp(zTable, "sqlite_sequence")==0 && !noSys ){
-    if( !dataOnly ) raw_printf(p->out, "DELETE FROM sqlite_sequence;\n");
+    /* The sqlite_sequence table is repopulated last.  Delete content
+    ** in the sqlite_sequence table added by prior repopulations prior to
+    ** repopulating sqlite_sequence itself.  But only do this if the
+    ** table is non-empty, because if it is empty the table might not
+    ** have been recreated by prior repopulations. See forum posts:
+    ** 2024-10-13T17:10:01z and 2025-10-29T19:38:43z
+    */
+    if( db_int(p->db, "SELECT count(*) FROM sqlite_sequence")>0 ){
+      if( !p->writableSchema ){
+        sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out);
+        p->writableSchema = 1;
+      }
+      sqlite3_fputs("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n"
+                    "DELETE FROM sqlite_sequence;\n", p->out);
+    }
   }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){
-    if( !dataOnly ) raw_printf(p->out, "ANALYZE sqlite_schema;\n");
+    if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
   }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){
     return 0;
   }else if( dataOnly ){
@@ -4574,7 +5029,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
   }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
     char *zIns;
     if( !p->writableSchema ){
-      raw_printf(p->out, "PRAGMA writable_schema=ON;\n");
+      sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out);
       p->writableSchema = 1;
     }
     zIns = sqlite3_mprintf(
@@ -4582,7 +5037,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
        "VALUES('table','%q','%q',0,'%q');",
        zTable, zTable, zSql);
     shell_check_oom(zIns);
-    utf8_printf(p->out, "%s\n", zIns);
+    sqlite3_fprintf(p->out, "%s\n", zIns);
     sqlite3_free(zIns);
     return 0;
   }else{
@@ -4640,13 +5095,13 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
 
     savedDestTable = p->zDestTable;
     savedMode = p->mode;
-    p->zDestTable = sTable.z;
+    p->zDestTable = sTable.zTxt;
     p->mode = p->cMode = MODE_Insert;
-    rc = shell_exec(p, sSelect.z, 0);
+    rc = shell_exec(p, sSelect.zTxt, 0);
     if( (rc&0xff)==SQLITE_CORRUPT ){
-      raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n");
+      sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
       toggleSelectOrder(p->db);
-      shell_exec(p, sSelect.z, 0);
+      shell_exec(p, sSelect.zTxt, 0);
       toggleSelectOrder(p->db);
     }
     p->zDestTable = savedDestTable;
@@ -4675,9 +5130,9 @@ static int run_schema_dump_query(
   if( rc==SQLITE_CORRUPT ){
     char *zQ2;
     int len = strlen30(zQuery);
-    raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n");
+    sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
     if( zErr ){
-      utf8_printf(p->out, "/****** %s ******/\n", zErr);
+      sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr);
       sqlite3_free(zErr);
       zErr = 0;
     }
@@ -4686,13 +5141,13 @@ static int run_schema_dump_query(
     sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery);
     rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr);
     if( rc ){
-      utf8_printf(p->out, "/****** ERROR: %s ******/\n", zErr);
+      sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr);
     }else{
       rc = SQLITE_CORRUPT;
     }
-    sqlite3_free(zErr);
     free(zQ2);
   }
+  sqlite3_free(zErr);
   return rc;
 }
 
@@ -4740,7 +5195,6 @@ static const char *(azHelp[]) = {
   "       --async             Write to FILE without journal and fsync()",
 #endif
   ".bail on|off             Stop after hitting an error.  Default OFF",
-  ".binary on|off           Turn binary output on or off.  Default OFF",
 #ifndef SQLITE_SHELL_FIDDLE
   ".cd DIRECTORY            Change the working directory to DIRECTORY",
 #endif
@@ -4750,11 +5204,13 @@ static const char *(azHelp[]) = {
   ".clone NEWDB             Clone data into NEWDB from the existing database",
 #endif
   ".connection [close] [#]  Open or close an auxiliary database connection",
+  ".crlf ?on|off?           Whether or not to use \\r\\n line endings",
   ".databases               List names and files of attached databases",
   ".dbconfig ?op? ?val?     List or change sqlite3_db_config() options",
 #if SQLITE_SHELL_HAVE_RECOVER
   ".dbinfo ?DB?             Show status information about the database",
 #endif
+  ".dbtotxt                 Hex dump of the database file",
   ".dump ?OBJECTS?          Render database content as SQL",
   "   Options:",
   "     --data-only            Output only INSERT statements",
@@ -4778,7 +5234,9 @@ static const char *(azHelp[]) = {
 #ifndef SQLITE_SHELL_FIDDLE
   ".exit ?CODE?             Exit this program with return-code CODE",
 #endif
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
   ".expert                  EXPERIMENTAL. Suggest indexes for queries",
+#endif
   ".explain ?on|off|auto?   Change the EXPLAIN formatting mode.  Default: auto",
   ".filectrl CMD ...        Run various sqlite3_file_control() operations",
   "   --schema SCHEMA         Use SCHEMA instead of \"main\"",
@@ -4803,11 +5261,12 @@ static const char *(azHelp[]) = {
   "        input text.",
 #endif
 #ifndef SQLITE_OMIT_TEST_CONTROL
-  ",imposter INDEX TABLE    Create imposter table TABLE on index INDEX",
+  ".imposter INDEX TABLE    Create imposter table TABLE on index INDEX",
 #endif
   ".indexes ?TABLE?         Show names of indexes",
   "                           If TABLE is specified, only show indexes for",
   "                           tables matching TABLE using the LIKE operator.",
+  ".intck ?STEPS_PER_UNLOCK?  Run an incremental integrity check on the db",
 #ifdef SQLITE_ENABLE_IOTRACE
   ",iotrace FILE            Enable I/O diagnostic logging to FILE",
 #endif
@@ -4823,7 +5282,7 @@ static const char *(azHelp[]) = {
 #else
   ".log on|off              Turn logging on or off.",
 #endif
-  ".mode MODE ?OPTIONS?     Set output mode",
+  ".mode ?MODE? ?OPTIONS?   Set output mode",
   "   MODE is one of:",
   "     ascii       Columns/rows delimited by 0x1F and 0x1E",
   "     box         Tables using unicode box-drawing characters",
@@ -4841,6 +5300,7 @@ static const char *(azHelp[]) = {
   "     tabs        Tab-separated values",
   "     tcl         TCL list elements",
   "   OPTIONS: (for columnar modes or insert mode):",
+  "     --escape T     ctrl-char escape; T is one of: symbol, ascii, off",
   "     --wrap N       Wrap output lines to no longer than N characters",
   "     --wordwrap B   Wrap or not at word boundaries per B (on/off)",
   "     --ww           Shorthand for \"--wordwrap 1\"",
@@ -4854,9 +5314,11 @@ static const char *(azHelp[]) = {
 #ifndef SQLITE_SHELL_FIDDLE
   ".once ?OPTIONS? ?FILE?   Output for the next SQL command only to FILE",
   "     If FILE begins with '|' then open as a pipe",
-  "       --bom  Put a UTF8 byte-order mark at the beginning",
-  "       -e     Send output to the system text editor",
-  "       -x     Send output as CSV to a spreadsheet (same as \".excel\")",
+  "       --bom    Put a UTF8 byte-order mark at the beginning",
+  "       -e       Send output to the system text editor",
+  "       --plain  Use text/plain output instead of HTML for -w option",
+  "       -w       Send output as HTML to a web browser (same as \".www\")",
+  "       -x       Send output as CSV to a spreadsheet (same as \".excel\")",
   /* Note that .open is (partially) available in WASM builds but is
   ** currently only intended to be used by the fiddle tool, not
   ** end users, so is "undocumented." */
@@ -4866,19 +5328,29 @@ static const char *(azHelp[]) = {
 #endif
 #ifndef SQLITE_OMIT_DESERIALIZE
   "        --deserialize   Load into memory using sqlite3_deserialize()",
+#endif
+/*"        --exclusive     Set the SQLITE_OPEN_EXCLUSIVE flag", UNDOCUMENTED */
+#ifndef SQLITE_OMIT_DESERIALIZE
   "        --hexdb         Load the output of \"dbtotxt\" as an in-memory db",
+#endif
+  "        --ifexist       Only open if FILE already exists",
+#ifndef SQLITE_OMIT_DESERIALIZE
   "        --maxsize N     Maximum size for --hexdb or --deserialized database",
 #endif
   "        --new           Initialize FILE to an empty database",
+  "        --normal        FILE is an ordinary SQLite database",
   "        --nofollow      Do not follow symbolic links",
   "        --readonly      Open FILE readonly",
   "        --zip           FILE is a ZIP archive",
 #ifndef SQLITE_SHELL_FIDDLE
   ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
   "   If FILE begins with '|' then open it as a pipe.",
+  "   If FILE is 'off' then output is disabled.",
   "   Options:",
   "     --bom                 Prefix output with a UTF8 byte-order mark",
   "     -e                    Send output to the system text editor",
+  "     --plain               Use text/plain for -w option",
+  "     -w                    Send output to a web browser",
   "     -x                    Send output as CSV to a spreadsheet",
 #endif
   ".parameter CMD ...       Manage SQL parameter bindings",
@@ -4992,111 +5464,116 @@ static const char *(azHelp[]) = {
   ".vfsname ?AUX?           Print the name of the VFS stack",
   ".width NUM1 NUM2 ...     Set minimum column widths for columnar output",
   "     Negative values right-justify",
+#ifndef SQLITE_SHELL_FIDDLE
+  ".www                     Display output of the next command in web browser",
+  "    --plain                 Show results as text/plain, not as HTML",
+#endif
 };
 
 /*
-** Output help text.
+** Output help text for commands that match zPattern.
+**
+**    *   If zPattern is NULL, then show all documented commands, but
+**        only give a one-line summary of each.
+**
+**    *   If zPattern is "-a" or "-all" or "--all" then show all help text
+**        for all commands except undocumented commands.
 **
-** zPattern describes the set of commands for which help text is provided.
-** If zPattern is NULL, then show all commands, but only give a one-line
-** description of each.
+**    *   If zPattern is "0" then show all help for undocumented commands.
+**        Undocumented commands begin with "," instead of "." in the azHelp[]
+**        array.
 **
-** Return the number of matches.
+**    *   If zPattern is a prefix for one or more documented commands, then
+**        show help for those commands.  If only a single command matches the
+**        prefix, show the full text of the help.  If multiple commands match,
+**        Only show just the first line of each.
+**
+**    *   Otherwise, show the complete text of any documented command for which
+**        zPattern is a LIKE match for any text within that command help
+**        text.
+**
+** Return the number commands that match zPattern.
 */
 static int showHelp(FILE *out, const char *zPattern){
   int i = 0;
   int j = 0;
   int n = 0;
   char *zPat;
-  if( zPattern==0
-   || zPattern[0]=='0'
-   || cli_strcmp(zPattern,"-a")==0
-   || cli_strcmp(zPattern,"-all")==0
-   || cli_strcmp(zPattern,"--all")==0
+  if( zPattern==0 ){
+    /* Show just the first line for all help topics */
+    zPattern = "[a-z]";
+  }else if( cli_strcmp(zPattern,"-a")==0
+         || cli_strcmp(zPattern,"-all")==0
+         || cli_strcmp(zPattern,"--all")==0
   ){
-    enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 };
-    enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 };
-    /* Show all or most commands
-    ** *zPattern==0   => summary of documented commands only
-    ** *zPattern=='0' => whole help for undocumented commands
-    ** Otherwise      => whole help for documented commands
-    */
-    enum HelpWanted hw = HW_SummaryOnly;
-    enum HelpHave hh = HH_More;
-    if( zPattern!=0 ){
-      hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull;
-    }
-    for(i=0; ipAuxDb->zDbFilename;
   unsigned int x[16];
   char zLine[1000];
   if( zDbFilename ){
-    in = fopen(zDbFilename, "r");
+    in = sqlite3_fopen(zDbFilename, "r");
     if( in==0 ){
-      utf8_printf(stderr, "cannot open \"%s\" for reading\n", zDbFilename);
+      sqlite3_fprintf(stderr,"cannot open \"%s\" for reading\n", zDbFilename);
       return 0;
     }
     nLine = 0;
@@ -5270,20 +5763,25 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
   }
   *pnData = 0;
   nLine++;
-  if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
+  if( sqlite3_fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error;
   rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz);
   if( rc!=2 ) goto readHexDb_error;
   if( n<0 ) goto readHexDb_error;
-  if( pgsz<512 || pgsz>65536 || (pgsz&(pgsz-1))!=0 ) goto readHexDb_error;
-  n = (n+pgsz-1)&~(pgsz-1);  /* Round n up to the next multiple of pgsz */
-  a = sqlite3_malloc( n ? n : 1 );
-  shell_check_oom(a);
-  memset(a, 0, n);
   if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){
-    utf8_printf(stderr, "invalid pagesize\n");
+    sqlite3_fputs("invalid pagesize\n", stderr);
     goto readHexDb_error;
   }
-  for(nLine++; fgets(zLine, sizeof(zLine), in)!=0; nLine++){
+  sz = ((i64)n+pgsz-1)&~(pgsz-1); /* Round up to nearest multiple of pgsz */
+  a = sqlite3_malloc64( sz ? sz : 1 );
+  shell_check_oom(a);
+  memset(a, 0, sz);
+  for(nLine++; sqlite3_fgets(zLine, sizeof(zLine), in)!=0; nLine++){
+    int j = 0;                    /* Page number from "| page" line */
+    int k = 0;                    /* Offset from "| page" line */
+    if( nLine>=2000000000 ){
+      sqlite3_fprintf(stderr, "input too big\n");
+      goto readHexDb_error;
+    }
     rc = sscanf(zLine, "| page %d offset %d", &j, &k);
     if( rc==2 ){
       iOffset = k;
@@ -5296,14 +5794,14 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
                 &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7],
                 &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]);
     if( rc==17 ){
-      k = iOffset+j;
-      if( k+16<=n && k>=0 ){
+      i64 iOff = iOffset+j;
+      if( iOff+16<=sz && iOff>=0 ){
         int ii;
-        for(ii=0; ii<16; ii++) a[k+ii] = x[ii]&0xff;
+        for(ii=0; ii<16; ii++) a[iOff+ii] = x[ii]&0xff;
       }
     }
   }
-  *pnData = n;
+  *pnData = sz;
   if( in!=p->in ){
     fclose(in);
   }else{
@@ -5315,14 +5813,14 @@ readHexDb_error:
   if( in!=p->in ){
     fclose(in);
   }else{
-    while( fgets(zLine, sizeof(zLine), p->in)!=0 ){
+    while( sqlite3_fgets(zLine, sizeof(zLine), p->in)!=0 ){
       nLine++;
       if(cli_strncmp(zLine, "| end ", 6)==0 ) break;
     }
     p->lineno = nLine;
   }
   sqlite3_free(a);
-  utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine);
+  sqlite3_fprintf(stderr,"Error on line %lld of --hexdb input\n", nLine);
   return 0;
 }
 #endif /* SQLITE_OMIT_DESERIALIZE */
@@ -5341,6 +5839,39 @@ static void shellUSleepFunc(
   sqlite3_result_int(context, sleep);
 }
 
+/*
+** SQL function:  shell_module_schema(X)
+**
+** Return a fake schema for the table-valued function or eponymous virtual
+** table X.
+*/
+static void shellModuleSchema(
+  sqlite3_context *pCtx,
+  int nVal,
+  sqlite3_value **apVal
+){
+  const char *zName;
+  char *zFake;
+  ShellState *p = (ShellState*)sqlite3_user_data(pCtx);
+  FILE *pSavedLog = p->pLog;
+  UNUSED_PARAMETER(nVal);
+  zName = (const char*)sqlite3_value_text(apVal[0]);
+
+  /* Temporarily disable the ".log" when calling shellFakeSchema() because
+  ** shellFakeSchema() might generate failures for some ephemeral virtual
+  ** tables due to missing arguments.  Example: fts4aux.
+  ** https://sqlite.org/forum/forumpost/42fe6520b803be51 */
+  p->pLog = 0;
+  zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0;
+  p->pLog = pSavedLog;
+
+  if( zFake ){
+    sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake),
+                        -1, sqlite3_free);
+    sqlite3_free(zFake);
+  }
+}
+
 /* Flags for open_db().
 **
 ** The default behavior of open_db() is to exit(1) if the database fails to
@@ -5366,13 +5897,16 @@ static void open_db(ShellState *p, int openFlags){
         p->openMode = SHELL_OPEN_NORMAL;
       }else{
         p->openMode = (u8)deduceDatabaseType(zDbFilename,
-                             (openFlags & OPEN_DB_ZIPFILE)!=0);
+                             (openFlags & OPEN_DB_ZIPFILE)!=0, p->openFlags);
       }
     }
+    if( (p->openFlags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE))==0 ){
+      if( p->openFlags==0 ) p->openFlags = SQLITE_OPEN_CREATE;
+      p->openFlags |= SQLITE_OPEN_READWRITE;
+    }
     switch( p->openMode ){
       case SHELL_OPEN_APPENDVFS: {
-        sqlite3_open_v2(zDbFilename, &p->db,
-           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, "apndvfs");
+        sqlite3_open_v2(zDbFilename, &p->db, p->openFlags, "apndvfs");
         break;
       }
       case SHELL_OPEN_HEXDB:
@@ -5384,38 +5918,31 @@ static void open_db(ShellState *p, int openFlags){
         sqlite3_open(":memory:", &p->db);
         break;
       }
-      case SHELL_OPEN_READONLY: {
-        sqlite3_open_v2(zDbFilename, &p->db,
-            SQLITE_OPEN_READONLY|p->openFlags, 0);
-        break;
-      }
       case SHELL_OPEN_UNSPEC:
       case SHELL_OPEN_NORMAL: {
-        sqlite3_open_v2(zDbFilename, &p->db,
-           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|p->openFlags, 0);
+        sqlite3_open_v2(zDbFilename, &p->db, p->openFlags, 0);
         break;
       }
     }
-    globalDb = p->db;
     if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
-      utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n",
-          zDbFilename, sqlite3_errmsg(p->db));
+      sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n",
+            zDbFilename, sqlite3_errmsg(p->db));
       if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){
         exit(1);
       }
       sqlite3_close(p->db);
       sqlite3_open(":memory:", &p->db);
       if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
-        utf8_printf(stderr,
-          "Also: unable to open substitute in-memory database.\n"
-        );
+        sqlite3_fputs("Also: unable to open substitute in-memory database.\n",
+                      stderr);
         exit(1);
       }else{
-        utf8_printf(stderr,
-          "Notice: using substitute in-memory database instead of \"%s\"\n",
-          zDbFilename);
+        sqlite3_fprintf(stderr,
+              "Notice: using substitute in-memory database instead of \"%s\"\n",
+              zDbFilename);
       }
     }
+    globalDb = p->db;
     sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0);
 
     /* Reflect the use or absence of --unsafe-testing invocation. */
@@ -5428,8 +5955,10 @@ static void open_db(ShellState *p, int openFlags){
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
     sqlite3_enable_load_extension(p->db, 1);
 #endif
+    sqlite3_sha_init(p->db, 0, 0);
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_uint_init(p->db, 0, 0);
+    sqlite3_stmtrand_init(p->db, 0, 0);
     sqlite3_decimal_init(p->db, 0, 0);
     sqlite3_base64_init(p->db, 0, 0);
     sqlite3_base85_init(p->db, 0, 0);
@@ -5456,7 +5985,7 @@ static void open_db(ShellState *p, int openFlags){
     /* Let custom-included extensions get their ..._init() called.
      * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause
      * the extension's sqlite3_*_init( db, pzErrorMsg, pApi )
-     * inititialization routine to be called.
+     * initialization routine to be called.
      */
     {
       int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db);
@@ -5474,9 +6003,15 @@ static void open_db(ShellState *p, int openFlags){
     }
 #endif
 
+    sqlite3_create_function(p->db, "strtod", 1, SQLITE_UTF8, 0,
+                            shellStrtod, 0, 0);
+    sqlite3_create_function(p->db, "dtostr", 1, SQLITE_UTF8, 0,
+                            shellDtostr, 0, 0);
+    sqlite3_create_function(p->db, "dtostr", 2, SQLITE_UTF8, 0,
+                            shellDtostr, 0, 0);
     sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
-    sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0,
+    sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, p,
                             shellModuleSchema, 0, 0);
     sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
                             shellPutsFunc, 0, 0);
@@ -5514,7 +6049,7 @@ static void open_db(ShellState *p, int openFlags){
                    SQLITE_DESERIALIZE_RESIZEABLE |
                    SQLITE_DESERIALIZE_FREEONCLOSE);
       if( rc ){
-        utf8_printf(stderr, "Error: sqlite3_deserialize() returns %d\n", rc);
+        sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc);
       }
       if( p->szMax>0 ){
         sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
@@ -5523,9 +6058,11 @@ static void open_db(ShellState *p, int openFlags){
 #endif
   }
   if( p->db!=0 ){
+#ifndef SQLITE_OMIT_AUTHORIZATION
     if( p->bSafeModePersist ){
       sqlite3_set_authorizer(p->db, safeModeAuth, p);
     }
+#endif
     sqlite3_db_config(
         p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0
     );
@@ -5533,17 +6070,18 @@ static void open_db(ShellState *p, int openFlags){
 }
 
 /*
-** Attempt to close the databaes connection.  Report errors.
+** Attempt to close the database connection.  Report errors.
 */
 void close_db(sqlite3 *db){
   int rc = sqlite3_close(db);
   if( rc ){
-    utf8_printf(stderr, "Error: sqlite3_close() returns %d: %s\n",
-        rc, sqlite3_errmsg(db));
+    sqlite3_fprintf(stderr,
+        "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db));
   }
 }
 
-#if HAVE_READLINE || HAVE_EDITLINE
+#if (HAVE_READLINE || HAVE_EDITLINE) \
+  && !defined(SQLITE_OMIT_READLINE_COMPLETION)
 /*
 ** Readline completion callbacks
 */
@@ -5578,18 +6116,28 @@ static char **readline_completion(const char *zText, int iStart, int iEnd){
 
 #elif HAVE_LINENOISE
 /*
-** Linenoise completion callback
+** Linenoise completion callback. Note that the 3rd argument is from
+** the "msteveb" version of linenoise, not the "antirez" version.
 */
-static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){
+static void linenoise_completion(
+  const char *zLine,
+  linenoiseCompletions *lc
+#if HAVE_LINENOISE==2
+  ,void *pUserData
+#endif
+){
   i64 nLine = strlen(zLine);
   i64 i, iStart;
   sqlite3_stmt *pStmt = 0;
   char *zSql;
   char zBuf[1000];
 
+#if HAVE_LINENOISE==2
+  UNUSED_PARAMETER(pUserData);
+#endif
   if( nLine>(i64)sizeof(zBuf)-30 ) return;
   if( zLine[0]=='.' || zLine[0]=='#') return;
-  for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){}
+  for(i=nLine-1; i>=0 && (IsAlnum(zLine[i]) || zLine[i]=='_'); i--){}
   if( i==nLine-1 ) return;
   iStart = i+1;
   memcpy(zBuf, zLine, iStart);
@@ -5700,8 +6248,8 @@ static int booleanValue(const char *zArg){
   if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){
     return 0;
   }
-  utf8_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n",
-          zArg);
+  sqlite3_fprintf(stderr,
+       "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg);
   return 0;
 }
 
@@ -5728,7 +6276,7 @@ static void output_file_close(FILE *f){
 ** recognized and do the right thing.  NULL is returned if the output
 ** filename is "off".
 */
-static FILE *output_file_open(const char *zFile, int bTextMode){
+static FILE *output_file_open(const char *zFile){
   FILE *f;
   if( cli_strcmp(zFile,"stdout")==0 ){
     f = stdout;
@@ -5737,9 +6285,9 @@ static FILE *output_file_open(const char *zFile, int bTextMode){
   }else if( cli_strcmp(zFile, "off")==0 ){
     f = 0;
   }else{
-    f = fopen(zFile, bTextMode ? "w" : "wb");
+    f = sqlite3_fopen(zFile, "w");
     if( f==0 ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
     }
   }
   return f;
@@ -5761,7 +6309,7 @@ static int sql_trace_callback(
   i64 nSql;
   if( p->traceOut==0 ) return 0;
   if( mType==SQLITE_TRACE_CLOSE ){
-    utf8_printf(p->traceOut, "-- closing database connection\n");
+    sputz(p->traceOut, "-- closing database connection\n");
     return 0;
   }
   if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){
@@ -5792,12 +6340,13 @@ static int sql_trace_callback(
   switch( mType ){
     case SQLITE_TRACE_ROW:
     case SQLITE_TRACE_STMT: {
-      utf8_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
+      sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
       break;
     }
     case SQLITE_TRACE_PROFILE: {
       sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0;
-      utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
+      sqlite3_fprintf(p->traceOut,
+                      "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
       break;
     }
   }
@@ -5826,8 +6375,8 @@ struct ImportCtx {
   FILE *in;           /* Read the CSV text from this input stream */
   int (SQLITE_CDECL *xCloser)(FILE*);      /* Func to close in */
   char *z;            /* Accumulated text for a field */
-  int n;              /* Number of bytes in z */
-  int nAlloc;         /* Space allocated for z[] */
+  i64 n;              /* Number of bytes in z */
+  i64 nAlloc;         /* Space allocated for z[] */
   int nLine;          /* Current line number */
   int nRow;           /* Number of rows imported */
   int nErr;           /* Number of errors encountered */
@@ -5904,12 +6453,12 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
         break;
       }
       if( pc==cQuote && c!='\r' ){
-        utf8_printf(stderr, "%s:%d: unescaped %c character\n",
-                p->zFile, p->nLine, cQuote);
+        sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", 
+                        p->zFile, p->nLine, cQuote);
       }
       if( c==EOF ){
-        utf8_printf(stderr, "%s:%d: unterminated %c-quoted field\n",
-                p->zFile, startLine, cQuote);
+        sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n",
+              p->zFile, startLine, cQuote);
         p->cTerm = c;
         break;
       }
@@ -6007,9 +6556,8 @@ static void tryToCloneData(
   shell_check_oom(zQuery);
   rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
   if( rc ){
-    utf8_printf(stderr, "Error %d: %s on [%s]\n",
-            sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
-            zQuery);
+    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
+          sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
     goto end_data_xfer;
   }
   n = sqlite3_column_count(pQuery);
@@ -6025,9 +6573,8 @@ static void tryToCloneData(
   memcpy(zInsert+i, ");", 3);
   rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0);
   if( rc ){
-    utf8_printf(stderr, "Error %d: %s on [%s]\n",
-            sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb),
-            zQuery);
+    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
+          sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert);
     goto end_data_xfer;
   }
   for(k=0; k<2; k++){
@@ -6062,8 +6609,8 @@ static void tryToCloneData(
       } /* End for */
       rc = sqlite3_step(pInsert);
       if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
-        utf8_printf(stderr, "Error %d: %s\n", sqlite3_extended_errcode(newDb),
-                        sqlite3_errmsg(newDb));
+        sqlite3_fprintf(stderr,"Error %d: %s\n",
+              sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb));
       }
       sqlite3_reset(pInsert);
       cnt++;
@@ -6080,7 +6627,7 @@ static void tryToCloneData(
     shell_check_oom(zQuery);
     rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
     if( rc ){
-      utf8_printf(stderr, "Warning: cannot step \"%s\" backwards", zTable);
+      sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable);
       break;
     }
   } /* End for(k=0...) */
@@ -6117,9 +6664,9 @@ static void tryToCloneSchema(
   shell_check_oom(zQuery);
   rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
   if( rc ){
-    utf8_printf(stderr, "Error: (%d) %s on [%s]\n",
-                    sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
-                    zQuery);
+    sqlite3_fprintf(stderr,
+          "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db),
+          sqlite3_errmsg(p->db), zQuery);
     goto end_schema_xfer;
   }
   while( (rc = sqlite3_step(pQuery))==SQLITE_ROW ){
@@ -6127,10 +6674,10 @@ static void tryToCloneSchema(
     zSql = sqlite3_column_text(pQuery, 1);
     if( zName==0 || zSql==0 ) continue;
     if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){
-      printf("%s... ", zName); fflush(stdout);
+      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
       sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
       if( zErrMsg ){
-        utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
+        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
         sqlite3_free(zErrMsg);
         zErrMsg = 0;
       }
@@ -6138,7 +6685,7 @@ static void tryToCloneSchema(
     if( xForEach ){
       xForEach(p, newDb, (const char*)zName);
     }
-    printf("done\n");
+    sputz(stdout, "done\n");
   }
   if( rc!=SQLITE_DONE ){
     sqlite3_finalize(pQuery);
@@ -6148,9 +6695,8 @@ static void tryToCloneSchema(
     shell_check_oom(zQuery);
     rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
     if( rc ){
-      utf8_printf(stderr, "Error: (%d) %s on [%s]\n",
-                      sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db),
-                      zQuery);
+      sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n",
+            sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
       goto end_schema_xfer;
     }
     while( sqlite3_step(pQuery)==SQLITE_ROW ){
@@ -6158,17 +6704,17 @@ static void tryToCloneSchema(
       zSql = sqlite3_column_text(pQuery, 1);
       if( zName==0 || zSql==0 ) continue;
       if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue;
-      printf("%s... ", zName); fflush(stdout);
+      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
       sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
       if( zErrMsg ){
-        utf8_printf(stderr, "Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
+        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
         sqlite3_free(zErrMsg);
         zErrMsg = 0;
       }
       if( xForEach ){
         xForEach(p, newDb, (const char*)zName);
       }
-      printf("done\n");
+      sputz(stdout, "done\n");
     }
   }
 end_schema_xfer:
@@ -6185,13 +6731,13 @@ static void tryToClone(ShellState *p, const char *zNewDb){
   int rc;
   sqlite3 *newDb = 0;
   if( access(zNewDb,0)==0 ){
-    utf8_printf(stderr, "File \"%s\" already exists.\n", zNewDb);
+    sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb);
     return;
   }
   rc = sqlite3_open(zNewDb, &newDb);
   if( rc ){
-    utf8_printf(stderr, "Cannot create output database: %s\n",
-            sqlite3_errmsg(newDb));
+    sqlite3_fprintf(stderr,
+        "Cannot create output database: %s\n", sqlite3_errmsg(newDb));
   }else{
     sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0);
     sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0);
@@ -6203,6 +6749,26 @@ static void tryToClone(ShellState *p, const char *zNewDb){
   close_db(newDb);
 }
 
+#ifndef SQLITE_SHELL_FIDDLE
+/*
+** Change the output stream (file or pipe or console) to something else.
+*/
+static void output_redir(ShellState *p, FILE *pfNew){
+  if( p->out != stdout ){
+    sqlite3_fputs("Output already redirected.\n", stderr);
+  }else{
+    p->out = pfNew;
+    setCrlfMode(p);
+    if( p->mode==MODE_Www ){
+      sqlite3_fputs(
+        "\n"
+        "
    \n",
+        p->out
+      );
+    }
+  }
+}
+
 /*
 ** Change the output file back to stdout.
 **
@@ -6216,6 +6782,9 @@ static void output_reset(ShellState *p){
     pclose(p->out);
 #endif
   }else{
+    if( p->mode==MODE_Www ){
+      sqlite3_fputs("
    \n", p->out);
+    }
     output_file_close(p->out);
 #ifndef SQLITE_NOHAVE_SYSTEM
     if( p->doXdgOpen ){
@@ -6230,7 +6799,7 @@ static void output_reset(ShellState *p){
       char *zCmd;
       zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile);
       if( system(zCmd) ){
-        utf8_printf(stderr, "Failed: [%s]\n", zCmd);
+        sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd);
       }else{
         /* Give the start/open/xdg-open command some time to get
         ** going before we continue, and potential delete the
@@ -6245,19 +6814,30 @@ static void output_reset(ShellState *p){
   }
   p->outfile[0] = 0;
   p->out = stdout;
+  setCrlfMode(p);
 }
+#else
+# define output_redir(SS,pfO)
+# define output_reset(SS)
+#endif
 
 /*
 ** Run an SQL command and return the single integer result.
 */
-static int db_int(sqlite3 *db, const char *zSql){
+static int db_int(sqlite3 *db, const char *zSql, ...){
   sqlite3_stmt *pStmt;
   int res = 0;
-  sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  char *z;
+  va_list ap;
+  va_start(ap, zSql);
+  z = sqlite3_vmprintf(zSql, ap);
+  va_end(ap);
+  sqlite3_prepare_v2(db, z, -1, &pStmt, 0);
   if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
     res = sqlite3_column_int(pStmt,0);
   }
   sqlite3_finalize(pStmt);
+  sqlite3_free(z);
   return res;
 }
 
@@ -6266,10 +6846,13 @@ static int db_int(sqlite3 *db, const char *zSql){
 ** Convert a 2-byte or 4-byte big-endian integer into a native integer
 */
 static unsigned int get2byteInt(unsigned char *a){
-  return (a[0]<<8) + a[1];
+  return ((unsigned int)a[0]<<8) + (unsigned int)a[1];
 }
 static unsigned int get4byteInt(unsigned char *a){
-  return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3];
+  return ((unsigned int)a[0]<<24)
+       + ((unsigned int)a[1]<<16)
+       + ((unsigned int)a[2]<<8)
+       + (unsigned int)a[3];
 }
 
 /*
@@ -6316,7 +6899,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
              "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1",
              -1, &pStmt, 0);
   if( rc ){
-    utf8_printf(stderr, "error: %s\n", sqlite3_errmsg(p->db));
+    sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db));
     sqlite3_finalize(pStmt);
     return 1;
   }
@@ -6329,28 +6912,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
     memcpy(aHdr, pb, 100);
     sqlite3_finalize(pStmt);
   }else{
-    raw_printf(stderr, "unable to read database header\n");
+    sqlite3_fputs("unable to read database header\n", stderr);
     sqlite3_finalize(pStmt);
     return 1;
   }
   i = get2byteInt(aHdr+16);
   if( i==1 ) i = 65536;
-  utf8_printf(p->out, "%-20s %d\n", "database page size:", i);
-  utf8_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]);
-  utf8_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]);
-  utf8_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]);
+  sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]);
   for(i=0; iout, "%-20s %u", aField[i].zName, val);
+    sqlite3_fprintf(p->out, "%-20s %u", aField[i].zName, val);
     switch( ofst ){
       case 56: {
-        if( val==1 ) raw_printf(p->out, " (utf8)");
-        if( val==2 ) raw_printf(p->out, " (utf16le)");
-        if( val==3 ) raw_printf(p->out, " (utf16be)");
+        if( val==1 ) sqlite3_fputs(" (utf8)", p->out);
+        if( val==2 ) sqlite3_fputs(" (utf16le)", p->out);
+        if( val==3 ) sqlite3_fputs(" (utf16be)", p->out);
       }
     }
-    raw_printf(p->out, "\n");
+    sqlite3_fputs("\n", p->out);
   }
   if( zDb==0 ){
     zSchemaTab = sqlite3_mprintf("main.sqlite_schema");
@@ -6360,24 +6943,125 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
     zSchemaTab = sqlite3_mprintf("\"%w\".sqlite_schema", zDb);
   }
   for(i=0; idb, zSql);
-    sqlite3_free(zSql);
-    utf8_printf(p->out, "%-20s %d\n", aQuery[i].zName, val);
+    int val = db_int(p->db, aQuery[i].zSql, zSchemaTab);
+    sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val);
   }
   sqlite3_free(zSchemaTab);
   sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion);
-  utf8_printf(p->out, "%-20s %u\n", "data version", iDataVersion);
+  sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion);
   return 0;
 }
 #endif /* SQLITE_SHELL_HAVE_RECOVER */
 
+/*
+** Implementation of the ".dbtotxt" command.
+**
+** Return 1 on error, 2 to exit, and 0 otherwise.
+*/
+static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){
+  sqlite3_stmt *pStmt = 0;
+  sqlite3_int64 nPage = 0;
+  int pgSz = 0;
+  const char *zTail;
+  char *zName = 0;
+  int rc, i, j;
+  unsigned char bShow[256];   /* Characters ok to display */
+
+  UNUSED_PARAMETER(nArg);
+  UNUSED_PARAMETER(azArg);
+  memset(bShow, '.', sizeof(bShow));
+  for(i=' '; i<='~'; i++){
+    if( i!='{' && i!='}' && i!='"' && i!='\\' ) bShow[i] = (unsigned char)i;
+  }
+  rc = sqlite3_prepare_v2(p->db, "PRAGMA page_size", -1, &pStmt, 0);
+  if( rc ) goto dbtotxt_error;
+  rc = 0;
+  if( sqlite3_step(pStmt)!=SQLITE_ROW ) goto dbtotxt_error;
+  pgSz = sqlite3_column_int(pStmt, 0);
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+  if( pgSz<512 || pgSz>65536 || (pgSz&(pgSz-1))!=0 ) goto dbtotxt_error;
+  rc = sqlite3_prepare_v2(p->db, "PRAGMA page_count", -1, &pStmt, 0);
+  if( rc ) goto dbtotxt_error;
+  rc = 0;
+  if( sqlite3_step(pStmt)!=SQLITE_ROW ) goto dbtotxt_error;
+  nPage = sqlite3_column_int64(pStmt, 0);
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+  if( nPage<1 ) goto dbtotxt_error;
+  rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
+  if( rc ) goto dbtotxt_error;
+  if( sqlite3_step(pStmt)!=SQLITE_ROW ){
+    zTail = "unk.db";
+  }else{
+    const char *zFilename = (const char*)sqlite3_column_text(pStmt, 2);
+    if( zFilename==0 || zFilename[0]==0 ) zFilename = "unk.db";
+    zTail = strrchr(zFilename, '/');
+#if defined(_WIN32)
+    if( zTail==0 ) zTail = strrchr(zFilename, '\\');
+#endif
+    if( zTail==0 ){
+      zTail = zFilename;
+    }else if( zTail[1]!=0 ){
+      zTail++;
+    }
+  }
+  zName = strdup(zTail);
+  shell_check_oom(zName);
+  sqlite3_fprintf(p->out, "| size %lld pagesize %d filename %s\n",
+                  nPage*pgSz, pgSz, zName);
+  sqlite3_finalize(pStmt);
+  pStmt = 0;
+  rc = sqlite3_prepare_v2(p->db,
+           "SELECT pgno, data FROM sqlite_dbpage ORDER BY pgno", -1, &pStmt, 0);
+  if( rc ) goto dbtotxt_error;
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    sqlite3_int64 pgno = sqlite3_column_int64(pStmt, 0);
+    const u8 *aData = sqlite3_column_blob(pStmt, 1);
+    int seenPageLabel = 0;
+    for(i=0; iout, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz);
+        seenPageLabel = 1;
+      }
+      sqlite3_fprintf(p->out, "|  %5d:", i);
+      for(j=0; j<16; j++) sqlite3_fprintf(p->out, " %02x", aLine[j]);
+      sqlite3_fprintf(p->out, "   ");
+      for(j=0; j<16; j++){
+        unsigned char c = (unsigned char)aLine[j];
+        sqlite3_fprintf(p->out, "%c", bShow[c]);
+      }
+      sqlite3_fprintf(p->out, "\n");
+    }
+  }
+  sqlite3_finalize(pStmt);
+  sqlite3_fprintf(p->out, "| end %s\n", zName);
+  free(zName);
+  return 0;
+
+dbtotxt_error:
+  if( rc ){
+    sqlite3_fprintf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db));
+  }
+  sqlite3_finalize(pStmt);
+  free(zName);
+  return 1;
+}
+
+/*
+** Print the given string as an error message.
+*/
+static void shellEmitError(const char *zErr){
+  sqlite3_fprintf(stderr,"Error: %s\n", zErr);
+}
 /*
 ** Print the current sqlite3_errmsg() value to stderr and return 1.
 */
 static int shellDatabaseError(sqlite3 *db){
-  const char *zErr = sqlite3_errmsg(db);
-  utf8_printf(stderr, "Error: %s\n", zErr);
+  shellEmitError(sqlite3_errmsg(db));
   return 1;
 }
 
@@ -6490,6 +7174,43 @@ static int optionMatch(const char *zStr, const char *zOpt){
   return cli_strcmp(zStr, zOpt)==0;
 }
 
+/*
+** The input zFN is guaranteed to start with "file:" and is thus a URI
+** filename.  Extract the actual filename and return a pointer to that
+** filename in spaced obtained from sqlite3_malloc().
+**
+** The caller is responsible for freeing space using sqlite3_free() when
+** it has finished with the filename.
+*/
+static char *shellFilenameFromUri(const char *zFN){
+  char *zOut;
+  int i, j, d1, d2;
+
+  assert( cli_strncmp(zFN,"file:",5)==0 );
+  zOut = sqlite3_mprintf("%s", zFN+5);
+  shell_check_oom(zOut);
+  for(i=j=0; zOut[i]!=0 && zOut[i]!='?'; i++){
+    if( zOut[i]!='%' ){
+      zOut[j++] = zOut[i];
+      continue;
+    }
+    d1 = hexDigitValue(zOut[i+1]);
+    if( d1<0 ){
+      zOut[j] = 0;
+      break;
+    }
+    d2 = hexDigitValue(zOut[i+2]);
+    if( d2<0 ){
+      zOut[j] = 0;
+      break;
+    }
+    zOut[j++] = d1*16 + d2;
+    i += 2;
+  }
+  zOut[j] = 0;
+  return zOut;
+}
+
 /*
 ** Delete a file.
 */
@@ -6612,13 +7333,13 @@ static int lintFkeyIndexes(
   int nArg                        /* Number of entries in azArg[] */
 ){
   sqlite3 *db = pState->db;       /* Database handle to query "main" db of */
-  FILE *out = pState->out;        /* Stream to write non-error output to */
   int bVerbose = 0;               /* If -verbose is present */
   int bGroupByParent = 0;         /* If -groupbyparent is present */
   int i;                          /* To iterate through azArg[] */
   const char *zIndent = "";       /* How much to indent CREATE INDEX by */
   int rc;                         /* Return code */
   sqlite3_stmt *pSql = 0;         /* Compiled version of SQL statement below */
+  FILE *out = pState->out;        /* Send output here */
 
   /*
   ** This SELECT statement returns one row for each foreign key constraint
@@ -6694,9 +7415,8 @@ static int lintFkeyIndexes(
       zIndent = "    ";
     }
     else{
-      raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n",
-          azArg[0], azArg[1]
-      );
+      sqlite3_fprintf(stderr,
+           "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]);
       return SQLITE_ERROR;
     }
   }
@@ -6740,23 +7460,24 @@ static int lintFkeyIndexes(
       if( rc!=SQLITE_OK ) break;
 
       if( res<0 ){
-        raw_printf(stderr, "Error: internal error");
+        sqlite3_fputs("Error: internal error", stderr);
         break;
       }else{
         if( bGroupByParent
         && (bVerbose || res==0)
         && (zPrev==0 || sqlite3_stricmp(zParent, zPrev))
         ){
-          raw_printf(out, "-- Parent table %s\n", zParent);
+          sqlite3_fprintf(out, "-- Parent table %s\n", zParent);
           sqlite3_free(zPrev);
           zPrev = sqlite3_mprintf("%s", zParent);
         }
 
         if( res==0 ){
-          raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget);
+          sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget);
         }else if( bVerbose ){
-          raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n",
-              zIndent, zFrom, zTarget
+          sqlite3_fprintf(out,
+                "%s/* no extra indexes required for %s -> %s */\n",
+                zIndent, zFrom, zTarget
           );
         }
       }
@@ -6764,16 +7485,16 @@ static int lintFkeyIndexes(
     sqlite3_free(zPrev);
 
     if( rc!=SQLITE_OK ){
-      raw_printf(stderr, "%s\n", sqlite3_errmsg(db));
+      sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
     }
 
     rc2 = sqlite3_finalize(pSql);
     if( rc==SQLITE_OK && rc2!=SQLITE_OK ){
       rc = rc2;
-      raw_printf(stderr, "%s\n", sqlite3_errmsg(db));
+      sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
     }
   }else{
-    raw_printf(stderr, "%s\n", sqlite3_errmsg(db));
+    sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db));
   }
 
   return rc;
@@ -6793,13 +7514,12 @@ static int lintDotCommand(
   return lintFkeyIndexes(pState, azArg, nArg);
 
  usage:
-  raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]);
-  raw_printf(stderr, "Where sub-commands are:\n");
-  raw_printf(stderr, "    fkey-indexes\n");
+  sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]);
+  sqlite3_fprintf(stderr, "Where sub-commands are:\n");
+  sqlite3_fprintf(stderr, "    fkey-indexes\n");
   return SQLITE_ERROR;
 }
 
-#if !defined SQLITE_OMIT_VIRTUALTABLE
 static void shellPrepare(
   sqlite3 *db,
   int *pRc,
@@ -6810,9 +7530,8 @@ static void shellPrepare(
   if( *pRc==SQLITE_OK ){
     int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
     if( rc!=SQLITE_OK ){
-      raw_printf(stderr, "sql error: %s (%d)\n",
-          sqlite3_errmsg(db), sqlite3_errcode(db)
-      );
+      sqlite3_fprintf(stderr,
+         "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db));
       *pRc = rc;
     }
   }
@@ -6820,12 +7539,8 @@ static void shellPrepare(
 
 /*
 ** Create a prepared statement using printf-style arguments for the SQL.
-**
-** This routine is could be marked "static".  But it is not always used,
-** depending on compile-time options.  By omitting the "static", we avoid
-** nuisance compiler warnings about "defined but not used".
 */
-void shellPreparePrintf(
+static void shellPreparePrintf(
   sqlite3 *db,
   int *pRc,
   sqlite3_stmt **ppStmt,
@@ -6848,13 +7563,10 @@ void shellPreparePrintf(
   }
 }
 
-/* Finalize the prepared statement created using shellPreparePrintf().
-**
-** This routine is could be marked "static".  But it is not always used,
-** depending on compile-time options.  By omitting the "static", we avoid
-** nuisance compiler warnings about "defined but not used".
+/* 
+** Finalize the prepared statement created using shellPreparePrintf().
 */
-void shellFinalize(
+static void shellFinalize(
   int *pRc,
   sqlite3_stmt *pStmt
 ){
@@ -6863,13 +7575,14 @@ void shellFinalize(
     int rc = sqlite3_finalize(pStmt);
     if( *pRc==SQLITE_OK ){
       if( rc!=SQLITE_OK ){
-        raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db));
+        sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db));
       }
       *pRc = rc;
     }
   }
 }
 
+#if !defined SQLITE_OMIT_VIRTUALTABLE
 /* Reset the prepared statement created using shellPreparePrintf().
 **
 ** This routine is could be marked "static".  But it is not always used,
@@ -6884,7 +7597,7 @@ void shellReset(
   if( *pRc==SQLITE_OK ){
     if( rc!=SQLITE_OK ){
       sqlite3 *db = sqlite3_db_handle(pStmt);
-      raw_printf(stderr, "SQL error: %s\n", sqlite3_errmsg(db));
+      sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db));
     }
     *pRc = rc;
   }
@@ -6913,6 +7626,7 @@ struct ArCommand {
   const char *zDir;               /* --directory argument, or NULL */
   char **azArg;                   /* Array of command arguments */
   ShellState *p;                  /* Shell state */
+  FILE *out;                      /* Output to this stream */
   sqlite3 *db;                    /* Database containing the archive */
 };
 
@@ -6934,11 +7648,11 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){
   va_start(ap, zFmt);
   z = sqlite3_vmprintf(zFmt, ap);
   va_end(ap);
-  utf8_printf(stderr, "Error: %s\n", z);
+  shellEmitError(z);
   if( pAr->fromCmdLine ){
-    utf8_printf(stderr, "Use \"-A\" for more help\n");
+    sqlite3_fputs("Use \"-A\" for more help\n", stderr);
   }else{
-    utf8_printf(stderr, "Use \".archive --help\" for more help\n");
+    sqlite3_fputs("Use \".archive --help\" for more help\n", stderr);
   }
   sqlite3_free(z);
   return SQLITE_ERROR;
@@ -6991,7 +7705,7 @@ static int arProcessSwitch(ArCommand *pAr, int eSwitch, const char *zArg){
       break;
     case AR_SWITCH_APPEND:
       pAr->bAppend = 1;
-      deliberate_fall_through;
+      deliberate_fall_through; /* FALLTHRU */
     case AR_SWITCH_FILE:
       pAr->zFile = zArg;
       break;
@@ -7038,7 +7752,7 @@ static int arParseCommand(
   struct ArSwitch *pEnd = &aSwitch[nSwitch];
 
   if( nArg<=1 ){
-    utf8_printf(stderr, "Wrong number of arguments.  Usage:\n");
+    sqlite3_fprintf(stderr, "Wrong number of arguments.  Usage:\n");
     return arUsage(stderr);
   }else{
     char *z = azArg[1];
@@ -7144,7 +7858,7 @@ static int arParseCommand(
     }
   }
   if( pAr->eCmd==0 ){
-    utf8_printf(stderr, "Required argument missing.  Usage:\n");
+    sqlite3_fprintf(stderr, "Required argument missing.  Usage:\n");
     return arUsage(stderr);
   }
   return SQLITE_OK;
@@ -7187,7 +7901,7 @@ static int arCheckEntries(ArCommand *pAr){
       }
       shellReset(&rc, pTest);
       if( rc==SQLITE_OK && bOk==0 ){
-        utf8_printf(stderr, "not found in archive: %s\n", z);
+        sqlite3_fprintf(stderr,"not found in archive: %s\n", z);
         rc = SQLITE_ERROR;
       }
     }
@@ -7210,25 +7924,41 @@ static void arWhereClause(
   char **pzWhere                  /* OUT: New WHERE clause */
 ){
   char *zWhere = 0;
-  const char *zSameOp = (pAr->bGlob)? "GLOB" : "=";
   if( *pRc==SQLITE_OK ){
     if( pAr->nArg==0 ){
       zWhere = sqlite3_mprintf("1");
     }else{
+      char *z1 = sqlite3_mprintf(pAr->bGlob ? "" : "name IN(");
+      char *z2 = sqlite3_mprintf("");
+      const char *zSep1 = "";
+      const char *zSep2 = "";
+
       int i;
-      const char *zSep = "";
-      for(i=0; inArg; i++){
+      for(i=0; inArg && z1 && z2; i++){
         const char *z = pAr->azArg[i];
-        zWhere = sqlite3_mprintf(
-          "%z%s name %s '%q' OR substr(name,1,%d) %s '%q/'",
-          zWhere, zSep, zSameOp, z, strlen30(z)+1, zSameOp, z
-        );
-        if( zWhere==0 ){
-          *pRc = SQLITE_NOMEM;
-          break;
+        int n = strlen30(z);
+
+        if( pAr->bGlob ){
+          z1 = sqlite3_mprintf("%z%sname GLOB '%q'", z1, zSep2, z);
+          z2 = sqlite3_mprintf(
+              "%z%ssubstr(name,1,%d) GLOB '%q/'", z2, zSep2, n+1,z
+          );
+        }else{
+          z1 = sqlite3_mprintf("%z%s'%q'", z1, zSep1, z);
+          z2 = sqlite3_mprintf("%z%ssubstr(name,1,%d) = '%q/'",z2,zSep2,n+1,z);
         }
-        zSep = " OR ";
+        zSep1 = ", ";
+        zSep2 = " OR ";
       }
+      if( z1==0 || z2==0 ){
+        *pRc = SQLITE_NOMEM;
+      }else{
+        zWhere = sqlite3_mprintf("(%s%s OR (name GLOB '*/*' AND (%s))) ",
+            z1, pAr->bGlob==0 ? ")" : "", z2
+        );
+      }
+      sqlite3_free(z1);
+      sqlite3_free(z2);
     }
   }
   *pzWhere = zWhere;
@@ -7254,18 +7984,15 @@ static int arListCommand(ArCommand *pAr){
   shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose],
                      pAr->zSrcTable, zWhere);
   if( pAr->bDryRun ){
-    utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql));
+    sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql));
   }else{
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       if( pAr->bVerbose ){
-        utf8_printf(pAr->p->out, "%s % 10d  %s  %s\n",
-            sqlite3_column_text(pSql, 0),
-            sqlite3_column_int(pSql, 1),
-            sqlite3_column_text(pSql, 2),
-            sqlite3_column_text(pSql, 3)
-        );
+        sqlite3_fprintf(pAr->out, "%s % 10d  %s  %s\n",
+              sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1),
+              sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3));
       }else{
-        utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0));
+        sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0));
       }
     }
   }
@@ -7274,7 +8001,6 @@ static int arListCommand(ArCommand *pAr){
   return rc;
 }
 
-
 /*
 ** Implementation of .ar "Remove" command.
 */
@@ -7293,7 +8019,7 @@ static int arRemoveCommand(ArCommand *pAr){
     zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;",
                            pAr->zSrcTable, zWhere);
     if( pAr->bDryRun ){
-      utf8_printf(pAr->p->out, "%s\n", zSql);
+      sqlite3_fprintf(pAr->out, "%s\n", zSql);
     }else{
       char *zErr = 0;
       rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0);
@@ -7306,7 +8032,7 @@ static int arRemoveCommand(ArCommand *pAr){
         }
       }
       if( zErr ){
-        utf8_printf(stdout, "ERROR: %s\n", zErr);
+        sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */
         sqlite3_free(zErr);
       }
     }
@@ -7370,11 +8096,11 @@ static int arExtractCommand(ArCommand *pAr){
       j = sqlite3_bind_parameter_index(pSql, "$dirOnly");
       sqlite3_bind_int(pSql, j, i);
       if( pAr->bDryRun ){
-        utf8_printf(pAr->p->out, "%s\n", sqlite3_sql(pSql));
+        sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql));
       }else{
         while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
           if( i==0 && pAr->bVerbose ){
-            utf8_printf(pAr->p->out, "%s\n", sqlite3_column_text(pSql, 0));
+            sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0));
           }
         }
       }
@@ -7394,13 +8120,13 @@ static int arExtractCommand(ArCommand *pAr){
 static int arExecSql(ArCommand *pAr, const char *zSql){
   int rc;
   if( pAr->bDryRun ){
-    utf8_printf(pAr->p->out, "%s\n", zSql);
+    sqlite3_fprintf(pAr->out, "%s\n", zSql);
     rc = SQLITE_OK;
   }else{
     char *zErr = 0;
     rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr);
     if( zErr ){
-      utf8_printf(stdout, "ERROR: %s\n", zErr);
+      sqlite3_fprintf(stdout, "ERROR: %s\n", zErr);
       sqlite3_free(zErr);
     }
   }
@@ -7549,9 +8275,10 @@ static int arDotCommand(
   if( rc==SQLITE_OK ){
     int eDbType = SHELL_OPEN_UNSPEC;
     cmd.p = pState;
+    cmd.out = pState->out;
     cmd.db = pState->db;
     if( cmd.zFile ){
-      eDbType = deduceDatabaseType(cmd.zFile, 1);
+      eDbType = deduceDatabaseType(cmd.zFile, 1, 0);
     }else{
       eDbType = pState->openMode;
     }
@@ -7575,15 +8302,14 @@ static int arDotCommand(
       }
       cmd.db = 0;
       if( cmd.bDryRun ){
-        utf8_printf(pState->out, "-- open database '%s'%s\n", cmd.zFile,
-             eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : "");
+        sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile,
+              eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : "");
       }
       rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags,
              eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0);
       if( rc!=SQLITE_OK ){
-        utf8_printf(stderr, "cannot open file: %s (%s)\n",
-            cmd.zFile, sqlite3_errmsg(cmd.db)
-        );
+        sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n",
+                        cmd.zFile, sqlite3_errmsg(cmd.db));
         goto end_ar_command;
       }
       sqlite3_fileio_init(cmd.db, 0, 0);
@@ -7596,7 +8322,7 @@ static int arDotCommand(
       if( cmd.eCmd!=AR_CMD_CREATE
        && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0)
       ){
-        utf8_printf(stderr, "database does not contain an 'sqlar' table\n");
+        sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n");
         rc = SQLITE_ERROR;
         goto end_ar_command;
       }
@@ -7654,7 +8380,7 @@ end_ar_command:
 */
 static int recoverSqlCb(void *pCtx, const char *zSql){
   ShellState *pState = (ShellState*)pCtx;
-  utf8_printf(pState->out, "%s;\n", zSql);
+  sqlite3_fprintf(pState->out, "%s;\n", zSql);
   return SQLITE_OK;
 }
 
@@ -7697,7 +8423,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
       bRowids = 0;
     }
     else{
-      utf8_printf(stderr, "unexpected option: %s\n", azArg[i]);
+      sqlite3_fprintf(stderr,"unexpected option: %s\n", azArg[i]);
       showHelp(pState->out, azArg[0]);
       return 1;
     }
@@ -7712,17 +8438,52 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
   sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids);
   sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist);
 
+  sqlite3_fprintf(pState->out, ".dbconfig defensive off\n");
   sqlite3_recover_run(p);
   if( sqlite3_recover_errcode(p)!=SQLITE_OK ){
     const char *zErr = sqlite3_recover_errmsg(p);
     int errCode = sqlite3_recover_errcode(p);
-    raw_printf(stderr, "sql error: %s (%d)\n", zErr, errCode);
+    sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode);
   }
   rc = sqlite3_recover_finish(p);
   return rc;
 }
 #endif /* SQLITE_SHELL_HAVE_RECOVER */
 
+/*
+** Implementation of ".intck STEPS_PER_UNLOCK" command.
+*/
+static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){
+  sqlite3_intck *p = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_intck_open(pState->db, "main", &p);
+  if( rc==SQLITE_OK ){
+    i64 nStep = 0;
+    i64 nError = 0;
+    const char *zErr = 0;
+    while( SQLITE_OK==sqlite3_intck_step(p) ){
+      const char *zMsg = sqlite3_intck_message(p);
+      if( zMsg ){
+        sqlite3_fprintf(pState->out, "%s\n", zMsg);
+        nError++;
+      }
+      nStep++;
+      if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){
+        sqlite3_intck_unlock(p);
+      }
+    }
+    rc = sqlite3_intck_error(p, &zErr);
+    if( zErr ){
+      sqlite3_fprintf(stderr,"%s\n", zErr);
+    }
+    sqlite3_intck_close(p);
+
+    sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError);
+  }
+
+  return rc;
+}
 
 /*
  * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it.
@@ -7741,7 +8502,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){
 #define rc_err_oom_die(rc) \
   if( rc==SQLITE_NOMEM ) shell_check_oom(0); \
   else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \
-    fprintf(stderr,"E:%d\n",rc), assert(0)
+    sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0)
 #else
 static void rc_err_oom_die(int rc){
   if( rc==SQLITE_NOMEM ) shell_check_oom(0);
@@ -7881,6 +8642,7 @@ FROM (\
         sqlite3_exec(*pDb,"drop table if exists ColNames;"
                      "drop view if exists RepeatedNames;",0,0,0);
 #endif
+#undef SHELL_COLFIX_DB
       rc = sqlite3_exec(*pDb, zTabMake, 0, 0, 0);
       rc_err_oom_die(rc);
     }
@@ -7898,8 +8660,8 @@ FROM (\
   }else{
     /* Formulate the columns spec, close the DB, zero *pDb. */
     char *zColsSpec = 0;
-    int hasDupes = db_int(*pDb, zHasDupes);
-    int nDigits = (hasDupes)? db_int(*pDb, zColDigits) : 0;
+    int hasDupes = db_int(*pDb, "%s", zHasDupes);
+    int nDigits = (hasDupes)? db_int(*pDb, "%s", zColDigits) : 0;
     if( hasDupes ){
 #ifdef SHELL_COLUMN_RENAME_CLEAN
       rc = sqlite3_exec(*pDb, zDedoctor, 0, 0, 0);
@@ -7914,7 +8676,7 @@ FROM (\
       sqlite3_finalize(pStmt);
       if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM);
     }
-    assert(db_int(*pDb, zHasDupes)==0); /* Consider: remove this */
+    assert(db_int(*pDb, "%s", zHasDupes)==0); /* Consider: remove this */
     rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0);
     rc_err_oom_die(rc);
     rc = sqlite3_step(pStmt);
@@ -7941,6 +8703,72 @@ FROM (\
   }
 }
 
+/*
+** Check if the sqlite_schema table contains one or more virtual tables. If
+** parameter zLike is not NULL, then it is an SQL expression that the
+** sqlite_schema row must also match. If one or more such rows are found,
+** print the following warning to the output:
+**
+** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled
+*/
+static int outputDumpWarning(ShellState *p, const char *zLike){
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pStmt = 0;
+  shellPreparePrintf(p->db, &rc, &pStmt,
+    "SELECT 1 FROM sqlite_schema o WHERE "
+    "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true"
+  );
+  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+    sqlite3_fputs("/* WARNING: "
+          "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n",
+          p->out
+    );
+  }
+  shellFinalize(&rc, pStmt);
+  return rc;
+}
+
+/*
+** Fault-Simulator state and logic.
+*/
+static struct {
+  int iId;           /* ID that triggers a simulated fault.  -1 means "any" */
+  int iErr;          /* The error code to return on a fault */
+  int iCnt;          /* Trigger the fault only if iCnt is already zero */
+  int iInterval;     /* Reset iCnt to this value after each fault */
+  int eVerbose;      /* When to print output */
+  int nHit;          /* Number of hits seen so far */
+  int nRepeat;       /* Turn off after this many hits.  0 for never */
+  int nSkip;         /* Skip this many before first fault */
+} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+** This is the fault-sim callback
+*/
+static int faultsim_callback(int iArg){
+  if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){
+    return SQLITE_OK;
+  }
+  if( faultsim_state.iCnt ){
+    if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--;
+    if( faultsim_state.eVerbose>=2 ){
+      sqlite3_fprintf(stdout, 
+         "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt);
+    }
+    return SQLITE_OK;
+  }
+  if( faultsim_state.eVerbose>=1 ){
+    sqlite3_fprintf(stdout, 
+         "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr);
+  }
+  faultsim_state.iCnt = faultsim_state.iInterval;
+  faultsim_state.nHit++;
+  if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){
+    faultsim_state.iCnt = -1;
+  }
+  return faultsim_state.iErr;
+}
+
 /*
 ** If an input line begins with "." then invoke this routine to
 ** process that line.
@@ -7954,7 +8782,7 @@ static int do_meta_command(char *zLine, ShellState *p){
   int rc = 0;
   char *azArg[52];
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
   if( p->expert.pExpert ){
     expertFinish(p, 1, 0);
   }
@@ -7980,7 +8808,6 @@ static int do_meta_command(char *zLine, ShellState *p){
       azArg[nArg++] = &zLine[h];
       while( zLine[h] && !IsSpace(zLine[h]) ){ h++; }
       if( zLine[h] ) zLine[h++] = 0;
-      resolve_backslashes(azArg[nArg-1]);
     }
   }
   azArg[nArg] = 0;
@@ -7995,7 +8822,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 #ifndef SQLITE_OMIT_AUTHORIZATION
   if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){
     if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .auth ON|OFF\n");
+      sqlite3_fprintf(stderr, "Usage: .auth ON|OFF\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -8042,7 +8869,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           bAsync = 1;
         }else
         {
-          utf8_printf(stderr, "unknown option: %s\n", azArg[j]);
+          sqlite3_fprintf(stderr,"unknown option: %s\n", azArg[j]);
           return 1;
         }
       }else if( zDestFile==0 ){
@@ -8051,19 +8878,19 @@ static int do_meta_command(char *zLine, ShellState *p){
         zDb = zDestFile;
         zDestFile = azArg[j];
       }else{
-        raw_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
+        sqlite3_fprintf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n");
         return 1;
       }
     }
     if( zDestFile==0 ){
-      raw_printf(stderr, "missing FILENAME argument on .backup\n");
+      sqlite3_fprintf(stderr, "missing FILENAME argument on .backup\n");
       return 1;
     }
     if( zDb==0 ) zDb = "main";
     rc = sqlite3_open_v2(zDestFile, &pDest,
                   SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs);
     if( rc!=SQLITE_OK ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zDestFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zDestFile);
       close_db(pDest);
       return 1;
     }
@@ -8074,7 +8901,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     open_db(p, 0);
     pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
     if( pBackup==0 ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
+      shellDatabaseError(pDest);
       close_db(pDest);
       return 1;
     }
@@ -8083,7 +8910,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( rc==SQLITE_DONE ){
       rc = 0;
     }else{
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(pDest));
+      shellDatabaseError(pDest);
       rc = 1;
     }
     close_db(pDest);
@@ -8094,22 +8921,15 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg==2 ){
       bail_on_error = booleanValue(azArg[1]);
     }else{
-      raw_printf(stderr, "Usage: .bail on|off\n");
+      eputz("Usage: .bail on|off\n");
       rc = 1;
     }
   }else
 
+  /* Undocumented.  Legacy only.  See "crlf" below */
   if( c=='b' && n>=3 && cli_strncmp(azArg[0], "binary", n)==0 ){
-    if( nArg==2 ){
-      if( booleanValue(azArg[1]) ){
-        setBinaryMode(p->out, 1);
-      }else{
-        setTextMode(p->out, 1);
-      }
-    }else{
-      raw_printf(stderr, "Usage: .binary on|off\n");
-      rc = 1;
-    }
+    eputz("The \".binary\" command is deprecated.\n");
+    rc = 1;
   }else
 
   /* The undocumented ".breakpoint" command causes a call to the no-op
@@ -8131,11 +8951,11 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = chdir(azArg[1]);
 #endif
       if( rc ){
-        utf8_printf(stderr, "Cannot change to directory \"%s\"\n", azArg[1]);
+        sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]);
         rc = 1;
       }
     }else{
-      raw_printf(stderr, "Usage: .cd DIRECTORY\n");
+      eputz("Usage: .cd DIRECTORY\n");
       rc = 1;
     }
   }else
@@ -8145,7 +8965,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg==2 ){
       setOrClearFlag(p, SHFLG_CountChanges, azArg[1]);
     }else{
-      raw_printf(stderr, "Usage: .changes on|off\n");
+      eputz("Usage: .changes on|off\n");
       rc = 1;
     }
   }else
@@ -8159,17 +8979,17 @@ static int do_meta_command(char *zLine, ShellState *p){
     char *zRes = 0;
     output_reset(p);
     if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .check GLOB-PATTERN\n");
+      eputz("Usage: .check GLOB-PATTERN\n");
       rc = 2;
     }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){
       rc = 2;
     }else if( testcase_glob(azArg[1],zRes)==0 ){
-      utf8_printf(stderr,
-                 "testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
-                 p->zTestcase, azArg[1], zRes);
+      sqlite3_fprintf(stderr,
+            "testcase-%s FAILED\n Expected: [%s]\n      Got: [%s]\n",
+            p->zTestcase, azArg[1], zRes);
       rc = 1;
     }else{
-      utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase);
+      sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase);
       p->nCheck++;
     }
     sqlite3_free(zRes);
@@ -8182,7 +9002,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg==2 ){
       tryToClone(p, azArg[1]);
     }else{
-      raw_printf(stderr, "Usage: .clone FILENAME\n");
+      eputz("Usage: .clone FILENAME\n");
       rc = 1;
     }
   }else
@@ -8202,9 +9022,9 @@ static int do_meta_command(char *zLine, ShellState *p){
           zFile = "(temporary-file)";
         }
         if( p->pAuxDb == &p->aAuxDb[i] ){
-          utf8_printf(stdout, "ACTIVE %d: %s\n", i, zFile);
+          sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile);
         }else if( p->aAuxDb[i].db!=0 ){
-          utf8_printf(stdout, "       %d: %s\n", i, zFile);
+          sqlite3_fprintf(stdout, "       %d: %s\n", i, zFile);
         }
       }
     }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){
@@ -8221,7 +9041,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( i<0 || i>=ArraySize(p->aAuxDb) ){
         /* No-op */
       }else if( p->pAuxDb == &p->aAuxDb[i] ){
-        raw_printf(stderr, "cannot close the active database connection\n");
+        eputz("cannot close the active database connection\n");
         rc = 1;
       }else if( p->aAuxDb[i].db ){
         session_close_all(p, i);
@@ -8229,11 +9049,25 @@ static int do_meta_command(char *zLine, ShellState *p){
         p->aAuxDb[i].db = 0;
       }
     }else{
-      raw_printf(stderr, "Usage: .connection [close] [CONNECTION-NUMBER]\n");
+      eputz("Usage: .connection [close] [CONNECTION-NUMBER]\n");
       rc = 1;
     }
   }else
 
+  if( c=='c' && n==4
+   && (cli_strncmp(azArg[0], "crlf", n)==0
+       || cli_strncmp(azArg[0], "crnl",n)==0)
+  ){
+    if( nArg==2 ){
+#ifdef _WIN32
+      p->crlfMode = booleanValue(azArg[1]);
+#else
+      p->crlfMode = 0;
+#endif
+    }
+    sqlite3_fprintf(stderr, "crlf is %s\n", p->crlfMode ? "ON" : "OFF");
+  }else
+
   if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){
     char **azName = 0;
     int nName = 0;
@@ -8242,14 +9076,14 @@ static int do_meta_command(char *zLine, ShellState *p){
     open_db(p, 0);
     rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
     if( rc ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
       rc = 1;
     }else{
       while( sqlite3_step(pStmt)==SQLITE_ROW ){
         const char *zSchema = (const char *)sqlite3_column_text(pStmt,1);
         const char *zFile = (const char*)sqlite3_column_text(pStmt,2);
         if( zSchema==0 || zFile==0 ) continue;
-        azName = sqlite3_realloc(azName, (nName+1)*2*sizeof(char*));
+        azName = sqlite3_realloc64(azName, (nName+1)*2*sizeof(char*));
         shell_check_oom(azName);
         azName[nName*2] = strdup(zSchema);
         azName[nName*2+1] = strdup(zFile);
@@ -8261,11 +9095,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       int eTxn = sqlite3_txn_state(p->db, azName[i*2]);
       int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]);
       const char *z = azName[i*2+1];
-      utf8_printf(p->out, "%s: %s %s%s\n",
-         azName[i*2],
-         z && z[0] ? z : "\"\"",
-         bRdonly ? "r/o" : "r/w",
-         eTxn==SQLITE_TXN_NONE ? "" :
+      sqlite3_fprintf(p->out, "%s: %s %s%s\n",
+            azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w",
+            eTxn==SQLITE_TXN_NONE ? "" :
             eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn");
       free(azName[i*2]);
       free(azName[i*2+1]);
@@ -8278,6 +9110,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       const char *zName;
       int op;
     } aDbConfig[] = {
+        { "attach_create",      SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE  },
+        { "attach_write",       SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE   },
+        { "comments",           SQLITE_DBCONFIG_ENABLE_COMMENTS       },
         { "defensive",          SQLITE_DBCONFIG_DEFENSIVE             },
         { "dqs_ddl",            SQLITE_DBCONFIG_DQS_DDL               },
         { "dqs_dml",            SQLITE_DBCONFIG_DQS_DML               },
@@ -8305,12 +9140,13 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0);
       }
       sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v);
-      utf8_printf(p->out, "%19s %s\n", aDbConfig[ii].zName, v ? "on" : "off");
+      sqlite3_fprintf(p->out, "%19s %s\n",
+                      aDbConfig[ii].zName, v ? "on" : "off");
       if( nArg>1 ) break;
     }
     if( nArg>1 && ii==ArraySize(aDbConfig) ){
-      utf8_printf(stderr, "Error: unknown dbconfig \"%s\"\n", azArg[1]);
-      utf8_printf(stderr, "Enter \".dbconfig\" with no arguments for a list\n");
+      sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]);
+      eputz("Enter \".dbconfig\" with no arguments for a list\n");
     }
   }else
 
@@ -8340,8 +9176,8 @@ static int do_meta_command(char *zLine, ShellState *p){
         if( z[0]=='-' ) z++;
         if( cli_strcmp(z,"preserve-rowids")==0 ){
 #ifdef SQLITE_OMIT_VIRTUALTABLE
-          raw_printf(stderr, "The --preserve-rowids option is not compatible"
-                             " with SQLITE_OMIT_VIRTUALTABLE\n");
+          eputz("The --preserve-rowids option is not compatible"
+                " with SQLITE_OMIT_VIRTUALTABLE\n");
           rc = 1;
           sqlite3_free(zLike);
           goto meta_command_exit;
@@ -8359,7 +9195,8 @@ static int do_meta_command(char *zLine, ShellState *p){
           ShellSetFlag(p, SHFLG_DumpNoSys);
         }else
         {
-          raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]);
+          sqlite3_fprintf(stderr,
+               "Unknown option \"%s\" on \".dump\"\n", azArg[i]);
           rc = 1;
           sqlite3_free(zLike);
           goto meta_command_exit;
@@ -8389,12 +9226,13 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     open_db(p, 0);
 
+    outputDumpWarning(p, zLike);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
       /* When playing back a "dump", the content might appear in an order
       ** which causes immediate foreign key constraints to be violated.
       ** So disable foreign-key constraint enforcement to prevent problems. */
-      raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
-      raw_printf(p->out, "BEGIN TRANSACTION;\n");
+      sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out);
+      sqlite3_fputs("BEGIN TRANSACTION;\n", p->out);
     }
     p->writableSchema = 0;
     p->showHeader = 0;
@@ -8417,7 +9255,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       zSql = sqlite3_mprintf(
         "SELECT sql FROM sqlite_schema AS o "
         "WHERE (%s) AND sql NOT NULL"
-        "  AND type IN ('index','trigger','view')",
+        "  AND type IN ('index','trigger','view') "
+        "ORDER BY type COLLATE NOCASE DESC",
         zLike
       );
       run_table_dump_query(p, zSql);
@@ -8425,27 +9264,33 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     sqlite3_free(zLike);
     if( p->writableSchema ){
-      raw_printf(p->out, "PRAGMA writable_schema=OFF;\n");
+      sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out);
       p->writableSchema = 0;
     }
     sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
     sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
     if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){
-      raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n");
+      sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out);
     }
     p->showHeader = savedShowHeader;
     p->shellFlgs = savedShellFlags;
+    rc = p->nErr>0;
   }else
 
   if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){
     if( nArg==2 ){
       setOrClearFlag(p, SHFLG_Echo, azArg[1]);
     }else{
-      raw_printf(stderr, "Usage: .echo on|off\n");
+      eputz("Usage: .echo on|off\n");
       rc = 1;
     }
   }else
 
+  if( c=='d' && n>=3 && cli_strncmp(azArg[0], "dbtotxt", n)==0 ){
+    open_db(p, 0);
+    rc = shell_dbtotxt_command(p, nArg, azArg);
+  }else
+
   if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){
     if( nArg==2 ){
       p->autoEQPtest = 0;
@@ -8472,7 +9317,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         p->autoEQP = (u8)booleanValue(azArg[1]);
       }
     }else{
-      raw_printf(stderr, "Usage: .eqp off|on|trace|trigger|full\n");
+      eputz("Usage: .eqp off|on|trace|trigger|full\n");
       rc = 1;
     }
   }else
@@ -8508,12 +9353,12 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
   }else
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
   if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){
     if( p->bSafeMode ){
-      raw_printf(stderr,
-        "Cannot run experimental commands such as \"%s\" in safe mode\n",
-        azArg[0]);
+      sqlite3_fprintf(stderr,
+            "Cannot run experimental commands such as \"%s\" in safe mode\n",
+            azArg[0]);
       rc = 1;
     }else{
       open_db(p, 0);
@@ -8569,10 +9414,10 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     /* --help lists all file-controls */
     if( cli_strcmp(zCmd,"help")==0 ){
-      utf8_printf(p->out, "Available file-controls:\n");
+      sqlite3_fputs("Available file-controls:\n", p->out);
       for(i=0; iout, "  .filectrl %s %s\n",
-                    aCtrl[i].zCtrlName, aCtrl[i].zUsage);
+        sqlite3_fprintf(p->out,
+               "  .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage);
       }
       rc = 1;
       goto meta_command_exit;
@@ -8587,16 +9432,16 @@ static int do_meta_command(char *zLine, ShellState *p){
           filectrl = aCtrl[i].ctrlCode;
           iCtrl = i;
         }else{
-          utf8_printf(stderr, "Error: ambiguous file-control: \"%s\"\n"
-                              "Use \".filectrl --help\" for help\n", zCmd);
+          sqlite3_fprintf(stderr,"Error: ambiguous file-control: \"%s\"\n"
+                "Use \".filectrl --help\" for help\n", zCmd);
           rc = 1;
           goto meta_command_exit;
         }
       }
     }
     if( filectrl<0 ){
-      utf8_printf(stderr,"Error: unknown file-control: %s\n"
-                         "Use \".filectrl --help\" for help\n", zCmd);
+      sqlite3_fprintf(stderr,"Error: unknown file-control: %s\n"
+            "Use \".filectrl --help\" for help\n", zCmd);
     }else{
       switch(filectrl){
         case SQLITE_FCNTL_SIZE_LIMIT: {
@@ -8639,7 +9484,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( nArg!=2 ) break;
           sqlite3_file_control(p->db, zSchema, filectrl, &z);
           if( z ){
-            utf8_printf(p->out, "%s\n", z);
+            sqlite3_fprintf(p->out, "%s\n", z);
             sqlite3_free(z);
           }
           isOk = 2;
@@ -8653,19 +9498,20 @@ static int do_meta_command(char *zLine, ShellState *p){
           }
           x = -1;
           sqlite3_file_control(p->db, zSchema, filectrl, &x);
-          utf8_printf(p->out,"%d\n", x);
+          sqlite3_fprintf(p->out, "%d\n", x);
           isOk = 2;
           break;
         }
       }
     }
     if( isOk==0 && iCtrl>=0 ){
-      utf8_printf(p->out, "Usage: .filectrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
+      sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n",
+                      zCmd, aCtrl[iCtrl].zUsage);
       rc = 1;
     }else if( isOk==1 ){
       char zBuf[100];
       sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes);
-      raw_printf(p->out, "%s\n", zBuf);
+      sqlite3_fprintf(p->out, "%s\n", zBuf);
     }
   }else
 
@@ -8680,7 +9526,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       nArg = 1;
     }
     if( nArg!=1 ){
-      raw_printf(stderr, "Usage: .fullschema ?--indent?\n");
+      eputz("Usage: .fullschema ?--indent?\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -8690,7 +9536,8 @@ static int do_meta_command(char *zLine, ShellState *p){
        "  (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x"
        "     FROM sqlite_schema UNION ALL"
        "   SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) "
-       "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' "
+       "WHERE type!='meta' AND sql NOTNULL"
+       "  AND name NOT LIKE 'sqlite__%' ESCAPE '_' "
        "ORDER BY x",
        callback, &data, 0
     );
@@ -8700,19 +9547,21 @@ static int do_meta_command(char *zLine, ShellState *p){
                "SELECT rowid FROM sqlite_schema"
                " WHERE name GLOB 'sqlite_stat[134]'",
                -1, &pStmt, 0);
-      doStats = sqlite3_step(pStmt)==SQLITE_ROW;
-      sqlite3_finalize(pStmt);
+      if( rc==SQLITE_OK ){
+        doStats = sqlite3_step(pStmt)==SQLITE_ROW;
+        sqlite3_finalize(pStmt);
+      }
     }
     if( doStats==0 ){
-      raw_printf(p->out, "/* No STAT tables available */\n");
+      sqlite3_fputs("/* No STAT tables available */\n", p->out);
     }else{
-      raw_printf(p->out, "ANALYZE sqlite_schema;\n");
+      sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
       data.cMode = data.mode = MODE_Insert;
       data.zDestTable = "sqlite_stat1";
       shell_exec(&data, "SELECT * FROM sqlite_stat1", 0);
       data.zDestTable = "sqlite_stat4";
       shell_exec(&data, "SELECT * FROM sqlite_stat4", 0);
-      raw_printf(p->out, "ANALYZE sqlite_schema;\n");
+      sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
     }
   }else
 
@@ -8721,7 +9570,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       p->showHeader = booleanValue(azArg[1]);
       p->shellFlgs |= SHFLG_HeaderSet;
     }else{
-      raw_printf(stderr, "Usage: .headers on|off\n");
+      eputz("Usage: .headers on|off\n");
       rc = 1;
     }
   }else
@@ -8730,7 +9579,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg>=2 ){
       n = showHelp(p->out, azArg[1]);
       if( n==0 ){
-        utf8_printf(p->out, "Nothing matches '%s'\n", azArg[1]);
+        sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]);
       }
     }else{
       showHelp(p->out, 0);
@@ -8740,20 +9589,19 @@ static int do_meta_command(char *zLine, ShellState *p){
 #ifndef SQLITE_SHELL_FIDDLE
   if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){
     char *zTable = 0;           /* Insert data into this table */
-    char *zSchema = 0;          /* within this schema (may default to "main") */
+    char *zSchema = 0;          /* Schema of zTable */
     char *zFile = 0;            /* Name of file to extra content from */
     sqlite3_stmt *pStmt = NULL; /* A statement */
     int nCol;                   /* Number of columns in the table */
-    int nByte;                  /* Number of bytes in an SQL string */
+    i64 nByte;                  /* Number of bytes in an SQL string */
     int i, j;                   /* Loop counters */
     int needCommit;             /* True to COMMIT or ROLLBACK at end */
     int nSep;                   /* Number of bytes in p->colSeparator[] */
-    char *zSql;                 /* An SQL statement */
-    char *zFullTabName;         /* Table name with schema if applicable */
+    char *zSql = 0;             /* An SQL statement */
     ImportCtx sCtx;             /* Reader context */
     char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */
     int eVerbose = 0;           /* Larger for more console output */
-    int nSkip = 0;              /* Initial lines to skip */
+    i64 nSkip = 0;              /* Initial lines to skip */
     int useOutputMode = 1;      /* Use output mode to determine separators */
     char *zCreate = 0;          /* CREATE TABLE statement text */
 
@@ -8774,7 +9622,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }else if( zTable==0 ){
           zTable = z;
         }else{
-          utf8_printf(p->out, "ERROR: extra argument: \"%s\".  Usage:\n", z);
+          sqlite3_fprintf(p->out, "ERROR: extra argument: \"%s\". Usage:\n",z);
           showHelp(p->out, "import");
           goto meta_command_exit;
         }
@@ -8795,14 +9643,14 @@ static int do_meta_command(char *zLine, ShellState *p){
         xRead = csv_read_one_field;
         useOutputMode = 0;
       }else{
-        utf8_printf(p->out, "ERROR: unknown option: \"%s\".  Usage:\n", z);
+        sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\".  Usage:\n", z);
         showHelp(p->out, "import");
         goto meta_command_exit;
       }
     }
     if( zTable==0 ){
-      utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n",
-                  zFile==0 ? "FILE" : "TABLE");
+      sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n",
+            zFile==0 ? "FILE" : "TABLE");
       showHelp(p->out, "import");
       goto meta_command_exit;
     }
@@ -8813,20 +9661,17 @@ static int do_meta_command(char *zLine, ShellState *p){
       ** the column and row separator characters from the output mode. */
       nSep = strlen30(p->colSeparator);
       if( nSep==0 ){
-        raw_printf(stderr,
-                   "Error: non-null column separator required for import\n");
+        eputz("Error: non-null column separator required for import\n");
         goto meta_command_exit;
       }
       if( nSep>1 ){
-        raw_printf(stderr,
-              "Error: multi-character column separators not allowed"
+        eputz("Error: multi-character column separators not allowed"
               " for import\n");
         goto meta_command_exit;
       }
       nSep = strlen30(p->rowSeparator);
       if( nSep==0 ){
-        raw_printf(stderr,
-            "Error: non-null row separator required for import\n");
+        eputz("Error: non-null row separator required for import\n");
         goto meta_command_exit;
       }
       if( nSep==2 && p->mode==MODE_Csv
@@ -8840,8 +9685,8 @@ static int do_meta_command(char *zLine, ShellState *p){
         nSep = strlen30(p->rowSeparator);
       }
       if( nSep>1 ){
-        raw_printf(stderr, "Error: multi-character row separators not allowed"
-                           " for import\n");
+        eputz("Error: multi-character row separators not allowed"
+              " for import\n");
         goto meta_command_exit;
       }
       sCtx.cColSep = (u8)p->colSeparator[0];
@@ -8851,31 +9696,31 @@ static int do_meta_command(char *zLine, ShellState *p){
     sCtx.nLine = 1;
     if( sCtx.zFile[0]=='|' ){
 #ifdef SQLITE_OMIT_POPEN
-      raw_printf(stderr, "Error: pipes are not supported in this OS\n");
+      eputz("Error: pipes are not supported in this OS\n");
       goto meta_command_exit;
 #else
-      sCtx.in = popen(sCtx.zFile+1, "r");
+      sCtx.in = sqlite3_popen(sCtx.zFile+1, "r");
       sCtx.zFile = "";
       sCtx.xCloser = pclose;
 #endif
     }else{
-      sCtx.in = fopen(sCtx.zFile, "rb");
+      sCtx.in = sqlite3_fopen(sCtx.zFile, "rb");
       sCtx.xCloser = fclose;
     }
     if( sCtx.in==0 ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
       goto meta_command_exit;
     }
     if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){
       char zSep[2];
       zSep[1] = 0;
       zSep[0] = sCtx.cColSep;
-      utf8_printf(p->out, "Column separator ");
+      sqlite3_fputs("Column separator ", p->out);
       output_c_string(p->out, zSep);
-      utf8_printf(p->out, ", row separator ");
+      sqlite3_fputs(", row separator ", p->out);
       zSep[0] = sCtx.cRowSep;
       output_c_string(p->out, zSep);
-      utf8_printf(p->out, "\n");
+      sqlite3_fputs("\n", p->out);
     }
     sCtx.z = sqlite3_malloc64(120);
     if( sCtx.z==0 ){
@@ -8883,78 +9728,102 @@ static int do_meta_command(char *zLine, ShellState *p){
       shell_out_of_memory();
     }
     /* Below, resources must be freed before exit. */
-    while( (nSkip--)>0 ){
+    while( nSkip>0 ){
+      nSkip--;
       while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){}
     }
-    if( zSchema!=0 ){
-      zFullTabName = sqlite3_mprintf("\"%w\".\"%w\"", zSchema, zTable);
-    }else{
-      zFullTabName = sqlite3_mprintf("\"%w\"", zTable);
-    }
-    zSql = sqlite3_mprintf("SELECT * FROM %s", zFullTabName);
-    if( zSql==0 || zFullTabName==0 ){
-      import_cleanup(&sCtx);
-      shell_out_of_memory();
-    }
-    nByte = strlen30(zSql);
-    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
     import_append_char(&sCtx, 0);    /* To ensure sCtx.z is allocated */
-    if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(p->db))==0 ){
+    if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) 
+     && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema"
+                         " WHERE name=%Q AND type='view'",
+                         zSchema ? zSchema : "main", zTable)
+    ){
+      /* Table does not exist.  Create it. */
       sqlite3 *dbCols = 0;
       char *zRenames = 0;
       char *zColDefs;
-      zCreate = sqlite3_mprintf("CREATE TABLE %s", zFullTabName);
+      zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", 
+                    zSchema ? zSchema : "main", zTable);
       while( xRead(&sCtx) ){
         zAutoColumn(sCtx.z, &dbCols, 0);
         if( sCtx.cTerm!=sCtx.cColSep ) break;
       }
       zColDefs = zAutoColumn(0, &dbCols, &zRenames);
       if( zRenames!=0 ){
-        utf8_printf((stdin_is_interactive && p->in==stdin)? p->out : stderr,
-                    "Columns renamed during .import %s due to duplicates:\n"
-                    "%s\n", sCtx.zFile, zRenames);
+        sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr,
+              "Columns renamed during .import %s due to duplicates:\n"
+              "%s\n", sCtx.zFile, zRenames);
         sqlite3_free(zRenames);
       }
       assert(dbCols==0);
       if( zColDefs==0 ){
-        utf8_printf(stderr,"%s: empty file\n", sCtx.zFile);
-      import_fail:
-        sqlite3_free(zCreate);
-        sqlite3_free(zSql);
-        sqlite3_free(zFullTabName);
+        sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile);
         import_cleanup(&sCtx);
         rc = 1;
+        sqlite3_free(zCreate);
         goto meta_command_exit;
       }
       zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs);
+      if( zCreate==0 ){
+        import_cleanup(&sCtx);
+        shell_out_of_memory();
+      }
       if( eVerbose>=1 ){
-        utf8_printf(p->out, "%s\n", zCreate);
+        sqlite3_fprintf(p->out, "%s\n", zCreate);
       }
       rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
       if( rc ){
-        utf8_printf(stderr, "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db));
-        goto import_fail;
+        sqlite3_fprintf(stderr,
+             "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db));
+      }
+      sqlite3_free(zCreate);
+      zCreate = 0;
+      if( rc ){
+        import_cleanup(&sCtx);
+        rc = 1;
+        goto meta_command_exit;
       }
-      sqlite3_free(zCreate);
-      zCreate = 0;
-      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
     }
+    zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);",
+                           zTable, zSchema);
+    if( zSql==0 ){
+      import_cleanup(&sCtx);
+      shell_out_of_memory();
+    }
+    rc =  sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    zSql = 0;
     if( rc ){
       if (pStmt) sqlite3_finalize(pStmt);
-      utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db));
-      goto import_fail;
+      shellDatabaseError(p->db);
+      import_cleanup(&sCtx);
+      rc = 1;
+      goto meta_command_exit;
+    }
+    if( sqlite3_step(pStmt)==SQLITE_ROW ){
+      nCol = sqlite3_column_int(pStmt, 0);
+    }else{
+      nCol = 0;
     }
-    sqlite3_free(zSql);
-    nCol = sqlite3_column_count(pStmt);
     sqlite3_finalize(pStmt);
     pStmt = 0;
     if( nCol==0 ) return 0; /* no columns, no error */
-    zSql = sqlite3_malloc64( nByte*2 + 20 + nCol*2 );
+
+    nByte = 64                 /* space for "INSERT INTO", "VALUES(", ")\0" */
+          + (zSchema ? strlen(zSchema)*2 + 2: 0)  /* Quoted schema name */
+          + strlen(zTable)*2 + 2                  /* Quoted table name */
+          + nCol*2;            /* Space for ",?" for each column */
+    zSql = sqlite3_malloc64( nByte );
     if( zSql==0 ){
       import_cleanup(&sCtx);
       shell_out_of_memory();
     }
-    sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zFullTabName);
+    if( zSchema ){
+      sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", 
+                       zSchema, zTable);
+    }else{
+      sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
+    }
     j = strlen30(zSql);
     for(i=1; i=2 ){
-      utf8_printf(p->out, "Insert using: %s\n", zSql);
+      sqlite3_fprintf(p->out, "Insert using: %s\n", zSql);
     }
     rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+    zSql = 0;
     if( rc ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
       if (pStmt) sqlite3_finalize(pStmt);
-      goto import_fail;
+      import_cleanup(&sCtx);
+      rc = 1;
+      goto meta_command_exit;
     }
-    sqlite3_free(zSql);
-    sqlite3_free(zFullTabName);
     needCommit = sqlite3_get_autocommit(p->db);
     if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
     do{
@@ -8990,11 +9862,19 @@ static int do_meta_command(char *zLine, ShellState *p){
         ** the remaining columns.
         */
         if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break;
+        /*
+        ** For CSV mode, per RFC 4180, accept EOF in lieu of final
+        ** record terminator but only for last field of multi-field row.
+        ** (If there are too few fields, it's not valid CSV anyway.)
+        */
+        if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){
+          z = "";
+        }
         sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
         if( i=nCol ){
         sqlite3_step(pStmt);
         rc = sqlite3_reset(pStmt);
         if( rc!=SQLITE_OK ){
-          utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile,
-                      startLine, sqlite3_errmsg(p->db));
+          sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n",
+                sCtx.zFile, startLine, sqlite3_errmsg(p->db));
           sCtx.nErr++;
         }else{
           sCtx.nRow++;
@@ -9025,9 +9905,9 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_finalize(pStmt);
     if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
     if( eVerbose>0 ){
-      utf8_printf(p->out,
-          "Added %d rows with %d errors using %d lines of input\n",
-          sCtx.nRow, sCtx.nErr, sCtx.nLine-1);
+      sqlite3_fprintf(p->out,
+            "Added %d rows with %d errors using %d lines of input\n",
+            sCtx.nRow, sCtx.nErr, sCtx.nLine-1);
     }
   }else
 #endif /* !defined(SQLITE_SHELL_FIDDLE) */
@@ -9041,15 +9921,9 @@ static int do_meta_command(char *zLine, ShellState *p){
     int isWO = 0;  /* True if making an imposter of a WITHOUT ROWID table */
     int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */
     int i;
-    if( !ShellHasFlag(p,SHFLG_TestingMode) ){
-      utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n",
-                  "imposter");
-      rc = 1;
-      goto meta_command_exit;
-    }
     if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){
-      utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"
-                          "       .imposter off\n");
+      eputz("Usage: .imposter INDEX IMPOSTER\n"
+            "       .imposter off\n");
       /* Also allowed, but not documented:
       **
       **    .imposter TABLE IMPOSTER
@@ -9108,7 +9982,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     sqlite3_finalize(pStmt);
     if( i==0 || tnum==0 ){
-      utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]);
+      sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]);
       rc = 1;
       sqlite3_free(zCollist);
       goto meta_command_exit;
@@ -9118,27 +9992,39 @@ static int do_meta_command(char *zLine, ShellState *p){
           "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%.*s))WITHOUT ROWID",
           azArg[2], zCollist, lenPK, zCollist);
     sqlite3_free(zCollist);
-    rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum);
+    rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 2, tnum);
     if( rc==SQLITE_OK ){
       rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
       sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0);
       if( rc ){
-        utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db));
+        sqlite3_fprintf(stderr,
+              "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db));
       }else{
-        utf8_printf(stdout, "%s;\n", zSql);
-        raw_printf(stdout,
-          "WARNING: writing to an imposter table will corrupt the \"%s\" %s!\n",
-          azArg[1], isWO ? "table" : "index"
-        );
+        sqlite3_fprintf(stdout, "%s;\n", zSql);
       }
     }else{
-      raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc);
+      sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc);
       rc = 1;
     }
     sqlite3_free(zSql);
   }else
 #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */
 
+  if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){
+    i64 iArg = 0;
+    if( nArg==2 ){
+      iArg = integerValue(azArg[1]);
+      if( iArg==0 ) iArg = -1;
+    }
+    if( (nArg!=1 && nArg!=2) || iArg<0 ){
+      sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n");
+      rc = 1;
+      goto meta_command_exit;
+    }
+    open_db(p, 0);
+    rc = intckDatabaseCmd(p, iArg);
+  }else
+
 #ifdef SQLITE_ENABLE_IOTRACE
   if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){
     SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...);
@@ -9150,9 +10036,9 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3IoTrace = iotracePrintf;
       iotrace = stdout;
     }else{
-      iotrace = fopen(azArg[1], "w");
+      iotrace = sqlite3_fopen(azArg[1], "w");
       if( iotrace==0 ){
-        utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]);
+        sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
         sqlite3IoTrace = 0;
         rc = 1;
       }else{
@@ -9184,11 +10070,11 @@ static int do_meta_command(char *zLine, ShellState *p){
     open_db(p, 0);
     if( nArg==1 ){
       for(i=0; idb, aLimit[i].limitCode, -1));
+        sqlite3_fprintf(stdout, "%20s %d\n", aLimit[i].zLimitName,
+              sqlite3_limit(p->db, aLimit[i].limitCode, -1));
       }
     }else if( nArg>3 ){
-      raw_printf(stderr, "Usage: .limit NAME ?NEW-VALUE?\n");
+      eputz("Usage: .limit NAME ?NEW-VALUE?\n");
       rc = 1;
       goto meta_command_exit;
     }else{
@@ -9199,16 +10085,16 @@ static int do_meta_command(char *zLine, ShellState *p){
           if( iLimit<0 ){
             iLimit = i;
           }else{
-            utf8_printf(stderr, "ambiguous limit: \"%s\"\n", azArg[1]);
+            sqlite3_fprintf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]);
             rc = 1;
             goto meta_command_exit;
           }
         }
       }
       if( iLimit<0 ){
-        utf8_printf(stderr, "unknown limit: \"%s\"\n"
-                        "enter \".limits\" with no arguments for a list.\n",
-                         azArg[1]);
+        sqlite3_fprintf(stderr,"unknown limit: \"%s\"\n"
+              "enter \".limits\" with no arguments for a list.\n",
+              azArg[1]);
         rc = 1;
         goto meta_command_exit;
       }
@@ -9216,8 +10102,8 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_limit(p->db, aLimit[iLimit].limitCode,
                       (int)integerValue(azArg[2]));
       }
-      printf("%20s %d\n", aLimit[iLimit].zLimitName,
-             sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1));
+      sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName,
+            sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1));
     }
   }else
 
@@ -9233,7 +10119,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     failIfSafeMode(p, "cannot run .load in safe mode");
     if( nArg<2 || azArg[1][0]==0 ){
       /* Must have a non-empty FILE. (Will not load self.) */
-      raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n");
+      eputz("Usage: .load FILE ?ENTRYPOINT?\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -9242,7 +10128,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     open_db(p, 0);
     rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg);
     if( rc!=SQLITE_OK ){
-      utf8_printf(stderr, "Error: %s\n", zErrMsg);
+      shellEmitError(zErrMsg);
       sqlite3_free(zErrMsg);
       rc = 1;
     }
@@ -9251,7 +10137,7 @@ static int do_meta_command(char *zLine, ShellState *p){
 
   if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){
     if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .log FILENAME\n");
+      eputz("Usage: .log FILENAME\n");
       rc = 1;
     }else{
       const char *zFile = azArg[1];
@@ -9259,13 +10145,13 @@ static int do_meta_command(char *zLine, ShellState *p){
        && cli_strcmp(zFile,"on")!=0
        && cli_strcmp(zFile,"off")!=0
       ){
-        raw_printf(stdout, "cannot set .log to anything other "
-                   "than \"on\" or \"off\"\n");
+        sputz(stdout, "cannot set .log to anything other"
+              " than \"on\" or \"off\"\n");
         zFile = "off";
       }
       output_file_close(p->pLog);
       if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout";
-      p->pLog = output_file_open(zFile, 0);
+      p->pLog = output_file_open(zFile);
     }
   }else
 
@@ -9273,24 +10159,52 @@ static int do_meta_command(char *zLine, ShellState *p){
     const char *zMode = 0;
     const char *zTabname = 0;
     int i, n2;
+    int chng = 0;       /* 0x01:  change to cmopts.  0x02:  Any other change */
     ColModeOpts cmOpts = ColModeOpts_default;
     for(i=1; ieEscMode = k;
+            chng |= 2;
+            break;
+          }
+        }
+        if( k>=ArraySize(shell_EscModeNames) ){
+          sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\""
+                                  " - choices:", zEsc);
+          for(k=0; kmode==MODE_Column
        || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
       ){
-        raw_printf
-          (p->out,
-           "current output mode: %s --wrap %d --wordwrap %s --%squote\n",
-           modeDescr[p->mode], p->cmOpts.iWrap,
-           p->cmOpts.bWordWrap ? "on" : "off",
-           p->cmOpts.bQuote ? "" : "no");
+        sqlite3_fprintf(p->out,
+              "current output mode: %s --wrap %d --wordwrap %s "
+              "--%squote --escape %s\n",
+              modeDescr[p->mode], p->cmOpts.iWrap,
+              p->cmOpts.bWordWrap ? "on" : "off",
+              p->cmOpts.bQuote ? "" : "no",
+              shell_EscModeNames[p->eEscMode]
+        );
       }else{
-        raw_printf(p->out, "current output mode: %s\n", modeDescr[p->mode]);
+        sqlite3_fprintf(p->out,
+              "current output mode: %s --escape %s\n",
+              modeDescr[p->mode],
+              shell_EscModeNames[p->eEscMode]
+        );
       }
+    }
+    if( zMode==0 ){
       zMode = modeDescr[p->mode];
+      if( (chng&1)==0 ) cmOpts = p->cmOpts;
     }
     n2 = strlen30(zMode);
     if( cli_strncmp(zMode,"lines",n2)==0 ){
@@ -9360,6 +10284,11 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else if( cli_strncmp(zMode,"insert",n2)==0 ){
       p->mode = MODE_Insert;
       set_table_name(p, zTabname ? zTabname : "table");
+      if( p->eEscMode==SHELL_ESC_OFF ){
+        ShellSetFlag(p, SHFLG_Newlines);
+      }else{
+        ShellClearFlag(p, SHFLG_Newlines);
+      }
     }else if( cli_strncmp(zMode,"quote",n2)==0 ){
       p->mode = MODE_Quote;
       sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
@@ -9384,9 +10313,9 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else if( cli_strncmp(zMode,"json",n2)==0 ){
       p->mode = MODE_Json;
     }else{
-      raw_printf(stderr, "Error: mode should be one of: "
-         "ascii box column csv html insert json line list markdown "
-         "qbox quote table tabs tcl\n");
+      eputz("Error: mode should be one of: "
+            "ascii box column csv html insert json line list markdown "
+            "qbox quote table tabs tcl\n");
       rc = 1;
     }
     p->cMode = p->mode;
@@ -9395,11 +10324,11 @@ static int do_meta_command(char *zLine, ShellState *p){
 #ifndef SQLITE_SHELL_FIDDLE
   if( c=='n' && cli_strcmp(azArg[0], "nonce")==0 ){
     if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .nonce NONCE\n");
+      eputz("Usage: .nonce NONCE\n");
       rc = 1;
     }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){
-      raw_printf(stderr, "line %d: incorrect nonce: \"%s\"\n",
-                 p->lineno, azArg[1]);
+      sqlite3_fprintf(stderr,"line %lld: incorrect nonce: \"%s\"\n",
+            p->lineno, azArg[1]);
       exit(1);
     }else{
       p->bSafeMode = 0;
@@ -9414,7 +10343,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3_snprintf(sizeof(p->nullValue), p->nullValue,
                        "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]);
     }else{
-      raw_printf(stderr, "Usage: .nullvalue STRING\n");
+      eputz("Usage: .nullvalue STRING\n");
       rc = 1;
     }
   }else
@@ -9425,6 +10354,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     int iName = 1;           /* Index in azArg[] of the filename */
     int newFlag = 0;         /* True to delete file before opening */
     int openMode = SHELL_OPEN_UNSPEC;
+    int openFlags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE;
 
     /* Check for command-line arguments */
     for(iName=1; iNameopenFlags |= SQLITE_OPEN_NOFOLLOW;
+        openFlags |= SQLITE_OPEN_NOFOLLOW;
 #ifndef SQLITE_OMIT_DESERIALIZE
       }else if( optionMatch(z, "deserialize") ){
         openMode = SHELL_OPEN_DESERIALIZE;
       }else if( optionMatch(z, "hexdb") ){
         openMode = SHELL_OPEN_HEXDB;
+      }else if( optionMatch(z, "normal") ){
+        openMode = SHELL_OPEN_NORMAL;
       }else if( optionMatch(z, "maxsize") && iName+1szMax = integerValue(azArg[++iName]);
 #endif /* SQLITE_OMIT_DESERIALIZE */
       }else
 #endif /* !SQLITE_SHELL_FIDDLE */
       if( z[0]=='-' ){
-        utf8_printf(stderr, "unknown option: %s\n", z);
+        sqlite3_fprintf(stderr,"unknown option: %s\n", z);
         rc = 1;
         goto meta_command_exit;
       }else if( zFN ){
-        utf8_printf(stderr, "extra argument: \"%s\"\n", z);
+        sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z);
         rc = 1;
         goto meta_command_exit;
       }else{
@@ -9473,12 +10410,21 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_free(p->pAuxDb->zFreeOnClose);
     p->pAuxDb->zFreeOnClose = 0;
     p->openMode = openMode;
-    p->openFlags = 0;
+    p->openFlags = openFlags;
     p->szMax = 0;
 
     /* If a filename is specified, try to open it first */
     if( zFN || p->openMode==SHELL_OPEN_HEXDB ){
-      if( newFlag && zFN && !p->bSafeMode ) shellDeleteFile(zFN);
+      if( newFlag && zFN && !p->bSafeMode ){
+        if( cli_strncmp(zFN,"file:",5)==0 ){
+          char *zDel = shellFilenameFromUri(zFN);
+          shell_check_oom(zDel);
+          shellDeleteFile(zDel);
+          sqlite3_free(zDel);
+        }else{
+          shellDeleteFile(zFN);
+        }
+      }
 #ifndef SQLITE_SHELL_FIDDLE
       if( p->bSafeMode
        && p->openMode!=SHELL_OPEN_HEXDB
@@ -9499,7 +10445,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       p->pAuxDb->zDbFilename = zNewFilename;
       open_db(p, OPEN_DB_KEEPALIVE);
       if( p->db==0 ){
-        utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename);
+        sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename);
         sqlite3_free(zNewFilename);
       }else{
         p->pAuxDb->zFreeOnClose = zNewFilename;
@@ -9517,19 +10463,23 @@ static int do_meta_command(char *zLine, ShellState *p){
         && (cli_strncmp(azArg[0], "output", n)==0
             || cli_strncmp(azArg[0], "once", n)==0))
    || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0)
+   || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0)
   ){
     char *zFile = 0;
-    int bTxtMode = 0;
     int i;
-    int eMode = 0;
-    int bOnce = 0;            /* 0: .output, 1: .once, 2: .excel */
-    unsigned char zBOM[4];    /* Byte-order mark to using if --bom is present */
+    int eMode = 0;          /* 0: .outout/.once, 'x'=.excel, 'w'=.www */
+    int bOnce = 0;          /* 0: .output, 1: .once, 2: .excel/.www */
+    int bPlain = 0;         /* --plain option */
+    static const char *zBomUtf8 = "\357\273\277";
+    const char *zBom = 0;
 
-    zBOM[0] = 0;
     failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]);
     if( c=='e' ){
       eMode = 'x';
       bOnce = 2;
+    }else if( c=='w' ){
+      eMode = 'w';
+      bOnce = 2;
     }else if( cli_strncmp(azArg[0],"once",n)==0 ){
       bOnce = 1;
     }
@@ -9538,30 +10488,40 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( z[0]=='-' ){
         if( z[1]=='-' ) z++;
         if( cli_strcmp(z,"-bom")==0 ){
-          zBOM[0] = 0xef;
-          zBOM[1] = 0xbb;
-          zBOM[2] = 0xbf;
-          zBOM[3] = 0;
-        }else if( c!='e' && cli_strcmp(z,"-x")==0 ){
+          zBom = zBomUtf8;
+        }else if( cli_strcmp(z,"-plain")==0 ){
+          bPlain = 1;
+        }else if( c=='o' && cli_strcmp(z,"-x")==0 ){
           eMode = 'x';  /* spreadsheet */
-        }else if( c!='e' && cli_strcmp(z,"-e")==0 ){
+        }else if( c=='o' && cli_strcmp(z,"-e")==0 ){
           eMode = 'e';  /* text editor */
+        }else if( c=='o' && cli_strcmp(z,"-w")==0 ){
+          eMode = 'w';  /* Web browser */
         }else{
-          utf8_printf(p->out, "ERROR: unknown option: \"%s\".  Usage:\n",
-                      azArg[i]);
+          sqlite3_fprintf(p->out,
+                          "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]);
           showHelp(p->out, azArg[0]);
           rc = 1;
+          sqlite3_free(zFile);
           goto meta_command_exit;
         }
-      }else if( zFile==0 && eMode!='e' && eMode!='x' ){
-        zFile = sqlite3_mprintf("%s", z);
+      }else if( zFile==0 && eMode==0 ){
+        if( cli_strcmp(z, "off")==0 ){
+#ifdef _WIN32
+          zFile = sqlite3_mprintf("nul");
+#else
+          zFile = sqlite3_mprintf("/dev/null");
+#endif
+        }else{
+          zFile = sqlite3_mprintf("%s", z);
+        }
         if( zFile && zFile[0]=='|' ){
           while( i+1out,"ERROR: extra parameter: \"%s\".  Usage:\n",
-                    azArg[i]);
+        sqlite3_fprintf(p->out,
+            "ERROR: extra parameter: \"%s\".  Usage:\n", azArg[i]);
         showHelp(p->out, azArg[0]);
         rc = 1;
         sqlite3_free(zFile);
@@ -9571,6 +10531,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( zFile==0 ){
       zFile = sqlite3_mprintf("stdout");
     }
+    shell_check_oom(zFile);
     if( bOnce ){
       p->outCount = 2;
     }else{
@@ -9578,7 +10539,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }
     output_reset(p);
 #ifndef SQLITE_NOHAVE_SYSTEM
-    if( eMode=='e' || eMode=='x' ){
+    if( eMode=='e' || eMode=='x' || eMode=='w' ){
       p->doXdgOpen = 1;
       outputModePush(p);
       if( eMode=='x' ){
@@ -9588,10 +10549,17 @@ static int do_meta_command(char *zLine, ShellState *p){
         p->mode = MODE_Csv;
         sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma);
         sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf);
+#ifdef _WIN32
+        zBom = zBomUtf8;  /* Always include the BOM on Windows, as Excel does
+                          ** not work without it. */
+#endif
+      }else if( eMode=='w' ){
+        /* web-browser mode. */
+        newTempFile(p, "html");
+        if( !bPlain ) p->mode = MODE_Www;
       }else{
         /* text editor mode */
         newTempFile(p, "txt");
-        bTxtMode = 1;
       }
       sqlite3_free(zFile);
       zFile = sqlite3_mprintf("%s", p->zTempFile);
@@ -9600,30 +10568,38 @@ static int do_meta_command(char *zLine, ShellState *p){
     shell_check_oom(zFile);
     if( zFile[0]=='|' ){
 #ifdef SQLITE_OMIT_POPEN
-      raw_printf(stderr, "Error: pipes are not supported in this OS\n");
+      eputz("Error: pipes are not supported in this OS\n");
       rc = 1;
-      p->out = stdout;
+      output_redir(p, stdout);
 #else
-      p->out = popen(zFile + 1, "w");
-      if( p->out==0 ){
-        utf8_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1);
-        p->out = stdout;
+      FILE *pfPipe = sqlite3_popen(zFile + 1, "w");
+      if( pfPipe==0 ){
+        assert( stderr!=NULL );
+        sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1);
         rc = 1;
       }else{
-        if( zBOM[0] ) fwrite(zBOM, 1, 3, p->out);
+        output_redir(p, pfPipe);
+        if( zBom ) sqlite3_fputs(zBom, pfPipe);
         sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
       }
 #endif
     }else{
-      p->out = output_file_open(zFile, bTxtMode);
-      if( p->out==0 ){
+      FILE *pfFile = output_file_open(zFile);
+      if( pfFile==0 ){
         if( cli_strcmp(zFile,"off")!=0 ){
-          utf8_printf(stderr,"Error: cannot write to \"%s\"\n", zFile);
+         assert( stderr!=NULL );
+         sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile);
         }
-        p->out = stdout;
         rc = 1;
       } else {
-        if( zBOM[0] ) fwrite(zBOM, 1, 3, p->out);
+        output_redir(p, pfFile);
+        if( zBom ) sqlite3_fputs(zBom, pfFile);
+        if( bPlain && eMode=='w' ){
+          sqlite3_fputs(
+            "\n\n\n",
+            pfFile
+          );
+        }
         sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile);
       }
     }
@@ -9664,8 +10640,9 @@ static int do_meta_command(char *zLine, ShellState *p){
              "SELECT key, quote(value) "
              "FROM temp.sqlite_parameters;", -1, &pStmt, 0);
         while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
-          utf8_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0),
-                      sqlite3_column_text(pStmt,1));
+          sqlite3_fprintf(p->out,
+                "%-*s %s\n", len, sqlite3_column_text(pStmt,0),
+                sqlite3_column_text(pStmt,1));
         }
         sqlite3_finalize(pStmt);
       }
@@ -9709,12 +10686,13 @@ static int do_meta_command(char *zLine, ShellState *p){
         rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
         sqlite3_free(zSql);
         if( rx!=SQLITE_OK ){
-          utf8_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db));
+          sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db));
           sqlite3_finalize(pStmt);
           pStmt = 0;
           rc = 1;
         }
       }
+      bind_prepared_stmt(p, pStmt);
       sqlite3_step(pStmt);
       sqlite3_finalize(pStmt);
     }else
@@ -9738,10 +10716,10 @@ static int do_meta_command(char *zLine, ShellState *p){
   if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){
     int i;
     for(i=1; i<nArg; i++){
-      if( i>1 ) raw_printf(p->out, " ");
-      utf8_printf(p->out, "%s", azArg[i]);
+      if( i>1 ) sqlite3_fputs(" ", p->out);
+      sqlite3_fputs(azArg[i], p->out);
     }
-    raw_printf(p->out, "\n");
+    sqlite3_fputs("\n", p->out);
   }else
 
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
@@ -9770,7 +10748,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         }
         if( cli_strcmp(z,"limit")==0 ){
           if( i+1>=nArg ){
-            utf8_printf(stderr, "Error: missing argument on --limit\n");
+            eputz("Error: missing argument on --limit\n");
             rc = 1;
             goto meta_command_exit;
           }else{
@@ -9778,7 +10756,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           }
           continue;
         }
-        utf8_printf(stderr, "Error: unknown option: \"%s\"\n", azArg[i]);
+        sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]);
         rc = 1;
         goto meta_command_exit;
       }else{
@@ -9808,33 +10786,32 @@ static int do_meta_command(char *zLine, ShellState *p){
 #ifndef SQLITE_SHELL_FIDDLE
   if( c=='r' && n>=3 && cli_strncmp(azArg[0], "read", n)==0 ){
     FILE *inSaved = p->in;
-    int savedLineno = p->lineno;
+    i64 savedLineno = p->lineno;
     failIfSafeMode(p, "cannot run .read in safe mode");
     if( nArg!=2 ){
-      raw_printf(stderr, "Usage: .read FILE\n");
+      eputz("Usage: .read FILE\n");
       rc = 1;
       goto meta_command_exit;
     }
     if( azArg[1][0]=='|' ){
 #ifdef SQLITE_OMIT_POPEN
-      raw_printf(stderr, "Error: pipes are not supported in this OS\n");
+      eputz("Error: pipes are not supported in this OS\n");
       rc = 1;
-      p->out = stdout;
 #else
-      p->in = popen(azArg[1]+1, "r");
+      p->in = sqlite3_popen(azArg[1]+1, "r");
       if( p->in==0 ){
-        utf8_printf(stderr, "Error: cannot open \"%s\"\n", azArg[1]);
+        sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
         rc = 1;
       }else{
-        rc = process_input(p);
+        rc = process_input(p, "<pipe>");
         pclose(p->in);
       }
 #endif
     }else if( (p->in = openChrSource(azArg[1]))==0 ){
-      utf8_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]);
       rc = 1;
     }else{
-      rc = process_input(p);
+      rc = process_input(p, azArg[1]);
       fclose(p->in);
     }
     p->in = inSaved;
@@ -9858,20 +10835,20 @@ static int do_meta_command(char *zLine, ShellState *p){
       zSrcFile = azArg[2];
       zDb = azArg[1];
     }else{
-      raw_printf(stderr, "Usage: .restore ?DB? FILE\n");
+      eputz("Usage: .restore ?DB? FILE\n");
       rc = 1;
       goto meta_command_exit;
     }
     rc = sqlite3_open(zSrcFile, &pSrc);
     if( rc!=SQLITE_OK ){
-      utf8_printf(stderr, "Error: cannot open \"%s\"\n", zSrcFile);
+      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile);
       close_db(pSrc);
       return 1;
     }
     open_db(p, 0);
     pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main");
     if( pBackup==0 ){
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
       close_db(pSrc);
       return 1;
     }
@@ -9886,18 +10863,24 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( rc==SQLITE_DONE ){
       rc = 0;
     }else if( rc==SQLITE_BUSY || rc==SQLITE_LOCKED ){
-      raw_printf(stderr, "Error: source database is busy\n");
+      eputz("Error: source database is busy\n");
       rc = 1;
     }else{
-      utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
       rc = 1;
     }
     close_db(pSrc);
   }else
 #endif /* !defined(SQLITE_SHELL_FIDDLE) */
 
-  if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){
+  if( c=='s' &&
+     (cli_strncmp(azArg[0], "scanstats",  n)==0 ||
+      cli_strncmp(azArg[0], "scanstatus", n)==0)
+  ){
     if( nArg==2 ){
+      if( cli_strcmp(azArg[1], "vm")==0 ){
+        p->scanstatsOn = 3;
+      }else
       if( cli_strcmp(azArg[1], "est")==0 ){
         p->scanstatsOn = 2;
       }else{
@@ -9907,11 +10890,15 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3_db_config(
           p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0
       );
-#ifndef SQLITE_ENABLE_STMT_SCANSTATUS
-      raw_printf(stderr, "Warning: .scanstats not available in this build.\n");
+#if !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+      eputz("Warning: .scanstats not available in this build.\n");
+#elif !defined(SQLITE_ENABLE_BYTECODE_VTAB)
+      if( p->scanstatsOn==3 ){
+        eputz("Warning: \".scanstats vm\" not available in this build.\n");
+      }
 #endif
     }else{
-      raw_printf(stderr, "Usage: .scanstats on|off|est\n");
+      eputz("Usage: .scanstats on|off|est\n");
       rc = 1;
     }
   }else
@@ -9940,14 +10927,13 @@ static int do_meta_command(char *zLine, ShellState *p){
       }else if( optionMatch(azArg[ii],"nosys") ){
         bNoSystemTabs = 1;
       }else if( azArg[ii][0]=='-' ){
-        utf8_printf(stderr, "Unknown option: \"%s\"\n", azArg[ii]);
+        sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]);
         rc = 1;
         goto meta_command_exit;
       }else if( zName==0 ){
         zName = azArg[ii];
       }else{
-        raw_printf(stderr,
-                   "Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n");
+        eputz("Usage: .schema ?--indent? ?--nosys? ?LIKE-PATTERN?\n");
         rc = 1;
         goto meta_command_exit;
       }
@@ -9980,7 +10966,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list",
                               -1, &pStmt, 0);
       if( rc ){
-        utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
+        shellDatabaseError(p->db);
         sqlite3_finalize(pStmt);
         rc = 1;
         goto meta_command_exit;
@@ -10037,23 +11023,23 @@ static int do_meta_command(char *zLine, ShellState *p){
         sqlite3_free(zQarg);
       }
       if( bNoSystemTabs ){
-        appendText(&sSelect, "name NOT LIKE 'sqlite_%%' AND ", 0);
+        appendText(&sSelect, "name NOT LIKE 'sqlite__%%' ESCAPE '_' AND ", 0);
       }
       appendText(&sSelect, "sql IS NOT NULL"
                            " ORDER BY snum, rowid", 0);
       if( bDebug ){
-        utf8_printf(p->out, "SQL: %s;\n", sSelect.z);
+        sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.zTxt);
       }else{
-        rc = sqlite3_exec(p->db, sSelect.z, callback, &data, &zErrMsg);
+        rc = sqlite3_exec(p->db, sSelect.zTxt, callback, &data, &zErrMsg);
       }
       freeText(&sSelect);
     }
     if( zErrMsg ){
-      utf8_printf(stderr,"Error: %s\n", zErrMsg);
+      shellEmitError(zErrMsg);
       sqlite3_free(zErrMsg);
       rc = 1;
     }else if( rc != SQLITE_OK ){
-      raw_printf(stderr,"Error: querying schema information\n");
+      eputz("Error: querying schema information\n");
       rc = 1;
     }else{
       rc = 0;
@@ -10099,11 +11085,12 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( nCmd!=2 ) goto session_syntax_error;
       if( pSession->p==0 ){
         session_not_open:
-        raw_printf(stderr, "ERROR: No sessions are open\n");
+        eputz("ERROR: No sessions are open\n");
       }else{
         rc = sqlite3session_attach(pSession->p, azCmd[1]);
         if( rc ){
-          raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc);
+          sqlite3_fprintf(stderr,
+               "ERROR: sqlite3session_attach() returns %d\n",rc);
           rc = 0;
         }
       }
@@ -10120,10 +11107,10 @@ static int do_meta_command(char *zLine, ShellState *p){
       failIfSafeMode(p, "cannot run \".session %s\" in safe mode", azCmd[0]);
       if( nCmd!=2 ) goto session_syntax_error;
       if( pSession->p==0 ) goto session_not_open;
-      out = fopen(azCmd[1], "wb");
+      out = sqlite3_fopen(azCmd[1], "wb");
       if( out==0 ){
-        utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n",
-                    azCmd[1]);
+        sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n",
+              azCmd[1]);
       }else{
         int szChng;
         void *pChng;
@@ -10133,13 +11120,13 @@ static int do_meta_command(char *zLine, ShellState *p){
           rc = sqlite3session_patchset(pSession->p, &szChng, &pChng);
         }
         if( rc ){
-          printf("Error: error code %d\n", rc);
+          sqlite3_fprintf(stdout, "Error: error code %d\n", rc);
           rc = 0;
         }
         if( pChng
           && fwrite(pChng, szChng, 1, out)!=1 ){
-          raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n",
-                  szChng);
+          sqlite3_fprintf(stderr,
+              "ERROR: Failed to write entire %d-byte output\n", szChng);
         }
         sqlite3_free(pChng);
         fclose(out);
@@ -10166,8 +11153,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
       if( pAuxDb->nSession ){
         ii = sqlite3session_enable(pSession->p, ii);
-        utf8_printf(p->out, "session %s enable flag = %d\n",
-                    pSession->zName, ii);
+        sqlite3_fprintf(p->out,
+            "session %s enable flag = %d\n", pSession->zName, ii);
       }
     }else
 
@@ -10175,7 +11162,8 @@ static int do_meta_command(char *zLine, ShellState *p){
     ** Set a list of GLOB patterns of table names to be excluded.
     */
     if( cli_strcmp(azCmd[0], "filter")==0 ){
-      int ii, nByte;
+      int ii;
+      i64 nByte;
       if( nCmd<2 ) goto session_syntax_error;
       if( pAuxDb->nSession ){
         for(ii=0; ii<pSession->nFilter; ii++){
@@ -10183,11 +11171,8 @@ static int do_meta_command(char *zLine, ShellState *p){
         }
         sqlite3_free(pSession->azFilter);
         nByte = sizeof(pSession->azFilter[0])*(nCmd-1);
-        pSession->azFilter = sqlite3_malloc( nByte );
-        if( pSession->azFilter==0 ){
-          raw_printf(stderr, "Error: out or memory\n");
-          exit(1);
-        }
+        pSession->azFilter = sqlite3_malloc64( nByte );
+        shell_check_oom( pSession->azFilter );
         for(ii=1; ii<nCmd; ii++){
           char *x = pSession->azFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]);
           shell_check_oom(x);
@@ -10205,8 +11190,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       ii = nCmd==1 ? -1 : booleanValue(azCmd[1]);
       if( pAuxDb->nSession ){
         ii = sqlite3session_indirect(pSession->p, ii);
-        utf8_printf(p->out, "session %s indirect flag = %d\n",
-                    pSession->zName, ii);
+        sqlite3_fprintf(p->out,
+            "session %s indirect flag = %d\n", pSession->zName, ii);
       }
     }else
 
@@ -10218,8 +11203,8 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( nCmd!=1 ) goto session_syntax_error;
       if( pAuxDb->nSession ){
         ii = sqlite3session_isempty(pSession->p);
-        utf8_printf(p->out, "session %s isempty flag = %d\n",
-                    pSession->zName, ii);
+        sqlite3_fprintf(p->out,
+             "session %s isempty flag = %d\n", pSession->zName, ii);
       }
     }else
 
@@ -10228,7 +11213,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     */
     if( cli_strcmp(azCmd[0],"list")==0 ){
       for(i=0; i<pAuxDb->nSession; i++){
-        utf8_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName);
+        sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName);
       }
     }else
 
@@ -10243,19 +11228,19 @@ static int do_meta_command(char *zLine, ShellState *p){
       if( zName[0]==0 ) goto session_syntax_error;
       for(i=0; i<pAuxDb->nSession; i++){
         if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){
-          utf8_printf(stderr, "Session \"%s\" already exists\n", zName);
+          sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName);
           goto meta_command_exit;
         }
       }
       if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){
-        raw_printf(stderr,
-                   "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession));
+        sqlite3_fprintf(stderr,
+           "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession));
         goto meta_command_exit;
       }
       pSession = &pAuxDb->aSession[pAuxDb->nSession];
       rc = sqlite3session_create(p->db, azCmd[1], &pSession->p);
       if( rc ){
-        raw_printf(stderr, "Cannot open session: error code=%d\n", rc);
+        sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc);
         rc = 0;
         goto meta_command_exit;
       }
@@ -10279,7 +11264,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       int i, v;
       for(i=1; i<nArg; i++){
         v = booleanValue(azArg[i]);
-        utf8_printf(p->out, "%s: %d 0x%x\n", azArg[i], v, v);
+        sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v);
       }
     }
     if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){
@@ -10288,7 +11273,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         char zBuf[200];
         v = integerValue(azArg[i]);
         sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v);
-        utf8_printf(p->out, "%s", zBuf);
+        sqlite3_fputs(zBuf, p->out);
       }
     }
   }else
@@ -10315,9 +11300,9 @@ static int do_meta_command(char *zLine, ShellState *p){
         bVerbose++;
       }else
       {
-        utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n",
-                    azArg[i], azArg[0]);
-        raw_printf(stderr, "Should be one of: --init -v\n");
+        sqlite3_fprintf(stderr,
+              "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
+        sqlite3_fputs("Should be one of: --init -v\n", stderr);
         rc = 1;
         goto meta_command_exit;
       }
@@ -10346,7 +11331,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           -1, &pStmt, 0);
       }
       if( rc ){
-        raw_printf(stderr, "Error querying the selftest table\n");
+        eputz("Error querying the selftest table\n");
         rc = 1;
         sqlite3_finalize(pStmt);
         goto meta_command_exit;
@@ -10362,35 +11347,35 @@ static int do_meta_command(char *zLine, ShellState *p){
         if( zAns==0 ) continue;
         k = 0;
         if( bVerbose>0 ){
-          printf("%d: %s %s\n", tno, zOp, zSql);
+          sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql);
         }
         if( cli_strcmp(zOp,"memo")==0 ){
-          utf8_printf(p->out, "%s\n", zSql);
+          sqlite3_fprintf(p->out, "%s\n", zSql);
         }else
         if( cli_strcmp(zOp,"run")==0 ){
           char *zErrMsg = 0;
           str.n = 0;
-          str.z[0] = 0;
+          str.zTxt[0] = 0;
           rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg);
           nTest++;
           if( bVerbose ){
-            utf8_printf(p->out, "Result: %s\n", str.z);
+            sqlite3_fprintf(p->out, "Result: %s\n", str.zTxt);
           }
           if( rc || zErrMsg ){
             nErr++;
             rc = 1;
-            utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg);
+            sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg);
             sqlite3_free(zErrMsg);
-          }else if( cli_strcmp(zAns,str.z)!=0 ){
+          }else if( cli_strcmp(zAns,str.zTxt)!=0 ){
             nErr++;
             rc = 1;
-            utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns);
-            utf8_printf(p->out, "%d:      Got: [%s]\n", tno, str.z);
+            sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns);
+            sqlite3_fprintf(p->out, "%d:      Got: [%s]\n", tno, str.zTxt);
           }
-        }else
-        {
-          utf8_printf(stderr,
-            "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
+        }
+        else{
+          sqlite3_fprintf(stderr,
+                "Unknown operation \"%s\" on selftest line %d\n", zOp, tno);
           rc = 1;
           break;
         }
@@ -10398,12 +11383,12 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3_finalize(pStmt);
     } /* End loop over k */
     freeText(&str);
-    utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest);
+    sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest);
   }else
 
   if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){
     if( nArg<2 || nArg>3 ){
-      raw_printf(stderr, "Usage: .separator COL ?ROW?\n");
+      eputz("Usage: .separator COL ?ROW?\n");
       rc = 1;
     }
     if( nArg>=2 ){
@@ -10446,14 +11431,14 @@ static int do_meta_command(char *zLine, ShellState *p){
           bDebug = 1;
         }else
         {
-          utf8_printf(stderr, "Unknown option \"%s\" on \"%s\"\n",
-                      azArg[i], azArg[0]);
+          sqlite3_fprintf(stderr,
+                  "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]);
           showHelp(p->out, azArg[0]);
           rc = 1;
           goto meta_command_exit;
         }
       }else if( zLike ){
-        raw_printf(stderr, "Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n");
+        eputz("Usage: .sha3sum ?OPTIONS? ?LIKE-PATTERN?\n");
         rc = 1;
         goto meta_command_exit;
       }else{
@@ -10470,7 +11455,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else{
       zSql = "SELECT lower(name) as tname FROM sqlite_schema"
              " WHERE type='table' AND coalesce(rootpage,0)>1"
-             " AND name NOT LIKE 'sqlite_%'"
+             " AND name NOT LIKE 'sqlite__%' ESCAPE '_'"
              " ORDER BY 1 collate nocase";
     }
     sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
@@ -10501,7 +11486,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0);
       }
       appendText(&sSql, zSep, 0);
-      appendText(&sSql, sQuery.z, '\'');
+      appendText(&sSql, sQuery.zTxt, '\'');
       sQuery.n = 0;
       appendText(&sSql, ",", 0);
       appendText(&sSql, zTab, '\'');
@@ -10513,19 +11498,19 @@ static int do_meta_command(char *zLine, ShellState *p){
           "%s))"
           " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label"
           "   FROM [sha3sum$query]",
-          sSql.z, iSize);
+          sSql.zTxt, iSize);
     }else{
       zSql = sqlite3_mprintf(
           "%s))"
           " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash"
           "   FROM [sha3sum$query]",
-          sSql.z, iSize);
+          sSql.zTxt, iSize);
     }
     shell_check_oom(zSql);
     freeText(&sQuery);
     freeText(&sSql);
     if( bDebug ){
-      utf8_printf(p->out, "%s\n", zSql);
+      sqlite3_fprintf(p->out, "%s\n", zSql);
     }else{
       shell_exec(p, zSql, 0);
     }
@@ -10535,7 +11520,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       char *zRevText = /* Query for reversible to-blob-to-text check */
         "SELECT lower(name) as tname FROM sqlite_schema\n"
         "WHERE type='table' AND coalesce(rootpage,0)>1\n"
-        "AND name NOT LIKE 'sqlite_%%'%s\n"
+        "AND name NOT LIKE 'sqlite__%%' ESCAPE '_'%s\n"
         "ORDER BY 1 collate nocase";
       zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : "");
       zRevText = sqlite3_mprintf(
@@ -10543,7 +11528,8 @@ static int do_meta_command(char *zLine, ShellState *p){
           "with tabcols as materialized(\n"
           "select tname, cname\n"
           "from ("
-          " select ss.tname as tname, ti.name as cname\n"
+          " select printf('\"%%w\"',ss.tname) as tname,"
+          " printf('\"%%w\"',ti.name) as cname\n"
           " from (%z) ss\n inner join pragma_table_info(tname) ti))\n"
           "select 'SELECT total(bad_text_count) AS bad_text_count\n"
           "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n"
@@ -10554,7 +11540,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           "' OR ') as query, tname from tabcols group by tname)"
           , zRevText);
       shell_check_oom(zRevText);
-      if( bDebug ) utf8_printf(p->out, "%s\n", zRevText);
+      if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText);
       lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
       if( lrc!=SQLITE_OK ){
         /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the
@@ -10567,7 +11553,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0);
           sqlite3_stmt *pCheckStmt;
           lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
-          if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery);
+          if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery);
           if( lrc!=SQLITE_OK ){
             rc = 1;
           }else{
@@ -10575,9 +11561,9 @@ static int do_meta_command(char *zLine, ShellState *p){
               double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
               if( countIrreversible>0 ){
                 int sz = (int)(countIrreversible + 0.5);
-                utf8_printf(stderr,
-                     "Digest includes %d invalidly encoded text field%s.\n",
-                            sz, (sz>1)? "s": "");
+                sqlite3_fprintf(stderr,
+                      "Digest includes %d invalidly encoded text field%s.\n",
+                      sz, (sz>1)? "s": "");
               }
             }
             sqlite3_finalize(pCheckStmt);
@@ -10585,7 +11571,7 @@ static int do_meta_command(char *zLine, ShellState *p){
           sqlite3_finalize(pStmt);
         }
       }
-      if( rc ) utf8_printf(stderr, ".sha3sum failed.\n");
+      if( rc ) eputz(".sha3sum failed.\n");
       sqlite3_free(zRevText);
     }
 #endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */
@@ -10601,7 +11587,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     int i, x;
     failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]);
     if( nArg<2 ){
-      raw_printf(stderr, "Usage: .system COMMAND\n");
+      eputz("Usage: .system COMMAND\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -10610,9 +11596,11 @@ static int do_meta_command(char *zLine, ShellState *p){
       zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"",
                              zCmd, azArg[i]);
     }
+    /*consoleRestore();*/
     x = zCmd!=0 ? system(zCmd) : 1;
+    /*consoleRenewSetup();*/
     sqlite3_free(zCmd);
-    if( x ) raw_printf(stderr, "System command returns %d\n", x);
+    if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x);
   }else
 #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */
 
@@ -10621,52 +11609,53 @@ static int do_meta_command(char *zLine, ShellState *p){
     const char *zOut;
     int i;
     if( nArg!=1 ){
-      raw_printf(stderr, "Usage: .show\n");
+      eputz("Usage: .show\n");
       rc = 1;
       goto meta_command_exit;
     }
-    utf8_printf(p->out, "%12.12s: %s\n","echo",
-                azBool[ShellHasFlag(p, SHFLG_Echo)]);
-    utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]);
-    utf8_printf(p->out, "%12.12s: %s\n","explain",
-         p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off");
-    utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","echo",
+          azBool[ShellHasFlag(p, SHFLG_Echo)]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","explain",
+          p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off");
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","headers",
+          azBool[p->showHeader!=0]);
     if( p->mode==MODE_Column
      || (p->mode>=MODE_Markdown && p->mode<=MODE_Box)
     ){
-      utf8_printf
-        (p->out, "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode",
-         modeDescr[p->mode], p->cmOpts.iWrap,
-         p->cmOpts.bWordWrap ? "on" : "off",
-         p->cmOpts.bQuote ? "" : "no");
+      sqlite3_fprintf(p->out,
+            "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode",
+            modeDescr[p->mode], p->cmOpts.iWrap,
+            p->cmOpts.bWordWrap ? "on" : "off",
+            p->cmOpts.bQuote ? "" : "no");
     }else{
-      utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]);
-    }
-    utf8_printf(p->out, "%12.12s: ", "nullvalue");
-      output_c_string(p->out, p->nullValue);
-      raw_printf(p->out, "\n");
-    utf8_printf(p->out,"%12.12s: %s\n","output",
-            strlen30(p->outfile) ? p->outfile : "stdout");
-    utf8_printf(p->out,"%12.12s: ", "colseparator");
-      output_c_string(p->out, p->colSeparator);
-      raw_printf(p->out, "\n");
-    utf8_printf(p->out,"%12.12s: ", "rowseparator");
-      output_c_string(p->out, p->rowSeparator);
-      raw_printf(p->out, "\n");
+      sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]);
+    }
+    sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue");
+    output_c_string(p->out, p->nullValue);
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","output",
+          strlen30(p->outfile) ? p->outfile : "stdout");
+    sqlite3_fprintf(p->out, "%12.12s: ", "colseparator");
+    output_c_string(p->out, p->colSeparator);
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator");
+    output_c_string(p->out, p->rowSeparator);
+    sqlite3_fputs("\n", p->out);
     switch( p->statsOn ){
       case 0:  zOut = "off";     break;
       default: zOut = "on";      break;
       case 2:  zOut = "stmt";    break;
       case 3:  zOut = "vmstep";  break;
     }
-    utf8_printf(p->out, "%12.12s: %s\n","stats", zOut);
-    utf8_printf(p->out, "%12.12s: ", "width");
+    sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut);
+    sqlite3_fprintf(p->out, "%12.12s: ", "width");
     for (i=0;i<p->nWidth;i++) {
-      raw_printf(p->out, "%d ", p->colWidth[i]);
+      sqlite3_fprintf(p->out, "%d ", p->colWidth[i]);
     }
-    raw_printf(p->out, "\n");
-    utf8_printf(p->out, "%12.12s: %s\n", "filename",
-                p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : "");
+    sqlite3_fputs("\n", p->out);
+    sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename",
+          p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : "");
   }else
 
   if( c=='s' && cli_strncmp(azArg[0], "stats", n)==0 ){
@@ -10681,7 +11670,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else if( nArg==1 ){
       display_stats(p->db, p, 0);
     }else{
-      raw_printf(stderr, "Usage: .stats ?on|off|stmt|vmstep?\n");
+      eputz("Usage: .stats ?on|off|stmt|vmstep?\n");
       rc = 1;
     }
   }else
@@ -10707,7 +11696,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       /* It is an historical accident that the .indexes command shows an error
       ** when called with the wrong number of arguments whereas the .tables
       ** command does not. */
-      raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n");
+      eputz("Usage: .indexes ?LIKE-PATTERN?\n");
       rc = 1;
       sqlite3_finalize(pStmt);
       goto meta_command_exit;
@@ -10715,7 +11704,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){
       const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1);
       if( zDbName==0 ) continue;
-      if( s.z && s.z[0] ) appendText(&s, " UNION ALL ", 0);
+      if( s.zTxt && s.zTxt[0] ) appendText(&s, " UNION ALL ", 0);
       if( sqlite3_stricmp(zDbName, "main")==0 ){
         appendText(&s, "SELECT name FROM ", 0);
       }else{
@@ -10727,7 +11716,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       appendText(&s, ".sqlite_schema ", 0);
       if( c=='t' ){
         appendText(&s," WHERE type IN ('table','view')"
-                      "   AND name NOT LIKE 'sqlite_%'"
+                      "   AND name NOT LIKE 'sqlite__%' ESCAPE '_'"
                       "   AND name LIKE ?1", 0);
       }else{
         appendText(&s," WHERE type='index'"
@@ -10737,7 +11726,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     rc = sqlite3_finalize(pStmt);
     if( rc==SQLITE_OK ){
       appendText(&s, " ORDER BY 1", 0);
-      rc = sqlite3_prepare_v2(p->db, s.z, -1, &pStmt, 0);
+      rc = sqlite3_prepare_v2(p->db, s.zTxt, -1, &pStmt, 0);
     }
     freeText(&s);
     if( rc ) return shellDatabaseError(p->db);
@@ -10754,10 +11743,10 @@ static int do_meta_command(char *zLine, ShellState *p){
     while( sqlite3_step(pStmt)==SQLITE_ROW ){
       if( nRow>=nAlloc ){
         char **azNew;
-        int n2 = nAlloc*2 + 10;
+        sqlite3_int64 n2 = 2*(sqlite3_int64)nAlloc + 10;
         azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2);
         shell_check_oom(azNew);
-        nAlloc = n2;
+        nAlloc = (int)n2;
         azResult = azNew;
       }
       azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0));
@@ -10783,10 +11772,10 @@ static int do_meta_command(char *zLine, ShellState *p){
       for(i=0; i<nPrintRow; i++){
         for(j=i; j<nRow; j+=nPrintRow){
           char *zSp = j<nPrintRow ? "" : "  ";
-          utf8_printf(p->out, "%s%-*s", zSp, maxlen,
-                      azResult[j] ? azResult[j]:"");
+          sqlite3_fprintf(p->out,
+               "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:"");
         }
-        raw_printf(p->out, "\n");
+        sqlite3_fputs("\n", p->out);
       }
     }
 
@@ -10798,9 +11787,9 @@ static int do_meta_command(char *zLine, ShellState *p){
   /* Begin redirecting output to the file "testcase-out.txt" */
   if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){
     output_reset(p);
-    p->out = output_file_open("testcase-out.txt", 0);
+    p->out = output_file_open("testcase-out.txt");
     if( p->out==0 ){
-      raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n");
+      eputz("Error: cannot open 'testcase-out.txt'\n");
     }
     if( nArg>=2 ){
       sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]);
@@ -10815,25 +11804,27 @@ static int do_meta_command(char *zLine, ShellState *p){
     static const struct {
        const char *zCtrlName;   /* Name of a test-control option */
        int ctrlCode;            /* Integer code for that option */
-       int unSafe;              /* Not valid for --safe mode */
+       int unSafe;              /* Not valid unless --unsafe-testing */
        const char *zUsage;      /* Usage notes */
     } aCtrl[] = {
     {"always",             SQLITE_TESTCTRL_ALWAYS, 1,     "BOOLEAN"         },
     {"assert",             SQLITE_TESTCTRL_ASSERT, 1,     "BOOLEAN"         },
   /*{"benign_malloc_hooks",SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS,1, ""        },*/
-  /*{"bitvec_test",        SQLITE_TESTCTRL_BITVEC_TEST, 1,  ""              },*/
+    {"bitvec_test",        SQLITE_TESTCTRL_BITVEC_TEST, 1, "SIZE INT-ARRAY"},
     {"byteorder",          SQLITE_TESTCTRL_BYTEORDER, 0,  ""                },
     {"extra_schema_checks",SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS,0,"BOOLEAN"  },
-  /*{"fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, 1,""              },*/
+    {"fault_install",      SQLITE_TESTCTRL_FAULT_INSTALL, 1,"args..."       },
+    {"fk_no_action",       SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN"       },
     {"imposter",         SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
     {"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,""          },
+    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
     {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
     {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
-    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK"   },
+    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK ..."},
 #ifdef YYCOVERAGE
     {"parser_coverage",    SQLITE_TESTCTRL_PARSER_COVERAGE,0,""             },
 #endif
-    {"pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,0, "OFFSET  "       },
+    {"pending_byte",       SQLITE_TESTCTRL_PENDING_BYTE,1, "OFFSET  "       },
     {"prng_restore",       SQLITE_TESTCTRL_PRNG_RESTORE,0, ""               },
     {"prng_save",          SQLITE_TESTCTRL_PRNG_SAVE,   0, ""               },
     {"prng_seed",          SQLITE_TESTCTRL_PRNG_SEED,   0, "SEED ?db?"      },
@@ -10848,12 +11839,6 @@ static int do_meta_command(char *zLine, ShellState *p){
     int i, n2;
     const char *zCmd = 0;
 
-    if( !ShellHasFlag(p,SHFLG_TestingMode) ){
-      utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n",
-                  "testctrl");
-      rc = 1;
-      goto meta_command_exit;
-    }
     open_db(p, 0);
     zCmd = nArg>=2 ? azArg[1] : "help";
 
@@ -10865,10 +11850,11 @@ static int do_meta_command(char *zLine, ShellState *p){
 
     /* --help lists all test-controls */
     if( cli_strcmp(zCmd,"help")==0 ){
-      utf8_printf(p->out, "Available test-controls:\n");
+      sqlite3_fputs("Available test-controls:\n", p->out);
       for(i=0; i<ArraySize(aCtrl); i++){
-        utf8_printf(p->out, "  .testctrl %s %s\n",
-                    aCtrl[i].zCtrlName, aCtrl[i].zUsage);
+        if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue;
+        sqlite3_fprintf(p->out, "  .testctrl %s %s\n",
+              aCtrl[i].zCtrlName, aCtrl[i].zUsage);
       }
       rc = 1;
       goto meta_command_exit;
@@ -10878,31 +11864,147 @@ static int do_meta_command(char *zLine, ShellState *p){
     ** of the option name, or a numerical value. */
     n2 = strlen30(zCmd);
     for(i=0; i<ArraySize(aCtrl); i++){
+      if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue;
       if( cli_strncmp(zCmd, aCtrl[i].zCtrlName, n2)==0 ){
         if( testctrl<0 ){
           testctrl = aCtrl[i].ctrlCode;
           iCtrl = i;
         }else{
-          utf8_printf(stderr, "Error: ambiguous test-control: \"%s\"\n"
-                              "Use \".testctrl --help\" for help\n", zCmd);
+          sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n"
+                "Use \".testctrl --help\" for help\n", zCmd);
           rc = 1;
           goto meta_command_exit;
         }
       }
     }
     if( testctrl<0 ){
-      utf8_printf(stderr,"Error: unknown test-control: %s\n"
-                         "Use \".testctrl --help\" for help\n", zCmd);
-    }else if( aCtrl[iCtrl].unSafe && p->bSafeMode ){
-      utf8_printf(stderr,
-         "line %d: \".testctrl %s\" may not be used in safe mode\n",
-         p->lineno, aCtrl[iCtrl].zCtrlName);
-      exit(1);
+      sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n"
+            "Use \".testctrl --help\" for help\n", zCmd);
     }else{
       switch(testctrl){
 
+        /* Special processing for .testctrl opt MASK ...
+        ** Each MASK argument can be one of:
+        **
+        **      +LABEL       Enable the named optimization 
+        **
+        **      -LABEL       Disable the named optimization
+        **
+        **      INTEGER      Mask of optimizations to disable
+        */
+        case SQLITE_TESTCTRL_OPTIMIZATIONS: {
+          static const struct {
+             unsigned int mask;    /* Mask for this optimization */
+             unsigned int bDsply;  /* Display this on output */
+             const char *zLabel;   /* Name of optimization */
+          } aLabel[] = {
+            { 0x00000001, 1, "QueryFlattener" },
+            { 0x00000001, 0, "Flatten" },
+            { 0x00000002, 1, "WindowFunc" },
+            { 0x00000004, 1, "GroupByOrder" },
+            { 0x00000008, 1, "FactorOutConst" },
+            { 0x00000010, 1, "DistinctOpt" },
+            { 0x00000020, 1, "CoverIdxScan" },
+            { 0x00000040, 1, "OrderByIdxJoin" },
+            { 0x00000080, 1, "Transitive" },
+            { 0x00000100, 1, "OmitNoopJoin" },
+            { 0x00000200, 1, "CountOfView" },
+            { 0x00000400, 1, "CursorHints" },
+            { 0x00000800, 1, "Stat4" },
+            { 0x00001000, 1, "PushDown" },
+            { 0x00002000, 1, "SimplifyJoin" },
+            { 0x00004000, 1, "SkipScan" },
+            { 0x00008000, 1, "PropagateConst" },
+            { 0x00010000, 1, "MinMaxOpt" },
+            { 0x00020000, 1, "SeekScan" },
+            { 0x00040000, 1, "OmitOrderBy" },
+            { 0x00080000, 1, "BloomFilter" },
+            { 0x00100000, 1, "BloomPulldown" },
+            { 0x00200000, 1, "BalancedMerge" },
+            { 0x00400000, 1, "ReleaseReg" },
+            { 0x00800000, 1, "FlttnUnionAll" },
+            { 0x01000000, 1, "IndexedEXpr" },
+            { 0x02000000, 1, "Coroutines" },
+            { 0x04000000, 1, "NullUnusedCols" },
+            { 0x08000000, 1, "OnePass" },
+            { 0x10000000, 1, "OrderBySubq" },
+            { 0x20000000, 1, "StarQuery" },
+            { 0x40000000, 1, "ExistsToJoin" },
+            { 0xffffffff, 0, "All" },
+          };
+          unsigned int curOpt;
+          unsigned int newOpt;
+          unsigned int m;
+          int ii;
+          int nOff;
+          sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, p->db, &curOpt);
+          newOpt = curOpt;
+          for(ii=2; ii<nArg; ii++){
+            const char *z = azArg[ii];
+            int useLabel = 0;
+            const char *zLabel = 0;
+            if( (z[0]=='+'|| z[0]=='-') && !IsDigit(z[1]) ){
+              useLabel = z[0];
+              zLabel = &z[1];
+            }else if( !IsDigit(z[0]) && z[0]!=0 && !IsDigit(z[1]) ){
+              useLabel = '+';
+              zLabel = z;
+            }else{
+              newOpt = (unsigned int)strtol(z,0,0);
+            }
+            if( useLabel ){
+              int jj;
+              for(jj=0; jj<ArraySize(aLabel); jj++){
+                if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break;
+              }
+              if( jj>=ArraySize(aLabel) ){
+                sqlite3_fprintf(stderr,
+                    "Error: no such optimization: \"%s\"\n", zLabel);
+                sqlite3_fputs("Should be one of:", stderr);
+                for(jj=0; jj<ArraySize(aLabel); jj++){
+                  sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel);
+                }
+                sqlite3_fputs("\n", stderr);
+                rc = 1;
+                goto meta_command_exit;
+              }
+              if( useLabel=='+' ){
+                newOpt &= ~aLabel[jj].mask;
+              }else{
+                newOpt |= aLabel[jj].mask;
+              }
+            }
+          }
+          if( curOpt!=newOpt ){
+            sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,p->db,newOpt);
+          }
+          for(ii=nOff=0, m=1; ii<32; ii++, m <<= 1){
+            if( m & newOpt ) nOff++;
+          }
+          if( nOff<12 ){
+            sqlite3_fputs("+All", p->out);
+            for(ii=0; ii<ArraySize(aLabel); ii++){
+              if( !aLabel[ii].bDsply  ) continue;
+              if( (newOpt & aLabel[ii].mask)!=0 ){
+                sqlite3_fprintf(p->out, " -%s", aLabel[ii].zLabel);
+              }
+            }
+          }else{
+            sqlite3_fputs("-All", p->out);
+            for(ii=0; ii<ArraySize(aLabel); ii++){
+              if( !aLabel[ii].bDsply  ) continue;
+              if( (newOpt & aLabel[ii].mask)==0 ){
+                sqlite3_fprintf(p->out, " +%s", aLabel[ii].zLabel);
+              }
+            }
+          }
+          sqlite3_fputs("\n", p->out);
+          rc2 = isOk = 3;
+          break;
+        }
+
         /* sqlite3_test_control(int, db, int) */
-        case SQLITE_TESTCTRL_OPTIMIZATIONS:
+        case SQLITE_TESTCTRL_FK_NO_ACTION:
           if( nArg==3 ){
             unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0);
             rc2 = sqlite3_test_control(testctrl, p->db, opt);
@@ -10936,7 +12038,7 @@ static int do_meta_command(char *zLine, ShellState *p){
             sqlite3 *db;
             if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){
               sqlite3_randomness(sizeof(ii),&ii);
-              printf("-- random seed: %d\n", ii);
+              sqlite3_fprintf(stdout, "-- random seed: %d\n", ii);
             }
             if( nArg==3 ){
               db = 0;
@@ -10989,7 +12091,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         case SQLITE_TESTCTRL_SEEK_COUNT: {
           u64 x = 0;
           rc2 = sqlite3_test_control(testctrl, p->db, &x);
-          utf8_printf(p->out, "%llu\n", x);
+          sqlite3_fprintf(p->out, "%llu\n", x);
           isOk = 3;
           break;
         }
@@ -11020,11 +12122,11 @@ static int do_meta_command(char *zLine, ShellState *p){
               int val = 0;
               rc2 = sqlite3_test_control(testctrl, -id, &val);
               if( rc2!=SQLITE_OK ) break;
-              if( id>1 ) utf8_printf(p->out, "  ");
-              utf8_printf(p->out, "%d: %d", id, val);
+              if( id>1 ) sqlite3_fputs("  ", p->out);
+              sqlite3_fprintf(p->out, "%d: %d", id, val);
               id++;
             }
-            if( id>1 ) utf8_printf(p->out, "\n");
+            if( id>1 ) sqlite3_fputs("\n", p->out);
             isOk = 3;
           }
           break;
@@ -11037,15 +12139,149 @@ static int do_meta_command(char *zLine, ShellState *p){
             isOk = 3;
           }
           break;
+        case SQLITE_TESTCTRL_JSON_SELFCHECK:
+          if( nArg==2 ){
+            rc2 = -1;
+            isOk = 1;
+          }else{
+            rc2 = booleanValue(azArg[2]);
+            isOk = 3;
+          }
+          sqlite3_test_control(testctrl, &rc2);
+          break;
+        case SQLITE_TESTCTRL_BITVEC_TEST: {
+          /* Examples:
+          **   .testctrl bitvec_test 100   6,1       -- Show BITVEC constants
+          **   .testctrl bitvec_test 1000  1,12,7,3  -- Simple test
+          **                         ----  --------
+          **      size of Bitvec -----^        ^---  aOp array. 0 added at end.
+          **
+          ** See comments on sqlite3BitvecBuiltinTest() for more information
+          ** about the aOp[] array.
+          */
+          int iSize;
+          const char *zTestArg;
+          int nOp;
+          int ii, jj, x;
+          int *aOp;
+          if( nArg!=4 ){
+            sqlite3_fprintf(stderr,
+              "ERROR - should be:  \".testctrl bitvec_test SIZE  INT-ARRAY\"\n"
+            );
+            rc = 1;
+            goto meta_command_exit;
+          }
+          isOk = 3;
+          iSize = (int)integerValue(azArg[2]);
+          zTestArg = azArg[3];
+          nOp = (int)strlen(zTestArg)+1;
+          aOp = malloc( sizeof(int)*(nOp+1) );
+          shell_check_oom(aOp);
+          memset(aOp, 0, sizeof(int)*(nOp+1) );
+          for(ii = jj = x = 0; zTestArg[ii]!=0; ii++){
+            if( IsDigit(zTestArg[ii]) ){
+              x = x*10 + zTestArg[ii] - '0';
+            }else{
+              aOp[jj++] = x;
+              x = 0;
+            }
+          }
+          aOp[jj] = x;
+          x = sqlite3_test_control(testctrl, iSize, aOp);
+          sqlite3_fprintf(p->out, "result: %d\n", x);
+          free(aOp);
+          break;
+        }
+        case SQLITE_TESTCTRL_FAULT_INSTALL: {
+          int kk;
+          int bShowHelp = nArg<=2;
+          isOk = 3;
+          for(kk=2; kk<nArg; kk++){
+            const char *z = azArg[kk];
+            if( z[0]=='-' && z[1]=='-' ) z++;
+            if( cli_strcmp(z,"off")==0 ){
+              sqlite3_test_control(testctrl, 0);
+            }else if( cli_strcmp(z,"on")==0 ){
+              faultsim_state.iCnt = faultsim_state.nSkip;
+              if( faultsim_state.iErr==0 ) faultsim_state.iErr = 1;
+              faultsim_state.nHit = 0;
+              sqlite3_test_control(testctrl, faultsim_callback);
+            }else if( cli_strcmp(z,"reset")==0 ){
+              faultsim_state.iCnt = faultsim_state.nSkip;
+              faultsim_state.nHit = 0;
+              sqlite3_test_control(testctrl, faultsim_callback);
+            }else if( cli_strcmp(z,"status")==0 ){
+              sqlite3_fprintf(p->out, "faultsim.iId:       %d\n",
+                              faultsim_state.iId);
+              sqlite3_fprintf(p->out, "faultsim.iErr:      %d\n",
+                              faultsim_state.iErr);
+              sqlite3_fprintf(p->out, "faultsim.iCnt:      %d\n",
+                              faultsim_state.iCnt);
+              sqlite3_fprintf(p->out, "faultsim.nHit:      %d\n",
+                              faultsim_state.nHit);
+              sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n",
+                              faultsim_state.iInterval);
+              sqlite3_fprintf(p->out, "faultsim.eVerbose:  %d\n",
+                              faultsim_state.eVerbose);
+              sqlite3_fprintf(p->out, "faultsim.nRepeat:   %d\n",
+                              faultsim_state.nRepeat);
+              sqlite3_fprintf(p->out, "faultsim.nSkip:     %d\n",
+                              faultsim_state.nSkip);
+            }else if( cli_strcmp(z,"-v")==0 ){
+              if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++;
+            }else if( cli_strcmp(z,"-q")==0 ){
+              if( faultsim_state.eVerbose>0 ) faultsim_state.eVerbose--;
+            }else if( cli_strcmp(z,"-id")==0 && kk+1<nArg ){
+              faultsim_state.iId = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-errcode")==0 && kk+1<nArg ){
+              faultsim_state.iErr = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-interval")==0 && kk+1<nArg ){
+              faultsim_state.iInterval = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-repeat")==0 && kk+1<nArg ){
+              faultsim_state.nRepeat = atoi(azArg[++kk]);
+           }else if( cli_strcmp(z,"-skip")==0 && kk+1<nArg ){
+              faultsim_state.nSkip = atoi(azArg[++kk]);
+            }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){
+              bShowHelp = 1;
+            }else{
+              sqlite3_fprintf(stderr,
+                  "Unrecognized fault_install argument: \"%s\"\n",
+                  azArg[kk]);
+              rc = 1;
+              bShowHelp = 1;
+              break;
+            }
+          }
+          if( bShowHelp ){
+            sqlite3_fputs(
+               "Usage: .testctrl fault_install ARGS\n"
+               "Possible arguments:\n"
+               "   off               Disable faultsim\n"
+               "   on                Activate faultsim\n"
+               "   reset             Reset the trigger counter\n"
+               "   status            Show current status\n"
+               "   -v                Increase verbosity\n"
+               "   -q                Decrease verbosity\n"
+               "   --errcode N       When triggered, return N as error code\n"
+               "   --id ID           Trigger only for the ID specified\n"
+               "   --interval N      Trigger only after every N-th call\n"
+               "   --repeat N        Turn off after N hits.  0 means never\n"
+               "   --skip N          Skip the first N encounters\n"
+               ,p->out
+            );
+          }
+          break;
+        }
       }
     }
     if( isOk==0 && iCtrl>=0 ){
-      utf8_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
+      sqlite3_fprintf(p->out,
+          "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage);
       rc = 1;
     }else if( isOk==1 ){
-      raw_printf(p->out, "%d\n", rc2);
+      sqlite3_fprintf(p->out, "%d\n", rc2);
     }else if( isOk==2 ){
-      raw_printf(p->out, "0x%08x\n", rc2);
+      sqlite3_fprintf(p->out, "0x%08x\n", rc2);
     }
   }else
 #endif /* !defined(SQLITE_UNTESTABLE) */
@@ -11059,11 +12295,11 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( nArg==2 ){
       enableTimer = booleanValue(azArg[1]);
       if( enableTimer && !HAS_TIMER ){
-        raw_printf(stderr, "Error: timer not available on this system.\n");
+        eputz("Error: timer not available on this system.\n");
         enableTimer = 0;
       }
     }else{
-      raw_printf(stderr, "Usage: .timer on|off\n");
+      eputz("Usage: .timer on|off\n");
       rc = 1;
     }
   }else
@@ -11100,13 +12336,13 @@ static int do_meta_command(char *zLine, ShellState *p){
           mType |= SQLITE_TRACE_CLOSE;
         }
         else {
-          raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z);
+          sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z);
           rc = 1;
           goto meta_command_exit;
         }
       }else{
         output_file_close(p->traceOut);
-        p->traceOut = output_file_open(z, 0);
+        p->traceOut = output_file_open(z);
       }
     }
     if( p->traceOut==0 ){
@@ -11124,7 +12360,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     int lenOpt;
     char *zOpt;
     if( nArg<2 ){
-      raw_printf(stderr, "Usage: .unmodule [--allexcept] NAME ...\n");
+      eputz("Usage: .unmodule [--allexcept] NAME ...\n");
       rc = 1;
       goto meta_command_exit;
     }
@@ -11143,95 +12379,33 @@ static int do_meta_command(char *zLine, ShellState *p){
   }else
 #endif
 
-#if SQLITE_USER_AUTHENTICATION
-  if( c=='u' && cli_strncmp(azArg[0], "user", n)==0 ){
-    if( nArg<2 ){
-      raw_printf(stderr, "Usage: .user SUBCOMMAND ...\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-    open_db(p, 0);
-    if( cli_strcmp(azArg[1],"login")==0 ){
-      if( nArg!=4 ){
-        raw_printf(stderr, "Usage: .user login USER PASSWORD\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3],
-                                     strlen30(azArg[3]));
-      if( rc ){
-        utf8_printf(stderr, "Authentication failed for user %s\n", azArg[2]);
-        rc = 1;
-      }
-    }else if( cli_strcmp(azArg[1],"add")==0 ){
-      if( nArg!=5 ){
-        raw_printf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_add(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
-                            booleanValue(azArg[4]));
-      if( rc ){
-        raw_printf(stderr, "User-Add failed: %d\n", rc);
-        rc = 1;
-      }
-    }else if( cli_strcmp(azArg[1],"edit")==0 ){
-      if( nArg!=5 ){
-        raw_printf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_change(p->db, azArg[2], azArg[3], strlen30(azArg[3]),
-                              booleanValue(azArg[4]));
-      if( rc ){
-        raw_printf(stderr, "User-Edit failed: %d\n", rc);
-        rc = 1;
-      }
-    }else if( cli_strcmp(azArg[1],"delete")==0 ){
-      if( nArg!=3 ){
-        raw_printf(stderr, "Usage: .user delete USER\n");
-        rc = 1;
-        goto meta_command_exit;
-      }
-      rc = sqlite3_user_delete(p->db, azArg[2]);
-      if( rc ){
-        raw_printf(stderr, "User-Delete failed: %d\n", rc);
-        rc = 1;
-      }
-    }else{
-      raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n");
-      rc = 1;
-      goto meta_command_exit;
-    }
-  }else
-#endif /* SQLITE_USER_AUTHENTICATION */
-
   if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){
-    utf8_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
-        sqlite3_libversion(), sqlite3_sourceid());
+    char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit";
+    sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/,
+          sqlite3_libversion(), sqlite3_sourceid());
 /* BEGIN SQLCIPHER */
 #ifdef SQLITE_HAS_CODEC
     {
       extern char* sqlcipher_version();
       char *sqlcipher_ver = sqlcipher_version();
-      utf8_printf(p->out, "SQLCipher %s\n", sqlcipher_ver);
+      sqlite3_fprintf(p->out, "SQLCipher %s\n", sqlcipher_ver);
       sqlite3_free(sqlcipher_ver);
    }
 #endif
 /* END SQLCIPHER */
 #if SQLITE_HAVE_ZLIB
-    utf8_printf(p->out, "zlib version %s\n", zlibVersion());
+    sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion());
 #endif
 #define CTIMEOPT_VAL_(opt) #opt
 #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt)
 #if defined(__clang__) && defined(__clang_major__)
-    utf8_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "."
-                    CTIMEOPT_VAL(__clang_minor__) "."
-                    CTIMEOPT_VAL(__clang_patchlevel__) "\n");
+    sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "."
+          CTIMEOPT_VAL(__clang_minor__) "."
+          CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz);
 #elif defined(_MSC_VER)
-    utf8_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) "\n");
+    sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz);
 #elif defined(__GNUC__) && defined(__VERSION__)
-    utf8_printf(p->out, "gcc-" __VERSION__ "\n");
+    sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz);
 #endif
   }else
 
@@ -11241,10 +12415,10 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( p->db ){
       sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs);
       if( pVfs ){
-        utf8_printf(p->out, "vfs.zName      = \"%s\"\n", pVfs->zName);
-        raw_printf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
-        raw_printf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
-        raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
+        sqlite3_fprintf(p->out, "vfs.zName      = \"%s\"\n", pVfs->zName);
+        sqlite3_fprintf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
+        sqlite3_fprintf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
+        sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
       }
     }
   }else
@@ -11256,13 +12430,13 @@ static int do_meta_command(char *zLine, ShellState *p){
       sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent);
     }
     for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){
-      utf8_printf(p->out, "vfs.zName      = \"%s\"%s\n", pVfs->zName,
-           pVfs==pCurrent ? "  <--- CURRENT" : "");
-      raw_printf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
-      raw_printf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
-      raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
+      sqlite3_fprintf(p->out, "vfs.zName      = \"%s\"%s\n", pVfs->zName,
+            pVfs==pCurrent ? "  <--- CURRENT" : "");
+      sqlite3_fprintf(p->out, "vfs.iVersion   = %d\n", pVfs->iVersion);
+      sqlite3_fprintf(p->out, "vfs.szOsFile   = %d\n", pVfs->szOsFile);
+      sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname);
       if( pVfs->pNext ){
-        raw_printf(p->out, "-----------------------------------\n");
+        sqlite3_fputs("-----------------------------------\n", p->out);
       }
     }
   }else
@@ -11273,7 +12447,7 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( p->db ){
       sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName);
       if( zVfsName ){
-        utf8_printf(p->out, "%s\n", zVfsName);
+        sqlite3_fprintf(p->out, "%s\n", zVfsName);
         sqlite3_free(zVfsName);
       }
     }
@@ -11292,13 +12466,16 @@ static int do_meta_command(char *zLine, ShellState *p){
     if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory();
     if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth];
     for(j=1; j<nArg; j++){
-      p->colWidth[j-1] = (int)integerValue(azArg[j]);
+      i64 w = integerValue(azArg[j]);
+      if( w < -30000 ) w = -30000;
+      if( w > +30000 ) w = +30000;
+      p->colWidth[j-1] = (int)w;
     }
   }else
 
   {
-    utf8_printf(stderr, "Error: unknown command or invalid arguments: "
-      " \"%s\". Enter \".help\" for help\n", azArg[0]);
+    sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: "
+          " \"%s\". Enter \".help\" for help\n", azArg[0]);
     rc = 1;
   }
 
@@ -11338,7 +12515,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
   char cWait = (char)qss; /* intentional narrowing loss */
   if( cWait==0 ){
   PlainScan:
-    assert( cWait==0 );
     while( (cin = *zLine++)!=0 ){
       if( IsSpace(cin) )
         continue;
@@ -11364,7 +12540,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
         break;
       case '[':
         cin = ']';
-        deliberate_fall_through;
+        deliberate_fall_through; /* FALLTHRU */
       case '`': case '\'': case '"':
         cWait = cin;
         qss = QSS_HasDark | cWait;
@@ -11390,7 +12566,6 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
           if( *zLine != '/' )
             continue;
           ++zLine;
-          cWait = 0;
           CONTINUE_PROMPT_AWAITC(pst, 0);
           qss = QSS_SETV(qss, 0);
           goto PlainScan;
@@ -11400,9 +12575,8 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
             ++zLine;
             continue;
           }
-          deliberate_fall_through;
+          deliberate_fall_through; /* FALLTHRU */
         case ']':
-          cWait = 0;
           CONTINUE_PROMPT_AWAITC(pst, 0);
           qss = QSS_SETV(qss, 0);
           goto PlainScan;
@@ -11435,7 +12609,7 @@ static int line_is_command_terminator(char *zLine){
 ** out of the build if compiling with SQLITE_OMIT_COMPLETE.
 */
 #ifdef SQLITE_OMIT_COMPLETE
-# error the CLI application is imcompatable with SQLITE_OMIT_COMPLETE.
+# error the CLI application is incompatible with SQLITE_OMIT_COMPLETE.
 #endif
 
 /*
@@ -11452,6 +12626,91 @@ static int line_is_complete(char *zSql, int nSql){
   return rc;
 }
 
+/*
+** This function is called after processing each line of SQL in the
+** runOneSqlLine() function. Its purpose is to detect scenarios where
+** defensive mode should be automatically turned off. Specifically, when
+**
+**   1. The first line of input is "PRAGMA foreign_keys=OFF;",
+**   2. The second line of input is "BEGIN TRANSACTION;",
+**   3. The database is empty, and
+**   4. The shell is not running in --safe mode.
+** 
+** The implementation uses the ShellState.eRestoreState to maintain state:
+**
+**    0: Have not seen any SQL.
+**    1: Have seen "PRAGMA foreign_keys=OFF;".
+**    2-6: Currently running .dump transaction. If the "2" bit is set,
+**         disable DEFENSIVE when done. If "4" is set, disable DQS_DDL.
+**    7: Nothing left to do. This function becomes a no-op.
+*/
+static int doAutoDetectRestore(ShellState *p, const char *zSql){
+  int rc = SQLITE_OK;
+
+  if( p->eRestoreState<7 ){
+    switch( p->eRestoreState ){
+      case 0: {
+        const char *zExpect = "PRAGMA foreign_keys=OFF;";
+        assert( strlen(zExpect)==24 );
+        if( p->bSafeMode==0
+         && strlen(zSql)>=24
+         && memcmp(zSql, zExpect, 25)==0
+        ){
+          p->eRestoreState = 1;
+        }else{
+          p->eRestoreState = 7;
+        }
+        break;
+      };
+  
+      case 1: {
+        int bIsDump = 0;
+        const char *zExpect = "BEGIN TRANSACTION;";
+        assert( strlen(zExpect)==18 );
+        if( memcmp(zSql, zExpect, 19)==0 ){
+          /* Now check if the database is empty. */
+          const char *zQuery = "SELECT 1 FROM sqlite_schema LIMIT 1";
+          sqlite3_stmt *pStmt = 0;
+  
+          bIsDump = 1;
+          shellPrepare(p->db, &rc, zQuery, &pStmt);
+          if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+            bIsDump = 0;
+          }
+          shellFinalize(&rc, pStmt);
+        }
+        if( bIsDump && rc==SQLITE_OK ){
+          int bDefense = 0;
+          int bDqsDdl = 0;
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDefense);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, -1, &bDqsDdl);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+          sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 1, 0);
+          p->eRestoreState = (bDefense ? 2 : 0) + (bDqsDdl ? 4 : 0);
+        }else{
+          p->eRestoreState = 7;
+        }
+        break;
+      }
+  
+      default: {
+        if( sqlite3_get_autocommit(p->db) ){
+          if( (p->eRestoreState & 2) ){
+            sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
+          }
+          if( (p->eRestoreState & 4) ){
+            sqlite3_db_config(p->db, SQLITE_DBCONFIG_DQS_DDL, 0, 0);
+          }
+          p->eRestoreState = 7;
+        }
+        break;
+      }
+    }
+  }
+
+  return rc;
+}
+
 /*
 ** Run a single line of SQL.  Return the number of errors.
 */
@@ -11464,7 +12723,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
   if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0;
   BEGIN_TIMER;
   rc = shell_exec(p, zSql, &zErrMsg);
-  END_TIMER;
+  END_TIMER(p->out);
   if( rc || zErrMsg ){
     char zPrefix[100];
     const char *zErrorTail;
@@ -11488,7 +12747,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
     }else{
       sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType);
     }
-    utf8_printf(stderr, "%s %s\n", zPrefix, zErrorTail);
+    sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail);
     sqlite3_free(zErrMsg);
     zErrMsg = 0;
     return 1;
@@ -11497,13 +12756,18 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){
     sqlite3_snprintf(sizeof(zLineBuf), zLineBuf,
             "changes: %lld   total_changes: %lld",
             sqlite3_changes64(p->db), sqlite3_total_changes64(p->db));
-    raw_printf(p->out, "%s\n", zLineBuf);
+    sqlite3_fprintf(p->out, "%s\n", zLineBuf);
   }
+
+  if( doAutoDetectRestore(p, zSql) ) return 1;
   return 0;
 }
 
 static void echo_group_input(ShellState *p, const char *zDo){
-  if( ShellHasFlag(p, SHFLG_Echo) ) utf8_printf(p->out, "%s\n", zDo);
+  if( ShellHasFlag(p, SHFLG_Echo) ){
+    sqlite3_fprintf(p->out, "%s\n", zDo);
+    fflush(p->out);
+  }
 }
 
 #ifdef SQLITE_SHELL_FIDDLE
@@ -11524,7 +12788,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
   if(!z || !*z){
     return 0;
   }
-  while(*z && isspace(*z)) ++z;
+  while(*z && IsSpace(*z)) ++z;
   zBegin = z;
   for(; *z && '\n'!=*z; ++nZ, ++z){}
   if(nZ>0 && '\r'==zBegin[nZ-1]){
@@ -11548,7 +12812,7 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){
 **
 ** Return the number of errors.
 */
-static int process_input(ShellState *p){
+static int process_input(ShellState *p, const char *zSrc){
   char *zLine = 0;          /* A single input line */
   char *zSql = 0;           /* Accumulated SQL text */
   i64 nLine;                /* Length of current line */
@@ -11561,8 +12825,8 @@ static int process_input(ShellState *p){
 
   if( p->inputNesting==MAX_INPUT_NESTING ){
     /* This will be more informative in a later version. */
-    utf8_printf(stderr,"Input nesting limit (%d) reached at line %d."
-                " Check recursion.\n", MAX_INPUT_NESTING, p->lineno);
+    sqlite3_fprintf(stderr,"%s: Input nesting limit (%d) reached at line %lld."
+          " Check recursion.\n", zSrc, MAX_INPUT_NESTING, p->lineno);
     return 1;
   }
   ++p->inputNesting;
@@ -11573,7 +12837,7 @@ static int process_input(ShellState *p){
     zLine = one_input_line(p->in, zLine, nSql>0);
     if( zLine==0 ){
       /* End of input */
-      if( p->in==0 && stdin_is_interactive ) printf("\n");
+      if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out);
       break;
     }
     if( seenInterrupt ){
@@ -11627,7 +12891,15 @@ static int process_input(ShellState *p){
       memcpy(zSql+nSql, zLine, nLine+1);
       nSql += nLine;
     }
-    if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){
+    if( nSql>0x7fff0000 ){
+      char zSize[100];
+      sqlite3_snprintf(sizeof(zSize),zSize,"%,lld",nSql);
+      sqlite3_fprintf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n",
+                      zSrc, startline, zSize);
+      nSql = 0;
+      errCnt++;
+      break;
+    }else if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){
       echo_group_input(p, zSql);
       errCnt += runOneSqlLine(p, zSql, p->in, startline);
       CONTINUE_PROMPT_RESET;
@@ -11728,83 +13000,106 @@ static char *find_home_dir(int clearFlag){
 }
 
 /*
-** On non-Windows platforms, look for $XDG_CONFIG_HOME.
-** If ${XDG_CONFIG_HOME}/sqlite3/sqliterc is found, return
-** the path to it, else return 0. The result is cached for
-** subsequent calls.
+** On non-Windows platforms, look for:
+**
+** - ${zEnvVar}/${zBaseName}
+** - ${HOME}/${zSubdir}/${zBaseName}
+**
+** $zEnvVar is intended to be the name of an XDG_... environment
+** variable, e.g. XDG_CONFIG_HOME or XDG_STATE_HOME.  If zEnvVar is
+** NULL or getenv(zEnvVar) is NULL then fall back to the second
+** option. If the selected option is not found in the filesystem,
+** return 0.
+**
+** zSubdir may be NULL or empty, in which case ${HOME}/${zBaseName}
+** becomes the fallback.
+**
+** Both zSubdir and zBaseName may contain subdirectory parts. zSubdir
+** will conventionally be ".config" or ".local/state", which, not
+** coincidentally, is the typical subdir of the corresponding XDG_...
+** var with the XDG var's $HOME prefix.
+**
+** The returned string is obtained from sqlite3_malloc() and should be
+** sqlite3_free()'d by the caller.
 */
-static const char *find_xdg_config(void){
+static char *find_xdg_file(const char *zEnvVar, const char *zSubdir,
+                           const char *zBaseName){
 #if defined(_WIN32) || defined(WIN32) || defined(_WIN32_WCE) \
      || defined(__RTP__) || defined(_WRS_KERNEL)
   return 0;
 #else
-  static int alreadyTried = 0;
-  static char *zConfig = 0;
-  const char *zXdgHome;
+  char *zConfigFile = 0;
+  const char *zXdgDir;
 
-  if( alreadyTried!=0 ){
-    return zConfig;
-  }
-  alreadyTried = 1;
-  zXdgHome = getenv("XDG_CONFIG_HOME");
-  if( zXdgHome==0 ){
-    return 0;
+  zXdgDir = zEnvVar ? getenv(zEnvVar) : 0;
+  if( zXdgDir ){
+    zConfigFile = sqlite3_mprintf("%s/%s", zXdgDir, zBaseName);
+  }else{
+    const char * zHome = find_home_dir(0);
+    if( zHome==0 ) return 0;
+    zConfigFile = (zSubdir && *zSubdir)
+      ? sqlite3_mprintf("%s/%s/%s", zHome, zSubdir, zBaseName)
+      : sqlite3_mprintf("%s/%s", zHome, zBaseName);
   }
-  zConfig = sqlite3_mprintf("%s/sqlite3/sqliterc", zXdgHome);
-  shell_check_oom(zConfig);
-  if( access(zConfig,0)!=0 ){
-    sqlite3_free(zConfig);
-    zConfig = 0;
+  shell_check_oom(zConfigFile);
+  if( access(zConfigFile,0)!=0 ){
+    sqlite3_free(zConfigFile);
+    zConfigFile = 0;
   }
-  return zConfig;
+  return zConfigFile;
 #endif
 }
 
 /*
-** Read input from the file given by sqliterc_override.  Or if that
-** parameter is NULL, take input from the first of find_xdg_config()
-** or ~/.sqliterc which is found.
+** Read input from the file sqliterc_override.  If that parameter is
+** NULL, take it from find_xdg_file(), if found, or fall back to
+** ~/.sqliterc.
 **
-** Returns the number of errors.
+** Failure to read the config is only considered a failure if
+** sqliterc_override is not NULL, in which case this function may emit
+** a warning or, if ::bail_on_error is true, fail fatally if the file
+** named by sqliterc_override is not found.
 */
 static void process_sqliterc(
   ShellState *p,                  /* Configuration data */
   const char *sqliterc_override   /* Name of config file. NULL to use default */
 ){
   char *home_dir = NULL;
-  const char *sqliterc = sqliterc_override;
-  char *zBuf = 0;
+  char *sqliterc = (char*)sqliterc_override;
   FILE *inSaved = p->in;
-  int savedLineno = p->lineno;
+  i64 savedLineno = p->lineno;
 
   if( sqliterc == NULL ){
-    sqliterc = find_xdg_config();
+    sqliterc = find_xdg_file("XDG_CONFIG_HOME",
+                             ".config",
+                             "sqlite3/sqliterc");
   }
   if( sqliterc == NULL ){
     home_dir = find_home_dir(0);
     if( home_dir==0 ){
-      raw_printf(stderr, "-- warning: cannot find home directory;"
-                      " cannot read ~/.sqliterc\n");
+      eputz("-- warning: cannot find home directory;"
+            " cannot read ~/.sqliterc\n");
       return;
     }
-    zBuf = sqlite3_mprintf("%s/.sqliterc",home_dir);
-    shell_check_oom(zBuf);
-    sqliterc = zBuf;
+    sqliterc = sqlite3_mprintf("%s/.sqliterc",home_dir);
+    shell_check_oom(sqliterc);
   }
-  p->in = fopen(sqliterc,"rb");
+  p->in = sqliterc ? sqlite3_fopen(sqliterc,"rb") : 0;
   if( p->in ){
     if( stdin_is_interactive ){
-      utf8_printf(stderr,"-- Loading resources from %s\n",sqliterc);
+      sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc);
     }
-    if( process_input(p) && bail_on_error ) exit(1);
+    if( process_input(p, sqliterc) && bail_on_error ) exit(1);
     fclose(p->in);
   }else if( sqliterc_override!=0 ){
-    utf8_printf(stderr,"cannot open: \"%s\"\n", sqliterc);
+    sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc);
     if( bail_on_error ) exit(1);
   }
   p->in = inSaved;
   p->lineno = savedLineno;
-  sqlite3_free(zBuf);
+  if( sqliterc != sqliterc_override ){
+    sqlite3_free(sqliterc);
+  }
 }
 
 /*
@@ -11827,6 +13122,7 @@ static const char zOptions[] =
   "   -deserialize         open the database using sqlite3_deserialize()\n"
 #endif
   "   -echo                print inputs before execution\n"
+  "   -escape T            ctrl-char escape; T is one of: symbol, ascii, off\n"
   "   -init FILENAME       read/process named file\n"
   "   -[no]header          turn headers on or off\n"
 #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
@@ -11834,6 +13130,7 @@ static const char zOptions[] =
 #endif
   "   -help                show this message\n"
   "   -html                set output mode to HTML\n"
+  "   -ifexists            only open if database already exists\n"
   "   -interactive         force interactive I/O\n"
   "   -json                set output mode to 'json'\n"
   "   -line                set output mode to 'line'\n"
@@ -11851,8 +13148,10 @@ static const char zOptions[] =
   "   -newline SEP         set output row separator. Default: '\\n'\n"
   "   -nofollow            refuse to open symbolic links to database files\n"
   "   -nonce STRING        set the safe-mode escape nonce\n"
+  "   -no-rowid-in-view    Disable rowid-in-view using sqlite3_config()\n"
   "   -nullvalue TEXT      set text string for NULL values. Default ''\n"
   "   -pagecache SIZE N    use N slots of SZ bytes each for page cache memory\n"
+  "   -pcachetrace         trace all page cache operations\n"
   "   -quote               set output mode to 'quote'\n"
   "   -readonly            open the database read-only\n"
   "   -safe                enable safe-mode\n"
@@ -11864,29 +13163,23 @@ static const char zOptions[] =
   "   -table               set output mode to 'table'\n"
   "   -tabs                set output mode to 'tabs'\n"
   "   -unsafe-testing      allow unsafe commands and modes for testing\n"
-#if SHELL_WIN_UTF8_OPT
-  "   -utf8                setup interactive console code page for UTF-8\n"
-#endif
   "   -version             show SQLite version\n"
   "   -vfs NAME            use NAME as the default VFS\n"
-#ifdef SQLITE_ENABLE_VFSTRACE
   "   -vfstrace            enable tracing of all VFS calls\n"
-#endif
 #ifdef SQLITE_HAVE_ZLIB
   "   -zip                 open the file as a ZIP Archive\n"
 #endif
 ;
 static void usage(int showDetail){
-  utf8_printf(stderr,
-      "Usage: %s [OPTIONS] [FILENAME [SQL]]\n"
-      "FILENAME is the name of an SQLite database. A new database is created\n"
-      "if the file does not previously exist. Defaults to :memory:.\n", Argv0);
+  sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n"
+       "FILENAME is the name of an SQLite database. A new database is created\n"
+       "if the file does not previously exist. Defaults to :memory:.\n", Argv0);
   if( showDetail ){
-    utf8_printf(stderr, "OPTIONS include:\n%s", zOptions);
+    sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions);
   }else{
-    raw_printf(stderr, "Use the -help option for additional information\n");
+    eputz("Use the -help option for additional information\n");
   }
-  exit(1);
+  exit(0);
 }
 
 /*
@@ -11895,8 +13188,8 @@ static void usage(int showDetail){
 */
 static void verify_uninitialized(void){
   if( sqlite3_config(-1)==SQLITE_MISUSE ){
-    utf8_printf(stdout, "WARNING: attempt to configure SQLite after"
-                        " initialization.\n");
+    sputz(stdout, "WARNING: attempt to configure SQLite after"
+          " initialization.\n");
   }
 }
 
@@ -11907,6 +13200,9 @@ static void main_init(ShellState *data) {
   memset(data, 0, sizeof(*data));
   data->normalMode = data->cMode = data->mode = MODE_List;
   data->autoExplain = 1;
+#ifdef _WIN32
+  data->crlfMode = 1;
+#endif
   data->pAuxDb = &data->aAuxDb[0];
   memcpy(data->colSeparator,SEP_Column, 2);
   memcpy(data->rowSeparator,SEP_Row, 2);
@@ -11925,7 +13221,7 @@ static void main_init(ShellState *data) {
 /*
 ** Output text to the console in a font that attracts extra attention.
 */
-#ifdef _WIN32
+#if defined(_WIN32) || defined(WIN32)
 static void printBold(const char *zText){
 #if !SQLITE_OS_WINRT
   HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE);
@@ -11935,14 +13231,14 @@ static void printBold(const char *zText){
          FOREGROUND_RED|FOREGROUND_INTENSITY
   );
 #endif
-  printf("%s", zText);
+  sputz(stdout, zText);
 #if !SQLITE_OS_WINRT
   SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes);
 #endif
 }
 #else
 static void printBold(const char *zText){
-  printf("\033[1m%s\033[0m", zText);
+  sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText);
 }
 #endif
 
@@ -11952,15 +13248,24 @@ static void printBold(const char *zText){
 */
 static char *cmdline_option_value(int argc, char **argv, int i){
   if( i==argc ){
-    utf8_printf(stderr, "%s: Error: missing argument to %s\n",
-            argv[0], argv[argc-1]);
+    sqlite3_fprintf(stderr,
+            "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]);
     exit(1);
   }
   return argv[i];
 }
 
 static void sayAbnormalExit(void){
-  if( seenInterrupt ) fprintf(stderr, "Program interrupted.\n");
+  if( seenInterrupt ) eputz("Program interrupted.\n");
+}
+
+/* Routine to output from vfstrace
+*/
+static int vfstraceOut(const char *z, void *pArg){
+  ShellState *p = (ShellState*)pArg;
+  sqlite3_fputs(z, p->out);
+  fflush(p->out);
+  return 1;
 }
 
 #ifndef SQLITE_SHELL_IS_UTF8
@@ -11998,6 +13303,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   int readStdin = 1;
   int nCmd = 0;
   int nOptsEnd = argc;
+  int bEnableVfstrace = 0;
   char **azCmd = 0;
   const char *zVfs = 0;           /* Value of -vfs command-line option */
 #if !SQLITE_SHELL_IS_UTF8
@@ -12013,9 +13319,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #else
   stdin_is_interactive = isatty(0);
   stdout_is_console = isatty(1);
-#endif
-#if SHELL_WIN_UTF8_OPT
-  atexit(console_restore); /* Needs revision for CLI as library call */
 #endif
   atexit(sayAbnormalExit);
 #ifdef SQLITE_DEBUG
@@ -12024,10 +13327,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #if !defined(_WIN32_WCE)
   if( getenv("SQLITE_DEBUG_BREAK") ){
     if( isatty(0) && isatty(2) ){
-      fprintf(stderr,
-          "attach debugger to process %d and press any key to continue.\n",
-          GETPID());
-      fgetc(stdin);
+      char zLine[100];
+      sqlite3_fprintf(stderr,
+            "attach debugger to process %d and press ENTER to continue...",
+            GETPID());
+      if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0
+       && cli_strcmp(zLine,"stop")==0
+      ){
+        exit(1);
+      }
     }else{
 #if defined(_WIN32) || defined(WIN32)
 #if SQLITE_OS_WINRT
@@ -12046,14 +13354,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   signal(SIGINT, interrupt_handler);
 #elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE)
   if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){
-    fprintf(stderr, "No ^C handler.\n");
+    eputz("No ^C handler.\n");
   }
 #endif
 
 #if USE_SYSTEM_SQLITE+0!=1
   if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){
-    utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n",
-            sqlite3_sourceid(), SQLITE_SOURCE_ID);
+    sqlite3_fprintf(stderr,
+          "SQLite header and source version mismatch\n%s\n%s\n",
+          sqlite3_sourceid(), SQLITE_SOURCE_ID);
     exit(1);
   }
 #endif
@@ -12102,8 +13411,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 
   /* Do an initial pass through the command-line argument to locate
   ** the name of the database file, the name of the initialization file,
-  ** the size of the alternative malloc heap,
-  ** and the first command to execute.
+  ** the size of the alternative malloc heap, options affecting commands
+  ** or SQL run from the command line, and the first command to execute.
   */
 #ifndef SQLITE_SHELL_FIDDLE
   verify_uninitialized();
@@ -12115,7 +13424,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       if( data.aAuxDb->zDbFilename==0 ){
         data.aAuxDb->zDbFilename = z;
       }else{
-        /* Excesss arguments are interpreted as SQL (or dot-commands) and
+        /* Excess arguments are interpreted as SQL (or dot-commands) and
         ** mean that nothing is read from stdin */
         readStdin = 0;
         nCmd++;
@@ -12137,12 +13446,19 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       (void)cmdline_option_value(argc, argv, ++i);
     }else if( cli_strcmp(z,"-init")==0 ){
       zInitFile = cmdline_option_value(argc, argv, ++i);
+    }else if( cli_strcmp(z,"-interactive")==0 ){
     }else if( cli_strcmp(z,"-batch")==0 ){
       /* Need to check for batch mode here to so we can avoid printing
       ** informational messages (like from process_sqliterc) before
       ** we do the actual processing of arguments later in a second pass.
       */
       stdin_is_interactive = 0;
+    }else if( cli_strcmp(z,"-utf8")==0 ){
+    }else if( cli_strcmp(z,"-no-utf8")==0 ){
+    }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){
+      int val = 0;
+      sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW, &val);
+      assert( val==0 );
     }else if( cli_strcmp(z,"-heap")==0 ){
 #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5)
       const char *zSize;
@@ -12159,12 +13475,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-pagecache")==0 ){
       sqlite3_int64 n, sz;
       sz = integerValue(cmdline_option_value(argc,argv,++i));
-      if( sz>70000 ) sz = 70000;
+      if( sz>65536 ) sz = 65536;
       if( sz<0 ) sz = 0;
       n = integerValue(cmdline_option_value(argc,argv,++i));
       if( sz>0 && n>0 && 0xffffffffffffLL/sz<n ){
         n = 0xffffffffffffLL/sz;
       }
+      if( sz>0 && (sz & (sz-1))==0 ){
+        /* If SIZE is a power of two, round it up by the PCACHE_HDRSZ */
+        int szHdr = 0;
+        sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &szHdr);
+        sz += szHdr;
+        sqlite3_fprintf(stdout, "Page cache size increased to %d to accommodate"
+                        " the %d-byte headers\n", (int)sz, szHdr);
+      }
       verify_uninitialized();
       sqlite3_config(SQLITE_CONFIG_PAGECACHE,
                     (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n);
@@ -12177,7 +13501,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       if( n<0 ) n = 0;
       verify_uninitialized();
       sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n);
-      if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside;
+      if( (i64)sz*(i64)n==0 ) data.shellFlgs &= ~SHFLG_Lookaside;
     }else if( cli_strcmp(z,"-threadsafe")==0 ){
       int n;
       n = (int)integerValue(cmdline_option_value(argc,argv,++i));
@@ -12187,20 +13511,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
          case 2:  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);   break;
          default: sqlite3_config(SQLITE_CONFIG_SERIALIZED);    break;
       }
-#ifdef SQLITE_ENABLE_VFSTRACE
     }else if( cli_strcmp(z,"-vfstrace")==0 ){
-      extern int vfstrace_register(
-         const char *zTraceName,
-         const char *zOldVfsName,
-         int (*xOut)(const char*,void*),
-         void *pOutArg,
-         int makeDefault
-      );
-      vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1);
-#endif
+      bEnableVfstrace = 1;
 #ifdef SQLITE_ENABLE_MULTIPLEX
     }else if( cli_strcmp(z,"-multiplex")==0 ){
-      extern int sqlite3_multiple_initialize(const char*,int);
+      extern int sqlite3_multiplex_initialize(const char*,int);
       sqlite3_multiplex_initialize(0, 1);
 #endif
     }else if( cli_strcmp(z,"-mmap")==0 ){
@@ -12228,9 +13543,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       data.szMax = integerValue(argv[++i]);
 #endif
     }else if( cli_strcmp(z,"-readonly")==0 ){
-      data.openMode = SHELL_OPEN_READONLY;
+      data.openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
+      data.openFlags |= SQLITE_OPEN_READONLY;
     }else if( cli_strcmp(z,"-nofollow")==0 ){
-      data.openFlags = SQLITE_OPEN_NOFOLLOW;
+      data.openFlags |= SQLITE_OPEN_NOFOLLOW;
+    }else if( cli_strcmp(z,"-exclusive")==0 ){  /* UNDOCUMENTED */
+      data.openFlags |= SQLITE_OPEN_EXCLUSIVE;
+    }else if( cli_strcmp(z,"-ifexists")==0 ){
+      data.openFlags &= ~(SQLITE_OPEN_CREATE);
+      if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE;
 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
     }else if( cli_strncmp(z, "-A",2)==0 ){
       /* All remaining command-line arguments are passed to the ".archive"
@@ -12239,19 +13560,24 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #endif
     }else if( cli_strcmp(z, "-memtrace")==0 ){
       sqlite3MemTraceActivate(stderr);
+    }else if( cli_strcmp(z, "-pcachetrace")==0 ){
+      sqlite3PcacheTraceActivate(stderr);
     }else if( cli_strcmp(z,"-bail")==0 ){
       bail_on_error = 1;
     }else if( cli_strcmp(z,"-nonce")==0 ){
       free(data.zNonce);
-      data.zNonce = strdup(argv[++i]);
+      data.zNonce = strdup(cmdline_option_value(argc, argv, ++i));
     }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
       ShellSetFlag(&data,SHFLG_TestingMode);
     }else if( cli_strcmp(z,"-safe")==0 ){
       /* no-op - catch this on the second pass */
+    }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){
+      /* skip over the argument */
+      i++;
     }
   }
 #ifndef SQLITE_SHELL_FIDDLE
-  verify_uninitialized();
+  if( !bEnableVfstrace ) verify_uninitialized();
 #endif
 
 
@@ -12275,7 +13601,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     if( pVfs ){
       sqlite3_vfs_register(pVfs, 1);
     }else{
-      utf8_printf(stderr, "no such VFS: \"%s\"\n", zVfs);
+      sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs);
       exit(1);
     }
   }
@@ -12285,11 +13611,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     data.pAuxDb->zDbFilename = ":memory:";
     warnInmemoryDb = argc==1;
 #else
-    utf8_printf(stderr,"%s: Error: no database filename specified\n", Argv0);
+    sqlite3_fprintf(stderr,
+                    "%s: Error: no database filename specified\n", Argv0);
     return 1;
 #endif
   }
   data.out = stdout;
+  if( bEnableVfstrace ){
+    vfstrace_register("trace",0,vfstraceOut, &data, 1);
+  }
 #ifndef SQLITE_SHELL_FIDDLE
   sqlite3_appendvfs_init(0,0,0);
 #endif
@@ -12343,6 +13673,25 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-csv")==0 ){
       data.mode = MODE_Csv;
       memcpy(data.colSeparator,",",2);
+    }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){
+      /* See similar code at tag-20250224-1 */
+      const char *zEsc = argv[++i];
+      int k;
+      for(k=0; k<ArraySize(shell_EscModeNames); k++){
+        if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){
+          data.eEscMode = k;
+          break;
+        }
+      }
+      if( k>=ArraySize(shell_EscModeNames) ){
+        sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\""
+                                " - choices:", zEsc);
+        for(k=0; k<ArraySize(shell_EscModeNames); k++){
+          sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]);
+        }
+        sqlite3_fprintf(stderr, "\n");
+        exit(1);
+      }
 #ifdef SQLITE_HAVE_ZLIB
     }else if( cli_strcmp(z,"-zip")==0 ){
       data.openMode = SHELL_OPEN_ZIPFILE;
@@ -12356,9 +13705,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       data.szMax = integerValue(argv[++i]);
 #endif
     }else if( cli_strcmp(z,"-readonly")==0 ){
-      data.openMode = SHELL_OPEN_READONLY;
+      data.openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE);
+      data.openFlags |= SQLITE_OPEN_READONLY;
     }else if( cli_strcmp(z,"-nofollow")==0 ){
       data.openFlags |= SQLITE_OPEN_NOFOLLOW;
+    }else if( cli_strcmp(z,"-exclusive")==0 ){  /* UNDOCUMENTED */
+      data.openFlags |= SQLITE_OPEN_EXCLUSIVE;
+    }else if( cli_strcmp(z,"-ifexists")==0 ){
+      data.openFlags &= ~(SQLITE_OPEN_CREATE);
+      if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE;
     }else if( cli_strcmp(z,"-ascii")==0 ){
       data.mode = MODE_Ascii;
       sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit);
@@ -12406,22 +13761,29 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
 #ifdef SQLITE_HAS_CODEC
       extern char* sqlcipher_version();
       char *sqlcipher_ver = sqlcipher_version();
-      printf("%s %s", sqlite3_libversion(), sqlite3_sourceid());
-      printf(" (SQLCipher %s)\n", sqlcipher_ver);
+      sqlite3_fprintf(stdout, "%s %s (%d-bit)",
+            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
+      sqlite3_fprintf(stdout, " (SQLCipher %s)\n", sqlcipher_ver);
       sqlite3_free(sqlcipher_ver);
 #else
-      printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid());
+      sqlite3_fprintf(stdout, "%s %s (%d-bit)\n",
+            sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*));
 #endif
 /* END SQLCIPHER */
       return 0;
     }else if( cli_strcmp(z,"-interactive")==0 ){
+      /* Need to check for interactive override here to so that it can
+      ** affect console setup (for Windows only) and testing thereof.
+      */
       stdin_is_interactive = 1;
     }else if( cli_strcmp(z,"-batch")==0 ){
-      stdin_is_interactive = 0;
+      /* already handled */
     }else if( cli_strcmp(z,"-utf8")==0 ){
-#if SHELL_WIN_UTF8_OPT
-      console_utf8 = 1;
-#endif /* SHELL_WIN_UTF8_OPT */
+      /* already handled */
+    }else if( cli_strcmp(z,"-no-utf8")==0 ){
+      /* already handled */
+    }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){
+      /* already handled */
     }else if( cli_strcmp(z,"-heap")==0 ){
       i++;
     }else if( cli_strcmp(z,"-pagecache")==0 ){
@@ -12436,16 +13798,16 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       i++;
     }else if( cli_strcmp(z,"-memtrace")==0 ){
       i++;
+    }else if( cli_strcmp(z,"-pcachetrace")==0 ){
+      i++;
 #ifdef SQLITE_ENABLE_SORTER_REFERENCES
     }else if( cli_strcmp(z,"-sorterref")==0 ){
       i++;
 #endif
     }else if( cli_strcmp(z,"-vfs")==0 ){
       i++;
-#ifdef SQLITE_ENABLE_VFSTRACE
     }else if( cli_strcmp(z,"-vfstrace")==0 ){
       i++;
-#endif
 #ifdef SQLITE_ENABLE_MULTIPLEX
     }else if( cli_strcmp(z,"-multiplex")==0 ){
       i++;
@@ -12466,19 +13828,21 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
         open_db(&data, 0);
         rc = shell_exec(&data, z, &zErrMsg);
         if( zErrMsg!=0 ){
-          utf8_printf(stderr,"Error: %s\n", zErrMsg);
+          shellEmitError(zErrMsg);
+          sqlite3_free(zErrMsg);
           if( bail_on_error ) return rc!=0 ? rc : 1;
         }else if( rc!=0 ){
-          utf8_printf(stderr,"Error: unable to process SQL \"%s\"\n", z);
+          sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z);
           if( bail_on_error ) return rc;
         }
       }
 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB)
     }else if( cli_strncmp(z, "-A", 2)==0 ){
       if( nCmd>0 ){
-        utf8_printf(stderr, "Error: cannot mix regular SQL or dot-commands"
-                            " with \"%s\"\n", z);
-        return 1;
+        sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands"
+              " with \"%s\"\n", z);
+        rc = 1;
+        goto shell_main_exit;
       }
       open_db(&data, OPEN_DB_ZIPFILE);
       if( z[2] ){
@@ -12495,20 +13859,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }else if( cli_strcmp(z,"-unsafe-testing")==0 ){
       /* Acted upon in first pass. */
     }else{
-      utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
-      raw_printf(stderr,"Use -help for a list of options.\n");
+      sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z);
+      eputz("Use -help for a list of options.\n");
       return 1;
     }
     data.cMode = data.mode;
   }
-#if SHELL_WIN_UTF8_OPT
-  if( console_utf8 && stdin_is_interactive ){
-    console_prepare();
-  }else{
-    setBinaryMode(stdin, 0);
-    console_utf8 = 0;
-  }
-#endif
 
   if( !readStdin ){
     /* Run all arguments that do not begin with '-' as if they were separate
@@ -12516,25 +13872,26 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     ** the database filename.
     */
     for(i=0; i<nCmd; i++){
+      echo_group_input(&data, azCmd[i]);
       if( azCmd[i][0]=='.' ){
         rc = do_meta_command(azCmd[i], &data);
         if( rc ){
-          free(azCmd);
-          return rc==2 ? 0 : rc;
+          if( rc==2 ) rc = 0;
+          goto shell_main_exit;
         }
       }else{
         open_db(&data, 0);
-        echo_group_input(&data, azCmd[i]);
         rc = shell_exec(&data, azCmd[i], &zErrMsg);
         if( zErrMsg || rc ){
           if( zErrMsg!=0 ){
-            utf8_printf(stderr,"Error: %s\n", zErrMsg);
+            shellEmitError(zErrMsg);
           }else{
-            utf8_printf(stderr,"Error: unable to process SQL: %s\n", azCmd[i]);
+            sqlite3_fprintf(stderr,
+                            "Error: unable to process SQL: %s\n", azCmd[i]);
           }
           sqlite3_free(zErrMsg);
-          free(azCmd);
-          return rc!=0 ? rc : 1;
+          if( rc==0 ) rc = 1;
+          goto shell_main_exit;
         }
       }
     }
@@ -12544,62 +13901,73 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     if( stdin_is_interactive ){
       char *zHome;
       char *zHistory;
-      int nHistory;
 /* BEGIN SQLCIPHER */
 #ifdef SQLITE_HAS_CODEC
+      {
       extern char* sqlcipher_version();
       char *sqlcipher_ver = sqlcipher_version();
-      printf(
-        "SQLite version %s %.19s" /*extra-version-info*/
-        " (SQLCipher %s)\n" /*sqlcipher version info*/
-        "Enter \".help\" for usage hints.\n",
-        sqlite3_libversion(), sqlite3_sourceid(), sqlcipher_ver
-      );
+      sqlite3_fprintf(stdout,
+            "SQLite version %s %.19s" /*extra-version-info*/
+            " (SQLCipher %s)\n" /*sqlcipher version info*/
+            "Enter \".help\" for usage hints.\n",
+            sqlite3_libversion(), sqlite3_sourceid(), sqlcipher_ver);
       sqlite3_free(sqlcipher_ver);
+      }
 #else
-      printf(
-        "SQLite version %s %.19s\n" /*extra-version-info*/
-        "Enter \".help\" for usage hints.\n",
-        sqlite3_libversion(), sqlite3_sourceid()
-      );
+      sqlite3_fprintf(stdout,
+            "SQLite version %s %.19s\n" /*extra-version-info*/
+            "Enter \".help\" for usage hints.\n",
+            sqlite3_libversion(), sqlite3_sourceid());
 #endif
 /* END SQLCIPHER */
       if( warnInmemoryDb ){
-        printf("Connected to a ");
+        sputz(stdout, "Connected to a ");
         printBold("transient in-memory database");
-        printf(".\nUse \".open FILENAME\" to reopen on a "
-               "persistent database.\n");
+        sputz(stdout, ".\nUse \".open FILENAME\" to reopen on a"
+              " persistent database.\n");
       }
       zHistory = getenv("SQLITE_HISTORY");
       if( zHistory ){
-        zHistory = strdup(zHistory);
-      }else if( (zHome = find_home_dir(0))!=0 ){
-        nHistory = strlen30(zHome) + 20;
-        if( (zHistory = malloc(nHistory))!=0 ){
-          sqlite3_snprintf(nHistory, zHistory,"%s/.sqlite_history", zHome);
+        zHistory = sqlite3_mprintf("%s", zHistory);
+        shell_check_oom(zHistory);
+      }else{
+        zHistory = find_xdg_file("XDG_STATE_HOME",
+                                 ".local/state",
+                                 "sqlite_history");
+        if( 0==zHistory && (zHome = find_home_dir(0))!=0 ){
+          zHistory = sqlite3_mprintf("%s/.sqlite_history", zHome);
+          shell_check_oom(zHistory);
         }
       }
       if( zHistory ){ shell_read_history(zHistory); }
-#if HAVE_READLINE || HAVE_EDITLINE
+#if (HAVE_READLINE || HAVE_EDITLINE) && !defined(SQLITE_OMIT_READLINE_COMPLETION)
       rl_attempted_completion_function = readline_completion;
-#elif HAVE_LINENOISE
+#elif HAVE_LINENOISE==1
       linenoiseSetCompletionCallback(linenoise_completion);
+#elif HAVE_LINENOISE==2
+      linenoiseSetCompletionCallback(linenoise_completion, NULL);
 #endif
       data.in = 0;
-      rc = process_input(&data);
+      rc = process_input(&data, "<stdin>");
       if( zHistory ){
         shell_stifle_history(2000);
         shell_write_history(zHistory);
-        free(zHistory);
+        sqlite3_free(zHistory);
       }
     }else{
       data.in = stdin;
-      rc = process_input(&data);
+      rc = process_input(&data, "<stdin>");
     }
   }
 #ifndef SQLITE_SHELL_FIDDLE
   /* In WASM mode we have to leave the db state in place so that
   ** client code can "push" SQL into it after this call returns. */
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
+  if( data.expert.pExpert ){
+    expertFinish(&data, 1, 0);
+  }
+#endif
+ shell_main_exit:
   free(azCmd);
   set_table_name(&data, 0);
   if( data.db ){
@@ -12626,13 +13994,18 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
   /* Clear the global data structure so that valgrind will detect memory
   ** leaks */
   memset(&data, 0, sizeof(data));
+  if( bEnableVfstrace ){
+    vfstrace_unregister("trace");
+  }
 #ifdef SQLITE_DEBUG
   if( sqlite3_memory_used()>mem_main_enter ){
-    utf8_printf(stderr, "Memory leaked: %u bytes\n",
-                (unsigned int)(sqlite3_memory_used()-mem_main_enter));
+    sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n",
+          (unsigned int)(sqlite3_memory_used()-mem_main_enter));
   }
 #endif
-#endif /* !SQLITE_SHELL_FIDDLE */
+#else /* SQLITE_SHELL_FIDDLE... */
+  shell_main_exit:
+#endif
   return rc;
 }
 
@@ -12666,7 +14039,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){
 
 /* Only for emcc experimentation purposes. */
 sqlite3 * fiddle_db_arg(sqlite3 *arg){
-    printf("fiddle_db_arg(%p)\n", (const void*)arg);
+    sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg);
     return arg;
 }
 
@@ -12692,12 +14065,22 @@ const char * fiddle_db_filename(const char * zDbName){
 
 /*
 ** Completely wipes out the contents of the currently-opened database
-** but leaves its storage intact for reuse.
+** but leaves its storage intact for reuse. If any transactions are
+** active, they are forcibly rolled back.
 */
 void fiddle_reset_db(void){
   if( globalDb ){
-    int rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
-    if( 0==rc ) rc = sqlite3_exec(globalDb, "VACUUM", 0, 0, 0);
+    int rc;
+    while( sqlite3_txn_state(globalDb,0)>0 ){
+      /*
+      ** Resolve problem reported in
+      ** https://sqlite.org/forum/forumpost/0b41a25d65
+      */
+      sqlite3_fputs("Rolling back in-progress transaction.\n", stdout);
+      sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0);
+    }
+    rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
+    if( 0==rc ) sqlite3_exec(globalDb, "VACUUM", 0, 0, 0);
     sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0);
   }
 }
@@ -12755,7 +14138,7 @@ void fiddle_exec(const char * zSql){
     if('.'==*zSql) puts(zSql);
     shellState.wasm.zInput = zSql;
     shellState.wasm.zPos = zSql;
-    process_input(&shellState);
+    process_input(&shellState, "<stdin>");
     shellState.wasm.zInput = shellState.wasm.zPos = 0;
   }
 }
diff --git a/src/sqlcipher.c b/src/sqlcipher.c
new file mode 100644
index 0000000000..889e0b7259
--- /dev/null
+++ b/src/sqlcipher.c
@@ -0,0 +1,3889 @@
+/* 
+** SQLCipher
+** http://zetetic.net
+** 
+** Copyright (c) 2008-2024, ZETETIC LLC
+** All rights reserved.
+** 
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are met:
+**     * Redistributions of source code must retain the above copyright
+**       notice, this list of conditions and the following disclaimer.
+**     * Redistributions in binary form must reproduce the above copyright
+**       notice, this list of conditions and the following disclaimer in the
+**       documentation and/or other materials provided with the distribution.
+**     * Neither the name of the ZETETIC LLC nor the
+**       names of its contributors may be used to endorse or promote products
+**       derived from this software without specific prior written permission.
+** 
+** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY
+** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY
+** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**  
+*/
+/* BEGIN SQLCIPHER */
+#ifdef SQLITE_HAS_CODEC
+
+#if !defined(SQLCIPHER_OMIT_LOG_DEVICE)
+#if defined(__ANDROID__)
+#include <android/log.h>
+#elif defined(__APPLE__)
+#include <TargetConditionals.h>
+#include <os/log.h>
+#endif
+#endif
+
+#include <time.h>
+
+#if defined(_WIN32) || defined(SQLITE_OS_WINRT)
+#include <windows.h> /*  amalgamator: dontcache */
+#else
+#include <sys/time.h> /* amalgamator: dontcache */
+#endif
+
+#ifndef OMIT_MEMLOCK
+#if defined(__unix__) || defined(__APPLE__) || defined(_AIX)
+#include <errno.h> /* amalgamator: dontcache */
+#include <unistd.h> /* amalgamator: dontcache */
+#include <sys/resource.h> /* amalgamator: dontcache */
+#include <sys/mman.h> /* amalgamator: dontcache */
+#endif
+#endif
+
+#include <assert.h>
+#include "sqlcipher.h"
+#include "btreeInt.h"
+#include "pager.h"
+#include "vdbeInt.h"
+
+#if !defined(SQLITE_EXTRA_INIT) || !defined(SQLITE_EXTRA_SHUTDOWN)
+#error "SQLCipher must be compiled with -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown"
+#endif
+
+#if !defined(SQLITE_THREADSAFE) || !(SQLITE_THREADSAFE == 1 || SQLITE_THREADSAFE == 2)
+#error "SQLCipher must be compiled with -DSQLITE_THREADSAFE=<1 or 2>"
+#endif
+
+#if !defined(SQLITE_TEMP_STORE) || SQLITE_TEMP_STORE == 0 || SQLITE_TEMP_STORE == 1
+#error "SQLCipher must be compiled with -DSQLITE_TEMP_STORE=<2 or 3>"
+#endif
+
+/* extensions defined in pager.c */ 
+void *sqlcipherPagerGetCodec(Pager*);
+void sqlcipherPagerSetCodec(Pager*, void *(*)(void*,void*,Pgno,int),  void (*)(void*,int,int),  void (*)(void*), void *);
+int sqlite3pager_is_sj_pgno(Pager*, Pgno);
+void sqlite3pager_error(Pager*, int);
+void sqlite3pager_reset(Pager *pPager);
+/* end extensions defined in pager.c */
+
+#if !defined (SQLCIPHER_CRYPTO_CC) \
+   && !defined (SQLCIPHER_CRYPTO_OPENSSL) \
+   && !defined (SQLCIPHER_CRYPTO_CUSTOM)
+#define SQLCIPHER_CRYPTO_OPENSSL
+#endif
+
+#define FILE_HEADER_SZ 16
+
+#define CIPHER_XSTR(s) CIPHER_STR(s)
+#define CIPHER_STR(s) #s
+
+#ifndef CIPHER_VERSION_NUMBER
+#define CIPHER_VERSION_NUMBER 4.13.0
+#endif
+
+#ifndef CIPHER_VERSION_BUILD
+#define CIPHER_VERSION_BUILD community
+#endif
+
+#define CIPHER_READ_CTX 0
+#define CIPHER_WRITE_CTX 1
+#define CIPHER_READWRITE_CTX 2
+
+#ifndef PBKDF2_ITER
+#define PBKDF2_ITER 256000
+#endif
+
+#define SQLCIPHER_FLAG_GET(FLAG,BIT) ((FLAG & BIT) != 0)
+#define SQLCIPHER_FLAG_SET(FLAG,BIT) FLAG |= BIT
+#define SQLCIPHER_FLAG_UNSET(FLAG,BIT) FLAG &= ~BIT
+
+/* possible flags for codec_ctx->flags */
+#define CIPHER_FLAG_HMAC          (1 << 0)
+#define CIPHER_FLAG_LE_PGNO       (1 << 1)
+#define CIPHER_FLAG_BE_PGNO       (1 << 2)
+#define CIPHER_FLAG_KEY_USED      (1 << 3)
+#define CIPHER_FLAG_HAS_KDF_SALT  (1 << 4)
+
+
+#ifndef DEFAULT_CIPHER_FLAGS
+#define DEFAULT_CIPHER_FLAGS CIPHER_FLAG_HMAC | CIPHER_FLAG_LE_PGNO
+#endif
+
+
+/* by default, sqlcipher will use a reduced number of iterations to generate
+   the HMAC key / or transform a raw cipher key 
+   */
+#ifndef FAST_PBKDF2_ITER
+#define FAST_PBKDF2_ITER 2
+#endif
+
+/* this if a fixed random array that will be xor'd with the database salt to ensure that the
+   salt passed to the HMAC key derivation function is not the same as that used to derive
+   the encryption key. This can be overridden at compile time but it will make the resulting
+   binary incompatible with the default builds when using HMAC. A future version of SQLcipher
+   will likely allow this to be defined at runtime via pragma */ 
+#ifndef HMAC_SALT_MASK
+#define HMAC_SALT_MASK 0x3a
+#endif
+
+#ifndef CIPHER_MAX_IV_SZ
+#define CIPHER_MAX_IV_SZ 16
+#endif
+
+#ifndef CIPHER_MAX_KEY_SZ
+#define CIPHER_MAX_KEY_SZ 64
+#endif
+
+
+/* the default implementation of SQLCipher uses a cipher_ctx
+   to keep track of read / write state separately. The following
+   struct and associated functions are defined here */
+typedef struct {
+  int derive_key;
+  int pass_sz;
+  unsigned char *key;
+  unsigned char *hmac_key;
+  unsigned char *pass;
+} cipher_ctx;
+
+
+typedef struct {
+  int store_pass;
+  int kdf_iter;
+  int fast_kdf_iter;
+  int kdf_salt_sz;
+  int key_sz;
+  int iv_sz;
+  int block_sz;
+  int page_sz;
+  int reserve_sz;
+  int hmac_sz;
+  int plaintext_header_sz;
+  int hmac_algorithm;
+  int kdf_algorithm;
+  int error;
+  unsigned int flags;
+  unsigned char *kdf_salt;
+  unsigned char *hmac_kdf_salt;
+  unsigned char *buffer;
+  Btree *pBt;
+  cipher_ctx *read_ctx;
+  cipher_ctx *write_ctx;
+  sqlcipher_provider *provider;
+  void *provider_ctx;
+} codec_ctx ;
+
+typedef struct private_block private_block;
+struct private_block {
+  private_block *next;
+  u32 size;
+  u32 is_used;
+};
+
+/* implementation of simple, fast PSRNG function using xoshiro256++ (XOR/shift/rotate)
+ * https://prng.di.unimi.it/ under the public domain via https://prng.di.unimi.it/xoshiro256plusplus.c
+ * xoshiro is NEVER used for any cryptographic functions as CSPRNG. It is solely used for
+ * generating random data for testing, debugging, and forensic purposes (overwriting memory segments) */
+static volatile uint64_t xoshiro_s[4];
+
+static inline uint64_t xoshiro_rotl(const uint64_t x, int k) {
+  return (x << k) | (x >> (64 - k));
+}
+
+uint64_t xoshiro_next(void) {
+  volatile uint64_t result = xoshiro_rotl(xoshiro_s[0] + xoshiro_s[3], 23) + xoshiro_s[0];
+  volatile uint64_t t = xoshiro_s[1] << 17;
+
+  xoshiro_s[2] ^= xoshiro_s[0];
+  xoshiro_s[3] ^= xoshiro_s[1];
+  xoshiro_s[1] ^= xoshiro_s[2];
+  xoshiro_s[0] ^= xoshiro_s[3];
+
+  xoshiro_s[2] ^= t;
+
+  xoshiro_s[3] = xoshiro_rotl(xoshiro_s[3], 45);
+
+  return result;
+}
+
+static void xoshiro_randomness(unsigned char *ptr, int sz) {
+  volatile uint64_t val;
+  volatile int to_copy;
+  while (sz > 0) {
+    val = xoshiro_next();
+    to_copy = (sz >= sizeof(val)) ? sizeof(val) : sz;
+    memcpy(ptr, (void *) &val, to_copy);
+    ptr += to_copy;
+    sz -= to_copy;
+  }
+}
+
+#ifdef SQLCIPHER_TEST
+/* possible flags for simulating specific test conditions */
+#define TEST_FAIL_ENCRYPT 0x01
+#define TEST_FAIL_DECRYPT 0x02
+#define TEST_FAIL_MIGRATE 0x04
+
+static volatile unsigned int cipher_test_flags = 0;
+static volatile int cipher_test_rand = 0;
+
+static int sqlcipher_get_test_fail() {
+  int x;
+
+  /* if cipher_test_rand is not set to a non-zero value always fail (return true) */
+  if (cipher_test_rand == 0) return 1;
+
+  xoshiro_randomness((unsigned char *) &x, sizeof(x));
+  return ((x % cipher_test_rand) == 0);
+}
+#endif
+
+static volatile unsigned int default_flags = DEFAULT_CIPHER_FLAGS;
+static volatile unsigned char hmac_salt_mask = HMAC_SALT_MASK;
+static volatile int default_kdf_iter = PBKDF2_ITER;
+static volatile int default_page_size = 4096;
+static volatile int default_plaintext_header_size = 0;
+static volatile int default_hmac_algorithm = SQLCIPHER_HMAC_SHA512;
+static volatile int default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA512;
+static volatile int sqlcipher_mem_security_on = 0;
+static volatile int sqlcipher_mem_executed = 0;
+static volatile int sqlcipher_mem_initialized = 0;
+static volatile sqlite3_mem_methods default_mem_methods;
+static sqlcipher_provider *default_provider = NULL;
+
+static sqlite3_mutex* sqlcipher_static_mutex[SQLCIPHER_MUTEX_COUNT];
+
+#ifndef SQLCIPHER_LOG_LEVEL_DEFAULT
+#define SQLCIPHER_LOG_LEVEL_DEFAULT SQLCIPHER_LOG_WARN
+#endif
+static FILE* sqlcipher_log_file = NULL;
+static volatile int sqlcipher_log_device = 0;
+static volatile unsigned int sqlcipher_log_level = SQLCIPHER_LOG_NONE;
+static volatile unsigned int sqlcipher_log_source = SQLCIPHER_LOG_ANY;
+static volatile int sqlcipher_log_set = 0;
+
+static size_t sqlcipher_shield_mask_sz  = 32;
+static u8* sqlcipher_shield_mask = NULL;
+
+/* Establish the default size of the private heap. This can be overriden 
+ * at compile time by setting -DSQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT=X */
+#ifndef SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT
+/* On android, the maximim amount of memory that can be memlocked in 64k. This also
+ * seems to be a popular ulimit on linux distributions, containsers, etc. Therefore
+ * the default heap size is chosen as 48K, which is either 4 (with 4k page size)
+ * or 1 (with 16k page size) page less than the max. We choose to allocate slightly
+ * less than the max just in case the app has locked some other page(s). This
+ * initial allocation should be enough to support at least 10 concurrent
+ * sqlcipher-enabled database connections at the same time without requiring any
+ * overflow allocations */
+#define SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT 49152
+#endif
+/* if default allocation fails, we'll reduce the size by this amount
+ * and try again. This is also the minimium of the private heap. The minimum
+ * size will be 4 4K pages or 1 16K page (possible with latest android)*/
+#define SQLCIPHER_PRIVATE_HEAP_SIZE_STEP 16384
+
+static volatile size_t private_heap_sz = SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT;
+static u8* private_heap = NULL;
+static volatile size_t private_heap_used = 0; /* bytes currently used on private heap */
+static volatile size_t private_heap_hwm = 0; /* larged number of bytes used on the private heap at one time */
+static volatile size_t private_heap_alloc = 0; /* total bytes allocated on private heap over time */
+static volatile u32 private_heap_allocs = 0; /* total number of allocations on private heap over time */
+static volatile size_t private_heap_overflow = 0; /* total bytes overflowing private heap over time */
+static volatile u32 private_heap_overflows = 0; /* number of overlow allocations over time */
+
+/* to prevent excessive fragmentation blocks will
+ * only be split if there are at least this many
+ * bytes available after the split. This should allow for at
+ * least two addtional small allocations */
+#define SQLCIPHER_PRIVATE_HEAP_MIN_SPLIT_SIZE 32
+
+/* requested sizes will be rounded up to the nearest 8 bytes for alignment */
+#define SQLCIPHER_PRIVATE_HEAP_ALIGNMENT 8
+#define SQLCIPHER_PRIVATE_HEAP_ROUNDUP(x) ((x % SQLCIPHER_PRIVATE_HEAP_ALIGNMENT) ? \
+  ((x / SQLCIPHER_PRIVATE_HEAP_ALIGNMENT) + 1) * SQLCIPHER_PRIVATE_HEAP_ALIGNMENT : x)
+
+static volatile int sqlcipher_init = 0;
+static volatile int sqlcipher_shutdown = 0;
+static volatile int sqlcipher_cleanup = 0;
+static int sqlcipher_init_error = SQLITE_ERROR;
+
+static void sqlcipher_internal_free(void *, sqlite_uint64);
+static void *sqlcipher_internal_malloc(sqlite_uint64);
+
+
+/*
+**  Simple shared routines for converting hex char strings to binary data
+ */
+static int cipher_hex2int(char c) {
+  return (c>='0' && c<='9') ? (c)-'0' :
+         (c>='A' && c<='F') ? (c)-'A'+10 :
+         (c>='a' && c<='f') ? (c)-'a'+10 : 0;
+}
+
+static void cipher_hex2bin(const unsigned char *hex, int sz, unsigned char *out){
+  int i;
+  for(i = 0; i < sz; i += 2){
+    out[i/2] = (cipher_hex2int(hex[i])<<4) | cipher_hex2int(hex[i+1]);
+  }
+}
+
+static void cipher_bin2hex(const unsigned char* in, int sz, char *out) {
+    int i;
+    for(i=0; i < sz; i++) {
+      sqlite3_snprintf(3, out + (i*2), "%02x ", in[i]);
+    } 
+}
+
+static int cipher_isHex(const unsigned char *hex, int sz){
+  int i;
+  for(i = 0; i < sz; i++) {
+    unsigned char c = hex[i];
+    if ((c < '0' || c > '9') &&
+        (c < 'A' || c > 'F') &&
+        (c < 'a' || c > 'f')) {
+      return 0;
+    }
+  }
+  return 1;
+}
+
+sqlite3_mutex* sqlcipher_mutex(int mutex) {
+  if(mutex < 0 || mutex >= SQLCIPHER_MUTEX_COUNT) return NULL;
+  return sqlcipher_static_mutex[mutex];
+}
+
+static void sqlcipher_atexit(void) {
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__);
+  sqlcipher_extra_shutdown();
+}
+
+static void sqlcipher_fini(void) {
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__);
+  sqlcipher_extra_shutdown();
+}
+
+#if defined(_WIN32)
+  #ifndef SQLCIPHER_OMIT_DLLMAIN
+  BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
+    switch (fdwReason) {
+      case DLL_PROCESS_DETACH:
+        sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__);
+        sqlcipher_extra_shutdown();
+        break;
+      default:
+        break;
+    }
+    return TRUE;
+  }
+  #endif
+#elif defined(__APPLE__)
+  #if defined(__has_feature)
+    #if __has_feature(address_sanitizer)
+    static void sqlcipher_cleanup_destructor(void) __attribute__((destructor));
+    static void sqlcipher_cleanup_destructor(void) { sqlcipher_fini(); }
+    #else
+    static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini;
+    #endif
+  #else
+    static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini;
+  #endif
+#else
+static void (*const sqlcipher_fini_func)(void) __attribute__((used, section(".fini_array"))) = sqlcipher_fini;
+#endif
+
+static void sqlcipher_exportFunc(sqlite3_context*, int, sqlite3_value**);
+
+static int sqlcipher_export_init(sqlite3* db, const char** errmsg, const struct sqlite3_api_routines* api) { 
+  sqlite3_create_function_v2(db, "sqlcipher_export", -1, SQLITE_UTF8, 0, sqlcipher_exportFunc, 0, 0, 0);
+  return SQLITE_OK;
+}
+
+/* The extra_init function is called by sqlite3_init automaticay by virtue of
+ * being defined with SQLITE_EXTRA_INIT. This function sets up 
+ * static mutexes used internally by SQLCipher and initializes
+ * the internal private heap */
+int sqlcipher_extra_init(const char* arg) {
+  int rc = SQLITE_OK, i=0;
+  void* provider_ctx = NULL;
+
+  sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER));
+
+  if(sqlcipher_init) {
+    /* if this init routine already completed successfully return immediately */
+    sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER));
+    return SQLITE_OK;
+  }
+
+  /* only register cleanup handlers once per process */
+  if(!sqlcipher_cleanup) {
+    atexit(sqlcipher_atexit);
+    sqlcipher_cleanup = 1;
+  }
+
+#ifndef SQLCIPHER_OMIT_DEFAULT_LOGGING
+  /* when sqlcipher is first activated, set a default log target and level of WARN if the
+     logging settings have not yet been initialized. Use the "device log" for 
+     android (logcat) or apple (console). Use stderr on all other platforms. */  
+  if(!sqlcipher_log_set) {
+
+    /* set log level if it is different than the uninitalized default value of NONE */ 
+    if(sqlcipher_log_level == SQLCIPHER_LOG_NONE) {
+      sqlcipher_log_level = SQLCIPHER_LOG_LEVEL_DEFAULT;
+    }
+
+    /* set the default file or device if neither is already set */
+    if(sqlcipher_log_device == 0 && sqlcipher_log_file == NULL) {
+#if defined(__ANDROID__) || defined(__APPLE__)
+      sqlcipher_log_device = 1;
+#else
+      sqlcipher_log_file = stderr;
+#endif
+    }
+    sqlcipher_log_set = 1;
+  }
+#endif
+
+  /* allocate static mutexe, and return error if any fail to allocate */
+  for(i = 0; i < SQLCIPHER_MUTEX_COUNT; i++) {
+    if(sqlcipher_static_mutex[i] == NULL) {
+      if((sqlcipher_static_mutex[i] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST)) == NULL) {
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to allocate static mutex %d", __func__, i);
+        rc = SQLITE_NOMEM; 
+        goto error;
+      }
+    }
+  }
+
+  /* initialize the private heap for use in internal SQLCipher memory allocations */
+  if(private_heap == NULL) {
+    while(private_heap_sz >= SQLCIPHER_PRIVATE_HEAP_SIZE_STEP) {
+      /* attempt to allocate the private heap. If allocation fails, reduce the size and try again */
+      if((private_heap = sqlcipher_internal_malloc(private_heap_sz))) {
+        xoshiro_randomness(private_heap, (int) private_heap_sz);
+        /* initialize the head block of the linked list at the start of the heap */ 
+        private_block *head = (private_block *) private_heap; 
+        head->is_used = 0;
+        head->size = (u32) private_heap_sz - sizeof(private_block);
+        head->next = NULL;
+        break;
+      }
+
+      /* allocation failed, reduce the requested size of the heap */
+      private_heap_sz -= SQLCIPHER_PRIVATE_HEAP_SIZE_STEP;
+    }
+  }
+  if(!private_heap) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to allocate private heap", __func__); 
+    rc = SQLITE_NOMEM; 
+    goto error;
+  }
+
+  /* check to see if there is a provider registered at this point
+     if there no provider registered at this point, register the 
+     default provider */
+  if(sqlcipher_get_provider() == NULL) {
+    sqlcipher_provider *p = sqlcipher_malloc(sizeof(sqlcipher_provider)); 
+#if defined (SQLCIPHER_CRYPTO_CC)
+    extern int sqlcipher_cc_setup(sqlcipher_provider *p);
+    sqlcipher_cc_setup(p);
+#elif defined (SQLCIPHER_CRYPTO_OPENSSL)
+    extern int sqlcipher_openssl_setup(sqlcipher_provider *p);
+    sqlcipher_openssl_setup(p);
+#elif defined (SQLCIPHER_CRYPTO_OSSL3)
+    extern int sqlcipher_ossl3_setup(sqlcipher_provider *p);
+    sqlcipher_ossl3_setup(p);
+#elif defined (SQLCIPHER_CRYPTO_CUSTOM)
+    extern int SQLCIPHER_CRYPTO_CUSTOM(sqlcipher_provider *p);
+    SQLCIPHER_CRYPTO_CUSTOM(p);
+#else
+#error "NO DEFAULT SQLCIPHER CRYPTO PROVIDER DEFINED"
+#endif
+    if((rc = sqlcipher_register_provider(p)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed to register provider %p %d", __func__, p, rc); 
+      goto error;
+    }
+  }
+
+  /* required random data */
+  if((rc = default_provider->ctx_init(&provider_ctx)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to initilize provider context %d", __func__, rc);
+    goto error; 
+  }
+
+  if((rc = default_provider->random(provider_ctx, (void *)xoshiro_s, sizeof(xoshiro_s))) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to generate xoshiro seed %d", __func__, rc); 
+    goto error; 
+  }
+
+  if(!sqlcipher_shield_mask) {
+    if(!(sqlcipher_shield_mask = sqlcipher_internal_malloc(sqlcipher_shield_mask_sz))) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to allocate shield mask", __func__); 
+      goto error; 
+    }
+    if((rc = default_provider->random(provider_ctx, sqlcipher_shield_mask, (int) sqlcipher_shield_mask_sz)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to generate requisite random mask data %d", __func__, rc); 
+      goto error; 
+    }
+  }
+
+  default_provider->ctx_free(&provider_ctx);
+
+  sqlcipher_init = 1;
+  sqlcipher_shutdown = 0;
+
+  /* leave the master mutex so we can proceed with auto extension registration */
+  sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER));
+
+  /* finally, extension registration occurs outside of the mutex because it is
+   * uses SQLITE_MUTEX_STATIC_MASTER itself */
+  sqlite3_auto_extension((void (*)(void))sqlcipher_export_init);
+
+  return SQLITE_OK;
+ 
+error:
+  /* if an error occurs during initialization, tear down everything that was setup */
+  if(private_heap) {
+    sqlcipher_internal_free(private_heap, private_heap_sz);
+    private_heap = NULL;
+  }
+  if(sqlcipher_shield_mask) {
+    sqlcipher_internal_free(sqlcipher_shield_mask, sqlcipher_shield_mask_sz);
+    sqlcipher_shield_mask = NULL;
+  }
+  for(i = 0; i < SQLCIPHER_MUTEX_COUNT; i++) {
+    if(sqlcipher_static_mutex[i]) {
+      sqlite3_mutex_free(sqlcipher_static_mutex[i]);
+      sqlcipher_static_mutex[i] = NULL;
+    }
+  }
+
+  /* post cleanup return the error code back up to sqlite3_init() */
+  sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER));
+  sqlcipher_init_error = rc;
+  return rc;
+}
+
+/* The extra_shutdown function is called by sqlite3_shutdown()
+ * because it is defined with SQLITE_EXTRA_SHUTDOWN. In addition it will
+ * be called via atexit(), finalizer, and DllMain. The function will
+ * cleanup resources allocated by SQLCipher including mutexes,
+ * the private heap, and default provider. */
+void sqlcipher_extra_shutdown(void) {
+  int i = 0;
+  sqlcipher_provider *provider = NULL;
+
+  /* if sqlcipher hasn't been initialized or the shutdown already completed exit early */
+  if(!sqlcipher_init || sqlcipher_shutdown) {
+    goto cleanup;
+  }
+
+  if(sqlcipher_shield_mask) {
+    sqlcipher_internal_free(sqlcipher_shield_mask, sqlcipher_shield_mask_sz);
+    sqlcipher_shield_mask = NULL;
+  }
+
+  /* free the provider list. start at the default provider and move through the list
+   * freeing each one. If a provider has a shutdown function, call it before freeing.
+   * finally NULL out the default_provider */
+  provider = default_provider;
+  while(provider) {
+    sqlcipher_provider *next = provider->next;
+    if(provider->shutdown) {
+      provider->shutdown();
+    }
+    sqlcipher_free(provider, sizeof(sqlcipher_provider));
+    provider = next;
+  }
+  default_provider = NULL;
+
+  /* free private heap. If SQLCipher is compiled in test mode, it will deliberately
+     not free the heap (leaking it) if the heap is not empty. This will allow tooling
+     to detect memory issues like unfreed private heap memory */
+  if(private_heap) {
+#ifdef SQLCIPHER_TEST
+    size_t used = 0;
+    private_block *block = NULL;
+    block = (private_block *) private_heap;
+    while (block != NULL) {
+      if(block->is_used) {
+        used+= block->size;
+        i++;
+      }
+      block = block->next;
+    }
+    if(used > 0) {
+      /* don't free the heap so that sqlite treats this as unfreed memory */ 
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, 
+        "%s: SQLCipher private heap unfreed memory %u bytes in %d allocations", __func__, used, i);
+    } else {
+      sqlcipher_internal_free(private_heap, private_heap_sz);
+      private_heap = NULL;
+    }
+#else
+    sqlcipher_internal_free(private_heap, private_heap_sz);
+    private_heap = NULL;
+#endif
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY,
+        "%s: SQLCipher private heap stats: size=%u, hwm=%u, alloc=%u, allocs=%u, overflow=%u, overflows=%u", __func__,
+        private_heap_sz, private_heap_hwm, private_heap_alloc, private_heap_allocs, private_heap_overflow, private_heap_overflows
+    );
+  }
+
+  /* free all of sqlcipher's static mutexes */
+  for(i = 0; i < SQLCIPHER_MUTEX_COUNT; i++) {
+    if(sqlcipher_static_mutex[i]) {
+      sqlite3_mutex_free(sqlcipher_static_mutex[i]);
+      sqlcipher_static_mutex[i] = NULL;
+    }
+  }
+
+cleanup: 
+  sqlcipher_init = 0;
+  sqlcipher_init_error = SQLITE_ERROR;
+  sqlcipher_shutdown = 1;
+}
+
+static void sqlcipher_shield(unsigned char *in, int sz) {
+  int i = 0;
+  for(i = 0; i < sz; i++) {
+    in[i] ^= sqlcipher_shield_mask[i % sqlcipher_shield_mask_sz];
+  }
+}
+
+/* constant time memset using volitile to avoid having the memset
+   optimized out by the compiler. 
+   Note: As suggested by Joachim Schipper (joachim.schipper@fox-it.com)
+*/
+void* sqlcipher_memset(void *v, unsigned char value, sqlite_uint64 len) {
+  volatile sqlite_uint64 i = 0;
+  volatile unsigned char *a = v;
+
+  if (v == NULL) return v;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_memset: setting %p[0-%u]=%d)", a, len, value);
+  for(i = 0; i < len; i++) {
+    a[i] = value;
+  }
+
+  return v;
+}
+
+/* constant time memory check tests every position of a memory segement
+   matches a single value (i.e. the memory is all zeros)
+   returns 0 if match, 1 of no match */
+int sqlcipher_ismemset(const void *v, unsigned char value, sqlite_uint64 len) {
+  const volatile unsigned char *a = v;
+  volatile sqlite_uint64 i = 0, result = 0;
+
+  for(i = 0; i < len; i++) {
+    result |= a[i] ^ value;
+  }
+
+  return (result != 0);
+}
+
+/* constant time memory comparison routine. 
+   returns 0 if match, 1 if no match */
+int sqlcipher_memcmp(const void *v0, const void *v1, int len) {
+  const volatile unsigned char *a0 = v0, *a1 = v1;
+  volatile int i = 0, result = 0;
+
+  for(i = 0; i < len; i++) {
+    result |= a0[i] ^ a1[i];
+  }
+  
+  return (result != 0);
+}
+
+static void sqlcipher_mlock(void *ptr, sqlite_uint64 sz) {
+#ifndef OMIT_MEMLOCK
+#if defined(__unix__) || defined(__APPLE__)
+  int rc;
+  unsigned long pagesize = sysconf(_SC_PAGESIZE);
+  unsigned long offset = (unsigned long) ptr % pagesize;
+
+  if(ptr == NULL || sz == 0) return;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_mlock: calling mlock(%p,%lu); _SC_PAGESIZE=%lu", ptr - offset, sz + offset, pagesize);
+  rc = mlock(ptr - offset, sz + offset);
+  if(rc!=0) {
+    sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_MEMORY, "sqlcipher_mlock: mlock() returned %d errno=%d", rc, errno);
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "sqlcipher_mlock: mlock(%p,%lu) returned %d errno=%d", ptr - offset, sz + offset, rc, errno);
+  }
+#elif defined(_WIN32)
+#if !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_PC_APP))
+  int rc;
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_mlock: calling VirtualLock(%p,%d)", ptr, sz);
+  rc = VirtualLock(ptr, sz);
+  if(rc==0) {
+    sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_MEMORY, "sqlcipher_mlock: VirtualLock() returned %d LastError=%d", rc, GetLastError());
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "sqlcipher_mlock: VirtualLock(%p,%d) returned %d LastError=%d", ptr, sz, rc, GetLastError());
+  }
+#endif
+#endif
+#endif
+}
+
+static void sqlcipher_munlock(void *ptr, sqlite_uint64 sz) {
+#ifndef OMIT_MEMLOCK
+#if defined(__unix__) || defined(__APPLE__)
+  int rc;
+  unsigned long pagesize = sysconf(_SC_PAGESIZE);
+  unsigned long offset = (unsigned long) ptr % pagesize;
+
+  if(ptr == NULL || sz == 0) return;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_munlock: calling munlock(%p,%lu)", ptr - offset, sz + offset);
+  rc = munlock(ptr - offset, sz + offset);
+  if(rc!=0) {
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "sqlcipher_munlock: munlock(%p,%lu) returned %d errno=%d", ptr - offset, sz + offset, rc, errno);
+  }
+#elif defined(_WIN32)
+#if !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_PC_APP))
+  int rc;
+
+  if(ptr == NULL || sz == 0) return;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_munlock: calling VirtualUnlock(%p,%d)", ptr, sz);
+  rc = VirtualUnlock(ptr, sz);
+
+  /* because memory allocations may be made from the same individual page, it is possible for VirtualUnlock to be called
+   * multiple times for the same page. Subsequent calls will return an error, but this can be safely ignored (i.e. because
+   * the previous call for that page unlocked the memory already). Log an info level event only in that case. */
+  if(!rc) {
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "sqlcipher_munlock: VirtualUnlock(%p,%d) returned %d LastError=%d", ptr, sz, rc, GetLastError());
+  }
+#endif
+#endif
+#endif
+}
+
+/** sqlcipher wraps the default memory subsystem so it can optionally provide the
+  * memory security feature which will lock and sanitize ALL memory used by 
+  * the sqlite library internally. Memory security feature is disabled by default
+  * but but the wrapper is used regardless, it just forwards to the default
+  * memory management implementation when disabled
+  */
+static int sqlcipher_mem_init(void *pAppData) {
+  return default_mem_methods.xInit(pAppData);
+}
+static void sqlcipher_mem_shutdown(void *pAppData) {
+  default_mem_methods.xShutdown(pAppData);
+}
+static void *sqlcipher_mem_malloc(int n) {
+  void *ptr = default_mem_methods.xMalloc(n);
+  if(!sqlcipher_mem_executed) sqlcipher_mem_executed = 1;
+  if(sqlcipher_mem_security_on) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_mem_malloc: calling sqlcipher_mlock(%p,%d)", ptr, n);
+    sqlcipher_mlock(ptr, n); 
+  }
+  return ptr;
+}
+static int sqlcipher_mem_size(void *p) {
+  return default_mem_methods.xSize(p);
+}
+static void sqlcipher_mem_free(void *p) {
+  int sz;
+  if(!sqlcipher_mem_executed) sqlcipher_mem_executed = 1;
+  if(sqlcipher_mem_security_on) {
+    sz = sqlcipher_mem_size(p);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s: calling xoshiro_randomness(%p,%d) and sqlcipher_munlock(%p, %d)", __func__, p, sz, p, sz);
+    xoshiro_randomness(p, sz);
+    sqlcipher_munlock(p, sz);
+  }
+  default_mem_methods.xFree(p);
+}
+static void *sqlcipher_mem_realloc(void *p, int n) {
+  void *new = NULL;
+  int orig_sz = 0;
+  if(sqlcipher_mem_security_on) {
+    orig_sz = sqlcipher_mem_size(p);
+    if (n==0) {
+      sqlcipher_mem_free(p);
+      return NULL;
+    } else if (!p) {
+      return sqlcipher_mem_malloc(n);
+    } else if(n <= orig_sz) {
+      return p;
+    } else {
+      new = sqlcipher_mem_malloc(n);
+      if(new) {
+        memcpy(new, p, orig_sz);
+        sqlcipher_mem_free(p);
+      }
+      return new;
+    }
+  } else {
+    return default_mem_methods.xRealloc(p, n);
+  }
+}
+
+static int sqlcipher_mem_roundup(int n) {
+  return default_mem_methods.xRoundup(n);
+}
+
+static sqlite3_mem_methods sqlcipher_mem_methods = {
+  sqlcipher_mem_malloc,
+  sqlcipher_mem_free,
+  sqlcipher_mem_realloc,
+  sqlcipher_mem_size,
+  sqlcipher_mem_roundup,
+  sqlcipher_mem_init,
+  sqlcipher_mem_shutdown,
+  0
+};
+
+void sqlcipher_init_memmethods() {
+  if(sqlcipher_mem_initialized) return;
+  if(sqlite3_config(SQLITE_CONFIG_GETMALLOC, &default_mem_methods) != SQLITE_OK ||
+    sqlite3_config(SQLITE_CONFIG_MALLOC, &sqlcipher_mem_methods)  != SQLITE_OK) {
+    sqlcipher_mem_security_on = sqlcipher_mem_executed = sqlcipher_mem_initialized = 0;
+  } else {
+    sqlcipher_mem_initialized = 1;
+  }
+}
+
+/**
+  * Free and wipe memory. Uses SQLites internal sqlite3_free so that memory
+  * can be countend and memory leak detection works in the test suite. 
+  * If ptr is not null memory will be freed. 
+  * If sz is greater than zero, the memory will be overwritten with zero before it is freed
+  * If sz is > 0, and not compiled with OMIT_MEMLOCK, system will attempt to unlock the
+  * memory segment so it can be paged
+  */
+static void sqlcipher_internal_free(void *ptr, sqlite_uint64 sz) {
+  xoshiro_randomness(ptr, sz);
+  sqlcipher_munlock(ptr, sz);
+  sqlite3_free(ptr);
+}
+
+/**
+  * allocate memory. Uses sqlite's internall malloc wrapper so memory can be 
+  * reference counted and leak detection works. Unless compiled with OMIT_MEMLOCK
+  * attempts to lock the memory pages so sensitive information won't be swapped
+  */
+static void* sqlcipher_internal_malloc(sqlite_uint64 sz) {
+  void *ptr;
+  ptr = sqlite3_malloc(sz);
+  sqlcipher_memset(ptr, 0, sz);
+  sqlcipher_mlock(ptr, sz);
+  return ptr;
+}
+
+void *sqlcipher_malloc(sqlite3_uint64 size) {
+  void *alloc = NULL;
+  private_block *block = NULL, *split = NULL;
+
+  if(size < 1) return NULL;
+
+  size = SQLCIPHER_PRIVATE_HEAP_ROUNDUP(size);
+
+  block = (private_block *) private_heap;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering SQLCIPHER_MUTEX_MEM", __func__);
+  sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_MEM));
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered SQLCIPHER_MUTEX_MEM", __func__);
+
+    /* iterate through the blocks in the heap to find one which is big enough to hold
+       the requested allocation. Stop when one is found. */
+  while(block != NULL && alloc == NULL) {
+    if(!block->is_used && block->size >= size) {
+      /* mark the block as in use and set the return pointer to the start
+         of the block free space */
+      block->is_used = 1;
+      alloc = ((u8*)block) + sizeof(private_block);
+      sqlcipher_memset(alloc, 0, size);
+
+      /* if there is at least the minimim amount of required space left after allocation, 
+         split off a new free block  and insert it after the in-use block */ 
+      if(block->size >= size + sizeof(private_block) + SQLCIPHER_PRIVATE_HEAP_MIN_SPLIT_SIZE) {
+        split = (private_block*) (((u8*) block) + size + sizeof(private_block));
+        split->is_used = 0;
+        split->size = block->size - size - sizeof(private_block);
+
+        /* insert inbetween current block and next */
+        split->next = block->next;
+        block->next = split;
+        
+        /* only set the size of the current block to the requested amount 
+           if the block was split. otherwise, size will be the full amount
+           of the block, which will actually be larger than the requested amount */
+        block->size = size;
+      } 
+    }
+    block = block->next;
+  }
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving SQLCIPHER_MUTEX_MEM", __func__);
+  sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_MEM));
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left SQLCIPHER_MUTEX_MEM", __func__);
+
+  /* If we were unable to locate a free block large enough to service the request, the fallback
+     behavior will simply attempt to allocate additional memory using malloc. */
+  if(alloc == NULL) {
+    private_heap_overflow += size;
+    private_heap_overflows++;
+    alloc = sqlcipher_internal_malloc(size);
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "%s: unable to allocate %u bytes on private heap, allocated %p using sqlcipher_internal_malloc fallback", __func__, size, alloc);
+  } else {
+    private_heap_used += size;
+    if(private_heap_used > private_heap_hwm) {
+      /* if the current bytes allocated on the private heap are greater than the high water mark, set the HWM to the new amount */
+      private_heap_hwm = private_heap_used;
+    }
+    private_heap_alloc += size;
+    private_heap_allocs++;
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s allocated %u bytes on private heap at %p", __func__, size, alloc);
+  }
+
+  return alloc;
+}
+
+void sqlcipher_free(void *mem, sqlite3_uint64 sz) {
+  private_block *block = NULL, *prev = NULL;
+  void *alloc = NULL;
+  u32 block_size = 0;
+  block = (private_block *) private_heap;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering SQLCIPHER_MUTEX_MEM", __func__);
+  sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_MEM));
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered SQLCIPHER_MUTEX_MEM", __func__);
+
+  /* search the heap for the block that contains this address */
+  while(block != NULL) {
+    alloc = ((u8*)block)+sizeof(private_block);
+    /* if the memory address to be freed corresponds to this block's
+       allocation, mark it as unused. If they don't match, move
+       on to the next block */
+    if(mem == alloc) {
+      block->is_used = 0;
+      block_size = block->size; /* retain the acual size of the block in use for stats adjustment */
+      xoshiro_randomness(alloc, block->size);
+
+      /* check whether the previous block is free, if so merge*/
+      if(prev && !prev->is_used) {
+        prev->size = prev->size + sizeof(private_block) + block->size;
+        prev->next = block->next;
+        block = prev;
+      }
+
+      /* check to see whether the next block is free, if so merge */
+      if(block->next && !block->next->is_used) {
+        block->size = block->size + sizeof(private_block) + block->next->size;
+        block->next = block->next->next;
+      }
+
+      /* once the block has been identified, marked free, and optionally
+         consolidated with it's neighbors, exit the loop, but leave
+         the block pointer intact so we know we found it in the heap */
+      break;
+    }
+
+    prev = block;
+    block = block->next;
+  }
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving SQLCIPHER_MUTEX_MEM", __func__);
+  sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_MEM));
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left SQLCIPHER_MUTEX_MEM", __func__);
+
+  /* If the memory address couldn't be found in the private heap
+     then it was allocated by the fallback mechanism and should
+     be deallocated with free() */
+  if(!block) {
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "%s: unable to find %p with %u bytes on private heap, calling sqlcipher_internal_free fallback", __func__, mem, sz);
+    sqlcipher_internal_free(mem, sz);
+  } else {
+    private_heap_used -= block_size;
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s freed %u bytes (%u total) on private heap at %p", __func__, sz, block_size, mem);
+  }
+}
+
+int sqlcipher_register_provider(sqlcipher_provider *p) {
+  int preexisting = 0, rc = SQLITE_OK;
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering SQLCIPHER_MUTEX_PROVIDER", __func__);
+  sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER));
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered SQLCIPHER_MUTEX_PROVIDER", __func__);
+
+  if(!p || default_provider == p) {
+    goto cleanup;
+  }
+
+  if(!default_provider) {
+    /* initial provider registration, NULL out previous pointer*/
+    p->next = NULL;
+  } else {
+    /* one or more previous provider has already been registered, search through
+     * the list to see if the new provider has already been registered and handle
+     * appropriately */
+    sqlcipher_provider *previous = default_provider;
+    sqlcipher_provider *current = default_provider->next;
+    while(current) {
+      if(current == p) {
+        /* this is a duplicate provider registration, and the provider in question
+         * already exists on the list. In that case, pop it out so it can be moved up to default.
+         * note that if we found an existing match we should avoid re-initializing the provider */
+        previous->next = current->next;
+        preexisting = 1;
+        break;
+      }
+      previous = current;
+      current = current->next;
+    }
+    /* the current default_provider gets tacked on the list */
+    p->next = default_provider;
+  }
+
+  /* the new provider is elevated to default. if the provider was not preexisting and it has an initializer, call it */
+  if(!preexisting && p->init) {
+    rc = p->init();
+  }
+ 
+  if(rc == SQLITE_OK) {
+    default_provider = p;   
+  }
+cleanup:
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving SQLCIPHER_MUTEX_PROVIDER", __func__);
+  sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER));
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left SQLCIPHER_MUTEX_PROVIDER", __func__);
+
+  return rc;
+}
+
+/* return a pointer to the currently registered provider. This will
+   allow an application to fetch the current registered provider and
+   make minor changes to it */
+sqlcipher_provider* sqlcipher_get_provider() {
+  return default_provider;
+}
+
+char* sqlcipher_version(void) {
+#ifdef CIPHER_VERSION_QUALIFIER
+    char *version = sqlite3_mprintf("%s %s %s", CIPHER_XSTR(CIPHER_VERSION_NUMBER), CIPHER_XSTR(CIPHER_VERSION_QUALIFIER), CIPHER_XSTR(CIPHER_VERSION_BUILD));
+#else
+    char *version = sqlite3_mprintf("%s %s", CIPHER_XSTR(CIPHER_VERSION_NUMBER), CIPHER_XSTR(CIPHER_VERSION_BUILD));
+#endif
+    return version;
+}
+
+/**
+  * Initialize new cipher_ctx struct. This function will allocate memory
+  * for the cipher context and for the key
+  * 
+  * returns SQLITE_OK if initialization was successful
+  * returns SQLITE_NOMEM if an error occured allocating memory
+  */
+static int sqlcipher_cipher_ctx_init(codec_ctx *ctx, cipher_ctx **iCtx) {
+  cipher_ctx *c_ctx;
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_cipher_ctx_init: allocating context");
+  *iCtx = (cipher_ctx *) sqlcipher_malloc(sizeof(cipher_ctx));
+  c_ctx = *iCtx;
+  if(c_ctx == NULL) return SQLITE_NOMEM;
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_cipher_ctx_init: allocating key");
+  c_ctx->key = (unsigned char *) sqlcipher_malloc(ctx->key_sz);
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_cipher_ctx_init: allocating hmac_key");
+  c_ctx->hmac_key = (unsigned char *) sqlcipher_malloc(ctx->key_sz);
+
+  if(!c_ctx->key || !c_ctx->hmac_key) return SQLITE_NOMEM;
+
+  return SQLITE_OK;
+}
+
+/**
+  * Free and wipe memory associated with a cipher_ctx
+  */
+static void sqlcipher_cipher_ctx_free(codec_ctx* ctx, cipher_ctx **iCtx) {
+  cipher_ctx *c_ctx = *iCtx;
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "cipher_ctx_free: iCtx=%p", iCtx);
+  if(c_ctx->key) sqlcipher_free(c_ctx->key, ctx->key_sz);
+  if(c_ctx->hmac_key) sqlcipher_free(c_ctx->hmac_key, ctx->key_sz);
+  if(c_ctx->pass) sqlcipher_free(c_ctx->pass, c_ctx->pass_sz);
+  sqlcipher_free(c_ctx, sizeof(cipher_ctx)); 
+}
+
+static int sqlcipher_codec_ctx_reserve_setup(codec_ctx *ctx) {
+  int base_reserve = ctx->iv_sz; /* base reserve size will be IV only */ 
+  int reserve = base_reserve;
+
+  ctx->hmac_sz = ctx->provider->get_hmac_sz(ctx->provider_ctx, ctx->hmac_algorithm); 
+
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC))
+    reserve += ctx->hmac_sz; /* if reserve will include hmac, update that size */
+
+  /* calculate the amount of reserve needed in even increments of the cipher block size */
+  if(ctx->block_sz > 0) {
+    reserve = ((reserve % ctx->block_sz) == 0) ? reserve :
+               ((reserve / ctx->block_sz) + 1) * ctx->block_sz;  
+  }
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_reserve_setup: base_reserve=%d block_sz=%d md_size=%d reserve=%d", 
+                base_reserve, ctx->block_sz, ctx->hmac_sz, reserve); 
+
+  ctx->reserve_sz = reserve;
+
+  return SQLITE_OK;
+}
+
+/**
+  * Compare one cipher_ctx to another.
+  *
+  * returns 0 if all the parameters (except the derived key data) are the same
+  * returns 1 otherwise
+  */
+static int sqlcipher_cipher_ctx_cmp(cipher_ctx *c1, cipher_ctx *c2) {
+  int are_equal = (
+    c1->pass_sz == c2->pass_sz
+    && (
+      c1->pass == c2->pass
+      || !sqlcipher_memcmp((const unsigned char*)c1->pass,
+                           (const unsigned char*)c2->pass,
+                           c1->pass_sz)
+    ));
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_cipher_ctx_cmp: c1=%p c2=%p sqlcipher_memcmp(c1->pass, c2_pass)=%d are_equal=%d",
+    c1, c2,
+    (c1->pass == NULL || c2->pass == NULL) ?
+      -1 :
+      sqlcipher_memcmp(
+        (const unsigned char*)c1->pass,
+        (const unsigned char*)c2->pass,
+        c1->pass_sz
+      ),
+    are_equal
+  );
+
+  return !are_equal; /* return 0 if they are the same, 1 otherwise */
+}
+
+/**
+  * Copy one cipher_ctx to another. For instance, assuming that read_ctx is a 
+  * fully initialized context, you could copy it to write_ctx and all yet data
+  * and pass information across
+  *
+  * returns SQLITE_OK if initialization was successful
+  * returns SQLITE_NOMEM if an error occured allocating memory
+  */
+static int sqlcipher_cipher_ctx_copy(codec_ctx *ctx, cipher_ctx *target, cipher_ctx *source) {
+  void *key = target->key; 
+  void *hmac_key = target->hmac_key; 
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_cipher_ctx_copy: target=%p, source=%p", target, source);
+  if(target->pass) sqlcipher_free(target->pass, target->pass_sz);
+  memcpy(target, source, sizeof(cipher_ctx));
+
+  target->key = key; /* restore pointer to previously allocated key data */
+  memcpy(target->key, source->key, ctx->key_sz);
+
+  target->hmac_key = hmac_key; /* restore pointer to previously allocated hmac key data */
+  memcpy(target->hmac_key, source->hmac_key, ctx->key_sz);
+
+  if(source->pass && source->pass_sz) {
+    target->pass = sqlcipher_malloc(source->pass_sz);
+    if(target->pass == NULL) return SQLITE_NOMEM;
+    memcpy(target->pass, source->pass, source->pass_sz);
+  }
+  return SQLITE_OK;
+}
+
+/**
+  * Get the keyspec for the cipher_ctx
+  * 
+  * returns SQLITE_OK if assignment was successfull
+  * returns SQLITE_NOMEM if an error occured allocating memory
+  */
+static int sqlcipher_cipher_ctx_get_keyspec(codec_ctx *ctx, cipher_ctx *c_ctx, char **keyspec_ptr, int *keyspec_sz) {
+  int sz = 0;
+  char *keyspec = NULL, *out = NULL;
+
+  if(keyspec_ptr == NULL) return SQLITE_NOMEM;
+
+  /* establish the size for a hex-formated key specification, containing the
+   * raw encryption key, optional hmac key, and the salt used to generate it.
+   * The format will be either:
+   *   x'hex(key)...hex(hmac_key)...hex(salt)'
+   *     or
+   *   x'hex(key)...hex(salt)'
+   *. The contents are SQLite BLOB formatted, so oversize by 3 bytes for the leading
+   * x' and trailing ' characters required by the spec*/
+  if(ctx->flags & CIPHER_FLAG_HMAC) { /* if HMAC is enabled, encode key, hmac key, and salt */
+    sz = ((ctx->key_sz + ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3;
+  } else { /* otherwise encode key and salt */
+    sz = ((ctx->key_sz + ctx->kdf_salt_sz) * 2) + 3;
+  }
+
+  *keyspec_ptr = sqlcipher_malloc(sz);
+  keyspec = *keyspec_ptr;
+
+  if(keyspec == NULL) return SQLITE_NOMEM;
+
+  keyspec[0] = 'x';
+  keyspec[1] = '\'';
+
+  out = keyspec + 2;
+
+  /* start with the encryption key */
+  sqlcipher_shield(c_ctx->key, ctx->key_sz);
+  cipher_bin2hex(c_ctx->key, ctx->key_sz, out);
+  sqlcipher_shield(c_ctx->key, ctx->key_sz);
+  out += ctx->key_sz * 2;
+
+  if(ctx->flags & CIPHER_FLAG_HMAC) {
+    /* add the hmac key after the encryption key if HMAC is in use*/
+    sqlcipher_shield(c_ctx->hmac_key, ctx->key_sz);
+    cipher_bin2hex(c_ctx->hmac_key, ctx->key_sz, out);
+    sqlcipher_shield(c_ctx->hmac_key, ctx->key_sz);
+    out += ctx->key_sz * 2;
+  }
+
+  /* finally encode the salt last */
+  cipher_bin2hex(ctx->kdf_salt, ctx->kdf_salt_sz, out);
+
+  keyspec[sz - 1] = '\'';
+  *keyspec_sz = sz;
+
+  return SQLITE_OK;
+}
+
+static void sqlcipher_set_derive_key(codec_ctx *ctx, int derive) {
+  if(ctx->read_ctx != NULL) ctx->read_ctx->derive_key = derive;
+  if(ctx->write_ctx != NULL) ctx->write_ctx->derive_key = derive;
+}
+
+/**
+  * Set the passphrase for the cipher_ctx
+  * 
+  * returns SQLITE_OK if assignment was successfull
+  * returns SQLITE_NOMEM if an error occured allocating memory
+  */
+static int sqlcipher_cipher_ctx_set_pass(cipher_ctx *ctx, const void *zKey, int nKey) {
+  /* free, zero existing pointers and size */
+  if(ctx->pass) sqlcipher_free(ctx->pass, ctx->pass_sz);
+  ctx->pass = NULL;
+  ctx->pass_sz = 0;
+
+  if(zKey && nKey) { /* if new password is provided, copy it */
+    ctx->pass_sz = nKey;
+    ctx->pass = sqlcipher_malloc(nKey);
+    if(ctx->pass == NULL) return SQLITE_NOMEM;
+    memcpy(ctx->pass, zKey, nKey);
+  } 
+  return SQLITE_OK;
+}
+
+static int sqlcipher_codec_ctx_set_pass(codec_ctx *ctx, const void *zKey, int nKey, int for_ctx) {
+  cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx;
+  int rc;
+
+  if((rc = sqlcipher_cipher_ctx_set_pass(c_ctx, zKey, nKey)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_set_pass: error %d from sqlcipher_cipher_ctx_set_pass", rc);
+    return rc;
+  }
+
+  c_ctx->derive_key = 1;
+
+  if(for_ctx == 2) {
+    if((rc = sqlcipher_cipher_ctx_copy(ctx, for_ctx ? ctx->read_ctx : ctx->write_ctx, c_ctx)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_set_pass: error %d from sqlcipher_cipher_ctx_copy", rc);
+      return rc;
+    }
+  }
+
+  return SQLITE_OK;
+} 
+
+static int sqlcipher_codec_ctx_set_kdf_iter(codec_ctx *ctx, int kdf_iter) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+  ctx->kdf_iter = kdf_iter;
+  sqlcipher_set_derive_key(ctx, 1);
+  return SQLITE_OK;
+}
+
+static int sqlcipher_codec_ctx_set_fast_kdf_iter(codec_ctx *ctx, int fast_kdf_iter) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+
+  ctx->fast_kdf_iter = fast_kdf_iter;
+  sqlcipher_set_derive_key(ctx, 1);
+  return SQLITE_OK;
+}
+
+/* set the global default flag for HMAC */
+static void sqlcipher_set_default_use_hmac(int use) {
+  if(use) SQLCIPHER_FLAG_SET(default_flags, CIPHER_FLAG_HMAC);
+  else SQLCIPHER_FLAG_UNSET(default_flags,CIPHER_FLAG_HMAC);
+}
+
+/* set the codec flag for whether this individual database should be using hmac */
+static int sqlcipher_codec_ctx_set_use_hmac(codec_ctx *ctx, int use) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+
+  if(use) {
+    SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_HMAC);
+  } else {
+    SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_HMAC);
+  } 
+
+  return sqlcipher_codec_ctx_reserve_setup(ctx);
+}
+
+/* the length of plaintext header size must be:
+ * 1. greater than or equal to zero
+ * 2. a multiple of the cipher block size
+ * 3. less than or equal to the non-reserve size of the first database page
+ *
+ * Note: it is possible to leave the entire first page in plaintext. This is discouraged since it will
+ * likely leak some small amount of schema data, but it's required to support use of the recovery VFS.
+ * see comment in sqlcipher_page_cipher for more details.
+ */
+static int sqlcipher_codec_ctx_set_plaintext_header_size(codec_ctx *ctx, int size) {
+  if(size >= 0 && ctx->block_sz > 0 && (size % ctx->block_sz) == 0 && size <= (ctx->page_sz - ctx->reserve_sz)) {
+    ctx->plaintext_header_sz = size;
+    return SQLITE_OK;
+  }
+  ctx->plaintext_header_sz = -1;
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: attempt to set invalid plantext_header_size %d", __func__, size);
+  return SQLITE_ERROR;
+} 
+
+static int sqlcipher_codec_ctx_set_hmac_algorithm(codec_ctx *ctx, int algorithm) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+
+  ctx->hmac_algorithm = algorithm;
+  return sqlcipher_codec_ctx_reserve_setup(ctx);
+} 
+
+static int sqlcipher_codec_ctx_set_kdf_algorithm(codec_ctx *ctx, int algorithm) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+
+  ctx->kdf_algorithm = algorithm;
+  return SQLITE_OK;
+} 
+
+static void sqlcipher_codec_ctx_set_error(codec_ctx *ctx, int error) {
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_set_error %d", error);
+  sqlite3pager_error(sqlite3BtreePager(ctx->pBt), error);
+  ctx->pBt->pBt->db->errCode = error;
+  ctx->error = error;
+}
+
+static int sqlcipher_codec_ctx_init_kdf_salt(codec_ctx *ctx) {
+  sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(ctx->pBt));
+
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) {
+    return SQLITE_OK; /* don't reload salt when not needed */
+  }
+
+  /* read salt from header, if present, otherwise generate a new random salt */
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init_kdf_salt: obtaining salt");
+  if(fd == NULL || fd->pMethods == 0 || sqlite3OsRead(fd, ctx->kdf_salt, ctx->kdf_salt_sz, 0) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init_kdf_salt: unable to read salt from file header, generating random");
+    if(ctx->provider->random(ctx->provider_ctx, ctx->kdf_salt, ctx->kdf_salt_sz) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init_kdf_salt: error retrieving random bytes from provider");
+      return SQLITE_ERROR;
+    }
+  }
+  SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT);
+  return SQLITE_OK; 
+}
+
+static int sqlcipher_codec_ctx_set_kdf_salt(codec_ctx *ctx, unsigned char *salt, int size) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+
+  if(size >= ctx->kdf_salt_sz) {
+    memcpy(ctx->kdf_salt, salt, ctx->kdf_salt_sz);
+    SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT);
+    return SQLITE_OK;
+  }
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_set_kdf_salt: attempt to set salt of incorrect size %d", size);
+  return SQLITE_ERROR;
+}
+
+static int sqlcipher_codec_ctx_get_kdf_salt(codec_ctx *ctx, void** salt) {
+  int rc = SQLITE_OK;
+  if(!SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) {
+    if((rc = sqlcipher_codec_ctx_init_kdf_salt(ctx)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_get_kdf_salt: error %d from sqlcipher_codec_ctx_init_kdf_salt", rc);
+    }
+  }
+  *salt = ctx->kdf_salt;
+
+  return rc;
+}
+
+static int sqlcipher_codec_ctx_set_pagesize(codec_ctx *ctx, int size) {
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) return SQLITE_OK;
+
+  if(!((size != 0) && ((size & (size - 1)) == 0)) || size < 512 || size > 65536) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "cipher_page_size not a power of 2 and between 512 and 65536 inclusive");
+    return SQLITE_ERROR;
+  }
+  /* attempt to free the existing page buffer */
+  if(ctx->buffer) sqlcipher_free(ctx->buffer,ctx->page_sz);
+  ctx->page_sz = size;
+
+  /* pre-allocate a page buffer of PageSize bytes. This will
+     be used as a persistent buffer for encryption and decryption 
+     operations to avoid overhead of multiple memory allocations*/
+  ctx->buffer = sqlcipher_malloc(size);
+  if(ctx->buffer == NULL) return SQLITE_NOMEM;
+
+  return SQLITE_OK;
+}
+
+static int sqlcipher_codec_ctx_init(codec_ctx **iCtx, Db *pDb, Pager *pPager, const void *zKey, int nKey) {
+  int rc;
+  codec_ctx *ctx;
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_codec_ctx_init: allocating context");
+
+  *iCtx = sqlcipher_malloc(sizeof(codec_ctx));
+  ctx = *iCtx;
+
+  if(ctx == NULL) return SQLITE_NOMEM;
+
+  ctx->pBt = pDb->pBt; /* assign pointer to database btree structure */
+
+  /* allocate space for salt data. Then read the first 16 bytes 
+       directly off the database file. This is the salt for the
+       key derivation function. If we get a short read allocate
+       a new random salt value */
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_codec_ctx_init: allocating kdf_salt");
+  ctx->kdf_salt_sz = FILE_HEADER_SZ;
+  ctx->kdf_salt = sqlcipher_malloc(ctx->kdf_salt_sz);
+  if(ctx->kdf_salt == NULL) return SQLITE_NOMEM;
+
+  /* allocate space for separate hmac salt data. We want the
+     HMAC derivation salt to be different than the encryption
+     key derivation salt */
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_codec_ctx_init: allocating hmac_kdf_salt");
+  ctx->hmac_kdf_salt = sqlcipher_malloc(ctx->kdf_salt_sz);
+  if(ctx->hmac_kdf_salt == NULL) return SQLITE_NOMEM;
+
+  /* setup default flags */
+  ctx->flags = default_flags;
+
+  /* the context will use the current default crypto provider  */
+  ctx->provider = default_provider;
+
+  if((rc = ctx->provider->ctx_init(&ctx->provider_ctx)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d returned from ctx_init", rc);
+    return rc;
+  }
+
+  ctx->key_sz = ctx->provider->get_key_sz(ctx->provider_ctx);
+  ctx->iv_sz = ctx->provider->get_iv_sz(ctx->provider_ctx);
+  ctx->block_sz = ctx->provider->get_block_sz(ctx->provider_ctx);
+
+  /*
+     Always overwrite page size and set to the default because the first page of the database
+     in encrypted and thus sqlite can't effectively determine the pagesize. this causes an issue in 
+     cases where bytes 16 & 17 of the page header are a power of 2 as reported by John Lehman
+  */
+  if((rc = sqlcipher_codec_ctx_set_pagesize(ctx, default_page_size)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d returned from sqlcipher_codec_ctx_set_pagesize with %d", rc, default_page_size);
+    return rc;
+  }
+
+  /* establish settings for the KDF iterations and fast (HMAC) KDF iterations */
+  if((rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, default_kdf_iter)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting default_kdf_iter %d", rc, default_kdf_iter);
+    return rc;
+  }
+
+  if((rc = sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, FAST_PBKDF2_ITER)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting fast_kdf_iter to %d", rc, FAST_PBKDF2_ITER);
+    return rc;
+  }
+
+  /* set the default HMAC and KDF algorithms which will determine the reserve size */
+  if((rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, default_hmac_algorithm)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting sqlcipher_codec_ctx_set_hmac_algorithm with %d", rc, default_hmac_algorithm);
+    return rc;
+  }
+
+  /* Note that use_hmac is a special case that requires recalculation of page size
+     so we call set_use_hmac to perform setup */
+  if((rc = sqlcipher_codec_ctx_set_use_hmac(ctx, SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC))) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting use_hmac %d", rc, SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC));
+    return rc;
+  }
+
+  if((rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, default_kdf_algorithm)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting sqlcipher_codec_ctx_set_kdf_algorithm with %d", rc, default_kdf_algorithm);
+    return rc;
+  }
+
+  /* setup the default plaintext header size */
+  if((rc = sqlcipher_codec_ctx_set_plaintext_header_size(ctx, default_plaintext_header_size)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting sqlcipher_codec_ctx_set_plaintext_header_size with %d", rc, default_plaintext_header_size);
+    return rc;
+  }
+
+  /* initialize the read and write sub-contexts. this must happen after key_sz is established  */
+  if((rc = sqlcipher_cipher_ctx_init(ctx, &ctx->read_ctx)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d initializing read_ctx", rc);
+    return rc;
+  } 
+
+  if((rc = sqlcipher_cipher_ctx_init(ctx, &ctx->write_ctx)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d initializing write_ctx", rc);
+    return rc; 
+  }
+
+  /* set the key material on one of the sub cipher contexts and sync them up */
+  if((rc = sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, 0)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d setting pass key", rc);
+    return rc;
+  }
+
+  if((rc = sqlcipher_cipher_ctx_copy(ctx, ctx->write_ctx, ctx->read_ctx)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_init: error %d copying write_ctx to read_ctx", rc);
+    return rc;
+  }
+
+  return SQLITE_OK;
+}
+
+/**
+  * Free and wipe memory associated with a cipher_ctx, including the allocated
+  * read_ctx and write_ctx.
+  */
+static void sqlcipher_codec_ctx_free(codec_ctx **iCtx) {
+  codec_ctx *ctx = *iCtx;
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "codec_ctx_free: iCtx=%p", iCtx);
+  if(ctx->kdf_salt) sqlcipher_free(ctx->kdf_salt, ctx->kdf_salt_sz);
+  if(ctx->hmac_kdf_salt) sqlcipher_free(ctx->hmac_kdf_salt, ctx->kdf_salt_sz);
+  if(ctx->buffer) sqlcipher_free(ctx->buffer, ctx->page_sz);
+  if(ctx->provider) ctx->provider->ctx_free(&ctx->provider_ctx);
+
+  sqlcipher_cipher_ctx_free(ctx, &ctx->read_ctx);
+  sqlcipher_cipher_ctx_free(ctx, &ctx->write_ctx);
+  sqlcipher_free(ctx, sizeof(codec_ctx)); 
+}
+
+/** convert a 32bit unsigned integer to little endian byte ordering */
+static void sqlcipher_put4byte_le(unsigned char *p, u32 v) { 
+  p[0] = (u8)v;
+  p[1] = (u8)(v>>8);
+  p[2] = (u8)(v>>16);
+  p[3] = (u8)(v>>24);
+}
+
+static int sqlcipher_page_hmac(codec_ctx *ctx, cipher_ctx *c_ctx, Pgno pgno, unsigned char *in, int in_sz, unsigned char *out) {
+  unsigned char pgno_raw[sizeof(pgno)];
+  int rc;
+  /* we may convert page number to consistent representation before calculating MAC for
+     compatibility across big-endian and little-endian platforms. 
+
+     Note: The public release of sqlcipher 2.0.0 to 2.0.6 had a bug where the bytes of pgno 
+     were used directly in the MAC. SQLCipher convert's to little endian by default to preserve
+     backwards compatibility on the most popular platforms, but can optionally be configured
+     to use either big endian or native byte ordering via pragma. */
+
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_LE_PGNO)) { /* compute hmac using little endian pgno*/
+    sqlcipher_put4byte_le(pgno_raw, pgno);
+  } else if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_BE_PGNO)) { /* compute hmac using big endian pgno */
+    sqlite3Put4byte(pgno_raw, pgno); /* sqlite3Put4byte converts 32bit uint to big endian  */
+  } else { /* use native byte ordering */
+    memcpy(pgno_raw, &pgno, sizeof(pgno));
+  }
+
+  /* include the encrypted page data,  initialization vector, and page number in HMAC. This will 
+     prevent both tampering with the ciphertext, manipulation of the IV, or resequencing otherwise
+     valid pages out of order in a database */ 
+  sqlcipher_shield(c_ctx->hmac_key, ctx->key_sz);
+  rc = ctx->provider->hmac(
+    ctx->provider_ctx, ctx->hmac_algorithm, c_ctx->hmac_key,
+    ctx->key_sz, in,
+    in_sz, (unsigned char*) &pgno_raw,
+    sizeof(pgno), out);
+  sqlcipher_shield(c_ctx->hmac_key, ctx->key_sz);
+
+  return rc;
+}
+
+/*
+ * ctx - codec context
+ * pgno - page number in database
+ * size - size in bytes of input and output buffers
+ * mode - 1 to encrypt, 0 to decrypt
+ * in - pointer to input bytes
+ * out - pouter to output bytes
+ */
+static int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mode, int page_sz, unsigned char *in, unsigned char *out) {
+  cipher_ctx *c_ctx = for_ctx ? ctx->write_ctx : ctx->read_ctx;
+  unsigned char *iv_in, *iv_out, *hmac_in, *hmac_out, *out_start;
+  int size, rc;
+
+  /* calculate some required positions into various buffers */
+  size = page_sz - ctx->reserve_sz; /* adjust size to useable size and memset reserve at end of page */
+  iv_out = out + size;
+  iv_in = in + size;
+
+  /* if the full amount of the first page (excluding reserve size), e.g. 4016 bytes for a 4096 byte page size with HMAC_SHA512,
+   * is used as a plaintext header, then the entire first page will be completely plaintext, and this function should just return early.
+   * This should almost never occur during normal usage, so we will log at WARN level, but it is required in the special case that
+   * a user wants to attempt recovery on an encrypted database. In that case, the database header must be completely plaintext so that
+   * the recovery VFS can be used with it's special 1st page logic */
+  if(pgno == 1 && size == 0) {
+    sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "%s: skipping encryption/decryption for fully  plaintext header", __func__);
+    return SQLITE_OK;
+  }
+
+  /* hmac will be written immediately after the initialization vector. the remainder of the page reserve will contain
+     random bytes. note, these pointers are only valid when using hmac */
+  hmac_in = in + size + ctx->iv_sz; 
+  hmac_out = out + size + ctx->iv_sz;
+  out_start = out; /* note the original position of the output buffer pointer, as out will be rewritten during encryption */
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: pgno=%d, mode=%d, size=%d", __func__, pgno, mode, size);
+  CODEC_HEXDUMP("sqlcipher_page_cipher: input page data", in, page_sz);
+
+  /* the key size should never be zero. If it is, error out. */
+  if(ctx->key_sz == 0) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: error possible context corruption, key_sz is zero for pgno=%d", __func__, pgno);
+    goto error;
+  } 
+
+  if(mode == SQLCIPHER_ENCRYPT) {
+    /* start at front of the reserve block, write random data to the end */
+    if(ctx->provider->random(ctx->provider_ctx, iv_out, ctx->reserve_sz) != SQLITE_OK) goto error;
+  } else { /* SQLCIPHER_DECRYPT */
+    memcpy(iv_out, iv_in, ctx->iv_sz); /* copy the iv from the input to output buffer */
+  } 
+
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == SQLCIPHER_DECRYPT)) {
+    if(sqlcipher_page_hmac(ctx, c_ctx, pgno, in, size + ctx->iv_sz, hmac_out) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: hmac operation on decrypt failed for pgno=%d", __func__, pgno);
+      goto error;
+    }
+
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: comparing hmac on in=%p out=%p hmac_sz=%d", __func__, hmac_in, hmac_out, ctx->hmac_sz);
+    if(sqlcipher_memcmp(hmac_in, hmac_out, ctx->hmac_sz) != 0) { /* the hmac check failed */ 
+      if(sqlite3BtreeGetAutoVacuum(ctx->pBt) != BTREE_AUTOVACUUM_NONE && sqlcipher_ismemset(in, 0, page_sz) == 0) {
+        /* first check if the entire contents of the page is zeros. If so, this page 
+           resulted from a short read (i.e. sqlite attempted to pull a page after the end of the file. these 
+           short read failures must be ignored for autovaccum mode to work so wipe the output buffer 
+           and return SQLITE_OK to skip the decryption step. */
+        sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: zeroed page (short read) for pgno %d with autovacuum enabled", __func__, pgno);
+        sqlcipher_memset(out, 0, page_sz); 
+        return SQLITE_OK;
+      } else {
+        /* if the page memory is not all zeros, it means the there was data and a hmac on the page. 
+           since the check failed, the page was either tampered with or corrupted. wipe the output buffer,
+           and return SQLITE_ERROR to the caller */
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: hmac check failed for pgno=%d", __func__, pgno);
+        goto error;
+      }
+    }
+  } 
+  
+  sqlcipher_shield(c_ctx->key, ctx->key_sz);
+  rc = ctx->provider->cipher(ctx->provider_ctx, mode, c_ctx->key, ctx->key_sz, iv_out, in, size, out);
+  sqlcipher_shield(c_ctx->key, ctx->key_sz);
+
+  if(rc != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: cipher operation mode=%d failed for pgno=%d", __func__, mode, pgno);
+    goto error;
+  };
+ 
+  if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == SQLCIPHER_ENCRYPT)) {
+    if(sqlcipher_page_hmac(ctx, c_ctx, pgno, out_start, size + ctx->iv_sz, hmac_out) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: hmac operation on encrypt failed for pgno=%d", __func__, pgno);
+      goto error;
+    }; 
+  }
+
+  CODEC_HEXDUMP("sqlcipher_page_cipher: output page data", out_start, page_sz);
+
+  return SQLITE_OK;
+error:
+  sqlcipher_memset(out, 0, page_sz); 
+  return SQLITE_ERROR;
+}
+
+/**
+  * Derive an encryption key for a cipher contex key based on the raw password.
+  *
+  * If the raw key data is formated as x'hex' and there are exactly enough hex chars to fill
+  * the key (i.e 64 hex chars for a 256 bit key) then the key data will be used directly. 
+
+  * Else, if the raw key data is formated as x'hex' and there are exactly enough hex chars to fill
+  * the key and the salt (i.e 92 hex chars for a 256 bit key and 16 byte salt) then it will be unpacked
+  * as the key followed by the salt.
+  * 
+  * Otherwise, a key data will be derived using PBKDF2
+  * 
+  * returns SQLITE_OK if initialization was successful
+  * returns SQLITE_ERROR if the key could't be derived (for instance if pass is NULL or pass_sz is 0)
+  */
+static int sqlcipher_cipher_ctx_key_derive(codec_ctx *ctx, cipher_ctx *c_ctx) {
+  int rc, raw_key_sz = 0, raw_salt_sz = 0, blob_format = 0, derive_hmac_key = 1;
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: ctx->kdf_salt_sz=%d ctx->kdf_iter=%d ctx->fast_kdf_iter=%d ctx->key_sz=%d",
+    __func__, ctx->kdf_salt_sz, ctx->kdf_iter, ctx->fast_kdf_iter, ctx->key_sz);
+
+  /* if key material is present on the context for derivation */
+  if(!c_ctx->pass || !c_ctx->pass_sz) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_cipher_ctx_key_derive: key material is not present on the context for key derivation");
+    return SQLITE_ERROR;
+  }
+
+  /* if necessary, initialize the salt from the header or random source */
+  if(!SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) {
+    if((rc = sqlcipher_codec_ctx_init_kdf_salt(ctx)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: error %d from sqlcipher_codec_ctx_init_kdf_salt", __func__, rc);
+      goto error;
+    }
+  }
+
+  /* raw hey hex encoded is 2x long */
+  raw_key_sz = ctx->key_sz * 2;
+  raw_salt_sz = ctx->kdf_salt_sz *2;
+
+  /* raw key must be BLOB formatted:
+   * 1. greater than or equal to 5 characters long
+   * 2. starting with x'
+   * 3. ending with '
+   * 4. length of contents between the x' and ' must be a power of 2
+   * 5. contents must be hex */
+  blob_format =
+    c_ctx->pass_sz >= 5
+    && sqlite3StrNICmp((const char *)c_ctx->pass ,"x'", 2) == 0
+    && c_ctx->pass[c_ctx->pass_sz - 1] == '\''
+    && (c_ctx->pass_sz - 3) % 2 == 0
+    && cipher_isHex(c_ctx->pass + 2, c_ctx->pass_sz - 3);
+
+  if(blob_format && c_ctx->pass_sz == raw_key_sz + 3) {
+    /* option 1 - raw key consisting of only the encryption key */
+    const unsigned char *z = c_ctx->pass + 2; /* adjust lead offset of x' */
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: using raw key only", __func__);
+    cipher_hex2bin(z, raw_key_sz, c_ctx->key);
+  } else if(blob_format && c_ctx->pass_sz == raw_key_sz + raw_salt_sz + 3) {
+    /* option 2 - raw key consisting of the encryption key and salt */
+    const unsigned char *z = c_ctx->pass + 2;
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: using raw key and salt", __func__);
+    cipher_hex2bin(z, raw_key_sz, c_ctx->key);
+    cipher_hex2bin(z + raw_key_sz, raw_salt_sz, ctx->kdf_salt);
+  } else if(blob_format && c_ctx->pass_sz == raw_key_sz + raw_key_sz + raw_salt_sz + 3) {
+    /* option 3 - raw key consisting of the encryption key, then hmac key, then salt */
+    const unsigned char *z = c_ctx->pass + 2;
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: using raw key, hmac key, and salt", __func__);
+    cipher_hex2bin(z, raw_key_sz, c_ctx->key);
+    cipher_hex2bin(z + raw_key_sz, raw_key_sz, c_ctx->hmac_key);
+    cipher_hex2bin(z + raw_key_sz + raw_key_sz, raw_salt_sz, ctx->kdf_salt);
+    derive_hmac_key = 0;
+  } else {
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: deriving key using PBKDF2 with %d iterations", __func__, ctx->kdf_iter);
+    if((rc = ctx->provider->kdf(ctx->provider_ctx, ctx->kdf_algorithm, c_ctx->pass, c_ctx->pass_sz,
+                  ctx->kdf_salt, ctx->kdf_salt_sz, ctx->kdf_iter,
+                  ctx->key_sz, c_ctx->key)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: error %d occurred from provider kdf generating encryption key", __func__, rc);
+      goto error;
+    }
+  }
+
+  /* if this context is setup to use hmac checks, and we didn't already get an hmac
+   * key inbound via keyspec / raw key, then generate a seperate
+   * key for HMAC. In this case, we use the output of the previous KDF as the input to
+   * this KDF run. This ensures a distinct but predictable HMAC key. */
+  if(ctx->flags & CIPHER_FLAG_HMAC && derive_hmac_key) {
+    int i;
+
+    /* start by copying the kdf key into the hmac salt slot
+       then XOR it with the fixed hmac salt defined at compile time
+       this ensures that the salt passed in to derive the hmac key, while
+       easy to derive and publically known, is not the same as the salt used
+       to generate the encryption key */
+    memcpy(ctx->hmac_kdf_salt, ctx->kdf_salt, ctx->kdf_salt_sz);
+    for(i = 0; i < ctx->kdf_salt_sz; i++) {
+      ctx->hmac_kdf_salt[i] ^= hmac_salt_mask;
+    }
+
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: deriving hmac key from encryption key using PBKDF2 with %d iterations",
+      __func__, ctx->fast_kdf_iter);
+
+    if((rc = ctx->provider->kdf(ctx->provider_ctx, ctx->kdf_algorithm, c_ctx->key, ctx->key_sz,
+                  ctx->hmac_kdf_salt, ctx->kdf_salt_sz, ctx->fast_kdf_iter,
+                  ctx->key_sz, c_ctx->hmac_key)) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: error occurred from provider kdf generating HMAC key", __func__, rc);
+      goto error;
+    }
+  }
+
+  sqlcipher_shield(c_ctx->key, ctx->key_sz);
+  sqlcipher_shield(c_ctx->hmac_key, ctx->key_sz);
+  c_ctx->derive_key = 0;
+  return SQLITE_OK;
+
+error:
+  /* if an error occurred, overwrite any derived key material */
+  xoshiro_randomness(c_ctx->key, ctx->key_sz);
+  xoshiro_randomness(c_ctx->hmac_key, ctx->key_sz);
+  return SQLITE_ERROR;
+}
+
+static int sqlcipher_codec_key_derive(codec_ctx *ctx) {
+  /* derive key on first use if necessary */
+  if(ctx->read_ctx->derive_key) {
+    if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->read_ctx) != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_key_derive: error occurred deriving read_ctx key");
+      return SQLITE_ERROR;
+    }
+  }
+
+  if(ctx->write_ctx->derive_key) {
+    if(sqlcipher_cipher_ctx_cmp(ctx->write_ctx, ctx->read_ctx) == 0) {
+      /* the relevant parameters are the same, just copy read key */
+      if(sqlcipher_cipher_ctx_copy(ctx, ctx->write_ctx, ctx->read_ctx) != SQLITE_OK) {
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_key_derive: error occurred copying read_ctx to write_ctx");
+        return SQLITE_ERROR;
+      }
+    } else {
+      if(sqlcipher_cipher_ctx_key_derive(ctx, ctx->write_ctx) != SQLITE_OK) {
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_key_derive: error occurred deriving write_ctx key");
+        return SQLITE_ERROR;
+      }
+    }
+  }
+
+  /* wipe and free passphrase after key derivation */
+  if(ctx->store_pass  != 1) {
+    sqlcipher_cipher_ctx_set_pass(ctx->read_ctx, NULL, 0);
+    sqlcipher_cipher_ctx_set_pass(ctx->write_ctx, NULL, 0);
+  }
+
+  return SQLITE_OK; 
+}
+
+static int sqlcipher_codec_key_copy(codec_ctx *ctx, int source) {
+  if(source == CIPHER_READ_CTX) { 
+      return sqlcipher_cipher_ctx_copy(ctx, ctx->write_ctx, ctx->read_ctx); 
+  } else {
+      return sqlcipher_cipher_ctx_copy(ctx, ctx->read_ctx, ctx->write_ctx); 
+  }
+}
+
+static int sqlcipher_check_connection(const char *filename, char *key, int key_sz, char *sql, int *user_version, char** journal_mode) {
+  int rc;
+  sqlite3 *db = NULL;
+  sqlite3_stmt *statement = NULL;
+  char *query_journal_mode = "PRAGMA journal_mode;";
+  char *query_user_version = "PRAGMA user_version;";
+ 
+  rc = sqlite3_open(filename, &db);
+  if(rc != SQLITE_OK) goto cleanup; 
+    
+  rc = sqlite3_key(db, key, key_sz);
+  if(rc != SQLITE_OK) goto cleanup; 
+    
+  rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
+  if(rc != SQLITE_OK) goto cleanup; 
+
+  /* start by querying the user version. 
+     this will fail if the key is incorrect */
+  rc = sqlite3_prepare(db, query_user_version, -1, &statement, NULL);
+  if(rc != SQLITE_OK) goto cleanup; 
+    
+  rc = sqlite3_step(statement);
+  if(rc == SQLITE_ROW) {
+    *user_version = sqlite3_column_int(statement, 0);
+  } else {
+    goto cleanup;
+  }
+  sqlite3_finalize(statement); 
+
+  rc = sqlite3_prepare(db, query_journal_mode, -1, &statement, NULL);
+  if(rc != SQLITE_OK) goto cleanup; 
+    
+  rc = sqlite3_step(statement);
+  if(rc == SQLITE_ROW) {
+    *journal_mode = sqlite3_mprintf("%s", sqlite3_column_text(statement, 0)); 
+  } else {
+    goto cleanup; 
+  }
+  rc = SQLITE_OK;
+  /* cleanup will finalize open statement */
+  
+cleanup:
+  if(statement) sqlite3_finalize(statement); 
+  if(db) sqlite3_close(db); 
+  return rc;
+}
+
+static int sqlcipher_codec_ctx_integrity_check(codec_ctx *ctx, Parse *pParse, char *column) {
+  Pgno page = 1;
+  int rc = 0;
+  char *result;
+  unsigned char *hmac_out = NULL;
+  sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(ctx->pBt));
+  i64 file_sz;
+
+  Vdbe *v = sqlite3GetVdbe(pParse);
+  sqlite3VdbeSetNumCols(v, 1);
+  sqlite3VdbeSetColName(v, 0, COLNAME_NAME, column, SQLITE_STATIC);
+
+  if(fd == NULL || fd->pMethods == 0) {
+    sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, "database file is undefined", P4_TRANSIENT);
+    sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+    goto cleanup;
+  }
+
+  if(!(ctx->flags & CIPHER_FLAG_HMAC)) {
+    sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, "HMAC is not enabled, unable to integrity check", P4_TRANSIENT);
+    sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+    goto cleanup;
+  }
+
+  if((rc = sqlcipher_codec_key_derive(ctx)) != SQLITE_OK) {
+    sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, "unable to derive keys", P4_TRANSIENT);
+    sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+    goto cleanup;
+  }
+
+  sqlite3OsFileSize(fd, &file_sz);
+  hmac_out = sqlcipher_malloc(ctx->hmac_sz);
+
+  for(page = 1; page <= file_sz / ctx->page_sz; page++) {
+    i64 offset = (page - 1) * ctx->page_sz;
+    int payload_sz = ctx->page_sz - ctx->reserve_sz + ctx->iv_sz;
+    int read_sz = ctx->page_sz;
+
+    /* skip integrity check on PAGER_SJ_PGNO since it will have no valid content */
+    if(sqlite3pager_is_sj_pgno(sqlite3BtreePager(ctx->pBt), page)) continue;
+
+    if(page==1) {
+      int page1_offset = ctx->plaintext_header_sz ? ctx->plaintext_header_sz : FILE_HEADER_SZ;
+      read_sz = read_sz - page1_offset;
+      payload_sz = payload_sz - page1_offset;
+      offset += page1_offset;
+    }
+
+    sqlcipher_memset(ctx->buffer, 0, ctx->page_sz);
+    sqlcipher_memset(hmac_out, 0, ctx->hmac_sz);
+    if(sqlite3OsRead(fd, ctx->buffer, read_sz, offset) != SQLITE_OK) {
+      result = sqlite3_mprintf("error reading %d bytes from file page %d at offset %d", read_sz, page, offset);
+      sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC);
+      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+    } else if(sqlcipher_page_hmac(ctx, ctx->read_ctx, page, ctx->buffer, payload_sz, hmac_out) != SQLITE_OK) {
+      result = sqlite3_mprintf("HMAC operation failed for page %d", page);
+      sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC);
+      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+    } else if(sqlcipher_memcmp(ctx->buffer + payload_sz, hmac_out, ctx->hmac_sz) != 0) {
+      result = sqlite3_mprintf("HMAC verification failed for page %d", page);
+      sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC);
+      sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+    }
+  }
+
+  if(file_sz % ctx->page_sz != 0) {
+    result = sqlite3_mprintf("page %d has an invalid size of %lld bytes (expected %d bytes)", page, file_sz - ((file_sz / ctx->page_sz) * ctx->page_sz), ctx->page_sz);
+    sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, result, P4_DYNAMIC);
+    sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+  }
+
+cleanup:
+  if(hmac_out != NULL) sqlcipher_free(hmac_out, ctx->hmac_sz);
+  return SQLITE_OK;
+}
+
+static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) {
+  int i, pass_sz, keyspec_sz, nRes, user_version, rc, rc_cleanup, oflags;
+  Db *pDb = 0;
+  sqlite3 *db = ctx->pBt->db;
+  const char *db_filename = sqlite3_db_filename(db, "main");
+  char *set_user_version = NULL, *pass = NULL, *attach_command = NULL, *migrated_db_filename = NULL, *keyspec = NULL, *temp = NULL, *journal_mode = NULL, *set_journal_mode = NULL, *pragma_compat = NULL;
+  Btree *pDest = NULL, *pSrc = NULL;
+  sqlite3_file *srcfile, *destfile;
+#if defined(_WIN32) || defined(SQLITE_OS_WINRT)
+  LPWSTR w_db_filename = NULL, w_migrated_db_filename = NULL;
+  int w_db_filename_sz = 0, w_migrated_db_filename_sz = 0;
+#endif
+  pass_sz = keyspec_sz = rc = user_version = 0;
+
+  if(!db_filename || sqlite3Strlen30(db_filename) < 1) 
+    goto cleanup; /* exit immediately if this is an in memory database */ 
+  
+  /* pull the provided password / key material off the current codec context */
+  pass_sz = ctx->read_ctx->pass_sz;
+  pass = sqlcipher_malloc(pass_sz+1);
+  memset(pass, 0, pass_sz+1);
+  memcpy(pass, ctx->read_ctx->pass, pass_sz);
+
+  /* Version 4 - current, no upgrade required, so exit immediately */
+  rc = sqlcipher_check_connection(db_filename, pass, pass_sz, "", &user_version, &journal_mode);
+  if(rc == SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: no upgrade required - exiting");
+    goto cleanup;
+  }
+
+  for(i = 3; i > 0; i--) {
+    pragma_compat = sqlite3_mprintf("PRAGMA cipher_compatibility = %d;", i);
+    rc = sqlcipher_check_connection(db_filename, pass, pass_sz, pragma_compat, &user_version, &journal_mode);
+    if(rc == SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: version %d format found", i);
+      goto migrate;
+    }
+    if(pragma_compat) sqlcipher_free(pragma_compat, sqlite3Strlen30(pragma_compat)); 
+    pragma_compat = NULL;
+  }
+  
+  /* if we exit the loop normally we failed to determine the version, this is an error */
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: unable to determine format version for upgrade: this may indicate custom settings were used ");
+  goto handle_error;
+
+migrate:
+
+  temp = sqlite3_mprintf("%s-migrated", db_filename);
+  /* overallocate migrated_db_filename, because sqlite3OsOpen will read past the null terminator
+   * to determine whether the filename was URI formatted */
+  migrated_db_filename = sqlcipher_malloc(sqlite3Strlen30(temp)+2); 
+  memcpy(migrated_db_filename, temp, sqlite3Strlen30(temp));
+  sqlcipher_free(temp, sqlite3Strlen30(temp));
+
+  attach_command = sqlite3_mprintf("ATTACH DATABASE '%s' as migrate;", migrated_db_filename, pass); 
+  set_user_version = sqlite3_mprintf("PRAGMA migrate.user_version = %d;", user_version);
+
+  rc = sqlite3_exec(db, pragma_compat, NULL, NULL, NULL);
+  if(rc != SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: set compatibility mode failed, error code %d", rc);
+    goto handle_error;
+  }
+
+  /* force journal mode to DELETE, we will set it back later if different */
+  rc = sqlite3_exec(db, "PRAGMA journal_mode = delete;", NULL, NULL, NULL);
+  if(rc != SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: force journal mode DELETE failed, error code %d", rc);
+    goto handle_error;
+  }
+
+  rc = sqlite3_exec(db, attach_command, NULL, NULL, NULL);
+  if(rc != SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: attach failed, error code %d", rc);
+    goto handle_error;
+  }
+
+  rc = sqlite3_key_v2(db, "migrate", pass, pass_sz);
+  if(rc != SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: keying attached database failed, error code %d", rc);
+    goto handle_error;
+  }
+
+  rc = sqlite3_exec(db, "SELECT sqlcipher_export('migrate');", NULL, NULL, NULL);
+  if(rc != SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: sqlcipher_export failed, error code %d", rc);
+    goto handle_error;
+  }
+
+#ifdef SQLCIPHER_TEST
+  if((cipher_test_flags & TEST_FAIL_MIGRATE) > 0) {
+    rc = SQLITE_ERROR;
+    sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: simulated migrate failure, error code %d", rc);
+    goto handle_error;
+  }
+#endif
+
+  rc = sqlite3_exec(db, set_user_version, NULL, NULL, NULL);
+  if(rc != SQLITE_OK){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: set user version failed, error code %d", rc);
+    goto handle_error;
+  }
+
+  if( !db->autoCommit ){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: cannot migrate from within a transaction");
+    goto handle_error;
+  }
+  if( db->nVdbeActive>1 ){
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: cannot migrate - SQL statements in progress");
+    goto handle_error;
+  }
+
+  pDest = db->aDb[0].pBt;
+  pDb = &(db->aDb[db->nDb-1]);
+  pSrc = pDb->pBt;
+
+  nRes = sqlite3BtreeGetRequestedReserve(pSrc);
+  /* unset the BTS_PAGESIZE_FIXED flag to avoid SQLITE_READONLY */
+  pDest->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; 
+  rc = sqlite3BtreeSetPageSize(pDest, default_page_size, nRes, 0);
+  if(rc != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: failed to set btree page size to %d res %d rc %d", default_page_size, nRes, rc);
+    goto handle_error;
+  }
+
+  sqlcipherCodecGetKey(db, db->nDb - 1, (void**)&keyspec, &keyspec_sz);
+  SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_KEY_USED);
+  sqlcipherCodecAttach(db, 0, keyspec, keyspec_sz);
+  
+  srcfile = sqlite3PagerFile(sqlite3BtreePager(pSrc));
+  destfile = sqlite3PagerFile(sqlite3BtreePager(pDest));
+
+  sqlite3OsClose(srcfile);
+  sqlite3OsClose(destfile); 
+
+#if defined(_WIN32) || defined(SQLITE_OS_WINRT)
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: performing windows MoveFileExA");
+
+  w_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) db_filename, -1, NULL, 0);
+  w_db_filename = sqlcipher_malloc(w_db_filename_sz * sizeof(wchar_t));
+  w_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) db_filename, -1, (const LPWSTR) w_db_filename, w_db_filename_sz);
+
+  w_migrated_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) migrated_db_filename, -1, NULL, 0);
+  w_migrated_db_filename = sqlcipher_malloc(w_migrated_db_filename_sz * sizeof(wchar_t));
+  w_migrated_db_filename_sz = MultiByteToWideChar(CP_UTF8, 0, (LPCCH) migrated_db_filename, -1, (const LPWSTR) w_migrated_db_filename, w_migrated_db_filename_sz);
+
+  if(!MoveFileExW(w_migrated_db_filename, w_db_filename, MOVEFILE_REPLACE_EXISTING)) {
+    rc = SQLITE_ERROR;
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: error occurred while renaming migration files %d", rc);
+    goto handle_error;
+  }
+#else
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: performing POSIX rename");
+  if ((rc = rename(migrated_db_filename, db_filename)) != 0) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: error occurred while renaming migration files %s to %s: %d", migrated_db_filename, db_filename, rc);
+    goto handle_error;
+  }
+#endif
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: renamed migration database %s to main database %s: %d", migrated_db_filename, db_filename, rc);
+
+  rc = sqlite3OsOpen(db->pVfs, migrated_db_filename, srcfile, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB, &oflags);
+  if(rc != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: failed to reopen migration database %s: %d", migrated_db_filename, rc);
+    goto handle_error;
+  }
+
+  rc = sqlite3OsOpen(db->pVfs, db_filename, destfile, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB, &oflags);
+  if(rc != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: failed to reopen main database %s: %d", db_filename, rc);
+    goto handle_error;
+  }
+
+  sqlite3pager_reset(sqlite3BtreePager(pDest));
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: reset pager");
+
+handle_error:
+  rc_cleanup = sqlite3_exec(db, "DETACH DATABASE migrate;", NULL, NULL, NULL);
+  if(rc_cleanup != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: DETACH DATABASE migrate failed: %d", rc_cleanup);
+    /* only overwrite the rc in the cleanup stage if it is currently not an error. This will prevent overwriting a previous error that occured earlier in migration */
+    if(rc == SQLITE_OK) { 
+      rc = rc_cleanup;
+    }
+  }
+
+  sqlite3ResetAllSchemasOfConnection(db);
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: reset all schemas");
+
+  if(journal_mode) {
+    set_journal_mode = sqlite3_mprintf("PRAGMA journal_mode = %s;", journal_mode);
+    rc_cleanup = sqlite3_exec(db, set_journal_mode, NULL, NULL, NULL);
+    if(rc_cleanup != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: failed to re-set journal mode via %s: %d", set_journal_mode, rc_cleanup);
+      if(rc == SQLITE_OK) {
+        rc = rc_cleanup;
+      }
+    }
+  }
+
+  if(migrated_db_filename) {
+    int del_rc = sqlite3OsDelete(db->pVfs, migrated_db_filename, 0);
+    if(del_rc != SQLITE_OK) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: failed to delete migration database %s: %d", migrated_db_filename, del_rc);
+    }
+  }
+
+  if(rc != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: an error occurred attempting to migrate the database - last error %d", rc);
+    sqlite3pager_reset(sqlite3BtreePager(ctx->pBt));
+    ctx->error = rc; /* set flag for deferred error */
+  }
+
+cleanup:
+  if(pass) sqlcipher_free(pass, pass_sz);
+  if(keyspec) sqlcipher_free(keyspec, keyspec_sz);
+  if(attach_command) sqlcipher_free(attach_command, sqlite3Strlen30(attach_command)); 
+  if(migrated_db_filename) sqlcipher_free(migrated_db_filename, sqlite3Strlen30(migrated_db_filename)); 
+  if(set_user_version) sqlcipher_free(set_user_version, sqlite3Strlen30(set_user_version)); 
+  if(set_journal_mode) sqlcipher_free(set_journal_mode, sqlite3Strlen30(set_journal_mode)); 
+  if(journal_mode) sqlcipher_free(journal_mode, sqlite3Strlen30(journal_mode)); 
+  if(pragma_compat) sqlcipher_free(pragma_compat, sqlite3Strlen30(pragma_compat)); 
+#if defined(_WIN32) || defined(SQLITE_OS_WINRT)
+  if(w_db_filename) sqlcipher_free(w_db_filename, w_db_filename_sz);
+  if(w_migrated_db_filename) sqlcipher_free(w_migrated_db_filename, w_migrated_db_filename_sz);
+#endif
+
+  return rc;
+}
+
+static int sqlcipher_codec_add_random(codec_ctx *ctx, const char *zRight, int random_sz){
+  const char *suffix = &zRight[random_sz-1];
+  int n = random_sz - 3; /* adjust for leading x' and tailing ' */
+  if (n > 0 &&
+      sqlite3StrNICmp((const char *)zRight ,"x'", 2) == 0 &&
+      sqlite3StrNICmp(suffix, "'", 1) == 0 &&
+      n % 2 == 0) {
+    int rc = 0;
+    int buffer_sz = n / 2;
+    unsigned char *random;
+    const unsigned char *z = (const unsigned char *)zRight + 2; /* adjust lead offset of x' */
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_add_random: using raw random blob from hex");
+    random = sqlcipher_malloc(buffer_sz);
+    memset(random, 0, buffer_sz);
+    cipher_hex2bin(z, n, random);
+    rc = ctx->provider->add_random(ctx->provider_ctx, random, buffer_sz);
+    sqlcipher_free(random, buffer_sz);
+    return rc;
+  }
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_add_random: attemt to add random with invalid format");
+  return SQLITE_ERROR;
+}
+
+#if defined(_WIN32)
+/* On windows convert to utf-16 when writing to stderr or stdout to avoid
+ * a potential exception when writing mixed context to those streams
+ * when using the shell. */
+static int sqlcipher_fprintf(FILE* stream, const char* format, ...) {
+  int sz;
+  va_list ap;
+
+  if (stream == stderr || stream == stdout) {
+    char* buffer = NULL;
+    wchar_t* wbuffer = NULL;
+
+    va_start(ap, format);
+    buffer = sqlite3_vmprintf(format, ap);
+    va_end(ap);
+    sz = (int)strlen(buffer);
+
+    wbuffer = sqlite3_malloc((sz + 1) * sizeof(wchar_t));
+    if (wbuffer == NULL) return -1;
+
+    sz = MultiByteToWideChar(CP_UTF8, 0, buffer, sz, wbuffer, sz);
+    wbuffer[sz] = (wchar_t) 0;
+    fputws(wbuffer, stream);
+
+    sqlite3_free(wbuffer);
+    sqlite3_free(buffer);
+  } else {
+    va_start(ap, format);
+    sz = vfprintf(stream, format, ap);
+    va_end(ap);
+  }
+  return sz;
+}
+#else
+#define sqlcipher_fprintf fprintf
+#endif
+
+#if !defined(SQLITE_OMIT_TRACE)
+
+#define SQLCIPHER_PROFILE_FMT        "Elapsed time:%.3f ms - %s\n"
+#define SQLCIPHER_PROFILE_FMT_OSLOG  "Elapsed time:%{public}.3f ms - %{public}s\n"
+
+static int sqlcipher_profile_callback(unsigned int trace, void *file, void *stmt, void *run_time){
+  FILE *f = (FILE*) file;
+  double elapsed = (*((sqlite3_uint64*)run_time))/1000000.0;
+  if(f == NULL) {
+#if !defined(SQLCIPHER_OMIT_LOG_DEVICE)
+#if defined(__ANDROID__)
+    __android_log_print(ANDROID_LOG_DEBUG, "sqlcipher", SQLCIPHER_PROFILE_FMT, elapsed, sqlite3_sql((sqlite3_stmt*)stmt));
+#elif defined(__APPLE__)
+    os_log(OS_LOG_DEFAULT, SQLCIPHER_PROFILE_FMT_OSLOG, elapsed, sqlite3_sql((sqlite3_stmt*)stmt));
+#endif
+#endif
+  } else {
+    sqlcipher_fprintf(f, SQLCIPHER_PROFILE_FMT, elapsed, sqlite3_sql((sqlite3_stmt*)stmt));
+  }
+  return SQLITE_OK;
+}
+#endif
+
+static int sqlcipher_cipher_profile(sqlite3 *db, const char *destination){
+#if defined(SQLITE_OMIT_TRACE)
+  return SQLITE_ERROR;
+#else
+  FILE *f = NULL;
+  if(sqlite3_stricmp(destination, "off") == 0){
+    sqlite3_trace_v2(db, 0, NULL, NULL); /* disable tracing */
+  } else {
+    if(sqlite3_stricmp(destination, "stdout") == 0){
+      f = stdout;
+    }else if(sqlite3_stricmp(destination, "stderr") == 0){
+      f = stderr;
+    }else if(sqlite3_stricmp(destination, "logcat") == 0 || sqlite3_stricmp(destination, "device") == 0){
+      f = NULL; /* file pointer will be NULL indicating the device target (i.e. logcat or oslog). We will accept logcat for backwards compatibility */
+    }else{
+#if !defined(SQLCIPHER_PROFILE_USE_FOPEN) && (defined(_WIN32) && (__STDC_VERSION__ > 199901L) || defined(SQLITE_OS_WINRT))
+      if(fopen_s(&f, destination, "a") != 0) return SQLITE_ERROR;
+#else
+      if((f = fopen(destination, "a")) == 0) return SQLITE_ERROR;
+#endif    
+    }
+    sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, sqlcipher_profile_callback, f);
+  }
+  return SQLITE_OK;
+#endif
+}
+
+static char *sqlcipher_get_log_level_str(unsigned int level) {
+  switch(level) {
+    case SQLCIPHER_LOG_ERROR:
+      return "ERROR";
+    case SQLCIPHER_LOG_WARN:
+      return "WARN";
+    case SQLCIPHER_LOG_INFO:
+      return "INFO";
+    case SQLCIPHER_LOG_DEBUG:
+      return "DEBUG";
+    case SQLCIPHER_LOG_TRACE:
+      return "TRACE";
+    case SQLCIPHER_LOG_ANY:
+      return "ANY";
+  }
+  return "NONE";
+}
+
+static char *sqlcipher_get_log_source_str(unsigned int source) {
+  switch(source) {
+    case SQLCIPHER_LOG_NONE:
+      return "NONE";
+    case SQLCIPHER_LOG_CORE:
+      return "CORE";
+    case SQLCIPHER_LOG_MEMORY:
+      return "MEMORY";
+    case SQLCIPHER_LOG_MUTEX:
+      return "MUTEX";
+    case SQLCIPHER_LOG_PROVIDER:
+      return "PROVIDER";
+  }
+  return "ANY";
+}
+
+static char *sqlcipher_get_log_sources_str(unsigned int source) {
+  if(source == SQLCIPHER_LOG_NONE) {
+    return sqlite3_mprintf("%s", "NONE");
+  } else if (source == SQLCIPHER_LOG_ANY) {
+    return sqlite3_mprintf("%s", "ANY");
+  } else {
+    char *sources = NULL;
+    unsigned int flag;
+    for(flag = SQLCIPHER_LOG_CORE; flag != 0; flag = flag << 1) {
+      if(SQLCIPHER_FLAG_GET(source, flag)) {
+        char *src = sqlcipher_get_log_source_str(flag);
+        if(sources) {
+          char *tmp = sqlite3_mprintf("%s %s", sources, src);
+          sqlite3_free(sources);
+          sources = tmp;
+        } else {
+          sources = sqlite3_mprintf("%s", src);
+        }
+      }
+    }
+    return sources;
+  }
+}
+
+#ifndef SQLCIPHER_OMIT_LOG
+/* constants from https://github.com/Alexpux/mingw-w64/blob/master/mingw-w64-crt/misc/gettimeofday.c */
+#define FILETIME_1970 116444736000000000ull /* seconds between 1/1/1601 and 1/1/1970 */
+#define HECTONANOSEC_PER_SEC 10000000ull
+#define MAX_LOG_LEN 8192
+void sqlcipher_log(unsigned int level, unsigned int source, const char *message, ...) {
+  va_list params;
+  va_start(params, message);
+  char formatted[MAX_LOG_LEN];
+  size_t len = 0;
+
+#ifdef CODEC_DEBUG
+#if defined(SQLCIPHER_OMIT_LOG_DEVICE) || (!defined(__ANDROID__) && !defined(__APPLE__))
+    sqlite3_vsnprintf(MAX_LOG_LEN, formatted, message, params);
+    sqlcipher_fprintf(stderr, formatted);
+    sqlcipher_fprintf(stderr, "\n");
+    goto end;
+#else
+#if defined(__ANDROID__)
+    __android_log_vprint(ANDROID_LOG_DEBUG, "sqlcipher", message, params);
+    goto end;
+#elif defined(__APPLE__)
+    sqlite3_vsnprintf(MAX_LOG_LEN, formatted, message, params);
+    os_log(OS_LOG_DEFAULT, "%{public}s", formatted);
+    goto end;
+#endif
+#endif
+#endif
+  if(
+    level > sqlcipher_log_level /* log level is higher, e.g. level filter is at ERROR but this message is DEBUG */
+    || !SQLCIPHER_FLAG_GET(sqlcipher_log_source, source) /* source filter doesn't match this message source */
+    || (sqlcipher_log_device == 0 && sqlcipher_log_file == NULL) /* no configured log target */
+  ) {
+    /* skip logging this message */
+    goto end;
+  }
+
+  sqlite3_snprintf(MAX_LOG_LEN, formatted, "%s %s ", sqlcipher_get_log_level_str(level), sqlcipher_get_log_source_str(source));
+  len = strlen(formatted);
+  sqlite3_vsnprintf(MAX_LOG_LEN - (int) len, formatted + (int) len, message, params);
+
+#if !defined(SQLCIPHER_OMIT_LOG_DEVICE)
+  if(sqlcipher_log_device) {
+#if defined(__ANDROID__)
+    __android_log_write(ANDROID_LOG_DEBUG, "sqlcipher", formatted);
+    goto end;
+#elif defined(__APPLE__)
+    os_log(OS_LOG_DEFAULT, "%{public}s", formatted);
+    goto end;
+#endif
+  }
+#endif
+
+  if(sqlcipher_log_file != NULL){
+    char buffer[24];
+    struct tm tt;
+    int ms;
+    time_t sec;
+#ifdef _WIN32
+    SYSTEMTIME st;
+    FILETIME ft;
+    GetSystemTime(&st);
+    SystemTimeToFileTime(&st, &ft);
+    sec = (time_t) ((*((sqlite_int64*)&ft) - FILETIME_1970) / HECTONANOSEC_PER_SEC);
+    ms = st.wMilliseconds;
+    localtime_s(&tt, &sec);
+#else
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    sec = tv.tv_sec;
+    ms = tv.tv_usec/1000.0;
+    localtime_r(&sec, &tt);
+#endif
+    if(strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tt)) {
+      sqlcipher_fprintf((FILE*)sqlcipher_log_file, "%s.%03d: %s\n", buffer, ms, formatted);
+      goto end;
+    }
+  }
+
+end:
+  va_end(params);
+}
+#endif
+
+static int sqlcipher_set_log(const char *destination){
+#ifdef SQLCIPHER_OMIT_LOG
+  return SQLITE_ERROR;
+#else
+  /* close open trace file if it is not stdout or stderr, then
+     reset trace settings */
+  if(sqlcipher_log_file != NULL && sqlcipher_log_file != stdout && sqlcipher_log_file != stderr) {
+    fclose((FILE*)sqlcipher_log_file);
+  }
+  sqlcipher_log_file = NULL;
+  sqlcipher_log_device = 0;
+
+  if(sqlite3_stricmp(destination, "logcat") == 0 || sqlite3_stricmp(destination, "device") == 0){
+    /* use the appropriate device log. accept logcat for backwards compatibility */
+    sqlcipher_log_device = 1;
+  } else if(sqlite3_stricmp(destination, "stdout") == 0){
+    sqlcipher_log_file = stdout;
+  }else if(sqlite3_stricmp(destination, "stderr") == 0){
+    sqlcipher_log_file = stderr;
+  }else if(sqlite3_stricmp(destination, "off") != 0){
+#if !defined(SQLCIPHER_PROFILE_USE_FOPEN) && (defined(_WIN32) && (__STDC_VERSION__ > 199901L) || defined(SQLITE_OS_WINRT))
+    if(fopen_s(&sqlcipher_log_file, destination, "a") != 0) return SQLITE_ERROR;
+#else
+    if((sqlcipher_log_file = fopen(destination, "a")) == 0) return SQLITE_ERROR;
+#endif
+  }
+  sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "sqlcipher_set_log: set log to %s", destination);
+  return SQLITE_OK;
+#endif
+}
+
+static void sqlcipher_vdbe_return_string(Parse *pParse, const char *zLabel, const char *value, int value_type){
+  Vdbe *v = sqlite3GetVdbe(pParse);
+  sqlite3VdbeSetNumCols(v, 1);
+  sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLabel, SQLITE_STATIC);
+  sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, value, value_type);
+  sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1);
+}
+
+static int codec_set_btree_to_codec_pagesize(sqlite3 *db, Db *pDb, codec_ctx *ctx) {
+  int rc;
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "codec_set_btree_to_codec_pagesize: sqlite3BtreeSetPageSize() size=%d reserve=%d", ctx->page_sz, ctx->reserve_sz);
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "codec_set_btree_to_codec_pagesize: entering database mutex %p", db->mutex);
+  sqlite3_mutex_enter(db->mutex);
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "codec_set_btree_to_codec_pagesize: entered database mutex %p", db->mutex);
+  db->nextPagesize = ctx->page_sz; 
+
+  /* before forcing the page size we need to unset the BTS_PAGESIZE_FIXED flag, else  
+     sqliteBtreeSetPageSize will block the change  */
+  pDb->pBt->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED;
+  rc = sqlite3BtreeSetPageSize(pDb->pBt, ctx->page_sz, ctx->reserve_sz, 0);
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "codec_set_btree_to_codec_pagesize: sqlite3BtreeSetPageSize returned %d", rc);
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "codec_set_btree_to_codec_pagesize: leaving database mutex %p", db->mutex);
+  sqlite3_mutex_leave(db->mutex);
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "codec_set_btree_to_codec_pagesize: left database mutex %p", db->mutex);
+
+  return rc;
+}
+
+static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, int for_ctx) {
+  struct Db *pDb = &db->aDb[nDb];
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "codec_set_pass_key: db=%p nDb=%d for_ctx=%d", db, nDb, for_ctx);
+  if(pDb->pBt) {
+    codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt));
+
+    if(ctx) {
+      return sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, for_ctx);
+    } else {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "codec_set_pass_key: error ocurred fetching codec from pager on db %d", nDb);
+      return SQLITE_ERROR;
+    }
+  }
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "codec_set_pass_key: no btree present on db %d", nDb);
+  return SQLITE_ERROR;
+} 
+
+int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLeft, const char *zRight) {
+  struct Db *pDb = &db->aDb[iDb];
+  codec_ctx *ctx = NULL;
+  int rc;
+
+  if(pDb->pBt) {
+    ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt));
+  }
+
+  if(sqlite3_stricmp(zLeft, "key") !=0 && sqlite3_stricmp(zLeft, "rekey") != 0) {
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_pragma: db=%p iDb=%d pParse=%p zLeft=%s zRight=%s ctx=%p", db, iDb, pParse, zLeft, zRight, ctx);
+  }
+
+#ifdef SQLCIPHER_TEST
+  if( sqlite3_stricmp(zLeft,"cipher_test_on")==0 ){
+    if( zRight ) {
+      if(sqlite3_stricmp(zRight, "fail_encrypt")==0) {
+        SQLCIPHER_FLAG_SET(cipher_test_flags,TEST_FAIL_ENCRYPT);
+      } else
+      if(sqlite3_stricmp(zRight, "fail_decrypt")==0) {
+        SQLCIPHER_FLAG_SET(cipher_test_flags,TEST_FAIL_DECRYPT);
+      } else
+      if(sqlite3_stricmp(zRight, "fail_migrate")==0) {
+        SQLCIPHER_FLAG_SET(cipher_test_flags,TEST_FAIL_MIGRATE);
+      }
+    }
+  } else
+  if( sqlite3_stricmp(zLeft,"cipher_test_off")==0 ){
+    if( zRight ) {
+      if(sqlite3_stricmp(zRight, "fail_encrypt")==0) {
+        SQLCIPHER_FLAG_UNSET(cipher_test_flags,TEST_FAIL_ENCRYPT);
+      } else
+      if(sqlite3_stricmp(zRight, "fail_decrypt")==0) {
+        SQLCIPHER_FLAG_UNSET(cipher_test_flags,TEST_FAIL_DECRYPT);
+      } else
+      if(sqlite3_stricmp(zRight, "fail_migrate")==0) {
+        SQLCIPHER_FLAG_UNSET(cipher_test_flags,TEST_FAIL_MIGRATE);
+      }
+    }
+  } else
+  if( sqlite3_stricmp(zLeft,"cipher_test")==0 ){
+    char *flags = sqlite3_mprintf("%u", cipher_test_flags);
+    sqlcipher_vdbe_return_string(pParse, "cipher_test", flags, P4_DYNAMIC);
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_test_rand")==0 ){
+    if( zRight ) {
+      int rand = atoi(zRight);
+      cipher_test_rand = rand;
+    } else {
+      char *rand = sqlite3_mprintf("%d", cipher_test_rand);
+      sqlcipher_vdbe_return_string(pParse, "cipher_test_rand", rand, P4_DYNAMIC);
+    }
+  } else
+#endif
+  if( sqlite3_stricmp(zLeft, "cipher_fips_status")== 0 && !zRight ){
+    if(ctx) {
+      char *fips_mode_status = sqlite3_mprintf("%d", ctx->provider->fips_status(ctx->provider_ctx));
+      sqlcipher_vdbe_return_string(pParse, "cipher_fips_status", fips_mode_status, P4_DYNAMIC);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_status")== 0 && !zRight ){
+    if(ctx && ctx->error == SQLITE_OK) {
+      sqlcipher_vdbe_return_string(pParse, "cipher_status", "1", P4_TRANSIENT);
+    } else {
+      sqlcipher_vdbe_return_string(pParse, "cipher_status", "0", P4_TRANSIENT);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_store_pass")==0 && zRight ) {
+    if(ctx) {
+      char *deprecation = "PRAGMA cipher_store_pass is deprecated, please remove from use";
+      ctx->store_pass = sqlite3GetBoolean(zRight, 1);
+      sqlcipher_vdbe_return_string(pParse, "cipher_store_pass", deprecation, P4_TRANSIENT);
+      sqlite3_log(SQLITE_WARNING, deprecation);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_store_pass")==0 && !zRight ) {
+    if(ctx){
+      char *store_pass_value = sqlite3_mprintf("%d", ctx->store_pass);
+      sqlcipher_vdbe_return_string(pParse, "cipher_store_pass", store_pass_value, P4_DYNAMIC);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_profile")== 0 && zRight ){
+      char *profile_status = sqlite3_mprintf("%d", sqlcipher_cipher_profile(db, zRight));
+      sqlcipher_vdbe_return_string(pParse, "cipher_profile", profile_status, P4_DYNAMIC);
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_add_random")==0 && zRight ){
+    if(ctx) {
+      char *add_random_status = sqlite3_mprintf("%d", sqlcipher_codec_add_random(ctx, zRight, sqlite3Strlen30(zRight)));
+      sqlcipher_vdbe_return_string(pParse, "cipher_add_random", add_random_status, P4_DYNAMIC);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_migrate")==0 && !zRight ){
+    if(ctx){
+      int status = sqlcipher_codec_ctx_migrate(ctx); 
+      char *migrate_status = sqlite3_mprintf("%d", status);
+      sqlcipher_vdbe_return_string(pParse, "cipher_migrate", migrate_status, P4_DYNAMIC);
+      if(status != SQLITE_OK) {
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_pragma: error occurred during cipher_migrate: %d", status);
+      }
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_provider")==0 && !zRight ){
+    if(ctx) {
+      sqlcipher_vdbe_return_string(pParse, "cipher_provider",
+        ctx->provider->get_provider_name(ctx->provider_ctx), P4_TRANSIENT);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_provider_version")==0 && !zRight){
+    if(ctx) {
+      sqlcipher_vdbe_return_string(pParse, "cipher_provider_version",
+        ctx->provider->get_provider_version(ctx->provider_ctx), P4_TRANSIENT);
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_version")==0 && !zRight ){
+    sqlcipher_vdbe_return_string(pParse, "cipher_version", sqlcipher_version(), P4_DYNAMIC);
+  }else
+  if( sqlite3_stricmp(zLeft, "cipher")==0 ){
+    if(ctx) {
+      if( zRight ) {
+        const char* message = "PRAGMA cipher is no longer supported.";
+        sqlcipher_vdbe_return_string(pParse, "cipher", message, P4_TRANSIENT);
+        sqlite3_log(SQLITE_WARNING, message);
+      }else {
+        sqlcipher_vdbe_return_string(pParse, "cipher", 
+          ctx->provider->get_cipher(ctx->provider_ctx), P4_TRANSIENT); 
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft, "rekey_cipher")==0 && zRight ){
+    const char* message = "PRAGMA rekey_cipher is no longer supported.";
+    sqlcipher_vdbe_return_string(pParse, "rekey_cipher", message, P4_TRANSIENT);
+    sqlite3_log(SQLITE_WARNING, message);
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_default_kdf_iter")==0 ){
+    if( zRight ) {
+      default_kdf_iter = atoi(zRight); /* change default KDF iterations */
+    } else {
+      char *kdf_iter = sqlite3_mprintf("%d", default_kdf_iter);
+      sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_iter", kdf_iter, P4_DYNAMIC);
+    }
+  }else
+  if( sqlite3_stricmp(zLeft, "kdf_iter")==0 ){
+    if(ctx) {
+      if( zRight ) {
+        sqlcipher_codec_ctx_set_kdf_iter(ctx, atoi(zRight)); /* change of RW PBKDF2 iteration */
+      } else {
+        char *kdf_iter = sqlite3_mprintf("%d", ctx->kdf_iter);
+        sqlcipher_vdbe_return_string(pParse, "kdf_iter", kdf_iter, P4_DYNAMIC);
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft, "fast_kdf_iter")==0){
+    if(ctx) {
+      if( zRight ) {
+        char *deprecation = "PRAGMA fast_kdf_iter is deprecated, please remove from use";
+        sqlcipher_codec_ctx_set_fast_kdf_iter(ctx, atoi(zRight)); /* change of RW PBKDF2 iteration */
+        sqlcipher_vdbe_return_string(pParse, "fast_kdf_iter", deprecation, P4_TRANSIENT);
+        sqlite3_log(SQLITE_WARNING, deprecation);
+      } else {
+        char *fast_kdf_iter = sqlite3_mprintf("%d", ctx->fast_kdf_iter);
+        sqlcipher_vdbe_return_string(pParse, "fast_kdf_iter", fast_kdf_iter, P4_DYNAMIC);
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft, "rekey_kdf_iter")==0 && zRight ){
+    const char* message = "PRAGMA rekey_kdf_iter is no longer supported.";
+    sqlcipher_vdbe_return_string(pParse, "rekey_kdf_iter", message, P4_TRANSIENT);
+    sqlite3_log(SQLITE_WARNING, message);
+  }else
+  if( sqlite3_stricmp(zLeft,"page_size")==0 || sqlite3_stricmp(zLeft,"cipher_page_size")==0 ){
+    /* PRAGMA cipher_page_size will alter the size of the database pages while ensuring that the
+       required reserve space is allocated at the end of each page. This will also override the
+       standard SQLite PRAGMA page_size behavior if a codec context is attached to the database handle.
+       If PRAGMA page_size is invoked but a codec context is not attached (i.e. dealing with a standard
+       unencrypted database) then return early and allow the standard PRAGMA page_size logic to apply. */
+    if(ctx) {
+      if( zRight ) {
+        int size = atoi(zRight);
+        rc = sqlcipher_codec_ctx_set_pagesize(ctx, size);
+        if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc);
+        rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx);
+        if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc);
+      } else {
+        char * page_size = sqlite3_mprintf("%d", ctx->page_sz);
+        sqlcipher_vdbe_return_string(pParse, "cipher_page_size", page_size, P4_DYNAMIC);
+      }
+    } else {
+      return 0; /* return early so that the PragTyp_PAGE_SIZE case logic in pragma.c will take effect */
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_default_page_size")==0 ){
+    if( zRight ) {
+      default_page_size = atoi(zRight);
+    } else {
+      char *page_size = sqlite3_mprintf("%d", default_page_size);
+      sqlcipher_vdbe_return_string(pParse, "cipher_default_page_size", page_size, P4_DYNAMIC);
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_default_use_hmac")==0 ){
+    if( zRight ) {
+      sqlcipher_set_default_use_hmac(sqlite3GetBoolean(zRight,1));
+    } else {
+      char *default_use_hmac = sqlite3_mprintf("%d", SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC));
+      sqlcipher_vdbe_return_string(pParse, "cipher_default_use_hmac", default_use_hmac, P4_DYNAMIC);
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_use_hmac")==0 ){
+    if(ctx) {
+      if( zRight ) {
+        rc = sqlcipher_codec_ctx_set_use_hmac(ctx, sqlite3GetBoolean(zRight,1));
+        if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc);
+        /* since the use of hmac has changed, the page size may also change */
+        rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx);
+        if(rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, rc);
+      } else {
+        char *hmac_flag = sqlite3_mprintf("%d", SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC));
+        sqlcipher_vdbe_return_string(pParse, "cipher_use_hmac", hmac_flag, P4_DYNAMIC);
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_hmac_pgno")==0 ){
+    if(ctx) {
+      if(zRight) {
+        char *deprecation = "PRAGMA cipher_hmac_pgno is deprecated, please remove from use";
+        /* clear both pgno endian flags */
+        if(sqlite3_stricmp(zRight, "le") == 0) {
+          SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_BE_PGNO);
+          SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_LE_PGNO);
+        } else if(sqlite3_stricmp(zRight, "be") == 0) {
+          SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_LE_PGNO);
+          SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_BE_PGNO);
+        } else if(sqlite3_stricmp(zRight, "native") == 0) {
+          SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_LE_PGNO);
+          SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_BE_PGNO);
+        }
+        sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", deprecation, P4_TRANSIENT);
+        sqlite3_log(SQLITE_WARNING, deprecation);
+ 
+      } else {
+        if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_LE_PGNO)) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", "le", P4_TRANSIENT);
+        } else if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_BE_PGNO)) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", "be", P4_TRANSIENT);
+        } else {
+          sqlcipher_vdbe_return_string(pParse, "cipher_hmac_pgno", "native", P4_TRANSIENT);
+        }
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_hmac_salt_mask")==0 ){
+    if(ctx) {
+      if(zRight) {
+        char *deprecation = "PRAGMA cipher_hmac_salt_mask is deprecated, please remove from use";
+        if (sqlite3StrNICmp(zRight ,"x'", 2) == 0 && sqlite3Strlen30(zRight) == 5) {
+          unsigned char mask = 0;
+          const unsigned char *hex = (const unsigned char *)zRight+2;
+          cipher_hex2bin(hex,2,&mask);
+          hmac_salt_mask = mask;
+        }
+        sqlcipher_vdbe_return_string(pParse, "cipher_hmac_salt_mask", deprecation, P4_TRANSIENT);
+        sqlite3_log(SQLITE_WARNING, deprecation);
+      } else {
+        char *mask = sqlite3_mprintf("%02x", hmac_salt_mask);
+        sqlcipher_vdbe_return_string(pParse, "cipher_hmac_salt_mask", mask, P4_DYNAMIC);
+      }
+    }
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_plaintext_header_size")==0 ){
+    if(ctx) {
+      if( zRight ) {
+        int size = atoi(zRight);
+        /* deliberately ignore result code, if size is invalid it will be set to -1
+           and trip the error later in the codec */
+        sqlcipher_codec_ctx_set_plaintext_header_size(ctx, size);
+      } else {
+        char *size = sqlite3_mprintf("%d", ctx->plaintext_header_sz);
+        sqlcipher_vdbe_return_string(pParse, "cipher_plaintext_header_size", size, P4_DYNAMIC);
+      }
+    }
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_default_plaintext_header_size")==0 ){
+    if( zRight ) {
+      default_plaintext_header_size = atoi(zRight);
+    } else {
+      char *size = sqlite3_mprintf("%d", default_plaintext_header_size);
+      sqlcipher_vdbe_return_string(pParse, "cipher_default_plaintext_header_size", size, P4_DYNAMIC);
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_salt")==0 ){
+    if(ctx) {
+      if(zRight) {
+        if (sqlite3StrNICmp(zRight ,"x'", 2) == 0 && sqlite3Strlen30(zRight) == (FILE_HEADER_SZ*2)+3) {
+          unsigned char *salt = (unsigned char*) sqlite3_malloc(FILE_HEADER_SZ);
+          const unsigned char *hex = (const unsigned char *)zRight+2;
+          cipher_hex2bin(hex,FILE_HEADER_SZ*2,salt);
+          sqlcipher_codec_ctx_set_kdf_salt(ctx, salt, FILE_HEADER_SZ);
+          sqlite3_free(salt);
+        }
+      } else {
+        void *salt;
+        char *hexsalt = (char*) sqlite3_malloc((FILE_HEADER_SZ*2)+1);
+        if((rc = sqlcipher_codec_ctx_get_kdf_salt(ctx, &salt)) == SQLITE_OK) {
+          cipher_bin2hex(salt, FILE_HEADER_SZ, hexsalt);
+          sqlcipher_vdbe_return_string(pParse, "cipher_salt", hexsalt, P4_DYNAMIC);
+        } else {
+          sqlite3_free(hexsalt);
+          sqlcipher_codec_ctx_set_error(ctx, rc);
+        }
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_hmac_algorithm")==0 ){
+    if(ctx) {
+      if(zRight) {
+        rc = SQLITE_ERROR;
+        if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA1_LABEL) == 0) {
+          rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1);
+        } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA256_LABEL) == 0) {
+          rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA256);
+        } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA512_LABEL) == 0) {
+          rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA512);
+        }
+        if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+        rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx);
+        if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+      } else {
+        int algorithm = ctx->hmac_algorithm;
+        if(ctx->hmac_algorithm == SQLCIPHER_HMAC_SHA1) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_hmac_algorithm", SQLCIPHER_HMAC_SHA1_LABEL, P4_TRANSIENT);
+        } else if(algorithm == SQLCIPHER_HMAC_SHA256) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_hmac_algorithm", SQLCIPHER_HMAC_SHA256_LABEL, P4_TRANSIENT);
+        } else if(algorithm == SQLCIPHER_HMAC_SHA512) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_hmac_algorithm", SQLCIPHER_HMAC_SHA512_LABEL, P4_TRANSIENT);
+        }
+      }
+    }
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_default_hmac_algorithm")==0 ){
+    if(zRight) {
+      rc = SQLITE_OK;
+      if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA1_LABEL) == 0) {
+        default_hmac_algorithm = SQLCIPHER_HMAC_SHA1;
+      } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA256_LABEL) == 0) {
+        default_hmac_algorithm = SQLCIPHER_HMAC_SHA256;
+      } else if(sqlite3_stricmp(zRight, SQLCIPHER_HMAC_SHA512_LABEL) == 0) {
+        default_hmac_algorithm = SQLCIPHER_HMAC_SHA512;
+      }
+    } else {
+      if(default_hmac_algorithm == SQLCIPHER_HMAC_SHA1) {
+        sqlcipher_vdbe_return_string(pParse, "cipher_default_hmac_algorithm", SQLCIPHER_HMAC_SHA1_LABEL, P4_TRANSIENT);
+      } else if(default_hmac_algorithm == SQLCIPHER_HMAC_SHA256) {
+        sqlcipher_vdbe_return_string(pParse, "cipher_default_hmac_algorithm", SQLCIPHER_HMAC_SHA256_LABEL, P4_TRANSIENT);
+      } else if(default_hmac_algorithm == SQLCIPHER_HMAC_SHA512) {
+        sqlcipher_vdbe_return_string(pParse, "cipher_default_hmac_algorithm", SQLCIPHER_HMAC_SHA512_LABEL, P4_TRANSIENT);
+      }
+    }
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_kdf_algorithm")==0 ){
+    if(ctx) {
+      if(zRight) {
+        rc = SQLITE_ERROR;
+        if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL) == 0) {
+          rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1);
+        } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL) == 0) {
+          rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA256);
+        } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL) == 0) {
+          rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA512);
+        }
+        if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+      } else {
+        if(ctx->kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL, P4_TRANSIENT);
+        } else if(ctx->kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL, P4_TRANSIENT);
+        } else if(ctx->kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) {
+          sqlcipher_vdbe_return_string(pParse, "cipher_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL, P4_TRANSIENT);
+        }
+      }
+    }
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_default_kdf_algorithm")==0 ){
+    if(zRight) {
+      rc = SQLITE_OK;
+      if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL) == 0) {
+        default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA1;
+      } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL) == 0) {
+        default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA256;
+      } else if(sqlite3_stricmp(zRight, SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL) == 0) {
+        default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA512;
+      }
+    } else {
+      if(default_kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) {
+        sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL, P4_TRANSIENT);
+      } else if(default_kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) {
+        sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL, P4_TRANSIENT);
+      } else if(default_kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) {
+        sqlcipher_vdbe_return_string(pParse, "cipher_default_kdf_algorithm", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL, P4_TRANSIENT);
+      }
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_compatibility")==0 ){
+    if(ctx) {
+      if(zRight) {
+        int version = atoi(zRight); 
+
+        switch(version) {
+          case 1: 
+            rc = sqlcipher_codec_ctx_set_pagesize(ctx, 1024);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 4000); 
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 0);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            break;
+
+          case 2: 
+            rc = sqlcipher_codec_ctx_set_pagesize(ctx, 1024);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 4000); 
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            break;
+
+          case 3:
+            rc = sqlcipher_codec_ctx_set_pagesize(ctx, 1024);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 64000); 
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            break;
+
+          default:
+            rc = sqlcipher_codec_ctx_set_pagesize(ctx, 4096);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_hmac_algorithm(ctx, SQLCIPHER_HMAC_SHA512);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_algorithm(ctx, SQLCIPHER_PBKDF2_HMAC_SHA512);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_kdf_iter(ctx, 256000); 
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            rc = sqlcipher_codec_ctx_set_use_hmac(ctx, 1);
+            if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+            break;
+        }  
+
+        rc = codec_set_btree_to_codec_pagesize(db, pDb, ctx);
+        if (rc != SQLITE_OK) sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+      } 
+    }
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_default_compatibility")==0 ){
+    if(zRight) {
+      int version = atoi(zRight); 
+      switch(version) {
+        case 1: 
+          default_page_size = 1024;
+          default_hmac_algorithm = SQLCIPHER_HMAC_SHA1;
+          default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA1;
+          default_kdf_iter = 4000;
+          sqlcipher_set_default_use_hmac(0);
+          break;
+
+        case 2: 
+          default_page_size = 1024;
+          default_hmac_algorithm = SQLCIPHER_HMAC_SHA1;
+          default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA1;
+          default_kdf_iter = 4000;
+          sqlcipher_set_default_use_hmac(1);
+          break;
+
+        case 3:
+          default_page_size = 1024;
+          default_hmac_algorithm = SQLCIPHER_HMAC_SHA1;
+          default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA1;
+          default_kdf_iter = 64000;
+          sqlcipher_set_default_use_hmac(1);
+          break;
+
+        default:
+          default_page_size = 4096;
+          default_hmac_algorithm = SQLCIPHER_HMAC_SHA512;
+          default_kdf_algorithm = SQLCIPHER_PBKDF2_HMAC_SHA512;
+          default_kdf_iter = 256000;
+          sqlcipher_set_default_use_hmac(1);
+          break;
+      }  
+    } 
+  }else 
+  if( sqlite3_stricmp(zLeft,"cipher_memory_security")==0 ){
+    if( zRight ) {
+      if(sqlite3GetBoolean(zRight,1)) {
+        /* memory security can only be enabled, not disabled */
+        sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_set_mem_security: on");
+        sqlcipher_mem_security_on = 1;
+      }
+    } else {
+      /* only report that memory security is enabled if pragma cipher_memory_security is ON and
+         SQLCipher's allocator/deallocator was run at least one time */
+      int state = sqlcipher_mem_security_on && sqlcipher_mem_executed;
+      char *on = sqlite3_mprintf("%d", state);
+      sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE,
+        "sqlcipher_get_mem_security: sqlcipher_mem_security_on = %d, sqlcipher_mem_executed = %d", 
+        sqlcipher_mem_security_on, sqlcipher_mem_executed);
+      sqlcipher_vdbe_return_string(pParse, "cipher_memory_security", on, P4_DYNAMIC);
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_settings")==0 ){
+    if(ctx) {
+      int algorithm;
+      char *pragma;
+
+      pragma = sqlite3_mprintf("PRAGMA kdf_iter = %d;", ctx->kdf_iter);
+      sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+      pragma = sqlite3_mprintf("PRAGMA cipher_page_size = %d;", ctx->page_sz);
+      sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+      pragma = sqlite3_mprintf("PRAGMA cipher_use_hmac = %d;", SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC));
+      sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+      pragma = sqlite3_mprintf("PRAGMA cipher_plaintext_header_size = %d;", ctx->plaintext_header_sz);
+      sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+      algorithm = ctx->hmac_algorithm;
+      pragma = NULL;
+      if(algorithm == SQLCIPHER_HMAC_SHA1) {
+        pragma = sqlite3_mprintf("PRAGMA cipher_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA1_LABEL);
+      } else if(algorithm == SQLCIPHER_HMAC_SHA256) {
+        pragma = sqlite3_mprintf("PRAGMA cipher_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA256_LABEL);
+      } else if(algorithm == SQLCIPHER_HMAC_SHA512) {
+        pragma = sqlite3_mprintf("PRAGMA cipher_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA512_LABEL);
+      }
+      sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+      algorithm = ctx->kdf_algorithm;
+      pragma = NULL;
+      if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) {
+        pragma = sqlite3_mprintf("PRAGMA cipher_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL);
+      } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) {
+        pragma = sqlite3_mprintf("PRAGMA cipher_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL);
+      } else if(algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) {
+        pragma = sqlite3_mprintf("PRAGMA cipher_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL);
+      }
+      sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+    }
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_default_settings")==0 ){
+    char *pragma;
+
+    pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_iter = %d;", default_kdf_iter);
+    sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+    pragma = sqlite3_mprintf("PRAGMA cipher_default_page_size = %d;", default_page_size);
+    sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+    pragma = sqlite3_mprintf("PRAGMA cipher_default_use_hmac = %d;", SQLCIPHER_FLAG_GET(default_flags, CIPHER_FLAG_HMAC)); 
+    sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+    pragma = sqlite3_mprintf("PRAGMA cipher_default_plaintext_header_size = %d;", default_plaintext_header_size);
+    sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+    pragma = NULL;
+    if(default_hmac_algorithm == SQLCIPHER_HMAC_SHA1) {
+      pragma = sqlite3_mprintf("PRAGMA cipher_default_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA1_LABEL);
+    } else if(default_hmac_algorithm == SQLCIPHER_HMAC_SHA256) {
+      pragma = sqlite3_mprintf("PRAGMA cipher_default_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA256_LABEL);
+    } else if(default_hmac_algorithm == SQLCIPHER_HMAC_SHA512) {
+      pragma = sqlite3_mprintf("PRAGMA cipher_default_hmac_algorithm = %s;", SQLCIPHER_HMAC_SHA512_LABEL);
+    }
+    sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+
+    pragma = NULL;
+    if(default_kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA1) {
+      pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA1_LABEL);
+    } else if(default_kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA256) {
+      pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA256_LABEL);
+    } else if(default_kdf_algorithm == SQLCIPHER_PBKDF2_HMAC_SHA512) {
+      pragma = sqlite3_mprintf("PRAGMA cipher_default_kdf_algorithm = %s;", SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL);
+    }
+    sqlcipher_vdbe_return_string(pParse, "pragma", pragma, P4_DYNAMIC);
+  }else
+  if( sqlite3_stricmp(zLeft,"cipher_integrity_check")==0 ){
+    if(ctx) {
+      sqlcipher_codec_ctx_integrity_check(ctx, pParse, "cipher_integrity_check");
+    }
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_log_level")==0 ){
+    if(zRight) {
+      sqlcipher_log_level = SQLCIPHER_LOG_NONE;
+      if(sqlite3_stricmp(zRight,      "ERROR")==0) sqlcipher_log_level = SQLCIPHER_LOG_ERROR;
+      else if(sqlite3_stricmp(zRight, "WARN" )==0) sqlcipher_log_level = SQLCIPHER_LOG_WARN;
+      else if(sqlite3_stricmp(zRight, "INFO" )==0) sqlcipher_log_level = SQLCIPHER_LOG_INFO;
+      else if(sqlite3_stricmp(zRight, "DEBUG")==0) sqlcipher_log_level = SQLCIPHER_LOG_DEBUG;
+      else if(sqlite3_stricmp(zRight, "TRACE")==0) sqlcipher_log_level = SQLCIPHER_LOG_TRACE;
+    }
+    sqlcipher_vdbe_return_string(pParse, "cipher_log_level", sqlcipher_get_log_level_str(sqlcipher_log_level), P4_TRANSIENT);
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_log_source")==0 ){
+    if(zRight) {
+      if(sqlite3_stricmp(zRight,      "NONE"    )==0) sqlcipher_log_source = SQLCIPHER_LOG_NONE;
+      else if(sqlite3_stricmp(zRight, "ANY"     )==0) sqlcipher_log_source = SQLCIPHER_LOG_ANY;
+      else {
+        if(sqlite3_stricmp(zRight,      "CORE"    )==0) SQLCIPHER_FLAG_SET(sqlcipher_log_source, SQLCIPHER_LOG_CORE);
+        else if(sqlite3_stricmp(zRight, "MEMORY"  )==0) SQLCIPHER_FLAG_SET(sqlcipher_log_source, SQLCIPHER_LOG_MEMORY);
+        else if(sqlite3_stricmp(zRight, "MUTEX"   )==0) SQLCIPHER_FLAG_SET(sqlcipher_log_source, SQLCIPHER_LOG_MUTEX);
+        else if(sqlite3_stricmp(zRight, "PROVIDER")==0) SQLCIPHER_FLAG_SET(sqlcipher_log_source, SQLCIPHER_LOG_PROVIDER);
+      }
+    }
+    sqlcipher_vdbe_return_string(pParse, "cipher_log_source", sqlcipher_get_log_sources_str(sqlcipher_log_source), P4_DYNAMIC);
+  } else
+  if( sqlite3_stricmp(zLeft, "cipher_log")== 0 && zRight ){
+      char *status = sqlite3_mprintf("%d", sqlcipher_set_log(zRight));
+      sqlcipher_vdbe_return_string(pParse, "cipher_log", status, P4_DYNAMIC);
+  }else {
+    return 0;
+  }
+  return 1;
+}
+
+/* these constants are used internally within SQLite's pager.c to differentiate between
+   operations on the main database or journal pages. This is important in the context
+   of a rekey operations, where the journal must be written using the original key 
+   material (to allow a transactional rollback), while the new database pages are being
+   written with the new key material*/
+#define CODEC_READ_OP 3
+#define CODEC_WRITE_OP 6
+#define CODEC_JOURNAL_OP 7
+
+/*
+ * sqlite3Codec can be called in multiple modes.
+ * encrypt mode - expected to return a pointer to the 
+ *   encrypted data without altering pData.
+ * decrypt mode - expected to return a pointer to pData, with
+ *   the data decrypted in the input buffer
+ */
+static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) {
+  codec_ctx *ctx = (codec_ctx *) iCtx;
+  int offset = 0, rc = 0;
+  unsigned char *pData = (unsigned char *) data;
+  int cctx = CIPHER_READ_CTX;
+  void *out = NULL;
+  sqlite3_mutex *mutex = ctx->pBt->sharable ? sqlcipher_mutex(SQLCIPHER_MUTEX_SHAREDCACHE) : NULL;
+
+  /* in shared cache mode, this needs to be mutexed to prevent a separate database handle from
+   * nuking the context on the shared Btree */
+  if(mutex) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering mutex %p", __func__, mutex);
+    sqlite3_mutex_enter(mutex);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered mutex %p", __func__, mutex);
+  }
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3Codec: pgno=%d, mode=%d, ctx->page_sz=%d", pgno, mode, ctx->page_sz);
+
+  if(ctx->error != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: identified deferred error condition: %d", __func__, rc);
+    sqlcipher_codec_ctx_set_error(ctx, ctx->error);
+    goto cleanup;
+  }
+
+  /* call to derive keys if not present yet */
+  if((rc = sqlcipher_codec_key_derive(ctx)) != SQLITE_OK) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error occurred during key derivation: %d", rc);
+    sqlcipher_codec_ctx_set_error(ctx, rc);
+    goto cleanup;
+  }
+
+  /* if the plaintext_header_size is negative that means an invalid size was set via 
+     PRAGMA. We can't set the error state on the pager at that point because the pager
+     may not be open yet. However, this is a fatal error state, so abort the codec */
+  if(ctx->plaintext_header_sz < 0) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error invalid ctx->plaintext_header_sz: %d", ctx->plaintext_header_sz);
+    sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR);
+    goto cleanup;
+  }
+
+  if(pgno == 1) /* adjust starting pointers in data page for header offset on first page*/   
+    offset = ctx->plaintext_header_sz ? ctx->plaintext_header_sz : FILE_HEADER_SZ; 
+  
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3Codec: switch mode=%d offset=%d",  mode, offset);
+  switch(mode) {
+    case CODEC_READ_OP: /* decrypt */
+      if(pgno == 1) /* copy initial part of file header or SQLite magic to buffer */ 
+        memcpy(ctx->buffer, ctx->plaintext_header_sz ? pData : (void *) SQLITE_FILE_HEADER, offset); 
+
+      rc = sqlcipher_page_cipher(ctx, cctx, pgno, SQLCIPHER_DECRYPT, ctx->page_sz - offset, pData + offset, (unsigned char*)ctx->buffer + offset);
+#ifdef SQLCIPHER_TEST
+      if((cipher_test_flags & TEST_FAIL_DECRYPT) > 0 && sqlcipher_get_test_fail()) {
+        rc = SQLITE_ERROR;
+        sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlite3Codec: simulating decryption failure for pgno=%d, mode=%d, ctx->page_sz=%d", pgno, mode, ctx->page_sz);
+      }
+#endif
+      if(rc != SQLITE_OK) {
+        /* failure to decrypt a page is considered a permanent error and will render the pager unusable
+         * in order to prevent inconsistent data being loaded into page cache. The only exception here is when a database is being "recovered",
+         * which we consider to be the case if the plaintext header size is set to the full non-reserved size of a page. If that is the case we consider
+         * this to be operating in recovery mode, and will log the error but not permanently put the codec into an error state */
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error decrypting page %d data: %d", pgno, rc);
+        sqlcipher_memset((unsigned char*) ctx->buffer+offset, 0, ctx->page_sz-offset);
+        if(ctx->plaintext_header_sz == ctx->page_sz - ctx->reserve_sz) {
+          sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: plaintext header size of %d indicates recovery mode, suppressing permanent error", __func__, ctx->plaintext_header_sz);
+        } else {
+          sqlcipher_codec_ctx_set_error(ctx, rc);
+        }
+      } else {
+        SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_KEY_USED);
+      }
+      memcpy(pData, ctx->buffer, ctx->page_sz); /* copy buffer data back to pData and return */
+      out = pData;
+      goto cleanup;
+      break;
+
+    case CODEC_WRITE_OP: /* encrypt database page, operate on write context and fall through to case 7, so the write context is used*/
+      cctx = CIPHER_WRITE_CTX; 
+
+    case CODEC_JOURNAL_OP: /* encrypt journal page, operate on read context use to get the original page data from the database */ 
+      if(pgno == 1) { /* copy initial part of file header or salt to buffer */ 
+        void *kdf_salt = NULL; 
+        /* retrieve the kdf salt */
+        if((rc = sqlcipher_codec_ctx_get_kdf_salt(ctx, &kdf_salt)) != SQLITE_OK) {
+          sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error retrieving salt: %d", rc);
+          sqlcipher_codec_ctx_set_error(ctx, rc); 
+          goto cleanup;
+        }
+        memcpy(ctx->buffer, ctx->plaintext_header_sz ? pData : kdf_salt, offset);
+      }
+      rc = sqlcipher_page_cipher(ctx, cctx, pgno, SQLCIPHER_ENCRYPT, ctx->page_sz - offset, pData + offset, (unsigned char*)ctx->buffer + offset);
+#ifdef SQLCIPHER_TEST
+      if((cipher_test_flags & TEST_FAIL_ENCRYPT) > 0 && sqlcipher_get_test_fail()) {
+        rc = SQLITE_ERROR;
+        sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlite3Codec: simulating encryption failure for pgno=%d, mode=%d, ctx->page_sz=%d", pgno, mode, ctx->page_sz);
+      }
+#endif
+      if(rc != SQLITE_OK) {
+        /* failure to encrypt a page is considered a permanent error and will render the pager unusable
+           in order to prevent corrupted pages from being written to the main databased when using WAL */
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error encrypting page %d data: %d", pgno, rc);
+        sqlcipher_memset((unsigned char*)ctx->buffer+offset, 0, ctx->page_sz-offset);
+        sqlcipher_codec_ctx_set_error(ctx, rc);
+        goto cleanup;
+      }
+      SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_KEY_USED);
+      out = ctx->buffer; /* return persistent buffer data, pData remains intact */
+      goto cleanup;
+      break;
+
+    default:
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error unsupported codec mode %d", mode);
+      sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); /* unsupported mode, set error */
+      out = pData;
+      goto cleanup;
+      break;
+  }
+
+cleanup:
+  if(mutex) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving mutex %p", __func__, mutex);
+    sqlite3_mutex_leave(mutex);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left mutex %p", __func__, mutex);
+  }
+  return out;
+}
+
+/* This callback will be invoked when a database connection is closed. It is basically a light wrapper
+ * ariund sqlciher_codec_ctx_free that locks the shared cache mutex if necessary */
+static void sqlite3FreeCodecArg(void *pCodecArg) {
+  codec_ctx *ctx = (codec_ctx *) pCodecArg;
+  sqlite3_mutex *mutex = ctx->pBt->sharable ? sqlcipher_mutex(SQLCIPHER_MUTEX_SHAREDCACHE) : NULL;
+
+  if(pCodecArg == NULL) return;
+
+  /* in shared cache mode, this needs to be mutexed to prevent a codec context from being deallocated when
+   * it is in use by the codec due to cross-database handle access to the shared Btree */
+  if(mutex) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering mutex %p", __func__, mutex);
+    sqlite3_mutex_enter(mutex);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered mutex %p", __func__, mutex);
+  }
+
+  sqlcipher_codec_ctx_free(&ctx); /* wipe and free allocated memory for the context */
+
+  if(mutex) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving mutex %p", __func__, mutex);
+    sqlite3_mutex_leave(mutex);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left mutex %p", __func__, mutex);
+  }
+}
+
+int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) {
+  struct Db *pDb = NULL;
+  sqlite3_file *fd = NULL;
+  codec_ctx *ctx = NULL;
+  Pager *pPager = NULL;
+  int rc = SQLITE_OK;
+  sqlite3_mutex *extra_mutex = NULL;
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: db=%p, nDb=%d", __func__, db, nDb);
+
+  if(!sqlcipher_init) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: sqlcipher not initialized %d", __func__, sqlcipher_init_error);
+    return sqlcipher_init_error;
+  }
+
+  /* error pKey is not null and nKey is > 0 */
+  if(!(nKey && zKey)) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: no key", __func__);
+    return SQLITE_MISUSE;
+  }
+
+  if(!(db && nDb >= 0 && nDb < db->nDb && (pDb = &db->aDb[nDb]))) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: invalid database %p %d", __func__, db, nDb);
+    return SQLITE_MISUSE;
+  }
+
+  /* After this point, early returns for API misuse are complete, lock on a mutex and ensure it is cleaned
+   * up later. If shared cache is enabled then enter a specially defined "global" recursive mutex specifically
+   * for isolating shared cache connections, otherwise use the built-in databse mutex */ 
+  extra_mutex = pDb->pBt->sharable ? sqlcipher_mutex(SQLCIPHER_MUTEX_SHAREDCACHE) : NULL;
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering database mutex %p", __func__, db->mutex);
+  sqlite3_mutex_enter(db->mutex);
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered database mutex %p", __func__, db->mutex);
+
+  if(extra_mutex) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering mutex %p", __func__, extra_mutex);
+    sqlite3_mutex_enter(extra_mutex);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered mutex %p", __func__, extra_mutex);
+  }
+
+  pPager = sqlite3BtreePager(pDb->pBt);
+  ctx = (codec_ctx*) sqlcipherPagerGetCodec(pPager);
+
+  if(ctx != NULL) {
+    /* There is already a codec attached to this database */
+    if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) {
+       /* The key was derived and used successfully, so return early */
+      sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: disregarding attempt to set key on an previously keyed database connection handle", __func__);
+      goto cleanup;
+#ifndef SQLITE_DEBUG
+    } else if (pDb->pBt->sharable) {
+      /* This Btree is participating in shared cache. It would be usafe to reset and reattach a new codec, so return early.
+       *
+       * When compiled with SQLITE_DEBUG, all database connections have shared cached enabled. This behavior of disallowing reset
+       * of the codec on a shared cache connection will break several tests that depend on the the ability to reset the codec,
+       * like migration tests, repeat-keying tests, etc. Asa result we will disable shared cache handling when compiled with
+       * SQLIE_DEBUG enabled.*/
+      sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: disregarding attempt to set key on an shared cache handle", __func__);
+      goto cleanup;
+#endif
+    } else {
+      /* To preseve legacy functionality where an incorrect key could be replaced by a correct key without closing the database,
+       * if the key has not been used, and shared cache is not enabled, reset the codec on this pager entirely.
+       * This will call sqlcipher_codec_ctx_free directly instead of through sqlite3FreeCodecArg because this function already
+       * holds the shared cache mutex if it is necessary, and that avoids requiring a more expensive recursive mutex */
+      sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: resetting existing codec on pager", __func__);
+      sqlcipher_codec_ctx_free(&ctx);
+      sqlcipherPagerSetCodec(pPager, NULL, NULL, NULL, NULL);
+      ctx = NULL;
+    }
+  }
+
+  /* check if the sqlite3_file is open, and if not force handle to NULL */
+  if((fd = sqlite3PagerFile(pPager))->pMethods == 0) fd = NULL;
+
+  /* point the internal codec argument against the contet to be prepared */
+  rc = sqlcipher_codec_ctx_init(&ctx, pDb, pPager, zKey, nKey);
+
+  if(rc != SQLITE_OK) {
+    /* initialization failed, do not attach potentially corrupted context */
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: context initialization failed, forcing error state with rc=%d", __func__, rc);
+    /* force an error at the pager level, such that even the upstream caller ignores the return code
+       the pager will be in an error state and will process no further operations */
+    sqlite3pager_error(pPager, rc);
+    pDb->pBt->pBt->db->errCode = rc;
+    goto cleanup;
+  }
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipherPagerSetCodec()", __func__);
+  sqlcipherPagerSetCodec(pPager, sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx);
+
+  codec_set_btree_to_codec_pagesize(db, pDb, ctx);
+
+  /* force secure delete. This has the benefit of wiping internal data when deleted
+     and also ensures that all pages are written to disk (i.e. not skipped by
+     sqlite3PagerDontWrite optimizations) */
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlite3BtreeSecureDelete()", __func__);
+  sqlite3BtreeSecureDelete(pDb->pBt, 1);
+
+  /* if fd is null, then this is an in-memory database and
+     we dont' want to overwrite the AutoVacuum settings
+     if not null, then set to the default */
+  if(fd != NULL) {
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlite3BtreeSetAutoVacuum()", __func__);
+    sqlite3BtreeSetAutoVacuum(pDb->pBt, SQLITE_DEFAULT_AUTOVACUUM);
+  }
+
+cleanup:
+
+  if(extra_mutex) {
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving mutex %p", __func__, extra_mutex);
+    sqlite3_mutex_leave(extra_mutex);
+    sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left mutex %p", __func__, extra_mutex);
+  }
+
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving database mutex %p", __func__, db->mutex);
+  sqlite3_mutex_leave(db->mutex);
+  sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left database mutex %p", __func__, db->mutex);
+
+  return rc;
+}
+
+/* search for the index of the named database by comparing db names. main is
+ * always 0, temp 1, and other attached databses follow. If the name is
+ * NULL or empty the main database will be used consistent with sqlite defaults. If
+ * sqlite3 handle is NULL or the database can't be found by name, return -1 indicating
+ * an invalid database */
+int sqlcipher_find_db_index(sqlite3 *db, const char *zDb) {
+  int db_index;
+
+  if(!db) return -1;
+  if(!zDb || sqlite3_stricmp(zDb,"")==0) return 0;
+
+  for(db_index = 0; db_index < db->nDb; db_index++) {
+    struct Db *pDb = &db->aDb[db_index];
+    if(sqlite3_stricmp(pDb->zDbSName, zDb) == 0) {
+      return db_index;
+    }
+  }
+  return -1;
+}
+
+void sqlite3_activate_see(const char* in) {
+  /* do nothing, security enhancements are always active */
+}
+
+int sqlite3_key(sqlite3 *db, const void *pKey, int nKey) {
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: db=%p", __func__, db);
+  return sqlite3_key_v2(db, "main", pKey, nKey);
+}
+
+int sqlite3_key_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) {
+  int db_index = sqlcipher_find_db_index(db, zDb);
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: db=%p zDb=%s db_index=%d", __func__, db, zDb, db_index);
+  return sqlcipherCodecAttach(db, db_index, pKey, nKey);
+}
+
+int sqlite3_rekey(sqlite3 *db, const void *pKey, int nKey) {
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3_rekey: db=%p", db);
+  return sqlite3_rekey_v2(db, "main", pKey, nKey);
+}
+
+/* sqlite3_rekey_v2
+** Given a database, this will reencrypt the database using a new key.
+** There is only one possible modes of operation - to encrypt a database
+** that is already encrpyted. If the database is not already encrypted
+** this should do nothing
+** The proposed logic for this function follows:
+** 1. Determine if the database is already encryptped
+** 2. If there is NOT already a key present do nothing
+** 3. If there is a key present, re-encrypt the database with the new key
+*/
+int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) {
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: db=%p zDb=%s", db, zDb);
+
+  if(!sqlcipher_init) {
+    sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: sqlcipher not initialized %d",__func__, sqlcipher_init_error);
+    return sqlcipher_init_error;
+  }
+
+  if(db && pKey && nKey) {
+    int db_index = sqlcipher_find_db_index(db, zDb);
+    struct Db *pDb = NULL;
+
+    if(!(db_index >= 0 && db_index < db->nDb)) {
+      sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: invalid database zDb=%p", __func__, zDb);
+      return SQLITE_MISUSE;
+    }
+
+    pDb = &db->aDb[db_index];
+    sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: database zDb=%p db_index:%d", zDb, db_index);
+
+    if(pDb->pBt) {
+      codec_ctx *ctx;
+      int rc, page_count;
+      Pgno pgno;
+      PgHdr *page;
+      Pager *pPager = sqlite3BtreePager(pDb->pBt);
+
+      ctx = (codec_ctx*) sqlcipherPagerGetCodec(pPager);
+     
+      if(ctx == NULL) { 
+        /* there was no codec attached to this database, so this should do nothing! */ 
+        sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: no codec attached to db %s: rekey can't be used on an unencrypted database", zDb);
+        return SQLITE_MISUSE;
+      }
+
+      sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlite3_rekey_v2: entering database mutex %p", db->mutex);
+      sqlite3_mutex_enter(db->mutex);
+      sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlite3_rekey_v2: entered database mutex %p", db->mutex);
+
+      codec_set_pass_key(db, db_index, pKey, nKey, CIPHER_WRITE_CTX);
+    
+      /* do stuff here to rewrite the database 
+      ** 1. Create a transaction on the database
+      ** 2. Iterate through each page, reading it and then writing it.
+      ** 3. If that goes ok then commit and put ctx->rekey into ctx->key
+      **    note: don't deallocate rekey since it may be used in a subsequent iteration 
+      */
+      rc = sqlite3BtreeBeginTrans(pDb->pBt, 1, 0); /* begin write transaction */
+      sqlite3PagerPagecount(pPager, &page_count);
+      for(pgno = 1; rc == SQLITE_OK && pgno <= (unsigned int)page_count; pgno++) { /* pgno's start at 1 see pager.c:pagerAcquire */
+        if(!sqlite3pager_is_sj_pgno(pPager, pgno)) { /* skip this page (see pager.c:pagerAcquire for reasoning) */
+          rc = sqlite3PagerGet(pPager, pgno, &page, 0);
+          if(rc == SQLITE_OK) { /* write page see pager_incr_changecounter for example */
+            rc = sqlite3PagerWrite(page);
+            if(rc == SQLITE_OK) {
+              sqlite3PagerUnref(page);
+            } else {
+             sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: error %d occurred writing page %d", rc, pgno);  
+            }
+          } else {
+             sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: error %d occurred reading page %d", rc, pgno);  
+          }
+        } 
+      }
+
+      /* if commit was successful commit and copy the rekey data to current key, else rollback to release locks */
+      if(rc == SQLITE_OK) { 
+        sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: committing");
+        rc = sqlite3BtreeCommit(pDb->pBt); 
+        sqlcipher_codec_key_copy(ctx, CIPHER_WRITE_CTX);
+      } else {
+        sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: rollback");
+        sqlite3BtreeRollback(pDb->pBt, SQLITE_ABORT_ROLLBACK, 0);
+      }
+
+      sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlite3_rekey_v2: leaving database mutex %p", db->mutex);
+      sqlite3_mutex_leave(db->mutex);
+      sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlite3_rekey_v2: left database mutex %p", db->mutex);
+    }
+    return SQLITE_OK;
+  }
+  sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: no key provided for db %s: rekey can't be used to decrypt an encrypted database", zDb);
+  return SQLITE_MISUSE;
+}
+
+/*
+ * Retrieves the current key attached to the database if there is a codec attached to it.
+ * The key will be passed back using internally allocated memory and must be freed using
+ * sqlcipher_free to avoid memory leaks. If no key is present, zKey will be set to NULL
+ * and nKey to 0.
+ *
+ * If the encryption key has not yet been derived or the key material is stored, it will
+ * be passed back directly. Otherwise, a "keyspec" consisting of the raw key and salt
+ * will be used instead. */
+void sqlcipherCodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) {
+  struct Db *pDb = NULL;
+
+  sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipherCodecGetKey:db=%p, nDb=%d", db, nDb);
+
+  *zKey = NULL;
+  *nKey = 0;
+
+  if(!(db && nDb >= 0 && nDb < db->nDb)) return; /* invalid database */
+
+  pDb = &db->aDb[nDb];
+
+  if( pDb->pBt ) {
+    codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt));
+    
+    if(ctx) {
+      /* if the key has not been derived yet, or the key is stored (vi PRAGMA cipher_store_pass)
+       * then return the key material. Other wise pass back the keyspec */
+      if(ctx->read_ctx->derive_key || ctx->store_pass == 1) {
+        *zKey = sqlcipher_malloc(ctx->read_ctx->pass_sz);
+        *nKey = ctx->read_ctx->pass_sz;
+        memcpy(*zKey, ctx->read_ctx->pass, ctx->read_ctx->pass_sz);
+      } else {
+        sqlcipher_cipher_ctx_get_keyspec(ctx, ctx->read_ctx, (char**) zKey, nKey);
+      }
+    }
+  }
+}
+
+/*
+ * Implementation of an "export" function that allows a caller
+ * to duplicate the main database to an attached database. This is intended
+ * as a conveneince for users who need to:
+ * 
+ *   1. migrate from an non-encrypted database to an encrypted database
+ *   2. move from an encrypted database to a non-encrypted database
+ *   3. convert beween the various flavors of encrypted databases.  
+ *
+ * This implementation is based heavily on the procedure and code used
+ * in vacuum.c, but is exposed as a function that allows export to any
+ * named attached database.
+ */
+
+/*
+** Finalize a prepared statement.  If there was an error, store the
+** text of the error message in *pzErrMsg.  Return the result code.
+** 
+** Based on vacuumFinalize from vacuum.c
+*/
+static int sqlcipher_finalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){
+  int rc;
+  rc = sqlite3VdbeFinalize((Vdbe*)pStmt);
+  if( rc ){
+    sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db));
+  }
+  return rc;
+}
+
+/*
+** Execute zSql on database db. Return an error code.
+** 
+** Based on execSql from vacuum.c
+*/
+static int sqlcipher_execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
+  sqlite3_stmt *pStmt;
+  VVA_ONLY( int rc; )
+  if( !zSql ){
+    return SQLITE_NOMEM;
+  }
+  if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){
+    sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db));
+    return sqlite3_errcode(db);
+  }
+  VVA_ONLY( rc = ) sqlite3_step(pStmt);
+  assert( rc!=SQLITE_ROW );
+  return sqlcipher_finalize(db, pStmt, pzErrMsg);
+}
+
+/*
+** Execute zSql on database db. The statement returns exactly
+** one column. Execute this as SQL on the same database.
+** 
+** Based on execExecSql from vacuum.c
+*/
+static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
+  sqlite3_stmt *pStmt;
+  int rc;
+
+  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ) return rc;
+
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    rc = sqlcipher_execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0));
+    if( rc!=SQLITE_OK ){
+      sqlcipher_finalize(db, pStmt, pzErrMsg);
+      return rc;
+    }
+  }
+
+  return sqlcipher_finalize(db, pStmt, pzErrMsg);
+}
+
+/*
+ * copy database and schema from the main database to an attached database
+ * 
+ * Based on sqlite3RunVacuum from vacuum.c
+*/
+static void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) {
+  sqlite3 *db = sqlite3_context_db_handle(context);
+  const char* targetDb, *sourceDb; 
+  int targetDb_idx = 0;
+  u64 saved_flags = db->flags;        /* Saved value of the db->flags */
+  u32 saved_mDbFlags = db->mDbFlags;        /* Saved value of the db->mDbFlags */
+  int saved_nChange = db->nChange;      /* Saved value of db->nChange */
+  int saved_nTotalChange = db->nTotalChange; /* Saved value of db->nTotalChange */
+  u8 saved_mTrace = db->mTrace;        /* Saved value of db->mTrace */
+  int rc = SQLITE_OK;     /* Return code from service routines */
+  char *zSql = NULL;         /* SQL statements */
+  char *pzErrMsg = NULL;
+
+  if(argc != 1 && argc != 2) {
+    rc = SQLITE_ERROR;
+    pzErrMsg = sqlite3_mprintf("invalid number of arguments (%d) passed to sqlcipher_export", argc);
+    goto end_of_export;
+  }
+
+  if(sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+    rc = SQLITE_ERROR;
+    pzErrMsg = sqlite3_mprintf("target database can't be NULL");
+    goto end_of_export;
+  }
+
+  targetDb = (const char*) sqlite3_value_text(argv[0]); 
+  sourceDb = "main";
+
+  if(argc == 2) {
+    if(sqlite3_value_type(argv[1]) == SQLITE_NULL) {
+      rc = SQLITE_ERROR;
+      pzErrMsg = sqlite3_mprintf("target database can't be NULL");
+      goto end_of_export;
+    }
+    sourceDb = (char *) sqlite3_value_text(argv[1]);
+  }
+
+
+  /* if the target database is not valid, do not proceed. */
+  targetDb_idx =  sqlcipher_find_db_index(db, targetDb);
+  if(targetDb_idx < 0) {
+    rc = SQLITE_ERROR;
+    pzErrMsg = sqlite3_mprintf("unknown database %s", targetDb);
+    goto end_of_export;
+  }
+  db->init.iDb = targetDb_idx;
+
+  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; 
+  db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
+  db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); 
+  db->mTrace = 0;
+
+  /* Query the schema of the main database. Create a mirror schema
+  ** in the temporary database.
+  */
+  zSql = sqlite3_mprintf(
+    "SELECT sql "
+    "  FROM %s.sqlite_schema WHERE type='table' AND name!='sqlite_sequence'"
+    "   AND rootpage>0"
+  , sourceDb);
+  rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); 
+  if( rc!=SQLITE_OK ) goto end_of_export;
+  sqlite3_free(zSql);
+
+  zSql = sqlite3_mprintf(
+    "SELECT sql "
+    "  FROM %s.sqlite_schema WHERE sql LIKE 'CREATE INDEX %%' "
+  , sourceDb);
+  rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); 
+  if( rc!=SQLITE_OK ) goto end_of_export;
+  sqlite3_free(zSql);
+
+  zSql = sqlite3_mprintf(
+    "SELECT sql "
+    "  FROM %s.sqlite_schema WHERE sql LIKE 'CREATE UNIQUE INDEX %%'"
+  , sourceDb);
+  rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); 
+  if( rc!=SQLITE_OK ) goto end_of_export;
+  sqlite3_free(zSql);
+
+  /* Loop through the tables in the main database. For each, do
+  ** an "INSERT INTO rekey_db.xxx SELECT * FROM main.xxx;" to copy
+  ** the contents to the temporary database.
+  */
+  zSql = sqlite3_mprintf(
+    "SELECT 'INSERT INTO %s.' || quote(name) "
+    "|| ' SELECT * FROM %s.' || quote(name) || ';'"
+    "FROM %s.sqlite_schema "
+    "WHERE type = 'table' AND name!='sqlite_sequence' "
+    "  AND rootpage>0"
+  , targetDb, sourceDb, sourceDb);
+  rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); 
+  if( rc!=SQLITE_OK ) goto end_of_export;
+  sqlite3_free(zSql);
+
+  /* Copy over the contents of the sequence table
+  */
+  zSql = sqlite3_mprintf(
+    "SELECT 'INSERT INTO %s.' || quote(name) "
+    "|| ' SELECT * FROM %s.' || quote(name) || ';' "
+    "FROM %s.sqlite_schema WHERE name=='sqlite_sequence';"
+  , targetDb, sourceDb, targetDb);
+  rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execExecSql(db, &pzErrMsg, zSql); 
+  if( rc!=SQLITE_OK ) goto end_of_export;
+  sqlite3_free(zSql);
+
+  /* Copy the triggers, views, and virtual tables from the main database
+  ** over to the temporary database.  None of these objects has any
+  ** associated storage, so all we have to do is copy their entries
+  ** from the SQLITE_MASTER table.
+  */
+  zSql = sqlite3_mprintf(
+    "INSERT INTO %s.sqlite_schema "
+    "  SELECT type, name, tbl_name, rootpage, sql"
+    "    FROM %s.sqlite_schema"
+    "   WHERE type='view' OR type='trigger'"
+    "      OR (type='table' AND rootpage=0)"
+  , targetDb, sourceDb);
+  rc = (zSql == NULL) ? SQLITE_NOMEM : sqlcipher_execSql(db, &pzErrMsg, zSql); 
+  if( rc!=SQLITE_OK ) goto end_of_export;
+  sqlite3_free(zSql);
+
+  zSql = NULL;
+end_of_export:
+  db->init.iDb = 0;
+  db->flags = saved_flags;
+  db->mDbFlags = saved_mDbFlags;
+  db->nChange = saved_nChange;
+  db->nTotalChange = saved_nTotalChange;
+  db->mTrace = saved_mTrace;
+
+  if(zSql) sqlite3_free(zSql);
+
+  if(rc) {
+    if(pzErrMsg != NULL) {
+      sqlite3_result_error(context, pzErrMsg, -1);
+      sqlite3DbFree(db, pzErrMsg);
+    } else {
+      sqlite3_result_error(context, sqlite3ErrStr(rc), -1);
+    }
+  }
+}
+#endif
+/* END SQLCIPHER */
diff --git a/src/sqlcipher.h b/src/sqlcipher.h
index 545a05d7fc..de25d0b1fe 100644
--- a/src/sqlcipher.h
+++ b/src/sqlcipher.h
@@ -36,6 +36,10 @@
 #define SQLCIPHER_H
 
 #include "sqlite3.h"
+#include "sqliteInt.h"
+
+#define SQLCIPHER_DECRYPT 0
+#define SQLCIPHER_ENCRYPT 1
 
 #define SQLCIPHER_HMAC_SHA1 0
 #define SQLCIPHER_HMAC_SHA1_LABEL "HMAC_SHA1"
@@ -52,16 +56,28 @@
 #define SQLCIPHER_PBKDF2_HMAC_SHA512 2
 #define SQLCIPHER_PBKDF2_HMAC_SHA512_LABEL "PBKDF2_HMAC_SHA512"
 
-
-typedef struct {
-  int (*activate)(void *ctx);
-  int (*deactivate)(void *ctx);
+typedef struct sqlcipher_provider sqlcipher_provider;
+struct sqlcipher_provider {
+  int (*init)(void);
+  void (*shutdown)(void);
   const char* (*get_provider_name)(void *ctx);
-  int (*add_random)(void *ctx, void *buffer, int length);
+  int (*add_random)(void *ctx, const void *buffer, int length);
   int (*random)(void *ctx, void *buffer, int length);
-  int (*hmac)(void *ctx, int algorithm, unsigned char *hmac_key, int key_sz, unsigned char *in, int in_sz, unsigned char *in2, int in2_sz, unsigned char *out);
-  int (*kdf)(void *ctx, int algorithm, const unsigned char *pass, int pass_sz, unsigned char* salt, int salt_sz, int workfactor, int key_sz, unsigned char *key);
-  int (*cipher)(void *ctx, int mode, unsigned char *key, int key_sz, unsigned char *iv, unsigned char *in, int in_sz, unsigned char *out);
+  int (*hmac)(void *ctx, int algorithm,
+              const unsigned char *hmac_key, int key_sz,
+              const unsigned char *in, int in_sz,
+              const unsigned char *in2, int in2_sz,
+              unsigned char *out);
+  int (*kdf)(void *ctx, int algorithm,
+              const unsigned char *pass, int pass_sz,
+              const unsigned char* salt, int salt_sz,
+              int workfactor,
+              int key_sz, unsigned char *key);
+  int (*cipher)(void *ctx, int mode,
+              const unsigned char *key, int key_sz,
+              const unsigned char *iv,
+              const unsigned char *in, int in_sz,
+              unsigned char *out);
   const char* (*get_cipher)(void *ctx);
   int (*get_key_sz)(void *ctx);
   int (*get_iv_sz)(void *ctx);
@@ -71,17 +87,25 @@ typedef struct {
   int (*ctx_free)(void **ctx);
   int (*fips_status)(void *ctx);
   const char* (*get_provider_version)(void *ctx);
-} sqlcipher_provider;
+  sqlcipher_provider *next;
+};
+
+/* public interfaces called externally */
+int sqlcipher_extra_init(const char*);
+void sqlcipher_extra_shutdown(void);
+void sqlcipher_init_memmethods(void);
+int sqlcipher_codec_pragma(sqlite3*, int, Parse*, const char *, const char*);
+int sqlcipherCodecAttach(sqlite3*, int, const void *, int);
+void sqlcipherCodecGetKey(sqlite3*, int, void**, int*);
+int sqlcipher_find_db_index(sqlite3 *, const char *);
 
 /* utility functions */
-void* sqlcipher_malloc(sqlite_uint64);
-void sqlcipher_mlock(void *, sqlite_uint64);
-void sqlcipher_munlock(void *, sqlite_uint64);
 void* sqlcipher_memset(void *, unsigned char, sqlite_uint64);
 int sqlcipher_ismemset(const void *, unsigned char, sqlite_uint64);
 int sqlcipher_memcmp(const void *, const void *, int);
+void* sqlcipher_malloc(sqlite_uint64);
 void sqlcipher_free(void *, sqlite_uint64);
-char* sqlcipher_version();
+char* sqlcipher_version(void);
 
 /* provider interfaces */
 int sqlcipher_register_provider(sqlcipher_provider *);
@@ -93,10 +117,48 @@ sqlcipher_provider* sqlcipher_get_provider(void);
 #define SQLCIPHER_MUTEX_RESERVED1         3
 #define SQLCIPHER_MUTEX_RESERVED2         4
 #define SQLCIPHER_MUTEX_RESERVED3         5
-#define SQLCIPHER_MUTEX_COUNT             6
+#define SQLCIPHER_MUTEX_MEM               6
+#define SQLCIPHER_MUTEX_SHAREDCACHE       7
+#define SQLCIPHER_MUTEX_COUNT             8
 
 sqlite3_mutex* sqlcipher_mutex(int);
 
+#define SQLCIPHER_LOG_NONE          0
+#define SQLCIPHER_LOG_ANY           0xffffffff
+
+#define SQLCIPHER_LOG_ERROR         (1<<0)
+#define SQLCIPHER_LOG_WARN          (1<<1)
+#define SQLCIPHER_LOG_INFO          (1<<2)
+#define SQLCIPHER_LOG_DEBUG         (1<<3)
+#define SQLCIPHER_LOG_TRACE         (1<<4)
+
+#define SQLCIPHER_LOG_CORE          (1<<0)
+#define SQLCIPHER_LOG_MEMORY        (1<<1)
+#define SQLCIPHER_LOG_MUTEX         (1<<2)
+#define SQLCIPHER_LOG_PROVIDER      (1<<3)
+
+#ifdef SQLCIPHER_OMIT_LOG
+#define sqlcipher_log(level, source, message, ...)
+#else
+void sqlcipher_log(unsigned int level, unsigned int source, const char *message, ...);
+#endif
+
+#ifdef CODEC_DEBUG_PAGEDATA
+#define CODEC_HEXDUMP(DESC,BUFFER,LEN)  \
+  { \
+    int __pctr; \
+    printf(DESC); \
+    for(__pctr=0; __pctr < LEN; __pctr++) { \
+      if(__pctr % 16 == 0) printf("\n%05x: ",__pctr); \
+      printf("%02x ",((unsigned char*) BUFFER)[__pctr]); \
+    } \
+    printf("\n"); \
+    fflush(stdout); \
+  }
+#else
+#define CODEC_HEXDUMP(DESC,BUFFER,LEN)
+#endif
+
 #endif
 #endif
 /* END SQLCIPHER */
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index be7422cc11..7f82dd0e78 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -131,9 +131,9 @@ extern "C" {
 ** be held constant and Z will be incremented or else Y will be incremented
 ** and Z will be reset to zero.
 **
-** Since [version 3.6.18] ([dateof:3.6.18]), 
+** Since [version 3.6.18] ([dateof:3.6.18]),
 ** SQLite source code has been stored in the
-** <a href="http://www.fossil-scm.org/">Fossil configuration management
+** <a href="http://fossil-scm.org/">Fossil configuration management
 ** system</a>.  ^The SQLITE_SOURCE_ID macro evaluates to
 ** a string which identifies a particular check-in of SQLite
 ** within its configuration management system.  ^The SQLITE_SOURCE_ID
@@ -149,6 +149,9 @@ extern "C" {
 #define SQLITE_VERSION        "--VERS--"
 #define SQLITE_VERSION_NUMBER --VERSION-NUMBER--
 #define SQLITE_SOURCE_ID      "--SOURCE-ID--"
+#define SQLITE_SCM_BRANCH     "--SCM-BRANCH--"
+#define SQLITE_SCM_TAGS       "--SCM-TAGS--"
+#define SQLITE_SCM_DATETIME   "--SCM-DATETIME--"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -168,9 +171,9 @@ extern "C" {
 ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
 ** </pre></blockquote>)^
 **
-** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION]
-** macro.  ^The sqlite3_libversion() function returns a pointer to the
-** to the sqlite3_version[] string constant.  The sqlite3_libversion()
+** ^The sqlite3_version[] string constant contains the text of the
+** [SQLITE_VERSION] macro.  ^The sqlite3_libversion() function returns a
+** pointer to the sqlite3_version[] string constant.  The sqlite3_libversion()
 ** function is provided for use in DLLs since DLL users usually do not have
 ** direct access to string constants within the DLL.  ^The
 ** sqlite3_libversion_number() function returns an integer equal to
@@ -370,7 +373,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** without having to use a lot of C code.
 **
 ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded,
-** semicolon-separate SQL statements passed into its 2nd argument,
+** semicolon-separated SQL statements passed into its 2nd argument,
 ** in the context of the [database connection] passed in as its 1st
 ** argument.  ^If the callback function of the 3rd argument to
 ** sqlite3_exec() is not NULL, then it is invoked for each result row
@@ -403,7 +406,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** result row is NULL then the corresponding string pointer for the
 ** sqlite3_exec() callback is a NULL pointer.  ^The 4th argument to the
 ** sqlite3_exec() callback is an array of pointers to strings where each
-** entry represents the name of corresponding result column as obtained
+** entry represents the name of a corresponding result column as obtained
 ** from [sqlite3_column_name()].
 **
 ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer
@@ -420,6 +423,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 **      the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
 ** <li> The application must not modify the SQL statement text passed into
 **      the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running.
+** <li> The application must not dereference the arrays or string pointers
+**       passed as the 3rd and 4th callback parameters after it returns.
 ** </ul>
 */
 int sqlite3_exec(
@@ -495,6 +500,9 @@ int sqlite3_exec(
 #define SQLITE_ERROR_MISSING_COLLSEQ   (SQLITE_ERROR | (1<<8))
 #define SQLITE_ERROR_RETRY             (SQLITE_ERROR | (2<<8))
 #define SQLITE_ERROR_SNAPSHOT          (SQLITE_ERROR | (3<<8))
+#define SQLITE_ERROR_RESERVESIZE       (SQLITE_ERROR | (4<<8))
+#define SQLITE_ERROR_KEY               (SQLITE_ERROR | (5<<8))
+#define SQLITE_ERROR_UNABLE            (SQLITE_ERROR | (6<<8))
 #define SQLITE_IOERR_READ              (SQLITE_IOERR | (1<<8))
 #define SQLITE_IOERR_SHORT_READ        (SQLITE_IOERR | (2<<8))
 #define SQLITE_IOERR_WRITE             (SQLITE_IOERR | (3<<8))
@@ -528,6 +536,9 @@ int sqlite3_exec(
 #define SQLITE_IOERR_ROLLBACK_ATOMIC   (SQLITE_IOERR | (31<<8))
 #define SQLITE_IOERR_DATA              (SQLITE_IOERR | (32<<8))
 #define SQLITE_IOERR_CORRUPTFS         (SQLITE_IOERR | (33<<8))
+#define SQLITE_IOERR_IN_PAGE           (SQLITE_IOERR | (34<<8))
+#define SQLITE_IOERR_BADKEY            (SQLITE_IOERR | (35<<8))
+#define SQLITE_IOERR_CODEC             (SQLITE_IOERR | (36<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_LOCKED_VTAB             (SQLITE_LOCKED |  (2<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
@@ -586,7 +597,7 @@ int sqlite3_exec(
 ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into
 ** [sqlite3_open_v2()] does *not* cause the underlying database file
 ** to be opened using O_EXCL.  Passing SQLITE_OPEN_EXCLUSIVE into
-** [sqlite3_open_v2()] has historically be a no-op and might become an
+** [sqlite3_open_v2()] has historically been a no-op and might become an
 ** error in future versions of SQLite.
 */
 #define SQLITE_OPEN_READONLY         0x00000001  /* Ok for sqlite3_open_v2() */
@@ -649,6 +660,13 @@ int sqlite3_exec(
 ** filesystem supports doing multiple write operations atomically when those
 ** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and
 ** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE].
+**
+** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read
+** from the database file in amounts that are not a multiple of the
+** page size and that do not begin at a page boundary.  Without this
+** property, SQLite is careful to only do full-page reads and write
+** on aligned pages, with the one exception that it will do a sub-page
+** read of the first page to access the database header.
 */
 #define SQLITE_IOCAP_ATOMIC                 0x00000001
 #define SQLITE_IOCAP_ATOMIC512              0x00000002
@@ -665,6 +683,7 @@ int sqlite3_exec(
 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE    0x00001000
 #define SQLITE_IOCAP_IMMUTABLE              0x00002000
 #define SQLITE_IOCAP_BATCH_ATOMIC           0x00004000
+#define SQLITE_IOCAP_SUBPAGE_READ           0x00008000
 
 /*
 ** CAPI3REF: File Locking Levels
@@ -672,7 +691,7 @@ int sqlite3_exec(
 ** SQLite uses one of these integer values as the second
 ** argument to calls it makes to the xLock() and xUnlock() methods
 ** of an [sqlite3_io_methods] object.  These values are ordered from
-** lest restrictive to most restrictive.
+** least restrictive to most restrictive.
 **
 ** The argument to xLock() is always SHARED or higher.  The argument to
 ** xUnlock is either SHARED or NONE.
@@ -716,7 +735,7 @@ int sqlite3_exec(
 /*
 ** CAPI3REF: OS Interface Open File Handle
 **
-** An [sqlite3_file] object represents an open file in the 
+** An [sqlite3_file] object represents an open file in the
 ** [sqlite3_vfs | OS interface layer].  Individual OS interface
 ** implementations will
 ** want to subclass this object by appending additional fields
@@ -738,7 +757,7 @@ struct sqlite3_file {
 ** This object defines the methods used to perform various operations
 ** against the open file represented by the [sqlite3_file] object.
 **
-** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element 
+** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element
 ** to a non-NULL pointer, then the sqlite3_io_methods.xClose method
 ** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed.  The
 ** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen]
@@ -761,16 +780,16 @@ struct sqlite3_file {
 ** </ul>
 ** xLock() upgrades the database file lock.  In other words, xLock() moves the
 ** database file lock in the direction NONE toward EXCLUSIVE. The argument to
-** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never
+** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never
 ** SQLITE_LOCK_NONE.  If the database file lock is already at or above the
 ** requested lock, then the call to xLock() is a no-op.
 ** xUnlock() downgrades the database file lock to either SHARED or NONE.
-*  If the lock is already at or below the requested lock state, then the call
+** If the lock is already at or below the requested lock state, then the call
 ** to xUnlock() is a no-op.
 ** The xCheckReservedLock() method checks whether any database connection,
 ** either in this process or in some other process, is holding a RESERVED,
-** PENDING, or EXCLUSIVE lock on the file.  It returns true
-** if such a lock exists and false otherwise.
+** PENDING, or EXCLUSIVE lock on the file.  It returns, via its output
+** pointer parameter, true if such a lock exists and false otherwise.
 **
 ** The xFileControl() method is a generic interface that allows custom
 ** VFS implementations to directly control an open file using the
@@ -811,6 +830,7 @@ struct sqlite3_file {
 ** <li> [SQLITE_IOCAP_POWERSAFE_OVERWRITE]
 ** <li> [SQLITE_IOCAP_IMMUTABLE]
 ** <li> [SQLITE_IOCAP_BATCH_ATOMIC]
+** <li> [SQLITE_IOCAP_SUBPAGE_READ]
 ** </ul>
 **
 ** The SQLITE_IOCAP_ATOMIC property means that all writes of
@@ -912,7 +932,7 @@ struct sqlite3_io_methods {
 ** connection.  See also [SQLITE_FCNTL_FILE_POINTER].
 **
 ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]]
-** No longer in use.
+** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used.
 **
 ** <li>[[SQLITE_FCNTL_SYNC]]
 ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and
@@ -982,13 +1002,13 @@ struct sqlite3_io_methods {
 ** <li>[[SQLITE_FCNTL_OVERWRITE]]
 ** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening
 ** a write transaction to indicate that, unless it is rolled back for some
-** reason, the entire database file will be overwritten by the current 
+** reason, the entire database file will be overwritten by the current
 ** transaction. This is used by VACUUM operations.
 **
 ** <li>[[SQLITE_FCNTL_VFSNAME]]
 ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of
-** all [VFSes] in the VFS stack.  The names are of all VFS shims and the
-** final bottom-level VFS are written into memory obtained from 
+** all [VFSes] in the VFS stack.  The names of all VFS shims and the
+** final bottom-level VFS are written into memory obtained from
 ** [sqlite3_malloc()] and the result is stored in the char* variable
 ** that the fourth parameter of [sqlite3_file_control()] points to.
 ** The caller is responsible for freeing the memory when done.  As with
@@ -1001,7 +1021,7 @@ struct sqlite3_io_methods {
 ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level
 ** [VFSes] currently in use.  ^(The argument X in
 ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be
-** of type "[sqlite3_vfs] **".  This opcodes will set *X
+** of type "[sqlite3_vfs] **".  This opcode will set *X
 ** to a pointer to the top-level VFS.)^
 ** ^When there are multiple VFS shims in the stack, this opcode finds the
 ** upper-most shim only.
@@ -1088,6 +1108,11 @@ struct sqlite3_io_methods {
 ** pointed to by the pArg argument.  This capability is used during testing
 ** and only needs to be supported when SQLITE_TEST is defined.
 **
+** <li>[[SQLITE_FCNTL_NULL_IO]]
+** The [SQLITE_FCNTL_NULL_IO] opcode sets the low-level file descriptor
+** or file handle for the [sqlite3_file] object such that it will no longer
+** read or write to the database file.
+**
 ** <li>[[SQLITE_FCNTL_WAL_BLOCK]]
 ** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might
 ** be advantageous to block on the next WAL lock if the lock is not immediately
@@ -1146,6 +1171,12 @@ struct sqlite3_io_methods {
 ** the value that M is to be set to. Before returning, the 32-bit signed
 ** integer is overwritten with the previous value of M.
 **
+** <li>[[SQLITE_FCNTL_BLOCK_ON_CONNECT]]
+** The [SQLITE_FCNTL_BLOCK_ON_CONNECT] opcode is used to configure the
+** VFS to block when taking a SHARED lock to connect to a wal mode database.
+** This is used to implement the functionality associated with
+** SQLITE_SETLK_BLOCK_ON_CONNECT.
+**
 ** <li>[[SQLITE_FCNTL_DATA_VERSION]]
 ** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to
 ** a database file.  The argument is a pointer to a 32-bit unsigned integer.
@@ -1179,25 +1210,34 @@ struct sqlite3_io_methods {
 **
 ** <li>[[SQLITE_FCNTL_EXTERNAL_READER]]
 ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect
-** whether or not there is a database client in another process with a wal-mode 
-** transaction open on the database or not. It is only available on unix.The
+** whether or not there is a database client in another process with a wal-mode
+** transaction open on the database or not. It is only available on unix. The
 ** (void*) argument passed with this file-control should be a pointer to a
 ** value of type (int). The integer value is set to 1 if the database is a wal
 ** mode database and there exists at least one client in another process that
-** currently has an SQL transaction open on the database. It is set to 0 if 
+** currently has an SQL transaction open on the database. It is set to 0 if
 ** the database is not a wal-mode db, or if there is no such connection in any
 ** other process. This opcode cannot be used to detect transactions opened
 ** by clients within the current process, only within other processes.
 **
 ** <li>[[SQLITE_FCNTL_CKSM_FILE]]
-** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use interally by the
+** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the
 ** [checksum VFS shim] only.
 **
 ** <li>[[SQLITE_FCNTL_RESET_CACHE]]
-** If there is currently no transaction open on the database, and the 
+** If there is currently no transaction open on the database, and the
 ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control
 ** purges the contents of the in-memory page cache. If there is an open
 ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error.
+**
+** <li>[[SQLITE_FCNTL_FILESTAT]]
+** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information
+** about the [sqlite3_file] objects used access the database and journal files
+** for the given schema.  The fourth parameter to [sqlite3_file_control()]
+** should be an initialized [sqlite3_str] pointer.  JSON text describing
+** various aspects of the sqlite3_file object is appended to the sqlite3_str.
+** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time
+** options are used to enable it.
 ** </ul>
 */
 #define SQLITE_FCNTL_LOCKSTATE               1
@@ -1241,6 +1281,9 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_EXTERNAL_READER        40
 #define SQLITE_FCNTL_CKSM_FILE              41
 #define SQLITE_FCNTL_RESET_CACHE            42
+#define SQLITE_FCNTL_NULL_IO                43
+#define SQLITE_FCNTL_BLOCK_ON_CONNECT       44
+#define SQLITE_FCNTL_FILESTAT               45
 
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
@@ -1496,7 +1539,7 @@ struct sqlite3_vfs {
   /*
   ** The methods above are in versions 1 through 3 of the sqlite_vfs object.
   ** New fields may be appended in future versions.  The iVersion
-  ** value will increment whenever this happens. 
+  ** value will increment whenever this happens.
   */
 };
 
@@ -1603,7 +1646,7 @@ struct sqlite3_vfs {
 ** SQLite interfaces so that an application usually does not need to
 ** invoke sqlite3_initialize() directly.  For example, [sqlite3_open()]
 ** calls sqlite3_initialize() so the SQLite library will be automatically
-** initialized when [sqlite3_open()] is called if it has not be initialized
+** initialized when [sqlite3_open()] is called if it has not been initialized
 ** already.  ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT]
 ** compile-time option, then the automatic calls to sqlite3_initialize()
 ** are omitted and the application must call sqlite3_initialize() directly
@@ -1688,7 +1731,7 @@ int sqlite3_config(int, ...);
 ** [database connection] (specified in the first argument).
 **
 ** The second argument to sqlite3_db_config(D,V,...)  is the
-** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code 
+** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code
 ** that indicates what aspect of the [database connection] is being configured.
 ** Subsequent arguments vary depending on the configuration verb.
 **
@@ -1736,7 +1779,7 @@ int sqlite3_db_config(sqlite3*, int op, ...);
 ** allocators round up memory allocations at least to the next multiple
 ** of 8.  Some allocators round up to a larger multiple or to a power of 2.
 ** Every memory allocation request coming in through [sqlite3_malloc()]
-** or [sqlite3_realloc()] first calls xRoundup.  If xRoundup returns 0, 
+** or [sqlite3_realloc()] first calls xRoundup.  If xRoundup returns 0,
 ** that causes the corresponding memory allocation to fail.
 **
 ** The xInit method initializes the memory allocator.  For example,
@@ -1811,7 +1854,7 @@ struct sqlite3_mem_methods {
 ** by a single thread.   ^If SQLite is compiled with
 ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then
 ** it is not possible to change the [threading mode] from its default
-** value of Single-thread and so [sqlite3_config()] will return 
+** value of Single-thread and so [sqlite3_config()] will return
 ** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD
 ** configuration option.</dd>
 **
@@ -1860,21 +1903,21 @@ struct sqlite3_mem_methods {
 ** The [sqlite3_mem_methods]
 ** structure is filled with the currently defined memory allocation routines.)^
 ** This option can be used to overload the default memory allocation
-** routines with a wrapper that simulations memory allocation failure or
+** routines with a wrapper that simulates memory allocation failure or
 ** tracks memory usage, for example. </dd>
 **
 ** [[SQLITE_CONFIG_SMALL_MALLOC]] <dt>SQLITE_CONFIG_SMALL_MALLOC</dt>
-** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of
+** <dd> ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of
 ** type int, interpreted as a boolean, which if true provides a hint to
 ** SQLite that it should avoid large memory allocations if possible.
 ** SQLite will run faster if it is free to make large memory allocations,
-** but some application might prefer to run slower in exchange for
+** but some applications might prefer to run slower in exchange for
 ** guarantees about memory fragmentation that are possible if large
 ** allocations are avoided.  This hint is normally off.
 ** </dd>
 **
 ** [[SQLITE_CONFIG_MEMSTATUS]] <dt>SQLITE_CONFIG_MEMSTATUS</dt>
-** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int,
+** <dd> ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int,
 ** interpreted as a boolean, which enables or disables the collection of
 ** memory allocation statistics. ^(When memory allocation statistics are
 ** disabled, the following SQLite interfaces become non-operational:
@@ -1919,7 +1962,7 @@ struct sqlite3_mem_methods {
 ** ^If pMem is NULL and N is non-zero, then each database connection
 ** does an initial bulk allocation for page cache memory
 ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or
-** of -1024*N bytes if N is negative, . ^If additional
+** of -1024*N bytes if N is negative. ^If additional
 ** page cache memory is needed beyond what is provided by the initial
 ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each
 ** additional cache line. </dd>
@@ -1948,7 +1991,7 @@ struct sqlite3_mem_methods {
 ** <dd> ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a
 ** pointer to an instance of the [sqlite3_mutex_methods] structure.
 ** The argument specifies alternative low-level mutex routines to be used
-** in place the mutex routines built into SQLite.)^  ^SQLite makes a copy of
+** in place of the mutex routines built into SQLite.)^  ^SQLite makes a copy of
 ** the content of the [sqlite3_mutex_methods] structure before the call to
 ** [sqlite3_config()] returns. ^If SQLite is compiled with
 ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then
@@ -1971,13 +2014,16 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_CONFIG_LOOKASIDE]] <dt>SQLITE_CONFIG_LOOKASIDE</dt>
 ** <dd> ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine
-** the default size of lookaside memory on each [database connection].
+** the default size of [lookaside memory] on each [database connection].
 ** The first argument is the
-** size of each lookaside buffer slot and the second is the number of
-** slots allocated to each database connection.)^  ^(SQLITE_CONFIG_LOOKASIDE
-** sets the <i>default</i> lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE]
-** option to [sqlite3_db_config()] can be used to change the lookaside
-** configuration on individual connections.)^ </dd>
+** size of each lookaside buffer slot ("sz") and the second is the number of
+** slots allocated to each database connection ("cnt").)^
+** ^(SQLITE_CONFIG_LOOKASIDE sets the <i>default</i> lookaside size.
+** The [SQLITE_DBCONFIG_LOOKASIDE] option to [sqlite3_db_config()] can
+** be used to change the lookaside configuration on individual connections.)^
+** The [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to change the
+** default lookaside configuration at compile-time.
+** </dd>
 **
 ** [[SQLITE_CONFIG_PCACHE2]] <dt>SQLITE_CONFIG_PCACHE2</dt>
 ** <dd> ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is
@@ -1987,7 +2033,7 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_CONFIG_GETPCACHE2]] <dt>SQLITE_CONFIG_GETPCACHE2</dt>
 ** <dd> ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which
-** is a pointer to an [sqlite3_pcache_methods2] object.  SQLite copies of
+** is a pointer to an [sqlite3_pcache_methods2] object.  SQLite copies off
 ** the current page cache implementation into that object.)^ </dd>
 **
 ** [[SQLITE_CONFIG_LOG]] <dt>SQLITE_CONFIG_LOG</dt>
@@ -2004,7 +2050,7 @@ struct sqlite3_mem_methods {
 ** the logger function is a copy of the first parameter to the corresponding
 ** [sqlite3_log()] call and is intended to be a [result code] or an
 ** [extended result code].  ^The third parameter passed to the logger is
-** log message after formatting via [sqlite3_snprintf()].
+** a log message after formatting via [sqlite3_snprintf()].
 ** The SQLite logging interface is not reentrant; the logger function
 ** supplied by the application must not invoke any SQLite interface.
 ** In a multi-threaded application, the application-defined logger
@@ -2126,7 +2172,7 @@ struct sqlite3_mem_methods {
 ** is stored in each sorted record and the required column values loaded
 ** from the database as records are returned in sorted order. The default
 ** value for this option is to never use this optimization. Specifying a
-** negative value for this option restores the default behaviour.
+** negative value for this option restores the default behavior.
 ** This option is only available if SQLite is compiled with the
 ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.
 **
@@ -2140,6 +2186,22 @@ struct sqlite3_mem_methods {
 ** configuration setting is never used, then the default maximum is determined
 ** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option.  If that
 ** compile-time option is not set, then the default maximum is 1073741824.
+**
+** [[SQLITE_CONFIG_ROWID_IN_VIEW]]
+** <dt>SQLITE_CONFIG_ROWID_IN_VIEW
+** <dd>The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability
+** for VIEWs to have a ROWID.  The capability can only be enabled if SQLite is
+** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability
+** defaults to on.  This configuration option queries the current setting or
+** changes the setting to off or on.  The argument is a pointer to an integer.
+** If that integer initially holds a value of 1, then the ability for VIEWs to
+** have ROWIDs is activated.  If the integer initially holds zero, then the
+** ability is deactivated.  Any other initial value for the integer leaves the
+** setting unchanged.  After changes, if any, the integer is written with
+** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off.  If SQLite
+** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and
+** recommended case) then the integer is always filled with zero, regardless
+** if its initial value.
 ** </dl>
 */
 #define SQLITE_CONFIG_SINGLETHREAD         1  /* nil */
@@ -2171,12 +2233,21 @@ struct sqlite3_mem_methods {
 #define SQLITE_CONFIG_SMALL_MALLOC        27  /* boolean */
 #define SQLITE_CONFIG_SORTERREF_SIZE      28  /* int nByte */
 #define SQLITE_CONFIG_MEMDB_MAXSIZE       29  /* sqlite3_int64 */
+#define SQLITE_CONFIG_ROWID_IN_VIEW       30  /* int* */
 
 /*
 ** CAPI3REF: Database Connection Configuration Options
 **
 ** These constants are the available integer configuration options that
-** can be passed as the second argument to the [sqlite3_db_config()] interface.
+** can be passed as the second parameter to the [sqlite3_db_config()] interface.
+**
+** The [sqlite3_db_config()] interface is a var-args function.  It takes a
+** variable number of parameters, though always at least two.  The number of
+** parameters passed into sqlite3_db_config() depends on which of these
+** constants is given as the second parameter.  This documentation page
+** refers to parameters beyond the second as "arguments".  Thus, when this
+** page says "the N-th argument" it means "the N-th parameter past the
+** configuration option" or "the (N+2)-th parameter to sqlite3_db_config()".
 **
 ** New configuration options may be added in future releases of SQLite.
 ** Existing configuration options might be discontinued.  Applications
@@ -2188,31 +2259,57 @@ struct sqlite3_mem_methods {
 ** <dl>
 ** [[SQLITE_DBCONFIG_LOOKASIDE]]
 ** <dt>SQLITE_DBCONFIG_LOOKASIDE</dt>
-** <dd> ^This option takes three additional arguments that determine the
-** [lookaside memory allocator] configuration for the [database connection].
-** ^The first argument (the third parameter to [sqlite3_db_config()] is a
+** <dd> The SQLITE_DBCONFIG_LOOKASIDE option is used to adjust the 
+** configuration of the [lookaside memory allocator] within a database
+** connection.
+** The arguments to the SQLITE_DBCONFIG_LOOKASIDE option are <i>not</i>
+** in the [DBCONFIG arguments|usual format].
+** The SQLITE_DBCONFIG_LOOKASIDE option takes three arguments, not two,
+** so that a call to [sqlite3_db_config()] that uses SQLITE_DBCONFIG_LOOKASIDE
+** should have a total of five parameters.
+** <ol>
+** <li><p>The first argument ("buf") is a
 ** pointer to a memory buffer to use for lookaside memory.
-** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb
-** may be NULL in which case SQLite will allocate the
-** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the
-** size of each lookaside buffer slot.  ^The third argument is the number of
-** slots.  The size of the buffer in the first argument must be greater than
-** or equal to the product of the second and third arguments.  The buffer
-** must be aligned to an 8-byte boundary.  ^If the second argument to
-** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally
-** rounded down to the next smaller multiple of 8.  ^(The lookaside memory
+** The first argument may be NULL in which case SQLite will allocate the
+** lookaside buffer itself using [sqlite3_malloc()].
+** <li><P>The second argument ("sz") is the
+** size of each lookaside buffer slot.  Lookaside is disabled if "sz"
+** is less than 8.  The "sz" argument should be a multiple of 8 less than
+** 65536.  If "sz" does not meet this constraint, it is reduced in size until
+** it does.
+** <li><p>The third argument ("cnt") is the number of slots. Lookaside is disabled
+** if "cnt"is less than 1.  The "cnt" value will be reduced, if necessary, so
+** that the product of "sz" and "cnt" does not exceed 2,147,418,112.  The "cnt"
+** parameter is usually chosen so that the product of "sz" and "cnt" is less
+** than 1,000,000.
+** </ol>
+** <p>If the "buf" argument is not NULL, then it must
+** point to a memory buffer with a size that is greater than
+** or equal to the product of "sz" and "cnt".
+** The buffer must be aligned to an 8-byte boundary.
+** The lookaside memory
 ** configuration for a database connection can only be changed when that
 ** connection is not currently using lookaside memory, or in other words
-** when the "current value" returned by
-** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero.
+** when the value returned by [SQLITE_DBSTATUS_LOOKASIDE_USED] is zero.
 ** Any attempt to change the lookaside memory configuration when lookaside
 ** memory is in use leaves the configuration unchanged and returns
-** [SQLITE_BUSY].)^</dd>
+** [SQLITE_BUSY].
+** If the "buf" argument is NULL and an attempt
+** to allocate memory based on "sz" and "cnt" fails, then
+** lookaside is silently disabled.
+** <p>
+** The [SQLITE_CONFIG_LOOKASIDE] configuration option can be used to set the
+** default lookaside configuration at initialization.  The
+** [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to set the default lookaside
+** configuration at compile-time.  Typical values for lookaside are 1200 for
+** "sz" and 40 to 100 for "cnt".
+** </dd>
 **
 ** [[SQLITE_DBCONFIG_ENABLE_FKEY]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_FKEY</dt>
 ** <dd> ^This option is used to enable or disable the enforcement of
-** [foreign key constraints].  There should be two additional arguments.
+** [foreign key constraints].  This is the same setting that is
+** enabled or disabled by the [PRAGMA foreign_keys] statement.
 ** The first argument is an integer which is 0 to disable FK enforcement,
 ** positive to enable FK enforcement or negative to leave FK enforcement
 ** unchanged.  The second parameter is a pointer to an integer into which
@@ -2234,13 +2331,13 @@ struct sqlite3_mem_methods {
 ** <p>Originally this option disabled all triggers.  ^(However, since
 ** SQLite version 3.35.0, TEMP triggers are still allowed even if
 ** this option is off.  So, in other words, this option now only disables
-** triggers in the main database schema or in the schemas of ATTACH-ed
+** triggers in the main database schema or in the schemas of [ATTACH]-ed
 ** databases.)^ </dd>
 **
 ** [[SQLITE_DBCONFIG_ENABLE_VIEW]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_VIEW</dt>
 ** <dd> ^This option is used to enable or disable [CREATE VIEW | views].
-** There should be two additional arguments.
+** There must be two additional arguments.
 ** The first argument is an integer which is 0 to disable views,
 ** positive to enable views or negative to leave the setting unchanged.
 ** The second parameter is a pointer to an integer into which
@@ -2256,17 +2353,20 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER</dt>
-** <dd> ^This option is used to enable or disable the
-** [fts3_tokenizer()] function which is part of the
-** [FTS3] full-text search engine extension.
-** There should be two additional arguments.
-** The first argument is an integer which is 0 to disable fts3_tokenizer() or
-** positive to enable fts3_tokenizer() or negative to leave the setting
-** unchanged.
-** The second parameter is a pointer to an integer into which
-** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled
-** following this call.  The second parameter may be a NULL pointer, in
-** which case the new setting is not reported back. </dd>
+** <dd> ^This option is used to enable or disable using the
+** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine
+** extension - without using bound parameters as the parameters. Doing so
+** is disabled by default. There must be two additional arguments. The first
+** argument is an integer. If it is passed 0, then using fts3_tokenizer()
+** without bound parameters is disabled. If it is passed a positive value, 
+** then calling fts3_tokenizer without bound parameters is enabled. If it 
+** is passed a negative value, this setting is not modified - this can be 
+** used to query for the current setting. The second parameter is a pointer 
+** to an integer into which is written 0 or 1 to indicate the current value 
+** of this setting (after it is modified, if applicable).  The second 
+** parameter may be a NULL pointer, in which case the value of the setting 
+** is not reported back. Refer to [FTS3] documentation for further details.
+** </dd> 
 **
 ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]]
 ** <dt>SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION</dt>
@@ -2274,12 +2374,12 @@ struct sqlite3_mem_methods {
 ** interface independently of the [load_extension()] SQL function.
 ** The [sqlite3_enable_load_extension()] API enables or disables both the
 ** C-API [sqlite3_load_extension()] and the SQL function [load_extension()].
-** There should be two additional arguments.
+** There must be two additional arguments.
 ** When the first argument to this interface is 1, then only the C-API is
 ** enabled and the SQL function remains disabled.  If the first argument to
 ** this interface is 0, then both the C-API and the SQL function are disabled.
-** If the first argument is -1, then no changes are made to state of either the
-** C-API or the SQL function.
+** If the first argument is -1, then no changes are made to the state of either
+** the C-API or the SQL function.
 ** The second parameter is a pointer to an integer into which
 ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface
 ** is disabled or enabled following this call.  The second parameter may
@@ -2288,23 +2388,30 @@ struct sqlite3_mem_methods {
 **
 ** [[SQLITE_DBCONFIG_MAINDBNAME]] <dt>SQLITE_DBCONFIG_MAINDBNAME</dt>
 ** <dd> ^This option is used to change the name of the "main" database
-** schema.  ^The sole argument is a pointer to a constant UTF8 string
-** which will become the new schema name in place of "main".  ^SQLite
-** does not make a copy of the new main schema name string, so the application
-** must ensure that the argument passed into this DBCONFIG option is unchanged
-** until after the database connection closes.
+** schema.  This option does not follow the
+** [DBCONFIG arguments|usual SQLITE_DBCONFIG argument format].
+** This option takes exactly one additional argument so that the 
+** [sqlite3_db_config()] call has a total of three parameters.  The
+** extra argument must be a pointer to a constant UTF8 string which
+** will become the new schema name in place of "main".  ^SQLite does
+** not make a copy of the new main schema name string, so the application
+** must ensure that the argument passed into SQLITE_DBCONFIG MAINDBNAME
+** is unchanged until after the database connection closes.
 ** </dd>
 **
-** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] 
+** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]]
 ** <dt>SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE</dt>
-** <dd> Usually, when a database in wal mode is closed or detached from a 
-** database handle, SQLite checks if this will mean that there are now no 
-** connections at all to the database. If so, it performs a checkpoint 
-** operation before closing the connection. This option may be used to
-** override this behaviour. The first parameter passed to this operation
-** is an integer - positive to disable checkpoints-on-close, or zero (the
-** default) to enable them, and negative to leave the setting unchanged.
-** The second parameter is a pointer to an integer
+** <dd> Usually, when a database in [WAL mode] is closed or detached from a
+** database handle, SQLite checks if if there are other connections to the
+** same database, and if there are no other database connection (if the
+** connection being closed is the last open connection to the database),
+** then SQLite performs a [checkpoint] before closing the connection and
+** deletes the WAL file.  The SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE option can
+** be used to override that behavior. The first argument passed to this
+** operation (the third parameter to [sqlite3_db_config()]) is an integer
+** which is positive to disable checkpoints-on-close, or zero (the default)
+** to enable them, and negative to leave the setting unchanged.
+** The second argument (the fourth parameter) is a pointer to an integer
 ** into which is written 0 or 1 to indicate whether checkpoints-on-close
 ** have been disabled - 0 if they are not disabled, 1 if they are.
 ** </dd>
@@ -2318,7 +2425,7 @@ struct sqlite3_mem_methods {
 ** slower.  But the QPSG has the advantage of more predictable behavior.  With
 ** the QPSG active, SQLite will always use the same query plan in the field as
 ** was used during testing in the lab.
-** The first argument to this setting is an integer which is 0 to disable 
+** The first argument to this setting is an integer which is 0 to disable
 ** the QPSG, positive to enable QPSG, or negative to leave the setting
 ** unchanged. The second parameter is a pointer to an integer into which
 ** is written 0 or 1 to indicate whether the QPSG is disabled or enabled
@@ -2326,15 +2433,15 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_TRIGGER_EQP]] <dt>SQLITE_DBCONFIG_TRIGGER_EQP</dt>
-** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not 
+** <dd> By default, the output of EXPLAIN QUERY PLAN commands does not
 ** include output for any operations performed by trigger programs. This
 ** option is used to set or clear (the default) a flag that governs this
 ** behavior. The first parameter passed to this operation is an integer -
 ** positive to enable output for trigger programs, or zero to disable it,
 ** or negative to leave the setting unchanged.
-** The second parameter is a pointer to an integer into which is written 
-** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if 
-** it is not disabled, 1 if it is.  
+** The second parameter is a pointer to an integer into which is written
+** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if
+** it is not disabled, 1 if it is. 
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_RESET_DATABASE]] <dt>SQLITE_DBCONFIG_RESET_DATABASE</dt>
@@ -2348,7 +2455,7 @@ struct sqlite3_mem_methods {
 **      database, or calling sqlite3_table_column_metadata(), ignoring any
 **      errors.  This step is only necessary if the application desires to keep
 **      the database in WAL mode after the reset if it was in WAL mode before
-**      the reset.  
+**      the reset. 
 ** <li> sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
 ** <li> [sqlite3_exec](db, "[VACUUM]", 0, 0, 0);
 ** <li> sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0);
@@ -2364,7 +2471,7 @@ struct sqlite3_mem_methods {
 ** [[SQLITE_DBCONFIG_DEFENSIVE]] <dt>SQLITE_DBCONFIG_DEFENSIVE</dt>
 ** <dd>The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the
 ** "defensive" flag for a database connection.  When the defensive
-** flag is enabled, language features that allow ordinary SQL to 
+** flag is enabled, language features that allow ordinary SQL to
 ** deliberately corrupt the database file are disabled.  The disabled
 ** features include but are not limited to the following:
 ** <ul>
@@ -2380,7 +2487,7 @@ struct sqlite3_mem_methods {
 ** <dd>The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the
 ** "writable_schema" flag. This has the same effect and is logically equivalent
 ** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF].
-** The first argument to this setting is an integer which is 0 to disable 
+** The first argument to this setting is an integer which is 0 to disable
 ** the writable_schema, positive to enable writable_schema, or negative to
 ** leave the setting unchanged. The second parameter is a pointer to an
 ** integer into which is written 0 or 1 to indicate whether the writable_schema
@@ -2390,7 +2497,7 @@ struct sqlite3_mem_methods {
 ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]]
 ** <dt>SQLITE_DBCONFIG_LEGACY_ALTER_TABLE</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates
-** the legacy behavior of the [ALTER TABLE RENAME] command such it
+** the legacy behavior of the [ALTER TABLE RENAME] command such that it
 ** behaves as it did prior to [version 3.24.0] (2018-06-04).  See the
 ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for
 ** additional information. This feature can also be turned on and off
@@ -2424,7 +2531,7 @@ struct sqlite3_mem_methods {
 ** including:
 ** <ul>
 ** <li> Prohibit the use of SQL functions inside triggers, views,
-** CHECK constraints, DEFAULT clauses, expression indexes, 
+** CHECK constraints, DEFAULT clauses, expression indexes,
 ** partial indexes, or generated columns
 ** unless those functions are tagged with [SQLITE_INNOCUOUS].
 ** <li> Prohibit the use of virtual tables inside of triggers or views
@@ -2439,13 +2546,13 @@ struct sqlite3_mem_methods {
 ** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates
 ** the legacy file format flag.  When activated, this flag causes all newly
-** created database file to have a schema format version number (the 4-byte
+** created database files to have a schema format version number (the 4-byte
 ** integer found at offset 44 into the database header) of 1.  This in turn
 ** means that the resulting database file will be readable and writable by
 ** any SQLite version back to 3.0.0 ([dateof:3.0.0]).  Without this setting,
 ** newly created databases are generally not understandable by SQLite versions
 ** prior to 3.3.0 ([dateof:3.3.0]).  As these words are written, there
-** is now scarcely any need to generate database files that are compatible 
+** is now scarcely any need to generate database files that are compatible
 ** all the way back to version 3.0.0, and so this setting is of little
 ** practical use, but is provided so that SQLite can continue to claim the
 ** ability to generate new database files that are compatible with  version
@@ -2454,7 +2561,7 @@ struct sqlite3_mem_methods {
 ** the [VACUUM] command will fail with an obscure error when attempting to
 ** process a table with generated columns and a descending index.  This is
 ** not considered a bug since SQLite versions 3.3.0 and earlier do not support
-** either generated columns or decending indexes.
+** either generated columns or descending indexes.
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]]
@@ -2463,10 +2570,10 @@ struct sqlite3_mem_methods {
 ** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears
 ** a flag that enables collection of the sqlite3_stmt_scanstatus_v2()
 ** statistics. For statistics to be collected, the flag must be set on
-** the database handle both when the SQL statement is prepared and when it 
+** the database handle both when the SQL statement is prepared and when it
 ** is stepped. The flag is set (collection of statistics is enabled)
-** by default.  This option takes two arguments: an integer and a pointer to
-** an integer..  The first argument is 1, 0, or -1 to enable, disable, or
+** by default. <p>This option takes two arguments: an integer and a pointer to
+** an integer.  The first argument is 1, 0, or -1 to enable, disable, or
 ** leave unchanged the statement scanstatus option.  If the second argument
 ** is not NULL, then the value of the statement scanstatus setting after
 ** processing the first argument is written into the integer that the second
@@ -2479,7 +2586,7 @@ struct sqlite3_mem_methods {
 ** in which tables and indexes are scanned so that the scans start at the end
 ** and work toward the beginning rather than starting at the beginning and
 ** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the
-** same as setting [PRAGMA reverse_unordered_selects].  This option takes
+** same as setting [PRAGMA reverse_unordered_selects]. <p>This option takes
 ** two arguments which are an integer and a pointer to an integer.  The first
 ** argument is 1, 0, or -1 to enable, disable, or leave unchanged the
 ** reverse scan order flag, respectively.  If the second argument is not NULL,
@@ -2488,7 +2595,76 @@ struct sqlite3_mem_methods {
 ** first argument.
 ** </dd>
 **
+** [[SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE]]
+** <dt>SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE</dt>
+** <dd>The SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE option enables or disables
+** the ability of the [ATTACH DATABASE] SQL command to create a new database
+** file if the database filed named in the ATTACH command does not already
+** exist.  This ability of ATTACH to create a new database is enabled by
+** default.  Applications can disable or reenable the ability for ATTACH to
+** create new database files using this DBCONFIG option.<p>
+** This option takes two arguments which are an integer and a pointer
+** to an integer.  The first argument is 1, 0, or -1 to enable, disable, or
+** leave unchanged the attach-create flag, respectively.  If the second
+** argument is not NULL, then 0 or 1 is written into the integer that the
+** second argument points to depending on if the attach-create flag is set
+** after processing the first argument. 
+** </dd>
+**
+** [[SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE]]
+** <dt>SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE</dt>
+** <dd>The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the
+** ability of the [ATTACH DATABASE] SQL command to open a database for writing.
+** This capability is enabled by default.  Applications can disable or
+** reenable this capability using the current DBCONFIG option.  If
+** this capability is disabled, the [ATTACH] command will still work,
+** but the database will be opened read-only.  If this option is disabled,
+** then the ability to create a new database using [ATTACH] is also disabled,
+** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE]
+** option.<p>
+** This option takes two arguments which are an integer and a pointer
+** to an integer.  The first argument is 1, 0, or -1 to enable, disable, or
+** leave unchanged the ability to ATTACH another database for writing,
+** respectively.  If the second argument is not NULL, then 0 or 1 is written
+** into the integer to which the second argument points, depending on whether
+** the ability to ATTACH a read/write database is enabled or disabled
+** after processing the first argument.
+** </dd>
+**
+** [[SQLITE_DBCONFIG_ENABLE_COMMENTS]]
+** <dt>SQLITE_DBCONFIG_ENABLE_COMMENTS</dt>
+** <dd>The SQLITE_DBCONFIG_ENABLE_COMMENTS option enables or disables the
+** ability to include comments in SQL text.  Comments are enabled by default.
+** An application can disable or reenable comments in SQL text using this
+** DBCONFIG option.<p>
+** This option takes two arguments which are an integer and a pointer
+** to an integer.  The first argument is 1, 0, or -1 to enable, disable, or
+** leave unchanged the ability to use comments in SQL text,
+** respectively.  If the second argument is not NULL, then 0 or 1 is written
+** into the integer that the second argument points to depending on if 
+** comments are allowed in SQL text after processing the first argument.
+** </dd>
+**
 ** </dl>
+**
+** [[DBCONFIG arguments]] <h3>Arguments To SQLITE_DBCONFIG Options</h3>
+**
+** <p>Most of the SQLITE_DBCONFIG options take two arguments, so that the
+** overall call to [sqlite3_db_config()] has a total of four parameters.
+** The first argument (the third parameter to sqlite3_db_config()) is an integer.
+** The second argument is a pointer to an integer.  If the first argument is 1,
+** then the option becomes enabled.  If the first integer argument is 0, then the
+** option is disabled.  If the first argument is -1, then the option setting
+** is unchanged.  The second argument, the pointer to an integer, may be NULL.
+** If the second argument is not NULL, then a value of 0 or 1 is written into
+** the integer to which the second argument points, depending on whether the
+** setting is disabled or enabled after applying any changes specified by
+** the first argument.
+**
+** <p>While most SQLITE_DBCONFIG options use the argument format
+** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME]
+** and [SQLITE_DBCONFIG_LOOKASIDE] options are different.  See the
+** documentation of those exceptional options for details.
 */
 #define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
 #define SQLITE_DBCONFIG_LOOKASIDE             1001 /* void* int int */
@@ -2510,7 +2686,10 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_TRUSTED_SCHEMA        1017 /* int int* */
 #define SQLITE_DBCONFIG_STMT_SCANSTATUS       1018 /* int int* */
 #define SQLITE_DBCONFIG_REVERSE_SCANORDER     1019 /* int int* */
-#define SQLITE_DBCONFIG_MAX                   1019 /* Largest DBCONFIG */
+#define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE  1020 /* int int* */
+#define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE   1021 /* int int* */
+#define SQLITE_DBCONFIG_ENABLE_COMMENTS       1022 /* int int* */
+#define SQLITE_DBCONFIG_MAX                   1022 /* Largest DBCONFIG */
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
@@ -2537,8 +2716,8 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff);
 ** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of
 ** the most recent successful [INSERT] into a rowid table or [virtual table]
 ** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not
-** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred 
-** on the database connection D, then sqlite3_last_insert_rowid(D) returns 
+** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred
+** on the database connection D, then sqlite3_last_insert_rowid(D) returns
 ** zero.
 **
 ** As well as being set automatically as rows are inserted into database
@@ -2548,15 +2727,15 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff);
 ** Some virtual table implementations may INSERT rows into rowid tables as
 ** part of committing a transaction (e.g. to flush data accumulated in memory
 ** to disk). In this case subsequent calls to this function return the rowid
-** associated with these internal INSERT operations, which leads to 
+** associated with these internal INSERT operations, which leads to
 ** unintuitive results. Virtual table implementations that do write to rowid
-** tables in this way can avoid this problem by restoring the original 
-** rowid value using [sqlite3_set_last_insert_rowid()] before returning 
+** tables in this way can avoid this problem by restoring the original
+** rowid value using [sqlite3_set_last_insert_rowid()] before returning
 ** control to the user.
 **
-** ^(If an [INSERT] occurs within a trigger then this routine will 
-** return the [rowid] of the inserted row as long as the trigger is 
-** running. Once the trigger program ends, the value returned 
+** ^(If an [INSERT] occurs within a trigger then this routine will
+** return the [rowid] of the inserted row as long as the trigger is
+** running. Once the trigger program ends, the value returned
 ** by this routine reverts to what it was before the trigger was fired.)^
 **
 ** ^An [INSERT] that fails due to a constraint violation is not a
@@ -2589,7 +2768,7 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*);
 ** METHOD: sqlite3
 **
 ** The sqlite3_set_last_insert_rowid(D, R) method allows the application to
-** set the value returned by calling sqlite3_last_insert_rowid(D) to R 
+** set the value returned by calling sqlite3_last_insert_rowid(D) to R
 ** without inserting a row into the database.
 */
 void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64);
@@ -2602,43 +2781,47 @@ void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64);
 ** deleted by the most recently completed INSERT, UPDATE or DELETE
 ** statement on the database connection specified by the only parameter.
 ** The two functions are identical except for the type of the return value
-** and that if the number of rows modified by the most recent INSERT, UPDATE 
+** and that if the number of rows modified by the most recent INSERT, UPDATE,
 ** or DELETE is greater than the maximum value supported by type "int", then
 ** the return value of sqlite3_changes() is undefined. ^Executing any other
 ** type of SQL statement does not modify the value returned by these functions.
+** For the purposes of this interface, a CREATE TABLE AS SELECT statement
+** does not count as an INSERT, UPDATE or DELETE statement and hence the rows
+** added to the new table by the CREATE TABLE AS SELECT statement are not
+** counted.
 **
 ** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are
-** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], 
+** considered - auxiliary changes caused by [CREATE TRIGGER | triggers],
 ** [foreign key actions] or [REPLACE] constraint resolution are not counted.
-** 
-** Changes to a view that are intercepted by 
-** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value 
-** returned by sqlite3_changes() immediately after an INSERT, UPDATE or 
-** DELETE statement run on a view is always zero. Only changes made to real 
+**
+** Changes to a view that are intercepted by
+** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value
+** returned by sqlite3_changes() immediately after an INSERT, UPDATE or
+** DELETE statement run on a view is always zero. Only changes made to real
 ** tables are counted.
 **
 ** Things are more complicated if the sqlite3_changes() function is
 ** executed while a trigger program is running. This may happen if the
 ** program uses the [changes() SQL function], or if some other callback
 ** function invokes sqlite3_changes() directly. Essentially:
-** 
+**
 ** <ul>
 **   <li> ^(Before entering a trigger program the value returned by
-**        sqlite3_changes() function is saved. After the trigger program 
+**        sqlite3_changes() function is saved. After the trigger program
 **        has finished, the original value is restored.)^
-** 
-**   <li> ^(Within a trigger program each INSERT, UPDATE and DELETE 
-**        statement sets the value returned by sqlite3_changes() 
-**        upon completion as normal. Of course, this value will not include 
-**        any changes performed by sub-triggers, as the sqlite3_changes() 
+**
+**   <li> ^(Within a trigger program each INSERT, UPDATE and DELETE
+**        statement sets the value returned by sqlite3_changes()
+**        upon completion as normal. Of course, this value will not include
+**        any changes performed by sub-triggers, as the sqlite3_changes()
 **        value will be saved and restored after each sub-trigger has run.)^
 ** </ul>
-** 
+**
 ** ^This means that if the changes() SQL function (or similar) is used
-** by the first INSERT, UPDATE or DELETE statement within a trigger, it 
+** by the first INSERT, UPDATE or DELETE statement within a trigger, it
 ** returns the value as set when the calling statement began executing.
-** ^If it is used by the second or subsequent such statement within a trigger 
-** program, the value returned reflects the number of rows modified by the 
+** ^If it is used by the second or subsequent such statement within a trigger
+** program, the value returned reflects the number of rows modified by the
 ** previous INSERT, UPDATE or DELETE statement within the same trigger.
 **
 ** If a separate thread makes changes on the same database connection
@@ -2663,16 +2846,16 @@ sqlite3_int64 sqlite3_changes64(sqlite3*);
 ** ^These functions return the total number of rows inserted, modified or
 ** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed
 ** since the database connection was opened, including those executed as
-** part of trigger programs. The two functions are identical except for the 
-** type of the return value and that if the number of rows modified by the 
+** part of trigger programs. The two functions are identical except for the
+** type of the return value and that if the number of rows modified by the
 ** connection exceeds the maximum value supported by type "int", then
-** the return value of sqlite3_total_changes() is undefined. ^Executing 
-** any other type of SQL statement does not affect the value returned by 
+** the return value of sqlite3_total_changes() is undefined. ^Executing
+** any other type of SQL statement does not affect the value returned by
 ** sqlite3_total_changes().
-** 
+**
 ** ^Changes made as part of [foreign key actions] are included in the
 ** count, but those made as part of REPLACE constraint resolution are
-** not. ^Changes to a view that are intercepted by INSTEAD OF triggers 
+** not. ^Changes to a view that are intercepted by INSTEAD OF triggers
 ** are not counted.
 **
 ** The [sqlite3_total_changes(D)] interface only reports the number
@@ -2681,7 +2864,7 @@ sqlite3_int64 sqlite3_changes64(sqlite3*);
 ** To detect changes against a database file from other database
 ** connections use the [PRAGMA data_version] command or the
 ** [SQLITE_FCNTL_DATA_VERSION] [file control].
-** 
+**
 ** If a separate thread makes changes on the same database connection
 ** while [sqlite3_total_changes()] is running then the value
 ** returned is unpredictable and not meaningful.
@@ -2724,7 +2907,7 @@ sqlite3_int64 sqlite3_total_changes64(sqlite3*);
 **
 ** ^The sqlite3_interrupt(D) call is in effect until all currently running
 ** SQL statements on [database connection] D complete.  ^Any new SQL statements
-** that are started after the sqlite3_interrupt() call and before the 
+** that are started after the sqlite3_interrupt() call and before the
 ** running statement count reaches zero are interrupted as if they had been
 ** running prior to the sqlite3_interrupt() call.  ^New SQL statements
 ** that are started after the running statement count reaches zero are
@@ -2735,6 +2918,7 @@ sqlite3_int64 sqlite3_total_changes64(sqlite3*);
 **
 ** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether
 ** or not an interrupt is currently in effect for [database connection] D.
+** It returns 1 if an interrupt is currently in effect, or 0 otherwise.
 */
 void sqlite3_interrupt(sqlite3*);
 int sqlite3_is_interrupted(sqlite3*);
@@ -2757,10 +2941,10 @@ int sqlite3_is_interrupted(sqlite3*);
 ** ^These routines return 0 if the statement is incomplete.  ^If a
 ** memory allocation fails, then SQLITE_NOMEM is returned.
 **
-** ^These routines do not parse the SQL statements thus
+** ^These routines do not parse the SQL statements and thus
 ** will not detect syntactically incorrect SQL.
 **
-** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior 
+** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior
 ** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked
 ** automatically by sqlite3_complete16().  If that initialization fails,
 ** then the return value from sqlite3_complete16() will be non-zero
@@ -2805,7 +2989,7 @@ int sqlite3_complete16(const void *sql);
 ** The presence of a busy handler does not guarantee that it will be invoked
 ** when there is lock contention. ^If SQLite determines that invoking the busy
 ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY]
-** to the application instead of invoking the 
+** to the application instead of invoking the
 ** busy handler.
 ** Consider a scenario where one process is holding a read lock that
 ** it is trying to promote to a reserved lock and
@@ -2830,7 +3014,7 @@ int sqlite3_complete16(const void *sql);
 ** database connection that invoked the busy handler.  In other words,
 ** the busy handler is not reentrant.  Any such actions
 ** result in undefined behavior.
-** 
+**
 ** A busy handler must not close the database connection
 ** or [prepared statement] that invoked the busy handler.
 */
@@ -2859,6 +3043,44 @@ int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*);
 */
 int sqlite3_busy_timeout(sqlite3*, int ms);
 
+/*
+** CAPI3REF: Set the Setlk Timeout
+** METHOD: sqlite3
+**
+** This routine is only useful in SQLITE_ENABLE_SETLK_TIMEOUT builds. If 
+** the VFS supports blocking locks, it sets the timeout in ms used by 
+** eligible locks taken on wal mode databases by the specified database 
+** handle. In non-SQLITE_ENABLE_SETLK_TIMEOUT builds, or if the VFS does
+** not support blocking locks, this function is a no-op.
+**
+** Passing 0 to this function disables blocking locks altogether. Passing 
+** -1 to this function requests that the VFS blocks for a long time - 
+** indefinitely if possible. The results of passing any other negative value 
+** are undefined.
+**
+** Internally, each SQLite database handle stores two timeout values - the
+** busy-timeout (used for rollback mode databases, or if the VFS does not
+** support blocking locks) and the setlk-timeout (used for blocking locks
+** on wal-mode databases). The sqlite3_busy_timeout() method sets both
+** values, this function sets only the setlk-timeout value. Therefore,
+** to configure separate busy-timeout and setlk-timeout values for a single
+** database handle, call sqlite3_busy_timeout() followed by this function.
+**
+** Whenever the number of connections to a wal mode database falls from
+** 1 to 0, the last connection takes an exclusive lock on the database, 
+** then checkpoints and deletes the wal file. While it is doing this, any
+** new connection that tries to read from the database fails with an
+** SQLITE_BUSY error. Or, if the SQLITE_SETLK_BLOCK_ON_CONNECT flag is
+** passed to this API, the new connection blocks until the exclusive lock
+** has been released.
+*/
+int sqlite3_setlk_timeout(sqlite3*, int ms, int flags);
+
+/*
+** CAPI3REF: Flags for sqlite3_setlk_timeout()
+*/
+#define SQLITE_SETLK_BLOCK_ON_CONNECT 0x01
+
 /*
 ** CAPI3REF: Convenience Routines For Running Queries
 ** METHOD: sqlite3
@@ -2866,7 +3088,7 @@ int sqlite3_busy_timeout(sqlite3*, int ms);
 ** This is a legacy interface that is preserved for backwards compatibility.
 ** Use of this interface is not recommended.
 **
-** Definition: A <b>result table</b> is memory data structure created by the
+** Definition: A <b>result table</b> is a memory data structure created by the
 ** [sqlite3_get_table()] interface.  A result table records the
 ** complete query results from one or more queries.
 **
@@ -2948,7 +3170,7 @@ void sqlite3_free_table(char **result);
 ** These routines are work-alikes of the "printf()" family of functions
 ** from the standard C library.
 ** These routines understand most of the common formatting options from
-** the standard library printf() 
+** the standard library printf()
 ** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]).
 ** See the [built-in printf()] documentation for details.
 **
@@ -3009,7 +3231,7 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** ^Calling sqlite3_free() with a pointer previously returned
 ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so
 ** that it might be reused.  ^The sqlite3_free() routine is
-** a no-op if is called with a NULL pointer.  Passing a NULL pointer
+** a no-op if it is called with a NULL pointer.  Passing a NULL pointer
 ** to sqlite3_free() is harmless.  After being freed, memory
 ** should neither be read nor written.  Even reading previously freed
 ** memory might result in a segmentation fault or other severe error.
@@ -3027,13 +3249,13 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** sqlite3_free(X).
 ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation
 ** of at least N bytes in size or NULL if insufficient memory is available.
-** ^If M is the size of the prior allocation, then min(N,M) bytes
-** of the prior allocation are copied into the beginning of buffer returned
+** ^If M is the size of the prior allocation, then min(N,M) bytes of the
+** prior allocation are copied into the beginning of the buffer returned
 ** by sqlite3_realloc(X,N) and the prior allocation is freed.
 ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the
 ** prior allocation is not freed.
 **
-** ^The sqlite3_realloc64(X,N) interfaces works the same as
+** ^The sqlite3_realloc64(X,N) interface works the same as
 ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead
 ** of a 32-bit signed integer.
 **
@@ -3083,7 +3305,7 @@ sqlite3_uint64 sqlite3_msize(void*);
 ** was last reset.  ^The values returned by [sqlite3_memory_used()] and
 ** [sqlite3_memory_highwater()] include any overhead
 ** added by SQLite in its implementation of [sqlite3_malloc()],
-** but not overhead added by the any underlying system library
+** but not overhead added by any underlying system library
 ** routines that [sqlite3_malloc()] may call.
 **
 ** ^The memory high-water mark is reset to the current value of
@@ -3144,7 +3366,7 @@ void sqlite3_randomness(int N, void *P);
 ** requested is ok.  ^When the callback returns [SQLITE_DENY], the
 ** [sqlite3_prepare_v2()] or equivalent call that triggered the
 ** authorizer will fail with an error message explaining that
-** access is denied. 
+** access is denied.
 **
 ** ^The first parameter to the authorizer callback is a copy of the third
 ** parameter to the sqlite3_set_authorizer() interface. ^The second parameter
@@ -3197,7 +3419,7 @@ void sqlite3_randomness(int N, void *P);
 ** database connections for the meaning of "modify" in this paragraph.
 **
 ** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the
-** statement might be re-prepared during [sqlite3_step()] due to a 
+** statement might be re-prepared during [sqlite3_step()] due to a
 ** schema change.  Hence, the application should ensure that the
 ** correct authorizer callback remains in place during the [sqlite3_step()].
 **
@@ -3284,8 +3506,8 @@ int sqlite3_set_authorizer(
 #define SQLITE_RECURSIVE            33   /* NULL            NULL            */
 
 /*
-** CAPI3REF: Tracing And Profiling Functions
-** METHOD: sqlite3
+** CAPI3REF: Deprecated Tracing And Profiling Functions
+** DEPRECATED
 **
 ** These routines are deprecated. Use the [sqlite3_trace_v2()] interface
 ** instead of the routines described here.
@@ -3388,10 +3610,12 @@ SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
 ** M argument should be the bitwise OR-ed combination of
 ** zero or more [SQLITE_TRACE] constants.
 **
-** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides 
-** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2().
+** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P)
+** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or
+** sqlite3_trace_v2(D,M,X,P) for the [database connection] D.  Each
+** database connection may have at most one trace callback.
 **
-** ^The X callback is invoked whenever any of the events identified by 
+** ^The X callback is invoked whenever any of the events identified by
 ** mask M occur.  ^The integer return value from the callback is currently
 ** ignored, though this may change in future releases.  Callback
 ** implementations should return zero to ensure future compatibility.
@@ -3423,8 +3647,8 @@ int sqlite3_trace_v2(
 ** database connection D.  An example use for this
 ** interface is to keep a GUI updated during a large query.
 **
-** ^The parameter P is passed through as the only parameter to the 
-** callback function X.  ^The parameter N is the approximate number of 
+** ^The parameter P is passed through as the only parameter to the
+** callback function X.  ^The parameter N is the approximate number of
 ** [virtual machine instructions] that are evaluated between successive
 ** invocations of the callback X.  ^If N is less than one then the progress
 ** handler is disabled.
@@ -3533,7 +3757,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** there is no harm in trying.)
 **
 ** ^(<dt>[SQLITE_OPEN_SHAREDCACHE]</dt>
-** <dd>The database is opened [shared cache] enabled, overriding
+** <dd>The database is opened with [shared cache] enabled, overriding
 ** the default shared cache setting provided by
 ** [sqlite3_enable_shared_cache()].)^
 ** The [use of shared cache mode is discouraged] and hence shared cache
@@ -3541,14 +3765,14 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** this option is a no-op.
 **
 ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt>
-** <dd>The database is opened [shared cache] disabled, overriding
+** <dd>The database is opened with [shared cache] disabled, overriding
 ** the default shared cache setting provided by
 ** [sqlite3_enable_shared_cache()].)^
 **
 ** [[OPEN_EXRESCODE]] ^(<dt>[SQLITE_OPEN_EXRESCODE]</dt>
 ** <dd>The database connection comes up in "extended result code mode".
-** In other words, the database behaves has if
-** [sqlite3_extended_result_codes(db,1)] where called on the database
+** In other words, the database behaves as if
+** [sqlite3_extended_result_codes(db,1)] were called on the database
 ** connection as soon as the connection is created. In addition to setting
 ** the extended result code mode, this flag also causes [sqlite3_open_v2()]
 ** to return an extended result code.</dd>
@@ -3601,17 +3825,17 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** information.
 **
 ** URI filenames are parsed according to RFC 3986. ^If the URI contains an
-** authority, then it must be either an empty string or the string 
-** "localhost". ^If the authority is not an empty string or "localhost", an 
-** error is returned to the caller. ^The fragment component of a URI, if 
+** authority, then it must be either an empty string or the string
+** "localhost". ^If the authority is not an empty string or "localhost", an
+** error is returned to the caller. ^The fragment component of a URI, if
 ** present, is ignored.
 **
 ** ^SQLite uses the path component of the URI as the name of the disk file
-** which contains the database. ^If the path begins with a '/' character, 
-** then it is interpreted as an absolute path. ^If the path does not begin 
+** which contains the database. ^If the path begins with a '/' character,
+** then it is interpreted as an absolute path. ^If the path does not begin
 ** with a '/' (meaning that the authority section is omitted from the URI)
-** then the path is interpreted as a relative path. 
-** ^(On windows, the first component of an absolute path 
+** then the path is interpreted as a relative path.
+** ^(On windows, the first component of an absolute path
 ** is a drive specification (e.g. "C:").)^
 **
 ** [[core URI query parameters]]
@@ -3631,13 +3855,13 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **
 **   <li> <b>mode</b>: ^(The mode parameter may be set to either "ro", "rw",
 **     "rwc", or "memory". Attempting to set it to any other value is
-**     an error)^. 
-**     ^If "ro" is specified, then the database is opened for read-only 
-**     access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the 
-**     third argument to sqlite3_open_v2(). ^If the mode option is set to 
-**     "rw", then the database is opened for read-write (but not create) 
-**     access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had 
-**     been set. ^Value "rwc" is equivalent to setting both 
+**     an error)^.
+**     ^If "ro" is specified, then the database is opened for read-only
+**     access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the
+**     third argument to sqlite3_open_v2(). ^If the mode option is set to
+**     "rw", then the database is opened for read-write (but not create)
+**     access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had
+**     been set. ^Value "rwc" is equivalent to setting both
 **     SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE.  ^If the mode option is
 **     set to "memory" then a pure [in-memory database] that never reads
 **     or writes from disk is used. ^It is an error to specify a value for
@@ -3647,7 +3871,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **   <li> <b>cache</b>: ^The cache parameter may be set to either "shared" or
 **     "private". ^Setting it to "shared" is equivalent to setting the
 **     SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to
-**     sqlite3_open_v2(). ^Setting the cache parameter to "private" is 
+**     sqlite3_open_v2(). ^Setting the cache parameter to "private" is
 **     equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit.
 **     ^If sqlite3_open_v2() is used and the "cache" parameter is present in
 **     a URI filename, its value overrides any behavior requested by setting
@@ -3673,7 +3897,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **     property on a database file that does in fact change can result
 **     in incorrect query results and/or [SQLITE_CORRUPT] errors.
 **     See also: [SQLITE_IOCAP_IMMUTABLE].
-**       
+**      
 ** </ul>
 **
 ** ^Specifying an unknown parameter in the query component of a URI is not an
@@ -3685,37 +3909,37 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **
 ** <table border="1" align=center cellpadding=5>
 ** <tr><th> URI filenames <th> Results
-** <tr><td> file:data.db <td> 
+** <tr><td> file:data.db <td>
 **          Open the file "data.db" in the current directory.
 ** <tr><td> file:/home/fred/data.db<br>
-**          file:///home/fred/data.db <br> 
-**          file://localhost/home/fred/data.db <br> <td> 
+**          file:///home/fred/data.db <br>
+**          file://localhost/home/fred/data.db <br> <td>
 **          Open the database file "/home/fred/data.db".
-** <tr><td> file://darkstar/home/fred/data.db <td> 
+** <tr><td> file://darkstar/home/fred/data.db <td>
 **          An error. "darkstar" is not a recognized authority.
-** <tr><td style="white-space:nowrap"> 
+** <tr><td style="white-space:nowrap">
 **          file:///C:/Documents%20and%20Settings/fred/Desktop/data.db
 **     <td> Windows only: Open the file "data.db" on fred's desktop on drive
-**          C:. Note that the %20 escaping in this example is not strictly 
+**          C:. Note that the %20 escaping in this example is not strictly
 **          necessary - space characters can be used literally
 **          in URI filenames.
-** <tr><td> file:data.db?mode=ro&cache=private <td> 
+** <tr><td> file:data.db?mode=ro&cache=private <td>
 **          Open file "data.db" in the current directory for read-only access.
 **          Regardless of whether or not shared-cache mode is enabled by
 **          default, use a private cache.
 ** <tr><td> file:/home/fred/data.db?vfs=unix-dotfile <td>
 **          Open file "/home/fred/data.db". Use the special VFS "unix-dotfile"
 **          that uses dot-files in place of posix advisory locking.
-** <tr><td> file:data.db?mode=readonly <td> 
+** <tr><td> file:data.db?mode=readonly <td>
 **          An error. "readonly" is not a valid option for the "mode" parameter.
 **          Use "ro" instead:  "file:data.db?mode=ro".
 ** </table>
 **
 ** ^URI hexadecimal escape sequences (%HH) are supported within the path and
 ** query components of a URI. A hexadecimal escape sequence consists of a
-** percent sign - "%" - followed by exactly two hexadecimal digits 
+** percent sign - "%" - followed by exactly two hexadecimal digits
 ** specifying an octet value. ^Before the path or query components of a
-** URI filename are interpreted, they are encoded using UTF-8 and all 
+** URI filename are interpreted, they are encoded using UTF-8 and all
 ** hexadecimal escape sequences replaced by a single byte containing the
 ** corresponding octet. If this process generates an invalid UTF-8 encoding,
 ** the results are undefined.
@@ -3751,14 +3975,14 @@ int sqlite3_open_v2(
 ** CAPI3REF: Obtain Values For URI Parameters
 **
 ** These are utility routines, useful to [VFS|custom VFS implementations],
-** that check if a database file was a URI that contained a specific query 
+** that check if a database file was a URI that contained a specific query
 ** parameter, and if so obtains the value of that query parameter.
 **
 ** The first parameter to these interfaces (hereafter referred to
 ** as F) must be one of:
 ** <ul>
 ** <li> A database filename pointer created by the SQLite core and
-** passed into the xOpen() method of a VFS implemention, or
+** passed into the xOpen() method of a VFS implementation, or
 ** <li> A filename obtained from [sqlite3_db_filename()], or
 ** <li> A new filename constructed using [sqlite3_create_filename()].
 ** </ul>
@@ -3769,7 +3993,7 @@ int sqlite3_open_v2(
 ** If F is a suitable filename (as described in the previous paragraph)
 ** and if P is the name of the query parameter, then
 ** sqlite3_uri_parameter(F,P) returns the value of the P
-** parameter if it exists or a NULL pointer if P does not appear as a 
+** parameter if it exists or a NULL pointer if P does not appear as a
 ** query parameter on F.  If P is a query parameter of F and it
 ** has no explicit value, then sqlite3_uri_parameter(F,P) returns
 ** a pointer to an empty string.
@@ -3778,7 +4002,7 @@ int sqlite3_open_v2(
 ** parameter and returns true (1) or false (0) according to the value
 ** of P.  The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the
 ** value of query parameter P is one of "yes", "true", or "on" in any
-** case or if the value begins with a non-zero number.  The 
+** case or if the value begins with a non-zero number.  The
 ** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of
 ** query parameter P is one of "no", "false", or "off" in any case or
 ** if the value begins with a numeric zero.  If P is not a query
@@ -3796,7 +4020,7 @@ int sqlite3_open_v2(
 ** parameters minus 1.  The N value is zero-based so N should be 0 to obtain
 ** the name of the first query parameter, 1 for the second parameter, and
 ** so forth.
-** 
+**
 ** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and
 ** sqlite3_uri_boolean(F,P,B) returns B.  If F is not a NULL pointer and
 ** is not a database file pathname pointer that the SQLite core passed
@@ -3853,14 +4077,14 @@ const char *sqlite3_filename_wal(sqlite3_filename);
 ** CAPI3REF:  Database File Corresponding To A Journal
 **
 ** ^If X is the name of a rollback or WAL-mode journal file that is
-** passed into the xOpen method of [sqlite3_vfs], then 
+** passed into the xOpen method of [sqlite3_vfs], then
 ** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file]
 ** object that represents the main database file.
 **
 ** This routine is intended for use in custom [VFS] implementations
 ** only.  It is not a general-purpose interface.
 ** The argument sqlite3_file_object(X) must be a filename pointer that
-** has been passed into [sqlite3_vfs].xOpen method where the 
+** has been passed into [sqlite3_vfs].xOpen method where the
 ** flags parameter to xOpen contains one of the bits
 ** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL].  Any other use
 ** of this routine results in undefined and probably undesirable
@@ -3871,19 +4095,19 @@ sqlite3_file *sqlite3_database_file_object(const char*);
 /*
 ** CAPI3REF: Create and Destroy VFS Filenames
 **
-** These interfces are provided for use by [VFS shim] implementations and
+** These interfaces are provided for use by [VFS shim] implementations and
 ** are not useful outside of that context.
 **
 ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of
 ** database filename D with corresponding journal file J and WAL file W and
-** with N URI parameters key/values pairs in the array P.  The result from
+** an array P of N URI Key/Value pairs.  The result from
 ** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that
 ** is safe to pass to routines like:
 ** <ul>
 ** <li> [sqlite3_uri_parameter()],
 ** <li> [sqlite3_uri_boolean()],
 ** <li> [sqlite3_uri_int64()],
-** <li> [sqlite3_uri_key()], 
+** <li> [sqlite3_uri_key()],
 ** <li> [sqlite3_filename_database()],
 ** <li> [sqlite3_filename_journal()], or
 ** <li> [sqlite3_filename_wal()].
@@ -3907,7 +4131,7 @@ sqlite3_file *sqlite3_database_file_object(const char*);
 ** If the Y parameter to sqlite3_free_filename(Y) is anything other
 ** than a NULL pointer or a pointer previously acquired from
 ** sqlite3_create_filename(), then bad things such as heap
-** corruption or segfaults may occur. The value Y should not be 
+** corruption or segfaults may occur. The value Y should not be
 ** used again after sqlite3_free_filename(Y) has been called.  This means
 ** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y,
 ** then the corresponding [sqlite3_module.xClose() method should also be
@@ -3926,12 +4150,12 @@ void sqlite3_free_filename(sqlite3_filename);
 ** CAPI3REF: Error Codes And Messages
 ** METHOD: sqlite3
 **
-** ^If the most recent sqlite3_* API call associated with 
+** ^If the most recent sqlite3_* API call associated with
 ** [database connection] D failed, then the sqlite3_errcode(D) interface
 ** returns the numeric [result code] or [extended result code] for that
 ** API call.
 ** ^The sqlite3_extended_errcode()
-** interface is the same except that it always returns the 
+** interface is the same except that it always returns the
 ** [extended result code] even when extended result codes are
 ** disabled.
 **
@@ -3950,21 +4174,24 @@ void sqlite3_free_filename(sqlite3_filename);
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
-** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** text that describes the error, as either UTF-8 or UTF-16 respectively,
+** or NULL if no error message is available.
+** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
 ** subsequent calls to other SQLite interface functions.)^
 **
-** ^The sqlite3_errstr() interface returns the English-language text
-** that describes the [result code], as UTF-8.
+** ^The sqlite3_errstr(E) interface returns the English-language text
+** that describes the [result code] E, as UTF-8, or NULL if E is not a
+** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
 ** ^If the most recent error references a specific token in the input
 ** SQL, the sqlite3_error_offset() interface returns the byte offset
 ** of the start of that token.  ^The byte offset returned by
-** sqlite3_error_offset() assumes that the input SQL is UTF8.
+** sqlite3_error_offset() assumes that the input SQL is UTF-8.
 ** ^If the most recent error does not reference a specific token in the input
 ** SQL, then the sqlite3_error_offset() function returns -1.
 **
@@ -3989,6 +4216,34 @@ const void *sqlite3_errmsg16(sqlite3*);
 const char *sqlite3_errstr(int);
 int sqlite3_error_offset(sqlite3 *db);
 
+/*
+** CAPI3REF: Set Error Codes And Message
+** METHOD: sqlite3
+**
+** Set the error code of the database handle passed as the first argument
+** to errcode, and the error message to a copy of nul-terminated string
+** zErrMsg. If zErrMsg is passed NULL, then the error message is set to
+** the default message associated with the supplied error code.  Subsequent
+** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will
+** return the values set by this routine in place of what was previously
+** set by SQLite itself.
+**
+** This function returns SQLITE_OK if the error code and error message are
+** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if
+** the database handle is NULL or invalid.
+**
+** The error code and message set by this routine remains in effect until
+** they are changed, either by another call to this routine or until they are
+** changed to by SQLite itself to reflect the result of some subsquent
+** API call.
+**
+** This function is intended for use by SQLite extensions or wrappers.  The
+** idea is that an extension or wrapper can use this routine to set error
+** messages and error codes and thus behave more like a core SQLite
+** feature from the point of view of an application.
+*/
+int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg);
+
 /*
 ** CAPI3REF: Prepared Statement Object
 ** KEYWORDS: {prepared statement} {prepared statements}
@@ -3997,7 +4252,7 @@ int sqlite3_error_offset(sqlite3 *db);
 ** has been compiled into binary form and is ready to be evaluated.
 **
 ** Think of each SQL statement as a separate computer program.  The
-** original SQL text is source code.  A prepared statement object 
+** original SQL text is source code.  A prepared statement object
 ** is the compiled object code.  All SQL must be converted into a
 ** prepared statement before it can be run.
 **
@@ -4027,7 +4282,7 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 ** new limit for that construct.)^
 **
 ** ^If the new limit is a negative number, the limit is unchanged.
-** ^(For each limit category SQLITE_LIMIT_<i>NAME</i> there is a 
+** ^(For each limit category SQLITE_LIMIT_<i>NAME</i> there is a
 ** [limits | hard upper bound]
 ** set at compile-time by a C preprocessor macro called
 ** [limits | SQLITE_MAX_<i>NAME</i>].
@@ -4035,7 +4290,7 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 ** ^Attempts to increase a limit above its hard upper bound are
 ** silently truncated to the hard upper bound.
 **
-** ^Regardless of whether or not the limit was changed, the 
+** ^Regardless of whether or not the limit was changed, the
 ** [sqlite3_limit()] interface returns the prior value of the limit.
 ** ^Hence, to find the current value of a limit without changing it,
 ** simply invoke this interface with the third parameter set to -1.
@@ -4063,8 +4318,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 **
 ** These constants define various performance limits
 ** that can be lowered at run-time using [sqlite3_limit()].
-** The synopsis of the meanings of the various limits is shown below.
-** Additional information is available at [limits | Limits in SQLite].
+** A concise description of these limits follows, and additional information
+** is available at [limits | Limits in SQLite].
 **
 ** <dl>
 ** [[SQLITE_LIMIT_LENGTH]] ^(<dt>SQLITE_LIMIT_LENGTH</dt>
@@ -4129,7 +4384,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 /*
 ** CAPI3REF: Prepare Flags
 **
-** These constants define various flags that can be passed into
+** These constants define various flags that can be passed into the
 ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and
 ** [sqlite3_prepare16_v3()] interfaces.
 **
@@ -4140,7 +4395,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** <dd>The SQLITE_PREPARE_PERSISTENT flag is a hint to the query planner
 ** that the prepared statement will be retained for a long time and
 ** probably reused many times.)^ ^Without this flag, [sqlite3_prepare_v3()]
-** and [sqlite3_prepare16_v3()] assume that the prepared statement will 
+** and [sqlite3_prepare16_v3()] assume that the prepared statement will
 ** be used just once or at most a few times and then destroyed using
 ** [sqlite3_finalize()] relatively soon. The current implementation acts
 ** on this hint by avoiding the use of [lookaside memory] so as not to
@@ -4159,11 +4414,22 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** <dd>The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler
 ** to return an error (error code SQLITE_ERROR) if the statement uses
 ** any virtual tables.
+**
+** [[SQLITE_PREPARE_DONT_LOG]] <dt>SQLITE_PREPARE_DONT_LOG</dt>
+** <dd>The SQLITE_PREPARE_DONT_LOG flag prevents SQL compiler
+** errors from being sent to the error log defined by
+** [SQLITE_CONFIG_LOG].  This can be used, for example, to do test
+** compiles to see if some SQL syntax is well-formed, without generating
+** messages on the global error log when it is not.  If the test compile
+** fails, the sqlite3_prepare_v3() call returns the same error indications
+** with or without this flag; it just omits the call to [sqlite3_log()] that
+** logs the error.
 ** </dl>
 */
 #define SQLITE_PREPARE_PERSISTENT              0x01
 #define SQLITE_PREPARE_NORMALIZE               0x02
 #define SQLITE_PREPARE_NO_VTAB                 0x04
+#define SQLITE_PREPARE_DONT_LOG                0x10
 
 /*
 ** CAPI3REF: Compiling An SQL Statement
@@ -4196,13 +4462,17 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** and sqlite3_prepare16_v3() use UTF-16.
 **
 ** ^If the nByte argument is negative, then zSql is read up to the
-** first zero terminator. ^If nByte is positive, then it is the
-** number of bytes read from zSql.  ^If nByte is zero, then no prepared
+** first zero terminator. ^If nByte is positive, then it is the maximum
+** number of bytes read from zSql.  When nByte is positive, zSql is read
+** up to the first zero terminator or until the nByte bytes have been read,
+** whichever comes first.  ^If nByte is zero, then no prepared
 ** statement is generated.
 ** If the caller knows that the supplied string is nul-terminated, then
 ** there is a small performance advantage to passing an nByte parameter that
 ** is the number of bytes in the input string <i>including</i>
 ** the nul-terminator.
+** Note that nByte measures the length of the input in bytes, not
+** characters, even for the UTF-16 interfaces.
 **
 ** ^If pzTail is not NULL then *pzTail is made to point to the first byte
 ** past the end of the first SQL statement in zSql.  These routines only
@@ -4247,12 +4517,12 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** </li>
 **
 ** <li>
-** ^If the specific value bound to a [parameter | host parameter] in the 
+** ^If the specific value bound to a [parameter | host parameter] in the
 ** WHERE clause might influence the choice of query plan for a statement,
-** then the statement will be automatically recompiled, as if there had been 
+** then the statement will be automatically recompiled, as if there had been
 ** a schema change, on the first [sqlite3_step()] call following any change
-** to the [sqlite3_bind_text | bindings] of that [parameter]. 
-** ^The specific value of a WHERE-clause [parameter] might influence the 
+** to the [sqlite3_bind_text | bindings] of that [parameter].
+** ^The specific value of a WHERE-clause [parameter] might influence the
 ** choice of query plan if the parameter is the left-hand side of a [LIKE]
 ** or [GLOB] operator or if the parameter is compared to an indexed column
 ** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled.
@@ -4335,7 +4605,7 @@ int sqlite3_prepare16_v3(
 **
 ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory
 ** is available to hold the result, or if the result would exceed the
-** the maximum string length determined by the [SQLITE_LIMIT_LENGTH].
+** maximum string length determined by the [SQLITE_LIMIT_LENGTH].
 **
 ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of
 ** bound parameter expansions.  ^The [SQLITE_OMIT_TRACE] compile-time
@@ -4366,8 +4636,8 @@ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
 ** the content of the database file.
 **
 ** Note that [application-defined SQL functions] or
-** [virtual tables] might change the database indirectly as a side effect.  
-** ^(For example, if an application defines a function "eval()" that 
+** [virtual tables] might change the database indirectly as a side effect. 
+** ^(For example, if an application defines a function "eval()" that
 ** calls [sqlite3_exec()], then the following SQL statement would
 ** change the database file through side-effects:
 **
@@ -4381,10 +4651,10 @@ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
 ** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK],
 ** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true,
 ** since the statements themselves do not actually modify the database but
-** rather they control the timing of when other statements modify the 
+** rather they control the timing of when other statements modify the
 ** database.  ^The [ATTACH] and [DETACH] statements also cause
 ** sqlite3_stmt_readonly() to return true since, while those statements
-** change the configuration of a database connection, they do not make 
+** change the configuration of a database connection, they do not make
 ** changes to the content of the database files on disk.
 ** ^The sqlite3_stmt_readonly() interface returns true for [BEGIN] since
 ** [BEGIN] merely sets internal flags, but the [BEGIN|BEGIN IMMEDIATE] and
@@ -4397,7 +4667,7 @@ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
 ** ^For example, an UPDATE statement might have a WHERE clause that
 ** makes it a no-op, but the sqlite3_stmt_readonly() result would still
 ** be false.  ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a
-** read-only no-op if the table already exists, but 
+** read-only no-op if the table already exists, but
 ** sqlite3_stmt_readonly() still returns false for such a statement.
 **
 ** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN]
@@ -4418,23 +4688,58 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 */
 int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt);
 
+/*
+** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement
+** METHOD: sqlite3_stmt
+**
+** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
+** setting for [prepared statement] S.  If E is zero, then S becomes
+** a normal prepared statement.  If E is 1, then S behaves as if
+** its SQL text began with "[EXPLAIN]".  If E is 2, then S behaves as if
+** its SQL text began with "[EXPLAIN QUERY PLAN]".
+**
+** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared.
+** SQLite tries to avoid a reprepare, but a reprepare might be necessary
+** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode.
+**
+** Because of the potential need to reprepare, a call to
+** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be
+** reprepared because it was created using [sqlite3_prepare()] instead of
+** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and
+** hence has no saved SQL text with which to reprepare.
+**
+** Changing the explain setting for a prepared statement does not change
+** the original SQL text for the statement.  Hence, if the SQL text originally
+** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0)
+** is called to convert the statement into an ordinary statement, the EXPLAIN
+** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S)
+** output, even though the statement now acts like a normal SQL statement.
+**
+** This routine returns SQLITE_OK if the explain mode is successfully
+** changed, or an error code if the explain mode could not be changed.
+** The explain mode cannot be changed while a statement is active.
+** Hence, it is good practice to call [sqlite3_reset(S)]
+** immediately prior to calling sqlite3_stmt_explain(S,E).
+*/
+int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode);
+
 /*
 ** CAPI3REF: Determine If A Prepared Statement Has Been Reset
 ** METHOD: sqlite3_stmt
 **
 ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the
-** [prepared statement] S has been stepped at least once using 
+** [prepared statement] S has been stepped at least once using
 ** [sqlite3_step(S)] but has neither run to completion (returned
 ** [SQLITE_DONE] from [sqlite3_step(S)]) nor
 ** been reset using [sqlite3_reset(S)].  ^The sqlite3_stmt_busy(S)
-** interface returns false if S is a NULL pointer.  If S is not a 
+** interface returns false if S is a NULL pointer.  If S is not a
 ** NULL pointer and is not a pointer to a valid [prepared statement]
 ** object, then the behavior is undefined and probably undesirable.
 **
 ** This interface can be used in combination [sqlite3_next_stmt()]
-** to locate all prepared statements associated with a database 
+** to locate all prepared statements associated with a database
 ** connection that are in need of being reset.  This can be used,
-** for example, in diagnostic routines to search for prepared 
+** for example, in diagnostic routines to search for prepared
 ** statements that are holding a transaction open.
 */
 int sqlite3_stmt_busy(sqlite3_stmt*);
@@ -4453,7 +4758,7 @@ int sqlite3_stmt_busy(sqlite3_stmt*);
 ** will accept either a protected or an unprotected sqlite3_value.
 ** Every interface that accepts sqlite3_value arguments specifies
 ** whether or not it requires a protected sqlite3_value.  The
-** [sqlite3_value_dup()] interface can be used to construct a new 
+** [sqlite3_value_dup()] interface can be used to construct a new
 ** protected sqlite3_value from an unprotected sqlite3_value.
 **
 ** The terms "protected" and "unprotected" refer to whether or not
@@ -4461,7 +4766,7 @@ int sqlite3_stmt_busy(sqlite3_stmt*);
 ** sqlite3_value object but no mutex is held for an unprotected
 ** sqlite3_value object.  If SQLite is compiled to be single-threaded
 ** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0)
-** or if SQLite is run in one of reduced mutex modes 
+** or if SQLite is run in one of reduced mutex modes
 ** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD]
 ** then there is no distinction between protected and unprotected
 ** sqlite3_value objects and they can be used interchangeably.  However,
@@ -4488,7 +4793,7 @@ typedef struct sqlite3_value sqlite3_value;
 **
 ** The context in which an SQL function executes is stored in an
 ** sqlite3_context object.  ^A pointer to an sqlite3_context object
-** is always first parameter to [application-defined SQL functions].
+** is always the first parameter to [application-defined SQL functions].
 ** The application-defined SQL function implementation will pass this
 ** pointer through into calls to [sqlite3_result_int | sqlite3_result()],
 ** [sqlite3_aggregate_context()], [sqlite3_user_data()],
@@ -4504,7 +4809,7 @@ typedef struct sqlite3_context sqlite3_context;
 ** METHOD: sqlite3_stmt
 **
 ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants,
-** literals may be replaced by a [parameter] that matches one of following
+** literals may be replaced by a [parameter] that matches one of the following
 ** templates:
 **
 ** <ul>
@@ -4549,10 +4854,10 @@ typedef struct sqlite3_context sqlite3_context;
 **
 ** [[byte-order determination rules]] ^The byte-order of
 ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF)
-** found in first character, which is removed, or in the absence of a BOM
+** found in the first character, which is removed, or in the absence of a BOM
 ** the byte order is the native byte order of the host
 ** machine for sqlite3_bind_text16() or the byte order specified in
-** the 6th parameter for sqlite3_bind_text64().)^ 
+** the 6th parameter for sqlite3_bind_text64().)^
 ** ^If UTF16 input text contains invalid unicode
 ** characters, then SQLite might change those invalid characters
 ** into the unicode replacement character: U+FFFD.
@@ -4569,7 +4874,7 @@ typedef struct sqlite3_context sqlite3_context;
 ** or sqlite3_bind_text16() or sqlite3_bind_text64() then
 ** that parameter must be the byte offset
 ** where the NUL terminator would occur assuming the string were NUL
-** terminated.  If any NUL characters occurs at byte offsets less than 
+** terminated.  If any NUL characters occur at byte offsets less than
 ** the value of the fourth parameter then the resulting string value will
 ** contain embedded NULs.  The result of expressions involving strings
 ** with embedded NULs is undefined.
@@ -4581,7 +4886,7 @@ typedef struct sqlite3_context sqlite3_context;
 ** with it may be passed. ^It is called to dispose of the BLOB or string even
 ** if the call to the bind API fails, except the destructor is not called if
 ** the third parameter is a NULL pointer or the fourth parameter is negative.
-** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that
+** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that
 ** the application remains responsible for disposing of the object. ^In this
 ** case, the object and the provided pointer to it must remain valid until
 ** either the prepared statement is finalized or the same SQL parameter is
@@ -4612,9 +4917,11 @@ typedef struct sqlite3_context sqlite3_context;
 ** associated with the pointer P of type T.  ^D is either a NULL pointer or
 ** a pointer to a destructor function for P. ^SQLite will invoke the
 ** destructor D with a single argument of P when it is finished using
-** P.  The T parameter should be a static string, preferably a string
-** literal. The sqlite3_bind_pointer() routine is part of the
-** [pointer passing interface] added for SQLite 3.20.0.
+** P, even if the call to sqlite3_bind_pointer() fails.  Due to a
+** historical design quirk, results are undefined if D is
+** SQLITE_TRANSIENT. The T parameter should be a static string,
+** preferably a string literal. The sqlite3_bind_pointer() routine is
+** part of the [pointer passing interface] added for SQLite 3.20.0.
 **
 ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer
 ** for the [prepared statement] or with a prepared statement for which
@@ -4735,7 +5042,7 @@ int sqlite3_clear_bindings(sqlite3_stmt*);
 ** METHOD: sqlite3_stmt
 **
 ** ^Return the number of columns in the result set returned by the
-** [prepared statement]. ^If this routine returns 0, that means the 
+** [prepared statement]. ^If this routine returns 0, that means the
 ** [prepared statement] returns no data (for example an [UPDATE]).
 ** ^However, just because this routine returns a positive number does not
 ** mean that one or more rows of data will be returned.  ^A SELECT statement
@@ -4781,7 +5088,7 @@ const void *sqlite3_column_name16(sqlite3_stmt*, int N);
 ** METHOD: sqlite3_stmt
 **
 ** ^These routines provide a means to determine the database, table, and
-** table column that is the origin of a particular result column in
+** table column that is the origin of a particular result column in a
 ** [SELECT] statement.
 ** ^The name of the database or table or column can be returned as
 ** either a UTF-8 or UTF-16 string.  ^The _database_ routines return
@@ -4917,9 +5224,9 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 ** For all versions of SQLite up to and including 3.6.23.1, a call to
 ** [sqlite3_reset()] was required after sqlite3_step() returned anything
 ** other than [SQLITE_ROW] before any subsequent invocation of
-** sqlite3_step().  Failure to reset the prepared statement using 
+** sqlite3_step().  Failure to reset the prepared statement using
 ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from
-** sqlite3_step().  But after [version 3.6.23.1] ([dateof:3.6.23.1],
+** sqlite3_step().  But after [version 3.6.23.1] ([dateof:3.6.23.1]),
 ** sqlite3_step() began
 ** calling [sqlite3_reset()] automatically in this circumstance rather
 ** than returning [SQLITE_MISUSE].  This is not considered a compatibility
@@ -5008,7 +5315,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** <tr><td><b>sqlite3_column_int64</b><td>&rarr;<td>64-bit INTEGER result
 ** <tr><td><b>sqlite3_column_text</b><td>&rarr;<td>UTF-8 TEXT result
 ** <tr><td><b>sqlite3_column_text16</b><td>&rarr;<td>UTF-16 TEXT result
-** <tr><td><b>sqlite3_column_value</b><td>&rarr;<td>The result as an 
+** <tr><td><b>sqlite3_column_value</b><td>&rarr;<td>The result as an
 ** [sqlite3_value|unprotected sqlite3_value] object.
 ** <tr><td>&nbsp;<td>&nbsp;<td>&nbsp;
 ** <tr><td><b>sqlite3_column_bytes</b><td>&rarr;<td>Size of a BLOB
@@ -5056,7 +5363,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** The return value of sqlite3_column_type() can be used to decide which
 ** of the first six interface should be used to extract the column value.
 ** The value returned by sqlite3_column_type() is only meaningful if no
-** automatic type conversions have occurred for the value in question.  
+** automatic type conversions have occurred for the value in question. 
 ** After a type conversion, the result of calling sqlite3_column_type()
 ** is undefined, though harmless.  Future
 ** versions of SQLite may change the behavior of sqlite3_column_type()
@@ -5084,7 +5391,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** the number of bytes in that string.
 ** ^If the result is NULL, then sqlite3_column_bytes16() returns zero.
 **
-** ^The values returned by [sqlite3_column_bytes()] and 
+** ^The values returned by [sqlite3_column_bytes()] and
 ** [sqlite3_column_bytes16()] do not include the zero terminators at the end
 ** of the string.  ^For clarity: the values returned by
 ** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of
@@ -5107,7 +5414,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** to routines like [sqlite3_value_int()], [sqlite3_value_text()],
 ** or [sqlite3_value_bytes()], the behavior is not threadsafe.
 ** Hence, the sqlite3_column_value() interface
-** is normally only useful within the implementation of 
+** is normally only useful within the implementation of
 ** [application-defined SQL functions] or [virtual tables], not within
 ** top-level application code.
 **
@@ -5225,7 +5532,7 @@ int sqlite3_column_type(sqlite3_stmt*, int iCol);
 **
 ** ^The sqlite3_finalize() function is called to delete a [prepared statement].
 ** ^If the most recent evaluation of the statement encountered no errors
-** or if the statement is never been evaluated, then sqlite3_finalize() returns
+** or if the statement has never been evaluated, then sqlite3_finalize() returns
 ** SQLITE_OK.  ^If the most recent evaluation of statement S failed, then
 ** sqlite3_finalize(S) returns the appropriate [error code] or
 ** [extended error code].
@@ -5260,20 +5567,33 @@ int sqlite3_finalize(sqlite3_stmt *pStmt);
 ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S
 ** back to the beginning of its program.
 **
-** ^If the most recent call to [sqlite3_step(S)] for the
-** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE],
-** or if [sqlite3_step(S)] has never before been called on S,
-** then [sqlite3_reset(S)] returns [SQLITE_OK].
+** ^The return code from [sqlite3_reset(S)] indicates whether or not
+** the previous evaluation of prepared statement S completed successfully.
+** ^If [sqlite3_step(S)] has never before been called on S or if
+** [sqlite3_step(S)] has not been called since the previous call
+** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return
+** [SQLITE_OK].
 **
 ** ^If the most recent call to [sqlite3_step(S)] for the
 ** [prepared statement] S indicated an error, then
 ** [sqlite3_reset(S)] returns an appropriate [error code].
+** ^The [sqlite3_reset(S)] interface might also return an [error code]
+** if there were no prior errors but the process of resetting
+** the prepared statement caused a new error. ^For example, if an
+** [INSERT] statement with a [RETURNING] clause is only stepped one time,
+** that one call to [sqlite3_step(S)] might return SQLITE_ROW but
+** the overall statement might still fail and the [sqlite3_reset(S)] call
+** might return SQLITE_BUSY if locking constraints prevent the
+** database change from committing.  Therefore, it is important that
+** applications check the return code from [sqlite3_reset(S)] even if
+** no prior call to [sqlite3_step(S)] indicated a problem.
 **
 ** ^The [sqlite3_reset(S)] interface does not change the values
 ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S.
 */
 int sqlite3_reset(sqlite3_stmt *pStmt);
 
+
 /*
 ** CAPI3REF: Create Or Redefine SQL Functions
 ** KEYWORDS: {function creation routines}
@@ -5282,8 +5602,8 @@ int sqlite3_reset(sqlite3_stmt *pStmt);
 ** ^These functions (collectively known as "function creation routines")
 ** are used to add SQL functions or aggregates or to redefine the behavior
 ** of existing SQL functions or aggregates. The only differences between
-** the three "sqlite3_create_function*" routines are the text encoding 
-** expected for the second parameter (the name of the function being 
+** the three "sqlite3_create_function*" routines are the text encoding
+** expected for the second parameter (the name of the function being
 ** created) and the presence or absence of a destructor callback for
 ** the application data pointer. Function sqlite3_create_window_function()
 ** is similar, but allows the user to supply the extra callback functions
@@ -5297,7 +5617,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt);
 ** ^The second parameter is the name of the SQL function to be created or
 ** redefined.  ^The length of the name is limited to 255 bytes in a UTF-8
 ** representation, exclusive of the zero-terminator.  ^Note that the name
-** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes.  
+** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. 
 ** ^Any attempt to create a function with a longer name
 ** will result in [SQLITE_MISUSE] being returned.
 **
@@ -5312,7 +5632,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt);
 ** ^The fourth parameter, eTextRep, specifies what
 ** [SQLITE_UTF8 | text encoding] this SQL function prefers for
 ** its parameters.  The application should set this parameter to
-** [SQLITE_UTF16LE] if the function implementation invokes 
+** [SQLITE_UTF16LE] if the function implementation invokes
 ** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the
 ** implementation invokes [sqlite3_value_text16be()] on an input, or
 ** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8]
@@ -5337,8 +5657,8 @@ int sqlite3_reset(sqlite3_stmt *pStmt);
 **
 ** For best security, the [SQLITE_DIRECTONLY] flag is recommended for
 ** all application-defined SQL functions that do not need to be
-** used inside of triggers, view, CHECK constraints, or other elements of
-** the database schema.  This flags is especially recommended for SQL 
+** used inside of triggers, views, CHECK constraints, or other elements of
+** the database schema.  This flag is especially recommended for SQL
 ** functions that have side effects or reveal internal application state.
 ** Without this flag, an attacker might be able to modify the schema of
 ** a database file to include invocations of the function with parameters
@@ -5358,21 +5678,21 @@ int sqlite3_reset(sqlite3_stmt *pStmt);
 ** SQL function or aggregate, pass NULL pointers for all three function
 ** callbacks.
 **
-** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue 
+** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue
 ** and xInverse) passed to sqlite3_create_window_function are pointers to
 ** C-language callbacks that implement the new function. xStep and xFinal
 ** must both be non-NULL. xValue and xInverse may either both be NULL, in
-** which case a regular aggregate function is created, or must both be 
+** which case a regular aggregate function is created, or must both be
 ** non-NULL, in which case the new function may be used as either an aggregate
 ** or aggregate window function. More details regarding the implementation
-** of aggregate window functions are 
+** of aggregate window functions are
 ** [user-defined window functions|available here].
 **
 ** ^(If the final parameter to sqlite3_create_function_v2() or
-** sqlite3_create_window_function() is not NULL, then it is destructor for
-** the application data pointer. The destructor is invoked when the function 
-** is deleted, either by being overloaded or when the database connection 
-** closes.)^ ^The destructor is also invoked if the call to 
+** sqlite3_create_window_function() is not NULL, then it is the destructor for
+** the application data pointer. The destructor is invoked when the function
+** is deleted, either by being overloaded or when the database connection
+** closes.)^ ^The destructor is also invoked if the call to
 ** sqlite3_create_function_v2() fails.  ^When the destructor callback is
 ** invoked, it is passed a single argument which is a copy of the application
 ** data pointer which was the fifth parameter to sqlite3_create_function_v2().
@@ -5385,7 +5705,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt);
 ** nArg parameter is a better match than a function implementation with
 ** a negative nArg.  ^A function where the preferred text encoding
 ** matches the database encoding is a better
-** match than a function where the encoding is different.  
+** match than a function where the encoding is different. 
 ** ^A function where the encoding difference is between UTF16le and UTF16be
 ** is a closer match than a function where the encoding difference is
 ** between UTF8 and UTF16.
@@ -5444,7 +5764,7 @@ int sqlite3_create_window_function(
 /*
 ** CAPI3REF: Text Encodings
 **
-** These constant define integer codes that represent the various
+** These constants define integer codes that represent the various
 ** text encodings supported by SQLite.
 */
 #define SQLITE_UTF8           1    /* IMP: R-37514-35566 */
@@ -5457,7 +5777,7 @@ int sqlite3_create_window_function(
 /*
 ** CAPI3REF: Function Flags
 **
-** These constants may be ORed together with the 
+** These constants may be ORed together with the
 ** [SQLITE_UTF8 | preferred text encoding] as the fourth argument
 ** to [sqlite3_create_function()], [sqlite3_create_function16()], or
 ** [sqlite3_create_function_v2()].
@@ -5473,18 +5793,18 @@ int sqlite3_create_window_function(
 ** SQLite might also optimize deterministic functions by factoring them
 ** out of inner loops.
 ** </dd>
-** 
+**
 ** [[SQLITE_DIRECTONLY]] <dt>SQLITE_DIRECTONLY</dt><dd>
 ** The SQLITE_DIRECTONLY flag means that the function may only be invoked
-** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in 
+** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in
 ** schema structures such as [CHECK constraints], [DEFAULT clauses],
 ** [expression indexes], [partial indexes], or [generated columns].
 ** <p>
-** The SQLITE_DIRECTONLY flag is recommended for any 
+** The SQLITE_DIRECTONLY flag is recommended for any
 ** [application-defined SQL function]
 ** that has side-effects or that could potentially leak sensitive information.
 ** This will prevent attacks in which an application is tricked
-** into using a database file that has had its schema surreptiously
+** into using a database file that has had its schema surreptitiously
 ** modified to invoke the application-defined function in ways that are
 ** harmful.
 ** <p>
@@ -5520,13 +5840,36 @@ int sqlite3_create_window_function(
 ** </dd>
 **
 ** [[SQLITE_SUBTYPE]] <dt>SQLITE_SUBTYPE</dt><dd>
-** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call
+** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call
 ** [sqlite3_value_subtype()] to inspect the sub-types of its arguments.
-** Specifying this flag makes no difference for scalar or aggregate user
-** functions. However, if it is not specified for a user-defined window
-** function, then any sub-types belonging to arguments passed to the window
-** function may be discarded before the window function is called (i.e.
-** sqlite3_value_subtype() will always return 0).
+** This flag instructs SQLite to omit some corner-case optimizations that
+** might disrupt the operation of the [sqlite3_value_subtype()] function,
+** causing it to return zero rather than the correct subtype().
+** All SQL functions that invoke [sqlite3_value_subtype()] should have this
+** property.  If the SQLITE_SUBTYPE property is omitted, then the return
+** value from [sqlite3_value_subtype()] might sometimes be zero even though
+** a non-zero subtype was specified by the function argument expression.
+**
+** [[SQLITE_RESULT_SUBTYPE]] <dt>SQLITE_RESULT_SUBTYPE</dt><dd>
+** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call
+** [sqlite3_result_subtype()] to cause a sub-type to be associated with its
+** result.
+** Every function that invokes [sqlite3_result_subtype()] should have this
+** property.  If it does not, then the call to [sqlite3_result_subtype()]
+** might become a no-op if the function is used as a term in an
+** [expression index].  On the other hand, SQL functions that never invoke
+** [sqlite3_result_subtype()] should avoid setting this property, as the
+** purpose of this property is to disable certain optimizations that are
+** incompatible with subtypes.
+**
+** [[SQLITE_SELFORDER1]] <dt>SQLITE_SELFORDER1</dt><dd>
+** The SQLITE_SELFORDER1 flag indicates that the function is an aggregate
+** that internally orders the values provided to the first argument.  The
+** ordered-set aggregate SQL notation with a single ORDER BY term can be
+** used to invoke this function.  If the ordered-set aggregate notation is
+** used on a function that lacks this flag, then an error is raised. Note
+** that the ordered-set aggregate syntax is only available if SQLite is
+** built using the -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES compile-time option.
 ** </dd>
 ** </dl>
 */
@@ -5534,13 +5877,15 @@ int sqlite3_create_window_function(
 #define SQLITE_DIRECTONLY       0x000080000
 #define SQLITE_SUBTYPE          0x000100000
 #define SQLITE_INNOCUOUS        0x000200000
+#define SQLITE_RESULT_SUBTYPE   0x001000000
+#define SQLITE_SELFORDER1       0x002000000
 
 /*
 ** CAPI3REF: Deprecated Functions
 ** DEPRECATED
 **
 ** These functions are [deprecated].  In order to maintain
-** backwards compatibility with older code, these functions continue 
+** backwards compatibility with older code, these functions continue
 ** to be supported.  However, new applications should avoid
 ** the use of these functions.  To encourage programmers to avoid
 ** these functions, we will not explain what they do.
@@ -5608,11 +5953,11 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),
 ** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces
 ** extract UTF-16 strings as big-endian and little-endian respectively.
 **
-** ^If [sqlite3_value] object V was initialized 
+** ^If [sqlite3_value] object V was initialized
 ** using [sqlite3_bind_pointer(S,I,P,X,D)] or [sqlite3_result_pointer(C,P,X,D)]
 ** and if X and Y are strings that compare equal according to strcmp(X,Y),
 ** then sqlite3_value_pointer(V,Y) will return the pointer P.  ^Otherwise,
-** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer() 
+** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer()
 ** routine is part of the [pointer passing interface] added for SQLite 3.20.0.
 **
 ** ^(The sqlite3_value_type(V) interface returns the
@@ -5638,7 +5983,7 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),
 ** sqlite3_value_nochange(X) interface returns true if and only if
 ** the column corresponding to X is unchanged by the UPDATE operation
 ** that the xUpdate method call was invoked to implement and if
-** and the prior [xColumn] method call that was invoked to extracted
+** the prior [xColumn] method call that was invoked to extract
 ** the value for that column returned without setting a result (probably
 ** because it queried [sqlite3_vtab_nochange()] and found that the column
 ** was unchanging).  ^Within an [xUpdate] method, any value for which
@@ -5730,6 +6075,12 @@ int sqlite3_value_encoding(sqlite3_value*);
 ** information can be used to pass a limited amount of context from
 ** one SQL function to another.  Use the [sqlite3_result_subtype()]
 ** routine to set the subtype for the return value of an SQL function.
+**
+** Every [application-defined SQL function] that invokes this interface
+** should include the [SQLITE_SUBTYPE] property in the text
+** encoding argument when the function is [sqlite3_create_function|registered].
+** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype()
+** might return zero instead of the upstream subtype in some corner cases.
 */
 unsigned int sqlite3_value_subtype(sqlite3_value*);
 
@@ -5738,7 +6089,7 @@ unsigned int sqlite3_value_subtype(sqlite3_value*);
 ** METHOD: sqlite3_value
 **
 ** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value]
-** object D and returns a pointer to that copy.  ^The [sqlite3_value] returned
+** object V and returns a pointer to that copy.  ^The [sqlite3_value] returned
 ** is a [protected sqlite3_value] object even if the input is not.
 ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a
 ** memory allocation fails. ^If V is a [pointer value], then the result
@@ -5758,7 +6109,7 @@ void sqlite3_value_free(sqlite3_value*);
 ** Implementations of aggregate SQL functions use this
 ** routine to allocate memory for storing their state.
 **
-** ^The first time the sqlite3_aggregate_context(C,N) routine is called 
+** ^The first time the sqlite3_aggregate_context(C,N) routine is called
 ** for a particular aggregate function, SQLite allocates
 ** N bytes of memory, zeroes out that memory, and returns a pointer
 ** to the new memory. ^On second and subsequent calls to
@@ -5771,19 +6122,19 @@ void sqlite3_value_free(sqlite3_value*);
 ** In those cases, sqlite3_aggregate_context() might be called for the
 ** first time from within xFinal().)^
 **
-** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer 
+** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer
 ** when first called if N is less than or equal to zero or if a memory
 ** allocation error occurs.
 **
 ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is
-** determined by the N parameter on first successful call.  Changing the
+** determined by the N parameter on the first successful call.  Changing the
 ** value of N in any subsequent call to sqlite3_aggregate_context() within
 ** the same aggregate function instance will not resize the memory
 ** allocation.)^  Within the xFinal callback, it is customary to set
-** N=0 in calls to sqlite3_aggregate_context(C,N) so that no 
+** N=0 in calls to sqlite3_aggregate_context(C,N) so that no
 ** pointless memory allocations occur.
 **
-** ^SQLite automatically frees the memory allocated by 
+** ^SQLite automatically frees the memory allocated by
 ** sqlite3_aggregate_context() when the aggregate query concludes.
 **
 ** The first parameter must be a copy of the
@@ -5828,48 +6179,56 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 ** METHOD: sqlite3_context
 **
 ** These functions may be used by (non-aggregate) SQL functions to
-** associate metadata with argument values. If the same value is passed to
-** multiple invocations of the same SQL function during query execution, under
-** some circumstances the associated metadata may be preserved.  An example
-** of where this might be useful is in a regular-expression matching
-** function. The compiled version of the regular expression can be stored as
-** metadata associated with the pattern string.  
+** associate auxiliary data with argument values. If the same argument
+** value is passed to multiple invocations of the same SQL function during
+** query execution, under some circumstances the associated auxiliary data
+** might be preserved.  An example of where this might be useful is in a
+** regular-expression matching function. The compiled version of the regular
+** expression can be stored as auxiliary data associated with the pattern string. 
 ** Then as long as the pattern string remains the same,
 ** the compiled regular expression can be reused on multiple
 ** invocations of the same function.
 **
-** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata
+** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data
 ** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument
 ** value to the application-defined function.  ^N is zero for the left-most
-** function argument.  ^If there is no metadata
+** function argument.  ^If there is no auxiliary data
 ** associated with the function argument, the sqlite3_get_auxdata(C,N) interface
 ** returns a NULL pointer.
 **
-** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th
-** argument of the application-defined function.  ^Subsequent
+** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the
+** N-th argument of the application-defined function.  ^Subsequent
 ** calls to sqlite3_get_auxdata(C,N) return P from the most recent
-** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or
-** NULL if the metadata has been discarded.
+** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or
+** NULL if the auxiliary data has been discarded.
 ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL,
 ** SQLite will invoke the destructor function X with parameter P exactly
-** once, when the metadata is discarded.
-** SQLite is free to discard the metadata at any time, including: <ul>
+** once, when the auxiliary data is discarded.
+** SQLite is free to discard the auxiliary data at any time, including: <ul>
 ** <li> ^(when the corresponding function parameter changes)^, or
 ** <li> ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the
 **      SQL statement)^, or
 ** <li> ^(when sqlite3_set_auxdata() is invoked again on the same
 **       parameter)^, or
-** <li> ^(during the original sqlite3_set_auxdata() call when a memory 
-**      allocation error occurs.)^ </ul>
+** <li> ^(during the original sqlite3_set_auxdata() call when a memory
+**      allocation error occurs.)^
+** <li> ^(during the original sqlite3_set_auxdata() call if the function
+**      is evaluated during query planning instead of during query execution,
+**      as sometimes happens with [SQLITE_ENABLE_STAT4].)^ </ul>
 **
-** Note the last bullet in particular.  The destructor X in 
+** Note the last two bullets in particular.  The destructor X in
 ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the
 ** sqlite3_set_auxdata() interface even returns.  Hence sqlite3_set_auxdata()
 ** should be called near the end of the function implementation and the
 ** function implementation should not make any use of P after
-** sqlite3_set_auxdata() has been called.
-**
-** ^(In practice, metadata is preserved between function calls for
+** sqlite3_set_auxdata() has been called.  Furthermore, a call to
+** sqlite3_get_auxdata() that occurs immediately after a corresponding call
+** to sqlite3_set_auxdata() might still return NULL if an out-of-memory
+** condition occurred during the sqlite3_set_auxdata() call or if the
+** function is being evaluated during query planning rather than during
+** query execution.
+**
+** ^(In practice, auxiliary data is preserved between function calls for
 ** function parameters that are compile-time constants, including literal
 ** values and [parameters] and expressions composed from the same.)^
 **
@@ -5879,10 +6238,68 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 **
 ** These routines must be called from the same thread in which
 ** the SQL function is running.
+**
+** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()].
 */
 void *sqlite3_get_auxdata(sqlite3_context*, int N);
 void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*));
 
+/*
+** CAPI3REF: Database Connection Client Data
+** METHOD: sqlite3
+**
+** These functions are used to associate one or more named pointers
+** with a [database connection].
+** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P
+** to be attached to [database connection] D using name N.  Subsequent
+** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P
+** or a NULL pointer if there were no prior calls to
+** sqlite3_set_clientdata() with the same values of D and N.
+** Names are compared using strcmp() and are thus case sensitive.
+** It returns 0 on success and SQLITE_NOMEM on allocation failure.
+**
+** If P and X are both non-NULL, then the destructor X is invoked with
+** argument P on the first of the following occurrences:
+** <ul>
+** <li> An out-of-memory error occurs during the call to
+**      sqlite3_set_clientdata() which attempts to register pointer P.
+** <li> A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made
+**      with the same D and N parameters.
+** <li> The database connection closes.  SQLite does not make any guarantees
+**      about the order in which destructors are called, only that all
+**      destructors will be called exactly once at some point during the
+**      database connection closing process.
+** </ul>
+**
+** SQLite does not do anything with client data other than invoke
+** destructors on the client data at the appropriate time.  The intended
+** use for client data is to provide a mechanism for wrapper libraries
+** to store additional information about an SQLite database connection.
+**
+** There is no limit (other than available memory) on the number of different
+** client data pointers (with different names) that can be attached to a
+** single database connection.  However, the implementation is optimized
+** for the case of having only one or two different client data names.
+** Applications and wrapper libraries are discouraged from using more than
+** one client data name each.
+**
+** There is no way to enumerate the client data pointers
+** associated with a database connection.  The N parameter can be thought
+** of as a secret key such that only code that knows the secret key is able
+** to access the associated data.
+**
+** Security Warning:  These interfaces should not be exposed in scripting
+** languages or in other circumstances where it might be possible for an
+** attacker to invoke them.  Any agent that can invoke these interfaces
+** can probably also take control of the process.
+**
+** Database connection client data is only available for SQLite
+** version 3.44.0 ([dateof:3.44.0]) and later.
+**
+** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()].
+*/
+void *sqlite3_get_clientdata(sqlite3*,const char*);
+int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*));
 
 /*
 ** CAPI3REF: Constants Defining Special Destructor Behavior
@@ -5987,7 +6404,7 @@ typedef void (*sqlite3_destructor_type)(void*);
 ** pointed to by the 2nd parameter are taken as the application-defined
 ** function result.  If the 3rd parameter is non-negative, then it
 ** must be the byte offset into the string where the NUL terminator would
-** appear if the string where NUL terminated.  If any NUL characters occur
+** appear if the string were NUL terminated.  If any NUL characters occur
 ** in the string at a byte offset that is less than the value of the 3rd
 ** parameter, then the resulting string will contain embedded NULs and the
 ** result of expressions operating on strings with embedded NULs is undefined.
@@ -6036,7 +6453,7 @@ typedef void (*sqlite3_destructor_type)(void*);
 **
 ** ^The sqlite3_result_pointer(C,P,T,D) interface sets the result to an
 ** SQL NULL value, just like [sqlite3_result_null(C)], except that it
-** also associates the host-language pointer P or type T with that 
+** also associates the host-language pointer P or type T with that
 ** NULL value such that the pointer can be retrieved within an
 ** [application-defined SQL function] using [sqlite3_value_pointer()].
 ** ^If the D parameter is not NULL, then it is a pointer to a destructor
@@ -6045,7 +6462,7 @@ typedef void (*sqlite3_destructor_type)(void*);
 ** string and preferably a string literal. The sqlite3_result_pointer()
 ** routine is part of the [pointer passing interface] added for SQLite 3.20.0.
 **
-** If these routines are called from within the different thread
+** If these routines are called from within a different thread
 ** than the one containing the application-defined function that received
 ** the [sqlite3_context] pointer, the results are undefined.
 */
@@ -6078,12 +6495,26 @@ int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n);
 ** METHOD: sqlite3_context
 **
 ** The sqlite3_result_subtype(C,T) function causes the subtype of
-** the result from the [application-defined SQL function] with 
-** [sqlite3_context] C to be the value T.  Only the lower 8 bits 
+** the result from the [application-defined SQL function] with
+** [sqlite3_context] C to be the value T.  Only the lower 8 bits
 ** of the subtype T are preserved in current versions of SQLite;
 ** higher order bits are discarded.
 ** The number of subtype bytes preserved by SQLite might increase
 ** in future releases of SQLite.
+**
+** Every [application-defined SQL function] that invokes this interface
+** should include the [SQLITE_RESULT_SUBTYPE] property in its
+** text encoding argument when the SQL function is
+** [sqlite3_create_function|registered].  If the [SQLITE_RESULT_SUBTYPE]
+** property is omitted from the function that invokes sqlite3_result_subtype(),
+** then in some cases the sqlite3_result_subtype() might fail to set
+** the result subtype.
+**
+** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any
+** SQL function that invokes the sqlite3_result_subtype() interface
+** and that does not have the SQLITE_RESULT_SUBTYPE property will raise
+** an error.  Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1
+** by default.
 */
 void sqlite3_result_subtype(sqlite3_context*,unsigned int);
 
@@ -6126,7 +6557,7 @@ void sqlite3_result_subtype(sqlite3_context*,unsigned int);
 ** deleted.  ^When all collating functions having the same name are deleted,
 ** that collation is no longer usable.
 **
-** ^The collating function callback is invoked with a copy of the pArg 
+** ^The collating function callback is invoked with a copy of the pArg
 ** application data pointer and with two strings in the encoding specified
 ** by the eTextRep argument.  The two integer parameters to the collating
 ** function callback are the length of the two strings, in bytes. The collating
@@ -6157,36 +6588,36 @@ void sqlite3_result_subtype(sqlite3_context*,unsigned int);
 ** calls to the collation creation functions or when the
 ** [database connection] is closed using [sqlite3_close()].
 **
-** ^The xDestroy callback is <u>not</u> called if the 
+** ^The xDestroy callback is <u>not</u> called if the
 ** sqlite3_create_collation_v2() function fails.  Applications that invoke
-** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should 
+** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should
 ** check the return code and dispose of the application data pointer
 ** themselves rather than expecting SQLite to deal with it for them.
-** This is different from every other SQLite interface.  The inconsistency 
-** is unfortunate but cannot be changed without breaking backwards 
+** This is different from every other SQLite interface.  The inconsistency
+** is unfortunate but cannot be changed without breaking backwards
 ** compatibility.
 **
 ** See also:  [sqlite3_collation_needed()] and [sqlite3_collation_needed16()].
 */
 int sqlite3_create_collation(
-  sqlite3*, 
-  const char *zName, 
-  int eTextRep, 
+  sqlite3*,
+  const char *zName,
+  int eTextRep,
   void *pArg,
   int(*xCompare)(void*,int,const void*,int,const void*)
 );
 int sqlite3_create_collation_v2(
-  sqlite3*, 
-  const char *zName, 
-  int eTextRep, 
+  sqlite3*,
+  const char *zName,
+  int eTextRep,
   void *pArg,
   int(*xCompare)(void*,int,const void*,int,const void*),
   void(*xDestroy)(void*)
 );
 int sqlite3_create_collation16(
-  sqlite3*, 
+  sqlite3*,
   const void *zName,
-  int eTextRep, 
+  int eTextRep,
   void *pArg,
   int(*xCompare)(void*,int,const void*,int,const void*)
 );
@@ -6219,12 +6650,12 @@ int sqlite3_create_collation16(
 ** [sqlite3_create_collation_v2()].
 */
 int sqlite3_collation_needed(
-  sqlite3*, 
-  void*, 
+  sqlite3*,
+  void*,
   void(*)(void*,sqlite3*,int eTextRep,const char*)
 );
 int sqlite3_collation_needed16(
-  sqlite3*, 
+  sqlite3*,
   void*,
   void(*)(void*,sqlite3*,int eTextRep,const void*)
 );
@@ -6291,7 +6722,7 @@ void sqlite3_activate_see(
 
 #ifdef SQLITE_ENABLE_CEROD
 /*
-** Specify the activation key for a CEROD database.  Unless 
+** Specify the activation key for a CEROD database.  Unless
 ** activated, none of the CEROD routines will work.
 */
 void sqlite3_activate_cerod(
@@ -6354,7 +6785,7 @@ int sqlite3_sleep(int);
 ** ^The [temp_store_directory pragma] may modify this variable and cause
 ** it to point to memory obtained from [sqlite3_malloc].  ^Furthermore,
 ** the [temp_store_directory pragma] always assumes that any string
-** that this variable points to is held in memory obtained from 
+** that this variable points to is held in memory obtained from
 ** [sqlite3_malloc] and the pragma may attempt to free that memory
 ** using [sqlite3_free].
 ** Hence, if this variable is modified directly, either it should be
@@ -6411,7 +6842,7 @@ SQLITE_EXTERN char *sqlite3_temp_directory;
 ** ^The [data_store_directory pragma] may modify this variable and cause
 ** it to point to memory obtained from [sqlite3_malloc].  ^Furthermore,
 ** the [data_store_directory pragma] always assumes that any string
-** that this variable points to is held in memory obtained from 
+** that this variable points to is held in memory obtained from
 ** [sqlite3_malloc] and the pragma may attempt to free that memory
 ** using [sqlite3_free].
 ** Hence, if this variable is modified directly, either it should be
@@ -6497,7 +6928,7 @@ sqlite3 *sqlite3_db_handle(sqlite3_stmt*);
 ** METHOD: sqlite3
 **
 ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name
-** for the N-th database on database connection D, or a NULL pointer of N is
+** for the N-th database on database connection D, or a NULL pointer if N is
 ** out of range.  An N value of 0 means the main database file.  An N of 1 is
 ** the "temp" schema.  Larger values of N correspond to various ATTACH-ed
 ** databases.
@@ -6575,7 +7006,7 @@ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName);
 int sqlite3_txn_state(sqlite3*,const char *zSchema);
 
 /*
-** CAPI3REF: Allowed return values from [sqlite3_txn_state()]
+** CAPI3REF: Allowed return values from sqlite3_txn_state()
 ** KEYWORDS: {transaction state}
 **
 ** These constants define the current transaction state of a database file.
@@ -6592,7 +7023,7 @@ int sqlite3_txn_state(sqlite3*,const char *zSchema);
 ** <dd>The SQLITE_TXN_READ state means that the database is currently
 ** in a read transaction.  Content has been read from the database file
 ** but nothing in the database file has changed.  The transaction state
-** will advanced to SQLITE_TXN_WRITE if any changes occur and there are
+** will be advanced to SQLITE_TXN_WRITE if any changes occur and there are
 ** no other conflicting concurrent write transactions.  The transaction
 ** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or
 ** [COMMIT].</dd>
@@ -6601,7 +7032,7 @@ int sqlite3_txn_state(sqlite3*,const char *zSchema);
 ** <dd>The SQLITE_TXN_WRITE state means that the database is currently
 ** in a write transaction.  Content has been written to the database file
 ** but has not yet committed.  The transaction state will change to
-** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].</dd>
+** SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].</dd>
 */
 #define SQLITE_TXN_NONE  0
 #define SQLITE_TXN_READ  1
@@ -6677,7 +7108,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
 ** CAPI3REF: Autovacuum Compaction Amount Callback
 ** METHOD: sqlite3
 **
-** ^The sqlite3_autovacuum_pages(D,C,P,X) interface registers a callback 
+** ^The sqlite3_autovacuum_pages(D,C,P,X) interface registers a callback
 ** function C that is invoked prior to each autovacuum of the database
 ** file.  ^The callback is passed a copy of the generic data pointer (P),
 ** the schema-name of the attached database that is being autovacuumed,
@@ -6707,7 +7138,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
 ** ^Each call to the sqlite3_autovacuum_pages() interface overrides all
 ** previous invocations for that database connection.  ^If the callback
 ** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer,
-** then the autovacuum steps callback is cancelled.  The return value
+** then the autovacuum steps callback is canceled.  The return value
 ** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might
 ** be some other error code if something goes wrong.  The current
 ** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other
@@ -6752,6 +7183,8 @@ int sqlite3_autovacuum_pages(
 **
 ** ^The second argument is a pointer to the function to invoke when a
 ** row is updated, inserted or deleted in a rowid table.
+** ^The update hook is disabled by invoking sqlite3_update_hook()
+** with a NULL pointer as the second parameter.
 ** ^The first argument to the callback is a copy of the third argument
 ** to sqlite3_update_hook().
 ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE],
@@ -6773,6 +7206,12 @@ int sqlite3_autovacuum_pages(
 ** The exceptions defined in this paragraph might change in a future
 ** release of SQLite.
 **
+** Whether the update hook is invoked before or after the
+** corresponding change is currently unspecified and may differ
+** depending on the type of change. Do not rely on the order of the
+** hook call with regards to the final result of the operation which
+** triggers the hook.
+**
 ** The update hook implementation must not do anything that will modify
 ** the database connection that invoked the update hook.  Any actions
 ** to modify the database connection must be deferred until after the
@@ -6789,7 +7228,7 @@ int sqlite3_autovacuum_pages(
 ** and [sqlite3_preupdate_hook()] interfaces.
 */
 void *sqlite3_update_hook(
-  sqlite3*, 
+  sqlite3*,
   void(*)(void *,int ,char const *,char const *,sqlite3_int64),
   void*
 );
@@ -6808,7 +7247,7 @@ void *sqlite3_update_hook(
 ** [use of shared cache mode is discouraged].
 **
 ** ^Cache sharing is enabled and disabled for an entire process.
-** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). 
+** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]).
 ** In prior versions of SQLite,
 ** sharing was enabled or disabled for each thread separately.
 **
@@ -6829,8 +7268,8 @@ void *sqlite3_update_hook(
 ** with the [SQLITE_OPEN_SHAREDCACHE] flag.
 **
 ** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0
-** and will always return SQLITE_MISUSE. On those systems, 
-** shared cache mode should be enabled per-database connection via 
+** and will always return SQLITE_MISUSE. On those systems,
+** shared cache mode should be enabled per-database connection via
 ** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE].
 **
 ** This interface is threadsafe on processors where writing a
@@ -6874,7 +7313,7 @@ int sqlite3_db_release_memory(sqlite3*);
 ** CAPI3REF: Impose A Limit On Heap Size
 **
 ** These interfaces impose limits on the amount of heap memory that will be
-** by all database connections within a single process.
+** used by all database connections within a single process.
 **
 ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the
 ** soft limit on the amount of heap memory that may be allocated by SQLite.
@@ -6883,7 +7322,7 @@ int sqlite3_db_release_memory(sqlite3*);
 ** as heap memory usages approaches the limit.
 ** ^The soft heap limit is "soft" because even though SQLite strives to stay
 ** below the limit, it will exceed the limit rather than generate
-** an [SQLITE_NOMEM] error.  In other words, the soft heap limit 
+** an [SQLITE_NOMEM] error.  In other words, the soft heap limit
 ** is advisory only.
 **
 ** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of
@@ -6932,7 +7371,7 @@ int sqlite3_db_release_memory(sqlite3*);
 ** </ul>)^
 **
 ** The circumstances under which SQLite will enforce the heap limits may
-** changes in future releases of SQLite.
+** change in future releases of SQLite.
 */
 sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N);
 sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N);
@@ -6999,7 +7438,7 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
 **
 ** ^If the specified table is actually a view, an [error code] is returned.
 **
-** ^If the specified column is "rowid", "oid" or "_rowid_" and the table 
+** ^If the specified column is "rowid", "oid" or "_rowid_" and the table
 ** is not a [WITHOUT ROWID] table and an
 ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output
 ** parameters are set for the explicitly declared column. ^(If there is no
@@ -7047,8 +7486,8 @@ int sqlite3_table_column_metadata(
 ** ^The entry point is zProc.
 ** ^(zProc may be 0, in which case SQLite will try to come up with an
 ** entry point name on its own.  It first tries "sqlite3_extension_init".
-** If that does not work, it constructs a name "sqlite3_X_init" where the
-** X is consists of the lower-case equivalent of all ASCII alphabetic
+** If that does not work, it constructs a name "sqlite3_X_init" where
+** X consists of the lower-case equivalent of all ASCII alphabetic
 ** characters in the filename from the last "/" to the first following
 ** "." and omitting any initial "lib".)^
 ** ^The sqlite3_load_extension() interface returns
@@ -7065,7 +7504,7 @@ int sqlite3_table_column_metadata(
 ** prior to calling this API,
 ** otherwise an error will be returned.
 **
-** <b>Security warning:</b> It is recommended that the 
+** <b>Security warning:</b> It is recommended that the
 ** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this
 ** interface.  The use of the [sqlite3_enable_load_extension()] interface
 ** should be avoided.  This will keep the SQL function [load_extension()]
@@ -7119,7 +7558,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
 ** ^(Even though the function prototype shows that xEntryPoint() takes
 ** no arguments and returns void, SQLite invokes xEntryPoint() with three
 ** arguments and expects an integer result as if the signature of the
-** entry point where as follows:
+** entry point were as follows:
 **
 ** <blockquote><pre>
 ** &nbsp;  int xEntryPoint(
@@ -7152,7 +7591,7 @@ int sqlite3_auto_extension(void(*xEntryPoint)(void));
 ** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the
 ** initialization routine X that was registered using a prior call to
 ** [sqlite3_auto_extension(X)].  ^The [sqlite3_cancel_auto_extension(X)]
-** routine returns 1 if initialization routine X was successfully 
+** routine returns 1 if initialization routine X was successfully
 ** unregistered and it returns 0 if X was not on the list of initialization
 ** routines.
 */
@@ -7178,8 +7617,8 @@ typedef struct sqlite3_module sqlite3_module;
 ** CAPI3REF: Virtual Table Object
 ** KEYWORDS: sqlite3_module {virtual table module}
 **
-** This structure, sometimes called a "virtual table module", 
-** defines the implementation of a [virtual table].  
+** This structure, sometimes called a "virtual table module",
+** defines the implementation of a [virtual table]. 
 ** This structure consists mostly of methods for the module.
 **
 ** ^A virtual table module is created by filling in a persistent
@@ -7218,7 +7657,7 @@ struct sqlite3_module {
                        void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
                        void **ppArg);
   int (*xRename)(sqlite3_vtab *pVtab, const char *zNew);
-  /* The methods above are in version 1 of the sqlite_module object. Those 
+  /* The methods above are in version 1 of the sqlite_module object. Those
   ** below are for version 2 and greater. */
   int (*xSavepoint)(sqlite3_vtab *pVTab, int);
   int (*xRelease)(sqlite3_vtab *pVTab, int);
@@ -7226,6 +7665,10 @@ struct sqlite3_module {
   /* The methods above are in versions 1 and 2 of the sqlite_module object.
   ** Those below are for version 3 and greater. */
   int (*xShadowName)(const char*);
+  /* The methods above are in versions 1 through 3 of the sqlite_module object.
+  ** Those below are for version 4 and greater. */
+  int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema,
+                    const char *zTabName, int mFlags, char **pzErr);
 };
 
 /*
@@ -7268,7 +7711,7 @@ struct sqlite3_module {
 ** required by SQLite. If the table has at least 64 columns and any column
 ** to the right of the first 63 is required, then bit 63 of colUsed is also
 ** set. In other words, column iCol may be required if the expression
-** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to 
+** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to
 ** non-zero.
 **
 ** The [xBestIndex] method must fill aConstraintUsage[] with information
@@ -7279,7 +7722,7 @@ struct sqlite3_module {
 ** virtual table and might not be checked again by the byte code.)^ ^(The
 ** aConstraintUsage[].omit flag is an optimization hint. When the omit flag
 ** is left in its default setting of false, the constraint will always be
-** checked separately in byte code.  If the omit flag is change to true, then
+** checked separately in byte code.  If the omit flag is changed to true, then
 ** the constraint may or may not be checked in byte code.  In other words,
 ** when the omit flag is true there is no guarantee that the constraint will
 ** not be checked again using byte code.)^
@@ -7295,17 +7738,19 @@ struct sqlite3_module {
 **
 ** ^The estimatedCost value is an estimate of the cost of a particular
 ** strategy. A cost of N indicates that the cost of the strategy is similar
-** to a linear scan of an SQLite table with N rows. A cost of log(N) 
+** to a linear scan of an SQLite table with N rows. A cost of log(N)
 ** indicates that the expense of the operation is similar to that of a
 ** binary search on a unique indexed field of an SQLite table with N rows.
 **
 ** ^The estimatedRows value is an estimate of the number of rows that
 ** will be returned by the strategy.
 **
-** The xBestIndex method may optionally populate the idxFlags field with a 
-** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
-** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
-** assumes that the strategy may visit at most one row. 
+** The xBestIndex method may optionally populate the idxFlags field with a
+** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
+** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
+** output to show the idxNum as hex instead of as decimal.  Another flag is
+** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will
+** return at most one row.
 **
 ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
 ** SQLite also assumes that if a call to the xUpdate() method is made as
@@ -7318,14 +7763,14 @@ struct sqlite3_module {
 ** the xUpdate method are automatically rolled back by SQLite.
 **
 ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
-** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). 
+** structure for SQLite [version 3.8.2] ([dateof:3.8.2]).
 ** If a virtual table extension is
-** used with an SQLite version earlier than 3.8.2, the results of attempting 
-** to read or write the estimatedRows field are undefined (but are likely 
+** used with an SQLite version earlier than 3.8.2, the results of attempting
+** to read or write the estimatedRows field are undefined (but are likely
 ** to include crashing the application). The estimatedRows field should
 ** therefore only be used if [sqlite3_libversion_number()] returns a
 ** value greater than or equal to 3008002. Similarly, the idxFlags field
-** was added for [version 3.9.0] ([dateof:3.9.0]). 
+** was added for [version 3.9.0] ([dateof:3.9.0]).
 ** It may therefore only be used if
 ** sqlite3_libversion_number() returns a value greater than or equal to
 ** 3009000.
@@ -7365,11 +7810,13 @@ struct sqlite3_index_info {
 /*
 ** CAPI3REF: Virtual Table Scan Flags
 **
-** Virtual table implementations are allowed to set the 
+** Virtual table implementations are allowed to set the
 ** [sqlite3_index_info].idxFlags field to some combination of
 ** these bits.
 */
-#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
+#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */
+#define SQLITE_INDEX_SCAN_HEX    0x00000002 /* Display idxNum as hex */
+                                            /* in EXPLAIN QUERY PLAN */
 
 /*
 ** CAPI3REF: Virtual Table Constraint Operator Codes
@@ -7437,12 +7884,12 @@ struct sqlite3_index_info {
 ** preexisting [virtual table] for the module.
 **
 ** ^The module name is registered on the [database connection] specified
-** by the first parameter.  ^The name of the module is given by the 
+** by the first parameter.  ^The name of the module is given by the
 ** second parameter.  ^The third parameter is a pointer to
 ** the implementation of the [virtual table module].   ^The fourth
 ** parameter is an arbitrary client data pointer that is passed through
 ** into the [xCreate] and [xConnect] methods of the virtual table module
-** when a new virtual table is be being created or reinitialized.
+** when a new virtual table is being created or reinitialized.
 **
 ** ^The sqlite3_create_module_v2() interface has a fifth parameter which
 ** is a pointer to a destructor for the pClientData.  ^SQLite will
@@ -7552,7 +7999,7 @@ int sqlite3_declare_vtab(sqlite3*, const char *zSQL);
 ** METHOD: sqlite3
 **
 ** ^(Virtual tables can provide alternative implementations of functions
-** using the [xFindFunction] method of the [virtual table module].  
+** using the [xFindFunction] method of the [virtual table module]. 
 ** But global versions of those functions
 ** must exist in order to be overloaded.)^
 **
@@ -7593,7 +8040,7 @@ typedef struct sqlite3_blob sqlite3_blob;
 **     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
 ** </pre>)^
 **
-** ^(Parameter zDb is not the filename that contains the database, but 
+** ^(Parameter zDb is not the filename that contains the database, but
 ** rather the symbolic name of the database. For attached databases, this is
 ** the name that appears after the AS keyword in the [ATTACH] statement.
 ** For the main database file, the database name is "main". For TEMP
@@ -7606,28 +8053,28 @@ typedef struct sqlite3_blob sqlite3_blob;
 ** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored
 ** in *ppBlob. Otherwise an [error code] is returned and, unless the error
 ** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided
-** the API is not misused, it is always safe to call [sqlite3_blob_close()] 
-** on *ppBlob after this function it returns.
+** the API is not misused, it is always safe to call [sqlite3_blob_close()]
+** on *ppBlob after this function returns.
 **
 ** This function fails with SQLITE_ERROR if any of the following are true:
 ** <ul>
-**   <li> ^(Database zDb does not exist)^, 
-**   <li> ^(Table zTable does not exist within database zDb)^, 
-**   <li> ^(Table zTable is a WITHOUT ROWID table)^, 
+**   <li> ^(Database zDb does not exist)^,
+**   <li> ^(Table zTable does not exist within database zDb)^,
+**   <li> ^(Table zTable is a WITHOUT ROWID table)^,
 **   <li> ^(Column zColumn does not exist)^,
 **   <li> ^(Row iRow is not present in the table)^,
 **   <li> ^(The specified column of row iRow contains a value that is not
 **         a TEXT or BLOB value)^,
-**   <li> ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE 
+**   <li> ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE
 **         constraint and the blob is being opened for read/write access)^,
-**   <li> ^([foreign key constraints | Foreign key constraints] are enabled, 
+**   <li> ^([foreign key constraints | Foreign key constraints] are enabled,
 **         column zColumn is part of a [child key] definition and the blob is
 **         being opened for read/write access)^.
 ** </ul>
 **
-** ^Unless it returns SQLITE_MISUSE, this function sets the 
-** [database connection] error code and message accessible via 
-** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. 
+** ^Unless it returns SQLITE_MISUSE, this function sets the
+** [database connection] error code and message accessible via
+** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions.
 **
 ** A BLOB referenced by sqlite3_blob_open() may be read using the
 ** [sqlite3_blob_read()] interface and modified by using
@@ -7653,7 +8100,7 @@ typedef struct sqlite3_blob sqlite3_blob;
 ** blob.
 **
 ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces
-** and the built-in [zeroblob] SQL function may be used to create a 
+** and the built-in [zeroblob] SQL function may be used to create a
 ** zero-filled blob to read or write using the incremental-blob interface.
 **
 ** To avoid a resource leak, every open [BLOB handle] should eventually
@@ -7703,7 +8150,7 @@ int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
 ** DESTRUCTOR: sqlite3_blob
 **
 ** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed
-** unconditionally.  Even if this routine returns an error code, the 
+** unconditionally.  Even if this routine returns an error code, the
 ** handle is still closed.)^
 **
 ** ^If the blob handle being closed was opened for read-write access, and if
@@ -7713,10 +8160,10 @@ int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
 ** code is returned and the transaction rolled back.
 **
 ** Calling this function with an argument that is not a NULL pointer or an
-** open blob handle results in undefined behaviour. ^Calling this routine 
-** with a null pointer (such as would be returned by a failed call to 
+** open blob handle results in undefined behavior. ^Calling this routine
+** with a null pointer (such as would be returned by a failed call to
 ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function
-** is passed a valid open blob handle, the values returned by the 
+** is passed a valid open blob handle, the values returned by the
 ** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning.
 */
 int sqlite3_blob_close(sqlite3_blob *);
@@ -7725,9 +8172,9 @@ int sqlite3_blob_close(sqlite3_blob *);
 ** CAPI3REF: Return The Size Of An Open BLOB
 ** METHOD: sqlite3_blob
 **
-** ^Returns the size in bytes of the BLOB accessible via the 
+** ^Returns the size in bytes of the BLOB accessible via the
 ** successfully opened [BLOB handle] in its only argument.  ^The
-** incremental blob I/O routines can only read or overwriting existing
+** incremental blob I/O routines can only read or overwrite existing
 ** blob content; they cannot change the size of a blob.
 **
 ** This routine only works on a [BLOB handle] which has been created
@@ -7776,9 +8223,9 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
 **
 ** ^(On success, sqlite3_blob_write() returns SQLITE_OK.
 ** Otherwise, an  [error code] or an [extended error code] is returned.)^
-** ^Unless SQLITE_MISUSE is returned, this function sets the 
-** [database connection] error code and message accessible via 
-** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. 
+** ^Unless SQLITE_MISUSE is returned, this function sets the
+** [database connection] error code and message accessible via
+** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions.
 **
 ** ^If the [BLOB handle] passed as the first argument was not opened for
 ** writing (the flags parameter to [sqlite3_blob_open()] was zero),
@@ -7787,9 +8234,9 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
 ** This function may only modify the contents of the BLOB; it is
 ** not possible to increase the size of a BLOB using this API.
 ** ^If offset iOffset is less than N bytes from the end of the BLOB,
-** [SQLITE_ERROR] is returned and no data is written. The size of the 
-** BLOB (and hence the maximum value of N+iOffset) can be determined 
-** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less 
+** [SQLITE_ERROR] is returned and no data is written. The size of the
+** BLOB (and hence the maximum value of N+iOffset) can be determined
+** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less
 ** than zero [SQLITE_ERROR] is returned and no data is written.
 **
 ** ^An attempt to write to an expired [BLOB handle] fails with an
@@ -7877,7 +8324,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*);
 ** ^The sqlite3_mutex_alloc() routine allocates a new
 ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc()
 ** routine returns NULL if it is unable to allocate the requested
-** mutex.  The argument to sqlite3_mutex_alloc() must one of these
+** mutex.  The argument to sqlite3_mutex_alloc() must be one of these
 ** integer constants:
 **
 ** <ul>
@@ -7940,9 +8387,11 @@ int sqlite3_vfs_unregister(sqlite3_vfs*);
 **
 ** ^(Some systems (for example, Windows 95) do not support the operation
 ** implemented by sqlite3_mutex_try().  On those systems, sqlite3_mutex_try()
-** will always return SQLITE_BUSY. The SQLite core only ever uses
-** sqlite3_mutex_try() as an optimization so this is acceptable
-** behavior.)^
+** will always return SQLITE_BUSY. In most cases the SQLite core only uses
+** sqlite3_mutex_try() as an optimization, so this is acceptable
+** behavior. The exceptions are unix builds that set the 
+** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working
+** sqlite3_mutex_try() is required.)^
 **
 ** ^The sqlite3_mutex_leave() routine exits a mutex that was
 ** previously entered by the same thread.   The behavior
@@ -8108,7 +8557,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*);
 ** CAPI3REF: Retrieve the mutex for a database connection
 ** METHOD: sqlite3
 **
-** ^This interface returns a pointer the [sqlite3_mutex] object that 
+** ^This interface returns a pointer to the [sqlite3_mutex] object that
 ** serializes access to the [database connection] given in the argument
 ** when the [threading mode] is Serialized.
 ** ^If the [threading mode] is Single-thread or Multi-thread then this
@@ -8135,7 +8584,7 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3*);
 ** method becomes the return value of this routine.
 **
 ** A few opcodes for [sqlite3_file_control()] are handled directly
-** by the SQLite core and never invoke the 
+** by the SQLite core and never invoke the
 ** sqlite3_io_methods.xFileControl method.
 ** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes
 ** a pointer to the underlying [sqlite3_file] object to be written into
@@ -8193,6 +8642,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_PRNG_SAVE                5
 #define SQLITE_TESTCTRL_PRNG_RESTORE             6
 #define SQLITE_TESTCTRL_PRNG_RESET               7  /* NOT USED */
+#define SQLITE_TESTCTRL_FK_NO_ACTION             7
 #define SQLITE_TESTCTRL_BITVEC_TEST              8
 #define SQLITE_TESTCTRL_FAULT_INSTALL            9
 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS     10
@@ -8200,8 +8650,10 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ASSERT                  12
 #define SQLITE_TESTCTRL_ALWAYS                  13
 #define SQLITE_TESTCTRL_RESERVE                 14  /* NOT USED */
+#define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
+#define SQLITE_TESTCTRL_GETOPT                  16
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
 #define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS      17
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
@@ -8221,20 +8673,21 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_TRACEFLAGS              31
 #define SQLITE_TESTCTRL_TUNE                    32
 #define SQLITE_TESTCTRL_LOGEST                  33
-#define SQLITE_TESTCTRL_LAST                    33  /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_USELONGDOUBLE           34  /* NOT USED */
+#define SQLITE_TESTCTRL_LAST                    34  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQL Keyword Checking
 **
-** These routines provide access to the set of SQL language keywords 
-** recognized by SQLite.  Applications can uses these routines to determine
+** These routines provide access to the set of SQL language keywords
+** recognized by SQLite.  Applications can use these routines to determine
 ** whether or not a specific identifier needs to be escaped (for example,
 ** by enclosing in double-quotes) so as not to confuse the parser.
 **
 ** The sqlite3_keyword_count() interface returns the number of distinct
 ** keywords understood by SQLite.
 **
-** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and
+** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and
 ** makes *Z point to that keyword expressed as UTF8 and writes the number
 ** of bytes in the keyword into *L.  The string that *Z points to is not
 ** zero-terminated.  The sqlite3_keyword_name(N,Z,L) routine returns
@@ -8298,14 +8751,14 @@ typedef struct sqlite3_str sqlite3_str;
 **
 ** ^The [sqlite3_str_new(D)] interface allocates and initializes
 ** a new [sqlite3_str] object.  To avoid memory leaks, the object returned by
-** [sqlite3_str_new()] must be freed by a subsequent call to 
+** [sqlite3_str_new()] must be freed by a subsequent call to
 ** [sqlite3_str_finish(X)].
 **
 ** ^The [sqlite3_str_new(D)] interface always returns a pointer to a
 ** valid [sqlite3_str] object, though in the event of an out-of-memory
 ** error the returned object might be a special singleton that will
-** silently reject new text, always return SQLITE_NOMEM from 
-** [sqlite3_str_errcode()], always return 0 for 
+** silently reject new text, always return SQLITE_NOMEM from
+** [sqlite3_str_errcode()], always return 0 for
 ** [sqlite3_str_length()], and always return NULL from
 ** [sqlite3_str_finish(X)].  It is always safe to use the value
 ** returned by [sqlite3_str_new(D)] as the sqlite3_str parameter
@@ -8341,9 +8794,9 @@ char *sqlite3_str_finish(sqlite3_str*);
 ** These interfaces add content to an sqlite3_str object previously obtained
 ** from [sqlite3_str_new()].
 **
-** ^The [sqlite3_str_appendf(X,F,...)] and 
+** ^The [sqlite3_str_appendf(X,F,...)] and
 ** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf]
-** functionality of SQLite to append formatted text onto the end of 
+** functionality of SQLite to append formatted text onto the end of
 ** [sqlite3_str] object X.
 **
 ** ^The [sqlite3_str_append(X,S,N)] method appends exactly N bytes from string S
@@ -8360,7 +8813,7 @@ char *sqlite3_str_finish(sqlite3_str*);
 ** ^This method can be used, for example, to add whitespace indentation.
 **
 ** ^The [sqlite3_str_reset(X)] method resets the string under construction
-** inside [sqlite3_str] object X back to zero bytes in length.  
+** inside [sqlite3_str] object X back to zero bytes in length. 
 **
 ** These methods do not return a result code.  ^If an error occurs, that fact
 ** is recorded in the [sqlite3_str] object and can be recovered by a
@@ -8395,7 +8848,7 @@ void sqlite3_str_reset(sqlite3_str*);
 ** content of the dynamic string under construction in X.  The value
 ** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X
 ** and might be freed or altered by any subsequent method on the same
-** [sqlite3_str] object.  Applications must not used the pointer returned
+** [sqlite3_str] object.  Applications must not use the pointer returned by
 ** [sqlite3_str_value(X)] after any subsequent method call on the same
 ** object.  ^Applications may change the content of the string returned
 ** by [sqlite3_str_value(X)] as long as they do not write into any bytes
@@ -8462,7 +8915,7 @@ int sqlite3_status64(
 ** <dd>This parameter records the largest memory allocation request
 ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their
 ** internal equivalents).  Only the value returned in the
-** *pHighwater parameter to [sqlite3_status()] is of interest.  
+** *pHighwater parameter to [sqlite3_status()] is of interest. 
 ** The value written into the *pCurrent parameter is undefined.</dd>)^
 **
 ** [[SQLITE_STATUS_MALLOC_COUNT]] ^(<dt>SQLITE_STATUS_MALLOC_COUNT</dt>
@@ -8471,24 +8924,24 @@ int sqlite3_status64(
 **
 ** [[SQLITE_STATUS_PAGECACHE_USED]] ^(<dt>SQLITE_STATUS_PAGECACHE_USED</dt>
 ** <dd>This parameter returns the number of pages used out of the
-** [pagecache memory allocator] that was configured using 
+** [pagecache memory allocator] that was configured using
 ** [SQLITE_CONFIG_PAGECACHE].  The
 ** value returned is in pages, not in bytes.</dd>)^
 **
-** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] 
+** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]]
 ** ^(<dt>SQLITE_STATUS_PAGECACHE_OVERFLOW</dt>
 ** <dd>This parameter returns the number of bytes of page cache
 ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE]
 ** buffer and where forced to overflow to [sqlite3_malloc()].  The
 ** returned value includes allocations that overflowed because they
-** where too large (they were larger than the "sz" parameter to
+** were too large (they were larger than the "sz" parameter to
 ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because
 ** no space was left in the page cache.</dd>)^
 **
 ** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(<dt>SQLITE_STATUS_PAGECACHE_SIZE</dt>
 ** <dd>This parameter records the largest memory allocation request
 ** handed to the [pagecache memory allocator].  Only the value returned in the
-** *pHighwater parameter to [sqlite3_status()] is of interest.  
+** *pHighwater parameter to [sqlite3_status()] is of interest. 
 ** The value written into the *pCurrent parameter is undefined.</dd>)^
 **
 ** [[SQLITE_STATUS_SCRATCH_USED]] <dt>SQLITE_STATUS_SCRATCH_USED</dt>
@@ -8501,7 +8954,7 @@ int sqlite3_status64(
 ** <dd>No longer used.</dd>
 **
 ** [[SQLITE_STATUS_PARSER_STACK]] ^(<dt>SQLITE_STATUS_PARSER_STACK</dt>
-** <dd>The *pHighwater parameter records the deepest parser stack. 
+** <dd>The *pHighwater parameter records the deepest parser stack.
 ** The *pCurrent value is undefined.  The *pHighwater value is only
 ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].</dd>)^
 ** </dl>
@@ -8523,12 +8976,12 @@ int sqlite3_status64(
 ** CAPI3REF: Database Connection Status
 ** METHOD: sqlite3
 **
-** ^This interface is used to retrieve runtime status information 
+** ^This interface is used to retrieve runtime status information
 ** about a single [database connection].  ^The first argument is the
 ** database connection object to be interrogated.  ^The second argument
 ** is an integer constant, taken from the set of
 ** [SQLITE_DBSTATUS options], that
-** determines the parameter to interrogate.  The set of 
+** determines the parameter to interrogate.  The set of
 ** [SQLITE_DBSTATUS options] is likely
 ** to grow in future releases of SQLite.
 **
@@ -8540,9 +8993,18 @@ int sqlite3_status64(
 ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a
 ** non-zero [error code] on failure.
 **
+** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same
+** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H
+** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead
+** of pointers to 32-bit integers, which allows larger status values
+** to be returned.  If a status value exceeds 2,147,483,647 then
+** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64()
+** will return the full value.
+**
 ** See also: [sqlite3_status()] and [sqlite3_stmt_status()].
 */
 int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
+int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int);
 
 /*
 ** CAPI3REF: Status Parameters for database connections
@@ -8563,51 +9025,53 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** checked out.</dd>)^
 **
 ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_HIT</dt>
-** <dd>This parameter returns the number of malloc attempts that were 
+** <dd>This parameter returns the number of malloc attempts that were
 ** satisfied using lookaside memory. Only the high-water value is meaningful;
-** the current value is always zero.)^
+** the current value is always zero.</dd>)^
 **
 ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]]
 ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE</dt>
-** <dd>This parameter returns the number malloc attempts that might have
+** <dd>This parameter returns the number of malloc attempts that might have
 ** been satisfied using lookaside memory but failed due to the amount of
 ** memory requested being larger than the lookaside slot size.
 ** Only the high-water value is meaningful;
-** the current value is always zero.)^
+** the current value is always zero.</dd>)^
 **
 ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]]
 ** ^(<dt>SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL</dt>
-** <dd>This parameter returns the number malloc attempts that might have
+** <dd>This parameter returns the number of malloc attempts that might have
 ** been satisfied using lookaside memory but failed due to all lookaside
 ** memory already being in use.
 ** Only the high-water value is meaningful;
-** the current value is always zero.)^
+** the current value is always zero.</dd>)^
 **
 ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(<dt>SQLITE_DBSTATUS_CACHE_USED</dt>
 ** <dd>This parameter returns the approximate number of bytes of heap
 ** memory used by all pager caches associated with the database connection.)^
 ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0.
+** </dd>
 **
-** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] 
+** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]]
 ** ^(<dt>SQLITE_DBSTATUS_CACHE_USED_SHARED</dt>
 ** <dd>This parameter is similar to DBSTATUS_CACHE_USED, except that if a
 ** pager cache is shared between two or more connections the bytes of heap
 ** memory used by that pager cache is divided evenly between the attached
 ** connections.)^  In other words, if none of the pager caches associated
 ** with the database connection are shared, this request returns the same
-** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are
+** value as DBSTATUS_CACHE_USED. Or, if one or more of the pager caches are
 ** shared, the value returned by this call will be smaller than that returned
 ** by DBSTATUS_CACHE_USED. ^The highwater mark associated with
-** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0.
+** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0.</dd>
 **
 ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(<dt>SQLITE_DBSTATUS_SCHEMA_USED</dt>
 ** <dd>This parameter returns the approximate number of bytes of heap
 ** memory used to store the schema for all databases associated
-** with the connection - main, temp, and any [ATTACH]-ed databases.)^ 
+** with the connection - main, temp, and any [ATTACH]-ed databases.)^
 ** ^The full amount of memory used by the schemas is reported, even if the
 ** schema memory is shared with other database connections due to
 ** [shared cache mode] being enabled.
 ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0.
+** </dd>
 **
 ** [[SQLITE_DBSTATUS_STMT_USED]] ^(<dt>SQLITE_DBSTATUS_STMT_USED</dt>
 ** <dd>This parameter returns the approximate number of bytes of heap
@@ -8618,13 +9082,13 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 **
 ** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(<dt>SQLITE_DBSTATUS_CACHE_HIT</dt>
 ** <dd>This parameter returns the number of pager cache hits that have
-** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT 
+** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT
 ** is always 0.
 ** </dd>
 **
 ** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(<dt>SQLITE_DBSTATUS_CACHE_MISS</dt>
 ** <dd>This parameter returns the number of pager cache misses that have
-** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS 
+** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS
 ** is always 0.
 ** </dd>
 **
@@ -8637,6 +9101,10 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** If an IO or other error occurs while writing a page to disk, the effect
 ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The
 ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0.
+** <p>
+** ^(There is overlap between the quantities measured by this parameter
+** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL.
+** Resetting one will reduce the other.)^
 ** </dd>
 **
 ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(<dt>SQLITE_DBSTATUS_CACHE_SPILL</dt>
@@ -8644,7 +9112,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** been written to disk in the middle of a transaction due to the page
 ** cache overflowing. Transactions are more efficient if they are written
 ** to disk all at once. When pages spill mid-transaction, that introduces
-** additional overhead. This parameter can be used help identify
+** additional overhead. This parameter can be used to help identify
 ** inefficiencies that can be resolved by increasing the cache size.
 ** </dd>
 **
@@ -8652,6 +9120,18 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** <dd>This parameter returns zero for the current value if and only if
 ** all foreign key constraints (deferred or immediate) have been
 ** resolved.)^  ^The highwater mark is always 0.
+**
+** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(<dt>SQLITE_DBSTATUS_TEMPBUF_SPILL</dt>
+** <dd>^(This parameter returns the number of bytes written to temporary
+** files on disk that could have been kept in memory had sufficient memory
+** been available.  This value includes writes to intermediate tables that
+** are part of complex queries, external sorts that spill to disk, and
+** writes to TEMP tables.)^
+** ^The highwater mark is always 0.
+** <p>
+** ^(There is overlap between the quantities measured by this parameter
+** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE.
+** Resetting one will reduce the other.)^
 ** </dd>
 ** </dl>
 */
@@ -8668,7 +9148,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 #define SQLITE_DBSTATUS_DEFERRED_FKS        10
 #define SQLITE_DBSTATUS_CACHE_USED_SHARED   11
 #define SQLITE_DBSTATUS_CACHE_SPILL         12
-#define SQLITE_DBSTATUS_MAX                 12   /* Largest defined DBSTATUS */
+#define SQLITE_DBSTATUS_TEMPBUF_SPILL       13
+#define SQLITE_DBSTATUS_MAX                 13   /* Largest defined DBSTATUS */
 
 
 /*
@@ -8682,7 +9163,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** statements.  For example, if the number of table steps greatly exceeds
 ** the number of table searches or result rows, that would tend to indicate
 ** that the prepared statement is using a full table scan rather than
-** an index.  
+** an index. 
 **
 ** ^(This interface is used to retrieve and reset counter values from
 ** a [prepared statement].  The first argument is the prepared statement
@@ -8709,50 +9190,50 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
 ** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]] <dt>SQLITE_STMTSTATUS_FULLSCAN_STEP</dt>
 ** <dd>^This is the number of times that SQLite has stepped forward in
 ** a table as part of a full table scan.  Large numbers for this counter
-** may indicate opportunities for performance improvement through 
+** may indicate opportunities for performance improvement through
 ** careful use of indices.</dd>
 **
 ** [[SQLITE_STMTSTATUS_SORT]] <dt>SQLITE_STMTSTATUS_SORT</dt>
 ** <dd>^This is the number of sort operations that have occurred.
 ** A non-zero value in this counter may indicate an opportunity to
-** improvement performance through careful use of indices.</dd>
+** improve performance through careful use of indices.</dd>
 **
 ** [[SQLITE_STMTSTATUS_AUTOINDEX]] <dt>SQLITE_STMTSTATUS_AUTOINDEX</dt>
 ** <dd>^This is the number of rows inserted into transient indices that
 ** were created automatically in order to help joins run faster.
 ** A non-zero value in this counter may indicate an opportunity to
-** improvement performance by adding permanent indices that do not
+** improve performance by adding permanent indices that do not
 ** need to be reinitialized each time the statement is run.</dd>
 **
 ** [[SQLITE_STMTSTATUS_VM_STEP]] <dt>SQLITE_STMTSTATUS_VM_STEP</dt>
 ** <dd>^This is the number of virtual machine operations executed
 ** by the prepared statement if that number is less than or equal
-** to 2147483647.  The number of virtual machine operations can be 
+** to 2147483647.  The number of virtual machine operations can be
 ** used as a proxy for the total work done by the prepared statement.
 ** If the number of virtual machine operations exceeds 2147483647
-** then the value returned by this statement status code is undefined.
+** then the value returned by this statement status code is undefined.</dd>
 **
 ** [[SQLITE_STMTSTATUS_REPREPARE]] <dt>SQLITE_STMTSTATUS_REPREPARE</dt>
 ** <dd>^This is the number of times that the prepare statement has been
-** automatically regenerated due to schema changes or changes to 
-** [bound parameters] that might affect the query plan.
+** automatically regenerated due to schema changes or changes to
+** [bound parameters] that might affect the query plan.</dd>
 **
 ** [[SQLITE_STMTSTATUS_RUN]] <dt>SQLITE_STMTSTATUS_RUN</dt>
 ** <dd>^This is the number of times that the prepared statement has
 ** been run.  A single "run" for the purposes of this counter is one
 ** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()].
 ** The counter is incremented on the first [sqlite3_step()] call of each
-** cycle.
+** cycle.</dd>
 **
 ** [[SQLITE_STMTSTATUS_FILTER_MISS]]
-** [[SQLITE_STMTSTATUS_FILTER HIT]] 
+** [[SQLITE_STMTSTATUS_FILTER HIT]]
 ** <dt>SQLITE_STMTSTATUS_FILTER_HIT<br>
 ** SQLITE_STMTSTATUS_FILTER_MISS</dt>
 ** <dd>^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join
 ** step was bypassed because a Bloom filter returned not-found.  The
 ** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of
 ** times that the Bloom filter returned a find, and thus the join step
-** had to be processed as normal.
+** had to be processed as normal.</dd>
 **
 ** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt>
 ** <dd>^This is the approximate number of bytes of heap memory
@@ -8806,15 +9287,15 @@ struct sqlite3_pcache_page {
 ** KEYWORDS: {page cache}
 **
 ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can
-** register an alternative page cache implementation by passing in an 
+** register an alternative page cache implementation by passing in an
 ** instance of the sqlite3_pcache_methods2 structure.)^
-** In many applications, most of the heap memory allocated by 
+** In many applications, most of the heap memory allocated by
 ** SQLite is used for the page cache.
-** By implementing a 
+** By implementing a
 ** custom page cache using this API, an application can better control
-** the amount of memory consumed by SQLite, the way in which 
-** that memory is allocated and released, and the policies used to 
-** determine exactly which parts of a database file are cached and for 
+** the amount of memory consumed by SQLite, the way in which
+** that memory is allocated and released, and the policies used to
+** determine exactly which parts of a database file are cached and for
 ** how long.
 **
 ** The alternative page cache mechanism is an
@@ -8827,19 +9308,19 @@ struct sqlite3_pcache_page {
 ** [sqlite3_config()] returns.)^
 **
 ** [[the xInit() page cache method]]
-** ^(The xInit() method is called once for each effective 
+** ^(The xInit() method is called once for each effective
 ** call to [sqlite3_initialize()])^
 ** (usually only once during the lifetime of the process). ^(The xInit()
 ** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^
-** The intent of the xInit() method is to set up global data structures 
-** required by the custom page cache implementation. 
-** ^(If the xInit() method is NULL, then the 
+** The intent of the xInit() method is to set up global data structures
+** required by the custom page cache implementation.
+** ^(If the xInit() method is NULL, then the
 ** built-in default page cache is used instead of the application defined
 ** page cache.)^
 **
 ** [[the xShutdown() page cache method]]
 ** ^The xShutdown() method is called by [sqlite3_shutdown()].
-** It can be used to clean up 
+** It can be used to clean up
 ** any outstanding resources before process shutdown, if required.
 ** ^The xShutdown() method may be NULL.
 **
@@ -8857,9 +9338,9 @@ struct sqlite3_pcache_page {
 ** SQLite will typically create one cache instance for each open database file,
 ** though this is not guaranteed. ^The
 ** first parameter, szPage, is the size in bytes of the pages that must
-** be allocated by the cache.  ^szPage will always a power of two.  ^The
-** second parameter szExtra is a number of bytes of extra storage 
-** associated with each page cache entry.  ^The szExtra parameter will
+** be allocated by the cache.  ^szPage will always be a power of two.  ^The
+** second parameter szExtra is a number of bytes of extra storage
+** associated with each page cache entry.  ^The szExtra parameter will be
 ** a number less than 250.  SQLite will use the
 ** extra szExtra bytes on each page to store metadata about the underlying
 ** database page on disk.  The value passed into szExtra depends
@@ -8867,17 +9348,17 @@ struct sqlite3_pcache_page {
 ** ^The third argument to xCreate(), bPurgeable, is true if the cache being
 ** created will be used to cache database pages of a file stored on disk, or
 ** false if it is used for an in-memory database. The cache implementation
-** does not have to do anything special based with the value of bPurgeable;
+** does not have to do anything special based upon the value of bPurgeable;
 ** it is purely advisory.  ^On a cache where bPurgeable is false, SQLite will
 ** never invoke xUnpin() except to deliberately delete a page.
 ** ^In other words, calls to xUnpin() on a cache with bPurgeable set to
-** false will always have the "discard" flag set to true.  
-** ^Hence, a cache created with bPurgeable false will
+** false will always have the "discard" flag set to true. 
+** ^Hence, a cache created with bPurgeable set to false will
 ** never contain any unpinned pages.
 **
 ** [[the xCachesize() page cache method]]
 ** ^(The xCachesize() method may be called at any time by SQLite to set the
-** suggested maximum cache-size (number of pages stored by) the cache
+** suggested maximum cache-size (number of pages stored) for the cache
 ** instance passed as the first argument. This is the value configured using
 ** the SQLite "[PRAGMA cache_size]" command.)^  As with the bPurgeable
 ** parameter, the implementation is not required to do anything with this
@@ -8886,12 +9367,12 @@ struct sqlite3_pcache_page {
 ** [[the xPagecount() page cache methods]]
 ** The xPagecount() method must return the number of pages currently
 ** stored in the cache, both pinned and unpinned.
-** 
+**
 ** [[the xFetch() page cache methods]]
-** The xFetch() method locates a page in the cache and returns a pointer to 
+** The xFetch() method locates a page in the cache and returns a pointer to
 ** an sqlite3_pcache_page object associated with that page, or a NULL pointer.
 ** The pBuf element of the returned sqlite3_pcache_page object will be a
-** pointer to a buffer of szPage bytes used to store the content of a 
+** pointer to a buffer of szPage bytes used to store the content of a
 ** single database page.  The pExtra element of sqlite3_pcache_page will be
 ** a pointer to the szExtra bytes of extra storage that SQLite has requested
 ** for each entry in the page cache.
@@ -8904,12 +9385,12 @@ struct sqlite3_pcache_page {
 ** implementation must return a pointer to the page buffer with its content
 ** intact.  If the requested page is not already in the cache, then the
 ** cache implementation should use the value of the createFlag
-** parameter to help it determined what action to take:
+** parameter to help it determine what action to take:
 **
 ** <table border=1 width=85% align=center>
 ** <tr><th> createFlag <th> Behavior when page is not already in cache
 ** <tr><td> 0 <td> Do not allocate a new page.  Return NULL.
-** <tr><td> 1 <td> Allocate a new page if it easy and convenient to do so.
+** <tr><td> 1 <td> Allocate a new page if it is easy and convenient to do so.
 **                 Otherwise return NULL.
 ** <tr><td> 2 <td> Make every effort to allocate a new page.  Only return
 **                 NULL if allocating a new page is effectively impossible.
@@ -8926,12 +9407,12 @@ struct sqlite3_pcache_page {
 ** as its second argument.  If the third parameter, discard, is non-zero,
 ** then the page must be evicted from the cache.
 ** ^If the discard parameter is
-** zero, then the page may be discarded or retained at the discretion of
+** zero, then the page may be discarded or retained at the discretion of the
 ** page cache implementation. ^The page cache implementation
 ** may choose to evict unpinned pages at any time.
 **
-** The cache must not perform any reference counting. A single 
-** call to xUnpin() unpins the page regardless of the number of prior calls 
+** The cache must not perform any reference counting. A single
+** call to xUnpin() unpins the page regardless of the number of prior calls
 ** to xFetch().
 **
 ** [[the xRekey() page cache methods]]
@@ -8944,7 +9425,7 @@ struct sqlite3_pcache_page {
 ** When SQLite calls the xTruncate() method, the cache must discard all
 ** existing cache entries with page numbers (keys) greater than or equal
 ** to the value of the iLimit parameter passed to xTruncate(). If any
-** of these pages are pinned, they are implicitly unpinned, meaning that
+** of these pages are pinned, they become implicitly unpinned, meaning that
 ** they can be safely discarded.
 **
 ** [[the xDestroy() page cache method]]
@@ -8971,7 +9452,7 @@ struct sqlite3_pcache_methods2 {
   int (*xPagecount)(sqlite3_pcache*);
   sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag);
   void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard);
-  void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, 
+  void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*,
       unsigned oldKey, unsigned newKey);
   void (*xTruncate)(sqlite3_pcache*, unsigned iLimit);
   void (*xDestroy)(sqlite3_pcache*);
@@ -9016,7 +9497,7 @@ typedef struct sqlite3_backup sqlite3_backup;
 **
 ** The backup API copies the content of one database into another.
 ** It is useful either for creating backups of databases or
-** for copying in-memory databases to or from persistent files. 
+** for copying in-memory databases to or from persistent files.
 **
 ** See Also: [Using the SQLite Online Backup API]
 **
@@ -9027,36 +9508,36 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** ^Thus, the backup may be performed on a live source database without
 ** preventing other database connections from
 ** reading or writing to the source database while the backup is underway.
-** 
-** ^(To perform a backup operation: 
+**
+** ^(To perform a backup operation:
 **   <ol>
 **     <li><b>sqlite3_backup_init()</b> is called once to initialize the
-**         backup, 
-**     <li><b>sqlite3_backup_step()</b> is called one or more times to transfer 
+**         backup,
+**     <li><b>sqlite3_backup_step()</b> is called one or more times to transfer
 **         the data between the two databases, and finally
-**     <li><b>sqlite3_backup_finish()</b> is called to release all resources 
-**         associated with the backup operation. 
+**     <li><b>sqlite3_backup_finish()</b> is called to release all resources
+**         associated with the backup operation.
 **   </ol>)^
 ** There should be exactly one call to sqlite3_backup_finish() for each
 ** successful call to sqlite3_backup_init().
 **
 ** [[sqlite3_backup_init()]] <b>sqlite3_backup_init()</b>
 **
-** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the 
-** [database connection] associated with the destination database 
+** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the
+** [database connection] associated with the destination database
 ** and the database name, respectively.
 ** ^The database name is "main" for the main database, "temp" for the
 ** temporary database, or the name specified after the AS keyword in
 ** an [ATTACH] statement for an attached database.
-** ^The S and M arguments passed to 
+** ^The S and M arguments passed to
 ** sqlite3_backup_init(D,N,S,M) identify the [database connection]
 ** and database name of the source database, respectively.
 ** ^The source and destination [database connections] (parameters S and D)
 ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with
 ** an error.
 **
-** ^A call to sqlite3_backup_init() will fail, returning NULL, if 
-** there is already a read or read-write transaction open on the 
+** ^A call to sqlite3_backup_init() will fail, returning NULL, if
+** there is already a read or read-write transaction open on the
 ** destination database.
 **
 ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is
@@ -9068,14 +9549,14 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** ^A successful call to sqlite3_backup_init() returns a pointer to an
 ** [sqlite3_backup] object.
 ** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and
-** sqlite3_backup_finish() functions to perform the specified backup 
+** sqlite3_backup_finish() functions to perform the specified backup
 ** operation.
 **
 ** [[sqlite3_backup_step()]] <b>sqlite3_backup_step()</b>
 **
-** ^Function sqlite3_backup_step(B,N) will copy up to N pages between 
+** ^Function sqlite3_backup_step(B,N) will copy up to N pages between
 ** the source and destination databases specified by [sqlite3_backup] object B.
-** ^If N is negative, all remaining source pages are copied. 
+** ^If N is negative, all remaining source pages are copied.
 ** ^If sqlite3_backup_step(B,N) successfully copies N pages and there
 ** are still more pages to be copied, then the function returns [SQLITE_OK].
 ** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages
@@ -9097,8 +9578,8 @@ typedef struct sqlite3_backup sqlite3_backup;
 **
 ** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then
 ** the [sqlite3_busy_handler | busy-handler function]
-** is invoked (if one is specified). ^If the 
-** busy-handler returns non-zero before the lock is available, then 
+** is invoked (if one is specified). ^If the
+** busy-handler returns non-zero before the lock is available, then
 ** [SQLITE_BUSY] is returned to the caller. ^In this case the call to
 ** sqlite3_backup_step() can be retried later. ^If the source
 ** [database connection]
@@ -9106,15 +9587,15 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this
 ** case the call to sqlite3_backup_step() can be retried later on. ^(If
 ** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or
-** [SQLITE_READONLY] is returned, then 
-** there is no point in retrying the call to sqlite3_backup_step(). These 
-** errors are considered fatal.)^  The application must accept 
-** that the backup operation has failed and pass the backup operation handle 
+** [SQLITE_READONLY] is returned, then
+** there is no point in retrying the call to sqlite3_backup_step(). These
+** errors are considered fatal.)^  The application must accept
+** that the backup operation has failed and pass the backup operation handle
 ** to the sqlite3_backup_finish() to release associated resources.
 **
 ** ^The first call to sqlite3_backup_step() obtains an exclusive lock
-** on the destination file. ^The exclusive lock is not released until either 
-** sqlite3_backup_finish() is called or the backup operation is complete 
+** on the destination file. ^The exclusive lock is not released until either
+** sqlite3_backup_finish() is called or the backup operation is complete
 ** and sqlite3_backup_step() returns [SQLITE_DONE].  ^Every call to
 ** sqlite3_backup_step() obtains a [shared lock] on the source database that
 ** lasts for the duration of the sqlite3_backup_step() call.
@@ -9123,25 +9604,25 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** through the backup process.  ^If the source database is modified by an
 ** external process or via a database connection other than the one being
 ** used by the backup operation, then the backup will be automatically
-** restarted by the next call to sqlite3_backup_step(). ^If the source 
-** database is modified by the using the same database connection as is used
+** restarted by the next call to sqlite3_backup_step(). ^If the source
+** database is modified by using the same database connection as is used
 ** by the backup operation, then the backup database is automatically
 ** updated at the same time.
 **
 ** [[sqlite3_backup_finish()]] <b>sqlite3_backup_finish()</b>
 **
-** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the 
+** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the
 ** application wishes to abandon the backup operation, the application
 ** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish().
 ** ^The sqlite3_backup_finish() interfaces releases all
-** resources associated with the [sqlite3_backup] object. 
+** resources associated with the [sqlite3_backup] object.
 ** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any
 ** active write-transaction on the destination database is rolled back.
 ** The [sqlite3_backup] object is invalid
 ** and may not be used following a call to sqlite3_backup_finish().
 **
 ** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no
-** sqlite3_backup_step() errors occurred, regardless or whether or not
+** sqlite3_backup_step() errors occurred, regardless of whether or not
 ** sqlite3_backup_step() completed.
 ** ^If an out-of-memory condition or IO error occurred during any prior
 ** sqlite3_backup_step() call on the same [sqlite3_backup] object, then
@@ -9174,8 +9655,8 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** connections, then the source database connection may be used concurrently
 ** from within other threads.
 **
-** However, the application must guarantee that the destination 
-** [database connection] is not passed to any other API (by any thread) after 
+** However, the application must guarantee that the destination
+** [database connection] is not passed to any other API (by any thread) after
 ** sqlite3_backup_init() is called and before the corresponding call to
 ** sqlite3_backup_finish().  SQLite does not currently check to see
 ** if the application incorrectly accesses the destination [database connection]
@@ -9186,16 +9667,26 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** If running in [shared cache mode], the application must
 ** guarantee that the shared cache used by the destination database
 ** is not accessed while the backup is running. In practice this means
-** that the application must guarantee that the disk file being 
+** that the application must guarantee that the disk file being
 ** backed up to is not accessed by any connection within the process,
 ** not just the specific connection that was passed to sqlite3_backup_init().
 **
-** The [sqlite3_backup] object itself is partially threadsafe. Multiple 
+** The [sqlite3_backup] object itself is partially threadsafe. Multiple
 ** threads may safely make multiple concurrent calls to sqlite3_backup_step().
 ** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount()
 ** APIs are not strictly speaking threadsafe. If they are invoked at the
 ** same time as another thread is invoking sqlite3_backup_step() it is
 ** possible that they return invalid values.
+**
+** <b>Alternatives To Using The Backup API</b>
+**
+** Other techniques for safely creating a consistent backup of an SQLite
+** database include:
+**
+** <ul>
+** <li> The [VACUUM INTO] command.
+** <li> The [sqlite3_rsync] utility program.
+** </ul>
 */
 sqlite3_backup *sqlite3_backup_init(
   sqlite3 *pDest,                        /* Destination database handle */
@@ -9215,8 +9706,8 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 ** ^When running in shared-cache mode, a database operation may fail with
 ** an [SQLITE_LOCKED] error if the required locks on the shared-cache or
 ** individual tables within the shared-cache cannot be obtained. See
-** [SQLite Shared-Cache Mode] for a description of shared-cache locking. 
-** ^This API may be used to register a callback that SQLite will invoke 
+** [SQLite Shared-Cache Mode] for a description of shared-cache locking.
+** ^This API may be used to register a callback that SQLite will invoke
 ** when the connection currently holding the required lock relinquishes it.
 ** ^This API is only available if the library was compiled with the
 ** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined.
@@ -9224,16 +9715,16 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 ** See Also: [Using the SQLite Unlock Notification Feature].
 **
 ** ^Shared-cache locks are released when a database connection concludes
-** its current transaction, either by committing it or rolling it back. 
+** its current transaction, either by committing it or rolling it back.
 **
 ** ^When a connection (known as the blocked connection) fails to obtain a
 ** shared-cache lock and SQLITE_LOCKED is returned to the caller, the
 ** identity of the database connection (the blocking connection) that
-** has locked the required resource is stored internally. ^After an 
+** has locked the required resource is stored internally. ^After an
 ** application receives an SQLITE_LOCKED error, it may call the
-** sqlite3_unlock_notify() method with the blocked connection handle as 
+** sqlite3_unlock_notify() method with the blocked connection handle as
 ** the first argument to register for a callback that will be invoked
-** when the blocking connections current transaction is concluded. ^The
+** when the blocking connection's current transaction is concluded. ^The
 ** callback is invoked from within the [sqlite3_step] or [sqlite3_close]
 ** call that concludes the blocking connection's transaction.
 **
@@ -9245,15 +9736,15 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 **
 ** ^If the blocked connection is attempting to obtain a write-lock on a
 ** shared-cache table, and more than one other connection currently holds
-** a read-lock on the same table, then SQLite arbitrarily selects one of 
+** a read-lock on the same table, then SQLite arbitrarily selects one of
 ** the other connections to use as the blocking connection.
 **
-** ^(There may be at most one unlock-notify callback registered by a 
+** ^(There may be at most one unlock-notify callback registered by a
 ** blocked connection. If sqlite3_unlock_notify() is called when the
 ** blocked connection already has a registered unlock-notify callback,
 ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is
 ** called with a NULL pointer as its second argument, then any existing
-** unlock-notify callback is canceled. ^The blocked connections 
+** unlock-notify callback is canceled. ^The blocked connection's
 ** unlock-notify callback may also be canceled by closing the blocked
 ** connection using [sqlite3_close()].
 **
@@ -9266,7 +9757,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 **
 ** <b>Callback Invocation Details</b>
 **
-** When an unlock-notify callback is registered, the application provides a 
+** When an unlock-notify callback is registered, the application provides a
 ** single void* pointer that is passed to the callback when it is invoked.
 ** However, the signature of the callback function allows SQLite to pass
 ** it an array of void* context pointers. The first argument passed to
@@ -9279,12 +9770,12 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 ** same callback function, then instead of invoking the callback function
 ** multiple times, it is invoked once with the set of void* context pointers
 ** specified by the blocked connections bundled together into an array.
-** This gives the application an opportunity to prioritize any actions 
+** This gives the application an opportunity to prioritize any actions
 ** related to the set of unblocked database connections.
 **
 ** <b>Deadlock Detection</b>
 **
-** Assuming that after registering for an unlock-notify callback a 
+** Assuming that after registering for an unlock-notify callback a
 ** database waits for the callback to be issued before taking any further
 ** action (a reasonable assumption), then using this API may cause the
 ** application to deadlock. For example, if connection X is waiting for
@@ -9307,7 +9798,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 **
 ** <b>The "DROP TABLE" Exception</b>
 **
-** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost 
+** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost
 ** always appropriate to call sqlite3_unlock_notify(). There is however,
 ** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement,
 ** SQLite checks if there are any currently executing SELECT statements
@@ -9320,7 +9811,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p);
 ** One way around this problem is to check the extended error code returned
 ** by an sqlite3_step() call. ^(If there is a blocking connection, then the
 ** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in
-** the special "DROP TABLE/INDEX" case, the extended error code is just 
+** the special "DROP TABLE/INDEX" case, the extended error code is just
 ** SQLITE_LOCKED.)^
 */
 int sqlite3_unlock_notify(
@@ -9411,8 +9902,8 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...);
 ** ^The [sqlite3_wal_hook()] function is used to register a callback that
 ** is invoked each time data is committed to a database in wal mode.
 **
-** ^(The callback is invoked by SQLite after the commit has taken place and 
-** the associated write-lock on the database released)^, so the implementation 
+** ^(The callback is invoked by SQLite after the commit has taken place and
+** the associated write-lock on the database released)^, so the implementation
 ** may read, write or [checkpoint] the database as required.
 **
 ** ^The first parameter passed to the callback function when it is invoked
@@ -9423,7 +9914,7 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...);
 ** is the number of pages currently in the write-ahead log file,
 ** including those that were just committed.
 **
-** The callback function should normally return [SQLITE_OK].  ^If an error
+** ^The callback function should normally return [SQLITE_OK].  ^If an error
 ** code is returned, that error will propagate back up through the
 ** SQLite code base to cause the statement that provoked the callback
 ** to report an error, though the commit will have still occurred. If the
@@ -9431,16 +9922,29 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...);
 ** that does not correspond to any valid SQLite error code, the results
 ** are undefined.
 **
-** A single database handle may have at most a single write-ahead log callback 
-** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any
-** previously registered write-ahead log callback. ^The return value is
-** a copy of the third parameter from the previous call, if any, or 0.
-** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the
-** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will
-** overwrite any prior [sqlite3_wal_hook()] settings.
+** ^A single database handle may have at most a single write-ahead log
+** callback registered at one time. ^Calling [sqlite3_wal_hook()]
+** replaces the default behavior or previously registered write-ahead
+** log callback.
+**
+** ^The return value is a copy of the third parameter from the
+** previous call, if any, or 0.
+**
+** ^The [sqlite3_wal_autocheckpoint()] interface and the
+** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and
+** will overwrite any prior [sqlite3_wal_hook()] settings.
+**
+** ^If a write-ahead log callback is set using this function then
+** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint]
+** should be invoked periodically to keep the write-ahead log file
+** from growing without bound.
+**
+** ^Passing a NULL pointer for the callback disables automatic
+** checkpointing entirely. To re-enable the default behavior, call
+** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint].
 */
 void *sqlite3_wal_hook(
-  sqlite3*, 
+  sqlite3*,
   int(*)(void *,sqlite3*,const char*,int),
   void*
 );
@@ -9453,8 +9957,8 @@ void *sqlite3_wal_hook(
 ** [sqlite3_wal_hook()] that causes any database on [database connection] D
 ** to automatically [checkpoint]
 ** after committing a transaction if there are N or
-** more frames in the [write-ahead log] file.  ^Passing zero or 
-** a negative value as the nFrame parameter disables automatic
+** more frames in the [write-ahead log] file.  ^Passing zero or
+** a negative value as the N parameter disables automatic
 ** checkpoints entirely.
 **
 ** ^The callback registered by this function replaces any existing callback
@@ -9470,9 +9974,10 @@ void *sqlite3_wal_hook(
 **
 ** ^Every new [database connection] defaults to having the auto-checkpoint
 ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT]
-** pages.  The use of this interface
-** is only necessary if the default setting is found to be suboptimal
-** for a particular application.
+** pages.
+**
+** ^The use of this interface is only necessary if the default setting
+** is found to be suboptimal for a particular application.
 */
 int sqlite3_wal_autocheckpoint(sqlite3 *db, int N);
 
@@ -9483,7 +9988,7 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N);
 ** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to
 ** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^
 **
-** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the 
+** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the
 ** [write-ahead log] for database X on [database connection] D to be
 ** transferred into the database file and for the write-ahead log to
 ** be reset.  See the [checkpointing] documentation for addition
@@ -9509,10 +10014,10 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb);
 **
 ** <dl>
 ** <dt>SQLITE_CHECKPOINT_PASSIVE<dd>
-**   ^Checkpoint as many frames as possible without waiting for any database 
-**   readers or writers to finish, then sync the database file if all frames 
+**   ^Checkpoint as many frames as possible without waiting for any database
+**   readers or writers to finish, then sync the database file if all frames
 **   in the log were checkpointed. ^The [busy-handler callback]
-**   is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode.  
+**   is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. 
 **   ^On the other hand, passive mode might leave the checkpoint unfinished
 **   if there are concurrent readers or writers.
 **
@@ -9526,9 +10031,9 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb);
 **
 ** <dt>SQLITE_CHECKPOINT_RESTART<dd>
 **   ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition
-**   that after checkpointing the log file it blocks (calls the 
+**   that after checkpointing the log file it blocks (calls the
 **   [busy-handler callback])
-**   until all readers are reading from the database file only. ^This ensures 
+**   until all readers are reading from the database file only. ^This ensures
 **   that the next writer will restart the log file from the beginning.
 **   ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new
 **   database writer attempts while it is pending, but does not impede readers.
@@ -9537,6 +10042,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb);
 **   ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the
 **   addition that it also truncates the log file to zero bytes just prior
 **   to a successful return.
+**
+** <dt>SQLITE_CHECKPOINT_NOOP<dd>
+**   ^This mode always checkpoints zero frames. The only reason to invoke
+**   a NOOP checkpoint is to access the values returned by
+**   sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt.
 ** </dl>
 **
 ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in
@@ -9550,31 +10060,31 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb);
 ** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero.
 **
 ** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If
-** any other process is running a checkpoint operation at the same time, the 
-** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a 
+** any other process is running a checkpoint operation at the same time, the
+** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a
 ** busy-handler configured, it will not be invoked in this case.
 **
-** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the 
+** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the
 ** exclusive "writer" lock on the database file. ^If the writer lock cannot be
 ** obtained immediately, and a busy-handler is configured, it is invoked and
 ** the writer lock retried until either the busy-handler returns 0 or the lock
 ** is successfully obtained. ^The busy-handler is also invoked while waiting for
 ** database readers as described above. ^If the busy-handler returns 0 before
 ** the writer lock is obtained or while waiting for database readers, the
-** checkpoint operation proceeds from that point in the same way as 
-** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible 
+** checkpoint operation proceeds from that point in the same way as
+** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible
 ** without blocking any further. ^SQLITE_BUSY is returned in this case.
 **
 ** ^If parameter zDb is NULL or points to a zero length string, then the
-** specified operation is attempted on all WAL databases [attached] to 
+** specified operation is attempted on all WAL databases [attached] to
 ** [database connection] db.  In this case the
-** values written to output parameters *pnLog and *pnCkpt are undefined. ^If 
-** an SQLITE_BUSY error is encountered when processing one or more of the 
-** attached WAL databases, the operation is still attempted on any remaining 
-** attached databases and SQLITE_BUSY is returned at the end. ^If any other 
-** error occurs while processing an attached database, processing is abandoned 
-** and the error code is returned to the caller immediately. ^If no error 
-** (SQLITE_BUSY or otherwise) is encountered while processing the attached 
+** values written to output parameters *pnLog and *pnCkpt are undefined. ^If
+** an SQLITE_BUSY error is encountered when processing one or more of the
+** attached WAL databases, the operation is still attempted on any remaining
+** attached databases and SQLITE_BUSY is returned at the end. ^If any other
+** error occurs while processing an attached database, processing is abandoned
+** and the error code is returned to the caller immediately. ^If no error
+** (SQLITE_BUSY or otherwise) is encountered while processing the attached
 ** databases, SQLITE_OK is returned.
 **
 ** ^If database zDb is the name of an attached database that is not in WAL
@@ -9607,6 +10117,7 @@ int sqlite3_wal_checkpoint_v2(
 ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the
 ** meaning of each of these checkpoint modes.
 */
+#define SQLITE_CHECKPOINT_NOOP    -1  /* Do no work at all */
 #define SQLITE_CHECKPOINT_PASSIVE  0  /* Do as much as possible w/o blocking */
 #define SQLITE_CHECKPOINT_FULL     1  /* Wait for writers, then checkpoint */
 #define SQLITE_CHECKPOINT_RESTART  2  /* Like FULL but wait for readers */
@@ -9634,7 +10145,7 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 
 /*
 ** CAPI3REF: Virtual Table Configuration Options
-** KEYWORDS: {virtual table configuration options} 
+** KEYWORDS: {virtual table configuration options}
 ** KEYWORDS: {virtual table configuration option}
 **
 ** These macros define the various options to the
@@ -9651,33 +10162,33 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 ** support constraints.  In this configuration (which is the default) if
 ** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire
 ** statement is rolled back as if [ON CONFLICT | OR ABORT] had been
-** specified as part of the users SQL statement, regardless of the actual
+** specified as part of the user's SQL statement, regardless of the actual
 ** ON CONFLICT mode specified.
 **
 ** If X is non-zero, then the virtual table implementation guarantees
 ** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before
 ** any modifications to internal or persistent data structures have been made.
-** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite 
+** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite
 ** is able to roll back a statement or database transaction, and abandon
-** or continue processing the current SQL statement as appropriate. 
+** or continue processing the current SQL statement as appropriate.
 ** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns
 ** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode
 ** had been ABORT.
 **
 ** Virtual table implementations that are required to handle OR REPLACE
-** must do so within the [xUpdate] method. If a call to the 
-** [sqlite3_vtab_on_conflict()] function indicates that the current ON 
-** CONFLICT policy is REPLACE, the virtual table implementation should 
+** must do so within the [xUpdate] method. If a call to the
+** [sqlite3_vtab_on_conflict()] function indicates that the current ON
+** CONFLICT policy is REPLACE, the virtual table implementation should
 ** silently replace the appropriate rows within the xUpdate callback and
 ** return SQLITE_OK. Or, if this is not possible, it may return
-** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT 
+** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT
 ** constraint handling.
 ** </dd>
 **
 ** [[SQLITE_VTAB_DIRECTONLY]]<dt>SQLITE_VTAB_DIRECTONLY</dt>
 ** <dd>Calls of the form
 ** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the
-** the [xConnect] or [xCreate] methods of a [virtual table] implmentation
+** the [xConnect] or [xCreate] methods of a [virtual table] implementation
 ** prohibits that virtual table from being used from within triggers and
 ** views.
 ** </dd>
@@ -9685,7 +10196,7 @@ int sqlite3_vtab_config(sqlite3*, int op, ...);
 ** [[SQLITE_VTAB_INNOCUOUS]]<dt>SQLITE_VTAB_INNOCUOUS</dt>
 ** <dd>Calls of the form
 ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the
-** the [xConnect] or [xCreate] methods of a [virtual table] implementation
+** [xConnect] or [xCreate] methods of a [virtual table] implementation
 ** identify that virtual table as being safe to use from within triggers
 ** and views.  Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the
 ** virtual table can do no serious harm even if it is controlled by a
@@ -9813,26 +10324,47 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
 ** <li value="2"><p>
 ** ^(If the sqlite3_vtab_distinct() interface returns 2, that means
 ** that the query planner does not need the rows returned in any particular
-** order, as long as rows with the same values in all "aOrderBy" columns
-** are adjacent.)^  ^(Furthermore, only a single row for each particular
-** combination of values in the columns identified by the "aOrderBy" field
-** needs to be returned.)^  ^It is always ok for two or more rows with the same
-** values in all "aOrderBy" columns to be returned, as long as all such rows
-** are adjacent.  ^The virtual table may, if it chooses, omit extra rows
-** that have the same value for all columns identified by "aOrderBy".
-** ^However omitting the extra rows is optional.
+** order, as long as rows with the same values in all columns identified
+** by "aOrderBy" are adjacent.)^  ^(Furthermore, when two or more rows 
+** contain the same values for all columns identified by "colUsed", all but
+** one such row may optionally be omitted from the result.)^
+** The virtual table is not required to omit rows that are duplicates
+** over the "colUsed" columns, but if the virtual table can do that without
+** too much extra effort, it could potentially help the query to run faster.
 ** This mode is used for a DISTINCT query.
 ** <li value="3"><p>
-** ^(If the sqlite3_vtab_distinct() interface returns 3, that means
-** that the query planner needs only distinct rows but it does need the
-** rows to be sorted.)^ ^The virtual table implementation is free to omit
-** rows that are identical in all aOrderBy columns, if it wants to, but
-** it is not required to omit any rows.  This mode is used for queries
+** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the
+** virtual table must return rows in the order defined by "aOrderBy" as
+** if the sqlite3_vtab_distinct() interface had returned 0.  However if
+** two or more rows in the result have the same values for all columns
+** identified by "colUsed", then all but one such row may optionally be
+** omitted.)^  Like when the return value is 2, the virtual table
+** is not required to omit rows that are duplicates over the "colUsed"
+** columns, but if the virtual table can do that without
+** too much extra effort, it could potentially help the query to run faster.
+** This mode is used for queries
 ** that have both DISTINCT and ORDER BY clauses.
 ** </ol>
 **
+** <p>The following table summarizes the conditions under which the
+** virtual table is allowed to set the "orderByConsumed" flag based on
+** the value returned by sqlite3_vtab_distinct().  This table is a
+** restatement of the previous four paragraphs:
+**
+** <table border=1 cellspacing=0 cellpadding=10 width="90%">
+** <tr>
+** <td valign="top">sqlite3_vtab_distinct() return value
+** <td valign="top">Rows are returned in aOrderBy order
+** <td valign="top">Rows with the same value in all aOrderBy columns are adjacent
+** <td valign="top">Duplicates over all colUsed columns may be omitted
+** <tr><td>0<td>yes<td>yes<td>no
+** <tr><td>1<td>no<td>yes<td>no
+** <tr><td>2<td>no<td>yes<td>yes
+** <tr><td>3<td>yes<td>yes<td>yes
+** </table>
+**
 ** ^For the purposes of comparing virtual table output values to see if the
-** values are same value for sorting purposes, two NULL values are considered
+** values are the same value for sorting purposes, two NULL values are considered
 ** to be the same.  In other words, the comparison operator is "IS"
 ** (or "IS NOT DISTINCT FROM") and not "==".
 **
@@ -9842,7 +10374,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
 **
 ** ^A virtual table implementation is always free to return rows in any order
 ** it wants, as long as the "orderByConsumed" flag is not set.  ^When the
-** the "orderByConsumed" flag is unset, the query planner will add extra
+** "orderByConsumed" flag is unset, the query planner will add extra
 ** [bytecode] to ensure that the final results returned by the SQL query are
 ** ordered correctly.  The use of the "orderByConsumed" flag and the
 ** sqlite3_vtab_distinct() interface is merely an optimization.  ^Careful
@@ -9857,7 +10389,7 @@ int sqlite3_vtab_distinct(sqlite3_index_info*);
 /*
 ** CAPI3REF: Identify and handle IN constraints in xBestIndex
 **
-** This interface may only be used from within an 
+** This interface may only be used from within an
 ** [xBestIndex|xBestIndex() method] of a [virtual table] implementation.
 ** The result of invoking this interface from any other context is
 ** undefined and probably harmful.
@@ -9867,7 +10399,7 @@ int sqlite3_vtab_distinct(sqlite3_index_info*);
 ** communicated to the xBestIndex method as a
 ** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^  If xBestIndex wants to use
 ** this constraint, it must set the corresponding
-** aConstraintUsage[].argvIndex to a postive integer.  ^(Then, under
+** aConstraintUsage[].argvIndex to a positive integer.  ^(Then, under
 ** the usual mode of handling IN operators, SQLite generates [bytecode]
 ** that invokes the [xFilter|xFilter() method] once for each value
 ** on the right-hand side of the IN operator.)^  Thus the virtual table
@@ -9916,7 +10448,7 @@ int sqlite3_vtab_distinct(sqlite3_index_info*);
 ** <li><p> The last call to sqlite3_vtab_in(P,N,F) for which F was
 ** non-negative had F>=1.
 ** </ol>)^
-** 
+**
 ** ^If either or both of the conditions above are false, then SQLite uses
 ** the traditional one-at-a-time processing strategy for the IN constraint.
 ** ^If both conditions are true, then the argvIndex-th parameter to the
@@ -9939,7 +10471,7 @@ int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle);
 ** sqlite3_vtab_in_next(X,P) should be one of the parameters to the
 ** xFilter method which invokes these routines, and specifically
 ** a parameter that was previously selected for all-at-once IN constraint
-** processing use the [sqlite3_vtab_in()] interface in the
+** processing using the [sqlite3_vtab_in()] interface in the
 ** [xBestIndex|xBestIndex method].  ^(If the X parameter is not
 ** an xFilter argument that was selected for all-at-once IN constraint
 ** processing, then these routines return [SQLITE_ERROR].)^
@@ -9954,7 +10486,7 @@ int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle);
 ** &nbsp;  ){
 ** &nbsp;    // do something with pVal
 ** &nbsp;  }
-** &nbsp;  if( rc!=SQLITE_OK ){
+** &nbsp;  if( rc!=SQLITE_DONE ){
 ** &nbsp;    // an error has occurred
 ** &nbsp;  }
 ** </pre></blockquote>)^
@@ -9991,10 +10523,10 @@ int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut);
 ** that constraint if the right-hand operand is known.  ^If the
 ** right-hand operand is not known, then *V is set to a NULL pointer.
 ** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if
-** and only if *V is set to a value.  ^The sqlite3_vtab_rhs_value(P,J,V) 
+** and only if *V is set to a value.  ^The sqlite3_vtab_rhs_value(P,J,V)
 ** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th
 ** constraint is not available.  ^The sqlite3_vtab_rhs_value() interface
-** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if
+** can return a result code other than SQLITE_OK or SQLITE_NOTFOUND if
 ** something goes wrong.
 **
 ** The sqlite3_vtab_rhs_value() interface is usually only successful if
@@ -10022,8 +10554,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal);
 ** KEYWORDS: {conflict resolution mode}
 **
 ** These constants are returned by [sqlite3_vtab_on_conflict()] to
-** inform a [virtual table] implementation what the [ON CONFLICT] mode
-** is for the SQL statement being evaluated.
+** inform a [virtual table] implementation of the [ON CONFLICT] mode
+** for the SQL statement being evaluated.
 **
 ** Note that the [SQLITE_IGNORE] constant is also used as a potential
 ** return value from the [sqlite3_set_authorizer()] callback and that
@@ -10063,39 +10595,39 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal);
 ** [[SQLITE_SCANSTAT_EST]] <dt>SQLITE_SCANSTAT_EST</dt>
 ** <dd>^The "double" variable pointed to by the V parameter will be set to the
 ** query planner's estimate for the average number of rows output from each
-** iteration of the X-th loop.  If the query planner's estimates was accurate,
+** iteration of the X-th loop.  If the query planner's estimate was accurate,
 ** then this value will approximate the quotient NVISIT/NLOOP and the
 ** product of this value for all prior loops with the same SELECTID will
-** be the NLOOP value for the current loop.
+** be the NLOOP value for the current loop.</dd>
 **
 ** [[SQLITE_SCANSTAT_NAME]] <dt>SQLITE_SCANSTAT_NAME</dt>
 ** <dd>^The "const char *" variable pointed to by the V parameter will be set
 ** to a zero-terminated UTF-8 string containing the name of the index or table
-** used for the X-th loop.
+** used for the X-th loop.</dd>
 **
 ** [[SQLITE_SCANSTAT_EXPLAIN]] <dt>SQLITE_SCANSTAT_EXPLAIN</dt>
 ** <dd>^The "const char *" variable pointed to by the V parameter will be set
 ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN]
-** description for the X-th loop.
+** description for the X-th loop.</dd>
 **
 ** [[SQLITE_SCANSTAT_SELECTID]] <dt>SQLITE_SCANSTAT_SELECTID</dt>
 ** <dd>^The "int" variable pointed to by the V parameter will be set to the
 ** id for the X-th query plan element. The id value is unique within the
-** statement. The select-id is the same value as is output in the first 
-** column of an [EXPLAIN QUERY PLAN] query.
+** statement. The select-id is the same value as is output in the first
+** column of an [EXPLAIN QUERY PLAN] query.</dd>
 **
 ** [[SQLITE_SCANSTAT_PARENTID]] <dt>SQLITE_SCANSTAT_PARENTID</dt>
 ** <dd>The "int" variable pointed to by the V parameter will be set to the
-** the id of the parent of the current query element, if applicable, or
+** id of the parent of the current query element, if applicable, or
 ** to zero if the query element has no parent. This is the same value as
-** returned in the second column of an [EXPLAIN QUERY PLAN] query. 
+** returned in the second column of an [EXPLAIN QUERY PLAN] query.</dd>
 **
 ** [[SQLITE_SCANSTAT_NCYCLE]] <dt>SQLITE_SCANSTAT_NCYCLE</dt>
 ** <dd>The sqlite3_int64 output value is set to the number of cycles,
 ** according to the processor time-stamp counter, that elapsed while the
 ** query element was being processed. This value is not available for
 ** all query elements - if it is unavailable the output variable is
-** set to -1.
+** set to -1.</dd>
 ** </dl>
 */
 #define SQLITE_SCANSTAT_NLOOP    0
@@ -10122,23 +10654,23 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal);
 **
 ** The "iScanStatusOp" parameter determines which status information to return.
 ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior
-** of this interface is undefined. ^The requested measurement is written into 
+** of this interface is undefined. ^The requested measurement is written into
 ** a variable pointed to by the "pOut" parameter.
 **
 ** The "flags" parameter must be passed a mask of flags. At present only
 ** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX
-** is specified, then status information is available for all elements 
+** is specified, then status information is available for all elements
 ** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If
 ** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements
 ** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of
 ** the EXPLAIN QUERY PLAN output) are available. Invoking API
-** sqlite3_stmt_scanstatus() is equivalent to calling 
+** sqlite3_stmt_scanstatus() is equivalent to calling
 ** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter.
-** 
+**
 ** Parameter "idx" identifies the specific query element to retrieve statistics
-** for. Query elements are numbered starting from zero. A value of -1 may be
-** to query for statistics regarding the entire query. ^If idx is out of range
-** - less than -1 or greater than or equal to the total number of query 
+** for. Query elements are numbered starting from zero. A value of -1 may
+** retrieve statistics for the entire query. ^If idx is out of range
+** - less than -1 or greater than or equal to the total number of query
 ** elements used to implement the statement - a non-zero value is returned and
 ** the variable that pOut points to is unchanged.
 **
@@ -10149,14 +10681,14 @@ int sqlite3_stmt_scanstatus(
   int idx,                  /* Index of loop to report on */
   int iScanStatusOp,        /* Information desired.  SQLITE_SCANSTAT_* */
   void *pOut                /* Result written here */
-);     
+);
 int sqlite3_stmt_scanstatus_v2(
   sqlite3_stmt *pStmt,      /* Prepared statement for which info desired */
   int idx,                  /* Index of loop to report on */
   int iScanStatusOp,        /* Information desired.  SQLITE_SCANSTAT_* */
   int flags,                /* Mask of flags defined below */
   void *pOut                /* Result written here */
-);     
+);
 
 /*
 ** CAPI3REF: Prepared Statement Scan Status
@@ -10180,16 +10712,16 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*);
 ** METHOD: sqlite3
 **
 ** ^If a write-transaction is open on [database connection] D when the
-** [sqlite3_db_cacheflush(D)] interface invoked, any dirty
-** pages in the pager-cache that are not currently in use are written out 
+** [sqlite3_db_cacheflush(D)] interface is invoked, any dirty
+** pages in the pager-cache that are not currently in use are written out
 ** to disk. A dirty page may be in use if a database cursor created by an
 ** active SQL statement is reading from it, or if it is page 1 of a database
 ** file (page 1 is always "in use").  ^The [sqlite3_db_cacheflush(D)]
 ** interface flushes caches for all schemas - "main", "temp", and
 ** any [attached] databases.
 **
-** ^If this function needs to obtain extra database locks before dirty pages 
-** can be flushed to disk, it does so. ^If those locks cannot be obtained 
+** ^If this function needs to obtain extra database locks before dirty pages
+** can be flushed to disk, it does so. ^If those locks cannot be obtained
 ** immediately and there is a busy-handler callback configured, it is invoked
 ** in the usual manner. ^If the required lock still cannot be obtained, then
 ** the database is skipped and an attempt made to flush any dirty pages
@@ -10237,16 +10769,16 @@ int sqlite3_db_cacheflush(sqlite3*);
 ** kind of update operation that is about to occur.
 ** ^(The fourth parameter to the preupdate callback is the name of the
 ** database within the database connection that is being modified.  This
-** will be "main" for the main database or "temp" for TEMP tables or 
+** will be "main" for the main database or "temp" for TEMP tables or
 ** the name given after the AS keyword in the [ATTACH] statement for attached
 ** databases.)^
 ** ^The fifth parameter to the preupdate callback is the name of the
 ** table that is being modified.
 **
 ** For an UPDATE or DELETE operation on a [rowid table], the sixth
-** parameter passed to the preupdate callback is the initial [rowid] of the 
+** parameter passed to the preupdate callback is the initial [rowid] of the
 ** row being modified or deleted. For an INSERT operation on a rowid table,
-** or any operation on a WITHOUT ROWID table, the value of the sixth 
+** or any operation on a WITHOUT ROWID table, the value of the sixth
 ** parameter is undefined. For an INSERT or UPDATE on a rowid table the
 ** seventh parameter is the final rowid value of the row being inserted
 ** or updated. The value of the seventh parameter passed to the callback
@@ -10289,14 +10821,14 @@ int sqlite3_db_cacheflush(sqlite3*);
 **
 ** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate
 ** callback was invoked as a result of a direct insert, update, or delete
-** operation; or 1 for inserts, updates, or deletes invoked by top-level 
+** operation; or 1 for inserts, updates, or deletes invoked by top-level
 ** triggers; or 2 for changes resulting from triggers called by top-level
 ** triggers; and so forth.
 **
 ** When the [sqlite3_blob_write()] API is used to update a blob column,
-** the pre-update hook is invoked with SQLITE_DELETE. This is because the
-** in this case the new values are not available. In this case, when a
-** callback made with op==SQLITE_DELETE is actuall a write using the
+** the pre-update hook is invoked with SQLITE_DELETE, because
+** the new values are not yet available. In this case, when a
+** callback made with op==SQLITE_DELETE is actually a write using the
 ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns
 ** the index of the column being written. In other cases, where the
 ** pre-update hook is being invoked for some other reason, including a
@@ -10334,7 +10866,7 @@ int sqlite3_preupdate_blobwrite(sqlite3 *);
 ** The return value is OS-dependent.  For example, on unix systems, after
 ** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be
 ** called to get back the underlying "errno" that caused the problem, such
-** as ENOSPC, EAUTH, EISDIR, and so forth.  
+** as ENOSPC, EAUTH, EISDIR, and so forth. 
 */
 int sqlite3_system_errno(sqlite3*);
 
@@ -10372,12 +10904,20 @@ typedef struct sqlite3_snapshot {
 ** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly
 ** created [sqlite3_snapshot] object into *P and returns SQLITE_OK.
 ** If there is not already a read-transaction open on schema S when
-** this function is called, one is opened automatically. 
+** this function is called, one is opened automatically.
+**
+** If a read-transaction is opened by this function, then it is guaranteed 
+** that the returned snapshot object may not be invalidated by a database 
+** writer or checkpointer until after the read-transaction is closed. This 
+** is not guaranteed if a read-transaction is already open when this 
+** function is called. In that case, any subsequent write or checkpoint 
+** operation on the database may invalidate the returned snapshot handle, 
+** even while the read-transaction remains open.
 **
 ** The following must be true for this function to succeed. If any of
 ** the following statements are false when sqlite3_snapshot_get() is
 ** called, SQLITE_ERROR is returned. The final value of *P is undefined
-** in this case. 
+** in this case.
 **
 ** <ul>
 **   <li> The database handle must not be in [autocommit mode].
@@ -10389,13 +10929,13 @@ typedef struct sqlite3_snapshot {
 **
 **   <li> One or more transactions must have been written to the current wal
 **        file since it was created on disk (by any connection). This means
-**        that a snapshot cannot be taken on a wal mode database with no wal 
+**        that a snapshot cannot be taken on a wal mode database with no wal
 **        file immediately after it is first opened. At least one transaction
 **        must be written to it first.
 ** </ul>
 **
 ** This function may also return SQLITE_NOMEM.  If it is called with the
-** database handle in autocommit mode but fails for some other reason, 
+** database handle in autocommit mode but fails for some other reason,
 ** whether or not a read transaction is opened on schema S is undefined.
 **
 ** The [sqlite3_snapshot] object returned from a successful call to
@@ -10405,7 +10945,7 @@ typedef struct sqlite3_snapshot {
 ** The [sqlite3_snapshot_get()] interface is only available when the
 ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used.
 */
-SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
+int sqlite3_snapshot_get(
   sqlite3 *db,
   const char *zSchema,
   sqlite3_snapshot **ppSnapshot
@@ -10415,38 +10955,38 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
 ** CAPI3REF: Start a read transaction on an historical snapshot
 ** METHOD: sqlite3_snapshot
 **
-** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read 
-** transaction or upgrades an existing one for schema S of 
-** [database connection] D such that the read transaction refers to 
-** historical [snapshot] P, rather than the most recent change to the 
-** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK 
+** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read
+** transaction or upgrades an existing one for schema S of
+** [database connection] D such that the read transaction refers to
+** historical [snapshot] P, rather than the most recent change to the
+** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK
 ** on success or an appropriate [error code] if it fails.
 **
-** ^In order to succeed, the database connection must not be in 
+** ^In order to succeed, the database connection must not be in
 ** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there
 ** is already a read transaction open on schema S, then the database handle
 ** must have no active statements (SELECT statements that have been passed
-** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). 
+** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()).
 ** SQLITE_ERROR is returned if either of these conditions is violated, or
 ** if schema S does not exist, or if the snapshot object is invalid.
 **
 ** ^A call to sqlite3_snapshot_open() will fail to open if the specified
-** snapshot has been overwritten by a [checkpoint]. In this case 
+** snapshot has been overwritten by a [checkpoint]. In this case
 ** SQLITE_ERROR_SNAPSHOT is returned.
 **
-** If there is already a read transaction open when this function is 
+** If there is already a read transaction open when this function is
 ** invoked, then the same read transaction remains open (on the same
 ** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT
 ** is returned. If another error code - for example SQLITE_PROTOCOL or an
 ** SQLITE_IOERR error code - is returned, then the final state of the
-** read transaction is undefined. If SQLITE_OK is returned, then the 
+** read transaction is undefined. If SQLITE_OK is returned, then the
 ** read transaction is now open on database snapshot P.
 **
 ** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the
 ** database connection D does not know that the database file for
 ** schema S is in [WAL mode].  A database connection might not know
 ** that the database file is in [WAL mode] if there has been no prior
-** I/O on that database connection, or if the database entered [WAL mode] 
+** I/O on that database connection, or if the database entered [WAL mode]
 ** after the most recent I/O on the database connection.)^
 ** (Hint: Run "[PRAGMA application_id]" against a newly opened
 ** database connection in order to make it ready to use snapshots.)
@@ -10454,7 +10994,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
 ** The [sqlite3_snapshot_open()] interface is only available when the
 ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used.
 */
-SQLITE_EXPERIMENTAL int sqlite3_snapshot_open(
+int sqlite3_snapshot_open(
   sqlite3 *db,
   const char *zSchema,
   sqlite3_snapshot *pSnapshot
@@ -10471,24 +11011,24 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_open(
 ** The [sqlite3_snapshot_free()] interface is only available when the
 ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used.
 */
-SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
+void sqlite3_snapshot_free(sqlite3_snapshot*);
 
 /*
 ** CAPI3REF: Compare the ages of two snapshot handles.
 ** METHOD: sqlite3_snapshot
 **
 ** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages
-** of two valid snapshot handles. 
+** of two valid snapshot handles.
 **
-** If the two snapshot handles are not associated with the same database 
-** file, the result of the comparison is undefined. 
+** If the two snapshot handles are not associated with the same database
+** file, the result of the comparison is undefined.
 **
 ** Additionally, the result of the comparison is only valid if both of the
 ** snapshot handles were obtained by calling sqlite3_snapshot_get() since the
 ** last time the wal file was deleted. The wal file is deleted when the
 ** database is changed back to rollback mode or when the number of database
-** clients drops to zero. If either snapshot handle was obtained before the 
-** wal file was last deleted, the value returned by this function 
+** clients drops to zero. If either snapshot handle was obtained before the
+** wal file was last deleted, the value returned by this function
 ** is undefined.
 **
 ** Otherwise, this API returns a negative value if P1 refers to an older
@@ -10498,7 +11038,7 @@ SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
 ** This interface is only available if SQLite is compiled with the
 ** [SQLITE_ENABLE_SNAPSHOT] option.
 */
-SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
+int sqlite3_snapshot_cmp(
   sqlite3_snapshot *p1,
   sqlite3_snapshot *p2
 );
@@ -10526,20 +11066,21 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
 ** This interface is only available if SQLite is compiled with the
 ** [SQLITE_ENABLE_SNAPSHOT] option.
 */
-SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
+int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 
 /*
 ** CAPI3REF: Serialize a database
 **
-** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory
-** that is a serialization of the S database on [database connection] D.
+** The sqlite3_serialize(D,S,P,F) interface returns a pointer to
+** memory that is a serialization of the S database on
+** [database connection] D.  If S is a NULL pointer, the main database is used.
 ** If P is not a NULL pointer, then the size of the database in bytes
 ** is written into *P.
 **
 ** For an ordinary on-disk database file, the serialization is just a
 ** copy of the disk file.  For an in-memory database or a "TEMP" database,
 ** the serialization is the same sequence of bytes which would be written
-** to disk if that database where backed up to disk.
+** to disk if that database were backed up to disk.
 **
 ** The usual case is that sqlite3_serialize() copies the serialization of
 ** the database into memory obtained from [sqlite3_malloc64()] and returns
@@ -10548,15 +11089,22 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
 ** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations
 ** are made, and the sqlite3_serialize() function will return a pointer
 ** to the contiguous memory representation of the database that SQLite
-** is currently using for that database, or NULL if the no such contiguous
+** is currently using for that database, or NULL if no such contiguous
 ** memory representation of the database exists.  A contiguous memory
 ** representation of the database will usually only exist if there has
 ** been a prior call to [sqlite3_deserialize(D,S,...)] with the same
 ** values of D and S.
-** The size of the database is written into *P even if the 
+** The size of the database is written into *P even if the
 ** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy
 ** of the database exists.
 **
+** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set,
+** the returned buffer content will remain accessible and unchanged
+** until either the next write operation on the connection or when
+** the connection is closed, and applications must not modify the
+** buffer. If the bit had been clear, the returned buffer will not
+** be accessed by SQLite after the call.
+**
 ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the
 ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory
 ** allocation error occurs.
@@ -10590,14 +11138,15 @@ unsigned char *sqlite3_serialize(
 /*
 ** CAPI3REF: Deserialize a database
 **
-** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the 
+** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the
 ** [database connection] D to disconnect from database S and then
-** reopen S as an in-memory database based on the serialization contained
-** in P.  The serialized database P is N bytes in size.  M is the size of
-** the buffer P, which might be larger than N.  If M is larger than N, and
-** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is
-** permitted to add content to the in-memory database as long as the total
-** size does not exceed M bytes.
+** reopen S as an in-memory database based on the serialization
+** contained in P.  If S is a NULL pointer, the main database is
+** used. The serialized database P is N bytes in size.  M is the size
+** of the buffer P, which might be larger than N.  If M is larger than
+** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then
+** SQLite is permitted to add content to the in-memory database as
+** long as the total size does not exceed M bytes.
 **
 ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will
 ** invoke sqlite3_free() on the serialization buffer when the database
@@ -10605,15 +11154,25 @@ unsigned char *sqlite3_serialize(
 ** SQLite will try to increase the buffer size using sqlite3_realloc64()
 ** if writes on the database cause it to grow larger than M bytes.
 **
+** Applications must not modify the buffer P or invalidate it before
+** the database connection D is closed.
+**
 ** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the
 ** database is currently in a read transaction or is involved in a backup
 ** operation.
 **
-** It is not possible to deserialized into the TEMP database.  If the
+** It is not possible to deserialize into the TEMP database.  If the
 ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the
 ** function returns SQLITE_ERROR.
 **
-** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the 
+** The deserialized database should not be in [WAL mode].  If the database
+** is in WAL mode, then any attempt to use the database file will result
+** in an [SQLITE_CANTOPEN] error.  The application can set the
+** [file format version numbers] (bytes 18 and 19) of the input database P
+** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the
+** database file into rollback mode and work around this limitation.
+**
+** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the
 ** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then
 ** [sqlite3_free()] is invoked on argument P prior to returning.
 **
@@ -10624,7 +11183,7 @@ int sqlite3_deserialize(
   sqlite3 *db,            /* The database connection */
   const char *zSchema,    /* Which DB to reopen with the deserialization */
   unsigned char *pData,   /* The serialized database content */
-  sqlite3_int64 szDb,     /* Number bytes in the deserialization */
+  sqlite3_int64 szDb,     /* Number of bytes in the deserialization */
   sqlite3_int64 szBuf,    /* Total size of buffer pData[] */
   unsigned mFlags         /* Zero or more SQLITE_DESERIALIZE_* flags */
 );
@@ -10632,7 +11191,7 @@ int sqlite3_deserialize(
 /*
 ** CAPI3REF: Flags for sqlite3_deserialize()
 **
-** The following are allowed values for 6th argument (the F argument) to
+** The following are allowed values for the 6th argument (the F argument) to
 ** the [sqlite3_deserialize(D,S,P,N,M,F)] interface.
 **
 ** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization
@@ -10654,6 +11213,54 @@ int sqlite3_deserialize(
 #define SQLITE_DESERIALIZE_RESIZEABLE  2 /* Resize using sqlite3_realloc64() */
 #define SQLITE_DESERIALIZE_READONLY    4 /* Database is read-only */
 
+/*
+** CAPI3REF: Bind array values to the CARRAY table-valued function
+**
+** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to
+** one of the first argument of the [carray() table-valued function].  The
+** S parameter is a pointer to the [prepared statement] that uses the carray()
+** functions.  I is the parameter index to be bound.  P is a pointer to the
+** array to be bound, and N is the number of eements in the array.  The
+** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64],
+** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to
+** indicate the datatype of the array being bound.  The X argument is not a
+** NULL pointer, then SQLite will invoke the function X on the P parameter
+** after it has finished using P, even if the call to
+** sqlite3_carray_bind() fails. The special-case finalizer
+** SQLITE_TRANSIENT has no effect here.
+*/
+int sqlite3_carray_bind(
+  sqlite3_stmt *pStmt,        /* Statement to be bound */
+  int i,                      /* Parameter index */
+  void *aData,                /* Pointer to array data */
+  int nData,                  /* Number of data elements */
+  int mFlags,                 /* CARRAY flags */
+  void (*xDel)(void*)         /* Destructor for aData */
+);
+
+/*
+** CAPI3REF: Datatypes for the CARRAY table-valued function
+**
+** The fifth argument to the [sqlite3_carray_bind()] interface musts be
+** one of the following constants, to specify the datatype of the array
+** that is being bound into the [carray table-valued function].
+*/
+#define SQLITE_CARRAY_INT32     0    /* Data is 32-bit signed integers */
+#define SQLITE_CARRAY_INT64     1    /* Data is 64-bit signed integers */
+#define SQLITE_CARRAY_DOUBLE    2    /* Data is doubles */
+#define SQLITE_CARRAY_TEXT      3    /* Data is char* */
+#define SQLITE_CARRAY_BLOB      4    /* Data is struct iovec */
+
+/*
+** Versions of the above #defines that omit the initial SQLITE_, for
+** legacy compatibility.
+*/
+#define CARRAY_INT32     0    /* Data is 32-bit signed integers */
+#define CARRAY_INT64     1    /* Data is 64-bit signed integers */
+#define CARRAY_DOUBLE    2    /* Data is doubles */
+#define CARRAY_TEXT      3    /* Data is char* */
+#define CARRAY_BLOB      4    /* Data is struct iovec */
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
@@ -10665,8 +11272,6 @@ int sqlite3_deserialize(
 #if defined(__wasi__)
 # undef SQLITE_WASI
 # define SQLITE_WASI 1
-# undef SQLITE_OMIT_WAL
-# define SQLITE_OMIT_WAL 1/* because it requires shared memory APIs */
 # ifndef SQLITE_OMIT_LOAD_EXTENSION
 #  define SQLITE_OMIT_LOAD_EXTENSION
 # endif
@@ -10678,4 +11283,4 @@ int sqlite3_deserialize(
 #ifdef __cplusplus
 }  /* End of the 'extern "C"' block */
 #endif
-#endif /* SQLITE3_H */
+/* #endif for SQLITE3_H will be added by mksqlite3.tcl */
diff --git a/src/sqlite3.rc b/src/sqlite3.rc
index 5a856490d6..aad468d349 100644
--- a/src/sqlite3.rc
+++ b/src/sqlite3.rc
@@ -70,7 +70,7 @@ BEGIN
       VALUE "FileDescription", "SQLite is a software library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine."
       VALUE "FileVersion", SQLITE_VERSION
       VALUE "InternalName", "sqlite3"
-      VALUE "LegalCopyright", "http://www.sqlite.org/copyright.html"
+      VALUE "LegalCopyright", "http://sqlite.org/copyright.html"
       VALUE "ProductName", "SQLite"
       VALUE "ProductVersion", SQLITE_VERSION
       VALUE "SourceId", SQLITE_SOURCE_ID
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 19e030028a..5258faaed3 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -361,6 +361,17 @@ struct sqlite3_api_routines {
   int (*value_encoding)(sqlite3_value*);
   /* Version 3.41.0 and later */
   int (*is_interrupted)(sqlite3*);
+  /* Version 3.43.0 and later */
+  int (*stmt_explain)(sqlite3_stmt*,int);
+  /* Version 3.44.0 and later */
+  void *(*get_clientdata)(sqlite3*,const char*);
+  int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*));
+  /* Version 3.50.0 and later */
+  int (*setlk_timeout)(sqlite3*,int,int);
+  /* Version 3.51.0 and later */
+  int (*set_errmsg)(sqlite3*,int,const char*);
+  int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int);
+  
 };
 
 /*
@@ -689,6 +700,16 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_value_encoding         sqlite3_api->value_encoding
 /* Version 3.41.0 and later */
 #define sqlite3_is_interrupted         sqlite3_api->is_interrupted
+/* Version 3.43.0 and later */
+#define sqlite3_stmt_explain           sqlite3_api->stmt_explain
+/* Version 3.44.0 and later */
+#define sqlite3_get_clientdata         sqlite3_api->get_clientdata
+#define sqlite3_set_clientdata         sqlite3_api->set_clientdata
+/* Version 3.50.0 and later */
+#define sqlite3_setlk_timeout          sqlite3_api->setlk_timeout
+/* Version 3.51.0 and later */
+#define sqlite3_set_errmsg             sqlite3_api->set_errmsg
+#define sqlite3_db_status64            sqlite3_api->db_status64
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index aeb5e07135..9bcc903e23 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -25,20 +25,20 @@
 **                                  used on lines of code that actually
 **                                  implement parts of coverage testing.
 **
-**    OPTIMIZATION-IF-TRUE        - This branch is allowed to alway be false
+**    OPTIMIZATION-IF-TRUE        - This branch is allowed to always be false
 **                                  and the correct answer is still obtained,
 **                                  though perhaps more slowly.
 **
-**    OPTIMIZATION-IF-FALSE       - This branch is allowed to alway be true
+**    OPTIMIZATION-IF-FALSE       - This branch is allowed to always be true
 **                                  and the correct answer is still obtained,
 **                                  though perhaps more slowly.
 **
 **    PREVENTS-HARMLESS-OVERREAD  - This branch prevents a buffer overread
 **                                  that would be harmless and undetectable
-**                                  if it did occur.  
+**                                  if it did occur.
 **
 ** In all cases, the special comment must be enclosed in the usual
-** slash-asterisk...asterisk-slash comment marks, with no spaces between the 
+** slash-asterisk...asterisk-slash comment marks, with no spaces between the
 ** asterisks and the comment text.
 */
 
@@ -143,10 +143,13 @@
 /*
 ** Macro to disable warnings about missing "break" at the end of a "case".
 */
-#if GCC_VERSION>=7000000
-# define deliberate_fall_through __attribute__((fallthrough));
-#else
-# define deliberate_fall_through
+#if defined(__has_attribute)
+#  if __has_attribute(fallthrough)
+#    define deliberate_fall_through __attribute__((fallthrough));
+#  endif
+#endif
+#if !defined(deliberate_fall_through)
+#  define deliberate_fall_through
 #endif
 
 /*
@@ -182,7 +185,7 @@
 #endif
 
 /* Optionally #include a user-defined header, whereby compilation options
-** may be set prior to where they take effect, but after platform setup. 
+** may be set prior to where they take effect, but after platform setup.
 ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include
 ** file.
 */
@@ -230,7 +233,7 @@
 #ifndef __has_extension
 # define __has_extension(x) 0     /* compatibility with non-clang compilers */
 #endif
-#if GCC_VERSION>=4007000 || __has_extension(c_atomic) 
+#if GCC_VERSION>=4007000 || __has_extension(c_atomic)
 # define SQLITE_ATOMIC_INTRINSICS 1
 # define AtomicLoad(PTR)       __atomic_load_n((PTR),__ATOMIC_RELAXED)
 # define AtomicStore(PTR,VAL)  __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED)
@@ -318,6 +321,29 @@
 #  endif
 #endif
 
+/*
+** Enable SQLITE_USE_SEH by default on MSVC builds.  Only omit
+** SEH support if the -DSQLITE_OMIT_SEH option is given.
+*/
+#if defined(_MSC_VER) && !defined(SQLITE_OMIT_SEH)
+# define SQLITE_USE_SEH 1
+#else
+# undef SQLITE_USE_SEH
+#endif
+
+/*
+** Enable SQLITE_DIRECT_OVERFLOW_READ, unless the build explicitly
+** disables it using -DSQLITE_DIRECT_OVERFLOW_READ=0
+*/
+#if defined(SQLITE_DIRECT_OVERFLOW_READ) && SQLITE_DIRECT_OVERFLOW_READ+1==1
+  /* Disable if -DSQLITE_DIRECT_OVERFLOW_READ=0 */
+# undef SQLITE_DIRECT_OVERFLOW_READ
+#else
+  /* In all other cases, enable */
+# define SQLITE_DIRECT_OVERFLOW_READ 1
+#endif
+
+
 /*
 ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2.
 ** 0 means mutexes are permanently disable and the library is never
@@ -586,6 +612,8 @@
 # define SQLITE_OMIT_ALTERTABLE
 #endif
 
+#define SQLITE_DIGIT_SEPARATOR '_'
+
 /*
 ** Return true (non-zero) if the input is an integer that is too large
 ** to fit in 32-bits.  This macro is used inside of various testcase()
@@ -610,6 +638,7 @@
 #include <string.h>
 #include <assert.h>
 #include <stddef.h>
+#include <ctype.h>
 
 /*
 ** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY.
@@ -630,7 +659,8 @@
 #ifdef SQLITE_OMIT_FLOATING_POINT
 # define double sqlite_int64
 # define float sqlite_int64
-# define LONGDOUBLE_TYPE sqlite_int64
+# define fabs(X) ((X)<0?-(X):(X))
+# define sqlite3IsOverflow(X) 0
 # ifndef SQLITE_BIG_DBL
 #   define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50)
 # endif
@@ -722,7 +752,7 @@
 #endif
 
 /*
-** The compile-time options SQLITE_MMAP_READWRITE and 
+** The compile-time options SQLITE_MMAP_READWRITE and
 ** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another.
 ** You must choose one or the other (or neither) but not both.
 */
@@ -735,7 +765,17 @@
 ** ourselves.
 */
 #ifndef offsetof
-#define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD))
+# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0))
+#endif
+
+/*
+** Work around C99 "flex-array" syntax for pre-C99 compilers, so as
+** to avoid complaints from -fsanitize=strict-bounds.
+*/
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+# define FLEXARRAY
+#else
+# define FLEXARRAY 1
 #endif
 
 /*
@@ -805,9 +845,6 @@
 #  define INT8_TYPE signed char
 # endif
 #endif
-#ifndef LONGDOUBLE_TYPE
-# define LONGDOUBLE_TYPE long double
-#endif
 typedef sqlite_int64 i64;          /* 8-byte signed integer */
 typedef sqlite_uint64 u64;         /* 8-byte unsigned integer */
 typedef UINT32_TYPE u32;           /* 4-byte unsigned integer */
@@ -816,6 +853,11 @@ typedef INT16_TYPE i16;            /* 2-byte signed integer */
 typedef UINT8_TYPE u8;             /* 1-byte unsigned integer */
 typedef INT8_TYPE i8;              /* 1-byte signed integer */
 
+/* A bitfield type for use inside of structures.  Always follow with :N where
+** N is the number of bits.
+*/
+typedef unsigned bft;  /* Bit Field Type */
+
 /*
 ** SQLITE_MAX_U32 is a u64 constant that is the maximum u64 value
 ** that can be stored in a u32 without loss of data.  The value
@@ -854,6 +896,8 @@ typedef u64 tRowcnt;
 **    0.5 -> -10           0.1 -> -33        0.0625 -> -40
 */
 typedef INT16_TYPE LogEst;
+#define LOGEST_MIN (-32768)
+#define LOGEST_MAX (32767)
 
 /*
 ** Set the SQLITE_PTRSIZE macro to the number of bytes in a pointer
@@ -863,7 +907,7 @@ typedef INT16_TYPE LogEst;
 #   define SQLITE_PTRSIZE __SIZEOF_POINTER__
 # elif defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
        defined(_M_ARM)   || defined(__arm__)    || defined(__x86)   ||    \
-      (defined(__APPLE__) && defined(__POWERPC__)) ||                     \
+      (defined(__APPLE__) && defined(__ppc__)) ||                         \
       (defined(__TOS_AIX__) && !defined(__64BIT__))
 #   define SQLITE_PTRSIZE 4
 # else
@@ -889,8 +933,31 @@ typedef INT16_TYPE LogEst;
 ** the end of buffer S.  This macro returns true if P points to something
 ** contained within the buffer S.
 */
-#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E)))
+#define SQLITE_WITHIN(P,S,E)   (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E)))
 
+/*
+** P is one byte past the end of a large buffer. Return true if a span of bytes
+** between S..E crosses the end of that buffer.  In other words, return true
+** if the sub-buffer S..E-1 overflows the buffer whose last byte is P-1.
+**
+** S is the start of the span.  E is one byte past the end of end of span.
+**
+**                        P
+**     |-----------------|                FALSE
+**               |-------|
+**               S        E
+**
+**                        P
+**     |-----------------|
+**                    |-------|           TRUE
+**                    S        E
+**
+**                        P
+**     |-----------------|               
+**                        |-------|       FALSE
+**                        S        E
+*/
+#define SQLITE_OVERFLOW(P,S,E) (((uptr)(S)<(uptr)(P))&&((uptr)(E)>(uptr)(P)))
 
 /*
 ** Macros to determine whether the machine is big or little endian,
@@ -900,16 +967,33 @@ typedef INT16_TYPE LogEst;
 ** using C-preprocessor macros.  If that is unsuccessful, or if
 ** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined
 ** at run-time.
+**
+** If you are building SQLite on some obscure platform for which the
+** following ifdef magic does not work, you can always include either:
+**
+**    -DSQLITE_BYTEORDER=1234
+**
+** or
+**
+**    -DSQLITE_BYTEORDER=4321
+**
+** to cause the build to work for little-endian or big-endian processors,
+** respectively.
 */
-#ifndef SQLITE_BYTEORDER
-# if defined(i386)      || defined(__i386__)      || defined(_M_IX86) ||    \
+#ifndef SQLITE_BYTEORDER  /* Replicate changes at tag-20230904a */
+# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__
+#   define SQLITE_BYTEORDER 4321
+# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__
+#   define SQLITE_BYTEORDER 1234
+# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1
+#   define SQLITE_BYTEORDER 4321
+# elif defined(i386)    || defined(__i386__)      || defined(_M_IX86) ||    \
      defined(__x86_64)  || defined(__x86_64__)    || defined(_M_X64)  ||    \
      defined(_M_AMD64)  || defined(_M_ARM)        || defined(__x86)   ||    \
      defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64)
-#   define SQLITE_BYTEORDER    1234
-# elif defined(sparc)     || defined(__ppc__) || \
-       defined(__ARMEB__) || defined(__AARCH64EB__)
-#   define SQLITE_BYTEORDER    4321
+#   define SQLITE_BYTEORDER 1234
+# elif defined(sparc)   || defined(__ARMEB__)     || defined(__AARCH64EB__)
+#   define SQLITE_BYTEORDER 4321
 # else
 #   define SQLITE_BYTEORDER 0
 # endif
@@ -942,6 +1026,14 @@ typedef INT16_TYPE LogEst;
 #define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32))
 #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
 
+/*
+** Macro SMXV(n) return the maximum value that can be held in variable n,
+** assuming n is a signed integer type.  UMXV(n) is similar for unsigned
+** integer types.
+*/
+#define SMXV(n) ((((i64)1)<<(sizeof(n)*8-1))-1)
+#define UMXV(n) ((((i64)1)<<(sizeof(n)*8))-1)
+
 /*
 ** Round up a number to the next larger multiple of 8.  This is used
 ** to force 8-byte alignment on 64-bit architectures.
@@ -1060,6 +1152,9 @@ extern u32 sqlite3TreeTrace;
 **   0x00010000     Beginning of DELETE/INSERT/UPDATE processing
 **   0x00020000     Transform DISTINCT into GROUP BY
 **   0x00040000     SELECT tree dump after all code has been generated
+**   0x00080000     NOT NULL strength reduction
+**   0x00100000     Pointers are all shown as zero
+**   0x00200000     EXISTS-to-JOIN optimization
 */
 
 /*
@@ -1083,14 +1178,14 @@ extern u32 sqlite3WhereTrace;
 ** 0xFFFF----   Low-level debug messages
 **
 ** 0x00000001   Code generation
-** 0x00000002   Solver
+** 0x00000002   Solver (Use 0x40000 for less detail)
 ** 0x00000004   Solver costs
 ** 0x00000008   WhereLoop inserts
 **
 ** 0x00000010   Display sqlite3_index_info xBestIndex calls
 ** 0x00000020   Range an equality scan metrics
 ** 0x00000040   IN operator decisions
-** 0x00000080   WhereLoop cost adjustements
+** 0x00000080   WhereLoop cost adjustments
 ** 0x00000100
 ** 0x00000200   Covering index decisions
 ** 0x00000400   OR optimization
@@ -1102,6 +1197,9 @@ extern u32 sqlite3WhereTrace;
 **
 ** 0x00010000   Show more detail when printing WHERE terms
 ** 0x00020000   Show WHERE terms returned from whereScanNext()
+** 0x00040000   Solver overview messages
+** 0x00080000   Star-query heuristic
+** 0x00100000   Pointers are all shown as zero
 */
 
 
@@ -1124,7 +1222,7 @@ struct BusyHandler {
 /*
 ** Name of table that holds the database schema.
 **
-** The PREFERRED names are used whereever possible.  But LEGACY is also
+** The PREFERRED names are used wherever possible.  But LEGACY is also
 ** used for backwards compatibility.
 **
 **  1.  Queries can use either the PREFERRED or the LEGACY names
@@ -1174,7 +1272,7 @@ struct BusyHandler {
 ** pointer will work here as long as it is distinct from SQLITE_STATIC
 ** and SQLITE_TRANSIENT.
 */
-#define SQLITE_DYNAMIC   ((sqlite3_destructor_type)sqlite3OomClear)
+#define SQLITE_DYNAMIC   ((sqlite3_destructor_type)sqlite3RowSetClear)
 
 /*
 ** When SQLITE_OMIT_WSD is defined, it means that the target platform does
@@ -1233,11 +1331,13 @@ typedef struct Column Column;
 typedef struct Cte Cte;
 typedef struct CteUse CteUse;
 typedef struct Db Db;
+typedef struct DbClientData DbClientData;
 typedef struct DbFixer DbFixer;
 typedef struct Schema Schema;
 typedef struct Expr Expr;
 typedef struct ExprList ExprList;
 typedef struct FKey FKey;
+typedef struct FpDecode FpDecode;
 typedef struct FuncDestructor FuncDestructor;
 typedef struct FuncDef FuncDef;
 typedef struct FuncDefHash FuncDefHash;
@@ -1256,6 +1356,7 @@ typedef struct Parse Parse;
 typedef struct ParseCleanup ParseCleanup;
 typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
+typedef struct RCStr RCStr;
 typedef struct RenameToken RenameToken;
 typedef struct Returning Returning;
 typedef struct RowSet RowSet;
@@ -1263,6 +1364,7 @@ typedef struct Savepoint Savepoint;
 typedef struct Select Select;
 typedef struct SQLiteThread SQLiteThread;
 typedef struct SelectDest SelectDest;
+typedef struct Subquery Subquery;
 typedef struct SrcItem SrcItem;
 typedef struct SrcList SrcList;
 typedef struct sqlite3_str StrAccum; /* Internal alias for sqlite3_str */
@@ -1342,7 +1444,7 @@ typedef int VList;
 /*
 ** Default synchronous levels.
 **
-** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ
+** Note that (for historical reasons) the PAGER_SYNCHRONOUS_* macros differ
 ** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1.
 **
 **           PAGER_SYNCHRONOUS       DEFAULT_SYNCHRONOUS
@@ -1381,7 +1483,7 @@ struct Db {
 ** An instance of the following structure stores a database schema.
 **
 ** Most Schema objects are associated with a Btree.  The exception is
-** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing.
+** the Schema for the TEMP database (sqlite3.aDb[1]) which is free-standing.
 ** In shared cache mode, a single Schema object can be shared by multiple
 ** Btrees that refer to the same underlying BtShared object.
 **
@@ -1492,7 +1594,7 @@ struct Lookaside {
   LookasideSlot *pInit;   /* List of buffers not previously used */
   LookasideSlot *pFree;   /* List of available buffers */
 #ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE
-  LookasideSlot *pSmallInit; /* List of small buffers not prediously used */
+  LookasideSlot *pSmallInit; /* List of small buffers not previously used */
   LookasideSlot *pSmallFree; /* List of available small buffers */
   void *pMiddle;          /* First byte past end of full-size buffers and
                           ** the first byte of LOOKASIDE_SMALL buffers */
@@ -1509,7 +1611,7 @@ struct LookasideSlot {
 #define EnableLookaside   db->lookaside.bDisable--;\
    db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue
 
-/* Size of the smaller allocations in two-size lookside */
+/* Size of the smaller allocations in two-size lookaside */
 #ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE
 #  define LOOKASIDE_SMALL           0
 #else
@@ -1530,43 +1632,11 @@ struct FuncDefHash {
 };
 #define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ)
 
-#ifdef SQLITE_USER_AUTHENTICATION
-/*
-** Information held in the "sqlite3" database connection object and used
-** to manage user authentication.
-*/
-typedef struct sqlite3_userauth sqlite3_userauth;
-struct sqlite3_userauth {
-  u8 authLevel;                 /* Current authentication level */
-  int nAuthPW;                  /* Size of the zAuthPW in bytes */
-  char *zAuthPW;                /* Password used to authenticate */
-  char *zAuthUser;              /* User name used to authenticate */
-};
-
-/* Allowed values for sqlite3_userauth.authLevel */
-#define UAUTH_Unknown     0     /* Authentication not yet checked */
-#define UAUTH_Fail        1     /* User authentication failed */
-#define UAUTH_User        2     /* Authenticated as a normal user */
-#define UAUTH_Admin       3     /* Authenticated as an administrator */
-
-/* Functions used only by user authorization logic */
-int sqlite3UserAuthTable(const char*);
-int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*);
-void sqlite3UserAuthInit(sqlite3*);
-void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**);
-
-#endif /* SQLITE_USER_AUTHENTICATION */
-
 /*
 ** typedef for the authorization callback function.
 */
-#ifdef SQLITE_USER_AUTHENTICATION
-  typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*,
-                               const char*, const char*);
-#else
-  typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*,
-                               const char*);
-#endif
+typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*,
+                             const char*);
 
 #ifndef SQLITE_OMIT_DEPRECATED
 /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
@@ -1631,7 +1701,7 @@ struct sqlite3 {
     u8 iDb;                     /* Which db file is being initialized */
     u8 busy;                    /* TRUE if currently initializing */
     unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */
-    unsigned imposterTable : 1; /* Building an imposter table */
+    unsigned imposterTable : 2; /* Building an imposter table */
     unsigned reopenMemdb : 1;   /* ATTACH is really a reopen using MemDB */
     const char **azInit;        /* "type", "name", and "tbl_name" columns */
   } init;
@@ -1704,11 +1774,17 @@ struct sqlite3 {
   Savepoint *pSavepoint;        /* List of active savepoints */
   int nAnalysisLimit;           /* Number of index rows to ANALYZE */
   int busyTimeout;              /* Busy handler timeout, in msec */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  int setlkTimeout;             /* Blocking lock timeout, in msec. -1 -> inf. */
+  int setlkFlags;               /* Flags passed to setlk_timeout() */
+#endif
   int nSavepoint;               /* Number of non-transaction savepoints */
   int nStatement;               /* Number of nested statement-transactions  */
   i64 nDeferredCons;            /* Net deferred constraints this transaction. */
   i64 nDeferredImmCons;         /* Net deferred immediate constraints */
   int *pnBytesFreed;            /* If not NULL, increment this in DbFree() */
+  DbClientData *pDbData;        /* sqlite3_set_clientdata() content */
+  u64 nSpill;                   /* TEMP content spilled to disk */
 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
   /* The following variables are all protected by the STATIC_MAIN
   ** mutex, not by sqlite3.mutex. They are used by code in notify.c.
@@ -1726,9 +1802,6 @@ struct sqlite3 {
   void (*xUnlockNotify)(void **, int);  /* Unlock notify callback */
   sqlite3 *pNextBlocked;        /* Next in list of all blocked connections */
 #endif
-#ifdef SQLITE_USER_AUTHENTICATION
-  sqlite3_userauth auth;        /* User authentication information */
-#endif
 };
 
 /*
@@ -1791,6 +1864,10 @@ struct sqlite3 {
                                           /*   the count using a callback. */
 #define SQLITE_CorruptRdOnly  HI(0x00002) /* Prohibit writes due to error */
 #define SQLITE_ReadUncommit   HI(0x00004) /* READ UNCOMMITTED in shared-cache */
+#define SQLITE_FkNoAction     HI(0x00008) /* Treat all FK as NO ACTION */
+#define SQLITE_AttachCreate   HI(0x00010) /* ATTACH allowed to create new dbs */
+#define SQLITE_AttachWrite    HI(0x00020) /* ATTACH allowed to open for write */
+#define SQLITE_Comments       HI(0x00040) /* Enable SQL comments */
 
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
@@ -1831,7 +1908,7 @@ struct sqlite3 {
 #define SQLITE_CursorHints    0x00000400 /* Add OP_CursorHint opcodes */
 #define SQLITE_Stat4          0x00000800 /* Use STAT4 data */
    /* TH3 expects this value  ^^^^^^^^^^ to be 0x0000800. Don't change it */
-#define SQLITE_PushDown       0x00001000 /* The push-down optimization */
+#define SQLITE_PushDown       0x00001000 /* WHERE-clause push-down opt */
 #define SQLITE_SimplifyJoin   0x00002000 /* Convert LEFT JOIN to JOIN */
 #define SQLITE_SkipScan       0x00004000 /* Skip-scans */
 #define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */
@@ -1848,6 +1925,10 @@ struct sqlite3 {
 #define SQLITE_IndexedExpr    0x01000000 /* Pull exprs from index when able */
 #define SQLITE_Coroutines     0x02000000 /* Co-routines for subqueries */
 #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */
+#define SQLITE_OnePass        0x08000000 /* Single-pass DELETE and UPDATE */
+#define SQLITE_OrderBySubq    0x10000000 /* ORDER BY in subquery helps outer */
+#define SQLITE_StarQuery      0x20000000 /* Heurists for star queries */
+#define SQLITE_ExistsToJoin   0x40000000 /* The EXISTS-to-JOIN optimization */
 #define SQLITE_AllOpts        0xffffffff /* All optimizations */
 
 /*
@@ -1884,7 +1965,7 @@ struct sqlite3 {
 ** field is used by per-connection app-def functions.
 */
 struct FuncDef {
-  i8 nArg;             /* Number of arguments.  -1 means unlimited */
+  i16 nArg;            /* Number of arguments.  -1 means unlimited */
   u32 funcFlags;       /* Some combination of SQLITE_FUNC_* */
   void *pUserData;     /* User data parameter */
   FuncDef *pNext;      /* Next function with same name */
@@ -1930,6 +2011,7 @@ struct FuncDestructor {
 **     SQLITE_FUNC_ANYORDER    ==  NC_OrderAgg       == SF_OrderByReqd
 **     SQLITE_FUNC_LENGTH      ==  OPFLAG_LENGTHARG
 **     SQLITE_FUNC_TYPEOF      ==  OPFLAG_TYPEOFARG
+**     SQLITE_FUNC_BYTELEN     ==  OPFLAG_BYTELENARG
 **     SQLITE_FUNC_CONSTANT    ==  SQLITE_DETERMINISTIC from the API
 **     SQLITE_FUNC_DIRECT      ==  SQLITE_DIRECTONLY from the API
 **     SQLITE_FUNC_UNSAFE      ==  SQLITE_INNOCUOUS  -- opposite meanings!!!
@@ -1937,7 +2019,7 @@ struct FuncDestructor {
 **
 ** Note that even though SQLITE_FUNC_UNSAFE and SQLITE_INNOCUOUS have the
 ** same bit value, their meanings are inverted.  SQLITE_FUNC_UNSAFE is
-** used internally and if set means tha the function has side effects.
+** used internally and if set means that the function has side effects.
 ** SQLITE_INNOCUOUS is used by application code and means "not unsafe".
 ** See multiple instances of tag-20230109-1.
 */
@@ -1948,6 +2030,7 @@ struct FuncDestructor {
 #define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/
 #define SQLITE_FUNC_LENGTH   0x0040 /* Built-in length() function */
 #define SQLITE_FUNC_TYPEOF   0x0080 /* Built-in typeof() function */
+#define SQLITE_FUNC_BYTELEN  0x00c0 /* Built-in octet_length() function */
 #define SQLITE_FUNC_COUNT    0x0100 /* Built-in count(*) aggregate */
 /*                           0x0200 -- available for reuse */
 #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */
@@ -1956,21 +2039,22 @@ struct FuncDestructor {
 #define SQLITE_FUNC_SLOCHNG  0x2000 /* "Slow Change". Value constant during a
                                     ** single query - might change over time */
 #define SQLITE_FUNC_TEST     0x4000 /* Built-in testing functions */
-/*                           0x8000 -- available for reuse */
+#define SQLITE_FUNC_RUNONLY  0x8000 /* Cannot be used by valueFromFunction */
 #define SQLITE_FUNC_WINDOW   0x00010000 /* Built-in window-only function */
 #define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */
 #define SQLITE_FUNC_DIRECT   0x00080000 /* Not for use in TRIGGERs or VIEWs */
-#define SQLITE_FUNC_SUBTYPE  0x00100000 /* Result likely to have sub-type */
+/* SQLITE_SUBTYPE            0x00100000 // Consumer of subtypes */
 #define SQLITE_FUNC_UNSAFE   0x00200000 /* Function has side effects */
 #define SQLITE_FUNC_INLINE   0x00400000 /* Functions implemented in-line */
 #define SQLITE_FUNC_BUILTIN  0x00800000 /* This is a built-in function */
+/*  SQLITE_RESULT_SUBTYPE    0x01000000 // Generator of subtypes */
 #define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */
 
 /* Identifier numbers for each in-line function */
 #define INLINEFUNC_coalesce             0
 #define INLINEFUNC_implies_nonnull_row  1
 #define INLINEFUNC_expr_implies_expr    2
-#define INLINEFUNC_expr_compare         3      
+#define INLINEFUNC_expr_compare         3
 #define INLINEFUNC_affinity             4
 #define INLINEFUNC_iif                  5
 #define INLINEFUNC_sqlite_offset        6
@@ -2019,7 +2103,7 @@ struct FuncDestructor {
 **   PURE_DATE(zName, nArg, iArg, bNC, xFunc)
 **     Used for "pure" date/time functions, this macro is like DFUNCTION
 **     except that it does set the SQLITE_FUNC_CONSTANT flags.  iArg is
-**     ignored and the user-data for these functions is set to an 
+**     ignored and the user-data for these functions is set to an
 **     arbitrary non-NULL pointer.  The bNC parameter is not used.
 **
 **   AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal)
@@ -2055,10 +2139,11 @@ struct FuncDestructor {
 #define MFUNCTION(zName, nArg, xPtr, xFunc) \
   {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \
    xPtr, 0, xFunc, 0, 0, 0, #zName, {0} }
-#define JFUNCTION(zName, nArg, iArg, xFunc) \
-  {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|\
-   SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} }
+#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc) \
+  {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\
+   SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\
+   ((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \
+   SQLITE_INT_TO_PTR(iArg|((bJsonB)*JSON_BLOB)),0,xFunc,0, 0, 0, #zName, {0} }
 #define INLINE_FUNC(zName, nArg, iArg, mFlags) \
   {nArg, SQLITE_FUNC_BUILTIN|\
    SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \
@@ -2082,7 +2167,7 @@ struct FuncDestructor {
 #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_BUILTIN|\
    SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   pArg, 0, xFunc, 0, 0, 0, #zName, }
+   pArg, 0, xFunc, 0, 0, 0, #zName, {0} }
 #define LIKEFUNC(zName, nArg, arg, flags) \
   {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \
    (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} }
@@ -2249,6 +2334,7 @@ struct CollSeq {
 #define SQLITE_AFF_INTEGER  0x44  /* 'D' */
 #define SQLITE_AFF_REAL     0x45  /* 'E' */
 #define SQLITE_AFF_FLEXNUM  0x46  /* 'F' */
+#define SQLITE_AFF_DEFER    0x58  /* 'X'  - defer computation until later */
 
 #define sqlite3IsNumericAffinity(X)  ((X)>=SQLITE_AFF_NUMERIC)
 
@@ -2373,6 +2459,7 @@ struct Table {
   } u;
   Trigger *pTrigger;   /* List of triggers on this object */
   Schema *pSchema;     /* Schema that contains this table */
+  u8 aHx[16];          /* Column aHt[K%sizeof(aHt)] might have hash K */
 };
 
 /*
@@ -2399,8 +2486,7 @@ struct Table {
 #define TF_HasStored      0x00000040 /* Has one or more STORED columns */
 #define TF_HasGenerated   0x00000060 /* Combo: HasVirtual + HasStored */
 #define TF_WithoutRowid   0x00000080 /* No rowid.  PRIMARY KEY is the key */
-#define TF_StatsUsed      0x00000100 /* Query planner decisions affected by
-                                     ** Index.aiRowLogEst[] values */
+#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */
 #define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */
 #define TF_OOOHidden      0x00000400 /* Out-of-Order hidden columns */
 #define TF_HasNotNull     0x00000800 /* Contains NOT NULL constraints */
@@ -2409,6 +2495,7 @@ struct Table {
 #define TF_Ephemeral      0x00004000 /* An ephemeral table */
 #define TF_Eponymous      0x00008000 /* An eponymous virtual table */
 #define TF_Strict         0x00010000 /* STRICT mode */
+#define TF_Imposter       0x00020000 /* An imposter table */
 
 /*
 ** Allowed values for Table.eTabType
@@ -2456,6 +2543,15 @@ struct Table {
 #define HasRowid(X)     (((X)->tabFlags & TF_WithoutRowid)==0)
 #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0)
 
+/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is
+** available.  By default, this macro is false
+*/
+#ifndef SQLITE_ALLOW_ROWID_IN_VIEW
+# define ViewCanHaveRowid     0
+#else
+# define ViewCanHaveRowid     (sqlite3Config.mNoVisibleRowid==0)
+#endif
+
 /*
 ** Each foreign key constraint is an instance of the following structure.
 **
@@ -2498,9 +2594,13 @@ struct FKey {
   struct sColMap {      /* Mapping of columns in pFrom to columns in zTo */
     int iFrom;            /* Index of column in pFrom */
     char *zCol;           /* Name of column in zTo.  If NULL use PRIMARY KEY */
-  } aCol[1];            /* One entry for each of nCol columns */
+  } aCol[FLEXARRAY];      /* One entry for each of nCol columns */
 };
 
+/* The size (in bytes) of an FKey object holding N columns.  The answer
+** does NOT include space to hold the zTo name. */
+#define SZ_FKEY(N)  (offsetof(FKey,aCol)+(N)*sizeof(struct sColMap))
+
 /*
 ** SQLite supports many different ways to resolve a constraint
 ** error.  ROLLBACK processing means that a constraint violation
@@ -2527,7 +2627,7 @@ struct FKey {
 ** foreign key.
 **
 ** The OE_Default value is a place holder that means to use whatever
-** conflict resolution algorthm is required from context.
+** conflict resolution algorithm is required from context.
 **
 ** The following symbolic values are used to record which type
 ** of conflict resolution action to take.
@@ -2551,9 +2651,15 @@ struct FKey {
 ** argument to sqlite3VdbeKeyCompare and is used to control the
 ** comparison of the two index keys.
 **
-** Note that aSortOrder[] and aColl[] have nField+1 slots.  There
-** are nField slots for the columns of an index then one extra slot
-** for the rowid at the end.
+** The aSortOrder[] and aColl[] arrays have nAllField slots each. There
+** are nKeyField slots for the columns of an index then extra slots
+** for the rowid or key at the end.  The aSortOrder array is located after
+** the aColl[] array.
+**
+** If SQLITE_ENABLE_PREUPDATE_HOOK is defined, then aSortFlags might be NULL
+** to indicate that this object is for use by a preupdate hook.  When aSortFlags
+** is NULL, then nAllField is uninitialized and no space is allocated for
+** aColl[], so those fields may not be used.
 */
 struct KeyInfo {
   u32 nRef;           /* Number of references to this KeyInfo object */
@@ -2562,9 +2668,21 @@ struct KeyInfo {
   u16 nAllField;      /* Total columns, including key plus others */
   sqlite3 *db;        /* The database connection */
   u8 *aSortFlags;     /* Sort order for each column. */
-  CollSeq *aColl[1];  /* Collating sequence for each term of the key */
+  CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */
 };
 
+/* The size (in bytes) of a KeyInfo object with up to N fields.  This includes
+** the main body of the KeyInfo object and the aColl[] array of N elements,
+** but does not count the memory used to hold aSortFlags[]. */
+#define SZ_KEYINFO(N)  (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*))
+
+/* The size of a bare KeyInfo with no aColl[] entries */
+#if FLEXARRAY+1 > 1
+# define SZ_KEYINFO_0   offsetof(KeyInfo,aColl)
+#else
+# define SZ_KEYINFO_0   sizeof(KeyInfo)
+#endif
+
 /*
 ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array.
 */
@@ -2583,9 +2701,8 @@ struct KeyInfo {
 **
 ** An instance of this object serves as a "key" for doing a search on
 ** an index b+tree. The goal of the search is to find the entry that
-** is closed to the key described by this object.  This object might hold
-** just a prefix of the key.  The number of fields is given by
-** pKeyInfo->nField.
+** is closest to the key described by this object.  This object might hold
+** just a prefix of the key.  The number of fields is given by nField.
 **
 ** The r1 and r2 fields are the values to return if this key is less than
 ** or greater than a key in the btree, respectively.  These are normally
@@ -2595,7 +2712,7 @@ struct KeyInfo {
 ** The key comparison functions actually return default_rc when they find
 ** an equals comparison.  default_rc can be -1, 0, or +1.  If there are
 ** multiple entries in the b-tree with the same key (when only looking
-** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to
+** at the first nField elements) then default_rc can be set to -1 to
 ** cause the search to find the last match, or +1 to cause the search to
 ** find the first match.
 **
@@ -2607,8 +2724,8 @@ struct KeyInfo {
 ** b-tree.
 */
 struct UnpackedRecord {
-  KeyInfo *pKeyInfo;  /* Collation and sort-order information */
-  Mem *aMem;          /* Values */
+  KeyInfo *pKeyInfo;  /* Comparison info for the index that is unpacked */
+  Mem *aMem;          /* Values for columns of the index */
   union {
     char *z;            /* Cache of aMem[0].z for vdbeRecordCompareString() */
     i64 i;              /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */
@@ -2684,7 +2801,7 @@ struct Index {
   Pgno tnum;               /* DB Page containing root of this index */
   LogEst szIdxRow;         /* Estimated average row size in bytes */
   u16 nKeyCol;             /* Number of columns forming the key */
-  u16 nColumn;             /* Number of columns stored in the index */
+  u16 nColumn;             /* Nr columns in btree. Can be 2*Table.nCol */
   u8 onError;              /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
   unsigned idxType:2;      /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */
   unsigned bUnordered:1;   /* Use this index for == or IN queries only */
@@ -2782,7 +2899,7 @@ struct AggInfo {
                           ** from source tables rather than from accumulators */
   u8 useSortingIdx;       /* In direct mode, reference the sorting index rather
                           ** than the source table */
-  u16 nSortingColumn;     /* Number of columns in the sorting index */
+  u32 nSortingColumn;     /* Number of columns in the sorting index */
   int sortingIdx;         /* Cursor number of the sorting index */
   int sortingIdxPTab;     /* Cursor number of pseudo-table */
   int iFirstReg;          /* First register in range for aCol[] and aFunc[] */
@@ -2791,8 +2908,8 @@ struct AggInfo {
     Table *pTab;             /* Source table */
     Expr *pCExpr;            /* The original expression */
     int iTable;              /* Cursor number of the source table */
-    i16 iColumn;             /* Column number within the source table */
-    i16 iSorterColumn;       /* Column number in the sorting index */
+    int iColumn;             /* Column number within the source table */
+    int iSorterColumn;       /* Column number in the sorting index */
   } *aCol;
   int nColumn;            /* Number of used entries in aCol[] */
   int nAccumulator;       /* Number of columns that show through to the output.
@@ -2803,6 +2920,10 @@ struct AggInfo {
     FuncDef *pFunc;          /* The aggregate function implementation */
     int iDistinct;           /* Ephemeral table used to enforce DISTINCT */
     int iDistAddr;           /* Address of OP_OpenEphemeral */
+    int iOBTab;              /* Ephemeral table to implement ORDER BY */
+    u8 bOBPayload;           /* iOBTab has payload columns separate from key */
+    u8 bOBUnique;            /* Enforce uniqueness on iOBTab keys */
+    u8 bUseSubtype;          /* Transfer subtype info through sorter */
   } *aFunc;
   int nFunc;              /* Number of entries in aFunc[] */
   u32 selId;              /* Select to which this AggInfo belongs */
@@ -2814,13 +2935,19 @@ struct AggInfo {
 /*
 ** Macros to compute aCol[] and aFunc[] register numbers.
 **
-** These macros should not be used prior to the call to 
+** These macros should not be used prior to the call to
 ** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg.
 ** The assert()s that are part of this macro verify that constraint.
 */
+#ifndef NDEBUG
 #define AggInfoColumnReg(A,I)  (assert((A)->iFirstReg),(A)->iFirstReg+(I))
 #define AggInfoFuncReg(A,I)    \
                       (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I))
+#else
+#define AggInfoColumnReg(A,I)  ((A)->iFirstReg+(I))
+#define AggInfoFuncReg(A,I)    \
+                      ((A)->iFirstReg+(A)->nColumn+(I))
+#endif
 
 /*
 ** The datatype ynVar is a signed integer, either 16-bit or 32-bit.
@@ -2941,7 +3068,7 @@ struct Expr {
                          ** TK_REGISTER: register number
                          ** TK_TRIGGER: 1 -> new, 0 -> old
                          ** EP_Unlikely:  134217728 times likelihood
-                         ** TK_IN: ephemerial table holding RHS
+                         ** TK_IN: ephemeral table holding RHS
                          ** TK_SELECT_COLUMN: Number of columns on the LHS
                          ** TK_SELECT: 1st register of result vector */
   ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
@@ -2957,6 +3084,7 @@ struct Expr {
     Table *pTab;           /* TK_COLUMN: Table containing column. Can be NULL
                            ** for a column of an index on an expression */
     Window *pWin;          /* EP_WinFunc: Window/Filter defn for a function */
+    int nReg;              /* TK_NULLS: Number of registers to NULL out */
     struct {               /* TK_IN, TK_SELECT, and TK_EXISTS */
       int iAddr;             /* Subroutine entry address */
       int regReturn;         /* Register used to hold return address */
@@ -2987,7 +3115,7 @@ struct Expr {
 #define EP_Reduced    0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */
 #define EP_Win        0x008000 /* Contains window functions */
 #define EP_TokenOnly  0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */
-                   /* 0x020000 // Available for reuse */
+#define EP_FullSize   0x020000 /* Expr structure must remain full sized */
 #define EP_IfNullRow  0x040000 /* The TK_IF_NULL_ROW opcode */
 #define EP_Unlikely   0x080000 /* unlikely() or likelihood() function */
 #define EP_ConstFunc  0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */
@@ -3001,7 +3129,7 @@ struct Expr {
 #define EP_IsTrue   0x10000000 /* Always has boolean value of TRUE */
 #define EP_IsFalse  0x20000000 /* Always has boolean value of FALSE */
 #define EP_FromDDL  0x40000000 /* Originates from sqlite_schema */
-               /*   0x80000000 // Available */
+#define EP_SubtArg  0x80000000 /* Is argument to SQLITE_SUBTYPE function */
 
 /* The EP_Propagate mask is a set of properties that automatically propagate
 ** upwards into parent nodes.
@@ -3011,18 +3139,21 @@ struct Expr {
 /* Macros can be used to test, set, or clear bits in the
 ** Expr.flags field.
 */
-#define ExprHasProperty(E,P)     (((E)->flags&(P))!=0)
-#define ExprHasAllProperty(E,P)  (((E)->flags&(P))==(P))
-#define ExprSetProperty(E,P)     (E)->flags|=(P)
-#define ExprClearProperty(E,P)   (E)->flags&=~(P)
+#define ExprHasProperty(E,P)     (((E)->flags&(u32)(P))!=0)
+#define ExprHasAllProperty(E,P)  (((E)->flags&(u32)(P))==(u32)(P))
+#define ExprSetProperty(E,P)     (E)->flags|=(u32)(P)
+#define ExprClearProperty(E,P)   (E)->flags&=~(u32)(P)
 #define ExprAlwaysTrue(E)   (((E)->flags&(EP_OuterON|EP_IsTrue))==EP_IsTrue)
 #define ExprAlwaysFalse(E)  (((E)->flags&(EP_OuterON|EP_IsFalse))==EP_IsFalse)
+#define ExprIsFullSize(E)   (((E)->flags&(EP_Reduced|EP_TokenOnly))==0)
 
 /* Macros used to ensure that the correct members of unions are accessed
 ** in Expr.
 */
 #define ExprUseUToken(E)    (((E)->flags&EP_IntValue)==0)
 #define ExprUseUValue(E)    (((E)->flags&EP_IntValue)!=0)
+#define ExprUseWOfst(E)     (((E)->flags&(EP_InnerON|EP_OuterON))==0)
+#define ExprUseWJoin(E)     (((E)->flags&(EP_InnerON|EP_OuterON))!=0)
 #define ExprUseXList(E)     (((E)->flags&EP_xIsSelect)==0)
 #define ExprUseXSelect(E)   (((E)->flags&EP_xIsSelect)!=0)
 #define ExprUseYTab(E)      (((E)->flags&(EP_WinFunc|EP_Subrtn))==0)
@@ -3123,15 +3254,21 @@ struct ExprList {
       int iConstExprReg;   /* Register in which Expr value is cached. Used only
                            ** by Parse.pConstExpr */
     } u;
-  } a[1];                  /* One slot for each expression in the list */
+  } a[FLEXARRAY];          /* One slot for each expression in the list */
 };
 
+/* The size (in bytes) of an ExprList object that is big enough to hold
+** as many as N expressions. */
+#define SZ_EXPRLIST(N)  \
+             (offsetof(ExprList,a) + (N)*sizeof(struct ExprList_item))
+
 /*
 ** Allowed values for Expr.a.eEName
 */
 #define ENAME_NAME  0       /* The AS clause of a result set */
 #define ENAME_SPAN  1       /* Complete text of the result set expression */
 #define ENAME_TAB   2       /* "DB.TABLE.NAME" for the result set */
+#define ENAME_ROWID 3       /* "DB.TABLE._rowid_" for * expansion of rowid */
 
 /*
 ** An instance of this structure can hold a simple list of identifiers,
@@ -3150,16 +3287,14 @@ struct ExprList {
 */
 struct IdList {
   int nId;         /* Number of identifiers on the list */
-  u8 eU4;          /* Which element of a.u4 is valid */
   struct IdList_item {
     char *zName;      /* Name of the identifier */
-    union {
-      int idx;          /* Index in some Table.aCol[] of a column named zName */
-      Expr *pExpr;      /* Expr to implement a USING variable -- NOT USED */
-    } u4;
-  } a[1];
+  } a[FLEXARRAY];
 };
 
+/* The size (in bytes) of an IdList object that can hold up to N IDs. */
+#define SZ_IDLIST(N)  (offsetof(IdList,a)+(N)*sizeof(struct IdList_item))
+
 /*
 ** Allowed values for IdList.eType, which determines which value of the a.u4
 ** is valid.
@@ -3168,6 +3303,16 @@ struct IdList {
 #define EU4_IDX    1   /* Uses IdList.a.u4.idx */
 #define EU4_EXPR   2   /* Uses IdList.a.u4.pExpr -- NOT CURRENTLY USED */
 
+/*
+** Details of the implementation of a subquery.
+*/
+struct Subquery {
+  Select *pSelect;  /* A SELECT statement used in place of a table name */
+  int addrFillSub;  /* Address of subroutine to initialize a subquery */
+  int regReturn;    /* Register holding return address of addrFillSub */
+  int regResult;    /* Registers holding results of a co-routine */
+};
+
 /*
 ** The SrcItem object represents a single term in the FROM clause of a query.
 ** The SrcList object is mostly an array of SrcItems.
@@ -3180,27 +3325,40 @@ struct IdList {
 ** In the colUsed field, the high-order bit (bit 63) is set if the table
 ** contains more than 63 columns and the 64-th or later column is used.
 **
-** Union member validity:
+** Aggressive use of "union" helps keep the size of the object small.  This
+** has been shown to boost performance, in addition to saving memory.
+** Access to union elements is gated by the following rules which should
+** always be checked, either by an if-statement or by an assert().
+**
+**    Field              Only access if this is true
+**    ---------------    -----------------------------------
+**    u1.zIndexedBy      fg.isIndexedBy
+**    u1.pFuncArg        fg.isTabFunc
+**    u1.nRow            !fg.isTabFunc  && !fg.isIndexedBy
+**
+**    u2.pIBIndex        fg.isIndexedBy
+**    u2.pCteUse         fg.isCte
+**
+**    u3.pOn             !fg.isUsing
+**    u3.pUsing          fg.isUsing
 **
-**    u1.zIndexedBy          fg.isIndexedBy && !fg.isTabFunc
-**    u1.pFuncArg            fg.isTabFunc   && !fg.isIndexedBy
-**    u2.pIBIndex            fg.isIndexedBy && !fg.isCte
-**    u2.pCteUse             fg.isCte       && !fg.isIndexedBy
+**    u4.zDatabase       !fg.fixedSchema && !fg.isSubquery
+**    u4.pSchema         fg.fixedSchema
+**    u4.pSubq           fg.isSubquery
+**
+** See also the sqlite3SrcListDelete() routine for assert() statements that
+** check invariants on the fields of this object, especially the flags
+** inside the fg struct.
 */
 struct SrcItem {
-  Schema *pSchema;  /* Schema to which this item is fixed */
-  char *zDatabase;  /* Name of database holding this table */
   char *zName;      /* Name of the table */
   char *zAlias;     /* The "B" part of a "A AS B" phrase.  zName is the "A" */
-  Table *pTab;      /* An SQL table corresponding to zName */
-  Select *pSelect;  /* A SELECT statement used in place of a table name */
-  int addrFillSub;  /* Address of subroutine to manifest a subquery */
-  int regReturn;    /* Register holding return address of addrFillSub */
-  int regResult;    /* Registers holding results of a co-routine */
+  Table *pSTab;     /* Table object for zName. Mnemonic: Srcitem-TABle */
   struct {
     u8 jointype;      /* Type of join between this table and the previous */
     unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
     unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
+    unsigned isSubquery :1;    /* True if this term is a subquery */
     unsigned isTabFunc :1;     /* True if table-valued-function syntax */
     unsigned isCorrelated :1;  /* True if sub-query is correlated */
     unsigned isMaterialized:1; /* This is a materialized view */
@@ -3211,23 +3369,33 @@ struct SrcItem {
     unsigned notCte :1;        /* This item may not match a CTE */
     unsigned isUsing :1;       /* u3.pUsing is valid */
     unsigned isOn :1;          /* u3.pOn was once valid and non-NULL */
-    unsigned isSynthUsing :1;  /* u3.pUsing is synthensized from NATURAL */
+    unsigned isSynthUsing :1;  /* u3.pUsing is synthesized from NATURAL */
     unsigned isNestedFrom :1;  /* pSelect is a SF_NestedFrom subquery */
+    unsigned rowidUsed :1;     /* The ROWID of this table is referenced */
+    unsigned fixedSchema :1;   /* Uses u4.pSchema, not u4.zDatabase */
+    unsigned hadSchema :1;     /* Had u4.zDatabase before u4.pSchema */
+    unsigned fromExists :1;    /* Comes from WHERE EXISTS(...) */
   } fg;
   int iCursor;      /* The VDBE cursor number used to access this table */
-  union {
-    Expr *pOn;        /* fg.isUsing==0 =>  The ON clause of a join */
-    IdList *pUsing;   /* fg.isUsing==1 =>  The USING clause of a join */
-  } u3;
   Bitmask colUsed;  /* Bit N set if column N used. Details above for N>62 */
   union {
     char *zIndexedBy;    /* Identifier from "INDEXED BY <zIndex>" clause */
     ExprList *pFuncArg;  /* Arguments to table-valued-function */
+    u32 nRow;            /* Number of rows in a VALUES clause */
   } u1;
   union {
     Index *pIBIndex;  /* Index structure corresponding to u1.zIndexedBy */
     CteUse *pCteUse;  /* CTE Usage info when fg.isCte is true */
   } u2;
+  union {
+    Expr *pOn;        /* fg.isUsing==0 =>  The ON clause of a join */
+    IdList *pUsing;   /* fg.isUsing==1 =>  The USING clause of a join */
+  } u3;
+  union {
+    Schema *pSchema;  /* Schema to which this item is fixed */
+    char *zDatabase;  /* Name of database holding this table */
+    Subquery *pSubq;  /* Description of a subquery */
+  } u4;
 };
 
 /*
@@ -3247,11 +3415,19 @@ struct OnOrUsing {
 **
 */
 struct SrcList {
-  int nSrc;        /* Number of tables or subqueries in the FROM clause */
-  u32 nAlloc;      /* Number of entries allocated in a[] below */
-  SrcItem a[1];    /* One entry for each identifier on the list */
+  int nSrc;             /* Number of tables or subqueries in the FROM clause */
+  u32 nAlloc;           /* Number of entries allocated in a[] below */
+  SrcItem a[FLEXARRAY]; /* One entry for each identifier on the list */
 };
 
+/* Size (in bytes) of a SrcList object that can hold as many as N
+** SrcItem objects. */
+#define SZ_SRCLIST(N) (offsetof(SrcList,a)+(N)*sizeof(SrcItem))
+
+/* Size (in bytes( of a SrcList object that holds 1 SrcItem.  This is a
+** special case of SZ_SRCITEM(1) that comes up often. */
+#define SZ_SRCLIST_1  (offsetof(SrcList,a)+sizeof(SrcItem))
+
 /*
 ** Permitted values of the SrcList.a.jointype field
 */
@@ -3287,7 +3463,7 @@ struct SrcList {
 #define WHERE_AGG_DISTINCT     0x0400 /* Query is "SELECT agg(DISTINCT ...)" */
 #define WHERE_ORDERBY_LIMIT    0x0800 /* ORDERBY+LIMIT on the inner loop */
 #define WHERE_RIGHT_JOIN       0x1000 /* Processing a RIGHT JOIN */
-                        /*     0x2000    not currently used */
+#define WHERE_KEEP_ALL_JOINS   0x2000 /* Do not do the omit-noop-join opt */
 #define WHERE_USE_LIMIT        0x4000 /* Use the LIMIT in cost estimates */
                         /*     0x8000    not currently used */
 
@@ -3332,6 +3508,7 @@ struct NameContext {
   int nRef;            /* Number of names resolved by this context */
   int nNcErr;          /* Number of errors encountered while resolving names */
   int ncFlags;         /* Zero or more NC_* flags defined below */
+  u32 nNestedSelect;   /* Number of nested selects using this NC */
   Select *pWinSelect;  /* SELECT statement for any window functions */
 };
 
@@ -3358,13 +3535,14 @@ struct NameContext {
 #define NC_UUpsert   0x000200 /* True if uNC.pUpsert is used */
 #define NC_UBaseReg  0x000400 /* True if uNC.iBaseReg is used */
 #define NC_MinMaxAgg 0x001000 /* min/max aggregates seen.  See note above */
-#define NC_Complex   0x002000 /* True if a function or subquery seen */
+/*                   0x002000 // available for reuse */
 #define NC_AllowWin  0x004000 /* Window functions are allowed here */
 #define NC_HasWin    0x008000 /* One or more window functions seen */
 #define NC_IsDDL     0x010000 /* Resolving names in a CREATE statement */
 #define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */
 #define NC_FromDDL   0x040000 /* SQL text comes from sqlite_schema */
 #define NC_NoSelect  0x080000 /* Do not descend into sub-selects */
+#define NC_Where     0x100000 /* Processing WHERE clause of a SELECT */
 #define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */
 
 /*
@@ -3376,7 +3554,7 @@ struct NameContext {
 ** conflict-target clause.)  The pUpsertTargetWhere is the optional
 ** WHERE clause used to identify partial unique indexes.
 **
-** pUpsertSet is the list of column=expr terms of the UPDATE statement. 
+** pUpsertSet is the list of column=expr terms of the UPDATE statement.
 ** The pUpsertSet field is NULL for a ON CONFLICT DO NOTHING.  The
 ** pUpsertWhere is the WHERE clause for the UPDATE and is NULL if the
 ** WHERE clause is omitted.
@@ -3388,6 +3566,7 @@ struct Upsert {
   Expr *pUpsertWhere;       /* WHERE clause for the ON CONFLICT UPDATE */
   Upsert *pNextUpsert;      /* Next ON CONFLICT clause in the list */
   u8 isDoUpdate;            /* True for DO UPDATE.  False for DO NOTHING */
+  u8 isDup;                 /* True if 2nd or later with same pUpsertIdx */
   /* Above this point is the parse tree for the ON CONFLICT clauses.
   ** The next group of fields stores intermediate data. */
   void *pToFree;            /* Free memory when deleting the Upsert object */
@@ -3477,14 +3656,18 @@ struct Select {
 #define SF_View          0x0200000 /* SELECT statement is a view */
 #define SF_NoopOrderBy   0x0400000 /* ORDER BY is ignored for this query */
 #define SF_UFSrcCheck    0x0800000 /* Check pSrc as required by UPDATE...FROM */
-#define SF_PushDown      0x1000000 /* SELECT has be modified by push-down opt */
+#define SF_PushDown      0x1000000 /* Modified by WHERE-clause push-down opt */
 #define SF_MultiPart     0x2000000 /* Has multiple incompatible PARTITIONs */
 #define SF_CopyCte       0x4000000 /* SELECT statement is a copy of a CTE */
 #define SF_OrderByReqd   0x8000000 /* The ORDER BY clause may not be omitted */
 #define SF_UpdateFrom   0x10000000 /* Query originates with UPDATE FROM */
+#define SF_Correlated   0x20000000 /* True if references the outer context */
+#define SF_OnToWhere    0x40000000 /* One or more ON clauses moved to WHERE */
 
-/* True if S exists and has SF_NestedFrom */
-#define IsNestedFrom(S) ((S)!=0 && ((S)->selFlags&SF_NestedFrom)!=0)
+/* True if SrcItem X is a subquery that has SF_NestedFrom */
+#define IsNestedFrom(X) \
+   ((X)->fg.isSubquery && \
+    ((X)->u4.pSubq->pSelect->selFlags&SF_NestedFrom)!=0)
 
 /*
 ** The results of a SELECT can be distributed in several ways, as defined
@@ -3514,7 +3697,11 @@ struct Select {
 **     SRT_Set         The result must be a single column.  Store each
 **                     row of result as the key in table pDest->iSDParm.
 **                     Apply the affinity pDest->affSdst before storing
-**                     results.  Used to implement "IN (SELECT ...)".
+**                     results.  if pDest->iSDParm2 is positive, then it is
+**                     a register holding a Bloom filter for the IN operator
+**                     that should be populated in addition to the
+**                     pDest->iSDParm table.  This SRT is used to
+**                     implement "IN (SELECT ...)".
 **
 **     SRT_EphemTab    Create an temporary table pDest->iSDParm and store
 **                     the result there. The cursor is left open after
@@ -3710,23 +3897,33 @@ struct Parse {
   char *zErrMsg;       /* An error message */
   Vdbe *pVdbe;         /* An engine for executing database bytecode */
   int rc;              /* Return code from execution */
-  u8 colNamesSet;      /* TRUE after OP_ColumnName has been issued to pVdbe */
-  u8 checkSchema;      /* Causes schema cookie check after an error */
+  LogEst nQueryLoop;   /* Est number of iterations of a query (10*log2(N)) */
   u8 nested;           /* Number of nested calls to the parser/code generator */
   u8 nTempReg;         /* Number of temporary registers in aTempReg[] */
   u8 isMultiWrite;     /* True if statement may modify/insert multiple rows */
   u8 mayAbort;         /* True if statement may throw an ABORT exception */
   u8 hasCompound;      /* Need to invoke convertCompoundSelectToSubquery() */
-  u8 okConstFactor;    /* OK to factor out constants */
   u8 disableLookaside; /* Number of times lookaside has been disabled */
   u8 prepFlags;        /* SQLITE_PREPARE_* flags */
   u8 withinRJSubrtn;   /* Nesting level for RIGHT JOIN body subroutines */
+  u8 bHasExists;       /* Has a correlated "EXISTS (SELECT ....)" expression */
+  u8 mSubrtnSig;       /* mini Bloom filter on available SubrtnSig.selId */
+  u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
+  u8 bReturning;       /* Coding a RETURNING trigger */
+  u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
+  u8 disableTriggers;  /* True to disable triggers */
 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
   u8 earlyCleanup;     /* OOM inside sqlite3ParserAddCleanup() */
 #endif
 #ifdef SQLITE_DEBUG
   u8 ifNotExists;      /* Might be true if IF NOT EXISTS.  Assert()s only */
+  u8 isCreate;         /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER)
+                       ** and ALTER TABLE ADD COLUMN. */
 #endif
+  bft colNamesSet :1;   /* TRUE after OP_ColumnName has been issued to pVdbe */
+  bft bHasWith :1;      /* True if statement contains WITH */
+  bft okConstFactor :1; /* OK to factor out constants */
+  bft checkSchema :1;   /* Causes schema cookie check after an error */
   int nRangeReg;       /* Size of the temporary register block */
   int iRangeReg;       /* First register in temporary register block */
   int nErr;            /* Number of errors seen */
@@ -3740,13 +3937,14 @@ struct Parse {
   int *aLabel;         /* Space to hold the labels */
   ExprList *pConstExpr;/* Constant expressions */
   IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */
-  Token constraintName;/* Name of the constraint currently being parsed */
+  IndexedExpr *pIdxPartExpr; /* Exprs constrained by index WHERE clauses */
   yDbMask writeMask;   /* Start a write transaction on these databases */
   yDbMask cookieMask;  /* Bitmask of schema verified databases */
-  int regRowid;        /* Register holding rowid of CREATE TABLE entry */
-  int regRoot;         /* Register holding root page number for new objects */
-  int nMaxArg;         /* Max args passed to user function by sub-program */
+  int nMaxArg;         /* Max args to xUpdate and xFilter vtab methods */
   int nSelect;         /* Number of SELECT stmts. Counter for Select.selId */
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+  u32 nProgressSteps;  /* xProgress steps taken during sqlite3_prepare() */
+#endif
 #ifndef SQLITE_OMIT_SHARED_CACHE
   int nTableLock;        /* Number of locks in aTableLock */
   TableLock *aTableLock; /* Required table locks for shared-cache mode */
@@ -3756,20 +3954,6 @@ struct Parse {
   Table *pTriggerTab;  /* Table triggers are being coded for */
   TriggerPrg *pTriggerPrg;  /* Linked list of coded triggers */
   ParseCleanup *pCleanup;   /* List of cleanup operations to run after parse */
-  union {
-    int addrCrTab;         /* Address of OP_CreateBtree on CREATE TABLE */
-    Returning *pReturning; /* The RETURNING clause */
-  } u1;
-  u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
-  u32 oldmask;         /* Mask of old.* columns referenced */
-  u32 newmask;         /* Mask of new.* columns referenced */
-#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
-  u32 nProgressSteps;  /* xProgress steps taken during sqlite3_prepare() */
-#endif
-  u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
-  u8 bReturning;       /* Coding a RETURNING trigger */
-  u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
-  u8 disableTriggers;  /* True to disable triggers */
 
   /**************************************************************************
   ** Fields above must be initialized to zero.  The fields that follow,
@@ -3781,6 +3965,19 @@ struct Parse {
   int aTempReg[8];        /* Holding area for temporary registers */
   Parse *pOuterParse;     /* Outer Parse object when nested */
   Token sNameToken;       /* Token with unqualified schema object name */
+  u32 oldmask;            /* Mask of old.* columns referenced */
+  u32 newmask;            /* Mask of new.* columns referenced */
+  union {
+    struct {  /* These fields available when isCreate is true */
+      int addrCrTab;        /* Address of OP_CreateBtree on CREATE TABLE */
+      int regRowid;         /* Register holding rowid of CREATE TABLE entry */
+      int regRoot;          /* Register holding root page for new objects */
+      Token constraintName; /* Name of the constraint currently being parsed */
+    } cr;
+    struct {  /* These fields available to all other statements */
+      Returning *pReturning; /* The RETURNING clause */
+    } d;
+  } u1;
 
   /************************************************************************
   ** Above is constant between recursions.  Below is reset before and after
@@ -3798,9 +3995,7 @@ struct Parse {
   int nVtabLock;            /* Number of virtual tables to lock */
 #endif
   int nHeight;              /* Expression tree height of current sub-select */
-#ifndef SQLITE_OMIT_EXPLAIN
   int addrExplain;          /* Address of current OP_Explain opcode */
-#endif
   VList *pVList;            /* Mapping between variable names and numbers */
   Vdbe *pReprepare;         /* VM being reprepared (sqlite3Reprepare()) */
   const char *zTail;        /* All SQL text past the last semicolon parsed */
@@ -3889,6 +4084,7 @@ struct AuthContext {
 #define OPFLAG_ISNOOP        0x40    /* OP_Delete does pre-update-hook only */
 #define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
 #define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
+#define OPFLAG_BYTELENARG    0xc0    /* OP_Column only for octet_length() */
 #define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
 #define OPFLAG_SEEKEQ        0x02    /* OP_Open** cursor uses EQ seek only */
 #define OPFLAG_FORDELETE     0x08    /* OP_Open should use BTREE_FORDELETE */
@@ -3897,7 +4093,7 @@ struct AuthContext {
 #define OPFLAG_SAVEPOSITION  0x02    /* OP_Delete/Insert: save cursor pos */
 #define OPFLAG_AUXDELETE     0x04    /* OP_Delete: index in a DELETE op */
 #define OPFLAG_NOCHNG_MAGIC  0x6d    /* OP_MakeRecord: serialtype 10 is ok */
-#define OPFLAG_PREFORMAT     0x80    /* OP_Insert uses preformatted cell */ 
+#define OPFLAG_PREFORMAT     0x80    /* OP_Insert uses preformatted cell */
 
 /*
 ** Each trigger present in the database schema is stored as an instance of
@@ -4010,10 +4206,11 @@ struct Returning {
   int iRetCur;          /* Transient table holding RETURNING results */
   int nRetCol;          /* Number of in pReturnEL after expansion */
   int iRetReg;          /* Register array for holding a row of RETURNING */
+  char zName[40];       /* Name of trigger: "sqlite_returning_%p" */
 };
 
 /*
-** An objected used to accumulate the text of a string where we
+** An object used to accumulate the text of a string where we
 ** do not necessarily know how big the string will be in the end.
 */
 struct sqlite3_str {
@@ -4027,10 +4224,32 @@ struct sqlite3_str {
 };
 #define SQLITE_PRINTF_INTERNAL 0x01  /* Internal-use-only converters allowed */
 #define SQLITE_PRINTF_SQLFUNC  0x02  /* SQL function arguments to VXPrintf */
-#define SQLITE_PRINTF_MALLOCED 0x04  /* True if xText is allocated space */
+#define SQLITE_PRINTF_MALLOCED 0x04  /* True if zText is allocated space */
 
 #define isMalloced(X)  (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0)
 
+/*
+** The following object is the header for an "RCStr" or "reference-counted
+** string".  An RCStr is passed around and used like any other char*
+** that has been dynamically allocated.  The important interface
+** differences:
+**
+**   1.  RCStr strings are reference counted.  They are deallocated
+**       when the reference count reaches zero.
+**
+**   2.  Use sqlite3RCStrUnref() to free an RCStr string rather than
+**       sqlite3_free()
+**
+**   3.  Make a (read-only) copy of a read-only RCStr string using
+**       sqlite3RCStrRef().
+**
+** "String" is in the name, but an RCStr object can also be used to hold
+** binary data.
+*/
+struct RCStr {
+  u64 nRCRef;            /* Number of references */
+  /* Total structure size should be a multiple of 8 bytes for alignment */
+};
 
 /*
 ** A pointer to this structure is used to communicate information
@@ -4057,7 +4276,7 @@ typedef struct {
 /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled
 ** on debug-builds of the CLI using ".testctrl tune ID VALUE".  Tuning
 ** parameters are for temporary use during development, to help find
-** optimial values for parameters in the query planner.  The should not
+** optimal values for parameters in the query planner.  The should not
 ** be used on trunk check-ins.  They are a temporary mechanism available
 ** for transient development builds only.
 **
@@ -4083,6 +4302,9 @@ struct Sqlite3Config {
   u8 bUseCis;                       /* Use covering indices for full-scans */
   u8 bSmallMalloc;                  /* Avoid large memory allocations if true */
   u8 bExtraSchemaChecks;            /* Verify type,name,tbl_name in schema */
+#ifdef SQLITE_DEBUG
+  u8 bJsonSelfcheck;                /* Double-check JSON parsing */
+#endif
   int mxStrlen;                     /* Maximum string length */
   int neverCorrupt;                 /* Database is always well-formed */
   int szLookaside;                  /* Default lookaside buffer size */
@@ -4129,6 +4351,11 @@ struct Sqlite3Config {
 #endif
 #ifndef SQLITE_UNTESTABLE
   int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
+#endif
+#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
+  u32 mNoVisibleRowid;              /* TF_NoVisibleRowid if the ROWID_IN_VIEW
+                                    ** feature is disabled.  0 if rowids can
+                                    ** occur in views. */
 #endif
   int bLocaltimeFault;              /* True to fail localtime() calls */
   int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */
@@ -4169,6 +4396,7 @@ struct Walker {
   void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */
   int walkerDepth;                          /* Number of subqueries */
   u16 eCode;                                /* A small processing code */
+  u16 mWFlags;                              /* Use-dependent flags */
   union {                                   /* Extra data for callback */
     NameContext *pNC;                         /* Naming context */
     int n;                                    /* A counter */
@@ -4188,6 +4416,7 @@ struct Walker {
     SrcItem *pSrcItem;                        /* A single FROM clause item */
     DbFixer *pFix;                            /* See sqlite3FixSelect() */
     Mem *aMem;                                /* See sqlite3BtreeCursorHint() */
+    struct CheckOnCtx *pCheckOnCtx;           /* See selectCheckOnClauses() */
   } u;
 };
 
@@ -4208,6 +4437,7 @@ struct DbFixer {
 
 /* Forward declarations */
 int sqlite3WalkExpr(Walker*, Expr*);
+int sqlite3WalkExprNN(Walker*, Expr*);
 int sqlite3WalkExprList(Walker*, ExprList*);
 int sqlite3WalkSelect(Walker*, Select*);
 int sqlite3WalkSelectExpr(Walker*, Select*);
@@ -4264,9 +4494,13 @@ struct With {
   int nCte;               /* Number of CTEs in the WITH clause */
   int bView;              /* Belongs to the outermost Select of a view */
   With *pOuter;           /* Containing WITH clause, or NULL */
-  Cte a[1];               /* For each CTE in the WITH clause.... */
+  Cte a[FLEXARRAY];       /* For each CTE in the WITH clause.... */
 };
 
+/* The size (in bytes) of a With object that can hold as many
+** as N different CTEs. */
+#define SZ_WITH(N)  (offsetof(With,a) + (N)*sizeof(Cte))
+
 /*
 ** The Cte object is not guaranteed to persist for the entire duration
 ** of code generation.  (The query flattener or other parser tree
@@ -4288,6 +4522,20 @@ struct CteUse {
 };
 
 
+/* Client data associated with sqlite3_set_clientdata() and
+** sqlite3_get_clientdata().
+*/
+struct DbClientData {
+  DbClientData *pNext;        /* Next in a linked list */
+  void *pData;                /* The data */
+  void (*xDestructor)(void*); /* Destructor.  Might be NULL */
+  char zName[FLEXARRAY];      /* Name of this client data. MUST BE LAST */
+};
+
+/* The size (in bytes) of a DbClientData object that can has a name
+** that is N bytes long, including the zero-terminator. */
+#define SZ_DBCLIENTDATA(N) (offsetof(DbClientData,zName)+(N))
+
 #ifdef SQLITE_DEBUG
 /*
 ** An instance of the TreeView object is used for printing the content of
@@ -4354,6 +4602,9 @@ struct Window {
                           ** due to the SQLITE_SUBTYPE flag */
 };
 
+Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow);
+void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal);
+
 #ifndef SQLITE_OMIT_WINDOWFUNC
 void sqlite3WindowDelete(sqlite3*, Window*);
 void sqlite3WindowUnlinkFromSelect(Window*);
@@ -4434,15 +4685,6 @@ int sqlite3CantopenError(int);
 # define SQLITE_ENABLE_FTS3 1
 #endif
 
-/*
-** The ctype.h header is needed for non-ASCII systems.  It is also
-** needed by FTS3 when FTS3 is included in the amalgamation.
-*/
-#if !defined(SQLITE_ASCII) || \
-    (defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_AMALGAMATION))
-# include <ctype.h>
-#endif
-
 /*
 ** The following macros mimic the standard library functions toupper(),
 ** isspace(), isalnum(), isdigit() and isxdigit(), respectively. The
@@ -4573,10 +4815,13 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex*);
 # define EXP754 (((u64)0x7ff)<<52)
 # define MAN754 ((((u64)1)<<52)-1)
 # define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0)
+# define IsOvfl(X) (((X)&EXP754)==EXP754)
   int sqlite3IsNaN(double);
+  int sqlite3IsOverflow(double);
 #else
-# define IsNaN(X)         0
-# define sqlite3IsNaN(X)  0
+# define IsNaN(X)             0
+# define sqlite3IsNaN(X)      0
+# define sqlite3IsOVerflow(X) 0
 #endif
 
 /*
@@ -4589,6 +4834,20 @@ struct PrintfArguments {
   sqlite3_value **apArg;   /* The argument values */
 };
 
+/*
+** An instance of this object receives the decoding of a floating point
+** value into an approximate decimal representation.
+*/
+struct FpDecode {
+  char sign;           /* '+' or '-' */
+  char isSpecial;      /* 1: Infinity  2: NaN */
+  int n;               /* Significant digits in the decode */
+  int iDP;             /* Location of the decimal point */
+  char *z;             /* Start of significant digits */
+  char zBuf[24];       /* Storage for significant digits */
+};
+
+void sqlite3FpDecode(FpDecode*,double,int,int);
 char *sqlite3MPrintf(sqlite3*,const char*, ...);
 char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
 #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE)
@@ -4645,6 +4904,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
   void sqlite3ShowWindow(const Window*);
   void sqlite3ShowWinFunc(const Window*);
 #endif
+  void sqlite3ShowBitvec(Bitvec*);
 #endif
 
 void sqlite3SetString(char **, sqlite3*, const char*);
@@ -4654,6 +4914,7 @@ int sqlite3ErrorToParser(sqlite3*,int);
 void sqlite3Dequote(char*);
 void sqlite3DequoteExpr(Expr*);
 void sqlite3DequoteToken(Token*);
+void sqlite3DequoteNumber(Parse*, Expr*);
 void sqlite3TokenInit(Token*,char*);
 int sqlite3KeywordCode(const unsigned char*, int);
 int sqlite3RunParser(Parse*, const char*);
@@ -4678,10 +4939,13 @@ void sqlite3PExprAddSelect(Parse*, Expr*, Select*);
 Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*);
 Expr *sqlite3ExprSimplifiedAndOr(Expr*);
 Expr *sqlite3ExprFunction(Parse*,ExprList*, const Token*, int);
+void sqlite3ExprAddFunctionOrderBy(Parse*,Expr*,ExprList*);
+void sqlite3ExprOrderByAggregateError(Parse*,Expr*);
 void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*);
 void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32);
 void sqlite3ExprDelete(sqlite3*, Expr*);
-void sqlite3ExprDeferredDelete(Parse*, Expr*);
+void sqlite3ExprDeleteGeneric(sqlite3*,void*);
+int sqlite3ExprDeferredDelete(Parse*, Expr*);
 void sqlite3ExprUnmapAndDelete(Parse*, Expr*);
 ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
 ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*);
@@ -4690,6 +4954,7 @@ void sqlite3ExprListSetSortOrder(ExprList*,int,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*);
 void sqlite3ExprListDelete(sqlite3*, ExprList*);
+void sqlite3ExprListDeleteGeneric(sqlite3*,void*);
 u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3IndexHasDuplicateRootPage(Index*);
 int sqlite3Init(sqlite3*, char**);
@@ -4714,7 +4979,7 @@ void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char);
 Table *sqlite3ResultSetOfSelect(Parse*,Select*,char);
 void sqlite3OpenSchemaTable(Parse *, int);
 Index *sqlite3PrimaryKeyIndex(Table*);
-i16 sqlite3TableColumnToIndex(Index*, i16);
+int sqlite3TableColumnToIndex(Index*, int);
 #ifdef SQLITE_OMIT_GENERATED_COLUMNS
 # define sqlite3TableColumnToStorage(T,X) (X)  /* No-op pass-through */
 # define sqlite3StorageColumnToTable(T,X) (X)  /* No-op pass-through */
@@ -4786,6 +5051,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int);
 void sqlite3DropTable(Parse*, SrcList*, int, int);
 void sqlite3CodeDropTable(Parse*, Table*, int, int);
 void sqlite3DeleteTable(sqlite3*, Table*);
+void sqlite3DeleteTableGeneric(sqlite3*, void*);
 void sqlite3FreeIndex(sqlite3*, Index*);
 #ifndef SQLITE_OMIT_AUTOINCREMENT
   void sqlite3AutoincrementBegin(Parse *pParse);
@@ -4804,6 +5070,9 @@ int sqlite3IdListIndex(IdList*,const char*);
 SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int);
 SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2);
 SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*);
+void sqlite3SubqueryDelete(sqlite3*,Subquery*);
+Select *sqlite3SubqueryDetach(sqlite3*,SrcItem*);
+int sqlite3SrcItemAttachSubquery(Parse*, SrcItem*, Select*, int);
 SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
                                       Token*, Select*, OnOrUsing*);
 void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *);
@@ -4814,7 +5083,7 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*);
 void sqlite3IdListDelete(sqlite3*, IdList*);
 void sqlite3ClearOnOrUsing(sqlite3*, OnOrUsing*);
 void sqlite3SrcListDelete(sqlite3*, SrcList*);
-Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**);
+Index *sqlite3AllocateIndexObject(sqlite3*,int,int,char**);
 void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
                           Expr*, int, int, u8);
 void sqlite3DropIndex(Parse*, SrcList*, int);
@@ -4822,6 +5091,7 @@ int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
                          Expr*,ExprList*,u32,Expr*);
 void sqlite3SelectDelete(sqlite3*, Select*);
+void sqlite3SelectDeleteGeneric(sqlite3*,void*);
 Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
 void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
@@ -4852,6 +5122,7 @@ void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int);
 int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8);
 void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
 void sqlite3ExprCodeMove(Parse*, int, int, int);
+void sqlite3ExprToRegister(Expr *pExpr, int iReg);
 void sqlite3ExprCode(Parse*, Expr*, int);
 #ifndef SQLITE_OMIT_GENERATED_COLUMNS
 void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int);
@@ -4859,6 +5130,7 @@ void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int);
 void sqlite3ExprCodeCopy(Parse*, Expr*, int);
 void sqlite3ExprCodeFactorable(Parse*, Expr*, int);
 int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int);
+void sqlite3ExprNullRegisterRange(Parse*, int, int);
 int sqlite3ExprCodeTemp(Parse*, Expr*, int*);
 int sqlite3ExprCodeTarget(Parse*, Expr*, int);
 int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8);
@@ -4885,7 +5157,7 @@ int sqlite3ExprCompare(const Parse*,const Expr*,const Expr*, int);
 int sqlite3ExprCompareSkip(Expr*,Expr*,int);
 int sqlite3ExprListCompare(const ExprList*,const ExprList*, int);
 int sqlite3ExprImpliesExpr(const Parse*,const Expr*,const Expr*, int);
-int sqlite3ExprImpliesNonNullRow(Expr*,int);
+int sqlite3ExprImpliesNonNullRow(Expr*,int,int);
 void sqlite3AggInfoPersistWalkerInit(Walker*,Parse*);
 void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
 void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*);
@@ -4907,19 +5179,18 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*);
 u32 sqlite3IsTrueOrFalse(const char*);
 int sqlite3ExprIdToTrueFalse(Expr*);
 int sqlite3ExprTruthValue(const Expr*);
-int sqlite3ExprIsConstant(Expr*);
-int sqlite3ExprIsConstantNotJoin(Expr*);
+int sqlite3ExprIsConstant(Parse*,Expr*);
 int sqlite3ExprIsConstantOrFunction(Expr*, u8);
 int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*);
-int sqlite3ExprIsTableConstant(Expr*,int);
-int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int);
+int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int);
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
 int sqlite3ExprContainsSubquery(Expr*);
 #endif
-int sqlite3ExprIsInteger(const Expr*, int*);
+int sqlite3ExprIsInteger(const Expr*, int*, Parse*);
 int sqlite3ExprCanBeNull(const Expr*);
 int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
 int sqlite3IsRowid(const char*);
+const char *sqlite3RowidAlias(Table *pTab);
 void sqlite3GenerateRowDelete(
     Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
 void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
@@ -4949,19 +5220,24 @@ Select *sqlite3SelectDup(sqlite3*,const Select*,int);
 FuncDef *sqlite3FunctionSearch(int,const char*);
 void sqlite3InsertBuiltinFuncs(FuncDef*,int);
 FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
-void sqlite3QuoteValue(StrAccum*,sqlite3_value*);
+void sqlite3QuoteValue(StrAccum*,sqlite3_value*,int);
+int sqlite3AppendOneUtf8Character(char*, u32);
 void sqlite3RegisterBuiltinFunctions(void);
 void sqlite3RegisterDateTimeFunctions(void);
 void sqlite3RegisterJsonFunctions(void);
 void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON)
-  int sqlite3JsonTableFunctions(sqlite3*);
+  Module *sqlite3JsonVtabRegister(sqlite3*,const char*);
 #endif
 int sqlite3SafetyCheckOk(sqlite3*);
 int sqlite3SafetyCheckSickOrOk(sqlite3*);
 void sqlite3ChangeCookie(Parse*, int);
 With *sqlite3WithDup(sqlite3 *db, With *p);
 
+#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY)
+  Module *sqlite3CarrayRegister(sqlite3*);
+#endif
+
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
 void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int);
 #endif
@@ -5034,6 +5310,7 @@ int sqlite3FixSrcList(DbFixer*, SrcList*);
 int sqlite3FixSelect(DbFixer*, Select*);
 int sqlite3FixExpr(DbFixer*, Expr*);
 int sqlite3FixTriggerStep(DbFixer*, TriggerStep*);
+
 int sqlite3RealSameAsInt(double,sqlite3_int64);
 i64 sqlite3RealToI64(double);
 int sqlite3Int64ToText(i64,char*);
@@ -5042,10 +5319,11 @@ int sqlite3GetInt32(const char *, int*);
 int sqlite3GetUInt32(const char*, u32*);
 int sqlite3Atoi(const char*);
 #ifndef SQLITE_OMIT_UTF16
-int sqlite3Utf16ByteLen(const void *pData, int nChar);
+int sqlite3Utf16ByteLen(const void *pData, int nByte, int nChar);
 #endif
 int sqlite3Utf8CharLen(const char *pData, int nByte);
 u32 sqlite3Utf8Read(const u8**);
+int sqlite3Utf8ReadLimited(const u8*, int, u32*);
 LogEst sqlite3LogEst(u64);
 LogEst sqlite3LogEstAdd(LogEst,LogEst);
 LogEst sqlite3LogEstFromDouble(double);
@@ -5094,7 +5372,9 @@ void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...);
 void sqlite3Error(sqlite3*,int);
 void sqlite3ErrorClear(sqlite3*);
 void sqlite3SystemError(sqlite3*,int);
+#if !defined(SQLITE_OMIT_BLOB_LITERAL)
 void *sqlite3HexToBlob(sqlite3*, const char *z, int n);
+#endif
 u8 sqlite3HexToInt(int h);
 int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
 
@@ -5138,6 +5418,7 @@ void sqlite3FileSuffix3(const char*, char*);
 u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
+int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*));
 int sqlite3ValueBytes(sqlite3_value*, u8);
 void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
@@ -5177,7 +5458,7 @@ void sqlite3Reindex(Parse*, Token*, Token*);
 void sqlite3AlterFunctions(void);
 void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
 void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
-int sqlite3GetToken(const unsigned char *, int *);
+i64 sqlite3GetToken(const unsigned char *, int *);
 void sqlite3NestedParse(Parse*, const char*, ...);
 void sqlite3ExpirePreparedStatements(sqlite3*, int);
 void sqlite3CodeRhsOfIN(Parse*, Expr*, int);
@@ -5189,7 +5470,8 @@ int sqlite3MatchEName(
   const struct ExprList_item*,
   const char*,
   const char*,
-  const char*
+  const char*,
+  int*
 );
 Bitmask sqlite3ExprColUsed(Expr*);
 u8 sqlite3StrIHash(const char*);
@@ -5233,10 +5515,10 @@ int sqlite3KeyInfoIsWriteable(KeyInfo*);
 #endif
 int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *,
   void (*)(sqlite3_context*,int,sqlite3_value **),
-  void (*)(sqlite3_context*,int,sqlite3_value **), 
+  void (*)(sqlite3_context*,int,sqlite3_value **),
   void (*)(sqlite3_context*),
   void (*)(sqlite3_context*),
-  void (*)(sqlite3_context*,int,sqlite3_value **), 
+  void (*)(sqlite3_context*,int,sqlite3_value **),
   FuncDestructor *pDestructor
 );
 void sqlite3NoopDestructor(void*);
@@ -5245,6 +5527,11 @@ void sqlite3OomClear(sqlite3*);
 int sqlite3ApiExit(sqlite3 *db, int);
 int sqlite3OpenTempDatabase(Parse *);
 
+char *sqlite3RCStrRef(char*);
+void sqlite3RCStrUnref(void*);
+char *sqlite3RCStrNew(u64);
+char *sqlite3RCStrResize(char*,u64);
+
 void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int);
 int sqlite3StrAccumEnlarge(StrAccum*, i64);
 char *sqlite3StrAccumFinish(StrAccum*);
@@ -5385,6 +5672,7 @@ const char *sqlite3JournalModename(int);
   void sqlite3CteDelete(sqlite3*,Cte*);
   With *sqlite3WithAdd(Parse*,With*,Cte*);
   void sqlite3WithDelete(sqlite3*,With*);
+  void sqlite3WithDeleteGeneric(sqlite3*,void*);
   With *sqlite3WithPush(Parse*, With*, u8);
 #else
 # define sqlite3CteNew(P,T,E,S)   ((void*)0)
@@ -5397,7 +5685,7 @@ const char *sqlite3JournalModename(int);
   Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*);
   void sqlite3UpsertDelete(sqlite3*,Upsert*);
   Upsert *sqlite3UpsertDup(sqlite3*,Upsert*);
-  int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*);
+  int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*);
   void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int);
   Upsert *sqlite3UpsertOfIndex(Upsert*,Index*);
   int sqlite3UpsertNextIsIPK(Upsert*);
@@ -5496,6 +5784,7 @@ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p);
   #define sqlite3SelectExprHeight(x) 0
   #define sqlite3ExprCheckHeight(x,y)
 #endif
+void sqlite3ExprSetErrorOffset(Expr*,int);
 
 u32 sqlite3Get4byte(const u8*);
 void sqlite3Put4byte(u8*, u32);
diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h
index 08703cb73a..6b6bb7167a 100644
--- a/src/sqliteLimit.h
+++ b/src/sqliteLimit.h
@@ -9,7 +9,7 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** 
+**
 ** This file defines various limits of what SQLite can process.
 */
 
@@ -23,6 +23,7 @@
 #ifndef SQLITE_MAX_LENGTH
 # define SQLITE_MAX_LENGTH 1000000000
 #endif
+#define SQLITE_MIN_LENGTH 30   /* Minimum value for the length limit */
 
 /*
 ** This is the maximum number of
@@ -35,14 +36,22 @@
 **    * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement.
 **    * Terms in the VALUES clause of an INSERT statement
 **
-** The hard upper limit here is 32676.  Most database people will
+** The hard upper limit here is 32767.  Most database people will
 ** tell you that in a well-normalized database, you usually should
 ** not have more than a dozen or so columns in any table.  And if
 ** that is the case, there is no point in having more than a few
 ** dozen values in any of the other situations described above.
+**
+** An index can only have SQLITE_MAX_COLUMN columns from the user
+** point of view, but the underlying b-tree that implements the index
+** might have up to twice as many columns in a WITHOUT ROWID table,
+** since must also store the primary key at the end.  Hence the
+** column count for Index is u16 instead of i16.
 */
-#ifndef SQLITE_MAX_COLUMN
+#if !defined(SQLITE_MAX_COLUMN)
 # define SQLITE_MAX_COLUMN 2000
+#elif SQLITE_MAX_COLUMN>32767
+# error SQLITE_MAX_COLUMN may not exceed 32767
 #endif
 
 /*
@@ -57,9 +66,9 @@
 #endif
 
 /*
-** The maximum depth of an expression tree. This is limited to 
-** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might 
-** want to place more severe limits on the complexity of an 
+** The maximum depth of an expression tree. This is limited to
+** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might
+** want to place more severe limits on the complexity of an
 ** expression. A value of 0 means that there is no limit.
 */
 #ifndef SQLITE_MAX_EXPR_DEPTH
@@ -72,7 +81,7 @@
 ** level of recursion for each term.  A stack overflow can result
 ** if the number of terms is too large.  In practice, most SQL
 ** never has more than 3 or 4 terms.  Use a value of 0 to disable
-** any limit on the number of terms in a compount SELECT.
+** any limit on the number of terms in a compound SELECT.
 */
 #ifndef SQLITE_MAX_COMPOUND_SELECT
 # define SQLITE_MAX_COMPOUND_SELECT 500
@@ -88,9 +97,13 @@
 
 /*
 ** The maximum number of arguments to an SQL function.
+**
+** This value has a hard upper limit of 32767 due to storage
+** constraints (it needs to fit inside a i16).  We keep it
+** lower than that to prevent abuse.
 */
 #ifndef SQLITE_MAX_FUNCTION_ARG
-# define SQLITE_MAX_FUNCTION_ARG 127
+# define SQLITE_MAX_FUNCTION_ARG 1000
 #endif
 
 /*
@@ -140,10 +153,10 @@
 **
 ** Earlier versions of SQLite allowed the user to change this value at
 ** compile time. This is no longer permitted, on the grounds that it creates
-** a library that is technically incompatible with an SQLite library 
-** compiled with a different limit. If a process operating on a database 
-** with a page-size of 65536 bytes crashes, then an instance of SQLite 
-** compiled with the default page-size limit will not be able to rollback 
+** a library that is technically incompatible with an SQLite library
+** compiled with a different limit. If a process operating on a database
+** with a page-size of 65536 bytes crashes, then an instance of SQLite
+** compiled with the default page-size limit will not be able to rollback
 ** the aborted transaction. This could lead to database corruption.
 */
 #ifdef SQLITE_MAX_PAGE_SIZE
@@ -183,11 +196,11 @@
 ** Maximum number of pages in one database file.
 **
 ** This is really just the default value for the max_page_count pragma.
-** This value can be lowered (or raised) at run-time using that the
+** This value can be lowered (or raised) at run-time using the
 ** max_page_count macro.
 */
 #ifndef SQLITE_MAX_PAGE_COUNT
-# define SQLITE_MAX_PAGE_COUNT 1073741823
+# define SQLITE_MAX_PAGE_COUNT 0xfffffffe /* 4294967294 */
 #endif
 
 /*
@@ -202,7 +215,7 @@
 ** Maximum depth of recursion for triggers.
 **
 ** A value of 1 means that a trigger program will not be able to itself
-** fire any triggers. A value of 0 means that no trigger programs at all 
+** fire any triggers. A value of 0 means that no trigger programs at all
 ** may be executed.
 */
 #ifndef SQLITE_MAX_TRIGGER_DEPTH
diff --git a/src/status.c b/src/status.c
index 7840167002..5db67b87b5 100644
--- a/src/status.c
+++ b/src/status.c
@@ -192,30 +192,33 @@ int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){
   nInit += countLookasideSlots(db->lookaside.pSmallInit);
   nFree += countLookasideSlots(db->lookaside.pSmallFree);
 #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */
-  if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit;
-  return db->lookaside.nSlot - (nInit+nFree);
+  assert( db->lookaside.nSlot >= nInit+nFree );
+  if( pHighwater ) *pHighwater = (int)(db->lookaside.nSlot - nInit);
+  return (int)(db->lookaside.nSlot - (nInit+nFree));
 }
 
 /*
 ** Query status information for a single database connection
 */
-int sqlite3_db_status(
-  sqlite3 *db,          /* The database connection whose status is desired */
-  int op,               /* Status verb */
-  int *pCurrent,        /* Write current value here */
-  int *pHighwater,      /* Write high-water mark here */
-  int resetFlag         /* Reset high-water mark if true */
+int sqlite3_db_status64(
+  sqlite3 *db,             /* The database connection whose status is desired */
+  int op,                  /* Status verb */
+  sqlite3_int64 *pCurrent, /* Write current value here */
+  sqlite3_int64 *pHighwtr, /* Write high-water mark here */
+  int resetFlag            /* Reset high-water mark if true */
 ){
   int rc = SQLITE_OK;   /* Return code */
 #ifdef SQLITE_ENABLE_API_ARMOR
-  if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){
+  if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){
     return SQLITE_MISUSE_BKPT;
   }
 #endif
   sqlite3_mutex_enter(db->mutex);
   switch( op ){
     case SQLITE_DBSTATUS_LOOKASIDE_USED: {
-      *pCurrent = sqlite3LookasideUsed(db, pHighwater);
+      int H = 0;
+      *pCurrent = sqlite3LookasideUsed(db, &H);
+      *pHighwtr = H;
       if( resetFlag ){
         LookasideSlot *p = db->lookaside.pFree;
         if( p ){
@@ -246,7 +249,7 @@ int sqlite3_db_status(
       assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 );
       assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 );
       *pCurrent = 0;
-      *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT];
+      *pHighwtr = db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT];
       if( resetFlag ){
         db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0;
       }
@@ -260,7 +263,7 @@ int sqlite3_db_status(
     */
     case SQLITE_DBSTATUS_CACHE_USED_SHARED:
     case SQLITE_DBSTATUS_CACHE_USED: {
-      int totalUsed = 0;
+      sqlite3_int64 totalUsed = 0;
       int i;
       sqlite3BtreeEnterAll(db);
       for(i=0; i<db->nDb; i++){
@@ -276,18 +279,18 @@ int sqlite3_db_status(
       }
       sqlite3BtreeLeaveAll(db);
       *pCurrent = totalUsed;
-      *pHighwater = 0;
+      *pHighwtr = 0;
       break;
     }
 
     /*
     ** *pCurrent gets an accurate estimate of the amount of memory used
     ** to store the schema for all databases (main, temp, and any ATTACHed
-    ** databases.  *pHighwater is set to zero.
+    ** databases.  *pHighwtr is set to zero.
     */
     case SQLITE_DBSTATUS_SCHEMA_USED: {
-      int i;                      /* Used to iterate through schemas */
-      int nByte = 0;              /* Used to accumulate return value */
+      int i;          /* Used to iterate through schemas */
+      int nByte = 0;  /* Used to accumulate return value */
 
       sqlite3BtreeEnterAll(db);
       db->pnBytesFreed = &nByte;
@@ -321,7 +324,7 @@ int sqlite3_db_status(
       db->lookaside.pEnd = db->lookaside.pTrueEnd;
       sqlite3BtreeLeaveAll(db);
 
-      *pHighwater = 0;
+      *pHighwtr = 0;
       *pCurrent = nByte;
       break;
     }
@@ -329,7 +332,7 @@ int sqlite3_db_status(
     /*
     ** *pCurrent gets an accurate estimate of the amount of memory used
     ** to store all prepared statements.
-    ** *pHighwater is set to zero.
+    ** *pHighwtr is set to zero.
     */
     case SQLITE_DBSTATUS_STMT_USED: {
       struct Vdbe *pVdbe;         /* Used to iterate through VMs */
@@ -344,7 +347,7 @@ int sqlite3_db_status(
       db->lookaside.pEnd = db->lookaside.pTrueEnd;
       db->pnBytesFreed = 0;
 
-      *pHighwater = 0;  /* IMP: R-64479-57858 */
+      *pHighwtr = 0;  /* IMP: R-64479-57858 */
       *pCurrent = nByte;
 
       break;
@@ -352,7 +355,7 @@ int sqlite3_db_status(
 
     /*
     ** Set *pCurrent to the total cache hits or misses encountered by all
-    ** pagers the database handle is connected to. *pHighwater is always set 
+    ** pagers the database handle is connected to. *pHighwtr is always set 
     ** to zero.
     */
     case SQLITE_DBSTATUS_CACHE_SPILL:
@@ -362,7 +365,7 @@ int sqlite3_db_status(
     case SQLITE_DBSTATUS_CACHE_MISS:
     case SQLITE_DBSTATUS_CACHE_WRITE:{
       int i;
-      int nRet = 0;
+      u64 nRet = 0;
       assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 );
       assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 );
 
@@ -372,19 +375,39 @@ int sqlite3_db_status(
           sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet);
         }
       }
-      *pHighwater = 0; /* IMP: R-42420-56072 */
+      *pHighwtr = 0; /* IMP: R-42420-56072 */
                        /* IMP: R-54100-20147 */
                        /* IMP: R-29431-39229 */
       *pCurrent = nRet;
       break;
     }
 
+    /* Set *pCurrent to the number of bytes that the db database connection
+    ** has spilled to the filesystem in temporary files that could have been
+    ** stored in memory, had sufficient memory been available.
+    ** The *pHighwater is always set to zero.
+    */
+    case SQLITE_DBSTATUS_TEMPBUF_SPILL: {
+      u64 nRet = 0;
+      if( db->aDb[1].pBt ){
+        Pager *pPager = sqlite3BtreePager(db->aDb[1].pBt);
+        sqlite3PagerCacheStat(pPager, SQLITE_DBSTATUS_CACHE_WRITE,
+                              resetFlag, &nRet);
+        nRet *= sqlite3BtreeGetPageSize(db->aDb[1].pBt);
+      }
+      nRet += db->nSpill;
+      if( resetFlag ) db->nSpill = 0;
+      *pHighwtr = 0;
+      *pCurrent = nRet;
+      break;
+    }
+
     /* Set *pCurrent to non-zero if there are unresolved deferred foreign
     ** key constraints.  Set *pCurrent to zero if all foreign key constraints
-    ** have been satisfied.  The *pHighwater is always set to zero.
+    ** have been satisfied.  The *pHighwtr is always set to zero.
     */
     case SQLITE_DBSTATUS_DEFERRED_FKS: {
-      *pHighwater = 0;  /* IMP: R-11967-56545 */
+      *pHighwtr = 0;  /* IMP: R-11967-56545 */
       *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0;
       break;
     }
@@ -396,3 +419,28 @@ int sqlite3_db_status(
   sqlite3_mutex_leave(db->mutex);
   return rc;
 }
+
+/*
+** 32-bit variant of sqlite3_db_status64()
+*/
+int sqlite3_db_status(
+  sqlite3 *db,             /* The database connection whose status is desired */
+  int op,                  /* Status verb */
+  int *pCurrent,           /* Write current value here */
+  int *pHighwtr,           /* Write high-water mark here */
+  int resetFlag            /* Reset high-water mark if true */
+){
+  sqlite3_int64 C = 0, H = 0;
+  int rc;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+  rc = sqlite3_db_status64(db, op, &C, &H, resetFlag);
+  if( rc==0 ){
+    *pCurrent = C & 0x7fffffff;
+    *pHighwtr = H & 0x7fffffff;
+  }
+  return rc;
+}
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 0adbb41697..32d6d6a120 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -35,14 +35,27 @@
 # include "msvc.h"
 #endif
 
+/****** Copy of tclsqlite.h ******/
 #if defined(INCLUDE_SQLITE_TCL_H)
-# include "sqlite_tcl.h"
+# include "sqlite_tcl.h"   /* Special case for Windows using STDCALL */
 #else
-# include "tcl.h"
+# include <tcl.h>          /* All normal cases */
 # ifndef SQLITE_TCLAPI
-#  define SQLITE_TCLAPI
+#   define SQLITE_TCLAPI
 # endif
 #endif
+/* Compatability between Tcl8.6 and Tcl9.0 */
+#if TCL_MAJOR_VERSION==9
+# define CONST const
+#elif !defined(Tcl_Size)
+  typedef int Tcl_Size;
+# ifndef Tcl_BounceRefCount
+#  define Tcl_BounceRefCount(X) Tcl_IncrRefCount(X); Tcl_DecrRefCount(X)
+   /* https://www.tcl-lang.org/man/tcl9.0/TclLib/Object.html */
+# endif
+#endif
+/**** End copy of tclsqlite.h ****/
+
 #include <errno.h>
 
 /*
@@ -55,6 +68,27 @@
 # include <string.h>
 # include <assert.h>
   typedef unsigned char u8;
+# ifndef SQLITE_PTRSIZE
+#   if defined(__SIZEOF_POINTER__)
+#     define SQLITE_PTRSIZE __SIZEOF_POINTER__
+#   elif defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
+         defined(_M_ARM)   || defined(__arm__)    || defined(__x86)   ||    \
+        (defined(__APPLE__) && defined(__POWERPC__)) ||                     \
+        (defined(__TOS_AIX__) && !defined(__64BIT__))
+#     define SQLITE_PTRSIZE 4
+#   else
+#     define SQLITE_PTRSIZE 8
+#   endif
+# endif /* SQLITE_PTRSIZE */
+# if defined(HAVE_STDINT_H) || (defined(__STDC_VERSION__) &&  \
+                                (__STDC_VERSION__ >= 199901L))
+#   include <stdint.h>
+    typedef uintptr_t uptr;
+# elif SQLITE_PTRSIZE==4
+    typedef unsigned int uptr;
+# else
+    typedef sqlite3_uint64 uptr;
+# endif
 #endif
 #include <ctype.h>
 
@@ -190,7 +224,8 @@ struct SqliteDb {
 struct IncrblobChannel {
   sqlite3_blob *pBlob;      /* sqlite3 blob handle */
   SqliteDb *pDb;            /* Associated database connection */
-  int iSeek;                /* Current seek offset */
+  sqlite3_int64 iSeek;      /* Current seek offset */
+  unsigned int isClosed;    /* TCL_CLOSE_READ or TCL_CLOSE_WRITE */
   Tcl_Channel channel;      /* Channel identifier */
   IncrblobChannel *pNext;   /* Linked list of all open incrblob channels */
   IncrblobChannel *pPrev;   /* Linked list of all open incrblob channels */
@@ -230,14 +265,23 @@ static void closeIncrblobChannels(SqliteDb *pDb){
 /*
 ** Close an incremental blob channel.
 */
-static int SQLITE_TCLAPI incrblobClose(
+static int SQLITE_TCLAPI incrblobClose2(
   ClientData instanceData,
-  Tcl_Interp *interp
+  Tcl_Interp *interp,
+  int flags
 ){
   IncrblobChannel *p = (IncrblobChannel *)instanceData;
-  int rc = sqlite3_blob_close(p->pBlob);
+  int  rc;
   sqlite3 *db = p->pDb->db;
 
+  if( flags ){
+    p->isClosed |= flags;
+    return TCL_OK;
+  }
+
+  /* If we reach this point, then we really do need to close the channel */
+  rc = sqlite3_blob_close(p->pBlob);
+
   /* Remove the channel from the SqliteDb.pIncrblob list. */
   if( p->pNext ){
     p->pNext->pPrev = p->pPrev;
@@ -258,6 +302,13 @@ static int SQLITE_TCLAPI incrblobClose(
   }
   return TCL_OK;
 }
+static int SQLITE_TCLAPI incrblobClose(
+  ClientData instanceData,
+  Tcl_Interp *interp
+){
+  return incrblobClose2(instanceData, interp, 0);
+}
+
 
 /*
 ** Read data from an incremental blob channel.
@@ -269,9 +320,9 @@ static int SQLITE_TCLAPI incrblobInput(
   int *errorCodePtr
 ){
   IncrblobChannel *p = (IncrblobChannel *)instanceData;
-  int nRead = bufSize;         /* Number of bytes to read */
-  int nBlob;                   /* Total size of the blob */
-  int rc;                      /* sqlite error code */
+  sqlite3_int64 nRead = bufSize;   /* Number of bytes to read */
+  sqlite3_int64 nBlob;             /* Total size of the blob */
+  int rc;                          /* sqlite error code */
 
   nBlob = sqlite3_blob_bytes(p->pBlob);
   if( (p->iSeek+nRead)>nBlob ){
@@ -281,7 +332,7 @@ static int SQLITE_TCLAPI incrblobInput(
     return 0;
   }
 
-  rc = sqlite3_blob_read(p->pBlob, (void *)buf, nRead, p->iSeek);
+  rc = sqlite3_blob_read(p->pBlob, (void *)buf, (int)nRead, (int)p->iSeek);
   if( rc!=SQLITE_OK ){
     *errorCodePtr = rc;
     return -1;
@@ -296,14 +347,14 @@ static int SQLITE_TCLAPI incrblobInput(
 */
 static int SQLITE_TCLAPI incrblobOutput(
   ClientData instanceData,
-  CONST char *buf,
+  const char *buf,
   int toWrite,
   int *errorCodePtr
 ){
   IncrblobChannel *p = (IncrblobChannel *)instanceData;
-  int nWrite = toWrite;        /* Number of bytes to write */
-  int nBlob;                   /* Total size of the blob */
-  int rc;                      /* sqlite error code */
+  sqlite3_int64 nWrite = toWrite;   /* Number of bytes to write */
+  sqlite3_int64 nBlob;              /* Total size of the blob */
+  int rc;                           /* sqlite error code */
 
   nBlob = sqlite3_blob_bytes(p->pBlob);
   if( (p->iSeek+nWrite)>nBlob ){
@@ -314,7 +365,7 @@ static int SQLITE_TCLAPI incrblobOutput(
     return 0;
   }
 
-  rc = sqlite3_blob_write(p->pBlob, (void *)buf, nWrite, p->iSeek);
+  rc = sqlite3_blob_write(p->pBlob, (void*)buf,(int)nWrite, (int)p->iSeek);
   if( rc!=SQLITE_OK ){
     *errorCodePtr = EIO;
     return -1;
@@ -324,12 +375,19 @@ static int SQLITE_TCLAPI incrblobOutput(
   return nWrite;
 }
 
+/* The datatype of Tcl_DriverWideSeekProc changes between tcl8.6 and tcl9.0 */
+#if TCL_MAJOR_VERSION==9
+# define WideSeekProcType long long
+#else
+# define WideSeekProcType Tcl_WideInt
+#endif
+
 /*
 ** Seek an incremental blob channel.
 */
-static int SQLITE_TCLAPI incrblobSeek(
+static WideSeekProcType SQLITE_TCLAPI incrblobWideSeek(
   ClientData instanceData,
-  long offset,
+  WideSeekProcType offset,
   int seekMode,
   int *errorCodePtr
 ){
@@ -351,6 +409,14 @@ static int SQLITE_TCLAPI incrblobSeek(
 
   return p->iSeek;
 }
+static int SQLITE_TCLAPI incrblobSeek(
+  ClientData instanceData,
+  long offset,
+  int seekMode,
+  int *errorCodePtr
+){
+  return incrblobWideSeek(instanceData,offset,seekMode,errorCodePtr);
+}
 
 
 static void SQLITE_TCLAPI incrblobWatch(
@@ -369,7 +435,7 @@ static int SQLITE_TCLAPI incrblobHandle(
 
 static Tcl_ChannelType IncrblobChannelType = {
   "incrblob",                        /* typeName                             */
-  TCL_CHANNEL_VERSION_2,             /* version                              */
+  TCL_CHANNEL_VERSION_5,             /* version                              */
   incrblobClose,                     /* closeProc                            */
   incrblobInput,                     /* inputProc                            */
   incrblobOutput,                    /* outputProc                           */
@@ -378,11 +444,11 @@ static Tcl_ChannelType IncrblobChannelType = {
   0,                                 /* getOptionProc                        */
   incrblobWatch,                     /* watchProc (this is a no-op)          */
   incrblobHandle,                    /* getHandleProc (always returns error) */
-  0,                                 /* close2Proc                           */
+  incrblobClose2,                    /* close2Proc                           */
   0,                                 /* blockModeProc                        */
   0,                                 /* flushProc                            */
   0,                                 /* handlerProc                          */
-  0,                                 /* wideSeekProc                         */
+  incrblobWideSeek,                  /* wideSeekProc                         */
 };
 
 /*
@@ -414,8 +480,9 @@ static int createIncrblobChannel(
   }
 
   p = (IncrblobChannel *)Tcl_Alloc(sizeof(IncrblobChannel));
-  p->iSeek = 0;
+  memset(p, 0, sizeof(*p));
   p->pBlob = pBlob;
+  if( (flags & TCL_WRITABLE)==0 ) p->isClosed |= TCL_CLOSE_WRITE;
 
   sqlite3_snprintf(sizeof(zChannel), zChannel, "incrblob_%d", ++count);
   p->channel = Tcl_CreateChannel(&IncrblobChannelType, zChannel, p, flags);
@@ -449,13 +516,13 @@ static int createIncrblobChannel(
 ** or {...} or ; to be seen anywhere.  Most callback scripts consist
 ** of just a single procedure name and they meet this requirement.
 */
-static int safeToUseEvalObjv(Tcl_Interp *interp, Tcl_Obj *pCmd){
+static int safeToUseEvalObjv(Tcl_Obj *pCmd){
   /* We could try to do something with Tcl_Parse().  But we will instead
   ** just do a search for forbidden characters.  If any of the forbidden
   ** characters appear in pCmd, we will report the string as unsafe.
   */
   const char *z;
-  int n;
+  Tcl_Size n;
   z = Tcl_GetStringFromObj(pCmd, &n);
   while( n-- > 0 ){
     int c = *(z++);
@@ -675,7 +742,7 @@ static int DbTraceV2Handler(
       pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
       Tcl_IncrRefCount(pCmd);
       Tcl_ListObjAppendElement(pDb->interp, pCmd,
-                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+                               Tcl_NewWideIntObj((Tcl_WideInt)(uptr)pStmt));
       Tcl_ListObjAppendElement(pDb->interp, pCmd,
                                Tcl_NewStringObj(zSql, -1));
       Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
@@ -690,7 +757,7 @@ static int DbTraceV2Handler(
       pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
       Tcl_IncrRefCount(pCmd);
       Tcl_ListObjAppendElement(pDb->interp, pCmd,
-                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+                               Tcl_NewWideIntObj((Tcl_WideInt)(uptr)pStmt));
       Tcl_ListObjAppendElement(pDb->interp, pCmd,
                                Tcl_NewWideIntObj((Tcl_WideInt)ns));
       Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
@@ -704,7 +771,7 @@ static int DbTraceV2Handler(
       pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
       Tcl_IncrRefCount(pCmd);
       Tcl_ListObjAppendElement(pDb->interp, pCmd,
-                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+                               Tcl_NewWideIntObj((Tcl_WideInt)(uptr)pStmt));
       Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
       Tcl_DecrRefCount(pCmd);
       Tcl_ResetResult(pDb->interp);
@@ -716,7 +783,7 @@ static int DbTraceV2Handler(
       pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
       Tcl_IncrRefCount(pCmd);
       Tcl_ListObjAppendElement(pDb->interp, pCmd,
-                               Tcl_NewWideIntObj((Tcl_WideInt)db));
+                               Tcl_NewWideIntObj((Tcl_WideInt)(uptr)db));
       Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
       Tcl_DecrRefCount(pCmd);
       Tcl_ResetResult(pDb->interp);
@@ -962,7 +1029,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
     ** be preserved and reused on the next invocation.
     */
     Tcl_Obj **aArg;
-    int nArg;
+    Tcl_Size nArg;
     if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){
       sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
       return;
@@ -1021,11 +1088,13 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
     Tcl_DecrRefCount(pCmd);
   }
 
-  if( rc && rc!=TCL_RETURN ){
+  if( TCL_BREAK==rc ){
+    sqlite3_result_null(context);
+  }else if( rc && rc!=TCL_RETURN ){
     sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
   }else{
     Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
-    int n;
+    Tcl_Size n;
     u8 *data;
     const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
     char c = zType[0];
@@ -1036,9 +1105,10 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
         /* Only return a BLOB type if the Tcl variable is a bytearray and
         ** has no string representation. */
         eType = SQLITE_BLOB;
-      }else if( (c=='b' && strcmp(zType,"boolean")==0)
+      }else if( (c=='b' && pVar->bytes==0 && strcmp(zType,"boolean")==0 )
+             || (c=='b' && pVar->bytes==0 && strcmp(zType,"booleanString")==0 )
              || (c=='w' && strcmp(zType,"wideInt")==0)
-             || (c=='i' && strcmp(zType,"int")==0) 
+             || (c=='i' && strcmp(zType,"int")==0)
       ){
         eType = SQLITE_INTEGER;
       }else if( c=='d' && strcmp(zType,"double")==0 ){
@@ -1072,7 +1142,8 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
       }
       default: {
         data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
-        sqlite3_result_text(context, (char *)data, n, SQLITE_TRANSIENT);
+        sqlite3_result_text64(context, (char *)data, n, SQLITE_TRANSIENT,
+                              SQLITE_UTF8);
         break;
       }
     }
@@ -1094,9 +1165,6 @@ static int auth_callback(
   const char *zArg2,
   const char *zArg3,
   const char *zArg4
-#ifdef SQLITE_USER_AUTHENTICATION
-  ,const char *zArg5
-#endif
 ){
   const char *zCode;
   Tcl_DString str;
@@ -1156,9 +1224,6 @@ static int auth_callback(
   Tcl_DStringAppendElement(&str, zArg2 ? zArg2 : "");
   Tcl_DStringAppendElement(&str, zArg3 ? zArg3 : "");
   Tcl_DStringAppendElement(&str, zArg4 ? zArg4 : "");
-#ifdef SQLITE_USER_AUTHENTICATION
-  Tcl_DStringAppendElement(&str, zArg5 ? zArg5 : "");
-#endif
   rc = Tcl_GlobalEval(pDb->interp, Tcl_DStringValue(&str));
   Tcl_DStringFree(&str);
   zReply = rc==TCL_OK ? Tcl_GetStringResult(pDb->interp) : "SQLITE_DENY";
@@ -1175,6 +1240,7 @@ static int auth_callback(
 }
 #endif /* SQLITE_OMIT_AUTHORIZATION */
 
+#if 0
 /*
 ** This routine reads a line of text from FILE in, stores
 ** the text in memory obtained from malloc() and returns a pointer
@@ -1219,6 +1285,7 @@ static char *local_getline(char *zPrompt, FILE *in){
   zLine = realloc( zLine, n+1 );
   return zLine;
 }
+#endif
 
 
 /*
@@ -1436,7 +1503,7 @@ static int dbPrepareAndBind(
         }
       }
       if( pVar ){
-        int n;
+        Tcl_Size n;
         u8 *data;
         const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
         c = zType[0];
@@ -1449,9 +1516,13 @@ static int dbPrepareAndBind(
           sqlite3_bind_blob(pStmt, i, data, n, SQLITE_STATIC);
           Tcl_IncrRefCount(pVar);
           pPreStmt->apParm[iParm++] = pVar;
-        }else if( c=='b' && strcmp(zType,"boolean")==0 ){
-          Tcl_GetIntFromObj(interp, pVar, &n);
-          sqlite3_bind_int(pStmt, i, n);
+        }else if( c=='b' && pVar->bytes==0 
+               && (strcmp(zType,"booleanString")==0
+                   || strcmp(zType,"boolean")==0)
+        ){
+          int nn;
+          Tcl_GetBooleanFromObj(interp, pVar, &nn);
+          sqlite3_bind_int(pStmt, i, nn);
         }else if( c=='d' && strcmp(zType,"double")==0 ){
           double r;
           Tcl_GetDoubleFromObj(interp, pVar, &r);
@@ -1463,7 +1534,8 @@ static int dbPrepareAndBind(
           sqlite3_bind_int64(pStmt, i, v);
         }else{
           data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
-          sqlite3_bind_text(pStmt, i, (char *)data, n, SQLITE_STATIC);
+          sqlite3_bind_text64(pStmt, i, (char *)data, n, SQLITE_STATIC,
+                              SQLITE_UTF8);
           Tcl_IncrRefCount(pVar);
           pPreStmt->apParm[iParm++] = pVar;
         }
@@ -1550,11 +1622,12 @@ struct DbEvalContext {
   SqlPreparedStmt *pPreStmt;      /* Current statement */
   int nCol;                       /* Number of columns returned by pStmt */
   int evalFlags;                  /* Flags used */
-  Tcl_Obj *pArray;                /* Name of array variable */
+  Tcl_Obj *pVarName;              /* Name of target array/dict variable */
   Tcl_Obj **apColName;            /* Array of column names */
 };
 
 #define SQLITE_EVAL_WITHOUTNULLS  0x00001  /* Unset array(*) for NULL */
+#define SQLITE_EVAL_ASDICT        0x00002  /* Use dict instead of array */
 
 /*
 ** Release any cache of column names currently held as part of
@@ -1575,20 +1648,20 @@ static void dbReleaseColumnNames(DbEvalContext *p){
 /*
 ** Initialize a DbEvalContext structure.
 **
-** If pArray is not NULL, then it contains the name of a Tcl array
+** If pVarName is not NULL, then it contains the name of a Tcl array
 ** variable. The "*" member of this array is set to a list containing
 ** the names of the columns returned by the statement as part of each
 ** call to dbEvalStep(), in order from left to right. e.g. if the names
 ** of the returned columns are a, b and c, it does the equivalent of the
 ** tcl command:
 **
-**     set ${pArray}(*) {a b c}
+**     set ${pVarName}(*) {a b c}
 */
 static void dbEvalInit(
   DbEvalContext *p,               /* Pointer to structure to initialize */
   SqliteDb *pDb,                  /* Database handle */
   Tcl_Obj *pSql,                  /* Object containing SQL script */
-  Tcl_Obj *pArray,                /* Name of Tcl array to set (*) element of */
+  Tcl_Obj *pVarName,              /* Name of Tcl array to set (*) element of */
   int evalFlags                   /* Flags controlling evaluation */
 ){
   memset(p, 0, sizeof(DbEvalContext));
@@ -1596,9 +1669,9 @@ static void dbEvalInit(
   p->zSql = Tcl_GetString(pSql);
   p->pSql = pSql;
   Tcl_IncrRefCount(pSql);
-  if( pArray ){
-    p->pArray = pArray;
-    Tcl_IncrRefCount(pArray);
+  if( pVarName ){
+    p->pVarName = pVarName;
+    Tcl_IncrRefCount(pVarName);
   }
   p->evalFlags = evalFlags;
   addDatabaseRef(p->pDb);
@@ -1621,7 +1694,7 @@ static void dbEvalRowInfo(
     Tcl_Obj **apColName = 0;      /* Array of column names */
 
     p->nCol = nCol = sqlite3_column_count(pStmt);
-    if( nCol>0 && (papColName || p->pArray) ){
+    if( nCol>0 && (papColName || p->pVarName) ){
       apColName = (Tcl_Obj**)Tcl_Alloc( sizeof(Tcl_Obj*)*nCol );
       for(i=0; i<nCol; i++){
         apColName[i] = Tcl_NewStringObj(sqlite3_column_name(pStmt,i), -1);
@@ -1630,20 +1703,35 @@ static void dbEvalRowInfo(
       p->apColName = apColName;
     }
 
-    /* If results are being stored in an array variable, then create
-    ** the array(*) entry for that array
+    /* If results are being stored in a variable then create the
+    ** array(*) or dict(*) entry for that variable.
     */
-    if( p->pArray ){
+    if( p->pVarName ){
       Tcl_Interp *interp = p->pDb->interp;
       Tcl_Obj *pColList = Tcl_NewObj();
       Tcl_Obj *pStar = Tcl_NewStringObj("*", -1);
 
+      Tcl_IncrRefCount(pColList);
+      Tcl_IncrRefCount(pStar);
       for(i=0; i<nCol; i++){
         Tcl_ListObjAppendElement(interp, pColList, apColName[i]);
       }
-      Tcl_IncrRefCount(pStar);
-      Tcl_ObjSetVar2(interp, p->pArray, pStar, pColList, 0);
+      if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){
+        Tcl_ObjSetVar2(interp, p->pVarName, pStar, pColList, 0);
+      }else{
+        Tcl_Obj * pDict = Tcl_ObjGetVar2(interp, p->pVarName, NULL, 0);
+        if( !pDict ){
+          pDict = Tcl_NewDictObj();
+        }else if( Tcl_IsShared(pDict) ){
+          pDict = Tcl_DuplicateObj(pDict);
+        }
+        if( Tcl_DictObjPut(interp, pDict, pStar, pColList)==TCL_OK ){
+          Tcl_ObjSetVar2(interp, p->pVarName, NULL, pDict, 0);
+        }
+        Tcl_BounceRefCount(pDict);
+      }
       Tcl_DecrRefCount(pStar);
+      Tcl_DecrRefCount(pColList);
     }
   }
 
@@ -1685,7 +1773,7 @@ static int dbEvalStep(DbEvalContext *p){
       if( rcs==SQLITE_ROW ){
         return TCL_OK;
       }
-      if( p->pArray ){
+      if( p->pVarName ){
         dbEvalRowInfo(p, 0, 0);
       }
       rcs = sqlite3_reset(pStmt);
@@ -1736,9 +1824,9 @@ static void dbEvalFinalize(DbEvalContext *p){
     dbReleaseStmt(p->pDb, p->pPreStmt, 0);
     p->pPreStmt = 0;
   }
-  if( p->pArray ){
-    Tcl_DecrRefCount(p->pArray);
-    p->pArray = 0;
+  if( p->pVarName ){
+    Tcl_DecrRefCount(p->pVarName);
+    p->pVarName = 0;
   }
   Tcl_DecrRefCount(p->pSql);
   dbReleaseColumnNames(p);
@@ -1780,12 +1868,13 @@ static Tcl_Obj *dbEvalColumnValue(DbEvalContext *p, int iCol){
 
 /*
 ** If using Tcl version 8.6 or greater, use the NR functions to avoid
-** recursive evalution of scripts by the [db eval] and [db trans]
+** recursive evaluation of scripts by the [db eval] and [db trans]
 ** commands. Even if the headers used while compiling the extension
 ** are 8.6 or newer, the code still tests the Tcl version at runtime.
 ** This allows stubs-enabled builds to be used with older Tcl libraries.
 */
-#if TCL_MAJOR_VERSION>8 || (TCL_MAJOR_VERSION==8 && TCL_MINOR_VERSION>=6)
+#if TCL_MAJOR_VERSION>8 || !defined(TCL_MINOR_VERSION) \
+                        || TCL_MINOR_VERSION>=6
 # define SQLITE_TCL_NRE 1
 static int DbUseNre(void){
   int major, minor;
@@ -1812,7 +1901,7 @@ static int DbUseNre(void){
 /*
 ** This function is part of the implementation of the command:
 **
-**   $db eval SQL ?ARRAYNAME? SCRIPT
+**   $db eval SQL ?TGT-NAME? SCRIPT
 */
 static int SQLITE_TCLAPI DbEvalNextCmd(
   ClientData data[],                   /* data[0] is the (DbEvalContext*) */
@@ -1826,8 +1915,8 @@ static int SQLITE_TCLAPI DbEvalNextCmd(
   ** is a pointer to a Tcl_Obj containing the script to run for each row
   ** returned by the queries encapsulated in data[0]. */
   DbEvalContext *p = (DbEvalContext *)data[0];
-  Tcl_Obj *pScript = (Tcl_Obj *)data[1];
-  Tcl_Obj *pArray = p->pArray;
+  Tcl_Obj * const pScript = (Tcl_Obj *)data[1];
+  Tcl_Obj * const pVarName = p->pVarName;
 
   while( (rc==TCL_OK || rc==TCL_CONTINUE) && TCL_OK==(rc = dbEvalStep(p)) ){
     int i;
@@ -1835,15 +1924,46 @@ static int SQLITE_TCLAPI DbEvalNextCmd(
     Tcl_Obj **apColName;
     dbEvalRowInfo(p, &nCol, &apColName);
     for(i=0; i<nCol; i++){
-      if( pArray==0 ){
+      if( pVarName==0 ){
         Tcl_ObjSetVar2(interp, apColName[i], 0, dbEvalColumnValue(p,i), 0);
       }else if( (p->evalFlags & SQLITE_EVAL_WITHOUTNULLS)!=0
-             && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL 
+             && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL
       ){
-        Tcl_UnsetVar2(interp, Tcl_GetString(pArray), 
-                      Tcl_GetString(apColName[i]), 0);
+        /* Remove NULL-containing column from the target container... */
+        if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){
+          /* Target is an array */
+          Tcl_UnsetVar2(interp, Tcl_GetString(pVarName),
+                        Tcl_GetString(apColName[i]), 0);
+        }else{
+          /* Target is a dict */
+          Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0);
+          if( pDict ){
+            if( Tcl_IsShared(pDict) ){
+              pDict = Tcl_DuplicateObj(pDict);
+            }
+            if( Tcl_DictObjRemove(interp, pDict, apColName[i])==TCL_OK ){
+              Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0);
+            }
+            Tcl_BounceRefCount(pDict);
+          }
+        }
+      }else if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){
+        /* Target is an array: set target(colName) = colValue */
+        Tcl_ObjSetVar2(interp, pVarName, apColName[i],
+                       dbEvalColumnValue(p,i), 0);
       }else{
-        Tcl_ObjSetVar2(interp, pArray, apColName[i], dbEvalColumnValue(p,i), 0);
+        /* Target is a dict: set target(colName) = colValue */
+        Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0);
+        if( !pDict ){
+          pDict = Tcl_NewDictObj();
+        }else if( Tcl_IsShared(pDict) ){
+          pDict = Tcl_DuplicateObj(pDict);
+        }
+        if( Tcl_DictObjPut(interp, pDict, apColName[i],
+                           dbEvalColumnValue(p,i))==TCL_OK ){
+          Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0);
+        }
+        Tcl_BounceRefCount(pDict);
       }
     }
 
@@ -1901,7 +2021,7 @@ static void DbHookCmd(
   }
   if( pArg ){
     assert( !(*ppHook) );
-    if( Tcl_GetCharLength(pArg)>0 ){
+    if( Tcl_GetString(pArg)[0] ){
       *ppHook = pArg;
       Tcl_IncrRefCount(*ppHook);
     }
@@ -1952,7 +2072,7 @@ static int SQLITE_TCLAPI DbObjCmd(
     "timeout",                "total_changes",         "trace",
     "trace_v2",               "transaction",           "unlock_notify",
     "update_hook",            "version",               "wal_hook",
-    0                        
+    0
   };
   enum DB_enum {
     DB_AUTHORIZER,            DB_BACKUP,               DB_BIND_FALLBACK,
@@ -1994,7 +2114,7 @@ static int SQLITE_TCLAPI DbObjCmd(
   **   (4) Name of the database (ex: "main", "temp")
   **   (5) Name of trigger that is doing the access
   **
-  ** The callback should return on of the following strings: SQLITE_OK,
+  ** The callback should return one of the following strings: SQLITE_OK,
   ** SQLITE_IGNORE, or SQLITE_DENY.  Any other return value is an error.
   **
   ** If this method is invoked with no arguments, the current authorization
@@ -2015,7 +2135,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       char *zAuth;
-      int len;
+      Tcl_Size len;
       if( pDb->zAuth ){
         Tcl_Free(pDb->zAuth);
       }
@@ -2118,7 +2238,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       char *zCallback;
-      int len;
+      Tcl_Size len;
       if( pDb->zBindFallback ){
         Tcl_Free(pDb->zBindFallback);
       }
@@ -2148,7 +2268,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       char *zBusy;
-      int len;
+      Tcl_Size len;
       if( pDb->zBusy ){
         Tcl_Free(pDb->zBusy);
       }
@@ -2255,7 +2375,7 @@ static int SQLITE_TCLAPI DbObjCmd(
     SqlCollate *pCollate;
     char *zName;
     char *zScript;
-    int nScript;
+    Tcl_Size nScript;
     if( objc!=4 ){
       Tcl_WrongNumArgs(interp, 2, objv, "NAME SCRIPT");
       return TCL_ERROR;
@@ -2314,7 +2434,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       const char *zCommit;
-      int len;
+      Tcl_Size len;
       if( pDb->zCommit ){
         Tcl_Free(pDb->zCommit);
       }
@@ -2441,7 +2561,7 @@ static int SQLITE_TCLAPI DbObjCmd(
   **
   ** This command usage is equivalent to the sqlite2.x COPY statement,
   ** which imports file data into a table using the PostgreSQL COPY file format:
-  **   $db copy $conflit_algo $table_name $filename \t \\N
+  **   $db copy $conflict_algorithm $table_name $filename \t \\N
   */
   case DB_COPY: {
     char *zTable;               /* Insert data into this table */
@@ -2457,9 +2577,10 @@ static int SQLITE_TCLAPI DbObjCmd(
     char *zLine;                /* A single line of input from the file */
     char **azCol;               /* zLine[] broken up into columns */
     const char *zCommit;        /* How to commit changes */
-    FILE *in;                   /* The input file */
+    Tcl_Channel in;             /* The input file */
     int lineno = 0;             /* Line number of input file */
     char zLineNum[80];          /* Line number print buffer */
+    Tcl_Obj *str;
     Tcl_Obj *pResult;           /* interp result */
 
     const char *zSep;
@@ -2538,23 +2659,27 @@ static int SQLITE_TCLAPI DbObjCmd(
       sqlite3_finalize(pStmt);
       return TCL_ERROR;
     }
-    in = fopen(zFile, "rb");
+    in = Tcl_OpenFileChannel(interp, zFile, "rb", 0666);
     if( in==0 ){
-      Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, (char*)0);
       sqlite3_finalize(pStmt);
       return TCL_ERROR;
     }
+    Tcl_SetChannelOption(NULL, in, "-translation", "auto");
     azCol = malloc( sizeof(azCol[0])*(nCol+1) );
     if( azCol==0 ) {
       Tcl_AppendResult(interp, "Error: can't malloc()", (char*)0);
-      fclose(in);
+      Tcl_Close(interp, in);
       return TCL_ERROR;
     }
+    str = Tcl_NewObj();
+    Tcl_IncrRefCount(str);
     (void)sqlite3_exec(pDb->db, "BEGIN", 0, 0, 0);
     zCommit = "COMMIT";
-    while( (zLine = local_getline(0, in))!=0 ){
+    while( Tcl_GetsObj(in, str)>=0 ) {
       char *z;
+      Tcl_Size byteLen;
       lineno++;
+      zLine = (char *)Tcl_GetByteArrayFromObj(str, &byteLen);
       azCol[0] = zLine;
       for(i=0, z=zLine; *z; z++){
         if( *z==zSep[0] && strncmp(z, zSep, nSep)==0 ){
@@ -2592,15 +2717,16 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
       sqlite3_step(pStmt);
       rc = sqlite3_reset(pStmt);
-      free(zLine);
+      Tcl_SetObjLength(str, 0);
       if( rc!=SQLITE_OK ){
         Tcl_AppendResult(interp,"Error: ", sqlite3_errmsg(pDb->db), (char*)0);
         zCommit = "ROLLBACK";
         break;
       }
     }
+    Tcl_DecrRefCount(str);
     free(azCol);
-    fclose(in);
+    Tcl_Close(interp, in);
     sqlite3_finalize(pStmt);
     (void)sqlite3_exec(pDb->db, zCommit, 0, 0, 0);
 
@@ -2634,7 +2760,8 @@ static int SQLITE_TCLAPI DbObjCmd(
     Tcl_Obj *pValue = 0;
     unsigned char *pBA;
     unsigned char *pData;
-    int len, xrc;
+    Tcl_Size len;
+    int xrc;
     sqlite3_int64 mxSize = 0;
     int i;
     int isReadonly = 0;
@@ -2779,13 +2906,15 @@ static int SQLITE_TCLAPI DbObjCmd(
   }
 
   /*
-  **    $db eval ?options? $sql ?array? ?{  ...code... }?
+  **    $db eval ?options? $sql ?varName? ?{  ...code... }?
   **
-  ** The SQL statement in $sql is evaluated.  For each row, the values are
-  ** placed in elements of the array named "array" and ...code... is executed.
-  ** If "array" and "code" are omitted, then no callback is every invoked.
-  ** If "array" is an empty string, then the values are placed in variables
-  ** that have the same name as the fields extracted by the query.
+  ** The SQL statement in $sql is evaluated.  For each row, the values
+  ** are placed in elements of the array or dict named $varName and
+  ** ...code... is executed.  If $varName and $code are omitted, then
+  ** no callback is ever invoked.  If $varName is an empty string,
+  ** then the values are placed in variables that have the same name
+  ** as the fields extracted by the query, and those variables are
+  ** accessible during the eval of $code.
   */
   case DB_EVAL: {
     int evalFlags = 0;
@@ -2793,8 +2922,9 @@ static int SQLITE_TCLAPI DbObjCmd(
     while( objc>3 && (zOpt = Tcl_GetString(objv[2]))!=0 && zOpt[0]=='-' ){
       if( strcmp(zOpt, "-withoutnulls")==0 ){
         evalFlags |= SQLITE_EVAL_WITHOUTNULLS;
-      }
-      else{
+      }else if( strcmp(zOpt, "-asdict")==0 ){
+        evalFlags |= SQLITE_EVAL_ASDICT;
+      }else{
         Tcl_AppendResult(interp, "unknown option: \"", zOpt, "\"", (void*)0);
         return TCL_ERROR;
       }
@@ -2802,8 +2932,8 @@ static int SQLITE_TCLAPI DbObjCmd(
       objv++;
     }
     if( objc<3 || objc>5 ){
-      Tcl_WrongNumArgs(interp, 2, objv, 
-          "?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?");
+      Tcl_WrongNumArgs(interp, 2, objv,
+          "?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?");
       return TCL_ERROR;
     }
 
@@ -2829,17 +2959,17 @@ static int SQLITE_TCLAPI DbObjCmd(
     }else{
       ClientData cd2[2];
       DbEvalContext *p;
-      Tcl_Obj *pArray = 0;
+      Tcl_Obj *pVarName = 0;
       Tcl_Obj *pScript;
 
       if( objc>=5 && *(char *)Tcl_GetString(objv[3]) ){
-        pArray = objv[3];
+        pVarName = objv[3];
       }
       pScript = objv[objc-1];
       Tcl_IncrRefCount(pScript);
 
       p = (DbEvalContext *)Tcl_Alloc(sizeof(DbEvalContext));
-      dbEvalInit(p, pDb, objv[2], pArray, evalFlags);
+      dbEvalInit(p, pDb, objv[2], pVarName, evalFlags);
 
       cd2[0] = (void *)p;
       cd2[1] = (void *)pScript;
@@ -2929,7 +3059,7 @@ static int SQLITE_TCLAPI DbObjCmd(
     }
     pFunc->pScript = pScript;
     Tcl_IncrRefCount(pScript);
-    pFunc->useEvalObjv = safeToUseEvalObjv(interp, pScript);
+    pFunc->useEvalObjv = safeToUseEvalObjv(pScript);
     pFunc->eType = eType;
     rc = sqlite3_create_function(pDb->db, zName, nArg, flags,
         pFunc, tclSqlFunc, 0, 0);
@@ -3005,7 +3135,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       return TCL_ERROR;
     }
     if( objc==3 ){
-      int len;
+      Tcl_Size len;
       char *zNull = Tcl_GetStringFromObj(objv[2], &len);
       if( pDb->zNull ){
         Tcl_Free(pDb->zNull);
@@ -3059,7 +3189,7 @@ static int SQLITE_TCLAPI DbObjCmd(
 #endif
     }else if( objc==4 ){
       char *zProgress;
-      int len;
+      Tcl_Size len;
       int N;
       if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){
         return TCL_ERROR;
@@ -3105,7 +3235,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       char *zProfile;
-      int len;
+      Tcl_Size len;
       if( pDb->zProfile ){
         Tcl_Free(pDb->zProfile);
       }
@@ -3332,7 +3462,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       char *zTrace;
-      int len;
+      Tcl_Size len;
       if( pDb->zTrace ){
         Tcl_Free(pDb->zTrace);
       }
@@ -3372,7 +3502,7 @@ static int SQLITE_TCLAPI DbObjCmd(
       }
     }else{
       char *zTraceV2;
-      int len;
+      Tcl_Size len;
       Tcl_WideInt wMask = 0;
       if( objc==4 ){
         static const char *TTYPE_strs[] = {
@@ -3381,7 +3511,7 @@ static int SQLITE_TCLAPI DbObjCmd(
         enum TTYPE_enum {
           TTYPE_STMT, TTYPE_PROFILE, TTYPE_ROW, TTYPE_CLOSE
         };
-        int i;
+        Tcl_Size i;
         if( TCL_OK!=Tcl_ListObjLength(interp, objv[3], &len) ){
           return TCL_ERROR;
         }
@@ -3443,7 +3573,7 @@ static int SQLITE_TCLAPI DbObjCmd(
   ** Start a new transaction (if we are not already in the midst of a
   ** transaction) and execute the TCL script SCRIPT.  After SCRIPT
   ** completes, either commit the transaction or roll it back if SCRIPT
-  ** throws an exception.  Or if no new transation was started, do nothing.
+  ** throws an exception.  Or if no new transaction was started, do nothing.
   ** pass the exception on up the stack.
   **
   ** This command was inspired by Dave Thomas's talk on Ruby at the
@@ -3962,7 +4092,7 @@ static int SQLITE_TCLAPI DbMain(
 ** The EXTERN macros are required by TCL in order to work on windows.
 */
 EXTERN int Sqlite3_Init(Tcl_Interp *interp){
-  int rc = Tcl_InitStubs(interp, "8.4", 0) ? TCL_OK : TCL_ERROR;
+  int rc = Tcl_InitStubs(interp, "8.5-", 0) ? TCL_OK : TCL_ERROR;
   if( rc==TCL_OK ){
     Tcl_CreateObjCommand(interp, "sqlite3", (Tcl_ObjCmdProc*)DbMain, 0, 0);
 #ifndef SQLITE_3_SUFFIX_ONLY
@@ -3986,14 +4116,27 @@ EXTERN int Tclsqlite3_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
 EXTERN int Sqlite3_SafeInit(Tcl_Interp *interp){ return TCL_ERROR; }
 EXTERN int Sqlite3_SafeUnload(Tcl_Interp *interp, int flags){return TCL_ERROR;}
 
+/*
+** Versions of all of the above entry points that omit the "3" at the end
+** of the name.  Years ago (circa 2004) the "3" was necessary to distinguish
+** SQLite version 3 from Sqlite version 2.  But two decades have elapsed.
+** SQLite2 is not longer a conflict.  So it is ok to omit the "3".
+**
+** Omitting the "3" helps TCL find the entry point.
+*/
+EXTERN int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);}
+EXTERN int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); }
+EXTERN int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
+EXTERN int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
+EXTERN int Sqlite_SafeInit(Tcl_Interp *interp){ return TCL_ERROR; }
+EXTERN int Sqlite_SafeUnload(Tcl_Interp *interp, int flags){return TCL_ERROR;}
 
+/* Also variants with a lowercase "s".  I'm told that these are
+** deprecated in Tcl9, but they continue to be included for backwards
+** compatibility. */
+EXTERN int sqlite3_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);}
+EXTERN int sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);}
 
-#ifndef SQLITE_3_SUFFIX_ONLY
-int Sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); }
-int Tclsqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp); }
-int Sqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
-int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
-#endif
 
 /*
 ** If the TCLSH macro is defined, add code to make a stand-alone program.
@@ -4001,12 +4144,29 @@ int Tclsqlite_Unload(Tcl_Interp *interp, int flags){ return TCL_OK; }
 #if defined(TCLSH)
 
 /* This is the main routine for an ordinary TCL shell.  If there are
-** are arguments, run the first argument as a script.  Otherwise,
-** read TCL commands from standard input
+** arguments, run the first argument as a script.  Otherwise, read TCL
+** commands from standard input
 */
 static const char *tclsh_main_loop(void){
   static const char zMainloop[] =
     "if {[llength $argv]>=1} {\n"
+#ifdef WIN32
+      "set new [list]\n"
+      "foreach arg $argv {\n"
+        "if {[string match -* $arg] || [file exists $arg]} {\n"
+          "lappend new $arg\n"
+        "} else {\n"
+          "set once 0\n"
+          "foreach match [lsort [glob -nocomplain $arg]] {\n"
+            "lappend new $match\n"
+            "set once 1\n"
+          "}\n"
+          "if {!$once} {lappend new $arg}\n"
+        "}\n"
+      "}\n"
+      "set argv $new\n"
+      "unset new\n"
+#endif
       "set argv0 [lindex $argv 0]\n"
       "set argv [lrange $argv 1 end]\n"
       "source $argv0\n"
diff --git a/src/tclsqlite.h b/src/tclsqlite.h
new file mode 100644
index 0000000000..f71ec9a7c2
--- /dev/null
+++ b/src/tclsqlite.h
@@ -0,0 +1,42 @@
+/*
+** 2024-07-30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This header file defines the interface to TCL as used by SQLite.
+** SQLite subcomponents that use TCL (the libsqlite3.c interface library
+** and various test*.c pieces) should #include this file rather than
+** including tcl.h directly.
+*/
+/******  Any edits to this file must mirrored in tclsqlite.c ***********/
+
+/* When compiling for Windows using STDCALL instead of CDECL calling
+** conventions, the MSVC makefile has to build a customized version of
+** the "tcl.h" header that specifies the calling conventions for each
+** interface.  That customized "tcl.h" is named "sqlite_tcl.h".
+*/
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"   /* Special case for Windows using STDCALL */
+#else
+# include <tcl.h>          /* All normal cases */
+# ifndef SQLITE_TCLAPI
+#   define SQLITE_TCLAPI
+# endif
+#endif
+
+/******  Any edits to this file must mirrored in tclsqlite.c ***********/
+
+/* Compatibility between Tcl8.6 and Tcl9.0 */
+#if TCL_MAJOR_VERSION==9
+# define CONST const
+#elif !defined(Tcl_Size)
+# define Tcl_Size int
+#endif
+
+/******  Any edits to this file must mirrored in tclsqlite.c ***********/
diff --git a/src/test1.c b/src/test1.c
index 7a9ed9c9fb..41b3324050 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -26,11 +26,7 @@
 #endif
 
 #include "vdbeInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -106,7 +102,7 @@ static int SQLITE_TCLAPI get_sqlite_pointer(
   }
   p = (struct SqliteDb*)cmdInfo.objClientData;
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%p", p->db);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -157,7 +153,7 @@ int sqlite3TestErrCode(Tcl_Interp *interp, sqlite3 *db, int rc){
        "error code %s (%d) does not match sqlite3_errcode %s (%d)",
        t1ErrorName(rc), rc, t1ErrorName(r2), r2);
     Tcl_ResetResult(interp);
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
     return 1;
   }
   return 0;
@@ -508,7 +504,7 @@ static int SQLITE_TCLAPI test_mprintf_z(
   for(i=2; i<argc && (i==2 || zResult); i++){
     zResult = sqlite3_mprintf("%z%s%s", zResult, argv[1], argv[i]);
   }
-  Tcl_AppendResult(interp, zResult, 0);
+  Tcl_AppendResult(interp, zResult, NULL);
   sqlite3_free(zResult);
   return TCL_OK;
 }
@@ -556,7 +552,7 @@ static int SQLITE_TCLAPI test_snprintf_int(
   if( n>sizeof(zStr) ) n = sizeof(zStr);
   sqlite3_snprintf(sizeof(zStr), zStr, "abcdefghijklmnopqrstuvwxyz");
   sqlite3_snprintf(n, zStr, zFormat, a1);
-  Tcl_AppendResult(interp, zStr, 0);
+  Tcl_AppendResult(interp, zStr, NULL);
   return TCL_OK;
 }
 
@@ -604,6 +600,7 @@ static int SQLITE_TCLAPI test_get_table_printf(
   }
   sqlite3_free(zSql);
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", rc);
+  Tcl_ResetResult(interp);
   Tcl_AppendElement(interp, zBuf);
   if( rc==SQLITE_OK ){
     if( argc==4 ){
@@ -642,12 +639,12 @@ static int SQLITE_TCLAPI test_last_rowid(
   char zBuf[30];
 
   if( argc!=2 ){
-    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB\"", 0);
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB\"", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR;
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", sqlite3_last_insert_rowid(db));
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return SQLITE_OK;
 }
 
@@ -1019,6 +1016,103 @@ static void intrealFunction(
   sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, context);
 }
 
+/*
+** These SQL functions attempt to return a value (their first argument)
+** that has been modified to have multiple datatypes.  For example both
+** TEXT and INTEGER.
+*/
+static void addTextTypeFunction(
+  sqlite3_context *context, 
+  int argc,  
+  sqlite3_value **argv
+){
+  (void)sqlite3_value_text(argv[0]);
+  (void)argc;
+  sqlite3_result_value(context, argv[0]);
+}
+static void addIntTypeFunction(
+  sqlite3_context *context, 
+  int argc,  
+  sqlite3_value **argv
+){
+  (void)sqlite3_value_int64(argv[0]);
+  (void)argc;
+  sqlite3_result_value(context, argv[0]);
+}
+static void addRealTypeFunction(
+  sqlite3_context *context, 
+  int argc,  
+  sqlite3_value **argv
+){
+  (void)sqlite3_value_double(argv[0]);
+  (void)argc;
+  sqlite3_result_value(context, argv[0]);
+}
+
+/*
+** SQL function:  strtod(X)
+**
+** Use the C-library strtod() function to convert string X into a double.
+** Used for comparing the accuracy of SQLite's internal text-to-float conversion
+** routines against the C-library.
+*/
+static void shellStrtod(
+  sqlite3_context *pCtx,
+  int nVal,
+  sqlite3_value **apVal
+){
+  char *z = (char*)sqlite3_value_text(apVal[0]);
+  UNUSED_PARAMETER(nVal);
+  if( z==0 ) return;
+  sqlite3_result_double(pCtx, strtod(z,0));
+}
+
+/*
+** SQL function:  dtostr(X)
+**
+** Use the C-library printf() function to convert real value X into a string.
+** Used for comparing the accuracy of SQLite's internal float-to-text conversion
+** routines against the C-library.
+*/
+static void shellDtostr(
+  sqlite3_context *pCtx,
+  int nVal,
+  sqlite3_value **apVal
+){
+  double r = sqlite3_value_double(apVal[0]);
+  int n = nVal>=2 ? sqlite3_value_int(apVal[1]) : 26;
+  char z[400];
+  if( n<1 ) n = 1;
+  if( n>350 ) n = 350;
+  sprintf(z, "%#+.*e", n, r);
+  sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT);
+}
+
+/*
+** We need a method for setting the pointer values created by the
+** intarray_addr, int64array_addr, doublearray_addr, and textarray_addr
+** routines below.  The inttoptr(X) SQL function accomplishes
+** this.  Tcl scripts will bind an array address as an integer X and
+** the inttoptr() SQL function will use sqlite3_result_pointer() to
+** convert that integer into a pointer usable by carray().
+*/
+static void inttoptrFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  void *p;
+  sqlite3_int64 i64;
+  i64 = sqlite3_value_int64(argv[0]);
+  if( sizeof(i64)==sizeof(p) ){
+    memcpy(&p, &i64, sizeof(p));
+  }else{
+    int i32 = i64 & 0xffffffff;
+    memcpy(&p, &i32, sizeof(p));
+  }
+  sqlite3_result_pointer(context, p, "carray", 0);
+}
+
 /*
 ** Usage:  sqlite3_create_function DB
 **
@@ -1032,7 +1126,8 @@ static void intrealFunction(
 **
 ** The original motivation for this routine was to be able to call the
 ** sqlite3_create_function function while a query is in progress in order
-** to test the SQLITE_MISUSE detection logic.
+** to test the SQLITE_MISUSE detection logic.  It is now also used to register
+** a bunch of SQL functions that are useful for testing.
 */
 static int SQLITE_TCLAPI test_create_function(
   void *NotUsed,
@@ -1091,6 +1186,47 @@ static int SQLITE_TCLAPI test_create_function(
           0, intrealFunction, 0, 0);
   }
 
+  /* The add_text_type(), add_int_type(), and add_real_type() functions
+  ** attempt to return a value that has multiple datatypes.
+  */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "add_text_type", 1, SQLITE_UTF8,
+          0, addTextTypeFunction, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "add_int_type", 1, SQLITE_UTF8,
+          0, addIntTypeFunction, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "add_real_type", 1, SQLITE_UTF8,
+          0, addRealTypeFunction, 0, 0);
+  }
+
+  /* Functions strtod() and dtostr() work as in the shell.  These routines
+  ** use the standard C library to convert between floating point and
+  ** text.  This is used to compare SQLite's internal conversion routines
+  ** against the standard library conversion routines.
+  **
+  ** Both routines copy/pasted from the shell.c.in implementation
+  ** on 2023-07-03.
+  */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "strtod", 1, SQLITE_UTF8, 0,
+                                 shellStrtod, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "dtostr", 1, SQLITE_UTF8, 0,
+                                 shellDtostr, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "dtostr", 2, SQLITE_UTF8, 0,
+                                 shellDtostr, 0, 0);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0,
+                                 inttoptrFunc, 0, 0);
+  }
+
 #ifndef SQLITE_OMIT_UTF16
   /* Use the sqlite3_create_function16() API here. Mainly for fun, but also 
   ** because it is not tested anywhere else. */
@@ -1306,7 +1442,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_int(
     if( Tcl_GetInt(interp, argv[i], &a[i-2]) ) return TCL_ERROR;
   }
   z = sqlite3_mprintf(argv[1], a[0], a[1], a[2]);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1332,12 +1468,12 @@ static int SQLITE_TCLAPI sqlite3_mprintf_int64(
   }
   for(i=2; i<5; i++){
     if( sqlite3Atoi64(argv[i], &a[i-2], sqlite3Strlen30(argv[i]), SQLITE_UTF8) ){
-      Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", 0);
+      Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", NULL);
       return TCL_ERROR;
     }
   }
   z = sqlite3_mprintf(argv[1], a[0], a[1], a[2]);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1370,7 +1506,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_long(
     a[i-2] &= (((u64)1)<<(sizeof(int)*8))-1;
   }
   z = sqlite3_mprintf(argv[1], a[0], a[1], a[2]);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1397,7 +1533,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_str(
     if( Tcl_GetInt(interp, argv[i], &a[i-2]) ) return TCL_ERROR;
   }
   z = sqlite3_mprintf(argv[1], a[0], a[1], argc>4 ? argv[4] : NULL);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1423,7 +1559,7 @@ static int SQLITE_TCLAPI sqlite3_snprintf_str(
   }
   if( Tcl_GetInt(interp, argv[1], &n) ) return TCL_ERROR;
   if( n<0 ){
-    Tcl_AppendResult(interp, "N must be non-negative", 0);
+    Tcl_AppendResult(interp, "N must be non-negative", NULL);
     return TCL_ERROR;
   }
   for(i=3; i<5; i++){
@@ -1431,7 +1567,7 @@ static int SQLITE_TCLAPI sqlite3_snprintf_str(
   }
   z = sqlite3_malloc( n+1 );
   sqlite3_snprintf(n, z, argv[2], a[0], a[1], argc>4 ? argv[5] : NULL);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1460,7 +1596,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_double(
   }
   if( Tcl_GetDouble(interp, argv[4], &r) ) return TCL_ERROR;
   z = sqlite3_mprintf(argv[1], a[0], a[1], r);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1490,7 +1626,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_scaled(
     if( Tcl_GetDouble(interp, argv[i], &r[i-2]) ) return TCL_ERROR;
   }
   z = sqlite3_mprintf(argv[1], r[0]*r[1]);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1515,7 +1651,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_stronly(
     return TCL_ERROR;
   }
   z = sqlite3_mprintf(argv[1], argv[2]);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1542,14 +1678,14 @@ static int SQLITE_TCLAPI sqlite3_mprintf_hexdouble(
     return TCL_ERROR;
   }
   if( sscanf(argv[2], "%08x%08x", &x2, &x1)!=2 ){
-    Tcl_AppendResult(interp, "2nd argument should be 16-characters of hex", 0);
+    Tcl_AppendResult(interp, "2nd argument should be 16-characters of hex", NULL);
     return TCL_ERROR;
   }
   d = x2;
   d = (d<<32) + x1;
   memcpy(&r, &d, sizeof(r));
   z = sqlite3_mprintf(argv[1], r);
-  Tcl_AppendResult(interp, z, 0);
+  Tcl_AppendResult(interp, z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 }
@@ -1667,7 +1803,7 @@ static int SQLITE_TCLAPI test_table_column_metadata(
       &zDatatype, &zCollseq, &notnull, &primarykey, &autoincrement);
 
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
@@ -1690,7 +1826,7 @@ static int SQLITE_TCLAPI blobHandleFromObj(
   sqlite3_blob **ppBlob
 ){
   char *z;
-  int n;
+  Tcl_Size n;
 
   z = Tcl_GetStringFromObj(pObj, &n);
   if( n==0 ){
@@ -1944,7 +2080,7 @@ static int SQLITE_TCLAPI test_create_function_v2(
   );
   if( rc!=SQLITE_OK ){
     Tcl_ResetResult(interp);
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -2133,7 +2269,7 @@ static int SQLITE_TCLAPI test_register_func(
   rc = sqlite3_create_function(db, argv[2], -1, SQLITE_UTF8, 0, 
       testFunc, 0, 0);
   if( rc!=0 ){
-    Tcl_AppendResult(interp, sqlite3ErrStr(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrStr(rc), NULL);
     return TCL_ERROR;
   }
   if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR;
@@ -2244,6 +2380,7 @@ static int SQLITE_TCLAPI test_stmt_scanstatus(
   int flags = 0;
   int iSelectId = 0;
   int iParentId = 0;
+  int bDebug = 0;
 
   if( objc==5 ){
     struct Flag {
@@ -2251,23 +2388,28 @@ static int SQLITE_TCLAPI test_stmt_scanstatus(
       int flag;
     } aTbl[] = {
       {"complex", SQLITE_SCANSTAT_COMPLEX},
+      {"debug", -1},
       {0, 0}
     };
 
     Tcl_Obj **aFlag = 0;
-    int nFlag = 0;
+    Tcl_Size nFlag = 0;
     int ii;
 
     if( Tcl_ListObjGetElements(interp, objv[2], &nFlag, &aFlag) ){
       return TCL_ERROR;
     }
-    for(ii=0; ii<nFlag; ii++){
+    for(ii=0; ii<(int)nFlag; ii++){
       int iVal = 0;
-      int res = Tcl_GetIndexFromObjStruct(
+      res = Tcl_GetIndexFromObjStruct(
           interp, aFlag[ii], aTbl, sizeof(aTbl[0]), "flag", 0, &iVal
       );
       if( res ) return TCL_ERROR;
-      flags |= aTbl[iVal].flag;
+      if( aTbl[iVal].flag==-1 ){
+        bDebug = 1;
+      }else{
+        flags |= aTbl[iVal].flag;
+      }
     }
   }
 
@@ -2281,6 +2423,13 @@ static int SQLITE_TCLAPI test_stmt_scanstatus(
     return TCL_ERROR;
   }
 
+  if( bDebug && 0==(flags & SQLITE_SCANSTAT_COMPLEX) ){
+    Tcl_SetObjResult(interp, 
+        Tcl_NewStringObj("may not specify debug without complex", -1)
+    );
+    return TCL_ERROR;
+  }
+
   if( idx<0 ){
     Tcl_Obj *pRet = Tcl_NewObj();
     res = sqlite3_stmt_scanstatus_v2(
@@ -2327,6 +2476,36 @@ static int SQLITE_TCLAPI test_stmt_scanstatus(
           pStmt, idx, SQLITE_SCANSTAT_NCYCLE, flags, (void*)&nCycle);
       Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nCycle", -1));
       Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nCycle));
+
+      if( bDebug ){
+        int ii;
+        ScanStatus *pScan = &((Vdbe*)pStmt)->aScan[idx];
+        Tcl_Obj *pRange = Tcl_NewObj();
+        Tcl_Obj *pCsr = Tcl_NewObj();
+
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("debug_loop", -1));
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(pScan->addrLoop));
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("debug_visit", -1));
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(pScan->addrVisit));
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("debug_explain",-1));
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(pScan->addrExplain));
+        for(ii=0; ii<ArraySize(pScan->aAddrRange)/2; ii++){
+          int iStart = pScan->aAddrRange[ii*2];
+          int iEnd = pScan->aAddrRange[ii*2+1];
+          if( iStart>0 ){
+            Tcl_ListObjAppendElement(0, pRange, Tcl_NewIntObj(iStart));
+            Tcl_ListObjAppendElement(0, pRange, Tcl_NewIntObj(iEnd));
+          }else if( iStart<0 ){
+            Tcl_ListObjAppendElement(0, pCsr, Tcl_NewIntObj(iEnd));
+          }
+        }
+
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("debug_range", -1));
+        Tcl_ListObjAppendElement(0, pRet, pRange);
+        Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("debug_csr", -1));
+        Tcl_ListObjAppendElement(0, pRet, pCsr);
+      }
+
       Tcl_SetObjResult(interp, pRet);
     }else{
       Tcl_ResetResult(interp);
@@ -2641,7 +2820,7 @@ static int SQLITE_TCLAPI test_snapshot_open_blob(
   sqlite3 *db;
   char *zName;
   unsigned char *pBlob;
-  int nBlob;
+  Tcl_Size nBlob;
 
   if( objc!=4 ){
     Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME SNAPSHOT");
@@ -2651,7 +2830,7 @@ static int SQLITE_TCLAPI test_snapshot_open_blob(
   zName = Tcl_GetString(objv[2]);
   pBlob = Tcl_GetByteArrayFromObj(objv[3], &nBlob);
   if( nBlob!=sizeof(sqlite3_snapshot) ){
-    Tcl_AppendResult(interp, "bad SNAPSHOT", 0);
+    Tcl_AppendResult(interp, "bad SNAPSHOT", NULL);
     return TCL_ERROR;
   }
   rc = sqlite3_snapshot_open(db, zName, (sqlite3_snapshot*)pBlob);
@@ -2676,8 +2855,8 @@ static int SQLITE_TCLAPI test_snapshot_cmp_blob(
   int res;
   unsigned char *p1;
   unsigned char *p2;
-  int n1;
-  int n2;
+  Tcl_Size n1;
+  Tcl_Size n2;
 
   if( objc!=3 ){
     Tcl_WrongNumArgs(interp, 1, objv, "SNAPSHOT1 SNAPSHOT2");
@@ -2688,7 +2867,7 @@ static int SQLITE_TCLAPI test_snapshot_cmp_blob(
   p2 = Tcl_GetByteArrayFromObj(objv[2], &n2);
 
   if( n1!=sizeof(sqlite3_snapshot) || n1!=n2 ){
-    Tcl_AppendResult(interp, "bad SNAPSHOT", 0);
+    Tcl_AppendResult(interp, "bad SNAPSHOT", NULL);
     return TCL_ERROR;
   }
 
@@ -2745,7 +2924,7 @@ static int SQLITE_TCLAPI test_atomic_batch_write(
 
   rc = sqlite3_open(zFile, &db);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, sqlite3_errmsg(db), NULL);
     sqlite3_close(db);
     return TCL_ERROR;
   }
@@ -2764,7 +2943,7 @@ static int SQLITE_TCLAPI test_atomic_batch_write(
 /*
 ** Usage:  sqlite3_next_stmt  DB  STMT
 **
-** Return the next statment in sequence after STMT.
+** Return the next statement in sequence after STMT.
 */
 static int SQLITE_TCLAPI test_next_stmt(
   void * clientData,
@@ -2787,7 +2966,7 @@ static int SQLITE_TCLAPI test_next_stmt(
   pStmt = sqlite3_next_stmt(db, pStmt);
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
@@ -2846,6 +3025,34 @@ static int SQLITE_TCLAPI test_stmt_isexplain(
   return TCL_OK;
 }
 
+/*
+** Usage:  sqlite3_stmt_explain  STMT  INT
+**
+** Set the explain to normal (0), EXPLAIN (1) or EXPLAIN QUERY PLAN (2).
+*/
+static int SQLITE_TCLAPI test_stmt_explain(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+  int eMode = 0;
+  int rc;
+
+  if( objc!=3 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"",
+        Tcl_GetStringFromObj(objv[0], 0), " STMT INT", 0);
+    return TCL_ERROR;
+  }
+
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  if( Tcl_GetIntFromObj(interp, objv[2], &eMode) ) return TCL_ERROR;
+  rc = sqlite3_stmt_explain(pStmt, eMode);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_stmt_busy  STMT
 **
@@ -3061,7 +3268,7 @@ static int SQLITE_TCLAPI test_bind(
   if( rc ){
     char zBuf[50];
     sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
-    Tcl_AppendResult(interp, zBuf, sqlite3ErrStr(rc), 0);
+    Tcl_AppendResult(interp, zBuf, sqlite3ErrStr(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -3193,14 +3400,14 @@ static int SQLITE_TCLAPI test_collate(
   if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR;
   
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
 
 bad_args:
   Tcl_AppendResult(interp, "wrong # args: should be \"",
-      Tcl_GetStringFromObj(objv[0], 0), " <DB> <utf8> <utf16le> <utf16be>", 0);
+      Tcl_GetStringFromObj(objv[0], 0), " <DB> <utf8> <utf16le> <utf16be>", NULL);
   return TCL_ERROR;
 }
 
@@ -3479,7 +3686,7 @@ static int SQLITE_TCLAPI test_function(
   return TCL_OK;
 bad_args:
   Tcl_AppendResult(interp, "wrong # args: should be \"",
-      Tcl_GetStringFromObj(objv[0], 0), " <DB> <utf8> <utf16le> <utf16be>", 0);
+      Tcl_GetStringFromObj(objv[0], 0), " <DB> <utf8> <utf16le> <utf16be>", NULL);
 #endif /* SQLITE_OMIT_UTF16 */
   return TCL_ERROR;
 }
@@ -3487,10 +3694,10 @@ static int SQLITE_TCLAPI test_function(
 /*
 ** Usage:         sqlite3_test_errstr <err code>
 **
-** Test that the english language string equivalents for sqlite error codes
+** Test that the English language string equivalents for sqlite error codes
 ** are sane. The parameter is an integer representing an sqlite error code.
 ** The result is a list of two elements, the string representation of the
-** error code and the english language explanation.
+** error code and the English language explanation.
 */
 static int SQLITE_TCLAPI test_errstr(
   void * clientData,
@@ -3600,7 +3807,7 @@ static int SQLITE_TCLAPI test_bind_zeroblob64(
   rc = sqlite3_bind_zeroblob64(pStmt, idx, n);
   if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR;
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
 
@@ -3627,7 +3834,7 @@ static int SQLITE_TCLAPI test_bind_int(
 
   if( objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", NULL);
     return TCL_ERROR;
   }
 
@@ -3676,11 +3883,11 @@ static int SQLITE_TCLAPI test_intarray_addr(
       }
     }
   }  
-  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((uptr)p));
   return TCL_OK;
 }
 /*
-** Usage:   intarray_addr  INT  ...
+** Usage:   int64array_addr  INT  ...
 **
 ** Return the address of a C-language array of 32-bit integers.
 **
@@ -3712,7 +3919,7 @@ static int SQLITE_TCLAPI test_int64array_addr(
       p[i] = v;
     }
   }  
-  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((uptr)p));
   return TCL_OK;
 }
 /*
@@ -3746,7 +3953,7 @@ static int SQLITE_TCLAPI test_doublearray_addr(
       }
     }
   }  
-  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((uptr)p));
   return TCL_OK;
 }
 /*
@@ -3779,11 +3986,10 @@ static int SQLITE_TCLAPI test_textarray_addr(
     }
   }
   n = objc-1;
-  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((sqlite3_int64)p));
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj((uptr)p));
   return TCL_OK;
 }
 
-
 /*
 ** Usage:   sqlite3_bind_int64  STMT N VALUE
 **
@@ -3804,7 +4010,7 @@ static int SQLITE_TCLAPI test_bind_int64(
 
   if( objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", NULL);
     return TCL_ERROR;
   }
 
@@ -3860,7 +4066,7 @@ static int SQLITE_TCLAPI test_bind_double(
 
   if( objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", NULL);
     return TCL_ERROR;
   }
 
@@ -3917,7 +4123,7 @@ static int SQLITE_TCLAPI test_bind_null(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N", NULL);
     return TCL_ERROR;
   }
 
@@ -3949,7 +4155,7 @@ static int SQLITE_TCLAPI test_bind_text(
 ){
   sqlite3_stmt *pStmt;
   int idx;
-  int trueLength = 0;
+  Tcl_Size trueLength = 0;
   int bytes;
   char *value;
   int rc;
@@ -3957,7 +4163,7 @@ static int SQLITE_TCLAPI test_bind_text(
 
   if( objc!=5 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", NULL);
     return TCL_ERROR;
   }
 
@@ -4007,7 +4213,7 @@ static int SQLITE_TCLAPI test_bind_text16(
   char *value;
   char *toFree = 0;
   int rc;
-  int trueLength = 0;
+  Tcl_Size trueLength = 0;
 
   void (*xDel)(void*) = (objc==6?SQLITE_STATIC:SQLITE_TRANSIENT);
   Tcl_Obj *oStmt    = objv[objc-4];
@@ -4017,7 +4223,7 @@ static int SQLITE_TCLAPI test_bind_text16(
 
   if( objc!=5 && objc!=6){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", NULL);
     return TCL_ERROR;
   }
 
@@ -4039,7 +4245,7 @@ static int SQLITE_TCLAPI test_bind_text16(
   free(toFree);
   if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR;
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
 
@@ -4061,7 +4267,8 @@ static int SQLITE_TCLAPI test_bind_blob(
   Tcl_Obj *CONST objv[]
 ){
   sqlite3_stmt *pStmt;
-  int len, idx;
+  Tcl_Size len;
+  int idx;
   int bytes;
   char *value;
   int rc;
@@ -4069,7 +4276,7 @@ static int SQLITE_TCLAPI test_bind_blob(
 
   if( objc!=5 && objc!=6 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " STMT N DATA BYTES", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " STMT N DATA BYTES", NULL);
     return TCL_ERROR;
   }
 
@@ -4087,7 +4294,7 @@ static int SQLITE_TCLAPI test_bind_blob(
   if( bytes>len ){
     char zBuf[200];
     sqlite3_snprintf(sizeof(zBuf), zBuf,
-                     "cannot use %d blob bytes, have %d", bytes, len);
+                     "cannot use %d blob bytes, have %d", bytes, (int)len);
     Tcl_AppendResult(interp, zBuf, (char*)0);
     return TCL_ERROR;
   }
@@ -4118,9 +4325,11 @@ static int SQLITE_TCLAPI test_bind_value_from_preupdate(
   sqlite3_stmt *pStmt;
   int idx;
   int bidx;
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   const char *z3 = 0;
   sqlite3 *db = 0;
   sqlite3_value *pVal = 0;
+#endif
 
   if( objc!=5 ){
     Tcl_WrongNumArgs(interp, 1, objv, "STMT N NEW|OLD IDX");
@@ -4129,11 +4338,11 @@ static int SQLITE_TCLAPI test_bind_value_from_preupdate(
 
   if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
   if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR;
-  z3 = Tcl_GetString(objv[3]);
   if( Tcl_GetIntFromObj(interp, objv[4], &bidx) ) return TCL_ERROR;
-  db = sqlite3_db_handle(pStmt);
 
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  z3 = Tcl_GetString(objv[3]);
+  db = sqlite3_db_handle(pStmt);
   if( z3[0]=='n' ){
     sqlite3_preupdate_new(db, bidx, &pVal);
   }else if( z3[0]=='o' ){
@@ -4207,10 +4416,71 @@ static int SQLITE_TCLAPI test_bind_value_from_select(
 #endif
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** These two are used by the -malloc option to sqlite3_carray_bind()
+*/
+static void *testCarrayAlloc(int n){
+  u8 *pRet = (u8*)sqlite3_malloc(n+16);
+  if( pRet ){
+    pRet = &pRet[16];
+  }
+  return (void*)pRet;
+}
+static void testCarrayFree(void *p){
+  if( p ){
+    u8 *p2 = (u8*)p;
+    sqlite3_free(&p2[-16]);
+  }
+}
+
+static void delIntptr(void *p){
+  ckfree(p);
+}
+
+/*
+** bind_carray_intptr STMT IPARAM  INT0 INT1 INT2...
+*/
+static int SQLITE_TCLAPI bind_carray_intptr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt = 0;
+  int iVar = 0;
+  int *aInt = 0;
+  int nInt = 0;
+  int ii = 0;
+  int rc = SQLITE_OK;
+
+  if( objc<3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "STMT");
+    return TCL_ERROR;
+  }
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  if( Tcl_GetIntFromObj(interp, objv[2], &iVar) ) return TCL_ERROR;
+  nInt = objc - 3;
+
+  aInt = ckalloc((nInt+1) * sizeof(int));
+  for(ii=0; ii<nInt; ii++){
+    if( Tcl_GetIntFromObj(interp, objv[3+ii], &aInt[ii]) ){
+      ckfree(aInt);
+      return TCL_ERROR;
+    }
+  }
+
+  rc = sqlite3_bind_pointer(pStmt, iVar, (void*)aInt, "carray", delIntptr);
+  Tcl_SetResult(interp, (char *)t1ErrorName(rc), 0);
+
+  return TCL_OK;
+}
+
 /*
 ** sqlite3_carray_bind [options...] STMT NAME VALUE ...
 **
 ** Options:
+**    -malloc
 **    -transient
 **    -static
 **    -int32
@@ -4230,25 +4500,19 @@ static int SQLITE_TCLAPI test_carray_bind(
 ){
   sqlite3_stmt *pStmt;
   int eType = 0;   /* CARRAY_INT32 */
+  int mFlagsOverride = 0;
   int nData = 0;
   void *aData = 0;
   int isTransient = 0;
   int isStatic = 0;
+  int isMalloc = 0;               /* True to use custom xDel function */
   int idx;
   int i, j;
-  int rc;
+  int rc = SQLITE_OK;
   void (*xDel)(void*) = sqlite3_free;
   static void *aStaticData = 0;
   static int nStaticData = 0;
   static int eStaticType = 0;
-  extern int sqlite3_carray_bind(
-    sqlite3_stmt *pStmt,
-    int i,
-    void *aData,
-    int nData,
-    int mFlags,
-    void (*xDestroy)(void*)
-  );
   
   if( aStaticData ){
     /* Always clear preexisting static data on every call */
@@ -4279,6 +4543,10 @@ static int SQLITE_TCLAPI test_carray_bind(
       isStatic = 1;
       xDel = SQLITE_STATIC;
     }else
+    if( strcmp(z, "-malloc")==0 ){
+      isMalloc = 1;
+      xDel = testCarrayFree;
+    }else
     if( strcmp(z, "-int32")==0 ){
       eType = 0;  /* CARRAY_INT32 */
     }else
@@ -4294,6 +4562,12 @@ static int SQLITE_TCLAPI test_carray_bind(
     if( strcmp(z, "-blob")==0 ){
       eType = 4;  /* CARRAY_BLOB */
     }else
+    if( i<(objc-1) && strcmp(z, "-flags")==0 ){
+      i++;
+      if( Tcl_GetIntFromObj(interp, objv[i], &mFlagsOverride) ){
+        return TCL_ERROR;
+      }
+    }else
     if( strcmp(z, "--")==0 ){
       break;
     }else
@@ -4371,24 +4645,36 @@ static int SQLITE_TCLAPI test_carray_bind(
     }
     case 3: { /* TEXT */
       char **a = sqlite3_malloc( sizeof(char*)*nData );
-      if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; }
-      for(j=0; j<nData; j++){
+      if( a==0 ){ 
+        rc = SQLITE_NOMEM;
+      }else{
+        memset(a, 0, sizeof(char*)*nData);
+      }
+      for(j=0; rc==SQLITE_OK && j<nData; j++){
         const char *v = Tcl_GetString(objv[i+j]);
-        a[j] = sqlite3_mprintf("%s", v);
+        if( v && strcmp(v, "NULL") ){
+          a[j] = sqlite3_mprintf("%s", v);
+          if( a[j]==0 ) rc = SQLITE_NOMEM;
+        }
       }
       aData = a;
       break;
     }
     case 4: { /* BLOB */
       struct iovec *a = sqlite3_malloc( sizeof(struct iovec)*nData );
-      if( a==0 ){ rc = SQLITE_NOMEM; goto carray_bind_done; }
-      for(j=0; j<nData; j++){
-        int n = 0;
+      if( a==0 ){ 
+        rc = SQLITE_NOMEM; 
+      }else{
+        memset(a, 0, sizeof(struct iovec)*nData);
+      }
+      for(j=0; rc==SQLITE_OK && j<nData; j++){
+        Tcl_Size n = 0;
         unsigned char *v = Tcl_GetByteArrayFromObj(objv[i+i], &n);
-        a[j].iov_len = n;
+        a[j].iov_len = (size_t)n;
         a[j].iov_base = sqlite3_malloc64( n );
         if( a[j].iov_base==0 ){
           a[j].iov_len = 0;
+          rc = SQLITE_NOMEM;
         }else{
           memcpy(a[j].iov_base, v, n);
         }
@@ -4404,17 +4690,38 @@ static int SQLITE_TCLAPI test_carray_bind(
       break;
     }
   }
-  if( isStatic ){
-    aStaticData = aData;
-    nStaticData = nData;
-    eStaticType = eType;
+
+  if( rc==SQLITE_OK ){
+    if( isStatic ){
+      aStaticData = aData;
+      nStaticData = nData;
+      eStaticType = eType;
+    }
+    else if( isMalloc ){
+      int nByte = ((eType==0) ? sizeof(int) : sizeof(i64)) * nData;
+      void *aByte = testCarrayAlloc(nByte);
+      if( aByte==0 ){
+        sqlite3_free(aData);
+        rc = SQLITE_NOMEM;
+      }else{
+        memcpy(aByte, aData, nByte);
+        sqlite3_free(aData);
+        aData = aByte;
+        xDel = testCarrayFree;
+      }
+      assert( eType==0 || eType==1 || eType==2 );
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    if( mFlagsOverride==0 ) mFlagsOverride = eType;
+    rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel);
   }
-  rc = sqlite3_carray_bind(pStmt, idx, aData, nData, eType, xDel);
   if( isTransient ){
-    if( eType==3 ){
+    if( eType==3 && aData ){
       for(i=0; i<nData; i++) sqlite3_free(((char**)aData)[i]);
     }
-    if( eType==4 ){
+    if( eType==4 && aData ){
       for(i=0; i<nData; i++) sqlite3_free(((struct iovec*)aData)[i].iov_base);
     }
     sqlite3_free(aData);
@@ -4565,12 +4872,12 @@ static int SQLITE_TCLAPI test_ex_errcode(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB", 0);
+       Tcl_GetString(objv[0]), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
   rc = sqlite3_extended_errcode(db);
-  Tcl_AppendResult(interp, (char *)t1ErrorName(rc), 0);
+  Tcl_AppendResult(interp, (char *)t1ErrorName(rc), NULL);
   return TCL_OK;
 }
 
@@ -4592,12 +4899,12 @@ static int SQLITE_TCLAPI test_errcode(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB", 0);
+       Tcl_GetString(objv[0]), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
   rc = sqlite3_errcode(db);
-  Tcl_AppendResult(interp, (char *)t1ErrorName(rc), 0);
+  Tcl_AppendResult(interp, (char *)t1ErrorName(rc), NULL);
   return TCL_OK;
 }
 
@@ -4618,7 +4925,7 @@ static int SQLITE_TCLAPI test_errmsg(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB", 0);
+       Tcl_GetString(objv[0]), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4646,7 +4953,7 @@ static int SQLITE_TCLAPI test_error_offset(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB", 0);
+       Tcl_GetString(objv[0]), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4678,7 +4985,7 @@ static int SQLITE_TCLAPI test_errmsg16(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB", 0);
+       Tcl_GetString(objv[0]), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4693,6 +5000,37 @@ static int SQLITE_TCLAPI test_errmsg16(
   return TCL_OK;
 }
 
+/*
+** Usage:   sqlite3_set_errmsg DB ERRCODE ERRMSG
+*/
+static int SQLITE_TCLAPI test_set_errmsg(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *zDb = 0;
+  const char *zErr = 0;
+  int iErr = 0;
+  sqlite3 *db = 0;
+  int rc;
+
+  if( objc!=4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB ERRCODE ERRMSG");
+    return TCL_ERROR;
+  }
+  zDb = Tcl_GetString(objv[1]);
+  if( zDb[0] ){
+    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  }
+  if( Tcl_GetIntFromObj(interp, objv[2], &iErr) ) return TCL_ERROR;
+  zErr = Tcl_GetString(objv[3]);
+
+  rc = sqlite3_set_errmsg(db, iErr, (zErr[0] ? zErr : 0));
+  Tcl_SetResult(interp, (char *)t1ErrorName(rc), 0);
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_prepare DB sql bytes ?tailvar?
 **
@@ -4717,7 +5055,7 @@ static int SQLITE_TCLAPI test_prepare(
 
   if( objc!=5 && objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", 0);
+       Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4739,13 +5077,13 @@ static int SQLITE_TCLAPI test_prepare(
   if( rc!=SQLITE_OK ){
     assert( pStmt==0 );
     sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
-    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
@@ -4776,7 +5114,7 @@ static int SQLITE_TCLAPI test_prepare_v2(
 
   if( objc!=5 && objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB sql bytes tailvar", 0);
+       Tcl_GetString(objv[0]), " DB sql bytes tailvar", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4813,13 +5151,13 @@ static int SQLITE_TCLAPI test_prepare_v2(
   if( rc!=SQLITE_OK ){
     assert( pStmt==0 );
     sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
-    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
@@ -4850,7 +5188,7 @@ static int SQLITE_TCLAPI test_prepare_v3(
 
   if( objc!=6 && objc!=5 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB sql bytes flags tailvar", 0);
+       Tcl_GetString(objv[0]), " DB sql bytes flags tailvar", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4871,8 +5209,8 @@ static int SQLITE_TCLAPI test_prepare_v3(
   }
   pzTail = objc>=6 ? &zTail : 0;
   rc = sqlite3_prepare_v3(db, zCopy, bytes, (unsigned int)flags,&pStmt,pzTail);
-  free(zCopy);
   zTail = &zSql[(zTail - zCopy)];
+  free(zCopy);
 
   assert(rc==SQLITE_OK || pStmt==0);
   Tcl_ResetResult(interp);
@@ -4886,13 +5224,13 @@ static int SQLITE_TCLAPI test_prepare_v3(
   if( rc!=SQLITE_OK ){
     assert( pStmt==0 );
     sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
-    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
@@ -4917,7 +5255,7 @@ static int SQLITE_TCLAPI test_prepare_tkt3134(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB sql bytes tailvar", 0);
+       Tcl_GetString(objv[0]), " DB sql bytes tailvar", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4927,13 +5265,13 @@ static int SQLITE_TCLAPI test_prepare_tkt3134(
   if( rc!=SQLITE_OK ){
     assert( pStmt==0 );
     sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
-    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
@@ -4961,11 +5299,11 @@ static int SQLITE_TCLAPI test_prepare16(
   char zBuf[50]; 
   int rc;
   int bytes;                /* The integer specified as arg 3 */
-  int objlen;               /* The byte-array length of arg 2 */
+  Tcl_Size objlen;          /* The byte-array length of arg 2 */
 
   if( objc!=5 && objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", 0);
+       Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -4993,7 +5331,7 @@ static int SQLITE_TCLAPI test_prepare16(
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
   }
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
 #endif /* SQLITE_OMIT_UTF16 */
   return TCL_OK;
 }
@@ -5021,11 +5359,11 @@ static int SQLITE_TCLAPI test_prepare16_v2(
   char zBuf[50]; 
   int rc;
   int bytes;                /* The integer specified as arg 3 */
-  int objlen;               /* The byte-array length of arg 2 */
+  Tcl_Size objlen;          /* The byte-array length of arg 2 */
 
   if( objc!=5 && objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", 0);
+       Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -5053,7 +5391,7 @@ static int SQLITE_TCLAPI test_prepare16_v2(
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
   }
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
 #endif /* SQLITE_OMIT_UTF16 */
   return TCL_OK;
 }
@@ -5073,7 +5411,7 @@ static int SQLITE_TCLAPI test_open(
 
   if( objc!=3 && objc!=2 && objc!=1 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " filename options-list", 0);
+       Tcl_GetString(objv[0]), " filename options-list", NULL);
     return TCL_ERROR;
   }
 
@@ -5081,7 +5419,7 @@ static int SQLITE_TCLAPI test_open(
   sqlite3_open(zFilename, &db);
   
   if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR;
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -5101,9 +5439,9 @@ static int SQLITE_TCLAPI test_open_v2(
   int rc;
   char zBuf[100];
 
-  int nFlag;
+  Tcl_Size nFlag;
   Tcl_Obj **apFlag;
-  int i;
+  Tcl_Size i;
 
   if( objc!=4 ){
     Tcl_WrongNumArgs(interp, 1, objv, "FILENAME FLAGS VFS");
@@ -5152,7 +5490,7 @@ static int SQLITE_TCLAPI test_open_v2(
 
   rc = sqlite3_open_v2(zFilename, &db, flags, zVfs);
   if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR;
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -5172,7 +5510,7 @@ static int SQLITE_TCLAPI test_open16(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " filename options-list", 0);
+       Tcl_GetString(objv[0]), " filename options-list", NULL);
     return TCL_ERROR;
   }
 
@@ -5180,7 +5518,7 @@ static int SQLITE_TCLAPI test_open16(
   sqlite3_open16(zFilename, &db);
   
   if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR;
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
 #endif /* SQLITE_OMIT_UTF16 */
   return TCL_OK;
 }
@@ -5256,7 +5594,7 @@ static int SQLITE_TCLAPI test_step(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT", 0);
+       Tcl_GetString(objv[0]), " STMT", NULL);
     return TCL_ERROR;
   }
 
@@ -5340,7 +5678,7 @@ static int SQLITE_TCLAPI test_column_count(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5367,7 +5705,7 @@ static int SQLITE_TCLAPI test_column_type(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5416,7 +5754,7 @@ static int SQLITE_TCLAPI test_column_int64(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5445,7 +5783,7 @@ static int SQLITE_TCLAPI test_column_blob(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5475,7 +5813,7 @@ static int SQLITE_TCLAPI test_column_double(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5502,7 +5840,7 @@ static int SQLITE_TCLAPI test_data_count(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5528,18 +5866,24 @@ static int SQLITE_TCLAPI test_stmt_utf8(
   sqlite3_stmt *pStmt;
   int col;
   const char *(*xFunc)(sqlite3_stmt*, int);
+  const unsigned char *(*xFuncU)(sqlite3_stmt*, int);
   const char *zRet;
 
   xFunc = (const char *(*)(sqlite3_stmt*, int))clientData;
+  xFuncU = (const unsigned char*(*)(sqlite3_stmt*,int))xFunc;
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
   if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
   if( Tcl_GetIntFromObj(interp, objv[2], &col) ) return TCL_ERROR;
-  zRet = xFunc(pStmt, col);
+  if( xFunc==sqlite3_column_name || xFunc==sqlite3_column_decltype ){
+    zRet = xFunc(pStmt, col);
+  }else{
+    zRet = (const char*)xFuncU(pStmt, col);
+  }
   if( zRet ){
     Tcl_SetResult(interp, (char *)zRet, 0);
   }
@@ -5587,7 +5931,7 @@ static int SQLITE_TCLAPI test_stmt_utf16(
   xFunc = (const void *(*)(sqlite3_stmt*, int))clientData;
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5628,7 +5972,7 @@ static int SQLITE_TCLAPI test_stmt_int(
   xFunc = (int (*)(sqlite3_stmt*, int))clientData;
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
-       Tcl_GetString(objv[0]), " STMT column", 0);
+       Tcl_GetString(objv[0]), " STMT column", NULL);
     return TCL_ERROR;
   }
 
@@ -5652,7 +5996,7 @@ static int SQLITE_TCLAPI test_interrupt(
 ){
   sqlite3 *db;
   if( argc!=2 ){
-    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", 0);
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR;
@@ -5674,7 +6018,7 @@ static int SQLITE_TCLAPI test_is_interrupted(
   sqlite3 *db;
   int rc;
   if( argc!=2 ){
-    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", 0);
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR;
@@ -5756,7 +6100,7 @@ static int SQLITE_TCLAPI get_autocommit(
   }
   if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR;
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", sqlite3_get_autocommit(db));
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -5777,13 +6121,49 @@ static int SQLITE_TCLAPI test_busy_timeout(
   sqlite3 *db;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], 
-        " DB", 0);
+        " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR;
   if( Tcl_GetInt(interp, argv[2], &ms) ) return TCL_ERROR;
   rc = sqlite3_busy_timeout(db, ms);
-  Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+  Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
+  return TCL_OK;
+}
+
+/*
+** Usage: sqlite3_setlk_timeout ?-blockonconnect? DB MS
+**
+** Set the setlk timeout.
+*/
+static int SQLITE_TCLAPI test_setlk_timeout(
+  void * clientData,
+  Tcl_Interp *interp,
+  int argc,
+  char **argv
+){
+  int rc, ms;
+  sqlite3 *db;
+  int bBlockOnConnect = 0;
+
+  if( argc==4 ){
+    const char *zArg = argv[1];
+    const size_t nArg = strlen(zArg);
+    if( nArg>=2 && nArg<=15 && memcmp(zArg, "-blockonconnect", nArg)==0 ){
+      bBlockOnConnect = 1;
+    }
+  }
+  if( argc!=(3+bBlockOnConnect) ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], 
+        " ?-blockonconnect? DB MS", NULL);
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, argv[argc-2], &db) ) return TCL_ERROR;
+  if( Tcl_GetInt(interp, argv[argc-1], &ms) ) return TCL_ERROR;
+  rc = sqlite3_setlk_timeout(
+      db, ms, (bBlockOnConnect ? SQLITE_SETLK_BLOCK_ON_CONNECT : 0)
+  );
+  Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
   return TCL_OK;
 }
 
@@ -5812,6 +6192,145 @@ static int SQLITE_TCLAPI tcl_variable_type(
   return TCL_OK;
 }
 
+#include <ctype.h>
+
+/*
+** Usage:  fpnum_compare STRING1 STRING2
+**
+** Compare two strings.  Return true if the strings are the same and
+** false if they differ.
+**
+** For this comparison, the strings are analyzed as a sequenced of
+** whitespace separated tokens.  The whitespace is ignored.  Only the
+** tokens are compared.  Comparison rules:
+**
+**   A.  Tokens that are not floating-point numbers must match exactly.
+**
+**   B.  Floating point number must have exactly the same digits before
+**       the decimal point.
+**
+**   C.  Digits must match after the decimal point up to 15 digits,
+**       taking rounding into consideration.
+**
+**   D.  An exponent on a floating point of the form "e+NN" will
+**       match "e+N" if NN==N.  Likewise for the negative exponent.
+**
+** This routine is used for comparing results that might involve floating
+** point values.  Tcl9.0 and Tcl8.6 differ in the number of significant
+** digits that they show, so there is no way to write a portable test result
+** without this routine.
+**
+** This routine is only called after [string compare] fails, which is seldom,
+** so performance is not a pressing concern.  Better to get the correct answer
+** slowly.
+*/
+static int SQLITE_TCLAPI fpnum_compare(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const unsigned char *zA;
+  const unsigned char *zB;
+  int i, j;
+  int nDigit;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "STRING1 STRING2");
+    return TCL_ERROR;
+  }
+  zA = (const unsigned char*)Tcl_GetString(objv[1]);
+  zB = (const unsigned char*)Tcl_GetString(objv[2]);
+  i = j = 0;
+  while( 1 ){
+    /* Skip whitespace before and after tokens */
+    while( isspace(zA[i]) ){ i++; }
+    while( isspace(zB[j]) ){ j++; }
+
+    if( zA[i]!=zB[j] ) break;                /* First character must match */
+    if( zA[i]=='-' && isdigit(zA[i+1]) ){ i++; j++; }  /* Skip initial '-' */
+    if( !isdigit(zA[i]) ){
+      /* Not a number.  Must match exactly */
+      while( !isspace(zA[i]) && zA[i] && zA[i]==zB[j] ){ i++; j++; }
+      if( zA[i]!=zB[j] ) break;
+      if( isspace(zA[i]) ) continue;
+      break;
+    }
+
+    /* At this point we know we are dealing with a number zA[i] and zB[j]
+    ** are both digits (leading "-" have been skipped).  See if they are
+    ** the same number.  Start by matching digits before the decimal
+    ** point, which must all be the same. */
+    nDigit = 0;
+    while( zA[i]==zB[j] && isdigit(zA[i]) ){ i++; j++; nDigit++; }
+    if( zA[i]!=zB[j] ) break;
+    if( zA[i]==0 ) break;
+    if( zA[i]=='.' && zB[j]=='.' ){
+      /* Count more matching digits after the decimal point */
+      i++;
+      j++;
+      while( zA[i]==zB[j] && isdigit(zA[i]) ){ i++; j++; nDigit++; }
+      if( zA[i]==0 ){
+        while( zB[j]=='0' || (isdigit(zB[j]) && nDigit>=15) ){ j++; nDigit++; }
+        break;
+      }
+      if( zB[j]==0 ){
+        while( zA[i]=='0' || (isdigit(zA[i]) && nDigit>=15) ){ i++; nDigit++; }
+        break;
+      }
+      if( isspace(zA[i]) && isspace(zB[j]) ) continue;
+
+      if( isdigit(zA[i]) && isdigit(zB[j]) ){
+        /* A and B are both digits, but different digits */
+        if( zA[i]==zB[j]+1 && !isdigit(zA[i+1]) && isdigit(zB[j+1]) ){
+          /* Is A a rounded up version of B? */
+          j++;
+          while( zB[j]=='9' ){ j++; nDigit++; }
+          if( nDigit<14 && (!isdigit(zB[j]) || zB[j]<5) ) break;
+          while( isdigit(zB[j]) ){ j++; }
+          i++;
+        }else if( zB[j]==zA[i]+1 && !isdigit(zB[j+1]) && isdigit(zA[i+1]) ){
+          /* Is B a rounded up version of A? */
+          i++;
+          while( zA[i]=='9' ){ i++; nDigit++; }
+          if( nDigit<14 && (!isdigit(zA[i]) || zA[i]<5) ) break;
+          while( isdigit(zA[i]) ){ i++; }
+          j++;
+        }else{
+          break;
+        }
+      }else if( !isdigit(zA[i]) && isdigit(zB[j]) ){
+        while( zB[j]=='0' ){ j++; nDigit++; }
+        if( nDigit<15 ) break;
+        while( isdigit(zB[j]) ){ j++; }
+      }else if( !isdigit(zB[j]) && isdigit(zA[i]) ){
+        while( zA[i]=='0' ){ i++; nDigit++; }
+        if( nDigit<15 ) break;
+        while( isdigit(zA[i]) ){ i++; }
+      }else{
+        break;
+      }
+    }
+    if( zA[i]=='e' && zB[j]=='e' ){
+      i++;
+      j++;
+      if( (zA[i]=='+' || zA[i]=='-') && zB[j]==zA[i] ){  i++;  j++; }
+      if( zA[i]!=zB[j] ){
+        if( zA[i]=='0' && zA[i+1]==zB[j] ){ i++; }
+        if( zB[j]=='0' && zB[j+1]==zA[i] ){ j++; }
+      }
+      while( zA[i]==zB[j] && isdigit(zA[i]) ){ i++; j++; }
+      if( zA[i]!=zB[j] ) break;
+      if( zA[i]==0 ) break;
+      continue;
+    }
+  }
+  while( isspace(zA[i]) ){ i++; }
+  while( isspace(zB[j]) ){ j++; }  
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(zA[i]==0 && zB[j]==0));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_release_memory ?N?
 **
@@ -6056,7 +6575,7 @@ static int SQLITE_TCLAPI test_pager_refcounts(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -6143,7 +6662,7 @@ static int SQLITE_TCLAPI vfs_unlink_test(
   assert( sqlite3_vfs_find("__two")==&two );
 
   /* Calling sqlite_vfs_register with non-zero second parameter changes the
-  ** default VFS, even if the 1st parameter is an existig VFS that is
+  ** default VFS, even if the 1st parameter is an existing VFS that is
   ** previously registered as the non-default.
   */
   sqlite3_vfs_register(&one, 1);
@@ -6298,7 +6817,7 @@ static int SQLITE_TCLAPI file_control_test(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -6333,7 +6852,7 @@ static int SQLITE_TCLAPI file_control_lasterrno_test(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6346,7 +6865,7 @@ static int SQLITE_TCLAPI file_control_lasterrno_test(
   }
   if( iArg!=0 ) {
     Tcl_AppendResult(interp, "Unexpected non-zero errno: ",
-                     Tcl_GetStringFromObj(Tcl_NewIntObj(iArg), 0), " ", 0);
+                     Tcl_GetStringFromObj(Tcl_NewIntObj(iArg), 0), " ", NULL);
     return TCL_ERROR;
   }
   return TCL_OK;  
@@ -6482,7 +7001,7 @@ static int SQLITE_TCLAPI file_control_lockproxy_test(
   
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-                     Tcl_GetStringFromObj(objv[0], 0), " DB PWD", 0);
+                     Tcl_GetStringFromObj(objv[0], 0), " DB PWD", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6500,7 +7019,7 @@ static int SQLITE_TCLAPI file_control_lockproxy_test(
   {
     char *testPath;
     int rc;
-    int nPwd;
+    Tcl_Size nPwd;
     const char *zPwd;
     char proxyPath[400];
     
@@ -6518,7 +7037,7 @@ static int SQLITE_TCLAPI file_control_lockproxy_test(
     rc = sqlite3_file_control(db, NULL, SQLITE_GET_LOCKPROXYFILE, &testPath);
     if( strncmp(proxyPath,testPath,11) ){
       Tcl_AppendResult(interp, "Lock proxy file did not match the "
-                               "previously assigned value", 0);
+                               "previously assigned value", NULL);
       return TCL_ERROR;
     }
     if( rc ){
@@ -6555,7 +7074,7 @@ static int SQLITE_TCLAPI file_control_win32_av_retry(
 
   if( objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6588,7 +7107,7 @@ static int file_control_win32_get_handle(
 
   if( objc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6620,7 +7139,7 @@ static int SQLITE_TCLAPI file_control_win32_set_handle(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB HANDLE", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB HANDLE", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6656,7 +7175,7 @@ static int SQLITE_TCLAPI file_control_persist_wal(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6688,7 +7207,7 @@ static int SQLITE_TCLAPI file_control_powersafe_overwrite(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6719,7 +7238,7 @@ static int SQLITE_TCLAPI file_control_vfsname(
 
   if( objc!=2 && objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6781,7 +7300,7 @@ static int SQLITE_TCLAPI file_control_tempfilename(
 
   if( objc!=2 && objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6814,7 +7333,7 @@ static int SQLITE_TCLAPI file_control_external_reader(
 
   if( objc!=2 && objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
@@ -6898,7 +7417,7 @@ static int SQLITE_TCLAPI test_limit(
 
   if( objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " DB ID VALUE", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " DB ID VALUE", NULL);
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
@@ -7209,7 +7728,7 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2(
   if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
     const char *zErrCode = sqlite3ErrName(rc);
     Tcl_ResetResult(interp);
-    Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
@@ -7261,12 +7780,16 @@ static int SQLITE_TCLAPI test_wal_autocheckpoint(
 
 /*
 ** tclcmd:  test_sqlite3_log ?SCRIPT?
+**
+** Caution:  If you register a log callback, you must deregister it (by
+** invoking test_sqlite3_log with no arguments) prior to closing the
+** Tcl interpreter or else a memory error will occur.
 */
 static struct LogCallback {
   Tcl_Interp *pInterp;
   Tcl_Obj *pObj;
 } logcallback = {0, 0};
-static void xLogcallback(void *unused, int err, char *zMsg){
+static void xLogcallback(void *unused, int err, const char *zMsg){
   Tcl_Obj *pNew = Tcl_DuplicateObj(logcallback.pObj);
   Tcl_IncrRefCount(pNew);
   Tcl_ListObjAppendElement(
@@ -7292,7 +7815,7 @@ static int SQLITE_TCLAPI test_sqlite3_log(
     logcallback.pInterp = 0;
     sqlite3_config(SQLITE_CONFIG_LOG, (void*)0, (void*)0);
   }
-  if( objc>1 ){
+  if( objc>1 && Tcl_GetString(objv[1])[0]!=0 ){
     logcallback.pObj = objv[1];
     Tcl_IncrRefCount(logcallback.pObj);
     logcallback.pInterp = interp;
@@ -7444,6 +7967,61 @@ static int testLocaltime(const void *aliasT, void *aliasTM){
   return t==959609760; /* Special case: 2000-05-29 14:16:00 fails */
 }
 
+/*
+** TCLCMD:  strftime FORMAT UNIXTIMESTAMP
+**
+** Access to the C-library strftime() routine, so that its results
+** can be compared against SQLite's internal strftime() SQL function
+** implementation.
+*/
+static int SQLITE_TCLAPI strftime_cmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  Tcl_WideInt ts;
+  time_t t;
+  struct tm *pTm;
+  const char *zFmt;
+  size_t n;
+  char zBuf[1000];
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "FORMAT UNIXTIMESTAMP");
+    return TCL_ERROR;
+  }
+  if( Tcl_GetWideIntFromObj(interp,  objv[2], &ts) ) return TCL_ERROR;
+  zFmt = Tcl_GetString(objv[1]);
+  t = (time_t)ts;
+  pTm = gmtime(&t);
+  n = strftime(zBuf, sizeof(zBuf)-1, zFmt, pTm);
+  if( n>=0 && n<sizeof(zBuf) ){
+    zBuf[n] = 0;
+    Tcl_SetResult(interp, zBuf, TCL_VOLATILE);
+  }
+  return TCL_OK;
+}
+
+/*
+** .treetrace N
+*/
+static int SQLITE_TCLAPI test_treetrace(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  unsigned int v = 0;
+  if( objc>=2 ){
+    if( Tcl_GetIntFromObj(interp, objv[1], (int*)&v)==TCL_OK ){
+      sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &v);
+    }
+  }
+  sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &v);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj((int)v));
+  return TCL_OK;
+}
+
 /*
 ** sqlite3_test_control VERB ARGS...
 */
@@ -7461,6 +8039,7 @@ static int SQLITE_TCLAPI test_test_control(
     { "SQLITE_TESTCTRL_SORTER_MMAP",        SQLITE_TESTCTRL_SORTER_MMAP     }, 
     { "SQLITE_TESTCTRL_IMPOSTER",           SQLITE_TESTCTRL_IMPOSTER        },
     { "SQLITE_TESTCTRL_INTERNAL_FUNCTIONS", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS},
+    { "SQLITE_TESTCTRL_FK_NO_ACTION",       SQLITE_TESTCTRL_FK_NO_ACTION},
     { 0, 0 }
   };
   int iVerb;
@@ -7500,6 +8079,20 @@ static int SQLITE_TCLAPI test_test_control(
       break;
     }
 
+    case SQLITE_TESTCTRL_FK_NO_ACTION: {
+      int val = 0;
+      sqlite3 *db = 0;
+      if( objc!=4 ){
+        Tcl_WrongNumArgs(interp, 2, objv, "DB BOOLEAN");
+        return TCL_ERROR;
+      }
+      if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR;
+      if( Tcl_GetBooleanFromObj(interp, objv[3], &val) ) return TCL_ERROR;
+
+      sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, db, val);
+      break;
+    }
+
     case SQLITE_TESTCTRL_SORTER_MMAP: {
       int val;
       sqlite3 *db;
@@ -7567,7 +8160,7 @@ static int SQLITE_TCLAPI test_getrusage(
 */
 struct win32FileLocker {
   char *evName;       /* Name of event to signal thread startup */
-  HANDLE h;           /* Handle of the file to be locked */
+  sqlite3_file *pFd;  /* Handle of the file to be locked */
   int delay1;         /* Delay before locking */
   int delay2;         /* Delay before unlocking */
   int ok;             /* Finished ok */
@@ -7576,13 +8169,15 @@ struct win32FileLocker {
 #endif
 
 
-#if SQLITE_OS_WIN
+#ifdef _WIN32
 #include <process.h>
 /*
 ** The background thread that does file locking.
 */
 static void SQLITE_CDECL win32_file_locker(void *pAppData){
   struct win32FileLocker *p = (struct win32FileLocker*)pAppData;
+  sqlite3_file *pFd = p->pFd;
+  HANDLE h = INVALID_HANDLE_VALUE;
   if( p->evName ){
     HANDLE ev = OpenEvent(EVENT_MODIFY_STATE, FALSE, p->evName);
     if ( ev ){
@@ -7591,21 +8186,23 @@ static void SQLITE_CDECL win32_file_locker(void *pAppData){
     }
   }
   if( p->delay1 ) Sleep(p->delay1);
-  if( LockFile(p->h, 0, 0, 100000000, 0) ){
+  pFd->pMethods->xFileControl(pFd, SQLITE_FCNTL_WIN32_GET_HANDLE, (void*)&h);
+  if( LockFile(h, 0, 0, 100000000, 0) ){
     Sleep(p->delay2);
-    UnlockFile(p->h, 0, 0, 100000000, 0);
+    UnlockFile(h, 0, 0, 100000000, 0);
     p->ok = 1;
   }else{
     p->err = 1;
   }
-  CloseHandle(p->h);
-  p->h = 0;
+  pFd->pMethods->xClose(pFd);
+  sqlite3_free(pFd);
+  p->pFd = 0;
   p->delay1 = 0;
   p->delay2 = 0;
 }
 #endif
 
-#if SQLITE_OS_WIN
+#ifdef _WIN32
 /*
 **      lock_win32_file FILENAME DELAY1 DELAY2
 **
@@ -7619,37 +8216,56 @@ static int SQLITE_TCLAPI win32_file_lock(
   Tcl_Obj *CONST objv[]
 ){
   static struct win32FileLocker x = { "win32_file_lock", 0, 0, 0, 0, 0 };
-  const char *zFilename;
+  const char *zFilename = 0;
+  Tcl_Size nFilename = 0;
+  char *zTerm = 0;
   char zBuf[200];
   int retry = 0;
   HANDLE ev;
   DWORD wResult;
+  sqlite3_vfs *pVfs = 0;
+  int flags = SQLITE_OPEN_MAIN_DB | SQLITE_OPEN_READWRITE;
+  int rc = SQLITE_OK;
   
   if( objc!=4 && objc!=1 ){
     Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2");
     return TCL_ERROR;
   }
   if( objc==1 ){
+    HANDLE h = INVALID_HANDLE_VALUE;
+    if( x.pFd ){
+      x.pFd->pMethods->xFileControl(
+          x.pFd, SQLITE_FCNTL_WIN32_GET_HANDLE, (void*)&h
+      );
+    }
     sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d",
-                     x.ok, x.err, x.delay1, x.delay2, x.h);
+                     x.ok, x.err, x.delay1, x.delay2, h);
     Tcl_AppendResult(interp, zBuf, (char*)0);
     return TCL_OK;
   }
-  while( x.h && retry<30 ){
+  while( x.pFd && retry<30 ){
     retry++;
     Sleep(100);
   }
-  if( x.h ){
+  if( x.pFd ){
     Tcl_AppendResult(interp, "busy", (char*)0);
     return TCL_ERROR;
   }
   if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR;
   if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR;
-  zFilename = Tcl_GetString(objv[1]);
-  x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE,
-              FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS,
-              FILE_ATTRIBUTE_NORMAL, 0);
-  if( !x.h ){
+  pVfs = sqlite3_vfs_find(0);
+  x.pFd = (sqlite3_file*)sqlite3_malloc(pVfs->szOsFile);
+
+  /* xOpen() must be passed a dual-nul-terminated string preceded in memory
+  ** by 4 0x00 bytes.  */
+  zFilename = Tcl_GetStringFromObj(objv[1], &nFilename);
+  zTerm = (char*)sqlite3_malloc(nFilename+6);
+  memset(zTerm, 0, nFilename+6);
+  memcpy(&zTerm[4], zFilename, nFilename);
+  rc = pVfs->xOpen(pVfs, &zTerm[4], x.pFd, flags, &flags);
+  sqlite3_free(zTerm);
+
+  if( rc!=SQLITE_OK ){
     Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0);
     return TCL_ERROR;
   }
@@ -7669,145 +8285,6 @@ static int SQLITE_TCLAPI win32_file_lock(
   CloseHandle(ev);
   return TCL_OK;
 }
-
-/*
-**      exists_win32_path PATH
-**
-** Returns non-zero if the specified path exists, whose fully qualified name
-** may exceed 260 characters if it is prefixed with "\\?\".
-*/
-static int SQLITE_TCLAPI win32_exists_path(
-  void *clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "PATH");
-    return TCL_ERROR;
-  }
-  Tcl_SetObjResult(interp, Tcl_NewBooleanObj(
-      GetFileAttributesW( Tcl_GetUnicode(objv[1]))!=INVALID_FILE_ATTRIBUTES ));
-  return TCL_OK;
-}
-
-/*
-**      find_win32_file PATTERN
-**
-** Returns a list of entries in a directory that match the specified pattern,
-** whose fully qualified name may exceed 248 characters if it is prefixed with
-** "\\?\".
-*/
-static int SQLITE_TCLAPI win32_find_file(
-  void *clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  HANDLE hFindFile = INVALID_HANDLE_VALUE;
-  WIN32_FIND_DATAW findData;
-  Tcl_Obj *listObj;
-  DWORD lastErrno;
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "PATTERN");
-    return TCL_ERROR;
-  }
-  hFindFile = FindFirstFileW(Tcl_GetUnicode(objv[1]), &findData);
-  if( hFindFile==INVALID_HANDLE_VALUE ){
-    Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError()));
-    return TCL_ERROR;
-  }
-  listObj = Tcl_NewObj();
-  Tcl_IncrRefCount(listObj);
-  do {
-    Tcl_ListObjAppendElement(interp, listObj, Tcl_NewUnicodeObj(
-        findData.cFileName, -1));
-    Tcl_ListObjAppendElement(interp, listObj, Tcl_NewWideIntObj(
-        findData.dwFileAttributes));
-  } while( FindNextFileW(hFindFile, &findData) );
-  lastErrno = GetLastError();
-  if( lastErrno!=NO_ERROR && lastErrno!=ERROR_NO_MORE_FILES ){
-    FindClose(hFindFile);
-    Tcl_DecrRefCount(listObj);
-    Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError()));
-    return TCL_ERROR;
-  }
-  FindClose(hFindFile);
-  Tcl_SetObjResult(interp, listObj);
-  return TCL_OK;
-}
-
-/*
-**      delete_win32_file FILENAME
-**
-** Deletes the specified file, whose fully qualified name may exceed 260
-** characters if it is prefixed with "\\?\".
-*/
-static int SQLITE_TCLAPI win32_delete_file(
-  void *clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "FILENAME");
-    return TCL_ERROR;
-  }
-  if( !DeleteFileW(Tcl_GetUnicode(objv[1])) ){
-    Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError()));
-    return TCL_ERROR;
-  }
-  Tcl_ResetResult(interp);
-  return TCL_OK;
-}
-
-/*
-**      make_win32_dir DIRECTORY
-**
-** Creates the specified directory, whose fully qualified name may exceed 248
-** characters if it is prefixed with "\\?\".
-*/
-static int SQLITE_TCLAPI win32_mkdir(
-  void *clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DIRECTORY");
-    return TCL_ERROR;
-  }
-  if( !CreateDirectoryW(Tcl_GetUnicode(objv[1]), NULL) ){
-    Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError()));
-    return TCL_ERROR;
-  }
-  Tcl_ResetResult(interp);
-  return TCL_OK;
-}
-
-/*
-**      remove_win32_dir DIRECTORY
-**
-** Removes the specified directory, whose fully qualified name may exceed 248
-** characters if it is prefixed with "\\?\".
-*/
-static int SQLITE_TCLAPI win32_rmdir(
-  void *clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DIRECTORY");
-    return TCL_ERROR;
-  }
-  if( !RemoveDirectoryW(Tcl_GetUnicode(objv[1])) ){
-    Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError()));
-    return TCL_ERROR;
-  }
-  Tcl_ResetResult(interp);
-  return TCL_OK;
-}
 #endif
 
 
@@ -7847,6 +8324,7 @@ static int SQLITE_TCLAPI optimization_control(
     { "distinct-opt",        SQLITE_DistinctOpt    },
     { "cover-idx-scan",      SQLITE_CoverIdxScan   },
     { "order-by-idx-join",   SQLITE_OrderByIdxJoin },
+    { "order-by-subquery",   SQLITE_OrderBySubq    },
     { "transitive",          SQLITE_Transitive     },
     { "omit-noop-join",      SQLITE_OmitNoopJoin   },
     { "stat4",               SQLITE_Stat4          },
@@ -7854,6 +8332,8 @@ static int SQLITE_TCLAPI optimization_control(
     { "push-down",           SQLITE_PushDown       },
     { "balanced-merge",      SQLITE_BalancedMerge  },
     { "propagate-const",     SQLITE_PropagateConst },
+    { "one-pass",            SQLITE_OnePass        },
+    { "exists-to-join",      SQLITE_ExistsToJoin   },
   };
 
   if( objc!=4 ){
@@ -7897,7 +8377,6 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_appendvfs_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_basexx_init(sqlite3*,char**,const sqlite3_api_routines*);
-  extern int sqlite3_carray_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_csv_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -7907,15 +8386,16 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*);
-  extern int sqlite3_percentile_init(sqlite3*,char**,const sqlite3_api_routines*);
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   extern int sqlite3_prefixes_init(sqlite3*,char**,const sqlite3_api_routines*);
 #endif
   extern int sqlite3_qpvtab_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_remember_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_stmtrand_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_unionvtab_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -7929,7 +8409,6 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
     { "amatch",                sqlite3_amatch_init               },
     { "appendvfs",             sqlite3_appendvfs_init            },
     { "basexx",                sqlite3_basexx_init               },
-    { "carray",                sqlite3_carray_init               },
     { "closure",               sqlite3_closure_init              },
     { "csv",                   sqlite3_csv_init                  },
     { "decimal",               sqlite3_decimal_init              },
@@ -7939,15 +8418,16 @@ static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
     { "fuzzer",                sqlite3_fuzzer_init               },
     { "ieee754",               sqlite3_ieee_init                 },
     { "nextchar",              sqlite3_nextchar_init             },
-    { "percentile",            sqlite3_percentile_init           },
 #ifndef SQLITE_OMIT_VIRTUALTABLE
     { "prefixes",              sqlite3_prefixes_init             },
 #endif
     { "qpvtab",                sqlite3_qpvtab_init               },
+    { "randomjson",            sqlite3_randomjson_init           },
     { "regexp",                sqlite3_regexp_init               },
     { "remember",              sqlite3_remember_init             },
     { "series",                sqlite3_series_init               },
     { "spellfix",              sqlite3_spellfix_init             },
+    { "stmtrand",              sqlite3_stmtrand_init             },
     { "totype",                sqlite3_totype_init               },
     { "unionvtab",             sqlite3_unionvtab_init            },
     { "wholenumber",           sqlite3_wholenumber_init          },
@@ -8066,7 +8546,7 @@ static int SQLITE_TCLAPI sorter_test_sort4_helper(
   for(iStep=0; iStep<nStep && SQLITE_ROW==sqlite3_step(pStmt); iStep++){
     int a = sqlite3_column_int(pStmt, 0);
     if( a!=sqlite3_column_int(pStmt, iB) ){
-      Tcl_AppendResult(interp, "data error: (a!=b)", 0);
+      Tcl_AppendResult(interp, "data error: (a!=b)", (void*)0);
       return TCL_ERROR;
     }
 
@@ -8085,200 +8565,17 @@ static int SQLITE_TCLAPI sorter_test_sort4_helper(
   if( rc!=SQLITE_OK ) goto sql_error;
 
   if( iCksum1!=iCksum2 ){
-    Tcl_AppendResult(interp, "checksum mismatch", 0);
+    Tcl_AppendResult(interp, "checksum mismatch", (void*)0);
     return TCL_ERROR;
   }
 
   return TCL_OK;
  sql_error:
-  Tcl_AppendResult(interp, "sql error: ", sqlite3_errmsg(db), 0);
+  Tcl_AppendResult(interp, "sql error: ", sqlite3_errmsg(db),  (void*)0);
   return TCL_ERROR;
 }
 
 
-#ifdef SQLITE_USER_AUTHENTICATION
-#include "sqlite3userauth.h"
-/*
-** tclcmd:  sqlite3_user_authenticate DB USERNAME PASSWORD
-*/
-static int SQLITE_TCLAPI test_user_authenticate(
-  ClientData clientData, /* Unused */
-  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
-  int objc,              /* Number of arguments */
-  Tcl_Obj *CONST objv[]  /* Command arguments */
-){
-  char *zUser = 0;
-  char *zPasswd = 0;
-  int nPasswd = 0;
-  sqlite3 *db;
-  int rc;
-
-  if( objc!=4 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME PASSWORD");
-    return TCL_ERROR;
-  }
-  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
-    return TCL_ERROR;
-  }
-  zUser = Tcl_GetString(objv[2]);
-  zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd);
-  rc = sqlite3_user_authenticate(db, zUser, zPasswd, nPasswd);
-  Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC);
-  return TCL_OK;
-}
-#endif /* SQLITE_USER_AUTHENTICATION */
-
-#ifdef SQLITE_USER_AUTHENTICATION
-/*
-** tclcmd:  sqlite3_user_add DB USERNAME PASSWORD ISADMIN
-*/
-static int SQLITE_TCLAPI test_user_add(
-  ClientData clientData, /* Unused */
-  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
-  int objc,              /* Number of arguments */
-  Tcl_Obj *CONST objv[]  /* Command arguments */
-){
-  char *zUser = 0;
-  char *zPasswd = 0;
-  int nPasswd = 0;
-  int isAdmin = 0;
-  sqlite3 *db;
-  int rc;
-
-  if( objc!=5 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME PASSWORD ISADMIN");
-    return TCL_ERROR;
-  }
-  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
-    return TCL_ERROR;
-  }
-  zUser = Tcl_GetString(objv[2]);
-  zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd);
-  Tcl_GetBooleanFromObj(interp, objv[4], &isAdmin);
-  rc = sqlite3_user_add(db, zUser, zPasswd, nPasswd, isAdmin);
-  Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC);
-  return TCL_OK;
-}
-#endif /* SQLITE_USER_AUTHENTICATION */
-
-#ifdef SQLITE_USER_AUTHENTICATION
-/*
-** tclcmd:  sqlite3_user_change DB USERNAME PASSWORD ISADMIN
-*/
-static int SQLITE_TCLAPI test_user_change(
-  ClientData clientData, /* Unused */
-  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
-  int objc,              /* Number of arguments */
-  Tcl_Obj *CONST objv[]  /* Command arguments */
-){
-  char *zUser = 0;
-  char *zPasswd = 0;
-  int nPasswd = 0;
-  int isAdmin = 0;
-  sqlite3 *db;
-  int rc;
-
-  if( objc!=5 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME PASSWORD ISADMIN");
-    return TCL_ERROR;
-  }
-  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
-    return TCL_ERROR;
-  }
-  zUser = Tcl_GetString(objv[2]);
-  zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd);
-  Tcl_GetBooleanFromObj(interp, objv[4], &isAdmin);
-  rc = sqlite3_user_change(db, zUser, zPasswd, nPasswd, isAdmin);
-  Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC);
-  return TCL_OK;
-}
-#endif /* SQLITE_USER_AUTHENTICATION */
-
-#ifdef SQLITE_USER_AUTHENTICATION
-/*
-** tclcmd:  sqlite3_user_delete DB USERNAME
-*/
-static int SQLITE_TCLAPI test_user_delete(
-  ClientData clientData, /* Unused */
-  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
-  int objc,              /* Number of arguments */
-  Tcl_Obj *CONST objv[]  /* Command arguments */
-){
-  char *zUser = 0;
-  sqlite3 *db;
-  int rc;
-
-  if( objc!=3 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME");
-    return TCL_ERROR;
-  }
-  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
-    return TCL_ERROR;
-  }
-  zUser = Tcl_GetString(objv[2]);
-  rc = sqlite3_user_delete(db, zUser);
-  Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC);
-  return TCL_OK;
-}
-#endif /* SQLITE_USER_AUTHENTICATION */
-
-/*
-** tclcmd: bad_behavior TYPE
-**
-** Do some things that should trigger a valgrind or -fsanitize=undefined
-** warning.  This is used to verify that errors and warnings output by those
-** tools are detected by the test scripts.
-**
-**       TYPE       BEHAVIOR
-**       1          Overflow a signed integer
-**       2          Jump based on an uninitialized variable
-**       3          Read after free
-**       4          Panic
-*/
-static int SQLITE_TCLAPI test_bad_behavior(
-  ClientData clientData, /* Pointer to an integer containing zero */
-  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
-  int objc,              /* Number of arguments */
-  Tcl_Obj *CONST objv[]  /* Command arguments */
-){
-  int iType;
-  int xyz;
-  int i = *(int*)clientData;
-  int j;
-  int w[10];
-  int *a;
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "TYPE");
-    return TCL_ERROR;
-  }
-  if( Tcl_GetIntFromObj(interp, objv[1], &iType) ) return TCL_ERROR;
-  switch( iType ){
-    case 1: {
-      xyz = 0x7fffff00 - i;
-      xyz += 0x100;
-      Tcl_SetObjResult(interp, Tcl_NewIntObj(xyz));
-      break;
-    }
-    case 2: {
-      w[1] = 5;
-      if( w[i]>0 ) w[1]++;
-      Tcl_SetObjResult(interp, Tcl_NewIntObj(w[1]));
-      break;
-    }
-    case 3: {
-      a = malloc( sizeof(int)*10 );
-      for(j=0; j<10; j++) a[j] = j;
-      free(a);
-      Tcl_SetObjResult(interp, Tcl_NewIntObj(a[i]));
-      break;
-    }
-    case 4: {
-      Tcl_Panic("Deliberate panic");
-      break;
-    }
-  }
-  return TCL_OK;
-}  
 
 /*
 ** tclcmd:   register_dbstat_vtab DB
@@ -8343,7 +8640,12 @@ static int SQLITE_TCLAPI test_sqlite3_db_config(
     { "DQS_DML",            SQLITE_DBCONFIG_DQS_DML },
     { "DQS_DDL",            SQLITE_DBCONFIG_DQS_DDL },
     { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT },
+    { "TRUSTED_SCHEMA",     SQLITE_DBCONFIG_TRUSTED_SCHEMA },
     { "STMT_SCANSTATUS",    SQLITE_DBCONFIG_STMT_SCANSTATUS },
+    { "REVERSE_SCANORDER",  SQLITE_DBCONFIG_REVERSE_SCANORDER },
+    { "ATTACH_CREATE",      SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE },
+    { "ATTACH_WRITE",       SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE },
+    { "COMMENTS",           SQLITE_DBCONFIG_ENABLE_COMMENTS },
   };
   int i;
   int v = 0;
@@ -8472,7 +8774,7 @@ static int SQLITE_TCLAPI test_write_db(
   sqlite3 *db = 0;
   Tcl_WideInt iOff = 0;
   const unsigned char *aData = 0;
-  int nData = 0;
+  Tcl_Size nData = 0;
   sqlite3_file *pFile = 0;
   int rc;
 
@@ -8485,7 +8787,7 @@ static int SQLITE_TCLAPI test_write_db(
   aData = Tcl_GetByteArrayFromObj(objv[3], &nData);
 
   sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFile);
-  rc = pFile->pMethods->xWrite(pFile, aData, nData, iOff);
+  rc = pFile->pMethods->xWrite(pFile, aData, (int)(nData&0x7fffffff), iOff);
 
   Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
   return TCL_OK;
@@ -8551,7 +8853,6 @@ static int SQLITE_TCLAPI test_decode_hexdb(
   const char *zIn = 0;
   unsigned char *a = 0;
   int n = 0;
-  int lineno = 0;
   int i, iNext;
   int iOffset = 0;
   int j, k;
@@ -8563,7 +8864,6 @@ static int SQLITE_TCLAPI test_decode_hexdb(
   }
   zIn = Tcl_GetString(objv[1]);
   for(i=0; zIn[i]; i=iNext){
-    lineno++;
     for(iNext=i; zIn[iNext] && zIn[iNext]!='\n'; iNext++){}
     if( zIn[iNext]=='\n' ) iNext++;
     while( zIn[i]==' ' || zIn[i]=='\t' ){ i++; }
@@ -8713,7 +9013,7 @@ static int SQLITE_TCLAPI guess_number_of_cores(
   Tcl_Obj *CONST objv[]
 ){
   unsigned int nCore = 1;
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   SYSTEM_INFO sysinfo;
   GetSystemInfo(&sysinfo);
   nCore = (unsigned int)sysinfo.dwNumberOfProcessors;
@@ -8751,7 +9051,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
   extern int sqlite3_max_blobsize;
   extern int SQLITE_TCLAPI sqlite3BtreeSharedCacheReport(void*,
                                           Tcl_Interp*,int,Tcl_Obj*CONST*);
-  static int iZero = 0;
   static struct {
      char *zName;
      Tcl_CmdProc *xProc;
@@ -8795,6 +9094,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite_delete_collation",       (Tcl_CmdProc*)delete_collation      },
      { "sqlite3_get_autocommit",        (Tcl_CmdProc*)get_autocommit        },
      { "sqlite3_busy_timeout",          (Tcl_CmdProc*)test_busy_timeout     },
+     { "sqlite3_setlk_timeout",         (Tcl_CmdProc*)test_setlk_timeout    },
      { "printf",                        (Tcl_CmdProc*)test_printf           },
      { "sqlite3IoTrace",              (Tcl_CmdProc*)test_io_trace         },
      { "clang_sanitize_address",        (Tcl_CmdProc*)clang_sanitize_address },
@@ -8806,7 +9106,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
   } aObjCmd[] = {
      { "sqlite3_db_config",             test_sqlite3_db_config, 0 },
      { "sqlite3_txn_state",             test_sqlite3_txn_state, 0 },
-     { "bad_behavior",                  test_bad_behavior,  (void*)&iZero },
      { "register_dbstat_vtab",          test_register_dbstat_vtab  },
      { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
      { "intarray_addr",                 test_intarray_addr, 0 },
@@ -8826,6 +9125,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_bind_value_from_preupdate",test_bind_value_from_preupdate ,0 },
 #ifndef SQLITE_OMIT_VIRTUALTABLE
      { "sqlite3_carray_bind",           test_carray_bind   ,0 },
+     { "bind_carray_intptr",            bind_carray_intptr   ,0 },
 #endif
      { "sqlite3_bind_parameter_count",  test_bind_parameter_count, 0},
      { "sqlite3_bind_parameter_name",   test_bind_parameter_name,  0},
@@ -8837,6 +9137,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_errmsg",                test_errmsg        ,0 },
      { "sqlite3_error_offset",          test_error_offset  ,0 },
      { "sqlite3_errmsg16",              test_errmsg16      ,0 },
+     { "sqlite3_set_errmsg",            test_set_errmsg    ,0 },
      { "sqlite3_open",                  test_open          ,0 },
      { "sqlite3_open16",                test_open16        ,0 },
      { "sqlite3_open_v2",               test_open_v2       ,0 },
@@ -8864,6 +9165,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_next_stmt",             test_next_stmt     ,0 },
      { "sqlite3_stmt_readonly",         test_stmt_readonly ,0 },
      { "sqlite3_stmt_isexplain",        test_stmt_isexplain,0 },
+     { "sqlite3_stmt_explain",          test_stmt_explain  ,0 },
      { "sqlite3_stmt_busy",             test_stmt_busy     ,0 },
      { "uses_stmt_journal",             uses_stmt_journal ,0 },
 
@@ -8893,13 +9195,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "database_never_corrupt",        database_never_corrupt, 0},
      { "database_may_be_corrupt",       database_may_be_corrupt, 0},
      { "optimization_control",          optimization_control,0},
-#if SQLITE_OS_WIN
+#ifdef _WIN32
      { "lock_win32_file",               win32_file_lock,    0 },
-     { "exists_win32_path",             win32_exists_path,  0 },
-     { "find_win32_file",               win32_find_file,    0 },
-     { "delete_win32_file",             win32_delete_file,  0 },
-     { "make_win32_dir",                win32_mkdir,        0 },
-     { "remove_win32_dir",              win32_rmdir,        0 },
 #endif
      { "tcl_objproc",                   runAsObjProc,       0 },
 
@@ -8974,6 +9271,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 #endif
      { "sqlite3_test_errstr",     test_errstr, 0             },
      { "tcl_variable_type",       tcl_variable_type, 0       },
+     { "fpnum_compare",           fpnum_compare, 0           },
 #ifndef SQLITE_OMIT_SHARED_CACHE
      { "sqlite3_enable_shared_cache", test_enable_shared, 0  },
      { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0},
@@ -8994,19 +9292,15 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 #ifndef SQLITE_OMIT_EXPLAIN
      { "print_explain_query_plan", test_print_eqp, 0  },
 #endif
+     { "strftime",             strftime_cmd      },
      { "sqlite3_test_control", test_test_control },
+     { ".treetrace",           test_treetrace    },
 #if SQLITE_OS_UNIX
      { "getrusage", test_getrusage },
 #endif
      { "load_static_extension", tclLoadStaticExtensionCmd },
      { "sorter_test_fakeheap", sorter_test_fakeheap },
      { "sorter_test_sort4_helper", sorter_test_sort4_helper },
-#ifdef SQLITE_USER_AUTHENTICATION
-     { "sqlite3_user_authenticate", test_user_authenticate, 0 },
-     { "sqlite3_user_add",          test_user_add,          0 },
-     { "sqlite3_user_change",       test_user_change,       0 },
-     { "sqlite3_user_delete",       test_user_delete,       0 },
-#endif
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
      { "sqlite3_stmt_scanstatus",       test_stmt_scanstatus,   0 },
      { "sqlite3_stmt_scanstatus_reset", test_stmt_scanstatus_reset,   0 },
@@ -9040,7 +9334,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 #endif
   };
   static int bitmask_size = sizeof(Bitmask)*8;
-  static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
   int i;
   extern int sqlite3_sync_count, sqlite3_fullsync_count;
   extern int sqlite3_opentemp_count;
@@ -9050,6 +9343,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
   extern int sqlite3_pager_writedb_count;
   extern int sqlite3_pager_writej_count;
 #if SQLITE_OS_WIN
+  extern int sqlite3_win_test_unc_locking;
   extern LONG volatile sqlite3_os_type;
 #endif
 #ifdef SQLITE_DEBUG
@@ -9109,6 +9403,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 #if SQLITE_OS_WIN
   Tcl_LinkVar(interp, "sqlite_os_type",
       (char*)&sqlite3_os_type, TCL_LINK_LONG);
+  Tcl_LinkVar(interp, "sqlite3_win_test_unc_locking",
+      (char*)&sqlite3_win_test_unc_locking, TCL_LINK_INT);
 #endif
 #ifdef SQLITE_TEST
   {
@@ -9141,8 +9437,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
       (char*)&sqlite3_data_directory, TCL_LINK_STRING);
   Tcl_LinkVar(interp, "bitmask_size",
       (char*)&bitmask_size, TCL_LINK_INT|TCL_LINK_READ_ONLY);
-  Tcl_LinkVar(interp, "longdouble_size",
-      (char*)&longdouble_size, TCL_LINK_INT|TCL_LINK_READ_ONLY);
   Tcl_LinkVar(interp, "sqlite_sync_count",
       (char*)&sqlite3_sync_count, TCL_LINK_INT);
   Tcl_LinkVar(interp, "sqlite_fullsync_count",
diff --git a/src/test2.c b/src/test2.c
index d5db3867b8..899728ead9 100644
--- a/src/test2.c
+++ b/src/test2.c
@@ -14,11 +14,7 @@
 ** testing of the SQLite library.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
@@ -55,7 +51,7 @@ static int SQLITE_TCLAPI pager_open(
   char zBuf[100];
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " FILENAME N-PAGE\"", 0);
+       " FILENAME N-PAGE\"", NULL);
     return TCL_ERROR;
   }
   if( Tcl_GetInt(interp, argv[2], &nPage) ) return TCL_ERROR;
@@ -63,14 +59,14 @@ static int SQLITE_TCLAPI pager_open(
       SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB,
       pager_test_reiniter);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   sqlite3PagerSetCachesize(pPager, nPage);
   pageSize = test_pagesize;
   sqlite3PagerSetPagesize(pPager, &pageSize, -1);
   sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPager);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -89,13 +85,13 @@ static int SQLITE_TCLAPI pager_close(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerClose(pPager, 0);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -116,13 +112,13 @@ static int SQLITE_TCLAPI pager_rollback(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerRollback(pPager);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -143,18 +139,18 @@ static int SQLITE_TCLAPI pager_commit(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   rc = sqlite3PagerCommitPhaseTwo(pPager);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -175,13 +171,13 @@ static int SQLITE_TCLAPI pager_stmt_begin(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerOpenSavepoint(pPager, 1);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -202,14 +198,14 @@ static int SQLITE_TCLAPI pager_stmt_rollback(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0);
   sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -230,13 +226,13 @@ static int SQLITE_TCLAPI pager_stmt_commit(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -257,7 +253,7 @@ static int SQLITE_TCLAPI pager_stats(
   int i, *a;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
@@ -291,13 +287,13 @@ static int SQLITE_TCLAPI pager_pagecount(
   int nPage;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
   sqlite3PagerPagecount(pPager, &nPage);
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", nPage);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -319,7 +315,7 @@ static int SQLITE_TCLAPI page_get(
   int rc;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID PGNO\"", 0);
+       " ID PGNO\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
@@ -329,11 +325,11 @@ static int SQLITE_TCLAPI page_get(
     rc = sqlite3PagerGet(pPager, pgno, &pPage, 0);
   }
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -355,7 +351,7 @@ static int SQLITE_TCLAPI page_lookup(
   int pgno;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID PGNO\"", 0);
+       " ID PGNO\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
@@ -363,7 +359,7 @@ static int SQLITE_TCLAPI page_lookup(
   pPage = sqlite3PagerLookup(pPager, pgno);
   if( pPage ){
     sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage);
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
@@ -381,7 +377,7 @@ static int SQLITE_TCLAPI pager_truncate(
   int pgno;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID PGNO\"", 0);
+       " ID PGNO\"", NULL);
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
@@ -405,7 +401,7 @@ static int SQLITE_TCLAPI page_unref(
   DbPage *pPage;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " PAGE\"", 0);
+       " PAGE\"", NULL);
     return TCL_ERROR;
   }
   pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]);
@@ -428,12 +424,12 @@ static int SQLITE_TCLAPI page_read(
   DbPage *pPage;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " PAGE\"", 0);
+       " PAGE\"", NULL);
     return TCL_ERROR;
   }
   pPage = sqlite3TestTextToPtr(argv[1]);
   memcpy(zBuf, sqlite3PagerGetData(pPage), sizeof(zBuf));
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -452,12 +448,12 @@ static int SQLITE_TCLAPI page_number(
   DbPage *pPage;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " PAGE\"", 0);
+       " PAGE\"", NULL);
     return TCL_ERROR;
   }
   pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]);
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", sqlite3PagerPagenumber(pPage));
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -477,13 +473,13 @@ static int SQLITE_TCLAPI page_write(
   int rc;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " PAGE DATA\"", 0);
+       " PAGE DATA\"", NULL);
     return TCL_ERROR;
   }
   pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3PagerWrite(pPage);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   pData = sqlite3PagerGetData(pPage);
@@ -517,7 +513,7 @@ static int SQLITE_TCLAPI fake_big_file(
   int nFile;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " N-MEGABYTES FILE\"", 0);
+       " N-MEGABYTES FILE\"", NULL);
     return TCL_ERROR;
   }
   if( Tcl_GetInt(interp, argv[1], &n) ) return TCL_ERROR;
@@ -540,7 +536,7 @@ static int SQLITE_TCLAPI fake_big_file(
       (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0
   );
   if( rc ){
-    Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), NULL);
     sqlite3_free(zFile);
     return TCL_ERROR;
   }
@@ -550,7 +546,7 @@ static int SQLITE_TCLAPI fake_big_file(
   sqlite3OsCloseFree(fd);
   sqlite3_free(zFile);
   if( rc ){
-    Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -608,7 +604,7 @@ static int faultSimCallback(int x){
       zInt[i] = (x%10) + '0';
     }
     if( isNeg ) zInt[i--] = '-';
-    memcpy(faultSimScript+faultSimScriptSize, zInt+i+1, sizeof(zInt)-i);
+    memcpy(faultSimScript+faultSimScriptSize, zInt+i+1, sizeof(zInt)-i-1);
   }
   rc = Tcl_Eval(faultSimInterp, faultSimScript);
   if( rc ){
diff --git a/src/test3.c b/src/test3.c
index d1626b6ef4..8fbb96a80d 100644
--- a/src/test3.c
+++ b/src/test3.c
@@ -15,11 +15,7 @@
 */
 #include "sqliteInt.h"
 #include "btreeInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -50,9 +46,10 @@ static int SQLITE_TCLAPI btree_open(
   char *zFilename;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " FILENAME NCACHE FLAGS\"", 0);
+       " FILENAME NCACHE FLAGS\"", NULL);
     return TCL_ERROR;
   }
+
   if( Tcl_GetInt(interp, argv[2], &nCache) ) return TCL_ERROR;
   nRefSqlite3++;
   if( nRefSqlite3==1 ){
@@ -69,12 +66,12 @@ static int SQLITE_TCLAPI btree_open(
      SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB);
   sqlite3_free(zFilename);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   sqlite3BtreeSetCacheSize(pBt, nCache);
   sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pBt);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -93,13 +90,13 @@ static int SQLITE_TCLAPI btree_close(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pBt = sqlite3TestTextToPtr(argv[1]);
   rc = sqlite3BtreeClose(pBt);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   nRefSqlite3--;
@@ -128,7 +125,7 @@ static int SQLITE_TCLAPI btree_begin_transaction(
   int rc;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pBt = sqlite3TestTextToPtr(argv[1]);
@@ -136,7 +133,7 @@ static int SQLITE_TCLAPI btree_begin_transaction(
   rc = sqlite3BtreeBeginTrans(pBt, 1, 0);
   sqlite3BtreeLeave(pBt);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
@@ -159,7 +156,7 @@ static int SQLITE_TCLAPI btree_pager_stats(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pBt = sqlite3TestTextToPtr(argv[1]);
@@ -212,7 +209,7 @@ static int SQLITE_TCLAPI btree_cursor(
 
   if( argc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID TABLENUM WRITEABLE\"", 0);
+       " ID TABLENUM WRITEABLE\"", NULL);
     return TCL_ERROR;
   }
   pBt = sqlite3TestTextToPtr(argv[1]);
@@ -233,11 +230,11 @@ static int SQLITE_TCLAPI btree_cursor(
   sqlite3_mutex_leave(pBt->db->mutex);
   if( rc ){
     ckfree((char *)pCur);
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return SQLITE_OK;
 }
 
@@ -257,7 +254,7 @@ static int SQLITE_TCLAPI btree_close_cursor(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
@@ -275,7 +272,7 @@ static int SQLITE_TCLAPI btree_close_cursor(
 #endif
   ckfree((char *)pCur);
   if( rc ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return SQLITE_OK;
@@ -301,7 +298,7 @@ static int SQLITE_TCLAPI btree_next(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
@@ -313,11 +310,11 @@ static int SQLITE_TCLAPI btree_next(
   }
   sqlite3BtreeLeave(pCur->pBtree);
   if( rc ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return SQLITE_OK;
 }
 
@@ -340,7 +337,7 @@ static int SQLITE_TCLAPI btree_first(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
@@ -348,11 +345,11 @@ static int SQLITE_TCLAPI btree_first(
   rc = sqlite3BtreeFirst(pCur, &res);
   sqlite3BtreeLeave(pCur->pBtree);
   if( rc ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return SQLITE_OK;
 }
 
@@ -374,7 +371,7 @@ static int SQLITE_TCLAPI btree_eof(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
@@ -382,7 +379,7 @@ static int SQLITE_TCLAPI btree_eof(
   rc = sqlite3BtreeEof(pCur);
   sqlite3BtreeLeave(pCur->pBtree);
   sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", rc);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return SQLITE_OK;
 }
 
@@ -403,7 +400,7 @@ static int SQLITE_TCLAPI btree_payload_size(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
@@ -411,7 +408,7 @@ static int SQLITE_TCLAPI btree_payload_size(
   n = sqlite3BtreePayloadSize(pCur);
   sqlite3BtreeLeave(pCur->pBtree);
   sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return SQLITE_OK;
 }
 
@@ -422,7 +419,7 @@ static int SQLITE_TCLAPI btree_payload_size(
 ** routines, both for accuracy and for speed.
 **
 ** An integer is written using putVarint() and read back with
-** getVarint() and varified to be unchanged.  This repeats COUNT
+** getVarint() and verified to be unchanged.  This repeats COUNT
 ** times.  The first integer is START*MULTIPLIER.  Each iteration
 ** increases the integer by INCREMENT.
 **
@@ -441,7 +438,7 @@ static int SQLITE_TCLAPI btree_varint_test(
   unsigned char zBuf[100];
   if( argc!=5 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " START MULTIPLIER COUNT INCREMENT\"", 0);
+       " START MULTIPLIER COUNT INCREMENT\"", NULL);
     return TCL_ERROR;
   }
   if( Tcl_GetInt(interp, argv[1], (int*)&start) ) return TCL_ERROR;
@@ -456,20 +453,20 @@ static int SQLITE_TCLAPI btree_varint_test(
     if( n1>9 || n1<1 ){
       sqlite3_snprintf(sizeof(zErr), zErr,
          "putVarint returned %d - should be between 1 and 9", n1);
-      Tcl_AppendResult(interp, zErr, 0);
+      Tcl_AppendResult(interp, zErr, NULL);
       return TCL_ERROR;
     }
     n2 = getVarint(zBuf, &out);
     if( n1!=n2 ){
       sqlite3_snprintf(sizeof(zErr), zErr,
           "putVarint returned %d and getVarint returned %d", n1, n2);
-      Tcl_AppendResult(interp, zErr, 0);
+      Tcl_AppendResult(interp, zErr, NULL);
       return TCL_ERROR;
     }
     if( in!=out ){
       sqlite3_snprintf(sizeof(zErr), zErr,
           "Wrote 0x%016llx and got back 0x%016llx", in, out);
-      Tcl_AppendResult(interp, zErr, 0);
+      Tcl_AppendResult(interp, zErr, NULL);
       return TCL_ERROR;
     }
     if( (in & 0xffffffff)==in ){
@@ -480,14 +477,14 @@ static int SQLITE_TCLAPI btree_varint_test(
         sqlite3_snprintf(sizeof(zErr), zErr,
           "putVarint returned %d and GetVarint32 returned %d", 
                   n1, n2);
-        Tcl_AppendResult(interp, zErr, 0);
+        Tcl_AppendResult(interp, zErr, NULL);
         return TCL_ERROR;
       }
       if( in!=out ){
         sqlite3_snprintf(sizeof(zErr), zErr,
           "Wrote 0x%016llx and got back 0x%016llx from GetVarint32",
             in, out);
-        Tcl_AppendResult(interp, zErr, 0);
+        Tcl_AppendResult(interp, zErr, NULL);
         return TCL_ERROR;
       }
     }
@@ -527,12 +524,12 @@ static int SQLITE_TCLAPI btree_from_db(
 
   if( argc!=2 && argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " DB-HANDLE ?N?\"", 0);
+       " DB-HANDLE ?N?\"", NULL);
     return TCL_ERROR;
   }
 
   if( 1!=Tcl_GetCommandInfo(interp, argv[1], &info) ){
-    Tcl_AppendResult(interp, "No such db-handle: \"", argv[1], "\"", 0);
+    Tcl_AppendResult(interp, "No such db-handle: \"", argv[1], "\"", NULL);
     return TCL_ERROR;
   }
   if( argc==3 ){
@@ -565,7 +562,7 @@ static int SQLITE_TCLAPI btree_ismemdb(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID\"", 0);
+       " ID\"", NULL);
     return TCL_ERROR;
   }
   pBt = sqlite3TestTextToPtr(argv[1]);
@@ -595,7 +592,7 @@ static int SQLITE_TCLAPI btree_set_cache_size(
   
   if( argc!=3 ){
     Tcl_AppendResult(
-        interp, "wrong # args: should be \"", argv[0], " BT NCACHE\"", 0);
+        interp, "wrong # args: should be \"", argv[0], " BT NCACHE\"", NULL);
     return TCL_ERROR;
   }
   pBt = sqlite3TestTextToPtr(argv[1]);
@@ -623,6 +620,7 @@ static int SQLITE_TCLAPI btree_insert(
   BtCursor *pCur;
   int rc;
   BtreePayload x;
+  Tcl_Size n;
 
   if( objc!=4 && objc!=3 ){
     Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE");
@@ -633,10 +631,11 @@ static int SQLITE_TCLAPI btree_insert(
   if( objc==4 ){
     if( Tcl_GetIntFromObj(interp, objv[2], &rc) ) return TCL_ERROR;
     x.nKey = rc;
-    x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &x.nData);
+    x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &n);
+    x.nData = (int)n;
   }else{
-    x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &rc);
-    x.nKey = rc;
+    x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &n);
+    x.nKey = (int)n;
   }
   pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
 
@@ -648,7 +647,7 @@ static int SQLITE_TCLAPI btree_insert(
 
   Tcl_ResetResult(interp);
   if( rc ){
-    Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL);
     return TCL_ERROR;
   }
   return TCL_OK;
diff --git a/src/test4.c b/src/test4.c
index 2043a33830..07236b3e5d 100644
--- a/src/test4.c
+++ b/src/test4.c
@@ -12,11 +12,7 @@
 ** Code for testing the SQLite library in a multithreaded environment.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #if SQLITE_OS_UNIX && SQLITE_THREADSAFE
 #include <stdlib.h>
 #include <string.h>
@@ -123,7 +119,7 @@ static void *test_thread_main(void *pArg){
 */
 static int parse_thread_id(Tcl_Interp *interp, const char *zArg){
   if( zArg==0 || zArg[0]==0 || zArg[1]!=0 || !isupper((unsigned char)zArg[0]) ){
-    Tcl_AppendResult(interp, "thread ID must be an upper case letter", 0);
+    Tcl_AppendResult(interp, "thread ID must be an upper case letter", NULL);
     return -1;
   }
   return zArg[0] - 'A';
@@ -147,13 +143,13 @@ static int SQLITE_TCLAPI tcl_thread_create(
 
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID FILENAME", 0);
+       " ID FILENAME", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( threadset[i].busy ){
-    Tcl_AppendResult(interp, "thread ", argv[1], " is already running", 0);
+    Tcl_AppendResult(interp, "thread ", argv[1], " is already running", NULL);
     return TCL_ERROR;
   }
   threadset[i].busy = 1;
@@ -163,7 +159,7 @@ static int SQLITE_TCLAPI tcl_thread_create(
   threadset[i].completed = 0;
   rc = pthread_create(&x, 0, test_thread_main, &threadset[i]);
   if( rc ){
-    Tcl_AppendResult(interp, "failed to create the thread", 0);
+    Tcl_AppendResult(interp, "failed to create the thread", NULL);
     sqlite3_free(threadset[i].zFilename);
     threadset[i].busy = 0;
     return TCL_ERROR;
@@ -196,13 +192,13 @@ static int SQLITE_TCLAPI tcl_thread_wait(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
@@ -240,7 +236,7 @@ static int SQLITE_TCLAPI tcl_thread_halt(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   if( argv[1][0]=='*' && argv[1][1]==0 ){
@@ -251,7 +247,7 @@ static int SQLITE_TCLAPI tcl_thread_halt(
     i = parse_thread_id(interp, argv[1]);
     if( i<0 ) return TCL_ERROR;
     if( !threadset[i].busy ){
-      Tcl_AppendResult(interp, "no such thread", 0);
+      Tcl_AppendResult(interp, "no such thread", NULL);
       return TCL_ERROR;
     }
     test_stop_thread(&threadset[i]);
@@ -276,18 +272,18 @@ static int SQLITE_TCLAPI tcl_thread_argc(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
   sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", threadset[i].argc);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   return TCL_OK;
 }
 
@@ -308,22 +304,22 @@ static int SQLITE_TCLAPI tcl_thread_argv(
 
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID N", 0);
+       " ID N", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   if( Tcl_GetInt(interp, argv[2], &n) ) return TCL_ERROR;
   test_thread_wait(&threadset[i]);
   if( n<0 || n>=threadset[i].argc ){
-    Tcl_AppendResult(interp, "column number out of range", 0);
+    Tcl_AppendResult(interp, "column number out of range", NULL);
     return TCL_ERROR;
   }
-  Tcl_AppendResult(interp, threadset[i].argv[n], 0);
+  Tcl_AppendResult(interp, threadset[i].argv[n], NULL);
   return TCL_OK;
 }
 
@@ -344,22 +340,22 @@ static int SQLITE_TCLAPI tcl_thread_colname(
 
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID N", 0);
+       " ID N", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   if( Tcl_GetInt(interp, argv[2], &n) ) return TCL_ERROR;
   test_thread_wait(&threadset[i]);
   if( n<0 || n>=threadset[i].argc ){
-    Tcl_AppendResult(interp, "column number out of range", 0);
+    Tcl_AppendResult(interp, "column number out of range", NULL);
     return TCL_ERROR;
   }
-  Tcl_AppendResult(interp, threadset[i].colv[n], 0);
+  Tcl_AppendResult(interp, threadset[i].colv[n], NULL);
   return TCL_OK;
 }
 
@@ -380,18 +376,18 @@ static int SQLITE_TCLAPI tcl_thread_result(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
   zName = sqlite3ErrName(threadset[i].rc);
-  Tcl_AppendResult(interp, zName, 0);
+  Tcl_AppendResult(interp, zName, NULL);
   return TCL_OK;
 }
 
@@ -411,17 +407,17 @@ static int SQLITE_TCLAPI tcl_thread_error(
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
-  Tcl_AppendResult(interp, threadset[i].zErr, 0);
+  Tcl_AppendResult(interp, threadset[i].zErr, NULL);
   return TCL_OK;
 }
 
@@ -455,13 +451,13 @@ static int SQLITE_TCLAPI tcl_thread_compile(
   int i;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID SQL", 0);
+       " ID SQL", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
@@ -509,13 +505,13 @@ static int SQLITE_TCLAPI tcl_thread_step(
   int i;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " IDL", 0);
+       " IDL", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
@@ -552,13 +548,13 @@ static int SQLITE_TCLAPI tcl_thread_finalize(
   int i;
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " IDL", 0);
+       " IDL", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
@@ -585,20 +581,20 @@ static int SQLITE_TCLAPI tcl_thread_swap(
   sqlite3 *temp;
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID1 ID2", 0);
+       " ID1 ID2", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
   j = parse_thread_id(interp, argv[2]);
   if( j<0 ) return TCL_ERROR;
   if( !threadset[j].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[j]);
@@ -626,13 +622,13 @@ static int SQLITE_TCLAPI tcl_thread_db_get(
   extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*);
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
@@ -657,13 +653,13 @@ static int SQLITE_TCLAPI tcl_thread_db_put(
   extern void *sqlite3TestTextToPtr(const char *);
   if( argc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID DB", 0);
+       " ID DB", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
@@ -689,13 +685,13 @@ static int SQLITE_TCLAPI tcl_thread_stmt_get(
   extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*);
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
-       " ID", 0);
+       " ID", NULL);
     return TCL_ERROR;
   }
   i = parse_thread_id(interp, argv[1]);
   if( i<0 ) return TCL_ERROR;
   if( !threadset[i].busy ){
-    Tcl_AppendResult(interp, "no such thread", 0);
+    Tcl_AppendResult(interp, "no such thread", NULL);
     return TCL_ERROR;
   }
   test_thread_wait(&threadset[i]);
diff --git a/src/test5.c b/src/test5.c
index 0d9242862b..06d2de911d 100644
--- a/src/test5.c
+++ b/src/test5.c
@@ -17,11 +17,7 @@
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -36,7 +32,7 @@ static int SQLITE_TCLAPI binarize(
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int len;
+  Tcl_Size len;
   char *bytes;
   Tcl_Obj *pRet;
   assert(objc==2);
@@ -71,7 +67,7 @@ static int SQLITE_TCLAPI test_value_overhead(
 
   if( objc!=3 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
-        Tcl_GetStringFromObj(objv[0], 0), " <repeat-count> <do-calls>", 0);
+        Tcl_GetStringFromObj(objv[0], 0), " <repeat-count> <do-calls>", NULL);
     return TCL_ERROR;
   }
 
@@ -110,7 +106,7 @@ static u8 name_to_enc(Tcl_Interp *interp, Tcl_Obj *pObj){
     }
   }
   if( !pEnc->enc ){
-    Tcl_AppendResult(interp, "No such encoding: ", z, 0);
+    Tcl_AppendResult(interp, "No such encoding: ", z, NULL);
   }
   if( pEnc->enc==SQLITE_UTF16 ){
     return SQLITE_UTF16NATIVE;
@@ -133,13 +129,13 @@ static int SQLITE_TCLAPI test_translate(
   sqlite3_value *pVal;
 
   char *z;
-  int len;
+  Tcl_Size len;
   void (*xDel)(void *p) = SQLITE_STATIC;
 
   if( objc!=4 && objc!=5 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"",
         Tcl_GetStringFromObj(objv[0], 0), 
-        " <string/blob> <from enc> <to enc>", 0
+        " <string/blob> <from enc> <to enc>", NULL
     );
     return TCL_ERROR;
   }
@@ -164,7 +160,7 @@ static int SQLITE_TCLAPI test_translate(
     z = (char*)Tcl_GetByteArrayFromObj(objv[1], &len);
     if( objc==5 ){
       char *zTmp = z;
-      z = sqlite3_malloc(len);
+      z = sqlite3_malloc64(len);
       memcpy(z, zTmp, len);
     }
     sqlite3ValueSetStr(pVal, -1, z, enc_from, xDel);
diff --git a/src/test6.c b/src/test6.c
index aa94b9b5ea..aee7bf12aa 100644
--- a/src/test6.c
+++ b/src/test6.c
@@ -16,11 +16,7 @@
 */
 #if SQLITE_TEST          /* This file is used for testing only */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 #ifndef SQLITE_OMIT_DISKIO  /* This file is a no-op if disk I/O is disabled */
 
@@ -33,25 +29,25 @@ typedef struct WriteBuffer WriteBuffer;
 /*
 ** Method:
 **
-**   This layer is implemented as a wrapper around the "real" 
-**   sqlite3_file object for the host system. Each time data is 
+**   This layer is implemented as a wrapper around the "real"
+**   sqlite3_file object for the host system. Each time data is
 **   written to the file object, instead of being written to the
-**   underlying file, the write operation is stored in an in-memory 
+**   underlying file, the write operation is stored in an in-memory
 **   structure (type WriteBuffer). This structure is placed at the
 **   end of a global ordered list (the write-list).
 **
 **   When data is read from a file object, the requested region is
-**   first retrieved from the real file. The write-list is then 
-**   traversed and data copied from any overlapping WriteBuffer 
+**   first retrieved from the real file. The write-list is then
+**   traversed and data copied from any overlapping WriteBuffer
 **   structures to the output buffer. i.e. a read() operation following
 **   one or more write() operations works as expected, even if no
 **   data has actually been written out to the real file.
 **
-**   When a fsync() operation is performed, an operating system crash 
-**   may be simulated, in which case exit(-1) is called (the call to 
+**   When a fsync() operation is performed, an operating system crash
+**   may be simulated, in which case exit(-1) is called (the call to
 **   xSync() never returns). Whether or not a crash is simulated,
-**   the data associated with a subset of the WriteBuffer structures 
-**   stored in the write-list is written to the real underlying files 
+**   the data associated with a subset of the WriteBuffer structures
+**   stored in the write-list is written to the real underlying files
 **   and the entries removed from the write-list. If a crash is simulated,
 **   a subset of the buffers may be corrupted before the data is written.
 **
@@ -63,13 +59,13 @@ typedef struct WriteBuffer WriteBuffer;
 **   Normal mode is used when the simulated device has none of the
 **   SQLITE_IOCAP_XXX flags set.
 **
-**   In normal mode, if the fsync() is not a simulated crash, the 
+**   In normal mode, if the fsync() is not a simulated crash, the
 **   write-list is traversed from beginning to end. Each WriteBuffer
 **   structure associated with the file handle used to call xSync()
 **   is written to the real file and removed from the write-list.
 **
-**   If a crash is simulated, one of the following takes place for 
-**   each WriteBuffer in the write-list, regardless of which 
+**   If a crash is simulated, one of the following takes place for
+**   each WriteBuffer in the write-list, regardless of which
 **   file-handle it is associated with:
 **
 **     1. The buffer is correctly written to the file, just as if
@@ -77,14 +73,14 @@ typedef struct WriteBuffer WriteBuffer;
 **
 **     2. Nothing is done.
 **
-**     3. Garbage data is written to all sectors of the file that 
+**     3. Garbage data is written to all sectors of the file that
 **        overlap the region specified by the WriteBuffer. Or garbage
-**        data is written to some contiguous section within the 
+**        data is written to some contiguous section within the
 **        overlapped sectors.
 **
 ** Device Characteristic flag handling:
 **
-**   If the IOCAP_ATOMIC flag is set, then option (3) above is 
+**   If the IOCAP_ATOMIC flag is set, then option (3) above is
 **   never selected.
 **
 **   If the IOCAP_ATOMIC512 flag is set, and the WriteBuffer represents
@@ -96,11 +92,11 @@ typedef struct WriteBuffer WriteBuffer;
 **
 **   If either the IOCAP_SAFEAPPEND or IOCAP_SEQUENTIAL flags are set
 **   and a crash is being simulated, then an entry of the write-list is
-**   selected at random. Everything in the list after the selected entry 
+**   selected at random. Everything in the list after the selected entry
 **   is discarded before processing begins.
 **
-**   If IOCAP_SEQUENTIAL is set and a crash is being simulated, option 
-**   (1) is selected for all write-list entries except the last. If a 
+**   If IOCAP_SEQUENTIAL is set and a crash is being simulated, option
+**   (1) is selected for all write-list entries except the last. If a
 **   crash is not being simulated, then all entries in the write-list
 **   that occur before at least one write() on the file-handle specified
 **   as part of the xSync() are written to their associated real files.
@@ -114,7 +110,7 @@ typedef struct WriteBuffer WriteBuffer;
 ** Each write operation in the write-list is represented by an instance
 ** of the following structure.
 **
-** If zBuf is 0, then this structure represents a call to xTruncate(), 
+** If zBuf is 0, then this structure represents a call to xTruncate(),
 ** not xWrite(). In that case, iOffset is the size that the file is
 ** truncated to.
 */
@@ -133,7 +129,7 @@ struct CrashFile {
   char *zName;
   int flags;                           /* Flags the file was opened with */
 
-  /* Cache of the entire file. This is used to speed up OsRead() and 
+  /* Cache of the entire file. This is used to speed up OsRead() and
   ** OsFileSize() calls. Although both could be done by traversing the
   ** write-list, in practice this is impractically slow.
   */
@@ -150,7 +146,7 @@ struct CrashGlobal {
   int iDeviceCharacteristics;  /* Value of simulated device characteristics */
 
   int iCrash;                  /* Crash on the iCrash'th call to xSync() */
-  char zCrashFile[500];        /* Crash during an xSync() on this file */ 
+  char zCrashFile[500];        /* Crash during an xSync() on this file */
 };
 
 static CrashGlobal g = {0, 0, SQLITE_DEFAULT_SECTOR_SIZE, 0, 0};
@@ -172,7 +168,7 @@ static void *crash_realloc(void *p, int n){
 
 /*
 ** Wrapper around the sqlite3OsWrite() function that avoids writing to the
-** 512 byte block begining at offset PENDING_BYTE.
+** 512 byte block beginning at offset PENDING_BYTE.
 */
 static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){
   int rc = SQLITE_OK;
@@ -194,7 +190,7 @@ static int writeListSync(CrashFile *pFile, int isCrash){
   WriteBuffer *pWrite;
   WriteBuffer **ppPtr;
 
-  /* If this is not a crash simulation, set pFinal to point to the 
+  /* If this is not a crash simulation, set pFinal to point to the
   ** last element of the write-list that is associated with file handle
   ** pFile.
   **
@@ -242,7 +238,7 @@ static int writeListSync(CrashFile *pFile, int isCrash){
       char random;
       sqlite3_randomness(1, &random);
 
-      /* Do not select option 3 (sector trashing) if the IOCAP_ATOMIC flag 
+      /* Do not select option 3 (sector trashing) if the IOCAP_ATOMIC flag
       ** is set or this is an OsTruncate(), not an Oswrite().
       */
       if( (iDc&SQLITE_IOCAP_ATOMIC) || (pWrite->zBuf==0) ){
@@ -288,7 +284,7 @@ static int writeListSync(CrashFile *pFile, int isCrash){
         *ppPtr = pWrite->pNext;
 #ifdef TRACE_CRASHTEST
         if( isCrash ){
-          printf("Writing %d bytes @ %d (%s)\n", 
+          printf("Writing %d bytes @ %d (%s)\n",
             pWrite->nBuf, (int)pWrite->iOffset, pWrite->pFile->zName
           );
         }
@@ -300,7 +296,7 @@ static int writeListSync(CrashFile *pFile, int isCrash){
         ppPtr = &pWrite->pNext;
 #ifdef TRACE_CRASHTEST
         if( isCrash ){
-          printf("Omiting %d bytes @ %d (%s)\n", 
+          printf("Omiting %d bytes @ %d (%s)\n",
             pWrite->nBuf, (int)pWrite->iOffset, pWrite->pFile->zName
           );
         }
@@ -315,7 +311,7 @@ static int writeListSync(CrashFile *pFile, int isCrash){
         assert(pWrite->zBuf);
 
 #ifdef TRACE_CRASHTEST
-        printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", 
+        printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n",
             1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize,
             pWrite->iOffset, iFirst, pWrite->pFile->zName
         );
@@ -325,7 +321,7 @@ static int writeListSync(CrashFile *pFile, int isCrash){
         if( zGarbage ){
           sqlite3_int64 i;
           for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){
-            sqlite3_randomness(g.iSectorSize, zGarbage); 
+            sqlite3_randomness(g.iSectorSize, zGarbage);
             rc = writeDbFile(
               pWrite->pFile, zGarbage, g.iSectorSize, i*g.iSectorSize
             );
@@ -389,7 +385,7 @@ static int writeListAppend(
     g.pWriteList = pNew;
   }
   g.pWriteListEnd = pNew;
-  
+ 
   return SQLITE_OK;
 }
 
@@ -407,9 +403,9 @@ static int cfClose(sqlite3_file *pFile){
 ** Read data from a crash-file.
 */
 static int cfRead(
-  sqlite3_file *pFile, 
-  void *zBuf, 
-  int iAmt, 
+  sqlite3_file *pFile,
+  void *zBuf,
+  int iAmt,
   sqlite_int64 iOfst
 ){
   CrashFile *pCrash = (CrashFile *)pFile;
@@ -431,9 +427,9 @@ static int cfRead(
 ** Write data to a crash-file.
 */
 static int cfWrite(
-  sqlite3_file *pFile, 
-  const void *zBuf, 
-  int iAmt, 
+  sqlite3_file *pFile,
+  const void *zBuf,
+  int iAmt,
   sqlite_int64 iOfst
 ){
   CrashFile *pCrash = (CrashFile *)pFile;
@@ -604,7 +600,7 @@ struct crashAppData {
 **
 ** The caller will have allocated pVfs->szOsFile bytes of space
 ** at pFile. This file uses this space for the CrashFile structure
-** and allocates space for the "real" file structure using 
+** and allocates space for the "real" file structure using
 ** sqlite3_malloc(). The assumption here is (pVfs->szOsFile) is
 ** equal or greater than sizeof(CrashFile).
 */
@@ -667,17 +663,17 @@ static int cfDelete(sqlite3_vfs *pCfVfs, const char *zPath, int dirSync){
   return pVfs->xDelete(pVfs, zPath, dirSync);
 }
 static int cfAccess(
-  sqlite3_vfs *pCfVfs, 
-  const char *zPath, 
-  int flags, 
+  sqlite3_vfs *pCfVfs,
+  const char *zPath,
+  int flags,
   int *pResOut
 ){
   sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
   return pVfs->xAccess(pVfs, zPath, flags, pResOut);
 }
 static int cfFullPathname(
-  sqlite3_vfs *pCfVfs, 
-  const char *zPath, 
+  sqlite3_vfs *pCfVfs,
+  const char *zPath,
   int nPathOut,
   char *zPathOut
 ){
@@ -751,20 +747,20 @@ static int processDevSymArgs(
   int setDeviceChar = 0;
 
   for(i=0; i<objc; i+=2){
-    int nOpt;
+    Tcl_Size nOpt;
     char *zOpt = Tcl_GetStringFromObj(objv[i], &nOpt);
 
-    if( (nOpt>11 || nOpt<2 || strncmp("-sectorsize", zOpt, nOpt)) 
+    if( (nOpt>11 || nOpt<2 || strncmp("-sectorsize", zOpt, nOpt))
      && (nOpt>16 || nOpt<2 || strncmp("-characteristics", zOpt, nOpt))
     ){
-      Tcl_AppendResult(interp, 
-        "Bad option: \"", zOpt, 
-        "\" - must be \"-characteristics\" or \"-sectorsize\"", 0
+      Tcl_AppendResult(interp,
+        "Bad option: \"", zOpt,
+        "\" - must be \"-characteristics\" or \"-sectorsize\"", NULL
       );
       return TCL_ERROR;
     }
     if( i==objc-1 ){
-      Tcl_AppendResult(interp, "Option requires an argument: \"", zOpt, "\"",0);
+      Tcl_AppendResult(interp, "Option requires an argument: \"", zOpt, "\"", NULL);
       return TCL_ERROR;
     }
 
@@ -776,17 +772,17 @@ static int processDevSymArgs(
     }else{
       int j;
       Tcl_Obj **apObj;
-      int nObj;
+      Tcl_Size nObj;
       if( Tcl_ListObjGetElements(interp, objv[i+1], &nObj, &apObj) ){
         return TCL_ERROR;
       }
-      for(j=0; j<nObj; j++){
+      for(j=0; j<(int)nObj; j++){
         int rc;
         int iChoice;
         Tcl_Obj *pFlag = Tcl_DuplicateObj(apObj[j]);
         Tcl_IncrRefCount(pFlag);
         Tcl_UtfToLower(Tcl_GetString(pFlag));
- 
+
         rc = Tcl_GetIndexFromObjStruct(
             interp, pFlag, aFlag, sizeof(aFlag[0]), "no such flag", 0, &iChoice
         );
@@ -814,7 +810,7 @@ static int processDevSymArgs(
 /*
 ** tclcmd:   sqlite3_crash_now
 **
-** Simulate a crash immediately. This function does not return 
+** Simulate a crash immediately. This function does not return
 ** (writeListSync() calls exit(-1)).
 */
 static int SQLITE_TCLAPI crashNowCmd(
@@ -853,7 +849,7 @@ static int SQLITE_TCLAPI crashEnableCmd(
     0,                  /* pNext */
     "crash",            /* zName */
     0,                  /* pAppData */
-  
+ 
     cfOpen,               /* xOpen */
     cfDelete,             /* xDelete */
     cfAccess,             /* xAccess */
@@ -909,7 +905,7 @@ static int SQLITE_TCLAPI crashEnableCmd(
 ** an argument. For -sectorsize, this is the simulated sector size in
 ** bytes. For -characteristics, the argument must be a list of io-capability
 ** flags to simulate. Valid flags are "atomic", "atomic512", "atomic1K",
-** "atomic2K", "atomic4K", "atomic8K", "atomic16K", "atomic32K", 
+** "atomic2K", "atomic4K", "atomic8K", "atomic16K", "atomic32K",
 ** "atomic64K", "sequential" and "safe_append".
 **
 ** Example:
@@ -925,7 +921,8 @@ static int SQLITE_TCLAPI crashParamsObjCmd(
 ){
   int iDelay;
   const char *zCrashFile;
-  int nCrashFile, iDc, iSectorSize;
+  Tcl_Size nCrashFile;
+  int iDc, iSectorSize;
 
   iDc = -1;
   iSectorSize = -1;
@@ -937,7 +934,7 @@ static int SQLITE_TCLAPI crashParamsObjCmd(
 
   zCrashFile = Tcl_GetStringFromObj(objv[objc-1], &nCrashFile);
   if( nCrashFile>=sizeof(g.zCrashFile) ){
-    Tcl_AppendResult(interp, "Filename is too long: \"", zCrashFile, "\"", 0);
+    Tcl_AppendResult(interp, "Filename is too long: \"", zCrashFile, "\"", NULL);
     goto error;
   }
   if( Tcl_GetIntFromObj(interp, objv[objc-2], &iDelay) ){
@@ -1046,8 +1043,8 @@ static int SQLITE_TCLAPI jtObjCmd(
   zParent = Tcl_GetString(objv[1]);
   if( objc==3 ){
     if( strcmp(zParent, "-default") ){
-      Tcl_AppendResult(interp, 
-          "bad option \"", zParent, "\": must be -default", 0
+      Tcl_AppendResult(interp,
+          "bad option \"", zParent, "\": must be -default", NULL
       );
       return TCL_ERROR;
     }
@@ -1058,7 +1055,7 @@ static int SQLITE_TCLAPI jtObjCmd(
     zParent = 0;
   }
   if( jt_register(zParent, objc==3) ){
-    Tcl_AppendResult(interp, "Error in jt_register", 0);
+    Tcl_AppendResult(interp, "Error in jt_register", NULL);
     return TCL_ERROR;
   }
 
diff --git a/src/test8.c b/src/test8.c
index 7a532346ed..8a13f5d556 100644
--- a/src/test8.c
+++ b/src/test8.c
@@ -14,11 +14,7 @@
 ** testing of the SQLite library.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -29,7 +25,7 @@ typedef struct echo_cursor echo_cursor;
 
 /*
 ** The test module defined in this file uses four global Tcl variables to
-** commicate with test-scripts:
+** communicate with test-scripts:
 **
 **     $::echo_module
 **     $::echo_module_sync_fail
@@ -1317,7 +1313,12 @@ static sqlite3_module echoModule = {
   echoCommit,                /* xCommit - commit transaction */
   echoRollback,              /* xRollback - rollback transaction */
   echoFindFunction,          /* xFindFunction - function overloading */
-  echoRename                 /* xRename - rename the table */
+  echoRename,                /* xRename - rename the table */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+  0                          /* xIntegrity */
 };
 
 static sqlite3_module echoModuleV2 = {
@@ -1343,7 +1344,9 @@ static sqlite3_module echoModuleV2 = {
   echoRename,                /* xRename - rename the table */
   echoSavepoint,
   echoRelease,
-  echoRollbackTo
+  echoRollbackTo,
+  0,                         /* xShadowName */
+  0                          /* xIntegrity  */
 };
 
 /*
diff --git a/src/test9.c b/src/test9.c
index 5b139e8a56..62b16795e3 100644
--- a/src/test9.c
+++ b/src/test9.c
@@ -15,11 +15,7 @@
 ** as there is not much point in binding to Tcl.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -60,7 +56,7 @@ static int SQLITE_TCLAPI c_collation_test(
 
 error_out:
   Tcl_ResetResult(interp);
-  Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, 0);
+  Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, NULL);
   return TCL_ERROR;
 }
 
@@ -100,7 +96,7 @@ static int SQLITE_TCLAPI c_realloc_test(
 
 error_out:
   Tcl_ResetResult(interp);
-  Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, 0);
+  Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, NULL);
   return TCL_ERROR;
 }
 
@@ -178,7 +174,7 @@ static int SQLITE_TCLAPI c_misuse_test(
 
 error_out:
   Tcl_ResetResult(interp);
-  Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, 0);
+  Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, NULL);
   return TCL_ERROR;
 }
 
diff --git a/src/test_async.c b/src/test_async.c
deleted file mode 100644
index c32c74c660..0000000000
--- a/src/test_async.c
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
-** 2005 December 14
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-**
-** This file contains a binding of the asynchronous IO extension interface
-** (defined in ext/async/sqlite3async.h) to Tcl.
-*/
-
-#define TCL_THREADS 
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
-
-#ifdef SQLITE_ENABLE_ASYNCIO
-
-#include "sqlite3async.h"
-#include "sqlite3.h"
-#include <assert.h>
-
-/* From main.c */
-extern const char *sqlite3ErrName(int);
-
-
-struct TestAsyncGlobal {
-  int isInstalled;                     /* True when async VFS is installed */
-} testasync_g = { 0 };
-
-TCL_DECLARE_MUTEX(testasync_g_writerMutex);
-
-/*
-** sqlite3async_initialize PARENT-VFS ISDEFAULT
-*/
-static int SQLITE_TCLAPI testAsyncInit(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  const char *zParent;
-  int isDefault;
-  int rc;
-
-  if( objc!=3 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "PARENT-VFS ISDEFAULT");
-    return TCL_ERROR;
-  }
-  zParent = Tcl_GetString(objv[1]);
-  if( !*zParent ) {
-    zParent = 0;
-  }
-  if( Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){
-    return TCL_ERROR;
-  }
-
-  rc = sqlite3async_initialize(zParent, isDefault);
-  if( rc!=SQLITE_OK ){
-    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
-    return TCL_ERROR;
-  }
-  return TCL_OK;
-}
-
-/*
-** sqlite3async_shutdown
-*/
-static int SQLITE_TCLAPI testAsyncShutdown(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  sqlite3async_shutdown();
-  return TCL_OK;
-}
-
-static Tcl_ThreadCreateType tclWriterThread(ClientData pIsStarted){
-  Tcl_MutexLock(&testasync_g_writerMutex);
-  *((int *)pIsStarted) = 1;
-  sqlite3async_run();
-  Tcl_MutexUnlock(&testasync_g_writerMutex);
-  Tcl_ExitThread(0);
-  TCL_THREAD_CREATE_RETURN;
-}
-
-/*
-** sqlite3async_start
-**
-** Start a new writer thread.
-*/
-static int SQLITE_TCLAPI testAsyncStart(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  volatile int isStarted = 0;
-  ClientData threadData = (ClientData)&isStarted;
-
-  Tcl_ThreadId x;
-  const int nStack = TCL_THREAD_STACK_DEFAULT;
-  const int flags = TCL_THREAD_NOFLAGS;
-  int rc;
-
-  rc = Tcl_CreateThread(&x, tclWriterThread, threadData, nStack, flags);
-  if( rc!=TCL_OK ){
-    Tcl_AppendResult(interp, "Tcl_CreateThread() failed", 0);
-    return TCL_ERROR;
-  }
-
-  while( isStarted==0 ) { /* Busy loop */ }
-  return TCL_OK;
-}
-
-/*
-** sqlite3async_wait
-**
-** Wait for the current writer thread to terminate.
-**
-** If the current writer thread is set to run forever then this
-** command would block forever.  To prevent that, an error is returned. 
-*/
-static int SQLITE_TCLAPI testAsyncWait(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  int eCond;
-  if( objc!=1 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "");
-    return TCL_ERROR;
-  }
-
-  sqlite3async_control(SQLITEASYNC_GET_HALT, &eCond);
-  if( eCond==SQLITEASYNC_HALT_NEVER ){
-    Tcl_AppendResult(interp, "would block forever", (char*)0);
-    return TCL_ERROR;
-  }
-
-  Tcl_MutexLock(&testasync_g_writerMutex);
-  Tcl_MutexUnlock(&testasync_g_writerMutex);
-  return TCL_OK;
-}
-
-/*
-** sqlite3async_control OPTION ?VALUE?
-*/
-static int SQLITE_TCLAPI testAsyncControl(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  int rc = SQLITE_OK;
-  int aeOpt[] = { SQLITEASYNC_HALT, SQLITEASYNC_DELAY, SQLITEASYNC_LOCKFILES };
-  const char *azOpt[] = { "halt", "delay", "lockfiles", 0 };
-  const char *az[] = { "never", "now", "idle", 0 };
-  int iVal;
-  int eOpt;
-
-  if( objc!=2 && objc!=3 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "OPTION ?VALUE?");
-    return TCL_ERROR;
-  }
-  if( Tcl_GetIndexFromObj(interp, objv[1], azOpt, "option", 0, &eOpt) ){
-    return TCL_ERROR;
-  }
-  eOpt = aeOpt[eOpt];
-
-  if( objc==3 ){
-    switch( eOpt ){
-      case SQLITEASYNC_HALT: {
-        assert( SQLITEASYNC_HALT_NEVER==0 );
-        assert( SQLITEASYNC_HALT_NOW==1 );
-        assert( SQLITEASYNC_HALT_IDLE==2 );
-        if( Tcl_GetIndexFromObj(interp, objv[2], az, "value", 0, &iVal) ){
-          return TCL_ERROR;
-        }
-        break;
-      }
-      case SQLITEASYNC_DELAY:
-        if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ){
-          return TCL_ERROR;
-        }
-        break;
-
-      case SQLITEASYNC_LOCKFILES:
-        if( Tcl_GetBooleanFromObj(interp, objv[2], &iVal) ){
-          return TCL_ERROR;
-        }
-        break;
-    }
-
-    rc = sqlite3async_control(eOpt, iVal);
-  }
-
-  if( rc==SQLITE_OK ){
-    rc = sqlite3async_control(
-        eOpt==SQLITEASYNC_HALT ? SQLITEASYNC_GET_HALT :
-        eOpt==SQLITEASYNC_DELAY ? SQLITEASYNC_GET_DELAY :
-        SQLITEASYNC_GET_LOCKFILES, &iVal);
-  }
-
-  if( rc!=SQLITE_OK ){
-    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
-    return TCL_ERROR;
-  }
-
-  if( eOpt==SQLITEASYNC_HALT ){
-    Tcl_SetObjResult(interp, Tcl_NewStringObj(az[iVal], -1));
-  }else{
-    Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
-  }
-
-  return TCL_OK;
-}
-
-#endif  /* SQLITE_ENABLE_ASYNCIO */
-
-/*
-** This routine registers the custom TCL commands defined in this
-** module.  This should be the only procedure visible from outside
-** of this module.
-*/
-int Sqlitetestasync_Init(Tcl_Interp *interp){
-#ifdef SQLITE_ENABLE_ASYNCIO
-  Tcl_CreateObjCommand(interp,"sqlite3async_start",testAsyncStart,0,0);
-  Tcl_CreateObjCommand(interp,"sqlite3async_wait",testAsyncWait,0,0);
-
-  Tcl_CreateObjCommand(interp,"sqlite3async_control",testAsyncControl,0,0);
-  Tcl_CreateObjCommand(interp,"sqlite3async_initialize",testAsyncInit,0,0);
-  Tcl_CreateObjCommand(interp,"sqlite3async_shutdown",testAsyncShutdown,0,0);
-#endif  /* SQLITE_ENABLE_ASYNCIO */
-  return TCL_OK;
-}
diff --git a/src/test_autoext.c b/src/test_autoext.c
index e23e41a08a..74ca55879f 100644
--- a/src/test_autoext.c
+++ b/src/test_autoext.c
@@ -11,14 +11,7 @@
 *************************************************************************
 ** Test extension for testing the sqlite3_auto_extension() function.
 */
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 #include "sqlite3ext.h"
 
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
diff --git a/src/test_backup.c b/src/test_backup.c
index 9b684a28f6..ae2348ebc2 100644
--- a/src/test_backup.c
+++ b/src/test_backup.c
@@ -13,14 +13,7 @@
 **
 */
 
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 #include "sqlite3.h"
 #include <assert.h>
 
@@ -142,7 +135,7 @@ static int SQLITE_TCLAPI backupTestInit(
 
   pBackup = sqlite3_backup_init(pDestDb, zDestName, pSrcDb, zSrcName);
   if( !pBackup ){
-    Tcl_AppendResult(interp, "sqlite3_backup_init() failed", 0);
+    Tcl_AppendResult(interp, "sqlite3_backup_init() failed", NULL);
     return TCL_ERROR;
   }
 
diff --git a/src/test_bestindex.c b/src/test_bestindex.c
index f6e0678ce0..f6b5db0fbe 100644
--- a/src/test_bestindex.c
+++ b/src/test_bestindex.c
@@ -93,11 +93,7 @@
 
 
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
@@ -105,6 +101,7 @@
 typedef struct tcl_vtab tcl_vtab;
 typedef struct tcl_cursor tcl_cursor;
 typedef struct TestFindFunction TestFindFunction;
+typedef struct TestVtabContext TestVtabContext;
 
 /* 
 ** A fs virtual-table object 
@@ -129,6 +126,10 @@ struct TestFindFunction {
   TestFindFunction *pNext;
 };
 
+struct TestVtabContext {
+  Tcl_Interp *interp;
+  Tcl_Obj *pDefault;
+};
 
 /*
 ** Dequote string z in place.
@@ -182,25 +183,33 @@ static int tclConnect(
   sqlite3_vtab **ppVtab,
   char **pzErr
 ){
-  Tcl_Interp *interp = (Tcl_Interp*)pAux;
+  TestVtabContext *pCtx = (TestVtabContext*)pAux;
+  Tcl_Interp *interp = pCtx->interp;
   tcl_vtab *pTab = 0;
   char *zCmd = 0;
   Tcl_Obj *pScript = 0;
   int rc = SQLITE_OK;
 
-  if( argc!=4 ){
+  if( argc!=4 && (argc!=3 || pCtx->pDefault==0) ){
     *pzErr = sqlite3_mprintf("wrong number of arguments");
     return SQLITE_ERROR;
   }
 
-  zCmd = sqlite3_malloc64(strlen(argv[3])+1);
+  if( argc==4 ){
+    zCmd = sqlite3_malloc64(strlen(argv[3])+1);
+  }
   pTab = (tcl_vtab*)sqlite3_malloc64(sizeof(tcl_vtab));
-  if( zCmd && pTab ){
-    memcpy(zCmd, argv[3], strlen(argv[3])+1);
-    tclDequote(zCmd);
+  if( (zCmd || argc==3) && pTab ){
     memset(pTab, 0, sizeof(tcl_vtab));
 
-    pTab->pCmd = Tcl_NewStringObj(zCmd, -1);
+    if( zCmd ){
+      memcpy(zCmd, argv[3], strlen(argv[3])+1);
+      tclDequote(zCmd);
+      pTab->pCmd = Tcl_NewStringObj(zCmd, -1);
+    }else{
+      pTab->pCmd = Tcl_DuplicateObj(pCtx->pDefault);
+    }
+
     pTab->interp = interp;
     pTab->db = db;
     Tcl_IncrRefCount(pTab->pCmd);
@@ -212,9 +221,16 @@ static int tclConnect(
     rc = Tcl_EvalObjEx(interp, pScript, TCL_EVAL_GLOBAL);
     if( rc!=TCL_OK ){
       *pzErr = sqlite3_mprintf("%s", Tcl_GetStringResult(interp));
-      rc = SQLITE_ERROR;
+      if( sqlite3_stricmp(*pzErr, "database schema has changed")==0 ){
+        rc = SQLITE_SCHEMA;
+      }else{
+        rc = SQLITE_ERROR;
+      }
     }else{
       rc = sqlite3_declare_vtab(db, Tcl_GetStringResult(interp));
+      if( rc!=SQLITE_OK ){
+        *pzErr = sqlite3_mprintf("declare_vtab: %s", sqlite3_errmsg(db));
+      }
     }
 
     if( rc!=SQLITE_OK ){
@@ -226,7 +242,7 @@ static int tclConnect(
   }
 
   sqlite3_free(zCmd);
-  *ppVtab = &pTab->base;
+  *ppVtab = pTab ? &pTab->base : 0;
   return rc;
 }
 
@@ -302,11 +318,9 @@ static int tclFilter(
   Tcl_IncrRefCount(pScript);
   Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("xFilter", -1));
   Tcl_ListObjAppendElement(interp, pScript, Tcl_NewIntObj(idxNum));
-  if( idxStr ){
-    Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj(idxStr, -1));
-  }else{
-    Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("", -1));
-  }
+  Tcl_ListObjAppendElement(
+      interp, pScript, Tcl_NewStringObj(idxStr ? idxStr : "", -1)
+  );
 
   pArg = Tcl_NewObj();
   Tcl_IncrRefCount(pArg);
@@ -351,14 +365,14 @@ static int tclFilter(
     */
     Tcl_Obj *pRes = Tcl_GetObjResult(interp);
     Tcl_Obj **apElem = 0;
-    int nElem;
+    Tcl_Size nElem;
     rc = Tcl_ListObjGetElements(interp, pRes, &nElem, &apElem);
     if( rc!=TCL_OK ){
       const char *zErr = Tcl_GetStringResult(interp);
       rc = SQLITE_ERROR;
       pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr);
     }else{
-      for(ii=0; rc==SQLITE_OK && ii<nElem; ii+=2){
+      for(ii=0; rc==SQLITE_OK && ii<(int)nElem; ii+=2){
         const char *zCmd = Tcl_GetString(apElem[ii]);
         Tcl_Obj *p = apElem[ii+1];
         if( sqlite3_stricmp("sql", zCmd)==0 ){
@@ -527,6 +541,7 @@ static int SQLITE_TCLAPI testBestIndexObj(
     "distinct",                   /* 3 */
     "in",                         /* 4 */
     "rhs_value",                  /* 5 */
+    "collation",                  /* 6 */
     0
   };
   int ii;
@@ -607,6 +622,17 @@ static int SQLITE_TCLAPI testBestIndexObj(
       Tcl_SetObjResult(interp, Tcl_NewStringObj(zVal, -1));
       break;
     }
+
+    case 6: assert( sqlite3_stricmp(azSub[ii], "collation")==0 ); {
+      int iCons = 0;
+      const char *zColl = "";
+      if( Tcl_GetIntFromObj(interp, objv[2], &iCons) ){
+        return TCL_ERROR;
+      }
+      zColl = sqlite3_vtab_collation(pIdxInfo, iCons);
+      Tcl_SetObjResult(interp, Tcl_NewStringObj(zColl, -1));
+      break;
+    }
   }
 
   return TCL_OK;
@@ -651,7 +677,7 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
     */
     Tcl_Obj *pRes = Tcl_GetObjResult(interp);
     Tcl_Obj **apElem = 0;
-    int nElem;
+    Tcl_Size nElem;
     rc = Tcl_ListObjGetElements(interp, pRes, &nElem, &apElem);
     if( rc!=TCL_OK ){
       const char *zErr = Tcl_GetStringResult(interp);
@@ -660,7 +686,7 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
     }else{
       int ii;
       int iArgv = 1;
-      for(ii=0; rc==SQLITE_OK && ii<nElem; ii+=2){
+      for(ii=0; rc==SQLITE_OK && ii<(int)nElem; ii+=2){
         const char *zCmd = Tcl_GetString(apElem[ii]);
         Tcl_Obj *p = apElem[ii+1];
         if( sqlite3_stricmp("cost", zCmd)==0 ){
@@ -697,6 +723,10 @@ static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
               pIdxInfo->aConstraintUsage[iCons].omit = bOmit;
             }
           }
+        }else
+        if( sqlite3_stricmp("constraint", zCmd)==0 ){
+          rc = SQLITE_CONSTRAINT;
+          pTab->base.zErrMsg = sqlite3_mprintf("%s", Tcl_GetString(p));
         }else{
           rc = SQLITE_ERROR;
           pTab->base.zErrMsg = sqlite3_mprintf("unexpected: %s", zCmd);
@@ -789,6 +819,39 @@ static int tclFindFunction(
   return iRet;
 }
 
+static int tclUpdate(
+  sqlite3_vtab *tab, 
+  int nArg, 
+  sqlite3_value **apVal, 
+  sqlite3_int64 *piRowid
+){
+  tcl_vtab *pTab = (tcl_vtab*)tab;
+  Tcl_Interp *interp = pTab->interp; 
+  Tcl_Obj *pEval = Tcl_DuplicateObj(pTab->pCmd);
+  Tcl_Obj *pRes = 0;
+  int rc = TCL_OK;
+
+  Tcl_IncrRefCount(pEval);
+  Tcl_ListObjAppendElement(interp, pEval, Tcl_NewStringObj("xUpdate",-1));
+
+  rc = Tcl_EvalObjEx(interp, pEval, TCL_EVAL_GLOBAL);
+  Tcl_DecrRefCount(pEval);
+
+  if( rc==TCL_OK ){
+    Tcl_Obj *pRes = Tcl_GetObjResult(interp);
+    Tcl_WideInt v;
+    rc = Tcl_GetWideIntFromObj(interp, pRes, &v);
+    *piRowid = (sqlite3_int64)v;
+  }
+
+  if( rc!=TCL_OK ){
+    tab->zErrMsg = sqlite3_mprintf("%s", Tcl_GetStringResult(pTab->interp));
+    return rc;
+  }
+
+  return SQLITE_OK;
+}
+
 /*
 ** A virtual table module that provides read-only access to a
 ** Tcl global variable namespace.
@@ -814,6 +877,38 @@ static sqlite3_module tclModule = {
   0,                           /* xRollback */
   tclFindFunction,             /* xFindFunction */
   0,                           /* xRename */
+  0,                           /* xSavepoint */
+  0,                           /* xRelease */
+  0,                           /* xRollbackTo */
+  0,                           /* xShadowName */
+  0                            /* xIntegrity */
+};
+static sqlite3_module tclModuleUpdate = {
+  0,                         /* iVersion */
+  tclConnect,
+  tclConnect,
+  tclBestIndex,
+  tclDisconnect, 
+  tclDisconnect,
+  tclOpen,                      /* xOpen - open a cursor */
+  tclClose,                     /* xClose - close a cursor */
+  tclFilter,                    /* xFilter - configure scan constraints */
+  tclNext,                      /* xNext - advance a cursor */
+  tclEof,                       /* xEof - check for end of scan */
+  tclColumn,                    /* xColumn - read data */
+  tclRowid,                     /* xRowid - read data */
+  tclUpdate,                   /* xUpdate */
+  0,                           /* xBegin */
+  0,                           /* xSync */
+  0,                           /* xCommit */
+  0,                           /* xRollback */
+  tclFindFunction,             /* xFindFunction */
+  0,                           /* xRename */
+  0,                           /* xSavepoint */
+  0,                           /* xRelease */
+  0,                           /* xRollbackTo */
+  0,                           /* xShadowName */
+  0                            /* xIntegrity */
 };
 
 /*
@@ -821,6 +916,14 @@ static sqlite3_module tclModule = {
 */
 extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 
+static void delTestVtabCtx(void *p){
+  TestVtabContext *pCtx = (TestVtabContext*)p;
+  if( pCtx->pDefault ){
+    Tcl_DecrRefCount(pCtx->pDefault);
+  }
+  ckfree(pCtx);
+}
+
 /*
 ** Register the echo virtual table module.
 */
@@ -831,13 +934,25 @@ static int SQLITE_TCLAPI register_tcl_module(
   Tcl_Obj *CONST objv[]  /* Command arguments */
 ){
   sqlite3 *db;
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+  if( objc!=2 && objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB ?DEFAULT-CMD?");
     return TCL_ERROR;
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  sqlite3_create_module(db, "tcl", &tclModule, (void *)interp);
+  {
+    sqlite3_module *pMod = &tclModule;
+    TestVtabContext *pCtx = (TestVtabContext*)ckalloc(sizeof(TestVtabContext));
+    pCtx->interp = interp;
+    pCtx->pDefault = 0;
+    if( objc==3 ){
+      pCtx->pDefault = objv[2];
+      Tcl_IncrRefCount(pCtx->pDefault);
+    }
+
+    if( objc==3 ){ pMod = &tclModuleUpdate; }
+    sqlite3_create_module_v2(db, "tcl", pMod, (void*)pCtx, delTestVtabCtx);
+  }
 #endif
   return TCL_OK;
 }
diff --git a/src/test_blob.c b/src/test_blob.c
index cbdf9f069f..ae5a734179 100644
--- a/src/test_blob.c
+++ b/src/test_blob.c
@@ -12,11 +12,7 @@
 **
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -58,7 +54,7 @@ static int blobHandleFromObj(
   sqlite3_blob **ppBlob
 ){
   char *z;
-  int n;
+  Tcl_Size n;
 
   z = Tcl_GetStringFromObj(pObj, &n);
   if( n==0 ){
@@ -88,7 +84,7 @@ static int blobHandleFromObj(
 ** NULL Pointer is returned.
 */
 static char *blobStringFromObj(Tcl_Obj *pObj){
-  int n;
+  Tcl_Size n;
   char *z;
   z = Tcl_GetStringFromObj(pObj, &n);
   return (n ? z : 0);
@@ -112,7 +108,7 @@ static int SQLITE_TCLAPI test_blob_open(
   Tcl_WideInt iRowid;
   int flags;
   const char *zVarname;
-  int nVarname;
+  Tcl_Size nVarname;
 
   sqlite3_blob *pBlob = (sqlite3_blob*)&flags;   /* Non-zero initialization */
   int rc;
@@ -241,7 +237,7 @@ static int SQLITE_TCLAPI test_blob_read(
   if( nByte>0 ){
     zBuf = (unsigned char *)Tcl_AttemptAlloc(nByte);
     if( zBuf==0 ){
-      Tcl_AppendResult(interp, "out of memory in " __FILE__, 0);
+      Tcl_AppendResult(interp, "out of memory in " __FILE__, NULL);
       return TCL_ERROR;
     }
   }
@@ -281,7 +277,8 @@ static int SQLITE_TCLAPI test_blob_write(
   int rc;
 
   unsigned char *zBuf;
-  int nBuf;
+  Tcl_Size nBuf;
+  int n;
   
   if( objc!=4 && objc!=5 ){
     Tcl_WrongNumArgs(interp, 1, objv, "HANDLE OFFSET DATA ?NDATA?");
@@ -294,10 +291,11 @@ static int SQLITE_TCLAPI test_blob_write(
   }
 
   zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf);
-  if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){
+  n = (int)(nBuf & 0x7fffffff);
+  if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &n) ){
     return TCL_ERROR;
   }
-  rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset);
+  rc = sqlite3_blob_write(pBlob, zBuf, n, iOffset);
   if( rc!=SQLITE_OK ){
     Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE);
   }
diff --git a/src/test_btree.c b/src/test_btree.c
index 03b8b207c9..168a10f1f3 100644
--- a/src/test_btree.c
+++ b/src/test_btree.c
@@ -14,11 +14,7 @@
 ** testing of the SQLite library.
 */
 #include "btreeInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 /*
 ** Usage: sqlite3_shared_cache_report
diff --git a/src/test_config.c b/src/test_config.c
index 71ffd50297..e66d15009c 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -24,11 +24,7 @@
 #  include "os_win.h"
 #endif
 
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -59,6 +55,20 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "rowid32", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
+  Tcl_SetVar2(
+      interp, "sqlite_options", "allow_rowid_in_view", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(
+      interp, "sqlite_options", "allow_rowid_in_view", "0", TCL_GLOBAL_ONLY);
+#endif
+
+#if defined(SQLITE_ENABLE_CARRAY)
+  Tcl_SetVar2(interp, "sqlite_options","carray","1",TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options","carray","0",TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_CASE_SENSITIVE_LIKE
   Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","1",TCL_GLOBAL_ONLY);
 #else
@@ -84,6 +94,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "win32malloc", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#if defined(SQLITE_OS_WINRT) && SQLITE_OS_WINRT
+  Tcl_SetVar2(interp, "sqlite_options", "winrt", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "winrt", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_DEBUG
   Tcl_SetVar2(interp, "sqlite_options", "debug", "1", TCL_GLOBAL_ONLY);
 #else
@@ -185,6 +201,14 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "offset_sql_func","0",TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES
+  Tcl_SetVar2(interp, "sqlite_options",
+                      "ordered_set_aggregates","1",TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options",
+                      "ordered_set_aggregates","0",TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   Tcl_SetVar2(interp, "sqlite_options", "preupdate", "1", TCL_GLOBAL_ONLY);
 #else
@@ -343,6 +367,14 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_ORDEREDSETFUNC
+  Tcl_SetVar2(interp, "sqlite_options", "ordered_set_funcs", "1",
+              TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "ordered_set_funcs", "0",
+              TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK
   Tcl_SetVar2(interp, "sqlite_options", "oversize_cell_check", "1",
               TCL_GLOBAL_ONLY);
@@ -414,18 +446,6 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "foreignkey", "1", TCL_GLOBAL_ONLY);
 #endif
 
-#ifdef SQLITE_ENABLE_FTS1
-  Tcl_SetVar2(interp, "sqlite_options", "fts1", "1", TCL_GLOBAL_ONLY);
-#else
-  Tcl_SetVar2(interp, "sqlite_options", "fts1", "0", TCL_GLOBAL_ONLY);
-#endif
-
-#ifdef SQLITE_ENABLE_FTS2
-  Tcl_SetVar2(interp, "sqlite_options", "fts2", "1", TCL_GLOBAL_ONLY);
-#else
-  Tcl_SetVar2(interp, "sqlite_options", "fts2", "0", TCL_GLOBAL_ONLY);
-#endif
-
 #ifdef SQLITE_ENABLE_FTS3
   Tcl_SetVar2(interp, "sqlite_options", "fts3", "1", TCL_GLOBAL_ONLY);
 #else
@@ -510,10 +530,6 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "lookaside", "1", TCL_GLOBAL_ONLY);
 #endif
 
-Tcl_SetVar2(interp, "sqlite_options", "long_double",
-              sizeof(LONGDOUBLE_TYPE)>sizeof(double) ? "1" : "0",
-              TCL_GLOBAL_ONLY);
-
 #ifdef SQLITE_OMIT_MEMORYDB
   Tcl_SetVar2(interp, "sqlite_options", "memorydb", "0", TCL_GLOBAL_ONLY);
 #else
@@ -747,12 +763,6 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "secure_delete", "0", TCL_GLOBAL_ONLY);
 #endif
 
-#ifdef SQLITE_USER_AUTHENTICATION
-  Tcl_SetVar2(interp, "sqlite_options", "userauth", "1", TCL_GLOBAL_ONLY);
-#else
-  Tcl_SetVar2(interp, "sqlite_options", "userauth", "0", TCL_GLOBAL_ONLY);
-#endif
-
 #ifdef SQLITE_MULTIPLEX_EXT_OVWR
   Tcl_SetVar2(interp, "sqlite_options", "multiplex_ext_overwrite", "1", TCL_GLOBAL_ONLY);
 #else
@@ -789,6 +799,13 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "windowfunc", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#if !defined(SQLITE_ENABLE_SETLK_TIMEOUT)
+  Tcl_SetVar2(interp, "sqlite_options", "setlk_timeout", "0", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "setlk_timeout", 
+      STRINGVALUE(SQLITE_ENABLE_SETLK_TIMEOUT), TCL_GLOBAL_ONLY);
+#endif
+
 #define LINKVAR(x) { \
     static const int cv_ ## x = SQLITE_ ## x; \
     Tcl_LinkVar(interp, "SQLITE_" #x, (char *)&(cv_ ## x), \
diff --git a/src/test_delete.c b/src/test_delete.c
index 68fdbc6a75..3f34a72c46 100644
--- a/src/test_delete.c
+++ b/src/test_delete.c
@@ -63,7 +63,7 @@ static int sqlite3DeleteUnlinkIfExists(
   int *pbExists
 ){
   int rc = SQLITE_ERROR;
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   if( pVfs ){
     if( pbExists ) *pbExists = 1;
     rc = pVfs->xDelete(pVfs, zFile, 0);
@@ -115,7 +115,7 @@ SQLITE_API int sqlite3_delete_database(
     { "%s-wal%03d",     SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 },
   };
 
-#ifdef SQLITE_OS_WIN
+#ifdef _WIN32
   sqlite3_vfs *pVfs = sqlite3_vfs_find("win32");
 #else
   sqlite3_vfs *pVfs = 0;
diff --git a/src/test_demovfs.c b/src/test_demovfs.c
index e990e98f29..e92fd56138 100644
--- a/src/test_demovfs.c
+++ b/src/test_demovfs.c
@@ -645,14 +645,7 @@ sqlite3_vfs *sqlite3_demovfs(void){
 
 #ifdef SQLITE_TEST
 
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 
 #if SQLITE_OS_UNIX
 static int SQLITE_TCLAPI register_demovfs(
diff --git a/src/test_devsym.c b/src/test_devsym.c
index 7847bc300b..86b78f15a1 100644
--- a/src/test_devsym.c
+++ b/src/test_devsym.c
@@ -187,7 +187,7 @@ static int devsymDeviceCharacteristics(sqlite3_file *pFile){
 }
 
 /*
-** Shared-memory methods are all pass-thrus.
+** Shared-memory methods are all pass-throughs.
 */
 static int devsymShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
   devsym_file *p = (devsym_file *)pFile;
diff --git a/src/test_fs.c b/src/test_fs.c
index ddfdc7fb59..6879f82734 100644
--- a/src/test_fs.c
+++ b/src/test_fs.c
@@ -62,30 +62,18 @@
 **   SELECT * FROM fstree WHERE path LIKE '/home/dan/sqlite/%'
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
-
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 
-#if SQLITE_OS_UNIX || defined(__MINGW_H)
+#if !defined(_WIN32) || defined(__MSVCRT__)
 # include <unistd.h>
 # include <dirent.h>
-# ifndef DIRENT
-#  define DIRENT dirent
-# endif
-#endif
-#if SQLITE_OS_WIN
-# include <io.h>
-# if !defined(__MINGW_H)
-#  include "test_windirent.h"
-# endif
+#else
+# include "windirent.h"
 # ifndef S_ISREG
 #  define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG)
 # endif
@@ -129,7 +117,7 @@ struct FsdirCsr {
   char *zDir;                     /* Buffer containing directory scanned */
   DIR *pDir;                      /* Open directory */
   sqlite3_int64 iRowid;
-  struct DIRENT *pEntry;
+  struct dirent *pEntry;
 };
 
 /*
@@ -490,10 +478,10 @@ static int fstreeFilter(
   int nDir;
   char aWild[2] = { '\0', '\0' };
 
-#if SQLITE_OS_WIN
-  const char *zDrive = windirent_getenv("fstreeDrive");
+#ifdef _WIN32
+  const char *zDrive = getenv("fstreeDrive");
   if( zDrive==0 ){
-    zDrive = windirent_getenv("SystemDrive");
+    zDrive = getenv("SystemDrive");
   }
   zRoot = sqlite3_mprintf("%s%c", zDrive, '/');
   nRoot = sqlite3Strlen30(zRoot);
@@ -543,7 +531,7 @@ static int fstreeFilter(
   sqlite3_bind_text(pCsr->pStmt, 2, zRoot, nRoot, SQLITE_TRANSIENT);
   sqlite3_bind_text(pCsr->pStmt, 3, zPrefix, nPrefix, SQLITE_TRANSIENT);
 
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   sqlite3_free(zPrefix);
   sqlite3_free(zRoot);
 #endif
@@ -816,6 +804,11 @@ static sqlite3_module fsModule = {
   0,                           /* xRollback */
   0,                           /* xFindMethod */
   0,                           /* xRename */
+  0,                           /* xSavepoint */
+  0,                           /* xRelease */
+  0,                           /* xRollbackTo */
+  0,                           /* xShadowName */
+  0                            /* xIntegrity */
 };
 
 static sqlite3_module fsdirModule = {
@@ -839,6 +832,11 @@ static sqlite3_module fsdirModule = {
   0,                              /* xRollback */
   0,                              /* xFindMethod */
   0,                              /* xRename */
+  0,                              /* xSavepoint */
+  0,                              /* xRelease */
+  0,                              /* xRollbackTo */
+  0,                              /* xShadowName */
+  0                               /* xIntegrity */
 };
 
 static sqlite3_module fstreeModule = {
@@ -862,6 +860,11 @@ static sqlite3_module fstreeModule = {
   0,                              /* xRollback */
   0,                              /* xFindMethod */
   0,                              /* xRename */
+  0,                              /* xSavepoint */
+  0,                              /* xRelease */
+  0,                              /* xRollbackTo */
+  0,                              /* xShadowName */
+  0                               /* xIntegrity */
 };
 
 /*
diff --git a/src/test_func.c b/src/test_func.c
index fa52438261..82f7b3d9ca 100644
--- a/src/test_func.c
+++ b/src/test_func.c
@@ -13,11 +13,7 @@
 ** implements new SQL functions used by the test scripts.
 */
 #include "sqlite3.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -694,7 +690,8 @@ static int registerTestFunctions(
     { "test_extract",          2, SQLITE_UTF8, test_extract},
     { "test_zeroblob",  1, SQLITE_UTF8|SQLITE_DETERMINISTIC, test_zeroblob},
     { "test_getsubtype",       1, SQLITE_UTF8, test_getsubtype},
-    { "test_setsubtype",       2, SQLITE_UTF8, test_setsubtype},
+    { "test_setsubtype",       2, SQLITE_UTF8|SQLITE_RESULT_SUBTYPE,
+                                               test_setsubtype},
     { "test_frombind",        -1, SQLITE_UTF8, test_frombind},
   };
   int i;
@@ -776,7 +773,7 @@ static int SQLITE_TCLAPI abuse_create_function(
   rc = sqlite3_create_function(db, "tx", -2, SQLITE_UTF8, 0, tStep, 0, 0);
   if( rc!=SQLITE_MISUSE ) goto abuse_err;
 
-  rc = sqlite3_create_function(db, "tx", 128, SQLITE_UTF8, 0, tStep, 0, 0);
+  rc = sqlite3_create_function(db, "tx", 32768, SQLITE_UTF8, 0, tStep, 0, 0);
   if( rc!=SQLITE_MISUSE ) goto abuse_err;
 
   rc = sqlite3_create_function(db, "funcxx"
@@ -792,7 +789,7 @@ static int SQLITE_TCLAPI abuse_create_function(
   ** a no-op function (that always returns NULL) and which has the
   ** maximum-length function name and the maximum number of parameters.
   */
-  sqlite3_limit(db, SQLITE_LIMIT_FUNCTION_ARG, 10000);
+  sqlite3_limit(db, SQLITE_LIMIT_FUNCTION_ARG, 1000000);
   mxArg = sqlite3_limit(db, SQLITE_LIMIT_FUNCTION_ARG, -1);
   rc = sqlite3_create_function(db, "nullx"
        "_123456789_123456789_123456789_123456789_123456789"
diff --git a/src/test_hexio.c b/src/test_hexio.c
index 61a41d5b1c..048ab13246 100644
--- a/src/test_hexio.c
+++ b/src/test_hexio.c
@@ -18,11 +18,7 @@
 ** easier and safer to build our own mechanism.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -126,7 +122,7 @@ static int SQLITE_TCLAPI hexio_read(
     in = fopen(zFile, "r");
   }
   if( in==0 ){
-    Tcl_AppendResult(interp, "cannot open input file ", zFile, 0);
+    Tcl_AppendResult(interp, "cannot open input file ", zFile, NULL);
     return TCL_ERROR;
   }
   fseek(in, offset, SEEK_SET);
@@ -136,7 +132,7 @@ static int SQLITE_TCLAPI hexio_read(
     got = 0;
   }
   sqlite3TestBinToHex(zBuf, got);
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
   sqlite3_free(zBuf);
   return TCL_OK;
 }
@@ -155,7 +151,8 @@ static int SQLITE_TCLAPI hexio_write(
   Tcl_Obj *CONST objv[]
 ){
   int offset;
-  int nIn, nOut, written;
+  Tcl_Size nIn;
+  int nOut, written;
   const char *zFile;
   const unsigned char *zIn;
   unsigned char *aOut;
@@ -168,17 +165,17 @@ static int SQLITE_TCLAPI hexio_write(
   if( Tcl_GetIntFromObj(interp, objv[2], &offset) ) return TCL_ERROR;
   zFile = Tcl_GetString(objv[1]);
   zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[3], &nIn);
-  aOut = sqlite3_malloc( 1 + nIn/2 );
+  aOut = sqlite3_malloc64( 1 + nIn/2 );
   if( aOut==0 ){
     return TCL_ERROR;
   }
-  nOut = sqlite3TestHexToBin(zIn, nIn, aOut);
+  nOut = sqlite3TestHexToBin(zIn, (int)nIn, aOut);
   out = fopen(zFile, "r+b");
   if( out==0 ){
     out = fopen(zFile, "r+");
   }
   if( out==0 ){
-    Tcl_AppendResult(interp, "cannot open output file ", zFile, 0);
+    Tcl_AppendResult(interp, "cannot open output file ", zFile, NULL);
     return TCL_ERROR;
   }
   fseek(out, offset, SEEK_SET);
@@ -190,7 +187,7 @@ static int SQLITE_TCLAPI hexio_write(
 }
 
 /*
-** USAGE:   hexio_get_int   HEXDATA
+** USAGE:   hexio_get_int [-littleendian] HEXDATA
 **
 ** Interpret the HEXDATA argument as a big-endian integer.  Return
 ** the value of that integer.  HEXDATA can contain between 2 and 8
@@ -203,21 +200,30 @@ static int SQLITE_TCLAPI hexio_get_int(
   Tcl_Obj *CONST objv[]
 ){
   int val;
-  int nIn, nOut;
+  Tcl_Size nIn;
+  int nOut;
   const unsigned char *zIn;
   unsigned char *aOut;
   unsigned char aNum[4];
+  int bLittle = 0;
 
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA");
+  if( objc==3 ){
+    Tcl_Size n;
+    char *z = Tcl_GetStringFromObj(objv[1], &n);
+    if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){
+      bLittle = 1;
+    }
+  }
+  if( (objc-bLittle)!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA");
     return TCL_ERROR;
   }
-  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn);
-  aOut = sqlite3_malloc( 1 + nIn/2 );
+  zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn);
+  aOut = sqlite3_malloc64( 1 + nIn/2 );
   if( aOut==0 ){
     return TCL_ERROR;
   }
-  nOut = sqlite3TestHexToBin(zIn, nIn, aOut);
+  nOut = sqlite3TestHexToBin(zIn, (int)nIn, aOut);
   if( nOut>=4 ){
     memcpy(aNum, aOut, 4);
   }else{
@@ -225,7 +231,11 @@ static int SQLITE_TCLAPI hexio_get_int(
     memcpy(&aNum[4-nOut], aOut, nOut);
   }
   sqlite3_free(aOut);
-  val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  if( bLittle ){
+    val = (int)((u32)aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0];
+  }else{
+    val = (int)((u32)aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3];
+  }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(val));
   return TCL_OK;
 }
@@ -300,7 +310,7 @@ static int SQLITE_TCLAPI utf8_to_utf8(
   Tcl_Obj *CONST objv[]
 ){
 #ifdef SQLITE_DEBUG
-  int n;
+  Tcl_Size n;
   int nOut;
   const unsigned char *zOrig;
   unsigned char *z;
@@ -309,17 +319,17 @@ static int SQLITE_TCLAPI utf8_to_utf8(
     return TCL_ERROR;
   }
   zOrig = (unsigned char *)Tcl_GetStringFromObj(objv[1], &n);
-  z = sqlite3_malloc( n+4 );
-  n = sqlite3TestHexToBin(zOrig, n, z);
+  z = sqlite3_malloc64( n+4 );
+  n = sqlite3TestHexToBin(zOrig, (int)n, z);
   z[n] = 0;
   nOut = sqlite3Utf8To8(z);
   sqlite3TestBinToHex(z,nOut);
-  Tcl_AppendResult(interp, (char*)z, 0);
+  Tcl_AppendResult(interp, (char*)z, NULL);
   sqlite3_free(z);
   return TCL_OK;
 #else
   Tcl_AppendResult(interp, 
-      "[utf8_to_utf8] unavailable - SQLITE_DEBUG not defined", 0
+      "[utf8_to_utf8] unavailable - SQLITE_DEBUG not defined", NULL
   );
   return TCL_ERROR;
 #endif
@@ -361,7 +371,7 @@ static int SQLITE_TCLAPI read_fts3varint(
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int nBlob;
+  Tcl_Size nBlob;
   unsigned char *zBlob;
   sqlite3_int64 iVal;
   int nVal;
@@ -388,10 +398,10 @@ static int SQLITE_TCLAPI make_fts3record(
   Tcl_Obj *CONST objv[]
 ){
   Tcl_Obj **aArg = 0;
-  int nArg = 0;
+  Tcl_Size nArg = 0;
   unsigned char *aOut = 0;
-  int nOut = 0;
-  int nAlloc = 0;
+  sqlite3_int64 nOut = 0;
+  sqlite3_int64 nAlloc = 0;
   int i;
 
   if( objc!=2 ){
@@ -402,8 +412,8 @@ static int SQLITE_TCLAPI make_fts3record(
     return TCL_ERROR;
   }
 
-  for(i=0; i<nArg; i++){
-    sqlite3_int64 iVal;
+  for(i=0; i<(int)nArg; i++){
+    Tcl_WideInt iVal;
     if( TCL_OK==Tcl_GetWideIntFromObj(0, aArg[i], &iVal) ){
       if( nOut+10>nAlloc ){
         int nNew = nAlloc?nAlloc*2:128;
@@ -417,11 +427,11 @@ static int SQLITE_TCLAPI make_fts3record(
       }
       nOut += putFts3Varint((char*)&aOut[nOut], iVal);
     }else{
-      int nVal = 0;
+      Tcl_Size nVal = 0;
       char *zVal = Tcl_GetStringFromObj(aArg[i], &nVal);
       while( (nOut + nVal)>nAlloc ){
-        int nNew = nAlloc?nAlloc*2:128;
-        unsigned char *aNew = sqlite3_realloc(aOut, nNew);
+        sqlite3_int64 nNew = nAlloc?nAlloc*2:128;
+        unsigned char *aNew = sqlite3_realloc64(aOut, nNew);
         if( aNew==0 ){
           sqlite3_free(aOut);
           return TCL_ERROR;
diff --git a/src/test_init.c b/src/test_init.c
index 58465785d8..0c6ac8eb56 100644
--- a/src/test_init.c
+++ b/src/test_init.c
@@ -27,23 +27,19 @@
 
 #include "sqliteInt.h"
 #include <string.h>
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 static struct Wrapped {
   sqlite3_pcache_methods2 pcache;
   sqlite3_mem_methods     mem;
   sqlite3_mutex_methods   mutex;
 
-  int mem_init;                /* True if mem subsystem is initalized */
-  int mem_fail;                /* True to fail mem subsystem inialization */
-  int mutex_init;              /* True if mutex subsystem is initalized */
-  int mutex_fail;              /* True to fail mutex subsystem inialization */
-  int pcache_init;             /* True if pcache subsystem is initalized */
-  int pcache_fail;             /* True to fail pcache subsystem inialization */
+  int mem_init;              /* True if mem subsystem is initialized */
+  int mem_fail;              /* True to fail mem subsystem initialization */
+  int mutex_init;            /* True if mutex subsystem is initialized */
+  int mutex_fail;            /* True to fail mutex subsystem initialization */
+  int pcache_init;           /* True if pcache subsystem is initialized */
+  int pcache_fail;           /* True to fail pcache subsystem initialization */
 } wrapped;
 
 static int wrMemInit(void *pAppData){
@@ -205,7 +201,7 @@ static int SQLITE_TCLAPI init_wrapper_install(
     }else if( strcmp(z, "pcache")==0 ){
       wrapped.pcache_fail = 1;
     }else{
-      Tcl_AppendResult(interp, "Unknown argument: \"", z, "\"");
+      Tcl_AppendResult(interp, "Unknown argument: \"", z, "\"", NULL);
       return TCL_ERROR;
     }
   }
diff --git a/src/test_intarray.c b/src/test_intarray.c
index 8c74a04156..9e4629467e 100644
--- a/src/test_intarray.c
+++ b/src/test_intarray.c
@@ -61,7 +61,8 @@ struct intarray_cursor {
 /*
 ** Free an sqlite3_intarray object.
 */
-static void intarrayFree(sqlite3_intarray *p){
+static void intarrayFree(void *pX){
+  sqlite3_intarray *p = (sqlite3_intarray*)pX;
   if( p->xFree ){
     p->xFree(p->a);
   }
@@ -205,6 +206,11 @@ static sqlite3_module intarrayModule = {
   0,                           /* xRollback */
   0,                           /* xFindMethod */
   0,                           /* xRename */
+  0,                           /* xSavepoint */
+  0,                           /* xRelease */
+  0,                           /* xRollbackTo */
+  0,                           /* xShadowName */
+  0                            /* xIntegrity */
 };
 
 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
@@ -274,14 +280,7 @@ SQLITE_API int sqlite3_intarray_bind(
 ** Everything below is interface for testing this module.
 */
 #ifdef SQLITE_TEST
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 
 /*
 ** Routines to encode and decode pointers
diff --git a/src/test_intarray.h b/src/test_intarray.h
index 116e3bdc07..b68233b12a 100644
--- a/src/test_intarray.h
+++ b/src/test_intarray.h
@@ -23,7 +23,7 @@
 **
 **     SELECT * FROM table WHERE x IN (?,?,?,...,?);
 **
-** And then binding indivdual integers to each of ? slots, a C-language
+** And then binding individual integers to each of ? slots, a C-language
 ** application can create an intarray object (named "ex1" in the following
 ** example), prepare a statement like this:
 **
diff --git a/src/test_malloc.c b/src/test_malloc.c
index 8146501c9c..68b2c367da 100644
--- a/src/test_malloc.c
+++ b/src/test_malloc.c
@@ -14,11 +14,7 @@
 ** memory allocation subsystem.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -45,8 +41,9 @@ static struct MemFault {
 ** fire on any simulated malloc() failure.
 */
 static void sqlite3Fault(void){
-  static int cnt = 0;
+  static u64 cnt = 0;
   cnt++;
+  if( cnt>((u64)1<<63) ) abort();
 }
 
 /*
@@ -56,8 +53,9 @@ static void sqlite3Fault(void){
 ** This routine only runs on the first such failure.
 */
 static void sqlite3FirstFault(void){
-  static int cnt2 = 0;
+  static u64 cnt2 = 0;
   cnt2++;
+  if( cnt2>((u64)1<<63) ) abort();
 }
 
 /*
@@ -387,7 +385,8 @@ static int SQLITE_TCLAPI test_memset(
   Tcl_Obj *CONST objv[]
 ){
   void *p;
-  int size, n, i;
+  int size, i;
+  Tcl_Size n;
   char *zHex;
   char *zOut;
   char zBin[100];
@@ -409,7 +408,7 @@ static int SQLITE_TCLAPI test_memset(
   }
   zHex = Tcl_GetStringFromObj(objv[3], &n);
   if( n>sizeof(zBin)*2 ) n = sizeof(zBin)*2;
-  n = sqlite3TestHexToBin(zHex, n, zBin);
+  n = sqlite3TestHexToBin(zHex, (int)n, zBin);
   if( n==0 ){
     Tcl_AppendResult(interp, "no data", (char*)0);
     return TCL_ERROR;
@@ -624,7 +623,7 @@ static int SQLITE_TCLAPI test_memdebug_fail(
   if( Tcl_GetIntFromObj(interp, objv[1], &iFail) ) return TCL_ERROR;
 
   for(ii=2; ii<objc; ii+=2){
-    int nOption;
+    Tcl_Size nOption;
     char *zOption = Tcl_GetStringFromObj(objv[ii], &nOption);
     char *zErr = 0;
 
@@ -647,7 +646,7 @@ static int SQLITE_TCLAPI test_memdebug_fail(
     }
 
     if( zErr ){
-      Tcl_AppendResult(interp, zErr, zOption, 0);
+      Tcl_AppendResult(interp, zErr, zOption, NULL);
       return TCL_ERROR;
     }
   }
@@ -1369,6 +1368,7 @@ static int SQLITE_TCLAPI test_db_status(
     { "DEFERRED_FKS",        SQLITE_DBSTATUS_DEFERRED_FKS        },
     { "CACHE_USED_SHARED",   SQLITE_DBSTATUS_CACHE_USED_SHARED   },
     { "CACHE_SPILL",         SQLITE_DBSTATUS_CACHE_SPILL         },
+    { "TEMPBUF_SPILL",       SQLITE_DBSTATUS_TEMPBUF_SPILL       },
   };
   Tcl_Obj *pResult;
   if( objc!=4 ){
diff --git a/src/test_md5.c b/src/test_md5.c
index b670026861..a09f1ad6c4 100644
--- a/src/test_md5.c
+++ b/src/test_md5.c
@@ -16,14 +16,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "sqlite3.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-# include "sqlite_tcl.h"
-#else
-# include "tcl.h"
-# ifndef SQLITE_TCLAPI
-#  define SQLITE_TCLAPI
-# endif
-#endif
+#include "tclsqlite.h"
 
 /*
  * This code implements the MD5 message-digest algorithm.
@@ -295,7 +288,7 @@ static void MD5DigestToBase16(unsigned char *digest, char *zBuf){
 
 
 /*
-** Convert a 128-bit MD5 digest into sequency of eight 5-digit integers
+** Convert a 128-bit MD5 digest into sequences of eight 5-digit integers
 ** each representing 16 bits of the digest and separated from each
 ** other by a "-" character.
 */
diff --git a/src/test_multiplex.c b/src/test_multiplex.c
index 226131f75c..82551200f2 100644
--- a/src/test_multiplex.c
+++ b/src/test_multiplex.c
@@ -28,9 +28,9 @@
 **
 ** The procedure call above will create and register a new VFS shim named
 ** "multiplex".  The multiplex VFS will use the VFS named by zOrigVfsName to
-** do the actual disk I/O.  (The zOrigVfsName parameter may be NULL, in 
+** do the actual disk I/O.  (The zOrigVfsName parameter may be NULL, in
 ** which case the default VFS at the moment sqlite3_multiplex_initialize()
-** is called will be used as the underlying real VFS.)  
+** is called will be used as the underlying real VFS.)
 **
 ** If the makeDefault parameter is TRUE then multiplex becomes the new
 ** default VFS.  Otherwise, you can use the multiplex VFS by specifying
@@ -39,7 +39,7 @@
 ** URI.
 **
 ** The multiplex VFS allows databases up to 32 GiB in size.  But it splits
-** the files up into smaller pieces, so that they will work even on 
+** the files up into smaller pieces, so that they will work even on
 ** filesystems that do not support large files.  The default chunk size
 ** is 2147418112 bytes (which is 64KiB less than 2GiB) but this can be
 ** changed at compile-time by defining the SQLITE_MULTIPLEX_CHUNK_SIZE
@@ -58,10 +58,10 @@
 #endif
 #include "sqlite3ext.h"
 
-/* 
-** These should be defined to be the same as the values in 
+/*
+** These should be defined to be the same as the values in
 ** sqliteInt.h.  They are defined separately here so that
-** the multiplex VFS shim can be built as a loadable 
+** the multiplex VFS shim can be built as a loadable
 ** module.
 */
 #define UNUSED_PARAMETER(x) (void)(x)
@@ -83,7 +83,7 @@
 #endif
 
 /* This is the limit on the chunk size.  It may be changed by calling
-** the xFileControl() interface.  It will be rounded up to a 
+** the xFileControl() interface.  It will be rounded up to a
 ** multiple of MAX_PAGE_SIZE.  We default it here to 2GiB less 64KiB.
 */
 #ifndef SQLITE_MULTIPLEX_CHUNK_SIZE
@@ -130,7 +130,7 @@ struct multiplexGroup {
 
 /*
 ** An instance of the following object represents each open connection
-** to a file that is multiplex'ed.  This object is a 
+** to a file that is multiplex'ed.  This object is a
 ** subclass of sqlite3_file.  The sqlite3_file object for the underlying
 ** VFS is appended to this structure.
 */
@@ -157,11 +157,11 @@ static struct {
   */
   sqlite3_vfs sThisVfs;
 
-  /* The sIoMethods defines the methods used by sqlite3_file objects 
+  /* The sIoMethods defines the methods used by sqlite3_file objects
   ** associated with this shim.  It is initialized at start-time and does
   ** not require a mutex.
   **
-  ** When the underlying VFS is called to open a file, it might return 
+  ** When the underlying VFS is called to open a file, it might return
   ** either a version 1 or a version 2 sqlite3_file object.  This shim
   ** has to create a wrapper sqlite3_file of the same version.  Hence
   ** there are two I/O method structures, one for version 1 and the other
@@ -199,16 +199,16 @@ static int multiplexStrlen30(const char *z){
 ** nul-terminator.
 **
 ** If iChunk is 0 (or 400 - the number for the first journal file chunk),
-** the output is a copy of the input string. Otherwise, if 
+** the output is a copy of the input string. Otherwise, if
 ** SQLITE_ENABLE_8_3_NAMES is not defined or the input buffer does not contain
-** a "." character, then the output is a copy of the input string with the 
-** three-digit zero-padded decimal representation if iChunk appended to it. 
+** a "." character, then the output is a copy of the input string with the
+** three-digit zero-padded decimal representation if iChunk appended to it.
 ** For example:
 **
 **   zBase="test.db", iChunk=4  ->  zOut="test.db004"
 **
 ** Or, if SQLITE_ENABLE_8_3_NAMES is defined and the input buffer contains
-** a "." character, then everything after the "." is replaced by the 
+** a "." character, then everything after the "." is replaced by the
 ** three-digit representation of iChunk.
 **
 **   zBase="test.db", iChunk=4  ->  zOut="test.004"
@@ -232,12 +232,12 @@ static void multiplexFilename(
     if( i>=n-4 ) n = i+1;
     if( flags & SQLITE_OPEN_MAIN_JOURNAL ){
       /* The extensions on overflow files for main databases are 001, 002,
-      ** 003 and so forth.  To avoid name collisions, add 400 to the 
+      ** 003 and so forth.  To avoid name collisions, add 400 to the
       ** extensions of journal files so that they are 401, 402, 403, ....
       */
       iChunk += SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET;
     }else if( flags & SQLITE_OPEN_WAL ){
-      /* To avoid name collisions, add 700 to the 
+      /* To avoid name collisions, add 700 to the
       ** extensions of WAL files so that they are 701, 702, 703, ....
       */
       iChunk += SQLITE_MULTIPLEX_WAL_8_3_OFFSET;
@@ -297,8 +297,8 @@ static sqlite3_file *multiplexSubOpen(
   sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;        /* Real VFS */
 
 #ifdef SQLITE_ENABLE_8_3_NAMES
-  /* If JOURNAL_8_3_OFFSET is set to (say) 400, then any overflow files are 
-  ** part of a database journal are named db.401, db.402, and so on. A 
+  /* If JOURNAL_8_3_OFFSET is set to (say) 400, then any overflow files are
+  ** part of a database journal are named db.401, db.402, and so on. A
   ** database may therefore not grow to larger than 400 chunks. Attempting
   ** to open chunk 401 indicates the database is full. */
   if( iChunk>=SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET ){
@@ -351,8 +351,8 @@ static sqlite3_file *multiplexSubOpen(
 
 /*
 ** Return the size, in bytes, of chunk number iChunk.  If that chunk
-** does not exist, then return 0.  This function does not distingish between
-** non-existant files and zero-length files.
+** does not exist, then return 0.  This function does not distinguish between
+** non-existent files and zero-length files.
 */
 static sqlite3_int64 multiplexSubSize(
   multiplexGroup *pGroup,    /* The multiplexor group */
@@ -367,7 +367,7 @@ static sqlite3_int64 multiplexSubSize(
   if( pSub==0 ) return 0;
   *rc = pSub->pMethods->xFileSize(pSub, &sz);
   return sz;
-}    
+}  
 
 /*
 ** This is the implementation of the multiplex_control() SQL function.
@@ -382,22 +382,22 @@ static void multiplexControlFunc(
   int op = 0;
   int iVal;
 
-  if( !db || argc!=2 ){ 
-    rc = SQLITE_ERROR; 
+  if( !db || argc!=2 ){
+    rc = SQLITE_ERROR;
   }else{
     /* extract params */
     op = sqlite3_value_int(argv[0]);
     iVal = sqlite3_value_int(argv[1]);
     /* map function op to file_control op */
     switch( op ){
-      case 1: 
-        op = MULTIPLEX_CTRL_ENABLE; 
+      case 1:
+        op = MULTIPLEX_CTRL_ENABLE;
         break;
-      case 2: 
-        op = MULTIPLEX_CTRL_SET_CHUNK_SIZE; 
+      case 2:
+        op = MULTIPLEX_CTRL_SET_CHUNK_SIZE;
         break;
-      case 3: 
-        op = MULTIPLEX_CTRL_SET_MAX_CHUNKS; 
+      case 3:
+        op = MULTIPLEX_CTRL_SET_MAX_CHUNKS;
         break;
       default:
         rc = SQLITE_NOTFOUND;
@@ -411,16 +411,16 @@ static void multiplexControlFunc(
 }
 
 /*
-** This is the entry point to register the auto-extension for the 
+** This is the entry point to register the auto-extension for the
 ** multiplex_control() function.
 */
 static int multiplexFuncInit(
-  sqlite3 *db, 
-  char **pzErrMsg, 
+  sqlite3 *db,
+  char **pzErrMsg,
   const sqlite3_api_routines *pApi
 ){
   int rc;
-  rc = sqlite3_create_function(db, "multiplex_control", 2, SQLITE_ANY, 
+  rc = sqlite3_create_function(db, "multiplex_control", 2, SQLITE_ANY,
       0, multiplexControlFunc, 0, 0);
   return rc;
 }
@@ -508,7 +508,7 @@ static int multiplexOpen(
     memset(pGroup, 0, sz);
     pMultiplexOpen->pGroup = pGroup;
     pGroup->bEnabled = (unsigned char)-1;
-    pGroup->bTruncate = (unsigned char)sqlite3_uri_boolean(zUri, "truncate", 
+    pGroup->bTruncate = (unsigned char)sqlite3_uri_boolean(zUri, "truncate",
                                    (flags & SQLITE_OPEN_MAIN_DB)==0);
     pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize",
                                         SQLITE_MULTIPLEX_CHUNK_SIZE);
@@ -551,11 +551,11 @@ static int multiplexOpen(
         if( sz64==0 ){
           if( flags & SQLITE_OPEN_MAIN_JOURNAL ){
             /* If opening a main journal file and the first chunk is zero
-            ** bytes in size, delete any subsequent chunks from the 
+            ** bytes in size, delete any subsequent chunks from the
             ** file-system. */
             int iChunk = 1;
             do {
-              rc = pOrigVfs->xAccess(pOrigVfs, 
+              rc = pOrigVfs->xAccess(pOrigVfs,
                   pGroup->aReal[iChunk].z, SQLITE_ACCESS_EXISTS, &bExists
               );
               if( rc==SQLITE_OK && bExists ){
@@ -573,8 +573,8 @@ static int multiplexOpen(
           **
           ** Or, if the first overflow file does not exist and the main file is
           ** larger than the chunk size, that means the chunk size is too small.
-          ** But we have no way of determining the intended chunk size, so 
-          ** just disable the multiplexor all togethre.
+          ** But we have no way of determining the intended chunk size, so
+          ** just disable the multiplexor all together.
           */
           rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z,
               SQLITE_ACCESS_EXISTS, &bExists);
@@ -618,7 +618,7 @@ static int multiplexDelete(
   rc = pOrigVfs->xDelete(pOrigVfs, zName, syncDir);
   if( rc==SQLITE_OK ){
     /* If the main chunk was deleted successfully, also delete any subsequent
-    ** chunks - starting with the last (highest numbered). 
+    ** chunks - starting with the last (highest numbered).
     */
     int nName = (int)strlen(zName);
     char *z;
@@ -695,7 +695,7 @@ static int multiplexCurrentTimeInt64(sqlite3_vfs *a, sqlite3_int64 *b){
 
 /* xClose requests get passed through to the original VFS.
 ** We loop over all open chunk handles and close them.
-** The group structure for this file is unlinked from 
+** The group structure for this file is unlinked from
 ** our list of groups and freed.
 */
 static int multiplexClose(sqlite3_file *pConn){
@@ -1008,7 +1008,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){
         /*
         ** PRAGMA multiplex_chunksize;
         **
-        ** Return the chunksize for the multiplexor, or no-op if the 
+        ** Return the chunksize for the multiplexor, or no-op if the
         ** multiplexor is not active.
         */
         if( sqlite3_stricmp(aFcntl[1],"multiplex_chunksize")==0
@@ -1138,8 +1138,8 @@ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){
 /*
 ** CAPI: Initialize the multiplex VFS shim - sqlite3_multiplex_initialize()
 **
-** Use the VFS named zOrigVfsName as the VFS that does the actual work.  
-** Use the default if zOrigVfsName==NULL.  
+** Use the VFS named zOrigVfsName as the VFS that does the actual work.
+** Use the default if zOrigVfsName==NULL.
 **
 ** The multiplex VFS shim is named "multiplex".  It will become the default
 ** VFS if makeDefault is non-zero.
@@ -1219,14 +1219,7 @@ int sqlite3_multiplex_shutdown(int eForce){
 
 /***************************** Test Code ***********************************/
 #ifdef SQLITE_TEST
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 extern const char *sqlite3ErrName(int);
 
 
@@ -1322,8 +1315,8 @@ static int SQLITE_TCLAPI test_multiplex_control(
   }
 
   if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){
-    Tcl_AppendResult(interp, "expected database handle, got \"", 0);
-    Tcl_AppendResult(interp, Tcl_GetString(objv[1]), "\"", 0);
+    Tcl_AppendResult(interp, "expected database handle, got \"", NULL);
+    Tcl_AppendResult(interp, Tcl_GetString(objv[1]), "\"", NULL);
     return TCL_ERROR;
   }else{
     db = *(sqlite3 **)cmdInfo.objClientData;
diff --git a/src/test_multiplex.h b/src/test_multiplex.h
index 790c778a35..065fac7ea9 100644
--- a/src/test_multiplex.h
+++ b/src/test_multiplex.h
@@ -13,8 +13,8 @@
 ** This file contains a VFS "shim" - a layer that sits in between the
 ** pager and the real VFS.
 **
-** This particular shim enforces a multiplex system on DB files.  
-** This shim shards/partitions a single DB file into smaller 
+** This particular shim enforces a multiplex system on DB files. 
+** This shim shards/partitions a single DB file into smaller
 ** "chunks" such that the total DB file size may exceed the maximum
 ** file size of the underlying file system.
 **
@@ -33,14 +33,14 @@
 **   shim.
 **
 ** MULTIPLEX_CTRL_SET_CHUNK_SIZE:
-**   This file control is used to set the maximum allowed chunk 
-**   size for a multiplex file set.  The chunk size should be 
+**   This file control is used to set the maximum allowed chunk
+**   size for a multiplex file set.  The chunk size should be
 **   a multiple of SQLITE_MAX_PAGE_SIZE, and will be rounded up
 **   if not.
 **
 ** MULTIPLEX_CTRL_SET_MAX_CHUNKS:
 **   This file control is used to set the maximum number of chunks
-**   allowed to be used for a mutliplex file set.
+**   allowed to be used for a multiplex file set.
 */
 #define MULTIPLEX_CTRL_ENABLE          214014
 #define MULTIPLEX_CTRL_SET_CHUNK_SIZE  214015
@@ -53,26 +53,26 @@ extern "C" {
 /*
 ** CAPI: Initialize the multiplex VFS shim - sqlite3_multiplex_initialize()
 **
-** Use the VFS named zOrigVfsName as the VFS that does the actual work.  
-** Use the default if zOrigVfsName==NULL.  
+** Use the VFS named zOrigVfsName as the VFS that does the actual work. 
+** Use the default if zOrigVfsName==NULL. 
 **
 ** The multiplex VFS shim is named "multiplex".  It will become the default
 ** VFS if makeDefault is non-zero.
 **
-** An auto-extension is registered which will make the function 
+** An auto-extension is registered which will make the function
 ** multiplex_control() available to database connections.  This
-** function gives access to the xFileControl interface of the 
+** function gives access to the xFileControl interface of the
 ** multiplex VFS shim.
 **
 ** SELECT multiplex_control(<op>,<val>);
-** 
+**
 **   <op>=1 MULTIPLEX_CTRL_ENABLE
 **   <val>=0 disable
 **   <val>=1 enable
-** 
+**
 **   <op>=2 MULTIPLEX_CTRL_SET_CHUNK_SIZE
 **   <val> int, chunk size
-** 
+**
 **   <op>=3 MULTIPLEX_CTRL_SET_MAX_CHUNKS
 **   <val> int, max chunks
 **
diff --git a/src/test_mutex.c b/src/test_mutex.c
index 8bd9ff85a0..de064de4c4 100644
--- a/src/test_mutex.c
+++ b/src/test_mutex.c
@@ -11,12 +11,7 @@
 *************************************************************************
 ** This file contains test logic for the sqlite3_mutex interfaces.
 */
-
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include "sqlite3.h"
 #include "sqliteInt.h"
 #include <stdlib.h>
@@ -45,7 +40,7 @@ struct sqlite3_mutex {
 /* State variables */
 static struct test_mutex_globals {
   int isInstalled;           /* True if installed */
-  int disableInit;           /* True to cause sqlite3_initalize() to fail */
+  int disableInit;           /* True to cause sqlite3_initialize() to fail */
   int disableTry;            /* True to force sqlite3_mutex_try() to fail */
   int isInit;                /* True if initialized */
   sqlite3_mutex_methods m;   /* Interface to "real" mutex system */
@@ -230,8 +225,8 @@ static int SQLITE_TCLAPI test_install_mutex_counters(
   assert(isInstall==0 || isInstall==1);
   assert(g.isInstalled==0 || g.isInstalled==1);
   if( isInstall==g.isInstalled ){
-    Tcl_AppendResult(interp, "mutex counters are ", 0);
-    Tcl_AppendResult(interp, isInstall?"already installed":"not installed", 0);
+    Tcl_AppendResult(interp, "mutex counters are ", NULL);
+    Tcl_AppendResult(interp, isInstall?"already installed":"not installed", NULL);
     return TCL_ERROR;
   }
 
diff --git a/src/test_osinst.c b/src/test_osinst.c
index 3e698c0324..e776d89e55 100644
--- a/src/test_osinst.c
+++ b/src/test_osinst.c
@@ -71,9 +71,8 @@
 
 #include "sqlite3.h"
 
-#include "os_setup.h"
-#if SQLITE_OS_WIN
-#  include "os_win.h"
+#ifdef _WIN32
+#include <windows.h>
 #endif
 
 #include <string.h>
@@ -219,14 +218,7 @@ static sqlite3_io_methods vfslog_io_methods = {
   vfslogShmUnmap                  /* xShmUnmap */
 };
 
-#if SQLITE_OS_UNIX && !defined(NO_GETTOD)
-#include <sys/time.h>
-static sqlite3_uint64 vfslog_time(){
-  struct timeval sTime;
-  gettimeofday(&sTime, 0);
-  return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000;
-}
-#elif SQLITE_OS_WIN
+#ifdef _WIN32
 #include <time.h>
 static sqlite3_uint64 vfslog_time(){
   FILETIME ft;
@@ -241,6 +233,13 @@ static sqlite3_uint64 vfslog_time(){
   /* ft is 100-nanosecond intervals, we want microseconds */
   return u64time /(sqlite3_uint64)10;
 }
+#elif !defined(NO_GETTOD)
+#include <sys/time.h>
+static sqlite3_uint64 vfslog_time(){
+  struct timeval sTime;
+  gettimeofday(&sTime, 0);
+  return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000;
+}
 #else
 static sqlite3_uint64 vfslog_time(){
   return 0;
@@ -1090,7 +1089,12 @@ int sqlite3_vfslog_register(sqlite3 *db){
     0,                            /* xRollback */
     0,                            /* xFindMethod */
     0,                            /* xRename */
-  };
+    0,                            /* xSavepoint */
+    0,                            /* xRelease */
+    0,                            /* xRollbackTo */
+    0,                            /* xShadowName */
+    0                             /* xIntegrity */
+ };
 
   sqlite3_create_module(db, "vfslog", &vfslog_module, 0);
   return SQLITE_OK;
@@ -1104,14 +1108,7 @@ int sqlite3_vfslog_register(sqlite3 *db){
 
 #if defined(SQLITE_TEST) || defined(TCLSH)
 
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 
 static int SQLITE_TCLAPI test_vfslog(
   void *clientData,
@@ -1148,7 +1145,7 @@ static int SQLITE_TCLAPI test_vfslog(
       zMsg = Tcl_GetString(objv[3]);
       rc = sqlite3_vfslog_annotate(zVfs, zMsg);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, "failed", 0);
+        Tcl_AppendResult(interp, "failed", (char*)0);
         return TCL_ERROR;
       }
       break;
@@ -1162,7 +1159,7 @@ static int SQLITE_TCLAPI test_vfslog(
       zVfs = Tcl_GetString(objv[2]);
       rc = sqlite3_vfslog_finalize(zVfs);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, "failed", 0);
+        Tcl_AppendResult(interp, "failed", (char*)0);
         return TCL_ERROR;
       }
       break;
@@ -1182,7 +1179,7 @@ static int SQLITE_TCLAPI test_vfslog(
       if( *zParent=='\0' ) zParent = 0;
       rc = sqlite3_vfslog_new(zVfs, zParent, zLog);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, "failed", 0);
+        Tcl_AppendResult(interp, "failed", (char*)0);
         return TCL_ERROR;
       }
       break;
diff --git a/src/test_pcache.c b/src/test_pcache.c
index 8fcfe7e26e..ceefa13e57 100644
--- a/src/test_pcache.c
+++ b/src/test_pcache.c
@@ -9,17 +9,17 @@
 **    May you share freely, never taking more than you give.
 **
 *************************************************************************
-** 
+**
 ** This file contains code used for testing the SQLite system.
 ** None of the code in this file goes into a deliverable build.
-** 
+**
 ** This file contains an application-defined pager cache
 ** implementation that can be plugged in in place of the
 ** default pcache.  This alternative pager cache will throw
 ** some errors that the default cache does not.
 **
 ** This pagecache implementation is designed for simplicity
-** not speed.  
+** not speed.
 */
 #include "sqlite3.h"
 #include <string.h>
@@ -36,7 +36,7 @@ struct testpcacheGlobalType {
   int nInstance;            /* Number of current instances */
   unsigned discardChance;   /* Chance of discarding on an unpin (0-100) */
   unsigned prngSeed;        /* Seed for the PRNG */
-  unsigned highStress;      /* Call xStress agressively */
+  unsigned highStress;      /* Call xStress aggressively */
 };
 static testpcacheGlobalType testpcacheGlobal;
 
@@ -99,7 +99,7 @@ static void testpcacheShutdown(void *pArg){
 */
 typedef struct testpcache testpcache;
 struct testpcache {
-  int szPage;               /* Size of each page.  Multiple of 8. */
+  sqlite3_int64 szPage;     /* Size of each page.  Multiple of 8. */
   int szExtra;              /* Size of extra data that accompanies each page */
   int bPurgeable;           /* True if the page cache is purgeable */
   int nFree;                /* Number of unused slots in a[] */
@@ -131,8 +131,8 @@ static unsigned testpcacheRandom(testpcache *p){
 ** Allocate a new page cache instance.
 */
 static sqlite3_pcache *testpcacheCreate(
-  int szPage, 
-  int szExtra, 
+  int szPage,
+  int szExtra,
   int bPurgeable
 ){
   int nMem;
@@ -141,6 +141,7 @@ static sqlite3_pcache *testpcacheCreate(
   int i;
   assert( testpcacheGlobal.pDummy!=0 );
   szPage = (szPage+7)&~7;
+  szExtra = (szPage+7)&~7;
   nMem = sizeof(testpcache) + TESTPCACHE_NPAGE*(szPage+szExtra);
   p = sqlite3_malloc( nMem );
   if( p==0 ) return 0;
@@ -225,7 +226,7 @@ static sqlite3_pcache_page *testpcacheFetch(
     return 0;
   }
 
-  /* Do not allocate if highStress is enabled and createFlag is not 2.  
+  /* Do not allocate if highStress is enabled and createFlag is not 2.
   **
   ** The highStress setting causes pagerStress() to be called much more
   ** often, which exercises the pager logic more intensely.
@@ -428,7 +429,7 @@ void installTestPCache(
   int installFlag,            /* True to install.  False to uninstall. */
   unsigned discardChance,     /* 0-100.  Chance to discard on unpin */
   unsigned prngSeed,          /* Seed for the PRNG */
-  unsigned highStress         /* Call xStress agressively */
+  unsigned highStress         /* Call xStress aggressively */
 ){
   static const sqlite3_pcache_methods2 testPcache = {
     1,
diff --git a/src/test_quota.c b/src/test_quota.c
index e87e9772f6..d2f9cddd11 100644
--- a/src/test_quota.c
+++ b/src/test_quota.c
@@ -44,14 +44,12 @@
 #define sqlite3_mutex_notheld(X)  ((void)(X),1)
 #endif /* SQLITE_THREADSAFE==0 */
 
-#include "os_setup.h"
 
-#if SQLITE_OS_UNIX
-# include <unistd.h>
-#endif
-#if SQLITE_OS_WIN
-# include "os_win.h"
+#ifdef _WIN32
+# include <windows.h>
 # include <io.h>
+#else
+# include <unistd.h>
 #endif
 
 
@@ -130,7 +128,7 @@ struct quota_FILE {
   FILE *f;                /* Open stdio file pointer */
   sqlite3_int64 iOfst;    /* Current offset into the file */
   quotaFile *pFile;       /* The file record in the quota system */
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   char *zMbcsName;        /* Full MBCS pathname of the file */
 #endif
 };
@@ -375,7 +373,7 @@ static quotaFile *quotaFindFile(
 ** used to store the returned pointer when done.
 */
 static char *quota_utf8_to_mbcs(const char *zUtf8){
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   size_t n;          /* Bytes in zUtf8 */
   int nWide;         /* number of UTF-16 characters */
   int nMbcs;         /* Bytes of MBCS */
@@ -389,7 +387,11 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){
   zTmpWide = (LPWSTR)sqlite3_malloc( (nWide+1)*sizeof(zTmpWide[0]) );
   if( zTmpWide==0 ) return 0;
   MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zTmpWide, nWide);
+#ifdef SQLITE_OS_WINRT
+  codepage = CP_ACP;
+#else
   codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP;
+#endif
   nMbcs = WideCharToMultiByte(codepage, 0, zTmpWide, nWide, 0, 0, 0, 0);
   zMbcs = nMbcs ? (char*)sqlite3_malloc( nMbcs+1 ) : 0;
   if( zMbcs ){
@@ -406,7 +408,7 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){
 ** Deallocate any memory allocated by quota_utf8_to_mbcs().
 */
 static void quota_mbcs_free(char *zOld){
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   sqlite3_free(zOld);
 #else
   /* No-op on unix */
@@ -966,7 +968,7 @@ quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){
   }
   quotaLeave();
   sqlite3_free(zFull);
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   p->zMbcsName = zFullTranslated;
 #endif
   return p;
@@ -999,7 +1001,7 @@ size_t sqlite3_quota_fwrite(
   const void *pBuf,      /* Take content to write from here */
   size_t size,           /* Size of each element */
   size_t nmemb,          /* Number of elements */
-  quota_FILE *p          /* Write to this quota_FILE objecct */
+  quota_FILE *p          /* Write to this quota_FILE object */
 ){
   sqlite3_int64 iOfst;
   sqlite3_int64 iEnd;
@@ -1069,7 +1071,7 @@ int sqlite3_quota_fclose(quota_FILE *p){
     }
     quotaLeave();
   }
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   quota_mbcs_free(p->zMbcsName);
 #endif
   sqlite3_free(p);
@@ -1083,11 +1085,10 @@ int sqlite3_quota_fflush(quota_FILE *p, int doFsync){
   int rc;
   rc = fflush(p->f);
   if( rc==0 && doFsync ){
-#if SQLITE_OS_UNIX
-    rc = fsync(fileno(p->f));
-#endif
-#if SQLITE_OS_WIN
+#ifdef _WIN32
     rc = _commit(_fileno(p->f));
+#else
+    rc = fsync(fileno(p->f));
 #endif
   }
   return rc!=0;
@@ -1139,17 +1140,16 @@ int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){
     pGroup->iSize += szNew - pFile->iSize;
     quotaLeave();
   }
-#if SQLITE_OS_UNIX
-  rc = ftruncate(fileno(p->f), szNew);
-#endif
-#if SQLITE_OS_WIN
-#  if defined(__MINGW32__) && defined(SQLITE_TEST)
+#ifdef _WIN32
+#  if defined(__MSVCRT__) && defined(SQLITE_TEST)
      /* _chsize_s() is missing from MingW (as of 2012-11-06).  Use
      ** _chsize() as a work-around for testing purposes. */
      rc = _chsize(_fileno(p->f), (long)szNew);
 #  else
      rc = _chsize_s(_fileno(p->f), szNew);
 #  endif
+#else
+  rc = ftruncate(fileno(p->f), szNew);
 #endif
   if( pFile && rc==0 ){
     quotaGroup *pGroup = pFile->pGroup;
@@ -1168,13 +1168,12 @@ int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){
 */
 int sqlite3_quota_file_mtime(quota_FILE *p, time_t *pTime){
   int rc;
-#if SQLITE_OS_UNIX
-  struct stat buf;
-  rc = fstat(fileno(p->f), &buf);
-#endif
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   struct _stati64 buf;
   rc = _stati64(p->zMbcsName, &buf);
+#else
+  struct stat buf;
+  rc = fstat(fileno(p->f), &buf);
 #endif
   if( rc==0 ) *pTime = buf.st_mtime;
   return rc;
@@ -1186,13 +1185,12 @@ int sqlite3_quota_file_mtime(quota_FILE *p, time_t *pTime){
 */
 sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE *p){
   int rc;
-#if SQLITE_OS_UNIX
-  struct stat buf;
-  rc = fstat(fileno(p->f), &buf);
-#endif
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   struct _stati64 buf;
   rc = _stati64(p->zMbcsName, &buf);
+#else
+  struct stat buf;
+  rc = fstat(fileno(p->f), &buf);
 #endif
   return rc==0 ? buf.st_size : -1;
 }
@@ -1278,14 +1276,7 @@ int sqlite3_quota_remove(const char *zFilename){
 
 /***************************** Test Code ***********************************/
 #ifdef SQLITE_TEST
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 
 /*
 ** Argument passed to a TCL quota-over-limit callback.
@@ -1420,7 +1411,7 @@ static int SQLITE_TCLAPI test_quota_set(
   Tcl_Obj *pScript;               /* Tcl script to invoke to increase quota */
   int rc;                         /* Value returned by quota_set() */
   TclQuotaCallback *p;            /* Callback object */
-  int nScript;                    /* Length of callback script */
+  Tcl_Size nScript;               /* Length of callback script */
   void (*xDestroy)(void*);        /* Optional destructor for pArg */
   void (*xCallback)(const char *, sqlite3_int64 *, sqlite3_int64, void *);
 
@@ -1876,7 +1867,7 @@ static int SQLITE_TCLAPI test_quota_glob(
   Tcl_Obj *CONST objv[]
 ){
   const char *zPattern;          /* The glob pattern */
-  const char *zText;             /* Text to compare agains the pattern */
+  const char *zText;             /* Text to compare against the pattern */
   int rc;
   if( objc!=3 ){
     Tcl_WrongNumArgs(interp, 1, objv, "PATTERN TEXT");
diff --git a/src/test_rtree.c b/src/test_rtree.c
index 0c6dbf3cd7..53af6e5cfe 100644
--- a/src/test_rtree.c
+++ b/src/test_rtree.c
@@ -14,11 +14,7 @@
 */
 
 #include "sqlite3.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 /* Solely for the UNUSED_PARAMETER() macro. */
 #include "sqliteInt.h"
@@ -357,11 +353,7 @@ static int bfs_query_func(sqlite3_rtree_query_info *p){
 *************************************************************************/
 
 #include <assert.h>
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 typedef struct Cube Cube;
 struct Cube {
diff --git a/src/test_schema.c b/src/test_schema.c
index d2cae7f2aa..660d21ea4e 100644
--- a/src/test_schema.c
+++ b/src/test_schema.c
@@ -36,11 +36,7 @@
 */
 #ifdef SQLITE_TEST
 #  include "sqliteInt.h"
-#  if defined(INCLUDE_SQLITE_TCL_H)
-#    include "sqlite_tcl.h"
-#  else
-#    include "tcl.h"
-#  endif
+#  include "tclsqlite.h"
 #else
 #  include "sqlite3ext.h"
   SQLITE_EXTENSION_INIT1
@@ -292,6 +288,11 @@ static sqlite3_module schemaModule = {
   0,                           /* xRollback */
   0,                           /* xFindMethod */
   0,                           /* xRename */
+  0,                           /* xSavepoint */
+  0,                           /* xRelease */
+  0,                           /* xRollbackTo */
+  0,                           /* xShadowName */
+  0                            /* xIntegrity */
 };
 
 #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */
diff --git a/src/test_sqllog.c b/src/test_sqllog.c
index 9ae0e50685..5abf59a8bf 100644
--- a/src/test_sqllog.c
+++ b/src/test_sqllog.c
@@ -84,7 +84,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 static int getProcessId(void){
-#if SQLITE_OS_WIN
+#ifdef _WIN32
   return (int)_getpid();
 #else
   return (int)getpid();
diff --git a/src/test_superlock.c b/src/test_superlock.c
index 45d0d623a0..82997927c4 100644
--- a/src/test_superlock.c
+++ b/src/test_superlock.c
@@ -256,14 +256,7 @@ int sqlite3demo_superlock(
 
 #ifdef SQLITE_TEST
 
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 
 struct InterpAndScript {
   Tcl_Interp *interp;
@@ -345,7 +338,7 @@ static int SQLITE_TCLAPI superlock_cmd(
   if( rc!=SQLITE_OK ){
     extern const char *sqlite3ErrStr(int);
     Tcl_ResetResult(interp);
-    Tcl_AppendResult(interp, sqlite3ErrStr(rc), 0);
+    Tcl_AppendResult(interp, sqlite3ErrStr(rc), NULL);
     return TCL_ERROR;
   }
 
diff --git a/src/test_syscall.c b/src/test_syscall.c
index 3cd1034d3f..35c303f8ee 100644
--- a/src/test_syscall.c
+++ b/src/test_syscall.c
@@ -76,11 +76,7 @@
 
 #include "sqliteInt.h"
 #include "sqlite3.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
@@ -197,7 +193,7 @@ static int tsIsFail(void){
 */
 static int tsErrno(const char *zFunc){
   int i;
-  int nFunc = strlen(zFunc);
+  size_t nFunc = strlen(zFunc);
   for(i=0; aSyscall[i].zName; i++){
     if( strlen(aSyscall[i].zName)!=nFunc ) continue;
     if( memcmp(aSyscall[i].zName, zFunc, nFunc) ) continue;
@@ -429,7 +425,7 @@ static int SQLITE_TCLAPI test_syscall_install(
   Tcl_Obj *CONST objv[]
 ){
   sqlite3_vfs *pVfs; 
-  int nElem;
+  Tcl_Size nElem;
   int i;
   Tcl_Obj **apElem;
 
@@ -442,7 +438,7 @@ static int SQLITE_TCLAPI test_syscall_install(
   }
   pVfs = sqlite3_vfs_find(0);
 
-  for(i=0; i<nElem; i++){
+  for(i=0; i<(int)nElem; i++){
     int iCall;
     int rc = Tcl_GetIndexFromObjStruct(interp, 
         apElem[i], aSyscall, sizeof(aSyscall[0]), "system-call", 0, &iCall
@@ -502,7 +498,7 @@ static int SQLITE_TCLAPI test_syscall_reset(
     rc = pVfs->xSetSystemCall(pVfs, 0, 0);
     for(i=0; aSyscall[i].zName; i++) aSyscall[i].xOrig = 0;
   }else{
-    int nFunc;
+    Tcl_Size nFunc;
     char *zFunc = Tcl_GetStringFromObj(objv[2], &nFunc);
     rc = pVfs->xSetSystemCall(pVfs, Tcl_GetString(objv[2]), 0);
     for(i=0; rc==SQLITE_OK && aSyscall[i].zName; i++){
@@ -690,7 +686,7 @@ static int SQLITE_TCLAPI test_syscall_pagesize(
     }
   }else{
     if( pgsz<512 || (pgsz & (pgsz-1)) ){
-      Tcl_AppendResult(interp, "pgsz out of range", 0);
+      Tcl_AppendResult(interp, "pgsz out of range", NULL);
       return TCL_ERROR;
     }
     gSyscall.orig_getpagesize = pVfs->xGetSystemCall(pVfs, "getpagesize");
@@ -733,7 +729,7 @@ static int SQLITE_TCLAPI test_syscall(
     return TCL_ERROR;
   }
   if( pVfs->iVersion<3 || pVfs->xSetSystemCall==0 ){
-    Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", 0);
+    Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", NULL);
     rc = TCL_ERROR;
   }else{
     rc = Tcl_GetIndexFromObjStruct(interp, 
diff --git a/src/test_tclsh.c b/src/test_tclsh.c
index 32aee42675..989cb97a62 100644
--- a/src/test_tclsh.c
+++ b/src/test_tclsh.c
@@ -20,14 +20,7 @@
 ** in an effort to keep the tclsqlite.c file pure.
 */
 #include "sqlite3.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-# include "sqlite_tcl.h"
-#else
-# include "tcl.h"
-# ifndef SQLITE_TCLAPI
-#  define SQLITE_TCLAPI
-# endif
-#endif
+#include "tclsqlite.h"
 
 /* Needed for the setrlimit() system call on unix */
 #if defined(unix)
@@ -66,7 +59,6 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   extern int Sqlitetest6_Init(Tcl_Interp*);
   extern int Sqlitetest8_Init(Tcl_Interp*);
   extern int Sqlitetest9_Init(Tcl_Interp*);
-  extern int Sqlitetestasync_Init(Tcl_Interp*);
   extern int Sqlitetest_autoext_Init(Tcl_Interp*);
   extern int Sqlitetest_blob_Init(Tcl_Interp*);
   extern int Sqlitetest_demovfs_Init(Tcl_Interp *);
@@ -108,6 +100,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   extern int Sqlitetest_window_Init(Tcl_Interp *);
   extern int Sqlitetestvdbecov_Init(Tcl_Interp *);
   extern int TestRecover_Init(Tcl_Interp*);
+  extern int Sqlitetestintck_Init(Tcl_Interp*);
 
   Tcl_CmdInfo cmdInfo;
 
@@ -137,7 +130,6 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   Sqlitetest6_Init(interp);
   Sqlitetest8_Init(interp);
   Sqlitetest9_Init(interp);
-  Sqlitetestasync_Init(interp);
   Sqlitetest_autoext_Init(interp);
   Sqlitetest_blob_Init(interp);
   Sqlitetest_demovfs_Init(interp);
@@ -175,6 +167,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){
   Sqlitetest_window_Init(interp);
   Sqlitetestvdbecov_Init(interp);
   TestRecover_Init(interp);
+  Sqlitetestintck_Init(interp);
 
   Tcl_CreateObjCommand(
       interp, "load_testfixture_extensions", load_testfixture_extensions,0,0
diff --git a/src/test_tclvar.c b/src/test_tclvar.c
index bf99a8eadb..6299960a6c 100644
--- a/src/test_tclvar.c
+++ b/src/test_tclvar.c
@@ -36,11 +36,7 @@
 ** according to "fullname" and "value" only.
 */
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <stdlib.h>
 #include <string.h>
 
@@ -72,8 +68,8 @@ struct tclvar_cursor {
 
   Tcl_Obj *pList1;     /* Result of [info vars ?pattern?] */
   Tcl_Obj *pList2;     /* Result of [array names [lindex $pList1 $i1]] */
-  int i1;              /* Current item in pList1 */
-  int i2;              /* Current item (if any) in pList2 */
+  Tcl_Size i1;              /* Current item in pList1 */
+  Tcl_Size i2;              /* Current item (if any) in pList2 */
 };
 
 /* Methods for the tclvar module */
@@ -150,7 +146,7 @@ static int next2(Tcl_Interp *interp, tclvar_cursor *pCur, Tcl_Obj *pObj){
       Tcl_IncrRefCount(pCur->pList2);
       assert( pCur->i2==0 );
     }else{
-      int n = 0;
+      Tcl_Size n = 0;
       pCur->i2++;
       Tcl_ListObjLength(0, pCur->pList2, &n);
       if( pCur->i2>=n ){
@@ -167,7 +163,7 @@ static int next2(Tcl_Interp *interp, tclvar_cursor *pCur, Tcl_Obj *pObj){
 
 static int tclvarNext(sqlite3_vtab_cursor *cur){
   Tcl_Obj *pObj;
-  int n = 0;
+  Tcl_Size n = 0;
   int ok = 0;
 
   tclvar_cursor *pCur = (tclvar_cursor *)cur;
@@ -487,6 +483,11 @@ static sqlite3_module tclvarModule = {
   0,                           /* xRollback */
   0,                           /* xFindMethod */
   0,                           /* xRename */
+  0,                           /* xSavepoint */
+  0,                           /* xRelease */
+  0,                           /* xRollbackTo */
+  0,                           /* xShadowName */
+  0                            /* xIntegrity */
 };
 
 /*
diff --git a/src/test_thread.c b/src/test_thread.c
index 4ba1d96f84..62d9227caa 100644
--- a/src/test_thread.c
+++ b/src/test_thread.c
@@ -16,11 +16,7 @@
 */
 
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 #if SQLITE_THREADSAFE
 
@@ -94,7 +90,7 @@ static int SQLITE_TCLAPI tclScriptEvent(Tcl_Event *evPtr, int flags){
 static void postToParent(SqlThread *p, Tcl_Obj *pScript){
   EvalEvent *pEvent;
   char *zMsg;
-  int nMsg;
+  Tcl_Size nMsg;
 
   zMsg = Tcl_GetStringFromObj(pScript, &nMsg); 
   pEvent = (EvalEvent *)ckalloc(sizeof(EvalEvent)+nMsg+1);
@@ -181,8 +177,8 @@ static int SQLITE_TCLAPI sqlthread_spawn(
   SqlThread *pNew;
   int rc;
 
-  int nVarname; char *zVarname;
-  int nScript; char *zScript;
+  Tcl_Size nVarname; char *zVarname;
+  Tcl_Size nScript; char *zScript;
 
   /* Parameters for thread creation */
   const int nStack = TCL_THREAD_STACK_DEFAULT;
@@ -205,7 +201,7 @@ static int SQLITE_TCLAPI sqlthread_spawn(
 
   rc = Tcl_CreateThread(&x, tclScriptThread, (void *)pNew, nStack, flags);
   if( rc!=TCL_OK ){
-    Tcl_AppendResult(interp, "Error in Tcl_CreateThread()", 0);
+    Tcl_AppendResult(interp, "Error in Tcl_CreateThread()", NULL);
     ckfree((char *)pNew);
     return TCL_ERROR;
   }
@@ -232,14 +228,14 @@ static int SQLITE_TCLAPI sqlthread_parent(
 ){
   EvalEvent *pEvent;
   char *zMsg;
-  int nMsg;
+  Tcl_Size nMsg;
   SqlThread *p = (SqlThread *)clientData;
 
   assert(objc==3);
   UNUSED_PARAMETER(objc);
 
   if( p==0 ){
-    Tcl_AppendResult(interp, "no parent thread", 0);
+    Tcl_AppendResult(interp, "no parent thread", NULL);
     return TCL_ERROR;
   }
 
@@ -309,7 +305,7 @@ static int SQLITE_TCLAPI sqlthread_open(
   sqlite3_busy_handler(db, xBusy, 0);
   
   if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR;
-  Tcl_AppendResult(interp, zBuf, 0);
+  Tcl_AppendResult(interp, zBuf, NULL);
 
   return TCL_OK;
 }
@@ -636,13 +632,13 @@ static int SQLITE_TCLAPI blocking_prepare_v2_proc(
   if( rc!=SQLITE_OK ){
     assert( pStmt==0 );
     sqlite3_snprintf(sizeof(zBuf), zBuf, "%s ", (char *)sqlite3ErrName(rc));
-    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL);
     return TCL_ERROR;
   }
 
   if( pStmt ){
     if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
-    Tcl_AppendResult(interp, zBuf, 0);
+    Tcl_AppendResult(interp, zBuf, NULL);
   }
   return TCL_OK;
 }
diff --git a/src/test_vdbecov.c b/src/test_vdbecov.c
index a001b1df0a..283936aeb7 100644
--- a/src/test_vdbecov.c
+++ b/src/test_vdbecov.c
@@ -15,11 +15,7 @@
 
 #include "sqlite3.h"
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 #ifdef SQLITE_VDBE_COVERAGE
 
diff --git a/src/test_vfs.c b/src/test_vfs.c
index 312e1a1be6..0d90a53a53 100644
--- a/src/test_vfs.c
+++ b/src/test_vfs.c
@@ -28,11 +28,7 @@
 
 #include "sqlite3.h"
 #include "sqliteInt.h"
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 
 typedef struct Testvfs Testvfs;
 typedef struct TestvfsShm TestvfsShm;
@@ -134,8 +130,9 @@ struct Testvfs {
 #define TESTVFS_LOCK_MASK         0x00040000
 #define TESTVFS_CKLOCK_MASK       0x00080000
 #define TESTVFS_FCNTL_MASK        0x00100000
+#define TESTVFS_SLEEP_MASK        0x00200000
 
-#define TESTVFS_ALL_MASK          0x001FFFFF
+#define TESTVFS_ALL_MASK          0x003FFFFF
 
 
 #define TESTVFS_MAX_PAGES 1024
@@ -817,6 +814,10 @@ static int tvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
 ** actually slept.
 */
 static int tvfsSleep(sqlite3_vfs *pVfs, int nMicro){
+  Testvfs *p = (Testvfs *)pVfs->pAppData;
+  if( p->pScript && (p->mask&TESTVFS_SLEEP_MASK) ){
+    tvfsExecTcl(p, "xSleep", Tcl_NewIntObj(nMicro), 0, 0, 0);
+  }
   return sqlite3OsSleep(PARENTVFS(pVfs), nMicro);
 }
 
@@ -1137,7 +1138,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
       );
       if( rc!=SQLITE_OK ){
         Tcl_AppendResult(interp, "failed to get full path: ",
-                         Tcl_GetString(objv[2]), 0);
+                         Tcl_GetString(objv[2]), NULL);
         ckfree(zName);
         return TCL_ERROR;
       }
@@ -1146,19 +1147,19 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
       }
       ckfree(zName);
       if( !pBuffer ){
-        Tcl_AppendResult(interp, "no such file: ", Tcl_GetString(objv[2]), 0);
+        Tcl_AppendResult(interp, "no such file: ", Tcl_GetString(objv[2]), NULL);
         return TCL_ERROR;
       }
       if( objc==4 ){
-        int n;
+        Tcl_Size n;
         u8 *a = Tcl_GetByteArrayFromObj(objv[3], &n);
         int pgsz = pBuffer->pgsz;
         if( pgsz==0 ) pgsz = 65536;
-        for(i=0; i*pgsz<n; i++){
+        for(i=0; i*pgsz<(int)n; i++){
           int nByte = pgsz;
           tvfsAllocPage(pBuffer, i, pgsz);
           if( n-i*pgsz<pgsz ){
-            nByte = n;
+            nByte = (int)n;
           }
           memcpy(pBuffer->aPage[i], &a[i*pgsz], nByte);
         }
@@ -1201,9 +1202,10 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
         { "xLock",              TESTVFS_LOCK_MASK },
         { "xCheckReservedLock", TESTVFS_CKLOCK_MASK },
         { "xFileControl",       TESTVFS_FCNTL_MASK },
+        { "xSleep",             TESTVFS_SLEEP_MASK },
       };
       Tcl_Obj **apElem = 0;
-      int nElem = 0;
+      Tcl_Size nElem = 0;
       int mask = 0;
       if( objc!=3 ){
         Tcl_WrongNumArgs(interp, 2, objv, "LIST");
@@ -1213,7 +1215,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
         return TCL_ERROR;
       }
       Tcl_ResetResult(interp);
-      for(i=0; i<nElem; i++){
+      for(i=0; i<(int)nElem; i++){
         int iMethod;
         char *zElem = Tcl_GetString(apElem[i]);
         for(iMethod=0; iMethod<ArraySize(vfsmethod); iMethod++){
@@ -1223,7 +1225,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
           }
         }
         if( iMethod==ArraySize(vfsmethod) ){
-          Tcl_AppendResult(interp, "unknown method: ", zElem, 0);
+          Tcl_AppendResult(interp, "unknown method: ", zElem, NULL);
           return TCL_ERROR;
         }
       }
@@ -1239,7 +1241,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
     */
     case CMD_SCRIPT: {
       if( objc==3 ){
-        int nByte;
+        Tcl_Size nByte;
         if( p->pScript ){
           Tcl_DecrRefCount(p->pScript);
           p->pScript = 0;
@@ -1337,13 +1339,13 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
         int j;
         int iNew = 0;
         Tcl_Obj **flags = 0;
-        int nFlags = 0;
+        Tcl_Size nFlags = 0;
 
         if( Tcl_ListObjGetElements(interp, objv[2], &nFlags, &flags) ){
           return TCL_ERROR;
         }
 
-        for(j=0; j<nFlags; j++){
+        for(j=0; j<(int)nFlags; j++){
           int idx = 0;
           if( Tcl_GetIndexFromObjStruct(interp, flags[j], aFlag, 
                 sizeof(aFlag[0]), "flag", 0, &idx) 
@@ -1351,7 +1353,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd(
             return TCL_ERROR;
           }
           if( aFlag[idx].iValue<0 && nFlags>1 ){
-            Tcl_AppendResult(interp, "bad flags: ", Tcl_GetString(objv[2]), 0);
+            Tcl_AppendResult(interp, "bad flags: ", Tcl_GetString(objv[2]), NULL);
             return TCL_ERROR;
           }
           iNew |= aFlag[idx].iValue;
@@ -1491,7 +1493,7 @@ static int SQLITE_TCLAPI testvfs_cmd(
 
   if( objc<2 || 0!=(objc%2) ) goto bad_args;
   for(i=2; i<objc; i += 2){
-    int nSwitch;
+    Tcl_Size nSwitch;
     char *zSwitch;
     zSwitch = Tcl_GetStringFromObj(objv[i], &nSwitch); 
 
@@ -1671,7 +1673,7 @@ static int SQLITE_TCLAPI test_vfs_set_readmark(
     return TCL_ERROR;
   }
   if( pShm==0 ){
-    Tcl_AppendResult(interp, "*-shm is not yet mapped", 0);
+    Tcl_AppendResult(interp, "*-shm is not yet mapped", NULL);
     return TCL_ERROR;
   }
   aShm = (u32*)pShm;
diff --git a/src/test_windirent.c b/src/test_windirent.c
deleted file mode 100644
index 62165c4bea..0000000000
--- a/src/test_windirent.c
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
-** 2015 November 30
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file contains code to implement most of the opendir() family of
-** POSIX functions on Win32 using the MSVCRT.
-*/
-
-#if defined(_WIN32) && defined(_MSC_VER)
-#include "test_windirent.h"
-
-/*
-** Implementation of the POSIX getenv() function using the Win32 API.
-** This function is not thread-safe.
-*/
-const char *windirent_getenv(
-  const char *name
-){
-  static char value[32768]; /* Maximum length, per MSDN */
-  DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */
-  DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */
-
-  memset(value, 0, sizeof(value));
-  dwRet = GetEnvironmentVariableA(name, value, dwSize);
-  if( dwRet==0 || dwRet>dwSize ){
-    /*
-    ** The function call to GetEnvironmentVariableA() failed -OR-
-    ** the buffer is not large enough.  Either way, return NULL.
-    */
-    return 0;
-  }else{
-    /*
-    ** The function call to GetEnvironmentVariableA() succeeded
-    ** -AND- the buffer contains the entire value.
-    */
-    return value;
-  }
-}
-
-/*
-** Implementation of the POSIX opendir() function using the MSVCRT.
-*/
-LPDIR opendir(
-  const char *dirname
-){
-  struct _finddata_t data;
-  LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR));
-  SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]);
-
-  if( dirp==NULL ) return NULL;
-  memset(dirp, 0, sizeof(DIR));
-
-  /* TODO: Remove this if Unix-style root paths are not used. */
-  if( sqlite3_stricmp(dirname, "/")==0 ){
-    dirname = windirent_getenv("SystemDrive");
-  }
-
-  memset(&data, 0, sizeof(struct _finddata_t));
-  _snprintf(data.name, namesize, "%s\\*", dirname);
-  dirp->d_handle = _findfirst(data.name, &data);
-
-  if( dirp->d_handle==BAD_INTPTR_T ){
-    closedir(dirp);
-    return NULL;
-  }
-
-  /* TODO: Remove this block to allow hidden and/or system files. */
-  if( is_filtered(data) ){
-next:
-
-    memset(&data, 0, sizeof(struct _finddata_t));
-    if( _findnext(dirp->d_handle, &data)==-1 ){
-      closedir(dirp);
-      return NULL;
-    }
-
-    /* TODO: Remove this block to allow hidden and/or system files. */
-    if( is_filtered(data) ) goto next;
-  }
-
-  dirp->d_first.d_attributes = data.attrib;
-  strncpy(dirp->d_first.d_name, data.name, NAME_MAX);
-  dirp->d_first.d_name[NAME_MAX] = '\0';
-
-  return dirp;
-}
-
-/*
-** Implementation of the POSIX readdir() function using the MSVCRT.
-*/
-LPDIRENT readdir(
-  LPDIR dirp
-){
-  struct _finddata_t data;
-
-  if( dirp==NULL ) return NULL;
-
-  if( dirp->d_first.d_ino==0 ){
-    dirp->d_first.d_ino++;
-    dirp->d_next.d_ino++;
-
-    return &dirp->d_first;
-  }
-
-next:
-
-  memset(&data, 0, sizeof(struct _finddata_t));
-  if( _findnext(dirp->d_handle, &data)==-1 ) return NULL;
-
-  /* TODO: Remove this block to allow hidden and/or system files. */
-  if( is_filtered(data) ) goto next;
-
-  dirp->d_next.d_ino++;
-  dirp->d_next.d_attributes = data.attrib;
-  strncpy(dirp->d_next.d_name, data.name, NAME_MAX);
-  dirp->d_next.d_name[NAME_MAX] = '\0';
-
-  return &dirp->d_next;
-}
-
-/*
-** Implementation of the POSIX readdir_r() function using the MSVCRT.
-*/
-INT readdir_r(
-  LPDIR dirp,
-  LPDIRENT entry,
-  LPDIRENT *result
-){
-  struct _finddata_t data;
-
-  if( dirp==NULL ) return EBADF;
-
-  if( dirp->d_first.d_ino==0 ){
-    dirp->d_first.d_ino++;
-    dirp->d_next.d_ino++;
-
-    entry->d_ino = dirp->d_first.d_ino;
-    entry->d_attributes = dirp->d_first.d_attributes;
-    strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX);
-    entry->d_name[NAME_MAX] = '\0';
-
-    *result = entry;
-    return 0;
-  }
-
-next:
-
-  memset(&data, 0, sizeof(struct _finddata_t));
-  if( _findnext(dirp->d_handle, &data)==-1 ){
-    *result = NULL;
-    return ENOENT;
-  }
-
-  /* TODO: Remove this block to allow hidden and/or system files. */
-  if( is_filtered(data) ) goto next;
-
-  entry->d_ino = (ino_t)-1; /* not available */
-  entry->d_attributes = data.attrib;
-  strncpy(entry->d_name, data.name, NAME_MAX);
-  entry->d_name[NAME_MAX] = '\0';
-
-  *result = entry;
-  return 0;
-}
-
-/*
-** Implementation of the POSIX closedir() function using the MSVCRT.
-*/
-INT closedir(
-  LPDIR dirp
-){
-  INT result = 0;
-
-  if( dirp==NULL ) return EINVAL;
-
-  if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){
-    result = _findclose(dirp->d_handle);
-  }
-
-  sqlite3_free(dirp);
-  return result;
-}
-
-#endif /* defined(WIN32) && defined(_MSC_VER) */
diff --git a/src/test_windirent.h b/src/test_windirent.h
deleted file mode 100644
index ada5322530..0000000000
--- a/src/test_windirent.h
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
-** 2015 November 30
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file contains declarations for most of the opendir() family of
-** POSIX functions on Win32 using the MSVCRT.
-*/
-
-#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H)
-#define SQLITE_WINDIRENT_H
-
-/*
-** We need several data types from the Windows SDK header.
-*/
-
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-
-#include "windows.h"
-
-/*
-** We need several support functions from the SQLite core.
-*/
-
-#include "sqlite3.h"
-
-/*
-** We need several things from the ANSI and MSVCRT headers.
-*/
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <io.h>
-#include <limits.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-
-/*
-** We may need several defines that should have been in "sys/stat.h".
-*/
-
-#ifndef S_ISREG
-#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG)
-#endif
-
-#ifndef S_ISDIR
-#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
-#endif
-
-#ifndef S_ISLNK
-#define S_ISLNK(mode) (0)
-#endif
-
-/*
-** We may need to provide the "mode_t" type.
-*/
-
-#ifndef MODE_T_DEFINED
-  #define MODE_T_DEFINED
-  typedef unsigned short mode_t;
-#endif
-
-/*
-** We may need to provide the "ino_t" type.
-*/
-
-#ifndef INO_T_DEFINED
-  #define INO_T_DEFINED
-  typedef unsigned short ino_t;
-#endif
-
-/*
-** We need to define "NAME_MAX" if it was not present in "limits.h".
-*/
-
-#ifndef NAME_MAX
-#  ifdef FILENAME_MAX
-#    define NAME_MAX (FILENAME_MAX)
-#  else
-#    define NAME_MAX (260)
-#  endif
-#endif
-
-/*
-** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T".
-*/
-
-#ifndef NULL_INTPTR_T
-#  define NULL_INTPTR_T ((intptr_t)(0))
-#endif
-
-#ifndef BAD_INTPTR_T
-#  define BAD_INTPTR_T ((intptr_t)(-1))
-#endif
-
-/*
-** We need to provide the necessary structures and related types.
-*/
-
-#ifndef DIRENT_DEFINED
-#define DIRENT_DEFINED
-typedef struct DIRENT DIRENT;
-typedef DIRENT *LPDIRENT;
-struct DIRENT {
-  ino_t d_ino;               /* Sequence number, do not use. */
-  unsigned d_attributes;     /* Win32 file attributes. */
-  char d_name[NAME_MAX + 1]; /* Name within the directory. */
-};
-#endif
-
-#ifndef DIR_DEFINED
-#define DIR_DEFINED
-typedef struct DIR DIR;
-typedef DIR *LPDIR;
-struct DIR {
-  intptr_t d_handle; /* Value returned by "_findfirst". */
-  DIRENT d_first;    /* DIRENT constructed based on "_findfirst". */
-  DIRENT d_next;     /* DIRENT constructed based on "_findnext". */
-};
-#endif
-
-/*
-** Provide a macro, for use by the implementation, to determine if a
-** particular directory entry should be skipped over when searching for
-** the next directory entry that should be returned by the readdir() or
-** readdir_r() functions.
-*/
-
-#ifndef is_filtered
-#  define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM))
-#endif
-
-/*
-** Provide the function prototype for the POSIX compatiable getenv()
-** function.  This function is not thread-safe.
-*/
-
-extern const char *windirent_getenv(const char *name);
-
-/*
-** Finally, we can provide the function prototypes for the opendir(),
-** readdir(), readdir_r(), and closedir() POSIX functions.
-*/
-
-extern LPDIR opendir(const char *dirname);
-extern LPDIRENT readdir(LPDIR dirp);
-extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result);
-extern INT closedir(LPDIR dirp);
-
-#endif /* defined(WIN32) && defined(_MSC_VER) */
diff --git a/src/test_window.c b/src/test_window.c
index 48ab022116..631b20162c 100644
--- a/src/test_window.c
+++ b/src/test_window.c
@@ -16,7 +16,7 @@
 #ifdef SQLITE_TEST
 
 #include "sqliteInt.h"
-#include <tcl.h>
+#include "tclsqlite.h"
 
 extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 extern const char *sqlite3ErrName(int);
diff --git a/src/tokenize.c b/src/tokenize.c
index 8dac9ece52..152ada64f6 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -199,7 +199,7 @@ static int getToken(const unsigned char **pz){
   int t;                          /* Token type to return */
   do {
     z += sqlite3GetToken(z, &t);
-  }while( t==TK_SPACE );
+  }while( t==TK_SPACE || t==TK_COMMENT );
   if( t==TK_ID 
    || t==TK_STRING 
    || t==TK_JOIN_KW 
@@ -270,8 +270,9 @@ static int analyzeFilterKeyword(const unsigned char *z, int lastToken){
 ** Return the length (in bytes) of the token that begins at z[0]. 
 ** Store the token type in *tokenType before returning.
 */
-int sqlite3GetToken(const unsigned char *z, int *tokenType){
-  int i, c;
+i64 sqlite3GetToken(const unsigned char *z, int *tokenType){
+  i64 i;
+  int c;
   switch( aiClass[*z] ){  /* Switch on the character-class of the first byte
                           ** of the token. See the comment on the CC_ defines
                           ** above. */
@@ -288,7 +289,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
     case CC_MINUS: {
       if( z[1]=='-' ){
         for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
-        *tokenType = TK_SPACE;   /* IMP: R-22934-25134 */
+        *tokenType = TK_COMMENT;
         return i;
       }else if( z[1]=='>' ){
         *tokenType = TK_PTR;
@@ -324,7 +325,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
       }
       for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
       if( c ) i++;
-      *tokenType = TK_SPACE;   /* IMP: R-22934-25134 */
+      *tokenType = TK_COMMENT;
       return i;
     }
     case CC_PERCENT: {
@@ -433,31 +434,62 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
       testcase( z[0]=='0' );  testcase( z[0]=='1' );  testcase( z[0]=='2' );
       testcase( z[0]=='3' );  testcase( z[0]=='4' );  testcase( z[0]=='5' );
       testcase( z[0]=='6' );  testcase( z[0]=='7' );  testcase( z[0]=='8' );
-      testcase( z[0]=='9' );
+      testcase( z[0]=='9' );  testcase( z[0]=='.' );
       *tokenType = TK_INTEGER;
 #ifndef SQLITE_OMIT_HEX_INTEGER
       if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
-        for(i=3; sqlite3Isxdigit(z[i]); i++){}
-        return i;
-      }
+        for(i=3; 1; i++){
+          if( sqlite3Isxdigit(z[i])==0 ){
+            if( z[i]==SQLITE_DIGIT_SEPARATOR ){
+              *tokenType = TK_QNUMBER;
+            }else{
+              break;
+            }
+          }
+        }
+      }else
 #endif
-      for(i=0; sqlite3Isdigit(z[i]); i++){}
+        {
+        for(i=0; 1; i++){
+          if( sqlite3Isdigit(z[i])==0 ){
+            if( z[i]==SQLITE_DIGIT_SEPARATOR ){
+              *tokenType = TK_QNUMBER;
+            }else{
+              break;
+            }
+          }
+        }
 #ifndef SQLITE_OMIT_FLOATING_POINT
-      if( z[i]=='.' ){
-        i++;
-        while( sqlite3Isdigit(z[i]) ){ i++; }
-        *tokenType = TK_FLOAT;
-      }
-      if( (z[i]=='e' || z[i]=='E') &&
-           ( sqlite3Isdigit(z[i+1]) 
-            || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
-           )
-      ){
-        i += 2;
-        while( sqlite3Isdigit(z[i]) ){ i++; }
-        *tokenType = TK_FLOAT;
-      }
+        if( z[i]=='.' ){
+          if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT;
+          for(i++; 1; i++){
+            if( sqlite3Isdigit(z[i])==0 ){
+              if( z[i]==SQLITE_DIGIT_SEPARATOR ){
+                *tokenType = TK_QNUMBER;
+              }else{
+                break;
+              }
+            }
+          }
+        }
+        if( (z[i]=='e' || z[i]=='E') &&
+             ( sqlite3Isdigit(z[i+1]) 
+              || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+             )
+        ){
+          if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT;
+          for(i+=2; 1; i++){
+            if( sqlite3Isdigit(z[i])==0 ){
+              if( z[i]==SQLITE_DIGIT_SEPARATOR ){
+                *tokenType = TK_QNUMBER;
+              }else{
+                break;
+              }
+            }
+          }
+        }
 #endif
+      }
       while( IdChar(z[i]) ){
         *tokenType = TK_ILLEGAL;
         i++;
@@ -505,7 +537,8 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
       return i;
     }
     case CC_KYWD0: {
-      for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+      if( aiClass[z[1]]>CC_KYWD ){ i = 1;  break; }
+      for(i=2; aiClass[z[i]]<=CC_KYWD; i++){}
       if( IdChar(z[i]) ){
         /* This token started out using characters that can appear in keywords,
         ** but z[i] is a character not allowed within keywords, so this must
@@ -567,7 +600,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
 int sqlite3RunParser(Parse *pParse, const char *zSql){
   int nErr = 0;                   /* Number of errors encountered */
   void *pEngine;                  /* The LEMON-generated LALR(1) parser */
-  int n = 0;                      /* Length of the next token token */
+  i64 n = 0;                      /* Length of the next token token */
   int tokenType;                  /* type of the next token */
   int lastTokenParsed = -1;       /* type of the previous token */
   sqlite3 *db = pParse->db;       /* The database connection */
@@ -621,10 +654,13 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){
     if( tokenType>=TK_WINDOW ){
       assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER
            || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW 
+           || tokenType==TK_QNUMBER || tokenType==TK_COMMENT
       );
 #else
     if( tokenType>=TK_SPACE ){
-      assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
+      assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL 
+           || tokenType==TK_QNUMBER || tokenType==TK_COMMENT
+      );
 #endif /* SQLITE_OMIT_WINDOWFUNC */
       if( AtomicLoad(&db->u1.isInterrupted) ){
         pParse->rc = SQLITE_INTERRUPT;
@@ -657,16 +693,23 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){
         assert( n==6 );
         tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed);
 #endif /* SQLITE_OMIT_WINDOWFUNC */
-      }else{
+      }else if( tokenType==TK_COMMENT
+             && (db->init.busy || (db->flags & SQLITE_Comments)!=0)
+      ){
+        /* Ignore SQL comments if either (1) we are reparsing the schema or
+        ** (2) SQLITE_DBCONFIG_ENABLE_COMMENTS is turned on (the default). */
+        zSql += n;
+        continue;
+      }else if( tokenType!=TK_QNUMBER ){
         Token x;
         x.z = zSql;
-        x.n = n;
+        x.n = (u32)n;
         sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x);
         break;
       }
     }
     pParse->sLastToken.z = zSql;
-    pParse->sLastToken.n = n;
+    pParse->sLastToken.n = (u32)n;
     sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
     lastTokenParsed = tokenType;
     zSql += n;
@@ -693,7 +736,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){
     if( pParse->zErrMsg==0 ){
       pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc));
     }
-    sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail);
+    if( (pParse->prepFlags & SQLITE_PREPARE_DONT_LOG)==0 ){
+      sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail);
+    }
     nErr++;
   }
   pParse->zTail = zSql;
@@ -740,7 +785,7 @@ char *sqlite3Normalize(
 ){
   sqlite3 *db;       /* The database connection */
   int i;             /* Next unread byte of zSql[] */
-  int n;             /* length of current token */
+  i64 n;             /* length of current token */
   int tokenType;     /* type of current token */
   int prevType = 0;  /* Previous non-whitespace token */
   int nParen;        /* Number of nested levels of parentheses */
@@ -761,6 +806,7 @@ char *sqlite3Normalize(
     n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
     if( NEVER(n<=0) ) break;
     switch( tokenType ){
+      case TK_COMMENT:
       case TK_SPACE: {
         break;
       }
diff --git a/src/treeview.c b/src/treeview.c
index 9f630b1561..153fec88d4 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -193,9 +193,11 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){
     sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0);
     x.printfFlags |= SQLITE_PRINTF_INTERNAL;
     sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem);
-    if( pItem->pTab ){
-      sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx",
-           pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed);
+    if( pItem->pSTab ){
+      sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s",
+           pItem->pSTab->zName, pItem->pSTab->nCol, pItem->pSTab, 
+           pItem->colUsed,
+           pItem->fg.rowidUsed ? "+rowid" : "");
     }
     if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){
       sqlite3_str_appendf(&x, " FULL-OUTER-JOIN");
@@ -213,10 +215,13 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){
       sqlite3_str_appendf(&x, " DDL");
     }
     if( pItem->fg.isCte ){
-      sqlite3_str_appendf(&x, " CteUse=0x%p", pItem->u2.pCteUse);
+      static const char *aMat[] = {",MAT", "", ",NO-MAT"};
+      sqlite3_str_appendf(&x, " CteUse=%d%s",
+                          pItem->u2.pCteUse->nUse,
+                          aMat[pItem->u2.pCteUse->eM10d]);
     }
     if( pItem->fg.isOn || (pItem->fg.isUsing==0 && pItem->u3.pOn!=0) ){
-      sqlite3_str_appendf(&x, " ON");
+      sqlite3_str_appendf(&x, " isOn");
     }
     if( pItem->fg.isTabFunc )      sqlite3_str_appendf(&x, " isTabFunc");
     if( pItem->fg.isCorrelated )   sqlite3_str_appendf(&x, " isCorrelated");
@@ -224,23 +229,31 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){
     if( pItem->fg.viaCoroutine )   sqlite3_str_appendf(&x, " viaCoroutine");
     if( pItem->fg.notCte )         sqlite3_str_appendf(&x, " notCte");
     if( pItem->fg.isNestedFrom )   sqlite3_str_appendf(&x, " isNestedFrom");
+    if( pItem->fg.fixedSchema )    sqlite3_str_appendf(&x, " fixedSchema");
+    if( pItem->fg.hadSchema )      sqlite3_str_appendf(&x, " hadSchema");
+    if( pItem->fg.isSubquery )     sqlite3_str_appendf(&x, " isSubquery");
 
     sqlite3StrAccumFinish(&x);
     sqlite3TreeViewItem(pView, zLine, i<pSrc->nSrc-1);
     n = 0;
-    if( pItem->pSelect ) n++;
+    if( pItem->fg.isSubquery ) n++;
     if( pItem->fg.isTabFunc ) n++;
-    if( pItem->fg.isUsing ) n++;
+    if( pItem->fg.isUsing || pItem->u3.pOn!=0 ) n++;
     if( pItem->fg.isUsing ){
       sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING");
+    }else if( pItem->u3.pOn!=0 ){
+      sqlite3TreeViewItem(pView, "ON", (--n)>0);
+      sqlite3TreeViewExpr(pView, pItem->u3.pOn, 0);
+      sqlite3TreeViewPop(&pView);
     }
-    if( pItem->pSelect ){
-      if( pItem->pTab ){
-        Table *pTab = pItem->pTab;
+    if( pItem->fg.isSubquery ){
+      assert( n==1 );
+      if( pItem->pSTab ){
+        Table *pTab = pItem->pSTab;
         sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1);
       }
-      assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem->pSelect) );
-      sqlite3TreeViewSelect(pView, pItem->pSelect, (--n)>0);
+      assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) );
+      sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, 0);
     }
     if( pItem->fg.isTabFunc ){
       sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:");
@@ -282,7 +295,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
       n = 1000;
     }else{
       n = 0;
-      if( p->pSrc && p->pSrc->nSrc ) n++;
+      if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ) n++;
       if( p->pWhere ) n++;
       if( p->pGroupBy ) n++;
       if( p->pHaving ) n++;
@@ -308,7 +321,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
       sqlite3TreeViewPop(&pView);
     }
 #endif
-    if( p->pSrc && p->pSrc->nSrc ){
+    if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ){
       sqlite3TreeViewPush(&pView, (n--)>0);
       sqlite3TreeViewLine(pView, "FROM");
       sqlite3TreeViewSrcList(pView, p->pSrc);
@@ -344,7 +357,7 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
       sqlite3TreeViewItem(pView, "LIMIT", (n--)>0);
       sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0);
       if( p->pLimit->pRight ){
-        sqlite3TreeViewItem(pView, "OFFSET", (n--)>0);
+        sqlite3TreeViewItem(pView, "OFFSET", 0);
         sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0);
         sqlite3TreeViewPop(&pView);
       }
@@ -412,6 +425,7 @@ void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){
     sqlite3TreeViewItem(pView, "FILTER", 1);
     sqlite3TreeViewExpr(pView, pWin->pFilter, 0);
     sqlite3TreeViewPop(&pView);
+    if( pWin->eFrmType==TK_FILTER ) return;
   }
   sqlite3TreeViewPush(&pView, more);
   if( pWin->zName ){
@@ -421,7 +435,7 @@ void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){
   }
   if( pWin->zBase )    nElement++;
   if( pWin->pOrderBy ) nElement++;
-  if( pWin->eFrmType ) nElement++;
+  if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ) nElement++;
   if( pWin->eExclude ) nElement++;
   if( pWin->zBase ){
     sqlite3TreeViewPush(&pView, (--nElement)>0);
@@ -434,7 +448,7 @@ void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){
   if( pWin->pOrderBy ){
     sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY");
   }
-  if( pWin->eFrmType ){
+  if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ){
     char zBuf[30];
     const char *zFrmType = "ROWS";
     if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE";
@@ -643,7 +657,8 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       };
       assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT );
       assert( pExpr->pRight );
-      assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE );
+      assert( sqlite3ExprSkipCollateAndLikely(pExpr->pRight)->op
+                  == TK_TRUEFALSE );
       x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight);
       zUniOp = azOp[x];
       break;
@@ -681,7 +696,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
         assert( ExprUseXList(pExpr) );
         pFarg = pExpr->x.pList;
 #ifndef SQLITE_OMIT_WINDOWFUNC
-        pWin = ExprHasProperty(pExpr, EP_WinFunc) ? pExpr->y.pWin : 0;
+        pWin = IsWindowFunc(pExpr) ? pExpr->y.pWin : 0;
 #else
         pWin = 0;
 #endif 
@@ -707,7 +722,13 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
         sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs);
       }
       if( pFarg ){
-        sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0);
+        sqlite3TreeViewExprList(pView, pFarg, pWin!=0 || pExpr->pLeft, 0);
+        if( pExpr->pLeft ){
+          Expr *pOB = pExpr->pLeft;
+          assert( pOB->op==TK_ORDER );
+          assert( ExprUseXList(pOB) );
+          sqlite3TreeViewExprList(pView, pOB->x.pList, pWin!=0, "ORDERBY");
+        }
       }
 #ifndef SQLITE_OMIT_WINDOWFUNC
       if( pWin ){
@@ -716,6 +737,10 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
 #endif
       break;
     }
+    case TK_ORDER: {
+      sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, "ORDERBY");
+      break;
+    }
 #ifndef SQLITE_OMIT_SUBQUERY
     case TK_EXISTS: {
       assert( ExprUseXSelect(pExpr) );
@@ -769,7 +794,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       assert( pExpr->x.pList->nExpr==2 );
       pY = pExpr->x.pList->a[0].pExpr;
       pZ = pExpr->x.pList->a[1].pExpr;
-      sqlite3TreeViewLine(pView, "BETWEEN");
+      sqlite3TreeViewLine(pView, "BETWEEN%s", zFlgs);
       sqlite3TreeViewExpr(pView, pX, 1);
       sqlite3TreeViewExpr(pView, pY, 1);
       sqlite3TreeViewExpr(pView, pZ, 0);
@@ -804,7 +829,8 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
         case OE_Ignore:     zType = "ignore";    break;
       }
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
-      sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken);
+      sqlite3TreeViewLine(pView, "RAISE %s", zType);
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
       break;
     }
 #endif
@@ -884,9 +910,10 @@ void sqlite3TreeViewBareExprList(
     sqlite3TreeViewLine(pView, "%s", zLabel);
     for(i=0; i<pList->nExpr; i++){
       int j = pList->a[i].u.x.iOrderByCol;
+      u8 sortFlags = pList->a[i].fg.sortFlags;
       char *zName = pList->a[i].zEName;
       int moreToFollow = i<pList->nExpr - 1;
-      if( j || zName ){
+      if( j || zName || sortFlags ){
         sqlite3TreeViewPush(&pView, moreToFollow);
         moreToFollow = 0;
         sqlite3TreeViewLine(pView, 0);
@@ -907,13 +934,18 @@ void sqlite3TreeViewBareExprList(
           }
         }
         if( j ){
-          fprintf(stdout, "iOrderByCol=%d", j);
+          fprintf(stdout, "iOrderByCol=%d ", j);
+        }
+        if( sortFlags & KEYINFO_ORDER_DESC ){
+          fprintf(stdout, "DESC ");
+        }else if( sortFlags & KEYINFO_ORDER_BIGNULL ){
+          fprintf(stdout, "NULLS-LAST");
         }
         fprintf(stdout, "\n");
         fflush(stdout);
       }
       sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow);
-      if( j || zName ){
+      if( j || zName || sortFlags ){
         sqlite3TreeViewPop(&pView);
       }
     }
@@ -950,21 +982,7 @@ void sqlite3TreeViewBareIdList(
       if( zName==0 ) zName = "(null)";
       sqlite3TreeViewPush(&pView, moreToFollow);
       sqlite3TreeViewLine(pView, 0);
-      if( pList->eU4==EU4_NONE ){
-        fprintf(stdout, "%s\n", zName);
-      }else if( pList->eU4==EU4_IDX ){
-        fprintf(stdout, "%s (%d)\n", zName, pList->a[i].u4.idx);
-      }else{
-        assert( pList->eU4==EU4_EXPR );
-        if( pList->a[i].u4.pExpr==0 ){
-          fprintf(stdout, "%s (pExpr=NULL)\n", zName);
-        }else{
-          fprintf(stdout, "%s\n", zName);
-          sqlite3TreeViewPush(&pView, i<pList->nId-1);
-          sqlite3TreeViewExpr(pView, pList->a[i].u4.pExpr, 0);
-          sqlite3TreeViewPop(&pView);
-        }
-      }
+      fprintf(stdout, "%s\n", zName);
       sqlite3TreeViewPop(&pView);
     }
   }
@@ -1274,6 +1292,10 @@ void sqlite3TreeViewTrigger(
 ** accessible to the debugging, and to avoid warnings about unused
 ** functions.  But these routines only exist in debugging builds, so they
 ** do not contaminate the interface.
+**
+** See Also:
+**
+**     sqlite3ShowWhereTerm() in where.c
 */
 void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); }
 void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);}
diff --git a/src/trigger.c b/src/trigger.c
index bcb2132f0b..799fbe57fa 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -70,7 +70,8 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){
       assert( pParse->db->pVtabCtx==0 );
 #endif
       assert( pParse->bReturning );
-      assert( &(pParse->u1.pReturning->retTrig) == pTrig );
+      assert( !pParse->isCreate );
+      assert( &(pParse->u1.d.pReturning->retTrig) == pTrig );
       pTrig->table = pTab->zName;
       pTrig->pTabSchema = pTab->pSchema;
       pTrig->pNext = pList;
@@ -152,8 +153,10 @@ void sqlite3BeginTrigger(
   ** name on pTableName if we are reparsing out of the schema table
   */
   if( db->init.busy && iDb!=1 ){
-    sqlite3DbFree(db, pTableName->a[0].zDatabase);
-    pTableName->a[0].zDatabase = 0;
+    assert( pTableName->a[0].fg.fixedSchema==0 );
+    assert( pTableName->a[0].fg.isSubquery==0 );
+    sqlite3DbFree(db, pTableName->a[0].u4.zDatabase);
+    pTableName->a[0].u4.zDatabase = 0;
   }
 
   /* If the trigger name was unqualified, and the table is a temp table,
@@ -183,6 +186,10 @@ void sqlite3BeginTrigger(
     sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables");
     goto trigger_orphan_error;
   }
+  if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){
+    sqlite3ErrorMsg(pParse, "cannot create triggers on shadow tables");
+    goto trigger_orphan_error;
+  }
 
   /* Check that the trigger name is not reserved and that no trigger of the
   ** specified name exists */
@@ -627,7 +634,8 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){
   }
 
   assert( pName->nSrc==1 );
-  zDb = pName->a[0].zDatabase;
+  assert( pName->a[0].fg.fixedSchema==0 && pName->a[0].fg.isSubquery==0 );
+  zDb = pName->a[0].u4.zDatabase;
   zName = pName->a[0].zName;
   assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) );
   for(i=OMIT_TEMPDB; i<db->nDb; i++){
@@ -864,7 +872,9 @@ SrcList *sqlite3TriggerStepSrc(
     Schema *pSchema = pStep->pTrig->pSchema;
     pSrc->a[0].zName = zName;
     if( pSchema!=db->aDb[1].pSchema ){
-      pSrc->a[0].pSchema = pSchema;
+      assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 );
+      pSrc->a[0].u4.pSchema = pSchema;
+      pSrc->a[0].fg.fixedSchema = 1;
     }
     if( pStep->pFrom ){
       SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0);
@@ -947,6 +957,72 @@ static ExprList *sqlite3ExpandReturning(
   return pNew;
 }
 
+/* If the Expr node is a subquery or an EXISTS operator or an IN operator that
+** uses a subquery, and if the subquery is SF_Correlated, then mark the
+** expression as EP_VarSelect.
+*/
+static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){
+  UNUSED_PARAMETER(NotUsed);
+  if( ExprUseXSelect(pExpr)
+   && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0
+  ){
+    testcase( ExprHasProperty(pExpr, EP_VarSelect) );
+    ExprSetProperty(pExpr, EP_VarSelect);
+  }
+  return WRC_Continue;
+}
+
+
+/*
+** If the SELECT references the table pWalker->u.pTab, then do two things:
+**
+**    (1) Mark the SELECT as as SF_Correlated.
+**    (2) Set pWalker->eCode to non-zero so that the caller will know
+**        that (1) has happened.
+*/
+static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){
+  int i;
+  SrcList *pSrc;
+  assert( pSelect!=0 );
+  pSrc = pSelect->pSrc;
+  assert( pSrc!=0 );
+  for(i=0; i<pSrc->nSrc; i++){
+    if( pSrc->a[i].pSTab==pWalker->u.pTab ){
+      testcase( pSelect->selFlags & SF_Correlated );
+      pSelect->selFlags |= SF_Correlated;
+      pWalker->eCode = 1;
+      break;
+    }
+  }
+  return WRC_Continue;
+}
+
+/*
+** Scan the expression list that is the argument to RETURNING looking
+** for subqueries that depend on the table which is being modified in the
+** statement that is hosting the RETURNING clause (pTab).  Mark all such
+** subqueries as SF_Correlated.  If the subqueries are part of an
+** expression, mark the expression as EP_VarSelect.
+**
+** https://sqlite.org/forum/forumpost/2c83569ce8945d39
+*/
+static void sqlite3ProcessReturningSubqueries(
+  ExprList *pEList,
+  Table *pTab
+){
+  Walker w;
+  memset(&w, 0, sizeof(w));
+  w.xExprCallback = sqlite3ExprWalkNoop;
+  w.xSelectCallback = sqlite3ReturningSubqueryCorrelated;
+  w.u.pTab = pTab;
+  sqlite3WalkExprList(&w, pEList);
+  if( w.eCode ){
+    w.xExprCallback = sqlite3ReturningSubqueryVarSelect;
+    w.xSelectCallback = sqlite3SelectWalkNoop;
+    sqlite3WalkExprList(&w, pEList);
+  }
+}
+
 /*
 ** Generate code for the RETURNING trigger.  Unlike other triggers
 ** that invoke a subprogram in the bytecode, the code for RETURNING
@@ -963,20 +1039,34 @@ static void codeReturningTrigger(
   ExprList *pNew;
   Returning *pReturning;
   Select sSelect;
-  SrcList sFrom;
+  SrcList *pFrom;
+  union {
+    SrcList sSrc;
+    u8 fromSpace[SZ_SRCLIST_1];
+  } uSrc;
 
   assert( v!=0 );
-  assert( pParse->bReturning );
+  if( !pParse->bReturning ){
+    /* This RETURNING trigger must be for a different statement as
+    ** this statement lacks a RETURNING clause. */
+    return;
+  }
   assert( db->pParse==pParse );
-  pReturning = pParse->u1.pReturning;
-  assert( pTrigger == &(pReturning->retTrig) );
+  assert( !pParse->isCreate );
+  pReturning = pParse->u1.d.pReturning;
+  if( pTrigger != &(pReturning->retTrig) ){
+    /* This RETURNING trigger is for a different statement */
+    return;
+  }
   memset(&sSelect, 0, sizeof(sSelect));
-  memset(&sFrom, 0, sizeof(sFrom));
+  memset(&uSrc, 0, sizeof(uSrc));
+  pFrom = &uSrc.sSrc;
   sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0);
-  sSelect.pSrc = &sFrom;
-  sFrom.nSrc = 1;
-  sFrom.a[0].pTab = pTab;
-  sFrom.a[0].iCursor = -1;
+  sSelect.pSrc = pFrom;
+  pFrom->nSrc = 1;
+  pFrom->a[0].pSTab = pTab;
+  pFrom->a[0].zName = pTab->zName; /* tag-20240424-1 */
+  pFrom->a[0].iCursor = -1;
   sqlite3SelectPrep(pParse, &sSelect, 0);
   if( pParse->nErr==0 ){
     assert( db->mallocFailed==0 );
@@ -1002,6 +1092,7 @@ static void codeReturningTrigger(
       int i;
       int nCol = pNew->nExpr;
       int reg = pParse->nMem+1;
+      sqlite3ProcessReturningSubqueries(pNew, pTab);
       pParse->nMem += nCol+2;
       pReturning->iRetReg = reg;
       for(i=0; i<nCol; i++){
@@ -1193,6 +1284,8 @@ static TriggerPrg *codeRowTrigger(
   sSubParse.eTriggerOp = pTrigger->op;
   sSubParse.nQueryLoop = pParse->nQueryLoop;
   sSubParse.prepFlags = pParse->prepFlags;
+  sSubParse.oldmask = 0;
+  sSubParse.newmask = 0;
 
   v = sqlite3GetVdbe(&sSubParse);
   if( v ){
@@ -1325,7 +1418,7 @@ void sqlite3CodeRowTriggerDirect(
     ** invocation is disallowed if (a) the sub-program is really a trigger,
     ** not a foreign key action, and (b) the flag to enable recursive triggers
     ** is clear.  */
-    sqlite3VdbeChangeP5(v, (u8)bRecursive);
+    sqlite3VdbeChangeP5(v, (u16)bRecursive);
   }
 }
 
diff --git a/src/update.c b/src/update.c
index c1f1363980..979afea1f5 100644
--- a/src/update.c
+++ b/src/update.c
@@ -30,10 +30,10 @@ static void updateVirtualTable(
 
 /*
 ** The most recently coded instruction was an OP_Column to retrieve the
-** i-th column of table pTab. This routine sets the P4 parameter of the 
+** i-th column of table pTab. This routine sets the P4 parameter of the
 ** OP_Column to the default value, if any.
 **
-** The default value of a column is specified by a DEFAULT clause in the 
+** The default value of a column is specified by a DEFAULT clause in the
 ** column definition. This was either supplied by the user when the table
 ** was created, or added later to the table definition by an ALTER TABLE
 ** command. If the latter, then the row-records in the table btree on disk
@@ -42,9 +42,9 @@ static void updateVirtualTable(
 ** If the former, then all row-records are guaranteed to include a value
 ** for the column and the P4 value is not required.
 **
-** Column definitions created by an ALTER TABLE command may only have 
+** Column definitions created by an ALTER TABLE command may only have
 ** literal default values specified: a number, null or a string. (If a more
-** complicated default expression value was provided, it is evaluated 
+** complicated default expression value was provided, it is evaluated
 ** when the ALTER TABLE is executed and one of the literal values written
 ** into the sqlite_schema table.)
 **
@@ -69,8 +69,8 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){
     assert( !IsView(pTab) );
     VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName));
     assert( i<pTab->nCol );
-    sqlite3ValueFromExpr(sqlite3VdbeDb(v), 
-                         sqlite3ColumnExpr(pTab,pCol), enc, 
+    sqlite3ValueFromExpr(sqlite3VdbeDb(v),
+                         sqlite3ColumnExpr(pTab,pCol), enc,
                          pCol->affinity, &pValue);
     if( pValue ){
       sqlite3VdbeAppendP4(v, pValue, P4_MEM);
@@ -150,17 +150,17 @@ static Expr *exprRowColumn(Parse *pParse, int iCol){
 ** Assuming both the pLimit and pOrderBy parameters are NULL, this function
 ** generates VM code to run the query:
 **
-**   SELECT <other-columns>, pChanges FROM pTabList WHERE pWhere 
+**   SELECT <other-columns>, pChanges FROM pTabList WHERE pWhere
 **
-** and write the results to the ephemeral table already opened as cursor 
-** iEph. None of pChanges, pTabList or pWhere are modified or consumed by 
+** and write the results to the ephemeral table already opened as cursor
+** iEph. None of pChanges, pTabList or pWhere are modified or consumed by
 ** this function, they must be deleted by the caller.
 **
 ** Or, if pLimit and pOrderBy are not NULL, and pTab is not a view:
 **
-**   SELECT <other-columns>, pChanges FROM pTabList 
+**   SELECT <other-columns>, pChanges FROM pTabList
 **   WHERE pWhere
-**   GROUP BY <other-columns> 
+**   GROUP BY <other-columns>
 **   ORDER BY pOrderBy LIMIT pLimit
 **
 ** If pTab is a view, the GROUP BY clause is omitted.
@@ -178,11 +178,11 @@ static Expr *exprRowColumn(Parse *pParse, int iCol){
 ** the view. The results are written to the ephemeral table iEph as records
 ** with automatically assigned integer keys.
 **
-** If the table is a virtual or ordinary intkey table, then <other-columns> 
+** If the table is a virtual or ordinary intkey table, then <other-columns>
 ** is its rowid. For a virtual table, the results are written to iEph as
 ** records with automatically assigned integer keys For intkey tables, the
-** rowid value in <other-columns> is used as the integer key, and the 
-** remaining fields make up the table record. 
+** rowid value in <other-columns> is used as the integer key, and the
+** remaining fields make up the table record.
 */
 static void updateFromSelect(
   Parse *pParse,                  /* Parse context */
@@ -202,7 +202,7 @@ static void updateFromSelect(
   Expr *pLimit2 = 0;
   ExprList *pOrderBy2 = 0;
   sqlite3 *db = pParse->db;
-  Table *pTab = pTabList->a[0].pTab;
+  Table *pTab = pTabList->a[0].pSTab;
   SrcList *pSrc;
   Expr *pWhere2;
   int eDest;
@@ -224,10 +224,10 @@ static void updateFromSelect(
 
   assert( pTabList->nSrc>1 );
   if( pSrc ){
-    pSrc->a[0].fg.notCte = 1;
+    assert( pSrc->a[0].fg.notCte );
     pSrc->a[0].iCursor = -1;
-    pSrc->a[0].pTab->nTabRef--;
-    pSrc->a[0].pTab = 0;
+    pSrc->a[0].pSTab->nTabRef--;
+    pSrc->a[0].pSTab = 0;
   }
   if( pPk ){
     for(i=0; i<pPk->nKeyCol; i++){
@@ -257,13 +257,13 @@ static void updateFromSelect(
   assert( pChanges!=0 || pParse->db->mallocFailed );
   if( pChanges ){
     for(i=0; i<pChanges->nExpr; i++){
-      pList = sqlite3ExprListAppend(pParse, pList, 
+      pList = sqlite3ExprListAppend(pParse, pList,
           sqlite3ExprDup(db, pChanges->a[i].pExpr, 0)
       );
     }
   }
-  pSelect = sqlite3SelectNew(pParse, pList, 
-      pSrc, pWhere2, pGrp, 0, pOrderBy2, 
+  pSelect = sqlite3SelectNew(pParse, pList,
+      pSrc, pWhere2, pGrp, 0, pOrderBy2,
       SF_UFSrcCheck|SF_IncludeHidden|SF_UpdateFrom, pLimit2
   );
   if( pSelect ) pSelect->selFlags |= SF_OrderByReqd;
@@ -357,7 +357,7 @@ void sqlite3Update(
   }
   assert( db->mallocFailed==0 );
 
-  /* Locate the table which we want to update. 
+  /* Locate the table which we want to update.
   */
   pTab = sqlite3SrcListLookup(pParse, pTabList);
   if( pTab==0 ) goto update_cleanup;
@@ -435,7 +435,7 @@ void sqlite3Update(
   }
   pTabList->a[0].iCursor = iDataCur;
 
-  /* Allocate space for aXRef[], aRegIdx[], and aToOpen[].  
+  /* Allocate space for aXRef[], aRegIdx[], and aToOpen[].
   ** Initialize aXRef[] and aToOpen[] to their default values.
   */
   aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx+1) + nIdx+2 );
@@ -465,38 +465,32 @@ void sqlite3Update(
   */
   chngRowid = chngPk = 0;
   for(i=0; i<pChanges->nExpr; i++){
-    u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName);
     /* If this is an UPDATE with a FROM clause, do not resolve expressions
     ** here. The call to sqlite3Select() below will do that. */
     if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){
       goto update_cleanup;
     }
-    for(j=0; j<pTab->nCol; j++){
-      if( pTab->aCol[j].hName==hCol
-       && sqlite3StrICmp(pTab->aCol[j].zCnName, pChanges->a[i].zEName)==0
-      ){
-        if( j==pTab->iPKey ){
-          chngRowid = 1;
-          pRowidExpr = pChanges->a[i].pExpr;
-          iRowidExpr = i;
-        }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){
-          chngPk = 1;
-        }
+    j = sqlite3ColumnIndex(pTab, pChanges->a[i].zEName);
+    if( j>=0 ){
+      if( j==pTab->iPKey ){
+        chngRowid = 1;
+        pRowidExpr = pChanges->a[i].pExpr;
+        iRowidExpr = i;
+      }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){
+        chngPk = 1;
+      }
 #ifndef SQLITE_OMIT_GENERATED_COLUMNS
-        else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){
-          testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL );
-          testcase( pTab->aCol[j].colFlags & COLFLAG_STORED );
-          sqlite3ErrorMsg(pParse, 
-             "cannot UPDATE generated column \"%s\"",
-             pTab->aCol[j].zCnName);
-          goto update_cleanup;
-        }
-#endif
-        aXRef[j] = i;
-        break;
+      else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){
+        testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL );
+        testcase( pTab->aCol[j].colFlags & COLFLAG_STORED );
+        sqlite3ErrorMsg(pParse,
+           "cannot UPDATE generated column \"%s\"",
+           pTab->aCol[j].zCnName);
+        goto update_cleanup;
       }
-    }
-    if( j>=pTab->nCol ){
+#endif
+      aXRef[j] = i;
+    }else{
       if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){
         j = -1;
         chngRowid = 1;
@@ -529,11 +523,11 @@ void sqlite3Update(
 
 #ifndef SQLITE_OMIT_GENERATED_COLUMNS
   /* Mark generated columns as changing if their generator expressions
-  ** reference any changing column.  The actual aXRef[] value for 
+  ** reference any changing column.  The actual aXRef[] value for
   ** generated expressions is not used, other than to check to see that it
   ** is non-negative, so the value of aXRef[] for generated columns can be
   ** set to any non-negative number.  We use 99999 so that the value is
-  ** obvious when looking at aXRef[] in a symbolic debugger. 
+  ** obvious when looking at aXRef[] in a symbolic debugger.
   */
   if( pTab->tabFlags & TF_HasGenerated ){
     int bProgress;
@@ -556,7 +550,7 @@ void sqlite3Update(
   }
 #endif
 
-  /* The SET expressions are not actually used inside the WHERE loop.  
+  /* The SET expressions are not actually used inside the WHERE loop.
   ** So reset the colUsed mask. Unless this is a virtual table. In that
   ** case, set all bits of the colUsed mask (to ensure that the virtual
   ** table implementation makes all columns available).
@@ -595,7 +589,7 @@ void sqlite3Update(
   }
   aRegIdx[nAllIdx] = ++pParse->nMem;  /* Register storing the table record */
   if( bReplace ){
-    /* If REPLACE conflict resolution might be invoked, open cursors on all 
+    /* If REPLACE conflict resolution might be invoked, open cursors on all
     ** indexes in case they are needed to delete records.  */
     memset(aToOpen, 1, nIdx+1);
   }
@@ -634,7 +628,7 @@ void sqlite3Update(
   */
 #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER)
   if( nChangeFrom==0 && isView ){
-    sqlite3MaterializeView(pParse, pTab, 
+    sqlite3MaterializeView(pParse, pTab,
         pWhere, pOrderBy, pLimit, iDataCur
     );
     pOrderBy = 0;
@@ -706,7 +700,7 @@ void sqlite3Update(
       }
     }
   }
-  
+
   if( nChangeFrom ){
     sqlite3MultiWrite(pParse);
     eOnePass = ONEPASS_OFF;
@@ -724,7 +718,7 @@ void sqlite3Update(
       sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL);
       bFinishSeek = 0;
     }else{
-      /* Begin the database scan. 
+      /* Begin the database scan.
       **
       ** Do not consider a single-pass strategy for a multi-row update if
       ** there is anything that might disrupt the cursor being used to do
@@ -741,7 +735,7 @@ void sqlite3Update(
        && !hasFK
        && !chngKey
        && !bReplace
-       && (sNC.ncFlags & NC_Subquery)==0
+       && (pWhere==0 || !ExprHasProperty(pWhere, EP_Subquery))
       ){
         flags |= WHERE_ONEPASS_MULTIROW;
       }
@@ -787,7 +781,7 @@ void sqlite3Update(
       /* Read the PK of the current row into an array of registers. In
       ** ONEPASS_OFF mode, serialize the array into a record and store it in
       ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change
-      ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table 
+      ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table
       ** is not required) and leave the PK fields in the array of registers.  */
       for(i=0; i<nPk; i++){
         assert( pPk->aiColumn[i]>=0 );
@@ -810,26 +804,28 @@ void sqlite3Update(
     if( nChangeFrom==0 && eOnePass!=ONEPASS_MULTI ){
       sqlite3WhereEnd(pWInfo);
     }
-  
+
     if( !isView ){
       int addrOnce = 0;
-  
+      int iNotUsed1 = 0;
+      int iNotUsed2 = 0;
+
       /* Open every index that needs updating. */
       if( eOnePass!=ONEPASS_OFF ){
         if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
         if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
       }
-  
+
       if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
         addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
       }
       sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur,
-                                 aToOpen, 0, 0);
+                                 aToOpen, &iNotUsed1, &iNotUsed2);
       if( addrOnce ){
         sqlite3VdbeJumpHereOrPopInst(v, addrOnce);
       }
     }
-  
+
     /* Top of the update loop */
     if( eOnePass!=ONEPASS_OFF ){
       if( aiCurOnePass[0]!=iDataCur
@@ -902,7 +898,7 @@ void sqlite3Update(
   ** information is needed */
   if( chngPk || hasFK || pTrigger ){
     u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0);
-    oldmask |= sqlite3TriggerColmask(pParse, 
+    oldmask |= sqlite3TriggerColmask(pParse,
         pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError
     );
     for(i=0; i<pTab->nCol; i++){
@@ -919,6 +915,9 @@ void sqlite3Update(
       }
     }
     if( chngRowid==0 && pPk==0 ){
+#ifdef SQLITE_ALLOW_ROWID_IN_VIEW
+      if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
+#endif
       sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid);
     }
   }
@@ -931,8 +930,8 @@ void sqlite3Update(
   ** If there are one or more BEFORE triggers, then do not populate the
   ** registers associated with columns that are (a) not modified by
   ** this UPDATE statement and (b) not accessed by new.* references. The
-  ** values for registers not modified by the UPDATE must be reloaded from 
-  ** the database after the BEFORE triggers are fired anyway (as the trigger 
+  ** values for registers not modified by the UPDATE must be reloaded from
+  ** the database after the BEFORE triggers are fired anyway (as the trigger
   ** may have modified them). So not loading those that are not going to
   ** be used eliminates some redundant opcodes.
   */
@@ -955,7 +954,7 @@ void sqlite3Update(
           sqlite3ExprCode(pParse, pChanges->a[j].pExpr, k);
         }
       }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask & MASKBIT32(i)) ){
-        /* This branch loads the value of a column that will not be changed 
+        /* This branch loads the value of a column that will not be changed
         ** into a register. This is done if there are no BEFORE triggers, or
         ** if there are one or more BEFORE triggers that use this value via
         ** a new.* reference in a trigger program.
@@ -982,12 +981,12 @@ void sqlite3Update(
   */
   if( tmask&TRIGGER_BEFORE ){
     sqlite3TableAffinity(v, pTab, regNew);
-    sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, 
+    sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges,
         TRIGGER_BEFORE, pTab, regOldRowid, onError, labelContinue);
 
     if( !isView ){
       /* The row-trigger may have deleted the row being updated. In this
-      ** case, jump to the next row. No updates or AFTER triggers are 
+      ** case, jump to the next row. No updates or AFTER triggers are
       ** required. This behavior - what happens when the row being updated
       ** is deleted or renamed by a BEFORE trigger - is left undefined in the
       ** documentation.
@@ -1001,8 +1000,8 @@ void sqlite3Update(
       }
 
       /* After-BEFORE-trigger-reload-loop:
-      ** If it did not delete it, the BEFORE trigger may still have modified 
-      ** some of the columns of the row being updated. Load the values for 
+      ** If it did not delete it, the BEFORE trigger may still have modified
+      ** some of the columns of the row being updated. Load the values for
       ** all columns not modified by the update statement into their registers
       ** in case this has happened. Only unmodified columns are reloaded.
       ** The values computed for modified columns use the values before the
@@ -1022,7 +1021,7 @@ void sqlite3Update(
         testcase( pTab->tabFlags & TF_HasStored );
         sqlite3ComputeGeneratedColumns(pParse, regNew, pTab);
       }
-#endif 
+#endif
     }
   }
 
@@ -1066,7 +1065,7 @@ void sqlite3Update(
     ** to process, delete the old record. Otherwise, add a noop OP_Delete
     ** to invoke the pre-update hook.
     **
-    ** That (regNew==regnewRowid+1) is true is also important for the 
+    ** That (regNew==regnewRowid+1) is true is also important for the
     ** pre-update hook. If the caller invokes preupdate_new(), the returned
     ** value is copied from memory cell (regNewRowid+1+iCol), where iCol
     ** is the column index supplied by the user.
@@ -1093,30 +1092,32 @@ void sqlite3Update(
     if( hasFK ){
       sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey);
     }
-  
+
     /* Insert the new index entries and the new record. */
     sqlite3CompleteInsertion(
-        pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, 
-        OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), 
+        pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx,
+        OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0),
         0, 0
     );
 
     /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to
     ** handle rows (possibly in other tables) that refer via a foreign key
-    ** to the row just updated. */ 
+    ** to the row just updated. */
     if( hasFK ){
       sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey);
     }
   }
 
-  /* Increment the row counter 
+  /* Increment the row counter
   */
   if( regRowCount ){
     sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1);
   }
 
-  sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, 
-      TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue);
+  if( pTrigger ){
+    sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges,
+        TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue);
+  }
 
   /* Repeat the above with the next record to be updated, until
   ** all record selected by the WHERE clause have been updated.
@@ -1154,7 +1155,7 @@ void sqlite3Update(
   sqlite3SrcListDelete(db, pTabList);
   sqlite3ExprListDelete(db, pChanges);
   sqlite3ExprDelete(db, pWhere);
-#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) 
+#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT)
   sqlite3ExprListDelete(db, pOrderBy);
   sqlite3ExprDelete(db, pLimit);
 #endif
@@ -1174,8 +1175,8 @@ void sqlite3Update(
 /*
 ** Generate code for an UPDATE of a virtual table.
 **
-** There are two possible strategies - the default and the special 
-** "onepass" strategy. Onepass is only used if the virtual table 
+** There are two possible strategies - the default and the special
+** "onepass" strategy. Onepass is only used if the virtual table
 ** implementation indicates that pWhere may match at most one row.
 **
 ** The default strategy is to create an ephemeral table that contains
@@ -1211,7 +1212,7 @@ static void updateVirtualTable(
   int nArg = 2 + pTab->nCol;      /* Number of arguments to VUpdate */
   int regArg;                     /* First register in VUpdate arg array */
   int regRec;                     /* Register in which to assemble record */
-  int regRowid;                   /* Register for ephem table rowid */
+  int regRowid;                   /* Register for ephemeral table rowid */
   int iCsr = pSrc->a[0].iCursor;  /* Cursor used for virtual table scan */
   int aDummy[2];                  /* Unused arg for sqlite3WhereOkOnePass() */
   int eOnePass;                   /* True to use onepass strategy */
@@ -1255,7 +1256,9 @@ static void updateVirtualTable(
           sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0)
         );
       }else{
-        pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i));
+        Expr *pRowExpr = exprRowColumn(pParse, i);
+        if( pRowExpr ) pRowExpr->op2 = OPFLAG_NOCHNG;
+        pList = sqlite3ExprListAppend(pParse, pList, pRowExpr);
       }
     }
 
@@ -1332,10 +1335,10 @@ static void updateVirtualTable(
       sqlite3WhereEnd(pWInfo);
     }
 
-    /* Begin scannning through the ephemeral table. */
+    /* Begin scanning through the ephemeral table. */
     addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v);
 
-    /* Extract arguments from the current row of the ephemeral table and 
+    /* Extract arguments from the current row of the ephemeral table and
     ** invoke the VUpdate method.  */
     for(i=0; i<nArg; i++){
       sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i, regArg+i);
diff --git a/src/upsert.c b/src/upsert.c
index 85994020cf..82295d52ae 100644
--- a/src/upsert.c
+++ b/src/upsert.c
@@ -90,7 +90,8 @@ Upsert *sqlite3UpsertNew(
 int sqlite3UpsertAnalyzeTarget(
   Parse *pParse,     /* The parsing context */
   SrcList *pTabList, /* Table into which we are inserting */
-  Upsert *pUpsert    /* The ON CONFLICT clauses */
+  Upsert *pUpsert,   /* The ON CONFLICT clauses */
+  Upsert *pAll       /* Complete list of all ON CONFLICT clauses */
 ){
   Table *pTab;            /* That table into which we are inserting */
   int rc;                 /* Result code */
@@ -103,7 +104,7 @@ int sqlite3UpsertAnalyzeTarget(
   int nClause = 0;        /* Counter of ON CONFLICT clauses */
 
   assert( pTabList->nSrc==1 );
-  assert( pTabList->a[0].pTab!=0 );
+  assert( pTabList->a[0].pSTab!=0 );
   assert( pUpsert!=0 );
   assert( pUpsert->pUpsertTarget!=0 );
 
@@ -122,7 +123,7 @@ int sqlite3UpsertAnalyzeTarget(
     if( rc ) return rc;
   
     /* Check to see if the conflict target matches the rowid. */  
-    pTab = pTabList->a[0].pTab;
+    pTab = pTabList->a[0].pSTab;
     pTarget = pUpsert->pUpsertTarget;
     iCursor = pTabList->a[0].iCursor;
     if( HasRowid(pTab) 
@@ -178,7 +179,7 @@ int sqlite3UpsertAnalyzeTarget(
           pExpr = &sCol[0];
         }
         for(jj=0; jj<nn; jj++){
-          if( sqlite3ExprCompare(pParse,pTarget->a[jj].pExpr,pExpr,iCursor)<2 ){
+          if( sqlite3ExprCompare(0,pTarget->a[jj].pExpr,pExpr,iCursor)<2 ){
             break;  /* Column ii of the index matches column jj of target */
           }
         }
@@ -193,6 +194,14 @@ int sqlite3UpsertAnalyzeTarget(
         continue;
       }
       pUpsert->pUpsertIdx = pIdx;
+      if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){
+        /* Really this should be an error.  The isDup ON CONFLICT clause will
+        ** never fire.  But this problem was not discovered until three years
+        ** after multi-CONFLICT upsert was added, and so we silently ignore
+        ** the problem to prevent breaking applications that might actually
+        ** have redundant ON CONFLICT clauses. */
+        pUpsert->isDup = 1;
+      }
       break;
     }
     if( pUpsert->pUpsertIdx==0 ){
@@ -219,9 +228,13 @@ int sqlite3UpsertNextIsIPK(Upsert *pUpsert){
   Upsert *pNext;
   if( NEVER(pUpsert==0) ) return 0;
   pNext = pUpsert->pNextUpsert;
-  if( pNext==0 ) return 1;
-  if( pNext->pUpsertTarget==0 ) return 1;
-  if( pNext->pUpsertIdx==0 ) return 1;
+  while( 1 /*exit-by-return*/ ){
+    if( pNext==0 ) return 1;
+    if( pNext->pUpsertTarget==0 ) return 1;
+    if( pNext->pUpsertIdx==0 ) return 1;
+    if( !pNext->isDup ) return 0;
+    pNext = pNext->pNextUpsert;
+  }
   return 0;
 }
 
diff --git a/src/utf.c b/src/utf.c
index 5f27babdfc..2efcd67912 100644
--- a/src/utf.c
+++ b/src/utf.c
@@ -105,6 +105,35 @@ static const unsigned char sqlite3Utf8Trans1[] = {
   }                                                                 \
 }
 
+/*
+** Write a single UTF8 character whose value is v into the
+** buffer starting at zOut.  zOut must be sized to hold at
+** least four bytes.  Return the number of bytes needed
+** to encode the new character.
+*/
+int sqlite3AppendOneUtf8Character(char *zOut, u32 v){
+  if( v<0x00080 ){
+    zOut[0] = (u8)(v & 0xff);
+    return 1;
+  }
+  if( v<0x00800 ){
+    zOut[0] = 0xc0 + (u8)((v>>6) & 0x1f);
+    zOut[1] = 0x80 + (u8)(v & 0x3f);
+    return 2;
+  }
+  if( v<0x10000 ){
+    zOut[0] = 0xe0 + (u8)((v>>12) & 0x0f);
+    zOut[1] = 0x80 + (u8)((v>>6) & 0x3f);
+    zOut[2] = 0x80 + (u8)(v & 0x3f);
+    return 3;
+  }
+  zOut[0] = 0xf0 + (u8)((v>>18) & 0x07);
+  zOut[1] = 0x80 + (u8)((v>>12) & 0x3f);
+  zOut[2] = 0x80 + (u8)((v>>6) & 0x3f);
+  zOut[3] = 0x80 + (u8)(v & 0x3f);
+  return 4;
+}
+
 /*
 ** Translate a single UTF-8 character.  Return the unicode value.
 **
@@ -136,7 +165,7 @@ static const unsigned char sqlite3Utf8Trans1[] = {
   c = *(zIn++);                                            \
   if( c>=0xc0 ){                                           \
     c = sqlite3Utf8Trans1[c-0xc0];                         \
-    while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){            \
+    while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){             \
       c = (c<<6) + (0x3f & *(zIn++));                      \
     }                                                      \
     if( c<0x80                                             \
@@ -164,7 +193,38 @@ u32 sqlite3Utf8Read(
   return c;
 }
 
-
+/*
+** Read a single UTF8 character out of buffer z[], but reading no
+** more than n characters from the buffer.  z[] is not zero-terminated.
+**
+** Return the number of bytes used to construct the character.
+**
+** Invalid UTF8 might generate a strange result.  No effort is made
+** to detect invalid UTF8.
+**
+** At most 4 bytes will be read out of z[].  The return value will always
+** be between 1 and 4.
+*/
+int sqlite3Utf8ReadLimited(
+  const u8 *z,
+  int n,
+  u32 *piOut
+){
+  u32 c;
+  int i = 1;
+  assert( n>0 );
+  c = z[0];
+  if( c>=0xc0 ){
+    c = sqlite3Utf8Trans1[c-0xc0];
+    if( n>4 ) n = 4;
+    while( i<n && (z[i] & 0xc0)==0x80 ){
+      c = (c<<6) + (0x3f & z[i]);
+      i++;
+    }
+  }
+  *piOut = c;
+  return i;
+}
 
 
 /*
@@ -483,20 +543,22 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){
 }
 
 /*
-** zIn is a UTF-16 encoded unicode string at least nChar characters long.
+** zIn is a UTF-16 encoded unicode string at least nByte bytes long.
 ** Return the number of bytes in the first nChar unicode characters
-** in pZ.  nChar must be non-negative.
+** in pZ.  nChar must be non-negative.  Surrogate pairs count as a single
+** character.
 */
-int sqlite3Utf16ByteLen(const void *zIn, int nChar){
+int sqlite3Utf16ByteLen(const void *zIn, int nByte, int nChar){
   int c;
   unsigned char const *z = zIn;
+  unsigned char const *zEnd = &z[nByte-1];
   int n = 0;
   
   if( SQLITE_UTF16NATIVE==SQLITE_UTF16LE ) z++;
-  while( n<nChar ){
+  while( n<nChar && z<=zEnd ){
     c = z[0];
     z += 2;
-    if( c>=0xd8 && c<0xdc && z[0]>=0xdc && z[0]<0xe0 ) z += 2;
+    if( c>=0xd8 && c<0xdc && z<=zEnd && z[0]>=0xdc && z[0]<0xe0 ) z += 2;
     n++;
   }
   return (int)(z-(unsigned char const *)zIn) 
diff --git a/src/util.c b/src/util.c
index 9bfcd253b0..8f3e75846b 100644
--- a/src/util.c
+++ b/src/util.c
@@ -23,8 +23,8 @@
 
 /*
 ** Calls to sqlite3FaultSim() are used to simulate a failure during testing,
-** or to bypass normal error detection during testing in order to let 
-** execute proceed futher downstream.
+** or to bypass normal error detection during testing in order to let
+** execute proceed further downstream.
 **
 ** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0).  The
 ** sqlite3FaultSim() function only returns non-zero during testing.
@@ -68,6 +68,19 @@ int sqlite3IsNaN(double x){
 }
 #endif /* SQLITE_OMIT_FLOATING_POINT */
 
+#ifndef SQLITE_OMIT_FLOATING_POINT
+/*
+** Return true if the floating point value is NaN or +Inf or -Inf.
+*/
+int sqlite3IsOverflow(double x){
+  int rc;   /* The value return */
+  u64 y;
+  memcpy(&y,&x,sizeof(y));
+  rc = IsOvfl(y);
+  return rc;
+}
+#endif /* SQLITE_OMIT_FLOATING_POINT */
+
 /*
 ** Compute a string length that is limited to what can be stored in
 ** lower 30 bits of a 32-bit signed integer.
@@ -82,7 +95,7 @@ int sqlite3Strlen30(const char *z){
 }
 
 /*
-** Return the declared type of a column.  Or return zDflt if the column 
+** Return the declared type of a column.  Or return zDflt if the column
 ** has no declared type.
 **
 ** The column type is an extra string stored after the zero-terminator on
@@ -141,6 +154,23 @@ void sqlite3ErrorClear(sqlite3 *db){
 */
 void sqlite3SystemError(sqlite3 *db, int rc){
   if( rc==SQLITE_IOERR_NOMEM ) return;
+#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL)
+  if( rc==SQLITE_IOERR_IN_PAGE ){
+    int ii;
+    int iErr;
+    sqlite3BtreeEnterAll(db);
+    for(ii=0; ii<db->nDb; ii++){
+      if( db->aDb[ii].pBt ){
+        iErr = sqlite3PagerWalSystemErrno(sqlite3BtreePager(db->aDb[ii].pBt));
+        if( iErr ){
+          db->iSysErrno = iErr;
+        }
+      }
+    }
+    sqlite3BtreeLeaveAll(db);
+    return;
+  }
+#endif
   rc &= 0xff;
   if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){
     db->iSysErrno = sqlite3OsGetLastError(db->pVfs);
@@ -185,12 +215,16 @@ void sqlite3ProgressCheck(Parse *p){
     p->rc = SQLITE_INTERRUPT;
   }
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
-  if( db->xProgress && (++p->nProgressSteps)>=db->nProgressOps ){
-    if( db->xProgress(db->pProgressArg) ){
-      p->nErr++;
-      p->rc = SQLITE_INTERRUPT;
+  if( db->xProgress ){
+    if( p->rc==SQLITE_INTERRUPT ){
+      p->nProgressSteps = 0;
+    }else if( (++p->nProgressSteps)>=db->nProgressOps ){
+      if( db->xProgress(db->pProgressArg) ){
+        p->nErr++;
+        p->rc = SQLITE_INTERRUPT;
+      }
+      p->nProgressSteps = 0;
     }
-    p->nProgressSteps = 0;
   }
 #endif
 }
@@ -290,6 +324,44 @@ void sqlite3DequoteExpr(Expr *p){
   sqlite3Dequote(p->u.zToken);
 }
 
+/*
+** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken
+** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those
+** that contain '_' characters that must be removed before further processing.
+*/
+void sqlite3DequoteNumber(Parse *pParse, Expr *p){
+  assert( p!=0 || pParse->db->mallocFailed );
+  if( p ){
+    const char *pIn = p->u.zToken;
+    char *pOut = p->u.zToken;
+    int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X'));
+    int iValue;
+    assert( p->op==TK_QNUMBER );
+    p->op = TK_INTEGER;
+    do {
+      if( *pIn!=SQLITE_DIGIT_SEPARATOR ){
+        *pOut++ = *pIn;
+        if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT;
+      }else{
+        if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1])))
+         || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1])))
+        ){
+          sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken);
+        }
+      }
+    }while( *pIn++ );
+    if( bHex ) p->op = TK_INTEGER;
+
+    /* tag-20240227-a: If after dequoting, the number is an integer that
+    ** fits in 32 bits, then it must be converted into EP_IntValue.  Other
+    ** parts of the code expect this.  See also tag-20240227-b. */
+    if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){
+      p->u.iValue = iValue;
+      p->flags |= EP_IntValue;
+    }
+  }
+}
+
 /*
 ** If the input token p is quoted, try to adjust the token to remove
 ** the quotes.  This is not always possible:
@@ -386,43 +458,40 @@ u8 sqlite3StrIHash(const char *z){
   return h;
 }
 
-/*
-** Compute 10 to the E-th power.  Examples:  E==1 results in 10.
-** E==2 results in 100.  E==50 results in 1.0e50.
+/* Double-Double multiplication.  (x[0],x[1]) *= (y,yy)
 **
-** This routine only works for values of E between 1 and 341.
+** Reference:
+**   T. J. Dekker, "A Floating-Point Technique for Extending the
+**   Available Precision".  1971-07-26.
 */
-static LONGDOUBLE_TYPE sqlite3Pow10(int E){
-#if defined(_MSC_VER)
-  static const LONGDOUBLE_TYPE x[] = {
-    1.0e+001L,
-    1.0e+002L,
-    1.0e+004L,
-    1.0e+008L,
-    1.0e+016L,
-    1.0e+032L,
-    1.0e+064L,
-    1.0e+128L,
-    1.0e+256L
-  };
-  LONGDOUBLE_TYPE r = 1.0;
-  int i;
-  assert( E>=0 && E<=307 );
-  for(i=0; E!=0; i++, E >>=1){
-    if( E & 1 ) r *= x[i];
-  }
-  return r;
-#else
-  LONGDOUBLE_TYPE x = 10.0;
-  LONGDOUBLE_TYPE r = 1.0;
-  while(1){
-    if( E & 1 ) r *= x;
-    E >>= 1;
-    if( E==0 ) break;
-    x *= x;
-  }
-  return r; 
-#endif
+static void dekkerMul2(volatile double *x, double y, double yy){
+  /*
+  ** The "volatile" keywords on parameter x[] and on local variables
+  ** below are needed force intermediate results to be truncated to
+  ** binary64 rather than be carried around in an extended-precision
+  ** format.  The truncation is necessary for the Dekker algorithm to
+  ** work.  Intel x86 floating point might omit the truncation without
+  ** the use of volatile. 
+  */
+  volatile double tx, ty, p, q, c, cc;
+  double hx, hy;
+  u64 m;
+  memcpy(&m, (void*)&x[0], 8);
+  m &= 0xfffffffffc000000LL;
+  memcpy(&hx, &m, 8);
+  tx = x[0] - hx;
+  memcpy(&m, &y, 8);
+  m &= 0xfffffffffc000000LL;
+  memcpy(&hy, &m, 8);
+  ty = y - hy;
+  p = hx*hy;
+  q = hx*ty + tx*hy;
+  c = p+q;
+  cc = p - c + q + tx*ty;
+  cc = x[0]*yy + x[1]*y + cc;
+  x[0] = c + cc;
+  x[1] = c - x[0];
+  x[1] += cc;
 }
 
 /*
@@ -438,7 +507,7 @@ static LONGDOUBLE_TYPE sqlite3Pow10(int E){
 **      1          =>  The input string is a pure integer
 **      2 or more  =>  The input has a decimal point or eNNN clause
 **      0 or less  =>  The input string is not a valid number
-**     -1          =>  Not a valid number, but has a valid prefix which 
+**     -1          =>  Not a valid number, but has a valid prefix which
 **                     includes a decimal point and/or an eNNN clause
 **
 ** Valid numbers are in one of these formats:
@@ -463,14 +532,15 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   const char *zEnd;
   /* sign * significand * (10 ^ (esign * exponent)) */
   int sign = 1;    /* sign of significand */
-  i64 s = 0;       /* significand */
+  u64 s = 0;       /* significand */
   int d = 0;       /* adjust exponent for shifting decimal point */
   int esign = 1;   /* sign of exponent */
   int e = 0;       /* exponent */
   int eValid = 1;  /* True exponent is either not used or is well-formed */
-  double result;
   int nDigit = 0;  /* Number of digits processed */
   int eType = 1;   /* 1: pure integer,  2+: fractional  -1 or less: bad UTF16 */
+  u64 s2;          /* round-tripped significand */
+  double rr[2];
 
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
   *pResult = 0.0;   /* Default return value, in case of an error */
@@ -508,7 +578,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   while( z<zEnd && sqlite3Isdigit(*z) ){
     s = s*10 + (*z - '0');
     z+=incr; nDigit++;
-    if( s>=((LARGEST_INT64-9)/10) ){
+    if( s>=((LARGEST_UINT64-9)/10) ){
       /* skip non-significant significand digits
       ** (increase exponent by d to shift decimal left) */
       while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; }
@@ -523,7 +593,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
     /* copy digits from after decimal to significand
     ** (decrease exponent by d to shift decimal right) */
     while( z<zEnd && sqlite3Isdigit(*z) ){
-      if( s<((LARGEST_INT64-9)/10) ){
+      if( s<((LARGEST_UINT64-9)/10) ){
         s = s*10 + (*z - '0');
         d--;
         nDigit++;
@@ -539,7 +609,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
     eValid = 0;
     eType++;
 
-    /* This branch is needed to avoid a (harmless) buffer overread.  The 
+    /* This branch is needed to avoid a (harmless) buffer overread.  The
     ** special comment alerts the mutation tester that the correct answer
     ** is obtained even if the branch is omitted */
     if( z>=zEnd ) goto do_atof_calc;              /*PREVENTS-HARMLESS-OVERREAD*/
@@ -563,79 +633,76 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   while( z<zEnd && sqlite3Isspace(*z) ) z+=incr;
 
 do_atof_calc:
+  /* Zero is a special case */
+  if( s==0 ){
+    *pResult = sign<0 ? -0.0 : +0.0;
+    goto atof_return;
+  }
+
   /* adjust exponent by d, and update sign */
   e = (e*esign) + d;
-  if( e<0 ) {
-    esign = -1;
-    e *= -1;
-  } else {
-    esign = 1;
-  }
-
-  if( s==0 ) {
-    /* In the IEEE 754 standard, zero is signed. */
-    result = sign<0 ? -(double)0 : (double)0;
-  } else {
-    /* Attempt to reduce exponent.
-    **
-    ** Branches that are not required for the correct answer but which only
-    ** help to obtain the correct answer faster are marked with special
-    ** comments, as a hint to the mutation tester.
-    */
-    while( e>0 ){                                       /*OPTIMIZATION-IF-TRUE*/
-      if( esign>0 ){
-        if( s>=(LARGEST_INT64/10) ) break;             /*OPTIMIZATION-IF-FALSE*/
-        s *= 10;
-      }else{
-        if( s%10!=0 ) break;                           /*OPTIMIZATION-IF-FALSE*/
-        s /= 10;
-      }
-      e--;
-    }
 
-    /* adjust the sign of significand */
-    s = sign<0 ? -s : s;
+  /* Try to adjust the exponent to make it smaller */
+  while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){
+    s *= 10;
+    e--;
+  }
+  while( e<0 && (s%10)==0 ){
+    s /= 10;
+    e++;
+  }
 
-    if( e==0 ){                                         /*OPTIMIZATION-IF-TRUE*/
-      result = (double)s;
-    }else{
-      /* attempt to handle extremely small/large numbers better */
-      if( e>307 ){                                      /*OPTIMIZATION-IF-TRUE*/
-        if( e<342 ){                                    /*OPTIMIZATION-IF-TRUE*/
-          LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308);
-          if( esign<0 ){
-            result = s / scale;
-            result /= 1.0e+308;
-          }else{
-            result = s * scale;
-            result *= 1.0e+308;
-          }
-        }else{ assert( e>=342 );
-          if( esign<0 ){
-            result = 0.0*s;
-          }else{
-#ifdef INFINITY
-            result = INFINITY*s;
-#else
-            result = 1e308*1e308*s;  /* Infinity */
+  rr[0] = (double)s;
+  assert( sizeof(s2)==sizeof(rr[0]) );
+#ifdef SQLITE_DEBUG
+  rr[1] = 18446744073709549568.0;
+  memcpy(&s2, &rr[1], sizeof(s2));
+  assert( s2==0x43efffffffffffffLL );
 #endif
-          }
-        }
-      }else{
-        LONGDOUBLE_TYPE scale = sqlite3Pow10(e);
-        if( esign<0 ){
-          result = s / scale;
-        }else{
-          result = s * scale;
-        }
-      }
+  /* Largest double that can be safely converted to u64
+  **         vvvvvvvvvvvvvvvvvvvvvv   */
+  if( rr[0]<=18446744073709549568.0 ){
+    s2 = (u64)rr[0];
+    rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s);
+  }else{
+    rr[1] = 0.0;
+  }
+  assert( rr[1]<=1.0e-10*rr[0] );  /* Equal only when rr[0]==0.0 */
+  
+  if( e>0 ){
+    while( e>=100  ){
+      e -= 100;
+      dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83);
+    }
+    while( e>=10   ){
+      e -= 10;
+      dekkerMul2(rr, 1.0e+10, 0.0);
+    }
+    while( e>=1    ){
+      e -= 1;
+      dekkerMul2(rr, 1.0e+01, 0.0);
+    }
+  }else{
+    while( e<=-100 ){
+      e += 100;
+      dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117);
+    }
+    while( e<=-10  ){
+      e += 10;
+      dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27);
+    }
+    while( e<=-1   ){
+      e += 1;
+      dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18);
     }
   }
+  *pResult = rr[0]+rr[1];
+  if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300;
+  if( sign<0 ) *pResult = -*pResult;
+  assert( !sqlite3IsNaN(*pResult) );
 
-  /* store the result */
-  *pResult = result;
-
-  /* return true if number and no extra non-whitespace chracters after */
+atof_return:
+  /* return true if number and no extra non-whitespace characters after */
   if( z==zEnd && nDigit>0 && eValid && eType>0 ){
     return eType;
   }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){
@@ -771,7 +838,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
     /* This test and assignment is needed only to suppress UB warnings
     ** from clang and -fsanitize=undefined.  This test and assignment make
     ** the code a little larger and slower, and no harm comes from omitting
-    ** them, but we must appaise the undefined-behavior pharisees. */
+    ** them, but we must appease the undefined-behavior pharisees. */
     *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64;
   }else if( neg ){
     *pNum = -(i64)u;
@@ -849,7 +916,9 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
   }else
 #endif /* SQLITE_OMIT_HEX_INTEGER */
   {
-    return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8);
+    int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789"));
+    if( z[n] ) n++;
+    return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8);
   }
 }
 
@@ -928,6 +997,146 @@ int sqlite3Atoi(const char *z){
   return x;
 }
 
+/*
+** Decode a floating-point value into an approximate decimal
+** representation.
+**
+** If iRound<=0 then round to -iRound significant digits to the
+** the left of the decimal point, or to a maximum of mxRound total
+** significant digits.
+**
+** If iRound>0 round to min(iRound,mxRound) significant digits total.
+**
+** mxRound must be positive.
+**
+** The significant digits of the decimal representation are
+** stored in p->z[] which is a often (but not always) a pointer
+** into the middle of p->zBuf[].  There are p->n significant digits.
+** The p->z[] array is *not* zero-terminated.
+*/
+void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){
+  int i;
+  u64 v;
+  int e, exp = 0;
+  double rr[2];
+
+  p->isSpecial = 0;
+  p->z = p->zBuf;
+  assert( mxRound>0 );
+
+  /* Convert negative numbers to positive.  Deal with Infinity, 0.0, and
+  ** NaN. */
+  if( r<0.0 ){
+    p->sign = '-';
+    r = -r;
+  }else if( r==0.0 ){
+    p->sign = '+';
+    p->n = 1;
+    p->iDP = 1;
+    p->z = "0";
+    return;
+  }else{
+    p->sign = '+';
+  }
+  memcpy(&v,&r,8);
+  e = v>>52;
+  if( (e&0x7ff)==0x7ff ){
+    p->isSpecial = 1 + (v!=0x7ff0000000000000LL);
+    p->n = 0;
+    p->iDP = 0;
+    return;
+  }
+
+  /* Multiply r by powers of ten until it lands somewhere in between
+  ** 1.0e+19 and 1.0e+17.
+  **
+  ** Use Dekker-style double-double computation to increase the
+  ** precision.
+  **
+  ** The error terms on constants like 1.0e+100 computed using the
+  ** decimal extension, for example as follows:
+  **
+  **   SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100)));
+  */
+  rr[0] = r;
+  rr[1] = 0.0;
+  if( rr[0]>9.223372036854774784e+18 ){
+    while( rr[0]>9.223372036854774784e+118 ){
+      exp += 100;
+      dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117);
+    }
+    while( rr[0]>9.223372036854774784e+28 ){
+      exp += 10;
+      dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27);
+    }
+    while( rr[0]>9.223372036854774784e+18 ){
+      exp += 1;
+      dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18);
+    }
+  }else{
+    while( rr[0]<9.223372036854774784e-83  ){
+      exp -= 100;
+      dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83);
+    }
+    while( rr[0]<9.223372036854774784e+07  ){
+      exp -= 10;
+      dekkerMul2(rr, 1.0e+10, 0.0);
+    }
+    while( rr[0]<9.22337203685477478e+17  ){
+      exp -= 1;
+      dekkerMul2(rr, 1.0e+01, 0.0);
+    }
+  }
+  v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1];
+
+  /* Extract significant digits. */
+  i = sizeof(p->zBuf)-1;
+  assert( v>0 );
+  while( v ){  p->zBuf[i--] = (v%10) + '0'; v /= 10; }
+  assert( i>=0 && i<sizeof(p->zBuf)-1 );
+  p->n = sizeof(p->zBuf) - 1 - i;
+  assert( p->n>0 );
+  assert( p->n<sizeof(p->zBuf) );
+  p->iDP = p->n + exp;
+  if( iRound<=0 ){
+    iRound = p->iDP - iRound;
+    if( iRound==0 && p->zBuf[i+1]>='5' ){
+      iRound = 1;
+      p->zBuf[i--] = '0';
+      p->n++;
+      p->iDP++;
+    }
+  }
+  if( iRound>0 && (iRound<p->n || p->n>mxRound) ){
+    char *z = &p->zBuf[i+1];
+    if( iRound>mxRound ) iRound = mxRound;
+    p->n = iRound;
+    if( z[iRound]>='5' ){
+      int j = iRound-1;
+      while( 1 /*exit-by-break*/ ){
+        z[j]++;
+        if( z[j]<='9' ) break;
+        z[j] = '0';
+        if( j==0 ){
+          p->z[i--] = '1';
+          p->n++;
+          p->iDP++;
+          break;
+        }else{
+          j--;
+        }
+      }
+    }
+  }
+  p->z = &p->zBuf[i+1];
+  assert( i+p->n < sizeof(p->zBuf) );
+  assert( p->n>0 );
+  while( p->z[p->n-1]=='0' ){
+    p->n--;
+    assert( p->n>0 );
+  }
+}
+
 /*
 ** Try to convert z into an unsigned 32-bit integer.  Return true on
 ** success and false if there is an error.
@@ -986,7 +1195,7 @@ static int SQLITE_NOINLINE putVarint64(unsigned char *p, u64 v){
       v >>= 7;
     }
     return 9;
-  }    
+  }   
   n = 0;
   do{
     buf[n++] = (u8)((v & 0x7f) | 0x80);
@@ -1186,126 +1395,37 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){
 ** If the varint stored in p[0] is larger than can fit in a 32-bit unsigned
 ** integer, then set *v to 0xffffffff.
 **
-** A MACRO version, getVarint32, is provided which inlines the 
-** single-byte case.  All code should use the MACRO version as 
+** A MACRO version, getVarint32, is provided which inlines the
+** single-byte case.  All code should use the MACRO version as
 ** this function assumes the single-byte case has already been handled.
 */
 u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){
-  u32 a,b;
+  u64 v64;
+  u8 n;
 
-  /* The 1-byte case.  Overwhelmingly the most common.  Handled inline
-  ** by the getVarin32() macro */
-  a = *p;
-  /* a: p0 (unmasked) */
-#ifndef getVarint32
-  if (!(a&0x80))
-  {
-    /* Values between 0 and 127 */
-    *v = a;
-    return 1;
-  }
-#endif
+  /* Assume that the single-byte case has already been handled by
+  ** the getVarint32() macro */
+  assert( (p[0] & 0x80)!=0 );
 
-  /* The 2-byte case */
-  p++;
-  b = *p;
-  /* b: p1 (unmasked) */
-  if (!(b&0x80))
-  {
-    /* Values between 128 and 16383 */
-    a &= 0x7f;
-    a = a<<7;
-    *v = a | b;
+  if( (p[1] & 0x80)==0 ){
+    /* This is the two-byte case */
+    *v = ((p[0]&0x7f)<<7) | p[1];
     return 2;
   }
-
-  /* The 3-byte case */
-  p++;
-  a = a<<14;
-  a |= *p;
-  /* a: p0<<14 | p2 (unmasked) */
-  if (!(a&0x80))
-  {
-    /* Values between 16384 and 2097151 */
-    a &= (0x7f<<14)|(0x7f);
-    b &= 0x7f;
-    b = b<<7;
-    *v = a | b;
+  if( (p[2] & 0x80)==0 ){
+    /* This is the three-byte case */
+    *v = ((p[0]&0x7f)<<14) | ((p[1]&0x7f)<<7) | p[2];
     return 3;
   }
-
-  /* A 32-bit varint is used to store size information in btrees.
-  ** Objects are rarely larger than 2MiB limit of a 3-byte varint.
-  ** A 3-byte varint is sufficient, for example, to record the size
-  ** of a 1048569-byte BLOB or string.
-  **
-  ** We only unroll the first 1-, 2-, and 3- byte cases.  The very
-  ** rare larger cases can be handled by the slower 64-bit varint
-  ** routine.
-  */
-#if 1
-  {
-    u64 v64;
-    u8 n;
-
-    n = sqlite3GetVarint(p-2, &v64);
-    assert( n>3 && n<=9 );
-    if( (v64 & SQLITE_MAX_U32)!=v64 ){
-      *v = 0xffffffff;
-    }else{
-      *v = (u32)v64;
-    }
-    return n;
-  }
-
-#else
-  /* For following code (kept for historical record only) shows an
-  ** unrolling for the 3- and 4-byte varint cases.  This code is
-  ** slightly faster, but it is also larger and much harder to test.
-  */
-  p++;
-  b = b<<14;
-  b |= *p;
-  /* b: p1<<14 | p3 (unmasked) */
-  if (!(b&0x80))
-  {
-    /* Values between 2097152 and 268435455 */
-    b &= (0x7f<<14)|(0x7f);
-    a &= (0x7f<<14)|(0x7f);
-    a = a<<7;
-    *v = a | b;
-    return 4;
-  }
-
-  p++;
-  a = a<<14;
-  a |= *p;
-  /* a: p0<<28 | p2<<14 | p4 (unmasked) */
-  if (!(a&0x80))
-  {
-    /* Values  between 268435456 and 34359738367 */
-    a &= SLOT_4_2_0;
-    b &= SLOT_4_2_0;
-    b = b<<7;
-    *v = a | b;
-    return 5;
-  }
-
-  /* We can only reach this point when reading a corrupt database
-  ** file.  In that case we are not in any hurry.  Use the (relatively
-  ** slow) general-purpose sqlite3GetVarint() routine to extract the
-  ** value. */
-  {
-    u64 v64;
-    u8 n;
-
-    p -= 4;
-    n = sqlite3GetVarint(p, &v64);
-    assert( n>5 && n<=9 );
+  /* four or more bytes */
+  n = sqlite3GetVarint(p, &v64);
+  assert( n>3 && n<=9 );
+  if( (v64 & SQLITE_MAX_U32)!=v64 ){
+    *v = 0xffffffff;
+  }else{
     *v = (u32)v64;
-    return n;
   }
-#endif
+  return n;
 }
 
 /*
@@ -1406,7 +1526,7 @@ void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){
 ** argument.  The zType is a word like "NULL" or "closed" or "invalid".
 */
 static void logBadConnection(const char *zType){
-  sqlite3_log(SQLITE_MISUSE, 
+  sqlite3_log(SQLITE_MISUSE,
      "API call with %s database connection pointer",
      zType
   );
@@ -1458,7 +1578,7 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){
 }
 
 /*
-** Attempt to add, substract, or multiply the 64-bit signed value iB against
+** Attempt to add, subtract, or multiply the 64-bit signed value iB against
 ** the other 64-bit signed integer at *pA and store the result in *pA.
 ** Return 0 on success.  Or if the operation would have resulted in an
 ** overflow, leave *pA unchanged and return 1.
@@ -1480,7 +1600,7 @@ int sqlite3AddInt64(i64 *pA, i64 iB){
     if( iA<0 && -(iA + LARGEST_INT64) > iB + 1 ) return 1;
   }
   *pA += iB;
-  return 0; 
+  return 0;
 #endif
 }
 int sqlite3SubInt64(i64 *pA, i64 iB){
@@ -1521,7 +1641,7 @@ int sqlite3MulInt64(i64 *pA, i64 iB){
 }
 
 /*
-** Compute the absolute value of a 32-bit signed integer, of possible.  Or 
+** Compute the absolute value of a 32-bit signed integer, if possible.  Or
 ** if the integer has a value of -2147483648, return +2147483647
 */
 int sqlite3AbsInt32(int x){
@@ -1561,11 +1681,11 @@ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){
 }
 #endif
 
-/* 
+/*
 ** Find (an approximate) sum of two LogEst values.  This computation is
 ** not a simple "+" operator because LogEst is stored as a logarithmic
 ** value.
-** 
+**
 */
 LogEst sqlite3LogEstAdd(LogEst a, LogEst b){
   static const unsigned char x[] = {
@@ -1663,8 +1783,8 @@ u64 sqlite3LogEstToInt(LogEst x){
 ** Conceptually:
 **
 **    struct VList {
-**      int nAlloc;   // Number of allocated slots 
-**      int nUsed;    // Number of used slots 
+**      int nAlloc;   // Number of allocated slots
+**      int nUsed;    // Number of used slots
 **      struct VListEntry {
 **        int iValue;    // Value for this entry
 **        int nSlot;     // Slots used by this entry
@@ -1673,7 +1793,7 @@ u64 sqlite3LogEstToInt(LogEst x){
 **    }
 **
 ** During code generation, pointers to the variable names within the
-** VList are taken.  When that happens, nAlloc is set to zero as an 
+** VList are taken.  When that happens, nAlloc is set to zero as an
 ** indication that the VList may never again be enlarged, since the
 ** accompanying realloc() would invalidate the pointers.
 */
@@ -1743,12 +1863,3 @@ int sqlite3VListNameToNum(VList *pIn, const char *zName, int nName){
   }while( i<mx );
   return 0;
 }
-
-/*
-** High-resolution hardware timer used for debugging and testing only.
-*/
-#if defined(VDBE_PROFILE)  \
- || defined(SQLITE_PERFORMANCE_TRACE) \
- || defined(SQLITE_ENABLE_STMT_SCANSTATUS) 
-# include "hwtime.h"
-#endif
diff --git a/src/vacuum.c b/src/vacuum.c
index 5153cf35a6..8e3ce29eb1 100644
--- a/src/vacuum.c
+++ b/src/vacuum.c
@@ -116,7 +116,7 @@ void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){
 #else
     /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
     ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
-    ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
+    ** occurred on 2016-08-19 (https://sqlite.org/src/info/083f9e6270).
     ** The buggy behavior is required for binary compatibility with some
     ** legacy applications. */
     iDb = sqlite3FindDb(pParse->db, pNm);
@@ -162,6 +162,9 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
   const char *zDbMain;    /* Schema name of database to vacuum */
   const char *zOut;       /* Name of output file */
   u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */
+  u64 iRandom;            /* Random value used for zDbVacuum[] */
+  char zDbVacuum[42];     /* Name of the ATTACH-ed database used for vacuum */
+
 
   if( !db->autoCommit ){
     sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
@@ -192,7 +195,8 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
   saved_nChange = db->nChange;
   saved_nTotalChange = db->nTotalChange;
   saved_mTrace = db->mTrace;
-  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
+  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments
+               | SQLITE_AttachCreate | SQLITE_AttachWrite;
   db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
   db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder
                    | SQLITE_Defensive | SQLITE_CountRows);
@@ -202,27 +206,29 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
   pMain = db->aDb[iDb].pBt;
   isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));
 
-  /* Attach the temporary database as 'vacuum_db'. The synchronous pragma
+  /* Attach the temporary database as 'vacuum_XXXXXX'. The synchronous pragma
   ** can be set to 'off' for this file, as it is not recovered if a crash
   ** occurs anyway. The integrity of the database is maintained by a
   ** (possibly synchronous) transaction opened on the main database before
   ** sqlite3BtreeCopyFile() is called.
   **
-  ** An optimisation would be to use a non-journaled pager.
-  ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but
+  ** An optimization would be to use a non-journaled pager.
+  ** (Later:) I tried setting "PRAGMA vacuum_XXXXXX.journal_mode=OFF" but
   ** that actually made the VACUUM run slower.  Very little journalling
   ** actually occurs when doing a vacuum since the vacuum_db is initially
   ** empty.  Only the journal header is written.  Apparently it takes more
   ** time to parse and run the PRAGMA to turn journalling off than it does
   ** to write the journal header file.
   */
+  sqlite3_randomness(sizeof(iRandom),&iRandom);
+  sqlite3_snprintf(sizeof(zDbVacuum), zDbVacuum, "vacuum_%016llx", iRandom);
   nDb = db->nDb;
-  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut);
+  rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS %s", zOut, zDbVacuum);
   db->openFlags = saved_openFlags;
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
   assert( (db->nDb-1)==nDb );
   pDb = &db->aDb[nDb];
-  assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
+  assert( strcmp(pDb->zDbSName,zDbVacuum)==0 );
   pTemp = pDb->pBt;
   if( pOut ){
     sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp));
@@ -246,10 +252,12 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
 #ifdef SQLITE_HAS_CODEC
   if( db->nextPagesize ){
     extern void sqlcipherCodecGetKey(sqlite3*, int, void**, int*);
+    extern void sqlcipher_free(void*, sqlite3_uint64);
     int nKey;
     char *zKey;
     sqlcipherCodecGetKey(db, iDb, (void**)&zKey, &nKey);
     if( nKey ) db->nextPagesize = 0;
+    if(nKey) sqlcipher_free(zKey, nKey);
   }
 #endif
 /* END SQLCIPHER */
@@ -312,11 +320,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
   ** the contents to the temporary database.
   */
   rc = execSqlF(db, pzErrMsg,
-      "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+      "SELECT'INSERT INTO %s.'||quote(name)"
       "||' SELECT*FROM\"%w\".'||quote(name)"
-      "FROM vacuum_db.sqlite_schema "
+      "FROM %s.sqlite_schema "
       "WHERE type='table'AND coalesce(rootpage,1)>0",
-      zDbMain
+      zDbVacuum, zDbMain, zDbVacuum
   );
   assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 );
   db->mDbFlags &= ~DBFLAG_Vacuum;
@@ -328,11 +336,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum(
   ** from the schema table.
   */
   rc = execSqlF(db, pzErrMsg,
-      "INSERT INTO vacuum_db.sqlite_schema"
+      "INSERT INTO %s.sqlite_schema"
       " SELECT*FROM \"%w\".sqlite_schema"
       " WHERE type IN('view','trigger')"
       " OR(type='table'AND rootpage=0)",
-      zDbMain
+      zDbVacuum, zDbMain
   );
   if( rc ) goto end_of_vacuum;
 
diff --git a/src/vdbe.c b/src/vdbe.c
index 2aa4e6df24..b5a262e636 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -21,6 +21,15 @@
 #include "sqliteInt.h"
 #include "vdbeInt.h"
 
+/*
+** High-resolution hardware timer used for debugging and testing only.
+*/
+#if defined(VDBE_PROFILE)  \
+ || defined(SQLITE_PERFORMANCE_TRACE) \
+ || defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+# include "hwtime.h"
+#endif
+
 /*
 ** Invoke this macro on memory cells just prior to changing the
 ** value of the cell.  This macro verifies that shallow copies are
@@ -132,11 +141,12 @@ int sqlite3_found_count = 0;
 **   sqlite3CantopenError(lineno)
 */
 static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){
-  static int n = 0;
+  static u64 n = 0;
   (void)pc;
   (void)pOp;
   (void)v;
   n++;
+  if( n==LARGEST_UINT64 ) abort(); /* So that n is used, preventing a warning */
 }
 #endif
 
@@ -154,7 +164,7 @@ static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){
 **
 ** In other words, if M is 2, then I is either 0 (for fall-through) or
 ** 1 (for when the branch is taken).  If M is 3, the I is 0 for an
-** ordinary fall-through, I is 1 if the branch was taken, and I is 2 
+** ordinary fall-through, I is 1 if the branch was taken, and I is 2
 ** if the result of comparison is NULL.  For M=3, I=2 the jump may or
 ** may not be taken, depending on the SQLITE_JUMPIFNULL flags in p5.
 ** When M is 4, that means that an OP_Jump is being run.  I is 0, 1, or 2
@@ -247,7 +257,7 @@ static VdbeCursor *allocateCursor(
   u8 eCurType           /* Type of the new cursor */
 ){
   /* Find the memory cell that will be used to store the blob of memory
-  ** required for this VdbeCursor structure. It is convenient to use a 
+  ** required for this VdbeCursor structure. It is convenient to use a
   ** vdbe memory cell to manage the memory allocation required for a
   ** VdbeCursor structure for the following reasons:
   **
@@ -266,11 +276,11 @@ static VdbeCursor *allocateCursor(
   */
   Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem;
 
-  int nByte;
+  i64 nByte;
   VdbeCursor *pCx = 0;
-  nByte = 
-      ROUND8P(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + 
-      (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0);
+  nByte = SZ_VDBECURSOR(nField);
+  assert( ROUND8(nByte)==nByte );
+  if( eCurType==CURTYPE_BTREE ) nByte += sqlite3BtreeCursorSize();
 
   assert( iCur>=0 && iCur<p->nCursor );
   if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/
@@ -294,7 +304,7 @@ static VdbeCursor *allocateCursor(
       pMem->szMalloc = 0;
       return 0;
     }
-    pMem->szMalloc = nByte;
+    pMem->szMalloc = (int)nByte;
   }
 
   p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->zMalloc;
@@ -303,8 +313,8 @@ static VdbeCursor *allocateCursor(
   pCx->nField = nField;
   pCx->aOffset = &pCx->aType[nField];
   if( eCurType==CURTYPE_BTREE ){
-    pCx->uc.pCursor = (BtCursor*)
-        &pMem->z[ROUND8P(sizeof(VdbeCursor))+2*sizeof(u32)*nField];
+    assert( ROUND8(SZ_VDBECURSOR(nField))==SZ_VDBECURSOR(nField) );
+    pCx->uc.pCursor = (BtCursor*)&pMem->z[SZ_VDBECURSOR(nField)];
     sqlite3BtreeCursorZero(pCx->uc.pCursor);
   }
   return pCx;
@@ -368,7 +378,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){
 ** SQLITE_AFF_INTEGER:
 ** SQLITE_AFF_REAL:
 ** SQLITE_AFF_NUMERIC:
-**    Try to convert pRec to an integer representation or a 
+**    Try to convert pRec to an integer representation or a
 **    floating-point representation if an integer representation
 **    is not possible.  Note that the integer representation is
 **    always preferred, even if the affinity is REAL, because
@@ -403,7 +413,7 @@ static void applyAffinity(
   }else if( affinity==SQLITE_AFF_TEXT ){
     /* Only attempt the conversion to TEXT if there is an integer or real
     ** representation (blob and NULL do not get converted) but no string
-    ** representation.  It would be harmless to repeat the conversion if 
+    ** representation.  It would be harmless to repeat the conversion if
     ** there is already a string rep, but it is pointless to waste those
     ** CPU cycles. */
     if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/
@@ -435,12 +445,12 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){
 }
 
 /*
-** Exported version of applyAffinity(). This one works on sqlite3_value*, 
+** Exported version of applyAffinity(). This one works on sqlite3_value*,
 ** not the internal Mem* type.
 */
 void sqlite3ValueApplyAffinity(
-  sqlite3_value *pVal, 
-  u8 affinity, 
+  sqlite3_value *pVal,
+  u8 affinity,
   u8 enc
 ){
   applyAffinity((Mem *)pVal, affinity, enc);
@@ -478,7 +488,7 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){
 
 /*
 ** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or
-** none.  
+** none. 
 **
 ** Unlike applyNumericAffinity(), this routine does not modify pMem->flags.
 ** But it does set pMem->u.r and pMem->u.i appropriately.
@@ -556,6 +566,9 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){
       sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.');
     }
     sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]);
+    if( f & MEM_Term ){
+      sqlite3_str_appendf(pStr, "(0-term)");
+    }
   }
 }
 #endif
@@ -594,6 +607,7 @@ static void registerTrace(int iReg, Mem *p){
   printf("R[%d] = ", iReg);
   memTracePrint(p);
   if( p->pScopyFrom ){
+    assert( p->pScopyFrom->bScopy );
     printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg]));
   }
   printf("\n");
@@ -628,9 +642,9 @@ void sqlite3VdbeRegisterDump(Vdbe *v){
 /*
 ** This function is only called from within an assert() expression. It
 ** checks that the sqlite3.nTransaction variable is correctly set to
-** the number of non-transaction savepoints currently in the 
+** the number of non-transaction savepoints currently in the
 ** linked list starting at sqlite3.pSavepoint.
-** 
+**
 ** Usage:
 **
 **     assert( checkSavepointCount(db) );
@@ -692,6 +706,123 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){
   return h;
 }
 
+
+/*
+** For OP_Column, factor out the case where content is loaded from
+** overflow pages, so that the code to implement this case is separate
+** the common case where all content fits on the page.  Factoring out
+** the code reduces register pressure and helps the common case
+** to run faster.
+*/
+static SQLITE_NOINLINE int vdbeColumnFromOverflow(
+  VdbeCursor *pC,       /* The BTree cursor from which we are reading */
+  int iCol,             /* The column to read */
+  u32 t,                /* The serial-type code for the column value */
+  i64 iOffset,          /* Offset to the start of the content value */
+  u32 cacheStatus,      /* Current Vdbe.cacheCtr value */
+  u32 colCacheCtr,      /* Current value of the column cache counter */
+  Mem *pDest            /* Store the value into this register. */
+){
+  int rc;
+  sqlite3 *db = pDest->db;
+  int encoding = pDest->enc;
+  int len = sqlite3VdbeSerialTypeLen(t);
+  assert( pC->eCurType==CURTYPE_BTREE );
+  if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG;
+  if( len > 4000 && pC->pKeyInfo==0 ){
+    /* Cache large column values that are on overflow pages using
+    ** an RCStr (reference counted string) so that if they are reloaded,
+    ** that do not have to be copied a second time.  The overhead of
+    ** creating and managing the cache is such that this is only
+    ** profitable for larger TEXT and BLOB values.
+    **
+    ** Only do this on table-btrees so that writes to index-btrees do not
+    ** need to clear the cache.  This buys performance in the common case
+    ** in exchange for generality.
+    */
+    VdbeTxtBlbCache *pCache;
+    char *pBuf;
+    if( pC->colCache==0 ){
+      pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) );
+      if( pC->pCache==0 ) return SQLITE_NOMEM;
+      pC->colCache = 1;
+    }
+    pCache = pC->pCache;
+    if( pCache->pCValue==0
+     || pCache->iCol!=iCol
+     || pCache->cacheStatus!=cacheStatus
+     || pCache->colCacheCtr!=colCacheCtr
+     || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor)
+    ){
+      if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue);
+      pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 );
+      if( pBuf==0 ) return SQLITE_NOMEM;
+      rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf);
+      if( rc ) return rc;
+      pBuf[len] = 0;
+      pBuf[len+1] = 0;
+      pBuf[len+2] = 0;
+      pCache->iCol = iCol;
+      pCache->cacheStatus = cacheStatus;
+      pCache->colCacheCtr = colCacheCtr;
+      pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor);
+    }else{
+      pBuf = pCache->pCValue;
+    }
+    assert( t>=12 );
+    sqlite3RCStrRef(pBuf);
+    if( t&1 ){
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding,
+                                sqlite3RCStrUnref);
+      pDest->flags |= MEM_Term;
+    }else{
+      rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0,
+                                sqlite3RCStrUnref);
+    }
+  }else{
+    rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest);
+    if( rc ) return rc;
+    sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
+    if( (t&1)!=0 && encoding==SQLITE_UTF8 ){
+      pDest->z[len] = 0;
+      pDest->flags |= MEM_Term;
+    }
+  }
+  pDest->flags &= ~MEM_Ephem;
+  return rc;
+}
+
+/*
+** Send a "statement aborts" message to the error log.
+*/
+static SQLITE_NOINLINE void sqlite3VdbeLogAbort(
+  Vdbe *p,     /* The statement that is running at the time of failure */
+  int rc,      /* Error code */
+  Op *pOp,     /* Opcode that filed */
+  Op *aOp      /* All opcodes */
+){
+  const char *zSql = p->zSql;   /* Original SQL text */
+  const char *zPrefix = "";     /* Prefix added to SQL text */
+  int pc;                       /* Opcode address */
+  char zXtra[100];              /* Buffer space to store zPrefix */
+
+  if( p->pFrame ){
+    assert( aOp[0].opcode==OP_Init );
+    if( aOp[0].p4.z!=0 ){
+      assert( aOp[0].p4.z[0]=='-' 
+           && aOp[0].p4.z[1]=='-' 
+           && aOp[0].p4.z[2]==' ' );
+      sqlite3_snprintf(sizeof(zXtra), zXtra,"/* %s */ ",aOp[0].p4.z+3);
+      zPrefix = zXtra;
+    }else{
+      zPrefix = "/* unknown trigger */ ";
+    }
+  }
+  pc = (int)(pOp - aOp);
+  sqlite3_log(rc, "statement aborts at %d: %s; [%s%s]",
+                   pc, p->zErrMsg, zPrefix, zSql);
+}
+
 /*
 ** Return the symbolic name for the data type of a pMem
 */
@@ -708,7 +839,7 @@ static const char *vdbeMemTypeName(Mem *pMem){
 
 /*
 ** Execute as much of a VDBE program as we can.
-** This is the core of sqlite3_step().  
+** This is the core of sqlite3_step(). 
 */
 int sqlite3VdbeExec(
   Vdbe *p                    /* The VDBE */
@@ -734,6 +865,7 @@ int sqlite3VdbeExec(
   Mem *pIn2 = 0;             /* 2nd input operand */
   Mem *pIn3 = 0;             /* 3rd input operand */
   Mem *pOut = 0;             /* Output operand */
+  u32 colCacheCtr = 0;       /* Column cache counter */
 #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
   u64 *pnCycle = 0;
   int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0;
@@ -822,7 +954,7 @@ int sqlite3VdbeExec(
       test_trace_breakpoint((int)(pOp - aOp),pOp,p);
     }
 #endif
-      
+     
 
     /* Check to see if we need to simulate an interrupt.  This only happens
     ** if we have a special test build.
@@ -876,7 +1008,7 @@ int sqlite3VdbeExec(
 #ifdef SQLITE_DEBUG
     pOrigOp = pOp;
 #endif
-  
+ 
     switch( pOp->opcode ){
 
 /*****************************************************************************
@@ -917,7 +1049,7 @@ int sqlite3VdbeExec(
 /* Opcode:  Goto * P2 * * *
 **
 ** An unconditional jump to address P2.
-** The next instruction executed will be 
+** The next instruction executed will be
 ** the one at index P2 from the beginning of
 ** the program.
 **
@@ -929,8 +1061,8 @@ int sqlite3VdbeExec(
 case OP_Goto: {             /* jump */
 
 #ifdef SQLITE_DEBUG
-  /* In debuggging mode, when the p5 flags is set on an OP_Goto, that
-  ** means we should really jump back to the preceeding OP_ReleaseReg
+  /* In debugging mode, when the p5 flags is set on an OP_Goto, that
+  ** means we should really jump back to the preceding OP_ReleaseReg
   ** instruction. */
   if( pOp->p5 ){
     assert( pOp->p2 < (int)(pOp - aOp) );
@@ -947,7 +1079,7 @@ case OP_Goto: {             /* jump */
   /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev,
   ** OP_VNext, or OP_SorterNext) all jump here upon
   ** completion.  Check to see if sqlite3_interrupt() has been called
-  ** or if the progress callback needs to be invoked. 
+  ** or if the progress callback needs to be invoked.
   **
   ** This code uses unstructured "goto" statements and does not look clean.
   ** But that is not due to sloppy coding habits. The code is written this
@@ -973,7 +1105,7 @@ case OP_Goto: {             /* jump */
     }
   }
 #endif
-  
+ 
   break;
 }
 
@@ -1037,7 +1169,7 @@ case OP_Return: {           /* in1 */
 **
 ** See also: EndCoroutine
 */
-case OP_InitCoroutine: {     /* jump */
+case OP_InitCoroutine: {     /* jump0 */
   assert( pOp->p1>0 &&  pOp->p1<=(p->nMem+1 - p->nCursor) );
   assert( pOp->p2>=0 && pOp->p2<p->nOp );
   assert( pOp->p3>=0 && pOp->p3<p->nOp );
@@ -1060,7 +1192,9 @@ case OP_InitCoroutine: {     /* jump */
 **
 ** The instruction at the address in register P1 is a Yield.
 ** Jump to the P2 parameter of that Yield.
-** After the jump, register P1 becomes undefined.
+** After the jump, the value register P1 is left with a value
+** such that subsequent OP_Yields go back to the this same
+** OP_EndCoroutine instruction.
 **
 ** See also: InitCoroutine
 */
@@ -1072,8 +1206,8 @@ case OP_EndCoroutine: {           /* in1 */
   pCaller = &aOp[pIn1->u.i];
   assert( pCaller->opcode==OP_Yield );
   assert( pCaller->p2>=0 && pCaller->p2<p->nOp );
+  pIn1->u.i = (int)(pOp - p->aOp) - 1;
   pOp = &aOp[pCaller->p2 - 1];
-  pIn1->flags = MEM_Undefined;
   break;
 }
 
@@ -1090,7 +1224,7 @@ case OP_EndCoroutine: {           /* in1 */
 **
 ** See also: InitCoroutine
 */
-case OP_Yield: {            /* in1, jump */
+case OP_Yield: {            /* in1, jump0 */
   int pcDest;
   pIn1 = &aMem[pOp->p1];
   assert( VdbeMemDynamic(pIn1)==0 );
@@ -1120,7 +1254,7 @@ case OP_HaltIfNull: {      /* in3 */
   /* no break */ deliberate_fall_through
 }
 
-/* Opcode:  Halt P1 P2 * P4 P5
+/* Opcode:  Halt P1 P2 P3 P4 P5
 **
 ** Exit immediately.  All open cursors, etc are closed
 ** automatically.
@@ -1131,20 +1265,24 @@ case OP_HaltIfNull: {      /* in3 */
 ** whether or not to rollback the current transaction.  Do not rollback
 ** if P2==OE_Fail. Do the rollback if P2==OE_Rollback.  If P2==OE_Abort,
 ** then back out all changes that have occurred during this execution of the
-** VDBE, but do not rollback the transaction. 
+** VDBE, but do not rollback the transaction.
 **
-** If P4 is not null then it is an error message string.
+** If P3 is not zero and P4 is NULL, then P3 is a register that holds the
+** text of an error message.
 **
-** P5 is a value between 0 and 4, inclusive, that modifies the P4 string.
+** If P3 is zero and P4 is not null then the error message string is held
+** in P4.
 **
-**    0:  (no change)
-**    1:  NOT NULL contraint failed: P4
+** P5 is a value between 1 and 4, inclusive, then the P4 error message
+** string is modified as follows:
+**
+**    1:  NOT NULL constraint failed: P4
 **    2:  UNIQUE constraint failed: P4
 **    3:  CHECK constraint failed: P4
 **    4:  FOREIGN KEY constraint failed: P4
 **
-** If P5 is not zero and P4 is NULL, then everything after the ":" is
-** omitted.
+** If P3 is zero and P5 is not zero and P4 is NULL, then everything after
+** the ":" is omitted.
 **
 ** There is an implied "Halt 0 0 0" instruction inserted at the very end of
 ** every program.  So a jump past the last instruction of the program
@@ -1157,6 +1295,9 @@ case OP_Halt: {
 #ifdef SQLITE_DEBUG
   if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); }
 #endif
+  assert( pOp->p4type==P4_NOTUSED
+       || pOp->p4type==P4_STATIC
+       || pOp->p4type==P4_DYNAMIC );
 
   /* A deliberately coded "OP_Halt SQLITE_INTERNAL * * * *" opcode indicates
   ** something is wrong with the code generator.  Raise an assertion in order
@@ -1171,7 +1312,7 @@ case OP_Halt: {
     sqlite3VdbeSetChanges(db, p->nChange);
     pcx = sqlite3VdbeFrameRestore(pFrame);
     if( pOp->p2==OE_Ignore ){
-      /* Instruction pcx is the OP_Program that invoked the sub-program 
+      /* Instruction pcx is the OP_Program that invoked the sub-program
       ** currently being halted. If the p2 instruction of this OP_Halt
       ** instruction is set to OE_Ignore, then the sub-program is throwing
       ** an IGNORE exception. In this case jump to the address specified
@@ -1187,7 +1328,12 @@ case OP_Halt: {
   p->errorAction = (u8)pOp->p2;
   assert( pOp->p5<=4 );
   if( p->rc ){
-    if( pOp->p5 ){
+    if( pOp->p3>0 && pOp->p4type==P4_NOTUSED ){
+      const char *zErr;
+      assert( pOp->p3<=(p->nMem + 1 - p->nCursor) );
+      zErr = sqlite3ValueText(&aMem[pOp->p3], SQLITE_UTF8);
+      sqlite3VdbeError(p, "%s", zErr);
+    }else if( pOp->p5 ){
       static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK",
                                              "FOREIGN KEY" };
       testcase( pOp->p5==1 );
@@ -1201,8 +1347,7 @@ case OP_Halt: {
     }else{
       sqlite3VdbeError(p, "%s", pOp->p4.z);
     }
-    pcx = (int)(pOp - aOp);
-    sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg);
+    sqlite3VdbeLogAbort(p, pOp->p1, pOp, aOp);
   }
   rc = sqlite3VdbeHalt(p);
   assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR );
@@ -1259,7 +1404,7 @@ case OP_Real: {            /* same as TK_FLOAT, out2 */
 /* Opcode: String8 * P2 * P4 *
 ** Synopsis: r[P2]='P4'
 **
-** P4 points to a nul terminated UTF-8 string. This opcode is transformed 
+** P4 points to a nul terminated UTF-8 string. This opcode is transformed
 ** into a String opcode before it is executed for the first time.  During
 ** this transformation, the length of string P4 is computed and stored
 ** as the P1 parameter.
@@ -1295,7 +1440,7 @@ case OP_String8: {         /* same as TK_STRING, out2 */
   /* Fall through to the next case, OP_String */
   /* no break */ deliberate_fall_through
 }
-  
+ 
 /* Opcode: String P1 P2 P3 P4 P5
 ** Synopsis: r[P2]='P4' (len=P1)
 **
@@ -1420,19 +1565,15 @@ case OP_Blob: {                /* out2 */
   break;
 }
 
-/* Opcode: Variable P1 P2 * P4 *
-** Synopsis: r[P2]=parameter(P1,P4)
+/* Opcode: Variable P1 P2 * * *
+** Synopsis: r[P2]=parameter(P1)
 **
 ** Transfer the values of bound parameter P1 into register P2
-**
-** If the parameter is named, then its name appears in P4.
-** The P4 value is used by sqlite3_bind_parameter_name().
 */
 case OP_Variable: {            /* out2 */
   Mem *pVar;       /* Value being transferred */
 
   assert( pOp->p1>0 && pOp->p1<=p->nVar );
-  assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) );
   pVar = &p->aVar[pOp->p1 - 1];
   if( sqlite3VdbeMemTooBig(pVar) ){
     goto too_big;
@@ -1479,6 +1620,7 @@ case OP_Move: {
     { int i;
       for(i=1; i<p->nMem; i++){
         if( aMem[i].pScopyFrom==pIn1 ){
+          assert( aMem[i].bScopy );
           aMem[i].pScopyFrom = pOut;
         }
       }
@@ -1551,6 +1693,7 @@ case OP_SCopy: {            /* out2 */
 #ifdef SQLITE_DEBUG
   pOut->pScopyFrom = pIn1;
   pOut->mScopyFlags = pIn1->flags;
+  pIn1->bScopy = 1;
 #endif
   break;
 }
@@ -1579,11 +1722,11 @@ case OP_IntCopy: {            /* out2 */
 **
 ** FK constraint violations are also checked when the prepared statement
 ** exits.  This opcode is used to raise foreign key constraint errors prior
-** to returning results such as a row change count or the result of a 
+** to returning results such as a row change count or the result of a
 ** RETURNING clause.
 */
 case OP_FkCheck: {
-  if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){
+  if( (rc = sqlite3VdbeCheckFkImmediate(p))!=SQLITE_OK ){
     goto abort_due_to_error;
   }
   break;
@@ -1675,10 +1818,14 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
     if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem;
     flags2 = pIn2->flags & ~MEM_Str;
   }
-  nByte = pIn1->n + pIn2->n;
+  nByte = pIn1->n;
+  nByte += pIn2->n;
   if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){
     goto too_big;
   }
+#if SQLITE_MAX_LENGTH>2147483645
+  if( nByte>2147483645 ){ goto too_big; }
+#endif
   if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){
     goto no_mem;
   }
@@ -1727,15 +1874,15 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 ** Synopsis: r[P3]=r[P2]/r[P1]
 **
 ** Divide the value in register P1 by the value in register P2
-** and store the result in register P3 (P3=P2/P1). If the value in 
-** register P1 is zero, then the result is NULL. If either input is 
+** and store the result in register P3 (P3=P2/P1). If the value in
+** register P1 is zero, then the result is NULL. If either input is
 ** NULL, the result is NULL.
 */
 /* Opcode: Remainder P1 P2 P3 * *
 ** Synopsis: r[P3]=r[P2]%r[P1]
 **
-** Compute the remainder after integer register P2 is divided by 
-** register P1 and store the result in register P3. 
+** Compute the remainder after integer register P2 is divided by
+** register P1 and store the result in register P3.
 ** If the value in register P1 is zero the result is NULL.
 ** If either operand is NULL, the result is NULL.
 */
@@ -1932,7 +2079,7 @@ case OP_ShiftRight: {           /* same as TK_RSHIFT, in1, in2, out3 */
 
 /* Opcode: AddImm  P1 P2 * * *
 ** Synopsis: r[P1]=r[P1]+P2
-** 
+**
 ** Add the constant P2 to the value in register P1.
 ** The result is always an integer.
 **
@@ -1942,18 +2089,18 @@ case OP_AddImm: {            /* in1 */
   pIn1 = &aMem[pOp->p1];
   memAboutToChange(p, pIn1);
   sqlite3VdbeMemIntegerify(pIn1);
-  pIn1->u.i += pOp->p2;
+  *(u64*)&pIn1->u.i += (u64)pOp->p2;
   break;
 }
 
 /* Opcode: MustBeInt P1 P2 * * *
-** 
+**
 ** Force the value in register P1 to be an integer.  If the value
 ** in P1 is not an integer and cannot be converted into an integer
 ** without data loss, then jump immediately to P2, or if P2==0
 ** raise an SQLITE_MISMATCH exception.
 */
-case OP_MustBeInt: {            /* jump, in1 */
+case OP_MustBeInt: {            /* jump0, in1 */
   pIn1 = &aMem[pOp->p1];
   if( (pIn1->flags & MEM_Int)==0 ){
     applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
@@ -1994,12 +2141,12 @@ case OP_RealAffinity: {                  /* in1 */
 }
 #endif
 
-#ifndef SQLITE_OMIT_CAST
+#if !defined(SQLITE_OMIT_CAST) || !defined(SQLITE_OMIT_ANALYZE)
 /* Opcode: Cast P1 P2 * * *
 ** Synopsis: affinity(r[P1])
 **
 ** Force the value in register P1 to be the type defined by P2.
-** 
+**
 ** <ul>
 ** <li> P2=='A' &rarr; BLOB
 ** <li> P2=='B' &rarr; TEXT
@@ -2033,17 +2180,17 @@ case OP_Cast: {                  /* in1 */
 ** Synopsis: IF r[P3]==r[P1]
 **
 ** Compare the values in register P1 and P3.  If reg(P3)==reg(P1) then
-** jump to address P2. 
+** jump to address P2.
 **
 ** The SQLITE_AFF_MASK portion of P5 must be an affinity character -
-** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made 
+** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made
 ** to coerce both inputs according to this affinity before the
 ** comparison is made. If the SQLITE_AFF_MASK is 0x00, then numeric
 ** affinity is used. Note that the affinity conversions are stored
 ** back into the input registers P1 and P3.  So this opcode can cause
 ** persistent changes to registers P1 and P3.
 **
-** Once any conversions have taken place, and neither value is NULL, 
+** Once any conversions have taken place, and neither value is NULL,
 ** the values are compared. If both values are blobs then memcmp() is
 ** used to determine the results of the comparison.  If both values
 ** are text, then the appropriate collating function specified in
@@ -2076,18 +2223,18 @@ case OP_Cast: {                  /* in1 */
 ** jump to address P2.
 **
 ** If the SQLITE_JUMPIFNULL bit of P5 is set and either reg(P1) or
-** reg(P3) is NULL then the take the jump.  If the SQLITE_JUMPIFNULL 
+** reg(P3) is NULL then the take the jump.  If the SQLITE_JUMPIFNULL
 ** bit is clear then fall through if either operand is NULL.
 **
 ** The SQLITE_AFF_MASK portion of P5 must be an affinity character -
-** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made 
+** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made
 ** to coerce both inputs according to this affinity before the
 ** comparison is made. If the SQLITE_AFF_MASK is 0x00, then numeric
 ** affinity is used. Note that the affinity conversions are stored
 ** back into the input registers P1 and P3.  So this opcode can cause
 ** persistent changes to registers P1 and P3.
 **
-** Once any conversions have taken place, and neither value is NULL, 
+** Once any conversions have taken place, and neither value is NULL,
 ** the values are compared. If both values are blobs then memcmp() is
 ** used to determine the results of the comparison.  If both values
 ** are text, then the appropriate collating function specified in
@@ -2209,7 +2356,9 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
         }
       }
     }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){
-      if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
+      if( (flags1 & MEM_Str)!=0 ){
+        pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal);
+      }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
         testcase( pIn1->flags & MEM_Int );
         testcase( pIn1->flags & MEM_Real );
         testcase( pIn1->flags & MEM_IntReal );
@@ -2218,7 +2367,9 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
         flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
         if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str;
       }
-      if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
+      if( (flags3 & MEM_Str)!=0 ){
+        pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal);
+      }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
         testcase( pIn3->flags & MEM_Int );
         testcase( pIn3->flags & MEM_Real );
         testcase( pIn3->flags & MEM_IntReal );
@@ -2267,12 +2418,12 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
 ** This opcode must follow an OP_Lt or OP_Gt comparison operator.  There
 ** can be zero or more OP_ReleaseReg opcodes intervening, but no other
 ** opcodes are allowed to occur between this instruction and the previous
-** OP_Lt or OP_Gt. 
+** OP_Lt or OP_Gt.
 **
-** If result of an OP_Eq comparison on the same two operands as the
-** prior OP_Lt or OP_Gt would have been true, then jump to P2.
-** If the result of an OP_Eq comparison on the two previous
-** operands would have been false or NULL, then fall through.
+** If the result of an OP_Eq comparison on the same two operands as
+** the prior OP_Lt or OP_Gt would have been true, then jump to P2.  If
+** the result of an OP_Eq comparison on the two previous operands
+** would have been false or NULL, then fall through.
 */
 case OP_ElseEq: {       /* same as TK_ESCAPE, jump */
 
@@ -2358,6 +2509,7 @@ case OP_Compare: {
   pKeyInfo = pOp->p4.pKeyInfo;
   assert( n>0 );
   assert( pKeyInfo!=0 );
+  assert( pKeyInfo->aSortFlags!=0 );
   p1 = pOp->p1;
   p2 = pOp->p2;
 #ifdef SQLITE_DEBUG
@@ -2383,7 +2535,7 @@ case OP_Compare: {
     iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl);
     VVA_ONLY( iCompareIsInit = 1; )
     if( iCompare ){
-      if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) 
+      if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL)
        && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null))
       ){
         iCompare = -iCompare;
@@ -2468,13 +2620,13 @@ case OP_Or: {             /* same as TK_OR, in1, in2, out3 */
 ** IS NOT FALSE operators.
 **
 ** Interpret the value in register P1 as a boolean value.  Store that
-** boolean (a 0 or 1) in register P2.  Or if the value in register P1 is 
+** boolean (a 0 or 1) in register P2.  Or if the value in register P1 is
 ** NULL, then the P3 is stored in register P2.  Invert the answer if P4
 ** is 1.
 **
 ** The logic is summarized like this:
 **
-** <ul> 
+** <ul>
 ** <li> If P3==0 and P4==0  then  r[P2] := r[P1] IS TRUE
 ** <li> If P3==1 and P4==1  then  r[P2] := r[P1] IS FALSE
 ** <li> If P3==0 and P4==1  then  r[P2] := r[P1] IS NOT TRUE
@@ -2494,7 +2646,7 @@ case OP_IsTrue: {               /* in1, out2 */
 ** Synopsis: r[P2]= !r[P1]
 **
 ** Interpret the value in register P1 as a boolean value.  Store the
-** boolean complement in register P2.  If the value in register P1 is 
+** boolean complement in register P2.  If the value in register P1 is
 ** NULL, then a NULL is stored in P2.
 */
 case OP_Not: {                /* same as TK_NOT, in1, out2 */
@@ -2526,7 +2678,7 @@ case OP_BitNot: {             /* same as TK_BITNOT, in1, out2 */
   break;
 }
 
-/* Opcode: Once P1 P2 * * *
+/* Opcode: Once P1 P2 P3 * *
 **
 ** Fall through to the next instruction the first time this opcode is
 ** encountered on each invocation of the byte-code program.  Jump to P2
@@ -2542,6 +2694,12 @@ case OP_BitNot: {             /* same as TK_BITNOT, in1, out2 */
 ** whether or not the jump should be taken.  The bitmask is necessary
 ** because the self-altering code trick does not work for recursive
 ** triggers.
+**
+** The P3 operand is not used directly by this opcode.  However P3 is
+** used by the code generator as follows:  If this opcode is the start
+** of a subroutine and that subroutine uses a Bloom filter, then P3 will
+** be the register that holds that Bloom filter.  See tag-202407032019
+** in the source code for implementation details.
 */
 case OP_Once: {             /* jump */
   u32 iAddr;                /* Address of this instruction */
@@ -2702,7 +2860,7 @@ case OP_IsType: {        /* jump */
 /* Opcode: ZeroOrNull P1 P2 P3 * *
 ** Synopsis: r[P2] = 0 OR NULL
 **
-** If all both registers P1 and P3 are NOT NULL, then store a zero in
+** If both registers P1 and P3 are NOT NULL, then store a zero in
 ** register P2.  If either registers P1 or P3 are NULL then put
 ** a NULL in register P2.
 */
@@ -2720,7 +2878,7 @@ case OP_ZeroOrNull: {            /* in1, in2, out2, in3 */
 /* Opcode: NotNull P1 P2 * * *
 ** Synopsis: if r[P1]!=NULL goto P2
 **
-** Jump to P2 if the value in register P1 is not NULL.  
+** Jump to P2 if the value in register P1 is not NULL. 
 */
 case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
   pIn1 = &aMem[pOp->p1];
@@ -2796,7 +2954,7 @@ case OP_Offset: {          /* out3 */
 ** Interpret the data that cursor P1 points to as a structure built using
 ** the MakeRecord instruction.  (See the MakeRecord opcode for additional
 ** information about the format of the data.)  Extract the P2-th column
-** from this record.  If there are less than (P2+1) 
+** from this record.  If there are less than (P2+1)
 ** values in the record, extract a NULL.
 **
 ** The value extracted is stored in register P3.
@@ -2942,7 +3100,7 @@ case OP_Column: {            /* ncycle */
   */
   if( pC->nHdrParsed<=p2 ){
     /* If there is more header available for parsing in the record, try
-    ** to extract additional fields up through the p2+1-th field 
+    ** to extract additional fields up through the p2+1-th field
     */
     if( pC->iHdrOffset<aOffset[0] ){
       /* Make sure zData points to enough of the record to cover the header. */
@@ -2954,7 +3112,7 @@ case OP_Column: {            /* ncycle */
       }else{
         zData = pC->aRow;
       }
-  
+ 
       /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */
     op_column_read_header:
       i = pC->nHdrParsed;
@@ -3056,18 +3214,23 @@ case OP_Column: {            /* ncycle */
       pDest->flags = aFlag[t&1];
     }
   }else{
+    u8 p5;
     pDest->enc = encoding;
+    assert( pDest->db==db );
     /* This branch happens only when content is on overflow pages */
-    if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
-          && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
-     || (len = sqlite3VdbeSerialTypeLen(t))==0
+    if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0
+          && (p5==OPFLAG_TYPEOFARG
+              || (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG))
+             )
+        )
+     || sqlite3VdbeSerialTypeLen(t)==0
     ){
       /* Content is irrelevant for
       **    1. the typeof() function,
       **    2. the length(X) function if X is a blob, and
       **    3. if the content length is zero.
       ** So we might as well use bogus content rather than reading
-      ** content from disk. 
+      ** content from disk.
       **
       ** Although sqlite3VdbeSerialGet() may read at most 8 bytes from the
       ** buffer passed to it, debugging function VdbeMemPrettyPrint() may
@@ -3077,11 +3240,13 @@ case OP_Column: {            /* ncycle */
       */
       sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest);
     }else{
-      if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big;
-      rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest);
-      if( rc!=SQLITE_OK ) goto abort_due_to_error;
-      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
-      pDest->flags &= ~MEM_Ephem;
+      rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2],
+                p->cacheCtr, colCacheCtr, pDest);
+      if( rc ){
+        if( rc==SQLITE_NOMEM ) goto no_mem;
+        if( rc==SQLITE_TOOBIG ) goto too_big;
+        goto abort_due_to_error;
+      }
     }
   }
 
@@ -3107,6 +3272,15 @@ case OP_Column: {            /* ncycle */
 ** Take the affinities from the Table object in P4.  If any value
 ** cannot be coerced into the correct type, then raise an error.
 **
+** If P3==0, then omit checking of VIRTUAL columns.
+**
+** If P3==1, then omit checking of all generated column, both VIRTUAL
+** and STORED.
+**
+** If P3>=2, then only check column number P3-2 in the table (which will
+** be a VIRTUAL column) against the value in reg[P1].  In this case,
+** P2 will be 1.
+**
 ** This opcode is similar to OP_Affinity except that this opcode
 ** forces the register type to the Table column type.  This is used
 ** to implement "strict affinity".
@@ -3120,8 +3294,8 @@ case OP_Column: {            /* ncycle */
 **
 ** <ul>
 ** <li> P2 should be the number of non-virtual columns in the
-**      table of P4.
-** <li> Table P4 should be a STRICT table.
+**      table of P4 unless P3>1, in which case P2 will be 1.
+** <li> Table P4 is a STRICT table.
 ** </ul>
 **
 ** If any precondition is false, an assertion fault occurs.
@@ -3130,16 +3304,28 @@ case OP_TypeCheck: {
   Table *pTab;
   Column *aCol;
   int i;
+  int nCol;
 
   assert( pOp->p4type==P4_TABLE );
   pTab = pOp->p4.pTab;
   assert( pTab->tabFlags & TF_Strict );
-  assert( pTab->nNVCol==pOp->p2 );
+  assert( pOp->p3>=0 && pOp->p3<pTab->nCol+2 );
   aCol = pTab->aCol;
   pIn1 = &aMem[pOp->p1];
-  for(i=0; i<pTab->nCol; i++){
-    if( aCol[i].colFlags & COLFLAG_GENERATED ){
-      if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue;
+  if( pOp->p3<2 ){
+    assert( pTab->nNVCol==pOp->p2 );
+    i = 0;
+    nCol = pTab->nCol;
+  }else{
+    i = pOp->p3-2;
+    nCol = i+1;
+    assert( i<pTab->nCol );
+    assert( aCol[i].colFlags & COLFLAG_VIRTUAL );
+    assert( pOp->p2==1 );
+  }
+  for(; i<nCol; i++){
+    if( (aCol[i].colFlags & COLFLAG_GENERATED)!=0 && pOp->p3<2 ){
+      if( (aCol[i].colFlags & COLFLAG_VIRTUAL)!=0 ) continue;
       if( pOp->p3 ){ pIn1++; continue; }
     }
     assert( pIn1 < &aMem[pOp->p1+pOp->p2] );
@@ -3298,13 +3484,13 @@ case OP_MakeRecord: {
   ** like this:
   **
   ** ------------------------------------------------------------------------
-  ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | 
+  ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 |
   ** ------------------------------------------------------------------------
   **
   ** Data(0) is taken from register P1.  Data(1) comes from register P1+1
   ** and so forth.
   **
-  ** Each type field is a varint representing the serial type of the 
+  ** Each type field is a varint representing the serial type of the
   ** corresponding data element (see sqlite3VdbeSerialType()). The
   ** hdr-size field is also a varint which is the offset from the beginning
   ** of the record to data0.
@@ -3461,7 +3647,7 @@ case OP_MakeRecord: {
       len = (u32)pRec->n;
       serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0);
       if( pRec->flags & MEM_Zero ){
-        serial_type += pRec->u.nZero*2;
+        serial_type += (u32)pRec->u.nZero*2;
         if( nData ){
           if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem;
           len += pRec->u.nZero;
@@ -3494,7 +3680,7 @@ case OP_MakeRecord: {
   }
   nByte = nHdr+nData;
 
-  /* Make sure the output register has a buffer large enough to store 
+  /* Make sure the output register has a buffer large enough to store
   ** the new record. The output register (pOp->p3) is not allowed to
   ** be one of the input registers (because the following call to
   ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used).
@@ -3543,7 +3729,6 @@ case OP_MakeRecord: {
         /* NULL value.  No change in zPayload */
       }else{
         u64 v;
-        u32 i;
         if( serial_type==7 ){
           assert( sizeof(v)==sizeof(pRec->u.r) );
           memcpy(&v, &pRec->u.r, sizeof(v));
@@ -3551,12 +3736,22 @@ case OP_MakeRecord: {
         }else{
           v = pRec->u.i;
         }
-        len = i = sqlite3SmallTypeSizes[serial_type];
-        assert( i>0 );
-        while( 1 /*exit-by-break*/ ){
-          zPayload[--i] = (u8)(v&0xFF);
-          if( i==0 ) break;
-          v >>= 8;
+        len = sqlite3SmallTypeSizes[serial_type];
+        assert( len>=1 && len<=8 && len!=5 && len!=7 );
+        switch( len ){
+          default: zPayload[7] = (u8)(v&0xff); v >>= 8;
+                   zPayload[6] = (u8)(v&0xff); v >>= 8;
+                   /* no break */ deliberate_fall_through
+          case 6:  zPayload[5] = (u8)(v&0xff); v >>= 8;
+                   zPayload[4] = (u8)(v&0xff); v >>= 8;
+                   /* no break */ deliberate_fall_through
+          case 4:  zPayload[3] = (u8)(v&0xff); v >>= 8;
+                   /* no break */ deliberate_fall_through
+          case 3:  zPayload[2] = (u8)(v&0xff); v >>= 8;
+                   /* no break */ deliberate_fall_through
+          case 2:  zPayload[1] = (u8)(v&0xff); v >>= 8;
+                   /* no break */ deliberate_fall_through
+          case 1:  zPayload[0] = (u8)(v&0xff);
         }
         zPayload += len;
       }
@@ -3571,6 +3766,7 @@ case OP_MakeRecord: {
       zHdr += sqlite3PutVarint(zHdr, serial_type);
       if( pRec->n ){
         assert( pRec->z!=0 );
+        assert( pRec->z!=(const char*)sqlite3CtypeMap );
         memcpy(zPayload, pRec->z, pRec->n);
         zPayload += pRec->n;
       }
@@ -3589,12 +3785,12 @@ case OP_MakeRecord: {
 /* Opcode: Count P1 P2 P3 * *
 ** Synopsis: r[P2]=count()
 **
-** Store the number of entries (an integer value) in the table or index 
+** Store the number of entries (an integer value) in the table or index
 ** opened by cursor P1 in register P2.
 **
 ** If P3==0, then an exact count is obtained, which involves visiting
 ** every btree page of the table.  But if P3 is non-zero, an estimate
-** is returned based on the current cursor position.  
+** is returned based on the current cursor position. 
 */
 case OP_Count: {         /* out2 */
   i64 nEntry;
@@ -3636,7 +3832,7 @@ case OP_Savepoint: {
   zName = pOp->p4.z;
 
   /* Assert that the p1 parameter is valid. Also that if there is no open
-  ** transaction, then there cannot be any savepoints. 
+  ** transaction, then there cannot be any savepoints.
   */
   assert( db->pSavepoint==0 || db->autoCommit==0 );
   assert( p1==SAVEPOINT_BEGIN||p1==SAVEPOINT_RELEASE||p1==SAVEPOINT_ROLLBACK );
@@ -3646,7 +3842,7 @@ case OP_Savepoint: {
 
   if( p1==SAVEPOINT_BEGIN ){
     if( db->nVdbeWrite>0 ){
-      /* A new savepoint cannot be created if there are active write 
+      /* A new savepoint cannot be created if there are active write
       ** statements (i.e. open read/write incremental blob handles).
       */
       sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress");
@@ -3670,7 +3866,7 @@ case OP_Savepoint: {
       if( pNew ){
         pNew->zName = (char *)&pNew[1];
         memcpy(pNew->zName, zName, nName+1);
-    
+   
         /* If there is no open transaction, then mark this as a special
         ** "transaction savepoint". */
         if( db->autoCommit ){
@@ -3694,7 +3890,7 @@ case OP_Savepoint: {
     /* Find the named savepoint. If there is no such savepoint, then an
     ** an error is returned to the user.  */
     for(
-      pSavepoint = db->pSavepoint; 
+      pSavepoint = db->pSavepoint;
       pSavepoint && sqlite3StrICmp(pSavepoint->zName, zName);
       pSavepoint = pSavepoint->pNext
     ){
@@ -3704,7 +3900,7 @@ case OP_Savepoint: {
       sqlite3VdbeError(p, "no such savepoint: %s", zName);
       rc = SQLITE_ERROR;
     }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){
-      /* It is not possible to release (commit) a savepoint if there are 
+      /* It is not possible to release (commit) a savepoint if there are
       ** active write statements.
       */
       sqlite3VdbeError(p, "cannot release savepoint - "
@@ -3713,12 +3909,12 @@ case OP_Savepoint: {
     }else{
 
       /* Determine whether or not this is a transaction savepoint. If so,
-      ** and this is a RELEASE command, then the current transaction 
-      ** is committed. 
+      ** and this is a RELEASE command, then the current transaction
+      ** is committed.
       */
       int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint;
       if( isTransaction && p1==SAVEPOINT_RELEASE ){
-        if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
+        if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){
           goto vdbe_return;
         }
         db->autoCommit = 1;
@@ -3762,8 +3958,8 @@ case OP_Savepoint: {
         }
       }
       if( rc ) goto abort_due_to_error;
-  
-      /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all 
+ 
+      /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all
       ** savepoints nested inside of the savepoint being operated on. */
       while( db->pSavepoint!=pSavepoint ){
         pTmp = db->pSavepoint;
@@ -3772,8 +3968,8 @@ case OP_Savepoint: {
         db->nSavepoint--;
       }
 
-      /* If it is a RELEASE, then destroy the savepoint being operated on 
-      ** too. If it is a ROLLBACK TO, then set the number of deferred 
+      /* If it is a RELEASE, then destroy the savepoint being operated on
+      ** too. If it is a ROLLBACK TO, then set the number of deferred
       ** constraint violations present in the database to the value stored
       ** when the savepoint was created.  */
       if( p1==SAVEPOINT_RELEASE ){
@@ -3830,13 +4026,13 @@ case OP_AutoCommit: {
       db->autoCommit = 1;
     }else if( desiredAutoCommit && db->nVdbeWrite>0 ){
       /* If this instruction implements a COMMIT and other VMs are writing
-      ** return an error indicating that the other VMs must complete first. 
+      ** return an error indicating that the other VMs must complete first.
       */
       sqlite3VdbeError(p, "cannot commit transaction - "
                           "SQL statements in progress");
       rc = SQLITE_BUSY;
       goto abort_due_to_error;
-    }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
+    }else if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){
       goto vdbe_return;
     }else{
       db->autoCommit = (u8)desiredAutoCommit;
@@ -3859,7 +4055,7 @@ case OP_AutoCommit: {
         (!desiredAutoCommit)?"cannot start a transaction within a transaction":(
         (iRollback)?"cannot rollback - no transaction is active":
                    "cannot commit - no transaction is active"));
-         
+        
     rc = SQLITE_ERROR;
     goto abort_due_to_error;
   }
@@ -3870,7 +4066,7 @@ case OP_AutoCommit: {
 **
 ** Begin a transaction on database P1 if a transaction is not already
 ** active.
-** If P2 is non-zero, then a write-transaction is started, or if a 
+** If P2 is non-zero, then a write-transaction is started, or if a
 ** read-transaction is already active, it is upgraded to a write-transaction.
 ** If P2 is zero, then a read-transaction is started.  If P2 is 2 or more
 ** then an exclusive transaction is started.
@@ -3941,12 +4137,12 @@ case OP_Transaction: {
 
     if( p->usesStmtJournal
      && pOp->p2
-     && (db->autoCommit==0 || db->nVdbeRead>1) 
+     && (db->autoCommit==0 || db->nVdbeRead>1)
     ){
       assert( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE );
       if( p->iStatement==0 ){
         assert( db->nStatement>=0 && db->nSavepoint>=0 );
-        db->nStatement++; 
+        db->nStatement++;
         p->iStatement = db->nSavepoint + db->nStatement;
       }
 
@@ -3974,7 +4170,7 @@ case OP_Transaction: {
     */
     sqlite3DbFree(db, p->zErrMsg);
     p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed");
-    /* If the schema-cookie from the database file matches the cookie 
+    /* If the schema-cookie from the database file matches the cookie
     ** stored with the in-memory representation of the schema, do
     ** not reload the schema from the database file.
     **
@@ -3984,7 +4180,7 @@ case OP_Transaction: {
     ** prepared queries. If such a query is out-of-date, we do not want to
     ** discard the database schema, as the user code implementing the
     ** v-table would have to be ready for the sqlite3_vtab structure itself
-    ** to be invalidated whenever sqlite3_step() is called from within 
+    ** to be invalidated whenever sqlite3_step() is called from within
     ** a v-table method.
     */
     if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){
@@ -4037,8 +4233,8 @@ case OP_ReadCookie: {               /* out2 */
 **
 ** Write the integer value P3 into cookie number P2 of database P1.
 ** P2==1 is the schema version.  P2==2 is the database format.
-** P2==3 is the recommended pager cache 
-** size, and so forth.  P1==0 is the main database file and P1==1 is the 
+** P2==3 is the recommended pager cache
+** size, and so forth.  P1==0 is the main database file and P1==1 is the
 ** database file used to store temporary tables.
 **
 ** A transaction must be started before executing this opcode.
@@ -4084,8 +4280,8 @@ case OP_SetCookie: {
 ** Synopsis: root=P2 iDb=P3
 **
 ** Open a read-only cursor for the database table whose root page is
-** P2 in a database file.  The database file is determined by P3. 
-** P3==0 means the main database, P3==1 means the database used for 
+** P2 in a database file.  The database file is determined by P3.
+** P3==0 means the main database, P3==1 means the database used for
 ** temporary tables, and P3>1 means used the corresponding attached
 ** database.  Give the new cursor an identifier of P1.  The P1
 ** values need not be contiguous but all P1 values should be small integers.
@@ -4099,10 +4295,10 @@ case OP_SetCookie: {
 ** </ul>
 **
 ** The P4 value may be either an integer (P4_INT32) or a pointer to
-** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo 
+** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo
 ** object, then table being opened must be an [index b-tree] where the
-** KeyInfo object defines the content and collating 
-** sequence of that index b-tree. Otherwise, if P4 is an integer 
+** KeyInfo object defines the content and collating
+** sequence of that index b-tree. Otherwise, if P4 is an integer
 ** value, then the table being opened must be a [table b-tree] with a
 ** number of columns no less than the value of P4.
 **
@@ -4138,10 +4334,10 @@ case OP_SetCookie: {
 ** OPFLAG_P2ISREG bit is set in P5 - see below).
 **
 ** The P4 value may be either an integer (P4_INT32) or a pointer to
-** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo 
+** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo
 ** object, then table being opened must be an [index b-tree] where the
-** KeyInfo object defines the content and collating 
-** sequence of that index b-tree. Otherwise, if P4 is an integer 
+** KeyInfo object defines the content and collating
+** sequence of that index b-tree. Otherwise, if P4 is an integer
 ** value, then the table being opened must be a [table b-tree] with a
 ** number of columns no less than the value of P4.
 **
@@ -4214,23 +4410,23 @@ case OP_OpenWrite:
     if( pDb->pSchema->file_format < p->minWriteFileFormat ){
       p->minWriteFileFormat = pDb->pSchema->file_format;
     }
+    if( pOp->p5 & OPFLAG_P2ISREG ){
+      assert( p2>0 );
+      assert( p2<=(u32)(p->nMem+1 - p->nCursor) );
+      pIn2 = &aMem[p2];
+      assert( memIsValid(pIn2) );
+      assert( (pIn2->flags & MEM_Int)!=0 );
+      sqlite3VdbeMemIntegerify(pIn2);
+      p2 = (int)pIn2->u.i;
+      /* The p2 value always comes from a prior OP_CreateBtree opcode and
+      ** that opcode will always set the p2 value to 2 or more or else fail.
+      ** If there were a failure, the prepared statement would have halted
+      ** before reaching this instruction. */
+      assert( p2>=2 );
+    }
   }else{
     wrFlag = 0;
-  }
-  if( pOp->p5 & OPFLAG_P2ISREG ){
-    assert( p2>0 );
-    assert( p2<=(u32)(p->nMem+1 - p->nCursor) );
-    assert( pOp->opcode==OP_OpenWrite );
-    pIn2 = &aMem[p2];
-    assert( memIsValid(pIn2) );
-    assert( (pIn2->flags & MEM_Int)!=0 );
-    sqlite3VdbeMemIntegerify(pIn2);
-    p2 = (int)pIn2->u.i;
-    /* The p2 value always comes from a prior OP_CreateBtree opcode and
-    ** that opcode will always set the p2 value to 2 or more or else fail.
-    ** If there were a failure, the prepared statement would have halted
-    ** before reaching this instruction. */
-    assert( p2>=2 );
+    assert( (pOp->p5 & OPFLAG_P2ISREG)==0 );
   }
   if( pOp->p4type==P4_KEYINFO ){
     pKeyInfo = pOp->p4.pKeyInfo;
@@ -4257,7 +4453,7 @@ case OP_OpenWrite:
   /* Set the VdbeCursor.isTable variable. Previous versions of
   ** SQLite used to check if the root-page flags were sane at this point
   ** and report database corruption if they were not, but this check has
-  ** since moved into the btree layer.  */  
+  ** since moved into the btree layer.  */ 
   pCur->isTable = pOp->p4type!=P4_KEYINFO;
 
 open_cursor_set_hints:
@@ -4298,7 +4494,7 @@ case OP_OpenDup: {           /* ncycle */
   pCx->ub.pBtx = pOrig->ub.pBtx;
   pCx->noReuse = 1;
   pOrig->noReuse = 1;
-  rc = sqlite3BtreeCursor(pCx->ub.pBtx, pCx->pgnoRoot, BTREE_WRCSR, 
+  rc = sqlite3BtreeCursor(pCx->ub.pBtx, pCx->pgnoRoot, BTREE_WRCSR,
                           pCx->pKeyInfo, pCx->uc.pCursor);
   /* The sqlite3BtreeCursor() routine can only fail for the first cursor
   ** opened for a database.  Since there is already an open cursor when this
@@ -4312,7 +4508,7 @@ case OP_OpenDup: {           /* ncycle */
 ** Synopsis: nColumn=P2
 **
 ** Open a new cursor P1 to a transient table.
-** The cursor is always opened read/write even if 
+** The cursor is always opened read/write even if
 ** the main database is read-only.  The ephemeral
 ** table is deleted automatically when the cursor is closed.
 **
@@ -4346,7 +4542,7 @@ case OP_OpenEphemeral: {     /* ncycle */
   VdbeCursor *pCx;
   KeyInfo *pKeyInfo;
 
-  static const int vfsFlags = 
+  static const int vfsFlags =
       SQLITE_OPEN_READWRITE |
       SQLITE_OPEN_CREATE |
       SQLITE_OPEN_EXCLUSIVE |
@@ -4365,7 +4561,7 @@ case OP_OpenEphemeral: {     /* ncycle */
   }
   pCx = p->apCsr[pOp->p1];
   if( pCx && !pCx->noReuse &&  ALWAYS(pOp->p2<=pCx->nField) ){
-    /* If the ephermeral table is already open and has no duplicates from
+    /* If the ephemeral table is already open and has no duplicates from
     ** OP_OpenDup, then erase all existing content so that the table is
     ** empty again, rather than creating a new table. */
     assert( pCx->isEphemeral );
@@ -4376,7 +4572,7 @@ case OP_OpenEphemeral: {     /* ncycle */
     pCx = allocateCursor(p, pOp->p1, pOp->p2, CURTYPE_BTREE);
     if( pCx==0 ) goto no_mem;
     pCx->isEphemeral = 1;
-    rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->ub.pBtx, 
+    rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->ub.pBtx,
                           BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5,
                           vfsFlags);
     if( rc==SQLITE_OK ){
@@ -4390,7 +4586,7 @@ case OP_OpenEphemeral: {     /* ncycle */
         if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){
           assert( pOp->p4type==P4_KEYINFO );
           rc = sqlite3BtreeCreateTable(pCx->ub.pBtx, &pCx->pgnoRoot,
-              BTREE_BLOBKEY | pOp->p5); 
+              BTREE_BLOBKEY | pOp->p5);
           if( rc==SQLITE_OK ){
             assert( pCx->pgnoRoot==SCHEMA_ROOT+1 );
             assert( pKeyInfo->db==db );
@@ -4407,8 +4603,13 @@ case OP_OpenEphemeral: {     /* ncycle */
         }
       }
       pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED);
+      assert( p->apCsr[pOp->p1]==pCx );
       if( rc ){
+        assert( !sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) );
         sqlite3BtreeClose(pCx->ub.pBtx);
+        p->apCsr[pOp->p1] = 0;  /* Not required; helps with static analysis */
+      }else{
+        assert( sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) );
       }
     }
   }
@@ -4465,7 +4666,7 @@ case OP_SequenceTest: {
 **
 ** Open a new cursor that points to a fake table that contains a single
 ** row of data.  The content of that one row is the content of memory
-** register P2.  In other words, cursor P1 becomes an alias for the 
+** register P2.  In other words, cursor P1 becomes an alias for the
 ** MEM_Blob content contained in register P2.
 **
 ** A pseudo-table created by this opcode is used to hold a single
@@ -4474,7 +4675,8 @@ case OP_SequenceTest: {
 ** is the only cursor opcode that works with a pseudo-table.
 **
 ** P3 is the number of fields in the records that will be stored by
-** the pseudo-table.
+** the pseudo-table.  If P2 is 0 or negative then the pseudo-cursor
+** will return NULL for every column.
 */
 case OP_OpenPseudo: {
   VdbeCursor *pCx;
@@ -4530,13 +4732,13 @@ case OP_ColumnsUsed: {
 /* Opcode: SeekGE P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), 
-** use the value in register P3 as the key.  If cursor P1 refers 
-** to an SQL index, then P3 is the first in an array of P4 registers 
-** that are used as an unpacked index key. 
+** If cursor P1 refers to an SQL table (B-Tree that uses integer keys),
+** use the value in register P3 as the key.  If cursor P1 refers
+** to an SQL index, then P3 is the first in an array of P4 registers
+** that are used as an unpacked index key.
 **
-** Reposition cursor P1 so that  it points to the smallest entry that 
-** is greater than or equal to the key value. If there are no records 
+** Reposition cursor P1 so that  it points to the smallest entry that
+** is greater than or equal to the key value. If there are no records
 ** greater than or equal to the key and P2 is not zero, then jump to P2.
 **
 ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this
@@ -4544,7 +4746,7 @@ case OP_ColumnsUsed: {
 ** else it will cause a jump to P2.  When the cursor is OPFLAG_SEEKEQ,
 ** this opcode must be followed by an IdxLE opcode with the same arguments.
 ** The IdxGT opcode will be skipped if this opcode succeeds, but the
-** IdxGT opcode will be used on subsequent loop iterations.  The 
+** IdxGT opcode will be used on subsequent loop iterations.  The
 ** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this
 ** is an equality search.
 **
@@ -4557,13 +4759,13 @@ case OP_ColumnsUsed: {
 /* Opcode: SeekGT P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), 
-** use the value in register P3 as a key. If cursor P1 refers 
-** to an SQL index, then P3 is the first in an array of P4 registers 
-** that are used as an unpacked index key. 
+** If cursor P1 refers to an SQL table (B-Tree that uses integer keys),
+** use the value in register P3 as a key. If cursor P1 refers
+** to an SQL index, then P3 is the first in an array of P4 registers
+** that are used as an unpacked index key.
 **
-** Reposition cursor P1 so that it points to the smallest entry that 
-** is greater than the key value. If there are no records greater than 
+** Reposition cursor P1 so that it points to the smallest entry that
+** is greater than the key value. If there are no records greater than
 ** the key and P2 is not zero, then jump to P2.
 **
 ** This opcode leaves the cursor configured to move in forward order,
@@ -4572,16 +4774,16 @@ case OP_ColumnsUsed: {
 **
 ** See also: Found, NotFound, SeekLt, SeekGe, SeekLe
 */
-/* Opcode: SeekLT P1 P2 P3 P4 * 
+/* Opcode: SeekLT P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), 
-** use the value in register P3 as a key. If cursor P1 refers 
-** to an SQL index, then P3 is the first in an array of P4 registers 
-** that are used as an unpacked index key. 
+** If cursor P1 refers to an SQL table (B-Tree that uses integer keys),
+** use the value in register P3 as a key. If cursor P1 refers
+** to an SQL index, then P3 is the first in an array of P4 registers
+** that are used as an unpacked index key.
 **
-** Reposition cursor P1 so that  it points to the largest entry that 
-** is less than the key value. If there are no records less than 
+** Reposition cursor P1 so that  it points to the largest entry that
+** is less than the key value. If there are no records less than
 ** the key and P2 is not zero, then jump to P2.
 **
 ** This opcode leaves the cursor configured to move in reverse order,
@@ -4593,13 +4795,13 @@ case OP_ColumnsUsed: {
 /* Opcode: SeekLE P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), 
-** use the value in register P3 as a key. If cursor P1 refers 
-** to an SQL index, then P3 is the first in an array of P4 registers 
-** that are used as an unpacked index key. 
+** If cursor P1 refers to an SQL table (B-Tree that uses integer keys),
+** use the value in register P3 as a key. If cursor P1 refers
+** to an SQL index, then P3 is the first in an array of P4 registers
+** that are used as an unpacked index key.
 **
-** Reposition cursor P1 so that it points to the largest entry that 
-** is less than or equal to the key value. If there are no records 
+** Reposition cursor P1 so that it points to the largest entry that
+** is less than or equal to the key value. If there are no records
 ** less than or equal to the key and P2 is not zero, then jump to P2.
 **
 ** This opcode leaves the cursor configured to move in reverse order,
@@ -4611,16 +4813,16 @@ case OP_ColumnsUsed: {
 ** else it will cause a jump to P2.  When the cursor is OPFLAG_SEEKEQ,
 ** this opcode must be followed by an IdxLE opcode with the same arguments.
 ** The IdxGE opcode will be skipped if this opcode succeeds, but the
-** IdxGE opcode will be used on subsequent loop iterations.  The 
+** IdxGE opcode will be used on subsequent loop iterations.  The
 ** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this
 ** is an equality search.
 **
 ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt
 */
-case OP_SeekLT:         /* jump, in3, group, ncycle */
-case OP_SeekLE:         /* jump, in3, group, ncycle */
-case OP_SeekGE:         /* jump, in3, group, ncycle */
-case OP_SeekGT: {       /* jump, in3, group, ncycle */
+case OP_SeekLT:         /* jump0, in3, group, ncycle */
+case OP_SeekLE:         /* jump0, in3, group, ncycle */
+case OP_SeekGE:         /* jump0, in3, group, ncycle */
+case OP_SeekGT: {       /* jump0, in3, group, ncycle */
   int res;           /* Comparison result */
   int oc;            /* Opcode */
   VdbeCursor *pC;    /* The cursor to seek */
@@ -4856,7 +5058,7 @@ case OP_SeekGT: {       /* jump, in3, group, ncycle */
 ** row.  If This.P5 is false (0) then a jump is made to SeekGE.P2.  If
 ** This.P5 is true (non-zero) then a jump is made to This.P2.  The P5==0
 ** case occurs when there are no inequality constraints to the right of
-** the IN constraing.  The jump to SeekGE.P2 ends the loop.  The P5!=0 case
+** the IN constraint.  The jump to SeekGE.P2 ends the loop.  The P5!=0 case
 ** occurs when there are inequality constraints to the right of the IN
 ** operator.  In that case, the This.P2 will point either directly to or
 ** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for
@@ -4864,7 +5066,7 @@ case OP_SeekGT: {       /* jump, in3, group, ncycle */
 **
 ** Possible outcomes from this opcode:<ol>
 **
-** <li> If the cursor is initally not pointed to any valid row, then
+** <li> If the cursor is initially not pointed to any valid row, then
 **      fall through into the subsequent OP_SeekGE opcode.
 **
 ** <li> If the cursor is left pointing to a row that is before the target
@@ -4881,7 +5083,7 @@ case OP_SeekGT: {       /* jump, in3, group, ncycle */
 **      (indicating that the target row definitely does not exist in the
 **      btree) then jump to SeekGE.P2, ending the loop.
 **
-** <li> If the cursor ends up on a valid row that is past the target row 
+** <li> If the cursor ends up on a valid row that is past the target row
 **      (indicating that the target row does not exist in the btree) then
 **      jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0.
 ** </ol>
@@ -4904,7 +5106,7 @@ case OP_SeekScan: {          /* ncycle */
     assert( pOp[1].p1==aOp[pOp->p2-1].p1 );
     assert( pOp[1].p2==aOp[pOp->p2-1].p2 );
     assert( pOp[1].p3==aOp[pOp->p2-1].p3 );
-    assert( aOp[pOp->p2-1].opcode==OP_IdxGT 
+    assert( aOp[pOp->p2-1].opcode==OP_IdxGT
          || aOp[pOp->p2-1].opcode==OP_IdxGE );
     testcase( aOp[pOp->p2-1].opcode==OP_IdxGE );
   }else{
@@ -4923,7 +5125,7 @@ case OP_SeekScan: {          /* ncycle */
 #ifdef SQLITE_DEBUG
      if( db->flags&SQLITE_VdbeTrace ){
        printf("... cursor not valid - fall through\n");
-     }        
+     }       
 #endif
     break;
   }
@@ -4952,7 +5154,7 @@ case OP_SeekScan: {          /* ncycle */
 #ifdef SQLITE_DEBUG
       if( db->flags&SQLITE_VdbeTrace ){
         printf("... %d steps and then skip\n", pOp->p1 - nStep);
-      }        
+      }       
 #endif
       VdbeBranchTaken(1,3);
       pOp++;
@@ -4963,7 +5165,7 @@ case OP_SeekScan: {          /* ncycle */
 #ifdef SQLITE_DEBUG
       if( db->flags&SQLITE_VdbeTrace ){
         printf("... %d steps and then success\n", pOp->p1 - nStep);
-      }        
+      }       
 #endif
       VdbeBranchTaken(2,3);
       goto jump_to_p2;
@@ -4973,7 +5175,7 @@ case OP_SeekScan: {          /* ncycle */
 #ifdef SQLITE_DEBUG
       if( db->flags&SQLITE_VdbeTrace ){
         printf("... fall through after %d steps\n", pOp->p1);
-      }        
+      }       
 #endif
       VdbeBranchTaken(0,3);
       break;
@@ -4990,7 +5192,7 @@ case OP_SeekScan: {          /* ncycle */
       }
     }
   }
-  
+ 
   break;
 }
 
@@ -5019,14 +5221,14 @@ case OP_SeekHit: {           /* ncycle */
 #ifdef SQLITE_DEBUG
     if( db->flags&SQLITE_VdbeTrace ){
       printf("seekHit changes from %d to %d\n", pC->seekHit, pOp->p2);
-    }        
+    }       
 #endif
     pC->seekHit = pOp->p2;
   }else if( pC->seekHit>pOp->p3 ){
 #ifdef SQLITE_DEBUG
     if( db->flags&SQLITE_VdbeTrace ){
       printf("seekHit changes from %d to %d\n", pC->seekHit, pOp->p3);
-    }        
+    }       
 #endif
     pC->seekHit = pOp->p3;
   }
@@ -5074,9 +5276,9 @@ case OP_IfNotOpen: {        /* jump */
 ** If P4==0 then register P3 holds a blob constructed by MakeRecord.  If
 ** P4>0 then register P3 is the first of P4 registers that form an unpacked
 ** record.
-** 
+**
 ** Cursor P1 is on an index btree.  If the record identified by P3 and P4
-** is not the prefix of any entry in P1 then a jump is made to P2.  If P1 
+** is not the prefix of any entry in P1 then a jump is made to P2.  If P1
 ** does contain an entry whose prefix matches the P3/P4 record then control
 ** falls through to the next instruction and P1 is left pointing at the
 ** matching entry.
@@ -5096,13 +5298,13 @@ case OP_IfNotOpen: {        /* jump */
 ** operands to OP_NotFound and OP_IdxGT.
 **
 ** This opcode is an optimization attempt only.  If this opcode always
-** falls through, the correct answer is still obtained, but extra works
+** falls through, the correct answer is still obtained, but extra work
 ** is performed.
 **
 ** A value of N in the seekHit flag of cursor P1 means that there exists
 ** a key P3:N that will match some record in the index.  We want to know
 ** if it is possible for a record P3:P4 to match some record in the
-** index.  If it is not possible, we can skips some work.  So if seekHit
+** index.  If it is not possible, we can skip some work.  So if seekHit
 ** is less than P4, attempt to find out if a match is possible by running
 ** OP_NotFound.
 **
@@ -5124,7 +5326,7 @@ case OP_IfNotOpen: {        /* jump */
 ** If P4==0 then register P3 holds a blob constructed by MakeRecord.  If
 ** P4>0 then register P3 is the first of P4 registers that form an unpacked
 ** record.
-** 
+**
 ** Cursor P1 is on an index btree.  If the record identified by P3 and P4
 ** contains any NULL value, jump immediately to P2.  If all terms of the
 ** record are not-NULL then a check is done to determine if any row in the
@@ -5149,7 +5351,7 @@ case OP_IfNoHope: {     /* jump, in3, ncycle */
 #ifdef SQLITE_DEBUG
   if( db->flags&SQLITE_VdbeTrace ){
     printf("seekHit is %d\n", pC->seekHit);
-  }        
+  }       
 #endif
   if( pC->seekHit>=pOp->p4.i ) break;
   /* Fall through into OP_NotFound */
@@ -5185,6 +5387,7 @@ case OP_Found: {        /* jump, in3, ncycle */
     r.pKeyInfo = pC->pKeyInfo;
     r.default_rc = 0;
 #ifdef SQLITE_DEBUG
+    (void)sqlite3FaultSim(50);  /* For use by --counter in TH3 */
     for(ii=0; ii<r.nField; ii++){
       assert( memIsValid(&r.aMem[ii]) );
       assert( (r.aMem[ii].flags & MEM_Zero)==0 || r.aMem[ii].n==0 );
@@ -5201,7 +5404,7 @@ case OP_Found: {        /* jump, in3, ncycle */
     if( rc ) goto no_mem;
     pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo);
     if( pIdxKey==0 ) goto no_mem;
-    sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey);
+    sqlite3VdbeRecordUnpack(r.aMem->n, r.aMem->z, pIdxKey);
     pIdxKey->default_rc = 0;
     rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult);
     sqlite3DbFreeNN(db, pIdxKey);
@@ -5245,9 +5448,9 @@ case OP_Found: {        /* jump, in3, ncycle */
 **
 ** P1 is the index of a cursor open on an SQL table btree (with integer
 ** keys).  If register P3 does not contain an integer or if P1 does not
-** contain a record with rowid P3 then jump immediately to P2.  
+** contain a record with rowid P3 then jump immediately to P2. 
 ** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain
-** a record with rowid P3 then 
+** a record with rowid P3 then
 ** leave the cursor pointing at that record and fall through to the next
 ** instruction.
 **
@@ -5270,7 +5473,7 @@ case OP_Found: {        /* jump, in3, ncycle */
 ** P1 is the index of a cursor open on an SQL table btree (with integer
 ** keys).  P3 is an integer rowid.  If P1 does not contain a record with
 ** rowid P3 then jump immediately to P2.  Or, if P2 is 0, raise an
-** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then 
+** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then
 ** leave the cursor pointing at that record and fall through to the next
 ** instruction.
 **
@@ -5287,7 +5490,7 @@ case OP_Found: {        /* jump, in3, ncycle */
 **
 ** See also: Found, NotFound, NoConflict, SeekRowid
 */
-case OP_SeekRowid: {        /* jump, in3, ncycle */
+case OP_SeekRowid: {        /* jump0, in3, ncycle */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
@@ -5354,7 +5557,7 @@ case OP_NotExists:          /* jump, in3, ncycle */
 ** Find the next available sequence number for cursor P1.
 ** Write the sequence number into register P2.
 ** The sequence number on the cursor is incremented after this
-** instruction.  
+** instruction. 
 */
 case OP_Sequence: {           /* out2 */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
@@ -5374,9 +5577,9 @@ case OP_Sequence: {           /* out2 */
 ** table that cursor P1 points to.  The new record number is written
 ** written to register P2.
 **
-** If P3>0 then P3 is a register in the root frame of this VDBE that holds 
+** If P3>0 then P3 is a register in the root frame of this VDBE that holds
 ** the largest previously generated record number. No new record numbers are
-** allowed to be less than this value. When this value reaches its maximum, 
+** allowed to be less than this value. When this value reaches its maximum,
 ** an SQLITE_FULL error is generated. The P3 register is updated with the '
 ** generated record number. This P3 mechanism is used to help implement the
 ** AUTOINCREMENT feature.
@@ -5527,8 +5730,8 @@ case OP_NewRowid: {           /* out2 */
 ** is part of an INSERT operation.  The difference is only important to
 ** the update hook.
 **
-** Parameter P4 may point to a Table structure, or may be NULL. If it is 
-** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked 
+** Parameter P4 may point to a Table structure, or may be NULL. If it is
+** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked
 ** following a successful insert.
 **
 ** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically
@@ -5609,11 +5812,12 @@ case OP_Insert: {
   x.pKey = 0;
   assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT );
   rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
-      (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), 
+      (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)),
       seekResult
   );
   pC->deferredMoveto = 0;
   pC->cacheStatus = CACHE_STALE;
+  colCacheCtr++;
 
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
@@ -5662,27 +5866,32 @@ case OP_RowCell: {
 ** the cursor will be left pointing at  either the next or the previous
 ** record in the table. If it is left pointing at the next record, then
 ** the next Next instruction will be a no-op. As a result, in this case
-** it is ok to delete a record from within a Next loop. If 
+** it is ok to delete a record from within a Next loop. If
 ** OPFLAG_SAVEPOSITION bit of P5 is clear, then the cursor will be
 ** left in an undefined state.
 **
 ** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this
-** delete one of several associated with deleting a table row and all its
-** associated index entries.  Exactly one of those deletes is the "primary"
-** delete.  The others are all on OPFLAG_FORDELETE cursors or else are
-** marked with the AUXDELETE flag.
+** delete is one of several associated with deleting a table row and
+** all its associated index entries.  Exactly one of those deletes is
+** the "primary" delete.  The others are all on OPFLAG_FORDELETE
+** cursors or else are marked with the AUXDELETE flag.
+**
+** If the OPFLAG_NCHANGE (0x01) flag of P2 (NB: P2 not P5) is set, then
+** the row change count is incremented (otherwise not).
 **
-** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row
-** change count is incremented (otherwise not).
+** If the OPFLAG_ISNOOP (0x40) flag of P2 (not P5!) is set, then the
+** pre-update-hook for deletes is run, but the btree is otherwise unchanged.
+** This happens when the OP_Delete is to be shortly followed by an OP_Insert
+** with the same key, causing the btree entry to be overwritten.
 **
 ** P1 must not be pseudo-table.  It has to be a real table with
 ** multiple rows.
 **
-** If P4 is not NULL then it points to a Table object. In this case either 
+** If P4 is not NULL then it points to a Table object. In this case either
 ** the update or pre-update hook, or both, may be invoked. The P1 cursor must
-** have been positioned using OP_NotFound prior to invoking this opcode in 
-** this case. Specifically, if one is configured, the pre-update hook is 
-** invoked if P4 is not NULL. The update-hook is invoked if one is configured, 
+** have been positioned using OP_NotFound prior to invoking this opcode in
+** this case. Specifically, if one is configured, the pre-update hook is
+** invoked if P4 is not NULL. The update-hook is invoked if one is configured,
 ** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2.
 **
 ** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address
@@ -5721,7 +5930,7 @@ case OP_Delete: {
   /* If the update-hook or pre-update-hook will be invoked, set zDb to
   ** the name of the db to pass as to it. Also set local pTab to a copy
   ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was
-  ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set 
+  ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set
   ** VdbeCursor.movetoTarget to the current rowid.  */
   if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
     assert( pC->iDb>=0 );
@@ -5740,20 +5949,20 @@ case OP_Delete: {
   /* Invoke the pre-update-hook if required. */
   assert( db->xPreUpdateCallback==0 || pTab==pOp->p4.pTab );
   if( db->xPreUpdateCallback && pTab ){
-    assert( !(opflags & OPFLAG_ISUPDATE) 
-         || HasRowid(pTab)==0 
-         || (aMem[pOp->p3].flags & MEM_Int) 
+    assert( !(opflags & OPFLAG_ISUPDATE)
+         || HasRowid(pTab)==0
+         || (aMem[pOp->p3].flags & MEM_Int)
     );
     sqlite3VdbePreUpdateHook(p, pC,
-        (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, 
+        (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE,
         zDb, pTab, pC->movetoTarget,
         pOp->p3, -1
     );
   }
   if( opflags & OPFLAG_ISNOOP ) break;
 #endif
- 
-  /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ 
+
+  /* Only flags that can be set are SAVEPOISTION and AUXDELETE */
   assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 );
   assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION );
   assert( OPFLAG_AUXDELETE==BTREE_AUXDELETE );
@@ -5774,6 +5983,7 @@ case OP_Delete: {
 
   rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5);
   pC->cacheStatus = CACHE_STALE;
+  colCacheCtr++;
   pC->seekResult = 0;
   if( rc ) goto abort_due_to_error;
 
@@ -5806,7 +6016,7 @@ case OP_ResetCount: {
 ** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2
 **
 ** P1 is a sorter cursor. This instruction compares a prefix of the
-** record blob in register P3 against a prefix of the entry that 
+** record blob in register P3 against a prefix of the entry that
 ** the sorter cursor currently points to.  Only the first P4 fields
 ** of r[P3] and the sorter record are compared.
 **
@@ -5841,13 +6051,13 @@ case OP_SorterCompare: {
 ** Write into register P2 the current sorter data for sorter cursor P1.
 ** Then clear the column header cache on cursor P3.
 **
-** This opcode is normally use to move a record out of the sorter and into
+** This opcode is normally used to move a record out of the sorter and into
 ** a register that is the source for a pseudo-table cursor created using
 ** OpenPseudo.  That pseudo-table cursor is the one that is identified by
 ** parameter P3.  Clearing the P3 column cache as part of this opcode saves
 ** us from having to issue a separate NullRow instruction to clear that cache.
 */
-case OP_SorterData: {
+case OP_SorterData: {       /* ncycle */
   VdbeCursor *pC;
 
   pOut = &aMem[pOp->p2];
@@ -5864,10 +6074,10 @@ case OP_SorterData: {
 /* Opcode: RowData P1 P2 P3 * *
 ** Synopsis: r[P2]=data
 **
-** Write into register P2 the complete row content for the row at 
+** Write into register P2 the complete row content for the row at
 ** which cursor P1 is currently pointing.
-** There is no interpretation of the data.  
-** It is just copied onto the P2 register exactly as 
+** There is no interpretation of the data. 
+** It is just copied onto the P2 register exactly as
 ** it is found in the database file.
 **
 ** If cursor P1 is an index, then the content is the key of the row.
@@ -5908,7 +6118,7 @@ case OP_RowData: {
   /* The OP_RowData opcodes always follow OP_NotExists or
   ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions
   ** that might invalidate the cursor.
-  ** If this where not the case, on of the following assert()s
+  ** If this were not the case, one of the following assert()s
   ** would fail.  Should this ever change (because of changes in the code
   ** generator) then the fix would be to insert a call to
   ** sqlite3VdbeCursorMoveto().
@@ -6028,7 +6238,7 @@ case OP_NullRow: {
 */
 /* Opcode: Last P1 P2 * * *
 **
-** The next use of the Rowid or Column or Prev instruction for P1 
+** The next use of the Rowid or Column or Prev instruction for P1
 ** will refer to the last entry in the database table or index.
 ** If the table or index is empty and P2>0, then jump immediately to P2.
 ** If P2 is 0 or if the table or index is not empty, fall through
@@ -6039,7 +6249,7 @@ case OP_NullRow: {
 ** configured to use Prev, not Next.
 */
 case OP_SeekEnd:             /* ncycle */
-case OP_Last: {              /* jump, ncycle */
+case OP_Last: {              /* jump0, ncycle */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
@@ -6073,28 +6283,38 @@ case OP_Last: {              /* jump, ncycle */
   break;
 }
 
-/* Opcode: IfSmaller P1 P2 P3 * *
+/* Opcode: IfSizeBetween P1 P2 P3 P4 *
+**
+** Let N be the approximate number of rows in the table or index
+** with cursor P1 and let X be 10*log2(N) if N is positive or -1
+** if N is zero.
 **
-** Estimate the number of rows in the table P1.  Jump to P2 if that
-** estimate is less than approximately 2**(0.1*P3).
+** Jump to P2 if X is in between P3 and P4, inclusive.
 */
-case OP_IfSmaller: {        /* jump */
+case OP_IfSizeBetween: {        /* jump */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
   i64 sz;
 
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  assert( pOp->p4type==P4_INT32 );
+  assert( pOp->p3>=-1 && pOp->p3<=640*2 );
+  assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
   pCrsr = pC->uc.pCursor;
   assert( pCrsr );
   rc = sqlite3BtreeFirst(pCrsr, &res);
   if( rc ) goto abort_due_to_error;
-  if( res==0 ){
+  if( res!=0 ){
+    sz = -1;  /* -Infinity encoding */
+  }else{
     sz = sqlite3BtreeRowCountEst(pCrsr);
-    if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)<pOp->p3 ) res = 1;
+    assert( sz>0 );
+    sz = sqlite3LogEst((u64)sz);
   }
+  res = sz>=pOp->p3 && sz<=pOp->p4.i;
   VdbeBranchTaken(res!=0,2);
   if( res ) goto jump_to_p2;
   break;
@@ -6122,8 +6342,8 @@ case OP_IfSmaller: {        /* jump */
 ** regression tests can determine whether or not the optimizer is
 ** correctly optimizing out sorts.
 */
-case OP_SorterSort:    /* jump */
-case OP_Sort: {        /* jump */
+case OP_SorterSort:    /* jump ncycle */
+case OP_Sort: {        /* jump ncycle */
 #ifdef SQLITE_TEST
   sqlite3_sort_count++;
   sqlite3_search_count--;
@@ -6134,10 +6354,10 @@ case OP_Sort: {        /* jump */
 }
 /* Opcode: Rewind P1 P2 * * *
 **
-** The next use of the Rowid or Column or Next instruction for P1 
+** The next use of the Rowid or Column or Next instruction for P1
 ** will refer to the first entry in the database table or index.
 ** If the table or index is empty, jump immediately to P2.
-** If the table or index is not empty, fall through to the following 
+** If the table or index is not empty, fall through to the following
 ** instruction.
 **
 ** If P2 is zero, that is an assertion that the P1 table is never
@@ -6147,7 +6367,7 @@ case OP_Sort: {        /* jump */
 ** from the beginning toward the end.  In other words, the cursor is
 ** configured to use Next, not Prev.
 */
-case OP_Rewind: {        /* jump, ncycle */
+case OP_Rewind: {        /* jump0, ncycle */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
@@ -6182,6 +6402,32 @@ case OP_Rewind: {        /* jump, ncycle */
   break;
 }
 
+/* Opcode: IfEmpty P1 P2 * * *
+** Synopsis: if( empty(P1) ) goto P2
+**
+** Check to see if the b-tree table that cursor P1 references is empty
+** and jump to P2 if it is.
+*/
+case OP_IfEmpty: {        /* jump */
+  VdbeCursor *pC;
+  BtCursor *pCrsr;
+  int res;
+
+  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  assert( pOp->p2>=0 && pOp->p2<p->nOp );
+
+  pC = p->apCsr[pOp->p1];
+  assert( pC!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
+  pCrsr = pC->uc.pCursor;
+  assert( pCrsr );
+  rc = sqlite3BtreeIsEmpty(pCrsr, &res);
+  if( rc ) goto abort_due_to_error;
+  VdbeBranchTaken(res!=0,2);
+  if( res ) goto jump_to_p2;
+  break;
+}
+
 /* Opcode: Next P1 P2 P3 * P5
 **
 ** Advance cursor P1 so that it points to the next key/data pair in its
@@ -6314,7 +6560,7 @@ case OP_Next:          /* jump, ncycle */
 ** run faster by avoiding an unnecessary seek on cursor P1.  However,
 ** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior
 ** seeks on the cursor or if the most recent seek used a key equivalent
-** to P2. 
+** to P2.
 **
 ** This instruction only works for indices.  The equivalent instruction
 ** for tables is OP_Insert.
@@ -6340,7 +6586,7 @@ case OP_IdxInsert: {        /* in2 */
   x.aMem = aMem + pOp->p3;
   x.nMem = (u16)pOp->p4.i;
   rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
-       (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), 
+       (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)),
       ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
       );
   assert( pC->deferredMoveto==0 );
@@ -6378,7 +6624,7 @@ case OP_SorterInsert: {     /* in2 */
 ** Synopsis: key=r[P2@P3]
 **
 ** The content of P3 registers starting at register P2 form
-** an unpacked index key. This opcode removes that entry from the 
+** an unpacked index key. This opcode removes that entry from the
 ** index opened by cursor P1.
 **
 ** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error
@@ -6436,8 +6682,8 @@ case OP_IdxDelete: {
 **
 ** P4 may be an array of integers (type P4_INTARRAY) containing
 ** one entry for each column in the P3 table.  If array entry a(i)
-** is non-zero, then reading column a(i)-1 from cursor P3 is 
-** equivalent to performing the deferred seek and then reading column i 
+** is non-zero, then reading column a(i)-1 from cursor P3 is
+** equivalent to performing the deferred seek and then reading column i
 ** from P1.  This information is stored in P3 and used to redirect
 ** reads against P3 over to P1, thus possibly avoiding the need to
 ** seek and read cursor P3.
@@ -6509,7 +6755,7 @@ case OP_IdxRowid: {           /* out2, ncycle */
 }
 
 /* Opcode: FinishSeek P1 * * * *
-** 
+**
 ** If cursor P1 was previously moved via OP_DeferredSeek, complete that
 ** seek operation now, without further delay.  If the cursor seek has
 ** already occurred, this instruction is a no-op.
@@ -6529,9 +6775,9 @@ case OP_FinishSeek: {        /* ncycle */
 /* Opcode: IdxGE P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** The P4 register values beginning with P3 form an unpacked index 
-** key that omits the PRIMARY KEY.  Compare this key value against the index 
-** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID 
+** The P4 register values beginning with P3 form an unpacked index
+** key that omits the PRIMARY KEY.  Compare this key value against the index
+** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID
 ** fields at the end.
 **
 ** If the P1 index entry is greater than or equal to the key value
@@ -6540,9 +6786,9 @@ case OP_FinishSeek: {        /* ncycle */
 /* Opcode: IdxGT P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** The P4 register values beginning with P3 form an unpacked index 
-** key that omits the PRIMARY KEY.  Compare this key value against the index 
-** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID 
+** The P4 register values beginning with P3 form an unpacked index
+** key that omits the PRIMARY KEY.  Compare this key value against the index
+** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID
 ** fields at the end.
 **
 ** If the P1 index entry is greater than the key value
@@ -6551,7 +6797,7 @@ case OP_FinishSeek: {        /* ncycle */
 /* Opcode: IdxLT P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** The P4 register values beginning with P3 form an unpacked index 
+** The P4 register values beginning with P3 form an unpacked index
 ** key that omits the PRIMARY KEY or ROWID.  Compare this key value against
 ** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or
 ** ROWID on the P1 index.
@@ -6562,7 +6808,7 @@ case OP_FinishSeek: {        /* ncycle */
 /* Opcode: IdxLE P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
 **
-** The P4 register values beginning with P3 form an unpacked index 
+** The P4 register values beginning with P3 form an unpacked index
 ** key that omits the PRIMARY KEY or ROWID.  Compare this key value against
 ** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or
 ** ROWID on the P1 index.
@@ -6650,7 +6896,7 @@ case OP_IdxGE:  {       /* jump, ncycle */
 ** file is given by P1.
 **
 ** The table being destroyed is in the main database file if P3==0.  If
-** P3==1 then the table to be clear is in the auxiliary database file
+** P3==1 then the table to be destroyed is in the auxiliary database file
 ** that is used to store tables create using CREATE TEMPORARY TABLE.
 **
 ** If AUTOVACUUM is enabled then it is possible that another root page
@@ -6658,15 +6904,15 @@ case OP_IdxGE:  {       /* jump, ncycle */
 ** root pages contiguous at the beginning of the database.  The former
 ** value of the root page that moved - its value before the move occurred -
 ** is stored in register P2. If no page movement was required (because the
-** table being dropped was already the last one in the database) then a 
-** zero is stored in register P2.  If AUTOVACUUM is disabled then a zero 
+** table being dropped was already the last one in the database) then a
+** zero is stored in register P2.  If AUTOVACUUM is disabled then a zero
 ** is stored in register P2.
 **
 ** This opcode throws an error if there are any active reader VMs when
-** it is invoked. This is done to avoid the difficulty associated with 
-** updating existing cursors when a root page is moved in an AUTOVACUUM 
-** database. This error is thrown even if the database is not an AUTOVACUUM 
-** db in order to avoid introducing an incompatibility between autovacuum 
+** it is invoked. This is done to avoid the difficulty associated with
+** updating existing cursors when a root page is moved in an AUTOVACUUM
+** database. This error is thrown even if the database is not an AUTOVACUUM
+** db in order to avoid introducing an incompatibility between autovacuum
 ** and non-autovacuum modes.
 **
 ** See also: Clear
@@ -6710,8 +6956,8 @@ case OP_Destroy: {     /* out2 */
 ** in the database file is given by P1.  But, unlike Destroy, do not
 ** remove the table or index from the database file.
 **
-** The table being clear is in the main database file if P2==0.  If
-** P2==1 then the table to be clear is in the auxiliary database file
+** The table being cleared is in the main database file if P2==0.  If
+** P2==1 then the table to be cleared is in the auxiliary database file
 ** that is used to store tables create using CREATE TEMPORARY TABLE.
 **
 ** If the P3 value is non-zero, then the row change count is incremented
@@ -6723,7 +6969,7 @@ case OP_Destroy: {     /* out2 */
 */
 case OP_Clear: {
   i64 nChange;
- 
+
   sqlite3VdbeIncrWriteCounter(p, 0);
   nChange = 0;
   assert( p->readOnly==0 );
@@ -6751,7 +6997,7 @@ case OP_Clear: {
 */
 case OP_ResetSorter: {
   VdbeCursor *pC;
- 
+
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
@@ -6794,16 +7040,57 @@ case OP_CreateBtree: {          /* out2 */
   break;
 }
 
-/* Opcode: SqlExec * * * P4 *
+/* Opcode: SqlExec P1 P2 * P4 *
 **
 ** Run the SQL statement or statements specified in the P4 string.
+**
+** The P1 parameter is a bitmask of options:
+**
+**    0x0001     Disable Auth and Trace callbacks while the statements
+**               in P4 are running.
+**
+**    0x0002     Set db->nAnalysisLimit to P2 while the statements in
+**               P4 are running.
+**
 */
 case OP_SqlExec: {
+  char *zErr;
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  sqlite3_xauth xAuth;
+#endif
+  u8 mTrace;
+  int savedAnalysisLimit;
+
   sqlite3VdbeIncrWriteCounter(p, 0);
   db->nSqlExec++;
-  rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0);
+  zErr = 0;
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  xAuth = db->xAuth;
+#endif
+  mTrace = db->mTrace;
+  savedAnalysisLimit = db->nAnalysisLimit;
+  if( pOp->p1 & 0x0001 ){
+#ifndef SQLITE_OMIT_AUTHORIZATION
+    db->xAuth = 0;
+#endif
+    db->mTrace = 0;
+  }
+  if( pOp->p1 & 0x0002 ){
+    db->nAnalysisLimit = pOp->p2;
+  }
+  rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr);
   db->nSqlExec--;
-  if( rc ) goto abort_due_to_error;
+#ifndef SQLITE_OMIT_AUTHORIZATION
+  db->xAuth = xAuth;
+#endif
+  db->mTrace = mTrace;
+  db->nAnalysisLimit = savedAnalysisLimit;
+  if( zErr || rc ){
+    sqlite3VdbeError(p, "%s", zErr);
+    sqlite3_free(zErr);
+    if( rc==SQLITE_NOMEM ) goto no_mem;
+    goto abort_due_to_error;
+  }
   break;
 }
 
@@ -6823,7 +7110,7 @@ case OP_ParseSchema: {
   InitData initData;
 
   /* Any prepared statement that invokes this opcode will hold mutexes
-  ** on every btree.  This is a prerequisite for invoking 
+  ** on every btree.  This is a prerequisite for invoking
   ** sqlite3InitCallback().
   */
 #ifdef SQLITE_DEBUG
@@ -6884,7 +7171,7 @@ case OP_ParseSchema: {
     }
     goto abort_due_to_error;
   }
-  break;  
+  break; 
 }
 
 #if !defined(SQLITE_OMIT_ANALYZE)
@@ -6898,7 +7185,7 @@ case OP_LoadAnalysis: {
   assert( pOp->p1>=0 && pOp->p1<db->nDb );
   rc = sqlite3AnalysisLoad(db, pOp->p1);
   if( rc ) goto abort_due_to_error;
-  break;  
+  break; 
 }
 #endif /* !defined(SQLITE_OMIT_ANALYZE) */
 
@@ -6906,7 +7193,7 @@ case OP_LoadAnalysis: {
 **
 ** Remove the internal (in-memory) data structures that describe
 ** the table named P4 in database P1.  This is called after a table
-** is dropped from disk (using the Destroy opcode) in order to keep 
+** is dropped from disk (using the Destroy opcode) in order to keep
 ** the internal representation of the
 ** schema consistent with what is on disk.
 */
@@ -6934,7 +7221,7 @@ case OP_DropIndex: {
 **
 ** Remove the internal (in-memory) data structures that describe
 ** the trigger named P4 in database P1.  This is called after a trigger
-** is dropped from disk (using the Destroy opcode) in order to keep 
+** is dropped from disk (using the Destroy opcode) in order to keep
 ** the internal representation of the
 ** schema consistent with what is on disk.
 */
@@ -6949,12 +7236,12 @@ case OP_DropTrigger: {
 /* Opcode: IntegrityCk P1 P2 P3 P4 P5
 **
 ** Do an analysis of the currently open database.  Store in
-** register P1 the text of an error message describing any problems.
-** If no problems are found, store a NULL in register P1.
+** register (P1+1) the text of an error message describing any problems.
+** If no problems are found, store a NULL in register (P1+1).
 **
-** The register P3 contains one less than the maximum number of allowed errors.
-** At most reg(P3) errors will be reported.
-** In other words, the analysis stops as soon as reg(P1) errors are 
+** The register (P1) contains one less than the maximum number of allowed
+** errors.  At most reg(P1) errors will be reported.
+** In other words, the analysis stops as soon as reg(P1) errors are
 ** seen.  Reg(P1) is updated with the number of errors remaining.
 **
 ** The root page numbers of all tables in the database are integers
@@ -6973,19 +7260,21 @@ case OP_IntegrityCk: {
   Mem *pnErr;     /* Register keeping track of errors remaining */
 
   assert( p->bIsReader );
+  assert( pOp->p4type==P4_INTARRAY );
   nRoot = pOp->p2;
   aRoot = pOp->p4.ai;
   assert( nRoot>0 );
+  assert( aRoot!=0 );
   assert( aRoot[0]==(Pgno)nRoot );
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
-  pnErr = &aMem[pOp->p3];
+  assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) );
+  pnErr = &aMem[pOp->p1];
   assert( (pnErr->flags & MEM_Int)!=0 );
   assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 );
-  pIn1 = &aMem[pOp->p1];
+  pIn1 = &aMem[pOp->p1+1];
   assert( pOp->p5<db->nDb );
   assert( DbMaskTest(p->btreeMask, pOp->p5) );
-  rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot,
-                                 (int)pnErr->u.i+1, &nErr, &z);
+  rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], 
+      &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z);
   sqlite3VdbeMemSetNull(pIn1);
   if( nErr==0 ){
     assert( z==0 );
@@ -7035,7 +7324,7 @@ case OP_RowSetRead: {       /* jump, in1, out3 */
 
   pIn1 = &aMem[pOp->p1];
   assert( (pIn1->flags & MEM_Blob)==0 || sqlite3VdbeMemIsRowSet(pIn1) );
-  if( (pIn1->flags & MEM_Blob)==0 
+  if( (pIn1->flags & MEM_Blob)==0
    || sqlite3RowSetNext((RowSet*)pIn1->z, &val)==0
   ){
     /* The boolean index is empty */
@@ -7107,22 +7396,24 @@ case OP_RowSetTest: {                     /* jump, in1, in3 */
 
 /* Opcode: Program P1 P2 P3 P4 P5
 **
-** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). 
+** Execute the trigger program passed as P4 (type P4_SUBPROGRAM).
 **
-** P1 contains the address of the memory cell that contains the first memory 
-** cell in an array of values used as arguments to the sub-program. P2 
-** contains the address to jump to if the sub-program throws an IGNORE 
-** exception using the RAISE() function. Register P3 contains the address 
-** of a memory cell in this (the parent) VM that is used to allocate the 
+** P1 contains the address of the memory cell that contains the first memory
+** cell in an array of values used as arguments to the sub-program. P2
+** contains the address to jump to if the sub-program throws an IGNORE
+** exception using the RAISE() function. P2 might be zero, if there is
+** no possibility that an IGNORE exception will be raised.
+** Register P3 contains the address
+** of a memory cell in this (the parent) VM that is used to allocate the
 ** memory required by the sub-vdbe at runtime.
 **
 ** P4 is a pointer to the VM containing the trigger program.
 **
 ** If P5 is non-zero, then recursive program invocation is enabled.
 */
-case OP_Program: {        /* jump */
+case OP_Program: {        /* jump0 */
   int nMem;               /* Number of memory registers for sub-program */
-  int nByte;              /* Bytes of runtime space required for sub-program */
+  i64 nByte;              /* Bytes of runtime space required for sub-program */
   Mem *pRt;               /* Register to allocate runtime space */
   Mem *pMem;              /* Used to iterate through memory cells */
   Mem *pEnd;              /* Last memory cell in new array */
@@ -7133,17 +7424,17 @@ case OP_Program: {        /* jump */
   pProgram = pOp->p4.pProgram;
   pRt = &aMem[pOp->p3];
   assert( pProgram->nOp>0 );
-  
-  /* If the p5 flag is clear, then recursive invocation of triggers is 
+ 
+  /* If the p5 flag is clear, then recursive invocation of triggers is
   ** disabled for backwards compatibility (p5 is set if this sub-program
   ** is really a trigger, not a foreign key action, and the flag set
   ** and cleared by the "PRAGMA recursive_triggers" command is clear).
-  ** 
-  ** It is recursive invocation of triggers, at the SQL level, that is 
-  ** disabled. In some cases a single trigger may generate more than one 
-  ** SubProgram (if the trigger may be executed with more than one different 
+  **
+  ** It is recursive invocation of triggers, at the SQL level, that is
+  ** disabled. In some cases a single trigger may generate more than one
+  ** SubProgram (if the trigger may be executed with more than one different
   ** ON CONFLICT algorithm). SubProgram structures associated with a
-  ** single trigger all have the same value for the SubProgram.token 
+  ** single trigger all have the same value for the SubProgram.token
   ** variable.  */
   if( pOp->p5 ){
     t = pProgram->token;
@@ -7159,10 +7450,10 @@ case OP_Program: {        /* jump */
 
   /* Register pRt is used to store the memory required to save the state
   ** of the current program, and the memory required at runtime to execute
-  ** the trigger program. If this trigger has been fired before, then pRt 
+  ** the trigger program. If this trigger has been fired before, then pRt
   ** is already allocated. Otherwise, it must be initialized.  */
   if( (pRt->flags&MEM_Blob)==0 ){
-    /* SubProgram.nMem is set to the number of memory cells used by the 
+    /* SubProgram.nMem is set to the number of memory cells used by the
     ** program stored in SubProgram.aOp. As well as these, one memory
     ** cell is required for each cursor used by the program. Set local
     ** variable nMem (and later, VdbeFrame.nChildMem) to this value.
@@ -7173,7 +7464,7 @@ case OP_Program: {        /* jump */
     nByte = ROUND8(sizeof(VdbeFrame))
               + nMem * sizeof(Mem)
               + pProgram->nCsr * sizeof(VdbeCursor*)
-              + (pProgram->nOp + 7)/8;
+              + (7 + (i64)pProgram->nOp)/8;
     pFrame = sqlite3DbMallocZero(db, nByte);
     if( !pFrame ){
       goto no_mem;
@@ -7181,7 +7472,7 @@ case OP_Program: {        /* jump */
     sqlite3VdbeMemRelease(pRt);
     pRt->flags = MEM_Blob|MEM_Dyn;
     pRt->z = (char*)pFrame;
-    pRt->n = nByte;
+    pRt->n = (int)nByte;
     pRt->xDel = sqlite3VdbeFrameMemDel;
 
     pFrame->v = p;
@@ -7207,7 +7498,7 @@ case OP_Program: {        /* jump */
   }else{
     pFrame = (VdbeFrame*)pRt->z;
     assert( pRt->xDel==sqlite3VdbeFrameMemDel );
-    assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem 
+    assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem
         || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) );
     assert( pProgram->nCsr==pFrame->nChildCsr );
     assert( (int)(pOp - aOp)==pFrame->pc );
@@ -7248,10 +7539,10 @@ case OP_Program: {        /* jump */
 
 /* Opcode: Param P1 P2 * * *
 **
-** This opcode is only ever present in sub-programs called via the 
-** OP_Program instruction. Copy a value currently stored in a memory 
-** cell of the calling (parent) frame to cell P2 in the current frames 
-** address space. This is used by trigger programs to access the new.* 
+** This opcode is only ever present in sub-programs called via the
+** OP_Program instruction. Copy a value currently stored in a memory
+** cell of the calling (parent) frame to cell P2 in the current frames
+** address space. This is used by trigger programs to access the new.*
 ** and old.* values.
 **
 ** The address of the cell in the parent frame is determined by adding
@@ -7263,7 +7554,7 @@ case OP_Param: {           /* out2 */
   Mem *pIn;
   pOut = out2Prerelease(p, pOp);
   pFrame = p->pFrame;
-  pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1];   
+  pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1];  
   sqlite3VdbeMemShallowCopy(pOut, pIn, MEM_Ephem);
   break;
 }
@@ -7275,17 +7566,19 @@ case OP_Param: {           /* out2 */
 ** Synopsis: fkctr[P1]+=P2
 **
 ** Increment a "constraint counter" by P2 (P2 may be negative or positive).
-** If P1 is non-zero, the database constraint counter is incremented 
-** (deferred foreign key constraints). Otherwise, if P1 is zero, the 
+** If P1 is non-zero, the database constraint counter is incremented
+** (deferred foreign key constraints). Otherwise, if P1 is zero, the
 ** statement counter is incremented (immediate foreign key constraints).
 */
 case OP_FkCounter: {
-  if( db->flags & SQLITE_DeferFKs ){
-    db->nDeferredImmCons += pOp->p2;
-  }else if( pOp->p1 ){
+  if( pOp->p1 ){
     db->nDeferredCons += pOp->p2;
   }else{
-    p->nFkConstraint += pOp->p2;
+    if( db->flags & SQLITE_DeferFKs ){
+      db->nDeferredImmCons += pOp->p2;
+    }else{
+      p->nFkConstraint += pOp->p2;
+    }
   }
   break;
 }
@@ -7294,7 +7587,7 @@ case OP_FkCounter: {
 ** Synopsis: if fkctr[P1]==0 goto P2
 **
 ** This opcode tests if a foreign key constraint-counter is currently zero.
-** If so, jump to instruction P2. Otherwise, fall through to the next 
+** If so, jump to instruction P2. Otherwise, fall through to the next
 ** instruction.
 **
 ** If P1 is non-zero, then the jump is taken if the database constraint-counter
@@ -7320,7 +7613,7 @@ case OP_FkIfZero: {         /* jump */
 **
 ** P1 is a register in the root frame of this VM (the root frame is
 ** different from the current frame if this instruction is being executed
-** within a sub-program). Set the value of register P1 to the maximum of 
+** within a sub-program). Set the value of register P1 to the maximum of
 ** its current value and the value in register P2.
 **
 ** This instruction throws an error if the memory cell is not initially
@@ -7380,7 +7673,7 @@ case OP_IfPos: {        /* jump, in1 */
 ** and r[P2] is set to be the value of the LIMIT, r[P1].
 **
 ** if r[P1] is zero or negative, that means there is no LIMIT
-** and r[P2] is set to -1. 
+** and r[P2] is set to -1.
 **
 ** Otherwise, r[P2] is set to the sum of r[P1] and r[P3].
 */
@@ -7412,7 +7705,7 @@ case OP_OffsetLimit: {    /* in1, out2, in3 */
 **
 ** Register P1 must contain an integer.  If the content of register P1 is
 ** initially greater than zero, then decrement the value in register P1.
-** If it is non-zero (negative or positive) and then also jump to P2.  
+** If it is non-zero (negative or positive) and then also jump to P2. 
 ** If register P1 is initially zero, leave it unchanged and fall through.
 */
 case OP_IfNotZero: {        /* jump, in1 */
@@ -7446,7 +7739,7 @@ case OP_DecrJumpZero: {      /* jump, in1 */
 ** Synopsis: accum=r[P3] step(r[P2@P5])
 **
 ** Execute the xStep function for an aggregate.
-** The function has P5 arguments.  P4 is a pointer to the 
+** The function has P5 arguments.  P4 is a pointer to the
 ** FuncDef structure that specifies the function.  Register P3 is the
 ** accumulator.
 **
@@ -7457,7 +7750,7 @@ case OP_DecrJumpZero: {      /* jump, in1 */
 ** Synopsis: accum=r[P3] inverse(r[P2@P5])
 **
 ** Execute the xInverse function for an aggregate.
-** The function has P5 arguments.  P4 is a pointer to the 
+** The function has P5 arguments.  P4 is a pointer to the
 ** FuncDef structure that specifies the function.  Register P3 is the
 ** accumulator.
 **
@@ -7468,7 +7761,7 @@ case OP_DecrJumpZero: {      /* jump, in1 */
 ** Synopsis: accum=r[P3] step(r[P2@P5])
 **
 ** Execute the xStep (if P1==0) or xInverse (if P1!=0) function for an
-** aggregate.  The function has P5 arguments.  P4 is a pointer to the 
+** aggregate.  The function has P5 arguments.  P4 is a pointer to the
 ** FuncDef structure that specifies the function.  Register P3 is the
 ** accumulator.
 **
@@ -7485,18 +7778,29 @@ case OP_AggInverse:
 case OP_AggStep: {
   int n;
   sqlite3_context *pCtx;
+  u64 nAlloc;
 
   assert( pOp->p4type==P4_FUNCDEF );
   n = pOp->p5;
   assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) );
   assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+n );
-  pCtx = sqlite3DbMallocRawNN(db, n*sizeof(sqlite3_value*) +
-               (sizeof(pCtx[0]) + sizeof(Mem) - sizeof(sqlite3_value*)));
+
+  /* Allocate space for (a) the context object and (n-1) extra pointers
+  ** to append to the sqlite3_context.argv[1] array, and (b) a memory
+  ** cell in which to store the accumulation. Be careful that the memory
+  ** cell is 8-byte aligned, even on platforms where a pointer is 32-bits.
+  **
+  ** Note: We could avoid this by using a regular memory cell from aMem[] for 
+  ** the accumulator, instead of allocating one here. */
+  nAlloc = ROUND8P( SZ_CONTEXT(n) );
+  pCtx = sqlite3DbMallocRawNN(db, nAlloc + sizeof(Mem));
   if( pCtx==0 ) goto no_mem;
-  pCtx->pMem = 0;
-  pCtx->pOut = (Mem*)&(pCtx->argv[n]);
+  pCtx->pOut = (Mem*)((u8*)pCtx + nAlloc);
+  assert( EIGHT_BYTE_ALIGNMENT(pCtx->pOut) );
+
   sqlite3VdbeMemInit(pCtx->pOut, db, MEM_Null);
+  pCtx->pMem = 0;
   pCtx->pFunc = pOp->p4.pFunc;
   pCtx->iOp = (int)(pOp - aOp);
   pCtx->pVdbe = p;
@@ -7537,7 +7841,7 @@ case OP_AggStep1: {
   /* If this function is inside of a trigger, the register array in aMem[]
   ** might change from one evaluation to the next.  The next block of code
   ** checks to see if the register array has changed, and if so it
-  ** reinitializes the relavant parts of the sqlite3_context object */
+  ** reinitializes the relevant parts of the sqlite3_context object */
   if( pCtx->pMem != pMem ){
     pCtx->pMem = pMem;
     for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i];
@@ -7586,7 +7890,7 @@ case OP_AggStep1: {
 ** Synopsis: accum=r[P1] N=P2
 **
 ** P1 is the memory location that is the accumulator for an aggregate
-** or window function.  Execute the finalizer function 
+** or window function.  Execute the finalizer function
 ** for an aggregate and store the result in P1.
 **
 ** P2 is the number of arguments that the step function takes and
@@ -7625,7 +7929,7 @@ case OP_AggFinal: {
   {
     rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc);
   }
-  
+ 
   if( rc ){
     sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem));
     goto abort_due_to_error;
@@ -7660,6 +7964,7 @@ case OP_Checkpoint: {
        || pOp->p2==SQLITE_CHECKPOINT_FULL
        || pOp->p2==SQLITE_CHECKPOINT_RESTART
        || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE
+       || pOp->p2==SQLITE_CHECKPOINT_NOOP
   );
   rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]);
   if( rc ){
@@ -7669,9 +7974,9 @@ case OP_Checkpoint: {
   }
   for(i=0, pMem = &aMem[pOp->p3]; i<3; i++, pMem++){
     sqlite3VdbeMemSetInt64(pMem, (i64)aRes[i]);
-  }    
+  }   
   break;
-};  
+}; 
 #endif
 
 #ifndef SQLITE_OMIT_PRAGMA
@@ -7697,9 +8002,9 @@ case OP_JournalMode: {    /* out2 */
 
   pOut = out2Prerelease(p, pOp);
   eNew = pOp->p3;
-  assert( eNew==PAGER_JOURNALMODE_DELETE 
-       || eNew==PAGER_JOURNALMODE_TRUNCATE 
-       || eNew==PAGER_JOURNALMODE_PERSIST 
+  assert( eNew==PAGER_JOURNALMODE_DELETE
+       || eNew==PAGER_JOURNALMODE_TRUNCATE
+       || eNew==PAGER_JOURNALMODE_PERSIST
        || eNew==PAGER_JOURNALMODE_OFF
        || eNew==PAGER_JOURNALMODE_MEMORY
        || eNew==PAGER_JOURNALMODE_WAL
@@ -7719,7 +8024,7 @@ case OP_JournalMode: {    /* out2 */
   zFilename = sqlite3PagerFilename(pPager, 1);
 
   /* Do not allow a transition to journal_mode=WAL for a database
-  ** in temporary storage or if the VFS does not support shared memory 
+  ** in temporary storage or if the VFS does not support shared memory
   */
   if( eNew==PAGER_JOURNALMODE_WAL
    && (sqlite3Strlen30(zFilename)==0           /* Temp file */
@@ -7739,12 +8044,12 @@ case OP_JournalMode: {    /* out2 */
       );
       goto abort_due_to_error;
     }else{
- 
+
       if( eOld==PAGER_JOURNALMODE_WAL ){
         /* If leaving WAL mode, close the log file. If successful, the call
-        ** to PagerCloseWal() checkpoints and deletes the write-ahead-log 
-        ** file. An EXCLUSIVE lock may still be held on the database file 
-        ** after a successful return. 
+        ** to PagerCloseWal() checkpoints and deletes the write-ahead-log
+        ** file. An EXCLUSIVE lock may still be held on the database file
+        ** after a successful return.
         */
         rc = sqlite3PagerCloseWal(pPager, db);
         if( rc==SQLITE_OK ){
@@ -7755,7 +8060,7 @@ case OP_JournalMode: {    /* out2 */
         ** as an intermediate */
         sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF);
       }
-  
+ 
       /* Open a transaction on the database file. Regardless of the journal
       ** mode, this transaction always uses a rollback journal.
       */
@@ -7830,7 +8135,7 @@ case OP_IncrVacuum: {        /* jump */
 ** is executed using sqlite3_step() it will either automatically
 ** reprepare itself (if it was originally created using sqlite3_prepare_v2())
 ** or it will fail with SQLITE_SCHEMA.
-** 
+**
 ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero,
 ** then only the currently executing statement is expired.
 **
@@ -7885,7 +8190,7 @@ case OP_CursorUnlock: {
 ** Synopsis: iDb=P1 root=P2 write=P3
 **
 ** Obtain a lock on a particular table. This instruction is only used when
-** the shared-cache feature is enabled. 
+** the shared-cache feature is enabled.
 **
 ** P1 is the index of the database in sqlite3.aDb[] of the database
 ** on which the lock is acquired.  A readlock is obtained if P3==0 or
@@ -7899,7 +8204,7 @@ case OP_CursorUnlock: {
 case OP_TableLock: {
   u8 isWriteLock = (u8)pOp->p3;
   if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){
-    int p1 = pOp->p1; 
+    int p1 = pOp->p1;
     assert( p1>=0 && p1<db->nDb );
     assert( DbMaskTest(p->btreeMask, p1) );
     assert( isWriteLock==0 || isWriteLock==1 );
@@ -7919,7 +8224,7 @@ case OP_TableLock: {
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /* Opcode: VBegin * * * P4 *
 **
-** P4 may be a pointer to an sqlite3_vtab structure. If so, call the 
+** P4 may be a pointer to an sqlite3_vtab structure. If so, call the
 ** xBegin method for that table.
 **
 ** Also, whether or not P4 is set, check that this is not being called from
@@ -7939,7 +8244,7 @@ case OP_VBegin: {
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /* Opcode: VCreate P1 P2 * * *
 **
-** P2 is a register that holds the name of a virtual table in database 
+** P2 is a register that holds the name of a virtual table in database
 ** P1. Call the xCreate method for that table.
 */
 case OP_VCreate: {
@@ -7995,7 +8300,14 @@ case OP_VOpen: {             /* ncycle */
   const sqlite3_module *pModule;
 
   assert( p->bIsReader );
-  pCur = 0;
+  pCur = p->apCsr[pOp->p1];
+  if( pCur!=0
+   && ALWAYS( pCur->eCurType==CURTYPE_VTAB )
+   && ALWAYS( pCur->uc.pVCur->pVtab==pOp->p4.pVtab->pVtab )
+  ){
+    /* This opcode is a no-op if the cursor is already open */
+    break;
+  }
   pVCur = 0;
   pVtab = pOp->p4.pVtab->pVtab;
   if( pVtab==0 || NEVER(pVtab->pModule==0) ){
@@ -8024,6 +8336,52 @@ case OP_VOpen: {             /* ncycle */
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+/* Opcode: VCheck P1 P2 P3 P4 *
+**
+** P4 is a pointer to a Table object that is a virtual table in schema P1
+** that supports the xIntegrity() method.  This opcode runs the xIntegrity()
+** method for that virtual table, using P3 as the integer argument.  If
+** an error is reported back, the table name is prepended to the error
+** message and that message is stored in P2.  If no errors are seen,
+** register P2 is set to NULL.
+*/
+case OP_VCheck: {             /* out2 */
+  Table *pTab;
+  sqlite3_vtab *pVtab;
+  const sqlite3_module *pModule;
+  char *zErr = 0;
+
+  pOut = &aMem[pOp->p2];
+  sqlite3VdbeMemSetNull(pOut);  /* Innocent until proven guilty */
+  assert( pOp->p4type==P4_TABLEREF );
+  pTab = pOp->p4.pTab;
+  assert( pTab!=0 );
+  assert( pTab->nTabRef>0 );
+  assert( IsVirtual(pTab) );
+  if( pTab->u.vtab.p==0 ) break;
+  pVtab = pTab->u.vtab.p->pVtab;
+  assert( pVtab!=0 );
+  pModule = pVtab->pModule;
+  assert( pModule!=0 );
+  assert( pModule->iVersion>=4 );
+  assert( pModule->xIntegrity!=0 );
+  sqlite3VtabLock(pTab->u.vtab.p);
+  assert( pOp->p1>=0 && pOp->p1<db->nDb );
+  rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName,
+                           pOp->p3, &zErr);
+  sqlite3VtabUnlock(pTab->u.vtab.p);
+  if( rc ){
+    sqlite3_free(zErr);
+    goto abort_due_to_error;
+  }
+  if( zErr ){
+    sqlite3VdbeMemSetStr(pOut, zErr, -1, SQLITE_UTF8, sqlite3_free);
+  }
+  break;
+}
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /* Opcode: VInitIn P1 P2 P3 * *
 ** Synopsis: r[P2]=ValueList(P1,P3)
@@ -8103,6 +8461,7 @@ case OP_VFilter: {   /* jump, ncycle */
 
   /* Invoke the xFilter method */
   apArg = p->apArg;
+  assert( nArg<=p->napArg );
   for(i = 0; i<nArg; i++){
     apArg[i] = &pArgc[i+1];
   }
@@ -8137,6 +8496,7 @@ case OP_VColumn: {           /* ncycle */
   const sqlite3_module *pModule;
   Mem *pDest;
   sqlite3_context sContext;
+  FuncDef nullFunc;
 
   VdbeCursor *pCur = p->apCsr[pOp->p1];
   assert( pCur!=0 );
@@ -8154,6 +8514,9 @@ case OP_VColumn: {           /* ncycle */
   memset(&sContext, 0, sizeof(sContext));
   sContext.pOut = pDest;
   sContext.enc = encoding;
+  nullFunc.pUserData = 0;
+  nullFunc.funcFlags = SQLITE_RESULT_SUBTYPE;
+  sContext.pFunc = &nullFunc;
   assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 );
   if( pOp->p5 & OPFLAG_NOCHNG ){
     sqlite3VdbeMemSetNull(pDest);
@@ -8202,7 +8565,7 @@ case OP_VNext: {   /* jump, ncycle */
 
   /* Invoke the xNext() method of the module. There is no way for the
   ** underlying implementation to return an error if one occurs during
-  ** xNext(). Instead, if an error occurs, true is returned (indicating that 
+  ** xNext(). Instead, if an error occurs, true is returned (indicating that
   ** data is available) and the error code returned when xColumn or
   ** some other method is next invoked on the save virtual table cursor.
   */
@@ -8230,7 +8593,7 @@ case OP_VRename: {
   sqlite3_vtab *pVtab;
   Mem *pName;
   int isLegacy;
-  
+ 
   isLegacy = (db->flags & SQLITE_LegacyAlter);
   db->flags |= SQLITE_LegacyAlter;
   pVtab = pOp->p4.pVtab->pVtab;
@@ -8260,23 +8623,23 @@ case OP_VRename: {
 **
 ** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
 ** This opcode invokes the corresponding xUpdate method. P2 values
-** are contiguous memory cells starting at P3 to pass to the xUpdate 
-** invocation. The value in register (P3+P2-1) corresponds to the 
+** are contiguous memory cells starting at P3 to pass to the xUpdate
+** invocation. The value in register (P3+P2-1) corresponds to the
 ** p2th element of the argv array passed to xUpdate.
 **
 ** The xUpdate method will do a DELETE or an INSERT or both.
 ** The argv[0] element (which corresponds to memory cell P3)
-** is the rowid of a row to delete.  If argv[0] is NULL then no 
-** deletion occurs.  The argv[1] element is the rowid of the new 
-** row.  This can be NULL to have the virtual table select the new 
-** rowid for itself.  The subsequent elements in the array are 
+** is the rowid of a row to delete.  If argv[0] is NULL then no
+** deletion occurs.  The argv[1] element is the rowid of the new
+** row.  This can be NULL to have the virtual table select the new
+** rowid for itself.  The subsequent elements in the array are
 ** the values of columns in the new row.
 **
 ** If P2==1 then no insert is performed.  argv[0] is the rowid of
 ** a row to delete.
 **
 ** P1 is a boolean flag. If it is set to true and the xUpdate call
-** is successful, then the value returned by sqlite3_last_insert_rowid() 
+** is successful, then the value returned by sqlite3_last_insert_rowid()
 ** is set to the value of the rowid for the row just inserted.
 **
 ** P5 is the error actions (OE_Replace, OE_Fail, OE_Ignore, etc) to
@@ -8291,7 +8654,7 @@ case OP_VUpdate: {
   Mem **apArg;
   Mem *pX;
 
-  assert( pOp->p2==1        || pOp->p5==OE_Fail   || pOp->p5==OE_Rollback 
+  assert( pOp->p2==1        || pOp->p5==OE_Fail   || pOp->p5==OE_Rollback
        || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace
   );
   assert( p->readOnly==0 );
@@ -8309,6 +8672,7 @@ case OP_VUpdate: {
     u8 vtabOnConflict = db->vtabOnConflict;
     apArg = p->apArg;
     pX = &aMem[pOp->p3];
+    assert( nArg<=p->napArg );
     for(i=0; i<nArg; i++){
       assert( memIsValid(pX) );
       memAboutToChange(p, pX);
@@ -8386,7 +8750,7 @@ case OP_MaxPgcnt: {            /* out2 */
 ** The result of the function is stored
 ** in register P3.  Register P3 must not be one of the function inputs.
 **
-** P1 is a 32-bit bitmask indicating whether or not each argument to the 
+** P1 is a 32-bit bitmask indicating whether or not each argument to the
 ** function was determined to be constant at compile time. If the first
 ** argument was constant then bit 0 of P1 is set. This is used to determine
 ** whether meta data associated with a user function argument using the
@@ -8405,7 +8769,7 @@ case OP_MaxPgcnt: {            /* out2 */
 ** The result of the function is stored
 ** in register P3.  Register P3 must not be one of the function inputs.
 **
-** P1 is a 32-bit bitmask indicating whether or not each argument to the 
+** P1 is a 32-bit bitmask indicating whether or not each argument to the
 ** function was determined to be constant at compile time. If the first
 ** argument was constant then bit 0 of P1 is set. This is used to determine
 ** whether meta data associated with a user function argument using the
@@ -8415,7 +8779,7 @@ case OP_MaxPgcnt: {            /* out2 */
 ** This opcode works exactly like OP_Function.  The only difference is in
 ** its name.  This opcode is used in places where the function must be
 ** purely non-deterministic.  Some built-in date/time functions can be
-** either determinitic of non-deterministic, depending on their arguments.
+** either deterministic of non-deterministic, depending on their arguments.
 ** When those function are used in a non-deterministic way, they will check
 ** to see if they were called using OP_PureFunc instead of OP_Function, and
 ** if they were, they throw an error.
@@ -8433,7 +8797,7 @@ case OP_Function: {            /* group */
   /* If this function is inside of a trigger, the register array in aMem[]
   ** might change from one evaluation to the next.  The next block of code
   ** checks to see if the register array has changed, and if so it
-  ** reinitializes the relavant parts of the sqlite3_context object */
+  ** reinitializes the relevant parts of the sqlite3_context object */
   pOut = &aMem[pOp->p3];
   if( pCtx->pOut != pOut ){
     pCtx->pVdbe = p;
@@ -8465,7 +8829,7 @@ case OP_Function: {            /* group */
     if( rc ) goto abort_due_to_error;
   }
 
-  assert( (pOut->flags&MEM_Str)==0 
+  assert( (pOut->flags&MEM_Str)==0
        || pOut->enc==encoding
        || db->mallocFailed );
   assert( !sqlite3VdbeMemTooBig(pOut) );
@@ -8486,6 +8850,42 @@ case OP_ClrSubtype: {   /* in1 */
   break;
 }
 
+/* Opcode: GetSubtype P1 P2 * * *
+** Synopsis:  r[P2] = r[P1].subtype
+**
+** Extract the subtype value from register P1 and write that subtype
+** into register P2.  If P1 has no subtype, then P1 gets a NULL.
+*/
+case OP_GetSubtype: {   /* in1 out2 */
+  pIn1 = &aMem[pOp->p1];
+  pOut = &aMem[pOp->p2];
+  if( pIn1->flags & MEM_Subtype ){
+    sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype);
+  }else{
+    sqlite3VdbeMemSetNull(pOut);
+  }
+  break;
+}
+
+/* Opcode: SetSubtype P1 P2 * * *
+** Synopsis:  r[P2].subtype = r[P1]
+**
+** Set the subtype value of register P2 to the integer from register P1.
+** If P1 is NULL, clear the subtype from p2.
+*/
+case OP_SetSubtype: {   /* in1 out2 */
+  pIn1 = &aMem[pOp->p1];
+  pOut = &aMem[pOp->p2];
+  if( pIn1->flags & MEM_Null ){
+    pOut->flags &= ~MEM_Subtype;
+  }else{
+    assert( pIn1->flags & MEM_Int );
+    pOut->flags |= MEM_Subtype;
+    pOut->eSubtype = (u8)(pIn1->u.i & 0xff);
+  }
+  break;
+}
+
 /* Opcode: FilterAdd P1 * P3 P4 *
 ** Synopsis: filter(P1) += key(P3@P4)
 **
@@ -8509,7 +8909,7 @@ case OP_FilterAdd: {
     printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n));
   }
 #endif
-  h %= pIn1->n;
+  h %= (pIn1->n*8);
   pIn1->z[h/8] |= 1<<(h&7);
   break;
 }
@@ -8545,7 +8945,7 @@ case OP_Filter: {          /* jump */
     printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n));
   }
 #endif
-  h %= pIn1->n;
+  h %= (pIn1->n*8);
   if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){
     VdbeBranchTaken(1, 2);
     p->aCounter[SQLITE_STMTSTATUS_FILTER_HIT]++;
@@ -8583,7 +8983,7 @@ case OP_Filter: {          /* jump */
 ** error is encountered.
 */
 case OP_Trace:
-case OP_Init: {          /* jump */
+case OP_Init: {          /* jump0 */
   int i;
 #ifndef SQLITE_OMIT_TRACE
   char *zTrace;
@@ -8744,14 +9144,29 @@ case OP_ReleaseReg: {
 
 /* Opcode: Noop * * * * *
 **
-** Do nothing.  This instruction is often useful as a jump
-** destination.
+** Do nothing.  Continue downward to the next opcode.
 */
-/*
-** The magic Explain opcode are only inserted when explain==2 (which
-** is to say when the EXPLAIN QUERY PLAN syntax is used.)
-** This opcode records information from the optimizer.  It is the
-** the same as a no-op.  This opcodesnever appears in a real VM program.
+/* Opcode: Explain P1 P2 P3 P4 *
+**
+** This is the same as OP_Noop during normal query execution.  The
+** purpose of this opcode is to hold information about the query
+** plan for the purpose of EXPLAIN QUERY PLAN output.
+**
+** The P4 value is human-readable text that describes the query plan
+** element.  Something like "SCAN t1" or "SEARCH t2 USING INDEX t2x1".
+**
+** The P1 value is the ID of the current element and P2 is the parent
+** element for the case of nested query plan elements.  If P2 is zero
+** then this element is a top-level element.
+**
+** For loop elements, P3 is the estimated code of each invocation of this
+** element.
+**
+** As with all opcodes, the meanings of the parameters for OP_Explain
+** are subject to change from one release to the next.  Applications
+** should not attempt to interpret or use any of the information
+** contained in the OP_Explain opcode.  The information provided by this
+** opcode is intended for testing and debugging use only.
 */
 default: {          /* This is really OP_Noop, OP_Explain */
   assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain );
@@ -8797,7 +9212,7 @@ default: {          /* This is really OP_Noop, OP_Explain */
       }
       if( opProperty==0xff ){
         /* Never happens.  This code exists to avoid a harmless linkage
-        ** warning aboud sqlite3VdbeRegisterDump() being defined but not
+        ** warning about sqlite3VdbeRegisterDump() being defined but not
         ** used. */
         sqlite3VdbeRegisterDump(p);
       }
@@ -8834,8 +9249,7 @@ default: {          /* This is really OP_Noop, OP_Explain */
   p->rc = rc;
   sqlite3SystemError(db, rc);
   testcase( sqlite3GlobalConfig.xLog!=0 );
-  sqlite3_log(rc, "statement aborts at %d: [%s] %s", 
-                   (int)(pOp - aOp), p->zSql, p->zErrMsg);
+  sqlite3VdbeLogAbort(p, rc, pOp, aOp);
   if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p);
   if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db);
   if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){
@@ -8876,8 +9290,8 @@ default: {          /* This is really OP_Noop, OP_Explain */
   if( DbMaskNonZero(p->lockMask) ){
     sqlite3VdbeLeave(p);
   }
-  assert( rc!=SQLITE_OK || nExtraDelete==0 
-       || sqlite3_strlike("DELETE%",p->zSql,0)!=0 
+  assert( rc!=SQLITE_OK || nExtraDelete==0
+       || sqlite3_strlike("DELETE%",p->zSql,0)!=0
   );
   return rc;
 
diff --git a/src/vdbe.h b/src/vdbe.h
index d28837f944..28df764bc6 100644
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -32,6 +32,20 @@ typedef struct Vdbe Vdbe;
 */
 typedef struct sqlite3_value Mem;
 typedef struct SubProgram SubProgram;
+typedef struct SubrtnSig SubrtnSig;
+
+/*
+** A signature for a reusable subroutine that materializes the RHS of
+** an IN operator.
+*/
+struct SubrtnSig {
+  int selId;          /* SELECT-id for the SELECT statement on the RHS */
+  u8 bComplete;       /* True if fully coded and available for reusable */
+  char *zAff;         /* Affinity of the overall IN expression */
+  int iTable;         /* Ephemeral table generated by the subroutine */
+  int iAddr;          /* Subroutine entry address */
+  int regReturn;      /* Register used to hold return address */
+};
 
 /*
 ** A single instruction of the virtual machine has an opcode
@@ -60,6 +74,7 @@ struct VdbeOp {
     u32 *ai;               /* Used when p4type is P4_INTARRAY */
     SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
     Table *pTab;           /* Used when p4type is P4_TABLE */
+    SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
     Expr *pExpr;           /* Used when p4type is P4_EXPR */
 #endif
@@ -126,6 +141,8 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_INT64      (-13) /* P4 is a 64-bit signed integer */
 #define P4_INTARRAY   (-14) /* P4 is a vector of 32-bit integers */
 #define P4_FUNCCTX    (-15) /* P4 is a pointer to an sqlite3_context object */
+#define P4_TABLEREF   (-16) /* Like P4_TABLE, but reference counted */
+#define P4_SUBRTNSIG  (-17) /* P4 is a SubrtnSig pointer */
 
 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1
@@ -169,7 +186,7 @@ typedef struct VdbeOpList VdbeOpList;
 ** Additional non-public SQLITE_PREPARE_* flags
 */
 #define SQLITE_PREPARE_SAVESQL  0x80  /* Preserve SQL text */
-#define SQLITE_PREPARE_MASK     0x0f  /* Mask of public flags */
+#define SQLITE_PREPARE_MASK     0x1f  /* Mask of public flags */
 
 /*
 ** Prototypes for the VDBE interface.  See comments on the implementation
@@ -283,8 +300,11 @@ void sqlite3VdbeSetVarmask(Vdbe*, int);
 #endif
 int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*);
 int sqlite3BlobCompare(const Mem*, const Mem*);
+#ifdef SQLITE_ENABLE_PERCENTILE
+  const char *sqlite3VdbeFuncName(const sqlite3_context*);
+#endif
 
-void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*);
+void sqlite3VdbeRecordUnpack(int,const void*,UnpackedRecord*);
 int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*);
 int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int);
 UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*);
@@ -295,13 +315,17 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*);
 void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *);
 int sqlite3VdbeHasSubProgram(Vdbe*);
 
+void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val);
+
+#ifndef SQLITE_OMIT_DATETIME_FUNCS
 int sqlite3NotPureFunc(sqlite3_context*);
+#endif
 #ifdef SQLITE_ENABLE_BYTECODE_VTAB
 int sqlite3VdbeBytecodeVtabInit(sqlite3*);
 #endif
 
-/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on
-** each VDBE opcode.
+/* Use SQLITE_ENABLE_EXPLAIN_COMMENTS to enable generation of extra
+** comments on each VDBE opcode.
 **
 ** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op
 ** comments in VDBE programs that show key decision points in the code
@@ -327,7 +351,7 @@ int sqlite3VdbeBytecodeVtabInit(sqlite3*);
 ** The VdbeCoverage macros are used to set a coverage testing point
 ** for VDBE branch instructions.  The coverage testing points are line
 ** numbers in the sqlite3.c source file.  VDBE branch coverage testing
-** only works with an amalagmation build.  That's ok since a VDBE branch
+** only works with an amalgamation build.  That's ok since a VDBE branch
 ** coverage build designed for testing the test suite only.  No application
 ** should ever ship with VDBE branch coverage measuring turned on.
 **
@@ -345,7 +369,7 @@ int sqlite3VdbeBytecodeVtabInit(sqlite3*);
 **                                     // NULL option is not possible
 **
 **    VdbeCoverageEqNe(v)              // Previous OP_Jump is only interested
-**                                     // in distingishing equal and not-equal.
+**                                     // in distinguishing equal and not-equal.
 **
 ** Every VDBE branch operation must be tagged with one of the macros above.
 ** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and
@@ -355,7 +379,7 @@ int sqlite3VdbeBytecodeVtabInit(sqlite3*);
 ** During testing, the test application will invoke
 ** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback
 ** routine that is invoked as each bytecode branch is taken.  The callback
-** contains the sqlite3.c source line number ov the VdbeCoverage macro and
+** contains the sqlite3.c source line number of the VdbeCoverage macro and
 ** flags to indicate whether or not the branch was taken.  The test application
 ** is responsible for keeping track of this and reporting byte-code branches
 ** that are never taken.
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index b901a01801..8b68c339af 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -56,6 +56,9 @@ typedef struct VdbeSorter VdbeSorter;
 /* Elements of the linked list at Vdbe.pAuxData */
 typedef struct AuxData AuxData;
 
+/* A cache of large TEXT or BLOB values in a VdbeCursor */
+typedef struct VdbeTxtBlbCache VdbeTxtBlbCache;
+
 /* Types of VDBE cursors */
 #define CURTYPE_BTREE       0
 #define CURTYPE_SORTER      1
@@ -87,6 +90,7 @@ struct VdbeCursor {
   Bool useRandomRowid:1;  /* Generate new record numbers semi-randomly */
   Bool isOrdered:1;       /* True if the table is not BTREE_UNORDERED */
   Bool noReuse:1;         /* OpenEphemeral may not reuse this cursor */
+  Bool colCache:1;        /* pCache pointer is initialized and non-NULL */
   u16 seekHit;            /* See the OP_SeekHit and OP_IfNoHope opcodes */
   union {                 /* pBtx for isEphermeral.  pAltMap otherwise */
     Btree *pBtx;            /* Separate file holding temporary table */
@@ -127,29 +131,50 @@ struct VdbeCursor {
 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
   u64 maskUsed;           /* Mask of columns used by this cursor */
 #endif
+  VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */
 
-  /* 2*nField extra array elements allocated for aType[], beyond the one
-  ** static element declared in the structure.  nField total array slots for
-  ** aType[] and nField+1 array slots for aOffset[] */
-  u32 aType[1];           /* Type values record decode.  MUST BE LAST */
+  /* Space is allocated for aType to hold at least 2*nField+1 entries:
+  ** nField slots for aType[] and nField+1 array slots for aOffset[] */
+  u32 aType[FLEXARRAY];    /* Type values record decode.  MUST BE LAST */
 };
 
+/*
+** The size (in bytes) of a VdbeCursor object that has an nField value of N
+** or less.  The value of SZ_VDBECURSOR(n) is guaranteed to be a multiple
+** of 8.
+*/
+#define SZ_VDBECURSOR(N) \
+    (ROUND8(offsetof(VdbeCursor,aType)) + ((N)+1)*sizeof(u64))
+
 /* Return true if P is a null-only cursor
 */
 #define IsNullCursor(P) \
   ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0)
 
-
 /*
 ** A value for VdbeCursor.cacheStatus that means the cache is always invalid.
 */
 #define CACHE_STALE 0
 
+/*
+** Large TEXT or BLOB values can be slow to load, so we want to avoid
+** loading them more than once.  For that reason, large TEXT and BLOB values
+** can be stored in a cache defined by this object, and attached to the
+** VdbeCursor using the pCache field.
+*/
+struct VdbeTxtBlbCache {
+  char *pCValue;        /* A RCStr buffer to hold the value */
+  i64 iOffset;          /* File offset of the row being cached */
+  int iCol;             /* Column for which the cache is valid */
+  u32 cacheStatus;      /* Vdbe.cacheCtr value */
+  u32 colCacheCtr;      /* Column cache counter */
+};
+
 /*
 ** When a sub-program is executed (OP_Program), a structure of this type
 ** is allocated to store the current value of the program counter, as
 ** well as the current memory cell array and various other frame specific
-** values stored in the Vdbe struct. When the sub-program is finished, 
+** values stored in the Vdbe struct. When the sub-program is finished,
 ** these values are copied back to the Vdbe from the VdbeFrame structure,
 ** restoring the state of the VM to as it was before the sub-program
 ** began executing.
@@ -226,6 +251,7 @@ struct sqlite3_value {
 #ifdef SQLITE_DEBUG
   Mem *pScopyFrom;    /* This Mem is a shallow copy of pScopyFrom */
   u16 mScopyFlags;    /* flags value immediately after the shallow copy */
+  u8  bScopy;         /* The pScopyFrom of some other Mem *might* point here */
 #endif
 };
 
@@ -262,8 +288,8 @@ struct sqlite3_value {
 **                             MEM_Int, MEM_Real, and MEM_IntReal.
 **
 **  *  MEM_Blob|MEM_Zero       A blob in Mem.z of length Mem.n plus
-**                             MEM.u.i extra 0x00 bytes at the end.
-**                                       
+**                             Mem.u.nZero extra 0x00 bytes at the end.
+**
 **  *  MEM_Int                 Integer stored in Mem.u.i.
 **
 **  *  MEM_Real                Real stored in Mem.u.r.
@@ -277,7 +303,7 @@ struct sqlite3_value {
 ** If the MEM_Str flag is set then Mem.z points at a string representation.
 ** Usually this is encoded in the same unicode encoding as the main
 ** database (see below for exceptions). If the MEM_Term flag is also
-** set, then the string is nul terminated. The MEM_Int and MEM_Real 
+** set, then the string is nul terminated. The MEM_Int and MEM_Real
 ** flags may coexist with the MEM_Str flag.
 */
 #define MEM_Undefined 0x0000   /* Value is undefined */
@@ -330,7 +356,7 @@ struct sqlite3_value {
 ** Return true if a memory cell has been initialized and is valid.
 ** is for use inside assert() statements only.
 **
-** A Memory cell is initialized if at least one of the 
+** A Memory cell is initialized if at least one of the
 ** MEM_Null, MEM_Str, MEM_Int, MEM_Real, MEM_Blob, or MEM_IntReal bits
 ** is set.  It is "undefined" if all those bits are zero.
 */
@@ -339,7 +365,7 @@ struct sqlite3_value {
 #endif
 
 /*
-** Each auxiliary data pointer stored by a user defined function 
+** Each auxiliary data pointer stored by a user defined function
 ** implementation calling sqlite3_set_auxdata() is stored in an instance
 ** of this structure. All such structures associated with a single VM
 ** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed
@@ -375,20 +401,23 @@ struct sqlite3_context {
   int isError;            /* Error code returned by the function. */
   u8 enc;                 /* Encoding to use for results */
   u8 skipFlag;            /* Skip accumulator loading if true */
-  u8 argc;                /* Number of arguments */
-  sqlite3_value *argv[1]; /* Argument set */
+  u16 argc;               /* Number of arguments */
+  sqlite3_value *argv[FLEXARRAY]; /* Argument set */
 };
 
-/* A bitfield type for use inside of structures.  Always follow with :N where
-** N is the number of bits.
+/*
+** The size (in bytes) of an sqlite3_context object that holds N
+** argv[] arguments.
 */
-typedef unsigned bft;  /* Bit Field Type */
+#define SZ_CONTEXT(N)  \
+   (offsetof(sqlite3_context,argv)+(N)*sizeof(sqlite3_value*))
+
 
 /* The ScanStatus object holds a single value for the
 ** sqlite3_stmt_scanstatus() interface.
 **
 ** aAddrRange[]:
-**   This array is used by ScanStatus elements associated with EQP 
+**   This array is used by ScanStatus elements associated with EQP
 **   notes that make an SQLITE_SCANSTAT_NCYCLE value available. It is
 **   an array of up to 3 ranges of VM addresses for which the Vdbe.anCycle[]
 **   values should be summed to calculate the NCYCLE value. Each pair of
@@ -443,7 +472,7 @@ struct Vdbe {
   i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
   i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   Mem *aMem;              /* The memory locations */
-  Mem **apArg;            /* Arguments to currently executing user function */
+  Mem **apArg;            /* Arguments xUpdate and xFilter vtab methods */
   VdbeCursor **apCsr;     /* One element of this array for each open cursor */
   Mem *aVar;              /* Values for the OP_Variable opcode. */
 
@@ -463,18 +492,21 @@ struct Vdbe {
 #ifdef SQLITE_DEBUG
   int rcApp;              /* errcode set by sqlite3_result_error_code() */
   u32 nWrite;             /* Number of write operations that have occurred */
+  int napArg;             /* Size of the apArg[] array */
 #endif
   u16 nResColumn;         /* Number of columns in one row of the result set */
+  u16 nResAlloc;          /* Column slots allocated to aColName[] */
   u8 errorAction;         /* Recovery action to do in case of an error */
   u8 minWriteFileFormat;  /* Minimum file format for writable database files */
   u8 prepFlags;           /* SQLITE_PREPARE_* flags */
   u8 eVdbeState;          /* On of the VDBE_*_STATE values */
   bft expired:2;          /* 1: recompile VM immediately  2: when convenient */
-  bft explain:2;          /* True if EXPLAIN present on SQL command */
+  bft explain:2;          /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */
   bft changeCntOn:1;      /* True to update the change-counter */
   bft usesStmtJournal:1;  /* True if uses a statement journal */
   bft readOnly:1;         /* True for statements that do not write */
   bft bIsReader:1;        /* True for statements that read */
+  bft haveEqpOps:1;       /* Bytecode supports EXPLAIN QUERY PLAN */
   yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   u32 aCounter[9];        /* Counters used by sqlite3_stmt_status() */
@@ -505,7 +537,7 @@ struct Vdbe {
 #define VDBE_HALT_STATE     3   /* Finished.  Need reset() or finalize() */
 
 /*
-** Structure used to store the context required by the 
+** Structure used to store the context required by the
 ** sqlite3_preupdate_*() API functions.
 */
 struct PreUpdate {
@@ -513,16 +545,21 @@ struct PreUpdate {
   VdbeCursor *pCsr;               /* Cursor to read old values from */
   int op;                         /* One of SQLITE_INSERT, UPDATE, DELETE */
   u8 *aRecord;                    /* old.* database record */
-  KeyInfo keyinfo;
+  KeyInfo *pKeyinfo;              /* Key information */
   UnpackedRecord *pUnpacked;      /* Unpacked version of aRecord[] */
   UnpackedRecord *pNewUnpacked;   /* Unpacked version of new.* record */
   int iNewReg;                    /* Register for new.* values */
   int iBlobWrite;                 /* Value returned by preupdate_blobwrite() */
   i64 iKey1;                      /* First key value passed to hook */
   i64 iKey2;                      /* Second key value passed to hook */
+  Mem oldipk;                     /* Memory cell holding "old" IPK value */
   Mem *aNew;                      /* Array of new.* values */
-  Table *pTab;                    /* Schema object being upated */          
+  Table *pTab;                    /* Schema object being updated */
   Index *pPk;                     /* PK index if pTab is WITHOUT ROWID */
+  sqlite3_value **apDflt;         /* Array of default values, if required */
+  struct {
+    u8 keyinfoSpace[SZ_KEYINFO_0];  /* Space to hold pKeyinfo[0] content */
+  } uKey;
 };
 
 /*
@@ -611,6 +648,7 @@ int sqlite3VdbeMemSetZeroBlob(Mem*,int);
 int sqlite3VdbeMemIsRowSet(const Mem*);
 #endif
 int sqlite3VdbeMemSetRowSet(Mem*);
+void sqlite3VdbeMemZeroTerminateIfAble(Mem*);
 int sqlite3VdbeMemMakeWriteable(Mem*);
 int sqlite3VdbeMemStringify(Mem*, u8, u8);
 int sqlite3IntFloatCompare(i64,double);
@@ -667,7 +705,7 @@ void sqlite3VdbeValueListFree(void*);
 # define sqlite3VdbeAssertAbortable(V)
 #endif
 
-#if !defined(SQLITE_OMIT_SHARED_CACHE) 
+#if !defined(SQLITE_OMIT_SHARED_CACHE)
   void sqlite3VdbeEnter(Vdbe*);
 #else
 # define sqlite3VdbeEnter(X)
@@ -685,9 +723,11 @@ int sqlite3VdbeCheckMemInvariants(Mem*);
 #endif
 
 #ifndef SQLITE_OMIT_FOREIGN_KEY
-int sqlite3VdbeCheckFk(Vdbe *, int);
+int sqlite3VdbeCheckFkImmediate(Vdbe*);
+int sqlite3VdbeCheckFkDeferred(Vdbe*);
 #else
-# define sqlite3VdbeCheckFk(p,i) 0
+# define sqlite3VdbeCheckFkImmediate(p) 0
+# define sqlite3VdbeCheckFkDeferred(p) 0
 #endif
 
 #ifdef SQLITE_DEBUG
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index d8fcda96df..1118481d1c 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -63,7 +63,6 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
   sqlite3_int64 iNow;
   sqlite3_int64 iElapse;
   assert( p->startTime>0 );
-  assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 );
   assert( db->init.busy==0 );
   assert( p->zSql!=0 );
   sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
@@ -152,7 +151,15 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){
   int rc = SQLITE_OK;
   Vdbe *p = (Vdbe*)pStmt;
 #if SQLITE_THREADSAFE
-  sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex;
+  sqlite3_mutex *mutex;
+#endif
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pStmt==0 ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+#if SQLITE_THREADSAFE
+  mutex = p->db->mutex;
 #endif
   sqlite3_mutex_enter(mutex);
   for(i=0; i<p->nVar; i++){
@@ -364,7 +371,7 @@ sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){
 void sqlite3_value_free(sqlite3_value *pOld){
   sqlite3ValueFree(pOld);
 }
-  
+ 
 
 /**************************** sqlite3_result_  *******************************
 ** The following routines are used by user-defined functions to specify
@@ -375,7 +382,7 @@ void sqlite3_value_free(sqlite3_value *pOld){
 ** is too big or if an OOM occurs.
 **
 ** The invokeValueDestructor(P,X) routine invokes destructor function X()
-** on value P is not going to be used and need to be destroyed.
+** on value P if P is not going to be used and need to be destroyed.
 */
 static void setResultStrOrError(
   sqlite3_context *pCtx,  /* Function context */
@@ -405,7 +412,7 @@ static void setResultStrOrError(
 static int invokeValueDestructor(
   const void *p,             /* Value to destroy */
   void (*xDel)(void*),       /* The destructor */
-  sqlite3_context *pCtx      /* Set a SQLITE_TOOBIG error if no NULL */
+  sqlite3_context *pCtx      /* Set a SQLITE_TOOBIG error if not NULL */
 ){
   assert( xDel!=SQLITE_DYNAMIC );
   if( xDel==0 ){
@@ -415,27 +422,46 @@ static int invokeValueDestructor(
   }else{
     xDel((void*)p);
   }
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx!=0 ){
+    sqlite3_result_error_toobig(pCtx);
+  }
+#else
+  assert( pCtx!=0 );
   sqlite3_result_error_toobig(pCtx);
+#endif
   return SQLITE_TOOBIG;
 }
 void sqlite3_result_blob(
-  sqlite3_context *pCtx, 
-  const void *z, 
-  int n, 
+  sqlite3_context *pCtx,
+  const void *z,
+  int n,
   void (*xDel)(void *)
 ){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 || n<0 ){
+    invokeValueDestructor(z, xDel, pCtx);
+    return;
+  }
+#endif
   assert( n>=0 );
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   setResultStrOrError(pCtx, z, n, 0, xDel);
 }
 void sqlite3_result_blob64(
-  sqlite3_context *pCtx, 
-  const void *z, 
+  sqlite3_context *pCtx,
+  const void *z,
   sqlite3_uint64 n,
   void (*xDel)(void *)
 ){
-  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   assert( xDel!=SQLITE_DYNAMIC );
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ){
+    invokeValueDestructor(z, xDel, 0);
+    return;
+  }
+#endif
+  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   if( n>0x7fffffff ){
     (void)invokeValueDestructor(z, xDel, pCtx);
   }else{
@@ -443,30 +469,48 @@ void sqlite3_result_blob64(
   }
 }
 void sqlite3_result_double(sqlite3_context *pCtx, double rVal){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetDouble(pCtx->pOut, rVal);
 }
 void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   pCtx->isError = SQLITE_ERROR;
   sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT);
 }
 #ifndef SQLITE_OMIT_UTF16
 void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   pCtx->isError = SQLITE_ERROR;
   sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT);
 }
 #endif
 void sqlite3_result_int(sqlite3_context *pCtx, int iVal){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal);
 }
 void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetInt64(pCtx->pOut, iVal);
 }
 void sqlite3_result_null(sqlite3_context *pCtx){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetNull(pCtx->pOut);
 }
@@ -476,34 +520,69 @@ void sqlite3_result_pointer(
   const char *zPType,
   void (*xDestructor)(void*)
 ){
-  Mem *pOut = pCtx->pOut;
+  Mem *pOut;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ){
+    invokeValueDestructor(pPtr, xDestructor, 0);
+    return;
+  }
+#endif
+  pOut = pCtx->pOut;
   assert( sqlite3_mutex_held(pOut->db->mutex) );
   sqlite3VdbeMemRelease(pOut);
   pOut->flags = MEM_Null;
   sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor);
 }
 void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){
-  Mem *pOut = pCtx->pOut;
+  Mem *pOut;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
+#if defined(SQLITE_STRICT_SUBTYPE) && SQLITE_STRICT_SUBTYPE+0!=0
+  if( pCtx->pFunc!=0
+   && (pCtx->pFunc->funcFlags & SQLITE_RESULT_SUBTYPE)==0
+  ){
+    char zErr[200];
+    sqlite3_snprintf(sizeof(zErr), zErr,
+                     "misuse of sqlite3_result_subtype() by %s()", 
+                     pCtx->pFunc->zName);
+    sqlite3_result_error(pCtx, zErr, -1);
+    return;
+  }
+#endif /* SQLITE_STRICT_SUBTYPE */
+  pOut = pCtx->pOut;
   assert( sqlite3_mutex_held(pOut->db->mutex) );
   pOut->eSubtype = eSubtype & 0xff;
   pOut->flags |= MEM_Subtype;
 }
 void sqlite3_result_text(
-  sqlite3_context *pCtx, 
-  const char *z, 
+  sqlite3_context *pCtx,
+  const char *z,
   int n,
   void (*xDel)(void *)
 ){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ){
+    invokeValueDestructor(z, xDel, 0);
+    return;
+  }
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel);
 }
 void sqlite3_result_text64(
-  sqlite3_context *pCtx, 
-  const char *z, 
+  sqlite3_context *pCtx,
+  const char *z,
   sqlite3_uint64 n,
   void (*xDel)(void *),
   unsigned char enc
 ){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ){
+    invokeValueDestructor(z, xDel, 0);
+    return;
+  }
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   assert( xDel!=SQLITE_DYNAMIC );
   if( enc!=SQLITE_UTF8 ){
@@ -514,31 +593,32 @@ void sqlite3_result_text64(
     (void)invokeValueDestructor(z, xDel, pCtx);
   }else{
     setResultStrOrError(pCtx, z, (int)n, enc, xDel);
+    sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut);
   }
 }
 #ifndef SQLITE_OMIT_UTF16
 void sqlite3_result_text16(
-  sqlite3_context *pCtx, 
-  const void *z, 
-  int n, 
+  sqlite3_context *pCtx,
+  const void *z,
+  int n,
   void (*xDel)(void *)
 ){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16NATIVE, xDel);
 }
 void sqlite3_result_text16be(
-  sqlite3_context *pCtx, 
-  const void *z, 
-  int n, 
+  sqlite3_context *pCtx,
+  const void *z,
+  int n,
   void (*xDel)(void *)
 ){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16BE, xDel);
 }
 void sqlite3_result_text16le(
-  sqlite3_context *pCtx, 
-  const void *z, 
-  int n, 
+  sqlite3_context *pCtx,
+  const void *z,
+  int n,
   void (*xDel)(void *)
 ){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
@@ -546,7 +626,16 @@ void sqlite3_result_text16le(
 }
 #endif /* SQLITE_OMIT_UTF16 */
 void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){
-  Mem *pOut = pCtx->pOut;
+  Mem *pOut;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+  if( pValue==0 ){
+    sqlite3_result_null(pCtx);
+    return;
+  }
+#endif
+  pOut = pCtx->pOut;
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemCopy(pOut, pValue);
   sqlite3VdbeChangeEncoding(pOut, pCtx->enc);
@@ -558,7 +647,12 @@ void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){
   sqlite3_result_zeroblob64(pCtx, n>0 ? n : 0);
 }
 int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){
-  Mem *pOut = pCtx->pOut;
+  Mem *pOut;
+
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+  pOut = pCtx->pOut;
   assert( sqlite3_mutex_held(pOut->db->mutex) );
   if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){
     sqlite3_result_error_toobig(pCtx);
@@ -572,6 +666,9 @@ int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){
 #endif
 }
 void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   pCtx->isError = errCode ? errCode : -1;
 #ifdef SQLITE_DEBUG
   if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode;
@@ -584,14 +681,20 @@ void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){
 
 /* Force an SQLITE_TOOBIG error. */
 void sqlite3_result_error_toobig(sqlite3_context *pCtx){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   pCtx->isError = SQLITE_TOOBIG;
-  sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, 
+  sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1,
                        SQLITE_UTF8, SQLITE_STATIC);
 }
 
 /* An SQLITE_NOMEM error. */
 void sqlite3_result_error_nomem(sqlite3_context *pCtx){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetNull(pCtx->pOut);
   pCtx->isError = SQLITE_NOMEM_BKPT;
@@ -603,7 +706,7 @@ void sqlite3_result_error_nomem(sqlite3_context *pCtx){
 ** a MEM_IntReal value.  See the SQLITE_TESTCTRL_RESULT_INTREAL
 ** test-control.
 */
-void sqlite3ResultIntReal(sqlite3_context *pCtx){ 
+void sqlite3ResultIntReal(sqlite3_context *pCtx){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   if( pCtx->pOut->flags & MEM_Int ){
     pCtx->pOut->flags &= ~MEM_Int;
@@ -614,7 +717,7 @@ void sqlite3ResultIntReal(sqlite3_context *pCtx){
 
 
 /*
-** This function is called after a transaction has been committed. It 
+** This function is called after a transaction has been committed. It
 ** invokes callbacks registered with sqlite3_wal_hook() as required.
 */
 static int doWalCallbacks(sqlite3 *db){
@@ -643,7 +746,7 @@ static int doWalCallbacks(sqlite3 *db){
 ** statement is completely executed or an error occurs.
 **
 ** This routine implements the bulk of the logic behind the sqlite_step()
-** API.  The only thing omitted is the automatic recompile if a 
+** API.  The only thing omitted is the automatic recompile if a
 ** schema change has occurred.  That detail is handled by the
 ** outer sqlite3_step() wrapper procedure.
 */
@@ -660,11 +763,11 @@ static int sqlite3Step(Vdbe *p){
         p->rc = SQLITE_SCHEMA;
         rc = SQLITE_ERROR;
         if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){
-          /* If this statement was prepared using saved SQL and an 
+          /* If this statement was prepared using saved SQL and an
           ** error has occurred, then return the error code in p->rc to the
           ** caller. Set the error code in the database handle to the same
           ** value.
-          */ 
+          */
           rc = sqlite3VdbeTransferError(p);
         }
         goto end_of_step;
@@ -678,8 +781,8 @@ static int sqlite3Step(Vdbe *p){
         AtomicStore(&db->u1.isInterrupted, 0);
       }
 
-      assert( db->nVdbeWrite>0 || db->autoCommit==0 
-          || (db->nDeferredCons==0 && db->nDeferredImmCons==0)
+      assert( db->nVdbeWrite>0 || db->autoCommit==0
+          || ((db->nDeferredCons + db->nDeferredImmCons)==0)
       );
 
 #ifndef SQLITE_OMIT_TRACE
@@ -703,15 +806,15 @@ static int sqlite3Step(Vdbe *p){
       ** sqlite3_step() after any error or after SQLITE_DONE.  But beginning
       ** with version 3.7.0, we changed this so that sqlite3_reset() would
       ** be called automatically instead of throwing the SQLITE_MISUSE error.
-      ** This "automatic-reset" change is not technically an incompatibility, 
+      ** This "automatic-reset" change is not technically an incompatibility,
       ** since any application that receives an SQLITE_MISUSE is broken by
       ** definition.
       **
       ** Nevertheless, some published applications that were originally written
-      ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE 
+      ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE
       ** returns, and those were broken by the automatic-reset change.  As a
       ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the
-      ** legacy behavior of returning SQLITE_MISUSE for cases where the 
+      ** legacy behavior of returning SQLITE_MISUSE for cases where the
       ** previous sqlite3_step() returned something other than a SQLITE_LOCKED
       ** or SQLITE_BUSY error.
       */
@@ -761,10 +864,10 @@ static int sqlite3Step(Vdbe *p){
         rc = SQLITE_ERROR;
       }
     }else if( rc!=SQLITE_DONE && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){
-      /* If this statement was prepared using saved SQL and an 
+      /* If this statement was prepared using saved SQL and an
       ** error has occurred, then return the error code in p->rc to the
       ** caller. Set the error code in the database handle to the same value.
-      */ 
+      */
       rc = sqlite3VdbeTransferError(p);
     }
   }
@@ -778,7 +881,7 @@ static int sqlite3Step(Vdbe *p){
   /* There are only a limited number of result codes allowed from the
   ** statements prepared using the legacy sqlite3_prepare() interface */
   assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0
-       || rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR 
+       || rc==SQLITE_ROW  || rc==SQLITE_DONE   || rc==SQLITE_ERROR
        || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE
   );
   return (rc&db->errMask);
@@ -805,15 +908,15 @@ int sqlite3_step(sqlite3_stmt *pStmt){
     int savedPc = v->pc;
     rc = sqlite3Reprepare(v);
     if( rc!=SQLITE_OK ){
-      /* This case occurs after failing to recompile an sql statement. 
-      ** The error message from the SQL compiler has already been loaded 
-      ** into the database handle. This block copies the error message 
+      /* This case occurs after failing to recompile an sql statement.
+      ** The error message from the SQL compiler has already been loaded
+      ** into the database handle. This block copies the error message
       ** from the database handle into the statement and sets the statement
-      ** program counter to 0 to ensure that when the statement is 
+      ** program counter to 0 to ensure that when the statement is
       ** finalized or reset the parser error message is available via
       ** sqlite3_errmsg() and sqlite3_errcode().
       */
-      const char *zErr = (const char *)sqlite3_value_text(db->pErr); 
+      const char *zErr = (const char *)sqlite3_value_text(db->pErr);
       sqlite3DbFree(db, v->zErrMsg);
       if( !db->mallocFailed ){
         v->zErrMsg = sqlite3DbStrDup(db, zErr);
@@ -844,6 +947,9 @@ int sqlite3_step(sqlite3_stmt *pStmt){
 ** pointer to it.
 */
 void *sqlite3_user_data(sqlite3_context *p){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( p==0 ) return 0;
+#endif
   assert( p && p->pFunc );
   return p->pFunc->pUserData;
 }
@@ -859,7 +965,11 @@ void *sqlite3_user_data(sqlite3_context *p){
 ** application defined function.
 */
 sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( p==0 ) return 0;
+#else
   assert( p && p->pOut );
+#endif
   return p->pOut->db;
 }
 
@@ -878,7 +988,11 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){
 ** value, as a signal to the xUpdate routine that the column is unchanged.
 */
 int sqlite3_vtab_nochange(sqlite3_context *p){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( p==0 ) return 0;
+#else
   assert( p );
+#endif
   return sqlite3_value_nochange(p->pOut);
 }
 
@@ -886,7 +1000,7 @@ int sqlite3_vtab_nochange(sqlite3_context *p){
 ** The destructor function for a ValueList object.  This needs to be
 ** a separate function, unknowable to the application, to ensure that
 ** calls to sqlite3_vtab_in_first()/sqlite3_vtab_in_next() that are not
-** preceeded by activation of IN processing via sqlite3_vtab_int() do not
+** preceded by activation of IN processing via sqlite3_vtab_int() do not
 ** try to access a fake ValueList object inserted by a hostile extension.
 */
 void sqlite3VdbeValueListFree(void *pToDelete){
@@ -906,7 +1020,7 @@ static int valueFromValueList(
   ValueList *pRhs;
 
   *ppOut = 0;
-  if( pVal==0 ) return SQLITE_MISUSE;
+  if( pVal==0 ) return SQLITE_MISUSE_BKPT;
   if( (pVal->flags & MEM_Dyn)==0 || pVal->xDel!=sqlite3VdbeValueListFree ){
     return SQLITE_ERROR;
   }else{
@@ -1037,6 +1151,9 @@ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){
 void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){
   AuxData *pAuxData;
 
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return 0;
+#endif
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
 #if SQLITE_ENABLE_STAT4
   if( pCtx->pVdbe==0 ) return 0;
@@ -1063,14 +1180,18 @@ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){
 ** access code.
 */
 void sqlite3_set_auxdata(
-  sqlite3_context *pCtx, 
-  int iArg, 
-  void *pAux, 
+  sqlite3_context *pCtx,
+  int iArg,
+  void *pAux,
   void (*xDelete)(void*)
 ){
   AuxData *pAuxData;
-  Vdbe *pVdbe = pCtx->pVdbe;
+  Vdbe *pVdbe;
 
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pCtx==0 ) return;
+#endif
+  pVdbe= pCtx->pVdbe;
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
 #ifdef SQLITE_ENABLE_STAT4
   if( pVdbe==0 ) goto failed;
@@ -1107,7 +1228,7 @@ void sqlite3_set_auxdata(
 
 #ifndef SQLITE_OMIT_DEPRECATED
 /*
-** Return the number of times the Step function of an aggregate has been 
+** Return the number of times the Step function of an aggregate has been
 ** called.
 **
 ** This function is deprecated.  Do not use it for new code.  It is
@@ -1126,7 +1247,8 @@ int sqlite3_aggregate_count(sqlite3_context *p){
 */
 int sqlite3_column_count(sqlite3_stmt *pStmt){
   Vdbe *pVm = (Vdbe *)pStmt;
-  return pVm ? pVm->nResColumn : 0;
+  if( pVm==0 ) return 0;
+  return pVm->nResColumn;
 }
 
 /*
@@ -1152,9 +1274,9 @@ static const Mem *columnNullValue(void){
   ** these assert()s from failing, when building with SQLITE_DEBUG defined
   ** using gcc, we force nullMem to be 8-byte aligned using the magical
   ** __attribute__((aligned(8))) macro.  */
-  static const Mem nullMem 
+  static const Mem nullMem
 #if defined(SQLITE_DEBUG) && defined(__GNUC__)
-    __attribute__((aligned(8))) 
+    __attribute__((aligned(8)))
 #endif
     = {
         /* .u          = */ {0},
@@ -1171,6 +1293,7 @@ static const Mem *columnNullValue(void){
 #ifdef SQLITE_DEBUG
         /* .pScopyFrom = */ (Mem*)0,
         /* .mScopyFlags= */ 0,
+        /* .bScopy     = */ 0,
 #endif
       };
   return &nullMem;
@@ -1200,9 +1323,9 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){
 }
 
 /*
-** This function is called after invoking an sqlite3_value_XXX function on a 
+** This function is called after invoking an sqlite3_value_XXX function on a
 ** column value (i.e. a value returned by evaluating an SQL expression in the
-** select list of a SELECT statement) that may cause a malloc() failure. If 
+** select list of a SELECT statement) that may cause a malloc() failure. If
 ** malloc() has failed, the threads mallocFailed flag is cleared and the result
 ** code of statement pStmt set to SQLITE_NOMEM.
 **
@@ -1212,10 +1335,10 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){
 **     sqlite3_column_int64()
 **     sqlite3_column_text()
 **     sqlite3_column_text16()
-**     sqlite3_column_real()
+**     sqlite3_column_double()
 **     sqlite3_column_bytes()
 **     sqlite3_column_bytes16()
-**     sqiite3_column_blob()
+**     sqlite3_column_blob()
 */
 static void columnMallocFailure(sqlite3_stmt *pStmt)
 {
@@ -1241,8 +1364,8 @@ const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){
   const void *val;
   val = sqlite3_value_blob( columnMem(pStmt,i) );
   /* Even though there is no encoding conversion, value_blob() might
-  ** need to call malloc() to expand the result of a zeroblob() 
-  ** expression. 
+  ** need to call malloc() to expand the result of a zeroblob()
+  ** expression.
   */
   columnMallocFailure(pStmt);
   return val;
@@ -1299,6 +1422,32 @@ int sqlite3_column_type(sqlite3_stmt *pStmt, int i){
   return iType;
 }
 
+/*
+** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN.
+*/
+static const char * const azExplainColNames8[] = {
+   "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",  /* EXPLAIN */
+   "id", "parent", "notused", "detail"                         /* EQP */
+};
+static const u16 azExplainColNames16data[] = {
+  /*   0 */  'a', 'd', 'd', 'r',                0,
+  /*   5 */  'o', 'p', 'c', 'o', 'd', 'e',      0,
+  /*  12 */  'p', '1',                          0, 
+  /*  15 */  'p', '2',                          0,
+  /*  18 */  'p', '3',                          0,
+  /*  21 */  'p', '4',                          0,
+  /*  24 */  'p', '5',                          0,
+  /*  27 */  'c', 'o', 'm', 'm', 'e', 'n', 't', 0,
+  /*  35 */  'i', 'd',                          0,
+  /*  38 */  'p', 'a', 'r', 'e', 'n', 't',      0,
+  /*  45 */  'n', 'o', 't', 'u', 's', 'e', 'd', 0,
+  /*  53 */  'd', 'e', 't', 'a', 'i', 'l',      0
+};
+static const u8 iExplainColNames16[] = {
+  0, 5, 12, 15, 18, 21, 24, 27,
+  35, 38, 45, 53
+};
+
 /*
 ** Convert the N-th element of pStmt->pColName[] into a string using
 ** xFunc() then return that string.  If N is out of range, return 0.
@@ -1331,15 +1480,29 @@ static const void *columnName(
     return 0;
   }
 #endif
+  if( N<0 ) return 0;
   ret = 0;
   p = (Vdbe *)pStmt;
   db = p->db;
   assert( db!=0 );
-  n = sqlite3_column_count(pStmt);
-  if( N<n && N>=0 ){
+  sqlite3_mutex_enter(db->mutex);
+
+  if( p->explain ){
+    if( useType>0 ) goto columnName_end;
+    n = p->explain==1 ? 8 : 4;
+    if( N>=n ) goto columnName_end;
+    if( useUtf16 ){
+      int i = iExplainColNames16[N + 8*p->explain - 8];
+      ret = (void*)&azExplainColNames16data[i];
+    }else{
+      ret = (void*)azExplainColNames8[N + 8*p->explain - 8];
+    }
+    goto columnName_end;
+  }
+  n = p->nResColumn;
+  if( N<n ){
     u8 prior_mallocFailed = db->mallocFailed;
     N += useType*n;
-    sqlite3_mutex_enter(db->mutex);
 #ifndef SQLITE_OMIT_UTF16
     if( useUtf16 ){
       ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]);
@@ -1356,8 +1519,9 @@ static const void *columnName(
       sqlite3OomClear(db);
       ret = 0;
     }
-    sqlite3_mutex_leave(db->mutex);
   }
+columnName_end:
+  sqlite3_mutex_leave(db->mutex);
   return ret;
 }
 
@@ -1444,19 +1608,30 @@ const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){
 
 
 /******************************* sqlite3_bind_  ***************************
-** 
+**
 ** Routines used to attach values to wildcards in a compiled SQL statement.
 */
 /*
-** Unbind the value bound to variable i in virtual machine p. This is the 
+** Unbind the value bound to variable i in virtual machine p. This is the
 ** the same as binding a NULL value to the column. If the "i" parameter is
-** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK.
+** out of range, then SQLITE_RANGE is returned. Otherwise SQLITE_OK.
 **
 ** A successful evaluation of this routine acquires the mutex on p.
 ** the mutex is released if any kind of error occurs.
 **
 ** The error code stored in database p->db is overwritten with the return
 ** value in any case.
+**
+** (tag-20240917-01) If  vdbeUnbind(p,(u32)(i-1))  returns SQLITE_OK,
+** that means all of the the following will be true:
+**
+**     p!=0
+**     p->pVar!=0
+**     i>0
+**     i<=p->nVar
+**
+** An assert() is normally added after vdbeUnbind() to help static analyzers
+** realize this.
 */
 static int vdbeUnbind(Vdbe *p, unsigned int i){
   Mem *pVar;
@@ -1465,9 +1640,9 @@ static int vdbeUnbind(Vdbe *p, unsigned int i){
   }
   sqlite3_mutex_enter(p->db->mutex);
   if( p->eVdbeState!=VDBE_READY_STATE ){
-    sqlite3Error(p->db, SQLITE_MISUSE);
+    sqlite3Error(p->db, SQLITE_MISUSE_BKPT);
     sqlite3_mutex_leave(p->db->mutex);
-    sqlite3_log(SQLITE_MISUSE, 
+    sqlite3_log(SQLITE_MISUSE,
         "bind on a busy prepared statement: [%s]", p->zSql);
     return SQLITE_MISUSE_BKPT;
   }
@@ -1481,7 +1656,7 @@ static int vdbeUnbind(Vdbe *p, unsigned int i){
   pVar->flags = MEM_Null;
   p->db->errCode = SQLITE_OK;
 
-  /* If the bit corresponding to this variable in Vdbe.expmask is set, then 
+  /* If the bit corresponding to this variable in Vdbe.expmask is set, then
   ** binding a new value to this variable invalidates the current query plan.
   **
   ** IMPLEMENTATION-OF: R-57496-20354 If the specific value bound to a host
@@ -1514,11 +1689,16 @@ static int bindText(
 
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     if( zData!=0 ){
       pVar = &p->aVar[i-1];
       rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel);
-      if( rc==SQLITE_OK && encoding!=0 ){
-        rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db));
+      if( rc==SQLITE_OK ){
+        if( encoding==0 ){
+          pVar->enc = ENC(p->db);
+        }else{
+          rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db));
+        }
       }
       if( rc ){
         sqlite3Error(p->db, rc);
@@ -1537,10 +1717,10 @@ static int bindText(
 ** Bind a blob value to an SQL statement variable.
 */
 int sqlite3_bind_blob(
-  sqlite3_stmt *pStmt, 
-  int i, 
-  const void *zData, 
-  int nData, 
+  sqlite3_stmt *pStmt,
+  int i,
+  const void *zData,
+  int nData,
   void (*xDel)(void*)
 ){
 #ifdef SQLITE_ENABLE_API_ARMOR
@@ -1549,10 +1729,10 @@ int sqlite3_bind_blob(
   return bindText(pStmt, i, zData, nData, xDel, 0);
 }
 int sqlite3_bind_blob64(
-  sqlite3_stmt *pStmt, 
-  int i, 
-  const void *zData, 
-  sqlite3_uint64 nData, 
+  sqlite3_stmt *pStmt,
+  int i,
+  const void *zData,
+  sqlite3_uint64 nData,
   void (*xDel)(void*)
 ){
   assert( xDel!=SQLITE_DYNAMIC );
@@ -1563,6 +1743,7 @@ int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){
   Vdbe *p = (Vdbe *)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue);
     sqlite3_mutex_leave(p->db->mutex);
   }
@@ -1576,6 +1757,7 @@ int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){
   Vdbe *p = (Vdbe *)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue);
     sqlite3_mutex_leave(p->db->mutex);
   }
@@ -1586,6 +1768,7 @@ int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){
   Vdbe *p = (Vdbe*)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3_mutex_leave(p->db->mutex);
   }
   return rc;
@@ -1601,6 +1784,7 @@ int sqlite3_bind_pointer(
   Vdbe *p = (Vdbe*)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
     sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor);
     sqlite3_mutex_leave(p->db->mutex);
   }else if( xDestructor ){
@@ -1608,36 +1792,36 @@ int sqlite3_bind_pointer(
   }
   return rc;
 }
-int sqlite3_bind_text( 
-  sqlite3_stmt *pStmt, 
-  int i, 
-  const char *zData, 
-  int nData, 
+int sqlite3_bind_text(
+  sqlite3_stmt *pStmt,
+  int i,
+  const char *zData,
+  int nData,
   void (*xDel)(void*)
 ){
   return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8);
 }
-int sqlite3_bind_text64( 
-  sqlite3_stmt *pStmt, 
-  int i, 
-  const char *zData, 
-  sqlite3_uint64 nData, 
+int sqlite3_bind_text64(
+  sqlite3_stmt *pStmt,
+  int i,
+  const char *zData,
+  sqlite3_uint64 nData,
   void (*xDel)(void*),
   unsigned char enc
 ){
   assert( xDel!=SQLITE_DYNAMIC );
   if( enc!=SQLITE_UTF8 ){
     if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE;
-    nData &= ~(u16)1;
+    nData &= ~(u64)1;
   }
   return bindText(pStmt, i, zData, nData, xDel, enc);
 }
 #ifndef SQLITE_OMIT_UTF16
 int sqlite3_bind_text16(
-  sqlite3_stmt *pStmt, 
-  int i, 
-  const void *zData, 
-  int n, 
+  sqlite3_stmt *pStmt,
+  int i,
+  const void *zData,
+  int n,
   void (*xDel)(void*)
 ){
   return bindText(pStmt, i, zData, n & ~(u64)1, xDel, SQLITE_UTF16NATIVE);
@@ -1652,7 +1836,7 @@ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){
     }
     case SQLITE_FLOAT: {
       assert( pValue->flags & (MEM_Real|MEM_IntReal) );
-      rc = sqlite3_bind_double(pStmt, i, 
+      rc = sqlite3_bind_double(pStmt, i,
           (pValue->flags & MEM_Real) ? pValue->u.r : (double)pValue->u.i
       );
       break;
@@ -1682,6 +1866,7 @@ int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){
   Vdbe *p = (Vdbe *)pStmt;
   rc = vdbeUnbind(p, (u32)(i-1));
   if( rc==SQLITE_OK ){
+    assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */
 #ifndef SQLITE_OMIT_INCRBLOB
     sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n);
 #else
@@ -1694,6 +1879,9 @@ int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){
 int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){
   int rc;
   Vdbe *p = (Vdbe *)pStmt;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( p==0 ) return SQLITE_MISUSE_BKPT;
+#endif
   sqlite3_mutex_enter(p->db->mutex);
   if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){
     rc = SQLITE_TOOBIG;
@@ -1708,7 +1896,7 @@ int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){
 
 /*
 ** Return the number of wildcards that can be potentially bound to.
-** This routine is added to support DBD::SQLite.  
+** This routine is added to support DBD::SQLite. 
 */
 int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){
   Vdbe *p = (Vdbe*)pStmt;
@@ -1814,6 +2002,42 @@ int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){
   return pStmt ? ((Vdbe*)pStmt)->explain : 0;
 }
 
+/*
+** Set the explain mode for a statement.
+*/
+int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){
+  Vdbe *v = (Vdbe*)pStmt;
+  int rc;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( pStmt==0 ) return SQLITE_MISUSE_BKPT;
+#endif
+  sqlite3_mutex_enter(v->db->mutex);
+  if( ((int)v->explain)==eMode ){
+    rc = SQLITE_OK;
+  }else if( eMode<0 || eMode>2 ){
+    rc = SQLITE_ERROR;
+  }else if( (v->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){
+    rc = SQLITE_ERROR;
+  }else if( v->eVdbeState!=VDBE_READY_STATE ){
+    rc = SQLITE_BUSY;
+  }else if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){
+    /* No reprepare necessary */
+    v->explain = eMode;
+    rc = SQLITE_OK;
+  }else{
+    v->explain = eMode;
+    rc = sqlite3Reprepare(v);
+    v->haveEqpOps = eMode==2;
+  }
+  if( v->explain ){
+    v->nResColumn = 12 - 4*v->explain;
+  }else{
+    v->nResColumn = v->nResAlloc;
+  }
+  sqlite3_mutex_leave(v->db->mutex);
+  return rc;
+}
+
 /*
 ** Return true if the prepared statement is in need of being reset.
 */
@@ -1853,7 +2077,7 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){
   Vdbe *pVdbe = (Vdbe*)pStmt;
   u32 v;
 #ifdef SQLITE_ENABLE_API_ARMOR
-  if( !pStmt 
+  if( !pStmt
    || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter)))
   ){
     (void)SQLITE_MISUSE_BKPT;
@@ -1934,8 +2158,8 @@ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
 ** if successful, or a NULL pointer if an OOM error is encountered.
 */
 static UnpackedRecord *vdbeUnpackRecord(
-  KeyInfo *pKeyInfo, 
-  int nKey, 
+  KeyInfo *pKeyInfo,
+  int nKey,
   const void *pKey
 ){
   UnpackedRecord *pRet;           /* Return value */
@@ -1943,7 +2167,7 @@ static UnpackedRecord *vdbeUnpackRecord(
   pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo);
   if( pRet ){
     memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1));
-    sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet);
+    sqlite3VdbeRecordUnpack(nKey, pKey, pRet);
   }
   return pRet;
 }
@@ -1953,10 +2177,17 @@ static UnpackedRecord *vdbeUnpackRecord(
 ** a field of the row currently being updated or deleted.
 */
 int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
-  PreUpdate *p = db->pPreUpdate;
+  PreUpdate *p;
   Mem *pMem;
   int rc = SQLITE_OK;
+  int iStore = 0;
 
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( db==0 || ppValue==0 ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+  p = db->pPreUpdate;
   /* Test that this call is being made from within an SQLITE_DELETE or
   ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */
   if( !p || p->op==SQLITE_INSERT ){
@@ -1964,44 +2195,78 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
     goto preupdate_old_out;
   }
   if( p->pPk ){
-    iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx);
+    iStore = sqlite3TableColumnToIndex(p->pPk, iIdx);
+  }else if( iIdx >= p->pTab->nCol ){
+    rc = SQLITE_MISUSE_BKPT;
+    goto preupdate_old_out;
+  }else{
+    iStore = sqlite3TableColumnToStorage(p->pTab, iIdx);
   }
-  if( iIdx>=p->pCsr->nField || iIdx<0 ){
+  if( iStore>=p->pCsr->nField || iStore<0 ){
     rc = SQLITE_RANGE;
     goto preupdate_old_out;
   }
 
-  /* If the old.* record has not yet been loaded into memory, do so now. */
-  if( p->pUnpacked==0 ){
-    u32 nRec;
-    u8 *aRec;
+  if( iIdx==p->pTab->iPKey ){
+    *ppValue = pMem = &p->oldipk;
+    sqlite3VdbeMemSetInt64(pMem, p->iKey1);
+  }else{
 
-    assert( p->pCsr->eCurType==CURTYPE_BTREE );
-    nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
-    aRec = sqlite3DbMallocRaw(db, nRec);
-    if( !aRec ) goto preupdate_old_out;
-    rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec);
-    if( rc==SQLITE_OK ){
-      p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec);
-      if( !p->pUnpacked ) rc = SQLITE_NOMEM;
-    }
-    if( rc!=SQLITE_OK ){
-      sqlite3DbFree(db, aRec);
-      goto preupdate_old_out;
+    /* If the old.* record has not yet been loaded into memory, do so now. */
+    if( p->pUnpacked==0 ){
+      u32 nRec;
+      u8 *aRec;
+
+      assert( p->pCsr->eCurType==CURTYPE_BTREE );
+      nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
+      aRec = sqlite3DbMallocRaw(db, nRec);
+      if( !aRec ) goto preupdate_old_out;
+      rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec);
+      if( rc==SQLITE_OK ){
+        p->pUnpacked = vdbeUnpackRecord(p->pKeyinfo, nRec, aRec);
+        if( !p->pUnpacked ) rc = SQLITE_NOMEM;
+      }
+      if( rc!=SQLITE_OK ){
+        sqlite3DbFree(db, aRec);
+        goto preupdate_old_out;
+      }
+      p->aRecord = aRec;
     }
-    p->aRecord = aRec;
-  }
 
-  pMem = *ppValue = &p->pUnpacked->aMem[iIdx];
-  if( iIdx==p->pTab->iPKey ){
-    sqlite3VdbeMemSetInt64(pMem, p->iKey1);
-  }else if( iIdx>=p->pUnpacked->nField ){
-    *ppValue = (sqlite3_value *)columnNullValue();
-  }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
-    if( pMem->flags & (MEM_Int|MEM_IntReal) ){
-      testcase( pMem->flags & MEM_Int );
-      testcase( pMem->flags & MEM_IntReal );
-      sqlite3VdbeMemRealify(pMem);
+    pMem = *ppValue = &p->pUnpacked->aMem[iStore];
+    if( iStore>=p->pUnpacked->nField ){
+      /* This occurs when the table has been extended using ALTER TABLE
+      ** ADD COLUMN. The value to return is the default value of the column. */
+      Column *pCol = &p->pTab->aCol[iIdx];
+      if( pCol->iDflt>0 ){
+        if( p->apDflt==0 ){
+          int nByte;
+          assert( sizeof(sqlite3_value*)*UMXV(p->pTab->nCol) < 0x7fffffff );
+          nByte = sizeof(sqlite3_value*)*p->pTab->nCol;
+          p->apDflt = (sqlite3_value**)sqlite3DbMallocZero(db, nByte);
+          if( p->apDflt==0 ) goto preupdate_old_out;
+        }
+        if( p->apDflt[iIdx]==0 ){
+          sqlite3_value *pVal = 0;
+          Expr *pDflt;
+          assert( p->pTab!=0 && IsOrdinaryTable(p->pTab) );
+          pDflt = p->pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr;
+          rc = sqlite3ValueFromExpr(db, pDflt, ENC(db), pCol->affinity, &pVal);
+          if( rc==SQLITE_OK && pVal==0 ){
+            rc = SQLITE_CORRUPT_BKPT;
+          }
+          p->apDflt[iIdx] = pVal;
+        }
+        *ppValue = p->apDflt[iIdx];
+      }else{
+        *ppValue = (sqlite3_value *)columnNullValue();
+      }
+    }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
+      if( pMem->flags & (MEM_Int|MEM_IntReal) ){
+        testcase( pMem->flags & MEM_Int );
+        testcase( pMem->flags & MEM_IntReal );
+        sqlite3VdbeMemRealify(pMem);
+      }
     }
   }
 
@@ -2017,8 +2282,13 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
 ** the number of columns in the row being updated, deleted or inserted.
 */
 int sqlite3_preupdate_count(sqlite3 *db){
-  PreUpdate *p = db->pPreUpdate;
-  return (p ? p->keyinfo.nKeyField : 0);
+  PreUpdate *p;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  p = db!=0 ? db->pPreUpdate : 0;
+#else
+  p = db->pPreUpdate;
+#endif
+  return (p ? p->pKeyinfo->nKeyField : 0);
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 
@@ -2028,14 +2298,19 @@ int sqlite3_preupdate_count(sqlite3 *db){
 ** only. It returns zero if the change that caused the callback was made
 ** immediately by a user SQL statement. Or, if the change was made by a
 ** trigger program, it returns the number of trigger programs currently
-** on the stack (1 for a top-level trigger, 2 for a trigger fired by a 
+** on the stack (1 for a top-level trigger, 2 for a trigger fired by a
 ** top-level trigger etc.).
 **
 ** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL
 ** or SET DEFAULT action is considered a trigger.
 */
 int sqlite3_preupdate_depth(sqlite3 *db){
-  PreUpdate *p = db->pPreUpdate;
+  PreUpdate *p;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  p = db!=0 ? db->pPreUpdate : 0;
+#else
+  p = db->pPreUpdate;
+#endif
   return (p ? p->v->nFrame : 0);
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
@@ -2043,10 +2318,15 @@ int sqlite3_preupdate_depth(sqlite3 *db){
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
 /*
 ** This function is designed to be called from within a pre-update callback
-** only. 
+** only.
 */
 int sqlite3_preupdate_blobwrite(sqlite3 *db){
-  PreUpdate *p = db->pPreUpdate;
+  PreUpdate *p;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  p = db!=0 ? db->pPreUpdate : 0;
+#else
+  p = db->pPreUpdate;
+#endif
   return (p ? p->iBlobWrite : -1);
 }
 #endif
@@ -2057,18 +2337,30 @@ int sqlite3_preupdate_blobwrite(sqlite3 *db){
 ** a field of the row currently being updated or inserted.
 */
 int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
-  PreUpdate *p = db->pPreUpdate;
+  PreUpdate *p;
   int rc = SQLITE_OK;
   Mem *pMem;
+  int iStore = 0;
 
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( db==0 || ppValue==0 ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+  p = db->pPreUpdate;
   if( !p || p->op==SQLITE_DELETE ){
     rc = SQLITE_MISUSE_BKPT;
     goto preupdate_new_out;
   }
   if( p->pPk && p->op!=SQLITE_UPDATE ){
-    iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx);
+    iStore = sqlite3TableColumnToIndex(p->pPk, iIdx);
+  }else if( iIdx >= p->pTab->nCol ){
+    return SQLITE_MISUSE_BKPT;
+  }else{
+    iStore = sqlite3TableColumnToStorage(p->pTab, iIdx);
   }
-  if( iIdx>=p->pCsr->nField || iIdx<0 ){
+
+  if( iStore>=p->pCsr->nField || iStore<0 ){
     rc = SQLITE_RANGE;
     goto preupdate_new_out;
   }
@@ -2081,40 +2373,41 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
       Mem *pData = &p->v->aMem[p->iNewReg];
       rc = ExpandBlob(pData);
       if( rc!=SQLITE_OK ) goto preupdate_new_out;
-      pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z);
+      pUnpack = vdbeUnpackRecord(p->pKeyinfo, pData->n, pData->z);
       if( !pUnpack ){
         rc = SQLITE_NOMEM;
         goto preupdate_new_out;
       }
       p->pNewUnpacked = pUnpack;
     }
-    pMem = &pUnpack->aMem[iIdx];
+    pMem = &pUnpack->aMem[iStore];
     if( iIdx==p->pTab->iPKey ){
       sqlite3VdbeMemSetInt64(pMem, p->iKey2);
-    }else if( iIdx>=pUnpack->nField ){
+    }else if( iStore>=pUnpack->nField ){
       pMem = (sqlite3_value *)columnNullValue();
     }
   }else{
-    /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required
+    /* For an UPDATE, memory cell (p->iNewReg+1+iStore) contains the required
     ** value. Make a copy of the cell contents and return a pointer to it.
     ** It is not safe to return a pointer to the memory cell itself as the
     ** caller may modify the value text encoding.
     */
     assert( p->op==SQLITE_UPDATE );
     if( !p->aNew ){
-      p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField);
+      assert( sizeof(Mem)*UMXV(p->pCsr->nField) < 0x7fffffff );
+      p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem)*p->pCsr->nField);
       if( !p->aNew ){
         rc = SQLITE_NOMEM;
         goto preupdate_new_out;
       }
     }
-    assert( iIdx>=0 && iIdx<p->pCsr->nField );
-    pMem = &p->aNew[iIdx];
+    assert( iStore>=0 && iStore<p->pCsr->nField );
+    pMem = &p->aNew[iStore];
     if( pMem->flags==0 ){
       if( iIdx==p->pTab->iPKey ){
         sqlite3VdbeMemSetInt64(pMem, p->iKey2);
       }else{
-        rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]);
+        rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iStore]);
         if( rc!=SQLITE_OK ) goto preupdate_new_out;
       }
     }
@@ -2139,11 +2432,20 @@ int sqlite3_stmt_scanstatus_v2(
   void *pOut                      /* OUT: Write the answer here */
 ){
   Vdbe *p = (Vdbe*)pStmt;
-  VdbeOp *aOp = p->aOp;
-  int nOp = p->nOp;
+  VdbeOp *aOp;
+  int nOp;
   ScanStatus *pScan = 0;
   int idx;
 
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( p==0 || pOut==0
+      || iScanStatusOp<SQLITE_SCANSTAT_NLOOP
+      || iScanStatusOp>SQLITE_SCANSTAT_NCYCLE ){
+    return 1;
+  }
+#endif
+  aOp = p->aOp;
+  nOp = p->nOp;
   if( p->pFrame ){
     VdbeFrame *pFrame;
     for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent);
@@ -2165,9 +2467,8 @@ int sqlite3_stmt_scanstatus_v2(
   }
   if( flags & SQLITE_SCANSTAT_COMPLEX ){
     idx = iScan;
-    pScan = &p->aScan[idx];
   }else{
-    /* If the COMPLEX flag is clear, then this function must ignore any 
+    /* If the COMPLEX flag is clear, then this function must ignore any
     ** ScanStatus structures with ScanStatus.addrLoop set to 0. */
     for(idx=0; idx<p->nScan; idx++){
       pScan = &p->aScan[idx];
@@ -2178,6 +2479,8 @@ int sqlite3_stmt_scanstatus_v2(
     }
   }
   if( idx>=p->nScan ) return 1;
+  assert( pScan==0 || pScan==&p->aScan[idx] );
+  pScan = &p->aScan[idx];
 
   switch( iScanStatusOp ){
     case SQLITE_SCANSTAT_NLOOP: {
@@ -2290,7 +2593,7 @@ int sqlite3_stmt_scanstatus(
 void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){
   Vdbe *p = (Vdbe*)pStmt;
   int ii;
-  for(ii=0; ii<p->nOp; ii++){
+  for(ii=0; p!=0 && ii<p->nOp; ii++){
     Op *pOp = &p->aOp[ii];
     pOp->nExec = 0;
     pOp->nCycle = 0;
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index ecbf2d892e..5368c0c420 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -10,7 +10,7 @@
 **
 *************************************************************************
 ** This file contains code used for creating, destroying, and populating
-** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) 
+** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.)
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -152,13 +152,13 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
 }
 
 /*
-** Resize the Vdbe.aOp array so that it is at least nOp elements larger 
+** Resize the Vdbe.aOp array so that it is at least nOp elements larger
 ** than its current size. nOp is guaranteed to be less than or equal
 ** to 1024/sizeof(Op).
 **
 ** If an out-of-memory error occurs while resizing the array, return
-** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain 
-** unchanged (this is so that any opcodes already allocated can be 
+** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain
+** unchanged (this is so that any opcodes already allocated can be
 ** correctly deallocated along with the rest of the Vdbe).
 */
 static int growOpArray(Vdbe *v, int nOp){
@@ -166,7 +166,7 @@ static int growOpArray(Vdbe *v, int nOp){
   Parse *p = v->pParse;
 
   /* The SQLITE_TEST_REALLOC_STRESS compile-time option is designed to force
-  ** more frequent reallocs and hence provide more opportunities for 
+  ** more frequent reallocs and hence provide more opportunities for
   ** simulated OOM faults.  SQLITE_TEST_REALLOC_STRESS is generally used
   ** during testing only.  With SQLITE_TEST_REALLOC_STRESS grow the op array
   ** by the minimum* amount required until the size reaches 512.  Normal
@@ -211,13 +211,43 @@ static int growOpArray(Vdbe *v, int nOp){
 **   sqlite3CantopenError(lineno)
 */
 static void test_addop_breakpoint(int pc, Op *pOp){
-  static int n = 0;
+  static u64 n = 0;
   (void)pc;
   (void)pOp;
   n++;
+  if( n==LARGEST_UINT64 ) abort(); /* so that n is used, preventing a warning */
 }
 #endif
 
+/*
+** Slow paths for sqlite3VdbeAddOp3() and sqlite3VdbeAddOp4Int() for the
+** unusual case when we need to increase the size of the Vdbe.aOp[] array
+** before adding the new opcode.
+*/
+static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){
+  assert( p->nOpAlloc<=p->nOp );
+  if( growOpArray(p, 1) ) return 1;
+  assert( p->nOpAlloc>p->nOp );
+  return sqlite3VdbeAddOp3(p, op, p1, p2, p3);
+}
+static SQLITE_NOINLINE int addOp4IntSlow(
+  Vdbe *p,            /* Add the opcode to this VM */
+  int op,             /* The new opcode */
+  int p1,             /* The P1 operand */
+  int p2,             /* The P2 operand */
+  int p3,             /* The P3 operand */
+  int p4              /* The P4 operand as an integer */
+){
+  int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
+  if( p->db->mallocFailed==0 ){
+    VdbeOp *pOp = &p->aOp[addr];
+    pOp->p4type = P4_INT32;
+    pOp->p4.i = p4;
+  }
+  return addr;
+}
+
+
 /*
 ** Add a new instruction to the list of instructions current in the
 ** VDBE.  Return the address of the new instruction.
@@ -228,17 +258,16 @@ static void test_addop_breakpoint(int pc, Op *pOp){
 **
 **    op              The opcode for this instruction
 **
-**    p1, p2, p3      Operands
-**
-** Use the sqlite3VdbeResolveLabel() function to fix an address and
-** the sqlite3VdbeChangeP4() function to change the value of the P4
-** operand.
+**    p1, p2, p3, p4  Operands
 */
-static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){
-  assert( p->nOpAlloc<=p->nOp );
-  if( growOpArray(p, 1) ) return 1;
-  assert( p->nOpAlloc>p->nOp );
-  return sqlite3VdbeAddOp3(p, op, p1, p2, p3);
+int sqlite3VdbeAddOp0(Vdbe *p, int op){
+  return sqlite3VdbeAddOp3(p, op, 0, 0, 0);
+}
+int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){
+  return sqlite3VdbeAddOp3(p, op, p1, 0, 0);
+}
+int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){
+  return sqlite3VdbeAddOp3(p, op, p1, p2, 0);
 }
 int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
   int i;
@@ -261,6 +290,9 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
   pOp->p3 = p3;
   pOp->p4.p = 0;
   pOp->p4type = P4_NOTUSED;
+
+  /* Replicate this logic in sqlite3VdbeAddOp4Int()
+  ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv   */
 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
   pOp->zComment = 0;
 #endif
@@ -277,16 +309,59 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
 #ifdef SQLITE_VDBE_COVERAGE
   pOp->iSrcLine = 0;
 #endif
+  /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  ** Replicate in sqlite3VdbeAddOp4Int() */
+
   return i;
 }
-int sqlite3VdbeAddOp0(Vdbe *p, int op){
-  return sqlite3VdbeAddOp3(p, op, 0, 0, 0);
-}
-int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){
-  return sqlite3VdbeAddOp3(p, op, p1, 0, 0);
-}
-int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){
-  return sqlite3VdbeAddOp3(p, op, p1, p2, 0);
+int sqlite3VdbeAddOp4Int(
+  Vdbe *p,            /* Add the opcode to this VM */
+  int op,             /* The new opcode */
+  int p1,             /* The P1 operand */
+  int p2,             /* The P2 operand */
+  int p3,             /* The P3 operand */
+  int p4              /* The P4 operand as an integer */
+){
+  int i;
+  VdbeOp *pOp;
+
+  i = p->nOp;
+  if( p->nOpAlloc<=i ){
+    return addOp4IntSlow(p, op, p1, p2, p3, p4);
+  }
+  p->nOp++;
+  pOp = &p->aOp[i];
+  assert( pOp!=0 );
+  pOp->opcode = (u8)op;
+  pOp->p5 = 0;
+  pOp->p1 = p1;
+  pOp->p2 = p2;
+  pOp->p3 = p3;
+  pOp->p4.i = p4;
+  pOp->p4type = P4_INT32;
+
+  /* Replicate this logic in sqlite3VdbeAddOp3()
+  ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv   */
+#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
+  pOp->zComment = 0;
+#endif
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
+  pOp->nExec = 0;
+  pOp->nCycle = 0;
+#endif
+#ifdef SQLITE_DEBUG
+  if( p->db->flags & SQLITE_VdbeAddopTrace ){
+    sqlite3VdbePrintOp(0, i, &p->aOp[i]);
+    test_addop_breakpoint(i, &p->aOp[i]);
+  }
+#endif
+#ifdef SQLITE_VDBE_COVERAGE
+  pOp->iSrcLine = 0;
+#endif
+  /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  ** Replicate in sqlite3VdbeAddOp3() */
+
+  return i;
 }
 
 /* Generate code for an unconditional jump to instruction iDest
@@ -370,12 +445,10 @@ int sqlite3VdbeAddFunctionCall(
   int eCallCtx          /* Calling context */
 ){
   Vdbe *v = pParse->pVdbe;
-  int nByte;
   int addr;
   sqlite3_context *pCtx;
   assert( v );
-  nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*);
-  pCtx = sqlite3DbMallocRawNN(pParse->db, nByte);
+  pCtx = sqlite3DbMallocRawNN(pParse->db, SZ_CONTEXT(nArg));
   if( pCtx==0 ){
     assert( pParse->db->mallocFailed );
     freeEphemeralFunction(pParse->db, (FuncDef*)pFunc);
@@ -464,7 +537,7 @@ int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
     if( bPush){
       pParse->addrExplain = iThis;
     }
-    sqlite3VdbeScanStatus(v, iThis, 0, 0, 0, 0);
+    sqlite3VdbeScanStatus(v, iThis, -1, -1, 0, 0);
   }
   return addr;
 }
@@ -494,26 +567,6 @@ void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere, u16 p5){
   sqlite3MayAbort(p->pParse);
 }
 
-/*
-** Add an opcode that includes the p4 value as an integer.
-*/
-int sqlite3VdbeAddOp4Int(
-  Vdbe *p,            /* Add the opcode to this VM */
-  int op,             /* The new opcode */
-  int p1,             /* The P1 operand */
-  int p2,             /* The P2 operand */
-  int p3,             /* The P3 operand */
-  int p4              /* The P4 operand as an integer */
-){
-  int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
-  if( p->db->mallocFailed==0 ){
-    VdbeOp *pOp = &p->aOp[addr];
-    pOp->p4type = P4_INT32;
-    pOp->p4.i = p4;
-  }
-  return addr;
-}
-
 /* Insert the end of a co-routine
 */
 void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){
@@ -624,19 +677,19 @@ void sqlite3VdbeReusable(Vdbe *p){
 
 /*
 ** The following type and function are used to iterate through all opcodes
-** in a Vdbe main program and each of the sub-programs (triggers) it may 
+** in a Vdbe main program and each of the sub-programs (triggers) it may
 ** invoke directly or indirectly. It should be used as follows:
 **
 **   Op *pOp;
 **   VdbeOpIter sIter;
 **
 **   memset(&sIter, 0, sizeof(sIter));
-**   sIter.v = v;                            // v is of type Vdbe* 
+**   sIter.v = v;                            // v is of type Vdbe*
 **   while( (pOp = opIterNext(&sIter)) ){
 **     // Do something with pOp
 **   }
 **   sqlite3DbFree(v->db, sIter.apSub);
-** 
+**
 */
 typedef struct VdbeOpIter VdbeOpIter;
 struct VdbeOpIter {
@@ -669,9 +722,9 @@ static Op *opIterNext(VdbeOpIter *p){
       p->iSub++;
       p->iAddr = 0;
     }
-  
+ 
     if( pRet->p4type==P4_SUBPROGRAM ){
-      int nByte = (p->nSub+1)*sizeof(SubProgram*);
+      i64 nByte = (1+(u64)p->nSub)*sizeof(SubProgram*);
       int j;
       for(j=0; j<p->nSub; j++){
         if( p->apSub[j]==pRet->p4.pProgram ) break;
@@ -703,7 +756,7 @@ static Op *opIterNext(VdbeOpIter *p){
 **   *  OP_VCreate
 **   *  OP_VRename
 **   *  OP_FkCounter with P2==0 (immediate foreign key constraint)
-**   *  OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine 
+**   *  OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine
 **      (for CREATE TABLE AS SELECT ...)
 **
 ** Then check that the value of Parse.mayAbort is true if an
@@ -728,12 +781,12 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
 
   while( (pOp = opIterNext(&sIter))!=0 ){
     int opcode = pOp->opcode;
-    if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename 
+    if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename
      || opcode==OP_VDestroy
      || opcode==OP_VCreate
      || opcode==OP_ParseSchema
      || opcode==OP_Function || opcode==OP_PureFunc
-     || ((opcode==OP_Halt || opcode==OP_HaltIfNull) 
+     || ((opcode==OP_Halt || opcode==OP_HaltIfNull)
       && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort))
     ){
       hasAbort = 1;
@@ -742,7 +795,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
     if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1;
     if( mayAbort ){
       /* hasCreateIndex may also be set for some DELETE statements that use
-      ** OP_Clear. So this routine may end up returning true in the case 
+      ** OP_Clear. So this routine may end up returning true in the case
       ** where a "DELETE FROM tbl" has a statement-journal but does not
       ** require one. This is not so bad - it is an inefficiency, not a bug. */
       if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1;
@@ -801,8 +854,8 @@ void sqlite3VdbeAssertAbortable(Vdbe *p){
 ** (1) For each jump instruction with a negative P2 value (a label)
 **     resolve the P2 value to an actual address.
 **
-** (2) Compute the maximum number of arguments used by any SQL function
-**     and store that value in *pMaxFuncArgs.
+** (2) Compute the maximum number of arguments used by the xUpdate/xFilter
+**     methods of any virtual table and store that value in *pMaxVtabArgs.
 **
 ** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately
 **     indicate what the prepared statement actually does.
@@ -815,8 +868,8 @@ void sqlite3VdbeAssertAbortable(Vdbe *p){
 ** script numbers the opcodes correctly.  Changes to this routine must be
 ** coordinated with changes to mkopcodeh.tcl.
 */
-static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
-  int nMaxArgs = *pMaxFuncArgs;
+static void resolveP2Values(Vdbe *p, int *pMaxVtabArgs){
+  int nMaxVtabArgs = *pMaxVtabArgs;
   Op *pOp;
   Parse *pParse = p->pParse;
   int *aLabel = pParse->aLabel;
@@ -826,7 +879,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
   p->bIsReader = 0;
   pOp = &p->aOp[p->nOp-1];
   assert( p->aOp[0].opcode==OP_Init );
-  while( 1 /* Loop termates when it reaches the OP_Init opcode */ ){
+  while( 1 /* Loop terminates when it reaches the OP_Init opcode */ ){
     /* Only JUMP opcodes and the short list of special opcodes in the switch
     ** below need to be considered.  The mkopcodeh.tcl generator script groups
     ** all these opcodes together near the front of the opcode list.  Skip
@@ -861,15 +914,19 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
         }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
         case OP_VUpdate: {
-          if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
+          if( pOp->p2>nMaxVtabArgs ) nMaxVtabArgs = pOp->p2;
           break;
         }
         case OP_VFilter: {
           int n;
+          /* The instruction immediately prior to VFilter will be an
+          ** OP_Integer that sets the "argc" value for the VFilter.  See
+          ** the code where OP_VFilter is generated at tag-20250207a. */
           assert( (pOp - p->aOp) >= 3 );
           assert( pOp[-1].opcode==OP_Integer );
+          assert( pOp[-1].p2==pOp->p3+1 );
           n = pOp[-1].p1;
-          if( n>nMaxArgs ) nMaxArgs = n;
+          if( n>nMaxVtabArgs ) nMaxVtabArgs = n;
           /* Fall through into the default case */
           /* no break */ deliberate_fall_through
         }
@@ -884,6 +941,15 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
             assert( aLabel!=0 );  /* True because of tag-20230419-1 */
             pOp->p2 = aLabel[ADDR(pOp->p2)];
           }
+
+          /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes
+          ** might */
+          assert( pOp->p2>0 
+                  || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 );
+
+          /* Jumps never go off the end of the bytecode array */
+          assert( pOp->p2<p->nOp
+                  || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 );
           break;
         }
       }
@@ -892,7 +958,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
       ** have non-negative values for P2. */
       assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0);
     }
-    assert( pOp>p->aOp );    
+    assert( pOp>p->aOp );   
     pOp--;
   }
 resolve_p2_values_loop_exit:
@@ -901,7 +967,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
     pParse->aLabel = 0;
   }
   pParse->nLabel = 0;
-  *pMaxFuncArgs = nMaxArgs;
+  *pMaxVtabArgs = nMaxVtabArgs;
   assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) );
 }
 
@@ -948,6 +1014,10 @@ void sqlite3VdbeNoJumpsOutsideSubrtn(
       int iDest = pOp->p2;   /* Jump destination */
       if( iDest==0 ) continue;
       if( pOp->opcode==OP_Gosub ) continue;
+      if( pOp->p3==20230325 && pOp->opcode==OP_NotNull ){
+        /* This is a deliberately taken illegal branch.  tag-20230325-2 */
+        continue;
+      }
       if( iDest<0 ){
         int j = ADDR(iDest);
         assert( j>=0 );
@@ -1041,12 +1111,12 @@ void sqlite3VdbeVerifyAbortable(Vdbe *p, int onError){
 /*
 ** This function returns a pointer to the array of opcodes associated with
 ** the Vdbe passed as the first argument. It is the callers responsibility
-** to arrange for the returned array to be eventually freed using the 
+** to arrange for the returned array to be eventually freed using the
 ** vdbeFreeOpArray() function.
 **
 ** Before returning, *pnOp is set to the number of entries in the returned
-** array. Also, *pnMaxArg is set to the larger of its current value and 
-** the number of entries in the Vdbe.apArg[] array required to execute the 
+** array. Also, *pnMaxArg is set to the larger of its current value and
+** the number of entries in the Vdbe.apArg[] array required to execute the
 ** returned program.
 */
 VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){
@@ -1120,13 +1190,13 @@ VdbeOp *sqlite3VdbeAddOpList(
 void sqlite3VdbeScanStatus(
   Vdbe *p,                        /* VM to add scanstatus() to */
   int addrExplain,                /* Address of OP_Explain (or 0) */
-  int addrLoop,                   /* Address of loop counter */ 
+  int addrLoop,                   /* Address of loop counter */
   int addrVisit,                  /* Address of rows visited counter */
   LogEst nEst,                    /* Estimated number of output rows */
   const char *zName               /* Name of table or index being scanned */
 ){
   if( IS_STMT_SCANSTATUS(p->db) ){
-    sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus);
+    i64 nByte = (1+(i64)p->nScan) * sizeof(ScanStatus);
     ScanStatus *aNew;
     aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte);
     if( aNew ){
@@ -1146,13 +1216,13 @@ void sqlite3VdbeScanStatus(
 ** Add the range of instructions from addrStart to addrEnd (inclusive) to
 ** the set of those corresponding to the sqlite3_stmt_scanstatus() counters
 ** associated with the OP_Explain instruction at addrExplain. The
-** sum of the sqlite3Hwtime() values for each of these instructions 
+** sum of the sqlite3Hwtime() values for each of these instructions
 ** will be returned for SQLITE_SCANSTAT_NCYCLE requests.
 */
 void sqlite3VdbeScanStatusRange(
-  Vdbe *p, 
-  int addrExplain, 
-  int addrStart, 
+  Vdbe *p,
+  int addrExplain,
+  int addrStart,
   int addrEnd
 ){
   if( IS_STMT_SCANSTATUS(p->db) ){
@@ -1182,9 +1252,9 @@ void sqlite3VdbeScanStatusRange(
 ** addrExplain.
 */
 void sqlite3VdbeScanStatusCounters(
-  Vdbe *p, 
-  int addrExplain, 
-  int addrLoop, 
+  Vdbe *p,
+  int addrExplain,
+  int addrLoop,
   int addrVisit
 ){
   if( IS_STMT_SCANSTATUS(p->db) ){
@@ -1196,8 +1266,8 @@ void sqlite3VdbeScanStatusCounters(
       pScan = 0;
     }
     if( pScan ){
-      pScan->addrLoop = addrLoop;
-      pScan->addrVisit = addrVisit;
+      if( addrLoop>0 ) pScan->addrLoop = addrLoop;
+      if( addrVisit>0 ) pScan->addrVisit = addrVisit;
     }
   }
 }
@@ -1236,6 +1306,9 @@ void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){
 */
 void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){
   VdbeOp *pOp = sqlite3VdbeGetLastOp(p);
+#ifdef SQLITE_DEBUG
+  while( pOp->opcode==OP_ReleaseReg ) pOp--;
+#endif
   if( pOp->p3==iDest && pOp->opcode==OP_Column ){
     pOp->p5 |= OPFLAG_TYPEOFARG;
   }
@@ -1280,7 +1353,7 @@ void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){
 
 /*
 ** If the input FuncDef structure is ephemeral, then free it.  If
-** the FuncDef is not ephermal, then do nothing.
+** the FuncDef is not ephemeral, then do nothing.
 */
 static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
   assert( db!=0 );
@@ -1341,13 +1414,23 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
       if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
       break;
     }
+    case P4_TABLEREF: {
+      if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
+      break;
+    }
+    case P4_SUBRTNSIG: {
+      SubrtnSig *pSig = (SubrtnSig*)p4;
+      sqlite3DbFree(db, pSig->zAff);
+      sqlite3DbFree(db, pSig);
+      break;
+    }
   }
 }
 
 /*
 ** Free the space allocated for aOp and any p4 values allocated for the
-** opcodes contained within. If aOp is not NULL it is assumed to contain 
-** nOp entries. 
+** opcodes contained within. If aOp is not NULL it is assumed to contain
+** nOp entries.
 */
 static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){
   assert( nOp>=0 );
@@ -1358,7 +1441,7 @@ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){
       if( pOp->p4type <= P4_FREE_IF_LE ) freeP4(db, pOp->p4type, pOp->p4.p);
 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
       sqlite3DbFree(db, pOp->zComment);
-#endif     
+#endif    
       if( pOp==aOp ) break;
       pOp--;
     }
@@ -1444,7 +1527,6 @@ void sqlite3VdbeReleaseRegisters(
 }
 #endif /* SQLITE_DEBUG */
 
-
 /*
 ** Change the value of the P4 operand for a specific instruction.
 ** This routine is useful when a large program is loaded from a
@@ -1455,7 +1537,7 @@ void sqlite3VdbeReleaseRegisters(
 ** the string is made into memory obtained from sqlite3_malloc().
 ** A value of n==0 means copy bytes of zP4 up to and including the
 ** first null byte.  If n>0 then copy n+1 bytes of zP4.
-** 
+**
 ** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points
 ** to a string or structure that is guaranteed to exist for the lifetime of
 ** the Vdbe. In these cases we can just copy the pointer.
@@ -1469,7 +1551,7 @@ static void SQLITE_NOINLINE vdbeChangeP4Full(
   int n
 ){
   if( pOp->p4type ){
-    freeP4(p->db, pOp->p4type, pOp->p4.p);
+    assert( pOp->p4type > P4_FREE_IF_LE );
     pOp->p4type = 0;
     pOp->p4.p = 0;
   }
@@ -1516,7 +1598,7 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){
 }
 
 /*
-** Change the P4 operand of the most recently coded instruction 
+** Change the P4 operand of the most recently coded instruction
 ** to the value defined by the arguments.  This is a high-speed
 ** version of sqlite3VdbeChangeP4().
 **
@@ -1605,7 +1687,7 @@ void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){
 ** routine, then a pointer to a dummy VdbeOp will be returned.  That opcode
 ** is readable but not writable, though it is cast to a writable value.
 ** The return of a dummy opcode allows the call to continue functioning
-** after an OOM fault without having to check to see if the return from 
+** after an OOM fault without having to check to see if the return from
 ** this routine is a valid pointer.  But because the dummy.opcode is 0,
 ** dummy will never be written to.  This is verified by code inspection and
 ** by running with Valgrind.
@@ -1835,9 +1917,9 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){
         CollSeq *pColl = pKeyInfo->aColl[j];
         const char *zColl = pColl ? pColl->zName : "";
         if( strcmp(zColl, "BINARY")==0 ) zColl = "B";
-        sqlite3_str_appendf(&x, ",%s%s%s", 
-               (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "", 
-               (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "", 
+        sqlite3_str_appendf(&x, ",%s%s%s",
+               (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "",
+               (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "",
                zColl);
       }
       sqlite3_str_append(&x, ")", 1);
@@ -1921,6 +2003,11 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){
       zP4 = pOp->p4.pTab->zName;
       break;
     }
+    case P4_SUBRTNSIG: {
+      SubrtnSig *pSig = pOp->p4.pSubrtnSig;
+      sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff);
+      break;
+    }
     default: {
       zP4 = pOp->p4.z;
     }
@@ -1960,13 +2047,13 @@ void sqlite3VdbeUsesBtree(Vdbe *p, int i){
 **
 ** If SQLite is not threadsafe but does support shared-cache mode, then
 ** sqlite3BtreeEnter() is invoked to set the BtShared.db variables
-** of all of BtShared structures accessible via the database handle 
+** of all of BtShared structures accessible via the database handle
 ** associated with the VM.
 **
 ** If SQLite is not threadsafe and does not support shared-cache mode, this
 ** function is a no-op.
 **
-** The p->btreeMask field is a bitmask of all btrees that the prepared 
+** The p->btreeMask field is a bitmask of all btrees that the prepared
 ** statement p will ever use.  Let N be the number of bits in p->btreeMask
 ** corresponding to btrees that use shared cache.  Then the runtime of
 ** this routine is N*N.  But as N is rarely more than 1, this should not
@@ -2034,8 +2121,8 @@ void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){
   /* NB:  The sqlite3OpcodeName() function is implemented by code created
   ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the
   ** information from the vdbe.c source text */
-  fprintf(pOut, zFormat1, pc, 
-      sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, 
+  fprintf(pOut, zFormat1, pc,
+      sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3,
       zP4 ? zP4 : "", pOp->p5,
       zCom ? zCom : ""
   );
@@ -2062,6 +2149,7 @@ void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){
 ** will be initialized before use.
 */
 static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){
+  assert( db!=0 );
   if( N>0 ){
     do{
       p->flags = flags;
@@ -2069,6 +2157,7 @@ static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){
       p->szMalloc = 0;
 #ifdef SQLITE_DEBUG
       p->pScopyFrom = 0;
+      p->bScopy = 0;
 #endif
       p++;
     }while( (--N)>0 );
@@ -2087,6 +2176,7 @@ static void releaseMemArray(Mem *p, int N){
   if( p && N ){
     Mem *pEnd = &p[N];
     sqlite3 *db = p->db;
+    assert( db!=0 );
     if( db->pnBytesFreed ){
       do{
         if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
@@ -2098,15 +2188,15 @@ static void releaseMemArray(Mem *p, int N){
       assert( sqlite3VdbeCheckMemInvariants(p) );
 
       /* This block is really an inlined version of sqlite3VdbeMemRelease()
-      ** that takes advantage of the fact that the memory cell value is 
+      ** that takes advantage of the fact that the memory cell value is
       ** being set to NULL after releasing any dynamic resources.
       **
-      ** The justification for duplicating code is that according to 
-      ** callgrind, this causes a certain test case to hit the CPU 4.7 
-      ** percent less (x86 linux, gcc version 4.1.2, -O6) than if 
+      ** The justification for duplicating code is that according to
+      ** callgrind, this causes a certain test case to hit the CPU 4.7
+      ** percent less (x86 linux, gcc version 4.1.2, -O6) than if
       ** sqlite3MemRelease() were called from here. With -O2, this jumps
-      ** to 6.6 percent. The test case is inserting 1000 rows into a table 
-      ** with no indexes using a single prepared INSERT statement, bind() 
+      ** to 6.6 percent. The test case is inserting 1000 rows into a table
+      ** with no indexes using a single prepared INSERT statement, bind()
       ** and reset(). Inserts are grouped into a transaction.
       */
       testcase( p->flags & MEM_Agg );
@@ -2256,7 +2346,7 @@ int sqlite3VdbeNextOpcode(
       Op *pOp = aOp + i;
       if( pOp->opcode==OP_OpenRead ) break;
       if( pOp->opcode==OP_OpenWrite && (pOp->p5 & OPFLAG_P2ISREG)==0 ) break;
-      if( pOp->opcode==OP_ReopenIdx ) break;      
+      if( pOp->opcode==OP_ReopenIdx ) break;     
     }else
 #endif
     {
@@ -2364,8 +2454,8 @@ int sqlite3VdbeList(
         sqlite3VdbeMemSetInt64(pMem, pOp->p1);
         sqlite3VdbeMemSetInt64(pMem+1, pOp->p2);
         sqlite3VdbeMemSetInt64(pMem+2, pOp->p3);
-        sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free);     
-        p->nResColumn = 4;
+        sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free);
+        assert( p->nResColumn==4 );
       }else{
         sqlite3VdbeMemSetInt64(pMem+0, i);
         sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode),
@@ -2384,7 +2474,7 @@ int sqlite3VdbeList(
         sqlite3VdbeMemSetNull(pMem+7);
 #endif
         sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free);
-        p->nResColumn = 8;
+        assert( p->nResColumn==8 );
       }
       p->pResultRow = pMem;
       if( db->mallocFailed ){
@@ -2537,11 +2627,11 @@ void sqlite3VdbeRewind(Vdbe *p){
 ** creating the virtual machine.  This involves things such
 ** as allocating registers and initializing the program counter.
 ** After the VDBE has be prepped, it can be executed by one or more
-** calls to sqlite3VdbeExec().  
+** calls to sqlite3VdbeExec(). 
 **
 ** This function may be called exactly once on each virtual machine.
 ** After this routine is called the VM has been "packaged" and is ready
-** to run.  After this routine is called, further calls to 
+** to run.  After this routine is called, further calls to
 ** sqlite3VdbeAddOp() functions are prohibited.  This routine disconnects
 ** the Vdbe from the Parse object that helped generate it so that the
 ** the Vdbe becomes an independent entity and the Parse object can be
@@ -2558,7 +2648,7 @@ void sqlite3VdbeMakeReady(
   int nVar;                      /* Number of parameters */
   int nMem;                      /* Number of VM memory registers */
   int nCursor;                   /* Number of cursors required */
-  int nArg;                      /* Number of arguments in subprograms */
+  int nArg;                      /* Max number args to xFilter or xUpdate */
   int n;                         /* Loop counter */
   struct ReusableSpace x;        /* Reusable bulk memory */
 
@@ -2567,6 +2657,7 @@ void sqlite3VdbeMakeReady(
   assert( pParse!=0 );
   assert( p->eVdbeState==VDBE_INIT_STATE );
   assert( pParse==p->pParse );
+  assert( pParse->db==p->db );
   p->pVList = pParse->pVList;
   pParse->pVList =  0;
   db = p->db;
@@ -2575,7 +2666,7 @@ void sqlite3VdbeMakeReady(
   nMem = pParse->nMem;
   nCursor = pParse->nTab;
   nArg = pParse->nMaxArg;
-  
+ 
   /* Each cursor uses a memory cell.  The first cursor (cursor 0) can
   ** use aMem[0] which is not otherwise used by the VDBE program.  Allocate
   ** space at the end of aMem[] for cursors 1 and greater.
@@ -2598,34 +2689,17 @@ void sqlite3VdbeMakeReady(
   resolveP2Values(p, &nArg);
   p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
   if( pParse->explain ){
-    static const char * const azColName[] = {
-       "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment",
-       "id", "parent", "notused", "detail"
-    };
-    int iFirst, mx, i;
     if( nMem<10 ) nMem = 10;
     p->explain = pParse->explain;
-    if( pParse->explain==2 ){
-      sqlite3VdbeSetNumCols(p, 4);
-      iFirst = 8;
-      mx = 12;
-    }else{
-      sqlite3VdbeSetNumCols(p, 8);
-      iFirst = 0;
-      mx = 8;
-    }
-    for(i=iFirst; i<mx; i++){
-      sqlite3VdbeSetColName(p, i-iFirst, COLNAME_NAME,
-                            azColName[i], SQLITE_STATIC);
-    }
+    p->nResColumn = 12 - 4*p->explain;
   }
   p->expired = 0;
 
   /* Memory for registers, parameters, cursor, etc, is allocated in one or two
-  ** passes.  On the first pass, we try to reuse unused memory at the 
+  ** passes.  On the first pass, we try to reuse unused memory at the
   ** end of the opcode array.  If we are unable to satisfy all memory
   ** requirements by reusing the opcode array tail, then the second
-  ** pass will fill in the remainder using a fresh memory allocation.  
+  ** pass will fill in the remainder using a fresh memory allocation. 
   **
   ** This two-pass approach that reuses as much memory as possible from
   ** the leftover memory at the end of the opcode array.  This can significantly
@@ -2646,6 +2720,9 @@ void sqlite3VdbeMakeReady(
       p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*));
     }
   }
+#ifdef SQLITE_DEBUG
+  p->napArg = nArg;
+#endif
 
   if( db->mallocFailed ){
     p->nVar = 0;
@@ -2663,13 +2740,29 @@ void sqlite3VdbeMakeReady(
 }
 
 /*
-** Close a VDBE cursor and release all the resources that cursor 
+** Close a VDBE cursor and release all the resources that cursor
 ** happens to hold.
 */
 void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
   if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx);
 }
+static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){
+  VdbeTxtBlbCache *pCache = pCx->pCache;
+  assert( pCx->colCache );
+  pCx->colCache = 0;
+  pCx->pCache = 0;
+  if( pCache->pCValue ){
+    sqlite3RCStrUnref(pCache->pCValue);
+    pCache->pCValue = 0;
+  }
+  sqlite3DbFree(p->db, pCache);
+  sqlite3VdbeFreeCursorNN(p, pCx);
+}
 void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){
+  if( pCx->colCache ){
+    freeCursorWithCache(p, pCx);
+    return;
+  }
   switch( pCx->eCurType ){
     case CURTYPE_SORTER: {
       sqlite3VdbeSorterClose(p->db, pCx);
@@ -2733,7 +2826,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){
 /*
 ** Close all cursors.
 **
-** Also release any dynamic memory held by the VM in the Vdbe.aMem memory 
+** Also release any dynamic memory held by the VM in the Vdbe.aMem memory
 ** cell array. This is necessary as the memory cell array may contain
 ** pointers to VdbeFrame objects, which may in turn contain pointers to
 ** open cursors.
@@ -2770,12 +2863,12 @@ void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){
   int n;
   sqlite3 *db = p->db;
 
-  if( p->nResColumn ){
-    releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
+  if( p->nResAlloc ){
+    releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N);
     sqlite3DbFree(db, p->aColName);
   }
   n = nResColumn*COLNAME_N;
-  p->nResColumn = (u16)nResColumn;
+  p->nResColumn = p->nResAlloc = (u16)nResColumn;
   p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n );
   if( p->aColName==0 ) return;
   initMemArray(p->aColName, n, db, MEM_Null);
@@ -2800,14 +2893,14 @@ int sqlite3VdbeSetColName(
 ){
   int rc;
   Mem *pColName;
-  assert( idx<p->nResColumn );
+  assert( idx<p->nResAlloc );
   assert( var<COLNAME_N );
   if( p->db->mallocFailed ){
     assert( !zName || xDel!=SQLITE_DYNAMIC );
     return SQLITE_NOMEM_BKPT;
   }
   assert( p->aColName!=0 );
-  pColName = &(p->aColName[idx+var*p->nResColumn]);
+  pColName = &(p->aColName[idx+var*p->nResAlloc]);
   rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel);
   assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 );
   return rc;
@@ -2828,27 +2921,27 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
   int needXcommit = 0;
 
 #ifdef SQLITE_OMIT_VIRTUALTABLE
-  /* With this option, sqlite3VtabSync() is defined to be simply 
-  ** SQLITE_OK so p is not used. 
+  /* With this option, sqlite3VtabSync() is defined to be simply
+  ** SQLITE_OK so p is not used.
   */
   UNUSED_PARAMETER(p);
 #endif
 
   /* Before doing anything else, call the xSync() callback for any
   ** virtual module tables written in this transaction. This has to
-  ** be done before determining whether a super-journal file is 
+  ** be done before determining whether a super-journal file is
   ** required, as an xSync() callback may add an attached database
   ** to the transaction.
   */
   rc = sqlite3VtabSync(db, p);
 
   /* This loop determines (a) if the commit hook should be invoked and
-  ** (b) how many database files have open write transactions, not 
-  ** including the temp database. (b) is important because if more than 
+  ** (b) how many database files have open write transactions, not
+  ** including the temp database. (b) is important because if more than
   ** one database file has an open write transaction, a super-journal
   ** file is required for an atomic commit.
-  */ 
-  for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ 
+  */
+  for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
     Btree *pBt = db->aDb[i].pBt;
     if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){
       /* Whether or not a database might need a super-journal depends upon
@@ -2869,7 +2962,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
       if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF
        && aMJNeeded[sqlite3PagerGetJournalMode(pPager)]
        && sqlite3PagerIsMemdb(pPager)==0
-      ){ 
+      ){
         assert( i!=1 );
         nTrans++;
       }
@@ -2894,28 +2987,32 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
   ** super-journal.
   **
   ** If the return value of sqlite3BtreeGetFilename() is a zero length
-  ** string, it means the main database is :memory: or a temp file.  In 
-  ** that case we do not support atomic multi-file commits, so use the 
+  ** string, it means the main database is :memory: or a temp file.  In
+  ** that case we do not support atomic multi-file commits, so use the
   ** simple case then too.
   */
   if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt))
    || nTrans<=1
   ){
-    for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
-      Btree *pBt = db->aDb[i].pBt;
-      if( pBt ){
-        rc = sqlite3BtreeCommitPhaseOne(pBt, 0);
+    if( needXcommit ){
+      for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
+        Btree *pBt = db->aDb[i].pBt;
+        if( sqlite3BtreeTxnState(pBt)>=SQLITE_TXN_WRITE ){
+          rc = sqlite3BtreeCommitPhaseOne(pBt, 0);
+        }
       }
     }
 
-    /* Do the commit only if all databases successfully complete phase 1. 
+    /* Do the commit only if all databases successfully complete phase 1.
     ** If one of the BtreeCommitPhaseOne() calls fails, this indicates an
     ** IO error while deleting or truncating a journal file. It is unlikely,
     ** but could happen. In this case abandon processing and return the error.
     */
     for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
       Btree *pBt = db->aDb[i].pBt;
-      if( pBt ){
+      int txn = sqlite3BtreeTxnState(pBt);
+      if( txn!=SQLITE_TXN_NONE ){
+        assert( needXcommit || txn==SQLITE_TXN_READ );
         rc = sqlite3BtreeCommitPhaseTwo(pBt, 0);
       }
     }
@@ -2967,7 +3064,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
     }while( rc==SQLITE_OK && res );
     if( rc==SQLITE_OK ){
       /* Open the super-journal. */
-      rc = sqlite3OsOpenMalloc(pVfs, zSuper, &pSuperJrnl, 
+      rc = sqlite3OsOpenMalloc(pVfs, zSuper, &pSuperJrnl,
           SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|
           SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_SUPER_JOURNAL, 0
       );
@@ -2976,7 +3073,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
       sqlite3DbFree(db, zSuper-4);
       return rc;
     }
- 
+
     /* Write the name of each database file in the transaction into the new
     ** super-journal file. If an error occurs at this point close
     ** and delete the super-journal file. All the individual journal files
@@ -3024,7 +3121,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
     ** in case the super-journal file name was written into the journal
     ** file before the failure occurred.
     */
-    for(i=0; rc==SQLITE_OK && i<db->nDb; i++){ 
+    for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
       Btree *pBt = db->aDb[i].pBt;
       if( pBt ){
         rc = sqlite3BtreeCommitPhaseOne(pBt, zSuper);
@@ -3057,7 +3154,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
     */
     disable_simulated_io_errors();
     sqlite3BeginBenignMalloc();
-    for(i=0; i<db->nDb; i++){ 
+    for(i=0; i<db->nDb; i++){
       Btree *pBt = db->aDb[i].pBt;
       if( pBt ){
         sqlite3BtreeCommitPhaseTwo(pBt, 1);
@@ -3073,7 +3170,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
   return rc;
 }
 
-/* 
+/*
 ** This routine checks that the sqlite3.nVdbeActive count variable
 ** matches the number of vdbe's in the list sqlite3.pVdbe that are
 ** currently active. An assertion fails if the two counts do not match.
@@ -3109,10 +3206,10 @@ static void checkActiveVdbeCnt(sqlite3 *db){
 ** If the Vdbe passed as the first argument opened a statement-transaction,
 ** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or
 ** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement
-** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the 
+** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the
 ** statement transaction is committed.
 **
-** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. 
+** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned.
 ** Otherwise SQLITE_OK.
 */
 static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){
@@ -3125,7 +3222,7 @@ static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){
   assert( db->nStatement>0 );
   assert( p->iStatement==(db->nStatement+db->nSavepoint) );
 
-  for(i=0; i<db->nDb; i++){ 
+  for(i=0; i<db->nDb; i++){
     int rc2 = SQLITE_OK;
     Btree *pBt = db->aDb[i].pBt;
     if( pBt ){
@@ -3152,8 +3249,8 @@ static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){
     }
   }
 
-  /* If the statement transaction is being rolled back, also restore the 
-  ** database handles deferred constraint counter to the value it had when 
+  /* If the statement transaction is being rolled back, also restore the
+  ** database handles deferred constraint counter to the value it had when
   ** the statement transaction was opened.  */
   if( eOp==SAVEPOINT_ROLLBACK ){
     db->nDeferredCons = p->nStmtDefCons;
@@ -3170,28 +3267,31 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){
 
 
 /*
-** This function is called when a transaction opened by the database 
-** handle associated with the VM passed as an argument is about to be 
-** committed. If there are outstanding deferred foreign key constraint
-** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK.
+** These functions are called when a transaction opened by the database
+** handle associated with the VM passed as an argument is about to be
+** committed. If there are outstanding foreign key constraint violations
+** return an error code. Otherwise, SQLITE_OK.
 **
-** If there are outstanding FK violations and this function returns 
-** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY
-** and write an error message to it. Then return SQLITE_ERROR.
+** If there are outstanding FK violations and this function returns
+** non-zero, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY
+** and write an error message to it.
 */
 #ifndef SQLITE_OMIT_FOREIGN_KEY
-int sqlite3VdbeCheckFk(Vdbe *p, int deferred){
+static SQLITE_NOINLINE int vdbeFkError(Vdbe *p){
+  p->rc = SQLITE_CONSTRAINT_FOREIGNKEY;
+  p->errorAction = OE_Abort;
+  sqlite3VdbeError(p, "FOREIGN KEY constraint failed");
+  if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR;
+  return SQLITE_CONSTRAINT_FOREIGNKEY;
+}
+int sqlite3VdbeCheckFkImmediate(Vdbe *p){
+  if( p->nFkConstraint==0 ) return SQLITE_OK;
+  return vdbeFkError(p);
+}
+int sqlite3VdbeCheckFkDeferred(Vdbe *p){
   sqlite3 *db = p->db;
-  if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) 
-   || (!deferred && p->nFkConstraint>0) 
-  ){
-    p->rc = SQLITE_CONSTRAINT_FOREIGNKEY;
-    p->errorAction = OE_Abort;
-    sqlite3VdbeError(p, "FOREIGN KEY constraint failed");
-    if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR;
-    return SQLITE_CONSTRAINT_FOREIGNKEY;
-  }
-  return SQLITE_OK;
+  if( (db->nDeferredCons+db->nDeferredImmCons)==0 ) return SQLITE_OK;
+  return vdbeFkError(p);
 }
 #endif
 
@@ -3214,7 +3314,7 @@ int sqlite3VdbeHalt(Vdbe *p){
 
   /* This function contains the logic that determines if a statement or
   ** transaction will be committed or rolled back as a result of the
-  ** execution of this virtual machine. 
+  ** execution of this virtual machine.
   **
   ** If any of the following errors occur:
   **
@@ -3256,16 +3356,16 @@ int sqlite3VdbeHalt(Vdbe *p){
       mrc = isSpecialError = 0;
     }
     if( isSpecialError ){
-      /* If the query was read-only and the error code is SQLITE_INTERRUPT, 
-      ** no rollback is necessary. Otherwise, at least a savepoint 
-      ** transaction must be rolled back to restore the database to a 
+      /* If the query was read-only and the error code is SQLITE_INTERRUPT,
+      ** no rollback is necessary. Otherwise, at least a savepoint
+      ** transaction must be rolled back to restore the database to a
       ** consistent state.
       **
       ** Even if the statement is read-only, it is important to perform
-      ** a statement or transaction rollback operation. If the error 
+      ** a statement or transaction rollback operation. If the error
       ** occurred while writing to the journal, sub-journal or database
       ** file as part of an effort to free up cache space (see function
-      ** pagerStress() in pager.c), the rollback is required to restore 
+      ** pagerStress() in pager.c), the rollback is required to restore
       ** the pager to a consistent state.
       */
       if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){
@@ -3285,21 +3385,21 @@ int sqlite3VdbeHalt(Vdbe *p){
 
     /* Check for immediate foreign key violations. */
     if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){
-      sqlite3VdbeCheckFk(p, 0);
+      (void)sqlite3VdbeCheckFkImmediate(p);
     }
-  
-    /* If the auto-commit flag is set and this is the only active writer 
-    ** VM, then we do either a commit or rollback of the current transaction. 
+
+    /* If the auto-commit flag is set and this is the only active writer
+    ** VM, then we do either a commit or rollback of the current transaction.
     **
-    ** Note: This block also runs if one of the special errors handled 
-    ** above has occurred. 
+    ** Note: This block also runs if one of the special errors handled
+    ** above has occurred.
     */
-    if( !sqlite3VtabInSync(db) 
-     && db->autoCommit 
-     && db->nVdbeWrite==(p->readOnly==0) 
+    if( !sqlite3VtabInSync(db)
+     && db->autoCommit
+     && db->nVdbeWrite==(p->readOnly==0)
     ){
       if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){
-        rc = sqlite3VdbeCheckFk(p, 1);
+        rc = sqlite3VdbeCheckFkDeferred(p);
         if( rc!=SQLITE_OK ){
           if( NEVER(p->readOnly) ){
             sqlite3VdbeLeave(p);
@@ -3309,10 +3409,10 @@ int sqlite3VdbeHalt(Vdbe *p){
         }else if( db->flags & SQLITE_CorruptRdOnly ){
           rc = SQLITE_CORRUPT;
           db->flags &= ~SQLITE_CorruptRdOnly;
-        }else{ 
-          /* The auto-commit flag is true, the vdbe program was successful 
+        }else{
+          /* The auto-commit flag is true, the vdbe program was successful
           ** or hit an 'OR FAIL' constraint and there are no deferred foreign
-          ** key constraints to hold up the transaction. This means a commit 
+          ** key constraints to hold up the transaction. This means a commit
           ** is required. */
           rc = vdbeCommit(db, p);
         }
@@ -3320,6 +3420,7 @@ int sqlite3VdbeHalt(Vdbe *p){
           sqlite3VdbeLeave(p);
           return SQLITE_BUSY;
         }else if( rc!=SQLITE_OK ){
+          sqlite3SystemError(db, rc);
           p->rc = rc;
           sqlite3RollbackAll(db, SQLITE_OK);
           p->nChange = 0;
@@ -3348,7 +3449,7 @@ int sqlite3VdbeHalt(Vdbe *p){
         p->nChange = 0;
       }
     }
-  
+ 
     /* If eStatementOp is non-zero, then a statement transaction needs to
     ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to
     ** do so. If this operation returns an error, and the current statement
@@ -3369,9 +3470,9 @@ int sqlite3VdbeHalt(Vdbe *p){
         p->nChange = 0;
       }
     }
-  
+ 
     /* If this was an INSERT, UPDATE or DELETE and no statement transaction
-    ** has been rolled back, update the database connection change-counter. 
+    ** has been rolled back, update the database connection change-counter.
     */
     if( p->changeCntOn ){
       if( eStatementOp!=SAVEPOINT_ROLLBACK ){
@@ -3400,7 +3501,7 @@ int sqlite3VdbeHalt(Vdbe *p){
   }
 
   /* If the auto-commit flag is set to true, then any locks that were held
-  ** by connection db have now been released. Call sqlite3ConnectionUnlocked() 
+  ** by connection db have now been released. Call sqlite3ConnectionUnlocked()
   ** to invoke any required unlock-notify callbacks.
   */
   if( db->autoCommit ){
@@ -3422,7 +3523,7 @@ void sqlite3VdbeResetStepResult(Vdbe *p){
 
 /*
 ** Copy the error code and error message belonging to the VDBE passed
-** as the first argument to its database handle (so that they will be 
+** as the first argument to its database handle (so that they will be
 ** returned by calls to sqlite3_errcode() and sqlite3_errmsg()).
 **
 ** This function does not clear the VDBE error code or message, just
@@ -3448,7 +3549,7 @@ int sqlite3VdbeTransferError(Vdbe *p){
 
 #ifdef SQLITE_ENABLE_SQLLOG
 /*
-** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, 
+** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run,
 ** invoke it.
 */
 static void vdbeInvokeSqllog(Vdbe *v){
@@ -3509,7 +3610,7 @@ int sqlite3VdbeReset(Vdbe *p){
   /* Reset register contents and reclaim error message memory.
   */
 #ifdef SQLITE_DEBUG
-  /* Execute assert() statements to ensure that the Vdbe.apCsr[] and 
+  /* Execute assert() statements to ensure that the Vdbe.apCsr[] and
   ** Vdbe.aMem[] arrays have already been cleaned up.  */
   if( p->apCsr ) for(i=0; i<p->nCursor; i++) assert( p->apCsr[i]==0 );
   if( p->aMem ){
@@ -3564,7 +3665,7 @@ int sqlite3VdbeReset(Vdbe *p){
 #endif
   return p->rc & db->errMask;
 }
- 
+
 /*
 ** Clean up and delete a VDBE after execution.  Return an integer which is
 ** the result code.  Write any error message text into *pzErrMsg.
@@ -3588,8 +3689,8 @@ int sqlite3VdbeFinalize(Vdbe *p){
 ** the first argument.
 **
 ** Or, if iOp is greater than or equal to zero, then the destructor is
-** only invoked for those auxiliary data pointers created by the user 
-** function invoked by the OP_Function opcode at instruction iOp of 
+** only invoked for those auxiliary data pointers created by the user
+** function invoked by the OP_Function opcode at instruction iOp of
 ** VM pVdbe, and only then if:
 **
 **    * the associated function parameter is the 32nd or later (counting
@@ -3631,7 +3732,7 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
   assert( db!=0 );
   assert( p->db==0 || p->db==db );
   if( p->aColName ){
-    releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
+    releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N);
     sqlite3DbNNFreeNN(db, p->aColName);
   }
   for(pSub=p->pProgram; pSub; pSub=pNext){
@@ -3861,7 +3962,7 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){
 ** The sizes for serial types less than 128
 */
 const u8 sqlite3SmallTypeSizes[128] = {
-        /*  0   1   2   3   4   5   6   7   8   9 */   
+        /*  0   1   2   3   4   5   6   7   8   9 */  
 /*   0 */   0,  1,  2,  3,  4,  6,  8,  8,  0,  0,
 /*  10 */   0,  0,  0,  0,  1,  1,  2,  2,  3,  3,
 /*  20 */   4,  4,  5,  5,  6,  6,  7,  7,  8,  8,
@@ -3884,19 +3985,19 @@ u32 sqlite3VdbeSerialTypeLen(u32 serial_type){
   if( serial_type>=128 ){
     return (serial_type-12)/2;
   }else{
-    assert( serial_type<12 
+    assert( serial_type<12
             || sqlite3SmallTypeSizes[serial_type]==(serial_type - 12)/2 );
     return sqlite3SmallTypeSizes[serial_type];
   }
 }
 u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){
   assert( serial_type<128 );
-  return sqlite3SmallTypeSizes[serial_type];  
+  return sqlite3SmallTypeSizes[serial_type]; 
 }
 
 /*
-** If we are on an architecture with mixed-endian floating 
-** points (ex: ARM7) then swap the lower 4 bytes with the 
+** If we are on an architecture with mixed-endian floating
+** points (ex: ARM7) then swap the lower 4 bytes with the
 ** upper 4 bytes.  Return the result.
 **
 ** For most architectures, this is a no-op.
@@ -3918,7 +4019,7 @@ u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){
 ** (2007-08-30)  Frank van Vugt has studied this problem closely
 ** and has send his findings to the SQLite developers.  Frank
 ** writes that some Linux kernels offer floating point hardware
-** emulation that uses only 32-bit mantissas instead of a full 
+** emulation that uses only 32-bit mantissas instead of a full
 ** 48-bits as required by the IEEE standard.  (This is the
 ** CONFIG_FPE_FASTFPE option.)  On such systems, floating point
 ** byte swapping becomes very complicated.  To avoid problems,
@@ -3962,7 +4063,7 @@ u64 sqlite3FloatSwap(u64 in){
 ** The few cases that require local variables are broken out into a separate
 ** routine so that in most cases the overhead of moving the stack pointer
 ** is avoided.
-*/ 
+*/
 static void serialGet(
   const unsigned char *buf,     /* Buffer to deserialize from */
   u32 serial_type,              /* Serial type to deserialize */
@@ -3998,6 +4099,23 @@ static void serialGet(
     pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real;
   }
 }
+static int serialGet7(
+  const unsigned char *buf,     /* Buffer to deserialize from */
+  Mem *pMem                     /* Memory cell to write value into */
+){
+  u64 x = FOUR_BYTE_UINT(buf);
+  u32 y = FOUR_BYTE_UINT(buf+4);
+  x = (x<<32) + y;
+  assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 );
+  swapMixedEndianFloat(x);
+  memcpy(&pMem->u.r, &x, sizeof(x));
+  if( IsNaN(x) ){
+    pMem->flags = MEM_Null;
+    return 1;
+  }
+  pMem->flags = MEM_Real;
+  return 0;
+}
 void sqlite3VdbeSerialGet(
   const unsigned char *buf,     /* Buffer to deserialize from */
   u32 serial_type,              /* Serial type to deserialize */
@@ -4045,7 +4163,7 @@ void sqlite3VdbeSerialGet(
       /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit
       ** twos-complement integer. */
       pMem->u.i = FOUR_BYTE_INT(buf);
-#ifdef __HP_cc 
+#ifdef __HP_cc
       /* Work around a sign-extension bug in the HP compiler for HP/UX */
       if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL;
 #endif
@@ -4091,51 +4209,44 @@ void sqlite3VdbeSerialGet(
   return;
 }
 /*
-** This routine is used to allocate sufficient space for an UnpackedRecord
-** structure large enough to be used with sqlite3VdbeRecordUnpack() if
-** the first argument is a pointer to KeyInfo structure pKeyInfo.
-**
-** The space is either allocated using sqlite3DbMallocRaw() or from within
-** the unaligned buffer passed via the second and third arguments (presumably
-** stack space). If the former, then *ppFree is set to a pointer that should
-** be eventually freed by the caller using sqlite3DbFree(). Or, if the 
-** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL
-** before returning.
+** Allocate sufficient space for an UnpackedRecord structure large enough
+** to hold a decoded index record for pKeyInfo.
 **
-** If an OOM error occurs, NULL is returned.
+** The space is allocated using sqlite3DbMallocRaw().  If an OOM error
+** occurs, NULL is returned.
 */
 UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(
   KeyInfo *pKeyInfo               /* Description of the record */
 ){
   UnpackedRecord *p;              /* Unpacked record to return */
-  int nByte;                      /* Number of bytes required for *p */
+  u64 nByte;                      /* Number of bytes required for *p */
+  assert( sizeof(UnpackedRecord) + sizeof(Mem)*65536 < 0x7fffffff );
   nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1);
   p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte);
   if( !p ) return 0;
   p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))];
-  assert( pKeyInfo->aSortFlags!=0 );
   p->pKeyInfo = pKeyInfo;
   p->nField = pKeyInfo->nKeyField + 1;
   return p;
 }
 
 /*
-** Given the nKey-byte encoding of a record in pKey[], populate the 
+** Given the nKey-byte encoding of a record in pKey[], populate the
 ** UnpackedRecord structure indicated by the fourth argument with the
 ** contents of the decoded record.
-*/ 
+*/
 void sqlite3VdbeRecordUnpack(
-  KeyInfo *pKeyInfo,     /* Information about the record format */
   int nKey,              /* Size of the binary record */
   const void *pKey,      /* The binary record */
   UnpackedRecord *p      /* Populate this structure before returning. */
 ){
   const unsigned char *aKey = (const unsigned char *)pKey;
-  u32 d; 
+  u32 d;
   u32 idx;                        /* Offset in aKey[] to read from */
   u16 u;                          /* Unsigned loop counter */
   u32 szHdr;
   Mem *pMem = p->aMem;
+  KeyInfo *pKeyInfo = p->pKeyInfo;
 
   p->default_rc = 0;
   assert( EIGHT_BYTE_ALIGNMENT(pMem) );
@@ -4153,16 +4264,18 @@ void sqlite3VdbeRecordUnpack(
     pMem->z = 0;
     sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem);
     d += sqlite3VdbeSerialTypeLen(serial_type);
-    pMem++;
     if( (++u)>=p->nField ) break;
+    pMem++;
   }
   if( d>(u32)nKey && u ){
     assert( CORRUPT_DB );
-    /* In a corrupt record entry, the last pMem might have been set up using 
+    /* In a corrupt record entry, the last pMem might have been set up using
     ** uninitialized memory. Overwrite its value with NULL, to prevent
     ** warnings from MSAN. */
-    sqlite3VdbeMemSetNull(pMem-1);
+    sqlite3VdbeMemSetNull(pMem-(u<p->nField));
   }
+  testcase( u == pKeyInfo->nKeyField + 1 );
+  testcase( u < pKeyInfo->nKeyField + 1 );
   assert( u<=pKeyInfo->nKeyField + 1 );
   p->nField = u;
 }
@@ -4202,13 +4315,13 @@ static int vdbeRecordCompareDebug(
 
   /* Compilers may complain that mem1.u.i is potentially uninitialized.
   ** We could initialize it, as shown here, to silence those complaints.
-  ** But in fact, mem1.u.i will never actually be used uninitialized, and doing 
+  ** But in fact, mem1.u.i will never actually be used uninitialized, and doing
   ** the unnecessary initialization has a measurable negative performance
   ** impact, since this routine is a very high runner.  And so, we choose
   ** to ignore the compiler warnings and leave this variable uninitialized.
   */
   /*  mem1.u.i = 0;  // not needed, here to silence compiler warning */
-  
+ 
   idx1 = getVarint32(aKey1, szHdr1);
   if( szHdr1>98307 ) return SQLITE_CORRUPT;
   d1 = szHdr1;
@@ -4229,8 +4342,17 @@ static int vdbeRecordCompareDebug(
     ** sqlite3VdbeSerialTypeLen() in the common case.
     */
     if( d1+(u64)serial_type1+2>(u64)nKey1
-     && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 
+     && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1
     ){
+      if( serial_type1>=1
+       && serial_type1<=7
+       && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)<=(u64)nKey1+8
+       && CORRUPT_DB
+      ){
+        return 1;  /* corrupt record not detected by
+                   ** sqlite3VdbeRecordCompareWithSkip().  Return true
+                   ** to avoid firing the assert() */
+      }
       break;
     }
 
@@ -4246,7 +4368,7 @@ static int vdbeRecordCompareDebug(
     if( rc!=0 ){
       assert( mem1.szMalloc==0 );  /* See comment below */
       if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL)
-       && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null)) 
+       && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null))
       ){
         rc = -rc;
       }
@@ -4292,7 +4414,7 @@ static int vdbeRecordCompareDebug(
 ** incorrectly.
 */
 static void vdbeAssertFieldCountWithinLimits(
-  int nKey, const void *pKey,   /* The record to verify */ 
+  int nKey, const void *pKey,   /* The record to verify */
   const KeyInfo *pKeyInfo       /* Compare size with this KeyInfo */
 ){
   int nField = 0;
@@ -4318,9 +4440,35 @@ static void vdbeAssertFieldCountWithinLimits(
 /*
 ** Both *pMem1 and *pMem2 contain string values. Compare the two values
 ** using the collation sequence pColl. As usual, return a negative , zero
-** or positive value if *pMem1 is less than, equal to or greater than 
+** or positive value if *pMem1 is less than, equal to or greater than
 ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);".
 */
+static SQLITE_NOINLINE int vdbeCompareMemStringWithEncodingChange(
+  const Mem *pMem1,
+  const Mem *pMem2,
+  const CollSeq *pColl,
+  u8 *prcErr                      /* If an OOM occurs, set to SQLITE_NOMEM */
+){
+  int rc;
+  const void *v1, *v2;
+  Mem c1;
+  Mem c2;
+  sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null);
+  sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null);
+  sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem);
+  sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem);
+  v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc);
+  v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc);
+  if( (v1==0 || v2==0) ){
+    if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT;
+    rc = 0;
+  }else{
+    rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2);
+  }
+  sqlite3VdbeMemReleaseMalloc(&c1);
+  sqlite3VdbeMemReleaseMalloc(&c2);
+  return rc;
+}
 static int vdbeCompareMemString(
   const Mem *pMem1,
   const Mem *pMem2,
@@ -4332,25 +4480,7 @@ static int vdbeCompareMemString(
      ** comparison function directly */
     return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z);
   }else{
-    int rc;
-    const void *v1, *v2;
-    Mem c1;
-    Mem c2;
-    sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null);
-    sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null);
-    sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem);
-    sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem);
-    v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc);
-    v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc);
-    if( (v1==0 || v2==0) ){
-      if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT;
-      rc = 0;
-    }else{
-      rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2);
-    }
-    sqlite3VdbeMemReleaseMalloc(&c1);
-    sqlite3VdbeMemReleaseMalloc(&c2);
-    return rc;
+    return vdbeCompareMemStringWithEncodingChange(pMem1,pMem2,pColl,prcErr);
   }
 }
 
@@ -4399,32 +4529,37 @@ SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){
   return n1 - n2;
 }
 
+/* The following two functions are used only within testcase() to prove
+** test coverage.  These functions do no exist for production builds.
+** We must use separate SQLITE_NOINLINE functions here, since otherwise
+** optimizer code movement causes gcov to become very confused.
+*/
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG)
+static int SQLITE_NOINLINE doubleLt(double a, double b){ return a<b; }
+static int SQLITE_NOINLINE doubleEq(double a, double b){ return a==b; }
+#endif
+
 /*
 ** Do a comparison between a 64-bit signed integer and a 64-bit floating-point
 ** number.  Return negative, zero, or positive if the first (i64) is less than,
 ** equal to, or greater than the second (double).
 */
 int sqlite3IntFloatCompare(i64 i, double r){
-  if( sizeof(LONGDOUBLE_TYPE)>8 ){
-    LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i;
-    testcase( x<r );
-    testcase( x>r );
-    testcase( x==r );
-    if( x<r ) return -1;
-    if( x>r ) return +1;  /*NO_TEST*/ /* work around bugs in gcov */
-    return 0;             /*NO_TEST*/ /* work around bugs in gcov */
+  if( sqlite3IsNaN(r) ){
+    /* SQLite considers NaN to be a NULL. And all integer values are greater
+    ** than NULL */
+    return 1;
   }else{
     i64 y;
-    double s;
     if( r<-9223372036854775808.0 ) return +1;
     if( r>=9223372036854775808.0 ) return -1;
     y = (i64)r;
     if( i<y ) return -1;
     if( i>y ) return +1;
-    s = (double)i;
-    if( s<r ) return -1;
-    if( s>r ) return +1;
-    return 0;
+    testcase( doubleLt(((double)i),r) );
+    testcase( doubleLt(r,((double)i)) );
+    testcase( doubleEq(r,((double)i)) );
+    return (((double)i)<r) ? -1 : (((double)i)>r);
   }
 }
 
@@ -4445,7 +4580,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
   f2 = pMem2->flags;
   combined_flags = f1|f2;
   assert( !sqlite3VdbeMemIsRowSet(pMem1) && !sqlite3VdbeMemIsRowSet(pMem2) );
- 
+
   /* If one value is NULL, it is less than the other. If both values
   ** are NULL, return 0.
   */
@@ -4508,7 +4643,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
     }
 
     assert( pMem1->enc==pMem2->enc || pMem1->db->mallocFailed );
-    assert( pMem1->enc==SQLITE_UTF8 || 
+    assert( pMem1->enc==SQLITE_UTF8 ||
             pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE );
 
     /* The collation sequence must be defined at this point, even if
@@ -4523,7 +4658,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
     /* If a NULL pointer was passed as the collate function, fall through
     ** to the blob case and use memcmp().  */
   }
- 
+
   /* Both values must be blobs.  Compare using memcmp().  */
   return sqlite3BlobCompare(pMem1, pMem2);
 }
@@ -4531,7 +4666,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
 
 /*
 ** The first argument passed to this function is a serial-type that
-** corresponds to an integer - all values between 1 and 9 inclusive 
+** corresponds to an integer - all values between 1 and 9 inclusive
 ** except 7. The second points to a buffer containing an integer value
 ** serialized according to serial_type. This function deserializes
 ** and returns the value.
@@ -4573,7 +4708,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){
 /*
 ** This function compares the two table rows or index records
 ** specified by {nKey1, pKey1} and pPKey2.  It returns a negative, zero
-** or positive integer if key1 is less than, equal to or 
+** or positive integer if key1 is less than, equal to or
 ** greater than key2.  The {nKey1, pKey1} key must be a blob
 ** created by the OP_MakeRecord opcode of the VDBE.  The pPKey2
 ** key must be a parsed key such as obtained from
@@ -4582,12 +4717,12 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){
 ** If argument bSkip is non-zero, it is assumed that the caller has already
 ** determined that the first fields of the keys are equal.
 **
-** Key1 and Key2 do not have to contain the same number of fields. If all 
-** fields that appear in both keys are equal, then pPKey2->default_rc is 
+** Key1 and Key2 do not have to contain the same number of fields. If all
+** fields that appear in both keys are equal, then pPKey2->default_rc is
 ** returned.
 **
-** If database corruption is discovered, set pPKey2->errCode to 
-** SQLITE_CORRUPT and return 0. If an OOM error is encountered, 
+** If database corruption is discovered, set pPKey2->errCode to
+** SQLITE_CORRUPT and return 0. If an OOM error is encountered,
 ** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the
 ** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db).
 */
@@ -4629,13 +4764,13 @@ int sqlite3VdbeRecordCompareWithSkip(
     d1 = szHdr1;
     i = 0;
   }
-  if( d1>(unsigned)nKey1 ){ 
+  if( d1>(unsigned)nKey1 ){
     pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT;
     return 0;  /* Corruption */
   }
 
   VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */
-  assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField 
+  assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField
        || CORRUPT_DB );
   assert( pPKey2->pKeyInfo->aSortFlags!=0 );
   assert( pPKey2->pKeyInfo->nKeyField>0 );
@@ -4654,7 +4789,7 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else if( serial_type==7 ){
-        sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
+        serialGet7(&aKey1[d1], &mem1);
         rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r);
       }else{
         i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]);
@@ -4672,21 +4807,25 @@ int sqlite3VdbeRecordCompareWithSkip(
       serial_type = aKey1[idx1];
       if( serial_type>=10 ){
         /* Serial types 12 or greater are strings and blobs (greater than
-        ** numbers). Types 10 and 11 are currently "reserved for future 
+        ** numbers). Types 10 and 11 are currently "reserved for future
         ** use", so it doesn't really matter what the results of comparing
-        ** them to numberic values are.  */
+        ** them to numeric values are.  */
         rc = serial_type==10 ? -1 : +1;
       }else if( serial_type==0 ){
         rc = -1;
       }else{
-        sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
         if( serial_type==7 ){
-          if( mem1.u.r<pRhs->u.r ){
+          if( serialGet7(&aKey1[d1], &mem1) ){
+            rc = -1;  /* mem1 is a NaN */
+          }else if( mem1.u.r<pRhs->u.r ){
             rc = -1;
           }else if( mem1.u.r>pRhs->u.r ){
             rc = +1;
+          }else{
+            assert( rc==0 );
           }
         }else{
+          sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
           rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r);
         }
       }
@@ -4720,7 +4859,7 @@ int sqlite3VdbeRecordCompareWithSkip(
         }else{
           int nCmp = MIN(mem1.n, pRhs->n);
           rc = memcmp(&aKey1[d1], pRhs->z, nCmp);
-          if( rc==0 ) rc = mem1.n - pRhs->n; 
+          if( rc==0 ) rc = mem1.n - pRhs->n;
         }
       }
     }
@@ -4756,7 +4895,14 @@ int sqlite3VdbeRecordCompareWithSkip(
     /* RHS is null */
     else{
       serial_type = aKey1[idx1];
-      rc = (serial_type!=0 && serial_type!=10);
+      if( serial_type==0
+       || serial_type==10
+       || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0)
+      ){
+        assert( rc==0 );
+      }else{
+        rc = 1;
+      }
     }
 
     if( rc!=0 ){
@@ -4794,8 +4940,8 @@ int sqlite3VdbeRecordCompareWithSkip(
   /* rc==0 here means that one or both of the keys ran out of fields and
   ** all the fields up to that point were equal. Return the default_rc
   ** value.  */
-  assert( CORRUPT_DB 
-       || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) 
+  assert( CORRUPT_DB
+       || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc)
        || pPKey2->pKeyInfo->db->mallocFailed
   );
   pPKey2->eqSeen = 1;
@@ -4810,8 +4956,8 @@ int sqlite3VdbeRecordCompare(
 
 
 /*
-** This function is an optimized version of sqlite3VdbeRecordCompare() 
-** that (a) the first field of pPKey2 is an integer, and (b) the 
+** This function is an optimized version of sqlite3VdbeRecordCompare()
+** that (a) the first field of pPKey2 is an integer, and (b) the
 ** size-of-header varint at the start of (pKey1/nKey1) fits in a single
 ** byte (i.e. is less than 128).
 **
@@ -4866,7 +5012,7 @@ static int vdbeRecordCompareInt(
       testcase( lhs<0 );
       break;
     }
-    case 8: 
+    case 8:
       lhs = 0;
       break;
     case 9:
@@ -4874,11 +5020,11 @@ static int vdbeRecordCompareInt(
       break;
 
     /* This case could be removed without changing the results of running
-    ** this code. Including it causes gcc to generate a faster switch 
+    ** this code. Including it causes gcc to generate a faster switch
     ** statement (since the range of switch targets now starts at zero and
     ** is contiguous) but does not cause any duplicate code to be generated
-    ** (as gcc is clever enough to combine the two like cases). Other 
-    ** compilers might be similar.  */ 
+    ** (as gcc is clever enough to combine the two like cases). Other
+    ** compilers might be similar.  */
     case 0: case 7:
       return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2);
 
@@ -4893,7 +5039,7 @@ static int vdbeRecordCompareInt(
   }else if( v<lhs ){
     res = pPKey2->r2;
   }else if( pPKey2->nField>1 ){
-    /* The first fields of the two keys are equal. Compare the trailing 
+    /* The first fields of the two keys are equal. Compare the trailing
     ** fields.  */
     res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1);
   }else{
@@ -4908,9 +5054,9 @@ static int vdbeRecordCompareInt(
 }
 
 /*
-** This function is an optimized version of sqlite3VdbeRecordCompare() 
+** This function is an optimized version of sqlite3VdbeRecordCompare()
 ** that (a) the first field of pPKey2 is a string, that (b) the first field
-** uses the collation sequence BINARY and (c) that the size-of-header varint 
+** uses the collation sequence BINARY and (c) that the size-of-header varint
 ** at the start of (pKey1/nKey1) fits in a single byte.
 */
 static int vdbeRecordCompareString(
@@ -4935,7 +5081,7 @@ static int vdbeRecordCompareString(
       assert( CORRUPT_DB );
     }
     res = pPKey2->r1;      /* (pKey1/nKey1) is a number or a null */
-  }else if( !(serial_type & 0x01) ){ 
+  }else if( !(serial_type & 0x01) ){
     res = pPKey2->r2;      /* (pKey1/nKey1) is a blob */
   }else{
     int nCmp;
@@ -4987,7 +5133,7 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){
   /* varintRecordCompareInt() and varintRecordCompareString() both assume
   ** that the size-of-header varint that occurs at the start of each record
   ** fits in a single byte (i.e. is 127 or less). varintRecordCompareInt()
-  ** also assumes that it is safe to overread a buffer by at least the 
+  ** also assumes that it is safe to overread a buffer by at least the
   ** maximum possible legal header size plus 8 bytes. Because there is
   ** guaranteed to be at least 74 (but not 136) bytes of padding following each
   ** buffer passed to varintRecordCompareInt() this makes it convenient to
@@ -4997,6 +5143,7 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){
   ** The easiest way to enforce this limit is to consider only records with
   ** 13 fields or less. If the first field is an integer, the maximum legal
   ** header size is (12*5 + 1 + 1) bytes.  */
+  assert( p->pKeyInfo->aSortFlags!=0 );
   if( p->pKeyInfo->nAllField<=13 ){
     int flags = p->aMem[0].flags;
     if( p->pKeyInfo->aSortFlags[0] ){
@@ -5048,7 +5195,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){
   /* Get the size of the index entry.  Only indices entries of less
   ** than 2GiB are support - anything large must be database corruption.
   ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so
-  ** this code can safely assume that nCellKey is 32-bits  
+  ** this code can safely assume that nCellKey is 32-bits 
   */
   assert( sqlite3BtreeCursorIsValid(pCur) );
   nCellKey = sqlite3BtreePayloadSize(pCur);
@@ -5113,7 +5260,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){
 **
 ** pUnpacked is either created without a rowid or is truncated so that it
 ** omits the rowid at the end.  The rowid at the end of the index entry
-** is ignored as well.  Hence, this routine only compares the prefixes 
+** is ignored as well.  Hence, this routine only compares the prefixes
 ** of the keys prior to the final rowid, not the entire key.
 */
 int sqlite3VdbeIdxKeyCompare(
@@ -5149,7 +5296,7 @@ int sqlite3VdbeIdxKeyCompare(
 
 /*
 ** This routine sets the value to be returned by subsequent calls to
-** sqlite3_changes() on the database handle 'db'. 
+** sqlite3_changes() on the database handle 'db'.
 */
 void sqlite3VdbeSetChanges(sqlite3 *db, i64 nChange){
   assert( sqlite3_mutex_held(db->mutex) );
@@ -5206,7 +5353,7 @@ u8 sqlite3VdbePrepareFlags(Vdbe *v){
 
 /*
 ** Return a pointer to an sqlite3_value structure containing the value bound
-** parameter iVar of VM v. Except, if the value is an SQL NULL, return 
+** parameter iVar of VM v. Except, if the value is an SQL NULL, return
 ** 0 instead. Unless it is NULL, apply affinity aff (one of the SQLITE_AFF_*
 ** constants) to the value before returning it.
 **
@@ -5216,7 +5363,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){
   assert( iVar>0 );
   if( v ){
     Mem *pMem = &v->aVar[iVar-1];
-    assert( (v->db->flags & SQLITE_EnableQPSG)==0 );
+    assert( (v->db->flags & SQLITE_EnableQPSG)==0 
+         || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 );
     if( 0==(pMem->flags & MEM_Null) ){
       sqlite3_value *pRet = sqlite3ValueNew(v->db);
       if( pRet ){
@@ -5236,7 +5384,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){
 */
 void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){
   assert( iVar>0 );
-  assert( (v->db->flags & SQLITE_EnableQPSG)==0 );
+  assert( (v->db->flags & SQLITE_EnableQPSG)==0 
+       || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 );
   if( iVar>=32 ){
     v->expmask |= 0x80000000;
   }else{
@@ -5244,6 +5393,7 @@ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){
   }
 }
 
+#ifndef SQLITE_OMIT_DATETIME_FUNCS
 /*
 ** Cause a function to throw an error if it was call from OP_PureFunc
 ** rather than OP_Function.
@@ -5277,11 +5427,12 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){
   }
   return 1;
 }
+#endif /* SQLITE_OMIT_DATETIME_FUNCS */
 
 #if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG)
 /*
 ** This Walker callback is used to help verify that calls to
-** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have 
+** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have
 ** byte-code register values correctly initialized.
 */
 int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){
@@ -5312,7 +5463,7 @@ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
 
 /*
-** If the second argument is not NULL, release any allocations associated 
+** If the second argument is not NULL, release any allocations associated
 ** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord
 ** structure itself, using sqlite3DbFree().
 **
@@ -5353,7 +5504,6 @@ void sqlite3VdbePreUpdateHook(
   i64 iKey2;
   PreUpdate preupdate;
   const char *zTbl = pTab->zName;
-  static const u8 fakeSortOrder = 0;
 #ifdef SQLITE_DEBUG
   int nRealCol;
   if( pTab->tabFlags & TF_WithoutRowid ){
@@ -5380,7 +5530,7 @@ void sqlite3VdbePreUpdateHook(
 
   assert( pCsr!=0 );
   assert( pCsr->eCurType==CURTYPE_BTREE );
-  assert( pCsr->nField==nRealCol 
+  assert( pCsr->nField==nRealCol
        || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1)
   );
 
@@ -5388,10 +5538,11 @@ void sqlite3VdbePreUpdateHook(
   preupdate.pCsr = pCsr;
   preupdate.op = op;
   preupdate.iNewReg = iReg;
-  preupdate.keyinfo.db = db;
-  preupdate.keyinfo.enc = ENC(db);
-  preupdate.keyinfo.nKeyField = pTab->nCol;
-  preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder;
+  preupdate.pKeyinfo = (KeyInfo*)&preupdate.uKey;
+  preupdate.pKeyinfo->db = db;
+  preupdate.pKeyinfo->enc = ENC(db);
+  preupdate.pKeyinfo->nKeyField = pTab->nCol;
+  preupdate.pKeyinfo->aSortFlags = 0; /* Indicate .aColl, .nAllField uninit */
   preupdate.iKey1 = iKey1;
   preupdate.iKey2 = iKey2;
   preupdate.pTab = pTab;
@@ -5401,8 +5552,9 @@ void sqlite3VdbePreUpdateHook(
   db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2);
   db->pPreUpdate = 0;
   sqlite3DbFree(db, preupdate.aRecord);
-  vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked);
-  vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked);
+  vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pUnpacked);
+  vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pNewUnpacked);
+  sqlite3VdbeMemRelease(&preupdate.oldipk);
   if( preupdate.aNew ){
     int i;
     for(i=0; i<pCsr->nField; i++){
@@ -5410,5 +5562,23 @@ void sqlite3VdbePreUpdateHook(
     }
     sqlite3DbNNFreeNN(db, preupdate.aNew);
   }
+  if( preupdate.apDflt ){
+    int i;
+    for(i=0; i<pTab->nCol; i++){
+      sqlite3ValueFree(preupdate.apDflt[i]);
+    }
+    sqlite3DbFree(db, preupdate.apDflt);
+  }
 }
 #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PERCENTILE
+/*
+** Return the name of an SQL function associated with the sqlite3_context.
+*/
+const char *sqlite3VdbeFuncName(const sqlite3_context *pCtx){
+  assert( pCtx!=0 );
+  assert( pCtx->pFunc!=0 );
+  return pCtx->pFunc->zName;
+}
+#endif /* SQLITE_ENABLE_PERCENTILE */
diff --git a/src/vdbeblob.c b/src/vdbeblob.c
index 32987da137..a15fec6c48 100644
--- a/src/vdbeblob.c
+++ b/src/vdbeblob.c
@@ -59,8 +59,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
   /* Set the value of register r[1] in the SQL statement to integer iRow. 
   ** This is done directly as a performance optimization
   */
-  v->aMem[1].flags = MEM_Int;
-  v->aMem[1].u.i = iRow;
+  sqlite3VdbeMemSetInt64(&v->aMem[1], iRow);
 
   /* If the statement has been run before (and is paused at the OP_ResultRow)
   ** then back it up to the point where it does the OP_NotExists.  This could
@@ -134,6 +133,7 @@ int sqlite3_blob_open(
   char *zErr = 0;
   Table *pTab;
   Incrblob *pBlob = 0;
+  int iDb;
   Parse sParse;
 
 #ifdef SQLITE_ENABLE_API_ARMOR
@@ -143,7 +143,7 @@ int sqlite3_blob_open(
 #endif
   *ppBlob = 0;
 #ifdef SQLITE_ENABLE_API_ARMOR
-  if( !sqlite3SafetyCheckOk(db) || zTable==0 ){
+  if( !sqlite3SafetyCheckOk(db) || zTable==0 || zColumn==0 ){
     return SQLITE_MISUSE_BKPT;
   }
 #endif
@@ -168,13 +168,21 @@ int sqlite3_blob_open(
       pTab = 0;
       sqlite3ErrorMsg(&sParse, "cannot open table without rowid: %s", zTable);
     }
+    if( pTab && (pTab->tabFlags&TF_HasGenerated)!=0 ){
+      pTab = 0;
+      sqlite3ErrorMsg(&sParse, "cannot open table with generated columns: %s",
+                      zTable);
+    }
 #ifndef SQLITE_OMIT_VIEW
     if( pTab && IsView(pTab) ){
       pTab = 0;
       sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable);
     }
 #endif
-    if( !pTab ){
+    if( pTab==0
+     || ((iDb = sqlite3SchemaToIndex(db, pTab->pSchema))==1 &&
+         sqlite3OpenTempDatabase(&sParse))
+    ){
       if( sParse.zErrMsg ){
         sqlite3DbFree(db, zErr);
         zErr = sParse.zErrMsg;
@@ -185,15 +193,11 @@ int sqlite3_blob_open(
       goto blob_open_out;
     }
     pBlob->pTab = pTab;
-    pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName;
+    pBlob->zDb = db->aDb[iDb].zDbSName;
 
     /* Now search pTab for the exact column. */
-    for(iCol=0; iCol<pTab->nCol; iCol++) {
-      if( sqlite3StrICmp(pTab->aCol[iCol].zCnName, zColumn)==0 ){
-        break;
-      }
-    }
-    if( iCol==pTab->nCol ){
+    iCol = sqlite3ColumnIndex(pTab, zColumn);
+    if( iCol<0 ){
       sqlite3DbFree(db, zErr);
       zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn);
       rc = SQLITE_ERROR;
@@ -273,7 +277,6 @@ int sqlite3_blob_open(
         {OP_Halt,           0, 0, 0},  /* 5  */
       };
       Vdbe *v = (Vdbe *)pBlob->pStmt;
-      int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
       VdbeOp *aOp;
 
       sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, 
@@ -382,7 +385,7 @@ static int blobReadWrite(
   int iOffset, 
   int (*xCall)(BtCursor*, u32, u32, void*)
 ){
-  int rc;
+  int rc = SQLITE_OK;
   Incrblob *p = (Incrblob *)pBlob;
   Vdbe *v;
   sqlite3 *db;
@@ -422,17 +425,32 @@ static int blobReadWrite(
       ** using the incremental-blob API, this works. For the sessions module
       ** anyhow.
       */
-      sqlite3_int64 iKey;
-      iKey = sqlite3BtreeIntegerKey(p->pCsr);
-      assert( v->apCsr[0]!=0 );
-      assert( v->apCsr[0]->eCurType==CURTYPE_BTREE );
-      sqlite3VdbePreUpdateHook(
-          v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol
-      );
+      if( sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ){
+        /* If the cursor is not currently valid, try to reseek it. This 
+        ** always either fails or finds the correct row - the cursor will
+        ** have been marked permanently CURSOR_INVALID if the open row has
+        ** been deleted.  */
+        int bDiff = 0;
+        rc = sqlite3BtreeCursorRestore(p->pCsr, &bDiff);
+        assert( bDiff==0 || sqlite3BtreeCursorIsValidNN(p->pCsr)==0 );
+      }
+      if( sqlite3BtreeCursorIsValidNN(p->pCsr) ){
+        sqlite3_int64 iKey;
+        iKey = sqlite3BtreeIntegerKey(p->pCsr);
+        assert( v->apCsr[0]!=0 );
+        assert( v->apCsr[0]->eCurType==CURTYPE_BTREE );
+        sqlite3VdbePreUpdateHook(
+            v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol
+        );
+      }
+    }
+    if( rc==SQLITE_OK ){
+      rc = xCall(p->pCsr, iOffset+p->iOffset, n, z);
     }
+#else
+    rc = xCall(p->pCsr, iOffset+p->iOffset, n, z);
 #endif
 
-    rc = xCall(p->pCsr, iOffset+p->iOffset, n, z);
     sqlite3BtreeLeaveCursor(p->pCsr);
     if( rc==SQLITE_ABORT ){
       sqlite3VdbeFinalize(v);
diff --git a/src/vdbemem.c b/src/vdbemem.c
index d3cd55ba9f..2c4d1c5681 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -141,7 +141,7 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){
 ** corresponding string value, then it is important that the string be
 ** derived from the numeric value, not the other way around, to ensure
 ** that the index and table are consistent.  See ticket
-** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for
+** https://sqlite.org/src/info/343634942dd54ab (2018-01-31) for
 ** an example.
 **
 ** This routine looks at pMem to verify that if it has both a numeric
@@ -314,6 +314,40 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){
   return SQLITE_OK;
 }
 
+/*
+** If pMem is already a string, detect if it is a zero-terminated
+** string, or make it into one if possible, and mark it as such.
+**
+** This is an optimization.  Correct operation continues even if
+** this routine is a no-op.
+*/
+void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){
+  if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){
+    /* pMem must be a string, and it cannot be an ephemeral or static string */
+    return;
+  }
+  if( pMem->enc!=SQLITE_UTF8 ) return;
+  assert( pMem->z!=0 );
+  if( pMem->flags & MEM_Dyn ){
+    if( pMem->xDel==sqlite3_free
+     && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1)
+    ){
+      pMem->z[pMem->n] = 0;
+      pMem->flags |= MEM_Term;
+      return;
+    }
+    if( pMem->xDel==sqlite3RCStrUnref ){
+      /* Blindly assume that all RCStr objects are zero-terminated */
+      pMem->flags |= MEM_Term;
+      return;
+    }
+  }else if( pMem->szMalloc >= pMem->n+1 ){
+    pMem->z[pMem->n] = 0;
+    pMem->flags |= MEM_Term;
+    return;
+  }
+}
+
 /*
 ** It is already known that pMem contains an unterminated string.
 ** Add the zero terminator.
@@ -575,36 +609,6 @@ void sqlite3VdbeMemReleaseMalloc(Mem *p){
   if( p->szMalloc ) vdbeMemClear(p);
 }
 
-/*
-** Convert a 64-bit IEEE double into a 64-bit signed integer.
-** If the double is out of range of a 64-bit signed integer then
-** return the closest available 64-bit signed integer.
-*/
-static SQLITE_NOINLINE i64 doubleToInt64(double r){
-#ifdef SQLITE_OMIT_FLOATING_POINT
-  /* When floating-point is omitted, double and int64 are the same thing */
-  return r;
-#else
-  /*
-  ** Many compilers we encounter do not define constants for the
-  ** minimum and maximum 64-bit integers, or they define them
-  ** inconsistently.  And many do not understand the "LL" notation.
-  ** So we define our own static constants here using nothing
-  ** larger than a 32-bit integer constant.
-  */
-  static const i64 maxInt = LARGEST_INT64;
-  static const i64 minInt = SMALLEST_INT64;
-
-  if( r<=(double)minInt ){
-    return minInt;
-  }else if( r>=(double)maxInt ){
-    return maxInt;
-  }else{
-    return (i64)r;
-  }
-#endif
-}
-
 /*
 ** Return some kind of integer value which is the best we can do
 ** at representing the value that *pMem describes as an integer.
@@ -631,7 +635,7 @@ i64 sqlite3VdbeIntValue(const Mem *pMem){
     testcase( flags & MEM_IntReal );
     return pMem->u.i;
   }else if( flags & MEM_Real ){
-    return doubleToInt64(pMem->u.r);
+    return sqlite3RealToI64(pMem->u.r);
   }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){
     return memIntValue(pMem);
   }else{
@@ -693,7 +697,7 @@ void sqlite3VdbeIntegerAffinity(Mem *pMem){
   if( pMem->flags & MEM_IntReal ){
     MemSetTypeFlag(pMem, MEM_Int);
   }else{
-    i64 ix = doubleToInt64(pMem->u.r);
+    i64 ix = sqlite3RealToI64(pMem->u.r);
 
     /* Only mark the value as an integer if
     **
@@ -761,8 +765,8 @@ int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){
 ** from UBSAN.
 */
 i64 sqlite3RealToI64(double r){
-  if( r<=(double)SMALLEST_INT64 ) return SMALLEST_INT64;
-  if( r>=(double)LARGEST_INT64) return LARGEST_INT64;
+  if( r<-9223372036854774784.0 ) return SMALLEST_INT64;
+  if( r>+9223372036854774784.0 ) return LARGEST_INT64;
   return (i64)r;
 }
 
@@ -833,6 +837,7 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){
       break;
     }
     default: {
+      int rc;
       assert( aff==SQLITE_AFF_TEXT );
       assert( MEM_Str==(MEM_Blob>>3) );
       pMem->flags |= (pMem->flags&MEM_Blob)>>3;
@@ -840,7 +845,9 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){
       assert( pMem->flags & MEM_Str || pMem->db->mallocFailed );
       pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero);
       if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1;
-      return sqlite3VdbeChangeEncoding(pMem, encoding);
+      rc = sqlite3VdbeChangeEncoding(pMem, encoding);
+      if( rc ) return rc;
+      sqlite3VdbeMemZeroTerminateIfAble(pMem);
     }
   }
   return SQLITE_OK;
@@ -936,6 +943,13 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){
   }
 }
 
+/*
+** Set the iIdx'th entry of array aMem[] to contain integer value val.
+*/
+void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){
+  sqlite3VdbeMemSetInt64(&aMem[iIdx], val);
+}
+
 /* A no-op destructor */
 void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); }
 
@@ -1032,27 +1046,30 @@ int sqlite3VdbeMemTooBig(Mem *p){
 void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){
   int i;
   Mem *pX;
-  for(i=1, pX=pVdbe->aMem+1; i<pVdbe->nMem; i++, pX++){
-    if( pX->pScopyFrom==pMem ){
-      u16 mFlags;
-      if( pVdbe->db->flags & SQLITE_VdbeTrace ){
-        sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n",
-          (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem));
+  if( pMem->bScopy ){
+    for(i=1, pX=pVdbe->aMem+1; i<pVdbe->nMem; i++, pX++){
+      if( pX->pScopyFrom==pMem ){
+        u16 mFlags;
+        if( pVdbe->db->flags & SQLITE_VdbeTrace ){
+          sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n",
+            (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem));
+        }
+        /* If pX is marked as a shallow copy of pMem, then try to verify that
+        ** no significant changes have been made to pX since the OP_SCopy.
+        ** A significant change would indicated a missed call to this
+        ** function for pX.  Minor changes, such as adding or removing a
+        ** dual type, are allowed, as long as the underlying value is the
+        ** same. */
+        mFlags = pMem->flags & pX->flags & pX->mScopyFlags;
+        assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i );
+        
+        /* pMem is the register that is changing.  But also mark pX as
+        ** undefined so that we can quickly detect the shallow-copy error */
+        pX->flags = MEM_Undefined;
+        pX->pScopyFrom = 0;
       }
-      /* If pX is marked as a shallow copy of pMem, then try to verify that
-      ** no significant changes have been made to pX since the OP_SCopy.
-      ** A significant change would indicated a missed call to this
-      ** function for pX.  Minor changes, such as adding or removing a
-      ** dual type, are allowed, as long as the underlying value is the
-      ** same. */
-      mFlags = pMem->flags & pX->flags & pX->mScopyFlags;
-      assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i );
-      
-      /* pMem is the register that is changing.  But also mark pX as
-      ** undefined so that we can quickly detect the shallow-copy error */
-      pX->flags = MEM_Undefined;
-      pX->pScopyFrom = 0;
     }
+    pMem->bScopy = 0;
   }
   pMem->pScopyFrom = 0;
 }
@@ -1209,6 +1226,7 @@ int sqlite3VdbeMemSetStr(
     if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){
       return SQLITE_NOMEM_BKPT;
     }
+    assert( pMem->z!=0 );
     memcpy(pMem->z, z, nAlloc);
   }else{
     sqlite3VdbeMemRelease(pMem);
@@ -1364,6 +1382,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){
   return valueToText(pVal, enc);
 }
 
+/* Return true if sqlit3_value object pVal is a string or blob value
+** that uses the destructor specified in the second argument.
+**
+** TODO:  Maybe someday promote this interface into a published API so
+** that third-party extensions can get access to it?
+*/
+int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){
+  if( ALWAYS(pVal!=0)
+   && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0)
+   && (pVal->flags & MEM_Dyn)!=0
+   && pVal->xDel==xFree
+  ){
+    return 1;
+  }else{
+    return 0;
+  }
+}
+
 /*
 ** Create a new sqlite3_value object.
 */
@@ -1405,7 +1441,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){
 
     if( pRec==0 ){
       Index *pIdx = p->pIdx;      /* Index being probed */
-      int nByte;                  /* Bytes of space to allocate */
+      i64 nByte;                  /* Bytes of space to allocate */
       int i;                      /* Counter variable */
       int nCol = pIdx->nColumn;   /* Number of index columns including rowid */
   
@@ -1431,6 +1467,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){
     }
   
     pRec->nField = p->iVal+1;
+    sqlite3VdbeMemSetNull(&pRec->aMem[p->iVal]);
     return &pRec->aMem[p->iVal];
   }
 #else
@@ -1470,7 +1507,7 @@ static int valueFromFunction(
 ){
   sqlite3_context ctx;            /* Context object for function invocation */
   sqlite3_value **apVal = 0;      /* Function arguments */
-  int nVal = 0;                   /* Size of apVal[] array */
+  int nVal = 0;                   /* Number of function arguments */
   FuncDef *pFunc = 0;             /* Function definition */
   sqlite3_value *pVal = 0;        /* New value */
   int rc = SQLITE_OK;             /* Return code */
@@ -1489,7 +1526,7 @@ static int valueFromFunction(
 #endif
   assert( pFunc );
   if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0
-   || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
+   || (pFunc->funcFlags & (SQLITE_FUNC_NEEDCOLL|SQLITE_FUNC_RUNONLY))!=0
   ){
     return SQLITE_OK;
   }
@@ -1501,7 +1538,8 @@ static int valueFromFunction(
       goto value_from_function_out;
     }
     for(i=0; i<nVal; i++){
-      rc = sqlite3ValueFromExpr(db, pList->a[i].pExpr, enc, aff, &apVal[i]);
+      rc = sqlite3Stat4ValueFromExpr(pCtx->pParse, pList->a[i].pExpr, aff,
+                                     &apVal[i]);
       if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out;
     }
   }
@@ -1605,14 +1643,20 @@ static int valueFromExpr(
   }
 
   /* Handle negative integers in a single step.  This is needed in the
-  ** case when the value is -9223372036854775808.
-  */
-  if( op==TK_UMINUS
-   && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){
-    pExpr = pExpr->pLeft;
-    op = pExpr->op;
-    negInt = -1;
-    zNeg = "-";
+  ** case when the value is -9223372036854775808. Except - do not do this
+  ** for hexadecimal literals.  */
+  if( op==TK_UMINUS ){
+    Expr *pLeft = pExpr->pLeft;
+    if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){
+      if( ExprHasProperty(pLeft, EP_IntValue)
+       || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X'
+      ){
+        pExpr = pLeft;
+        op = pExpr->op;
+        negInt = -1;
+        zNeg = "-";
+      }
+    }
   }
 
   if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
@@ -1621,12 +1665,26 @@ static int valueFromExpr(
     if( ExprHasProperty(pExpr, EP_IntValue) ){
       sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt);
     }else{
-      zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken);
-      if( zVal==0 ) goto no_mem;
-      sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC);
+      i64 iVal;
+      if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){
+        sqlite3VdbeMemSetInt64(pVal, iVal*negInt);
+      }else{
+        zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken);
+        if( zVal==0 ) goto no_mem;
+        sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC);
+      }
     }
-    if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){
-      sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8);
+    if( affinity==SQLITE_AFF_BLOB ){
+      if( op==TK_FLOAT ){
+        assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) );
+        sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8);
+        pVal->flags = MEM_Real;
+      }else if( op==TK_INTEGER ){
+        /* This case is required by -9223372036854775808 and other strings
+        ** that look like integers but cannot be handled by the
+        ** sqlite3DecOrHexToI64() call above.  */
+        sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8);
+      }
     }else{
       sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8);
     }
@@ -1690,6 +1748,7 @@ static int valueFromExpr(
     if( pVal ){
       pVal->flags = MEM_Int;
       pVal->u.i = pExpr->u.zToken[4]==0;
+      sqlite3ValueApplyAffinity(pVal, affinity, enc);
     }
   }
 
@@ -1895,17 +1954,17 @@ int sqlite3Stat4Column(
   sqlite3_value **ppVal           /* OUT: Extracted value */
 ){
   u32 t = 0;                      /* a column type code */
-  int nHdr;                       /* Size of the header in the record */
-  int iHdr;                       /* Next unread header byte */
-  int iField;                     /* Next unread data byte */
-  int szField = 0;                /* Size of the current data field */
+  u32 nHdr;                       /* Size of the header in the record */
+  u32 iHdr;                       /* Next unread header byte */
+  i64 iField;                     /* Next unread data byte */
+  u32 szField = 0;                /* Size of the current data field */
   int i;                          /* Column index */
   u8 *a = (u8*)pRec;              /* Typecast byte array */
   Mem *pMem = *ppVal;             /* Write result into this Mem object */
 
   assert( iCol>0 );
   iHdr = getVarint32(a, nHdr);
-  if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT;
+  if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT;
   iField = nHdr;
   for(i=0; i<=iCol; i++){
     iHdr += getVarint32(&a[iHdr], t);
diff --git a/src/vdbesort.c b/src/vdbesort.c
index 3958662cc6..73f3301116 100644
--- a/src/vdbesort.c
+++ b/src/vdbesort.c
@@ -55,7 +55,7 @@
 **                                  is like Close() followed by Init() only
 **                                  much faster.
 **
-** The interfaces above must be called in a particular order.  Write() can 
+** The interfaces above must be called in a particular order.  Write() can
 ** only occur in between Init()/Reset() and Rewind().  Next(), Rowkey(), and
 ** Compare() can only occur in between Rewind() and Close()/Reset(). i.e.
 **
@@ -63,16 +63,16 @@
 **   for each record: Write()
 **   Rewind()
 **     Rowkey()/Compare()
-**   Next() 
+**   Next()
 **   Close()
 **
 ** Algorithm:
 **
-** Records passed to the sorter via calls to Write() are initially held 
+** Records passed to the sorter via calls to Write() are initially held
 ** unsorted in main memory. Assuming the amount of memory used never exceeds
 ** a threshold, when Rewind() is called the set of records is sorted using
 ** an in-memory merge sort. In this case, no temporary files are required
-** and subsequent calls to Rowkey(), Next() and Compare() read records 
+** and subsequent calls to Rowkey(), Next() and Compare() read records
 ** directly from main memory.
 **
 ** If the amount of space used to store records in main memory exceeds the
@@ -82,10 +82,10 @@
 ** of PMAs may be created by merging existing PMAs together - for example
 ** merging two or more level-0 PMAs together creates a level-1 PMA.
 **
-** The threshold for the amount of main memory to use before flushing 
+** The threshold for the amount of main memory to use before flushing
 ** records to a PMA is roughly the same as the limit configured for the
-** page-cache of the main database. Specifically, the threshold is set to 
-** the value returned by "PRAGMA main.page_size" multipled by 
+** page-cache of the main database. Specifically, the threshold is set to
+** the value returned by "PRAGMA main.page_size" multiplied by
 ** that returned by "PRAGMA main.cache_size", in bytes.
 **
 ** If the sorter is running in single-threaded mode, then all PMAs generated
@@ -102,28 +102,28 @@
 ** than zero, and (b) worker threads have been enabled at runtime by calling
 ** "PRAGMA threads=N" with some value of N greater than 0.
 **
-** When Rewind() is called, any data remaining in memory is flushed to a 
+** When Rewind() is called, any data remaining in memory is flushed to a
 ** final PMA. So at this point the data is stored in some number of sorted
 ** PMAs within temporary files on disk.
 **
 ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the
 ** sorter is running in single-threaded mode, then these PMAs are merged
-** incrementally as keys are retreived from the sorter by the VDBE.  The
+** incrementally as keys are retrieved from the sorter by the VDBE.  The
 ** MergeEngine object, described in further detail below, performs this
 ** merge.
 **
 ** Or, if running in multi-threaded mode, then a background thread is
 ** launched to merge the existing PMAs. Once the background thread has
-** merged T bytes of data into a single sorted PMA, the main thread 
+** merged T bytes of data into a single sorted PMA, the main thread
 ** begins reading keys from that PMA while the background thread proceeds
 ** with merging the next T bytes of data. And so on.
 **
-** Parameter T is set to half the value of the memory threshold used 
+** Parameter T is set to half the value of the memory threshold used
 ** by Write() above to determine when to create a new PMA.
 **
-** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when 
-** Rewind() is called, then a hierarchy of incremental-merges is used. 
-** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on 
+** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when
+** Rewind() is called, then a hierarchy of incremental-merges is used.
+** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on
 ** disk are merged together. Then T bytes of data from the second set, and
 ** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT
 ** PMAs at a time. This done is to improve locality.
@@ -138,7 +138,7 @@
 #include "sqliteInt.h"
 #include "vdbeInt.h"
 
-/* 
+/*
 ** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various
 ** messages to stderr that may be helpful in understanding the performance
 ** characteristics of the sorter in multi-threaded mode.
@@ -167,7 +167,7 @@ typedef struct SorterList SorterList;       /* In-memory list of records */
 typedef struct IncrMerger IncrMerger;       /* Read & merge multiple PMAs */
 
 /*
-** A container for a temp file handle and the current amount of data 
+** A container for a temp file handle and the current amount of data
 ** stored in the file.
 */
 struct SorterFile {
@@ -186,7 +186,7 @@ struct SorterFile {
 struct SorterList {
   SorterRecord *pList;            /* Linked list of records */
   u8 *aMemory;                    /* If non-NULL, bulk memory to hold pList */
-  int szPMA;                      /* Size of pList as PMA in bytes */
+  i64 szPMA;                      /* Size of pList as PMA in bytes */
 };
 
 /*
@@ -207,17 +207,17 @@ struct SorterList {
 ** the MergeEngine.nTree variable.
 **
 ** The final (N/2) elements of aTree[] contain the results of comparing
-** pairs of PMA keys together. Element i contains the result of 
+** pairs of PMA keys together. Element i contains the result of
 ** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the
-** aTree element is set to the index of it. 
+** aTree element is set to the index of it.
 **
 ** For the purposes of this comparison, EOF is considered greater than any
 ** other key value. If the keys are equal (only possible with two EOF
 ** values), it doesn't matter which index is stored.
 **
-** The (N/4) elements of aTree[] that precede the final (N/2) described 
+** The (N/4) elements of aTree[] that precede the final (N/2) described
 ** above contains the index of the smallest of each block of 4 PmaReaders
-** And so on. So that aTree[1] contains the index of the PmaReader that 
+** And so on. So that aTree[1] contains the index of the PmaReader that
 ** currently points to the smallest key value. aTree[0] is unused.
 **
 ** Example:
@@ -233,7 +233,7 @@ struct SorterList {
 **
 **     aTree[] = { X, 5   0, 5    0, 3, 5, 6 }
 **
-** The current element is "Apple" (the value of the key indicated by 
+** The current element is "Apple" (the value of the key indicated by
 ** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will
 ** be advanced to the next key in its segment. Say the next key is
 ** "Eggplant":
@@ -271,11 +271,11 @@ struct MergeEngine {
 **
 ** Essentially, this structure contains all those fields of the VdbeSorter
 ** structure for which each thread requires a separate instance. For example,
-** each thread requries its own UnpackedRecord object to unpack records in
+** each thread requeries its own UnpackedRecord object to unpack records in
 ** as part of comparison operations.
 **
-** Before a background thread is launched, variable bDone is set to 0. Then, 
-** right before it exits, the thread itself sets bDone to 1. This is used for 
+** Before a background thread is launched, variable bDone is set to 0. Then,
+** right before it exits, the thread itself sets bDone to 1. This is used for
 ** two purposes:
 **
 **   1. When flushing the contents of memory to a level-0 PMA on disk, to
@@ -295,18 +295,19 @@ typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int);
 struct SortSubtask {
   SQLiteThread *pThread;          /* Background thread, if any */
   int bDone;                      /* Set if thread is finished but not joined */
+  int nPMA;                       /* Number of PMAs currently in file */
   VdbeSorter *pSorter;            /* Sorter that owns this sub-task */
   UnpackedRecord *pUnpacked;      /* Space to unpack a record */
   SorterList list;                /* List for thread to write to a PMA */
-  int nPMA;                       /* Number of PMAs currently in file */
   SorterCompare xCompare;         /* Compare function to use */
   SorterFile file;                /* Temp file for level-0 PMAs */
   SorterFile file2;               /* Space for other PMAs */
+  u64 nSpill;                     /* Total bytes written by this task */
 };
 
 
 /*
-** Main sorter structure. A single instance of this is allocated for each 
+** Main sorter structure. A single instance of this is allocated for each
 ** sorter cursor created by the VDBE.
 **
 ** mxKeysize:
@@ -332,9 +333,12 @@ struct VdbeSorter {
   u8 iPrev;                       /* Previous thread used to flush PMA */
   u8 nTask;                       /* Size of aTask[] array */
   u8 typeMask;
-  SortSubtask aTask[1];           /* One or more subtasks */
+  SortSubtask aTask[FLEXARRAY];   /* One or more subtasks */
 };
 
+/* Size (in bytes) of a VdbeSorter object that works with N or fewer subtasks */
+#define SZ_VDBESORTER(N)  (offsetof(VdbeSorter,aTask)+(N)*sizeof(SortSubtask))
+
 #define SORTER_TYPE_INTEGER 0x01
 #define SORTER_TYPE_TEXT    0x02
 
@@ -343,7 +347,7 @@ struct VdbeSorter {
 ** PMA, in sorted order.  The next key to be read is cached in nKey/aKey.
 ** aKey might point into aMap or into aBuffer.  If neither of those locations
 ** contain a contiguous representation of the key, then aAlloc is allocated
-** and the key is copied into aAlloc and aKey is made to poitn to aAlloc.
+** and the key is copied into aAlloc and aKey is made to point to aAlloc.
 **
 ** pFd==0 at EOF.
 */
@@ -362,21 +366,21 @@ struct PmaReader {
 };
 
 /*
-** Normally, a PmaReader object iterates through an existing PMA stored 
+** Normally, a PmaReader object iterates through an existing PMA stored
 ** within a temp file. However, if the PmaReader.pIncr variable points to
 ** an object of the following type, it may be used to iterate/merge through
 ** multiple PMAs simultaneously.
 **
-** There are two types of IncrMerger object - single (bUseThread==0) and 
-** multi-threaded (bUseThread==1). 
+** There are two types of IncrMerger object - single (bUseThread==0) and
+** multi-threaded (bUseThread==1).
 **
-** A multi-threaded IncrMerger object uses two temporary files - aFile[0] 
-** and aFile[1]. Neither file is allowed to grow to more than mxSz bytes in 
-** size. When the IncrMerger is initialized, it reads enough data from 
-** pMerger to populate aFile[0]. It then sets variables within the 
-** corresponding PmaReader object to read from that file and kicks off 
-** a background thread to populate aFile[1] with the next mxSz bytes of 
-** sorted record data from pMerger. 
+** A multi-threaded IncrMerger object uses two temporary files - aFile[0]
+** and aFile[1]. Neither file is allowed to grow to more than mxSz bytes in
+** size. When the IncrMerger is initialized, it reads enough data from
+** pMerger to populate aFile[0]. It then sets variables within the
+** corresponding PmaReader object to read from that file and kicks off
+** a background thread to populate aFile[1] with the next mxSz bytes of
+** sorted record data from pMerger.
 **
 ** When the PmaReader reaches the end of aFile[0], it blocks until the
 ** background thread has finished populating aFile[1]. It then exchanges
@@ -387,7 +391,7 @@ struct PmaReader {
 **
 ** A single-threaded IncrMerger does not open any temporary files of its
 ** own. Instead, it has exclusive access to mxSz bytes of space beginning
-** at offset iStartOff of file pTask->file2. And instead of using a 
+** at offset iStartOff of file pTask->file2. And instead of using a
 ** background thread to prepare data for the PmaReader, with a single
 ** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with
 ** keys from pMerger by the calling thread whenever the PmaReader runs out
@@ -419,6 +423,7 @@ struct PmaWriter {
   int iBufEnd;                    /* Last byte of buffer to write */
   i64 iWriteOff;                  /* Offset of start of buffer in file */
   sqlite3_file *pFd;              /* File handle to write to */
+  u64 nPmaSpill;                  /* Total number of bytes written */
 };
 
 /*
@@ -499,7 +504,7 @@ static int vdbePmaReadBlob(
 
   assert( p->aBuffer );
 
-  /* If there is no more data to be read from the buffer, read the next 
+  /* If there is no more data to be read from the buffer, read the next
   ** p->nBuffer bytes of data from the file into it. Or, if there are less
   ** than p->nBuffer bytes remaining in the PMA, read all remaining data.  */
   iBuf = p->iReadOff % p->nBuffer;
@@ -520,11 +525,11 @@ static int vdbePmaReadBlob(
     assert( rc!=SQLITE_IOERR_SHORT_READ );
     if( rc!=SQLITE_OK ) return rc;
   }
-  nAvail = p->nBuffer - iBuf; 
+  nAvail = p->nBuffer - iBuf;
 
   if( nByte<=nAvail ){
     /* The requested data is available in the in-memory buffer. In this
-    ** case there is no need to make a copy of the data, just return a 
+    ** case there is no need to make a copy of the data, just return a
     ** pointer into the buffer to the caller.  */
     *ppOut = &p->aBuffer[iBuf];
     p->iReadOff += nByte;
@@ -556,13 +561,14 @@ static int vdbePmaReadBlob(
     while( nRem>0 ){
       int rc;                     /* vdbePmaReadBlob() return code */
       int nCopy;                  /* Number of bytes to copy */
-      u8 *aNext;                  /* Pointer to buffer to copy data from */
+      u8 *aNext = 0;              /* Pointer to buffer to copy data from */
 
       nCopy = nRem;
       if( nRem>p->nBuffer ) nCopy = p->nBuffer;
       rc = vdbePmaReadBlob(p, nCopy, &aNext);
       if( rc!=SQLITE_OK ) return rc;
       assert( aNext!=p->aAlloc );
+      assert( aNext!=0 );
       memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy);
       nRem -= nCopy;
     }
@@ -603,7 +609,7 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){
 
 /*
 ** Attempt to memory map file pFile. If successful, set *pp to point to the
-** new mapping and return SQLITE_OK. If the mapping is not attempted 
+** new mapping and return SQLITE_OK. If the mapping is not attempted
 ** (because the file is too large or the VFS layer is configured not to use
 ** mmap), return SQLITE_OK and set *pp to NULL.
 **
@@ -624,7 +630,7 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){
 
 /*
 ** Attach PmaReader pReadr to file pFile (if it is not already attached to
-** that file) and seek it to offset iOff within the file.  Return SQLITE_OK 
+** that file) and seek it to offset iOff within the file.  Return SQLITE_OK
 ** if successful, or an SQLite error code if an error occurs.
 */
 static int vdbePmaReaderSeek(
@@ -714,11 +720,11 @@ static int vdbePmaReaderNext(PmaReader *pReadr){
 
 /*
 ** Initialize PmaReader pReadr to scan through the PMA stored in file pFile
-** starting at offset iStart and ending at offset iEof-1. This function 
-** leaves the PmaReader pointing to the first key in the PMA (or EOF if the 
+** starting at offset iStart and ending at offset iEof-1. This function
+** leaves the PmaReader pointing to the first key in the PMA (or EOF if the
 ** PMA is empty).
 **
-** If the pnByte parameter is NULL, then it is assumed that the file 
+** If the pnByte parameter is NULL, then it is assumed that the file
 ** contains a single PMA, and that that PMA omits the initial length varint.
 */
 static int vdbePmaReaderInit(
@@ -751,7 +757,7 @@ static int vdbePmaReaderInit(
 
 /*
 ** A version of vdbeSorterCompare() that assumes that it has already been
-** determined that the first field of key1 is equal to the first field of 
+** determined that the first field of key1 is equal to the first field of
 ** key2.
 */
 static int vdbeSorterCompareTail(
@@ -762,14 +768,14 @@ static int vdbeSorterCompareTail(
 ){
   UnpackedRecord *r2 = pTask->pUnpacked;
   if( *pbKey2Cached==0 ){
-    sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2);
+    sqlite3VdbeRecordUnpack(nKey2, pKey2, r2);
     *pbKey2Cached = 1;
   }
   return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1);
 }
 
 /*
-** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, 
+** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2,
 ** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences
 ** used by the comparison. Return the result of the comparison.
 **
@@ -789,7 +795,7 @@ static int vdbeSorterCompare(
 ){
   UnpackedRecord *r2 = pTask->pUnpacked;
   if( !*pbKey2Cached ){
-    sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2);
+    sqlite3VdbeRecordUnpack(nKey2, pKey2, r2);
     *pbKey2Cached = 1;
   }
   return sqlite3VdbeRecordCompare(nKey1, pKey1, r2);
@@ -829,6 +835,7 @@ static int vdbeSorterCompareText(
       );
     }
   }else{
+    assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 );
     assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) );
     if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){
       res = res * -1;
@@ -892,6 +899,7 @@ static int vdbeSorterCompareInt(
     }
   }
 
+  assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 );
   if( res==0 ){
     if( pTask->pSorter->pKeyInfo->nKeyField>1 ){
       res = vdbeSorterCompareTail(
@@ -915,7 +923,7 @@ static int vdbeSorterCompareInt(
 ** is non-zero and the sorter is able to guarantee a stable sort, nField
 ** is used instead. This is used when sorting records for a CREATE INDEX
 ** statement. In this case, keys are always delivered to the sorter in
-** order of the primary key, which happens to be make up the final part 
+** order of the primary key, which happens to be make up the final part
 ** of the records being sorted. So if the sort is stable, there is never
 ** any reason to compare PK fields and they can be ignored for a small
 ** performance boost.
@@ -935,7 +943,7 @@ int sqlite3VdbeSorterInit(
   VdbeSorter *pSorter;            /* The new sorter */
   KeyInfo *pKeyInfo;              /* Copy of pCsr->pKeyInfo with db==0 */
   int szKeyInfo;                  /* Size of pCsr->pKeyInfo in bytes */
-  int sz;                         /* Size of pSorter in bytes */
+  i64 sz;                         /* Size of pSorter in bytes */
   int rc = SQLITE_OK;
 #if SQLITE_MAX_WORKER_THREADS==0
 # define nWorker 0
@@ -963,8 +971,11 @@ int sqlite3VdbeSorterInit(
   assert( pCsr->pKeyInfo );
   assert( !pCsr->isEphemeral );
   assert( pCsr->eCurType==CURTYPE_SORTER );
-  szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*);
-  sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask);
+  assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*)
+               < 0x7fffffff );
+  assert( pCsr->pKeyInfo->nKeyField<=pCsr->pKeyInfo->nAllField );
+  szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nAllField);
+  sz = SZ_VDBESORTER(nWorker+1);
 
   pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo);
   pCsr->uc.pSorter = pSorter;
@@ -977,7 +988,12 @@ int sqlite3VdbeSorterInit(
     pKeyInfo->db = 0;
     if( nField && nWorker==0 ){
       pKeyInfo->nKeyField = nField;
+      assert( nField<=pCsr->pKeyInfo->nAllField );
     }
+    /* It is OK that pKeyInfo reuses the aSortFlags field from pCsr->pKeyInfo,
+    ** since the pCsr->pKeyInfo->aSortFlags[] array is invariant and lives
+    ** longer that pSorter. */
+    assert( pKeyInfo->aSortFlags==pCsr->pKeyInfo->aSortFlags );
     sqlite3BtreeEnter(pBt);
     pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt);
     sqlite3BtreeLeave(pBt);
@@ -1016,7 +1032,7 @@ int sqlite3VdbeSorterInit(
       }
     }
 
-    if( pKeyInfo->nAllField<13 
+    if( pKeyInfo->nAllField<13
      && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl)
      && (pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL)==0
     ){
@@ -1041,7 +1057,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){
 }
 
 /*
-** Free all resources owned by the object indicated by argument pTask. All 
+** Free all resources owned by the object indicated by argument pTask. All
 ** fields of *pTask are zeroed before returning.
 */
 static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){
@@ -1141,7 +1157,7 @@ static int vdbeSorterCreateThread(
 }
 
 /*
-** Join all outstanding threads launched by SorterWrite() to create 
+** Join all outstanding threads launched by SorterWrite() to create
 ** level-0 PMAs.
 */
 static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){
@@ -1150,10 +1166,10 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){
 
   /* This function is always called by the main user thread.
   **
-  ** If this function is being called after SorterRewind() has been called, 
+  ** If this function is being called after SorterRewind() has been called,
   ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread
   ** is currently attempt to join one of the other threads. To avoid a race
-  ** condition where this thread also attempts to join the same object, join 
+  ** condition where this thread also attempts to join the same object, join
   ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */
   for(i=pSorter->nTask-1; i>=0; i--){
     SortSubtask *pTask = &pSorter->aTask[i];
@@ -1176,7 +1192,7 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){
 */
 static MergeEngine *vdbeMergeEngineNew(int nReader){
   int N = 2;                      /* Smallest power of two >= nReader */
-  int nByte;                      /* Total bytes of space to allocate */
+  i64 nByte;                      /* Total bytes of space to allocate */
   MergeEngine *pNew;              /* Pointer to allocated object to return */
 
   assert( nReader<=SORTER_MAX_MERGE_COUNT );
@@ -1266,6 +1282,12 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
   assert( pCsr->eCurType==CURTYPE_SORTER );
   pSorter = pCsr->uc.pSorter;
   if( pSorter ){
+    /* Increment db->nSpill by the total number of bytes of data written
+    ** to temp files by this sort operation.  */
+    int ii;
+    for(ii=0; ii<pSorter->nTask; ii++){
+      db->nSpill += pSorter->aTask[ii].nSpill;
+    }
     sqlite3VdbeSorterReset(db, pSorter);
     sqlite3_free(pSorter->list.aMemory);
     sqlite3DbFree(db, pSorter);
@@ -1325,8 +1347,8 @@ static int vdbeSorterOpenTempFile(
 }
 
 /*
-** If it has not already been allocated, allocate the UnpackedRecord 
-** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or 
+** If it has not already been allocated, allocate the UnpackedRecord
+** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or
 ** if no allocation was required), or SQLITE_NOMEM otherwise.
 */
 static int vdbeSortAllocUnpacked(SortSubtask *pTask){
@@ -1389,14 +1411,14 @@ static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){
   if( p->typeMask==SORTER_TYPE_INTEGER ){
     return vdbeSorterCompareInt;
   }else if( p->typeMask==SORTER_TYPE_TEXT ){
-    return vdbeSorterCompareText; 
+    return vdbeSorterCompareText;
   }
   return vdbeSorterCompare;
 }
 
 /*
-** Sort the linked list of records headed at pTask->pList. Return 
-** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if 
+** Sort the linked list of records headed at pTask->pList. Return
+** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if
 ** an error occurs.
 */
 static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
@@ -1428,6 +1450,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
     p->u.pNext = 0;
     for(i=0; aSlot[i]; i++){
       p = vdbeSorterMerge(pTask, p, aSlot[i]);
+      /* ,--Each aSlot[] holds twice as much as the previous. So we cannot use
+      ** |  up all 64 aSlots[] with only a 64-bit address space.
+      ** v                                                                */
+      assert( i<ArraySize(aSlot) );
       aSlot[i] = 0;
     }
     aSlot[i] = p;
@@ -1441,8 +1467,8 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
   }
   pList->pList = p;
 
-  assert( pTask->pUnpacked->errCode==SQLITE_OK 
-       || pTask->pUnpacked->errCode==SQLITE_NOMEM 
+  assert( pTask->pUnpacked->errCode==SQLITE_OK
+       || pTask->pUnpacked->errCode==SQLITE_NOMEM
   );
   return pTask->pUnpacked->errCode;
 }
@@ -1483,10 +1509,11 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){
     memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy);
     p->iBufEnd += nCopy;
     if( p->iBufEnd==p->nBuffer ){
-      p->eFWErr = sqlite3OsWrite(p->pFd, 
-          &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, 
+      p->eFWErr = sqlite3OsWrite(p->pFd,
+          &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart,
           p->iWriteOff + p->iBufStart
       );
+      p->nPmaSpill += (p->iBufEnd - p->iBufStart);
       p->iBufStart = p->iBufEnd = 0;
       p->iWriteOff += p->nBuffer;
     }
@@ -1499,21 +1526,24 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){
 /*
 ** Flush any buffered data to disk and clean up the PMA-writer object.
 ** The results of using the PMA-writer after this call are undefined.
-** Return SQLITE_OK if flushing the buffered data succeeds or is not 
+** Return SQLITE_OK if flushing the buffered data succeeds or is not
 ** required. Otherwise, return an SQLite error code.
 **
 ** Before returning, set *piEof to the offset immediately following the
-** last byte written to the file.
+** last byte written to the file. Also, increment (*pnSpill) by the total
+** number of bytes written to the file.
 */
-static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){
+static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof, u64 *pnSpill){
   int rc;
   if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){
-    p->eFWErr = sqlite3OsWrite(p->pFd, 
-        &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, 
+    p->eFWErr = sqlite3OsWrite(p->pFd,
+        &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart,
         p->iWriteOff + p->iBufStart
     );
+    p->nPmaSpill += (p->iBufEnd - p->iBufStart);
   }
   *piEof = (p->iWriteOff + p->iBufEnd);
+  *pnSpill += p->nPmaSpill;
   sqlite3_free(p->aBuffer);
   rc = p->eFWErr;
   memset(p, 0, sizeof(PmaWriter));
@@ -1521,11 +1551,11 @@ static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){
 }
 
 /*
-** Write value iVal encoded as a varint to the PMA. Return 
+** Write value iVal encoded as a varint to the PMA. Return
 ** SQLITE_OK if successful, or an SQLite error code if an error occurs.
 */
 static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){
-  int nByte; 
+  int nByte;
   u8 aByte[10];
   nByte = sqlite3PutVarint(aByte, iVal);
   vdbePmaWriteBlob(p, aByte, nByte);
@@ -1533,7 +1563,7 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){
 
 /*
 ** Write the current contents of in-memory linked-list pList to a level-0
-** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if 
+** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if
 ** successful, or an SQLite error code otherwise.
 **
 ** The format of a PMA is:
@@ -1541,8 +1571,8 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){
 **     * A varint. This varint contains the total number of bytes of content
 **       in the PMA (not including the varint itself).
 **
-**     * One or more records packed end-to-end in order of ascending keys. 
-**       Each record consists of a varint followed by a blob of data (the 
+**     * One or more records packed end-to-end in order of ascending keys.
+**       Each record consists of a varint followed by a blob of data (the
 **       key). The varint is the number of bytes in the blob of data.
 */
 static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){
@@ -1551,7 +1581,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){
   PmaWriter writer;               /* Object used to write to the file */
 
 #ifdef SQLITE_DEBUG
-  /* Set iSz to the expected size of file pTask->file after writing the PMA. 
+  /* Set iSz to the expected size of file pTask->file after writing the PMA.
   ** This is used by an assert() statement at the end of this function.  */
   i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof;
 #endif
@@ -1593,7 +1623,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){
       if( pList->aMemory==0 ) sqlite3_free(p);
     }
     pList->pList = p;
-    rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof);
+    rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof, &pTask->nSpill);
   }
 
   vdbeSorterWorkDebug(pTask, "exit");
@@ -1704,7 +1734,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){
   SortSubtask *pTask = 0;    /* Thread context used to create new PMA */
   int nWorker = (pSorter->nTask-1);
 
-  /* Set the flag to indicate that at least one PMA has been written. 
+  /* Set the flag to indicate that at least one PMA has been written.
   ** Or will be, anyhow.  */
   pSorter->bUsePMA = 1;
 
@@ -1714,7 +1744,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){
   ** the background thread from a sub-tasks previous turn is still running,
   ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy,
   ** fall back to using the final sub-task. The first (pSorter->nTask-1)
-  ** sub-tasks are prefered as they use background threads - the final 
+  ** sub-tasks are preferred as they use background threads - the final
   ** sub-task uses the main thread. */
   for(i=0; i<nWorker; i++){
     int iTest = (pSorter->iPrev + i + 1) % nWorker;
@@ -1772,8 +1802,8 @@ int sqlite3VdbeSorterWrite(
   int rc = SQLITE_OK;             /* Return Code */
   SorterRecord *pNew;             /* New list element */
   int bFlush;                     /* True to flush contents of memory to PMA */
-  int nReq;                       /* Bytes of memory required */
-  int nPMA;                       /* Bytes of PMA space required */
+  i64 nReq;                       /* Bytes of memory required */
+  i64 nPMA;                       /* Bytes of PMA space required */
   int t;                          /* serial type of first record field */
 
   assert( pCsr->eCurType==CURTYPE_SORTER );
@@ -1795,14 +1825,14 @@ int sqlite3VdbeSorterWrite(
   ** If using the single large allocation mode (pSorter->aMemory!=0), then
   ** flush the contents of memory to a new PMA if (a) at least one value is
   ** already in memory and (b) the new value will not fit in memory.
-  ** 
+  **
   ** Or, if using separate allocations for each record, flush the contents
   ** of memory to a PMA if either of the following are true:
   **
-  **   * The total memory allocated for the in-memory list is greater 
+  **   * The total memory allocated for the in-memory list is greater
   **     than (page-size * cache-size), or
   **
-  **   * The total memory allocated for the in-memory list is greater 
+  **   * The total memory allocated for the in-memory list is greater
   **     than (page-size * 10) and sqlite3HeapNearlyFull() returns true.
   */
   nReq = pVal->n + sizeof(SorterRecord);
@@ -1907,7 +1937,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){
     rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy);
   }
 
-  rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof);
+  rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof, &pTask->nSpill);
   if( rc==SQLITE_OK ) rc = rc2;
   vdbeSorterPopulateDebug(pTask, "exit");
   return rc;
@@ -1941,11 +1971,11 @@ static int vdbeIncrBgPopulate(IncrMerger *pIncr){
 ** aFile[0] such that the PmaReader should start rereading it from the
 ** beginning.
 **
-** For single-threaded objects, this is accomplished by literally reading 
-** keys from pIncr->pMerger and repopulating aFile[0]. 
+** For single-threaded objects, this is accomplished by literally reading
+** keys from pIncr->pMerger and repopulating aFile[0].
 **
-** For multi-threaded objects, all that is required is to wait until the 
-** background thread is finished (if it is not already) and then swap 
+** For multi-threaded objects, all that is required is to wait until the
+** background thread is finished (if it is not already) and then swap
 ** aFile[0] and aFile[1] in place. If the contents of pMerger have not
 ** been exhausted, this function also launches a new background thread
 ** to populate the new aFile[1].
@@ -2086,7 +2116,7 @@ static void vdbeMergeEngineCompare(
 #define INCRINIT_TASK   1
 #define INCRINIT_ROOT   2
 
-/* 
+/*
 ** Forward reference required as the vdbeIncrMergeInit() and
 ** vdbePmaReaderIncrInit() routines are called mutually recursively when
 ** building a merge tree.
@@ -2095,7 +2125,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode);
 
 /*
 ** Initialize the MergeEngine object passed as the second argument. Once this
-** function returns, the first key of merged data may be read from the 
+** function returns, the first key of merged data may be read from the
 ** MergeEngine object in the usual fashion.
 **
 ** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge
@@ -2105,8 +2135,8 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode);
 ** required is to call vdbePmaReaderNext() on each PmaReader to point it at
 ** its first key.
 **
-** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use 
-** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data 
+** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use
+** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data
 ** to pMerger.
 **
 ** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
@@ -2161,19 +2191,19 @@ static int vdbeMergeEngineInit(
 ** object at (pReadr->pIncr).
 **
 ** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders
-** in the sub-tree headed by pReadr are also initialized. Data is then 
-** loaded into the buffers belonging to pReadr and it is set to point to 
+** in the sub-tree headed by pReadr are also initialized. Data is then
+** loaded into the buffers belonging to pReadr and it is set to point to
 ** the first key in its range.
 **
 ** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed
 ** to be a multi-threaded PmaReader and this function is being called in a
-** background thread. In this case all PmaReaders in the sub-tree are 
+** background thread. In this case all PmaReaders in the sub-tree are
 ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to
 ** pReadr is populated. However, pReadr itself is not set up to point
 ** to its first key. A call to vdbePmaReaderNext() is still required to do
-** that. 
+** that.
 **
-** The reason this function does not call vdbePmaReaderNext() immediately 
+** The reason this function does not call vdbePmaReaderNext() immediately
 ** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has
 ** to block on thread (pTask->thread) before accessing aFile[1]. But, since
 ** this entire function is being run by thread (pTask->thread), that will
@@ -2198,7 +2228,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){
 
   rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode);
 
-  /* Set up the required files for pIncr. A multi-theaded IncrMerge object
+  /* Set up the required files for pIncr. A multi-threaded IncrMerge object
   ** requires two temp files to itself, whereas a single-threaded object
   ** only requires a region of pTask->file2. */
   if( rc==SQLITE_OK ){
@@ -2229,12 +2259,12 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){
   if( rc==SQLITE_OK && pIncr->bUseThread ){
     /* Use the current thread to populate aFile[1], even though this
     ** PmaReader is multi-threaded. If this is an INCRINIT_TASK object,
-    ** then this function is already running in background thread 
-    ** pIncr->pTask->thread. 
+    ** then this function is already running in background thread
+    ** pIncr->pTask->thread.
     **
-    ** If this is the INCRINIT_ROOT object, then it is running in the 
+    ** If this is the INCRINIT_ROOT object, then it is running in the
     ** main VDBE thread. But that is Ok, as that thread cannot return
-    ** control to the VDBE or proceed with anything useful until the 
+    ** control to the VDBE or proceed with anything useful until the
     ** first results are ready from this merger object anyway.
     */
     assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK );
@@ -2251,7 +2281,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){
 
 #if SQLITE_MAX_WORKER_THREADS>0
 /*
-** The main routine for vdbePmaReaderIncrMergeInit() operations run in 
+** The main routine for vdbePmaReaderIncrMergeInit() operations run in
 ** background threads.
 */
 static void *vdbePmaReaderBgIncrInit(void *pCtx){
@@ -2269,8 +2299,8 @@ static void *vdbePmaReaderBgIncrInit(void *pCtx){
 ** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it invokes
 ** the vdbePmaReaderIncrMergeInit() function with the parameters passed to
 ** this routine to initialize the incremental merge.
-** 
-** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), 
+**
+** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1),
 ** then a background thread is launched to call vdbePmaReaderIncrMergeInit().
 ** Or, if the IncrMerger is single threaded, the same function is called
 ** using the current thread.
@@ -2300,7 +2330,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){
 ** to NULL and return an SQLite error code.
 **
 ** When this function is called, *piOffset is set to the offset of the
-** first PMA to read from pTask->file. Assuming no error occurs, it is 
+** first PMA to read from pTask->file. Assuming no error occurs, it is
 ** set to the offset immediately following the last byte of the last
 ** PMA before returning. If an error does occur, then the final value of
 ** *piOffset is undefined.
@@ -2410,12 +2440,12 @@ static int vdbeSorterAddToTree(
 /*
 ** This function is called as part of a SorterRewind() operation on a sorter
 ** that has already written two or more level-0 PMAs to one or more temp
-** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that 
+** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that
 ** can be used to incrementally merge all PMAs on disk.
 **
 ** If successful, SQLITE_OK is returned and *ppOut set to point to the
 ** MergeEngine object at the root of the tree before returning. Or, if an
-** error occurs, an SQLite error code is returned and the final value 
+** error occurs, an SQLite error code is returned and the final value
 ** of *ppOut is undefined.
 */
 static int vdbeSorterMergeTreeBuild(
@@ -2427,8 +2457,8 @@ static int vdbeSorterMergeTreeBuild(
   int iTask;
 
 #if SQLITE_MAX_WORKER_THREADS>0
-  /* If the sorter uses more than one task, then create the top-level 
-  ** MergeEngine here. This MergeEngine will read data from exactly 
+  /* If the sorter uses more than one task, then create the top-level
+  ** MergeEngine here. This MergeEngine will read data from exactly
   ** one PmaReader per sub-task.  */
   assert( pSorter->bUseThreads || pSorter->nTask==1 );
   if( pSorter->nTask>1 ){
@@ -2537,7 +2567,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
           }
           for(iTask=0; rc==SQLITE_OK && iTask<pSorter->nTask; iTask++){
             /* Check that:
-            **   
+            **  
             **   a) The incremental merge object is configured to use the
             **      right task, and
             **   b) If it is using task (nTask-1), it is configured to run
@@ -2600,7 +2630,7 @@ int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){
     return rc;
   }
 
-  /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() 
+  /* Write the current in-memory list to a PMA. When the VdbeSorterWrite()
   ** function flushes the contents of memory to disk, it immediately always
   ** creates a new list consisting of a single key immediately afterwards.
   ** So the list is never empty at this point.  */
@@ -2612,7 +2642,7 @@ int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){
 
   vdbeSorterRewindDebug("rewind");
 
-  /* Assuming no errors have occurred, set up a merger structure to 
+  /* Assuming no errors have occurred, set up a merger structure to
   ** incrementally read and merge all remaining PMAs.  */
   assert( pSorter->pReader==0 );
   if( rc==SQLITE_OK ){
@@ -2666,7 +2696,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr){
 }
 
 /*
-** Return a pointer to a buffer owned by the sorter that contains the 
+** Return a pointer to a buffer owned by the sorter that contains the
 ** current key.
 */
 static void *vdbeSorterRowkey(
@@ -2753,7 +2783,7 @@ int sqlite3VdbeSorterCompare(
   assert( r2->nField==nKeyCol );
 
   pKey = vdbeSorterRowkey(pSorter, &nKey);
-  sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2);
+  sqlite3VdbeRecordUnpack(nKey, pKey, r2);
   for(i=0; i<nKeyCol; i++){
     if( r2->aMem[i].flags & MEM_Null ){
       *pRes = -1;
diff --git a/src/vdbetrace.c b/src/vdbetrace.c
index ae8ad3115f..1a59f0e4d5 100644
--- a/src/vdbetrace.c
+++ b/src/vdbetrace.c
@@ -26,10 +26,10 @@
 ** a host parameter.  If the text contains no host parameters, return
 ** the total number of bytes in the text.
 */
-static int findNextHostParameter(const char *zSql, int *pnToken){
+static i64 findNextHostParameter(const char *zSql, i64 *pnToken){
   int tokenType;
-  int nTotal = 0;
-  int n;
+  i64 nTotal = 0;
+  i64 n;
 
   *pnToken = 0;
   while( zSql[0] ){
@@ -76,8 +76,8 @@ char *sqlite3VdbeExpandSql(
   sqlite3 *db;             /* The database connection */
   int idx = 0;             /* Index of a host parameter */
   int nextIndex = 1;       /* Index of next ? host parameter */
-  int n;                   /* Length of a token prefix */
-  int nToken;              /* Length of the parameter token */
+  i64 n;                   /* Length of a token prefix */
+  i64 nToken;              /* Length of the parameter token */
   int i;                   /* Loop counter */
   Mem *pVar;               /* Value of a host parameter */
   StrAccum out;            /* Accumulate the output here */
diff --git a/src/vdbevtab.c b/src/vdbevtab.c
index 6557d8cb01..1c9909a1aa 100644
--- a/src/vdbevtab.c
+++ b/src/vdbevtab.c
@@ -69,6 +69,8 @@ static int bytecodevtabConnect(
       "p5 INT,"
       "comment TEXT,"
       "subprog TEXT," 
+      "nexec INT,"
+      "ncycle INT,"
       "stmt HIDDEN"
     ");",
 
@@ -231,7 +233,7 @@ static int bytecodevtabColumn(
           }
         }
       }
-      i += 10;
+      i += 20;
     }
   }
   switch( i ){
@@ -281,16 +283,31 @@ static int bytecodevtabColumn(
       }
       break;
     }
-    case 10:  /* tables_used.type */
+
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+    case 9:     /* nexec */
+      sqlite3_result_int64(ctx, pOp->nExec);
+      break;
+    case 10:    /* ncycle */
+      sqlite3_result_int64(ctx, pOp->nCycle);
+      break;
+#else
+    case 9:     /* nexec */
+    case 10:    /* ncycle */
+      sqlite3_result_int(ctx, 0);
+      break;
+#endif
+
+    case 20:  /* tables_used.type */
       sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC);
       break;
-    case 11:  /* tables_used.schema */
+    case 21:  /* tables_used.schema */
       sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC);
       break;
-    case 12:  /* tables_used.name */
+    case 22:  /* tables_used.name */
       sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC);
       break;
-    case 13:  /* tables_used.wr */
+    case 23:  /* tables_used.wr */
       sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite);
       break;
   }
@@ -364,7 +381,7 @@ static int bytecodevtabBestIndex(
   int rc = SQLITE_CONSTRAINT;
   struct sqlite3_index_constraint *p;
   bytecodevtab *pVTab = (bytecodevtab*)tab;
-  int iBaseCol = pVTab->bTablesUsed ? 4 : 8;
+  int iBaseCol = pVTab->bTablesUsed ? 4 : 10;
   pIdxInfo->estimatedCost = (double)100;
   pIdxInfo->estimatedRows = 100;
   pIdxInfo->idxNum = 0;
@@ -411,7 +428,8 @@ static sqlite3_module bytecodevtabModule = {
   /* xSavepoint  */ 0,
   /* xRelease    */ 0,
   /* xRollbackTo */ 0,
-  /* xShadowName */ 0
+  /* xShadowName */ 0,
+  /* xIntegrity  */ 0
 };
 
 
diff --git a/src/vtab.c b/src/vtab.c
index ad629bb031..ed4b0afaf4 100644
--- a/src/vtab.c
+++ b/src/vtab.c
@@ -17,7 +17,7 @@
 /*
 ** Before a virtual table xCreate() or xConnect() method is invoked, the
 ** sqlite3.pVtabCtx member variable is set to point to an instance of
-** this struct allocated on the stack. It is used by the implementation of 
+** this struct allocated on the stack. It is used by the implementation of
 ** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which
 ** are invoked only from within xCreate and xConnect methods.
 */
@@ -174,7 +174,7 @@ void sqlite3VtabModuleUnref(sqlite3 *db, Module *pMod){
 /*
 ** Lock the virtual table so that it cannot be disconnected.
 ** Locks nest.  Every lock should have a corresponding unlock.
-** If an unlock is omitted, resources leaks will occur.  
+** If an unlock is omitted, resources leaks will occur.
 **
 ** If a disconnect is attempted while a virtual table is locked,
 ** the disconnect is deferred until all locks have been removed.
@@ -186,7 +186,7 @@ void sqlite3VtabLock(VTable *pVTab){
 
 /*
 ** pTab is a pointer to a Table structure representing a virtual-table.
-** Return a pointer to the VTable object used by connection db to access 
+** Return a pointer to the VTable object used by connection db to access
 ** this virtual-table, if one has been created, or NULL otherwise.
 */
 VTable *sqlite3GetVTable(sqlite3 *db, Table *pTab){
@@ -222,7 +222,7 @@ void sqlite3VtabUnlock(VTable *pVTab){
 /*
 ** Table p is a virtual table. This function moves all elements in the
 ** p->u.vtab.p list to the sqlite3.pDisconnect lists of their associated
-** database connections to be disconnected at the next opportunity. 
+** database connections to be disconnected at the next opportunity.
 ** Except, if argument db is not NULL, then the entry associated with
 ** connection db is left in the p->u.vtab.p list.
 */
@@ -234,8 +234,8 @@ static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){
   pVTable = p->u.vtab.p;
   p->u.vtab.p = 0;
 
-  /* Assert that the mutex (if any) associated with the BtShared database 
-  ** that contains table p is held by the caller. See header comments 
+  /* Assert that the mutex (if any) associated with the BtShared database
+  ** that contains table p is held by the caller. See header comments
   ** above function sqlite3VtabUnlockList() for an explanation of why
   ** this makes it safe to access the sqlite3.pDisconnect list of any
   ** database connection that may have an entry in the p->u.vtab.p list.
@@ -291,7 +291,7 @@ void sqlite3VtabDisconnect(sqlite3 *db, Table *p){
 ** Disconnect all the virtual table objects in the sqlite3.pDisconnect list.
 **
 ** This function may only be called when the mutexes associated with all
-** shared b-tree databases opened using connection db are held by the 
+** shared b-tree databases opened using connection db are held by the
 ** caller. This is done to protect the sqlite3.pDisconnect list. The
 ** sqlite3.pDisconnect list is accessed only as follows:
 **
@@ -304,7 +304,7 @@ void sqlite3VtabDisconnect(sqlite3 *db, Table *p){
 **      or, if the virtual table is stored in a non-sharable database, then
 **      the database handle mutex is held.
 **
-** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously 
+** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously
 ** by multiple threads. It is thread-safe.
 */
 void sqlite3VtabUnlockList(sqlite3 *db){
@@ -315,7 +315,6 @@ void sqlite3VtabUnlockList(sqlite3 *db){
 
   if( p ){
     db->pDisconnect = 0;
-    sqlite3ExpirePreparedStatements(db, 0);
     do {
       VTable *pNext = p->pNext;
       sqlite3VtabUnlock(p);
@@ -330,12 +329,12 @@ void sqlite3VtabUnlockList(sqlite3 *db){
 ** record.
 **
 ** Since it is a virtual-table, the Table structure contains a pointer
-** to the head of a linked list of VTable structures. Each VTable 
+** to the head of a linked list of VTable structures. Each VTable
 ** structure is associated with a single sqlite3* user of the schema.
-** The reference count of the VTable structure associated with database 
-** connection db is decremented immediately (which may lead to the 
+** The reference count of the VTable structure associated with database
+** connection db is decremented immediately (which may lead to the
 ** structure being xDisconnected and free). Any other VTable structures
-** in the list are moved to the sqlite3.pDisconnect list of the associated 
+** in the list are moved to the sqlite3.pDisconnect list of the associated
 ** database connection.
 */
 void sqlite3VtabClear(sqlite3 *db, Table *p){
@@ -421,7 +420,7 @@ void sqlite3VtabBeginParse(
   if( pTable->u.vtab.azArg ){
     int iDb = sqlite3SchemaToIndex(db, pTable->pSchema);
     assert( iDb>=0 ); /* The database the table is being created in */
-    sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, 
+    sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName,
             pTable->u.vtab.azArg[0], pParse->db->aDb[iDb].zDbSName);
   }
 #endif
@@ -454,7 +453,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
   addArgumentToVtab(pParse);
   pParse->sArg.z = 0;
   if( pTab->u.vtab.nArg<1 ) return;
-  
+
   /* If the CREATE VIRTUAL TABLE statement is being entered for the
   ** first time (in other words if the virtual table is actually being
   ** created now instead of just being read out of sqlite_schema) then
@@ -476,15 +475,16 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
     }
     zStmt = sqlite3MPrintf(db, "CREATE VIRTUAL TABLE %T", &pParse->sNameToken);
 
-    /* A slot for the record has already been allocated in the 
+    /* A slot for the record has already been allocated in the
     ** schema table.  We just need to update that slot with all
-    ** the information we've collected.  
+    ** the information we've collected.
     **
-    ** The VM register number pParse->regRowid holds the rowid of an
-    ** entry in the sqlite_schema table tht was created for this vtab
+    ** The VM register number pParse->u1.cr.regRowid holds the rowid of an
+    ** entry in the sqlite_schema table that was created for this vtab
     ** by sqlite3StartTable().
     */
     iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+    assert( pParse->isCreate );
     sqlite3NestedParse(pParse,
       "UPDATE %Q." LEGACY_SCHEMA_TABLE " "
          "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q "
@@ -493,7 +493,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
       pTab->zName,
       pTab->zName,
       zStmt,
-      pParse->regRowid
+      pParse->u1.cr.regRowid
     );
     v = sqlite3GetVdbe(pParse);
     sqlite3ChangeCookie(pParse, iDb);
@@ -555,7 +555,7 @@ void sqlite3VtabArgExtend(Parse *pParse, Token *p){
 ** to this procedure.
 */
 static int vtabCallConstructor(
-  sqlite3 *db, 
+  sqlite3 *db,
   Table *pTab,
   Module *pMod,
   int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**),
@@ -577,7 +577,7 @@ static int vtabCallConstructor(
   /* Check that the virtual-table is not already being initialized */
   for(pCtx=db->pVtabCtx; pCtx; pCtx=pCtx->pPrior){
     if( pCtx->pTab==pTab ){
-      *pzErr = sqlite3MPrintf(db, 
+      *pzErr = sqlite3MPrintf(db,
           "vtable constructor called recursively: %s", pTab->zName
       );
       return SQLITE_LOCKED;
@@ -612,6 +612,8 @@ static int vtabCallConstructor(
   db->pVtabCtx = &sCtx;
   pTab->nTabRef++;
   rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr);
+  assert( pTab!=0 );
+  assert( pTab->nTabRef>1 || rc!=SQLITE_OK );
   sqlite3DeleteTable(db, pTab);
   db->pVtabCtx = sCtx.pPrior;
   if( rc==SQLITE_NOMEM ) sqlite3OomFault(db);
@@ -634,14 +636,14 @@ static int vtabCallConstructor(
     pVTable->nRef = 1;
     if( sCtx.bDeclared==0 ){
       const char *zFormat = "vtable constructor did not declare schema: %s";
-      *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName);
+      *pzErr = sqlite3MPrintf(db, zFormat, zModuleName);
       sqlite3VtabUnlock(pVTable);
       rc = SQLITE_ERROR;
     }else{
       int iCol;
       u16 oooHidden = 0;
       /* If everything went according to plan, link the new VTable structure
-      ** into the linked list headed by pTab->u.vtab.p. Then loop through the 
+      ** into the linked list headed by pTab->u.vtab.p. Then loop through the
       ** columns of the table to see if any of them contain the token "hidden".
       ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from
       ** the type string.  */
@@ -687,7 +689,7 @@ static int vtabCallConstructor(
 
 /*
 ** This function is invoked by the parser to call the xConnect() method
-** of the virtual table pTab. If an error occurs, an error code is returned 
+** of the virtual table pTab. If an error occurs, an error code is returned
 ** and an error left in pParse.
 **
 ** This call is a no-op if table pTab is not a virtual table.
@@ -759,7 +761,7 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){
 
 /*
 ** This function is invoked by the vdbe to call the xCreate method
-** of the virtual table named zTab in database iDb. 
+** of the virtual table named zTab in database iDb.
 **
 ** If an error occurs, *pzErr is set to point to an English language
 ** description of the error and an SQLITE_XXX error code is returned.
@@ -778,8 +780,8 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){
   zMod = pTab->u.vtab.azArg[0];
   pMod = (Module*)sqlite3HashFind(&db->aModule, zMod);
 
-  /* If the module has been registered and includes a Create method, 
-  ** invoke it now. If the module has not been registered, return an 
+  /* If the module has been registered and includes a Create method,
+  ** invoke it now. If the module has not been registered, return an
   ** error. Otherwise, do nothing.
   */
   if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){
@@ -812,19 +814,40 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
   Table *pTab;
   Parse sParse;
   int initBusy;
+  int i;
+  const unsigned char *z;
+  static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 };
 
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){
     return SQLITE_MISUSE_BKPT;
   }
 #endif
+
+  /* Verify that the first two keywords in the CREATE TABLE statement
+  ** really are "CREATE" and "TABLE".  If this is not the case, then
+  ** sqlite3_declare_vtab() is being misused.
+  */
+  z = (const unsigned char*)zCreateTable;
+  for(i=0; aKeyword[i]; i++){
+    int tokenType = 0;
+    do{
+      z += sqlite3GetToken(z, &tokenType);
+    }while( tokenType==TK_SPACE || tokenType==TK_COMMENT );
+    if( tokenType!=aKeyword[i] ){
+      sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error");
+      return SQLITE_ERROR;
+    }
+  }
+ 
   sqlite3_mutex_enter(db->mutex);
   pCtx = db->pVtabCtx;
   if( !pCtx || pCtx->bDeclared ){
-    sqlite3Error(db, SQLITE_MISUSE);
+    sqlite3Error(db, SQLITE_MISUSE_BKPT);
     sqlite3_mutex_leave(db->mutex);
     return SQLITE_MISUSE_BKPT;
   }
+
   pTab = pCtx->pTab;
   assert( IsVirtual(pTab) );
 
@@ -838,16 +861,16 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
   initBusy = db->init.busy;
   db->init.busy = 0;
   sParse.nQueryLoop = 1;
-  if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) 
-   && ALWAYS(sParse.pNewTable!=0)
-   && ALWAYS(!db->mallocFailed)
-   && IsOrdinaryTable(sParse.pNewTable)
-  ){
+  if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){
+    assert( sParse.pNewTable!=0 );
+    assert( !db->mallocFailed );
+    assert( IsOrdinaryTable(sParse.pNewTable) );
     assert( sParse.zErrMsg==0 );
     if( !pTab->aCol ){
       Table *pNew = sParse.pNewTable;
       Index *pIdx;
       pTab->aCol = pNew->aCol;
+      assert( IsOrdinaryTable(pNew) );
       sqlite3ExprListDelete(db, pNew->u.tab.pDfltList);
       pTab->nNVCol = pTab->nCol = pNew->nCol;
       pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
@@ -942,7 +965,7 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){
 ** called is identified by the second argument, "offset", which is
 ** the offset of the method to call in the sqlite3_module structure.
 **
-** The array is cleared after invoking the callbacks. 
+** The array is cleared after invoking the callbacks.
 */
 static void callFinaliser(sqlite3 *db, int offset){
   int i;
@@ -991,7 +1014,7 @@ int sqlite3VtabSync(sqlite3 *db, Vdbe *p){
 }
 
 /*
-** Invoke the xRollback method of all virtual tables in the 
+** Invoke the xRollback method of all virtual tables in the
 ** sqlite3.aVTrans array. Then clear the array itself.
 */
 int sqlite3VtabRollback(sqlite3 *db){
@@ -1000,7 +1023,7 @@ int sqlite3VtabRollback(sqlite3 *db){
 }
 
 /*
-** Invoke the xCommit method of all virtual tables in the 
+** Invoke the xCommit method of all virtual tables in the
 ** sqlite3.aVTrans array. Then clear the array itself.
 */
 int sqlite3VtabCommit(sqlite3 *db){
@@ -1022,7 +1045,7 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){
 
   /* Special case: If db->aVTrans is NULL and db->nVTrans is greater
   ** than zero, then this function is being called from within a
-  ** virtual module xSync() callback. It is illegal to write to 
+  ** virtual module xSync() callback. It is illegal to write to
   ** virtual module tables in this case, so return SQLITE_LOCKED.
   */
   if( sqlite3VtabInSync(db) ){
@@ -1030,7 +1053,7 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){
   }
   if( !pVTab ){
     return SQLITE_OK;
-  } 
+  }
   pModule = pVTab->pVtab->pModule;
 
   if( pModule->xBegin ){
@@ -1043,7 +1066,7 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){
       }
     }
 
-    /* Invoke the xBegin method. If successful, add the vtab to the 
+    /* Invoke the xBegin method. If successful, add the vtab to the
     ** sqlite3.aVTrans[] array. */
     rc = growVTrans(db);
     if( rc==SQLITE_OK ){
@@ -1067,11 +1090,11 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){
 ** as the second argument to the virtual table method invoked.
 **
 ** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is
-** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is 
+** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is
 ** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with
 ** an open transaction is invoked.
 **
-** If any virtual table method returns an error code other than SQLITE_OK, 
+** If any virtual table method returns an error code other than SQLITE_OK,
 ** processing is abandoned and the error returned to the caller of this
 ** function immediately. If all calls to virtual table methods are successful,
 ** SQLITE_OK is returned.
@@ -1123,7 +1146,7 @@ int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){
 ** This routine is used to allow virtual table implementations to
 ** overload MATCH, LIKE, GLOB, and REGEXP operators.
 **
-** Return either the pDef argument (indicating no change) or a 
+** Return either the pDef argument (indicating no change) or a
 ** new FuncDef structure that is marked as ephemeral using the
 ** SQLITE_FUNC_EPHEM flag.
 */
@@ -1153,7 +1176,7 @@ FuncDef *sqlite3VtabOverloadFunction(
   assert( pVtab->pModule!=0 );
   pMod = (sqlite3_module *)pVtab->pModule;
   if( pMod->xFindFunction==0 ) return pDef;
- 
+
   /* Call the xFindFunction method on the virtual table implementation
   ** to see if the implementation wants to overload this function.
   **
@@ -1219,13 +1242,13 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){
 /*
 ** Check to see if virtual table module pMod can be have an eponymous
 ** virtual table instance.  If it can, create one if one does not already
-** exist. Return non-zero if either the eponymous virtual table instance 
+** exist. Return non-zero if either the eponymous virtual table instance
 ** exists when this routine returns or if an attempt to create it failed
 ** and an error message was left in pParse.
 **
 ** An eponymous virtual table instance is one that is named after its
 ** module, and more importantly, does not require a CREATE VIRTUAL TABLE
-** statement in order to come into existance.  Eponymous virtual table
+** statement in order to come into existence.  Eponymous virtual table
 ** instances always exist.  They cannot be DROP-ed.
 **
 ** Any virtual table module for which xConnect and xCreate are the same
@@ -1256,9 +1279,12 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
   addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName));
   addModuleArgument(pParse, pTab, 0);
   addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName));
+  db->nSchemaLock++;
   rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr);
+  db->nSchemaLock--;
   if( rc ){
     sqlite3ErrorMsg(pParse, "%s", zErr);
+    pParse->rc = rc;
     sqlite3DbFree(db, zErr);
     sqlite3VtabEponymousTableClear(db, pMod);
   }
@@ -1273,7 +1299,7 @@ void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
   Table *pTab = pMod->pEpoTab;
   if( pTab!=0 ){
     /* Mark the table as Ephemeral prior to deleting it, so that the
-    ** sqlite3DeleteTable() routine will know that it is not stored in 
+    ** sqlite3DeleteTable() routine will know that it is not stored in
     ** the schema. */
     pTab->tabFlags |= TF_Ephemeral;
     sqlite3DeleteTable(db, pTab);
@@ -1289,8 +1315,8 @@ void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
 ** within an xUpdate method.
 */
 int sqlite3_vtab_on_conflict(sqlite3 *db){
-  static const unsigned char aMap[] = { 
-    SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE 
+  static const unsigned char aMap[] = {
+    SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE
   };
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT;
@@ -1302,7 +1328,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *db){
 }
 
 /*
-** Call from within the xCreate() or xConnect() methods to provide 
+** Call from within the xCreate() or xConnect() methods to provide
 ** the SQLite core with additional information about the behavior
 ** of the virtual table being implemented.
 */
diff --git a/src/vxworks.h b/src/vxworks.h
index e7013c3f66..3a95779d2c 100644
--- a/src/vxworks.h
+++ b/src/vxworks.h
@@ -25,7 +25,9 @@
 #define HAVE_UTIME 1
 #else
 /* This is not VxWorks. */
-#define OS_VXWORKS 0
+#ifndef OS_VXWORKS
+#  define OS_VXWORKS 0
+#endif
 #define HAVE_FCHOWN 1
 #define HAVE_READLINK 1
 #define HAVE_LSTAT 1
diff --git a/src/wal.c b/src/wal.c
index 89cc53ca31..d4d568d628 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -10,7 +10,7 @@
 **
 *************************************************************************
 **
-** This file contains the implementation of a write-ahead log (WAL) used in 
+** This file contains the implementation of a write-ahead log (WAL) used in
 ** "journal_mode=WAL" mode.
 **
 ** WRITE-AHEAD LOG (WAL) FILE FORMAT
@@ -19,7 +19,7 @@
 ** Each frame records the revised content of a single page from the
 ** database file.  All changes to the database are recorded by writing
 ** frames into the WAL.  Transactions commit when a frame is written that
-** contains a commit marker.  A single WAL can and usually does record 
+** contains a commit marker.  A single WAL can and usually does record
 ** multiple transactions.  Periodically, the content of the WAL is
 ** transferred back into the database file in an operation called a
 ** "checkpoint".
@@ -44,12 +44,12 @@
 **    28: Checksum-2 (second part of checksum for first 24 bytes of header).
 **
 ** Immediately following the wal-header are zero or more frames. Each
-** frame consists of a 24-byte frame-header followed by a <page-size> bytes
-** of page data. The frame-header is six big-endian 32-bit unsigned 
+** frame consists of a 24-byte frame-header followed by <page-size> bytes
+** of page data. The frame-header is six big-endian 32-bit unsigned
 ** integer values, as follows:
 **
 **     0: Page number.
-**     4: For commit records, the size of the database image in pages 
+**     4: For commit records, the size of the database image in pages
 **        after the commit. For all other records, zero.
 **     8: Salt-1 (copied from the header)
 **    12: Salt-2 (copied from the header)
@@ -75,7 +75,7 @@
 ** the checksum.  The checksum is computed by interpreting the input as
 ** an even number of unsigned 32-bit integers: x[0] through x[N].  The
 ** algorithm used for the checksum is as follows:
-** 
+**
 **   for i from 0 to n-1 step 2:
 **     s0 += x[i] + s1;
 **     s1 += x[i+1] + s0;
@@ -83,7 +83,7 @@
 **
 ** Note that s0 and s1 are both weighted checksums using fibonacci weights
 ** in reverse order (the largest fibonacci weight occurs on the first element
-** of the sequence being summed.)  The s1 value spans all 32-bit 
+** of the sequence being summed.)  The s1 value spans all 32-bit
 ** terms of the sequence whereas s0 omits the final term.
 **
 ** On a checkpoint, the WAL is first VFS.xSync-ed, then valid content of the
@@ -116,19 +116,19 @@
 ** multiple concurrent readers to view different versions of the database
 ** content simultaneously.
 **
-** The reader algorithm in the previous paragraphs works correctly, but 
+** The reader algorithm in the previous paragraphs works correctly, but
 ** because frames for page P can appear anywhere within the WAL, the
 ** reader has to scan the entire WAL looking for page P frames.  If the
 ** WAL is large (multiple megabytes is typical) that scan can be slow,
 ** and read performance suffers.  To overcome this problem, a separate
 ** data structure called the wal-index is maintained to expedite the
 ** search for frames of a particular page.
-** 
+**
 ** WAL-INDEX FORMAT
 **
 ** Conceptually, the wal-index is shared memory, though VFS implementations
 ** might choose to implement the wal-index using a mmapped file.  Because
-** the wal-index is shared memory, SQLite does not support journal_mode=WAL 
+** the wal-index is shared memory, SQLite does not support journal_mode=WAL
 ** on a network filesystem.  All users of the database must be able to
 ** share memory.
 **
@@ -146,19 +146,19 @@
 ** byte order of the host computer.
 **
 ** The purpose of the wal-index is to answer this question quickly:  Given
-** a page number P and a maximum frame index M, return the index of the 
+** a page number P and a maximum frame index M, return the index of the
 ** last frame in the wal before frame M for page P in the WAL, or return
 ** NULL if there are no frames for page P in the WAL prior to M.
 **
 ** The wal-index consists of a header region, followed by an one or
-** more index blocks.  
+** more index blocks.
 **
 ** The wal-index header contains the total number of frames within the WAL
 ** in the mxFrame field.
 **
-** Each index block except for the first contains information on 
+** Each index block except for the first contains information on
 ** HASHTABLE_NPAGE frames. The first index block contains information on
-** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and 
+** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and
 ** HASHTABLE_NPAGE are selected so that together the wal-index header and
 ** first index block are the same size as all other index blocks in the
 ** wal-index.  The values are:
@@ -167,10 +167,10 @@
 **   HASHTABLE_NPAGE_ONE  4062
 **
 ** Each index block contains two sections, a page-mapping that contains the
-** database page number associated with each wal frame, and a hash-table 
+** database page number associated with each wal frame, and a hash-table
 ** that allows readers to query an index block for a specific page number.
 ** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE
-** for the first index block) 32-bit page numbers. The first entry in the 
+** for the first index block) 32-bit page numbers. The first entry in the
 ** first index-block contains the database page number corresponding to the
 ** first frame in the WAL file. The first entry in the second index block
 ** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in
@@ -191,8 +191,8 @@
 **
 ** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers.
 ** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the
-** hash table for each page number in the mapping section, so the hash 
-** table is never more than half full.  The expected number of collisions 
+** hash table for each page number in the mapping section, so the hash
+** table is never more than half full.  The expected number of collisions
 ** prior to finding a match is 1.  Each entry of the hash table is an
 ** 1-based index of an entry in the mapping section of the same
 ** index block.   Let K be the 1-based index of the largest entry in
@@ -211,12 +211,12 @@
 ** reached) until an unused hash slot is found. Let the first unused slot
 ** be at index iUnused.  (iUnused might be less than iKey if there was
 ** wrap-around.) Because the hash table is never more than half full,
-** the search is guaranteed to eventually hit an unused entry.  Let 
+** the search is guaranteed to eventually hit an unused entry.  Let
 ** iMax be the value between iKey and iUnused, closest to iUnused,
 ** where aHash[iMax]==P.  If there is no iMax entry (if there exists
 ** no hash slot such that aHash[i]==p) then page P is not in the
 ** current index block.  Otherwise the iMax-th mapping entry of the
-** current index block corresponds to the last entry that references 
+** current index block corresponds to the last entry that references
 ** page P.
 **
 ** A hash search begins with the last index block and moves toward the
@@ -241,7 +241,7 @@
 ** if no values greater than K0 had ever been inserted into the hash table
 ** in the first place - which is what reader one wants.  Meanwhile, the
 ** second reader using K1 will see additional values that were inserted
-** later, which is exactly what reader two wants.  
+** later, which is exactly what reader two wants.
 **
 ** When a rollback occurs, the value of K is decreased. Hash table entries
 ** that correspond to frames greater than the new K value are removed
@@ -269,7 +269,7 @@ int sqlite3WalTrace = 0;
 ** values in the wal-header are correct and (b) the version field is not
 ** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN.
 **
-** Similarly, if a client successfully reads a wal-index header (i.e. the 
+** Similarly, if a client successfully reads a wal-index header (i.e. the
 ** checksum test is successful) and finds that the version field is not
 ** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite
 ** returns SQLITE_CANTOPEN.
@@ -284,7 +284,7 @@ int sqlite3WalTrace = 0;
 **
 ** Technically, the various VFSes are free to implement these locks however
 ** they see fit.  However, compatibility is encouraged so that VFSes can
-** interoperate.  The standard implemention used on both unix and windows
+** interoperate.  The standard implementation used on both unix and windows
 ** is for the index number to indicate a byte offset into the
 ** WalCkptInfo.aLock[] array in the wal-index header.  In other words, all
 ** locks are on the shm file.  The WALINDEX_LOCK_OFFSET constant (which
@@ -316,7 +316,7 @@ typedef struct WalCkptInfo WalCkptInfo;
 **
 ** The szPage value can be any power of 2 between 512 and 32768, inclusive.
 ** Or it can be 1 to represent a 65536-byte page.  The latter case was
-** added in 3.7.1 when support for 64K pages was added.  
+** added in 3.7.1 when support for 64K pages was added.
 */
 struct WalIndexHdr {
   u32 iVersion;                   /* Wal-index version */
@@ -358,9 +358,9 @@ struct WalIndexHdr {
 ** There is one entry in aReadMark[] for each reader lock.  If a reader
 ** holds read-lock K, then the value in aReadMark[K] is no greater than
 ** the mxFrame for that reader.  The value READMARK_NOT_USED (0xffffffff)
-** for any aReadMark[] means that entry is unused.  aReadMark[0] is 
+** for any aReadMark[] means that entry is unused.  aReadMark[0] is
 ** a special case; its value is never used and it exists as a place-holder
-** to avoid having to offset aReadMark[] indexs by one.  Readers holding
+** to avoid having to offset aReadMark[] indexes by one.  Readers holding
 ** WAL_READ_LOCK(0) always ignore the entire WAL and read all content
 ** directly from the database.
 **
@@ -378,7 +378,7 @@ struct WalIndexHdr {
 ** previous sentence is when nBackfill equals mxFrame (meaning that everything
 ** in the WAL has been backfilled into the database) then new readers
 ** will choose aReadMark[0] which has value 0 and hence such reader will
-** get all their all content directly from the database file and ignore 
+** get all their all content directly from the database file and ignore
 ** the WAL.
 **
 ** Writers normally append new frames to the end of the WAL.  However,
@@ -484,14 +484,14 @@ struct WalCkptInfo {
 ** big-endian format in the first 4 bytes of a WAL file.
 **
 ** If the LSB is set, then the checksums for each frame within the WAL
-** file are calculated by treating all data as an array of 32-bit 
-** big-endian words. Otherwise, they are calculated by interpreting 
+** file are calculated by treating all data as an array of 32-bit
+** big-endian words. Otherwise, they are calculated by interpreting
 ** all data as 32-bit little-endian words.
 */
 #define WAL_MAGIC 0x377f0682
 
 /*
-** Return the offset of frame iFrame in the write-ahead log file, 
+** Return the offset of frame iFrame in the write-ahead log file,
 ** assuming a database page size of szPage bytes. The offset returned
 ** is to the start of the write-ahead log frame-header.
 */
@@ -502,6 +502,11 @@ struct WalCkptInfo {
 /*
 ** An open write-ahead log file is represented by an instance of the
 ** following object.
+**
+** writeLock:
+**   This is usually set to 1 whenever the WRITER lock is held. However,
+**   if it is set to 2, then the WRITER lock is held but must be released
+**   by walHandleException() if a SEH exception is thrown.
 */
 struct Wal {
   sqlite3_vfs *pVfs;         /* The VFS used to create pDbFd */
@@ -528,11 +533,20 @@ struct Wal {
   u32 iReCksum;              /* On commit, recalculate checksums from here */
   const char *zWalName;      /* Name of WAL file */
   u32 nCkpt;                 /* Checkpoint sequence counter in the wal-header */
+#ifdef SQLITE_USE_SEH
+  u32 lockMask;              /* Mask of locks held */
+  void *pFree;               /* Pointer to sqlite3_free() if exception thrown */
+  u32 *pWiValue;             /* Value to write into apWiData[iWiPg] */
+  int iWiPg;                 /* Write pWiValue into apWiData[iWiPg] */
+  int iSysErrno;             /* System error code following exception */
+#endif
 #ifdef SQLITE_DEBUG
+  int nSehTry;               /* Number of nested SEH_TRY{} blocks */
   u8 lockError;              /* True if a locking error has occurred */
 #endif
 #ifdef SQLITE_ENABLE_SNAPSHOT
   WalIndexHdr *pSnapshot;    /* Start transaction here if not NULL */
+  int bGetSnapshot;          /* Transaction opened for sqlite3_get_snapshot() */
 #endif
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
   sqlite3 *db;
@@ -543,7 +557,7 @@ struct Wal {
 ** Candidate values for Wal.exclusiveMode.
 */
 #define WAL_NORMAL_MODE     0
-#define WAL_EXCLUSIVE_MODE  1     
+#define WAL_EXCLUSIVE_MODE  1
 #define WAL_HEAPMEMORY_MODE 2
 
 /*
@@ -562,7 +576,7 @@ typedef u16 ht_slot;
 /*
 ** This structure is used to implement an iterator that loops through
 ** all frames in the WAL in database page order. Where two or more frames
-** correspond to the same database page, the iterator visits only the 
+** correspond to the same database page, the iterator visits only the
 ** frame most recently written to the WAL (in other words, the frame with
 ** the largest index).
 **
@@ -583,9 +597,13 @@ struct WalIterator {
     u32 *aPgno;                   /* Array of page numbers. */
     int nEntry;                   /* Nr. of entries in aPgno[] and aIndex[] */
     int iZero;                    /* Frame number associated with aPgno[0] */
-  } aSegment[1];                  /* One for every 32KB page in the wal-index */
+  } aSegment[FLEXARRAY];          /* One for every 32KB page in the wal-index */
 };
 
+/* Size (in bytes) of a WalIterator object suitable for N or fewer segments */
+#define SZ_WALITERATOR(N)  \
+     (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment))
+
 /*
 ** Define the parameters of the hash tables in the wal-index file. There
 ** is a hash-table following every HASHTABLE_NPAGE page numbers in the
@@ -598,7 +616,7 @@ struct WalIterator {
 #define HASHTABLE_HASH_1     383                  /* Should be prime */
 #define HASHTABLE_NSLOT      (HASHTABLE_NPAGE*2)  /* Must be a power of 2 */
 
-/* 
+/*
 ** The block of page numbers associated with the first hash-table in a
 ** wal-index is smaller than usual. This is so that there is a complete
 ** hash-table on each aligned 32KB page of the wal-index.
@@ -610,6 +628,113 @@ struct WalIterator {
     sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \
 )
 
+/*
+** Structured Exception Handling (SEH) is a Windows-specific technique
+** for catching exceptions raised while accessing memory-mapped files.
+**
+** The -DSQLITE_USE_SEH compile-time option means to use SEH to catch and
+** deal with system-level errors that arise during WAL -shm file processing.
+** Without this compile-time option, any system-level faults that appear
+** while accessing the memory-mapped -shm file will cause a process-wide
+** signal to be deliver, which will more than likely cause the entire
+** process to exit.
+*/
+#ifdef SQLITE_USE_SEH
+#include <Windows.h>
+
+/* Beginning of a block of code in which an exception might occur */
+# define SEH_TRY    __try { \
+   assert( walAssertLockmask(pWal) && pWal->nSehTry==0 ); \
+   VVA_ONLY(pWal->nSehTry++);
+
+/* The end of a block of code in which an exception might occur */
+# define SEH_EXCEPT(X) \
+   VVA_ONLY(pWal->nSehTry--); \
+   assert( pWal->nSehTry==0 ); \
+   } __except( sehExceptionFilter(pWal, GetExceptionCode(), GetExceptionInformation() ) ){ X }
+
+/* Simulate a memory-mapping fault in the -shm file for testing purposes */
+# define SEH_INJECT_FAULT sehInjectFault(pWal) 
+
+/*
+** The second argument is the return value of GetExceptionCode() for the 
+** current exception. Return EXCEPTION_EXECUTE_HANDLER if the exception code
+** indicates that the exception may have been caused by accessing the *-shm 
+** file mapping. Or EXCEPTION_CONTINUE_SEARCH otherwise.
+*/
+static int sehExceptionFilter(Wal *pWal, int eCode, EXCEPTION_POINTERS *p){
+  VVA_ONLY(pWal->nSehTry--);
+  if( eCode==EXCEPTION_IN_PAGE_ERROR ){
+    if( p && p->ExceptionRecord && p->ExceptionRecord->NumberParameters>=3 ){
+      /* From MSDN: For this type of exception, the first element of the
+      ** ExceptionInformation[] array is a read-write flag - 0 if the exception
+      ** was thrown while reading, 1 if while writing. The second element is
+      ** the virtual address being accessed. The "third array element specifies
+      ** the underlying NTSTATUS code that resulted in the exception". */
+      pWal->iSysErrno = (int)p->ExceptionRecord->ExceptionInformation[2];
+    }
+    return EXCEPTION_EXECUTE_HANDLER;
+  }
+  return EXCEPTION_CONTINUE_SEARCH;
+}
+
+/*
+** If one is configured, invoke the xTestCallback callback with 650 as
+** the argument. If it returns true, throw the same exception that is
+** thrown by the system if the *-shm file mapping is accessed after it
+** has been invalidated.
+*/
+static void sehInjectFault(Wal *pWal){
+  int res;
+  assert( pWal->nSehTry>0 );
+
+  res = sqlite3FaultSim(650);
+  if( res!=0 ){
+    ULONG_PTR aArg[3];
+    aArg[0] = 0;
+    aArg[1] = 0;
+    aArg[2] = (ULONG_PTR)res;
+    RaiseException(EXCEPTION_IN_PAGE_ERROR, 0, 3, (const ULONG_PTR*)aArg);
+  }
+}
+
+/*
+** There are two ways to use this macro. To set a pointer to be freed
+** if an exception is thrown:
+**
+**   SEH_FREE_ON_ERROR(0, pPtr);
+**
+** and to cancel the same:
+**
+**   SEH_FREE_ON_ERROR(pPtr, 0);
+**
+** In the first case, there must not already be a pointer registered to
+** be freed. In the second case, pPtr must be the registered pointer.
+*/
+#define SEH_FREE_ON_ERROR(X,Y) \
+  assert( (X==0 || Y==0) && pWal->pFree==X ); pWal->pFree = Y
+
+/*
+** There are two ways to use this macro. To arrange for pWal->apWiData[iPg]
+** to be set to pValue if an exception is thrown:
+**
+**   SEH_SET_ON_ERROR(iPg, pValue);
+**
+** and to cancel the same:
+**
+**   SEH_SET_ON_ERROR(0, 0);
+*/
+#define SEH_SET_ON_ERROR(X,Y)  pWal->iWiPg = X; pWal->pWiValue = Y
+
+#else
+# define SEH_TRY          VVA_ONLY(pWal->nSehTry++);
+# define SEH_EXCEPT(X)    VVA_ONLY(pWal->nSehTry--); assert( pWal->nSehTry==0 );
+# define SEH_INJECT_FAULT assert( pWal->nSehTry>0 );
+# define SEH_FREE_ON_ERROR(X,Y)
+# define SEH_SET_ON_ERROR(X,Y)
+#endif /* ifdef SQLITE_USE_SEH */
+
+
 /*
 ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index
 ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are
@@ -624,7 +749,7 @@ struct WalIterator {
 **
 **   (1)  rc==SQLITE_OK    and *ppPage==Requested-Wal-Index-Page
 **   (2)  rc>=SQLITE_ERROR and *ppPage==NULL
-**   (3)  rc==SQLITE_OK    and *ppPage==NULL  // only if iPage==0 
+**   (3)  rc==SQLITE_OK    and *ppPage==NULL  // only if iPage==0
 **
 ** Scenario (3) can only occur when pWal->writeLock is false and iPage==0
 */
@@ -637,7 +762,7 @@ static SQLITE_NOINLINE int walIndexPageRealloc(
 
   /* Enlarge the pWal->apWiData[] array if required */
   if( pWal->nWiData<=iPage ){
-    sqlite3_int64 nByte = sizeof(u32*)*(iPage+1);
+    sqlite3_int64 nByte = sizeof(u32*)*(1+(i64)iPage);
     volatile u32 **apNew;
     apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte);
     if( !apNew ){
@@ -656,7 +781,7 @@ static SQLITE_NOINLINE int walIndexPageRealloc(
     pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
     if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT;
   }else{
-    rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
+    rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ,
         pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
     );
     assert( pWal->apWiData[iPage]!=0
@@ -682,6 +807,7 @@ static int walIndexPage(
   int iPage,               /* The page we seek */
   volatile u32 **ppPage    /* Write the page pointer here */
 ){
+  SEH_INJECT_FAULT;
   if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){
     return walIndexPageRealloc(pWal, iPage, ppPage);
   }
@@ -693,6 +819,7 @@ static int walIndexPage(
 */
 static volatile WalCkptInfo *walCkptInfo(Wal *pWal){
   assert( pWal->nWiData>0 && pWal->apWiData[0] );
+  SEH_INJECT_FAULT;
   return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]);
 }
 
@@ -701,6 +828,7 @@ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){
 */
 static volatile WalIndexHdr *walIndexHdr(Wal *pWal){
   assert( pWal->nWiData>0 && pWal->apWiData[0] );
+  SEH_INJECT_FAULT;
   return (volatile WalIndexHdr*)pWal->apWiData[0];
 }
 
@@ -717,7 +845,7 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){
 )
 
 /*
-** Generate or extend an 8 byte checksum based on the data in 
+** Generate or extend an 8 byte checksum based on the data in
 ** array aByte[] and the initial values of aIn[0] and aIn[1] (or
 ** initial values of 0 and 0 if aIn==NULL).
 **
@@ -743,10 +871,8 @@ static void walChecksumBytes(
     s1 = s2 = 0;
   }
 
-  assert( nByte>=8 );
-  assert( (nByte&0x00000007)==0 );
-  assert( nByte<=65536 );
-  assert( nByte%4==0 );
+  /* nByte is a multiple of 8 between 8 and 65536 */
+  assert( nByte>=8 && (nByte&7)==0 && nByte<=65536 );
 
   if( !nativeCksum ){
     do {
@@ -829,11 +955,11 @@ static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){
 
 /*
 ** This function encodes a single frame header and writes it to a buffer
-** supplied by the caller. A frame-header is made up of a series of 
+** supplied by the caller. A frame-header is made up of a series of
 ** 4-byte big-endian integers, as follows:
 **
 **     0: Page number.
-**     4: For commit records, the size of the database image in pages 
+**     4: For commit records, the size of the database image in pages
 **        after the commit. For all other records, zero.
 **     8: Salt-1 (copied from the wal-header)
 **    12: Salt-2 (copied from the wal-header)
@@ -884,13 +1010,13 @@ static int walDecodeFrame(
   assert( WAL_FRAME_HDRSIZE==24 );
 
   /* A frame is only valid if the salt values in the frame-header
-  ** match the salt values in the wal-header. 
+  ** match the salt values in the wal-header.
   */
   if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){
     return 0;
   }
 
-  /* A frame is only valid if the page number is creater than zero.
+  /* A frame is only valid if the page number is greater than zero.
   */
   pgno = sqlite3Get4byte(&aFrame[0]);
   if( pgno==0 ){
@@ -898,15 +1024,15 @@ static int walDecodeFrame(
   }
 
   /* A frame is only valid if a checksum of the WAL header,
-  ** all prior frams, the first 16 bytes of this frame-header, 
-  ** and the frame-data matches the checksum in the last 8 
+  ** all prior frames, the first 16 bytes of this frame-header,
+  ** and the frame-data matches the checksum in the last 8
   ** bytes of this frame-header.
   */
   nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN);
   walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum);
   walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum);
-  if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) 
-   || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) 
+  if( aCksum[0]!=sqlite3Get4byte(&aFrame[16])
+   || aCksum[1]!=sqlite3Get4byte(&aFrame[20])
   ){
     /* Checksum failed. */
     return 0;
@@ -941,7 +1067,7 @@ static const char *walLockName(int lockIdx){
   }
 }
 #endif /*defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */
-    
+
 
 /*
 ** Set or release locks on the WAL.  Locks are either shared or exclusive.
@@ -958,12 +1084,18 @@ static int walLockShared(Wal *pWal, int lockIdx){
   WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal,
             walLockName(lockIdx), rc ? "failed" : "ok"));
   VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); )
+#ifdef SQLITE_USE_SEH
+  if( rc==SQLITE_OK ) pWal->lockMask |= (1 << lockIdx);
+#endif
   return rc;
 }
 static void walUnlockShared(Wal *pWal, int lockIdx){
   if( pWal->exclusiveMode ) return;
   (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1,
                          SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED);
+#ifdef SQLITE_USE_SEH
+  pWal->lockMask &= ~(1 << lockIdx);
+#endif
   WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx)));
 }
 static int walLockExclusive(Wal *pWal, int lockIdx, int n){
@@ -974,12 +1106,20 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){
   WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal,
             walLockName(lockIdx), n, rc ? "failed" : "ok"));
   VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); )
+#ifdef SQLITE_USE_SEH
+  if( rc==SQLITE_OK ){
+    pWal->lockMask |= (((1<<n)-1) << (SQLITE_SHM_NLOCK+lockIdx));
+  }
+#endif
   return rc;
 }
 static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){
   if( pWal->exclusiveMode ) return;
   (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n,
                          SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE);
+#ifdef SQLITE_USE_SEH
+  pWal->lockMask &= ~(((1<<n)-1) << (SQLITE_SHM_NLOCK+lockIdx));
+#endif
   WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal,
              walLockName(lockIdx), n));
 }
@@ -1010,15 +1150,15 @@ struct WalHashLoc {
   u32 iZero;                /* One less than the frame number of first indexed*/
 };
 
-/* 
+/*
 ** Return pointers to the hash table and page number array stored on
 ** page iHash of the wal-index. The wal-index is broken into 32KB pages
 ** numbered starting from 0.
 **
 ** Set output variable pLoc->aHash to point to the start of the hash table
-** in the wal-index file. Set pLoc->iZero to one less than the frame 
+** in the wal-index file. Set pLoc->iZero to one less than the frame
 ** number of the first frame indexed by this hash table. If a
-** slot in the hash table is set to N, it refers to frame number 
+** slot in the hash table is set to N, it refers to frame number
 ** (pLoc->iZero+N) in the log.
 **
 ** Finally, set pLoc->aPgno so that pLoc->aPgno[0] is the page number of the
@@ -1051,7 +1191,7 @@ static int walHashGet(
 /*
 ** Return the number of the wal-index page that contains the hash-table
 ** and page-number array that contain entries corresponding to WAL frame
-** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages 
+** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages
 ** are numbered starting from 0.
 */
 static int walFramePage(u32 iFrame){
@@ -1071,6 +1211,7 @@ static int walFramePage(u32 iFrame){
 */
 static u32 walFramePgno(Wal *pWal, u32 iFrame){
   int iHash = walFramePage(iFrame);
+  SEH_INJECT_FAULT;
   if( iHash==0 ){
     return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1];
   }
@@ -1102,7 +1243,7 @@ static void walCleanupHash(Wal *pWal){
 
   if( pWal->hdr.mxFrame==0 ) return;
 
-  /* Obtain pointers to the hash-table and page-number array containing 
+  /* Obtain pointers to the hash-table and page-number array containing
   ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed
   ** that the page said hash-table and array reside on is already mapped.(1)
   */
@@ -1121,9 +1262,9 @@ static void walCleanupHash(Wal *pWal){
       sLoc.aHash[i] = 0;
     }
   }
-  
+
   /* Zero the entries in the aPgno array that correspond to frames with
-  ** frame numbers greater than pWal->hdr.mxFrame. 
+  ** frame numbers greater than pWal->hdr.mxFrame.
   */
   nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit]);
   assert( nByte>=0 );
@@ -1167,9 +1308,9 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
 
     idx = iFrame - sLoc.iZero;
     assert( idx <= HASHTABLE_NSLOT/2 + 1 );
-    
+
     /* If this is the first entry to be added to this hash-table, zero the
-    ** entire hash table and aPgno[] array before proceeding. 
+    ** entire hash table and aPgno[] array before proceeding.
     */
     if( idx==1 ){
       int nByte = (int)((u8*)&sLoc.aHash[HASHTABLE_NSLOT] - (u8*)sLoc.aPgno);
@@ -1179,8 +1320,8 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
 
     /* If the entry in aPgno[] is already set, then the previous writer
     ** must have exited unexpectedly in the middle of a transaction (after
-    ** writing one or more dirty pages to the WAL to free up memory). 
-    ** Remove the remnants of that writers uncommitted transaction from 
+    ** writing one or more dirty pages to the WAL to free up memory).
+    ** Remove the remnants of that writers uncommitted transaction from
     ** the hash-table before writing any new entries.
     */
     if( sLoc.aPgno[idx-1] ){
@@ -1231,7 +1372,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){
 
 
 /*
-** Recover the wal-index by reading the write-ahead log file. 
+** Recover the wal-index by reading the write-ahead log file.
 **
 ** This routine first tries to establish an exclusive lock on the
 ** wal-index to prevent other threads/processes from doing anything
@@ -1291,16 +1432,16 @@ static int walIndexRecover(Wal *pWal){
     }
 
     /* If the database page size is not a power of two, or is greater than
-    ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid 
+    ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid
     ** data. Similarly, if the 'magic' value is invalid, ignore the whole
     ** WAL file.
     */
     magic = sqlite3Get4byte(&aBuf[0]);
     szPage = sqlite3Get4byte(&aBuf[8]);
-    if( (magic&0xFFFFFFFE)!=WAL_MAGIC 
-     || szPage&(szPage-1) 
-     || szPage>SQLITE_MAX_PAGE_SIZE 
-     || szPage<512 
+    if( (magic&0xFFFFFFFE)!=WAL_MAGIC
+     || szPage&(szPage-1)
+     || szPage>SQLITE_MAX_PAGE_SIZE
+     || szPage<512
     ){
       goto finished;
     }
@@ -1310,7 +1451,7 @@ static int walIndexRecover(Wal *pWal){
     memcpy(&pWal->hdr.aSalt, &aBuf[16], 8);
 
     /* Verify that the WAL header checksum is correct */
-    walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, 
+    walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN,
         aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum
     );
     if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24])
@@ -1330,6 +1471,7 @@ static int walIndexRecover(Wal *pWal){
     /* Malloc a buffer to read frames into. */
     szFrame = szPage + WAL_FRAME_HDRSIZE;
     aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ);
+    SEH_FREE_ON_ERROR(0, aFrame);
     if( !aFrame ){
       rc = SQLITE_NOMEM_BKPT;
       goto recovery_error;
@@ -1348,8 +1490,9 @@ static int walIndexRecover(Wal *pWal){
       rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare);
       assert( aShare!=0 || rc!=SQLITE_OK );
       if( aShare==0 ) break;
+      SEH_SET_ON_ERROR(iPg, aShare);
       pWal->apWiData[iPg] = aPrivate;
-      
+
       for(iFrame=iFirst; iFrame<=iLast; iFrame++){
         i64 iOffset = walFrameOffset(iFrame, szPage);
         u32 pgno;                 /* Database page number for frame */
@@ -1375,6 +1518,7 @@ static int walIndexRecover(Wal *pWal){
         }
       }
       pWal->apWiData[iPg] = aShare;
+      SEH_SET_ON_ERROR(0,0);
       nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0);
       nHdr32 = nHdr / sizeof(u32);
 #ifndef SQLITE_SAFER_WALINDEX_RECOVERY
@@ -1405,9 +1549,11 @@ static int walIndexRecover(Wal *pWal){
         }
       }
 #endif
+      SEH_INJECT_FAULT;
       if( iFrame<=iLast ) break;
     }
 
+    SEH_FREE_ON_ERROR(aFrame, 0);
     sqlite3_free(aFrame);
   }
 
@@ -1419,8 +1565,8 @@ static int walIndexRecover(Wal *pWal){
     pWal->hdr.aFrameCksum[1] = aFrameCksum[1];
     walIndexWriteHdr(pWal);
 
-    /* Reset the checkpoint-header. This is safe because this thread is 
-    ** currently holding locks that exclude all other writers and 
+    /* Reset the checkpoint-header. This is safe because this thread is
+    ** currently holding locks that exclude all other writers and
     ** checkpointers. Then set the values of read-mark slots 1 through N.
     */
     pInfo = walCkptInfo(pWal);
@@ -1435,6 +1581,7 @@ static int walIndexRecover(Wal *pWal){
         }else{
           pInfo->aReadMark[i] = READMARK_NOT_USED;
         }
+        SEH_INJECT_FAULT;
         walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
       }else if( rc!=SQLITE_BUSY ){
         goto recovery_error;
@@ -1476,8 +1623,8 @@ static void walIndexClose(Wal *pWal, int isDelete){
   }
 }
 
-/* 
-** Open a connection to the WAL file zWalName. The database file must 
+/*
+** Open a connection to the WAL file zWalName. The database file must
 ** already be opened on connection pDbFd. The buffer that zWalName points
 ** to must remain valid for the lifetime of the returned Wal* handle.
 **
@@ -1487,7 +1634,7 @@ static void walIndexClose(Wal *pWal, int isDelete){
 ** were to do this just after this client opened one of these files, the
 ** system would be badly broken.
 **
-** If the log file is successfully opened, SQLITE_OK is returned and 
+** If the log file is successfully opened, SQLITE_OK is returned and
 ** *ppWal is set to point to a new WAL handle. If an error occurs,
 ** an SQLite error code is returned and *ppWal is left unmodified.
 */
@@ -1592,7 +1739,7 @@ int sqlite3WalOpen(
 }
 
 /*
-** Change the size to which the WAL file is trucated on each reset.
+** Change the size to which the WAL file is truncated on each reset.
 */
 void sqlite3WalLimit(Wal *pWal, i64 iLimit){
   if( pWal ) pWal->mxWalSize = iLimit;
@@ -1680,7 +1827,7 @@ static void walMerge(
     ht_slot logpage;
     Pgno dbpage;
 
-    if( (iLeft<nLeft) 
+    if( (iLeft<nLeft)
      && (iRight>=nRight || aContent[aLeft[iLeft]]<aContent[aRight[iRight]])
     ){
       logpage = aLeft[iLeft++];
@@ -1778,7 +1925,7 @@ static void walMergesort(
 #endif
 }
 
-/* 
+/*
 ** Free an iterator allocated by walIteratorInit().
 */
 static void walIteratorFree(WalIterator *p){
@@ -1786,7 +1933,7 @@ static void walIteratorFree(WalIterator *p){
 }
 
 /*
-** Construct a WalInterator object that can be used to loop over all 
+** Construct a WalInterator object that can be used to loop over all
 ** pages in the WAL following frame nBackfill in ascending order. Frames
 ** nBackfill or earlier may be included - excluding them is an optimization
 ** only. The caller must hold the checkpoint lock.
@@ -1815,26 +1962,18 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
 
   /* Allocate space for the WalIterator object. */
   nSegment = walFramePage(iLast) + 1;
-  nByte = sizeof(WalIterator) 
-        + (nSegment-1)*sizeof(struct WalSegment)
+  nByte = SZ_WALITERATOR(nSegment)
         + iLast*sizeof(ht_slot);
-  p = (WalIterator *)sqlite3_malloc64(nByte);
+  p = (WalIterator *)sqlite3_malloc64(nByte
+      + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast)
+  );
   if( !p ){
     return SQLITE_NOMEM_BKPT;
   }
   memset(p, 0, nByte);
   p->nSegment = nSegment;
-
-  /* Allocate temporary space used by the merge-sort routine. This block
-  ** of memory will be freed before this function returns.
-  */
-  aTmp = (ht_slot *)sqlite3_malloc64(
-      sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast)
-  );
-  if( !aTmp ){
-    rc = SQLITE_NOMEM_BKPT;
-  }
-
+  aTmp = (ht_slot*)&(((u8*)p)[nByte]);
+  SEH_FREE_ON_ERROR(0, p);
   for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && i<nSegment; i++){
     WalHashLoc sLoc;
 
@@ -1851,7 +1990,7 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
       }
       aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero];
       sLoc.iZero++;
-  
+
       for(j=0; j<nEntry; j++){
         aIndex[j] = (ht_slot)j;
       }
@@ -1862,9 +2001,8 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
       p->aSegment[i].aPgno = (u32 *)sLoc.aPgno;
     }
   }
-  sqlite3_free(aTmp);
-
   if( rc!=SQLITE_OK ){
+    SEH_FREE_ON_ERROR(p, 0);
     walIteratorFree(p);
     p = 0;
   }
@@ -1873,22 +2011,31 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
 }
 
 #ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+
+
+/*
+** Attempt to enable blocking locks that block for nMs ms. Return 1 if 
+** blocking locks are successfully enabled, or 0 otherwise.
+*/
+static int walEnableBlockingMs(Wal *pWal, int nMs){
+  int rc = sqlite3OsFileControl(
+      pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&nMs
+  );
+  return (rc==SQLITE_OK);
+}
+
 /*
 ** Attempt to enable blocking locks. Blocking locks are enabled only if (a)
-** they are supported by the VFS, and (b) the database handle is configured 
-** with a busy-timeout. Return 1 if blocking locks are successfully enabled, 
+** they are supported by the VFS, and (b) the database handle is configured
+** with a busy-timeout. Return 1 if blocking locks are successfully enabled,
 ** or 0 otherwise.
 */
 static int walEnableBlocking(Wal *pWal){
   int res = 0;
   if( pWal->db ){
-    int tmout = pWal->db->busyTimeout;
+    int tmout = pWal->db->setlkTimeout;
     if( tmout ){
-      int rc;
-      rc = sqlite3OsFileControl(
-          pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout
-      );
-      res = (rc==SQLITE_OK);
+      res = walEnableBlockingMs(pWal, tmout);
     }
   }
   return res;
@@ -1905,7 +2052,7 @@ static void walDisableBlocking(Wal *pWal){
 /*
 ** If parameter bLock is true, attempt to enable blocking locks, take
 ** the WRITER lock, and then disable blocking locks. If blocking locks
-** cannot be enabled, no attempt to obtain the WRITER lock is made. Return 
+** cannot be enabled, no attempt to obtain the WRITER lock is made. Return
 ** an SQLite error code if an error occurs, or SQLITE_OK otherwise. It is not
 ** an error if blocking locks can not be enabled.
 **
@@ -1937,20 +2084,10 @@ void sqlite3WalDb(Wal *pWal, sqlite3 *db){
   pWal->db = db;
 }
 
-/*
-** Take an exclusive WRITE lock. Blocking if so configured.
-*/
-static int walLockWriter(Wal *pWal){
-  int rc;
-  walEnableBlocking(pWal);
-  rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1);
-  walDisableBlocking(pWal);
-  return rc;
-}
 #else
 # define walEnableBlocking(x) 0
 # define walDisableBlocking(x)
-# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1)
+# define walEnableBlockingMs(pWal, ms) 0
 # define sqlite3WalDb(pWal, db)
 #endif   /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */
 
@@ -2002,8 +2139,8 @@ static int walPagesize(Wal *pWal){
 ** client to write to the database (which may be this one) does so by
 ** writing frames into the start of the log file.
 **
-** The value of parameter salt1 is used as the aSalt[1] value in the 
-** new wal-index header. It should be passed a pseudo-random value (i.e. 
+** The value of parameter salt1 is used as the aSalt[1] value in the
+** new wal-index header. It should be passed a pseudo-random value (i.e.
 ** one obtained from sqlite3_randomness()).
 */
 static void walRestartHdr(Wal *pWal, u32 salt1){
@@ -2031,8 +2168,8 @@ static void walRestartHdr(Wal *pWal, u32 salt1){
 ** that a concurrent reader might be using.
 **
 ** All I/O barrier operations (a.k.a fsyncs) occur in this routine when
-** SQLite is in WAL-mode in synchronous=NORMAL.  That means that if 
-** checkpoints are always run by a background thread or background 
+** SQLite is in WAL-mode in synchronous=NORMAL.  That means that if
+** checkpoints are always run by a background thread or background
 ** process, foreground threads will never block on a lengthy fsync call.
 **
 ** Fsync is called on the WAL before writing content out of the WAL and
@@ -2045,7 +2182,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){
 ** database file.
 **
 ** This routine uses and updates the nBackfill field of the wal-index header.
-** This is the only routine that will increase the value of nBackfill.  
+** This is the only routine that will increase the value of nBackfill.
 ** (A WAL reset or recovery will revert nBackfill to zero, but not increase
 ** its value.)
 **
@@ -2090,13 +2227,13 @@ static int walCheckpoint(
     mxSafeFrame = pWal->hdr.mxFrame;
     mxPage = pWal->hdr.nPage;
     for(i=1; i<WAL_NREADER; i++){
-      u32 y = AtomicLoad(pInfo->aReadMark+i);
+      u32 y = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT;
       if( mxSafeFrame>y ){
         assert( y<=pWal->hdr.mxFrame );
         rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
         if( rc==SQLITE_OK ){
           u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED);
-          AtomicStore(pInfo->aReadMark+i, iMark);
+          AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT;
           walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
         }else if( rc==SQLITE_BUSY ){
           mxSafeFrame = y;
@@ -2117,8 +2254,7 @@ static int walCheckpoint(
      && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK
     ){
       u32 nBackfill = pInfo->nBackfill;
-
-      pInfo->nBackfillAttempted = mxSafeFrame;
+      pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT;
 
       /* Sync the WAL to disk */
       rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags));
@@ -2135,7 +2271,7 @@ static int walCheckpoint(
           if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){
             /* If the size of the final database is larger than the current
             ** database plus the amount of data in the wal file, plus the
-            ** maximum size of the pending-byte page (65536 bytes), then 
+            ** maximum size of the pending-byte page (65536 bytes), then
             ** must be corruption somewhere.  */
             rc = SQLITE_CORRUPT_BKPT;
           }else{
@@ -2149,6 +2285,7 @@ static int walCheckpoint(
       while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
         i64 iOffset;
         assert( walFramePgno(pWal, iFrame)==iDbpage );
+        SEH_INJECT_FAULT;
         if( AtomicLoad(&db->u1.isInterrupted) ){
           rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT;
           break;
@@ -2178,7 +2315,7 @@ static int walCheckpoint(
           }
         }
         if( rc==SQLITE_OK ){
-          AtomicStore(&pInfo->nBackfill, mxSafeFrame);
+          AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT;
         }
       }
 
@@ -2194,12 +2331,13 @@ static int walCheckpoint(
   }
 
   /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the
-  ** entire wal file has been copied into the database file, then block 
-  ** until all readers have finished using the wal file. This ensures that 
+  ** entire wal file has been copied into the database file, then block
+  ** until all readers have finished using the wal file. This ensures that
   ** the next process to write to the database restarts the wal file.
   */
   if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){
     assert( pWal->writeLock );
+    SEH_INJECT_FAULT;
     if( pInfo->nBackfill<pWal->hdr.mxFrame ){
       rc = SQLITE_BUSY;
     }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){
@@ -2219,7 +2357,7 @@ static int walCheckpoint(
           ** writer clients should see that the entire log file has been
           ** checkpointed and behave accordingly. This seems unsafe though,
           ** as it would leave the system in a state where the contents of
-          ** the wal-index header do not match the contents of the 
+          ** the wal-index header do not match the contents of the
           ** file-system. To avoid this, update the wal-index header to
           ** indicate that the log file contains zero valid frames.  */
           walRestartHdr(pWal, salt1);
@@ -2231,6 +2369,7 @@ static int walCheckpoint(
   }
 
  walcheckpoint_out:
+  SEH_FREE_ON_ERROR(pIter, 0);
   walIteratorFree(pIter);
   return rc;
 }
@@ -2253,6 +2392,95 @@ static void walLimitSize(Wal *pWal, i64 nMax){
   }
 }
 
+#ifdef SQLITE_USE_SEH
+/*
+** This is the "standard" exception handler used in a few places to handle 
+** an exception thrown by reading from the *-shm mapping after it has become
+** invalid in SQLITE_USE_SEH builds. It is used as follows:
+**
+**   SEH_TRY { ... }
+**   SEH_EXCEPT( rc = walHandleException(pWal); )
+**
+** This function does three things:
+**
+**   1) Determines the locks that should be held, based on the contents of
+**      the Wal.readLock, Wal.writeLock and Wal.ckptLock variables. All other
+**      held locks are assumed to be transient locks that would have been
+**      released had the exception not been thrown and are dropped.
+**
+**   2) Frees the pointer at Wal.pFree, if any, using sqlite3_free().
+**
+**   3) Set pWal->apWiData[pWal->iWiPg] to pWal->pWiValue if not NULL
+**
+**   4) Returns SQLITE_IOERR.
+*/
+static int walHandleException(Wal *pWal){
+  if( pWal->exclusiveMode==0 ){
+    static const int S = 1;
+    static const int E = (1<<SQLITE_SHM_NLOCK);
+    int ii;
+    u32 mUnlock;
+    if( pWal->writeLock==2 ) pWal->writeLock = 0;
+    mUnlock = pWal->lockMask & ~(
+        (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock)))
+        | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0)
+        | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0)
+        );
+    for(ii=0; ii<SQLITE_SHM_NLOCK; ii++){
+      if( (S<<ii) & mUnlock ) walUnlockShared(pWal, ii);
+      if( (E<<ii) & mUnlock ) walUnlockExclusive(pWal, ii, 1);
+    }
+  }
+  sqlite3_free(pWal->pFree);
+  pWal->pFree = 0;
+  if( pWal->pWiValue ){
+    pWal->apWiData[pWal->iWiPg] = pWal->pWiValue;
+    pWal->pWiValue = 0;
+  }
+  return SQLITE_IOERR_IN_PAGE;
+}
+
+/*
+** Assert that the Wal.lockMask mask, which indicates the locks held
+** by the connection, is consistent with the Wal.readLock, Wal.writeLock
+** and Wal.ckptLock variables. To be used as:
+**
+**   assert( walAssertLockmask(pWal) );
+*/
+static int walAssertLockmask(Wal *pWal){
+  if( pWal->exclusiveMode==0 ){
+    static const int S = 1;
+    static const int E = (1<<SQLITE_SHM_NLOCK);
+    u32 mExpect = (
+        (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock)))
+      | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0)
+      | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0)
+#ifdef SQLITE_ENABLE_SNAPSHOT
+      | (pWal->pSnapshot ? (pWal->lockMask & (1 << WAL_CKPT_LOCK)) : 0)
+#endif
+    );
+    assert( mExpect==pWal->lockMask );
+  }
+  return 1;
+}
+
+/*
+** Return and zero the "system error" field set when an 
+** EXCEPTION_IN_PAGE_ERROR exception is caught.
+*/
+int sqlite3WalSystemErrno(Wal *pWal){
+  int iRet = 0;
+  if( pWal ){
+    iRet = pWal->iSysErrno;
+    pWal->iSysErrno = 0;
+  }
+  return iRet;
+}
+
+#else
+# define walAssertLockmask(x) 1
+#endif /* ifdef SQLITE_USE_SEH */
+
 /*
 ** Close a connection to a log file.
 */
@@ -2267,6 +2495,8 @@ int sqlite3WalClose(
   if( pWal ){
     int isDelete = 0;             /* True to unlink wal and wal-index files */
 
+    assert( walAssertLockmask(pWal) );
+
     /* If an EXCLUSIVE lock can be obtained on the database file (using the
     ** ordinary, rollback-mode locking methods, this guarantees that the
     ** connection associated with this log file is the only connection to
@@ -2281,7 +2511,7 @@ int sqlite3WalClose(
       if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
-      rc = sqlite3WalCheckpoint(pWal, db, 
+      rc = sqlite3WalCheckpoint(pWal, db,
           SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
       );
       if( rc==SQLITE_OK ){
@@ -2291,7 +2521,7 @@ int sqlite3WalClose(
         );
         if( bPersist!=1 ){
           /* Try to delete the WAL file if the checkpoint completed and
-          ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal
+          ** fsynced (rc==SQLITE_OK) and if we are not in persistent-wal
           ** mode (!bPersist) */
           isDelete = 1;
         }else if( pWal->mxWalSize>=0 ){
@@ -2358,7 +2588,7 @@ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){
   ** give false-positive warnings about these accesses because the tools do not
   ** account for the double-read and the memory barrier. The use of mutexes
   ** here would be problematic as the memory being accessed is potentially
-  ** shared among multiple processes and not all mutex implementions work
+  ** shared among multiple processes and not all mutex implementations work
   ** reliably in that environment.
   */
   aHdr = walIndexHdr(pWal);
@@ -2368,7 +2598,7 @@ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){
 
   if( memcmp(&h1, &h2, sizeof(h1))!=0 ){
     return 1;   /* Dirty read */
-  }  
+  }
   if( h1.isInit==0 ){
     return 1;   /* Malformed header - probably all zeros */
   }
@@ -2404,7 +2634,7 @@ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){
 ** changed by this operation.  If pWal->hdr is unchanged, set *pChanged
 ** to 0.
 **
-** If the wal-index header is successfully read, return SQLITE_OK. 
+** If the wal-index header is successfully read, return SQLITE_OK.
 ** Otherwise an SQLite error code.
 */
 static int walIndexReadHdr(Wal *pWal, int *pChanged){
@@ -2412,7 +2642,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
   int badHdr;                     /* True if a header read failed */
   volatile u32 *page0;            /* Chunk of wal-index containing header */
 
-  /* Ensure that page 0 of the wal-index (the page that contains the 
+  /* Ensure that page 0 of the wal-index (the page that contains the
   ** wal-index header) is mapped. Return early if an error occurs here.
   */
   assert( pChanged );
@@ -2444,7 +2674,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 
   /* If the first page of the wal-index has been mapped, try to read the
   ** wal-index header immediately, without holding any lock. This usually
-  ** works, but may fail if the wal-index header is corrupt or currently 
+  ** works, but may fail if the wal-index header is corrupt or currently
   ** being modified by another thread or process.
   */
   badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1);
@@ -2460,15 +2690,23 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
       }
     }else{
       int bWriteLock = pWal->writeLock;
-      if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){
-        pWal->writeLock = 1;
+      if( bWriteLock 
+       || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) 
+      ){
+        /* If the write-lock was just obtained, set writeLock to 2 instead of
+        ** the usual 1. This causes walIndexPage() to behave as if the 
+        ** write-lock were held (so that it allocates new pages as required),
+        ** and walHandleException() to unlock the write-lock if a SEH exception
+        ** is thrown.  */
+        if( !bWriteLock ) pWal->writeLock = 2;
         if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){
           badHdr = walIndexTryHdr(pWal, pChanged);
           if( badHdr ){
             /* If the wal-index header is still malformed even while holding
             ** a WRITE lock, it can only mean that the header is corrupted and
             ** needs to be reconstructed.  So run recovery to do exactly that.
-            */
+            ** Disable blocking locks first.  */
+            walDisableBlocking(pWal);
             rc = walIndexRecover(pWal);
             *pChanged = 1;
           }
@@ -2519,15 +2757,15 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){
 **
 **    The *-wal file has been read and an appropriate wal-index has been
 **    constructed in pWal->apWiData[] using heap memory instead of shared
-**    memory. 
+**    memory.
 **
 ** If this function returns SQLITE_OK, then the read transaction has
-** been successfully opened. In this case output variable (*pChanged) 
+** been successfully opened. In this case output variable (*pChanged)
 ** is set to true before returning if the caller should discard the
-** contents of the page cache before proceeding. Or, if it returns 
-** WAL_RETRY, then the heap memory wal-index has been discarded and 
-** the caller should retry opening the read transaction from the 
-** beginning (including attempting to map the *-shm file). 
+** contents of the page cache before proceeding. Or, if it returns
+** WAL_RETRY, then the heap memory wal-index has been discarded and
+** the caller should retry opening the read transaction from the
+** beginning (including attempting to map the *-shm file).
 **
 ** If an error occurs, an SQLite error code is returned.
 */
@@ -2640,8 +2878,8 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   ** the caller.  */
   aSaveCksum[0] = pWal->hdr.aFrameCksum[0];
   aSaveCksum[1] = pWal->hdr.aFrameCksum[1];
-  for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->szPage); 
-      iOffset+szFrame<=szWal; 
+  for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->szPage);
+      iOffset+szFrame<=szWal;
       iOffset+=szFrame
   ){
     u32 pgno;                   /* Database page number for frame */
@@ -2678,6 +2916,37 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
   return rc;
 }
 
+/*
+** The final argument passed to walTryBeginRead() is of type (int*). The
+** caller should invoke walTryBeginRead as follows:
+**
+**   int cnt = 0;
+**   do {
+**     rc = walTryBeginRead(..., &cnt);
+**   }while( rc==WAL_RETRY );
+**
+** The final value of "cnt" is of no use to the caller. It is used by
+** the implementation of walTryBeginRead() as follows:
+**
+**   + Each time walTryBeginRead() is called, it is incremented. Once
+**     it reaches WAL_RETRY_PROTOCOL_LIMIT - indicating that walTryBeginRead()
+**     has many times been invoked and failed with WAL_RETRY - walTryBeginRead()
+**     returns SQLITE_PROTOCOL.
+**
+**   + If SQLITE_ENABLE_SETLK_TIMEOUT is defined and walTryBeginRead() failed
+**     because a blocking lock timed out (SQLITE_BUSY_TIMEOUT from the OS
+**     layer), the WAL_RETRY_BLOCKED_MASK bit is set in "cnt". In this case
+**     the next invocation of walTryBeginRead() may omit an expected call to 
+**     sqlite3OsSleep(). There has already been a delay when the previous call
+**     waited on a lock.
+*/
+#define WAL_RETRY_PROTOCOL_LIMIT 100
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+# define WAL_RETRY_BLOCKED_MASK    0x10000000
+#else
+# define WAL_RETRY_BLOCKED_MASK    0
+#endif
+
 /*
 ** Attempt to start a read transaction.  This might fail due to a race or
 ** other transient condition.  When that happens, it returns WAL_RETRY to
@@ -2689,10 +2958,10 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
 **
 ** The useWal parameter is true to force the use of the WAL and disable
 ** the case where the WAL is bypassed because it has been completely
-** checkpointed.  If useWal==0 then this routine calls walIndexReadHdr() 
-** to make a copy of the wal-index header into pWal->hdr.  If the 
-** wal-index header has changed, *pChanged is set to 1 (as an indication 
-** to the caller that the local page cache is obsolete and needs to be 
+** checkpointed.  If useWal==0 then this routine calls walIndexReadHdr()
+** to make a copy of the wal-index header into pWal->hdr.  If the
+** wal-index header has changed, *pChanged is set to 1 (as an indication
+** to the caller that the local page cache is obsolete and needs to be
 ** flushed.)  When useWal==1, the wal-index header is assumed to already
 ** be loaded and the pChanged parameter is unused.
 **
@@ -2707,7 +2976,7 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
 ** bad luck when there is lots of contention for the wal-index, but that
 ** possibility is so small that it can be safely neglected, we believe.
 **
-** On success, this routine obtains a read lock on 
+** On success, this routine obtains a read lock on
 ** WAL_READ_LOCK(pWal->readLock).  The pWal->readLock integer is
 ** in the range 0 <= pWal->readLock < WAL_NREADER.  If pWal->readLock==(-1)
 ** that means the Wal does not hold any read lock.  The reader must not
@@ -2728,13 +2997,12 @@ static int walBeginShmUnreliable(Wal *pWal, int *pChanged){
 ** so it takes care to hold an exclusive lock on the corresponding
 ** WAL_READ_LOCK() while changing values.
 */
-static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
+static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){
   volatile WalCkptInfo *pInfo;    /* Checkpoint information in wal-index */
-  u32 mxReadMark;                 /* Largest aReadMark[] value */
-  int mxI;                        /* Index of largest aReadMark[] value */
-  int i;                          /* Loop counter */
   int rc = SQLITE_OK;             /* Return code  */
-  u32 mxFrame;                    /* Wal frame to lock to */
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+  int nBlockTmout = 0;
+#endif
 
   assert( pWal->readLock<0 );     /* Not currently locked */
 
@@ -2745,27 +3013,47 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   **
   ** Circumstances that cause a RETRY should only last for the briefest
   ** instances of time.  No I/O or other system calls are done while the
-  ** locks are held, so the locks should not be held for very long. But 
+  ** locks are held, so the locks should not be held for very long. But
   ** if we are unlucky, another process that is holding a lock might get
-  ** paged out or take a page-fault that is time-consuming to resolve, 
+  ** paged out or take a page-fault that is time-consuming to resolve,
   ** during the few nanoseconds that it is holding the lock.  In that case,
   ** it might take longer than normal for the lock to free.
   **
   ** After 5 RETRYs, we begin calling sqlite3OsSleep().  The first few
   ** calls to sqlite3OsSleep() have a delay of 1 microsecond.  Really this
   ** is more of a scheduler yield than an actual delay.  But on the 10th
-  ** an subsequent retries, the delays start becoming longer and longer, 
+  ** an subsequent retries, the delays start becoming longer and longer,
   ** so that on the 100th (and last) RETRY we delay for 323 milliseconds.
   ** The total delay time before giving up is less than 10 seconds.
   */
-  if( cnt>5 ){
+  (*pCnt)++;
+  if( *pCnt>5 ){
     int nDelay = 1;                      /* Pause time in microseconds */
-    if( cnt>100 ){
+    int cnt = (*pCnt & ~WAL_RETRY_BLOCKED_MASK);
+    if( cnt>WAL_RETRY_PROTOCOL_LIMIT ){
       VVA_ONLY( pWal->lockError = 1; )
       return SQLITE_PROTOCOL;
     }
-    if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39;
+    if( *pCnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39;
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor
+    ** to block for locks for approximately nDelay us. This affects three
+    ** locks: (a) the shared lock taken on the DMS slot in os_unix.c (if
+    ** using os_unix.c), (b) the WRITER lock taken in walIndexReadHdr() if the
+    ** first attempted read fails, and (c) the shared lock taken on the 
+    ** read-mark.  
+    **
+    ** If the previous call failed due to an SQLITE_BUSY_TIMEOUT error,
+    ** then sleep for the minimum of 1us. The previous call already provided 
+    ** an extra delay while it was blocking on the lock.
+    */
+    nBlockTmout = (nDelay+998) / 1000;
+    if( !useWal && walEnableBlockingMs(pWal, nBlockTmout) ){
+      if( *pCnt & WAL_RETRY_BLOCKED_MASK ) nDelay = 1;
+    }
+#endif
     sqlite3OsSleep(pWal->pVfs, nDelay);
+    *pCnt &= ~WAL_RETRY_BLOCKED_MASK;
   }
 
   if( !useWal ){
@@ -2773,6 +3061,12 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
     if( pWal->bShmUnreliable==0 ){
       rc = walIndexReadHdr(pWal, pChanged);
     }
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+    if( rc==SQLITE_BUSY_TIMEOUT ){
+      rc = SQLITE_BUSY;
+      *pCnt |= WAL_RETRY_BLOCKED_MASK;
+    }
+#endif
     if( rc==SQLITE_BUSY ){
       /* If there is not a recovery running in another thread or process
       ** then convert BUSY errors to WAL_RETRY.  If recovery is known to
@@ -2782,12 +3076,13 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
       ** WAL_RETRY this routine will be called again and will probably be
       ** right on the second iteration.
       */
+      (void)walEnableBlocking(pWal);
       if( pWal->apWiData[0]==0 ){
         /* This branch is taken when the xShmMap() method returns SQLITE_BUSY.
         ** We assume this is a transient condition, so return WAL_RETRY. The
-        ** xShmMap() implementation used by the default unix and win32 VFS 
-        ** modules may return SQLITE_BUSY due to a race condition in the 
-        ** code that determines whether or not the shared-memory region 
+        ** xShmMap() implementation used by the default unix and win32 VFS
+        ** modules may return SQLITE_BUSY due to a race condition in the
+        ** code that determines whether or not the shared-memory region
         ** must be zeroed before the requested page is returned.
         */
         rc = WAL_RETRY;
@@ -2798,6 +3093,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
         rc = SQLITE_BUSY_RECOVERY;
       }
     }
+    walDisableBlocking(pWal);
     if( rc!=SQLITE_OK ){
       return rc;
     }
@@ -2809,146 +3105,211 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   assert( pWal->nWiData>0 );
   assert( pWal->apWiData[0]!=0 );
   pInfo = walCkptInfo(pWal);
-  if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame
+  SEH_INJECT_FAULT;
+  {
+    u32 mxReadMark;               /* Largest aReadMark[] value */
+    int mxI;                      /* Index of largest aReadMark[] value */
+    int i;                        /* Loop counter */
+    u32 mxFrame;                  /* Wal frame to lock to */
+    if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame
 #ifdef SQLITE_ENABLE_SNAPSHOT
-   && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0)
+     && ((pWal->bGetSnapshot==0 && pWal->pSnapshot==0) || pWal->hdr.mxFrame==0)
 #endif
-  ){
-    /* The WAL has been completely backfilled (or it is empty).
-    ** and can be safely ignored.
-    */
-    rc = walLockShared(pWal, WAL_READ_LOCK(0));
-    walShmBarrier(pWal);
-    if( rc==SQLITE_OK ){
-      if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){
-        /* It is not safe to allow the reader to continue here if frames
-        ** may have been appended to the log before READ_LOCK(0) was obtained.
-        ** When holding READ_LOCK(0), the reader ignores the entire log file,
-        ** which implies that the database file contains a trustworthy
-        ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from
-        ** happening, this is usually correct.
-        **
-        ** However, if frames have been appended to the log (or if the log 
-        ** is wrapped and written for that matter) before the READ_LOCK(0)
-        ** is obtained, that is not necessarily true. A checkpointer may
-        ** have started to backfill the appended frames but crashed before
-        ** it finished. Leaving a corrupt image in the database file.
-        */
-        walUnlockShared(pWal, WAL_READ_LOCK(0));
-        return WAL_RETRY;
+    ){
+      /* The WAL has been completely backfilled (or it is empty).
+      ** and can be safely ignored.
+      */
+      rc = walLockShared(pWal, WAL_READ_LOCK(0));
+      walShmBarrier(pWal);
+      if( rc==SQLITE_OK ){
+        if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr,sizeof(WalIndexHdr)) ){
+          /* It is not safe to allow the reader to continue here if frames
+          ** may have been appended to the log before READ_LOCK(0) was obtained.
+          ** When holding READ_LOCK(0), the reader ignores the entire log file,
+          ** which implies that the database file contains a trustworthy
+          ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from
+          ** happening, this is usually correct.
+          **
+          ** However, if frames have been appended to the log (or if the log
+          ** is wrapped and written for that matter) before the READ_LOCK(0)
+          ** is obtained, that is not necessarily true. A checkpointer may
+          ** have started to backfill the appended frames but crashed before
+          ** it finished. Leaving a corrupt image in the database file.
+          */
+          walUnlockShared(pWal, WAL_READ_LOCK(0));
+          return WAL_RETRY;
+        }
+        pWal->readLock = 0;
+        return SQLITE_OK;
+      }else if( rc!=SQLITE_BUSY ){
+        return rc;
       }
-      pWal->readLock = 0;
-      return SQLITE_OK;
-    }else if( rc!=SQLITE_BUSY ){
-      return rc;
     }
-  }
-
-  /* If we get this far, it means that the reader will want to use
-  ** the WAL to get at content from recent commits.  The job now is
-  ** to select one of the aReadMark[] entries that is closest to
-  ** but not exceeding pWal->hdr.mxFrame and lock that entry.
-  */
-  mxReadMark = 0;
-  mxI = 0;
-  mxFrame = pWal->hdr.mxFrame;
+  
+    /* If we get this far, it means that the reader will want to use
+    ** the WAL to get at content from recent commits.  The job now is
+    ** to select one of the aReadMark[] entries that is closest to
+    ** but not exceeding pWal->hdr.mxFrame and lock that entry.
+    */
+    mxReadMark = 0;
+    mxI = 0;
+    mxFrame = pWal->hdr.mxFrame;
 #ifdef SQLITE_ENABLE_SNAPSHOT
-  if( pWal->pSnapshot && pWal->pSnapshot->mxFrame<mxFrame ){
-    mxFrame = pWal->pSnapshot->mxFrame;
-  }
-#endif
-  for(i=1; i<WAL_NREADER; i++){
-    u32 thisMark = AtomicLoad(pInfo->aReadMark+i);
-    if( mxReadMark<=thisMark && thisMark<=mxFrame ){
-      assert( thisMark!=READMARK_NOT_USED );
-      mxReadMark = thisMark;
-      mxI = i;
+    if( pWal->pSnapshot && pWal->pSnapshot->mxFrame<mxFrame ){
+      mxFrame = pWal->pSnapshot->mxFrame;
     }
-  }
-  if( (pWal->readOnly & WAL_SHM_RDONLY)==0
-   && (mxReadMark<mxFrame || mxI==0)
-  ){
+#endif
     for(i=1; i<WAL_NREADER; i++){
-      rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      if( rc==SQLITE_OK ){
-        AtomicStore(pInfo->aReadMark+i,mxFrame);
-        mxReadMark = mxFrame;
+      u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT;
+      if( mxReadMark<=thisMark && thisMark<=mxFrame ){
+        assert( thisMark!=READMARK_NOT_USED );
+        mxReadMark = thisMark;
         mxI = i;
-        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-        break;
-      }else if( rc!=SQLITE_BUSY ){
-        return rc;
       }
     }
+    if( (pWal->readOnly & WAL_SHM_RDONLY)==0
+     && (mxReadMark<mxFrame || mxI==0)
+    ){
+      for(i=1; i<WAL_NREADER; i++){
+        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        if( rc==SQLITE_OK ){
+          AtomicStore(pInfo->aReadMark+i,mxFrame);
+          mxReadMark = mxFrame;
+          mxI = i;
+          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+          break;
+        }else if( rc!=SQLITE_BUSY ){
+          return rc;
+        }
+      }
+    }
+    if( mxI==0 ){
+      assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
+      return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
+    }
+  
+    (void)walEnableBlockingMs(pWal, nBlockTmout);
+    rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
+    walDisableBlocking(pWal);
+    if( rc ){
+#ifdef SQLITE_ENABLE_SETLK_TIMEOUT
+      if( rc==SQLITE_BUSY_TIMEOUT ){
+        *pCnt |= WAL_RETRY_BLOCKED_MASK;
+      }
+#else
+      assert( rc!=SQLITE_BUSY_TIMEOUT );
+#endif
+      assert((rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT);
+      return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc;
+    }
+    /* Now that the read-lock has been obtained, check that neither the
+    ** value in the aReadMark[] array or the contents of the wal-index
+    ** header have changed.
+    **
+    ** It is necessary to check that the wal-index header did not change
+    ** between the time it was read and when the shared-lock was obtained
+    ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility
+    ** that the log file may have been wrapped by a writer, or that frames
+    ** that occur later in the log than pWal->hdr.mxFrame may have been
+    ** copied into the database by a checkpointer. If either of these things
+    ** happened, then reading the database with the current value of
+    ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry
+    ** instead.
+    **
+    ** Before checking that the live wal-index header has not changed
+    ** since it was read, set Wal.minFrame to the first frame in the wal
+    ** file that has not yet been checkpointed. This client will not need
+    ** to read any frames earlier than minFrame from the wal file - they
+    ** can be safely read directly from the database file.
+    **
+    ** Because a ShmBarrier() call is made between taking the copy of
+    ** nBackfill and checking that the wal-header in shared-memory still
+    ** matches the one cached in pWal->hdr, it is guaranteed that the
+    ** checkpointer that set nBackfill was not working with a wal-index
+    ** header newer than that cached in pWal->hdr. If it were, that could
+    ** cause a problem. The checkpointer could omit to checkpoint
+    ** a version of page X that lies before pWal->minFrame (call that version
+    ** A) on the basis that there is a newer version (version B) of the same
+    ** page later in the wal file. But if version B happens to like past
+    ** frame pWal->hdr.mxFrame - then the client would incorrectly assume
+    ** that it can read version A from the database file. However, since
+    ** we can guarantee that the checkpointer that set nBackfill could not
+    ** see any pages past pWal->hdr.mxFrame, this problem does not come up.
+    */
+    pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT;
+    walShmBarrier(pWal);
+    if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark
+     || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr))
+    ){
+      walUnlockShared(pWal, WAL_READ_LOCK(mxI));
+      return WAL_RETRY;
+    }else{
+      assert( mxReadMark<=pWal->hdr.mxFrame );
+      pWal->readLock = (i16)mxI;
+    }
   }
-  if( mxI==0 ){
-    assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 );
-    return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT;
-  }
+  return rc;
+}
 
-  rc = walLockShared(pWal, WAL_READ_LOCK(mxI));
-  if( rc ){
-    return rc==SQLITE_BUSY ? WAL_RETRY : rc;
-  }
-  /* Now that the read-lock has been obtained, check that neither the
-  ** value in the aReadMark[] array or the contents of the wal-index
-  ** header have changed.
-  **
-  ** It is necessary to check that the wal-index header did not change
-  ** between the time it was read and when the shared-lock was obtained
-  ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility
-  ** that the log file may have been wrapped by a writer, or that frames
-  ** that occur later in the log than pWal->hdr.mxFrame may have been
-  ** copied into the database by a checkpointer. If either of these things
-  ** happened, then reading the database with the current value of
-  ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry
-  ** instead.
-  **
-  ** Before checking that the live wal-index header has not changed
-  ** since it was read, set Wal.minFrame to the first frame in the wal
-  ** file that has not yet been checkpointed. This client will not need
-  ** to read any frames earlier than minFrame from the wal file - they
-  ** can be safely read directly from the database file.
-  **
-  ** Because a ShmBarrier() call is made between taking the copy of 
-  ** nBackfill and checking that the wal-header in shared-memory still
-  ** matches the one cached in pWal->hdr, it is guaranteed that the 
-  ** checkpointer that set nBackfill was not working with a wal-index
-  ** header newer than that cached in pWal->hdr. If it were, that could
-  ** cause a problem. The checkpointer could omit to checkpoint
-  ** a version of page X that lies before pWal->minFrame (call that version
-  ** A) on the basis that there is a newer version (version B) of the same
-  ** page later in the wal file. But if version B happens to like past
-  ** frame pWal->hdr.mxFrame - then the client would incorrectly assume
-  ** that it can read version A from the database file. However, since
-  ** we can guarantee that the checkpointer that set nBackfill could not
-  ** see any pages past pWal->hdr.mxFrame, this problem does not come up.
-  */
-  pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1;
-  walShmBarrier(pWal);
-  if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark
-   || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr))
-  ){
-    walUnlockShared(pWal, WAL_READ_LOCK(mxI));
-    return WAL_RETRY;
-  }else{
-    assert( mxReadMark<=pWal->hdr.mxFrame );
-    pWal->readLock = (i16)mxI;
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** This function does the work of sqlite3WalSnapshotRecover().
+*/
+static int walSnapshotRecover(
+  Wal *pWal,                      /* WAL handle */
+  void *pBuf1,                    /* Temp buffer pWal->szPage bytes in size */
+  void *pBuf2                     /* Temp buffer pWal->szPage bytes in size */
+){
+  int szPage = (int)pWal->szPage;
+  int rc;
+  i64 szDb;                       /* Size of db file in bytes */
+
+  rc = sqlite3OsFileSize(pWal->pDbFd, &szDb);
+  if( rc==SQLITE_OK ){
+    volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
+    u32 i = pInfo->nBackfillAttempted;
+    for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){
+      WalHashLoc sLoc;          /* Hash table location */
+      u32 pgno;                 /* Page number in db file */
+      i64 iDbOff;               /* Offset of db file entry */
+      i64 iWalOff;              /* Offset of wal file entry */
+
+      rc = walHashGet(pWal, walFramePage(i), &sLoc);
+      if( rc!=SQLITE_OK ) break;
+      assert( i - sLoc.iZero - 1 >=0 );
+      pgno = sLoc.aPgno[i-sLoc.iZero-1];
+      iDbOff = (i64)(pgno-1) * szPage;
+
+      if( iDbOff+szPage<=szDb ){
+        iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE;
+        rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff);
+
+        if( rc==SQLITE_OK ){
+          rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff);
+        }
+
+        if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){
+          break;
+        }
+      }
+
+      pInfo->nBackfillAttempted = i-1;
+    }
   }
+
   return rc;
 }
 
-#ifdef SQLITE_ENABLE_SNAPSHOT
 /*
-** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted 
+** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted
 ** variable so that older snapshots can be accessed. To do this, loop
-** through all wal frames from nBackfillAttempted to (nBackfill+1), 
+** through all wal frames from nBackfillAttempted to (nBackfill+1),
 ** comparing their content to the corresponding page with the database
 ** file, if any. Set nBackfillAttempted to the frame number of the
 ** first frame for which the wal file content matches the db file.
 **
-** This is only really safe if the file-system is such that any page 
-** writes made by earlier checkpointers were atomic operations, which 
+** This is only really safe if the file-system is such that any page
+** writes made by earlier checkpointers were atomic operations, which
 ** is not always true. It is also possible that nBackfillAttempted
 ** may be left set to a value larger than expected, if a wal frame
 ** contains content that duplicate of an earlier version of the same
@@ -2964,50 +3325,21 @@ int sqlite3WalSnapshotRecover(Wal *pWal){
   assert( pWal->readLock>=0 );
   rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
   if( rc==SQLITE_OK ){
-    volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
-    int szPage = (int)pWal->szPage;
-    i64 szDb;                   /* Size of db file in bytes */
-
-    rc = sqlite3OsFileSize(pWal->pDbFd, &szDb);
-    if( rc==SQLITE_OK ){
-      void *pBuf1 = sqlite3_malloc(szPage);
-      void *pBuf2 = sqlite3_malloc(szPage);
-      if( pBuf1==0 || pBuf2==0 ){
-        rc = SQLITE_NOMEM;
-      }else{
-        u32 i = pInfo->nBackfillAttempted;
-        for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){
-          WalHashLoc sLoc;          /* Hash table location */
-          u32 pgno;                 /* Page number in db file */
-          i64 iDbOff;               /* Offset of db file entry */
-          i64 iWalOff;              /* Offset of wal file entry */
-
-          rc = walHashGet(pWal, walFramePage(i), &sLoc);
-          if( rc!=SQLITE_OK ) break;
-          assert( i - sLoc.iZero - 1 >=0 );
-          pgno = sLoc.aPgno[i-sLoc.iZero-1];
-          iDbOff = (i64)(pgno-1) * szPage;
-
-          if( iDbOff+szPage<=szDb ){
-            iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE;
-            rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff);
-
-            if( rc==SQLITE_OK ){
-              rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff);
-            }
-
-            if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){
-              break;
-            }
-          }
-
-          pInfo->nBackfillAttempted = i-1;
-        }
+    void *pBuf1 = sqlite3_malloc(pWal->szPage);
+    void *pBuf2 = sqlite3_malloc(pWal->szPage);
+    if( pBuf1==0 || pBuf2==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      pWal->ckptLock = 1;
+      SEH_TRY {
+        rc = walSnapshotRecover(pWal, pBuf1, pBuf2);
       }
-
-      sqlite3_free(pBuf1);
-      sqlite3_free(pBuf2);
+      SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+      pWal->ckptLock = 0;
     }
+
+    sqlite3_free(pBuf1);
+    sqlite3_free(pBuf2);
     walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
   }
 
@@ -3016,28 +3348,20 @@ int sqlite3WalSnapshotRecover(Wal *pWal){
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
 /*
-** Begin a read transaction on the database.
-**
-** This routine used to be called sqlite3OpenSnapshot() and with good reason:
-** it takes a snapshot of the state of the WAL and wal-index for the current
-** instant in time.  The current thread will continue to use this snapshot.
-** Other threads might append new content to the WAL and wal-index but
-** that extra content is ignored by the current thread.
-**
-** If the database contents have changes since the previous read
-** transaction, then *pChanged is set to 1 before returning.  The
-** Pager layer will use this to know that its cache is stale and
-** needs to be flushed.
+** This function does the work of sqlite3WalBeginReadTransaction() (see 
+** below). That function simply calls this one inside an SEH_TRY{...} block.
 */
-int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
+static int walBeginReadTransaction(Wal *pWal, int *pChanged){
   int rc;                         /* Return code */
   int cnt = 0;                    /* Number of TryBeginRead attempts */
 #ifdef SQLITE_ENABLE_SNAPSHOT
+  int ckptLock = 0;
   int bChanged = 0;
   WalIndexHdr *pSnapshot = pWal->pSnapshot;
 #endif
 
   assert( pWal->ckptLock==0 );
+  assert( pWal->nSehTry>0 );
 
 #ifdef SQLITE_ENABLE_SNAPSHOT
   if( pSnapshot ){
@@ -3045,13 +3369,13 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
       bChanged = 1;
     }
 
-    /* It is possible that there is a checkpointer thread running 
+    /* It is possible that there is a checkpointer thread running
     ** concurrent with this code. If this is the case, it may be that the
-    ** checkpointer has already determined that it will checkpoint 
-    ** snapshot X, where X is later in the wal file than pSnapshot, but 
-    ** has not yet set the pInfo->nBackfillAttempted variable to indicate 
+    ** checkpointer has already determined that it will checkpoint
+    ** snapshot X, where X is later in the wal file than pSnapshot, but
+    ** has not yet set the pInfo->nBackfillAttempted variable to indicate
     ** its intent. To avoid the race condition this leads to, ensure that
-    ** there is no checkpointer process by taking a shared CKPT lock 
+    ** there is no checkpointer process by taking a shared CKPT lock
     ** before checking pInfo->nBackfillAttempted.  */
     (void)walEnableBlocking(pWal);
     rc = walLockShared(pWal, WAL_CKPT_LOCK);
@@ -3060,12 +3384,12 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
     if( rc!=SQLITE_OK ){
       return rc;
     }
-    pWal->ckptLock = 1;
+    ckptLock = 1;
   }
 #endif
 
   do{
-    rc = walTryBeginRead(pWal, pChanged, 0, ++cnt);
+    rc = walTryBeginRead(pWal, pChanged, 0, &cnt);
   }while( rc==WAL_RETRY );
   testcase( (rc&0xff)==SQLITE_BUSY );
   testcase( (rc&0xff)==SQLITE_IOERR );
@@ -3112,7 +3436,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
       /* A client using a non-current snapshot may not ignore any frames
       ** from the start of the wal file. This is because, for a system
       ** where (minFrame < iSnapshot < maxFrame), a checkpointer may
-      ** have omitted to checkpoint a frame earlier than minFrame in 
+      ** have omitted to checkpoint a frame earlier than minFrame in
       ** the file because there exists a frame after iSnapshot that
       ** is the same database page.  */
       pWal->minFrame = 1;
@@ -3124,22 +3448,47 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
   }
 
   /* Release the shared CKPT lock obtained above. */
-  if( pWal->ckptLock ){
+  if( ckptLock ){
     assert( pSnapshot );
     walUnlockShared(pWal, WAL_CKPT_LOCK);
-    pWal->ckptLock = 0;
   }
 #endif
   return rc;
 }
 
+/*
+** Begin a read transaction on the database.
+**
+** This routine used to be called sqlite3OpenSnapshot() and with good reason:
+** it takes a snapshot of the state of the WAL and wal-index for the current
+** instant in time.  The current thread will continue to use this snapshot.
+** Other threads might append new content to the WAL and wal-index but
+** that extra content is ignored by the current thread.
+**
+** If the database contents have changes since the previous read
+** transaction, then *pChanged is set to 1 before returning.  The
+** Pager layer will use this to know that its cache is stale and
+** needs to be flushed.
+*/
+int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
+  int rc;
+  SEH_TRY {
+    rc = walBeginReadTransaction(pWal, pChanged);
+  }
+  SEH_EXCEPT( rc = walHandleException(pWal); )
+  return rc;
+}
+
 /*
 ** Finish with a read transaction.  All this does is release the
 ** read-lock.
 */
 void sqlite3WalEndReadTransaction(Wal *pWal){
-  sqlite3WalEndWriteTransaction(pWal);
+#ifndef SQLITE_ENABLE_SETLK_TIMEOUT
+  assert( pWal->writeLock==0 || pWal->readLock<0 );
+#endif
   if( pWal->readLock>=0 ){
+    (void)sqlite3WalEndWriteTransaction(pWal);
     walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock));
     pWal->readLock = -1;
   }
@@ -3153,7 +3502,7 @@ void sqlite3WalEndReadTransaction(Wal *pWal){
 ** Return SQLITE_OK if successful, or an error code if an error occurs. If an
 ** error does occur, the final value of *piRead is undefined.
 */
-int sqlite3WalFindFrame(
+static int walFindFrame(
   Wal *pWal,                      /* WAL handle */
   Pgno pgno,                      /* Database page number to read data for */
   u32 *piRead                     /* OUT: Frame number (or zero) */
@@ -3168,8 +3517,8 @@ int sqlite3WalFindFrame(
 
   /* If the "last page" field of the wal-index header snapshot is 0, then
   ** no data will be read from the wal under any circumstances. Return early
-  ** in this case as an optimization.  Likewise, if pWal->readLock==0, 
-  ** then the WAL is ignored by the reader so return early, as if the 
+  ** in this case as an optimization.  Likewise, if pWal->readLock==0,
+  ** then the WAL is ignored by the reader so return early, as if the
   ** WAL were empty.
   */
   if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){
@@ -3182,9 +3531,9 @@ int sqlite3WalFindFrame(
   ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames).
   **
   ** This code might run concurrently to the code in walIndexAppend()
-  ** that adds entries to the wal-index (and possibly to this hash 
-  ** table). This means the value just read from the hash 
-  ** slot (aHash[iKey]) may have been added before or after the 
+  ** that adds entries to the wal-index (and possibly to this hash
+  ** table). This means the value just read from the hash
+  ** slot (aHash[iKey]) may have been added before or after the
   ** current read transaction was opened. Values added after the
   ** read transaction was opened may have been written incorrectly -
   ** i.e. these slots may contain garbage data. However, we assume
@@ -3192,13 +3541,13 @@ int sqlite3WalFindFrame(
   ** opened remain unmodified.
   **
   ** For the reasons above, the if(...) condition featured in the inner
-  ** loop of the following block is more stringent that would be required 
+  ** loop of the following block is more stringent that would be required
   ** if we had exclusive access to the hash-table:
   **
-  **   (aPgno[iFrame]==pgno): 
+  **   (aPgno[iFrame]==pgno):
   **     This condition filters out normal hash-table collisions.
   **
-  **   (iFrame<=iLast): 
+  **   (iFrame<=iLast):
   **     This condition filters out entries that were added to the hash
   **     table after the current read-transaction had started.
   */
@@ -3216,6 +3565,7 @@ int sqlite3WalFindFrame(
     }
     nCollide = HASHTABLE_NSLOT;
     iKey = walHash(pgno);
+    SEH_INJECT_FAULT;
     while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){
       u32 iFrame = iH + sLoc.iZero;
       if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){
@@ -3223,6 +3573,7 @@ int sqlite3WalFindFrame(
         iRead = iFrame;
       }
       if( (nCollide--)==0 ){
+        *piRead = 0;
         return SQLITE_CORRUPT_BKPT;
       }
       iKey = walNextHash(iKey);
@@ -3252,6 +3603,30 @@ int sqlite3WalFindFrame(
   return SQLITE_OK;
 }
 
+/*
+** Search the wal file for page pgno. If found, set *piRead to the frame that
+** contains the page. Otherwise, if pgno is not in the wal file, set *piRead
+** to zero.
+**
+** Return SQLITE_OK if successful, or an error code if an error occurs. If an
+** error does occur, the final value of *piRead is undefined.
+**
+** The difference between this function and walFindFrame() is that this
+** function wraps walFindFrame() in an SEH_TRY{...} block.
+*/
+int sqlite3WalFindFrame(
+  Wal *pWal,                      /* WAL handle */
+  Pgno pgno,                      /* Database page number to read data for */
+  u32 *piRead                     /* OUT: Frame number (or zero) */
+){
+  int rc;
+  SEH_TRY {
+    rc = walFindFrame(pWal, pgno, piRead);
+  }
+  SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+  return rc;
+}
+
 /*
 ** Read the contents of frame iRead from the wal file into buffer pOut
 ** (which is nOut bytes in size). Return SQLITE_OK if successful, or an
@@ -3274,7 +3649,7 @@ int sqlite3WalReadFrame(
   return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset);
 }
 
-/* 
+/*
 ** Return the size of the database in pages (or zero, if unknown).
 */
 Pgno sqlite3WalDbsize(Wal *pWal){
@@ -3285,7 +3660,7 @@ Pgno sqlite3WalDbsize(Wal *pWal){
 }
 
 
-/* 
+/*
 ** This function starts a write transaction on the WAL.
 **
 ** A read transaction must have already been started by a prior call
@@ -3306,7 +3681,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){
   ** read-transaction was even opened, making this call a no-op.
   ** Return early. */
   if( pWal->writeLock ){
-    assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) );
+    assert( !memcmp(&pWal->hdr,(void*)pWal->apWiData[0],sizeof(WalIndexHdr)) );
     return SQLITE_OK;
   }
 #endif
@@ -3333,12 +3708,17 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){
   ** time the read transaction on this connection was started, then
   ** the write is disallowed.
   */
-  if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
+  SEH_TRY {
+    if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){
+      rc = SQLITE_BUSY_SNAPSHOT;
+    }
+  }
+  SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+
+  if( rc!=SQLITE_OK ){
     walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1);
     pWal->writeLock = 0;
-    rc = SQLITE_BUSY_SNAPSHOT;
   }
-
   return rc;
 }
 
@@ -3373,39 +3753,43 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){
   if( ALWAYS(pWal->writeLock) ){
     Pgno iMax = pWal->hdr.mxFrame;
     Pgno iFrame;
-  
-    /* Restore the clients cache of the wal-index header to the state it
-    ** was in before the client began writing to the database. 
-    */
-    memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));
 
-    for(iFrame=pWal->hdr.mxFrame+1; 
-        ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; 
-        iFrame++
-    ){
-      /* This call cannot fail. Unless the page for which the page number
-      ** is passed as the second argument is (a) in the cache and 
-      ** (b) has an outstanding reference, then xUndo is either a no-op
-      ** (if (a) is false) or simply expels the page from the cache (if (b)
-      ** is false).
-      **
-      ** If the upper layer is doing a rollback, it is guaranteed that there
-      ** are no outstanding references to any page other than page 1. And
-      ** page 1 is never written to the log until the transaction is
-      ** committed. As a result, the call to xUndo may not fail.
+    SEH_TRY {
+      /* Restore the clients cache of the wal-index header to the state it
+      ** was in before the client began writing to the database.
       */
-      assert( walFramePgno(pWal, iFrame)!=1 );
-      rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame));
+      memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr));
+
+      for(iFrame=pWal->hdr.mxFrame+1;
+          ALWAYS(rc==SQLITE_OK) && iFrame<=iMax;
+          iFrame++
+      ){
+        /* This call cannot fail. Unless the page for which the page number
+        ** is passed as the second argument is (a) in the cache and
+        ** (b) has an outstanding reference, then xUndo is either a no-op
+        ** (if (a) is false) or simply expels the page from the cache (if (b)
+        ** is false).
+        **
+        ** If the upper layer is doing a rollback, it is guaranteed that there
+        ** are no outstanding references to any page other than page 1. And
+        ** page 1 is never written to the log until the transaction is
+        ** committed. As a result, the call to xUndo may not fail.
+        */
+        assert( walFramePgno(pWal, iFrame)!=1 );
+        rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame));
+      }
+      if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal);
     }
-    if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal);
+    SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+    pWal->iReCksum = 0;
   }
   return rc;
 }
 
-/* 
-** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 
-** values. This function populates the array with values required to 
-** "rollback" the write position of the WAL handle back to the current 
+/*
+** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32
+** values. This function populates the array with values required to
+** "rollback" the write position of the WAL handle back to the current
 ** point in the event of a savepoint rollback (via WalSavepointUndo()).
 */
 void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
@@ -3416,7 +3800,7 @@ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){
   aWalData[3] = pWal->nCkpt;
 }
 
-/* 
+/*
 ** Move the write position of the WAL back to the point identified by
 ** the values in the aWalData[] array. aWalData must point to an array
 ** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated
@@ -3441,7 +3825,13 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){
     pWal->hdr.mxFrame = aWalData[0];
     pWal->hdr.aFrameCksum[0] = aWalData[1];
     pWal->hdr.aFrameCksum[1] = aWalData[2];
-    walCleanupHash(pWal);
+    SEH_TRY {
+      walCleanupHash(pWal);
+    }
+    SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; )
+    if( pWal->iReCksum>pWal->hdr.mxFrame ){
+      pWal->iReCksum = 0;
+    }
   }
 
   return rc;
@@ -3491,7 +3881,7 @@ static int walRestartLog(Wal *pWal){
     cnt = 0;
     do{
       int notUsed;
-      rc = walTryBeginRead(pWal, &notUsed, 1, ++cnt);
+      rc = walTryBeginRead(pWal, &notUsed, 1, &cnt);
     }while( rc==WAL_RETRY );
     assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */
     testcase( (rc&0xff)==SQLITE_IOERR );
@@ -3622,11 +4012,11 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){
   return rc;
 }
 
-/* 
+/*
 ** Write a set of frames to the log. The caller must hold the write-lock
 ** on the log file (obtained using sqlite3WalBeginWriteTransaction()).
 */
-int sqlite3WalFrames(
+static int walFrames(
   Wal *pWal,                      /* Wal handle to write to */
   int szPage,                     /* Database page-size in bytes */
   PgHdr *pList,                   /* List of dirty pages to write */
@@ -3689,7 +4079,7 @@ int sqlite3WalFrames(
     walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum);
     sqlite3Put4byte(&aWalHdr[24], aCksum[0]);
     sqlite3Put4byte(&aWalHdr[28], aCksum[1]);
-    
+
     pWal->szPage = szPage;
     pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN;
     pWal->hdr.aFrameCksum[0] = aCksum[0];
@@ -3733,11 +4123,11 @@ int sqlite3WalFrames(
 
     /* Check if this page has already been written into the wal file by
     ** the current transaction. If so, overwrite the existing frame and
-    ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that 
+    ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that
     ** checksums must be recomputed when the transaction is committed.  */
     if( iFirst && (p->pDirty || isCommit==0) ){
       u32 iWrite = 0;
-      VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite);
+      VVA_ONLY(rc =) walFindFrame(pWal, p->pgno, &iWrite);
       assert( rc==SQLITE_OK || iWrite==0 );
       if( iWrite>=iFirst ){
         i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE;
@@ -3821,7 +4211,7 @@ int sqlite3WalFrames(
     pWal->truncateOnCommit = 0;
   }
 
-  /* Append data to the wal-index. It is not necessary to lock the 
+  /* Append data to the wal-index. It is not necessary to lock the
   ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index
   ** guarantees that there are no other writers, and no data that may
   ** be in use by existing readers is being overwritten.
@@ -3861,6 +4251,29 @@ int sqlite3WalFrames(
 }
 
 /* 
+** Write a set of frames to the log. The caller must hold the write-lock
+** on the log file (obtained using sqlite3WalBeginWriteTransaction()).
+**
+** The difference between this function and walFrames() is that this
+** function wraps walFrames() in an SEH_TRY{...} block.
+*/
+int sqlite3WalFrames(
+  Wal *pWal,                      /* Wal handle to write to */
+  int szPage,                     /* Database page-size in bytes */
+  PgHdr *pList,                   /* List of dirty pages to write */
+  Pgno nTruncate,                 /* Database size after this commit */
+  int isCommit,                   /* True if this is a commit */
+  int sync_flags                  /* Flags to pass to OsSync() (or 0) */
+){
+  int rc;
+  SEH_TRY {
+    rc = walFrames(pWal, szPage, pList, nTruncate, isCommit, sync_flags);
+  }
+  SEH_EXCEPT( rc = walHandleException(pWal); )
+  return rc;
+}
+
+/*
 ** This routine is called to implement sqlite3_wal_checkpoint() and
 ** related interfaces.
 **
@@ -3892,17 +4305,17 @@ int sqlite3WalCheckpoint(
 
   /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked
   ** in the SQLITE_CHECKPOINT_PASSIVE mode. */
-  assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
+  assert( SQLITE_CHECKPOINT_NOOP<SQLITE_CHECKPOINT_PASSIVE );
+  assert( eMode>SQLITE_CHECKPOINT_PASSIVE || xBusy==0 );
 
   if( pWal->readOnly ) return SQLITE_READONLY;
   WALTRACE(("WAL%p: checkpoint begins\n", pWal));
 
-  /* Enable blocking locks, if possible. If blocking locks are successfully
-  ** enabled, set xBusy2=0 so that the busy-handler is never invoked. */
+  /* Enable blocking locks, if possible. */
   sqlite3WalDb(pWal, db);
-  (void)walEnableBlocking(pWal);
+  if( xBusy2 ) (void)walEnableBlocking(pWal);
 
-  /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive 
+  /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive
   ** "checkpoint" lock on the database file.
   ** EVIDENCE-OF: R-10421-19736 If any other process is running a
   ** checkpoint operation at the same time, the lock cannot be obtained and
@@ -3910,62 +4323,74 @@ int sqlite3WalCheckpoint(
   ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured,
   ** it will not be invoked in this case.
   */
-  rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
-  testcase( rc==SQLITE_BUSY );
-  testcase( rc!=SQLITE_OK && xBusy2!=0 );
-  if( rc==SQLITE_OK ){
-    pWal->ckptLock = 1;
+  if( eMode!=SQLITE_CHECKPOINT_NOOP ){
+    rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
+    testcase( rc==SQLITE_BUSY );
+    testcase( rc!=SQLITE_OK && xBusy2!=0 );
+    if( rc==SQLITE_OK ){
+      pWal->ckptLock = 1;
 
-    /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and
-    ** TRUNCATE modes also obtain the exclusive "writer" lock on the database
-    ** file.
-    **
-    ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained
-    ** immediately, and a busy-handler is configured, it is invoked and the
-    ** writer lock retried until either the busy-handler returns 0 or the
-    ** lock is successfully obtained.
-    */
-    if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){
-      rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1);
-      if( rc==SQLITE_OK ){
-        pWal->writeLock = 1;
-      }else if( rc==SQLITE_BUSY ){
-        eMode2 = SQLITE_CHECKPOINT_PASSIVE;
-        xBusy2 = 0;
-        rc = SQLITE_OK;
+      /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART 
+      ** and TRUNCATE modes also obtain the exclusive "writer" lock on the 
+      ** database file.
+      **
+      ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained
+      ** immediately, and a busy-handler is configured, it is invoked and the
+      ** writer lock retried until either the busy-handler returns 0 or the
+      ** lock is successfully obtained.
+      */
+      if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){
+        rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1);
+        if( rc==SQLITE_OK ){
+          pWal->writeLock = 1;
+        }else if( rc==SQLITE_BUSY ){
+          eMode2 = SQLITE_CHECKPOINT_PASSIVE;
+          xBusy2 = 0;
+          rc = SQLITE_OK;
+        }
       }
     }
+  }else{
+    rc = SQLITE_OK;
   }
 
 
   /* Read the wal-index header. */
-  if( rc==SQLITE_OK ){
-    walDisableBlocking(pWal);
-    rc = walIndexReadHdr(pWal, &isChanged);
-    (void)walEnableBlocking(pWal);
-    if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){
-      sqlite3OsUnfetch(pWal->pDbFd, 0, 0);
-    }
-  }
-
-  /* Copy data from the log to the database file. */
-  if( rc==SQLITE_OK ){
-
-    if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
-      rc = SQLITE_CORRUPT_BKPT;
-    }else{
-      rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf);
+  SEH_TRY {
+    if( rc==SQLITE_OK ){
+      /* For a passive checkpoint, do not re-enable blocking locks after
+      ** reading the wal-index header. A passive checkpoint should not block 
+      ** or invoke the busy handler. The only lock such a checkpoint may 
+      ** attempt to obtain is a lock on a read-slot, and it should give up
+      ** immediately and do a partial checkpoint if it cannot obtain it. */
+      walDisableBlocking(pWal);
+      rc = walIndexReadHdr(pWal, &isChanged);
+      if( eMode2>SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal);
+      if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){
+        sqlite3OsUnfetch(pWal->pDbFd, 0, 0);
+      }
     }
+  
+    /* Copy data from the log to the database file. */
+    if( rc==SQLITE_OK ){
+      if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){
+        rc = SQLITE_CORRUPT_BKPT;
+      }else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){
+        rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf);
+      }
 
-    /* If no error occurred, set the output variables. */
-    if( rc==SQLITE_OK || rc==SQLITE_BUSY ){
-      if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame;
-      if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill);
+      /* If no error occurred, set the output variables. */
+      if( rc==SQLITE_OK || rc==SQLITE_BUSY ){
+        if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame;
+        SEH_INJECT_FAULT;
+        if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill);
+      }
     }
   }
+  SEH_EXCEPT( rc = walHandleException(pWal); )
 
   if( isChanged ){
-    /* If a new wal-index header was loaded before the checkpoint was 
+    /* If a new wal-index header was loaded before the checkpoint was
     ** performed, then the pager-cache associated with pWal is now
     ** out of date. So zero the cached wal-index header to ensure that
     ** next time the pager opens a snapshot on this database it knows that
@@ -3978,7 +4403,7 @@ int sqlite3WalCheckpoint(
   sqlite3WalDb(pWal, 0);
 
   /* Release the locks. */
-  sqlite3WalEndWriteTransaction(pWal);
+  (void)sqlite3WalEndWriteTransaction(pWal);
   if( pWal->ckptLock ){
     walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
     pWal->ckptLock = 0;
@@ -4016,7 +4441,7 @@ int sqlite3WalCallback(Wal *pWal){
 ** operation must occur while the pager is still holding the exclusive
 ** lock on the main database file.
 **
-** If op is one, then change from locking_mode=NORMAL into 
+** If op is one, then change from locking_mode=NORMAL into
 ** locking_mode=EXCLUSIVE.  This means that the pWal->readLock must
 ** be released.  Return 1 if the transition is made and 0 if the
 ** WAL is already in exclusive-locking mode - meaning that this
@@ -4033,13 +4458,15 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){
   assert( pWal->writeLock==0 );
   assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 );
 
-  /* pWal->readLock is usually set, but might be -1 if there was a 
-  ** prior error while attempting to acquire are read-lock. This cannot 
+  /* pWal->readLock is usually set, but might be -1 if there was a
+  ** prior error while attempting to acquire are read-lock. This cannot
   ** happen if the connection is actually in exclusive mode (as no xShmLock
   ** locks are taken in this case). Nor should the pager attempt to
   ** upgrade to exclusive-mode following such an error.
   */
+#ifndef SQLITE_USE_SEH
   assert( pWal->readLock>=0 || pWal->lockError );
+#endif
   assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) );
 
   if( op==0 ){
@@ -4065,10 +4492,10 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){
   return rc;
 }
 
-/* 
+/*
 ** Return true if the argument is non-NULL and the WAL module is using
 ** heap-memory for the wal-index. Otherwise, if the argument is NULL or the
-** WAL module is using shared-memory, return false. 
+** WAL module is using shared-memory, return false.
 */
 int sqlite3WalHeapMemory(Wal *pWal){
   return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE );
@@ -4104,13 +4531,26 @@ int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){
 /* Try to open on pSnapshot when the next read-transaction starts
 */
 void sqlite3WalSnapshotOpen(
-  Wal *pWal, 
+  Wal *pWal,
   sqlite3_snapshot *pSnapshot
 ){
-  pWal->pSnapshot = (WalIndexHdr*)pSnapshot;
+  if( pSnapshot && ((WalIndexHdr*)pSnapshot)->iVersion==0 ){
+    /* iVersion==0 means that this is a call to sqlite3_snapshot_get().  In
+    ** this case set the bGetSnapshot flag so that if the call to
+    ** sqlite3_snapshot_get() is about to read transaction on this wal 
+    ** file, it does not take read-lock 0 if the wal file has been completely
+    ** checkpointed. Taking read-lock 0 would work, but then it would be
+    ** possible for a subsequent writer to destroy the snapshot even while 
+    ** this connection is holding its read-transaction open. This is contrary
+    ** to user expectations, so we avoid it by not taking read-lock 0. */
+    pWal->bGetSnapshot = 1;
+  }else{
+    pWal->pSnapshot = (WalIndexHdr*)pSnapshot;
+    pWal->bGetSnapshot = 0;
+  }
 }
 
-/* 
+/*
 ** Return a +ve value if snapshot p1 is newer than p2. A -ve value if
 ** p1 is older than p2 and zero if p1 and p2 are the same snapshot.
 */
@@ -4130,7 +4570,7 @@ int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){
 /*
 ** The caller currently has a read transaction open on the database.
 ** This function takes a SHARED lock on the CHECKPOINTER slot and then
-** checks if the snapshot passed as the second argument is still 
+** checks if the snapshot passed as the second argument is still
 ** available. If so, SQLITE_OK is returned.
 **
 ** If the snapshot is not available, SQLITE_ERROR is returned. Or, if
@@ -4140,16 +4580,19 @@ int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){
 */
 int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){
   int rc;
-  rc = walLockShared(pWal, WAL_CKPT_LOCK);
-  if( rc==SQLITE_OK ){
-    WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot;
-    if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt))
-     || pNew->mxFrame<walCkptInfo(pWal)->nBackfillAttempted
-    ){
-      rc = SQLITE_ERROR_SNAPSHOT;
-      walUnlockShared(pWal, WAL_CKPT_LOCK);
+  SEH_TRY {
+    rc = walLockShared(pWal, WAL_CKPT_LOCK);
+    if( rc==SQLITE_OK ){
+      WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot;
+      if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt))
+       || pNew->mxFrame<walCkptInfo(pWal)->nBackfillAttempted
+      ){
+        rc = SQLITE_ERROR_SNAPSHOT;
+        walUnlockShared(pWal, WAL_CKPT_LOCK);
+      }
     }
   }
+  SEH_EXCEPT( rc = walHandleException(pWal); )
   return rc;
 }
 
diff --git a/src/wal.h b/src/wal.h
index 02e2bab360..1b17d2dfbe 100644
--- a/src/wal.h
+++ b/src/wal.h
@@ -45,6 +45,7 @@
 # define sqlite3WalFramesize(z)                  0
 # define sqlite3WalFindFrame(x,y,z)              0
 # define sqlite3WalFile(x)                       0
+# undef SQLITE_USE_SEH
 #else
 
 #define WAL_SAVEPOINT_NDATA 4
@@ -151,5 +152,9 @@ int sqlite3WalWriteLock(Wal *pWal, int bLock);
 void sqlite3WalDb(Wal *pWal, sqlite3 *db);
 #endif
 
+#ifdef SQLITE_USE_SEH
+int sqlite3WalSystemErrno(Wal*);
+#endif
+
 #endif /* ifndef SQLITE_OMIT_WAL */
 #endif /* SQLITE_WAL_H */
diff --git a/src/walker.c b/src/walker.c
index f24c052d6d..c8735e39b8 100644
--- a/src/walker.c
+++ b/src/walker.c
@@ -61,7 +61,7 @@ static int walkWindowList(Walker *pWalker, Window *pList, int bOneOnly){
 ** The return value from this routine is WRC_Abort to abandon the tree walk
 ** and WRC_Continue to continue.
 */
-static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
+SQLITE_NOINLINE int sqlite3WalkExprNN(Walker *pWalker, Expr *pExpr){
   int rc;
   testcase( ExprHasProperty(pExpr, EP_TokenOnly) );
   testcase( ExprHasProperty(pExpr, EP_Reduced) );
@@ -70,7 +70,9 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
     if( rc ) return rc & WRC_Abort;
     if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){
       assert( pExpr->x.pList==0 || pExpr->pRight==0 );
-      if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
+      if( pExpr->pLeft && sqlite3WalkExprNN(pWalker, pExpr->pLeft) ){
+        return WRC_Abort;
+      }
       if( pExpr->pRight ){
         assert( !ExprHasProperty(pExpr, EP_WinFunc) );
         pExpr = pExpr->pRight;
@@ -94,7 +96,7 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
   return WRC_Continue;
 }
 int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){
-  return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue;
+  return pExpr ? sqlite3WalkExprNN(pWalker,pExpr) : WRC_Continue;
 }
 
 /*
@@ -169,7 +171,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){
   pSrc = p->pSrc;
   if( ALWAYS(pSrc) ){
     for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){
-      if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){
+      if( pItem->fg.isSubquery
+       && sqlite3WalkSelect(pWalker, pItem->u4.pSubq->pSelect)
+      ){
         return WRC_Abort;
       }
       if( pItem->fg.isTabFunc
@@ -220,7 +224,7 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){
 }
 
 /* Increase the walkerDepth when entering a subquery, and
-** descrease when leaving the subquery.
+** decrease when leaving the subquery.
 */
 int sqlite3WalkerDepthIncrease(Walker *pWalker, Select *pSelect){
   UNUSED_PARAMETER(pSelect);
diff --git a/src/where.c b/src/where.c
index 25405cedf4..c4f2c55435 100644
--- a/src/where.c
+++ b/src/where.c
@@ -35,11 +35,16 @@ struct HiddenIndexInfo {
   int eDistinct;           /* Value to return from sqlite3_vtab_distinct() */
   u32 mIn;                 /* Mask of terms that are <col> IN (...) */
   u32 mHandleIn;           /* Terms that vtab will handle as <col> IN (...) */
-  sqlite3_value *aRhs[1];  /* RHS values for constraints. MUST BE LAST
-                           ** because extra space is allocated to hold up
-                           ** to nTerm such values */
+  sqlite3_value *aRhs[FLEXARRAY];  /* RHS values for constraints. MUST BE LAST
+                                   ** Extra space is allocated to hold up
+                                   ** to nTerm such values */
 };
 
+/* Size (in bytes) of a HiddenIndeInfo object sufficient to hold as
+** many as N constraints */
+#define SZ_HIDDENINDEXINFO(N) \
+                  (offsetof(HiddenIndexInfo,aRhs) + (N)*sizeof(sqlite3_value*))
+
 /* Forward declaration of methods */
 static int whereLoopResize(sqlite3*, WhereLoop*, int);
 
@@ -88,7 +93,7 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
 ** be the continuation for the inner-most loop.
 **
 ** It is always safe for this routine to return the continuation of the
-** inner-most loop, in the sense that a correct answer will result.  
+** inner-most loop, in the sense that a correct answer will result. 
 ** Returning the continuation the second inner loop is an optimization
 ** that might make the code run a little faster, but should not change
 ** the final answer.
@@ -96,7 +101,7 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
 int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){
   WhereLevel *pInner;
   if( !pWInfo->bOrderedInnerLoop ){
-    /* The ORDER BY LIMIT optimization does not apply.  Jump to the 
+    /* The ORDER BY LIMIT optimization does not apply.  Jump to the
     ** continuation of the inner-most loop. */
     return pWInfo->iContinue;
   }
@@ -153,7 +158,7 @@ int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
 ** operate directly on the rowids returned by a WHERE clause.  Return
 ** ONEPASS_SINGLE (1) if the statement can operation directly because only
 ** a single row is to be changed.  Return ONEPASS_MULTI (2) if the one-pass
-** optimization can be used on multiple 
+** optimization can be used on multiple
 **
 ** If the ONEPASS optimization is used (if this routine returns true)
 ** then also write the indices of open cursors used by ONEPASS
@@ -302,6 +307,42 @@ static Expr *whereRightSubexprIsColumn(Expr *p){
   return 0;
 }
 
+/*
+** Term pTerm is guaranteed to be a WO_IN term. It may be a component term
+** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)".
+** This function checks to see if the term is compatible with an index
+** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so,
+** it returns a pointer to the name of the collation sequence (e.g. "BINARY"
+** or "NOCASE") used by the comparison in pTerm. If it is not compatible
+** with affinity idxaff, NULL is returned.
+*/
+static SQLITE_NOINLINE const char *indexInAffinityOk(
+  Parse *pParse, 
+  WhereTerm *pTerm, 
+  u8 idxaff
+){
+  Expr *pX = pTerm->pExpr;
+  Expr inexpr;
+
+  assert( pTerm->eOperator & WO_IN );
+
+  if( sqlite3ExprIsVector(pX->pLeft) ){
+    int iField = pTerm->u.x.iField - 1;
+    inexpr.flags = 0;
+    inexpr.op = TK_EQ;
+    inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr;
+    assert( ExprUseXSelect(pX) );
+    inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr;
+    pX = &inexpr;
+  }
+
+  if( sqlite3IndexAffinityOk(pX, idxaff) ){
+    CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX);
+    return pRet ? pRet->zName : sqlite3StrBINARY;
+  }
+  return 0;
+}
+
 /*
 ** Advance to the next WhereTerm that matches according to the criteria
 ** established when the pScan object was initialized by whereScanInit().
@@ -352,16 +393,24 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
           if( (pTerm->eOperator & pScan->opMask)!=0 ){
             /* Verify the affinity and collating sequence match */
             if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){
-              CollSeq *pColl;
+              const char *zCollName;
               Parse *pParse = pWC->pWInfo->pParse;
               pX = pTerm->pExpr;
-              if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
-                continue;
+
+              if( (pTerm->eOperator & WO_IN) ){
+                zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff);
+                if( !zCollName ) continue;
+              }else{
+                CollSeq *pColl;
+                if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){
+                  continue;
+                }
+                assert(pX->pLeft);
+                pColl = sqlite3ExprCompareCollSeq(pParse, pX);
+                zCollName = pColl ? pColl->zName : sqlite3StrBINARY;
               }
-              assert(pX->pLeft);
-              pColl = sqlite3ExprCompareCollSeq(pParse, pX);
-              if( pColl==0 ) pColl = pParse->db->pDfltColl;
-              if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){
+
+              if( sqlite3StrICmp(zCollName, pScan->zCollName) ){
                 continue;
               }
             }
@@ -377,11 +426,11 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
             pScan->pWC = pWC;
             pScan->k = k+1;
 #ifdef WHERETRACE_ENABLED
-            if( sqlite3WhereTrace & 0x20000 ){
+            if( (sqlite3WhereTrace & 0x20000)!=0 && pScan->nEquiv>1 ){
               int ii;
-              sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d",
-                 pTerm, pScan->nEquiv);
-              for(ii=0; ii<pScan->nEquiv; ii++){
+              sqlite3DebugPrintf("EQUIVALENT TO {%d:%d} (due to TERM-%d):",
+                 pScan->aiCur[0], pScan->aiColumn[0], pTerm->iTerm);
+              for(ii=1; ii<pScan->nEquiv; ii++){
                 sqlite3DebugPrintf(" {%d:%d}",
                    pScan->aiCur[ii], pScan->aiColumn[ii]);
               }
@@ -478,7 +527,7 @@ static WhereTerm *whereScanInit(
 ** if pIdx!=0 and <op> is one of the WO_xx operator codes specified by
 ** the op parameter.  Return a pointer to the term.  Return 0 if not found.
 **
-** If pIdx!=0 then it must be one of the indexes of table iCur.  
+** If pIdx!=0 then it must be one of the indexes of table iCur. 
 ** Search for terms matching the iColumn-th column of pIdx
 ** rather than the iColumn-th column of table iCur.
 **
@@ -592,17 +641,17 @@ static int isDistinctRedundant(
 ){
   Table *pTab;
   Index *pIdx;
-  int i;                          
+  int i;                         
   int iBase;
 
   /* If there is more than one table or sub-select in the FROM clause of
-  ** this query, then it will not be possible to show that the DISTINCT 
+  ** this query, then it will not be possible to show that the DISTINCT
   ** clause is redundant. */
   if( pTabList->nSrc!=1 ) return 0;
   iBase = pTabList->a[0].iCursor;
-  pTab = pTabList->a[0].pTab;
+  pTab = pTabList->a[0].pSTab;
 
-  /* If any of the expressions is an IPK column on table iBase, then return 
+  /* If any of the expressions is an IPK column on table iBase, then return
   ** true. Note: The (p->iTable==iBase) part of this test may be false if the
   ** current SELECT is a correlated sub-query.
   */
@@ -656,7 +705,7 @@ static LogEst estLog(LogEst N){
 ** Convert OP_Column opcodes to OP_Copy in previously generated code.
 **
 ** This routine runs over generated VDBE code and translates OP_Column
-** opcodes into OP_Copy when the table is being accessed via co-routine 
+** opcodes into OP_Copy when the table is being accessed via co-routine
 ** instead of via table lookup.
 **
 ** If the iAutoidxCur is not zero, then any OP_Rowid instructions on
@@ -675,15 +724,31 @@ static void translateColumnToCopy(
   VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
   int iEnd = sqlite3VdbeCurrentAddr(v);
   if( pParse->db->mallocFailed ) return;
+#ifdef SQLITE_DEBUG
+  if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
+    printf("CHECKING for column-to-copy on cursor %d for %d..%d\n",
+            iTabCur, iStart, iEnd);
+  }
+#endif
   for(; iStart<iEnd; iStart++, pOp++){
     if( pOp->p1!=iTabCur ) continue;
     if( pOp->opcode==OP_Column ){
+#ifdef SQLITE_DEBUG
+      if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
+        printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart);
+      }
+#endif
       pOp->opcode = OP_Copy;
       pOp->p1 = pOp->p2 + iRegister;
       pOp->p2 = pOp->p3;
       pOp->p3 = 0;
       pOp->p5 = 2;  /* Cause the MEM_Subtype flag to be cleared */
     }else if( pOp->opcode==OP_Rowid ){
+#ifdef SQLITE_DEBUG
+      if( pParse->db->flags & SQLITE_VdbeAddopTrace ){
+        printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart);
+      }
+#endif
       pOp->opcode = OP_Sequence;
       pOp->p1 = iAutoidxCur;
 #ifdef SQLITE_ALLOW_ROWID_IN_VIEW
@@ -703,9 +768,13 @@ static void translateColumnToCopy(
 ** are no-ops.
 */
 #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED)
-static void whereTraceIndexInfoInputs(sqlite3_index_info *p){
+static void whereTraceIndexInfoInputs(
+  sqlite3_index_info *p,   /* The IndexInfo object */
+  Table *pTab              /* The TABLE that is the virtual table */
+){
   int i;
   if( (sqlite3WhereTrace & 0x10)==0 ) return;
+  sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName);
   for(i=0; i<p->nConstraint; i++){
     sqlite3DebugPrintf(
        "  constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n",
@@ -723,9 +792,13 @@ static void whereTraceIndexInfoInputs(sqlite3_index_info *p){
        p->aOrderBy[i].desc);
   }
 }
-static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
+static void whereTraceIndexInfoOutputs(
+  sqlite3_index_info *p,   /* The IndexInfo object */
+  Table *pTab              /* The TABLE that is the virtual table */
+){
   int i;
   if( (sqlite3WhereTrace & 0x10)==0 ) return;
+  sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName);
   for(i=0; i<p->nConstraint; i++){
     sqlite3DebugPrintf("  usage[%d]: argvIdx=%d omit=%d\n",
        i,
@@ -739,8 +812,8 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){
   sqlite3DebugPrintf("  estimatedRows=%lld\n", p->estimatedRows);
 }
 #else
-#define whereTraceIndexInfoInputs(A)
-#define whereTraceIndexInfoOutputs(A)
+#define whereTraceIndexInfoInputs(A,B)
+#define whereTraceIndexInfoOutputs(A,B)
 #endif
 
 /*
@@ -771,13 +844,52 @@ static int constraintCompatibleWithOuterJoin(
     return 0;
   }
   if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0
-   && ExprHasProperty(pTerm->pExpr, EP_InnerON)
+   && NEVER(ExprHasProperty(pTerm->pExpr, EP_InnerON))
   ){
     return 0;
   }
   return 1;
 }
- 
+
+#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
+/*
+** Return true if column iCol of table pTab seem like it might be a
+** good column to use as part of a query-time index.
+**
+** Current algorithm (subject to improvement!):
+**
+**   1.   If iCol is already the left-most column of some other index,
+**        then return false.
+**
+**   2.   If iCol is part of an existing index that has an aiRowLogEst of
+**        more than 20, then return false.
+**
+**   3.   If no disqualifying conditions above are found, return true.
+**
+** 2025-01-03: I experimented with a new rule that returns false if the
+** the datatype of the column is "BOOLEAN". This did not improve
+** performance on any queries at hand, but it did burn CPU cycles, so the
+** idea was not committed.
+*/
+static SQLITE_NOINLINE int columnIsGoodIndexCandidate(
+  const Table *pTab,
+  int iCol
+){
+  const Index *pIdx;
+  for(pIdx = pTab->pIndex; pIdx!=0; pIdx=pIdx->pNext){
+    int j;
+    for(j=0; j<pIdx->nKeyCol; j++){
+       if( pIdx->aiColumn[j]==iCol ){
+         if( j==0 ) return 0;
+         if( pIdx->hasStat1 && pIdx->aiRowLogEst[j+1]>20 ) return 0;
+         break;
+       }
+    }
+  }
+  return 1;
+}
+#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
+
 
 
 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
@@ -792,6 +904,8 @@ static int termCanDriveIndex(
   const Bitmask notReady         /* Tables in outer loops of the join */
 ){
   char aff;
+  int leftCol;
+  
   if( pTerm->leftCursor!=pSrc->iCursor ) return 0;
   if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0;
   assert( (pSrc->fg.jointype & JT_RIGHT)==0 );
@@ -802,11 +916,12 @@ static int termCanDriveIndex(
   }
   if( (pTerm->prereqRight & notReady)!=0 ) return 0;
   assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
-  if( pTerm->u.x.leftColumn<0 ) return 0;
-  aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity;
+  leftCol = pTerm->u.x.leftColumn;
+  if( leftCol<0 ) return 0;
+  aff = pSrc->pSTab->aCol[leftCol].affinity;
   if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0;
   testcase( pTerm->pExpr->op==TK_IS );
-  return 1;
+  return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol);
 }
 #endif
 
@@ -822,7 +937,7 @@ static int termCanDriveIndex(
 **
 ** This is only required if sqlite3_stmt_scanstatus() is enabled, to
 ** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
-** values with. In order to avoid breaking legacy code and test cases, 
+** values with. In order to avoid breaking legacy code and test cases,
 ** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
 */
 static void explainAutomaticIndex(
@@ -839,7 +954,7 @@ static void explainAutomaticIndex(
     sqlite3_str *pStr = sqlite3_str_new(pParse->db);
     sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
     assert( pIdx->nColumn>1 );
-    assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID );
+    assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID || !HasRowid(pTab) );
     for(ii=0; ii<(pIdx->nColumn-1); ii++){
       const char *zName = 0;
       int iCol = pIdx->aiColumn[ii];
@@ -914,7 +1029,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   nKeyCol = 0;
   pTabList = pWC->pWInfo->pTabList;
   pSrc = &pTabList->a[pLevel->iFrom];
-  pTable = pSrc->pTab;
+  pTable = pSrc->pSTab;
   pWCEnd = &pWC->a[pWC->nTerm];
   pLoop = pLevel->pWLoop;
   idxCols = 0;
@@ -924,7 +1039,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
     ** WHERE clause (or the ON clause of a LEFT join) that constrain which
     ** rows of the target table (pSrc) that can be used. */
     if( (pTerm->wtFlags & TERM_VIRTUAL)==0
-     && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom)
+     && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0)
     ){
       pPartial = sqlite3ExprAnd(pParse, pPartial,
                                 sqlite3ExprDup(pParse->db, pExpr, 0));
@@ -966,10 +1081,23 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   ** if they go out of sync.
   */
   if( IsView(pTable) ){
-    extraCols = ALLBITS;
+    extraCols = ALLBITS & ~idxCols;
   }else{
     extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
   }
+  if( !HasRowid(pTable) ){
+    /* For WITHOUT ROWID tables, ensure that all PRIMARY KEY columns are
+    ** either in the idxCols mask or in the extraCols mask */
+    for(i=0; i<pTable->nCol; i++){
+      if( (pTable->aCol[i].colFlags & COLFLAG_PRIMKEY)==0 ) continue;
+      if( i>=BMS-1 ){
+        extraCols |= MASKBIT(BMS-1);
+        break;
+      }
+      if( idxCols & MASKBIT(i) ) continue;
+      extraCols |= MASKBIT(i);
+    }
+  }
   mxBitCol = MIN(BMS-1,pTable->nCol);
   testcase( pTable->nCol==BMS-1 );
   testcase( pTable->nCol==BMS-2 );
@@ -981,7 +1109,10 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   }
 
   /* Construct the Index object to describe this index */
-  pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed);
+  assert( nKeyCol <= pTable->nCol + MAX(0, pTable->nCol - BMS + 1) );
+  /* ^-- This guarantees that the number of index columns will fit in the u16 */
+  pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+HasRowid(pTable),
+                                    0, &zNotUsed);
   if( pIdx==0 ) goto end_auto_index_create;
   pLoop->u.btree.pIndex = pIdx;
   pIdx->zName = "auto-index";
@@ -1037,8 +1168,10 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
     }
   }
   assert( n==nKeyCol );
-  pIdx->aiColumn[n] = XN_ROWID;
-  pIdx->azColl[n] = sqlite3StrBINARY;
+  if( HasRowid(pTable) ){
+    pIdx->aiColumn[n] = XN_ROWID;
+    pIdx->azColl[n] = sqlite3StrBINARY;
+  }
 
   /* Create the automatic index */
   explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
@@ -1056,14 +1189,21 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   /* Fill the automatic index with content */
   assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
   if( pSrc->fg.viaCoroutine ){
-    int regYield = pSrc->regReturn;
+    int regYield;
+    Subquery *pSubq;
+    assert( pSrc->fg.isSubquery );
+    pSubq = pSrc->u4.pSubq;
+    assert( pSubq!=0 );
+    regYield = pSubq->regReturn;
     addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
-    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub);
+    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub);
     addrTop =  sqlite3VdbeAddOp1(v, OP_Yield, regYield);
     VdbeCoverage(v);
-    VdbeComment((v, "next row of %s", pSrc->pTab->zName));
+    VdbeComment((v, "next row of %s", pSrc->pSTab->zName));
   }else{
-    addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
+    assert( pLevel->addrHalt );
+    addrTop = sqlite3VdbeAddOp2(v, OP_Rewind,pLevel->iTabCur,pLevel->addrHalt);
+    VdbeCoverage(v);
   }
   if( pPartial ){
     iContinue = sqlite3VdbeMakeLabel(pParse);
@@ -1083,20 +1223,24 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
   if( pSrc->fg.viaCoroutine ){
+    assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 );
     sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
     testcase( pParse->db->mallocFailed );
     assert( pLevel->iIdxCur>0 );
     translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
-                          pSrc->regResult, pLevel->iIdxCur);
+                          pSrc->u4.pSubq->regResult, pLevel->iIdxCur);
     sqlite3VdbeGoto(v, addrTop);
     pSrc->fg.viaCoroutine = 0;
+    sqlite3VdbeJumpHere(v, addrTop);
   }else{
     sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
     sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
+    if( (pSrc->fg.jointype & JT_LEFT)!=0 ){
+      sqlite3VdbeJumpHere(v, addrTop);
+    }
   }
-  sqlite3VdbeJumpHere(v, addrTop);
   sqlite3ReleaseTempReg(pParse, regRecord);
-  
+ 
   /* Jump here when skipping the initialization */
   sqlite3VdbeJumpHere(v, addrInit);
   sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
@@ -1142,13 +1286,17 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
   WhereLoop *pLoop = pLevel->pWLoop;   /* The loop being coded */
   int iCur;                            /* Cursor for table getting the filter */
   IndexedExpr *saved_pIdxEpr;          /* saved copy of Parse.pIdxEpr */
+  IndexedExpr *saved_pIdxPartExpr;     /* saved copy of Parse.pIdxPartExpr */
 
   saved_pIdxEpr = pParse->pIdxEpr;
+  saved_pIdxPartExpr = pParse->pIdxPartExpr;
   pParse->pIdxEpr = 0;
+  pParse->pIdxPartExpr = 0;
 
   assert( pLoop!=0 );
   assert( v!=0 );
   assert( pLoop->wsFlags & WHERE_BLOOMFILTER );
+  assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 );
 
   addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   do{
@@ -1174,7 +1322,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
     iSrc = pLevel->iFrom;
     pItem = &pTabList->a[iSrc];
     assert( pItem!=0 );
-    pTab = pItem->pTab;
+    pTab = pItem->pSTab;
     assert( pTab!=0 );
     sz = sqlite3LogEstToInt(pTab->nRowLogEst);
     if( sz<10000 ){
@@ -1189,7 +1337,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
     for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
       Expr *pExpr = pTerm->pExpr;
       if( (pTerm->wtFlags & TERM_VIRTUAL)==0
-       && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc)
+       && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0)
       ){
         sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
       }
@@ -1205,7 +1353,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
       int r1 = sqlite3GetTempRange(pParse, n);
       int jj;
       for(jj=0; jj<n; jj++){
-        assert( pIdx->pTable==pItem->pTab );
+        assert( pIdx->pTable==pItem->pSTab );
         sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj);
       }
       sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n);
@@ -1238,12 +1386,27 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
   }while( iLevel < pWInfo->nLevel );
   sqlite3VdbeJumpHere(v, addrOnce);
   pParse->pIdxEpr = saved_pIdxEpr;
+  pParse->pIdxPartExpr = saved_pIdxPartExpr;
 }
 
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
-** Allocate and populate an sqlite3_index_info structure. It is the 
+** Return term iTerm of the WhereClause passed as the first argument. Terms
+** are numbered from 0 upwards, starting with the terms in pWC->a[], then
+** those in pWC->pOuter->a[] (if any), and so on.
+*/
+static WhereTerm *termFromWhereClause(WhereClause *pWC, int iTerm){
+  WhereClause *p;
+  for(p=pWC; p; p=p->pOuter){
+    if( iTerm<p->nTerm ) return &p->a[iTerm];
+    iTerm -= p->nTerm;
+  }
+  return 0;
+}
+
+/*
+** Allocate and populate an sqlite3_index_info structure. It is the
 ** responsibility of the caller to eventually release the structure
 ** by passing the pointer returned by this function to freeIndexInfo().
 */
@@ -1268,9 +1431,10 @@ static sqlite3_index_info *allocateIndexInfo(
   const Table *pTab;
   int eDistinct = 0;
   ExprList *pOrderBy = pWInfo->pOrderBy;
- 
+  WhereClause *p;
+
   assert( pSrc!=0 );
-  pTab = pSrc->pTab;
+  pTab = pSrc->pSTab;
   assert( pTab!=0 );
   assert( IsVirtual(pTab) );
 
@@ -1278,31 +1442,33 @@ static sqlite3_index_info *allocateIndexInfo(
   ** Mark each term with the TERM_OK flag.  Set nTerm to the number of
   ** terms found.
   */
-  for(i=nTerm=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
-    pTerm->wtFlags &= ~TERM_OK;
-    if( pTerm->leftCursor != pSrc->iCursor ) continue;
-    if( pTerm->prereqRight & mUnusable ) continue;
-    assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
-    testcase( pTerm->eOperator & WO_IN );
-    testcase( pTerm->eOperator & WO_ISNULL );
-    testcase( pTerm->eOperator & WO_IS );
-    testcase( pTerm->eOperator & WO_ALL );
-    if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
-    if( pTerm->wtFlags & TERM_VNULL ) continue;
+  for(p=pWC, nTerm=0; p; p=p->pOuter){
+    for(i=0, pTerm=p->a; i<p->nTerm; i++, pTerm++){
+      pTerm->wtFlags &= ~TERM_OK;
+      if( pTerm->leftCursor != pSrc->iCursor ) continue;
+      if( pTerm->prereqRight & mUnusable ) continue;
+      assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) );
+      testcase( pTerm->eOperator & WO_IN );
+      testcase( pTerm->eOperator & WO_ISNULL );
+      testcase( pTerm->eOperator & WO_IS );
+      testcase( pTerm->eOperator & WO_ALL );
+      if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue;
+      if( pTerm->wtFlags & TERM_VNULL ) continue;
 
-    assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
-    assert( pTerm->u.x.leftColumn>=XN_ROWID );
-    assert( pTerm->u.x.leftColumn<pTab->nCol );
-    if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
-     && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
-    ){
-      continue;
+      assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 );
+      assert( pTerm->u.x.leftColumn>=XN_ROWID );
+      assert( pTerm->u.x.leftColumn<pTab->nCol );
+      if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0
+          && !constraintCompatibleWithOuterJoin(pTerm,pSrc)
+        ){
+        continue;
+      }
+      nTerm++;
+      pTerm->wtFlags |= TERM_OK;
     }
-    nTerm++;
-    pTerm->wtFlags |= TERM_OK;
   }
 
-  /* If the ORDER BY clause contains only columns in the current 
+  /* If the ORDER BY clause contains only columns in the current
   ** virtual table then allocate space for the aOrderBy part of
   ** the sqlite3_index_info structure.
   */
@@ -1314,7 +1480,7 @@ static sqlite3_index_info *allocateIndexInfo(
       Expr *pE2;
 
       /* Skip over constant terms in the ORDER BY clause */
-      if( sqlite3ExprIsConstant(pExpr) ){
+      if( sqlite3ExprIsConstant(0, pExpr) ){
         continue;
       }
 
@@ -1349,7 +1515,7 @@ static sqlite3_index_info *allocateIndexInfo(
     }
     if( i==n ){
       nOrderBy = n;
-      if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) ){
+      if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){
         eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0);
       }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){
         eDistinct = 1;
@@ -1361,8 +1527,8 @@ static sqlite3_index_info *allocateIndexInfo(
   */
   pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo)
                            + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm
-                           + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden)
-                           + sizeof(sqlite3_value*)*nTerm );
+                           + sizeof(*pIdxOrderBy)*nOrderBy
+                           + SZ_HIDDENINDEXINFO(nTerm) );
   if( pIdxInfo==0 ){
     sqlite3ErrorMsg(pParse, "out of memory");
     return 0;
@@ -1374,59 +1540,75 @@ static sqlite3_index_info *allocateIndexInfo(
   pIdxInfo->aConstraint = pIdxCons;
   pIdxInfo->aOrderBy = pIdxOrderBy;
   pIdxInfo->aConstraintUsage = pUsage;
+  pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
+  if( HasRowid(pTab)==0 ){
+    /* Ensure that all bits associated with PK columns are set. This is to
+    ** ensure they are available for cases like RIGHT joins or OR loops. */
+    Index *pPk = sqlite3PrimaryKeyIndex((Table*)pTab);
+    assert( pPk!=0 );
+    for(i=0; i<pPk->nKeyCol; i++){
+      int iCol = pPk->aiColumn[i];
+      assert( iCol>=0 );
+      if( iCol>=BMS-1 ) iCol = BMS-1;
+      pIdxInfo->colUsed |= MASKBIT(iCol);
+    }
+  }
   pHidden->pWC = pWC;
   pHidden->pParse = pParse;
   pHidden->eDistinct = eDistinct;
   pHidden->mIn = 0;
-  for(i=j=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
-    u16 op;
-    if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
-    pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
-    pIdxCons[j].iTermOffset = i;
-    op = pTerm->eOperator & WO_ALL;
-    if( op==WO_IN ){
-      if( (pTerm->wtFlags & TERM_SLICE)==0 ){
-        pHidden->mIn |= SMASKBIT32(j);
+  for(p=pWC, i=j=0; p; p=p->pOuter){
+    int nLast = i+p->nTerm;;
+    for(pTerm=p->a; i<nLast; i++, pTerm++){
+      u16 op;
+      if( (pTerm->wtFlags & TERM_OK)==0 ) continue;
+      pIdxCons[j].iColumn = pTerm->u.x.leftColumn;
+      pIdxCons[j].iTermOffset = i;
+      op = pTerm->eOperator & WO_ALL;
+      if( op==WO_IN ){
+        if( (pTerm->wtFlags & TERM_SLICE)==0 ){
+          pHidden->mIn |= SMASKBIT32(j);
+        }
+        op = WO_EQ;
       }
-      op = WO_EQ;
-    }
-    if( op==WO_AUX ){
-      pIdxCons[j].op = pTerm->eMatchOp;
-    }else if( op & (WO_ISNULL|WO_IS) ){
-      if( op==WO_ISNULL ){
-        pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
+      if( op==WO_AUX ){
+        pIdxCons[j].op = pTerm->eMatchOp;
+      }else if( op & (WO_ISNULL|WO_IS) ){
+        if( op==WO_ISNULL ){
+          pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL;
+        }else{
+          pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
+        }
       }else{
-        pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS;
-      }
-    }else{
-      pIdxCons[j].op = (u8)op;
-      /* The direct assignment in the previous line is possible only because
-      ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical.  The
-      ** following asserts verify this fact. */
-      assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
-      assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
-      assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
-      assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
-      assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
-      assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
-
-      if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
-       && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
-      ){
-        testcase( j!=i );
-        if( j<16 ) mNoOmit |= (1 << j);
-        if( op==WO_LT ) pIdxCons[j].op = WO_LE;
-        if( op==WO_GT ) pIdxCons[j].op = WO_GE;
+        pIdxCons[j].op = (u8)op;
+        /* The direct assignment in the previous line is possible only because
+        ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical.  The
+        ** following asserts verify this fact. */
+        assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ );
+        assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT );
+        assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE );
+        assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT );
+        assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
+        assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) );
+
+        if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
+            && sqlite3ExprIsVector(pTerm->pExpr->pRight)
+          ){
+          testcase( j!=i );
+          if( j<16 ) mNoOmit |= (1 << j);
+          if( op==WO_LT ) pIdxCons[j].op = WO_LE;
+          if( op==WO_GT ) pIdxCons[j].op = WO_GE;
+        }
       }
-    }
 
-    j++;
+      j++;
+    }
   }
   assert( j==nTerm );
   pIdxInfo->nConstraint = j;
   for(i=j=0; i<nOrderBy; i++){
     Expr *pExpr = pOrderBy->a[i].pExpr;
-    if( sqlite3ExprIsConstant(pExpr) ) continue;
+    if( sqlite3ExprIsConstant(0, pExpr) ) continue;
     assert( pExpr->op==TK_COLUMN
          || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN
               && pExpr->iColumn==pExpr->pLeft->iColumn) );
@@ -1440,6 +1622,17 @@ static sqlite3_index_info *allocateIndexInfo(
   return pIdxInfo;
 }
 
+/*
+** Free and zero the sqlite3_index_info.idxStr value if needed.
+*/
+static void freeIdxStr(sqlite3_index_info *pIdxInfo){
+  if( pIdxInfo->needToFreeIdxStr ){
+    sqlite3_free(pIdxInfo->idxStr);
+    pIdxInfo->idxStr = 0;
+    pIdxInfo->needToFreeIdxStr = 0;
+  }
+}  
+
 /*
 ** Free an sqlite3_index_info structure allocated by allocateIndexInfo()
 ** and possibly modified by xBestIndex methods.
@@ -1455,6 +1648,7 @@ static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
     sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */
     pHidden->aRhs[i] = 0;
   }
+  freeIdxStr(pIdxInfo);
   sqlite3DbFree(db, pIdxInfo);
 }
 
@@ -1475,14 +1669,16 @@ static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){
 ** that this is required.
 */
 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
-  sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
   int rc;
+  sqlite3_vtab *pVtab;
 
-  whereTraceIndexInfoInputs(p);
+  assert( IsVirtual(pTab) );
+  pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
+  whereTraceIndexInfoInputs(p, pTab);
   pParse->db->nSchemaLock++;
   rc = pVtab->pModule->xBestIndex(pVtab, p);
   pParse->db->nSchemaLock--;
-  whereTraceIndexInfoOutputs(p);
+  whereTraceIndexInfoOutputs(p, pTab);
 
   if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){
     if( rc==SQLITE_NOMEM ){
@@ -1513,8 +1709,8 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
 ** Return the index of the sample that is the smallest sample that
 ** is greater than or equal to pRec. Note that this index is not an index
 ** into the aSample[] array - it is an index into a virtual set of samples
-** based on the contents of aSample[] and the number of fields in record 
-** pRec. 
+** based on the contents of aSample[] and the number of fields in record
+** pRec.
 */
 static int whereKeyStats(
   Parse *pParse,              /* Database connection */
@@ -1539,7 +1735,7 @@ static int whereKeyStats(
   assert( pRec!=0 );
   assert( pIdx->nSample>0 );
   assert( pRec->nField>0 );
- 
+
 
   /* Do a binary search to find the first sample greater than or equal
   ** to pRec. If pRec contains a single field, the set of samples to search
@@ -1551,38 +1747,38 @@ static int whereKeyStats(
   ** consider prefixes of those samples. For example, if the set of samples
   ** in aSample is:
   **
-  **     aSample[0] = (a, 5) 
-  **     aSample[1] = (a, 10) 
-  **     aSample[2] = (b, 5) 
-  **     aSample[3] = (c, 100) 
+  **     aSample[0] = (a, 5)
+  **     aSample[1] = (a, 10)
+  **     aSample[2] = (b, 5)
+  **     aSample[3] = (c, 100)
   **     aSample[4] = (c, 105)
   **
-  ** Then the search space should ideally be the samples above and the 
-  ** unique prefixes [a], [b] and [c]. But since that is hard to organize, 
+  ** Then the search space should ideally be the samples above and the
+  ** unique prefixes [a], [b] and [c]. But since that is hard to organize,
   ** the code actually searches this set:
   **
-  **     0: (a) 
-  **     1: (a, 5) 
-  **     2: (a, 10) 
-  **     3: (a, 10) 
-  **     4: (b) 
-  **     5: (b, 5) 
-  **     6: (c) 
-  **     7: (c, 100) 
+  **     0: (a)
+  **     1: (a, 5)
+  **     2: (a, 10)
+  **     3: (a, 10)
+  **     4: (b)
+  **     5: (b, 5)
+  **     6: (c)
+  **     7: (c, 100)
   **     8: (c, 105)
   **     9: (c, 105)
   **
   ** For each sample in the aSample[] array, N samples are present in the
-  ** effective sample array. In the above, samples 0 and 1 are based on 
+  ** effective sample array. In the above, samples 0 and 1 are based on
   ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc.
   **
   ** Often, sample i of each block of N effective samples has (i+1) fields.
   ** Except, each sample may be extended to ensure that it is greater than or
-  ** equal to the previous sample in the array. For example, in the above, 
-  ** sample 2 is the first sample of a block of N samples, so at first it 
-  ** appears that it should be 1 field in size. However, that would make it 
-  ** smaller than sample 1, so the binary search would not work. As a result, 
-  ** it is extended to two fields. The duplicates that this creates do not 
+  ** equal to the previous sample in the array. For example, in the above,
+  ** sample 2 is the first sample of a block of N samples, so at first it
+  ** appears that it should be 1 field in size. However, that would make it
+  ** smaller than sample 1, so the binary search would not work. As a result,
+  ** it is extended to two fields. The duplicates that this creates do not
   ** cause any problems.
   */
   if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
@@ -1601,7 +1797,7 @@ static int whereKeyStats(
     iSamp = iTest / nField;
     if( iSamp>0 ){
       /* The proposed effective sample is a prefix of sample aSample[iSamp].
-      ** Specifically, the shortest prefix of at least (1 + iTest%nField) 
+      ** Specifically, the shortest prefix of at least (1 + iTest%nField)
       ** fields that is greater than the previous effective sample.  */
       for(n=(iTest % nField) + 1; n<nField; n++){
         if( aSample[iSamp-1].anLt[n-1]!=aSample[iSamp].anLt[n-1] ) break;
@@ -1636,8 +1832,8 @@ static int whereKeyStats(
       assert( i<pIdx->nSample );
       assert( iCol==nField-1 );
       pRec->nField = nField;
-      assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) 
-           || pParse->db->mallocFailed 
+      assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)
+           || pParse->db->mallocFailed
       );
     }else{
       /* Unless i==pIdx->nSample, indicating that pRec is larger than
@@ -1645,7 +1841,7 @@ static int whereKeyStats(
       ** (iCol+1) field prefix of sample i.  */
       assert( i<=pIdx->nSample && i>=0 );
       pRec->nField = iCol+1;
-      assert( i==pIdx->nSample 
+      assert( i==pIdx->nSample
            || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0
            || pParse->db->mallocFailed );
 
@@ -1673,7 +1869,7 @@ static int whereKeyStats(
     aStat[0] = aSample[i].anLt[iCol];
     aStat[1] = aSample[i].anEq[iCol];
   }else{
-    /* At this point, the (iCol+1) field prefix of aSample[i] is the first 
+    /* At this point, the (iCol+1) field prefix of aSample[i] is the first
     ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec
     ** is larger than all samples in the array. */
     tRowcnt iUpper, iGap;
@@ -1705,7 +1901,7 @@ static int whereKeyStats(
 
 /*
 ** If it is not NULL, pTerm is a term that provides an upper or lower
-** bound on a range scan. Without considering pTerm, it is estimated 
+** bound on a range scan. Without considering pTerm, it is estimated
 ** that the scan will visit nNew rows. This function returns the number
 ** estimated to be visited after taking pTerm into account.
 **
@@ -1743,18 +1939,18 @@ char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
 
 
 #ifdef SQLITE_ENABLE_STAT4
-/* 
+/*
 ** This function is called to estimate the number of rows visited by a
 ** range-scan on a skip-scan index. For example:
 **
 **   CREATE INDEX i1 ON t1(a, b, c);
 **   SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?;
 **
-** Value pLoop->nOut is currently set to the estimated number of rows 
-** visited for scanning (a=? AND b=?). This function reduces that estimate 
+** Value pLoop->nOut is currently set to the estimated number of rows
+** visited for scanning (a=? AND b=?). This function reduces that estimate
 ** by some factor to account for the (c BETWEEN ? AND ?) expression based
-** on the stat4 data for the index. this scan will be peformed multiple 
-** times (once for each (a,b) combination that matches a=?) is dealt with 
+** on the stat4 data for the index. this scan will be performed multiple
+** times (once for each (a,b) combination that matches a=?) is dealt with
 ** by the caller.
 **
 ** It does this by scanning through all stat4 samples, comparing values
@@ -1775,7 +1971,7 @@ char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
 ** estimate of the number of rows delivered remains unchanged), *pbDone
 ** is left as is.
 **
-** If an error occurs, an SQLite error code is returned. Otherwise, 
+** If an error occurs, an SQLite error code is returned. Otherwise,
 ** SQLITE_OK.
 */
 static int whereRangeSkipScanEst(
@@ -1793,7 +1989,7 @@ static int whereRangeSkipScanEst(
   int rc = SQLITE_OK;
   u8 aff = sqlite3IndexColumnAffinity(db, p, nEq);
   CollSeq *pColl;
-  
+ 
   sqlite3_value *p1 = 0;          /* Value extracted from pLower */
   sqlite3_value *p2 = 0;          /* Value extracted from pUpper */
   sqlite3_value *pVal = 0;        /* Value extracted from record */
@@ -1825,7 +2021,7 @@ static int whereRangeSkipScanEst(
     nDiff = (nUpper - nLower);
     if( nDiff<=0 ) nDiff = 1;
 
-    /* If there is both an upper and lower bound specified, and the 
+    /* If there is both an upper and lower bound specified, and the
     ** comparisons indicate that they are close together, use the fallback
     ** method (assume that the scan visits 1/64 of the rows) for estimating
     ** the number of rows visited. Otherwise, estimate the number of rows
@@ -1872,7 +2068,7 @@ static int whereRangeSkipScanEst(
 **
 **   ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
 **
-** then nEq is set to 1 (as the range restricted column, b, is the second 
+** then nEq is set to 1 (as the range restricted column, b, is the second
 ** left-most column of the index). Or, if the query is:
 **
 **   ... FROM t1 WHERE a > ? AND a < ? ...
@@ -1880,13 +2076,13 @@ static int whereRangeSkipScanEst(
 ** then nEq is set to 0.
 **
 ** When this function is called, *pnOut is set to the sqlite3LogEst() of the
-** number of rows that the index scan is expected to visit without 
-** considering the range constraints. If nEq is 0, then *pnOut is the number of 
+** number of rows that the index scan is expected to visit without
+** considering the range constraints. If nEq is 0, then *pnOut is the number of
 ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced)
 ** to account for the range constraints pLower and pUpper.
-** 
+**
 ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be
-** used, a single range inequality reduces the search space by a factor of 4. 
+** used, a single range inequality reduces the search space by a factor of 4.
 ** and a pair of constraints (x>? AND x<?) reduces the expected number of
 ** rows visited by a factor of 64.
 */
@@ -1914,7 +2110,7 @@ static int whereRangeScanEst(
       int nBtm = pLoop->u.btree.nBtm;
       int nTop = pLoop->u.btree.nTop;
 
-      /* Variable iLower will be set to the estimate of the number of rows in 
+      /* Variable iLower will be set to the estimate of the number of rows in
       ** the index that are less than the lower bound of the range query. The
       ** lower bound being the concatenation of $P and $L, where $P is the
       ** key-prefix formed by the nEq values matched against the nEq left-most
@@ -1923,7 +2119,7 @@ static int whereRangeScanEst(
       ** Or, if pLower is NULL or $L cannot be extracted from it (because it
       ** is not a simple variable or literal value), the lower bound of the
       ** range is $P. Due to a quirk in the way whereKeyStats() works, even
-      ** if $L is available, whereKeyStats() is called for both ($P) and 
+      ** if $L is available, whereKeyStats() is called for both ($P) and
       ** ($P:$L) and the larger of the two returned values is used.
       **
       ** Similarly, iUpper is to be set to the estimate of the number of rows
@@ -1947,7 +2143,7 @@ static int whereRangeScanEst(
         iLower = 0;
         iUpper = p->nRowEst0;
       }else{
-        /* Note: this call could be optimized away - since the same values must 
+        /* Note: this call could be optimized away - since the same values must
         ** have been requested when testing key $P in whereEqualScanEst().  */
         whereKeyStats(pParse, p, pRec, 0, a);
         iLower = a[0];
@@ -2005,7 +2201,8 @@ static int whereRangeScanEst(
           ** sample, then assume they are 4x more selective.  This brings
           ** the estimated selectivity more in line with what it would be
           ** if estimated without the use of STAT4 tables. */
-          if( iLwrIdx==iUprIdx ) nNew -= 20;  assert( 20==sqlite3LogEst(4) );
+          if( iLwrIdx==iUprIdx ){ nNew -= 20; }
+          assert( 20==sqlite3LogEst(4) );
         }else{
           nNew = 10;        assert( 10==sqlite3LogEst(2) );
         }
@@ -2035,7 +2232,7 @@ static int whereRangeScanEst(
   ** reduced by an additional 75%. This means that, by default, an open-ended
   ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the
   ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to
-  ** match 1/64 of the index. */ 
+  ** match 1/64 of the index. */
   if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){
     nNew -= 20;
   }
@@ -2062,7 +2259,7 @@ static int whereRangeScanEst(
 ** for that index.  When pExpr==NULL that means the constraint is
 ** "x IS NULL" instead of "x=VALUE".
 **
-** Write the estimated row count into *pnRow and return SQLITE_OK. 
+** Write the estimated row count into *pnRow and return SQLITE_OK.
 ** If unable to make an estimate, leave *pnRow unchanged and return
 ** non-zero.
 **
@@ -2113,7 +2310,7 @@ static int whereEqualScanEst(
   WHERETRACE(0x20,("equality scan regions %s(%d): %d\n",
                    p->zName, nEq-1, (int)a[1]));
   *pnRow = a[1];
-  
+ 
   return rc;
 }
 #endif /* SQLITE_ENABLE_STAT4 */
@@ -2126,7 +2323,7 @@ static int whereEqualScanEst(
 **
 **        WHERE x IN (1,2,3,4)
 **
-** Write the estimated row count into *pnRow and return SQLITE_OK. 
+** Write the estimated row count into *pnRow and return SQLITE_OK.
 ** If unable to make an estimate, leave *pnRow unchanged and return
 ** non-zero.
 **
@@ -2168,7 +2365,7 @@ static int whereInScanEst(
 #endif /* SQLITE_ENABLE_STAT4 */
 
 
-#ifdef WHERETRACE_ENABLED
+#if defined(WHERETRACE_ENABLED) || defined(SQLITE_DEBUG)
 /*
 ** Print the content of a WhereTerm object
 */
@@ -2188,11 +2385,12 @@ void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
       sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
                        pTerm->leftCursor, pTerm->u.x.leftColumn);
     }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
-      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx", 
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx",
                        pTerm->u.pOrInfo->indexable);
     }else{
       sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
     }
+    iTerm = pTerm->iTerm = MAX(iTerm,pTerm->iTerm);
     sqlite3DebugPrintf(
        "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x",
        iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags);
@@ -2212,6 +2410,9 @@ void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){
     sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
   }
 }
+void sqlite3ShowWhereTerm(WhereTerm *pTerm){
+  sqlite3WhereTermPrint(pTerm, 0);
+}
 #endif
 
 #ifdef WHERETRACE_ENABLED
@@ -2229,17 +2430,36 @@ void sqlite3WhereClausePrint(WhereClause *pWC){
 #ifdef WHERETRACE_ENABLED
 /*
 ** Print a WhereLoop object for debugging purposes
+**
+** Format example:
+**
+**     .--- Position in WHERE clause           rSetup, rRun, nOut ---.
+**     |                                                             |
+**     |  .--- selfMask                       nTerm ------.          |
+**     |  |                                               |          |
+**     |  |   .-- prereq    Idx          wsFlags----.     |          |
+**     |  |   |             Name                    |     |          |
+**     |  |   |           __|__        nEq ---.  ___|__   |        __|__
+**     | / \ / \         /     \              | /      \ / \      /     \
+**     1.002.001         t2.t2xy              2 f 010241 N 2 cost 0,56,31
 */
-void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
-  WhereInfo *pWInfo = pWC->pWInfo;
-  int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
-  SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
-  Table *pTab = pItem->pTab;
-  Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
-  sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
-                     p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
-  sqlite3DebugPrintf(" %12s",
-                     pItem->zAlias ? pItem->zAlias : pTab->zName);
+void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){
+  WhereInfo *pWInfo;
+  if( pWC ){
+    pWInfo = pWC->pWInfo;
+    int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
+    SrcItem *pItem = pWInfo->pTabList->a + p->iTab;
+    Table *pTab = pItem->pSTab;
+    Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
+    sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
+                       p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
+    sqlite3DebugPrintf(" %12s",
+                       pItem->zAlias ? pItem->zAlias : pTab->zName);
+  }else{
+    pWInfo = 0;
+    sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d",
+         p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab);
+  }
   if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
     const char *zName;
     if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){
@@ -2268,7 +2488,12 @@ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
   }else{
     sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm);
   }
-  sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
+  if( pWInfo && pWInfo->bStarUsed && p->rStarDelta!=0 ){
+    sqlite3DebugPrintf(" cost %d,%d,%d delta=%d\n",
+                       p->rSetup, p->rRun, p->nOut, p->rStarDelta);
+  }else{
+    sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
+  }
   if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){
     int i;
     for(i=0; i<p->nLTerm; i++){
@@ -2276,6 +2501,15 @@ void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){
     }
   }
 }
+void sqlite3ShowWhereLoop(const WhereLoop *p){
+  if( p ) sqlite3WhereLoopPrint(p, 0);
+}
+void sqlite3ShowWhereLoopList(const WhereLoop *p){
+  while( p ){
+    sqlite3ShowWhereLoop(p);
+    p = p->pNextLoop;
+  }
+}
 #endif
 
 /*
@@ -2388,46 +2622,60 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
 }
 
 /*
-** Return TRUE if all of the following are true:
-**
-**   (1)  X has the same or lower cost, or returns the same or fewer rows, 
-**        than Y.
-**   (2)  X uses fewer WHERE clause terms than Y
-**   (3)  Every WHERE clause term used by X is also used by Y
-**   (4)  X skips at least as many columns as Y
-**   (5)  If X is a covering index, than Y is too
-**
-** Conditions (2) and (3) mean that X is a "proper subset" of Y.
-** If X is a proper subset of Y then Y is a better choice and ought
-** to have a lower cost.  This routine returns TRUE when that cost 
-** relationship is inverted and needs to be adjusted.  Constraint (4)
-** was added because if X uses skip-scan less than Y it still might
-** deserve a lower cost even if it is a proper subset of Y.  Constraint (5)
-** was added because a covering index probably deserves to have a lower cost
-** than a non-covering index even if it is a proper subset.
+** Return TRUE if X is a proper subset of Y but is of equal or less cost.
+** In other words, return true if all constraints of X are also part of Y
+** and Y has additional constraints that might speed the search that X lacks
+** but the cost of running X is not more than the cost of running Y.
+**
+** In other words, return true if the cost relationship between X and Y
+** is inverted and needs to be adjusted.
+**
+** Case 1:
+**
+**   (1a)  X and Y use the same index.
+**   (1b)  X has fewer == terms than Y
+**   (1c)  Neither X nor Y use skip-scan
+**   (1d)  X does not have a a greater cost than Y
+**
+** Case 2:
+**
+**   (2a)  X has the same or lower cost, or returns the same or fewer rows,
+**         than Y.
+**   (2b)  X uses fewer WHERE clause terms than Y
+**   (2c)  Every WHERE clause term used by X is also used by Y
+**   (2d)  X skips at least as many columns as Y
+**   (2e)  If X is a covering index, than Y is too
 */
 static int whereLoopCheaperProperSubset(
   const WhereLoop *pX,       /* First WhereLoop to compare */
   const WhereLoop *pY        /* Compare against this WhereLoop */
 ){
   int i, j;
+  if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */
+  assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 );
+  assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 );
+  if( pX->u.btree.nEq < pY->u.btree.nEq                  /* (1b) */
+   && pX->u.btree.pIndex==pY->u.btree.pIndex             /* (1a) */
+   && pX->nSkip==0 && pY->nSkip==0                       /* (1c) */
+  ){
+    return 1;  /* Case 1 is true */
+  }
   if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
-    return 0; /* X is not a subset of Y */
+    return 0;                                            /* (2b) */
   }
-  if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0;
-  if( pY->nSkip > pX->nSkip ) return 0;
+  if( pY->nSkip > pX->nSkip ) return 0;                  /* (2d) */
   for(i=pX->nLTerm-1; i>=0; i--){
     if( pX->aLTerm[i]==0 ) continue;
     for(j=pY->nLTerm-1; j>=0; j--){
       if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
     }
-    if( j<0 ) return 0;  /* X not a subset of Y since term X[i] not used by Y */
+    if( j<0 ) return 0;                                  /* (2c) */
   }
-  if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 
+  if( (pX->wsFlags&WHERE_IDX_ONLY)!=0
    && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){
-    return 0;  /* Constraint (5) */
+    return 0;                                            /* (2e) */
   }
-  return 1;  /* All conditions meet */
+  return 1;  /* Case 2 is true */
 }
 
 /*
@@ -2450,10 +2698,10 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
     if( p->iTab!=pTemplate->iTab ) continue;
     if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
     if( whereLoopCheaperProperSubset(p, pTemplate) ){
-      /* Adjust pTemplate cost downward so that it is cheaper than its 
+      /* Adjust pTemplate cost downward so that it is cheaper than its
       ** subset p. */
       WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
-                       pTemplate->rRun, pTemplate->nOut, 
+                       pTemplate->rRun, pTemplate->nOut,
                        MIN(p->rRun, pTemplate->rRun),
                        MIN(p->nOut - 1, pTemplate->nOut)));
       pTemplate->rRun = MIN(p->rRun, pTemplate->rRun);
@@ -2462,7 +2710,7 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
       /* Adjust pTemplate cost upward so that it is costlier than p since
       ** pTemplate is a proper subset of p */
       WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
-                       pTemplate->rRun, pTemplate->nOut, 
+                       pTemplate->rRun, pTemplate->nOut,
                        MAX(p->rRun, pTemplate->rRun),
                        MAX(p->nOut + 1, pTemplate->nOut)));
       pTemplate->rRun = MAX(p->rRun, pTemplate->rRun);
@@ -2500,7 +2748,7 @@ static WhereLoop **whereLoopFindLesser(
     /* In the current implementation, the rSetup value is either zero
     ** or the cost of building an automatic index (NlogN) and the NlogN
     ** is the same for compatible WhereLoops. */
-    assert( p->rSetup==0 || pTemplate->rSetup==0 
+    assert( p->rSetup==0 || pTemplate->rSetup==0
                  || p->rSetup==pTemplate->rSetup );
 
     /* whereLoopAddBtree() always generates and inserts the automatic index
@@ -2508,7 +2756,7 @@ static WhereLoop **whereLoopFindLesser(
     ** rSetup. Call this SETUP-INVARIANT */
     assert( p->rSetup>=pTemplate->rSetup );
 
-    /* Any loop using an appliation-defined index (or PRIMARY KEY or
+    /* Any loop using an application-defined index (or PRIMARY KEY or
     ** UNIQUE constraint) with one or more == constraints is better
     ** than an automatic index. Unless it is a skip-scan. */
     if( (p->wsFlags & WHERE_AUTO_INDEX)!=0
@@ -2535,7 +2783,7 @@ static WhereLoop **whereLoopFindLesser(
 
     /* If pTemplate is always better than p, then cause p to be overwritten
     ** with pTemplate.  pTemplate is better than p if:
-    **   (1)  pTemplate has no more dependences than p, and
+    **   (1)  pTemplate has no more dependencies than p, and
     **   (2)  pTemplate has an equal or lower cost than p.
     */
     if( (p->prereq & pTemplate->prereq)==pTemplate->prereq   /* (1)  */
@@ -2565,7 +2813,7 @@ static WhereLoop **whereLoopFindLesser(
 **
 ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we
 ** still might overwrite similar loops with the new template if the
-** new template is better.  Loops may be overwritten if the following 
+** new template is better.  Loops may be overwritten if the following
 ** conditions are met:
 **
 **    (1)  They have the same iTab.
@@ -2623,7 +2871,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
       sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC);
     }
 #endif
-    return SQLITE_OK;  
+    return SQLITE_OK; 
   }else{
     p = *ppPrev;
   }
@@ -2653,7 +2901,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
   }else{
     /* We will be overwriting WhereLoop p[].  But before we do, first
     ** go through the rest of the list and delete any other entries besides
-    ** p[] that are also supplated by pTemplate */
+    ** p[] that are also supplanted by pTemplate */
     WhereLoop **ppTail = &p->pNextLoop;
     WhereLoop *pToDel;
     while( *ppTail ){
@@ -2765,7 +3013,7 @@ static void whereLoopOutputAdjust(
           Expr *pRight = pTerm->pExpr->pRight;
           int k = 0;
           testcase( pTerm->pExpr->op==TK_IS );
-          if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
+          if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){
             k = 10;
           }else{
             k = 20;
@@ -2783,7 +3031,7 @@ static void whereLoopOutputAdjust(
   }
 }
 
-/* 
+/*
 ** Term pTerm is a vector range comparison operation. The first comparison
 ** in the vector can be optimized using column nEq of the index. This
 ** function returns the total number of vector elements that can be used
@@ -2812,7 +3060,7 @@ static int whereRangeVectorLen(
 
   nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
   for(i=1; i<nCmp; i++){
-    /* Test if comparison i of pTerm is compatible with column (i+nEq) 
+    /* Test if comparison i of pTerm is compatible with column (i+nEq)
     ** of the index. If not, exit the loop.  */
     char aff;                     /* Comparison affinity */
     char idxaff = 0;              /* Indexed columns affinity */
@@ -2832,9 +3080,9 @@ static int whereRangeVectorLen(
     ** the right column of the right source table. And that the sort
     ** order of the index column is the same as the sort order of the
     ** leftmost index column.  */
-    if( pLhs->op!=TK_COLUMN 
-     || pLhs->iTable!=iCur 
-     || pLhs->iColumn!=pIdx->aiColumn[i+nEq] 
+    if( pLhs->op!=TK_COLUMN
+     || pLhs->iTable!=iCur
+     || pLhs->iColumn!=pIdx->aiColumn[i+nEq]
      || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
     ){
       break;
@@ -2853,7 +3101,7 @@ static int whereRangeVectorLen(
 }
 
 /*
-** Adjust the cost C by the costMult facter T.  This only occurs if
+** Adjust the cost C by the costMult factor T.  This only occurs if
 ** compiled with -DSQLITE_ENABLE_COSTMULT
 */
 #ifdef SQLITE_ENABLE_COSTMULT
@@ -2863,15 +3111,15 @@ static int whereRangeVectorLen(
 #endif
 
 /*
-** We have so far matched pBuilder->pNew->u.btree.nEq terms of the 
+** We have so far matched pBuilder->pNew->u.btree.nEq terms of the
 ** index pIndex. Try to match one more.
 **
-** When this function is called, pBuilder->pNew->nOut contains the 
-** number of rows expected to be visited by filtering using the nEq 
-** terms only. If it is modified, this value is restored before this 
+** When this function is called, pBuilder->pNew->nOut contains the
+** number of rows expected to be visited by filtering using the nEq
+** terms only. If it is modified, this value is restored before this
 ** function returns.
 **
-** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is 
+** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is
 ** a fake index used for the INTEGER PRIMARY KEY.
 */
 static int whereLoopAddBtreeIndex(
@@ -2880,7 +3128,7 @@ static int whereLoopAddBtreeIndex(
   Index *pProbe,                  /* An index on pSrc */
   LogEst nInMul                   /* log(Number of iterations due to IN) */
 ){
-  WhereInfo *pWInfo = pBuilder->pWInfo;  /* WHERE analyse context */
+  WhereInfo *pWInfo = pBuilder->pWInfo;  /* WHERE analyze context */
   Parse *pParse = pWInfo->pParse;        /* Parsing context */
   sqlite3 *db = pParse->db;       /* Database connection malloc context */
   WhereLoop *pNew;                /* Template WhereLoop under construction */
@@ -2917,7 +3165,9 @@ static int whereLoopAddBtreeIndex(
     assert( pNew->u.btree.nBtm==0 );
     opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
   }
-  if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
+  if( pProbe->bUnordered ){
+    opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
+  }
 
   assert( pNew->u.btree.nEq<pProbe->nColumn );
   assert( pNew->u.btree.nEq<pProbe->nKeyCol
@@ -2979,9 +3229,9 @@ static int whereLoopAddBtreeIndex(
     pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf;
 
     assert( nInMul==0
-        || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 
-        || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 
-        || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 
+        || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0
+        || (pNew->wsFlags & WHERE_COLUMN_IN)!=0
+        || (pNew->wsFlags & WHERE_SKIPSCAN)!=0
     );
 
     if( eOp & WO_IN ){
@@ -2989,6 +3239,7 @@ static int whereLoopAddBtreeIndex(
       if( ExprUseXSelect(pExpr) ){
         /* "x IN (SELECT ...)":  TUNING: the SELECT returns 25 rows */
         int i;
+        int bRedundant = 0;
         nIn = 46;  assert( 46==sqlite3LogEst(25) );
 
         /* The expression may actually be of the form (x, y) IN (SELECT...).
@@ -2997,7 +3248,20 @@ static int whereLoopAddBtreeIndex(
         ** for each such term. The following loop checks that pTerm is the
         ** first such term in use, and sets nIn back to 0 if it is not. */
         for(i=0; i<pNew->nLTerm-1; i++){
-          if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
+          if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ){
+            nIn = 0;
+            if( pNew->aLTerm[i]->u.x.iField == pTerm->u.x.iField ){
+              /* Detect when two or more columns of an index match the same 
+              ** column of a vector IN operater, and avoid adding the column
+              ** to the WhereLoop more than once.  See tag-20250707-01
+              ** in test/rowvalue.test */
+              bRedundant = 1;
+            }
+          }
+        }
+        if( bRedundant ){
+          pNew->nLTerm--;
+          continue;
         }
       }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
         /* "x IN (value, value, ...)" */
@@ -3008,7 +3272,7 @@ static int whereLoopAddBtreeIndex(
         /* Let:
         **   N = the total number of rows in the table
         **   K = the number of entries on the RHS of the IN operator
-        **   M = the number of rows in the table that match terms to the 
+        **   M = the number of rows in the table that match terms to the
         **       to the left in the same index.  If the IN operator is on
         **       the left-most index column, M==N.
         **
@@ -3053,11 +3317,11 @@ static int whereLoopAddBtreeIndex(
       int iCol = pProbe->aiColumn[saved_nEq];
       pNew->wsFlags |= WHERE_COLUMN_EQ;
       assert( saved_nEq==pNew->u.btree.nEq );
-      if( iCol==XN_ROWID 
+      if( iCol==XN_ROWID
        || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1)
       ){
-        if( iCol==XN_ROWID || pProbe->uniqNotNull 
-         || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) 
+        if( iCol==XN_ROWID || pProbe->uniqNotNull
+         || (pProbe->nKeyCol==1 && pProbe->onError && (eOp & WO_EQ))
         ){
           pNew->wsFlags |= WHERE_ONEROW;
         }else{
@@ -3104,7 +3368,7 @@ static int whereLoopAddBtreeIndex(
 
     /* At this point pNew->nOut is set to the number of rows expected to
     ** be visited by the index scan before considering term pTerm, or the
-    ** values of nIn and nInMul. In other words, assuming that all 
+    ** values of nIn and nInMul. In other words, assuming that all
     ** "x IN(...)" terms are replaced with "x = ?". This block updates
     ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul).  */
     assert( pNew->nOut==saved_nOut );
@@ -3125,8 +3389,8 @@ static int whereLoopAddBtreeIndex(
       }else{
 #ifdef SQLITE_ENABLE_STAT4
         tRowcnt nOut = 0;
-        if( nInMul==0 
-         && pProbe->nSample 
+        if( nInMul==0
+         && pProbe->nSample
          && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol)
          && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr))
          && OptimizationEnabled(db, SQLITE_Stat4)
@@ -3174,8 +3438,8 @@ static int whereLoopAddBtreeIndex(
         {
           pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]);
           if( eOp & WO_ISNULL ){
-            /* TUNING: If there is no likelihood() value, assume that a 
-            ** "col IS NULL" expression matches twice as many rows 
+            /* TUNING: If there is no likelihood() value, assume that a
+            ** "col IS NULL" expression matches twice as many rows
             ** as (col=?). */
             pNew->nOut += 10;
           }
@@ -3183,21 +3447,32 @@ static int whereLoopAddBtreeIndex(
       }
     }
 
-    /* Set rCostIdx to the cost of visiting selected rows in index. Add
-    ** it to pNew->rRun, which is currently set to the cost of the index
-    ** seek only. Then, if this is a non-covering index, add the cost of
-    ** visiting the rows in the main table.  */
-    assert( pSrc->pTab->szTabRow>0 );
+    /* Set rCostIdx to the estimated cost of visiting selected rows in the
+    ** index.  The estimate is the sum of two values:
+    **   1.  The cost of doing one search-by-key to find the first matching
+    **       entry
+    **   2.  Stepping forward in the index pNew->nOut times to find all
+    **       additional matching entries.
+    */
+    assert( pSrc->pSTab->szTabRow>0 );
     if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){
       /* The pProbe->szIdxRow is low for an IPK table since the interior
-      ** pages are small.  Thuse szIdxRow gives a good estimate of seek cost.
+      ** pages are small.  Thus szIdxRow gives a good estimate of seek cost.
       ** But the leaf pages are full-size, so pProbe->szIdxRow would badly
       ** under-estimate the scanning cost. */
       rCostIdx = pNew->nOut + 16;
     }else{
-      rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow;
+      rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow;
     }
-    pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx);
+    rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx);
+
+    /* Estimate the cost of running the loop.  If all data is coming
+    ** from the index, then this is just the cost of doing the index
+    ** lookup and scan.  But if some data is coming out of the main table,
+    ** we also have to add in the cost of doing pNew->nOut searches to
+    ** locate the row in the main table that corresponds to the index entry.
+    */
+    pNew->rRun = rCostIdx;
     if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){
       pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16);
     }
@@ -3218,7 +3493,7 @@ static int whereLoopAddBtreeIndex(
     if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
      && pNew->u.btree.nEq<pProbe->nColumn
      && (pNew->u.btree.nEq<pProbe->nKeyCol ||
-           pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
+          pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
     ){
       if( pNew->u.btree.nEq>3 ){
         sqlite3ProgressCheck(pParse);
@@ -3241,12 +3516,12 @@ static int whereLoopAddBtreeIndex(
 
   /* Consider using a skip-scan if there are no WHERE clause constraints
   ** available for the left-most terms of the index, and if the average
-  ** number of repeats in the left-most terms is at least 18. 
+  ** number of repeats in the left-most terms is at least 18.
   **
   ** The magic number 18 is selected on the basis that scanning 17 rows
   ** is almost always quicker than an index seek (even though if the index
   ** contains fewer than 2^17 rows we assume otherwise in other parts of
-  ** the code). And, even if it is not, it should not be too much slower. 
+  ** the code). And, even if it is not, it should not be too much slower.
   ** On the other hand, the extra seeks could end up being significantly
   ** more expensive.  */
   assert( 42==sqlite3LogEst(18) );
@@ -3257,6 +3532,7 @@ static int whereLoopAddBtreeIndex(
    && pProbe->hasStat1!=0
    && OptimizationEnabled(db, SQLITE_SkipScan)
    && pProbe->aiRowLogEst[saved_nEq+1]>=42  /* TUNING: Minimum for skip-scan */
+   && pSrc->fg.fromExists==0
    && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
   ){
     LogEst nIter;
@@ -3303,7 +3579,9 @@ static int indexMightHelpWithOrderBy(
   for(ii=0; ii<pOB->nExpr; ii++){
     Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr);
     if( NEVER(pExpr==0) ) continue;
-    if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){
+    if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) 
+     && pExpr->iTable==iCursor 
+    ){
       if( pExpr->iColumn<0 ) return 1;
       for(jj=0; jj<pIndex->nKeyCol; jj++){
         if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1;
@@ -3339,13 +3617,13 @@ static int whereUsablePartialIndex(
     if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0;
     pWhere = pWhere->pRight;
   }
-  if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0;
   for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
     Expr *pExpr;
     pExpr = pTerm->pExpr;
     if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab)
      && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON))
      && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab)
+     && !sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, -1)
      && (pTerm->wtFlags & TERM_VNULL)==0
     ){
       return 1;
@@ -3393,7 +3671,7 @@ struct CoveringIndexCheck {
 ** all columns less than 63 (really BMS-1) are covered, so we don't need
 ** to check them.  But we do need to check any column at 63 or greater.
 **
-** If the index does not cover the column, then set pWalk->eCode to 
+** If the index does not cover the column, then set pWalk->eCode to
 ** non-zero and return WRC_Abort to stop the search.
 **
 ** If this node does not disprove that the index can be a covering index,
@@ -3497,6 +3775,100 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex(
   return rc;
 }
 
+/*
+** This is an sqlite3ParserAddCleanup() callback that is invoked to
+** free the Parse->pIdxEpr list when the Parse object is destroyed.
+*/
+static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
+  IndexedExpr **pp = (IndexedExpr**)pObject;
+  while( *pp!=0 ){
+    IndexedExpr *p = *pp;
+    *pp = p->pIENext;
+    sqlite3ExprDelete(db, p->pExpr);
+    sqlite3DbFreeNN(db, p);
+  }
+}
+
+/*
+** This function is called for a partial index - one with a WHERE clause - in 
+** two scenarios. In both cases, it determines whether or not the WHERE 
+** clause on the index implies that a column of the table may be safely
+** replaced by a constant expression. For example, in the following 
+** SELECT:
+**
+**   CREATE INDEX i1 ON t1(b, c) WHERE a=<expr>;
+**   SELECT a, b, c FROM t1 WHERE a=<expr> AND b=?;
+**
+** The "a" in the select-list may be replaced by <expr>, iff:
+**
+**    (a) <expr> is a constant expression, and
+**    (b) The (a=<expr>) comparison uses the BINARY collation sequence, and
+**    (c) Column "a" has an affinity other than NONE or BLOB.
+**
+** If argument pItem is NULL, then pMask must not be NULL. In this case this 
+** function is being called as part of determining whether or not pIdx
+** is a covering index. This function clears any bits in (*pMask) 
+** corresponding to columns that may be replaced by constants as described
+** above.
+**
+** Otherwise, if pItem is not NULL, then this function is being called
+** as part of coding a loop that uses index pIdx. In this case, add entries
+** to the Parse.pIdxPartExpr list for each column that can be replaced
+** by a constant.
+*/
+static void wherePartIdxExpr(
+  Parse *pParse,                  /* Parse context */
+  Index *pIdx,                    /* Partial index being processed */
+  Expr *pPart,                    /* WHERE clause being processed */
+  Bitmask *pMask,                 /* Mask to clear bits in */
+  int iIdxCur,                    /* Cursor number for index */
+  SrcItem *pItem                  /* The FROM clause entry for the table */
+){
+  assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 );
+  assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) );
+
+  if( pPart->op==TK_AND ){
+    wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem);
+    pPart = pPart->pLeft;
+  }
+
+  if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){
+    Expr *pLeft = pPart->pLeft;
+    Expr *pRight = pPart->pRight;
+    u8 aff;
+
+    if( pLeft->op!=TK_COLUMN ) return;
+    if( !sqlite3ExprIsConstant(0, pRight) ) return;
+    if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return;
+    if( pLeft->iColumn<0 ) return;
+    aff = pIdx->pTable->aCol[pLeft->iColumn].affinity;
+    if( aff>=SQLITE_AFF_TEXT ){
+      if( pItem ){
+        sqlite3 *db = pParse->db;
+        IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p));
+        if( p ){
+          int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0;
+          p->pExpr = sqlite3ExprDup(db, pRight, 0);
+          p->iDataCur = pItem->iCursor;
+          p->iIdxCur = iIdxCur;
+          p->iIdxCol = pLeft->iColumn;
+          p->bMaybeNullRow = bNullRow;
+          p->pIENext = pParse->pIdxPartExpr;
+          p->aff = aff;
+          pParse->pIdxPartExpr = p;
+          if( p->pIENext==0 ){
+            void *pArg = (void*)&pParse->pIdxPartExpr;
+            sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
+          }
+        }
+      }else if( pLeft->iColumn<(BMS-1) ){
+        *pMask &= ~((Bitmask)1 << pLeft->iColumn);
+      }
+    }
+  }
+}
+
+
 /*
 ** Add all WhereLoop objects for a single table of the join where the table
 ** is identified by pBuilder->pNew->iTab.  That table is guaranteed to be
@@ -3511,18 +3883,18 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex(
 **     cost = nRow * K                      // scan of covering index
 **     cost = nRow * (K+3.0)                // scan of non-covering index
 **
-** where K is a value between 1.1 and 3.0 set based on the relative 
+** where K is a value between 1.1 and 3.0 set based on the relative
 ** estimated average size of the index and table records.
 **
 ** For an index scan, where nVisit is the number of index rows visited
-** by the scan, and nSeek is the number of seek operations required on 
+** by the scan, and nSeek is the number of seek operations required on
 ** the index b-tree:
 **
 **     cost = nSeek * (log(nRow) + K * nVisit)          // covering index
 **     cost = nSeek * (log(nRow) + (K+3.0) * nVisit)    // non-covering index
 **
-** Normally, nSeek is 1. nSeek values greater than 1 come about if the 
-** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when 
+** Normally, nSeek is 1. nSeek values greater than 1 come about if the
+** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when
 ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans.
 **
 ** The estimated values (nRow, nVisit, nSeek) often contain a large amount
@@ -3535,7 +3907,7 @@ static SQLITE_NOINLINE u32 whereIsCoveringIndex(
 */
 static int whereLoopAddBtree(
   WhereLoopBuilder *pBuilder, /* WHERE clause information */
-  Bitmask mPrereq             /* Extra prerequesites for using this table */
+  Bitmask mPrereq             /* Extra prerequisites for using this table */
 ){
   WhereInfo *pWInfo;          /* WHERE analysis context */
   Index *pProbe;              /* An index we are evaluating */
@@ -3551,14 +3923,14 @@ static int whereLoopAddBtree(
   LogEst rSize;               /* number of rows in the table */
   WhereClause *pWC;           /* The parsed WHERE clause */
   Table *pTab;                /* Table being queried */
-  
+ 
   pNew = pBuilder->pNew;
   pWInfo = pBuilder->pWInfo;
   pTabList = pWInfo->pTabList;
   pSrc = pTabList->a + pNew->iTab;
-  pTab = pSrc->pTab;
+  pTab = pSrc->pSTab;
   pWC = pBuilder->pWC;
-  assert( !IsVirtual(pSrc->pTab) );
+  assert( !IsVirtual(pSrc->pSTab) );
 
   if( pSrc->fg.isIndexedBy ){
     assert( pSrc->fg.isCte==0 );
@@ -3583,7 +3955,7 @@ static int whereLoopAddBtree(
     sPk.idxType = SQLITE_IDXTYPE_IPK;
     aiRowEstPk[0] = pTab->nRowLogEst;
     aiRowEstPk[1] = 0;
-    pFirst = pSrc->pTab->pIndex;
+    pFirst = pSrc->pSTab->pIndex;
     if( pSrc->fg.notIndexed==0 ){
       /* The real indices of the table are only considered if the
       ** NOT INDEXED qualifier is omitted from the FROM clause */
@@ -3600,7 +3972,6 @@ static int whereLoopAddBtree(
    && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
    && !pSrc->fg.isIndexedBy  /* Has no INDEXED BY clause */
    && !pSrc->fg.notIndexed   /* Has no NOT INDEXED clause */
-   && HasRowid(pTab)         /* Not WITHOUT ROWID table. (FIXME: Why not?) */
    && !pSrc->fg.isCorrelated /* Not a correlated subquery */
    && !pSrc->fg.isRecursive  /* Not a recursive common table expression. */
    && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */
@@ -3649,9 +4020,9 @@ static int whereLoopAddBtree(
   }
 #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */
 
-  /* Loop over all indices. If there was an INDEXED BY clause, then only 
+  /* Loop over all indices. If there was an INDEXED BY clause, then only
   ** consider index pProbe.  */
-  for(; rc==SQLITE_OK && pProbe; 
+  for(; rc==SQLITE_OK && pProbe;
       pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++
   ){
     if( pProbe->pPartIdxWhere!=0
@@ -3666,6 +4037,7 @@ static int whereLoopAddBtree(
     pNew->u.btree.nEq = 0;
     pNew->u.btree.nBtm = 0;
     pNew->u.btree.nTop = 0;
+    pNew->u.btree.nDistinctCol = 0;
     pNew->nSkip = 0;
     pNew->nLTerm = 0;
     pNew->iSortIdx = 0;
@@ -3673,6 +4045,7 @@ static int whereLoopAddBtree(
     pNew->prereq = mPrereq;
     pNew->nOut = rSize;
     pNew->u.btree.pIndex = pProbe;
+    pNew->u.btree.pOrderBy = 0;
     b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
 
     /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
@@ -3700,11 +4073,12 @@ static int whereLoopAddBtree(
 #else
       pNew->rRun = rSize + 16;
 #endif
-      if( IsView(pTab) || (pTab->tabFlags & TF_Ephemeral)!=0 ){
-        pNew->wsFlags |= WHERE_VIEWSCAN;
-      }
       ApplyCostMultiplier(pNew->rRun, pTab->costMult);
       whereLoopOutputAdjust(pWC, pNew, rSize);
+      if( pSrc->fg.isSubquery ){
+        if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE;
+        pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy;
+      }
       rc = whereLoopInsert(pBuilder, pNew);
       pNew->nOut = rSize;
       if( rc ) break;
@@ -3715,6 +4089,11 @@ static int whereLoopAddBtree(
         pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
       }else{
         m = pSrc->colUsed & pProbe->colNotIdxed;
+        if( pProbe->pPartIdxWhere ){
+          wherePartIdxExpr(
+              pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0
+          );
+        }
         pNew->wsFlags = WHERE_INDEXED;
         if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){
           u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor);
@@ -3737,9 +4116,11 @@ static int whereLoopAddBtree(
                   " according to whereIsCoveringIndex()\n", pProbe->zName));
             }
           }
-        }else if( m==0 ){
+        }else if( m==0 
+           && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700))
+        ){
           WHERETRACE(0x200,
-             ("-> %s a covering index according to bitmasks\n",
+             ("-> %s is a covering index according to bitmasks\n",
              pProbe->zName, m==0 ? "is" : "is not"));
           pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED;
         }
@@ -3788,7 +4169,7 @@ static int whereLoopAddBtree(
               if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
             }
           }
-          
+         
           pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
         }
         ApplyCostMultiplier(pNew->rRun, pTab->costMult);
@@ -3813,7 +4194,7 @@ static int whereLoopAddBtree(
       ** unique index is used (making the index functionally non-unique)
       ** then the sqlite_stat1 data becomes important for scoring the
       ** plan */
-      pTab->tabFlags |= TF_StatsUsed;
+      pTab->tabFlags |= TF_MaybeReanalyze;
     }
 #ifdef SQLITE_ENABLE_STAT4
     sqlite3Stat4ProbeFree(pBuilder->pRec);
@@ -3831,10 +4212,25 @@ static int whereLoopAddBtree(
 */
 static int isLimitTerm(WhereTerm *pTerm){
   assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 );
-  return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT 
+  return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT
       && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET;
 }
 
+/*
+** Return true if the first nCons constraints in the pUsage array are
+** marked as in-use (have argvIndex>0). False otherwise.
+*/
+static int allConstraintsUsed(
+  struct sqlite3_index_constraint_usage *aUsage, 
+  int nCons
+){
+  int ii;
+  for(ii=0; ii<nCons; ii++){
+    if( aUsage[ii].argvIndex<=0 ) return 0;
+  }
+  return 1;
+}
+
 /*
 ** Argument pIdxInfo is already populated with all constraints that may
 ** be used by the virtual table identified by pBuilder->pNew->iTab. This
@@ -3881,13 +4277,13 @@ static int whereLoopAddVirtualOne(
   *pbIn = 0;
   pNew->prereq = mPrereq;
 
-  /* Set the usable flag on the subset of constraints identified by 
+  /* Set the usable flag on the subset of constraints identified by
   ** arguments mUsable and mExclude. */
   pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
   for(i=0; i<nConstraint; i++, pIdxCons++){
-    WhereTerm *pTerm = &pWC->a[pIdxCons->iTermOffset];
+    WhereTerm *pTerm = termFromWhereClause(pWC, pIdxCons->iTermOffset);
     pIdxCons->usable = 0;
-    if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight 
+    if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight
      && (pTerm->eOperator & mExclude)==0
      && (pbRetryLimit || !isLimitTerm(pTerm))
     ){
@@ -3904,11 +4300,10 @@ static int whereLoopAddVirtualOne(
   pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
   pIdxInfo->estimatedRows = 25;
   pIdxInfo->idxFlags = 0;
-  pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
   pHidden->mHandleIn = 0;
 
   /* Invoke the virtual table xBestIndex() method */
-  rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
+  rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo);
   if( rc ){
     if( rc==SQLITE_CONSTRAINT ){
       /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means
@@ -3916,6 +4311,7 @@ static int whereLoopAddVirtualOne(
       ** Make no entries in the loop table.
       */
       WHERETRACE(0xffffffff, ("  ^^^^--- non-viable plan rejected!\n"));
+      freeIdxStr(pIdxInfo);
       return SQLITE_OK;
     }
     return rc;
@@ -3933,18 +4329,17 @@ static int whereLoopAddVirtualOne(
       int j = pIdxCons->iTermOffset;
       if( iTerm>=nConstraint
        || j<0
-       || j>=pWC->nTerm
+       || (pTerm = termFromWhereClause(pWC, j))==0
        || pNew->aLTerm[iTerm]!=0
        || pIdxCons->usable==0
       ){
-        sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
-        testcase( pIdxInfo->needToFreeIdxStr );
+        sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName);
+        freeIdxStr(pIdxInfo);
         return SQLITE_ERROR;
       }
       testcase( iTerm==nConstraint-1 );
       testcase( j==0 );
       testcase( j==pWC->nTerm-1 );
-      pTerm = &pWC->a[j];
       pNew->prereq |= pTerm->prereqRight;
       assert( iTerm<pNew->nLSlot );
       pNew->aLTerm[iTerm] = pTerm;
@@ -3962,7 +4357,7 @@ static int whereLoopAddVirtualOne(
           pNew->u.vtab.bOmitOffset = 1;
         }
       }
-      if( SMASKBIT32(i) & pHidden->mHandleIn ){ 
+      if( SMASKBIT32(i) & pHidden->mHandleIn ){
         pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm);
       }else if( (pTerm->eOperator & WO_IN)!=0 ){
         /* A virtual table that is constrained by an IN clause may not
@@ -3975,18 +4370,21 @@ static int whereLoopAddVirtualOne(
         *pbIn = 1; assert( (mExclude & WO_IN)==0 );
       }
 
+      /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET
+      ** terms. And if there are any, they should follow all other terms. */
       assert( pbRetryLimit || !isLimitTerm(pTerm) );
-      if( isLimitTerm(pTerm) && *pbIn ){
+      assert( !isLimitTerm(pTerm) || i>=nConstraint-2 );
+      assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) );
+
+      if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){
         /* If there is an IN(...) term handled as an == (separate call to
         ** xFilter for each value on the RHS of the IN) and a LIMIT or
-        ** OFFSET term handled as well, the plan is unusable. Set output
-        ** variable *pbRetryLimit to true to tell the caller to retry with
-        ** LIMIT and OFFSET disabled. */
-        if( pIdxInfo->needToFreeIdxStr ){
-          sqlite3_free(pIdxInfo->idxStr);
-          pIdxInfo->idxStr = 0;
-          pIdxInfo->needToFreeIdxStr = 0;
-        }
+        ** OFFSET term handled as well, the plan is unusable. Similarly,
+        ** if there is a LIMIT/OFFSET and there are other unused terms,
+        ** the plan cannot be used. In these cases set variable *pbRetryLimit
+        ** to true to tell the caller to retry with LIMIT and OFFSET 
+        ** disabled. */
+        freeIdxStr(pIdxInfo);
         *pbRetryLimit = 1;
         return SQLITE_OK;
       }
@@ -3998,8 +4396,8 @@ static int whereLoopAddVirtualOne(
     if( pNew->aLTerm[i]==0 ){
       /* The non-zero argvIdx values must be contiguous.  Raise an
       ** error if they are not */
-      sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
-      testcase( pIdxInfo->needToFreeIdxStr );
+      sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName);
+      freeIdxStr(pIdxInfo);
       return SQLITE_ERROR;
     }
   }
@@ -4010,6 +4408,7 @@ static int whereLoopAddVirtualOne(
   pNew->u.vtab.idxStr = pIdxInfo->idxStr;
   pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
       pIdxInfo->nOrderBy : 0);
+  pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0;
   pNew->rSetup = 0;
   pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
   pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
@@ -4042,7 +4441,7 @@ static int whereLoopAddVirtualOne(
 **
 ** Return a pointer to the collation name:
 **
-**    1. If there is an explicit COLLATE operator on the constaint, return it.
+**    1. If there is an explicit COLLATE operator on the constraint, return it.
 **
 **    2. Else, if the column has an alternative collation, return that.
 **
@@ -4054,7 +4453,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){
   if( iCons>=0 && iCons<pIdxInfo->nConstraint ){
     CollSeq *pC = 0;
     int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset;
-    Expr *pX = pHidden->pWC->a[iTerm].pExpr;
+    Expr *pX = termFromWhereClause(pHidden->pWC, iTerm)->pExpr;
     if( pX->pLeft ){
       pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX);
     }
@@ -4072,7 +4471,7 @@ int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
   HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
   u32 m = SMASKBIT32(iCons);
   if( m & pHidden->mIn ){
-    if( bHandle==0 ){ 
+    if( bHandle==0 ){
       pHidden->mHandleIn &= ~m;
     }else if( bHandle>0 ){
       pHidden->mHandleIn |= m;
@@ -4085,7 +4484,7 @@ int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){
 /*
 ** This interface is callable from within the xBestIndex callback only.
 **
-** If possible, set (*ppVal) to point to an object containing the value 
+** If possible, set (*ppVal) to point to an object containing the value
 ** on the right-hand-side of constraint iCons.
 */
 int sqlite3_vtab_rhs_value(
@@ -4097,10 +4496,12 @@ int sqlite3_vtab_rhs_value(
   sqlite3_value *pVal = 0;
   int rc = SQLITE_OK;
   if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
-    rc = SQLITE_MISUSE; /* EV: R-30545-25046 */
+    rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */
   }else{
     if( pH->aRhs[iCons]==0 ){
-      WhereTerm *pTerm = &pH->pWC->a[pIdxInfo->aConstraint[iCons].iTermOffset];
+      WhereTerm *pTerm = termFromWhereClause(
+          pH->pWC, pIdxInfo->aConstraint[iCons].iTermOffset
+      );
       rc = sqlite3ValueFromExpr(
           pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db),
           SQLITE_AFF_BLOB, &pH->aRhs[iCons]
@@ -4158,8 +4559,8 @@ void sqlite3VtabUsesAllSchemas(Parse *pParse){
 ** entries that occur before the virtual table in the FROM clause and are
 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
 ** mUnusable mask contains all FROM clause entries that occur after the
-** virtual table and are separated from it by at least one LEFT or 
-** CROSS JOIN. 
+** virtual table and are separated from it by at least one LEFT or
+** CROSS JOIN.
 **
 ** For example, if the query were:
 **
@@ -4167,9 +4568,9 @@ void sqlite3VtabUsesAllSchemas(Parse *pParse){
 **
 ** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
 **
-** All the tables in mPrereq must be scanned before the current virtual 
-** table. So any terms for which all prerequisites are satisfied by 
-** mPrereq may be specified as "usable" in all calls to xBestIndex. 
+** All the tables in mPrereq must be scanned before the current virtual
+** table. So any terms for which all prerequisites are satisfied by
+** mPrereq may be specified as "usable" in all calls to xBestIndex.
 ** Conversely, all tables in mUnusable must be scanned after the current
 ** virtual table, so any terms for which the prerequisites overlap with
 ** mUnusable should always be configured as "not-usable" for xBestIndex.
@@ -4198,7 +4599,7 @@ static int whereLoopAddVirtual(
   pWC = pBuilder->pWC;
   pNew = pBuilder->pNew;
   pSrc = &pWInfo->pTabList->a[pNew->iTab];
-  assert( IsVirtual(pSrc->pTab) );
+  assert( IsVirtual(pSrc->pSTab) );
   p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit);
   if( p==0 ) return SQLITE_NOMEM_BKPT;
   pNew->rSetup = 0;
@@ -4212,7 +4613,7 @@ static int whereLoopAddVirtual(
   }
 
   /* First call xBestIndex() with all constraints usable. */
-  WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName));
+  WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName));
   WHERETRACE(0x800, ("  VirtualOne: all usable\n"));
   rc = whereLoopAddVirtualOne(
       pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry
@@ -4226,7 +4627,7 @@ static int whereLoopAddVirtual(
 
   /* If the call to xBestIndex() with all terms enabled produced a plan
   ** that does not require any source tables (IOW: a plan with mBest==0)
-  ** and does not use an IN(...) operator, then there is no point in making 
+  ** and does not use an IN(...) operator, then there is no point in making
   ** any further calls to xBestIndex() since they will all return the same
   ** result (if the xBestIndex() implementation is sane). */
   if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){
@@ -4249,16 +4650,15 @@ static int whereLoopAddVirtual(
       }
     }
 
-    /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) 
+    /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq)
     ** in the set of terms that apply to the current virtual table.  */
     while( rc==SQLITE_OK ){
       int i;
       Bitmask mNext = ALLBITS;
       assert( mNext>0 );
       for(i=0; i<nConstraint; i++){
-        Bitmask mThis = (
-            pWC->a[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
-        );
+        int iTerm = p->aConstraint[i].iTermOffset;
+        Bitmask mThis = termFromWhereClause(pWC, iTerm)->prereqRight & ~mPrereq;
         if( mThis>mPrev && mThis<mNext ) mNext = mThis;
       }
       mPrev = mNext;
@@ -4294,9 +4694,8 @@ static int whereLoopAddVirtual(
     }
   }
 
-  if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
   freeIndexInfo(pParse->db, p);
-  WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc));
+  WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc));
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -4306,8 +4705,8 @@ static int whereLoopAddVirtual(
 ** btrees or virtual tables.
 */
 static int whereLoopAddOr(
-  WhereLoopBuilder *pBuilder, 
-  Bitmask mPrereq, 
+  WhereLoopBuilder *pBuilder,
+  Bitmask mPrereq,
   Bitmask mUnusable
 ){
   WhereInfo *pWInfo = pBuilder->pWInfo;
@@ -4320,7 +4719,7 @@ static int whereLoopAddOr(
   WhereLoopBuilder sSubBuild;
   WhereOrSet sSum, sCur;
   SrcItem *pItem;
-  
+ 
   pWC = pBuilder->pWC;
   pWCEnd = pWC->a + pWC->nTerm;
   pNew = pBuilder->pNew;
@@ -4333,14 +4732,14 @@ static int whereLoopAddOr(
 
   for(pTerm=pWC->a; pTerm<pWCEnd && rc==SQLITE_OK; pTerm++){
     if( (pTerm->eOperator & WO_OR)!=0
-     && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 
+     && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0
     ){
       WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
       WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
       WhereTerm *pOrTerm;
       int once = 1;
       int i, j;
-    
+   
       sSubBuild = *pBuilder;
       sSubBuild.pOrSet = &sCur;
 
@@ -4361,14 +4760,14 @@ static int whereLoopAddOr(
         }
         sCur.n = 0;
 #ifdef WHERETRACE_ENABLED
-        WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", 
+        WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n",
                    (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
         if( sqlite3WhereTrace & 0x20000 ){
           sqlite3WhereClausePrint(sSubBuild.pWC);
         }
 #endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-        if( IsVirtual(pItem->pTab) ){
+        if( IsVirtual(pItem->pSTab) ){
           rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
         }else
 #endif
@@ -4409,8 +4808,8 @@ static int whereLoopAddOr(
         /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs
         ** of all sub-scans required by the OR-scan. However, due to rounding
         ** errors, it may be that the cost of the OR-scan is equal to its
-        ** most expensive sub-scan. Add the smallest possible penalty 
-        ** (equivalent to multiplying the cost by 1.07) to ensure that 
+        ** most expensive sub-scan. Add the smallest possible penalty
+        ** (equivalent to multiplying the cost by 1.07) to ensure that
         ** this does not happen. Otherwise, for WHERE clauses such as the
         ** following where there is an index on "y":
         **
@@ -4430,7 +4829,7 @@ static int whereLoopAddOr(
 }
 
 /*
-** Add all WhereLoop objects for all tables 
+** Add all WhereLoop objects for all tables
 */
 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
   WhereInfo *pWInfo = pBuilder->pWInfo;
@@ -4462,7 +4861,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
     pNew->iTab = iTab;
     pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR;
     pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor);
-    if( bFirstPastRJ 
+    if( bFirstPastRJ
      || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0
     ){
       /* Add prerequisites to prevent reordering of FROM clause terms
@@ -4482,7 +4881,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
       mPrereq = 0;
     }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-    if( IsVirtual(pItem->pTab) ){
+    if( IsVirtual(pItem->pSTab) ){
       SrcItem *p;
       for(p=&pItem[1]; p<pEnd; p++){
         if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){
@@ -4514,21 +4913,112 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
   return rc;
 }
 
+/* Implementation of the order-by-subquery optimization:
+**
+** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really
+** a subquery or CTE that has an ORDER BY clause.  See if any of the terms
+** in the subquery ORDER BY clause will satisfy pOrderBy from the outer
+** query.  Mark off all satisfied terms (by setting bits in *pOBSat) and
+** return TRUE if they do.  If not, return false.
+**
+** Example:
+**
+**    CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b));
+**    CREATE TABLE t2(x,y);
+**    WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y)
+**       SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b;
+**
+** The CTE named "t3" comes out in the natural order of "p", so the first
+** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3"
+** and sorting only needs to occur on the second term "b".
+**
+** Limitations:
+**
+** (1)  The optimization is not applied if the outer ORDER BY contains
+**      a COLLATE clause.  The optimization might be applied if the
+**      outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as
+**      long as the subquery ORDER BY does the same.  But if the
+**      outer ORDER BY uses COLLATE, even a redundant COLLATE, the
+**      optimization is bypassed.
+**
+** (2)  The subquery ORDER BY terms must exactly match subquery result
+**      columns, including any COLLATE annotations.  This routine relies
+**      on iOrderByCol to do matching between order by terms and result
+**      columns, and iOrderByCol will not be set if the result column
+**      and ORDER BY collations differ.
+**
+** (3)  The subquery and outer ORDER BY can be in opposite directions as
+**      long as  the subquery is materialized.  If the subquery is
+**      implemented as a co-routine, the sort orders must be in the same
+**      direction because there is no way to run a co-routine backwards.
+*/
+static SQLITE_NOINLINE int wherePathMatchSubqueryOB(
+  WhereInfo *pWInfo,      /* The WHERE clause */
+  WhereLoop *pLoop,       /* The nested loop term that is a subquery */
+  int iLoop,              /* Which level of the nested loop.  0==outermost */
+  int iCur,               /* Cursor used by the this loop */
+  ExprList *pOrderBy,     /* The ORDER BY clause on the whole query */
+  Bitmask *pRevMask,      /* When loops need to go in reverse order */
+  Bitmask *pOBSat         /* Which terms of pOrderBy are satisfied so far */
+){
+  int iOB;                /* Index into pOrderBy->a[] */
+  int jSub;               /* Index into pSubOB->a[] */
+  u8 rev = 0;             /* True if iOB and jSub sort in opposite directions */
+  u8 revIdx = 0;          /* Sort direction for jSub */
+  Expr *pOBExpr;          /* Current term of outer ORDER BY */
+  ExprList *pSubOB;       /* Complete ORDER BY on the subquery */
+
+  pSubOB = pLoop->u.btree.pOrderBy;
+  assert( pSubOB!=0 );
+  for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){}
+  for(jSub=0; jSub<pSubOB->nExpr && iOB<pOrderBy->nExpr; jSub++, iOB++){
+    if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break;
+    pOBExpr = pOrderBy->a[iOB].pExpr;
+    if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break;
+    if( pOBExpr->iTable!=iCur ) break;
+    if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break;
+    if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){
+      u8 sfOB = pOrderBy->a[iOB].fg.sortFlags;   /* sortFlags for iOB */
+      u8 sfSub = pSubOB->a[jSub].fg.sortFlags;   /* sortFlags for jSub */
+      if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){
+        break;
+      }
+      revIdx = sfSub & KEYINFO_ORDER_DESC;
+      if( jSub>0 ){
+        if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){
+          break;
+        }
+      }else{
+        rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC);
+        if( rev ){
+          if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){
+            /* Cannot run a co-routine in reverse order */
+            break;
+          }
+          *pRevMask |= MASKBIT(iLoop);
+        }
+      }
+    }
+    *pOBSat |= MASKBIT(iOB);
+  }
+  return jSub>0;
+}
+
 /*
 ** Examine a WherePath (with the addition of the extra WhereLoop of the 6th
 ** parameters) to see if it outputs rows in the requested ORDER BY
 ** (or GROUP BY) without requiring a separate sort operation.  Return N:
-** 
+**
 **   N>0:   N terms of the ORDER BY clause are satisfied
 **   N==0:  No terms of the ORDER BY clause are satisfied
-**   N<0:   Unknown yet how many terms of ORDER BY might be satisfied.   
+**   N<0:   Unknown yet how many terms of ORDER BY might be satisfied.  
 **
 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
 ** strict.  With GROUP BY and DISTINCT the only requirement is that
 ** equivalent rows appear immediately adjacent to one another.  GROUP BY
 ** and DISTINCT do not require rows to appear in any particular order as long
 ** as equivalent rows are grouped together.  Thus for GROUP BY and DISTINCT
-** the pOrderBy terms can be matched in any order.  With ORDER BY, the 
+** the pOrderBy terms can be matched in any order.  With ORDER BY, the
 ** pOrderBy terms must be matched in strict left-to-right order.
 */
 static i8 wherePathSatisfiesOrderBy(
@@ -4578,7 +5068,7 @@ static i8 wherePathSatisfiesOrderBy(
   ** row of the WhereLoop.  Every one-row WhereLoop is automatically
   ** order-distinct.   A WhereLoop that has no columns in the ORDER BY clause
   ** is not order-distinct. To be order-distinct is not quite the same as being
-  ** UNIQUE since a UNIQUE column or index can have multiple rows that 
+  ** UNIQUE since a UNIQUE column or index can have multiple rows that
   ** are NULL and NULL values are equivalent for the purpose of order-distinct.
   ** To be order-distinct, the columns must be UNIQUE and NOT NULL.
   **
@@ -4610,14 +5100,16 @@ static i8 wherePathSatisfiesOrderBy(
       pLoop = pLast;
     }
     if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
-      if( pLoop->u.vtab.isOrdered 
+      if( pLoop->u.vtab.isOrdered
        && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY)
       ){
         obSat = obDone;
+      }else{
+        /* No further ORDER BY terms may be matched. So this call should
+        ** return >=0, not -1. Clear isOrderDistinct to ensure it does so. */
+        isOrderDistinct = 0;
       }
       break;
-    }else if( wctrlFlags & WHERE_DISTINCTBY ){
-      pLoop->u.btree.nDistinctCol = 0;
     }
     iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor;
 
@@ -4636,10 +5128,10 @@ static i8 wherePathSatisfiesOrderBy(
                        ~ready, eqOpMask, 0);
       if( pTerm==0 ) continue;
       if( pTerm->eOperator==WO_IN ){
-        /* IN terms are only valid for sorting in the ORDER BY LIMIT 
+        /* IN terms are only valid for sorting in the ORDER BY LIMIT
         ** optimization, and then only if they are actually used
         ** by the query plan */
-        assert( wctrlFlags & 
+        assert( wctrlFlags &
                (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) );
         for(j=0; j<pLoop->nLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
         if( j>=pLoop->nLTerm ) continue;
@@ -4659,9 +5151,18 @@ static i8 wherePathSatisfiesOrderBy(
 
     if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){
       if( pLoop->wsFlags & WHERE_IPK ){
+        if( pLoop->u.btree.pOrderBy
+         && OptimizationEnabled(db, SQLITE_OrderBySubq)
+         &&  wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur,
+                                     pOrderBy,pRevMask, &obSat)
+        ){
+          nColumn = 0;
+          isOrderDistinct = 0;
+        }else{
+          nColumn = 1;
+        }
         pIndex = 0;
         nKeyCol = 0;
-        nColumn = 1;
       }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){
         return 0;
       }else{
@@ -4671,7 +5172,7 @@ static i8 wherePathSatisfiesOrderBy(
         assert( pIndex->aiColumn[nColumn-1]==XN_ROWID
                           || !HasRowid(pIndex->pTable));
         /* All relevant terms of the index must also be non-NULL in order
-        ** for isOrderDistinct to be true.  So the isOrderDistint value
+        ** for isOrderDistinct to be true.  So the isOrderDistinct value
         ** computed here might be a false positive.  Corrections will be
         ** made at tag-20210426-1 below */
         isOrderDistinct = IsUniqueIndex(pIndex)
@@ -4686,7 +5187,7 @@ static i8 wherePathSatisfiesOrderBy(
       for(j=0; j<nColumn; j++){
         u8 bOnce = 1; /* True to run the ORDER BY search loop */
 
-        assert( j>=pLoop->u.btree.nEq 
+        assert( j>=pLoop->u.btree.nEq
             || (pLoop->aLTerm[j]==0)==(j<pLoop->nSkip)
         );
         if( j<pLoop->u.btree.nEq && j>=pLoop->nSkip ){
@@ -4698,7 +5199,7 @@ static i8 wherePathSatisfiesOrderBy(
           ** the loop need to be marked as not order-distinct because it can
           ** have repeated NULL rows.
           **
-          ** If the current term is a column of an ((?,?) IN (SELECT...)) 
+          ** If the current term is a column of an ((?,?) IN (SELECT...))
           ** expression for which the SELECT returns more than one column,
           ** check that it is the only column used by this loop. Otherwise,
           ** if it is one of two or more, none of the columns can be
@@ -4711,7 +5212,7 @@ static i8 wherePathSatisfiesOrderBy(
               testcase( isOrderDistinct );
               isOrderDistinct = 0;
             }
-            continue;  
+            continue; 
           }else if( ALWAYS(eOp & WO_IN) ){
             /* ALWAYS() justification: eOp is an equality operator due to the
             ** j<pLoop->u.btree.nEq constraint above.  Any equality other
@@ -4753,10 +5254,10 @@ static i8 wherePathSatisfiesOrderBy(
           if( iColumn==XN_EXPR ){
             isOrderDistinct = 0;
           }
-        } 
+        }
 
         /* Find the ORDER BY term that corresponds to the j-th column
-        ** of the index and mark that ORDER BY term off 
+        ** of the index and mark that ORDER BY term having been satisfied.
         */
         isMatch = 0;
         for(i=0; bOnce && i<nOrderBy; i++){
@@ -4790,7 +5291,7 @@ static i8 wherePathSatisfiesOrderBy(
           /* Make sure the sort order is compatible in an ORDER BY clause.
           ** Sort order is irrelevant for a GROUP BY clause. */
           if( revSet ){
-            if( (rev ^ revIdx) 
+            if( (rev ^ revIdx)
                            != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC)
             ){
               isMatch = 0;
@@ -4838,7 +5339,7 @@ static i8 wherePathSatisfiesOrderBy(
         if( MASKBIT(i) & obSat ) continue;
         p = pOrderBy->a[i].pExpr;
         mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p);
-        if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue;
+        if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue;
         if( (mTerm&~orderDistinctMask)==0 ){
           obSat |= MASKBIT(i);
         }
@@ -4899,7 +5400,7 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){
 #endif
 
 /*
-** Return the cost of sorting nRow rows, assuming that the keys have 
+** Return the cost of sorting nRow rows, assuming that the keys have
 ** nOrderby columns and that the first nSorted columns are already in
 ** order.
 */
@@ -4909,13 +5410,13 @@ static LogEst whereSortingCost(
   int nOrderBy,      /* Number of ORDER BY clause terms */
   int nSorted        /* Number of initial ORDER BY terms naturally in order */
 ){
-  /* Estimated cost of a full external sort, where N is 
+  /* Estimated cost of a full external sort, where N is
   ** the number of rows to sort is:
   **
   **   cost = (K * N * log(N)).
-  ** 
-  ** Or, if the order-by clause has X terms but only the last Y 
-  ** terms are out of order, then block-sorting will reduce the 
+  **
+  ** Or, if the order-by clause has X terms but only the last Y
+  ** terms are out of order, then block-sorting will reduce the
   ** sorting cost to:
   **
   **   cost = (K * N * log(N)) * (Y/X)
@@ -4923,8 +5424,8 @@ static LogEst whereSortingCost(
   ** The constant K is at least 2.0 but will be larger if there are a
   ** large number of columns to be sorted, as the sorting time is
   ** proportional to the amount of content to be sorted.  The algorithm
-  ** does not currently distinguish between fat columns (BLOBs and TEXTs) 
-  ** and skinny columns (INTs).  It just uses the number of columns as 
+  ** does not currently distinguish between fat columns (BLOBs and TEXTs)
+  ** and skinny columns (INTs).  It just uses the number of columns as
   ** an approximation for the row width.
   **
   ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort
@@ -4963,6 +5464,216 @@ static LogEst whereSortingCost(
   return rSortCost;
 }
 
+/*
+** Compute the maximum number of paths in the solver algorithm, for
+** queries that have three or more terms in the FROM clause.  Queries with
+** two or fewer FROM clause terms are handled by the caller.
+**
+** Query planning is NP-hard.  We must limit the number of paths at
+** each step of the solver search algorithm to avoid exponential behavior.
+**
+** The value returned is a tuning parameter.  Currently the value is:
+**
+**     18    for star queries
+**     12    otherwise
+**
+** For the purposes of this heuristic, a star-query is defined as a query
+** with a large central table that is joined using an INNER JOIN,
+** not CROSS or OUTER JOINs, against four or more smaller tables.
+** The central table is called the "fact" table.  The smaller tables
+** that get joined are "dimension tables".  Also, any table that is
+** self-joined cannot be a dimension table; we assume that dimension
+** tables may only be joined against fact tables.
+**
+** SIDE EFFECT:  (and really the whole point of this subroutine)
+**
+** If pWInfo describes a star-query, then the cost for SCANs of dimension
+** WhereLoops is increased to be slightly larger than the cost of a SCAN
+** in the fact table.  Only SCAN costs are increased.  SEARCH costs are
+** unchanged. This heuristic helps keep fact tables in outer loops. Without
+** this heuristic, paths with fact tables in outer loops tend to get pruned
+** by the mxChoice limit on the number of paths, resulting in poor query
+** plans.  See the starschema1.test test module for examples of queries
+** that need this heuristic to find good query plans.
+**
+** This heuristic can be completely disabled, so that no query is
+** considered a star-query, using SQLITE_TESTCTRL_OPTIMIZATION to
+** disable the SQLITE_StarQuery optimization.  In the CLI, the command
+** to do that is:  ".testctrl opt -starquery".
+**
+** HISTORICAL NOTES:
+**
+** This optimization was first added on 2024-05-09 by check-in 38db9b5c83d.
+** The original optimization reduced the cost and output size estimate for
+** fact tables to help them move to outer loops.  But months later (as people
+** started upgrading) performance regression reports started caming in,
+** including:
+**
+**    forum post b18ef983e68d06d1 (2024-12-21)
+**    forum post 0025389d0860af82 (2025-01-14)
+**    forum post d87570a145599033 (2025-01-17)
+**
+** To address these, the criteria for a star-query was tightened to exclude
+** cases where the fact and dimensions are separated by an outer join, and
+** the affect of star-schema detection was changed to increase the rRun cost
+** on just full table scans of dimension tables, rather than reducing costs
+** in the all access methods of the fact table.
+*/
+static int computeMxChoice(WhereInfo *pWInfo){
+  int nLoop = pWInfo->nLevel;    /* Number of terms in the join */
+  WhereLoop *pWLoop;             /* For looping over WhereLoops */
+
+#ifdef SQLITE_DEBUG
+  /* The star-query detection code below makes use of the following
+  ** properties of the WhereLoop list, so verify them before
+  ** continuing:
+  **    (1)  .maskSelf is the bitmask corresponding to .iTab
+  **    (2)  The WhereLoop list is in ascending .iTab order
+  */
+  for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
+    assert( pWLoop->maskSelf==MASKBIT(pWLoop->iTab) );
+    assert( pWLoop->pNextLoop==0 || pWLoop->iTab<=pWLoop->pNextLoop->iTab );
+  }
+#endif /* SQLITE_DEBUG */
+
+  if( nLoop>=5
+   && !pWInfo->bStarDone
+   && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery)
+  ){
+    SrcItem *aFromTabs;    /* All terms of the FROM clause */
+    int iFromIdx;          /* Term of FROM clause is the candidate fact-table */
+    Bitmask m;             /* Bitmask for candidate fact-table */
+    Bitmask mSelfJoin = 0; /* Tables that cannot be dimension tables */       
+    WhereLoop *pStart;     /* Where to start searching for dimension-tables */
+
+    pWInfo->bStarDone = 1; /* Only do this computation once */
+
+    /* Look for fact tables with four or more dimensions where the
+    ** dimension tables are not separately from the fact tables by an outer
+    ** or cross join.  Adjust cost weights if found.
+    */
+    assert( !pWInfo->bStarUsed );
+    aFromTabs = pWInfo->pTabList->a;
+    pStart = pWInfo->pLoops;
+    for(iFromIdx=0, m=1; iFromIdx<nLoop; iFromIdx++, m<<=1){
+      int nDep = 0;             /* Number of dimension tables */
+      LogEst mxRun;             /* Maximum SCAN cost of a fact table */
+      Bitmask mSeen = 0;        /* Mask of dimension tables */
+      SrcItem *pFactTab;        /* The candidate fact table */
+
+      pFactTab = aFromTabs + iFromIdx;
+      if( (pFactTab->fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){
+        /* If the candidate fact-table is the right table of an outer join
+        ** restrict the search for dimension-tables to be tables to the right
+        ** of the fact-table. */
+        if( iFromIdx+4 > nLoop ) break;  /* Impossible to reach nDep>=4 */
+        while( pStart && pStart->iTab<=iFromIdx ){
+          pStart = pStart->pNextLoop;
+        }
+      }
+      for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){
+        if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){
+          /* Fact-tables and dimension-tables cannot be separated by an
+          ** outer join (at least for the definition of fact- and dimension-
+          ** used by this heuristic). */
+          break;
+        }
+        if( (pWLoop->prereq & m)!=0        /* pWInfo depends on iFromIdx */
+         && (pWLoop->maskSelf & mSeen)==0  /* pWInfo not already a dependency */
+         && (pWLoop->maskSelf & mSelfJoin)==0 /* Not a self-join */
+        ){
+          if( aFromTabs[pWLoop->iTab].pSTab==pFactTab->pSTab ){
+            mSelfJoin |= m;
+          }else{
+            nDep++;
+            mSeen |= pWLoop->maskSelf;
+          }
+        }
+      }
+      if( nDep<=3 ) continue;
+
+      /* If we reach this point, it means that pFactTab is a fact table
+      ** with four or more dimensions connected by inner joins.  Proceed
+      ** to make cost adjustments. */
+     
+#ifdef WHERETRACE_ENABLED
+      /* Make sure rStarDelta values are initialized */
+      if( !pWInfo->bStarUsed ){
+        for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
+          pWLoop->rStarDelta = 0;
+        }
+      }
+#endif
+      pWInfo->bStarUsed = 1;
+
+      /* Compute the maximum cost of any WhereLoop for the
+      ** fact table plus one epsilon */
+      mxRun = LOGEST_MIN;
+      for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){
+        if( pWLoop->iTab<iFromIdx ) continue;
+        if( pWLoop->iTab>iFromIdx ) break;
+        if( pWLoop->rRun>mxRun ) mxRun = pWLoop->rRun;
+      }
+      if( ALWAYS(mxRun<LOGEST_MAX) ) mxRun++;
+
+      /* Increase the cost of table scans for dimension tables to be
+      ** slightly more than the maximum cost of the fact table */
+      for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){
+        if( (pWLoop->maskSelf & mSeen)==0 ) continue;
+        if( pWLoop->nLTerm ) continue;
+        if( pWLoop->rRun<mxRun ){
+#ifdef WHERETRACE_ENABLED /* 0x80000 */
+          if( sqlite3WhereTrace & 0x80000 ){
+            SrcItem *pDim = aFromTabs + pWLoop->iTab;
+            sqlite3DebugPrintf(
+              "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n",
+              pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab,
+              pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName,
+              iFromIdx, mxRun
+            );
+          }
+          pWLoop->rStarDelta = mxRun - pWLoop->rRun;
+#endif /* WHERETRACE_ENABLED */
+          pWLoop->rRun = mxRun;
+        }
+      }
+    }
+#ifdef WHERETRACE_ENABLED /* 0x80000 */
+    if( (sqlite3WhereTrace & 0x80000)!=0 && pWInfo->bStarUsed ){
+      sqlite3DebugPrintf("WhereLoops changed by star-query heuristic:\n");
+      for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
+        if( pWLoop->rStarDelta ){
+          sqlite3WhereLoopPrint(pWLoop, &pWInfo->sWC);
+        }
+      }
+    }
+#endif
+  }
+  return pWInfo->bStarUsed ? 18 : 12;
+}
+
+/*
+** Two WhereLoop objects, pCandidate and pBaseline, are known to have the
+** same cost.  Look deep into each to see if pCandidate is even slightly
+** better than pBaseline.  Return false if it is, if pCandidate is is preferred.
+** Return true if pBaseline is preferred or if we cannot tell the difference.
+**
+**    Result       Meaning
+**    --------     ----------------------------------------------------------
+**    true         We cannot tell the difference in pCandidate and pBaseline
+**    false        pCandidate seems like a better choice than pBaseline
+*/
+static SQLITE_NOINLINE int whereLoopIsNoBetter(
+  const WhereLoop *pCandidate,
+  const WhereLoop *pBaseline
+){
+  if( (pCandidate->wsFlags & WHERE_INDEXED)==0 ) return 1;
+  if( (pBaseline->wsFlags & WHERE_INDEXED)==0 ) return 1;
+  if( pCandidate->u.btree.pIndex->szIdxRow <
+        pBaseline->u.btree.pIndex->szIdxRow ) return 0;
+  return 1;
+}
+
 /*
 ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine
 ** attempts to find the lowest cost path that visits each WhereLoop
@@ -4984,7 +5695,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
   int mxI = 0;              /* Index of next entry to replace */
   int nOrderBy;             /* Number of ORDER BY clause terms */
   LogEst mxCost = 0;        /* Maximum cost of a set of paths */
-  LogEst mxUnsorted = 0;    /* Maximum unsorted cost of a set of path */
+  LogEst mxUnsort = 0;      /* Maximum unsorted cost of a set of path */
   int nTo, nFrom;           /* Number of valid entries in aTo[] and aFrom[] */
   WherePath *aFrom;         /* All nFrom paths at the previous level */
   WherePath *aTo;           /* The nTo best paths at the current level */
@@ -4998,12 +5709,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
 
   pParse = pWInfo->pParse;
   nLoop = pWInfo->nLevel;
-  /* TUNING: For simple queries, only the best path is tracked.
-  ** For 2-way joins, the 5 best paths are followed.
-  ** For joins of 3 or more tables, track the 10 best paths */
-  mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10);
+  WHERETRACE(0x002, ("---- begin solver.  (nRowEst=%d, nQueryLoop=%d)\n",
+                     nRowEst, pParse->nQueryLoop));
+  /* TUNING: mxChoice is the maximum number of possible paths to preserve
+  ** at each step.  Based on the number of loops in the FROM clause:
+  **
+  **     nLoop      mxChoice
+  **     -----      --------
+  **       1            1            // the most common case
+  **       2            5
+  **       3+        12 or 18        // see computeMxChoice()
+  */
+  if( nLoop<=1 ){
+    mxChoice = 1;
+  }else if( nLoop==2 ){
+    mxChoice = 5;
+  }else if( pParse->nErr ){
+    mxChoice = 1;
+  }else{
+    mxChoice = computeMxChoice(pWInfo);
+  }
   assert( nLoop<=pWInfo->pTabList->nSrc );
-  WHERETRACE(0x002, ("---- begin solver.  (nRowEst=%d)\n", nRowEst));
 
   /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this
   ** case the purpose of this call is to estimate the number of rows returned
@@ -5033,7 +5759,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
     ** space for the aSortCost[] array. Each element of the aSortCost array
     ** is either zero - meaning it has not yet been initialized - or the
     ** cost of sorting nRowEst rows of data where the first X terms of
-    ** the ORDER BY clause are already in order, where X is the array 
+    ** the ORDER BY clause are already in order, where X is the array
     ** index.  */
     aSortCost = (LogEst*)pX;
     memset(aSortCost, 0, sizeof(LogEst) * nOrderBy);
@@ -5054,7 +5780,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
     ** in this case the query may return a maximum of one row, the results
     ** are already in the requested order. Set isOrdered to nOrderBy to
     ** indicate this. Or, if nLoop is greater than zero, set isOrdered to
-    ** -1, indicating that the result set may or may not be ordered, 
+    ** -1, indicating that the result set may or may not be ordered,
     ** depending on the loops added to the current plan.  */
     aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy;
   }
@@ -5068,7 +5794,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
         LogEst nOut;                      /* Rows visited by (pFrom+pWLoop) */
         LogEst rCost;                     /* Cost of path (pFrom+pWLoop) */
-        LogEst rUnsorted;                 /* Unsorted cost of (pFrom+pWLoop) */
+        LogEst rUnsort;                   /* Unsorted cost of (pFrom+pWLoop) */
         i8 isOrdered;                     /* isOrdered for (pFrom+pWLoop) */
         Bitmask maskNew;                  /* Mask of src visited by (..) */
         Bitmask revMask;                  /* Mask of rev-order loops for (..) */
@@ -5084,10 +5810,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
           continue;
         }
 
-        /* At this point, pWLoop is a candidate to be the next loop. 
+        /* At this point, pWLoop is a candidate to be the next loop.
         ** Compute its cost */
-        rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
-        rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted);
+        rUnsort = pWLoop->rRun + pFrom->nRow;
+        if( pWLoop->rSetup ){
+          rUnsort = sqlite3LogEstAdd(pWLoop->rSetup, rUnsort);
+        }
+        rUnsort = sqlite3LogEstAdd(rUnsort, pFrom->rUnsort);
         nOut = pFrom->nRow + pWLoop->nOut;
         maskNew = pFrom->maskLoop | pWLoop->maskSelf;
         isOrdered = pFrom->isOrdered;
@@ -5106,41 +5835,42 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
             );
           }
           /* TUNING:  Add a small extra penalty (3) to sorting as an
-          ** extra encouragment to the query planner to select a plan
+          ** extra encouragement to the query planner to select a plan
           ** where the rows emerge in the correct order without any sorting
           ** required. */
-          rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3;
+          rCost = sqlite3LogEstAdd(rUnsort, aSortCost[isOrdered]) + 3;
 
           WHERETRACE(0x002,
               ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n",
-               aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, 
-               rUnsorted, rCost));
+               aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy,
+               rUnsort, rCost));
         }else{
-          rCost = rUnsorted;
-          rUnsorted -= 2;  /* TUNING:  Slight bias in favor of no-sort plans */
-        }
-
-        /* TUNING:  A full-scan of a VIEW or subquery in the outer loop
-        ** is not so bad. */
-        if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){
-          rCost += -10;
-          nOut += -30;
+          rCost = rUnsort;
+          rUnsort -= 2;  /* TUNING:  Slight bias in favor of no-sort plans */
         }
 
         /* Check to see if pWLoop should be added to the set of
         ** mxChoice best-so-far paths.
         **
         ** First look for an existing path among best-so-far paths
-        ** that covers the same set of loops and has the same isOrdered
-        ** setting as the current path candidate.
+        ** that:
+        **     (1) covers the same set of loops, and
+        **     (2) has a compatible isOrdered value.
+        **
+        ** "Compatible isOrdered value" means either
+        **     (A) both have isOrdered==-1, or
+        **     (B) both have isOrder>=0, or
+        **     (C) ordering does not matter because this is the last round
+        **         of the solver.
         **
         ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent
         ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range
         ** of legal values for isOrdered, -1..64.
         */
+        testcase( nTo==0 );
         for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
           if( pTo->maskLoop==maskNew
-           && ((pTo->isOrdered^isOrdered)&0x80)==0
+           && ( ((pTo->isOrdered^isOrdered)&0x80)==0 || iLoop==nLoop-1 )
           ){
             testcase( jj==nTo-1 );
             break;
@@ -5149,7 +5879,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         if( jj>=nTo ){
           /* None of the existing best-so-far paths match the candidate. */
           if( nTo>=mxChoice
-           && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted))
+           && (rCost>mxCost || (rCost==mxCost && rUnsort>=mxUnsort))
           ){
             /* The current candidate is no better than any of the mxChoice
             ** paths currently in the best-so-far buffer.  So discard
@@ -5157,7 +5887,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
 #ifdef WHERETRACE_ENABLED /* 0x4 */
             if( sqlite3WhereTrace&0x4 ){
               sqlite3DebugPrintf("Skip   %s cost=%-3d,%3d,%3d order=%c\n",
-                  wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
+                  wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
                   isOrdered>=0 ? isOrdered+'0' : '?');
             }
 #endif
@@ -5176,7 +5906,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
 #ifdef WHERETRACE_ENABLED /* 0x4 */
           if( sqlite3WhereTrace&0x4 ){
             sqlite3DebugPrintf("New    %s cost=%-3d,%3d,%3d order=%c\n",
-                wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
+                wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
                 isOrdered>=0 ? isOrdered+'0' : '?');
           }
 #endif
@@ -5185,26 +5915,25 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
           ** same set of loops and has the same isOrdered setting as the
           ** candidate path.  Check to see if the candidate should replace
           ** pTo or if the candidate should be skipped.
-          ** 
+          **
           ** The conditional is an expanded vector comparison equivalent to:
-          **   (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted)
+          **   (pTo->rCost,pTo->nRow,pTo->rUnsort) <= (rCost,nOut,rUnsort)
           */
-          if( pTo->rCost<rCost 
-           || (pTo->rCost==rCost
-               && (pTo->nRow<nOut
-                   || (pTo->nRow==nOut && pTo->rUnsorted<=rUnsorted)
-                  )
-              )
+          if( (pTo->rCost<rCost)
+           || (pTo->rCost==rCost && pTo->nRow<nOut)
+           || (pTo->rCost==rCost && pTo->nRow==nOut && pTo->rUnsort<rUnsort)
+           || (pTo->rCost==rCost && pTo->nRow==nOut && pTo->rUnsort==rUnsort
+                  && whereLoopIsNoBetter(pWLoop, pTo->aLoop[iLoop]) )
           ){
 #ifdef WHERETRACE_ENABLED /* 0x4 */
             if( sqlite3WhereTrace&0x4 ){
               sqlite3DebugPrintf(
                   "Skip   %s cost=%-3d,%3d,%3d order=%c",
-                  wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
+                  wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
                   isOrdered>=0 ? isOrdered+'0' : '?');
               sqlite3DebugPrintf("   vs %s cost=%-3d,%3d,%3d order=%c\n",
                   wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
-                  pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
+                  pTo->rUnsort, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
             }
 #endif
             /* Discard the candidate path from further consideration */
@@ -5218,11 +5947,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
           if( sqlite3WhereTrace&0x4 ){
             sqlite3DebugPrintf(
                 "Update %s cost=%-3d,%3d,%3d order=%c",
-                wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted,
+                wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsort,
                 isOrdered>=0 ? isOrdered+'0' : '?');
             sqlite3DebugPrintf("  was %s cost=%-3d,%3d,%3d order=%c\n",
                 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
-                pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
+                pTo->rUnsort, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
           }
 #endif
         }
@@ -5231,20 +5960,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         pTo->revLoop = revMask;
         pTo->nRow = nOut;
         pTo->rCost = rCost;
-        pTo->rUnsorted = rUnsorted;
+        pTo->rUnsort = rUnsort;
         pTo->isOrdered = isOrdered;
         memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
         pTo->aLoop[iLoop] = pWLoop;
         if( nTo>=mxChoice ){
           mxI = 0;
           mxCost = aTo[0].rCost;
-          mxUnsorted = aTo[0].nRow;
+          mxUnsort = aTo[0].nRow;
           for(jj=1, pTo=&aTo[1]; jj<mxChoice; jj++, pTo++){
-            if( pTo->rCost>mxCost 
-             || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) 
+            if( pTo->rCost>mxCost
+             || (pTo->rCost==mxCost && pTo->rUnsort>mxUnsort)
             ){
               mxCost = pTo->rCost;
-              mxUnsorted = pTo->rUnsorted;
+              mxUnsort = pTo->rUnsort;
               mxI = jj;
             }
           }
@@ -5254,17 +5983,32 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
 
 #ifdef WHERETRACE_ENABLED  /* >=2 */
     if( sqlite3WhereTrace & 0x02 ){
+      LogEst rMin, rFloor = 0;
+      int nDone = 0;
+      int nProgress;
       sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
-      for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
-        sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
-           wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
-           pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
-        if( pTo->isOrdered>0 ){
-          sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
-        }else{
-          sqlite3DebugPrintf("\n");
+      do{
+        nProgress = 0;
+        rMin = 0x7fff;
+        for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
+          if( pTo->rCost>rFloor && pTo->rCost<rMin ) rMin = pTo->rCost;
         }
-      }
+        for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
+          if( pTo->rCost==rMin ){
+            sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
+               wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
+               pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
+            if( pTo->isOrdered>0 ){
+              sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
+            }else{
+              sqlite3DebugPrintf("\n");
+            }
+            nDone++;
+            nProgress++;
+          }
+        }
+        rFloor = rMin;
+      }while( nDone<nTo && nProgress>0 );
     }
 #endif
 
@@ -5280,12 +6024,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
     sqlite3StackFreeNN(pParse->db, pSpace);
     return SQLITE_ERROR;
   }
-  
-  /* Find the lowest cost path.  pFrom will be left pointing to that path */
+ 
+  /* Only one path is available, which is the best path */
+  assert( nFrom==1 );
   pFrom = aFrom;
-  for(ii=1; ii<nFrom; ii++){
-    if( pFrom->rCost>aFrom[ii].rCost ) pFrom = &aFrom[ii];
-  }
+
   assert( pWInfo->nLevel==nLoop );
   /* Load the lowest cost path into pWInfo */
   for(iLoop=0; iLoop<nLoop; iLoop++){
@@ -5313,17 +6056,16 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
         pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
       }
-      if( pWInfo->pSelect->pOrderBy
-       && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){
-        pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr;
-      }
+      /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */
+      assert( pWInfo->pSelect->pOrderBy==0
+           || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr );
     }else{
       pWInfo->revMask = pFrom->revLoop;
       if( pWInfo->nOBSat<=0 ){
         pWInfo->nOBSat = 0;
         if( nLoop>0 ){
           u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
-          if( (wsFlags & WHERE_ONEROW)==0 
+          if( (wsFlags & WHERE_ONEROW)==0
            && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
           ){
             Bitmask m = 0;
@@ -5348,7 +6090,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
     ){
       Bitmask revMask = 0;
-      int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, 
+      int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy,
           pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
       );
       assert( pWInfo->sorted==0 );
@@ -5359,14 +6101,96 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
     }
   }
 
-
   pWInfo->nRowOut = pFrom->nRow;
+#ifdef WHERETRACE_ENABLED
+  pWInfo->rTotalCost = pFrom->rCost;
+#endif
 
   /* Free temporary memory and return success */
   sqlite3StackFreeNN(pParse->db, pSpace);
   return SQLITE_OK;
 }
 
+/*
+** This routine implements a heuristic designed to improve query planning.
+** This routine is called in between the first and second call to
+** wherePathSolver().  Hence the name "Interstage" "Heuristic".
+**
+** The first call to wherePathSolver() (hereafter just "solver()") computes
+** the best path without regard to the order of the outputs.  The second call
+** to the solver() builds upon the first call to try to find an alternative
+** path that satisfies the ORDER BY clause.
+**
+** This routine looks at the results of the first solver() run, and for
+** every FROM clause term in the resulting query plan that uses an equality
+** constraint against an index, disable other WhereLoops for that same
+** FROM clause term that would try to do a full-table scan.  This prevents
+** an index search from being converted into a full-table scan in order to
+** satisfy an ORDER BY clause, since even though we might get slightly better
+** performance using the full-scan without sorting if the output size
+** estimates are very precise, we might also get severe performance
+** degradation using the full-scan if the output size estimate is too large.
+** It is better to err on the side of caution.
+**
+** Except, if the first solver() call generated a full-table scan in an outer
+** loop then stop this analysis at the first full-scan, since the second
+** solver() run might try to swap that full-scan for another in order to
+** get the output into the correct order.  In other words, we allow a
+** rewrite like this:
+**
+**     First Solver()                      Second Solver()
+**       |-- SCAN t1                         |-- SCAN t2
+**       |-- SEARCH t2                       `-- SEARCH t1
+**       `-- SORT USING B-TREE
+**
+** The purpose of this routine is to disallow rewrites such as:
+**
+**     First Solver()                      Second Solver()
+**       |-- SEARCH t1                       |-- SCAN t2     <--- bad!
+**       |-- SEARCH t2                       `-- SEARCH t1
+**       `-- SORT USING B-TREE
+**
+** See test cases in test/whereN.test for the real-world query that
+** originally provoked this heuristic.
+*/
+static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){
+  int i;
+#ifdef WHERETRACE_ENABLED
+  int once = 0;
+#endif
+  for(i=0; i<pWInfo->nLevel; i++){
+    WhereLoop *p = pWInfo->a[i].pWLoop;
+    if( p==0 ) break;
+    if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){
+      /* Treat a vtab scan as similar to a full-table scan */
+      break;
+    }
+    if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){
+      u8 iTab = p->iTab;
+      WhereLoop *pLoop;
+      for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){
+        if( pLoop->iTab!=iTab ) continue;
+        if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){
+          /* Auto-index and index-constrained loops allowed to remain */
+          continue;
+        }
+#ifdef WHERETRACE_ENABLED
+        if( sqlite3WhereTrace & 0x80 ){
+          if( once==0 ){
+            sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n");
+            once = 1;
+          }
+          sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC);
+        }
+#endif /* WHERETRACE_ENABLED */
+        pLoop->prereq = ALLBITS;  /* Prevent 2nd solver() from using this one */
+      }
+    }else{
+      break;
+    }
+  }
+}
+
 /*
 ** Most queries use only a single table (they are not joins) and have
 ** simple == constraints against indexed fields.  This routine attempts
@@ -5375,7 +6199,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
 ** times for the common case.
 **
 ** Return non-zero on success, if this query can be handled by this
-** no-frills query planner.  Return zero if this query needs the 
+** no-frills query planner.  Return zero if this query needs the
 ** general-purpose query planner.
 */
 static int whereShortCut(WhereLoopBuilder *pBuilder){
@@ -5394,7 +6218,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
   assert( pWInfo->pTabList->nSrc>=1 );
   pItem = pWInfo->pTabList->a;
-  pTab = pItem->pTab;
+  pTab = pItem->pSTab;
   if( IsVirtual(pTab) ) return 0;
   if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){
     testcase( pItem->fg.isIndexedBy );
@@ -5421,8 +6245,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
       int opMask;
       assert( pLoop->aLTermSpace==pLoop->aLTerm );
       if( !IsUniqueIndex(pIdx)
-       || pIdx->pPartIdxWhere!=0 
-       || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) 
+       || pIdx->pPartIdxWhere!=0
+       || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace)
       ) continue;
       opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ;
       for(j=0; j<pIdx->nKeyCol; j++){
@@ -5482,8 +6306,8 @@ static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){
 }
 
 /*
-** Return true if the expression contains no non-deterministic SQL 
-** functions. Do not consider non-deterministic SQL functions that are 
+** Return true if the expression contains no non-deterministic SQL
+** functions. Do not consider non-deterministic SQL functions that are
 ** part of sub-select statements.
 */
 static int exprIsDeterministic(Expr *p){
@@ -5496,7 +6320,7 @@ static int exprIsDeterministic(Expr *p){
   return w.eCode;
 }
 
-  
+ 
 #ifdef WHERETRACE_ENABLED
 /*
 ** Display all WhereLoops in pWInfo
@@ -5524,7 +6348,7 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
 **   1) The query must not be an aggregate.
 **   2) The table must be the RHS of a LEFT JOIN.
 **   3) Either the query must be DISTINCT, or else the ON or USING clause
-**      must contain a constraint that limits the scan of the table to 
+**      must contain a constraint that limits the scan of the table to
 **      at most a single row.
 **   4) The table must not be referenced by any part of the query apart
 **      from its own USING or ON clause.
@@ -5535,6 +6359,10 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
 **      the right-most table of a subquery that was flattened into the
 **      main query and that subquery was the right-hand operand of an
 **      inner join that held an ON or USING clause.
+**   6) The ORDER BY clause has 63 or fewer terms
+**   7) The omit-noop-join optimization is enabled.
+**
+** Items (1), (6), and (7) are checked by the caller.
 **
 ** For example, given:
 **
@@ -5544,13 +6372,13 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
 **
 ** then table t2 can be omitted from the following:
 **
-**     SELECT v1, v3 FROM t1 
+**     SELECT v1, v3 FROM t1
 **       LEFT JOIN t2 ON (t1.ipk=t2.ipk)
 **       LEFT JOIN t3 ON (t1.ipk=t3.ipk)
 **
 ** or from:
 **
-**     SELECT DISTINCT v1, v3 FROM t1 
+**     SELECT DISTINCT v1, v3 FROM t1
 **       LEFT JOIN t2
 **       LEFT JOIN t3 ON (t1.ipk=t3.ipk)
 */
@@ -5580,6 +6408,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
     WhereTerm *pTerm, *pEnd;
     SrcItem *pItem;
     WhereLoop *pLoop;
+    Bitmask m1;
     pLoop = pWInfo->a[i].pWLoop;
     pItem = &pWInfo->pTabList->a[pLoop->iTab];
     if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue;
@@ -5600,13 +6429,16 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
       }
       if( hasRightJoin
        && ExprHasProperty(pTerm->pExpr, EP_InnerON)
-       && pTerm->pExpr->w.iJoin==pItem->iCursor
+       && NEVER(pTerm->pExpr->w.iJoin==pItem->iCursor)
       ){
         break;  /* restriction (5) */
       }
     }
     if( pTerm<pEnd ) continue;
-    WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId));
+    WHERETRACE(0xffffffff,("-> omit unused FROM-clause term %c\n",pLoop->cId));
+    m1 = MASKBIT(i)-1;
+    testcase( ((pWInfo->revMask>>1) & ~m1)!=0 );
+    pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1);
     notReady &= ~pLoop->maskSelf;
     for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){
       if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){
@@ -5653,9 +6485,9 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
     WhereLoop *pLoop = pWInfo->a[i].pWLoop;
     const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ);
     SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab];
-    Table *pTab = pItem->pTab;
+    Table *pTab = pItem->pSTab;
     if( (pTab->tabFlags & TF_HasStat1)==0 ) break;
-    pTab->tabFlags |= TF_StatsUsed;
+    pTab->tabFlags |= TF_MaybeReanalyze;
     if( i>=1
      && (pLoop->wsFlags & reqFlags)==reqFlags
      /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */
@@ -5676,20 +6508,6 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
   }
 }
 
-/*
-** This is an sqlite3ParserAddCleanup() callback that is invoked to
-** free the Parse->pIdxEpr list when the Parse object is destroyed.
-*/
-static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){
-  Parse *pParse = (Parse*)pObject;
-  while( pParse->pIdxEpr!=0 ){
-    IndexedExpr *p = pParse->pIdxEpr;
-    pParse->pIdxEpr = p->pIENext;
-    sqlite3ExprDelete(db, p->pExpr);
-    sqlite3DbFreeNN(db, p);
-  }
-}
-
 /*
 ** The index pIdx is used by a query and contains one or more expressions.
 ** In other words pIdx is an index on an expression.  iIdxCur is the cursor
@@ -5715,20 +6533,14 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
   for(i=0; i<pIdx->nColumn; i++){
     Expr *pExpr;
     int j = pIdx->aiColumn[i];
-    int bMaybeNullRow;
     if( j==XN_EXPR ){
       pExpr = pIdx->aColExpr->a[i].pExpr;
-      testcase( pTabItem->fg.jointype & JT_LEFT );
-      testcase( pTabItem->fg.jointype & JT_RIGHT );
-      testcase( pTabItem->fg.jointype & JT_LTORJ );
-      bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
     }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){
       pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]);
-      bMaybeNullRow = 0;
     }else{
       continue;
     }
-    if( sqlite3ExprIsConstant(pExpr) ) continue;
+    if( sqlite3ExprIsConstant(0,pExpr) ) continue;
     p = sqlite3DbMallocRaw(pParse->db,  sizeof(IndexedExpr));
     if( p==0 ) break;
     p->pIENext = pParse->pIdxEpr;
@@ -5742,7 +6554,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
     p->iDataCur = pTabItem->iCursor;
     p->iIdxCur = iIdxCur;
     p->iIdxCol = i;
-    p->bMaybeNullRow = bMaybeNullRow;
+    p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0;
     if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){
       p->aff = pIdx->zColAff[i];
     }
@@ -5751,7 +6563,30 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
 #endif
     pParse->pIdxEpr = p;
     if( p->pIENext==0 ){
-      sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse);
+      void *pArg = (void*)&pParse->pIdxEpr;
+      sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg);
+    }
+  }
+}
+
+/*
+** Set the reverse-scan order mask to one for all tables in the query
+** with the exception of MATERIALIZED common table expressions that have
+** their own internal ORDER BY clauses.
+**
+** This implements the PRAGMA reverse_unordered_selects=ON setting.
+** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER).
+*/
+static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){
+  int ii;
+  for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){
+    SrcItem *pItem = &pWInfo->pTabList->a[ii];
+    if( !pItem->fg.isCte
+     || pItem->u2.pCteUse->eM10d!=M10d_Yes
+     || NEVER(pItem->fg.isSubquery==0)
+     || pItem->u4.pSubq->pSelect->pOrderBy==0
+    ){
+      pWInfo->revMask |= MASKBIT(ii);
     }
   }
 }
@@ -5814,7 +6649,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
 **
 ** OUTER JOINS
 **
-** An outer join of tables t1 and t2 is conceptally coded as follows:
+** An outer join of tables t1 and t2 is conceptually coded as follows:
 **
 **    foreach row1 in t1 do
 **      flag = 0
@@ -5836,7 +6671,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
 ** if there is one.  If there is no ORDER BY clause or if this routine
 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
 **
-** The iIdxCur parameter is the cursor number of an index.  If 
+** The iIdxCur parameter is the cursor number of an index.  If
 ** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
 ** to use for OR clause processing.  The WHERE clause should use this
 ** specific cursor.  If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
@@ -5870,8 +6705,8 @@ WhereInfo *sqlite3WhereBegin(
   u8 bFordelete = 0;         /* OPFLAG_FORDELETE or zero, as appropriate */
 
   assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
-        (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
-     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 
+        (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0
+     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
   ));
 
   /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
@@ -5884,10 +6719,14 @@ WhereInfo *sqlite3WhereBegin(
 
   /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
   testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
-  if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
+  if( pOrderBy && pOrderBy->nExpr>=BMS ){
+    pOrderBy = 0;
+    wctrlFlags &= ~WHERE_WANT_DISTINCT;
+    wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */
+  }
 
   /* The number of tables in the FROM clause is limited by the number of
-  ** bits in a Bitmask 
+  ** bits in a Bitmask
   */
   testcase( pTabList->nSrc==BMS );
   if( pTabList->nSrc>BMS ){
@@ -5895,7 +6734,7 @@ WhereInfo *sqlite3WhereBegin(
     return 0;
   }
 
-  /* This function normally generates a nested loop for all tables in 
+  /* This function normally generates a nested loop for all tables in
   ** pTabList.  But if the WHERE_OR_SUBCLAUSE flag is set, then we should
   ** only generate code for the first table in pTabList and assume that
   ** any cursors associated with subsequent tables are uninitialized.
@@ -5909,7 +6748,7 @@ WhereInfo *sqlite3WhereBegin(
   ** field (type Bitmask) it must be aligned on an 8-byte boundary on
   ** some architectures. Hence the ROUND8() below.
   */
-  nByteWInfo = ROUND8P(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel));
+  nByteWInfo = SZ_WHEREINFO(nTabList);
   pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
   if( db->mallocFailed ){
     sqlite3DbFree(db, pWInfo);
@@ -5930,7 +6769,7 @@ WhereInfo *sqlite3WhereBegin(
   pWInfo->iLimit = iAuxArg;
   pWInfo->savedNQueryLoop = pParse->nQueryLoop;
   pWInfo->pSelect = pSelect;
-  memset(&pWInfo->nOBSat, 0, 
+  memset(&pWInfo->nOBSat, 0,
          offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
   memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
   assert( pWInfo->eOnePass==ONEPASS_OFF );  /* ONEPASS defaults to OFF */
@@ -5953,7 +6792,7 @@ WhereInfo *sqlite3WhereBegin(
   */
   sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo);
   sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND);
-    
+   
   /* Special case: No FROM clause
   */
   if( nTabList==0 ){
@@ -5963,13 +6802,17 @@ WhereInfo *sqlite3WhereBegin(
     ){
       pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
     }
-    ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
+    if( ALWAYS(pWInfo->pSelect)
+     && (pWInfo->pSelect->selFlags & SF_MultiValue)==0
+    ){
+      ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW"));
+    }
   }else{
     /* Assign a bit from the bitmask to every term in the FROM clause.
     **
     ** The N-th term of the FROM clause is assigned a bitmask of 1<<N.
     **
-    ** The rule of the previous sentence ensures thta if X is the bitmask for
+    ** The rule of the previous sentence ensures that if X is the bitmask for
     ** a table T, then X-1 is the bitmask for all other tables to the left of T.
     ** Knowing the bitmask for all tables to the left of a left join is
     ** important.  Ticket #3015.
@@ -5995,7 +6838,7 @@ WhereInfo *sqlite3WhereBegin(
     }
   #endif
   }
-  
+ 
   /* Analyze all of the subexpressions. */
   sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC);
   if( pSelect && pSelect->pLimit ){
@@ -6099,7 +6942,7 @@ WhereInfo *sqlite3WhereBegin(
     ** loops will be built using the revised truthProb values. */
     if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
       WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
-      WHERETRACE(0xffffffff, 
+      WHERETRACE(0xffffffff,
            ("**** Redo all loop computations due to"
             " TERM_HIGHTRUTH changes ****\n"));
       while( pWInfo->pLoops ){
@@ -6112,16 +6955,29 @@ WhereInfo *sqlite3WhereBegin(
     }
 #endif
     WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
-  
+ 
     wherePathSolver(pWInfo, 0);
     if( db->mallocFailed ) goto whereBeginError;
     if( pWInfo->pOrderBy ){
-       wherePathSolver(pWInfo, pWInfo->nRowOut+1);
+       whereInterstageHeuristic(pWInfo);
+       wherePathSolver(pWInfo, pWInfo->nRowOut<0 ? 1 : pWInfo->nRowOut+1);
        if( db->mallocFailed ) goto whereBeginError;
     }
+
+    /* TUNING:  Assume that a DISTINCT clause on a subquery reduces
+    ** the output size by a factor of 8 (LogEst -30).  Search for
+    ** tag-20250414a to see other cases.
+    */
+    if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){
+      WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n",
+                         pWInfo->nRowOut, pWInfo->nRowOut-30));
+      pWInfo->nRowOut -= 30;
+    }
+
   }
+  assert( pWInfo->pTabList!=0 );
   if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
-     pWInfo->revMask = ALLBITS;
+    whereReverseScanOrder(pWInfo);
   }
   if( pParse->nErr ){
     goto whereBeginError;
@@ -6129,7 +6985,8 @@ WhereInfo *sqlite3WhereBegin(
   assert( db->mallocFailed==0 );
 #ifdef WHERETRACE_ENABLED
   if( sqlite3WhereTrace ){
-    sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut);
+    sqlite3DebugPrintf("---- Solution cost=%d, nRow=%d",
+                       pWInfo->rTotalCost, pWInfo->nRowOut);
     if( pWInfo->nOBSat>0 ){
       sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask);
     }
@@ -6160,15 +7017,15 @@ WhereInfo *sqlite3WhereBegin(
   ** This query optimization is factored out into a separate "no-inline"
   ** procedure to keep the sqlite3WhereBegin() procedure from becoming
   ** too large.  If sqlite3WhereBegin() becomes too large, that prevents
-  ** some C-compiler optimizers from in-lining the 
+  ** some C-compiler optimizers from in-lining the
   ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to
   ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons.
   */
   notReady = ~(Bitmask)0;
-  if( pWInfo->nLevel>=2
-   && pResultSet!=0                         /* these two combine to guarantee */
-   && 0==(wctrlFlags & WHERE_AGG_DISTINCT)  /* condition (1) above */
-   && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
+  if( pWInfo->nLevel>=2       /* Must be a join, or this opt8n is pointless */
+   && pResultSet!=0           /* Condition (1) */
+   && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */
+   && OptimizationEnabled(db, SQLITE_OmitNoopJoin)                /* (7) */
   ){
     notReady = whereOmitNoopJoin(pWInfo, notReady);
     nTabList = pWInfo->nLevel;
@@ -6216,14 +7073,15 @@ WhereInfo *sqlite3WhereBegin(
   if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){
     int wsFlags = pWInfo->a[0].pWLoop->wsFlags;
     int bOnerow = (wsFlags & WHERE_ONEROW)!=0;
-    assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) );
+    assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) );
     if( bOnerow || (
         0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW)
-     && !IsVirtual(pTabList->a[0].pTab)
+     && !IsVirtual(pTabList->a[0].pSTab)
      && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK))
+     && OptimizationEnabled(db, SQLITE_OnePass)
     )){
       pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
-      if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){
+      if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){
         if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){
           bFordelete = OPFLAG_FORDELETE;
         }
@@ -6241,9 +7099,17 @@ WhereInfo *sqlite3WhereBegin(
     SrcItem *pTabItem;
 
     pTabItem = &pTabList->a[pLevel->iFrom];
-    pTab = pTabItem->pTab;
+    pTab = pTabItem->pSTab;
     iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
     pLoop = pLevel->pWLoop;
+    pLevel->addrBrk = sqlite3VdbeMakeLabel(pParse);
+    if( ii==0 || (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){
+      pLevel->addrHalt = pLevel->addrBrk;
+    }else if( pWInfo->a[ii-1].pRJ ){
+      pLevel->addrHalt = pWInfo->a[ii-1].addrBrk;
+    }else{
+      pLevel->addrHalt = pWInfo->a[ii-1].addrHalt;
+    }
     if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){
       /* Do nothing */
     }else
@@ -6269,7 +7135,7 @@ WhereInfo *sqlite3WhereBegin(
       assert( pTabItem->iCursor==pLevel->iTabCur );
       testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
       testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
-      if( pWInfo->eOnePass==ONEPASS_OFF 
+      if( pWInfo->eOnePass==ONEPASS_OFF
        && pTab->nCol<BMS
        && (pTab->tabFlags & (TF_HasGenerated|TF_WithoutRowid))==0
        && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0
@@ -6295,6 +7161,13 @@ WhereInfo *sqlite3WhereBegin(
       sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0,
                             (const u8*)&pTabItem->colUsed, P4_INT64);
 #endif
+      if( ii>=2
+       && (pTabItem[0].fg.jointype & (JT_LTORJ|JT_LEFT))==0 
+       && pLevel->addrHalt==pWInfo->a[0].addrHalt
+      ){
+        sqlite3VdbeAddOp2(v, OP_IfEmpty, pTabItem->iCursor, pWInfo->iBreak);
+        VdbeCoverage(v);
+      }
     }else{
       sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
     }
@@ -6312,7 +7185,7 @@ WhereInfo *sqlite3WhereBegin(
         iIndexCur = pLevel->iTabCur;
         op = 0;
       }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
-        Index *pJ = pTabItem->pTab->pIndex;
+        Index *pJ = pTabItem->pSTab->pIndex;
         iIndexCur = iAuxArg;
         assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
         while( ALWAYS(pJ) && pJ!=pIx ){
@@ -6329,6 +7202,11 @@ WhereInfo *sqlite3WhereBegin(
         if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){
           whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem);
         }
+        if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){
+          wherePartIdxExpr(
+              pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem
+          );
+        }
       }
       pLevel->iIdxCur = iIndexCur;
       assert( pIx!=0 );
@@ -6374,7 +7252,7 @@ WhereInfo *sqlite3WhereBegin(
       sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom);
       pRJ->regReturn = ++pParse->nMem;
       sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn);
-      assert( pTab==pTabItem->pTab );
+      assert( pTab==pTabItem->pSTab );
       if( HasRowid(pTab) ){
         KeyInfo *pInfo;
         sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1);
@@ -6413,13 +7291,18 @@ WhereInfo *sqlite3WhereBegin(
     wsFlags = pLevel->pWLoop->wsFlags;
     pSrc = &pTabList->a[pLevel->iFrom];
     if( pSrc->fg.isMaterialized ){
-      if( pSrc->fg.isCorrelated ){
-        sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
+      Subquery *pSubq;
+      int iOnce = 0;
+      assert( pSrc->fg.isSubquery );
+      pSubq = pSrc->u4.pSubq;
+      if( pSrc->fg.isCorrelated==0 ){
+        iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
       }else{
-        int iOnce = sqlite3VdbeAddOp0(v, OP_Once);  VdbeCoverage(v);
-        sqlite3VdbeAddOp2(v, OP_Gosub, pSrc->regReturn, pSrc->addrFillSub);
-        sqlite3VdbeJumpHere(v, iOnce);
+        iOnce = 0;
       }
+      sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub);
+      VdbeComment((v, "materialize %!S", pSrc));
+      if( iOnce )  sqlite3VdbeJumpHere(v, iOnce);
     }
     assert( pTabList == pWInfo->pTabList );
     if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
@@ -6454,6 +7337,11 @@ WhereInfo *sqlite3WhereBegin(
     pParse->nQueryLoop = pWInfo->savedNQueryLoop;
     whereInfoFree(db, pWInfo);
   }
+#ifdef WHERETRACE_ENABLED
+  /* Prevent harmless compiler warnings about debugging routines
+  ** being declared but never used */
+  sqlite3ShowWhereLoopList(0);
+#endif /* WHERETRACE_ENABLED */
   return 0;
 }
 
@@ -6474,31 +7362,12 @@ WhereInfo *sqlite3WhereBegin(
   ){
     if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return;
     sqlite3VdbePrintOp(0, pc, pOp);
+    sqlite3ShowWhereTerm(0); /* So compiler won't complain about unused func */
   }
 #endif
 
-#ifdef SQLITE_DEBUG
 /*
-** Return true if cursor iCur is opened by instruction k of the
-** bytecode.  Used inside of assert() only.
-*/
-static int cursorIsOpen(Vdbe *v, int iCur, int k){
-  while( k>=0 ){
-    VdbeOp *pOp = sqlite3VdbeGetOp(v,k--);
-    if( pOp->p1!=iCur ) continue;
-    if( pOp->opcode==OP_Close ) return 0;
-    if( pOp->opcode==OP_OpenRead ) return 1;
-    if( pOp->opcode==OP_OpenWrite ) return 1;
-    if( pOp->opcode==OP_OpenDup ) return 1;
-    if( pOp->opcode==OP_OpenAutoindex ) return 1;
-    if( pOp->opcode==OP_OpenEphemeral ) return 1;
-  }
-  return 0;
-}
-#endif /* SQLITE_DEBUG */
-
-/*
-** Generate the end of the WHERE loop.  See comments on 
+** Generate the end of the WHERE loop.  See comments on
 ** sqlite3WhereBegin() for additional information.
 */
 void sqlite3WhereEnd(WhereInfo *pWInfo){
@@ -6511,6 +7380,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
   sqlite3 *db = pParse->db;
   int iEnd = sqlite3VdbeCurrentAddr(v);
   int nRJ = 0;
+#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
+  int addrSeek = 0;
+#endif
 
   /* Generate loop termination code.
   */
@@ -6523,7 +7395,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       ** the RIGHT JOIN table */
       WhereRightJoin *pRJ = pLevel->pRJ;
       sqlite3VdbeResolveLabel(v, pLevel->addrCont);
-      pLevel->addrCont = 0;
+      /* Replace addrCont with a new label that will never be used, just so
+      ** the subsequent call to resolve pLevel->addrCont will have something
+      ** to resolve. */
+      pLevel->addrCont = sqlite3VdbeMakeLabel(pParse);
       pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v);
       sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1);
       VdbeCoverage(v);
@@ -6532,7 +7407,6 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     pLoop = pLevel->pWLoop;
     if( pLevel->op!=OP_Noop ){
 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
-      int addrSeek = 0;
       Index *pIdx;
       int n;
       if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
@@ -6555,8 +7429,26 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
         sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
       }
 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
-      /* The common case: Advance to the next row */
-      if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
+    }
+    if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){
+      /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS
+      ** loop(s) will be the inner-most loops of the join. There might be
+      ** multiple EXISTS loops, but they will all be nested, and the join
+      ** order will not have been changed by the query planner.  If the
+      ** inner-most EXISTS loop sees a single successful row, it should
+      ** break out of *all* EXISTS loops.  But only the inner-most of the
+      ** nested EXISTS loops should do this breakout. */
+      int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */
+      while( nOuter<i ){
+        if( !pTabList->a[pLevel[-nOuter-1].iFrom].fg.fromExists ) break;
+        nOuter++;
+      }
+      testcase( nOuter>0 );
+      sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk);
+      VdbeComment((v, "EXISTS break"));
+    }
+    sqlite3VdbeResolveLabel(v, pLevel->addrCont);
+    if( pLevel->op!=OP_Noop ){
       sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);
       sqlite3VdbeChangeP5(v, pLevel->p5);
       VdbeCoverage(v);
@@ -6569,10 +7461,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
         VdbeCoverage(v);
       }
 #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT
-      if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek);
+      if( addrSeek ){
+        sqlite3VdbeJumpHere(v, addrSeek);
+        addrSeek = 0;
+      }
 #endif
-    }else if( pLevel->addrCont ){
-      sqlite3VdbeResolveLabel(v, pLevel->addrCont);
     }
     if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){
       struct InLoop *pIn;
@@ -6584,7 +7477,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
         sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
         if( pIn->eEndLoopOp!=OP_Noop ){
           if( pIn->nPrefix ){
-            int bEarlyOut = 
+            int bEarlyOut =
                 (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
                  && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0;
             if( pLevel->iLeftJoin ){
@@ -6596,7 +7489,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
               ** return the null-row. So, if the cursor is not open yet,
               ** jump over the OP_Next or OP_Prev instruction about to
               ** be coded.  */
-              sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, 
+              sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur,
                   sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut);
               VdbeCoverage(v);
             }
@@ -6605,7 +7498,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
                   sqlite3VdbeCurrentAddr(v)+2,
                   pIn->iBase, pIn->nPrefix);
               VdbeCoverage(v);
-              /* Retarget the OP_IsNull against the left operand of IN so 
+              /* Retarget the OP_IsNull against the left operand of IN so
               ** it jumps past the OP_IfNoHope.  This is because the
               ** OP_IsNull also bypasses the OP_Affinity opcode that is
               ** required by OP_IfNoHope. */
@@ -6643,11 +7536,20 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
       assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
       if( (ws & WHERE_IDX_ONLY)==0 ){
-        assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor );
+        SrcItem *pSrc = &pTabList->a[pLevel->iFrom];
+        assert( pLevel->iTabCur==pSrc->iCursor );
+        if( pSrc->fg.viaCoroutine ){
+          int m, n;
+          assert( pSrc->fg.isSubquery );
+          n = pSrc->u4.pSubq->regResult;
+          assert( pSrc->pSTab!=0 );
+          m = pSrc->pSTab->nCol;
+          sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1);
+        }
         sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur);
       }
-      if( (ws & WHERE_INDEXED) 
-       || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx) 
+      if( (ws & WHERE_INDEXED)
+       || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx)
       ){
         if( ws & WHERE_MULTI_OR ){
           Index *pIx = pLevel->u.pCoveringIdx;
@@ -6665,7 +7567,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       sqlite3VdbeJumpHere(v, addr);
     }
     VdbeModuleComment((v, "End WHERE-loop%d: %s", i,
-                     pWInfo->pTabList->a[pLevel->iFrom].pTab->zName));
+                     pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName));
   }
 
   assert( pWInfo->nLevel<=pTabList->nSrc );
@@ -6674,7 +7576,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     VdbeOp *pOp, *pLastOp;
     Index *pIdx = 0;
     SrcItem *pTabItem = &pTabList->a[pLevel->iFrom];
-    Table *pTab = pTabItem->pTab;
+    Table *pTab = pTabItem->pSTab;
     assert( pTab!=0 );
     pLoop = pLevel->pWLoop;
 
@@ -6693,8 +7595,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     */
     if( pTabItem->fg.viaCoroutine ){
       testcase( pParse->db->mallocFailed );
+      assert( pTabItem->fg.isSubquery );
+      assert( pTabItem->u4.pSubq->regResult>=0 );
       translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
-                            pTabItem->regResult, 0);
+                            pTabItem->u4.pSubq->regResult, 0);
       continue;
     }
 
@@ -6702,7 +7606,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     ** from the index instead of from the table where possible.  In some cases
     ** this optimization prevents the table from ever being read, which can
     ** yield a significant performance boost.
-    ** 
+    **
     ** Calls to the code generator in between sqlite3WhereBegin and
     ** sqlite3WhereEnd will have created code that references the table
     ** directly.  This loop scans all that code looking for opcodes
@@ -6782,21 +7686,29 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
             pOp->p2 = x;
             pOp->p1 = pLevel->iIdxCur;
             OpcodeRewriteTrace(db, k, pOp);
-          }else{
-            /* Unable to translate the table reference into an index
-            ** reference.  Verify that this is harmless - that the
-            ** table being referenced really is open.
-            */
-#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
-            assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-                 || cursorIsOpen(v,pOp->p1,k)
-                 || pOp->opcode==OP_Offset
-            );
-#else
-            assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-                 || cursorIsOpen(v,pOp->p1,k)
-            );
-#endif
+          }else if( pLoop->wsFlags & (WHERE_IDX_ONLY|WHERE_EXPRIDX) ){
+            if( pLoop->wsFlags & WHERE_IDX_ONLY ){
+              /* An error. pLoop is supposed to be a covering index loop,
+              ** and yet the VM code refers to a column of the table that 
+              ** is not part of the index.  */
+              sqlite3ErrorMsg(pParse, "internal query planner error");
+              pParse->rc = SQLITE_INTERNAL;
+            }else{
+              /* The WHERE_EXPRIDX flag is set by the planner when it is likely
+              ** that pLoop is a covering index loop, but it is not possible
+              ** to be 100% sure. In this case, any OP_Explain opcode
+              ** corresponding to this loop describes the index as a "COVERING
+              ** INDEX". But, pOp proves that pLoop is not actually a covering 
+              ** index loop. So clear the WHERE_EXPRIDX flag and rewrite the
+              ** text that accompanies the OP_Explain opcode, if any.  */
+              pLoop->wsFlags &= ~WHERE_EXPRIDX;
+              sqlite3WhereAddExplainText(pParse,
+                  pLevel->addrBody-1,
+                  pTabList,
+                  pLevel,
+                  pWInfo->wctrlFlags
+              );
+            }
           }
         }else if( pOp->opcode==OP_Rowid ){
           pOp->p1 = pLevel->iIdxCur;
diff --git a/src/whereInt.h b/src/whereInt.h
index b89a4513e3..09e02c8c73 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -37,7 +37,7 @@ typedef struct WhereRightJoin WhereRightJoin;
 
 /*
 ** This object is a header on a block of allocated memory that will be
-** automatically freed when its WInfo oject is destructed.
+** automatically freed when its WInfo object is destructed.
 */
 struct WhereMemBlock {
   WhereMemBlock *pNext;      /* Next block in the chain */
@@ -75,6 +75,7 @@ struct WhereLevel {
   int iTabCur;          /* The VDBE cursor used to access the table */
   int iIdxCur;          /* The VDBE cursor used to access pIdx */
   int addrBrk;          /* Jump here to break out of the loop */
+  int addrHalt;         /* Abort the query due to empty table or similar */
   int addrNxt;          /* Jump here to start the next IN combination */
   int addrSkip;         /* Jump here for next iteration of skip-scan */
   int addrCont;         /* Jump here to continue with the next loop cycle */
@@ -98,7 +99,7 @@ struct WhereLevel {
         int iCur;              /* The VDBE cursor used by this IN operator */
         int addrInTop;         /* Top of the IN loop */
         int iBase;             /* Base register of multi-key index record */
-        int nPrefix;           /* Number of prior entires in the key */
+        int nPrefix;           /* Number of prior entries in the key */
         u8 eEndLoopOp;         /* IN Loop terminator. OP_Next or OP_Prev */
       } *aInLoop;           /* Information about each nested IN operator */
     } in;                 /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */
@@ -143,11 +144,13 @@ struct WhereLoop {
       u16 nTop;              /* Size of TOP vector */
       u16 nDistinctCol;      /* Index columns used to sort for DISTINCT */
       Index *pIndex;         /* Index used, or NULL */
+      ExprList *pOrderBy;    /* ORDER BY clause if this is really a subquery */
     } btree;
     struct {               /* Information for virtual tables */
       int idxNum;            /* Index number */
       u32 needFree : 1;      /* True if sqlite3_free(idxStr) is needed */
       u32 bOmitOffset : 1;   /* True to let virtual table handle offset */
+      u32 bIdxNumHex : 1;    /* Show idxNum as hex in EXPLAIN QUERY PLAN */
       i8 isOrdered;          /* True if satisfies ORDER BY */
       u16 omitMask;          /* Terms that may be omitted */
       char *idxStr;          /* Index identifier string */
@@ -160,6 +163,10 @@ struct WhereLoop {
   /**** whereLoopXfer() copies fields above ***********************/
 # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot)
   u16 nLSlot;           /* Number of slots allocated for aLTerm[] */
+#ifdef WHERETRACE_ENABLED
+  LogEst rStarDelta;    /* Cost delta due to star-schema heuristic.  Not
+                        ** initialized unless pWInfo->bStarUsed */
+#endif
   WhereTerm **aLTerm;   /* WhereTerms used */
   WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */
   WhereTerm *aLTermSpace[3];  /* Initial aLTerm[] space */
@@ -167,7 +174,7 @@ struct WhereLoop {
 
 /* This object holds the prerequisites and the cost of running a
 ** subquery on one operand of an OR operator in the WHERE clause.
-** See WhereOrSet for additional information 
+** See WhereOrSet for additional information
 */
 struct WhereOrCost {
   Bitmask prereq;     /* Prerequisites */
@@ -208,7 +215,7 @@ struct WherePath {
   Bitmask revLoop;      /* aLoop[]s that should be reversed for ORDER BY */
   LogEst nRow;          /* Estimated number of rows generated by this path */
   LogEst rCost;         /* Total cost of this path */
-  LogEst rUnsorted;     /* Total cost of this path ignoring sorting costs */
+  LogEst rUnsort;       /* Total cost of this path ignoring sorting costs */
   i8 isOrdered;         /* No. of ORDER BY terms satisfied. -1 for unknown */
   WhereLoop **aLoop;    /* Array of WhereLoop objects implementing this path */
 };
@@ -219,7 +226,7 @@ struct WherePath {
 ** clause subexpression is separated from the others by AND operators,
 ** usually, or sometimes subexpressions separated by OR.
 **
-** All WhereTerms are collected into a single WhereClause structure.  
+** All WhereTerms are collected into a single WhereClause structure. 
 ** The following identity holds:
 **
 **        WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm
@@ -274,6 +281,9 @@ struct WhereTerm {
   u8 eMatchOp;            /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */
   int iParent;            /* Disable pWC->a[iParent] when this term disabled */
   int leftCursor;         /* Cursor number of X in "X <op> <expr>" */
+#ifdef SQLITE_DEBUG
+  int iTerm;              /* Which WhereTerm is this, for debug purposes */
+#endif
   union {
     struct {
       int leftColumn;         /* Column number of X in "X <op> <expr>" */
@@ -348,7 +358,7 @@ struct WhereClause {
   int nTerm;               /* Number of terms */
   int nSlot;               /* Number of entries in a[] */
   int nBase;               /* Number of terms through the last non-Virtual */
-  WhereTerm *a;            /* Each a[] describes a term of the WHERE cluase */
+  WhereTerm *a;            /* Each a[] describes a term of the WHERE clause */
 #if defined(SQLITE_SMALL_STACK)
   WhereTerm aStatic[1];    /* Initial static space for a[] */
 #else
@@ -377,8 +387,8 @@ struct WhereAndInfo {
 ** An instance of the following structure keeps track of a mapping
 ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm.
 **
-** The VDBE cursor numbers are small integers contained in 
-** SrcItem.iCursor and Expr.iTable fields.  For any given WHERE 
+** The VDBE cursor numbers are small integers contained in
+** SrcItem.iCursor and Expr.iTable fields.  For any given WHERE
 ** clause, the cursor numbers might not begin with 0 and they might
 ** contain gaps in the numbering sequence.  But we want to make maximum
 ** use of the bits in our bitmasks.  This structure provides a mapping
@@ -481,8 +491,13 @@ struct WhereInfo {
   unsigned bDeferredSeek :1;   /* Uses OP_DeferredSeek */
   unsigned untestedTerms :1;   /* Not all WHERE terms resolved by outer loop */
   unsigned bOrderedInnerLoop:1;/* True if only the inner-most loop is ordered */
-  unsigned sorted :1;          /* True if really sorted (not just grouped) */
+  unsigned sorted        :1;   /* True if really sorted (not just grouped) */
+  unsigned bStarDone     :1;   /* True if check for star-query is complete */
+  unsigned bStarUsed     :1;   /* True if star-query heuristic is used */
   LogEst nRowOut;           /* Estimated number of output rows */
+#ifdef WHERETRACE_ENABLED
+  LogEst rTotalCost;        /* Total cost of the solution */
+#endif
   int iTop;                 /* The very beginning of the WHERE loop */
   int iEndWhere;            /* End of the WHERE clause itself */
   WhereLoop *pLoops;        /* List of all WhereLoop objects */
@@ -490,9 +505,14 @@ struct WhereInfo {
   Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
   WhereClause sWC;          /* Decomposition of the WHERE clause */
   WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
-  WhereLevel a[1];          /* Information about each nest loop in WHERE */
+  WhereLevel a[FLEXARRAY];  /* Information about each nest loop in WHERE */
 };
 
+/*
+** The size (in bytes) of a WhereInfo object that holds N WhereLevels.
+*/
+#define SZ_WHEREINFO(N) ROUND8(offsetof(WhereInfo,a)+(N)*sizeof(WhereLevel))
+
 /*
 ** Private interfaces - callable only by other where.c routines.
 **
@@ -502,7 +522,7 @@ Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
 #ifdef WHERETRACE_ENABLED
 void sqlite3WhereClausePrint(WhereClause *pWC);
 void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm);
-void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC);
+void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC);
 #endif
 WhereTerm *sqlite3WhereFindTerm(
   WhereClause *pWC,     /* The WHERE clause to be searched */
@@ -528,9 +548,17 @@ int sqlite3WhereExplainBloomFilter(
   const WhereInfo *pWInfo,        /* WHERE clause */
   const WhereLevel *pLevel        /* Bloom filter on this level */
 );
+void sqlite3WhereAddExplainText(
+  Parse *pParse,                  /* Parse context */
+  int addr, 
+  SrcList *pTabList,              /* Table list this loop refers to */
+  WhereLevel *pLevel,             /* Scan to write OP_Explain opcode for */
+  u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
+);
 #else
 # define sqlite3WhereExplainOneScan(u,v,w,x) 0
 # define sqlite3WhereExplainBloomFilter(u,v,w) 0
+# define  sqlite3WhereAddExplainText(u,v,w,x,y)
 #endif /* SQLITE_OMIT_EXPLAIN */
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
 void sqlite3WhereAddScanStatus(
@@ -633,7 +661,8 @@ void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*);
 #define WHERE_BLOOMFILTER  0x00400000  /* Consider using a Bloom-filter */
 #define WHERE_SELFCULL     0x00800000  /* nOut reduced by extra WHERE terms */
 #define WHERE_OMIT_OFFSET  0x01000000  /* Set offset counter to zero */
-#define WHERE_VIEWSCAN     0x02000000  /* A full-scan of a VIEW or subquery */
+#define WHERE_COROUTINE    0x02000000  /* Implemented by co-routine.
+                                       ** NB: False-negatives are possible */
 #define WHERE_EXPRIDX      0x04000000  /* Uses an index-on-expressions */
 
 #endif /* !defined(SQLITE_WHEREINT_H) */
diff --git a/src/wherecode.c b/src/wherecode.c
index a998c0a4f4..1efa34a5da 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -71,7 +71,7 @@ static void explainAppendTerm(
 }
 
 /*
-** Argument pLevel describes a strategy for scanning table pTab. This 
+** Argument pLevel describes a strategy for scanning table pTab. This
 ** function appends text to pStr that describes the subset of table
 ** rows scanned by the strategy in the form of an SQL expression.
 **
@@ -110,38 +110,37 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){
 }
 
 /*
-** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
-** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG 
-** was defined at compile-time. If it is not a no-op, a single OP_Explain
-** opcode is added to the output to describe the table scan strategy in pLevel.
-**
-** If an OP_Explain opcode is added to the VM, its address is returned.
-** Otherwise, if no OP_Explain is coded, zero is returned.
+** This function sets the P4 value of an existing OP_Explain opcode to
+** text describing the loop in pLevel. If the OP_Explain opcode already has
+** a P4 value, it is freed before it is overwritten.
 */
-int sqlite3WhereExplainOneScan(
+void sqlite3WhereAddExplainText(
   Parse *pParse,                  /* Parse context */
+  int addr,                       /* Address of OP_Explain opcode */
   SrcList *pTabList,              /* Table list this loop refers to */
   WhereLevel *pLevel,             /* Scan to write OP_Explain opcode for */
   u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
 ){
-  int ret = 0;
 #if !defined(SQLITE_DEBUG)
   if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) )
 #endif
   {
+    VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe, addr);
     SrcItem *pItem = &pTabList->a[pLevel->iFrom];
-    Vdbe *v = pParse->pVdbe;      /* VM being constructed */
     sqlite3 *db = pParse->db;     /* Database handle */
     int isSearch;                 /* True for a SEARCH. False for SCAN. */
     WhereLoop *pLoop;             /* The controlling WhereLoop object */
     u32 flags;                    /* Flags that describe this loop */
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN)
     char *zMsg;                   /* Text to add to EQP output */
+#endif
     StrAccum str;                 /* EQP output string */
     char zBuf[100];               /* Initial space for EQP output string */
 
+    if( db->mallocFailed ) return;
+
     pLoop = pLevel->pWLoop;
     flags = pLoop->wsFlags;
-    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0;
 
     isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
             || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
@@ -149,7 +148,10 @@ int sqlite3WhereExplainOneScan(
 
     sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
     str.printfFlags = SQLITE_PRINTF_INTERNAL;
-    sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem);
+    sqlite3_str_appendf(&str, "%s %S%s",
+       isSearch ? "SEARCH" : "SCAN",
+       pItem,
+       pItem->fg.fromExists ? " EXISTS" : "");
     if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){
       const char *zFmt = 0;
       Index *pIdx;
@@ -157,7 +159,7 @@ int sqlite3WhereExplainOneScan(
       assert( pLoop->u.btree.pIndex!=0 );
       pIdx = pLoop->u.btree.pIndex;
       assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) );
-      if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){
+      if( !HasRowid(pItem->pSTab) && IsPrimaryKeyIndex(pIdx) ){
         if( isSearch ){
           zFmt = "PRIMARY KEY";
         }
@@ -165,7 +167,7 @@ int sqlite3WhereExplainOneScan(
         zFmt = "AUTOMATIC PARTIAL COVERING INDEX";
       }else if( flags & WHERE_AUTO_INDEX ){
         zFmt = "AUTOMATIC COVERING INDEX";
-      }else if( flags & WHERE_IDX_ONLY ){
+      }else if( flags & (WHERE_IDX_ONLY|WHERE_EXPRIDX) ){
         zFmt = "COVERING INDEX %s";
       }else{
         zFmt = "INDEX %s";
@@ -200,7 +202,9 @@ int sqlite3WhereExplainOneScan(
     }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
     else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
-      sqlite3_str_appendf(&str, " VIRTUAL TABLE INDEX %d:%s",
+      sqlite3_str_appendall(&str, " VIRTUAL TABLE INDEX ");
+      sqlite3_str_appendf(&str,
+                  pLoop->u.vtab.bIdxNumHex ? "0x%x:%s" : "%d:%s",
                   pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
     }
 #endif
@@ -215,10 +219,50 @@ int sqlite3WhereExplainOneScan(
       sqlite3_str_append(&str, " (~1 row)", 9);
     }
 #endif
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN)
     zMsg = sqlite3StrAccumFinish(&str);
     sqlite3ExplainBreakpoint("",zMsg);
-    ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v),
-                            pParse->addrExplain, 0, zMsg,P4_DYNAMIC);
+#endif
+
+    assert( pOp->opcode==OP_Explain );
+    assert( pOp->p4type==P4_DYNAMIC || pOp->p4.z==0 );
+    sqlite3DbFree(db, pOp->p4.z);
+    pOp->p4type = P4_DYNAMIC;
+    pOp->p4.z = sqlite3StrAccumFinish(&str);
+  }
+}
+
+
+/*
+** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
+** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG
+** was defined at compile-time. If it is not a no-op, a single OP_Explain
+** opcode is added to the output to describe the table scan strategy in pLevel.
+**
+** If an OP_Explain opcode is added to the VM, its address is returned.
+** Otherwise, if no OP_Explain is coded, zero is returned.
+*/
+int sqlite3WhereExplainOneScan(
+  Parse *pParse,                  /* Parse context */
+  SrcList *pTabList,              /* Table list this loop refers to */
+  WhereLevel *pLevel,             /* Scan to write OP_Explain opcode for */
+  u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
+){
+  int ret = 0;
+#if !defined(SQLITE_DEBUG)
+  if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) )
+#endif
+  {
+    if( (pLevel->pWLoop->wsFlags & WHERE_MULTI_OR)==0
+     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
+    ){
+      Vdbe *v = pParse->pVdbe;
+      int addr = sqlite3VdbeCurrentAddr(v);
+      ret = sqlite3VdbeAddOp3(
+          v, OP_Explain, addr, pParse->addrExplain, pLevel->pWLoop->rRun
+      );
+      sqlite3WhereAddExplainText(pParse, addr, pTabList, pLevel, wctrlFlags);
+    }
   }
   return ret;
 }
@@ -253,7 +297,7 @@ int sqlite3WhereExplainBloomFilter(
   sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem);
   pLoop = pLevel->pWLoop;
   if( pLoop->wsFlags & WHERE_IPK ){
-    const Table *pTab = pItem->pTab;
+    const Table *pTab = pItem->pSTab;
     if( pTab->iPKey>=0 ){
       sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName);
     }else{
@@ -279,11 +323,11 @@ int sqlite3WhereExplainBloomFilter(
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
 /*
 ** Configure the VM passed as the first argument with an
-** sqlite3_stmt_scanstatus() entry corresponding to the scan used to 
-** implement level pLvl. Argument pSrclist is a pointer to the FROM 
+** sqlite3_stmt_scanstatus() entry corresponding to the scan used to
+** implement level pLvl. Argument pSrclist is a pointer to the FROM
 ** clause that the scan reads data from.
 **
-** If argument addrExplain is not 0, it must be the address of an 
+** If argument addrExplain is not 0, it must be the address of an
 ** OP_Explain instruction that describes the same loop.
 */
 void sqlite3WhereAddScanStatus(
@@ -315,6 +359,15 @@ void sqlite3WhereAddScanStatus(
       if( wsFlags & WHERE_INDEXED ){
         sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur);
       }
+    }else{
+      int addr;
+      VdbeOp *pOp;
+      assert( pSrclist->a[pLvl->iFrom].fg.isSubquery );
+      addr = pSrclist->a[pLvl->iFrom].u4.pSubq->addrFillSub;
+      pOp = sqlite3VdbeGetOp(v, addr-1);
+      assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine );
+      assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr );
+      sqlite3VdbeScanStatusRange(v, addrExplain, addr, pOp->p2-1);
     }
   }
 }
@@ -354,7 +407,7 @@ void sqlite3WhereAddScanStatus(
 **
 ** Only the parent term was in the original WHERE clause.  The child1
 ** and child2 terms were added by the LIKE optimization.  If both of
-** the virtual child terms are valid, then testing of the parent can be 
+** the virtual child terms are valid, then testing of the parent can be
 ** skipped.
 **
 ** Usually the parent term is marked as TERM_CODED.  But if the parent
@@ -392,7 +445,7 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){
 
 /*
 ** Code an OP_Affinity opcode to apply the column affinity string zAff
-** to the n registers starting at base. 
+** to the n registers starting at base.
 **
 ** As an optimization, SQLITE_AFF_BLOB and SQLITE_AFF_NONE entries (which
 ** are no-ops) at the beginning and end of zAff are ignored.  If all entries
@@ -429,7 +482,7 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){
 }
 
 /*
-** Expression pRight, which is the RHS of a comparison operation, is 
+** Expression pRight, which is the RHS of a comparison operation, is
 ** either a vector of n elements or, if n==1, a scalar expression.
 ** Before the comparison operation, affinity zAff is to be applied
 ** to the pRight values. This function modifies characters within the
@@ -454,11 +507,44 @@ static void updateRangeAffinityStr(
   }
 }
 
+/*
+** The pOrderBy->a[].u.x.iOrderByCol values might be incorrect because
+** columns might have been rearranged in the result set.  This routine
+** fixes them up.
+**
+** pEList is the new result set.  The pEList->a[].u.x.iOrderByCol values
+** contain the *old* locations of each expression.  This is a temporary
+** use of u.x.iOrderByCol, not its intended use.  The caller must reset
+** u.x.iOrderByCol back to zero for all entries in pEList before the
+** caller returns.
+**
+** This routine changes pOrderBy->a[].u.x.iOrderByCol values from
+** pEList->a[N].u.x.iOrderByCol into N+1.  (The "+1" is because of the 1-based
+** indexing used by iOrderByCol.)  Or if no match, iOrderByCol is set to zero.
+*/
+static void adjustOrderByCol(ExprList *pOrderBy, ExprList *pEList){
+  int i, j;
+  if( pOrderBy==0 ) return;
+  for(i=0; i<pOrderBy->nExpr; i++){
+    int t = pOrderBy->a[i].u.x.iOrderByCol;
+    if( t==0 ) continue;
+    for(j=0; j<pEList->nExpr; j++){
+      if( pEList->a[j].u.x.iOrderByCol==t ){
+        pOrderBy->a[i].u.x.iOrderByCol = j+1;
+        break;
+      }
+    }
+    if( j>=pEList->nExpr ){
+      pOrderBy->a[i].u.x.iOrderByCol = 0;
+    }
+  }
+}
+
 
 /*
 ** pX is an expression of the form:  (vector) IN (SELECT ...)
 ** In other words, it is a vector IN operator with a SELECT clause on the
-** LHS.  But not all terms in the vector are indexable and the terms might
+** RHS.  But not all terms in the vector are indexable and the terms might
 ** not be in the correct order for indexing.
 **
 ** This routine makes a copy of the input pX expression and then adjusts
@@ -514,9 +600,12 @@ static Expr *removeUnindexableInClauseTerms(
           int iField;
           assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 );
           iField = pLoop->aLTerm[i]->u.x.iField - 1;
-          if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */
+          if( NEVER(pOrigRhs->a[iField].pExpr==0) ){
+            continue; /* Duplicate PK column */
+          }
           pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr);
           pOrigRhs->a[iField].pExpr = 0;
+          if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1;
           if( pOrigLhs ){
             assert( pOrigLhs->a[iField].pExpr!=0 );
             pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr);
@@ -530,6 +619,7 @@ static Expr *removeUnindexableInClauseTerms(
         pNew->pLeft->x.pList = pLhs;
       }
       pSelect->pEList = pRhs;
+      pSelect->selId = ++pParse->nSelect; /* Req'd for SubrtnSig validity */
       if( pLhs && pLhs->nExpr==1 ){
         /* Take care here not to generate a TK_VECTOR containing only a
         ** single value. Since the parser never creates such a vector, some
@@ -539,18 +629,16 @@ static Expr *removeUnindexableInClauseTerms(
         sqlite3ExprDelete(db, pNew->pLeft);
         pNew->pLeft = p;
       }
-      if( pSelect->pOrderBy ){
-        /* If the SELECT statement has an ORDER BY clause, zero the 
-        ** iOrderByCol variables. These are set to non-zero when an 
-        ** ORDER BY term exactly matches one of the terms of the 
-        ** result-set. Since the result-set of the SELECT statement may
-        ** have been modified or reordered, these variables are no longer 
-        ** set correctly.  Since setting them is just an optimization, 
-        ** it's easiest just to zero them here.  */
-        ExprList *pOrderBy = pSelect->pOrderBy;
-        for(i=0; i<pOrderBy->nExpr; i++){
-          pOrderBy->a[i].u.x.iOrderByCol = 0;
-        }
+
+      /* If either the ORDER BY clause or the GROUP BY clause contains
+      ** references to result-set columns, those references might now be
+      ** obsolete.  So fix them up.
+      */
+      assert( pRhs!=0 || db->mallocFailed );
+      if( pRhs ){
+        adjustOrderByCol(pSelect->pOrderBy, pRhs);
+        adjustOrderByCol(pSelect->pGroupBy, pRhs);
+        for(i=0; i<pRhs->nExpr; i++) pRhs->a[i].u.x.iOrderByCol = 0;
       }
 
 #if 0
@@ -565,9 +653,141 @@ static Expr *removeUnindexableInClauseTerms(
 }
 
 
+#ifndef SQLITE_OMIT_SUBQUERY
+/*
+** Generate code for a single X IN (....) term of the WHERE clause.
+**
+** This is a special-case of codeEqualityTerm() that works for IN operators
+** only.  It is broken out into a subroutine because this case is
+** uncommon and by splitting it off into a subroutine, the common case
+** runs faster.
+**
+** The current value for the constraint is left in  register iTarget.
+** This routine sets up a loop that will iterate over all values of X.
+*/
+static SQLITE_NOINLINE void codeINTerm(
+  Parse *pParse,      /* The parsing context */
+  WhereTerm *pTerm,   /* The term of the WHERE clause to be coded */
+  WhereLevel *pLevel, /* The level of the FROM clause we are working on */
+  int iEq,            /* Index of the equality term within this level */
+  int bRev,           /* True for reverse-order IN operations */
+  int iTarget         /* Attempt to leave results in this register */
+){
+  Expr *pX = pTerm->pExpr;
+  int eType = IN_INDEX_NOOP;
+  int iTab;
+  struct InLoop *pIn;
+  WhereLoop *pLoop = pLevel->pWLoop;
+  Vdbe *v = pParse->pVdbe;
+  int i;
+  int nEq = 0;
+  int *aiMap = 0;
+
+  if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
+    && pLoop->u.btree.pIndex!=0
+    && pLoop->u.btree.pIndex->aSortOrder[iEq]
+  ){
+    testcase( iEq==0 );
+    testcase( bRev );
+    bRev = !bRev;
+  }
+  assert( pX->op==TK_IN );
+
+  for(i=0; i<iEq; i++){
+    if( pLoop->aLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){
+      disableTerm(pLevel, pTerm);
+      return;
+    }
+  }
+  for(i=iEq; i<pLoop->nLTerm; i++){
+    assert( pLoop->aLTerm[i]!=0 );
+    if( pLoop->aLTerm[i]->pExpr==pX ) nEq++;
+  }
+
+  iTab = 0;
+  if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){
+    eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab);
+  }else{
+    sqlite3 *db = pParse->db;
+    Expr *pXMod = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX);
+    if( !db->mallocFailed ){
+      aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nEq);
+      eType = sqlite3FindInIndex(pParse, pXMod, IN_INDEX_LOOP, 0, aiMap, &iTab);
+    }
+    sqlite3ExprDelete(db, pXMod);
+  }
+
+  if( eType==IN_INDEX_INDEX_DESC ){
+    testcase( bRev );
+    bRev = !bRev;
+  }
+  sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0);
+  VdbeCoverageIf(v, bRev);
+  VdbeCoverageIf(v, !bRev);
+
+  assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 );
+  pLoop->wsFlags |= WHERE_IN_ABLE;
+  if( pLevel->u.in.nIn==0 ){
+    pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse);
+  }
+  if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){
+    pLoop->wsFlags |= WHERE_IN_EARLYOUT;
+  }
+
+  i = pLevel->u.in.nIn;
+  pLevel->u.in.nIn += nEq;
+  pLevel->u.in.aInLoop =
+     sqlite3WhereRealloc(pTerm->pWC->pWInfo,
+                         pLevel->u.in.aInLoop,
+                         sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn);
+  pIn = pLevel->u.in.aInLoop;
+  if( pIn ){
+    int iMap = 0;               /* Index in aiMap[] */
+    pIn += i;
+    for(i=iEq; i<pLoop->nLTerm; i++){
+      if( pLoop->aLTerm[i]->pExpr==pX ){
+        int iOut = iTarget + i - iEq;
+        if( eType==IN_INDEX_ROWID ){
+          pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut);
+        }else{
+          int iCol = aiMap ? aiMap[iMap++] : 0;
+          pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut);
+        }
+        sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v);
+        if( i==iEq ){
+          pIn->iCur = iTab;
+          pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next;
+          if( iEq>0 ){
+            pIn->iBase = iTarget - i;
+            pIn->nPrefix = i;
+          }else{
+            pIn->nPrefix = 0;
+          }
+        }else{
+          pIn->eEndLoopOp = OP_Noop;
+        }
+        pIn++;
+      }
+    }
+    testcase( iEq>0
+              && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
+              && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 );
+    if( iEq>0
+     && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0
+    ){
+      sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq);
+    }
+  }else{
+    pLevel->u.in.nIn = 0;
+  }
+  sqlite3DbFree(pParse->db, aiMap);
+}
+#endif
+
+
 /*
 ** Generate code for a single equality term of the WHERE clause.  An equality
-** term can be either X=expr or X IN (...).   pTerm is the term to be 
+** term can be either X=expr or X IN (...).   pTerm is the term to be
 ** coded.
 **
 ** The current value for the constraint is left in a register, the index
@@ -589,7 +809,6 @@ static int codeEqualityTerm(
   int iTarget         /* Attempt to leave results in this register */
 ){
   Expr *pX = pTerm->pExpr;
-  Vdbe *v = pParse->pVdbe;
   int iReg;                  /* Register holding results */
 
   assert( pLevel->pWLoop->aLTerm[iEq]==pTerm );
@@ -598,125 +817,12 @@ static int codeEqualityTerm(
     iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget);
   }else if( pX->op==TK_ISNULL ){
     iReg = iTarget;
-    sqlite3VdbeAddOp2(v, OP_Null, 0, iReg);
+    sqlite3VdbeAddOp2(pParse->pVdbe, OP_Null, 0, iReg);
 #ifndef SQLITE_OMIT_SUBQUERY
   }else{
-    int eType = IN_INDEX_NOOP;
-    int iTab;
-    struct InLoop *pIn;
-    WhereLoop *pLoop = pLevel->pWLoop;
-    int i;
-    int nEq = 0;
-    int *aiMap = 0;
-
-    if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
-      && pLoop->u.btree.pIndex!=0
-      && pLoop->u.btree.pIndex->aSortOrder[iEq]
-    ){
-      testcase( iEq==0 );
-      testcase( bRev );
-      bRev = !bRev;
-    }
     assert( pX->op==TK_IN );
     iReg = iTarget;
-
-    for(i=0; i<iEq; i++){
-      if( pLoop->aLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){
-        disableTerm(pLevel, pTerm);
-        return iTarget;
-      }
-    }
-    for(i=iEq;i<pLoop->nLTerm; i++){
-      assert( pLoop->aLTerm[i]!=0 );
-      if( pLoop->aLTerm[i]->pExpr==pX ) nEq++;
-    }
-
-    iTab = 0;
-    if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){
-      eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab);
-    }else{
-      Expr *pExpr = pTerm->pExpr;
-      if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){
-        sqlite3 *db = pParse->db;
-        pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX);
-        if( !db->mallocFailed ){
-          aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq);
-          eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab);
-          pExpr->iTable = iTab;
-        }
-        sqlite3ExprDelete(db, pX);
-      }else{
-        int n = sqlite3ExprVectorSize(pX->pLeft);
-        aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n));
-        eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab);
-      }
-      pX = pExpr;
-    }
-
-    if( eType==IN_INDEX_INDEX_DESC ){
-      testcase( bRev );
-      bRev = !bRev;
-    }
-    sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0);
-    VdbeCoverageIf(v, bRev);
-    VdbeCoverageIf(v, !bRev);
-
-    assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 );
-    pLoop->wsFlags |= WHERE_IN_ABLE;
-    if( pLevel->u.in.nIn==0 ){
-      pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse);
-    }
-    if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){
-      pLoop->wsFlags |= WHERE_IN_EARLYOUT;
-    }
-
-    i = pLevel->u.in.nIn;
-    pLevel->u.in.nIn += nEq;
-    pLevel->u.in.aInLoop =
-       sqlite3WhereRealloc(pTerm->pWC->pWInfo,
-                           pLevel->u.in.aInLoop,
-                           sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn);
-    pIn = pLevel->u.in.aInLoop;
-    if( pIn ){
-      int iMap = 0;               /* Index in aiMap[] */
-      pIn += i;
-      for(i=iEq;i<pLoop->nLTerm; i++){
-        if( pLoop->aLTerm[i]->pExpr==pX ){
-          int iOut = iReg + i - iEq;
-          if( eType==IN_INDEX_ROWID ){
-            pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut);
-          }else{
-            int iCol = aiMap ? aiMap[iMap++] : 0;
-            pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut);
-          }
-          sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v);
-          if( i==iEq ){
-            pIn->iCur = iTab;
-            pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next;
-            if( iEq>0 ){
-              pIn->iBase = iReg - i;
-              pIn->nPrefix = i;
-            }else{
-              pIn->nPrefix = 0;
-            }
-          }else{
-            pIn->eEndLoopOp = OP_Noop;
-          }
-          pIn++;
-        }
-      }
-      testcase( iEq>0
-                && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0
-                && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 );
-      if( iEq>0 
-       && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0
-      ){
-        sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq);
-      }
-    }else{
-      pLevel->u.in.nIn = 0;
-    }
-    sqlite3DbFree(pParse->db, aiMap);
+    codeINTerm(pParse, pTerm, pLevel, iEq, bRev, iTarget);
 #endif
   }
 
@@ -745,7 +851,7 @@ static int codeEqualityTerm(
 ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
 ** Suppose the WHERE clause is this:  a==5 AND b IN (1,2,3) AND c>5 AND c<10
 ** The index has as many as three equality constraints, but in this
-** example, the third "c" value is an inequality.  So only two 
+** example, the third "c" value is an inequality.  So only two
 ** constraints are coded.  This routine will generate code to evaluate
 ** a==5 and b IN (1,2,3).  The current values for a and b will be stored
 ** in consecutive registers and the index of the first register is returned.
@@ -812,7 +918,7 @@ static int codeAllEqualityTerms(
   /* Figure out how many memory cells we will need then allocate them.
   */
   regBase = pParse->nMem + 1;
-  nReg = pLoop->u.btree.nEq + nExtraReg;
+  nReg = nEq + nExtraReg;
   pParse->nMem += nReg;
 
   zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx));
@@ -837,7 +943,7 @@ static int codeAllEqualityTerms(
       testcase( pIdx->aiColumn[j]==XN_EXPR );
       VdbeComment((v, "%s", explainIndexColumnName(pIdx, j)));
     }
-  }    
+  }
 
   /* Evaluate the equality constraints
   */
@@ -846,7 +952,7 @@ static int codeAllEqualityTerms(
     int r1;
     pTerm = pLoop->aLTerm[j];
     assert( pTerm!=0 );
-    /* The following testcase is true for indices with redundant columns. 
+    /* The following testcase is true for indices with redundant columns.
     ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */
     testcase( (pTerm->wtFlags & TERM_CODED)!=0 );
     testcase( pTerm->wtFlags & TERM_VIRTUAL );
@@ -859,14 +965,11 @@ static int codeAllEqualityTerms(
         sqlite3VdbeAddOp2(v, OP_Copy, r1, regBase+j);
       }
     }
-  }
-  for(j=nSkip; j<nEq; j++){
-    pTerm = pLoop->aLTerm[j];
     if( pTerm->eOperator & WO_IN ){
       if( pTerm->pExpr->flags & EP_xIsSelect ){
         /* No affinity ever needs to be (or should be) applied to a value
-        ** from the RHS of an "? IN (SELECT ...)" expression. The 
-        ** sqlite3FindInIndex() routine has already ensured that the 
+        ** from the RHS of an "? IN (SELECT ...)" expression. The
+        ** sqlite3FindInIndex() routine has already ensured that the
         ** affinity of the comparison has been applied to the value.  */
         if( zAff ) zAff[j] = SQLITE_AFF_BLOB;
       }
@@ -894,7 +997,7 @@ static int codeAllEqualityTerms(
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
 /*
 ** If the most recently coded instruction is a constant range constraint
-** (a string literal) that originated from the LIKE optimization, then 
+** (a string literal) that originated from the LIKE optimization, then
 ** set P3 and P5 on the OP_String opcode so that the string will be cast
 ** to a BLOB at appropriate times.
 **
@@ -919,7 +1022,7 @@ static void whereLikeOptimizationStringFixup(
     assert( pLevel->iLikeRepCntr>0 );
     pOp = sqlite3VdbeGetLastOp(v);
     assert( pOp!=0 );
-    assert( pOp->opcode==OP_String8 
+    assert( pOp->opcode==OP_String8
             || pTerm->pWC->pWInfo->pParse->db->mallocFailed );
     pOp->p3 = (int)(pLevel->iLikeRepCntr>>1);  /* Register holding counter */
     pOp->p5 = (u8)(pLevel->iLikeRepCntr&1);    /* ASC or DESC */
@@ -962,7 +1065,7 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
 /*
 ** Test whether or not expression pExpr, which was part of a WHERE clause,
 ** should be included in the cursor-hint for a table that is on the rhs
-** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the 
+** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the
 ** expression is not suitable.
 **
 ** An expression is unsuitable if it might evaluate to non NULL even if
@@ -975,9 +1078,9 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
 **   CASE WHEN col THEN 0 ELSE 1 END
 */
 static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
-  if( pExpr->op==TK_IS 
-   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT 
-   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE 
+  if( pExpr->op==TK_IS
+   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT
+   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE
   ){
     pWalker->eCode = 1;
   }else if( pExpr->op==TK_FUNCTION ){
@@ -998,13 +1101,13 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
 ** that accesses any table other than the one identified by
 ** CCurHint.iTabCur, then do the following:
 **
-**   1) allocate a register and code an OP_Column instruction to read 
+**   1) allocate a register and code an OP_Column instruction to read
 **      the specified column into the new register, and
 **
-**   2) transform the expression node to a TK_REGISTER node that reads 
+**   2) transform the expression node to a TK_REGISTER node that reads
 **      from the newly populated register.
 **
-** Also, if the node is a TK_COLUMN that does access the table idenified
+** Also, if the node is a TK_COLUMN that does access the table identified
 ** by pCCurHint.iTabCur, and an index is being used (which we will
 ** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into
 ** an access of the index rather than the original table.
@@ -1073,18 +1176,18 @@ static void codeCursorHint(
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( pTerm->prereqAll & pLevel->notReady ) continue;
 
-    /* Any terms specified as part of the ON(...) clause for any LEFT 
+    /* Any terms specified as part of the ON(...) clause for any LEFT
     ** JOIN for which the current table is not the rhs are omitted
-    ** from the cursor-hint. 
+    ** from the cursor-hint.
     **
-    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms 
+    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms
     ** that were specified as part of the WHERE clause must be excluded.
     ** This is to address the following:
     **
     **   SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL;
     **
     ** Say there is a single row in t2 that matches (t1.a=t2.b), but its
-    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is 
+    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is
     ** pushed down to the cursor, this row is filtered out, causing
     ** SQLite to synthesize a row of NULL values. Which does match the
     ** WHERE clause, and so the query returns a row. Which is incorrect.
@@ -1097,7 +1200,7 @@ static void codeCursorHint(
     */
     if( pTabItem->fg.jointype & JT_LEFT ){
       Expr *pExpr = pTerm->pExpr;
-      if( !ExprHasProperty(pExpr, EP_OuterON) 
+      if( !ExprHasProperty(pExpr, EP_OuterON)
        || pExpr->w.iJoin!=pTabItem->iCursor
       ){
         sWalker.eCode = 0;
@@ -1135,7 +1238,7 @@ static void codeCursorHint(
   if( pExpr!=0 ){
     sWalker.xExprCallback = codeCursorHintFixExpr;
     if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr);
-    sqlite3VdbeAddOp4(v, OP_CursorHint, 
+    sqlite3VdbeAddOp4(v, OP_CursorHint,
                       (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0,
                       (const char*)pExpr, P4_EXPR);
   }
@@ -1147,7 +1250,7 @@ static void codeCursorHint(
 /*
 ** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains
 ** a rowid value just read from cursor iIdxCur, open on index pIdx. This
-** function generates code to do a deferred seek of cursor iCur to the 
+** function generates code to do a deferred seek of cursor iCur to the
 ** rowid stored in register iRowid.
 **
 ** Normally, this is just:
@@ -1181,7 +1284,7 @@ static void codeDeferredSeek(
 
   assert( iIdxCur>0 );
   assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 );
-  
+
   pWInfo->bDeferredSeek = 1;
   sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur);
   if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))
@@ -1292,6 +1395,7 @@ static SQLITE_NOINLINE void filterPullDown(
   int addrNxt,         /* Jump here to bypass inner loops */
   Bitmask notReady     /* Loops that are not ready */
 ){
+  int saved_addrBrk;
   while( ++iLevel < pWInfo->nLevel ){
     WhereLevel *pLevel = &pWInfo->a[iLevel];
     WhereLoop *pLoop = pLevel->pWLoop;
@@ -1300,7 +1404,7 @@ static SQLITE_NOINLINE void filterPullDown(
     /*         ,--- Because sqlite3ConstructBloomFilter() has will not have set
     **  vvvvv--'    pLevel->regFilter if this were true. */
     if( NEVER(pLoop->prereq & notReady) ) continue;
-    assert( pLevel->addrBrk==0 );
+    saved_addrBrk = pLevel->addrBrk;
     pLevel->addrBrk = addrNxt;
     if( pLoop->wsFlags & WHERE_IPK ){
       WhereTerm *pTerm = pLoop->aLTerm[0];
@@ -1330,8 +1434,29 @@ static SQLITE_NOINLINE void filterPullDown(
       VdbeCoverage(pParse->pVdbe);
     }
     pLevel->regFilter = 0;
-    pLevel->addrBrk = 0;
+    pLevel->addrBrk = saved_addrBrk;
+  }
+}
+
+/*
+** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...)
+** operator. Return true if level pLoop is guaranteed to visit only one
+** row for each key generated for the index.
+*/
+static int whereLoopIsOneRow(WhereLoop *pLoop){
+  if( pLoop->u.btree.pIndex->onError
+   && pLoop->nSkip==0
+   && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol
+  ){
+    int ii;
+    for(ii=0; ii<pLoop->u.btree.nEq; ii++){
+      if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){
+        return 0;
+      }
+    }
+    return 1;
   }
+  return 0;
 }
 
 /*
@@ -1356,7 +1481,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   sqlite3 *db;                    /* Database connection */
   SrcItem *pTabItem;              /* FROM clause term being coded */
   int addrBrk;                    /* Jump here to break out of the loop */
-  int addrHalt;                   /* addrBrk for the outermost loop */
   int addrCont;                   /* Jump here to continue with next cycle */
   int iRowidReg = 0;        /* Rowid is stored in this register, if not zero */
   int iReleaseReg = 0;      /* Temp register to free before returning */
@@ -1370,7 +1494,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   iCur = pTabItem->iCursor;
   pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur);
   bRev = (pWInfo->revMask>>iLevel)&1;
-  VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName));
+  VdbeModuleComment((v, "Begin WHERE-loop%d: %s",
+                     iLevel, pTabItem->pSTab->zName));
 #if WHERETRACE_ENABLED /* 0x4001 */
   if( sqlite3WhereTrace & 0x1 ){
     sqlite3DebugPrintf("Coding level %d of %d:  notReady=%llx  iFrom=%d\n",
@@ -1399,7 +1524,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   ** there are no IN operators in the constraints, the "addrNxt" label
   ** is the same as "addrBrk".
   */
-  addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse);
+  addrBrk = pLevel->addrNxt = pLevel->addrBrk;
   addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse);
 
   /* If this is the right table of a LEFT OUTER JOIN, allocate and
@@ -1412,24 +1537,20 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){
     pLevel->iLeftJoin = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin);
-    VdbeComment((v, "init LEFT JOIN no-match flag"));
+    VdbeComment((v, "init LEFT JOIN match flag"));
   }
 
-  /* Compute a safe address to jump to if we discover that the table for
-  ** this loop is empty and can never contribute content. */
-  for(j=iLevel; j>0; j--){
-    if( pWInfo->a[j].iLeftJoin ) break;
-    if( pWInfo->a[j].pRJ ) break;
-  }
-  addrHalt = pWInfo->a[j].addrBrk;
-
   /* Special case of a FROM clause subquery implemented as a co-routine */
   if( pTabItem->fg.viaCoroutine ){
-    int regYield = pTabItem->regReturn;
-    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub);
+    int regYield;
+    Subquery *pSubq;
+    assert( pTabItem->fg.isSubquery && pTabItem->u4.pSubq!=0 );
+    pSubq = pTabItem->u4.pSubq;
+    regYield = pSubq->regReturn;
+    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub);
     pLevel->p2 =  sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk);
     VdbeCoverage(v);
-    VdbeComment((v, "next row of %s", pTabItem->pTab->zName));
+    VdbeComment((v, "next row of %s", pTabItem->pSTab->zName));
     pLevel->op = OP_Goto;
   }else
 
@@ -1474,6 +1595,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     }
     sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg);
     sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1);
+    /* The instruction immediately prior to OP_VFilter must be an OP_Integer
+    ** that sets the "argc" value for xVFilter.  This is necessary for
+    ** resolveP2() to work correctly.  See tag-20250207a. */
     sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg,
                       pLoop->u.vtab.idxStr,
                       pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC);
@@ -1518,7 +1642,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
           }
         }
 
-        /* Generate code that will continue to the next row if 
+        /* Generate code that will continue to the next row if
         ** the IN constraint is not satisfied
         */
         pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0);
@@ -1611,7 +1735,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       int r1, rTemp;        /* Registers for holding the start boundary */
       int op;               /* Cursor seek operation */
 
-      /* The following constant maps TK_xx codes into corresponding 
+      /* The following constant maps TK_xx codes into corresponding
       ** seek opcodes.  It depends on a particular ordering of TK_xx
       */
       const u8 aMoveOp[] = {
@@ -1622,7 +1746,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       };
       assert( TK_LE==TK_GT+1 );      /* Make sure the ordering.. */
       assert( TK_LT==TK_GT+2 );      /*  ... of the TK_xx values... */
-      assert( TK_GE==TK_GT+3 );      /*  ... is correcct. */
+      assert( TK_GE==TK_GT+3 );      /*  ... is correct. */
 
       assert( (pStart->wtFlags & TERM_VNULL)==0 );
       testcase( pStart->wtFlags & TERM_VIRTUAL );
@@ -1654,7 +1778,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       VdbeCoverageIf(v, pX->op==TK_GE);
       sqlite3ReleaseTempReg(pParse, rTemp);
     }else{
-      sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt);
+      sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, pLevel->addrHalt);
       VdbeCoverageIf(v, bRev==0);
       VdbeCoverageIf(v, bRev!=0);
     }
@@ -1667,8 +1791,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       testcase( pEnd->wtFlags & TERM_VIRTUAL );
       memEndValue = ++pParse->nMem;
       codeExprOrVector(pParse, pX->pRight, memEndValue, 1);
-      if( 0==sqlite3ExprIsVector(pX->pRight) 
-       && (pX->op==TK_LT || pX->op==TK_GT) 
+      if( 0==sqlite3ExprIsVector(pX->pRight)
+       && (pX->op==TK_LT || pX->op==TK_GT)
       ){
         testOp = bRev ? OP_Le : OP_Ge;
       }else{
@@ -1694,37 +1818,37 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL);
     }
   }else if( pLoop->wsFlags & WHERE_INDEXED ){
-    /* Case 4: A scan using an index.
+    /* Case 4: Search using an index.
     **
-    **         The WHERE clause may contain zero or more equality 
-    **         terms ("==" or "IN" operators) that refer to the N
-    **         left-most columns of the index. It may also contain
-    **         inequality constraints (>, <, >= or <=) on the indexed
-    **         column that immediately follows the N equalities. Only 
-    **         the right-most column can be an inequality - the rest must
-    **         use the "==" and "IN" operators. For example, if the 
-    **         index is on (x,y,z), then the following clauses are all 
-    **         optimized:
+    ** The WHERE clause may contain zero or more equality
+    ** terms ("==" or "IN" or "IS" operators) that refer to the N
+    ** left-most columns of the index. It may also contain
+    ** inequality constraints (>, <, >= or <=) on the indexed
+    ** column that immediately follows the N equalities. Only
+    ** the right-most column can be an inequality - the rest must
+    ** use the "==", "IN", or "IS" operators. For example, if the
+    ** index is on (x,y,z), then the following clauses are all
+    ** optimized:
     **
-    **            x=5
-    **            x=5 AND y=10
-    **            x=5 AND y<10
-    **            x=5 AND y>5 AND y<10
-    **            x=5 AND y=5 AND z<=10
+    **    x=5
+    **    x=5 AND y=10
+    **    x=5 AND y<10
+    **    x=5 AND y>5 AND y<10
+    **    x=5 AND y=5 AND z<=10
     **
-    **         The z<10 term of the following cannot be used, only
-    **         the x=5 term:
+    ** The z<10 term of the following cannot be used, only
+    ** the x=5 term:
     **
-    **            x=5 AND z<10
+    **    x=5 AND z<10
     **
-    **         N may be zero if there are inequality constraints.
-    **         If there are no inequality constraints, then N is at
-    **         least one.
+    ** N may be zero if there are inequality constraints.
+    ** If there are no inequality constraints, then N is at
+    ** least one.
     **
-    **         This case is also used when there are no WHERE clause
-    **         constraints but an index is selected anyway, in order
-    **         to force the output order to conform to an ORDER BY.
-    */  
+    ** This case is also used when there are no WHERE clause
+    ** constraints but an index is selected anyway, in order
+    ** to force the output order to conform to an ORDER BY.
+    */
     static const u8 aStartOp[] = {
       0,
       0,
@@ -1766,15 +1890,15 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     iIdxCur = pLevel->iIdxCur;
     assert( nEq>=pLoop->nSkip );
 
-    /* Find any inequality constraint terms for the start and end 
-    ** of the range. 
+    /* Find any inequality constraint terms for the start and end
+    ** of the range.
     */
     j = nEq;
     if( pLoop->wsFlags & WHERE_BTM_LIMIT ){
       pRangeStart = pLoop->aLTerm[j++];
       nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm);
       /* Like optimization range constraints always occur in pairs */
-      assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || 
+      assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 ||
               (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 );
     }
     if( pLoop->wsFlags & WHERE_TOP_LIMIT ){
@@ -1807,7 +1931,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 );
 
     /* If the WHERE_BIGNULL_SORT flag is set, then index column nEq uses
-    ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS 
+    ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS
     ** FIRST). In both cases separate ordered scans are made of those
     ** index entries for which the column is null and for those for which
     ** it is not. For an ASC sort, the non-NULL entries are scanned first.
@@ -1829,7 +1953,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     }
 
     /* If we are doing a reverse order scan on an ascending index, or
-    ** a forward order scan on a descending index, interchange the 
+    ** a forward order scan on a descending index, interchange the
     ** start and end terms (pRangeStart and pRangeEnd).
     */
     if( (nEq<pIdx->nColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) ){
@@ -1878,7 +2002,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       }
       if( zStartAff ){
         updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]);
-      }  
+      }
       nConstraint += nBtm;
       testcase( pRangeStart->wtFlags & TERM_VIRTUAL );
       if( sqlite3ExprIsVector(pRight)==0 ){
@@ -1926,7 +2050,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         ** of entries in the tree, so basing the number of steps to try
         ** on the estimated number of rows in the btree seems like a good
         ** guess. */
-        addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, 
+        addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan,
                                          (pIdx->aiRowLogEst[0]+9)/10);
         if( pRangeStart || pRangeEnd ){
           sqlite3VdbeChangeP5(v, 1);
@@ -1951,7 +2075,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         assert( bStopAtNull==startEq );
         sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2);
         op = aStartOp[(nConstraint>1)*4 + 2 + bRev];
-        sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, 
+        sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase,
                              nConstraint-startEq);
         VdbeCoverage(v);
         VdbeCoverageIf(v, op==OP_Rewind);  testcase( op==OP_Rewind );
@@ -2044,7 +2168,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     }
 
     /* Seek the table cursor, if required */
-    omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 
+    omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0
            && (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0;
     if( omitTable ){
       /* pIdx is a covering index.  No need to access the main table. */
@@ -2064,12 +2188,13 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( pLevel->iLeftJoin==0 ){
       /* If a partial index is driving the loop, try to eliminate WHERE clause
       ** terms from the query that must be true due to the WHERE clause of
-      ** the partial index.
+      ** the partial index.  This optimization does not work on an outer join,
+      ** as shown by:
       **
-      ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work
-      ** for a LEFT JOIN.
+      ** 2019-11-02 ticket 623eff57e76d45f6      (LEFT JOIN)
+      ** 2025-05-29 forum post 7dee41d32506c4ae  (RIGHT JOIN)
       */
-      if( pIdx->pPartIdxWhere ){
+      if( pIdx->pPartIdxWhere && pLevel->pRJ==0 ){
         whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC);
       }
     }else{
@@ -2079,9 +2204,11 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       ** a LEFT JOIN: */
       assert( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 );
     }
-  
+
     /* Record the instruction used to terminate the loop. */
-    if( pLoop->wsFlags & WHERE_ONEROW ){
+    if( (pLoop->wsFlags & WHERE_ONEROW)
+     || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop))
+    ){
       pLevel->op = OP_Noop;
     }else if( bRev ){
       pLevel->op = OP_Prev;
@@ -2156,7 +2283,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     int untestedTerms = 0;             /* Some terms not completely tested */
     int ii;                            /* Loop counter */
     Expr *pAndExpr = 0;                /* An ".. AND (...)" expression */
-    Table *pTab = pTabItem->pTab;
+    Table *pTab = pTabItem->pSTab;
 
     pTerm = pLoop->aLTerm[0];
     assert( pTerm!=0 );
@@ -2174,8 +2301,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       int nNotReady;                 /* The number of notReady tables */
       SrcItem *origSrc;              /* Original list of tables */
       nNotReady = pWInfo->nLevel - iLevel - 1;
-      pOrTab = sqlite3DbMallocRawNN(db,
-                            sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0]));
+      pOrTab = sqlite3DbMallocRawNN(db, SZ_SRCLIST(nNotReady+1));
       if( pOrTab==0 ) return notReady;
       pOrTab->nAlloc = (u8)(nNotReady + 1);
       pOrTab->nSrc = pOrTab->nAlloc;
@@ -2188,15 +2314,15 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pOrTab = pWInfo->pTabList;
     }
 
-    /* Initialize the rowset register to contain NULL. An SQL NULL is 
+    /* Initialize the rowset register to contain NULL. An SQL NULL is
     ** equivalent to an empty rowset.  Or, create an ephemeral index
     ** capable of holding primary keys in the case of a WITHOUT ROWID.
     **
-    ** Also initialize regReturn to contain the address of the instruction 
+    ** Also initialize regReturn to contain the address of the instruction
     ** immediately following the OP_Return at the bottom of the loop. This
     ** is required in a few obscure LEFT JOIN cases where control jumps
-    ** over the top of the loop into the body of it. In this case the 
-    ** correct response for the end-of-loop code (the OP_Return) is to 
+    ** over the top of the loop into the body of it. In this case the
+    ** correct response for the end-of-loop code (the OP_Return) is to
     ** fall through to the next instruction, just as an OP_Next does if
     ** called on an uninitialized cursor.
     */
@@ -2221,12 +2347,12 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     **
     ** Actually, each subexpression is converted to "xN AND w" where w is
     ** the "interesting" terms of z - terms that did not originate in the
-    ** ON or USING clause of a LEFT JOIN, and terms that are usable as 
+    ** ON or USING clause of a LEFT JOIN, and terms that are usable as
     ** indices.
     **
     ** This optimization also only applies if the (x1 OR x2 OR ...) term
     ** is not contained in the ON clause of a LEFT JOIN.
-    ** See ticket http://www.sqlite.org/src/info/f2369304e4
+    ** See ticket http://sqlite.org/src/info/f2369304e4
     **
     ** 2022-02-04:  Do not push down slices of a row-value comparison.
     ** In other words, "w" or "y" may not be a slice of a vector.  Otherwise,
@@ -2262,7 +2388,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         /* The extra 0x10000 bit on the opcode is masked off and does not
         ** become part of the new Expr.op.  However, it does make the
         ** op==TK_AND comparison inside of sqlite3PExpr() false, and this
-        ** prevents sqlite3PExpr() from applying the AND short-circuit 
+        ** prevents sqlite3PExpr() from applying the AND short-circuit
         ** optimization, which we do not want here. */
         pAndExpr = sqlite3PExpr(pParse, TK_AND|0x10000, 0, pAndExpr);
       }
@@ -2338,9 +2464,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
               **
               ** Use some of the same optimizations as OP_RowSetTest: If iSet
               ** is zero, assume that the key cannot already be present in
-              ** the temp table. And if iSet is -1, assume that there is no 
-              ** need to insert the key into the temp table, as it will never 
-              ** be tested for.  */ 
+              ** the temp table. And if iSet is -1, assume that there is no
+              ** need to insert the key into the temp table, as it will never
+              ** be tested for.  */
               if( iSet ){
                 jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk);
                 VdbeCoverage(v);
@@ -2379,8 +2505,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
           ** If the call to sqlite3WhereBegin() above resulted in a scan that
           ** uses an index, and this is either the first OR-connected term
           ** processed or the index is the same as that used by all previous
-          ** terms, set pCov to the candidate covering index. Otherwise, set 
-          ** pCov to NULL to indicate that no candidate covering index will 
+          ** terms, set pCov to the candidate covering index. Otherwise, set
+          ** pCov to NULL to indicate that no candidate covering index will
           ** be available.
           */
           pSubLoop = pSubWInfo->a[0].pWLoop;
@@ -2447,7 +2573,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       codeCursorHint(pTabItem, pWInfo, pLevel, 0);
       pLevel->op = aStep[bRev];
       pLevel->p1 = iCur;
-      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt);
+      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev],iCur,pLevel->addrHalt);
       VdbeCoverageIf(v, bRev==0);
       VdbeCoverageIf(v, bRev!=0);
       pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
@@ -2467,10 +2593,16 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   **
   ** iLoop==1: Code only expressions that are entirely covered by pIdx.
   ** iLoop==2: Code remaining expressions that do not contain correlated
-  **           sub-queries.  
+  **           sub-queries.
   ** iLoop==3: Code all remaining expressions.
   **
   ** An effort is made to skip unnecessary iterations of the loop.
+  **
+  ** This optimization of causing simple query restrictions to occur before
+  ** more complex one is call the "push-down" optimization in MySQL.  Here
+  ** in SQLite, the name is "MySQL push-down", since there is also another
+  ** totally unrelated optimization called "WHERE-clause push-down".
+  ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware.
   */
   iLoop = (pIdx ? 1 : 2);
   do{
@@ -2578,7 +2710,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                     WO_EQ|WO_IN|WO_IS, 0);
     if( pAlt==0 ) continue;
     if( pAlt->wtFlags & (TERM_CODED) ) continue;
-    if( (pAlt->eOperator & WO_IN) 
+    if( (pAlt->eOperator & WO_IN)
      && ExprUseXSelect(pAlt->pExpr)
      && (pAlt->pExpr->x.pSelect->pEList->nExpr>1)
     ){
@@ -2609,7 +2741,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ** least once.  This is accomplished by storing the PK for the row in
     ** both the iMatch index and the regBloom Bloom filter.
     */
-    pTab = pWInfo->pTabList->a[pLevel->iFrom].pTab;
+    pTab = pWInfo->pTabList->a[pLevel->iFrom].pSTab;
     if( HasRowid(pTab) ){
       r = sqlite3GetTempRange(pParse, 2);
       sqlite3ExprCodeGetColumnOfTable(v, pTab, pLevel->iTabCur, -1, r+1);
@@ -2636,7 +2768,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   }
 
   /* For a LEFT OUTER JOIN, generate code that will record the fact that
-  ** at least one row of the right table has matched the left table.  
+  ** at least one row of the right table has matched the left table.
   */
   if( pLevel->iLeftJoin ){
     pLevel->addrFirst = sqlite3VdbeCurrentAddr(v);
@@ -2712,16 +2844,33 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop(
   WhereInfo *pSubWInfo;
   WhereLoop *pLoop = pLevel->pWLoop;
   SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
-  SrcList sFrom;
+  SrcList *pFrom;
+  union {
+    SrcList sSrc;
+    u8 fromSpace[SZ_SRCLIST_1];
+  } uSrc;
   Bitmask mAll = 0;
   int k;
 
-  ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pTab->zName));
+  ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pSTab->zName));
   sqlite3VdbeNoJumpsOutsideSubrtn(v, pRJ->addrSubrtn, pRJ->endSubrtn,
                                   pRJ->regReturn);
   for(k=0; k<iLevel; k++){
     int iIdxCur;
+    SrcItem *pRight;
+    assert( pWInfo->a[k].pWLoop->iTab == pWInfo->a[k].iFrom );
+    pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom];
     mAll |= pWInfo->a[k].pWLoop->maskSelf;
+    if( pRight->fg.viaCoroutine ){
+      Subquery *pSubq;
+      assert( pRight->fg.isSubquery && pRight->u4.pSubq!=0 );
+      pSubq = pRight->u4.pSubq;
+      assert( pSubq->pSelect!=0 && pSubq->pSelect->pEList!=0 );
+      sqlite3VdbeAddOp3(
+          v, OP_Null, 0, pSubq->regResult,
+          pSubq->regResult + pSubq->pSelect->pEList->nExpr-1
+      );
+    }
     sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur);
     iIdxCur = pWInfo->a[k].iIdxCur;
     if( iIdxCur ){
@@ -2743,13 +2892,14 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop(
                                  sqlite3ExprDup(pParse->db, pTerm->pExpr, 0));
     }
   }
-  sFrom.nSrc = 1;
-  sFrom.nAlloc = 1;
-  memcpy(&sFrom.a[0], pTabItem, sizeof(SrcItem));
-  sFrom.a[0].fg.jointype = 0;
+  pFrom = &uSrc.sSrc;
+  pFrom->nSrc = 1;
+  pFrom->nAlloc = 1;
+  memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem));
+  pFrom->a[0].fg.jointype = 0;
   assert( pParse->withinRJSubrtn < 100 );
   pParse->withinRJSubrtn++;
-  pSubWInfo = sqlite3WhereBegin(pParse, &sFrom, pSubWhere, 0, 0, 0,
+  pSubWInfo = sqlite3WhereBegin(pParse, pFrom, pSubWhere, 0, 0, 0,
                                 WHERE_RIGHT_JOIN, 0);
   if( pSubWInfo ){
     int iCur = pLevel->iTabCur;
@@ -2757,7 +2907,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop(
     int nPk;
     int jmp;
     int addrCont = sqlite3WhereContinueLabel(pSubWInfo);
-    Table *pTab = pTabItem->pTab;
+    Table *pTab = pTabItem->pSTab;
     if( HasRowid(pTab) ){
       sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r);
       nPk = 1;
diff --git a/src/whereexpr.c b/src/whereexpr.c
index a979b6f2b8..0d99ca85e6 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -13,7 +13,7 @@
 ** the WHERE clause of SQL statements.
 **
 ** This file was originally part of where.c but was split out to improve
-** readability and editabiliity.  This file contains utility routines for
+** readability and editability.  This file contains utility routines for
 ** analyzing Expr objects in the WHERE clause.
 */
 #include "sqliteInt.h"
@@ -101,7 +101,12 @@ static int allowedOp(int op){
   assert( TK_LT>TK_EQ && TK_LT<TK_GE );
   assert( TK_LE>TK_EQ && TK_LE<TK_GE );
   assert( TK_GE==TK_EQ+4 );
-  return op==TK_IN || (op>=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS;
+  assert( TK_IN<TK_EQ );
+  assert( TK_IS<TK_EQ );
+  assert( TK_ISNULL<TK_EQ );
+  if( op>TK_GE ) return 0;
+  if( op>=TK_EQ ) return 1;
+  return op==TK_IN || op==TK_ISNULL || op==TK_IS;
 }
 
 /*
@@ -134,15 +139,16 @@ static u16 exprCommute(Parse *pParse, Expr *pExpr){
 static u16 operatorMask(int op){
   u16 c;
   assert( allowedOp(op) );
-  if( op==TK_IN ){
+  if( op>=TK_EQ ){
+    assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff );
+    c = (u16)(WO_EQ<<(op-TK_EQ));
+  }else if( op==TK_IN ){
     c = WO_IN;
   }else if( op==TK_ISNULL ){
     c = WO_ISNULL;
-  }else if( op==TK_IS ){
-    c = WO_IS;
   }else{
-    assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff );
-    c = (u16)(WO_EQ<<(op-TK_EQ));
+    assert( op==TK_IS );
+    c = WO_IS;
   }
   assert( op!=TK_ISNULL || c==WO_ISNULL );
   assert( op!=TK_IN || c==WO_IN );
@@ -213,12 +219,28 @@ static int isLikeOrGlob(
      z = (u8*)pRight->u.zToken;
   }
   if( z ){
-
-    /* Count the number of prefix characters prior to the first wildcard */
+    /* Count the number of prefix bytes prior to the first wildcard,
+    ** U+fffd character, or malformed utf-8. If the underlying database
+    ** has a UTF16LE encoding, then only consider ASCII characters.  Note that
+    ** the encoding of z[] is UTF8 - we are dealing with only UTF8 here in this
+    ** code, but the database engine itself might be processing content using a
+    ** different encoding. */
     cnt = 0;
     while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){
       cnt++;
-      if( c==wc[3] && z[cnt]!=0 ) cnt++;
+      if( c==wc[3] && z[cnt]>0 && z[cnt]<0x80 ){
+        cnt++;
+      }else if( c>=0x80 ){
+        const u8 *z2 = z+cnt-1;
+        if( c==0xff || sqlite3Utf8Read(&z2)==0xfffd  /* bad utf-8 */
+         || ENC(db)==SQLITE_UTF16LE 
+        ){
+          cnt--;
+          break;
+        }else{
+          cnt = (int)(z2-z);
+        }
+      }
     }
 
     /* The optimization is possible only if (1) the pattern does not begin
@@ -229,11 +251,11 @@ static int isLikeOrGlob(
     ** range search. The third is because the caller assumes that the pattern
     ** consists of at least one character after all escapes have been
     ** removed.  */
-    if( cnt!=0 && 255!=(u8)z[cnt-1] && (cnt>1 || z[0]!=wc[3]) ){
+    if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && ALWAYS(255!=(u8)z[cnt-1]) ){
       Expr *pPrefix;
 
       /* A "complete" match if the pattern ends with "*" or "%" */
-      *pisComplete = c==wc[0] && z[cnt+1]==0;
+      *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE;
 
       /* Get the pattern prefix.  Remove all escapes from the prefix. */
       pPrefix = sqlite3Expr(db, TK_STRING, (char*)z);
@@ -263,8 +285,8 @@ static int isLikeOrGlob(
         **    2019-06-14 https://sqlite.org/src/info/ce8717f0885af975
         **    2019-09-03 https://sqlite.org/src/info/0f0428096f17252a
         */
-        if( pLeft->op!=TK_COLUMN 
-         || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT 
+        if( pLeft->op!=TK_COLUMN
+         || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT
          || (ALWAYS( ExprUseYTab(pLeft) )
              && ALWAYS(pLeft->y.pTab)
              && IsVirtual(pLeft->y.pTab))  /* Might be numeric */
@@ -302,7 +324,7 @@ static int isLikeOrGlob(
           ** function, then no OP_Variable will be added to the program.
           ** This causes problems for the sqlite3_bind_parameter_name()
           ** API. To work around them, add a dummy OP_Variable here.
-          */ 
+          */
           int r1 = sqlite3GetTempReg(pParse);
           sqlite3ExprCodeTarget(pParse, pRight, r1);
           sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0);
@@ -339,7 +361,7 @@ static int isLikeOrGlob(
 **      9.  column IS NOT NULL           SQLITE_INDEX_CONSTRAINT_ISNOTNULL
 **
 ** In every case, "column" must be a column of a virtual table.  If there
-** is a match, set *ppLeft to the "column" expression, set *ppRight to the 
+** is a match, set *ppLeft to the "column" expression, set *ppRight to the
 ** "expr" expression (even though in forms (6) and (8) the column is on the
 ** right and the expression is on the left).  Also set *peOp2 to the
 ** appropriate virtual table operator.  The return value is 1 or 2 if there
@@ -429,6 +451,13 @@ static int isAuxiliaryVtabOperator(
         }
       }
     }
+  }else if( pExpr->op>=TK_EQ ){
+    /* Comparison operators are a common case.  Save a few comparisons for
+    ** that common case by terminating early. */
+    assert( TK_NE < TK_EQ );
+    assert( TK_ISNOT < TK_EQ );
+    assert( TK_NOTNULL < TK_EQ );
+    return 0;
   }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){
     int res = 0;
     Expr *pLeft = pExpr->pLeft;
@@ -507,7 +536,7 @@ static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){
 **
 ** The following is NOT generated:
 **
-**    x<y OR x>y    -->     x!=y     
+**    x<y OR x>y    -->     x!=y    
 */
 static void whereCombineDisjuncts(
   SrcList *pSrc,         /* the FROM clause */
@@ -605,10 +634,10 @@ static void whereCombineDisjuncts(
 **     WhereTerm.u.pOrInfo->indexable  |=  the cursor number for table T
 **
 ** A subterm is "indexable" if it is of the form
-** "T.C <op> <expr>" where C is any column of table T and 
+** "T.C <op> <expr>" where C is any column of table T and
 ** <op> is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN".
 ** A subterm is also indexable if it is an AND of two or more
-** subsubterms at least one of which is indexable.  Indexable AND 
+** subsubterms at least one of which is indexable.  Indexable AND
 ** subterms have their eOperator set to WO_AND and they have
 ** u.pAndInfo set to a dynamically allocated WhereAndTerm object.
 **
@@ -700,7 +729,7 @@ static void exprAnalyzeOrTerm(
         if( !db->mallocFailed ){
           for(j=0, pAndTerm=pAndWC->a; j<pAndWC->nTerm; j++, pAndTerm++){
             assert( pAndTerm->pExpr );
-            if( allowedOp(pAndTerm->pExpr->op) 
+            if( allowedOp(pAndTerm->pExpr->op)
              || pAndTerm->eOperator==WO_AUX
             ){
               b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
@@ -802,7 +831,7 @@ static void exprAnalyzeOrTerm(
                                             pOrTerm->leftCursor))==0 ){
           /* This term must be of the form t1.a==t2.b where t2 is in the
           ** chngToIN set but t1 is not.  This term will be either preceded
-          ** or follwed by an inverted copy (t2.b==t1.a).  Skip this term 
+          ** or followed by an inverted copy (t2.b==t1.a).  Skip this term
           ** and use its inversion. */
           testcase( pOrTerm->wtFlags & TERM_COPIED );
           testcase( pOrTerm->wtFlags & TERM_VIRTUAL );
@@ -833,7 +862,7 @@ static void exprAnalyzeOrTerm(
         assert( (pOrTerm->eOperator & (WO_OR|WO_AND))==0 );
         if( pOrTerm->leftCursor!=iCursor ){
           pOrTerm->wtFlags &= ~TERM_OK;
-        }else if( pOrTerm->u.x.leftColumn!=iColumn || (iColumn==XN_EXPR 
+        }else if( pOrTerm->u.x.leftColumn!=iColumn || (iColumn==XN_EXPR
                && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1)
         )){
           okToChngToIN = 0;
@@ -855,7 +884,7 @@ static void exprAnalyzeOrTerm(
     }
 
     /* At this point, okToChngToIN is true if original pTerm satisfies
-    ** case 1.  In that case, construct a new virtual term that is 
+    ** case 1.  In that case, construct a new virtual term that is
     ** pTerm converted into an IN operator.
     */
     if( okToChngToIN ){
@@ -902,30 +931,42 @@ static void exprAnalyzeOrTerm(
 **   1.  The SQLITE_Transitive optimization must be enabled
 **   2.  Must be either an == or an IS operator
 **   3.  Not originating in the ON clause of an OUTER JOIN
-**   4.  The affinities of A and B must be compatible
-**   5a. Both operands use the same collating sequence OR
-**   5b. The overall collating sequence is BINARY
+**   4.  The operator is not IS or else the query does not contain RIGHT JOIN
+**   5.  The affinities of A and B must be compatible
+**   6a. Both operands use the same collating sequence OR
+**   6b. The overall collating sequence is BINARY
 ** If this routine returns TRUE, that means that the RHS can be substituted
 ** for the LHS anyplace else in the WHERE clause where the LHS column occurs.
 ** This is an optimization.  No harm comes from returning 0.  But if 1 is
 ** returned when it should not be, then incorrect answers might result.
 */
-static int termIsEquivalence(Parse *pParse, Expr *pExpr){
+static int termIsEquivalence(Parse *pParse, Expr *pExpr, SrcList *pSrc){
   char aff1, aff2;
   CollSeq *pColl;
-  if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0;
-  if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0;
-  if( ExprHasProperty(pExpr, EP_OuterON) ) return 0;
+  if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0;  /* (1) */
+  if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0;                 /* (2) */
+  if( ExprHasProperty(pExpr, EP_OuterON) ) return 0;                   /* (3) */
+  assert( pSrc!=0 );
+  if( pExpr->op==TK_IS
+   && pSrc->nSrc>=2
+   && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0
+  ){
+    return 0;                                                          /* (4) */
+  }
   aff1 = sqlite3ExprAffinity(pExpr->pLeft);
   aff2 = sqlite3ExprAffinity(pExpr->pRight);
   if( aff1!=aff2
    && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2))
   ){
-    return 0;
+    return 0;                                                          /* (5) */
   }
   pColl = sqlite3ExprCompareCollSeq(pParse, pExpr);
-  if( sqlite3IsBinary(pColl) ) return 1;
-  return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight);
+  if( !sqlite3IsBinary(pColl)
+   && !sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight)
+  ){
+    return 0;                                                          /* (6) */
+  }
+  return 1;
 }
 
 /*
@@ -945,7 +986,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
     if( ALWAYS(pSrc!=0) ){
       int i;
       for(i=0; i<pSrc->nSrc; i++){
-        mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect);
+        if( pSrc->a[i].fg.isSubquery ){
+          mask |= exprSelectUsage(pMaskSet, pSrc->a[i].u4.pSubq->pSelect);
+        }
         if( pSrc->a[i].fg.isUsing==0 ){
           mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn);
         }
@@ -983,13 +1026,13 @@ static SQLITE_NOINLINE int exprMightBeIndexed2(
   int iCur;
   do{
     iCur = pFrom->a[j].iCursor;
-    for(pIdx=pFrom->a[j].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    for(pIdx=pFrom->a[j].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){
       if( pIdx->aColExpr==0 ) continue;
       for(i=0; i<pIdx->nKeyCol; i++){
         if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
         assert( pIdx->bHasExpr );
-        if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 
-          && pExpr->op!=TK_STRING
+        if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0
+         && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr)
         ){
           aiCurCol[0] = iCur;
           aiCurCol[1] = XN_EXPR;
@@ -1008,8 +1051,8 @@ static int exprMightBeIndexed(
 ){
   int i;
 
-  /* If this expression is a vector to the left or right of a 
-  ** inequality constraint (>, <, >= or <=), perform the processing 
+  /* If this expression is a vector to the left or right of a
+  ** inequality constraint (>, <, >= or <=), perform the processing
   ** on the first element of the vector.  */
   assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE );
   assert( TK_IS<TK_GE && TK_ISNULL<TK_GE && TK_IN<TK_GE );
@@ -1027,7 +1070,7 @@ static int exprMightBeIndexed(
 
   for(i=0; i<pFrom->nSrc; i++){
     Index *pIdx;
-    for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
+    for(pIdx=pFrom->a[i].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){
       if( pIdx->aColExpr ){
         return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i);
       }
@@ -1064,8 +1107,8 @@ static void exprAnalyze(
   WhereTerm *pTerm;                /* The term to be analyzed */
   WhereMaskSet *pMaskSet;          /* Set of table index masks */
   Expr *pExpr;                     /* The expression to be analyzed */
-  Bitmask prereqLeft;              /* Prerequesites of the pExpr->pLeft */
-  Bitmask prereqAll;               /* Prerequesites of pExpr */
+  Bitmask prereqLeft;              /* Prerequisites of the pExpr->pLeft */
+  Bitmask prereqAll;               /* Prerequisites of pExpr */
   Bitmask extraRight = 0;          /* Extra dependencies on LEFT JOIN */
   Expr *pStr1 = 0;                 /* RHS of LIKE/GLOB operator */
   int isComplete = 0;              /* RHS of LIKE/GLOB ends with wildcard */
@@ -1081,6 +1124,9 @@ static void exprAnalyze(
   }
   assert( pWC->nTerm > idxTerm );
   pTerm = &pWC->a[idxTerm];
+#ifdef SQLITE_DEBUG
+  pTerm->iTerm = idxTerm;
+#endif
   pMaskSet = &pWInfo->sMaskSet;
   pExpr = pTerm->pExpr;
   assert( pExpr!=0 ); /* Because malloc() has not failed */
@@ -1124,21 +1170,7 @@ static void exprAnalyze(
       prereqAll |= x;
       extraRight = x-1;  /* ON clause terms may not be used with an index
                          ** on left table of a LEFT JOIN.  Ticket #3015 */
-      if( (prereqAll>>1)>=x ){
-        sqlite3ErrorMsg(pParse, "ON clause references tables to its right");
-        return;
-      }
     }else if( (prereqAll>>1)>=x ){
-      /* The ON clause of an INNER JOIN references a table to its right.
-      ** Most other SQL database engines raise an error.  But SQLite versions
-      ** 3.0 through 3.38 just put the ON clause constraint into the WHERE
-      ** clause and carried on.   Beginning with 3.39, raise an error only
-      ** if there is a RIGHT or FULL JOIN in the query.  This makes SQLite
-      ** more like other systems, and also preserves legacy. */
-      if( ALWAYS(pSrc->nSrc>0) && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){
-        sqlite3ErrorMsg(pParse, "ON clause references tables to its right");
-        return;
-      }
       ExprClearProperty(pExpr, EP_InnerON);
     }
   }
@@ -1166,7 +1198,7 @@ static void exprAnalyze(
       pTerm->eOperator = operatorMask(op) & opMask;
     }
     if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
-    if( pRight 
+    if( pRight
      && exprMightBeIndexed(pSrc, aiCurCol, pRight, op)
      && !ExprHasProperty(pRight, EP_FixedCol)
     ){
@@ -1188,8 +1220,8 @@ static void exprAnalyze(
         if( op==TK_IS ) pNew->wtFlags |= TERM_IS;
         pTerm = &pWC->a[idxTerm];
         pTerm->wtFlags |= TERM_COPIED;
-
-        if( termIsEquivalence(pParse, pDup) ){
+        assert( pWInfo->pTabList!=0 );
+        if( termIsEquivalence(pParse, pDup, pWInfo->pTabList) ){
           pTerm->eOperator |= WO_EQUIV;
           eExtraOp = WO_EQUIV;
         }
@@ -1205,7 +1237,7 @@ static void exprAnalyze(
       pNew->prereqRight = prereqLeft | extraRight;
       pNew->prereqAll = prereqAll;
       pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask;
-    }else 
+    }else
     if( op==TK_ISNULL
      && !ExprHasProperty(pExpr,EP_OuterON)
      && 0==sqlite3ExprCanBeNull(pLeft)
@@ -1246,7 +1278,7 @@ static void exprAnalyze(
     for(i=0; i<2; i++){
       Expr *pNewExpr;
       int idxNew;
-      pNewExpr = sqlite3PExpr(pParse, ops[i], 
+      pNewExpr = sqlite3PExpr(pParse, ops[i],
                              sqlite3ExprDup(db, pExpr->pLeft, 0),
                              sqlite3ExprDup(db, pList->a[i].pExpr, 0));
       transferJoinMarkings(pNewExpr, pExpr);
@@ -1284,11 +1316,11 @@ static void exprAnalyze(
       Expr *pLeft = pExpr->pLeft;
       int idxNew;
       WhereTerm *pNewTerm;
-  
+ 
       pNewExpr = sqlite3PExpr(pParse, TK_GT,
                               sqlite3ExprDup(db, pLeft, 0),
                               sqlite3ExprAlloc(db, TK_NULL, 0, 0));
-  
+ 
       idxNew = whereClauseInsert(pWC, pNewExpr,
                                 TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL);
       if( idxNew ){
@@ -1338,7 +1370,7 @@ static void exprAnalyze(
     pStr2 = sqlite3ExprDup(db, pStr1, 0);
     assert( pStr1==0 || !ExprHasProperty(pStr1, EP_IntValue) );
     assert( pStr2==0 || !ExprHasProperty(pStr2, EP_IntValue) );
- 
+
 
     /* Convert the lower bound to upper-case and the upper bound to
     ** lower-case (upper-case is less than lower-case in ASCII) so that
@@ -1355,20 +1387,26 @@ static void exprAnalyze(
     }
 
     if( !db->mallocFailed ){
-      u8 c, *pC;       /* Last character before the first wildcard */
+      u8 *pC;       /* Last character before the first wildcard */
       pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1];
-      c = *pC;
       if( noCase ){
         /* The point is to increment the last character before the first
         ** wildcard.  But if we increment '@', that will push it into the
-        ** alphabetic range where case conversions will mess up the 
+        ** alphabetic range where case conversions will mess up the
         ** inequality.  To avoid this, make sure to also run the full
         ** LIKE on all candidate expressions by clearing the isComplete flag
         */
-        if( c=='A'-1 ) isComplete = 0;
-        c = sqlite3UpperToLower[c];
+        if( *pC=='A'-1 ) isComplete = 0;
+        *pC = sqlite3UpperToLower[*pC];
       }
-      *pC = c + 1;
+
+      /* Increment the value of the last utf8 character in the prefix. */
+      while( *pC==0xBF && pC>(u8*)pStr2->u.zToken ){
+        *pC = 0x80;
+        pC--;
+      }
+      assert( *pC!=0xFF );        /* isLikeOrGlob() guarantees this */
+      (*pC)++;
     }
     zCollSeqName = noCase ? "NOCASE" : sqlite3StrBINARY;
     pNewExpr1 = sqlite3ExprDup(db, pLeft, 0);
@@ -1408,7 +1446,7 @@ static void exprAnalyze(
   if( (pExpr->op==TK_EQ || pExpr->op==TK_IS)
    && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1
    && sqlite3ExprVectorSize(pExpr->pRight)==nLeft
-   && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 
+   && ( (pExpr->pLeft->flags & EP_xIsSelect)==0
      || (pExpr->pRight->flags & EP_xIsSelect)==0)
    && pWC->op==TK_AND
   ){
@@ -1431,7 +1469,7 @@ static void exprAnalyze(
 
   /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create
   ** a virtual term for each vector component. The expression object
-  ** used by each such virtual term is pExpr (the full vector IN(...) 
+  ** used by each such virtual term is pExpr (the full vector IN(...)
   ** expression). The WhereTerm.u.x.iField variable identifies the index within
   ** the vector on the LHS that the virtual term represents.
   **
@@ -1480,7 +1518,7 @@ static void exprAnalyze(
       prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft);
       if( (prereqExpr & prereqColumn)==0 ){
         Expr *pNewExpr;
-        pNewExpr = sqlite3PExpr(pParse, TK_MATCH, 
+        pNewExpr = sqlite3PExpr(pParse, TK_MATCH,
             0, sqlite3ExprDup(db, pRight, 0));
         if( ExprHasProperty(pExpr, EP_OuterON) && pNewExpr ){
           ExprSetProperty(pNewExpr, EP_OuterON);
@@ -1489,7 +1527,7 @@ static void exprAnalyze(
         idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
         testcase( idxNew==0 );
         pNewTerm = &pWC->a[idxNew];
-        pNewTerm->prereqRight = prereqExpr;
+        pNewTerm->prereqRight = prereqExpr | extraRight;
         pNewTerm->leftCursor = pLeft->iTable;
         pNewTerm->u.x.leftColumn = pLeft->iColumn;
         pNewTerm->eOperator = WO_AUX;
@@ -1548,12 +1586,12 @@ void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){
 }
 
 /*
-** Add either a LIMIT (if eMatchOp==SQLITE_INDEX_CONSTRAINT_LIMIT) or 
-** OFFSET (if eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET) term to the 
+** Add either a LIMIT (if eMatchOp==SQLITE_INDEX_CONSTRAINT_LIMIT) or
+** OFFSET (if eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET) term to the
 ** where-clause passed as the first argument. The value for the term
 ** is found in register iReg.
 **
-** In the common case where the value is a simple integer 
+** In the common case where the value is a simple integer
 ** (example: "LIMIT 5 OFFSET 10") then the expression codes as a
 ** TK_INTEGER so that it will be available to sqlite3_vtab_rhs_value().
 ** If not, then it codes as a TK_REGISTER expression.
@@ -1570,7 +1608,7 @@ static void whereAddLimitExpr(
   Expr *pNew;
   int iVal = 0;
 
-  if( sqlite3ExprIsInteger(pExpr, &iVal) && iVal>=0 ){
+  if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){
     Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0);
     if( pVal==0 ) return;
     ExprSetProperty(pVal, EP_IntValue);
@@ -1600,7 +1638,7 @@ static void whereAddLimitExpr(
 **
 **   1. The SELECT statement has a LIMIT clause, and
 **   2. The SELECT statement is not an aggregate or DISTINCT query, and
-**   3. The SELECT statement has exactly one object in its from clause, and
+**   3. The SELECT statement has exactly one object in its FROM clause, and
 **      that object is a virtual table, and
 **   4. There are no terms in the WHERE clause that will not be passed
 **      to the virtual table xBestIndex method.
@@ -1615,7 +1653,7 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){
   assert( p!=0 && p->pLimit!=0 );                 /* 1 -- checked by caller */
   if( p->pGroupBy==0
    && (p->selFlags & (SF_Distinct|SF_Aggregate))==0             /* 2 */
-   && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pTab))       /* 3 */
+   && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pSTab))      /* 3 */
   ){
     ExprList *pOrderBy = p->pOrderBy;
     int iCsr = p->pSrc->a[0].iCursor;
@@ -1633,11 +1671,26 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){
       if( pWC->a[ii].nChild ){
         /* If this term has child terms, then they are also part of the
         ** pWC->a[] array. So this term can be ignored, as a LIMIT clause
-        ** will only be added if each of the child terms passes the 
+        ** will only be added if each of the child terms passes the
         ** (leftCursor==iCsr) test below.  */
         continue;
       }
-      if( pWC->a[ii].leftCursor!=iCsr ) return;
+      if( pWC->a[ii].leftCursor==iCsr && pWC->a[ii].prereqRight==0 ) continue;
+
+      /* If this term has a parent with exactly one child, and the parent will
+      ** be passed through to xBestIndex, then this term can be ignored.  */
+      if( pWC->a[ii].iParent>=0 ){
+        WhereTerm *pParent = &pWC->a[ pWC->a[ii].iParent ];
+        if( pParent->leftCursor==iCsr
+         && pParent->prereqRight==0
+         && pParent->nChild==1
+        ){
+          continue;
+        }
+      }
+
+      /* This term will not be passed through. Do not add a LIMIT clause. */
+      return;
     }
 
     /* Check condition (5). Return early if it is not met. */
@@ -1652,12 +1705,14 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){
 
     /* All conditions are met. Add the terms to the where-clause object. */
     assert( p->pLimit->op==TK_LIMIT );
-    whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft,
-                      iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT);
-    if( p->iOffset>0 ){
+    if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){
       whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight,
                         iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET);
     }
+    if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){
+      whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft,
+                        iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT);
+    }
   }
 }
 
@@ -1798,7 +1853,7 @@ Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){
 
 
 /*
-** Call exprAnalyze on all terms in a WHERE clause.  
+** Call exprAnalyze on all terms in a WHERE clause. 
 **
 ** Note that exprAnalyze() might add new virtual terms onto the
 ** end of the WHERE clause.  We do not want to analyze these new
@@ -1817,7 +1872,7 @@ void sqlite3WhereExprAnalyze(
 
 /*
 ** For table-valued-functions, transform the function arguments into
-** new WHERE clause terms.  
+** new WHERE clause terms. 
 **
 ** Each function argument translates into an equality constraint against
 ** a HIDDEN column in the table.
@@ -1833,7 +1888,7 @@ void sqlite3WhereTabFuncArgs(
   Expr *pColRef;
   Expr *pTerm;
   if( pItem->fg.isTabFunc==0 ) return;
-  pTab = pItem->pTab;
+  pTab = pItem->pSTab;
   assert( pTab!=0 );
   pArgs = pItem->u1.pFuncArg;
   if( pArgs==0 ) return;
@@ -1853,7 +1908,7 @@ void sqlite3WhereTabFuncArgs(
     assert( ExprUseYTab(pColRef) );
     pColRef->y.pTab = pTab;
     pItem->colUsed |= sqlite3ExprColUsed(pColRef);
-    pRhs = sqlite3PExpr(pParse, TK_UPLUS, 
+    pRhs = sqlite3PExpr(pParse, TK_UPLUS,
         sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0);
     pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs);
     if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){
diff --git a/src/window.c b/src/window.c
index a8081aa244..1f22ab1945 100644
--- a/src/window.c
+++ b/src/window.c
@@ -40,12 +40,12 @@
 **   (in this case max()) to process rows sorted in order of (c, d), which
 **   makes things easier for obvious reasons. More generally:
 **
-**     * FROM, WHERE, GROUP BY and HAVING clauses are all moved to 
+**     * FROM, WHERE, GROUP BY and HAVING clauses are all moved to
 **       the sub-query.
 **
 **     * ORDER BY, LIMIT and OFFSET remain part of the parent query.
 **
-**     * Terminals from each of the expression trees that make up the 
+**     * Terminals from each of the expression trees that make up the
 **       select-list and ORDER BY expressions in the parent query are
 **       selected by the sub-query. For the purposes of the transformation,
 **       terminals are column references and aggregate functions.
@@ -54,14 +54,14 @@
 **   the same window declaration (the OVER bit), then a single scan may
 **   be used to process more than one window function. For example:
 **
-**     SELECT max(b) OVER (PARTITION BY c ORDER BY d), 
-**            min(e) OVER (PARTITION BY c ORDER BY d) 
+**     SELECT max(b) OVER (PARTITION BY c ORDER BY d),
+**            min(e) OVER (PARTITION BY c ORDER BY d)
 **     FROM t1;
 **
 **   is transformed in the same way as the example above. However:
 **
-**     SELECT max(b) OVER (PARTITION BY c ORDER BY d), 
-**            min(e) OVER (PARTITION BY a ORDER BY b) 
+**     SELECT max(b) OVER (PARTITION BY c ORDER BY d),
+**            min(e) OVER (PARTITION BY a ORDER BY b)
 **     FROM t1;
 **
 **   Must be transformed to:
@@ -114,15 +114,15 @@
 **     first_value(expr)
 **     last_value(expr)
 **     nth_value(expr, N)
-**   
-**   These are the same built-in window functions supported by Postgres. 
+**  
+**   These are the same built-in window functions supported by Postgres.
 **   Although the behaviour of aggregate window functions (functions that
-**   can be used as either aggregates or window funtions) allows them to
+**   can be used as either aggregates or window functions) allows them to
 **   be implemented using an API, built-in window functions are much more
-**   esoteric. Additionally, some window functions (e.g. nth_value()) 
+**   esoteric. Additionally, some window functions (e.g. nth_value())
 **   may only be implemented by caching the entire partition in memory.
 **   As such, some built-in window functions use the same API as aggregate
-**   window functions and some are implemented directly using VDBE 
+**   window functions and some are implemented directly using VDBE
 **   instructions. Additionally, for those functions that use the API, the
 **   window frame is sometimes modified before the SELECT statement is
 **   rewritten. For example, regardless of the specified window frame, the
@@ -134,7 +134,7 @@
 **
 **   As well as some of the built-in window functions, aggregate window
 **   functions min() and max() are implemented using VDBE instructions if
-**   the start of the window frame is declared as anything other than 
+**   the start of the window frame is declared as anything other than
 **   UNBOUNDED PRECEDING.
 */
 
@@ -145,7 +145,7 @@
 **   ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
 */
 static void row_numberStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -173,10 +173,10 @@ struct CallCount {
 ** Implementation of built-in window function dense_rank(). Assumes that
 ** the window frame has been set to:
 **
-**   RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW 
+**   RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
 */
 static void dense_rankStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -208,7 +208,7 @@ struct NthValueCtx {
   sqlite3_value *pValue;
 };
 static void nth_valueStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -261,7 +261,7 @@ static void nth_valueFinalizeFunc(sqlite3_context *pCtx){
 #define nth_valueValueFunc noopValueFunc
 
 static void first_valueStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -292,10 +292,10 @@ static void first_valueFinalizeFunc(sqlite3_context *pCtx){
 ** Implementation of built-in window function rank(). Assumes that
 ** the window frame has been set to:
 **
-**   RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW 
+**   RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
 */
 static void rankStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -326,7 +326,7 @@ static void rankValueFunc(sqlite3_context *pCtx){
 **   GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
 */
 static void percent_rankStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -339,7 +339,7 @@ static void percent_rankStepFunc(
   }
 }
 static void percent_rankInvFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -371,7 +371,7 @@ static void percent_rankValueFunc(sqlite3_context *pCtx){
 **   GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING
 */
 static void cume_distStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -384,7 +384,7 @@ static void cume_distStepFunc(
   }
 }
 static void cume_distInvFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -420,7 +420,7 @@ struct NtileCtx {
 **   ROWS CURRENT ROW AND UNBOUNDED FOLLOWING
 */
 static void ntileStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -440,7 +440,7 @@ static void ntileStepFunc(
   }
 }
 static void ntileInvFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -486,7 +486,7 @@ struct LastValueCtx {
 ** Implementation of last_value().
 */
 static void last_valueStepFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -504,7 +504,7 @@ static void last_valueStepFunc(
   }
 }
 static void last_valueInvFunc(
-  sqlite3_context *pCtx, 
+  sqlite3_context *pCtx,
   int nArg,
   sqlite3_value **apArg
 ){
@@ -647,7 +647,7 @@ static Window *windowFind(Parse *pParse, Window *pList, const char *zName){
 ** is the Window object representing the associated OVER clause. This
 ** function updates the contents of pWin as follows:
 **
-**   * If the OVER clause refered to a named window (as in "max(x) OVER win"),
+**   * If the OVER clause referred to a named window (as in "max(x) OVER win"),
 **     search list pList for a matching WINDOW definition, and update pWin
 **     accordingly. If no such WINDOW clause can be found, leave an error
 **     in pParse.
@@ -657,7 +657,7 @@ static Window *windowFind(Parse *pParse, Window *pList, const char *zName){
 **     of this file), pWin is updated here.
 */
 void sqlite3WindowUpdate(
-  Parse *pParse, 
+  Parse *pParse,
   Window *pList,                  /* List of named windows for this SELECT */
   Window *pWin,                   /* Window frame to update */
   FuncDef *pFunc                  /* Window function definition */
@@ -677,17 +677,17 @@ void sqlite3WindowUpdate(
     sqlite3WindowChain(pParse, pWin, pList);
   }
   if( (pWin->eFrmType==TK_RANGE)
-   && (pWin->pStart || pWin->pEnd) 
+   && (pWin->pStart || pWin->pEnd)
    && (pWin->pOrderBy==0 || pWin->pOrderBy->nExpr!=1)
   ){
-    sqlite3ErrorMsg(pParse, 
+    sqlite3ErrorMsg(pParse,
       "RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression"
     );
   }else
   if( pFunc->funcFlags & SQLITE_FUNC_WINDOW ){
     sqlite3 *db = pParse->db;
     if( pWin->pFilter ){
-      sqlite3ErrorMsg(pParse, 
+      sqlite3ErrorMsg(pParse,
           "FILTER clause may only be used with aggregate window functions"
       );
     }else{
@@ -697,14 +697,14 @@ void sqlite3WindowUpdate(
         int eStart;
         int eEnd;
       } aUp[] = {
-        { row_numberName,   TK_ROWS,   TK_UNBOUNDED, TK_CURRENT }, 
-        { dense_rankName,   TK_RANGE,  TK_UNBOUNDED, TK_CURRENT }, 
-        { rankName,         TK_RANGE,  TK_UNBOUNDED, TK_CURRENT }, 
-        { percent_rankName, TK_GROUPS, TK_CURRENT,   TK_UNBOUNDED }, 
-        { cume_distName,    TK_GROUPS, TK_FOLLOWING, TK_UNBOUNDED }, 
-        { ntileName,        TK_ROWS,   TK_CURRENT,   TK_UNBOUNDED }, 
-        { leadName,         TK_ROWS,   TK_UNBOUNDED, TK_UNBOUNDED }, 
-        { lagName,          TK_ROWS,   TK_UNBOUNDED, TK_CURRENT }, 
+        { row_numberName,   TK_ROWS,   TK_UNBOUNDED, TK_CURRENT },
+        { dense_rankName,   TK_RANGE,  TK_UNBOUNDED, TK_CURRENT },
+        { rankName,         TK_RANGE,  TK_UNBOUNDED, TK_CURRENT },
+        { percent_rankName, TK_GROUPS, TK_CURRENT,   TK_UNBOUNDED },
+        { cume_distName,    TK_GROUPS, TK_FOLLOWING, TK_UNBOUNDED },
+        { ntileName,        TK_ROWS,   TK_CURRENT,   TK_UNBOUNDED },
+        { leadName,         TK_ROWS,   TK_UNBOUNDED, TK_UNBOUNDED },
+        { lagName,          TK_ROWS,   TK_UNBOUNDED, TK_CURRENT },
       };
       int i;
       for(i=0; i<ArraySize(aUp); i++){
@@ -742,7 +742,7 @@ struct WindowRewrite {
 
 /*
 ** Callback function used by selectWindowRewriteEList(). If necessary,
-** this function appends to the output expression-list and updates 
+** this function appends to the output expression-list and updates
 ** expression (*ppExpr) in place.
 */
 static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){
@@ -847,16 +847,16 @@ static int selectWindowRewriteSelectCb(Walker *pWalker, Select *pSelect){
 **
 **   * TK_COLUMN,
 **   * aggregate function, or
-**   * window function with a Window object that is not a member of the 
+**   * window function with a Window object that is not a member of the
 **     Window list passed as the second argument (pWin).
 **
 ** Append the node to output expression-list (*ppSub). And replace it
-** with a TK_COLUMN that reads the (N-1)th element of table 
+** with a TK_COLUMN that reads the (N-1)th element of table
 ** pWin->iEphCsr, where N is the number of elements in (*ppSub) after
 ** appending the new one.
 */
 static void selectWindowRewriteEList(
-  Parse *pParse, 
+  Parse *pParse,
   Window *pWin,
   SrcList *pSrc,
   ExprList *pEList,               /* Rewrite expressions in this list */
@@ -909,7 +909,7 @@ static ExprList *exprListAppendList(
         int iDummy;
         Expr *pSub;
         pSub = sqlite3ExprSkipCollateAndLikely(pDup);
-        if( sqlite3ExprIsInteger(pSub, &iDummy) ){
+        if( sqlite3ExprIsInteger(pSub, &iDummy, 0) ){
           pSub->op = TK_NULL;
           pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse);
           pSub->u.zToken = 0;
@@ -950,7 +950,7 @@ static int disallowAggregatesInOrderByCb(Walker *pWalker, Expr *pExpr){
 
 /*
 ** If the SELECT statement passed as the second argument does not invoke
-** any SQL window functions, this function is a no-op. Otherwise, it 
+** any SQL window functions, this function is a no-op. Otherwise, it
 ** rewrites the SELECT statement so that window function xStep functions
 ** are invoked in the correct order as described under "SELECT REWRITING"
 ** at the top of this file.
@@ -995,7 +995,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
     p->pWhere = 0;
     p->pGroupBy = 0;
     p->pHaving = 0;
-    p->selFlags &= ~SF_Aggregate;
+    p->selFlags &= ~(u32)SF_Aggregate;
     p->selFlags |= SF_WinRewrite;
 
     /* Create the ORDER BY clause for the sub-select. This is the concatenation
@@ -1023,8 +1023,8 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
     selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, pTab, &pSublist);
     pMWin->nBufferCol = (pSublist ? pSublist->nExpr : 0);
 
-    /* Append the PARTITION BY and ORDER BY expressions to the to the 
-    ** sub-select expression list. They are required to figure out where 
+    /* Append the PARTITION BY and ORDER BY expressions to the to the
+    ** sub-select expression list. They are required to figure out where
     ** boundaries for partitions and sets of peer rows lie.  */
     pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition, 0);
     pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy, 0);
@@ -1038,7 +1038,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
       assert( ExprUseXList(pWin->pOwner) );
       assert( pWin->pWFunc!=0 );
       pArgs = pWin->pOwner->x.pList;
-      if( pWin->pWFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){
+      if( pWin->pWFunc->funcFlags & SQLITE_SUBTYPE ){
         selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist);
         pWin->iArgCol = (pSublist ? pSublist->nExpr : 0);
         pWin->bExprArgs = 1;
@@ -1058,11 +1058,11 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
     /* If there is no ORDER BY or PARTITION BY clause, and the window
     ** function accepts zero arguments, and there are no other columns
     ** selected (e.g. "SELECT row_number() OVER () FROM t1"), it is possible
-    ** that pSublist is still NULL here. Add a constant expression here to 
-    ** keep everything legal in this case. 
+    ** that pSublist is still NULL here. Add a constant expression here to
+    ** keep everything legal in this case.
     */
     if( pSublist==0 ){
-      pSublist = sqlite3ExprListAppend(pParse, 0, 
+      pSublist = sqlite3ExprListAppend(pParse, 0,
         sqlite3Expr(db, TK_INTEGER, "0")
       );
     }
@@ -1077,9 +1077,10 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
     assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside
                                      ** of sqlite3DbMallocRawNN() called from
                                      ** sqlite3SrcListAppend() */
-    if( p->pSrc ){
+    if( p->pSrc==0 ){
+      sqlite3SelectDelete(db, pSub);
+    }else if( sqlite3SrcItemAttachSubquery(pParse, &p->pSrc->a[0], pSub, 0) ){
       Table *pTab2;
-      p->pSrc->a[0].pSelect = pSub;
       p->pSrc->a[0].fg.isCorrelated = 1;
       sqlite3SrcListAssignCursors(pParse, p->pSrc);
       pSub->selFlags |= SF_Expanded|SF_OrderByReqd;
@@ -1093,7 +1094,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
       }else{
         memcpy(pTab, pTab2, sizeof(Table));
         pTab->tabFlags |= TF_Ephemeral;
-        p->pSrc->a[0].pTab = pTab;
+        p->pSrc->a[0].pSTab = pTab;
         pTab = pTab2;
         memset(&w, 0, sizeof(w));
         w.xExprCallback = sqlite3WindowExtraAggFuncDepth;
@@ -1101,8 +1102,6 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){
         w.xSelectCallback2 = sqlite3WalkerDepthDecrease;
         sqlite3WalkSelect(&w, pSub);
       }
-    }else{
-      sqlite3SelectDelete(db, pSub);
     }
     if( db->mallocFailed ) rc = SQLITE_NOMEM;
 
@@ -1164,7 +1163,7 @@ void sqlite3WindowListDelete(sqlite3 *db, Window *p){
 ** variable values in the expression tree.
 */
 static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){
-  if( 0==sqlite3ExprIsConstant(pExpr) ){
+  if( 0==sqlite3ExprIsConstant(0,pExpr) ){
     if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr);
     sqlite3ExprDelete(pParse->db, pExpr);
     pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0);
@@ -1248,10 +1247,10 @@ Window *sqlite3WindowAlloc(
 ** equivalent nul-terminated string.
 */
 Window *sqlite3WindowAssemble(
-  Parse *pParse, 
-  Window *pWin, 
-  ExprList *pPartition, 
-  ExprList *pOrderBy, 
+  Parse *pParse,
+  Window *pWin,
+  ExprList *pPartition,
+  ExprList *pOrderBy,
   Token *pBase
 ){
   if( pWin ){
@@ -1268,7 +1267,7 @@ Window *sqlite3WindowAssemble(
 }
 
 /*
-** Window *pWin has just been created from a WINDOW clause. Tokne pBase
+** Window *pWin has just been created from a WINDOW clause. Token pBase
 ** is the base window. Earlier windows from the same WINDOW clause are
 ** stored in the linked list starting at pWin->pNextWin. This function
 ** either updates *pWin according to the base specification, or else
@@ -1289,7 +1288,7 @@ void sqlite3WindowChain(Parse *pParse, Window *pWin, Window *pList){
         zErr = "frame specification";
       }
       if( zErr ){
-        sqlite3ErrorMsg(pParse, 
+        sqlite3ErrorMsg(pParse,
             "cannot override %s of window: %s", zErr, pWin->zBase
         );
       }else{
@@ -1312,8 +1311,9 @@ void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){
   if( p ){
     assert( p->op==TK_FUNCTION );
     assert( pWin );
+    assert( ExprIsFullSize(p) );
     p->y.pWin = pWin;
-    ExprSetProperty(p, EP_WinFunc);
+    ExprSetProperty(p, EP_WinFunc|EP_FullSize);
     pWin->pOwner = p;
     if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){
       sqlite3ErrorMsg(pParse,
@@ -1388,10 +1388,15 @@ int sqlite3WindowCompare(
 ** and initialize registers and cursors used by sqlite3WindowCodeStep().
 */
 void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){
-  int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr;
-  Window *pMWin = pSelect->pWin;
   Window *pWin;
-  Vdbe *v = sqlite3GetVdbe(pParse);
+  int nEphExpr;
+  Window *pMWin;
+  Vdbe *v;
+
+  assert( pSelect->pSrc->a[0].fg.isSubquery );
+  nEphExpr = pSelect->pSrc->a[0].u4.pSubq->pSelect->pEList->nExpr;
+  pMWin = pSelect->pWin;
+  v = sqlite3GetVdbe(pParse);
 
   sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr);
   sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr);
@@ -1540,7 +1545,7 @@ struct WindowCsrAndReg {
 };
 
 /*
-** A single instance of this structure is allocated on the stack by 
+** A single instance of this structure is allocated on the stack by
 ** sqlite3WindowCodeStep() and a pointer to it passed to the various helper
 ** routines. This is to reduce the number of arguments required by each
 ** helper function.
@@ -1574,7 +1579,7 @@ struct WindowCsrAndReg {
 **
 **     (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING)
 **
-**   The windows functions implmentation caches the input rows in a temp
+**   The windows functions implementation caches the input rows in a temp
 **   table, sorted by "a, b" (it actually populates the cache lazily, and
 **   aggressively removes rows once they are no longer required, but that's
 **   a mere detail). It keeps three cursors open on the temp table. One
@@ -1587,7 +1592,7 @@ struct WindowCsrAndReg {
 **
 **   Each cursor (start, current and end) consists of a VDBE cursor
 **   (WindowCsrAndReg.csr) and an array of registers (starting at
-**   WindowCodeArg.reg) that always contains a copy of the peer values 
+**   WindowCodeArg.reg) that always contains a copy of the peer values
 **   read from the corresponding cursor.
 **
 **   Depending on the window-frame in question, all three cursors may not
@@ -1632,8 +1637,8 @@ static void windowReadPeerValues(
 }
 
 /*
-** Generate VM code to invoke either xStep() (if bInverse is 0) or 
-** xInverse (if bInverse is non-zero) for each window function in the 
+** Generate VM code to invoke either xStep() (if bInverse is 0) or
+** xInverse (if bInverse is non-zero) for each window function in the
 ** linked list starting at pMWin. Or, for built-in window functions
 ** that do not use the standard function API, generate the required
 ** inline VM code.
@@ -1665,6 +1670,7 @@ static void windowAggStep(
     int regArg;
     int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin);
     int i;
+    int addrIf = 0;
 
     assert( bInverse==0 || pWin->eStart!=TK_UNBOUNDED );
 
@@ -1681,8 +1687,20 @@ static void windowAggStep(
     }
     regArg = reg;
 
+    if( pWin->pFilter ){
+      int regTmp;
+      assert( ExprUseXList(pWin->pOwner) );
+      assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr );
+      assert( pWin->bExprArgs || nArg  ||pWin->pOwner->x.pList==0 );
+      regTmp = sqlite3GetTempReg(pParse);
+      sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp);
+      addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1);
+      VdbeCoverage(v);
+      sqlite3ReleaseTempReg(pParse, regTmp);
+    }
+
     if( pMWin->regStartRowid==0
-     && (pFunc->funcFlags & SQLITE_FUNC_MINMAX) 
+     && (pFunc->funcFlags & SQLITE_FUNC_MINMAX)
      && (pWin->eStart!=TK_UNBOUNDED)
     ){
       int addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regArg);
@@ -1700,25 +1718,13 @@ static void windowAggStep(
       }
       sqlite3VdbeJumpHere(v, addrIsNull);
     }else if( pWin->regApp ){
+      assert( pWin->pFilter==0 );
       assert( pFunc->zName==nth_valueName
            || pFunc->zName==first_valueName
       );
       assert( bInverse==0 || bInverse==1 );
       sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1);
     }else if( pFunc->xSFunc!=noopStepFunc ){
-      int addrIf = 0;
-      if( pWin->pFilter ){
-        int regTmp;
-        assert( ExprUseXList(pWin->pOwner) );
-        assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr );
-        assert( pWin->bExprArgs || nArg  ||pWin->pOwner->x.pList==0 );
-        regTmp = sqlite3GetTempReg(pParse);
-        sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp);
-        addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1);
-        VdbeCoverage(v);
-        sqlite3ReleaseTempReg(pParse, regTmp);
-      }
-      
       if( pWin->bExprArgs ){
         int iOp = sqlite3VdbeCurrentAddr(v);
         int iEnd;
@@ -1742,15 +1748,16 @@ static void windowAggStep(
         pColl = sqlite3ExprNNCollSeq(pParse, pWin->pOwner->x.pList->a[0].pExpr);
         sqlite3VdbeAddOp4(v, OP_CollSeq, 0,0,0, (const char*)pColl, P4_COLLSEQ);
       }
-      sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, 
+      sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep,
                         bInverse, regArg, pWin->regAccum);
       sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF);
-      sqlite3VdbeChangeP5(v, (u8)nArg);
+      sqlite3VdbeChangeP5(v, (u16)nArg);
       if( pWin->bExprArgs ){
         sqlite3ReleaseTempRange(pParse, regArg, nArg);
       }
-      if( addrIf ) sqlite3VdbeJumpHere(v, addrIf);
     }
+
+    if( addrIf ) sqlite3VdbeJumpHere(v, addrIf);
   }
 }
 
@@ -1775,7 +1782,7 @@ static void windowAggFinal(WindowCodeArg *p, int bFin){
 
   for(pWin=pMWin; pWin; pWin=pWin->pNextWin){
     if( pMWin->regStartRowid==0
-     && (pWin->pWFunc->funcFlags & SQLITE_FUNC_MINMAX) 
+     && (pWin->pWFunc->funcFlags & SQLITE_FUNC_MINMAX)
      && (pWin->eStart!=TK_UNBOUNDED)
     ){
       sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult);
@@ -1932,7 +1939,7 @@ static void windowReturnOneRow(WindowCodeArg *p){
         int lbl = sqlite3VdbeMakeLabel(pParse);
         int tmpReg = sqlite3GetTempReg(pParse);
         sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult);
-  
+ 
         if( pFunc->zName==nth_valueName ){
           sqlite3VdbeAddOp3(v, OP_Column,pMWin->iEphCsr,pWin->iArgCol+1,tmpReg);
           windowCheckValue(pParse, tmpReg, 2);
@@ -1954,7 +1961,7 @@ static void windowReturnOneRow(WindowCodeArg *p){
         int lbl = sqlite3VdbeMakeLabel(pParse);
         int tmpReg = sqlite3GetTempReg(pParse);
         int iEph = pMWin->iEphCsr;
-  
+ 
         if( nArg<3 ){
           sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult);
         }else{
@@ -1971,7 +1978,7 @@ static void windowReturnOneRow(WindowCodeArg *p){
           sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg);
           sqlite3ReleaseTempReg(pParse, tmpReg2);
         }
-  
+ 
         sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg);
         VdbeCoverage(v);
         sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult);
@@ -2017,7 +2024,7 @@ static int windowInitAccum(Parse *pParse, Window *pMWin){
   return regArg;
 }
 
-/* 
+/*
 ** Return true if the current frame should be cached in the ephemeral table,
 ** even if there are no xInverse() calls required.
 */
@@ -2041,9 +2048,9 @@ static int windowCacheFrame(Window *pMWin){
 ** regOld and regNew are each the first register in an array of size
 ** pOrderBy->nExpr. This function generates code to compare the two
 ** arrays of registers using the collation sequences and other comparison
-** parameters specified by pOrderBy. 
+** parameters specified by pOrderBy.
 **
-** If the two arrays are not equal, the contents of regNew is copied to 
+** If the two arrays are not equal, the contents of regNew is copied to
 ** regOld and control falls through. Otherwise, if the contents of the arrays
 ** are equal, an OP_Goto is executed. The address of the OP_Goto is returned.
 */
@@ -2060,7 +2067,7 @@ static void windowIfNewPeer(
     KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0);
     sqlite3VdbeAddOp3(v, OP_Compare, regOld, regNew, nVal);
     sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO);
-    sqlite3VdbeAddOp3(v, OP_Jump, 
+    sqlite3VdbeAddOp3(v, OP_Jump,
       sqlite3VdbeCurrentAddr(v)+1, addr, sqlite3VdbeCurrentAddr(v)+1
     );
     VdbeCoverageEqNe(v);
@@ -2094,7 +2101,7 @@ static void windowIfNewPeer(
 ** or subtraction is a a copy of csr1.peerVal.
 */
 static void windowCodeRangeTest(
-  WindowCodeArg *p, 
+  WindowCodeArg *p,
   int op,                         /* OP_Ge, OP_Gt, or OP_Le */
   int csr1,                       /* Cursor number for cursor 1 */
   int regVal,                     /* Register containing non-negative number */
@@ -2132,8 +2139,8 @@ static void windowCodeRangeTest(
       ((op==OP_Ge) ? ">=" : (op==OP_Le) ? "<=" : (op==OP_Gt) ? ">" : "<"), reg2
   ));
 
-  /* If the BIGNULL flag is set for the ORDER BY, then it is required to 
-  ** consider NULL values to be larger than all other values, instead of 
+  /* If the BIGNULL flag is set for the ORDER BY, then it is required to
+  ** consider NULL values to be larger than all other values, instead of
   ** the usual smaller. The VDBE opcodes OP_Ge and so on do not handle this
   ** (and adding that capability causes a performance regression), so
   ** instead if the BIGNULL flag is set then cases where either reg1 or
@@ -2148,23 +2155,23 @@ static void windowCodeRangeTest(
   **     if( op==OP_Le ) goto lbl;
   **   }
   **
-  ** Additionally, if either reg1 or reg2 are NULL but the jump to lbl is 
+  ** Additionally, if either reg1 or reg2 are NULL but the jump to lbl is
   ** not taken, control jumps over the comparison operator coded below this
   ** block.  */
   if( pOrderBy->a[0].fg.sortFlags & KEYINFO_ORDER_BIGNULL ){
     /* This block runs if reg1 contains a NULL. */
     int addr = sqlite3VdbeAddOp1(v, OP_NotNull, reg1); VdbeCoverage(v);
     switch( op ){
-      case OP_Ge: 
-        sqlite3VdbeAddOp2(v, OP_Goto, 0, lbl); 
+      case OP_Ge:
+        sqlite3VdbeAddOp2(v, OP_Goto, 0, lbl);
         break;
-      case OP_Gt: 
-        sqlite3VdbeAddOp2(v, OP_NotNull, reg2, lbl); 
-        VdbeCoverage(v); 
+      case OP_Gt:
+        sqlite3VdbeAddOp2(v, OP_NotNull, reg2, lbl);
+        VdbeCoverage(v);
         break;
-      case OP_Le: 
-        sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); 
-        VdbeCoverage(v); 
+      case OP_Le:
+        sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl);
+        VdbeCoverage(v);
         break;
       default: assert( op==OP_Lt ); /* no-op */ break;
     }
@@ -2221,7 +2228,7 @@ static void windowCodeRangeTest(
 
 /*
 ** Helper function for sqlite3WindowCodeStep(). Each call to this function
-** generates VM code for a single RETURN_ROW, AGGSTEP or AGGINVERSE 
+** generates VM code for a single RETURN_ROW, AGGSTEP or AGGINVERSE
 ** operation. Refer to the header comment for sqlite3WindowCodeStep() for
 ** details.
 */
@@ -2280,8 +2287,8 @@ static int windowCodeOp(
   addrContinue = sqlite3VdbeCurrentAddr(v);
 
   /* If this is a (RANGE BETWEEN a FOLLOWING AND b FOLLOWING) or
-  ** (RANGE BETWEEN b PRECEDING AND a PRECEDING) frame, ensure the 
-  ** start cursor does not advance past the end cursor within the 
+  ** (RANGE BETWEEN b PRECEDING AND a PRECEDING) frame, ensure the
+  ** start cursor does not advance past the end cursor within the
   ** temporary table. It otherwise might, if (a>b). Also ensure that,
   ** if the input cursor is still finding new rows, that the end
   ** cursor does not go past it to EOF. */
@@ -2422,11 +2429,11 @@ Window *sqlite3WindowListDup(sqlite3 *db, Window *p){
 }
 
 /*
-** Return true if it can be determined at compile time that expression 
-** pExpr evaluates to a value that, when cast to an integer, is greater 
+** Return true if it can be determined at compile time that expression
+** pExpr evaluates to a value that, when cast to an integer, is greater
 ** than zero. False otherwise.
 **
-** If an OOM error occurs, this function sets the Parse.db.mallocFailed 
+** If an OOM error occurs, this function sets the Parse.db.mallocFailed
 ** flag and returns zero.
 */
 static int windowExprGtZero(Parse *pParse, Expr *pExpr){
@@ -2442,11 +2449,11 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 }
 
 /*
-** sqlite3WhereBegin() has already been called for the SELECT statement 
+** sqlite3WhereBegin() has already been called for the SELECT statement
 ** passed as the second argument when this function is invoked. It generates
-** code to populate the Window.regResult register for each window function 
+** code to populate the Window.regResult register for each window function
 ** and invoke the sub-routine at instruction addrGosub once for each row.
-** sqlite3WhereEnd() is always called before returning. 
+** sqlite3WhereEnd() is always called before returning.
 **
 ** This function handles several different types of window frames, which
 ** require slightly different processing. The following pseudo code is
@@ -2461,17 +2468,17 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 **         Gosub flush
 **       }
 **       Insert new row into eph table.
-**       
+**      
 **       if( first row of partition ){
 **         // Rewind three cursors, all open on the eph table.
 **         Rewind(csrEnd);
 **         Rewind(csrStart);
 **         Rewind(csrCurrent);
-**       
+**      
 **         regEnd = <expr2>          // FOLLOWING expression
 **         regStart = <expr1>        // PRECEDING expression
 **       }else{
-**         // First time this branch is taken, the eph table contains two 
+**         // First time this branch is taken, the eph table contains two
 **         // rows. The first row in the partition, which all three cursors
 **         // currently point to, and the following row.
 **         AGGSTEP
@@ -2500,17 +2507,17 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 **               with arguments read from the current row of cursor csrEnd, then
 **               step cursor csrEnd forward one row (i.e. sqlite3BtreeNext()).
 **
-**   RETURN_ROW: return a row to the caller based on the contents of the 
-**               current row of csrCurrent and the current state of all 
+**   RETURN_ROW: return a row to the caller based on the contents of the
+**               current row of csrCurrent and the current state of all
 **               aggregates. Then step cursor csrCurrent forward one row.
 **
-**   AGGINVERSE: invoke the aggregate xInverse() function for each window 
+**   AGGINVERSE: invoke the aggregate xInverse() function for each window
 **               functions with arguments read from the current row of cursor
 **               csrStart. Then step csrStart forward one row.
 **
 ** There are two other ROWS window frames that are handled significantly
 ** differently from the above - "BETWEEN <expr> PRECEDING AND <expr> PRECEDING"
-** and "BETWEEN <expr> FOLLOWING AND <expr> FOLLOWING". These are special 
+** and "BETWEEN <expr> FOLLOWING AND <expr> FOLLOWING". These are special
 ** cases because they change the order in which the three cursors (csrStart,
 ** csrCurrent and csrEnd) iterate through the ephemeral table. Cases that
 ** use UNBOUNDED or CURRENT ROW are much simpler variations on one of these
@@ -2546,7 +2553,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 **
 **   ROWS BETWEEN <expr1> FOLLOWING AND <expr2> FOLLOWING
 **
-**     ... loop started by sqlite3WhereBegin() ...
+**   ... loop started by sqlite3WhereBegin() ...
 **     if( new partition ){
 **       Gosub flush
 **     }
@@ -2583,7 +2590,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 **
 ** For the most part, the patterns above are adapted to support UNBOUNDED by
 ** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and
-** CURRENT ROW by assuming that it is equivilent to "0 PRECEDING/FOLLOWING".
+** CURRENT ROW by assuming that it is equivalent to "0 PRECEDING/FOLLOWING".
 ** This is optimized of course - branches that will never be taken and
 ** conditions that are always true are omitted from the VM code. The only
 ** exceptional case is:
@@ -2660,15 +2667,15 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 **           regEnd = <expr2>
 **           regStart = <expr1>
 **         }else if( new group ){
-**           ... 
+**           ...
 **         }
 **       }
 **
-**   2. Instead of processing a single row, each RETURN_ROW, AGGSTEP or 
+**   2. Instead of processing a single row, each RETURN_ROW, AGGSTEP or
 **      AGGINVERSE step processes the current row of the relevant cursor and
 **      all subsequent rows belonging to the same group.
 **
-** RANGE window frames are a little different again. As for GROUPS, the 
+** RANGE window frames are a little different again. As for GROUPS, the
 ** main loop runs once per group only. And RETURN_ROW, AGGSTEP and AGGINVERSE
 ** deal in groups instead of rows. As for ROWS and GROUPS, there are three
 ** basic cases:
@@ -2705,7 +2712,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){
 **         }
 **       }
 **
-** In the above notation, "csr.key" means the current value of the ORDER BY 
+** In the above notation, "csr.key" means the current value of the ORDER BY
 ** expression (there is only ever 1 for a RANGE that uses an <expr> FOLLOWING
 ** or <expr PRECEDING) read from cursor csr.
 **
@@ -2788,7 +2795,7 @@ void sqlite3WindowCodeStep(
   Vdbe *v = sqlite3GetVdbe(pParse);
   int csrWrite;                   /* Cursor used to write to eph. table */
   int csrInput = p->pSrc->a[0].iCursor;     /* Cursor of sub-select */
-  int nInput = p->pSrc->a[0].pTab->nCol;    /* Number of cols returned by sub */
+  int nInput = p->pSrc->a[0].pSTab->nCol;   /* Number of cols returned by sub */
   int iInput;                               /* To iterate through sub cols */
   int addrNe;                     /* Address of OP_Ne */
   int addrGosubFlush = 0;         /* Address of OP_Gosub to flush: */
@@ -2804,11 +2811,11 @@ void sqlite3WindowCodeStep(
   int regStart = 0;               /* Value of <expr> PRECEDING */
   int regEnd = 0;                 /* Value of <expr> FOLLOWING */
 
-  assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_CURRENT 
-       || pMWin->eStart==TK_FOLLOWING || pMWin->eStart==TK_UNBOUNDED 
+  assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_CURRENT
+       || pMWin->eStart==TK_FOLLOWING || pMWin->eStart==TK_UNBOUNDED
   );
-  assert( pMWin->eEnd==TK_FOLLOWING || pMWin->eEnd==TK_CURRENT 
-       || pMWin->eEnd==TK_UNBOUNDED || pMWin->eEnd==TK_PRECEDING 
+  assert( pMWin->eEnd==TK_FOLLOWING || pMWin->eEnd==TK_CURRENT
+       || pMWin->eEnd==TK_UNBOUNDED || pMWin->eEnd==TK_PRECEDING
   );
   assert( pMWin->eExclude==0 || pMWin->eExclude==TK_CURRENT
        || pMWin->eExclude==TK_GROUP || pMWin->eExclude==TK_TIES
@@ -2830,9 +2837,9 @@ void sqlite3WindowCodeStep(
   s.end.csr = s.current.csr+3;
 
   /* Figure out when rows may be deleted from the ephemeral table. There
-  ** are four options - they may never be deleted (eDelete==0), they may 
+  ** are four options - they may never be deleted (eDelete==0), they may
   ** be deleted as soon as they are no longer part of the window frame
-  ** (eDelete==WINDOW_AGGINVERSE), they may be deleted as after the row 
+  ** (eDelete==WINDOW_AGGINVERSE), they may be deleted as after the row
   ** has been returned to the caller (WINDOW_RETURN_ROW), or they may
   ** be deleted after they enter the frame (WINDOW_AGGSTEP). */
   switch( pMWin->eStart ){
@@ -2862,7 +2869,7 @@ void sqlite3WindowCodeStep(
   }
 
   /* Allocate registers for the array of values from the sub-query, the
-  ** samve values in record form, and the rowid used to insert said record
+  ** same values in record form, and the rowid used to insert said record
   ** into the ephemeral table.  */
   regNew = pParse->nMem+1;
   pParse->nMem += nInput;
@@ -2880,7 +2887,7 @@ void sqlite3WindowCodeStep(
   }
 
   /* If this is not a "ROWS BETWEEN ..." frame, then allocate arrays of
-  ** registers to store copies of the ORDER BY expressions (peer values) 
+  ** registers to store copies of the ORDER BY expressions (peer values)
   ** for the main loop, and for each cursor (start, current and end). */
   if( pMWin->eFrmType!=TK_ROWS ){
     int nPeer = (pOrderBy ? pOrderBy->nExpr : 0);
@@ -2901,7 +2908,7 @@ void sqlite3WindowCodeStep(
   sqlite3VdbeAddOp3(v, OP_MakeRecord, regNew, nInput, regRecord);
 
   /* An input row has just been read into an array of registers starting
-  ** at regNew. If the window has a PARTITION clause, this block generates 
+  ** at regNew. If the window has a PARTITION clause, this block generates
   ** VM code to check if the input row is the start of a new partition.
   ** If so, it does an OP_Gosub to an address to be filled in later. The
   ** address of the OP_Gosub is stored in local variable addrGosubFlush. */
@@ -3064,6 +3071,12 @@ void sqlite3WindowCodeStep(
       addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1);
     }else{
       assert( pMWin->eEnd==TK_FOLLOWING );
+      /* assert( regStart>=0 );
+      ** regEnd = regEnd - regStart;
+      ** regStart = 0;   */ 
+      sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regEnd);
+      sqlite3VdbeAddOp2(v, OP_Integer, 0, regStart);
+
       addrStart = sqlite3VdbeCurrentAddr(v);
       addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1);
       addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1);
diff --git a/test/affinity2.test b/test/affinity2.test
index 6ad257ac36..59f9dada53 100644
--- a/test/affinity2.test
+++ b/test/affinity2.test
@@ -116,7 +116,7 @@ do_execsql_test 507 {
   SELECT * FROM t0 WHERE +-+'ce' >= t0.c0;
 } {-1 {}}
  
-# 2019-08-30 ticket https://www.sqlite.org/src/info/40812aea1fde9594
+# 2019-08-30 ticket https://sqlite.org/src/info/40812aea1fde9594
 #
 # Due to some differences in floating point computations, these tests do not
 # work under valgrind.
diff --git a/test/affinity3.test b/test/affinity3.test
index 3695ea8479..be415de46c 100644
--- a/test/affinity3.test
+++ b/test/affinity3.test
@@ -11,14 +11,14 @@
 #
 # Test cases for bugs:
 #
-#    https://www.sqlite.org/src/info/91e2e8ba6ff2e2
-#    https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf
+#    https://sqlite.org/src/info/91e2e8ba6ff2e2
+#    https://sqlite.org/src/info/7ffd1ca1d2ad4ecf
 #
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-# Ticket https://www.sqlite.org/src/info/91e2e8ba6ff2e2 (2011-09-19)
+# Ticket https://sqlite.org/src/info/91e2e8ba6ff2e2 (2011-09-19)
 # Automatic index causes undesired type conversions
 #
 do_execsql_test affinity3-100 {
@@ -87,7 +87,7 @@ do_execsql_test affinity3-142 {
   SELECT id, (apr / 100), typeof(apr) apr_type  FROM v2rjrj;
 } {1 0.12 real 2 0.1201 real}
 
-# Ticket https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf  (2017-01-16)
+# Ticket https://sqlite.org/src/info/7ffd1ca1d2ad4ecf  (2017-01-16)
 # Incorrect affinity when using automatic indexes 
 #
 do_execsql_test affinity3-200 {
diff --git a/test/aggnested.test b/test/aggnested.test
index 1b8b608803..f3539076bd 100644
--- a/test/aggnested.test
+++ b/test/aggnested.test
@@ -25,19 +25,19 @@ do_test aggnested-1.1 {
     INSERT INTO t1 VALUES(1), (2), (3);
     CREATE TABLE t2(b1 INTEGER);
     INSERT INTO t2 VALUES(4), (5);
-    SELECT (SELECT group_concat(a1,'x') FROM t2) FROM t1;
+    SELECT (SELECT string_agg(a1,'x') FROM t2) FROM t1;
   }
 } {1x2x3}
 do_test aggnested-1.2 {
   db eval {
     SELECT
-     (SELECT group_concat(a1,'x') || '-' || group_concat(b1,'y') FROM t2)
+     (SELECT string_agg(a1,'x') || '-' || string_agg(b1,'y') FROM t2)
     FROM t1;
   }
 } {1x2x3-4y5}
 do_test aggnested-1.3 {
   db eval {
-    SELECT (SELECT group_concat(b1,a1) FROM t2) FROM t1;
+    SELECT (SELECT string_agg(b1,a1) FROM t2) FROM t1;
   }
 } {415 425 435}
 do_test aggnested-1.4 {
@@ -309,7 +309,7 @@ do_execsql_test 5.4 {
 do_execsql_test 5.5 {
   CREATE TABLE a(b);
   WITH c AS(SELECT a)
-    SELECT(SELECT(SELECT group_concat(b, b)
+    SELECT(SELECT(SELECT string_agg(b, b)
           LIMIT(SELECT 0.100000 *
             AVG(DISTINCT(SELECT 0 FROM a ORDER BY b, b, b))))
         FROM a GROUP BY b,
@@ -358,8 +358,130 @@ do_execsql_test 6.2.2 {
   FROM t2 GROUP BY 'constant_string';
 } {{}}
 
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 7.0 {
+  CREATE TABLE invoice (
+      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+      amount DOUBLE PRECISION DEFAULT NULL,
+      name VARCHAR(100) DEFAULT NULL
+  );
+
+  INSERT INTO invoice (amount, name) VALUES 
+      (4.0, 'Michael'), (15.0, 'Bara'), (4.0, 'Michael'), (6.0, 'John');
+}
+
+do_execsql_test 7.1 {
+  SELECT sum(amount), name
+    from invoice
+  group by name
+  having (select v > 6 from (select sum(amount) v) t)
+} {
+  15.0 Bara
+  8.0 Michael
+}
+
+do_execsql_test 7.2 {
+  SELECT (select 1 from (select sum(amount))) FROM invoice
+} {1}
+
+do_execsql_test 8.0 {
+  CREATE TABLE t1(x INT);
+  INSERT INTO t1 VALUES(100);
+  INSERT INTO t1 VALUES(20);
+  INSERT INTO t1 VALUES(3);
+  SELECT (SELECT y FROM (SELECT sum(x) AS y) AS t2 ) FROM t1;
+} {123}
+
+do_execsql_test 8.1 {
+  SELECT (
+    SELECT y FROM (
+      SELECT z AS y FROM (SELECT sum(x) AS z) AS t2 
+    ) 
+  ) FROM t1;
+} {123}
+
+do_execsql_test 8.2 {
+  SELECT (
+    SELECT a FROM (
+      SELECT y AS a FROM (
+        SELECT z AS y FROM (SELECT sum(x) AS z) AS t2 
+      ) 
+    )
+  ) FROM t1;
+} {123}
+
+#-------------------------------------------------------------------------
+# dbsqlfuzz 04408efc51ae46897c4c122b407412045ed221b4
+#
+reset_db
 
+do_execsql_test 9.1 {
+  WITH out(i, j, k) AS ( 
+      VALUES(1234, 5678, 9012) 
+  )
+  SELECT (
+    SELECT (
+      SELECT min(abc) = ( SELECT ( SELECT 1234 fROM (SELECT abc) ) ) 
+      FROM (
+        SELECT sum( out.i ) + ( SELECT sum( out.i ) ) AS abc FROM (SELECT out.j)
+      )
+    ) 
+  ) FROM out;
+} {0}
 
- 
+do_execsql_test 9.2 {
+  CREATE TABLE t1(a);
+  CREATE TABLE t2(b);
+  INSERT INTO t1 VALUES(1), (2), (3);
+  INSERT INTO t2 VALUES(4), (5), (6);
+
+  SELECT ( 
+    SELECT min(y) + (SELECT x) FROM (
+      SELECT sum(a) AS x, b AS y FROM t2
+    )
+  )
+  FROM t1;
+} {10}
+
+do_execsql_test 9.3 {
+  SELECT ( 
+    SELECT min(y) + (SELECT (SELECT x)) FROM (
+      SELECT sum(a) AS x, b AS y FROM t2
+    )
+  )
+  FROM t1;
+} {10}
+
+do_execsql_test 9.4 {
+  SELECT (
+    SELECT (SELECT x) FROM (
+      SELECT sum(a) AS x, b AS y FROM t2
+      ) GROUP BY y
+    )
+  FROM t1;
+} {6}
+
+do_execsql_test 9.5 {
+  SELECT (
+    SELECT (SELECT (SELECT x)) FROM (
+      SELECT sum(a) AS x, b AS y FROM t2
+      ) GROUP BY y
+    )
+  FROM t1;
+} {6}
+
+# 2023-12-16
+# New test case for check-in [4470f657d2069972] from 2023-11-02
+# https://bugs.chromium.org/p/chromium/issues/detail?id=1511689
+#
+do_execsql_test 10.1 {
+  DROP TABLE IF EXISTS t0;
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t0(c1, c2);  INSERT INTO t0 VALUES(1,2);
+  CREATE TABLE t1(c3, c4);  INSERT INTO t1 VALUES(3,4);
+  SELECT * FROM t0 WHERE EXISTS (SELECT 1 FROM t1 GROUP BY c3 HAVING ( SELECT count(*) FROM (SELECT 1 UNION ALL SELECT sum(DISTINCT c1) ) ) ) BETWEEN 1 AND 1;
+} {1 2}
 
 finish_test
diff --git a/test/aggorderby.test b/test/aggorderby.test
new file mode 100644
index 0000000000..466074815a
--- /dev/null
+++ b/test/aggorderby.test
@@ -0,0 +1,174 @@
+# 2023-10-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for ORDER BY on aggregate functions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test aggorderby-1.1 {
+  CREATE TABLE t1(a TEXT,b INT,c INT,d INT);
+  WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<9)
+  INSERT INTO t1(a,b,c,d) SELECT printf('%d',(x*7)%10),1,x,10-x FROM c;
+  INSERT INTO t1(a,b,c,d) SELECT a, 2, c, 10-d FROM t1;
+  CREATE INDEX t1b ON t1(b);
+}
+do_catchsql_test aggorderby-1.2 {
+  SELECT b, group_concat(a ORDER BY max(d)) FROM t1 GROUP BY b;
+} {1 {misuse of aggregate function max()}}
+do_catchsql_test aggorderby-1.3 {
+  SELECT abs(a ORDER BY max(d)) FROM t1;
+} {1 {ORDER BY may not be used with non-aggregate abs()}}
+
+do_execsql_test aggorderby-2.0 {
+  SELECT group_concat(a ORDER BY a) FROM t1 WHERE b=1;
+} {0,1,2,3,4,5,6,7,8,9}
+do_execsql_test aggorderby-2.1 {
+  SELECT group_concat(a ORDER BY c) FROM t1 WHERE b=1;
+} {0,7,4,1,8,5,2,9,6,3}
+do_execsql_test aggorderby-2.2 {
+  SELECT group_concat(a ORDER BY b, d) FROM t1;
+} {3,6,9,2,5,8,1,4,7,0,0,7,4,1,8,5,2,9,6,3}
+do_execsql_test aggorderby-2.3 {
+  SELECT string_agg(a, ',' ORDER BY b DESC, d) FROM t1;
+} {0,7,4,1,8,5,2,9,6,3,3,6,9,2,5,8,1,4,7,0}
+do_execsql_test aggorderby-2.4 {
+  SELECT b, group_concat(a ORDER BY d) FROM t1 GROUP BY b ORDER BY b;
+} {1 3,6,9,2,5,8,1,4,7,0 2 0,7,4,1,8,5,2,9,6,3}
+
+do_execsql_test aggorderby-3.0 {
+  SELECT group_concat(DISTINCT a ORDER BY a) FROM t1;
+} {0,1,2,3,4,5,6,7,8,9}
+do_execsql_test aggorderby-3.1 {
+  SELECT group_concat(DISTINCT a ORDER BY c) FROM t1;
+} {0,7,4,1,8,5,2,9,6,3}
+
+do_execsql_test aggorderby-4.0 {
+  SELECT count(ORDER BY a) FROM t1;
+} 20
+do_execsql_test aggorderby-4.1 {
+  SELECT c, max(a ORDER BY a) FROM t1;
+} {7 9}
+
+
+do_execsql_test aggorderby-5.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t3;
+  CREATE TABLE t1(a TEXT);  INSERT INTO t1 VALUES('aaa'),('bbb');
+  CREATE TABLE t3(d TEXT);  INSERT INTO t3 VALUES('/'),('-');
+  SELECT (SELECT string_agg(a,d) FROM t3) FROM t1;
+} {aaa-aaa bbb-bbb}
+do_execsql_test aggorderby-5.1 {
+  SELECT (SELECT group_concat(a,d ORDER BY d) FROM t3) FROM t1;
+} {aaa/aaa bbb/bbb}
+do_execsql_test aggorderby-5.2 {
+  SELECT (SELECT string_agg(a,d ORDER BY d DESC) FROM t3) FROM t1;
+} {aaa-aaa bbb-bbb}
+do_execsql_test aggorderby-5.3 {
+  SELECT (SELECT string_agg(a,'#' ORDER BY d) FROM t3) FROM t1;
+} {aaa#aaa bbb#bbb}
+
+# COLLATE works on the ORDER BY.
+#
+do_execsql_test aggorderby-6.0 {
+  WITH c(x) AS (VALUES('abc'),('DEF'),('xyz'),('ABC'),('XYZ'))
+  SELECT string_agg(x,',' ORDER BY x COLLATE nocase),
+         string_agg(x,',' ORDER BY x) FROM c;
+} {abc,ABC,DEF,xyz,XYZ ABC,DEF,XYZ,abc,xyz}
+do_execsql_test aggorderby-6.1 {
+  WITH c(x,y) AS (VALUES(1,'a'),(2,'B'),(3,'c'),(4,'D'))
+  SELECT group_concat(x ORDER BY y COLLATE nocase),
+         group_concat(x ORDER BY y COLLATE binary) FROM c;
+} {1,2,3,4 2,4,1,3}
+
+# NULLS FIRST and NULLS LAST work on the ORDER BY
+#
+do_execsql_test aggorderby-7.0 {
+  WITH c(x) AS (VALUES(1),(NULL),(2.5),(NULL),('three'))
+  SELECT json_group_array(x ORDER BY x NULLS FIRST),
+         json_group_array(x ORDER BY x NULLS LAST) FROM c;
+} {[null,null,1,2.5,"three"] [1,2.5,"three",null,null]}
+do_execsql_test aggorderby-7.1 {
+  WITH c(x,y) AS (VALUES(1,9),(2,null),(3,5),(4,null),(5,1))
+  SELECT json_group_array(x ORDER BY y NULLS FIRST, x),
+         json_group_array(x ORDER BY y NULLS LAST, x) FROM c;
+} {[2,4,5,3,1] [5,3,1,2,4]}
+
+# The DISTINCT only applies to the function arguments, not to the
+# ORDER BY arguments.
+#
+do_execsql_test aggorderby-8.0 {
+  WITH c(x,y,z) AS (VALUES('a',4,5),('b',3,6),('c',2,7),('c',1,8))
+  SELECT group_concat(DISTINCT x ORDER BY y, z) FROM c;
+} {c,b,a}
+do_execsql_test aggorderby-8.1 {
+  WITH c(x,y,z) AS (VALUES('a',4,5),('b',3,6),('b',2,7),('c',1,8))
+  SELECT group_concat(DISTINCT x ORDER BY y, z) FROM c;
+} {c,b,a}
+do_execsql_test aggorderby-8.2 {
+  WITH c(x,y) AS (VALUES(1,1),(2,2),(3,3),(3,4),(3,5),(3,6))
+  SELECT sum(DISTINCT x ORDER BY y) FROM c;
+} 6
+
+# Subtype information is transfered through the sorter for aggregates
+# that make use of subtype info.
+#
+do_execsql_test aggorderby-9.0 {
+  WITH c(x,y) AS (VALUES
+    ('{a:3}', 3),
+    ('[1,1]', 1),
+    ('[4,4]', 4),
+    ('{x:2}', 2))
+  SELECT json_group_array(json(x) ORDER BY y) FROM c;
+} {{[[1,1],{"x":2},{"a":3},[4,4]]}}
+do_execsql_test aggorderby-9.1 {
+  WITH c(x,y) AS (VALUES
+    ('[4,4]', 4),
+    ('{a:3}', 3),
+    ('[4,4]', 4),
+    ('[1,1]', 1),
+    ('[4,4]', 4),
+    ('{x:2}', 2))
+  SELECT json_group_array(DISTINCT json(x) ORDER BY y) FROM c;
+} {{[[1,1],{"x":2},{"a":3},[4,4]]}}
+do_execsql_test aggorderby-9.2 {
+  WITH c(x,y) AS (VALUES
+    ('{a:3}', 3),
+    ('[1,1]', 1),
+    ('[4,4]', 4),
+    ('{x:2}', 2))
+  SELECT json_group_array(json(x) ORDER BY json(x)) FROM c;
+} {{[[1,1],[4,4],{"a":3},{"x":2}]}}
+do_execsql_test aggorderby-9.3 {
+  WITH c(x,y) AS (VALUES
+    ('[4,4]', 4),
+    ('{a:3}', 3),
+    ('[4,4]', 4),
+    ('[1,1]', 1),
+    ('[4,4]', 4),
+    ('{x:2}', 2))
+  SELECT json_group_array(DISTINCT json(x) ORDER BY json(x)) FROM c;
+} {{[[1,1],[4,4],{"a":3},{"x":2}]}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test aggorderby-10.0 {
+  CREATE TABLE t1(w, x);
+  INSERT INTO t1 VALUES(1, 2);
+}
+
+for {set i 0} {$i < 70000} {incr i} { lappend lExpr x }
+do_catchsql_test aggorderby-10.1 "
+  SELECT group_concat(w ORDER BY [join $lExpr ,]) FROM t1
+" {1 {too many terms in ORDER BY clause}}
+
+
+finish_test
diff --git a/test/all.test b/test/all.test
index 46e8115fba..22d7b8daed 100644
--- a/test/all.test
+++ b/test/all.test
@@ -42,7 +42,7 @@ run_test_suite pcache100
 run_test_suite prepare
 run_test_suite mmap
 
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(platform) eq "unix"} {
   ifcapable !default_autovacuum {
     run_test_suite autovacuum_crash
   }
diff --git a/test/alter.test b/test/alter.test
index 0088858a15..9201f40adc 100644
--- a/test/alter.test
+++ b/test/alter.test
@@ -934,5 +934,53 @@ do_execsql_test alter-19.3 {
   SELECT name FROM sqlite_schema WHERE sql LIKE '%t3%' ORDER BY name;
 } {r1 t3}
 
+# 2023-10-14
+# On an ALTER TABLE ADD COLUMN with a DEFAULT clause on a STRICT table
+# make sure that the DEFAULT has a compatible type.
+#
+reset_db
+do_execsql_test alter-20.1 {
+  CREATE TABLE t1(a INT) STRICT;
+  INSERT INTO t1(a) VALUES(45);
+} {}
+do_catchsql_test alter-20.2 {
+  ALTER TABLE t1 ADD COLUMN b TEXT DEFAULT x'313233';
+} {1 {type mismatch on DEFAULT}}
+do_execsql_test alter-20.2 {
+  DELETE FROM t1;
+  ALTER TABLE t1 ADD COLUMN b TEXT DEFAULT x'313233';
+} {}
+do_catchsql_test alter-20.3 {
+  INSERT INTO t1(a) VALUES(45);
+} {1 {cannot store BLOB value in TEXT column t1.b}}
+
+# 2023-11-17 dbsqlfuzz e0900262dadd5c78c2226ad6a435c7f0255be2cd
+# Assertion fault associated with ALTER TABLE and an
+# aggregate ORDER BY within an unknown aggregate function.
+#
+reset_db
+do_execsql_test alter-21.1 {
+  CREATE TABLE t1(a,b,c,d);
+  CREATE TABLE t2(a,b,c,d,x);
+  CREATE TRIGGER r1 AFTER INSERT ON t2 BEGIN
+    SELECT unknown_function(a ORDER BY (SELECT group_concat(DISTINCT a ORDER BY a) FROM t1)) FROM t1;
+  END;
+  ALTER TABLE t2 RENAME TO e;
+} {}
+do_execsql_test alter-21.2 {
+  SELECT name, type FROM sqlite_schema ORDER BY name;
+} {e table r1 trigger t1 table}
+do_execsql_test alter-21.3 {
+  DROP TRIGGER r1;
+  CREATE TRIGGER r2 AFTER INSERT ON e BEGIN
+    SELECT unknown_function(a ORDER BY (SELECT group_concat(a ORDER BY a) FROM (SELECT b FROM t1))) FROM t1;
+  END;
+  ALTER TABLE e RENAME TO t99;
+}
+do_execsql_test alter-21.4 {
+  SELECT name, type FROM sqlite_schema ORDER BY name;
+} {r2 trigger t1 table t99 table}
+
+
 
 finish_test
diff --git a/test/alter2.test b/test/alter2.test
index aae0061ad4..20b75b59ee 100644
--- a/test/alter2.test
+++ b/test/alter2.test
@@ -371,7 +371,7 @@ do_test alter2-7.5 {
   execsql {
     SELECT a, typeof(a), b, typeof(b), c, typeof(c) FROM t1 LIMIT 1;
   }
-} {1 integer -123 integer 5 text}
+} {1 integer -123.0 real 5 text}
 
 #-----------------------------------------------------------------------
 # Test that UPDATE trigger tables work with default values, and that when
@@ -397,11 +397,11 @@ do_test alter2-8.2 {
     UPDATE t1 SET c = 10 WHERE a = 1;
     SELECT a, typeof(a), b, typeof(b), c, typeof(c) FROM t1 LIMIT 1;
   }
-} {1 integer -123 integer 10 text}
+} {1 integer -123.0 real 10 text}
 ifcapable trigger {
   do_test alter2-8.3 {
     set ::val
-  } {-123 integer 5 text -123 integer 10 text}
+  } {-123.0 real 5 text -123.0 real 10 text}
 }
 
 #-----------------------------------------------------------------------
@@ -425,7 +425,7 @@ ifcapable trigger {
       DELETE FROM t1 WHERE a = 2;
     }
     set ::val
-  } {-123 integer 5 text}
+  } {-123.0 real 5 text}
 }
 
 #-----------------------------------------------------------------------
diff --git a/test/alter3.test b/test/alter3.test
index c6f26b0c50..464e20aee2 100644
--- a/test/alter3.test
+++ b/test/alter3.test
@@ -101,7 +101,7 @@ do_test alter3-1.7 {
 } {{CREATE TABLE t3(a, b, c VARCHAR(10, 20), UNIQUE(a, b))}}
 do_test alter3-1.99 {
   catchsql {
-    # May not exist if foriegn-keys are omitted at compile time.
+    # May not exist if foreign-keys are omitted at compile time.
     DROP TABLE t2; 
   }
   execsql {
diff --git a/test/alter4.test b/test/alter4.test
index c63ba6b072..7e2d7e66a8 100644
--- a/test/alter4.test
+++ b/test/alter4.test
@@ -110,7 +110,7 @@ do_test alter4-1.7 {
 } {{CREATE TABLE t3(a, b, c VARCHAR(10, 20), UNIQUE(a, b))}}
 do_test alter4-1.99 {
   catchsql {
-    # May not exist if foriegn-keys are omitted at compile time.
+    # May not exist if foreign-keys are omitted at compile time.
     DROP TABLE t2; 
   }
   execsql {
@@ -379,7 +379,7 @@ do_execsql_test alter4-9.3 {
 # Confirm that doing an ALTER TABLE on a legacy format database
 # does not corrupt DESC indexes.
 #
-# Ticket https://www.sqlite.org/src/tktview/f68bf68513a1c
+# Ticket https://sqlite.org/src/tktview/f68bf68513a1c
 #
 do_test alter4-10.1 {
   db close
diff --git a/test/altercol.test b/test/altercol.test
index e39793aa9f..5f7de57a4e 100644
--- a/test/altercol.test
+++ b/test/altercol.test
@@ -343,6 +343,21 @@ do_catchsql_test 8.4.5 {
   ALTER TABLE b1 RENAME a TO aaa;
 } {1 {error in view zzz: no such column: george}}
 
+do_execsql_test 8.5 {
+  DROP VIEW zzz;
+  CREATE TABLE t5(a TEXT, b INT);
+  INSERT INTO t5(a,b) VALUES('aaa',7),('bbb',3),('ccc',4);
+  CREATE VIEW vt5(x) AS SELECT group_concat(a ORDER BY b) FROM t5;
+  SELECT x FROM vt5;
+} {bbb,ccc,aaa}
+do_execsql_test 8.5.1 {
+  ALTER TABLE t5 RENAME COLUMN b TO bbb;
+  SELECT sql FROM sqlite_schema WHERE name='vt5';
+} {{CREATE VIEW vt5(x) AS SELECT group_concat(a ORDER BY bbb) FROM t5}}
+do_execsql_test 8.5.2 {
+  SELECT x FROM vt5;
+} {bbb,ccc,aaa}
+
 #-------------------------------------------------------------------------
 # More triggers.
 #
@@ -781,7 +796,7 @@ do_execsql_test 19.1 {
   {CREATE VIEW v2(e) AS SELECT coalesce(t2.c,t1.f) FROM t1, t2 WHERE t1.b=t2.d}
 }
 
-# 2019-01-08: https://www.sqlite.org/src/tktview/bc8d94f0fbd633fd9a051e3
+# 2019-01-08: https://sqlite.org/src/tktview/bc8d94f0fbd633fd9a051e3
 #
 # ALTER TABLE RENAME COLUMN does not work for tables that have redundant
 # UNIQUE constraints.
diff --git a/test/altermalloc3.test b/test/altermalloc3.test
index 2dc0b46f29..47efebe228 100644
--- a/test/altermalloc3.test
+++ b/test/altermalloc3.test
@@ -26,6 +26,7 @@ set ::TMPDBERROR [list 1 \
   {unable to open a temporary database file for storing temporary tables}
 ]
 
+sqlite3_db_config db SQLITE_DBCONFIG_DQS_DDL 1
 do_execsql_test 1.0 {
   CREATE TABLE x1(
       one, two, three, PRIMARY KEY(one), 
diff --git a/test/alterqf.test b/test/alterqf.test
index 423a9fa865..b248c7c7f4 100644
--- a/test/alterqf.test
+++ b/test/alterqf.test
@@ -65,7 +65,7 @@ foreach {tn before after} {
 
  11 {CREATE TRIGGER ott AFTER UPDATE ON t1 BEGIN 
       SELECT max("str", new."a") FROM t1 
-          WHERE group_concat("b", ",") OVER (ORDER BY c||"str");
+          WHERE string_agg("b", ",") OVER (ORDER BY c||"str");
       UPDATE t1 SET c= b + "str";
       DELETE FROM t1 WHERE EXISTS (
         SELECT 1 FROM t1 AS o WHERE o."a" = "o.a" AND t1.b IN("t1.b")
@@ -73,7 +73,7 @@ foreach {tn before after} {
     END;
  } {CREATE TRIGGER ott AFTER UPDATE ON t1 BEGIN 
       SELECT max('str', new."a") FROM t1 
-          WHERE group_concat("b", ',') OVER (ORDER BY c||'str');
+          WHERE string_agg("b", ',') OVER (ORDER BY c||'str');
       UPDATE t1 SET c= b + 'str';
       DELETE FROM t1 WHERE EXISTS (
         SELECT 1 FROM t1 AS o WHERE o."a" = 'o.a' AND t1.b IN('t1.b')
diff --git a/test/altertab2.test b/test/altertab2.test
index def9e56686..f2a1d74c40 100644
--- a/test/altertab2.test
+++ b/test/altertab2.test
@@ -351,13 +351,35 @@ do_execsql_test 8.5 {
   SELECT sql FROM sqlite_master WHERE name = 'v4'
 } {{CREATE VIEW v4 AS SELECT * FROM t4 WHERE (c=1 AND 0) OR b=2}}
 
-# 2019-06-10 https://www.sqlite.org/src/info/533010b8cacebe82
+# 2019-06-10 https://sqlite.org/src/info/533010b8cacebe82
 reset_db
 do_catchsql_test 8.6 {
   CREATE TABLE t0(c0);
   CREATE INDEX i0 ON t0(likelihood(1,2) AND 0);
   ALTER TABLE t0 RENAME TO t1;
   SELECT sql FROM sqlite_master WHERE name='i0';
-} {1 {error in index i0: second argument to likelihood() must be a constant between 0.0 and 1.0}}
+} {1 {second argument to likelihood() must be a constant between 0.0 and 1.0}}
+
+
+reset_db
+
+do_execsql_test 9.0 {
+  CREATE TABLE t1(a,b,c,d);
+  CREATE TABLE t2(a,b,c,d,x);
+
+  CREATE TRIGGER AFTER INSERT ON t2 BEGIN
+
+    SELECT group_conct(
+        123 ORDER BY (
+          SELECT 1 FROM ( VALUES(a, 'b'), ('c') )
+          )) 
+    FROM t1;
+
+  END;
+}
+
+do_catchsql_test 9.1 {
+  ALTER TABLE t2 RENAME TO newname;
+} {1 {error in trigger AFTER: all VALUES must have the same number of terms}}
 
 finish_test
diff --git a/test/altertab3.test b/test/altertab3.test
index 5fd17f3a2f..92060fb41c 100644
--- a/test/altertab3.test
+++ b/test/altertab3.test
@@ -190,14 +190,14 @@ do_execsql_test 8.1 {
 }
 do_execsql_test 8.2.1 {
   CREATE TABLE t2 (c0);
-  CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 100) IN ()));
+  CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 1.0) IN ()));
   ALTER TABLE t2 RENAME COLUMN c0 TO c1;
 }
 do_execsql_test 8.2.2 {
   SELECT sql FROM sqlite_master WHERE tbl_name = 't2';
 } {
   {CREATE TABLE t2 (c1)} 
-  {CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 100) IN ()))}
+  {CREATE INDEX i2 ON t2((LIKELIHOOD(c1, 1.0) IN ()))}
 }
 do_test 8.2.3 {
   sqlite3 db2 test.db
@@ -662,14 +662,6 @@ do_execsql_test 27.2 {
   {CREATE TABLE t1(a, b AS ((WITH w1 (xyz) AS  ( SELECT t1.b FROM t1 )  SELECT 123) IN ()))}
 }
 
-do_execsql_test 27.3 {
-  CREATE TABLE t0(c0 , c1 AS (CASE TRUE   NOT IN () WHEN NULL   THEN CASE + 0xa     ISNULL  WHEN NOT + 0x9     THEN t0.c1  ELSE CURRENT_TIME   LIKE CAST (t0.c1 REGEXP '-([1-9]\d*.\d*|0\.\d*[1-9]\d*)'ESCAPE (c1) COLLATE BINARY  BETWEEN c1  AND c1   NOT IN (WITH t4 (c0) AS  (WITH t3 (c0) AS NOT MATERIALIZED  (WITH RECURSIVE t2 (c0) AS  (WITH RECURSIVE t1 AS  (VALUES (x'717171ff71717171' )  )  SELECT DISTINCT t0.c0  FROM t0 NOT INDEXED  WHERE t0.c0 =t0.c0 GROUP BY 0x9      )  SELECT DISTINCT t0.c0  FROM t0 NOT INDEXED  WHERE t0.c0 =t0.c1   )  SELECT DISTINCT t0.c0  FROM t0 NOT INDEXED  WHERE t0.c0 =t0.c0 GROUP BY typeof(0x9   )    )  SELECT DISTINCT t0.c0  FROM t0 NOT INDEXED  WHERE t0.c0 =t0.c0 GROUP BY typeof(typeof(0x9    )  )    ) IN t0   BETWEEN typeof(typeof(typeof(hex(*) FILTER (WHERE + x'5ccd1e68'   )  )  )  )  AND 1   >0xa      AS BLOB (+4.4E4 , -0xe  ) )  END  <> c1  IN ()  END  ) VIRTUAL   , c35 PRIMARY KEY   ,  c60 , c64 NUMERIC (-6.8 , -0xE  )  ) WITHOUT ROWID ;
-} {}
-
-do_execsql_test 27.4 {
-  ALTER TABLE t0 DROP COLUMN c60;
-} {}
-
 #-------------------------------------------------------------------------
 reset_db
 do_execsql_test 28.1 {
@@ -736,4 +728,54 @@ do_execsql_test 29.7 {
   END}
 }
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 30.0 {
+  CREATE TABLE t1(a, b);
+  CREATE VIEW v1 AS 
+      SELECT ( VALUES(a), (b) ) FROM (
+        SELECT a, b FROM t1
+      )
+  ;
+}
+
+do_execsql_test 30.1 {
+  SELECT * FROM v1
+}
+
+do_execsql_test 30.1 {
+  ALTER TABLE t1 RENAME TO t2;
+}
+do_execsql_test 30.2 {
+  SELECT sql FROM sqlite_schema WHERE type='view'
+} {
+  {CREATE VIEW v1 AS 
+      SELECT ( VALUES(a), (b) ) FROM (
+        SELECT a, b FROM "t2"
+      )}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 31.0 {
+  CREATE TABLE t1(ii INTEGER PRIMARY KEY, tt INTEGER, rr REAL);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000
+  )
+  INSERT INTO t1 SELECT NULL, i, 5.0 FROM s;
+}
+
+do_test 31.1 {
+  set pg [db one {PRAGMA page_count}]
+  execsql {
+    ALTER TABLE t1 DROP COLUMN tt;
+  }
+  set pg2 [db one {PRAGMA page_count}]
+  expr $pg==$pg2
+} {1}
+
+do_execsql_test 31.2 {
+  SELECT rr FROM t1 LIMIT 1
+} {5.0}
+
 finish_test
diff --git a/test/altertrig.test b/test/altertrig.test
index 934a636669..556dc3fea4 100644
--- a/test/altertrig.test
+++ b/test/altertrig.test
@@ -160,4 +160,3 @@ foreach {tn alter update final} {
 }
 
 finish_test
-
diff --git a/test/analyze.test b/test/analyze.test
index ca6c9b096d..f97c78aff1 100644
--- a/test/analyze.test
+++ b/test/analyze.test
@@ -377,4 +377,23 @@ do_execsql_test analyze-6.1 {
   SELECT tbl FROM sqlite_stat1 WHERE idx IS NULL ORDER BY tbl;
 } {SQLiteDemo2 sqliteDemo t1}
 
+# The following caused a small buffer overread in STAT4 processing prior
+# to check-in [b99135288b157044].
+#
+ifcapable stat4 {
+  reset_db
+  database_may_be_corrupt
+  do_execsql_test analyze-7.1 {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER);
+    INSERT INTO t1 VALUES(1, 7223372036854775);
+    INSERT INTO t1 VALUES(2, 7223372036854776);
+    INSERT INTO t1 VALUES(3, 7223372036854777);
+    CREATE INDEX i1 ON t1(b);
+    ANALYZE;
+    UPDATE sqlite_stat4 SET sample = substr(sample, 0, 4);
+    ANALYZE sqlite_schema;
+    SELECT * FROM t1 WHERE b>7223372036854775
+  } {2 7223372036854776 3 7223372036854777}
+}
+
 finish_test
diff --git a/test/analyze3.test b/test/analyze3.test
index c5d7a7cb13..033dfa5dff 100644
--- a/test/analyze3.test
+++ b/test/analyze3.test
@@ -48,7 +48,7 @@ if {[permutation]=="prepare"} {
 #               query plan when there is no way in which replanning the
 #               query may produce a superior outcome.
 #
-# analyze3-4.*: Test that SQL or authorization callback errors occuring
+# analyze3-4.*: Test that SQL or authorization callback errors occurring
 #               within sqlite3Reprepare() are handled correctly.
 #
 # analyze3-5.*: Check that the query plans of applicable statements are
diff --git a/test/analyzeC.test b/test/analyzeC.test
index 2f43d57a1e..f5bcaeb781 100644
--- a/test/analyzeC.test
+++ b/test/analyzeC.test
@@ -133,7 +133,7 @@ do_execsql_test 4.3 {
 } {/.*INDEX t1ca.*/}
 
 # 2019-08-15.
-# Ticket https://www.sqlite.org/src/tktview/e4598ecbdd18bd82945f602901
+# Ticket https://sqlite.org/src/tktview/e4598ecbdd18bd82945f602901
 # The sz=N parameter in the sqlite_stat1 table needs to have a value of
 # 2 or more to avoid a division by zero in the query planner.
 #
diff --git a/test/analyzer1.test b/test/analyzer1.test
index 51b5f8b6af..1f0b0f6376 100644
--- a/test/analyzer1.test
+++ b/test/analyzer1.test
@@ -19,7 +19,7 @@ ifcapable !vtab {
   return
 }
 
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
   set PROG "sqlite3_analyzer.exe"
 } else {
   set PROG "./sqlite3_analyzer"
diff --git a/test/async.test b/test/async.test
deleted file mode 100644
index e1bc08642e..0000000000
--- a/test/async.test
+++ /dev/null
@@ -1,90 +0,0 @@
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file runs all tests.
-#
-# $Id: async.test,v 1.21 2009/06/05 17:09:12 drh Exp $
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-if {[info commands sqlite3async_initialize] eq ""} {
-  # The async logic is not built into this system
-  finish_test
-  return
-}
-
-rename finish_test async_really_finish_test
-proc finish_test {} {
-  catch {db close}
-  catch {db2 close}
-  catch {db3 close}
-}
-if {[info exists G(isquick)]} { set ASYNC_SAVE_ISQUICK $G(isquick) }
-set G(isquick) 1
-
-set ASYNC_INCLUDE {
-  insert.test
-  insert2.test
-  insert3.test
-  lock.test
-  lock2.test
-  lock3.test
-  select1.test
-  select2.test
-  select3.test
-  select4.test
-  trans.test
-}
-
-# Enable asynchronous IO.
-sqlite3async_initialize "" 1
-
-# This proc flushes the contents of the async-IO queue through to the 
-# underlying VFS. A couple of the test scripts identified in $ASYNC_INCLUDE
-# above contain lines like "catch flush_async_queue" in places where 
-# this is required for the tests to work in async mode.
-#
-proc flush_async_queue {} {
-  sqlite3async_control halt idle
-  sqlite3async_start
-  sqlite3async_wait
-  sqlite3async_control halt never
-}
-
-rename do_test async_really_do_test
-proc do_test {name args} {
-  uplevel async_really_do_test async_io-$name $args
-  flush_async_queue
-}
-
-foreach testfile [lsort -dictionary [glob $testdir/*.test]] {
-  set tail [file tail $testfile]
-  if {[lsearch -exact $ASYNC_INCLUDE $tail]<0} continue
-  source $testfile
-
-  # Make sure everything is flushed through. This is because [source]ing 
-  # the next test file will delete the database file on disk (using
-  # [delete_file]). If the asynchronous backend still has the file
-  # open, it will become confused.
-  #
-  flush_async_queue
-}
-
-# Flush the write-queue and disable asynchronous IO. This should ensure
-# all allocated memory is cleaned up.
-set sqlite3async_trace 1
-flush_async_queue
-sqlite3async_shutdown
-set sqlite3async_trace 0
-
-rename do_test {}
-rename async_really_do_test do_test
-rename finish_test {}
-rename async_really_finish_test finish_test
-
-if {[info exists ASYNC_SAVE_ISQUICK]} { set G(isquick) $ASYNC_SAVE_ISQUICK }
-finish_test
diff --git a/test/async2.test b/test/async2.test
deleted file mode 100644
index 7994a7219d..0000000000
--- a/test/async2.test
+++ /dev/null
@@ -1,126 +0,0 @@
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# $Id: async2.test,v 1.12 2009/04/25 08:39:15 danielk1977 Exp $
-
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-if {
-  [info commands sqlite3async_initialize]=="" ||
-  [info command sqlite3_memdebug_fail]==""
-} {
-  # The async logic is not built into this system
-  puts "Skipping async2 tests: not compiled with required features"
-  finish_test
-  return
-}
-
-# Enable asynchronous IO.
-
-set setup_script {
-  CREATE TABLE counter(c);
-  INSERT INTO counter(c) VALUES (1);
-} 
-
-set sql_script {
-  BEGIN;
-    UPDATE counter SET c = 2;
-    CREATE TABLE t1(a PRIMARY KEY, b, c);
-    CREATE TABLE t2(a PRIMARY KEY, b, c);
-  COMMIT;
-
-  BEGIN;
-    UPDATE counter SET c = 3;
-    INSERT INTO t1 VALUES('abcdefghij', 'four', 'score');
-    INSERT INTO t2 VALUES('klmnopqrst', 'and', 'seven');
-  COMMIT;
-
-  UPDATE counter SET c = 'FIN';
-}
-
-db close
-
-foreach err [list ioerr malloc-transient malloc-persistent] {
-  set ::go 10
-  for {set n 1} {$::go} {incr n} {
-    set ::sqlite_io_error_pending 0
-    sqlite3_memdebug_fail -1
-    forcedelete test.db test.db-journal
-    sqlite3 db test.db
-    execsql $::setup_script
-    db close
-  
-    sqlite3async_initialize "" 1
-    sqlite3 db test.db
-    sqlite3_db_config_lookaside db 0 0 0
-  
-    switch -- $err {
-      ioerr             { set ::sqlite_io_error_pending $n }
-      malloc-persistent { sqlite3_memdebug_fail $n -repeat 1 }
-      malloc-transient  { sqlite3_memdebug_fail $n -repeat 0 }
-    }
-
-    catchsql $::sql_script
-    db close
-
-    sqlite3async_control halt idle
-    sqlite3async_start
-    sqlite3async_wait
-    sqlite3async_control halt never
-    sqlite3async_shutdown
-
-    set ::sqlite_io_error_pending 0
-    sqlite3_memdebug_fail -1
-
-    sqlite3 db test.db
-    set c [db one {SELECT c FROM counter LIMIT 1}]
-    switch -- $c {
-      1 {
-        do_test async-$err-1.1.$n {
-          execsql {
-            SELECT name FROM sqlite_master;
-          }
-        } {counter}
-      }
-      2 {
-        do_test async-$err-1.2.$n.1 {
-          execsql {
-            SELECT * FROM t1;
-          }
-        } {}
-        do_test async-$err-1.2.$n.2 {
-          execsql {
-            SELECT * FROM t2;
-          }
-        } {}
-      }
-      3 {
-        do_test async-$err-1.3.$n.1 {
-          execsql {
-            SELECT * FROM t1;
-          }
-        } {abcdefghij four score}
-        do_test async-$err-1.3.$n.2 {
-          execsql {
-            SELECT * FROM t2;
-          }
-        } {klmnopqrst and seven}
-      }
-      FIN {
-        incr ::go -1
-      }
-    }
-  
-    db close
-  }
-}
-
-catch {db close}
-
-finish_test
diff --git a/test/async3.test b/test/async3.test
deleted file mode 100644
index 9336b66058..0000000000
--- a/test/async3.test
+++ /dev/null
@@ -1,76 +0,0 @@
-# 2007 September 5
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# The focus of this file is testing the code in test_async.c.
-# Specifically, it tests that the xFullPathname() method of
-# of the asynchronous vfs works correctly.
-#
-# $Id: async3.test,v 1.5 2009/04/25 08:39:15 danielk1977 Exp $
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-if { [info commands sqlite3async_initialize]==""  } {
-  # The async logic is not built into this system
-  puts "Skipping async3 tests: not compiled with required features"
-  finish_test
-  return
-}
-
-db close
-sqlite3async_initialize "" 1
-#set sqlite3async_trace 1
-sqlite3async_start
-
-set paths {
-  chocolate/banana/vanilla/file.db
-  chocolate//banana/vanilla/file.db
-  chocolate/./banana//vanilla/file.db
-  chocolate/banana/./vanilla/file.db
-  chocolate/banana/../banana/vanilla/file.db
-  chocolate/banana/./vanilla/extra_bit/../file.db
-}
-
-do_test async3-1.0 {
-  file mkdir [file join chocolate banana vanilla]
-  forcedelete chocolate/banana/vanilla/file.db
-  forcedelete chocolate/banana/vanilla/file.db-journal
-} {}
-
-do_test async3-1.1 {
-  sqlite3 db chocolate/banana/vanilla/file.db
-  execsql {
-    CREATE TABLE abc(a, b, c);
-    BEGIN;
-    INSERT INTO abc VALUES(1, 2, 3);
-  }
-} {}
-
-set N 2
-foreach p $paths {
-  sqlite3 db2 $p
-  do_test async3-1.$N.1 {
-    execsql {SELECT * FROM abc} db2
-  } {}
-  do_test async3-1.$N.2 {
-    catchsql {INSERT INTO abc VALUES(4, 5, 6)} db2
-  } {1 {database is locked}}
-  db2 close
-  incr N
-}
-
-db close
-
-sqlite3async_control halt idle
-sqlite3async_wait
-sqlite3async_control halt never
-sqlite3async_shutdown
-finish_test
diff --git a/test/async4.test b/test/async4.test
deleted file mode 100644
index 92a820173e..0000000000
--- a/test/async4.test
+++ /dev/null
@@ -1,168 +0,0 @@
-# 2009 April 25
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# $Id: async4.test,v 1.4 2009/06/05 17:09:12 drh Exp $
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# Do not use a codec for tests in this file, as the database file is
-# manipulated directly using tcl scripts (using the [hexio_write] command).
-#
-do_not_use_codec
-
-# These tests only work for Tcl version 8.5 and later on Windows (for now)
-#
-if {$tcl_platform(platform)=="windows"} {
-  scan $::tcl_version %f vx
-  if {$vx<8.5} {
-    finish_test
-    return
-  }
-}
-
-if {[info commands sqlite3async_initialize] eq ""} {
-  # The async logic is not built into this system
-  finish_test
-  return
-}
-db close
-
-# Test layout:
-#
-#   async4.1.*: Test the lockfiles parameter.
-#   async4.2.*: Test the delay parameter.
-
-do_test async4.1.1 {
-  sqlite3async_initialize {} 0
-  sqlite3async_control lockfiles
-} {1}
-do_test async4.1.2 {
-  sqlite3async_control lockfiles false
-} {0}
-do_test async4.1.3 {
-  sqlite3async_control lockfiles
-} {0}
-do_test async4.1.4 {
-  sqlite3async_control lockfiles true
-} {1}
-
-do_test async4.1.5 {
-  sqlite3 db test.db -vfs sqlite3async
-  execsql { CREATE TABLE t1(a, b, c) }
-} {}
-do_test async4.1.6 {
-  list [file exists test.db] [file size test.db]
-} {1 0}
-do_test async4.1.7 {
-  sqlite3 db2 test.db
-  catchsql { CREATE TABLE t2(a, b, c) } db2
-} {1 {database is locked}}
-do_test async4.1.8 {
-  sqlite3async_control halt idle
-  sqlite3async_start
-  sqlite3async_wait
-} {}
-do_test async4.1.9 {
-  catchsql { CREATE TABLE t2(a, b, c) } db2
-} {0 {}}
-do_test async4.1.10 {
-  list [catch {sqlite3async_control lockfiles false} msg] $msg
-} {1 SQLITE_MISUSE}
-do_test async4.1.11 {
-  db close
-  list [catch {sqlite3async_control lockfiles false} msg] $msg
-} {1 SQLITE_MISUSE}
-do_test async4.1.12 {
-  sqlite3async_start
-  sqlite3async_wait
-  sqlite3async_control lockfiles false
-} {0}
-do_test async4.1.13 {
-  sqlite3 db test.db -vfs sqlite3async
-  execsql { CREATE TABLE t3(a, b, c) } db
-} {}
-do_test async4.1.14 {
-  execsql { 
-    CREATE INDEX i1 ON t2(a);
-    CREATE INDEX i2 ON t1(a);
-  } db2
-} {}
-do_test async4.1.15 {
-  sqlite3async_start
-  sqlite3async_wait
-  hexio_write test.db 28 00000000
-  execsql { pragma integrity_check } db2
-} {{*** in database main ***
-Page 5 is never used}}
-do_test async4.1.16 {
-  db close
-  db2 close
-  sqlite3async_start
-  sqlite3async_wait
-} {}
-do_test async4.1.17 {
-  sqlite3async_control lockfiles true
-} {1}
-
-do_test async4.2.1 {
-  sqlite3async_control delay
-} {0}
-do_test async4.2.2 {
-  sqlite3async_control delay 23
-} {23}
-do_test async4.2.3 {
-  sqlite3async_control delay
-} {23}
-do_test async4.2.4 {
-  sqlite3async_control delay 0
-} {0}
-do_test async4.2.5 {
-  sqlite3 db test.db -vfs sqlite3async
-
-  execsql { CREATE TABLE t4(a, b) }
-  set T1 [lindex [time {
-    sqlite3async_start
-    sqlite3async_wait
-  }] 0]
-
-  sqlite3async_control delay 100
-  execsql { CREATE TABLE t5(a, b) }
-  set T2 [lindex [time {
-    sqlite3async_start
-    sqlite3async_wait
-  }] 0]
-
-  expr {($T1+1000000) < $T2}
-} {1}
-
-do_test async4.2.6 {
-  sqlite3async_control delay 0
-  execsql { CREATE TABLE t6(a, b) }
-  set T1 [lindex [time {
-    sqlite3async_start
-    sqlite3async_wait
-  }] 0]
-
-  expr {($T1+1000000) < $T2}
-} {1}
-
-do_test async4.2.7 {
-  list [catch { sqlite3async_control delay -1 } msg] $msg
-} {1 SQLITE_MISUSE}
-
-do_test async4.2.8 {
-  db close
-  sqlite3async_start
-  sqlite3async_wait
-} {}
-
-finish_test
diff --git a/test/async5.test b/test/async5.test
deleted file mode 100644
index abac11f750..0000000000
--- a/test/async5.test
+++ /dev/null
@@ -1,68 +0,0 @@
-# 2009 July 19
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file tests that asynchronous IO is compatible with multi-file
-# transactions.
-#
-# $Id: async5.test,v 1.1 2009/07/18 11:52:04 danielk1977 Exp $
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-if {[info commands sqlite3async_initialize] eq ""} {
-  # The async logic is not built into this system
-  finish_test
-  return
-}
-
-db close
-forcedelete test2.db
-sqlite3async_initialize "" 1
-sqlite3async_control halt never
-sqlite3 db test.db
-
-do_test async5-1.1 {
-  execsql {
-    ATTACH 'test2.db' AS next;
-    CREATE TABLE main.t1(a, b);
-    CREATE TABLE next.t2(a, b);
-    BEGIN;
-      INSERT INTO t1 VALUES(1, 2);
-      INSERT INTO t2 VALUES(3, 4);
-    COMMIT;
-  }
-} {}
-do_test async5-1.2 {
-  execsql { SELECT * FROM t1 }
-} {1 2}
-do_test async5-1.3 {
-  execsql { SELECT * FROM t2 }
-} {3 4}
-do_test async5-1.4 {
-  execsql {
-    BEGIN;
-      INSERT INTO t1 VALUES('a', 'b');
-      INSERT INTO t2 VALUES('c', 'd');
-    COMMIT;
-  }
-} {}
-do_test async5-1.5 {
-  execsql { SELECT * FROM t1 }
-} {1 2 a b}
-do_test async5-1.6 {
-  execsql { SELECT * FROM t2 }
-} {3 4 c d}
-
-db close
-
-sqlite3async_control halt idle
-sqlite3async_start
-sqlite3async_wait
-sqlite3async_control halt never
-sqlite3async_shutdown
-set sqlite3async_trace 0
-finish_test
diff --git a/test/atof1.test b/test/atof1.test
index 5959c5d5b9..39747e5da5 100644
--- a/test/atof1.test
+++ b/test/atof1.test
@@ -15,18 +15,10 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-if {$::longdouble_size<=8} {
-  finish_test
-  return
-}
-if {$::tcl_platform(machine)!="x86_64"} {
-  finish_test
-  return
-}
-
+set mxpow 35
 expr srand(1)
 for {set i 1} {$i<20000} {incr i} {
-  set pow [expr {int((rand()-0.5)*100)}]
+  set pow [expr {int((rand()-0.5)*$mxpow)}]
   set x [expr {pow((rand()-0.5)*2*rand(),$pow)}]
   set xf [format %.32e $x]
 
@@ -51,10 +43,16 @@ for {set i 1} {$i<20000} {incr i} {
     set y [db eval {SELECT $x=CAST(quote($x) AS real)}]
     if {!$y} {
       db eval {SELECT real2hex($x) a, real2hex(CAST(quote($x) AS real)) b} {}
-      puts "\nIN:    $a $xf"
-      puts [format {QUOTE: %16s %s} {} [db eval {SELECT quote($x)}]]
+      puts ""
+      if {$x<0} {
+        puts "[format {!SCALE: %17s 1 23456789 123456789 123456789} {}]"
+      } else {
+        puts "[format {!SCALE: %16s 1 23456789 123456789 123456789} {}]"
+      }
+      puts "!IN:    $a $xf"
+      puts [format {!QUOTE: %16s %s} {} [db eval {SELECT quote($x)}]]
       db eval {SELECT CAST(quote($x) AS real) c} {}
-      puts "OUT:   $b [format %.32e $c]"
+      puts "!OUT:   $b [format %.32e $c]"
     }
     set y
   } {1}
@@ -84,5 +82,43 @@ do_execsql_test atof1-2.40 {
   SELECT randomblob(0) - 1;
 } {-1}
 
+# 2024-12-07 https://sqlite.org/forum/forumpost/569a7209179a7f5e
+# Incorrect conversion of floating point or integer literals that
+# have significant digits that begin with 1844674407370955 followed
+# by more digits in the range 0592 throgh 1609.
+#
+do_execsql_test atof-3.1 {
+  WITH RECURSIVE bigval(i,vtxt) AS (
+    SELECT 0, '18446744073709550000'
+    UNION ALL
+    SELECT i+1, format('1844674407370955%04d',i+1) FROM bigval
+     WHERE i+1<=9999
+  )
+  SELECT vtxt, CAST(vtxt AS REAL) FROM bigval
+   WHERE CAST(vtxt AS REAL) NOT GLOB '1.8446744073709[56]*';
+} {}
+do_execsql_test atof-3.2 {
+  WITH RECURSIVE bigval(i,vtxt) AS (
+    SELECT 0, '18.446744073709550000'
+    UNION ALL
+    SELECT i+1, format('18.44674407370955%04d',i+1) FROM bigval
+     WHERE i+1<=9999
+  )
+  SELECT vtxt, CAST(vtxt AS REAL) FROM bigval
+   WHERE CAST(vtxt AS REAL) NOT GLOB '18.446744073709*';
+} {}
+do_execsql_test atof-3.3 {
+  WITH RECURSIVE exp(n,v1,v2) AS (
+    SELECT -200, '1.8446744073709550592e-200', '1.8446744073709551609e-200'
+    UNION ALL
+    SELECT n+1, ('1.8446744073709550592e'||n),('1.8446744073709551609e'||n)
+      FROM exp WHERE n<200
+  )
+  SELECT n, v1, v2
+    FROM exp
+   WHERE format('%.10e',CAST(v1 AS REAL)) NOT GLOB '1.8446*'
+      OR format('%.10e',CAST(v2 AS REAL)) NOT GLOB '1.8446*';
+} {}
+
 
 finish_test
diff --git a/test/attach.test b/test/attach.test
index 557201d654..3445c43fa7 100644
--- a/test/attach.test
+++ b/test/attach.test
@@ -759,7 +759,7 @@ do_test attach-6.1 {
     ATTACH DATABASE 'no-such-file' AS nosuch;
   }
 } {0 {}}
-if {$tcl_platform(platform)=="unix"} {
+if {$tcl_platform(platform) eq "unix"} {
   do_test attach-6.2 {
     sqlite3 dbx cannot-read
     dbx eval {CREATE TABLE t1(a,b,c)}
diff --git a/test/autoinc.test b/test/autoinc.test
index 2c7ee2a7e8..9f869f35ea 100644
--- a/test/autoinc.test
+++ b/test/autoinc.test
@@ -669,7 +669,7 @@ ifcapable trigger {
   } {1 124 2 10123}
 }
 
-# 2016-10-03 ticket https://www.sqlite.org/src/tktview/7b3328086a5c1
+# 2016-10-03 ticket https://sqlite.org/src/tktview/7b3328086a5c1
 # Make sure autoincrement plays nicely with the xfer optimization
 #
 do_execsql_test autoinc-10.1 {
diff --git a/test/autoindex1.test b/test/autoindex1.test
index 08bd9f74e6..1c8ce007f0 100644
--- a/test/autoindex1.test
+++ b/test/autoindex1.test
@@ -185,7 +185,8 @@ do_eqp_test autoindex1-500.1 {
   QUERY PLAN
   |--SEARCH t501 USING INTEGER PRIMARY KEY (rowid=?)
   `--LIST SUBQUERY xxxxxx
-     `--SCAN t502
+     |--SCAN t502
+     `--CREATE BLOOM FILTER
 }
 do_eqp_test autoindex1-501 {
   SELECT b FROM t501
@@ -562,4 +563,32 @@ do_execsql_test autoindex-1120 {
   SELECT * FROM t1 LEFT JOIN t2 ON (t2.c=+t1.a) LEFT JOIN t3 ON (t2.d IS NULL);
 } {1 1 1 2 {} {}}
 
+# 2025-01-18
+# Added support for automatic indexes on WITHOUT ROWID tables.
+#
+reset_db
+do_execsql_test autoindex-1200 {
+  CREATE TABLE t1(a INT, b INT, x INT, PRIMARY KEY(a,b)) WITHOUT ROWID;
+  INSERT INTO t1 VALUES(1,2,90),(1,3,91),(1,4,92);
+  CREATE TABLE t2a(c INTEGER PRIMARY KEY, i1 INT);
+  CREATE TABLE t2b(i1 INTEGER PRIMARY KEY, d INT);
+  CREATE VIEW t2(c,d) AS SELECT c, d FROM t2a NATURAL JOIN t2b;
+  INSERT INTO t2a VALUES(3,93),(4,94),(5,95),(6,96),(7,97);
+  INSERT INTO t2b VALUES(91,11),(92,22),(93,33),(94,44),(95,55);
+  CREATE TABLE dual(dummy TEXT);
+  INSERT INTO dual(dummy) VALUES('x');
+}
+db null NULL
+do_execsql_test autoindex-1210 {
+  SELECT t1.*, t2.* FROM t2 LEFT OUTER JOIN t1 ON b=c ORDER BY +b;
+} {
+  NULL NULL NULL 5    55
+  1    3    91   3    33
+  1    4    92   4    44
+}
+do_execsql_test autoindex-1211 {
+  EXPLAIN QUERY PLAN
+  SELECT t1.*, t2.* FROM t2 LEFT OUTER JOIN t1 ON b=c ORDER BY +b;
+} {/SEARCH t1 USING AUTOMATIC COVERING INDEX/}
+
 finish_test
diff --git a/test/autoindex3.test b/test/autoindex3.test
index 3da7a70586..aa6aa00128 100644
--- a/test/autoindex3.test
+++ b/test/autoindex3.test
@@ -90,5 +90,40 @@ do_eqp_test 220 {
   `--SEARCH u USING AUTOMATIC COVERING INDEX (b=?)
 }
 
+# 2024-05-27
+# ticket https://sqlite.org/src/tktview/8ff324e120
+# forum post https://sqlite.org/forum/forumpost/b21c2101a559be0a
+#
+# If an index with STAT1 data indicates that a column is not very
+# selective, then do not attempt to create an automatic index on
+# that column.
+#
+reset_db
+do_execsql_test 300 {
+  CREATE TABLE t1(id INTEGER PRIMARY KEY);
+  CREATE TABLE t2(cid INT, pid INT, rx INT, PRIMARY KEY(cid, pid, rx));
+  CREATE INDEX x1 ON t2(pid, rx);
+  ANALYZE sqlite_schema;
+  REPLACE INTO sqlite_stat1(tbl, idx, stat) VALUES
+    ('t2', 'x1', '500000 250 250'),
+    ('t2','sqlite_autoindex_t2_1','500000 1 1 1');
+  ANALYZE sqlite_schema;
+}
+do_eqp_test 310 {
+  WITH RECURSIVE children(id) AS (
+    SELECT cid FROM t2 WHERE pid = ?1 AND rx = ?2
+    UNION 
+    SELECT cid FROM t2 JOIN children ON t2.pid = children.id AND rx = ?2
+  ) SELECT count(id) FROM children;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE children
+  |  |--SETUP
+  |  |  `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?)
+  |  `--RECURSIVE STEP
+  |     |--SCAN children
+  |     `--SEARCH t2 USING INDEX x1 (pid=? AND rx=?)
+  `--SCAN children
+}  
 
 finish_test
diff --git a/test/autoindex4.test b/test/autoindex4.test
index d9ab783e42..6af99f5e15 100644
--- a/test/autoindex4.test
+++ b/test/autoindex4.test
@@ -141,7 +141,7 @@ foreach {id data1 data2 jointype onclause whereclause answer} {
   {coalesce(y,4)==4}
   {3 4 3 4}
 
-  5
+  5.1
   VALUES(1,2),(3,4),(NULL,4)
   VALUES(1,2),(3,4)
   {LEFT JOIN}
@@ -149,6 +149,22 @@ foreach {id data1 data2 jointype onclause whereclause answer} {
   {y=4 OR y IS NULL}
   {3 4 3 4 {} 4 {} {}}
 
+  5.2
+  VALUES(1,2),(3,4),(NULL,4)
+  VALUES(1,2),(3,4)
+  {LEFT JOIN}
+  a=x
+  {y NOT IN ()}
+  {1 2 1 2 3 4 3 4 {} 4 {} {}}
+
+  5.3
+  VALUES(1,2),(3,4),(NULL,4)
+  VALUES(1,2),(3,4)
+  {LEFT JOIN}
+  a=x
+  {y NOT IN (SELECT 1 WHERE false)}
+  {1 2 1 2 3 4 3 4 {} 4 {} {}}
+
   6
   VALUES(1,2),(3,4)
   VALUES(1,2),(3,4),(NULL,4)
@@ -193,6 +209,12 @@ foreach {id data1 data2 jointype onclause whereclause answer} {
     db eval {PRAGMA automatic_index=OFF;}
     db eval $sql
   } $answer
+  do_test autoindex4-4.$id.3 {
+    db eval {PRAGMA automatic_index=ON;}
+    optimization_control db all 0
+    db eval $sql
+  } $answer
+  optimization_control db all 1
 }
 
 
diff --git a/test/autoindex5.test b/test/autoindex5.test
index aa8dec27d9..adfc3e5f76 100644
--- a/test/autoindex5.test
+++ b/test/autoindex5.test
@@ -142,7 +142,7 @@ do_catchsql_test 2.2 {
   );
 } {0 9}
 
-# Ticket https://www.sqlite.org/src/info/787fa716be3a7f65
+# Ticket https://sqlite.org/src/info/787fa716be3a7f65
 # Segfault due to multiple uses of the same subquery where the
 # subquery is implemented via coroutine.
 #
diff --git a/test/avfs.test b/test/avfs.test
index 2ebd608baa..ffd6b309fc 100644
--- a/test/avfs.test
+++ b/test/avfs.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 # 
 # This file implements tests for the appendvfs extension.
 #
diff --git a/test/avtrans.test b/test/avtrans.test
index 6fc4a3e393..b483c71a44 100644
--- a/test/avtrans.test
+++ b/test/avtrans.test
@@ -903,7 +903,7 @@ for {set i 2} {$i<=$limit} {incr i} {
          INSERT INTO t3 SELECT randstr(10,400) FROM t3 WHERE random()%10==0;
        }
     } {}
-    if {$tcl_platform(platform)=="unix"} {
+    if {$tcl_platform(platform) eq "unix"} {
       do_test avtrans-9.$i.4-$cnt {
          expr {$sqlite_sync_count>0}
       } 1
diff --git a/test/backcompat.test b/test/backcompat.test
index 87ffc4b3ea..d477d4466c 100644
--- a/test/backcompat.test
+++ b/test/backcompat.test
@@ -112,7 +112,7 @@ proc read_file {zFile} {
   set zData {}
   if {[file exists $zFile]} {
     set fd [open $zFile]
-    fconfigure $fd -translation binary -encoding binary
+    fconfigure $fd -translation binary
 
     if {[file size $zFile]<=$::sqlite_pending_byte || $zFile != "test.db"} {
       set zData [read $fd]
@@ -129,7 +129,7 @@ proc read_file {zFile} {
 }
 proc write_file {zFile zData} {
   set fd [open $zFile w]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   puts -nonewline $fd $zData
   close $fd
 }
diff --git a/test/backup2.test b/test/backup2.test
index 1822e2dbf0..095ecc752d 100644
--- a/test/backup2.test
+++ b/test/backup2.test
@@ -141,7 +141,7 @@ do_test backup2-9 {
 
 # Try to restore from an unreadable file.
 #
-if {$tcl_platform(platform)=="windows"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   set msg {cannot open source database: unable to open database file}
 } elseif {[string match *BSD $tcl_platform(os)]} {
   set msg {}
diff --git a/test/basexx1.test b/test/basexx1.test
index 947a5678f3..b34b25ff36 100644
--- a/test/basexx1.test
+++ b/test/basexx1.test
@@ -39,6 +39,12 @@ do_execsql_test 102 {
 } {AAECAw==
 }}
 
+# Buffer size testing
+do_execsql_test 102-b {
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c wHERE n<5000)
+  SELECT sum(length(base64(randomblob(n)))) FROM c;
+} {16910656}
+
 # Basic base64 decoding with pad chars
 do_execsql_test 103 {
   SELECT hex(base64('AAECAwQF'));
@@ -152,4 +158,11 @@ do_execsql_test 117 {
   SELECT num FROM bs WHERE base85(base85(b))!=b;
 } {}
 
+do_catchsql_test 118 {
+  SELECT base64(zeroblob(2000_000_000))
+} {/1.*too big.*/}
+do_catchsql_test 119 {
+  SELECT base85(zeroblob(2000_000_000))
+} {/1.*too big.*/}
+
 finish_test
diff --git a/test/bc_common.tcl b/test/bc_common.tcl
index c47f99681f..953ce62621 100644
--- a/test/bc_common.tcl
+++ b/test/bc_common.tcl
@@ -9,7 +9,7 @@ proc bc_find_binaries {zCaption} {
   set binaries [list]
   set self [info nameofexec]
   set pattern "$self?*"
-  if {$::tcl_platform(platform)=="windows"} {
+  if {$::tcl_platform(platform) eq "windows"} {
     set pattern [string map {\.exe {}} $pattern]
   }
   foreach file [glob -nocomplain $pattern] {
diff --git a/test/bestindex8.test b/test/bestindex8.test
index e95c3c6dc2..3ed7f6703e 100644
--- a/test/bestindex8.test
+++ b/test/bestindex8.test
@@ -158,7 +158,7 @@ do_test 2.2 {
   set ::lFilterArgs [list]
   execsql { SELECT * FROM vt1 LIMIT 5 OFFSET 50 }
   set ::lFilterArgs
-} {{5 50}}
+} {{50 5}}
 
 do_test 2.3 {
   set ::lFilterArgs [list]
diff --git a/test/bestindex9.test b/test/bestindex9.test
index 94b9da0d38..d591b30efe 100644
--- a/test/bestindex9.test
+++ b/test/bestindex9.test
@@ -102,7 +102,3 @@ do_bestindex9_test 4 {
 
 
 finish_test
-
-
-
-
diff --git a/test/bestindexA.test b/test/bestindexA.test
index 650404eaa0..1976986471 100644
--- a/test/bestindexA.test
+++ b/test/bestindexA.test
@@ -133,6 +133,3 @@ do_xbestindex_test 1.9 {
 
 
 finish_test
-
-
-
diff --git a/test/bestindexB.test b/test/bestindexB.test
new file mode 100644
index 0000000000..b50e74fee3
--- /dev/null
+++ b/test/bestindexB.test
@@ -0,0 +1,87 @@
+# 2023-10-26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindexB
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set clist [$hdl constraints]
+      set orderby [$hdl orderby]
+
+      if {[info exists ::xbestindex_sql]} { 
+        explain_i $::xbestindex_sql 
+        set ::xbestindex_res [ execsql $::xbestindex_sql ]
+      }
+
+      return "cost 1000000 rows 1000000 idxnum 0 idxstr hello"
+    }
+
+    xFilter {
+      return "sql {SELECT 0, 1, 2, 3}"
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+  CREATE TABLE y1(a, b);
+  CREATE TABLE y2(a, b);
+} {}
+
+do_execsql_test 1.1 {
+  SELECT * FROM x1
+} {1 2 3}
+
+do_execsql_test 1.2 {
+  INSERT INTO y1 VALUES(1, 2) RETURNING rowid;
+} {1}
+
+do_execsql_test 1.3 {
+  CREATE TRIGGER y1tr BEFORE INSERT ON y1 BEGIN
+    SELECT * FROM x1;
+  END;
+  INSERT INTO y1 VALUES(3, 4) RETURNING rowid;
+} {2}
+
+
+# This time, rig the xBestIndex() method of the vtab to invoke an SQL
+# statement that uses RETURNING.
+set ::xbestindex_sql {
+  INSERT INTO y2 VALUES(NULL, NULL) RETURNING rowid;
+}
+do_execsql_test 1.4 {
+  INSERT INTO y1 VALUES(5, 6) RETURNING rowid;
+} {3}
+
+do_test 1.5 {
+  set ::xbestindex_res
+} {1}
+
+finish_test
diff --git a/test/bestindexC.test b/test/bestindexC.test
new file mode 100644
index 0000000000..8b96a19e6c
--- /dev/null
+++ b/test/bestindexC.test
@@ -0,0 +1,426 @@
+# 2024-04-26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindexC
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc vtab_command {lVal method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a)"
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set clist [$hdl constraints]
+      set orderby [$hdl orderby]
+
+      set idxstr [list]
+      set res [list]
+
+      set idx 0
+      foreach c $clist {
+        array set a $c
+        if {$a(usable)==0} continue
+        if {$a(op)=="limit" && ![info exists ::do_not_use_limit]} { 
+          lappend idxstr limit
+          lappend res omit $idx
+        }
+        if {$a(op)=="offset" && ![info exists ::do_not_use_offset]} { 
+          lappend idxstr offset
+          lappend res omit $idx
+        }
+        incr idx
+      }
+
+      return "cost 1000000 rows 1000000 idxnum 0 idxstr {$idxstr} $res"
+    }
+
+    xFilter {
+      set idxstr [lindex $args 1]
+      set LIMIT ""
+      foreach a $idxstr b [lindex $args 2] {
+        set x($a) $b
+      }
+
+      if {![info exists x(limit)]} { set x(limit) -1 }
+      if {![info exists x(offset)]} { set x(offset) -1 }
+      set LIMIT " LIMIT $x(limit) OFFSET $x(offset)"
+
+      set idx 1
+      foreach v $lVal {
+        lappend lRow "($idx, '$v')"
+        incr idx
+      }
+
+      return [list sql "
+        SELECT * FROM ( VALUES [join $lRow ,]) $LIMIT
+      "]
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command "a b c d e f");
+  CREATE VIRTUAL TABLE x2 USING tcl(vtab_command "A B C D E F a b");
+} {}
+
+do_execsql_test 1.1 {
+  CREATE TEMP TABLE t_unionall AS 
+    SELECT * FROM x1 UNION ALL SELECT * FROM x2;
+
+  CREATE TEMP TABLE t_intersect AS 
+    SELECT * FROM x1 INTERSECT SELECT * FROM x2;
+
+  CREATE TEMP TABLE t_union AS 
+    SELECT * FROM x1 UNION SELECT * FROM x2;
+
+  CREATE TEMP TABLE t_except AS 
+    SELECT * FROM x1 EXCEPT SELECT * FROM x2;
+}
+
+foreach {tn limit} {
+  1 "LIMIT 8" 
+  2 "LIMIT 4" 
+  3 "LIMIT 4 OFFSET 2" 
+  4 "LIMIT 8 OFFSET 4" 
+} {
+
+  foreach {op tbl} {
+    "UNION ALL" t_unionall
+    "UNION"     t_union
+    "INTERSECT" t_intersect
+    "EXCEPT"    t_except
+  } {
+
+    set expect [execsql "SELECT * FROM $tbl $limit"]
+    do_execsql_test 1.2.$tbl.$tn "SELECT * FROM (
+      SELECT * FROM x1 $op SELECT * FROM x2
+    ) $limit" $expect
+
+  }
+
+}
+
+#-------------------------------------------------------------------------
+reset_db
+register_tcl_module db
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command "a b c d e f");
+  CREATE VIRTUAL TABLE x2 USING tcl(vtab_command "a b e f");
+} {}
+
+do_execsql_test 2.1 {
+  SELECT * FROM x1 
+    EXCEPT
+  SELECT * FROM x2
+  LIMIT 3
+} {c d}
+
+#-------------------------------------------------------------------------
+reset_db
+register_tcl_module db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "1 2 3 4 5 6 7 8 9 10");
+} {}
+
+do_execsql_test 3.1 {
+  SELECT * FROM y1 WHERE a = COALESCE('8', a) LIMIT 3
+} {8}
+
+do_execsql_test 3.2 {
+  SELECT * FROM y1 WHERE a = '2' LIMIT 3
+} {2}
+
+load_static_extension db series
+do_execsql_test 3.3 {
+  SELECT * FROM generate_series(1, 5) WHERE value = (value & 14) LIMIT 3
+} {2 4}
+
+do_execsql_test 3.4 {
+  SELECT value FROM generate_series(1,10) WHERE value>2 LIMIT 4 OFFSET 1;
+} {4 5 6 7}
+
+set ::do_not_use_limit 1
+do_execsql_test 3.5 {
+  SELECT * FROM y1 LIMIT 5 OFFSET 3
+} {4 5 6 7 8}
+unset ::do_not_use_limit
+set ::do_not_use_offset 1
+do_execsql_test 3.6 {
+  SELECT * FROM y1 LIMIT 5 OFFSET 3
+} {4 5 6 7 8}
+unset ::do_not_use_offset
+
+#-------------------------------------------------------------------------
+reset_db
+proc vtab_command {lVal method args} {
+  switch -- $method {
+    xConnect { error "not happy!" }
+  }
+
+  return {}
+}
+
+register_tcl_module db
+do_catchsql_test 4.0 {
+  CREATE VIRTUAL TABLE y1 USING tcl(vtab_command 1);
+} {1 {not happy!}}
+do_test 4.1 {
+  sqlite3_errcode db
+} SQLITE_ERROR
+
+proc vtab_command {lVal method args} {
+  switch -- $method {
+    xConnect {
+      return $lVal
+    }
+  }
+  return {}
+}
+
+do_catchsql_test 4.2 {
+  CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "PRAGMA page_size=1024");
+} {1 {declare_vtab: syntax error}}
+do_catchsql_test 4.3 {
+  CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "CREATE TABLE x1(");
+} {1 {declare_vtab: incomplete input}}
+do_catchsql_test 4.4 {
+  CREATE VIRTUAL TABLE y1 USING tcl(vtab_command "CREATE TABLE x1(insert)");
+} {1 {declare_vtab: near "insert": syntax error}}
+
+#-------------------------------------------------------------------------
+reset_db
+register_tcl_module db
+
+proc quote {str} {
+  return "'[string map {' ''} $str]'"
+}
+
+proc vtab_command {lVal method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c, d)"
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set clist [$hdl constraints]
+
+      set res [list]
+      set idx 0
+      set idxnum 0
+
+      set cols(0) a
+      set cols(1) b
+      set cols(2) c
+
+      set lCons [list]
+
+      foreach c $clist {
+        array set a $c
+        if {$a(usable)==0} continue
+
+        if {($a(op)=="eq" || $a(op)=="is") && [info exists cols($a(column))]} {
+          lappend res omit $idx
+          set coll [$hdl collation $idx]
+          lappend lCons "$cols($a(column)) = %[llength $lCons]% COLLATE $coll"
+
+          set idxnum [expr {$idx + (1 << $a(column))}]
+          catch { unset cols($a(column)) }
+        }
+
+        incr idx
+      }
+
+      if {[llength [array names cols]]>0} {
+        set missing [list]
+        for {set i 0} {$i < 3} {incr i} {
+          catch { lappend missing $cols($i) }
+        }
+        set msg "missing required constraints: [join $missing ,]"
+        return [list constraint $msg]
+      }
+
+      set idxstr [join $lCons " AND "]
+      return "cost 1000 rows 1000 idxnum $idxnum $res idxstr {$idxstr}"
+    }
+
+    xFilter {
+      foreach {idxnum idxstr lArg} $args {}
+      set i 0
+      set where $idxstr
+      foreach a $lArg {
+        set where [string map [list %$i% [quote $a]] $where]
+        incr i
+      }
+   #   puts $where
+      return [list sql "SELECT rowid, * FROM $lVal WHERE $where"]
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command t1);
+  CREATE TABLE t1(a, b, c, d);
+}
+
+foreach {tn where ok} {
+  0    "WHERE a=? AND b=? AND c=? AND c=?"              1
+  1    "WHERE a=? AND b=? AND c=?"                      1
+  2    "WHERE a=? AND b=? AND (c=? OR c=?)"             1
+  3    "WHERE a=? AND b=? AND (c=? OR c=? OR c=?)"      1
+  4    "WHERE a=? AND b=? AND (c IS ? OR c IS ?)"       1
+  5    "WHERE a=? AND ((b=? AND c=?) OR (c=? AND b=?))" 1
+  6    "WHERE a=? AND ((b=? AND c=?) OR (c=?))"         0
+} {
+  do_test 5.2.$tn {
+    catch { execsql "SELECT * FROM x1 $::where" } msg
+#    if {$tn==0 || $tn==2 || $tn==3} { puts "MSG: $msg" }
+  } [expr !$ok]
+}
+
+do_execsql_test 5.3 {
+  SELECT * FROM x1 WHERE (a, b, c) = (?, ?, ?);
+}
+
+do_execsql_test 5.4 {
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(1, 'x', 'y', 'z', 'one');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(2, 'X', 'Y', 'Z', 'two');
+  SELECT * FROM x1 WHERE (a, b, c) = ('X', 'Y', 'Z');
+} {X Y Z two}
+do_execsql_test 5.5 {
+  SELECT * FROM x1 WHERE a='x' AND b='y' AND c='z';
+} {x y z one}
+do_execsql_test 5.6 {
+  SELECT * FROM x1 
+  WHERE a='x' COLLATE nocase AND b='y' COLLATE nocase AND c='z'COLLATE nocase;
+} {x y z one X Y Z two}
+
+do_execsql_test 5.7 {
+  DELETE FROM t1;
+
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(0, 'x', 'y', 'z', 'zero');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(1, 'x', 'y', 'Z', 'one');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(2, 'x', 'Y', 'z', 'two');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(3, 'x', 'Y', 'Z', 'three');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(4, 'X', 'y', 'z', 'four');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(5, 'X', 'y', 'Z', 'five');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(6, 'X', 'Y', 'z', 'six');
+  INSERT INTO t1(rowid, a, b, c, d) VALUES(7, 'X', 'Y', 'z', 'seven');
+}
+
+do_execsql_test 5.8 {
+  SELECT d FROM x1 
+  WHERE a='x' AND ((b='y' AND c='z') OR (b='Y' AND c='z' COLLATE nocase))
+} {
+  zero two three
+}
+
+do_execsql_test 5.9 {
+  SELECT d FROM x1 
+  WHERE a='x' COLLATE nocase 
+  AND ((b='y' AND c='z') OR (b='Y' AND c='z' COLLATE nocase))
+} {
+  zero four two
+  three six seven
+}
+
+#--------------------------------------------------------------------------
+
+reset_db
+register_tcl_module db
+
+proc quote {str} {
+  return "'[string map {' ''} $str]'"
+}
+
+proc vtab_command {lVal method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c, d)"
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set clist [$hdl constraints]
+
+      set idx 0
+      set idxnum 0
+
+      foreach c $clist {
+        array set a $c
+        if {$a(usable)==0} continue
+
+        if {$a(op)=="limit"} {
+          set idxnum [$hdl rhs_value $idx 555]
+        }
+
+        incr idx
+      }
+
+      return "cost 1000 rows 1000 idxnum $idxnum"
+
+    }
+
+    xFilter {
+      foreach {idxnum idxstr lArg} $args {}
+      return [list sql "SELECT 0, $idxnum, $idxnum, $idxnum, $idxnum"]
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(2, 2);
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command t1);
+}
+
+do_execsql_test 6.1 { SELECT * FROM x1 LIMIT 50 } {50 50 50 50}
+
+do_execsql_test 6.2 { SELECT * FROM x1 WHERE b=c LIMIT 5 } {0 0 0 0}
+
+do_execsql_test 6.3 { 
+  SELECT (SELECT a FROM x1 WHERE t1.x=t1.y LIMIT 10) FROM t1
+} {0}
+
+do_execsql_test 6.4 { 
+  SELECT (SELECT a FROM x1 WHERE x1.a=1) FROM t1
+} {1}
+
+do_execsql_test 6.5 { 
+  SELECT (SELECT a FROM x1 WHERE x1.a=1 LIMIT 1) FROM t1
+} {1}
+
+do_execsql_test 6.6 { 
+  SELECT (SELECT a FROM x1 WHERE x1.a=555 LIMIT 2) FROM t1
+} {555}
+
+finish_test
+
+
diff --git a/test/bestindexD.test b/test/bestindexD.test
new file mode 100644
index 0000000000..b06d6b4270
--- /dev/null
+++ b/test/bestindexD.test
@@ -0,0 +1,93 @@
+# 2024-08-03
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindexD
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID"
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set ::colUsed [$hdl mask]
+
+      set cost 1000000
+      set used ""
+
+      set cons 0
+      foreach c [$hdl constraints] {
+        set cost [expr $cost/10]
+        append used " use $cons"
+        incr cons
+      }
+
+      return "cost $cost rows $cost $used"
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+
+  CREATE TABLE t2(a, b);
+} {}
+
+# This proc assumes that there is only one use of a virtual table - x1 -
+# in SQL statement $sql. It tests that the colUsed value passed to the
+# xBestIndex method matches the actual columns used, which is ascertained 
+# by searching the compiled VM code for VColumn instructions.
+#
+proc do_colsused_test {tn sql} {
+  set ::colUsed ""
+  execsql $sql
+  set got $::colUsed
+
+  set expect 0
+  db eval "EXPLAIN $sql" x {
+    if {$x(opcode)=="VColumn"} {
+      set expect [expr $expect | (1<<$x(p2))]
+    }
+  }
+
+  uplevel [list do_test $tn.($expect/$got) [list expr ($expect & $got)] $expect]
+}
+
+do_colsused_test 1.1 { SELECT a FROM x1 }   
+do_colsused_test 1.2 { SELECT a,c FROM x1 } 
+do_colsused_test 1.3 { SELECT b FROM x1 }   
+do_colsused_test 1.4 { SELECT b FROM x1 WHERE c=? } 
+
+do_colsused_test 1.5 {
+  select 1 from t2 full join x1;
+}
+
+do_colsused_test 1.6 {
+  select 1 from x1 WHERE (b=? AND c=?) OR (b=? AND c=?)
+}
+
+finish_test
+
+
diff --git a/test/bestindexE.test b/test/bestindexE.test
new file mode 100644
index 0000000000..d225d74aef
--- /dev/null
+++ b/test/bestindexE.test
@@ -0,0 +1,191 @@
+# 2025-09-02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindexE
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc pretty_constraint {lCol cons} {
+  array set C $cons
+
+  set ret ""
+  if {$C(usable)} {
+    set OP(eq) =
+    set ret "[lindex $lCol $C(column)]$OP($C(op))?"
+  }
+
+  return $ret
+}
+
+proc vtab_command {zName lCol method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE $zName ([join $lCol ,]) "
+    }
+
+    xBestIndex {
+      set hdl [lindex $args 0]
+      set conslist [$hdl constraints]
+
+      set ret [list]
+      foreach cons $conslist {
+        set pretty [pretty_constraint $lCol $cons]
+        if {$pretty != ""} { lappend ret $pretty }
+      }
+
+      lappend ::xBestIndex "$zName: [join $ret { AND }]"
+
+      return "cost 1000 rows 1000 idxnum 555"
+    }
+  }
+
+  return {}
+}
+
+proc do_bestindex_test {tn sql lCons} {
+  set ::xBestIndex [list]
+  do_execsql_test $tn.1 $sql
+  uplevel [list do_test $tn.2 [list set ::xBestIndex] [list {*}$lCons]]
+}
+
+proc create_vtab {tname clist} {
+  set cmd [list vtab_command $tname $clist]
+  execsql "
+    CREATE VIRTUAL TABLE $tname USING tcl('$cmd')
+  "
+}
+
+do_test 1.0 {
+  create_vtab x1 {a b c}
+} {}
+
+do_bestindex_test 1.1 {
+  SELECT * FROM x1 WHERE a=?
+} {{x1: a=?}}
+
+do_bestindex_test 1.2 {
+  SELECT * FROM x1 WHERE a=? AND b=?
+} {{x1: a=? AND b=?}}
+
+#--------------------------------------------------------------------------
+reset_db
+register_tcl_module db
+
+do_test 2.0 {
+  create_vtab Delivery       {id customer}
+  create_vtab ReturnDelivery {id customer}
+  create_vtab Customer       {oid name}
+} {}
+
+do_bestindex_test 2.1 {
+  SELECT Delivery.ID, Customer.Name
+  FROM Delivery LEFT JOIN
+  Customer ON Delivery.Customer = Customer.OID
+} {
+  {Delivery: }
+  {Customer: oid=?}
+}
+
+do_bestindex_test 2.2 {
+  SELECT * FROM
+  (
+     SELECT Delivery.ID, Customer.Name
+     FROM Delivery LEFT JOIN
+     Customer ON Delivery.Customer = Customer.OID
+
+     UNION
+
+     SELECT ReturnDelivery.ID, Customer.Name
+     FROM ReturnDelivery LEFT JOIN
+     Customer ON ReturnDelivery.Customer = Customer.OID
+  )
+  WHERE ID = 1
+} {
+  {Delivery: id=?} 
+  {Customer: oid=?} 
+  {ReturnDelivery: id=?} 
+  {Customer: oid=?}
+}
+
+#--------------------------------------------------------------------------
+reset_db
+register_tcl_module db eponymous_cmd
+
+proc eponymous_cmd {method args} {
+  switch -- $method {
+    xConnect {
+      db eval { SELECT * FROM sqlite_schema }
+      return "CREATE TABLE t1 (a, b)"
+    }
+
+    xBestIndex {
+      return "idxnum 555"
+    }
+
+    xFilter {
+      return [list sql {SELECT 123, 'A', 'B'}]
+    }
+
+    xUpdate {
+      return 123
+    }
+
+  }
+
+  return {}
+}
+
+do_execsql_test 3.1.0 {
+  PRAGMA table_info = tcl
+} {
+  0 a {} 0 {} 0 1 b {} 0 {} 0
+}
+do_execsql_test 3.1.1 {
+  SELECT rowid, * FROM tcl
+} {123 A B}
+do_execsql_test 3.1.2 {
+  INSERT INTO tcl VALUES('i', 'ii') RETURNING *;
+} {i ii}
+do_test 3.1.3 {
+  db last_insert_rowid
+} {123}
+
+do_execsql_test 3.1.4 {
+  CREATE TABLE x1(x);
+}
+
+db close
+sqlite3 db  test.db
+register_tcl_module db eponymous_cmd
+sqlite3 db2 test.db
+
+# Load the schema into connection [db]
+do_execsql_test 3.2.1 { SELECT * FROM x1 }
+
+# Modify the schema on disk
+do_execsql_test -db db2 3.2.2 { CREATE TABLE x2(x) }
+
+# Insert with RETURNING trigger on eponymous table.
+do_execsql_test 3.2.3 {
+  INSERT INTO tcl VALUES('i', 'ii') RETURNING *;
+} {i ii}
+
+finish_test
+
diff --git a/test/between.test b/test/between.test
index 16c3913d12..5e02ef9b22 100644
--- a/test/between.test
+++ b/test/between.test
@@ -140,4 +140,21 @@ foreach {tn expr res} {
   do_execsql_test between-2.1.$tn $sql $res
 }
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test between-3.0 {
+  CREATE TABLE t1(x, y);
+  CREATE INDEX i1 ON t1(x);
+  INSERT INTO t1 VALUES(4, 4);
+  CREATE TABLE t2(a, b);
+}
+
+do_execsql_test between-3.1 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (x BETWEEN 1 AND 3);
+} {4 4 {} {}}
+
+do_execsql_test between-3.2 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (x BETWEEN 5 AND 7);
+} {4 4 {} {}}
+
 finish_test
diff --git a/test/bigrow.test b/test/bigrow.test
index fa59c36252..83270cb764 100644
--- a/test/bigrow.test
+++ b/test/bigrow.test
@@ -220,4 +220,15 @@ do_test bigrow-5.99 {
   execsql {DROP TABLE t1}
 } {}
 
+if {$SQLITE_MAX_LENGTH > 1200*1000*1000} {
+  # If SQLITE_MAX_LENGTH is large enough, check that a serial type greater
+  # than 0x7FFFFFFF does not cause trouble.
+  set v [string repeat A 1100000000]
+  do_execsql_test bigrow-6.0 {
+    CREATE TABLE docs(content TEXT);
+    INSERT INTO docs VALUES ( $v );
+  }
+  do_execsql_test bigrow-6.1 { SELECT content FROM docs } [list $v]
+}
+
 finish_test
diff --git a/test/bloom1.test b/test/bloom1.test
index 151f364ae0..09553c3b9b 100644
--- a/test/bloom1.test
+++ b/test/bloom1.test
@@ -183,6 +183,59 @@ do_execsql_test 4.3 {
 do_execsql_test 4.4 {
   SELECT * FROM t0 LEFT JOIN t1 LEFT JOIN t2 ON (b NOTNULL)==(c IN ()) WHERE c;
 } {xyz {} 7.0}
- 
+
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE t1 (c1);
+  INSERT INTO t1 VALUES (101);
+  CREATE TABLE t2 ( x );
+  INSERT INTO t2 VALUES(404);
+}
+
+do_execsql_test 5.1 {
+  SELECT 'val' in (
+      select 'val' from ( select 'valueB' from t1 order by 1 ) 
+      union all 
+      select 'val'
+  );
+} {1}
+
+do_execsql_test 5.2 {
+  select * from t2
+    where 'val' in (
+        select 'val' from ( select 'valueB' from t1 order by 1 ) 
+        union all 
+        select 'val'
+    );
+} {404}
+
+do_execsql_test 5.3 {
+  SELECT subq_1.c_0 as c_0 
+  FROM ( SELECT 0 as c_0) as subq_1
+  WHERE (subq_1.c_0) IN (
+    SELECT subq_2.c_0 as c_0
+    FROM (
+      SELECT 0 as c_0
+      FROM t1 as ref_1
+      WHERE (ref_1.c1) = (2)
+      ORDER BY c_0 desc
+    ) as subq_2
+    UNION ALL
+    SELECT 0 as c_0
+  );
+} {0}
+
+# 2025-04-30 https://sqlite.org/forum/forumpost/792a09cb3df9e69f
+# A continuation of the above.
+#
+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a);
+  SELECT 111 IN (
+    SELECT 222 FROM (SELECT 333 ORDER BY 1)
+    UNION ALL
+    SELECT 444 FROM (SELECT 555 FROM t1 ORDER BY 1)
+  );
+} 0
 
 finish_test
diff --git a/test/btree01.test b/test/btree01.test
index 6e4717ae65..b1909a3adb 100644
--- a/test/btree01.test
+++ b/test/btree01.test
@@ -17,7 +17,7 @@ source $testdir/tester.tcl
 set testprefix btree01
 
 # The refactoring on the b-tree balance() routine in check-in
-# http://www.sqlite.org/src/info/face33bea1ba3a (2014-10-27)
+# http://sqlite.org/src/info/face33bea1ba3a (2014-10-27)
 # caused the integrity_check on the following SQL to fail.
 #
 do_execsql_test btree01-1.1 {
diff --git a/test/busy.test b/test/busy.test
index be0515b013..896c7fa651 100644
--- a/test/busy.test
+++ b/test/busy.test
@@ -106,7 +106,7 @@ do_test 3.4 {
 proc busy_handler {n} { return 1 }
 do_test 3.5 {
   catchsql { PRAGMA optimize }
-} {0 {}}
+} {1 {database is locked}}
 
 do_test 3.6 {
   execsql { COMMIT } db2
diff --git a/test/capi3.test b/test/capi3.test
index 40d87ac8e3..6319d8284d 100644
--- a/test/capi3.test
+++ b/test/capi3.test
@@ -181,7 +181,7 @@ do_test capi3-3.4 {
 do_test capi3-3.5 {
   list [sqlite3_system_errno $db2] [sqlite3_close $db2]
 } [list $::capi3_errno SQLITE_OK]
-if {[clang_sanitize_address]==0} {
+if {[clang_sanitize_address]==0 && 0} {
   do_test capi3-3.6.1-misuse {
     sqlite3_close $db2
   } {SQLITE_MISUSE}
@@ -689,7 +689,9 @@ do_test capi3-6.3 {
   sqlite3_finalize $STMT
 } {SQLITE_OK}
 
-if {[clang_sanitize_address]==0} {
+if {0 && [clang_sanitize_address]==0} {
+  # This use-after-free occasionally causes segfaults during ordinary
+  # builds.  Let's just disable it completely.
   do_test capi3-6.4-misuse {
     db cache flush
     sqlite3_close $DB
diff --git a/test/carray01.test b/test/carray01.test
index 1af9cb66e3..86ea069961 100644
--- a/test/carray01.test
+++ b/test/carray01.test
@@ -20,7 +20,7 @@ ifcapable !vtab {
   finish_test
   return
 }
-load_static_extension db carray
+#load_static_extension db carray
 
 # Parameter $stmt must be a prepared statement created using
 # the sqlite3_prepare_v2 command and with parameters fully bound.
diff --git a/test/carray02.test b/test/carray02.test
new file mode 100644
index 0000000000..c75ca42719
--- /dev/null
+++ b/test/carray02.test
@@ -0,0 +1,162 @@
+# 2025-10-07
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file implements tests for CARRAY extension
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix carray02
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+proc run_stmt {stmt} {
+  set r {}
+  while {[sqlite3_step $stmt]=="SQLITE_ROW"} {
+    for {set i 0} {$i<[sqlite3_data_count $stmt]} {incr i} {
+      lappend r [sqlite3_column_text $stmt $i]
+    }
+  }
+  sqlite3_reset $stmt
+  return $r
+}
+
+# Bind some text values using sqlite3_carray_bind(). Including a NULL pointer.
+#
+set STMT [
+  sqlite3_prepare_v2 db "SELECT value, value IS NULL FROM carray(?)" -1 TAIL
+]
+do_test 1.0 {
+  sqlite3_carray_bind -transient -text $STMT 1 "abc" NULL "def"
+  run_stmt $STMT
+} {abc 0 {} 1 def 0}
+sqlite3_finalize $STMT
+
+# Pass bad values as the "flags" parameter to sqlite3_carray_bind().
+#
+set STMT [sqlite3_prepare_v2 db "SELECT value FROM carray(?)" -1 TAIL]
+do_test 2.0 {
+  list [catch {sqlite3_carray_bind -flags 5 -int32 $STMT 1 1 2 3 4} msg] $msg
+} {1 {SQL logic error}}
+do_test 2.1 {
+  list [catch {sqlite3_carray_bind -flags -1 -int32 $STMT 1 1 2 3 4} msg] $msg
+} {1 {SQL logic error}}
+sqlite3_finalize $STMT
+
+# In each of the following, carray(?) contains integer values 1 to 5, bound
+# using sqlite3_carray_bind().
+#
+foreach {tn sql res} {
+  1 { SELECT value FROM carray(?) WHERE value>2 } {3 4 5}
+  2 { 
+    WITH s(i) AS ( VALUES(1) UNION ALL VALUES(2) )
+    SELECT i, value FROM s, carray(?) WHERE i=value;
+  } {1 1 2 2}
+} {
+  do_test 2.2.$tn {
+    set STMT [sqlite3_prepare_v2 db $sql -1 TAIL]
+    sqlite3_carray_bind -int32 $STMT 1 1 2 3 4 5
+    set r [run_stmt $STMT]
+    sqlite3_finalize $STMT
+    set r
+  } $res
+}
+
+# In each of the following, ? is bound to an int array containing 1 to 5.
+# Bound using C code like:
+#
+#     sqlite3_bind_pointer(pStmt, 1, aInt, "carray", SQLITE_TRANSIENT)
+#
+foreach {tn sql res} {
+  1 { SELECT value FROM carray(?, 5) } {1 2 3 4 5}
+  2 { SELECT value FROM carray(?, 3, 'int32') } {1 2 3}
+  3 { SELECT value, pointer, count, ctype FROM carray(?, 5, 'int32') } 
+    {1 {} 5 int32 2 {} 5 int32 3 {} 5 int32 4 {} 5 int32 5 {} 5 int32}
+  4 { SELECT rowid, value FROM carray(?, 5, 'int32') } 
+    {1 1 2 2 3 3 4 4 5 5}
+} {
+  do_test 2.3.$tn {
+    set STMT [sqlite3_prepare_v2 db $sql -1 TAIL]
+    bind_carray_intptr $STMT 1   1 2 3 4 5
+    set r [run_stmt $STMT]
+    sqlite3_finalize $STMT
+    set r
+  } $res
+}
+
+# In each of the following. Both carray(?1) and carray(?2) contain integer
+# values 1 to 5. Bound by sqlite3_carray_bind().
+#
+foreach {tn sql res} {
+  1 { 
+    SELECT * FROM carray(?1) AS a, carray(?2) AS b 
+    WHERE a.value=b.value
+  } {1 1 2 2 3 3 4 4 5 5}
+
+  2 { 
+    SELECT * FROM carray(?1) AS a, carray(?2) AS b 
+    WHERE a.value=b.value AND a.value<3 AND b.value<3
+  } {1 1 2 2 3 3}
+
+  3 { 
+    SELECT * FROM carray(?1) AS a, carray(?2) AS b 
+    WHERE a.value<3 AND b.value<3 AND a.value=b.value
+  } {1 1 2 2 3 3}
+
+  4 { 
+    SELECT * FROM carray(?1) AS a, carray(?2, a.value) AS b 
+    WHERE a.value=b.value
+  } {1 1 2 2 3 3}
+
+} {
+  do_test 2.4.$tn {
+    set STMT [sqlite3_prepare_v2 db {
+      SELECT * FROM carray(?1) AS a, carray(?2) AS b WHERE a.value=b.value
+    } -1 TAIL]
+    sqlite3_carray_bind $STMT 1   1 2 3 4 5
+    sqlite3_carray_bind $STMT 2   1 2 3 4 5
+    set r [run_stmt $STMT]
+    sqlite3_finalize $STMT
+    set r
+  } {1 1 2 2 3 3 4 4 5 5}
+}
+
+# Test that not binding any pointer, or passing a value that is not a bound
+# pointer to carray() produces no rows of output.
+#
+do_execsql_test 3.0.0 {
+  SELECT * FROM carray
+} {}
+do_execsql_test 3.0.1 {
+  SELECT * FROM carray('0xFFFF', 5)
+} {}
+do_execsql_test 3.0.2 {
+  SELECT * FROM carray('0xFFFF')
+} {}
+do_execsql_test 3.0.3 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES('0xFFFF');
+  SELECT * FROM t1, carray WHERE carray.pointer = t1.x;
+} {}
+
+# Test passing an unknown datatype.
+#
+do_test 3.1 {
+  set STMT [sqlite3_prepare_v2 db {SELECT * FROM carray(?, 5, 'apples')} -1 T]
+  sqlite3_carray_bind $STMT 1   1 2 3 4 5
+  sqlite3_step $STMT
+  list [sqlite3_finalize $STMT] [sqlite3_errmsg db]
+} {SQLITE_ERROR {unknown datatype: 'apples'}}
+
+finish_test
diff --git a/test/carrayfault.test b/test/carrayfault.test
new file mode 100644
index 0000000000..0dd2cd21b7
--- /dev/null
+++ b/test/carrayfault.test
@@ -0,0 +1,87 @@
+# 2025-10-07
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix carrayfault
+
+ifcapable {!carray||!vtab} {
+  finish_test; return
+}
+
+sqlite3 db test.db
+
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a);
+}
+
+set STMT [sqlite3_prepare_v2 db "SELECT value FROM carray(?)" -1 TAIL]
+
+# Test OOM faults in sqlite3_carray_bind() when binding an integer
+# array.
+#
+foreach {tn mem} {
+  1 -static
+  2 -transient
+  3 -malloc
+} {
+  do_faultsim_test 2.$tn -faults oom* -prep {
+  } -body {
+    sqlite3_carray_bind $::mem -int64 $::STMT 1   100 200 300 400
+  } -test {
+    faultsim_test_result {0 {}} {1 {out of memory}}
+  }
+}
+
+# Test OOM faults in sqlite3_carray_bind() when binding a text array.
+#
+do_faultsim_test 3 -faults oom* -prep {
+} -body {
+    sqlite3_carray_bind -transient -text $::STMT 1 "hello" "world"
+} -test {
+  faultsim_test_result {0 {}} {1 {initialization of carray failed: }}
+}
+
+sqlite3_finalize $STMT
+
+
+# Test OOM faults when running queries against carray.
+#
+do_faultsim_test 4 -faults oom* -prep {
+  set ::STMT [sqlite3_prepare_v2 db "SELECT value FROM carray(?)" -1 TAIL]
+  sqlite3_carray_bind -transient -text $::STMT 1 "hello" "world"
+} -body {
+    set myres [list]
+    while { "SQLITE_ROW"==[sqlite3_step $::STMT] } {
+      lappend myres [sqlite3_column_text $::STMT 1]
+    }
+    sqlite3_reset $::STMT
+} -test {
+  faultsim_test_result {0 SQLITE_OK} {0 SQLITE_NOMEM}
+  sqlite3_finalize $::STMT
+}
+
+# Test OOM faults when preparing queries against carray.
+#
+do_faultsim_test 5 -faults oom* -prep {
+  sqlite3 db test.db
+} -body {
+  execsql "SELECT value FROM carray(?)"
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+
+
+sqlite3_carray_bind
+
+finish_test
diff --git a/test/cast.test b/test/cast.test
index ebfea5aa06..c1b564328a 100644
--- a/test/cast.test
+++ b/test/cast.test
@@ -234,6 +234,7 @@ do_test cast-3.1 {
 do_test cast-3.2 {
   execsql {SELECT CAST(9223372036854774800 AS numeric)}
 } 9223372036854774800
+breakpoint
 do_realnum_test cast-3.3 {
   execsql {SELECT CAST(9223372036854774800 AS real)}
 } 9.22337203685477e+18
@@ -261,11 +262,9 @@ do_test cast-3.12 {
 do_realnum_test cast-3.13 {
   execsql {SELECT CAST('9223372036854774800' AS real)}
 } 9.22337203685477e+18
-ifcapable long_double {
-  do_test cast-3.14 {
-    execsql {SELECT CAST(CAST('9223372036854774800' AS real) AS integer)}
-  } 9223372036854774784
-}
+do_test cast-3.14 {
+  execsql {SELECT CAST(CAST('9223372036854774800' AS real) AS integer)}
+} 9223372036854774784
 do_test cast-3.15 {
   execsql {SELECT CAST('-9223372036854774800' AS integer)}
 } -9223372036854774800
@@ -275,11 +274,9 @@ do_test cast-3.16 {
 do_realnum_test cast-3.17 {
   execsql {SELECT CAST('-9223372036854774800' AS real)}
 } -9.22337203685477e+18
-ifcapable long_double {
-  do_test cast-3.18 {
-    execsql {SELECT CAST(CAST('-9223372036854774800' AS real) AS integer)}
-  } -9223372036854774784
-}
+do_test cast-3.18 {
+  execsql {SELECT CAST(CAST('-9223372036854774800' AS real) AS integer)}
+} -9223372036854774784
 if {[db eval {PRAGMA encoding}]=="UTF-8"} {
   do_test cast-3.21 {
     execsql {SELECT CAST(x'39323233333732303336383534373734383030' AS integer)}
@@ -290,14 +287,12 @@ if {[db eval {PRAGMA encoding}]=="UTF-8"} {
   do_realnum_test cast-3.23 {
     execsql {SELECT CAST(x'39323233333732303336383534373734383030' AS real)}
   } 9.22337203685477e+18
-  ifcapable long_double {
-    do_test cast-3.24 {
-      execsql {
-        SELECT CAST(CAST(x'39323233333732303336383534373734383030' AS real)
-                    AS integer)
-      }
-    } 9223372036854774784
-  }
+  do_test cast-3.24 {
+    execsql {
+      SELECT CAST(CAST(x'39323233333732303336383534373734383030' AS real)
+                  AS integer)
+    }
+  } 9223372036854774784
 }
 do_test cast-3.31 {
   execsql {SELECT CAST(NULL AS numeric)}
@@ -390,7 +385,7 @@ do_execsql_test cast-6.1 {
 } {9000000000000000001 9000000000000000001 9000000000000000001 9000000000000000001}
 
 # 2019-06-07
-# https://www.sqlite.org/src/info/4c2d7639f076aa7c
+# https://sqlite.org/src/info/4c2d7639f076aa7c
 do_execsql_test cast-7.1 {
   SELECT CAST('-' AS NUMERIC);
 } {0}
@@ -405,7 +400,7 @@ do_execsql_test cast-7.4 {
 } {0}
 
 # 2019-06-07
-# https://www.sqlite.org/src/info/e8bedb2a184001bb
+# https://sqlite.org/src/info/e8bedb2a184001bb
 do_execsql_test cast-7.10 {
   SELECT '' - 2851427734582196970;
 } {-2851427734582196970}
@@ -417,7 +412,7 @@ do_execsql_test cast-7.12 {
 } {-1}
 
 # 2019-06-10
-# https://www.sqlite.org/src/info/dd6bffbfb6e61db9
+# https://sqlite.org/src/info/dd6bffbfb6e61db9
 #
 # EVIDENCE-OF: R-55084-10555 Casting a TEXT or BLOB value into NUMERIC
 # yields either an INTEGER or a REAL result.
@@ -446,7 +441,7 @@ do_execsql_test cast-7.33 {
 } 0
 
 # 2019-06-12
-# https://www.sqlite.org/src/info/674385aeba91c774
+# https://sqlite.org/src/info/674385aeba91c774
 #
 do_execsql_test cast-7.40 {
   SELECT CAST('-0.0' AS numeric);
diff --git a/test/changes.test b/test/changes.test
index 21db075f96..b3a2ae1eef 100644
--- a/test/changes.test
+++ b/test/changes.test
@@ -86,5 +86,3 @@ foreach {tn nRow wor} {
 }
 
 finish_test
-
-
diff --git a/test/changes2.test b/test/changes2.test
index 46e25c03a7..5b2684a8df 100644
--- a/test/changes2.test
+++ b/test/changes2.test
@@ -92,4 +92,3 @@ do_execsql_test 2.4 {
 } {{2 changes} {2 changes}}
 
 finish_test
-
diff --git a/test/check.test b/test/check.test
index 10d1cf4be6..c3beb2f5d8 100644
--- a/test/check.test
+++ b/test/check.test
@@ -612,4 +612,36 @@ do_catchsql_test 12.81 {
   INSERT INTO t1(a) VALUES(456);
 } {1 {CHECK constraint failed: a NOT BETWEEN +a AND 999999}}
 
+#-------------------------------------------------------------------------
+
+reset_db
+
+do_execsql_test 13.1.0 {
+  CREATE TABLE Table0 (Col0 , CHECK(Table0.Col0 NOT NULL ) ) ;
+  REPLACE INTO Table0 VALUES (hex(randomblob(100000)));
+}
+integrity_check 13.1.1
+do_execsql_test 13.1.2 {
+  UPDATE OR REPLACE Table0 SET Col0 = Table0.Col0 ;
+}
+integrity_check 13.1.3
+do_execsql_test 13.1.4 {
+  SELECT length(col0) FROM table0;
+} {200000}
+
+do_execsql_test 13.2.0 {
+  CREATE TABLE t2 (x , CHECK((NOT (x ISNULL) )));
+  REPLACE INTO t2 VALUES (hex(randomblob(100000)));
+}
+do_execsql_test 13.2.1 {
+  SELECT length(x) FROM t2
+} {200000}
+do_execsql_test 13.2.2 {
+  UPDATE OR REPLACE t2 SET x = x;
+}
+do_execsql_test 13.2.3 {
+  SELECT length(x) FROM t2
+} {200000}
+integrity_check 13.2.4
+
 finish_test
diff --git a/test/chunksize.test b/test/chunksize.test
index 47d118d841..be3795fc5c 100644
--- a/test/chunksize.test
+++ b/test/chunksize.test
@@ -14,7 +14,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix chunksize
 
-if {$::tcl_platform(platform)!="unix"} {
+if {$::tcl_platform(platform) ne "unix"} {
   finish_test
   return
 }
diff --git a/test/cksumvfs.test b/test/cksumvfs.test
new file mode 100644
index 0000000000..2bf4d91873
--- /dev/null
+++ b/test/cksumvfs.test
@@ -0,0 +1,93 @@
+# 2024 March 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix cksumvfs
+
+
+db close
+sqlite3_shutdown
+#test_sqlite3_log logfunc
+sqlite3_initialize
+
+proc logfunc {args} {
+  puts "LOG: $args"
+}
+
+sqlite3_register_cksumvfs
+sqlite3 db test.db
+
+file_control_reservebytes db 8
+execsql {
+  PRAGMA page_size = 4096;
+}
+
+set text [db one "SELECT hex(randomblob(5000))"]
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  INSERT INTO t1 VALUES(1, $text, NULL);
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1;
+} [list 1 $text {}]
+
+do_execsql_test 1.2 {
+  DELETE FROM t1;
+}
+
+do_test 1.3 {
+  execsql BEGIN
+  for {set ii 1500} {$ii < 10000} {incr ii} {
+    execsql { INSERT INTO t1 VALUES(NULL, randomblob(5000), randomblob($ii)) }
+  }
+  execsql COMMIT
+} {}
+
+do_execsql_test 1.4 {
+  SELECT count(b) FROM t1
+} {8500}
+
+do_execsql_test 1.5 {
+  PRAGMA journal_mode = wal;
+  DELETE FROM t1;
+} {wal}
+
+do_execsql_test 1.6 {
+  PRAGMA wal_checkpoint;
+} {/0 # #/}
+
+do_execsql_test 1.7 {
+  WITH s(i) AS (
+    VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO t1 SELECT NULL, randomblob(5000), randomblob(i) FROM s;
+  SELECT count(*) FROM t1;
+} {100}
+
+db_save_and_close
+db_restore_and_reopen
+
+do_execsql_test 1.8 {
+  SELECT count(*) FROM t1;
+} {100}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 1.9 {
+  SELECT count(*) FROM t1;
+} {100}
+
+finish_test
diff --git a/test/collate1.test b/test/collate1.test
index b65b850474..da662bee62 100644
--- a/test/collate1.test
+++ b/test/collate1.test
@@ -23,7 +23,7 @@ set testprefix collate1
 # collate1-1.* - Single-field ORDER BY with an explicit COLLATE clause.
 # collate1-2.* - Multi-field ORDER BY with an explicit COLLATE clause.
 # collate1-3.* - ORDER BY using a default collation type. Also that an 
-#                explict collate type overrides a default collate type.
+#                explicit collate type overrides a default collate type.
 # collate1-4.* - ORDER BY using a data type.
 #
 
diff --git a/test/collateB.test b/test/collateB.test
index 3c8d50d8e6..678eb5af0f 100644
--- a/test/collateB.test
+++ b/test/collateB.test
@@ -62,7 +62,7 @@ do_execsql_test collateB-1.17 {
 } {1 11 1 11 |}
 
 #-------------------------------------------------------------------------
-# Test an assert() failure that was occuring if an index were created
+# Test an assert() failure that was occurring if an index were created
 # on a column explicitly declared "COLLATE binary".
 reset_db
 do_execsql_test 2.1 {
diff --git a/test/colmeta.test b/test/colmeta.test
index 21b0e0f38a..43d2c83cb6 100644
--- a/test/colmeta.test
+++ b/test/colmeta.test
@@ -97,7 +97,7 @@ foreach {tn params results} $tests {
 }
 
 # Calling sqlite3_table_column_metadata with a NULL column name merely
-# checks for the existance of the table.
+# checks for the existence of the table.
 #
 do_test colmeta-300 {
   catch {sqlite3_table_column_metadata $::DB main xyzzy} res
diff --git a/test/colname.test b/test/colname.test
index 5fa0b601f9..0907df641e 100644
--- a/test/colname.test
+++ b/test/colname.test
@@ -378,7 +378,7 @@ do_test colname-9.210 {
   execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3}
 } {a 1 n 3}
 
-# 2017-12-23:  Ticket https://www.sqlite.org/src/info/3b4450072511e621
+# 2017-12-23:  Ticket https://sqlite.org/src/info/3b4450072511e621
 # Inconsistent column names in CREATE TABLE AS
 #
 # Verify that the names of columns in the created table of a CREATE TABLE AS
@@ -424,7 +424,7 @@ do_catchsql_test colname-9.400 {
 #
 do_catchsql_test colname-9.410 {
   CREATE TABLE t5 AS SELECT RAISE(abort,a);
-} {1 {RAISE() may only be used within a trigger-program}}
+} {1 {no such column: a}}
 
 # Make sure the quotation marks get removed from the column names
 # when constructing a new table from an aggregate SELECT.
diff --git a/test/conflict.test b/test/conflict.test
index ba3ad9b0dd..54c01d36de 100644
--- a/test/conflict.test
+++ b/test/conflict.test
@@ -824,7 +824,7 @@ do_test conflict-13.2 {
 } {1 3}
 
 
-# Ticket https://www.sqlite.org/src/tktview/e6f1f2e34dceeb1ed61531c7e9
+# Ticket https://sqlite.org/src/tktview/e6f1f2e34dceeb1ed61531c7e9
 # Verify that it is not possible to sneak a NULL value into a NOT NULL
 # column using REPLACE.
 #
diff --git a/test/corrupt.test b/test/corrupt.test
index 62ead4d9a7..ff61cc0bbe 100644
--- a/test/corrupt.test
+++ b/test/corrupt.test
@@ -290,7 +290,7 @@ ifcapable oversize_cell_check {
   # detecting corruption was not possible at that point, and an assert() failed.
   #
   set fd [open test.db r+]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   seek $fd [expr 1024+8]
   puts -nonewline $fd "\x03\x14"
   close $fd
diff --git a/test/corrupt2.test b/test/corrupt2.test
index 96d28490aa..fc24cb376f 100644
--- a/test/corrupt2.test
+++ b/test/corrupt2.test
@@ -69,7 +69,7 @@ do_test corrupt2-1.3 {
   forcedelete corrupt.db-journal
   forcecopy test.db corrupt.db
   set f [open corrupt.db RDWR]
-  fconfigure $f -encoding binary
+  fconfigure $f -translation binary
   seek $f 16 start
   puts -nonewline $f "\x00\xFF"
   close $f
@@ -89,7 +89,7 @@ do_test corrupt2-1.4 {
   forcedelete corrupt.db-journal
   forcecopy test.db corrupt.db
   set f [open corrupt.db RDWR]
-  fconfigure $f -encoding binary
+  fconfigure $f -translation binary
   seek $f 101 start
   puts -nonewline $f "\xFF\xFF"
   close $f
@@ -109,7 +109,7 @@ do_test corrupt2-1.5 {
   forcedelete corrupt.db-journal
   forcecopy test.db corrupt.db
   set f [open corrupt.db RDWR]
-  fconfigure $f -encoding binary
+  fconfigure $f -translation binary
   seek $f 101 start
   puts -nonewline $f "\x00\xC8"
   seek $f 200 start
@@ -179,7 +179,7 @@ do_test corrupt2-3.1 {
   # of the DROP TABLE operation below.
   #
   set fd [open corrupt.db r+]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   seek $fd [expr 1024*3 + 12]
   set zCelloffset [read $fd 2]
   binary scan $zCelloffset S iCelloffset
@@ -227,7 +227,7 @@ do_test corrupt2-5.1 {
   # This block links a page from table t2 into the t1 table structure.
   #
   set fd [open corrupt.db r+]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   seek $fd [expr 1024 + 12]
   set zCelloffset [read $fd 2]
   binary scan $zCelloffset S iCelloffset
@@ -248,7 +248,7 @@ do_test corrupt2-5.1 {
   }
   set result
 } {{*** in database main ***
-Tree 11 page 2 cell 0: 2nd reference to page 10
+Tree 2 page 2 cell 0: 2nd reference to page 10
 Page 4: never used}}
 
 db2 close
@@ -392,7 +392,7 @@ corruption_test -sqlprep $sqlprep -corrupt {
   # 0x0D (interpreted by SQLite as "leaf page of a table B-Tree").
   #
   set fd [open corrupt.db r+]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   seek $fd [expr 1024*2 + 8] 
   set zRightChild [read $fd 4]
   binary scan $zRightChild I iRightChild
@@ -410,7 +410,7 @@ corruption_test -sqlprep $sqlprep -corrupt {
   # The corruption is detected as part of an OP_Prev opcode.
   #
   set fd [open corrupt.db r+]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   seek $fd [expr 1024*2 + 12] 
   set zCellOffset [read $fd 2]
   binary scan $zCellOffset S iCellOffset
@@ -431,7 +431,7 @@ corruption_test -sqlprep $sqlprep -corrupt {
   # 0x0A (interpreted by SQLite as "leaf page of an index B-Tree").
   #
   set fd [open corrupt.db r+]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   seek $fd [expr 1024*1 + 8] 
   set zRightChild [read $fd 4]
   binary scan $zRightChild I iRightChild
@@ -459,7 +459,7 @@ corruption_test -sqlprep {
   CREATE TABLE x6(a, b, c); CREATE TABLE xD(a, b, c); CREATE TABLE xJ(a, b, c);
 } -corrupt {
   set fd [open corrupt.db r+]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   seek $fd 108
   set zRightChild [read $fd 4]
   binary scan $zRightChild I iRightChild
diff --git a/test/corrupt4.test b/test/corrupt4.test
index 5b0965a836..544ac33053 100644
--- a/test/corrupt4.test
+++ b/test/corrupt4.test
@@ -122,7 +122,7 @@ proc put4byte {fd offset val} {
 # the second rightmost child page number of page 1 to 1.
 #
 set fd [open test.db r+]
-fconfigure $fd -encoding binary -translation binary
+fconfigure $fd -translation binary
 set nChild [get2byte $fd 103]
 set offChild [get2byte $fd [expr 100+12+($nChild-2)*2]]
 set pgnoChild [get4byte $fd $offChild]
diff --git a/test/corruptC.test b/test/corruptC.test
index f5733a8186..bf324c120f 100644
--- a/test/corruptC.test
+++ b/test/corruptC.test
@@ -98,7 +98,7 @@ do_test corruptC-2.1 {
   sqlite3 db test.db
   catchsql {PRAGMA integrity_check}
 } {0 {{*** in database main ***
-Tree 3 page 3: free space corruption}}}
+Tree 3 page 3: free space corruption} {wrong # of entries in index t1i1}}}
 
 # test that a corrupt content offset size is handled (seed 5649)
 #
diff --git a/test/corruptD.test b/test/corruptD.test
index c35388adfb..381e52c808 100644
--- a/test/corruptD.test
+++ b/test/corruptD.test
@@ -113,7 +113,7 @@ do_test corruptD-1.1.1 {
   hexio_write test.db [expr 1024+1] FFFF
   catchsql { PRAGMA quick_check }
 } {0 {{*** in database main ***
-Tree 2 page 2: free space corruption}}}
+Tree 2 page 2: free space corruption} {wrong # of entries in index i1}}}
 do_test corruptD-1.1.2 {
   incr_change_counter
   hexio_write test.db [expr 1024+1] [hexio_render_int32 1021]
diff --git a/test/corruptK.test b/test/corruptK.test
index 1569afe4a8..0bdb3c45b5 100644
--- a/test/corruptK.test
+++ b/test/corruptK.test
@@ -60,7 +60,7 @@ do_test 1.2 {
   sqlite3 db test.db
 
   set fd [db incrblob t1 x 3]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   seek $fd 30
   puts -nonewline $fd "\x18"
   close $fd
@@ -102,7 +102,7 @@ do_test 2.2 {
 
   sqlite3 db test.db
   set fd [db incrblob t1 x 5]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
 
   seek $fd 22
   puts -nonewline $fd "\x5d"
diff --git a/test/corruptL.test b/test/corruptL.test
index 65d9821b35..52adf6fd72 100644
--- a/test/corruptL.test
+++ b/test/corruptL.test
@@ -1269,6 +1269,7 @@ do_test 15.0 {
 }]} {}
 
 extra_schema_checks 0
+optimization_control db one-pass off
 do_catchsql_test 15.1 {
   PRAGMA cell_size_check = 0;
   UPDATE c1 SET c= NOT EXISTS(SELECT 1 FROM c1 ORDER BY (SELECT 1 FROM c1 ORDER BY a)) +10 WHERE d BETWEEN 4 AND 7;
@@ -1504,4 +1505,89 @@ do_catchsql_test 19.4 {
   PRAGMA integrity_check;
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_test 18.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 20480 pagesize 4096 filename crash-a4150b729051e4.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     64: 00 00 00 00 00 00 00 00 00 00 ff f0 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 04 0e e5 00 0f c2 0f 75   ...............u
+|    112: 0f 19 0e e5 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|   3808: 00 00 00 00 00 32 04 06 17 17 11 01 4b 69 6e 64   .....2......Kind
+|   3824: 65 78 74 31 61 62 63 74 31 05 43 52 45 41 54 45   ext1abct1.CREATE
+|   3840: 20 49 4e 44 45 58 20 74 31 61 62 63 20 4f 4e 20    INDEX t1abc ON 
+|   3856: 74 31 28 61 2c 62 2c 63 29 5a 03 06 17 25 25 01   t1(a,b,c)Z...%%.
+|   3872: 79 74 61 62 6c 65 73 71 6c 69 74 65 5f 73 74 61   ytablesqlite_sta
+|   3888: 74 34 73 71 6c 69 74 65 5f 73 74 61 74 34 04 43   t4sqlite_stat4.C
+|   3904: 52 45 41 54 45 20 54 41 42 4c 45 20 73 71 6c 69   REATE TABLE sqli
+|   3920: 74 65 5f 73 74 61 74 34 28 74 62 6c 2c 69 64 78   te_stat4(tbl,idx
+|   3936: 2c 6e 65 71 2c 6e 6c 74 2c 6e 64 6c 74 2c 73 61   ,neq,nlt,ndlt,sa
+|   3952: 6d 70 6c 65 29 4b 02 06 17 25 25 01 5b 74 61 62   mple)K...%%.[tab
+|   3968: 6c 65 73 71 6c 69 74 65 5f 73 74 61 74 31 73 71   lesqlite_stat1sq
+|   3984: 6c 69 74 65 5f 73 74 61 74 31 03 43 52 45 41 54   lite_stat1.CREAT
+|   4000: 45 20 54 41 42 4c 45 20 73 71 6c 69 74 65 5f 73   E TABLE sqlite_s
+|   4016: 74 61 74 31 28 74 62 6c 2c 69 64 78 2c 73 74 61   tat1(tbl,idx,sta
+|   4032: 74 29 3c 01 06 17 11 11 01 65 74 61 62 6c 65 74   t)<......etablet
+|   4048: 31 74 31 02 43 52 45 41 54 45 20 54 41 42 4c 45   1t1.CREATE TABLE
+|   4064: 20 74 31 28 61 20 54 45 58 54 2c 20 62 20 49 4e    t1(a TEXT, b IN
+|   4080: 54 2c 20 63 20 49 4e 54 2c 20 64 20 49 4e 54 29   T, c INT, d INT)
+| page 2 offset 4096
+|      0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   4000: 0b 07 05 13 01 01 01 62 63 64 64 06 0b 0c 06 05   .......bcdd.....
+|   4016: 13 02 01 01 64 65 66 01 59 09 0a 0c 05 05 13 03   ....def.Y.......
+|   4032: 01 01 64 65 66 02 6f 08 09 0c 04 05 13 02 01 01   ..def.o.........
+|   4048: 61 62 63 01 59 07 08 0c 03 05 13 02 01 01 87 62   abc.Y..........b
+|   4064: 63 00 ea 06 07 0c 02 05 13 02 01 01 61 62 63 00   c...........abc.
+|   4080: ea 06 06 0b 01 05 13 01 01 01 61 62 63 7b 04 04   ..........abc...
+| page 3 offset 8192
+|      0: 0d 00 00 00 01 0f e0 00 0f e1 00 00 00 00 00 00   ................
+|   4064: 00 1d 01 04 11 17 31 74 31 74 31 61 62 63 31 30   ......1t1t1abc10
+|   4080: 30 30 30 20 35 30 30 30 20 32 30 30 30 20 31 30   000 5000 2000 10
+| page 4 offset 12288
+|      0: 0d 00 00 00 07 0e ac 00 0f d1 0f a0 0f 6f 0f 3e   .............o.>
+|     16: 0f 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+|   3744: 00 00 00 00 00 00 00 00 00 00 00 00 2f 07 07 11   ............/...
+|   3760: 17 1b 1b 1b 24 74 31 74 31 61 62 63 32 20 31 20   ....$t1t1abc2 1 
+|   3776: 31 20 31 35 20 36 20 36 20 36 32 20 35 20 36 20   1 15 6 6 62 5 6 
+|   3792: 36 05 13 02 01 01 64 65 66 02 37 08 05 2f 06 07   6.....def.7../..
+|   3808: 11 17 1b 1b 1b 24 74 41 74 31 61 62 63 32 20 31   .....$tAt1abc2 1
+|   3824: 20 31 20 31 35 20 35 20 55 20 35 32 20 34 20 35    1 15 5 U 52 4 5
+|   3840: 20 35 05 13 02 01 01 64 65 66 01 59 09 06 2e 05    5.....def.Y....
+|   3856: 07 11 17 1b 1b 1b 22 74 31 74 31 61 62 63 31 20   .......t1t1abc1 
+|   3872: 31 20 31 20 31 34 20 34 20 34 20 34 31 20 33 20   1 1 14 4 4 41 3 
+|   3888: 34 20 34 08 b3 cd f0 f1 62 63 64 64 06 07 2f 05   4 4.....bcdd../.
+|   3904: 07 11 17 1b 1b 1b 24 74 37 74 31 61 62 63 34 20   ......$t7t1abc4 
+|   3920: 31 20 31 20 31 30 20 33 20 33 20 33 30 20 32 20   1 1 10 3 3 30 2 
+|   3936: 33 20 33 05 13 02 01 01 61 62 63 01 59 07 04 2f   3 3.....abc.Y../
+|   3952: 03 07 11 17 1b 1b 1b 24 74 31 74 31 61 62 63 34   .......$t1t1abc4
+|   3968: 20 32 20 31 20 31 30 20 31 20 32 20 32 30 20 31    2 1 10 1 2 20 1
+|   3984: 20 32 20 32 05 13 02 01 01 61 62 63 00 ea 06 03    2 2.....abc....
+|   4000: 2f 02 07 11 17 1b 1b 1b 24 74 31 74 31 61 62 63   /.......$t1t1abc
+|   4016: 34 20 32 20 31 20 31 30 20 31 20 31 20 31 30 20   4 2 1 10 1 1 10 
+|   4032: 31 20 31 20 31 05 13 02 01 01 61 62 63 00 ea 05   1 1 1.....abc...
+|   4048: 02 2d 01 07 11 17 1b 1b 1b 20 74 31 74 31 61 62   .-....... t1t1ab
+|   4064: 63 34 20 31 20 31 20 31 30 20 30 20 30 1f 30 30   c4 1 1 10 0 0.00
+|   4080: 20 30 20 30 20 30 05 13 01 01 09 61 62 63 7b 04    0 0 0.....abc..
+| page 5 offset 16384
+|      0: 0a 00 00 00 07 0f a8 00 0f f5 00 00 00 00 00 00   ................
+|   4000: 00 00 00 00 00 00 00 00 0c 05 13 02 01 01 64 65   ..............de
+|   4016: 66 02 37 08 05 0c 05 13 02 01 01 64 65 66 01 59   f.7........def.Y
+|   4032: 09 06 0b 05 12 01 01 01 62 63 64 64 06 07 0c 05   ........bcdd....
+|   4048: 13 02 01 01 61 62 63 01 59 07 01 2c 05 13 02 01   ....abc.Y..,....
+|   4064: 01 61 62 63 00 ea 06 03 0c 05 13 02 01 01 61 62   .abc..........ab
+|   4080: 63 00 ea 05 00 00 00 00 00 00 00 00 00 00 00 00   c...............
+| end crash-a4150b729051e4.db
+}]} {}
+
+do_catchsql_test 18.1 {
+  SELECT a FROM t1 WHERE b GLOB b AND b GLOB '0^x]␅6␚xz]';
+} {1 {database disk image is malformed}}
+
 finish_test
diff --git a/test/corruptN.test b/test/corruptN.test
index 376325f5c5..2297991aba 100644
--- a/test/corruptN.test
+++ b/test/corruptN.test
@@ -141,15 +141,6 @@ do_test 2.0 {
 | end c-b92b.txt.db
 }]} {}
 
-# This test only works with the legacy RC4 PRNG
-if 0 {
-  prng_seed 0 db
-  do_catchsql_test 2.1 {
-  SELECT count(*) FROM sqlite_schema;
-  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
-  INSERT INTO t1(a) SELECT randomblob(null) FROM c;
-  } {1 {database disk image is malformed}}
-}
 
 reset_db
 if {![info exists ::G(perm:presql)]} {
@@ -276,4 +267,31 @@ do_catchsql_test 6.3 {
   UPDATE t1 SET x='hello world' WHERE rowid=1;
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  BEGIN;
+  CREATE TABLE p1(x PRIMARY KEY);
+  CREATE TABLE c1(y);
+
+  PRAGMA schema_version = 0;
+  PRAGMA writable_schema = RESET;
+
+  INSERT INTO c1 VALUES(1000);
+  ROLLBACK;
+}
+
+do_execsql_test 7.1 {
+  PRAGMA table_info = p1;
+} {0 x {} 0 {} 1}
+
+do_catchsql_test 7.2 {
+  SELECT * FROM p1;
+} {1 {database disk image is malformed}}
+
+do_catchsql_test 7.3 {
+  PRAGMA integrity_check
+} {1 {database disk image is malformed}}
+
+
 finish_test
diff --git a/test/cost.test b/test/cost.test
index 5684177a13..6106caba8c 100644
--- a/test/cost.test
+++ b/test/cost.test
@@ -103,7 +103,7 @@ do_eqp_test 5.2 {
 } {
   QUERY PLAN
   |--SCAN t2 USING INDEX t2i1
-  `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+  `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY
 }
 
 do_eqp_test 5.3 {
diff --git a/test/crash.test b/test/crash.test
index c1901daec6..f631636599 100644
--- a/test/crash.test
+++ b/test/crash.test
@@ -399,7 +399,7 @@ do_test crash-7.1 {
 
   # Change the checksum value for the master journal name.
   set f [open test.db-journal a]
-  fconfigure $f -encoding binary
+  fconfigure $f -translation binary
   seek $f [expr [file size test.db-journal] - 12]
   puts -nonewline $f "\00\00\00\00"
   close $f
diff --git a/test/crash8.test b/test/crash8.test
index c07829979f..b2b01183fd 100644
--- a/test/crash8.test
+++ b/test/crash8.test
@@ -356,7 +356,7 @@ ifcapable pragma {
 # is not created on F2FS file-systems that support atomic
 # write. So do not run these tests in that case either.
 #
-if {$::tcl_platform(platform)=="unix" && [atomic_batch_write test.db]==0 } {
+if {$::tcl_platform(os) ne "Windows NT" && [atomic_batch_write test.db]==0 } {
   for {set i 1} {$i < 10} {incr i} {
     catch { db close }
     forcedelete test.db test.db-journal
diff --git a/test/date.test b/test/date.test
index 3e93181896..5ff131f733 100644
--- a/test/date.test
+++ b/test/date.test
@@ -146,6 +146,8 @@ datetest 2.49 {datetime('2003-10-22 12:24','0000 second')} {2003-10-22 12:24:00}
 datetest 2.50 {datetime('2003-10-22 12:24','0001 second')} {2003-10-22 12:24:01}
 datetest 2.51 {datetime('2003-10-22 12:24','nonsense')} NULL
 
+datetest 2.60 {datetime('2023-02-31')} {2023-03-03 00:00:00}
+
 datetest 3.1 {strftime('%d','2003-10-31 12:34:56.432')} 31
 datetest 3.2.1 {strftime('pre%fpost','2003-10-31 12:34:56.432')} pre56.432post
 datetest 3.2.2 {strftime('%f','2003-10-31 12:34:59.9999999')} 59.999
@@ -207,15 +209,33 @@ datetest 3.16 "strftime('[repeat 200 %Y]','2003-10-31')" [repeat 200 2003]
 datetest 3.17 "strftime('[repeat 200 abc%m123]','2003-10-31')" \
     [repeat 200 abc10123]
 
-foreach c {a b c e g h i k l n o p q r t v x y z
-           A B C D E F G I K L N O P Q R T U V Z
+foreach c {a b c h i n o q r t v x y z
+           A B C D E K L N O Q Z
            0 1 2 3 4 5 6 6 7 9 _} {
   datetest 3.18.$c "strftime('%$c','2003-10-31')" NULL
 }
+datetest 3.20 {strftime('%e','2023-08-09')} { 9}
+datetest 3.21 {strftime('%F %T','2023-08-09 01:23')} {2023-08-09 01:23:00}
+datetest 3.22 {strftime('%k','2023-08-09 04:59:59')} { 4}
+datetest 3.23 {strftime('%I%P','2023-08-09 11:59:59')} {11am}
+datetest 3.24 {strftime('%I%p','2023-08-09 12:00:00')} {12PM}
+datetest 3.25 {strftime('%I%P','2023-08-09 12:59:59.9')} {12pm}
+datetest 3.26 {strftime('%I%p','2023-08-09 13:00:00')} {01PM}
+datetest 3.27 {strftime('%I%P','2023-08-09 23:59:59')} {11pm}
+datetest 3.28 {strftime('%I%p','2023-08-09 00:00:00')} {12AM}
+datetest 3.29 {strftime('%l:%M%P','2023-08-09 13:00:00')} { 1:00pm}
+datetest 3.30 {strftime('%F %R','2023-08-09 12:34:56')} {2023-08-09 12:34}
+datetest 3.31 {strftime('%w %u','2023-01-01')} {0 7}
+datetest 3.32 {strftime('%w %u','2023-01-02')} {1 1}
+datetest 3.33 {strftime('%w %u','2023-01-03')} {2 2}
+datetest 3.34 {strftime('%w %u','2023-01-04')} {3 3}
+datetest 3.35 {strftime('%w %u','2023-01-05')} {4 4}
+datetest 3.36 {strftime('%w %u','2023-01-06')} {5 5}
+datetest 3.37 {strftime('%w %u','2023-01-07')} {6 6}
 
 # Ticket #2276.  Make sure leading zeros are inserted where appropriate.
 #
-datetest 3.20 \
+datetest 3.40 \
    {strftime('%d/%f/%H/%W/%j/%m/%M/%S/%Y','0421-01-02 03:04:05.006')} \
    02/05.006/03/00/002/01/04/05/0421
 
@@ -242,7 +262,7 @@ datetest 5.15 {datetime('1994-04-16 14:00:00 +05:00 Z')} NULL
 # localtime->utc and utc->localtime conversions.
 #
 # Use SQLITE_TESTCTRL_LOCALTIME_FAULT=2 to set an alternative localtime_r()
-# implementation that is not locale-dependent.  This testing localtime_r()
+# implementation that is not locale-dependent.  The testing localtime_r()
 # operates as follows:
 #
 #     (1)  Localtime is 30 minutes earlier than (west of) UTC on
@@ -301,6 +321,58 @@ utc_to_local 6.22 {1800-10-29 12:30:00} {1800-10-29 12:00:00}
 local_to_utc 6.23 {3000-10-30 12:00:00} {3000-10-30 11:30:00}
 utc_to_local 6.24 {3000-10-30 11:30:00} {3000-10-30 12:00:00}
 
+# If the time is specified to be ZULU, or if it has an explicit
+# timezone extension, then the time will already be UTC and subsequent
+# 'utc' modifiers are no-ops.
+#
+# Forum post 2025-09-17T10:12:14 - The "+00:00" suffix should work like "Z"
+#
+do_execsql_test date-6.25.1 {
+  SELECT datetime('2000-10-29 12:00Z','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.25.2 {
+  SELECT datetime('2000-10-29 12:00 +00:00','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.25.3 {
+  SELECT datetime('2000-10-29 12:00+00:00','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.25.4 {
+  SELECT datetime('2000-10-29 12:00:00+00:00','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.25.5 {
+  SELECT datetime('2000-10-29 12:00 -00:00','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.25.6 {
+  SELECT datetime('2000-10-29 12:00-00:00','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.25.7 {
+  SELECT datetime('2000-10-29 12:00:00-00:00','utc','utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.26 {
+  SELECT datetime('2000-10-29 12:00:00+05:00');
+} {{2000-10-29 07:00:00}}
+do_execsql_test date-6.27 {
+  SELECT datetime('2000-10-29 12:00:00+05:00', 'utc');
+} {{2000-10-29 07:00:00}}
+
+# Multiple back-and-forth UTC to LOCAL to UTC...
+do_execsql_test date-6.28 {
+  SELECT datetime('2000-10-29 12:00:00Z', 'localtime');
+} {{2000-10-29 12:30:00}}
+do_execsql_test date-6.29 {
+  SELECT datetime('2000-10-29 12:00:00Z', 'utc', 'localtime');
+} {{2000-10-29 12:30:00}}
+do_execsql_test date-6.30 {
+  SELECT datetime('2000-10-29 12:00:00Z', 'utc', 'localtime', 'utc');
+} {{2000-10-29 12:00:00}}
+do_execsql_test date-6.31 {
+  SELECT datetime('2000-10-29 12:00:00Z', 'utc','localtime','utc','localtime');
+} {{2000-10-29 12:30:00}}
+do_execsql_test date-6.32 {
+  SELECT datetime('2000-10-29 12:00:00Z', 'localtime','localtime');
+} {{2000-10-29 12:30:00}}
+
+
 # Restore the use of the OS localtime_r() before going on...
 sqlite3_test_control SQLITE_TESTCTRL_LOCALTIME_FAULT 0
 
@@ -434,6 +506,9 @@ datetest 13.31 {date('2001-01-01','+1.5 years')} {2002-07-02}
 datetest 13.32 {date('2002-01-01','+1.5 years')} {2003-07-02}
 datetest 13.33 {date('2002-01-01','-1.5 years')} {2000-07-02}
 datetest 13.34 {date('2001-01-01','-1.5 years')} {1999-07-02}
+datetest 13.35 {date('2023-02-28')} {2023-02-28}
+datetest 13.36 {date('2023-02-29')} {2023-03-01}
+datetest 13.37 {date('2023-04-31')} {2023-05-01}
 
 # Test for issues reported by BareFeet (list.sql at tandb.com.au)
 # on mailing list on 2008-06-12.
@@ -529,7 +604,7 @@ datetest 16.30 {datetime(5373484,'-14712 years')} {-4713-12-31 12:00:00}
 datetest 16.31 {datetime(5373484,'-14713 years')} NULL
 
 # 2017-03-02: Wrong 'start of day' computation.
-# https://www.sqlite.org/src/info/6097cb92745327a1
+# https://sqlite.org/src/info/6097cb92745327a1
 #
 datetest 17.1 {datetime(2457754, 'start of day')} {2016-12-31 00:00:00}
 datetest 17.2 {datetime(2457828)} {2017-03-15 12:00:00}
@@ -550,4 +625,59 @@ datetest 18.2 {unixepoch('1970-01-01T00:00:00.1', 'subsec')} {0.1}
 datetest 18.3 {unixepoch('1970-01-01T00:00:00.2', 'subsecond')} {0.2}
 datetest 18.4 {julianday('-4713-11-24 13:40:48.864', 'subsec')} {0.07001}
 datetest 18.5 {typeof(unixepoch('now', 'subsecond'))} {real}
+
+# 2024-03-03 the 'ceiling' and 'floor' operators.
+#
+datetest 19.1 {date('2000-01-31','floor')} {2000-01-31}
+datetest 19.2a {date('2000-02-31','floor')} {2000-02-29}
+datetest 19.2b {date('1999-02-31','floor')} {1999-02-28}
+datetest 19.2c {date('1900-02-31','floor')} {1900-02-28}
+datetest 19.3 {date('2000-03-31','floor')} {2000-03-31}
+datetest 19.4 {date('2000-04-31','floor')} {2000-04-30}
+datetest 19.5 {date('2000-05-31','floor')} {2000-05-31}
+datetest 19.6 {date('2000-06-31','floor')} {2000-06-30}
+datetest 19.7 {date('2000-07-31','floor')} {2000-07-31}
+datetest 19.8 {date('2000-08-31','floor')} {2000-08-31}
+datetest 19.9 {date('2000-09-31','floor')} {2000-09-30}
+datetest 19.10 {date('2000-10-31','floor')} {2000-10-31}
+datetest 19.11 {date('2000-11-31','floor')} {2000-11-30}
+datetest 19.12 {date('2000-12-31','floor')} {2000-12-31}
+datetest 19.21 {date('2000-01-31','ceiling')} {2000-01-31}
+datetest 19.22a {date('2000-02-31','ceiling')} {2000-03-02}
+datetest 19.22b {date('1999-02-31','ceiling')} {1999-03-03}
+datetest 19.22c {date('1900-02-31','ceiling')} {1900-03-03}
+datetest 19.23 {date('2000-03-31','ceiling')} {2000-03-31}
+datetest 19.24 {date('2000-04-31','ceiling')} {2000-05-01}
+datetest 19.25 {date('2000-05-31','ceiling')} {2000-05-31}
+datetest 19.26 {date('2000-06-31','ceiling')} {2000-07-01}
+datetest 19.27 {date('2000-07-31','ceiling')} {2000-07-31}
+datetest 19.28 {date('2000-08-31','ceiling')} {2000-08-31}
+datetest 19.29 {date('2000-09-31','ceiling')} {2000-10-01}
+datetest 19.30 {date('2000-10-31','ceiling')} {2000-10-31}
+datetest 19.31 {date('2000-11-31','ceiling')} {2000-12-01}
+datetest 19.32 {date('2000-12-31','ceiling')} {2000-12-31}
+datetest 19.40 {date('2024-01-31','+1 month','ceiling')} {2024-03-02}
+datetest 19.41 {date('2024-01-31','+1 month','floor')} {2024-02-29}
+datetest 19.42 {date('2023-01-31','+1 month','ceiling')} {2023-03-03}
+datetest 19.43 {date('2023-01-31','+1 month','floor')} {2023-02-28}
+datetest 19.44 {date('2024-02-29','+1 year','ceiling')} {2025-03-01}
+datetest 19.45 {date('2024-02-29','+1 year','floor')} {2025-02-28}
+datetest 19.46 {date('2024-02-29','-110 years','ceiling')} {1914-03-01}
+datetest 19.47 {date('2024-02-29','-110 years','floor')} {1914-02-28}
+datetest 19.48 {date('2024-02-29','-0110-00-00','floor')} {1914-02-28}
+datetest 19.49 {date('2024-02-29','-0110-00-00','ceiling')} {1914-03-01}
+datetest 19.50 {date('2000-08-31','+0023-06-00','floor')} {2024-02-29}
+datetest 19.51 {date('2000-08-31','+0022-06-00','floor')} {2023-02-28}
+datetest 19.52 {date('2000-08-31','+0023-06-00','ceiling')} {2024-03-02}
+datetest 19.53 {date('2000-08-31','+0022-06-00','ceiling')} {2023-03-03}
+
+# 2025-01-21
+# https://sqlite.org/forum/forumpost/766a2c9231
+#
+datetest 20.1 {datetime('2024-12-31 23:59:59.9990')} {2024-12-31 23:59:59}
+datetest 20.2 {datetime('2024-12-31 23:59:59.9999999999999')} \
+                 {2024-12-31 23:59:59}
+datetest 20.3 {datetime('2024-12-31 23:59:59.9995')} {2024-12-31 23:59:59}
+datetest 20.4 {datetime('2024-12-31 23:59:58.9995')} {2024-12-31 23:59:58}
+
 finish_test
diff --git a/test/date4.test b/test/date4.test
new file mode 100644
index 0000000000..4e936b71c3
--- /dev/null
+++ b/test/date4.test
@@ -0,0 +1,44 @@
+# 2023-08-29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for the strftime() SQL function.  Comparisons against the
+# C-library strftime() function.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Skip this whole file if date and time functions are omitted
+# at compile-time
+#
+ifcapable {!datetime} {
+  finish_test
+  return
+}
+
+if {$tcl_platform(os)=="Linux"} {
+  if {"" eq [strftime {%P} 1]} {
+    # This is probably musl libc, which does not support
+    # %k, %l, %P
+    set FMT {%d,%e,%F,%H,%I,%j,%m,%M,%u,%w,%W,%Y,%%,%p,%U,%V,%G,%g}
+  } else {
+    set FMT {%d,%e,%F,%H,%k,%I,%l,%j,%m,%M,%u,%w,%W,%Y,%%,%P,%p,%U,%V,%G,%g}
+  }
+} else {
+  set FMT {%d,%e,%F,%H,%I,%j,%p,%R,%u,%w,%W,%%}
+}
+for {set i 0} {$i<=24858} {incr i} {
+  set TS [expr {$i*86390}]
+  do_execsql_test date4-$i {
+    SELECT strftime($::FMT,$::TS,'unixepoch');
+  } [list [strftime $FMT $TS]]
+}
+
+finish_test
diff --git a/test/date5.test b/test/date5.test
new file mode 100644
index 0000000000..688f84d0f1
--- /dev/null
+++ b/test/date5.test
@@ -0,0 +1,86 @@
+# 2024-08-19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# https://sqlite.org/forum/forumpost/eaa0a09786c6368b
+#
+# Apparently SQLite has been miscomputing leap-year dates before
+# the year 0400.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Skip this whole file if date and time functions are omitted
+# at compile-time
+#
+ifcapable {!datetime} {
+  finish_test
+  return
+}
+
+# Data sources:
+#   1-10  https://ssd.jpl.nasa.gov/tools/jdc/#/cd
+#   11    Jean Meeus, Astronomical Algorithms, ISBN 0-943396-61-1, p.59
+#   12    https://en.wikipedia.org/wiki/Julian_day
+#   
+# ID YEAR MONTH DAY JD
+set date5data {
+   1 2024     2  29 2460369.5
+   2 2024     3   1 2460370.5
+   3 2023     2  28 2460003.5
+   4 2023     3   1 2460004.5
+   5 2000     2  29 2451603.5
+   6 2000     3   1 2451604.5
+   7 1900     2  28 2415078.5
+   8 1900     3   1 2415079.5
+   9 1712     2  29 2346413.5
+  10 1712     3   1 2346414.5
+  11 1977     4  26 2443259.5
+  12 2013     1   1 2456293.5
+}
+
+foreach {id y m d jd} $date5data {
+  set date [format %04d-%02d-%02d $y $m $d]
+  do_execsql_test date5-jd$jd {
+    SELECT date($::jd);
+  } $date
+  do_execsql_test date5-cal/$date {
+    SELECT julianday($::date);
+  } $jd
+  for {set i 1} {$y+400*$i<=9999} {incr i} {
+    set y2 [expr {$y+400*$i}]
+    set date2 [format %04d-%02d-%02d $y2 $m $d]
+    set jd2 [expr {$jd+146097*$i}]
+    do_execsql_test date5-jd$jd2 {
+      SELECT date($::jd2);
+    } $date2
+    do_execsql_test date5-cal/$date2 {
+      SELECT julianday($::date2);
+    } $jd2
+  }
+  for {set i 1} {$y-400*$i>=-4712} {incr i} {
+    set y2 [expr {$y-400*$i}]
+    if {$y2<0} {
+      set date2 [format -%04d-%02d-%02d [expr {-$y2}] $m $d]
+    } else {
+      set date2 [format %04d-%02d-%02d $y2 $m $d]
+    }
+    set jd2 [expr {$jd-146097*$i}]
+    do_execsql_test date5-jd$jd2 {
+      SELECT date($::jd2);
+    } $date2
+    do_execsql_test date5-cal/$date2 {
+      SELECT julianday($::date2);
+    } $jd2
+  }
+}
+
+finish_test
diff --git a/test/dbfuzz.c b/test/dbfuzz.c
index ca1c6ea217..1587bacbbd 100644
--- a/test/dbfuzz.c
+++ b/test/dbfuzz.c
@@ -368,7 +368,7 @@ static int inmemDelete(
   return SQLITE_IOERR_DELETE;
 }
 
-/* Check for the existance of a file
+/* Check for the existence of a file
 */
 static int inmemAccess(
   sqlite3_vfs *pVfs,
diff --git a/test/dbfuzz001.test b/test/dbfuzz001.test
index 2a430de12e..228dd16db6 100644
--- a/test/dbfuzz001.test
+++ b/test/dbfuzz001.test
@@ -371,4 +371,27 @@ do_catchsql_test dbfuzz001-330 {
 } {1 {database disk image is malformed}}
 extra_schema_checks 1
 
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test dbfuzz001-430 {
+  CREATE TABLE t1(a INTEGER, b INT, c DEFAULT 0);
+}
+
+do_execsql_test dbfuzz001-420 {
+  PRAGMA locking_mode=EXCLUSIVE;
+  PRAGMA journal_mode = memory;
+  INSERT INTO t1 VALUES(1,2,3);
+  PRAGMA journal_mode=PERSIST;
+} {exclusive memory persist}
+
+do_execsql_test dbfuzz001-430 {
+  INSERT INTO t1 VALUES(4, 5, 6);
+}
+
+do_execsql_test dbfuzz001-440 {
+  PRAGMA journal_mode=MEMORY;
+  INSERT INTO t1 VALUES(7, 8, 9);
+} {memory}
+
 finish_test
diff --git a/test/dblwidth-a.sql b/test/dblwidth-a.sql
new file mode 100644
index 0000000000..38c219698d
--- /dev/null
+++ b/test/dblwidth-a.sql
@@ -0,0 +1,20 @@
+/*
+** Run this script using "sqlite3" to confirm that the command-line
+** shell properly handles the output of double-width characters.
+**
+** https://sqlite.org/forum/forumpost/008ac80276
+*/
+.mode box
+CREATE TABLE data(word TEXT, description TEXT);
+INSERT INTO data VALUES('〈οὐκέτι〉','Greek without dblwidth <...>');
+.print .mode box
+SELECT * FROM data;
+.mode table
+.print .mode table
+SELECT * FROM data;
+.mode qbox
+.print .mode qbox
+SELECT * FROM data;
+.mode column
+.print .mode column
+SELECT * FROM data;
diff --git a/test/dbpage.test b/test/dbpage.test
index 0646a70b02..8039e0e1be 100644
--- a/test/dbpage.test
+++ b/test/dbpage.test
@@ -108,4 +108,148 @@ do_execsql_test 300 {
   SELECT * FROM sqlite_temp_schema, sqlite_dbpage;
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 400 {
+  ATTACH ':memory:' AS aux1;
+  BEGIN;
+    CREATE VIRTUAL TABLE aux1.t1 USING sqlite_dbpage;
+    INSERT INTO t1 VALUES(17, NULL);
+  COMMIT;
+}
+
+#-------------------------------------------------------------------------
+reset_db
+forcedelete test.db2
+sqlite3 db2 test.db2
+db2 eval {
+  PRAGMA auto_vacuum=NONE;
+  CREATE TABLE t1(x, y);
+}
+
+do_execsql_test 500 {
+  PRAGMA auto_vacuum=NONE;
+  CREATE TABLE x1(a);
+  INSERT INTO x1 VALUES( hex(randomblob(2000)) );
+  INSERT INTO x1 VALUES( hex(randomblob(2000)) );
+  INSERT INTO x1 VALUES( hex(randomblob(2000)) );
+  INSERT INTO x1 VALUES( hex(randomblob(2000)) );
+  PRAGMA page_count;
+} {18}
+
+do_test 510 {
+  db eval  BEGIN 
+  db2 eval { PRAGMA page_count } {
+    db eval {
+      INSERT INTO sqlite_dbpage values($page_count, NULL);
+    }
+  }
+  db2 eval { SELECT pgno, data FROM sqlite_dbpage } {
+    db eval {
+      INSERT INTO sqlite_dbpage values($pgno, $data);
+    }
+  }
+
+  db eval COMMIT
+} {}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 520 {
+  PRAGMA page_count;
+  SELECT * FROM t1;
+} {2}
+
+db2 close
+
+#-------------------------------------------------------------------------
+reset_db
+forcedelete test.db2
+do_execsql_test 610 {
+  ATTACH 'test.db2' AS aux;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y);
+  INSERT INTO t1 VALUES(1234);
+  CREATE TABLE aux.x1(z);
+}
+
+set pgno [db one {SELECT max(rootpage) FROM sqlite_schema}]
+sqlite3 db2 test.db2
+db2 eval {
+  BEGIN;
+    SELECT * FROM x1;
+}
+
+do_catchsql_test 620 {
+  UPDATE sqlite_dbpage SET data = (
+    SELECT data FROM sqlite_dbpage WHERE pgno=$pgno-1
+  ) WHERE pgno = $pgno;
+} {1 {database is locked}}
+
+db2 eval {
+  COMMIT;
+}
+
+do_catchsql_test 630 {
+  UPDATE sqlite_dbpage SET data = (
+    SELECT data FROM sqlite_dbpage WHERE pgno=$pgno-1
+  ) WHERE pgno = $pgno;
+} {0 {}}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 640 {
+  SELECT * FROM t2;
+} {1234}
+
+db2 close
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 700 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES( hex(randomblob(1000)) );
+  INSERT INTO t1 VALUES( hex(randomblob(1000)) );
+  INSERT INTO t1 VALUES( hex(randomblob(1000)) );
+}
+
+forcedelete test.db2
+sqlite3 db2 test.db2
+db2 eval {
+  CREATE TABLE y1(y);
+  INSERT INTO y1 VALUES( hex(randomblob(1000)) );
+}
+
+set max [db2 one {PRAGMA page_count}]
+
+do_test 710 {
+  execsql {
+    BEGIN;
+  }
+
+  for {set ii 1} {$ii <= $max} {incr ii} {
+    set data [db2 one {SELECT data FROM sqlite_dbpage WHERE pgno=$ii}]
+    execsql {
+      UPDATE sqlite_dbpage SET data=$data WHERE pgno=$ii
+    }
+  }
+
+  execsql {
+      SAVEPOINT abc;
+        INSERT INTO sqlite_dbpage VALUES(2, NULL);
+      ROLLBACK TO abc;
+    COMMIT;
+  }
+} {}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 720 {
+  PRAGMA integrity_check
+} {ok}
+
+
 finish_test
diff --git a/test/dbpagefault.test b/test/dbpagefault.test
index 544d279ce9..e5b246fc94 100644
--- a/test/dbpagefault.test
+++ b/test/dbpagefault.test
@@ -82,7 +82,31 @@ do_catchsql_test 3.2 {
 #  faultsim_test_result {0 {}}
 #}
 
+reset_db
+forcedelete test.db2
+do_execsql_test 4.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES('one');
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES('two');
+  ATTACH 'test.db2' AS aux;
+  CREATE TABLE aux.x1(x);
+}
 
-finish_test
+set pgno [db one {SELECT max(rootpage) FROM sqlite_schema}]
 
+faultsim_save_and_close
+do_faultsim_test 4 -prep {
+  faultsim_restore_and_reopen
+  execsql { ATTACH 'test.db2' AS aux; }
+} -body {
+  execsql { 
+    UPDATE sqlite_dbpage SET data = (
+      SELECT data FROM sqlite_dbpage WHERE pgno=($pgno-1)
+    ) WHERE pgno = $pgno;
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}}
+}
 
+finish_test
diff --git a/test/dbstatus2.test b/test/dbstatus2.test
index 5e9ea888a6..526d8aa17b 100644
--- a/test/dbstatus2.test
+++ b/test/dbstatus2.test
@@ -41,6 +41,10 @@ proc db_spill {db {reset 0}} {
   sqlite3_db_status $db CACHE_SPILL $reset
 }
 
+proc db_temp_spill {db {reset 0}} {
+  sqlite3_db_status $db TEMPBUF_SPILL $reset
+}
+
 do_test 1.1 {
   db close
   sqlite3 db test.db
@@ -111,5 +115,50 @@ do_execsql_test 3.2 {
   UPDATE t1 SET b=randomblob(1000);
 } {delete}
 do_test 3.3 { db_spill db 0 } {0 8 0}
+
+
+if {$::TEMP_STORE<3} {
+  do_execsql_test 4.0 {
+    PRAGMA temp_store = file;
+    PRAGMA cache_size = -1024;
+  } {}
+
+  do_test 4.1 { db_temp_spill db 0 } {0 0 0}
+
+  do_execsql_test 4.2 {
+    CREATE TABLE data(a INTEGER, b BLOB);
+
+    -- Insert 5-6 MB of data.
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<75000 )
+    INSERT INTO data SELECT i, hex(randomblob(50)) FROM s;
+  }
+
+  do_test 4.3 { db_temp_spill db 0 } {0 0 0}
+
+  do_test 4.4 {
+    execsql { SELECT a, b FROM data ORDER BY a }
+    set nTmpSpill [lindex [db_temp_spill db 1] 1]
+    expr {($nTmpSpill>7*1000*1000) && ($nTmpSpill<10*1000*1000)?"ok":$nTmpSpill}
+  } ok
+
+  # The previous test case reset the status value.
+  do_test 4.5 { db_temp_spill db 0 } {0 0 0}
+
+  do_test 4.6 {
+    execsql { CREATE INDEX i1 ON data(a) }
+    set nTmpSpill [lindex [db_temp_spill db 1] 1]
+    expr {($nTmpSpill>384*1000) && ($nTmpSpill<768*1000)?"ok":$nTmpSpill}
+  } ok
+
+  # The previous test case reset the status value.
+  do_test 4.7 { db_temp_spill db 0 } {0 0 0}
+
+  # Same query as in (4.4). Now does not require temp space.
+  do_test 4.8 {
+    execsql { SELECT a, b FROM data ORDER BY a }
+    db_temp_spill db 0
+  } {0 0 0}
+}
+
  
 finish_test
diff --git a/test/decimal.test b/test/decimal.test
index a2e57997dd..07510faa1e 100644
--- a/test/decimal.test
+++ b/test/decimal.test
@@ -22,32 +22,44 @@ if {[catch {load_static_extension db decimal} error]} {
 do_execsql_test 1000 {
   SELECT decimal(1);
 } {1}
+do_execsql_test 1001 {
+  SELECT decimal('+0'), decimal('-0');
+} {0 0}
 do_execsql_test 1010 {
-  SELECT decimal(1.0);
+  SELECT decimal('1.0');
 } {1.0}
 do_execsql_test 1020 {
-  SELECT decimal(0001.0);
+  SELECT decimal('0001.0');
 } {1.0}
 do_execsql_test 1030 {
-  SELECT decimal(+0001.0);
+  SELECT decimal('+0001.0');
 } {1.0}
 do_execsql_test 1040 {
-  SELECT decimal(-0001.0);
+  SELECT decimal('-0001.0');
 } {-1.0}
+do_execsql_test 1041 {
+  SELECT decimal('-0000.0');
+} {0.0}
+do_execsql_test 1042 {
+  SELECT decimal(0.0)==decimal(-0.0);
+} {1}
 do_execsql_test 1050 {
-  SELECT decimal(1.0e72);
+  SELECT decimal('1.0e72');
 } {1000000000000000000000000000000000000000000000000000000000000000000000000}
 #   123456789 123456789 123456789 123456789 123456789 123456789 123456789 123
 do_execsql_test 1060 {
-  SELECT decimal(1.0e-72);
+  SELECT decimal('1.0e-72');
 } {0.0000000000000000000000000000000000000000000000000000000000000000000000010}
 #    123456789 123456789 123456789 123456789 123456789 123456789 123456789 123
 do_execsql_test 1070 {
-  SELECT decimal(-123e-4);
+  SELECT decimal('-123e-4');
 } {-0.0123}
 do_execsql_test 1080 {
-  SELECT decimal(+123e+4);
-} {1230000.0}
+  SELECT decimal('+123e+4');
+} {1230000}
+do_execsql_test 1081 {
+  SELECT decimal_exp('+123e+4');
+} {+1.23e+06}
 
 
 do_execsql_test 2000 {
@@ -70,6 +82,16 @@ do_execsql_test 2000 {
    WHERE a.seq<b.seq
      AND decimal_cmp(a.val,b.val)>=0;
 } {}
+do_execsql_test 2001 {
+  WITH vx(a,b) AS (VALUES
+    ('-0','+0'),
+    ('-000.000','0'),
+    ('1.2','1.2000')
+  )
+  SELECT *, '|' FROM vx 
+   WHERE decimal_cmp(a,b)!=0
+      OR decimal_cmp(b,a)!=0;
+} {}
 do_execsql_test 2010 {
   SELECT *, '|'
     FROM t1 AS a, t1 AS b
@@ -161,6 +183,10 @@ do_execsql_test 6010 {
 SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v)
   FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n);
 } {0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004940656458412465441765687928682213723650598026143247644255856825006755072702087518652998363616359923797965646954457177309266567103559397963987747960107818781263007131903114045278458171678489821036887186360569987307230500063874091535649843873124733972731696151400317153853980741262385655911710266585566867681870395603106249319452715914924553293054565444011274801297099995419319894090804165633245247571478690147267801593552386115501348035264934720193790268107107491703332226844753335720832431936092382893458368060106011506169809753078342277318329247904982524730776375927247874656084778203734469699533647017972677717585125660551199131504891101451037862738167250955837389733598993664809941164205702637090279242767544565229087538682506419718265533447265625}
+do_execsql_test 6011 {
+  WITH c(n) AS (SELECT ieee754_from_blob(x'0000000000000001'))
+SELECT decimal(c.n) FROM c;
+} {0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004940656458412465441765687928682213723650598026143247644255856825006755072702087518652998363616359923797965646954457177309266567103559397963987747960107818781263007131903114045278458171678489821036887186360569987307230500063874091535649843873124733972731696151400317153853980741262385655911710266585566867681870395603106249319452715914924553293054565444011274801297099995419319894090804165633245247571478690147267801593552386115501348035264934720193790268107107491703332226844753335720832431936092382893458368060106011506169809753078342277318329247904982524730776375927247874656084778203734469699533647017972677717585125660551199131504891101451037862738167250955837389733598993664809941164205702637090279242767544565229087538682506419718265533447265625}
 do_execsql_test 6020 {
   WITH c(n) AS (SELECT ieee754_from_blob(x'7fefffffffffffff'))
 SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v)
diff --git a/test/default.test b/test/default.test
index 06a180c1de..192b3d2ff3 100644
--- a/test/default.test
+++ b/test/default.test
@@ -135,6 +135,10 @@ reset_db
 do_catchsql_test default-5.1 {
   CREATE TABLE t1 (a,b DEFAULT(random() NOTNULL IN (RAISE(IGNORE),2,3)));
   INSERT INTO t1(a) VALUES(1);
-} {1 {RAISE() may only be used within a trigger-program}}
+} {1 {default value of column [b] is not constant}}
+do_catchsql_test default-5.2 {
+  CREATE TABLE Table0 (Col0 DEFAULT (RAISE(IGNORE) )  ) ; 
+  INSERT INTO Table0 DEFAULT VALUES ;
+} {1 {default value of column [Col0] is not constant}}
 
 finish_test
diff --git a/test/delete4.test b/test/delete4.test
index 8d6a1b8c7c..34151446b1 100644
--- a/test/delete4.test
+++ b/test/delete4.test
@@ -170,7 +170,7 @@ do_execsql_test 5.0 {
 } {}
 
 # 2016-05-02
-# Ticket https://www.sqlite.org/src/tktview/dc6ebeda93960877
+# Ticket https://sqlite.org/src/tktview/dc6ebeda93960877
 # A subquery in the WHERE clause of a one-pass DELETE can cause an
 # incorrect answer.
 #
diff --git a/test/distinct2.test b/test/distinct2.test
index 958d9634fd..980b0b1e3a 100644
--- a/test/distinct2.test
+++ b/test/distinct2.test
@@ -313,4 +313,71 @@ do_execsql_test 4020 {
   SELECT b FROM t1 UNION SELECT 1;
 } {1 {  }}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5010 {
+  CREATE TABLE cnt(a);
+  WITH RECURSIVE cnt2(x) AS (
+    VALUES(1) UNION ALL SELECT x+1 FROM cnt2 WHERE x<50
+  )
+  INSERT INTO cnt SELECT x FROM cnt2;   
+}
+
+do_execsql_test 5020 {
+  SELECT DISTINCT abs(random())%5 AS r FROM cnt ORDER BY r; 
+} {0 1 2 3 4}
+
+do_execsql_test 5030 {
+  SELECT abs(random())%5 AS r FROM cnt GROUP BY 1 ORDER BY 1; 
+} {0 1 2 3 4}
+
+do_execsql_test 5040 {
+  SELECT a FROM cnt WHERE a>45 GROUP BY 1;
+} {46 47 48 49 50}
+
+
+# 2024-06-03 dbsqlfuzz 8a44f675401a8b1f68a43bf813c4f4f72ad8f0ea
+# Use of uninitialized bytecode register due to the call-function-once
+# optimization at check-in 663f5dd32d9db832
+#
+db null NULL
+do_execsql_test 5050 {
+  CREATE TABLE t0(a TEXT); INSERT INTO t0 VALUES('abcd');
+  CREATE TABLE t1(b TEXT);
+  CREATE TABLE t2(c TEXT);
+  CREATE TABLE t3(d TEXT); INSERT INTO t3 VALUES('wxyz');
+  CREATE VIEW  v4(e) AS SELECT (SELECT t2.c FROM t0, t1 GROUP BY 1) FROM t2;
+  SELECT v4.e FROM t3 LEFT JOIN v4 ON true GROUP BY 1;
+} NULL
+do_execsql_test 5060 {
+  DROP VIEW v4;
+  CREATE VIEW  v4(e) AS SELECT (SELECT t2.c COLLATE nocase FROM t0, t1 GROUP BY 1) FROM t2;
+  SELECT v4.e FROM t3 LEFT JOIN v4 ON true GROUP BY 1;
+} NULL
+
+do_execsql_test 5070 {
+  DROP VIEW v4;
+  CREATE VIEW  v4(e) AS SELECT (SELECT unlikely(t2.c COLLATE nocase) FROM t0, t1 GROUP BY 1) FROM t2;
+  SELECT v4.e FROM t3 LEFT JOIN v4 ON true GROUP BY 1;
+} NULL
+
+# 2024-06-28 dbsqlfuzz 46343912848a603e32c6072cae792eb056bac897
+# Do not call sqlite3ExprToRegister() on an expression that is already
+# a register.
+#
+do_execsql_test 5080 {
+  CREATE TABLE dual(dummy TEXT);
+  INSERT INTO dual VALUES('X');
+  SELECT 11 = (
+                SELECT b
+                  FROM (
+                         SELECT a AS b
+                           FROM dual
+                           LEFT JOIN (SELECT 22 AS a FROM dual)
+                       )
+                 GROUP BY b, b
+              );
+} 0
+
 finish_test
diff --git a/test/distinctagg.test b/test/distinctagg.test
index 6e46c88616..9eedd35bd2 100644
--- a/test/distinctagg.test
+++ b/test/distinctagg.test
@@ -56,7 +56,7 @@ do_test distinctagg-2.1 {
 } {1 {DISTINCT aggregates must have exactly one argument}}
 do_test distinctagg-2.2 {
   catchsql {
-    SELECT group_concat(distinct a,b) FROM t1;
+    SELECT string_agg(distinct a,b) FROM t1;
   }
 } {1 {DISTINCT aggregates must have exactly one argument}}
 
@@ -95,7 +95,7 @@ foreach {tn use_eph sql res} {
   7  0  "SELECT count(DISTINCT a) FROM t2, t1"            5
   8  1  "SELECT count(DISTINCT a+b) FROM t1, t2, t2, t2"  6
   9  0  "SELECT count(DISTINCT c) FROM t1 WHERE c=2"      1
- 10  1  "SELECT count(DISTINCT t1.rowid) FROM t1, t2"    10
+ 10  0  "SELECT count(DISTINCT t1.rowid) FROM t1, t2"    10
 } {
   do_test 3.$tn.1 {
     set prg [db eval "EXPLAIN $sql"]
@@ -148,6 +148,10 @@ do_execsql_test 3.0 {
   CREATE TABLE t3(x, y, z);
   INSERT INTO t3 VALUES(1,1,1);
   INSERT INTO t3 VALUES(2,2,2);
+
+  CREATE TABLE t4(a);
+  CREATE INDEX t4a ON t4(a);
+  INSERT INTO t4 VALUES(1), (2), (2), (3), (1);
 }
 
 foreach {tn use_eph sql res} {
@@ -158,6 +162,9 @@ foreach {tn use_eph sql res} {
   4 0  "SELECT count(DISTINCT f) FROM t2 GROUP BY d, e" {1 2 2 3}
   5 1  "SELECT count(DISTINCT f) FROM t2 GROUP BY d" {2 3}
   6 0  "SELECT count(DISTINCT f) FROM t2 WHERE d IS 1 GROUP BY e" {1 2 2}
+
+  7 0  "SELECT count(DISTINCT a) FROM t1" {4}
+  8 0  "SELECT count(DISTINCT a) FROM t4" {3}
 } {
   do_test 4.$tn.1 {
     set prg [db eval "EXPLAIN $sql"]
@@ -215,4 +222,3 @@ do_execsql_test 7.0 {
 
 
 finish_test
-
diff --git a/test/e_expr.test b/test/e_expr.test
index 5ad5993bb4..81d2fd172c 100644
--- a/test/e_expr.test
+++ b/test/e_expr.test
@@ -278,6 +278,10 @@ set literals {
  13     NULL
 }
 foreach op $oplist {
+  if {$op eq "AND" || $op eq "OR"} {
+    # These tests do not work with AND and OR due to short-circuit evaluation
+    continue
+  }
   foreach {n1 rhs} $literals { 
   foreach {n2 lhs} $literals {
 
@@ -1232,12 +1236,18 @@ db nullvalue {}
 # EVIDENCE-OF: R-13943-13592 A NULL result is considered untrue when
 # evaluating WHEN terms.
 #
-do_execsql_test e_expr-21.4.1 {
+do_execsql_test e_expr-21.4.1a {
   SELECT CASE WHEN NULL THEN 'A' WHEN 1 THEN 'B' END, iif(NULL,8,99);
 } {B 99}
-do_execsql_test e_expr-21.4.2 {
+do_execsql_test e_expr-21.4.1b {
+  SELECT CASE WHEN NULL THEN 'A' WHEN 1 THEN 'B' END, if(NULL,8,99);
+} {B 99}
+do_execsql_test e_expr-21.4.2a {
   SELECT CASE WHEN 0 THEN 'A' WHEN NULL THEN 'B' ELSE 'C' END, iif(0,8,99);
 } {C 99}
+do_execsql_test e_expr-21.4.2b {
+  SELECT CASE WHEN 0 THEN 'A' WHEN NULL THEN 'B' ELSE 'C' END, if(0,8,99);
+} {C 99}
 
 # EVIDENCE-OF: R-38620-19499 In a CASE with a base expression, the base
 # expression is evaluated just once and the result is compared against
@@ -1930,7 +1940,7 @@ foreach {tn expr restype resval} {
     6  { ( SELECT y FROM t4 ORDER BY y DESC ) }        text    two
 
     7  { ( SELECT sum(x) FROM t4 )           }         integer 6
-    8  { ( SELECT group_concat(y,'') FROM t4 ) }       text    onetwothree
+    8  { ( SELECT string_agg(y,'') FROM t4 ) }       text    onetwothree
     9  { ( SELECT max(x) FROM t4 WHERE y LIKE '___') } integer 2 
 
 } {
@@ -1969,9 +1979,12 @@ do_execsql_test e_expr-37.5 {
 # EVIDENCE-OF: R-55532-10108 Values 1, 1.0, 0.1, -0.1 and '1english' are
 # considered to be true.
 #
-do_execsql_test e_expr-37.6 {
+do_execsql_test e_expr-37.6a {
    SELECT CASE WHEN 1 THEN 'true' ELSE 'false' END, iif(1,'true','false');
 } {true true}
+do_execsql_test e_expr-37.6b {
+   SELECT CASE WHEN 1 THEN 'true' ELSE 'false' END, if(1,'true');
+} {true true}
 do_execsql_test e_expr-37.7 {
    SELECT CASE WHEN 1.0 THEN 'true' ELSE 'false' END, iif(1.0,'true','false');
 } {true true}
diff --git a/test/e_reindex.test b/test/e_reindex.test
index 00291b76a6..50d2e7d516 100644
--- a/test/e_reindex.test
+++ b/test/e_reindex.test
@@ -73,12 +73,12 @@ sqlite3 db test.db
 do_execsql_test e_reindex-1.3 {
   PRAGMA integrity_check;
 } [list \
+  {wrong # of entries in index i2} \
+  {wrong # of entries in index i1} \
   {row 3 missing from index i2} \
   {row 3 missing from index i1} \
   {row 4 missing from index i2} \
-  {row 4 missing from index i1} \
-  {wrong # of entries in index i2} \
-  {wrong # of entries in index i1}
+  {row 4 missing from index i1} 
 ]
 
 do_execsql_test e_reindex-1.4 {
diff --git a/test/e_select.test b/test/e_select.test
index 5a3f0d30dc..e2e969dcf9 100644
--- a/test/e_select.test
+++ b/test/e_select.test
@@ -943,7 +943,7 @@ do_select_tests e_select-4.6 {
   4 "SELECT *, count(*) FROM a1 JOIN a2"                       {1 1 1 1 16}
   5 "SELECT *, sum(three) FROM a1 NATURAL JOIN a2"             {1 1 1 3}
   6 "SELECT *, sum(three) FROM a1 NATURAL JOIN a2"             {1 1 1 3}
-  7 "SELECT group_concat(three, ''), a1.* FROM a1 NATURAL JOIN a2" {12 1 1}
+  7 "SELECT string_agg(three, ''), a1.* FROM a1 NATURAL JOIN a2" {12 1 1}
 }
 
 # EVIDENCE-OF: R-04486-07266 Or, if the dataset contains zero rows, then
diff --git a/test/enc.test b/test/enc.test
index ffe24164bd..0aec224e2d 100644
--- a/test/enc.test
+++ b/test/enc.test
@@ -249,4 +249,33 @@ do_execsql_test enc-12.10 {
   SELECT * FROM t1;
 } {d e f xxx yyy zzz}
 
+
+# 2024-11-04
+# https://sqlite.org/forum/forumpost/bc75a4d20b756044
+#
+# Ensure the database encoding is detected in a timely manner,
+# and before we get too far along in generating and running
+# bytecode.
+#
+# See also check-in https://sqlite.org/src/info/a02da71f3a.
+#
+db close
+forcedelete utf16.db
+sqlite3 db utf16.db
+db eval {PRAGMA encoding=UTF16; CREATE TABLE t2(y); INSERT INTO t2 VALUES('utf16');}
+db close
+sqlite3 db utf16.db
+do_test enc-13.1 {
+  db eval {PRAGMA function_list} {db eval {SELECT * FROM sqlite_schema}}
+} {}
+ifcapable rtree {
+  db eval {CREATE VIRTUAL TABLE t3 USING rtree(id,x1,x2)}
+  db close
+  sqlite3 db utf16.db
+  do_execsql_test enc-13.2 {
+    WITH t1(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM t1 WHERE x<3)
+    SELECT rtreecheck('t3') FROM t1;
+  } {ok ok ok}
+}
+
 finish_test
diff --git a/test/enc2.test b/test/enc2.test
index 81e7bfd68c..f7446a40ef 100644
--- a/test/enc2.test
+++ b/test/enc2.test
@@ -63,7 +63,7 @@ set dbcontents {
   INSERT INTO t1 VALUES('one', 'I', 1);
 }
 # This proc tests that we can open and manipulate the test.db 
-# database, and that it is possible to retreive values in
+# database, and that it is possible to retrieve values in
 # various text encodings.
 #
 proc run_test_script {t enc} {
diff --git a/test/eqp.test b/test/eqp.test
index 61fd617d89..147b5ceafe 100644
--- a/test/eqp.test
+++ b/test/eqp.test
@@ -288,6 +288,7 @@ det 3.2.1 {
   |--SCAN (subquery-xxxxxx)
   `--USE TEMP B-TREE FOR ORDER BY
 }
+
 det 3.2.2 {
   SELECT * FROM 
     (SELECT * FROM t1 ORDER BY x LIMIT 10) AS x1,
@@ -305,13 +306,24 @@ det 3.2.2 {
   `--USE TEMP B-TREE FOR ORDER BY
 }
 
+det 3.2.3 {
+  SELECT * FROM (SELECT * FROM t1 ORDER BY x LIMIT 10) ORDER BY x LIMIT 5
+} {
+  QUERY PLAN
+  |--CO-ROUTINE (subquery-xxxxxx)
+  |  |--SCAN t1
+  |  `--USE TEMP B-TREE FOR ORDER BY
+  `--SCAN (subquery-xxxxxx)
+}
+
 det 3.3.1 {
   SELECT * FROM t1 WHERE y IN (SELECT y FROM t2)
 } {
   QUERY PLAN
   |--SCAN t1
   `--LIST SUBQUERY xxxxxx
-     `--SCAN t2
+     |--SCAN t2
+     `--CREATE BLOOM FILTER
 }
 det 3.3.2 {
   SELECT * FROM t1 WHERE y IN (SELECT y FROM t2 WHERE t1.x!=t2.x)
@@ -326,8 +338,7 @@ det 3.3.3 {
 } {
   QUERY PLAN
   |--SCAN t1
-  `--CORRELATED SCALAR SUBQUERY xxxxxx
-     `--SCAN t2
+  `--SCAN t2 EXISTS
 }
 
 #-------------------------------------------------------------------------
@@ -413,7 +424,7 @@ do_eqp_test 4.2.3 {
      |  `--USE TEMP B-TREE FOR ORDER BY
      `--RIGHT
         |--SCAN t2 USING INDEX t2i1
-        `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+        `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY
 }
 do_eqp_test 4.2.4 {
   SELECT * FROM t1 INTERSECT SELECT * FROM t2 ORDER BY 1
@@ -425,7 +436,7 @@ do_eqp_test 4.2.4 {
      |  `--USE TEMP B-TREE FOR ORDER BY
      `--RIGHT
         |--SCAN t2 USING INDEX t2i1
-        `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+        `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY
 }
 do_eqp_test 4.2.5 {
   SELECT * FROM t1 EXCEPT SELECT * FROM t2 ORDER BY 1
@@ -437,7 +448,7 @@ do_eqp_test 4.2.5 {
      |  `--USE TEMP B-TREE FOR ORDER BY
      `--RIGHT
         |--SCAN t2 USING INDEX t2i1
-        `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+        `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY
 }
 
 do_eqp_test 4.3.1 {
@@ -806,6 +817,7 @@ do_execsql_test 9.0 {
   CREATE INDEX event_i1 ON event(mtime);
   CREATE TABLE private(rid INTEGER PRIMARY KEY);
 }
+optimization_control db order-by-subquery off
 do_eqp_test 9.1 {
   WITH thread(age,duration,cnt,root,last) AS (
     SELECT
@@ -846,5 +858,46 @@ do_eqp_test 9.1 {
   |--SEARCH event USING INTEGER PRIMARY KEY (rowid=?)
   `--USE TEMP B-TREE FOR ORDER BY
 }
+optimization_control db all on
+db cache flush
+do_eqp_test 9.2 {
+  WITH thread(age,duration,cnt,root,last) AS (
+    SELECT
+      julianday('now') - max(fmtime) AS age,
+      max(fmtime) - min(fmtime) AS duration,
+      sum(fprev IS NULL) AS msg_count,
+      froot,
+      (SELECT fpid FROM forumpost
+        WHERE froot=x.froot
+          AND fpid NOT IN private
+        ORDER BY fmtime DESC LIMIT 1)
+    FROM forumpost AS x
+    WHERE fpid NOT IN private  --- Ensure this table mentioned in EQP output!
+    GROUP BY froot
+    ORDER BY 1 LIMIT 26 OFFSET 5
+  )
+  SELECT
+    thread.age,
+    thread.duration,
+    thread.cnt,
+    blob.uuid,
+    substr(event.comment,instr(event.comment,':')+1)
+  FROM thread, blob, event
+  WHERE blob.rid=thread.last
+    AND event.objid=thread.last
+  ORDER BY 1;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE thread
+  |  |--SCAN x USING INDEX forumthread
+  |  |--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR
+  |  |--CORRELATED SCALAR SUBQUERY xxxxxx
+  |  |  |--SEARCH forumpost USING COVERING INDEX forumthread (froot=?)
+  |  |  `--USING ROWID SEARCH ON TABLE private FOR IN-OPERATOR
+  |  `--USE TEMP B-TREE FOR ORDER BY
+  |--SCAN thread
+  |--SEARCH blob USING INTEGER PRIMARY KEY (rowid=?)
+  `--SEARCH event USING INTEGER PRIMARY KEY (rowid=?)
+}
 
 finish_test
diff --git a/test/eqp2.test b/test/eqp2.test
new file mode 100644
index 0000000000..3c634fc28a
--- /dev/null
+++ b/test/eqp2.test
@@ -0,0 +1,49 @@
+# 2024 March 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix eqp2
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c, d);
+  CREATE INDEX i1 ON t1(a, b, c);
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 ORDER BY a, b, c
+} {
+  QUERY PLAN
+  `--SCAN t1 USING INDEX i1
+}
+
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 ORDER BY a, b, +c
+} {
+  QUERY PLAN
+  |--SCAN t1 USING INDEX i1
+  `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t1 ORDER BY a, +b, +c
+} {
+  QUERY PLAN
+  |--SCAN t1 USING INDEX i1
+  `--USE TEMP B-TREE FOR LAST 2 TERMS OF ORDER BY
+}
+
+finish_test
+
+
diff --git a/test/errofst1.test b/test/errofst1.test
new file mode 100644
index 0000000000..f8876316e5
--- /dev/null
+++ b/test/errofst1.test
@@ -0,0 +1,31 @@
+# 2024-11-20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for sqlite3_error_offset()
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test errofst1-1.1 {
+  CREATE TABLE t1 as select 1 as aa;
+  CREATE VIEW t2 AS
+     WITH t3 AS (SELECT 1 FROM t1 AS bb, t1 AS cc WHERE cc.aa <= sts.aa)
+     SELECT 1 FROM t3 AS dd;
+}
+do_catchsql_test errofst1-1.2 {
+  SELECT * FROM t2;
+} {1 {no such column: sts.aa}}
+do_test errofst1-1.3 {
+  sqlite3_error_offset db
+} {-1}
+
+finish_test
diff --git a/test/exclusive2.test b/test/exclusive2.test
index 92fcd76ab3..0a5d2b6eb8 100644
--- a/test/exclusive2.test
+++ b/test/exclusive2.test
@@ -41,7 +41,7 @@ sqlite3_soft_heap_limit 0
 proc pagerChangeCounter {filename new {fd ""}} {
   if {$fd==""} {
     set fd [open $filename RDWR]
-    fconfigure $fd -translation binary -encoding binary
+    fconfigure $fd -translation binary
     set needClose 1
   } else {
     set needClose 0
@@ -70,7 +70,7 @@ proc pagerChangeCounter {filename new {fd ""}} {
 
 proc readPagerChangeCounter {filename} {
   set fd [open $filename RDONLY]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
 
   seek $fd 24
   foreach {a b c d} [list 0 0 0 0] {}
diff --git a/test/existsexpr.test b/test/existsexpr.test
new file mode 100644
index 0000000000..28029359b8
--- /dev/null
+++ b/test/existsexpr.test
@@ -0,0 +1,517 @@
+# 2024 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix existsexpr
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID;
+  INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6);
+  CREATE INDEX x1b ON x1(b);
+
+  CREATE TABLE x2(x, y);
+  INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+do_execsql_test 1.1 {
+  SELECT 1 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=5)
+} {1}
+
+do_execsql_test 1.2 {
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {1 2 3 4 5 6}
+
+# With "a=x", the UNIQUE index means the EXIST can be transformed to a join.
+# So no "SUBQUERY". With "b=x", the index is not UNIQUE and so there is a
+# "SUBQUERY".
+do_execsql_test 1.3.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {~/SUBQUERY/}
+do_execsql_test 1.3.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE b=x)
+} {~/SUBQUERY/}
+
+do_execsql_test 1.4.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE x=1 AND EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {~/SUBQUERY/}
+do_execsql_test 1.4.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y=2
+} {~/SUBQUERY/}
+
+do_execsql_test 1.5 {
+  SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {3}
+
+#-------------------------------------------------------------------------
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  ) INSERT INTO t1 SELECT i, i FROM s;
+
+  CREATE TABLE t2(c, d);
+  WITH s(i) AS (
+    SELECT 10 UNION ALL SELECT i+10 FROM s WHERE i<1000
+  ) INSERT INTO t2 SELECT i, i FROM s;
+}
+
+do_execsql_test 2.1 {
+  SELECT count(*) FROM t1;
+  SELECT count(*) FROM t2;
+} {1000 100}
+
+do_execsql_test 2.2 {
+  SELECT count(*) FROM t1, t2 WHERE a=c;
+} {100}
+
+do_execsql_test 2.3 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a)
+} {100}
+do_eqp_test 2.4 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a)
+} {SCAN t1}
+
+do_execsql_test 2.4.0 {
+  CREATE UNIQUE INDEX t2c ON t2(c);
+  CREATE UNIQUE INDEX t1a ON t1(a);
+}
+
+do_eqp_test 2.4.1 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a);
+} {SCAN t1*t2 EXISTS}
+do_execsql_test 2.4.2 {
+  ANALYZE;
+}
+do_eqp_test 2.4.3 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a);
+} {SCAN t1*t2 EXISTS}
+do_execsql_test 2.4.4 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a);
+} {100}
+
+do_execsql_test 2.5.1 {
+  EXPLAIN QUERY PLAN
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.rowid=a);
+} {~/SUBQUERY/}
+
+#-------------------------------------------------------------------------
+proc do_subquery_test {tn bSub sql res} {
+  set r1(0) ~/SUBQUERY/
+  set r1(1) /SUBQUERY/
+  do_execsql_test $tn.1 "explain query plan $sql" $r1($bSub)
+  do_execsql_test $tn.2 $sql $res
+}
+
+do_execsql_test 3.0 {
+  CREATE TABLE y1(a, b, c);
+  CREATE TABLE y2(x, y, z);
+  CREATE UNIQUE INDEX y2zy ON y2(z, y);
+
+  INSERT INTO y1 VALUES(1, 1, 1);
+  INSERT INTO y1 VALUES(2, 2, 2);
+  INSERT INTO y1 VALUES(3, 3, 3);
+  INSERT INTO y1 VALUES(4, 4, 4);
+
+  INSERT INTO y2 VALUES(1, 1, 1);
+  INSERT INTO y2 VALUES(3, 3, 3);
+}
+
+do_subquery_test 3.1 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=a AND y=b AND x=z
+  )
+} {
+  1 1 1          3 3 3
+}    
+
+do_subquery_test 3.2 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND x=z
+  )
+} {
+  1 1 1          3 3 3
+}    
+
+do_subquery_test 3.3 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND c!=3
+  )
+} {
+  1 1 1
+}    
+
+do_subquery_test 3.4 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=max(a,b) AND b=3
+  )
+} {
+  3 3 3
+}    
+
+do_subquery_test 3.5 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=a-1 AND y=a-1
+  )
+} {
+  2 2 2
+  4 4 4
+}    
+
+do_subquery_test 3.6 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=a-1 AND y+1=a
+  )
+} {
+  2 2 2
+  4 4 4
+}    
+
+do_subquery_test 3.7 1 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT count(*) FROM y2 WHERE z=a-1 AND y=a-1
+  )
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+}    
+
+do_subquery_test 3.8 0 {
+  SELECT * FROM y1 WHERE EXISTS ( SELECT a+1 FROM y2 )
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+}    
+
+do_subquery_test 3.9 1 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 one, y2 two WHERE one.z=a-1 AND one.y=a-1
+  )
+} {
+  2 2 2
+  4 4 4
+}    
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE tx1(a TEXT COLLATE nocase, b TEXT);
+  CREATE UNIQUE INDEX tx1ab ON tx1(a, b);
+
+  INSERT INTO tx1 VALUES('a', 'a');
+  INSERT INTO tx1 VALUES('B', 'b');
+  INSERT INTO tx1 VALUES('c', 'c');
+  INSERT INTO tx1 VALUES('D', 'd');
+  INSERT INTO tx1 VALUES('e', 'e');
+
+  CREATE TABLE tx2(x, y);
+  INSERT INTO tx2 VALUES('A', 'a');
+  INSERT INTO tx2 VALUES('b', 'b');
+  INSERT INTO tx2 VALUES('C', 'c');
+  INSERT INTO tx2 VALUES('D', 'd');
+}
+
+do_subquery_test 4.1 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND b=y
+  )
+} {
+  A a
+  b b
+  C c
+  D d
+}
+
+do_subquery_test 4.1.1 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE (a COLLATE nocase)=x AND b=y
+  )
+} {
+  A a   b b   C c   D d
+}
+do_subquery_test 4.1.2 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND (b COLLATE binary)=y
+  )
+} {
+  A a   b b   C c   D d
+}
+do_subquery_test 4.1.1 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE x=(a COLLATE nocase) AND b=y
+  )
+} {
+  A a   b b   C c   D d
+}
+do_subquery_test 4.1.2 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND y=(b COLLATE binary)
+  )
+} {
+  A a   b b   C c   D d
+}
+
+do_subquery_test 4.2 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND b=y COLLATE nocase
+  )
+} {
+  A a
+  b b
+  C c
+  D d
+}
+
+do_execsql_test 4.3 {
+  DROP INDEX tx1ab;
+  CREATE UNIQUE INDEX tx1ab ON tx1(a COLLATE binary, b);
+}
+
+do_subquery_test 4.4 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND b=y
+  )
+} {
+  A a
+  b b
+  C c
+  D d
+}
+
+do_subquery_test 4.4 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x COLLATE binary AND b=y
+  )
+} {
+  D d
+}
+
+do_subquery_test 4.4 1 {
+  SELECT EXISTS ( SELECT x FROM tx1 ) FROM tx2
+} {
+  1 1 1 1
+}
+
+do_subquery_test 4.4 1 {
+  SELECT (SELECT EXISTS ( SELECT x FROM tx1 ) WHERE 1) FROM tx2
+} {
+  1 1 1 1
+}
+
+#-------------------------------------------------------------------------
+proc cols {s f} {
+  set lCols [list]
+  for {set i $s} {$i<=$f} {incr i} {
+    lappend lCols [format "c%02d" $i]
+  }
+  join $lCols ", "
+}
+proc vals {n val} {
+  set lVal [list]
+  for {set i 0} {$i<$n} {incr i} {
+    lappend lVal $val
+  }
+  join $lVal ", "
+}
+proc exprs {s f} {
+  set lExpr [list]
+  for {set i $s} {$i<=$f} {incr i} {
+    lappend lExpr [format "c%02d = o" $i]
+  }
+  join $lExpr " AND "
+}
+
+
+do_execsql_test 5.0 "
+  CREATE TABLE a1( [cols 0 99] );
+"
+do_execsql_test 5.1 "
+  -- 63 column index
+  CREATE UNIQUE INDEX a1idx1 ON a1( [cols 0 62] );
+"
+do_execsql_test 5.2 "
+  -- 64 column index
+  CREATE UNIQUE INDEX a1idx2 ON a1( [cols 10 73] );
+"
+do_execsql_test 5.2 "
+  -- 65 column index
+  CREATE UNIQUE INDEX a1idx3 ON a1( [cols 20 84] );
+"
+
+do_test 5.3 {
+  foreach v {1 2 3 4 5 6} {
+    execsql "INSERT INTO a1 VALUES( [vals 100 $v] )"
+  }
+} {}
+
+do_execsql_test 5.4 {
+  CREATE TABLE a2(o);
+  INSERT INTO a2 VALUES(2), (5); 
+}
+
+do_subquery_test 5.5 0 "
+  SELECT o FROM a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE [exprs 0 62]
+  )
+" {
+  2 5
+}
+
+do_subquery_test 5.6 0 "
+  SELECT o FROM a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE [exprs 10 73]
+  )
+" {
+  2 5
+}
+
+do_subquery_test 5.7 0 "
+  SELECT o FROM a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE [exprs 20 84]
+  )
+" {
+  2 5
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a, b UNIQUE, c UNIQUE);
+  CREATE TABLE t2(a INfEGER PRIMARY KEY, b);
+  CREATE UNIQUE INDEX t2b ON t2(b);
+}
+
+do_catchsql_test 6.1 {
+  SELECT a FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c COLLATE f = a)
+} {1 {no such collation sequence: f}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y UNIQUE);
+
+  INSERT INTO t1 VALUES(1), (2);
+  INSERT INTO t2 VALUES(1), (3);
+
+  SELECT * FROM t1 one LEFT JOIN t1 two ON (one.x=two.x AND EXISTS (
+    SELECT 1 FROM t2 WHERE y=one.x
+  ));
+} {
+  1 1
+  2 {}
+}
+
+# https://sqlite.org/forum/forumpost/2025-07-23T10:59:14z
+reset_db
+do_execsql_test 8.0 {
+  CREATE TABLE t0 (c0 INT);  INSERT INTO t0(c0) VALUES (1);
+  CREATE TABLE t1(c0 INT);   INSERT INTO t1(c0) VALUES (2);
+  SELECT * FROM t1 WHERE EXISTS (SELECT 1 FROM t0 LIMIT 0);
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 9.0 {
+  CREATE TABLE t1(xx);
+  INSERT INTO t1 VALUES('big string value');
+} {}
+
+do_execsql_test 9.1 {
+  PRAGMA automatic_index = off;
+  CREATE TABLE t2(ii);
+  INSERT INTO t2 VALUES(100);
+  INSERT INTO t2 VALUES(200);
+}
+
+do_execsql_test 9.2 {
+  CREATE TABLE t3(yy);
+  INSERT INTO t3 VALUES(200);
+}
+
+do_execsql_test 9.3 {
+  SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii )
+} {1}
+
+do_execsql_test 9.4 {
+  SELECT EXISTS (
+    SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii )
+  )
+} {1}
+
+do_execsql_test 9.5 {
+  SELECT 1234 WHERE EXISTS (
+    SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii )
+  )
+} {1234}
+
+set Q {
+  SELECT * FROM t1 WHERE 
+    EXISTS (
+      SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii ) 
+    )
+}
+
+do_execsql_test 9.5 $Q {{big string value}}
+catch { optimization_control db exists-to-join 0 }
+db cache flush
+do_execsql_test 9.6 $Q {{big string value}}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 10.0 {
+  CREATE TABLE t1(a);
+  CREATE TABLE x1(x);
+}
+
+do_execsql_test 10.1 {
+  SELECT EXISTS( SELECT 1 FROM t1 ) aaa FROM x1 WHERE aaa AND aaa
+}
+
+do_execsql_test 10.2 {
+  SELECT 
+    EXISTS( SELECT 1 FROM t1 ) aaa
+  WHERE (
+    SELECT 1 FROM x1 WHERE aaa AND aaa
+  );
+}
+
+# https://sqlite.org/forum/forumpost/2026-01-03T14:05:48z
+do_execsql_test 11.0 {
+  CREATE TABLE parent (id TEXT PRIMARY KEY);
+  CREATE TABLE child_a (id TEXT);
+  CREATE TABLE child_b (id TEXT);
+  INSERT INTO parent VALUES ('p1');
+  INSERT INTO child_a VALUES ('p1');
+}
+do_execsql_test 11.1 {
+  SELECT count(*), parent.id FROM parent
+   WHERE EXISTS (
+      SELECT 1 FROM child_a WHERE child_a.id = parent.id
+      UNION
+      SELECT 1 FROM child_b WHERE child_b.id = parent.id
+   )
+   GROUP BY id;
+} {1 p1}
+
+finish_test
diff --git a/test/existsexpr2.test b/test/existsexpr2.test
new file mode 100644
index 0000000000..f7644bf802
--- /dev/null
+++ b/test/existsexpr2.test
@@ -0,0 +1,96 @@
+# 2024 June 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix existsexpr2
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID;
+  INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6);
+  CREATE INDEX x1b ON x1(b);
+
+  CREATE TABLE x2(x, y);
+  INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM x1 WHERE EXISTS (SELECT 1 FROM x2 WHERE a!=123)
+} {1 2   3 4   5 6}
+
+do_execsql_test 1.2 {
+  CREATE TABLE x3(u, v);
+  CREATE INDEX x3u ON x3(u);
+  INSERT INTO x3 VALUES
+    (1, 1), (1, 2), (1, 3),
+    (2, 1), (2, 2), (2, 3);
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM x1 WHERE EXISTS (
+    SELECT 1 FROM x3 WHERE u IN (1, 2, 3, 4) AND v=b
+  );
+} {
+  1 2
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1ab ON t1(a,b);
+
+  INSERT INTO t1 VALUES
+      ('abc', 1, 1),
+      ('abc', 2, 2),
+      ('abc', 2, 3),
+
+      ('def', 1, 1),
+      ('def', 2, 2),
+      ('def', 2, 3);
+
+  CREATE TABLE t2(x, y);
+  INSERT INTO t2 VALUES(1, 1), (2, 2), (3, 3);
+
+  ANALYZE;
+  DELETE FROM sqlite_stat1;
+  INSERT INTO sqlite_stat1 VALUES('t1','t1ab','10000 5000 2');
+  ANALYZE sqlite_master;
+}
+
+
+do_execsql_test 2.1 {
+  SELECT a,b,c FROM t1 WHERE b=2 ORDER BY a
+} {
+  abc 2 2
+  abc 2 3
+  def 2 2
+  def 2 3
+}
+
+do_execsql_test 2.2 {
+  SELECT x, y FROM t2 WHERE EXISTS (
+    SELECT 1 FROM t1 WHERE b=x
+  )
+} {
+  1 1
+  2 2
+}
+
+
+
+finish_test
+
+
diff --git a/test/existsfault.test b/test/existsfault.test
new file mode 100644
index 0000000000..4b335d84cd
--- /dev/null
+++ b/test/existsfault.test
@@ -0,0 +1,49 @@
+# 2024 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+set testprefix existsfault
+
+db close
+sqlite3_shutdown
+sqlite3_config_lookaside 0 0
+sqlite3_initialize
+autoinstall_test_functions
+sqlite3 db test.db
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b);
+  INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6);
+  CREATE UNIQUE INDEX x1a ON x1(a);
+  CREATE INDEX x1b ON x1(b);
+
+  CREATE TABLE x2(x, y);
+  INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+do_faultsim_test 1 -faults oom* -prep {
+  sqlite3 db test.db
+  execsql { SELECT * FROM sqlite_schema }
+} -body {
+  execsql {
+    SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y!=11
+  }
+} -test {
+  faultsim_test_result {0 3}
+}
+
+finish_test
+
+
diff --git a/test/expr.test b/test/expr.test
index 55b1dc9502..4f739e2e7d 100644
--- a/test/expr.test
+++ b/test/expr.test
@@ -71,6 +71,9 @@ test_expr expr-1.28 {i1=1, i2=2} {i1=2 AND i2=1} {0}
 test_expr expr-1.29 {i1=1, i2=2} {i1=1 AND i2=1} {0}
 test_expr expr-1.30 {i1=1, i2=2} {i1=2 AND i2=2} {0}
 test_expr expr-1.31 {i1=1, i2=2} {i1==1 OR i2=2} {1}
+test_expr expr-1.31b {i1=1} {0 OR 2} {1}
+test_expr expr-1.31c {i1=1} {false OR true} {1}
+test_expr expr-1.31d {i1=1} {99 OR false} {1}
 test_expr expr-1.32 {i1=1, i2=2} {i1=2 OR i2=1} {0}
 test_expr expr-1.33 {i1=1, i2=2} {i1=1 OR i2=1} {1}
 test_expr expr-1.34 {i1=1, i2=2} {i1=2 OR i2=2} {1}
@@ -1002,7 +1005,7 @@ do_execsql_test expr-13.9 {
   SELECT '' <= "";
 } {1}
 
-# 2018-02-26. Ticket https://www.sqlite.org/src/tktview/36fae083b450e3af85
+# 2018-02-26. Ticket https://sqlite.org/src/tktview/36fae083b450e3af85
 # 
 do_execsql_test expr-14.1 {
   DROP TABLE IF EXISTS t1;
diff --git a/test/exprfault2.test b/test/exprfault2.test
new file mode 100644
index 0000000000..acbead59f6
--- /dev/null
+++ b/test/exprfault2.test
@@ -0,0 +1,35 @@
+# 2024-05-11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix exprfault2
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a,b,c,d,f,PRIMARY KEY(b,b));
+  CREATE TABLE t2(x INT PRIMARY KEY, y, z);
+  CREATE TABLE t3(a,b,c,d,e,PRIMARY KEY(a,b))WITHOUT ROWID;
+}
+faultsim_save_and_close
+
+
+do_faultsim_test 1 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+  UPDATE t3 SET (d,d,d,d, a )=(SELECT EXISTS(SELECT 1 NOT IN(SELECT EXISTS(SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT (SELECT max( 1 IN(SELECT x NOT IN(SELECT 1 NOT IN(SELECT EXISTS(SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT (SELECT max( (SELECT x NOT IN(SELECT 1 NOT IN(SELECT EXISTS(SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT (SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT (SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) a)| (SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(ORDER BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) z)|9 AS blob) IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))EXCEPT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) z) ORDER BY 1) IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT (SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 x ORDER BY 1)))EXCEPT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) a)| (SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) z)|9 AS blob) IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))EXCEPT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) z) ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 5 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) ORDERa)| (SELECT 1 IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(ORDER BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) z)| 1 AS blob) IN(SELECT max( 1 IN(SELECT x ORDER BY 1)) OVER(PARTITION BY sum((SELECT DISTINCT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 UNION SELECT d ORDER BY 1) ORDER BY 1) z)| (SELECT 1 IN(SELECT max( 1 IN(SELECT c ORDER BY 1)) OVER(PARTITION BY sum((SELECT y FROM t2 UNION SELECT x ORDER BY 1)))INTERSECT SELECT (SELECT 1 FROM t2 UNION SELECT x ORDER BY 1) ORDER BY 1) e)|9 AS blob) FROM t2 WHERE a<x), e= BY 1) FROM t2 UNION SELECT 1 ORDER BY 1) ORDER BY 1)) a) FILTER (GROUP BY 1 HAVING b<= OVER(ORDER BY (SELECT max(x INPARTITION BY sum((SELECT y FROM t2 UNION SELECT x IN(SELECT 1 ORDER BY 1) ORDER 1)))INTERSECT SELECT EXISTS(SELECT 1 FROM t2 WHERE xBY 1) ORDER BY 1)) FROM77;
+  }
+} -test {
+  faultsim_test_result {1 {near ")": syntax error}}
+}
+
+finish_test
diff --git a/test/extension01.test b/test/extension01.test
index 97b772680e..ba8a44edb5 100644
--- a/test/extension01.test
+++ b/test/extension01.test
@@ -60,7 +60,7 @@ do_test 1.5 {
 } {0}
 
 do_test 1.6 {
-  if {$::tcl_platform(platform)=="unix"} {
+  if {$::tcl_platform(os) ne "Windows NT"} {
     file attributes ./file2.txt -permissions r--r--r--
   } else {
     file attributes ./file2.txt -readonly 1
@@ -70,7 +70,7 @@ do_test 1.6 {
   }
 } {nil}
 do_test 1.7 {
-  if {$::tcl_platform(platform)=="unix"} {
+  if {$::tcl_platform(os) ne "Windows NT"} {
     file attributes ./file2.txt -permissions rw-r--r--
   } else {
     file attributes ./file2.txt -readonly 0
diff --git a/test/external_reader.test b/test/external_reader.test
index 5d293981c5..d56aa4e261 100644
--- a/test/external_reader.test
+++ b/test/external_reader.test
@@ -19,7 +19,7 @@ ifcapable !wal {
   finish_test 
   return 
 }
-if {$::tcl_platform(platform)!="unix"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   finish_test 
   return
 }
diff --git a/test/filectrl.test b/test/filectrl.test
index 460b71e257..9b1a1c7589 100644
--- a/test/filectrl.test
+++ b/test/filectrl.test
@@ -36,16 +36,18 @@ do_test filectrl-1.5 {
   sqlite3 db test_control_lockproxy.db
   file_control_lockproxy_test db [get_pwd]
 } {}
-do_test filectrl-1.6 {
-  sqlite3 db test.db
-  set fn [file_control_tempfilename db]
-  set fn
-} {/etilqs_/}
+ifcapable !winrt {
+  do_test filectrl-1.6 {
+    sqlite3 db test.db
+    set fn [file_control_tempfilename db]
+    set fn
+  } {/etilqs_/}
+}
 db close
 forcedelete .test_control_lockproxy.db-conch test.proxy
 forcedelete test.db test2.db
 
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
   do_test filectrl-2.1 {
     sqlite3 db test2.db
     set size [file size test2.db]
diff --git a/test/filter2.test b/test/filter2.test
index 21ee6659ff..06cfd2a4c3 100644
--- a/test/filter2.test
+++ b/test/filter2.test
@@ -113,7 +113,7 @@ do_execsql_test 1.12 {
 
 do_execsql_test 1.13 {
   SELECT 
-    group_concat(CAST(b AS TEXT), '_') FILTER (WHERE b%2!=0),
+    string_agg(CAST(b AS TEXT), '_') FILTER (WHERE b%2!=0),
     group_concat(CAST(b AS TEXT), '_') FILTER (WHERE b%2!=1),
     count(*) FILTER (WHERE b%2!=0),
     count(*) FILTER (WHERE b%2!=1)
diff --git a/test/fkey6.test b/test/fkey6.test
index b658f20fea..415daeaf3e 100644
--- a/test/fkey6.test
+++ b/test/fkey6.test
@@ -225,5 +225,82 @@ do_execsql_test 3.3.4 {
   SELECT * FROM p2;
 } {0 one 1 deleted!}
 
+#-------------------------------------------------------------------------
+# Verify that, even with "PRAGMA defer_foreign_keys", a transaction cannot
+# be committed if there are outstanding foreign key violations.
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE p1(a INTEGER PRIMARY KEY, b UNIQUE);
+  CREATE TABLE c1(x REFERENCES p1(b));
+
+  INSERT INTO p1 VALUES(1, 'one'), (2, 'two'), (3, 'three');
+  INSERT INTO c1 VALUES('two');
+
+  PRAGMA foreign_keys = 1;
+  PRAGMA defer_foreign_keys = 1;
+}
+
+do_execsql_test 4.1 {
+  BEGIN;
+    DELETE FROM p1 WHERE a=2;
+}
+
+do_catchsql_test 4.2 {
+  COMMIT;
+} {1 {FOREIGN KEY constraint failed}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.0 {
+  PRAGMA foreign_keys = 1;
+  CREATE TABLE p1(a INTEGER PRIMARY KEY, b);
+  CREATE TABLE c1(x REFERENCES p1 DEFERRABLE INITIALLY DEFERRED);
+}
+
+do_execsql_test 5.1 {
+  BEGIN;
+    INSERT INTO c1 VALUES(123);
+    PRAGMA defer_foreign_keys = 1;
+    INSERT INTO p1 VALUES(123, 'one two three');
+  COMMIT;
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+
+ifcapable fts5 {
+if {[permutation]!="inmemory_journal"} {
+  do_execsql_test 6.1 {
+    PRAGMA auto_vacuum = 0;
+    PRAGMA writable_schema = 1;
+    INSERT INTO sqlite_schema 
+      VALUES('table', 't1', 't1', 2, 'CREATE TABLE t1(x INTEGER PRIMARY KEY)');
+  }
+  db close
+  sqlite3 db test.db
+  do_execsql_test 6.1 {
+    PRAGMA foreign_keys = 1;
+    PRAGMA writable_schema = 1;
+  }
+  do_execsql_test 6.2 {
+    CREATE TABLE t2(
+        y INTEGER PRIMARY KEY,
+        z INTEGER REFERENCES t1(x) DEFERRABLE INITIALLY DEFERRED
+    );
+  }
+  do_execsql_test 6.3 {
+    BEGIN;
+      INSERT INTO t2 VALUES(1,0),(2,1);
+      CREATE VIRTUAL TABLE t3 USING fts5(a, b, content='', tokendata=1);
+      INSERT INTO t3 VALUES(3,3);
+      PRAGMA defer_foreign_keys=ON;
+      DELETE FROM t2;
+    COMMIT;
+  }
+}
+}
 
 finish_test
diff --git a/test/fork-test.c b/test/fork-test.c
new file mode 100644
index 0000000000..8a9b4b4afa
--- /dev/null
+++ b/test/fork-test.c
@@ -0,0 +1,310 @@
+/*
+** The program demonstrates how a child process created using fork()
+** can continue to use SQLite for a database that the parent had opened and
+** and was writing into when the fork() occurred.
+**
+** This program executes the following steps:
+**
+**   1.  Create a new database file.  Open it, and populate it.
+**   2.  Start a transaction and make changes.
+**       ^-- close the transaction prior to fork() if --commit-before-fork
+**   3.  Fork()
+**   4.  In the child, close the database connection.  Special procedures
+**       are needed to close the database connection in the child.  See the
+**       implementation below.
+**   5.  Commit the transaction in the parent.
+**   6.  Verify that the transaction committed in the parent.
+**   7.  In the child, after a delay to allow time for (5) and (6),
+**       open a new database connection and verify that the transaction
+**       committed by (5) is seen.
+**   8.  Add make further changes and commit them in the child, using the
+**       new database connection.
+**   9.  In the parent, after a delay to account for (8), verify that
+**       the new transaction added by (8) can be seen.
+**
+** Usage:
+**
+**    fork-test FILENAME [options]
+**
+** Options:
+**
+**    --wal                       Run the database in WAL mode
+**    --vfstrace                  Enable VFS tracing for debugging
+**    --commit-before-fork        COMMIT prior to the fork() in step 3
+**    --delay-after-4 N           Pause for N seconds after step 4
+**
+** How To Compile:
+**
+**   gcc -O0 -g -Wall -I$(SQLITESRC) \
+**         f1.c $(SQLITESRC)/ext/misc/vfstrace.c $(SQLITESRC/sqlite3.c \
+**         -ldl -lpthread -lm
+**
+** Test procedure:
+**
+**   (1)  Run "fork-test x1.db".  Verify no I/O errors occur and that
+**        both parent and child see all three rows in the t1 table.
+**
+**   (2)  Repeat (1) adding the --wal option.
+**
+**   (3)  Repeat (1) and (2) adding the --commit-before-fork option.
+**
+**   (4)  Repeat all prior steps adding the --delay-after-4 option with
+**        a timeout of 15 seconds or so.  Then, while both parent and child
+**        are paused, run the CLI against the x1.db database from a separate
+**        window and verify that all the correct file locks are still working
+**        correctly.
+**
+** Take-Aways:
+**
+**   *   If a process has open SQLite database connections when it fork()s,
+**       the child can call exec() and all it well.  Nothing special needs
+**       to happen.
+**
+**   *   If a process has open SQLite database connections when it fork()s,
+**       the child can do anything that does not involve using SQLite without
+**       causing problems in the parent.  No special actions are needed in
+**       the child.
+**
+**   *   If a process has open SQLite database connections when it fork()s,
+**       the child can call sqlite3_close() on those database connections
+**       as long as there were no pending write transactions when the fork()
+**       occurred.
+**
+**   *   If a process has open SQLite database connections that are in the
+**       middle of a write transaction and then the processes fork()s, the
+**       child process should close the database connections using the
+**       procedures demonstrated in Step 4 below before trying to do anything
+**       else with SQLite.
+**
+**   *   If a child process can safely close SQLite database connections that
+**       it inherited via fork() using the procedures shown in Step 4 below
+**       even if the database connections were not involved in a write
+**       transaction at the time of the fork().  The special procedures are
+**       required if a write transaction was active.  They are optional
+**       otherwise.  No harm results from using the special procedures when
+**       they are not necessary.
+**
+**   *   Child processes that use SQLite should open their own database
+**       connections.  They should not attempt to use a database connection
+**       that is inherited from the parent.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include "sqlite3.h"
+
+/*
+** Process ID of the parent
+*/
+static pid_t parentPid = 0;
+
+/*
+** Return either "parent" or "child", as appropriate.
+*/
+static const char *whoAmI(void){
+  return getpid()==parentPid ? "parent" : "child";
+}
+
+/*
+** This is an sqlite3_exec() callback routine that prints all results.
+*/
+static int execCallback(void *pNotUsed, int nCol, char **aVal, char **aCol){
+  int i;
+  const char *zWho = whoAmI();
+  for(i=0; i<nCol; i++){
+    const char *zVal = aVal[i];
+    const char *zCol = aCol[i];
+    if( zVal==0 ) zVal = "NULL";
+    if( zCol==0 ) zCol = "NULL";
+    printf("%s: %s = %s\n", zWho, zCol, zVal);
+    fflush(stdout);
+  }
+  return 0;
+}
+
+/*
+** Execute one or more SQL statements.
+*/
+static void sqlExec(sqlite3 *db, const char *zSql, int bCallback){
+  int rc;
+  char *zErr = 0;
+  printf("%s: %s\n", whoAmI(), zSql);
+  fflush(stdout);
+  rc = sqlite3_exec(db, zSql, bCallback ? execCallback : 0, 0, &zErr);
+  if( rc || zErr!=0 ){
+    printf("%s: %s: rc=%d: %s\n", 
+      whoAmI(), zSql, rc, zErr);
+    exit(1);
+  }
+}
+
+/*
+** Trace callback for the vfstrace extension.
+*/
+static int vfsTraceCallback(const char *zMsg, void *pNotUsed){
+  printf("%s: %s", whoAmI(), zMsg);
+  fflush(stdout);
+  return 0;
+}
+
+/* External VFS module provided by ext/misc/vfstrace.c
+*/
+extern int vfstrace_register(
+  const char *zTraceName,         // Name of the newly constructed VFS
+  const char *zOldVfsName,        // Name of the underlying VFS
+  int (*xOut)(const char*,void*), // Output routine.  ex: fputs
+  void *pOutArg,                  // 2nd argument to xOut.  ex: stderr
+  int makeDefault                 // Make the new VFS the default
+);
+
+
+int main(int argc, char **argv){
+  sqlite3 *db;
+  int rc;
+  int i;
+  int useWal = 0;
+  const char *zFilename = 0;
+  pid_t child = 0, c2;
+  int status;
+  int bCommitBeforeFork = 0;
+  int nDelayAfter4 = 0;
+
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++;
+    if( strcmp(z, "-wal")==0 ){
+      useWal = 1;
+    }else if( strcmp(z, "-commit-before-fork")==0 ){
+      bCommitBeforeFork = 1;
+    }else if( strcmp(z, "-delay-after-4")==0 && i+1<argc ){
+      i++;
+      nDelayAfter4 = atoi(argv[i]);
+    }else if( strcmp(z, "-vfstrace")==0 ){
+      vfstrace_register("vfstrace", 0, vfsTraceCallback, 0, 1);
+    }else if( z[0]=='-' ){
+      printf("unknown option: \"%s\"\n", argv[i]);
+      exit(1);
+    }else if( zFilename!=0 ){
+      printf("unknown argument: \"%s\"\n", argv[i]);
+      exit(1);
+    }else{
+      zFilename = argv[i];
+    }
+  }
+  if( zFilename==0 ){
+    printf("Usage: %s FILENAME\n", argv[0]);
+    return 1;
+  }
+
+  /**  Step 1 **/
+  printf("Step 1:\n");
+  parentPid = getpid();
+  unlink(zFilename);
+  rc = sqlite3_open(zFilename, &db);
+  if( rc ){
+    printf("sqlite3_open() returns %d\n", rc);
+    exit(1);
+  }
+  if( useWal ){
+    sqlExec(db, "PRAGMA journal_mode=WAL;", 0);
+  }
+  sqlExec(db, "CREATE TABLE t1(x);", 0);
+  sqlExec(db, "INSERT INTO t1 VALUES('First row');", 0);
+  sqlExec(db, "SELECT x FROM t1;", 1);
+
+  /**  Step 2 **/
+  printf("Step 2:\n");
+  sqlExec(db, "BEGIN IMMEDIATE;", 0);
+  sqlExec(db, "INSERT INTO t1 VALUES('Second row');", 0);
+  sqlExec(db, "SELECT x FROM t1;", 1);
+  if( bCommitBeforeFork ) sqlExec(db, "COMMIT", 0);
+
+  /**  Step 3 **/
+  printf("Step 3:\n"); fflush(stdout);
+  child = fork();
+  if( child!=0 ){
+    printf("Parent = %d\nChild = %d\n", getpid(), child);
+  }
+
+  /**  Step 4 **/
+  if( child==0 ){
+    int k;
+    printf("Step 4:\n"); fflush(stdout);
+
+    /***********************************************************************
+    ** The following block of code closes the database connection without
+    ** rolling back or changing any files on disk.  This is necessary to
+    ** preservce the pending transaction in the parent. 
+    */
+    for(k=0; 1/*exit-by-break*/; k++){
+      const char *zDbName = sqlite3_db_name(db, k);
+      sqlite3_file *pJrnl = 0;
+      if( k==1 ) continue;
+      if( zDbName==0 ) break;
+      sqlite3_file_control(db, zDbName, SQLITE_FCNTL_NULL_IO, 0);
+      sqlite3_file_control(db, zDbName, SQLITE_FCNTL_JOURNAL_POINTER, &pJrnl);
+      if( pJrnl && pJrnl->pMethods && pJrnl->pMethods->xFileControl ){
+        pJrnl->pMethods->xFileControl(pJrnl, SQLITE_FCNTL_NULL_IO, 0);
+      }
+    }    
+    sqlite3_close(db);
+    /*
+    ** End of special close procedures for SQLite database connections
+    ** inherited via fork().
+    ***********************************************************************/
+
+    printf("%s: database connection closed\n", whoAmI()); fflush(stdout);
+  }else{
+    /* Pause the parent briefly to give the child a chance to close its
+    ** database connection */
+    sleep(1);
+  }
+
+  if( nDelayAfter4>0 ){
+    printf("%s: Delay for %d seconds\n", whoAmI(), nDelayAfter4);
+    fflush(stdout);
+    sleep(nDelayAfter4);
+    printf("%s: Continue after %d delay\n", whoAmI(), nDelayAfter4);
+    fflush(stdout);
+  }
+
+  /**  Step 5 **/
+  if( child!=0 ){
+    printf("Step 5:\n");
+    if( !bCommitBeforeFork ) sqlExec(db, "COMMIT", 0);
+    sqlExec(db, "SELECT x FROM t1;", 1);
+  }    
+  
+
+  /** Step 7 **/
+  if( child==0 ){
+    sleep(2);
+    printf("Steps 7 and 8:\n");
+    rc = sqlite3_open(zFilename, &db);
+    if( rc ){
+      printf("Child unable to reopen the database.  rc = %d\n", rc);
+      exit(1);
+    }
+    sqlExec(db, "SELECT * FROM t1;", 1);
+
+    /** Step 8 **/
+    sqlExec(db, "INSERT INTO t1 VALUES('Third row');", 0);
+    sqlExec(db, "SELECT * FROM t1;", 1);
+    sleep(1);
+    return 0;
+  }
+  c2 = wait(&status);
+  printf("Process %d finished with status %d\n", c2, status);
+
+  /** Step 9 */
+  if( child!=0 ){
+    printf("Step 9:\n");
+    sqlExec(db, "SELECT * FROM t1;", 1);
+  }
+
+  return 0;
+}
diff --git a/test/fp-speed-1.c b/test/fp-speed-1.c
new file mode 100644
index 0000000000..cb4e0409c3
--- /dev/null
+++ b/test/fp-speed-1.c
@@ -0,0 +1,154 @@
+/*
+** Performance testing of floating-point binary-to-decimal conversion for
+** SQLite versus the standard library.
+**
+** To compile:
+**
+**    make sqlite3.c
+**    gcc -Os -c sqlite3.c
+**    gcc -I. -Os test/fp-speed-1.c sqlite3.o -ldl -lm -lpthread
+**
+** To run the test:
+**
+**    time ./a.out 0 10000000     <-- standard library
+**    time ./a.out 1 10000000     <-- SQLite
+*/
+static double aVal[] = {
+  -1.0163830486285643089e+063,
+  +0.0049243807391586981e-019,
+  +7.3818732407343994867e-095,
+  +7.0678595219225717143e+000,
+  +9.2807266319850025655e+120,
+  +5.8871050861933394663e+135,
+  -2.2998023621259662805e-168,
+  -1.5903581924910847482e+181,
+  +2.4313441816844978216e-001,
+  -3.8290987328945399091e-147,
+  +1.8787914062744001349e-009,
+  +0.7270653560487146653e-014,
+  +0.0639577697913183665e+107,
+  +5.2292338879315861930e-103,
+  +6.3747682672872231319e-021,
+  +8.6972339538329106984e-129,
+  -9.5074486051597691937e-026,
+  -8.6480257845368753018e-005,
+  -3.5657595775797470332e+017,
+  -7.8323106179731761351e+161,
+  +7.7813875274120800372e+077,
+  -1.8739718928360196596e-050,
+  +8.6898145915593357218e-051,
+  +6.0566766359877837871e+002,
+  +4.1703379217148160678e+199,
+  +2.1283871288746651560e+150,
+  -6.8395236083891810637e+128,
+  -6.2114299763939529155e-147,
+  -2.0753525742614637350e-149,
+  +5.8727459803944290257e-007,
+  +8.5888991062002101817e+010,
+  +6.8624461031355917632e+026,
+  -3.3053986756670905167e-075,
+  -4.3596843152763444945e-108,
+  +0.0834139520104099673e+098,
+  -8.8581986548904222440e-192,
+  -3.6622095428727698202e+038,
+  -6.6965852297025063260e+005,
+  +1.8204169347406488441e-054,
+  +6.5234508038649000384e-065,
+  +1.5923006018219011453e+083,
+  +1.7362555291656389480e+018,
+  +7.2875431976854785882e+160,
+  +1.2835880105958926748e-146,
+  +8.0516253320320819420e-113,
+  +6.6324633399381145440e-030,
+  -1.7126500070280062058e-149,
+  +1.6995738341583712335e+042,
+  +7.6048766923738663725e-112,
+  +0.6159117235449455043e-004,
+  +5.7544894355415943289e-135,
+  +8.2970228592690581890e-023,
+  -6.5531925360163067447e+020,
+  +5.8321334606187030050e+120,
+  +5.6557166509571891727e+095,
+  +0.3322789708438408704e-114,
+  -7.1210648776698686623e-050,
+  -9.6721262526706343301e+179,
+  -3.4583916571377395084e-106,
+  +4.7896094323214750793e-172,
+  -9.6926028040004137875e-056,
+  +7.0683848275381385483e-198,
+  -5.2970114182162961964e-007,
+  -4.4287021200905393271e-170,
+  +0.0728891155732808799e-189,
+  -9.1855462025879447465e+175,
+  +3.7294126234131007796e+015,
+  +2.6857421882792719241e+003,
+  -4.7070607333624685339e+039,
+  +7.2175820768279334274e+136,
+  -8.3678412534261163481e-115,
+  +2.2174844304241822163e+019,
+  +0.1949824588606861016e+112,
+  -9.7334052955672071912e+151,
+  -9.7793887766936999879e-142,
+  -5.1561164587416931561e+139,
+  -7.5048993577765174789e-022,
+  +7.3556076568687784568e+107,
+  -5.0681628575533599865e-144,
+  +1.5209705642027747811e+165,
+  -7.5989782535048296040e-101,
+  +1.3654137203389775871e-016,
+  -1.6441720554651372066e+087,
+  -4.9042433196141125923e+000,
+  -7.7063611961649130777e+118,
+  +0.1699427460930766201e+174,
+  +8.3374317849572216870e-145,
+  -5.2355330480469580072e+081,
+  -3.8510045942194147919e+141,
+  -6.3513622544326339887e-147,
+  +2.3869303484454428988e+049,
+  +3.8352715871620360268e-165,
+  -3.1263120493136887902e+035,
+  -5.5794797002556490823e+051,
+  -8.8109874479595604379e+142,
+  -4.3727360120203216922e+070,
+  -3.1109951189668539974e+170,
+  -9.4841878031704268232e+011,
+  -3.7398451668304407277e+067,
+  +4.8984042008915959963e-091,
+};
+#define NN (sizeof(aVal)/sizeof(aVal[0]))
+
+#include "sqlite3.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv){
+  int i;
+  int cnt;
+  int fg;
+  char zBuf[1000];
+
+  if( argc!=3 ){
+    fprintf(stderr, "Usage:  %s FLAG COUNT\n", argv[0]);
+    return 1;
+  }
+  cnt = atoi(argv[2]);
+  fg = atoi(argv[1]);
+
+  switch( fg % 3 ){
+    case 0: {
+      printf("Doing %d calls to C-lib sprintf()\n", cnt);
+      for(i=0; i<cnt; i++){
+        sprintf(zBuf, "%.26g", aVal[i%NN]);
+      }
+      break;
+    }
+    case 1: {
+      printf("Doing %d calls to sqlite3_snprintf()\n", cnt);
+      for(i=0; i<cnt; i++){
+        sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.26g", aVal[i%NN]);
+      }
+      break;
+    }
+  }
+  return 0;
+}
diff --git a/test/fpconv1.test b/test/fpconv1.test
new file mode 100644
index 0000000000..195fdf9904
--- /dev/null
+++ b/test/fpconv1.test
@@ -0,0 +1,44 @@
+# 2023-07-03
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file contains a test that attempts to verify the claim that the
+# floatpoint-to-text conversion routines built into SQLite maintain at
+# least 15 significant digits of accuracy.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+if {[catch {load_static_extension db decimal} error]} {
+  puts "Skipping decimal tests, hit load error: $error"
+  finish_test; return
+}
+
+sqlite3_create_function db
+do_execsql_test fpconv1-1.0 {
+  WITH RECURSIVE
+       /* Number of random floating-point values to try.
+       ** On a circa 2016 x64 linux box, this test runs at
+       ** about 80000 cases per second  -------------------vvvvvv */
+    c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100000),
+    fp(y) AS MATERIALIZED (
+       SELECT CAST( format('%+d.%019d0e%+03d',
+                           random()%10,abs(random()),random()%200) AS real)
+        FROM c
+    )
+  SELECT y FROM fp
+   WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<15.0;
+                     /* Number of digits of accuracy required -------^^^^ */
+} {}
+#  ^---- Expect a empty set as the result.  The output is all tested numbers
+#        that fail to preserve at least 15 significant digits of accuracy.
+
+finish_test
diff --git a/test/fts1a.test b/test/fts1a.test
deleted file mode 100644
index b63e79a81b..0000000000
--- a/test/fts1a.test
+++ /dev/null
@@ -1,186 +0,0 @@
-# 2006 September 9
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS1 module.
-#
-# $Id: fts1a.test,v 1.4 2006/09/28 19:43:32 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Construct a full-text search table containing five keywords:
-# one, two, three, four, and five, in various combinations.  The
-# rowid for each will be a bitmask for the elements it contains.
-#
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(content);
-  INSERT INTO t1(content) VALUES('one');
-  INSERT INTO t1(content) VALUES('two');
-  INSERT INTO t1(content) VALUES('one two');
-  INSERT INTO t1(content) VALUES('three');
-  INSERT INTO t1(content) VALUES('one three');
-  INSERT INTO t1(content) VALUES('two three');
-  INSERT INTO t1(content) VALUES('one two three');
-  INSERT INTO t1(content) VALUES('four');
-  INSERT INTO t1(content) VALUES('one four');
-  INSERT INTO t1(content) VALUES('two four');
-  INSERT INTO t1(content) VALUES('one two four');
-  INSERT INTO t1(content) VALUES('three four');
-  INSERT INTO t1(content) VALUES('one three four');
-  INSERT INTO t1(content) VALUES('two three four');
-  INSERT INTO t1(content) VALUES('one two three four');
-  INSERT INTO t1(content) VALUES('five');
-  INSERT INTO t1(content) VALUES('one five');
-  INSERT INTO t1(content) VALUES('two five');
-  INSERT INTO t1(content) VALUES('one two five');
-  INSERT INTO t1(content) VALUES('three five');
-  INSERT INTO t1(content) VALUES('one three five');
-  INSERT INTO t1(content) VALUES('two three five');
-  INSERT INTO t1(content) VALUES('one two three five');
-  INSERT INTO t1(content) VALUES('four five');
-  INSERT INTO t1(content) VALUES('one four five');
-  INSERT INTO t1(content) VALUES('two four five');
-  INSERT INTO t1(content) VALUES('one two four five');
-  INSERT INTO t1(content) VALUES('three four five');
-  INSERT INTO t1(content) VALUES('one three four five');
-  INSERT INTO t1(content) VALUES('two three four five');
-  INSERT INTO t1(content) VALUES('one two three four five');
-}
-
-do_test fts1a-1.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts1a-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two'}
-} {3 7 11 15 19 23 27 31}
-do_test fts1a-1.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two one'}
-} {3 7 11 15 19 23 27 31}
-do_test fts1a-1.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two three'}
-} {7 15 23 31}
-do_test fts1a-1.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one three two'}
-} {7 15 23 31}
-do_test fts1a-1.6 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two three one'}
-} {7 15 23 31}
-do_test fts1a-1.7 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two one three'}
-} {7 15 23 31}
-do_test fts1a-1.8 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three one two'}
-} {7 15 23 31}
-do_test fts1a-1.9 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three two one'}
-} {7 15 23 31}
-do_test fts1a-1.10 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two THREE'}
-} {7 15 23 31}
-do_test fts1a-1.11 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '  ONE    Two   three  '}
-} {7 15 23 31}
-
-do_test fts1a-2.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one"'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts1a-2.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two"'}
-} {3 7 11 15 19 23 27 31}
-do_test fts1a-2.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"two one"'}
-} {}
-do_test fts1a-2.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two three"'}
-} {7 15 23 31}
-do_test fts1a-2.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three two"'}
-} {}
-do_test fts1a-2.6 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two three four"'}
-} {15 31}
-do_test fts1a-2.7 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three two four"'}
-} {}
-do_test fts1a-2.8 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three five"'}
-} {21}
-do_test fts1a-2.9 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three" five'}
-} {21 29}
-do_test fts1a-2.10 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five "one three"'}
-} {21 29}
-do_test fts1a-2.11 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five "one three" four'}
-} {29}
-do_test fts1a-2.12 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five four "one three"'}
-} {29}
-do_test fts1a-2.13 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three" four five'}
-} {29}
-
-do_test fts1a-3.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts1a-3.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one -two'}
-} {1 5 9 13 17 21 25 29}
-do_test fts1a-3.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '-two one'}
-} {1 5 9 13 17 21 25 29}
-
-do_test fts1a-4.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one OR two'}
-} {1 2 3 5 6 7 9 10 11 13 14 15 17 18 19 21 22 23 25 26 27 29 30 31}
-do_test fts1a-4.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two" OR three'}
-} {3 4 5 6 7 11 12 13 14 15 19 20 21 22 23 27 28 29 30 31}
-do_test fts1a-4.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three OR "one two"'}
-} {3 4 5 6 7 11 12 13 14 15 19 20 21 22 23 27 28 29 30 31}
-do_test fts1a-4.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two OR three'}
-} {3 5 7 11 13 15 19 21 23 27 29 31}
-do_test fts1a-4.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three OR two one'}
-} {3 5 7 11 13 15 19 21 23 27 29 31}
-do_test fts1a-4.6 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two OR three OR four'}
-} {3 5 7 9 11 13 15 19 21 23 25 27 29 31}
-do_test fts1a-4.7 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two OR three OR four one'}
-} {3 5 7 9 11 13 15 19 21 23 25 27 29 31}
-
-# Test the ability to handle NULL content
-#
-do_test fts1a-5.1 {
-  execsql {INSERT INTO t1(content) VALUES(NULL)}
-} {}
-do_test fts1a-5.2 {
-  set rowid [db last_insert_rowid]
-  execsql {SELECT content FROM t1 WHERE rowid=$rowid}
-} {{}}
-do_test fts1a-5.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH NULL}
-} {}
-
-
-
-finish_test
diff --git a/test/fts1b.test b/test/fts1b.test
deleted file mode 100644
index 2bbe1aab80..0000000000
--- a/test/fts1b.test
+++ /dev/null
@@ -1,147 +0,0 @@
-# 2006 September 13
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS1 module.
-#
-# $Id: fts1b.test,v 1.4 2006/09/18 02:12:48 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Fill the full-text index "t1" with phrases in english, spanish,
-# and german.  For the i-th row, fill in the names for the bits
-# that are set in the value of i.  The least significant bit is
-# 1.  For example,  the value 5 is 101 in binary which will be
-# converted to "one three" in english.
-#
-proc fill_multilanguage_fulltext_t1 {} {
-  set english {one two three four five}
-  set spanish {un dos tres cuatro cinco}
-  set german {eine zwei drei vier funf}
-  
-  for {set i 1} {$i<=31} {incr i} {
-    set cmd "INSERT INTO t1 VALUES"
-    set vset {}
-    foreach lang {english spanish german} {
-      set words {}
-      for {set j 0; set k 1} {$j<5} {incr j; incr k $k} {
-        if {$k&$i} {lappend words [lindex [set $lang] $j]}
-      }
-      lappend vset "'$words'"
-    }
-    set sql "INSERT INTO t1(english,spanish,german) VALUES([join $vset ,])"
-    # puts $sql
-    db eval $sql
-  }
-}
-
-# Construct a full-text search table containing five keywords:
-# one, two, three, four, and five, in various combinations.  The
-# rowid for each will be a bitmask for the elements it contains.
-#
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(english,spanish,german);
-}
-fill_multilanguage_fulltext_t1
-
-do_test fts1b-1.1 {
-  execsql {SELECT rowid FROM t1 WHERE english MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts1b-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE spanish MATCH 'one'}
-} {}
-do_test fts1b-1.3 {
-  execsql {SELECT rowid FROM t1 WHERE german MATCH 'one'}
-} {}
-do_test fts1b-1.4 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts1b-1.5 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'one dos drei'}
-} {7 15 23 31}
-do_test fts1b-1.6 {
-  execsql {SELECT english, spanish, german FROM t1 WHERE rowid=1}
-} {one un eine}
-do_test fts1b-1.7 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH '"one un"'}
-} {}
-
-do_test fts1b-2.1 {
-  execsql {
-    CREATE VIRTUAL TABLE t2 USING fts1(from,to);
-    INSERT INTO t2([from],[to]) VALUES ('one two three', 'four five six');
-    SELECT [from], [to] FROM t2
-  }
-} {{one two three} {four five six}}
-
-
-# Compute an SQL string that contains the words one, two, three,... to
-# describe bits set in the value $i.  Only the lower 5 bits are examined.
-#
-proc wordset {i} {
-  set x {}
-  for {set j 0; set k 1} {$j<5} {incr j; incr k $k} {
-    if {$k&$i} {lappend x [lindex {one two three four five} $j]}
-  }
-  return '$x'
-}
-
-# Create a new FTS table with three columns:
-#
-#    norm:      words for the bits of rowid
-#    plusone:   words for the bits of rowid+1
-#    invert:    words for the bits of ~rowid
-#
-db eval {
-   CREATE VIRTUAL TABLE t4 USING fts1([norm],'plusone',"invert");
-}
-for {set i 1} {$i<=15} {incr i} {
-  set vset [list [wordset $i] [wordset [expr {$i+1}]] [wordset [expr {~$i}]]]
-  db eval "INSERT INTO t4(norm,plusone,invert) VALUES([join $vset ,]);"
-}
-
-do_test fts1b-4.1 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one'}
-} {1 3 5 7 9 11 13 15}
-do_test fts1b-4.2 {
-  execsql {SELECT rowid FROM t4 WHERE norm MATCH 'one'}
-} {1 3 5 7 9 11 13 15}
-do_test fts1b-4.3 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'one'}
-} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}
-do_test fts1b-4.4 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'plusone:one'}
-} {2 4 6 8 10 12 14}
-do_test fts1b-4.5 {
-  execsql {SELECT rowid FROM t4 WHERE plusone MATCH 'one'}
-} {2 4 6 8 10 12 14}
-do_test fts1b-4.6 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one plusone:two'}
-} {1 5 9 13}
-do_test fts1b-4.7 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one two'}
-} {1 3 5 7 9 11 13 15}
-do_test fts1b-4.8 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'plusone:two norm:one'}
-} {1 5 9 13}
-do_test fts1b-4.9 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'two norm:one'}
-} {1 3 5 7 9 11 13 15}
-
-
-finish_test
diff --git a/test/fts1c.test b/test/fts1c.test
deleted file mode 100644
index a12469593a..0000000000
--- a/test/fts1c.test
+++ /dev/null
@@ -1,1213 +0,0 @@
-# 2006 September 14
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS1 module.
-#
-# $Id: fts1c.test,v 1.11 2006/10/04 17:35:28 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Create a table of sample email data.   The data comes from email
-# archives of Enron executives that was published as part of the
-# litigation against that company.
-#
-do_test fts1c-1.1 {
-  db eval {
-    CREATE VIRTUAL TABLE email USING fts1([from],[to],subject,body);
-    BEGIN TRANSACTION;
-INSERT INTO email([from],[to],subject,body) VALUES('savita.puthigai@enron.com', 'traders.eol@enron.com, traders.eol@enron.com', 'EnronOnline- Change to Autohedge', 'Effective Monday, October 22, 2001 the following changes will be made to the Autohedge functionality on EnronOnline.
-
-The volume on the hedge will now respect the minimum volume and volume increment settings on the parent product. See rules below: 
-
-?	If the transaction volume on the child is less than half of the parent''s minimum volume no hedge will occur.
-?	If the transaction volume on the child is more than half the parent''s minimum volume but less than half the volume increment on the parent, the hedge will volume will be the parent''s minimum volume.
-?	For all other volumes, the same rounding rules will apply based on the volume increment on the parent product.
-
-Please see example below:
-
-Parent''s Settings:
-Minimum: 	5000
-Increment:  1000
-
-Volume on Autohedge transaction			Volume Hedged
-1      - 2499							0
-2500 - 5499							5000
-5500 - 6499							6000');
-INSERT INTO email([from],[to],subject,body) VALUES('dana.davis@enron.com', 'laynie.east@enron.com, lisa.king@enron.com, lisa.best@enron.com,', 'Leaving Early', 'FYI:  
-If it''s ok with everyone''s needs, I would like to leave @4pm. If you think 
-you will need my assistance past the 4 o''clock hour just let me know;  I''ll 
-be more than willing to stay.');
-INSERT INTO email([from],[to],subject,body) VALUES('enron_update@concureworkplace.com', 'louise.kitchen@enron.com', '<<Concur Expense Document>> - CC02.06.02', 'The following expense report is ready for approval:
-
-Employee Name: Christopher F. Calger
-Status last changed by: Mollie E. Gustafson Ms
-Expense Report Name: CC02.06.02
-Report Total: $3,972.93
-Amount Due Employee: $3,972.93
-
-
-To approve this expense report, click on the following link for Concur Expense.
-http://expensexms.enron.com');
-INSERT INTO email([from],[to],subject,body) VALUES('jeff.duff@enron.com', 'julie.johnson@enron.com', 'Work request', 'Julie,
-
-Could you print off the current work request report by 1:30 today?
-
-Gentlemen,
-
-I''d like to review this today at 1:30 in our office.  Also, could you provide 
-me with your activity reports so I can have Julie enter this information.
-
-JD');
-INSERT INTO email([from],[to],subject,body) VALUES('v.weldon@enron.com', 'gary.l.carrier@usa.dupont.com, scott.joyce@bankofamerica.com', 'Enron News', 'This could turn into something big.... 
-http://biz.yahoo.com/rf/010129/n29305829.html');
-INSERT INTO email([from],[to],subject,body) VALUES('mark.haedicke@enron.com', 'paul.simons@enron.com', 'Re: First Polish Deal!', 'Congrats!  Things seem to be building rapidly now on the Continent.  Mark');
-INSERT INTO email([from],[to],subject,body) VALUES('e..carter@enron.com', 't..robinson@enron.com', 'FW: Producers Newsletter 9-24-2001', '
-The producer lumber pricing sheet.
- -----Original Message-----
-From: 	Johnson, Jay  
-Sent:	Tuesday, October 16, 2001 3:42 PM
-To:	Carter, Karen E.
-Subject:	FW: Producers Newsletter 9-24-2001
-
-
-
- -----Original Message-----
-From: 	Daigre, Sergai  
-Sent:	Friday, September 21, 2001 8:33 PM
-Subject:	Producers Newsletter 9-24-2001
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('david.delainey@enron.com', 'kenneth.lay@enron.com', 'Greater Houston Partnership', 'Ken, in response to the letter from Mr Miguel San Juan, my suggestion would 
-be to offer up the Falcon for their use; however, given the tight time frame 
-and your recent visit with Mr. Fox that it would be difficult for either you 
-or me to participate.
-
-I spoke to Max and he agrees with this approach.
-
-I hope this meets with your approval.
-
-Regards
-Delainey');
-INSERT INTO email([from],[to],subject,body) VALUES('lachandra.fenceroy@enron.com', 'lindy.donoho@enron.com', 'FW: Bus Applications Meeting Follow Up', 'Lindy,
-
-Here is the original memo we discussed earlier.  Please provide any information that you may have.
-
-Your cooperation is greatly appreciated.
-
-Thanks,
-
-lachandra.fenceroy@enron.com
-713.853.3884
-877.498.3401 Pager
-
- -----Original Message-----
-From: 	Bisbee, Joanne  
-Sent:	Wednesday, September 26, 2001 7:50 AM
-To:	Fenceroy, LaChandra
-Subject:	FW: Bus Applications Meeting Follow Up
-
-Lachandra, Please get with David Duff today and see what this is about.  Who are our TW accounting business users?
-
- -----Original Message-----
-From: 	Koh, Wendy  
-Sent:	Tuesday, September 25, 2001 2:41 PM
-To:	Bisbee, Joanne
-Subject:	Bus Applications Meeting Follow Up
-
-Lisa brought up a TW change effective Nov 1.  It involves eliminating a turnback surcharge.  I have no other information, but you might check with the business folks for any system changes required.
-
-Wendy');
-INSERT INTO email([from],[to],subject,body) VALUES('danny.mccarty@enron.com', 'fran.fagan@enron.com', 'RE: worksheets', 'Fran,
-    If Julie''s merit needs to be lump sum, just move it over to that column.  Also, send me Eric Gadd''s sheets as well.  Thanks.
-Dan
-
- -----Original Message-----
-From: 	Fagan, Fran  
-Sent:	Thursday, December 20, 2001 11:10 AM
-To:	McCarty, Danny
-Subject:	worksheets
-
-As discussed, attached are your sheets for bonus and merit.
-
-Thanks,
-
-Fran Fagan
-Sr. HR Rep
-713.853.5219
-
-
- << File: McCartyMerit.xls >>  << File: mccartyBonusCommercial_UnP.xls >> 
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('bert.meyers@enron.com', 'shift.dl-portland@enron.com', 'OCTOBER SCHEDULE', 'TEAM,
-
-PLEASE SEND ME ANY REQUESTS THAT YOU HAVE FOR OCTOBER.  SO FAR I HAVE THEM FOR LEAF.  I WOULD LIKE TO HAVE IT DONE BY THE 15TH OF THE MONTH.  ANY QUESTIONS PLEASE GIVE ME A CALL.
-
-BERT');
-INSERT INTO email([from],[to],subject,body) VALUES('errol.mclaughlin@enron.com', 'john.arnold@enron.com, bilal.bajwa@enron.com, john.griffith@enron.com,', 'TRV Notification:  (NG - PROPT P/L - 09/27/2001)', 'The report named: NG - PROPT P/L <http://trv.corp.enron.com/linkFromExcel.asp?report_cd=11&report_name=NG+-+PROPT+P/L&category_cd=5&category_name=FINANCIAL&toc_hide=1&sTV1=5&TV1Exp=Y&current_efct_date=09/27/2001>, published as of 09/27/2001 is now available for viewing on the website.');
-INSERT INTO email([from],[to],subject,body) VALUES('patrice.mims@enron.com', 'calvin.eakins@enron.com', 'Re: Small business supply assistance', 'Hi Calvin
-
-
-I spoke with Rickey (boy, is he long-winded!!).  Gave him the name of our 
-credit guy, Russell Diamond.
-
-Thank for your help!');
-INSERT INTO email([from],[to],subject,body) VALUES('legal <.hall@enron.com>', 'stephanie.panus@enron.com', 'Termination update', 'City of Vernon and Salt River Project terminated their contracts.  I will fax these notices to you.');
-INSERT INTO email([from],[to],subject,body) VALUES('d..steffes@enron.com', 'richard.shapiro@enron.com', 'EES / ENA Government Affairs Staffing & Outside Services', 'Rick --
-
-Here is the information on staffing and outside services.  Call if you need anything else.
-
-Jim
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('gelliott@industrialinfo.com', 'pcopello@industrialinfo.com', 'ECAAR (Gavin), WSCC (Diablo Canyon), & NPCC (Seabrook)', 'Dear Power Outage Database Customer, 
-Attached you will find an excel document. The outages contained within are forced or rescheduled outages. Your daily delivery will still contain these outages. 
-In addition to the two excel documents, there is a dbf file that is formatted like your daily deliveries you receive nightly. This will enable you to load the data into your regular database. Any questions please let me know. Thanks. 
-Greg Elliott 
-IIR, Inc. 
-713-783-5147 x 3481 
-outages@industrialinfo.com 
-THE INFORMATION CONTAINED IN THIS E-MAIL IS LEGALLY PRIVILEGED AND CONFIDENTIAL INFORMATION INTENDED ONLY FOR THE USE OF THE INDIVIDUAL OR ENTITY NAMED ABOVE.  YOU ARE HEREBY NOTIFIED THAT ANY DISSEMINATION, DISTRIBUTION, OR COPY OF THIS E-MAIL TO UNAUTHORIZED ENTITIES IS STRICTLY PROHIBITED. IF YOU HAVE RECEIVED THIS 
-E-MAIL IN ERROR, PLEASE DELETE IT.
- - OUTAGE.dbf 
- - 111201R.xls 
- - 111201.xls ');
-INSERT INTO email([from],[to],subject,body) VALUES('enron.announcements@enron.com', 'all_ena_egm_eim@enron.com', 'EWS Brown Bag', 'MARK YOUR LUNCH CALENDARS NOW !
-
-You are invited to attend the EWS Brown Bag Lunch Series
-
-Featuring:   RAY BOWEN, COO
-
-Topic:  Enron Industrial Markets
-
-Thursday, March 15, 2001
-11:30 am - 12:30 pm
-EB 5 C2
-
-
-You bring your lunch,           Limited Seating
-We provide drinks and dessert.          RSVP  x 3-9610');
-INSERT INTO email([from],[to],subject,body) VALUES('chris.germany@enron.com', 'ingrid.immer@williams.com', 'Re: About St Pauls', 'Sounds good to me.  I bet this is next to the Warick?? Hotel.
-
-
-
-
-"Immer, Ingrid" <Ingrid.Immer@Williams.com> on 12/21/2000 11:48:47 AM
-To: "''chris.germany@enron.com''" <chris.germany@enron.com>
-cc:  
-Subject: About St Pauls
-
-
-
-
- <<About St Pauls.url>>  
-? 
-?http://www.stpaulshouston.org/about.html 
-
-Chris, 
-
-I like the looks of this place.? What do you think about going here Christmas 
-eve?? They have an 11:00 a.m. service and a candlelight service at 5:00 p.m., 
-among others.
-
-Let me know.?? ii 
-
- - About St Pauls.url
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('nas@cpuc.ca.gov', 'skatz@sempratrading.com, kmccrea@sablaw.com, thompson@wrightlaw.com,', 'Reply Brief filed July 31, 2000', ' - CPUC01-#76371-v1-Revised_Reply_Brief__Due_today_7_31_.doc');
-INSERT INTO email([from],[to],subject,body) VALUES('gascontrol@aglresources.com', 'dscott4@enron.com, lcampbel@enron.com', 'Alert Posted 10:00 AM November 20,2000: E-GAS Request Reminder', 'Alert Posted 10:00 AM November 20,2000: E-GAS Request Reminder
-As discussed in the Winter Operations Meeting on Sept.29,2000, 
-E-Gas(Emergency Gas) will not be offered this winter as a service from AGLC.  
-Marketers and Poolers can receive gas via Peaking and IBSS nominations(daisy 
-chain) from other marketers up to the 6 p.m. Same Day 2 nomination cycle.
-');
-INSERT INTO email([from],[to],subject,body) VALUES('dutch.quigley@enron.com', 'rwolkwitz@powermerchants.com', '', ' 
-
-Here is a goody for you');
-INSERT INTO email([from],[to],subject,body) VALUES('ryan.o''rourke@enron.com', 'k..allen@enron.com, randy.bhatia@enron.com, frank.ermis@enron.com,', 'TRV Notification:  (West VaR - 11/07/2001)', 'The report named: West VaR <http://trv.corp.enron.com/linkFromExcel.asp?report_cd=36&report_name=West+VaR&category_cd=2&category_name=WEST&toc_hide=1&sTV1=2&TV1Exp=Y&current_efct_date=11/07/2001>, published as of 11/07/2001 is now available for viewing on the website.');
-INSERT INTO email([from],[to],subject,body) VALUES('mjones7@txu.com', 'cstone1@txu.com, ggreen2@txu.com, timpowell@txu.com,', 'Enron / HPL Actuals for July 10, 2000', 'Teco Tap       10.000 / Enron ; 110.000 / HPL IFERC
-
-LS HPL LSK IC       30.000 / Enron
-');
-INSERT INTO email([from],[to],subject,body) VALUES('susan.pereira@enron.com', 'kkw816@aol.com', 'soccer practice', 'Kathy-
-
-Is it safe to assume that practice is cancelled for tonight??
-
-Susan Pereira');
-INSERT INTO email([from],[to],subject,body) VALUES('mark.whitt@enron.com', 'barry.tycholiz@enron.com', 'Huber Internal Memo', 'Please look at this.  I didn''t know how deep to go with the desk.  Do you think this works.
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('m..forney@enron.com', 'george.phillips@enron.com', '', 'George,
-Give me a call and we will further discuss opportunities on the 13st floor.
-
-Thanks,
-JMForney
-3-7160');
-INSERT INTO email([from],[to],subject,body) VALUES('brad.mckay@enron.com', 'angusmcka@aol.com', 'Re: (no subject)', 'not yet');
-INSERT INTO email([from],[to],subject,body) VALUES('adam.bayer@enron.com', 'jonathan.mckay@enron.com', 'FW: Curve Fetch File', 'Here is the curve fetch file sent to me.  It has plenty of points in it.  If you give me a list of which ones you need we may be able to construct a secondary worksheet to vlookup the values.
-
-adam
-35227
-
-
- -----Original Message-----
-From: 	Royed, Jeff  
-Sent:	Tuesday, September 25, 2001 11:37 AM
-To:	Bayer, Adam
-Subject:	Curve Fetch File
-
-Let me know if it works.   It may be required to have a certain version of Oracle for it to work properly.
-
- 
-
-Jeff Royed
-Enron 
-Energy Operations
-Phone: 713-853-5295');
-INSERT INTO email([from],[to],subject,body) VALUES('matt.smith@enron.com', 'yan.wang@enron.com', 'Report Formats', 'Yan,
-
-The merged reports look great.  I believe the only orientation changes are to 
-"unmerge" the following six reports:  
-
-31 Keystone Receipts
-15 Questar Pipeline
-40 Rockies Production
-22 West_2
-23 West_3
-25 CIG_WIC
-
-The orientation of the individual reports should be correct.  Thanks.
-
-Mat
-
-PS.  Just a reminder to add the "*" by the title of calculated points.');
-INSERT INTO email([from],[to],subject,body) VALUES('michelle.lokay@enron.com', 'jimboman@bigfoot.com', 'Egyptian Festival', '---------------------- Forwarded by Michelle Lokay/ET&S/Enron on 09/07/2000 
-10:08 AM ---------------------------
-
-
-"Karkour, Randa" <Randa.Karkour@COMPAQ.com> on 09/07/2000 09:01:04 AM
-To: "''Agheb (E-mail)" <Agheb@aol.com>, "Leila Mankarious (E-mail)" 
-<Leila_Mankarious@mhhs.org>, "''Marymankarious (E-mail)" 
-<marymankarious@aol.com>, "Michelle lokay (E-mail)" <mlokay@enron.com>, "Ramy 
-Mankarious (E-mail)" <Mankarious@aol.com>
-cc:  
-
-Subject: Egyptian Festival
-
-
- <<Egyptian Festival.url>>
-
- http://www.egyptianfestival.com/
-
- - Egyptian Festival.url
-');
-INSERT INTO email([from],[to],subject,body) VALUES('errol.mclaughlin@enron.com', 'sherry.dawson@enron.com', 'Urgent!!! --- New EAST books', 'This has to be done..................................
-
-Thanks
----------------------- Forwarded by Errol McLaughlin/Corp/Enron on 12/20/2000 
-08:39 AM ---------------------------
-   
-	
-	
-	From:  William Kelly @ ECT                           12/20/2000 08:31 AM
-	
-
-To: Kam Keiser/HOU/ECT@ECT, Darron C Giron/HOU/ECT@ECT, David 
-Baumbach/HOU/ECT@ECT, Errol McLaughlin/Corp/Enron@ENRON
-cc: Kimat Singla/HOU/ECT@ECT, Kulvinder Fowler/NA/Enron@ENRON, Kyle R 
-Lilly/HOU/ECT@ECT, Jeff Royed/Corp/Enron@ENRON, Alejandra 
-Chavez/NA/Enron@ENRON, Crystal Hyde/HOU/ECT@ECT 
-
-Subject: New EAST books
-
-We have new book names in TAGG for our intramonth portfolios and it is 
-extremely important that any deal booked to the East is communicated quickly 
-to someone on my team.  I know it will take some time for the new names to 
-sink in and I do not want us to miss any positions or P&L.  
-
-Thanks for your help on this.
-
-New:
-Scott Neal :         East Northeast
-Dick Jenkins:     East Marketeast
-
-WK 
-');
-INSERT INTO email([from],[to],subject,body) VALUES('david.forster@enron.com', 'eol.wide@enron.com', 'Change to Stack Manager', 'Effective immediately, there is a change to the Stack Manager which will 
-affect any Inactive Child.
-
-An inactive Child with links to Parent products will not have their 
-calculated prices updated until the Child product is Activated.
-
-When the Child Product is activated, the price will be recalculated and 
-updated BEFORE it is displayed on the web.
-
-This means that if you are inputting a basis price on a Child product, you 
-will not see the final, calculated price until you Activate the product, at 
-which time the customer will also see it.
-
-If you have any questions, please contact the Help Desk on:
-
-Americas: 713 853 4357
-Europe: + 44 (0) 20 7783 7783
-Asia/Australia: +61 2 9229 2300
-
-Dave');
-INSERT INTO email([from],[to],subject,body) VALUES('vince.kaminski@enron.com', 'jhh1@email.msn.com', 'Re: Light reading - see pieces beginning on page 7', 'John,
-
-I saw it. Very interesting.
-
-Vince
-
-
-
-
-
-"John H Herbert" <jhh1@email.msn.com> on 07/28/2000 08:38:08 AM
-To: "Vince J Kaminski" <Vince_J_Kaminski@enron.com>
-cc:  
-Subject: Light reading - see pieces beginning on page 7
-
-
-Cheers and have a nice weekend,
-
-
-JHHerbert
-
-
-
-
- - gd000728.pdf
-
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('matthew.lenhart@enron.com', 'mmmarcantel@equiva.com', 'RE:', 'i will try to line up a pig for you ');
-INSERT INTO email([from],[to],subject,body) VALUES('jae.black@enron.com', 'claudette.harvey@enron.com, chaun.roberts@enron.com, judy.martinez@enron.com,', 'Disaster Recovery Equipment', 'As a reminder...there are several pieces of equipment that are set up on the 30th Floor, as well as on our floor, for the Disaster Recovery Team.  PLEASE DO NOT TAKE, BORROW OR USE this equipment.  Should you need to use another computer system, other than yours, or make conference calls please work with your Assistant to help find or set up equipment for you to use.
-
-Thanks for your understanding in this matter.
-
-T.Jae Black
-East Power Trading
-Assistant to Kevin Presto
-off. 713-853-5800
-fax 713-646-8272
-cell 713-539-4760');
-INSERT INTO email([from],[to],subject,body) VALUES('eric.bass@enron.com', 'dale.neuner@enron.com', '5 X 24', 'Dale,
-
-Have you heard anything more on the 5 X 24s?  We would like to get this 
-product out ASAP.
-
-
-Thanks,
-
-Eric');
-INSERT INTO email([from],[to],subject,body) VALUES('messenger@smartreminders.com', 'm..tholt@enron.com', '10% Coupon - PrintPal Printer Cartridges - 100% Guaranteed', '[IMAGE]
-[IMAGE][IMAGE][IMAGE] 
-Dear  SmartReminders Member,
-       [IMAGE]         [IMAGE]        [IMAGE]     [IMAGE]    [IMAGE]    [IMAGE]        [IMAGE]      [IMAGE]     	
-
-
-  
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
-
-We respect  your privacy and are a Certified Participant of the BBBOnLine
- Privacy Program.  To be removed from future offers,click  here. 
-SmartReminders.com  is a permission based service. To unsubscribe click  here .  ');
-INSERT INTO email([from],[to],subject,body) VALUES('benjamin.rogers@enron.com', 'mark.bernstein@enron.com', '', 'The guy you are talking about left CIN under a "cloud of suspicion" sort of 
-speak.  He was the one who got into several bad deals and PPA''s in California 
-for CIN, thus he left on a bad note.  Let me know if you need more detail 
-than that, I felt this was the type of info you were looking for.  Thanks!
-Ben');
-INSERT INTO email([from],[to],subject,body) VALUES('enron_update@concureworkplace.com', 'michelle.cash@enron.com', 'Expense Report Receipts Not Received', 'Employee Name: Michelle Cash
-Report Name:   Houston Cellular 8-11-01
-Report Date:   12/13/01
-Report ID:     594D37C9ED2111D5B452
-Submitted On:  12/13/01
-
-You are only allowed 2 reports with receipts outstanding.  Your expense reports will not be paid until you meet this requirement.');
-INSERT INTO email([from],[to],subject,body) VALUES('susan.mara@enron.com', 'ray.alvarez@enron.com, mark.palmer@enron.com, karen.denne@enron.com,', 'CAISO Emergency Motion -- to discontinue market-based rates for', 'FYI.  the latest broadside against the generators.
-
-Sue Mara
-Enron Corp.
-Tel: (415) 782-7802
-Fax:(415) 782-7854
------ Forwarded by Susan J Mara/NA/Enron on 06/08/2001 12:24 PM -----
-
-
-	"Milner, Marcie" <MMilner@coral-energy.com> 06/08/2001 11:13 AM 	   To: "''smara@enron.com''" <smara@enron.com>  cc:   Subject: CAISO Emergency Motion	
-
-
-Sue, did you see this emergency motion the CAISO filed today?  Apparently
-they are requesting that FERC discontinue market-based rates immediately and
-grant refunds plus interest on the difference between cost-based rates and
-market revenues received back to May 2000.  They are requesting the
-commission act within 14 days.  Have you heard anything about what they are
-doing?
-
-Marcie
-
-http://www.caiso.com/docs/2001/06/08/200106081005526469.pdf 
-');
-INSERT INTO email([from],[to],subject,body) VALUES('fletcher.sturm@enron.com', 'eloy.escobar@enron.com', 'Re: General Brinks Position Meeting', 'Eloy,
-
-Who is General Brinks?
-
-Fletch');
-INSERT INTO email([from],[to],subject,body) VALUES('nailia.dindarova@enron.com', 'richard.shapiro@enron.com', 'Documents for Mark Frevert (on EU developments and lessons from', 'Rick,
-
-Here are the documents that Peter has prepared for Mark Frevert. 
-
-Nailia
----------------------- Forwarded by Nailia Dindarova/LON/ECT on 25/06/2001 
-16:36 ---------------------------
-
-
-Nailia Dindarova
-25/06/2001 15:36
-To: Michael Brown/Enron@EUEnronXGate
-cc: Ross Sankey/Enron@EUEnronXGate, Eric Shaw/ENRON@EUEnronXGate, Peter 
-Styles/LON/ECT@ECT 
-
-Subject: Documents for Mark Frevert (on EU developments and lessons from 
-California)
-
-Michael,
-
-
-These are the documents that Peter promised to give to you for Mark Frevert. 
-He has now handed them to him in person but asked me to transmit them 
-electronically to you, as well as Eric and Ross.
-
-Nailia
-
-
-
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('peggy.a.kostial@accenture.com', 'dave.samuels@enron.com', 'EOL-Accenture Deal Sheet', 'Dave -
-
-Attached are our comments and suggested changes. Please call to review.
-
-On the time line for completion, we have four critical steps to complete:
-     Finalize market analysis to refine business case, specifically
-     projected revenue stream
-     Complete counterparty surveying, including targeting 3 CPs for letters
-     of intent
-     Review Enron asset base for potential reuse/ licensing
-     Contract negotiations
-
-Joe will come back to us with an updated time line, but it is my
-expectation that we are still on the same schedule (we just begun week
-three) with possibly a week or so slippage.....contract negotiations will
-probably be the critical path.
-
-We will send our cut at the actual time line here shortly. Thanks,
-
-Peggy
-
-(See attached file: accenture-dealpoints v2.doc)
- - accenture-dealpoints v2.doc ');
-INSERT INTO email([from],[to],subject,body) VALUES('thomas.martin@enron.com', 'thomas.martin@enron.com', 'Re: Guadalupe Power Partners LP', '---------------------- Forwarded by Thomas A Martin/HOU/ECT on 03/20/2001 
-03:49 PM ---------------------------
-
-
-Thomas A Martin
-10/11/2000 03:55 PM
-To: Patrick Wade/HOU/ECT@ECT
-cc:  
-Subject: Re: Guadalupe Power Partners LP  
-
-The deal is physically served at Oasis Waha or Oasis Katy and is priced at 
-either HSC, Waha or Katytailgate GD at buyers option three days prior to 
-NYMEX  close.
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('judy.townsend@enron.com', 'dan.junek@enron.com, chris.germany@enron.com', 'Columbia Distribution''s Capacity Available for Release - Sum', '---------------------- Forwarded by Judy Townsend/HOU/ECT on 03/09/2001 11:04 
-AM ---------------------------
-
-
-agoddard@nisource.com on 03/08/2001 09:16:57 AM
-To: "        -         *Koch, Kent" <kkoch@nisource.com>, "        -         
-*Millar, Debra" <dmillar@nisource.com>, "        -         *Burke, Lynn" 
-<lburke@nisource.com>
-cc: "        -         *Heckathorn, Tom" <theckathorn@nisource.com> 
-Subject: Columbia Distribution''s Capacity Available for Release - Sum
-
-
-Attached is Columbia Distribution''s notice of capacity available for release
-for
-the summer of 2001 (Apr. 2001 through Oct. 2001).
-
-Please note that the deadline for bids is 3:00pm EST on March 20, 2001.
-
-If you have any questions, feel free to contact any of the representatives
-listed
-at the bottom of the attachment.
-
-Aaron Goddard
-
-
-
-
- - 2001Summer.doc
-');
-INSERT INTO email([from],[to],subject,body) VALUES('rhonda.denton@enron.com', 'tim.belden@enron.com, dana.davis@enron.com, genia.fitzgerald@enron.com,', 'Split Rock Energy LLC', 'We have received the executed EEI contract from this CP dated 12/12/2000.  
-Copies will be distributed to Legal and Credit.');
-INSERT INTO email([from],[to],subject,body) VALUES('kerrymcelroy@dwt.com', 'jack.speer@alcoa.com, crow@millernash.com, michaelearly@earthlink.net,', 'Oral Argument Request', ' - Oral Argument Request.doc');
-INSERT INTO email([from],[to],subject,body) VALUES('mike.carson@enron.com', 'rlmichaelis@hormel.com', '', 'Did you come in town this wk end..... My new number at our house is : 
-713-668-3712...... my cell # is 281-381-7332
-
-the kid');
-INSERT INTO email([from],[to],subject,body) VALUES('cooper.richey@enron.com', 'trycooper@hotmail.com', 'FW: Contact Info', '
-
------Original Message-----
-From: Punja, Karim 
-Sent: Thursday, December 13, 2001 2:35 PM
-To: Richey, Cooper
-Subject: Contact Info
-
-
-Cooper,
-
-Its been a real pleasure working with you (even though it was for only a small amount of time)
-I hope we can stay in touch.
-
-Home# 234-0249
-email: kpunja@hotmail.com
-
-Take Care, 
-
-Karim.
-  ');
-INSERT INTO email([from],[to],subject,body) VALUES('bjm30@earthlink.net', 'mcguinn.k@enron.com, mcguinn.ian@enron.com, mcguinn.stephen@enron.com,', 'email address change', 'Hello all.
-
-I haven''t talked to many of you via email recently but I do want to give you
-my new address for your email file:
-
-    bjm30@earthlink.net
-
-I hope all is well.
-
-Brian McGuinn');
-INSERT INTO email([from],[to],subject,body) VALUES('shelley.corman@enron.com', 'steve.hotte@enron.com', 'Flat Panels', 'Can you please advise what is going on with the flat panels that we had planned to distribute to our gas logistics team.  It was in the budget and we had the okay, but now I''m hearing there is some hold-up & the units are stored on 44.
-
-Shelley');
-INSERT INTO email([from],[to],subject,body) VALUES('sara.davidson@enron.com', 'john.schwartzenburg@enron.com, scott.dieball@enron.com, recipients@enron.com,', '2001 Enron Law Conference (Distribution List 2)', '    Enron Law Conference
-
-San Antonio, Texas    May 2-4, 2001    Westin Riverwalk
-
-                   See attached memo for more details!!
-
-
-? Registration for the law conference this year will be handled through an 
-Online RSVP Form on the Enron Law Conference Website at 
-http://lawconference.corp.enron.com.  The website is still under construction 
-and will not be available until Thursday, March 15, 2001.  
-
-? We will send you another e-mail to confirm when the Law Conference Website 
-is operational. 
-
-? Please complete the Online RSVP Form as soon as it is available  and submit 
-it no later than Friday, March 30th.  
-
-
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('tori.kuykendall@enron.com', 'heath.b.taylor@accenture.com', 'Re:', 'hey - thats funny about john - he definitely remembers him - i''ll call pat 
-and let him know - we are coming on saturday - i just havent had a chance to 
-call you guys back --  looking forward to it -- i probably need the 
-directions again though');
-INSERT INTO email([from],[to],subject,body) VALUES('darron.giron@enron.com', 'bryce.baxter@enron.com', 'Re: Feedback for Audrey Cook', 'Bryce,
-
-I''ll get it done today.  
-
-DG    3-9573
-
-
-   
-	
-	
-	From:  Bryce Baxter                           06/12/2000 07:15 PM
-	
-
-To: Darron C Giron/HOU/ECT@ECT
-cc:  
-Subject: Feedback for Audrey Cook
-
-You were identified as a reviewer for Audrey Cook.  If possible, could you 
-complete her feedback by end of business Wednesday?  It will really help me 
-in the PRC process to have your input.  Thanks.
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('casey.evans@enron.com', 'stephanie.sever@enron.com', 'Gas EOL ID', 'Stephanie,
-
-In conjunction with the recent movement of several power traders, they are changing the names of their gas books as well.  The names of the new gas books and traders are as follows:
-
-PWR-NG-LT-SPP:  Mike Carson
-PWR-NG-LT-SERC:  Jeff King
-
-If you need to know their power desk to map their ID to their gas books, those desks are as follows:
-
-EPMI-LT-SPP:  Mike Carson
-EPMI-LT-SERC:  Jeff King
-
-I will be in training this afternoon, but will be back when class is over.  Let me know if you have any questions.
-
-Thanks for your help!
-Casey');
-INSERT INTO email([from],[to],subject,body) VALUES('darrell.schoolcraft@enron.com', 'david.roensch@enron.com, kimberly.watson@enron.com, michelle.lokay@enron.com,', 'Postings', 'Please see the attached.
-
-
-ds
-
-
-  
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('mcominsky@aol.com', 'cpatman@bracepatt.com, james_derrick@enron.com', 'Jurisprudence Luncheon', 'Carrin & Jim --
-
-It was an honor and a pleasure to meet both of you yesterday.  I know we will
-have fun working together on this very special event.
-
-Jeff left the jurisprudence luncheon lists for me before he left on vacation.
- I wasn''t sure whether he transmitted them to you as well.  Would you please
-advise me if you would like them sent to you?  I can email the MS Excel files
-or I can fax the hard copies to you.   Please advise what is most convenient.
-
-I plan to be in town through the holidays and can be reached by phone, email,
-or cell phone at any time.  My cell phone number is 713/705-4829.
-
-Thanks again for your interest in the ADL''s work.  Martin.
-
-Martin B. Cominsky
-Director, Southwest Region
-Anti-Defamation League
-713/627-3490, ext. 122
-713/627-2011 (fax)
-MCominsky@aol.com');
-INSERT INTO email([from],[to],subject,body) VALUES('phillip.love@enron.com', 'todagost@utmb.edu, gbsonnta@utmb.edu', 'New President', 'I had a little bird put a word in my ear.  Is there any possibility for Ben 
-Raimer to be Bush''s secretary of HHS?  Just curious about that infamous UTMB 
-rumor mill.  Hope things are well, happy holidays.
-PL');
-INSERT INTO email([from],[to],subject,body) VALUES('marie.heard@enron.com', 'ehamilton@fna.com', 'ISDA Master Agreement', 'Erin:
-
-Pursuant to your request, attached are the Schedule to the ISDA Master Agreement, together with Paragraph 13 to the ISDA Credit Support Annex.  Please let me know if you need anything else.  We look forward to hearing your comments.
-
-Marie
-
-Marie Heard
-Senior Legal Specialist
-Enron North America Corp.
-Phone:  (713) 853-3907
-Fax:  (713) 646-3490
-marie.heard@enron.com
-
-				 ');
-INSERT INTO email([from],[to],subject,body) VALUES('andrea.ring@enron.com', 'beverly.beaty@enron.com', 'Re: Tennessee Buy - Louis Dreyfus', 'Beverly -  once again thanks so much for your help on this.
-
-           
-
-                                                                     ');
-INSERT INTO email([from],[to],subject,body) VALUES('karolyn.criado@enron.com', 'j..bonin@enron.com, felicia.case@enron.com, b..clapp@enron.com,', 'Price List week of Oct. 8-9, 2001', '
-Please contact me if you have any questions regarding last weeks prices.
-
-Thank you,
-Karolyn Criado
-3-9441
-
-
- 
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('kevin.presto@enron.com', 'edward.baughman@enron.com, billy.braddock@enron.com', 'Associated', 'Please begin working on filling our Associated short position in 02.   I would like to take this risk off the books.
-
-In addition, please find out what a buy-out of VEPCO would cost us.   With Rogers transitioning to run our retail risk management, I would like to clean up our customer positions.
-
-We also need to continue to explore a JEA buy-out.
-
-Thanks.');
-INSERT INTO email([from],[to],subject,body) VALUES('stacy.dickson@enron.com', 'gregg.penman@enron.com', 'RE: Constellation TC 5-7-01', 'Gregg, 
-
-I am at home with a sick baby.  (Lots of fun!)  I will call you about this 
-tomorrow.
-
-Stacy');
-INSERT INTO email([from],[to],subject,body) VALUES('joe.quenet@enron.com', 'dfincher@utilicorp.com', '', 'hey big guy.....check this out.....
-
- w ww.gorelieberman-2000.com/');
-INSERT INTO email([from],[to],subject,body) VALUES('k..allen@enron.com', 'jacqestc@aol.com', '', 'Jacques,
-
-I sent you a fax of Kevin Kolb''s comments on the release.  The payoff on the note would be $36,248 ($36090(principal) + $158 (accrued interest)).
-This is assuming we wrap this up on Tuesday.  
-
-Please email to confirm that their changes are ok so I can set up a meeting on Tuesday to reach closure.
-
-Phillip');
-INSERT INTO email([from],[to],subject,body) VALUES('kourtney.nelson@enron.com', 'mike.swerzbin@enron.com', 'Adjusted L/R Balance', 'Mike,
-
-I placed the adjusted L/R Balance on the Enronwest site.  It is under the "Staff/Kourtney Nelson".  There are two links:  
-
-1)  "Adj L_R" is the same data/format from the weekly strategy meeting. 
-2)  "New Gen 2001_2002" link has all of the supply side info that is used to calculate the L/R balance
-	-Please note the Data Flag column, a value of "3" indicates the project was cancelled, on hold, etc and is not included in the calc.  
-
-Both of these sheets are interactive Excel spreadsheets and thus you can play around with the data as you please.  Also, James Bruce is working to get his gen report on the web.  That will help with your access to information on new gen.
-
-Please let me know if you have any questions or feedback,
-
-Kourtney
-
-
-
-Kourtney Nelson
-Fundamental Analysis 
-Enron North America
-(503) 464-8280
-kourtney.nelson@enron.com');
-INSERT INTO email([from],[to],subject,body) VALUES('d..thomas@enron.com', 'naveed.ahmed@enron.com', 'FW: Current Enron TCC Portfolio', '
-
------Original Message-----
-From: Grace, Rebecca M. 
-Sent: Monday, December 17, 2001 9:44 AM
-To: Thomas, Paul D.
-Cc: Cashion, Jim; Allen, Thresa A.; May, Tom
-Subject: RE: Current Enron TCC Portfolio
-
-
-Paul,
-
-I reviewed NY''s list.  I agree with all of their contracts numbers and mw amounts.
-
-Call if you have any more questions.
-
-Rebecca
-
-
-
- -----Original Message-----
-From: 	Thomas, Paul D.  
-Sent:	Monday, December 17, 2001 9:08 AM
-To:	Grace, Rebecca M.
-Subject:	FW: Current Enron TCC Portfolio
-
- << File: enrontccs.xls >> 
-Rebecca,
-Let me know if you see any differences.
-
-Paul
-X 3-0403
------Original Message-----
-From: Thomas, Paul D. 
-Sent: Monday, December 17, 2001 9:04 AM
-To: Ahmed, Naveed
-Subject: FW: Current Enron TCC Portfolio
-
-
-
-
------Original Message-----
-From: Thomas, Paul D. 
-Sent: Thursday, December 13, 2001 10:01 AM
-To: Baughman, Edward D.
-Subject: Current Enron TCC Portfolio
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('stephanie.panus@enron.com', 'william.bradford@enron.com, debbie.brackett@enron.com,', 'Coastal Merchant Energy/El Paso Merchant Energy', 'Coastal Merchant Energy, L.P. merged with and into El Paso Merchant Energy, 
-L.P., effective February 1, 2001, with the surviving entity being El Paso 
-Merchant Energy, L.P.  We currently have ISDA Master Agreements with both 
-counterparties.  Please see the attached memo regarding the existing Masters 
-and let us know which agreement should be terminated.
-
-Thanks,
-Stephanie
-');
-INSERT INTO email([from],[to],subject,body) VALUES('kam.keiser@enron.com', 'c..kenne@enron.com', 'RE: What about this too???', ' 
-
- -----Original Message-----
-From: 	Kenne, Dawn C.  
-Sent:	Wednesday, February 06, 2002 11:50 AM
-To:	Keiser, Kam
-Subject:	What about this too???
-
-
- << File: Netco Trader Matrix.xls >> 
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('chris.meyer@enron.com', 'joe.parks@enron.com', 'Centana', 'Talked to Chip.  We do need Cash Committe approval given the netting feature of your deal, which means Batch Funding Request.  Please update per my previous e-mail and forward.
-
-Thanks
-
-chris
-x31666');
-INSERT INTO email([from],[to],subject,body) VALUES('debra.perlingiere@enron.com', 'jworman@academyofhealth.com', '', 'Have a great weekend!   Happy Fathers Day!
-
-
-Debra Perlingiere
-Enron North America Corp.
-1400 Smith Street, EB 3885
-Houston, Texas 77002
-dperlin@enron.com
-Phone 713-853-7658
-Fax  713-646-3490');
-INSERT INTO email([from],[to],subject,body) VALUES('outlook.team@enron.com', '', 'Demo by Martha Janousek of Dashboard & Pipeline Profile / Julia  &', 'CALENDAR ENTRY:	APPOINTMENT
-
-Description:
-	Demo by Martha Janousek of Dashboard & Pipeline Profile / Julia  & Dir Rpts. - 4102
-
-Date:		1/5/2001
-Time:		9:00 AM - 10:00 AM (Central Standard Time)
-
-Chairperson:	Outlook Migration Team
-
-Detailed Description:');
-INSERT INTO email([from],[to],subject,body) VALUES('diana.seifert@enron.com', 'mark.taylor@enron.com', 'Guest access Chile', 'Hello Mark,
-
-Justin Boyd told me that your can help me with questions regarding Chile.
-We got a request for guest access through MG.
-The company is called Escondida and is a subsidiary of BHP Australia.
-
-Please advise if I can set up a guest account or not.
-F.Y.I.: MG is planning to put a "in w/h Chile" contract for Copper on-line as 
-soon as Enron has done the due diligence for this country.
-Thanks !
-
-
-Best regards
-
-Diana Seifert
-EOL PCG');
-INSERT INTO email([from],[to],subject,body) VALUES('enron_update@concureworkplace.com', 'mark.whitt@enron.com', '<<Concur Expense Document>> - 121001', 'The Approval status has changed on the following report:
-
-Status last changed by: Barry L. Tycholiz
-Expense Report Name: 121001
-Report Total: $198.98
-Amount Due Employee: $198.98
-Amount Approved: $198.98
-Amount Paid: $0.00
-Approval Status: Approved
-Payment Status: Pending
-
-
-To review this expense report, click on the following link for Concur Expense.
-http://expensexms.enron.com');
-INSERT INTO email([from],[to],subject,body) VALUES('kevin.hyatt@enron.com', '', 'Technical Support', 'Outside the U.S., please refer to the list below:
-
-Australia:
-1800 678-515
-support@palm-au.com
-
-Canada:
-1905 305-6530
-support@palm.com
-
-New Zealand:
-0800 446-398
-support@palm-nz.com
-
-U.K.:
-0171 867 0108
-eurosupport@palm.3com.com
-
-Please refer to the Worldwide Customer Support card for a complete technical support contact list.');
-INSERT INTO email([from],[to],subject,body) VALUES('geoff.storey@enron.com', 'dutch.quigley@enron.com', 'RE:', 'duke contact?
-
- -----Original Message-----
-From: 	Quigley, Dutch  
-Sent:	Wednesday, October 31, 2001 10:14 AM
-To:	Storey, Geoff
-Subject:	RE: 
-
-bp corp	Albert LaMore	281-366-4962
-
-running the reports now
-
-
- -----Original Message-----
-From: 	Storey, Geoff  
-Sent:	Wednesday, October 31, 2001 10:10 AM
-To:	Quigley, Dutch
-Subject:	RE: 
-
-give me a contact over there too
-BP
-
-
- -----Original Message-----
-From: 	Quigley, Dutch  
-Sent:	Wednesday, October 31, 2001 9:42 AM
-To:	Storey, Geoff
-Subject:	
-
-Coral	Jeff Whitnah	713-767-5374
-Relaint	Steve McGinn	713-207-4000');
-INSERT INTO email([from],[to],subject,body) VALUES('pete.davis@enron.com', 'pete.davis@enron.com', 'Start Date: 4/22/01; HourAhead hour: 3;  <CODESITE>', 'Start Date: 4/22/01; HourAhead hour: 3;  No ancillary schedules awarded.  
-Variances detected.
-Variances detected in Load schedule.
-
-    LOG MESSAGES:
-
-PARSING FILE -->> O:\Portland\WestDesk\California Scheduling\ISO Final 
-Schedules\2001042203.txt
-
----- Load Schedule ----
-$$$ Variance found in table tblLoads.
-     Details: (Hour: 3 / Preferred:   1.92 / Final:   1.89)
-  TRANS_TYPE: FINAL
-  LOAD_ID: PGE4
-  MKT_TYPE: 2
-  TRANS_DATE: 4/22/01
-  SC_ID: EPMI
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('john.postlethwaite@enron.com', 'john.zufferli@enron.com', 'Reference', 'John, hope things are going well up there for you. The big day is almost here for you and Jessica. I was wondering if I could use your name as a job reference if need be. I am just trying to get everything in order just in case something happens. 
-
-John');
-INSERT INTO email([from],[to],subject,body) VALUES('jeffrey.shankman@enron.com', 'lschiffm@jonesday.com', 'Re:', 'I saw you called on the cell this a.m.  Sorry I missed you.  (I was in the 
-shower).  I have had a shitty week--I suspect my silence (not only to you, 
-but others) after our phone call is a result of the week.  I''m seeing Glen at 
-11:15....talk to you');
-INSERT INTO email([from],[to],subject,body) VALUES('litebytz@enron.com', '', 'Lite Bytz RSVP', '
-This week''s Lite Bytz presentation will feature the following TOOLZ speaker:
-
-Richard McDougall
-Solaris 8
-Thursday, June 7, 2001
-
-If you have not already signed up, please RSVP via email to litebytz@enron.com by the end of the day Tuesday, June 5, 2001.
-
-*Remember: this is now a Brown Bag Event--so bring your lunch and we will provide cookies and drinks.
-
-Click below for more details.
-
-http://home.enron.com:84/messaging/litebytztoolzprint.jpg');
-    COMMIT;
-  }
-} {}
-
-###############################################################################
-# Everything above just builds an interesting test database.  The actual
-# tests come after this comment.
-###############################################################################
-
-do_test fts1c-1.2 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark'
-  }
-} {6 17 25 38 40 42 73 74}
-do_test fts1c-1.3 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'susan'
-  }
-} {24 40}
-do_test fts1c-1.4 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark susan'
-  }
-} {40}
-do_test fts1c-1.5 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'susan mark'
-  }
-} {40}
-do_test fts1c-1.6 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH '"mark susan"'
-  }
-} {}
-do_test fts1c-1.7 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark -susan'
-  }
-} {6 17 25 38 42 73 74}
-do_test fts1c-1.8 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH '-mark susan'
-  }
-} {24}
-do_test fts1c-1.9 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark OR susan'
-  }
-} {6 17 24 25 38 40 42 73 74}
-
-# Some simple tests of the automatic "offsets(email)" column.  In the sample
-# data set above, only one message, number 20, contains the words
-# "gas" and "reminder" in both body and subject.
-#
-do_test fts1c-2.1 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE email MATCH 'gas reminder'
-  }
-} {20 {2 0 42 3 2 1 54 8 3 0 42 3 3 1 54 8 3 0 129 3 3 0 143 3 3 0 240 3}}
-do_test fts1c-2.2 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE email MATCH 'subject:gas reminder'
-  }
-} {20 {2 0 42 3 2 1 54 8 3 1 54 8}}
-do_test fts1c-2.3 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE email MATCH 'body:gas reminder'
-  }
-} {20 {2 1 54 8 3 0 42 3 3 1 54 8 3 0 129 3 3 0 143 3 3 0 240 3}}
-do_test fts1c-2.4 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE subject MATCH 'gas reminder'
-  }
-} {20 {2 0 42 3 2 1 54 8}}
-do_test fts1c-2.5 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE body MATCH 'gas reminder'
-  }
-} {20 {3 0 42 3 3 1 54 8 3 0 129 3 3 0 143 3 3 0 240 3}}
-
-# Document 32 contains 5 instances of the world "child".  But only
-# 3 of them are paired with "product".  Make sure only those instances
-# that match the phrase appear in the offsets(email) list.
-#
-do_test fts1c-3.1 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE body MATCH 'child product' AND +rowid=32
-  }
-} {32 {3 0 94 5 3 0 114 5 3 0 207 5 3 1 213 7 3 0 245 5 3 1 251 7 3 0 409 5 3 1 415 7 3 1 493 7}}
-do_test fts1c-3.2 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE body MATCH '"child product"'
-  }
-} {32 {3 0 207 5 3 1 213 7 3 0 245 5 3 1 251 7 3 0 409 5 3 1 415 7}}
-
-# Snippet generator tests
-#
-do_test fts1c-4.1 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'subject:gas reminder'
-  }
-} {{Alert Posted 10:00 AM November 20,2000: E-<b>GAS</b> Request <b>Reminder</b>}}
-do_test fts1c-4.2 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'christmas candlelight'
-  }
-} {{<b>...</b> place.? What do you think about going here <b>Christmas</b> 
-eve?? They have an 11:00 a.m. service and a <b>candlelight</b> service at 5:00 p.m., 
-among others. <b>...</b>}}
-
-do_test fts1c-4.3 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'deal sheet potential reuse'
-  }
-} {{EOL-Accenture <b>Deal</b> <b>Sheet</b> <b>...</b> intent
-     Review Enron asset base for <b>potential</b> <b>reuse</b>/ licensing
-     Contract negotiations <b>...</b>}}
-do_test fts1c-4.4 {
-  execsql {
-    SELECT snippet(email,'<<<','>>>',' ') FROM email
-     WHERE email MATCH 'deal sheet potential reuse'
-  }
-} {{EOL-Accenture <<<Deal>>> <<<Sheet>>>  intent
-     Review Enron asset base for <<<potential>>> <<<reuse>>>/ licensing
-     Contract negotiations  }}
-do_test fts1c-4.5 {
-  execsql {
-    SELECT snippet(email,'<<<','>>>',' ') FROM email
-     WHERE email MATCH 'first things'
-  }
-} {{Re: <<<First>>> Polish Deal!  Congrats!  <<<Things>>> seem to be building rapidly now on the  }}
-do_test fts1c-4.6 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'chris is here'
-  }
-} {{<b>chris</b>.germany@enron.com <b>...</b> Sounds good to me.  I bet this <b>is</b> next to the Warick?? Hotel. <b>...</b> place.? What do you think about going <b>here</b> Christmas 
-eve?? They have an 11:00 a.m. <b>...</b>}}
-do_test fts1c-4.7 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH '"pursuant to"'
-  }
-} {{Erin:
-
-<b>Pursuant</b> <b>to</b> your request, attached are the Schedule to <b>...</b>}}
-do_test fts1c-4.8 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'ancillary load davis'
-  }
-} {{pete.<b>davis</b>@enron.com <b>...</b> Start Date: 4/22/01; HourAhead hour: 3;  No <b>ancillary</b> schedules awarded.  
-Variances detected.
-Variances detected in <b>Load</b> schedule.
-
-    LOG MESSAGES:
-
-PARSING <b>...</b>}}
-
-# Combinations of AND and OR operators:
-#
-do_test fts1c-5.1 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'questar enron OR com'
-  }
-} {{matt.smith@<b>enron</b>.<b>com</b> <b>...</b> six reports:  
-
-31 Keystone Receipts
-15 <b>Questar</b> Pipeline
-40 Rockies Production
-22 West_2 <b>...</b>}}
-do_test fts1c-5.2 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'enron OR com questar'
-  }
-} {{matt.smith@<b>enron</b>.<b>com</b> <b>...</b> six reports:  
-
-31 Keystone Receipts
-15 <b>Questar</b> Pipeline
-40 Rockies Production
-22 West_2 <b>...</b>}}
-
-finish_test
diff --git a/test/fts1d.test b/test/fts1d.test
deleted file mode 100644
index ea2303489c..0000000000
--- a/test/fts1d.test
+++ /dev/null
@@ -1,65 +0,0 @@
-# 2006 October 1
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS1 module, and in particular
-# the Porter stemmer.
-#
-# $Id: fts1d.test,v 1.1 2006/10/01 18:41:21 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-do_test fts1d-1.1 {
-  execsql {
-    CREATE VIRTUAL TABLE t1 USING fts1(content, tokenize porter);
-    INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping');
-    SELECT rowid FROM t1 WHERE content MATCH 'run jump';
-  }
-} {1}
-do_test fts1d-1.2 {
-  execsql {
-    SELECT snippet(t1) FROM t1 WHERE t1 MATCH 'run jump';
-  }
-} {{<b>running</b> and <b>jumping</b>}}
-do_test fts1d-1.3 {
-  execsql {
-    INSERT INTO t1(rowid, content) 
-          VALUES(2, 'abcdefghijklmnopqrstuvwyxz');
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH 'abcdefghijqrstuvwyxz'
-  }
-} {2 <b>abcdefghijklmnopqrstuvwyxz</b>}
-do_test fts1d-1.4 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH 'abcdefghijXXXXqrstuvwyxz'
-  }
-} {2 <b>abcdefghijklmnopqrstuvwyxz</b>}
-do_test fts1d-1.5 {
-  execsql {
-    INSERT INTO t1(rowid, content) 
-          VALUES(3, 'The value is 123456789');
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH '123789'
-  }
-} {3 {The value is <b>123456789</b>}}
-do_test fts1d-1.6 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH '123000000789'
-  }
-} {3 {The value is <b>123456789</b>}}
-
-
-finish_test
diff --git a/test/fts1e.test b/test/fts1e.test
deleted file mode 100644
index 479cfac91d..0000000000
--- a/test/fts1e.test
+++ /dev/null
@@ -1,85 +0,0 @@
-# 2006 October 19
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing deletions in the FTS1 module.
-#
-# $Id: fts1e.test,v 1.1 2006/10/19 23:28:35 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Construct a full-text search table containing keywords which are the
-# ordinal numbers of the bit positions set for a sequence of integers,
-# which are used for the rowid.  There are a total of 30 INSERT and
-# DELETE statements, so that we'll test both the segmentMerge() merge
-# (over the first 16) and the termSelect() merge (over the level-1
-# segment and 14 level-0 segments).
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, 'one');
-  INSERT INTO t1 (rowid, content) VALUES(2, 'two');
-  INSERT INTO t1 (rowid, content) VALUES(3, 'one two');
-  INSERT INTO t1 (rowid, content) VALUES(4, 'three');
-  DELETE FROM t1 WHERE rowid = 1;
-  INSERT INTO t1 (rowid, content) VALUES(5, 'one three');
-  INSERT INTO t1 (rowid, content) VALUES(6, 'two three');
-  INSERT INTO t1 (rowid, content) VALUES(7, 'one two three');
-  DELETE FROM t1 WHERE rowid = 4;
-  INSERT INTO t1 (rowid, content) VALUES(8, 'four');
-  INSERT INTO t1 (rowid, content) VALUES(9, 'one four');
-  INSERT INTO t1 (rowid, content) VALUES(10, 'two four');
-  DELETE FROM t1 WHERE rowid = 7;
-  INSERT INTO t1 (rowid, content) VALUES(11, 'one two four');
-  INSERT INTO t1 (rowid, content) VALUES(12, 'three four');
-  INSERT INTO t1 (rowid, content) VALUES(13, 'one three four');
-  DELETE FROM t1 WHERE rowid = 10;
-  INSERT INTO t1 (rowid, content) VALUES(14, 'two three four');
-  INSERT INTO t1 (rowid, content) VALUES(15, 'one two three four');
-  INSERT INTO t1 (rowid, content) VALUES(16, 'five');
-  DELETE FROM t1 WHERE rowid = 13;
-  INSERT INTO t1 (rowid, content) VALUES(17, 'one five');
-  INSERT INTO t1 (rowid, content) VALUES(18, 'two five');
-  INSERT INTO t1 (rowid, content) VALUES(19, 'one two five');
-  DELETE FROM t1 WHERE rowid = 16;
-  INSERT INTO t1 (rowid, content) VALUES(20, 'three five');
-  INSERT INTO t1 (rowid, content) VALUES(21, 'one three five');
-  INSERT INTO t1 (rowid, content) VALUES(22, 'two three five');
-  DELETE FROM t1 WHERE rowid = 19;
-  DELETE FROM t1 WHERE rowid = 22;
-}
-
-do_test fts1f-1.1 {
-  execsql {SELECT COUNT(*) FROM t1}
-} {14}
-
-do_test fts1e-2.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {3 5 9 11 15 17 21}
-
-do_test fts1e-2.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two'}
-} {2 3 6 11 14 15 18}
-
-do_test fts1e-2.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three'}
-} {5 6 12 14 15 20 21}
-
-do_test fts1e-2.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'four'}
-} {8 9 11 12 14 15}
-
-do_test fts1e-2.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five'}
-} {17 18 20 21}
-
-finish_test
diff --git a/test/fts1f.test b/test/fts1f.test
deleted file mode 100644
index 19dea0a329..0000000000
--- a/test/fts1f.test
+++ /dev/null
@@ -1,90 +0,0 @@
-# 2006 October 19
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing updates in the FTS1 module.
-#
-# $Id: fts1f.test,v 1.2 2007/02/23 00:14:06 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Construct a full-text search table containing keywords which are the
-# ordinal numbers of the bit positions set for a sequence of integers,
-# which are used for the rowid.  There are a total of 31 INSERT,
-# UPDATE, and DELETE statements, so that we'll test both the
-# segmentMerge() merge (over the first 16) and the termSelect() merge
-# (over the level-1 segment and 15 level-0 segments).
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, 'one');
-  INSERT INTO t1 (rowid, content) VALUES(2, 'two');
-  INSERT INTO t1 (rowid, content) VALUES(3, 'one two');
-  INSERT INTO t1 (rowid, content) VALUES(4, 'three');
-  INSERT INTO t1 (rowid, content) VALUES(5, 'one three');
-  INSERT INTO t1 (rowid, content) VALUES(6, 'two three');
-  INSERT INTO t1 (rowid, content) VALUES(7, 'one two three');
-  DELETE FROM t1 WHERE rowid = 4;
-  INSERT INTO t1 (rowid, content) VALUES(8, 'four');
-  UPDATE t1 SET content = 'update one three' WHERE rowid = 1;
-  INSERT INTO t1 (rowid, content) VALUES(9, 'one four');
-  INSERT INTO t1 (rowid, content) VALUES(10, 'two four');
-  DELETE FROM t1 WHERE rowid = 7;
-  INSERT INTO t1 (rowid, content) VALUES(11, 'one two four');
-  INSERT INTO t1 (rowid, content) VALUES(12, 'three four');
-  INSERT INTO t1 (rowid, content) VALUES(13, 'one three four');
-  DELETE FROM t1 WHERE rowid = 10;
-  INSERT INTO t1 (rowid, content) VALUES(14, 'two three four');
-  INSERT INTO t1 (rowid, content) VALUES(15, 'one two three four');
-  UPDATE t1 SET content = 'update two five' WHERE rowid = 8;
-  INSERT INTO t1 (rowid, content) VALUES(16, 'five');
-  DELETE FROM t1 WHERE rowid = 13;
-  INSERT INTO t1 (rowid, content) VALUES(17, 'one five');
-  INSERT INTO t1 (rowid, content) VALUES(18, 'two five');
-  INSERT INTO t1 (rowid, content) VALUES(19, 'one two five');
-  DELETE FROM t1 WHERE rowid = 16;
-  INSERT INTO t1 (rowid, content) VALUES(20, 'three five');
-  INSERT INTO t1 (rowid, content) VALUES(21, 'one three five');
-  INSERT INTO t1 (rowid, content) VALUES(22, 'two three five');
-  DELETE FROM t1 WHERE rowid = 19;
-  UPDATE t1 SET content = 'update' WHERE rowid = 15;
-}
-
-do_test fts1f-1.1 {
-  execsql {SELECT COUNT(*) FROM t1}
-} {16}
-
-do_test fts1f-2.0 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'update'}
-} {1 8 15}
-
-do_test fts1f-2.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {1 3 5 9 11 17 21}
-
-do_test fts1f-2.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two'}
-} {2 3 6 8 11 14 18 22}
-
-do_test fts1f-2.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three'}
-} {1 5 6 12 14 20 21 22}
-
-do_test fts1f-2.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'four'}
-} {9 11 12 14}
-
-do_test fts1f-2.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five'}
-} {8 17 18 20 21 22}
-
-finish_test
diff --git a/test/fts1i.test b/test/fts1i.test
deleted file mode 100644
index 803b93bf29..0000000000
--- a/test/fts1i.test
+++ /dev/null
@@ -1,88 +0,0 @@
-# 2007 January 17
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite fts1 library.  The
-# focus here is testing handling of UPDATE when using UTF-16-encoded
-# databases.
-#
-# $Id: fts1i.test,v 1.2 2007/01/24 03:43:20 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-
-# Return the UTF-16 representation of the supplied UTF-8 string $str.
-# If $nt is true, append two 0x00 bytes as a nul terminator.
-# NOTE(shess) Copied from capi3.test.
-proc utf16 {str {nt 1}} {
-  set r [encoding convertto unicode $str]
-  if {$nt} {
-    append r "\x00\x00"
-  }
-  return $r
-}
-
-db eval {
-  PRAGMA encoding = "UTF-16le";
-  CREATE VIRTUAL TABLE t1 USING fts1(content);
-}
-
-do_test fts1i-1.0 {
-  execsql {PRAGMA encoding}
-} {UTF-16le}
-
-do_test fts1i-1.1 {
-  execsql {INSERT INTO t1 (rowid, content) VALUES(1, 'one')}
-  execsql {SELECT content FROM t1 WHERE rowid = 1}
-} {one}
-
-do_test fts1i-1.2 {
-  set sql "INSERT INTO t1 (rowid, content) VALUES(2, 'two')"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 2}
-} {two}
-
-do_test fts1i-1.3 {
-  set sql "INSERT INTO t1 (rowid, content) VALUES(3, 'three')"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  set sql "UPDATE t1 SET content = 'trois' WHERE rowid = 3"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 3}
-} {trois}
-
-do_test fts1i-1.4 {
-  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(4, 'four')}]
-  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 4}
-} {four}
-
-do_test fts1i-1.5 {
-  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(5, 'five')}]
-  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  set sql "UPDATE t1 SET content = 'cinq' WHERE rowid = 5"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 5}
-} {cinq}
-
-finish_test
diff --git a/test/fts1j.test b/test/fts1j.test
deleted file mode 100644
index 4dac22abbf..0000000000
--- a/test/fts1j.test
+++ /dev/null
@@ -1,89 +0,0 @@
-# 2007 February 6
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  This
-# tests creating fts1 tables in an attached database.
-#
-# $Id: fts1j.test,v 1.1 2007/02/07 01:01:18 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Clean up anything left over from a previous pass.
-forcedelete test2.db
-forcedelete test2.db-journal
-sqlite3 db2 test2.db
-
-db eval {
-  CREATE VIRTUAL TABLE t3 USING fts1(content);
-  INSERT INTO t3 (rowid, content) VALUES(1, "hello world");
-}
-
-db2 eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, "hello world");
-  INSERT INTO t1 (rowid, content) VALUES(2, "hello there");
-  INSERT INTO t1 (rowid, content) VALUES(3, "cruel world");
-}
-
-# This has always worked because the t1_* tables used by fts1 will be
-# the defaults.
-do_test fts1j-1.1 {
-  execsql {
-    ATTACH DATABASE 'test2.db' AS two;
-    SELECT rowid FROM t1 WHERE t1 MATCH 'hello';
-    DETACH DATABASE two;
-  }
-} {1 2}
-# Make certain we're detached if there was an error.
-catch {db eval {DETACH DATABASE two}}
-
-# In older code, this appears to work fine, but the t2_* tables used
-# by fts1 will be created in database 'main' instead of database
-# 'two'.  It appears to work fine because the tables end up being the
-# defaults, but obviously is badly broken if you hope to use things
-# other than in the exact same ATTACH setup.
-do_test fts1j-1.2 {
-  execsql {
-    ATTACH DATABASE 'test2.db' AS two;
-    CREATE VIRTUAL TABLE two.t2 USING fts1(content);
-    INSERT INTO t2 (rowid, content) VALUES(1, "hello world");
-    INSERT INTO t2 (rowid, content) VALUES(2, "hello there");
-    INSERT INTO t2 (rowid, content) VALUES(3, "cruel world");
-    SELECT rowid FROM t2 WHERE t2 MATCH 'hello';
-    DETACH DATABASE two;
-  }
-} {1 2}
-catch {db eval {DETACH DATABASE two}}
-
-# In older code, this broke because the fts1 code attempted to create
-# t3_* tables in database 'main', but they already existed.  Normally
-# this wouldn't happen without t3 itself existing, in which case the
-# fts1 code would never be called in the first place.
-do_test fts1j-1.3 {
-  execsql {
-    ATTACH DATABASE 'test2.db' AS two;
-
-    CREATE VIRTUAL TABLE two.t3 USING fts1(content);
-    INSERT INTO two.t3 (rowid, content) VALUES(2, "hello there");
-    INSERT INTO two.t3 (rowid, content) VALUES(3, "cruel world");
-    SELECT rowid FROM two.t3 WHERE t3 MATCH 'hello';
-
-    DETACH DATABASE two;
-  } db2
-} {2}
-catch {db eval {DETACH DATABASE two}}
-
-catch {db2 close}
-forcedelete test2.db
-
-finish_test
diff --git a/test/fts1k.test b/test/fts1k.test
deleted file mode 100644
index 35b94d2ae7..0000000000
--- a/test/fts1k.test
+++ /dev/null
@@ -1,69 +0,0 @@
-# 2007 March 28
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing isspace/isalnum/tolower problems with the
-# FTS1 module.  Unfortunately, this code isn't a really principled set
-# of tests, because it is impossible to know where new uses of these
-# functions might appear.
-#
-# $Id: fts1k.test,v 1.2 2007/12/13 21:54:11 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Tests that startsWith() (calls isspace, tolower, isalnum) can handle
-# hi-bit chars.  parseSpec() also calls isalnum here.
-do_test fts1k-1.1 {
-  execsql "CREATE VIRTUAL TABLE t1 USING fts1(content, \x80)"
-} {}
-
-# Additionally tests isspace() call in getToken(), and isalnum() call
-# in tokenListToIdList().
-do_test fts1k-1.2 {
-  catch {
-    execsql "CREATE VIRTUAL TABLE t2 USING fts1(content, tokenize \x80)"
-  }
-  sqlite3_errmsg $DB
-} "unknown tokenizer: \x80"
-
-# Additionally test final isalnum() in startsWith().
-do_test fts1k-1.3 {
-  execsql "CREATE VIRTUAL TABLE t3 USING fts1(content, tokenize\x80)"
-} {}
-
-# The snippet-generation code has calls to isspace() which are sort of
-# hard to get to.  It finds convenient breakpoints by starting ~40
-# chars before and after the matched term, and scanning ~10 chars
-# around that position for isspace() characters.  The long word with
-# embedded hi-bit chars causes one of these isspace() calls to be
-# exercised.  The version with a couple extra spaces should cause the
-# other isspace() call to be exercised.  [Both cases have been tested
-# in the debugger, but I'm hoping to continue to catch it if simple
-# constant changes change things slightly.
-#
-# The trailing and leading hi-bit chars help with code which tests for
-# isspace() to coalesce multiple spaces.
-
-set word "\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80"
-set phrase1 "$word $word $word target $word $word $word"
-set phrase2 "$word $word $word    target    $word $word $word"
-
-db eval {CREATE VIRTUAL TABLE t4 USING fts1(content)}
-db eval "INSERT INTO t4 (content) VALUES ('$phrase1')"
-db eval "INSERT INTO t4 (content) VALUES ('$phrase2')"
-
-do_test fts1k-1.4 {
-  execsql {SELECT rowid, length(snippet(t4)) FROM t4 WHERE t4 MATCH 'target'}
-} {1 111 2 117}
-
-finish_test
diff --git a/test/fts1l.test b/test/fts1l.test
deleted file mode 100644
index 924be33801..0000000000
--- a/test/fts1l.test
+++ /dev/null
@@ -1,65 +0,0 @@
-# 2007 April 9
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  fts1
-# DELETE handling assumed all fields were non-null.  This was not
-# the intention at all.
-#
-# $Id: fts1l.test,v 1.1 2007/04/09 20:45:42 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(col_a, col_b);
-
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(1, 'testing', 'testing');
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(2, 'only a', null);
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(3, null, 'only b');
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(4, null, null);
-}
-
-do_test fts1m-1.0 {
-  execsql {
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {2 2 4}
-
-do_test fts1m-1.1 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 1;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {1 1 3}
-
-do_test fts1m-1.2 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 2;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {0 1 2}
-
-do_test fts1m-1.3 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 3;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {0 0 1}
-
-do_test fts1m-1.4 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 4;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {0 0 0}
-
-finish_test
diff --git a/test/fts1m.test b/test/fts1m.test
deleted file mode 100644
index c2f8f915a1..0000000000
--- a/test/fts1m.test
+++ /dev/null
@@ -1,50 +0,0 @@
-# 2007 July 27
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing the FTS1 module, specifically snippet
-# generation.  Extracted from fts2o.test.
-#
-# $Id: fts1m.test,v 1.1 2007/07/25 00:25:20 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is not defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-#---------------------------------------------------------------------
-# These tests, fts1m-1.*, test that ticket #2429 is fixed.
-#
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(a, b, c);
-  INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two');
-}
-do_test fts1m-1.1 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE c MATCH 'four';
-  }
-} {1 {one <b>four</b> two}}
-do_test fts1m-1.2 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE b MATCH 'four';
-  }
-} {1 {one <b>four</b>}}
-do_test fts1m-1.3 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE a MATCH 'four';
-  }
-} {1 {one three <b>four</b>}}
-
-finish_test
diff --git a/test/fts1n.test b/test/fts1n.test
deleted file mode 100644
index 2f102b4e2c..0000000000
--- a/test/fts1n.test
+++ /dev/null
@@ -1,45 +0,0 @@
-# 2007 July 24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing the FTS1 module for errors in the handling
-# of SQLITE_SCHEMA.
-#
-# $Id: fts1n.test,v 1.1 2007/07/25 00:38:06 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is not defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-do_test fts1m-1.1 {
-  execsql {
-    CREATE VIRTUAL TABLE t1 USING fts1(a, b, c);
-    INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one two');
-    SELECT a, b, c FROM t1 WHERE c MATCH 'two';
-  }
-} {{one three four} {one four} {one two}}
-
-# This test was crashing at one point.
-#
-do_test fts1m-1.2 {
-  execsql {
-    SELECT a, b, c FROM t1 WHERE c MATCH 'two';
-    CREATE TABLE t3(a, b, c);
-    SELECT a, b, c FROM t1 WHERE  c  MATCH 'two';
-  }
-} {{one three four} {one four} {one two} {one three four} {one four} {one two}}
-
-finish_test
diff --git a/test/fts1o.test b/test/fts1o.test
deleted file mode 100644
index 3ed095294d..0000000000
--- a/test/fts1o.test
+++ /dev/null
@@ -1,138 +0,0 @@
-# 2007 July 24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing the FTS1 module rename functionality.  Mostly
-# copied from fts2o.test.
-#
-# $Id: fts1o.test,v 1.2 2007/08/30 20:01:33 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is not defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(a, b, c);
-  INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two');
-}
-
-#---------------------------------------------------------------------
-# Test that it is possible to rename an fts1 table.
-#
-do_test fts1o-1.1 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {t1 t1_content t1_term}
-do_test fts1o-1.2 {
-  execsql { ALTER TABLE t1 RENAME to fts_t1; }
-} {}
-do_test fts1o-1.3 {
-  execsql { SELECT rowid, snippet(fts_t1) FROM fts_t1 WHERE a MATCH 'four'; }
-} {1 {one three <b>four</b>}}
-do_test fts1o-1.4 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {fts_t1 fts_t1_content fts_t1_term}
-
-# See what happens when renaming the fts1 table fails.
-#
-do_test fts1o-2.1 {
-  catchsql {
-    CREATE TABLE t1_term(a, b, c);
-    ALTER TABLE fts_t1 RENAME to t1;
-  }
-} {1 {SQL logic error}}
-do_test fts1o-2.2 {
-  execsql { SELECT rowid, snippet(fts_t1) FROM fts_t1 WHERE a MATCH 'four'; }
-} {1 {one three <b>four</b>}}
-do_test fts1o-2.3 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {fts_t1 fts_t1_content fts_t1_term t1_term}
-
-# See what happens when renaming the fts1 table fails inside a transaction.
-#
-do_test fts1o-3.1 {
-  execsql {
-    BEGIN;
-    INSERT INTO fts_t1(a, b, c) VALUES('one two three', 'one four', 'one two');
-  }
-} {}
-do_test fts1o-3.2 {
-  catchsql {
-    ALTER TABLE fts_t1 RENAME to t1;
-  }
-} {1 {SQL logic error}}
-# NOTE(shess) rowid AS rowid to defeat caching.  Otherwise, this
-# seg-faults, I suspect that there's something up with a stale
-# virtual-table reference, but I'm not quite sure how it happens here
-# but not for fts2o.test.
-do_test fts1o-3.3 {
-  execsql { SELECT rowid AS rowid, snippet(fts_t1) FROM fts_t1 WHERE a MATCH 'four'; }
-} {1 {one three <b>four</b>}}
-do_test fts1o-3.4 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {fts_t1 fts_t1_content fts_t1_term t1_term}
-do_test fts1o-3.5 {
-  execsql COMMIT
-  execsql {SELECT a FROM fts_t1}
-} {{one three four} {one two three}}
-do_test fts1o-3.6 {
-  execsql { SELECT a, b, c FROM fts_t1 WHERE c MATCH 'four'; }
-} {{one three four} {one four} {one four two}}
-
-#---------------------------------------------------------------------
-# Test that it is possible to rename an fts1 table in an attached 
-# database.
-#
-forcedelete test2.db test2.db-journal
-
-do_test fts1o-4.1 {
-  execsql {
-    DROP TABLE t1_term;
-    ALTER TABLE fts_t1 RENAME to t1;
-    SELECT a, b, c FROM t1 WHERE c MATCH 'two';
-  }
-} {{one three four} {one four} {one four two} {one two three} {one four} {one two}}
-
-do_test fts1o-4.2 {
-  execsql {
-    ATTACH 'test2.db' AS aux;
-    CREATE VIRTUAL TABLE aux.t1 USING fts1(a, b, c);
-    INSERT INTO aux.t1(a, b, c) VALUES(
-      'neung song sahm', 'neung see', 'neung see song'
-    );
-  }
-} {}
-
-do_test fts1o-4.3 {
-  execsql { SELECT a, b, c FROM aux.t1 WHERE a MATCH 'song'; }
-} {{neung song sahm} {neung see} {neung see song}}
-
-do_test fts1o-4.4 {
-  execsql { SELECT a, b, c FROM t1 WHERE c MATCH 'two'; }
-} {{one three four} {one four} {one four two} {one two three} {one four} {one two}}
-
-do_test fts1o-4.5 {
-  execsql { ALTER TABLE aux.t1 RENAME TO t2 }
-} {}
-
-do_test fts1o-4.6 {
-  execsql { SELECT a, b, c FROM t2 WHERE a MATCH 'song'; }
-} {{neung song sahm} {neung see} {neung see song}}
-
-do_test fts1o-4.7 {
-  execsql { SELECT a, b, c FROM t1 WHERE c MATCH 'two'; }
-} {{one three four} {one four} {one four two} {one two three} {one four} {one two}}
-
-finish_test
diff --git a/test/fts1porter.test b/test/fts1porter.test
deleted file mode 100644
index 0ca87a01ed..0000000000
--- a/test/fts1porter.test
+++ /dev/null
@@ -1,23590 +0,0 @@
-# 2006 October 1
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS1 module, and in particular
-# the Porter stemmer.
-#
-# $Id: fts1porter.test,v 1.5 2006/10/03 19:37:37 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS1 is defined, omit this file.
-ifcapable !fts1 {
-  finish_test
-  return
-}
-
-# Test data for the Porter stemmer.  The first word of each line
-# is the input.  The second word is the desired output.
-#
-# This test data is taken from http://www.tartarus.org/martin/PorterStemmer/
-# There is no claim of copyright made on that page, but you should
-# probably contact the author (Martin Porter - the inventor of the
-# Porter Stemmer algorithm) if you want to use this test data in a
-# commerical product of some kind.  The stemmer code in FTS1 is a
-# complete rewrite from scratch based on the algorithm specification
-# and does not contain any code under copyright.
-#
-set porter_test_data {
-    a                    a
-    aaron                aaron
-    abaissiez            abaissiez
-    abandon              abandon
-    abandoned            abandon
-    abase                abas
-    abash                abash
-    abate                abat
-    abated               abat
-    abatement            abat
-    abatements           abat
-    abates               abat
-    abbess               abbess
-    abbey                abbei
-    abbeys               abbei
-    abbominable          abbomin
-    abbot                abbot
-    abbots               abbot
-    abbreviated          abbrevi
-    abed                 ab
-    abel                 abel
-    aberga               aberga
-    abergavenny          abergavenni
-    abet                 abet
-    abetting             abet
-    abhominable          abhomin
-    abhor                abhor
-    abhorr               abhorr
-    abhorred             abhor
-    abhorring            abhor
-    abhors               abhor
-    abhorson             abhorson
-    abide                abid
-    abides               abid
-    abilities            abil
-    ability              abil
-    abject               abject
-    abjectly             abjectli
-    abjects              abject
-    abjur                abjur
-    abjure               abjur
-    able                 abl
-    abler                abler
-    aboard               aboard
-    abode                abod
-    aboded               abod
-    abodements           abod
-    aboding              abod
-    abominable           abomin
-    abominably           abomin
-    abominations         abomin
-    abortive             abort
-    abortives            abort
-    abound               abound
-    abounding            abound
-    about                about
-    above                abov
-    abr                  abr
-    abraham              abraham
-    abram                abram
-    abreast              abreast
-    abridg               abridg
-    abridge              abridg
-    abridged             abridg
-    abridgment           abridg
-    abroach              abroach
-    abroad               abroad
-    abrogate             abrog
-    abrook               abrook
-    abrupt               abrupt
-    abruption            abrupt
-    abruptly             abruptli
-    absence              absenc
-    absent               absent
-    absey                absei
-    absolute             absolut
-    absolutely           absolut
-    absolv               absolv
-    absolver             absolv
-    abstains             abstain
-    abstemious           abstemi
-    abstinence           abstin
-    abstract             abstract
-    absurd               absurd
-    absyrtus             absyrtu
-    abundance            abund
-    abundant             abund
-    abundantly           abundantli
-    abus                 abu
-    abuse                abus
-    abused               abus
-    abuser               abus
-    abuses               abus
-    abusing              abus
-    abutting             abut
-    aby                  abi
-    abysm                abysm
-    ac                   ac
-    academe              academ
-    academes             academ
-    accent               accent
-    accents              accent
-    accept               accept
-    acceptable           accept
-    acceptance           accept
-    accepted             accept
-    accepts              accept
-    access               access
-    accessary            accessari
-    accessible           access
-    accidence            accid
-    accident             accid
-    accidental           accident
-    accidentally         accident
-    accidents            accid
-    accite               accit
-    accited              accit
-    accites              accit
-    acclamations         acclam
-    accommodate          accommod
-    accommodated         accommod
-    accommodation        accommod
-    accommodations       accommod
-    accommodo            accommodo
-    accompanied          accompani
-    accompany            accompani
-    accompanying         accompani
-    accomplices          accomplic
-    accomplish           accomplish
-    accomplished         accomplish
-    accomplishing        accomplish
-    accomplishment       accomplish
-    accompt              accompt
-    accord               accord
-    accordant            accord
-    accorded             accord
-    accordeth            accordeth
-    according            accord
-    accordingly          accordingli
-    accords              accord
-    accost               accost
-    accosted             accost
-    account              account
-    accountant           account
-    accounted            account
-    accounts             account
-    accoutred            accoutr
-    accoutrement         accoutr
-    accoutrements        accoutr
-    accrue               accru
-    accumulate           accumul
-    accumulated          accumul
-    accumulation         accumul
-    accurs               accur
-    accursed             accurs
-    accurst              accurst
-    accus                accu
-    accusation           accus
-    accusations          accus
-    accusative           accus
-    accusativo           accusativo
-    accuse               accus
-    accused              accus
-    accuser              accus
-    accusers             accus
-    accuses              accus
-    accuseth             accuseth
-    accusing             accus
-    accustom             accustom
-    accustomed           accustom
-    ace                  ac
-    acerb                acerb
-    ache                 ach
-    acheron              acheron
-    aches                ach
-    achiev               achiev
-    achieve              achiev
-    achieved             achiev
-    achievement          achiev
-    achievements         achiev
-    achiever             achiev
-    achieves             achiev
-    achieving            achiev
-    achilles             achil
-    aching               ach
-    achitophel           achitophel
-    acknowledg           acknowledg
-    acknowledge          acknowledg
-    acknowledged         acknowledg
-    acknowledgment       acknowledg
-    acknown              acknown
-    acold                acold
-    aconitum             aconitum
-    acordo               acordo
-    acorn                acorn
-    acquaint             acquaint
-    acquaintance         acquaint
-    acquainted           acquaint
-    acquaints            acquaint
-    acquir               acquir
-    acquire              acquir
-    acquisition          acquisit
-    acquit               acquit
-    acquittance          acquitt
-    acquittances         acquitt
-    acquitted            acquit
-    acre                 acr
-    acres                acr
-    across               across
-    act                  act
-    actaeon              actaeon
-    acted                act
-    acting               act
-    action               action
-    actions              action
-    actium               actium
-    active               activ
-    actively             activ
-    activity             activ
-    actor                actor
-    actors               actor
-    acts                 act
-    actual               actual
-    acture               actur
-    acute                acut
-    acutely              acut
-    ad                   ad
-    adage                adag
-    adallas              adalla
-    adam                 adam
-    adamant              adam
-    add                  add
-    added                ad
-    adder                adder
-    adders               adder
-    addeth               addeth
-    addict               addict
-    addicted             addict
-    addiction            addict
-    adding               ad
-    addition             addit
-    additions            addit
-    addle                addl
-    address              address
-    addressing           address
-    addrest              addrest
-    adds                 add
-    adhere               adher
-    adheres              adher
-    adieu                adieu
-    adieus               adieu
-    adjacent             adjac
-    adjoin               adjoin
-    adjoining            adjoin
-    adjourn              adjourn
-    adjudg               adjudg
-    adjudged             adjudg
-    adjunct              adjunct
-    administer           administ
-    administration       administr
-    admir                admir
-    admirable            admir
-    admiral              admir
-    admiration           admir
-    admire               admir
-    admired              admir
-    admirer              admir
-    admiring             admir
-    admiringly           admiringli
-    admission            admiss
-    admit                admit
-    admits               admit
-    admittance           admitt
-    admitted             admit
-    admitting            admit
-    admonish             admonish
-    admonishing          admonish
-    admonishment         admonish
-    admonishments        admonish
-    admonition           admonit
-    ado                  ado
-    adonis               adoni
-    adopt                adopt
-    adopted              adopt
-    adoptedly            adoptedli
-    adoption             adopt
-    adoptious            adopti
-    adopts               adopt
-    ador                 ador
-    adoration            ador
-    adorations           ador
-    adore                ador
-    adorer               ador
-    adores               ador
-    adorest              adorest
-    adoreth              adoreth
-    adoring              ador
-    adorn                adorn
-    adorned              adorn
-    adornings            adorn
-    adornment            adorn
-    adorns               adorn
-    adown                adown
-    adramadio            adramadio
-    adrian               adrian
-    adriana              adriana
-    adriano              adriano
-    adriatic             adriat
-    adsum                adsum
-    adulation            adul
-    adulterate           adulter
-    adulterates          adulter
-    adulterers           adulter
-    adulteress           adulteress
-    adulteries           adulteri
-    adulterous           adulter
-    adultery             adulteri
-    adultress            adultress
-    advanc               advanc
-    advance              advanc
-    advanced             advanc
-    advancement          advanc
-    advancements         advanc
-    advances             advanc
-    advancing            advanc
-    advantage            advantag
-    advantageable        advantag
-    advantaged           advantag
-    advantageous         advantag
-    advantages           advantag
-    advantaging          advantag
-    advent               advent
-    adventur             adventur
-    adventure            adventur
-    adventures           adventur
-    adventuring          adventur
-    adventurous          adventur
-    adventurously        adventur
-    adversaries          adversari
-    adversary            adversari
-    adverse              advers
-    adversely            advers
-    adversities          advers
-    adversity            advers
-    advertis             adverti
-    advertise            advertis
-    advertised           advertis
-    advertisement        advertis
-    advertising          advertis
-    advice               advic
-    advis                advi
-    advise               advis
-    advised              advis
-    advisedly            advisedli
-    advises              advis
-    advisings            advis
-    advocate             advoc
-    advocation           advoc
-    aeacida              aeacida
-    aeacides             aeacid
-    aedile               aedil
-    aediles              aedil
-    aegeon               aegeon
-    aegion               aegion
-    aegles               aegl
-    aemelia              aemelia
-    aemilia              aemilia
-    aemilius             aemiliu
-    aeneas               aenea
-    aeolus               aeolu
-    aer                  aer
-    aerial               aerial
-    aery                 aeri
-    aesculapius          aesculapiu
-    aeson                aeson
-    aesop                aesop
-    aetna                aetna
-    afar                 afar
-    afear                afear
-    afeard               afeard
-    affability           affabl
-    affable              affabl
-    affair               affair
-    affaire              affair
-    affairs              affair
-    affect               affect
-    affectation          affect
-    affectations         affect
-    affected             affect
-    affectedly           affectedli
-    affecteth            affecteth
-    affecting            affect
-    affection            affect
-    affectionate         affection
-    affectionately       affection
-    affections           affect
-    affects              affect
-    affeer               affeer
-    affianc              affianc
-    affiance             affianc
-    affianced            affianc
-    affied               affi
-    affin                affin
-    affined              affin
-    affinity             affin
-    affirm               affirm
-    affirmation          affirm
-    affirmatives         affirm
-    afflict              afflict
-    afflicted            afflict
-    affliction           afflict
-    afflictions          afflict
-    afflicts             afflict
-    afford               afford
-    affordeth            affordeth
-    affords              afford
-    affray               affrai
-    affright             affright
-    affrighted           affright
-    affrights            affright
-    affront              affront
-    affronted            affront
-    affy                 affi
-    afield               afield
-    afire                afir
-    afloat               afloat
-    afoot                afoot
-    afore                afor
-    aforehand            aforehand
-    aforesaid            aforesaid
-    afraid               afraid
-    afresh               afresh
-    afric                afric
-    africa               africa
-    african              african
-    afront               afront
-    after                after
-    afternoon            afternoon
-    afterward            afterward
-    afterwards           afterward
-    ag                   ag
-    again                again
-    against              against
-    agamemmon            agamemmon
-    agamemnon            agamemnon
-    agate                agat
-    agaz                 agaz
-    age                  ag
-    aged                 ag
-    agenor               agenor
-    agent                agent
-    agents               agent
-    ages                 ag
-    aggravate            aggrav
-    aggrief              aggrief
-    agile                agil
-    agincourt            agincourt
-    agitation            agit
-    aglet                aglet
-    agnize               agniz
-    ago                  ago
-    agone                agon
-    agony                agoni
-    agree                agre
-    agreed               agre
-    agreeing             agre
-    agreement            agreement
-    agrees               agre
-    agrippa              agrippa
-    aground              aground
-    ague                 agu
-    aguecheek            aguecheek
-    agued                agu
-    agueface             aguefac
-    agues                agu
-    ah                   ah
-    aha                  aha
-    ahungry              ahungri
-    ai                   ai
-    aialvolio            aialvolio
-    aiaria               aiaria
-    aid                  aid
-    aidance              aidanc
-    aidant               aidant
-    aided                aid
-    aiding               aid
-    aidless              aidless
-    aids                 aid
-    ail                  ail
-    aim                  aim
-    aimed                aim
-    aimest               aimest
-    aiming               aim
-    aims                 aim
-    ainsi                ainsi
-    aio                  aio
-    air                  air
-    aired                air
-    airless              airless
-    airs                 air
-    airy                 airi
-    ajax                 ajax
-    akilling             akil
-    al                   al
-    alabaster            alabast
-    alack                alack
-    alacrity             alacr
-    alarbus              alarbu
-    alarm                alarm
-    alarms               alarm
-    alarum               alarum
-    alarums              alarum
-    alas                 ala
-    alb                  alb
-    alban                alban
-    albans               alban
-    albany               albani
-    albeit               albeit
-    albion               albion
-    alchemist            alchemist
-    alchemy              alchemi
-    alcibiades           alcibiad
-    alcides              alcid
-    alder                alder
-    alderman             alderman
-    aldermen             aldermen
-    ale                  al
-    alecto               alecto
-    alehouse             alehous
-    alehouses            alehous
-    alencon              alencon
-    alengon              alengon
-    aleppo               aleppo
-    ales                 al
-    alewife              alewif
-    alexander            alexand
-    alexanders           alexand
-    alexandria           alexandria
-    alexandrian          alexandrian
-    alexas               alexa
-    alias                alia
-    alice                alic
-    alien                alien
-    aliena               aliena
-    alight               alight
-    alighted             alight
-    alights              alight
-    aliis                alii
-    alike                alik
-    alisander            alisand
-    alive                aliv
-    all                  all
-    alla                 alla
-    allay                allai
-    allayed              allai
-    allaying             allai
-    allayment            allay
-    allayments           allay
-    allays               allai
-    allegation           alleg
-    allegations          alleg
-    allege               alleg
-    alleged              alleg
-    allegiance           allegi
-    allegiant            allegi
-    alley                allei
-    alleys               allei
-    allhallowmas         allhallowma
-    alliance             allianc
-    allicholy            allicholi
-    allied               alli
-    allies               alli
-    alligant             allig
-    alligator            allig
-    allons               allon
-    allot                allot
-    allots               allot
-    allotted             allot
-    allottery            allotteri
-    allow                allow
-    allowance            allow
-    allowed              allow
-    allowing             allow
-    allows               allow
-    allur                allur
-    allure               allur
-    allurement           allur
-    alluring             allur
-    allusion             allus
-    ally                 alli
-    allycholly           allycholli
-    almain               almain
-    almanac              almanac
-    almanack             almanack
-    almanacs             almanac
-    almighty             almighti
-    almond               almond
-    almost               almost
-    alms                 alm
-    almsman              almsman
-    aloes                alo
-    aloft                aloft
-    alone                alon
-    along                along
-    alonso               alonso
-    aloof                aloof
-    aloud                aloud
-    alphabet             alphabet
-    alphabetical         alphabet
-    alphonso             alphonso
-    alps                 alp
-    already              alreadi
-    also                 also
-    alt                  alt
-    altar                altar
-    altars               altar
-    alter                alter
-    alteration           alter
-    altered              alter
-    alters               alter
-    althaea              althaea
-    although             although
-    altitude             altitud
-    altogether           altogeth
-    alton                alton
-    alway                alwai
-    always               alwai
-    am                   am
-    amaimon              amaimon
-    amain                amain
-    amaking              amak
-    amamon               amamon
-    amaz                 amaz
-    amaze                amaz
-    amazed               amaz
-    amazedly             amazedli
-    amazedness           amazed
-    amazement            amaz
-    amazes               amaz
-    amazeth              amazeth
-    amazing              amaz
-    amazon               amazon
-    amazonian            amazonian
-    amazons              amazon
-    ambassador           ambassador
-    ambassadors          ambassador
-    amber                amber
-    ambiguides           ambiguid
-    ambiguities          ambigu
-    ambiguous            ambigu
-    ambition             ambit
-    ambitions            ambit
-    ambitious            ambiti
-    ambitiously          ambiti
-    amble                ambl
-    ambled               ambl
-    ambles               ambl
-    ambling              ambl
-    ambo                 ambo
-    ambuscadoes          ambuscado
-    ambush               ambush
-    amen                 amen
-    amend                amend
-    amended              amend
-    amendment            amend
-    amends               amend
-    amerce               amerc
-    america              america
-    ames                 am
-    amiable              amiabl
-    amid                 amid
-    amidst               amidst
-    amiens               amien
-    amis                 ami
-    amiss                amiss
-    amities              amiti
-    amity                amiti
-    amnipotent           amnipot
-    among                among
-    amongst              amongst
-    amorous              amor
-    amorously            amor
-    amort                amort
-    amount               amount
-    amounts              amount
-    amour                amour
-    amphimacus           amphimacu
-    ample                ampl
-    ampler               ampler
-    amplest              amplest
-    amplified            amplifi
-    amplify              amplifi
-    amply                ampli
-    ampthill             ampthil
-    amurath              amurath
-    amyntas              amynta
-    an                   an
-    anatomiz             anatomiz
-    anatomize            anatom
-    anatomy              anatomi
-    ancestor             ancestor
-    ancestors            ancestor
-    ancestry             ancestri
-    anchises             anchis
-    anchor               anchor
-    anchorage            anchorag
-    anchored             anchor
-    anchoring            anchor
-    anchors              anchor
-    anchovies            anchovi
-    ancient              ancient
-    ancientry            ancientri
-    ancients             ancient
-    ancus                ancu
-    and                  and
-    andirons             andiron
-    andpholus            andpholu
-    andren               andren
-    andrew               andrew
-    andromache           andromach
-    andronici            andronici
-    andronicus           andronicu
-    anew                 anew
-    ang                  ang
-    angel                angel
-    angelica             angelica
-    angelical            angel
-    angelo               angelo
-    angels               angel
-    anger                anger
-    angerly              angerli
-    angers               anger
-    anges                ang
-    angiers              angier
-    angl                 angl
-    anglais              anglai
-    angle                angl
-    angler               angler
-    angleterre           angleterr
-    angliae              anglia
-    angling              angl
-    anglish              anglish
-    angrily              angrili
-    angry                angri
-    anguish              anguish
-    angus                angu
-    animal               anim
-    animals              anim
-    animis               animi
-    anjou                anjou
-    ankle                ankl
-    anna                 anna
-    annals               annal
-    anne                 ann
-    annex                annex
-    annexed              annex
-    annexions            annexion
-    annexment            annex
-    annothanize          annothan
-    announces            announc
-    annoy                annoi
-    annoyance            annoy
-    annoying             annoi
-    annual               annual
-    anoint               anoint
-    anointed             anoint
-    anon                 anon
-    another              anoth
-    anselmo              anselmo
-    answer               answer
-    answerable           answer
-    answered             answer
-    answerest            answerest
-    answering            answer
-    answers              answer
-    ant                  ant
-    ante                 ant
-    antenor              antenor
-    antenorides          antenorid
-    anteroom             anteroom
-    anthem               anthem
-    anthems              anthem
-    anthony              anthoni
-    anthropophagi        anthropophagi
-    anthropophaginian    anthropophaginian
-    antiates             antiat
-    antic                antic
-    anticipate           anticip
-    anticipates          anticip
-    anticipatest         anticipatest
-    anticipating         anticip
-    anticipation         anticip
-    antick               antick
-    anticly              anticli
-    antics               antic
-    antidote             antidot
-    antidotes            antidot
-    antigonus            antigonu
-    antiopa              antiopa
-    antipathy            antipathi
-    antipholus           antipholu
-    antipholuses         antipholus
-    antipodes            antipod
-    antiquary            antiquari
-    antique              antiqu
-    antiquity            antiqu
-    antium               antium
-    antoniad             antoniad
-    antonio              antonio
-    antonius             antoniu
-    antony               antoni
-    antres               antr
-    anvil                anvil
-    any                  ani
-    anybody              anybodi
-    anyone               anyon
-    anything             anyth
-    anywhere             anywher
-    ap                   ap
-    apace                apac
-    apart                apart
-    apartment            apart
-    apartments           apart
-    ape                  ap
-    apemantus            apemantu
-    apennines            apennin
-    apes                 ap
-    apiece               apiec
-    apish                apish
-    apollinem            apollinem
-    apollo               apollo
-    apollodorus          apollodoru
-    apology              apolog
-    apoplex              apoplex
-    apoplexy             apoplexi
-    apostle              apostl
-    apostles             apostl
-    apostrophas          apostropha
-    apoth                apoth
-    apothecary           apothecari
-    appal                appal
-    appall               appal
-    appalled             appal
-    appals               appal
-    apparel              apparel
-    apparell             apparel
-    apparelled           apparel
-    apparent             appar
-    apparently           appar
-    apparition           apparit
-    apparitions          apparit
-    appeach              appeach
-    appeal               appeal
-    appeals              appeal
-    appear               appear
-    appearance           appear
-    appeared             appear
-    appeareth            appeareth
-    appearing            appear
-    appears              appear
-    appeas               appea
-    appease              appeas
-    appeased             appeas
-    appelant             appel
-    appele               appel
-    appelee              appele
-    appeles              appel
-    appelez              appelez
-    appellant            appel
-    appellants           appel
-    appelons             appelon
-    appendix             appendix
-    apperil              apperil
-    appertain            appertain
-    appertaining         appertain
-    appertainings        appertain
-    appertains           appertain
-    appertinent          appertin
-    appertinents         appertin
-    appetite             appetit
-    appetites            appetit
-    applaud              applaud
-    applauded            applaud
-    applauding           applaud
-    applause             applaus
-    applauses            applaus
-    apple                appl
-    apples               appl
-    appletart            appletart
-    appliance            applianc
-    appliances           applianc
-    applications         applic
-    applied              appli
-    applies              appli
-    apply                appli
-    applying             appli
-    appoint              appoint
-    appointed            appoint
-    appointment          appoint
-    appointments         appoint
-    appoints             appoint
-    apprehend            apprehend
-    apprehended          apprehend
-    apprehends           apprehend
-    apprehension         apprehens
-    apprehensions        apprehens
-    apprehensive         apprehens
-    apprendre            apprendr
-    apprenne             apprenn
-    apprenticehood       apprenticehood
-    appris               appri
-    approach             approach
-    approachers          approach
-    approaches           approach
-    approacheth          approacheth
-    approaching          approach
-    approbation          approb
-    approof              approof
-    appropriation        appropri
-    approv               approv
-    approve              approv
-    approved             approv
-    approvers            approv
-    approves             approv
-    appurtenance         appurten
-    appurtenances        appurten
-    apricocks            apricock
-    april                april
-    apron                apron
-    aprons               apron
-    apt                  apt
-    apter                apter
-    aptest               aptest
-    aptly                aptli
-    aptness              apt
-    aqua                 aqua
-    aquilon              aquilon
-    aquitaine            aquitain
-    arabia               arabia
-    arabian              arabian
-    araise               arais
-    arbitrate            arbitr
-    arbitrating          arbitr
-    arbitrator           arbitr
-    arbitrement          arbitr
-    arbors               arbor
-    arbour               arbour
-    arc                  arc
-    arch                 arch
-    archbishop           archbishop
-    archbishopric        archbishopr
-    archdeacon           archdeacon
-    arched               arch
-    archelaus            archelau
-    archer               archer
-    archers              archer
-    archery              archeri
-    archibald            archibald
-    archidamus           archidamu
-    architect            architect
-    arcu                 arcu
-    arde                 ard
-    arden                arden
-    ardent               ardent
-    ardour               ardour
-    are                  ar
-    argal                argal
-    argier               argier
-    argo                 argo
-    argosies             argosi
-    argosy               argosi
-    argu                 argu
-    argue                argu
-    argued               argu
-    argues               argu
-    arguing              argu
-    argument             argument
-    arguments            argument
-    argus                argu
-    ariachne             ariachn
-    ariadne              ariadn
-    ariel                ariel
-    aries                ari
-    aright               aright
-    arinado              arinado
-    arinies              arini
-    arion                arion
-    arise                aris
-    arises               aris
-    ariseth              ariseth
-    arising              aris
-    aristode             aristod
-    aristotle            aristotl
-    arithmetic           arithmet
-    arithmetician        arithmetician
-    ark                  ark
-    arm                  arm
-    arma                 arma
-    armado               armado
-    armadoes             armado
-    armagnac             armagnac
-    arme                 arm
-    armed                arm
-    armenia              armenia
-    armies               armi
-    armigero             armigero
-    arming               arm
-    armipotent           armipot
-    armor                armor
-    armour               armour
-    armourer             armour
-    armourers            armour
-    armours              armour
-    armoury              armouri
-    arms                 arm
-    army                 armi
-    arn                  arn
-    aroint               aroint
-    arose                aros
-    arouse               arous
-    aroused              arous
-    arragon              arragon
-    arraign              arraign
-    arraigned            arraign
-    arraigning           arraign
-    arraignment          arraign
-    arrant               arrant
-    arras                arra
-    array                arrai
-    arrearages           arrearag
-    arrest               arrest
-    arrested             arrest
-    arrests              arrest
-    arriv                arriv
-    arrival              arriv
-    arrivance            arriv
-    arrive               arriv
-    arrived              arriv
-    arrives              arriv
-    arriving             arriv
-    arrogance            arrog
-    arrogancy            arrog
-    arrogant             arrog
-    arrow                arrow
-    arrows               arrow
-    art                  art
-    artemidorus          artemidoru
-    arteries             arteri
-    arthur               arthur
-    article              articl
-    articles             articl
-    articulate           articul
-    artificer            artific
-    artificial           artifici
-    artillery            artilleri
-    artire               artir
-    artist               artist
-    artists              artist
-    artless              artless
-    artois               artoi
-    arts                 art
-    artus                artu
-    arviragus            arviragu
-    as                   as
-    asaph                asaph
-    ascanius             ascaniu
-    ascend               ascend
-    ascended             ascend
-    ascendeth            ascendeth
-    ascends              ascend
-    ascension            ascens
-    ascent               ascent
-    ascribe              ascrib
-    ascribes             ascrib
-    ash                  ash
-    asham                asham
-    ashamed              asham
-    asher                asher
-    ashes                ash
-    ashford              ashford
-    ashore               ashor
-    ashouting            ashout
-    ashy                 ashi
-    asia                 asia
-    aside                asid
-    ask                  ask
-    askance              askanc
-    asked                ask
-    asker                asker
-    asketh               asketh
-    asking               ask
-    asks                 ask
-    aslant               aslant
-    asleep               asleep
-    asmath               asmath
-    asp                  asp
-    aspect               aspect
-    aspects              aspect
-    aspen                aspen
-    aspersion            aspers
-    aspic                aspic
-    aspicious            aspici
-    aspics               aspic
-    aspir                aspir
-    aspiration           aspir
-    aspire               aspir
-    aspiring             aspir
-    asquint              asquint
-    ass                  ass
-    assail               assail
-    assailable           assail
-    assailant            assail
-    assailants           assail
-    assailed             assail
-    assaileth            assaileth
-    assailing            assail
-    assails              assail
-    assassination        assassin
-    assault              assault
-    assaulted            assault
-    assaults             assault
-    assay                assai
-    assaying             assai
-    assays               assai
-    assemblance          assembl
-    assemble             assembl
-    assembled            assembl
-    assemblies           assembl
-    assembly             assembl
-    assent               assent
-    asses                ass
-    assez                assez
-    assign               assign
-    assigned             assign
-    assigns              assign
-    assinico             assinico
-    assist               assist
-    assistance           assist
-    assistances          assist
-    assistant            assist
-    assistants           assist
-    assisted             assist
-    assisting            assist
-    associate            associ
-    associated           associ
-    associates           associ
-    assuage              assuag
-    assubjugate          assubjug
-    assum                assum
-    assume               assum
-    assumes              assum
-    assumption           assumpt
-    assur                assur
-    assurance            assur
-    assure               assur
-    assured              assur
-    assuredly            assuredli
-    assures              assur
-    assyrian             assyrian
-    astonish             astonish
-    astonished           astonish
-    astraea              astraea
-    astray               astrai
-    astrea               astrea
-    astronomer           astronom
-    astronomers          astronom
-    astronomical         astronom
-    astronomy            astronomi
-    asunder              asund
-    at                   at
-    atalanta             atalanta
-    ate                  at
-    ates                 at
-    athenian             athenian
-    athenians            athenian
-    athens               athen
-    athol                athol
-    athversary           athversari
-    athwart              athwart
-    atlas                atla
-    atomies              atomi
-    atomy                atomi
-    atone                aton
-    atonement            aton
-    atonements           aton
-    atropos              atropo
-    attach               attach
-    attached             attach
-    attachment           attach
-    attain               attain
-    attainder            attaind
-    attains              attain
-    attaint              attaint
-    attainted            attaint
-    attainture           attaintur
-    attempt              attempt
-    attemptable          attempt
-    attempted            attempt
-    attempting           attempt
-    attempts             attempt
-    attend               attend
-    attendance           attend
-    attendant            attend
-    attendants           attend
-    attended             attend
-    attendents           attend
-    attendeth            attendeth
-    attending            attend
-    attends              attend
-    attent               attent
-    attention            attent
-    attentive            attent
-    attentivenes         attentiven
-    attest               attest
-    attested             attest
-    attir                attir
-    attire               attir
-    attired              attir
-    attires              attir
-    attorney             attornei
-    attorneyed           attornei
-    attorneys            attornei
-    attorneyship         attorneyship
-    attract              attract
-    attraction           attract
-    attractive           attract
-    attracts             attract
-    attribute            attribut
-    attributed           attribut
-    attributes           attribut
-    attribution          attribut
-    attributive          attribut
-    atwain               atwain
-    au                   au
-    aubrey               aubrei
-    auburn               auburn
-    aucun                aucun
-    audacious            audaci
-    audaciously          audaci
-    audacity             audac
-    audible              audibl
-    audience             audienc
-    audis                audi
-    audit                audit
-    auditor              auditor
-    auditors             auditor
-    auditory             auditori
-    audre                audr
-    audrey               audrei
-    aufidius             aufidiu
-    aufidiuses           aufidius
-    auger                auger
-    aught                aught
-    augment              augment
-    augmentation         augment
-    augmented            augment
-    augmenting           augment
-    augurer              augur
-    augurers             augur
-    augures              augur
-    auguring             augur
-    augurs               augur
-    augury               auguri
-    august               august
-    augustus             augustu
-    auld                 auld
-    aumerle              aumerl
-    aunchient            aunchient
-    aunt                 aunt
-    aunts                aunt
-    auricular            auricular
-    aurora               aurora
-    auspicious           auspici
-    aussi                aussi
-    austere              auster
-    austerely            auster
-    austereness          auster
-    austerity            auster
-    austria              austria
-    aut                  aut
-    authentic            authent
-    author               author
-    authorities          author
-    authority            author
-    authorized           author
-    authorizing          author
-    authors              author
-    autolycus            autolycu
-    autre                autr
-    autumn               autumn
-    auvergne             auvergn
-    avail                avail
-    avails               avail
-    avarice              avaric
-    avaricious           avarici
-    avaunt               avaunt
-    ave                  av
-    aveng                aveng
-    avenge               aveng
-    avenged              aveng
-    averring             aver
-    avert                avert
-    aves                 av
-    avez                 avez
-    avis                 avi
-    avoid                avoid
-    avoided              avoid
-    avoiding             avoid
-    avoids               avoid
-    avoirdupois          avoirdupoi
-    avouch               avouch
-    avouched             avouch
-    avouches             avouch
-    avouchment           avouch
-    avow                 avow
-    aw                   aw
-    await                await
-    awaits               await
-    awak                 awak
-    awake                awak
-    awaked               awak
-    awaken               awaken
-    awakened             awaken
-    awakens              awaken
-    awakes               awak
-    awaking              awak
-    award                award
-    awards               award
-    awasy                awasi
-    away                 awai
-    awe                  aw
-    aweary               aweari
-    aweless              aweless
-    awful                aw
-    awhile               awhil
-    awkward              awkward
-    awl                  awl
-    awooing              awoo
-    awork                awork
-    awry                 awri
-    axe                  ax
-    axle                 axl
-    axletree             axletre
-    ay                   ay
-    aye                  ay
-    ayez                 ayez
-    ayli                 ayli
-    azur                 azur
-    azure                azur
-    b                    b
-    ba                   ba
-    baa                  baa
-    babbl                babbl
-    babble               babbl
-    babbling             babbl
-    babe                 babe
-    babes                babe
-    babies               babi
-    baboon               baboon
-    baboons              baboon
-    baby                 babi
-    babylon              babylon
-    bacare               bacar
-    bacchanals           bacchan
-    bacchus              bacchu
-    bach                 bach
-    bachelor             bachelor
-    bachelors            bachelor
-    back                 back
-    backbite             backbit
-    backbitten           backbitten
-    backing              back
-    backs                back
-    backward             backward
-    backwardly           backwardli
-    backwards            backward
-    bacon                bacon
-    bacons               bacon
-    bad                  bad
-    bade                 bade
-    badge                badg
-    badged               badg
-    badges               badg
-    badly                badli
-    badness              bad
-    baes                 bae
-    baffl                baffl
-    baffle               baffl
-    baffled              baffl
-    bag                  bag
-    baggage              baggag
-    bagot                bagot
-    bagpipe              bagpip
-    bags                 bag
-    bail                 bail
-    bailiff              bailiff
-    baillez              baillez
-    baily                baili
-    baisant              baisant
-    baisees              baise
-    baiser               baiser
-    bait                 bait
-    baited               bait
-    baiting              bait
-    baitings             bait
-    baits                bait
-    bajazet              bajazet
-    bak                  bak
-    bake                 bake
-    baked                bake
-    baker                baker
-    bakers               baker
-    bakes                bake
-    baking               bake
-    bal                  bal
-    balanc               balanc
-    balance              balanc
-    balcony              balconi
-    bald                 bald
-    baldrick             baldrick
-    bale                 bale
-    baleful              bale
-    balk                 balk
-    ball                 ball
-    ballad               ballad
-    ballads              ballad
-    ballast              ballast
-    ballasting           ballast
-    ballet               ballet
-    ballow               ballow
-    balls                ball
-    balm                 balm
-    balms                balm
-    balmy                balmi
-    balsam               balsam
-    balsamum             balsamum
-    balth                balth
-    balthasar            balthasar
-    balthazar            balthazar
-    bames                bame
-    ban                  ban
-    banbury              banburi
-    band                 band
-    bandied              bandi
-    banding              band
-    bandit               bandit
-    banditti             banditti
-    banditto             banditto
-    bands                band
-    bandy                bandi
-    bandying             bandi
-    bane                 bane
-    banes                bane
-    bang                 bang
-    bangor               bangor
-    banish               banish
-    banished             banish
-    banishers            banish
-    banishment           banish
-    banister             banist
-    bank                 bank
-    bankrout             bankrout
-    bankrupt             bankrupt
-    bankrupts            bankrupt
-    banks                bank
-    banner               banner
-    bannerets            banneret
-    banners              banner
-    banning              ban
-    banns                bann
-    banquet              banquet
-    banqueted            banquet
-    banqueting           banquet
-    banquets             banquet
-    banquo               banquo
-    bans                 ban
-    baptism              baptism
-    baptista             baptista
-    baptiz               baptiz
-    bar                  bar
-    barbarian            barbarian
-    barbarians           barbarian
-    barbarism            barbar
-    barbarous            barbar
-    barbary              barbari
-    barbason             barbason
-    barbed               barb
-    barber               barber
-    barbermonger         barbermong
-    bard                 bard
-    bardolph             bardolph
-    bards                bard
-    bare                 bare
-    bared                bare
-    barefac              barefac
-    barefaced            barefac
-    barefoot             barefoot
-    bareheaded           barehead
-    barely               bare
-    bareness             bare
-    barful               bar
-    bargain              bargain
-    bargains             bargain
-    barge                barg
-    bargulus             bargulu
-    baring               bare
-    bark                 bark
-    barking              bark
-    barkloughly          barkloughli
-    barks                bark
-    barky                barki
-    barley               barlei
-    barm                 barm
-    barn                 barn
-    barnacles            barnacl
-    barnardine           barnardin
-    barne                barn
-    barnes               barn
-    barnet               barnet
-    barns                barn
-    baron                baron
-    barons               baron
-    barony               baroni
-    barr                 barr
-    barrabas             barraba
-    barrel               barrel
-    barrels              barrel
-    barren               barren
-    barrenly             barrenli
-    barrenness           barren
-    barricado            barricado
-    barricadoes          barricado
-    barrow               barrow
-    bars                 bar
-    barson               barson
-    barter               barter
-    bartholomew          bartholomew
-    bas                  ba
-    basan                basan
-    base                 base
-    baseless             baseless
-    basely               base
-    baseness             base
-    baser                baser
-    bases                base
-    basest               basest
-    bashful              bash
-    bashfulness          bash
-    basilisco            basilisco
-    basilisk             basilisk
-    basilisks            basilisk
-    basimecu             basimecu
-    basin                basin
-    basingstoke          basingstok
-    basins               basin
-    basis                basi
-    bask                 bask
-    basket               basket
-    baskets              basket
-    bass                 bass
-    bassanio             bassanio
-    basset               basset
-    bassianus            bassianu
-    basta                basta
-    bastard              bastard
-    bastardizing         bastard
-    bastardly            bastardli
-    bastards             bastard
-    bastardy             bastardi
-    basted               bast
-    bastes               bast
-    bastinado            bastinado
-    basting              bast
-    bat                  bat
-    batailles            batail
-    batch                batch
-    bate                 bate
-    bated                bate
-    bates                bate
-    bath                 bath
-    bathe                bath
-    bathed               bath
-    bathing              bath
-    baths                bath
-    bating               bate
-    batler               batler
-    bats                 bat
-    batt                 batt
-    battalia             battalia
-    battalions           battalion
-    batten               batten
-    batter               batter
-    battering            batter
-    batters              batter
-    battery              batteri
-    battle               battl
-    battled              battl
-    battlefield          battlefield
-    battlements          battlement
-    battles              battl
-    batty                batti
-    bauble               baubl
-    baubles              baubl
-    baubling             baubl
-    baulk                baulk
-    bavin                bavin
-    bawcock              bawcock
-    bawd                 bawd
-    bawdry               bawdri
-    bawds                bawd
-    bawdy                bawdi
-    bawl                 bawl
-    bawling              bawl
-    bay                  bai
-    baying               bai
-    baynard              baynard
-    bayonne              bayonn
-    bays                 bai
-    be                   be
-    beach                beach
-    beached              beach
-    beachy               beachi
-    beacon               beacon
-    bead                 bead
-    beaded               bead
-    beadle               beadl
-    beadles              beadl
-    beads                bead
-    beadsmen             beadsmen
-    beagle               beagl
-    beagles              beagl
-    beak                 beak
-    beaks                beak
-    beam                 beam
-    beamed               beam
-    beams                beam
-    bean                 bean
-    beans                bean
-    bear                 bear
-    beard                beard
-    bearded              beard
-    beardless            beardless
-    beards               beard
-    bearer               bearer
-    bearers              bearer
-    bearest              bearest
-    beareth              beareth
-    bearing              bear
-    bears                bear
-    beast                beast
-    beastliest           beastliest
-    beastliness          beastli
-    beastly              beastli
-    beasts               beast
-    beat                 beat
-    beated               beat
-    beaten               beaten
-    beating              beat
-    beatrice             beatric
-    beats                beat
-    beau                 beau
-    beaufort             beaufort
-    beaumond             beaumond
-    beaumont             beaumont
-    beauteous            beauteou
-    beautied             beauti
-    beauties             beauti
-    beautified           beautifi
-    beautiful            beauti
-    beautify             beautifi
-    beauty               beauti
-    beaver               beaver
-    beavers              beaver
-    became               becam
-    because              becaus
-    bechanc              bechanc
-    bechance             bechanc
-    bechanced            bechanc
-    beck                 beck
-    beckon               beckon
-    beckons              beckon
-    becks                beck
-    becom                becom
-    become               becom
-    becomed              becom
-    becomes              becom
-    becoming             becom
-    becomings            becom
-    bed                  bed
-    bedabbled            bedabbl
-    bedash               bedash
-    bedaub               bedaub
-    bedazzled            bedazzl
-    bedchamber           bedchamb
-    bedclothes           bedcloth
-    bedded               bed
-    bedeck               bedeck
-    bedecking            bedeck
-    bedew                bedew
-    bedfellow            bedfellow
-    bedfellows           bedfellow
-    bedford              bedford
-    bedlam               bedlam
-    bedrench             bedrench
-    bedrid               bedrid
-    beds                 bed
-    bedtime              bedtim
-    bedward              bedward
-    bee                  bee
-    beef                 beef
-    beefs                beef
-    beehives             beehiv
-    been                 been
-    beer                 beer
-    bees                 bee
-    beest                beest
-    beetle               beetl
-    beetles              beetl
-    beeves               beev
-    befall               befal
-    befallen             befallen
-    befalls              befal
-    befell               befel
-    befits               befit
-    befitted             befit
-    befitting            befit
-    befor                befor
-    before               befor
-    beforehand           beforehand
-    befortune            befortun
-    befriend             befriend
-    befriended           befriend
-    befriends            befriend
-    beg                  beg
-    began                began
-    beget                beget
-    begets               beget
-    begetting            beget
-    begg                 begg
-    beggar               beggar
-    beggared             beggar
-    beggarly             beggarli
-    beggarman            beggarman
-    beggars              beggar
-    beggary              beggari
-    begging              beg
-    begin                begin
-    beginners            beginn
-    beginning            begin
-    beginnings           begin
-    begins               begin
-    begnawn              begnawn
-    begone               begon
-    begot                begot
-    begotten             begotten
-    begrimed             begrim
-    begs                 beg
-    beguil               beguil
-    beguile              beguil
-    beguiled             beguil
-    beguiles             beguil
-    beguiling            beguil
-    begun                begun
-    behalf               behalf
-    behalfs              behalf
-    behav                behav
-    behaved              behav
-    behavedst            behavedst
-    behavior             behavior
-    behaviors            behavior
-    behaviour            behaviour
-    behaviours           behaviour
-    behead               behead
-    beheaded             behead
-    beheld               beheld
-    behest               behest
-    behests              behest
-    behind               behind
-    behold               behold
-    beholder             behold
-    beholders            behold
-    beholdest            beholdest
-    beholding            behold
-    beholds              behold
-    behoof               behoof
-    behooffull           behoofful
-    behooves             behoov
-    behove               behov
-    behoves              behov
-    behowls              behowl
-    being                be
-    bel                  bel
-    belarius             belariu
-    belch                belch
-    belching             belch
-    beldam               beldam
-    beldame              beldam
-    beldams              beldam
-    belee                bele
-    belgia               belgia
-    belie                beli
-    belied               beli
-    belief               belief
-    beliest              beliest
-    believ               believ
-    believe              believ
-    believed             believ
-    believes             believ
-    believest            believest
-    believing            believ
-    belike               belik
-    bell                 bell
-    bellario             bellario
-    belle                bell
-    bellied              belli
-    bellies              belli
-    bellman              bellman
-    bellona              bellona
-    bellow               bellow
-    bellowed             bellow
-    bellowing            bellow
-    bellows              bellow
-    bells                bell
-    belly                belli
-    bellyful             belly
-    belman               belman
-    belmont              belmont
-    belock               belock
-    belong               belong
-    belonging            belong
-    belongings           belong
-    belongs              belong
-    belov                belov
-    beloved              belov
-    beloving             belov
-    below                below
-    belt                 belt
-    belzebub             belzebub
-    bemadding            bemad
-    bemet                bemet
-    bemete               bemet
-    bemoan               bemoan
-    bemoaned             bemoan
-    bemock               bemock
-    bemoil               bemoil
-    bemonster            bemonst
-    ben                  ben
-    bench                bench
-    bencher              bencher
-    benches              bench
-    bend                 bend
-    bended               bend
-    bending              bend
-    bends                bend
-    bene                 bene
-    beneath              beneath
-    benedicite           benedicit
-    benedick             benedick
-    benediction          benedict
-    benedictus           benedictu
-    benefactors          benefactor
-    benefice             benefic
-    beneficial           benefici
-    benefit              benefit
-    benefited            benefit
-    benefits             benefit
-    benetted             benet
-    benevolence          benevol
-    benevolences         benevol
-    benied               beni
-    benison              benison
-    bennet               bennet
-    bent                 bent
-    bentii               bentii
-    bentivolii           bentivolii
-    bents                bent
-    benumbed             benumb
-    benvolio             benvolio
-    bepaint              bepaint
-    bepray               beprai
-    bequeath             bequeath
-    bequeathed           bequeath
-    bequeathing          bequeath
-    bequest              bequest
-    ber                  ber
-    berard               berard
-    berattle             berattl
-    beray                berai
-    bere                 bere
-    bereave              bereav
-    bereaved             bereav
-    bereaves             bereav
-    bereft               bereft
-    bergamo              bergamo
-    bergomask            bergomask
-    berhym               berhym
-    berhyme              berhym
-    berkeley             berkelei
-    bermoothes           bermooth
-    bernardo             bernardo
-    berod                berod
-    berowne              berown
-    berri                berri
-    berries              berri
-    berrord              berrord
-    berry                berri
-    bertram              bertram
-    berwick              berwick
-    bescreen             bescreen
-    beseech              beseech
-    beseeched            beseech
-    beseechers           beseech
-    beseeching           beseech
-    beseek               beseek
-    beseem               beseem
-    beseemeth            beseemeth
-    beseeming            beseem
-    beseems              beseem
-    beset                beset
-    beshrew              beshrew
-    beside               besid
-    besides              besid
-    besieg               besieg
-    besiege              besieg
-    besieged             besieg
-    beslubber            beslubb
-    besmear              besmear
-    besmeared            besmear
-    besmirch             besmirch
-    besom                besom
-    besort               besort
-    besotted             besot
-    bespake              bespak
-    bespeak              bespeak
-    bespice              bespic
-    bespoke              bespok
-    bespotted            bespot
-    bess                 bess
-    bessy                bessi
-    best                 best
-    bestained            bestain
-    bested               best
-    bestial              bestial
-    bestir               bestir
-    bestirr              bestirr
-    bestow               bestow
-    bestowed             bestow
-    bestowing            bestow
-    bestows              bestow
-    bestraught           bestraught
-    bestrew              bestrew
-    bestrid              bestrid
-    bestride             bestrid
-    bestrides            bestrid
-    bet                  bet
-    betake               betak
-    beteem               beteem
-    bethink              bethink
-    bethought            bethought
-    bethrothed           bethroth
-    bethump              bethump
-    betid                betid
-    betide               betid
-    betideth             betideth
-    betime               betim
-    betimes              betim
-    betoken              betoken
-    betook               betook
-    betossed             betoss
-    betray               betrai
-    betrayed             betrai
-    betraying            betrai
-    betrays              betrai
-    betrims              betrim
-    betroth              betroth
-    betrothed            betroth
-    betroths             betroth
-    bett                 bett
-    betted               bet
-    better               better
-    bettered             better
-    bettering            better
-    betters              better
-    betting              bet
-    bettre               bettr
-    between              between
-    betwixt              betwixt
-    bevel                bevel
-    beverage             beverag
-    bevis                bevi
-    bevy                 bevi
-    bewail               bewail
-    bewailed             bewail
-    bewailing            bewail
-    bewails              bewail
-    beware               bewar
-    bewasted             bewast
-    beweep               beweep
-    bewept               bewept
-    bewet                bewet
-    bewhored             bewhor
-    bewitch              bewitch
-    bewitched            bewitch
-    bewitchment          bewitch
-    bewray               bewrai
-    beyond               beyond
-    bezonian             bezonian
-    bezonians            bezonian
-    bianca               bianca
-    bianco               bianco
-    bias                 bia
-    bibble               bibbl
-    bickerings           bicker
-    bid                  bid
-    bidden               bidden
-    bidding              bid
-    biddings             bid
-    biddy                biddi
-    bide                 bide
-    bides                bide
-    biding               bide
-    bids                 bid
-    bien                 bien
-    bier                 bier
-    bifold               bifold
-    big                  big
-    bigamy               bigami
-    biggen               biggen
-    bigger               bigger
-    bigness              big
-    bigot                bigot
-    bilberry             bilberri
-    bilbo                bilbo
-    bilboes              bilbo
-    bilbow               bilbow
-    bill                 bill
-    billeted             billet
-    billets              billet
-    billiards            billiard
-    billing              bill
-    billow               billow
-    billows              billow
-    bills                bill
-    bin                  bin
-    bind                 bind
-    bindeth              bindeth
-    binding              bind
-    binds                bind
-    biondello            biondello
-    birch                birch
-    bird                 bird
-    birding              bird
-    birdlime             birdlim
-    birds                bird
-    birnam               birnam
-    birth                birth
-    birthday             birthdai
-    birthdom             birthdom
-    birthplace           birthplac
-    birthright           birthright
-    birthrights          birthright
-    births               birth
-    bis                  bi
-    biscuit              biscuit
-    bishop               bishop
-    bishops              bishop
-    bisson               bisson
-    bit                  bit
-    bitch                bitch
-    bite                 bite
-    biter                biter
-    bites                bite
-    biting               bite
-    bits                 bit
-    bitt                 bitt
-    bitten               bitten
-    bitter               bitter
-    bitterest            bitterest
-    bitterly             bitterli
-    bitterness           bitter
-    blab                 blab
-    blabb                blabb
-    blabbing             blab
-    blabs                blab
-    black                black
-    blackamoor           blackamoor
-    blackamoors          blackamoor
-    blackberries         blackberri
-    blackberry           blackberri
-    blacker              blacker
-    blackest             blackest
-    blackfriars          blackfriar
-    blackheath           blackheath
-    blackmere            blackmer
-    blackness            black
-    blacks               black
-    bladder              bladder
-    bladders             bladder
-    blade                blade
-    bladed               blade
-    blades               blade
-    blains               blain
-    blam                 blam
-    blame                blame
-    blamed               blame
-    blameful             blame
-    blameless            blameless
-    blames               blame
-    blanc                blanc
-    blanca               blanca
-    blanch               blanch
-    blank                blank
-    blanket              blanket
-    blanks               blank
-    blaspheme            blasphem
-    blaspheming          blasphem
-    blasphemous          blasphem
-    blasphemy            blasphemi
-    blast                blast
-    blasted              blast
-    blasting             blast
-    blastments           blastment
-    blasts               blast
-    blaz                 blaz
-    blaze                blaze
-    blazes               blaze
-    blazing              blaze
-    blazon               blazon
-    blazoned             blazon
-    blazoning            blazon
-    bleach               bleach
-    bleaching            bleach
-    bleak                bleak
-    blear                blear
-    bleared              blear
-    bleat                bleat
-    bleated              bleat
-    bleats               bleat
-    bled                 bled
-    bleed                bleed
-    bleedest             bleedest
-    bleedeth             bleedeth
-    bleeding             bleed
-    bleeds               bleed
-    blemish              blemish
-    blemishes            blemish
-    blench               blench
-    blenches             blench
-    blend                blend
-    blended              blend
-    blent                blent
-    bless                bless
-    blessed              bless
-    blessedly            blessedli
-    blessedness          blessed
-    blesses              bless
-    blesseth             blesseth
-    blessing             bless
-    blessings            bless
-    blest                blest
-    blew                 blew
-    blind                blind
-    blinded              blind
-    blindfold            blindfold
-    blinding             blind
-    blindly              blindli
-    blindness            blind
-    blinds               blind
-    blink                blink
-    blinking             blink
-    bliss                bliss
-    blist                blist
-    blister              blister
-    blisters             blister
-    blithe               blith
-    blithild             blithild
-    bloat                bloat
-    block                block
-    blockish             blockish
-    blocks               block
-    blois                bloi
-    blood                blood
-    blooded              blood
-    bloodhound           bloodhound
-    bloodied             bloodi
-    bloodier             bloodier
-    bloodiest            bloodiest
-    bloodily             bloodili
-    bloodless            bloodless
-    bloods               blood
-    bloodshed            bloodsh
-    bloodshedding        bloodshed
-    bloodstained         bloodstain
-    bloody               bloodi
-    bloom                bloom
-    blooms               bloom
-    blossom              blossom
-    blossoming           blossom
-    blossoms             blossom
-    blot                 blot
-    blots                blot
-    blotted              blot
-    blotting             blot
-    blount               blount
-    blow                 blow
-    blowed               blow
-    blowers              blower
-    blowest              blowest
-    blowing              blow
-    blown                blown
-    blows                blow
-    blowse               blows
-    blubb                blubb
-    blubber              blubber
-    blubbering           blubber
-    blue                 blue
-    bluecaps             bluecap
-    bluest               bluest
-    blunt                blunt
-    blunted              blunt
-    blunter              blunter
-    bluntest             bluntest
-    blunting             blunt
-    bluntly              bluntli
-    bluntness            blunt
-    blunts               blunt
-    blur                 blur
-    blurr                blurr
-    blurs                blur
-    blush                blush
-    blushes              blush
-    blushest             blushest
-    blushing             blush
-    blust                blust
-    bluster              bluster
-    blusterer            bluster
-    blusters             bluster
-    bo                   bo
-    boar                 boar
-    board                board
-    boarded              board
-    boarding             board
-    boards               board
-    boarish              boarish
-    boars                boar
-    boast                boast
-    boasted              boast
-    boastful             boast
-    boasting             boast
-    boasts               boast
-    boat                 boat
-    boats                boat
-    boatswain            boatswain
-    bob                  bob
-    bobb                 bobb
-    boblibindo           boblibindo
-    bobtail              bobtail
-    bocchus              bocchu
-    bode                 bode
-    boded                bode
-    bodements            bodement
-    bodes                bode
-    bodg                 bodg
-    bodied               bodi
-    bodies               bodi
-    bodiless             bodiless
-    bodily               bodili
-    boding               bode
-    bodkin               bodkin
-    body                 bodi
-    bodykins             bodykin
-    bog                  bog
-    boggle               boggl
-    boggler              boggler
-    bogs                 bog
-    bohemia              bohemia
-    bohemian             bohemian
-    bohun                bohun
-    boil                 boil
-    boiling              boil
-    boils                boil
-    boist                boist
-    boisterous           boister
-    boisterously         boister
-    boitier              boitier
-    bold                 bold
-    bolden               bolden
-    bolder               bolder
-    boldest              boldest
-    boldly               boldli
-    boldness             bold
-    bolds                bold
-    bolingbroke          bolingbrok
-    bolster              bolster
-    bolt                 bolt
-    bolted               bolt
-    bolter               bolter
-    bolters              bolter
-    bolting              bolt
-    bolts                bolt
-    bombard              bombard
-    bombards             bombard
-    bombast              bombast
-    bon                  bon
-    bona                 bona
-    bond                 bond
-    bondage              bondag
-    bonded               bond
-    bondmaid             bondmaid
-    bondman              bondman
-    bondmen              bondmen
-    bonds                bond
-    bondslave            bondslav
-    bone                 bone
-    boneless             boneless
-    bones                bone
-    bonfire              bonfir
-    bonfires             bonfir
-    bonjour              bonjour
-    bonne                bonn
-    bonnet               bonnet
-    bonneted             bonnet
-    bonny                bonni
-    bonos                bono
-    bonto                bonto
-    bonville             bonvil
-    bood                 bood
-    book                 book
-    bookish              bookish
-    books                book
-    boon                 boon
-    boor                 boor
-    boorish              boorish
-    boors                boor
-    boot                 boot
-    booted               boot
-    booties              booti
-    bootless             bootless
-    boots                boot
-    booty                booti
-    bor                  bor
-    bora                 bora
-    borachio             borachio
-    bordeaux             bordeaux
-    border               border
-    bordered             border
-    borderers            border
-    borders              border
-    bore                 bore
-    boreas               borea
-    bores                bore
-    boring               bore
-    born                 born
-    borne                born
-    borough              borough
-    boroughs             borough
-    borrow               borrow
-    borrowed             borrow
-    borrower             borrow
-    borrowing            borrow
-    borrows              borrow
-    bosko                bosko
-    boskos               bosko
-    bosky                boski
-    bosom                bosom
-    bosoms               bosom
-    boson                boson
-    boss                 boss
-    bosworth             bosworth
-    botch                botch
-    botcher              botcher
-    botches              botch
-    botchy               botchi
-    both                 both
-    bots                 bot
-    bottle               bottl
-    bottled              bottl
-    bottles              bottl
-    bottom               bottom
-    bottomless           bottomless
-    bottoms              bottom
-    bouciqualt           bouciqualt
-    bouge                boug
-    bough                bough
-    boughs               bough
-    bought               bought
-    bounce               bounc
-    bouncing             bounc
-    bound                bound
-    bounded              bound
-    bounden              bounden
-    boundeth             boundeth
-    bounding             bound
-    boundless            boundless
-    bounds               bound
-    bounteous            bounteou
-    bounteously          bounteous
-    bounties             bounti
-    bountiful            bounti
-    bountifully          bountifulli
-    bounty               bounti
-    bourbier             bourbier
-    bourbon              bourbon
-    bourchier            bourchier
-    bourdeaux            bourdeaux
-    bourn                bourn
-    bout                 bout
-    bouts                bout
-    bove                 bove
-    bow                  bow
-    bowcase              bowcas
-    bowed                bow
-    bowels               bowel
-    bower                bower
-    bowing               bow
-    bowl                 bowl
-    bowler               bowler
-    bowling              bowl
-    bowls                bowl
-    bows                 bow
-    bowsprit             bowsprit
-    bowstring            bowstr
-    box                  box
-    boxes                box
-    boy                  boi
-    boyet                boyet
-    boyish               boyish
-    boys                 boi
-    brabant              brabant
-    brabantio            brabantio
-    brabble              brabbl
-    brabbler             brabbler
-    brac                 brac
-    brace                brace
-    bracelet             bracelet
-    bracelets            bracelet
-    brach                brach
-    bracy                braci
-    brag                 brag
-    bragg                bragg
-    braggardism          braggard
-    braggards            braggard
-    braggart             braggart
-    braggarts            braggart
-    bragged              brag
-    bragging             brag
-    bragless             bragless
-    brags                brag
-    braid                braid
-    braided              braid
-    brain                brain
-    brained              brain
-    brainford            brainford
-    brainish             brainish
-    brainless            brainless
-    brains               brain
-    brainsick            brainsick
-    brainsickly          brainsickli
-    brake                brake
-    brakenbury           brakenburi
-    brakes               brake
-    brambles             brambl
-    bran                 bran
-    branch               branch
-    branches             branch
-    branchless           branchless
-    brand                brand
-    branded              brand
-    brandish             brandish
-    brandon              brandon
-    brands               brand
-    bras                 bra
-    brass                brass
-    brassy               brassi
-    brat                 brat
-    brats                brat
-    brav                 brav
-    brave                brave
-    braved               brave
-    bravely              brave
-    braver               braver
-    bravery              braveri
-    braves               brave
-    bravest              bravest
-    braving              brave
-    brawl                brawl
-    brawler              brawler
-    brawling             brawl
-    brawls               brawl
-    brawn                brawn
-    brawns               brawn
-    bray                 brai
-    braying              brai
-    braz                 braz
-    brazen               brazen
-    brazier              brazier
-    breach               breach
-    breaches             breach
-    bread                bread
-    breadth              breadth
-    break                break
-    breaker              breaker
-    breakfast            breakfast
-    breaking             break
-    breaks               break
-    breast               breast
-    breasted             breast
-    breasting            breast
-    breastplate          breastplat
-    breasts              breast
-    breath               breath
-    breathe              breath
-    breathed             breath
-    breather             breather
-    breathers            breather
-    breathes             breath
-    breathest            breathest
-    breathing            breath
-    breathless           breathless
-    breaths              breath
-    brecknock            brecknock
-    bred                 bred
-    breech               breech
-    breeches             breech
-    breeching            breech
-    breed                breed
-    breeder              breeder
-    breeders             breeder
-    breeding             breed
-    breeds               breed
-    breese               brees
-    breeze               breez
-    breff                breff
-    bretagne             bretagn
-    brethen              brethen
-    bretheren            bretheren
-    brethren             brethren
-    brevis               brevi
-    brevity              breviti
-    brew                 brew
-    brewage              brewag
-    brewer               brewer
-    brewers              brewer
-    brewing              brew
-    brews                brew
-    briareus             briareu
-    briars               briar
-    brib                 brib
-    bribe                bribe
-    briber               briber
-    bribes               bribe
-    brick                brick
-    bricklayer           bricklay
-    bricks               brick
-    bridal               bridal
-    bride                bride
-    bridegroom           bridegroom
-    bridegrooms          bridegroom
-    brides               bride
-    bridge               bridg
-    bridgenorth          bridgenorth
-    bridges              bridg
-    bridget              bridget
-    bridle               bridl
-    bridled              bridl
-    brief                brief
-    briefer              briefer
-    briefest             briefest
-    briefly              briefli
-    briefness            brief
-    brier                brier
-    briers               brier
-    brigandine           brigandin
-    bright               bright
-    brighten             brighten
-    brightest            brightest
-    brightly             brightli
-    brightness           bright
-    brim                 brim
-    brimful              brim
-    brims                brim
-    brimstone            brimston
-    brinded              brind
-    brine                brine
-    bring                bring
-    bringer              bringer
-    bringeth             bringeth
-    bringing             bring
-    bringings            bring
-    brings               bring
-    brinish              brinish
-    brink                brink
-    brisk                brisk
-    brisky               briski
-    bristle              bristl
-    bristled             bristl
-    bristly              bristli
-    bristol              bristol
-    bristow              bristow
-    britain              britain
-    britaine             britain
-    britaines            britain
-    british              british
-    briton               briton
-    britons              briton
-    brittany             brittani
-    brittle              brittl
-    broach               broach
-    broached             broach
-    broad                broad
-    broader              broader
-    broadsides           broadsid
-    brocas               broca
-    brock                brock
-    brogues              brogu
-    broil                broil
-    broiling             broil
-    broils               broil
-    broke                broke
-    broken               broken
-    brokenly             brokenli
-    broker               broker
-    brokers              broker
-    brokes               broke
-    broking              broke
-    brooch               brooch
-    brooches             brooch
-    brood                brood
-    brooded              brood
-    brooding             brood
-    brook                brook
-    brooks               brook
-    broom                broom
-    broomstaff           broomstaff
-    broth                broth
-    brothel              brothel
-    brother              brother
-    brotherhood          brotherhood
-    brotherhoods         brotherhood
-    brotherly            brotherli
-    brothers             brother
-    broths               broth
-    brought              brought
-    brow                 brow
-    brown                brown
-    browner              browner
-    brownist             brownist
-    browny               browni
-    brows                brow
-    browse               brows
-    browsing             brows
-    bruis                brui
-    bruise               bruis
-    bruised              bruis
-    bruises              bruis
-    bruising             bruis
-    bruit                bruit
-    bruited              bruit
-    brundusium           brundusium
-    brunt                brunt
-    brush                brush
-    brushes              brush
-    brute                brute
-    brutish              brutish
-    brutus               brutu
-    bubble               bubbl
-    bubbles              bubbl
-    bubbling             bubbl
-    bubukles             bubukl
-    buck                 buck
-    bucket               bucket
-    buckets              bucket
-    bucking              buck
-    buckingham           buckingham
-    buckle               buckl
-    buckled              buckl
-    buckler              buckler
-    bucklers             buckler
-    bucklersbury         bucklersburi
-    buckles              buckl
-    buckram              buckram
-    bucks                buck
-    bud                  bud
-    budded               bud
-    budding              bud
-    budge                budg
-    budger               budger
-    budget               budget
-    buds                 bud
-    buff                 buff
-    buffet               buffet
-    buffeting            buffet
-    buffets              buffet
-    bug                  bug
-    bugbear              bugbear
-    bugle                bugl
-    bugs                 bug
-    build                build
-    builded              build
-    buildeth             buildeth
-    building             build
-    buildings            build
-    builds               build
-    built                built
-    bulk                 bulk
-    bulks                bulk
-    bull                 bull
-    bullcalf             bullcalf
-    bullen               bullen
-    bullens              bullen
-    bullet               bullet
-    bullets              bullet
-    bullocks             bullock
-    bulls                bull
-    bully                bulli
-    bulmer               bulmer
-    bulwark              bulwark
-    bulwarks             bulwark
-    bum                  bum
-    bumbast              bumbast
-    bump                 bump
-    bumper               bumper
-    bums                 bum
-    bunch                bunch
-    bunches              bunch
-    bundle               bundl
-    bung                 bung
-    bunghole             bunghol
-    bungle               bungl
-    bunting              bunt
-    buoy                 buoi
-    bur                  bur
-    burbolt              burbolt
-    burd                 burd
-    burden               burden
-    burdened             burden
-    burdening            burden
-    burdenous            burden
-    burdens              burden
-    burgh                burgh
-    burgher              burgher
-    burghers             burgher
-    burglary             burglari
-    burgomasters         burgomast
-    burgonet             burgonet
-    burgundy             burgundi
-    burial               burial
-    buried               buri
-    burier               burier
-    buriest              buriest
-    burly                burli
-    burn                 burn
-    burned               burn
-    burnet               burnet
-    burneth              burneth
-    burning              burn
-    burnish              burnish
-    burns                burn
-    burnt                burnt
-    burr                 burr
-    burrows              burrow
-    burs                 bur
-    burst                burst
-    bursting             burst
-    bursts               burst
-    burthen              burthen
-    burthens             burthen
-    burton               burton
-    bury                 buri
-    burying              buri
-    bush                 bush
-    bushels              bushel
-    bushes               bush
-    bushy                bushi
-    busied               busi
-    busily               busili
-    busines              busin
-    business             busi
-    businesses           busi
-    buskin               buskin
-    busky                buski
-    buss                 buss
-    busses               buss
-    bussing              buss
-    bustle               bustl
-    bustling             bustl
-    busy                 busi
-    but                  but
-    butcheed             butche
-    butcher              butcher
-    butchered            butcher
-    butcheries           butcheri
-    butcherly            butcherli
-    butchers             butcher
-    butchery             butcheri
-    butler               butler
-    butt                 butt
-    butter               butter
-    buttered             butter
-    butterflies          butterfli
-    butterfly            butterfli
-    butterwoman          butterwoman
-    buttery              butteri
-    buttock              buttock
-    buttocks             buttock
-    button               button
-    buttonhole           buttonhol
-    buttons              button
-    buttress             buttress
-    buttry               buttri
-    butts                butt
-    buxom                buxom
-    buy                  bui
-    buyer                buyer
-    buying               bui
-    buys                 bui
-    buzz                 buzz
-    buzzard              buzzard
-    buzzards             buzzard
-    buzzers              buzzer
-    buzzing              buzz
-    by                   by
-    bye                  bye
-    byzantium            byzantium
-    c                    c
-    ca                   ca
-    cabbage              cabbag
-    cabileros            cabilero
-    cabin                cabin
-    cabins               cabin
-    cable                cabl
-    cables               cabl
-    cackling             cackl
-    cacodemon            cacodemon
-    caddis               caddi
-    caddisses            caddiss
-    cade                 cade
-    cadence              cadenc
-    cadent               cadent
-    cades                cade
-    cadmus               cadmu
-    caduceus             caduceu
-    cadwal               cadwal
-    cadwallader          cadwallad
-    caelius              caeliu
-    caelo                caelo
-    caesar               caesar
-    caesarion            caesarion
-    caesars              caesar
-    cage                 cage
-    caged                cage
-    cagion               cagion
-    cain                 cain
-    caithness            caith
-    caitiff              caitiff
-    caitiffs             caitiff
-    caius                caiu
-    cak                  cak
-    cake                 cake
-    cakes                cake
-    calaber              calab
-    calais               calai
-    calamities           calam
-    calamity             calam
-    calchas              calcha
-    calculate            calcul
-    calen                calen
-    calendar             calendar
-    calendars            calendar
-    calf                 calf
-    caliban              caliban
-    calibans             caliban
-    calipolis            calipoli
-    cality               caliti
-    caliver              caliv
-    call                 call
-    callat               callat
-    called               call
-    callet               callet
-    calling              call
-    calls                call
-    calm                 calm
-    calmest              calmest
-    calmly               calmli
-    calmness             calm
-    calms                calm
-    calpurnia            calpurnia
-    calumniate           calumni
-    calumniating         calumni
-    calumnious           calumni
-    calumny              calumni
-    calve                calv
-    calved               calv
-    calves               calv
-    calveskins           calveskin
-    calydon              calydon
-    cam                  cam
-    cambio               cambio
-    cambria              cambria
-    cambric              cambric
-    cambrics             cambric
-    cambridge            cambridg
-    cambyses             cambys
-    came                 came
-    camel                camel
-    camelot              camelot
-    camels               camel
-    camest               camest
-    camillo              camillo
-    camlet               camlet
-    camomile             camomil
-    camp                 camp
-    campeius             campeiu
-    camping              camp
-    camps                camp
-    can                  can
-    canakin              canakin
-    canaries             canari
-    canary               canari
-    cancel               cancel
-    cancell              cancel
-    cancelled            cancel
-    cancelling           cancel
-    cancels              cancel
-    cancer               cancer
-    candidatus           candidatu
-    candied              candi
-    candle               candl
-    candles              candl
-    candlesticks         candlestick
-    candy                candi
-    canidius             canidiu
-    cank                 cank
-    canker               canker
-    cankerblossom        cankerblossom
-    cankers              canker
-    cannibally           cannib
-    cannibals            cannib
-    cannon               cannon
-    cannoneer            cannon
-    cannons              cannon
-    cannot               cannot
-    canon                canon
-    canoniz              canoniz
-    canonize             canon
-    canonized            canon
-    canons               canon
-    canopied             canopi
-    canopies             canopi
-    canopy               canopi
-    canst                canst
-    canstick             canstick
-    canterbury           canterburi
-    cantle               cantl
-    cantons              canton
-    canus                canu
-    canvas               canva
-    canvass              canvass
-    canzonet             canzonet
-    cap                  cap
-    capability           capabl
-    capable              capabl
-    capacities           capac
-    capacity             capac
-    caparison            caparison
-    capdv                capdv
-    cape                 cape
-    capel                capel
-    capels               capel
-    caper                caper
-    capers               caper
-    capet                capet
-    caphis               caphi
-    capilet              capilet
-    capitaine            capitain
-    capital              capit
-    capite               capit
-    capitol              capitol
-    capitulate           capitul
-    capocchia            capocchia
-    capon                capon
-    capons               capon
-    capp                 capp
-    cappadocia           cappadocia
-    capriccio            capriccio
-    capricious           caprici
-    caps                 cap
-    capt                 capt
-    captain              captain
-    captains             captain
-    captainship          captainship
-    captious             captiou
-    captivate            captiv
-    captivated           captiv
-    captivates           captiv
-    captive              captiv
-    captives             captiv
-    captivity            captiv
-    captum               captum
-    capucius             capuciu
-    capulet              capulet
-    capulets             capulet
-    car                  car
-    carack               carack
-    caracks              carack
-    carat                carat
-    caraways             carawai
-    carbonado            carbonado
-    carbuncle            carbuncl
-    carbuncled           carbuncl
-    carbuncles           carbuncl
-    carcanet             carcanet
-    carcase              carcas
-    carcases             carcas
-    carcass              carcass
-    carcasses            carcass
-    card                 card
-    cardecue             cardecu
-    carded               card
-    carders              carder
-    cardinal             cardin
-    cardinally           cardin
-    cardinals            cardin
-    cardmaker            cardmak
-    cards                card
-    carduus              carduu
-    care                 care
-    cared                care
-    career               career
-    careers              career
-    careful              care
-    carefully            carefulli
-    careless             careless
-    carelessly           carelessli
-    carelessness         careless
-    cares                care
-    caret                caret
-    cargo                cargo
-    carl                 carl
-    carlisle             carlisl
-    carlot               carlot
-    carman               carman
-    carmen               carmen
-    carnal               carnal
-    carnally             carnal
-    carnarvonshire       carnarvonshir
-    carnation            carnat
-    carnations           carnat
-    carol                carol
-    carous               carou
-    carouse              carous
-    caroused             carous
-    carouses             carous
-    carousing            carous
-    carp                 carp
-    carpenter            carpent
-    carper               carper
-    carpet               carpet
-    carpets              carpet
-    carping              carp
-    carriage             carriag
-    carriages            carriag
-    carried              carri
-    carrier              carrier
-    carriers             carrier
-    carries              carri
-    carrion              carrion
-    carrions             carrion
-    carry                carri
-    carrying             carri
-    cars                 car
-    cart                 cart
-    carters              carter
-    carthage             carthag
-    carts                cart
-    carv                 carv
-    carve                carv
-    carved               carv
-    carver               carver
-    carves               carv
-    carving              carv
-    cas                  ca
-    casa                 casa
-    casaer               casaer
-    casca                casca
-    case                 case
-    casement             casement
-    casements            casement
-    cases                case
-    cash                 cash
-    cashier              cashier
-    casing               case
-    cask                 cask
-    casket               casket
-    casketed             casket
-    caskets              casket
-    casque               casqu
-    casques              casqu
-    cassado              cassado
-    cassandra            cassandra
-    cassibelan           cassibelan
-    cassio               cassio
-    cassius              cassiu
-    cassocks             cassock
-    cast                 cast
-    castalion            castalion
-    castaway             castawai
-    castaways            castawai
-    casted               cast
-    caster               caster
-    castigate            castig
-    castigation          castig
-    castile              castil
-    castiliano           castiliano
-    casting              cast
-    castle               castl
-    castles              castl
-    casts                cast
-    casual               casual
-    casually             casual
-    casualties           casualti
-    casualty             casualti
-    cat                  cat
-    cataian              cataian
-    catalogue            catalogu
-    cataplasm            cataplasm
-    cataracts            cataract
-    catarrhs             catarrh
-    catastrophe          catastroph
-    catch                catch
-    catcher              catcher
-    catches              catch
-    catching             catch
-    cate                 cate
-    catechising          catechis
-    catechism            catech
-    catechize            catech
-    cater                cater
-    caterpillars         caterpillar
-    caters               cater
-    caterwauling         caterwaul
-    cates                cate
-    catesby              catesbi
-    cathedral            cathedr
-    catlike              catlik
-    catling              catl
-    catlings             catl
-    cato                 cato
-    cats                 cat
-    cattle               cattl
-    caucasus             caucasu
-    caudle               caudl
-    cauf                 cauf
-    caught               caught
-    cauldron             cauldron
-    caus                 cau
-    cause                caus
-    caused               caus
-    causeless            causeless
-    causer               causer
-    causes               caus
-    causest              causest
-    causeth              causeth
-    cautel               cautel
-    cautelous            cautel
-    cautels              cautel
-    cauterizing          cauter
-    caution              caution
-    cautions             caution
-    cavaleiro            cavaleiro
-    cavalery             cavaleri
-    cavaliers            cavali
-    cave                 cave
-    cavern               cavern
-    caverns              cavern
-    caves                cave
-    caveto               caveto
-    caviary              caviari
-    cavil                cavil
-    cavilling            cavil
-    cawdor               cawdor
-    cawdron              cawdron
-    cawing               caw
-    ce                   ce
-    ceas                 cea
-    cease                ceas
-    ceases               ceas
-    ceaseth              ceaseth
-    cedar                cedar
-    cedars               cedar
-    cedius               cediu
-    celebrate            celebr
-    celebrated           celebr
-    celebrates           celebr
-    celebration          celebr
-    celerity             celer
-    celestial            celesti
-    celia                celia
-    cell                 cell
-    cellar               cellar
-    cellarage            cellarag
-    celsa                celsa
-    cement               cement
-    censer               censer
-    censor               censor
-    censorinus           censorinu
-    censur               censur
-    censure              censur
-    censured             censur
-    censurers            censur
-    censures             censur
-    censuring            censur
-    centaur              centaur
-    centaurs             centaur
-    centre               centr
-    cents                cent
-    centuries            centuri
-    centurion            centurion
-    centurions           centurion
-    century              centuri
-    cerberus             cerberu
-    cerecloth            cerecloth
-    cerements            cerement
-    ceremonial           ceremoni
-    ceremonies           ceremoni
-    ceremonious          ceremoni
-    ceremoniously        ceremoni
-    ceremony             ceremoni
-    ceres                cere
-    cerns                cern
-    certain              certain
-    certainer            certain
-    certainly            certainli
-    certainties          certainti
-    certainty            certainti
-    certes               cert
-    certificate          certif
-    certified            certifi
-    certifies            certifi
-    certify              certifi
-    ces                  ce
-    cesario              cesario
-    cess                 cess
-    cesse                cess
-    cestern              cestern
-    cetera               cetera
-    cette                cett
-    chaces               chace
-    chaf                 chaf
-    chafe                chafe
-    chafed               chafe
-    chafes               chafe
-    chaff                chaff
-    chaffless            chaffless
-    chafing              chafe
-    chain                chain
-    chains               chain
-    chair                chair
-    chairs               chair
-    chalic               chalic
-    chalice              chalic
-    chalices             chalic
-    chalk                chalk
-    chalks               chalk
-    chalky               chalki
-    challeng             challeng
-    challenge            challeng
-    challenged           challeng
-    challenger           challeng
-    challengers          challeng
-    challenges           challeng
-    cham                 cham
-    chamber              chamber
-    chamberers           chamber
-    chamberlain          chamberlain
-    chamberlains         chamberlain
-    chambermaid          chambermaid
-    chambermaids         chambermaid
-    chambers             chamber
-    chameleon            chameleon
-    champ                champ
-    champagne            champagn
-    champain             champain
-    champains            champain
-    champion             champion
-    champions            champion
-    chanc                chanc
-    chance               chanc
-    chanced              chanc
-    chancellor           chancellor
-    chances              chanc
-    chandler             chandler
-    chang                chang
-    change               chang
-    changeable           changeabl
-    changed              chang
-    changeful            chang
-    changeling           changel
-    changelings          changel
-    changer              changer
-    changes              chang
-    changest             changest
-    changing             chang
-    channel              channel
-    channels             channel
-    chanson              chanson
-    chant                chant
-    chanticleer          chanticl
-    chanting             chant
-    chantries            chantri
-    chantry              chantri
-    chants               chant
-    chaos                chao
-    chap                 chap
-    chape                chape
-    chapel               chapel
-    chapeless            chapeless
-    chapels              chapel
-    chaplain             chaplain
-    chaplains            chaplain
-    chapless             chapless
-    chaplet              chaplet
-    chapmen              chapmen
-    chaps                chap
-    chapter              chapter
-    character            charact
-    charactered          charact
-    characterless        characterless
-    characters           charact
-    charactery           characteri
-    characts             charact
-    charbon              charbon
-    chare                chare
-    chares               chare
-    charg                charg
-    charge               charg
-    charged              charg
-    chargeful            charg
-    charges              charg
-    chargeth             chargeth
-    charging             charg
-    chariest             chariest
-    chariness            chari
-    charing              chare
-    chariot              chariot
-    chariots             chariot
-    charitable           charit
-    charitably           charit
-    charities            chariti
-    charity              chariti
-    charlemain           charlemain
-    charles              charl
-    charm                charm
-    charmed              charm
-    charmer              charmer
-    charmeth             charmeth
-    charmian             charmian
-    charming             charm
-    charmingly           charmingli
-    charms               charm
-    charneco             charneco
-    charnel              charnel
-    charolois            charoloi
-    charon               charon
-    charter              charter
-    charters             charter
-    chartreux            chartreux
-    chary                chari
-    charybdis            charybdi
-    chas                 cha
-    chase                chase
-    chased               chase
-    chaser               chaser
-    chaseth              chaseth
-    chasing              chase
-    chaste               chast
-    chastely             chast
-    chastis              chasti
-    chastise             chastis
-    chastised            chastis
-    chastisement         chastis
-    chastity             chastiti
-    chat                 chat
-    chatham              chatham
-    chatillon            chatillon
-    chats                chat
-    chatt                chatt
-    chattels             chattel
-    chatter              chatter
-    chattering           chatter
-    chattles             chattl
-    chaud                chaud
-    chaunted             chaunt
-    chaw                 chaw
-    chawdron             chawdron
-    che                  che
-    cheap                cheap
-    cheapen              cheapen
-    cheaper              cheaper
-    cheapest             cheapest
-    cheaply              cheapli
-    cheapside            cheapsid
-    cheat                cheat
-    cheated              cheat
-    cheater              cheater
-    cheaters             cheater
-    cheating             cheat
-    cheats               cheat
-    check                check
-    checked              check
-    checker              checker
-    checking             check
-    checks               check
-    cheek                cheek
-    cheeks               cheek
-    cheer                cheer
-    cheered              cheer
-    cheerer              cheerer
-    cheerful             cheer
-    cheerfully           cheerfulli
-    cheering             cheer
-    cheerless            cheerless
-    cheerly              cheerli
-    cheers               cheer
-    cheese               chees
-    chequer              chequer
-    cher                 cher
-    cherish              cherish
-    cherished            cherish
-    cherisher            cherish
-    cherishes            cherish
-    cherishing           cherish
-    cherries             cherri
-    cherry               cherri
-    cherrypit            cherrypit
-    chertsey             chertsei
-    cherub               cherub
-    cherubims            cherubim
-    cherubin             cherubin
-    cherubins            cherubin
-    cheshu               cheshu
-    chess                chess
-    chest                chest
-    chester              chester
-    chestnut             chestnut
-    chestnuts            chestnut
-    chests               chest
-    chetas               cheta
-    chev                 chev
-    cheval               cheval
-    chevalier            chevali
-    chevaliers           chevali
-    cheveril             cheveril
-    chew                 chew
-    chewed               chew
-    chewet               chewet
-    chewing              chew
-    chez                 chez
-    chi                  chi
-    chick                chick
-    chicken              chicken
-    chickens             chicken
-    chicurmurco          chicurmurco
-    chid                 chid
-    chidden              chidden
-    chide                chide
-    chiders              chider
-    chides               chide
-    chiding              chide
-    chief                chief
-    chiefest             chiefest
-    chiefly              chiefli
-    chien                chien
-    child                child
-    childed              child
-    childeric            childer
-    childhood            childhood
-    childhoods           childhood
-    childing             child
-    childish             childish
-    childishness         childish
-    childlike            childlik
-    childness            child
-    children             children
-    chill                chill
-    chilling             chill
-    chime                chime
-    chimes               chime
-    chimney              chimnei
-    chimneypiece         chimneypiec
-    chimneys             chimnei
-    chimurcho            chimurcho
-    chin                 chin
-    china                china
-    chine                chine
-    chines               chine
-    chink                chink
-    chinks               chink
-    chins                chin
-    chipp                chipp
-    chipper              chipper
-    chips                chip
-    chiron               chiron
-    chirping             chirp
-    chirrah              chirrah
-    chirurgeonly         chirurgeonli
-    chisel               chisel
-    chitopher            chitoph
-    chivalrous           chivalr
-    chivalry             chivalri
-    choice               choic
-    choicely             choic
-    choicest             choicest
-    choir                choir
-    choirs               choir
-    chok                 chok
-    choke                choke
-    choked               choke
-    chokes               choke
-    choking              choke
-    choler               choler
-    choleric             choler
-    cholers              choler
-    chollors             chollor
-    choose               choos
-    chooser              chooser
-    chooses              choos
-    chooseth             chooseth
-    choosing             choos
-    chop                 chop
-    chopine              chopin
-    choplogic            choplog
-    chopp                chopp
-    chopped              chop
-    chopping             chop
-    choppy               choppi
-    chops                chop
-    chopt                chopt
-    chor                 chor
-    choristers           chorist
-    chorus               choru
-    chose                chose
-    chosen               chosen
-    chough               chough
-    choughs              chough
-    chrish               chrish
-    christ               christ
-    christen             christen
-    christendom          christendom
-    christendoms         christendom
-    christening          christen
-    christenings         christen
-    christian            christian
-    christianlike        christianlik
-    christians           christian
-    christmas            christma
-    christom             christom
-    christopher          christoph
-    christophero         christophero
-    chronicle            chronicl
-    chronicled           chronicl
-    chronicler           chronicl
-    chroniclers          chronicl
-    chronicles           chronicl
-    chrysolite           chrysolit
-    chuck                chuck
-    chucks               chuck
-    chud                 chud
-    chuffs               chuff
-    church               church
-    churches             church
-    churchman            churchman
-    churchmen            churchmen
-    churchyard           churchyard
-    churchyards          churchyard
-    churl                churl
-    churlish             churlish
-    churlishly           churlishli
-    churls               churl
-    churn                churn
-    chus                 chu
-    cicatrice            cicatric
-    cicatrices           cicatric
-    cicely               cice
-    cicero               cicero
-    ciceter              cicet
-    ciel                 ciel
-    ciitzens             ciitzen
-    cilicia              cilicia
-    cimber               cimber
-    cimmerian            cimmerian
-    cinable              cinabl
-    cincture             cinctur
-    cinders              cinder
-    cine                 cine
-    cinna                cinna
-    cinque               cinqu
-    cipher               cipher
-    ciphers              cipher
-    circa                circa
-    circe                circ
-    circle               circl
-    circled              circl
-    circlets             circlet
-    circling             circl
-    circuit              circuit
-    circum               circum
-    circumcised          circumcis
-    circumference        circumfer
-    circummur            circummur
-    circumscrib          circumscrib
-    circumscribed        circumscrib
-    circumscription      circumscript
-    circumspect          circumspect
-    circumstance         circumst
-    circumstanced        circumstanc
-    circumstances        circumst
-    circumstantial       circumstanti
-    circumvent           circumv
-    circumvention        circumvent
-    cistern              cistern
-    citadel              citadel
-    cital                cital
-    cite                 cite
-    cited                cite
-    cites                cite
-    cities               citi
-    citing               cite
-    citizen              citizen
-    citizens             citizen
-    cittern              cittern
-    city                 citi
-    civet                civet
-    civil                civil
-    civility             civil
-    civilly              civilli
-    clack                clack
-    clad                 clad
-    claim                claim
-    claiming             claim
-    claims               claim
-    clamb                clamb
-    clamber              clamber
-    clammer              clammer
-    clamor               clamor
-    clamorous            clamor
-    clamors              clamor
-    clamour              clamour
-    clamours             clamour
-    clang                clang
-    clangor              clangor
-    clap                 clap
-    clapp                clapp
-    clapped              clap
-    clapper              clapper
-    clapping             clap
-    claps                clap
-    clare                clare
-    clarence             clarenc
-    claret               claret
-    claribel             claribel
-    clasp                clasp
-    clasps               clasp
-    clatter              clatter
-    claud                claud
-    claudio              claudio
-    claudius             claudiu
-    clause               claus
-    claw                 claw
-    clawed               claw
-    clawing              claw
-    claws                claw
-    clay                 clai
-    clays                clai
-    clean                clean
-    cleanliest           cleanliest
-    cleanly              cleanli
-    cleans               clean
-    cleanse              cleans
-    cleansing            cleans
-    clear                clear
-    clearer              clearer
-    clearest             clearest
-    clearly              clearli
-    clearness            clear
-    clears               clear
-    cleave               cleav
-    cleaving             cleav
-    clef                 clef
-    cleft                cleft
-    cleitus              cleitu
-    clemency             clemenc
-    clement              clement
-    cleomenes            cleomen
-    cleopatpa            cleopatpa
-    cleopatra            cleopatra
-    clepeth              clepeth
-    clept                clept
-    clerestories         clerestori
-    clergy               clergi
-    clergyman            clergyman
-    clergymen            clergymen
-    clerk                clerk
-    clerkly              clerkli
-    clerks               clerk
-    clew                 clew
-    client               client
-    clients              client
-    cliff                cliff
-    clifford             clifford
-    cliffords            clifford
-    cliffs               cliff
-    clifton              clifton
-    climate              climat
-    climature            climatur
-    climb                climb
-    climbed              climb
-    climber              climber
-    climbeth             climbeth
-    climbing             climb
-    climbs               climb
-    clime                clime
-    cling                cling
-    clink                clink
-    clinking             clink
-    clinquant            clinquant
-    clip                 clip
-    clipp                clipp
-    clipper              clipper
-    clippeth             clippeth
-    clipping             clip
-    clipt                clipt
-    clitus               clitu
-    clo                  clo
-    cloak                cloak
-    cloakbag             cloakbag
-    cloaks               cloak
-    clock                clock
-    clocks               clock
-    clod                 clod
-    cloddy               cloddi
-    clodpole             clodpol
-    clog                 clog
-    clogging             clog
-    clogs                clog
-    cloister             cloister
-    cloistress           cloistress
-    cloquence            cloquenc
-    clos                 clo
-    close                close
-    closed               close
-    closely              close
-    closeness            close
-    closer               closer
-    closes               close
-    closest              closest
-    closet               closet
-    closing              close
-    closure              closur
-    cloten               cloten
-    clotens              cloten
-    cloth                cloth
-    clothair             clothair
-    clotharius           clothariu
-    clothe               cloth
-    clothes              cloth
-    clothier             clothier
-    clothiers            clothier
-    clothing             cloth
-    cloths               cloth
-    clotpoles            clotpol
-    clotpoll             clotpol
-    cloud                cloud
-    clouded              cloud
-    cloudiness           cloudi
-    clouds               cloud
-    cloudy               cloudi
-    clout                clout
-    clouted              clout
-    clouts               clout
-    cloven               cloven
-    clover               clover
-    cloves               clove
-    clovest              clovest
-    clowder              clowder
-    clown                clown
-    clownish             clownish
-    clowns               clown
-    cloy                 cloi
-    cloyed               cloi
-    cloying              cloi
-    cloyless             cloyless
-    cloyment             cloyment
-    cloys                cloi
-    club                 club
-    clubs                club
-    cluck                cluck
-    clung                clung
-    clust                clust
-    clusters             cluster
-    clutch               clutch
-    clyster              clyster
-    cneius               cneiu
-    cnemies              cnemi
-    co                   co
-    coach                coach
-    coaches              coach
-    coachmakers          coachmak
-    coact                coact
-    coactive             coactiv
-    coagulate            coagul
-    coal                 coal
-    coals                coal
-    coarse               coars
-    coarsely             coars
-    coast                coast
-    coasting             coast
-    coasts               coast
-    coat                 coat
-    coated               coat
-    coats                coat
-    cobble               cobbl
-    cobbled              cobbl
-    cobbler              cobbler
-    cobham               cobham
-    cobloaf              cobloaf
-    cobweb               cobweb
-    cobwebs              cobweb
-    cock                 cock
-    cockatrice           cockatric
-    cockatrices          cockatric
-    cockle               cockl
-    cockled              cockl
-    cockney              cocknei
-    cockpit              cockpit
-    cocks                cock
-    cocksure             cocksur
-    coctus               coctu
-    cocytus              cocytu
-    cod                  cod
-    codding              cod
-    codling              codl
-    codpiece             codpiec
-    codpieces            codpiec
-    cods                 cod
-    coelestibus          coelestibu
-    coesar               coesar
-    coeur                coeur
-    coffer               coffer
-    coffers              coffer
-    coffin               coffin
-    coffins              coffin
-    cog                  cog
-    cogging              cog
-    cogitation           cogit
-    cogitations          cogit
-    cognition            cognit
-    cognizance           cogniz
-    cogscomb             cogscomb
-    cohabitants          cohabit
-    coher                coher
-    cohere               coher
-    coherence            coher
-    coherent             coher
-    cohorts              cohort
-    coif                 coif
-    coign                coign
-    coil                 coil
-    coin                 coin
-    coinage              coinag
-    coiner               coiner
-    coining              coin
-    coins                coin
-    col                  col
-    colbrand             colbrand
-    colchos              colcho
-    cold                 cold
-    colder               colder
-    coldest              coldest
-    coldly               coldli
-    coldness             cold
-    coldspur             coldspur
-    colebrook            colebrook
-    colic                colic
-    collar               collar
-    collars              collar
-    collateral           collater
-    colleagued           colleagu
-    collect              collect
-    collected            collect
-    collection           collect
-    college              colleg
-    colleges             colleg
-    collied              colli
-    collier              collier
-    colliers             collier
-    collop               collop
-    collusion            collus
-    colme                colm
-    colmekill            colmekil
-    coloquintida         coloquintida
-    color                color
-    colors               color
-    colossus             colossu
-    colour               colour
-    colourable           colour
-    coloured             colour
-    colouring            colour
-    colours              colour
-    colt                 colt
-    colted               colt
-    colts                colt
-    columbine            columbin
-    columbines           columbin
-    colville             colvil
-    com                  com
-    comagene             comagen
-    comart               comart
-    comb                 comb
-    combat               combat
-    combatant            combat
-    combatants           combat
-    combated             combat
-    combating            combat
-    combin               combin
-    combinate            combin
-    combination          combin
-    combine              combin
-    combined             combin
-    combless             combless
-    combustion           combust
-    come                 come
-    comedian             comedian
-    comedians            comedian
-    comedy               comedi
-    comeliness           comeli
-    comely               come
-    comer                comer
-    comers               comer
-    comes                come
-    comest               comest
-    comet                comet
-    cometh               cometh
-    comets               comet
-    comfect              comfect
-    comfit               comfit
-    comfits              comfit
-    comfort              comfort
-    comfortable          comfort
-    comforted            comfort
-    comforter            comfort
-    comforting           comfort
-    comfortless          comfortless
-    comforts             comfort
-    comic                comic
-    comical              comic
-    coming               come
-    comings              come
-    cominius             cominiu
-    comma                comma
-    command              command
-    commande             command
-    commanded            command
-    commander            command
-    commanders           command
-    commanding           command
-    commandment          command
-    commandments         command
-    commands             command
-    comme                comm
-    commenc              commenc
-    commence             commenc
-    commenced            commenc
-    commencement         commenc
-    commences            commenc
-    commencing           commenc
-    commend              commend
-    commendable          commend
-    commendation         commend
-    commendations        commend
-    commended            commend
-    commending           commend
-    commends             commend
-    comment              comment
-    commentaries         commentari
-    commenting           comment
-    comments             comment
-    commerce             commerc
-    commingled           commingl
-    commiseration        commiser
-    commission           commiss
-    commissioners        commission
-    commissions          commiss
-    commit               commit
-    commits              commit
-    committ              committ
-    committed            commit
-    committing           commit
-    commix               commix
-    commixed             commix
-    commixtion           commixt
-    commixture           commixtur
-    commodious           commodi
-    commodities          commod
-    commodity            commod
-    common               common
-    commonalty           commonalti
-    commoner             common
-    commoners            common
-    commonly             commonli
-    commons              common
-    commonweal           commonw
-    commonwealth         commonwealth
-    commotion            commot
-    commotions           commot
-    commune              commun
-    communicat           communicat
-    communicate          commun
-    communication        commun
-    communities          commun
-    community            commun
-    comonty              comonti
-    compact              compact
-    companies            compani
-    companion            companion
-    companions           companion
-    companionship        companionship
-    company              compani
-    compar               compar
-    comparative          compar
-    compare              compar
-    compared             compar
-    comparing            compar
-    comparison           comparison
-    comparisons          comparison
-    compartner           compartn
-    compass              compass
-    compasses            compass
-    compassing           compass
-    compassion           compass
-    compassionate        compassion
-    compeers             compeer
-    compel               compel
-    compell              compel
-    compelled            compel
-    compelling           compel
-    compels              compel
-    compensation         compens
-    competence           compet
-    competency           compet
-    competent            compet
-    competitor           competitor
-    competitors          competitor
-    compil               compil
-    compile              compil
-    compiled             compil
-    complain             complain
-    complainer           complain
-    complainest          complainest
-    complaining          complain
-    complainings         complain
-    complains            complain
-    complaint            complaint
-    complaints           complaint
-    complement           complement
-    complements          complement
-    complete             complet
-    complexion           complexion
-    complexioned         complexion
-    complexions          complexion
-    complices            complic
-    complies             compli
-    compliment           compliment
-    complimental         compliment
-    compliments          compliment
-    complot              complot
-    complots             complot
-    complotted           complot
-    comply               compli
-    compos               compo
-    compose              compos
-    composed             compos
-    composition          composit
-    compost              compost
-    composture           compostur
-    composure            composur
-    compound             compound
-    compounded           compound
-    compounds            compound
-    comprehend           comprehend
-    comprehended         comprehend
-    comprehends          comprehend
-    compremises          compremis
-    compris              compri
-    comprising           compris
-    compromis            compromi
-    compromise           compromis
-    compt                compt
-    comptible            comptibl
-    comptrollers         comptrol
-    compulsatory         compulsatori
-    compulsion           compuls
-    compulsive           compuls
-    compunctious         compuncti
-    computation          comput
-    comrade              comrad
-    comrades             comrad
-    comutual             comutu
-    con                  con
-    concave              concav
-    concavities          concav
-    conceal              conceal
-    concealed            conceal
-    concealing           conceal
-    concealment          conceal
-    concealments         conceal
-    conceals             conceal
-    conceit              conceit
-    conceited            conceit
-    conceitless          conceitless
-    conceits             conceit
-    conceiv              conceiv
-    conceive             conceiv
-    conceived            conceiv
-    conceives            conceiv
-    conceiving           conceiv
-    conception           concept
-    conceptions          concept
-    conceptious          concepti
-    concern              concern
-    concernancy          concern
-    concerneth           concerneth
-    concerning           concern
-    concernings          concern
-    concerns             concern
-    conclave             conclav
-    conclud              conclud
-    conclude             conclud
-    concluded            conclud
-    concludes            conclud
-    concluding           conclud
-    conclusion           conclus
-    conclusions          conclus
-    concolinel           concolinel
-    concord              concord
-    concubine            concubin
-    concupiscible        concupisc
-    concupy              concupi
-    concur               concur
-    concurring           concur
-    concurs              concur
-    condemn              condemn
-    condemnation         condemn
-    condemned            condemn
-    condemning           condemn
-    condemns             condemn
-    condescend           condescend
-    condign              condign
-    condition            condit
-    conditionally        condition
-    conditions           condit
-    condole              condol
-    condolement          condol
-    condoling            condol
-    conduce              conduc
-    conduct              conduct
-    conducted            conduct
-    conducting           conduct
-    conductor            conductor
-    conduit              conduit
-    conduits             conduit
-    conected             conect
-    coney                conei
-    confection           confect
-    confectionary        confectionari
-    confections          confect
-    confederacy          confederaci
-    confederate          confeder
-    confederates         confeder
-    confer               confer
-    conference           confer
-    conferr              conferr
-    conferring           confer
-    confess              confess
-    confessed            confess
-    confesses            confess
-    confesseth           confesseth
-    confessing           confess
-    confession           confess
-    confessions          confess
-    confessor            confessor
-    confidence           confid
-    confident            confid
-    confidently          confid
-    confin               confin
-    confine              confin
-    confined             confin
-    confineless          confineless
-    confiners            confin
-    confines             confin
-    confining            confin
-    confirm              confirm
-    confirmation         confirm
-    confirmations        confirm
-    confirmed            confirm
-    confirmer            confirm
-    confirmers           confirm
-    confirming           confirm
-    confirmities         confirm
-    confirms             confirm
-    confiscate           confisc
-    confiscated          confisc
-    confiscation         confisc
-    confixed             confix
-    conflict             conflict
-    conflicting          conflict
-    conflicts            conflict
-    confluence           confluenc
-    conflux              conflux
-    conform              conform
-    conformable          conform
-    confound             confound
-    confounded           confound
-    confounding          confound
-    confounds            confound
-    confront             confront
-    confronted           confront
-    confus               confu
-    confused             confus
-    confusedly           confusedli
-    confusion            confus
-    confusions           confus
-    confutation          confut
-    confutes             confut
-    congeal              congeal
-    congealed            congeal
-    congealment          congeal
-    congee               conge
-    conger               conger
-    congest              congest
-    congied              congi
-    congratulate         congratul
-    congreeing           congre
-    congreeted           congreet
-    congregate           congreg
-    congregated          congreg
-    congregation         congreg
-    congregations        congreg
-    congruent            congruent
-    congruing            congru
-    conies               coni
-    conjectural          conjectur
-    conjecture           conjectur
-    conjectures          conjectur
-    conjoin              conjoin
-    conjoined            conjoin
-    conjoins             conjoin
-    conjointly           conjointli
-    conjunct             conjunct
-    conjunction          conjunct
-    conjunctive          conjunct
-    conjur               conjur
-    conjuration          conjur
-    conjurations         conjur
-    conjure              conjur
-    conjured             conjur
-    conjurer             conjur
-    conjurers            conjur
-    conjures             conjur
-    conjuring            conjur
-    conjuro              conjuro
-    conn                 conn
-    connected            connect
-    connive              conniv
-    conqu                conqu
-    conquer              conquer
-    conquered            conquer
-    conquering           conquer
-    conqueror            conqueror
-    conquerors           conqueror
-    conquers             conquer
-    conquest             conquest
-    conquests            conquest
-    conquring            conqur
-    conrade              conrad
-    cons                 con
-    consanguineous       consanguin
-    consanguinity        consanguin
-    conscienc            conscienc
-    conscience           conscienc
-    consciences          conscienc
-    conscionable         conscion
-    consecrate           consecr
-    consecrated          consecr
-    consecrations        consecr
-    consent              consent
-    consented            consent
-    consenting           consent
-    consents             consent
-    consequence          consequ
-    consequences         consequ
-    consequently         consequ
-    conserve             conserv
-    conserved            conserv
-    conserves            conserv
-    consider             consid
-    considerance         consider
-    considerate          consider
-    consideration        consider
-    considerations       consider
-    considered           consid
-    considering          consid
-    considerings         consid
-    considers            consid
-    consign              consign
-    consigning           consign
-    consist              consist
-    consisteth           consisteth
-    consisting           consist
-    consistory           consistori
-    consists             consist
-    consolate            consol
-    consolation          consol
-    consonancy           conson
-    consonant            conson
-    consort              consort
-    consorted            consort
-    consortest           consortest
-    conspectuities       conspectu
-    conspir              conspir
-    conspiracy           conspiraci
-    conspirant           conspir
-    conspirator          conspir
-    conspirators         conspir
-    conspire             conspir
-    conspired            conspir
-    conspirers           conspir
-    conspires            conspir
-    conspiring           conspir
-    constable            constabl
-    constables           constabl
-    constance            constanc
-    constancies          constanc
-    constancy            constanc
-    constant             constant
-    constantine          constantin
-    constantinople       constantinopl
-    constantly           constantli
-    constellation        constel
-    constitution         constitut
-    constrain            constrain
-    constrained          constrain
-    constraineth         constraineth
-    constrains           constrain
-    constraint           constraint
-    constring            constr
-    construction         construct
-    construe             constru
-    consul               consul
-    consuls              consul
-    consulship           consulship
-    consulships          consulship
-    consult              consult
-    consulting           consult
-    consults             consult
-    consum               consum
-    consume              consum
-    consumed             consum
-    consumes             consum
-    consuming            consum
-    consummate           consumm
-    consummation         consumm
-    consumption          consumpt
-    consumptions         consumpt
-    contagion            contagion
-    contagious           contagi
-    contain              contain
-    containing           contain
-    contains             contain
-    contaminate          contamin
-    contaminated         contamin
-    contemn              contemn
-    contemned            contemn
-    contemning           contemn
-    contemns             contemn
-    contemplate          contempl
-    contemplation        contempl
-    contemplative        contempl
-    contempt             contempt
-    contemptible         contempt
-    contempts            contempt
-    contemptuous         contemptu
-    contemptuously       contemptu
-    contend              contend
-    contended            contend
-    contending           contend
-    contendon            contendon
-    content              content
-    contenta             contenta
-    contented            content
-    contenteth           contenteth
-    contention           content
-    contentious          contenti
-    contentless          contentless
-    contento             contento
-    contents             content
-    contest              contest
-    contestation         contest
-    continence           contin
-    continency           contin
-    continent            contin
-    continents           contin
-    continu              continu
-    continual            continu
-    continually          continu
-    continuance          continu
-    continuantly         continuantli
-    continuate           continu
-    continue             continu
-    continued            continu
-    continuer            continu
-    continues            continu
-    continuing           continu
-    contract             contract
-    contracted           contract
-    contracting          contract
-    contraction          contract
-    contradict           contradict
-    contradicted         contradict
-    contradiction        contradict
-    contradicts          contradict
-    contraries           contrari
-    contrarieties        contrarieti
-    contrariety          contrarieti
-    contrarious          contrari
-    contrariously        contrari
-    contrary             contrari
-    contre               contr
-    contribution         contribut
-    contributors         contributor
-    contrite             contrit
-    contriv              contriv
-    contrive             contriv
-    contrived            contriv
-    contriver            contriv
-    contrives            contriv
-    contriving           contriv
-    control              control
-    controll             control
-    controller           control
-    controlling          control
-    controlment          control
-    controls             control
-    controversy          controversi
-    contumelious         contumeli
-    contumeliously       contumeli
-    contumely            contum
-    contusions           contus
-    convenience          conveni
-    conveniences         conveni
-    conveniency          conveni
-    convenient           conveni
-    conveniently         conveni
-    convented            convent
-    conventicles         conventicl
-    convents             convent
-    convers              conver
-    conversant           convers
-    conversation         convers
-    conversations        convers
-    converse             convers
-    conversed            convers
-    converses            convers
-    conversing           convers
-    conversion           convers
-    convert              convert
-    converted            convert
-    convertest           convertest
-    converting           convert
-    convertite           convertit
-    convertites          convertit
-    converts             convert
-    convey               convei
-    conveyance           convey
-    conveyances          convey
-    conveyers            convey
-    conveying            convei
-    convict              convict
-    convicted            convict
-    convince             convinc
-    convinced            convinc
-    convinces            convinc
-    convive              conviv
-    convocation          convoc
-    convoy               convoi
-    convulsions          convuls
-    cony                 coni
-    cook                 cook
-    cookery              cookeri
-    cooks                cook
-    cool                 cool
-    cooled               cool
-    cooling              cool
-    cools                cool
-    coop                 coop
-    coops                coop
-    cop                  cop
-    copatain             copatain
-    cope                 cope
-    cophetua             cophetua
-    copied               copi
-    copies               copi
-    copious              copiou
-    copper               copper
-    copperspur           copperspur
-    coppice              coppic
-    copulation           copul
-    copulatives          copul
-    copy                 copi
-    cor                  cor
-    coragio              coragio
-    coral                coral
-    coram                coram
-    corambus             corambu
-    coranto              coranto
-    corantos             coranto
-    corbo                corbo
-    cord                 cord
-    corded               cord
-    cordelia             cordelia
-    cordial              cordial
-    cordis               cordi
-    cords                cord
-    core                 core
-    corin                corin
-    corinth              corinth
-    corinthian           corinthian
-    coriolanus           coriolanu
-    corioli              corioli
-    cork                 cork
-    corky                corki
-    cormorant            cormor
-    corn                 corn
-    cornelia             cornelia
-    cornelius            corneliu
-    corner               corner
-    corners              corner
-    cornerstone          cornerston
-    cornets              cornet
-    cornish              cornish
-    corns                corn
-    cornuto              cornuto
-    cornwall             cornwal
-    corollary            corollari
-    coronal              coron
-    coronation           coron
-    coronet              coronet
-    coronets             coronet
-    corporal             corpor
-    corporals            corpor
-    corporate            corpor
-    corpse               corps
-    corpulent            corpul
-    correct              correct
-    corrected            correct
-    correcting           correct
-    correction           correct
-    correctioner         correction
-    corrects             correct
-    correspondence       correspond
-    correspondent        correspond
-    corresponding        correspond
-    corresponsive        correspons
-    corrigible           corrig
-    corrival             corriv
-    corrivals            corriv
-    corroborate          corrobor
-    corrosive            corros
-    corrupt              corrupt
-    corrupted            corrupt
-    corrupter            corrupt
-    corrupters           corrupt
-    corruptible          corrupt
-    corruptibly          corrupt
-    corrupting           corrupt
-    corruption           corrupt
-    corruptly            corruptli
-    corrupts             corrupt
-    corse                cors
-    corses               cors
-    corslet              corslet
-    cosmo                cosmo
-    cost                 cost
-    costard              costard
-    costermongers        costermong
-    costlier             costlier
-    costly               costli
-    costs                cost
-    cot                  cot
-    cote                 cote
-    coted                cote
-    cotsall              cotsal
-    cotsole              cotsol
-    cotswold             cotswold
-    cottage              cottag
-    cottages             cottag
-    cotus                cotu
-    couch                couch
-    couched              couch
-    couching             couch
-    couchings            couch
-    coude                coud
-    cough                cough
-    coughing             cough
-    could                could
-    couldst              couldst
-    coulter              coulter
-    council              council
-    councillor           councillor
-    councils             council
-    counsel              counsel
-    counsell             counsel
-    counsellor           counsellor
-    counsellors          counsellor
-    counselor            counselor
-    counselors           counselor
-    counsels             counsel
-    count                count
-    counted              count
-    countenanc           countenanc
-    countenance          counten
-    countenances         counten
-    counter              counter
-    counterchange        counterchang
-    countercheck         countercheck
-    counterfeit          counterfeit
-    counterfeited        counterfeit
-    counterfeiting       counterfeit
-    counterfeitly        counterfeitli
-    counterfeits         counterfeit
-    countermand          countermand
-    countermands         countermand
-    countermines         countermin
-    counterpart          counterpart
-    counterpoints        counterpoint
-    counterpois          counterpoi
-    counterpoise         counterpois
-    counters             counter
-    countervail          countervail
-    countess             countess
-    countesses           countess
-    counties             counti
-    counting             count
-    countless            countless
-    countries            countri
-    countrv              countrv
-    country              countri
-    countryman           countryman
-    countrymen           countrymen
-    counts               count
-    county               counti
-    couper               couper
-    couple               coupl
-    coupled              coupl
-    couplement           couplement
-    couples              coupl
-    couplet              couplet
-    couplets             couplet
-    cour                 cour
-    courage              courag
-    courageous           courag
-    courageously         courag
-    courages             courag
-    courier              courier
-    couriers             courier
-    couronne             couronn
-    cours                cour
-    course               cours
-    coursed              cours
-    courser              courser
-    coursers             courser
-    courses              cours
-    coursing             cours
-    court                court
-    courted              court
-    courteous            courteou
-    courteously          courteous
-    courtesan            courtesan
-    courtesies           courtesi
-    courtesy             courtesi
-    courtezan            courtezan
-    courtezans           courtezan
-    courtier             courtier
-    courtiers            courtier
-    courtlike            courtlik
-    courtly              courtli
-    courtney             courtnei
-    courts               court
-    courtship            courtship
-    cousin               cousin
-    cousins              cousin
-    couterfeit           couterfeit
-    coutume              coutum
-    covenant             coven
-    covenants            coven
-    covent               covent
-    coventry             coventri
-    cover                cover
-    covered              cover
-    covering             cover
-    coverlet             coverlet
-    covers               cover
-    covert               covert
-    covertly             covertli
-    coverture            covertur
-    covet                covet
-    coveted              covet
-    coveting             covet
-    covetings            covet
-    covetous             covet
-    covetously           covet
-    covetousness         covet
-    covets               covet
-    cow                  cow
-    coward               coward
-    cowarded             coward
-    cowardice            cowardic
-    cowardly             cowardli
-    cowards              coward
-    cowardship           cowardship
-    cowish               cowish
-    cowl                 cowl
-    cowslip              cowslip
-    cowslips             cowslip
-    cox                  cox
-    coxcomb              coxcomb
-    coxcombs             coxcomb
-    coy                  coi
-    coystrill            coystril
-    coz                  coz
-    cozen                cozen
-    cozenage             cozenag
-    cozened              cozen
-    cozener              cozen
-    cozeners             cozen
-    cozening             cozen
-    coziers              cozier
-    crab                 crab
-    crabbed              crab
-    crabs                crab
-    crack                crack
-    cracked              crack
-    cracker              cracker
-    crackers             cracker
-    cracking             crack
-    cracks               crack
-    cradle               cradl
-    cradled              cradl
-    cradles              cradl
-    craft                craft
-    crafted              craft
-    craftied             crafti
-    craftier             craftier
-    craftily             craftili
-    crafts               craft
-    craftsmen            craftsmen
-    crafty               crafti
-    cram                 cram
-    cramm                cramm
-    cramp                cramp
-    cramps               cramp
-    crams                cram
-    cranking             crank
-    cranks               crank
-    cranmer              cranmer
-    crannied             cranni
-    crannies             cranni
-    cranny               cranni
-    crants               crant
-    crare                crare
-    crash                crash
-    crassus              crassu
-    crav                 crav
-    crave                crave
-    craved               crave
-    craven               craven
-    cravens              craven
-    craves               crave
-    craveth              craveth
-    craving              crave
-    crawl                crawl
-    crawling             crawl
-    crawls               crawl
-    craz                 craz
-    crazed               craze
-    crazy                crazi
-    creaking             creak
-    cream                cream
-    create               creat
-    created              creat
-    creates              creat
-    creating             creat
-    creation             creation
-    creator              creator
-    creature             creatur
-    creatures            creatur
-    credence             credenc
-    credent              credent
-    credible             credibl
-    credit               credit
-    creditor             creditor
-    creditors            creditor
-    credo                credo
-    credulity            credul
-    credulous            credul
-    creed                creed
-    creek                creek
-    creeks               creek
-    creep                creep
-    creeping             creep
-    creeps               creep
-    crept                crept
-    crescent             crescent
-    crescive             cresciv
-    cressets             cresset
-    cressid              cressid
-    cressida             cressida
-    cressids             cressid
-    cressy               cressi
-    crest                crest
-    crested              crest
-    crestfall            crestfal
-    crestless            crestless
-    crests               crest
-    cretan               cretan
-    crete                crete
-    crevice              crevic
-    crew                 crew
-    crews                crew
-    crib                 crib
-    cribb                cribb
-    cribs                crib
-    cricket              cricket
-    crickets             cricket
-    cried                cri
-    criedst              criedst
-    crier                crier
-    cries                cri
-    criest               criest
-    crieth               crieth
-    crime                crime
-    crimeful             crime
-    crimeless            crimeless
-    crimes               crime
-    criminal             crimin
-    crimson              crimson
-    cringe               cring
-    cripple              crippl
-    crisp                crisp
-    crisped              crisp
-    crispian             crispian
-    crispianus           crispianu
-    crispin              crispin
-    critic               critic
-    critical             critic
-    critics              critic
-    croak                croak
-    croaking             croak
-    croaks               croak
-    crocodile            crocodil
-    cromer               cromer
-    cromwell             cromwel
-    crone                crone
-    crook                crook
-    crookback            crookback
-    crooked              crook
-    crooking             crook
-    crop                 crop
-    cropp                cropp
-    crosby               crosbi
-    cross                cross
-    crossed              cross
-    crosses              cross
-    crossest             crossest
-    crossing             cross
-    crossings            cross
-    crossly              crossli
-    crossness            cross
-    crost                crost
-    crotchets            crotchet
-    crouch               crouch
-    crouching            crouch
-    crow                 crow
-    crowd                crowd
-    crowded              crowd
-    crowding             crowd
-    crowds               crowd
-    crowflowers          crowflow
-    crowing              crow
-    crowkeeper           crowkeep
-    crown                crown
-    crowned              crown
-    crowner              crowner
-    crownet              crownet
-    crownets             crownet
-    crowning             crown
-    crowns               crown
-    crows                crow
-    crudy                crudi
-    cruel                cruel
-    cruell               cruell
-    crueller             crueller
-    cruelly              cruelli
-    cruels               cruel
-    cruelty              cruelti
-    crum                 crum
-    crumble              crumbl
-    crumbs               crumb
-    crupper              crupper
-    crusadoes            crusado
-    crush                crush
-    crushed              crush
-    crushest             crushest
-    crushing             crush
-    crust                crust
-    crusts               crust
-    crusty               crusti
-    crutch               crutch
-    crutches             crutch
-    cry                  cry
-    crying               cry
-    crystal              crystal
-    crystalline          crystallin
-    crystals             crystal
-    cub                  cub
-    cubbert              cubbert
-    cubiculo             cubiculo
-    cubit                cubit
-    cubs                 cub
-    cuckold              cuckold
-    cuckoldly            cuckoldli
-    cuckolds             cuckold
-    cuckoo               cuckoo
-    cucullus             cucullu
-    cudgel               cudgel
-    cudgeled             cudgel
-    cudgell              cudgel
-    cudgelling           cudgel
-    cudgels              cudgel
-    cue                  cue
-    cues                 cue
-    cuff                 cuff
-    cuffs                cuff
-    cuique               cuiqu
-    cull                 cull
-    culling              cull
-    cullion              cullion
-    cullionly            cullionli
-    cullions             cullion
-    culpable             culpabl
-    culverin             culverin
-    cum                  cum
-    cumber               cumber
-    cumberland           cumberland
-    cunning              cun
-    cunningly            cunningli
-    cunnings             cun
-    cuore                cuor
-    cup                  cup
-    cupbearer            cupbear
-    cupboarding          cupboard
-    cupid                cupid
-    cupids               cupid
-    cuppele              cuppel
-    cups                 cup
-    cur                  cur
-    curan                curan
-    curate               curat
-    curb                 curb
-    curbed               curb
-    curbing              curb
-    curbs                curb
-    curd                 curd
-    curdied              curdi
-    curds                curd
-    cure                 cure
-    cured                cure
-    cureless             cureless
-    curer                curer
-    cures                cure
-    curfew               curfew
-    curing               cure
-    curio                curio
-    curiosity            curios
-    curious              curiou
-    curiously            curious
-    curl                 curl
-    curled               curl
-    curling              curl
-    curls                curl
-    currance             curranc
-    currants             currant
-    current              current
-    currents             current
-    currish              currish
-    curry                curri
-    curs                 cur
-    curse                curs
-    cursed               curs
-    curses               curs
-    cursies              cursi
-    cursing              curs
-    cursorary            cursorari
-    curst                curst
-    curster              curster
-    curstest             curstest
-    curstness            curst
-    cursy                cursi
-    curtail              curtail
-    curtain              curtain
-    curtains             curtain
-    curtal               curtal
-    curtis               curti
-    curtle               curtl
-    curtsied             curtsi
-    curtsies             curtsi
-    curtsy               curtsi
-    curvet               curvet
-    curvets              curvet
-    cushes               cush
-    cushion              cushion
-    cushions             cushion
-    custalorum           custalorum
-    custard              custard
-    custody              custodi
-    custom               custom
-    customary            customari
-    customed             custom
-    customer             custom
-    customers            custom
-    customs              custom
-    custure              custur
-    cut                  cut
-    cutler               cutler
-    cutpurse             cutpurs
-    cutpurses            cutpurs
-    cuts                 cut
-    cutter               cutter
-    cutting              cut
-    cuttle               cuttl
-    cxsar                cxsar
-    cyclops              cyclop
-    cydnus               cydnu
-    cygnet               cygnet
-    cygnets              cygnet
-    cym                  cym
-    cymbals              cymbal
-    cymbeline            cymbelin
-    cyme                 cyme
-    cynic                cynic
-    cynthia              cynthia
-    cypress              cypress
-    cypriot              cypriot
-    cyprus               cypru
-    cyrus                cyru
-    cytherea             cytherea
-    d                    d
-    dabbled              dabbl
-    dace                 dace
-    dad                  dad
-    daedalus             daedalu
-    daemon               daemon
-    daff                 daff
-    daffed               daf
-    daffest              daffest
-    daffodils            daffodil
-    dagger               dagger
-    daggers              dagger
-    dagonet              dagonet
-    daily                daili
-    daintier             daintier
-    dainties             dainti
-    daintiest            daintiest
-    daintily             daintili
-    daintiness           dainti
-    daintry              daintri
-    dainty               dainti
-    daisied              daisi
-    daisies              daisi
-    daisy                daisi
-    dale                 dale
-    dalliance            dallianc
-    dallied              dalli
-    dallies              dalli
-    dally                dalli
-    dallying             dalli
-    dalmatians           dalmatian
-    dam                  dam
-    damage               damag
-    damascus             damascu
-    damask               damask
-    damasked             damask
-    dame                 dame
-    dames                dame
-    damm                 damm
-    damn                 damn
-    damnable             damnabl
-    damnably             damnabl
-    damnation            damnat
-    damned               damn
-    damns                damn
-    damoiselle           damoisel
-    damon                damon
-    damosella            damosella
-    damp                 damp
-    dams                 dam
-    damsel               damsel
-    damsons              damson
-    dan                  dan
-    danc                 danc
-    dance                danc
-    dancer               dancer
-    dances               danc
-    dancing              danc
-    dandle               dandl
-    dandy                dandi
-    dane                 dane
-    dang                 dang
-    danger               danger
-    dangerous            danger
-    dangerously          danger
-    dangers              danger
-    dangling             dangl
-    daniel               daniel
-    danish               danish
-    dank                 dank
-    dankish              dankish
-    danskers             dansker
-    daphne               daphn
-    dappled              dappl
-    dapples              dappl
-    dar                  dar
-    dardan               dardan
-    dardanian            dardanian
-    dardanius            dardaniu
-    dare                 dare
-    dared                dare
-    dareful              dare
-    dares                dare
-    darest               darest
-    daring               dare
-    darius               dariu
-    dark                 dark
-    darken               darken
-    darkening            darken
-    darkens              darken
-    darker               darker
-    darkest              darkest
-    darkling             darkl
-    darkly               darkli
-    darkness             dark
-    darling              darl
-    darlings             darl
-    darnel               darnel
-    darraign             darraign
-    dart                 dart
-    darted               dart
-    darter               darter
-    dartford             dartford
-    darting              dart
-    darts                dart
-    dash                 dash
-    dashes               dash
-    dashing              dash
-    dastard              dastard
-    dastards             dastard
-    dat                  dat
-    datchet              datchet
-    date                 date
-    dated                date
-    dateless             dateless
-    dates                date
-    daub                 daub
-    daughter             daughter
-    daughters            daughter
-    daunt                daunt
-    daunted              daunt
-    dauntless            dauntless
-    dauphin              dauphin
-    daventry             daventri
-    davy                 davi
-    daw                  daw
-    dawn                 dawn
-    dawning              dawn
-    daws                 daw
-    day                  dai
-    daylight             daylight
-    days                 dai
-    dazzle               dazzl
-    dazzled              dazzl
-    dazzling             dazzl
-    de                   de
-    dead                 dead
-    deadly               deadli
-    deaf                 deaf
-    deafing              deaf
-    deafness             deaf
-    deafs                deaf
-    deal                 deal
-    dealer               dealer
-    dealers              dealer
-    dealest              dealest
-    dealing              deal
-    dealings             deal
-    deals                deal
-    dealt                dealt
-    dean                 dean
-    deanery              deaneri
-    dear                 dear
-    dearer               dearer
-    dearest              dearest
-    dearly               dearli
-    dearness             dear
-    dears                dear
-    dearth               dearth
-    dearths              dearth
-    death                death
-    deathbed             deathb
-    deathful             death
-    deaths               death
-    deathsman            deathsman
-    deathsmen            deathsmen
-    debarred             debar
-    debase               debas
-    debate               debat
-    debated              debat
-    debatement           debat
-    debateth             debateth
-    debating             debat
-    debauch              debauch
-    debile               debil
-    debility             debil
-    debitor              debitor
-    debonair             debonair
-    deborah              deborah
-    debosh               debosh
-    debt                 debt
-    debted               debt
-    debtor               debtor
-    debtors              debtor
-    debts                debt
-    debuty               debuti
-    decay                decai
-    decayed              decai
-    decayer              decay
-    decaying             decai
-    decays               decai
-    deceas               decea
-    decease              deceas
-    deceased             deceas
-    deceit               deceit
-    deceitful            deceit
-    deceits              deceit
-    deceiv               deceiv
-    deceivable           deceiv
-    deceive              deceiv
-    deceived             deceiv
-    deceiver             deceiv
-    deceivers            deceiv
-    deceives             deceiv
-    deceivest            deceivest
-    deceiveth            deceiveth
-    deceiving            deceiv
-    december             decemb
-    decent               decent
-    deceptious           decepti
-    decerns              decern
-    decide               decid
-    decides              decid
-    decimation           decim
-    decipher             deciph
-    deciphers            deciph
-    decision             decis
-    decius               deciu
-    deck                 deck
-    decking              deck
-    decks                deck
-    deckt                deckt
-    declare              declar
-    declares             declar
-    declension           declens
-    declensions          declens
-    declin               declin
-    decline              declin
-    declined             declin
-    declines             declin
-    declining            declin
-    decoct               decoct
-    decorum              decorum
-    decreas              decrea
-    decrease             decreas
-    decreasing           decreas
-    decree               decre
-    decreed              decre
-    decrees              decre
-    decrepit             decrepit
-    dedicate             dedic
-    dedicated            dedic
-    dedicates            dedic
-    dedication           dedic
-    deed                 deed
-    deedless             deedless
-    deeds                deed
-    deem                 deem
-    deemed               deem
-    deep                 deep
-    deeper               deeper
-    deepest              deepest
-    deeply               deepli
-    deeps                deep
-    deepvow              deepvow
-    deer                 deer
-    deesse               deess
-    defac                defac
-    deface               defac
-    defaced              defac
-    defacer              defac
-    defacers             defac
-    defacing             defac
-    defam                defam
-    default              default
-    defeat               defeat
-    defeated             defeat
-    defeats              defeat
-    defeatures           defeatur
-    defect               defect
-    defective            defect
-    defects              defect
-    defence              defenc
-    defences             defenc
-    defend               defend
-    defendant            defend
-    defended             defend
-    defender             defend
-    defenders            defend
-    defending            defend
-    defends              defend
-    defense              defens
-    defensible           defens
-    defensive            defens
-    defer                defer
-    deferr               deferr
-    defiance             defianc
-    deficient            defici
-    defied               defi
-    defies               defi
-    defil                defil
-    defile               defil
-    defiler              defil
-    defiles              defil
-    defiling             defil
-    define               defin
-    definement           defin
-    definite             definit
-    definitive           definit
-    definitively         definit
-    deflow               deflow
-    deflower             deflow
-    deflowered           deflow
-    deform               deform
-    deformed             deform
-    deformities          deform
-    deformity            deform
-    deftly               deftli
-    defunct              defunct
-    defunction           defunct
-    defuse               defus
-    defy                 defi
-    defying              defi
-    degenerate           degener
-    degraded             degrad
-    degree               degre
-    degrees              degre
-    deified              deifi
-    deifying             deifi
-    deign                deign
-    deigned              deign
-    deiphobus            deiphobu
-    deities              deiti
-    deity                deiti
-    deja                 deja
-    deject               deject
-    dejected             deject
-    delabreth            delabreth
-    delay                delai
-    delayed              delai
-    delaying             delai
-    delays               delai
-    delectable           delect
-    deliberate           deliber
-    delicate             delic
-    delicates            delic
-    delicious            delici
-    deliciousness        delici
-    delight              delight
-    delighted            delight
-    delightful           delight
-    delights             delight
-    delinquents          delinqu
-    deliv                deliv
-    deliver              deliv
-    deliverance          deliver
-    delivered            deliv
-    delivering           deliv
-    delivers             deliv
-    delivery             deliveri
-    delphos              delpho
-    deluded              delud
-    deluding             delud
-    deluge               delug
-    delve                delv
-    delver               delver
-    delves               delv
-    demand               demand
-    demanded             demand
-    demanding            demand
-    demands              demand
-    demean               demean
-    demeanor             demeanor
-    demeanour            demeanour
-    demerits             demerit
-    demesnes             demesn
-    demetrius            demetriu
-    demi                 demi
-    demigod              demigod
-    demise               demis
-    demoiselles          demoisel
-    demon                demon
-    demonstrable         demonstr
-    demonstrate          demonstr
-    demonstrated         demonstr
-    demonstrating        demonstr
-    demonstration        demonstr
-    demonstrative        demonstr
-    demure               demur
-    demurely             demur
-    demuring             demur
-    den                  den
-    denay                denai
-    deni                 deni
-    denial               denial
-    denials              denial
-    denied               deni
-    denier               denier
-    denies               deni
-    deniest              deniest
-    denis                deni
-    denmark              denmark
-    dennis               denni
-    denny                denni
-    denote               denot
-    denoted              denot
-    denotement           denot
-    denounc              denounc
-    denounce             denounc
-    denouncing           denounc
-    dens                 den
-    denunciation         denunci
-    deny                 deni
-    denying              deni
-    deo                  deo
-    depart               depart
-    departed             depart
-    departest            departest
-    departing            depart
-    departure            departur
-    depeche              depech
-    depend               depend
-    dependant            depend
-    dependants           depend
-    depended             depend
-    dependence           depend
-    dependences          depend
-    dependency           depend
-    dependent            depend
-    dependents           depend
-    depender             depend
-    depending            depend
-    depends              depend
-    deplore              deplor
-    deploring            deplor
-    depopulate           depopul
-    depos                depo
-    depose               depos
-    deposed              depos
-    deposing             depos
-    depositaries         depositari
-    deprav               deprav
-    depravation          deprav
-    deprave              deprav
-    depraved             deprav
-    depraves             deprav
-    depress              depress
-    depriv               depriv
-    deprive              depriv
-    depth                depth
-    depths               depth
-    deputation           deput
-    depute               deput
-    deputed              deput
-    deputies             deputi
-    deputing             deput
-    deputy               deputi
-    deracinate           deracin
-    derby                derbi
-    dercetas             derceta
-    dere                 dere
-    derides              derid
-    derision             deris
-    deriv                deriv
-    derivation           deriv
-    derivative           deriv
-    derive               deriv
-    derived              deriv
-    derives              deriv
-    derogate             derog
-    derogately           derog
-    derogation           derog
-    des                  de
-    desartless           desartless
-    descant              descant
-    descend              descend
-    descended            descend
-    descending           descend
-    descends             descend
-    descension           descens
-    descent              descent
-    descents             descent
-    describe             describ
-    described            describ
-    describes            describ
-    descried             descri
-    description          descript
-    descriptions         descript
-    descry               descri
-    desdemon             desdemon
-    desdemona            desdemona
-    desert               desert
-    deserts              desert
-    deserv               deserv
-    deserve              deserv
-    deserved             deserv
-    deservedly           deservedli
-    deserver             deserv
-    deservers            deserv
-    deserves             deserv
-    deservest            deservest
-    deserving            deserv
-    deservings           deserv
-    design               design
-    designment           design
-    designments          design
-    designs              design
-    desir                desir
-    desire               desir
-    desired              desir
-    desirers             desir
-    desires              desir
-    desirest             desirest
-    desiring             desir
-    desirous             desir
-    desist               desist
-    desk                 desk
-    desolate             desol
-    desolation           desol
-    desp                 desp
-    despair              despair
-    despairing           despair
-    despairs             despair
-    despatch             despatch
-    desperate            desper
-    desperately          desper
-    desperation          desper
-    despis               despi
-    despise              despis
-    despised             despis
-    despiser             despis
-    despiseth            despiseth
-    despising            despis
-    despite              despit
-    despiteful           despit
-    despoiled            despoil
-    dest                 dest
-    destin               destin
-    destined             destin
-    destinies            destini
-    destiny              destini
-    destitute            destitut
-    destroy              destroi
-    destroyed            destroi
-    destroyer            destroy
-    destroyers           destroy
-    destroying           destroi
-    destroys             destroi
-    destruction          destruct
-    destructions         destruct
-    det                  det
-    detain               detain
-    detains              detain
-    detect               detect
-    detected             detect
-    detecting            detect
-    detection            detect
-    detector             detector
-    detects              detect
-    detention            detent
-    determin             determin
-    determinate          determin
-    determination        determin
-    determinations       determin
-    determine            determin
-    determined           determin
-    determines           determin
-    detest               detest
-    detestable           detest
-    detested             detest
-    detesting            detest
-    detests              detest
-    detract              detract
-    detraction           detract
-    detractions          detract
-    deucalion            deucalion
-    deuce                deuc
-    deum                 deum
-    deux                 deux
-    devant               devant
-    devesting            devest
-    device               devic
-    devices              devic
-    devil                devil
-    devilish             devilish
-    devils               devil
-    devis                devi
-    devise               devis
-    devised              devis
-    devises              devis
-    devising             devis
-    devoid               devoid
-    devonshire           devonshir
-    devote               devot
-    devoted              devot
-    devotion             devot
-    devour               devour
-    devoured             devour
-    devourers            devour
-    devouring            devour
-    devours              devour
-    devout               devout
-    devoutly             devoutli
-    dew                  dew
-    dewberries           dewberri
-    dewdrops             dewdrop
-    dewlap               dewlap
-    dewlapp              dewlapp
-    dews                 dew
-    dewy                 dewi
-    dexter               dexter
-    dexteriously         dexteri
-    dexterity            dexter
-    di                   di
-    diable               diabl
-    diablo               diablo
-    diadem               diadem
-    dial                 dial
-    dialect              dialect
-    dialogue             dialogu
-    dialogued            dialogu
-    dials                dial
-    diameter             diamet
-    diamond              diamond
-    diamonds             diamond
-    dian                 dian
-    diana                diana
-    diaper               diaper
-    dibble               dibbl
-    dic                  dic
-    dice                 dice
-    dicers               dicer
-    dich                 dich
-    dick                 dick
-    dickens              dicken
-    dickon               dickon
-    dicky                dicki
-    dictator             dictat
-    diction              diction
-    dictynna             dictynna
-    did                  did
-    diddle               diddl
-    didest               didest
-    dido                 dido
-    didst                didst
-    die                  die
-    died                 di
-    diedst               diedst
-    dies                 di
-    diest                diest
-    diet                 diet
-    dieted               diet
-    dieter               dieter
-    dieu                 dieu
-    diff                 diff
-    differ               differ
-    difference           differ
-    differences          differ
-    differency           differ
-    different            differ
-    differing            differ
-    differs              differ
-    difficile            difficil
-    difficult            difficult
-    difficulties         difficulti
-    difficulty           difficulti
-    diffidence           diffid
-    diffidences          diffid
-    diffus               diffu
-    diffused             diffus
-    diffusest            diffusest
-    dig                  dig
-    digest               digest
-    digested             digest
-    digestion            digest
-    digestions           digest
-    digg                 digg
-    digging              dig
-    dighton              dighton
-    dignified            dignifi
-    dignifies            dignifi
-    dignify              dignifi
-    dignities            digniti
-    dignity              digniti
-    digress              digress
-    digressing           digress
-    digression           digress
-    digs                 dig
-    digt                 digt
-    dilate               dilat
-    dilated              dilat
-    dilations            dilat
-    dilatory             dilatori
-    dild                 dild
-    dildos               dildo
-    dilemma              dilemma
-    dilemmas             dilemma
-    diligence            dilig
-    diligent             dilig
-    diluculo             diluculo
-    dim                  dim
-    dimension            dimens
-    dimensions           dimens
-    diminish             diminish
-    diminishing          diminish
-    diminution           diminut
-    diminutive           diminut
-    diminutives          diminut
-    dimm                 dimm
-    dimmed               dim
-    dimming              dim
-    dimpled              dimpl
-    dimples              dimpl
-    dims                 dim
-    din                  din
-    dine                 dine
-    dined                dine
-    diner                diner
-    dines                dine
-    ding                 ding
-    dining               dine
-    dinner               dinner
-    dinners              dinner
-    dinnertime           dinnertim
-    dint                 dint
-    diomed               diom
-    diomede              diomed
-    diomedes             diomed
-    dion                 dion
-    dip                  dip
-    dipp                 dipp
-    dipping              dip
-    dips                 dip
-    dir                  dir
-    dire                 dire
-    direct               direct
-    directed             direct
-    directing            direct
-    direction            direct
-    directions           direct
-    directitude          directitud
-    directive            direct
-    directly             directli
-    directs              direct
-    direful              dire
-    direness             dire
-    direst               direst
-    dirge                dirg
-    dirges               dirg
-    dirt                 dirt
-    dirty                dirti
-    dis                  di
-    disability           disabl
-    disable              disabl
-    disabled             disabl
-    disabling            disabl
-    disadvantage         disadvantag
-    disagree             disagre
-    disallow             disallow
-    disanimates          disanim
-    disannul             disannul
-    disannuls            disannul
-    disappointed         disappoint
-    disarm               disarm
-    disarmed             disarm
-    disarmeth            disarmeth
-    disarms              disarm
-    disaster             disast
-    disasters            disast
-    disastrous           disastr
-    disbench             disbench
-    disbranch            disbranch
-    disburdened          disburden
-    disburs              disbur
-    disburse             disburs
-    disbursed            disburs
-    discandy             discandi
-    discandying          discandi
-    discard              discard
-    discarded            discard
-    discase              discas
-    discased             discas
-    discern              discern
-    discerner            discern
-    discerning           discern
-    discernings          discern
-    discerns             discern
-    discharg             discharg
-    discharge            discharg
-    discharged           discharg
-    discharging          discharg
-    discipled            discipl
-    disciples            discipl
-    disciplin            disciplin
-    discipline           disciplin
-    disciplined          disciplin
-    disciplines          disciplin
-    disclaim             disclaim
-    disclaiming          disclaim
-    disclaims            disclaim
-    disclos              disclo
-    disclose             disclos
-    disclosed            disclos
-    discloses            disclos
-    discolour            discolour
-    discoloured          discolour
-    discolours           discolour
-    discomfit            discomfit
-    discomfited          discomfit
-    discomfiture         discomfitur
-    discomfort           discomfort
-    discomfortable       discomfort
-    discommend           discommend
-    disconsolate         disconsol
-    discontent           discont
-    discontented         discont
-    discontentedly       discontentedli
-    discontenting        discont
-    discontents          discont
-    discontinue          discontinu
-    discontinued         discontinu
-    discord              discord
-    discordant           discord
-    discords             discord
-    discourse            discours
-    discoursed           discours
-    discourser           discours
-    discourses           discours
-    discoursive          discours
-    discourtesy          discourtesi
-    discov               discov
-    discover             discov
-    discovered           discov
-    discoverers          discover
-    discoveries          discoveri
-    discovering          discov
-    discovers            discov
-    discovery            discoveri
-    discredit            discredit
-    discredited          discredit
-    discredits           discredit
-    discreet             discreet
-    discreetly           discreetli
-    discretion           discret
-    discretions          discret
-    discuss              discuss
-    disdain              disdain
-    disdained            disdain
-    disdaineth           disdaineth
-    disdainful           disdain
-    disdainfully         disdainfulli
-    disdaining           disdain
-    disdains             disdain
-    disdnguish           disdnguish
-    diseas               disea
-    disease              diseas
-    diseased             diseas
-    diseases             diseas
-    disedg               disedg
-    disembark            disembark
-    disfigure            disfigur
-    disfigured           disfigur
-    disfurnish           disfurnish
-    disgorge             disgorg
-    disgrac              disgrac
-    disgrace             disgrac
-    disgraced            disgrac
-    disgraceful          disgrac
-    disgraces            disgrac
-    disgracing           disgrac
-    disgracious          disgraci
-    disguis              disgui
-    disguise             disguis
-    disguised            disguis
-    disguiser            disguis
-    disguises            disguis
-    disguising           disguis
-    dish                 dish
-    dishabited           dishabit
-    dishclout            dishclout
-    dishearten           dishearten
-    disheartens          dishearten
-    dishes               dish
-    dishonest            dishonest
-    dishonestly          dishonestli
-    dishonesty           dishonesti
-    dishonor             dishonor
-    dishonorable         dishonor
-    dishonors            dishonor
-    dishonour            dishonour
-    dishonourable        dishonour
-    dishonoured          dishonour
-    dishonours           dishonour
-    disinherit           disinherit
-    disinherited         disinherit
-    disjoin              disjoin
-    disjoining           disjoin
-    disjoins             disjoin
-    disjoint             disjoint
-    disjunction          disjunct
-    dislik               dislik
-    dislike              dislik
-    disliken             disliken
-    dislikes             dislik
-    dislimns             dislimn
-    dislocate            disloc
-    dislodg              dislodg
-    disloyal             disloy
-    disloyalty           disloyalti
-    dismal               dismal
-    dismantle            dismantl
-    dismantled           dismantl
-    dismask              dismask
-    dismay               dismai
-    dismayed             dismai
-    dismemb              dismemb
-    dismember            dismemb
-    dismes               dism
-    dismiss              dismiss
-    dismissed            dismiss
-    dismissing           dismiss
-    dismission           dismiss
-    dismount             dismount
-    dismounted           dismount
-    disnatur             disnatur
-    disobedience         disobedi
-    disobedient          disobedi
-    disobey              disobei
-    disobeys             disobei
-    disorb               disorb
-    disorder             disord
-    disordered           disord
-    disorderly           disorderli
-    disorders            disord
-    disparage            disparag
-    disparagement        disparag
-    disparagements       disparag
-    dispark              dispark
-    dispatch             dispatch
-    dispensation         dispens
-    dispense             dispens
-    dispenses            dispens
-    dispers              disper
-    disperse             dispers
-    dispersed            dispers
-    dispersedly          dispersedli
-    dispersing           dispers
-    dispiteous           dispit
-    displac              displac
-    displace             displac
-    displaced            displac
-    displant             displant
-    displanting          displant
-    display              displai
-    displayed            displai
-    displeas             displea
-    displease            displeas
-    displeased           displeas
-    displeasing          displeas
-    displeasure          displeasur
-    displeasures         displeasur
-    disponge             dispong
-    disport              disport
-    disports             disport
-    dispos               dispo
-    dispose              dispos
-    disposed             dispos
-    disposer             dispos
-    disposing            dispos
-    disposition          disposit
-    dispositions         disposit
-    dispossess           dispossess
-    dispossessing        dispossess
-    disprais             disprai
-    dispraise            disprais
-    dispraising          disprais
-    dispraisingly        dispraisingli
-    dispropertied        disproperti
-    disproportion        disproport
-    disproportioned      disproport
-    disprov              disprov
-    disprove             disprov
-    disproved            disprov
-    dispursed            dispurs
-    disputable           disput
-    disputation          disput
-    disputations         disput
-    dispute              disput
-    disputed             disput
-    disputes             disput
-    disputing            disput
-    disquantity          disquant
-    disquiet             disquiet
-    disquietly           disquietli
-    disrelish            disrelish
-    disrobe              disrob
-    disseat              disseat
-    dissemble            dissembl
-    dissembled           dissembl
-    dissembler           dissembl
-    dissemblers          dissembl
-    dissembling          dissembl
-    dissembly            dissembl
-    dissension           dissens
-    dissensions          dissens
-    dissentious          dissenti
-    dissever             dissev
-    dissipation          dissip
-    dissolute            dissolut
-    dissolutely          dissolut
-    dissolution          dissolut
-    dissolutions         dissolut
-    dissolv              dissolv
-    dissolve             dissolv
-    dissolved            dissolv
-    dissolves            dissolv
-    dissuade             dissuad
-    dissuaded            dissuad
-    distaff              distaff
-    distaffs             distaff
-    distain              distain
-    distains             distain
-    distance             distanc
-    distant              distant
-    distaste             distast
-    distasted            distast
-    distasteful          distast
-    distemp              distemp
-    distemper            distemp
-    distemperature       distemperatur
-    distemperatures      distemperatur
-    distempered          distemp
-    distempering         distemp
-    distil               distil
-    distill              distil
-    distillation         distil
-    distilled            distil
-    distills             distil
-    distilment           distil
-    distinct             distinct
-    distinction          distinct
-    distinctly           distinctli
-    distingue            distingu
-    distinguish          distinguish
-    distinguishes        distinguish
-    distinguishment      distinguish
-    distract             distract
-    distracted           distract
-    distractedly         distractedli
-    distraction          distract
-    distractions         distract
-    distracts            distract
-    distrain             distrain
-    distraught           distraught
-    distress             distress
-    distressed           distress
-    distresses           distress
-    distressful          distress
-    distribute           distribut
-    distributed          distribut
-    distribution         distribut
-    distrust             distrust
-    distrustful          distrust
-    disturb              disturb
-    disturbed            disturb
-    disturbers           disturb
-    disturbing           disturb
-    disunite             disunit
-    disvalued            disvalu
-    disvouch             disvouch
-    dit                  dit
-    ditch                ditch
-    ditchers             ditcher
-    ditches              ditch
-    dites                dite
-    ditties              ditti
-    ditty                ditti
-    diurnal              diurnal
-    div                  div
-    dive                 dive
-    diver                diver
-    divers               diver
-    diversely            divers
-    diversity            divers
-    divert               divert
-    diverted             divert
-    diverts              divert
-    dives                dive
-    divest               divest
-    dividable            divid
-    dividant             divid
-    divide               divid
-    divided              divid
-    divides              divid
-    divideth             divideth
-    divin                divin
-    divination           divin
-    divine               divin
-    divinely             divin
-    divineness           divin
-    diviner              divin
-    divines              divin
-    divinest             divinest
-    divining             divin
-    divinity             divin
-    division             divis
-    divisions            divis
-    divorc               divorc
-    divorce              divorc
-    divorced             divorc
-    divorcement          divorc
-    divorcing            divorc
-    divulg               divulg
-    divulge              divulg
-    divulged             divulg
-    divulging            divulg
-    dizy                 dizi
-    dizzy                dizzi
-    do                   do
-    doating              doat
-    dobbin               dobbin
-    dock                 dock
-    docks                dock
-    doct                 doct
-    doctor               doctor
-    doctors              doctor
-    doctrine             doctrin
-    document             document
-    dodge                dodg
-    doe                  doe
-    doer                 doer
-    doers                doer
-    does                 doe
-    doest                doest
-    doff                 doff
-    dog                  dog
-    dogberry             dogberri
-    dogfish              dogfish
-    dogg                 dogg
-    dogged               dog
-    dogs                 dog
-    doigts               doigt
-    doing                do
-    doings               do
-    doit                 doit
-    doits                doit
-    dolabella            dolabella
-    dole                 dole
-    doleful              dole
-    doll                 doll
-    dollar               dollar
-    dollars              dollar
-    dolor                dolor
-    dolorous             dolor
-    dolour               dolour
-    dolours              dolour
-    dolphin              dolphin
-    dolt                 dolt
-    dolts                dolt
-    domestic             domest
-    domestics            domest
-    dominance            domin
-    dominations          domin
-    dominator            domin
-    domine               domin
-    domineer             domin
-    domineering          domin
-    dominical            domin
-    dominion             dominion
-    dominions            dominion
-    domitius             domitiu
-    dommelton            dommelton
-    don                  don
-    donalbain            donalbain
-    donation             donat
-    donc                 donc
-    doncaster            doncast
-    done                 done
-    dong                 dong
-    donn                 donn
-    donne                donn
-    donner               donner
-    donnerai             donnerai
-    doom                 doom
-    doomsday             doomsdai
-    door                 door
-    doorkeeper           doorkeep
-    doors                door
-    dorcas               dorca
-    doreus               doreu
-    doricles             doricl
-    dormouse             dormous
-    dorothy              dorothi
-    dorset               dorset
-    dorsetshire          dorsetshir
-    dost                 dost
-    dotage               dotag
-    dotant               dotant
-    dotard               dotard
-    dotards              dotard
-    dote                 dote
-    doted                dote
-    doters               doter
-    dotes                dote
-    doteth               doteth
-    doth                 doth
-    doting               dote
-    double               doubl
-    doubled              doubl
-    doubleness           doubl
-    doubler              doubler
-    doublet              doublet
-    doublets             doublet
-    doubling             doubl
-    doubly               doubli
-    doubt                doubt
-    doubted              doubt
-    doubtful             doubt
-    doubtfully           doubtfulli
-    doubting             doubt
-    doubtless            doubtless
-    doubts               doubt
-    doug                 doug
-    dough                dough
-    doughty              doughti
-    doughy               doughi
-    douglas              dougla
-    dout                 dout
-    doute                dout
-    douts                dout
-    dove                 dove
-    dovehouse            dovehous
-    dover                dover
-    doves                dove
-    dow                  dow
-    dowager              dowag
-    dowdy                dowdi
-    dower                dower
-    dowerless            dowerless
-    dowers               dower
-    dowlas               dowla
-    dowle                dowl
-    down                 down
-    downfall             downfal
-    downright            downright
-    downs                down
-    downstairs           downstair
-    downtrod             downtrod
-    downward             downward
-    downwards            downward
-    downy                downi
-    dowries              dowri
-    dowry                dowri
-    dowsabel             dowsabel
-    doxy                 doxi
-    dozed                doze
-    dozen                dozen
-    dozens               dozen
-    dozy                 dozi
-    drab                 drab
-    drabbing             drab
-    drabs                drab
-    drachma              drachma
-    drachmas             drachma
-    draff                draff
-    drag                 drag
-    dragg                dragg
-    dragged              drag
-    dragging             drag
-    dragon               dragon
-    dragonish            dragonish
-    dragons              dragon
-    drain                drain
-    drained              drain
-    drains               drain
-    drake                drake
-    dram                 dram
-    dramatis             dramati
-    drank                drank
-    draught              draught
-    draughts             draught
-    drave                drave
-    draw                 draw
-    drawbridge           drawbridg
-    drawer               drawer
-    drawers              drawer
-    draweth              draweth
-    drawing              draw
-    drawling             drawl
-    drawn                drawn
-    draws                draw
-    drayman              drayman
-    draymen              draymen
-    dread                dread
-    dreaded              dread
-    dreadful             dread
-    dreadfully           dreadfulli
-    dreading             dread
-    dreads               dread
-    dream                dream
-    dreamer              dreamer
-    dreamers             dreamer
-    dreaming             dream
-    dreams               dream
-    dreamt               dreamt
-    drearning            drearn
-    dreary               dreari
-    dreg                 dreg
-    dregs                dreg
-    drench               drench
-    drenched             drench
-    dress                dress
-    dressed              dress
-    dresser              dresser
-    dressing             dress
-    dressings            dress
-    drest                drest
-    drew                 drew
-    dribbling            dribbl
-    dried                dri
-    drier                drier
-    dries                dri
-    drift                drift
-    drily                drili
-    drink                drink
-    drinketh             drinketh
-    drinking             drink
-    drinkings            drink
-    drinks               drink
-    driv                 driv
-    drive                drive
-    drivelling           drivel
-    driven               driven
-    drives               drive
-    driveth              driveth
-    driving              drive
-    drizzle              drizzl
-    drizzled             drizzl
-    drizzles             drizzl
-    droit                droit
-    drollery             drolleri
-    dromio               dromio
-    dromios              dromio
-    drone                drone
-    drones               drone
-    droop                droop
-    droopeth             droopeth
-    drooping             droop
-    droops               droop
-    drop                 drop
-    dropheir             dropheir
-    droplets             droplet
-    dropp                dropp
-    dropper              dropper
-    droppeth             droppeth
-    dropping             drop
-    droppings            drop
-    drops                drop
-    dropsied             dropsi
-    dropsies             dropsi
-    dropsy               dropsi
-    dropt                dropt
-    dross                dross
-    drossy               drossi
-    drought              drought
-    drove                drove
-    droven               droven
-    drovier              drovier
-    drown                drown
-    drowned              drown
-    drowning             drown
-    drowns               drown
-    drows                drow
-    drowse               drows
-    drowsily             drowsili
-    drowsiness           drowsi
-    drowsy               drowsi
-    drudge               drudg
-    drudgery             drudgeri
-    drudges              drudg
-    drug                 drug
-    drugg                drugg
-    drugs                drug
-    drum                 drum
-    drumble              drumbl
-    drummer              drummer
-    drumming             drum
-    drums                drum
-    drunk                drunk
-    drunkard             drunkard
-    drunkards            drunkard
-    drunken              drunken
-    drunkenly            drunkenli
-    drunkenness          drunken
-    dry                  dry
-    dryness              dryness
-    dst                  dst
-    du                   du
-    dub                  dub
-    dubb                 dubb
-    ducat                ducat
-    ducats               ducat
-    ducdame              ducdam
-    duchess              duchess
-    duchies              duchi
-    duchy                duchi
-    duck                 duck
-    ducking              duck
-    ducks                duck
-    dudgeon              dudgeon
-    due                  due
-    duellist             duellist
-    duello               duello
-    duer                 duer
-    dues                 due
-    duff                 duff
-    dug                  dug
-    dugs                 dug
-    duke                 duke
-    dukedom              dukedom
-    dukedoms             dukedom
-    dukes                duke
-    dulcet               dulcet
-    dulche               dulch
-    dull                 dull
-    dullard              dullard
-    duller               duller
-    dullest              dullest
-    dulling              dull
-    dullness             dull
-    dulls                dull
-    dully                dulli
-    dulness              dul
-    duly                 duli
-    dumain               dumain
-    dumb                 dumb
-    dumbe                dumb
-    dumbly               dumbl
-    dumbness             dumb
-    dump                 dump
-    dumps                dump
-    dun                  dun
-    duncan               duncan
-    dung                 dung
-    dungeon              dungeon
-    dungeons             dungeon
-    dunghill             dunghil
-    dunghills            dunghil
-    dungy                dungi
-    dunnest              dunnest
-    dunsinane            dunsinan
-    dunsmore             dunsmor
-    dunstable            dunstabl
-    dupp                 dupp
-    durance              duranc
-    during               dure
-    durst                durst
-    dusky                duski
-    dust                 dust
-    dusted               dust
-    dusty                dusti
-    dutch                dutch
-    dutchman             dutchman
-    duteous              duteou
-    duties               duti
-    dutiful              duti
-    duty                 duti
-    dwarf                dwarf
-    dwarfish             dwarfish
-    dwell                dwell
-    dwellers             dweller
-    dwelling             dwell
-    dwells               dwell
-    dwelt                dwelt
-    dwindle              dwindl
-    dy                   dy
-    dye                  dye
-    dyed                 dy
-    dyer                 dyer
-    dying                dy
-    e                    e
-    each                 each
-    eager                eager
-    eagerly              eagerli
-    eagerness            eager
-    eagle                eagl
-    eagles               eagl
-    eaning               ean
-    eanlings             eanl
-    ear                  ear
-    earing               ear
-    earl                 earl
-    earldom              earldom
-    earlier              earlier
-    earliest             earliest
-    earliness            earli
-    earls                earl
-    early                earli
-    earn                 earn
-    earned               earn
-    earnest              earnest
-    earnestly            earnestli
-    earnestness          earnest
-    earns                earn
-    ears                 ear
-    earth                earth
-    earthen              earthen
-    earthlier            earthlier
-    earthly              earthli
-    earthquake           earthquak
-    earthquakes          earthquak
-    earthy               earthi
-    eas                  ea
-    ease                 eas
-    eased                eas
-    easeful              eas
-    eases                eas
-    easier               easier
-    easiest              easiest
-    easiliest            easiliest
-    easily               easili
-    easiness             easi
-    easing               eas
-    east                 east
-    eastcheap            eastcheap
-    easter               easter
-    eastern              eastern
-    eastward             eastward
-    easy                 easi
-    eat                  eat
-    eaten                eaten
-    eater                eater
-    eaters               eater
-    eating               eat
-    eats                 eat
-    eaux                 eaux
-    eaves                eav
-    ebb                  ebb
-    ebbing               eb
-    ebbs                 ebb
-    ebon                 ebon
-    ebony                eboni
-    ebrew                ebrew
-    ecce                 ecc
-    echapper             echapp
-    echo                 echo
-    echoes               echo
-    eclips               eclip
-    eclipse              eclips
-    eclipses             eclips
-    ecolier              ecoli
-    ecoutez              ecoutez
-    ecstacy              ecstaci
-    ecstasies            ecstasi
-    ecstasy              ecstasi
-    ecus                 ecu
-    eden                 eden
-    edg                  edg
-    edgar                edgar
-    edge                 edg
-    edged                edg
-    edgeless             edgeless
-    edges                edg
-    edict                edict
-    edicts               edict
-    edifice              edific
-    edifices             edific
-    edified              edifi
-    edifies              edifi
-    edition              edit
-    edm                  edm
-    edmund               edmund
-    edmunds              edmund
-    edmundsbury          edmundsburi
-    educate              educ
-    educated             educ
-    education            educ
-    edward               edward
-    eel                  eel
-    eels                 eel
-    effect               effect
-    effected             effect
-    effectless           effectless
-    effects              effect
-    effectual            effectu
-    effectually          effectu
-    effeminate           effemin
-    effigies             effigi
-    effus                effu
-    effuse               effus
-    effusion             effus
-    eftest               eftest
-    egal                 egal
-    egally               egal
-    eget                 eget
-    egeus                egeu
-    egg                  egg
-    eggs                 egg
-    eggshell             eggshel
-    eglamour             eglamour
-    eglantine            eglantin
-    egma                 egma
-    ego                  ego
-    egregious            egregi
-    egregiously          egregi
-    egress               egress
-    egypt                egypt
-    egyptian             egyptian
-    egyptians            egyptian
-    eie                  eie
-    eight                eight
-    eighteen             eighteen
-    eighth               eighth
-    eightpenny           eightpenni
-    eighty               eighti
-    eisel                eisel
-    either               either
-    eject                eject
-    eke                  ek
-    el                   el
-    elbe                 elb
-    elbow                elbow
-    elbows               elbow
-    eld                  eld
-    elder                elder
-    elders               elder
-    eldest               eldest
-    eleanor              eleanor
-    elect                elect
-    elected              elect
-    election             elect
-    elegancy             eleg
-    elegies              elegi
-    element              element
-    elements             element
-    elephant             eleph
-    elephants            eleph
-    elevated             elev
-    eleven               eleven
-    eleventh             eleventh
-    elf                  elf
-    elflocks             elflock
-    eliads               eliad
-    elinor               elinor
-    elizabeth            elizabeth
-    ell                  ell
-    elle                 ell
-    ellen                ellen
-    elm                  elm
-    eloquence            eloqu
-    eloquent             eloqu
-    else                 els
-    elsewhere            elsewher
-    elsinore             elsinor
-    eltham               eltham
-    elves                elv
-    elvish               elvish
-    ely                  eli
-    elysium              elysium
-    em                   em
-    emballing            embal
-    embalm               embalm
-    embalms              embalm
-    embark               embark
-    embarked             embark
-    embarquements        embarqu
-    embassade            embassad
-    embassage            embassag
-    embassies            embassi
-    embassy              embassi
-    embattailed          embattail
-    embattl              embattl
-    embattle             embattl
-    embay                embai
-    embellished          embellish
-    embers               ember
-    emblaze              emblaz
-    emblem               emblem
-    emblems              emblem
-    embodied             embodi
-    embold               embold
-    emboldens            embolden
-    emboss               emboss
-    embossed             emboss
-    embounded            embound
-    embowel              embowel
-    embowell             embowel
-    embrac               embrac
-    embrace              embrac
-    embraced             embrac
-    embracement          embrac
-    embracements         embrac
-    embraces             embrac
-    embracing            embrac
-    embrasures           embrasur
-    embroider            embroid
-    embroidery           embroideri
-    emhracing            emhrac
-    emilia               emilia
-    eminence             emin
-    eminent              emin
-    eminently            emin
-    emmanuel             emmanuel
-    emnity               emniti
-    empale               empal
-    emperal              emper
-    emperess             emperess
-    emperial             emperi
-    emperor              emperor
-    empery               emperi
-    emphasis             emphasi
-    empire               empir
-    empirics             empir
-    empiricutic          empiricut
-    empleached           empleach
-    employ               emploi
-    employed             emploi
-    employer             employ
-    employment           employ
-    employments          employ
-    empoison             empoison
-    empress              empress
-    emptied              empti
-    emptier              emptier
-    empties              empti
-    emptiness            empti
-    empty                empti
-    emptying             empti
-    emulate              emul
-    emulation            emul
-    emulations           emul
-    emulator             emul
-    emulous              emul
-    en                   en
-    enact                enact
-    enacted              enact
-    enacts               enact
-    enactures            enactur
-    enamell              enamel
-    enamelled            enamel
-    enamour              enamour
-    enamoured            enamour
-    enanmour             enanmour
-    encamp               encamp
-    encamped             encamp
-    encave               encav
-    enceladus            enceladu
-    enchaf               enchaf
-    enchafed             enchaf
-    enchant              enchant
-    enchanted            enchant
-    enchanting           enchant
-    enchantingly         enchantingli
-    enchantment          enchant
-    enchantress          enchantress
-    enchants             enchant
-    enchas               encha
-    encircle             encircl
-    encircled            encircl
-    enclos               enclo
-    enclose              enclos
-    enclosed             enclos
-    encloses             enclos
-    encloseth            encloseth
-    enclosing            enclos
-    enclouded            encloud
-    encompass            encompass
-    encompassed          encompass
-    encompasseth         encompasseth
-    encompassment        encompass
-    encore               encor
-    encorporal           encorpor
-    encount              encount
-    encounter            encount
-    encountered          encount
-    encounters           encount
-    encourage            encourag
-    encouraged           encourag
-    encouragement        encourag
-    encrimsoned          encrimson
-    encroaching          encroach
-    encumb               encumb
-    end                  end
-    endamage             endamag
-    endamagement         endamag
-    endanger             endang
-    endart               endart
-    endear               endear
-    endeared             endear
-    endeavour            endeavour
-    endeavours           endeavour
-    ended                end
-    ender                ender
-    ending               end
-    endings              end
-    endite               endit
-    endless              endless
-    endow                endow
-    endowed              endow
-    endowments           endow
-    endows               endow
-    ends                 end
-    endu                 endu
-    endue                endu
-    endur                endur
-    endurance            endur
-    endure               endur
-    endured              endur
-    endures              endur
-    enduring             endur
-    endymion             endymion
-    eneas                enea
-    enemies              enemi
-    enemy                enemi
-    enernies             enerni
-    enew                 enew
-    enfeebled            enfeebl
-    enfeebles            enfeebl
-    enfeoff              enfeoff
-    enfetter             enfett
-    enfoldings           enfold
-    enforc               enforc
-    enforce              enforc
-    enforced             enforc
-    enforcedly           enforcedli
-    enforcement          enforc
-    enforces             enforc
-    enforcest            enforcest
-    enfranched           enfranch
-    enfranchis           enfranchi
-    enfranchise          enfranchis
-    enfranchised         enfranchis
-    enfranchisement      enfranchis
-    enfreed              enfre
-    enfreedoming         enfreedom
-    engag                engag
-    engage               engag
-    engaged              engag
-    engagements          engag
-    engaging             engag
-    engaol               engaol
-    engend               engend
-    engender             engend
-    engenders            engend
-    engilds              engild
-    engine               engin
-    engineer             engin
-    enginer              engin
-    engines              engin
-    engirt               engirt
-    england              england
-    english              english
-    englishman           englishman
-    englishmen           englishmen
-    engluts              englut
-    englutted            englut
-    engraffed            engraf
-    engraft              engraft
-    engrafted            engraft
-    engrav               engrav
-    engrave              engrav
-    engross              engross
-    engrossed            engross
-    engrossest           engrossest
-    engrossing           engross
-    engrossments         engross
-    enguard              enguard
-    enigma               enigma
-    enigmatical          enigmat
-    enjoin               enjoin
-    enjoined             enjoin
-    enjoy                enjoi
-    enjoyed              enjoi
-    enjoyer              enjoy
-    enjoying             enjoi
-    enjoys               enjoi
-    enkindle             enkindl
-    enkindled            enkindl
-    enlard               enlard
-    enlarg               enlarg
-    enlarge              enlarg
-    enlarged             enlarg
-    enlargement          enlarg
-    enlargeth            enlargeth
-    enlighten            enlighten
-    enlink               enlink
-    enmesh               enmesh
-    enmities             enmiti
-    enmity               enmiti
-    ennoble              ennobl
-    ennobled             ennobl
-    enobarb              enobarb
-    enobarbus            enobarbu
-    enon                 enon
-    enormity             enorm
-    enormous             enorm
-    enough               enough
-    enow                 enow
-    enpatron             enpatron
-    enpierced            enpierc
-    enquir               enquir
-    enquire              enquir
-    enquired             enquir
-    enrag                enrag
-    enrage               enrag
-    enraged              enrag
-    enrages              enrag
-    enrank               enrank
-    enrapt               enrapt
-    enrich               enrich
-    enriched             enrich
-    enriches             enrich
-    enridged             enridg
-    enrings              enr
-    enrob                enrob
-    enrobe               enrob
-    enroll               enrol
-    enrolled             enrol
-    enrooted             enroot
-    enrounded            enround
-    enschedul            enschedul
-    ensconce             ensconc
-    ensconcing           ensconc
-    enseamed             enseam
-    ensear               ensear
-    enseigne             enseign
-    enseignez            enseignez
-    ensemble             ensembl
-    enshelter            enshelt
-    enshielded           enshield
-    enshrines            enshrin
-    ensign               ensign
-    ensigns              ensign
-    enskied              enski
-    ensman               ensman
-    ensnare              ensnar
-    ensnared             ensnar
-    ensnareth            ensnareth
-    ensteep              ensteep
-    ensu                 ensu
-    ensue                ensu
-    ensued               ensu
-    ensues               ensu
-    ensuing              ensu
-    enswathed            enswath
-    ent                  ent
-    entail               entail
-    entame               entam
-    entangled            entangl
-    entangles            entangl
-    entendre             entendr
-    enter                enter
-    entered              enter
-    entering             enter
-    enterprise           enterpris
-    enterprises          enterpris
-    enters               enter
-    entertain            entertain
-    entertained          entertain
-    entertainer          entertain
-    entertaining         entertain
-    entertainment        entertain
-    entertainments       entertain
-    enthrall             enthral
-    enthralled           enthral
-    enthron              enthron
-    enthroned            enthron
-    entice               entic
-    enticements          entic
-    enticing             entic
-    entire               entir
-    entirely             entir
-    entitle              entitl
-    entitled             entitl
-    entitling            entitl
-    entomb               entomb
-    entombed             entomb
-    entrails             entrail
-    entrance             entranc
-    entrances            entranc
-    entrap               entrap
-    entrapp              entrapp
-    entre                entr
-    entreat              entreat
-    entreated            entreat
-    entreaties           entreati
-    entreating           entreat
-    entreatments         entreat
-    entreats             entreat
-    entreaty             entreati
-    entrench             entrench
-    entry                entri
-    entwist              entwist
-    envelop              envelop
-    envenom              envenom
-    envenomed            envenom
-    envenoms             envenom
-    envied               envi
-    envies               envi
-    envious              enviou
-    enviously            envious
-    environ              environ
-    environed            environ
-    envoy                envoi
-    envy                 envi
-    envying              envi
-    enwheel              enwheel
-    enwombed             enwomb
-    enwraps              enwrap
-    ephesian             ephesian
-    ephesians            ephesian
-    ephesus              ephesu
-    epicure              epicur
-    epicurean            epicurean
-    epicures             epicur
-    epicurism            epicur
-    epicurus             epicuru
-    epidamnum            epidamnum
-    epidaurus            epidauru
-    epigram              epigram
-    epilepsy             epilepsi
-    epileptic            epilept
-    epilogue             epilogu
-    epilogues            epilogu
-    epistles             epistl
-    epistrophus          epistrophu
-    epitaph              epitaph
-    epitaphs             epitaph
-    epithet              epithet
-    epitheton            epitheton
-    epithets             epithet
-    epitome              epitom
-    equal                equal
-    equalities           equal
-    equality             equal
-    equall               equal
-    equally              equal
-    equalness            equal
-    equals               equal
-    equinoctial          equinocti
-    equinox              equinox
-    equipage             equipag
-    equity               equiti
-    equivocal            equivoc
-    equivocate           equivoc
-    equivocates          equivoc
-    equivocation         equivoc
-    equivocator          equivoc
-    er                   er
-    erbear               erbear
-    erbearing            erbear
-    erbears              erbear
-    erbeat               erbeat
-    erblows              erblow
-    erboard              erboard
-    erborne              erborn
-    ercame               ercam
-    ercast               ercast
-    ercharg              ercharg
-    ercharged            ercharg
-    ercharging           ercharg
-    ercles               ercl
-    ercome               ercom
-    ercover              ercov
-    ercrows              ercrow
-    erdoing              erdo
-    ere                  er
-    erebus               erebu
-    erect                erect
-    erected              erect
-    erecting             erect
-    erection             erect
-    erects               erect
-    erewhile             erewhil
-    erflourish           erflourish
-    erflow               erflow
-    erflowing            erflow
-    erflows              erflow
-    erfraught            erfraught
-    erga                 erga
-    ergalled             ergal
-    erglanced            erglanc
-    ergo                 ergo
-    ergone               ergon
-    ergrow               ergrow
-    ergrown              ergrown
-    ergrowth             ergrowth
-    erhang               erhang
-    erhanging            erhang
-    erhasty              erhasti
-    erhear               erhear
-    erheard              erheard
-    eringoes             eringo
-    erjoy                erjoi
-    erleap               erleap
-    erleaps              erleap
-    erleavens            erleaven
-    erlook               erlook
-    erlooking            erlook
-    ermaster             ermast
-    ermengare            ermengar
-    ermount              ermount
-    ern                  ern
-    ernight              ernight
-    eros                 ero
-    erpaid               erpaid
-    erparted             erpart
-    erpast               erpast
-    erpays               erpai
-    erpeer               erpeer
-    erperch              erperch
-    erpicturing          erpictur
-    erpingham            erpingham
-    erposting            erpost
-    erpow                erpow
-    erpress              erpress
-    erpressed            erpress
-    err                  err
-    errand               errand
-    errands              errand
-    errant               errant
-    errate               errat
-    erraught             erraught
-    erreaches            erreach
-    erred                er
-    errest               errest
-    erring               er
-    erroneous            erron
-    error                error
-    errors               error
-    errs                 err
-    errule               errul
-    errun                errun
-    erset                erset
-    ershade              ershad
-    ershades             ershad
-    ershine              ershin
-    ershot               ershot
-    ersized              ersiz
-    erskip               erskip
-    erslips              erslip
-    erspreads            erspread
-    erst                 erst
-    erstare              erstar
-    erstep               erstep
-    erstunk              erstunk
-    ersway               erswai
-    ersways              erswai
-    erswell              erswel
-    erta                 erta
-    ertake               ertak
-    erteemed             erteem
-    erthrow              erthrow
-    erthrown             erthrown
-    erthrows             erthrow
-    ertook               ertook
-    ertop                ertop
-    ertopping            ertop
-    ertrip               ertrip
-    erturn               erturn
-    erudition            erudit
-    eruption             erupt
-    eruptions            erupt
-    ervalues             ervalu
-    erwalk               erwalk
-    erwatch              erwatch
-    erween               erween
-    erweens              erween
-    erweigh              erweigh
-    erweighs             erweigh
-    erwhelm              erwhelm
-    erwhelmed            erwhelm
-    erworn               erworn
-    es                   es
-    escalus              escalu
-    escap                escap
-    escape               escap
-    escaped              escap
-    escapes              escap
-    eschew               eschew
-    escoted              escot
-    esill                esil
-    especial             especi
-    especially           especi
-    esperance            esper
-    espials              espial
-    espied               espi
-    espies               espi
-    espous               espou
-    espouse              espous
-    espy                 espi
-    esquire              esquir
-    esquires             esquir
-    essay                essai
-    essays               essai
-    essence              essenc
-    essential            essenti
-    essentially          essenti
-    esses                ess
-    essex                essex
-    est                  est
-    establish            establish
-    established          establish
-    estate               estat
-    estates              estat
-    esteem               esteem
-    esteemed             esteem
-    esteemeth            esteemeth
-    esteeming            esteem
-    esteems              esteem
-    estimable            estim
-    estimate             estim
-    estimation           estim
-    estimations          estim
-    estime               estim
-    estranged            estrang
-    estridge             estridg
-    estridges            estridg
-    et                   et
-    etc                  etc
-    etceteras            etcetera
-    ete                  et
-    eternal              etern
-    eternally            etern
-    eterne               etern
-    eternity             etern
-    eterniz              eterniz
-    etes                 et
-    ethiop               ethiop
-    ethiope              ethiop
-    ethiopes             ethiop
-    ethiopian            ethiopian
-    etna                 etna
-    eton                 eton
-    etre                 etr
-    eunuch               eunuch
-    eunuchs              eunuch
-    euphrates            euphrat
-    euphronius           euphroniu
-    euriphile            euriphil
-    europa               europa
-    europe               europ
-    ev                   ev
-    evade                evad
-    evades               evad
-    evans                evan
-    evasion              evas
-    evasions             evas
-    eve                  ev
-    even                 even
-    evening              even
-    evenly               evenli
-    event                event
-    eventful             event
-    events               event
-    ever                 ever
-    everlasting          everlast
-    everlastingly        everlastingli
-    evermore             evermor
-    every                everi
-    everyone             everyon
-    everything           everyth
-    everywhere           everywher
-    evidence             evid
-    evidences            evid
-    evident              evid
-    evil                 evil
-    evilly               evilli
-    evils                evil
-    evitate              evit
-    ewe                  ew
-    ewer                 ewer
-    ewers                ewer
-    ewes                 ew
-    exact                exact
-    exacted              exact
-    exactest             exactest
-    exacting             exact
-    exaction             exact
-    exactions            exact
-    exactly              exactli
-    exacts               exact
-    exalt                exalt
-    exalted              exalt
-    examin               examin
-    examination          examin
-    examinations         examin
-    examine              examin
-    examined             examin
-    examines             examin
-    exampl               exampl
-    example              exampl
-    exampled             exampl
-    examples             exampl
-    exasperate           exasper
-    exasperates          exasper
-    exceed               exce
-    exceeded             exceed
-    exceedeth            exceedeth
-    exceeding            exceed
-    exceedingly          exceedingli
-    exceeds              exce
-    excel                excel
-    excelled             excel
-    excellence           excel
-    excellencies         excel
-    excellency           excel
-    excellent            excel
-    excellently          excel
-    excelling            excel
-    excels               excel
-    except               except
-    excepted             except
-    excepting            except
-    exception            except
-    exceptions           except
-    exceptless           exceptless
-    excess               excess
-    excessive            excess
-    exchang              exchang
-    exchange             exchang
-    exchanged            exchang
-    exchequer            exchequ
-    exchequers           exchequ
-    excite               excit
-    excited              excit
-    excitements          excit
-    excites              excit
-    exclaim              exclaim
-    exclaims             exclaim
-    exclamation          exclam
-    exclamations         exclam
-    excludes             exclud
-    excommunicate        excommun
-    excommunication      excommun
-    excrement            excrement
-    excrements           excrement
-    excursion            excurs
-    excursions           excurs
-    excus                excu
-    excusable            excus
-    excuse               excus
-    excused              excus
-    excuses              excus
-    excusez              excusez
-    excusing             excus
-    execrable            execr
-    execrations          execr
-    execute              execut
-    executed             execut
-    executing            execut
-    execution            execut
-    executioner          execution
-    executioners         execution
-    executor             executor
-    executors            executor
-    exempt               exempt
-    exempted             exempt
-    exequies             exequi
-    exercise             exercis
-    exercises            exercis
-    exeter               exet
-    exeunt               exeunt
-    exhal                exhal
-    exhalation           exhal
-    exhalations          exhal
-    exhale               exhal
-    exhales              exhal
-    exhaust              exhaust
-    exhibit              exhibit
-    exhibiters           exhibit
-    exhibition           exhibit
-    exhort               exhort
-    exhortation          exhort
-    exigent              exig
-    exil                 exil
-    exile                exil
-    exiled               exil
-    exion                exion
-    exist                exist
-    exists               exist
-    exit                 exit
-    exits                exit
-    exorciser            exorcis
-    exorcisms            exorc
-    exorcist             exorcist
-    expect               expect
-    expectance           expect
-    expectancy           expect
-    expectation          expect
-    expectations         expect
-    expected             expect
-    expecters            expect
-    expecting            expect
-    expects              expect
-    expedience           expedi
-    expedient            expedi
-    expediently          expedi
-    expedition           expedit
-    expeditious          expediti
-    expel                expel
-    expell               expel
-    expelling            expel
-    expels               expel
-    expend               expend
-    expense              expens
-    expenses             expens
-    experienc            experienc
-    experience           experi
-    experiences          experi
-    experiment           experi
-    experimental         experiment
-    experiments          experi
-    expert               expert
-    expertness           expert
-    expiate              expiat
-    expiation            expiat
-    expir                expir
-    expiration           expir
-    expire               expir
-    expired              expir
-    expires              expir
-    expiring             expir
-    explication          explic
-    exploit              exploit
-    exploits             exploit
-    expos                expo
-    expose               expos
-    exposing             expos
-    exposition           exposit
-    expositor            expositor
-    expostulate          expostul
-    expostulation        expostul
-    exposture            expostur
-    exposure             exposur
-    expound              expound
-    expounded            expound
-    express              express
-    expressed            express
-    expresseth           expresseth
-    expressing           express
-    expressive           express
-    expressly            expressli
-    expressure           expressur
-    expuls               expul
-    expulsion            expuls
-    exquisite            exquisit
-    exsufflicate         exsuffl
-    extant               extant
-    extemporal           extempor
-    extemporally         extempor
-    extempore            extempor
-    extend               extend
-    extended             extend
-    extends              extend
-    extent               extent
-    extenuate            extenu
-    extenuated           extenu
-    extenuates           extenu
-    extenuation          extenu
-    exterior             exterior
-    exteriorly           exteriorli
-    exteriors            exterior
-    extermin             extermin
-    extern               extern
-    external             extern
-    extinct              extinct
-    extincted            extinct
-    extincture           extinctur
-    extinguish           extinguish
-    extirp               extirp
-    extirpate            extirp
-    extirped             extirp
-    extol                extol
-    extoll               extol
-    extolment            extol
-    exton                exton
-    extort               extort
-    extorted             extort
-    extortion            extort
-    extortions           extort
-    extra                extra
-    extract              extract
-    extracted            extract
-    extracting           extract
-    extraordinarily      extraordinarili
-    extraordinary        extraordinari
-    extraught            extraught
-    extravagancy         extravag
-    extravagant          extravag
-    extreme              extrem
-    extremely            extrem
-    extremes             extrem
-    extremest            extremest
-    extremities          extrem
-    extremity            extrem
-    exuent               exuent
-    exult                exult
-    exultation           exult
-    ey                   ey
-    eyas                 eya
-    eyases               eyas
-    eye                  ey
-    eyeball              eyebal
-    eyeballs             eyebal
-    eyebrow              eyebrow
-    eyebrows             eyebrow
-    eyed                 ei
-    eyeless              eyeless
-    eyelid               eyelid
-    eyelids              eyelid
-    eyes                 ey
-    eyesight             eyesight
-    eyestrings           eyestr
-    eying                ei
-    eyne                 eyn
-    eyrie                eyri
-    fa                   fa
-    fabian               fabian
-    fable                fabl
-    fables               fabl
-    fabric               fabric
-    fabulous             fabul
-    fac                  fac
-    face                 face
-    faced                face
-    facere               facer
-    faces                face
-    faciant              faciant
-    facile               facil
-    facility             facil
-    facinerious          facineri
-    facing               face
-    facit                facit
-    fact                 fact
-    faction              faction
-    factionary           factionari
-    factions             faction
-    factious             factiou
-    factor               factor
-    factors              factor
-    faculties            faculti
-    faculty              faculti
-    fade                 fade
-    faded                fade
-    fadeth               fadeth
-    fadge                fadg
-    fading               fade
-    fadings              fade
-    fadom                fadom
-    fadoms               fadom
-    fagot                fagot
-    fagots               fagot
-    fail                 fail
-    failing              fail
-    fails                fail
-    fain                 fain
-    faint                faint
-    fainted              faint
-    fainter              fainter
-    fainting             faint
-    faintly              faintli
-    faintness            faint
-    faints               faint
-    fair                 fair
-    fairer               fairer
-    fairest              fairest
-    fairies              fairi
-    fairing              fair
-    fairings             fair
-    fairly               fairli
-    fairness             fair
-    fairs                fair
-    fairwell             fairwel
-    fairy                fairi
-    fais                 fai
-    fait                 fait
-    faites               fait
-    faith                faith
-    faithful             faith
-    faithfull            faithful
-    faithfully           faithfulli
-    faithless            faithless
-    faiths               faith
-    faitors              faitor
-    fal                  fal
-    falchion             falchion
-    falcon               falcon
-    falconbridge         falconbridg
-    falconer             falcon
-    falconers            falcon
-    fall                 fall
-    fallacy              fallaci
-    fallen               fallen
-    falleth              falleth
-    falliable            falliabl
-    fallible             fallibl
-    falling              fall
-    fallow               fallow
-    fallows              fallow
-    falls                fall
-    fally                falli
-    falorous             falor
-    false                fals
-    falsehood            falsehood
-    falsely              fals
-    falseness            fals
-    falser               falser
-    falsify              falsifi
-    falsing              fals
-    falstaff             falstaff
-    falstaffs            falstaff
-    falter               falter
-    fam                  fam
-    fame                 fame
-    famed                fame
-    familiar             familiar
-    familiarity          familiar
-    familiarly           familiarli
-    familiars            familiar
-    family               famili
-    famine               famin
-    famish               famish
-    famished             famish
-    famous               famou
-    famoused             famous
-    famously             famous
-    fan                  fan
-    fanatical            fanat
-    fancies              fanci
-    fancy                fanci
-    fane                 fane
-    fanes                fane
-    fang                 fang
-    fangled              fangl
-    fangless             fangless
-    fangs                fang
-    fann                 fann
-    fanning              fan
-    fans                 fan
-    fantasied            fantasi
-    fantasies            fantasi
-    fantastic            fantast
-    fantastical          fantast
-    fantastically        fantast
-    fantasticoes         fantastico
-    fantasy              fantasi
-    fap                  fap
-    far                  far
-    farborough           farborough
-    farced               farc
-    fardel               fardel
-    fardels              fardel
-    fare                 fare
-    fares                fare
-    farewell             farewel
-    farewells            farewel
-    fariner              farin
-    faring               fare
-    farm                 farm
-    farmer               farmer
-    farmhouse            farmhous
-    farms                farm
-    farre                farr
-    farrow               farrow
-    farther              farther
-    farthest             farthest
-    farthing             farth
-    farthingale          farthingal
-    farthingales         farthingal
-    farthings            farth
-    fartuous             fartuou
-    fas                  fa
-    fashion              fashion
-    fashionable          fashion
-    fashioning           fashion
-    fashions             fashion
-    fast                 fast
-    fasted               fast
-    fasten               fasten
-    fastened             fasten
-    faster               faster
-    fastest              fastest
-    fasting              fast
-    fastly               fastli
-    fastolfe             fastolf
-    fasts                fast
-    fat                  fat
-    fatal                fatal
-    fatally              fatal
-    fate                 fate
-    fated                fate
-    fates                fate
-    father               father
-    fathered             father
-    fatherless           fatherless
-    fatherly             fatherli
-    fathers              father
-    fathom               fathom
-    fathomless           fathomless
-    fathoms              fathom
-    fatigate             fatig
-    fatness              fat
-    fats                 fat
-    fatted               fat
-    fatter               fatter
-    fattest              fattest
-    fatting              fat
-    fatuus               fatuu
-    fauconbridge         fauconbridg
-    faulconbridge        faulconbridg
-    fault                fault
-    faultiness           faulti
-    faultless            faultless
-    faults               fault
-    faulty               faulti
-    fausse               fauss
-    fauste               faust
-    faustuses            faustus
-    faut                 faut
-    favor                favor
-    favorable            favor
-    favorably            favor
-    favors               favor
-    favour               favour
-    favourable           favour
-    favoured             favour
-    favouredly           favouredli
-    favourer             favour
-    favourers            favour
-    favouring            favour
-    favourite            favourit
-    favourites           favourit
-    favours              favour
-    favout               favout
-    fawn                 fawn
-    fawneth              fawneth
-    fawning              fawn
-    fawns                fawn
-    fay                  fai
-    fe                   fe
-    fealty               fealti
-    fear                 fear
-    feared               fear
-    fearest              fearest
-    fearful              fear
-    fearfull             fearful
-    fearfully            fearfulli
-    fearfulness          fear
-    fearing              fear
-    fearless             fearless
-    fears                fear
-    feast                feast
-    feasted              feast
-    feasting             feast
-    feasts               feast
-    feat                 feat
-    feated               feat
-    feater               feater
-    feather              feather
-    feathered            feather
-    feathers             feather
-    featly               featli
-    feats                feat
-    featur               featur
-    feature              featur
-    featured             featur
-    featureless          featureless
-    features             featur
-    february             februari
-    fecks                feck
-    fed                  fed
-    fedary               fedari
-    federary             federari
-    fee                  fee
-    feeble               feebl
-    feebled              feebl
-    feebleness           feebl
-    feebling             feebl
-    feebly               feebli
-    feed                 feed
-    feeder               feeder
-    feeders              feeder
-    feedeth              feedeth
-    feeding              feed
-    feeds                feed
-    feel                 feel
-    feeler               feeler
-    feeling              feel
-    feelingly            feelingli
-    feels                feel
-    fees                 fee
-    feet                 feet
-    fehemently           fehement
-    feign                feign
-    feigned              feign
-    feigning             feign
-    feil                 feil
-    feith                feith
-    felicitate           felicit
-    felicity             felic
-    fell                 fell
-    fellest              fellest
-    fellies              felli
-    fellow               fellow
-    fellowly             fellowli
-    fellows              fellow
-    fellowship           fellowship
-    fellowships          fellowship
-    fells                fell
-    felon                felon
-    felonious            feloni
-    felony               feloni
-    felt                 felt
-    female               femal
-    females              femal
-    feminine             feminin
-    fen                  fen
-    fenc                 fenc
-    fence                fenc
-    fencer               fencer
-    fencing              fenc
-    fends                fend
-    fennel               fennel
-    fenny                fenni
-    fens                 fen
-    fenton               fenton
-    fer                  fer
-    ferdinand            ferdinand
-    fere                 fere
-    fernseed             fernse
-    ferrara              ferrara
-    ferrers              ferrer
-    ferret               ferret
-    ferry                ferri
-    ferryman             ferryman
-    fertile              fertil
-    fertility            fertil
-    fervency             fervenc
-    fervour              fervour
-    fery                 feri
-    fest                 fest
-    feste                fest
-    fester               fester
-    festinate            festin
-    festinately          festin
-    festival             festiv
-    festivals            festiv
-    fet                  fet
-    fetch                fetch
-    fetches              fetch
-    fetching             fetch
-    fetlock              fetlock
-    fetlocks             fetlock
-    fett                 fett
-    fetter               fetter
-    fettering            fetter
-    fetters              fetter
-    fettle               fettl
-    feu                  feu
-    feud                 feud
-    fever                fever
-    feverous             fever
-    fevers               fever
-    few                  few
-    fewer                fewer
-    fewest               fewest
-    fewness              few
-    fickle               fickl
-    fickleness           fickl
-    fico                 fico
-    fiction              fiction
-    fiddle               fiddl
-    fiddler              fiddler
-    fiddlestick          fiddlestick
-    fidele               fidel
-    fidelicet            fidelicet
-    fidelity             fidel
-    fidius               fidiu
-    fie                  fie
-    field                field
-    fielded              field
-    fields               field
-    fiend                fiend
-    fiends               fiend
-    fierce               fierc
-    fiercely             fierc
-    fierceness           fierc
-    fiery                fieri
-    fife                 fife
-    fifes                fife
-    fifteen              fifteen
-    fifteens             fifteen
-    fifteenth            fifteenth
-    fifth                fifth
-    fifty                fifti
-    fiftyfold            fiftyfold
-    fig                  fig
-    fight                fight
-    fighter              fighter
-    fightest             fightest
-    fighteth             fighteth
-    fighting             fight
-    fights               fight
-    figo                 figo
-    figs                 fig
-    figur                figur
-    figure               figur
-    figured              figur
-    figures              figur
-    figuring             figur
-    fike                 fike
-    fil                  fil
-    filberts             filbert
-    filch                filch
-    filches              filch
-    filching             filch
-    file                 file
-    filed                file
-    files                file
-    filial               filial
-    filius               filiu
-    fill                 fill
-    filled               fill
-    fillet               fillet
-    filling              fill
-    fillip               fillip
-    fills                fill
-    filly                filli
-    film                 film
-    fils                 fil
-    filth                filth
-    filths               filth
-    filthy               filthi
-    fin                  fin
-    finally              final
-    finch                finch
-    find                 find
-    finder               finder
-    findeth              findeth
-    finding              find
-    findings             find
-    finds                find
-    fine                 fine
-    fineless             fineless
-    finely               fine
-    finem                finem
-    fineness             fine
-    finer                finer
-    fines                fine
-    finest               finest
-    fing                 fing
-    finger               finger
-    fingering            finger
-    fingers              finger
-    fingre               fingr
-    fingres              fingr
-    finical              finic
-    finish               finish
-    finished             finish
-    finisher             finish
-    finless              finless
-    finn                 finn
-    fins                 fin
-    finsbury             finsburi
-    fir                  fir
-    firago               firago
-    fire                 fire
-    firebrand            firebrand
-    firebrands           firebrand
-    fired                fire
-    fires                fire
-    firework             firework
-    fireworks            firework
-    firing               fire
-    firk                 firk
-    firm                 firm
-    firmament            firmament
-    firmly               firmli
-    firmness             firm
-    first                first
-    firstlings           firstl
-    fish                 fish
-    fisher               fisher
-    fishermen            fishermen
-    fishers              fisher
-    fishes               fish
-    fishified            fishifi
-    fishmonger           fishmong
-    fishpond             fishpond
-    fisnomy              fisnomi
-    fist                 fist
-    fisting              fist
-    fists                fist
-    fistula              fistula
-    fit                  fit
-    fitchew              fitchew
-    fitful               fit
-    fitly                fitli
-    fitment              fitment
-    fitness              fit
-    fits                 fit
-    fitted               fit
-    fitter               fitter
-    fittest              fittest
-    fitteth              fitteth
-    fitting              fit
-    fitzwater            fitzwat
-    five                 five
-    fivepence            fivep
-    fives                five
-    fix                  fix
-    fixed                fix
-    fixes                fix
-    fixeth               fixeth
-    fixing               fix
-    fixture              fixtur
-    fl                   fl
-    flag                 flag
-    flagging             flag
-    flagon               flagon
-    flagons              flagon
-    flags                flag
-    flail                flail
-    flakes               flake
-    flaky                flaki
-    flam                 flam
-    flame                flame
-    flamen               flamen
-    flamens              flamen
-    flames               flame
-    flaming              flame
-    flaminius            flaminiu
-    flanders             flander
-    flannel              flannel
-    flap                 flap
-    flaring              flare
-    flash                flash
-    flashes              flash
-    flashing             flash
-    flask                flask
-    flat                 flat
-    flatly               flatli
-    flatness             flat
-    flats                flat
-    flatt                flatt
-    flatter              flatter
-    flattered            flatter
-    flatterer            flatter
-    flatterers           flatter
-    flatterest           flatterest
-    flatteries           flatteri
-    flattering           flatter
-    flatters             flatter
-    flattery             flatteri
-    flaunts              flaunt
-    flavio               flavio
-    flavius              flaviu
-    flaw                 flaw
-    flaws                flaw
-    flax                 flax
-    flaxen               flaxen
-    flay                 flai
-    flaying              flai
-    flea                 flea
-    fleance              fleanc
-    fleas                flea
-    flecked              fleck
-    fled                 fled
-    fledge               fledg
-    flee                 flee
-    fleec                fleec
-    fleece               fleec
-    fleeces              fleec
-    fleer                fleer
-    fleering             fleer
-    fleers               fleer
-    fleet                fleet
-    fleeter              fleeter
-    fleeting             fleet
-    fleming              fleme
-    flemish              flemish
-    flesh                flesh
-    fleshes              flesh
-    fleshly              fleshli
-    fleshment            fleshment
-    fleshmonger          fleshmong
-    flew                 flew
-    flexible             flexibl
-    flexure              flexur
-    flibbertigibbet      flibbertigibbet
-    flickering           flicker
-    flidge               flidg
-    fliers               flier
-    flies                fli
-    flieth               flieth
-    flight               flight
-    flights              flight
-    flighty              flighti
-    flinch               flinch
-    fling                fling
-    flint                flint
-    flints               flint
-    flinty               flinti
-    flirt                flirt
-    float                float
-    floated              float
-    floating             float
-    flock                flock
-    flocks               flock
-    flood                flood
-    floodgates           floodgat
-    floods               flood
-    floor                floor
-    flora                flora
-    florence             florenc
-    florentine           florentin
-    florentines          florentin
-    florentius           florentiu
-    florizel             florizel
-    flote                flote
-    floulish             floulish
-    flour                flour
-    flourish             flourish
-    flourishes           flourish
-    flourisheth          flourisheth
-    flourishing          flourish
-    flout                flout
-    flouted              flout
-    flouting             flout
-    flouts               flout
-    flow                 flow
-    flowed               flow
-    flower               flower
-    flowerets            floweret
-    flowers              flower
-    flowing              flow
-    flown                flown
-    flows                flow
-    fluellen             fluellen
-    fluent               fluent
-    flung                flung
-    flush                flush
-    flushing             flush
-    fluster              fluster
-    flute                flute
-    flutes               flute
-    flutter              flutter
-    flux                 flux
-    fluxive              fluxiv
-    fly                  fly
-    flying               fly
-    fo                   fo
-    foal                 foal
-    foals                foal
-    foam                 foam
-    foamed               foam
-    foaming              foam
-    foams                foam
-    foamy                foami
-    fob                  fob
-    focative             foc
-    fodder               fodder
-    foe                  foe
-    foeman               foeman
-    foemen               foemen
-    foes                 foe
-    fog                  fog
-    foggy                foggi
-    fogs                 fog
-    foh                  foh
-    foi                  foi
-    foil                 foil
-    foiled               foil
-    foils                foil
-    foin                 foin
-    foining              foin
-    foins                foin
-    fois                 foi
-    foison               foison
-    foisons              foison
-    foist                foist
-    foix                 foix
-    fold                 fold
-    folded               fold
-    folds                fold
-    folio                folio
-    folk                 folk
-    folks                folk
-    follies              folli
-    follow               follow
-    followed             follow
-    follower             follow
-    followers            follow
-    followest            followest
-    following            follow
-    follows              follow
-    folly                folli
-    fond                 fond
-    fonder               fonder
-    fondly               fondli
-    fondness             fond
-    font                 font
-    fontibell            fontibel
-    food                 food
-    fool                 fool
-    fooleries            fooleri
-    foolery              fooleri
-    foolhardy            foolhardi
-    fooling              fool
-    foolish              foolish
-    foolishly            foolishli
-    foolishness          foolish
-    fools                fool
-    foot                 foot
-    football             footbal
-    footboy              footboi
-    footboys             footboi
-    footed               foot
-    footfall             footfal
-    footing              foot
-    footman              footman
-    footmen              footmen
-    footpath             footpath
-    footsteps            footstep
-    footstool            footstool
-    fopp                 fopp
-    fopped               fop
-    foppery              fopperi
-    foppish              foppish
-    fops                 fop
-    for                  for
-    forage               forag
-    foragers             forag
-    forbade              forbad
-    forbear              forbear
-    forbearance          forbear
-    forbears             forbear
-    forbid               forbid
-    forbidden            forbidden
-    forbiddenly          forbiddenli
-    forbids              forbid
-    forbod               forbod
-    forborne             forborn
-    forc                 forc
-    force                forc
-    forced               forc
-    forceful             forc
-    forceless            forceless
-    forces               forc
-    forcible             forcibl
-    forcibly             forcibl
-    forcing              forc
-    ford                 ford
-    fordid               fordid
-    fordo                fordo
-    fordoes              fordo
-    fordone              fordon
-    fore                 fore
-    forecast             forecast
-    forefather           forefath
-    forefathers          forefath
-    forefinger           forefing
-    forego               forego
-    foregone             foregon
-    forehand             forehand
-    forehead             forehead
-    foreheads            forehead
-    forehorse            forehors
-    foreign              foreign
-    foreigner            foreign
-    foreigners           foreign
-    foreknowing          foreknow
-    foreknowledge        foreknowledg
-    foremost             foremost
-    forenamed            forenam
-    forenoon             forenoon
-    forerun              forerun
-    forerunner           forerunn
-    forerunning          forerun
-    foreruns             forerun
-    foresaid             foresaid
-    foresaw              foresaw
-    foresay              foresai
-    foresee              forese
-    foreseeing           forese
-    foresees             forese
-    foreshow             foreshow
-    foreskirt            foreskirt
-    forespent            foresp
-    forest               forest
-    forestall            forestal
-    forestalled          forestal
-    forester             forest
-    foresters            forest
-    forests              forest
-    foretell             foretel
-    foretelling          foretel
-    foretells            foretel
-    forethink            forethink
-    forethought          forethought
-    foretold             foretold
-    forever              forev
-    foreward             foreward
-    forewarn             forewarn
-    forewarned           forewarn
-    forewarning          forewarn
-    forfeit              forfeit
-    forfeited            forfeit
-    forfeiters           forfeit
-    forfeiting           forfeit
-    forfeits             forfeit
-    forfeiture           forfeitur
-    forfeitures          forfeitur
-    forfend              forfend
-    forfended            forfend
-    forg                 forg
-    forgave              forgav
-    forge                forg
-    forged               forg
-    forgeries            forgeri
-    forgery              forgeri
-    forges               forg
-    forget               forget
-    forgetful            forget
-    forgetfulness        forget
-    forgetive            forget
-    forgets              forget
-    forgetting           forget
-    forgive              forgiv
-    forgiven             forgiven
-    forgiveness          forgiv
-    forgo                forgo
-    forgoing             forgo
-    forgone              forgon
-    forgot               forgot
-    forgotten            forgotten
-    fork                 fork
-    forked               fork
-    forks                fork
-    forlorn              forlorn
-    form                 form
-    formal               formal
-    formally             formal
-    formed               form
-    former               former
-    formerly             formerli
-    formless             formless
-    forms                form
-    fornication          fornic
-    fornications         fornic
-    fornicatress         fornicatress
-    forres               forr
-    forrest              forrest
-    forsake              forsak
-    forsaken             forsaken
-    forsaketh            forsaketh
-    forslow              forslow
-    forsook              forsook
-    forsooth             forsooth
-    forspent             forspent
-    forspoke             forspok
-    forswear             forswear
-    forswearing          forswear
-    forswore             forswor
-    forsworn             forsworn
-    fort                 fort
-    forted               fort
-    forth                forth
-    forthcoming          forthcom
-    forthlight           forthlight
-    forthright           forthright
-    forthwith            forthwith
-    fortification        fortif
-    fortifications       fortif
-    fortified            fortifi
-    fortifies            fortifi
-    fortify              fortifi
-    fortinbras           fortinbra
-    fortitude            fortitud
-    fortnight            fortnight
-    fortress             fortress
-    fortresses           fortress
-    forts                fort
-    fortun               fortun
-    fortuna              fortuna
-    fortunate            fortun
-    fortunately          fortun
-    fortune              fortun
-    fortuned             fortun
-    fortunes             fortun
-    fortward             fortward
-    forty                forti
-    forum                forum
-    forward              forward
-    forwarding           forward
-    forwardness          forward
-    forwards             forward
-    forwearied           forweari
-    fosset               fosset
-    fost                 fost
-    foster               foster
-    fostered             foster
-    fought               fought
-    foughten             foughten
-    foul                 foul
-    fouler               fouler
-    foulest              foulest
-    foully               foulli
-    foulness             foul
-    found                found
-    foundation           foundat
-    foundations          foundat
-    founded              found
-    founder              founder
-    fount                fount
-    fountain             fountain
-    fountains            fountain
-    founts               fount
-    four                 four
-    fourscore            fourscor
-    fourteen             fourteen
-    fourth               fourth
-    foutra               foutra
-    fowl                 fowl
-    fowler               fowler
-    fowling              fowl
-    fowls                fowl
-    fox                  fox
-    foxes                fox
-    foxship              foxship
-    fracted              fract
-    fraction             fraction
-    fractions            fraction
-    fragile              fragil
-    fragment             fragment
-    fragments            fragment
-    fragrant             fragrant
-    frail                frail
-    frailer              frailer
-    frailties            frailti
-    frailty              frailti
-    fram                 fram
-    frame                frame
-    framed               frame
-    frames               frame
-    frampold             frampold
-    fran                 fran
-    francais             francai
-    france               franc
-    frances              franc
-    franchise            franchis
-    franchised           franchis
-    franchisement        franchis
-    franchises           franchis
-    franciae             francia
-    francis              franci
-    francisca            francisca
-    franciscan           franciscan
-    francisco            francisco
-    frank                frank
-    franker              franker
-    frankfort            frankfort
-    franklin             franklin
-    franklins            franklin
-    frankly              frankli
-    frankness            frank
-    frantic              frantic
-    franticly            franticli
-    frateretto           frateretto
-    fratrum              fratrum
-    fraud                fraud
-    fraudful             fraud
-    fraught              fraught
-    fraughtage           fraughtag
-    fraughting           fraught
-    fray                 frai
-    frays                frai
-    freckl               freckl
-    freckled             freckl
-    freckles             freckl
-    frederick            frederick
-    free                 free
-    freed                freed
-    freedom              freedom
-    freedoms             freedom
-    freehearted          freeheart
-    freelier             freelier
-    freely               freeli
-    freeman              freeman
-    freemen              freemen
-    freeness             freeness
-    freer                freer
-    frees                free
-    freestone            freeston
-    freetown             freetown
-    freeze               freez
-    freezes              freez
-    freezing             freez
-    freezings            freez
-    french               french
-    frenchman            frenchman
-    frenchmen            frenchmen
-    frenchwoman          frenchwoman
-    frenzy               frenzi
-    frequent             frequent
-    frequents            frequent
-    fresh                fresh
-    fresher              fresher
-    freshes              fresh
-    freshest             freshest
-    freshly              freshli
-    freshness            fresh
-    fret                 fret
-    fretful              fret
-    frets                fret
-    fretted              fret
-    fretten              fretten
-    fretting             fret
-    friar                friar
-    friars               friar
-    friday               fridai
-    fridays              fridai
-    friend               friend
-    friended             friend
-    friending            friend
-    friendless           friendless
-    friendliness         friendli
-    friendly             friendli
-    friends              friend
-    friendship           friendship
-    friendships          friendship
-    frieze               friez
-    fright               fright
-    frighted             fright
-    frightened           frighten
-    frightful            fright
-    frighting            fright
-    frights              fright
-    fringe               fring
-    fringed              fring
-    frippery             fripperi
-    frisk                frisk
-    fritters             fritter
-    frivolous            frivol
-    fro                  fro
-    frock                frock
-    frog                 frog
-    frogmore             frogmor
-    froissart            froissart
-    frolic               frolic
-    from                 from
-    front                front
-    fronted              front
-    frontier             frontier
-    frontiers            frontier
-    fronting             front
-    frontlet             frontlet
-    fronts               front
-    frost                frost
-    frosts               frost
-    frosty               frosti
-    froth                froth
-    froward              froward
-    frown                frown
-    frowning             frown
-    frowningly           frowningli
-    frowns               frown
-    froze                froze
-    frozen               frozen
-    fructify             fructifi
-    frugal               frugal
-    fruit                fruit
-    fruiterer            fruiter
-    fruitful             fruit
-    fruitfully           fruitfulli
-    fruitfulness         fruit
-    fruition             fruition
-    fruitless            fruitless
-    fruits               fruit
-    frush                frush
-    frustrate            frustrat
-    frutify              frutifi
-    fry                  fry
-    fubb                 fubb
-    fuel                 fuel
-    fugitive             fugit
-    fulfil               fulfil
-    fulfill              fulfil
-    fulfilling           fulfil
-    fulfils              fulfil
-    full                 full
-    fullam               fullam
-    fuller               fuller
-    fullers              fuller
-    fullest              fullest
-    fullness             full
-    fully                fulli
-    fulness              ful
-    fulsome              fulsom
-    fulvia               fulvia
-    fum                  fum
-    fumble               fumbl
-    fumbles              fumbl
-    fumblest             fumblest
-    fumbling             fumbl
-    fume                 fume
-    fumes                fume
-    fuming               fume
-    fumiter              fumit
-    fumitory             fumitori
-    fun                  fun
-    function             function
-    functions            function
-    fundamental          fundament
-    funeral              funer
-    funerals             funer
-    fur                  fur
-    furbish              furbish
-    furies               furi
-    furious              furiou
-    furlongs             furlong
-    furnace              furnac
-    furnaces             furnac
-    furnish              furnish
-    furnished            furnish
-    furnishings          furnish
-    furniture            furnitur
-    furnival             furniv
-    furor                furor
-    furr                 furr
-    furrow               furrow
-    furrowed             furrow
-    furrows              furrow
-    furth                furth
-    further              further
-    furtherance          further
-    furtherer            further
-    furthermore          furthermor
-    furthest             furthest
-    fury                 furi
-    furze                furz
-    furzes               furz
-    fust                 fust
-    fustian              fustian
-    fustilarian          fustilarian
-    fusty                fusti
-    fut                  fut
-    future               futur
-    futurity             futur
-    g                    g
-    gabble               gabbl
-    gaberdine            gaberdin
-    gabriel              gabriel
-    gad                  gad
-    gadding              gad
-    gads                 gad
-    gadshill             gadshil
-    gag                  gag
-    gage                 gage
-    gaged                gage
-    gagg                 gagg
-    gaging               gage
-    gagne                gagn
-    gain                 gain
-    gained               gain
-    gainer               gainer
-    gaingiving           gaingiv
-    gains                gain
-    gainsaid             gainsaid
-    gainsay              gainsai
-    gainsaying           gainsai
-    gainsays             gainsai
-    gainst               gainst
-    gait                 gait
-    gaited               gait
-    galathe              galath
-    gale                 gale
-    galen                galen
-    gales                gale
-    gall                 gall
-    gallant              gallant
-    gallantly            gallantli
-    gallantry            gallantri
-    gallants             gallant
-    galled               gall
-    gallery              galleri
-    galley               gallei
-    galleys              gallei
-    gallia               gallia
-    gallian              gallian
-    galliard             galliard
-    galliasses           galliass
-    gallimaufry          gallimaufri
-    galling              gall
-    gallons              gallon
-    gallop               gallop
-    galloping            gallop
-    gallops              gallop
-    gallow               gallow
-    galloway             gallowai
-    gallowglasses        gallowglass
-    gallows              gallow
-    gallowses            gallows
-    galls                gall
-    gallus               gallu
-    gam                  gam
-    gambol               gambol
-    gambold              gambold
-    gambols              gambol
-    gamboys              gamboi
-    game                 game
-    gamers               gamer
-    games                game
-    gamesome             gamesom
-    gamester             gamest
-    gaming               game
-    gammon               gammon
-    gamut                gamut
-    gan                  gan
-    gangren              gangren
-    ganymede             ganymed
-    gaol                 gaol
-    gaoler               gaoler
-    gaolers              gaoler
-    gaols                gaol
-    gap                  gap
-    gape                 gape
-    gapes                gape
-    gaping               gape
-    gar                  gar
-    garb                 garb
-    garbage              garbag
-    garboils             garboil
-    garcon               garcon
-    gard                 gard
-    garde                gard
-    garden               garden
-    gardener             garden
-    gardeners            garden
-    gardens              garden
-    gardez               gardez
-    gardiner             gardin
-    gardon               gardon
-    gargantua            gargantua
-    gargrave             gargrav
-    garish               garish
-    garland              garland
-    garlands             garland
-    garlic               garlic
-    garment              garment
-    garments             garment
-    garmet               garmet
-    garner               garner
-    garners              garner
-    garnish              garnish
-    garnished            garnish
-    garret               garret
-    garrison             garrison
-    garrisons            garrison
-    gart                 gart
-    garter               garter
-    garterd              garterd
-    gartering            garter
-    garters              garter
-    gascony              gasconi
-    gash                 gash
-    gashes               gash
-    gaskins              gaskin
-    gasp                 gasp
-    gasping              gasp
-    gasted               gast
-    gastness             gast
-    gat                  gat
-    gate                 gate
-    gated                gate
-    gates                gate
-    gath                 gath
-    gather               gather
-    gathered             gather
-    gathering            gather
-    gathers              gather
-    gatories             gatori
-    gatory               gatori
-    gaud                 gaud
-    gaudeo               gaudeo
-    gaudy                gaudi
-    gauge                gaug
-    gaul                 gaul
-    gaultree             gaultre
-    gaunt                gaunt
-    gauntlet             gauntlet
-    gauntlets            gauntlet
-    gav                  gav
-    gave                 gave
-    gavest               gavest
-    gawded               gawd
-    gawds                gawd
-    gawsey               gawsei
-    gay                  gai
-    gayness              gay
-    gaz                  gaz
-    gaze                 gaze
-    gazed                gaze
-    gazer                gazer
-    gazers               gazer
-    gazes                gaze
-    gazeth               gazeth
-    gazing               gaze
-    gear                 gear
-    geck                 geck
-    geese                gees
-    geffrey              geffrei
-    geld                 geld
-    gelded               geld
-    gelding              geld
-    gelida               gelida
-    gelidus              gelidu
-    gelt                 gelt
-    gem                  gem
-    geminy               gemini
-    gems                 gem
-    gen                  gen
-    gender               gender
-    genders              gender
-    general              gener
-    generally            gener
-    generals             gener
-    generation           gener
-    generations          gener
-    generative           gener
-    generosity           generos
-    generous             gener
-    genitive             genit
-    genitivo             genitivo
-    genius               geniu
-    gennets              gennet
-    genoa                genoa
-    genoux               genoux
-    gens                 gen
-    gent                 gent
-    gentilhomme          gentilhomm
-    gentility            gentil
-    gentle               gentl
-    gentlefolks          gentlefolk
-    gentleman            gentleman
-    gentlemanlike        gentlemanlik
-    gentlemen            gentlemen
-    gentleness           gentl
-    gentler              gentler
-    gentles              gentl
-    gentlest             gentlest
-    gentlewoman          gentlewoman
-    gentlewomen          gentlewomen
-    gently               gentli
-    gentry               gentri
-    george               georg
-    gerard               gerard
-    germaines            germain
-    germains             germain
-    german               german
-    germane              german
-    germans              german
-    germany              germani
-    gertrude             gertrud
-    gest                 gest
-    gests                gest
-    gesture              gestur
-    gestures             gestur
-    get                  get
-    getrude              getrud
-    gets                 get
-    getter               getter
-    getting              get
-    ghastly              ghastli
-    ghost                ghost
-    ghosted              ghost
-    ghostly              ghostli
-    ghosts               ghost
-    gi                   gi
-    giant                giant
-    giantess             giantess
-    giantlike            giantlik
-    giants               giant
-    gib                  gib
-    gibber               gibber
-    gibbet               gibbet
-    gibbets              gibbet
-    gibe                 gibe
-    giber                giber
-    gibes                gibe
-    gibing               gibe
-    gibingly             gibingli
-    giddily              giddili
-    giddiness            giddi
-    giddy                giddi
-    gift                 gift
-    gifts                gift
-    gig                  gig
-    giglets              giglet
-    giglot               giglot
-    gilbert              gilbert
-    gild                 gild
-    gilded               gild
-    gilding              gild
-    gilliams             gilliam
-    gillian              gillian
-    gills                gill
-    gillyvors            gillyvor
-    gilt                 gilt
-    gimmal               gimmal
-    gimmers              gimmer
-    gin                  gin
-    ging                 ging
-    ginger               ginger
-    gingerbread          gingerbread
-    gingerly             gingerli
-    ginn                 ginn
-    gins                 gin
-    gioucestershire      gioucestershir
-    gipes                gipe
-    gipsies              gipsi
-    gipsy                gipsi
-    gird                 gird
-    girded               gird
-    girdle               girdl
-    girdled              girdl
-    girdles              girdl
-    girdling             girdl
-    girl                 girl
-    girls                girl
-    girt                 girt
-    girth                girth
-    gis                  gi
-    giv                  giv
-    give                 give
-    given                given
-    giver                giver
-    givers               giver
-    gives                give
-    givest               givest
-    giveth               giveth
-    giving               give
-    givings              give
-    glad                 glad
-    gladded              glad
-    gladding             glad
-    gladly               gladli
-    gladness             glad
-    glamis               glami
-    glanc                glanc
-    glance               glanc
-    glanced              glanc
-    glances              glanc
-    glancing             glanc
-    glanders             glander
-    glansdale            glansdal
-    glare                glare
-    glares               glare
-    glass                glass
-    glasses              glass
-    glassy               glassi
-    glaz                 glaz
-    glazed               glaze
-    gleams               gleam
-    glean                glean
-    gleaned              glean
-    gleaning             glean
-    gleeful              gleeful
-    gleek                gleek
-    gleeking             gleek
-    gleeks               gleek
-    glend                glend
-    glendower            glendow
-    glib                 glib
-    glide                glide
-    glided               glide
-    glides               glide
-    glideth              glideth
-    gliding              glide
-    glimmer              glimmer
-    glimmering           glimmer
-    glimmers             glimmer
-    glimpse              glimps
-    glimpses             glimps
-    glist                glist
-    glistening           glisten
-    glister              glister
-    glistering           glister
-    glisters             glister
-    glitt                glitt
-    glittering           glitter
-    globe                globe
-    globes               globe
-    glooming             gloom
-    gloomy               gloomi
-    glories              glori
-    glorified            glorifi
-    glorify              glorifi
-    glorious             gloriou
-    gloriously           glorious
-    glory                glori
-    glose                glose
-    gloss                gloss
-    glosses              gloss
-    glou                 glou
-    glouceste            gloucest
-    gloucester           gloucest
-    gloucestershire      gloucestershir
-    glove                glove
-    glover               glover
-    gloves               glove
-    glow                 glow
-    glowed               glow
-    glowing              glow
-    glowworm             glowworm
-    gloz                 gloz
-    gloze                gloze
-    glozes               gloze
-    glu                  glu
-    glue                 glue
-    glued                glu
-    glues                glue
-    glut                 glut
-    glutt                glutt
-    glutted              glut
-    glutton              glutton
-    gluttoning           glutton
-    gluttony             gluttoni
-    gnarled              gnarl
-    gnarling             gnarl
-    gnat                 gnat
-    gnats                gnat
-    gnaw                 gnaw
-    gnawing              gnaw
-    gnawn                gnawn
-    gnaws                gnaw
-    go                   go
-    goad                 goad
-    goaded               goad
-    goads                goad
-    goal                 goal
-    goat                 goat
-    goatish              goatish
-    goats                goat
-    gobbets              gobbet
-    gobbo                gobbo
-    goblet               goblet
-    goblets              goblet
-    goblin               goblin
-    goblins              goblin
-    god                  god
-    godded               god
-    godden               godden
-    goddess              goddess
-    goddesses            goddess
-    goddild              goddild
-    godfather            godfath
-    godfathers           godfath
-    godhead              godhead
-    godlike              godlik
-    godliness            godli
-    godly                godli
-    godmother            godmoth
-    gods                 god
-    godson               godson
-    goer                 goer
-    goers                goer
-    goes                 goe
-    goest                goest
-    goeth                goeth
-    goffe                goff
-    gogs                 gog
-    going                go
-    gold                 gold
-    golden               golden
-    goldenly             goldenli
-    goldsmith            goldsmith
-    goldsmiths           goldsmith
-    golgotha             golgotha
-    goliases             golias
-    goliath              goliath
-    gon                  gon
-    gondola              gondola
-    gondolier            gondoli
-    gone                 gone
-    goneril              goneril
-    gong                 gong
-    gonzago              gonzago
-    gonzalo              gonzalo
-    good                 good
-    goodfellow           goodfellow
-    goodlier             goodlier
-    goodliest            goodliest
-    goodly               goodli
-    goodman              goodman
-    goodness             good
-    goodnight            goodnight
-    goodrig              goodrig
-    goods                good
-    goodwife             goodwif
-    goodwill             goodwil
-    goodwin              goodwin
-    goodwins             goodwin
-    goodyear             goodyear
-    goodyears            goodyear
-    goose                goos
-    gooseberry           gooseberri
-    goosequills          goosequil
-    goot                 goot
-    gor                  gor
-    gorbellied           gorbelli
-    gorboduc             gorboduc
-    gordian              gordian
-    gore                 gore
-    gored                gore
-    gorg                 gorg
-    gorge                gorg
-    gorgeous             gorgeou
-    gorget               gorget
-    gorging              gorg
-    gorgon               gorgon
-    gormandize           gormand
-    gormandizing         gormand
-    gory                 gori
-    gosling              gosl
-    gospel               gospel
-    gospels              gospel
-    goss                 goss
-    gossamer             gossam
-    gossip               gossip
-    gossiping            gossip
-    gossiplike           gossiplik
-    gossips              gossip
-    got                  got
-    goth                 goth
-    goths                goth
-    gotten               gotten
-    gourd                gourd
-    gout                 gout
-    gouts                gout
-    gouty                gouti
-    govern               govern
-    governance           govern
-    governed             govern
-    governess            gover
-    government           govern
-    governor             governor
-    governors            governor
-    governs              govern
-    gower                gower
-    gown                 gown
-    gowns                gown
-    grac                 grac
-    grace                grace
-    graced               grace
-    graceful             grace
-    gracefully           gracefulli
-    graceless            graceless
-    graces               grace
-    gracing              grace
-    gracious             graciou
-    graciously           gracious
-    gradation            gradat
-    graff                graff
-    graffing             graf
-    graft                graft
-    grafted              graft
-    grafters             grafter
-    grain                grain
-    grained              grain
-    grains               grain
-    gramercies           gramerci
-    gramercy             gramerci
-    grammar              grammar
-    grand                grand
-    grandam              grandam
-    grandame             grandam
-    grandchild           grandchild
-    grande               grand
-    grandeur             grandeur
-    grandfather          grandfath
-    grandjurors          grandjuror
-    grandmother          grandmoth
-    grandpre             grandpr
-    grandsir             grandsir
-    grandsire            grandsir
-    grandsires           grandsir
-    grange               grang
-    grant                grant
-    granted              grant
-    granting             grant
-    grants               grant
-    grape                grape
-    grapes               grape
-    grapple              grappl
-    grapples             grappl
-    grappling            grappl
-    grasp                grasp
-    grasped              grasp
-    grasps               grasp
-    grass                grass
-    grasshoppers         grasshopp
-    grassy               grassi
-    grate                grate
-    grated               grate
-    grateful             grate
-    grates               grate
-    gratiano             gratiano
-    gratify              gratifi
-    gratii               gratii
-    gratillity           gratil
-    grating              grate
-    gratis               grati
-    gratitude            gratitud
-    gratulate            gratul
-    grav                 grav
-    grave                grave
-    gravediggers         gravedigg
-    gravel               gravel
-    graveless            graveless
-    gravell              gravel
-    gravely              grave
-    graven               graven
-    graveness            grave
-    graver               graver
-    graves               grave
-    gravest              gravest
-    gravestone           graveston
-    gravities            graviti
-    gravity              graviti
-    gravy                gravi
-    gray                 grai
-    graymalkin           graymalkin
-    graz                 graz
-    graze                graze
-    grazed               graze
-    grazing              graze
-    grease               greas
-    greases              greas
-    greasily             greasili
-    greasy               greasi
-    great                great
-    greater              greater
-    greatest             greatest
-    greatly              greatli
-    greatness            great
-    grecian              grecian
-    grecians             grecian
-    gree                 gree
-    greece               greec
-    greed                greed
-    greedily             greedili
-    greediness           greedi
-    greedy               greedi
-    greeing              gree
-    greek                greek
-    greekish             greekish
-    greeks               greek
-    green                green
-    greener              greener
-    greenly              greenli
-    greens               green
-    greensleeves         greensleev
-    greenwich            greenwich
-    greenwood            greenwood
-    greet                greet
-    greeted              greet
-    greeting             greet
-    greetings            greet
-    greets               greet
-    greg                 greg
-    gregory              gregori
-    gremio               gremio
-    grew                 grew
-    grey                 grei
-    greybeard            greybeard
-    greybeards           greybeard
-    greyhound            greyhound
-    greyhounds           greyhound
-    grief                grief
-    griefs               grief
-    griev                griev
-    grievance            grievanc
-    grievances           grievanc
-    grieve               griev
-    grieved              griev
-    grieves              griev
-    grievest             grievest
-    grieving             griev
-    grievingly           grievingli
-    grievous             grievou
-    grievously           grievous
-    griffin              griffin
-    griffith             griffith
-    grim                 grim
-    grime                grime
-    grimly               grimli
-    grin                 grin
-    grind                grind
-    grinding             grind
-    grindstone           grindston
-    grinning             grin
-    grip                 grip
-    gripe                gripe
-    gripes               gripe
-    griping              gripe
-    grise                grise
-    grisly               grisli
-    grissel              grissel
-    grize                grize
-    grizzle              grizzl
-    grizzled             grizzl
-    groan                groan
-    groaning             groan
-    groans               groan
-    groat                groat
-    groats               groat
-    groin                groin
-    groom                groom
-    grooms               groom
-    grop                 grop
-    groping              grope
-    gros                 gro
-    gross                gross
-    grosser              grosser
-    grossly              grossli
-    grossness            gross
-    ground               ground
-    grounded             ground
-    groundlings          groundl
-    grounds              ground
-    grove                grove
-    grovel               grovel
-    grovelling           grovel
-    groves               grove
-    grow                 grow
-    groweth              groweth
-    growing              grow
-    grown                grown
-    grows                grow
-    growth               growth
-    grub                 grub
-    grubb                grubb
-    grubs                grub
-    grudge               grudg
-    grudged              grudg
-    grudges              grudg
-    grudging             grudg
-    gruel                gruel
-    grumble              grumbl
-    grumblest            grumblest
-    grumbling            grumbl
-    grumblings           grumbl
-    grumio               grumio
-    grund                grund
-    grunt                grunt
-    gualtier             gualtier
-    guard                guard
-    guardage             guardag
-    guardant             guardant
-    guarded              guard
-    guardian             guardian
-    guardians            guardian
-    guards               guard
-    guardsman            guardsman
-    gud                  gud
-    gudgeon              gudgeon
-    guerdon              guerdon
-    guerra               guerra
-    guess                guess
-    guesses              guess
-    guessingly           guessingli
-    guest                guest
-    guests               guest
-    guiana               guiana
-    guichard             guichard
-    guide                guid
-    guided               guid
-    guider               guider
-    guiderius            guideriu
-    guides               guid
-    guiding              guid
-    guidon               guidon
-    guienne              guienn
-    guil                 guil
-    guildenstern         guildenstern
-    guilders             guilder
-    guildford            guildford
-    guildhall            guildhal
-    guile                guil
-    guiled               guil
-    guileful             guil
-    guilfords            guilford
-    guilt                guilt
-    guiltian             guiltian
-    guiltier             guiltier
-    guiltily             guiltili
-    guiltiness           guilti
-    guiltless            guiltless
-    guilts               guilt
-    guilty               guilti
-    guinea               guinea
-    guinever             guinev
-    guise                guis
-    gul                  gul
-    gules                gule
-    gulf                 gulf
-    gulfs                gulf
-    gull                 gull
-    gulls                gull
-    gum                  gum
-    gumm                 gumm
-    gums                 gum
-    gun                  gun
-    gunner               gunner
-    gunpowder            gunpowd
-    guns                 gun
-    gurnet               gurnet
-    gurney               gurnei
-    gust                 gust
-    gusts                gust
-    gusty                gusti
-    guts                 gut
-    gutter               gutter
-    guy                  gui
-    guynes               guyn
-    guysors              guysor
-    gypsy                gypsi
-    gyve                 gyve
-    gyved                gyve
-    gyves                gyve
-    h                    h
-    ha                   ha
-    haberdasher          haberdash
-    habiliment           habili
-    habiliments          habili
-    habit                habit
-    habitation           habit
-    habited              habit
-    habits               habit
-    habitude             habitud
-    hack                 hack
-    hacket               hacket
-    hackney              hacknei
-    hacks                hack
-    had                  had
-    hadst                hadst
-    haec                 haec
-    haeres               haer
-    hag                  hag
-    hagar                hagar
-    haggard              haggard
-    haggards             haggard
-    haggish              haggish
-    haggled              haggl
-    hags                 hag
-    hail                 hail
-    hailed               hail
-    hailstone            hailston
-    hailstones           hailston
-    hair                 hair
-    hairless             hairless
-    hairs                hair
-    hairy                hairi
-    hal                  hal
-    halberd              halberd
-    halberds             halberd
-    halcyon              halcyon
-    hale                 hale
-    haled                hale
-    hales                hale
-    half                 half
-    halfcan              halfcan
-    halfpence            halfpenc
-    halfpenny            halfpenni
-    halfpennyworth       halfpennyworth
-    halfway              halfwai
-    halidom              halidom
-    hall                 hall
-    halloa               halloa
-    halloing             hallo
-    hallond              hallond
-    halloo               halloo
-    hallooing            halloo
-    hallow               hallow
-    hallowed             hallow
-    hallowmas            hallowma
-    hallown              hallown
-    hals                 hal
-    halt                 halt
-    halter               halter
-    halters              halter
-    halting              halt
-    halts                halt
-    halves               halv
-    ham                  ham
-    hames                hame
-    hamlet               hamlet
-    hammer               hammer
-    hammered             hammer
-    hammering            hammer
-    hammers              hammer
-    hamper               hamper
-    hampton              hampton
-    hams                 ham
-    hamstring            hamstr
-    hand                 hand
-    handed               hand
-    handful              hand
-    handicraft           handicraft
-    handicraftsmen       handicraftsmen
-    handing              hand
-    handiwork            handiwork
-    handkercher          handkerch
-    handkerchers         handkerch
-    handkerchief         handkerchief
-    handle               handl
-    handled              handl
-    handles              handl
-    handless             handless
-    handlest             handlest
-    handling             handl
-    handmaid             handmaid
-    handmaids            handmaid
-    hands                hand
-    handsaw              handsaw
-    handsome             handsom
-    handsomely           handsom
-    handsomeness         handsom
-    handwriting          handwrit
-    handy                handi
-    hang                 hang
-    hanged               hang
-    hangers              hanger
-    hangeth              hangeth
-    hanging              hang
-    hangings             hang
-    hangman              hangman
-    hangmen              hangmen
-    hangs                hang
-    hannibal             hannib
-    hap                  hap
-    hapless              hapless
-    haply                hapli
-    happ                 happ
-    happen               happen
-    happened             happen
-    happier              happier
-    happies              happi
-    happiest             happiest
-    happily              happili
-    happiness            happi
-    happy                happi
-    haps                 hap
-    harbinger            harbing
-    harbingers           harbing
-    harbor               harbor
-    harbour              harbour
-    harbourage           harbourag
-    harbouring           harbour
-    harbours             harbour
-    harcourt             harcourt
-    hard                 hard
-    harder               harder
-    hardest              hardest
-    hardiest             hardiest
-    hardiment            hardiment
-    hardiness            hardi
-    hardly               hardli
-    hardness             hard
-    hardocks             hardock
-    hardy                hardi
-    hare                 hare
-    harelip              harelip
-    hares                hare
-    harfleur             harfleur
-    hark                 hark
-    harlot               harlot
-    harlotry             harlotri
-    harlots              harlot
-    harm                 harm
-    harmed               harm
-    harmful              harm
-    harming              harm
-    harmless             harmless
-    harmonious           harmoni
-    harmony              harmoni
-    harms                harm
-    harness              har
-    harp                 harp
-    harper               harper
-    harpier              harpier
-    harping              harp
-    harpy                harpi
-    harried              harri
-    harrow               harrow
-    harrows              harrow
-    harry                harri
-    harsh                harsh
-    harshly              harshli
-    harshness            harsh
-    hart                 hart
-    harts                hart
-    harum                harum
-    harvest              harvest
-    has                  ha
-    hast                 hast
-    haste                hast
-    hasted               hast
-    hasten               hasten
-    hastes               hast
-    hastily              hastili
-    hasting              hast
-    hastings             hast
-    hasty                hasti
-    hat                  hat
-    hatch                hatch
-    hatches              hatch
-    hatchet              hatchet
-    hatching             hatch
-    hatchment            hatchment
-    hate                 hate
-    hated                hate
-    hateful              hate
-    hater                hater
-    haters               hater
-    hates                hate
-    hateth               hateth
-    hatfield             hatfield
-    hath                 hath
-    hating               hate
-    hatred               hatr
-    hats                 hat
-    haud                 haud
-    hauf                 hauf
-    haught               haught
-    haughtiness          haughti
-    haughty              haughti
-    haunch               haunch
-    haunches             haunch
-    haunt                haunt
-    haunted              haunt
-    haunting             haunt
-    haunts               haunt
-    hautboy              hautboi
-    hautboys             hautboi
-    have                 have
-    haven                haven
-    havens               haven
-    haver                haver
-    having               have
-    havings              have
-    havior               havior
-    haviour              haviour
-    havoc                havoc
-    hawk                 hawk
-    hawking              hawk
-    hawks                hawk
-    hawthorn             hawthorn
-    hawthorns            hawthorn
-    hay                  hai
-    hazard               hazard
-    hazarded             hazard
-    hazards              hazard
-    hazel                hazel
-    hazelnut             hazelnut
-    he                   he
-    head                 head
-    headborough          headborough
-    headed               head
-    headier              headier
-    heading              head
-    headland             headland
-    headless             headless
-    headlong             headlong
-    heads                head
-    headsman             headsman
-    headstrong           headstrong
-    heady                headi
-    heal                 heal
-    healed               heal
-    healing              heal
-    heals                heal
-    health               health
-    healthful            health
-    healths              health
-    healthsome           healthsom
-    healthy              healthi
-    heap                 heap
-    heaping              heap
-    heaps                heap
-    hear                 hear
-    heard                heard
-    hearer               hearer
-    hearers              hearer
-    hearest              hearest
-    heareth              heareth
-    hearing              hear
-    hearings             hear
-    heark                heark
-    hearken              hearken
-    hearkens             hearken
-    hears                hear
-    hearsay              hearsai
-    hearse               hears
-    hearsed              hears
-    hearst               hearst
-    heart                heart
-    heartache            heartach
-    heartbreak           heartbreak
-    heartbreaking        heartbreak
-    hearted              heart
-    hearten              hearten
-    hearth               hearth
-    hearths              hearth
-    heartily             heartili
-    heartiness           hearti
-    heartless            heartless
-    heartlings           heartl
-    heartly              heartli
-    hearts               heart
-    heartsick            heartsick
-    heartstrings         heartstr
-    hearty               hearti
-    heat                 heat
-    heated               heat
-    heath                heath
-    heathen              heathen
-    heathenish           heathenish
-    heating              heat
-    heats                heat
-    heauties             heauti
-    heav                 heav
-    heave                heav
-    heaved               heav
-    heaven               heaven
-    heavenly             heavenli
-    heavens              heaven
-    heaves               heav
-    heavier              heavier
-    heaviest             heaviest
-    heavily              heavili
-    heaviness            heavi
-    heaving              heav
-    heavings             heav
-    heavy                heavi
-    hebona               hebona
-    hebrew               hebrew
-    hecate               hecat
-    hectic               hectic
-    hector               hector
-    hectors              hector
-    hecuba               hecuba
-    hedg                 hedg
-    hedge                hedg
-    hedgehog             hedgehog
-    hedgehogs            hedgehog
-    hedges               hedg
-    heed                 heed
-    heeded               heed
-    heedful              heed
-    heedfull             heedful
-    heedfully            heedfulli
-    heedless             heedless
-    heel                 heel
-    heels                heel
-    hefted               heft
-    hefts                heft
-    heifer               heifer
-    heifers              heifer
-    heigh                heigh
-    height               height
-    heighten             heighten
-    heinous              heinou
-    heinously            heinous
-    heir                 heir
-    heiress              heiress
-    heirless             heirless
-    heirs                heir
-    held                 held
-    helen                helen
-    helena               helena
-    helenus              helenu
-    helias               helia
-    helicons             helicon
-    hell                 hell
-    hellespont           hellespont
-    hellfire             hellfir
-    hellish              hellish
-    helm                 helm
-    helmed               helm
-    helmet               helmet
-    helmets              helmet
-    helms                helm
-    help                 help
-    helper               helper
-    helpers              helper
-    helpful              help
-    helping              help
-    helpless             helpless
-    helps                help
-    helter               helter
-    hem                  hem
-    heme                 heme
-    hemlock              hemlock
-    hemm                 hemm
-    hemp                 hemp
-    hempen               hempen
-    hems                 hem
-    hen                  hen
-    hence                henc
-    henceforth           henceforth
-    henceforward         henceforward
-    henchman             henchman
-    henri                henri
-    henricus             henricu
-    henry                henri
-    hens                 hen
-    hent                 hent
-    henton               henton
-    her                  her
-    herald               herald
-    heraldry             heraldri
-    heralds              herald
-    herb                 herb
-    herbert              herbert
-    herblets             herblet
-    herbs                herb
-    herculean            herculean
-    hercules             hercul
-    herd                 herd
-    herds                herd
-    herdsman             herdsman
-    herdsmen             herdsmen
-    here                 here
-    hereabout            hereabout
-    hereabouts           hereabout
-    hereafter            hereaft
-    hereby               herebi
-    hereditary           hereditari
-    hereford             hereford
-    herefordshire        herefordshir
-    herein               herein
-    hereof               hereof
-    heresies             heresi
-    heresy               heresi
-    heretic              heret
-    heretics             heret
-    hereto               hereto
-    hereupon             hereupon
-    heritage             heritag
-    heritier             heriti
-    hermes               herm
-    hermia               hermia
-    hermione             hermion
-    hermit               hermit
-    hermitage            hermitag
-    hermits              hermit
-    herne                hern
-    hero                 hero
-    herod                herod
-    herods               herod
-    heroes               hero
-    heroic               heroic
-    heroical             heroic
-    herring              her
-    herrings             her
-    hers                 her
-    herself              herself
-    hesperides           hesperid
-    hesperus             hesperu
-    hest                 hest
-    hests                hest
-    heure                heur
-    heureux              heureux
-    hew                  hew
-    hewgh                hewgh
-    hewing               hew
-    hewn                 hewn
-    hews                 hew
-    hey                  hei
-    heyday               heydai
-    hibocrates           hibocr
-    hic                  hic
-    hiccups              hiccup
-    hick                 hick
-    hid                  hid
-    hidden               hidden
-    hide                 hide
-    hideous              hideou
-    hideously            hideous
-    hideousness          hideous
-    hides                hide
-    hidest               hidest
-    hiding               hide
-    hie                  hie
-    hied                 hi
-    hiems                hiem
-    hies                 hi
-    hig                  hig
-    high                 high
-    higher               higher
-    highest              highest
-    highly               highli
-    highmost             highmost
-    highness             high
-    hight                hight
-    highway              highwai
-    highways             highwai
-    hilding              hild
-    hildings             hild
-    hill                 hill
-    hillo                hillo
-    hilloa               hilloa
-    hills                hill
-    hilt                 hilt
-    hilts                hilt
-    hily                 hili
-    him                  him
-    himself              himself
-    hinc                 hinc
-    hinckley             hincklei
-    hind                 hind
-    hinder               hinder
-    hindered             hinder
-    hinders              hinder
-    hindmost             hindmost
-    hinds                hind
-    hing                 hing
-    hinge                hing
-    hinges               hing
-    hint                 hint
-    hip                  hip
-    hipp                 hipp
-    hipparchus           hipparchu
-    hippolyta            hippolyta
-    hips                 hip
-    hir                  hir
-    hire                 hire
-    hired                hire
-    hiren                hiren
-    hirtius              hirtiu
-    his                  hi
-    hisperia             hisperia
-    hiss                 hiss
-    hisses               hiss
-    hissing              hiss
-    hist                 hist
-    historical           histor
-    history              histori
-    hit                  hit
-    hither               hither
-    hitherto             hitherto
-    hitherward           hitherward
-    hitherwards          hitherward
-    hits                 hit
-    hitting              hit
-    hive                 hive
-    hives                hive
-    hizzing              hizz
-    ho                   ho
-    hoa                  hoa
-    hoar                 hoar
-    hoard                hoard
-    hoarded              hoard
-    hoarding             hoard
-    hoars                hoar
-    hoarse               hoars
-    hoary                hoari
-    hob                  hob
-    hobbididence         hobbidid
-    hobby                hobbi
-    hobbyhorse           hobbyhors
-    hobgoblin            hobgoblin
-    hobnails             hobnail
-    hoc                  hoc
-    hod                  hod
-    hodge                hodg
-    hog                  hog
-    hogs                 hog
-    hogshead             hogshead
-    hogsheads            hogshead
-    hois                 hoi
-    hoise                hois
-    hoist                hoist
-    hoisted              hoist
-    hoists               hoist
-    holborn              holborn
-    hold                 hold
-    holden               holden
-    holder               holder
-    holdeth              holdeth
-    holdfast             holdfast
-    holding              hold
-    holds                hold
-    hole                 hole
-    holes                hole
-    holidam              holidam
-    holidame             holidam
-    holiday              holidai
-    holidays             holidai
-    holier               holier
-    holiest              holiest
-    holily               holili
-    holiness             holi
-    holla                holla
-    holland              holland
-    hollander            holland
-    hollanders           holland
-    holloa               holloa
-    holloaing            holloa
-    hollow               hollow
-    hollowly             hollowli
-    hollowness           hollow
-    holly                holli
-    holmedon             holmedon
-    holofernes           holofern
-    holp                 holp
-    holy                 holi
-    homage               homag
-    homager              homag
-    home                 home
-    homely               home
-    homes                home
-    homespuns            homespun
-    homeward             homeward
-    homewards            homeward
-    homicide             homicid
-    homicides            homicid
-    homily               homili
-    hominem              hominem
-    hommes               homm
-    homo                 homo
-    honest               honest
-    honester             honest
-    honestest            honestest
-    honestly             honestli
-    honesty              honesti
-    honey                honei
-    honeycomb            honeycomb
-    honeying             honei
-    honeyless            honeyless
-    honeysuckle          honeysuckl
-    honeysuckles         honeysuckl
-    honi                 honi
-    honneur              honneur
-    honor                honor
-    honorable            honor
-    honorably            honor
-    honorato             honorato
-    honors               honor
-    honour               honour
-    honourable           honour
-    honourably           honour
-    honoured             honour
-    honourest            honourest
-    honourible           honour
-    honouring            honour
-    honours              honour
-    hoo                  hoo
-    hood                 hood
-    hooded               hood
-    hoodman              hoodman
-    hoods                hood
-    hoodwink             hoodwink
-    hoof                 hoof
-    hoofs                hoof
-    hook                 hook
-    hooking              hook
-    hooks                hook
-    hoop                 hoop
-    hoops                hoop
-    hoot                 hoot
-    hooted               hoot
-    hooting              hoot
-    hoots                hoot
-    hop                  hop
-    hope                 hope
-    hopeful              hope
-    hopeless             hopeless
-    hopes                hope
-    hopest               hopest
-    hoping               hope
-    hopkins              hopkin
-    hoppedance           hopped
-    hor                  hor
-    horace               horac
-    horatio              horatio
-    horizon              horizon
-    horn                 horn
-    hornbook             hornbook
-    horned               horn
-    horner               horner
-    horning              horn
-    hornpipes            hornpip
-    horns                horn
-    horologe             horolog
-    horrible             horribl
-    horribly             horribl
-    horrid               horrid
-    horrider             horrid
-    horridly             horridli
-    horror               horror
-    horrors              horror
-    hors                 hor
-    horse                hors
-    horseback            horseback
-    horsed               hors
-    horsehairs           horsehair
-    horseman             horseman
-    horsemanship         horsemanship
-    horsemen             horsemen
-    horses               hors
-    horseway             horsewai
-    horsing              hors
-    hortensio            hortensio
-    hortensius           hortensiu
-    horum                horum
-    hose                 hose
-    hospitable           hospit
-    hospital             hospit
-    hospitality          hospit
-    host                 host
-    hostage              hostag
-    hostages             hostag
-    hostess              hostess
-    hostile              hostil
-    hostility            hostil
-    hostilius            hostiliu
-    hosts                host
-    hot                  hot
-    hotly                hotli
-    hotspur              hotspur
-    hotter               hotter
-    hottest              hottest
-    hound                hound
-    hounds               hound
-    hour                 hour
-    hourly               hourli
-    hours                hour
-    hous                 hou
-    house                hous
-    household            household
-    householder          household
-    householders         household
-    households           household
-    housekeeper          housekeep
-    housekeepers         housekeep
-    housekeeping         housekeep
-    houseless            houseless
-    houses               hous
-    housewife            housewif
-    housewifery          housewiferi
-    housewives           housew
-    hovel                hovel
-    hover                hover
-    hovered              hover
-    hovering             hover
-    hovers               hover
-    how                  how
-    howbeit              howbeit
-    howe                 how
-    howeer               howeer
-    however              howev
-    howl                 howl
-    howled               howl
-    howlet               howlet
-    howling              howl
-    howls                howl
-    howsoe               howso
-    howsoever            howsoev
-    howsome              howsom
-    hoxes                hox
-    hoy                  hoi
-    hoyday               hoydai
-    hubert               hubert
-    huddled              huddl
-    huddling             huddl
-    hue                  hue
-    hued                 hu
-    hues                 hue
-    hug                  hug
-    huge                 huge
-    hugely               huge
-    hugeness             huge
-    hugg                 hugg
-    hugger               hugger
-    hugh                 hugh
-    hugs                 hug
-    hujus                huju
-    hulk                 hulk
-    hulks                hulk
-    hull                 hull
-    hulling              hull
-    hullo                hullo
-    hum                  hum
-    human                human
-    humane               human
-    humanely             human
-    humanity             human
-    humble               humbl
-    humbled              humbl
-    humbleness           humbl
-    humbler              humbler
-    humbles              humbl
-    humblest             humblest
-    humbling             humbl
-    humbly               humbl
-    hume                 hume
-    humh                 humh
-    humidity             humid
-    humility             humil
-    humming              hum
-    humor                humor
-    humorous             humor
-    humors               humor
-    humour               humour
-    humourists           humourist
-    humours              humour
-    humphrey             humphrei
-    humphry              humphri
-    hums                 hum
-    hundred              hundr
-    hundreds             hundr
-    hundredth            hundredth
-    hung                 hung
-    hungarian            hungarian
-    hungary              hungari
-    hunger               hunger
-    hungerford           hungerford
-    hungerly             hungerli
-    hungry               hungri
-    hunt                 hunt
-    hunted               hunt
-    hunter               hunter
-    hunters              hunter
-    hunteth              hunteth
-    hunting              hunt
-    huntington           huntington
-    huntress             huntress
-    hunts                hunt
-    huntsman             huntsman
-    huntsmen             huntsmen
-    hurdle               hurdl
-    hurl                 hurl
-    hurling              hurl
-    hurls                hurl
-    hurly                hurli
-    hurlyburly           hurlyburli
-    hurricano            hurricano
-    hurricanoes          hurricano
-    hurried              hurri
-    hurries              hurri
-    hurry                hurri
-    hurt                 hurt
-    hurting              hurt
-    hurtled              hurtl
-    hurtless             hurtless
-    hurtling             hurtl
-    hurts                hurt
-    husband              husband
-    husbanded            husband
-    husbandless          husbandless
-    husbandry            husbandri
-    husbands             husband
-    hush                 hush
-    hushes               hush
-    husht                husht
-    husks                husk
-    huswife              huswif
-    huswifes             huswif
-    hutch                hutch
-    hybla                hybla
-    hydra                hydra
-    hyen                 hyen
-    hymen                hymen
-    hymenaeus            hymenaeu
-    hymn                 hymn
-    hymns                hymn
-    hyperboles           hyperbol
-    hyperbolical         hyperbol
-    hyperion             hyperion
-    hypocrisy            hypocrisi
-    hypocrite            hypocrit
-    hypocrites           hypocrit
-    hyrcan               hyrcan
-    hyrcania             hyrcania
-    hyrcanian            hyrcanian
-    hyssop               hyssop
-    hysterica            hysterica
-    i                    i
-    iachimo              iachimo
-    iaculis              iaculi
-    iago                 iago
-    iament               iament
-    ibat                 ibat
-    icarus               icaru
-    ice                  ic
-    iceland              iceland
-    ici                  ici
-    icicle               icicl
-    icicles              icicl
-    icy                  ici
-    idea                 idea
-    ideas                idea
-    idem                 idem
-    iden                 iden
-    ides                 id
-    idiot                idiot
-    idiots               idiot
-    idle                 idl
-    idleness             idl
-    idles                idl
-    idly                 idli
-    idol                 idol
-    idolatrous           idolatr
-    idolatry             idolatri
-    ield                 ield
-    if                   if
-    ifs                  if
-    ignis                igni
-    ignoble              ignobl
-    ignobly              ignobl
-    ignominious          ignomini
-    ignominy             ignomini
-    ignomy               ignomi
-    ignorance            ignor
-    ignorant             ignor
-    ii                   ii
-    iii                  iii
-    iiii                 iiii
-    il                   il
-    ilbow                ilbow
-    ild                  ild
-    ilion                ilion
-    ilium                ilium
-    ill                  ill
-    illegitimate         illegitim
-    illiterate           illiter
-    illness              ill
-    illo                 illo
-    ills                 ill
-    illume               illum
-    illumin              illumin
-    illuminate           illumin
-    illumineth           illumineth
-    illusion             illus
-    illusions            illus
-    illustrate           illustr
-    illustrated          illustr
-    illustrious          illustri
-    illyria              illyria
-    illyrian             illyrian
-    ils                  il
-    im                   im
-    image                imag
-    imagery              imageri
-    images               imag
-    imagin               imagin
-    imaginary            imaginari
-    imagination          imagin
-    imaginations         imagin
-    imagine              imagin
-    imagining            imagin
-    imaginings           imagin
-    imbar                imbar
-    imbecility           imbecil
-    imbrue               imbru
-    imitari              imitari
-    imitate              imit
-    imitated             imit
-    imitation            imit
-    imitations           imit
-    immaculate           immacul
-    immanity             imman
-    immask               immask
-    immaterial           immateri
-    immediacy            immediaci
-    immediate            immedi
-    immediately          immedi
-    imminence            immin
-    imminent             immin
-    immoderate           immoder
-    immoderately         immoder
-    immodest             immodest
-    immoment             immoment
-    immortal             immort
-    immortaliz           immortaliz
-    immortally           immort
-    immur                immur
-    immured              immur
-    immures              immur
-    imogen               imogen
-    imp                  imp
-    impaint              impaint
-    impair               impair
-    impairing            impair
-    impale               impal
-    impaled              impal
-    impanelled           impanel
-    impart               impart
-    imparted             impart
-    impartial            imparti
-    impartment           impart
-    imparts              impart
-    impasted             impast
-    impatience           impati
-    impatient            impati
-    impatiently          impati
-    impawn               impawn
-    impeach              impeach
-    impeached            impeach
-    impeachment          impeach
-    impeachments         impeach
-    impedes              imped
-    impediment           impedi
-    impediments          impedi
-    impenetrable         impenetr
-    imperator            imper
-    imperceiverant       imperceiver
-    imperfect            imperfect
-    imperfection         imperfect
-    imperfections        imperfect
-    imperfectly          imperfectli
-    imperial             imperi
-    imperious            imperi
-    imperiously          imperi
-    impertinency         impertin
-    impertinent          impertin
-    impeticos            impetico
-    impetuosity          impetuos
-    impetuous            impetu
-    impieties            impieti
-    impiety              impieti
-    impious              impiou
-    implacable           implac
-    implements           implement
-    implies              impli
-    implor               implor
-    implorators          implor
-    implore              implor
-    implored             implor
-    imploring            implor
-    impon                impon
-    import               import
-    importance           import
-    importancy           import
-    important            import
-    importantly          importantli
-    imported             import
-    importeth            importeth
-    importing            import
-    importless           importless
-    imports              import
-    importun             importun
-    importunacy          importunaci
-    importunate          importun
-    importune            importun
-    importunes           importun
-    importunity          importun
-    impos                impo
-    impose               impos
-    imposed              impos
-    imposition           imposit
-    impositions          imposit
-    impossibilities      imposs
-    impossibility        imposs
-    impossible           imposs
-    imposthume           imposthum
-    impostor             impostor
-    impostors            impostor
-    impotence            impot
-    impotent             impot
-    impounded            impound
-    impregnable          impregn
-    imprese              impres
-    impress              impress
-    impressed            impress
-    impressest           impressest
-    impression           impress
-    impressure           impressur
-    imprimendum          imprimendum
-    imprimis             imprimi
-    imprint              imprint
-    imprinted            imprint
-    imprison             imprison
-    imprisoned           imprison
-    imprisoning          imprison
-    imprisonment         imprison
-    improbable           improb
-    improper             improp
-    improve              improv
-    improvident          improvid
-    impudence            impud
-    impudency            impud
-    impudent             impud
-    impudently           impud
-    impudique            impudiqu
-    impugn               impugn
-    impugns              impugn
-    impure               impur
-    imputation           imput
-    impute               imput
-    in                   in
-    inaccessible         inaccess
-    inaidable            inaid
-    inaudible            inaud
-    inauspicious         inauspici
-    incaged              incag
-    incantations         incant
-    incapable            incap
-    incardinate          incardin
-    incarnadine          incarnadin
-    incarnate            incarn
-    incarnation          incarn
-    incens               incen
-    incense              incens
-    incensed             incens
-    incensement          incens
-    incenses             incens
-    incensing            incens
-    incertain            incertain
-    incertainties        incertainti
-    incertainty          incertainti
-    incessant            incess
-    incessantly          incessantli
-    incest               incest
-    incestuous           incestu
-    inch                 inch
-    incharitable         incharit
-    inches               inch
-    incidency            incid
-    incident             incid
-    incision             incis
-    incite               incit
-    incites              incit
-    incivil              incivil
-    incivility           incivil
-    inclin               inclin
-    inclinable           inclin
-    inclination          inclin
-    incline              inclin
-    inclined             inclin
-    inclines             inclin
-    inclining            inclin
-    inclips              inclip
-    include              includ
-    included             includ
-    includes             includ
-    inclusive            inclus
-    incomparable         incompar
-    incomprehensible     incomprehens
-    inconsiderate        inconsider
-    inconstancy          inconst
-    inconstant           inconst
-    incontinency         incontin
-    incontinent          incontin
-    incontinently        incontin
-    inconvenience        inconveni
-    inconveniences       inconveni
-    inconvenient         inconveni
-    incony               inconi
-    incorporate          incorpor
-    incorps              incorp
-    incorrect            incorrect
-    increas              increa
-    increase             increas
-    increases            increas
-    increaseth           increaseth
-    increasing           increas
-    incredible           incred
-    incredulous          incredul
-    incur                incur
-    incurable            incur
-    incurr               incurr
-    incurred             incur
-    incursions           incurs
-    ind                  ind
-    inde                 ind
-    indebted             indebt
-    indeed               inde
-    indent               indent
-    indented             indent
-    indenture            indentur
-    indentures           indentur
-    index                index
-    indexes              index
-    india                india
-    indian               indian
-    indict               indict
-    indicted             indict
-    indictment           indict
-    indies               indi
-    indifferency         indiffer
-    indifferent          indiffer
-    indifferently        indiffer
-    indigent             indig
-    indigest             indigest
-    indigested           indigest
-    indign               indign
-    indignation          indign
-    indignations         indign
-    indigne              indign
-    indignities          indign
-    indignity            indign
-    indirect             indirect
-    indirection          indirect
-    indirections         indirect
-    indirectly           indirectli
-    indiscreet           indiscreet
-    indiscretion         indiscret
-    indispos             indispo
-    indisposition        indisposit
-    indissoluble         indissolubl
-    indistinct           indistinct
-    indistinguish        indistinguish
-    indistinguishable    indistinguish
-    indited              indit
-    individable          individ
-    indrench             indrench
-    indu                 indu
-    indubitate           indubit
-    induc                induc
-    induce               induc
-    induced              induc
-    inducement           induc
-    induction            induct
-    inductions           induct
-    indue                indu
-    indued               indu
-    indues               indu
-    indulgence           indulg
-    indulgences          indulg
-    indulgent            indulg
-    indurance            indur
-    industrious          industri
-    industriously        industri
-    industry             industri
-    inequality           inequ
-    inestimable          inestim
-    inevitable           inevit
-    inexecrable          inexecr
-    inexorable           inexor
-    inexplicable         inexplic
-    infallible           infal
-    infallibly           infal
-    infamonize           infamon
-    infamous             infam
-    infamy               infami
-    infancy              infanc
-    infant               infant
-    infants              infant
-    infect               infect
-    infected             infect
-    infecting            infect
-    infection            infect
-    infections           infect
-    infectious           infecti
-    infectiously         infecti
-    infects              infect
-    infer                infer
-    inference            infer
-    inferior             inferior
-    inferiors            inferior
-    infernal             infern
-    inferr               inferr
-    inferreth            inferreth
-    inferring            infer
-    infest               infest
-    infidel              infidel
-    infidels             infidel
-    infinite             infinit
-    infinitely           infinit
-    infinitive           infinit
-    infirm               infirm
-    infirmities          infirm
-    infirmity            infirm
-    infixed              infix
-    infixing             infix
-    inflam               inflam
-    inflame              inflam
-    inflaming            inflam
-    inflammation         inflamm
-    inflict              inflict
-    infliction           inflict
-    influence            influenc
-    influences           influenc
-    infold               infold
-    inform               inform
-    informal             inform
-    information          inform
-    informations         inform
-    informed             inform
-    informer             inform
-    informs              inform
-    infortunate          infortun
-    infring              infr
-    infringe             infring
-    infringed            infring
-    infus                infu
-    infuse               infus
-    infused              infus
-    infusing             infus
-    infusion             infus
-    ingener              ingen
-    ingenious            ingeni
-    ingeniously          ingeni
-    inglorious           inglori
-    ingots               ingot
-    ingraffed            ingraf
-    ingraft              ingraft
-    ingrate              ingrat
-    ingrated             ingrat
-    ingrateful           ingrat
-    ingratitude          ingratitud
-    ingratitudes         ingratitud
-    ingredient           ingredi
-    ingredients          ingredi
-    ingross              ingross
-    inhabit              inhabit
-    inhabitable          inhabit
-    inhabitants          inhabit
-    inhabited            inhabit
-    inhabits             inhabit
-    inhearse             inhears
-    inhearsed            inhears
-    inherent             inher
-    inherit              inherit
-    inheritance          inherit
-    inherited            inherit
-    inheriting           inherit
-    inheritor            inheritor
-    inheritors           inheritor
-    inheritrix           inheritrix
-    inherits             inherit
-    inhibited            inhibit
-    inhibition           inhibit
-    inhoop               inhoop
-    inhuman              inhuman
-    iniquities           iniqu
-    iniquity             iniqu
-    initiate             initi
-    injointed            injoint
-    injunction           injunct
-    injunctions          injunct
-    injur                injur
-    injure               injur
-    injurer              injur
-    injuries             injuri
-    injurious            injuri
-    injury               injuri
-    injustice            injustic
-    ink                  ink
-    inkhorn              inkhorn
-    inkle                inkl
-    inkles               inkl
-    inkling              inkl
-    inky                 inki
-    inlaid               inlaid
-    inland               inland
-    inlay                inlai
-    inly                 inli
-    inmost               inmost
-    inn                  inn
-    inner                inner
-    innkeeper            innkeep
-    innocence            innoc
-    innocency            innoc
-    innocent             innoc
-    innocents            innoc
-    innovation           innov
-    innovator            innov
-    inns                 inn
-    innumerable          innumer
-    inoculate            inocul
-    inordinate           inordin
-    inprimis             inprimi
-    inquir               inquir
-    inquire              inquir
-    inquiry              inquiri
-    inquisition          inquisit
-    inquisitive          inquisit
-    inroads              inroad
-    insane               insan
-    insanie              insani
-    insatiate            insati
-    insconce             insconc
-    inscrib              inscrib
-    inscription          inscript
-    inscriptions         inscript
-    inscroll             inscrol
-    inscrutable          inscrut
-    insculp              insculp
-    insculpture          insculptur
-    insensible           insens
-    inseparable          insepar
-    inseparate           insepar
-    insert               insert
-    inserted             insert
-    inset                inset
-    inshell              inshel
-    inshipp              inshipp
-    inside               insid
-    insinewed            insinew
-    insinuate            insinu
-    insinuateth          insinuateth
-    insinuating          insinu
-    insinuation          insinu
-    insisted             insist
-    insisting            insist
-    insisture            insistur
-    insociable           insoci
-    insolence            insol
-    insolent             insol
-    insomuch             insomuch
-    inspir               inspir
-    inspiration          inspir
-    inspirations         inspir
-    inspire              inspir
-    inspired             inspir
-    install              instal
-    installed            instal
-    instalment           instal
-    instance             instanc
-    instances            instanc
-    instant              instant
-    instantly            instantli
-    instate              instat
-    instead              instead
-    insteeped            insteep
-    instigate            instig
-    instigated           instig
-    instigation          instig
-    instigations         instig
-    instigator           instig
-    instinct             instinct
-    instinctively        instinct
-    institute            institut
-    institutions         institut
-    instruct             instruct
-    instructed           instruct
-    instruction          instruct
-    instructions         instruct
-    instructs            instruct
-    instrument           instrument
-    instrumental         instrument
-    instruments          instrument
-    insubstantial        insubstanti
-    insufficience        insuffici
-    insufficiency        insuffici
-    insult               insult
-    insulted             insult
-    insulting            insult
-    insultment           insult
-    insults              insult
-    insupportable        insupport
-    insuppressive        insuppress
-    insurrection         insurrect
-    insurrections        insurrect
-    int                  int
-    integer              integ
-    integritas           integrita
-    integrity            integr
-    intellect            intellect
-    intellects           intellect
-    intellectual         intellectu
-    intelligence         intellig
-    intelligencer        intelligenc
-    intelligencing       intelligenc
-    intelligent          intellig
-    intelligis           intelligi
-    intelligo            intelligo
-    intemperance         intemper
-    intemperate          intemper
-    intend               intend
-    intended             intend
-    intendeth            intendeth
-    intending            intend
-    intendment           intend
-    intends              intend
-    intenible            inten
-    intent               intent
-    intention            intent
-    intentively          intent
-    intents              intent
-    inter                inter
-    intercept            intercept
-    intercepted          intercept
-    intercepter          intercept
-    interception         intercept
-    intercepts           intercept
-    intercession         intercess
-    intercessors         intercessor
-    interchained         interchain
-    interchang           interchang
-    interchange          interchang
-    interchangeably      interchang
-    interchangement      interchang
-    interchanging        interchang
-    interdiction         interdict
-    interest             interest
-    interim              interim
-    interims             interim
-    interior             interior
-    interjections        interject
-    interjoin            interjoin
-    interlude            interlud
-    intermingle          intermingl
-    intermission         intermiss
-    intermissive         intermiss
-    intermit             intermit
-    intermix             intermix
-    intermixed           intermix
-    interpose            interpos
-    interposer           interpos
-    interposes           interpos
-    interpret            interpret
-    interpretation       interpret
-    interpreted          interpret
-    interpreter          interpret
-    interpreters         interpret
-    interprets           interpret
-    interr               interr
-    interred             inter
-    interrogatories      interrogatori
-    interrupt            interrupt
-    interrupted          interrupt
-    interrupter          interrupt
-    interruptest         interruptest
-    interruption         interrupt
-    interrupts           interrupt
-    intertissued         intertissu
-    intervallums         intervallum
-    interview            interview
-    intestate            intest
-    intestine            intestin
-    intil                intil
-    intimate             intim
-    intimation           intim
-    intitled             intitl
-    intituled            intitul
-    into                 into
-    intolerable          intoler
-    intoxicates          intox
-    intreasured          intreasur
-    intreat              intreat
-    intrench             intrench
-    intrenchant          intrench
-    intricate            intric
-    intrinse             intrins
-    intrinsicate         intrins
-    intrude              intrud
-    intruder             intrud
-    intruding            intrud
-    intrusion            intrus
-    inundation           inund
-    inure                inur
-    inurn                inurn
-    invade               invad
-    invades              invad
-    invasion             invas
-    invasive             invas
-    invectively          invect
-    invectives           invect
-    inveigled            inveigl
-    invent               invent
-    invented             invent
-    invention            invent
-    inventions           invent
-    inventor             inventor
-    inventorially        inventori
-    inventoried          inventori
-    inventors            inventor
-    inventory            inventori
-    inverness            inver
-    invert               invert
-    invest               invest
-    invested             invest
-    investing            invest
-    investments          invest
-    inveterate           inveter
-    invincible           invinc
-    inviolable           inviol
-    invised              invis
-    invisible            invis
-    invitation           invit
-    invite               invit
-    invited              invit
-    invites              invit
-    inviting             invit
-    invitis              inviti
-    invocate             invoc
-    invocation           invoc
-    invoke               invok
-    invoked              invok
-    invulnerable         invulner
-    inward               inward
-    inwardly             inwardli
-    inwardness           inward
-    inwards              inward
-    ionia                ionia
-    ionian               ionian
-    ipse                 ips
-    ipswich              ipswich
-    ira                  ira
-    irae                 ira
-    iras                 ira
-    ire                  ir
-    ireful               ir
-    ireland              ireland
-    iris                 iri
-    irish                irish
-    irishman             irishman
-    irishmen             irishmen
-    irks                 irk
-    irksome              irksom
-    iron                 iron
-    irons                iron
-    irreconcil           irreconcil
-    irrecoverable        irrecover
-    irregular            irregular
-    irregulous           irregul
-    irreligious          irreligi
-    irremovable          irremov
-    irreparable          irrepar
-    irresolute           irresolut
-    irrevocable          irrevoc
-    is                   is
-    isabel               isabel
-    isabella             isabella
-    isbel                isbel
-    isbels               isbel
-    iscariot             iscariot
-    ise                  is
-    ish                  ish
-    isidore              isidor
-    isis                 isi
-    island               island
-    islander             island
-    islanders            island
-    islands              island
-    isle                 isl
-    isles                isl
-    israel               israel
-    issu                 issu
-    issue                issu
-    issued               issu
-    issueless            issueless
-    issues               issu
-    issuing              issu
-    ist                  ist
-    ista                 ista
-    it                   it
-    italian              italian
-    italy                itali
-    itch                 itch
-    itches               itch
-    itching              itch
-    item                 item
-    items                item
-    iteration            iter
-    ithaca               ithaca
-    its                  it
-    itself               itself
-    itshall              itshal
-    iv                   iv
-    ivory                ivori
-    ivy                  ivi
-    iwis                 iwi
-    ix                   ix
-    j                    j
-    jacet                jacet
-    jack                 jack
-    jackanapes           jackanap
-    jacks                jack
-    jacksauce            jacksauc
-    jackslave            jackslav
-    jacob                jacob
-    jade                 jade
-    jaded                jade
-    jades                jade
-    jail                 jail
-    jakes                jake
-    jamany               jamani
-    james                jame
-    jamy                 jami
-    jane                 jane
-    jangled              jangl
-    jangling             jangl
-    january              januari
-    janus                janu
-    japhet               japhet
-    jaquenetta           jaquenetta
-    jaques               jaqu
-    jar                  jar
-    jarring              jar
-    jars                 jar
-    jarteer              jarteer
-    jasons               jason
-    jaunce               jaunc
-    jauncing             jaunc
-    jaundice             jaundic
-    jaundies             jaundi
-    jaw                  jaw
-    jawbone              jawbon
-    jaws                 jaw
-    jay                  jai
-    jays                 jai
-    jc                   jc
-    je                   je
-    jealous              jealou
-    jealousies           jealousi
-    jealousy             jealousi
-    jeer                 jeer
-    jeering              jeer
-    jelly                jelli
-    jenny                jenni
-    jeopardy             jeopardi
-    jephtha              jephtha
-    jephthah             jephthah
-    jerkin               jerkin
-    jerkins              jerkin
-    jerks                jerk
-    jeronimy             jeronimi
-    jerusalem            jerusalem
-    jeshu                jeshu
-    jesses               jess
-    jessica              jessica
-    jest                 jest
-    jested               jest
-    jester               jester
-    jesters              jester
-    jesting              jest
-    jests                jest
-    jesu                 jesu
-    jesus                jesu
-    jet                  jet
-    jets                 jet
-    jew                  jew
-    jewel                jewel
-    jeweller             jewel
-    jewels               jewel
-    jewess               jewess
-    jewish               jewish
-    jewry                jewri
-    jews                 jew
-    jezebel              jezebel
-    jig                  jig
-    jigging              jig
-    jill                 jill
-    jills                jill
-    jingling             jingl
-    joan                 joan
-    job                  job
-    jockey               jockei
-    jocund               jocund
-    jog                  jog
-    jogging              jog
-    john                 john
-    johns                john
-    join                 join
-    joinder              joinder
-    joined               join
-    joiner               joiner
-    joineth              joineth
-    joins                join
-    joint                joint
-    jointed              joint
-    jointing             joint
-    jointly              jointli
-    jointress            jointress
-    joints               joint
-    jointure             jointur
-    jollity              jolliti
-    jolly                jolli
-    jolt                 jolt
-    joltheads            jolthead
-    jordan               jordan
-    joseph               joseph
-    joshua               joshua
-    jot                  jot
-    jour                 jour
-    jourdain             jourdain
-    journal              journal
-    journey              journei
-    journeying           journei
-    journeyman           journeyman
-    journeymen           journeymen
-    journeys             journei
-    jove                 jove
-    jovem                jovem
-    jovial               jovial
-    jowl                 jowl
-    jowls                jowl
-    joy                  joi
-    joyed                joi
-    joyful               joy
-    joyfully             joyfulli
-    joyless              joyless
-    joyous               joyou
-    joys                 joi
-    juan                 juan
-    jud                  jud
-    judas                juda
-    judases              judas
-    jude                 jude
-    judg                 judg
-    judge                judg
-    judged               judg
-    judgement            judgement
-    judges               judg
-    judgest              judgest
-    judging              judg
-    judgment             judgment
-    judgments            judgment
-    judicious            judici
-    jug                  jug
-    juggle               juggl
-    juggled              juggl
-    juggler              juggler
-    jugglers             juggler
-    juggling             juggl
-    jugs                 jug
-    juice                juic
-    juiced               juic
-    jul                  jul
-    jule                 jule
-    julia                julia
-    juliet               juliet
-    julietta             julietta
-    julio                julio
-    julius               juliu
-    july                 juli
-    jump                 jump
-    jumpeth              jumpeth
-    jumping              jump
-    jumps                jump
-    june                 june
-    junes                june
-    junior               junior
-    junius               juniu
-    junkets              junket
-    juno                 juno
-    jupiter              jupit
-    jure                 jure
-    jurement             jurement
-    jurisdiction         jurisdict
-    juror                juror
-    jurors               juror
-    jury                 juri
-    jurymen              jurymen
-    just                 just
-    justeius             justeiu
-    justest              justest
-    justice              justic
-    justicer             justic
-    justicers            justic
-    justices             justic
-    justification        justif
-    justified            justifi
-    justify              justifi
-    justle               justl
-    justled              justl
-    justles              justl
-    justling             justl
-    justly               justli
-    justness             just
-    justs                just
-    jutting              jut
-    jutty                jutti
-    juvenal              juven
-    kam                  kam
-    kate                 kate
-    kated                kate
-    kates                kate
-    katharine            katharin
-    katherina            katherina
-    katherine            katherin
-    kecksies             kecksi
-    keech                keech
-    keel                 keel
-    keels                keel
-    keen                 keen
-    keenness             keen
-    keep                 keep
-    keepdown             keepdown
-    keeper               keeper
-    keepers              keeper
-    keepest              keepest
-    keeping              keep
-    keeps                keep
-    keiser               keiser
-    ken                  ken
-    kendal               kendal
-    kennel               kennel
-    kent                 kent
-    kentish              kentish
-    kentishman           kentishman
-    kentishmen           kentishmen
-    kept                 kept
-    kerchief             kerchief
-    kerely               kere
-    kern                 kern
-    kernal               kernal
-    kernel               kernel
-    kernels              kernel
-    kerns                kern
-    kersey               kersei
-    kettle               kettl
-    kettledrum           kettledrum
-    kettledrums          kettledrum
-    key                  kei
-    keys                 kei
-    kibe                 kibe
-    kibes                kibe
-    kick                 kick
-    kicked               kick
-    kickshaws            kickshaw
-    kickshawses          kickshaws
-    kicky                kicki
-    kid                  kid
-    kidney               kidnei
-    kikely               kike
-    kildare              kildar
-    kill                 kill
-    killed               kill
-    killer               killer
-    killeth              killeth
-    killing              kill
-    killingworth         killingworth
-    kills                kill
-    kiln                 kiln
-    kimbolton            kimbolton
-    kin                  kin
-    kind                 kind
-    kinder               kinder
-    kindest              kindest
-    kindle               kindl
-    kindled              kindl
-    kindless             kindless
-    kindlier             kindlier
-    kindling             kindl
-    kindly               kindli
-    kindness             kind
-    kindnesses           kind
-    kindred              kindr
-    kindreds             kindr
-    kinds                kind
-    kine                 kine
-    king                 king
-    kingdom              kingdom
-    kingdoms             kingdom
-    kingly               kingli
-    kings                king
-    kinred               kinr
-    kins                 kin
-    kinsman              kinsman
-    kinsmen              kinsmen
-    kinswoman            kinswoman
-    kirtle               kirtl
-    kirtles              kirtl
-    kiss                 kiss
-    kissed               kiss
-    kisses               kiss
-    kissing              kiss
-    kitchen              kitchen
-    kitchens             kitchen
-    kite                 kite
-    kites                kite
-    kitten               kitten
-    kj                   kj
-    kl                   kl
-    klll                 klll
-    knack                knack
-    knacks               knack
-    knapp                knapp
-    knav                 knav
-    knave                knave
-    knaveries            knaveri
-    knavery              knaveri
-    knaves               knave
-    knavish              knavish
-    knead                knead
-    kneaded              knead
-    kneading             knead
-    knee                 knee
-    kneel                kneel
-    kneeling             kneel
-    kneels               kneel
-    knees                knee
-    knell                knell
-    knew                 knew
-    knewest              knewest
-    knife                knife
-    knight               knight
-    knighted             knight
-    knighthood           knighthood
-    knighthoods          knighthood
-    knightly             knightli
-    knights              knight
-    knit                 knit
-    knits                knit
-    knitters             knitter
-    knitteth             knitteth
-    knives               knive
-    knobs                knob
-    knock                knock
-    knocking             knock
-    knocks               knock
-    knog                 knog
-    knoll                knoll
-    knot                 knot
-    knots                knot
-    knotted              knot
-    knotty               knotti
-    know                 know
-    knower               knower
-    knowest              knowest
-    knowing              know
-    knowingly            knowingli
-    knowings             know
-    knowledge            knowledg
-    known                known
-    knows                know
-    l                    l
-    la                   la
-    laban                laban
-    label                label
-    labell               label
-    labienus             labienu
-    labio                labio
-    labor                labor
-    laboring             labor
-    labors               labor
-    labour               labour
-    laboured             labour
-    labourer             labour
-    labourers            labour
-    labouring            labour
-    labours              labour
-    laboursome           laboursom
-    labras               labra
-    labyrinth            labyrinth
-    lac                  lac
-    lace                 lace
-    laced                lace
-    lacedaemon           lacedaemon
-    laces                lace
-    lacies               laci
-    lack                 lack
-    lackbeard            lackbeard
-    lacked               lack
-    lackey               lackei
-    lackeying            lackei
-    lackeys              lackei
-    lacking              lack
-    lacks                lack
-    lad                  lad
-    ladder               ladder
-    ladders              ladder
-    lade                 lade
-    laden                laden
-    ladies               ladi
-    lading               lade
-    lads                 lad
-    lady                 ladi
-    ladybird             ladybird
-    ladyship             ladyship
-    ladyships            ladyship
-    laer                 laer
-    laertes              laert
-    lafeu                lafeu
-    lag                  lag
-    lagging              lag
-    laid                 laid
-    lain                 lain
-    laissez              laissez
-    lake                 lake
-    lakes                lake
-    lakin                lakin
-    lam                  lam
-    lamb                 lamb
-    lambert              lambert
-    lambkin              lambkin
-    lambkins             lambkin
-    lambs                lamb
-    lame                 lame
-    lamely               lame
-    lameness             lame
-    lament               lament
-    lamentable           lament
-    lamentably           lament
-    lamentation          lament
-    lamentations         lament
-    lamented             lament
-    lamenting            lament
-    lamentings           lament
-    laments              lament
-    lames                lame
-    laming               lame
-    lammas               lamma
-    lammastide           lammastid
-    lamound              lamound
-    lamp                 lamp
-    lampass              lampass
-    lamps                lamp
-    lanc                 lanc
-    lancaster            lancast
-    lance                lanc
-    lances               lanc
-    lanceth              lanceth
-    lanch                lanch
-    land                 land
-    landed               land
-    landing              land
-    landless             landless
-    landlord             landlord
-    landmen              landmen
-    lands                land
-    lane                 lane
-    lanes                lane
-    langage              langag
-    langley              langlei
-    langton              langton
-    language             languag
-    languageless         languageless
-    languages            languag
-    langues              langu
-    languish             languish
-    languished           languish
-    languishes           languish
-    languishing          languish
-    languishings         languish
-    languishment         languish
-    languor              languor
-    lank                 lank
-    lantern              lantern
-    lanterns             lantern
-    lanthorn             lanthorn
-    lap                  lap
-    lapis                lapi
-    lapland              lapland
-    lapp                 lapp
-    laps                 lap
-    lapse                laps
-    lapsed               laps
-    lapsing              laps
-    lapwing              lapw
-    laquais              laquai
-    larded               lard
-    larder               larder
-    larding              lard
-    lards                lard
-    large                larg
-    largely              larg
-    largeness            larg
-    larger               larger
-    largess              largess
-    largest              largest
-    lark                 lark
-    larks                lark
-    larron               larron
-    lartius              lartiu
-    larum                larum
-    larums               larum
-    las                  la
-    lascivious           lascivi
-    lash                 lash
-    lass                 lass
-    lasses               lass
-    last                 last
-    lasted               last
-    lasting              last
-    lastly               lastli
-    lasts                last
-    latch                latch
-    latches              latch
-    late                 late
-    lated                late
-    lately               late
-    later                later
-    latest               latest
-    lath                 lath
-    latin                latin
-    latten               latten
-    latter               latter
-    lattice              lattic
-    laud                 laud
-    laudable             laudabl
-    laudis               laudi
-    laugh                laugh
-    laughable            laughabl
-    laughed              laugh
-    laugher              laugher
-    laughest             laughest
-    laughing             laugh
-    laughs               laugh
-    laughter             laughter
-    launce               launc
-    launcelot            launcelot
-    launces              launc
-    launch               launch
-    laund                laund
-    laundress            laundress
-    laundry              laundri
-    laur                 laur
-    laura                laura
-    laurel               laurel
-    laurels              laurel
-    laurence             laurenc
-    laus                 lau
-    lavache              lavach
-    lave                 lave
-    lavee                lave
-    lavender             lavend
-    lavina               lavina
-    lavinia              lavinia
-    lavish               lavish
-    lavishly             lavishli
-    lavolt               lavolt
-    lavoltas             lavolta
-    law                  law
-    lawful               law
-    lawfully             lawfulli
-    lawless              lawless
-    lawlessly            lawlessli
-    lawn                 lawn
-    lawns                lawn
-    lawrence             lawrenc
-    laws                 law
-    lawyer               lawyer
-    lawyers              lawyer
-    lay                  lai
-    layer                layer
-    layest               layest
-    laying               lai
-    lays                 lai
-    lazar                lazar
-    lazars               lazar
-    lazarus              lazaru
-    lazy                 lazi
-    lc                   lc
-    ld                   ld
-    ldst                 ldst
-    le                   le
-    lead                 lead
-    leaden               leaden
-    leader               leader
-    leaders              leader
-    leadest              leadest
-    leading              lead
-    leads                lead
-    leaf                 leaf
-    leagu                leagu
-    league               leagu
-    leagued              leagu
-    leaguer              leaguer
-    leagues              leagu
-    leah                 leah
-    leak                 leak
-    leaky                leaki
-    lean                 lean
-    leander              leander
-    leaner               leaner
-    leaning              lean
-    leanness             lean
-    leans                lean
-    leap                 leap
-    leaped               leap
-    leaping              leap
-    leaps                leap
-    leapt                leapt
-    lear                 lear
-    learn                learn
-    learned              learn
-    learnedly            learnedli
-    learning             learn
-    learnings            learn
-    learns               learn
-    learnt               learnt
-    leas                 lea
-    lease                leas
-    leases               leas
-    leash                leash
-    leasing              leas
-    least                least
-    leather              leather
-    leathern             leathern
-    leav                 leav
-    leave                leav
-    leaven               leaven
-    leavening            leaven
-    leaver               leaver
-    leaves               leav
-    leaving              leav
-    leavy                leavi
-    lecher               lecher
-    lecherous            lecher
-    lechers              lecher
-    lechery              lecheri
-    lecon                lecon
-    lecture              lectur
-    lectures             lectur
-    led                  led
-    leda                 leda
-    leech                leech
-    leeches              leech
-    leek                 leek
-    leeks                leek
-    leer                 leer
-    leers                leer
-    lees                 lee
-    leese                lees
-    leet                 leet
-    leets                leet
-    left                 left
-    leg                  leg
-    legacies             legaci
-    legacy               legaci
-    legate               legat
-    legatine             legatin
-    lege                 lege
-    legerity             leger
-    leges                lege
-    legg                 legg
-    legion               legion
-    legions              legion
-    legitimate           legitim
-    legitimation         legitim
-    legs                 leg
-    leicester            leicest
-    leicestershire       leicestershir
-    leiger               leiger
-    leigers              leiger
-    leisure              leisur
-    leisurely            leisur
-    leisures             leisur
-    leman                leman
-    lemon                lemon
-    lena                 lena
-    lend                 lend
-    lender               lender
-    lending              lend
-    lendings             lend
-    lends                lend
-    length               length
-    lengthen             lengthen
-    lengthens            lengthen
-    lengths              length
-    lenity               leniti
-    lennox               lennox
-    lent                 lent
-    lenten               lenten
-    lentus               lentu
-    leo                  leo
-    leon                 leon
-    leonardo             leonardo
-    leonati              leonati
-    leonato              leonato
-    leonatus             leonatu
-    leontes              leont
-    leopard              leopard
-    leopards             leopard
-    leper                leper
-    leperous             leper
-    lepidus              lepidu
-    leprosy              leprosi
-    lequel               lequel
-    lers                 ler
-    les                  le
-    less                 less
-    lessen               lessen
-    lessens              lessen
-    lesser               lesser
-    lesson               lesson
-    lessoned             lesson
-    lessons              lesson
-    lest                 lest
-    lestrake             lestrak
-    let                  let
-    lethargied           lethargi
-    lethargies           lethargi
-    lethargy             lethargi
-    lethe                leth
-    lets                 let
-    lett                 lett
-    letter               letter
-    letters              letter
-    letting              let
-    lettuce              lettuc
-    leur                 leur
-    leve                 leve
-    level                level
-    levell               level
-    levelled             level
-    levels               level
-    leven                leven
-    levers               lever
-    leviathan            leviathan
-    leviathans           leviathan
-    levied               levi
-    levies               levi
-    levity               leviti
-    levy                 levi
-    levying              levi
-    lewd                 lewd
-    lewdly               lewdli
-    lewdness             lewd
-    lewdsters            lewdster
-    lewis                lewi
-    liable               liabl
-    liar                 liar
-    liars                liar
-    libbard              libbard
-    libelling            libel
-    libels               libel
-    liberal              liber
-    liberality           liber
-    liberte              libert
-    liberties            liberti
-    libertine            libertin
-    libertines           libertin
-    liberty              liberti
-    library              librari
-    libya                libya
-    licence              licenc
-    licens               licen
-    license              licens
-    licentious           licenti
-    lichas               licha
-    licio                licio
-    lick                 lick
-    licked               lick
-    licker               licker
-    lictors              lictor
-    lid                  lid
-    lids                 lid
-    lie                  lie
-    lied                 li
-    lief                 lief
-    liefest              liefest
-    liege                lieg
-    liegeman             liegeman
-    liegemen             liegemen
-    lien                 lien
-    lies                 li
-    liest                liest
-    lieth                lieth
-    lieu                 lieu
-    lieutenant           lieuten
-    lieutenantry         lieutenantri
-    lieutenants          lieuten
-    lieve                liev
-    life                 life
-    lifeblood            lifeblood
-    lifeless             lifeless
-    lifelings            lifel
-    lift                 lift
-    lifted               lift
-    lifter               lifter
-    lifteth              lifteth
-    lifting              lift
-    lifts                lift
-    lig                  lig
-    ligarius             ligariu
-    liggens              liggen
-    light                light
-    lighted              light
-    lighten              lighten
-    lightens             lighten
-    lighter              lighter
-    lightest             lightest
-    lightly              lightli
-    lightness            light
-    lightning            lightn
-    lightnings           lightn
-    lights               light
-    lik                  lik
-    like                 like
-    liked                like
-    likeliest            likeliest
-    likelihood           likelihood
-    likelihoods          likelihood
-    likely               like
-    likeness             like
-    liker                liker
-    likes                like
-    likest               likest
-    likewise             likewis
-    liking               like
-    likings              like
-    lilies               lili
-    lily                 lili
-    lim                  lim
-    limander             limand
-    limb                 limb
-    limbeck              limbeck
-    limbecks             limbeck
-    limber               limber
-    limbo                limbo
-    limbs                limb
-    lime                 lime
-    limed                lime
-    limehouse            limehous
-    limekilns            limekiln
-    limit                limit
-    limitation           limit
-    limited              limit
-    limits               limit
-    limn                 limn
-    limp                 limp
-    limping              limp
-    limps                limp
-    lin                  lin
-    lincoln              lincoln
-    lincolnshire         lincolnshir
-    line                 line
-    lineal               lineal
-    lineally             lineal
-    lineament            lineament
-    lineaments           lineament
-    lined                line
-    linen                linen
-    linens               linen
-    lines                line
-    ling                 ling
-    lingare              lingar
-    linger               linger
-    lingered             linger
-    lingers              linger
-    linguist             linguist
-    lining               line
-    link                 link
-    links                link
-    linsey               linsei
-    linstock             linstock
-    linta                linta
-    lion                 lion
-    lionel               lionel
-    lioness              lioness
-    lions                lion
-    lip                  lip
-    lipp                 lipp
-    lips                 lip
-    lipsbury             lipsburi
-    liquid               liquid
-    liquor               liquor
-    liquorish            liquorish
-    liquors              liquor
-    lirra                lirra
-    lisbon               lisbon
-    lisp                 lisp
-    lisping              lisp
-    list                 list
-    listen               listen
-    listening            listen
-    lists                list
-    literatured          literatur
-    lither               lither
-    litter               litter
-    little               littl
-    littlest             littlest
-    liv                  liv
-    live                 live
-    lived                live
-    livelier             liveli
-    livelihood           livelihood
-    livelong             livelong
-    lively               live
-    liver                liver
-    liveries             liveri
-    livers               liver
-    livery               liveri
-    lives                live
-    livest               livest
-    liveth               liveth
-    livia                livia
-    living               live
-    livings              live
-    lizard               lizard
-    lizards              lizard
-    ll                   ll
-    lll                  lll
-    llous                llou
-    lnd                  lnd
-    lo                   lo
-    loa                  loa
-    loach                loach
-    load                 load
-    loaden               loaden
-    loading              load
-    loads                load
-    loaf                 loaf
-    loam                 loam
-    loan                 loan
-    loath                loath
-    loathe               loath
-    loathed              loath
-    loather              loather
-    loathes              loath
-    loathing             loath
-    loathly              loathli
-    loathness            loath
-    loathsome            loathsom
-    loathsomeness        loathsom
-    loathsomest          loathsomest
-    loaves               loav
-    lob                  lob
-    lobbies              lobbi
-    lobby                lobbi
-    local                local
-    lochaber             lochab
-    lock                 lock
-    locked               lock
-    locking              lock
-    lockram              lockram
-    locks                lock
-    locusts              locust
-    lode                 lode
-    lodg                 lodg
-    lodge                lodg
-    lodged               lodg
-    lodgers              lodger
-    lodges               lodg
-    lodging              lodg
-    lodgings             lodg
-    lodovico             lodovico
-    lodowick             lodowick
-    lofty                lofti
-    log                  log
-    logger               logger
-    loggerhead           loggerhead
-    loggerheads          loggerhead
-    loggets              logget
-    logic                logic
-    logs                 log
-    loins                loin
-    loiter               loiter
-    loiterer             loiter
-    loiterers            loiter
-    loitering            loiter
-    lolling              loll
-    lolls                loll
-    lombardy             lombardi
-    london               london
-    londoners            london
-    lone                 lone
-    loneliness           loneli
-    lonely               lone
-    long                 long
-    longaville           longavil
-    longboat             longboat
-    longed               long
-    longer               longer
-    longest              longest
-    longeth              longeth
-    longing              long
-    longings             long
-    longly               longli
-    longs                long
-    longtail             longtail
-    loo                  loo
-    loof                 loof
-    look                 look
-    looked               look
-    looker               looker
-    lookers              looker
-    lookest              lookest
-    looking              look
-    looks                look
-    loon                 loon
-    loop                 loop
-    loos                 loo
-    loose                loos
-    loosed               loos
-    loosely              loos
-    loosen               loosen
-    loosing              loos
-    lop                  lop
-    lopp                 lopp
-    loquitur             loquitur
-    lord                 lord
-    lorded               lord
-    lording              lord
-    lordings             lord
-    lordliness           lordli
-    lordly               lordli
-    lords                lord
-    lordship             lordship
-    lordships            lordship
-    lorenzo              lorenzo
-    lorn                 lorn
-    lorraine             lorrain
-    lorship              lorship
-    los                  lo
-    lose                 lose
-    loser                loser
-    losers               loser
-    loses                lose
-    losest               losest
-    loseth               loseth
-    losing               lose
-    loss                 loss
-    losses               loss
-    lost                 lost
-    lot                  lot
-    lots                 lot
-    lott                 lott
-    lottery              lotteri
-    loud                 loud
-    louder               louder
-    loudly               loudli
-    lour                 lour
-    loureth              loureth
-    louring              lour
-    louse                lous
-    louses               lous
-    lousy                lousi
-    lout                 lout
-    louted               lout
-    louts                lout
-    louvre               louvr
-    lov                  lov
-    love                 love
-    loved                love
-    lovedst              lovedst
-    lovel                lovel
-    lovelier             loveli
-    loveliness           loveli
-    lovell               lovel
-    lovely               love
-    lover                lover
-    lovered              lover
-    lovers               lover
-    loves                love
-    lovest               lovest
-    loveth               loveth
-    loving               love
-    lovingly             lovingli
-    low                  low
-    lowe                 low
-    lower                lower
-    lowest               lowest
-    lowing               low
-    lowliness            lowli
-    lowly                lowli
-    lown                 lown
-    lowness              low
-    loyal                loyal
-    loyally              loyal
-    loyalties            loyalti
-    loyalty              loyalti
-    lozel                lozel
-    lt                   lt
-    lubber               lubber
-    lubberly             lubberli
-    luc                  luc
-    luccicos             luccico
-    luce                 luce
-    lucentio             lucentio
-    luces                luce
-    lucetta              lucetta
-    luciana              luciana
-    lucianus             lucianu
-    lucifer              lucif
-    lucifier             lucifi
-    lucilius             luciliu
-    lucina               lucina
-    lucio                lucio
-    lucius               luciu
-    luck                 luck
-    luckier              luckier
-    luckiest             luckiest
-    luckily              luckili
-    luckless             luckless
-    lucky                lucki
-    lucre                lucr
-    lucrece              lucrec
-    lucretia             lucretia
-    lucullius            luculliu
-    lucullus             lucullu
-    lucy                 luci
-    lud                  lud
-    ludlow               ludlow
-    lug                  lug
-    lugg                 lugg
-    luggage              luggag
-    luke                 luke
-    lukewarm             lukewarm
-    lull                 lull
-    lulla                lulla
-    lullaby              lullabi
-    lulls                lull
-    lumbert              lumbert
-    lump                 lump
-    lumpish              lumpish
-    luna                 luna
-    lunacies             lunaci
-    lunacy               lunaci
-    lunatic              lunat
-    lunatics             lunat
-    lunes                lune
-    lungs                lung
-    lupercal             luperc
-    lurch                lurch
-    lure                 lure
-    lurk                 lurk
-    lurketh              lurketh
-    lurking              lurk
-    lurks                lurk
-    luscious             lusciou
-    lush                 lush
-    lust                 lust
-    lusted               lust
-    luster               luster
-    lustful              lust
-    lustier              lustier
-    lustiest             lustiest
-    lustig               lustig
-    lustihood            lustihood
-    lustily              lustili
-    lustre               lustr
-    lustrous             lustrou
-    lusts                lust
-    lusty                lusti
-    lute                 lute
-    lutes                lute
-    lutestring           lutestr
-    lutheran             lutheran
-    luxurious            luxuri
-    luxuriously          luxuri
-    luxury               luxuri
-    ly                   ly
-    lycaonia             lycaonia
-    lycurguses           lycurgus
-    lydia                lydia
-    lye                  lye
-    lyen                 lyen
-    lying                ly
-    lym                  lym
-    lymoges              lymog
-    lynn                 lynn
-    lysander             lysand
-    m                    m
-    ma                   ma
-    maan                 maan
-    mab                  mab
-    macbeth              macbeth
-    maccabaeus           maccabaeu
-    macdonwald           macdonwald
-    macduff              macduff
-    mace                 mace
-    macedon              macedon
-    maces                mace
-    machiavel            machiavel
-    machination          machin
-    machinations         machin
-    machine              machin
-    mack                 mack
-    macmorris            macmorri
-    maculate             macul
-    maculation           macul
-    mad                  mad
-    madam                madam
-    madame               madam
-    madams               madam
-    madcap               madcap
-    madded               mad
-    madding              mad
-    made                 made
-    madeira              madeira
-    madly                madli
-    madman               madman
-    madmen               madmen
-    madness              mad
-    madonna              madonna
-    madrigals            madrig
-    mads                 mad
-    maecenas             maecena
-    maggot               maggot
-    maggots              maggot
-    magic                magic
-    magical              magic
-    magician             magician
-    magistrate           magistr
-    magistrates          magistr
-    magnanimity          magnanim
-    magnanimous          magnanim
-    magni                magni
-    magnifi              magnifi
-    magnificence         magnific
-    magnificent          magnific
-    magnifico            magnifico
-    magnificoes          magnifico
-    magnus               magnu
-    mahomet              mahomet
-    mahu                 mahu
-    maid                 maid
-    maiden               maiden
-    maidenhead           maidenhead
-    maidenheads          maidenhead
-    maidenhood           maidenhood
-    maidenhoods          maidenhood
-    maidenliest          maidenliest
-    maidenly             maidenli
-    maidens              maiden
-    maidhood             maidhood
-    maids                maid
-    mail                 mail
-    mailed               mail
-    mails                mail
-    maim                 maim
-    maimed               maim
-    maims                maim
-    main                 main
-    maincourse           maincours
-    maine                main
-    mainly               mainli
-    mainmast             mainmast
-    mains                main
-    maintain             maintain
-    maintained           maintain
-    maintains            maintain
-    maintenance          mainten
-    mais                 mai
-    maison               maison
-    majestas             majesta
-    majestee             majeste
-    majestic             majest
-    majestical           majest
-    majestically         majest
-    majesties            majesti
-    majesty              majesti
-    major                major
-    majority             major
-    mak                  mak
-    make                 make
-    makeless             makeless
-    maker                maker
-    makers               maker
-    makes                make
-    makest               makest
-    maketh               maketh
-    making               make
-    makings              make
-    mal                  mal
-    mala                 mala
-    maladies             maladi
-    malady               maladi
-    malapert             malapert
-    malcolm              malcolm
-    malcontent           malcont
-    malcontents          malcont
-    male                 male
-    maledictions         maledict
-    malefactions         malefact
-    malefactor           malefactor
-    malefactors          malefactor
-    males                male
-    malevolence          malevol
-    malevolent           malevol
-    malhecho             malhecho
-    malice               malic
-    malicious            malici
-    maliciously          malici
-    malign               malign
-    malignancy           malign
-    malignant            malign
-    malignantly          malignantli
-    malkin               malkin
-    mall                 mall
-    mallard              mallard
-    mallet               mallet
-    mallows              mallow
-    malmsey              malmsei
-    malt                 malt
-    maltworms            maltworm
-    malvolio             malvolio
-    mamillius            mamilliu
-    mammering            mammer
-    mammet               mammet
-    mammets              mammet
-    mammock              mammock
-    man                  man
-    manacle              manacl
-    manacles             manacl
-    manage               manag
-    managed              manag
-    manager              manag
-    managing             manag
-    manakin              manakin
-    manchus              manchu
-    mandate              mandat
-    mandragora           mandragora
-    mandrake             mandrak
-    mandrakes            mandrak
-    mane                 mane
-    manent               manent
-    manes                mane
-    manet                manet
-    manfully             manfulli
-    mangle               mangl
-    mangled              mangl
-    mangles              mangl
-    mangling             mangl
-    mangy                mangi
-    manhood              manhood
-    manhoods             manhood
-    manifest             manifest
-    manifested           manifest
-    manifests            manifest
-    manifold             manifold
-    manifoldly           manifoldli
-    manka                manka
-    mankind              mankind
-    manlike              manlik
-    manly                manli
-    mann                 mann
-    manna                manna
-    manner               manner
-    mannerly             mannerli
-    manners              manner
-    manningtree          manningtre
-    mannish              mannish
-    manor                manor
-    manors               manor
-    mans                 man
-    mansion              mansion
-    mansionry            mansionri
-    mansions             mansion
-    manslaughter         manslaught
-    mantle               mantl
-    mantled              mantl
-    mantles              mantl
-    mantua               mantua
-    mantuan              mantuan
-    manual               manual
-    manure               manur
-    manured              manur
-    manus                manu
-    many                 mani
-    map                  map
-    mapp                 mapp
-    maps                 map
-    mar                  mar
-    marble               marbl
-    marbled              marbl
-    marcade              marcad
-    marcellus            marcellu
-    march                march
-    marches              march
-    marcheth             marcheth
-    marching             march
-    marchioness          marchio
-    marchpane            marchpan
-    marcians             marcian
-    marcius              marciu
-    marcus               marcu
-    mardian              mardian
-    mare                 mare
-    mares                mare
-    marg                 marg
-    margarelon           margarelon
-    margaret             margaret
-    marge                marg
-    margent              margent
-    margery              margeri
-    maria                maria
-    marian               marian
-    mariana              mariana
-    maries               mari
-    marigold             marigold
-    mariner              marin
-    mariners             marin
-    maritime             maritim
-    marjoram             marjoram
-    mark                 mark
-    marked               mark
-    market               market
-    marketable           market
-    marketplace          marketplac
-    markets              market
-    marking              mark
-    markman              markman
-    marks                mark
-    marl                 marl
-    marle                marl
-    marmoset             marmoset
-    marquess             marquess
-    marquis              marqui
-    marr                 marr
-    marriage             marriag
-    marriages            marriag
-    married              marri
-    marries              marri
-    marring              mar
-    marrow               marrow
-    marrowless           marrowless
-    marrows              marrow
-    marry                marri
-    marrying             marri
-    mars                 mar
-    marseilles           marseil
-    marsh                marsh
-    marshal              marshal
-    marshalsea           marshalsea
-    marshalship          marshalship
-    mart                 mart
-    marted               mart
-    martem               martem
-    martext              martext
-    martial              martial
-    martin               martin
-    martino              martino
-    martius              martiu
-    martlemas            martlema
-    martlet              martlet
-    marts                mart
-    martyr               martyr
-    martyrs              martyr
-    marullus             marullu
-    marv                 marv
-    marvel               marvel
-    marvell              marvel
-    marvellous           marvel
-    marvellously         marvel
-    marvels              marvel
-    mary                 mari
-    mas                  ma
-    masculine            masculin
-    masham               masham
-    mask                 mask
-    masked               mask
-    masker               masker
-    maskers              masker
-    masking              mask
-    masks                mask
-    mason                mason
-    masonry              masonri
-    masons               mason
-    masque               masqu
-    masquers             masquer
-    masques              masqu
-    masquing             masqu
-    mass                 mass
-    massacre             massacr
-    massacres            massacr
-    masses               mass
-    massy                massi
-    mast                 mast
-    mastcr               mastcr
-    master               master
-    masterdom            masterdom
-    masterest            masterest
-    masterless           masterless
-    masterly             masterli
-    masterpiece          masterpiec
-    masters              master
-    mastership           mastership
-    mastic               mastic
-    mastiff              mastiff
-    mastiffs             mastiff
-    masts                mast
-    match                match
-    matches              match
-    matcheth             matcheth
-    matching             match
-    matchless            matchless
-    mate                 mate
-    mated                mate
-    mater                mater
-    material             materi
-    mates                mate
-    mathematics          mathemat
-    matin                matin
-    matron               matron
-    matrons              matron
-    matter               matter
-    matters              matter
-    matthew              matthew
-    mattock              mattock
-    mattress             mattress
-    mature               matur
-    maturity             matur
-    maud                 maud
-    maudlin              maudlin
-    maugre               maugr
-    maul                 maul
-    maund                maund
-    mauri                mauri
-    mauritania           mauritania
-    mauvais              mauvai
-    maw                  maw
-    maws                 maw
-    maxim                maxim
-    may                  mai
-    mayday               maydai
-    mayest               mayest
-    mayor                mayor
-    maypole              maypol
-    mayst                mayst
-    maz                  maz
-    maze                 maze
-    mazed                maze
-    mazes                maze
-    mazzard              mazzard
-    me                   me
-    meacock              meacock
-    mead                 mead
-    meadow               meadow
-    meadows              meadow
-    meads                mead
-    meagre               meagr
-    meal                 meal
-    meals                meal
-    mealy                meali
-    mean                 mean
-    meanders             meander
-    meaner               meaner
-    meanest              meanest
-    meaneth              meaneth
-    meaning              mean
-    meanings             mean
-    meanly               meanli
-    means                mean
-    meant                meant
-    meantime             meantim
-    meanwhile            meanwhil
-    measles              measl
-    measur               measur
-    measurable           measur
-    measure              measur
-    measured             measur
-    measureless          measureless
-    measures             measur
-    measuring            measur
-    meat                 meat
-    meats                meat
-    mechanic             mechan
-    mechanical           mechan
-    mechanicals          mechan
-    mechanics            mechan
-    mechante             mechant
-    med                  med
-    medal                medal
-    meddle               meddl
-    meddler              meddler
-    meddling             meddl
-    mede                 mede
-    medea                medea
-    media                media
-    mediation            mediat
-    mediators            mediat
-    medice               medic
-    medicinal            medicin
-    medicine             medicin
-    medicines            medicin
-    meditate             medit
-    meditates            medit
-    meditating           medit
-    meditation           medit
-    meditations          medit
-    mediterranean        mediterranean
-    mediterraneum        mediterraneum
-    medlar               medlar
-    medlars              medlar
-    meed                 meed
-    meeds                meed
-    meek                 meek
-    meekly               meekli
-    meekness             meek
-    meet                 meet
-    meeter               meeter
-    meetest              meetest
-    meeting              meet
-    meetings             meet
-    meetly               meetli
-    meetness             meet
-    meets                meet
-    meg                  meg
-    mehercle             mehercl
-    meilleur             meilleur
-    meiny                meini
-    meisen               meisen
-    melancholies         melancholi
-    melancholy           melancholi
-    melford              melford
-    mell                 mell
-    mellifluous          melliflu
-    mellow               mellow
-    mellowing            mellow
-    melodious            melodi
-    melody               melodi
-    melt                 melt
-    melted               melt
-    melteth              melteth
-    melting              melt
-    melts                melt
-    melun                melun
-    member               member
-    members              member
-    memento              memento
-    memorable            memor
-    memorandums          memorandum
-    memorial             memori
-    memorials            memori
-    memories             memori
-    memoriz              memoriz
-    memorize             memor
-    memory               memori
-    memphis              memphi
-    men                  men
-    menac                menac
-    menace               menac
-    menaces              menac
-    menaphon             menaphon
-    menas                mena
-    mend                 mend
-    mended               mend
-    mender               mender
-    mending              mend
-    mends                mend
-    menecrates           menecr
-    menelaus             menelau
-    menenius             meneniu
-    mental               mental
-    menteith             menteith
-    mention              mention
-    mentis               menti
-    menton               menton
-    mephostophilus       mephostophilu
-    mer                  mer
-    mercatante           mercatant
-    mercatio             mercatio
-    mercenaries          mercenari
-    mercenary            mercenari
-    mercer               mercer
-    merchandise          merchandis
-    merchandized         merchand
-    merchant             merchant
-    merchants            merchant
-    mercies              merci
-    merciful             merci
-    mercifully           mercifulli
-    merciless            merciless
-    mercurial            mercuri
-    mercuries            mercuri
-    mercury              mercuri
-    mercutio             mercutio
-    mercy                merci
-    mere                 mere
-    mered                mere
-    merely               mere
-    merest               merest
-    meridian             meridian
-    merit                merit
-    merited              merit
-    meritorious          meritori
-    merits               merit
-    merlin               merlin
-    mermaid              mermaid
-    mermaids             mermaid
-    merops               merop
-    merrier              merrier
-    merriest             merriest
-    merrily              merrili
-    merriman             merriman
-    merriment            merriment
-    merriments           merriment
-    merriness            merri
-    merry                merri
-    mervailous           mervail
-    mes                  me
-    mesh                 mesh
-    meshes               mesh
-    mesopotamia          mesopotamia
-    mess                 mess
-    message              messag
-    messages             messag
-    messala              messala
-    messaline            messalin
-    messenger            messeng
-    messengers           messeng
-    messes               mess
-    messina              messina
-    met                  met
-    metal                metal
-    metals               metal
-    metamorphis          metamorphi
-    metamorphoses        metamorphos
-    metaphor             metaphor
-    metaphysical         metaphys
-    metaphysics          metaphys
-    mete                 mete
-    metellus             metellu
-    meteor               meteor
-    meteors              meteor
-    meteyard             meteyard
-    metheglin            metheglin
-    metheglins           metheglin
-    methink              methink
-    methinks             methink
-    method               method
-    methods              method
-    methought            methought
-    methoughts           methought
-    metre                metr
-    metres               metr
-    metropolis           metropoli
-    mette                mett
-    mettle               mettl
-    mettled              mettl
-    meus                 meu
-    mew                  mew
-    mewed                mew
-    mewling              mewl
-    mexico               mexico
-    mi                   mi
-    mice                 mice
-    michael              michael
-    michaelmas           michaelma
-    micher               micher
-    miching              mich
-    mickle               mickl
-    microcosm            microcosm
-    mid                  mid
-    midas                mida
-    middest              middest
-    middle               middl
-    middleham            middleham
-    midnight             midnight
-    midriff              midriff
-    midst                midst
-    midsummer            midsumm
-    midway               midwai
-    midwife              midwif
-    midwives             midwiv
-    mienne               mienn
-    might                might
-    mightful             might
-    mightier             mightier
-    mightiest            mightiest
-    mightily             mightili
-    mightiness           mighti
-    mightst              mightst
-    mighty               mighti
-    milan                milan
-    milch                milch
-    mild                 mild
-    milder               milder
-    mildest              mildest
-    mildew               mildew
-    mildews              mildew
-    mildly               mildli
-    mildness             mild
-    mile                 mile
-    miles                mile
-    milford              milford
-    militarist           militarist
-    military             militari
-    milk                 milk
-    milking              milk
-    milkmaid             milkmaid
-    milks                milk
-    milksops             milksop
-    milky                milki
-    mill                 mill
-    mille                mill
-    miller               miller
-    milliner             millin
-    million              million
-    millioned            million
-    millions             million
-    mills                mill
-    millstones           millston
-    milo                 milo
-    mimic                mimic
-    minc                 minc
-    mince                minc
-    minces               minc
-    mincing              minc
-    mind                 mind
-    minded               mind
-    minding              mind
-    mindless             mindless
-    minds                mind
-    mine                 mine
-    mineral              miner
-    minerals             miner
-    minerva              minerva
-    mines                mine
-    mingle               mingl
-    mingled              mingl
-    mingling             mingl
-    minikin              minikin
-    minim                minim
-    minime               minim
-    minimo               minimo
-    minimus              minimu
-    mining               mine
-    minion               minion
-    minions              minion
-    minist               minist
-    minister             minist
-    ministers            minist
-    ministration         ministr
-    minnow               minnow
-    minnows              minnow
-    minola               minola
-    minority             minor
-    minos                mino
-    minotaurs            minotaur
-    minstrel             minstrel
-    minstrels            minstrel
-    minstrelsy           minstrelsi
-    mint                 mint
-    mints                mint
-    minute               minut
-    minutely             minut
-    minutes              minut
-    minx                 minx
-    mio                  mio
-    mir                  mir
-    mirable              mirabl
-    miracle              miracl
-    miracles             miracl
-    miraculous           miracul
-    miranda              miranda
-    mire                 mire
-    mirror               mirror
-    mirrors              mirror
-    mirth                mirth
-    mirthful             mirth
-    miry                 miri
-    mis                  mi
-    misadventur          misadventur
-    misadventure         misadventur
-    misanthropos         misanthropo
-    misapplied           misappli
-    misbecame            misbecam
-    misbecom             misbecom
-    misbecome            misbecom
-    misbegot             misbegot
-    misbegotten          misbegotten
-    misbeliever          misbeliev
-    misbelieving         misbeliev
-    misbhav              misbhav
-    miscall              miscal
-    miscalled            miscal
-    miscarried           miscarri
-    miscarries           miscarri
-    miscarry             miscarri
-    miscarrying          miscarri
-    mischance            mischanc
-    mischances           mischanc
-    mischief             mischief
-    mischiefs            mischief
-    mischievous          mischiev
-    misconceived         misconceiv
-    misconst             misconst
-    misconster           misconst
-    misconstruction      misconstruct
-    misconstrued         misconstru
-    misconstrues         misconstru
-    miscreant            miscreant
-    miscreate            miscreat
-    misdeed              misde
-    misdeeds             misde
-    misdemean            misdemean
-    misdemeanours        misdemeanour
-    misdoubt             misdoubt
-    misdoubteth          misdoubteth
-    misdoubts            misdoubt
-    misenum              misenum
-    miser                miser
-    miserable            miser
-    miserably            miser
-    misericorde          misericord
-    miseries             miseri
-    misers               miser
-    misery               miseri
-    misfortune           misfortun
-    misfortunes          misfortun
-    misgive              misgiv
-    misgives             misgiv
-    misgiving            misgiv
-    misgoverned          misgovern
-    misgovernment        misgovern
-    misgraffed           misgraf
-    misguide             misguid
-    mishap               mishap
-    mishaps              mishap
-    misheard             misheard
-    misinterpret         misinterpret
-    mislead              mislead
-    misleader            mislead
-    misleaders           mislead
-    misleading           mislead
-    misled               misl
-    mislike              mislik
-    misord               misord
-    misplac              misplac
-    misplaced            misplac
-    misplaces            misplac
-    mispris              mispri
-    misprised            mispris
-    misprision           mispris
-    misprizing           mispriz
-    misproud             misproud
-    misquote             misquot
-    misreport            misreport
-    miss                 miss
-    missed               miss
-    misses               miss
-    misshap              misshap
-    misshapen            misshapen
-    missheathed          missheath
-    missing              miss
-    missingly            missingli
-    missions             mission
-    missive              missiv
-    missives             missiv
-    misspoke             misspok
-    mist                 mist
-    mista                mista
-    mistak               mistak
-    mistake              mistak
-    mistaken             mistaken
-    mistakes             mistak
-    mistaketh            mistaketh
-    mistaking            mistak
-    mistakings           mistak
-    mistemp              mistemp
-    mistempered          mistemp
-    misterm              misterm
-    mistful              mist
-    misthink             misthink
-    misthought           misthought
-    mistletoe            mistleto
-    mistook              mistook
-    mistreadings         mistread
-    mistress             mistress
-    mistresses           mistress
-    mistresss            mistresss
-    mistriship           mistriship
-    mistrust             mistrust
-    mistrusted           mistrust
-    mistrustful          mistrust
-    mistrusting          mistrust
-    mists                mist
-    misty                misti
-    misus                misu
-    misuse               misus
-    misused              misus
-    misuses              misus
-    mites                mite
-    mithridates          mithrid
-    mitigate             mitig
-    mitigation           mitig
-    mix                  mix
-    mixed                mix
-    mixture              mixtur
-    mixtures             mixtur
-    mm                   mm
-    mnd                  mnd
-    moan                 moan
-    moans                moan
-    moat                 moat
-    moated               moat
-    mobled               mobl
-    mock                 mock
-    mockable             mockabl
-    mocker               mocker
-    mockeries            mockeri
-    mockers              mocker
-    mockery              mockeri
-    mocking              mock
-    mocks                mock
-    mockvater            mockvat
-    mockwater            mockwat
-    model                model
-    modena               modena
-    moderate             moder
-    moderately           moder
-    moderation           moder
-    modern               modern
-    modest               modest
-    modesties            modesti
-    modestly             modestli
-    modesty              modesti
-    modicums             modicum
-    modo                 modo
-    module               modul
-    moe                  moe
-    moi                  moi
-    moiety               moieti
-    moist                moist
-    moisten              moisten
-    moisture             moistur
-    moldwarp             moldwarp
-    mole                 mole
-    molehill             molehil
-    moles                mole
-    molest               molest
-    molestation          molest
-    mollification        mollif
-    mollis               molli
-    molten               molten
-    molto                molto
-    mome                 mome
-    moment               moment
-    momentary            momentari
-    moming               mome
-    mon                  mon
-    monachum             monachum
-    monarch              monarch
-    monarchies           monarchi
-    monarchize           monarch
-    monarcho             monarcho
-    monarchs             monarch
-    monarchy             monarchi
-    monast               monast
-    monastery            monasteri
-    monastic             monast
-    monday               mondai
-    monde                mond
-    money                monei
-    moneys               monei
-    mong                 mong
-    monger               monger
-    mongers              monger
-    monging              mong
-    mongrel              mongrel
-    mongrels             mongrel
-    mongst               mongst
-    monk                 monk
-    monkey               monkei
-    monkeys              monkei
-    monks                monk
-    monmouth             monmouth
-    monopoly             monopoli
-    mons                 mon
-    monsieur             monsieur
-    monsieurs            monsieur
-    monster              monster
-    monsters             monster
-    monstrous            monstrou
-    monstrously          monstrous
-    monstrousness        monstrous
-    monstruosity         monstruos
-    montacute            montacut
-    montage              montag
-    montague             montagu
-    montagues            montagu
-    montano              montano
-    montant              montant
-    montez               montez
-    montferrat           montferrat
-    montgomery           montgomeri
-    month                month
-    monthly              monthli
-    months               month
-    montjoy              montjoi
-    monument             monument
-    monumental           monument
-    monuments            monument
-    mood                 mood
-    moods                mood
-    moody                moodi
-    moon                 moon
-    moonbeams            moonbeam
-    moonish              moonish
-    moonlight            moonlight
-    moons                moon
-    moonshine            moonshin
-    moonshines           moonshin
-    moor                 moor
-    moorfields           moorfield
-    moors                moor
-    moorship             moorship
-    mop                  mop
-    mope                 mope
-    moping               mope
-    mopping              mop
-    mopsa                mopsa
-    moral                moral
-    moraler              moral
-    morality             moral
-    moralize             moral
-    mordake              mordak
-    more                 more
-    moreover             moreov
-    mores                more
-    morgan               morgan
-    mori                 mori
-    morisco              morisco
-    morn                 morn
-    morning              morn
-    mornings             morn
-    morocco              morocco
-    morris               morri
-    morrow               morrow
-    morrows              morrow
-    morsel               morsel
-    morsels              morsel
-    mort                 mort
-    mortal               mortal
-    mortality            mortal
-    mortally             mortal
-    mortals              mortal
-    mortar               mortar
-    mortgaged            mortgag
-    mortified            mortifi
-    mortifying           mortifi
-    mortimer             mortim
-    mortimers            mortim
-    mortis               morti
-    mortise              mortis
-    morton               morton
-    mose                 mose
-    moss                 moss
-    mossgrown            mossgrown
-    most                 most
-    mote                 mote
-    moth                 moth
-    mother               mother
-    mothers              mother
-    moths                moth
-    motion               motion
-    motionless           motionless
-    motions              motion
-    motive               motiv
-    motives              motiv
-    motley               motlei
-    mots                 mot
-    mought               mought
-    mould                mould
-    moulded              mould
-    mouldeth             mouldeth
-    moulds               mould
-    mouldy               mouldi
-    moult                moult
-    moulten              moulten
-    mounch               mounch
-    mounseur             mounseur
-    mounsieur            mounsieur
-    mount                mount
-    mountain             mountain
-    mountaineer          mountain
-    mountaineers         mountain
-    mountainous          mountain
-    mountains            mountain
-    mountant             mountant
-    mountanto            mountanto
-    mountebank           mountebank
-    mountebanks          mountebank
-    mounted              mount
-    mounteth             mounteth
-    mounting             mount
-    mounts               mount
-    mourn                mourn
-    mourned              mourn
-    mourner              mourner
-    mourners             mourner
-    mournful             mourn
-    mournfully           mournfulli
-    mourning             mourn
-    mourningly           mourningli
-    mournings            mourn
-    mourns               mourn
-    mous                 mou
-    mouse                mous
-    mousetrap            mousetrap
-    mousing              mous
-    mouth                mouth
-    mouthed              mouth
-    mouths               mouth
-    mov                  mov
-    movables             movabl
-    move                 move
-    moveable             moveabl
-    moveables            moveabl
-    moved                move
-    mover                mover
-    movers               mover
-    moves                move
-    moveth               moveth
-    moving               move
-    movingly             movingli
-    movousus             movousu
-    mow                  mow
-    mowbray              mowbrai
-    mower                mower
-    mowing               mow
-    mows                 mow
-    moy                  moi
-    moys                 moi
-    moyses               moys
-    mrs                  mr
-    much                 much
-    muck                 muck
-    mud                  mud
-    mudded               mud
-    muddied              muddi
-    muddy                muddi
-    muffins              muffin
-    muffl                muffl
-    muffle               muffl
-    muffled              muffl
-    muffler              muffler
-    muffling             muffl
-    mugger               mugger
-    mugs                 mug
-    mulberries           mulberri
-    mulberry             mulberri
-    mule                 mule
-    mules                mule
-    muleteers            mulet
-    mulier               mulier
-    mulieres             mulier
-    muliteus             muliteu
-    mull                 mull
-    mulmutius            mulmutiu
-    multiplied           multipli
-    multiply             multipli
-    multiplying          multipli
-    multipotent          multipot
-    multitude            multitud
-    multitudes           multitud
-    multitudinous        multitudin
-    mum                  mum
-    mumble               mumbl
-    mumbling             mumbl
-    mummers              mummer
-    mummy                mummi
-    mun                  mun
-    munch                munch
-    muniments            muniment
-    munition             munit
-    murd                 murd
-    murder               murder
-    murdered             murder
-    murderer             murder
-    murderers            murder
-    murdering            murder
-    murderous            murder
-    murders              murder
-    mure                 mure
-    murk                 murk
-    murkiest             murkiest
-    murky                murki
-    murmur               murmur
-    murmurers            murmur
-    murmuring            murmur
-    murrain              murrain
-    murray               murrai
-    murrion              murrion
-    murther              murther
-    murtherer            murther
-    murtherers           murther
-    murthering           murther
-    murtherous           murther
-    murthers             murther
-    mus                  mu
-    muscadel             muscadel
-    muscovites           muscovit
-    muscovits            muscovit
-    muscovy              muscovi
-    muse                 muse
-    muses                muse
-    mush                 mush
-    mushrooms            mushroom
-    music                music
-    musical              music
-    musician             musician
-    musicians            musician
-    musics               music
-    musing               muse
-    musings              muse
-    musk                 musk
-    musket               musket
-    muskets              musket
-    muskos               musko
-    muss                 muss
-    mussel               mussel
-    mussels              mussel
-    must                 must
-    mustachio            mustachio
-    mustard              mustard
-    mustardseed          mustardse
-    muster               muster
-    mustering            muster
-    musters              muster
-    musty                musti
-    mutability           mutabl
-    mutable              mutabl
-    mutation             mutat
-    mutations            mutat
-    mute                 mute
-    mutes                mute
-    mutest               mutest
-    mutine               mutin
-    mutineer             mutin
-    mutineers            mutin
-    mutines              mutin
-    mutinies             mutini
-    mutinous             mutin
-    mutiny               mutini
-    mutius               mutiu
-    mutter               mutter
-    muttered             mutter
-    mutton               mutton
-    muttons              mutton
-    mutual               mutual
-    mutualities          mutual
-    mutually             mutual
-    muzzl                muzzl
-    muzzle               muzzl
-    muzzled              muzzl
-    mv                   mv
-    mww                  mww
-    my                   my
-    mynheers             mynheer
-    myrmidon             myrmidon
-    myrmidons            myrmidon
-    myrtle               myrtl
-    myself               myself
-    myst                 myst
-    mysteries            mysteri
-    mystery              mysteri
-    n                    n
-    nag                  nag
-    nage                 nage
-    nags                 nag
-    naiads               naiad
-    nail                 nail
-    nails                nail
-    nak                  nak
-    naked                nake
-    nakedness            naked
-    nal                  nal
-    nam                  nam
-    name                 name
-    named                name
-    nameless             nameless
-    namely               name
-    names                name
-    namest               namest
-    naming               name
-    nan                  nan
-    nance                nanc
-    nap                  nap
-    nape                 nape
-    napes                nape
-    napkin               napkin
-    napkins              napkin
-    naples               napl
-    napless              napless
-    napping              nap
-    naps                 nap
-    narbon               narbon
-    narcissus            narcissu
-    narines              narin
-    narrow               narrow
-    narrowly             narrowli
-    naso                 naso
-    nasty                nasti
-    nathaniel            nathaniel
-    natifs               natif
-    nation               nation
-    nations              nation
-    native               nativ
-    nativity             nativ
-    natur                natur
-    natural              natur
-    naturalize           natur
-    naturally            natur
-    nature               natur
-    natured              natur
-    natures              natur
-    natus                natu
-    naught               naught
-    naughtily            naughtili
-    naughty              naughti
-    navarre              navarr
-    nave                 nave
-    navel                navel
-    navigation           navig
-    navy                 navi
-    nay                  nai
-    nayward              nayward
-    nayword              nayword
-    nazarite             nazarit
-    ne                   ne
-    neaf                 neaf
-    neamnoins            neamnoin
-    neanmoins            neanmoin
-    neapolitan           neapolitan
-    neapolitans          neapolitan
-    near                 near
-    nearer               nearer
-    nearest              nearest
-    nearly               nearli
-    nearness             near
-    neat                 neat
-    neatly               neatli
-    neb                  neb
-    nebour               nebour
-    nebuchadnezzar       nebuchadnezzar
-    nec                  nec
-    necessaries          necessari
-    necessarily          necessarili
-    necessary            necessari
-    necessitied          necess
-    necessities          necess
-    necessity            necess
-    neck                 neck
-    necklace             necklac
-    necks                neck
-    nectar               nectar
-    ned                  ned
-    nedar                nedar
-    need                 need
-    needed               need
-    needer               needer
-    needful              need
-    needfull             needful
-    needing              need
-    needle               needl
-    needles              needl
-    needless             needless
-    needly               needli
-    needs                need
-    needy                needi
-    neer                 neer
-    neeze                neez
-    nefas                nefa
-    negation             negat
-    negative             neg
-    negatives            neg
-    neglect              neglect
-    neglected            neglect
-    neglecting           neglect
-    neglectingly         neglectingli
-    neglection           neglect
-    negligence           neglig
-    negligent            neglig
-    negotiate            negoti
-    negotiations         negoti
-    negro                negro
-    neigh                neigh
-    neighbors            neighbor
-    neighbour            neighbour
-    neighbourhood        neighbourhood
-    neighbouring         neighbour
-    neighbourly          neighbourli
-    neighbours           neighbour
-    neighing             neigh
-    neighs               neigh
-    neither              neither
-    nell                 nell
-    nemean               nemean
-    nemesis              nemesi
-    neoptolemus          neoptolemu
-    nephew               nephew
-    nephews              nephew
-    neptune              neptun
-    ner                  ner
-    nereides             nereid
-    nerissa              nerissa
-    nero                 nero
-    neroes               nero
-    ners                 ner
-    nerve                nerv
-    nerves               nerv
-    nervii               nervii
-    nervy                nervi
-    nessus               nessu
-    nest                 nest
-    nestor               nestor
-    nests                nest
-    net                  net
-    nether               nether
-    netherlands          netherland
-    nets                 net
-    nettle               nettl
-    nettled              nettl
-    nettles              nettl
-    neuter               neuter
-    neutral              neutral
-    nev                  nev
-    never                never
-    nevil                nevil
-    nevils               nevil
-    new                  new
-    newborn              newborn
-    newer                newer
-    newest               newest
-    newgate              newgat
-    newly                newli
-    newness              new
-    news                 new
-    newsmongers          newsmong
-    newt                 newt
-    newts                newt
-    next                 next
-    nibbling             nibbl
-    nicanor              nicanor
-    nice                 nice
-    nicely               nice
-    niceness             nice
-    nicer                nicer
-    nicety               niceti
-    nicholas             nichola
-    nick                 nick
-    nickname             nicknam
-    nicks                nick
-    niece                niec
-    nieces               niec
-    niggard              niggard
-    niggarding           niggard
-    niggardly            niggardli
-    nigh                 nigh
-    night                night
-    nightcap             nightcap
-    nightcaps            nightcap
-    nighted              night
-    nightgown            nightgown
-    nightingale          nightingal
-    nightingales         nightingal
-    nightly              nightli
-    nightmare            nightmar
-    nights               night
-    nightwork            nightwork
-    nihil                nihil
-    nile                 nile
-    nill                 nill
-    nilus                nilu
-    nimble               nimbl
-    nimbleness           nimbl
-    nimbler              nimbler
-    nimbly               nimbl
-    nine                 nine
-    nineteen             nineteen
-    ning                 ning
-    ningly               ningli
-    ninny                ninni
-    ninth                ninth
-    ninus                ninu
-    niobe                niob
-    niobes               niob
-    nip                  nip
-    nipp                 nipp
-    nipping              nip
-    nipple               nippl
-    nips                 nip
-    nit                  nit
-    nly                  nly
-    nnight               nnight
-    nnights              nnight
-    no                   no
-    noah                 noah
-    nob                  nob
-    nobility             nobil
-    nobis                nobi
-    noble                nobl
-    nobleman             nobleman
-    noblemen             noblemen
-    nobleness            nobl
-    nobler               nobler
-    nobles               nobl
-    noblesse             nobless
-    noblest              noblest
-    nobly                nobli
-    nobody               nobodi
-    noces                noce
-    nod                  nod
-    nodded               nod
-    nodding              nod
-    noddle               noddl
-    noddles              noddl
-    noddy                noddi
-    nods                 nod
-    noes                 noe
-    nointed              noint
-    nois                 noi
-    noise                nois
-    noiseless            noiseless
-    noisemaker           noisemak
-    noises               nois
-    noisome              noisom
-    nole                 nole
-    nominate             nomin
-    nominated            nomin
-    nomination           nomin
-    nominativo           nominativo
-    non                  non
-    nonage               nonag
-    nonce                nonc
-    none                 none
-    nonino               nonino
-    nonny                nonni
-    nonpareil            nonpareil
-    nonsuits             nonsuit
-    nony                 noni
-    nook                 nook
-    nooks                nook
-    noon                 noon
-    noonday              noondai
-    noontide             noontid
-    nor                  nor
-    norbery              norberi
-    norfolk              norfolk
-    norman               norman
-    normandy             normandi
-    normans              norman
-    north                north
-    northampton          northampton
-    northamptonshire     northamptonshir
-    northerly            northerli
-    northern             northern
-    northgate            northgat
-    northumberland       northumberland
-    northumberlands      northumberland
-    northward            northward
-    norway               norwai
-    norways              norwai
-    norwegian            norwegian
-    norweyan             norweyan
-    nos                  no
-    nose                 nose
-    nosegays             nosegai
-    noseless             noseless
-    noses                nose
-    noster               noster
-    nostra               nostra
-    nostril              nostril
-    nostrils             nostril
-    not                  not
-    notable              notabl
-    notably              notabl
-    notary               notari
-    notch                notch
-    note                 note
-    notebook             notebook
-    noted                note
-    notedly              notedli
-    notes                note
-    notest               notest
-    noteworthy           noteworthi
-    nothing              noth
-    nothings             noth
-    notice               notic
-    notify               notifi
-    noting               note
-    notion               notion
-    notorious            notori
-    notoriously          notori
-    notre                notr
-    notwithstanding      notwithstand
-    nought               nought
-    noun                 noun
-    nouns                noun
-    nourish              nourish
-    nourished            nourish
-    nourisher            nourish
-    nourishes            nourish
-    nourisheth           nourisheth
-    nourishing           nourish
-    nourishment          nourish
-    nous                 nou
-    novel                novel
-    novelties            novelti
-    novelty              novelti
-    noverbs              noverb
-    novi                 novi
-    novice               novic
-    novices              novic
-    novum                novum
-    now                  now
-    nowhere              nowher
-    noyance              noyanc
-    ns                   ns
-    nt                   nt
-    nubibus              nubibu
-    numa                 numa
-    numb                 numb
-    number               number
-    numbered             number
-    numbering            number
-    numberless           numberless
-    numbers              number
-    numbness             numb
-    nun                  nun
-    nuncio               nuncio
-    nuncle               nuncl
-    nunnery              nunneri
-    nuns                 nun
-    nuntius              nuntiu
-    nuptial              nuptial
-    nurs                 nur
-    nurse                nurs
-    nursed               nurs
-    nurser               nurser
-    nursery              nurseri
-    nurses               nurs
-    nurseth              nurseth
-    nursh                nursh
-    nursing              nurs
-    nurtur               nurtur
-    nurture              nurtur
-    nut                  nut
-    nuthook              nuthook
-    nutmeg               nutmeg
-    nutmegs              nutmeg
-    nutriment            nutriment
-    nuts                 nut
-    nutshell             nutshel
-    ny                   ny
-    nym                  nym
-    nymph                nymph
-    nymphs               nymph
-    o                    o
-    oak                  oak
-    oaken                oaken
-    oaks                 oak
-    oared                oar
-    oars                 oar
-    oatcake              oatcak
-    oaten                oaten
-    oath                 oath
-    oathable             oathabl
-    oaths                oath
-    oats                 oat
-    ob                   ob
-    obduracy             obduraci
-    obdurate             obdur
-    obedience            obedi
-    obedient             obedi
-    obeisance            obeis
-    oberon               oberon
-    obey                 obei
-    obeyed               obei
-    obeying              obei
-    obeys                obei
-    obidicut             obidicut
-    object               object
-    objected             object
-    objections           object
-    objects              object
-    oblation             oblat
-    oblations            oblat
-    obligation           oblig
-    obligations          oblig
-    obliged              oblig
-    oblique              obliqu
-    oblivion             oblivion
-    oblivious            oblivi
-    obloquy              obloqui
-    obscene              obscen
-    obscenely            obscen
-    obscur               obscur
-    obscure              obscur
-    obscured             obscur
-    obscurely            obscur
-    obscures             obscur
-    obscuring            obscur
-    obscurity            obscur
-    obsequies            obsequi
-    obsequious           obsequi
-    obsequiously         obsequi
-    observ               observ
-    observance           observ
-    observances          observ
-    observancy           observ
-    observant            observ
-    observants           observ
-    observation          observ
-    observe              observ
-    observed             observ
-    observer             observ
-    observers            observ
-    observing            observ
-    observingly          observingli
-    obsque               obsqu
-    obstacle             obstacl
-    obstacles            obstacl
-    obstinacy            obstinaci
-    obstinate            obstin
-    obstinately          obstin
-    obstruct             obstruct
-    obstruction          obstruct
-    obstructions         obstruct
-    obtain               obtain
-    obtained             obtain
-    obtaining            obtain
-    occasion             occas
-    occasions            occas
-    occident             occid
-    occidental           occident
-    occulted             occult
-    occupat              occupat
-    occupation           occup
-    occupations          occup
-    occupied             occupi
-    occupies             occupi
-    occupy               occupi
-    occurrence           occurr
-    occurrences          occurr
-    occurrents           occurr
-    ocean                ocean
-    oceans               ocean
-    octavia              octavia
-    octavius             octaviu
-    ocular               ocular
-    od                   od
-    odd                  odd
-    oddest               oddest
-    oddly                oddli
-    odds                 odd
-    ode                  od
-    odes                 od
-    odious               odiou
-    odoriferous          odorifer
-    odorous              odor
-    odour                odour
-    odours               odour
-    ods                  od
-    oeillades            oeillad
-    oes                  oe
-    oeuvres              oeuvr
-    of                   of
-    ofephesus            ofephesu
-    off                  off
-    offal                offal
-    offence              offenc
-    offenceful           offenc
-    offences             offenc
-    offend               offend
-    offended             offend
-    offendendo           offendendo
-    offender             offend
-    offenders            offend
-    offendeth            offendeth
-    offending            offend
-    offendress           offendress
-    offends              offend
-    offense              offens
-    offenseless          offenseless
-    offenses             offens
-    offensive            offens
-    offer                offer
-    offered              offer
-    offering             offer
-    offerings            offer
-    offers               offer
-    offert               offert
-    offic                offic
-    office               offic
-    officed              offic
-    officer              offic
-    officers             offic
-    offices              offic
-    official             offici
-    officious            offici
-    offspring            offspr
-    oft                  oft
-    often                often
-    oftener              often
-    oftentimes           oftentim
-    oh                   oh
-    oil                  oil
-    oils                 oil
-    oily                 oili
-    old                  old
-    oldcastle            oldcastl
-    olden                olden
-    older                older
-    oldest               oldest
-    oldness              old
-    olive                oliv
-    oliver               oliv
-    olivers              oliv
-    olives               oliv
-    olivia               olivia
-    olympian             olympian
-    olympus              olympu
-    oman                 oman
-    omans                oman
-    omen                 omen
-    ominous              omin
-    omission             omiss
-    omit                 omit
-    omittance            omitt
-    omitted              omit
-    omitting             omit
-    omne                 omn
-    omnes                omn
-    omnipotent           omnipot
-    on                   on
-    once                 onc
-    one                  on
-    ones                 on
-    oneyers              oney
-    ongles               ongl
-    onion                onion
-    onions               onion
-    only                 onli
-    onset                onset
-    onward               onward
-    onwards              onward
-    oo                   oo
-    ooze                 ooz
-    oozes                ooz
-    oozy                 oozi
-    op                   op
-    opal                 opal
-    ope                  op
-    open                 open
-    opener               open
-    opening              open
-    openly               openli
-    openness             open
-    opens                open
-    operant              oper
-    operate              oper
-    operation            oper
-    operations           oper
-    operative            oper
-    opes                 op
-    oph                  oph
-    ophelia              ophelia
-    opinion              opinion
-    opinions             opinion
-    opportune            opportun
-    opportunities        opportun
-    opportunity          opportun
-    oppos                oppo
-    oppose               oppos
-    opposed              oppos
-    opposeless           opposeless
-    opposer              oppos
-    opposers             oppos
-    opposes              oppos
-    opposing             oppos
-    opposite             opposit
-    opposites            opposit
-    opposition           opposit
-    oppositions          opposit
-    oppress              oppress
-    oppressed            oppress
-    oppresses            oppress
-    oppresseth           oppresseth
-    oppressing           oppress
-    oppression           oppress
-    oppressor            oppressor
-    opprest              opprest
-    opprobriously        opprobri
-    oppugnancy           oppugn
-    opulency             opul
-    opulent              opul
-    or                   or
-    oracle               oracl
-    oracles              oracl
-    orange               orang
-    oration              orat
-    orator               orat
-    orators              orat
-    oratory              oratori
-    orb                  orb
-    orbed                orb
-    orbs                 orb
-    orchard              orchard
-    orchards             orchard
-    ord                  ord
-    ordain               ordain
-    ordained             ordain
-    ordaining            ordain
-    order                order
-    ordered              order
-    ordering             order
-    orderless            orderless
-    orderly              orderli
-    orders               order
-    ordinance            ordin
-    ordinant             ordin
-    ordinaries           ordinari
-    ordinary             ordinari
-    ordnance             ordnanc
-    ords                 ord
-    ordure               ordur
-    ore                  or
-    organ                organ
-    organs               organ
-    orgillous            orgil
-    orient               orient
-    orifex               orifex
-    origin               origin
-    original             origin
-    orisons              orison
-    ork                  ork
-    orlando              orlando
-    orld                 orld
-    orleans              orlean
-    ornament             ornament
-    ornaments            ornament
-    orodes               orod
-    orphan               orphan
-    orphans              orphan
-    orpheus              orpheu
-    orsino               orsino
-    ort                  ort
-    orthography          orthographi
-    orts                 ort
-    oscorbidulchos       oscorbidulcho
-    osier                osier
-    osiers               osier
-    osprey               osprei
-    osr                  osr
-    osric                osric
-    ossa                 ossa
-    ost                  ost
-    ostent               ostent
-    ostentare            ostentar
-    ostentation          ostent
-    ostents              ostent
-    ostler               ostler
-    ostlers              ostler
-    ostrich              ostrich
-    osw                  osw
-    oswald               oswald
-    othello              othello
-    other                other
-    othergates           otherg
-    others               other
-    otherwhere           otherwher
-    otherwhiles          otherwhil
-    otherwise            otherwis
-    otter                otter
-    ottoman              ottoman
-    ottomites            ottomit
-    oublie               oubli
-    ouches               ouch
-    ought                ought
-    oui                  oui
-    ounce                ounc
-    ounces               ounc
-    ouphes               ouph
-    our                  our
-    ours                 our
-    ourself              ourself
-    ourselves            ourselv
-    ousel                ousel
-    out                  out
-    outbids              outbid
-    outbrave             outbrav
-    outbraves            outbrav
-    outbreak             outbreak
-    outcast              outcast
-    outcries             outcri
-    outcry               outcri
-    outdar               outdar
-    outdare              outdar
-    outdares             outdar
-    outdone              outdon
-    outfac               outfac
-    outface              outfac
-    outfaced             outfac
-    outfacing            outfac
-    outfly               outfli
-    outfrown             outfrown
-    outgo                outgo
-    outgoes              outgo
-    outgrown             outgrown
-    outjest              outjest
-    outlaw               outlaw
-    outlawry             outlawri
-    outlaws              outlaw
-    outliv               outliv
-    outlive              outliv
-    outlives             outliv
-    outliving            outliv
-    outlook              outlook
-    outlustres           outlustr
-    outpriz              outpriz
-    outrage              outrag
-    outrageous           outrag
-    outrages             outrag
-    outran               outran
-    outright             outright
-    outroar              outroar
-    outrun               outrun
-    outrunning           outrun
-    outruns              outrun
-    outscold             outscold
-    outscorn             outscorn
-    outsell              outsel
-    outsells             outsel
-    outside              outsid
-    outsides             outsid
-    outspeaks            outspeak
-    outsport             outsport
-    outstare             outstar
-    outstay              outstai
-    outstood             outstood
-    outstretch           outstretch
-    outstretched         outstretch
-    outstrike            outstrik
-    outstrip             outstrip
-    outstripped          outstrip
-    outswear             outswear
-    outvenoms            outvenom
-    outward              outward
-    outwardly            outwardli
-    outwards             outward
-    outwear              outwear
-    outweighs            outweigh
-    outwent              outwent
-    outworn              outworn
-    outworths            outworth
-    oven                 oven
-    over                 over
-    overawe              overaw
-    overbear             overbear
-    overblown            overblown
-    overboard            overboard
-    overbold             overbold
-    overborne            overborn
-    overbulk             overbulk
-    overbuys             overbui
-    overcame             overcam
-    overcast             overcast
-    overcharg            overcharg
-    overcharged          overcharg
-    overcome             overcom
-    overcomes            overcom
-    overdone             overdon
-    overearnest          overearnest
-    overfar              overfar
-    overflow             overflow
-    overflown            overflown
-    overglance           overgl
-    overgo               overgo
-    overgone             overgon
-    overgorg             overgorg
-    overgrown            overgrown
-    overhead             overhead
-    overhear             overhear
-    overheard            overheard
-    overhold             overhold
-    overjoyed            overjoi
-    overkind             overkind
-    overland             overland
-    overleather          overleath
-    overlive             overl
-    overlook             overlook
-    overlooking          overlook
-    overlooks            overlook
-    overmaster           overmast
-    overmounting         overmount
-    overmuch             overmuch
-    overpass             overpass
-    overpeer             overp
-    overpeering          overp
-    overplus             overplu
-    overrul              overrul
-    overrun              overrun
-    overscutch           overscutch
-    overset              overset
-    overshades           overshad
-    overshine            overshin
-    overshines           overshin
-    overshot             overshot
-    oversights           oversight
-    overspread           overspread
-    overstain            overstain
-    overswear            overswear
-    overt                overt
-    overta               overta
-    overtake             overtak
-    overtaketh           overtaketh
-    overthrow            overthrow
-    overthrown           overthrown
-    overthrows           overthrow
-    overtook             overtook
-    overtopp             overtopp
-    overture             overtur
-    overturn             overturn
-    overwatch            overwatch
-    overween             overween
-    overweening          overween
-    overweigh            overweigh
-    overwhelm            overwhelm
-    overwhelming         overwhelm
-    overworn             overworn
-    ovid                 ovid
-    ovidius              ovidiu
-    ow                   ow
-    owe                  ow
-    owed                 ow
-    owedst               owedst
-    owen                 owen
-    owes                 ow
-    owest                owest
-    oweth                oweth
-    owing                ow
-    owl                  owl
-    owls                 owl
-    own                  own
-    owner                owner
-    owners               owner
-    owning               own
-    owns                 own
-    owy                  owi
-    ox                   ox
-    oxen                 oxen
-    oxford               oxford
-    oxfordshire          oxfordshir
-    oxlips               oxlip
-    oyes                 oy
-    oyster               oyster
-    p                    p
-    pabble               pabbl
-    pabylon              pabylon
-    pac                  pac
-    pace                 pace
-    paced                pace
-    paces                pace
-    pacified             pacifi
-    pacify               pacifi
-    pacing               pace
-    pack                 pack
-    packet               packet
-    packets              packet
-    packhorses           packhors
-    packing              pack
-    packings             pack
-    packs                pack
-    packthread           packthread
-    pacorus              pacoru
-    paction              paction
-    pad                  pad
-    paddle               paddl
-    paddling             paddl
-    paddock              paddock
-    padua                padua
-    pagan                pagan
-    pagans               pagan
-    page                 page
-    pageant              pageant
-    pageants             pageant
-    pages                page
-    pah                  pah
-    paid                 paid
-    pail                 pail
-    pailfuls             pail
-    pails                pail
-    pain                 pain
-    pained               pain
-    painful              pain
-    painfully            painfulli
-    pains                pain
-    paint                paint
-    painted              paint
-    painter              painter
-    painting             paint
-    paintings            paint
-    paints               paint
-    pair                 pair
-    paired               pair
-    pairs                pair
-    pajock               pajock
-    pal                  pal
-    palabras             palabra
-    palace               palac
-    palaces              palac
-    palamedes            palamed
-    palate               palat
-    palates              palat
-    palatine             palatin
-    palating             palat
-    pale                 pale
-    paled                pale
-    paleness             pale
-    paler                paler
-    pales                pale
-    palestine            palestin
-    palfrey              palfrei
-    palfreys             palfrei
-    palisadoes           palisado
-    pall                 pall
-    pallabris            pallabri
-    pallas               palla
-    pallets              pallet
-    palm                 palm
-    palmer               palmer
-    palmers              palmer
-    palms                palm
-    palmy                palmi
-    palpable             palpabl
-    palsied              palsi
-    palsies              palsi
-    palsy                palsi
-    palt                 palt
-    palter               palter
-    paltry               paltri
-    paly                 pali
-    pamp                 pamp
-    pamper               pamper
-    pamphlets            pamphlet
-    pan                  pan
-    pancackes            pancack
-    pancake              pancak
-    pancakes             pancak
-    pandar               pandar
-    pandars              pandar
-    pandarus             pandaru
-    pander               pander
-    panderly             panderli
-    panders              pander
-    pandulph             pandulph
-    panel                panel
-    pang                 pang
-    panging              pang
-    pangs                pang
-    pannier              pannier
-    pannonians           pannonian
-    pansa                pansa
-    pansies              pansi
-    pant                 pant
-    pantaloon            pantaloon
-    panted               pant
-    pantheon             pantheon
-    panther              panther
-    panthino             panthino
-    panting              pant
-    pantingly            pantingli
-    pantler              pantler
-    pantry               pantri
-    pants                pant
-    pap                  pap
-    papal                papal
-    paper                paper
-    papers               paper
-    paphlagonia          paphlagonia
-    paphos               papho
-    papist               papist
-    paps                 pap
-    par                  par
-    parable              parabl
-    paracelsus           paracelsu
-    paradise             paradis
-    paradox              paradox
-    paradoxes            paradox
-    paragon              paragon
-    paragons             paragon
-    parallel             parallel
-    parallels            parallel
-    paramour             paramour
-    paramours            paramour
-    parapets             parapet
-    paraquito            paraquito
-    parasite             parasit
-    parasites            parasit
-    parca                parca
-    parcel               parcel
-    parcell              parcel
-    parcels              parcel
-    parch                parch
-    parched              parch
-    parching             parch
-    parchment            parchment
-    pard                 pard
-    pardon               pardon
-    pardona              pardona
-    pardoned             pardon
-    pardoner             pardon
-    pardoning            pardon
-    pardonne             pardonn
-    pardonner            pardonn
-    pardonnez            pardonnez
-    pardons              pardon
-    pare                 pare
-    pared                pare
-    parel                parel
-    parent               parent
-    parentage            parentag
-    parents              parent
-    parfect              parfect
-    paring               pare
-    parings              pare
-    paris                pari
-    parish               parish
-    parishioners         parishion
-    parisians            parisian
-    paritors             paritor
-    park                 park
-    parks                park
-    parle                parl
-    parler               parler
-    parles               parl
-    parley               parlei
-    parlez               parlez
-    parliament           parliament
-    parlors              parlor
-    parlour              parlour
-    parlous              parlou
-    parmacity            parmac
-    parolles             parol
-    parricide            parricid
-    parricides           parricid
-    parrot               parrot
-    parrots              parrot
-    parsley              parslei
-    parson               parson
-    part                 part
-    partake              partak
-    partaken             partaken
-    partaker             partak
-    partakers            partak
-    parted               part
-    parthia              parthia
-    parthian             parthian
-    parthians            parthian
-    parti                parti
-    partial              partial
-    partialize           partial
-    partially            partial
-    participate          particip
-    participation        particip
-    particle             particl
-    particular           particular
-    particularities      particular
-    particularize        particular
-    particularly         particularli
-    particulars          particular
-    parties              parti
-    parting              part
-    partisan             partisan
-    partisans            partisan
-    partition            partit
-    partizan             partizan
-    partlet              partlet
-    partly               partli
-    partner              partner
-    partners             partner
-    partridge            partridg
-    parts                part
-    party                parti
-    pas                  pa
-    pash                 pash
-    pashed               pash
-    pashful              pash
-    pass                 pass
-    passable             passabl
-    passado              passado
-    passage              passag
-    passages             passag
-    passant              passant
-    passed               pass
-    passenger            passeng
-    passengers           passeng
-    passes               pass
-    passeth              passeth
-    passing              pass
-    passio               passio
-    passion              passion
-    passionate           passion
-    passioning           passion
-    passions             passion
-    passive              passiv
-    passport             passport
-    passy                passi
-    past                 past
-    paste                past
-    pasterns             pastern
-    pasties              pasti
-    pastime              pastim
-    pastimes             pastim
-    pastoral             pastor
-    pastorals            pastor
-    pastors              pastor
-    pastry               pastri
-    pasture              pastur
-    pastures             pastur
-    pasty                pasti
-    pat                  pat
-    patay                patai
-    patch                patch
-    patchery             patcheri
-    patches              patch
-    pate                 pate
-    pated                pate
-    patent               patent
-    patents              patent
-    paternal             patern
-    pates                pate
-    path                 path
-    pathetical           pathet
-    paths                path
-    pathway              pathwai
-    pathways             pathwai
-    patience             patienc
-    patient              patient
-    patiently            patient
-    patients             patient
-    patines              patin
-    patrician            patrician
-    patricians           patrician
-    patrick              patrick
-    patrimony            patrimoni
-    patroclus            patroclu
-    patron               patron
-    patronage            patronag
-    patroness            patro
-    patrons              patron
-    patrum               patrum
-    patter               patter
-    pattern              pattern
-    patterns             pattern
-    pattle               pattl
-    pauca                pauca
-    paucas               pauca
-    paul                 paul
-    paulina              paulina
-    paunch               paunch
-    paunches             paunch
-    pause                paus
-    pauser               pauser
-    pauses               paus
-    pausingly            pausingli
-    pauvres              pauvr
-    pav                  pav
-    paved                pave
-    pavement             pavement
-    pavilion             pavilion
-    pavilions            pavilion
-    pavin                pavin
-    paw                  paw
-    pawn                 pawn
-    pawns                pawn
-    paws                 paw
-    pax                  pax
-    pay                  pai
-    payest               payest
-    paying               pai
-    payment              payment
-    payments             payment
-    pays                 pai
-    paysan               paysan
-    paysans              paysan
-    pe                   pe
-    peace                peac
-    peaceable            peaceabl
-    peaceably            peaceabl
-    peaceful             peac
-    peacemakers          peacemak
-    peaces               peac
-    peach                peach
-    peaches              peach
-    peacock              peacock
-    peacocks             peacock
-    peak                 peak
-    peaking              peak
-    peal                 peal
-    peals                peal
-    pear                 pear
-    peard                peard
-    pearl                pearl
-    pearls               pearl
-    pears                pear
-    peas                 pea
-    peasant              peasant
-    peasantry            peasantri
-    peasants             peasant
-    peascod              peascod
-    pease                peas
-    peaseblossom         peaseblossom
-    peat                 peat
-    peaten               peaten
-    peating              peat
-    pebble               pebbl
-    pebbled              pebbl
-    pebbles              pebbl
-    peck                 peck
-    pecks                peck
-    peculiar             peculiar
-    pecus                pecu
-    pedant               pedant
-    pedantical           pedant
-    pedascule            pedascul
-    pede                 pede
-    pedestal             pedest
-    pedigree             pedigre
-    pedlar               pedlar
-    pedlars              pedlar
-    pedro                pedro
-    peds                 ped
-    peel                 peel
-    peep                 peep
-    peeped               peep
-    peeping              peep
-    peeps                peep
-    peer                 peer
-    peereth              peereth
-    peering              peer
-    peerless             peerless
-    peers                peer
-    peesel               peesel
-    peevish              peevish
-    peevishly            peevishli
-    peflur               peflur
-    peg                  peg
-    pegasus              pegasu
-    pegs                 peg
-    peise                peis
-    peised               peis
-    peize                peiz
-    pelf                 pelf
-    pelican              pelican
-    pelion               pelion
-    pell                 pell
-    pella                pella
-    pelleted             pellet
-    peloponnesus         peloponnesu
-    pelt                 pelt
-    pelting              pelt
-    pembroke             pembrok
-    pen                  pen
-    penalties            penalti
-    penalty              penalti
-    penance              penanc
-    pence                penc
-    pencil               pencil
-    pencill              pencil
-    pencils              pencil
-    pendant              pendant
-    pendent              pendent
-    pendragon            pendragon
-    pendulous            pendul
-    penelope             penelop
-    penetrable           penetr
-    penetrate            penetr
-    penetrative          penetr
-    penitence            penit
-    penitent             penit
-    penitential          penitenti
-    penitently           penit
-    penitents            penit
-    penker               penker
-    penknife             penknif
-    penn                 penn
-    penned               pen
-    penning              pen
-    pennons              pennon
-    penny                penni
-    pennyworth           pennyworth
-    pennyworths          pennyworth
-    pens                 pen
-    pense                pens
-    pension              pension
-    pensioners           pension
-    pensive              pensiv
-    pensived             pensiv
-    pensively            pensiv
-    pent                 pent
-    pentecost            pentecost
-    penthesilea          penthesilea
-    penthouse            penthous
-    penurious            penuri
-    penury               penuri
-    peopl                peopl
-    people               peopl
-    peopled              peopl
-    peoples              peopl
-    pepin                pepin
-    pepper               pepper
-    peppercorn           peppercorn
-    peppered             pepper
-    per                  per
-    peradventure         peradventur
-    peradventures        peradventur
-    perceiv              perceiv
-    perceive             perceiv
-    perceived            perceiv
-    perceives            perceiv
-    perceiveth           perceiveth
-    perch                perch
-    perchance            perchanc
-    percies              perci
-    percussion           percuss
-    percy                perci
-    perdie               perdi
-    perdita              perdita
-    perdition            perdit
-    perdonato            perdonato
-    perdu                perdu
-    perdurable           perdur
-    perdurably           perdur
-    perdy                perdi
-    pere                 pere
-    peregrinate          peregrin
-    peremptorily         peremptorili
-    peremptory           peremptori
-    perfect              perfect
-    perfected            perfect
-    perfecter            perfect
-    perfectest           perfectest
-    perfection           perfect
-    perfections          perfect
-    perfectly            perfectli
-    perfectness          perfect
-    perfidious           perfidi
-    perfidiously         perfidi
-    perforce             perforc
-    perform              perform
-    performance          perform
-    performances         perform
-    performed            perform
-    performer            perform
-    performers           perform
-    performing           perform
-    performs             perform
-    perfum               perfum
-    perfume              perfum
-    perfumed             perfum
-    perfumer             perfum
-    perfumes             perfum
-    perge                perg
-    perhaps              perhap
-    periapts             periapt
-    perigort             perigort
-    perigouna            perigouna
-    peril                peril
-    perilous             peril
-    perils               peril
-    period               period
-    periods              period
-    perish               perish
-    perished             perish
-    perishest            perishest
-    perisheth            perisheth
-    perishing            perish
-    periwig              periwig
-    perjur               perjur
-    perjure              perjur
-    perjured             perjur
-    perjuries            perjuri
-    perjury              perjuri
-    perk                 perk
-    perkes               perk
-    permafoy             permafoi
-    permanent            perman
-    permission           permiss
-    permissive           permiss
-    permit               permit
-    permitted            permit
-    pernicious           pernici
-    perniciously         pernici
-    peroration           peror
-    perpend              perpend
-    perpendicular        perpendicular
-    perpendicularly      perpendicularli
-    perpetual            perpetu
-    perpetually          perpetu
-    perpetuity           perpetu
-    perplex              perplex
-    perplexed            perplex
-    perplexity           perplex
-    pers                 per
-    persecuted           persecut
-    persecutions         persecut
-    persecutor           persecutor
-    perseus              perseu
-    persever             persev
-    perseverance         persever
-    persevers            persev
-    persia               persia
-    persian              persian
-    persist              persist
-    persisted            persist
-    persistency          persist
-    persistive           persist
-    persists             persist
-    person               person
-    personae             persona
-    personage            personag
-    personages           personag
-    personal             person
-    personally           person
-    personate            person
-    personated           person
-    personates           person
-    personating          person
-    persons              person
-    perspective          perspect
-    perspectively        perspect
-    perspectives         perspect
-    perspicuous          perspicu
-    persuade             persuad
-    persuaded            persuad
-    persuades            persuad
-    persuading           persuad
-    persuasion           persuas
-    persuasions          persuas
-    pert                 pert
-    pertain              pertain
-    pertaining           pertain
-    pertains             pertain
-    pertaunt             pertaunt
-    pertinent            pertin
-    pertly               pertli
-    perturb              perturb
-    perturbation         perturb
-    perturbations        perturb
-    perturbed            perturb
-    perus                peru
-    perusal              perus
-    peruse               perus
-    perused              perus
-    perusing             perus
-    perverse             pervers
-    perversely           pervers
-    perverseness         pervers
-    pervert              pervert
-    perverted            pervert
-    peseech              peseech
-    pest                 pest
-    pester               pester
-    pestiferous          pestifer
-    pestilence           pestil
-    pestilent            pestil
-    pet                  pet
-    petar                petar
-    peter                peter
-    petit                petit
-    petition             petit
-    petitionary          petitionari
-    petitioner           petition
-    petitioners          petition
-    petitions            petit
-    peto                 peto
-    petrarch             petrarch
-    petruchio            petruchio
-    petter               petter
-    petticoat            petticoat
-    petticoats           petticoat
-    pettiness            petti
-    pettish              pettish
-    pettitoes            pettito
-    petty                petti
-    peu                  peu
-    pew                  pew
-    pewter               pewter
-    pewterer             pewter
-    phaethon             phaethon
-    phaeton              phaeton
-    phantasime           phantasim
-    phantasimes          phantasim
-    phantasma            phantasma
-    pharamond            pharamond
-    pharaoh              pharaoh
-    pharsalia            pharsalia
-    pheasant             pheasant
-    pheazar              pheazar
-    phebe                phebe
-    phebes               phebe
-    pheebus              pheebu
-    pheeze               pheez
-    phibbus              phibbu
-    philadelphos         philadelpho
-    philario             philario
-    philarmonus          philarmonu
-    philemon             philemon
-    philip               philip
-    philippan            philippan
-    philippe             philipp
-    philippi             philippi
-    phillida             phillida
-    philo                philo
-    philomel             philomel
-    philomela            philomela
-    philosopher          philosoph
-    philosophers         philosoph
-    philosophical        philosoph
-    philosophy           philosophi
-    philostrate          philostr
-    philotus             philotu
-    phlegmatic           phlegmat
-    phoebe               phoeb
-    phoebus              phoebu
-    phoenicia            phoenicia
-    phoenicians          phoenician
-    phoenix              phoenix
-    phorbus              phorbu
-    photinus             photinu
-    phrase               phrase
-    phraseless           phraseless
-    phrases              phrase
-    phrygia              phrygia
-    phrygian             phrygian
-    phrynia              phrynia
-    physic               physic
-    physical             physic
-    physician            physician
-    physicians           physician
-    physics              physic
-    pia                  pia
-    pibble               pibbl
-    pible                pibl
-    picardy              picardi
-    pick                 pick
-    pickaxe              pickax
-    pickaxes             pickax
-    pickbone             pickbon
-    picked               pick
-    pickers              picker
-    picking              pick
-    pickle               pickl
-    picklock             picklock
-    pickpurse            pickpurs
-    picks                pick
-    pickt                pickt
-    pickthanks           pickthank
-    pictur               pictur
-    picture              pictur
-    pictured             pictur
-    pictures             pictur
-    pid                  pid
-    pie                  pie
-    piec                 piec
-    piece                piec
-    pieces               piec
-    piecing              piec
-    pied                 pi
-    piedness             pied
-    pier                 pier
-    pierc                pierc
-    pierce               pierc
-    pierced              pierc
-    pierces              pierc
-    pierceth             pierceth
-    piercing             pierc
-    piercy               pierci
-    piers                pier
-    pies                 pi
-    piety                pieti
-    pig                  pig
-    pigeon               pigeon
-    pigeons              pigeon
-    pight                pight
-    pigmy                pigmi
-    pigrogromitus        pigrogromitu
-    pike                 pike
-    pikes                pike
-    pil                  pil
-    pilate               pilat
-    pilates              pilat
-    pilchers             pilcher
-    pile                 pile
-    piles                pile
-    pilf                 pilf
-    pilfering            pilfer
-    pilgrim              pilgrim
-    pilgrimage           pilgrimag
-    pilgrims             pilgrim
-    pill                 pill
-    pillage              pillag
-    pillagers            pillag
-    pillar               pillar
-    pillars              pillar
-    pillicock            pillicock
-    pillory              pillori
-    pillow               pillow
-    pillows              pillow
-    pills                pill
-    pilot                pilot
-    pilots               pilot
-    pimpernell           pimpernel
-    pin                  pin
-    pinch                pinch
-    pinched              pinch
-    pinches              pinch
-    pinching             pinch
-    pindarus             pindaru
-    pine                 pine
-    pined                pine
-    pines                pine
-    pinfold              pinfold
-    pining               pine
-    pinion               pinion
-    pink                 pink
-    pinn                 pinn
-    pinnace              pinnac
-    pins                 pin
-    pinse                pins
-    pint                 pint
-    pintpot              pintpot
-    pioned               pion
-    pioneers             pioneer
-    pioner               pioner
-    pioners              pioner
-    pious                piou
-    pip                  pip
-    pipe                 pipe
-    piper                piper
-    pipers               piper
-    pipes                pipe
-    piping               pipe
-    pippin               pippin
-    pippins              pippin
-    pirate               pirat
-    pirates              pirat
-    pisa                 pisa
-    pisanio              pisanio
-    pish                 pish
-    pismires             pismir
-    piss                 piss
-    pissing              piss
-    pistol               pistol
-    pistols              pistol
-    pit                  pit
-    pitch                pitch
-    pitched              pitch
-    pitcher              pitcher
-    pitchers             pitcher
-    pitchy               pitchi
-    piteous              piteou
-    piteously            piteous
-    pitfall              pitfal
-    pith                 pith
-    pithless             pithless
-    pithy                pithi
-    pitie                piti
-    pitied               piti
-    pities               piti
-    pitiful              piti
-    pitifully            pitifulli
-    pitiless             pitiless
-    pits                 pit
-    pittance             pittanc
-    pittie               pitti
-    pittikins            pittikin
-    pity                 piti
-    pitying              piti
-    pius                 piu
-    plac                 plac
-    place                place
-    placed               place
-    placentio            placentio
-    places               place
-    placeth              placeth
-    placid               placid
-    placing              place
-    plack                plack
-    placket              placket
-    plackets             placket
-    plagu                plagu
-    plague               plagu
-    plagued              plagu
-    plagues              plagu
-    plaguing             plagu
-    plaguy               plagui
-    plain                plain
-    plainer              plainer
-    plainest             plainest
-    plaining             plain
-    plainings            plain
-    plainly              plainli
-    plainness            plain
-    plains               plain
-    plainsong            plainsong
-    plaintful            plaint
-    plaintiff            plaintiff
-    plaintiffs           plaintiff
-    plaints              plaint
-    planched             planch
-    planet               planet
-    planetary            planetari
-    planets              planet
-    planks               plank
-    plant                plant
-    plantage             plantag
-    plantagenet          plantagenet
-    plantagenets         plantagenet
-    plantain             plantain
-    plantation           plantat
-    planted              plant
-    planteth             planteth
-    plants               plant
-    plash                plash
-    plashy               plashi
-    plast                plast
-    plaster              plaster
-    plasterer            plaster
-    plat                 plat
-    plate                plate
-    plated               plate
-    plates               plate
-    platform             platform
-    platforms            platform
-    plats                plat
-    platted              plat
-    plausible            plausibl
-    plausive             plausiv
-    plautus              plautu
-    play                 plai
-    played               plai
-    player               player
-    players              player
-    playeth              playeth
-    playfellow           playfellow
-    playfellows          playfellow
-    playhouse            playhous
-    playing              plai
-    plays                plai
-    plea                 plea
-    pleach               pleach
-    pleached             pleach
-    plead                plead
-    pleaded              plead
-    pleader              pleader
-    pleaders             pleader
-    pleading             plead
-    pleads               plead
-    pleas                plea
-    pleasance            pleasanc
-    pleasant             pleasant
-    pleasantly           pleasantli
-    please               pleas
-    pleased              pleas
-    pleaser              pleaser
-    pleasers             pleaser
-    pleases              pleas
-    pleasest             pleasest
-    pleaseth             pleaseth
-    pleasing             pleas
-    pleasure             pleasur
-    pleasures            pleasur
-    plebeians            plebeian
-    plebeii              plebeii
-    plebs                pleb
-    pledge               pledg
-    pledges              pledg
-    pleines              plein
-    plenitude            plenitud
-    plenteous            plenteou
-    plenteously          plenteous
-    plenties             plenti
-    plentiful            plenti
-    plentifully          plentifulli
-    plenty               plenti
-    pless                pless
-    plessed              pless
-    plessing             pless
-    pliant               pliant
-    plied                pli
-    plies                pli
-    plight               plight
-    plighted             plight
-    plighter             plighter
-    plod                 plod
-    plodded              plod
-    plodders             plodder
-    plodding             plod
-    plods                plod
-    plood                plood
-    ploody               ploodi
-    plot                 plot
-    plots                plot
-    plotted              plot
-    plotter              plotter
-    plough               plough
-    ploughed             plough
-    ploughman            ploughman
-    ploughmen            ploughmen
-    plow                 plow
-    plows                plow
-    pluck                pluck
-    plucked              pluck
-    plucker              plucker
-    plucking             pluck
-    plucks               pluck
-    plue                 plue
-    plum                 plum
-    plume                plume
-    plumed               plume
-    plumes               plume
-    plummet              plummet
-    plump                plump
-    plumpy               plumpi
-    plums                plum
-    plung                plung
-    plunge               plung
-    plunged              plung
-    plural               plural
-    plurisy              plurisi
-    plus                 plu
-    pluto                pluto
-    plutus               plutu
-    ply                  ply
-    po                   po
-    pocket               pocket
-    pocketing            pocket
-    pockets              pocket
-    pocky                pocki
-    pody                 podi
-    poem                 poem
-    poesy                poesi
-    poet                 poet
-    poetical             poetic
-    poetry               poetri
-    poets                poet
-    poictiers            poictier
-    poinards             poinard
-    poins                poin
-    point                point
-    pointblank           pointblank
-    pointed              point
-    pointing             point
-    points               point
-    pois                 poi
-    poise                pois
-    poising              pois
-    poison               poison
-    poisoned             poison
-    poisoner             poison
-    poisoning            poison
-    poisonous            poison
-    poisons              poison
-    poke                 poke
-    poking               poke
-    pol                  pol
-    polack               polack
-    polacks              polack
-    poland               poland
-    pold                 pold
-    pole                 pole
-    poleaxe              poleax
-    polecat              polecat
-    polecats             polecat
-    polemon              polemon
-    poles                pole
-    poli                 poli
-    policies             polici
-    policy               polici
-    polish               polish
-    polished             polish
-    politic              polit
-    politician           politician
-    politicians          politician
-    politicly            politicli
-    polixenes            polixen
-    poll                 poll
-    polluted             pollut
-    pollution            pollut
-    polonius             poloniu
-    poltroons            poltroon
-    polusion             polus
-    polydamus            polydamu
-    polydore             polydor
-    polyxena             polyxena
-    pomander             pomand
-    pomegranate          pomegran
-    pomewater            pomewat
-    pomfret              pomfret
-    pomgarnet            pomgarnet
-    pommel               pommel
-    pomp                 pomp
-    pompeius             pompeiu
-    pompey               pompei
-    pompion              pompion
-    pompous              pompou
-    pomps                pomp
-    pond                 pond
-    ponder               ponder
-    ponderous            ponder
-    ponds                pond
-    poniard              poniard
-    poniards             poniard
-    pont                 pont
-    pontic               pontic
-    pontifical           pontif
-    ponton               ponton
-    pooh                 pooh
-    pool                 pool
-    poole                pool
-    poop                 poop
-    poor                 poor
-    poorer               poorer
-    poorest              poorest
-    poorly               poorli
-    pop                  pop
-    pope                 pope
-    popedom              popedom
-    popilius             popiliu
-    popingay             popingai
-    popish               popish
-    popp                 popp
-    poppy                poppi
-    pops                 pop
-    popular              popular
-    popularity           popular
-    populous             popul
-    porch                porch
-    porches              porch
-    pore                 pore
-    poring               pore
-    pork                 pork
-    porn                 porn
-    porpentine           porpentin
-    porridge             porridg
-    porringer            porring
-    port                 port
-    portable             portabl
-    portage              portag
-    portal               portal
-    portance             portanc
-    portcullis           portculli
-    portend              portend
-    portends             portend
-    portent              portent
-    portentous           portent
-    portents             portent
-    porter               porter
-    porters              porter
-    portia               portia
-    portion              portion
-    portly               portli
-    portotartarossa      portotartarossa
-    portrait             portrait
-    portraiture          portraitur
-    ports                port
-    portugal             portug
-    pose                 pose
-    posied               posi
-    posies               posi
-    position             posit
-    positive             posit
-    positively           posit
-    posse                poss
-    possess              possess
-    possessed            possess
-    possesses            possess
-    possesseth           possesseth
-    possessing           possess
-    possession           possess
-    possessions          possess
-    possessor            possessor
-    posset               posset
-    possets              posset
-    possibilities        possibl
-    possibility          possibl
-    possible             possibl
-    possibly             possibl
-    possitable           possit
-    post                 post
-    poste                post
-    posted               post
-    posterior            posterior
-    posteriors           posterior
-    posterity            poster
-    postern              postern
-    posterns             postern
-    posters              poster
-    posthorse            posthors
-    posthorses           posthors
-    posthumus            posthumu
-    posting              post
-    postmaster           postmast
-    posts                post
-    postscript           postscript
-    posture              postur
-    postures             postur
-    posy                 posi
-    pot                  pot
-    potable              potabl
-    potations            potat
-    potato               potato
-    potatoes             potato
-    potch                potch
-    potency              potenc
-    potent               potent
-    potentates           potent
-    potential            potenti
-    potently             potent
-    potents              potent
-    pothecary            pothecari
-    pother               pother
-    potion               potion
-    potions              potion
-    potpan               potpan
-    pots                 pot
-    potter               potter
-    potting              pot
-    pottle               pottl
-    pouch                pouch
-    poulter              poulter
-    poultice             poultic
-    poultney             poultnei
-    pouncet              pouncet
-    pound                pound
-    pounds               pound
-    pour                 pour
-    pourest              pourest
-    pouring              pour
-    pourquoi             pourquoi
-    pours                pour
-    pout                 pout
-    poverty              poverti
-    pow                  pow
-    powd                 powd
-    powder               powder
-    power                power
-    powerful             power
-    powerfully           powerfulli
-    powerless            powerless
-    powers               power
-    pox                  pox
-    poys                 poi
-    poysam               poysam
-    prabbles             prabbl
-    practic              practic
-    practice             practic
-    practiced            practic
-    practicer            practic
-    practices            practic
-    practicing           practic
-    practis              practi
-    practisants          practis
-    practise             practis
-    practiser            practis
-    practisers           practis
-    practises            practis
-    practising           practis
-    praeclarissimus      praeclarissimu
-    praemunire           praemunir
-    praetor              praetor
-    praetors             praetor
-    pragging             prag
-    prague               pragu
-    prain                prain
-    prains               prain
-    prais                prai
-    praise               prais
-    praised              prais
-    praises              prais
-    praisest             praisest
-    praiseworthy         praiseworthi
-    praising             prais
-    prancing             pranc
-    prank                prank
-    pranks               prank
-    prat                 prat
-    prate                prate
-    prated               prate
-    prater               prater
-    prating              prate
-    prattle              prattl
-    prattler             prattler
-    prattling            prattl
-    prave                prave
-    prawls               prawl
-    prawns               prawn
-    pray                 prai
-    prayer               prayer
-    prayers              prayer
-    praying              prai
-    prays                prai
-    pre                  pre
-    preach               preach
-    preached             preach
-    preachers            preacher
-    preaches             preach
-    preaching            preach
-    preachment           preachment
-    pread                pread
-    preambulate          preambul
-    precedence           preced
-    precedent            preced
-    preceding            preced
-    precept              precept
-    preceptial           precepti
-    precepts             precept
-    precinct             precinct
-    precious             preciou
-    preciously           precious
-    precipice            precipic
-    precipitating        precipit
-    precipitation        precipit
-    precise              precis
-    precisely            precis
-    preciseness          precis
-    precisian            precisian
-    precor               precor
-    precurse             precurs
-    precursors           precursor
-    predeceased          predeceas
-    predecessor          predecessor
-    predecessors         predecessor
-    predestinate         predestin
-    predicament          predica
-    predict              predict
-    prediction           predict
-    predictions          predict
-    predominance         predomin
-    predominant          predomin
-    predominate          predomin
-    preeches             preech
-    preeminence          preemin
-    preface              prefac
-    prefer               prefer
-    preferment           prefer
-    preferments          prefer
-    preferr              preferr
-    preferreth           preferreth
-    preferring           prefer
-    prefers              prefer
-    prefiguring          prefigur
-    prefix               prefix
-    prefixed             prefix
-    preformed            preform
-    pregnancy            pregnanc
-    pregnant             pregnant
-    pregnantly           pregnantli
-    prejudicates         prejud
-    prejudice            prejudic
-    prejudicial          prejudici
-    prelate              prelat
-    premeditated         premedit
-    premeditation        premedit
-    premised             premis
-    premises             premis
-    prenez               prenez
-    prenominate          prenomin
-    prentice             prentic
-    prentices            prentic
-    preordinance         preordin
-    prepar               prepar
-    preparation          prepar
-    preparations         prepar
-    prepare              prepar
-    prepared             prepar
-    preparedly           preparedli
-    prepares             prepar
-    preparing            prepar
-    prepost              prepost
-    preposterous         preposter
-    preposterously       preposter
-    prerogatifes         prerogatif
-    prerogative          prerog
-    prerogatived         prerogativ
-    presage              presag
-    presagers            presag
-    presages             presag
-    presageth            presageth
-    presaging            presag
-    prescience           prescienc
-    prescribe            prescrib
-    prescript            prescript
-    prescription         prescript
-    prescriptions        prescript
-    prescripts           prescript
-    presence             presenc
-    presences            presenc
-    present              present
-    presentation         present
-    presented            present
-    presenter            present
-    presenters           present
-    presenteth           presenteth
-    presenting           present
-    presently            present
-    presentment          present
-    presents             present
-    preserv              preserv
-    preservation         preserv
-    preservative         preserv
-    preserve             preserv
-    preserved            preserv
-    preserver            preserv
-    preservers           preserv
-    preserving           preserv
-    president            presid
-    press                press
-    pressed              press
-    presser              presser
-    presses              press
-    pressing             press
-    pressure             pressur
-    pressures            pressur
-    prest                prest
-    prester              prester
-    presume              presum
-    presumes             presum
-    presuming            presum
-    presumption          presumpt
-    presumptuous         presumptu
-    presuppos            presuppo
-    pret                 pret
-    pretence             pretenc
-    pretences            pretenc
-    pretend              pretend
-    pretended            pretend
-    pretending           pretend
-    pretense             pretens
-    pretext              pretext
-    pretia               pretia
-    prettier             prettier
-    prettiest            prettiest
-    prettily             prettili
-    prettiness           pretti
-    pretty               pretti
-    prevail              prevail
-    prevailed            prevail
-    prevaileth           prevaileth
-    prevailing           prevail
-    prevailment          prevail
-    prevails             prevail
-    prevent              prevent
-    prevented            prevent
-    prevention           prevent
-    preventions          prevent
-    prevents             prevent
-    prey                 prei
-    preyful              prey
-    preys                prei
-    priam                priam
-    priami               priami
-    priamus              priamu
-    pribbles             pribbl
-    price                price
-    prick                prick
-    pricked              prick
-    pricket              pricket
-    pricking             prick
-    pricks               prick
-    pricksong            pricksong
-    pride                pride
-    prides               pride
-    pridge               pridg
-    prie                 prie
-    pried                pri
-    prief                prief
-    pries                pri
-    priest               priest
-    priesthood           priesthood
-    priests              priest
-    prig                 prig
-    primal               primal
-    prime                prime
-    primer               primer
-    primero              primero
-    primest              primest
-    primitive            primit
-    primo                primo
-    primogenity          primogen
-    primrose             primros
-    primroses            primros
-    primy                primi
-    prince               princ
-    princely             princ
-    princes              princ
-    princess             princess
-    principal            princip
-    principalities       princip
-    principality         princip
-    principle            principl
-    principles           principl
-    princox              princox
-    prings               pring
-    print                print
-    printed              print
-    printing             print
-    printless            printless
-    prints               print
-    prioress             prioress
-    priories             priori
-    priority             prioriti
-    priory               priori
-    priscian             priscian
-    prison               prison
-    prisoner             prison
-    prisoners            prison
-    prisonment           prison
-    prisonnier           prisonni
-    prisons              prison
-    pristine             pristin
-    prithe               prith
-    prithee              prithe
-    privacy              privaci
-    private              privat
-    privately            privat
-    privates             privat
-    privilage            privilag
-    privileg             privileg
-    privilege            privileg
-    privileged           privileg
-    privileges           privileg
-    privilegio           privilegio
-    privily              privili
-    privity              priviti
-    privy                privi
-    priz                 priz
-    prize                prize
-    prized               prize
-    prizer               prizer
-    prizes               prize
-    prizest              prizest
-    prizing              prize
-    pro                  pro
-    probable             probabl
-    probal               probal
-    probation            probat
-    proceed              proce
-    proceeded            proceed
-    proceeders           proceed
-    proceeding           proceed
-    proceedings          proceed
-    proceeds             proce
-    process              process
-    procession           process
-    proclaim             proclaim
-    proclaimed           proclaim
-    proclaimeth          proclaimeth
-    proclaims            proclaim
-    proclamation         proclam
-    proclamations        proclam
-    proconsul            proconsul
-    procrastinate        procrastin
-    procreant            procreant
-    procreants           procreant
-    procreation          procreat
-    procrus              procru
-    proculeius           proculeiu
-    procur               procur
-    procurator           procur
-    procure              procur
-    procured             procur
-    procures             procur
-    procuring            procur
-    prodigal             prodig
-    prodigality          prodig
-    prodigally           prodig
-    prodigals            prodig
-    prodigies            prodigi
-    prodigious           prodigi
-    prodigiously         prodigi
-    prodigy              prodigi
-    proditor             proditor
-    produc               produc
-    produce              produc
-    produced             produc
-    produces             produc
-    producing            produc
-    proface              profac
-    profan               profan
-    profanation          profan
-    profane              profan
-    profaned             profan
-    profanely            profan
-    profaneness          profan
-    profaners            profan
-    profaning            profan
-    profess              profess
-    professed            profess
-    professes            profess
-    profession           profess
-    professions          profess
-    professors           professor
-    proffer              proffer
-    proffered            proffer
-    profferer            proffer
-    proffers             proffer
-    proficient           profici
-    profit               profit
-    profitable           profit
-    profitably           profit
-    profited             profit
-    profiting            profit
-    profitless           profitless
-    profits              profit
-    profound             profound
-    profoundest          profoundest
-    profoundly           profoundli
-    progenitors          progenitor
-    progeny              progeni
-    progne               progn
-    prognosticate        prognost
-    prognostication      prognost
-    progress             progress
-    progression          progress
-    prohibit             prohibit
-    prohibition          prohibit
-    project              project
-    projection           project
-    projects             project
-    prolixious           prolixi
-    prolixity            prolix
-    prologue             prologu
-    prologues            prologu
-    prolong              prolong
-    prolongs             prolong
-    promethean           promethean
-    prometheus           prometheu
-    promis               promi
-    promise              promis
-    promised             promis
-    promises             promis
-    promiseth            promiseth
-    promising            promis
-    promontory           promontori
-    promotion            promot
-    promotions           promot
-    prompt               prompt
-    prompted             prompt
-    promptement          promptement
-    prompter             prompter
-    prompting            prompt
-    prompts              prompt
-    prompture            promptur
-    promulgate           promulg
-    prone                prone
-    prononcer            prononc
-    prononcez            prononcez
-    pronoun              pronoun
-    pronounc             pronounc
-    pronounce            pronounc
-    pronounced           pronounc
-    pronouncing          pronounc
-    pronouns             pronoun
-    proof                proof
-    proofs               proof
-    prop                 prop
-    propagate            propag
-    propagation          propag
-    propend              propend
-    propension           propens
-    proper               proper
-    properer             proper
-    properly             properli
-    propertied           properti
-    properties           properti
-    property             properti
-    prophecies           propheci
-    prophecy             propheci
-    prophesied           prophesi
-    prophesier           prophesi
-    prophesy             prophesi
-    prophesying          prophesi
-    prophet              prophet
-    prophetess           prophetess
-    prophetic            prophet
-    prophetically        prophet
-    prophets             prophet
-    propinquity          propinqu
-    propontic            propont
-    proportion           proport
-    proportionable       proportion
-    proportions          proport
-    propos               propo
-    propose              propos
-    proposed             propos
-    proposer             propos
-    proposes             propos
-    proposing            propos
-    proposition          proposit
-    propositions         proposit
-    propounded           propound
-    propp                propp
-    propre               propr
-    propriety            proprieti
-    props                prop
-    propugnation         propugn
-    prorogue             prorogu
-    prorogued            prorogu
-    proscription         proscript
-    proscriptions        proscript
-    prose                prose
-    prosecute            prosecut
-    prosecution          prosecut
-    proselytes           proselyt
-    proserpina           proserpina
-    prosp                prosp
-    prospect             prospect
-    prosper              prosper
-    prosperity           prosper
-    prospero             prospero
-    prosperous           prosper
-    prosperously         prosper
-    prospers             prosper
-    prostitute           prostitut
-    prostrate            prostrat
-    protect              protect
-    protected            protect
-    protection           protect
-    protector            protector
-    protectors           protector
-    protectorship        protectorship
-    protectress          protectress
-    protects             protect
-    protest              protest
-    protestation         protest
-    protestations        protest
-    protested            protest
-    protester            protest
-    protesting           protest
-    protests             protest
-    proteus              proteu
-    protheus             protheu
-    protract             protract
-    protractive          protract
-    proud                proud
-    prouder              prouder
-    proudest             proudest
-    proudlier            proudlier
-    proudly              proudli
-    prouds               proud
-    prov                 prov
-    provand              provand
-    prove                prove
-    proved               prove
-    provender            provend
-    proverb              proverb
-    proverbs             proverb
-    proves               prove
-    proveth              proveth
-    provide              provid
-    provided             provid
-    providence           provid
-    provident            provid
-    providently          provid
-    provider             provid
-    provides             provid
-    province             provinc
-    provinces            provinc
-    provincial           provinci
-    proving              prove
-    provision            provis
-    proviso              proviso
-    provocation          provoc
-    provok               provok
-    provoke              provok
-    provoked             provok
-    provoker             provok
-    provokes             provok
-    provoketh            provoketh
-    provoking            provok
-    provost              provost
-    prowess              prowess
-    prudence             prudenc
-    prudent              prudent
-    prun                 prun
-    prune                prune
-    prunes               prune
-    pruning              prune
-    pry                  pry
-    prying               pry
-    psalm                psalm
-    psalmist             psalmist
-    psalms               psalm
-    psalteries           psalteri
-    ptolemies            ptolemi
-    ptolemy              ptolemi
-    public               public
-    publican             publican
-    publication          public
-    publicly             publicli
-    publicola            publicola
-    publish              publish
-    published            publish
-    publisher            publish
-    publishing           publish
-    publius              publiu
-    pucelle              pucel
-    puck                 puck
-    pudder               pudder
-    pudding              pud
-    puddings             pud
-    puddle               puddl
-    puddled              puddl
-    pudency              pudenc
-    pueritia             pueritia
-    puff                 puff
-    puffing              puf
-    puffs                puff
-    pugging              pug
-    puis                 pui
-    puissance            puissanc
-    puissant             puissant
-    puke                 puke
-    puking               puke
-    pulcher              pulcher
-    puling               pule
-    pull                 pull
-    puller               puller
-    pullet               pullet
-    pulling              pull
-    pulls                pull
-    pulpit               pulpit
-    pulpiter             pulpit
-    pulpits              pulpit
-    pulse                puls
-    pulsidge             pulsidg
-    pump                 pump
-    pumpion              pumpion
-    pumps                pump
-    pun                  pun
-    punched              punch
-    punish               punish
-    punished             punish
-    punishes             punish
-    punishment           punish
-    punishments          punish
-    punk                 punk
-    punto                punto
-    puny                 puni
-    pupil                pupil
-    pupils               pupil
-    puppet               puppet
-    puppets              puppet
-    puppies              puppi
-    puppy                puppi
-    pur                  pur
-    purblind             purblind
-    purchas              purcha
-    purchase             purchas
-    purchased            purchas
-    purchases            purchas
-    purchaseth           purchaseth
-    purchasing           purchas
-    pure                 pure
-    purely               pure
-    purer                purer
-    purest               purest
-    purg                 purg
-    purgation            purgat
-    purgative            purg
-    purgatory            purgatori
-    purge                purg
-    purged               purg
-    purgers              purger
-    purging              purg
-    purifies             purifi
-    purifying            purifi
-    puritan              puritan
-    purity               puriti
-    purlieus             purlieu
-    purple               purpl
-    purpled              purpl
-    purples              purpl
-    purport              purport
-    purpos               purpo
-    purpose              purpos
-    purposed             purpos
-    purposely            purpos
-    purposes             purpos
-    purposeth            purposeth
-    purposing            purpos
-    purr                 purr
-    purs                 pur
-    purse                purs
-    pursents             pursent
-    purses               purs
-    pursu                pursu
-    pursue               pursu
-    pursued              pursu
-    pursuers             pursuer
-    pursues              pursu
-    pursuest             pursuest
-    pursueth             pursueth
-    pursuing             pursu
-    pursuit              pursuit
-    pursuivant           pursuiv
-    pursuivants          pursuiv
-    pursy                pursi
-    purus                puru
-    purveyor             purveyor
-    push                 push
-    pushes               push
-    pusillanimity        pusillanim
-    put                  put
-    putrefy              putrefi
-    putrified            putrifi
-    puts                 put
-    putter               putter
-    putting              put
-    puttock              puttock
-    puzzel               puzzel
-    puzzle               puzzl
-    puzzled              puzzl
-    puzzles              puzzl
-    py                   py
-    pygmalion            pygmalion
-    pygmies              pygmi
-    pygmy                pygmi
-    pyramid              pyramid
-    pyramides            pyramid
-    pyramids             pyramid
-    pyramis              pyrami
-    pyramises            pyramis
-    pyramus              pyramu
-    pyrenean             pyrenean
-    pyrrhus              pyrrhu
-    pythagoras           pythagora
-    qu                   qu
-    quadrangle           quadrangl
-    quae                 quae
-    quaff                quaff
-    quaffing             quaf
-    quagmire             quagmir
-    quail                quail
-    quailing             quail
-    quails               quail
-    quaint               quaint
-    quaintly             quaintli
-    quak                 quak
-    quake                quak
-    quakes               quak
-    qualification        qualif
-    qualified            qualifi
-    qualifies            qualifi
-    qualify              qualifi
-    qualifying           qualifi
-    qualite              qualit
-    qualities            qualiti
-    quality              qualiti
-    qualm                qualm
-    qualmish             qualmish
-    quam                 quam
-    quand                quand
-    quando               quando
-    quantities           quantiti
-    quantity             quantiti
-    quare                quar
-    quarrel              quarrel
-    quarrell             quarrel
-    quarreller           quarrel
-    quarrelling          quarrel
-    quarrelous           quarrel
-    quarrels             quarrel
-    quarrelsome          quarrelsom
-    quarries             quarri
-    quarry               quarri
-    quart                quart
-    quarter              quarter
-    quartered            quarter
-    quartering           quarter
-    quarters             quarter
-    quarts               quart
-    quasi                quasi
-    quat                 quat
-    quatch               quatch
-    quay                 quai
-    que                  que
-    quean                quean
-    queas                quea
-    queasiness           queasi
-    queasy               queasi
-    queen                queen
-    queens               queen
-    quell                quell
-    queller              queller
-    quench               quench
-    quenched             quench
-    quenching            quench
-    quenchless           quenchless
-    quern                quern
-    quest                quest
-    questant             questant
-    question             question
-    questionable         question
-    questioned           question
-    questioning          question
-    questionless         questionless
-    questions            question
-    questrists           questrist
-    quests               quest
-    queubus              queubu
-    qui                  qui
-    quick                quick
-    quicken              quicken
-    quickens             quicken
-    quicker              quicker
-    quicklier            quicklier
-    quickly              quickli
-    quickness            quick
-    quicksand            quicksand
-    quicksands           quicksand
-    quicksilverr         quicksilverr
-    quid                 quid
-    quiddities           quidditi
-    quiddits             quiddit
-    quier                quier
-    quiet                quiet
-    quieter              quieter
-    quietly              quietli
-    quietness            quiet
-    quietus              quietu
-    quill                quill
-    quillets             quillet
-    quills               quill
-    quilt                quilt
-    quinapalus           quinapalu
-    quince               quinc
-    quinces              quinc
-    quintain             quintain
-    quintessence         quintess
-    quintus              quintu
-    quip                 quip
-    quips                quip
-    quire                quir
-    quiring              quir
-    quirk                quirk
-    quirks               quirk
-    quis                 qui
-    quit                 quit
-    quite                quit
-    quits                quit
-    quittance            quittanc
-    quitted              quit
-    quitting             quit
-    quiver               quiver
-    quivering            quiver
-    quivers              quiver
-    quo                  quo
-    quod                 quod
-    quoifs               quoif
-    quoint               quoint
-    quoit                quoit
-    quoits               quoit
-    quondam              quondam
-    quoniam              quoniam
-    quote                quot
-    quoted               quot
-    quotes               quot
-    quoth                quoth
-    quotidian            quotidian
-    r                    r
-    rabbit               rabbit
-    rabble               rabbl
-    rabblement           rabblement
-    race                 race
-    rack                 rack
-    rackers              racker
-    racket               racket
-    rackets              racket
-    racking              rack
-    racks                rack
-    radiance             radianc
-    radiant              radiant
-    radish               radish
-    rafe                 rafe
-    raft                 raft
-    rag                  rag
-    rage                 rage
-    rages                rage
-    rageth               rageth
-    ragg                 ragg
-    ragged               rag
-    raggedness           ragged
-    raging               rage
-    ragozine             ragozin
-    rags                 rag
-    rah                  rah
-    rail                 rail
-    railed               rail
-    railer               railer
-    railest              railest
-    raileth              raileth
-    railing              rail
-    rails                rail
-    raiment              raiment
-    rain                 rain
-    rainbow              rainbow
-    raineth              raineth
-    raining              rain
-    rainold              rainold
-    rains                rain
-    rainy                raini
-    rais                 rai
-    raise                rais
-    raised               rais
-    raises               rais
-    raising              rais
-    raisins              raisin
-    rak                  rak
-    rake                 rake
-    rakers               raker
-    rakes                rake
-    ral                  ral
-    rald                 rald
-    ralph                ralph
-    ram                  ram
-    rambures             rambur
-    ramm                 ramm
-    rampallian           rampallian
-    rampant              rampant
-    ramping              ramp
-    rampir               rampir
-    ramps                ramp
-    rams                 ram
-    ramsey               ramsei
-    ramston              ramston
-    ran                  ran
-    rance                ranc
-    rancorous            rancor
-    rancors              rancor
-    rancour              rancour
-    random               random
-    rang                 rang
-    range                rang
-    ranged               rang
-    rangers              ranger
-    ranges               rang
-    ranging              rang
-    rank                 rank
-    ranker               ranker
-    rankest              rankest
-    ranking              rank
-    rankle               rankl
-    rankly               rankli
-    rankness             rank
-    ranks                rank
-    ransack              ransack
-    ransacking           ransack
-    ransom               ransom
-    ransomed             ransom
-    ransoming            ransom
-    ransomless           ransomless
-    ransoms              ransom
-    rant                 rant
-    ranting              rant
-    rap                  rap
-    rape                 rape
-    rapes                rape
-    rapier               rapier
-    rapiers              rapier
-    rapine               rapin
-    raps                 rap
-    rapt                 rapt
-    rapture              raptur
-    raptures             raptur
-    rar                  rar
-    rare                 rare
-    rarely               rare
-    rareness             rare
-    rarer                rarer
-    rarest               rarest
-    rarities             rariti
-    rarity               rariti
-    rascal               rascal
-    rascalliest          rascalliest
-    rascally             rascal
-    rascals              rascal
-    rased                rase
-    rash                 rash
-    rasher               rasher
-    rashly               rashli
-    rashness             rash
-    rat                  rat
-    ratcatcher           ratcatch
-    ratcliff             ratcliff
-    rate                 rate
-    rated                rate
-    rately               rate
-    rates                rate
-    rather               rather
-    ratherest            ratherest
-    ratified             ratifi
-    ratifiers            ratifi
-    ratify               ratifi
-    rating               rate
-    rational             ration
-    ratolorum            ratolorum
-    rats                 rat
-    ratsbane             ratsban
-    rattle               rattl
-    rattles              rattl
-    rattling             rattl
-    rature               ratur
-    raught               raught
-    rav                  rav
-    rave                 rave
-    ravel                ravel
-    raven                raven
-    ravening             raven
-    ravenous             raven
-    ravens               raven
-    ravenspurgh          ravenspurgh
-    raves                rave
-    ravin                ravin
-    raving               rave
-    ravish               ravish
-    ravished             ravish
-    ravisher             ravish
-    ravishing            ravish
-    ravishments          ravish
-    raw                  raw
-    rawer                rawer
-    rawly                rawli
-    rawness              raw
-    ray                  rai
-    rayed                rai
-    rays                 rai
-    raz                  raz
-    raze                 raze
-    razed                raze
-    razes                raze
-    razeth               razeth
-    razing               raze
-    razor                razor
-    razorable            razor
-    razors               razor
-    razure               razur
-    re                   re
-    reach                reach
-    reaches              reach
-    reacheth             reacheth
-    reaching             reach
-    read                 read
-    reader               reader
-    readiest             readiest
-    readily              readili
-    readiness            readi
-    reading              read
-    readins              readin
-    reads                read
-    ready                readi
-    real                 real
-    really               realli
-    realm                realm
-    realms               realm
-    reap                 reap
-    reapers              reaper
-    reaping              reap
-    reaps                reap
-    rear                 rear
-    rears                rear
-    rearward             rearward
-    reason               reason
-    reasonable           reason
-    reasonably           reason
-    reasoned             reason
-    reasoning            reason
-    reasonless           reasonless
-    reasons              reason
-    reave                reav
-    rebate               rebat
-    rebato               rebato
-    rebeck               rebeck
-    rebel                rebel
-    rebell               rebel
-    rebelling            rebel
-    rebellion            rebellion
-    rebellious           rebelli
-    rebels               rebel
-    rebound              rebound
-    rebuk                rebuk
-    rebuke               rebuk
-    rebukeable           rebuk
-    rebuked              rebuk
-    rebukes              rebuk
-    rebus                rebu
-    recall               recal
-    recant               recant
-    recantation          recant
-    recanter             recant
-    recanting            recant
-    receipt              receipt
-    receipts             receipt
-    receiv               receiv
-    receive              receiv
-    received             receiv
-    receiver             receiv
-    receives             receiv
-    receivest            receivest
-    receiveth            receiveth
-    receiving            receiv
-    receptacle           receptacl
-    rechate              rechat
-    reciprocal           reciproc
-    reciprocally         reciproc
-    recite               recit
-    recited              recit
-    reciterai            reciterai
-    reck                 reck
-    recking              reck
-    reckless             reckless
-    reckon               reckon
-    reckoned             reckon
-    reckoning            reckon
-    reckonings           reckon
-    recks                reck
-    reclaim              reclaim
-    reclaims             reclaim
-    reclusive            reclus
-    recognizance         recogniz
-    recognizances        recogniz
-    recoil               recoil
-    recoiling            recoil
-    recollected          recollect
-    recomforted          recomfort
-    recomforture         recomfortur
-    recommend            recommend
-    recommended          recommend
-    recommends           recommend
-    recompens            recompen
-    recompense           recompens
-    reconcil             reconcil
-    reconcile            reconcil
-    reconciled           reconcil
-    reconcilement        reconcil
-    reconciler           reconcil
-    reconciles           reconcil
-    reconciliation       reconcili
-    record               record
-    recordation          record
-    recorded             record
-    recorder             record
-    recorders            record
-    records              record
-    recount              recount
-    recounted            recount
-    recounting           recount
-    recountments         recount
-    recounts             recount
-    recourse             recours
-    recov                recov
-    recover              recov
-    recoverable          recover
-    recovered            recov
-    recoveries           recoveri
-    recovers             recov
-    recovery             recoveri
-    recreant             recreant
-    recreants            recreant
-    recreate             recreat
-    recreation           recreat
-    rectify              rectifi
-    rector               rector
-    rectorship           rectorship
-    recure               recur
-    recured              recur
-    red                  red
-    redbreast            redbreast
-    redder               redder
-    reddest              reddest
-    rede                 rede
-    redeem               redeem
-    redeemed             redeem
-    redeemer             redeem
-    redeeming            redeem
-    redeems              redeem
-    redeliver            redeliv
-    redemption           redempt
-    redime               redim
-    redness              red
-    redoubled            redoubl
-    redoubted            redoubt
-    redound              redound
-    redress              redress
-    redressed            redress
-    redresses            redress
-    reduce               reduc
-    reechy               reechi
-    reed                 reed
-    reeds                reed
-    reek                 reek
-    reeking              reek
-    reeks                reek
-    reeky                reeki
-    reel                 reel
-    reeleth              reeleth
-    reeling              reel
-    reels                reel
-    refell               refel
-    refer                refer
-    reference            refer
-    referr               referr
-    referred             refer
-    refigured            refigur
-    refin                refin
-    refined              refin
-    reflect              reflect
-    reflecting           reflect
-    reflection           reflect
-    reflex               reflex
-    reform               reform
-    reformation          reform
-    reformed             reform
-    refractory           refractori
-    refrain              refrain
-    refresh              refresh
-    refreshing           refresh
-    reft                 reft
-    refts                reft
-    refuge               refug
-    refus                refu
-    refusal              refus
-    refuse               refus
-    refused              refus
-    refusest             refusest
-    refusing             refus
-    reg                  reg
-    regal                regal
-    regalia              regalia
-    regan                regan
-    regard               regard
-    regardance           regard
-    regarded             regard
-    regardfully          regardfulli
-    regarding            regard
-    regards              regard
-    regenerate           regener
-    regent               regent
-    regentship           regentship
-    regia                regia
-    regiment             regiment
-    regiments            regiment
-    regina               regina
-    region               region
-    regions              region
-    regist               regist
-    register             regist
-    registers            regist
-    regreet              regreet
-    regreets             regreet
-    regress              regress
-    reguerdon            reguerdon
-    regular              regular
-    rehears              rehear
-    rehearsal            rehears
-    rehearse             rehears
-    reign                reign
-    reigned              reign
-    reignier             reignier
-    reigning             reign
-    reigns               reign
-    rein                 rein
-    reinforc             reinforc
-    reinforce            reinforc
-    reinforcement        reinforc
-    reins                rein
-    reiterate            reiter
-    reject               reject
-    rejected             reject
-    rejoic               rejoic
-    rejoice              rejoic
-    rejoices             rejoic
-    rejoiceth            rejoiceth
-    rejoicing            rejoic
-    rejoicingly          rejoicingli
-    rejoindure           rejoindur
-    rejourn              rejourn
-    rel                  rel
-    relapse              relaps
-    relate               relat
-    relates              relat
-    relation             relat
-    relations            relat
-    relative             rel
-    releas               relea
-    release              releas
-    released             releas
-    releasing            releas
-    relent               relent
-    relenting            relent
-    relents              relent
-    reliances            relianc
-    relics               relic
-    relief               relief
-    reliev               reliev
-    relieve              reliev
-    relieved             reliev
-    relieves             reliev
-    relieving            reliev
-    religion             religion
-    religions            religion
-    religious            religi
-    religiously          religi
-    relinquish           relinquish
-    reliques             reliqu
-    reliquit             reliquit
-    relish               relish
-    relume               relum
-    rely                 reli
-    relying              reli
-    remain               remain
-    remainder            remaind
-    remainders           remaind
-    remained             remain
-    remaineth            remaineth
-    remaining            remain
-    remains              remain
-    remark               remark
-    remarkable           remark
-    remediate            remedi
-    remedied             remedi
-    remedies             remedi
-    remedy               remedi
-    rememb               rememb
-    remember             rememb
-    remembered           rememb
-    remembers            rememb
-    remembrance          remembr
-    remembrancer         remembranc
-    remembrances         remembr
-    remercimens          remercimen
-    remiss               remiss
-    remission            remiss
-    remissness           remiss
-    remit                remit
-    remnant              remnant
-    remnants             remnant
-    remonstrance         remonstr
-    remorse              remors
-    remorseful           remors
-    remorseless          remorseless
-    remote               remot
-    remotion             remot
-    remov                remov
-    remove               remov
-    removed              remov
-    removedness          removed
-    remover              remov
-    removes              remov
-    removing             remov
-    remunerate           remuner
-    remuneration         remuner
-    rence                renc
-    rend                 rend
-    render               render
-    rendered             render
-    renders              render
-    rendezvous           rendezv
-    renegado             renegado
-    renege               reneg
-    reneges              reneg
-    renew                renew
-    renewed              renew
-    renewest             renewest
-    renounce             renounc
-    renouncement         renounc
-    renouncing           renounc
-    renowmed             renowm
-    renown               renown
-    renowned             renown
-    rent                 rent
-    rents                rent
-    repaid               repaid
-    repair               repair
-    repaired             repair
-    repairing            repair
-    repairs              repair
-    repass               repass
-    repast               repast
-    repasture            repastur
-    repay                repai
-    repaying             repai
-    repays               repai
-    repeal               repeal
-    repealing            repeal
-    repeals              repeal
-    repeat               repeat
-    repeated             repeat
-    repeating            repeat
-    repeats              repeat
-    repel                repel
-    repent               repent
-    repentance           repent
-    repentant            repent
-    repented             repent
-    repenting            repent
-    repents              repent
-    repetition           repetit
-    repetitions          repetit
-    repin                repin
-    repine               repin
-    repining             repin
-    replant              replant
-    replenish            replenish
-    replenished          replenish
-    replete              replet
-    replication          replic
-    replied              repli
-    replies              repli
-    repliest             repliest
-    reply                repli
-    replying             repli
-    report               report
-    reported             report
-    reporter             report
-    reportest            reportest
-    reporting            report
-    reportingly          reportingli
-    reports              report
-    reposal              repos
-    repose               repos
-    reposeth             reposeth
-    reposing             repos
-    repossess            repossess
-    reprehend            reprehend
-    reprehended          reprehend
-    reprehending         reprehend
-    represent            repres
-    representing         repres
-    reprieve             repriev
-    reprieves            repriev
-    reprisal             repris
-    reproach             reproach
-    reproaches           reproach
-    reproachful          reproach
-    reproachfully        reproachfulli
-    reprobate            reprob
-    reprobation          reprob
-    reproof              reproof
-    reprov               reprov
-    reprove              reprov
-    reproveable          reprov
-    reproves             reprov
-    reproving            reprov
-    repugn               repugn
-    repugnancy           repugn
-    repugnant            repugn
-    repulse              repuls
-    repulsed             repuls
-    repurchas            repurcha
-    repured              repur
-    reputation           reput
-    repute               reput
-    reputed              reput
-    reputeless           reputeless
-    reputes              reput
-    reputing             reput
-    request              request
-    requested            request
-    requesting           request
-    requests             request
-    requiem              requiem
-    requir               requir
-    require              requir
-    required             requir
-    requires             requir
-    requireth            requireth
-    requiring            requir
-    requisite            requisit
-    requisites           requisit
-    requit               requit
-    requital             requit
-    requite              requit
-    requited             requit
-    requites             requit
-    rer                  rer
-    rere                 rere
-    rers                 rer
-    rescu                rescu
-    rescue               rescu
-    rescued              rescu
-    rescues              rescu
-    rescuing             rescu
-    resemblance          resembl
-    resemble             resembl
-    resembled            resembl
-    resembles            resembl
-    resembleth           resembleth
-    resembling           resembl
-    reserv               reserv
-    reservation          reserv
-    reserve              reserv
-    reserved             reserv
-    reserves             reserv
-    reside               resid
-    residence            resid
-    resident             resid
-    resides              resid
-    residing             resid
-    residue              residu
-    resign               resign
-    resignation          resign
-    resist               resist
-    resistance           resist
-    resisted             resist
-    resisting            resist
-    resists              resist
-    resolute             resolut
-    resolutely           resolut
-    resolutes            resolut
-    resolution           resolut
-    resolv               resolv
-    resolve              resolv
-    resolved             resolv
-    resolvedly           resolvedli
-    resolves             resolv
-    resolveth            resolveth
-    resort               resort
-    resorted             resort
-    resounding           resound
-    resounds             resound
-    respeaking           respeak
-    respect              respect
-    respected            respect
-    respecting           respect
-    respective           respect
-    respectively         respect
-    respects             respect
-    respice              respic
-    respite              respit
-    respites             respit
-    responsive           respons
-    respose              respos
-    ress                 ress
-    rest                 rest
-    rested               rest
-    resteth              resteth
-    restful              rest
-    resting              rest
-    restitution          restitut
-    restless             restless
-    restor               restor
-    restoration          restor
-    restorative          restor
-    restore              restor
-    restored             restor
-    restores             restor
-    restoring            restor
-    restrain             restrain
-    restrained           restrain
-    restraining          restrain
-    restrains            restrain
-    restraint            restraint
-    rests                rest
-    resty                resti
-    resum                resum
-    resume               resum
-    resumes              resum
-    resurrections        resurrect
-    retail               retail
-    retails              retail
-    retain               retain
-    retainers            retain
-    retaining            retain
-    retell               retel
-    retention            retent
-    retentive            retent
-    retinue              retinu
-    retir                retir
-    retire               retir
-    retired              retir
-    retirement           retir
-    retires              retir
-    retiring             retir
-    retold               retold
-    retort               retort
-    retorts              retort
-    retourne             retourn
-    retract              retract
-    retreat              retreat
-    retrograde           retrograd
-    rets                 ret
-    return               return
-    returned             return
-    returnest            returnest
-    returneth            returneth
-    returning            return
-    returns              return
-    revania              revania
-    reveal               reveal
-    reveals              reveal
-    revel                revel
-    reveler              revel
-    revell               revel
-    reveller             revel
-    revellers            revel
-    revelling            revel
-    revelry              revelri
-    revels               revel
-    reveng               reveng
-    revenge              reveng
-    revenged             reveng
-    revengeful           reveng
-    revengement          reveng
-    revenger             reveng
-    revengers            reveng
-    revenges             reveng
-    revenging            reveng
-    revengingly          revengingli
-    revenue              revenu
-    revenues             revenu
-    reverb               reverb
-    reverberate          reverber
-    reverbs              reverb
-    reverenc             reverenc
-    reverence            rever
-    reverend             reverend
-    reverent             rever
-    reverently           rever
-    revers               rever
-    reverse              revers
-    reversion            revers
-    reverted             revert
-    review               review
-    reviewest            reviewest
-    revil                revil
-    revile               revil
-    revisits             revisit
-    reviv                reviv
-    revive               reviv
-    revives              reviv
-    reviving             reviv
-    revok                revok
-    revoke               revok
-    revokement           revok
-    revolt               revolt
-    revolted             revolt
-    revolting            revolt
-    revolts              revolt
-    revolution           revolut
-    revolutions          revolut
-    revolve              revolv
-    revolving            revolv
-    reward               reward
-    rewarded             reward
-    rewarder             reward
-    rewarding            reward
-    rewards              reward
-    reword               reword
-    reworded             reword
-    rex                  rex
-    rey                  rei
-    reynaldo             reynaldo
-    rford                rford
-    rful                 rful
-    rfull                rfull
-    rhapsody             rhapsodi
-    rheims               rheim
-    rhenish              rhenish
-    rhesus               rhesu
-    rhetoric             rhetor
-    rheum                rheum
-    rheumatic            rheumat
-    rheums               rheum
-    rheumy               rheumi
-    rhinoceros           rhinocero
-    rhodes               rhode
-    rhodope              rhodop
-    rhubarb              rhubarb
-    rhym                 rhym
-    rhyme                rhyme
-    rhymers              rhymer
-    rhymes               rhyme
-    rhyming              rhyme
-    rialto               rialto
-    rib                  rib
-    ribald               ribald
-    riband               riband
-    ribands              riband
-    ribaudred            ribaudr
-    ribb                 ribb
-    ribbed               rib
-    ribbon               ribbon
-    ribbons              ribbon
-    ribs                 rib
-    rice                 rice
-    rich                 rich
-    richard              richard
-    richer               richer
-    riches               rich
-    richest              richest
-    richly               richli
-    richmond             richmond
-    richmonds            richmond
-    rid                  rid
-    riddance             riddanc
-    ridden               ridden
-    riddle               riddl
-    riddles              riddl
-    riddling             riddl
-    ride                 ride
-    rider                rider
-    riders               rider
-    rides                ride
-    ridest               ridest
-    rideth               rideth
-    ridge                ridg
-    ridges               ridg
-    ridiculous           ridicul
-    riding               ride
-    rids                 rid
-    rien                 rien
-    ries                 ri
-    rifle                rifl
-    rift                 rift
-    rifted               rift
-    rig                  rig
-    rigg                 rigg
-    riggish              riggish
-    right                right
-    righteous            righteou
-    righteously          righteous
-    rightful             right
-    rightfully           rightfulli
-    rightly              rightli
-    rights               right
-    rigol                rigol
-    rigorous             rigor
-    rigorously           rigor
-    rigour               rigour
-    ril                  ril
-    rim                  rim
-    rin                  rin
-    rinaldo              rinaldo
-    rind                 rind
-    ring                 ring
-    ringing              ring
-    ringleader           ringlead
-    ringlets             ringlet
-    rings                ring
-    ringwood             ringwood
-    riot                 riot
-    rioter               rioter
-    rioting              riot
-    riotous              riotou
-    riots                riot
-    rip                  rip
-    ripe                 ripe
-    ripely               ripe
-    ripen                ripen
-    ripened              ripen
-    ripeness             ripe
-    ripening             ripen
-    ripens               ripen
-    riper                riper
-    ripest               ripest
-    riping               ripe
-    ripp                 ripp
-    ripping              rip
-    rise                 rise
-    risen                risen
-    rises                rise
-    riseth               riseth
-    rish                 rish
-    rising               rise
-    rite                 rite
-    rites                rite
-    rivage               rivag
-    rival                rival
-    rivality             rival
-    rivall               rival
-    rivals               rival
-    rive                 rive
-    rived                rive
-    rivelled             rivel
-    river                river
-    rivers               river
-    rivet                rivet
-    riveted              rivet
-    rivets               rivet
-    rivo                 rivo
-    rj                   rj
-    rless                rless
-    road                 road
-    roads                road
-    roam                 roam
-    roaming              roam
-    roan                 roan
-    roar                 roar
-    roared               roar
-    roarers              roarer
-    roaring              roar
-    roars                roar
-    roast                roast
-    roasted              roast
-    rob                  rob
-    roba                 roba
-    robas                roba
-    robb                 robb
-    robbed               rob
-    robber               robber
-    robbers              robber
-    robbery              robberi
-    robbing              rob
-    robe                 robe
-    robed                robe
-    robert               robert
-    robes                robe
-    robin                robin
-    robs                 rob
-    robustious           robusti
-    rochester            rochest
-    rochford             rochford
-    rock                 rock
-    rocks                rock
-    rocky                rocki
-    rod                  rod
-    rode                 rode
-    roderigo             roderigo
-    rods                 rod
-    roe                  roe
-    roes                 roe
-    roger                roger
-    rogero               rogero
-    rogue                rogu
-    roguery              rogueri
-    rogues               rogu
-    roguish              roguish
-    roi                  roi
-    roisting             roist
-    roll                 roll
-    rolled               roll
-    rolling              roll
-    rolls                roll
-    rom                  rom
-    romage               romag
-    roman                roman
-    romano               romano
-    romanos              romano
-    romans               roman
-    rome                 rome
-    romeo                romeo
-    romish               romish
-    rondure              rondur
-    ronyon               ronyon
-    rood                 rood
-    roof                 roof
-    roofs                roof
-    rook                 rook
-    rooks                rook
-    rooky                rooki
-    room                 room
-    rooms                room
-    root                 root
-    rooted               root
-    rootedly             rootedli
-    rooteth              rooteth
-    rooting              root
-    roots                root
-    rope                 rope
-    ropery               roperi
-    ropes                rope
-    roping               rope
-    ros                  ro
-    rosalind             rosalind
-    rosalinda            rosalinda
-    rosalinde            rosalind
-    rosaline             rosalin
-    roscius              rosciu
-    rose                 rose
-    rosed                rose
-    rosemary             rosemari
-    rosencrantz          rosencrantz
-    roses                rose
-    ross                 ross
-    rosy                 rosi
-    rot                  rot
-    rote                 rote
-    roted                rote
-    rother               rother
-    rotherham            rotherham
-    rots                 rot
-    rotted               rot
-    rotten               rotten
-    rottenness           rotten
-    rotting              rot
-    rotundity            rotund
-    rouen                rouen
-    rough                rough
-    rougher              rougher
-    roughest             roughest
-    roughly              roughli
-    roughness            rough
-    round                round
-    rounded              round
-    roundel              roundel
-    rounder              rounder
-    roundest             roundest
-    rounding             round
-    roundly              roundli
-    rounds               round
-    roundure             roundur
-    rous                 rou
-    rouse                rous
-    roused               rous
-    rousillon            rousillon
-    rously               rousli
-    roussi               roussi
-    rout                 rout
-    routed               rout
-    routs                rout
-    rove                 rove
-    rover                rover
-    row                  row
-    rowel                rowel
-    rowland              rowland
-    rowlands             rowland
-    roy                  roi
-    royal                royal
-    royalize             royal
-    royally              royal
-    royalties            royalti
-    royalty              royalti
-    roynish              roynish
-    rs                   rs
-    rt                   rt
-    rub                  rub
-    rubb                 rubb
-    rubbing              rub
-    rubbish              rubbish
-    rubies               rubi
-    rubious              rubiou
-    rubs                 rub
-    ruby                 rubi
-    rud                  rud
-    rudand               rudand
-    rudder               rudder
-    ruddiness            ruddi
-    ruddock              ruddock
-    ruddy                ruddi
-    rude                 rude
-    rudely               rude
-    rudeness             rude
-    ruder                ruder
-    rudesby              rudesbi
-    rudest               rudest
-    rudiments            rudiment
-    rue                  rue
-    rued                 ru
-    ruff                 ruff
-    ruffian              ruffian
-    ruffians             ruffian
-    ruffle               ruffl
-    ruffling             ruffl
-    ruffs                ruff
-    rug                  rug
-    rugby                rugbi
-    rugemount            rugemount
-    rugged               rug
-    ruin                 ruin
-    ruinate              ruinat
-    ruined               ruin
-    ruining              ruin
-    ruinous              ruinou
-    ruins                ruin
-    rul                  rul
-    rule                 rule
-    ruled                rule
-    ruler                ruler
-    rulers               ruler
-    rules                rule
-    ruling               rule
-    rumble               rumbl
-    ruminaies            ruminai
-    ruminat              ruminat
-    ruminate             rumin
-    ruminated            rumin
-    ruminates            rumin
-    rumination           rumin
-    rumor                rumor
-    rumour               rumour
-    rumourer             rumour
-    rumours              rumour
-    rump                 rump
-    run                  run
-    runagate             runag
-    runagates            runag
-    runaway              runawai
-    runaways             runawai
-    rung                 rung
-    runn                 runn
-    runner               runner
-    runners              runner
-    running              run
-    runs                 run
-    rupture              ruptur
-    ruptures             ruptur
-    rural                rural
-    rush                 rush
-    rushes               rush
-    rushing              rush
-    rushling             rushl
-    rushy                rushi
-    russet               russet
-    russia               russia
-    russian              russian
-    russians             russian
-    rust                 rust
-    rusted               rust
-    rustic               rustic
-    rustically           rustic
-    rustics              rustic
-    rustle               rustl
-    rustling             rustl
-    rusts                rust
-    rusty                rusti
-    rut                  rut
-    ruth                 ruth
-    ruthful              ruth
-    ruthless             ruthless
-    rutland              rutland
-    ruttish              ruttish
-    ry                   ry
-    rye                  rye
-    rything              ryth
-    s                    s
-    sa                   sa
-    saba                 saba
-    sabbath              sabbath
-    sable                sabl
-    sables               sabl
-    sack                 sack
-    sackbuts             sackbut
-    sackcloth            sackcloth
-    sacked               sack
-    sackerson            sackerson
-    sacks                sack
-    sacrament            sacrament
-    sacred               sacr
-    sacrific             sacrif
-    sacrifice            sacrific
-    sacrificers          sacrific
-    sacrifices           sacrific
-    sacrificial          sacrifici
-    sacrificing          sacrif
-    sacrilegious         sacrilegi
-    sacring              sacr
-    sad                  sad
-    sadder               sadder
-    saddest              saddest
-    saddle               saddl
-    saddler              saddler
-    saddles              saddl
-    sadly                sadli
-    sadness              sad
-    saf                  saf
-    safe                 safe
-    safeguard            safeguard
-    safely               safe
-    safer                safer
-    safest               safest
-    safeties             safeti
-    safety               safeti
-    saffron              saffron
-    sag                  sag
-    sage                 sage
-    sagittary            sagittari
-    said                 said
-    saidst               saidst
-    sail                 sail
-    sailing              sail
-    sailmaker            sailmak
-    sailor               sailor
-    sailors              sailor
-    sails                sail
-    sain                 sain
-    saint                saint
-    sainted              saint
-    saintlike            saintlik
-    saints               saint
-    saith                saith
-    sake                 sake
-    sakes                sake
-    sala                 sala
-    salad                salad
-    salamander           salamand
-    salary               salari
-    sale                 sale
-    salerio              salerio
-    salicam              salicam
-    salique              saliqu
-    salisbury            salisburi
-    sall                 sall
-    sallet               sallet
-    sallets              sallet
-    sallies              salli
-    sallow               sallow
-    sally                salli
-    salmon               salmon
-    salmons              salmon
-    salt                 salt
-    salter               salter
-    saltiers             saltier
-    saltness             salt
-    saltpetre            saltpetr
-    salutation           salut
-    salutations          salut
-    salute               salut
-    saluted              salut
-    salutes              salut
-    saluteth             saluteth
-    salv                 salv
-    salvation            salvat
-    salve                salv
-    salving              salv
-    same                 same
-    samingo              samingo
-    samp                 samp
-    sampire              sampir
-    sample               sampl
-    sampler              sampler
-    sampson              sampson
-    samson               samson
-    samsons              samson
-    sancta               sancta
-    sanctified           sanctifi
-    sanctifies           sanctifi
-    sanctify             sanctifi
-    sanctimonies         sanctimoni
-    sanctimonious        sanctimoni
-    sanctimony           sanctimoni
-    sanctities           sanctiti
-    sanctity             sanctiti
-    sanctuarize          sanctuar
-    sanctuary            sanctuari
-    sand                 sand
-    sandal               sandal
-    sandbag              sandbag
-    sanded               sand
-    sands                sand
-    sandy                sandi
-    sandys               sandi
-    sang                 sang
-    sanguine             sanguin
-    sanguis              sangui
-    sanity               saniti
-    sans                 san
-    santrailles          santrail
-    sap                  sap
-    sapient              sapient
-    sapit                sapit
-    sapless              sapless
-    sapling              sapl
-    sapphire             sapphir
-    sapphires            sapphir
-    saracens             saracen
-    sarcenet             sarcenet
-    sard                 sard
-    sardians             sardian
-    sardinia             sardinia
-    sardis               sardi
-    sarum                sarum
-    sat                  sat
-    satan                satan
-    satchel              satchel
-    sate                 sate
-    sated                sate
-    satiate              satiat
-    satiety              satieti
-    satin                satin
-    satire               satir
-    satirical            satir
-    satis                sati
-    satisfaction         satisfact
-    satisfied            satisfi
-    satisfies            satisfi
-    satisfy              satisfi
-    satisfying           satisfi
-    saturday             saturdai
-    saturdays            saturdai
-    saturn               saturn
-    saturnine            saturnin
-    saturninus           saturninu
-    satyr                satyr
-    satyrs               satyr
-    sauc                 sauc
-    sauce                sauc
-    sauced               sauc
-    saucers              saucer
-    sauces               sauc
-    saucily              saucili
-    sauciness            sauci
-    saucy                sauci
-    sauf                 sauf
-    saunder              saunder
-    sav                  sav
-    savage               savag
-    savagely             savag
-    savageness           savag
-    savagery             savageri
-    savages              savag
-    save                 save
-    saved                save
-    saves                save
-    saving               save
-    saviour              saviour
-    savory               savori
-    savour               savour
-    savouring            savour
-    savours              savour
-    savoury              savouri
-    savoy                savoi
-    saw                  saw
-    sawed                saw
-    sawest               sawest
-    sawn                 sawn
-    sawpit               sawpit
-    saws                 saw
-    sawyer               sawyer
-    saxons               saxon
-    saxony               saxoni
-    saxton               saxton
-    say                  sai
-    sayest               sayest
-    saying               sai
-    sayings              sai
-    says                 sai
-    sayst                sayst
-    sblood               sblood
-    sc                   sc
-    scab                 scab
-    scabbard             scabbard
-    scabs                scab
-    scaffold             scaffold
-    scaffoldage          scaffoldag
-    scal                 scal
-    scald                scald
-    scalded              scald
-    scalding             scald
-    scale                scale
-    scaled               scale
-    scales               scale
-    scaling              scale
-    scall                scall
-    scalp                scalp
-    scalps               scalp
-    scaly                scali
-    scamble              scambl
-    scambling            scambl
-    scamels              scamel
-    scan                 scan
-    scandal              scandal
-    scandaliz            scandaliz
-    scandalous           scandal
-    scandy               scandi
-    scann                scann
-    scant                scant
-    scanted              scant
-    scanter              scanter
-    scanting             scant
-    scantling            scantl
-    scants               scant
-    scap                 scap
-    scape                scape
-    scaped               scape
-    scapes               scape
-    scapeth              scapeth
-    scar                 scar
-    scarce               scarc
-    scarcely             scarc
-    scarcity             scarciti
-    scare                scare
-    scarecrow            scarecrow
-    scarecrows           scarecrow
-    scarf                scarf
-    scarfed              scarf
-    scarfs               scarf
-    scaring              scare
-    scarlet              scarlet
-    scarr                scarr
-    scarre               scarr
-    scars                scar
-    scarus               scaru
-    scath                scath
-    scathe               scath
-    scathful             scath
-    scatt                scatt
-    scatter              scatter
-    scattered            scatter
-    scattering           scatter
-    scatters             scatter
-    scelera              scelera
-    scelerisque          scelerisqu
-    scene                scene
-    scenes               scene
-    scent                scent
-    scented              scent
-    scept                scept
-    scepter              scepter
-    sceptre              sceptr
-    sceptred             sceptr
-    sceptres             sceptr
-    schedule             schedul
-    schedules            schedul
-    scholar              scholar
-    scholarly            scholarli
-    scholars             scholar
-    school               school
-    schoolboy            schoolboi
-    schoolboys           schoolboi
-    schoolfellows        schoolfellow
-    schooling            school
-    schoolmaster         schoolmast
-    schoolmasters        schoolmast
-    schools              school
-    sciatica             sciatica
-    sciaticas            sciatica
-    science              scienc
-    sciences             scienc
-    scimitar             scimitar
-    scion                scion
-    scions               scion
-    scissors             scissor
-    scoff                scoff
-    scoffer              scoffer
-    scoffing             scof
-    scoffs               scoff
-    scoggin              scoggin
-    scold                scold
-    scolding             scold
-    scolds               scold
-    sconce               sconc
-    scone                scone
-    scope                scope
-    scopes               scope
-    scorch               scorch
-    scorched             scorch
-    score                score
-    scored               score
-    scores               score
-    scoring              score
-    scorn                scorn
-    scorned              scorn
-    scornful             scorn
-    scornfully           scornfulli
-    scorning             scorn
-    scorns               scorn
-    scorpion             scorpion
-    scorpions            scorpion
-    scot                 scot
-    scotch               scotch
-    scotches             scotch
-    scotland             scotland
-    scots                scot
-    scottish             scottish
-    scoundrels           scoundrel
-    scour                scour
-    scoured              scour
-    scourg               scourg
-    scourge              scourg
-    scouring             scour
-    scout                scout
-    scouts               scout
-    scowl                scowl
-    scrap                scrap
-    scrape               scrape
-    scraping             scrape
-    scraps               scrap
-    scratch              scratch
-    scratches            scratch
-    scratching           scratch
-    scream               scream
-    screams              scream
-    screech              screech
-    screeching           screech
-    screen               screen
-    screens              screen
-    screw                screw
-    screws               screw
-    scribbl              scribbl
-    scribbled            scribbl
-    scribe               scribe
-    scribes              scribe
-    scrimers             scrimer
-    scrip                scrip
-    scrippage            scrippag
-    scripture            scriptur
-    scriptures           scriptur
-    scrivener            scriven
-    scroll               scroll
-    scrolls              scroll
-    scroop               scroop
-    scrowl               scrowl
-    scroyles             scroyl
-    scrubbed             scrub
-    scruple              scrupl
-    scruples             scrupl
-    scrupulous           scrupul
-    scuffles             scuffl
-    scuffling            scuffl
-    scullion             scullion
-    sculls               scull
-    scum                 scum
-    scurril              scurril
-    scurrility           scurril
-    scurrilous           scurril
-    scurvy               scurvi
-    scuse                scuse
-    scut                 scut
-    scutcheon            scutcheon
-    scutcheons           scutcheon
-    scylla               scylla
-    scythe               scyth
-    scythed              scyth
-    scythia              scythia
-    scythian             scythian
-    sdeath               sdeath
-    se                   se
-    sea                  sea
-    seacoal              seacoal
-    seafaring            seafar
-    seal                 seal
-    sealed               seal
-    sealing              seal
-    seals                seal
-    seam                 seam
-    seamen               seamen
-    seamy                seami
-    seaport              seaport
-    sear                 sear
-    searce               searc
-    search               search
-    searchers            searcher
-    searches             search
-    searcheth            searcheth
-    searching            search
-    seared               sear
-    seas                 sea
-    seasick              seasick
-    seaside              seasid
-    season               season
-    seasoned             season
-    seasons              season
-    seat                 seat
-    seated               seat
-    seats                seat
-    sebastian            sebastian
-    second               second
-    secondarily          secondarili
-    secondary            secondari
-    seconded             second
-    seconds              second
-    secrecy              secreci
-    secret               secret
-    secretaries          secretari
-    secretary            secretari
-    secretly             secretli
-    secrets              secret
-    sect                 sect
-    sectary              sectari
-    sects                sect
-    secundo              secundo
-    secure               secur
-    securely             secur
-    securing             secur
-    security             secur
-    sedg                 sedg
-    sedge                sedg
-    sedges               sedg
-    sedgy                sedgi
-    sedition             sedit
-    seditious            sediti
-    seduc                seduc
-    seduce               seduc
-    seduced              seduc
-    seducer              seduc
-    seducing             seduc
-    see                  see
-    seed                 seed
-    seeded               seed
-    seedness             seed
-    seeds                seed
-    seedsman             seedsman
-    seein                seein
-    seeing               see
-    seek                 seek
-    seeking              seek
-    seeks                seek
-    seel                 seel
-    seeling              seel
-    seely                seeli
-    seem                 seem
-    seemed               seem
-    seemers              seemer
-    seemest              seemest
-    seemeth              seemeth
-    seeming              seem
-    seemingly            seemingli
-    seemly               seemli
-    seems                seem
-    seen                 seen
-    seer                 seer
-    sees                 see
-    seese                sees
-    seest                seest
-    seethe               seeth
-    seethes              seeth
-    seething             seeth
-    seeting              seet
-    segregation          segreg
-    seigneur             seigneur
-    seigneurs            seigneur
-    seiz                 seiz
-    seize                seiz
-    seized               seiz
-    seizes               seiz
-    seizeth              seizeth
-    seizing              seiz
-    seizure              seizur
-    seld                 seld
-    seldom               seldom
-    select               select
-    seleucus             seleucu
-    self                 self
-    selfsame             selfsam
-    sell                 sell
-    seller               seller
-    selling              sell
-    sells                sell
-    selves               selv
-    semblable            semblabl
-    semblably            semblabl
-    semblance            semblanc
-    semblances           semblanc
-    semblative           sembl
-    semi                 semi
-    semicircle           semicircl
-    semiramis            semirami
-    semper               semper
-    sempronius           semproniu
-    senate               senat
-    senator              senat
-    senators             senat
-    send                 send
-    sender               sender
-    sendeth              sendeth
-    sending              send
-    sends                send
-    seneca               seneca
-    senior               senior
-    seniory              seniori
-    senis                seni
-    sennet               sennet
-    senoys               senoi
-    sense                sens
-    senseless            senseless
-    senses               sens
-    sensible             sensibl
-    sensibly             sensibl
-    sensual              sensual
-    sensuality           sensual
-    sent                 sent
-    sentenc              sentenc
-    sentence             sentenc
-    sentences            sentenc
-    sententious          sententi
-    sentinel             sentinel
-    sentinels            sentinel
-    separable            separ
-    separate             separ
-    separated            separ
-    separates            separ
-    separation           separ
-    septentrion          septentrion
-    sepulchre            sepulchr
-    sepulchres           sepulchr
-    sepulchring          sepulchr
-    sequel               sequel
-    sequence             sequenc
-    sequent              sequent
-    sequest              sequest
-    sequester            sequest
-    sequestration        sequestr
-    sere                 sere
-    serenis              sereni
-    serge                serg
-    sergeant             sergeant
-    serious              seriou
-    seriously            serious
-    sermon               sermon
-    sermons              sermon
-    serpent              serpent
-    serpentine           serpentin
-    serpents             serpent
-    serpigo              serpigo
-    serv                 serv
-    servant              servant
-    servanted            servant
-    servants             servant
-    serve                serv
-    served               serv
-    server               server
-    serves               serv
-    serveth              serveth
-    service              servic
-    serviceable          servic
-    services             servic
-    servile              servil
-    servility            servil
-    servilius            serviliu
-    serving              serv
-    servingman           servingman
-    servingmen           servingmen
-    serviteur            serviteur
-    servitor             servitor
-    servitors            servitor
-    servitude            servitud
-    sessa                sessa
-    session              session
-    sessions             session
-    sestos               sesto
-    set                  set
-    setebos              setebo
-    sets                 set
-    setter               setter
-    setting              set
-    settle               settl
-    settled              settl
-    settlest             settlest
-    settling             settl
-    sev                  sev
-    seven                seven
-    sevenfold            sevenfold
-    sevennight           sevennight
-    seventeen            seventeen
-    seventh              seventh
-    seventy              seventi
-    sever                sever
-    several              sever
-    severally            sever
-    severals             sever
-    severe               sever
-    severed              sever
-    severely             sever
-    severest             severest
-    severing             sever
-    severity             sever
-    severn               severn
-    severs               sever
-    sew                  sew
-    seward               seward
-    sewer                sewer
-    sewing               sew
-    sex                  sex
-    sexes                sex
-    sexton               sexton
-    sextus               sextu
-    seymour              seymour
-    seyton               seyton
-    sfoot                sfoot
-    sh                   sh
-    shackle              shackl
-    shackles             shackl
-    shade                shade
-    shades               shade
-    shadow               shadow
-    shadowed             shadow
-    shadowing            shadow
-    shadows              shadow
-    shadowy              shadowi
-    shady                shadi
-    shafalus             shafalu
-    shaft                shaft
-    shafts               shaft
-    shag                 shag
-    shak                 shak
-    shake                shake
-    shaked               shake
-    shaken               shaken
-    shakes               shake
-    shaking              shake
-    shales               shale
-    shall                shall
-    shallenge            shalleng
-    shallow              shallow
-    shallowest           shallowest
-    shallowly            shallowli
-    shallows             shallow
-    shalt                shalt
-    sham                 sham
-    shambles             shambl
-    shame                shame
-    shamed               shame
-    shameful             shame
-    shamefully           shamefulli
-    shameless            shameless
-    shames               shame
-    shamest              shamest
-    shaming              shame
-    shank                shank
-    shanks               shank
-    shap                 shap
-    shape                shape
-    shaped               shape
-    shapeless            shapeless
-    shapen               shapen
-    shapes               shape
-    shaping              shape
-    shar                 shar
-    shard                shard
-    sharded              shard
-    shards               shard
-    share                share
-    shared               share
-    sharers              sharer
-    shares               share
-    sharing              share
-    shark                shark
-    sharp                sharp
-    sharpen              sharpen
-    sharpened            sharpen
-    sharpens             sharpen
-    sharper              sharper
-    sharpest             sharpest
-    sharply              sharpli
-    sharpness            sharp
-    sharps               sharp
-    shatter              shatter
-    shav                 shav
-    shave                shave
-    shaven               shaven
-    shaw                 shaw
-    she                  she
-    sheaf                sheaf
-    sheal                sheal
-    shear                shear
-    shearers             shearer
-    shearing             shear
-    shearman             shearman
-    shears               shear
-    sheath               sheath
-    sheathe              sheath
-    sheathed             sheath
-    sheathes             sheath
-    sheathing            sheath
-    sheaved              sheav
-    sheaves              sheav
-    shed                 shed
-    shedding             shed
-    sheds                shed
-    sheen                sheen
-    sheep                sheep
-    sheepcote            sheepcot
-    sheepcotes           sheepcot
-    sheeps               sheep
-    sheepskins           sheepskin
-    sheer                sheer
-    sheet                sheet
-    sheeted              sheet
-    sheets               sheet
-    sheffield            sheffield
-    shelf                shelf
-    shell                shell
-    shells               shell
-    shelt                shelt
-    shelter              shelter
-    shelters             shelter
-    shelves              shelv
-    shelving             shelv
-    shelvy               shelvi
-    shent                shent
-    shepherd             shepherd
-    shepherdes           shepherd
-    shepherdess          shepherdess
-    shepherdesses        shepherdess
-    shepherds            shepherd
-    sher                 sher
-    sheriff              sheriff
-    sherris              sherri
-    shes                 she
-    sheweth              sheweth
-    shield               shield
-    shielded             shield
-    shields              shield
-    shift                shift
-    shifted              shift
-    shifting             shift
-    shifts               shift
-    shilling             shill
-    shillings            shill
-    shin                 shin
-    shine                shine
-    shines               shine
-    shineth              shineth
-    shining              shine
-    shins                shin
-    shiny                shini
-    ship                 ship
-    shipboard            shipboard
-    shipman              shipman
-    shipmaster           shipmast
-    shipmen              shipmen
-    shipp                shipp
-    shipped              ship
-    shipping             ship
-    ships                ship
-    shipt                shipt
-    shipwreck            shipwreck
-    shipwrecking         shipwreck
-    shipwright           shipwright
-    shipwrights          shipwright
-    shire                shire
-    shirley              shirlei
-    shirt                shirt
-    shirts               shirt
-    shive                shive
-    shiver               shiver
-    shivering            shiver
-    shivers              shiver
-    shoal                shoal
-    shoals               shoal
-    shock                shock
-    shocks               shock
-    shod                 shod
-    shoe                 shoe
-    shoeing              shoe
-    shoemaker            shoemak
-    shoes                shoe
-    shog                 shog
-    shone                shone
-    shook                shook
-    shoon                shoon
-    shoot                shoot
-    shooter              shooter
-    shootie              shooti
-    shooting             shoot
-    shoots               shoot
-    shop                 shop
-    shops                shop
-    shore                shore
-    shores               shore
-    shorn                shorn
-    short                short
-    shortcake            shortcak
-    shorten              shorten
-    shortened            shorten
-    shortens             shorten
-    shorter              shorter
-    shortly              shortli
-    shortness            short
-    shot                 shot
-    shotten              shotten
-    shoughs              shough
-    should               should
-    shoulder             shoulder
-    shouldering          shoulder
-    shoulders            shoulder
-    shouldst             shouldst
-    shout                shout
-    shouted              shout
-    shouting             shout
-    shouts               shout
-    shov                 shov
-    shove                shove
-    shovel               shovel
-    shovels              shovel
-    show                 show
-    showed               show
-    shower               shower
-    showers              shower
-    showest              showest
-    showing              show
-    shown                shown
-    shows                show
-    shreds               shred
-    shrew                shrew
-    shrewd               shrewd
-    shrewdly             shrewdli
-    shrewdness           shrewd
-    shrewish             shrewish
-    shrewishly           shrewishli
-    shrewishness         shrewish
-    shrews               shrew
-    shrewsbury           shrewsburi
-    shriek               shriek
-    shrieking            shriek
-    shrieks              shriek
-    shrieve              shriev
-    shrift               shrift
-    shrill               shrill
-    shriller             shriller
-    shrills              shrill
-    shrilly              shrilli
-    shrimp               shrimp
-    shrine               shrine
-    shrink               shrink
-    shrinking            shrink
-    shrinks              shrink
-    shriv                shriv
-    shrive               shrive
-    shriver              shriver
-    shrives              shrive
-    shriving             shrive
-    shroud               shroud
-    shrouded             shroud
-    shrouding            shroud
-    shrouds              shroud
-    shrove               shrove
-    shrow                shrow
-    shrows               shrow
-    shrub                shrub
-    shrubs               shrub
-    shrug                shrug
-    shrugs               shrug
-    shrunk               shrunk
-    shudd                shudd
-    shudders             shudder
-    shuffl               shuffl
-    shuffle              shuffl
-    shuffled             shuffl
-    shuffling            shuffl
-    shun                 shun
-    shunless             shunless
-    shunn                shunn
-    shunned              shun
-    shunning             shun
-    shuns                shun
-    shut                 shut
-    shuts                shut
-    shuttle              shuttl
-    shy                  shy
-    shylock              shylock
-    si                   si
-    sibyl                sibyl
-    sibylla              sibylla
-    sibyls               sibyl
-    sicil                sicil
-    sicilia              sicilia
-    sicilian             sicilian
-    sicilius             siciliu
-    sicils               sicil
-    sicily               sicili
-    sicinius             siciniu
-    sick                 sick
-    sicken               sicken
-    sickens              sicken
-    sicker               sicker
-    sickle               sickl
-    sicklemen            sicklemen
-    sicklied             sickli
-    sickliness           sickli
-    sickly               sickli
-    sickness             sick
-    sicles               sicl
-    sicyon               sicyon
-    side                 side
-    sided                side
-    sides                side
-    siege                sieg
-    sieges               sieg
-    sienna               sienna
-    sies                 si
-    sieve                siev
-    sift                 sift
-    sifted               sift
-    sigeia               sigeia
-    sigh                 sigh
-    sighed               sigh
-    sighing              sigh
-    sighs                sigh
-    sight                sight
-    sighted              sight
-    sightless            sightless
-    sightly              sightli
-    sights               sight
-    sign                 sign
-    signal               signal
-    signet               signet
-    signieur             signieur
-    significant          signific
-    significants         signific
-    signified            signifi
-    signifies            signifi
-    signify              signifi
-    signifying           signifi
-    signior              signior
-    signiories           signiori
-    signiors             signior
-    signiory             signiori
-    signor               signor
-    signories            signori
-    signs                sign
-    signum               signum
-    silenc               silenc
-    silence              silenc
-    silenced             silenc
-    silencing            silenc
-    silent               silent
-    silently             silent
-    silius               siliu
-    silk                 silk
-    silken               silken
-    silkman              silkman
-    silks                silk
-    silliest             silliest
-    silliness            silli
-    silling              sill
-    silly                silli
-    silva                silva
-    silver               silver
-    silvered             silver
-    silverly             silverli
-    silvia               silvia
-    silvius              silviu
-    sima                 sima
-    simile               simil
-    similes              simil
-    simois               simoi
-    simon                simon
-    simony               simoni
-    simp                 simp
-    simpcox              simpcox
-    simple               simpl
-    simpleness           simpl
-    simpler              simpler
-    simples              simpl
-    simplicity           simplic
-    simply               simpli
-    simular              simular
-    simulation           simul
-    sin                  sin
-    since                sinc
-    sincere              sincer
-    sincerely            sincer
-    sincerity            sincer
-    sinel                sinel
-    sinew                sinew
-    sinewed              sinew
-    sinews               sinew
-    sinewy               sinewi
-    sinful               sin
-    sinfully             sinfulli
-    sing                 sing
-    singe                sing
-    singeing             sing
-    singer               singer
-    singes               sing
-    singeth              singeth
-    singing              sing
-    single               singl
-    singled              singl
-    singleness           singl
-    singly               singli
-    sings                sing
-    singular             singular
-    singulariter         singularit
-    singularities        singular
-    singularity          singular
-    singuled             singul
-    sinister             sinist
-    sink                 sink
-    sinking              sink
-    sinks                sink
-    sinn                 sinn
-    sinner               sinner
-    sinners              sinner
-    sinning              sin
-    sinon                sinon
-    sins                 sin
-    sip                  sip
-    sipping              sip
-    sir                  sir
-    sire                 sire
-    siren                siren
-    sirrah               sirrah
-    sirs                 sir
-    sist                 sist
-    sister               sister
-    sisterhood           sisterhood
-    sisterly             sisterli
-    sisters              sister
-    sit                  sit
-    sith                 sith
-    sithence             sithenc
-    sits                 sit
-    sitting              sit
-    situate              situat
-    situation            situat
-    situations           situat
-    siward               siward
-    six                  six
-    sixpence             sixpenc
-    sixpences            sixpenc
-    sixpenny             sixpenni
-    sixteen              sixteen
-    sixth                sixth
-    sixty                sixti
-    siz                  siz
-    size                 size
-    sizes                size
-    sizzle               sizzl
-    skains               skain
-    skamble              skambl
-    skein                skein
-    skelter              skelter
-    skies                ski
-    skilful              skil
-    skilfully            skilfulli
-    skill                skill
-    skilless             skilless
-    skillet              skillet
-    skillful             skill
-    skills               skill
-    skim                 skim
-    skimble              skimbl
-    skin                 skin
-    skinker              skinker
-    skinny               skinni
-    skins                skin
-    skip                 skip
-    skipp                skipp
-    skipper              skipper
-    skipping             skip
-    skirmish             skirmish
-    skirmishes           skirmish
-    skirr                skirr
-    skirted              skirt
-    skirts               skirt
-    skittish             skittish
-    skulking             skulk
-    skull                skull
-    skulls               skull
-    sky                  sky
-    skyey                skyei
-    skyish               skyish
-    slab                 slab
-    slack                slack
-    slackly              slackli
-    slackness            slack
-    slain                slain
-    slake                slake
-    sland                sland
-    slander              slander
-    slandered            slander
-    slanderer            slander
-    slanderers           slander
-    slandering           slander
-    slanderous           slander
-    slanders             slander
-    slash                slash
-    slaught              slaught
-    slaughter            slaughter
-    slaughtered          slaughter
-    slaughterer          slaughter
-    slaughterman         slaughterman
-    slaughtermen         slaughtermen
-    slaughterous         slaughter
-    slaughters           slaughter
-    slave                slave
-    slaver               slaver
-    slavery              slaveri
-    slaves               slave
-    slavish              slavish
-    slay                 slai
-    slayeth              slayeth
-    slaying              slai
-    slays                slai
-    sleave               sleav
-    sledded              sled
-    sleek                sleek
-    sleekly              sleekli
-    sleep                sleep
-    sleeper              sleeper
-    sleepers             sleeper
-    sleepest             sleepest
-    sleeping             sleep
-    sleeps               sleep
-    sleepy               sleepi
-    sleeve               sleev
-    sleeves              sleev
-    sleid                sleid
-    sleided              sleid
-    sleight              sleight
-    sleights             sleight
-    slender              slender
-    slenderer            slender
-    slenderly            slenderli
-    slept                slept
-    slew                 slew
-    slewest              slewest
-    slice                slice
-    slid                 slid
-    slide                slide
-    slides               slide
-    sliding              slide
-    slight               slight
-    slighted             slight
-    slightest            slightest
-    slightly             slightli
-    slightness           slight
-    slights              slight
-    slily                slili
-    slime                slime
-    slimy                slimi
-    slings               sling
-    slink                slink
-    slip                 slip
-    slipp                slipp
-    slipper              slipper
-    slippers             slipper
-    slippery             slipperi
-    slips                slip
-    slish                slish
-    slit                 slit
-    sliver               sliver
-    slobb                slobb
-    slomber              slomber
-    slop                 slop
-    slope                slope
-    slops                slop
-    sloth                sloth
-    slothful             sloth
-    slough               slough
-    slovenly             slovenli
-    slovenry             slovenri
-    slow                 slow
-    slower               slower
-    slowly               slowli
-    slowness             slow
-    slubber              slubber
-    slug                 slug
-    sluggard             sluggard
-    sluggardiz           sluggardiz
-    sluggish             sluggish
-    sluic                sluic
-    slumb                slumb
-    slumber              slumber
-    slumbers             slumber
-    slumbery             slumberi
-    slunk                slunk
-    slut                 slut
-    sluts                slut
-    sluttery             slutteri
-    sluttish             sluttish
-    sluttishness         sluttish
-    sly                  sly
-    slys                 sly
-    smack                smack
-    smacking             smack
-    smacks               smack
-    small                small
-    smaller              smaller
-    smallest             smallest
-    smallness            small
-    smalus               smalu
-    smart                smart
-    smarting             smart
-    smartly              smartli
-    smatch               smatch
-    smatter              smatter
-    smear                smear
-    smell                smell
-    smelling             smell
-    smells               smell
-    smelt                smelt
-    smil                 smil
-    smile                smile
-    smiled               smile
-    smiles               smile
-    smilest              smilest
-    smilets              smilet
-    smiling              smile
-    smilingly            smilingli
-    smirch               smirch
-    smirched             smirch
-    smit                 smit
-    smite                smite
-    smites               smite
-    smith                smith
-    smithfield           smithfield
-    smock                smock
-    smocks               smock
-    smok                 smok
-    smoke                smoke
-    smoked               smoke
-    smokes               smoke
-    smoking              smoke
-    smoky                smoki
-    smooth               smooth
-    smoothed             smooth
-    smoothing            smooth
-    smoothly             smoothli
-    smoothness           smooth
-    smooths              smooth
-    smote                smote
-    smoth                smoth
-    smother              smother
-    smothered            smother
-    smothering           smother
-    smug                 smug
-    smulkin              smulkin
-    smutch               smutch
-    snaffle              snaffl
-    snail                snail
-    snails               snail
-    snake                snake
-    snakes               snake
-    snaky                snaki
-    snap                 snap
-    snapp                snapp
-    snapper              snapper
-    snar                 snar
-    snare                snare
-    snares               snare
-    snarl                snarl
-    snarleth             snarleth
-    snarling             snarl
-    snatch               snatch
-    snatchers            snatcher
-    snatches             snatch
-    snatching            snatch
-    sneak                sneak
-    sneaking             sneak
-    sneap                sneap
-    sneaping             sneap
-    sneck                sneck
-    snip                 snip
-    snipe                snipe
-    snipt                snipt
-    snore                snore
-    snores               snore
-    snoring              snore
-    snorting             snort
-    snout                snout
-    snow                 snow
-    snowballs            snowbal
-    snowed               snow
-    snowy                snowi
-    snuff                snuff
-    snuffs               snuff
-    snug                 snug
-    so                   so
-    soak                 soak
-    soaking              soak
-    soaks                soak
-    soar                 soar
-    soaring              soar
-    soars                soar
-    sob                  sob
-    sobbing              sob
-    sober                sober
-    soberly              soberli
-    sobriety             sobrieti
-    sobs                 sob
-    sociable             sociabl
-    societies            societi
-    society              societi
-    socks                sock
-    socrates             socrat
-    sod                  sod
-    sodden               sodden
-    soe                  soe
-    soever               soever
-    soft                 soft
-    soften               soften
-    softens              soften
-    softer               softer
-    softest              softest
-    softly               softli
-    softness             soft
-    soil                 soil
-    soiled               soil
-    soilure              soilur
-    soit                 soit
-    sojourn              sojourn
-    sol                  sol
-    sola                 sola
-    solace               solac
-    solanio              solanio
-    sold                 sold
-    soldat               soldat
-    solder               solder
-    soldest              soldest
-    soldier              soldier
-    soldiers             soldier
-    soldiership          soldiership
-    sole                 sole
-    solely               sole
-    solem                solem
-    solemn               solemn
-    solemness            solem
-    solemnities          solemn
-    solemnity            solemn
-    solemniz             solemniz
-    solemnize            solemn
-    solemnized           solemn
-    solemnly             solemnli
-    soles                sole
-    solicit              solicit
-    solicitation         solicit
-    solicited            solicit
-    soliciting           solicit
-    solicitings          solicit
-    solicitor            solicitor
-    solicits             solicit
-    solid                solid
-    solidares            solidar
-    solidity             solid
-    solinus              solinu
-    solitary             solitari
-    solomon              solomon
-    solon                solon
-    solum                solum
-    solus                solu
-    solyman              solyman
-    some                 some
-    somebody             somebodi
-    someone              someon
-    somerset             somerset
-    somerville           somervil
-    something            someth
-    sometime             sometim
-    sometimes            sometim
-    somever              somev
-    somewhat             somewhat
-    somewhere            somewher
-    somewhither          somewhith
-    somme                somm
-    son                  son
-    sonance              sonanc
-    song                 song
-    songs                song
-    sonnet               sonnet
-    sonneting            sonnet
-    sonnets              sonnet
-    sons                 son
-    sont                 sont
-    sonties              sonti
-    soon                 soon
-    sooner               sooner
-    soonest              soonest
-    sooth                sooth
-    soothe               sooth
-    soothers             soother
-    soothing             sooth
-    soothsay             soothsai
-    soothsayer           soothsay
-    sooty                sooti
-    sop                  sop
-    sophister            sophist
-    sophisticated        sophist
-    sophy                sophi
-    sops                 sop
-    sorcerer             sorcer
-    sorcerers            sorcer
-    sorceress            sorceress
-    sorceries            sorceri
-    sorcery              sorceri
-    sore                 sore
-    sorel                sorel
-    sorely               sore
-    sorer                sorer
-    sores                sore
-    sorrier              sorrier
-    sorriest             sorriest
-    sorrow               sorrow
-    sorrowed             sorrow
-    sorrowest            sorrowest
-    sorrowful            sorrow
-    sorrowing            sorrow
-    sorrows              sorrow
-    sorry                sorri
-    sort                 sort
-    sortance             sortanc
-    sorted               sort
-    sorting              sort
-    sorts                sort
-    sossius              sossiu
-    sot                  sot
-    soto                 soto
-    sots                 sot
-    sottish              sottish
-    soud                 soud
-    sought               sought
-    soul                 soul
-    sould                sould
-    soulless             soulless
-    souls                soul
-    sound                sound
-    sounded              sound
-    sounder              sounder
-    soundest             soundest
-    sounding             sound
-    soundless            soundless
-    soundly              soundli
-    soundness            sound
-    soundpost            soundpost
-    sounds               sound
-    sour                 sour
-    source               sourc
-    sources              sourc
-    sourest              sourest
-    sourly               sourli
-    sours                sour
-    sous                 sou
-    souse                sous
-    south                south
-    southam              southam
-    southampton          southampton
-    southerly            southerli
-    southern             southern
-    southward            southward
-    southwark            southwark
-    southwell            southwel
-    souviendrai          souviendrai
-    sov                  sov
-    sovereign            sovereign
-    sovereignest         sovereignest
-    sovereignly          sovereignli
-    sovereignty          sovereignti
-    sovereignvours       sovereignvour
-    sow                  sow
-    sowing               sow
-    sowl                 sowl
-    sowter               sowter
-    space                space
-    spaces               space
-    spacious             spaciou
-    spade                spade
-    spades               spade
-    spain                spain
-    spak                 spak
-    spake                spake
-    spakest              spakest
-    span                 span
-    spangle              spangl
-    spangled             spangl
-    spaniard             spaniard
-    spaniel              spaniel
-    spaniels             spaniel
-    spanish              spanish
-    spann                spann
-    spans                span
-    spar                 spar
-    spare                spare
-    spares               spare
-    sparing              spare
-    sparingly            sparingli
-    spark                spark
-    sparkle              sparkl
-    sparkles             sparkl
-    sparkling            sparkl
-    sparks               spark
-    sparrow              sparrow
-    sparrows             sparrow
-    sparta               sparta
-    spartan              spartan
-    spavin               spavin
-    spavins              spavin
-    spawn                spawn
-    speak                speak
-    speaker              speaker
-    speakers             speaker
-    speakest             speakest
-    speaketh             speaketh
-    speaking             speak
-    speaks               speak
-    spear                spear
-    speargrass           speargrass
-    spears               spear
-    special              special
-    specialities         special
-    specially            special
-    specialties          specialti
-    specialty            specialti
-    specify              specifi
-    speciously           specious
-    spectacle            spectacl
-    spectacled           spectacl
-    spectacles           spectacl
-    spectators           spectat
-    spectatorship        spectatorship
-    speculation          specul
-    speculations         specul
-    speculative          specul
-    sped                 sped
-    speech               speech
-    speeches             speech
-    speechless           speechless
-    speed                speed
-    speeded              speed
-    speedier             speedier
-    speediest            speediest
-    speedily             speedili
-    speediness           speedi
-    speeding             speed
-    speeds               speed
-    speedy               speedi
-    speens               speen
-    spell                spell
-    spelling             spell
-    spells               spell
-    spelt                spelt
-    spencer              spencer
-    spend                spend
-    spendest             spendest
-    spending             spend
-    spends               spend
-    spendthrift          spendthrift
-    spent                spent
-    sperato              sperato
-    sperm                sperm
-    spero                spero
-    sperr                sperr
-    spher                spher
-    sphere               sphere
-    sphered              sphere
-    spheres              sphere
-    spherical            spheric
-    sphery               spheri
-    sphinx               sphinx
-    spice                spice
-    spiced               spice
-    spicery              spiceri
-    spices               spice
-    spider               spider
-    spiders              spider
-    spied                spi
-    spies                spi
-    spieth               spieth
-    spightfully          spightfulli
-    spigot               spigot
-    spill                spill
-    spilling             spill
-    spills               spill
-    spilt                spilt
-    spilth               spilth
-    spin                 spin
-    spinii               spinii
-    spinners             spinner
-    spinster             spinster
-    spinsters            spinster
-    spire                spire
-    spirit               spirit
-    spirited             spirit
-    spiritless           spiritless
-    spirits              spirit
-    spiritual            spiritu
-    spiritualty          spiritualti
-    spirt                spirt
-    spit                 spit
-    spital               spital
-    spite                spite
-    spited               spite
-    spiteful             spite
-    spites               spite
-    spits                spit
-    spitted              spit
-    spitting             spit
-    splay                splai
-    spleen               spleen
-    spleenful            spleen
-    spleens              spleen
-    spleeny              spleeni
-    splendour            splendour
-    splenitive           splenit
-    splinter             splinter
-    splinters            splinter
-    split                split
-    splits               split
-    splitted             split
-    splitting            split
-    spoil                spoil
-    spoils               spoil
-    spok                 spok
-    spoke                spoke
-    spoken               spoken
-    spokes               spoke
-    spokesman            spokesman
-    sponge               spong
-    spongy               spongi
-    spoon                spoon
-    spoons               spoon
-    sport                sport
-    sportful             sport
-    sporting             sport
-    sportive             sportiv
-    sports               sport
-    spot                 spot
-    spotless             spotless
-    spots                spot
-    spotted              spot
-    spousal              spousal
-    spouse               spous
-    spout                spout
-    spouting             spout
-    spouts               spout
-    sprag                sprag
-    sprang               sprang
-    sprat                sprat
-    sprawl               sprawl
-    spray                sprai
-    sprays               sprai
-    spread               spread
-    spreading            spread
-    spreads              spread
-    sprighted            spright
-    sprightful           spright
-    sprightly            sprightli
-    sprigs               sprig
-    spring               spring
-    springe              spring
-    springes             spring
-    springeth            springeth
-    springhalt           springhalt
-    springing            spring
-    springs              spring
-    springtime           springtim
-    sprinkle             sprinkl
-    sprinkles            sprinkl
-    sprite               sprite
-    sprited              sprite
-    spritely             sprite
-    sprites              sprite
-    spriting             sprite
-    sprout               sprout
-    spruce               spruce
-    sprung               sprung
-    spun                 spun
-    spur                 spur
-    spurio               spurio
-    spurn                spurn
-    spurns               spurn
-    spurr                spurr
-    spurrer              spurrer
-    spurring             spur
-    spurs                spur
-    spy                  spy
-    spying               spy
-    squabble             squabbl
-    squadron             squadron
-    squadrons            squadron
-    squand               squand
-    squar                squar
-    square               squar
-    squarer              squarer
-    squares              squar
-    squash               squash
-    squeak               squeak
-    squeaking            squeak
-    squeal               squeal
-    squealing            squeal
-    squeezes             squeez
-    squeezing            squeez
-    squele               squel
-    squier               squier
-    squints              squint
-    squiny               squini
-    squire               squir
-    squires              squir
-    squirrel             squirrel
-    st                   st
-    stab                 stab
-    stabb                stabb
-    stabbed              stab
-    stabbing             stab
-    stable               stabl
-    stableness           stabl
-    stables              stabl
-    stablish             stablish
-    stablishment         stablish
-    stabs                stab
-    stacks               stack
-    staff                staff
-    stafford             stafford
-    staffords            stafford
-    staffordshire        staffordshir
-    stag                 stag
-    stage                stage
-    stages               stage
-    stagger              stagger
-    staggering           stagger
-    staggers             stagger
-    stags                stag
-    staid                staid
-    staider              staider
-    stain                stain
-    stained              stain
-    staines              stain
-    staineth             staineth
-    staining             stain
-    stainless            stainless
-    stains               stain
-    stair                stair
-    stairs               stair
-    stake                stake
-    stakes               stake
-    stale                stale
-    staled               stale
-    stalk                stalk
-    stalking             stalk
-    stalks               stalk
-    stall                stall
-    stalling             stall
-    stalls               stall
-    stamford             stamford
-    stammer              stammer
-    stamp                stamp
-    stamped              stamp
-    stamps               stamp
-    stanch               stanch
-    stanchless           stanchless
-    stand                stand
-    standard             standard
-    standards            standard
-    stander              stander
-    standers             stander
-    standest             standest
-    standeth             standeth
-    standing             stand
-    stands               stand
-    staniel              staniel
-    stanley              stanlei
-    stanze               stanz
-    stanzo               stanzo
-    stanzos              stanzo
-    staple               stapl
-    staples              stapl
-    star                 star
-    stare                stare
-    stared               stare
-    stares               stare
-    staring              stare
-    starings             stare
-    stark                stark
-    starkly              starkli
-    starlight            starlight
-    starling             starl
-    starr                starr
-    starry               starri
-    stars                star
-    start                start
-    started              start
-    starting             start
-    startingly           startingli
-    startle              startl
-    startles             startl
-    starts               start
-    starv                starv
-    starve               starv
-    starved              starv
-    starvelackey         starvelackei
-    starveling           starvel
-    starveth             starveth
-    starving             starv
-    state                state
-    statelier            stateli
-    stately              state
-    states               state
-    statesman            statesman
-    statesmen            statesmen
-    statilius            statiliu
-    station              station
-    statist              statist
-    statists             statist
-    statue               statu
-    statues              statu
-    stature              statur
-    statures             statur
-    statute              statut
-    statutes             statut
-    stave                stave
-    staves               stave
-    stay                 stai
-    stayed               stai
-    stayest              stayest
-    staying              stai
-    stays                stai
-    stead                stead
-    steaded              stead
-    steadfast            steadfast
-    steadier             steadier
-    steads               stead
-    steal                steal
-    stealer              stealer
-    stealers             stealer
-    stealing             steal
-    steals               steal
-    stealth              stealth
-    stealthy             stealthi
-    steed                steed
-    steeds               steed
-    steel                steel
-    steeled              steel
-    steely               steeli
-    steep                steep
-    steeped              steep
-    steeple              steepl
-    steeples             steepl
-    steeps               steep
-    steepy               steepi
-    steer                steer
-    steerage             steerag
-    steering             steer
-    steers               steer
-    stelled              stell
-    stem                 stem
-    stemming             stem
-    stench               stench
-    step                 step
-    stepdame             stepdam
-    stephano             stephano
-    stephen              stephen
-    stepmothers          stepmoth
-    stepp                stepp
-    stepping             step
-    steps                step
-    sterile              steril
-    sterility            steril
-    sterling             sterl
-    stern                stern
-    sternage             sternag
-    sterner              sterner
-    sternest             sternest
-    sternness            stern
-    steterat             steterat
-    stew                 stew
-    steward              steward
-    stewards             steward
-    stewardship          stewardship
-    stewed               stew
-    stews                stew
-    stick                stick
-    sticking             stick
-    stickler             stickler
-    sticks               stick
-    stiff                stiff
-    stiffen              stiffen
-    stiffly              stiffli
-    stifle               stifl
-    stifled              stifl
-    stifles              stifl
-    stigmatic            stigmat
-    stigmatical          stigmat
-    stile                stile
-    still                still
-    stiller              stiller
-    stillest             stillest
-    stillness            still
-    stilly               stilli
-    sting                sting
-    stinging             sting
-    stingless            stingless
-    stings               sting
-    stink                stink
-    stinking             stink
-    stinkingly           stinkingli
-    stinks               stink
-    stint                stint
-    stinted              stint
-    stints               stint
-    stir                 stir
-    stirr                stirr
-    stirred              stir
-    stirrer              stirrer
-    stirrers             stirrer
-    stirreth             stirreth
-    stirring             stir
-    stirrup              stirrup
-    stirrups             stirrup
-    stirs                stir
-    stitchery            stitcheri
-    stitches             stitch
-    stithied             stithi
-    stithy               stithi
-    stoccadoes           stoccado
-    stoccata             stoccata
-    stock                stock
-    stockfish            stockfish
-    stocking             stock
-    stockings            stock
-    stockish             stockish
-    stocks               stock
-    stog                 stog
-    stogs                stog
-    stoics               stoic
-    stokesly             stokesli
-    stol                 stol
-    stole                stole
-    stolen               stolen
-    stolest              stolest
-    stomach              stomach
-    stomachers           stomach
-    stomaching           stomach
-    stomachs             stomach
-    ston                 ston
-    stone                stone
-    stonecutter          stonecutt
-    stones               stone
-    stonish              stonish
-    stony                stoni
-    stood                stood
-    stool                stool
-    stools               stool
-    stoop                stoop
-    stooping             stoop
-    stoops               stoop
-    stop                 stop
-    stope                stope
-    stopp                stopp
-    stopped              stop
-    stopping             stop
-    stops                stop
-    stor                 stor
-    store                store
-    storehouse           storehous
-    storehouses          storehous
-    stores               store
-    stories              stori
-    storm                storm
-    stormed              storm
-    storming             storm
-    storms               storm
-    stormy               stormi
-    story                stori
-    stoup                stoup
-    stoups               stoup
-    stout                stout
-    stouter              stouter
-    stoutly              stoutli
-    stoutness            stout
-    stover               stover
-    stow                 stow
-    stowage              stowag
-    stowed               stow
-    strachy              strachi
-    stragglers           straggler
-    straggling           straggl
-    straight             straight
-    straightest          straightest
-    straightway          straightwai
-    strain               strain
-    strained             strain
-    straining            strain
-    strains              strain
-    strait               strait
-    straited             strait
-    straiter             straiter
-    straitly             straitli
-    straitness           strait
-    straits              strait
-    strand               strand
-    strange              strang
-    strangely            strang
-    strangeness          strang
-    stranger             stranger
-    strangers            stranger
-    strangest            strangest
-    strangle             strangl
-    strangled            strangl
-    strangler            strangler
-    strangles            strangl
-    strangling           strangl
-    strappado            strappado
-    straps               strap
-    stratagem            stratagem
-    stratagems           stratagem
-    stratford            stratford
-    strato               strato
-    straw                straw
-    strawberries         strawberri
-    strawberry           strawberri
-    straws               straw
-    strawy               strawi
-    stray                strai
-    straying             strai
-    strays               strai
-    streak               streak
-    streaks              streak
-    stream               stream
-    streamers            streamer
-    streaming            stream
-    streams              stream
-    streching            strech
-    street               street
-    streets              street
-    strength             strength
-    strengthen           strengthen
-    strengthened         strengthen
-    strengthless         strengthless
-    strengths            strength
-    stretch              stretch
-    stretched            stretch
-    stretches            stretch
-    stretching           stretch
-    strew                strew
-    strewing             strew
-    strewings            strew
-    strewments           strewment
-    stricken             stricken
-    strict               strict
-    stricter             stricter
-    strictest            strictest
-    strictly             strictli
-    stricture            strictur
-    stride               stride
-    strides              stride
-    striding             stride
-    strife               strife
-    strifes              strife
-    strik                strik
-    strike               strike
-    strikers             striker
-    strikes              strike
-    strikest             strikest
-    striking             strike
-    string               string
-    stringless           stringless
-    strings              string
-    strip                strip
-    stripes              stripe
-    stripling            stripl
-    striplings           stripl
-    stripp               stripp
-    stripping            strip
-    striv                striv
-    strive               strive
-    strives              strive
-    striving             strive
-    strok                strok
-    stroke               stroke
-    strokes              stroke
-    strond               strond
-    stronds              strond
-    strong               strong
-    stronger             stronger
-    strongest            strongest
-    strongly             strongli
-    strooke              strook
-    strossers            strosser
-    strove               strove
-    strown               strown
-    stroy                stroi
-    struck               struck
-    strucken             strucken
-    struggle             struggl
-    struggles            struggl
-    struggling           struggl
-    strumpet             strumpet
-    strumpeted           strumpet
-    strumpets            strumpet
-    strung               strung
-    strut                strut
-    struts               strut
-    strutted             strut
-    strutting            strut
-    stubble              stubbl
-    stubborn             stubborn
-    stubbornest          stubbornest
-    stubbornly           stubbornli
-    stubbornness         stubborn
-    stuck                stuck
-    studded              stud
-    student              student
-    students             student
-    studied              studi
-    studies              studi
-    studious             studiou
-    studiously           studious
-    studs                stud
-    study                studi
-    studying             studi
-    stuff                stuff
-    stuffing             stuf
-    stuffs               stuff
-    stumble              stumbl
-    stumbled             stumbl
-    stumblest            stumblest
-    stumbling            stumbl
-    stump                stump
-    stumps               stump
-    stung                stung
-    stupefy              stupefi
-    stupid               stupid
-    stupified            stupifi
-    stuprum              stuprum
-    sturdy               sturdi
-    sty                  sty
-    styga                styga
-    stygian              stygian
-    styl                 styl
-    style                style
-    styx                 styx
-    su                   su
-    sub                  sub
-    subcontracted        subcontract
-    subdu                subdu
-    subdue               subdu
-    subdued              subdu
-    subduements          subduement
-    subdues              subdu
-    subduing             subdu
-    subject              subject
-    subjected            subject
-    subjection           subject
-    subjects             subject
-    submerg              submerg
-    submission           submiss
-    submissive           submiss
-    submit               submit
-    submits              submit
-    submitting           submit
-    suborn               suborn
-    subornation          suborn
-    suborned             suborn
-    subscrib             subscrib
-    subscribe            subscrib
-    subscribed           subscrib
-    subscribes           subscrib
-    subscription         subscript
-    subsequent           subsequ
-    subsidies            subsidi
-    subsidy              subsidi
-    subsist              subsist
-    subsisting           subsist
-    substance            substanc
-    substances           substanc
-    substantial          substanti
-    substitute           substitut
-    substituted          substitut
-    substitutes          substitut
-    substitution         substitut
-    subtile              subtil
-    subtilly             subtilli
-    subtle               subtl
-    subtleties           subtleti
-    subtlety             subtleti
-    subtly               subtli
-    subtractors          subtractor
-    suburbs              suburb
-    subversion           subvers
-    subverts             subvert
-    succedant            succed
-    succeed              succe
-    succeeded            succeed
-    succeeders           succeed
-    succeeding           succeed
-    succeeds             succe
-    success              success
-    successantly         successantli
-    successes            success
-    successful           success
-    successfully         successfulli
-    succession           success
-    successive           success
-    successively         success
-    successor            successor
-    successors           successor
-    succour              succour
-    succours             succour
-    such                 such
-    suck                 suck
-    sucker               sucker
-    suckers              sucker
-    sucking              suck
-    suckle               suckl
-    sucks                suck
-    sudden               sudden
-    suddenly             suddenli
-    sue                  sue
-    sued                 su
-    suerly               suerli
-    sues                 sue
-    sueth                sueth
-    suff                 suff
-    suffer               suffer
-    sufferance           suffer
-    sufferances          suffer
-    suffered             suffer
-    suffering            suffer
-    suffers              suffer
-    suffic               suffic
-    suffice              suffic
-    sufficed             suffic
-    suffices             suffic
-    sufficeth            sufficeth
-    sufficiency          suffici
-    sufficient           suffici
-    sufficiently         suffici
-    sufficing            suffic
-    sufficit             sufficit
-    suffigance           suffig
-    suffocate            suffoc
-    suffocating          suffoc
-    suffocation          suffoc
-    suffolk              suffolk
-    suffrage             suffrag
-    suffrages            suffrag
-    sug                  sug
-    sugar                sugar
-    sugarsop             sugarsop
-    suggest              suggest
-    suggested            suggest
-    suggesting           suggest
-    suggestion           suggest
-    suggestions          suggest
-    suggests             suggest
-    suis                 sui
-    suit                 suit
-    suitable             suitabl
-    suited               suit
-    suiting              suit
-    suitor               suitor
-    suitors              suitor
-    suits                suit
-    suivez               suivez
-    sullen               sullen
-    sullens              sullen
-    sullied              sulli
-    sullies              sulli
-    sully                sulli
-    sulph                sulph
-    sulpherous           sulpher
-    sulphur              sulphur
-    sulphurous           sulphur
-    sultan               sultan
-    sultry               sultri
-    sum                  sum
-    sumless              sumless
-    summ                 summ
-    summa                summa
-    summary              summari
-    summer               summer
-    summers              summer
-    summit               summit
-    summon               summon
-    summoners            summon
-    summons              summon
-    sumpter              sumpter
-    sumptuous            sumptuou
-    sumptuously          sumptuous
-    sums                 sum
-    sun                  sun
-    sunbeams             sunbeam
-    sunburning           sunburn
-    sunburnt             sunburnt
-    sund                 sund
-    sunday               sundai
-    sundays              sundai
-    sunder               sunder
-    sunders              sunder
-    sundry               sundri
-    sung                 sung
-    sunk                 sunk
-    sunken               sunken
-    sunny                sunni
-    sunrising            sunris
-    suns                 sun
-    sunset               sunset
-    sunshine             sunshin
-    sup                  sup
-    super                super
-    superficial          superfici
-    superficially        superfici
-    superfluity          superflu
-    superfluous          superflu
-    superfluously        superflu
-    superflux            superflux
-    superior             superior
-    supernal             supern
-    supernatural         supernatur
-    superpraise          superprais
-    superscript          superscript
-    superscription       superscript
-    superserviceable     superservic
-    superstition         superstit
-    superstitious        superstiti
-    superstitiously      superstiti
-    supersubtle          supersubtl
-    supervise            supervis
-    supervisor           supervisor
-    supp                 supp
-    supper               supper
-    suppers              supper
-    suppertime           suppertim
-    supping              sup
-    supplant             supplant
-    supple               suppl
-    suppler              suppler
-    suppliance           supplianc
-    suppliant            suppliant
-    suppliants           suppliant
-    supplicant           supplic
-    supplication         supplic
-    supplications        supplic
-    supplie              suppli
-    supplied             suppli
-    supplies             suppli
-    suppliest            suppliest
-    supply               suppli
-    supplyant            supplyant
-    supplying            suppli
-    supplyment           supplyment
-    support              support
-    supportable          support
-    supportance          support
-    supported            support
-    supporter            support
-    supporters           support
-    supporting           support
-    supportor            supportor
-    suppos               suppo
-    supposal             suppos
-    suppose              suppos
-    supposed             suppos
-    supposes             suppos
-    supposest            supposest
-    supposing            suppos
-    supposition          supposit
-    suppress             suppress
-    suppressed           suppress
-    suppresseth          suppresseth
-    supremacy            supremaci
-    supreme              suprem
-    sups                 sup
-    sur                  sur
-    surance              suranc
-    surcease             surceas
-    surd                 surd
-    sure                 sure
-    surecard             surecard
-    surely               sure
-    surer                surer
-    surest               surest
-    sureties             sureti
-    surety               sureti
-    surfeit              surfeit
-    surfeited            surfeit
-    surfeiter            surfeit
-    surfeiting           surfeit
-    surfeits             surfeit
-    surge                surg
-    surgeon              surgeon
-    surgeons             surgeon
-    surgere              surger
-    surgery              surgeri
-    surges               surg
-    surly                surli
-    surmis               surmi
-    surmise              surmis
-    surmised             surmis
-    surmises             surmis
-    surmount             surmount
-    surmounted           surmount
-    surmounts            surmount
-    surnam               surnam
-    surname              surnam
-    surnamed             surnam
-    surpasseth           surpasseth
-    surpassing           surpass
-    surplice             surplic
-    surplus              surplu
-    surpris              surpri
-    surprise             surpris
-    surprised            surpris
-    surrender            surrend
-    surrey               surrei
-    surreys              surrei
-    survey               survei
-    surveyest            surveyest
-    surveying            survei
-    surveyor             surveyor
-    surveyors            surveyor
-    surveys              survei
-    survive              surviv
-    survives             surviv
-    survivor             survivor
-    susan                susan
-    suspect              suspect
-    suspected            suspect
-    suspecting           suspect
-    suspects             suspect
-    suspend              suspend
-    suspense             suspens
-    suspicion            suspicion
-    suspicions           suspicion
-    suspicious           suspici
-    suspiration          suspir
-    suspire              suspir
-    sust                 sust
-    sustain              sustain
-    sustaining           sustain
-    sutler               sutler
-    sutton               sutton
-    suum                 suum
-    swabber              swabber
-    swaddling            swaddl
-    swag                 swag
-    swagg                swagg
-    swagger              swagger
-    swaggerer            swagger
-    swaggerers           swagger
-    swaggering           swagger
-    swain                swain
-    swains               swain
-    swallow              swallow
-    swallowed            swallow
-    swallowing           swallow
-    swallows             swallow
-    swam                 swam
-    swan                 swan
-    swans                swan
-    sward                sward
-    sware                sware
-    swarm                swarm
-    swarming             swarm
-    swart                swart
-    swarth               swarth
-    swarths              swarth
-    swarthy              swarthi
-    swashers             swasher
-    swashing             swash
-    swath                swath
-    swathing             swath
-    swathling            swathl
-    sway                 swai
-    swaying              swai
-    sways                swai
-    swear                swear
-    swearer              swearer
-    swearers             swearer
-    swearest             swearest
-    swearing             swear
-    swearings            swear
-    swears               swear
-    sweat                sweat
-    sweaten              sweaten
-    sweating             sweat
-    sweats               sweat
-    sweaty               sweati
-    sweep                sweep
-    sweepers             sweeper
-    sweeps               sweep
-    sweet                sweet
-    sweeten              sweeten
-    sweetens             sweeten
-    sweeter              sweeter
-    sweetest             sweetest
-    sweetheart           sweetheart
-    sweeting             sweet
-    sweetly              sweetli
-    sweetmeats           sweetmeat
-    sweetness            sweet
-    sweets               sweet
-    swell                swell
-    swelling             swell
-    swellings            swell
-    swells               swell
-    swelter              swelter
-    sweno                sweno
-    swept                swept
-    swerve               swerv
-    swerver              swerver
-    swerving             swerv
-    swift                swift
-    swifter              swifter
-    swiftest             swiftest
-    swiftly              swiftli
-    swiftness            swift
-    swill                swill
-    swills               swill
-    swim                 swim
-    swimmer              swimmer
-    swimmers             swimmer
-    swimming             swim
-    swims                swim
-    swine                swine
-    swineherds           swineherd
-    swing                swing
-    swinge               swing
-    swinish              swinish
-    swinstead            swinstead
-    switches             switch
-    swits                swit
-    switzers             switzer
-    swol                 swol
-    swoll                swoll
-    swoln                swoln
-    swoon                swoon
-    swooned              swoon
-    swooning             swoon
-    swoons               swoon
-    swoop                swoop
-    swoopstake           swoopstak
-    swor                 swor
-    sword                sword
-    sworder              sworder
-    swords               sword
-    swore                swore
-    sworn                sworn
-    swounded             swound
-    swounds              swound
-    swum                 swum
-    swung                swung
-    sy                   sy
-    sycamore             sycamor
-    sycorax              sycorax
-    sylla                sylla
-    syllable             syllabl
-    syllables            syllabl
-    syllogism            syllog
-    symbols              symbol
-    sympathise           sympathis
-    sympathiz            sympathiz
-    sympathize           sympath
-    sympathized          sympath
-    sympathy             sympathi
-    synagogue            synagogu
-    synod                synod
-    synods               synod
-    syracuse             syracus
-    syracusian           syracusian
-    syracusians          syracusian
-    syria                syria
-    syrups               syrup
-    t                    t
-    ta                   ta
-    taber                taber
-    table                tabl
-    tabled               tabl
-    tables               tabl
-    tablet               tablet
-    tabor                tabor
-    taborer              tabor
-    tabors               tabor
-    tabourines           tabourin
-    taciturnity          taciturn
-    tack                 tack
-    tackle               tackl
-    tackled              tackl
-    tackles              tackl
-    tackling             tackl
-    tacklings            tackl
-    taddle               taddl
-    tadpole              tadpol
-    taffeta              taffeta
-    taffety              taffeti
-    tag                  tag
-    tagrag               tagrag
-    tah                  tah
-    tail                 tail
-    tailor               tailor
-    tailors              tailor
-    tails                tail
-    taint                taint
-    tainted              taint
-    tainting             taint
-    taints               taint
-    tainture             taintur
-    tak                  tak
-    take                 take
-    taken                taken
-    taker                taker
-    takes                take
-    takest               takest
-    taketh               taketh
-    taking               take
-    tal                  tal
-    talbot               talbot
-    talbotites           talbotit
-    talbots              talbot
-    tale                 tale
-    talent               talent
-    talents              talent
-    taleporter           taleport
-    tales                tale
-    talk                 talk
-    talked               talk
-    talker               talker
-    talkers              talker
-    talkest              talkest
-    talking              talk
-    talks                talk
-    tall                 tall
-    taller               taller
-    tallest              tallest
-    tallies              talli
-    tallow               tallow
-    tally                talli
-    talons               talon
-    tam                  tam
-    tambourines          tambourin
-    tame                 tame
-    tamed                tame
-    tamely               tame
-    tameness             tame
-    tamer                tamer
-    tames                tame
-    taming               tame
-    tamora               tamora
-    tamworth             tamworth
-    tan                  tan
-    tang                 tang
-    tangle               tangl
-    tangled              tangl
-    tank                 tank
-    tanlings             tanl
-    tann                 tann
-    tanned               tan
-    tanner               tanner
-    tanquam              tanquam
-    tanta                tanta
-    tantaene             tantaen
-    tap                  tap
-    tape                 tape
-    taper                taper
-    tapers               taper
-    tapestries           tapestri
-    tapestry             tapestri
-    taphouse             taphous
-    tapp                 tapp
-    tapster              tapster
-    tapsters             tapster
-    tar                  tar
-    tardied              tardi
-    tardily              tardili
-    tardiness            tardi
-    tardy                tardi
-    tarentum             tarentum
-    targe                targ
-    targes               targ
-    target               target
-    targets              target
-    tarpeian             tarpeian
-    tarquin              tarquin
-    tarquins             tarquin
-    tarr                 tarr
-    tarre                tarr
-    tarriance            tarrianc
-    tarried              tarri
-    tarries              tarri
-    tarry                tarri
-    tarrying             tarri
-    tart                 tart
-    tartar               tartar
-    tartars              tartar
-    tartly               tartli
-    tartness             tart
-    task                 task
-    tasker               tasker
-    tasking              task
-    tasks                task
-    tassel               tassel
-    taste                tast
-    tasted               tast
-    tastes               tast
-    tasting              tast
-    tatt                 tatt
-    tatter               tatter
-    tattered             tatter
-    tatters              tatter
-    tattle               tattl
-    tattling             tattl
-    tattlings            tattl
-    taught               taught
-    taunt                taunt
-    taunted              taunt
-    taunting             taunt
-    tauntingly           tauntingli
-    taunts               taunt
-    taurus               tauru
-    tavern               tavern
-    taverns              tavern
-    tavy                 tavi
-    tawdry               tawdri
-    tawny                tawni
-    tax                  tax
-    taxation             taxat
-    taxations            taxat
-    taxes                tax
-    taxing               tax
-    tc                   tc
-    te                   te
-    teach                teach
-    teacher              teacher
-    teachers             teacher
-    teaches              teach
-    teachest             teachest
-    teacheth             teacheth
-    teaching             teach
-    team                 team
-    tear                 tear
-    tearful              tear
-    tearing              tear
-    tears                tear
-    tearsheet            tearsheet
-    teat                 teat
-    tedious              tediou
-    tediously            tedious
-    tediousness          tedious
-    teem                 teem
-    teeming              teem
-    teems                teem
-    teen                 teen
-    teeth                teeth
-    teipsum              teipsum
-    telamon              telamon
-    telamonius           telamoniu
-    tell                 tell
-    teller               teller
-    telling              tell
-    tells                tell
-    tellus               tellu
-    temp                 temp
-    temper               temper
-    temperality          temper
-    temperance           temper
-    temperate            temper
-    temperately          temper
-    tempers              temper
-    tempest              tempest
-    tempests             tempest
-    tempestuous          tempestu
-    temple               templ
-    temples              templ
-    temporal             tempor
-    temporary            temporari
-    temporiz             temporiz
-    temporize            tempor
-    temporizer           tempor
-    temps                temp
-    tempt                tempt
-    temptation           temptat
-    temptations          temptat
-    tempted              tempt
-    tempter              tempter
-    tempters             tempter
-    tempteth             tempteth
-    tempting             tempt
-    tempts               tempt
-    ten                  ten
-    tenable              tenabl
-    tenant               tenant
-    tenantius            tenantiu
-    tenantless           tenantless
-    tenants              tenant
-    tench                tench
-    tend                 tend
-    tendance             tendanc
-    tended               tend
-    tender               tender
-    tendered             tender
-    tenderly             tenderli
-    tenderness           tender
-    tenders              tender
-    tending              tend
-    tends                tend
-    tenedos              tenedo
-    tenement             tenement
-    tenements            tenement
-    tenfold              tenfold
-    tennis               tenni
-    tenour               tenour
-    tenours              tenour
-    tens                 ten
-    tent                 tent
-    tented               tent
-    tenth                tenth
-    tenths               tenth
-    tents                tent
-    tenure               tenur
-    tenures              tenur
-    tercel               tercel
-    tereus               tereu
-    term                 term
-    termagant            termag
-    termed               term
-    terminations         termin
-    termless             termless
-    terms                term
-    terra                terra
-    terrace              terrac
-    terram               terram
-    terras               terra
-    terre                terr
-    terrene              terren
-    terrestrial          terrestri
-    terrible             terribl
-    terribly             terribl
-    territories          territori
-    territory            territori
-    terror               terror
-    terrors              terror
-    tertian              tertian
-    tertio               tertio
-    test                 test
-    testament            testament
-    tested               test
-    tester               tester
-    testern              testern
-    testify              testifi
-    testimonied          testimoni
-    testimonies          testimoni
-    testimony            testimoni
-    testiness            testi
-    testril              testril
-    testy                testi
-    tetchy               tetchi
-    tether               tether
-    tetter               tetter
-    tevil                tevil
-    tewksbury            tewksburi
-    text                 text
-    tgv                  tgv
-    th                   th
-    thaes                thae
-    thames               thame
-    than                 than
-    thane                thane
-    thanes               thane
-    thank                thank
-    thanked              thank
-    thankful             thank
-    thankfully           thankfulli
-    thankfulness         thank
-    thanking             thank
-    thankings            thank
-    thankless            thankless
-    thanks               thank
-    thanksgiving         thanksgiv
-    thasos               thaso
-    that                 that
-    thatch               thatch
-    thaw                 thaw
-    thawing              thaw
-    thaws                thaw
-    the                  the
-    theatre              theatr
-    theban               theban
-    thebes               thebe
-    thee                 thee
-    theft                theft
-    thefts               theft
-    thein                thein
-    their                their
-    theirs               their
-    theise               theis
-    them                 them
-    theme                theme
-    themes               theme
-    themselves           themselv
-    then                 then
-    thence               thenc
-    thenceforth          thenceforth
-    theoric              theoric
-    there                there
-    thereabout           thereabout
-    thereabouts          thereabout
-    thereafter           thereaft
-    thereat              thereat
-    thereby              therebi
-    therefore            therefor
-    therein              therein
-    thereof              thereof
-    thereon              thereon
-    thereto              thereto
-    thereunto            thereunto
-    thereupon            thereupon
-    therewith            therewith
-    therewithal          therewith
-    thersites            thersit
-    these                these
-    theseus              theseu
-    thessalian           thessalian
-    thessaly             thessali
-    thetis               theti
-    thews                thew
-    they                 thei
-    thick                thick
-    thicken              thicken
-    thickens             thicken
-    thicker              thicker
-    thickest             thickest
-    thicket              thicket
-    thickskin            thickskin
-    thief                thief
-    thievery             thieveri
-    thieves              thiev
-    thievish             thievish
-    thigh                thigh
-    thighs               thigh
-    thimble              thimbl
-    thimbles             thimbl
-    thin                 thin
-    thine                thine
-    thing                thing
-    things               thing
-    think                think
-    thinkest             thinkest
-    thinking             think
-    thinkings            think
-    thinks               think
-    thinkst              thinkst
-    thinly               thinli
-    third                third
-    thirdly              thirdli
-    thirds               third
-    thirst               thirst
-    thirsting            thirst
-    thirsts              thirst
-    thirsty              thirsti
-    thirteen             thirteen
-    thirties             thirti
-    thirtieth            thirtieth
-    thirty               thirti
-    this                 thi
-    thisby               thisbi
-    thisne               thisn
-    thistle              thistl
-    thistles             thistl
-    thither              thither
-    thitherward          thitherward
-    thoas                thoa
-    thomas               thoma
-    thorn                thorn
-    thorns               thorn
-    thorny               thorni
-    thorough             thorough
-    thoroughly           thoroughli
-    those                those
-    thou                 thou
-    though               though
-    thought              thought
-    thoughtful           thought
-    thoughts             thought
-    thousand             thousand
-    thousands            thousand
-    thracian             thracian
-    thraldom             thraldom
-    thrall               thrall
-    thralled             thrall
-    thralls              thrall
-    thrash               thrash
-    thrasonical          thrason
-    thread               thread
-    threadbare           threadbar
-    threaden             threaden
-    threading            thread
-    threat               threat
-    threaten             threaten
-    threatening          threaten
-    threatens            threaten
-    threatest            threatest
-    threats              threat
-    three                three
-    threefold            threefold
-    threepence           threepenc
-    threepile            threepil
-    threes               three
-    threescore           threescor
-    thresher             thresher
-    threshold            threshold
-    threw                threw
-    thrice               thrice
-    thrift               thrift
-    thriftless           thriftless
-    thrifts              thrift
-    thrifty              thrifti
-    thrill               thrill
-    thrilling            thrill
-    thrills              thrill
-    thrive               thrive
-    thrived              thrive
-    thrivers             thriver
-    thrives              thrive
-    thriving             thrive
-    throat               throat
-    throats              throat
-    throbbing            throb
-    throbs               throb
-    throca               throca
-    throe                throe
-    throes               throe
-    thromuldo            thromuldo
-    thron                thron
-    throne               throne
-    throned              throne
-    thrones              throne
-    throng               throng
-    thronging            throng
-    throngs              throng
-    throstle             throstl
-    throttle             throttl
-    through              through
-    throughfare          throughfar
-    throughfares         throughfar
-    throughly            throughli
-    throughout           throughout
-    throw                throw
-    thrower              thrower
-    throwest             throwest
-    throwing             throw
-    thrown               thrown
-    throws               throw
-    thrum                thrum
-    thrumm               thrumm
-    thrush               thrush
-    thrust               thrust
-    thrusteth            thrusteth
-    thrusting            thrust
-    thrusts              thrust
-    thumb                thumb
-    thumbs               thumb
-    thump                thump
-    thund                thund
-    thunder              thunder
-    thunderbolt          thunderbolt
-    thunderbolts         thunderbolt
-    thunderer            thunder
-    thunders             thunder
-    thunderstone         thunderston
-    thunderstroke        thunderstrok
-    thurio               thurio
-    thursday             thursdai
-    thus                 thu
-    thwack               thwack
-    thwart               thwart
-    thwarted             thwart
-    thwarting            thwart
-    thwartings           thwart
-    thy                  thy
-    thyme                thyme
-    thymus               thymu
-    thyreus              thyreu
-    thyself              thyself
-    ti                   ti
-    tib                  tib
-    tiber                tiber
-    tiberio              tiberio
-    tibey                tibei
-    ticed                tice
-    tick                 tick
-    tickl                tickl
-    tickle               tickl
-    tickled              tickl
-    tickles              tickl
-    tickling             tickl
-    ticklish             ticklish
-    tiddle               tiddl
-    tide                 tide
-    tides                tide
-    tidings              tide
-    tidy                 tidi
-    tie                  tie
-    tied                 ti
-    ties                 ti
-    tiff                 tiff
-    tiger                tiger
-    tigers               tiger
-    tight                tight
-    tightly              tightli
-    tike                 tike
-    til                  til
-    tile                 tile
-    till                 till
-    tillage              tillag
-    tilly                tilli
-    tilt                 tilt
-    tilter               tilter
-    tilth                tilth
-    tilting              tilt
-    tilts                tilt
-    tiltyard             tiltyard
-    tim                  tim
-    timandra             timandra
-    timber               timber
-    time                 time
-    timeless             timeless
-    timelier             timeli
-    timely               time
-    times                time
-    timon                timon
-    timor                timor
-    timorous             timor
-    timorously           timor
-    tinct                tinct
-    tincture             tinctur
-    tinctures            tinctur
-    tinder               tinder
-    tingling             tingl
-    tinker               tinker
-    tinkers              tinker
-    tinsel               tinsel
-    tiny                 tini
-    tip                  tip
-    tipp                 tipp
-    tippling             tippl
-    tips                 tip
-    tipsy                tipsi
-    tiptoe               tipto
-    tir                  tir
-    tire                 tire
-    tired                tire
-    tires                tire
-    tirest               tirest
-    tiring               tire
-    tirra                tirra
-    tirrits              tirrit
-    tis                  ti
-    tish                 tish
-    tisick               tisick
-    tissue               tissu
-    titan                titan
-    titania              titania
-    tithe                tith
-    tithed               tith
-    tithing              tith
-    titinius             titiniu
-    title                titl
-    titled               titl
-    titleless            titleless
-    titles               titl
-    tittle               tittl
-    tittles              tittl
-    titular              titular
-    titus                titu
-    tn                   tn
-    to                   to
-    toad                 toad
-    toads                toad
-    toadstool            toadstool
-    toast                toast
-    toasted              toast
-    toasting             toast
-    toasts               toast
-    toaze                toaz
-    toby                 tobi
-    tock                 tock
-    tod                  tod
-    today                todai
-    todpole              todpol
-    tods                 tod
-    toe                  toe
-    toes                 toe
-    tofore               tofor
-    toge                 toge
-    toged                toge
-    together             togeth
-    toil                 toil
-    toiled               toil
-    toiling              toil
-    toils                toil
-    token                token
-    tokens               token
-    told                 told
-    toledo               toledo
-    tolerable            toler
-    toll                 toll
-    tolling              toll
-    tom                  tom
-    tomb                 tomb
-    tombe                tomb
-    tombed               tomb
-    tombless             tombless
-    tomboys              tomboi
-    tombs                tomb
-    tomorrow             tomorrow
-    tomyris              tomyri
-    ton                  ton
-    tongs                tong
-    tongu                tongu
-    tongue               tongu
-    tongued              tongu
-    tongueless           tongueless
-    tongues              tongu
-    tonight              tonight
-    too                  too
-    took                 took
-    tool                 tool
-    tools                tool
-    tooth                tooth
-    toothache            toothach
-    toothpick            toothpick
-    toothpicker          toothpick
-    top                  top
-    topas                topa
-    topful               top
-    topgallant           topgal
-    topless              topless
-    topmast              topmast
-    topp                 topp
-    topping              top
-    topple               toppl
-    topples              toppl
-    tops                 top
-    topsail              topsail
-    topsy                topsi
-    torch                torch
-    torchbearer          torchbear
-    torchbearers         torchbear
-    torcher              torcher
-    torches              torch
-    torchlight           torchlight
-    tore                 tore
-    torment              torment
-    tormenta             tormenta
-    tormente             torment
-    tormented            torment
-    tormenting           torment
-    tormentors           tormentor
-    torments             torment
-    torn                 torn
-    torrent              torrent
-    tortive              tortiv
-    tortoise             tortois
-    tortur               tortur
-    torture              tortur
-    tortured             tortur
-    torturer             tortur
-    torturers            tortur
-    tortures             tortur
-    torturest            torturest
-    torturing            tortur
-    toryne               toryn
-    toss                 toss
-    tossed               toss
-    tosseth              tosseth
-    tossing              toss
-    tot                  tot
-    total                total
-    totally              total
-    tott                 tott
-    tottered             totter
-    totters              totter
-    tou                  tou
-    touch                touch
-    touched              touch
-    touches              touch
-    toucheth             toucheth
-    touching             touch
-    touchstone           touchston
-    tough                tough
-    tougher              tougher
-    toughness            tough
-    touraine             tourain
-    tournaments          tournament
-    tours                tour
-    tous                 tou
-    tout                 tout
-    touze                touz
-    tow                  tow
-    toward               toward
-    towardly             towardli
-    towards              toward
-    tower                tower
-    towering             tower
-    towers               tower
-    town                 town
-    towns                town
-    township             township
-    townsman             townsman
-    townsmen             townsmen
-    towton               towton
-    toy                  toi
-    toys                 toi
-    trace                trace
-    traces               trace
-    track                track
-    tract                tract
-    tractable            tractabl
-    trade                trade
-    traded               trade
-    traders              trader
-    trades               trade
-    tradesman            tradesman
-    tradesmen            tradesmen
-    trading              trade
-    tradition            tradit
-    traditional          tradit
-    traduc               traduc
-    traduced             traduc
-    traducement          traduc
-    traffic              traffic
-    traffickers          traffick
-    traffics             traffic
-    tragedian            tragedian
-    tragedians           tragedian
-    tragedies            tragedi
-    tragedy              tragedi
-    tragic               tragic
-    tragical             tragic
-    trail                trail
-    train                train
-    trained              train
-    training             train
-    trains               train
-    trait                trait
-    traitor              traitor
-    traitorly            traitorli
-    traitorous           traitor
-    traitorously         traitor
-    traitors             traitor
-    traitress            traitress
-    traject              traject
-    trammel              trammel
-    trample              trampl
-    trampled             trampl
-    trampling            trampl
-    tranc                tranc
-    trance               tranc
-    tranio               tranio
-    tranquil             tranquil
-    tranquillity         tranquil
-    transcendence        transcend
-    transcends           transcend
-    transferred          transfer
-    transfigur           transfigur
-    transfix             transfix
-    transform            transform
-    transformation       transform
-    transformations      transform
-    transformed          transform
-    transgress           transgress
-    transgresses         transgress
-    transgressing        transgress
-    transgression        transgress
-    translate            translat
-    translated           translat
-    translates           translat
-    translation          translat
-    transmigrates        transmigr
-    transmutation        transmut
-    transparent          transpar
-    transport            transport
-    transportance        transport
-    transported          transport
-    transporting         transport
-    transports           transport
-    transpose            transpos
-    transshape           transshap
-    trap                 trap
-    trapp                trapp
-    trappings            trap
-    traps                trap
-    trash                trash
-    travail              travail
-    travails             travail
-    travel               travel
-    traveler             travel
-    traveling            travel
-    travell              travel
-    travelled            travel
-    traveller            travel
-    travellers           travel
-    travellest           travellest
-    travelling           travel
-    travels              travel
-    travers              traver
-    traverse             travers
-    tray                 trai
-    treacherous          treacher
-    treacherously        treacher
-    treachers            treacher
-    treachery            treacheri
-    tread                tread
-    treading             tread
-    treads               tread
-    treason              treason
-    treasonable          treason
-    treasonous           treason
-    treasons             treason
-    treasure             treasur
-    treasurer            treasur
-    treasures            treasur
-    treasuries           treasuri
-    treasury             treasuri
-    treat                treat
-    treaties             treati
-    treatise             treatis
-    treats               treat
-    treaty               treati
-    treble               trebl
-    trebled              trebl
-    trebles              trebl
-    trebonius            treboniu
-    tree                 tree
-    trees                tree
-    tremble              trembl
-    trembled             trembl
-    trembles             trembl
-    tremblest            tremblest
-    trembling            trembl
-    tremblingly          tremblingli
-    tremor               tremor
-    trempling            trempl
-    trench               trench
-    trenchant            trenchant
-    trenched             trench
-    trencher             trencher
-    trenchering          trencher
-    trencherman          trencherman
-    trenchers            trencher
-    trenches             trench
-    trenching            trench
-    trent                trent
-    tres                 tre
-    trespass             trespass
-    trespasses           trespass
-    tressel              tressel
-    tresses              tress
-    treys                trei
-    trial                trial
-    trials               trial
-    trib                 trib
-    tribe                tribe
-    tribes               tribe
-    tribulation          tribul
-    tribunal             tribun
-    tribune              tribun
-    tribunes             tribun
-    tributaries          tributari
-    tributary            tributari
-    tribute              tribut
-    tributes             tribut
-    trice                trice
-    trick                trick
-    tricking             trick
-    trickling            trickl
-    tricks               trick
-    tricksy              tricksi
-    trident              trident
-    tried                tri
-    trier                trier
-    trifle               trifl
-    trifled              trifl
-    trifler              trifler
-    trifles              trifl
-    trifling             trifl
-    trigon               trigon
-    trill                trill
-    trim                 trim
-    trimly               trimli
-    trimm                trimm
-    trimmed              trim
-    trimming             trim
-    trims                trim
-    trinculo             trinculo
-    trinculos            trinculo
-    trinkets             trinket
-    trip                 trip
-    tripartite           tripartit
-    tripe                tripe
-    triple               tripl
-    triplex              triplex
-    tripoli              tripoli
-    tripolis             tripoli
-    tripp                tripp
-    tripping             trip
-    trippingly           trippingli
-    trips                trip
-    tristful             trist
-    triton               triton
-    triumph              triumph
-    triumphant           triumphant
-    triumphantly         triumphantli
-    triumpher            triumpher
-    triumphers           triumpher
-    triumphing           triumph
-    triumphs             triumph
-    triumvir             triumvir
-    triumvirate          triumvir
-    triumvirs            triumvir
-    triumviry            triumviri
-    trivial              trivial
-    troat                troat
-    trod                 trod
-    trodden              trodden
-    troiant              troiant
-    troien               troien
-    troilus              troilu
-    troiluses            troilus
-    trojan               trojan
-    trojans              trojan
-    troll                troll
-    tromperies           tromperi
-    trompet              trompet
-    troop                troop
-    trooping             troop
-    troops               troop
-    trop                 trop
-    trophies             trophi
-    trophy               trophi
-    tropically           tropic
-    trot                 trot
-    troth                troth
-    trothed              troth
-    troths               troth
-    trots                trot
-    trotting             trot
-    trouble              troubl
-    troubled             troubl
-    troubler             troubler
-    troubles             troubl
-    troublesome          troublesom
-    troublest            troublest
-    troublous            troublou
-    trough               trough
-    trout                trout
-    trouts               trout
-    trovato              trovato
-    trow                 trow
-    trowel               trowel
-    trowest              trowest
-    troy                 troi
-    troyan               troyan
-    troyans              troyan
-    truant               truant
-    truce                truce
-    truckle              truckl
-    trudge               trudg
-    true                 true
-    trueborn             trueborn
-    truepenny            truepenni
-    truer                truer
-    truest               truest
-    truie                truie
-    trull                trull
-    trulls               trull
-    truly                truli
-    trump                trump
-    trumpery             trumperi
-    trumpet              trumpet
-    trumpeter            trumpet
-    trumpeters           trumpet
-    trumpets             trumpet
-    truncheon            truncheon
-    truncheoners         truncheon
-    trundle              trundl
-    trunk                trunk
-    trunks               trunk
-    trust                trust
-    trusted              trust
-    truster              truster
-    trusters             truster
-    trusting             trust
-    trusts               trust
-    trusty               trusti
-    truth                truth
-    truths               truth
-    try                  try
-    ts                   ts
-    tu                   tu
-    tuae                 tuae
-    tub                  tub
-    tubal                tubal
-    tubs                 tub
-    tuck                 tuck
-    tucket               tucket
-    tuesday              tuesdai
-    tuft                 tuft
-    tufts                tuft
-    tug                  tug
-    tugg                 tugg
-    tugging              tug
-    tuition              tuition
-    tullus               tullu
-    tully                tulli
-    tumble               tumbl
-    tumbled              tumbl
-    tumbler              tumbler
-    tumbling             tumbl
-    tumult               tumult
-    tumultuous           tumultu
-    tun                  tun
-    tune                 tune
-    tuneable             tuneabl
-    tuned                tune
-    tuners               tuner
-    tunes                tune
-    tunis                tuni
-    tuns                 tun
-    tupping              tup
-    turban               turban
-    turbans              turban
-    turbulence           turbul
-    turbulent            turbul
-    turd                 turd
-    turf                 turf
-    turfy                turfi
-    turk                 turk
-    turkey               turkei
-    turkeys              turkei
-    turkish              turkish
-    turks                turk
-    turlygod             turlygod
-    turmoil              turmoil
-    turmoiled            turmoil
-    turn                 turn
-    turnbull             turnbul
-    turncoat             turncoat
-    turncoats            turncoat
-    turned               turn
-    turneth              turneth
-    turning              turn
-    turnips              turnip
-    turns                turn
-    turph                turph
-    turpitude            turpitud
-    turquoise            turquois
-    turret               turret
-    turrets              turret
-    turtle               turtl
-    turtles              turtl
-    turvy                turvi
-    tuscan               tuscan
-    tush                 tush
-    tut                  tut
-    tutor                tutor
-    tutored              tutor
-    tutors               tutor
-    tutto                tutto
-    twain                twain
-    twang                twang
-    twangling            twangl
-    twas                 twa
-    tway                 twai
-    tweaks               tweak
-    tween                tween
-    twelfth              twelfth
-    twelve               twelv
-    twelvemonth          twelvemonth
-    twentieth            twentieth
-    twenty               twenti
-    twere                twere
-    twice                twice
-    twig                 twig
-    twiggen              twiggen
-    twigs                twig
-    twilight             twilight
-    twill                twill
-    twilled              twill
-    twin                 twin
-    twine                twine
-    twink                twink
-    twinkle              twinkl
-    twinkled             twinkl
-    twinkling            twinkl
-    twinn                twinn
-    twins                twin
-    twire                twire
-    twist                twist
-    twisted              twist
-    twit                 twit
-    twits                twit
-    twitting             twit
-    twixt                twixt
-    two                  two
-    twofold              twofold
-    twopence             twopenc
-    twopences            twopenc
-    twos                 two
-    twould               twould
-    tyb                  tyb
-    tybalt               tybalt
-    tybalts              tybalt
-    tyburn               tyburn
-    tying                ty
-    tyke                 tyke
-    tymbria              tymbria
-    type                 type
-    types                type
-    typhon               typhon
-    tyrannical           tyrann
-    tyrannically         tyrann
-    tyrannize            tyrann
-    tyrannous            tyrann
-    tyranny              tyranni
-    tyrant               tyrant
-    tyrants              tyrant
-    tyrian               tyrian
-    tyrrel               tyrrel
-    u                    u
-    ubique               ubiqu
-    udders               udder
-    udge                 udg
-    uds                  ud
-    uglier               uglier
-    ugliest              ugliest
-    ugly                 ugli
-    ulcer                ulcer
-    ulcerous             ulcer
-    ulysses              ulyss
-    um                   um
-    umber                umber
-    umbra                umbra
-    umbrage              umbrag
-    umfrevile            umfrevil
-    umpire               umpir
-    umpires              umpir
-    un                   un
-    unable               unabl
-    unaccommodated       unaccommod
-    unaccompanied        unaccompani
-    unaccustom           unaccustom
-    unaching             unach
-    unacquainted         unacquaint
-    unactive             unact
-    unadvis              unadvi
-    unadvised            unadvis
-    unadvisedly          unadvisedli
-    unagreeable          unagre
-    unanel               unanel
-    unanswer             unansw
-    unappeas             unappea
-    unapproved           unapprov
-    unapt                unapt
-    unaptness            unapt
-    unarm                unarm
-    unarmed              unarm
-    unarms               unarm
-    unassail             unassail
-    unassailable         unassail
-    unattainted          unattaint
-    unattempted          unattempt
-    unattended           unattend
-    unauspicious         unauspici
-    unauthorized         unauthor
-    unavoided            unavoid
-    unawares             unawar
-    unback               unback
-    unbak                unbak
-    unbanded             unband
-    unbar                unbar
-    unbarb               unbarb
-    unbashful            unbash
-    unbated              unbat
-    unbatter             unbatt
-    unbecoming           unbecom
-    unbefitting          unbefit
-    unbegot              unbegot
-    unbegotten           unbegotten
-    unbelieved           unbeliev
-    unbend               unbend
-    unbent               unbent
-    unbewail             unbewail
-    unbid                unbid
-    unbidden             unbidden
-    unbind               unbind
-    unbinds              unbind
-    unbitted             unbit
-    unbless              unbless
-    unblest              unblest
-    unbloodied           unbloodi
-    unblown              unblown
-    unbodied             unbodi
-    unbolt               unbolt
-    unbolted             unbolt
-    unbonneted           unbonnet
-    unbookish            unbookish
-    unborn               unborn
-    unbosom              unbosom
-    unbound              unbound
-    unbounded            unbound
-    unbow                unbow
-    unbowed              unbow
-    unbrac               unbrac
-    unbraced             unbrac
-    unbraided            unbraid
-    unbreathed           unbreath
-    unbred               unbr
-    unbreech             unbreech
-    unbridled            unbridl
-    unbroke              unbrok
-    unbruis              unbrui
-    unbruised            unbruis
-    unbuckle             unbuckl
-    unbuckles            unbuckl
-    unbuckling           unbuckl
-    unbuild              unbuild
-    unburden             unburden
-    unburdens            unburden
-    unburied             unburi
-    unburnt              unburnt
-    unburthen            unburthen
-    unbutton             unbutton
-    unbuttoning          unbutton
-    uncapable            uncap
-    uncape               uncap
-    uncase               uncas
-    uncasing             uncas
-    uncaught             uncaught
-    uncertain            uncertain
-    uncertainty          uncertainti
-    unchain              unchain
-    unchanging           unchang
-    uncharge             uncharg
-    uncharged            uncharg
-    uncharitably         uncharit
-    unchary              unchari
-    unchaste             unchast
-    uncheck              uncheck
-    unchilded            unchild
-    uncivil              uncivil
-    unclaim              unclaim
-    unclasp              unclasp
-    uncle                uncl
-    unclean              unclean
-    uncleanliness        uncleanli
-    uncleanly            uncleanli
-    uncleanness          unclean
-    uncles               uncl
-    unclew               unclew
-    unclog               unclog
-    uncoined             uncoin
-    uncolted             uncolt
-    uncomeliness         uncomeli
-    uncomfortable        uncomfort
-    uncompassionate      uncompassion
-    uncomprehensive      uncomprehens
-    unconfinable         unconfin
-    unconfirm            unconfirm
-    unconfirmed          unconfirm
-    unconquer            unconqu
-    unconquered          unconqu
-    unconsidered         unconsid
-    unconstant           unconst
-    unconstrain          unconstrain
-    unconstrained        unconstrain
-    uncontemn            uncontemn
-    uncontroll           uncontrol
-    uncorrected          uncorrect
-    uncounted            uncount
-    uncouple             uncoupl
-    uncourteous          uncourt
-    uncouth              uncouth
-    uncover              uncov
-    uncovered            uncov
-    uncropped            uncrop
-    uncross              uncross
-    uncrown              uncrown
-    unction              unction
-    unctuous             unctuou
-    uncuckolded          uncuckold
-    uncurable            uncur
-    uncurbable           uncurb
-    uncurbed             uncurb
-    uncurls              uncurl
-    uncurrent            uncurr
-    uncurse              uncurs
-    undaunted            undaunt
-    undeaf               undeaf
-    undeck               undeck
-    undeeded             undeed
-    under                under
-    underbearing         underbear
-    underborne           underborn
-    undercrest           undercrest
-    underfoot            underfoot
-    undergo              undergo
-    undergoes            undergo
-    undergoing           undergo
-    undergone            undergon
-    underground          underground
-    underhand            underhand
-    underlings           underl
-    undermine            undermin
-    underminers          undermin
-    underneath           underneath
-    underprizing         underpr
-    underprop            underprop
-    understand           understand
-    understandeth        understandeth
-    understanding        understand
-    understandings       understand
-    understands          understand
-    understood           understood
-    underta              underta
-    undertake            undertak
-    undertakeing         undertak
-    undertaker           undertak
-    undertakes           undertak
-    undertaking          undertak
-    undertakings         undertak
-    undertook            undertook
-    undervalu            undervalu
-    undervalued          undervalu
-    underwent            underw
-    underwrit            underwrit
-    underwrite           underwrit
-    undescried           undescri
-    undeserved           undeserv
-    undeserver           undeserv
-    undeservers          undeserv
-    undeserving          undeserv
-    undetermin           undetermin
-    undid                undid
-    undinted             undint
-    undiscernible        undiscern
-    undiscover           undiscov
-    undishonoured        undishonour
-    undispos             undispo
-    undistinguishable    undistinguish
-    undistinguished      undistinguish
-    undividable          undivid
-    undivided            undivid
-    undivulged           undivulg
-    undo                 undo
-    undoes               undo
-    undoing              undo
-    undone               undon
-    undoubted            undoubt
-    undoubtedly          undoubtedli
-    undream              undream
-    undress              undress
-    undressed            undress
-    undrown              undrown
-    unduteous            undut
-    undutiful            unduti
-    une                  un
-    uneared              unear
-    unearned             unearn
-    unearthly            unearthli
-    uneasines            uneasin
-    uneasy               uneasi
-    uneath               uneath
-    uneducated           uneduc
-    uneffectual          uneffectu
-    unelected            unelect
-    unequal              unequ
-    uneven               uneven
-    unexamin             unexamin
-    unexecuted           unexecut
-    unexpected           unexpect
-    unexperienc          unexperienc
-    unexperient          unexperi
-    unexpressive         unexpress
-    unfair               unfair
-    unfaithful           unfaith
-    unfallible           unfal
-    unfam                unfam
-    unfashionable        unfashion
-    unfasten             unfasten
-    unfather             unfath
-    unfathered           unfath
-    unfed                unf
-    unfeed               unfe
-    unfeeling            unfeel
-    unfeigned            unfeign
-    unfeignedly          unfeignedli
-    unfellowed           unfellow
-    unfelt               unfelt
-    unfenced             unfenc
-    unfilial             unfili
-    unfill               unfil
-    unfinish             unfinish
-    unfirm               unfirm
-    unfit                unfit
-    unfitness            unfit
-    unfix                unfix
-    unfledg              unfledg
-    unfold               unfold
-    unfolded             unfold
-    unfoldeth            unfoldeth
-    unfolding            unfold
-    unfolds              unfold
-    unfool               unfool
-    unforc               unforc
-    unforced             unforc
-    unforfeited          unforfeit
-    unfortified          unfortifi
-    unfortunate          unfortun
-    unfought             unfought
-    unfrequented         unfrequ
-    unfriended           unfriend
-    unfurnish            unfurnish
-    ungain               ungain
-    ungalled             ungal
-    ungart               ungart
-    ungarter             ungart
-    ungenitur            ungenitur
-    ungentle             ungentl
-    ungentleness         ungentl
-    ungently             ungent
-    ungird               ungird
-    ungodly              ungodli
-    ungor                ungor
-    ungot                ungot
-    ungotten             ungotten
-    ungovern             ungovern
-    ungracious           ungraci
-    ungrateful           ungrat
-    ungravely            ungrav
-    ungrown              ungrown
-    unguarded            unguard
-    unguem               unguem
-    unguided             unguid
-    unhack               unhack
-    unhair               unhair
-    unhallow             unhallow
-    unhallowed           unhallow
-    unhand               unhand
-    unhandled            unhandl
-    unhandsome           unhandsom
-    unhang               unhang
-    unhappied            unhappi
-    unhappily            unhappili
-    unhappiness          unhappi
-    unhappy              unhappi
-    unhardened           unharden
-    unharm               unharm
-    unhatch              unhatch
-    unheard              unheard
-    unhearts             unheart
-    unheedful            unheed
-    unheedfully          unheedfulli
-    unheedy              unheedi
-    unhelpful            unhelp
-    unhidden             unhidden
-    unholy               unholi
-    unhop                unhop
-    unhopefullest        unhopefullest
-    unhorse              unhors
-    unhospitable         unhospit
-    unhous               unhou
-    unhoused             unhous
-    unhurtful            unhurt
-    unicorn              unicorn
-    unicorns             unicorn
-    unimproved           unimprov
-    uninhabitable        uninhabit
-    uninhabited          uninhabit
-    unintelligent        unintellig
-    union                union
-    unions               union
-    unite                unit
-    united               unit
-    unity                uniti
-    universal            univers
-    universe             univers
-    universities         univers
-    university           univers
-    unjointed            unjoint
-    unjust               unjust
-    unjustice            unjustic
-    unjustly             unjustli
-    unkennel             unkennel
-    unkept               unkept
-    unkind               unkind
-    unkindest            unkindest
-    unkindly             unkindli
-    unkindness           unkind
-    unking               unk
-    unkinglike           unkinglik
-    unkiss               unkiss
-    unknit               unknit
-    unknowing            unknow
-    unknown              unknown
-    unlace               unlac
-    unlaid               unlaid
-    unlawful             unlaw
-    unlawfully           unlawfulli
-    unlearn              unlearn
-    unlearned            unlearn
-    unless               unless
-    unlesson             unlesson
-    unletter             unlett
-    unlettered           unlett
-    unlick               unlick
-    unlike               unlik
-    unlikely             unlik
-    unlimited            unlimit
-    unlineal             unlin
-    unlink               unlink
-    unload               unload
-    unloaded             unload
-    unloading            unload
-    unloads              unload
-    unlock               unlock
-    unlocks              unlock
-    unlook               unlook
-    unlooked             unlook
-    unloos               unloo
-    unloose              unloos
-    unlov                unlov
-    unloving             unlov
-    unluckily            unluckili
-    unlucky              unlucki
-    unmade               unmad
-    unmake               unmak
-    unmanly              unmanli
-    unmann               unmann
-    unmanner             unmann
-    unmannerd            unmannerd
-    unmannerly           unmannerli
-    unmarried            unmarri
-    unmask               unmask
-    unmasked             unmask
-    unmasking            unmask
-    unmasks              unmask
-    unmast               unmast
-    unmatch              unmatch
-    unmatchable          unmatch
-    unmatched            unmatch
-    unmeasurable         unmeasur
-    unmeet               unmeet
-    unmellowed           unmellow
-    unmerciful           unmerci
-    unmeritable          unmerit
-    unmeriting           unmerit
-    unminded             unmind
-    unmindfull           unmindful
-    unmingled            unmingl
-    unmitigable          unmitig
-    unmitigated          unmitig
-    unmix                unmix
-    unmoan               unmoan
-    unmov                unmov
-    unmoved              unmov
-    unmoving             unmov
-    unmuffles            unmuffl
-    unmuffling           unmuffl
-    unmusical            unmus
-    unmuzzle             unmuzzl
-    unmuzzled            unmuzzl
-    unnatural            unnatur
-    unnaturally          unnatur
-    unnaturalness        unnatur
-    unnecessarily        unnecessarili
-    unnecessary          unnecessari
-    unneighbourly        unneighbourli
-    unnerved             unnerv
-    unnoble              unnobl
-    unnoted              unnot
-    unnumb               unnumb
-    unnumber             unnumb
-    unowed               unow
-    unpack               unpack
-    unpaid               unpaid
-    unparagon            unparagon
-    unparallel           unparallel
-    unpartial            unparti
-    unpath               unpath
-    unpaved              unpav
-    unpay                unpai
-    unpeaceable          unpeac
-    unpeg                unpeg
-    unpeople             unpeopl
-    unpeopled            unpeopl
-    unperfect            unperfect
-    unperfectness        unperfect
-    unpick               unpick
-    unpin                unpin
-    unpink               unpink
-    unpitied             unpiti
-    unpitifully          unpitifulli
-    unplagu              unplagu
-    unplausive           unplaus
-    unpleas              unplea
-    unpleasant           unpleas
-    unpleasing           unpleas
-    unpolicied           unpolici
-    unpolish             unpolish
-    unpolished           unpolish
-    unpolluted           unpollut
-    unpossess            unpossess
-    unpossessing         unpossess
-    unpossible           unposs
-    unpractis            unpracti
-    unpregnant           unpregn
-    unpremeditated       unpremedit
-    unprepar             unprepar
-    unprepared           unprepar
-    unpress              unpress
-    unprevailing         unprevail
-    unprevented          unprev
-    unpriz               unpriz
-    unprizable           unpriz
-    unprofitable         unprofit
-    unprofited           unprofit
-    unproper             unprop
-    unproperly           unproperli
-    unproportion         unproport
-    unprovide            unprovid
-    unprovided           unprovid
-    unprovident          unprovid
-    unprovokes           unprovok
-    unprun               unprun
-    unpruned             unprun
-    unpublish            unpublish
-    unpurged             unpurg
-    unpurpos             unpurpo
-    unqualitied          unqual
-    unqueen              unqueen
-    unquestion           unquest
-    unquestionable       unquestion
-    unquiet              unquiet
-    unquietly            unquietli
-    unquietness          unquiet
-    unraised             unrais
-    unrak                unrak
-    unread               unread
-    unready              unreadi
-    unreal               unreal
-    unreasonable         unreason
-    unreasonably         unreason
-    unreclaimed          unreclaim
-    unreconciled         unreconcil
-    unreconciliable      unreconcili
-    unrecounted          unrecount
-    unrecuring           unrecur
-    unregarded           unregard
-    unregist             unregist
-    unrelenting          unrel
-    unremovable          unremov
-    unremovably          unremov
-    unreprievable        unrepriev
-    unresolv             unresolv
-    unrespected          unrespect
-    unrespective         unrespect
-    unrest               unrest
-    unrestor             unrestor
-    unrestrained         unrestrain
-    unreveng             unreveng
-    unreverend           unreverend
-    unreverent           unrever
-    unrevers             unrev
-    unrewarded           unreward
-    unrighteous          unright
-    unrightful           unright
-    unripe               unrip
-    unripp               unripp
-    unrivall             unrival
-    unroll               unrol
-    unroof               unroof
-    unroosted            unroost
-    unroot               unroot
-    unrough              unrough
-    unruly               unruli
-    unsafe               unsaf
-    unsaluted            unsalut
-    unsanctified         unsanctifi
-    unsatisfied          unsatisfi
-    unsavoury            unsavouri
-    unsay                unsai
-    unscalable           unscal
-    unscann              unscann
-    unscarr              unscarr
-    unschool             unschool
-    unscorch             unscorch
-    unscour              unscour
-    unscratch            unscratch
-    unseal               unseal
-    unseam               unseam
-    unsearch             unsearch
-    unseason             unseason
-    unseasonable         unseason
-    unseasonably         unseason
-    unseasoned           unseason
-    unseconded           unsecond
-    unsecret             unsecret
-    unseduc              unseduc
-    unseeing             unse
-    unseeming            unseem
-    unseemly             unseemli
-    unseen               unseen
-    unseminar            unseminar
-    unseparable          unsepar
-    unserviceable        unservic
-    unset                unset
-    unsettle             unsettl
-    unsettled            unsettl
-    unsever              unsev
-    unsex                unsex
-    unshak               unshak
-    unshaked             unshak
-    unshaken             unshaken
-    unshaped             unshap
-    unshapes             unshap
-    unsheath             unsheath
-    unsheathe            unsheath
-    unshorn              unshorn
-    unshout              unshout
-    unshown              unshown
-    unshrinking          unshrink
-    unshrubb             unshrubb
-    unshunn              unshunn
-    unshunnable          unshunn
-    unsifted             unsift
-    unsightly            unsightli
-    unsinew              unsinew
-    unsisting            unsist
-    unskilful            unskil
-    unskilfully          unskilfulli
-    unskillful           unskil
-    unslipping           unslip
-    unsmirched           unsmirch
-    unsoil               unsoil
-    unsolicited          unsolicit
-    unsorted             unsort
-    unsought             unsought
-    unsound              unsound
-    unsounded            unsound
-    unspeak              unspeak
-    unspeakable          unspeak
-    unspeaking           unspeak
-    unsphere             unspher
-    unspoke              unspok
-    unspoken             unspoken
-    unspotted            unspot
-    unsquar              unsquar
-    unstable             unstabl
-    unstaid              unstaid
-    unstain              unstain
-    unstained            unstain
-    unstanched           unstanch
-    unstate              unstat
-    unsteadfast          unsteadfast
-    unstooping           unstoop
-    unstringed           unstring
-    unstuff              unstuff
-    unsubstantial        unsubstanti
-    unsuitable           unsuit
-    unsuiting            unsuit
-    unsullied            unsulli
-    unsunn               unsunn
-    unsur                unsur
-    unsure               unsur
-    unsuspected          unsuspect
-    unsway               unswai
-    unswayable           unsway
-    unswayed             unswai
-    unswear              unswear
-    unswept              unswept
-    unsworn              unsworn
-    untainted            untaint
-    untalk               untalk
-    untangle             untangl
-    untangled            untangl
-    untasted             untast
-    untaught             untaught
-    untempering          untemp
-    untender             untend
-    untent               untent
-    untented             untent
-    unthankful           unthank
-    unthankfulness       unthank
-    unthink              unthink
-    unthought            unthought
-    unthread             unthread
-    unthrift             unthrift
-    unthrifts            unthrift
-    unthrifty            unthrifti
-    untie                unti
-    untied               unti
-    until                until
-    untimber             untimb
-    untimely             untim
-    untir                untir
-    untirable            untir
-    untired              untir
-    untitled             untitl
-    unto                 unto
-    untold               untold
-    untouch              untouch
-    untoward             untoward
-    untowardly           untowardli
-    untraded             untrad
-    untrain              untrain
-    untrained            untrain
-    untread              untread
-    untreasur            untreasur
-    untried              untri
-    untrimmed            untrim
-    untrod               untrod
-    untrodden            untrodden
-    untroubled           untroubl
-    untrue               untru
-    untrussing           untruss
-    untruth              untruth
-    untruths             untruth
-    untucked             untuck
-    untun                untun
-    untune               untun
-    untuneable           untun
-    untutor              untutor
-    untutored            untutor
-    untwine              untwin
-    unurg                unurg
-    unus                 unu
-    unused               unus
-    unusual              unusu
-    unvalued             unvalu
-    unvanquish           unvanquish
-    unvarnish            unvarnish
-    unveil               unveil
-    unveiling            unveil
-    unvenerable          unvener
-    unvex                unvex
-    unviolated           unviol
-    unvirtuous           unvirtu
-    unvisited            unvisit
-    unvulnerable         unvulner
-    unwares              unwar
-    unwarily             unwarili
-    unwash               unwash
-    unwatch              unwatch
-    unwearied            unweari
-    unwed                unw
-    unwedgeable          unwedg
-    unweeded             unweed
-    unweighed            unweigh
-    unweighing           unweigh
-    unwelcome            unwelcom
-    unwept               unwept
-    unwhipp              unwhipp
-    unwholesome          unwholesom
-    unwieldy             unwieldi
-    unwilling            unwil
-    unwillingly          unwillingli
-    unwillingness        unwilling
-    unwind               unwind
-    unwiped              unwip
-    unwise               unwis
-    unwisely             unwis
-    unwish               unwish
-    unwished             unwish
-    unwitted             unwit
-    unwittingly          unwittingli
-    unwonted             unwont
-    unwooed              unwoo
-    unworthier           unworthi
-    unworthiest          unworthiest
-    unworthily           unworthili
-    unworthiness         unworthi
-    unworthy             unworthi
-    unwrung              unwrung
-    unyok                unyok
-    unyoke               unyok
-    up                   up
-    upbraid              upbraid
-    upbraided            upbraid
-    upbraidings          upbraid
-    upbraids             upbraid
-    uphoarded            uphoard
-    uphold               uphold
-    upholdeth            upholdeth
-    upholding            uphold
-    upholds              uphold
-    uplift               uplift
-    uplifted             uplift
-    upmost               upmost
-    upon                 upon
-    upper                upper
-    uprear               uprear
-    upreared             uprear
-    upright              upright
-    uprighteously        upright
-    uprightness          upright
-    uprise               upris
-    uprising             upris
-    uproar               uproar
-    uproars              uproar
-    uprous               uprou
-    upshoot              upshoot
-    upshot               upshot
-    upside               upsid
-    upspring             upspr
-    upstairs             upstair
-    upstart              upstart
-    upturned             upturn
-    upward               upward
-    upwards              upward
-    urchin               urchin
-    urchinfield          urchinfield
-    urchins              urchin
-    urg                  urg
-    urge                 urg
-    urged                urg
-    urgent               urgent
-    urges                urg
-    urgest               urgest
-    urging               urg
-    urinal               urin
-    urinals              urin
-    urine                urin
-    urn                  urn
-    urns                 urn
-    urs                  ur
-    ursa                 ursa
-    ursley               urslei
-    ursula               ursula
-    urswick              urswick
-    us                   us
-    usage                usag
-    usance               usanc
-    usances              usanc
-    use                  us
-    used                 us
-    useful               us
-    useless              useless
-    user                 user
-    uses                 us
-    usest                usest
-    useth                useth
-    usher                usher
-    ushered              usher
-    ushering             usher
-    ushers               usher
-    using                us
-    usual                usual
-    usually              usual
-    usurer               usur
-    usurers              usur
-    usuries              usuri
-    usuring              usur
-    usurp                usurp
-    usurpation           usurp
-    usurped              usurp
-    usurper              usurp
-    usurpers             usurp
-    usurping             usurp
-    usurpingly           usurpingli
-    usurps               usurp
-    usury                usuri
-    ut                   ut
-    utensil              utensil
-    utensils             utensil
-    utility              util
-    utmost               utmost
-    utt                  utt
-    utter                utter
-    utterance            utter
-    uttered              utter
-    uttereth             uttereth
-    uttering             utter
-    utterly              utterli
-    uttermost            uttermost
-    utters               utter
-    uy                   uy
-    v                    v
-    va                   va
-    vacancy              vacanc
-    vacant               vacant
-    vacation             vacat
-    vade                 vade
-    vagabond             vagabond
-    vagabonds            vagabond
-    vagram               vagram
-    vagrom               vagrom
-    vail                 vail
-    vailed               vail
-    vailing              vail
-    vaillant             vaillant
-    vain                 vain
-    vainer               vainer
-    vainglory            vainglori
-    vainly               vainli
-    vainness             vain
-    vais                 vai
-    valanc               valanc
-    valance              valanc
-    vale                 vale
-    valence              valenc
-    valentine            valentin
-    valentinus           valentinu
-    valentio             valentio
-    valeria              valeria
-    valerius             valeriu
-    vales                vale
-    valiant              valiant
-    valiantly            valiantli
-    valiantness          valiant
-    validity             valid
-    vallant              vallant
-    valley               vallei
-    valleys              vallei
-    vally                valli
-    valor                valor
-    valorous             valor
-    valorously           valor
-    valour               valour
-    valu                 valu
-    valuation            valuat
-    value                valu
-    valued               valu
-    valueless            valueless
-    values               valu
-    valuing              valu
-    vane                 vane
-    vanish               vanish
-    vanished             vanish
-    vanishes             vanish
-    vanishest            vanishest
-    vanishing            vanish
-    vanities             vaniti
-    vanity               vaniti
-    vanquish             vanquish
-    vanquished           vanquish
-    vanquisher           vanquish
-    vanquishest          vanquishest
-    vanquisheth          vanquisheth
-    vant                 vant
-    vantage              vantag
-    vantages             vantag
-    vantbrace            vantbrac
-    vapians              vapian
-    vapor                vapor
-    vaporous             vapor
-    vapour               vapour
-    vapours              vapour
-    vara                 vara
-    variable             variabl
-    variance             varianc
-    variation            variat
-    variations           variat
-    varied               vari
-    variest              variest
-    variety              varieti
-    varld                varld
-    varlet               varlet
-    varletry             varletri
-    varlets              varlet
-    varletto             varletto
-    varnish              varnish
-    varrius              varriu
-    varro                varro
-    vary                 vari
-    varying              vari
-    vassal               vassal
-    vassalage            vassalag
-    vassals              vassal
-    vast                 vast
-    vastidity            vastid
-    vasty                vasti
-    vat                  vat
-    vater                vater
-    vaudemont            vaudemont
-    vaughan              vaughan
-    vault                vault
-    vaultages            vaultag
-    vaulted              vault
-    vaulting             vault
-    vaults               vault
-    vaulty               vaulti
-    vaumond              vaumond
-    vaunt                vaunt
-    vaunted              vaunt
-    vaunter              vaunter
-    vaunting             vaunt
-    vauntingly           vauntingli
-    vaunts               vaunt
-    vauvado              vauvado
-    vaux                 vaux
-    vaward               vaward
-    ve                   ve
-    veal                 veal
-    vede                 vede
-    vehemence            vehem
-    vehemency            vehem
-    vehement             vehement
-    vehor                vehor
-    veil                 veil
-    veiled               veil
-    veiling              veil
-    vein                 vein
-    veins                vein
-    vell                 vell
-    velure               velur
-    velutus              velutu
-    velvet               velvet
-    vendible             vendibl
-    venerable            vener
-    venereal             vener
-    venetia              venetia
-    venetian             venetian
-    venetians            venetian
-    veneys               venei
-    venge                veng
-    vengeance            vengeanc
-    vengeances           vengeanc
-    vengeful             veng
-    veni                 veni
-    venial               venial
-    venice               venic
-    venison              venison
-    venit                venit
-    venom                venom
-    venomous             venom
-    venomously           venom
-    vent                 vent
-    ventages             ventag
-    vented               vent
-    ventidius            ventidiu
-    ventricle            ventricl
-    vents                vent
-    ventur               ventur
-    venture              ventur
-    ventured             ventur
-    ventures             ventur
-    venturing            ventur
-    venturous            ventur
-    venue                venu
-    venus                venu
-    venuto               venuto
-    ver                  ver
-    verb                 verb
-    verba                verba
-    verbal               verbal
-    verbatim             verbatim
-    verbosity            verbos
-    verdict              verdict
-    verdun               verdun
-    verdure              verdur
-    vere                 vere
-    verefore             verefor
-    verg                 verg
-    verge                verg
-    vergers              verger
-    verges               verg
-    verier               verier
-    veriest              veriest
-    verified             verifi
-    verify               verifi
-    verily               verili
-    veritable            verit
-    verite               verit
-    verities             veriti
-    verity               veriti
-    vermilion            vermilion
-    vermin               vermin
-    vernon               vernon
-    verona               verona
-    veronesa             veronesa
-    versal               versal
-    verse                vers
-    verses               vers
-    versing              vers
-    vert                 vert
-    very                 veri
-    vesper               vesper
-    vessel               vessel
-    vessels              vessel
-    vestal               vestal
-    vestments            vestment
-    vesture              vestur
-    vetch                vetch
-    vetches              vetch
-    veux                 veux
-    vex                  vex
-    vexation             vexat
-    vexations            vexat
-    vexed                vex
-    vexes                vex
-    vexest               vexest
-    vexeth               vexeth
-    vexing               vex
-    vi                   vi
-    via                  via
-    vial                 vial
-    vials                vial
-    viand                viand
-    viands               viand
-    vic                  vic
-    vicar                vicar
-    vice                 vice
-    vicegerent           viceger
-    vicentio             vicentio
-    viceroy              viceroi
-    viceroys             viceroi
-    vices                vice
-    vici                 vici
-    vicious              viciou
-    viciousness          vicious
-    vict                 vict
-    victims              victim
-    victor               victor
-    victoress            victoress
-    victories            victori
-    victorious           victori
-    victors              victor
-    victory              victori
-    victual              victual
-    victuall             victual
-    victuals             victual
-    videlicet            videlicet
-    video                video
-    vides                vide
-    videsne              videsn
-    vidi                 vidi
-    vie                  vie
-    vied                 vi
-    vienna               vienna
-    view                 view
-    viewest              viewest
-    vieweth              vieweth
-    viewing              view
-    viewless             viewless
-    views                view
-    vigil                vigil
-    vigilance            vigil
-    vigilant             vigil
-    vigitant             vigit
-    vigour               vigour
-    vii                  vii
-    viii                 viii
-    vile                 vile
-    vilely               vile
-    vileness             vile
-    viler                viler
-    vilest               vilest
-    vill                 vill
-    village              villag
-    villager             villag
-    villagery            villageri
-    villages             villag
-    villain              villain
-    villainies           villaini
-    villainous           villain
-    villainously         villain
-    villains             villain
-    villainy             villaini
-    villanies            villani
-    villanous            villan
-    villany              villani
-    villiago             villiago
-    villian              villian
-    villianda            villianda
-    villians             villian
-    vinaigre             vinaigr
-    vincentio            vincentio
-    vincere              vincer
-    vindicative          vindic
-    vine                 vine
-    vinegar              vinegar
-    vines                vine
-    vineyard             vineyard
-    vineyards            vineyard
-    vint                 vint
-    vintner              vintner
-    viol                 viol
-    viola                viola
-    violate              violat
-    violated             violat
-    violates             violat
-    violation            violat
-    violator             violat
-    violence             violenc
-    violent              violent
-    violenta             violenta
-    violenteth           violenteth
-    violently            violent
-    violet               violet
-    violets              violet
-    viper                viper
-    viperous             viper
-    vipers               viper
-    vir                  vir
-    virgilia             virgilia
-    virgin               virgin
-    virginal             virgin
-    virginalling         virginal
-    virginity            virgin
-    virginius            virginiu
-    virgins              virgin
-    virgo                virgo
-    virtue               virtu
-    virtues              virtu
-    virtuous             virtuou
-    virtuously           virtuous
-    visag                visag
-    visage               visag
-    visages              visag
-    visard               visard
-    viscount             viscount
-    visible              visibl
-    visibly              visibl
-    vision               vision
-    visions              vision
-    visit                visit
-    visitation           visit
-    visitations          visit
-    visited              visit
-    visiting             visit
-    visitings            visit
-    visitor              visitor
-    visitors             visitor
-    visits               visit
-    visor                visor
-    vita                 vita
-    vitae                vita
-    vital                vital
-    vitement             vitement
-    vitruvio             vitruvio
-    vitx                 vitx
-    viva                 viva
-    vivant               vivant
-    vive                 vive
-    vixen                vixen
-    viz                  viz
-    vizaments            vizament
-    vizard               vizard
-    vizarded             vizard
-    vizards              vizard
-    vizor                vizor
-    vlouting             vlout
-    vocation             vocat
-    vocativo             vocativo
-    vocatur              vocatur
-    voce                 voce
-    voic                 voic
-    voice                voic
-    voices               voic
-    void                 void
-    voided               void
-    voiding              void
-    voke                 voke
-    volable              volabl
-    volant               volant
-    volivorco            volivorco
-    volley               vollei
-    volquessen           volquessen
-    volsce               volsc
-    volsces              volsc
-    volscian             volscian
-    volscians            volscian
-    volt                 volt
-    voltemand            voltemand
-    volubility           volubl
-    voluble              volubl
-    volume               volum
-    volumes              volum
-    volumnia             volumnia
-    volumnius            volumniu
-    voluntaries          voluntari
-    voluntary            voluntari
-    voluptuously         voluptu
-    voluptuousness       voluptu
-    vomissement          vomiss
-    vomit                vomit
-    vomits               vomit
-    vor                  vor
-    vore                 vore
-    vortnight            vortnight
-    vot                  vot
-    votaries             votari
-    votarist             votarist
-    votarists            votarist
-    votary               votari
-    votre                votr
-    vouch                vouch
-    voucher              voucher
-    vouchers             voucher
-    vouches              vouch
-    vouching             vouch
-    vouchsaf             vouchsaf
-    vouchsafe            vouchsaf
-    vouchsafed           vouchsaf
-    vouchsafes           vouchsaf
-    vouchsafing          vouchsaf
-    voudrais             voudrai
-    vour                 vour
-    vous                 vou
-    voutsafe             voutsaf
-    vow                  vow
-    vowed                vow
-    vowel                vowel
-    vowels               vowel
-    vowing               vow
-    vows                 vow
-    vox                  vox
-    voyage               voyag
-    voyages              voyag
-    vraiment             vraiment
-    vulcan               vulcan
-    vulgar               vulgar
-    vulgarly             vulgarli
-    vulgars              vulgar
-    vulgo                vulgo
-    vulnerable           vulner
-    vulture              vultur
-    vultures             vultur
-    vurther              vurther
-    w                    w
-    wad                  wad
-    waddled              waddl
-    wade                 wade
-    waded                wade
-    wafer                wafer
-    waft                 waft
-    waftage              waftag
-    wafting              waft
-    wafts                waft
-    wag                  wag
-    wage                 wage
-    wager                wager
-    wagers               wager
-    wages                wage
-    wagging              wag
-    waggish              waggish
-    waggling             waggl
-    waggon               waggon
-    waggoner             waggon
-    wagon                wagon
-    wagoner              wagon
-    wags                 wag
-    wagtail              wagtail
-    wail                 wail
-    wailful              wail
-    wailing              wail
-    wails                wail
-    wain                 wain
-    wainropes            wainrop
-    wainscot             wainscot
-    waist                waist
-    wait                 wait
-    waited               wait
-    waiter               waiter
-    waiteth              waiteth
-    waiting              wait
-    waits                wait
-    wak                  wak
-    wake                 wake
-    waked                wake
-    wakefield            wakefield
-    waken                waken
-    wakened              waken
-    wakes                wake
-    wakest               wakest
-    waking               wake
-    wales                wale
-    walk                 walk
-    walked               walk
-    walking              walk
-    walks                walk
-    wall                 wall
-    walled               wall
-    wallet               wallet
-    wallets              wallet
-    wallon               wallon
-    walloon              walloon
-    wallow               wallow
-    walls                wall
-    walnut               walnut
-    walter               walter
-    wan                  wan
-    wand                 wand
-    wander               wander
-    wanderer             wander
-    wanderers            wander
-    wandering            wander
-    wanders              wander
-    wands                wand
-    wane                 wane
-    waned                wane
-    wanes                wane
-    waning               wane
-    wann                 wann
-    want                 want
-    wanted               want
-    wanteth              wanteth
-    wanting              want
-    wanton               wanton
-    wantonly             wantonli
-    wantonness           wanton
-    wantons              wanton
-    wants                want
-    wappen               wappen
-    war                  war
-    warble               warbl
-    warbling             warbl
-    ward                 ward
-    warded               ward
-    warden               warden
-    warder               warder
-    warders              warder
-    wardrobe             wardrob
-    wardrop              wardrop
-    wards                ward
-    ware                 ware
-    wares                ware
-    warily               warili
-    warkworth            warkworth
-    warlike              warlik
-    warm                 warm
-    warmed               warm
-    warmer               warmer
-    warming              warm
-    warms                warm
-    warmth               warmth
-    warn                 warn
-    warned               warn
-    warning              warn
-    warnings             warn
-    warns                warn
-    warp                 warp
-    warped               warp
-    warr                 warr
-    warrant              warrant
-    warranted            warrant
-    warranteth           warranteth
-    warrantise           warrantis
-    warrantize           warrant
-    warrants             warrant
-    warranty             warranti
-    warren               warren
-    warrener             warren
-    warring              war
-    warrior              warrior
-    warriors             warrior
-    wars                 war
-    wart                 wart
-    warwick              warwick
-    warwickshire         warwickshir
-    wary                 wari
-    was                  wa
-    wash                 wash
-    washed               wash
-    washer               washer
-    washes               wash
-    washford             washford
-    washing              wash
-    wasp                 wasp
-    waspish              waspish
-    wasps                wasp
-    wassail              wassail
-    wassails             wassail
-    wast                 wast
-    waste                wast
-    wasted               wast
-    wasteful             wast
-    wasters              waster
-    wastes               wast
-    wasting              wast
-    wat                  wat
-    watch                watch
-    watched              watch
-    watchers             watcher
-    watches              watch
-    watchful             watch
-    watching             watch
-    watchings            watch
-    watchman             watchman
-    watchmen             watchmen
-    watchword            watchword
-    water                water
-    waterdrops           waterdrop
-    watered              water
-    waterfly             waterfli
-    waterford            waterford
-    watering             water
-    waterish             waterish
-    waterpots            waterpot
-    waterrugs            waterrug
-    waters               water
-    waterton             waterton
-    watery               wateri
-    wav                  wav
-    wave                 wave
-    waved                wave
-    waver                waver
-    waverer              waver
-    wavering             waver
-    waves                wave
-    waving               wave
-    waw                  waw
-    wawl                 wawl
-    wax                  wax
-    waxed                wax
-    waxen                waxen
-    waxes                wax
-    waxing               wax
-    way                  wai
-    waylaid              waylaid
-    waylay               waylai
-    ways                 wai
-    wayward              wayward
-    waywarder            wayward
-    waywardness          wayward
-    we                   we
-    weak                 weak
-    weaken               weaken
-    weakens              weaken
-    weaker               weaker
-    weakest              weakest
-    weakling             weakl
-    weakly               weakli
-    weakness             weak
-    weal                 weal
-    wealsmen             wealsmen
-    wealth               wealth
-    wealthiest           wealthiest
-    wealthily            wealthili
-    wealthy              wealthi
-    wealtlly             wealtlli
-    wean                 wean
-    weapon               weapon
-    weapons              weapon
-    wear                 wear
-    wearer               wearer
-    wearers              wearer
-    wearied              weari
-    wearies              weari
-    weariest             weariest
-    wearily              wearili
-    weariness            weari
-    wearing              wear
-    wearisome            wearisom
-    wears                wear
-    weary                weari
-    weasel               weasel
-    weather              weather
-    weathercock          weathercock
-    weathers             weather
-    weav                 weav
-    weave                weav
-    weaver               weaver
-    weavers              weaver
-    weaves               weav
-    weaving              weav
-    web                  web
-    wed                  wed
-    wedded               wed
-    wedding              wed
-    wedg                 wedg
-    wedged               wedg
-    wedges               wedg
-    wedlock              wedlock
-    wednesday            wednesdai
-    weed                 weed
-    weeded               weed
-    weeder               weeder
-    weeding              weed
-    weeds                weed
-    weedy                weedi
-    week                 week
-    weeke                week
-    weekly               weekli
-    weeks                week
-    ween                 ween
-    weening              ween
-    weep                 weep
-    weeper               weeper
-    weeping              weep
-    weepingly            weepingli
-    weepings             weep
-    weeps                weep
-    weet                 weet
-    weigh                weigh
-    weighed              weigh
-    weighing             weigh
-    weighs               weigh
-    weight               weight
-    weightier            weightier
-    weightless           weightless
-    weights              weight
-    weighty              weighti
-    weird                weird
-    welcom               welcom
-    welcome              welcom
-    welcomer             welcom
-    welcomes             welcom
-    welcomest            welcomest
-    welfare              welfar
-    welkin               welkin
-    well                 well
-    wells                well
-    welsh                welsh
-    welshman             welshman
-    welshmen             welshmen
-    welshwomen           welshwomen
-    wench                wench
-    wenches              wench
-    wenching             wench
-    wend                 wend
-    went                 went
-    wept                 wept
-    weraday              weradai
-    were                 were
-    wert                 wert
-    west                 west
-    western              western
-    westminster          westminst
-    westmoreland         westmoreland
-    westward             westward
-    wet                  wet
-    wether               wether
-    wetting              wet
-    wezand               wezand
-    whale                whale
-    whales               whale
-    wharf                wharf
-    wharfs               wharf
-    what                 what
-    whate                whate
-    whatever             whatev
-    whatsoe              whatso
-    whatsoever           whatsoev
-    whatsome             whatsom
-    whe                  whe
-    wheat                wheat
-    wheaten              wheaten
-    wheel                wheel
-    wheeling             wheel
-    wheels               wheel
-    wheer                wheer
-    wheeson              wheeson
-    wheezing             wheez
-    whelk                whelk
-    whelks               whelk
-    whelm                whelm
-    whelp                whelp
-    whelped              whelp
-    whelps               whelp
-    when                 when
-    whenas               whena
-    whence               whenc
-    whencesoever         whencesoev
-    whene                whene
-    whenever             whenev
-    whensoever           whensoev
-    where                where
-    whereabout           whereabout
-    whereas              wherea
-    whereat              whereat
-    whereby              wherebi
-    wherefore            wherefor
-    wherein              wherein
-    whereinto            whereinto
-    whereof              whereof
-    whereon              whereon
-    whereout             whereout
-    whereso              whereso
-    wheresoe             whereso
-    wheresoever          wheresoev
-    wheresome            wheresom
-    whereto              whereto
-    whereuntil           whereuntil
-    whereunto            whereunto
-    whereupon            whereupon
-    wherever             wherev
-    wherewith            wherewith
-    wherewithal          wherewith
-    whet                 whet
-    whether              whether
-    whetstone            whetston
-    whetted              whet
-    whew                 whew
-    whey                 whei
-    which                which
-    whiff                whiff
-    whiffler             whiffler
-    while                while
-    whiles               while
-    whilst               whilst
-    whin                 whin
-    whine                whine
-    whined               whine
-    whinid               whinid
-    whining              whine
-    whip                 whip
-    whipp                whipp
-    whippers             whipper
-    whipping             whip
-    whips                whip
-    whipster             whipster
-    whipstock            whipstock
-    whipt                whipt
-    whirl                whirl
-    whirled              whirl
-    whirligig            whirligig
-    whirling             whirl
-    whirlpool            whirlpool
-    whirls               whirl
-    whirlwind            whirlwind
-    whirlwinds           whirlwind
-    whisp                whisp
-    whisper              whisper
-    whispering           whisper
-    whisperings          whisper
-    whispers             whisper
-    whist                whist
-    whistle              whistl
-    whistles             whistl
-    whistling            whistl
-    whit                 whit
-    white                white
-    whitehall            whitehal
-    whitely              white
-    whiteness            white
-    whiter               whiter
-    whites               white
-    whitest              whitest
-    whither              whither
-    whiting              white
-    whitmore             whitmor
-    whitsters            whitster
-    whitsun              whitsun
-    whittle              whittl
-    whizzing             whizz
-    who                  who
-    whoa                 whoa
-    whoe                 whoe
-    whoever              whoever
-    whole                whole
-    wholesom             wholesom
-    wholesome            wholesom
-    wholly               wholli
-    whom                 whom
-    whoobub              whoobub
-    whoop                whoop
-    whooping             whoop
-    whor                 whor
-    whore                whore
-    whoremaster          whoremast
-    whoremasterly        whoremasterli
-    whoremonger          whoremong
-    whores               whore
-    whoreson             whoreson
-    whoresons            whoreson
-    whoring              whore
-    whorish              whorish
-    whose                whose
-    whoso                whoso
-    whosoe               whoso
-    whosoever            whosoev
-    why                  why
-    wi                   wi
-    wick                 wick
-    wicked               wick
-    wickednes            wickedn
-    wickedness           wicked
-    wicket               wicket
-    wicky                wicki
-    wid                  wid
-    wide                 wide
-    widens               widen
-    wider                wider
-    widow                widow
-    widowed              widow
-    widower              widow
-    widowhood            widowhood
-    widows               widow
-    wield                wield
-    wife                 wife
-    wight                wight
-    wights               wight
-    wild                 wild
-    wildcats             wildcat
-    wilder               wilder
-    wilderness           wilder
-    wildest              wildest
-    wildfire             wildfir
-    wildly               wildli
-    wildness             wild
-    wilds                wild
-    wiles                wile
-    wilful               wil
-    wilfull              wilful
-    wilfully             wilfulli
-    wilfulnes            wilfuln
-    wilfulness           wil
-    will                 will
-    willed               will
-    willers              willer
-    willeth              willeth
-    william              william
-    williams             william
-    willing              will
-    willingly            willingli
-    willingness          willing
-    willoughby           willoughbi
-    willow               willow
-    wills                will
-    wilt                 wilt
-    wiltshire            wiltshir
-    wimpled              wimpl
-    win                  win
-    wince                winc
-    winch                winch
-    winchester           winchest
-    wincot               wincot
-    wind                 wind
-    winded               wind
-    windgalls            windgal
-    winding              wind
-    windlasses           windlass
-    windmill             windmil
-    window               window
-    windows              window
-    windpipe             windpip
-    winds                wind
-    windsor              windsor
-    windy                windi
-    wine                 wine
-    wing                 wing
-    winged               wing
-    wingfield            wingfield
-    wingham              wingham
-    wings                wing
-    wink                 wink
-    winking              wink
-    winks                wink
-    winner               winner
-    winners              winner
-    winning              win
-    winnow               winnow
-    winnowed             winnow
-    winnows              winnow
-    wins                 win
-    winter               winter
-    winterly             winterli
-    winters              winter
-    wip                  wip
-    wipe                 wipe
-    wiped                wipe
-    wipes                wipe
-    wiping               wipe
-    wire                 wire
-    wires                wire
-    wiry                 wiri
-    wisdom               wisdom
-    wisdoms              wisdom
-    wise                 wise
-    wiselier             wiseli
-    wisely               wise
-    wiser                wiser
-    wisest               wisest
-    wish                 wish
-    wished               wish
-    wisher               wisher
-    wishers              wisher
-    wishes               wish
-    wishest              wishest
-    wisheth              wisheth
-    wishful              wish
-    wishing              wish
-    wishtly              wishtli
-    wisp                 wisp
-    wist                 wist
-    wit                  wit
-    witb                 witb
-    witch                witch
-    witchcraft           witchcraft
-    witches              witch
-    witching             witch
-    with                 with
-    withal               withal
-    withdraw             withdraw
-    withdrawing          withdraw
-    withdrawn            withdrawn
-    withdrew             withdrew
-    wither               wither
-    withered             wither
-    withering            wither
-    withers              wither
-    withheld             withheld
-    withhold             withhold
-    withholds            withhold
-    within               within
-    withold              withold
-    without              without
-    withstand            withstand
-    withstanding         withstand
-    withstood            withstood
-    witless              witless
-    witness              wit
-    witnesses            wit
-    witnesseth           witnesseth
-    witnessing           wit
-    wits                 wit
-    witted               wit
-    wittenberg           wittenberg
-    wittiest             wittiest
-    wittily              wittili
-    witting              wit
-    wittingly            wittingli
-    wittol               wittol
-    wittolly             wittolli
-    witty                witti
-    wiv                  wiv
-    wive                 wive
-    wived                wive
-    wives                wive
-    wiving               wive
-    wizard               wizard
-    wizards              wizard
-    wo                   wo
-    woe                  woe
-    woeful               woeful
-    woefull              woeful
-    woefullest           woefullest
-    woes                 woe
-    woful                woful
-    wolf                 wolf
-    wolfish              wolfish
-    wolsey               wolsei
-    wolves               wolv
-    wolvish              wolvish
-    woman                woman
-    womanhood            womanhood
-    womanish             womanish
-    womankind            womankind
-    womanly              womanli
-    womb                 womb
-    wombs                womb
-    womby                wombi
-    women                women
-    won                  won
-    woncot               woncot
-    wond                 wond
-    wonder               wonder
-    wondered             wonder
-    wonderful            wonder
-    wonderfully          wonderfulli
-    wondering            wonder
-    wonders              wonder
-    wondrous             wondrou
-    wondrously           wondrous
-    wont                 wont
-    wonted               wont
-    woo                  woo
-    wood                 wood
-    woodbine             woodbin
-    woodcock             woodcock
-    woodcocks            woodcock
-    wooden               wooden
-    woodland             woodland
-    woodman              woodman
-    woodmonger           woodmong
-    woods                wood
-    woodstock            woodstock
-    woodville            woodvil
-    wooed                woo
-    wooer                wooer
-    wooers               wooer
-    wooes                wooe
-    woof                 woof
-    wooing               woo
-    wooingly             wooingli
-    wool                 wool
-    woollen              woollen
-    woolly               woolli
-    woolsack             woolsack
-    woolsey              woolsei
-    woolward             woolward
-    woos                 woo
-    wor                  wor
-    worcester            worcest
-    word                 word
-    words                word
-    wore                 wore
-    worins               worin
-    work                 work
-    workers              worker
-    working              work
-    workings             work
-    workman              workman
-    workmanly            workmanli
-    workmanship          workmanship
-    workmen              workmen
-    works                work
-    worky                worki
-    world                world
-    worldlings           worldl
-    worldly              worldli
-    worlds               world
-    worm                 worm
-    worms                worm
-    wormwood             wormwood
-    wormy                wormi
-    worn                 worn
-    worried              worri
-    worries              worri
-    worry                worri
-    worrying             worri
-    worse                wors
-    worser               worser
-    worship              worship
-    worshipful           worship
-    worshipfully         worshipfulli
-    worshipp             worshipp
-    worshipper           worshipp
-    worshippers          worshipp
-    worshippest          worshippest
-    worships             worship
-    worst                worst
-    worsted              worst
-    wort                 wort
-    worth                worth
-    worthied             worthi
-    worthier             worthier
-    worthies             worthi
-    worthiest            worthiest
-    worthily             worthili
-    worthiness           worthi
-    worthless            worthless
-    worths               worth
-    worthy               worthi
-    worts                wort
-    wot                  wot
-    wots                 wot
-    wotting              wot
-    wouid                wouid
-    would                would
-    wouldest             wouldest
-    wouldst              wouldst
-    wound                wound
-    wounded              wound
-    wounding             wound
-    woundings            wound
-    woundless            woundless
-    wounds               wound
-    wouns                woun
-    woven                woven
-    wow                  wow
-    wrack                wrack
-    wrackful             wrack
-    wrangle              wrangl
-    wrangler             wrangler
-    wranglers            wrangler
-    wrangling            wrangl
-    wrap                 wrap
-    wrapp                wrapp
-    wraps                wrap
-    wrapt                wrapt
-    wrath                wrath
-    wrathful             wrath
-    wrathfully           wrathfulli
-    wraths               wrath
-    wreak                wreak
-    wreakful             wreak
-    wreaks               wreak
-    wreath               wreath
-    wreathed             wreath
-    wreathen             wreathen
-    wreaths              wreath
-    wreck                wreck
-    wrecked              wreck
-    wrecks               wreck
-    wren                 wren
-    wrench               wrench
-    wrenching            wrench
-    wrens                wren
-    wrest                wrest
-    wrested              wrest
-    wresting             wrest
-    wrestle              wrestl
-    wrestled             wrestl
-    wrestler             wrestler
-    wrestling            wrestl
-    wretch               wretch
-    wretchcd             wretchcd
-    wretched             wretch
-    wretchedness         wretched
-    wretches             wretch
-    wring                wring
-    wringer              wringer
-    wringing             wring
-    wrings               wring
-    wrinkle              wrinkl
-    wrinkled             wrinkl
-    wrinkles             wrinkl
-    wrist                wrist
-    wrists               wrist
-    writ                 writ
-    write                write
-    writer               writer
-    writers              writer
-    writes               write
-    writhled             writhl
-    writing              write
-    writings             write
-    writs                writ
-    written              written
-    wrong                wrong
-    wronged              wrong
-    wronger              wronger
-    wrongful             wrong
-    wrongfully           wrongfulli
-    wronging             wrong
-    wrongly              wrongli
-    wrongs               wrong
-    wronk                wronk
-    wrote                wrote
-    wroth                wroth
-    wrought              wrought
-    wrung                wrung
-    wry                  wry
-    wrying               wry
-    wt                   wt
-    wul                  wul
-    wye                  wye
-    x                    x
-    xanthippe            xanthipp
-    xi                   xi
-    xii                  xii
-    xiii                 xiii
-    xiv                  xiv
-    xv                   xv
-    y                    y
-    yard                 yard
-    yards                yard
-    yare                 yare
-    yarely               yare
-    yarn                 yarn
-    yaughan              yaughan
-    yaw                  yaw
-    yawn                 yawn
-    yawning              yawn
-    ycleped              yclepe
-    ycliped              yclipe
-    ye                   ye
-    yea                  yea
-    yead                 yead
-    year                 year
-    yearly               yearli
-    yearn                yearn
-    yearns               yearn
-    years                year
-    yeas                 yea
-    yeast                yeast
-    yedward              yedward
-    yell                 yell
-    yellow               yellow
-    yellowed             yellow
-    yellowing            yellow
-    yellowness           yellow
-    yellows              yellow
-    yells                yell
-    yelping              yelp
-    yeoman               yeoman
-    yeomen               yeomen
-    yerk                 yerk
-    yes                  ye
-    yesterday            yesterdai
-    yesterdays           yesterdai
-    yesternight          yesternight
-    yesty                yesti
-    yet                  yet
-    yew                  yew
-    yicld                yicld
-    yield                yield
-    yielded              yield
-    yielder              yielder
-    yielders             yielder
-    yielding             yield
-    yields               yield
-    yok                  yok
-    yoke                 yoke
-    yoked                yoke
-    yokefellow           yokefellow
-    yokes                yoke
-    yoketh               yoketh
-    yon                  yon
-    yond                 yond
-    yonder               yonder
-    yongrey              yongrei
-    yore                 yore
-    yorick               yorick
-    york                 york
-    yorkists             yorkist
-    yorks                york
-    yorkshire            yorkshir
-    you                  you
-    young                young
-    younger              younger
-    youngest             youngest
-    youngling            youngl
-    younglings           youngl
-    youngly              youngli
-    younker              younker
-    your                 your
-    yours                your
-    yourself             yourself
-    yourselves           yourselv
-    youth                youth
-    youthful             youth
-    youths               youth
-    youtli               youtli
-    zanies               zani
-    zany                 zani
-    zeal                 zeal
-    zealous              zealou
-    zeals                zeal
-    zed                  zed
-    zenelophon           zenelophon
-    zenith               zenith
-    zephyrs              zephyr
-    zir                  zir
-    zo                   zo
-    zodiac               zodiac
-    zodiacs              zodiac
-    zone                 zone
-    zounds               zound
-    zwagger              zwagger
-}                         
-
-# Create a full-text index to use for testing the stemmer.
-#
-db close
-sqlite3 db :memory:
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts1(word, tokenize Porter);
-}
-
-foreach {pfrom pto} $porter_test_data {
-  do_test fts1porter-$pfrom {
-    execsql {
-      DELETE FROM t1_term;
-      DELETE FROM t1_content;
-      INSERT INTO t1(word) VALUES($pfrom);
-      SELECT term FROM t1_term;
-    }
-  } $pto
-}
-
-finish_test
diff --git a/test/fts2.test b/test/fts2.test
deleted file mode 100644
index b1e2959366..0000000000
--- a/test/fts2.test
+++ /dev/null
@@ -1,67 +0,0 @@
-# 2008 July 22
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file runs all tests.
-#
-# $Id: fts2.test,v 1.2 2008/07/23 18:17:32 drh Exp $
-
-proc lshift {lvar} {
-  upvar $lvar l
-  set ret [lindex $l 0]
-  set l [lrange $l 1 end]
-  return $ret
-}
-while {[set arg [lshift argv]] != ""} {
-  switch -- $arg {
-    -sharedpagercache {
-      sqlite3_enable_shared_cache 1
-    }
-    -soak {
-       set G(issoak) 1
-    }
-    default {
-      set argv [linsert $argv 0 $arg]
-      break
-    }
-  }
-}
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  return
-}
-rename finish_test really_finish_test
-proc finish_test {} {}
-set G(isquick) 1
-
-set EXCLUDE {
-  fts2.test
-}
-
-# Files to include in the test.  If this list is empty then everything
-# that is not in the EXCLUDE list is run.
-#
-set INCLUDE {
-}
-
-foreach testfile [lsort -dictionary [glob $testdir/fts2*.test]] {
-  set tail [file tail $testfile]
-  if {[lsearch -exact $EXCLUDE $tail]>=0} continue
-  if {[llength $INCLUDE]>0 && [lsearch -exact $INCLUDE $tail]<0} continue
-  source $testfile
-  catch {db close}
-  if {$sqlite_open_file_count>0} {
-    puts "$tail did not close all files: $sqlite_open_file_count"
-    fail_test $tail
-    set sqlite_open_file_count 0
-  }
-}
-
-set sqlite_open_file_count 0
-really_finish_test
diff --git a/test/fts2a.test b/test/fts2a.test
deleted file mode 100644
index 2d1566fcce..0000000000
--- a/test/fts2a.test
+++ /dev/null
@@ -1,202 +0,0 @@
-# 2006 September 9
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS2 module.
-#
-# $Id: fts2a.test,v 1.2 2007/05/21 21:59:18 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Construct a full-text search table containing five keywords:
-# one, two, three, four, and five, in various combinations.  The
-# rowid for each will be a bitmask for the elements it contains.
-#
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1(content) VALUES('one');
-  INSERT INTO t1(content) VALUES('two');
-  INSERT INTO t1(content) VALUES('one two');
-  INSERT INTO t1(content) VALUES('three');
-  INSERT INTO t1(content) VALUES('one three');
-  INSERT INTO t1(content) VALUES('two three');
-  INSERT INTO t1(content) VALUES('one two three');
-  INSERT INTO t1(content) VALUES('four');
-  INSERT INTO t1(content) VALUES('one four');
-  INSERT INTO t1(content) VALUES('two four');
-  INSERT INTO t1(content) VALUES('one two four');
-  INSERT INTO t1(content) VALUES('three four');
-  INSERT INTO t1(content) VALUES('one three four');
-  INSERT INTO t1(content) VALUES('two three four');
-  INSERT INTO t1(content) VALUES('one two three four');
-  INSERT INTO t1(content) VALUES('five');
-  INSERT INTO t1(content) VALUES('one five');
-  INSERT INTO t1(content) VALUES('two five');
-  INSERT INTO t1(content) VALUES('one two five');
-  INSERT INTO t1(content) VALUES('three five');
-  INSERT INTO t1(content) VALUES('one three five');
-  INSERT INTO t1(content) VALUES('two three five');
-  INSERT INTO t1(content) VALUES('one two three five');
-  INSERT INTO t1(content) VALUES('four five');
-  INSERT INTO t1(content) VALUES('one four five');
-  INSERT INTO t1(content) VALUES('two four five');
-  INSERT INTO t1(content) VALUES('one two four five');
-  INSERT INTO t1(content) VALUES('three four five');
-  INSERT INTO t1(content) VALUES('one three four five');
-  INSERT INTO t1(content) VALUES('two three four five');
-  INSERT INTO t1(content) VALUES('one two three four five');
-}
-
-do_test fts2a-1.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts2a-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two'}
-} {3 7 11 15 19 23 27 31}
-do_test fts2a-1.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two one'}
-} {3 7 11 15 19 23 27 31}
-do_test fts2a-1.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two three'}
-} {7 15 23 31}
-do_test fts2a-1.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one three two'}
-} {7 15 23 31}
-do_test fts2a-1.6 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two three one'}
-} {7 15 23 31}
-do_test fts2a-1.7 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two one three'}
-} {7 15 23 31}
-do_test fts2a-1.8 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three one two'}
-} {7 15 23 31}
-do_test fts2a-1.9 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three two one'}
-} {7 15 23 31}
-do_test fts2a-1.10 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two THREE'}
-} {7 15 23 31}
-do_test fts2a-1.11 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '  ONE    Two   three  '}
-} {7 15 23 31}
-
-do_test fts2a-2.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one"'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts2a-2.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two"'}
-} {3 7 11 15 19 23 27 31}
-do_test fts2a-2.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"two one"'}
-} {}
-do_test fts2a-2.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two three"'}
-} {7 15 23 31}
-do_test fts2a-2.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three two"'}
-} {}
-do_test fts2a-2.6 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two three four"'}
-} {15 31}
-do_test fts2a-2.7 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three two four"'}
-} {}
-do_test fts2a-2.8 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three five"'}
-} {21}
-do_test fts2a-2.9 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three" five'}
-} {21 29}
-do_test fts2a-2.10 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five "one three"'}
-} {21 29}
-do_test fts2a-2.11 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five "one three" four'}
-} {29}
-do_test fts2a-2.12 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five four "one three"'}
-} {29}
-do_test fts2a-2.13 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one three" four five'}
-} {29}
-
-do_test fts2a-3.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts2a-3.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one -two'}
-} {1 5 9 13 17 21 25 29}
-do_test fts2a-3.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '-two one'}
-} {1 5 9 13 17 21 25 29}
-
-do_test fts2a-4.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one OR two'}
-} {1 2 3 5 6 7 9 10 11 13 14 15 17 18 19 21 22 23 25 26 27 29 30 31}
-do_test fts2a-4.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two" OR three'}
-} {3 4 5 6 7 11 12 13 14 15 19 20 21 22 23 27 28 29 30 31}
-do_test fts2a-4.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three OR "one two"'}
-} {3 4 5 6 7 11 12 13 14 15 19 20 21 22 23 27 28 29 30 31}
-do_test fts2a-4.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two OR three'}
-} {3 5 7 11 13 15 19 21 23 27 29 31}
-do_test fts2a-4.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three OR two one'}
-} {3 5 7 11 13 15 19 21 23 27 29 31}
-do_test fts2a-4.6 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one two OR three OR four'}
-} {3 5 7 9 11 13 15 19 21 23 25 27 29 31}
-do_test fts2a-4.7 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two OR three OR four one'}
-} {3 5 7 9 11 13 15 19 21 23 25 27 29 31}
-
-# Test the ability to handle NULL content
-#
-do_test fts2a-5.1 {
-  execsql {INSERT INTO t1(content) VALUES(NULL)}
-} {}
-do_test fts2a-5.2 {
-  set rowid [db last_insert_rowid]
-  execsql {SELECT content FROM t1 WHERE rowid=$rowid}
-} {{}}
-do_test fts2a-5.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH NULL}
-} {}
-
-# Test the ability to handle non-positive rowids
-#
-do_test fts2a-6.0 {
-  execsql {INSERT INTO t1(rowid, content) VALUES(0, 'four five')}
-} {}
-do_test fts2a-6.1 {
-  execsql {SELECT content FROM t1 WHERE rowid = 0}
-} {{four five}}
-do_test fts2a-6.2 {
-  execsql {INSERT INTO t1(rowid, content) VALUES(-1, 'three four')}
-} {}
-do_test fts2a-6.3 {
-  execsql {SELECT content FROM t1 WHERE rowid = -1}
-} {{three four}}
-do_test fts2a-6.4 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'four'}
-} {-1 0 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31}
-
-finish_test
diff --git a/test/fts2b.test b/test/fts2b.test
deleted file mode 100644
index 169cd8a0a3..0000000000
--- a/test/fts2b.test
+++ /dev/null
@@ -1,147 +0,0 @@
-# 2006 September 13
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS2 module.
-#
-# $Id: fts2b.test,v 1.1 2006/10/19 23:36:26 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Fill the full-text index "t1" with phrases in english, spanish,
-# and german.  For the i-th row, fill in the names for the bits
-# that are set in the value of i.  The least significant bit is
-# 1.  For example,  the value 5 is 101 in binary which will be
-# converted to "one three" in english.
-#
-proc fill_multilanguage_fulltext_t1 {} {
-  set english {one two three four five}
-  set spanish {un dos tres cuatro cinco}
-  set german {eine zwei drei vier funf}
-  
-  for {set i 1} {$i<=31} {incr i} {
-    set cmd "INSERT INTO t1 VALUES"
-    set vset {}
-    foreach lang {english spanish german} {
-      set words {}
-      for {set j 0; set k 1} {$j<5} {incr j; incr k $k} {
-        if {$k&$i} {lappend words [lindex [set $lang] $j]}
-      }
-      lappend vset "'$words'"
-    }
-    set sql "INSERT INTO t1(english,spanish,german) VALUES([join $vset ,])"
-    # puts $sql
-    db eval $sql
-  }
-}
-
-# Construct a full-text search table containing five keywords:
-# one, two, three, four, and five, in various combinations.  The
-# rowid for each will be a bitmask for the elements it contains.
-#
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(english,spanish,german);
-}
-fill_multilanguage_fulltext_t1
-
-do_test fts2b-1.1 {
-  execsql {SELECT rowid FROM t1 WHERE english MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts2b-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE spanish MATCH 'one'}
-} {}
-do_test fts2b-1.3 {
-  execsql {SELECT rowid FROM t1 WHERE german MATCH 'one'}
-} {}
-do_test fts2b-1.4 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'one'}
-} {1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31}
-do_test fts2b-1.5 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'one dos drei'}
-} {7 15 23 31}
-do_test fts2b-1.6 {
-  execsql {SELECT english, spanish, german FROM t1 WHERE rowid=1}
-} {one un eine}
-do_test fts2b-1.7 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH '"one un"'}
-} {}
-
-do_test fts2b-2.1 {
-  execsql {
-    CREATE VIRTUAL TABLE t2 USING fts2(from,to);
-    INSERT INTO t2([from],[to]) VALUES ('one two three', 'four five six');
-    SELECT [from], [to] FROM t2
-  }
-} {{one two three} {four five six}}
-
-
-# Compute an SQL string that contains the words one, two, three,... to
-# describe bits set in the value $i.  Only the lower 5 bits are examined.
-#
-proc wordset {i} {
-  set x {}
-  for {set j 0; set k 1} {$j<5} {incr j; incr k $k} {
-    if {$k&$i} {lappend x [lindex {one two three four five} $j]}
-  }
-  return '$x'
-}
-
-# Create a new FTS table with three columns:
-#
-#    norm:      words for the bits of rowid
-#    plusone:   words for the bits of rowid+1
-#    invert:    words for the bits of ~rowid
-#
-db eval {
-   CREATE VIRTUAL TABLE t4 USING fts2([norm],'plusone',"invert");
-}
-for {set i 1} {$i<=15} {incr i} {
-  set vset [list [wordset $i] [wordset [expr {$i+1}]] [wordset [expr {~$i}]]]
-  db eval "INSERT INTO t4(norm,plusone,invert) VALUES([join $vset ,]);"
-}
-
-do_test fts2b-4.1 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one'}
-} {1 3 5 7 9 11 13 15}
-do_test fts2b-4.2 {
-  execsql {SELECT rowid FROM t4 WHERE norm MATCH 'one'}
-} {1 3 5 7 9 11 13 15}
-do_test fts2b-4.3 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'one'}
-} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}
-do_test fts2b-4.4 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'plusone:one'}
-} {2 4 6 8 10 12 14}
-do_test fts2b-4.5 {
-  execsql {SELECT rowid FROM t4 WHERE plusone MATCH 'one'}
-} {2 4 6 8 10 12 14}
-do_test fts2b-4.6 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one plusone:two'}
-} {1 5 9 13}
-do_test fts2b-4.7 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one two'}
-} {1 3 5 7 9 11 13 15}
-do_test fts2b-4.8 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'plusone:two norm:one'}
-} {1 5 9 13}
-do_test fts2b-4.9 {
-  execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'two norm:one'}
-} {1 3 5 7 9 11 13 15}
-
-
-finish_test
diff --git a/test/fts2c.test b/test/fts2c.test
deleted file mode 100644
index cc6c9bbb5d..0000000000
--- a/test/fts2c.test
+++ /dev/null
@@ -1,1213 +0,0 @@
-# 2006 September 14
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS2 module.
-#
-# $Id: fts2c.test,v 1.1 2006/10/19 23:36:26 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Create a table of sample email data.   The data comes from email
-# archives of Enron executives that was published as part of the
-# litigation against that company.
-#
-do_test fts2c-1.1 {
-  db eval {
-    CREATE VIRTUAL TABLE email USING fts2([from],[to],subject,body);
-    BEGIN TRANSACTION;
-INSERT INTO email([from],[to],subject,body) VALUES('savita.puthigai@enron.com', 'traders.eol@enron.com, traders.eol@enron.com', 'EnronOnline- Change to Autohedge', 'Effective Monday, October 22, 2001 the following changes will be made to the Autohedge functionality on EnronOnline.
-
-The volume on the hedge will now respect the minimum volume and volume increment settings on the parent product. See rules below: 
-
-?	If the transaction volume on the child is less than half of the parent''s minimum volume no hedge will occur.
-?	If the transaction volume on the child is more than half the parent''s minimum volume but less than half the volume increment on the parent, the hedge will volume will be the parent''s minimum volume.
-?	For all other volumes, the same rounding rules will apply based on the volume increment on the parent product.
-
-Please see example below:
-
-Parent''s Settings:
-Minimum: 	5000
-Increment:  1000
-
-Volume on Autohedge transaction			Volume Hedged
-1      - 2499							0
-2500 - 5499							5000
-5500 - 6499							6000');
-INSERT INTO email([from],[to],subject,body) VALUES('dana.davis@enron.com', 'laynie.east@enron.com, lisa.king@enron.com, lisa.best@enron.com,', 'Leaving Early', 'FYI:  
-If it''s ok with everyone''s needs, I would like to leave @4pm. If you think 
-you will need my assistance past the 4 o''clock hour just let me know;  I''ll 
-be more than willing to stay.');
-INSERT INTO email([from],[to],subject,body) VALUES('enron_update@concureworkplace.com', 'louise.kitchen@enron.com', '<<Concur Expense Document>> - CC02.06.02', 'The following expense report is ready for approval:
-
-Employee Name: Christopher F. Calger
-Status last changed by: Mollie E. Gustafson Ms
-Expense Report Name: CC02.06.02
-Report Total: $3,972.93
-Amount Due Employee: $3,972.93
-
-
-To approve this expense report, click on the following link for Concur Expense.
-http://expensexms.enron.com');
-INSERT INTO email([from],[to],subject,body) VALUES('jeff.duff@enron.com', 'julie.johnson@enron.com', 'Work request', 'Julie,
-
-Could you print off the current work request report by 1:30 today?
-
-Gentlemen,
-
-I''d like to review this today at 1:30 in our office.  Also, could you provide 
-me with your activity reports so I can have Julie enter this information.
-
-JD');
-INSERT INTO email([from],[to],subject,body) VALUES('v.weldon@enron.com', 'gary.l.carrier@usa.dupont.com, scott.joyce@bankofamerica.com', 'Enron News', 'This could turn into something big.... 
-http://biz.yahoo.com/rf/010129/n29305829.html');
-INSERT INTO email([from],[to],subject,body) VALUES('mark.haedicke@enron.com', 'paul.simons@enron.com', 'Re: First Polish Deal!', 'Congrats!  Things seem to be building rapidly now on the Continent.  Mark');
-INSERT INTO email([from],[to],subject,body) VALUES('e..carter@enron.com', 't..robinson@enron.com', 'FW: Producers Newsletter 9-24-2001', '
-The producer lumber pricing sheet.
- -----Original Message-----
-From: 	Johnson, Jay  
-Sent:	Tuesday, October 16, 2001 3:42 PM
-To:	Carter, Karen E.
-Subject:	FW: Producers Newsletter 9-24-2001
-
-
-
- -----Original Message-----
-From: 	Daigre, Sergai  
-Sent:	Friday, September 21, 2001 8:33 PM
-Subject:	Producers Newsletter 9-24-2001
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('david.delainey@enron.com', 'kenneth.lay@enron.com', 'Greater Houston Partnership', 'Ken, in response to the letter from Mr Miguel San Juan, my suggestion would 
-be to offer up the Falcon for their use; however, given the tight time frame 
-and your recent visit with Mr. Fox that it would be difficult for either you 
-or me to participate.
-
-I spoke to Max and he agrees with this approach.
-
-I hope this meets with your approval.
-
-Regards
-Delainey');
-INSERT INTO email([from],[to],subject,body) VALUES('lachandra.fenceroy@enron.com', 'lindy.donoho@enron.com', 'FW: Bus Applications Meeting Follow Up', 'Lindy,
-
-Here is the original memo we discussed earlier.  Please provide any information that you may have.
-
-Your cooperation is greatly appreciated.
-
-Thanks,
-
-lachandra.fenceroy@enron.com
-713.853.3884
-877.498.3401 Pager
-
- -----Original Message-----
-From: 	Bisbee, Joanne  
-Sent:	Wednesday, September 26, 2001 7:50 AM
-To:	Fenceroy, LaChandra
-Subject:	FW: Bus Applications Meeting Follow Up
-
-Lachandra, Please get with David Duff today and see what this is about.  Who are our TW accounting business users?
-
- -----Original Message-----
-From: 	Koh, Wendy  
-Sent:	Tuesday, September 25, 2001 2:41 PM
-To:	Bisbee, Joanne
-Subject:	Bus Applications Meeting Follow Up
-
-Lisa brought up a TW change effective Nov 1.  It involves eliminating a turnback surcharge.  I have no other information, but you might check with the business folks for any system changes required.
-
-Wendy');
-INSERT INTO email([from],[to],subject,body) VALUES('danny.mccarty@enron.com', 'fran.fagan@enron.com', 'RE: worksheets', 'Fran,
-    If Julie''s merit needs to be lump sum, just move it over to that column.  Also, send me Eric Gadd''s sheets as well.  Thanks.
-Dan
-
- -----Original Message-----
-From: 	Fagan, Fran  
-Sent:	Thursday, December 20, 2001 11:10 AM
-To:	McCarty, Danny
-Subject:	worksheets
-
-As discussed, attached are your sheets for bonus and merit.
-
-Thanks,
-
-Fran Fagan
-Sr. HR Rep
-713.853.5219
-
-
- << File: McCartyMerit.xls >>  << File: mccartyBonusCommercial_UnP.xls >> 
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('bert.meyers@enron.com', 'shift.dl-portland@enron.com', 'OCTOBER SCHEDULE', 'TEAM,
-
-PLEASE SEND ME ANY REQUESTS THAT YOU HAVE FOR OCTOBER.  SO FAR I HAVE THEM FOR LEAF.  I WOULD LIKE TO HAVE IT DONE BY THE 15TH OF THE MONTH.  ANY QUESTIONS PLEASE GIVE ME A CALL.
-
-BERT');
-INSERT INTO email([from],[to],subject,body) VALUES('errol.mclaughlin@enron.com', 'john.arnold@enron.com, bilal.bajwa@enron.com, john.griffith@enron.com,', 'TRV Notification:  (NG - PROPT P/L - 09/27/2001)', 'The report named: NG - PROPT P/L <http://trv.corp.enron.com/linkFromExcel.asp?report_cd=11&report_name=NG+-+PROPT+P/L&category_cd=5&category_name=FINANCIAL&toc_hide=1&sTV1=5&TV1Exp=Y&current_efct_date=09/27/2001>, published as of 09/27/2001 is now available for viewing on the website.');
-INSERT INTO email([from],[to],subject,body) VALUES('patrice.mims@enron.com', 'calvin.eakins@enron.com', 'Re: Small business supply assistance', 'Hi Calvin
-
-
-I spoke with Rickey (boy, is he long-winded!!).  Gave him the name of our 
-credit guy, Russell Diamond.
-
-Thank for your help!');
-INSERT INTO email([from],[to],subject,body) VALUES('legal <.hall@enron.com>', 'stephanie.panus@enron.com', 'Termination update', 'City of Vernon and Salt River Project terminated their contracts.  I will fax these notices to you.');
-INSERT INTO email([from],[to],subject,body) VALUES('d..steffes@enron.com', 'richard.shapiro@enron.com', 'EES / ENA Government Affairs Staffing & Outside Services', 'Rick --
-
-Here is the information on staffing and outside services.  Call if you need anything else.
-
-Jim
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('gelliott@industrialinfo.com', 'pcopello@industrialinfo.com', 'ECAAR (Gavin), WSCC (Diablo Canyon), & NPCC (Seabrook)', 'Dear Power Outage Database Customer, 
-Attached you will find an excel document. The outages contained within are forced or rescheduled outages. Your daily delivery will still contain these outages. 
-In addition to the two excel documents, there is a dbf file that is formatted like your daily deliveries you receive nightly. This will enable you to load the data into your regular database. Any questions please let me know. Thanks. 
-Greg Elliott 
-IIR, Inc. 
-713-783-5147 x 3481 
-outages@industrialinfo.com 
-THE INFORMATION CONTAINED IN THIS E-MAIL IS LEGALLY PRIVILEGED AND CONFIDENTIAL INFORMATION INTENDED ONLY FOR THE USE OF THE INDIVIDUAL OR ENTITY NAMED ABOVE.  YOU ARE HEREBY NOTIFIED THAT ANY DISSEMINATION, DISTRIBUTION, OR COPY OF THIS E-MAIL TO UNAUTHORIZED ENTITIES IS STRICTLY PROHIBITED. IF YOU HAVE RECEIVED THIS 
-E-MAIL IN ERROR, PLEASE DELETE IT.
- - OUTAGE.dbf 
- - 111201R.xls 
- - 111201.xls ');
-INSERT INTO email([from],[to],subject,body) VALUES('enron.announcements@enron.com', 'all_ena_egm_eim@enron.com', 'EWS Brown Bag', 'MARK YOUR LUNCH CALENDARS NOW !
-
-You are invited to attend the EWS Brown Bag Lunch Series
-
-Featuring:   RAY BOWEN, COO
-
-Topic:  Enron Industrial Markets
-
-Thursday, March 15, 2001
-11:30 am - 12:30 pm
-EB 5 C2
-
-
-You bring your lunch,           Limited Seating
-We provide drinks and dessert.          RSVP  x 3-9610');
-INSERT INTO email([from],[to],subject,body) VALUES('chris.germany@enron.com', 'ingrid.immer@williams.com', 'Re: About St Pauls', 'Sounds good to me.  I bet this is next to the Warick?? Hotel.
-
-
-
-
-"Immer, Ingrid" <Ingrid.Immer@Williams.com> on 12/21/2000 11:48:47 AM
-To: "''chris.germany@enron.com''" <chris.germany@enron.com>
-cc:  
-Subject: About St Pauls
-
-
-
-
- <<About St Pauls.url>>  
-? 
-?http://www.stpaulshouston.org/about.html 
-
-Chris, 
-
-I like the looks of this place.? What do you think about going here Christmas 
-eve?? They have an 11:00 a.m. service and a candlelight service at 5:00 p.m., 
-among others.
-
-Let me know.?? ii 
-
- - About St Pauls.url
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('nas@cpuc.ca.gov', 'skatz@sempratrading.com, kmccrea@sablaw.com, thompson@wrightlaw.com,', 'Reply Brief filed July 31, 2000', ' - CPUC01-#76371-v1-Revised_Reply_Brief__Due_today_7_31_.doc');
-INSERT INTO email([from],[to],subject,body) VALUES('gascontrol@aglresources.com', 'dscott4@enron.com, lcampbel@enron.com', 'Alert Posted 10:00 AM November 20,2000: E-GAS Request Reminder', 'Alert Posted 10:00 AM November 20,2000: E-GAS Request Reminder
-As discussed in the Winter Operations Meeting on Sept.29,2000, 
-E-Gas(Emergency Gas) will not be offered this winter as a service from AGLC.  
-Marketers and Poolers can receive gas via Peaking and IBSS nominations(daisy 
-chain) from other marketers up to the 6 p.m. Same Day 2 nomination cycle.
-');
-INSERT INTO email([from],[to],subject,body) VALUES('dutch.quigley@enron.com', 'rwolkwitz@powermerchants.com', '', ' 
-
-Here is a goody for you');
-INSERT INTO email([from],[to],subject,body) VALUES('ryan.o''rourke@enron.com', 'k..allen@enron.com, randy.bhatia@enron.com, frank.ermis@enron.com,', 'TRV Notification:  (West VaR - 11/07/2001)', 'The report named: West VaR <http://trv.corp.enron.com/linkFromExcel.asp?report_cd=36&report_name=West+VaR&category_cd=2&category_name=WEST&toc_hide=1&sTV1=2&TV1Exp=Y&current_efct_date=11/07/2001>, published as of 11/07/2001 is now available for viewing on the website.');
-INSERT INTO email([from],[to],subject,body) VALUES('mjones7@txu.com', 'cstone1@txu.com, ggreen2@txu.com, timpowell@txu.com,', 'Enron / HPL Actuals for July 10, 2000', 'Teco Tap       10.000 / Enron ; 110.000 / HPL IFERC
-
-LS HPL LSK IC       30.000 / Enron
-');
-INSERT INTO email([from],[to],subject,body) VALUES('susan.pereira@enron.com', 'kkw816@aol.com', 'soccer practice', 'Kathy-
-
-Is it safe to assume that practice is cancelled for tonight??
-
-Susan Pereira');
-INSERT INTO email([from],[to],subject,body) VALUES('mark.whitt@enron.com', 'barry.tycholiz@enron.com', 'Huber Internal Memo', 'Please look at this.  I didn''t know how deep to go with the desk.  Do you think this works.
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('m..forney@enron.com', 'george.phillips@enron.com', '', 'George,
-Give me a call and we will further discuss opportunities on the 13st floor.
-
-Thanks,
-JMForney
-3-7160');
-INSERT INTO email([from],[to],subject,body) VALUES('brad.mckay@enron.com', 'angusmcka@aol.com', 'Re: (no subject)', 'not yet');
-INSERT INTO email([from],[to],subject,body) VALUES('adam.bayer@enron.com', 'jonathan.mckay@enron.com', 'FW: Curve Fetch File', 'Here is the curve fetch file sent to me.  It has plenty of points in it.  If you give me a list of which ones you need we may be able to construct a secondary worksheet to vlookup the values.
-
-adam
-35227
-
-
- -----Original Message-----
-From: 	Royed, Jeff  
-Sent:	Tuesday, September 25, 2001 11:37 AM
-To:	Bayer, Adam
-Subject:	Curve Fetch File
-
-Let me know if it works.   It may be required to have a certain version of Oracle for it to work properly.
-
- 
-
-Jeff Royed
-Enron 
-Energy Operations
-Phone: 713-853-5295');
-INSERT INTO email([from],[to],subject,body) VALUES('matt.smith@enron.com', 'yan.wang@enron.com', 'Report Formats', 'Yan,
-
-The merged reports look great.  I believe the only orientation changes are to 
-"unmerge" the following six reports:  
-
-31 Keystone Receipts
-15 Questar Pipeline
-40 Rockies Production
-22 West_2
-23 West_3
-25 CIG_WIC
-
-The orientation of the individual reports should be correct.  Thanks.
-
-Mat
-
-PS.  Just a reminder to add the "*" by the title of calculated points.');
-INSERT INTO email([from],[to],subject,body) VALUES('michelle.lokay@enron.com', 'jimboman@bigfoot.com', 'Egyptian Festival', '---------------------- Forwarded by Michelle Lokay/ET&S/Enron on 09/07/2000 
-10:08 AM ---------------------------
-
-
-"Karkour, Randa" <Randa.Karkour@COMPAQ.com> on 09/07/2000 09:01:04 AM
-To: "''Agheb (E-mail)" <Agheb@aol.com>, "Leila Mankarious (E-mail)" 
-<Leila_Mankarious@mhhs.org>, "''Marymankarious (E-mail)" 
-<marymankarious@aol.com>, "Michelle lokay (E-mail)" <mlokay@enron.com>, "Ramy 
-Mankarious (E-mail)" <Mankarious@aol.com>
-cc:  
-
-Subject: Egyptian Festival
-
-
- <<Egyptian Festival.url>>
-
- http://www.egyptianfestival.com/
-
- - Egyptian Festival.url
-');
-INSERT INTO email([from],[to],subject,body) VALUES('errol.mclaughlin@enron.com', 'sherry.dawson@enron.com', 'Urgent!!! --- New EAST books', 'This has to be done..................................
-
-Thanks
----------------------- Forwarded by Errol McLaughlin/Corp/Enron on 12/20/2000 
-08:39 AM ---------------------------
-   
-	
-	
-	From:  William Kelly @ ECT                           12/20/2000 08:31 AM
-	
-
-To: Kam Keiser/HOU/ECT@ECT, Darron C Giron/HOU/ECT@ECT, David 
-Baumbach/HOU/ECT@ECT, Errol McLaughlin/Corp/Enron@ENRON
-cc: Kimat Singla/HOU/ECT@ECT, Kulvinder Fowler/NA/Enron@ENRON, Kyle R 
-Lilly/HOU/ECT@ECT, Jeff Royed/Corp/Enron@ENRON, Alejandra 
-Chavez/NA/Enron@ENRON, Crystal Hyde/HOU/ECT@ECT 
-
-Subject: New EAST books
-
-We have new book names in TAGG for our intramonth portfolios and it is 
-extremely important that any deal booked to the East is communicated quickly 
-to someone on my team.  I know it will take some time for the new names to 
-sink in and I do not want us to miss any positions or P&L.  
-
-Thanks for your help on this.
-
-New:
-Scott Neal :         East Northeast
-Dick Jenkins:     East Marketeast
-
-WK 
-');
-INSERT INTO email([from],[to],subject,body) VALUES('david.forster@enron.com', 'eol.wide@enron.com', 'Change to Stack Manager', 'Effective immediately, there is a change to the Stack Manager which will 
-affect any Inactive Child.
-
-An inactive Child with links to Parent products will not have their 
-calculated prices updated until the Child product is Activated.
-
-When the Child Product is activated, the price will be recalculated and 
-updated BEFORE it is displayed on the web.
-
-This means that if you are inputting a basis price on a Child product, you 
-will not see the final, calculated price until you Activate the product, at 
-which time the customer will also see it.
-
-If you have any questions, please contact the Help Desk on:
-
-Americas: 713 853 4357
-Europe: + 44 (0) 20 7783 7783
-Asia/Australia: +61 2 9229 2300
-
-Dave');
-INSERT INTO email([from],[to],subject,body) VALUES('vince.kaminski@enron.com', 'jhh1@email.msn.com', 'Re: Light reading - see pieces beginning on page 7', 'John,
-
-I saw it. Very interesting.
-
-Vince
-
-
-
-
-
-"John H Herbert" <jhh1@email.msn.com> on 07/28/2000 08:38:08 AM
-To: "Vince J Kaminski" <Vince_J_Kaminski@enron.com>
-cc:  
-Subject: Light reading - see pieces beginning on page 7
-
-
-Cheers and have a nice weekend,
-
-
-JHHerbert
-
-
-
-
- - gd000728.pdf
-
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('matthew.lenhart@enron.com', 'mmmarcantel@equiva.com', 'RE:', 'i will try to line up a pig for you ');
-INSERT INTO email([from],[to],subject,body) VALUES('jae.black@enron.com', 'claudette.harvey@enron.com, chaun.roberts@enron.com, judy.martinez@enron.com,', 'Disaster Recovery Equipment', 'As a reminder...there are several pieces of equipment that are set up on the 30th Floor, as well as on our floor, for the Disaster Recovery Team.  PLEASE DO NOT TAKE, BORROW OR USE this equipment.  Should you need to use another computer system, other than yours, or make conference calls please work with your Assistant to help find or set up equipment for you to use.
-
-Thanks for your understanding in this matter.
-
-T.Jae Black
-East Power Trading
-Assistant to Kevin Presto
-off. 713-853-5800
-fax 713-646-8272
-cell 713-539-4760');
-INSERT INTO email([from],[to],subject,body) VALUES('eric.bass@enron.com', 'dale.neuner@enron.com', '5 X 24', 'Dale,
-
-Have you heard anything more on the 5 X 24s?  We would like to get this 
-product out ASAP.
-
-
-Thanks,
-
-Eric');
-INSERT INTO email([from],[to],subject,body) VALUES('messenger@smartreminders.com', 'm..tholt@enron.com', '10% Coupon - PrintPal Printer Cartridges - 100% Guaranteed', '[IMAGE]
-[IMAGE][IMAGE][IMAGE] 
-Dear  SmartReminders Member,
-       [IMAGE]         [IMAGE]        [IMAGE]     [IMAGE]    [IMAGE]    [IMAGE]        [IMAGE]      [IMAGE]     	
-
-
-  
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
- 
-
-We respect  your privacy and are a Certified Participant of the BBBOnLine
- Privacy Program.  To be removed from future offers,click  here. 
-SmartReminders.com  is a permission based service. To unsubscribe click  here .  ');
-INSERT INTO email([from],[to],subject,body) VALUES('benjamin.rogers@enron.com', 'mark.bernstein@enron.com', '', 'The guy you are talking about left CIN under a "cloud of suspicion" sort of 
-speak.  He was the one who got into several bad deals and PPA''s in California 
-for CIN, thus he left on a bad note.  Let me know if you need more detail 
-than that, I felt this was the type of info you were looking for.  Thanks!
-Ben');
-INSERT INTO email([from],[to],subject,body) VALUES('enron_update@concureworkplace.com', 'michelle.cash@enron.com', 'Expense Report Receipts Not Received', 'Employee Name: Michelle Cash
-Report Name:   Houston Cellular 8-11-01
-Report Date:   12/13/01
-Report ID:     594D37C9ED2111D5B452
-Submitted On:  12/13/01
-
-You are only allowed 2 reports with receipts outstanding.  Your expense reports will not be paid until you meet this requirement.');
-INSERT INTO email([from],[to],subject,body) VALUES('susan.mara@enron.com', 'ray.alvarez@enron.com, mark.palmer@enron.com, karen.denne@enron.com,', 'CAISO Emergency Motion -- to discontinue market-based rates for', 'FYI.  the latest broadside against the generators.
-
-Sue Mara
-Enron Corp.
-Tel: (415) 782-7802
-Fax:(415) 782-7854
------ Forwarded by Susan J Mara/NA/Enron on 06/08/2001 12:24 PM -----
-
-
-	"Milner, Marcie" <MMilner@coral-energy.com> 06/08/2001 11:13 AM 	   To: "''smara@enron.com''" <smara@enron.com>  cc:   Subject: CAISO Emergency Motion	
-
-
-Sue, did you see this emergency motion the CAISO filed today?  Apparently
-they are requesting that FERC discontinue market-based rates immediately and
-grant refunds plus interest on the difference between cost-based rates and
-market revenues received back to May 2000.  They are requesting the
-commission act within 14 days.  Have you heard anything about what they are
-doing?
-
-Marcie
-
-http://www.caiso.com/docs/2001/06/08/200106081005526469.pdf 
-');
-INSERT INTO email([from],[to],subject,body) VALUES('fletcher.sturm@enron.com', 'eloy.escobar@enron.com', 'Re: General Brinks Position Meeting', 'Eloy,
-
-Who is General Brinks?
-
-Fletch');
-INSERT INTO email([from],[to],subject,body) VALUES('nailia.dindarova@enron.com', 'richard.shapiro@enron.com', 'Documents for Mark Frevert (on EU developments and lessons from', 'Rick,
-
-Here are the documents that Peter has prepared for Mark Frevert. 
-
-Nailia
----------------------- Forwarded by Nailia Dindarova/LON/ECT on 25/06/2001 
-16:36 ---------------------------
-
-
-Nailia Dindarova
-25/06/2001 15:36
-To: Michael Brown/Enron@EUEnronXGate
-cc: Ross Sankey/Enron@EUEnronXGate, Eric Shaw/ENRON@EUEnronXGate, Peter 
-Styles/LON/ECT@ECT 
-
-Subject: Documents for Mark Frevert (on EU developments and lessons from 
-California)
-
-Michael,
-
-
-These are the documents that Peter promised to give to you for Mark Frevert. 
-He has now handed them to him in person but asked me to transmit them 
-electronically to you, as well as Eric and Ross.
-
-Nailia
-
-
-
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('peggy.a.kostial@accenture.com', 'dave.samuels@enron.com', 'EOL-Accenture Deal Sheet', 'Dave -
-
-Attached are our comments and suggested changes. Please call to review.
-
-On the time line for completion, we have four critical steps to complete:
-     Finalize market analysis to refine business case, specifically
-     projected revenue stream
-     Complete counterparty surveying, including targeting 3 CPs for letters
-     of intent
-     Review Enron asset base for potential reuse/ licensing
-     Contract negotiations
-
-Joe will come back to us with an updated time line, but it is my
-expectation that we are still on the same schedule (we just begun week
-three) with possibly a week or so slippage.....contract negotiations will
-probably be the critical path.
-
-We will send our cut at the actual time line here shortly. Thanks,
-
-Peggy
-
-(See attached file: accenture-dealpoints v2.doc)
- - accenture-dealpoints v2.doc ');
-INSERT INTO email([from],[to],subject,body) VALUES('thomas.martin@enron.com', 'thomas.martin@enron.com', 'Re: Guadalupe Power Partners LP', '---------------------- Forwarded by Thomas A Martin/HOU/ECT on 03/20/2001 
-03:49 PM ---------------------------
-
-
-Thomas A Martin
-10/11/2000 03:55 PM
-To: Patrick Wade/HOU/ECT@ECT
-cc:  
-Subject: Re: Guadalupe Power Partners LP  
-
-The deal is physically served at Oasis Waha or Oasis Katy and is priced at 
-either HSC, Waha or Katytailgate GD at buyers option three days prior to 
-NYMEX  close.
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('judy.townsend@enron.com', 'dan.junek@enron.com, chris.germany@enron.com', 'Columbia Distribution''s Capacity Available for Release - Sum', '---------------------- Forwarded by Judy Townsend/HOU/ECT on 03/09/2001 11:04 
-AM ---------------------------
-
-
-agoddard@nisource.com on 03/08/2001 09:16:57 AM
-To: "        -         *Koch, Kent" <kkoch@nisource.com>, "        -         
-*Millar, Debra" <dmillar@nisource.com>, "        -         *Burke, Lynn" 
-<lburke@nisource.com>
-cc: "        -         *Heckathorn, Tom" <theckathorn@nisource.com> 
-Subject: Columbia Distribution''s Capacity Available for Release - Sum
-
-
-Attached is Columbia Distribution''s notice of capacity available for release
-for
-the summer of 2001 (Apr. 2001 through Oct. 2001).
-
-Please note that the deadline for bids is 3:00pm EST on March 20, 2001.
-
-If you have any questions, feel free to contact any of the representatives
-listed
-at the bottom of the attachment.
-
-Aaron Goddard
-
-
-
-
- - 2001Summer.doc
-');
-INSERT INTO email([from],[to],subject,body) VALUES('rhonda.denton@enron.com', 'tim.belden@enron.com, dana.davis@enron.com, genia.fitzgerald@enron.com,', 'Split Rock Energy LLC', 'We have received the executed EEI contract from this CP dated 12/12/2000.  
-Copies will be distributed to Legal and Credit.');
-INSERT INTO email([from],[to],subject,body) VALUES('kerrymcelroy@dwt.com', 'jack.speer@alcoa.com, crow@millernash.com, michaelearly@earthlink.net,', 'Oral Argument Request', ' - Oral Argument Request.doc');
-INSERT INTO email([from],[to],subject,body) VALUES('mike.carson@enron.com', 'rlmichaelis@hormel.com', '', 'Did you come in town this wk end..... My new number at our house is : 
-713-668-3712...... my cell # is 281-381-7332
-
-the kid');
-INSERT INTO email([from],[to],subject,body) VALUES('cooper.richey@enron.com', 'trycooper@hotmail.com', 'FW: Contact Info', '
-
------Original Message-----
-From: Punja, Karim 
-Sent: Thursday, December 13, 2001 2:35 PM
-To: Richey, Cooper
-Subject: Contact Info
-
-
-Cooper,
-
-Its been a real pleasure working with you (even though it was for only a small amount of time)
-I hope we can stay in touch.
-
-Home# 234-0249
-email: kpunja@hotmail.com
-
-Take Care, 
-
-Karim.
-  ');
-INSERT INTO email([from],[to],subject,body) VALUES('bjm30@earthlink.net', 'mcguinn.k@enron.com, mcguinn.ian@enron.com, mcguinn.stephen@enron.com,', 'email address change', 'Hello all.
-
-I haven''t talked to many of you via email recently but I do want to give you
-my new address for your email file:
-
-    bjm30@earthlink.net
-
-I hope all is well.
-
-Brian McGuinn');
-INSERT INTO email([from],[to],subject,body) VALUES('shelley.corman@enron.com', 'steve.hotte@enron.com', 'Flat Panels', 'Can you please advise what is going on with the flat panels that we had planned to distribute to our gas logistics team.  It was in the budget and we had the okay, but now I''m hearing there is some hold-up & the units are stored on 44.
-
-Shelley');
-INSERT INTO email([from],[to],subject,body) VALUES('sara.davidson@enron.com', 'john.schwartzenburg@enron.com, scott.dieball@enron.com, recipients@enron.com,', '2001 Enron Law Conference (Distribution List 2)', '    Enron Law Conference
-
-San Antonio, Texas    May 2-4, 2001    Westin Riverwalk
-
-                   See attached memo for more details!!
-
-
-? Registration for the law conference this year will be handled through an 
-Online RSVP Form on the Enron Law Conference Website at 
-http://lawconference.corp.enron.com.  The website is still under construction 
-and will not be available until Thursday, March 15, 2001.  
-
-? We will send you another e-mail to confirm when the Law Conference Website 
-is operational. 
-
-? Please complete the Online RSVP Form as soon as it is available  and submit 
-it no later than Friday, March 30th.  
-
-
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('tori.kuykendall@enron.com', 'heath.b.taylor@accenture.com', 'Re:', 'hey - thats funny about john - he definitely remembers him - i''ll call pat 
-and let him know - we are coming on saturday - i just havent had a chance to 
-call you guys back --  looking forward to it -- i probably need the 
-directions again though');
-INSERT INTO email([from],[to],subject,body) VALUES('darron.giron@enron.com', 'bryce.baxter@enron.com', 'Re: Feedback for Audrey Cook', 'Bryce,
-
-I''ll get it done today.  
-
-DG    3-9573
-
-
-   
-	
-	
-	From:  Bryce Baxter                           06/12/2000 07:15 PM
-	
-
-To: Darron C Giron/HOU/ECT@ECT
-cc:  
-Subject: Feedback for Audrey Cook
-
-You were identified as a reviewer for Audrey Cook.  If possible, could you 
-complete her feedback by end of business Wednesday?  It will really help me 
-in the PRC process to have your input.  Thanks.
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('casey.evans@enron.com', 'stephanie.sever@enron.com', 'Gas EOL ID', 'Stephanie,
-
-In conjunction with the recent movement of several power traders, they are changing the names of their gas books as well.  The names of the new gas books and traders are as follows:
-
-PWR-NG-LT-SPP:  Mike Carson
-PWR-NG-LT-SERC:  Jeff King
-
-If you need to know their power desk to map their ID to their gas books, those desks are as follows:
-
-EPMI-LT-SPP:  Mike Carson
-EPMI-LT-SERC:  Jeff King
-
-I will be in training this afternoon, but will be back when class is over.  Let me know if you have any questions.
-
-Thanks for your help!
-Casey');
-INSERT INTO email([from],[to],subject,body) VALUES('darrell.schoolcraft@enron.com', 'david.roensch@enron.com, kimberly.watson@enron.com, michelle.lokay@enron.com,', 'Postings', 'Please see the attached.
-
-
-ds
-
-
-  
-
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('mcominsky@aol.com', 'cpatman@bracepatt.com, james_derrick@enron.com', 'Jurisprudence Luncheon', 'Carrin & Jim --
-
-It was an honor and a pleasure to meet both of you yesterday.  I know we will
-have fun working together on this very special event.
-
-Jeff left the jurisprudence luncheon lists for me before he left on vacation.
- I wasn''t sure whether he transmitted them to you as well.  Would you please
-advise me if you would like them sent to you?  I can email the MS Excel files
-or I can fax the hard copies to you.   Please advise what is most convenient.
-
-I plan to be in town through the holidays and can be reached by phone, email,
-or cell phone at any time.  My cell phone number is 713/705-4829.
-
-Thanks again for your interest in the ADL''s work.  Martin.
-
-Martin B. Cominsky
-Director, Southwest Region
-Anti-Defamation League
-713/627-3490, ext. 122
-713/627-2011 (fax)
-MCominsky@aol.com');
-INSERT INTO email([from],[to],subject,body) VALUES('phillip.love@enron.com', 'todagost@utmb.edu, gbsonnta@utmb.edu', 'New President', 'I had a little bird put a word in my ear.  Is there any possibility for Ben 
-Raimer to be Bush''s secretary of HHS?  Just curious about that infamous UTMB 
-rumor mill.  Hope things are well, happy holidays.
-PL');
-INSERT INTO email([from],[to],subject,body) VALUES('marie.heard@enron.com', 'ehamilton@fna.com', 'ISDA Master Agreement', 'Erin:
-
-Pursuant to your request, attached are the Schedule to the ISDA Master Agreement, together with Paragraph 13 to the ISDA Credit Support Annex.  Please let me know if you need anything else.  We look forward to hearing your comments.
-
-Marie
-
-Marie Heard
-Senior Legal Specialist
-Enron North America Corp.
-Phone:  (713) 853-3907
-Fax:  (713) 646-3490
-marie.heard@enron.com
-
-				 ');
-INSERT INTO email([from],[to],subject,body) VALUES('andrea.ring@enron.com', 'beverly.beaty@enron.com', 'Re: Tennessee Buy - Louis Dreyfus', 'Beverly -  once again thanks so much for your help on this.
-
-           
-
-                                                                     ');
-INSERT INTO email([from],[to],subject,body) VALUES('karolyn.criado@enron.com', 'j..bonin@enron.com, felicia.case@enron.com, b..clapp@enron.com,', 'Price List week of Oct. 8-9, 2001', '
-Please contact me if you have any questions regarding last weeks prices.
-
-Thank you,
-Karolyn Criado
-3-9441
-
-
- 
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('kevin.presto@enron.com', 'edward.baughman@enron.com, billy.braddock@enron.com', 'Associated', 'Please begin working on filling our Associated short position in 02.   I would like to take this risk off the books.
-
-In addition, please find out what a buy-out of VEPCO would cost us.   With Rogers transitioning to run our retail risk management, I would like to clean up our customer positions.
-
-We also need to continue to explore a JEA buy-out.
-
-Thanks.');
-INSERT INTO email([from],[to],subject,body) VALUES('stacy.dickson@enron.com', 'gregg.penman@enron.com', 'RE: Constellation TC 5-7-01', 'Gregg, 
-
-I am at home with a sick baby.  (Lots of fun!)  I will call you about this 
-tomorrow.
-
-Stacy');
-INSERT INTO email([from],[to],subject,body) VALUES('joe.quenet@enron.com', 'dfincher@utilicorp.com', '', 'hey big guy.....check this out.....
-
- w ww.gorelieberman-2000.com/');
-INSERT INTO email([from],[to],subject,body) VALUES('k..allen@enron.com', 'jacqestc@aol.com', '', 'Jacques,
-
-I sent you a fax of Kevin Kolb''s comments on the release.  The payoff on the note would be $36,248 ($36090(principal) + $158 (accrued interest)).
-This is assuming we wrap this up on Tuesday.  
-
-Please email to confirm that their changes are ok so I can set up a meeting on Tuesday to reach closure.
-
-Phillip');
-INSERT INTO email([from],[to],subject,body) VALUES('kourtney.nelson@enron.com', 'mike.swerzbin@enron.com', 'Adjusted L/R Balance', 'Mike,
-
-I placed the adjusted L/R Balance on the Enronwest site.  It is under the "Staff/Kourtney Nelson".  There are two links:  
-
-1)  "Adj L_R" is the same data/format from the weekly strategy meeting. 
-2)  "New Gen 2001_2002" link has all of the supply side info that is used to calculate the L/R balance
-	-Please note the Data Flag column, a value of "3" indicates the project was cancelled, on hold, etc and is not included in the calc.  
-
-Both of these sheets are interactive Excel spreadsheets and thus you can play around with the data as you please.  Also, James Bruce is working to get his gen report on the web.  That will help with your access to information on new gen.
-
-Please let me know if you have any questions or feedback,
-
-Kourtney
-
-
-
-Kourtney Nelson
-Fundamental Analysis 
-Enron North America
-(503) 464-8280
-kourtney.nelson@enron.com');
-INSERT INTO email([from],[to],subject,body) VALUES('d..thomas@enron.com', 'naveed.ahmed@enron.com', 'FW: Current Enron TCC Portfolio', '
-
------Original Message-----
-From: Grace, Rebecca M. 
-Sent: Monday, December 17, 2001 9:44 AM
-To: Thomas, Paul D.
-Cc: Cashion, Jim; Allen, Thresa A.; May, Tom
-Subject: RE: Current Enron TCC Portfolio
-
-
-Paul,
-
-I reviewed NY''s list.  I agree with all of their contracts numbers and mw amounts.
-
-Call if you have any more questions.
-
-Rebecca
-
-
-
- -----Original Message-----
-From: 	Thomas, Paul D.  
-Sent:	Monday, December 17, 2001 9:08 AM
-To:	Grace, Rebecca M.
-Subject:	FW: Current Enron TCC Portfolio
-
- << File: enrontccs.xls >> 
-Rebecca,
-Let me know if you see any differences.
-
-Paul
-X 3-0403
------Original Message-----
-From: Thomas, Paul D. 
-Sent: Monday, December 17, 2001 9:04 AM
-To: Ahmed, Naveed
-Subject: FW: Current Enron TCC Portfolio
-
-
-
-
------Original Message-----
-From: Thomas, Paul D. 
-Sent: Thursday, December 13, 2001 10:01 AM
-To: Baughman, Edward D.
-Subject: Current Enron TCC Portfolio
-
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('stephanie.panus@enron.com', 'william.bradford@enron.com, debbie.brackett@enron.com,', 'Coastal Merchant Energy/El Paso Merchant Energy', 'Coastal Merchant Energy, L.P. merged with and into El Paso Merchant Energy, 
-L.P., effective February 1, 2001, with the surviving entity being El Paso 
-Merchant Energy, L.P.  We currently have ISDA Master Agreements with both 
-counterparties.  Please see the attached memo regarding the existing Masters 
-and let us know which agreement should be terminated.
-
-Thanks,
-Stephanie
-');
-INSERT INTO email([from],[to],subject,body) VALUES('kam.keiser@enron.com', 'c..kenne@enron.com', 'RE: What about this too???', ' 
-
- -----Original Message-----
-From: 	Kenne, Dawn C.  
-Sent:	Wednesday, February 06, 2002 11:50 AM
-To:	Keiser, Kam
-Subject:	What about this too???
-
-
- << File: Netco Trader Matrix.xls >> 
- ');
-INSERT INTO email([from],[to],subject,body) VALUES('chris.meyer@enron.com', 'joe.parks@enron.com', 'Centana', 'Talked to Chip.  We do need Cash Committe approval given the netting feature of your deal, which means Batch Funding Request.  Please update per my previous e-mail and forward.
-
-Thanks
-
-chris
-x31666');
-INSERT INTO email([from],[to],subject,body) VALUES('debra.perlingiere@enron.com', 'jworman@academyofhealth.com', '', 'Have a great weekend!   Happy Fathers Day!
-
-
-Debra Perlingiere
-Enron North America Corp.
-1400 Smith Street, EB 3885
-Houston, Texas 77002
-dperlin@enron.com
-Phone 713-853-7658
-Fax  713-646-3490');
-INSERT INTO email([from],[to],subject,body) VALUES('outlook.team@enron.com', '', 'Demo by Martha Janousek of Dashboard & Pipeline Profile / Julia  &', 'CALENDAR ENTRY:	APPOINTMENT
-
-Description:
-	Demo by Martha Janousek of Dashboard & Pipeline Profile / Julia  & Dir Rpts. - 4102
-
-Date:		1/5/2001
-Time:		9:00 AM - 10:00 AM (Central Standard Time)
-
-Chairperson:	Outlook Migration Team
-
-Detailed Description:');
-INSERT INTO email([from],[to],subject,body) VALUES('diana.seifert@enron.com', 'mark.taylor@enron.com', 'Guest access Chile', 'Hello Mark,
-
-Justin Boyd told me that your can help me with questions regarding Chile.
-We got a request for guest access through MG.
-The company is called Escondida and is a subsidiary of BHP Australia.
-
-Please advise if I can set up a guest account or not.
-F.Y.I.: MG is planning to put a "in w/h Chile" contract for Copper on-line as 
-soon as Enron has done the due diligence for this country.
-Thanks !
-
-
-Best regards
-
-Diana Seifert
-EOL PCG');
-INSERT INTO email([from],[to],subject,body) VALUES('enron_update@concureworkplace.com', 'mark.whitt@enron.com', '<<Concur Expense Document>> - 121001', 'The Approval status has changed on the following report:
-
-Status last changed by: Barry L. Tycholiz
-Expense Report Name: 121001
-Report Total: $198.98
-Amount Due Employee: $198.98
-Amount Approved: $198.98
-Amount Paid: $0.00
-Approval Status: Approved
-Payment Status: Pending
-
-
-To review this expense report, click on the following link for Concur Expense.
-http://expensexms.enron.com');
-INSERT INTO email([from],[to],subject,body) VALUES('kevin.hyatt@enron.com', '', 'Technical Support', 'Outside the U.S., please refer to the list below:
-
-Australia:
-1800 678-515
-support@palm-au.com
-
-Canada:
-1905 305-6530
-support@palm.com
-
-New Zealand:
-0800 446-398
-support@palm-nz.com
-
-U.K.:
-0171 867 0108
-eurosupport@palm.3com.com
-
-Please refer to the Worldwide Customer Support card for a complete technical support contact list.');
-INSERT INTO email([from],[to],subject,body) VALUES('geoff.storey@enron.com', 'dutch.quigley@enron.com', 'RE:', 'duke contact?
-
- -----Original Message-----
-From: 	Quigley, Dutch  
-Sent:	Wednesday, October 31, 2001 10:14 AM
-To:	Storey, Geoff
-Subject:	RE: 
-
-bp corp	Albert LaMore	281-366-4962
-
-running the reports now
-
-
- -----Original Message-----
-From: 	Storey, Geoff  
-Sent:	Wednesday, October 31, 2001 10:10 AM
-To:	Quigley, Dutch
-Subject:	RE: 
-
-give me a contact over there too
-BP
-
-
- -----Original Message-----
-From: 	Quigley, Dutch  
-Sent:	Wednesday, October 31, 2001 9:42 AM
-To:	Storey, Geoff
-Subject:	
-
-Coral	Jeff Whitnah	713-767-5374
-Relaint	Steve McGinn	713-207-4000');
-INSERT INTO email([from],[to],subject,body) VALUES('pete.davis@enron.com', 'pete.davis@enron.com', 'Start Date: 4/22/01; HourAhead hour: 3;  <CODESITE>', 'Start Date: 4/22/01; HourAhead hour: 3;  No ancillary schedules awarded.  
-Variances detected.
-Variances detected in Load schedule.
-
-    LOG MESSAGES:
-
-PARSING FILE -->> O:\Portland\WestDesk\California Scheduling\ISO Final 
-Schedules\2001042203.txt
-
----- Load Schedule ----
-$$$ Variance found in table tblLoads.
-     Details: (Hour: 3 / Preferred:   1.92 / Final:   1.89)
-  TRANS_TYPE: FINAL
-  LOAD_ID: PGE4
-  MKT_TYPE: 2
-  TRANS_DATE: 4/22/01
-  SC_ID: EPMI
-
-');
-INSERT INTO email([from],[to],subject,body) VALUES('john.postlethwaite@enron.com', 'john.zufferli@enron.com', 'Reference', 'John, hope things are going well up there for you. The big day is almost here for you and Jessica. I was wondering if I could use your name as a job reference if need be. I am just trying to get everything in order just in case something happens. 
-
-John');
-INSERT INTO email([from],[to],subject,body) VALUES('jeffrey.shankman@enron.com', 'lschiffm@jonesday.com', 'Re:', 'I saw you called on the cell this a.m.  Sorry I missed you.  (I was in the 
-shower).  I have had a shitty week--I suspect my silence (not only to you, 
-but others) after our phone call is a result of the week.  I''m seeing Glen at 
-11:15....talk to you');
-INSERT INTO email([from],[to],subject,body) VALUES('litebytz@enron.com', '', 'Lite Bytz RSVP', '
-This week''s Lite Bytz presentation will feature the following TOOLZ speaker:
-
-Richard McDougall
-Solaris 8
-Thursday, June 7, 2001
-
-If you have not already signed up, please RSVP via email to litebytz@enron.com by the end of the day Tuesday, June 5, 2001.
-
-*Remember: this is now a Brown Bag Event--so bring your lunch and we will provide cookies and drinks.
-
-Click below for more details.
-
-http://home.enron.com:84/messaging/litebytztoolzprint.jpg');
-    COMMIT;
-  }
-} {}
-
-###############################################################################
-# Everything above just builds an interesting test database.  The actual
-# tests come after this comment.
-###############################################################################
-
-do_test fts2c-1.2 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark'
-  }
-} {6 17 25 38 40 42 73 74}
-do_test fts2c-1.3 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'susan'
-  }
-} {24 40}
-do_test fts2c-1.4 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark susan'
-  }
-} {40}
-do_test fts2c-1.5 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'susan mark'
-  }
-} {40}
-do_test fts2c-1.6 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH '"mark susan"'
-  }
-} {}
-do_test fts2c-1.7 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark -susan'
-  }
-} {6 17 25 38 42 73 74}
-do_test fts2c-1.8 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH '-mark susan'
-  }
-} {24}
-do_test fts2c-1.9 {
-  execsql {
-    SELECT rowid FROM email WHERE email MATCH 'mark OR susan'
-  }
-} {6 17 24 25 38 40 42 73 74}
-
-# Some simple tests of the automatic "offsets(email)" column.  In the sample
-# data set above, only one message, number 20, contains the words
-# "gas" and "reminder" in both body and subject.
-#
-do_test fts2c-2.1 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE email MATCH 'gas reminder'
-  }
-} {20 {2 0 42 3 2 1 54 8 3 0 42 3 3 1 54 8 3 0 129 3 3 0 143 3 3 0 240 3}}
-do_test fts2c-2.2 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE email MATCH 'subject:gas reminder'
-  }
-} {20 {2 0 42 3 2 1 54 8 3 1 54 8}}
-do_test fts2c-2.3 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE email MATCH 'body:gas reminder'
-  }
-} {20 {2 1 54 8 3 0 42 3 3 1 54 8 3 0 129 3 3 0 143 3 3 0 240 3}}
-do_test fts2c-2.4 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE subject MATCH 'gas reminder'
-  }
-} {20 {2 0 42 3 2 1 54 8}}
-do_test fts2c-2.5 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE body MATCH 'gas reminder'
-  }
-} {20 {3 0 42 3 3 1 54 8 3 0 129 3 3 0 143 3 3 0 240 3}}
-
-# Document 32 contains 5 instances of the world "child".  But only
-# 3 of them are paired with "product".  Make sure only those instances
-# that match the phrase appear in the offsets(email) list.
-#
-do_test fts2c-3.1 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE body MATCH 'child product' AND +rowid=32
-  }
-} {32 {3 0 94 5 3 0 114 5 3 0 207 5 3 1 213 7 3 0 245 5 3 1 251 7 3 0 409 5 3 1 415 7 3 1 493 7}}
-do_test fts2c-3.2 {
-  execsql {
-    SELECT rowid, offsets(email) FROM email
-     WHERE body MATCH '"child product"'
-  }
-} {32 {3 0 207 5 3 1 213 7 3 0 245 5 3 1 251 7 3 0 409 5 3 1 415 7}}
-
-# Snippet generator tests
-#
-do_test fts2c-4.1 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'subject:gas reminder'
-  }
-} {{Alert Posted 10:00 AM November 20,2000: E-<b>GAS</b> Request <b>Reminder</b>}}
-do_test fts2c-4.2 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'christmas candlelight'
-  }
-} {{<b>...</b> place.? What do you think about going here <b>Christmas</b> 
-eve?? They have an 11:00 a.m. service and a <b>candlelight</b> service at 5:00 p.m., 
-among others. <b>...</b>}}
-
-do_test fts2c-4.3 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'deal sheet potential reuse'
-  }
-} {{EOL-Accenture <b>Deal</b> <b>Sheet</b> <b>...</b> intent
-     Review Enron asset base for <b>potential</b> <b>reuse</b>/ licensing
-     Contract negotiations <b>...</b>}}
-do_test fts2c-4.4 {
-  execsql {
-    SELECT snippet(email,'<<<','>>>',' ') FROM email
-     WHERE email MATCH 'deal sheet potential reuse'
-  }
-} {{EOL-Accenture <<<Deal>>> <<<Sheet>>>  intent
-     Review Enron asset base for <<<potential>>> <<<reuse>>>/ licensing
-     Contract negotiations  }}
-do_test fts2c-4.5 {
-  execsql {
-    SELECT snippet(email,'<<<','>>>',' ') FROM email
-     WHERE email MATCH 'first things'
-  }
-} {{Re: <<<First>>> Polish Deal!  Congrats!  <<<Things>>> seem to be building rapidly now on the  }}
-do_test fts2c-4.6 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'chris is here'
-  }
-} {{<b>chris</b>.germany@enron.com <b>...</b> Sounds good to me.  I bet this <b>is</b> next to the Warick?? Hotel. <b>...</b> place.? What do you think about going <b>here</b> Christmas 
-eve?? They have an 11:00 a.m. <b>...</b>}}
-do_test fts2c-4.7 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH '"pursuant to"'
-  }
-} {{Erin:
-
-<b>Pursuant</b> <b>to</b> your request, attached are the Schedule to <b>...</b>}}
-do_test fts2c-4.8 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'ancillary load davis'
-  }
-} {{pete.<b>davis</b>@enron.com <b>...</b> Start Date: 4/22/01; HourAhead hour: 3;  No <b>ancillary</b> schedules awarded.  
-Variances detected.
-Variances detected in <b>Load</b> schedule.
-
-    LOG MESSAGES:
-
-PARSING <b>...</b>}}
-
-# Combinations of AND and OR operators:
-#
-do_test fts2c-5.1 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'questar enron OR com'
-  }
-} {{matt.smith@<b>enron</b>.<b>com</b> <b>...</b> six reports:  
-
-31 Keystone Receipts
-15 <b>Questar</b> Pipeline
-40 Rockies Production
-22 West_2 <b>...</b>}}
-do_test fts2c-5.2 {
-  execsql {
-    SELECT snippet(email) FROM email
-     WHERE email MATCH 'enron OR com questar'
-  }
-} {{matt.smith@<b>enron</b>.<b>com</b> <b>...</b> six reports:  
-
-31 Keystone Receipts
-15 <b>Questar</b> Pipeline
-40 Rockies Production
-22 West_2 <b>...</b>}}
-
-finish_test
diff --git a/test/fts2d.test b/test/fts2d.test
deleted file mode 100644
index d8090d8f0c..0000000000
--- a/test/fts2d.test
+++ /dev/null
@@ -1,65 +0,0 @@
-# 2006 October 1
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS2 module, and in particular
-# the Porter stemmer.
-#
-# $Id: fts2d.test,v 1.1 2006/10/19 23:36:26 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-do_test fts2d-1.1 {
-  execsql {
-    CREATE VIRTUAL TABLE t1 USING fts2(content, tokenize porter);
-    INSERT INTO t1(rowid, content) VALUES(1, 'running and jumping');
-    SELECT rowid FROM t1 WHERE content MATCH 'run jump';
-  }
-} {1}
-do_test fts2d-1.2 {
-  execsql {
-    SELECT snippet(t1) FROM t1 WHERE t1 MATCH 'run jump';
-  }
-} {{<b>running</b> and <b>jumping</b>}}
-do_test fts2d-1.3 {
-  execsql {
-    INSERT INTO t1(rowid, content) 
-          VALUES(2, 'abcdefghijklmnopqrstuvwyxz');
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH 'abcdefghijqrstuvwyxz'
-  }
-} {2 <b>abcdefghijklmnopqrstuvwyxz</b>}
-do_test fts2d-1.4 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH 'abcdefghijXXXXqrstuvwyxz'
-  }
-} {2 <b>abcdefghijklmnopqrstuvwyxz</b>}
-do_test fts2d-1.5 {
-  execsql {
-    INSERT INTO t1(rowid, content) 
-          VALUES(3, 'The value is 123456789');
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH '123789'
-  }
-} {3 {The value is <b>123456789</b>}}
-do_test fts2d-1.6 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE t1 MATCH '123000000789'
-  }
-} {3 {The value is <b>123456789</b>}}
-
-
-finish_test
diff --git a/test/fts2e.test b/test/fts2e.test
deleted file mode 100644
index 71845acdde..0000000000
--- a/test/fts2e.test
+++ /dev/null
@@ -1,85 +0,0 @@
-# 2006 October 19
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing deletions in the FTS2 module.
-#
-# $Id: fts2e.test,v 1.1 2006/10/19 23:36:26 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Construct a full-text search table containing keywords which are the
-# ordinal numbers of the bit positions set for a sequence of integers,
-# which are used for the rowid.  There are a total of 30 INSERT and
-# DELETE statements, so that we'll test both the segmentMerge() merge
-# (over the first 16) and the termSelect() merge (over the level-1
-# segment and 14 level-0 segments).
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, 'one');
-  INSERT INTO t1 (rowid, content) VALUES(2, 'two');
-  INSERT INTO t1 (rowid, content) VALUES(3, 'one two');
-  INSERT INTO t1 (rowid, content) VALUES(4, 'three');
-  DELETE FROM t1 WHERE rowid = 1;
-  INSERT INTO t1 (rowid, content) VALUES(5, 'one three');
-  INSERT INTO t1 (rowid, content) VALUES(6, 'two three');
-  INSERT INTO t1 (rowid, content) VALUES(7, 'one two three');
-  DELETE FROM t1 WHERE rowid = 4;
-  INSERT INTO t1 (rowid, content) VALUES(8, 'four');
-  INSERT INTO t1 (rowid, content) VALUES(9, 'one four');
-  INSERT INTO t1 (rowid, content) VALUES(10, 'two four');
-  DELETE FROM t1 WHERE rowid = 7;
-  INSERT INTO t1 (rowid, content) VALUES(11, 'one two four');
-  INSERT INTO t1 (rowid, content) VALUES(12, 'three four');
-  INSERT INTO t1 (rowid, content) VALUES(13, 'one three four');
-  DELETE FROM t1 WHERE rowid = 10;
-  INSERT INTO t1 (rowid, content) VALUES(14, 'two three four');
-  INSERT INTO t1 (rowid, content) VALUES(15, 'one two three four');
-  INSERT INTO t1 (rowid, content) VALUES(16, 'five');
-  DELETE FROM t1 WHERE rowid = 13;
-  INSERT INTO t1 (rowid, content) VALUES(17, 'one five');
-  INSERT INTO t1 (rowid, content) VALUES(18, 'two five');
-  INSERT INTO t1 (rowid, content) VALUES(19, 'one two five');
-  DELETE FROM t1 WHERE rowid = 16;
-  INSERT INTO t1 (rowid, content) VALUES(20, 'three five');
-  INSERT INTO t1 (rowid, content) VALUES(21, 'one three five');
-  INSERT INTO t1 (rowid, content) VALUES(22, 'two three five');
-  DELETE FROM t1 WHERE rowid = 19;
-  DELETE FROM t1 WHERE rowid = 22;
-}
-
-do_test fts2f-1.1 {
-  execsql {SELECT COUNT(*) FROM t1}
-} {14}
-
-do_test fts2e-2.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {3 5 9 11 15 17 21}
-
-do_test fts2e-2.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two'}
-} {2 3 6 11 14 15 18}
-
-do_test fts2e-2.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three'}
-} {5 6 12 14 15 20 21}
-
-do_test fts2e-2.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'four'}
-} {8 9 11 12 14 15}
-
-do_test fts2e-2.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five'}
-} {17 18 20 21}
-
-finish_test
diff --git a/test/fts2f.test b/test/fts2f.test
deleted file mode 100644
index 49cff14664..0000000000
--- a/test/fts2f.test
+++ /dev/null
@@ -1,90 +0,0 @@
-# 2006 October 19
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing updates in the FTS2 module.
-#
-# $Id: fts2f.test,v 1.2 2007/02/23 00:14:06 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Construct a full-text search table containing keywords which are the
-# ordinal numbers of the bit positions set for a sequence of integers,
-# which are used for the rowid.  There are a total of 31 INSERT,
-# UPDATE, and DELETE statements, so that we'll test both the
-# segmentMerge() merge (over the first 16) and the termSelect() merge
-# (over the level-1 segment and 15 level-0 segments).
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, 'one');
-  INSERT INTO t1 (rowid, content) VALUES(2, 'two');
-  INSERT INTO t1 (rowid, content) VALUES(3, 'one two');
-  INSERT INTO t1 (rowid, content) VALUES(4, 'three');
-  INSERT INTO t1 (rowid, content) VALUES(5, 'one three');
-  INSERT INTO t1 (rowid, content) VALUES(6, 'two three');
-  INSERT INTO t1 (rowid, content) VALUES(7, 'one two three');
-  DELETE FROM t1 WHERE rowid = 4;
-  INSERT INTO t1 (rowid, content) VALUES(8, 'four');
-  UPDATE t1 SET content = 'update one three' WHERE rowid = 1;
-  INSERT INTO t1 (rowid, content) VALUES(9, 'one four');
-  INSERT INTO t1 (rowid, content) VALUES(10, 'two four');
-  DELETE FROM t1 WHERE rowid = 7;
-  INSERT INTO t1 (rowid, content) VALUES(11, 'one two four');
-  INSERT INTO t1 (rowid, content) VALUES(12, 'three four');
-  INSERT INTO t1 (rowid, content) VALUES(13, 'one three four');
-  DELETE FROM t1 WHERE rowid = 10;
-  INSERT INTO t1 (rowid, content) VALUES(14, 'two three four');
-  INSERT INTO t1 (rowid, content) VALUES(15, 'one two three four');
-  UPDATE t1 SET content = 'update two five' WHERE rowid = 8;
-  INSERT INTO t1 (rowid, content) VALUES(16, 'five');
-  DELETE FROM t1 WHERE rowid = 13;
-  INSERT INTO t1 (rowid, content) VALUES(17, 'one five');
-  INSERT INTO t1 (rowid, content) VALUES(18, 'two five');
-  INSERT INTO t1 (rowid, content) VALUES(19, 'one two five');
-  DELETE FROM t1 WHERE rowid = 16;
-  INSERT INTO t1 (rowid, content) VALUES(20, 'three five');
-  INSERT INTO t1 (rowid, content) VALUES(21, 'one three five');
-  INSERT INTO t1 (rowid, content) VALUES(22, 'two three five');
-  DELETE FROM t1 WHERE rowid = 19;
-  UPDATE t1 SET content = 'update' WHERE rowid = 15;
-}
-
-do_test fts2f-1.1 {
-  execsql {SELECT COUNT(*) FROM t1}
-} {16}
-
-do_test fts2f-2.0 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'update'}
-} {1 8 15}
-
-do_test fts2f-2.1 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'}
-} {1 3 5 9 11 17 21}
-
-do_test fts2f-2.2 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'two'}
-} {2 3 6 8 11 14 18 22}
-
-do_test fts2f-2.3 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'three'}
-} {1 5 6 12 14 20 21 22}
-
-do_test fts2f-2.4 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'four'}
-} {9 11 12 14}
-
-do_test fts2f-2.5 {
-  execsql {SELECT rowid FROM t1 WHERE content MATCH 'five'}
-} {8 17 18 20 21 22}
-
-finish_test
diff --git a/test/fts2g.test b/test/fts2g.test
deleted file mode 100644
index 176c619aa3..0000000000
--- a/test/fts2g.test
+++ /dev/null
@@ -1,93 +0,0 @@
-# 2006 October 19
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing handling of edge cases for various doclist
-# merging functions in the FTS2 module query logic.
-#
-# $Id: fts2g.test,v 1.3 2007/11/16 00:23:08 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, 'this is a test');
-  INSERT INTO t1 (rowid, content) VALUES(2, 'also a test');
-}
-
-# No hits at all.  Returns empty doclists from termSelect().
-do_test fts2g-1.1 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something'}
-} {}
-
-# Empty left in docListExceptMerge().
-do_test fts2g-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH '-this something'}
-} {}
-
-# Empty right in docListExceptMerge().
-do_test fts2g-1.3 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this -something'}
-} {1}
-
-# Empty left in docListPhraseMerge().
-do_test fts2g-1.4 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH '"this something"'}
-} {}
-
-# Empty right in docListPhraseMerge().
-do_test fts2g-1.5 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH '"something is"'}
-} {}
-
-# Empty left in docListOrMerge().
-do_test fts2g-1.6 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something OR this'}
-} {1}
-
-# Empty right in docListOrMerge().
-do_test fts2g-1.7 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this OR something'}
-} {1}
-
-# Empty left in docListAndMerge().
-do_test fts2g-1.8 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something this'}
-} {}
-
-# Empty right in docListAndMerge().
-do_test fts2g-1.9 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this something'}
-} {}
-
-# No support for all-except queries.
-do_test fts2g-1.10 {
-  catchsql {SELECT rowid FROM t1 WHERE t1 MATCH '-this -something'}
-} {1 {SQL logic error}}
-
-# Test that docListOrMerge() correctly handles reaching the end of one
-# doclist before it reaches the end of the other.
-do_test fts2g-1.11 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this OR also'}
-} {1 2}
-do_test fts2g-1.12 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'also OR this'}
-} {1 2}
-
-# Empty left and right in docListOrMerge().  Each term matches neither
-# row, and when combined there was an assertion failure.
-do_test fts2g-1.13 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something OR nothing'}
-} {}
-
-finish_test
diff --git a/test/fts2h.test b/test/fts2h.test
deleted file mode 100644
index 72561d85bc..0000000000
--- a/test/fts2h.test
+++ /dev/null
@@ -1,76 +0,0 @@
-# 2006 October 31 (scaaarey)
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# here is testing correct handling of excessively long terms.
-#
-# $Id: fts2h.test,v 1.1 2006/11/29 21:03:01 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Generate a term of len copies of char.
-proc bigterm {char len} {
-  for {set term ""} {$len>0} {incr len -1} {
-    append term $char
-  }
-  return $term
-}
-
-# Generate a document of bigterms based on characters from the list
-# chars.
-proc bigtermdoc {chars len} {
-  set doc ""
-  foreach char $chars {
-    append doc " " [bigterm $char $len]
-  }
-  return $doc
-}
-
-set len 5000
-set doc1 [bigtermdoc {a b c d} $len]
-set doc2 [bigtermdoc {b d e f} $len]
-set doc3 [bigtermdoc {a c e} $len]
-
-set aterm [bigterm a $len]
-set bterm [bigterm b $len]
-set xterm [bigterm x $len]
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, $doc1);
-  INSERT INTO t1 (rowid, content) VALUES(2, $doc2);
-  INSERT INTO t1 (rowid, content) VALUES(3, $doc3);
-}
-
-# No hits at all.  Returns empty doclists from termSelect().
-do_test fts2h-1.1 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'something'}
-} {}
-
-do_test fts2h-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm}
-} {1 3}
-
-do_test fts2h-1.2 {
-  execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm}
-} {}
-
-do_test fts2h-1.3 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'"
-} {1 3}
-
-do_test fts2h-1.4 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'"
-} {1}
-
-finish_test
diff --git a/test/fts2i.test b/test/fts2i.test
deleted file mode 100644
index e732e6a8a9..0000000000
--- a/test/fts2i.test
+++ /dev/null
@@ -1,87 +0,0 @@
-# 2007 January 17
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite fts2 library.  The
-# focus here is testing handling of UPDATE when using UTF-16-encoded
-# databases.
-#
-# $Id: fts2i.test,v 1.2 2007/01/24 03:46:35 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Return the UTF-16 representation of the supplied UTF-8 string $str.
-# If $nt is true, append two 0x00 bytes as a nul terminator.
-# NOTE(shess) Copied from capi3.test.
-proc utf16 {str {nt 1}} {
-  set r [encoding convertto unicode $str]
-  if {$nt} {
-    append r "\x00\x00"
-  }
-  return $r
-}
-
-db eval {
-  PRAGMA encoding = "UTF-16le";
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-}
-
-do_test fts2i-1.0 {
-  execsql {PRAGMA encoding}
-} {UTF-16le}
-
-do_test fts2i-1.1 {
-  execsql {INSERT INTO t1 (rowid, content) VALUES(1, 'one')}
-  execsql {SELECT content FROM t1 WHERE rowid = 1}
-} {one}
-
-do_test fts2i-1.2 {
-  set sql "INSERT INTO t1 (rowid, content) VALUES(2, 'two')"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 2}
-} {two}
-
-do_test fts2i-1.3 {
-  set sql "INSERT INTO t1 (rowid, content) VALUES(3, 'three')"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  set sql "UPDATE t1 SET content = 'trois' WHERE rowid = 3"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 3}
-} {trois}
-
-do_test fts2i-1.4 {
-  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(4, 'four')}]
-  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 4}
-} {four}
-
-do_test fts2i-1.5 {
-  set sql16 [utf16 {INSERT INTO t1 (rowid, content) VALUES(5, 'five')}]
-  set STMT [sqlite3_prepare16 $DB $sql16 -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  set sql "UPDATE t1 SET content = 'cinq' WHERE rowid = 5"
-  set STMT [sqlite3_prepare $DB $sql -1 TAIL]
-  sqlite3_step $STMT
-  sqlite3_finalize $STMT
-  execsql {SELECT content FROM t1 WHERE rowid = 5}
-} {cinq}
-
-finish_test
diff --git a/test/fts2j.test b/test/fts2j.test
deleted file mode 100644
index a8a2c07c18..0000000000
--- a/test/fts2j.test
+++ /dev/null
@@ -1,89 +0,0 @@
-# 2007 February 6
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  This
-# tests creating fts2 tables in an attached database.
-#
-# $Id: fts2j.test,v 1.1 2007/02/07 01:01:18 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Clean up anything left over from a previous pass.
-forcedelete test2.db
-forcedelete test2.db-journal
-sqlite3 db2 test2.db
-
-db eval {
-  CREATE VIRTUAL TABLE t3 USING fts2(content);
-  INSERT INTO t3 (rowid, content) VALUES(1, "hello world");
-}
-
-db2 eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, "hello world");
-  INSERT INTO t1 (rowid, content) VALUES(2, "hello there");
-  INSERT INTO t1 (rowid, content) VALUES(3, "cruel world");
-}
-
-# This has always worked because the t1_* tables used by fts2 will be
-# the defaults.
-do_test fts2j-1.1 {
-  execsql {
-    ATTACH DATABASE 'test2.db' AS two;
-    SELECT rowid FROM t1 WHERE t1 MATCH 'hello';
-    DETACH DATABASE two;
-  }
-} {1 2}
-# Make certain we're detached if there was an error.
-catch {db eval {DETACH DATABASE two}}
-
-# In older code, this appears to work fine, but the t2_* tables used
-# by fts2 will be created in database 'main' instead of database
-# 'two'.  It appears to work fine because the tables end up being the
-# defaults, but obviously is badly broken if you hope to use things
-# other than in the exact same ATTACH setup.
-do_test fts2j-1.2 {
-  execsql {
-    ATTACH DATABASE 'test2.db' AS two;
-    CREATE VIRTUAL TABLE two.t2 USING fts2(content);
-    INSERT INTO t2 (rowid, content) VALUES(1, "hello world");
-    INSERT INTO t2 (rowid, content) VALUES(2, "hello there");
-    INSERT INTO t2 (rowid, content) VALUES(3, "cruel world");
-    SELECT rowid FROM t2 WHERE t2 MATCH 'hello';
-    DETACH DATABASE two;
-  }
-} {1 2}
-catch {db eval {DETACH DATABASE two}}
-
-# In older code, this broke because the fts2 code attempted to create
-# t3_* tables in database 'main', but they already existed.  Normally
-# this wouldn't happen without t3 itself existing, in which case the
-# fts2 code would never be called in the first place.
-do_test fts2j-1.3 {
-  execsql {
-    ATTACH DATABASE 'test2.db' AS two;
-
-    CREATE VIRTUAL TABLE two.t3 USING fts2(content);
-    INSERT INTO two.t3 (rowid, content) VALUES(2, "hello there");
-    INSERT INTO two.t3 (rowid, content) VALUES(3, "cruel world");
-    SELECT rowid FROM two.t3 WHERE t3 MATCH 'hello';
-
-    DETACH DATABASE two;
-  } db2
-} {2}
-catch {db eval {DETACH DATABASE two}}
-
-catch {db2 close}
-forcedelete test2.db
-
-finish_test
diff --git a/test/fts2k.test b/test/fts2k.test
deleted file mode 100644
index e7d5f0dff4..0000000000
--- a/test/fts2k.test
+++ /dev/null
@@ -1,105 +0,0 @@
-# 2007 March 9
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  These
-# make sure that fts2 insertion buffering is fully transparent when
-# using transactions.
-#
-# $Id: fts2k.test,v 1.2 2007/08/10 23:47:04 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(content);
-  INSERT INTO t1 (rowid, content) VALUES(1, "hello world");
-  INSERT INTO t1 (rowid, content) VALUES(2, "hello there");
-  INSERT INTO t1 (rowid, content) VALUES(3, "cruel world");
-}
-
-# Test that possibly-buffered inserts went through after commit.
-do_test fts2k-1.1 {
-  execsql {
-    BEGIN TRANSACTION;
-    INSERT INTO t1 (rowid, content) VALUES(4, "false world");
-    INSERT INTO t1 (rowid, content) VALUES(5, "false door");
-    COMMIT TRANSACTION;
-    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
-  }
-} {1 3 4}
-
-# Test that buffered inserts are seen by selects in the same
-# transaction.
-do_test fts2k-1.2 {
-  execsql {
-    BEGIN TRANSACTION;
-    INSERT INTO t1 (rowid, content) VALUES(6, "another world");
-    INSERT INTO t1 (rowid, content) VALUES(7, "another test");
-    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
-    COMMIT TRANSACTION;
-  }
-} {1 3 4 6}
-
-# Test that buffered inserts are seen within a transaction.  This is
-# really the same test as 1.2.
-do_test fts2k-1.3 {
-  execsql {
-    BEGIN TRANSACTION;
-    INSERT INTO t1 (rowid, content) VALUES(8, "second world");
-    INSERT INTO t1 (rowid, content) VALUES(9, "second sight");
-    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
-    ROLLBACK TRANSACTION;
-  }
-} {1 3 4 6 8}
-
-# Double-check that the previous result doesn't persist past the
-# rollback!
-do_test fts2k-1.4 {
-  execsql {
-    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
-  }
-} {1 3 4 6}
-
-# Test it all together.
-do_test fts2k-1.5 {
-  execsql {
-    BEGIN TRANSACTION;
-    INSERT INTO t1 (rowid, content) VALUES(10, "second world");
-    INSERT INTO t1 (rowid, content) VALUES(11, "second sight");
-    ROLLBACK TRANSACTION;
-    SELECT rowid FROM t1 WHERE t1 MATCH 'world';
-  }
-} {1 3 4 6}
-
-# Test that the obvious case works.
-do_test fts2k-1.6 {
-  execsql {
-    BEGIN;
-    INSERT INTO t1 (rowid, content) VALUES(12, "third world");
-    COMMIT;
-    SELECT rowid FROM t1 WHERE t1 MATCH 'third';
-  }
-} {12}
-
-# This is exactly the same as the previous test, except that older
-# code loses the INSERT due to an SQLITE_SCHEMA error.
-do_test fts2k-1.7 {
-  execsql {
-    BEGIN;
-    INSERT INTO t1 (rowid, content) VALUES(13, "third dimension");
-    CREATE TABLE x (c);
-    COMMIT;
-    SELECT rowid FROM t1 WHERE t1 MATCH 'dimension';
-  }
-} {13}
-
-finish_test
diff --git a/test/fts2l.test b/test/fts2l.test
deleted file mode 100644
index 42f5ba134f..0000000000
--- a/test/fts2l.test
+++ /dev/null
@@ -1,69 +0,0 @@
-# 2007 March 28
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing isspace/isalnum/tolower problems with the
-# FTS2 module.  Unfortunately, this code isn't a really principled set
-# of tests, because it is impossible to know where new uses of these
-# functions might appear.
-#
-# $Id: fts2l.test,v 1.2 2007/12/13 21:54:11 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# Tests that startsWith() (calls isspace, tolower, isalnum) can handle
-# hi-bit chars.  parseSpec() also calls isalnum here.
-do_test fts2l-1.1 {
-  execsql "CREATE VIRTUAL TABLE t1 USING fts2(content, \x80)"
-} {}
-
-# Additionally tests isspace() call in getToken(), and isalnum() call
-# in tokenListToIdList().
-do_test fts2l-1.2 {
-  catch {
-    execsql "CREATE VIRTUAL TABLE t2 USING fts2(content, tokenize \x80)"
-  }
-  sqlite3_errmsg $DB
-} "unknown tokenizer: \x80"
-
-# Additionally test final isalnum() in startsWith().
-do_test fts2l-1.3 {
-  execsql "CREATE VIRTUAL TABLE t3 USING fts2(content, tokenize\x80)"
-} {}
-
-# The snippet-generation code has calls to isspace() which are sort of
-# hard to get to.  It finds convenient breakpoints by starting ~40
-# chars before and after the matched term, and scanning ~10 chars
-# around that position for isspace() characters.  The long word with
-# embedded hi-bit chars causes one of these isspace() calls to be
-# exercised.  The version with a couple extra spaces should cause the
-# other isspace() call to be exercised.  [Both cases have been tested
-# in the debugger, but I'm hoping to continue to catch it if simple
-# constant changes change things slightly.
-#
-# The trailing and leading hi-bit chars help with code which tests for
-# isspace() to coalesce multiple spaces.
-
-set word "\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80"
-set phrase1 "$word $word $word target $word $word $word"
-set phrase2 "$word $word $word    target    $word $word $word"
-
-db eval {CREATE VIRTUAL TABLE t4 USING fts2(content)}
-db eval "INSERT INTO t4 (content) VALUES ('$phrase1')"
-db eval "INSERT INTO t4 (content) VALUES ('$phrase2')"
-
-do_test fts2l-1.4 {
-  execsql {SELECT rowid, length(snippet(t4)) FROM t4 WHERE t4 MATCH 'target'}
-} {1 111 2 117}
-
-finish_test
diff --git a/test/fts2m.test b/test/fts2m.test
deleted file mode 100644
index 6552637a62..0000000000
--- a/test/fts2m.test
+++ /dev/null
@@ -1,65 +0,0 @@
-# 2007 April 9
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  fts2
-# DELETE handling assumed all fields were non-null.  This was not
-# the intention at all.
-#
-# $Id: fts2m.test,v 1.1 2007/04/09 20:45:42 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(col_a, col_b);
-
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(1, 'testing', 'testing');
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(2, 'only a', null);
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(3, null, 'only b');
-  INSERT INTO t1(rowid, col_a, col_b) VALUES(4, null, null);
-}
-
-do_test fts2m-1.0 {
-  execsql {
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {2 2 4}
-
-do_test fts2m-1.1 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 1;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {1 1 3}
-
-do_test fts2m-1.2 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 2;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {0 1 2}
-
-do_test fts2m-1.3 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 3;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {0 0 1}
-
-do_test fts2m-1.4 {
-  execsql {
-    DELETE FROM t1 WHERE rowid = 4;
-    SELECT COUNT(col_a), COUNT(col_b), COUNT(*) FROM t1;
-  }
-} {0 0 0}
-
-finish_test
diff --git a/test/fts2n.test b/test/fts2n.test
deleted file mode 100644
index ca0b4fe9ff..0000000000
--- a/test/fts2n.test
+++ /dev/null
@@ -1,196 +0,0 @@
-# 2007 April 26
-#
-# The author disclaims copyright to this source code.
-#
-#*************************************************************************
-# This file implements tests for prefix-searching in the fts2
-# component of the SQLite library.
-#
-# $Id: fts2n.test,v 1.2 2007/12/13 21:54:11 drh Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-# A large string to prime the pump with.
-set text {
-  Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Maecenas
-  iaculis mollis ipsum. Praesent rhoncus placerat justo. Duis non quam
-  sed turpis posuere placerat. Curabitur et lorem in lorem porttitor
-  aliquet. Pellentesque bibendum tincidunt diam. Vestibulum blandit
-  ante nec elit. In sapien diam, facilisis eget, dictum sed, viverra
-  at, felis. Vestibulum magna. Sed magna dolor, vestibulum rhoncus,
-  ornare vel, vulputate sit amet, felis. Integer malesuada, tellus at
-  luctus gravida, diam nunc porta nibh, nec imperdiet massa metus eu
-  lectus. Aliquam nisi. Nunc fringilla nulla at lectus. Suspendisse
-  potenti. Cum sociis natoque penatibus et magnis dis parturient
-  montes, nascetur ridiculus mus. Pellentesque odio nulla, feugiat eu,
-  suscipit nec, consequat quis, risus.
-}
-
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-
-  INSERT INTO t1(rowid, c) VALUES(1, $text);
-  INSERT INTO t1(rowid, c) VALUES(2, 'Another lovely row');
-}
-
-# Exact match
-do_test fts2n-1.1 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lorem'"
-} {1}
-
-# And a prefix
-do_test fts2n-1.2 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lore*'"
-} {1}
-
-# Prefix includes exact match
-do_test fts2n-1.3 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lorem*'"
-} {1}
-
-# Make certain everything isn't considered a prefix!
-do_test fts2n-1.4 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lore'"
-} {}
-
-# Prefix across multiple rows.
-do_test fts2n-1.5 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lo*'"
-} {1 2}
-
-# Likewise, with multiple hits in one document.
-do_test fts2n-1.6 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'l*'"
-} {1 2}
-
-# Prefix which should only hit one document.
-do_test fts2n-1.7 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lov*'"
-} {2}
-
-# * not at end is dropped.
-do_test fts2n-1.8 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH 'lo *'"
-} {}
-
-# Stand-alone * is dropped.
-do_test fts2n-1.9 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '*'"
-} {}
-
-# Phrase-query prefix.
-do_test fts2n-1.10 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"lovely r*\"'"
-} {2}
-do_test fts2n-1.11 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"lovely r\"'"
-} {}
-
-# Phrase query with multiple prefix matches.
-do_test fts2n-1.12 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"a* l*\"'"
-} {1 2}
-
-# Phrase query with multiple prefix matches.
-do_test fts2n-1.13 {
-  execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"a* l* row\"'"
-} {2}
-
-
-
-
-# Test across updates (and, by implication, deletes).
-
-# Version of text without "lorem".
-regsub -all {[Ll]orem} $text '' ntext
-
-db eval {
-  CREATE VIRTUAL TABLE t2 USING fts2(c);
-
-  INSERT INTO t2(rowid, c) VALUES(1, $text);
-  INSERT INTO t2(rowid, c) VALUES(2, 'Another lovely row');
-  UPDATE t2 SET c = $ntext WHERE rowid = 1;
-}
-
-# Can't see lorem as an exact match.
-do_test fts2n-2.1 {
-  execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lorem'"
-} {}
-
-# Can't see a prefix of lorem, either.
-do_test fts2n-2.2 {
-  execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lore*'"
-} {}
-
-# Can see lovely in the other document.
-do_test fts2n-2.3 {
-  execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lo*'"
-} {2}
-
-# Can still see other hits.
-do_test fts2n-2.4 {
-  execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'l*'"
-} {1 2}
-
-# Prefix which should only hit one document.
-do_test fts2n-2.5 {
-  execsql "SELECT rowid FROM t2 WHERE t2 MATCH 'lov*'"
-} {2}
-
-
-
-# Test with a segment which will have multiple levels in the tree.
-
-# Build a big document with lots of unique terms.
-set bigtext $text
-foreach c {a b c d e} {
-  regsub -all {[A-Za-z]+} $bigtext "&$c" t
-  append bigtext $t
-}
-
-# Populate a table with many copies of the big document, so that we
-# can test the number of hits found.  Populate $ret with the expected
-# hit counts for each row.  offsets() returns 4 elements for every
-# hit.  We'll have 6 hits for row 1, 1 for row 2, and 6*(2^5)==192 for
-# $bigtext.
-set ret {6 1}
-db eval {
-  BEGIN;
-  CREATE VIRTUAL TABLE t3 USING fts2(c);
-
-  INSERT INTO t3(rowid, c) VALUES(1, $text);
-  INSERT INTO t3(rowid, c) VALUES(2, 'Another lovely row');
-}
-for {set i 0} {$i<100} {incr i} {
-  db eval {INSERT INTO t3(rowid, c) VALUES(3+$i, $bigtext)}
-  lappend ret 192
-}
-db eval {COMMIT;}
-
-# Test that we get the expected number of hits.
-do_test fts2n-3.1 {
-  set t {}
-  db eval {SELECT offsets(t3) as o FROM t3 WHERE t3 MATCH 'l*'} {
-    set l [llength $o]
-    lappend t [expr {$l/4}]
-  }
-  set t
-} $ret
-
-# TODO(shess) It would be useful to test a couple edge cases, but I
-# don't know if we have the precision to manage it from here at this
-# time.  Prefix hits can cross leaves, which the code above _should_
-# hit by virtue of size.  There are two variations on this.  If the
-# tree is 2 levels high, the code will find the leaf-node extent
-# directly, but if its higher, the code will have to follow two
-# separate interior branches down the tree.  Both should be tested.
-
-finish_test
diff --git a/test/fts2o.test b/test/fts2o.test
deleted file mode 100644
index de319ea96a..0000000000
--- a/test/fts2o.test
+++ /dev/null
@@ -1,169 +0,0 @@
-# 2007 June 20
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS2 module.
-#
-# $Id: fts2o.test,v 1.4 2007/07/02 10:16:50 danielk1977 Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is not defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-#---------------------------------------------------------------------
-# These tests, fts2o-1.*, test that ticket #2429 is fixed.
-#
-db eval {
-  CREATE VIRTUAL TABLE t1 USING fts2(a, b, c);
-  INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one four two');
-}
-do_test fts2o-1.1 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE c MATCH 'four';
-  }
-} {1 {one <b>four</b> two}}
-do_test fts2o-1.2 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE b MATCH 'four';
-  }
-} {1 {one <b>four</b>}}
-do_test fts2o-1.3 {
-  execsql {
-    SELECT rowid, snippet(t1) FROM t1 WHERE a MATCH 'four';
-  }
-} {1 {one three <b>four</b>}}
-
-#---------------------------------------------------------------------
-# Test that it is possible to rename an fts2 table.
-#
-do_test fts2o-2.1 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {t1 t1_content t1_segments t1_segdir}
-do_test fts2o-2.2 {
-  execsql { ALTER TABLE t1 RENAME to fts_t1; }
-} {}
-do_test fts2o-2.3 {
-  execsql { SELECT rowid, snippet(fts_t1) FROM fts_t1 WHERE a MATCH 'four'; }
-} {1 {one three <b>four</b>}}
-do_test fts2o-2.4 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {fts_t1 fts_t1_content fts_t1_segments fts_t1_segdir}
-
-# See what happens when renaming the fts2 table fails.
-#
-do_test fts2o-2.5 {
-  catchsql {
-    CREATE TABLE t1_segdir(a, b, c);
-    ALTER TABLE fts_t1 RENAME to t1;
-  }
-} {1 {SQL logic error}}
-do_test fts2o-2.6 {
-  execsql { SELECT rowid, snippet(fts_t1) FROM fts_t1 WHERE a MATCH 'four'; }
-} {1 {one three <b>four</b>}}
-do_test fts2o-2.7 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {fts_t1 fts_t1_content fts_t1_segments fts_t1_segdir t1_segdir}
-
-# See what happens when renaming the fts2 table fails inside a transaction.
-#
-do_test fts2o-2.8 {
-  execsql {
-    BEGIN;
-    INSERT INTO fts_t1(a, b, c) VALUES('one two three', 'one four', 'one two');
-  }
-} {}
-do_test fts2o-2.9 {
-  catchsql {
-    ALTER TABLE fts_t1 RENAME to t1;
-  }
-} {1 {SQL logic error}}
-do_test fts2o-2.10 {
-  execsql { SELECT rowid, snippet(fts_t1) FROM fts_t1 WHERE a MATCH 'four'; }
-} {1 {one three <b>four</b>}}
-do_test fts2o-2.11 {
-  execsql { SELECT tbl_name FROM sqlite_master WHERE type = 'table'}
-} {fts_t1 fts_t1_content fts_t1_segments fts_t1_segdir t1_segdir}
-do_test fts2o-2.12 {
-  execsql COMMIT
-  execsql {SELECT a FROM fts_t1}
-} {{one three four} {one two three}}
-do_test fts2o-2.12 {
-  execsql { SELECT a, b, c FROM fts_t1 WHERE c MATCH 'four'; }
-} {{one three four} {one four} {one four two}}
-
-#-------------------------------------------------------------------
-# Close, delete and reopen the database. The following test should 
-# be run on an initially empty db.
-#
-db close
-forcedelete test.db test.db-journal
-sqlite3 db test.db
-
-do_test fts2o-3.1 {
-  execsql {
-    CREATE VIRTUAL TABLE t1 USING fts2(a, b, c);
-    INSERT INTO t1(a, b, c) VALUES('one three four', 'one four', 'one two');
-    SELECT a, b, c FROM t1 WHERE c MATCH 'two';
-  }
-} {{one three four} {one four} {one two}}
-
-# This test was crashing at one point.
-#
-do_test fts2o-3.2 {
-  execsql {
-    SELECT a, b, c FROM t1 WHERE c MATCH 'two';
-    CREATE TABLE t3(a, b, c);
-    SELECT a, b, c FROM t1 WHERE  c  MATCH 'two';
-  }
-} {{one three four} {one four} {one two} {one three four} {one four} {one two}}
-
-#---------------------------------------------------------------------
-# Test that it is possible to rename an fts2 table in an attached 
-# database.
-#
-forcedelete test2.db test2.db-journal
-
-do_test fts2o-3.1 {
-  execsql {
-    ATTACH 'test2.db' AS aux;
-    CREATE VIRTUAL TABLE aux.t1 USING fts2(a, b, c);
-    INSERT INTO aux.t1(a, b, c) VALUES(
-      'neung song sahm', 'neung see', 'neung see song'
-    );
-  }
-} {}
-
-do_test fts2o-3.2 {
-  execsql { SELECT a, b, c FROM aux.t1 WHERE a MATCH 'song'; }
-} {{neung song sahm} {neung see} {neung see song}}
-
-do_test fts2o-3.3 {
-  execsql { SELECT a, b, c FROM t1 WHERE c MATCH 'two'; }
-} {{one three four} {one four} {one two}}
-
-do_test fts2o-3.4 {
-  execsql { ALTER TABLE aux.t1 RENAME TO t2 }
-} {}
-
-do_test fts2o-3.2 {
-  execsql { SELECT a, b, c FROM t2 WHERE a MATCH 'song'; }
-} {{neung song sahm} {neung see} {neung see song}}
-
-do_test fts2o-3.3 {
-  execsql { SELECT a, b, c FROM t1 WHERE c MATCH 'two'; }
-} {{one three four} {one four} {one two}}
-
-finish_test
diff --git a/test/fts2p.test b/test/fts2p.test
deleted file mode 100644
index 38a8079d8f..0000000000
--- a/test/fts2p.test
+++ /dev/null
@@ -1,357 +0,0 @@
-# 2008 June 26
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file exercises some new testing functions in the FTS2 module,
-# and then uses them to do some basic tests that FTS2 is internally
-# working as expected.
-#
-# $Id: fts2p.test,v 1.1 2008/07/22 23:32:28 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is not defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-#*************************************************************************
-# Probe to see if support for these functions is compiled in.
-# TODO(shess): Change main.mk to do the right thing and remove this test.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'x');
-}
-
-set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1}
-set r {1 {unable to use function dump_terms in the requested context}}
-if {[catchsql $s]==$r} {
-  finish_test
-  return
-}
-
-#*************************************************************************
-# Test that the new functions give appropriate errors.
-do_test fts2p-0.0 {
-  catchsql {
-    SELECT dump_terms(t1, 1) FROM t1 LIMIT 1;
-  }
-} {1 {dump_terms: incorrect arguments}}
-
-do_test fts2p-0.1 {
-  catchsql {
-    SELECT dump_terms(t1, 0, 0, 0) FROM t1 LIMIT 1;
-  }
-} {1 {dump_terms: incorrect arguments}}
-
-do_test fts2p-0.2 {
-  catchsql {
-    SELECT dump_terms(1, t1) FROM t1 LIMIT 1;
-  }
-} {1 {unable to use function dump_terms in the requested context}}
-
-do_test fts2p-0.3 {
-  catchsql {
-    SELECT dump_terms(t1, 16, 16) FROM t1 LIMIT 1;
-  }
-} {1 {dump_terms: segment not found}}
-
-do_test fts2p-0.4 {
-  catchsql {
-    SELECT dump_doclist(t1) FROM t1 LIMIT 1;
-  }
-} {1 {dump_doclist: incorrect arguments}}
-
-do_test fts2p-0.5 {
-  catchsql {
-    SELECT dump_doclist(t1, NULL) FROM t1 LIMIT 1;
-  }
-} {1 {dump_doclist: empty second argument}}
-
-do_test fts2p-0.6 {
-  catchsql {
-    SELECT dump_doclist(t1, '') FROM t1 LIMIT 1;
-  }
-} {1 {dump_doclist: empty second argument}}
-
-do_test fts2p-0.7 {
-  catchsql {
-    SELECT dump_doclist(t1, 'a', 0) FROM t1 LIMIT 1;
-  }
-} {1 {dump_doclist: incorrect arguments}}
-
-do_test fts2p-0.8 {
-  catchsql {
-    SELECT dump_doclist(t1, 'a', 0, 0, 0) FROM t1 LIMIT 1;
-  }
-} {1 {dump_doclist: incorrect arguments}}
-
-do_test fts2p-0.9 {
-  catchsql {
-    SELECT dump_doclist(t1, 'a', 16, 16) FROM t1 LIMIT 1;
-  }
-} {1 {dump_doclist: segment not found}}
-
-#*************************************************************************
-# Utility function to check for the expected terms in the segment
-# level/index.  _all version does same but for entire index.
-proc check_terms {test level index terms} {
-  # TODO(shess): Figure out why uplevel in do_test can't catch
-  # $level and $index directly.
-  set ::level $level
-  set ::index $index
-  do_test $test.terms {
-    execsql {
-      SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1;
-    }
-  } [list $terms]
-}
-proc check_terms_all {test terms} {
-  do_test $test.terms {
-    execsql {
-      SELECT dump_terms(t1) FROM t1 LIMIT 1;
-    }
-  } [list $terms]
-}
-
-# Utility function to check for the expected doclist for the term in
-# segment level/index.  _all version does same for entire index.
-proc check_doclist {test level index term doclist} {
-  # TODO(shess): Again, why can't the non-:: versions work?
-  set ::term $term
-  set ::level $level
-  set ::index $index
-  do_test $test {
-    execsql {
-      SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1;
-    }
-  } [list $doclist]
-}
-proc check_doclist_all {test term doclist} {
-  set ::term $term
-  do_test $test {
-    execsql {
-      SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1;
-    }
-  } [list $doclist]
-}
-
-#*************************************************************************
-# Test the segments resulting from straight-forward inserts.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-}
-
-# Check for expected segments and expected matches.
-do_test fts2p-1.0.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0 0 1 0 2}
-do_test fts2p-1.0.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \
-        {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \
-        {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}]
-
-# Check the specifics of the segments constructed.
-# Logical view of entire index.
-check_terms_all   fts2p-1.0.1   {a is test that this was}
-check_doclist_all fts2p-1.0.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]}
-check_doclist_all fts2p-1.0.1.2 is {[1 0[1]] [3 0[1]]}
-check_doclist_all fts2p-1.0.1.3 test {[1 0[3]] [2 0[3]] [3 0[3]]}
-check_doclist_all fts2p-1.0.1.4 that {[2 0[0]]}
-check_doclist_all fts2p-1.0.1.5 this {[1 0[0]] [3 0[0]]}
-check_doclist_all fts2p-1.0.1.6 was {[2 0[1]]}
-
-# Segment 0,0
-check_terms   fts2p-1.0.2   0 0 {a is test this}
-check_doclist fts2p-1.0.2.1 0 0 a {[1 0[2]]}
-check_doclist fts2p-1.0.2.2 0 0 is {[1 0[1]]}
-check_doclist fts2p-1.0.2.3 0 0 test {[1 0[3]]}
-check_doclist fts2p-1.0.2.4 0 0 this {[1 0[0]]}
-
-# Segment 0,1
-check_terms   fts2p-1.0.3   0 1 {a test that was}
-check_doclist fts2p-1.0.3.1 0 1 a {[2 0[2]]}
-check_doclist fts2p-1.0.3.2 0 1 test {[2 0[3]]}
-check_doclist fts2p-1.0.3.3 0 1 that {[2 0[0]]}
-check_doclist fts2p-1.0.3.4 0 1 was {[2 0[1]]}
-
-# Segment 0,2
-check_terms   fts2p-1.0.4   0 2 {a is test this}
-check_doclist fts2p-1.0.4.1 0 2 a {[3 0[2]]}
-check_doclist fts2p-1.0.4.2 0 2 is {[3 0[1]]}
-check_doclist fts2p-1.0.4.3 0 2 test {[3 0[3]]}
-check_doclist fts2p-1.0.4.4 0 2 this {[3 0[0]]}
-
-#*************************************************************************
-# Test the segments resulting from inserts followed by a delete.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-  DELETE FROM t1 WHERE rowid = 1;
-}
-
-do_test fts2p-1.1.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0 0 1 0 2 0 3}
-do_test fts2p-1.1.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}}
-
-check_terms_all fts2p-1.1.1 {a is test that this was}
-check_doclist_all fts2p-1.1.1.1 a {[2 0[2]] [3 0[2]]}
-check_doclist_all fts2p-1.1.1.2 is {[3 0[1]]}
-check_doclist_all fts2p-1.1.1.3 test {[2 0[3]] [3 0[3]]}
-check_doclist_all fts2p-1.1.1.4 that {[2 0[0]]}
-check_doclist_all fts2p-1.1.1.5 this {[3 0[0]]}
-check_doclist_all fts2p-1.1.1.6 was {[2 0[1]]}
-
-check_terms fts2p-1.1.2 0 0 {a is test this}
-check_doclist fts2p-1.1.2.1 0 0 a {[1 0[2]]}
-check_doclist fts2p-1.1.2.2 0 0 is {[1 0[1]]}
-check_doclist fts2p-1.1.2.3 0 0 test {[1 0[3]]}
-check_doclist fts2p-1.1.2.4 0 0 this {[1 0[0]]}
-
-check_terms fts2p-1.1.3 0 1 {a test that was}
-check_doclist fts2p-1.1.3.1 0 1 a {[2 0[2]]}
-check_doclist fts2p-1.1.3.2 0 1 test {[2 0[3]]}
-check_doclist fts2p-1.1.3.3 0 1 that {[2 0[0]]}
-check_doclist fts2p-1.1.3.4 0 1 was {[2 0[1]]}
-
-check_terms fts2p-1.1.4 0 2 {a is test this}
-check_doclist fts2p-1.1.4.1 0 2 a {[3 0[2]]}
-check_doclist fts2p-1.1.4.2 0 2 is {[3 0[1]]}
-check_doclist fts2p-1.1.4.3 0 2 test {[3 0[3]]}
-check_doclist fts2p-1.1.4.4 0 2 this {[3 0[0]]}
-
-check_terms fts2p-1.1.5 0 3 {a is test this}
-check_doclist fts2p-1.1.5.1 0 3 a {[1]}
-check_doclist fts2p-1.1.5.2 0 3 is {[1]}
-check_doclist fts2p-1.1.5.3 0 3 test {[1]}
-check_doclist fts2p-1.1.5.4 0 3 this {[1]}
-
-#*************************************************************************
-# Test results when all references to certain tokens are deleted.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-  DELETE FROM t1 WHERE rowid IN (1,3);
-}
-
-# Still 4 segments because 0,3 will contain deletes for rowid 1 and 3.
-do_test fts2p-1.2.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0 0 1 0 2 0 3}
-do_test fts2p-1.2.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}}
-
-check_terms_all fts2p-1.2.1 {a is test that this was}
-check_doclist_all fts2p-1.2.1.1 a {[2 0[2]]}
-check_doclist_all fts2p-1.2.1.2 is {}
-check_doclist_all fts2p-1.2.1.3 test {[2 0[3]]}
-check_doclist_all fts2p-1.2.1.4 that {[2 0[0]]}
-check_doclist_all fts2p-1.2.1.5 this {}
-check_doclist_all fts2p-1.2.1.6 was {[2 0[1]]}
-
-check_terms fts2p-1.2.2 0 0 {a is test this}
-check_doclist fts2p-1.2.2.1 0 0 a {[1 0[2]]}
-check_doclist fts2p-1.2.2.2 0 0 is {[1 0[1]]}
-check_doclist fts2p-1.2.2.3 0 0 test {[1 0[3]]}
-check_doclist fts2p-1.2.2.4 0 0 this {[1 0[0]]}
-
-check_terms fts2p-1.2.3 0 1 {a test that was}
-check_doclist fts2p-1.2.3.1 0 1 a {[2 0[2]]}
-check_doclist fts2p-1.2.3.2 0 1 test {[2 0[3]]}
-check_doclist fts2p-1.2.3.3 0 1 that {[2 0[0]]}
-check_doclist fts2p-1.2.3.4 0 1 was {[2 0[1]]}
-
-check_terms fts2p-1.2.4 0 2 {a is test this}
-check_doclist fts2p-1.2.4.1 0 2 a {[3 0[2]]}
-check_doclist fts2p-1.2.4.2 0 2 is {[3 0[1]]}
-check_doclist fts2p-1.2.4.3 0 2 test {[3 0[3]]}
-check_doclist fts2p-1.2.4.4 0 2 this {[3 0[0]]}
-
-check_terms fts2p-1.2.5 0 3 {a is test this}
-check_doclist fts2p-1.2.5.1 0 3 a {[1] [3]}
-check_doclist fts2p-1.2.5.2 0 3 is {[1] [3]}
-check_doclist fts2p-1.2.5.3 0 3 test {[1] [3]}
-check_doclist fts2p-1.2.5.4 0 3 this {[1] [3]}
-
-#*************************************************************************
-# Test results when everything is optimized manually.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-  DELETE FROM t1 WHERE rowid IN (1,3);
-  DROP TABLE IF EXISTS t1old;
-  ALTER TABLE t1 RENAME TO t1old;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) SELECT rowid, c FROM t1old;
-  DROP TABLE t1old;
-}
-
-# Should be a single optimal segment with the same logical results.
-do_test fts2p-1.3.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0}
-do_test fts2p-1.3.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}}
-
-check_terms_all fts2p-1.3.1 {a test that was}
-check_doclist_all fts2p-1.3.1.1 a {[2 0[2]]}
-check_doclist_all fts2p-1.3.1.2 test {[2 0[3]]}
-check_doclist_all fts2p-1.3.1.3 that {[2 0[0]]}
-check_doclist_all fts2p-1.3.1.4 was {[2 0[1]]}
-
-check_terms fts2p-1.3.2 0 0 {a test that was}
-check_doclist fts2p-1.3.2.1 0 0 a {[2 0[2]]}
-check_doclist fts2p-1.3.2.2 0 0 test {[2 0[3]]}
-check_doclist fts2p-1.3.2.3 0 0 that {[2 0[0]]}
-check_doclist fts2p-1.3.2.4 0 0 was {[2 0[1]]}
-
-finish_test
diff --git a/test/fts2q.test b/test/fts2q.test
deleted file mode 100644
index cba78d583f..0000000000
--- a/test/fts2q.test
+++ /dev/null
@@ -1,346 +0,0 @@
-# 2008 June 26
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The focus
-# of this script is testing the FTS2 module's optimize() function.
-#
-# $Id: fts2q.test,v 1.2 2008/07/22 23:49:44 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is not defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-#*************************************************************************
-# Probe to see if support for the FTS2 dump_* functions is compiled in.
-# TODO(shess): Change main.mk to do the right thing and remove this test.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'x');
-}
-
-set s {SELECT dump_terms(t1, 1) FROM t1 LIMIT 1}
-set r {1 {unable to use function dump_terms in the requested context}}
-if {[catchsql $s]==$r} {
-  finish_test
-  return
-}
-
-#*************************************************************************
-# Utility function to check for the expected terms in the segment
-# level/index.  _all version does same but for entire index.
-proc check_terms {test level index terms} {
-  # TODO(shess): Figure out why uplevel in do_test can't catch
-  # $level and $index directly.
-  set ::level $level
-  set ::index $index
-  do_test $test.terms {
-    execsql {
-      SELECT dump_terms(t1, $::level, $::index) FROM t1 LIMIT 1;
-    }
-  } [list $terms]
-}
-proc check_terms_all {test terms} {
-  do_test $test.terms {
-    execsql {
-      SELECT dump_terms(t1) FROM t1 LIMIT 1;
-    }
-  } [list $terms]
-}
-
-# Utility function to check for the expected doclist for the term in
-# segment level/index.  _all version does same for entire index.
-proc check_doclist {test level index term doclist} {
-  # TODO(shess): Again, why can't the non-:: versions work?
-  set ::term $term
-  set ::level $level
-  set ::index $index
-  do_test $test {
-    execsql {
-      SELECT dump_doclist(t1, $::term, $::level, $::index) FROM t1 LIMIT 1;
-    }
-  } [list $doclist]
-}
-proc check_doclist_all {test term doclist} {
-  set ::term $term
-  do_test $test {
-    execsql {
-      SELECT dump_doclist(t1, $::term) FROM t1 LIMIT 1;
-    }
-  } [list $doclist]
-}
-
-#*************************************************************************
-# Test results when all rows are deleted and one is added back.
-# Previously older segments would continue to exist, but now the index
-# should be dropped when the table is empty.  The results should look
-# exactly like we never added the earlier rows in the first place.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-  DELETE FROM t1 WHERE 1=1; -- Delete each row rather than dropping table.
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-}
-
-# Should be a single initial segment.
-do_test fts2q-1.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0}
-do_test fts2q-1.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} {{0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}}
-
-check_terms_all fts2q-1.1 {a is test this}
-check_doclist_all fts2q-1.1.1 a {[1 0[2]]}
-check_doclist_all fts2q-1.1.2 is {[1 0[1]]}
-check_doclist_all fts2q-1.1.3 test {[1 0[3]]}
-check_doclist_all fts2q-1.1.4 this {[1 0[0]]}
-
-check_terms   fts2q-1.2   0 0 {a is test this}
-check_doclist fts2q-1.2.1 0 0 a {[1 0[2]]}
-check_doclist fts2q-1.2.2 0 0 is {[1 0[1]]}
-check_doclist fts2q-1.2.3 0 0 test {[1 0[3]]}
-check_doclist fts2q-1.2.4 0 0 this {[1 0[0]]}
-
-#*************************************************************************
-# Test results when everything is optimized manually.
-# NOTE(shess): This is a copy of fts2c-1.3.  I've pulled a copy here
-# because fts2q-2 and fts2q-3 should have identical results.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-  DELETE FROM t1 WHERE rowid IN (1,3);
-  DROP TABLE IF EXISTS t1old;
-  ALTER TABLE t1 RENAME TO t1old;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) SELECT rowid, c FROM t1old;
-  DROP TABLE t1old;
-}
-
-# Should be a single optimal segment with the same logical results.
-do_test fts2q-2.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0}
-do_test fts2q-2.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}}
-
-check_terms_all fts2q-2.1 {a test that was}
-check_doclist_all fts2q-2.1.1 a {[2 0[2]]}
-check_doclist_all fts2q-2.1.2 test {[2 0[3]]}
-check_doclist_all fts2q-2.1.3 that {[2 0[0]]}
-check_doclist_all fts2q-2.1.4 was {[2 0[1]]}
-
-check_terms fts2q-2.2 0 0 {a test that was}
-check_doclist fts2q-2.2.1 0 0 a {[2 0[2]]}
-check_doclist fts2q-2.2.2 0 0 test {[2 0[3]]}
-check_doclist fts2q-2.2.3 0 0 that {[2 0[0]]}
-check_doclist fts2q-2.2.4 0 0 was {[2 0[1]]}
-
-#*************************************************************************
-# Test results when everything is optimized via optimize().
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-  DELETE FROM t1 WHERE rowid IN (1,3);
-  SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;
-}
-
-# Should be a single optimal segment with the same logical results.
-do_test fts2q-3.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0}
-do_test fts2q-3.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} {{0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4}}
-
-check_terms_all fts2q-3.1 {a test that was}
-check_doclist_all fts2q-3.1.1 a {[2 0[2]]}
-check_doclist_all fts2q-3.1.2 test {[2 0[3]]}
-check_doclist_all fts2q-3.1.3 that {[2 0[0]]}
-check_doclist_all fts2q-3.1.4 was {[2 0[1]]}
-
-check_terms fts2q-3.2 0 0 {a test that was}
-check_doclist fts2q-3.2.1 0 0 a {[2 0[2]]}
-check_doclist fts2q-3.2.2 0 0 test {[2 0[3]]}
-check_doclist fts2q-3.2.3 0 0 that {[2 0[0]]}
-check_doclist fts2q-3.2.4 0 0 was {[2 0[1]]}
-
-#*************************************************************************
-# Test optimize() against a table involving segment merges.
-# NOTE(shess): Since there's no transaction, each of the INSERT/UPDATE
-# statements generates a segment.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-
-  UPDATE t1 SET c = 'This is a test one' WHERE rowid = 1;
-  UPDATE t1 SET c = 'That was a test one' WHERE rowid = 2;
-  UPDATE t1 SET c = 'This is a test one' WHERE rowid = 3;
-
-  UPDATE t1 SET c = 'This is a test two' WHERE rowid = 1;
-  UPDATE t1 SET c = 'That was a test two' WHERE rowid = 2;
-  UPDATE t1 SET c = 'This is a test two' WHERE rowid = 3;
-
-  UPDATE t1 SET c = 'This is a test three' WHERE rowid = 1;
-  UPDATE t1 SET c = 'That was a test three' WHERE rowid = 2;
-  UPDATE t1 SET c = 'This is a test three' WHERE rowid = 3;
-
-  UPDATE t1 SET c = 'This is a test four' WHERE rowid = 1;
-  UPDATE t1 SET c = 'That was a test four' WHERE rowid = 2;
-  UPDATE t1 SET c = 'This is a test four' WHERE rowid = 3;
-
-  UPDATE t1 SET c = 'This is a test' WHERE rowid = 1;
-  UPDATE t1 SET c = 'That was a test' WHERE rowid = 2;
-  UPDATE t1 SET c = 'This is a test' WHERE rowid = 3;
-}
-
-# 2 segments in level 0, 1 in level 1 (18 segments created, 16
-# merged).
-do_test fts2q-4.segments {
-  execsql {
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {0 0 0 1 1 0}
-
-do_test fts2q-4.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \
-        {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \
-        {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}]
-
-check_terms_all fts2q-4.1      {a four is one test that this three two was}
-check_doclist_all fts2q-4.1.1  a {[1 0[2]] [2 0[2]] [3 0[2]]}
-check_doclist_all fts2q-4.1.2  four {}
-check_doclist_all fts2q-4.1.3  is {[1 0[1]] [3 0[1]]}
-check_doclist_all fts2q-4.1.4  one {}
-check_doclist_all fts2q-4.1.5  test {[1 0[3]] [2 0[3]] [3 0[3]]}
-check_doclist_all fts2q-4.1.6  that {[2 0[0]]}
-check_doclist_all fts2q-4.1.7  this {[1 0[0]] [3 0[0]]}
-check_doclist_all fts2q-4.1.8  three {}
-check_doclist_all fts2q-4.1.9  two {}
-check_doclist_all fts2q-4.1.10 was {[2 0[1]]}
-
-check_terms fts2q-4.2     0 0 {a four test that was}
-check_doclist fts2q-4.2.1 0 0 a {[2 0[2]]}
-check_doclist fts2q-4.2.2 0 0 four {[2]}
-check_doclist fts2q-4.2.3 0 0 test {[2 0[3]]}
-check_doclist fts2q-4.2.4 0 0 that {[2 0[0]]}
-check_doclist fts2q-4.2.5 0 0 was {[2 0[1]]}
-
-check_terms fts2q-4.3     0 1 {a four is test this}
-check_doclist fts2q-4.3.1 0 1 a {[3 0[2]]}
-check_doclist fts2q-4.3.2 0 1 four {[3]}
-check_doclist fts2q-4.3.3 0 1 is {[3 0[1]]}
-check_doclist fts2q-4.3.4 0 1 test {[3 0[3]]}
-check_doclist fts2q-4.3.5 0 1 this {[3 0[0]]}
-
-check_terms fts2q-4.4      1 0 {a four is one test that this three two was}
-check_doclist fts2q-4.4.1  1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]}
-check_doclist fts2q-4.4.2  1 0 four {[1] [2 0[4]] [3 0[4]]}
-check_doclist fts2q-4.4.3  1 0 is {[1 0[1]] [3 0[1]]}
-check_doclist fts2q-4.4.4  1 0 one {[1] [2] [3]}
-check_doclist fts2q-4.4.5  1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]}
-check_doclist fts2q-4.4.6  1 0 that {[2 0[0]]}
-check_doclist fts2q-4.4.7  1 0 this {[1 0[0]] [3 0[0]]}
-check_doclist fts2q-4.4.8  1 0 three {[1] [2] [3]}
-check_doclist fts2q-4.4.9  1 0 two {[1] [2] [3]}
-check_doclist fts2q-4.4.10 1 0 was {[2 0[1]]}
-
-# Optimize should leave the result in the level of the highest-level
-# prior segment.
-do_test fts2q-4.5 {
-  execsql {
-    SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {{Index optimized} 1 0}
-
-# Identical to fts2q-4.matches.
-do_test fts2q-4.5.matches {
-  execsql {
-    SELECT OFFSETS(t1) FROM t1
-     WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY rowid;
-  }
-} [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \
-        {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \
-        {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}]
-
-check_terms_all fts2q-4.5.1     {a is test that this was}
-check_doclist_all fts2q-4.5.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]}
-check_doclist_all fts2q-4.5.1.2 is {[1 0[1]] [3 0[1]]}
-check_doclist_all fts2q-4.5.1.3 test {[1 0[3]] [2 0[3]] [3 0[3]]}
-check_doclist_all fts2q-4.5.1.4 that {[2 0[0]]}
-check_doclist_all fts2q-4.5.1.5 this {[1 0[0]] [3 0[0]]}
-check_doclist_all fts2q-4.5.1.6 was {[2 0[1]]}
-
-check_terms fts2q-4.5.2     1 0 {a is test that this was}
-check_doclist fts2q-4.5.2.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]}
-check_doclist fts2q-4.5.2.2 1 0 is {[1 0[1]] [3 0[1]]}
-check_doclist fts2q-4.5.2.3 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]}
-check_doclist fts2q-4.5.2.4 1 0 that {[2 0[0]]}
-check_doclist fts2q-4.5.2.5 1 0 this {[1 0[0]] [3 0[0]]}
-check_doclist fts2q-4.5.2.6 1 0 was {[2 0[1]]}
-
-# Re-optimizing does nothing.
-do_test fts2q-5.0 {
-  execsql {
-    SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {{Index already optimal} 1 0}
-
-# Even if we move things around, still does nothing.
-do_test fts2q-5.1 {
-  execsql {
-    UPDATE t1_segdir SET level = 2 WHERE level = 1 AND idx = 0;
-    SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;
-    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
-  }
-} {{Index already optimal} 2 0}
-
-finish_test
diff --git a/test/fts2r.test b/test/fts2r.test
deleted file mode 100644
index c0be367115..0000000000
--- a/test/fts2r.test
+++ /dev/null
@@ -1,121 +0,0 @@
-# 2008 July 29
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# These tests exercise the various types of fts2 cursors.
-#
-# $Id: fts2r.test,v 1.1 2008/07/29 20:38:18 shess Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is not defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-#*************************************************************************
-# Test table scan (QUERY_GENERIC).  This kind of query happens for
-# queries with no WHERE clause, or for WHERE clauses which cannot be
-# satisfied by an index.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  INSERT INTO t1 (rowid, c) VALUES (1, 'This is a test');
-  INSERT INTO t1 (rowid, c) VALUES (2, 'That was a test');
-  INSERT INTO t1 (rowid, c) VALUES (3, 'This is a test');
-}
-
-do_test fts2e-1.1 {
-  execsql {
-    SELECT rowid FROM t1 ORDER BY rowid;
-  }
-} {1 2 3}
-
-do_test fts2e-1.2 {
-  execsql {
-    SELECT rowid FROM t1 WHERE c LIKE '%test' ORDER BY rowid;
-  }
-} {1 2 3}
-
-do_test fts2e-1.3 {
-  execsql {
-    SELECT rowid FROM t1 WHERE c LIKE 'That%' ORDER BY rowid;
-  }
-} {2}
-
-#*************************************************************************
-# Test lookup by rowid (QUERY_ROWID).  This kind of query happens for
-# queries which select by the rowid implicit index.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  DROP TABLE IF EXISTS t2;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  CREATE TABLE t2(id INTEGER PRIMARY KEY AUTOINCREMENT, weight INTEGER UNIQUE);
-  INSERT INTO t2 VALUES (null, 10);
-  INSERT INTO t1 (rowid, c) VALUES (last_insert_rowid(), 'This is a test');
-  INSERT INTO t2 VALUES (null, 5);
-  INSERT INTO t1 (rowid, c) VALUES (last_insert_rowid(), 'That was a test');
-  INSERT INTO t2 VALUES (null, 20);
-  INSERT INTO t1 (rowid, c) VALUES (last_insert_rowid(), 'This is a test');
-}
-
-# TODO(shess): This actually is doing QUERY_GENERIC?  I'd have
-# expected QUERY_ROWID in this case, as for a very large table the
-# full scan is less efficient.
-do_test fts2e-2.1 {
-  execsql {
-    SELECT rowid FROM t1 WHERE rowid in (1, 2, 10);
-  }
-} {1 2}
-
-do_test fts2e-2.2 {
-  execsql {
-    SELECT t1.rowid, weight FROM t1, t2 WHERE t2.id = t1.rowid ORDER BY weight;
-  }
-} {2 5 1 10 3 20}
-
-do_test fts2e-2.3 {
-  execsql {
-    SELECT t1.rowid, weight FROM t1, t2
-           WHERE t2.weight>5 AND t2.id = t1.rowid ORDER BY weight;
-  }
-} {1 10 3 20}
-
-#*************************************************************************
-# Test lookup by MATCH (QUERY_FULLTEXT).  This is the fulltext index.
-db eval {
-  DROP TABLE IF EXISTS t1;
-  DROP TABLE IF EXISTS t2;
-  CREATE VIRTUAL TABLE t1 USING fts2(c);
-  CREATE TABLE t2(id INTEGER PRIMARY KEY AUTOINCREMENT, weight INTEGER UNIQUE);
-  INSERT INTO t2 VALUES (null, 10);
-  INSERT INTO t1 (rowid, c) VALUES (last_insert_rowid(), 'This is a test');
-  INSERT INTO t2 VALUES (null, 5);
-  INSERT INTO t1 (rowid, c) VALUES (last_insert_rowid(), 'That was a test');
-  INSERT INTO t2 VALUES (null, 20);
-  INSERT INTO t1 (rowid, c) VALUES (last_insert_rowid(), 'This is a test');
-}
-
-do_test fts2e-3.1 {
-  execsql {
-    SELECT rowid FROM t1 WHERE t1 MATCH 'this' ORDER BY rowid;
-  }
-} {1 3}
-
-do_test fts2e-3.2 {
-  execsql {
-    SELECT t1.rowid, weight FROM t1, t2
-     WHERE t1 MATCH 'this' AND t1.rowid = t2.id ORDER BY weight;
-  }
-} {1 10 3 20}
-
-finish_test
diff --git a/test/fts2token.test b/test/fts2token.test
deleted file mode 100644
index de5f94d7fc..0000000000
--- a/test/fts2token.test
+++ /dev/null
@@ -1,174 +0,0 @@
-# 2007 June 21
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library. The focus 
-# of this script is testing the pluggable tokeniser feature of the 
-# FTS2 module.
-#
-# $Id: fts2token.test,v 1.3 2007/06/25 12:05:40 danielk1977 Exp $
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-# If SQLITE_ENABLE_FTS2 is defined, omit this file.
-ifcapable !fts2 {
-  finish_test
-  return
-}
-
-proc escape_string {str} {
-  set out ""
-  foreach char [split $str ""] {
-    scan $char %c i
-    if {$i<=127} {
-      append out $char
-    } else {
-      append out [format {\x%.4x} $i]
-    }
-  }
-  set out
-}
-
-#--------------------------------------------------------------------------
-# Test cases fts2token-1.* are the warm-body test for the SQL scalar
-# function fts2_tokenizer(). The procedure is as follows:
-#
-#   1: Verify that there is no such fts2 tokenizer as 'blah'.
-#
-#   2: Query for the built-in tokenizer 'simple'. Insert a copy of the
-#      retrieved value as tokenizer 'blah'.
-#
-#   3: Test that the value returned for tokenizer 'blah' is now the
-#      same as that retrieved for 'simple'.
-#
-#   4: Test that it is now possible to create an fts2 table using 
-#      tokenizer 'blah' (it was not possible in step 1).
-#
-#   5: Test that the table created to use tokenizer 'blah' is usable.
-#
-do_test fts2token-1.1 {
-  catchsql {
-    CREATE VIRTUAL TABLE t1 USING fts2(content, tokenize blah);
-  }
-} {1 {unknown tokenizer: blah}}
-do_test fts2token-1.2 {
-  execsql {
-    SELECT fts2_tokenizer('blah', fts2_tokenizer('simple')) IS NULL;
-  }
-} {0}
-do_test fts2token-1.3 {
-  execsql {
-    SELECT fts2_tokenizer('blah') == fts2_tokenizer('simple');
-  }
-} {1}
-do_test fts2token-1.4 {
-  catchsql {
-    CREATE VIRTUAL TABLE t1 USING fts2(content, tokenize blah);
-  }
-} {0 {}}
-do_test fts2token-1.5 {
-  execsql {
-    INSERT INTO t1(content) VALUES('There was movement at the station');
-    INSERT INTO t1(content) VALUES('For the word has passed around');
-    INSERT INTO t1(content) VALUES('That the colt from ol regret had got away');
-    SELECT content FROM t1 WHERE content MATCH 'movement'
-  }
-} {{There was movement at the station}}
-
-#--------------------------------------------------------------------------
-# Test cases fts2token-2.* test error cases in the scalar function based
-# API for getting and setting tokenizers.
-#
-do_test fts2token-2.1 {
-  catchsql {
-    SELECT fts2_tokenizer('nosuchtokenizer');
-  }
-} {1 {unknown tokenizer: nosuchtokenizer}}
-
-#--------------------------------------------------------------------------
-# Test cases fts2token-3.* test the three built-in tokenizers with a
-# simple input string via the built-in test function. This is as much
-# to test the test function as the tokenizer implementations.
-#
-do_test fts2token-3.1 {
-  execsql {
-    SELECT fts2_tokenizer_test('simple', 'I don''t see how');
-  }
-} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
-do_test fts2token-3.2 {
-  execsql {
-    SELECT fts2_tokenizer_test('porter', 'I don''t see how');
-  }
-} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
-ifcapable icu {
-  do_test fts2token-3.3 {
-    execsql {
-      SELECT fts2_tokenizer_test('icu', 'I don''t see how');
-    }
-  } {{0 i I 1 don't don't 2 see see 3 how how}}
-}
-
-#--------------------------------------------------------------------------
-# Test cases fts2token-4.* test the ICU tokenizer. In practice, this
-# tokenizer only has two modes - "thai" and "everybody else". Some other
-# Asian languages (Lao, Khmer etc.) require the same special treatment as 
-# Thai, but ICU doesn't support them yet.
-#
-ifcapable icu {
-
-  proc do_icu_test {name locale input output} {
-    set ::out [db eval { SELECT fts2_tokenizer_test('icu', $locale, $input) }]
-    do_test $name {
-      lindex $::out 0
-    } $output
-  }
-  
-  do_icu_test fts2token-4.1 en_US  {}   {}
-  do_icu_test fts2token-4.2 en_US {Test cases fts2} [list \
-    0 test Test 1 cases cases 2 fts2 fts2
-  ]
-
-  # The following test shows that ICU is smart enough to recognise
-  # Thai chararacters, even when the locale is set to English/United 
-  # States.
-  #
-  set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a"
-  set output    "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 "
-  append output "1 \u0e19\u0e30 \u0e19\u0e30 "
-  append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"
-
-  do_icu_test fts2token-4.3 th_TH  $input $output
-  do_icu_test fts2token-4.4 en_US  $input $output
-
-  # ICU handles an unknown locale by falling back to the default.
-  # So this is not an error.
-  do_icu_test fts2token-4.5 MiddleOfTheOcean  $input $output
-
-  set    longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"
-  append longtoken "AReallocInTheIcuTokenizerCode"
-
-  set    input "short tokens then "
-  append input $longtoken
-  set    output "0 short short "
-  append output "1 tokens tokens "
-  append output "2 then then "
-  append output "3 [string tolower $longtoken] $longtoken"
-
-  do_icu_test fts2token-4.6 MiddleOfTheOcean  $input $output
-  do_icu_test fts2token-4.7 th_TH  $input $output
-  do_icu_test fts2token-4.8 en_US  $input $output
-}
-
-do_test fts2token-internal {
-  execsql { SELECT fts2_tokenizer_internal_test() }
-} {ok}
-
-finish_test
diff --git a/test/fts3atoken2.test b/test/fts3atoken2.test
new file mode 100644
index 0000000000..f6a2a29ab1
--- /dev/null
+++ b/test/fts3atoken2.test
@@ -0,0 +1,106 @@
+# 2025 September 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library. The focus 
+# of this script is testing the pluggable tokeniser feature of the 
+# FTS3 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+set ::testprefix fts3atoken2
+
+
+reset_db
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0
+
+# With ENABLE_FTS3_TOKENIZER set to 0:
+#
+#   * It is not possible to get a pointer to a token implementation
+#     using single arg fts3_tokenize() unless the name of the tokenizer
+#     is a bound paramter - function should return NULL.
+#
+#   * But it is possible with a bound parameter.
+#
+do_execsql_test 1.1.1 {
+  SELECT typeof( fts3_tokenizer('simple') );
+} {null}
+set bound "simple"
+do_execsql_test 1.1.2 {
+  SELECT typeof( fts3_tokenizer($bound) );
+} {blob}
+
+# With ENABLE_FTS3_TOKENIZER set to 0:
+#
+#   * It is not possible to create a token implementation using anything
+#     other than a bound parameter.
+#
+#   * But it is possible with a bound parameter.
+#
+set literal [db one {SELECT quote( fts3_tokenizer($bound) )}]
+set blob [db one {SELECT fts3_tokenizer($bound) }]
+
+do_catchsql_test 1.2.1 "
+  SELECT fts3_tokenizer('mytok', $literal)
+" {1 {fts3tokenize disabled}}
+do_catchsql_test 1.2.2 {
+  CREATE VIRTUAL TABLE x1 USING fts3(col, tokenize=mytok);
+} {1 {unknown tokenizer: mytok}}
+do_catchsql_test 1.2.3 {
+  SELECT fts3_tokenizer('mytok', $blob)
+} {0 {{}}}
+do_execsql_test 1.2.4 {
+  CREATE VIRTUAL TABLE x1 USING fts3(col, tokenize=mytok);
+}
+
+# With ENABLE_FTS3_TOKENIZER set to 1:
+#
+#   * It is possible to get a pointer to a token implementation with either
+#     a bound parameter or a literal.
+#
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
+set bound "simple"
+do_execsql_test 1.3.1 {
+  SELECT typeof( fts3_tokenizer('simple') );
+} {blob}
+do_execsql_test 1.3.2 {
+  SELECT typeof( fts3_tokenizer($bound) );
+} {blob}
+
+# With ENABLE_FTS3_TOKENIZER set to 1:
+#
+#   * It is not possible to create a token implementation using either
+#     a bound parameter or a literal.
+#
+set literal [db one {SELECT quote( fts3_tokenizer($bound) )}]
+set blob    [db one {SELECT fts3_tokenizer($bound) }]
+
+do_execsql_test 1.4.1 "
+  SELECT typeof( fts3_tokenizer('mytok2', $literal) );
+" {blob}
+do_execsql_test 1.4.2 {
+  CREATE VIRTUAL TABLE x2 USING fts3(col, tokenize=mytok2);
+}
+do_execsql_test 1.4.3 {
+  SELECT typeof( fts3_tokenizer('mytok3', $blob) );
+} {blob}
+do_execsql_test 1.4.4 {
+  CREATE VIRTUAL TABLE x3 USING fts3(col, tokenize=mytok3);
+}
+
+finish_test
+
diff --git a/test/fts3conf.test b/test/fts3conf.test
index 6ceef2c47e..cd48290195 100644
--- a/test/fts3conf.test
+++ b/test/fts3conf.test
@@ -198,7 +198,8 @@ do_execsql_test 4.1.2 {
 do_execsql_test 4.1.3 {
   SELECT * FROM t0 WHERE t0 MATCH 'abc';
   INSERT INTO t0(t0) VALUES('integrity-check');
-} {}
+  PRAGMA integrity_check;
+} {ok}
 
 do_execsql_test 4.2.1 {
   CREATE VIRTUAL TABLE t01 USING fts4;
@@ -211,7 +212,8 @@ do_execsql_test 4.2.1 {
 do_execsql_test 4.2.2 {
   SELECT * FROM t01 WHERE t01 MATCH 'b';
   INSERT INTO t01(t01) VALUES('integrity-check');
-} {}
+  PRAGMA integrity_check;
+} {ok}
 
 do_execsql_test 4.3.1 {
   CREATE VIRTUAL TABLE t02 USING fts4;
diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test
index f8e89b3a75..01effa0850 100644
--- a/test/fts3corrupt4.test
+++ b/test/fts3corrupt4.test
@@ -2595,17 +2595,13 @@ do_execsql_test 17.1 {
     UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thread*';
 }
 
-do_catchsql_test 17.2 {
-  DROP TABLE IF EXISTS t1;
-} {1 {SQL logic error}}
-
-do_execsql_test 17.3 {
+do_execsql_test 17.2 {
   INSERT INTO t1(t1) VALUES('optimize');
 }
 
-do_catchsql_test 17.4 {
+do_catchsql_test 17.3 {
   DROP TABLE IF EXISTS t1;
-} {1 {SQL logic error}}
+} {0 {}}
 
 #-------------------------------------------------------------------------
 reset_db
@@ -4380,6 +4376,8 @@ do_test 25.0 {
 | end crash-dde9e76ed8ab2d.db
 }]} {}
 
+reset_prng_state
+
 do_catchsql_test 25.1 {
   PRAGMA writable_schema = 1;
   WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x%1 FROM c WHERE x<599237)
@@ -4405,10 +4403,32 @@ do_catchsql_test 25.5 {
     INSERT INTO t1( a ) SELECT randomblob(3000) FROM t2 ;
 } {0 {}}
 
-do_catchsql_test 25.6 {
-  INSERT INTO t1(t1) SELECT x FROM t2;
-  INSERT INTO t1(t1) SELECT x FROM t2;
-} {1 {database disk image is malformed}}
+if {$tcl_platform(byteOrder)=="littleEndian"} {
+  # The SQLITE_CORRUPT error depends on the specific random byte
+  # sequence generated by SQLite's PRNG.  But the SQLite PRNG
+  # uses ChaCha20, which generates a different byte sequence on
+  # big-endian and little-endian platforms.  The SQLITE_CORRUPT
+  # error only comes up when the pseudo-random byte sequence is
+  # the one generated on little-endian platforms.
+  #
+  # See Forum thread:
+  # https://sqlite.org/forum/forumpost/b5f89d813babfd88
+  #
+  do_catchsql_test 25.6a {
+    INSERT INTO t1(t1) SELECT x FROM t2;
+  } {1 {database disk image is malformed}}
+  do_catchsql_test 25.6b {
+    INSERT INTO t1(t1) SELECT x FROM t2;
+  } {1 {database disk image is malformed}}
+} else {
+  do_catchsql_test 25.6a {
+    INSERT INTO t1(t1) SELECT x FROM t2;
+  } {0 {}}
+  do_catchsql_test 25.6b {
+    INSERT INTO t1(t1) SELECT x FROM t2;
+  } {0 {}}
+}
+
 
 #-------------------------------------------------------------------------
 reset_db
@@ -5812,6 +5832,9 @@ do_execsql_test 35.0 {
 do_catchsql_test 35.1 {
   INSERT INTO f(f) VALUES ('integrity-check');
 } {1 {database disk image is malformed}}
+do_execsql_test 35.2 {
+  PRAGMA integrity_check;
+} {{malformed inverted index for FTS3 table main.f}}
 
 reset_db
 do_catchsql_test 36.0 {
diff --git a/test/fts3corrupt7.test b/test/fts3corrupt7.test
new file mode 100644
index 0000000000..6cf9c9a9dc
--- /dev/null
+++ b/test/fts3corrupt7.test
@@ -0,0 +1,280 @@
+# 2024 November 7
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/fts3_common.tcl
+set testprefix fts3corrupt7
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+sqlite3_fts3_may_be_corrupt 1
+database_may_be_corrupt
+extra_schema_checks 0
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 1.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+| size 28672 pagesize 4096 filename x.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 04 00 00 00 07   .....@  ........
+|     32: 00 00 00 02 00 00 00 01 00 00 00 04 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04   ................
+|     96: 00 2e 82 40 0d 0e b1 00 06 0d a4 00 0f 8d 0f 21   ...@...........!
+|    112: 0e b9 0d c8 0e 7e 0d a4 7d a4 00 00 00 00 00 00   .....~..........
+|   2512: 00 00 00 00 00 00 00 00 96 00 00 00 00 00 00 00   ................
+|   3488: 00 00 00 00 22 07 06 17 11 11 01 31 74 61 62 6c   ...........1tabl
+|   3504: 65 74 32 74 32 07 43 52 45 41 54 45 20 54 41 42   et2t2.CREATE TAB
+|   3520: 4c 45 20 74 32 28 78 29 81 33 05 07 17 1f 1f 01   LE t2(x).3......
+|   3536: 82 35 74 61 62 6c 65 74 31 5f 73 65 67 64 69 72   .5tablet1_segdir
+|   3552: 74 31 5f 73 65 67 64 69 72 05 43 52 45 41 54 45   t1_segdir.CREATE
+|   3568: 20 54 41 42 4c 45 20 27 74 31 5f 73 65 67 64 69    TABLE 't1_segdi
+|   3584: 72 27 28 6c 65 76 65 6c 20 49 4e 54 45 47 45 52   r'(level INTEGER
+|   3600: 2c 69 64 78 20 49 4e 54 45 47 45 52 2c 73 74 61   ,idx INTEGER,sta
+|   3616: 72 74 5f 62 6c 6f 63 6b 20 49 4e 54 45 47 45 52   rt_block INTEGER
+|   3632: 2c 6c 65 61 76 65 73 5f 65 6e 64 5f 62 6c 6f 63   ,leaves_end_bloc
+|   3648: 6b 20 49 4e 54 45 47 45 52 2c 65 6e 64 5f 62 6c   k INTEGER,end_bl
+|   3664: 6f 63 6b 20 49 4e 54 45 47 45 52 2c 72 6f 6f 74   ock INTEGER,root
+|   3680: 20 42 4c 4f 42 2c 50 52 49 4d 41 52 59 20 4b 45    BLOB,PRIMARY KE
+|   3696: 59 28 6c 65 76 65 6c 2c 20 69 64 78 29 29 31 06   Y(level, idx))1.
+|   3712: 06 17 45 1f 01 00 69 6e 64 65 78 73 71 6c 69 74   ..E...indexsqlit
+|   3728: 65 5f 61 75 74 6f 69 6e 64 65 78 5f 74 31 5f 73   e_autoindex_t1_s
+|   3744: 65 67 64 69 72 5f 31 74 31 5f 73 65 67 64 69 72   egdir_1t1_segdir
+|   3760: 06 0f c7 00 08 00 00 00 00 66 04 07 17 23 23 01   .........f...##.
+|   3776: 81 13 74 61 62 6c 65 74 31 5f 73 65 67 6d 65 6e   ..tablet1_segmen
+|   3792: 74 73 74 31 5f 73 65 67 6d 65 6e 74 73 04 43 52   tst1_segments.CR
+|   3808: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 73   EATE TABLE 't1_s
+|   3824: 65 67 6d 65 6e 74 73 27 28 62 6c 6f 63 6b 69 64   egments'(blockid
+|   3840: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59    INTEGER PRIMARY
+|   3856: 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42    KEY, block BLOB
+|   3872: 29 6a 03 07 17 21 21 01 81 1f 74 61 62 6c 65 74   )j...!!...tablet
+|   3888: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74   1_contentt1_cont
+|   3904: 65 6e 74 03 43 52 45 41 54 45 20 54 41 42 4c 45   ent.CREATE TABLE
+|   3920: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 64 6f    't1_content'(do
+|   3936: 63 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   cid INTEGER PRIM
+|   3952: 41 52 59 20 4b 45 59 2c 20 27 63 30 61 27 2c 20   ARY KEY, 'c0a', 
+|   3968: 27 63 31 62 27 2c 20 27 63 32 63 27 29 38 02 06   'c1b', 'c2c')8..
+|   3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52   ...._tablet1t1CR
+|   4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42   EATE VIRTUAL TAB
+|   4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 33   LE t1 USING fts3
+|   4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00   (a,b,c)...9.....
+| page 2 offset 4096
+|      0: 01 00 00 00 00 01 00 00 00 00 01 00 00 00 00 01   ................
+|     16: 00 00 00 00 02 00 00 00 00 05 00 00 00 03 02 00   ................
+|     32: 00 00 00 05 00 00 00 03 02 00 00 00 00 05 00 00   ................
+|     48: 00 03 02 00 00 00 00 05 00 00 00 03 02 00 00 00   ................
+|     64: 00 05 00 00 00 03 02 00 00 00 00 05 00 00 00 03   ................
+|     80: 02 00 00 00 00 05 00 00 00 03 02 00 00 00 00 05   ................
+|     96: 00 00 00 03 02 00 00 00 00 05 00 00 00 03 05 00   ................
+|    112: 00 00 03 03 00 00 00 23 02 00 00 00 00 03 00 00   .......#........
+|    128: 00 23 02 00 00 00 00 03 00 00 4d 5a 14 00 ae 7c   .#........MZ...|
+|   1088: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09   ................
+| page 3 offset 8192
+|      0: 0d 00 00 00 25 0b 48 00 0f d8 0f af 0f 86 0f 74   ....%.H........t
+|     16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5   .a.N./..........
+|     32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5   ...t.[.@.$......
+|     48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb   .......h.O.5....
+|     64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a   .......x.W.>.$..
+|     80: 0b 48 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .H..............
+|   2880: 00 00 00 00 00 00 00 00 81 3f 25 06 00 82 7f 00   .........?%.....
+|   2896: 00 43 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e   .COMPILER=gcc-5.
+|   2912: 34 2e 30 20 32 30 31 36 30 36 30 39 20 44 45 42   4.0 20160609 DEB
+|   2928: 55 47 20 45 4e 41 42 4c 45 20 44 42 53 54 41 54   UG ENABLE DBSTAT
+|   2944: 20 56 54 41 42 20 45 4e 41 42 4c 45 20 46 54 53    VTAB ENABLE FTS
+|   2960: 34 20 45 4e 41 42 4c 45 20 46 54 53 35 20 45 4e   4 ENABLE FTS5 EN
+|   2976: 41 42 4c 45 20 47 45 4f 50 4f 4c 59 20 45 4e 41   ABLE GEOPOLY ENA
+|   2992: 42 4c 45 20 4a 53 4f 4e 31 20 45 4e 41 42 4c 45   BLE JSON1 ENABLE
+|   3008: 20 4d 45 4d 53 59 53 35 20 45 4e 41 42 4c 45 20    MEMSYS5 ENABLE 
+|   3024: 52 54 52 45 45 20 4d 41 58 20 4d 45 4d 4f 52 59   RTREE MAX MEMORY
+|   3040: 3d 35 30 30 30 30 30 30 30 20 4f 4d 49 54 20 4c   =50000000 OMIT L
+|   3056: 4f 41 44 20 45 58 54 45 4e 53 49 4f 4e 20 54 48   OAD EXTENSION TH
+|   3072: 52 45 41 44 53 41 46 45 3d 30 18 24 05 00 25 0f   READSAFE=0.$..%.
+|   3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49   .THREADSAFE=0XBI
+|   3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41   NARY.#..%..THREA
+|   3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22   DSAFE=0XNOCASE..
+|   3136: 05 00 25 0f 17 54 48 52 45 41 44 53 41 46 45 3d   ..%..THREADSAFE=
+|   3152: 30 58 52 54 52 49 4d 1f 21 05 00 33 0f 19 4f 4d   0XRTRIM.!..3..OM
+|   3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f   IT LOAD EXTENSIO
+|   3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f   NXBINARY. ..3..O
+|   3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49   MIT LOAD EXTENSI
+|   3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17   ONXNOCASE....3..
+|   3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53   OMIT LOAD EXTENS
+|   3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 00 00   IONXRTRIM....3..
+|   4016: 54 41 42 4c 45 20 74 31 28 61 20 49 4e 54 45 47   TABLE t1(a INTEG
+|   4032: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 20 41   ER PRIMARY KEY A
+|   4048: 55 54 4f 49 4e 43 52 45 4d 45 4e 54 2c 0a 62 2c   UTOINCREMENT,.b,
+|   4064: 63 2c 64 2c 65 2c 66 2c 67 2c 68 2c 6a 2c 6b 2c   c,d,e,f,g,h,j,k,
+|   4080: 6c 2c 6d 2c 6e 2c 6f 2c 70 2c 71 2c 72 2c 73 29   l,m,n,o,p,q,r,s)
+| page 5 offset 16384
+|      0: 0d 00 00 00 02 0b a0 00 0c ad 0b a0 00 00 00 00   ................
+|   2976: 82 0a 02 08 08 09 08 08 17 84 06 30 20 32 35 33   ...........0 253
+|   2992: 00 01 30 04 25 06 1b 00 00 08 32 30 31 36 30 36   ..0.%.....201606
+|   3008: 30 39 03 25 07 00 00 01 34 03 25 05 00 00 01 35   09.%....4.%....5
+|   3024: 03 25 04 00 01 07 30 30 30 30 30 30 30 03 25 1a   .%....0000000.%.
+|   3040: 00 00 08 63 6f 6d 70 69 6c 65 72 03 25 02 00 00   ...compiler.%...
+|   3056: 06 64 62 73 74 61 74 03 25 0a 00 01 04 65 62 75   .dbstat.%....ebu
+|   3072: 67 03 25 08 00 00 06 65 6e 61 62 6c 65 09 25 09   g.%....enable.%.
+|   3088: 05 04 04 04 04 04 00 01 08 78 74 65 6e 73 69 6f   .........xtensio
+|   3104: 6e 03 25 1d 00 00 04 66 74 73 34 03 25 0d 00 03   n.%....fts4.%...
+|   3120: 01 35 03 25 0f 00 00 03 67 63 63 03 25 03 00 01   .5.%....gcc.%...
+|   3136: 06 65 6f 70 6f 6c 79 03 25 11 00 00 05 6a 73 6f   .eopoly.%....jso
+|   3152: 6e 31 03 25 13 00 00 04 6c 6f 61 64 03 25 1c 00   n1.%....load.%..
+|   3168: 00 03 6d 61 78 03 25 18 00 01 05 65 6d 6f 72 79   ..max.%....emory
+|   3184: 03 25 19 00 03 04 73 79 73 35 03 25 15 00 00 04   .%....sys5.%....
+|   3200: 6f 6d 69 74 03 25 1b 00 00 05 72 74 72 65 65 03   omit.%....rtree.
+|   3216: 25 01 00 d0 0a 07 68 72 65 61 64 73 61 66 65 03   %.....hreadsafe.
+|   3232: 25 1e 00 00 04 76 74 61 62 03 25 0b 00 86 50 01   %....vtab.%...P.
+|   3248: 08 08 08 08 08 17 8d 12 30 20 38 33 35 00 01 30   ........0 835..0
+|   3264: 12 01 06 00 01 06 00 01 06 00 1f 03 00 01 03 00   ................
+|   3280: 01 03 00 00 08 32 30 31 36 30 36 30 39 09 01 07   .....20160609...
+|   3296: 00 01 07 00 01 07 00 00 01 34 09 01 05 00 01 05   .........4......
+|   3312: 00 01 05 00 00 01 35 09 01 04 00 01 04 00 01 04   ......5.........
+|   3328: 00 01 07 30 30 30 30 30 30 30 09 1c 04 00 01 04   ...0000000......
+|   3344: 00 01 04 00 00 06 62 69 6e 61 72 79 3c 03 01 02   ......binary<...
+|   3360: 02 00 03 01 02 02 00 03 01 02 02 00 03 01 02 02   ................
+|   3376: 00 03 01 02 02 00 03 01 02 02 00 03 01 02 02 00   ................
+|   3392: 03 01 02 02 00 03 01 02 02 00 03 01 02 02 00 03   ................
+|   3408: 01 02 02 00 03 01 02 02 00 00 08 63 6f 6d 70 69   ...........compi
+|   3424: 6c 65 72 09 01 02 00 01 02 00 01 02 00 00 06 64   ler............d
+|   3440: 62 73 74 61 74 09 07 03 00 01 03 00 01 03 00 01   bstat...........
+|   3456: 04 65 62 75 67 09 04 02 00 01 02 00 01 02 00 00   .ebug...........
+|   3472: 06 65 6e 61 62 6c 65 3f 07 02 00 01 02 00 01 02   .enable?........
+|   3488: 00 01 02 00 01 02 00 01 02 00 01 02 00 01 02 00   ................
+|   3504: 01 02 00 01 02 00 01 02 00 01 02 00 01 02 00 01   ................
+|   3520: 02 00 01 02 00 01 02 00 01 02 00 01 02 00 01 02   ................
+|   3536: 00 01 02 00 01 02 00 01 08 78 74 65 6e 73 69 6f   .........xtensio
+|   3552: 6e 09 1f 04 00 01 04 00 01 04 00 00 04 66 74 73   n............fts
+|   3568: 34 09 0a 03 00 01 03 00 01 03 00 03 01 35 09 0d   4............5..
+|   3584: 03 00 01 03 00 01 03 00 00 03 67 63 63 09 01 03   ..........gcc...
+|   3600: 00 01 03 00 01 03 00 01 06 65 6f 70 6f 6c 79 09   .........eopoly.
+|   3616: 10 03 00 01 03 00 01 03 00 00 05 6a 73 6f 6e 31   ...........json1
+|   3632: 09 13 03 00 01 03 01 01 03 00 00 04 6c 6f 61 64   ............load
+|   3648: 09 1f 03 00 01 03 00 01 03 00 00 03 6d 61 78 09   ............max.
+|   3664: 1c 02 00 01 02 00 01 02 00 01 05 65 6d 6f 72 79   ...........emory
+|   3680: 09 1c 03 00 01 03 00 01 03 00 03 04 73 79 73 35   ............sys5
+|   3696: 09 16 03 00 01 03 00 01 03 00 00 06 6e 6f 63 61   ............noca
+|   3712: 73 65 3c 02 01 02 02 00 03 01 02 02 00 03 01 02   se<.............
+|   3744: 00 03 01 02 02 00 03 01 02 02 00 03 01 02 02 00   ................
+|   3760: 03 01 02 02 00 03 01 02 02 00 03 01 02 02 00 00   ................
+|   3776: 04 6f 6d 69 74 09 1f 02 00 01 02 00 01 02 00 00   .omit...........
+|   3792: 05 72 74 72 65 65 09 19 03 00 01 03 00 01 03 00   .rtree..........
+|   3808: 03 02 69 6d 3c 01 01 02 02 00 03 01 02 02 00 03   ..im<...........
+|   3824: 01 02 02 00 03 01 02 02 00 03 01 02 02 00 03 01   ................
+|   3840: 02 02 00 03 01 02 02 00 03 01 02 02 00 03 01 02   ................
+|   3856: 02 00 03 01 02 02 00 03 01 02 02 00 03 01 02 02   ................
+|   3872: 00 00 0a 74 68 72 65 61 64 73 61 66 65 09 22 02   ...threadsafe...
+|   3888: 00 01 02 00 01 02 00 00 04 76 74 61 62 09 07 04   .........vtab...
+|   3952: 01 01 02 00 01 01 01 02 00 01 01 01 02 00 01 01   ................
+|   3968: 01 02 00 01 01 01 02 00 01 01 01 02 00 01 01 01   ................
+|   3984: 02 00 01 01 01 02 00 01 01 01 02 00 01 01 01 02   ................
+|   4000: 00 01 01 01 02 00 01 01 01 02 00 01 01 01 02 00   ................
+|   4032: 01 01 02 00 01 01 01 02 00 01 01 01 02 00 01 01   ................
+|   4048: 01 02 00 01 01 01 02 00 01 01 01 02 00 01 01 01   ................
+|   4064: 02 00 01 01 01 02 00 01 01 01 02 00 01 01 01 02   ................
+|   4080: 00 01 01 01 02 00 01 01 01 02 00 01 01 01 02 00   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 02 0f f5 00 0f fb 0f f5 00 00 00 00   ................
+|   4080: 00 00 00 00 00 05 04 08 09 01 02 04 04 08 08 09   ................
+| end x.db
+}]} {}
+
+do_catchsql_test 1.1 {
+  SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'rtree NEAR rtree NEAR "json1 enable"';
+} {0 {}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_test 1.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+| size 24576 pagesize 4096 filename crash-10b0f1037e9c85.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 00 00 00 40 20 20 00 00 00 01 00 00 00 07   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01   ................
+|     96: 00 2e 82 40 0d 00 00 00 06 00 00 00 0f 8d 0f 21   ...@...........!
+|    112: 0e b9 0d c8 0e 7e 0d a4 00 00 00 00 00 00 00 00   .....~..........
+|   3488: 00 00 00 00 21 ff 06 17 10 10 01 30 74 61 62 6c   ....!......0tabl
+|   3504: 65 74 32 74 32 00 43 52 45 41 54 45 20 54 41 42   et2t2.CREATE TAB
+|   3520: 4c 45 20 74 32 28 70 29 81 33 00 07 17 1f 1f 01   LE t2(p).3......
+|   3536: 82 35 74 61 62 6c 65 74 31 5f 73 65 67 64 69 72   .5tablet1_segdir
+|   3552: 74 31 5f 73 65 67 64 69 72 05 43 52 45 41 54 45   t1_segdir.CREATE
+|   3568: 20 54 41 42 4c 45 20 27 74 31 5f 73 65 67 64 69    TABLE 't1_segdi
+|   3584: 72 27 28 6c 65 76 65 6c 20 09 4e 50 45 47 45 50   r'(level .NPEGEP
+|   3600: 2c 69 64 78 20 09 4e 50 45 47 45 50 2c 73 74 61   ,idx .NPEGEP,sta
+|   3616: 72 74 5f 62 6c 6f 63 6b 20 09 4e 50 45 47 45 50   rt_block .NPEGEP
+|   3632: 2c 6c 65 61 76 65 73 5f 65 6e 64 5f 62 6c 6f 63   ,leaves_end_bloc
+|   3648: 6b 20 09 4e 50 45 47 45 50 2c 65 6e 64 5f 62 6c   k .NPEGEP,end_bl
+|   3664: 6f 63 6b 20 09 4e 50 45 47 45 50 2c 72 6f 6f 74   ock .NPEGEP,root
+|   3680: 20 42 0c 4f 42 2c 50 52 49 4d 41 52 59 20 4b 45    B.OB,PRIMARY KE
+|   3696: 59 28 6c 65 76 65 6c 2c 20 69 64 78 29 29 31 00   Y(level, idx))1.
+|   3712: 06 17 45 1f 01 00 00 00 00 00 00 73 71 6c 69 74   ..E........sqlit
+|   3728: 65 5f 61 75 74 6f 69 6e 64 65 78 5f 74 31 5f 73   e_autoindex_t1_s
+|   3744: 65 67 64 69 72 5f 31 00 00 00 00 00 00 00 00 00   egdir_1.........
+|   3760: 06 00 00 00 00 00 00 00 00 66 00 07 17 23 23 01   .........f...##.
+|   3776: 81 13 74 61 62 6c 65 74 31 5f 73 65 67 6d 65 6e   ..tablet1_segmen
+|   3792: 74 73 74 31 5f 73 65 67 6d 65 6e 74 73 00 43 52   tst1_segments.CR
+|   3808: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 73   EATE TABLE 't1_s
+|   3824: 65 67 6d 65 6e 74 73 27 28 0c 6f 63 6b 09 64 0a   egments'(.ock.d.
+|   3840: 20 09 4e 50 45 47 45 50 20 50 50 09 04 31 50 09    .NPEGEP PP..1P.
+|   3856: 20 0b 45 09 0c 20 62 0c 6f 63 6b 20 42 0c 4f 42    .E.. b.ock B.OB
+|   3872: 29 6a 00 07 17 20 20 01 81 1f 74 61 62 6c 65 74   )j...  ...tablet
+|   3888: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74   1_contentt1_cont
+|   3904: 65 6e 74 00 43 52 45 41 54 45 20 54 41 42 4c 45   ent.CREATE TABLE
+|   3920: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 64 6f    't1_content'(do
+|   3936: 09 64 20 09 4e 50 45 47 45 50 20 50 50 09 0d 0c   .d .NPEGEP PP...
+|   3952: 50 09 20 0b 45 09 0c 20 27 03 03 01 27 0c 20 0a   P. .E.. '...'. .
+|   3968: 27 03 01 02 27 0c 20 27 03 02 03 27 29 38 00 06   '...'. '...')8..
+|   3984: 17 10 10 08 5f 74 61 62 6c 65 74 31 74 31 43 52   ...._tablet1t1CR
+|   4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42   EATE VIRTUAL TAB
+|   4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 33   LE t1 USING fts3
+| page 5 offset 16384
+|      0: 0d 00 00 00 02 00 00 00 00 00 0b a0 00 00 00 00   ................
+|   2976: 82 0a 02 08 00 00 00 00 17 84 06 00 00 00 00 00   ................
+|   2992: 00 01 00 04 00 00 00 00 00 08 00 00 00 00 00 00   ................
+|   3008: 00 00 03 00 00 00 00 01 00 03 00 00 00 00 01 00   ................
+|   3024: 03 00 00 00 00 07 00 00 00 00 00 00 00 03 00 00   ................
+|   3040: 00 00 08 00 00 00 00 00 00 00 00 03 00 00 00 00   ................
+|   3056: 06 00 00 00 00 00 00 03 00 00 00 00 04 00 00 00   ................
+|   3072: 00 03 00 00 00 00 06 65 6e 61 62 6c 65 09 25 09   .......enable.%.
+|   3088: 05 04 04 04 04 00 00 00 08 00 00 00 00 00 00 00   ................
+|   3104: 00 03 00 00 00 00 04 00 00 00 00 03 00 00 00 00   ................
+|   3120: 01 00 03 00 00 00 00 03 00 00 00 03 00 00 00 00   ................
+|   3136: 06 00 00 00 00 00 00 03 00 00 00 00 05 6a 73 6f   .............jso
+|   3152: 6e 31 03 25 13 00 00 04 00 00 00 00 03 00 00 00   n1.%............
+|   3168: 00 03 00 00 00 03 00 00 00 00 05 00 00 00 00 00   ................
+|   3184: 03 00 00 00 00 04 00 00 00 00 03 00 00 00 00 04   ................
+|   3200: 00 00 00 00 03 00 00 00 00 05 72 74 72 65 65 03   ..........rtree.
+|   3216: 25 01 00 0d 0a 07 08 01 ff ff ff ff ff 01 00 00   %...............
+| page 6 offset 20480
+|      0: 0a 00 00 00 02 00 00 00 0f fb 0f f5 00 00 00 00   ................
+|   4080: 00 00 00 00 00 05 04 09 00 01 02 04 00 00 00 00   ................
+| end crash-10b0f1037e9c85.db
+}]} {}
+
+do_catchsql_test 2.1 {
+  SELECT 0 FROM t1 WHERE t1 MATCH 'rtree NEAR rtree"json1 enable"';
+} {1 {database disk image is malformed}}
+
+finish_test
diff --git a/test/fts3cov.test b/test/fts3cov.test
index 5d83836576..d01791bbe5 100644
--- a/test/fts3cov.test
+++ b/test/fts3cov.test
@@ -25,7 +25,7 @@ set testprefix fts3cov
 # When it first needs to read a block from the %_segments table, the FTS3 
 # module compiles an SQL statement for that purpose. The statement is 
 # stored and reused each subsequent time a block is read. This test case 
-# tests the effects of an OOM error occuring while compiling the statement.
+# tests the effects of an OOM error occurring while compiling the statement.
 #
 # Similarly, when FTS3 first needs to scan through a set of segment leaves
 # to find a set of documents that matches a term, it allocates a string
@@ -277,7 +277,7 @@ do_test fts3cov-7.2 {
 # pending-terms table must be flushed each time a document with a docid
 # less than or equal to the previous docid is modified. 
 #
-# This test checks the effects of an OOM error occuring when the 
+# This test checks the effects of an OOM error occurring when the 
 # pending-terms table is flushed for this reason as part of a DELETE 
 # statement.
 #
diff --git a/test/fts3expr2.test b/test/fts3expr2.test
index c3d161730b..6fb133f17f 100644
--- a/test/fts3expr2.test
+++ b/test/fts3expr2.test
@@ -46,7 +46,7 @@ ifcapable !fts3 {
 #     * Whether or not superflous parenthesis are included. i.e. if
 #       "a OR b AND (c OR d)" or "a OR (b AND (c OR d))" is generated.
 #
-#     * Whether or not explict AND operators are used. i.e. if
+#     * Whether or not explicit AND operators are used. i.e. if
 #     "a OR b AND c" or "a OR b c" is generated.
 #
 
diff --git a/test/fts3fault.test b/test/fts3fault.test
index 21defd282f..20e5f25de5 100644
--- a/test/fts3fault.test
+++ b/test/fts3fault.test
@@ -216,6 +216,14 @@ do_faultsim_test 8.4 -prep {
 } -test {
   faultsim_test_result {0 3}
 }
+do_faultsim_test 8.5 -prep { 
+  faultsim_restore_and_reopen
+  db func mit mit
+} -body {
+  execsql { SELECT mit(matchinfo(t8, 'l')) FROM t8 WHERE t8 MATCH '"a b c"' }
+} -test {
+  faultsim_test_result {0 3}
+}
 
 do_test 9.0 {
   faultsim_delete_and_reopen
diff --git a/test/fts3fault3.test b/test/fts3fault3.test
new file mode 100644
index 0000000000..ae204718b4
--- /dev/null
+++ b/test/fts3fault3.test
@@ -0,0 +1,82 @@
+# 2023 October 23 
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set ::testprefix fts3fault
+
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
+ifcapable !fts3 { finish_test ; return }
+
+set ::TMPDBERROR [list 1 \
+  {unable to open a temporary database file for storing temporary tables}
+]
+
+
+# Test error handling in an "ALTER TABLE ... RENAME TO" statement on an
+# FTS3 table. Specifically, test renaming the table within a transaction
+# after it has been written to.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts3(a);
+  INSERT INTO t1 VALUES('test renaming the table');
+  INSERT INTO t1 VALUES(' after it has been written');
+  INSERT INTO t1 VALUES(' actually other stuff instead');
+}
+faultsim_save_and_close
+do_faultsim_test 1 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      DELETE FROM t1 WHERE rowid=2;
+  }
+} -body {
+  execsql {
+    DELETE FROM t1;
+  }
+} -test {
+  catchsql { COMMIT }
+  faultsim_integrity_check
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------
+reset_db
+
+do_execsql_test 2.0 {
+  BEGIN;
+  CREATE VIRTUAL TABLE t1 USING fts3(a);
+  WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
+  )
+  INSERT INTO t1 SELECT 'abc def ghi jkl mno pqr' FROM s;
+  COMMIT;
+}
+
+faultsim_save_and_close
+do_faultsim_test 2 -faults oom-t* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      CREATE TABLE x1(a PRIMARY KEY);
+  }
+} -body {
+  execsql {
+    PRAGMA integrity_check;
+  }
+} -test {
+  faultsim_test_result {0 ok} $::TMPDBERROR
+}
+
+
+finish_test
diff --git a/test/fts3fuzz001.test b/test/fts3fuzz001.test
index 41b22d33da..6b1ae90ee4 100644
--- a/test/fts3fuzz001.test
+++ b/test/fts3fuzz001.test
@@ -13,6 +13,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix fts3fuzz001
 
 ifcapable !deserialize||!fts3 {
   finish_test
@@ -110,5 +111,31 @@ do_test fts3fuzz001-121 {
   }
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 200 {
+  CREATE VIRTUAL TABLE x1 USING fts3(x);
+
+  INSERT INTO x1 VALUES('braes brag bragged bragger bragging');
+  INSERT INTO x1 VALUES('brags braid braided braiding braids');
+  INSERT INTO x1 VALUES('brain brainchild brained braining brains');
+  INSERT INTO x1 VALUES('brainstem brainstems brainstorm brainstorms');
+  INSERT INTO x1(x1) VALUES('nodesize=24');
+}
+
+do_execsql_test 210 {
+  PRAGMA integrity_check;
+} {ok}
+
+do_execsql_test 220 {
+  INSERT INTO x1(x1) VALUES('merge=10,2')
+}
+
+do_execsql_test 220 {
+  PRAGMA integrity_check;
+} {ok}
+
+
+
 
 finish_test
diff --git a/test/fts3integrity.test b/test/fts3integrity.test
new file mode 100644
index 0000000000..bcbc49dc33
--- /dev/null
+++ b/test/fts3integrity.test
@@ -0,0 +1,42 @@
+# 2023 December 16
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file runs all tests.
+#
+# $Id: fts3.test,v 1.2 2008/07/23 18:17:32 drh Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix fts3integrity
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+  
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts3(x);
+  INSERT INTO t1 VALUES('first row');
+  INSERT INTO t1 VALUES('second row');
+
+  CREATE TABLE t2(x PRIMARY KEY);
+  INSERT INTO t2 VALUES('first row');
+  INSERT INTO t2 VALUES('second row');
+}
+
+sqlite3 db2 test.db
+
+do_execsql_test -db db2 1.1 {
+  CREATE TABLE t3(x, y);
+}
+
+do_execsql_test 1.2 {
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test
diff --git a/test/fts3join.test b/test/fts3join.test
index cbd08b63f2..9171c817be 100644
--- a/test/fts3join.test
+++ b/test/fts3join.test
@@ -97,11 +97,8 @@ do_eqp_test 4.2 {
   WHERE t4.y = ?;
 } {
   QUERY PLAN
-  |--MATERIALIZE rr
-  |  `--SCAN ft4 VIRTUAL TABLE INDEX 3:
   |--SCAN t4
-  |--BLOOM FILTER ON rr (docid=?)
-  `--SEARCH rr USING AUTOMATIC COVERING INDEX (docid=?) LEFT-JOIN
+  `--SCAN ft4 VIRTUAL TABLE INDEX 3: LEFT-JOIN
 }
 
 finish_test
diff --git a/test/fts3snippet.test b/test/fts3snippet.test
index ae022b68a6..ad1fbb3bef 100644
--- a/test/fts3snippet.test
+++ b/test/fts3snippet.test
@@ -561,7 +561,6 @@ do_test 4.3 {
   }]
 } {64}
 
-
 #-------------------------------------------------------------------------
 # Request a snippet from a query with more than 64 phrases.
 #
@@ -588,5 +587,9 @@ do_execsql_test 5.1 {
   {[a70] [a71] [a72]}
 }
 
+do_execsql_test 5.2 {
+  SELECT snippet(t5, '[', ']', -1, 0) FROM t5 WHERE t5 MATCH 'a5'
+} {{a4 [a5] a6}}
+
 set sqlite_fts3_enable_parentheses 0
 finish_test
diff --git a/test/fts3snippet2.test b/test/fts3snippet2.test
index 607b01e0f8..f55a5f203a 100644
--- a/test/fts3snippet2.test
+++ b/test/fts3snippet2.test
@@ -55,5 +55,16 @@ do_execsql_test 2.2 {
   '(def AND (one NEAR abc)) OR one'
 } {<b>one</b>}
 
+#-------------------------------------------------------------------------
+
+do_execsql_test 3.0 {
+CREATE VIRTUAL TABLE f USING fts3(a,b);
+INSERT INTO f VALUES (101,x'056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f5010201010101014101017373737373737373737373737373737373737373737373737373737330737373737373737373737373737365056505650d051e05650565056528056505650d05650505656505650565650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565286505c705650565050565059494949494949494949494949494949494949494949494949494949494949494949494650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650505650565056505650565056505650565056500000000000000000000000000000000000000000000000000000000000000000565056505656505650565056505650d056500000000000000000000000000000000000000000000000000000000000000000100000000000000ed0000000000ffffffffffffffffffffff0007ffffff0001c5c50001c5c50001c5c50001c5c50001c5c50001c5c50001e5c50001c5c50001c5c50001c5c50001c5c50001c5c5000100000014720000000000000016dac5c50001c5c50001c5c50001c5c50001c5c50001c5c50d0505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800465056505650565056505651e650565056505650565056505650d05650505656505650565050565650584058005650565056505650565056505650565056522650565056505650d051e056505650561286505c70565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef650565058405056505650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565226505737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373737373733a73737373737373737373737373737373737373737373737373737373737373737373737373737373737c7373737365056505650d051e05650565056528650565056505650505650565056505650565056505650565056505e505650565056505656505650565056505650d05650505656505650565ef65056505');
+}
+
+do_execsql_test 3.1 {
+  SELECT  length(snippet(f))  FROM f WHERE b MATCH x'0565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056505650565056505650565056505650565050565056505640565056505650565056505651e05650565056522650565056505650d051e056505650565286505650565056505056505650565056505650565056505650565056505650565056505656505650565056505650d05650505656505650565ef65056505844c746e65650565056505650565056505650565056505650565058405800565056505650565056505651e650565056505650565056505650d056505056565056505650565056505840580056505650565056f05650565056505650565056505650565050565056505640565056505650565056505651e05650565056505650565056505650505656565056505650565056505651e0565056505650565056505650565052265056505650569056505650565056505650565056505650565056505650500406505650565056505650565056505000101e5c501014b010101c501c5c501010101f50102010101010141010141010001017bf15905000000000017';
+} {192}
+
 set sqlite_fts3_enable_parentheses 0
 finish_test
diff --git a/test/fts4check.test b/test/fts4check.test
index c94c35910b..7f1004d8b3 100644
--- a/test/fts4check.test
+++ b/test/fts4check.test
@@ -71,7 +71,10 @@ foreach {tn disruption} {
   do_catchsql_test 1.2.2.$tn {
     INSERT INTO t1 (t1) VALUES('integrity-check')
   } {1 {database disk image is malformed}}
-  do_execsql_test  1.2.3.$tn "ROLLBACK"
+  do_execsql_test 1.2.3.$tn {
+    PRAGMA integrity_check;
+  } {{malformed inverted index for FTS4 table main.t1}}
+  do_execsql_test  1.2.4.$tn "ROLLBACK"
 }
 
 do_test 1.3 { fts_integrity db t1 } {ok}
@@ -106,7 +109,10 @@ foreach {tn disruption} {
   do_catchsql_test 2.2.2.$tn {
     INSERT INTO t2 (t2) VALUES('integrity-check')
   } {1 {database disk image is malformed}}
-  do_execsql_test  2.2.3.$tn "ROLLBACK"
+  do_test 2.2.3.$tn {
+    db eval {PRAGMA integrity_check(t2);}
+  } {{malformed inverted index for FTS4 table main.t2}}
+  do_execsql_test  2.2.4.$tn "ROLLBACK"
 }
 
 
diff --git a/test/fts4intck1.test b/test/fts4intck1.test
new file mode 100644
index 0000000000..6596b2f997
--- /dev/null
+++ b/test/fts4intck1.test
@@ -0,0 +1,75 @@
+# 2023-10-23
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Test PRAGMA integrity_check against and FTS3/FTS4 table.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+ifcapable !fts3 { finish_test ; return }
+
+set ::testprefix fts4intck1
+
+proc slang {in} {
+  return [string map {th d e eh} $in]
+}
+
+db function slang -deterministic -innocuous slang
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b)));
+  INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog');
+  SELECT c FROM t1;
+} {{deh quick fox jumps ovehr deh lazy brown dog}}
+
+do_execsql_test 1.1 {
+  CREATE VIRTUAL TABLE t2 USING fts4(content="t1", c);
+  INSERT INTO t2(t2) VALUES('rebuild');
+  SELECT docid FROM t2 WHERE t2 MATCH 'deh';
+} {1}
+
+do_execsql_test 1.2 {
+  PRAGMA integrity_check(t2);
+} {ok}
+
+db close
+sqlite3 db test.db
+do_execsql_test 2.1 {
+  PRAGMA integrity_check(t2);
+} {{unable to validate the inverted index for FTS4 table main.t2: SQL logic error}}
+
+db function slang -deterministic -innocuous slang
+do_execsql_test 2.2 {
+  PRAGMA integrity_check(t2);
+} {ok}
+
+proc slang {in} {return $in}
+do_execsql_test 2.3 {
+  PRAGMA integrity_check(t2);
+} {{malformed inverted index for FTS4 table main.t2}}
+
+#-------------------------------------------------------------------------
+# Test that integrity-check works on a read-only database.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x1 USING fts4(a, b);
+  INSERT INTO x1 VALUES('one', 'two');
+  INSERT INTO x1 VALUES('three', 'four');
+}
+db close
+sqlite3 db test.db -readonly 1
+
+do_execsql_test 3.1 {
+  PRAGMA integrity_check;
+} {ok}
+
+
+
+finish_test
diff --git a/test/fts4langid.test b/test/fts4langid.test
index 45e851f940..84424ff3c7 100644
--- a/test/fts4langid.test
+++ b/test/fts4langid.test
@@ -435,7 +435,7 @@ do_test 5.3.2 {
   for {set i 0} {$i < 20} {incr i} {
     execsql {
       INSERT INTO t6(content, lid) VALUES(
-        'I (row '||$i||') belong to langauge N!', $lid
+        'I (row '||$i||') belong to language N!', $lid
       );
     }
   }
@@ -499,7 +499,8 @@ do_execsql_test 6.0 {
 }
 do_execsql_test 6.1 {
   INSERT INTO vt0(vt0) VALUES('integrity-check');
-}
+  PRAGMA integrity_check;
+} {ok}
 do_execsql_test 6.2 {
   COMMIT;
   INSERT INTO vt0(vt0) VALUES('integrity-check');
diff --git a/test/fts4merge.test b/test/fts4merge.test
index 3cd693209d..ffef0e9334 100644
--- a/test/fts4merge.test
+++ b/test/fts4merge.test
@@ -37,7 +37,7 @@ foreach mod {fts3 fts4} {
   do_test 1.0 { fts3_build_db_1 -module $mod 1004 } {}
   do_test 1.1 { fts3_integrity_check t1 } {ok}
   do_execsql_test 1.1 { 
-    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
+    SELECT level, string_agg(idx, ' ') FROM t1_segdir GROUP BY level 
   } {
     0 {0 1 2 3 4 5 6 7 8 9 10 11} 
     1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13}
@@ -67,7 +67,7 @@ foreach mod {fts3 fts4} {
   }
   
   do_execsql_test 1.5 { 
-    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
+    SELECT level, string_agg(idx, ' ') FROM t1_segdir GROUP BY level 
   } {
     3 0
   }
@@ -103,7 +103,7 @@ foreach mod {fts3 fts4} {
   do_test 3.1 { fts3_integrity_check t2 } {ok}
   
   do_execsql_test 3.2 { 
-    SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
+    SELECT level, string_agg(idx, ' ') FROM t2_segdir GROUP BY level 
   } {
     0 {0 1 2 3 4 5 6} 
     1 {0 1 2 3 4} 
@@ -132,7 +132,7 @@ foreach mod {fts3 fts4} {
     foreach x {a c b d e f g h i j k l m n o p} {
       execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')"
     }
-    execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level}
+    execsql {SELECT level, string_agg(idx, ' ') FROM t4_segdir GROUP BY level}
   } {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}}
   
   foreach {tn expect} {
@@ -160,7 +160,7 @@ foreach mod {fts3 fts4} {
   do_execsql_test 4.4.2 {
     DELETE FROM t4_stat WHERE rowid=1;
     INSERT INTO t4(t4) VALUES('merge=1,12');
-    SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
+    SELECT level, string_agg(idx, ' ') FROM t4_segdir GROUP BY level;
   } "0 {0 1 2 3 4 5}                     1 0"
   
   
@@ -194,7 +194,7 @@ foreach mod {fts3 fts4} {
   do_execsql_test 5.3 {
     INSERT INTO t1(t1) VALUES('merge=1,5');
     INSERT INTO t1(t1) VALUES('merge=1,5');
-    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
+    SELECT level, string_agg(idx, ' ') FROM t1_segdir GROUP BY level;
   } {
     1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 
     2 {0 1 2 3}
@@ -249,7 +249,7 @@ foreach mod {fts3 fts4} {
   
   do_execsql_test 5.11 {
     INSERT INTO t1(t1) VALUES('merge=1,6');
-    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
+    SELECT level, string_agg(idx, ' ') FROM t1_segdir GROUP BY level;
     SELECT quote(value) from t1_stat WHERE rowid=1;
   } {
     1 {0 1} 2 0 3 0 X'010E'
diff --git a/test/func.test b/test/func.test
index df9d4dacfa..4e5f617e74 100644
--- a/test/func.test
+++ b/test/func.test
@@ -42,6 +42,10 @@ do_test func-0.1 {
 do_test func-1.0 {
   execsql {SELECT length(t1) FROM tbl1 ORDER BY t1}
 } {4 2 7 8 4}
+set isutf16 [regexp 16 [db one {PRAGMA encoding}]]
+do_execsql_test func-1.0b {
+  SELECT octet_length(t1) FROM tbl1 ORDER BY t1;
+} [expr {$isutf16?"8 4 14 16 8":"4 2 7 8 4"}]
 do_test func-1.1 {
   set r [catch {execsql {SELECT length(*) FROM tbl1 ORDER BY t1}} msg]
   lappend r $msg
@@ -57,9 +61,29 @@ do_test func-1.3 {
 do_test func-1.4 {
   execsql {SELECT coalesce(length(a),-1) FROM t2}
 } {1 -1 3 -1 5}
+do_execsql_test func-1.5 {
+  SELECT octet_length(12345);
+} [expr {(1+($isutf16!=0))*5}]
+db null NULL
+do_execsql_test func-1.6 {
+  SELECT octet_length(NULL);
+} {NULL}
+do_execsql_test func-1.7 {
+  SELECT octet_length(7.5);
+} [expr {(1+($isutf16!=0))*3}]
+do_execsql_test func-1.8 {
+  SELECT octet_length(x'30313233');
+} {4}
+do_execsql_test func-1.9 {
+  WITH c(x) AS (VALUES(char(350,351,352,353,354)))
+  SELECT length(x), octet_length(x) FROM c;
+} {5 10}
+
+
 
 # Check out the substr() function
 #
+db null {}
 do_test func-2.0 {
   execsql {SELECT substr(t1,1,2) FROM tbl1 ORDER BY t1}
 } {fr is pr so th}
@@ -93,6 +117,15 @@ do_test func-2.9 {
 do_test func-2.10 {
   execsql {SELECT substr(a,2,2) FROM t2}
 } {{} {} 45 {} 78}
+do_test func-2.11 {
+  execsql {SELECT substr('abcdefg',0x100000001,2)}
+} {{}}
+do_test func-2.12 {
+  execsql {SELECT substr('abcdefg',1,0x100000002)}
+} {abcdefg}
+do_test func-2.13 {
+  execsql {SELECT quote(substr(x'313233343536373839',0x7ffffffffffffffe,5))}
+} {X''}
 
 # Only do the following tests if TCL has UTF-8 capabilities
 #
@@ -237,9 +270,6 @@ ifcapable floatingpoint {
     catchsql {SELECT round(b,2.0) FROM t1 ORDER BY b}
   } {0 {-2.0 1.23 2.0}}
   # Verify some values reported on the mailing list.
-  # Some of these fail on MSVC builds with 64-bit
-  # long doubles, but not on GCC builds with 80-bit
-  # long doubles.
   for {set i 1} {$i<999} {incr i} {
     set x1 [expr 40222.5 + $i]
     set x2 [expr 40223.0 + $i]
@@ -318,6 +348,9 @@ ifcapable floatingpoint {
   do_test func-4.39 {
     string tolower [db eval {SELECT round(1e500), round(-1e500);}]
   } {inf -inf}
+  do_execsql_test func-4.40 {
+    SELECT round(123.456 , 4294967297);
+  } {123.456}
 }
 
 # Test the upper() and lower() functions
@@ -762,6 +795,11 @@ do_test func-16.1 {
   }
 } {X'616263' NULL}
 
+# Test the quote function for +Inf and -Inf
+do_execsql_test func-16.2 {
+  SELECT quote(4.2e+859), quote(-7.8e+904);
+} {9.0e+999 -9.0e+999}
+
 # Correctly handle function error messages that include %.  Ticket #1354
 #
 do_test func-17.1 {
@@ -839,30 +877,13 @@ do_test func-18.11 {
   }
 } integer
 ifcapable floatingpoint {
-  do_test func-18.12 {
-    catchsql {
-      INSERT INTO t6 VALUES(1<<62);
-      SELECT sum(x) - ((1<<62)*2.0+1) from t6;
-    }
-  } {1 {integer overflow}}
-  do_test func-18.13 {
-    execsql {
-      SELECT total(x) - ((1<<62)*2.0+1) FROM t6
-    }
-  } 0.0
-}
-ifcapable !floatingpoint {
-  do_test func-18.12 {
-    catchsql {
-      INSERT INTO t6 VALUES(1<<62);
-      SELECT sum(x) - ((1<<62)*2+1) from t6;
-    }
+  do_catchsql_test func-18.12 {
+    INSERT INTO t6 VALUES(1<<62);
+    SELECT sum(x) - ((1<<62)*2.0+1) from t6;
   } {1 {integer overflow}}
-  do_test func-18.13 {
-    execsql {
-      SELECT total(x) - ((1<<62)*2+1) FROM t6
-    }
-  } 0.0
+  do_catchsql_test func-18.13 {
+    SELECT total(x) - ((1<<62)*2.0+1) FROM t6
+  } {0 0.0}
 }
 if {[working_64bit_int]} {
   do_test func-18.14 {
@@ -1035,6 +1056,9 @@ do_test func-21.8 {
     SELECT replace('aaaaaaa', 'a', '0123456789');
   }
 } {0123456789012345678901234567890123456789012345678901234567890123456789}
+do_execsql_test func-21.9 {
+  SELECT typeof(replace(1,'',0));
+} {text}
 
 ifcapable tclvar {
   do_test func-21.9 {
@@ -1129,18 +1153,18 @@ ifcapable deprecated {
   } {3}
 }
 
-# The group_concat() function.
+# The group_concat() and string_agg() functions.
 #
 do_test func-24.1 {
   execsql {
-    SELECT group_concat(t1) FROM tbl1
+    SELECT group_concat(t1), string_agg(t1,',') FROM tbl1
   }
-} {this,program,is,free,software}
+} {this,program,is,free,software this,program,is,free,software}
 do_test func-24.2 {
   execsql {
-    SELECT group_concat(t1,' ') FROM tbl1
+    SELECT group_concat(t1,' '), string_agg(t1,' ') FROM tbl1
   }
-} {{this program is free software}}
+} {{this program is free software} {this program is free software}}
 do_test func-24.3 {
   execsql {
     SELECT group_concat(t1,' ' || rowid || ' ') FROM tbl1
@@ -1153,9 +1177,9 @@ do_test func-24.4 {
 } {{}}
 do_test func-24.5 {
   execsql {
-    SELECT group_concat(t1,NULL) FROM tbl1
+    SELECT group_concat(t1,NULL), string_agg(t1,NULL) FROM tbl1
   }
-} {thisprogramisfreesoftware}
+} {thisprogramisfreesoftware thisprogramisfreesoftware}
 do_test func-24.6 {
   execsql {
     SELECT 'BEGIN-'||group_concat(t1) FROM tbl1
@@ -1171,7 +1195,9 @@ set midargs {}
 unset -nocomplain midres
 set midres {}
 unset -nocomplain result
-for {set i 1} {$i<[sqlite3_limit db SQLITE_LIMIT_FUNCTION_ARG -1]} {incr i} {
+set limit [sqlite3_limit db SQLITE_LIMIT_FUNCTION_ARG -1]
+if {$limit>400} {set limit 400}
+for {set i 1} {$i<$limit} {incr i} {
   append midargs ,'/$i'
   append midres /$i
   set result [md5 \
@@ -1215,7 +1241,7 @@ do_test func-24.12 {
                           WHEN 'program' THEN null ELSE t1 END) FROM tbl1
   }
 } {,is,free,software}
-# Tests to verify ticket http://www.sqlite.org/src/tktview/55746f9e65f8587c0
+# Tests to verify ticket http://sqlite.org/src/tktview/55746f9e65f8587c0
 do_test func-24.13 {
   execsql {
     SELECT typeof(group_concat(x)) FROM (SELECT '' AS x);
@@ -1249,7 +1275,8 @@ do_test func-26.1 {
 #
 do_test func-26.2 {
   set a {}
-  for {set i 1} {$i<=$::SQLITE_MAX_FUNCTION_ARG} {incr i} {
+  set limit $::SQLITE_MAX_FUNCTION_ARG
+  for {set i 1} {$i<=$limit} {incr i} {
     lappend a $i
   }
   db eval "
@@ -1267,7 +1294,8 @@ do_test func-26.3 {
 } {1 {too many arguments on function nullx_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789}}
 do_test func-26.4 {
   set a {}
-  for {set i 1} {$i<=$::SQLITE_MAX_FUNCTION_ARG-1} {incr i} {
+  set limit [expr {$::SQLITE_MAX_FUNCTION_ARG-1}]
+  for {set i 1} {$i<=$limit} {incr i} {
     lappend a $i
   }
   catchsql "
@@ -1520,6 +1548,51 @@ do_execsql_test func-36.110 {
   SELECT 123 ->> 456
 } {123->>456}
 
+# 2023-06-26
+# Enhanced precision of SUM().
+#
+reset_db
+do_catchsql_test func-37.100 {
+  WITH c(x) AS (VALUES(9223372036854775807),(9223372036854775807),
+                      (123),(-9223372036854775807),(-9223372036854775807))
+  SELECT sum(x) FROM c;
+} {1 {integer overflow}}
+do_catchsql_test func-37.110 {
+  WITH c(x) AS (VALUES(9223372036854775807),(1))
+  SELECT sum(x) FROM c;
+} {1 {integer overflow}}
+do_catchsql_test func-37.120 {
+  WITH c(x) AS (VALUES(9223372036854775807),(10000),(-10010))
+  SELECT sum(x) FROM c;
+} {1 {integer overflow}}
 
+# 2023-08-28 forum post https://sqlite.org/forum/forumpost/1c06ddcacc86032a
+# Incorrect handling of infinity by SUM().
+#
+do_execsql_test func-38.100 {
+  WITH t1(x) AS (VALUES(9e+999)) SELECT sum(x), avg(x), total(x) FROM t1;
+  WITH t1(x) AS (VALUES(-9e+999)) SELECT sum(x), avg(x), total(x) FROM t1;
+} {Inf Inf Inf -Inf -Inf -Inf}
+
+# 2024-03-21 https://sqlite.org/forum/forumpost/23b8688ef4
+# Another problem with Kahan-Babushka-Neumaier summation and
+# infinities.
+#
+do_execsql_test func-39.101 {
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1)
+  SELECT sum(1.7976931348623157e308),
+         avg(1.7976931348623157e308),
+         total(1.7976931348623157e308)
+    FROM c;
+} {1.79769313486232e+308 1.79769313486232e+308 1.79769313486232e+308}
+for {set i 2} {$i<10} {incr i} {
+  do_execsql_test func-39.[expr {10*$i+100}] {
+    WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<$i)
+    SELECT sum(1.7976931348623157e308),
+           avg(1.7976931348623157e308),
+           total(1.7976931348623157e308)
+      FROM c;
+  } {Inf Inf Inf}
+}
 
 finish_test
diff --git a/test/func2.test b/test/func2.test
index 08ad857509..a7c7ec3fd8 100644
--- a/test/func2.test
+++ b/test/func2.test
@@ -508,4 +508,27 @@ do_test func2-3.9.2 {
   bin_to_hex [lindex $blob 0]
 } "12"
 
+#-------------------------------------------------------------------------
+# At one point this was extremely slow to compile.
+#
+do_test func2-3.10 {
+  set tm [time {
+    execsql {
+      SELECT '' IN (zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(
+      zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(zerobloB(1)
+      )))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
+  }
+  }]
+
+  set tm [lindex $tm 0]
+  expr $tm<2000000
+} {1}
+
 finish_test
diff --git a/test/func3.test b/test/func3.test
index 0221a0dfd6..518bd51c79 100644
--- a/test/func3.test
+++ b/test/func3.test
@@ -154,7 +154,7 @@ do_test func3-5.39 {
 } [db eval {EXPLAIN SELECT min(1.0+'2.0',4*11)}]
 
 # Unlikely() does not preserve the affinity of X.
-# ticket https://www.sqlite.org/src/tktview/0c620df60b
+# ticket https://sqlite.org/src/tktview/0c620df60b
 #
 do_execsql_test func3-5.40 {
   SELECT likely(CAST(1 AS INT))=='1';
diff --git a/test/func4.test b/test/func4.test
index 924c1fa538..fb74b7d8d5 100644
--- a/test/func4.test
+++ b/test/func4.test
@@ -1,4 +1,4 @@
-# 2013 March 10
+# 2023-03-10
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
@@ -9,7 +9,10 @@
 #
 #***********************************************************************
 # This file implements regression tests for SQLite library. The focus of
-# this file is testing the tointeger() and toreal() functions.
+# this file is testing the tointeger() and toreal() functions that are
+# part of the "totype.c" extension.  This file does not test the core
+# SQLite library.  Failures of tests in this file are related to the
+# ext/misc/totype.c extension.
 #
 # Several of the toreal() tests are disabled on platforms where floating
 # point precision is not high enough to represent their constant integer
@@ -23,6 +26,20 @@ load_static_extension db totype
 
 set highPrecision(1) [expr \
     {[db eval {SELECT tointeger(9223372036854775807 + 1);}] eq {{}}}]
+set highPrecision(2) [expr \
+    {[db eval {SELECT toreal(-9223372036854775808 + 1);}] eq {{}}}]
+
+# highPrecision(3) is only known to be false on i586 with gcc-13 and -O2.
+# It is true on the exact same platform with -O0.  Both results seem
+# reasonable, so we'll just very the expectation accordingly.
+#
+set highPrecision(3) [expr \
+    {[db eval {SELECT toreal(9007199254740992 + 1);}] eq {{}}}]
+
+if {!$highPrecision(1) || !$highPrecision(2) || !$highPrecision(3)} {
+  puts "NOTICE:\
+        highPrecision: $highPrecision(1) $highPrecision(2) $highPrecision(3)"
+}
 
 do_execsql_test func4-1.1 {
   SELECT tointeger(NULL);
@@ -92,7 +109,7 @@ do_execsql_test func4-1.22 {
 } {{}}
 do_execsql_test func4-1.23 {
   SELECT tointeger(-9223372036854775808 - 1);
-} {-9223372036854775808}
+} {{}}
 do_execsql_test func4-1.24 {
   SELECT tointeger(-9223372036854775808);
 } {-9223372036854775808}
@@ -195,8 +212,6 @@ do_execsql_test func4-1.55 {
 } {{}}
 
 ifcapable floatingpoint {
-  set highPrecision(2) [expr \
-      {[db eval {SELECT toreal(-9223372036854775808 + 1);}] eq {{}}}]
 
   do_execsql_test func4-2.1 {
     SELECT toreal(NULL);
@@ -269,7 +284,7 @@ ifcapable floatingpoint {
   } {-9.223372036854776e+18}
   do_execsql_test func4-2.24 {
     SELECT toreal(-9223372036854775808);
-  } {-9.223372036854776e+18}
+  } {{}}
   if {$highPrecision(2)} {
     do_execsql_test func4-2.25 {
       SELECT toreal(-9223372036854775808 + 1);
@@ -277,7 +292,7 @@ ifcapable floatingpoint {
   }
   do_execsql_test func4-2.26 {
     SELECT toreal(-9223372036854775807 - 1);
-  } {-9.223372036854776e+18}
+  } {{}}
   if {$highPrecision(2)} {
     do_execsql_test func4-2.27 {
       SELECT toreal(-9223372036854775807);
@@ -341,10 +356,14 @@ ifcapable floatingpoint {
   do_execsql_test func4-2.45 {
     SELECT toreal(9007199254740992);
   } {9007199254740992.0}
-  if {$highPrecision(2)} {
+  if {$highPrecision(3)} {
     do_execsql_test func4-2.46 {
       SELECT toreal(9007199254740992 + 1);
     } {{}}
+  } else {
+    do_execsql_test func4-2.46 {
+      SELECT toreal(9007199254740992 + 1);
+    } {9007199254740992.0}
   }
   do_execsql_test func4-2.47 {
     SELECT toreal(9007199254740992 + 2);
@@ -461,7 +480,7 @@ ifcapable check {
     catchsql {
       INSERT INTO t1 (x) VALUES ('-9223372036854775809');
     }
-  } {0 {}}
+  } {1 {CHECK constraint failed: tointeger(x) IS NOT NULL}}
   if {$highPrecision(1)} {
     do_test func4-3.19 {
       catchsql {
@@ -573,10 +592,10 @@ ifcapable floatingpoint {
   } {1}
   do_execsql_test func4-5.6 {
     SELECT tointeger(toreal(-9223372036854775808 - 1));
-  } {-9223372036854775808}
+  } {{}}
   do_execsql_test func4-5.7 {
     SELECT tointeger(toreal(-9223372036854775808));
-  } {-9223372036854775808}
+  } {{}}
   if {$highPrecision(2)} {
     do_execsql_test func4-5.8 {
       SELECT tointeger(toreal(-9223372036854775808 + 1));
@@ -626,10 +645,14 @@ ifcapable floatingpoint {
   do_execsql_test func4-5.22 {
     SELECT tointeger(toreal(9007199254740992));
   } {9007199254740992}
-  if {$highPrecision(2)} {
+  if {$highPrecision(3)} {
     do_execsql_test func4-5.23 {
       SELECT tointeger(toreal(9007199254740992 + 1));
     } {{}}
+  } else {
+    do_execsql_test func4-5.23 {
+      SELECT tointeger(toreal(9007199254740992 + 1));
+    } {9007199254740992}
   }
   do_execsql_test func4-5.24 {
     SELECT tointeger(toreal(9007199254740992 + 2));
diff --git a/test/func6.test b/test/func6.test
index fe90a755d7..acca490f33 100644
--- a/test/func6.test
+++ b/test/func6.test
@@ -43,7 +43,7 @@ do_execsql_test func6-100 {
 # string.
 proc loadhex {file} {
   set fd [open $file]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   set data [read $fd]
   close $fd
   binary encode hex $data 
diff --git a/test/func7.test b/test/func7.test
index bb4f80b325..6026b557f5 100644
--- a/test/func7.test
+++ b/test/func7.test
@@ -100,13 +100,13 @@ do_execsql_test func7-pg-300 {
    SELECT acos(1);
 } {0.0}
 do_execsql_test func7-pg-301 {
-   SELECT degrees(acos(0.5));
+   SELECT format('%f',degrees(acos(0.5)));
 } {60.0}
 do_execsql_test func7-pg-310 {
    SELECT round( asin(1), 7);
 } {1.5707963}
 do_execsql_test func7-pg-311 {
-   SELECT degrees( asin(0.5) );
+   SELECT format('%f',degrees( asin(0.5) ));
 } {30.0}
 do_execsql_test func7-pg-320 {
    SELECT round( atan(1), 7);
@@ -139,7 +139,7 @@ do_execsql_test func7-pg-420 {
    SELECT round( tan(1), 7);
 } {1.5574077}
 do_execsql_test func7-pg-421 {
-   SELECT tan( radians(45) );
+   SELECT round(tan( radians(45) ),10);
 } {1.0}
 do_execsql_test func7-pg-500 {
    SELECT round( sinh(1), 7);
diff --git a/test/func9.test b/test/func9.test
new file mode 100644
index 0000000000..d24d5f7beb
--- /dev/null
+++ b/test/func9.test
@@ -0,0 +1,66 @@
+# 2023-08-29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# Test cases for some newer SQL functions
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test func9-100 {
+  SELECT concat('abc',123,null,'xyz');
+} {abc123xyz}
+do_execsql_test func9-110 {
+  SELECT typeof(concat(null));
+} {text}
+do_catchsql_test func9-120 {
+  SELECT concat();
+} {1 {wrong number of arguments to function concat()}}
+do_execsql_test func9-130 {
+  SELECT concat_ws(',',1,2,3,4,5,6,7,8,NULL,9,10,11,12);
+} {1,2,3,4,5,6,7,8,9,10,11,12}
+do_execsql_test func9-131 {
+  SELECT concat_ws(',',1,2,3,4,'',6,7,8,NULL,9,10,11,12);
+} {1,2,3,4,,6,7,8,9,10,11,12}
+do_execsql_test func9-132 {
+  SELECT concat_ws(',','',2,3,4,'',6,7,8,NULL,9,10,11,12);
+} {,2,3,4,,6,7,8,9,10,11,12}
+do_execsql_test func9-133 {
+  SELECT concat_ws(',',NULL,'',3,4,'',6,7,8,NULL,9,10,11,12);
+} {,3,4,,6,7,8,9,10,11,12}
+do_execsql_test func9-134 {
+  SELECT concat_ws(',',NULL,NULL,NULL,'',3,4,'',6,7,8,NULL,9,10,11,12);
+} {,3,4,,6,7,8,9,10,11,12}
+do_execsql_test func9-140 {
+  SELECT concat_ws(NULL,1,2,3,4,5,6,7,8,NULL,9,10,11,12);
+} {{}}
+do_catchsql_test func9-150 {
+  SELECT concat_ws();
+} {1 {wrong number of arguments to function concat_ws()}}
+do_catchsql_test func9-160 {
+  SELECT concat_ws(',');
+} {1 {wrong number of arguments to function concat_ws()}}
+
+# https://sqlite.org/forum/forumpost/4c344ca61f (2025-03-02)
+do_execsql_test func9-200 {
+  SELECT unistr('G\u00e4ste');
+} {Gäste}
+do_execsql_test func9-210 {
+  SELECT unistr_quote(unistr('G\u00e4ste'));
+} {'Gäste'}
+do_execsql_test func9-220 {
+  SELECT format('%#Q',unistr('G\u00e4ste'));
+} {'Gäste'}
+
+do_execsql_test func9-300 {
+  SELECT hex( unistr('\UFFFFFFFF') )
+} {F7BFBFBF}
+
+finish_test
diff --git a/test/fuzz.test b/test/fuzz.test
index 83dc79bc78..7002054be9 100644
--- a/test/fuzz.test
+++ b/test/fuzz.test
@@ -130,7 +130,7 @@ do_test fuzz-1.12.1 {
   # The following query was crashing. The later subquery (in the FROM)
   # clause was flattened into the parent, but the code was not repairng
   # the "b" reference in the other sub-query. When the query was executed,
-  # that "b" refered to a non-existant vdbe table-cursor.
+  # that "b" referred to a non-existant vdbe table-cursor.
   #
   execsql {
     SELECT 1 IN ( SELECT b UNION SELECT 1 ) FROM (SELECT b FROM abc);
diff --git a/test/fuzz3.test b/test/fuzz3.test
index bf778ed2bc..8ac1f4d58f 100644
--- a/test/fuzz3.test
+++ b/test/fuzz3.test
@@ -83,7 +83,7 @@ proc modify_database {iMod} {
   set offset [expr {$iMod>>8}]
 
   set fd [open test.db r+]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   seek $fd $offset
   set old_blob [read $fd 1]
   seek $fd $offset
diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c
index 59bddc96a8..a3377770a8 100644
--- a/test/fuzzcheck.c
+++ b/test/fuzzcheck.c
@@ -88,6 +88,11 @@
 #include "sqlite3recover.h"
 #define ISSPACE(X) isspace((unsigned char)(X))
 #define ISDIGIT(X) isdigit((unsigned char)(X))
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+# define FLEXARRAY
+#else
+# define FLEXARRAY 1
+#endif
 
 
 #ifdef __unix__
@@ -129,9 +134,12 @@ struct Blob {
   int id;                 /* Id of this Blob */
   int seq;                /* Sequence number */
   int sz;                 /* Size of this Blob in bytes */
-  unsigned char a[1];     /* Blob content.  Extra space allocated as needed. */
+  unsigned char a[FLEXARRAY]; /* Blob content. Allocated as needed. */
 };
 
+/* Size in bytes of a Blob object sufficient to store N byte of content */
+#define SZ_BLOB(N) (offsetof(Blob,a) + (((N)+7)&~7))
+
 /*
 ** Maximum number of files in the in-memory virtual filesystem.
 */
@@ -159,10 +167,10 @@ static struct GlobalVars {
 } g;
 
 /*
-** Include the external vt02.c module.
+** Include various extensions.
 */
-extern int sqlite3_vt02_init(sqlite3*,char***,void*);
-
+extern int sqlite3_vt02_init(sqlite3*,char**,const sqlite3_api_routines*);
+extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*);
 
 /*
 ** Print an error message and quit.
@@ -381,6 +389,21 @@ static void renderDbSqlForCLI(
   if( nSql>0 && zSql[nSql-1]!='\n' ) fprintf(out, "\n");
 }
 
+/*
+** Find the tail (the last component) of a pathname.
+*/
+static const char *pathTail(const char *zPath){
+  const char *zTail = zPath;
+  while( zPath[0] ){
+    if( zPath[0]=='/' && zPath[1]!=0 ) zTail = &zPath[1];
+#ifndef __unix__
+    if( zPath[0]=='\\' && zPath[1]!=0 ) zTail = &zPath[1];
+#endif
+    zPath++;
+  }
+  return zTail;
+}
+
 /*
 ** Read the complete content of a file into memory.  Add a 0x00 terminator
 ** and return a pointer to the result.
@@ -505,30 +528,37 @@ static void writefileFunc(
 static void blobListLoadFromDb(
   sqlite3 *db,             /* Read from this database */
   const char *zSql,        /* Query used to extract the blobs */
-  int onlyId,              /* Only load where id is this value */
+  int firstId,             /* First sqlid to load */
+  int lastId,              /* Last sqlid to load */
   int *pN,                 /* OUT: Write number of blobs loaded here */
   Blob **ppList            /* OUT: Write the head of the blob list here */
 ){
-  Blob head;
+  Blob *head;
   Blob *p;
   sqlite3_stmt *pStmt;
   int n = 0;
   int rc;
   char *z2;
-
-  if( onlyId>0 ){
-    z2 = sqlite3_mprintf("%s WHERE rowid=%d", zSql, onlyId);
+  union {
+    Blob sBlob;
+    unsigned char tmp[SZ_BLOB(8)];
+  } uBlob;
+
+  head = &uBlob.sBlob;
+  if( firstId>0 ){
+    z2 = sqlite3_mprintf("%s WHERE rowid BETWEEN %d AND %d", zSql,
+                         firstId, lastId);
   }else{
     z2 = sqlite3_mprintf("%s", zSql);
   }
   rc = sqlite3_prepare_v2(db, z2, -1, &pStmt, 0);
   sqlite3_free(z2);
   if( rc ) fatalError("%s", sqlite3_errmsg(db));
-  head.pNext = 0;
-  p = &head;
+  head->pNext = 0;
+  p = head;
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     int sz = sqlite3_column_bytes(pStmt, 1);
-    Blob *pNew = safe_realloc(0, sizeof(*pNew)+sz );
+    Blob *pNew = safe_realloc(0, SZ_BLOB(sz+1));
     pNew->id = sqlite3_column_int(pStmt, 0);
     pNew->sz = sz;
     pNew->seq = n++;
@@ -540,7 +570,7 @@ static void blobListLoadFromDb(
   }
   sqlite3_finalize(pStmt);
   *pN = n;
-  *ppList = head.pNext;
+  *ppList = head->pNext;
 }
 
 /*
@@ -978,7 +1008,8 @@ extern int fuzz_invariant(
   int iRow,               /* The row number for pStmt */
   int nRow,               /* Total number of output rows */
   int *pbCorrupt,         /* IN/OUT: Flag indicating a corrupt database file */
-  int eVerbosity          /* How much debugging output */
+  int eVerbosity,         /* How much debugging output */
+  unsigned int dbOpt      /* Default optimization flags */
 );
 
 /* Implementation of sqlite_dbdata and sqlite_dbptr */
@@ -990,7 +1021,7 @@ extern int sqlite3_dbdata_init(sqlite3*,const char**,void*);
 ** print the supplied SQL statement to stdout.
 */
 static int recoverSqlCb(void *pCtx, const char *zSql){
-  if( eVerbosity>=2 ){
+  if( eVerbosity>=2 && zSql ){
     printf("%s\n", zSql);
   }
   return SQLITE_OK;
@@ -1027,10 +1058,61 @@ static int recoverDatabase(sqlite3 *db){
   return rc;
 }
 
+/*
+** Special parameter binding, for testing and debugging purposes.
+**
+**     $int_NNN        ->   integer value NNN
+**     $text_TTTT      ->   floating point value TTT with destructor
+**     $carray_clr     ->   First argument to carray() for color names
+**     $carray_primes  ->   First argument to carray() for prime numbers
+*/
+static void bindDebugParameters(sqlite3_stmt *pStmt){
+  int nVar = sqlite3_bind_parameter_count(pStmt);
+  int i;
+  for(i=1; i<=nVar; i++){
+    const char *zVar = sqlite3_bind_parameter_name(pStmt, i);
+    if( zVar==0 ) continue;
+#ifdef SQLITE_ENABLE_CARRAY
+    if( strcmp(zVar,"$carray_clr")==0 ){
+      static char *azColorNames[] = {
+        "azure", "black", "blue",   "brown", "cyan",   "fuchsia", "gold",
+        "gray",  "green", "indigo", "khaki", "lime",   "magenta", "maroon",
+        "navy",  "olive", "orange", "pink",  "purple", "red",     "silver",
+        "tan",   "teal",  "violet", "white", "yellow"
+      };
+      sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0);
+    }else
+    if( strcmp(zVar,"$carray_primes")==0 ){
+      static int aPrimes[] = {
+        1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47,
+       53, 59, 61, 67, 71, 73, 79, 83, 89, 97
+      };
+      sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0);
+    }else
+#endif
+    if( strncmp(zVar, "$int_", 5)==0 ){
+      sqlite3_bind_int(pStmt, i, atoi(&zVar[5]));
+    }else
+    if( strncmp(zVar, "$text_", 6)==0 ){
+      size_t szVar = strlen(zVar);
+      char *zBuf = sqlite3_malloc64( szVar-5 );
+      if( zBuf ){
+        memcpy(zBuf, &zVar[6], szVar-5);
+        sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8);
+      }
+    }
+  }
+}
+
 /*
 ** Run the SQL text
 */
-static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){
+static int runDbSql(
+  sqlite3 *db,                /* Run SQL on this database connection */
+  const char *zSql,           /* The SQL to be run */
+  unsigned int *pBtsFlags,
+  unsigned int dbOpt          /* Default optimization flags */
+){
   int rc;
   sqlite3_stmt *pStmt;
   int bCorrupt = 0;
@@ -1044,6 +1126,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){
   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
   if( rc==SQLITE_OK ){
     int nRow = 0;
+    bindDebugParameters(pStmt);
     while( (rc = sqlite3_step(pStmt))==SQLITE_ROW ){
       nRow++;
       if( eVerbosity>=4 ){
@@ -1106,7 +1189,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){
           iRow++;
           for(iCnt=0; iCnt<99999; iCnt++){
             rc = fuzz_invariant(db, pStmt, iCnt, iRow, nRow,
-                                &bCorrupt, eVerbosity);
+                                &bCorrupt, eVerbosity, dbOpt);
             if( rc==SQLITE_DONE ) break;
             if( rc!=SQLITE_ERROR ) g.nInvariant++;
             if( eVerbosity>0 ){
@@ -1130,6 +1213,44 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){
   return sqlite3_finalize(pStmt);
 }
 
+/* Mappings into dbconfig settings for bits taken from bytes 72..75 of
+** the input database.
+**
+** This should be the same as in dbsqlfuzz.c.  Make sure those codes stay
+** in sync.
+*/
+static const struct {
+  unsigned int mask;
+  int iSetting;
+  char *zName;
+} aDbConfigSettings[] = {
+  {  0x0001, SQLITE_DBCONFIG_ENABLE_FKEY,        "enable_fkey"        },
+  {  0x0002, SQLITE_DBCONFIG_ENABLE_TRIGGER,     "enable_trigger"     },
+  {  0x0004, SQLITE_DBCONFIG_ENABLE_VIEW,        "enable_view"        },
+  {  0x0008, SQLITE_DBCONFIG_ENABLE_QPSG,        "enable_qpsg"        },
+  {  0x0010, SQLITE_DBCONFIG_TRIGGER_EQP,        "trigger_eqp"        },
+  {  0x0020, SQLITE_DBCONFIG_DEFENSIVE,          "defensive"          },
+  {  0x0040, SQLITE_DBCONFIG_WRITABLE_SCHEMA,    "writable_schema"    },
+  {  0x0080, SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, "legacy_alter_table" },
+  {  0x0100, SQLITE_DBCONFIG_STMT_SCANSTATUS,    "stmt_scanstatus"    },
+  {  0x0200, SQLITE_DBCONFIG_REVERSE_SCANORDER,  "reverse_scanorder"  },
+#ifdef SQLITE_DBCONFIG_STRICT_AGGREGATE
+  {  0x0400, SQLITE_DBCONFIG_STRICT_AGGREGATE,   "strict_aggregate"   },
+#endif
+  {  0x0800, SQLITE_DBCONFIG_DQS_DML,            "dqs_dml"            },
+  {  0x1000, SQLITE_DBCONFIG_DQS_DDL,            "dqs_ddl"            },
+  {  0x2000, SQLITE_DBCONFIG_TRUSTED_SCHEMA,     "trusted_schema"     },
+};
+
+/* Toggle a dbconfig setting
+*/
+static void toggleDbConfig(sqlite3 *db, int iSetting){
+  int v = 0;
+  sqlite3_db_config(db, iSetting, -1, &v);
+  v = !v;
+  sqlite3_db_config(db, iSetting, v, 0);
+}
+
 /* Invoke this routine to run a single test case */
 int runCombinedDbSqlInput(
   const uint8_t *aData,      /* Combined DB+SQL content */
@@ -1148,6 +1269,9 @@ int runCombinedDbSqlInput(
   int nSql;                  /* Bytes of SQL text */
   FuzzCtx cx;                /* Fuzzing context */
   unsigned int btsFlags = 0; /* Parsing flags */
+  unsigned int dbFlags = 0;  /* Flag values from db offset 72..75 */
+  unsigned int dbOpt = 0;    /* Flag values from db offset 76..79 */
+
 
   if( nByte<10 ) return 0;
   if( sqlite3_initialize() ) return 0;
@@ -1163,6 +1287,14 @@ int runCombinedDbSqlInput(
   memset(&cx, 0, sizeof(cx));
   iSql = decodeDatabase((unsigned char*)aData, (int)nByte, &aDb, &nDb);
   if( iSql<0 ) return 0;
+  if( nDb>=75 ){
+    dbFlags = ((unsigned int)aDb[72]<<24) + ((unsigned int)aDb[73]<<16) +
+              ((unsigned int)aDb[74]<<8) + (unsigned int)aDb[75];
+  }
+  if( nDb>=79 ){
+    dbOpt = ((unsigned int)aDb[76]<<24) + ((unsigned int)aDb[77]<<16) +
+            ((unsigned int)aDb[78]<<8) + (unsigned int)aDb[79];
+  }
   nSql = (int)(nByte - iSql);
   if( bScript ){
     char zName[100];
@@ -1183,7 +1315,12 @@ int runCombinedDbSqlInput(
     sqlite3_free(aDb);
     return 1;
   }
-  sqlite3_db_config(cx.db, SQLITE_DBCONFIG_STMT_SCANSTATUS, 1, 0);
+  sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, cx.db, dbOpt);
+  for(i=0; i<sizeof(aDbConfigSettings)/sizeof(aDbConfigSettings[0]); i++){
+    if( dbFlags & aDbConfigSettings[i].mask ){
+      toggleDbConfig(cx.db, aDbConfigSettings[i].iSetting);
+    }
+  }
   if( bVdbeDebug ){
     sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON", 0, 0, 0);
   }
@@ -1213,6 +1350,8 @@ int runCombinedDbSqlInput(
   }
   sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100);
   sqlite3_hard_heap_limit64(heapLimit);
+  rc = 1;
+  sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc);
 
   if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){
     aDb[18] = aDb[19] = 1;
@@ -1241,6 +1380,9 @@ int runCombinedDbSqlInput(
   /* Add the vt02 virtual table */
   sqlite3_vt02_init(cx.db, 0, 0);
 
+  /* Activate extensions */
+  sqlite3_randomjson_init(cx.db, 0, 0);
+
   /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used
   ** by the recovery API */
   sqlite3_dbdata_init(cx.db, 0, 0);
@@ -1270,7 +1412,7 @@ int runCombinedDbSqlInput(
         char cSaved = zSql[i+1];
         zSql[i+1] = 0;
         if( sqlite3_complete(zSql+j) ){
-          rc = runDbSql(cx.db, zSql+j, &btsFlags);
+          rc = runDbSql(cx.db, zSql+j, &btsFlags, dbOpt);
           j = i+1;
         }
         zSql[i+1] = cSaved;
@@ -1280,7 +1422,7 @@ int runCombinedDbSqlInput(
       }
     }
     if( j<i ){
-      runDbSql(cx.db, zSql+j, &btsFlags);
+      runDbSql(cx.db, zSql+j, &btsFlags, dbOpt);
     }
   }
 testrun_finished:
@@ -1498,7 +1640,7 @@ static int inmemDelete(
   return SQLITE_IOERR_DELETE;
 }
 
-/* Check for the existance of a file
+/* Check for the existence of a file
 */
 static int inmemAccess(
   sqlite3_vfs *pVfs,
@@ -1741,8 +1883,10 @@ static void showHelp(void){
 "Read databases and SQL scripts from SOURCE-DB and execute each script against\n"
 "each database, checking for crashes and memory leaks.\n"
 "Options:\n"
+"  --brief              Output only a summary of results at the end\n"
 "  --cell-size-check    Set the PRAGMA cell_size_check=ON\n"
-"  --dbid N             Use only the database where dbid=N\n"
+"  --dbid M..N          Use only the databases where dbid between M and N\n"
+"                       \"M..\" for M and afterwards. Just \"M\" for M only\n"
 "  --export-db DIR      Write databases to files(s) in DIR. Works with --dbid\n"
 "  --export-sql DIR     Write SQL to file(s) in DIR. Also works with --sqlid\n"
 "  --help               Show this help text\n"
@@ -1766,8 +1910,10 @@ static void showHelp(void){
 "  --result-trace       Show the results of each SQL command\n"
 "  --script             Output CLI script instead of running tests\n"
 "  --skip N             Skip the first N test cases\n"
+"  --slice M N          Run only the M-th out of each group of N tests\n"
 "  --spinner            Use a spinner to show progress\n"
-"  --sqlid N            Use only SQL where sqlid=N\n"
+"  --sqlid M..N         Use only SQL where sqlid between M..N\n"
+"                       \"M..\" for M and afterwards. Just \"M\" for M only\n"
 "  --timeout N          Maximum time for any one test in N millseconds\n"
 "  -v|--verbose         Increased output.  Repeat for more output.\n"
 "  --vdbe-debug         Activate VDBE debugging.\n"
@@ -1779,6 +1925,7 @@ static void showHelp(void){
 int main(int argc, char **argv){
   sqlite3_int64 iBegin;        /* Start time of this program */
   int quietFlag = 0;           /* True if --quiet or -q */
+  int briefFlag = 0;           /* Output summary report at the end */
   int verboseFlag = 0;         /* True if --verbose or -v */
   char *zInsSql = 0;           /* SQL statement for --load-db or --load-sql */
   int iFirstInsArg = 0;        /* First argv[] for --load-db or --load-sql */
@@ -1789,8 +1936,10 @@ int main(int argc, char **argv){
   Blob *pDb;                   /* For looping over template databases */
   int i;                       /* Loop index for the argv[] loop */
   int dbSqlOnly = 0;           /* Only use scripts that are dbsqlfuzz */
-  int onlySqlid = -1;          /* --sqlid */
-  int onlyDbid = -1;           /* --dbid */
+  int firstSqlid = -1;         /* First --sqlid range */
+  int lastSqlid = 0x7fffffff;  /* Last --sqlid range */
+  int firstDbid = -1;          /* --dbid */
+  int lastDbid = 0x7fffffff;   /* --dbid end */
   int nativeFlag = 0;          /* --native-vfs */
   int rebuildFlag = 0;         /* --rebuild */
   int vdbeLimitFlag = 0;       /* --limit-vdbe */
@@ -1823,8 +1972,12 @@ int main(int argc, char **argv){
   int bTimer = 0;              /* Show elapse time for each test */
   int nV;                      /* How much to increase verbosity with -vvvv */
   sqlite3_int64 tmStart;       /* Start of each test */
+  int iEstTime = 0;            /* LPF for the time-to-go */
+  int iSliceSz = 0;            /* Divide the test space into this many pieces */
+  int iSliceIdx = 0;           /* Only run the piece with this index */
 
   sqlite3_config(SQLITE_CONFIG_URI,1);
+  sqlite3_config(SQLITE_CONFIG_MEMSTATUS,1);
   registerOomSimulator();
   sqlite3_initialize();
   iBegin = timeOfDay();
@@ -1843,12 +1996,28 @@ int main(int argc, char **argv){
     if( z[0]=='-' ){
       z++;
       if( z[0]=='-' ) z++;
+      if( strcmp(z,"brief")==0 ){
+        briefFlag = 1;
+        quietFlag = 1;
+        verboseFlag = 1;
+        eVerbosity = 0;
+      }else
       if( strcmp(z,"cell-size-check")==0 ){
         cellSzCkFlag = 1;
       }else
       if( strcmp(z,"dbid")==0 ){
+        const char *zDotDot;
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
-        onlyDbid = integerValue(argv[++i]);
+        i++;
+        zDotDot = strstr(argv[i], "..");
+        if( zDotDot ){
+          firstDbid = atoi(argv[i]);
+          if( zDotDot[2] ){
+            lastDbid = atoi(&zDotDot[2]);
+          }
+        }else{
+          lastDbid = firstDbid = integerValue(argv[i]);
+        }
       }else
       if( strcmp(z,"export-db")==0 ){
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
@@ -1923,6 +2092,7 @@ int main(int argc, char **argv){
         g.uRandom = atoi(argv[++i]);
       }else
       if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){
+        briefFlag = 0;
         quietFlag = 1;
         verboseFlag = 0;
         eVerbosity = 0;
@@ -1941,15 +2111,33 @@ int main(int argc, char **argv){
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
         nSkip = atoi(argv[++i]);
       }else
+      if( strcmp(z,"slice")==0 ){
+        if( i>=argc-2 ) fatalError("missing arguments on %s", argv[i]);
+        iSliceIdx = integerValue(argv[++i]);
+        iSliceSz = integerValue(argv[++i]);
+        /* --slice implices --brief */
+        briefFlag = 1;
+        quietFlag = 1;
+        verboseFlag = 1;
+        eVerbosity = 0;
+      }else
       if( strcmp(z,"spinner")==0 ){
         bSpinner = 1;
       }else
-      if( strcmp(z,"timer")==0 ){
-        bTimer = 1;
-      }else
       if( strcmp(z,"sqlid")==0 ){
+        const char *zDotDot;
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
-        onlySqlid = integerValue(argv[++i]);
+        i++;
+        zDotDot = strstr(argv[i], "..");
+        if( zDotDot ){
+          firstSqlid = atoi(argv[i]);
+          if( zDotDot[2] ){
+            lastSqlid = atoi(&zDotDot[2]);
+          }
+        }else{
+          firstSqlid = integerValue(argv[i]);
+          lastSqlid = firstSqlid;
+        }
       }else
       if( strcmp(z,"timeout")==0 ){
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
@@ -1961,10 +2149,14 @@ int main(int argc, char **argv){
         fatalError("timeout is not available on non-unix systems");
 #endif
       }else
+      if( strcmp(z,"timer")==0 ){
+        bTimer = 1;
+      }else
       if( strcmp(z,"vdbe-debug")==0 ){
         bVdbeDebug = 1;
       }else
       if( strcmp(z,"verbose")==0 ){
+        briefFlag = 0;
         quietFlag = 0;
         verboseFlag++;
         eVerbosity++;
@@ -1979,7 +2171,9 @@ int main(int argc, char **argv){
       if( strcmp(z,"version")==0 ){
         int ii;
         const char *zz;
-        printf("SQLite %s %s\n", sqlite3_libversion(), sqlite3_sourceid());
+        printf("SQLite %s %s (%d-bit)\n",
+            sqlite3_libversion(), sqlite3_sourceid(),
+            8*(int)sizeof(char*));
         for(ii=0; (zz = sqlite3_compileoption_get(ii))!=0; ii++){
           printf("%s\n", zz);
         }
@@ -2029,6 +2223,12 @@ int main(int argc, char **argv){
       fatalError("cannot import into more than one database");
     }
   }
+  if( iSliceSz<=iSliceIdx 
+   || iSliceSz<=0
+   || iSliceIdx<0
+  ){
+    iSliceSz = iSliceIdx = 0;
+  }
 
   /* Process each source database separately */
   for(iSrcDb=0; iSrcDb<nSrcDb; iSrcDb++){
@@ -2195,13 +2395,14 @@ int main(int argc, char **argv){
         const char *zExDb = 
           "SELECT writefile(printf('%s/db%06d.db',?1,dbid),dbcontent),"
           "       dbid, printf('%s/db%06d.db',?1,dbid), length(dbcontent)"
-          "  FROM db WHERE ?2<0 OR dbid=?2;";
+          "  FROM db WHERE dbid BETWEEN ?2 AND ?3;";
         rc = sqlite3_prepare_v2(db, zExDb, -1, &pStmt, 0);
         if( rc ) fatalError("cannot prepare statement [%s]: %s",
                             zExDb, sqlite3_errmsg(db));
         sqlite3_bind_text64(pStmt, 1, zExpDb, strlen(zExpDb),
                             SQLITE_STATIC, SQLITE_UTF8);
-        sqlite3_bind_int(pStmt, 2, onlyDbid);
+        sqlite3_bind_int(pStmt, 2, firstDbid);
+        sqlite3_bind_int(pStmt, 3, lastDbid);
         while( sqlite3_step(pStmt)==SQLITE_ROW ){
           printf("write db-%d (%d bytes) into %s\n",
              sqlite3_column_int(pStmt,1),
@@ -2214,13 +2415,14 @@ int main(int argc, char **argv){
         const char *zExSql = 
           "SELECT writefile(printf('%s/sql%06d.txt',?1,sqlid),sqltext),"
           "       sqlid, printf('%s/sql%06d.txt',?1,sqlid), length(sqltext)"
-          "  FROM xsql WHERE ?2<0 OR sqlid=?2;";
+          "  FROM xsql WHERE sqlid BETWEEN ?2 AND ?3;";
         rc = sqlite3_prepare_v2(db, zExSql, -1, &pStmt, 0);
         if( rc ) fatalError("cannot prepare statement [%s]: %s",
                             zExSql, sqlite3_errmsg(db));
         sqlite3_bind_text64(pStmt, 1, zExpSql, strlen(zExpSql),
                             SQLITE_STATIC, SQLITE_UTF8);
-        sqlite3_bind_int(pStmt, 2, onlySqlid);
+        sqlite3_bind_int(pStmt, 2, firstSqlid);
+        sqlite3_bind_int(pStmt, 3, lastSqlid);
         while( sqlite3_step(pStmt)==SQLITE_ROW ){
           printf("write sql-%d (%d bytes) into %s\n",
              sqlite3_column_int(pStmt,1),
@@ -2236,11 +2438,11 @@ int main(int argc, char **argv){
     /* Load all SQL script content and all initial database images from the
     ** source db
     */
-    blobListLoadFromDb(db, "SELECT sqlid, sqltext FROM xsql", onlySqlid,
-                           &g.nSql, &g.pFirstSql);
+    blobListLoadFromDb(db, "SELECT sqlid, sqltext FROM xsql", firstSqlid,
+                           lastSqlid, &g.nSql, &g.pFirstSql);
     if( g.nSql==0 ) fatalError("need at least one SQL script");
-    blobListLoadFromDb(db, "SELECT dbid, dbcontent FROM db", onlyDbid,
-                       &g.nDb, &g.pFirstDb);
+    blobListLoadFromDb(db, "SELECT dbid, dbcontent FROM db", firstDbid,
+                       lastDbid, &g.nDb, &g.pFirstDb);
     if( g.nDb==0 ){
       g.pFirstDb = safe_realloc(0, sizeof(Blob));
       memset(g.pFirstDb, 0, sizeof(Blob));
@@ -2316,13 +2518,37 @@ int main(int argc, char **argv){
     for(pSql=g.pFirstSql; pSql; pSql=pSql->pNext){
       tmStart = timeOfDay();
       if( isDbSql(pSql->a, pSql->sz) ){
+        if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){
+          nTest++;
+          continue;
+        }
         sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id);
         if( bScript ){
           /* No progress output */
         }else if( bSpinner ){
-          int nTotal =g.nSql;
+          int nTotal = g.nSql;
           int idx = pSql->seq;
-          printf("\r%s: %d/%d   ", zDbName, idx, nTotal);
+          if( nSrcDb==1 && nTotal>idx && idx>=20 ){
+            int iToGo = (timeOfDay() - iBegin)*(nTotal-idx)/(idx*1000);
+            int hr, min, sec;
+            if( idx==20 ){
+              iEstTime = iToGo;
+            }else{
+              iEstTime = (iToGo + 7*iEstTime)/8;
+            }
+            hr = iEstTime/3600;
+            min = (iEstTime/60)%60;
+            sec = iEstTime%60;
+            if( hr>0 ){
+              printf("\r%s: %d/%d ETC %d:%02d:%02d  ",
+                     zDbName, idx, nTotal, hr, min, sec);
+            }else{
+              printf("\r%s: %d/%d ETC %02d:%02d    ",
+                     zDbName, idx, nTotal, min, sec);
+            }
+          }else{  
+            printf("\r%s: %d/%d           ", zDbName, idx, nTotal);
+          }
           fflush(stdout);
         }else if( verboseFlag>1 ){
           printf("%s\n", g.zTestName);
@@ -2354,6 +2580,10 @@ int main(int argc, char **argv){
       for(pDb=g.pFirstDb; pDb; pDb=pDb->pNext){
         int openFlags;
         const char *zVfs = "inmem";
+        if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){
+          nTest++;
+          continue;
+        }
         sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d",
                          pSql->id, pDb->id);
         if( bScript ){
@@ -2361,7 +2591,7 @@ int main(int argc, char **argv){
         }else if( bSpinner ){
           int nTotal = g.nDb*g.nSql;
           int idx = pSql->seq*g.nDb + pDb->id - 1;
-          printf("\r%s: %d/%d   ", zDbName, idx, nTotal);
+          printf("\r%s: %d/%d            ", zDbName, idx, nTotal);
           fflush(stdout);
         }else if( verboseFlag>1 ){
           printf("%s\n", g.zTestName);
@@ -2460,11 +2690,24 @@ int main(int argc, char **argv){
         }
       }
     }
-    if( bScript ){
+    if( briefFlag ){
+      sqlite3_int64 iElapse = timeOfDay() - iBegin;
+      if( iSliceSz>0 ){
+        printf("%s %s: slice %d/%d of %d tests, %d.%03d seconds\n",
+               pathTail(argv[0]), pathTail(g.zDbFile),
+               iSliceIdx, iSliceSz, nTest,
+               (int)(iElapse/1000), (int)(iElapse%1000));
+      }else{
+        printf("%s %s: 0 errors, %d tests, %d.%03d seconds\n",
+               pathTail(argv[0]), pathTail(g.zDbFile), nTest,
+               (int)(iElapse/1000), (int)(iElapse%1000));
+      }
+      iBegin = timeOfDay();
+    }else if( bScript ){
       /* No progress output */
     }else if( bSpinner ){
       int nTotal = g.nDb*g.nSql;
-      printf("\r%s: %d/%d   \n", zDbName, nTotal, nTotal);
+      printf("\r%s: %d/%d          \n", zDbName, nTotal, nTotal);
     }else if( !quietFlag && verboseFlag<2 ){
       printf(" 100%% - %d tests\n", g.nDb*g.nSql);
     }
@@ -2478,15 +2721,17 @@ int main(int argc, char **argv){
  
   } /* End loop over all source databases */
 
+           
   if( !quietFlag && !bScript ){
     sqlite3_int64 iElapse = timeOfDay() - iBegin;
     if( g.nInvariant ){
       printf("fuzzcheck: %u query invariants checked\n", g.nInvariant);
     }
     printf("fuzzcheck: 0 errors out of %d tests in %d.%03d seconds\n"
-           "SQLite %s %s\n",
+           "SQLite %s %s (%d-bit)\n",
            nTest, (int)(iElapse/1000), (int)(iElapse%1000),
-           sqlite3_libversion(), sqlite3_sourceid());
+           sqlite3_libversion(), sqlite3_sourceid(),
+           8*(int)sizeof(char*));
   }
   free(azSrcDb);
   free(pHeap);
diff --git a/test/fuzzdata6.db b/test/fuzzdata6.db
index b1424c21e4..67076d0026 100644
Binary files a/test/fuzzdata6.db and b/test/fuzzdata6.db differ
diff --git a/test/fuzzdata8.db b/test/fuzzdata8.db
index a4969047aa..bfa3e3ecd0 100644
Binary files a/test/fuzzdata8.db and b/test/fuzzdata8.db differ
diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c
index 5d473f19f3..6a5cfda689 100644
--- a/test/fuzzinvariants.c
+++ b/test/fuzzinvariants.c
@@ -30,7 +30,59 @@
 /* Forward references */
 static char *fuzz_invariant_sql(sqlite3_stmt*, int);
 static int sameValue(sqlite3_stmt*,int,sqlite3_stmt*,int,sqlite3_stmt*);
-static void reportInvariantFailed(sqlite3_stmt*,sqlite3_stmt*,int);
+static void reportInvariantFailed(
+  sqlite3_stmt *pOrig,   /* The original query */
+  sqlite3_stmt *pTest,   /* The alternative test query with a missing row */
+  int iRow,              /* Row number in pOrig */
+  unsigned int dbOpt,    /* Optimization flags on pOrig */
+  int noOpt              /* True if opt flags inverted for pTest */
+);
+
+/*
+** Special parameter binding, for testing and debugging purposes.
+**
+**     $int_NNN        ->   integer value NNN
+**     $text_TTTT      ->   floating point value TTT with destructor
+**     $carray_clr     ->   First argument to carray() for color names
+**     $carray_primes  ->   First argument to carray() for prime numbers
+*/
+static void bindDebugParameters(sqlite3_stmt *pStmt){
+  int nVar = sqlite3_bind_parameter_count(pStmt);
+  int i;
+  for(i=1; i<=nVar; i++){
+    const char *zVar = sqlite3_bind_parameter_name(pStmt, i);
+    if( zVar==0 ) continue;
+#ifdef SQLITE_ENABLE_CARRAY
+    if( strcmp(zVar,"$carray_clr")==0 ){
+      static char *azColorNames[] = {
+        "azure", "black", "blue",   "brown", "cyan",   "fuchsia", "gold",
+        "gray",  "green", "indigo", "khaki", "lime",   "magenta", "maroon",
+        "navy",  "olive", "orange", "pink",  "purple", "red",     "silver",
+        "tan",   "teal",  "violet", "white", "yellow"
+      };
+      sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0);
+    }else
+    if( strcmp(zVar,"$carray_primes")==0 ){
+      static int aPrimes[] = {
+        1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47,
+       53, 59, 61, 67, 71, 73, 79, 83, 89, 97
+      };
+      sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0);
+    }else
+#endif
+    if( strncmp(zVar, "$int_", 5)==0 ){
+      sqlite3_bind_int(pStmt, i, atoi(&zVar[5]));
+    }else
+    if( strncmp(zVar, "$text_", 6)==0 ){
+      size_t szVar = strlen(zVar);
+      char *zBuf = sqlite3_malloc64( szVar-5 );
+      if( zBuf ){
+        memcpy(zBuf, &zVar[6], szVar-5);
+        sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8);
+      }
+    }
+  }
+}
 
 /*
 ** Do an invariant check on pStmt.  iCnt determines which invariant check to
@@ -68,7 +120,8 @@ int fuzz_invariant(
   int iRow,               /* Current row number */
   int nRow,               /* Number of output rows from pStmt */
   int *pbCorrupt,         /* IN/OUT: Flag indicating a corrupt database file */
-  int eVerbosity          /* How much debugging output */
+  int eVerbosity,         /* How much debugging output */
+  unsigned int dbOpt      /* Default optimization flags */
 ){
   char *zTest;
   sqlite3_stmt *pTestStmt = 0;
@@ -76,13 +129,20 @@ int fuzz_invariant(
   int i;
   int nCol;
   int nParam;
+  int noOpt = (iCnt%3)==0;
 
   if( *pbCorrupt ) return SQLITE_DONE;
   nParam = sqlite3_bind_parameter_count(pStmt);
   if( nParam>100 ) return SQLITE_DONE;
   zTest = fuzz_invariant_sql(pStmt, iCnt);
   if( zTest==0 ) return SQLITE_DONE;
+  if( noOpt ){
+    sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, ~dbOpt);
+  }
   rc = sqlite3_prepare_v2(db, zTest, -1, &pTestStmt, 0);
+  if( noOpt ){
+    sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, db, dbOpt);
+  }
   if( rc ){
     if( eVerbosity ){
       printf("invariant compile failed: %s\n%s\n",
@@ -93,6 +153,7 @@ int fuzz_invariant(
     return rc;
   }
   sqlite3_free(zTest);
+  bindDebugParameters(pTestStmt);
   nCol = sqlite3_column_count(pStmt);
   for(i=0; i<nCol; i++){
     rc = sqlite3_bind_value(pTestStmt,i+1+nParam,sqlite3_column_value(pStmt,i));
@@ -157,6 +218,7 @@ int fuzz_invariant(
       printf("invariant-validity-check #2:\n%s\n", zSql);
       sqlite3_free(zSql);
     }
+    bindDebugParameters(pCk);
     while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){
       for(i=0; i<nCol; i++){
         if( !sameValue(pStmt, i, pTestStmt, i, 0) ) break;
@@ -185,6 +247,7 @@ int fuzz_invariant(
       }
 
       sqlite3_reset(pTestStmt);
+      bindDebugParameters(pCk);
       while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){
         for(i=0; i<nCol; i++){
           if( !sameValue(pStmt, i, pTestStmt, i, pCk) ) break;
@@ -212,7 +275,7 @@ int fuzz_invariant(
     }
     sqlite3_finalize(pCk);
     if( rc==SQLITE_DONE ){
-      reportInvariantFailed(pStmt, pTestStmt, iRow);
+      reportInvariantFailed(pStmt, pTestStmt, iRow, dbOpt, noOpt);
       return SQLITE_INTERNAL;
     }else if( eVerbosity>0 ){
       printf("invariant-error ignored due to the use of virtual tables\n");
@@ -223,7 +286,6 @@ int fuzz_invariant(
   return SQLITE_OK;
 }
 
-
 /*
 ** Generate SQL used to test a statement invariant.
 **
@@ -261,6 +323,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
   int bDistinct = 0;
   int bOrderBy = 0;
   int nParam = sqlite3_bind_parameter_count(pStmt);
+  int hasGroupBy = 0;
 
   switch( iCnt % 4 ){
     case 1:  bDistinct = 1;              break;
@@ -285,6 +348,8 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
     sqlite3_finalize(pBase);
     pBase = pStmt;
   }
+  hasGroupBy = sqlite3_strlike("%GROUP BY%",zIn,0)==0;
+  bindDebugParameters(pBase);
   for(i=0; i<sqlite3_column_count(pStmt); i++){
     const char *zColName = sqlite3_column_name(pBase,i);
     const char *zSuffix = zColName ? strrchr(zColName, ':') : 0;
@@ -308,7 +373,8 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){
     if( iCnt>1 && i+2!=iCnt ) continue;
     if( zColName==0 ) continue;
     if( sqlite3_column_type(pStmt, i)==SQLITE_NULL ){
-      sqlite3_str_appendf(pTest, " %s \"%w\" ISNULL", zAnd, zColName);
+      const char *zPlus = hasGroupBy ? "+" : "";
+      sqlite3_str_appendf(pTest, " %s %s\"%w\" ISNULL", zAnd, zPlus, zColName);
     }else{
       sqlite3_str_appendf(pTest, " %s \"%w\"=?%d", zAnd, zColName, 
                           i+1+nParam);
@@ -489,13 +555,17 @@ static void printRow(sqlite3_stmt *pStmt, int iRow){
 static void reportInvariantFailed(
   sqlite3_stmt *pOrig,   /* The original query */
   sqlite3_stmt *pTest,   /* The alternative test query with a missing row */
-  int iRow               /* Row number in pOrig */
+  int iRow,              /* Row number in pOrig */
+  unsigned int dbOpt,    /* Optimization flags on pOrig */
+  int noOpt              /* True if opt flags inverted for pTest */
 ){
   int iTestRow = 0;
   printf("Invariant check failed on row %d.\n", iRow);
-  printf("Original query --------------------------------------------------\n");
+  printf("Original query (opt-flags: 0x%08x) --------------------------\n",
+         dbOpt);
   printf("%s\n", sqlite3_expanded_sql(pOrig));
-  printf("Alternative query -----------------------------------------------\n");
+  printf("Alternative query (opt-flags: 0x%08x) -----------------------\n",
+         noOpt ? ~dbOpt : dbOpt);
   printf("%s\n", sqlite3_expanded_sql(pTest));
   printf("Result row that is missing from the alternative -----------------\n");
   printRow(pOrig, iRow);
diff --git a/test/gcfault.test b/test/gcfault.test
index d54b78fafc..2d77f9ef2c 100644
--- a/test/gcfault.test
+++ b/test/gcfault.test
@@ -10,7 +10,7 @@
 #***********************************************************************
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing OOM error handling within the built-in 
-# group_concat() function.
+# group_concat() and string_agg() functions.
 #
 
 set testdir [file dirname $argv0]
@@ -40,7 +40,7 @@ foreach {enc} {
   }
 
   do_faultsim_test 1.$enc.2 -faults oom-t* -body {
-    execsql { SELECT group_concat(e, (SELECT s FROM s WHERE i=2)) FROM e }
+    execsql { SELECT string_agg(e, (SELECT s FROM s WHERE i=2)) FROM e }
   }
 
   do_faultsim_test 1.$enc.3 -faults oom-t* -prep {
diff --git a/test/gencol1.test b/test/gencol1.test
index f3fbb0dfba..a247a7148d 100644
--- a/test/gencol1.test
+++ b/test/gencol1.test
@@ -211,9 +211,9 @@ do_catchsql_test gencol1-6.10 {
   REPLACE INTO t0(c1) VALUES(NULL);
 } {1 {NOT NULL constraint failed: t0.c0}}
 
-# 2019-11-06 ticket https://www.sqlite.org/src/info/2399f5986134f79c
-# 2019-12-27 ticket https://www.sqlite.org/src/info/5fbc159eeb092130
-# 2019-12-27 ticket https://www.sqlite.org/src/info/37823501c68a09f9
+# 2019-11-06 ticket https://sqlite.org/src/info/2399f5986134f79c
+# 2019-12-27 ticket https://sqlite.org/src/info/5fbc159eeb092130
+# 2019-12-27 ticket https://sqlite.org/src/info/37823501c68a09f9
 #
 # All of the above tickets deal with NOT NULL ON CONFLICT REPLACE
 # constraints on tables that have generated columns.
@@ -662,11 +662,8 @@ do_execsql_test gencol1-23.4 {
 
 # 2023-03-07 https://sqlite.org/forum/forumpost/b312e075b5
 #
-do_execsql_test gencol1-23.5 {
+do_catchsql_test gencol1-23.5 {
   CREATE TABLE v0(c1 INT, c2 AS (RAISE(IGNORE)));
-}
-do_catchsql_test gencol1-23.6 {
-  SELECT * FROM v0;
 } {1 {RAISE() may only be used within a trigger-program}}
 
 finish_test
diff --git a/test/hook.test b/test/hook.test
index 3d735875df..a4256732e1 100644
--- a/test/hook.test
+++ b/test/hook.test
@@ -488,11 +488,21 @@ proc preupdate_hook {args} {
   set type [lindex $args 0]
   eval lappend ::preupdate $args
   if {$type != "INSERT"} {
+    set x [catch {db preupdate old [db preupdate count]}]
+    if {!$x} {
+      lappend "ERROR: sqlite3_preupdate_old() accepted an out-of-bounds\
+               column index"
+    }
     for {set i 0} {$i < [db preupdate count]} {incr i} {
       lappend ::preupdate [db preupdate old $i]
     }
   }
   if {$type != "DELETE"} {
+    set x [catch {db preupdate new [db preupdate count]}]
+    if {!$x} {
+      lappend "ERROR: sqlite3_preupdate_old() accepted an out-of-bounds\
+               column index"
+    }
     for {set i 0} {$i < [db preupdate count]} {incr i} {
       set rc [catch { db preupdate new $i } v]
       lappend ::preupdate $v
@@ -706,11 +716,13 @@ ifcapable altertable {
   }
 }
 
-if 0 {
+if 1 {
   # At time of writing, these two are broken. They demonstrate that the
   # sqlite3_preupdate_old() method does not handle the case where ALTER TABLE
   # has been used to add a column with a default value other than NULL.
   #
+  # 2024-09-18: These are now fixed.
+  #
   do_preupdate_test 7.5.2.1 {
     DELETE FROM t8 WHERE a = 'one'
   } {
@@ -1022,4 +1034,37 @@ do_catchsql_test 12.6 {
   INSERT INTO t4 VALUES('def', 3);
 } {1 {UNIQUE constraint failed: t4.a}}
 
+#-------------------------------------------------------------------------
+# Test adding non-NULL default values using ALTER TABLE.
+#
+reset_db
+db preupdate hook preupdate_hook
+do_execsql_test 13.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(100), (200), (300), (400);
+}
+
+do_execsql_test 13.1 {
+  ALTER TABLE t1 ADD COLUMN b DEFAULT 1234;
+  ALTER TABLE t1 ADD COLUMN c DEFAULT 'abcdef';
+  ALTER TABLE t1 ADD COLUMN d DEFAULT NULL;
+}
+
+do_preupdate_test 13.2 {
+  DELETE FROM t1 WHERE a=300
+} {DELETE main t1 300 300 0 300 1234 abcdef {}}
+
+do_preupdate_test 13.3 {
+  UPDATE t1 SET d='hello world' WHERE a=200
+} {
+  UPDATE main t1 200 200 0 200 1234 abcdef {} 
+                           200 1234 abcdef {hello world}
+}
+
+do_preupdate_test 13.4 {
+  INSERT INTO t1 DEFAULT VALUES;
+} {
+  INSERT main t1 401 401 0 401 1234 abcdef {}
+}
+
 finish_test
diff --git a/test/icu.test b/test/icu.test
index 644cbb1f08..c1b5653d4f 100644
--- a/test/icu.test
+++ b/test/icu.test
@@ -149,7 +149,7 @@ ifcapable icu {
 # 2020-03-19
 # The ESCAPE clause on LIKE takes precedence over wildcards
 #
-do_execsql_test idu-6.0 {
+do_execsql_test icu-6.0 {
   DROP TABLE IF EXISTS t1;
   CREATE TABLE t1(id INTEGER PRIMARY KEY, x TEXT);
   INSERT INTO t1 VALUES
@@ -164,4 +164,20 @@ do_execsql_test icu-6.1 {
   SELECT id FROM t1 WHERE x LIKE 'abc__' ESCAPE '_';
 } {2}
 
+# 2024-04-02
+# Optional 3rd argument to icu_load_collation() that specifies
+# the "strength" of comparison.
+#
+reset_db
+do_catchsql_test icu-7.1 {
+  SELECT icu_load_collation('en_US','error','xyzzy');
+} {1 {unknown collation strength "xyzzy" - should be one of: PRIMARY SECONDARY TERTIARY DEFAULT QUARTERNARY IDENTICAL}}
+do_execsql_test icu-7.2 {
+  SELECT icu_load_collation('en_US','prim','PRIMARY'),
+         icu_load_collation('en_US','dflt','DEFAULT');
+} {{} {}}
+do_execsql_test icu-7.3 {
+  SELECT char(0x100)=='a', char(0x100)=='a' COLLATE dflt, char(0x100)=='a' COLLATE prim;
+} {0 0 1}
+
 finish_test
diff --git a/test/ieee754.test b/test/ieee754.test
index bd806d2cf3..467416dae8 100644
--- a/test/ieee754.test
+++ b/test/ieee754.test
@@ -57,4 +57,16 @@ do_execsql_test ieee754-112 {
   SELECT ieee754(4503599627370495,973) is null;
 } {1}
 
+do_execsql_test ieee754-200 {
+  SELECT ieee754(0.0), hex(ieee754_to_blob(ieee754(0,-1075)));
+} {ieee754(0,-1075) 0000000000000000}
+
+# Special case.  -0.0 is rendered as ieee754(-1,-3071).
+#
+do_execsql_test ieee754-201 {
+  SELECT ieee754(-0.0), hex(ieee754_to_blob(ieee754(-1,-3071)));
+} {ieee754(-1,-3071) 8000000000000000}
+
+
+
 finish_test
diff --git a/test/imposter1.test b/test/imposter1.test
index 196767be15..da58266be2 100644
--- a/test/imposter1.test
+++ b/test/imposter1.test
@@ -85,6 +85,7 @@ do_execsql_test imposter-1.2 {
 # database.
 #
 do_execsql_test imposter-1.3 {
+  PRAGMA writable_schema=on;
   DELETE FROM xt1 WHERE rowid=5;
   INSERT INTO xt1(rowid,a,b,c,d) VALUES(99,'hello',1099,2022,NULL);
   SELECT * FROM chnglog ORDER BY rowid;
diff --git a/test/in.test b/test/in.test
index 601c7e3b4d..67fdd4f555 100644
--- a/test/in.test
+++ b/test/in.test
@@ -652,7 +652,7 @@ do_execsql_test in-14.1 {
   SELECT * FROM c1 WHERE a IN (SELECT a FROM c1) ORDER BY 1
 } {1 2 3 4}
 
-# 2019-02-20 Ticket https://www.sqlite.org/src/tktview/df46dfb631f75694fbb97033b69
+# 2019-02-20 Ticket https://sqlite.org/src/tktview/df46dfb631f75694fbb97033b69
 #
 do_execsql_test in-15.0 {
   DROP TABLE IF EXISTS t1;
@@ -736,7 +736,7 @@ do_execsql_test in-16.2 {
 } {}
 
 # 2019-06-11
-# https://www.sqlite.org/src/info/57353f8243c637c0
+# https://sqlite.org/src/info/57353f8243c637c0
 #
 do_execsql_test in-17.1 {
   SELECT 1 IN ('1');
@@ -760,7 +760,7 @@ do_execsql_test in-18.1 {
   SELECT * FROM t0 WHERE '1' IN (t0.c0);
 } {}
 
-# 2019-09-02 ticket https://www.sqlite.org/src/info/2841e99d104c6436
+# 2019-09-02 ticket https://sqlite.org/src/info/2841e99d104c6436
 # For the IN_INDEX_NOOP optimization, apply REAL affinity to the LHS
 # values prior to comparison if the RHS has REAL affinity.
 #
diff --git a/test/in4.test b/test/in4.test
index a3fe22e787..07cd3e284d 100644
--- a/test/in4.test
+++ b/test/in4.test
@@ -442,7 +442,7 @@ do_execsql_test 10.0 {
 } {10}
 
 # 2021-06-13 dbsqlfuzz e41762333a4d6e90a49e628f488d0873b2dba4c5
-# The opcode that preceeds OP_SeekScan is usually OP_IdxGT, but can
+# The opcode that precedes OP_SeekScan is usually OP_IdxGT, but can
 # sometimes be OP_IdxGE
 #
 reset_db
@@ -458,14 +458,14 @@ do_execsql_test 11.0 {
 do_execsql_test 11.1 {
   SELECT * FROM t1
    WHERE b IN (345, (SELECT 1 FROM t1 
-                      WHERE b IN (345 NOT GLOB 510)
+                      WHERE b IN (coalesce(1,random()))
                         AND c GLOB 'abc*xyz'))
      AND c BETWEEN 'abc' AND 'xyz';
 } {xyz 1 abcdefxyz 99}
 do_execsql_test 11.2 {
   EXPLAIN SELECT * FROM t1
    WHERE b IN (345, (SELECT 1 FROM t1 
-                      WHERE b IN (345 NOT GLOB 510)
+                      WHERE b IN (coalesce(1,random()))
                         AND c GLOB 'abc*xyz'))
      AND c BETWEEN 'abc' AND 'xyz';
 } {/ SeekScan /}
diff --git a/test/in5.test b/test/in5.test
index 6680641ff3..933eb90026 100644
--- a/test/in5.test
+++ b/test/in5.test
@@ -266,4 +266,26 @@ do_execsql_test 9.2 {
   SELECT lower('1e500') FROM t0 WHERE rowid != lower('1e500');
 } {1e500}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+
+do_execsql_test 10.0 {
+  CREATE TABLE t1(a, b TEXT COLLATE NOCASE);
+  INSERT INTO t1 VALUES('abc', 'def');
+  INSERT INTO t1 VALUES('ghi', 'jkl');
+}
+
+do_execsql_test 10.1 {
+  SELECT rowid FROM t1 WHERE (a, b) IN ( VALUES('abc', 'def'), ('ghi', 'JKL') );
+} {1 2}
+
+do_execsql_test 10.2 {
+  CREATE INDEX i1 ON t1(a, b COLLATE BINARY);
+}
+
+do_execsql_test 10.3 {
+  SELECT rowid FROM t1 WHERE (a, b) IN ( VALUES('abc', 'def'), ('ghi', 'JKL') );
+} {1 2}
+
 finish_test
diff --git a/test/in7.test b/test/in7.test
new file mode 100644
index 0000000000..763396140a
--- /dev/null
+++ b/test/in7.test
@@ -0,0 +1,251 @@
+# 2024-05-01
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix in7
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c PRIMARY KEY);
+  CREATE TABLE t2(x, y, z);
+}
+
+foreach {tn nNext idx sql} {
+  1 1 {
+    CREATE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE (a, b) IN (SELECT x, y FROM t2)
+  } 
+
+  2 0 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE (a, b) IN (SELECT x, y FROM t2)
+  } 
+
+  3 0 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a = ? AND b = ?
+  } 
+
+  3 1 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a = ? AND b IS ?
+  } 
+
+  4 0 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a = ? AND b IN (?, ?, ?);
+  } 
+
+  5 1 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b, c);
+  } {
+    SELECT * FROM t1 WHERE a = ? AND b = ?
+  } 
+
+  6 0 {
+  } {
+    SELECT * FROM t1 WHERE c IN (SELECT z FROM t2)
+  } 
+
+  7 0 {
+  } {
+    SELECT * FROM t1 WHERE (a, c) IN (SELECT z, x FROM t2)
+  } 
+
+  8 1 {
+  } {
+    SELECT * FROM t1 WHERE a IN (SELECT z FROM t2)
+  } 
+
+  9 1 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a IN (SELECT z FROM t2) AND b IS ?
+  } 
+  10 0 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a IN (SELECT z FROM t2) AND b = ?
+  } 
+  11 1 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a IS NULL AND b IN (SELECT z FROM t2)
+  } 
+  12 0 {
+    CREATE UNIQUE INDEX i1 ON t1(a, b);
+  } {
+    SELECT * FROM t1 WHERE a = ? AND b IN (SELECT z FROM t2)
+  } 
+} {
+  do_test 1.1.$tn {
+    execsql BEGIN
+    execsql $idx
+
+    catch { array unset root_to_tbl }
+    catch { array unset csr_to_root }
+
+    db eval {SELECT rootpage, tbl_name FROM sqlite_schema} {
+      set root_to_tbl($rootpage) $tbl_name
+    }
+
+    set nSeen 0
+    db eval "explain $sql" {
+      if {$opcode=="OpenRead"} {
+        set csr_to_root($p1) $p2
+      }
+      if {$opcode=="Next"} {
+        catch {
+          set root $csr_to_root($p1)
+          set tbl $root_to_tbl($root)
+          if {$tbl=="t1"} {incr nSeen}
+        }
+      }
+    }
+
+    execsql ROLLBACK
+
+    set nSeen
+  } $nNext
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a TEXT PRIMARY KEY, b TEXT) WITHOUT ROWID;
+  INSERT INTO t1 VALUES('1', 'one');
+  INSERT INTO t1 VALUES('2', NULL);
+  INSERT INTO t1 VALUES('3', 'three');
+}
+
+do_execsql_test 2.1 {
+  SELECT b FROM t1 WHERE a IN (1,2,3) ORDER BY b ASC NULLS LAST;
+} {one three {}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE x1(a);
+  INSERT INTO x1 VALUES(1), (2), (3);
+
+  CREATE TABLE x2(b);
+  INSERT INTO x2 VALUES(4), (5), (6);
+
+  CREATE TABLE t1(u);
+  INSERT INTO t1 VALUES(1), (2), (3), (4), (5), (6);
+
+  CREATE VIEW v1 AS SELECT u FROM t1 WHERE u IN (
+    SELECT a FROM x1
+  );
+  CREATE VIEW v2 AS SELECT u FROM t1 WHERE u IN (
+    SELECT b FROM x2
+  );
+}
+
+do_execsql_test 3.1 {
+  SELECT * FROM v1
+} {
+  1 2 3
+}
+
+do_execsql_test 3.2 {
+  SELECT * FROM v2
+} {
+  4 5 6 
+}
+
+do_execsql_test 3.3 {
+  SELECT * FROM v2
+  UNION ALL
+  SELECT * FROM v1
+} {
+  4 5 6
+  1 2 3 
+}
+
+do_execsql_test 3.4 {
+  WITH w1 AS (
+    SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3
+  ),
+  w2 AS (
+    SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6
+  )
+  SELECT * FROM v1 WHERE u IN w1
+  UNION ALL
+  SELECT * FROM v2 WHERE u IN w2
+} {
+  1 2 3 4 5 6
+}
+
+# 2024-11-20 https://sqlite.org/forum/forumpost/0b9ded2f8428ac00
+#
+# Bug in SubrtnSig logic.  If a SELECT statement is copied and the copy
+# is subsequently modified, we need to change the Select.selId on the
+# copy so that when the copy is used to generate code, the SubrtnSig
+# logic won't try to substitute the original SELECT in place of the
+# copy which is now different.
+#
+do_execsql_test 3.5 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1 (a int UNIQUE);
+  CREATE TABLE t2 (b int UNIQUE);
+  INSERT INTO t1 VALUES (1);
+  INSERT INTO t2 VALUES (1), (2);
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) = (1, 1);
+} {1 1}
+do_execsql_test 3.6 {
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) IN ((1, 1));
+} {1 1}
+do_execsql_test 3.7 {
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) = (1, 2);
+} {1 2}
+do_execsql_test 3.8 {
+  SELECT t1.a, t2.b FROM t1, t2 WHERE (t1.a, t2.b) IN ((1, 2));
+} {1 2}
+
+# 2025-01-30 Inifinite loop in byte-code discovered by dbsqlfuzz
+# having to do with SubrtnSig logic.  The code was using a Subroutine
+# from within itself resulting in infinite recursion.
+#
+# This test will spin forever if the bug has not been fixed, or if
+# it reappears.
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(1,x'1111');
+  CREATE TABLE t2(c);
+  CREATE TABLE t3(d);
+  CREATE TRIGGER t1tr UPDATE ON t1 BEGIN
+    UPDATE t1 SET b=x'2222' FROM t2;
+    UPDATE t1
+       SET b = (SELECT a IN (SELECT a
+                               FROM t1
+                              WHERE (b,a) IN (SELECT rowid, d
+                                                FROM t3
+                                             )
+                            )
+                  FROM t1 NATURAL RIGHT JOIN t1
+               );
+  END;
+  UPDATE t1 SET b=x'3333';
+  SELECT quote(b) FROM t1;
+} {X'3333'}
+
+finish_test
diff --git a/test/incrblob4.test b/test/incrblob4.test
index dbff8eb7d5..c9bcee8a3b 100644
--- a/test/incrblob4.test
+++ b/test/incrblob4.test
@@ -106,4 +106,102 @@ do_test 4.4 {
 } {SQLITE_LOCKED}
 close $blob
 
+#-------------------------------------------------------------------------
+
+ifcapable preupdate {
+
+reset_db
+do_execsql_test 5.1 {
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t2 VALUES(1000, 'abcdefghijklmnopqrstuvwxyz');
+  INSERT INTO t2 VALUES(2000, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ');
+  INSERT INTO t2 VALUES(3000, 'abcdefghijklmnopqrstuvwxyz');
+}
+
+do_test 5.2.1 {
+  execsql BEGIN
+  set blob [db incrblob t2 b 2000]
+  seek $blob 0
+  puts -nonewline $blob "hello "
+  flush $blob
+  execsql ROLLBACK
+} {}
+
+do_test 5.2.2 {
+  puts -nonewline $blob "world"
+  set rc [catch { flush $blob } msg]
+  list $rc [regsub {input/output} $msg {I/O}]
+} "1 {error flushing \"$blob\": I/O error}"
+catch { close $blob }
+
+set preupdate_count 0
+proc preupdate {args} { incr ::preupdate_count ; return {} }
+db preupdate hook preupdate
+
+set preupdate_count 0
+do_test 5.3.1 {
+  execsql BEGIN
+  set blob [db incrblob t2 b 1000]
+  seek $blob 0
+  puts -nonewline $blob "hello "
+  flush $blob
+  execsql ROLLBACK
+} {}
+
+do_test 5.3.2 {
+  puts -nonewline $blob "world"
+  set rc [catch { flush $blob } msg]
+  list $rc [regsub {input/output} $msg {I/O}]
+} "1 {error flushing \"$blob\": I/O error}"
+catch { close $blob }
+
+do_test 5.3.3 {
+  set ::preupdate_count
+} {1}
+
+set preupdate_count 0
+do_test 5.4.1 {
+  execsql BEGIN
+  set blob [db incrblob t2 b 1000]
+  seek $blob 0
+  puts -nonewline $blob "hello "
+  flush $blob
+  execsql { DELETE FROM t2 WHERE a=3000; }
+} {}
+
+do_test 5.4.2 {
+  puts -nonewline $blob "world"
+  list [catch { flush $blob } msg] $msg
+} "0 {}"
+catch { close $blob }
+catchsql { ROLLBACK }
+
+do_test 5.3.3 {
+  set ::preupdate_count
+} {3}
+
+set preupdate_count 0
+do_test 5.4.3 {
+  execsql BEGIN
+  set blob [db incrblob t2 b 2000]
+  seek $blob 0
+  puts -nonewline $blob "hello "
+  flush $blob
+  execsql { UPDATE t2 SET b='abcdefghijklmnopqrstuvwxyz' WHERE a=2000 }
+} {}
+
+do_test 5.4.4 {
+  puts -nonewline $blob "world"
+  set rc [catch { flush $blob } msg]
+  list $rc [regsub {input/output} $msg {I/O}]
+} "1 {error flushing \"$blob\": I/O error}"
+catch { close $blob }
+catchsql { ROLLBACK }
+
+do_test 5.4.5 {
+  set ::preupdate_count
+} {2}
+
+}
+
 finish_test
diff --git a/test/incrvacuum3.test b/test/incrvacuum3.test
index e901bfe1f5..b4cbc8b0c8 100644
--- a/test/incrvacuum3.test
+++ b/test/incrvacuum3.test
@@ -50,8 +50,8 @@ proc check_on_disk {} {
   set sz [file size test.db]
   set fd [open test.db]
   set fd2 [open test2.db w]
-  fconfigure $fd  -encoding binary -translation binary
-  fconfigure $fd2 -encoding binary -translation binary
+  fconfigure $fd  -translation binary
+  fconfigure $fd2 -translation binary
   if {$sz>$::sqlite_pending_byte} {
     puts -nonewline $fd2 [read $fd $::sqlite_pending_byte]
     seek $fd [expr $::sqlite_pending_byte+512]
diff --git a/test/index.test b/test/index.test
index 11d3d7191c..25ff41762d 100644
--- a/test/index.test
+++ b/test/index.test
@@ -738,7 +738,7 @@ do_test index-21.2 {
   }
 } {0 {9 5 1}}
 
-# 2019-05-01 ticket https://www.sqlite.org/src/info/3be1295b264be2fa
+# 2019-05-01 ticket https://sqlite.org/src/info/3be1295b264be2fa
 do_execsql_test index-22.0 {
   DROP TABLE IF EXISTS t1;
   CREATE TABLE t1(a, b TEXT);
@@ -748,7 +748,7 @@ do_execsql_test index-22.0 {
   SELECT a, b, '|' FROM t1;
 } {a 1 | a 0 |}
 
-# 2019-05-10 ticket https://www.sqlite.org/src/info/ae0f637bddc5290b
+# 2019-05-10 ticket https://sqlite.org/src/info/ae0f637bddc5290b
 do_execsql_test index-23.0 {
   DROP TABLE t1;
   CREATE TABLE t1(a TEXT, b REAL);
diff --git a/test/index6.test b/test/index6.test
index 16370f521d..007823a283 100644
--- a/test/index6.test
+++ b/test/index6.test
@@ -391,7 +391,7 @@ do_execsql_test index6-11.2 {
 } {/USING INDEX t11x/}
 
 # 2018-12-08
-# Ticket https://www.sqlite.org/src/info/1d958d90596593a7
+# Ticket https://sqlite.org/src/info/1d958d90596593a7
 # NOT IN operator fails when using a partial index.
 #
 do_execsql_test index6-12.1 {
@@ -412,7 +412,7 @@ do_execsql_test index6-12.2 {
 } {1 2}
 
 # 2019-05-04
-# Ticket https://www.sqlite.org/src/tktview/5c6955204c392ae763a95
+# Ticket https://sqlite.org/src/tktview/5c6955204c392ae763a95
 # Theorem prover error
 #
 do_execsql_test index6-13.1 {
@@ -438,8 +438,8 @@ do_execsql_test index6-14.2 {
 } {{} row}
 
 # 2019-08-30
-# Ticket https://www.sqlite.org/src/info/a6408d42b9f44462
-# Ticket https://www.sqlite.org/src/info/fba33c8b1df6a915
+# Ticket https://sqlite.org/src/info/a6408d42b9f44462
+# Ticket https://sqlite.org/src/info/fba33c8b1df6a915
 # https://sqlite.org/src/info/bac716244fddac1fe841
 #
 do_execsql_test index6-15.1 {
diff --git a/test/indexA.test b/test/indexA.test
new file mode 100644
index 0000000000..518d7e18ad
--- /dev/null
+++ b/test/indexA.test
@@ -0,0 +1,350 @@
+# 2023 September 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix indexA
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a TEXT, b, c);
+  CREATE INDEX i1 ON t1(b, c) WHERE a='abc';
+  INSERT INTO t1 VALUES('abc', 1, 2);
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1 WHERE a='abc'
+} {abc 1 2}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a='abc'
+} {USING COVERING INDEX i1}
+
+do_execsql_test 1.3 {
+  CREATE INDEX i2 ON t1(b, c) WHERE a=5;
+  INSERT INTO t1 VALUES(5, 4, 3);
+
+  SELECT a, typeof(a), b, c FROM t1 WHERE a=5;
+} {5 text 4 3}
+
+do_execsql_test 1.4 {
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES('v');
+}
+
+do_execsql_test 1.5 {
+  SELECT x, a, b, c FROM t2 LEFT JOIN t1 ON (a=5 AND b=x)
+} {v {} {} {}}
+
+do_execsql_test 1.6 {
+  SELECT x, a, b, c FROM t2 RIGHT JOIN t1 ON (t1.a=5 AND t1.b=t2.x)
+} {{} abc 1 2   {} 5 4 3}
+
+do_eqp_test 1.7 {
+  SELECT x, a, b, c FROM t2 RIGHT JOIN t1 ON (t1.a=5 AND t1.b=t2.x)
+} {USING INDEX i2}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 2.0 {
+  CREATE TABLE x1(a TEXT, b, c);
+  INSERT INTO x1 VALUES('2', 'two', 'ii');
+  INSERT INTO x1 VALUES('2.0', 'twopointoh', 'ii.0');
+
+  CREATE TABLE x2(a NUMERIC, b, c);
+  INSERT INTO x2 VALUES('2', 'two', 'ii');
+  INSERT INTO x2 VALUES('2.0', 'twopointoh', 'ii.0');
+
+  CREATE TABLE x3(a REAL, b, c);
+  INSERT INTO x3 VALUES('2', 'two', 'ii');
+  INSERT INTO x3 VALUES('2.0', 'twopointoh', 'ii.0');
+}
+
+foreach {tn idx} {
+  0 {
+  }
+  1 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a=2;
+    CREATE INDEX i2 ON x2(b, c) WHERE a=2;
+    CREATE INDEX i3 ON x3(b, c) WHERE a=2;
+  }
+  2 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a=2.0;
+    CREATE INDEX i2 ON x2(b, c) WHERE a=2.0;
+    CREATE INDEX i3 ON x3(b, c) WHERE a=2.0;
+  }
+  3 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a='2.0';
+    CREATE INDEX i2 ON x2(b, c) WHERE a='2.0';
+    CREATE INDEX i3 ON x3(b, c) WHERE a='2.0';
+  }
+  4 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a='2';
+    CREATE INDEX i2 ON x2(b, c) WHERE a='2';
+    CREATE INDEX i3 ON x3(b, c) WHERE a='2';
+  }
+} {
+  execsql { DROP INDEX IF EXISTS i1 }
+  execsql { DROP INDEX IF EXISTS i2 }
+  execsql { DROP INDEX IF EXISTS i3 }
+
+  execsql $idx
+  do_execsql_test 2.1.$tn.1 {
+    SELECT *, typeof(a) FROM x1 WHERE a=2
+  } {2 two ii text}
+  do_execsql_test 2.1.$tn.2 {
+    SELECT *, typeof(a) FROM x1 WHERE a=2.0
+  } {2.0 twopointoh ii.0 text}
+  do_execsql_test 2.1.$tn.3 {
+    SELECT *, typeof(a) FROM x1 WHERE a='2'
+  } {2 two ii text}
+  do_execsql_test 2.1.$tn.4 {
+    SELECT *, typeof(a) FROM x1 WHERE a='2.0'
+  } {2.0 twopointoh ii.0 text}
+
+  do_execsql_test 2.1.$tn.5 {
+    SELECT *, typeof(a) FROM x2 WHERE a=2
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+  do_execsql_test 2.1.$tn.6 {
+    SELECT *, typeof(a) FROM x2 WHERE a=2.0
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+  do_execsql_test 2.1.$tn.7 {
+    SELECT *, typeof(a) FROM x2 WHERE a='2'
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+  do_execsql_test 2.1.$tn.8 {
+    SELECT *, typeof(a) FROM x2 WHERE a='2.0'
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+
+  do_execsql_test 2.1.$tn.9 {
+    SELECT *, typeof(a) FROM x3 WHERE a=2
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+  do_execsql_test 2.1.$tn.10 {
+    SELECT *, typeof(a) FROM x3 WHERE a=2.0
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+  do_execsql_test 2.1.$tn.11 {
+    SELECT *, typeof(a) FROM x3 WHERE a='2'
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+  do_execsql_test 2.1.$tn.12 {
+    SELECT *, typeof(a) FROM x3 WHERE a='2.0'
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+
+}
+
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE x1(a TEXT, d PRIMARY KEY, b, c) WITHOUT ROWID;
+  INSERT INTO x1 VALUES('2', 1, 'two', 'ii');
+  INSERT INTO x1 VALUES('2.0', 2, 'twopointoh', 'ii.0');
+
+  CREATE TABLE x2(a NUMERIC, b, c, d PRIMARY KEY) WITHOUT ROWID;
+  INSERT INTO x2 VALUES('2', 'two', 'ii', 1);
+  INSERT INTO x2 VALUES('2.0', 'twopointoh', 'ii.0', 2);
+
+  CREATE TABLE x3(d PRIMARY KEY, a REAL, b, c) WITHOUT ROWID;
+  INSERT INTO x3 VALUES(34, '2', 'two', 'ii');
+  INSERT INTO x3 VALUES(35, '2.0', 'twopointoh', 'ii.0');
+}
+
+foreach {tn idx} {
+  0 {
+  }
+  1 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a=2;
+    CREATE INDEX i2 ON x2(b, c) WHERE a=2;
+    CREATE INDEX i3 ON x3(b, c) WHERE a=2;
+  }
+  2 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a=2.0;
+    CREATE INDEX i2 ON x2(b, c) WHERE a=2.0;
+    CREATE INDEX i3 ON x3(b, c) WHERE a=2.0;
+  }
+  3 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a='2.0';
+    CREATE INDEX i2 ON x2(b, c) WHERE a='2.0';
+    CREATE INDEX i3 ON x3(b, c) WHERE a='2.0';
+  }
+  4 {
+    CREATE INDEX i1 ON x1(b, c) WHERE a='2';
+    CREATE INDEX i2 ON x2(b, c) WHERE a='2';
+    CREATE INDEX i3 ON x3(b, c) WHERE a='2';
+  }
+} {
+  execsql { DROP INDEX IF EXISTS i1 }
+  execsql { DROP INDEX IF EXISTS i2 }
+  execsql { DROP INDEX IF EXISTS i3 }
+
+  execsql $idx
+  do_execsql_test 3.1.$tn.1 {
+    SELECT a, b, c, typeof(a) FROM x1 WHERE a=2
+  } {2 two ii text}
+  do_execsql_test 3.1.$tn.2 {
+    SELECT a, b, c, typeof(a) FROM x1 WHERE a=2.0
+  } {2.0 twopointoh ii.0 text}
+  do_execsql_test 3.1.$tn.3 {
+    SELECT a, b, c, typeof(a) FROM x1 WHERE a='2'
+  } {2 two ii text}
+  do_execsql_test 3.1.$tn.4 {
+    SELECT a, b, c, typeof(a) FROM x1 WHERE a='2.0'
+  } {2.0 twopointoh ii.0 text}
+
+  do_execsql_test 3.1.$tn.5 {
+    SELECT a, b, c, typeof(a) FROM x2 WHERE a=2
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+  do_execsql_test 3.1.$tn.6 {
+    SELECT a, b, c, typeof(a) FROM x2 WHERE a=2.0
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+  do_execsql_test 3.1.$tn.7 {
+    SELECT a, b, c, typeof(a) FROM x2 WHERE a='2'
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+  do_execsql_test 3.1.$tn.8 {
+    SELECT a, b, c, typeof(a) FROM x2 WHERE a='2.0'
+  } {2 two ii integer 2 twopointoh ii.0 integer}
+
+  do_execsql_test 3.1.$tn.9 {
+    SELECT a, b, c, typeof(a) FROM x3 WHERE a=2
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+  do_execsql_test 3.1.$tn.10 {
+    SELECT a, b, c, typeof(a) FROM x3 WHERE a=2.0
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+  do_execsql_test 3.1.$tn.11 {
+    SELECT a, b, c, typeof(a) FROM x3 WHERE a='2'
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+  do_execsql_test 3.1.$tn.12 {
+    SELECT a, b, c, typeof(a) FROM x3 WHERE a='2.0'
+  } {2.0 two ii real 2.0 twopointoh ii.0 real}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t2(a INTEGER, b TEXT);
+  INSERT INTO t2 VALUES(1, 'two');
+  INSERT INTO t2 VALUES(2, 'two');
+  INSERT INTO t2 VALUES(3, 'two');
+  INSERT INTO t2 VALUES(1, 'three');
+  INSERT INTO t2 VALUES(2, 'three');
+  INSERT INTO t2 VALUES(3, 'three');
+
+  CREATE INDEX t2a_two ON t2(a) WHERE b='two';
+}
+
+# explain_i { SELECT sum(a), b FROM t2 WHERE b='two' }
+do_execsql_test 4.1.1 {
+  SELECT sum(a), b FROM t2 WHERE b='two'
+} {6 two}
+do_eqp_test 4.1.2 {
+  SELECT sum(a), b FROM t2 WHERE b='two'
+} {USING COVERING INDEX t2a_two}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE t1(a INTEGER PRIMQRY KEY, b, c);
+}
+do_catchsql_test 5.1 {
+  CREATE INDEX ex1 ON t1(c) WHERE b IS 'abc' COLLATE g;
+} {1 {no such collation sequence: g}}
+
+proc xyz {lhs rhs} {
+  return [string compare $lhs $rhs]
+}
+db collate xyz xyz
+do_execsql_test 5.2 {
+  CREATE INDEX ex1 ON t1(c) WHERE b IS 'abc' COLLATE xyz;
+}
+db close
+sqlite3 db test.db
+do_execsql_test 5.3 {
+  SELECT * FROM t1
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER, z INTEGER);
+  INSERT INTO t1 VALUES(1, 1, 1);
+  INSERT INTO t1 VALUES(2, 1, 2);
+  INSERT INTO t2 VALUES(1, 5, 1);
+  INSERT INTO t2 VALUES(2, 5, 2);
+
+  CREATE INDEX t2z ON t2(z) WHERE y=5;
+}
+
+do_execsql_test 6.1 {
+  ANALYZE;
+  UPDATE sqlite_stat1 SET stat = '50 1' WHERE idx='t2z';
+  UPDATE sqlite_stat1 SET stat = '50' WHERE tbl='t2' AND idx IS NULL;
+  UPDATE sqlite_stat1 SET stat = '5000' WHERE tbl='t1' AND idx IS NULL;
+  ANALYZE sqlite_schema;
+}
+
+do_execsql_test 6.2 {
+  SELECT * FROM t1, t2 WHERE b=1 AND z=c AND y=5;
+} {
+  1 1 1  1 5 1
+  2 1 2  2 5 2
+}
+
+do_eqp_test 6.3 {
+  SELECT * FROM t1, t2 WHERE b=1 AND z=c AND y=5;
+} {BLOOM FILTER ON t2}
+
+do_execsql_test 6.4 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (y=5) WHERE b=1 AND z IS c;
+} {
+  1 1 1  1 5 1
+  2 1 2  2 5 2
+}
+
+do_eqp_test 6.5 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (y=5) WHERE b=1 AND z IS c;
+} {BLOOM FILTER ON t2}
+
+do_execsql_test 6.6 {
+  CREATE INDEX t2yz ON t2(y, z) WHERE y=5;
+}
+
+do_execsql_test 6.7 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (y=5) WHERE b=1 AND z IS c;
+} {
+  1 1 1  1 5 1
+  2 1 2  2 5 2
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE TABLE t1(i INTEGER PRIMARY KEY, b TEXT, c TEXT);
+  CREATE INDEX i1 ON t1(c) WHERE b='abc' AND i=5;
+  INSERT INTO t1 VALUES(5, 'abc', 'xyz');
+  SELECT * FROM t1 INDEXED BY i1 WHERE b='abc' AND i=5 ORDER BY c;
+} {5 abc xyz}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 8.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX ex2 ON t1(a, 4);
+  CREATE INDEX ex1 ON t1(a) WHERE 4=b;
+  INSERT INTO t1 VALUES(1, 4, 1);
+  INSERT INTO t1 VALUES(1, 5, 1);
+  INSERT INTO t1 VALUES(2, 4, 2);
+}
+do_execsql_test 8.1 {
+  SELECT * FROM t1 WHERE b=4;
+} {
+  1 4 1  2 4 2
+}
+
+finish_test
diff --git a/test/indexedby.test b/test/indexedby.test
index 6a371112b3..de4bdaf185 100644
--- a/test/indexedby.test
+++ b/test/indexedby.test
@@ -187,7 +187,7 @@ do_test indexedby-4.4 {
 } {0 {}}
 
 # Test embedding an INDEXED BY in a CREATE VIEW statement. This block
-# also tests that nothing bad happens if an index refered to by
+# also tests that nothing bad happens if an index referred to by
 # a CREATE VIEW statement is dropped and recreated.
 #
 do_execsql_test indexedby-5.1 {
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index 51ef73bbf5..ca6682cda7 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -49,7 +49,7 @@ do_execsql_test indexexpr1-130 {
 do_execsql_test indexexpr1-130eqp {
   EXPLAIN QUERY PLAN
   SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c;
-} {/USING INDEX t1ba/}
+} {/USING COVERING INDEX t1ba/}
 
 do_execsql_test indexexpr1-140 {
   SELECT rowid, substr(a,b,3), '|' FROM t1 ORDER BY 2;
@@ -61,7 +61,7 @@ do_execsql_test indexexpr1-141 {
 do_execsql_test indexexpr1-141eqp {
   EXPLAIN QUERY PLAN
   SELECT rowid FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +rowid;
-} {/USING INDEX t1abx/}
+} {/USING COVERING INDEX t1abx/}
 do_execsql_test indexexpr1-142 {
   SELECT rowid FROM t1 WHERE +substr(a,b,3)<='and' ORDER BY +rowid;
 } {1 2 3}
@@ -73,7 +73,7 @@ do_execsql_test indexexpr1-150eqp {
   EXPLAIN QUERY PLAN
   SELECT rowid FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz')
    ORDER BY +rowid;
-} {/USING INDEX t1abx/}
+} {/USING COVERING INDEX t1abx/}
 
 ifcapable altertable {
   do_execsql_test indexexpr1-160 {
@@ -99,14 +99,14 @@ do_execsql_test indexexpr1-170 {
 do_execsql_test indexexpr1-170eqp {
   EXPLAIN QUERY PLAN
   SELECT length(a) FROM t1 ORDER BY length(a);
-} {/SCAN t1 USING INDEX t1alen/}
+} {/SCAN t1 USING COVERING INDEX t1alen/}
 do_execsql_test indexexpr1-171 {
   SELECT length(a) FROM t1 ORDER BY length(a) DESC;
 } {52 38 29 27 25 20}
 do_execsql_test indexexpr1-171eqp {
   EXPLAIN QUERY PLAN
   SELECT length(a) FROM t1 ORDER BY length(a) DESC;
-} {/SCAN t1 USING INDEX t1alen/}
+} {/SCAN t1 USING COVERING INDEX t1alen/}
 
 do_execsql_test indexexpr1-200 {
   DROP TABLE t1;
@@ -142,7 +142,7 @@ do_execsql_test indexexpr1-230 {
 do_execsql_test indexexpr1-230eqp {
   EXPLAIN QUERY PLAN
   SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c;
-} {/USING INDEX t1ba/}
+} {/USING COVERING INDEX t1ba/}
 
 do_execsql_test indexexpr1-240 {
   SELECT id, substr(a,b,3), '|' FROM t1 ORDER BY 2;
@@ -154,7 +154,7 @@ do_execsql_test indexexpr1-241 {
 do_execsql_test indexexpr1-241eqp {
   EXPLAIN QUERY PLAN
   SELECT id FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +id;
-} {/USING INDEX t1abx/}
+} {/USING COVERING INDEX t1abx/}
 do_execsql_test indexexpr1-242 {
   SELECT id FROM t1 WHERE +substr(a,b,3)<='and' ORDER BY +id;
 } {1 2 3}
@@ -166,7 +166,7 @@ do_execsql_test indexexpr1-250eqp {
   EXPLAIN QUERY PLAN
   SELECT id FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz')
    ORDER BY +id;
-} {/USING INDEX t1abx/}
+} {/USING COVERING INDEX t1abx/}
 
 ifcapable altertable {
   do_execsql_test indexexpr1-260 {
@@ -238,7 +238,7 @@ do_execsql_test indexexpr1-510 {
 do_execsql_test indexexpr1-510eqp {
   EXPLAIN QUERY PLAN
   SELECT substr(a,4,3) AS k FROM cnt, t5 WHERE k=printf('%03d',x);
-} {/USING INDEX t5ax/}
+} {/USING COVERING INDEX t5ax/}
 
 # Skip-scan on an indexed expression
 #
@@ -330,7 +330,7 @@ do_execsql_test indexexpr1-1010 {
 # 2016-10-10
 # Make sure indexes on expressions skip over initial NULL values in the
 # index as they are suppose to do.
-# Ticket https://www.sqlite.org/src/tktview/4baa46491212947
+# Ticket https://sqlite.org/src/tktview/4baa46491212947
 #
 do_execsql_test indexexpr1-1100 {
   DROP TABLE IF EXISTS t1;
@@ -374,7 +374,7 @@ do_execsql_test indexexpr1-1200.4 {
   0 0 0 2 0 4 2 0 2 2 4 0
 }
 
-# Ticket https://www.sqlite.org/src/tktview/eb703ba7b50c1a
+# Ticket https://sqlite.org/src/tktview/eb703ba7b50c1a
 # Incorrect result using an index on an expression with a collating function
 #
 do_execsql_test indexexpr1-1300.1 {
@@ -429,7 +429,7 @@ do_execsql_test indexexpr1-1510 {
   REPLACE INTO t1 SELECT a, randomblob(a) FROM t1
 } {}
 
-# 2018-01-31 https://www.sqlite.org/src/tktview/343634942dd54ab57b702411
+# 2018-01-31 https://sqlite.org/src/tktview/343634942dd54ab57b702411
 # When an index on an expression depends on the string representation of
 # a numeric table column, trouble can arise since there are multiple
 # string that can map to the same numeric value.  (Ex: 123, 0123, 000123).
@@ -449,7 +449,7 @@ do_execsql_test indexexpr1-1620 {
   SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b;
 } {}
 
-# 2019-08-09 https://www.sqlite.org/src/info/9080b6227fabb466
+# 2019-08-09 https://sqlite.org/src/info/9080b6227fabb466
 # ExprImpliesExpr theorem prover bug:
 # "(NULL IS FALSE) IS FALSE" does not imply "NULL IS NULL"
 #
@@ -461,7 +461,7 @@ do_execsql_test indexexpr1-1700 {
   SELECT * FROM t0 WHERE ((NULL IS FALSE) IS FALSE);
 } {0}
 
-# 2019-09-02 https://www.sqlite.org/src/tktview/57af00b6642ecd6848
+# 2019-09-02 https://sqlite.org/src/tktview/57af00b6642ecd6848
 # When the expression of an an index-on-expression references a
 # table column of type REAL that is actually holding an MEM_IntReal
 # value, be sure to use the REAL value and not the INT value when
@@ -547,7 +547,7 @@ do_execsql_test indexexpr1-2030 {
     (3, '{"x":1}',  6,  7);
   CREATE INDEX t1x ON t1(d, a, b->>'x', c);
 }
-do_execsql_test indexexpr1-2030 {
+do_execsql_test indexexpr1-2040 {
   SELECT a,
        SUM(1)                              AS t1,
        SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
@@ -555,7 +555,7 @@ do_execsql_test indexexpr1-2030 {
        SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
     FROM t1;
 } {1 6 4 54 46}
-do_execsql_test indexexpr1-2030 {
+do_execsql_test indexexpr1-2050 {
   explain query plan
   SELECT a,
        SUM(1)                              AS t1,
@@ -563,7 +563,7 @@ do_execsql_test indexexpr1-2030 {
        SUM(c)                              AS t3,
        SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
     FROM t1;
-} {/.*SCAN t1 USING INDEX t1x.*/}
+} {/.*SCAN t1 USING COVERING INDEX t1x.*/}
 
 reset_db
 do_execsql_test indexexpr1-2100 {
@@ -615,5 +615,71 @@ do_execsql_test indexexpr1-2200 {
            GROUP BY t2.type, t1.tag
          ) v ON v.type = 0 AND v.tag = u.tag;
 } {7 100 8 101}
+do_execsql_test indexexpr1-2210 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INT, y TEXT);
+  INSERT INTO t1(x,y) VALUES(1,'{b:5}');
+  SELECT json_insert('{}', '$.a', coalesce(null,json(y)))->>'$.a.b' FROM t1;
+} {5}
+db null NULL
+do_execsql_test indexexpr1-2211 {
+  CREATE INDEX t1j ON t1(coalesce(null,json(y)));
+  SELECT json_insert('{}', '$.a', coalesce(null,json(y)))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2220 {
+  DROP INDEX t1j;
+  SELECT json_insert('{}', '$.a', iif(1,json(y),123))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2221 {
+  CREATE INDEX t1j ON t1(iif(1,json(y),123));
+  SELECT json_insert('{}', '$.a', iif(1,json(y),123))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2230 {
+  DROP INDEX t1j;
+  SELECT json_insert('{}', '$.a', ifnull(NULL,json(y)))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2231 {
+  CREATE INDEX t1j ON t1(ifnull(NULL,json(y)));
+  SELECT json_insert('{}', '$.a', ifnull(NULL,json(y)))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2240 {
+  DROP INDEX t1j;
+  SELECT json_insert('{}', '$.a', nullif(json(y),8))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2241 {
+  CREATE INDEX t1j ON t1(nullif(json(y),8));
+  SELECT json_insert('{}', '$.a', nullif(json(y),8))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2250 {
+  DROP INDEX t1j;
+  SELECT json_insert('{}', '$.a', min('~',json(y)))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2251 {
+  CREATE INDEX t1j ON t1(min('~',json(y)));
+  SELECT json_insert('{}', '$.a', min('~',json(y)))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2260 {
+  DROP INDEX t1j;
+  SELECT json_insert('{}', '$.a', max('...',json(y)))->>'$.a.b' FROM t1;
+} {5}
+do_execsql_test indexexpr1-2261 {
+  CREATE INDEX t1j ON t1(max('...',json(y)));
+  SELECT json_insert('{}', '$.a', max('...',json(y)))->>'$.a.b' FROM t1;
+} {5}
+
+
+# 2023-11-08 Forum post https://sqlite.org/forum/forumpost/68d284c86b082c3e
+#
+# Functions that return subtypes and that are indexed cannot be used to
+# cover function calls from the main table, since the indexed value does
+# not know the subtype.
+#
+reset_db
+do_execsql_test indexexpr1-2300 {
+  CREATE TABLE t1(x INT, y TEXT);
+  INSERT INTO t1(x,y) VALUES(1,'{b:5}');
+  CREATE INDEX t1j ON t1(json(y));
+  SELECT json_insert('{}', '$.a', json(y)) FROM t1;
+} {{{"a":{"b":5}}}}
 
 finish_test
diff --git a/test/indexexpr3.test b/test/indexexpr3.test
new file mode 100644
index 0000000000..76d3331f75
--- /dev/null
+++ b/test/indexexpr3.test
@@ -0,0 +1,121 @@
+# 2024-10-05
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing indexes on expressions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix indexexpr3
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, j);
+  INSERT INTO t1 VALUES(1, '{x:"one"}');
+  INSERT INTO t1 VALUES(2, '{x:"two"}');
+  INSERT INTO t1 VALUES(3, '{x:"three"}');
+
+  CREATE INDEX i1 ON t1( json_extract(j, '$.x') );
+  CREATE INDEX i2 ON t1( a, json_extract(j, '$.x') );
+}
+
+proc do_hasfunction_test {tn sql res} {
+  set nFunction 0
+  db eval "EXPLAIN $sql" x {
+    if {$x(opcode)=="Function"} {
+      incr nFunction 
+    }
+  }
+
+  do_execsql_test $tn "
+    SELECT $nFunction;
+    $sql
+  " $res
+}
+
+do_hasfunction_test 1.1 {
+  SELECT json_extract(j, '$.x') FROM t1 ORDER BY 1;
+} {
+  0 one three two
+}
+
+do_hasfunction_test 1.2 {
+  SELECT json_extract(j, '$.x') FROM t1 WHERE a=2
+} {
+  0 two
+}
+
+do_hasfunction_test 1.3 {
+  SELECT coalesce(json_extract(j, '$.x'), 'five') FROM t1 WHERE a=2
+} {
+  0 two
+}
+
+do_hasfunction_test 1.4 {
+  SELECT json_extract(j, '$.x') || '.two' FROM t1 WHERE a=2
+} {
+  0 two.two
+}
+
+do_hasfunction_test 1.5 {
+  SELECT json_insert( '{}', '$.y', json_extract(j, '$.x') ) FROM t1 WHERE a=2
+} {
+  2 {{"y":"two"}}
+}
+
+do_hasfunction_test 1.6 {
+  SELECT json_insert( '{}', '$.y', coalesce( json_extract(j, '$.x'), 'five' ) )
+  FROM t1 WHERE a=2
+} {
+  2 {{"y":"two"}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b, j);
+  CREATE INDEX i1 ON t1( a, json_extract(j, '$.x') );
+}
+
+do_eqp_test 2.1 {
+  SELECT json_extract(j, '$.x') FROM t1 WHERE a=?
+} {
+  t1 USING COVERING INDEX i1
+}
+
+do_eqp_test 2.2 {
+  SELECT b, json_extract(j, '$.x') FROM t1 WHERE a=?
+} {
+  t1 USING INDEX i1
+}
+
+do_eqp_test 2.3 {
+  SELECT json_insert( '{}', json_extract(j, '$.x') ) FROM t1 WHERE a=?
+} {
+  t1 USING INDEX i1
+}
+
+do_eqp_test 2.4 {
+  SELECT sum( json_extract(j, '$.x') ) FROM t1 WHERE a=?
+} {
+  t1 USING COVERING INDEX i1
+}
+
+do_eqp_test 2.5 {
+  SELECT json_extract(j, '$.x'), sum( json_extract(j, '$.x') ) FROM t1 WHERE a=?
+} {
+  t1 USING INDEX i1
+}
+
+
+
+finish_test
+
diff --git a/test/insert.test b/test/insert.test
index 51e62268db..fd08eb43b8 100644
--- a/test/insert.test
+++ b/test/insert.test
@@ -413,7 +413,7 @@ do_execsql_test insert-11.1 {
 
 
 # More columns of input than there are columns in the table.
-# Ticket http://www.sqlite.org/src/info/e9654505cfda9361
+# Ticket http://sqlite.org/src/info/e9654505cfda9361
 #
 do_execsql_test insert-12.1 {
   CREATE TABLE t12a(a,b,c,d,e,f,g);
@@ -437,7 +437,7 @@ do_execsql_test insert-12.3 {
 # 2018-06-11.  From OSSFuzz.  A column cache malfunction in
 # the constraint checking on an index of expressions causes
 # an assertion fault in a REPLACE.  Ticket
-# https://www.sqlite.org/src/info/c2432ef9089ee73b
+# https://sqlite.org/src/info/c2432ef9089ee73b
 #
 do_execsql_test insert-13.1 {
   DROP TABLE IF EXISTS t13;
@@ -475,7 +475,7 @@ do_execsql_test insert-15.1 {
 } {4 33000}
 
 # 2019-10-16
-# ticket https://www.sqlite.org/src/info/a8a4847a2d96f5de
+# ticket https://sqlite.org/src/info/a8a4847a2d96f5de
 # On a REPLACE INTO, if an AFTER trigger adds back the conflicting
 # row, you can end up with the wrong number of rows in an index.
 #
diff --git a/test/instr.test b/test/instr.test
index d23d66c25c..326f8eb9ee 100644
--- a/test/instr.test
+++ b/test/instr.test
@@ -257,7 +257,7 @@ do_execsql_test instr-1.64 {
   SELECT instr(a, b) FROM x1;
 } 0
 
-# 2019-09-16 ticket https://www.sqlite.org/src/info/587791f92620090e
+# 2019-09-16 ticket https://sqlite.org/src/info/587791f92620090e
 #
 do_execsql_test instr-2.0 {
   DROP TABLE IF EXISTS t0;
diff --git a/test/intpkey.test b/test/intpkey.test
index d6b8833a27..a8d2fb2ffb 100644
--- a/test/intpkey.test
+++ b/test/intpkey.test
@@ -604,7 +604,7 @@ do_test intpkey-15.7 {
   }
 } {}
 
-# 2016-04-18 ticket https://www.sqlite.org/src/tktview/7d7525cb01b68712495d3a
+# 2016-04-18 ticket https://sqlite.org/src/tktview/7d7525cb01b68712495d3a
 # Be sure to escape quoted typenames.
 #
 do_execsql_test intpkey-16.0 {
@@ -614,7 +614,7 @@ do_execsql_test intpkey-16.1 {
   PRAGMA table_info=t16a;
 } {0 id INTEGER 0 {} 1 1 b TEXT 0 {} 0 2 c INT 0 {} 0}
 
-# 2016-05-06 ticket https://www.sqlite.org/src/tktview/16c9801ceba4923939085
+# 2016-05-06 ticket https://sqlite.org/src/tktview/16c9801ceba4923939085
 # When the schema contains an index on the IPK and no other index
 # and a WHERE clause on a delete uses an OR where both sides referencing
 # the IPK, then it is possible that the OP_Delete will fail because there
diff --git a/test/io.test b/test/io.test
index dfadcd1362..1dc84bdee0 100644
--- a/test/io.test
+++ b/test/io.test
@@ -443,7 +443,7 @@ sqlite3_simulate_device -char safe_append
 # on the journal file between steps (2) and (3) above.
 #
 set expected_sync_count 2
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
   ifcapable dirsync {
     incr expected_sync_count
   }
@@ -465,7 +465,7 @@ do_test io-4.2.1 {
   execsql { INSERT INTO abc VALUES('c', 'd') }
   file exists test.db-journal
 } {1}
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(platform) eq "unix"} {
   do_test io-4.2.2 {
     hexio_read test.db-journal 8 4
   } {FFFFFFFF}
diff --git a/test/ioerr.test b/test/ioerr.test
index 29a3dede8a..c76c9c1e6f 100644
--- a/test/ioerr.test
+++ b/test/ioerr.test
@@ -172,7 +172,7 @@ ifcapable crashtest&&attach {
 # These tests can't be run on windows because the windows version of 
 # SQLite holds a mandatory exclusive lock on journal files it has open.
 #
-if {$tcl_platform(platform)!="windows" && ![atomic_batch_write test.db]} {
+if {$tcl_platform(platform) ne"windows" && ![atomic_batch_write test.db]} {
   do_ioerr_test ioerr-7 -tclprep {
     db close
     sqlite3 db2 test2.db
@@ -211,7 +211,7 @@ do_ioerr_test ioerr-8 -ckrefcount true -tclprep {
 
 # For test coverage: Cause an IO error whilst reading the master-journal
 # name from a journal file.
-if {$tcl_platform(platform)=="unix" && [atomic_batch_write test.db]==0} {
+if {$tcl_platform(platform) eq "unix" && [atomic_batch_write test.db]==0} {
   do_ioerr_test ioerr-9 -ckrefcount true -tclprep {
     execsql {
       CREATE TABLE t1(a,b,c);
@@ -225,7 +225,7 @@ if {$tcl_platform(platform)=="unix" && [atomic_batch_write test.db]==0} {
     }
     forcecopy test2.db-journal test.db-journal
     set f [open test.db-journal a]
-    fconfigure $f -encoding binary
+    fconfigure $f -translation binary
     puts -nonewline $f "hello"
     puts -nonewline $f "\x00\x00\x00\x05\x01\x02\x03\x04"
     puts -nonewline $f "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7"
diff --git a/test/ioerr5.test b/test/ioerr5.test
index a430f53407..a0a74bd953 100644
--- a/test/ioerr5.test
+++ b/test/ioerr5.test
@@ -117,7 +117,7 @@ foreach locking_mode {normal exclusive} {
     # Read the contents of the database file into a Tcl variable.
     #
     set fd [open test.db]
-    fconfigure $fd -translation binary -encoding binary
+    fconfigure $fd -translation binary
     set zDatabase [read $fd]
     close $fd
 
@@ -138,7 +138,7 @@ foreach locking_mode {normal exclusive} {
     #
     do_test ioerr5-1.$locking_mode-$iFail.4 {
       set fd [open test.db]
-      fconfigure $fd -translation binary -encoding binary
+      fconfigure $fd -translation binary
       set zDatabase2 [read $fd]
       close $fd
       expr {$zDatabase eq $zDatabase2}
diff --git a/test/join.test b/test/join.test
index aa526aeb29..b33a7560a1 100644
--- a/test/join.test
+++ b/test/join.test
@@ -799,7 +799,7 @@ do_execsql_test join-14.9 {
 } {111 {}}
 
 # Verify the fix to ticket 
-# https://www.sqlite.org/src/tktview/7fde638e94287d2c948cd9389
+# https://sqlite.org/src/tktview/7fde638e94287d2c948cd9389
 #
 db close
 sqlite3 db :memory:
@@ -819,7 +819,7 @@ do_execsql_test join-14.12 {
 } {4 {} {} | 2 2 1 |}
 
 # Verify the fix for ticket
-# https://www.sqlite.org/src/info/892fc34f173e99d8
+# https://sqlite.org/src/info/892fc34f173e99d8
 #
 db close
 sqlite3 db :memory:
@@ -904,7 +904,7 @@ do_execsql_test join-15.110 {
    ORDER BY a1, a2, a3, a4, a5;
 } {1 {} {} {} {} 1 11 {} {} {} 1 12 {} {} {} 1 12 121 {} {} 1 13 {} {} {}}
 
-# 2019-02-05 Ticket https://www.sqlite.org/src/tktview/5948e09b8c415bc45da5c
+# 2019-02-05 Ticket https://sqlite.org/src/tktview/5948e09b8c415bc45da5c
 # Error in join due to the LEFT JOIN strength reduction optimization.
 #
 do_execsql_test join-16.100 {
@@ -1002,6 +1002,21 @@ do_execsql_test join-20.2 {
   SELECT * FROM t0 LEFT JOIN t1 WHERE NULL IN (c1);
 } {}
 
+# 2025-05-29 forum post 7dee41d32506c4ae
+# The complaint in the forum post appears to be the same as for the
+# ticket on 2019-11-02, only for RIGHT JOIN instead of LEFT JOIN.  Note
+# that RIGHT JOIN did not yet exist in SQLite when the ticket was
+# written and fixed.
+#
+do_execsql_test join-20.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INT);      INSERT INTO t1(x) VALUES(1);
+  CREATE TABLE t2(y BOOLEAN);  INSERT INTO t2(y) VALUES(false);
+  CREATE TABLE t3(z INT);      INSERT INTO t3(z) VALUES(3);
+  CREATE INDEX t2y ON t2(y) WHERE y;
+  SELECT quote(z) FROM t1 RIGHT JOIN t2 ON y LEFT JOIN t3 ON y;
+} {NULL}
+
 # 2019-11-30 ticket 7f39060a24b47353
 # Do not allow a WHERE clause term to qualify a partial index on the
 # right table of a LEFT JOIN.
@@ -1289,4 +1304,69 @@ do_execsql_test join-30.3 {
    WHERE x <= y;
 } {}
 
+# 2025-05-30 https://sqlite.org/forum/forumpost/4fc70203b61c7e12
+#
+# When converting a USING(x) or NATURAL into the constraint expression
+# t1.x==t2.x, mark the t1.x term as EP_CanBeNull if it is the left table
+# of a RIGHT JOIN.
+#
+reset_db
+db null NULL
+do_execsql_test join-31.1 {
+  CREATE TABLE t1(c0 INT , c1 INT); INSERT INTO t1(c0, c1) VALUES(NULL,11);
+  CREATE TABLE t2(c0 INT NOT NULL);
+  CREATE TABLE t2n(c0 INT);
+  CREATE TABLE t3(x INT);           INSERT INTO t3(x) VALUES(3);
+  CREATE TABLE t4(y INT);           INSERT INTO t4(y) VALUES(4);
+  CREATE TABLE t5(c0 INT, x INT);   INSERT INTO t5 VALUES(NULL, 5);
+}
+do_execsql_test join-31.2 {
+  SELECT * FROM t2 RIGHT JOIN t3 ON true LEFT JOIN t1 USING(c0);
+} {NULL 3 NULL}
+do_execsql_test join-31.3 {
+  SELECT * FROM t2 RIGHT JOIN t3 ON true NATURAL LEFT JOIN t1;
+} {NULL 3 NULL}
+do_execsql_test join-31.4 {
+  SELECT * FROM t2n RIGHT JOIN t3 ON true LEFT JOIN t1 USING(c0);
+} {NULL 3 NULL}
+do_execsql_test join-31.5 {
+  SELECT * FROM t5 LEFT JOIN t1 USING(c0);
+} {NULL 5 NULL}
+do_execsql_test join-31.6 {
+  SELECT * FROM t3 LEFT JOIN t2 ON true LEFT JOIN t1 USING(c0);
+} {3 NULL NULL}
+do_execsql_test join-31.7 {
+  SELECT * FROM t3 LEFT JOIN t2 ON true NATURAL LEFT JOIN t1;
+} {3 NULL NULL}
+do_execsql_test join-31.8 {
+  SELECT * FROM t3 LEFT JOIN t2 ON true JOIN t4 ON true NATURAL LEFT JOIN t1;
+} {3 NULL 4 NULL}
+
+# 2025-06-16 https://sqlite.org/forum/forumpost/68f29a2005
+#
+# The transitive-constraint optimization was not working for RIGHT JOIN.
+#
+reset_db
+db null NULL
+do_execsql_test join-32.1 {
+  CREATE TABLE t0(w INT);
+  CREATE TABLE t1(x INT);
+  CREATE TABLE t2(y INT UNIQUE);
+  CREATE VIEW v0(z) AS SELECT CAST(x AS INT) FROM t1 LEFT JOIN t2 ON true;
+  INSERT INTO t1(x) VALUES(123);
+  INSERT INTO t2(y) VALUES(NULL);
+}
+do_execsql_test join-32.2 {
+  SELECT *
+  FROM t0 JOIN v0 ON w=z
+          RIGHT JOIN t1 ON true
+          INNER JOIN t2 ON y IS z;
+} {NULL NULL 123 NULL}
+do_execsql_test join-32.3 {
+  SELECT *
+  FROM t0 JOIN v0 ON w=z
+          RIGHT JOIN t1 ON true
+          INNER JOIN t2 ON +y IS z;
+} {NULL NULL 123 NULL}
+
 finish_test
diff --git a/test/join2.test b/test/join2.test
index 15e76f965d..6f2fe1d770 100644
--- a/test/join2.test
+++ b/test/join2.test
@@ -428,4 +428,25 @@ do_eqp_test 12.3 {
   `--SEARCH t1 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN
 }
 
+# 2024-09-05 https://sqlite.org/forum/forumpost/8a1e467e905b8d27
+# When performing the Omit-Noop-Join optimization, if FROM clause terms
+# to the right of the omitted join have the reverse-order bit set in the
+# WhereInfo.revMask bitmask, those bits need to be shifted to account
+# for the omitted join.
+#
+reset_db
+do_execsql_test 13.0 {
+  CREATE TABLE t1(a1 INTEGER PRIMARY KEY, b1 INT);
+  CREATE TABLE t2(c2 INT, d2 INTEGER PRIMARY KEY);
+  CREATE TABLE t3(e3 INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(33,0);
+  INSERT INTO t2 VALUES(33,1),(33,2);
+}
+do_execsql_test 13.1 {
+  SELECT t1.a1, t2.d2
+    FROM (t1 LEFT JOIN t3 ON t3.e3=t1.b1) JOIN t2 ON t2.c2=t1.a1
+   WHERE t1.a1=33
+   ORDER BY t2.d2 DESC;
+} {33 2 33 1}
+
 finish_test
diff --git a/test/join5.test b/test/join5.test
index 85e09c57d6..eb8ba3c7b5 100644
--- a/test/join5.test
+++ b/test/join5.test
@@ -106,7 +106,7 @@ do_test join5-2.12 {
   execsql {SELECT * FROM xy LEFT JOIN ab ON NULL WHERE NULL}
 } {}
 
-# Ticket https://www.sqlite.org/src/tktview/6f2222d550f5b0ee7ed37601
+# Ticket https://sqlite.org/src/tktview/6f2222d550f5b0ee7ed37601
 # Incorrect output on a LEFT JOIN.
 #
 do_execsql_test join5-3.1 {
@@ -161,7 +161,7 @@ do_execsql_test join5-3.3 {
   SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b;
 } {}
 
-# Ticket https://www.sqlite.org/src/tktview/c2a19d81652f40568c770c43 on
+# Ticket https://sqlite.org/src/tktview/c2a19d81652f40568c770c43 on
 # 2015-08-20.  LEFT JOIN and the push-down optimization.
 #
 do_execsql_test join5-4.1 {
@@ -302,14 +302,34 @@ do_execsql_test 7.3 {
   ANALYZE sqlite_schema;
 }
 
+# If both sides of the OR reference the right-hand side of the LEFT JOIN
+# then simplify the LEFT JOIN.
+#
 do_eqp_test 7.4 {
   SELECT * FROM t3 LEFT JOIN t4 ON (t4.x = t3.x) WHERE (t4.y = ? OR t4.z = ?);
+} {
+  QUERY PLAN
+  |--SCAN t4
+  `--SEARCH t3 USING COVERING INDEX t3x (x=?)
+} 
+# If only one side of the OR references the right-hand side of the LEFT JOIN
+# then do not do the simplification
+#
+do_eqp_test 7.4b {
+  SELECT * FROM t3 LEFT JOIN t4 ON (t4.x = t3.x) WHERE (t4.y = ? OR t3.x = ?);
 } {
   QUERY PLAN
   |--SCAN t3
   `--SEARCH t4 USING INDEX t4xz (x=?) LEFT-JOIN
 } 
-do_eqp_test 7.4b {
+do_eqp_test 7.4c {
+  SELECT * FROM t3 LEFT JOIN t4 ON (t4.x = t3.x) WHERE (t3.x = ? OR t4.z = ?);
+} {
+  QUERY PLAN
+  |--SCAN t3
+  `--SEARCH t4 USING INDEX t4xz (x=?) LEFT-JOIN
+} 
+do_eqp_test 7.4d {
   SELECT * FROM t3 CROSS JOIN t4 ON (t4.x = t3.x) WHERE (+t4.y = ? OR t4.z = ?);
 } {
   QUERY PLAN
@@ -350,8 +370,12 @@ do_execsql_test 9.1 {
   ANALYZE sqlite_schema;
   INSERT INTO sqlite_stat1 VALUES('t1','t1x1','648 324 81 81 81 81 81 81 81081 81 81 81');
   ANALYZE sqlite_schema;
-  SELECT a FROM (SELECT a FROM t1 NATURAL LEFT JOIN t1) NATURAL LEFT JOIN t1 WHERE (rowid,1)<=(5,0);
-} {1}
+}
+do_catchsql_test 9.2 {
+  SELECT a FROM 
+      (SELECT a FROM t1 NATURAL LEFT JOIN t1) NATURAL LEFT JOIN t1 
+  WHERE (rowid,1)<=(5,0);
+} {0 1}
 
 # 2022-03-02 https://sqlite.org/forum/info/50a1bbe08ce4c29c
 # Bloom-filter pulldown is incompatible with skip-scan.
diff --git a/test/joinA.test b/test/joinA.test
index d6bb678c54..04d1e68cfd 100644
--- a/test/joinA.test
+++ b/test/joinA.test
@@ -210,5 +210,68 @@ foreach {id schema} {
     18  28  38  48  -   -   - 
     19  -   29  39  -   49  - 
   }
+
+  # Verified by PG-14
+  do_execsql_test joinA-$id.201 {
+    SELECT a,b,c,d,t2.e,f,t3.e,t1.a
+      FROM t1
+           FULL JOIN t2 USING(c,d)
+           FULL JOIN t3 USING(a,b,f)
+           FULL JOIN t4 USING(a,c,d,f)
+     WHERE t1.a!=0
+    ORDER BY 1 nulls first, 3 nulls first;
+  } {
+    11  21  31  41  -   -   -  11
+    12  22  32  42  -   -   -  12
+    15  25  35  45  -   -   -  15
+    18  28  38  48  -   -   -  18
+  }
+
+  # Verified by PG-14
+  do_execsql_test joinA-$id.202 {
+    SELECT a,b,c,d,t2.e,f,t3.e,t3.a
+      FROM t1
+           FULL JOIN t2 USING(c,d)
+           FULL JOIN t3 USING(a,b,f)
+           FULL JOIN t4 USING(a,c,d,f)
+     WHERE t3.a!=0
+    ORDER BY 1 nulls first, 3 nulls first;
+  } {
+    14  24  -   -   -   44  34  14
+    15  25  -   -   -   45  35  15
+    16  26  -   -   -   46  36  16
+  }
+
+  # Verified by PG-14
+  do_execsql_test joinA-$id.203 {
+    SELECT a,b,c,d,t2.e,f,t3.e,t4.a
+      FROM t1
+           FULL JOIN t2 USING(c,d)
+           FULL JOIN t3 USING(a,b,f)
+           FULL JOIN t4 USING(a,c,d,f)
+     WHERE t4.a!=0
+    ORDER BY 1 nulls first, 3 nulls first;
+  } {
+    11  -   21  31  -   41  -  11
+    13  -   23  33  -   43  -  13
+    16  -   26  36  -   46  -  16
+    19  -   29  39  -   49  -  19
+  }
+
+  # Verified by PG-14
+  do_execsql_test joinA-$id.204 {
+    SELECT a,b,c,d,t2.e,f,t3.e
+      FROM t1
+           FULL JOIN t2 USING(c,d)
+           FULL JOIN t3 USING(a,b,f)
+           FULL JOIN t4 USING(a,c,d,f)
+     WHERE t2.e!=0
+    ORDER BY 1 nulls first, 3 nulls first;
+  } {
+    -   -   12  22  32  42  - 
+    -   -   13  23  33  43  - 
+    -   -   15  25  35  45  - 
+    -   -   17  27  37  47  - 
+  }
 }
 finish_test
diff --git a/test/joinH.test b/test/joinH.test
index 78d1556293..56fa9c7ec3 100644
--- a/test/joinH.test
+++ b/test/joinH.test
@@ -120,4 +120,358 @@ do_execsql_test 5.5 {
   SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0)
 } {}
 
+
+reset_db
+db null NULL
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a INT);
+  CREATE TABLE t2(b INT);
+  INSERT INTO t1 VALUES(3);
+  SELECT CASE WHEN t2.b THEN 0 ELSE 1 END FROM t1 LEFT JOIN t2 ON true;
+} {1}
+do_execsql_test 6.1 {
+  SELECT * FROM t1 LEFT JOIN t2 ON true WHERE CASE WHEN t2.b THEN 0 ELSE 1 END;
+} {3 NULL}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c);
+  CREATE TABLE t3(d);
+
+  INSERT INTO t1 VALUES ('a', 'a');
+  INSERT INTO t2 VALUES ('ddd');
+  INSERT INTO t3 VALUES(1234);
+}
+
+do_execsql_test 7.1 {
+  SELECT t2.rowid FROM t1 JOIN (t2 JOIN t3);
+} {1}
+
+do_execsql_test 7.1 {
+  UPDATE t1 SET b = t2.rowid FROM t2, t3;
+}
+
+do_execsql_test 7.2 { 
+  SELECT * FROM t1
+} {a 1}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 8.0 {
+  CREATE TABLE x1(a INTEGER PRIMARY KEY, b);
+  CREATE TABLE x2(c, d);
+  CREATE TABLE x3(rowid, _rowid_);
+
+  CREATE TABLE x4(rowid, _rowid_, oid);
+
+  INSERT INTO x1 VALUES(1000, 'thousand');
+  INSERT INTO x2 VALUES('c', 'd');
+  INSERT INTO x3(oid, rowid, _rowid_) VALUES(43, 'hello', 'world');
+  INSERT INTO x4(oid, rowid, _rowid_) VALUES('forty three', 'hello', 'world');
+}
+
+do_execsql_test 8.1 {
+  SELECT x3.oid FROM x1 JOIN (x2 JOIN x3 ON c='c')
+} 43
+
+breakpoint
+do_execsql_test 8.2 {
+  SELECT x3.rowid FROM x1 JOIN (x2 JOIN x3 ON c='c')
+} {hello}
+
+do_execsql_test 8.3 {
+  SELECT x4.oid FROM x1 JOIN (x2 JOIN x4 ON c='c')
+} {{forty three}}
+
+
+#---------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 9.0 {
+  CREATE TABLE x1(a);
+  CREATE TABLE x2(b);
+  CREATE TABLE x3(c);
+
+  CREATE TABLE wo1(a PRIMARY KEY, b) WITHOUT ROWID;
+  CREATE TABLE wo2(a PRIMARY KEY, rowid) WITHOUT ROWID;
+  CREATE TABLE wo3(a PRIMARY KEY, b) WITHOUT ROWID;
+}
+
+do_catchsql_test 9.1 {
+  SELECT rowid FROM wo1, x1, x2;
+} {1 {ambiguous column name: rowid}}
+do_catchsql_test 9.2 {
+  SELECT rowid FROM wo1, (x1, x2);
+} {1 {ambiguous column name: rowid}}
+do_catchsql_test 9.3 {
+  SELECT rowid FROM wo1 JOIN (x1 JOIN x2);
+} {1 {ambiguous column name: rowid}}
+do_catchsql_test 9.4 {
+  SELECT a FROM wo1, x1, x2;
+} {1 {ambiguous column name: a}}
+
+
+# It is not possible to use "rowid" in a USING clause.
+#
+do_catchsql_test 9.5 {
+  SELECT * FROM x1 JOIN x2 USING (rowid);
+} {1 {cannot join using column rowid - column not present in both tables}}
+do_catchsql_test 9.6 {
+  SELECT * FROM wo2 JOIN x2 USING (rowid);
+} {1 {cannot join using column rowid - column not present in both tables}}
+
+# "rowid" columns are not matched by NATURAL JOIN. If they were, then
+# the SELECT below would return zero rows.
+do_execsql_test 9.7 {
+  INSERT INTO x1(rowid, a) VALUES(101, 'A');
+  INSERT INTO x2(rowid, b) VALUES(55, 'B');
+  SELECT * FROM x1 NATURAL JOIN x2;
+} {A B}
+
+do_execsql_test 9.8 {
+  INSERT INTO wo1(a, b) VALUES('mya', 'myb');
+  INSERT INTO wo2(a, rowid) VALUES('mypk', 'myrowid');
+  INSERT INTO wo3(a, b) VALUES('MYA', 'MYB');
+  INSERT INTO x3(rowid, c) VALUES(99, 'x3B');
+}
+
+do_catchsql_test 9.8 {
+  SELECT rowid FROM x1 JOIN (x2 JOIN wo2);
+} {0 myrowid}
+do_catchsql_test 9.9 {
+  SELECT _rowid_ FROM wo1 JOIN (wo3 JOIN x3)
+} {0 99}
+do_catchsql_test 9.10 {
+  SELECT oid FROM wo1 JOIN (wo3 JOIN x3)
+} {0 99}
+do_catchsql_test 9.11 {
+  SELECT oid FROM wo2 JOIN (wo3 JOIN x3)
+} {0 99}
+
+reset_db
+do_execsql_test 10.0 {
+  CREATE TABLE rt0 (c0 INTEGER, c1 INTEGER, c2 INTEGER, c3 INTEGER, c4 INTEGER);
+  CREATE TABLE rt3 (c3 INTEGER);
+
+  INSERT INTO rt0(c3, c1) VALUES (x'', '1');
+  INSERT INTO rt0(c3, c1) VALUES ('-1', -1e500);
+  INSERT INTO rt0(c3, c1) VALUES (1, x'');
+
+  CREATE VIEW v6(c0, c1, c2) AS SELECT 0, 0, 0;
+}
+
+do_execsql_test 10.1 {
+  SELECT COUNT(*) FROM rt0 LEFT JOIN rt3 JOIN v6 ON ((CASE v6.c0 WHEN rt0.c4 THEN rt3.c3 END) NOT BETWEEN (rt0.c4) AND (NULL)) WHERE (rt0.c1); -- 2
+} {0}
+
+do_execsql_test 10.2 {
+  SELECT COUNT(*) FROM rt0 LEFT JOIN rt3 RIGHT OUTER JOIN v6 ON ((CASE v6.c0 WHEN rt0.c4 THEN rt3.c3 END) NOT BETWEEN (rt0.c4) AND (NULL)) WHERE (rt0.c1); -- 2
+} {0}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 11.1 {
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+  CREATE TABLE t3(e, f);
+
+  INSERT INTO t1 VALUES(1, 1);
+  INSERT INTO t2 VALUES(2, 2);
+  INSERT INTO t3 VALUES(3, 3);
+}
+
+do_execsql_test 11.2 {
+  SELECT * FROM t1 LEFT JOIN t2 RIGHT JOIN t3 ON (t2.c=10)
+} {{} {} {} {} 3 3}
+
+do_execsql_test 11.3 {
+  SELECT * FROM t1 LEFT JOIN t2 RIGHT JOIN t3 ON (t2.c=10) WHERE t1.a=1
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 12.1 {
+  CREATE TABLE t1(a1 INT, b1 TEXT);
+  INSERT INTO t1 VALUES(88,'');
+  CREATE TABLE t2(c2 INT, d2 TEXT);
+  INSERT INTO t2 VALUES(88,'');
+  CREATE TABLE t3(e3 TEXT PRIMARY KEY);
+  INSERT INTO t3 VALUES('');
+}
+
+do_execsql_test 12.2 {
+  SELECT * FROM t1 LEFT JOIN t2 ON true RIGHT JOIN t3 ON d2=e3 WHERE c2 BETWEEN NULL AND a1;
+}
+do_execsql_test 12.3 {
+  SELECT * FROM t1 LEFT JOIN t2 ON true RIGHT JOIN t3 ON d2=e3 WHERE c2 BETWEEN NULL AND a1;
+}
+
+#-------------------------------------------------------------------------
+# 2024-04-05 dbsqlfuzz b9e65e2f110df998f1306571fae7af6c01e4d92b
+reset_db
+do_execsql_test 13.1 {
+  CREATE TABLE t1(a INT AS (b), b INT);
+  INSERT INTO t1(b) VALUES(123);
+  CREATE TABLE t2(a INT, c INT);
+  SELECT a FROM t2 NATURAL RIGHT JOIN t1;
+} {123}
+do_execsql_test 13.2 {
+  CREATE INDEX t1a ON t1(a);
+  SELECT a FROM t2 NATURAL RIGHT JOIN t1;
+} {123}
+# Further tests of the same logic (indexes on expressions
+# used by RIGHT JOIN) from check-in ffe23af73fcb324d and
+# forum post https://sqlite.org/forum/forumpost/9b491e1debf0b67a.
+db null NULL
+do_execsql_test 13.3 {
+  CREATE TABLE t3(a INT, b INT);
+  CREATE UNIQUE INDEX t3x ON t3(a, a+b);
+  INSERT INTO t3(a,b) VALUES(1,2),(4,8),(16,32),(4,80),(1,-300);
+  CREATE TABLE t4(x INT, y INT);
+  INSERT INTO t4(x,y) SELECT a, b FROM t3;
+  INSERT INTO t4(x,y) VALUES(99,99);
+  SELECT a1.a, sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t4 ON a=x
+   GROUP BY a1.a ORDER BY 1;
+} {NULL NULL 1 -592 4 192 16 48}
+do_execsql_test 13.4 {
+  SELECT sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t3 ON true
+   GROUP BY a1.a ORDER BY 1;
+} {-1480 240 480}
+
+#-------------------------------------------------------------------------
+# 2025-05-30
+# https://sqlite.org/forum/forumpost/5028c785b6
+#
+reset_db
+
+do_execsql_test 14.0 {
+  CREATE TABLE t1(c0 INT);
+  CREATE TABLE t2(c0 BLOB);
+  CREATE TABLE t3(c0 BLOB);
+  CREATE TABLE t4(c4 BLOB);
+  INSERT INTO t1(c0) VALUES(0);
+  INSERT INTO t3(c0) VALUES('0');
+}
+
+do_execsql_test 14.1.1 {
+  SELECT * FROM t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3;
+} {0}
+
+do_execsql_test 14.1.2 {
+  SELECT * FROM t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3 FULL JOIN t4 ON true;
+} {0 {}}
+
+do_execsql_test 14.1.3 {
+  SELECT * FROM (t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3) FULL JOIN t4 ON true;
+} {0 {}}
+
+do_execsql_test 14.1.4 {
+  SELECT * 
+  FROM (t1 NATURAL LEFT JOIN t2 NATURAL JOIN t3) AS qq FULL JOIN t4 ON true;
+} {0 {}}
+
+do_execsql_test 14.2.1 {
+  SELECT * FROM t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1;
+} {0}
+
+do_execsql_test 14.2.2 {
+  SELECT * FROM t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1 FULL JOIN t4 ON true;
+} {0 {}}
+
+do_execsql_test 14.2.3 {
+  SELECT * FROM (t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1) FULL JOIN t4 ON true;
+} {0 {}}
+
+do_execsql_test 14.2.4 {
+  SELECT * 
+  FROM (t3 NATURAL LEFT JOIN t2 NATURAL JOIN t1) AS qq FULL JOIN t4 ON true;
+} {0 {}}
+
+# 2025-06-01 
+#
+reset_db
+do_execsql_test 15.1 {
+  CREATE TABLE t0(c0);
+  CREATE TABLE t1(c0);
+  CREATE TABLE t2(c0);
+  INSERT INTO t0 VALUES ('1.0');
+  INSERT INTO t2(c0) VALUES (9);
+  SELECT t0.c0,t2.c0 FROM (SELECT CAST(t0.c0 as REAL) AS c0 FROM t0) as subquery NATURAL LEFT JOIN t1  NATURAL JOIN t0  RIGHT JOIN t2 ON 1;
+} {1.0 9}
+do_execsql_test 15.2 {
+  CREATE TABLE x1(x COLLATE nocase);
+  CREATE TABLE x2(x);
+  CREATE TABLE x3(x);
+  CREATE TABLE t4(y);
+  INSERT INTO x1 VALUES('ABC');
+  INSERT INTO x3 VALUES('abc');
+  SELECT lower(x), quote(y) FROM x1 LEFT JOIN x2 USING (x) JOIN x3 USING (x) FULL JOIN t4;
+} {abc NULL}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 16.0 {
+  CREATE TABLE t0_a (c0 INT);
+  CREATE TABLE t0_b (c0 INT);
+
+  CREATE TABLE t2 (c0 INT);
+  INSERT INTO t2 VALUES (1);
+}
+
+do_execsql_test 16.1 {
+  SELECT * FROM t2 LEFT JOIN t0_b
+} {1 {}}
+
+do_execsql_test 16.2.1 {
+  SELECT * FROM (t0_a RIGHT JOIN (              t2 LEFT JOIN t0_b))
+} {{} 1 {}}
+do_execsql_test 16.2.2 {
+  SELECT * FROM (t0_a RIGHT JOIN (SELECT * FROM t2 LEFT JOIN t0_b))
+} {{} 1 {}}
+
+  
+do_catchsql_test 16.3.1 {
+  SELECT * FROM (t0_a RIGHT JOIN (              t2 LEFT JOIN t0_b) USING (c0));
+} {1 {ambiguous column name: c0}}
+
+do_execsql_test 16.3.2 {
+  SELECT * FROM (t0_a RIGHT JOIN (SELECT * FROM t2 LEFT JOIN t0_b) USING (c0));
+} {1 {}}
+
+do_execsql_test 16.4.0 {
+  CREATE TABLE x0(a TEXT);
+  CREATE TABLE x1(a TEXT);
+  CREATE TABLE x2(a TEXT);
+
+  INSERT INTO x1 VALUES('blue');
+  INSERT INTO x2 VALUES('red');
+}
+
+do_catchsql_test 16.4.1 {
+  SELECT * FROM x0 RIGHT JOIN (              x1, x2) USING (a)
+} {1 {ambiguous column name: a}}
+
+do_execsql_test 16.4.2 {
+  SELECT * FROM x0 RIGHT JOIN (SELECT * FROM x1, x2) USING (a)
+} {blue red}
+
+reset_db
+do_execsql_test 16.5.0 {
+  CREATE TABLE t0 (c0 INT);
+  CREATE TABLE t1 (c0 INT);
+  CREATE TABLE t2 (c0 INT);
+
+  INSERT INTO t1(c0) VALUES (NULL);
+  INSERT INTO t2 VALUES (1);
+
+  CREATE VIEW v0(c0) AS SELECT 1 AS col_0 FROM t0;
+}
+
+do_catchsql_test 16.5.2 {
+  SELECT * FROM (t0 NATURAL RIGHT JOIN (t0 FULL JOIN (v0 NATURAL FULL JOIN t2) ON TRUE)) NATURAL FULL JOIN t1;
+} {1 {ambiguous column name: c0}}
+
 finish_test
diff --git a/test/joinI.test b/test/joinI.test
new file mode 100644
index 0000000000..577ca4c2c3
--- /dev/null
+++ b/test/joinI.test
@@ -0,0 +1,125 @@
+# 2022 May 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix joinI
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INT);
+  CREATE TABLE t2(b INT);
+  CREATE TABLE t3(c INT);
+}
+
+foreach {tn sql} {
+  1 "SELECT * FROM t1 RIGHT JOIN t2 ON t2.b=t3.c CROSS JOIN t3"
+  2 "SELECT * FROM t1 RIGHT JOIN t2 ON t2.b=(SELECT t3.c) CROSS JOIN t3"
+  3 "SELECT * FROM t1 RIGHT JOIN t2 ON CASE WHEN t2.b THEN t3.c ELSE 1 END CROSS JOIN t3"
+} {
+  do_catchsql_test 1.1.$tn $sql {1 {ON clause references tables to its right}}
+}
+
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 2.0 {
+  CREATE TABLE t0(c0 INT, c1 INT);
+  CREATE TABLE t1 (c0 INT);
+
+  CREATE VIEW v1(c0) AS SELECT t0.c0 FROM t0 NATURAL RIGHT JOIN t1;
+  CREATE VIEW v2(c0) AS SELECT 0 FROM v1;
+
+  INSERT INTO t0(c0, c1) VALUES (-1, 0);
+  INSERT INTO t1(c0) VALUES (NULL);
+
+  SELECT * FROM v1 INNER JOIN (v2 CROSS JOIN t0) ON (t0.c0 < t0.c1);
+} {{} 0 -1 0}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t0(c0, c1);
+  CREATE TABLE t1(v);
+  CREATE TABLE t2(w);
+  CREATE TABLE t3(x);
+  CREATE TABLE t4(y);
+  CREATE TABLE t5(z);
+}
+
+do_execsql_test 3.1 {
+  SELECT 1234 FROM t4 
+  RIGHT JOIN t5 
+  CROSS JOIN (t2 CROSS JOIN t0) AS a1 ON (a1.c0 < a1.c1);
+}
+
+do_execsql_test 3.2 {
+  SELECT 1234 FROM t4 
+  RIGHT JOIN t5 
+  CROSS JOIN (t2 CROSS JOIN t1 CROSS JOIN t0) AS a1 ON (a1.c0 < a1.c1);
+}
+
+do_execsql_test 3.3 {
+  SELECT 5678 FROM t0 RIGHT JOIN t1 ON (
+    SELECT 1 FROM t2 RIGHT JOIN t3 ON t2.w
+  ) CROSS JOIN t4;
+}
+
+do_catchsql_test 3.4 {
+  SELECT 5678 FROM t0 RIGHT JOIN t1 ON (
+    SELECT 1 FROM t2 RIGHT JOIN t3 ON t4.y
+  ) CROSS JOIN t4;
+} {1 {ON clause references tables to its right}}
+
+do_execsql_test 3.5 {
+  SELECT 5678 FROM t0 RIGHT JOIN t1 ON (
+    SELECT 1 FROM t2 RIGHT JOIN t3 ON 0=t1.v
+  ) CROSS JOIN t4;
+}
+
+do_catchsql_test 3.6 {
+  SELECT 5678 FROM t0 RIGHT JOIN t1 ON (
+    SELECT 1 FROM t2 RIGHT JOIN t3 ON max(0,t5.z) CROSS JOIN t5
+  ) CROSS JOIN t4;
+} {1 {ON clause references tables to its right}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE t1(a);
+  CREATE TABLE t2(b);
+  CREATE TABLE t3(c, d);
+}
+
+do_catchsql_test 4.1 {
+  SELECT c+d AS cd FROM t1 LEFT JOIN t2 ON (cd=5) CROSS JOIN t3;
+} {1 {ON clause references tables to its right}}
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE parent1(parent1key, child1key, Child2key, child3key);
+  CREATE TABLE child1 ( child1key NVARCHAR, value NVARCHAR );
+  CREATE TABLE child2 ( child2key NVARCHAR, value NVARCHAR );
+}
+
+do_execsql_test 5.1 {
+  SELECT parent1.parent1key, child1.value, child2.value
+    FROM parent1
+    LEFT OUTER JOIN child1 ON child1.child1key = parent1.child1key
+    INNER JOIN child2 ON child2.child2key = parent1.child2key;
+}
+
+finish_test
+
diff --git a/test/journal1.test b/test/journal1.test
index bcbafe30f6..56c862f055 100644
--- a/test/journal1.test
+++ b/test/journal1.test
@@ -25,7 +25,7 @@ source $testdir/tester.tcl
 # Or with atomic_batch_write systems, as journal files are
 # not created.
 #
-if {$tcl_platform(platform)=="windows"
+if {$tcl_platform(platform) eq "windows"
  || [atomic_batch_write test.db]
 } {
   finish_test
diff --git a/test/journal3.test b/test/journal3.test
index c3e3d12db6..a29b68d54a 100644
--- a/test/journal3.test
+++ b/test/journal3.test
@@ -20,7 +20,7 @@ source $testdir/malloc_common.tcl
 # If a connection is required to create a journal file, it creates it with 
 # the same file-system permissions as the database file itself. Test this.
 #
-if {$::tcl_platform(platform) == "unix"
+if {$::tcl_platform(os) ne "Windows NT"
  && [atomic_batch_write test.db]==0
 } {
 
diff --git a/test/json/README.md b/test/json/README.md
index 6a16114925..4ebbda6d3f 100644
--- a/test/json/README.md
+++ b/test/json/README.md
@@ -1,27 +1,66 @@
 The files in this subdirectory are used to help measure the performance
-of the SQLite JSON parser.
+of the SQLite JSON functions, especially in relation to handling large
+JSON inputs.
 
 # 1.0 Prerequisites
 
-  1.  Valgrind
+  *   Standard SQLite build environment (SQLite source tree, compiler, make, etc.)
 
-  2.  Fossil
+  *   Valgrind
+
+  *   Fossil (only the "fossil xdiff" command is used by this procedure)
+
+  *   tclsh
 
 # 2.0 Setup
 
-  1.  Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create
+  *   Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create
       the 100 megabyte test database.  Do this so that the "json100mb.db"
       file lands in the directory from which you will run tests, not in
       the test/json subdirectory of the source tree.
 
-  2.  Build the baseline sqlite3.c file.  ("`make sqlite3.c`")
+  *   Make a copy of "json100mb.db" into "jsonb100mb.db" - change the prefix
+      from "json" to "jsonb".
+
+  *   Bring up jsonb100mb.db in the sqlite3 command-line shell.
+      Convert all of the content into JSONB using a commands like this:
+
+>        UPDATE data1 SET x=jsonb(x);
+>        VACUUM;
+
+  *   Build the baseline sqlite3.c file with sqlite3.h and shell.c.
+
+>        make clean sqlite3.c
 
-  3.  Run "`sh json-speed-check-1.sh trunk`".   This creates the baseline
-      profile in "jout-trunk.txt".
+  *   Run "`sh json-speed-check.sh trunk`".   This creates the baseline
+      profile in "jout-trunk.txt" for the preformance test using text JSON.
+
+  *   Run "`sh json-speed-check.sh trunk --jsonb`".  This creates the
+      baseline profile in "joutb-trunk.txt" for the performance test
+      for processing JSONB
+
+  *   (Optional) Verify that the json100mb.db database really does contain
+      approximately 100MB of JSON content by running:
+
+>        SELECT sum(length(x)) FROM data1;
+>        SELECT * FROM data1 WHERE NOT json_valid(x);
 
 # 3.0 Testing
 
-  1.  Build the sqlite3.c to be tested.
+  *   Build the sqlite3.c (with sqlite3.h and shell.c) to be tested.
 
-  2.  Run "`sh json-speed-check-1.sh x1`".  The profile output will appear
+  *   Run "`sh json-speed-check.sh x1`".  The profile output will appear
       in jout-x1.txt.  Substitute any label you want in place of "x1".
+
+  *   Run "`sh json-speed-check.sh x1 --jsonb`".  The profile output will appear
+      in joutb-x1.txt.  Substitute any label you want in place of "x1".
+
+  *   Run the script shown below in the CLI.
+      Divide 2500 by the real elapse time from this test
+      to get an estimate for number of MB/s that the JSON parser is
+      able to process.
+
+>        .open json100mb.db
+>        .timer on
+>        WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<25)
+>        SELECT sum(json_valid(x)) FROM c, data1;
diff --git a/test/json/json-q1.txt b/test/json/json-q1.txt
index 0395f0c061..d122a2d826 100644
--- a/test/json/json-q1.txt
+++ b/test/json/json-q1.txt
@@ -2,3 +2,23 @@
 .timer on
 .param set $label 'q87'
 SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;
+
+CREATE TEMP TABLE t2(x JSON TEXT);
+WITH RECURSIVE
+  c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000),
+  array1(y) AS (
+    SELECT json_group_array(
+             json_object('x',x,'y',random(),'z',hex(randomblob(50)))
+           )
+      FROM c
+  ),
+  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
+INSERT INTO t2(x)
+  SELECT json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
+CREATE INDEX t2x1 ON t2(x->>'a');
+CREATE INDEX t2x2 ON t2(x->>'b');
+CREATE INDEX t2x3 ON t2(x->>'e');
+CREATE INDEX t2x4 ON t2(x->>'f');
+UPDATE t2 SET x=json_replace(x,'$.f',(x->>'f')+1);
+UPDATE t2 SET x=json_set(x,'$.e',(x->>'f')-1);
+UPDATE t2 SET x=json_remove(x,'$.d');
diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh
index f7e7f1be58..1eabbb3db2 100755
--- a/test/json/json-speed-check.sh
+++ b/test/json/json-speed-check.sh
@@ -5,7 +5,7 @@
 #
 #     sh speed-check.sh trunk  #  Baseline measurement of trunk
 #     sh speed-check.sh x1     # Measure some experimental change
-#     fossil xdiff --tk jout-trunk.txt jout-x1.txt   # View chanages
+#     fossil xdiff --tk jout-trunk.txt jout-x1.txt   # View changes
 #
 # There are multiple output files, all with a base name given by
 # the first argument:
@@ -35,11 +35,13 @@ LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE"
 LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
 BASELINE="trunk"
+TYPE="json"
 doExplain=0
 doCachegrind=1
 doVdbeProfile=0
 doWal=1
 doDiff=1
+doJsonB=0
 while test "$1" != ""; do
   case $1 in
     --nodiff)
@@ -54,6 +56,10 @@ while test "$1" != ""; do
     --gcc7)
         CC=gcc-7
         ;;
+    --jsonb)
+        doJsonB=1
+        TYPE="jsonb"
+        ;;
     -*)
         CC_OPTS="$CC_OPTS $1"
         ;;
@@ -69,12 +75,13 @@ rm -f cachegrind.out.* jsonshell
 $CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread
 ls -l jsonshell | tee -a summary-$NAME.txt
 home=`echo $0 | sed -e 's,/[^/]*$,,'`
-echo ./jsonshell json100mb.db "<$home/json-q1.txt"
-valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \
-      2>&1 | tee -a summary-$NAME.txt
-cg_anno.tcl cachegrind.out.* >jout-$NAME.txt
-echo '*****************************************************' >>jout-$NAME.txt
-sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>jout-$NAME.txt
-if test "$NAME" != "$BASELINE"; then
-  fossil xdiff --tk -c 20 jout-$BASELINE.txt jout-$NAME.txt
+DB=$TYPE''100mb.db
+echo ./jsonshell $DB "<$home/$TYPE-q1.txt"
+valgrind --tool=cachegrind ./jsonshell json100mb_b.db <$home/$TYPE-q1.txt \
+        2>&1 | tee -a summary-$NAME.txt
+cg_anno.tcl cachegrind.out.* >$TYPE-$NAME.txt
+echo '*****************************************************' >>$TYPE-$NAME.txt
+sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>$TYPE-$NAME.txt
+if test "$NAME" != "$BASELINE" -a $doDiff -ne 0; then
+  fossil xdiff --tk -c 20 $TYPE-$BASELINE.txt $TYPE-$NAME.txt
 fi
diff --git a/test/json/jsonb-q1.txt b/test/json/jsonb-q1.txt
new file mode 100644
index 0000000000..e78c63670a
--- /dev/null
+++ b/test/json/jsonb-q1.txt
@@ -0,0 +1,24 @@
+.mode qbox
+.timer on
+.param set $label 'q87'
+SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL;
+
+CREATE TEMP TABLE t2(x JSON TEXT);
+WITH RECURSIVE
+  c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000),
+  array1(y) AS (
+    SELECT json_group_array(
+             json_object('x',x,'y',random(),'z',hex(randomblob(50)))
+           )
+      FROM c
+  ),
+  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5)
+INSERT INTO t2(x)
+  SELECT jsonb_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2;
+CREATE INDEX t2x1 ON t2(x->>'a');
+CREATE INDEX t2x2 ON t2(x->>'b');
+CREATE INDEX t2x3 ON t2(x->>'e');
+CREATE INDEX t2x4 ON t2(x->>'f');
+UPDATE t2 SET x=jsonb_replace(x,'$.f',(x->>'f')+1);
+UPDATE t2 SET x=jsonb_set(x,'$.e',(x->>'f')-1);
+UPDATE t2 SET x=jsonb_remove(x,'$.d');
diff --git a/test/json101.test b/test/json101.test
index 543e4c71e7..7582d14a64 100644
--- a/test/json101.test
+++ b/test/json101.test
@@ -36,6 +36,9 @@ do_execsql_test json101-1.2 {
 do_catchsql_test json101-1.3 {
   SELECT json_array(1,printf('%.1000c','x'),x'abcd',3);
 } {1 {JSON cannot hold BLOB values}}
+do_catchsql_test json101-1.3b {
+  SELECT jsonb_array(1,printf('%.1000c','x'),x'abcd',3);
+} {1 {JSON cannot hold BLOB values}}
 do_execsql_test json101-1.4 {
   SELECT json_array(-9223372036854775808,9223372036854775807,0,1,-1,
                     0.0, 1.0, -1.0, -1e99, +2e100,
@@ -47,36 +50,83 @@ do_execsql_test json101-1.4 {
                     'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
                     99);
 } {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]}
+do_execsql_test json101-1.4b {
+  SELECT json(jsonb_array(-9223372036854775808,9223372036854775807,0,1,-1,
+                    0.0, 1.0, -1.0, -1e99, +2e100,
+                    'one','two','three',
+                    4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+                    19, NULL, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    99));
+} {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]}
 
 do_execsql_test json101-2.1 {
   SELECT json_object('a',1,'b',2.5,'c',null,'d','String Test');
 } {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}}
+do_execsql_test json101-2.1b {
+  SELECT json(jsonb_object('a',1,'b',2.5,'c',null,'d','String Test'));
+} {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}}
 do_catchsql_test json101-2.2 {
   SELECT json_object('a',printf('%.1000c','x'),2,2.5);
 } {1 {json_object() labels must be TEXT}}
+do_catchsql_test json101-2.2b {
+  SELECT jsonb_object('a',printf('%.1000c','x'),2,2.5);
+} {1 {json_object() labels must be TEXT}}
+do_execsql_test json101-2.2.2 {
+  SELECT json_object('a',json_array('xyx',77,4.5),'x',2.5);
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
+do_execsql_test json101-2.2.2b {
+  SELECT json(jsonb_object('a',json_array('xyx',77,4.5),'x',2.5));
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
+do_execsql_test json101-2.2.3 {
+  SELECT json_object('a',jsonb_array('xyx',77,4.5),'x',2.5);
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
+do_execsql_test json101-2.2.3b {
+  SELECT json(jsonb_object('a',jsonb_array('xyx',77,4.5),'x',2.5));
+} {{{"a":["xyx",77,4.5],"x":2.5}}}
 do_catchsql_test json101-2.3 {
   SELECT json_object('a',1,'b');
 } {1 {json_object() requires an even number of arguments}}
 do_catchsql_test json101-2.4 {
   SELECT json_object('a',printf('%.1000c','x'),'b',x'abcd');
 } {1 {JSON cannot hold BLOB values}}
+do_execsql_test json101-2.5 {
+  SELECT json_object('a',printf('%.10c','x'),'b',jsonb_array(1,2,3));
+} {{{"a":"xxxxxxxxxx","b":[1,2,3]}}}
 
 do_execsql_test json101-3.1 {
   SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]');
 } {{{"a":"[3,4,5]","b":2}}}
+do_execsql_test json101-3.1b {
+  SELECT json(jsonb_replace('{"a":1,"b":2}','$.a','[3,4,5]'));
+} {{{"a":"[3,4,5]","b":2}}}
 do_execsql_test json101-3.2 {
   SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]'));
 } {{{"a":[3,4,5],"b":2}}}
+do_execsql_test json101-3.2b {
+  SELECT json_replace('{"a":1,"b":2}','$.a',jsonb('[3,4,5]'));
+} {{{"a":[3,4,5],"b":2}}}
 do_execsql_test json101-3.3 {
   SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b');
 } {text}
+do_execsql_test json101-3.3b {
+  SELECT json_type(jsonb_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b');
+} {text}
 do_execsql_test json101-3.4 {
   SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b');
 } {object}
+do_execsql_test json101-3.4b {
+  SELECT json_type(jsonb_set('{"a":1,"b":2}','$.b',jsonb('{"x":3,"y":4}')),'$.b');
+} {object}
 ifcapable vtab {
-do_execsql_test json101-3.5 {
-  SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456));
-} {{$} {} | {$.x} 456 |}
+  do_execsql_test json101-3.5 {
+    SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456));
+  } {{$} {} | {$.x} 456 |}
+  do_execsql_test json101-3.5b {
+    SELECT fullkey, atom, '|' FROM json_tree(jsonb_set('{}','$.x',123,'$.x',456));
+  } {{$} {} | {$.x} 456 |}
 }
 
 # Per rfc7159, any JSON value is allowed at the top level, and whitespace
@@ -119,17 +169,27 @@ do_execsql_test json101-4.7 {
 do_execsql_test json101-4.8 {
   SELECT x FROM j1 WHERE json_insert(x)<>x;
 } {}
+do_execsql_test json101-4.9 {
+  SELECT json_insert('{"a":1}','$.b',CAST(x'0000' AS text));
+} {{{"a":1,"b":"\u0000\u0000"}}}
 
 # json_extract(JSON,'$') will return objects and arrays without change.
 #
-do_execsql_test json-4.10 {
+do_execsql_test json101-4.10 {
   SELECT count(*) FROM j1 WHERE json_type(x) IN ('object','array');
   SELECT x FROM j1
    WHERE json_extract(x,'$')<>x
      AND json_type(x) IN ('object','array');
 } {4}
+do_execsql_test json101-4.10b {
+  CREATE TABLE j1b AS SELECT jsonb(x) AS "x" FROM j1;
+  SELECT count(*) FROM j1b WHERE json_type(x) IN ('object','array');
+  SELECT json(x) FROM j1b
+   WHERE json_extract(x,'$')<>json(x)
+     AND json_type(x) IN ('object','array');
+} {4}
 
-do_execsql_test json-5.1 {
+do_execsql_test json101-5.1 {
   CREATE TABLE j2(id INTEGER PRIMARY KEY, json, src);
   INSERT INTO j2(id,json,src)
   VALUES(1,'{
@@ -255,11 +315,17 @@ do_execsql_test json-5.1 {
 	}
    ]','https://adobe.github.io/Spry/samples/data_region/JSONDataSetSample.html');
    SELECT count(*) FROM j2;
-} {3}
+   CREATE TABLE j2b(id INTEGER PRIMARY KEY, json, src);
+   INSERT INTO J2b(id,json,src) SELECT id, jsonb(json), src FROM j2;
+   SELECT count(*) FROM j2b;
+} {3 3}
 
-do_execsql_test json-5.2 {
+do_execsql_test json101-5.2 {
   SELECT id, json_valid(json), json_type(json), '|' FROM j2 ORDER BY id;
 } {1 1 object | 2 1 object | 3 1 array |}
+do_execsql_test json101-5.2b {
+  SELECT id, json_valid(json,5), json_type(json), '|' FROM j2b ORDER BY id;
+} {1 1 object | 2 1 object | 3 1 array |}
 
 ifcapable !vtab {
   finish_test
@@ -268,13 +334,19 @@ ifcapable !vtab {
 
 # fullkey is always the same as path+key (with appropriate formatting)
 #
-do_execsql_test json-5.3 {
+do_execsql_test json101-5.3 {
   SELECT j2.rowid, jx.rowid, fullkey, path, key
     FROM j2, json_tree(j2.json) AS jx
    WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']'
                                 ELSE '.'||key END);
 } {}
-do_execsql_test json-5.4 {
+do_execsql_test json101-5.3b {
+  SELECT j2b.rowid, jx.rowid, fullkey, path, key
+    FROM j2b, json_tree(j2b.json) AS jx
+   WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']'
+                                ELSE '.'||key END);
+} {}
+do_execsql_test json101-5.4 {
   SELECT j2.rowid, jx.rowid, fullkey, path, key
     FROM j2, json_each(j2.json) AS jx
    WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']'
@@ -285,58 +357,72 @@ do_execsql_test json-5.4 {
 # Verify that the json_each.json and json_tree.json output is always the
 # same as input.
 #
-do_execsql_test json-5.5 {
+do_execsql_test json101-5.5 {
   SELECT j2.rowid, jx.rowid, fullkey, path, key
     FROM j2, json_each(j2.json) AS jx
    WHERE jx.json<>j2.json;
 } {}
-do_execsql_test json-5.6 {
+do_execsql_test json101-5.6 {
   SELECT j2.rowid, jx.rowid, fullkey, path, key
     FROM j2, json_tree(j2.json) AS jx
    WHERE jx.json<>j2.json;
 } {}
-do_execsql_test json-5.7 {
+do_execsql_test json101-5.7 {
   SELECT j2.rowid, jx.rowid, fullkey, path, key
     FROM j2, json_each(j2.json) AS jx
    WHERE jx.value<>jx.atom AND type NOT IN ('array','object');
 } {}
-do_execsql_test json-5.8 {
+do_execsql_test json101-5.8 {
   SELECT j2.rowid, jx.rowid, fullkey, path, key
     FROM j2, json_tree(j2.json) AS jx
    WHERE jx.value<>jx.atom AND type NOT IN ('array','object');
 } {}
 
-do_execsql_test json-6.1 {
+# 2024-02-16 https://sqlite.org/forum/forumpost/ecb94cd210
+# Regression in json_tree()/json_each().  The value column
+# should have the "J" subtype if the value is an array or
+# object.
+#
+do_execsql_test json101-5.10 {
+  SELECT json_insert('{}','$.a',value) FROM json_tree('[1,2,3]') WHERE atom IS NULL;
+} {{{"a":[1,2,3]}}}
+#        ^^^^^^^--- In double-quotes, a string literal, prior to bug fix
+
+do_execsql_test json101-5.11 {
+  SELECT json_insert('{}','$.a',value) FROM json_tree('"[1,2,3]"');
+} {{{"a":"[1,2,3]"}}}
+
+do_execsql_test json101-6.1 {
   SELECT json_valid('{"a":55,"b":72,}');
 } {0}
-do_execsql_test json-6.2 {
+do_execsql_test json101-6.2 {
   SELECT json_error_position('{"a":55,"b":72,}');
 } {0}
-do_execsql_test json-6.3 {
+do_execsql_test json101-6.3 {
   SELECT json_valid(json('{"a":55,"b":72,}'));
 } {1}
-do_execsql_test json-6.4 {
+do_execsql_test json101-6.4 {
   SELECT json_valid('{"a":55,"b":72 , }');
 } {0}
-do_execsql_test json-6.5 {
+do_execsql_test json101-6.5 {
   SELECT json_error_position('{"a":55,"b":72 , }');
 } {0}
-do_execsql_test json-6.6 {
+do_execsql_test json101-6.6 {
   SELECT json_error_position('{"a":55,"b":72,,}');
 } {16}
-do_execsql_test json-6.7 {
+do_execsql_test json101-6.7 {
   SELECT json_valid('{"a":55,"b":72}');
 } {1}
-do_execsql_test json-6.8 {
+do_execsql_test json101-6.8 {
   SELECT json_error_position('["a",55,"b",72,]');
 } {0}
-do_execsql_test json-6.9 {
+do_execsql_test json101-6.9 {
   SELECT json_error_position('["a",55,"b",72 , ]');
 } {0}
-do_execsql_test json-6.10 {
+do_execsql_test json101-6.10 {
   SELECT json_error_position('["a",55,"b",72,,]');
 } {16}
-do_execsql_test json-6.11 {
+do_execsql_test json101-6.11 {
   SELECT json_valid('["a",55,"b",72]');
 } {1}
 
@@ -352,32 +438,39 @@ foreach {tn isvalid ws} {
   7.6  1  char(0x20,0x09,0x0a,0x0d,0x20)
   7.7  0  char(0x20,0x09,0x0a,0x0c,0x0d,0x20)
 } {
-  do_execsql_test json-$tn.1 \
+  do_execsql_test json101-$tn.1 \
     "SELECT json_valid(printf('%s{%s\"x\"%s:%s9%s}%s',
          $::ws,$::ws,$::ws,$::ws,$::ws,$::ws));" \
   $isvalid
 }
 
-# Ticket https://www.sqlite.org/src/info/ad2559db380abf8e
+# Ticket https://sqlite.org/src/info/ad2559db380abf8e
 # Control characters must be escaped in JSON strings.
 #
-do_execsql_test json-8.1 {
+do_execsql_test json101-8.1 {
   DROP TABLE IF EXISTS t8;
   CREATE TABLE t8(a,b);
   INSERT INTO t8(a) VALUES('abc' || char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35) || 'xyz');
   UPDATE t8 SET b=json_array(a);
   SELECT b FROM t8;
 } {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}}
-do_execsql_test json-8.2 {
+do_execsql_test json101-8.1b {
+  DROP TABLE IF EXISTS t8;
+  CREATE TABLE t8(a,b);
+  INSERT INTO t8(a) VALUES('abc' || char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35) || 'xyz');
+  UPDATE t8 SET b=jsonb_array(a);
+  SELECT json(b) FROM t8;
+} {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}}
+do_execsql_test json101-8.2 {
   SELECT a=json_extract(b,'$[0]') FROM t8;
 } {1}
 
 # 2017-04-12.  Regression reported on the mailing list by Rolf Ade
 #
-do_execsql_test json-8.3 {
+do_execsql_test json101-8.3 {
   SELECT json_valid(char(0x22,0xe4,0x22));
 } {1}
-do_execsql_test json-8.4 {
+do_execsql_test json101-8.4 {
   SELECT unicode(json_extract(char(0x22,228,0x22),'$'));
 } {228}
 
@@ -385,354 +478,354 @@ do_execsql_test json-8.4 {
 # String values are quoted and interior quotes are escaped.  NULL values
 # are rendered as the unquoted string "null".
 #
-do_execsql_test json-9.1 {
+do_execsql_test json101-9.1 {
   SELECT json_quote('abc"xyz');
 } {{"abc\"xyz"}}
-do_execsql_test json-9.2 {
+do_execsql_test json101-9.2 {
   SELECT json_quote(3.14159);
 } {3.14159}
-do_execsql_test json-9.3 {
+do_execsql_test json101-9.3 {
   SELECT json_quote(12345);
 } {12345}
-do_execsql_test json-9.4 {
+do_execsql_test json101-9.4 {
   SELECT json_quote(null);
 } {"null"}
-do_catchsql_test json-9.5 {
-  SELECT json_quote(x'30313233');
+do_catchsql_test json101-9.5 {
+  SELECT json_quote(x'3031323334');
 } {1 {JSON cannot hold BLOB values}}
-do_catchsql_test json-9.6 {
+do_catchsql_test json101-9.6 {
   SELECT json_quote(123,456)
 } {1 {wrong number of arguments to function json_quote()}}
-do_catchsql_test json-9.7 {
+do_catchsql_test json101-9.7 {
   SELECT json_quote()
 } {1 {wrong number of arguments to function json_quote()}}
 
 # Make sure only valid backslash-escapes are accepted.
 #
-do_execsql_test json-10.1 {
+do_execsql_test json101-10.1 {
   SELECT json_valid('" \  "');
 } {0}
-do_execsql_test json-10.2 {
+do_execsql_test json101-10.2 {
   SELECT json_valid('" \! "');
 } {0}
-do_execsql_test json-10.3 {
+do_execsql_test json101-10.3 {
   SELECT json_valid('" \" "');
 } {1}
-do_execsql_test json-10.4 {
+do_execsql_test json101-10.4 {
   SELECT json_valid('" \# "');
 } {0}
-do_execsql_test json-10.5 {
+do_execsql_test json101-10.5 {
   SELECT json_valid('" \$ "');
 } {0}
-do_execsql_test json-10.6 {
+do_execsql_test json101-10.6 {
   SELECT json_valid('" \% "');
 } {0}
-do_execsql_test json-10.7 {
+do_execsql_test json101-10.7 {
   SELECT json_valid('" \& "');
 } {0}
-do_execsql_test json-10.8 {
+do_execsql_test json101-10.8 {
   SELECT json_valid('" \'' "');
 } {0}
-do_execsql_test json-10.9 {
+do_execsql_test json101-10.9 {
   SELECT json_valid('" \( "');
 } {0}
-do_execsql_test json-10.10 {
+do_execsql_test json101-10.10 {
   SELECT json_valid('" \) "');
 } {0}
-do_execsql_test json-10.11 {
+do_execsql_test json101-10.11 {
   SELECT json_valid('" \* "');
 } {0}
-do_execsql_test json-10.12 {
+do_execsql_test json101-10.12 {
   SELECT json_valid('" \+ "');
 } {0}
-do_execsql_test json-10.13 {
+do_execsql_test json101-10.13 {
   SELECT json_valid('" \, "');
 } {0}
-do_execsql_test json-10.14 {
+do_execsql_test json101-10.14 {
   SELECT json_valid('" \- "');
 } {0}
-do_execsql_test json-10.15 {
+do_execsql_test json101-10.15 {
   SELECT json_valid('" \. "');
 } {0}
-do_execsql_test json-10.16 {
+do_execsql_test json101-10.16 {
   SELECT json_valid('" \/ "');
 } {1}
-do_execsql_test json-10.17 {
+do_execsql_test json101-10.17 {
   SELECT json_valid('" \0 "');
 } {0}
-do_execsql_test json-10.18 {
+do_execsql_test json101-10.18 {
   SELECT json_valid('" \1 "');
 } {0}
-do_execsql_test json-10.19 {
+do_execsql_test json101-10.19 {
   SELECT json_valid('" \2 "');
 } {0}
-do_execsql_test json-10.20 {
+do_execsql_test json101-10.20 {
   SELECT json_valid('" \3 "');
 } {0}
-do_execsql_test json-10.21 {
+do_execsql_test json101-10.21 {
   SELECT json_valid('" \4 "');
 } {0}
-do_execsql_test json-10.22 {
+do_execsql_test json101-10.22 {
   SELECT json_valid('" \5 "');
 } {0}
-do_execsql_test json-10.23 {
+do_execsql_test json101-10.23 {
   SELECT json_valid('" \6 "');
 } {0}
-do_execsql_test json-10.24 {
+do_execsql_test json101-10.24 {
   SELECT json_valid('" \7 "');
 } {0}
-do_execsql_test json-10.25 {
+do_execsql_test json101-10.25 {
   SELECT json_valid('" \8 "');
 } {0}
-do_execsql_test json-10.26 {
+do_execsql_test json101-10.26 {
   SELECT json_valid('" \9 "');
 } {0}
-do_execsql_test json-10.27 {
+do_execsql_test json101-10.27 {
   SELECT json_valid('" \: "');
 } {0}
-do_execsql_test json-10.28 {
+do_execsql_test json101-10.28 {
   SELECT json_valid('" \; "');
 } {0}
-do_execsql_test json-10.29 {
+do_execsql_test json101-10.29 {
   SELECT json_valid('" \< "');
 } {0}
-do_execsql_test json-10.30 {
+do_execsql_test json101-10.30 {
   SELECT json_valid('" \= "');
 } {0}
-do_execsql_test json-10.31 {
+do_execsql_test json101-10.31 {
   SELECT json_valid('" \> "');
 } {0}
-do_execsql_test json-10.32 {
+do_execsql_test json101-10.32 {
   SELECT json_valid('" \? "');
 } {0}
-do_execsql_test json-10.33 {
+do_execsql_test json101-10.33 {
   SELECT json_valid('" \@ "');
 } {0}
-do_execsql_test json-10.34 {
+do_execsql_test json101-10.34 {
   SELECT json_valid('" \A "');
 } {0}
-do_execsql_test json-10.35 {
+do_execsql_test json101-10.35 {
   SELECT json_valid('" \B "');
 } {0}
-do_execsql_test json-10.36 {
+do_execsql_test json101-10.36 {
   SELECT json_valid('" \C "');
 } {0}
-do_execsql_test json-10.37 {
+do_execsql_test json101-10.37 {
   SELECT json_valid('" \D "');
 } {0}
-do_execsql_test json-10.38 {
+do_execsql_test json101-10.38 {
   SELECT json_valid('" \E "');
 } {0}
-do_execsql_test json-10.39 {
+do_execsql_test json101-10.39 {
   SELECT json_valid('" \F "');
 } {0}
-do_execsql_test json-10.40 {
+do_execsql_test json101-10.40 {
   SELECT json_valid('" \G "');
 } {0}
-do_execsql_test json-10.41 {
+do_execsql_test json101-10.41 {
   SELECT json_valid('" \H "');
 } {0}
-do_execsql_test json-10.42 {
+do_execsql_test json101-10.42 {
   SELECT json_valid('" \I "');
 } {0}
-do_execsql_test json-10.43 {
+do_execsql_test json101-10.43 {
   SELECT json_valid('" \J "');
 } {0}
-do_execsql_test json-10.44 {
+do_execsql_test json101-10.44 {
   SELECT json_valid('" \K "');
 } {0}
-do_execsql_test json-10.45 {
+do_execsql_test json101-10.45 {
   SELECT json_valid('" \L "');
 } {0}
-do_execsql_test json-10.46 {
+do_execsql_test json101-10.46 {
   SELECT json_valid('" \M "');
 } {0}
-do_execsql_test json-10.47 {
+do_execsql_test json101-10.47 {
   SELECT json_valid('" \N "');
 } {0}
-do_execsql_test json-10.48 {
+do_execsql_test json101-10.48 {
   SELECT json_valid('" \O "');
 } {0}
-do_execsql_test json-10.49 {
+do_execsql_test json101-10.49 {
   SELECT json_valid('" \P "');
 } {0}
-do_execsql_test json-10.50 {
+do_execsql_test json101-10.50 {
   SELECT json_valid('" \Q "');
 } {0}
-do_execsql_test json-10.51 {
+do_execsql_test json101-10.51 {
   SELECT json_valid('" \R "');
 } {0}
-do_execsql_test json-10.52 {
+do_execsql_test json101-10.52 {
   SELECT json_valid('" \S "');
 } {0}
-do_execsql_test json-10.53 {
+do_execsql_test json101-10.53 {
   SELECT json_valid('" \T "');
 } {0}
-do_execsql_test json-10.54 {
+do_execsql_test json101-10.54 {
   SELECT json_valid('" \U "');
 } {0}
-do_execsql_test json-10.55 {
+do_execsql_test json101-10.55 {
   SELECT json_valid('" \V "');
 } {0}
-do_execsql_test json-10.56 {
+do_execsql_test json101-10.56 {
   SELECT json_valid('" \W "');
 } {0}
-do_execsql_test json-10.57 {
+do_execsql_test json101-10.57 {
   SELECT json_valid('" \X "');
 } {0}
-do_execsql_test json-10.58 {
+do_execsql_test json101-10.58 {
   SELECT json_valid('" \Y "');
 } {0}
-do_execsql_test json-10.59 {
+do_execsql_test json101-10.59 {
   SELECT json_valid('" \Z "');
 } {0}
-do_execsql_test json-10.60 {
+do_execsql_test json101-10.60 {
   SELECT json_valid('" \[ "');
 } {0}
-do_execsql_test json-10.61 {
+do_execsql_test json101-10.61 {
   SELECT json_valid('" \\ "');
 } {1}
-do_execsql_test json-10.62 {
+do_execsql_test json101-10.62 {
   SELECT json_valid('" \] "');
 } {0}
-do_execsql_test json-10.63 {
+do_execsql_test json101-10.63 {
   SELECT json_valid('" \^ "');
 } {0}
-do_execsql_test json-10.64 {
+do_execsql_test json101-10.64 {
   SELECT json_valid('" \_ "');
 } {0}
-do_execsql_test json-10.65 {
+do_execsql_test json101-10.65 {
   SELECT json_valid('" \` "');
 } {0}
-do_execsql_test json-10.66 {
+do_execsql_test json101-10.66 {
   SELECT json_valid('" \a "');
 } {0}
-do_execsql_test json-10.67 {
+do_execsql_test json101-10.67 {
   SELECT json_valid('" \b "');
 } {1}
-do_execsql_test json-10.68 {
+do_execsql_test json101-10.68 {
   SELECT json_valid('" \c "');
 } {0}
-do_execsql_test json-10.69 {
+do_execsql_test json101-10.69 {
   SELECT json_valid('" \d "');
 } {0}
-do_execsql_test json-10.70 {
+do_execsql_test json101-10.70 {
   SELECT json_valid('" \e "');
 } {0}
-do_execsql_test json-10.71 {
+do_execsql_test json101-10.71 {
   SELECT json_valid('" \f "');
 } {1}
-do_execsql_test json-10.72 {
+do_execsql_test json101-10.72 {
   SELECT json_valid('" \g "');
 } {0}
-do_execsql_test json-10.73 {
+do_execsql_test json101-10.73 {
   SELECT json_valid('" \h "');
 } {0}
-do_execsql_test json-10.74 {
+do_execsql_test json101-10.74 {
   SELECT json_valid('" \i "');
 } {0}
-do_execsql_test json-10.75 {
+do_execsql_test json101-10.75 {
   SELECT json_valid('" \j "');
 } {0}
-do_execsql_test json-10.76 {
+do_execsql_test json101-10.76 {
   SELECT json_valid('" \k "');
 } {0}
-do_execsql_test json-10.77 {
+do_execsql_test json101-10.77 {
   SELECT json_valid('" \l "');
 } {0}
-do_execsql_test json-10.78 {
+do_execsql_test json101-10.78 {
   SELECT json_valid('" \m "');
 } {0}
-do_execsql_test json-10.79 {
+do_execsql_test json101-10.79 {
   SELECT json_valid('" \n "');
 } {1}
-do_execsql_test json-10.80 {
+do_execsql_test json101-10.80 {
   SELECT json_valid('" \o "');
 } {0}
-do_execsql_test json-10.81 {
+do_execsql_test json101-10.81 {
   SELECT json_valid('" \p "');
 } {0}
-do_execsql_test json-10.82 {
+do_execsql_test json101-10.82 {
   SELECT json_valid('" \q "');
 } {0}
-do_execsql_test json-10.83 {
+do_execsql_test json101-10.83 {
   SELECT json_valid('" \r "');
 } {1}
-do_execsql_test json-10.84 {
+do_execsql_test json101-10.84 {
   SELECT json_valid('" \s "');
 } {0}
-do_execsql_test json-10.85 {
+do_execsql_test json101-10.85 {
   SELECT json_valid('" \t "');
 } {1}
-do_execsql_test json-10.86.0 {
+do_execsql_test json101-10.86.0 {
   SELECT json_valid('" \u "');
 } {0}
-do_execsql_test json-10.86.1 {
+do_execsql_test json101-10.86.1 {
   SELECT json_valid('" \ua "');
 } {0}
-do_execsql_test json-10.86.2 {
+do_execsql_test json101-10.86.2 {
   SELECT json_valid('" \uab "');
 } {0}
-do_execsql_test json-10.86.3 {
+do_execsql_test json101-10.86.3 {
   SELECT json_valid('" \uabc "');
 } {0}
-do_execsql_test json-10.86.4 {
+do_execsql_test json101-10.86.4 {
   SELECT json_valid('" \uabcd "');
 } {1}
-do_execsql_test json-10.86.5 {
+do_execsql_test json101-10.86.5 {
   SELECT json_valid('" \uFEDC "');
 } {1}
-do_execsql_test json-10.86.6 {
+do_execsql_test json101-10.86.6 {
   SELECT json_valid('" \u1234 "');
 } {1}
-do_execsql_test json-10.87 {
+do_execsql_test json101-10.87 {
   SELECT json_valid('" \v "');
 } {0}
-do_execsql_test json-10.88 {
+do_execsql_test json101-10.88 {
   SELECT json_valid('" \w "');
 } {0}
-do_execsql_test json-10.89 {
+do_execsql_test json101-10.89 {
   SELECT json_valid('" \x "');
 } {0}
-do_execsql_test json-10.90 {
+do_execsql_test json101-10.90 {
   SELECT json_valid('" \y "');
 } {0}
-do_execsql_test json-10.91 {
+do_execsql_test json101-10.91 {
   SELECT json_valid('" \z "');
 } {0}
-do_execsql_test json-10.92 {
+do_execsql_test json101-10.92 {
   SELECT json_valid('" \{ "');
 } {0}
-do_execsql_test json-10.93 {
+do_execsql_test json101-10.93 {
   SELECT json_valid('" \| "');
 } {0}
-do_execsql_test json-10.94 {
+do_execsql_test json101-10.94 {
   SELECT json_valid('" \} "');
 } {0}
-do_execsql_test json-10.95 {
+do_execsql_test json101-10.95 {
   SELECT json_valid('" \~ "');
 } {0}
 
 #--------------------------------------------------------------------------
-# 2017-04-11.  https://www.sqlite.org/src/info/981329adeef51011
+# 2017-04-11.  https://sqlite.org/src/info/981329adeef51011
 # Stack overflow on deeply nested JSON.
 #
 # The following tests confirm that deeply nested JSON is considered invalid.
 #
-do_execsql_test json-11.0 {
+do_execsql_test json101-11.0 {
   /* Shallow enough to be parsed */
   SELECT json_valid(printf('%.1000c0%.1000c','[',']'));
 } {1}
-do_execsql_test json-11.1 {
+do_execsql_test json101-11.1 {
   /* Too deep by one */
   SELECT json_valid(printf('%.1001c0%.1001c','[',']'));
 } {0}
-do_execsql_test json-11.2 {
+do_execsql_test json101-11.2 {
   /* Shallow enough to be parsed { */
   SELECT json_valid(replace(printf('%.1000c0%.1000c','[','}'),'[','{"a":'));
   /* } */
 } {1}
-do_execsql_test json-11.3 {
+do_execsql_test json101-11.3 {
   /* Too deep by one { */
   SELECT json_valid(replace(printf('%.1001c0%.1001c','[','}'),'[','{"a":'));
   /* } */
@@ -742,7 +835,7 @@ do_execsql_test json-11.3 {
 # a json structure even though the element name constains a "."
 # character, by quoting the element name in the path.
 #
-do_execsql_test json-12.100 {
+do_execsql_test json101-12.100 {
   CREATE TABLE t12(x);
   INSERT INTO t12(x) VALUES(
     '{"settings":
@@ -766,20 +859,29 @@ do_execsql_test json-12.100 {
         }
      }');
 } {}
-do_execsql_test json-12.110 {
+
+do_execsql_test json101-12.110 {
   SELECT json_remove(x, '$.settings.layer2."dis.legomenon".forceDisplay')
     FROM t12;
 } {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}}
-do_execsql_test json-12.120 {
+do_execsql_test json101-12.110b {
+  SELECT json_remove(jsonb(x), '$.settings.layer2."dis.legomenon".forceDisplay')
+    FROM t12;
+} {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}}
+do_execsql_test json101-12.120 {
   SELECT json_extract(x, '$.settings.layer2."tris.legomenon"."summary.report"')
     FROM t12;
 } {0}
+do_execsql_test json101-12.120b {
+  SELECT json_extract(jsonb(x), '$.settings.layer2."tris.legomenon"."summary.report"')
+    FROM t12;
+} {0}
 
 # 2018-01-26
-# ticket https://www.sqlite.org/src/tktview/80177f0c226ff54f6ddd41
+# ticket https://sqlite.org/src/tktview/80177f0c226ff54f6ddd41
 # Make sure the query planner knows about the arguments to table-valued functions.
 #
-do_execsql_test json-13.100 {
+do_execsql_test json101-13.100 {
   DROP TABLE IF EXISTS t1;
   DROP TABLE IF EXISTS t2;
   CREATE TABLE t1(id, json);
@@ -790,42 +892,42 @@ do_execsql_test json-13.100 {
   INSERT INTO t2(id,json) VALUES(4,'{"value":4}');
   INSERT INTO t2(id,json) VALUES(5,'{"value":5}');
   INSERT INTO t2(id,json) VALUES(6,'{"value":6}');
-  SELECT * FROM t1 CROSS JOIN t2
+  SELECT *, 'NL' FROM t1 CROSS JOIN t2
    WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z
                  WHERE Z.value==t2.id);
-} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}}
-do_execsql_test json-13.110 {
-  SELECT * FROM t2 CROSS JOIN t1
+} {1 {{"items":[3,5]}} 3 {{"value":3}} NL 1 {{"items":[3,5]}} 5 {{"value":5}} NL}
+do_execsql_test json101-13.110 {
+  SELECT *, 'NL' FROM t2 CROSS JOIN t1
    WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z
                  WHERE Z.value==t2.id);
-} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}}
+} {3 {{"value":3}} 1 {{"items":[3,5]}} NL 5 {{"value":5}} 1 {{"items":[3,5]}} NL}
 
 # 2018-05-16
 # Incorrect fullkey output from json_each()
 # when the input JSON is not an array or object.
 #
-do_execsql_test json-14.100 {
+do_execsql_test json101-14.100 {
   SELECT fullkey FROM json_each('123');
 } {$}
-do_execsql_test json-14.110 {
+do_execsql_test json101-14.110 {
   SELECT fullkey FROM json_each('123.56');
 } {$}
-do_execsql_test json-14.120 {
+do_execsql_test json101-14.120 {
   SELECT fullkey FROM json_each('"hello"');
 } {$}
-do_execsql_test json-14.130 {
+do_execsql_test json101-14.130 {
   SELECT fullkey FROM json_each('null');
 } {$}
-do_execsql_test json-14.140 {
+do_execsql_test json101-14.140 {
   SELECT fullkey FROM json_tree('123');
 } {$}
-do_execsql_test json-14.150 {
+do_execsql_test json101-14.150 {
   SELECT fullkey FROM json_tree('123.56');
 } {$}
-do_execsql_test json-14.160 {
+do_execsql_test json101-14.160 {
   SELECT fullkey FROM json_tree('"hello"');
 } {$}
-do_execsql_test json-14.170 {
+do_execsql_test json101-14.170 {
   SELECT fullkey FROM json_tree('null');
 } {$}
 
@@ -835,35 +937,35 @@ do_execsql_test json-14.170 {
 #
 # Bug reported via private email. See TH3 for more information.
 #
-do_execsql_test json-15.100 {
+do_execsql_test json101-15.100 {
   SELECT * FROM JSON_EACH('{"a":1, "b":2}');
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
-do_execsql_test json-15.110 {
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
+do_execsql_test json101-15.110 {
   SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz;
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
-do_execsql_test json-15.120 {
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
+do_execsql_test json101-15.120 {
   SELECT * FROM (JSON_EACH('{"a":1, "b":2}'));
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
-do_execsql_test json-15.130 {
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
+do_execsql_test json101-15.130 {
   SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz;
-} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}}
+} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}}
 
 # 2019-11-10
 # Mailing list bug report on the handling of surrogate pairs
 # in JSON.
 #
-do_execsql_test json-16.10 {
+do_execsql_test json101-16.10 {
   SELECT length(json_extract('"abc\uD834\uDD1Exyz"','$'));
 } {7}
-do_execsql_test json-16.20 {
+do_execsql_test json101-16.20 {
   SELECT length(json_extract('"\uD834\uDD1E"','$'));
 } {1}
-do_execsql_test json-16.30 {
+do_execsql_test json101-16.30 {
   SELECT unicode(json_extract('"\uD834\uDD1E"','$'));
 } {119070}
 
 # 2022-01-30 dbsqlfuzz 4678cf825d27f87c9b8343720121e12cf944b71a
-do_execsql_test json-17.1 {
+do_execsql_test json101-17.1 {
   DROP TABLE IF EXISTS t1;
   DROP TABLE IF EXISTS t2;
   CREATE TABLE t1(a,b,c);
@@ -872,49 +974,49 @@ do_execsql_test json-17.1 {
 } {}
 
 # 2022-04-04 forum post https://sqlite.org/forum/forumpost/c082aeab43
-do_execsql_test json-18.1 {
+do_execsql_test json101-18.1 {
   SELECT json_valid('{"":5}');
 } {1}
-do_execsql_test json-18.2 {
+do_execsql_test json101-18.2 {
   SELECT json_extract('{"":5}', '$.""');
 } {5}
-do_execsql_test json-18.3 {
+do_execsql_test json101-18.3 {
   SELECT json_extract('[3,{"a":4,"":[5,{"hi":6},7]},8]', '$[1].""[1].hi');
 } {6}
-do_execsql_test json-18.4 {
+do_execsql_test json101-18.4 {
   SELECT json_extract('[3,{"a":4,"":[5,{"hi":6},7]},8]', '$[1].""[1]."hi"');
 } {6}
-do_catchsql_test json-18.5 {
+do_catchsql_test json101-18.5 {
   SELECT json_extract('{"":8}', '$.');
-} {1 {JSON path error near ''}}
+} {1 {bad JSON path: '$.'}}
 
 # 2022-08-29 https://sqlite.org/forum/forumpost/9b9e4716c0d7bbd1
 # This is not a problem specifically with JSON functions.  It is
 # a problem with transaction control.  But the json() function makes
 # the problem more easily accessible, so it is tested here.
 #
-do_execsql_test json-19.1 {
+do_execsql_test json101-19.1 {
   DROP TABLE IF EXISTS t1;
   CREATE TABLE t1(x);
 } {}
-do_catchsql_test json-19.2 {
+do_catchsql_test json101-19.2 {
   BEGIN;
   INSERT INTO t1 VALUES(0), (json('not-valid-json'));
 } {1 {malformed JSON}}
-do_execsql_test json-19.3 {
+do_execsql_test json101-19.3 {
   COMMIT;
   SELECT * FROM t1;
 } {}
 
 # 2023-03-17 positive and negative infinities
 #
-do_execsql_test json-20.1 {
+do_execsql_test json101-20.1 {
   SELECT json_object('a',2e370,'b',-3e380);
 } {{{"a":9.0e+999,"b":-9.0e+999}}}
-do_execsql_test json-20.2 {
+do_execsql_test json101-20.2 {
   SELECT json_object('a',2e370,'b',-3e380)->>'a';
 } Inf
-do_execsql_test json-20.3 {
+do_execsql_test json101-20.3 {
   SELECT json_object('a',2e370,'b',-3e380)->>'b';
 } {-Inf}
 
@@ -923,92 +1025,157 @@ do_execsql_test json-20.3 {
 # a NULL value as the JSON input.
 #
 db null NULL
-do_execsql_test json-21.1 {
-  SELECT json_valid(NULL);
-} NULL
-do_execsql_test json-21.2 {
+if {[db exists {SELECT * FROM pragma_compile_options WHERE compile_options LIKE '%legacy_json_valid%'}]} {
+  do_execsql_test json101-21.1-legacy {
+    SELECT json_valid(NULL);
+  } 0
+} else {
+  do_execsql_test json101-21.1-correct {
+    SELECT json_valid(NULL);
+  } NULL
+}
+do_execsql_test json101-21.2 {
   SELECT json_error_position(NULL);
 } NULL
-do_execsql_test json-21.3 {
+do_execsql_test json101-21.3 {
   SELECT json(NULL);
 } NULL
-do_execsql_test json-21.4 {
+do_execsql_test json101-21.4 {
   SELECT json_array(NULL);
 } {[null]}
-do_execsql_test json-21.5 {
+do_execsql_test json101-21.5 {
   SELECT json_extract(NULL);
 } NULL
-do_execsql_test json-21.6 {
+do_execsql_test json101-21.6 {
   SELECT json_insert(NULL,'$',123);
 } NULL
-do_execsql_test json-21.7 {
+do_execsql_test json101-21.7 {
   SELECT NULL->0;
 } NULL
-do_execsql_test json-21.8 {
+do_execsql_test json101-21.8 {
   SELECT NULL->>0;
 } NULL
-do_execsql_test json-21.9 {
+do_execsql_test json101-21.9 {
   SELECT '{a:5}'->NULL;
 } NULL
-do_execsql_test json-21.10 {
+do_execsql_test json101-21.10 {
   SELECT '{a:5}'->>NULL;
 } NULL
-do_catchsql_test json-21.11 {
+do_catchsql_test json101-21.11 {
   SELECT json_object(NULL,5);
 } {1 {json_object() labels must be TEXT}}
-do_execsql_test json-21.12 {
+do_execsql_test json101-21.12 {
   SELECT json_patch(NULL,'{a:5}');
 } NULL
-do_execsql_test json-21.13 {
+do_execsql_test json101-21.13 {
   SELECT json_patch('{a:5}',NULL);
 } NULL
-do_execsql_test json-21.14 {
+do_execsql_test json101-21.14 {
   SELECT json_patch(NULL,NULL);
 } NULL
-do_execsql_test json-21.15 {
+do_execsql_test json101-21.15 {
   SELECT json_remove(NULL,'$');
 } NULL
-do_execsql_test json-21.16 {
+do_execsql_test json101-21.16 {
   SELECT json_remove('{a:5,b:7}',NULL);
 } NULL
-do_execsql_test json-21.17 {
+do_execsql_test json101-21.17 {
   SELECT json_replace(NULL,'$.a',123);
 } NULL
-do_execsql_test json-21.18 {
+do_execsql_test json101-21.18 {
   SELECT json_replace('{a:5,b:7}',NULL,NULL);
 } {{{"a":5,"b":7}}}
-do_execsql_test json-21.19 {
+do_execsql_test json101-21.19 {
   SELECT json_set(NULL,'$.a',123);
 } NULL
-do_execsql_test json-21.20 {
+do_execsql_test json101-21.20 {
   SELECT json_set('{a:5,b:7}',NULL,NULL);
 } {{{"a":5,"b":7}}}
-do_execsql_test json-21.21 {
+do_execsql_test json101-21.21 {
   SELECT json_type(NULL);
 } NULL
-do_execsql_test json-21.22 {
+do_execsql_test json101-21.22 {
   SELECT json_type('{a:5,b:7}',NULL);
 } NULL
-do_execsql_test json-21.23 {
+do_execsql_test json101-21.23 {
   SELECT json_quote(NULL);
 } null
-do_execsql_test json-21.24 {
+do_execsql_test json101-21.24 {
   SELECT count(*) FROM json_each(NULL);
 } 0
-do_execsql_test json-21.25 {
+do_execsql_test json101-21.25 {
   SELECT count(*) FROM json_tree(NULL);
 } 0
-do_execsql_test json-21.26 {
+do_execsql_test json101-21.26 {
   WITH c(x) AS (VALUES(1),(2.0),(NULL),('three'))
   SELECT json_group_array(x) FROM c;
 } {[1,2.0,null,"three"]}
-do_execsql_test json-21.27 {
+do_execsql_test json101-21.27 {
   WITH c(x,y) AS (VALUES('a',1),('b',2.0),('c',NULL),(NULL,'three'),('e','four'))
   SELECT json_group_object(x,y) FROM c;
-} {{{"a":1,"b":2.0,"c":null,:"three","e":"four"}}}
-
+} {{{"a":1,"b":2.0,"c":null,"e":"four"}}}
 
+# 2023-10-09 https://sqlite.org/forum/forumpost/b25edc1d46
+# UAF due to JSON cache overflow
+#
+do_execsql_test json101-22.1 {
+  SELECT json_set(
+    '{}',
+    '$.a', json('1'),
+    '$.a', json('2'),
+    '$.b', json('3'),
+    '$.b', json('4'),
+    '$.c', json('5'),
+    '$.c', json('6')
+  );
+} {{{"a":2,"b":4,"c":6}}}
+do_execsql_test json101-22.2 {
+  SELECT json_replace(
+    '{"a":7,"b":8,"c":9}',
+    '$.a', json('1'),
+    '$.a', json('2'),
+    '$.b', json('3'),
+    '$.b', json('4'),
+    '$.c', json('5'),
+    '$.c', json('6')
+  );
+} {{{"a":2,"b":4,"c":6}}}
 
+# 2023-10-17 https://sqlite.org/forum/forumpost/fc0e3f1e2a
+# Incorrect accesss to '$[0]' in parsed + edited JSON.
+#
+do_execsql_test json101-23.1 {
+  SELECT j, j->>0, j->>1
+    FROM (SELECT json_set(json_set('[]','$[#]',0), '$[#]',1) AS j);
+} {{[0,1]} 0 1}
+do_execsql_test json101-23.2 {
+  SELECT j, j->>0, j->>1
+    FROM (SELECT json_set('[]','$[#]',0,'$[#]',1) AS j);
+} {{[0,1]} 0 1}
 
+# Insert/Set/Replace where the path specifies substructure that
+# does not yet exist
+#
+proc tx x {return [string map [list ( \173 ) \175 ' \042 < \133 > \135] $x]}
+foreach {id start path ins set repl} {
+  1 {{}}       {$.a.b.c}     ('a':('b':('c':9)))      ('a':('b':('c':9)))      ()
+  2 {{a:4}}    {$.a.b.c}     ('a':4)                  ('a':4)                  ('a':4)
+  3 {{a:{}}}   {$.a.b.c}     ('a':('b':('c':9)))      ('a':('b':('c':9)))      ('a':())
+  4 {[0,1,2]}  {$[3].a[0].b} <0,1,2,('a':<('b':9)>)>  <0,1,2,('a':<('b':9)>)>  <0,1,2>
+  5 {[0,1,2]}  {$[1].a[0].b} <0,1,2>                  <0,1,2>                  <0,1,2>
+  6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':9)>),2>    <0,('a':<('b':9)>),2>    <0,(),2>
+  7 {[0,1,2]}  {$[3][0].b}   <0,1,2,<('b':9)>>        <0,1,2,<('b':9)>>        <0,1,2>
+  8 {[0,1,2]}  {$[1][0].b}   <0,1,2>                  <0,1,2>                  <0,1,2>
+} {
+  do_execsql_test json101-24.$id.insert {
+    SELECT json_insert($start,$path,9);
+  } [list [tx $ins]]
+  do_execsql_test json101-24.$id.set {
+    SELECT json_set($start,$path,9);
+  } [list [tx $set]]
+  do_execsql_test json101-24.$id.replace {
+    SELECT json_replace($start,$path,9);
+  } [list [tx $repl]]
+}
 
 finish_test
diff --git a/test/json102.test b/test/json102.test
index bfd5e7ed07..54a0e1e0e0 100644
--- a/test/json102.test
+++ b/test/json102.test
@@ -21,156 +21,492 @@ source $testdir/tester.tcl
 do_execsql_test json102-100 {
   SELECT json_object('ex','[52,3.14159]');
 } {{{"ex":"[52,3.14159]"}}}
+do_execsql_test json102-100b {
+  SELECT json(jsonb_object('ex','[52,3.14159]'));
+} {{{"ex":"[52,3.14159]"}}}
 do_execsql_test json102-110 {
   SELECT json_object('ex',json('[52,3.14159]'));
 } {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-110-2 {
+  SELECT json(jsonb_object('ex',json('[52,3.14159]')));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-110-3 {
+  SELECT json_object('ex',jsonb('[52,3.14159]'));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-110-3 {
+  SELECT json(jsonb_object('ex',jsonb('[52,3.14159]')));
+} {{{"ex":[52,3.14159]}}}
 do_execsql_test json102-120 {
   SELECT json_object('ex',json_array(52,3.14159));
 } {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120-2 {
+  SELECT json(jsonb_object('ex',json_array(52,3.14159)));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120-3 {
+  SELECT json_object('ex',jsonb_array(52,3.14159));
+} {{{"ex":[52,3.14159]}}}
+do_execsql_test json102-120-4 {
+  SELECT json(jsonb_object('ex',jsonb_array(52,3.14159)));
+} {{{"ex":[52,3.14159]}}}
 do_execsql_test json102-130 {
   SELECT json(' { "this" : "is", "a": [ "test" ] } ');
 } {{{"this":"is","a":["test"]}}}
+do_execsql_test json102-130b {
+  SELECT json(jsonb(' { "this" : "is", "a": [ "test" ] } '));
+} {{{"this":"is","a":["test"]}}}
 do_execsql_test json102-140 {
   SELECT json_array(1,2,'3',4);
 } {{[1,2,"3",4]}}
+do_execsql_test json102-140b {
+  SELECT json(jsonb_array(1,2,'3',4));
+} {{[1,2,"3",4]}}
 do_execsql_test json102-150 {
   SELECT json_array('[1,2]');
 } {{["[1,2]"]}}
+do_execsql_test json102-150b {
+  SELECT json(jsonb_array('[1,2]'));
+} {{["[1,2]"]}}
 do_execsql_test json102-160 {
   SELECT json_array(json_array(1,2));
 } {{[[1,2]]}}
+do_execsql_test json102-160-2 {
+  SELECT json_array(jsonb_array(1,2));
+} {{[[1,2]]}}
+do_execsql_test json102-160-3 {
+  SELECT json(jsonb_array(json_array(1,2)));
+} {{[[1,2]]}}
+do_execsql_test json102-160-4 {
+  SELECT json(jsonb_array(jsonb_array(1,2)));
+} {{[[1,2]]}}
 do_execsql_test json102-170 {
   SELECT json_array(1,null,'3','[4,5]','{"six":7.7}');
 } {{[1,null,"3","[4,5]","{\"six\":7.7}"]}}
+do_execsql_test json102-170b {
+  SELECT json(jsonb_array(1,null,'3','[4,5]','{"six":7.7}'));
+} {{[1,null,"3","[4,5]","{\"six\":7.7}"]}}
 do_execsql_test json102-180 {
   SELECT json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}'));
 } {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-180-2 {
+  SELECT json_array(1,null,'3',jsonb('[4,5]'),json('{"six":7.7}'));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-180-3 {
+  SELECT json(jsonb_array(1,null,'3',json('[4,5]'),json('{"six":7.7}')));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
+do_execsql_test json102-180-4 {
+  SELECT json(jsonb_array(1,null,'3',jsonb('[4,5]'),jsonb('{"six":7.7}')));
+} {{[1,null,"3",[4,5],{"six":7.7}]}}
 do_execsql_test json102-190 {
   SELECT json_array_length('[1,2,3,4]');
 } {{4}}
+do_execsql_test json102-190b {
+  SELECT json_array_length(jsonb('[1,2,3,4]'));
+} {{4}}
+do_execsql_test json102-191 {
+  SELECT json_array_length( json_remove('[1,2,3,4]','$[2]') );
+} {{3}}
+do_execsql_test json102-191b {
+  SELECT json_array_length( jsonb_remove('[1,2,3,4]','$[2]') );
+} {{3}}
 do_execsql_test json102-200 {
   SELECT json_array_length('[1,2,3,4]', '$');
 } {{4}}
+do_execsql_test json102-200b {
+  SELECT json_array_length(jsonb('[1,2,3,4]'), '$');
+} {{4}}
 do_execsql_test json102-210 {
   SELECT json_array_length('[1,2,3,4]', '$[2]');
 } {{0}}
+do_execsql_test json102-210b {
+  SELECT json_array_length(jsonb('[1,2,3,4]'), '$[2]');
+} {{0}}
+do_execsql_test json102-220 {
+  SELECT json_array_length('{"one":[1,2,3]}');
+} {{0}}
 do_execsql_test json102-220 {
   SELECT json_array_length('{"one":[1,2,3]}');
 } {{0}}
-do_execsql_test json102-230 {
-  SELECT json_array_length('{"one":[1,2,3]}', '$.one');
+do_execsql_test json102-230b {
+  SELECT json_array_length(jsonb('{"one":[1,2,3]}'), '$.one');
 } {{3}}
 do_execsql_test json102-240 {
   SELECT json_array_length('{"one":[1,2,3]}', '$.two');
 } {{}}
+do_execsql_test json102-240b {
+  SELECT json_array_length(jsonb('{"one":[1,2,3]}'), '$.two');
+} {{}}
 do_execsql_test json102-250 {
   SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$');
 } {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-250-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$');
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-250-3 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'));
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
+do_execsql_test json102-250-4 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$'));
+} {{{"a":2,"c":[4,5,{"f":7}]}}}
 do_execsql_test json102-260 {
   SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c');
 } {{[4,5,{"f":7}]}}
+do_execsql_test json102-260-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c');
+} {{[4,5,{"f":7}]}}
+do_execsql_test json102-260-3 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'));
+} {{[4,5,{"f":7}]}}
+do_execsql_test json102-260-4 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c'));
+} {{[4,5,{"f":7}]}}
 do_execsql_test json102-270 {
   SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]');
 } {{{"f":7}}}
+do_execsql_test json102-270-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c[2]');
+} {{{"f":7}}}
+do_execsql_test json102-270-3 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c[2]'));
+} {{{"f":7}}}
+do_execsql_test json102-270-4 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'));
+} {{{"f":7}}}
 do_execsql_test json102-280 {
   SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f');
 } {{7}}
+do_execsql_test json102-280b {
+  SELECT jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f');
+} {{7}}
 do_execsql_test json102-290 {
   SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a');
 } {{[[4,5],2]}}
+do_execsql_test json102-290-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5],"f":7}'),'$.c','$.a');
+} {{[[4,5],2]}}
+do_execsql_test json102-290-3 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'));
+} {{[[4,5],2]}}
+do_execsql_test json102-290-4 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5],"f":7}'),'$.c','$.a'));
+} {{[[4,5],2]}}
 do_execsql_test json102-300 {
   SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x');
 } {{}}
+do_execsql_test json102-300b {
+  SELECT jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x');
+} {{}}
 do_execsql_test json102-310 {
   SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a');
 } {{[null,2]}}
+do_execsql_test json102-310-2 {
+  SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.x', '$.a');
+} {{[null,2]}}
+do_execsql_test json102-310-3 {
+  SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.x', '$.a'));
+} {{[null,2]}}
+do_execsql_test json102-310-43 {
+  SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a'));
+} {{[null,2]}}
 do_execsql_test json102-320 {
   SELECT json_insert('{"a":2,"c":4}', '$.a', 99);
 } {{{"a":2,"c":4}}}
+do_execsql_test json102-320-2 {
+  SELECT json_insert(jsonb('{"a":2,"c":4}'), '$.a', 99);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-320-3 {
+  SELECT json(jsonb_insert('{"a":2,"c":4}', '$.a', 99));
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-320-4 {
+  SELECT json(jsonb_insert(jsonb('{"a":2,"c":4}'), '$.a', 99));
+} {{{"a":2,"c":4}}}
 do_execsql_test json102-330 {
   SELECT json_insert('{"a":2,"c":4}', '$.e', 99);
 } {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-330-2 {
+  SELECT json_insert(jsonb('{"a":2,"c":4}'), '$.e', 99);
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-330-3 {
+  SELECT json(jsonb_insert('{"a":2,"c":4}', '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-330-4 {
+  SELECT json(jsonb_insert(jsonb('{"a":2,"c":4}'), '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
 do_execsql_test json102-340 {
   SELECT json_replace('{"a":2,"c":4}', '$.a', 99);
 } {{{"a":99,"c":4}}}
+do_execsql_test json102-340-2 {
+  SELECT json_replace(jsonb('{"a":2,"c":4}'), '$.a', 99);
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-340-3 {
+  SELECT json(jsonb_replace('{"a":2,"c":4}', '$.a', 99));
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-340-4 {
+  SELECT json(jsonb_replace(jsonb('{"a":2,"c":4}'), '$.a', 99));
+} {{{"a":99,"c":4}}}
 do_execsql_test json102-350 {
   SELECT json_replace('{"a":2,"c":4}', '$.e', 99);
 } {{{"a":2,"c":4}}}
+do_execsql_test json102-350-2 {
+  SELECT json_replace(jsonb('{"a":2,"c":4}'), '$.e', 99);
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-350-3 {
+  SELECT json(jsonb_replace('{"a":2,"c":4}', '$.e', 99));
+} {{{"a":2,"c":4}}}
+do_execsql_test json102-350-4 {
+  SELECT json(jsonb_replace(jsonb('{"a":2,"c":4}'), '$.e', 99));
+} {{{"a":2,"c":4}}}
 do_execsql_test json102-360 {
   SELECT json_set('{"a":2,"c":4}', '$.a', 99);
 } {{{"a":99,"c":4}}}
+do_execsql_test json102-360-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.a', 99);
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-360-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.a', 99));
+} {{{"a":99,"c":4}}}
+do_execsql_test json102-360-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.a', 99));
+} {{{"a":99,"c":4}}}
 do_execsql_test json102-370 {
   SELECT json_set('{"a":2,"c":4}', '$.e', 99);
 } {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-370-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.e', 99);
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-370-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
+do_execsql_test json102-370-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.e', 99));
+} {{{"a":2,"c":4,"e":99}}}
 do_execsql_test json102-380 {
   SELECT json_set('{"a":2,"c":4}', '$.c', '[97,96]');
 } {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-380-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', '[97,96]');
+} {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-380-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', '[97,96]'));
+} {{{"a":2,"c":"[97,96]"}}}
+do_execsql_test json102-380-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', '[97,96]'));
+} {{{"a":2,"c":"[97,96]"}}}
 do_execsql_test json102-390 {
   SELECT json_set('{"a":2,"c":4}', '$.c', json('[97,96]'));
 } {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', json('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', json('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', json('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-5 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', jsonb('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-6 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb('[97,96]'));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-7 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', jsonb('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-390-8 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb('[97,96]')));
+} {{{"a":2,"c":[97,96]}}}
 do_execsql_test json102-400 {
   SELECT json_set('{"a":2,"c":4}', '$.c', json_array(97,96));
 } {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-2 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', json_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-3 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', json_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-4 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', json_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-5 {
+  SELECT json_set('{"a":2,"c":4}', '$.c', jsonb_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-6 {
+  SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb_array(97,96));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-7 {
+  SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', jsonb_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
+do_execsql_test json102-400-8 {
+  SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb_array(97,96)));
+} {{{"a":2,"c":[97,96]}}}
 do_execsql_test json102-410 {
   SELECT json_object('a',2,'c',4);
 } {{{"a":2,"c":4}}}
+do_execsql_test json102-410b {
+  SELECT json(jsonb_object('a',2,'c',4));
+} {{{"a":2,"c":4}}}
 do_execsql_test json102-420 {
   SELECT json_object('a',2,'c','{e:5}');
 } {{{"a":2,"c":"{e:5}"}}}
+do_execsql_test json102-420b {
+  SELECT json(jsonb_object('a',2,'c','{e:5}'));
+} {{{"a":2,"c":"{e:5}"}}}
 do_execsql_test json102-430 {
   SELECT json_object('a',2,'c',json_object('e',5));
 } {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-430-2 {
+  SELECT json(jsonb_object('a',2,'c',json_object('e',5)));
+} {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-430-3 {
+  SELECT json_object('a',2,'c',jsonb_object('e',5));
+} {{{"a":2,"c":{"e":5}}}}
+do_execsql_test json102-430-4 {
+  SELECT json(jsonb_object('a',2,'c',jsonb_object('e',5)));
+} {{{"a":2,"c":{"e":5}}}}
 do_execsql_test json102-440 {
   SELECT json_remove('[0,1,2,3,4]','$[2]');
 } {{[0,1,3,4]}}
+do_execsql_test json102-440-2 {
+  SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[2]');
+} {{[0,1,3,4]}}
+do_execsql_test json102-440-3 {
+  SELECT json(jsonb_remove('[0,1,2,3,4]','$[2]'));
+} {{[0,1,3,4]}}
+do_execsql_test json102-440-4 {
+  SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]'));
+} {{[0,1,3,4]}}
 do_execsql_test json102-450 {
   SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]');
 } {{[1,3,4]}}
+do_execsql_test json102-450-2 {
+  SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[2]','$[0]');
+} {{[1,3,4]}}
+do_execsql_test json102-450-3 {
+  SELECT json(jsonb_remove('[0,1,2,3,4]','$[2]','$[0]'));
+} {{[1,3,4]}}
+do_execsql_test json102-450-4 {
+  SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]','$[0]'));
+} {{[1,3,4]}}
 do_execsql_test json102-460 {
   SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]');
 } {{[1,2,4]}}
+do_execsql_test json102-460-2 {
+  SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[0]','$[2]');
+} {{[1,2,4]}}
+do_execsql_test json102-460-3 {
+  SELECT json(jsonb_remove('[0,1,2,3,4]','$[0]','$[2]'));
+} {{[1,2,4]}}
+do_execsql_test json102-460-4 {
+  SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[0]','$[2]'));
+} {{[1,2,4]}}
 do_execsql_test json102-470 {
   SELECT json_remove('{"x":25,"y":42}');
 } {{{"x":25,"y":42}}}
+do_execsql_test json102-470-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'));
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-470-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}'));
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-470-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}')));
+} {{{"x":25,"y":42}}}
 do_execsql_test json102-480 {
   SELECT json_remove('{"x":25,"y":42}','$.z');
 } {{{"x":25,"y":42}}}
+do_execsql_test json102-480-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'),'$.z');
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-480-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}','$.z'));
+} {{{"x":25,"y":42}}}
+do_execsql_test json102-480-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$.z'));
+} {{{"x":25,"y":42}}}
 do_execsql_test json102-490 {
   SELECT json_remove('{"x":25,"y":42}','$.y');
 } {{{"x":25}}}
+do_execsql_test json102-490-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'),'$.y');
+} {{{"x":25}}}
+do_execsql_test json102-490-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}','$.y'));
+} {{{"x":25}}}
+do_execsql_test json102-490-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$.y'));
+} {{{"x":25}}}
 do_execsql_test json102-500 {
   SELECT json_remove('{"x":25,"y":42}','$');
 } {{}}
+do_execsql_test json102-500-2 {
+  SELECT json_remove(jsonb('{"x":25,"y":42}'),'$');
+} {{}}
+do_execsql_test json102-500-3 {
+  SELECT json(jsonb_remove('{"x":25,"y":42}','$'));
+} {{}}
+do_execsql_test json102-500-4 {
+  SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$'));
+} {{}}
 do_execsql_test json102-510 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}');
 } {{object}}
+do_execsql_test json102-510b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778');
+} {{object}}
 do_execsql_test json102-520 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$');
 } {{object}}
+do_execsql_test json102-520b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$');
+} {{object}}
 do_execsql_test json102-530 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a');
 } {{array}}
+do_execsql_test json102-530b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a');
+} {{array}}
 do_execsql_test json102-540 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]');
 } {{integer}}
+do_execsql_test json102-540b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[0]');
+} {{integer}}
 do_execsql_test json102-550 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]');
 } {{real}}
+do_execsql_test json102-550b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[1]');
+} {{real}}
 do_execsql_test json102-560 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]');
 } {{true}}
+do_execsql_test json102-560b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[2]');
+} {{true}}
 do_execsql_test json102-570 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]');
 } {{false}}
+do_execsql_test json102-570b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[3]');
+} {{false}}
 do_execsql_test json102-580 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]');
 } {{null}}
+do_execsql_test json102-580b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[4]');
+} {{null}}
 do_execsql_test json102-590 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]');
 } {{text}}
+do_execsql_test json102-590b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[5]');
+} {{text}}
 do_execsql_test json102-600 {
   SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]');
 } {{}}
+do_execsql_test json102-600b {
+  SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[6]');
+} {{}}
 do_execsql_test json102-610 {
   SELECT json_valid(char(123)||'"x":35'||char(125));
 } {{1}}
@@ -180,17 +516,24 @@ do_execsql_test json102-620 {
 
 ifcapable vtab {
 do_execsql_test json102-1000 {
-  CREATE TABLE user(name,phone);
+  CREATE TABLE user(name,phone,phoneb);
   INSERT INTO user(name,phone) VALUES
      ('Alice','["919-555-2345","804-555-3621"]'),
      ('Bob','["201-555-8872"]'),
      ('Cindy','["704-555-9983"]'),
      ('Dave','["336-555-8421","704-555-4321","803-911-4421"]');
+  UPDATE user SET phoneb=jsonb(phone);
   SELECT DISTINCT user.name
     FROM user, json_each(user.phone)
    WHERE json_each.value LIKE '704-%'
    ORDER BY 1;
 } {Cindy Dave}
+do_execsql_test json102-1000b {
+  SELECT DISTINCT user.name
+    FROM user, json_each(user.phoneb)
+   WHERE json_each.value LIKE '704-%'
+   ORDER BY 1;
+} {Cindy Dave}
 
 do_execsql_test json102-1010 {
   UPDATE user
@@ -250,6 +593,12 @@ do_execsql_test json102-1110 {
    WHERE json_tree.type NOT IN ('object','array')
    ORDER BY +big.rowid, +json_tree.id
 } $correct_answer
+do_execsql_test json102-1110b {
+  SELECT big.rowid, fullkey, value
+    FROM big, json_tree(jsonb(big.json))
+   WHERE json_tree.type NOT IN ('object','array')
+   ORDER BY +big.rowid, +json_tree.id
+} $correct_answer
 do_execsql_test json102-1120 {
   SELECT big.rowid, fullkey, atom
     FROM big, json_tree(big.json)
@@ -396,5 +745,78 @@ do_execsql_test json102-1610 {
   5 {{"b":9}} json {{"b":9}} text    {{"b":9}} json       \
   6 {}        null {}        null    {}        null
 ]
+do_execsql_test json102-1620 {
+  DELETE FROM t1;
+  INSERT INTO t1(x) VALUES('[null,123,4.5,"six",[7,8],{"b":9}]');
+  WITH c(y) AS (VALUES(0),(1),(2),(3),(4),(5),(6))
+  SELECT
+    y,
+    x->y AS '->',
+    CASE WHEN subtype(if(json_valid(x),x->y)) THEN 'json'
+         ELSE typeof(x->y) END AS 'type',
+    x->>y AS '->>',
+    CASE WHEN subtype(x->>y) THEN 'json' ELSE typeof(x->>y) END AS 'type',
+    json_extract(x,format('$[%d]',y)) AS 'json_extract',
+    CASE WHEN subtype(json_extract(x,format('$[%d]',y)))
+      THEN 'json' ELSE typeof(json_extract(x,format('$[%d]',y))) END AS 'type'
+  FROM c, t1 ORDER BY y;    
+} [list \
+  0 null      json {}        null    {}        null       \
+  1 123       json 123       integer 123       integer    \
+  2 4.5       json 4.5       real    4.5       real       \
+  3 {"six"}   json six       text    six       text       \
+  4 {[7,8]}   json {[7,8]}   text    {[7,8]}   json       \
+  5 {{"b":9}} json {{"b":9}} text    {{"b":9}} json       \
+  6 {}        null {}        null    {}        null
+]
+
+reset_db
+do_execsql_test json102-1700 {
+  CREATE TABLE t1(a1 DATE, a2 INTEGER PRIMARY KEY, a3 INTEGER, memo TEXT);
+  CREATE INDEX t1x1 ON t1(a3, a1, memo->>'y');
+  INSERT INTO t1(a2,a1,a3,memo) VALUES (876, '2023-08-03', 5, '{"x":77,"y":4}');
+}
+do_execsql_test json102-1710 {
+  UPDATE t1 SET memo = JSON_REMOVE(memo, '$.y');
+  PRAGMA integrity_check;
+  SELECT * FROM t1;
+} {ok 2023-08-03 876 5 {{"x":77}}}
+do_execsql_test json102-1720 {
+  UPDATE t1 SET memo = JSON_SET(memo, '$.y', 6)
+    WHERE a2 IN (876) AND JSON_TYPE(memo, '$.y') IS NULL;
+  PRAGMA integrity_check;
+  SELECT * FROM t1;
+} {ok 2023-08-03 876 5 {{"x":77,"y":6}}}
+
+# 2024-05-21 https://sqlite.org/forum/forumpost/9e52cdfe15c3926e
+# What if the RHS of the -> or ->> operator is a string that looks
+# like a number?  PostgreSQL treats it as a string.
+#
+do_execsql_test json102-1800 {
+  SELECT '{"1":"one","2":"two","3":"three"}'->>'2';
+} two
+db null NULL
+do_execsql_test json102-1801 {
+  SELECT '{"1":"one","2":"two","3":"three"}'->>2;
+} NULL
+do_execsql_test json102-1810 {
+  SELECT '["zero","one","two"]'->>'1';
+} NULL
+do_execsql_test json102-1811 {
+  SELECT '["zero","one","two"]'->>1;
+} one
+do_execsql_test json102-1820 {
+  SELECT '{"1":"one","2":"two","3":"three"}'->'2';
+} {{"two"}}
+do_execsql_test json102-1821 {
+  SELECT '{"1":"one","2":"two","3":"three"}'->2;
+} {NULL}
+do_execsql_test json102-1830 {
+  SELECT '["zero","one","two"]'->'1';
+} {NULL}
+do_execsql_test json102-1831 {
+  SELECT '["zero","one","two"]'->1;
+} {{"one"}}
+
 
 finish_test
diff --git a/test/json103.test b/test/json103.test
index e7483073b6..f94217ac10 100644
--- a/test/json103.test
+++ b/test/json103.test
@@ -55,7 +55,7 @@ do_execsql_test json103-220 {
    WHERE rowid<7 GROUP BY b ORDER BY b;
 } {0 {{"n3":3,"n6":6}} 1 {{"n1":1,"n4":4}} 2 {{"n2":2,"n5":5}}}
 
-# ticket https://www.sqlite.org/src/info/f45ac567eaa9f93c 2016-01-30
+# ticket https://sqlite.org/src/info/f45ac567eaa9f93c 2016-01-30
 # Invalid JSON generated by json_group_array() 
 #
 # The underlying problem is a failure to reset Mem.eSubtype
diff --git a/test/json105.test b/test/json105.test
index 83face188e..509db94e11 100644
--- a/test/json105.test
+++ b/test/json105.test
@@ -13,7 +13,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-set testprefix json104
+set testprefix json105
 
 # This is the example from pages 2 and 3 of RFC-7396
 db eval {
@@ -96,18 +96,18 @@ json_replace_test 80 {'$.b[#-1]','AAA','$.b[#-1]','BBB'} \
 
 do_catchsql_test json105-6.10 {
   SELECT json_extract(j, '$.b[#-]') FROM t1;
-} {1 {JSON path error near '[#-]'}}
+} {1 {bad JSON path: '$.b[#-]'}}
 do_catchsql_test json105-6.20 {
   SELECT json_extract(j, '$.b[#9]') FROM t1;
-} {1 {JSON path error near '[#9]'}}
+} {1 {bad JSON path: '$.b[#9]'}}
 do_catchsql_test json105-6.30 {
   SELECT json_extract(j, '$.b[#+2]') FROM t1;
-} {1 {JSON path error near '[#+2]'}}
+} {1 {bad JSON path: '$.b[#+2]'}}
 do_catchsql_test json105-6.40 {
   SELECT json_extract(j, '$.b[#-1') FROM t1;
-} {1 {JSON path error near '[#-1'}}
+} {1 {bad JSON path: '$.b[#-1'}}
 do_catchsql_test json105-6.50 {
   SELECT json_extract(j, '$.b[#-1x]') FROM t1;
-} {1 {JSON path error near '[#-1x]'}}
+} {1 {bad JSON path: '$.b[#-1x]'}}
 
 finish_test
diff --git a/test/json106.test b/test/json106.test
new file mode 100644
index 0000000000..06859a10b4
--- /dev/null
+++ b/test/json106.test
@@ -0,0 +1,79 @@
+# 2023-12-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Invariant tests for JSON built around the randomjson extension
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix json106
+
+# These tests require virtual table "json_tree" to run.
+ifcapable !vtab { finish_test ; return }
+
+load_static_extension db randomjson
+db eval {
+  CREATE TEMP TABLE t1(j0,j5,p);
+  CREATE TEMP TABLE kv(n,key,val);
+}
+unset -nocomplain ii
+for {set ii 1} {$ii<=5000} {incr ii} {
+  do_execsql_test $ii.1 {
+    DELETE FROM t1;
+    INSERT INTO t1(j0,j5) VALUES(random_json($ii),random_json5($ii));
+    SELECT json_valid(j0), json_valid(j5,2) FROM t1;
+  } {1 1}
+  do_execsql_test $ii.2 {
+    SELECT count(*)
+      FROM t1, json_tree(j0) AS rt
+     WHERE rt.type NOT IN ('object','array')
+       AND rt.atom IS NOT (j0 ->> rt.fullkey);
+  } 0
+  do_execsql_test $ii.3 {
+    SELECT count(*)
+      FROM t1, json_tree(j5) AS rt
+     WHERE rt.type NOT IN ('object','array')
+       AND rt.atom IS NOT (j0 ->> rt.fullkey);
+  } 0
+  do_execsql_test $ii.4 {
+    DELETE FROM kv;
+    INSERT INTO kv
+      SELECT rt.rowid, rt.fullkey, rt.atom
+        FROM t1, json_tree(j0) AS rt
+       WHERE rt.type NOT IN ('object','array');
+  }
+  do_execsql_test $ii.5 {
+    SELECT count(*)
+      FROM t1, kv
+     WHERE key NOT LIKE '%]'
+       AND json_remove(j5,key)->>key IS NOT NULL
+  } 0
+  do_execsql_test $ii.6 {
+    SELECT count(*)
+      FROM t1, kv
+     WHERE key NOT LIKE '%]'
+       AND json_insert(json_remove(j5,key),key,val)->>key IS NOT val
+  } 0
+  do_execsql_test $ii.7 {
+    UPDATE t1 SET p=json_patch(j0,j5);
+    SELECT count(*)
+      FROM t1, kv
+     WHERE p->>key IS NOT val
+  } 0
+  do_execsql_test $ii.8 {
+    SELECT j0 FROM t1 WHERE json(j0)!=json(json_pretty(j0));
+  } {}
+  do_execsql_test $ii.9 {
+    SELECT j5 FROM t1 WHERE json(j5)!=json(json_pretty(j5));
+  } {}
+}
+
+
+finish_test
diff --git a/test/json107.test b/test/json107.test
new file mode 100644
index 0000000000..779b557fba
--- /dev/null
+++ b/test/json107.test
@@ -0,0 +1,86 @@
+# 2024-01-23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Legacy JSON bug:  If the input is a BLOB that when cast into TEXT looks
+# like valid JSON, then treat it as valid JSON.
+#
+# The original intent of the JSON functions was to raise an error on any
+# BLOB input.  That intent was clearly documented, but the code failed to
+# to implement it.  Subsequently, many applications began to depend on the
+# incorrect behavior, especially apps that used readfile() to read JSON
+# content, since readfile() returns a BLOB.  So we need to support the
+# bug moving forward.
+#
+# The tests in this fail verify that the original buggy behavior is
+# preserved.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix json107
+
+if {[db one {PRAGMA encoding}]!="UTF-8"} {
+  # These tests only work for a UTF-8 encoding.
+  finish_test
+  return
+}
+
+do_execsql_test 1.1 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB) );
+} 1
+do_execsql_test 1.1.1 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 1);
+} 1
+do_execsql_test 1.1.2 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 2);
+} 1
+do_execsql_test 1.1.4 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 4);
+} 0
+do_execsql_test 1.1.8 {
+  SELECT json_valid( CAST('{"a":1}' AS BLOB), 8);
+} 0
+
+do_execsql_test 1.2.1 {
+  SELECT CAST('{"a":123}' AS blob) -> 'a';
+} 123
+do_execsql_test 1.2.2 {
+  SELECT CAST('{"a":123}' AS blob) ->> 'a';
+} 123
+do_execsql_test 1.2.3 {
+  SELECT json_extract(CAST('{"a":123}' AS blob), '$.a');
+} 123
+do_execsql_test 1.3 {
+  SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456);
+} {{{"a":123,"b":456}}}
+do_execsql_test 1.4 {
+  SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a');
+} {{{"b":456}}}
+do_execsql_test 1.5 {
+  SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
+} {{{"a":789,"b":456}}}
+do_execsql_test 1.6 {
+  SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789);
+} {{{"a":789,"b":456}}}
+do_execsql_test 1.7 {
+  SELECT json_type(CAST('{"a":123,"b":456}' AS blob));
+} object
+do_execsql_test 1.8 {
+  SELECT json(CAST('{"a":123,"b":456}' AS blob));
+} {{{"a":123,"b":456}}}
+
+ifcapable vtab {
+  do_execsql_test 2.1 {
+    SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) )
+      WHERE atom;
+  } {a 123 b 456}
+} 
+finish_test
diff --git a/test/json108.test b/test/json108.test
new file mode 100644
index 0000000000..71f3814dce
--- /dev/null
+++ b/test/json108.test
@@ -0,0 +1,45 @@
+# 2024-03-06
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Invariant tests for JSON built around the randomjson extension
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix json108
+
+# These tests require virtual table "json_tree" to run.
+ifcapable !vtab { finish_test ; return }
+
+load_static_extension db randomjson
+db eval {
+  CREATE TEMP TABLE t1(j0,j5);
+  WITH RECURSIVE c(n) AS (VALUES(0) UNION ALL SELECT n+1 FROM c WHERE n<9)
+  INSERT INTO t1 SELECT random_json(n), random_json5(n) FROM c;
+}
+
+do_execsql_test 1.1 {
+  SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,NULL));
+} 10
+do_execsql_test 1.2 {
+  SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,NULL));
+} 10
+do_execsql_test 1.3 {
+  SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,''));
+} 10
+do_execsql_test 1.4 {
+  SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,char(9)));
+} 10
+do_execsql_test 1.5 {
+  SELECT count(*) FROM t1 WHERE json(j0)==json(json_pretty(j0,'/*hello*/'));
+} 10
+
+
+finish_test
diff --git a/test/json501.test b/test/json501.test
index a37326973a..bfd21055a7 100644
--- a/test/json501.test
+++ b/test/json501.test
@@ -252,13 +252,13 @@ do_execsql_test 8.11 {
 
 do_execsql_test 9.1 {
   WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {Inf {{"x":9.0e999}}}
+} {Inf {{"x":9e999}}}
 do_execsql_test 9.2 {
   WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {-Inf {{"x":-9.0e999}}}
+} {-Inf {{"x":-9e999}}}
 do_execsql_test 9.3 {
   WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c;
-} {Inf {{"x":9.0e999}}}
+} {Inf {{"x":9e999}}}
 do_execsql_test 9.4 {
   WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c;
 } {{} {{"x":null}}}
@@ -300,5 +300,37 @@ do_execsql_test 12.4 {
           || ' "xyz"}')->>'a';
 } xyz
 
+# 2023-11-08 forum/forumpost/ddcad3e884
+#
+do_execsql_test 13.1 {
+  SELECT json('{x:''a "b" c''}');
+} {{{"x":"a \"b\" c"}}}
+
+# 2024-01-31
+# Allow control characters within JSON5 string literals.
+#
+for {set c 1} {$c<=0x1f} {incr c} {
+  do_execsql_test 14.$c.1 {
+    SELECT json_valid('"abc' || char($c) || 'xyz"');
+  } {0}
+  do_execsql_test 14.$c.2 {
+    SELECT json_valid('"abc' || char($c) || 'xyz"', 2);
+  } {1}
+  switch $c {
+    8   {set e "\\b"}
+    9   {set e "\\t"}
+    10  {set e "\\n"}
+    12  {set e "\\f"}
+    13  {set e "\\r"}
+    default {set e [format "\\u00%02x" $c]}
+  }
+  do_execsql_test 14.$c.3 {
+    SELECT json('{label:"abc' || char($c) || 'xyz"}');
+  } "{{\"label\":\"abc${e}xyz\"}}"
+  do_execsql_test 14.$c.4 {
+    SELECT jsonb('{label:"abc' || char($c) || 'xyz"}') -> '$';
+  } "{{\"label\":\"abc${e}xyz\"}}"
+}
+
 
 finish_test
diff --git a/test/json502.test b/test/json502.test
index 595bf63318..1eba00dba5 100644
--- a/test/json502.test
+++ b/test/json502.test
@@ -36,5 +36,45 @@ do_catchsql_test 2.3 {
   SELECT '{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}'->'$h[#-1]';
 } {1 {malformed JSON}}
 
+# Verify that escaped label names are compared correctly.
+# 
+do_execsql_test 3.1 {
+  SELECT '{"a\x62c":123}' ->> 'abc';
+} 123
+do_execsql_test 3.2 {
+  SELECT '{"abc":123}' ->> 'a\x62c';
+} 123
+
+db null null
+do_execsql_test 3.3 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222));
+  INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222));
+  SELECT x->'$.a\', x->'$.a\\', x->'$."a\\"', x->'$."b\\"' FROM t1;
+} {111 null 111 222 111 null 111 222}
+
+do_execsql_test 3.4 {
+  SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc';
+} 456
+
+ifcapable vtab {
+  do_execsql_test 4.1 {
+    SELECT * FROM json_tree('{"\u0017":1}','$."\x17"');
+  } {{\x17} 1 integer 1 1 null {$."\x17"} {$}}
+}
+
+# JSON PATH parsing bug involving backslash escapes, reported via
+# private email from Florent De'Neve on 2024-09-04.
+#
+do_execsql_test 5.1 {
+  SELECT json_extract('{"A\"Key":1}', '$.A"Key');
+} 1
+do_execsql_test 5.2 {
+  SELECT json_extract('{"A\"Key":1}', '$."A\"Key"');
+} 1
+do_execsql_test 5.3 {
+  SELECT JSON_SET('{}', '$."\"Key"', 1);
+} {{{"\"Key":1}}}
 
 finish_test
diff --git a/test/jsonb01.test b/test/jsonb01.test
new file mode 100644
index 0000000000..8f16428dcc
--- /dev/null
+++ b/test/jsonb01.test
@@ -0,0 +1,53 @@
+# 2023-11-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Test cases for JSONB
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_execsql_test jsonb01-1.1 {
+  CREATE TABLE t1(x JSON BLOB);
+  INSERT INTO t1 VALUES(jsonb('{a:5,b:{x:10,y:11},c:[1,2,3,4]}'));
+}
+foreach {id path res} {
+  1 {$.a}    {{{"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+  2 {$.b}    {{{"a":5,"c":[1,2,3,4]}}}
+  3 {$.c}    {{{"a":5,"b":{"x":10,"y":11}}}}
+  4 {$.d}    {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+  5 {$.b.x}  {{{"a":5,"b":{"y":11},"c":[1,2,3,4]}}}
+  6 {$.b.y}  {{{"a":5,"b":{"x":10},"c":[1,2,3,4]}}}
+  7 {$.c[0]}  {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}}
+  8 {$.c[1]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}}
+  9 {$.c[2]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}}
+ 10 {$.c[3]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}}
+ 11 {$.c[4]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 12 {$.c[#]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 13 {$.c[#-1]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}}
+ 14 {$.c[#-2]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}}
+ 15 {$.c[#-3]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}}
+ 16 {$.c[#-4]}  {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}}
+ 17 {$.c[#-5]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+ 18 {$.c[#-6]}  {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}}
+} {
+  do_execsql_test jsonb01-1.2.$id.1 {
+    SELECT json(jsonb_remove(x,$path)) FROM t1;
+  } $res
+  do_execsql_test jsonb01-1.2.$id.2 {
+    SELECT json_remove(x,$path) FROM t1;
+  } $res
+}
+
+do_catchsql_test jsonb01-2.0 {
+  SELECT x'8ce6ffffffff171333' -> '$';
+} {1 {malformed JSON}}
+
+finish_test
diff --git a/test/kvtest.c b/test/kvtest.c
index 04dc01045c..624c80b746 100644
--- a/test/kvtest.c
+++ b/test/kvtest.c
@@ -161,7 +161,7 @@ static const char zHelp[] =
 #endif
 
 /*
-** Show thqe help text and quit.
+** Show the help text and quit.
 */
 static void showHelp(void){
   fprintf(stdout, "%s", zHelp);
diff --git a/test/lemon-test01.y b/test/lemon-test01.y
index 0fd514ff3a..67890c6376 100644
--- a/test/lemon-test01.y
+++ b/test/lemon-test01.y
@@ -2,6 +2,11 @@
 //
 //     lemon lemon-test01.y && gcc -g lemon-test01.c && ./a.out
 //
+// This testcase was made obsolete by check-in 7cca80808cef192f on
+// 2021-08-17 (associated with Forum Thread 
+// https://sqlite.org/forum/forumpost/bd91fd965c9803c4) and no longer
+// works.  It is retained for historical reference only.
+//
 %token_prefix TK_
 %token_type   int
 %default_type int
@@ -28,7 +33,7 @@ all ::=  error B.
   #include "lemon-test01.h"
   static int nTest = 0;
   static int nErr = 0;
-  static int testCase(int testId, int shouldBe, int actual){
+  static void testCase(int testId, int shouldBe, int actual){
     nTest++;
     if( shouldBe==actual ){
       printf("test %d: ok\n", testId);
diff --git a/test/like.test b/test/like.test
index e8662dc6c3..1f9a810e5d 100644
--- a/test/like.test
+++ b/test/like.test
@@ -731,16 +731,16 @@ ifcapable like_opt&&!icu {
   }
   do_test like-9.5.1 {
     set res [sqlite3_exec_hex db {
-       SELECT x FROM t2 WHERE x LIKE '%fe%25'
+       SELECT 1 FROM t2 WHERE x LIKE '%fe%25'
     }]
-  } {0 {}}
+  } {0 {1 1}}
   ifcapable explain {
     do_test like-9.5.2 {
       set res [sqlite3_exec_hex db {
          EXPLAIN QUERY PLAN SELECT x FROM t2 WHERE x LIKE '%fe%25'
       }]
       regexp {INDEX i2} $res
-    } {1}
+    } {0}
   }
 
   # Do an SQL statement.  Append the search count to the end of the result.
@@ -1013,7 +1013,7 @@ do_execsql_test like-12.16 {
   SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id;
 } {/SCAN/}
 
-# Ticket [https://www.sqlite.org/src/tktview/80369eddd5c94d49f7fbbcf5]
+# Ticket [https://sqlite.org/src/tktview/80369eddd5c94d49f7fbbcf5]
 # 2016-01-20
 #
 do_execsql_test like-13.1 {
@@ -1140,4 +1140,24 @@ do_execsql_test 17.1 {
 } {1}
 
 
+# 2023-08-15 https://sqlite.org/forum/forumpost/925dc9f67804c540
+#
+reset_db
+sqlite3_db_config db DEFENSIVE 1
+db eval {PRAGMA trusted_schema=OFF}
+do_execsql_test 18.0 {
+  CREATE TABLE t1(x INT, y TEXT);
+  INSERT INTO t1 VALUES(1,'abc'),(2,'ABC'),(3,'Abc');
+  CREATE VIEW t2 AS SELECT * FROM t1 WHERE y LIKE 'a%';
+  SELECT * FROM t2;
+} {1 abc 2 ABC 3 Abc}
+do_execsql_test 18.1 {
+  PRAGMA case_sensitive_like=OFF;
+  SELECT * FROM t2;
+} {1 abc 2 ABC 3 Abc}
+do_execsql_test 18.2 {
+  PRAGMA case_sensitive_like=ON;
+  SELECT * FROM t2;
+} {1 abc}
+
 finish_test
diff --git a/test/like3.test b/test/like3.test
index a93e113d62..03681606c3 100644
--- a/test/like3.test
+++ b/test/like3.test
@@ -112,7 +112,7 @@ do_execsql_test like3-4.2ck {
   SELECT quote(x) FROM t4 WHERE x LIKE 'ab%' ORDER BY +x ASC;
 } {'abc' 'abd' 'abe' X'616263' X'616264' X'616265'}
 
-# 2018-09-10 ticket https://www.sqlite.org/src/tktview/c94369cae9b561b1f996
+# 2018-09-10 ticket https://sqlite.org/src/tktview/c94369cae9b561b1f996
 # The like optimization fails for a column with numeric affinity if
 # the pattern '/%' or begins with the escape character.
 #
@@ -199,7 +199,7 @@ do_execsql_test like3-5.400 {
 } {./}
 
 # 2019-06-14
-# Ticket https://www.sqlite.org/src/info/ce8717f0885af975
+# Ticket https://sqlite.org/src/info/ce8717f0885af975
 do_execsql_test like3-5.410 {
   DROP TABLE IF EXISTS t0;
   CREATE TABLE t0(c0 INT UNIQUE COLLATE NOCASE);
@@ -208,7 +208,7 @@ do_execsql_test like3-5.410 {
 } {.1%}
 
 # 2019-09-03
-# Ticket https://www.sqlite.org/src/info/0f0428096f
+# Ticket https://sqlite.org/src/info/0f0428096f
 do_execsql_test like3-5.420 {
   DROP TABLE IF EXISTS t0;
   CREATE TABLE t0(c0 UNIQUE);
@@ -275,4 +275,92 @@ do_eqp_test like3-6.240 {
 }
 }
 
+#-------------------------------------------------------------------------
+
+ifcapable utf16 {
+  reset_db
+  do_execsql_test like3-7.0 {
+    PRAGMA encoding = 'UTF-16be';
+  
+    CREATE TABLE Example(word TEXT NOT NULL);
+    CREATE INDEX Example_word on Example(word);
+  
+    INSERT INTO Example VALUES(char(0x307F));
+  }
+  
+  do_execsql_test like3-7.1 {
+    SELECT char(0x307F)=='み';
+  } {1}
+  
+  do_execsql_test like3-7.1 {
+    SELECT * FROM Example WHERE word GLOB 'み*'
+  } {み}
+  
+  do_execsql_test like3-7.2 {
+    SELECT * FROM Example WHERE word >= char(0x307F) AND word < char(0x3080);
+  } {み}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+# See forum thread https://sqlite.org/forum/info/d7b90d92ffbfc61f
+foreach enc {
+  UTF-8
+  UTF-16le 
+  UTF-16be
+} {
+  ifcapable icu {
+    if {$enc=="UTF-8"} {
+      # The invalid UTF8 used in these tests is incompatible with ICU
+      # https://sqlite.org/forum/forumpost/2ca8a09a7e
+      continue
+    }
+  }
+  foreach {tn expr} {
+    1 "CAST (X'FF' AS TEXT)"
+    2 "CAST (X'FFBF' AS TEXT)"
+    3 "CAST (X'FFBFBF' AS TEXT)"
+    4 "CAST (X'FFBFBFBF' AS TEXT)"
+
+    5 "'abc' || CAST (X'FF' AS TEXT)"
+    6 "'def' || CAST (X'FFBF' AS TEXT)"
+    7 "'ghi' || CAST (X'FFBFBF' AS TEXT)"
+    8 "'jkl' || CAST (X'FFBFBFBF' AS TEXT)"
+  } {
+    reset_db
+    execsql "PRAGMA encoding = '$enc'"
+    set tn utf[string range $enc 4 end].$tn
+    do_execsql_test like3-8.$tn.1 {
+      CREATE TABLE t1(x);
+    }
+  
+    do_execsql_test like3-8.$tn.2 {
+      PRAGMA encoding
+    } $enc
+  
+    do_execsql_test like3-8.$tn.3 "
+      INSERT INTO t1 VALUES( $expr )
+    "
+  
+    do_execsql_test like3-8.$tn.4 {
+      SELECT typeof(x) FROM t1
+    } {text}
+  
+    set x [db one {SELECT x || '%' FROM t1}]
+  
+    do_execsql_test like3-8.$tn.5 {
+      SELECT rowid FROM t1 WHERE x LIKE $x
+    } 1
+  
+    do_execsql_test like3-8.$tn.6 {
+      CREATE INDEX i1 ON t1(x);
+    }
+  
+    do_execsql_test like3-8.$tn.7 {
+      SELECT rowid FROM t1 WHERE x LIKE $x
+    } 1
+  }
+}
+
 finish_test
diff --git a/test/limit2.test b/test/limit2.test
index c03f39cd9c..2ecd8ab5f5 100644
--- a/test/limit2.test
+++ b/test/limit2.test
@@ -98,7 +98,7 @@ do_execsql_test limit2-210 {
 } {1 1 {} {} | 3 3 {} {} | 4 4 {} {} |}
 
 # Bug in the ORDER BY LIMIT optimization reported on 2016-09-06.
-# Ticket https://www.sqlite.org/src/info/559733b09e96
+# Ticket https://sqlite.org/src/info/559733b09e96
 #
 do_execsql_test limit2-300 {
   CREATE TABLE t300(a,b,c);
@@ -150,11 +150,11 @@ do_execsql_test 502 {
   SELECT j FROM t502 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3;
 } {1 3 4}
 
-# Ticket https://www.sqlite.org/src/info/123c9ba32130a6c9 2017-12-13
+# Ticket https://sqlite.org/src/info/123c9ba32130a6c9 2017-12-13
 # Incorrect result when an idnex is used for an ordered join.
 #
 # This test case is in the limit2.test module because the problem was first
-# exposed by check-in https://www.sqlite.org/src/info/559733b09e which 
+# exposed by check-in https://sqlite.org/src/info/559733b09e which 
 # implemented the ORDER BY LIMIT optimization that limit2.test strives to
 # test.
 #
@@ -167,7 +167,7 @@ do_execsql_test 600 {
   SELECT y FROM t1, t2 WHERE a=x AND b<=y ORDER BY b DESC;
 } {3}
 
-# Ticket https://www.sqlite.org/src/info/9936b2fa443fec03 2018-09-08
+# Ticket https://sqlite.org/src/info/9936b2fa443fec03 2018-09-08
 # Infinite loop due to the ORDER BY LIMIT optimization.
 #
 do_execsql_test 700 {
diff --git a/test/literal.test b/test/literal.test
new file mode 100644
index 0000000000..5aa331e39b
--- /dev/null
+++ b/test/literal.test
@@ -0,0 +1,103 @@
+# 2024-01-19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements tests for SQL literals
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix literal
+
+proc test_literal {tn lit type val} {
+  do_execsql_test $tn.1 "SELECT typeof( $lit ), $lit" [list $type $val]
+
+  ifcapable altertable {
+    do_execsql_test $tn.2 "
+      DROP TABLE IF EXISTS x1;
+      CREATE TABLE x1(a);
+      INSERT INTO x1 VALUES(123);
+      ALTER TABLE x1 ADD COLUMN b DEFAULT $lit ;
+      SELECT typeof(b), b FROM x1;
+    " [list $type $val]
+  }
+
+  do_execsql_test $tn.3 "
+    DROP TABLE IF EXISTS x1;
+    CREATE TABLE x1(a DEFAULT $lit);
+    INSERT INTO x1 DEFAULT VALUES;
+    SELECT typeof(a), a FROM x1;
+  " [list $type $val]
+}
+
+proc test_literal_error {tn lit unrec} {
+  do_catchsql_test $tn "SELECT $lit" "1 {unrecognized token: \"$unrec\"}"
+}
+
+
+test_literal 1.0 45                  integer 45
+test_literal 1.1 0xFF                integer 255
+test_literal 1.2 0xFFFFFFFF          integer [expr 0xFFFFFFFF]
+test_literal 1.3 0x123FFFFFFFF       integer [expr 0x123FFFFFFFF]
+test_literal 1.4 -0x123FFFFFFFF      integer [expr -1 * 0x123FFFFFFFF]
+test_literal 1.5 0xFFFFFFFFFFFFFFFF  integer -1
+test_literal 1.7 0x7FFFFFFFFFFFFFFF  integer  [expr 0x7FFFFFFFFFFFFFFF]
+test_literal 1.8 -0x7FFFFFFFFFFFFFFF integer [expr -0x7FFFFFFFFFFFFFFF]
+test_literal 1.9 +0x7FFFFFFFFFFFFFFF integer [expr +0x7FFFFFFFFFFFFFFF]
+test_literal 1.10 -45                integer -45
+test_literal 1.11 '0xFF'             text 0xFF
+test_literal 1.12 '-0xFF'            text -0xFF
+test_literal 1.13 -'0xFF'            integer 0
+test_literal 1.14 -9223372036854775808 integer -9223372036854775808
+
+test_literal 2.1  1e12    real    1000000000000.0
+test_literal 2.2  1.0     real    1.0
+test_literal 2.3  1e1000  real    Inf
+test_literal 2.4  -1e1000  real   -Inf
+
+test_literal 3.1  1_000   integer 1000
+test_literal 3.2  1.1_1   real    1.11
+test_literal 3.3  1_0.1_1 real    10.11
+test_literal 3.4  1e1_000 real    Inf
+test_literal 3.5  12_3_456.7_8_9 real 123456.789
+test_literal 3.6  9_223_372_036_854_775_807 integer 9223372036854775807
+test_literal 3.7  9_223_372_036_854_775_808 real 9.22337203685478e+18
+test_literal 3.8  -9_223_372_036_854_775_808 integer -9223372036854775808
+
+foreach {tn lit unrec} {
+  0    123a456       123a456
+  1    1_            1_
+  2    1_.4          1_.4
+  3    1e_4          1e_4
+  4    1_e4          1_e4
+  5    1.4_e4        1.4_e4
+  6    1.4e+_4       1.4e
+  7    1.4e-_4       1.4e
+  8    1.4e4_        1.4e4_
+  9    1.4_e4        1.4_e4
+  10   1.4e_4        1.4e_4
+  11   12__34        12__34
+  12   1234_         1234_
+  13   12._34        12._34
+  14   12_.34        12_.34
+  15   12.34_        12.34_
+  16   1.0e1_______2 1.0e1_______2 
+} {
+  test_literal_error 4.$tn $lit $unrec
+}
+
+# dbsqlfuzz e3186a9e7826e9cd7f4085aa4452f8696485f9e1
+# See tag-20240224-a and -b
+#
+do_catchsql_test 5.1 {
+  SELECT 1 ORDER BY 2_3;
+} {1 {1st ORDER BY term out of range - should be between 1 and 1}}
+
+finish_test
diff --git a/test/literal2.tcl b/test/literal2.tcl
new file mode 100644
index 0000000000..e14a03587b
--- /dev/null
+++ b/test/literal2.tcl
@@ -0,0 +1,40 @@
+# 2018 May 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname $argv0] pg_common.tcl]
+
+#=========================================================================
+
+
+start_test literal2 "2024 Jan 23"
+
+execsql_test  1.0 { SELECT 123_456 }
+errorsql_test 1.1 { SELECT 123__456 }
+
+execsql_float_test 2.1 { SELECT 1.0e1_2 }
+
+
+execsql_test  3.0.0 { SELECT 0xFF_FF }
+execsql_test  3.0.1 { SELECT 0xFF_EF }
+errorsql_test  3.0.2 { SELECT 0xFF__EF }
+# errorsql_test   3.0.3 { SELECT 0x_FFEF }
+errorsql_test  3.0.4 { SELECT 0xFFEF_ }
+
+execsql_test  3.1.0 { SELECT 0XFF_FF }
+execsql_test  3.1.1 { SELECT 0XFF_EF }
+errorsql_test  3.1.2 { SELECT 0XFF__EF }
+# errorsql_test   3.1.3 { SELECT 0X_FFEF }
+errorsql_test  3.1.4 { SELECT 0XFFEF_ }
+
+finish_test
+
+
diff --git a/test/literal2.test b/test/literal2.test
new file mode 100644
index 0000000000..ed177ca261
--- /dev/null
+++ b/test/literal2.test
@@ -0,0 +1,84 @@
+# 2024 Jan 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+####################################################
+# DO NOT EDIT! THIS FILE IS AUTOMATICALLY GENERATED!
+####################################################
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix literal2
+
+do_execsql_test 1.0 {
+  SELECT 123_456
+} {123456}
+
+# PG says ERROR:  trailing junk after numeric literal at or near "123_"
+do_test 1.1 { catch { execsql {
+  SELECT 123__456
+} } } 1
+
+
+do_test 2.1 {
+  set myres {}
+  foreach r [db eval {SELECT 1.0e1_2}] {
+    lappend myres [format %.4f [set r]]
+  }
+  set res2 {1000000000000.0000}
+  set i 0
+  foreach r [set myres] r2 [set res2] {
+    if {[set r]<([set r2]-0.0001) || [set r]>([set r2]+0.0001)} {
+      error "list element [set i] does not match: got=[set r] expected=[set r2]"
+    }
+    incr i
+  }
+  set {} {}
+} {}
+
+do_execsql_test 3.0.0 {
+  SELECT 0xFF_FF
+} {65535}
+
+do_execsql_test 3.0.1 {
+  SELECT 0xFF_EF
+} {65519}
+
+# PG says ERROR:  trailing junk after numeric literal at or near "0xFF_"
+do_test 3.0.2 { catch { execsql {
+  SELECT 0xFF__EF
+} } } 1
+
+# PG says ERROR:  trailing junk after numeric literal at or near "0xFFEF_"
+do_test 3.0.4 { catch { execsql {
+  SELECT 0xFFEF_
+} } } 1
+
+do_execsql_test 3.1.0 {
+  SELECT 0XFF_FF
+} {65535}
+
+do_execsql_test 3.1.1 {
+  SELECT 0XFF_EF
+} {65519}
+
+# PG says ERROR:  trailing junk after numeric literal at or near "0XFF_"
+do_test 3.1.2 { catch { execsql {
+  SELECT 0XFF__EF
+} } } 1
+
+# PG says ERROR:  trailing junk after numeric literal at or near "0XFFEF_"
+do_test 3.1.4 { catch { execsql {
+  SELECT 0XFFEF_
+} } } 1
+
+finish_test
diff --git a/test/loadext.test b/test/loadext.test
index 6da8a52894..e95d5f6eb8 100644
--- a/test/loadext.test
+++ b/test/loadext.test
@@ -66,9 +66,13 @@ if {$::tcl_platform(os) eq "Darwin"} {
   set dlerror_nosymbol   {dlsym.XXX, %2$s.: symbol not found}
 }
 
-if {$::tcl_platform(platform) eq "windows"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   set dlerror_nosuchfile {The specified module could not be found.*}
-  set dlerror_notadll    {%%1 is not a valid Win32 application.*}
+  if {$::tcl_platform(platform) eq "unix"} {
+    set dlerror_notadll    $dlerror_nosuchfile
+  } else {
+    set dlerror_notadll    {%%1 is not a valid Win32 application.*}
+  }
   set dlerror_nosymbol   {The specified procedure could not be found.*}
 }
 
diff --git a/test/lock.test b/test/lock.test
index 534aa3b9a4..3f4631a623 100644
--- a/test/lock.test
+++ b/test/lock.test
@@ -21,7 +21,7 @@ source $testdir/tester.tcl
 #
 do_test lock-1.0 {
   # Give a complex pathname to stress the path simplification logic in
-  # the vxworks driver and in test_async.
+  # the vxworks driver.
   file mkdir tempdir/t1/t2
   sqlite3 db2 ./tempdir/../tempdir/t1/.//t2/../../..//test.db
   set dummy {}
@@ -145,7 +145,7 @@ do_test lock-1.21 {
 # connections, because UNIX supports reader/writer locks.  Under windows,
 # this is not possible.
 #
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(platform) eq "unix"} {
   do_test lock-1.22 {
     db eval {SELECT * FROM t1} qv {
       set r [catch {db2 eval {SELECT a FROM t1}} msg]
diff --git a/test/lock5.test b/test/lock5.test
index 99214afb19..8ebc277018 100644
--- a/test/lock5.test
+++ b/test/lock5.test
@@ -11,10 +11,10 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is database locks.
 #
-# $Id: lock5.test,v 1.6 2008/12/04 12:34:16 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix lock5
 
 # This file is only run if using the unix backend compiled with the
 # SQLITE_ENABLE_LOCKING_STYLE macro.
@@ -101,10 +101,7 @@ do_test lock5-dotfile.X {
 #####################################################################
 
 forcedelete test.db
-if {[catch {sqlite3 db test.db -vfs unix-flock} msg]} {
-  finish_test
-  return
-}
+if {0==[catch {sqlite3 db test.db -vfs unix-flock} msg]} {
 
 do_test lock5-flock.1 {
   sqlite3 db test.db -vfs unix-flock
@@ -149,13 +146,67 @@ do_test lock5-flock.8 {
   db2 close
 } {}
 
+do_test lock5-flock.9 {
+  sqlite3 db test.db -vfs unix-flock
+  execsql {
+    SELECT * FROM t1
+  }
+} {1 2}
+
+do_test lock5-flock.10 {
+  sqlite3 db2 test.db -vfs unix-flock
+  execsql {
+    SELECT * FROM t1
+  } db2
+} {1 2}
+
+do_test lock5-flock.10 {
+  execsql {
+    PRAGMA cache_size = 1;
+    BEGIN;
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000
+      )
+      INSERT INTO t1 SELECT i, i+1 FROM s;
+  }
+
+  catchsql {
+    SELECT * FROM t1
+  } db2
+} {1 {database is locked}}
+
+if {[permutation]!="inmemory_journal"} {
+  do_test lock5-flock.11 {
+    forcecopy test.db test.db2
+    forcecopy test.db-journal test.db2-journal
+    db2 close
+    sqlite3 db2 test.db2 -vfs unix-flock
+    catchsql {
+      SELECT * FROM t1
+    } db2
+  } {0 {1 2}}
+  
+  do_test lock5-flock.12 {
+    file exists test.db2-journal
+  } 0
+}
+
+db close
+db2 close
+
+}
+
 #####################################################################
 
+reset_db
+
 do_test lock5-none.1 {
   sqlite3 db test.db -vfs unix-none
   sqlite3 db2 test.db -vfs unix-none
   execsql { PRAGMA mmap_size = 0 } db2
   execsql {
+    CREATE TABLE t1(a, b);
+    INSERT INTO t1 VALUES(1, 2);
     BEGIN;
     INSERT INTO t1 VALUES(3, 4);
   }
@@ -178,10 +229,12 @@ do_test lock5-none.5 {
 } {1 2}
 
 ifcapable memorymanage {
-  do_test lock5-none.6 {
-    sqlite3_release_memory 1000000
-    execsql {SELECT * FROM t1} db2
-  } {1 2 3 4}
+  if {[permutation]!="memsubsys1" && [permutation]!="memsubsys2"} { 
+    do_test lock5-none.6 {
+      sqlite3_release_memory 1000000
+      execsql {SELECT * FROM t1} db2
+    } {1 2 3 4}
+  }
 }
 
 do_test lock5-none.X {
@@ -193,4 +246,74 @@ ifcapable lock_proxy_pragmas {
   set env(SQLITE_FORCE_PROXY_LOCKING) $::using_proxy
 }
 
+#####################################################################
+reset_db
+if {[permutation]!="inmemory_journal"} {
+  
+  # 1. Create a large database using the unix-dotfile VFS
+  # 2. Write a large transaction to the db, so that the cache spills, but do
+  #    not commit it.
+  # 3. Make a copy of the database files on disk.
+  # 4. Try to read from the copy using unix-dotfile VFS. This fails because
+  #    the dotfile still exists, so SQLite thinks the database is locked.
+  # 5. Remove the dotfile.
+  # 6. Try to read the db again. This time, the old transaction is rolled
+  #    back and the read permitted.
+  #
+  do_test 2.dotfile.1 {
+    sqlite3 db test.db -vfs unix-dotfile
+    execsql {
+      PRAGMA cache_size = 10;
+      CREATE TABLE t1(x, y, z);
+      CREATE INDEX t1x ON t1(x);
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+      )
+      INSERT INTO t1 SELECT hex(randomblob(20)), hex(randomblob(500)), i FROM s;
+    }
+  } {}
+  
+  do_execsql_test 2.dotfile.2 {
+    BEGIN;
+      UPDATE t1 SET z=z+1, x=hex(randomblob(20));
+  }
+  
+  do_test 2.dotfile.3 {
+    list                            \
+      [file exists test.db]         \
+      [file exists test.db-journal] \
+      [file exists test.db.lock]
+  } {1 1 1}
+  
+  do_test 2.dotfile.4 {
+    forcecopy test.db test.db2
+    forcecopy test.db-journal test.db2-journal
+    file mkdir test.db2.lock
+  
+    sqlite3 db2 test.db2 -vfs unix-dotfile
+    catchsql {
+      SELECT count(*) FROM t1;
+    } db2
+  } {1 {database is locked}}
+  
+  do_test 2.dotfile.5 {
+    file delete test.db2.lock
+    execsql {
+      PRAGMA integrity_check
+    } db2
+  } {ok}
+  
+  db2 close
+  
+  do_test 2.dotfile.6 {
+    forcecopy test.db test.db2
+    forcecopy test.db-journal test.db2-journal
+  
+    sqlite3 db2 file:test.db2?nolock=1 -vfs unix-dotfile -uri 1
+    catchsql {
+      SELECT count(*) FROM t1;
+    } db2
+  } {0 1000}
+}
+
 finish_test
diff --git a/test/lock_common.tcl b/test/lock_common.tcl
index 3e1821bab0..7c79e4b4c5 100644
--- a/test/lock_common.tcl
+++ b/test/lock_common.tcl
@@ -145,6 +145,7 @@ proc testfixture_nb_cb {varname chan} {
   }
 
   if { $line == "OVER" } {
+    if {[string range $varname 0 1]!="::"} { global $varname }
     set $varname [lindex $::tfnb($chan) 1]
     unset ::tfnb($chan)
     close $chan
diff --git a/test/main.test b/test/main.test
index 13a385b7c4..556a8bdfcc 100644
--- a/test/main.test
+++ b/test/main.test
@@ -472,67 +472,10 @@ do_test main-3.6 {
   catchsql {SELECT 'abc' + #9}
 } {1 {near "#9": syntax error}}
 
-# The following test-case tests the linked list code used to manage
-# sqlite3_vfs structures.
-if {$::tcl_platform(platform)=="unix" 
-     && [info command sqlite3async_initialize]!=""} {
-  ifcapable threadsafe {
-    do_test main-4.1 {
-      sqlite3_crash_enable 1
-      sqlite3_crash_enable 0
-    
-      sqlite3async_initialize "" 1
-      sqlite3async_shutdown
-    
-      sqlite3_crash_enable 1
-      sqlite3async_initialize "" 1
-      sqlite3_crash_enable 0
-      sqlite3async_shutdown
-    
-      sqlite3_crash_enable 1
-      sqlite3async_initialize "" 1
-      sqlite3async_shutdown
-      sqlite3_crash_enable 0
-    
-      sqlite3async_initialize "" 1
-      sqlite3_crash_enable 1
-      sqlite3_crash_enable 0
-      sqlite3async_shutdown
-    
-      sqlite3async_initialize "" 1
-      sqlite3_crash_enable 1
-      sqlite3async_shutdown
-      sqlite3_crash_enable 0
-    } {}
-    do_test main-4.2 {
-      set rc [catch {sqlite3 db test.db -vfs crash} msg]
-      list $rc $msg
-    } {1 {no such vfs: crash}}
-    do_test main-4.3 {
-      set rc [catch {sqlite3 db test.db -vfs async} msg]
-      list $rc $msg
-    } {1 {no such vfs: async}}
-  }
-}
-
 # Print the version number so that it can be picked up by releasetest.tcl.
 #
 puts [db one {SELECT 'VERSION: ' ||
                   sqlite_version() || ' ' ||
                   sqlite_source_id();}]
-
-# Do deliberate failures if the TEST_FAILURE environment variable is set.
-# This is done to verify that failure notifications are detected by the
-# releasetest.tcl script, or possibly by other scripts involved in automatic
-# testing.
-#
-if {[info exists ::env(TEST_FAILURE)]} {
-  set res 123
-  if {$::env(TEST_FAILURE)==0} {set res 234}
-  do_test main-99.1 {
-     bad_behavior $::env(TEST_FAILURE)
-     set x 123
-  } $res
-}
-    
+   
 finish_test
diff --git a/test/malloc.test b/test/malloc.test
index 5e82e8028b..e53bfcd192 100644
--- a/test/malloc.test
+++ b/test/malloc.test
@@ -329,7 +329,7 @@ ifcapable crashtest&&attach {
   }
 }
 
-if {$tcl_platform(platform)!="windows" && [atomic_batch_write test.db]==0} {
+if {$tcl_platform(platform) ne "windows" && [atomic_batch_write test.db]==0} {
   do_malloc_test 14 -tclprep {
     catch {db close}
     sqlite3 db2 test2.db
diff --git a/test/memdb1.test b/test/memdb1.test
index 5e219a4c01..c0510abae7 100644
--- a/test/memdb1.test
+++ b/test/memdb1.test
@@ -84,7 +84,6 @@ do_test 152 {
   catchsql {INSERT INTO t1 VALUES(3,4);}
 } {1 {attempt to write a readonly database}}
 
-breakpoint
 do_test 160 {
   db deserialize -maxsize 32768 $db1
   db eval {SELECT * FROM t1}
@@ -246,8 +245,9 @@ if {[wal_is_capable]} {
   db close
   
   set fd [open test.db]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   set data [read $fd [expr 20*1024]]
+  close $fd
   
   sqlite3 db ""
   db deserialize $data
@@ -267,4 +267,17 @@ if {[wal_is_capable]} {
   } {1 {database disk image is malformed}}
 }
 
+# 2024-01-20
+# https://sqlite.org/forum/forumpost/498777780e16880a
+#
+# Make sure a database is initialized before serializing it.
+#
+reset_db
+sqlite3 dbempty :memory:
+do_test 900 {
+  set len [string length [dbempty serialize]]
+  expr {$len>0}
+} 1
+dbempty close
+
 finish_test
diff --git a/test/memdb2.test b/test/memdb2.test
index 286bfc3f84..7c2144991f 100644
--- a/test/memdb2.test
+++ b/test/memdb2.test
@@ -74,4 +74,3 @@ foreach {tn fname} {
 }
 
 finish_test
-
diff --git a/test/memjournal2.test b/test/memjournal2.test
index ec5ba56da3..d08bcb5a6a 100644
--- a/test/memjournal2.test
+++ b/test/memjournal2.test
@@ -59,5 +59,3 @@ for {set jj 200} {$jj <= 300} {incr jj} {
 
 
 finish_test
-
-
diff --git a/test/memleak.test b/test/memleak.test
index a24a901f50..8443162ed6 100644
--- a/test/memleak.test
+++ b/test/memleak.test
@@ -38,8 +38,6 @@ set EXCLUDE {
   misuse.test
   memleak.test
   btree2.test
-  async.test
-  async2.test
   trans.test
   crash.test
   autovacuum_crash.test
diff --git a/test/merge1.test b/test/merge1.test
index 7ec4dab108..686271648a 100644
--- a/test/merge1.test
+++ b/test/merge1.test
@@ -64,21 +64,21 @@ do_eqp_test 101 {
      |     |--LEFT
      |     |  `--MERGE (UNION ALL)
      |     |     |--LEFT
-     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      |     |     `--RIGHT
-     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      |     `--RIGHT
-     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      `--RIGHT
         `--MERGE (UNION ALL)
            |--LEFT
            |  `--MERGE (UNION ALL)
            |     |--LEFT
-           |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+           |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
            |     `--RIGHT
-           |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+           |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
            `--RIGHT
-              `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+              `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
 }
 
 # Same test with the blanced-merge optimization
@@ -129,17 +129,17 @@ do_eqp_test 111 {
      |     |     |     |--LEFT
      |     |     |     |  `--MERGE (UNION ALL)
      |     |     |     |     |--LEFT
-     |     |     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      |     |     |     |     `--RIGHT
-     |     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      |     |     |     `--RIGHT
-     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      |     |     `--RIGHT
-     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      |     `--RIGHT
-     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      `--RIGHT
-        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
 }
 
 finish_test
diff --git a/test/minmax.test b/test/minmax.test
index 81bd46dbe2..232ac14a05 100644
--- a/test/minmax.test
+++ b/test/minmax.test
@@ -633,7 +633,7 @@ do_test_13_noopt 13.7 {
   SELECT min(c), count(c) FROM t1 WHERE a='a';
 } {1 5}
 
-# 2016-07-26.  https://www.sqlite.org/src/info/a0bac8b3c3d1bb75
+# 2016-07-26.  https://sqlite.org/src/info/a0bac8b3c3d1bb75
 # Incorrect result on a min() query after a CREATE INDEX.
 #
 do_execsql_test 14.1 {
diff --git a/test/misc1.test b/test/misc1.test
index 83acc752af..8110d38678 100644
--- a/test/misc1.test
+++ b/test/misc1.test
@@ -654,7 +654,7 @@ do_catchsql_test misc1-21.1 {
 } {1 {near "#0": syntax error}}
 do_catchsql_test misc1-21.2 {
   VALUES(0,0x0MATCH#0;
-} {1 {near ";": syntax error}}
+} {1 {unrecognized token: "0x0MATCH"}}
 
 # 2015-04-15
 do_execsql_test misc1-22.1 {
diff --git a/test/misc2.test b/test/misc2.test
index 4796d5d374..607799ea21 100644
--- a/test/misc2.test
+++ b/test/misc2.test
@@ -54,19 +54,34 @@ do_test misc2-2.1 {
   }
 } {}
 ifcapable subquery {
-  do_catchsql_test misc2-2.2 {
-    SELECT rowid, * FROM (SELECT * FROM t1, t2);
-  } {1 {no such column: rowid}}
+  ifcapable allow_rowid_in_view {
+    do_catchsql_test misc2-2.2 {
+      SELECT rowid, * FROM (SELECT * FROM t1, t2);
+    } {0 {{} 1 2 3 7 8 9}}
+  } else {
+    do_catchsql_test misc2-2.2 {
+      SELECT rowid, * FROM (SELECT * FROM t1, t2);
+    } {1 {no such column: rowid}}
+  }
   do_catchsql_test misc2-2.2b {
     SELECT 'rowid', * FROM (SELECT * FROM t1, t2);
   } {0 {rowid 1 2 3 7 8 9}}
 }
 
 ifcapable view {
-  do_catchsql_test misc2-2.3 {
-    CREATE VIEW v1 AS SELECT * FROM t1, t2;
-    SELECT rowid, * FROM v1;
-  } {1 {no such column: rowid}}
+  ifcapable allow_rowid_in_view {
+    do_catchsql_test misc2-2.3 {
+      CREATE VIEW v1 AS SELECT * FROM t1, t2;
+      SELECT rowid, * FROM v1;
+    } {0 {{} 1 2 3 7 8 9}}
+  } else {
+    do_catchsql_test misc2-2.3 {
+      CREATE VIEW v1 AS SELECT * FROM t1, t2;
+      SELECT rowid, * FROM v1;
+    } {1 {no such column: rowid}}
+  }
+
+
   do_catchsql_test misc2-2.3b {
     SELECT 'rowid', * FROM v1;
   } {0 {rowid 1 2 3 7 8 9}}
diff --git a/test/misc3.test b/test/misc3.test
index bc1f0ff911..f64b0fe1dd 100644
--- a/test/misc3.test
+++ b/test/misc3.test
@@ -88,8 +88,8 @@ do_test misc3-2.4 {
   execsql {SELECT 2e-25*0.5e250}
 } 1e+225
 do_test misc3-2.5 {
-  execsql {SELECT 2.0e-250*0.5e25}
-} 1e-225
+  execsql {SELECT format('%.15e',2.0e-250*0.5e25)}
+} {1.0000000000000e-225}
 do_test misc3-2.6 {
   execsql {SELECT '-2.0e-127' * '-0.5e27'}
 } 1e-100
diff --git a/test/misc5.test b/test/misc5.test
index f7c6048d97..43ee2781a1 100644
--- a/test/misc5.test
+++ b/test/misc5.test
@@ -523,6 +523,7 @@ if {[permutation] == ""} {
       CREATE TABLE t1(a,b,c);
     }
   } {1 {file is not a database}}
+  reset_db
 }
 
 # Ticket #1371.  Allow floating point numbers of the form .N  or N.
@@ -569,11 +570,11 @@ ifcapable subquery&&compound {
 }
 
 # Overflow the lemon parser stack by providing an overly complex
-# expression.  Make sure that the overflow is detected and reported.
+# expression.  Make sure that the overflow is detected and the
+# stack is grown automatically such that the application calling
+# SQLite never notices.
 #
-# This test fails when building with -DYYSTACKDEPTH=0
-#
-do_test misc5-7.1 {
+do_test misc5-7.1.1 {
   execsql {CREATE TABLE t1(x)}
   set sql "INSERT INTO t1 VALUES("
   set tail ""
@@ -581,9 +582,21 @@ do_test misc5-7.1 {
     append sql "(1+"
     append tail ")"
   }
-  append sql 2$tail
+  append sql "0$tail); SELECT * FROM t1;"
+  catchsql $sql
+} {0 200}
+do_test misc5-7.1.2 {
+  execsql {DELETE FROM t1}
+  set sql "INSERT INTO t1 VALUES("
+  set tail ""
+  for {set i 0} {$i<900} {incr i} {
+    append sql "(1+"
+    append tail ")"
+  }
+  append sql "0$tail); SELECT * FROM t1;"
   catchsql $sql
-} {1 {parser stack overflow}}
+} {0 900}
+
 
 # Parser stack overflow is silently ignored when it occurs while parsing the
 # schema and PRAGMA writable_schema is turned on.
diff --git a/test/misc7.test b/test/misc7.test
index add9014b08..f4ef2d2103 100644
--- a/test/misc7.test
+++ b/test/misc7.test
@@ -100,7 +100,7 @@ proc do_fileopen_test {prefix sql} {
 execsql { CREATE TABLE abc(a PRIMARY KEY, b, c); }
 db close
 
-if {$tcl_platform(platform)!="windows"} {
+if {$tcl_platform(platform) ne "windows"} {
   do_fileopen_test misc7-6.1 {
     BEGIN;
     INSERT INTO abc VALUES(1, 2, 3);
@@ -390,7 +390,7 @@ do_test misc7-16.X {
 # These tests do not work on windows due to restrictions in the
 # windows file system.
 #
-if {$tcl_platform(platform)!="windows"} {
+if {$tcl_platform(platform) ne "windows"} {
 
   # Some network filesystems (ex: AFP) do not support setting read-only
   # permissions.  Only run these tests if full unix permission setting
@@ -534,7 +534,7 @@ do_test misc7-22.4 {
 catch { db close }
 forcedelete test.db
 
-if {$::tcl_platform(platform)=="unix"
+if {$::tcl_platform(platform) eq "unix"
  && [atomic_batch_write test.db]==0
 } {
   reset_db
diff --git a/test/misc8.test b/test/misc8.test
index 32b3a597dc..60b44fe1c7 100644
--- a/test/misc8.test
+++ b/test/misc8.test
@@ -100,6 +100,11 @@ do_execsql_test misc8-2.1 {
 
 # 2016-02-26: An assertion fault found by the libFuzzer project
 #
+ifcapable allow_rowid_in_view {
+  set nosuch "1 {ambiguous column name: rowid}"
+} else {
+  set nosuch "1 {no such column: rowid}"
+}
 do_catchsql_test misc8-3.0 {
   SELECT *
     FROM
@@ -110,7 +115,7 @@ do_catchsql_test misc8-3.0 {
          (SELECT 6 AS j UNION ALL SELECT 7) AS x4
    WHERE i<rowid
    ORDER BY 1;
-} {1 {no such column: rowid}}
+} $nosuch
 
 # The SQLITE_DBCONFIG_MAINDBNAME interface
 #
diff --git a/test/misuse.test b/test/misuse.test
index e15cf3357e..640cb5a4d3 100644
--- a/test/misuse.test
+++ b/test/misuse.test
@@ -172,7 +172,11 @@ do_test misuse-4.3 {
   lappend v $msg $r
 } {0 {} SQLITE_BUSY}
 
-if {[clang_sanitize_address]==0} {
+# All of the following tests can potentially (though rarely)
+# lead to segfaults, which is unsettling.  So they are disabled
+# for now__________________________
+#                                  v
+if {[clang_sanitize_address]==0 && 0} {
   do_test misuse-4.4 {
   # Flush the TCL statement cache here, otherwise the sqlite3_close() will
   # fail because there are still un-finalized() VDBEs.
@@ -207,4 +211,16 @@ if {[clang_sanitize_address]==0} {
   } {1 {(21) bad parameter or other API misuse}}
 }
 
+#-------------------------------------------------------------------------
+reset_db
+do_test misuse-6.0 {
+  sqlite3_set_errmsg db 1 "an error has occurred"
+} {SQLITE_OK}
+do_test misuse-6.1 {
+  sqlite3_errmsg db
+} {an error has occurred}
+do_test misuse-6.2 {
+  sqlite3_set_errmsg "" 1 "an error has occurred"
+} {SQLITE_MISUSE}
+
 finish_test
diff --git a/test/mmap1.test b/test/mmap1.test
index 3362f7187f..6a9625427a 100644
--- a/test/mmap1.test
+++ b/test/mmap1.test
@@ -45,18 +45,18 @@ proc register_rblob_code {dbname seed} {
 }
 
 
-# For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on
-# unix and 9 on windows. The difference is that windows only ever maps
+# For cases 1.1 and 1.4, the number of pages read using xRead() is 8 on
+# unix and 12 on windows. The difference is that windows only ever maps
 # an integer number of OS pages (i.e. creates mappings that are a multiple
 # of 4KB in size). Whereas on unix any sized mapping may be created.
 #
 foreach {t mmap_size nRead c2init} {
-  1.1 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 0}
-  1.2 { PRAGMA mmap_size =    53248 } 150    {PRAGMA mmap_size = 0}
-  1.3 { PRAGMA mmap_size =        0 } 344    {PRAGMA mmap_size = 0}
-  1.4 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 67108864 }
-  1.5 { PRAGMA mmap_size =    53248 } 150    {PRAGMA mmap_size = 67108864 }
-  1.6 { PRAGMA mmap_size =        0 } 344    {PRAGMA mmap_size = 67108864 }
+  1.1 { PRAGMA mmap_size = 67108864 } /8|12/   {PRAGMA mmap_size = 0}
+  1.2 { PRAGMA mmap_size =    53248 } /15[34]/ {PRAGMA mmap_size = 0}
+  1.3 { PRAGMA mmap_size =        0 } 344      {PRAGMA mmap_size = 0}
+  1.4 { PRAGMA mmap_size = 67108864 } /12|8/   {PRAGMA mmap_size = 67108864 }
+  1.5 { PRAGMA mmap_size =    53248 } /15[34]/ {PRAGMA mmap_size = 67108864 }
+  1.6 { PRAGMA mmap_size =        0 } 344      {PRAGMA mmap_size = 67108864 }
 } {
 
   do_multiclient_test tn {
diff --git a/test/mmap2.test b/test/mmap2.test
index 1f8346b915..df96671dfb 100644
--- a/test/mmap2.test
+++ b/test/mmap2.test
@@ -21,10 +21,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix mmap2
 
-if {$::tcl_platform(platform)!="unix" || [test_syscall defaultvfs] != "unix"} {
+if {[llength [info commands test_syscall]]==0} {
   finish_test
   return
-}
+} 
 ifcapable !mmap {
   finish_test
   return
diff --git a/test/mmapcorrupt.test b/test/mmapcorrupt.test
new file mode 100644
index 0000000000..d434ec1836
--- /dev/null
+++ b/test/mmapcorrupt.test
@@ -0,0 +1,55 @@
+# 2024 January 23
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test special cases of corrupt database handling in mmap-mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix mmapcorrupt
+
+ifcapable !mmap {
+  finish_test
+  return
+}
+database_may_be_corrupt
+
+db close
+sqlite3_shutdown
+sqlite3_config_lookaside 0 0
+sqlite3_initialize
+
+reset_db
+do_execsql_test 1.0 {
+  PRAGMA page_size = 16384;
+  CREATE TABLE tn1(a PRIMARY KEY) WITHOUT ROWID;
+  CREATE TABLE t0(a PRIMARY KEY) WITHOUT ROWID;
+  CREATE TABLE t1(a PRIMARY KEY) WITHOUT ROWID;
+  INSERT INTO t1 VALUES('B');
+}
+db close
+
+set sz [file size test.db]
+hexio_write test.db [expr $sz-3] 800380
+
+sqlite3 db test.db
+do_execsql_test 2.1 {
+  PRAGMA mmap_size = 1000000;
+  SELECT sql FROM sqlite_schema LIMIT 1;
+  SELECT * FROM t0;
+} {1000000 {CREATE TABLE tn1(a PRIMARY KEY) WITHOUT ROWID}}
+
+do_execsql_test 2.2 {
+  INSERT INTO t0 SELECT * FROM t1;
+}
+
+finish_test
+
diff --git a/test/mutex1.test b/test/mutex1.test
index 0fc8368f76..cb189a7a8a 100644
--- a/test/mutex1.test
+++ b/test/mutex1.test
@@ -115,6 +115,14 @@ ifcapable threadsafe1&&shared_cache {
     }
   } {
 
+    # For journal_mode=memory, the static_prng mutex is not required. This
+    # is because the header of an in-memory journal does not contain
+    # any random bytes, and so no call to sqlite3_randomness() is made.
+    if {[permutation]=="inmemory_journal"} {
+      set idx [lsearch $mutexes static_prng]
+      if {$idx>=0} { set mutexes [lreplace $mutexes $idx $idx] }
+    }
+
     do_test mutex1.2.$mode.1 {
       catch {db close}
       sqlite3_shutdown
diff --git a/test/nan.test b/test/nan.test
index 615a4ad227..8d8a98ab3b 100644
--- a/test/nan.test
+++ b/test/nan.test
@@ -281,6 +281,7 @@ do_test nan-4.14 {
 
 # These tests test some really, really small floating point numbers.
 #
+load_static_extension db decimal
 if {$tcl_platform(platform) != "symbian"} {
   # These two are not run on symbian because tcl has trouble converting
   # the very small numbers back to text form (probably due to a difference
@@ -291,15 +292,15 @@ if {$tcl_platform(platform) != "symbian"} {
     set small \
         [string repeat 0 10000].[string repeat 0 323][string repeat 9 10000]
     db eval "INSERT INTO t1 VALUES($small)"
-    db eval {SELECT x, typeof(x) FROM t1}
-  } {9.88131291682493e-324 real}
+    db eval {SELECT decimal_exp(x), typeof(x) FROM t1}
+  } {/9\.88131291682493\d*e-324 real/}
   do_test nan-4.16 {
     db eval {DELETE FROM t1}
     set small \
         -[string repeat 0 10000].[string repeat 0 323][string repeat 9 10000]
     db eval "INSERT INTO t1 VALUES($small)"
-    db eval {SELECT x, typeof(x) FROM t1}
-  } {-9.88131291682493e-324 real}
+    db eval {SELECT decimal_exp(x), typeof(x) FROM t1}
+  } {/-9\.88131291682493\d*e-324 real/}
 }
 do_test nan-4.17 {
   db eval {DELETE FROM t1}
diff --git a/test/nockpt.test b/test/nockpt.test
index 4cc61d11e1..f5d11732be 100644
--- a/test/nockpt.test
+++ b/test/nockpt.test
@@ -61,7 +61,7 @@ do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1}
 do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete}
 do_test 1.15 { file exists test.db-wal } {0}
 
-if {$::tcl_platform(platform)!="windows"} {
+if {$::tcl_platform(platform) ne "windows"} {
 #-------------------------------------------------------------------------
 # Test an unusual scenario:
 #
@@ -102,7 +102,9 @@ sqlite3_close_v2 $::db1
 
 # Delete the database, wal and shm files.
 #
-forcedelete test.db test.db-wal test.db-shm
+catch {forcedelete test.db}
+catch {forcedelete test.db-wal}
+catch {forcedelete  test.db-shm}
 
 # Open and populate a new database file at the same file-system location
 # as the one just deleted. Contrive a partial checkpoint on it.
diff --git a/test/notnull2.test b/test/notnull2.test
index 7f68086810..67d7c26a8d 100644
--- a/test/notnull2.test
+++ b/test/notnull2.test
@@ -59,14 +59,14 @@ do_vmstep_test 1.4.2 {
 
 do_vmstep_test 1.5.1 {
   SELECT count(*) FROM t2 WHERE EXISTS(
-    SELECT t2.d IS NULL FROM t1 WHERE t1.a=450
+    SELECT 1 FROM t1 WHERE t1.a=450 AND t2.d IS NULL
   )
-} 10000 {1000}
+} 7000 {0}
 do_vmstep_test 1.5.2 {
   SELECT count(*) FROM t2 WHERE EXISTS(
-    SELECT t2.c IS NULL FROM t1 WHERE t1.a=450
+    SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL
   )
-} +100000 {1000}
+} 4000 {0}
 
 #-------------------------------------------------------------------------
 reset_db
@@ -111,4 +111,12 @@ do_execsql_test 4.1 {
   SELECT * FROM (SELECT 3 AS c FROM t1) AS t3 LEFT JOIN t2 ON c IS NULL;
 } {3 {}}
 
+# 2024-03-08 https://sqlite.org/forum/forumpost/440f2a2f17
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE t1(a INT NOT NULL);
+  SELECT a IS NULL, a IS NOT NULL, count(*) FROM t1;
+} {1 0 0}
+
 finish_test
diff --git a/test/orderby1.test b/test/orderby1.test
index 7432b5473d..73eda83992 100644
--- a/test/orderby1.test
+++ b/test/orderby1.test
@@ -520,7 +520,7 @@ do_eqp_test 8.1 {
 } {
   QUERY PLAN
   |--SCAN t1 USING INDEX i1
-  `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+  `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY
 }
 
 do_execsql_test 8.2 {
@@ -536,7 +536,7 @@ do_test 8.3 {
 } 5000
 
 #---------------------------------------------------------------------------
-# https://www.sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029
+# https://sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029
 #
 # Adverse interaction between scalar subqueries and the partial-sorting
 # logic.
diff --git a/test/orderby3.test b/test/orderby3.test
index f005f0d2e8..09e9156d0f 100644
--- a/test/orderby3.test
+++ b/test/orderby3.test
@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing that the optimizations that disable
 # ORDER BY clauses work correctly on a 3-way join.  See ticket
-# http://www.sqlite.org/src/956e4d7f89
+# http://sqlite.org/src/956e4d7f89
 #
 
 
diff --git a/test/orderby4.test b/test/orderby4.test
index ec6eb041f6..944a91f37d 100644
--- a/test/orderby4.test
+++ b/test/orderby4.test
@@ -12,7 +12,7 @@
 # focus of this file is testing that the optimizations that disable
 # ORDER BY clauses work correctly on multi-value primary keys and
 # unique indices when only some prefix of the terms in the key are
-# used.  See ticket http://www.sqlite.org/src/info/a179fe74659
+# used.  See ticket http://sqlite.org/src/info/a179fe74659
 #
 
 
diff --git a/test/orderbyB.test b/test/orderbyB.test
new file mode 100644
index 0000000000..42d2de7982
--- /dev/null
+++ b/test/orderbyB.test
@@ -0,0 +1,94 @@
+# 2024-08-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+# Specifically, it tests cases with order-by-subquery optimization in which
+# an ORDER BY in a subquery is used to help resolve an ORDER BY in the
+# outer query without having to do an extra sort.
+# 
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix orderbyb
+
+db null NULL
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a TEXT, b TEXT, c INT);
+  INSERT INTO t1 VALUES(NULL,NULL,NULL);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7)
+    INSERT INTO t1(a,b,c) SELECT char(p,p), char(q,q), n FROM
+            (SELECT ((n-1)%4)+0x61 AS p, abs(n*2-9+(n>=5))+0x60 AS q, n FROM c);
+  UPDATE t1 SET b=upper(b) WHERE c=1;
+  CREATE TABLE t2(k TEXT PRIMARY KEY, v INT) WITHOUT ROWID;
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7)
+    INSERT INTO t2(k,v) SELECT char(0x60+n,0x60+n), n FROM c;
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7)
+    INSERT INTO t2(k,v) SELECT char(0x40+n,0x40+n), n FROM c;
+  SELECT a,b,c,tx.v AS 'v-a', ty.v AS 'v-b'
+    FROM t1 LEFT JOIN t2 AS tx ON tx.k=a
+            LEFT JOIN t2 AS ty ON ty.k=b
+   ORDER BY c;
+} {
+  NULL  NULL  NULL  NULL  NULL
+  aa    GG    1     1     7
+  bb    ee    2     2     5
+  cc    cc    3     3     3
+  dd    aa    4     4     1
+  aa    bb    5     1     2
+  bb    dd    6     2     4
+  cc    ff    7     3     6
+}
+
+do_eqp_execsql_test 1.1 {
+  WITH t3(x,y) AS (SELECT a, b FROM t1 ORDER BY a, b LIMIT 8)
+    SELECT x, y, v FROM t3 LEFT JOIN t2 ON k=t3.y ORDER BY x, y COLLATE nocase;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE t3
+  |  |--SCAN t1
+  |  `--USE TEMP B-TREE FOR ORDER BY
+  |--SCAN t3
+  |--SEARCH t2 USING PRIMARY KEY (k=?) LEFT-JOIN
+  `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY
+} {
+  NULL  NULL  NULL
+  aa    bb    2
+  aa    GG    7
+  bb    dd    4
+  bb    ee    5
+  cc    cc    3
+  cc    ff    6
+  dd    aa    1
+}
+
+do_eqp_execsql_test 1.2 {
+  WITH t3(x,y) AS MATERIALIZED (SELECT a, b COLLATE nocase FROM t1 ORDER BY 1,2)
+    SELECT x, y, v FROM t3 LEFT JOIN t2 ON k=t3.y ORDER BY x,y;
+} {
+  QUERY PLAN
+  |--MATERIALIZE t3
+  |  |--SCAN t1
+  |  `--USE TEMP B-TREE FOR ORDER BY
+  |--SCAN t3
+  `--SEARCH t2 USING PRIMARY KEY (k=?) LEFT-JOIN
+} {
+  NULL  NULL  NULL
+  aa    bb    2
+  aa    GG    7
+  bb    dd    4
+  bb    ee    5
+  cc    cc    3
+  cc    ff    6
+  dd    aa    1
+}
+
+
+finish_test
diff --git a/test/oserror.test b/test/oserror.test
index a51301cc52..43d72569d9 100644
--- a/test/oserror.test
+++ b/test/oserror.test
@@ -16,7 +16,10 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$::tcl_platform(platform)!="unix"} { finish_test ; return }
+if {[llength [info commands test_syscall]]==0} {
+  finish_test
+  return
+} 
 set ::testprefix oserror
 
 db close
diff --git a/test/ossfuzz.c b/test/ossfuzz.c
index b0156a640e..8e80b98ef6 100644
--- a/test/ossfuzz.c
+++ b/test/ossfuzz.c
@@ -155,6 +155,11 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
   /* Set a limit on the maximum size of a prepared statement */
   sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000);
 
+  /* Set a limit on the maximum LIKE or GLOB pattern length due to
+  ** https://issues.oss-fuzz.com/issues/453240497.  The default is 50K
+  ** which is causing timeouts in OSS-Fuzz */
+  sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 250);
+
   /* Limit total memory available to SQLite to 20MB */
   sqlite3_hard_heap_limit64(20000000);
 
diff --git a/test/pager1.test b/test/pager1.test
index 79598e2a70..9a39e6f374 100644
--- a/test/pager1.test
+++ b/test/pager1.test
@@ -454,7 +454,7 @@ do_test pager1.4.2.1 {
   tstvfs delete
 } {}
 
-if {$::tcl_platform(platform)!="windows"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
 do_test pager1.4.2.2 {
   faultsim_restore_and_reopen
   execsql {
@@ -567,7 +567,7 @@ foreach {tn1 tcl} {
     # make sure SQLite doesn't get confused by this.
     #
     set nPadding [expr 511 - $::mj_filename_length]
-    if {$tcl_platform(platform)=="windows"} {
+    if {$tcl_platform(platform) eq "windows"} {
       # TBD need to figure out how to do this correctly for Windows!!!
       set nPadding [expr 255 - $::mj_filename_length]
     }
@@ -1714,7 +1714,7 @@ do_execsql_test pager1-13.1.1 {
   UPDATE t1 SET b = a_string(400);
 } {persist}
 
-if {$::tcl_platform(platform)!="windows"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
 # Run transactions of increasing sizes. Eventually, one (or more than one)
 # of these will write just enough content that one of the old headers created 
 # by the transaction in the block above lies immediately after the content
@@ -1739,7 +1739,7 @@ for {set nUp 1} {$nUp<64} {incr nUp} {
 }
 }
 
-if {$::tcl_platform(platform)!="windows"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
 # Same test as above. But this time with an index on the table.
 #
 do_execsql_test pager1-13.2.1 {
@@ -2538,7 +2538,7 @@ do_test pager1-30.1 {
 # file can still be rolled back. This is required for backward compatibility -
 # versions of SQLite prior to 3.5.8 always set this field to zero.
 #
-if {$tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
 do_test pager1-31.1 {
   faultsim_delete_and_reopen
   execsql {
@@ -2612,7 +2612,7 @@ forcedelete test.db
 # and the call to unlink() returns an ENOENT error, the COMMIT does not
 # succeed.
 #
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
   do_test pager1-33.1 {
     sqlite3 db test.db
     execsql {
diff --git a/test/pager4.test b/test/pager4.test
index 2cf73d1b17..537f529dd3 100644
--- a/test/pager4.test
+++ b/test/pager4.test
@@ -13,7 +13,7 @@
 # is unlinked or renamed out from under SQLite.
 #
 
-if {$tcl_platform(platform)!="unix"} return
+if {$tcl_platform(os) eq "Windows NT"} return
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
diff --git a/test/pagerfault.test b/test/pagerfault.test
index 3006dad7cc..6e82b90090 100644
--- a/test/pagerfault.test
+++ b/test/pagerfault.test
@@ -20,7 +20,7 @@ if {[permutation] == "inmemory_journal"} {
   return
 }
 
-if {$::tcl_platform(platform)=="windows"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   finish_test
   return
 }
diff --git a/test/parser1.test b/test/parser1.test
index ad95b49091..b8d3d8b420 100644
--- a/test/parser1.test
+++ b/test/parser1.test
@@ -100,4 +100,26 @@ do_execsql_test parser1-3.1 {
   PRAGMA foreign_key_list(t301);
 } {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE}
 
+# 2025-07-01 https://sqlite.org/forum/forumpost/f4878de3e7dd4764
+# Do not allow parse-time optimizations to omit aggregate functions,
+# because doing so can change the meaning of the query.
+#
+unset -nocomplain zero
+set zero [expr {0+0}]
+do_execsql_test parser1-4.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  SELECT max(x) AND $zero FROM t1;
+} 0
+do_execsql_test parser1-4.2 {
+  SELECT max(x) AND 0 FROM t1;
+} 0
+do_execsql_test parser1-4.3 {
+  SELECT max(x) IN () FROM t1;
+} 0
+do_execsql_test parser1-4.4 {
+  SELECT max(x) NOT IN () FROM t1;
+} 1
+
+
 finish_test
diff --git a/test/pendingrace.test b/test/pendingrace.test
index f0e1a18ffa..80160149a3 100644
--- a/test/pendingrace.test
+++ b/test/pendingrace.test
@@ -61,12 +61,12 @@ proc my_db_restore {} {
   forcecopy sv_test.db-journal test.db-journal
 
   set fd1 [open sv_test.db r]
-  fconfigure $fd1 -encoding binary -translation binary
+  fconfigure $fd1 -translation binary
   set data [read $fd1]
   close $fd1
 
   set fd1 [open test.db w]
-  fconfigure $fd1 -encoding binary -translation binary
+  fconfigure $fd1 -translation binary
   puts -nonewline $fd1 $data
   close $fd1
 }
@@ -121,6 +121,3 @@ tvfs delete
 tvfs2 delete
 
 finish_test
-
-
-
diff --git a/test/percentile.test b/test/percentile.test
index b2bd061e86..25096a953c 100644
--- a/test/percentile.test
+++ b/test/percentile.test
@@ -9,7 +9,8 @@
 #
 #***********************************************************************
 # This file implements regression tests for SQLite library.  The
-# focus of this file is percentile.c extension
+# focus of this file is percentile.c extension.  This also tests
+# the SQLITE_ENABLE_ORDERED_SET_AGGREGATES compile-time option.
 #
 
 set testdir [file dirname $argv0]
@@ -18,25 +19,71 @@ source $testdir/tester.tcl
 # Basic test of the percentile() function.
 #
 do_test percentile-1.0 {
-  load_static_extension db percentile
   execsql {
     CREATE TABLE t1(x);
     INSERT INTO t1 VALUES(1),(4),(6),(7),(8),(9),(11),(11),(11);
   }
   execsql {SELECT percentile(x,0) FROM t1}
 } {1.0}
-foreach {in out} {
-  100    11.0
-   50     8.0
-   12.5   4.0
-   15     4.4
-   20     5.2
-   80    11.0
-   89    11.0
+foreach {in out disc} {
+  100    11.0  11.0
+   50     8.0   8.0
+   12.5   4.0   4.0
+   15     4.4   4.0
+   20     5.2   4.0
+   80    11.0  11.0
+   89    11.0  11.0
 } {
-  do_test percentile-1.1.$in {
+  do_test percentile-1.1.$in.1 {
     execsql {SELECT percentile(x,$in) FROM t1}
   } $out
+  do_test percentile-1.1.$in.2 {
+    execsql {SELECT percentile_cont(x,$in*0.01) FROM t1}
+  } $out
+  do_test percentile-1.1.$in.3 {
+    execsql {SELECT percentile_disc(x,$in*0.01) FROM t1}
+  } $disc
+  if {$in==50} {
+    do_test percentile-1.1.$in.4 {
+      execsql {SELECT median(x) FROM t1}
+    } $out
+  }
+  ifcapable ordered_set_aggregates {
+    do_test percentile-1.1.$in.5 {
+      execsql {SELECT percentile($in)WITHIN GROUP(ORDER BY x) FROM t1}
+    } $out
+    do_test percentile-1.1.$in.6 {
+      execsql {SELECT percentile_cont($in*0.01) WITHIN GROUP(ORDER BY x)
+                 FROM t1}
+    } $out
+    do_test percentile-1.1.$in.7 {
+      execsql {SELECT percentile_disc($in*0.01) WITHIN GROUP(ORDER BY x)
+                 FROM t1}
+    } $disc
+    if {$in==50} {
+      do_test percentile-1.1.$in.8 {
+        execsql {SELECT median() WITHIN GROUP (ORDER BY x) FROM t1}
+      } $out
+    }
+  }
+}
+do_execsql_test percentile-1.1.median {
+  SELECT median(x) FROM t1;
+} 8.0
+ifcapable ordered_set_aggregates {
+  do_execsql_test percentile-1.1.median {
+    SELECT median() WITHIN GROUP (ORDER BY x) FROM t1;
+  } 8.0
+  do_execsql_test percentile-1.1.distinct.1 {
+    SELECT median(DISTINCT x) FROM t1;
+  } 7.0
+  do_catchsql_test percentile-1.1.distinct.2 {
+    SELECT percentile(DISTINCT 50) WITHIN GROUP (ORDER BY x) FROM t1;
+  } {1 {DISTINCT not allowed on ordered-set aggregate percentile()}}
+} else {
+  do_catchsql_test percentile-1.1.median {
+    SELECT median() WITHIN GROUP (ORDER BY x) FROM t1;
+  } {1 {near "(": syntax error}}
 }
 
 # Add some NULL values.
@@ -44,28 +91,69 @@ foreach {in out} {
 do_test percentile-1.2 {
   execsql {INSERT INTO t1 VALUES(NULL),(NULL);}
 } {}
-foreach {in out} {
-  100    11.0
-   50     8.0
-   12.5   4.0
-   15     4.4
-   20     5.2
-   80    11.0
-   89    11.0
+foreach {in out disc} {
+  100    11.0  11.0
+   50     8.0   8.0
+   12.5   4.0   4.0
+   15     4.4   4.0
+   20     5.2   4.0
+   80    11.0  11.0
+   89    11.0  11.0
 } {
-  do_test percentile-1.3.$in {
+  do_test percentile-1.3.$in.1 {
     execsql {SELECT percentile(x,$in) FROM t1}
   } $out
+  do_test percentile-1.3.$in.2 {
+    execsql {SELECT percentile_cont(x,$in*0.01) FROM t1}
+  } $out
+  do_test percentile-1.3.$in.3 {
+    execsql {SELECT percentile_disc(x,$in*0.01) FROM t1}
+  } $disc
+  if {$in==50} {
+    do_test percentile-1.3.$in.4 {
+      execsql {SELECT median(x) FROM t1}
+    } $out
+  }
+  ifcapable ordered_set_aggregates {
+    do_test percentile-1.3.$in.5 {
+      execsql {SELECT percentile($in)WITHIN GROUP(ORDER BY x) FROM t1}
+    } $out
+    do_test percentile-1.3.$in.6 {
+      execsql {SELECT percentile_cont($in*0.01) WITHIN GROUP(ORDER BY x)
+                 FROM t1}
+    } $out
+    do_test percentile-1.3.$in.7 {
+      execsql {SELECT percentile_disc($in*0.01) WITHIN GROUP(ORDER BY x)
+                 FROM t1}
+    } $disc
+    if {$in==50} {
+      do_test percentile-1.3.$in.8 {
+        execsql {SELECT median() WITHIN GROUP (ORDER BY x) FROM t1}
+      } $out
+    }
+  }
 }
 
 # The second argument to percentile can change some, but not much.
 #
-do_test percentile-1.4 {
+do_test percentile-1.4.1 {
   catchsql {SELECT round(percentile(x, 15+0.000001*rowid),1) FROM t1}
 } {0 4.4}
-do_test percentile-1.5 {
-  catchsql {SELECT round(percentile(x, 15+0.1*rowid),1) FROM t1}
-} {1 {2nd argument to percentile() is not the same for all input rows}}
+do_test percentile-1.4.2 {
+  catchsql {SELECT round(percentile_cont(x,(15+0.000001*rowid)*0.01),1) FROM t1}
+} {0 4.4}
+do_test percentile-1.4.3 {
+  catchsql {SELECT percentile_disc(x, (15+0.000001*rowid)*0.01) FROM t1}
+} {0 4.0}
+do_test percentile-1.5.1 {
+  catchsql {SELECT percentile(x, 15+0.1*rowid) FROM t1}
+} {1 {the fraction argument to percentile() is not the same for all input rows}}
+do_test percentile-1.5.2 {
+  catchsql {SELECT percentile_cont(x, (15+0.1*rowid)*0.01) FROM t1}
+} {1 {the fraction argument to percentile_cont() is not the same for all input rows}}
+do_test percentile-1.5.3 {
+  catchsql {SELECT percentile_disc(x, (15+0.1*rowid)*0.01) FROM t1}
+} {1 {the fraction argument to percentile_disc() is not the same for all input rows}}
 
 # Input values in a random order
 #
@@ -75,59 +163,152 @@ do_test percentile-1.6 {
     INSERT INTO t2 SELECT x+0.0 FROM t1 ORDER BY random();
   }
 } {}
-foreach {in out} {
-  100    11.0
-   50     8.0
-   12.5   4.0
-   15     4.4
-   20     5.2
-   80    11.0
-   89    11.0
+foreach {in out disc} {
+  100    11.0  11.0
+   50     8.0   8.0
+   12.5   4.0   4.0
+   15     4.4   4.0
+   20     5.2   4.0
+   80    11.0  11.0
+   89    11.0  11.0
 } {
-  do_test percentile-1.7.$in {
+  do_test percentile-1.7.$in.1 {
     execsql {SELECT percentile(x,$in) FROM t2}
   } $out
+  do_test percentile-1.7.$in.2 {
+    execsql {SELECT percentile_cont(x,$in*0.01) FROM t2}
+  } $out
+  do_test percentile-1.7.$in.3 {
+    execsql {SELECT percentile_disc(x,$in*0.01) FROM t2}
+  } $disc
+  if {$in==50} {
+    do_test percentile-1.7.$in.4 {
+      execsql {SELECT median(x) FROM t2}
+    } $out
+  }
+  ifcapable ordered_set_aggregates {
+    do_test percentile-1.7.$in.5 {
+      execsql {SELECT percentile($in)WITHIN GROUP(ORDER BY x) FROM t2}
+    } $out
+    do_test percentile-1.7.$in.6 {
+      execsql {SELECT percentile_cont($in*0.01) WITHIN GROUP(ORDER BY x)
+                 FROM t2}
+    } $out
+    do_test percentile-1.7.$in.7 {
+      execsql {SELECT percentile_disc($in*0.01) WITHIN GROUP(ORDER BY x)
+                 FROM t2}
+    } $disc
+    if {$in==50} {
+      do_test percentile-1.7.$in.8 {
+        execsql {SELECT median() WITHIN GROUP (ORDER BY x) FROM t2}
+      } $out
+    }
+  }
 }
 
 # Wrong number of arguments
 #
-do_test percentile-1.8 {
+do_test percentile-1.8.1 {
   catchsql {SELECT percentile(x,0,1) FROM t1}
 } {1 {wrong number of arguments to function percentile()}}
-do_test percentile-1.9 {
+do_test percentile-1.8.2 {
+  catchsql {SELECT percentile_cont(x,0,1) FROM t1}
+} {1 {wrong number of arguments to function percentile_cont()}}
+do_test percentile-1.8.3 {
+  catchsql {SELECT percentile_disc(x,0,1) FROM t1}
+} {1 {wrong number of arguments to function percentile_disc()}}
+do_test percentile-1.8.4 {
+  catchsql {SELECT median(x,0) FROM t1}
+} {1 {wrong number of arguments to function median()}}
+ifcapable ordered_set_aggregates {
+  do_test percentile-1.8.5 {
+    catchsql {SELECT percentile(0,1) WITHIN GROUP(ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function percentile()}}
+  do_test percentile-1.8.2 {
+    catchsql {SELECT percentile_cont(0,1)WITHIN GROUP (ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function percentile_cont()}}
+  do_test percentile-1.8.3 {
+    catchsql {SELECT percentile_disc(0,1)WITHIN GROUP (ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function percentile_disc()}}
+  do_test percentile-1.8.4 {
+    catchsql {SELECT median(x) WITHIN GROUP (ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function median()}}
+}
+do_test percentile-1.9.1 {
   catchsql {SELECT percentile(x) FROM t1}
 } {1 {wrong number of arguments to function percentile()}}
+do_test percentile-1.9.2 {
+  catchsql {SELECT percentile_cont(x) FROM t1}
+} {1 {wrong number of arguments to function percentile_cont()}}
+do_test percentile-1.9.3 {
+  catchsql {SELECT percentile_disc(x) FROM t1}
+} {1 {wrong number of arguments to function percentile_disc()}}
+do_test percentile-1.9.4 {
+  catchsql {SELECT median() FROM t1}
+} {1 {wrong number of arguments to function median()}}
+ifcapable ordered_set_aggregates {
+  do_test percentile-1.9.5 {
+    catchsql {SELECT percentile() WITHIN GROUP(ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function percentile()}}
+  do_test percentile-1.9.6 {
+    catchsql {SELECT percentile_cont()WITHIN GROUP (ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function percentile_cont()}}
+  do_test percentile-1.9.7 {
+    catchsql {SELECT percentile_disc()WITHIN GROUP (ORDER BY x) FROM t1}
+  } {1 {wrong number of arguments to function percentile_disc()}}
+}
 
 # Second argument must be numeric
 #
 do_test percentile-1.10 {
   catchsql {SELECT percentile(x,null) FROM t1}
-} {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}}
+} {1 {the fraction argument to percentile() is not between 0.0 and 100.0}}
 do_test percentile-1.11 {
   catchsql {SELECT percentile(x,'fifty') FROM t1}
-} {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}}
+} {1 {the fraction argument to percentile() is not between 0.0 and 100.0}}
 do_test percentile-1.12 {
   catchsql {SELECT percentile(x,x'3530') FROM t1}
-} {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}}
+} {1 {the fraction argument to percentile() is not between 0.0 and 100.0}}
 
 # Second argument is out of range
 #
 do_test percentile-1.13 {
   catchsql {SELECT percentile(x,-0.0000001) FROM t1}
-} {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}}
+} {1 {the fraction argument to percentile() is not between 0.0 and 100.0}}
 do_test percentile-1.14 {
   catchsql {SELECT percentile(x,100.0000001) FROM t1}
-} {1 {2nd argument to percentile() is not a number between 0.0 and 100.0}}
+} {1 {the fraction argument to percentile() is not between 0.0 and 100.0}}
+do_test percentile-1.14.2 {
+  catchsql {SELECT percentile_cont(x,1.0000001) FROM t1}
+} {1 {the fraction argument to percentile_cont() is not between 0.0 and 1.0}}
+do_test percentile-1.14.3 {
+  catchsql {SELECT percentile_disc(x,1.0000001) FROM t1}
+} {1 {the fraction argument to percentile_disc() is not between 0.0 and 1.0}}
 
 # First argument is not NULL and is not NUMERIC
 #
-do_test percentile-1.15 {
+do_test percentile-1.15.1 {
   catchsql {
     BEGIN;
     UPDATE t1 SET x='50' WHERE x IS NULL;
     SELECT percentile(x, 50) FROM t1;
   }
-} {1 {1st argument to percentile() is not numeric}}
+} {1 {input to percentile() is not numeric}}
+do_test percentile-1.15.2 {
+  catchsql {
+    SELECT percentile_cont(x, 0.50) FROM t1;
+  }
+} {1 {input to percentile_cont() is not numeric}}
+do_test percentile-1.15.3 {
+  catchsql {
+    SELECT percentile_disc(x, 0.50) FROM t1;
+  }
+} {1 {input to percentile_disc() is not numeric}}
+do_test percentile-1.15.4 {
+  catchsql {
+    SELECT median(x) FROM t1;
+  }
+} {1 {input to median() is not numeric}}
 do_test percentile-1.16 {
   catchsql {
     ROLLBACK;
@@ -135,7 +316,7 @@ do_test percentile-1.16 {
     UPDATE t1 SET x=x'3530' WHERE x IS NULL;
     SELECT percentile(x, 50) FROM t1;
   }
-} {1 {1st argument to percentile() is not numeric}}
+} {1 {input to percentile() is not numeric}}
 do_test percentile-1.17 {
   catchsql {
     ROLLBACK;
@@ -163,7 +344,7 @@ do_test percentile-1.19 {
 
 # Infinity as an input
 #
-do_test percentile-1.20 {
+do_test percentile-1.20.1 {
   catchsql {
     DELETE FROM t1;
     INSERT INTO t1 SELECT x+0.0 FROM t2;
@@ -171,6 +352,43 @@ do_test percentile-1.20 {
     SELECT percentile(x,50) from t1;
   }
 } {1 {Inf input to percentile()}}
+do_test percentile-1.20.2 {
+  catchsql {
+    SELECT percentile_cont(x,0.50) from t1;
+  }
+} {1 {Inf input to percentile_cont()}}
+do_test percentile-1.20.3 {
+  catchsql {
+    SELECT percentile_disc(x,0.50) from t1;
+  }
+} {1 {Inf input to percentile_disc()}}
+do_test percentile-1.20.4 {
+  catchsql {
+    SELECT median(x) from t1;
+  }
+} {1 {Inf input to median()}}
+ifcapable ordered_set_aggregates {
+  do_test percentile-1.20.5 {
+    catchsql {
+      SELECT percentile(50) WITHIN GROUP (ORDER BY x) from t1;
+    }
+  } {1 {Inf input to percentile()}}
+  do_test percentile-1.20.6 {
+    catchsql {
+      SELECT percentile_cont(0.50) WITHIN GROUP (ORDER BY x) from t1;
+    }
+  } {1 {Inf input to percentile_cont()}}
+  do_test percentile-1.20.7 {
+    catchsql {
+      SELECT percentile_disc(0.50) WITHIN GROUP(ORDER BY X) from t1;
+    }
+  } {1 {Inf input to percentile_disc()}}
+  do_test percentile-1.20.8 {
+    catchsql {
+      SELECT median() WITHIN GROUP (ORDER BY x) from t1;
+    }
+  } {1 {Inf input to median()}}
+}
 do_test percentile-1.21 {
   catchsql {
     UPDATE t1 SET x=-1.0e300*1.0e300 WHERE rowid=5;
@@ -206,4 +424,172 @@ ifcapable vtab {
   }
 }
 
+# median() as a window function.  (2024-08-31)
+#
+do_execsql_test percentile-3.0 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d);
+  INSERT INTO t1 VALUES (1, 'A', 'one',   8.4),
+                        (2, 'B', 'two',   7.1),
+                        (3, 'C', 'three', 5.9),
+                        (4, 'D', 'one',  11.0),
+                        (5, 'E', 'two',  12.5),
+                        (6, 'F', 'three', 0.0),
+                        (7, 'G', 'one',   2.7);
+}
+foreach {id oba expr} {
+  1 0 "median(d)"
+  2 0 "percentile(d,50)"
+  3 0 "percentile_cont(d,0.5)"
+  4 1 "median() WITHIN GROUP (ORDER BY d)"
+  5 1 "percentile(50) WITHIN GROUP (ORDER BY d)"
+  6 1 "percentile_cont(0.5) WITHIN GROUP (ORDER BY d)"
+} {
+  if {$oba} {
+    ifcapable !ordered_set_aggregates break
+  }
+  set sql "SELECT a, b, c, d, \
+                  group_concat(b,'.') OVER w1 AS 'elements', \
+                  $expr OVER w1 AS 'median' \
+            FROM t1 \
+          WINDOW w1 AS (ORDER BY c, a ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)"
+  do_execsql_test percentile-3.$id.1 $sql {
+    1  A  one    8.4   A.D       9.7   
+    4  D  one    11.0  A.D.G     8.4   
+    7  G  one    2.7   D.G.C     5.9   
+    3  C  three  5.9   G.C.F     2.7   
+    6  F  three  0.0   C.F.B     5.9   
+    2  B  two    7.1   F.B.E     7.1   
+    5  E  two    12.5  B.E       9.8   
+  }
+
+  set sql "SELECT a, b, c, d, \
+                  group_concat(b,'.') OVER w1 AS 'elements', \
+                  $expr OVER w1 AS 'median' \
+            FROM t1 \
+           WINDOW w1 AS (ORDER BY c, a \
+               ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)"
+  do_execsql_test percentile-3.$id.2 $sql {
+    1  A  one    8.4   A.D            9.7   
+    4  D  one    11.0  A.D.G          8.4   
+    7  G  one    2.7   A.D.G.C        7.15  
+    3  C  three  5.9   A.D.G.C.F      5.9   
+    6  F  three  0.0   A.D.G.C.F.B    6.5   
+    2  B  two    7.1   A.D.G.C.F.B.E  7.1   
+    5  E  two    12.5  A.D.G.C.F.B.E  7.1   
+  }
+
+  set sql "SELECT a, b, c, d, \
+                  group_concat(b,'.') OVER w1 AS 'elements', \
+                  $expr OVER w1 AS 'median' \
+            FROM t1 \
+           WINDOW w1 AS (ORDER BY c, a \
+               ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING)"
+  do_execsql_test percentile-3.$id.3 $sql {
+    1  A  one    8.4   A.D.G.C.F.B.E  7.1   
+    4  D  one    11.0  A.D.G.C.F.B.E  7.1   
+    7  G  one    2.7   D.G.C.F.B.E    6.5   
+    3  C  three  5.9   G.C.F.B.E      5.9   
+    6  F  three  0.0   C.F.B.E        6.5   
+    2  B  two    7.1   F.B.E          7.1   
+    5  E  two    12.5  B.E            9.8   
+  }
+}
+
+# Test case adapted from examples shown at
+# https://database.guide/3-functions-to-calculate-the-median-in-sql/
+#
+do_execsql_test percential-4.0 {
+  CREATE TABLE products(
+    vendorId INT,
+    productId INTEGER PRIMARY KEY,
+    productName REAL,
+    price REAL
+  );
+  INSERT INTO products VALUES
+    (1001, 17,  'Left-handed screwdriver', 25.99),
+    (1001, 49,  'Right-handed screwdriver', 25.99),
+    (1001, 216, 'Long weight (blue)', 14.75),
+    (1001, 31,  'Long weight (green)', 11.99),
+    (1002, 37,  'Sledge hammer', 33.49),
+    (1003, 7,   'Chainsaw', 245.00),
+    (1003, 8,   'Straw dog box', 55.99),
+    (1003, 12,  'Hammock', 11.01),
+    (1004, 113, 'Teapot', 12.45),
+    (1004, 117, 'Bottomless coffee mug', 9.99);
+}
+do_execsql_test percentile-4.1 {
+  SELECT VendorId, ProductId, /* ProductName,*/ Price,
+         avg(price) OVER (PARTITION BY vendorId) AS "Average",
+         median(price) OVER (PARTITION BY vendorId) AS "Median"
+    FROM products
+   ORDER BY vendorId, productId;
+} {
+  1001      17         25.99  19.68    20.37 
+  1001      31         11.99  19.68    20.37 
+  1001      49         25.99  19.68    20.37 
+  1001      216        14.75  19.68    20.37 
+  1002      37         33.49  33.49    33.49 
+  1003      7          245.0  104.0    55.99 
+  1003      8          55.99  104.0    55.99 
+  1003      12         11.01  104.0    55.99 
+  1004      113        12.45  11.22    11.22 
+  1004      117        9.99   11.22    11.22 
+}
+do_execsql_test percentile-4.2 {
+  SELECT vendorId, median(price) FROM products
+   GROUP BY 1 ORDER BY 1;
+} {1001 20.37 1002 33.49 1003 55.99 1004 11.22}
+
+do_execsql_test percentile-5.0 {
+  CREATE TABLE user(name TEXT, class TEXT, cost REAL);
+  INSERT INTO user VALUES
+    ('Alice', 'Y',  3578.27),
+    ('Bob',   'X',  3399.99),
+    ('Cindy', 'Z',  699.10),
+    ('Dave',  'Y',  3078.27),
+    ('Emma',  'Z',  2319.99),
+    ('Fred',  'Y',  539.99),
+    ('Gina',  'X',  2320.49),
+    ('Hank',  'W',  24.99),
+    ('Irma',  'W',  24.99),
+    ('Jake',  'X',  2234.99),
+    ('Kim',   'Y',  4319.99),
+    ('Liam',  'X',  4968.59),
+    ('Mia',   'W',  59.53),
+    ('Nate',  'W',  23.50);
+}
+do_execsql_test percentile-5.1 {
+  SELECT name, class, cost,
+    percentile(cost,   0) OVER w1 AS 'P0',
+    percentile(cost,  25) OVER w1 AS 'P1',
+    percentile(cost,  50) OVER w1 AS 'P2',
+    percentile(cost,  75) OVER w1 AS 'P3',
+    percentile(cost, 100) OVER w1 AS 'P4'
+  FROM user
+  WINDOW w1 AS (PARTITION BY class)
+  ORDER BY class, cost;
+} {
+  Nate   W  23.5     23.5     24.6175    24.99     33.625     59.53  
+  Hank   W  24.99    23.5     24.6175    24.99     33.625     59.53  
+  Irma   W  24.99    23.5     24.6175    24.99     33.625     59.53  
+  Mia    W  59.53    23.5     24.6175    24.99     33.625     59.53  
+  Jake   X  2234.99  2234.99  2299.115   2860.24   3792.14    4968.59
+  Gina   X  2320.49  2234.99  2299.115   2860.24   3792.14    4968.59
+  Bob    X  3399.99  2234.99  2299.115   2860.24   3792.14    4968.59
+  Liam   X  4968.59  2234.99  2299.115   2860.24   3792.14    4968.59
+  Fred   Y  539.99   539.99   2443.7     3328.27   3763.7     4319.99
+  Dave   Y  3078.27  539.99   2443.7     3328.27   3763.7     4319.99
+  Alice  Y  3578.27  539.99   2443.7     3328.27   3763.7     4319.99
+  Kim    Y  4319.99  539.99   2443.7     3328.27   3763.7     4319.99
+  Cindy  Z  699.1    699.1    1104.3225  1509.545  1914.7675  2319.99
+  Emma   Z  2319.99  699.1    1104.3225  1509.545  1914.7675  2319.99
+}
+
+# Fuzzer find.
+do_execsql_test percentile-6.0 {
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12)
+  SELECT median(iif(n%2,0.1,1.0)) FROM c;
+} 0.55
+
 finish_test
diff --git a/test/permutations.test b/test/permutations.test
index eb173f79b9..02f4827189 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -10,9 +10,11 @@
 #***********************************************************************
 #
 
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-db close
+if {[info vars ::trd::tcltest]==""} {
+  set testdir [file dirname $argv0]
+  source $testdir/tester.tcl
+  db close
+}
 
 #-------------------------------------------------------------------------
 # test_suite NAME OPTIONS
@@ -93,6 +95,7 @@ foreach f [glob -nocomplain            \
     $testdir/../ext/lsm1/test/*.test   \
     $testdir/../ext/recover/*.test     \
     $testdir/../ext/rbu/*.test         \
+    $testdir/../ext/intck/*.test       \
 ] {
   lappend alltests $f 
 }
@@ -101,18 +104,18 @@ foreach f [glob -nocomplain $testdir/../ext/session/*.test] {
 }
 unset f
 
-if {$::tcl_platform(platform)!="unix"} {
+if {$::tcl_platform(platform) ne "unix"} {
   set alltests [test_set $alltests -exclude crash.test crash2.test]
 }
 set alltests [test_set $alltests -exclude {
-  all.test        async.test         quick.test  veryquick.test
+  all.test        quick.test  veryquick.test
   memleak.test    permutations.test  soak.test   fts3.test
   mallocAll.test  rtree.test         full.test   extraquick.test
   session.test    rbu.test
 }]
 
 set allquicktests [test_set $alltests -exclude {
-  async2.test async3.test backup_ioerr.test corrupt.test
+  backup_ioerr.test corrupt.test
   corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test
   crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test
   fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test
@@ -787,7 +790,7 @@ test_suite "inmemory_journal" -description {
   journal3.test 8_3_names.test shmlock.test
   pendingrace.test
 
-  pager1.test async4.test corrupt.test filefmt.test pager2.test
+  pager1.test corrupt.test filefmt.test pager2.test
   corrupt5.test corruptA.test pageropt.test
 
   # Exclude stmt.test, which expects sub-journals to use temporary files.
@@ -813,117 +816,111 @@ test_suite "inmemory_journal" -description {
   recoverpgsz.test
 }]
 
-ifcapable mem3 {
-  test_suite "memsys3" -description {
-    Run tests using the allocator in mem3.c.
-  } -files [test_set $::allquicktests -exclude {
-    autovacuum.test           delete3.test              manydb.test
-    bigrow.test               incrblob2.test            memdb.test
-    bitvec.test               index2.test               memsubsys1.test
-    capi3c.test               ioerr.test                memsubsys2.test
-    capi3.test                join3.test                pagesize.test
-    collate5.test             limit.test                backup_ioerr.test
-    backup_malloc.test
-  }] -initialize {
-    catch {db close}
-    sqlite3_reset_auto_extension
-    sqlite3_shutdown
-    sqlite3_config_heap 25000000 0
-    sqlite3_config_lookaside 0 0
-    ifcapable mem5 {
-      # If both memsys3 and memsys5 are enabled in the build, the call to
-      # [sqlite3_config_heap] will initialize the system to use memsys5.
-      # The following overrides this preference and installs the memsys3
-      # allocator.
-      sqlite3_install_memsys3
-    }
-    install_malloc_faultsim 1 
-    sqlite3_initialize
-    autoinstall_test_functions
-  } -shutdown {
-    catch {db close}
-    sqlite3_shutdown
-    sqlite3_config_heap 0 0
-    sqlite3_config_lookaside 100 500
-    install_malloc_faultsim 1 
-    sqlite3_initialize
-    autoinstall_test_functions
+test_suite "memsys3" -description {
+  Run tests using the allocator in mem3.c.
+} -files [test_set $::allquicktests -exclude {
+  autovacuum.test           delete3.test              manydb.test
+  bigrow.test               incrblob2.test            memdb.test
+  bitvec.test               index2.test               memsubsys1.test
+  capi3c.test               ioerr.test                memsubsys2.test
+  capi3.test                join3.test                pagesize.test
+  collate5.test             limit.test                backup_ioerr.test
+  backup_malloc.test
+}] -initialize {
+  catch {db close}
+  sqlite3_reset_auto_extension
+  sqlite3_shutdown
+  sqlite3_config_heap 25000000 0
+  sqlite3_config_lookaside 0 0
+  ifcapable mem5 {
+    # If both memsys3 and memsys5 are enabled in the build, the call to
+    # [sqlite3_config_heap] will initialize the system to use memsys5.
+    # The following overrides this preference and installs the memsys3
+    # allocator.
+    sqlite3_install_memsys3
   }
+  install_malloc_faultsim 1 
+  sqlite3_initialize
+  autoinstall_test_functions
+} -shutdown {
+  catch {db close}
+  sqlite3_shutdown
+  sqlite3_config_heap 0 0
+  sqlite3_config_lookaside 100 500
+  install_malloc_faultsim 1 
+  sqlite3_initialize
+  autoinstall_test_functions
 }
 
-ifcapable mem5 {
-  test_suite "memsys5" -description {
-    Run tests using the allocator in mem5.c.
-  } -files [test_set $::allquicktests -exclude {
-    autovacuum.test           delete3.test              manydb.test
-    bigrow.test               incrblob2.test            memdb.test
-    bitvec.test               index2.test               memsubsys1.test
-    capi3c.test               ioerr.test                memsubsys2.test
-    capi3.test                join3.test                pagesize.test
-    collate5.test             limit.test                zeroblob.test
-  }] -initialize {
-    catch {db close}
-    sqlite3_shutdown
-    sqlite3_config_heap 25000000 64
-    sqlite3_config_lookaside 0 0
-    install_malloc_faultsim 1 
-    sqlite3_initialize
-    autoinstall_test_functions
-  } -shutdown {
-    catch {db close}
-    sqlite3_shutdown
-    sqlite3_config_heap 0 0
-    sqlite3_config_lookaside 100 500
-    install_malloc_faultsim 1 
-    sqlite3_initialize
-    autoinstall_test_functions
-  }
+test_suite "memsys5" -description {
+  Run tests using the allocator in mem5.c.
+} -files [test_set $::allquicktests -exclude {
+  autovacuum.test           delete3.test              manydb.test
+  bigrow.test               incrblob2.test            memdb.test
+  bitvec.test               index2.test               memsubsys1.test
+  capi3c.test               ioerr.test                memsubsys2.test
+  capi3.test                join3.test                pagesize.test
+  collate5.test             limit.test                zeroblob.test
+}] -initialize {
+  catch {db close}
+  sqlite3_shutdown
+  sqlite3_config_heap 25000000 64
+  sqlite3_config_lookaside 0 0
+  install_malloc_faultsim 1 
+  sqlite3_initialize
+  autoinstall_test_functions
+} -shutdown {
+  catch {db close}
+  sqlite3_shutdown
+  sqlite3_config_heap 0 0
+  sqlite3_config_lookaside 100 500
+  install_malloc_faultsim 1 
+  sqlite3_initialize
+  autoinstall_test_functions
+}
 
-  test_suite "memsys5-2" -description {
-    Run tests using the allocator in mem5.c in a different configuration.
-  } -files {
-    select1.test 
-  } -initialize {
-    catch {db close}
-    sqlite3_shutdown
-    sqlite3_config_memstatus 0
-    sqlite3_config_heap 40000000 16
-    sqlite3_config_lookaside 0 0
-    install_malloc_faultsim 1 
-    sqlite3_initialize
-    autoinstall_test_functions
-  } -shutdown {
-    catch {db close}
-    sqlite3_shutdown
-    sqlite3_config_heap 0 0
-    sqlite3_config_lookaside 100 500
-    install_malloc_faultsim 1 
-    sqlite3_initialize
-    autoinstall_test_functions
-  }
+test_suite "memsys5-2" -description {
+  Run tests using the allocator in mem5.c in a different configuration.
+} -files {
+  select1.test 
+} -initialize {
+  catch {db close}
+  sqlite3_shutdown
+  sqlite3_config_memstatus 0
+  sqlite3_config_heap 40000000 16
+  sqlite3_config_lookaside 0 0
+  install_malloc_faultsim 1 
+  sqlite3_initialize
+  autoinstall_test_functions
+} -shutdown {
+  catch {db close}
+  sqlite3_shutdown
+  sqlite3_config_heap 0 0
+  sqlite3_config_lookaside 100 500
+  install_malloc_faultsim 1 
+  sqlite3_initialize
+  autoinstall_test_functions
 }
 
-ifcapable threadsafe {
-  test_suite "no_mutex_try" -description {
-     The sqlite3_mutex_try() interface always fails
-  } -files [
-    test_set $::allquicktests -exclude mutex1.test mutex2.test
-  ] -initialize {
-    catch {db close}
-    sqlite3_shutdown
-    install_mutex_counters 1
-    set ::disable_mutex_try 1
-    sqlite3_initialize
-    autoinstall_test_functions
-  } -shutdown {
-    catch {db close}
-    catch {db2 close}
-    catch {db3 close}
-    sqlite3_shutdown
-    install_mutex_counters 0
-    sqlite3_initialize
-    autoinstall_test_functions
-  }
+test_suite "no_mutex_try" -description {
+   The sqlite3_mutex_try() interface always fails
+} -files [
+  test_set $::allquicktests -exclude mutex1.test mutex2.test
+] -initialize {
+  catch {db close}
+  sqlite3_shutdown
+  install_mutex_counters 1
+  set ::disable_mutex_try 1
+  sqlite3_initialize
+  autoinstall_test_functions
+} -shutdown {
+  catch {db close}
+  catch {db2 close}
+  catch {db3 close}
+  sqlite3_shutdown
+  install_mutex_counters 0
+  sqlite3_initialize
+  autoinstall_test_functions
 }
 
 # run_tests "crash_safe_append" -description {
@@ -954,12 +951,11 @@ test_suite "safe_append" -description {
   set ::G(perm:sqlite3_args) [list -vfs devsym]
   sqlite3_simulate_device -char safe_append
 } -files [
-  test_set $::allquicktests shared_err.test -exclude async3.test
+  test_set $::allquicktests shared_err.test
 ]
 
 # The set of tests to run on the alternative-pcache
 set perm-alt-pcache-testset {
-  async.test
   attach.test
   delete.test delete2.test
   index.test
@@ -1000,7 +996,7 @@ test_suite "journaltest" -description {
   unregister_jt_vfs
 } -files [test_set $::allquicktests -exclude {
   wal* incrvacuum.test ioerr.test corrupt4.test io.test crash8.test 
-  async4.test bigfile.test backcompat.test e_wal* fstat.test mmap2.test
+  bigfile.test backcompat.test e_wal* fstat.test mmap2.test
   pager1.test syscall.test tkt3457.test *malloc* mmap* multiplex* nolock*
   pager2.test *fault* rowal* snapshot* superlock* symlink.test
   delete_db.test shmlock.test chunksize.test
@@ -1123,6 +1119,18 @@ test_suite "maindbname" -prefix "" -description {
   dbconfig_maindbname_icecube $::dbhandle
 }
 
+test_suite "win_unc_locking" -prefix "" -description {
+  Run the "wal*" tests with UNC style single fd locking.
+} -files [
+  test_set                                      \
+      {*}[glob [file join $testdir wal*]]       \
+      -exclude *fault* *malloc* *crash* *slow* walsetlk.test
+] -initialize {
+  set sqlite3_win_test_unc_locking 1
+} -shutdown {
+  set sqlite3_win_test_unc_locking 0
+}
+
 # End of tests
 #############################################################################
 
diff --git a/test/pragma.test b/test/pragma.test
index 5b45a74400..5249cc7c41 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -372,27 +372,30 @@ ifcapable attach {
       db close
       sqlite3 db test.db
       execsql {PRAGMA integrity_check}
-    } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+    } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}
     do_test pragma-3.3 {
       execsql {PRAGMA integrity_check=1}
-    } {{row 1 missing from index i2}}
+    } {{wrong # of entries in index i2}}
     do_test pragma-3.4 {
       execsql {
         ATTACH DATABASE 'test.db' AS t2;
         PRAGMA integrity_check
       }
-    } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+    } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}
     do_test pragma-3.5 {
       execsql {
         PRAGMA integrity_check=4
       }
-    } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2}}
+    } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+    do_catchsql_test pragma-3.5.2 {
+      PRAGMA integrity_check='4'
+    } {1 {no such table: 4}}
     do_catchsql_test pragma-3.6 {
       PRAGMA integrity_check=xyz
     } {1 {no such table: xyz}}
     do_catchsql_test pragma-3.6b {
       PRAGMA integrity_check=t2
-    } {0 {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}}
+    } {0 {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}}
     do_catchsql_test pragma-3.6c {
       PRAGMA integrity_check=sqlite_schema
     } {0 ok}
@@ -400,7 +403,7 @@ ifcapable attach {
       execsql {
         PRAGMA integrity_check=0
       }
-    } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+    } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}
   
     # Add additional corruption by appending unused pages to the end of
     # the database file testerr.db
@@ -435,10 +438,10 @@ ifcapable attach {
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}
     do_execsql_test pragma-3.9b {
       PRAGMA t2.integrity_check=t2;
-    } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+    } {{wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}
     do_execsql_test pragma-3.9c {
       PRAGMA t2.integrity_check=sqlite_schema;
     } {ok}
@@ -455,7 +458,7 @@ Page 4: never used}}
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}}
+Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2}}
     do_test pragma-3.12 {
       execsql {
         PRAGMA integrity_check=4
@@ -463,7 +466,7 @@ Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}}
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2}}
+Page 6: never used} {wrong # of entries in index i2}}
     do_test pragma-3.13 {
       execsql {
         PRAGMA integrity_check=3
@@ -487,10 +490,10 @@ Page 5: never used}}
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 ***
+Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {*** in database t3 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}}
+Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2}}
     do_test pragma-3.16 {
       execsql {
         PRAGMA integrity_check(10)
@@ -498,10 +501,10 @@ Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 ***
+Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {*** in database t3 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2}}
+Page 6: never used} {wrong # of entries in index i2}}
     do_test pragma-3.17 {
       execsql {
         PRAGMA integrity_check=8
@@ -509,7 +512,7 @@ Page 6: never used} {row 1 missing from index i2}}
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 ***
+Page 6: never used} {wrong # of entries in index i2} {row 1 missing from index i2} {row 2 missing from index i2} {*** in database t3 ***
 Page 4: never used
 Page 5: never used}}
     do_test pragma-3.18 {
@@ -519,7 +522,7 @@ Page 5: never used}}
     } {{*** in database t2 ***
 Page 4: never used
 Page 5: never used
-Page 6: never used} {row 1 missing from index i2}}
+Page 6: never used} {wrong # of entries in index i2}}
   }
   do_test pragma-3.19 {
     catch {db close}
@@ -556,6 +559,21 @@ ifcapable altertable {
   do_execsql_test pragma-3.23 {
     PRAGMA integrity_check(1);
   } {{non-unique entry in index t1a}}
+
+  # forum post https://sqlite.org/forum/forumpost/ee4f6fa5ab
+  do_execsql_test pragma-3.24 {
+    DROP TABLE IF EXISTS t1;
+    CREATE TABLE t1(a);
+    INSERT INTO t1 VALUES (1);
+    ALTER TABLE t1 ADD COLUMN b NOT NULL DEFAULT 0.25;
+    SELECT * FROM t1;
+    PRAGMA integrity_check(t1);
+  } {1 0.25 ok}
+  do_execsql_test pragma-3.25 {
+    ALTER TABLE t1 ADD COLUMN c CHECK (1);
+    SELECT * FROM t1;
+    PRAGMA integrity_check(t1);
+  } {1 0.25 {} ok}
 }
 
 # PRAGMA integrity check (or more specifically the sqlite3BtreeCount()
@@ -1805,7 +1823,7 @@ do_test pragma-19.5 {
   file tail [lindex [execsql {PRAGMA filename}] 0]
 } {test.db}
 
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
 # Test data_store_directory pragma
 #
 db close
diff --git a/test/pragma4.test b/test/pragma4.test
index b82df81cbd..2ba87c0c60 100644
--- a/test/pragma4.test
+++ b/test/pragma4.test
@@ -83,7 +83,7 @@ foreach {tn sql} {
 # Verify that that P4_INTARRAY argument to OP_IntegrityCk is rendered
 # correctly.
 #
-db close
+catch {db close}
 forcedelete test.db
 sqlite3 db test.db
 do_test pragma4-2.100 {
@@ -97,7 +97,7 @@ do_test pragma4-2.100 {
   }
   string map {\[ x \] x \173 {} \175 {}} \
     [db eval {EXPLAIN PRAGMA integrity_check}]
-} {/ IntegrityCk 2 2 1 x[0-9]+,1x /}
+} {/ IntegrityCk 1 2 8 x[0-9]+,1x /}
 
 
 #--------------------------------------------------------------------------
@@ -264,5 +264,72 @@ do_execsql_test 5.0 {
   0 a {} 0 'abc' 0 1 b {} 0 -1 0 2 c {} 0 +4.0 0
 }
 
+# 2024-03-24 https://sqlite.org/forum/forumpost/85b6a8b6705fb77a
+#
+catch {db2 close}
+catch {db3 close}
+ifcapable vtab {
+  reset_db
+  do_execsql_test 6.0 {
+    DROP TABLE IF EXISTS t1;
+    DROP TABLE IF EXISTS t2;
+    CREATE TABLE t1(a INT PRIMARY KEY, b INT);
+    CREATE TABLE t2(c INT PRIMARY KEY, d INT REFERENCES t1);
+    SELECT t.name, f."table", f."from", i.name, i.pk
+      FROM pragma_table_list() AS t
+           JOIN pragma_foreign_key_list(t.name, t.schema) AS f
+           JOIN pragma_table_info(f."table", t.schema) AS i
+     WHERE i.pk;
+  } {t2 t1 d a 1}
+
+  # With a corrupt VIEW in the schema, the PRAGMA table_list command
+  # will generate internal errors.  Confirm that these internal errors
+  # do not appears on the log.  https://sqlite.org/src/forumpost/00ee467e
+  test_sqlite3_log [list lappend ::log]
+  set ::log {}
+  do_execsql_test 6.1 {
+    CREATE VIEW v1 AS SELECT abs(a) FROM t1;
+    PRAGMA writable_schema=ON;
+    UPDATE sqlite_schema
+       SET sql=replace(sql,'abs(a)','nosuchfunc(a)')
+     WHERE name='v1';
+    PRAGMA writable_schema=RESET;
+  } {}
+  do_execsql_test 6.2 {
+    PRAGMA table_list;
+  } {main v1 view 0 0 0 main t2 table 2 0 0 main t1 table 2 0 0 main sqlite_schema table 5 0 0 temp sqlite_temp_schema table 5 0 0}
+  do_test 6.3 {
+    set ::log
+  } {}
+  test_sqlite3_log
+}
+
+# 2024-05-08 https://sqlite.org/forum/forumpost/cf29a33e94
+#
+ifcapable vtab {
+  do_execsql_test 7.0 {
+    CREATE TABLE t3 ("a" TEXT, "b" TEXT);
+    CREATE TABLE t4 ("a" TEXT, "b" TEXT, "c" TEXT); 
+  }
+  
+  do_execsql_test 7.1 {
+    CREATE TABLE pragma_t3 AS SELECT * FROM pragma_table_info('t3');
+    CREATE TABLE pragma_t4 AS SELECT * FROM pragma_table_info('t4');
+  }
+  
+  do_execsql_test 7.2 {
+    SELECT pragma_t4.name, pragma_t3.name 
+      FROM pragma_t4 RIGHT JOIN pragma_t3 ON (pragma_t4.name=pragma_t3.name);
+  } {a a b b}
+  
+  do_execsql_test 7.3 {
+    SELECT t4.name, t3.name 
+    FROM pragma_table_info('t4') t4 
+    RIGHT JOIN pragma_table_info('t3') t3 ON (t4.name=t3.name);
+  } {a a b b}
+}
+
+
+
 
 finish_test
diff --git a/test/pragma6.test b/test/pragma6.test
new file mode 100644
index 0000000000..fc5566af10
--- /dev/null
+++ b/test/pragma6.test
@@ -0,0 +1,74 @@
+# 2024 February 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for PRAGMAs quick_check and integrity_check.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix pragma6
+
+database_may_be_corrupt
+
+#-------------------------------------------------------------------------
+#
+do_test 1.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+    .open --hexdb
+    | size 12288 pagesize 4096 filename crash-540f4c1eb1e7ac.db
+    | page 1 offset 0
+    |      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+    |     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 03   .....@  ........
+    |     32: 00 bb 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+    |     96: 00 00 00 00 0d 00 00 00 02 0f 7f 00 0f c3 0f 7f   ................
+    |   3952: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 42   ...............B
+    |   3968: 02 06 17 11 11 01 71 74 61 62 6c 65 74 32 74 32   ......qtablet2t2
+    |   3984: 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 32   .CREATE TABLE t2
+    |   4000: 28 61 20 49 4e 54 2c 20 62 20 41 53 20 28 61 2a   (a INT, b AS (a*
+    |   4016: 32 29 20 53 54 4f 52 45 44 20 4e 4f 54 20 4e 55   2) STORED NOT NU
+    |   4032: 4c 4c 29 3b 01 06 17 11 11 01 63 74 61 62 6c 65   LL);......ctable
+    |   4048: 74 31 74 31 02 43 52 45 41 54 45 20 54 41 42 4c   t1t1.CREATE TABL
+    |   4064: 45 20 74 31 28 61 20 49 4e 54 2c 20 62 20 41 53   E t1(a INT, b AS
+    |   4080: 20 28 61 2a 32 29 20 4e 4f 54 20 4e 55 4c 4c 29    (a*2) NOT NULL)
+    | page 2 offset 4096
+    |      0: 0d 00 00 00 05 0f e7 00 00 00 00 00 00 00 00 00   ................
+    |   4064: 00 00 00 00 00 00 00 00 03 05 02 01 05 03 04 02   ................
+    |   4080: 01 04 03 03 02 01 03 03 02 02 01 02 02 01 02 09   ................
+    | page 3 offset 8192
+    |      0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
+    |   4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 05   ................
+    |   4064: 03 01 01 05 0a 05 04 03 01 01 04 08 05 03 03 01   ................
+    |   4080: 01 03 06 05 02 03 00 00 00 00 00 00 00 00 00 00   ................
+    | end crash-540f4c1eb1e7ac.db
+  }]
+} {}
+
+do_test 1.1 {
+  execsql {
+    CREATE TEMP TABLE t2(
+        a t1 PRIMARY KEY default 27,
+        b default(current_timestamp),
+        d TEXT UNIQUE DEFAULT 'ch`arlie',
+        c TEXT UNIQUE DEFAULT 084,
+        UNIQUE(c,b,b,a,b)
+    ) WITHOUT ROWID;
+  }
+  catchsql { INSERT INTO t1(a) VALUES(zeroblob(40000)) }
+  set {} {}
+} {}
+
+do_test 1.2 {
+  execsql { PRAGMA integrity_check; }
+  execsql { PRAGMA quick_check; }
+  set {} {}
+} {}
+
+finish_test
diff --git a/test/printf.test b/test/printf.test
index 6d4ad71d28..cc439e6172 100644
--- a/test/printf.test
+++ b/test/printf.test
@@ -3833,4 +3833,21 @@ do_execsql_test printf-18.1 {
   SELECT length( format('%,.249f', -5.0e-300) );
 } {252}
 
+# 2024-02-16
+# https://sqlite.org/forum/info/393708f4a8
+#
+# The problem introduced by on 2023-07-03 by
+# https://sqlite.org/src/info/32befb224b254639
+#
+do_execsql_test printf-19.1 {
+  SELECT format('%0.0f %0.0g %0.0g', 0.9, 0.09, 1.9);
+} {{1 0.09 2}}
+do_execsql_test printf-19.2 {
+  SELECT format('%0.0f %#0.0f',0.0, 0.0);
+} {{0 0.}}
+do_execsql_test printf-19.3 {
+  SELECT format('%,.0f %,.0f',12345e+10, 12345e+11);
+} {{123,450,000,000,000 1,234,500,000,000,000}}
+
+
 finish_test
diff --git a/test/pushdown.test b/test/pushdown.test
index 1fbe6f34cd..cb9042d258 100644
--- a/test/pushdown.test
+++ b/test/pushdown.test
@@ -1,4 +1,4 @@
-# 2017 April 29
+# 2017-04-29
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
@@ -8,6 +8,26 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+#
+# Test cases for the push-down optimizations.
+#
+#
+# There are two different meanings for "push-down optimization".
+#
+#   (1)  "MySQL push-down" means that WHERE clause terms that can be
+#        evaluated using only the index and without reference to the
+#        table are run first, so that if they are false, unnecessary table
+#        seeks are avoided.  See https://sqlite.org/src/info/d7bb79ed3a40419d
+#        from 2017-04-29.
+#
+#   (2)  "WHERE-clause pushdown" means to push WHERE clause terms in
+#        outer queries down into subqueries.  See
+#        https://sqlite.org/src/info/6df18e949d367629 from 2015-06-02.
+#
+# This module started out as tests for MySQL push-down only.  But because
+# of naming ambiguity, it has picked up test cases for WHERE-clause push-down
+# over the years.
+#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -87,8 +107,8 @@ do_test 2.2 {
 } {three}
 
 # 2022-11-25 dbsqlfuzz crash-3a548de406a50e896c1bf7142692d35d339d697f
-# Disable the push-down optimization for compound subqueries if any
-# arm of the compound has an incompatible affinity.
+# Disable the WHERE-clause push-down optimization for compound subqueries
+# if any arm of the compound has an incompatible affinity.
 #
 reset_db
 do_execsql_test 3.1 {
@@ -185,7 +205,7 @@ do_eqp_test 3.8 {
 #   SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2)
 
 # 2023-05-09 https://sqlite.org/forum/forumpost/a7d4be7fb6
-# Restriction (9) on the push-down optimization.
+# Restriction (9) on the WHERE-clause push-down optimization.
 # 
 reset_db
 db null -
@@ -227,4 +247,113 @@ do_execsql_test 5.0 {
    WHERE e>0;
 } {- - 3 4 5}
 
+
+# 2024-04-05
+# Allow push-down of operators of the form "expr IN table".
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t01(w,x,y,z);
+  CREATE TABLE t02(w,x,y,z);
+  CREATE VIEW t0(w,x,y,z) AS
+    SELECT w,x,y,z FROM t01 UNION ALL SELECT w,x,y,z FROM t02;
+  CREATE INDEX t01x ON t01(w,x,y);
+  CREATE INDEX t02x ON t02(w,x,y);
+  CREATE VIEW v1(k) AS VALUES(77),(88),(99);
+  CREATE TABLE k1(k);
+  INSERT INTO k1 SELECT * FROM v1;
+}
+do_eqp_test 6.1 {
+  WITH k(n) AS (VALUES(77),(88),(99))
+  SELECT max(z) FROM t0 WHERE w=123 AND x IN k AND y BETWEEN 44 AND 55;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE t0
+  |  `--COMPOUND QUERY
+  |     |--LEFT-MOST SUBQUERY
+  |     |  |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?)
+  |     |  `--LIST SUBQUERY xxxxxx
+  |     |     |--MATERIALIZE k
+  |     |     |  `--SCAN 3 CONSTANT ROWS
+  |     |     |--SCAN k
+  |     |     `--CREATE BLOOM FILTER
+  |     `--UNION ALL
+  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
+  |        `--REUSE LIST SUBQUERY xxxxxx
+  |--SEARCH t0
+  `--REUSE LIST SUBQUERY xxxxxx
+}
+# ^^^^--- The key feature above is that the SEARCH for each subquery
+# uses all three fields of the index w, x, and y.  Prior to the push-down
+# of "expr IN table", only the w term of the index would be used.  Similar
+# for the following tests:
+#
+do_eqp_test 6.2 {
+  SELECT max(z) FROM t0 WHERE w=123 AND x IN v1 AND y BETWEEN 44 AND 55;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE t0
+  |  `--COMPOUND QUERY
+  |     |--LEFT-MOST SUBQUERY
+  |     |  |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?)
+  |     |  `--LIST SUBQUERY xxxxxx
+  |     |     |--CO-ROUTINE v1
+  |     |     |  `--SCAN 3 CONSTANT ROWS
+  |     |     |--SCAN v1
+  |     |     `--CREATE BLOOM FILTER
+  |     `--UNION ALL
+  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
+  |        `--REUSE LIST SUBQUERY xxxxxx
+  |--SEARCH t0
+  `--REUSE LIST SUBQUERY xxxxxx
+}
+do_eqp_test 6.3 {
+  SELECT max(z) FROM t0 WHERE w=123 AND x IN k1 AND y BETWEEN 44 AND 55;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE t0
+  |  `--COMPOUND QUERY
+  |     |--LEFT-MOST SUBQUERY
+  |     |  |--SEARCH t01 USING INDEX t01x (w=? AND x=? AND y>? AND y<?)
+  |     |  `--LIST SUBQUERY xxxxxx
+  |     |     |--SCAN k1
+  |     |     `--CREATE BLOOM FILTER
+  |     `--UNION ALL
+  |        |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
+  |        `--REUSE LIST SUBQUERY xxxxxx
+  |--SEARCH t0
+  `--REUSE LIST SUBQUERY xxxxxx
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE  TABLE  t0_1(a INT , b INT, c INT);
+  CREATE  TABLE  t0_2(a INT , b INT, c INT);
+
+  INSERT INTO t0_1 (a, b, c) VALUES (1, 0, 1);
+  INSERT INTO t0_2 (a, b, c) VALUES (1, 0, 1);
+
+  CREATE  TABLE  empty1(x);
+  CREATE  TABLE  empty2(y);
+}
+
+do_execsql_test 7.1 {
+  SELECT t0_2.c
+    FROM (SELECT '0000' AS c0 FROM empty2 RIGHT JOIN t0_1 ON 1) AS v0 
+    LEFT JOIN empty1 ON v0.c0, t0_2 
+    RIGHT JOIN (
+           SELECT 5678 AS col0 FROM (SELECT 0)
+    ) AS sub1 ON 1;
+} {1}
+
+do_execsql_test 7.2 {
+  SELECT t0_2.c
+    FROM (SELECT '0000' AS c0 FROM empty2 RIGHT JOIN t0_1 ON 1) AS v0 
+    LEFT JOIN empty1 ON v0.c0, t0_2 
+    RIGHT JOIN (
+           SELECT 5678 AS col0 FROM (SELECT 0)
+    ) AS sub1 ON 1 WHERE +t0_2.c;
+} {1}
+
 finish_test
diff --git a/test/quickcheck.test b/test/quickcheck.test
index 94016e845f..18c42a13d0 100644
--- a/test/quickcheck.test
+++ b/test/quickcheck.test
@@ -31,4 +31,3 @@ do_execsql_test 1.1 {
 }
 
 finish_test
-
diff --git a/test/quota.test b/test/quota.test
index 5d0bda3ad3..f49600043e 100644
--- a/test/quota.test
+++ b/test/quota.test
@@ -216,7 +216,7 @@ do_test quota-3.2.9 {
 set ::quota [list]
 proc quota_callback {file limitvar size} {
   upvar $limitvar limit
-  if {$::tcl_platform(platform)=="windows"} {
+  if {$::tcl_platform(platform) eq "windows"} {
     set file [ lindex [string map {\\ \/} $file] 0 ]
   }
   lappend ::quota $file $size
@@ -351,7 +351,7 @@ do_test quota-4.3.1 {
 } {}
 
 unset -nocomplain quotagroup
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
   set quotagroup *\\quota-test-A?.db
 } else {
   set quotagroup */quota-test-A?.db
@@ -402,7 +402,7 @@ do_test quota-4.4.7 {
 } [expr {[file size quota-test-A1.db]+[file size quota-test-A2.db]}]
 
 unset -nocomplain quotagroup
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
   set quotagroup *\\quota-test-B*
 } else {
   set quotagroup */quota-test-B*
diff --git a/test/quote.test b/test/quote.test
index 6d7b317ea1..4e40a9e57f 100644
--- a/test/quote.test
+++ b/test/quote.test
@@ -103,7 +103,7 @@ foreach {tn sql errname} {
   3 { CREATE INDEX i3 ON t1("w") }                   w
   4 { CREATE INDEX i4 ON t1(x) WHERE z="w" }         w
 } {
-  do_catchsql_test 2.1.$tn $sql [list 1 "no such column: $errname"]
+  do_catchsql_test 2.1.$tn $sql [list 1 "no such column: \"$errname\" - should this be a string literal in single-quotes?"]
 }
 
 do_execsql_test 2.2 {
@@ -147,19 +147,19 @@ ifcapable altertable {
       CREATE TABLE t1(a,b);
       CREATE INDEX x1 on t1("b");
       ALTER TABLE t1 DROP COLUMN b;
-    } {1 {error in index x1 after drop column: no such column: b}}
+    } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}}
   do_catchsql_test 3.1 {
     DROP TABLE t1;
     CREATE TABLE t1(a,"b");
     CREATE INDEX x1 on t1("b");
     ALTER TABLE t1 DROP COLUMN b;
-  } {1 {error in index x1 after drop column: no such column: b}}
+  } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}}
   do_catchsql_test 3.2 {
     DROP TABLE t1;
     CREATE TABLE t1(a,'b');
     CREATE INDEX x1 on t1("b");
     ALTER TABLE t1 DROP COLUMN b;
-  } {1 {error in index x1 after drop column: no such column: b}}
+  } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}}
   do_catchsql_test 3.3 {
     DROP TABLE t1;
     CREATE TABLE t1(a,"b");
@@ -172,7 +172,7 @@ ifcapable altertable {
     CREATE INDEX x1 ON t1("a"||"b");
     INSERT INTO t1 VALUES(1,2,3),(1,4,5);
     ALTER TABLE t1 DROP COLUMN b;
-  } {1 {error in index x1 after drop column: no such column: b}}
+  } {1 {error in index x1 after drop column: no such column: "b" - should this be a string literal in single-quotes?}}
   sqlite3_db_config db SQLITE_DBCONFIG_DQS_DDL 1
   do_catchsql_test 3.5 {
     DROP TABLE t1;
diff --git a/test/readonly.test b/test/readonly.test
new file mode 100644
index 0000000000..00392266eb
--- /dev/null
+++ b/test/readonly.test
@@ -0,0 +1,58 @@
+# 2024 March 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for using databases in read-only mode on
+# unix.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+if {$tcl_platform(platform) eq "windows"} {
+  finish_test
+  return
+}
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+set ::testprefix readonly
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+db close
+file attributes test.db -permissions r--r--r--
+
+sqlite3 db test.db
+
+do_catchsql_test 1.1 {
+  INSERT INTO t1 VALUES(7, 8);
+} {1 {attempt to write a readonly database}}
+
+do_execsql_test 1.2 {
+  BEGIN;
+    SELECT * FROM t1;
+} {1 2 3 4 5 6}
+
+# The following attempts to open a read/write fd on the database 20,000 
+# times. And each time instead opens a read-only fd. At one point this
+# was failing to reuse cached fds, causing a "too many open file-descriptors"
+# error.
+do_test 1.3 {
+  for {set ii 0} {$ii < 20000} {incr ii} {
+    sqlite3 db2 test.db
+    db2 close
+  }
+  set {} {} 
+} {}
+
+
+finish_test
diff --git a/test/recover.test b/test/recover.test
index 8d9ad013c0..ad6b7298dc 100644
--- a/test/recover.test
+++ b/test/recover.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # Test the shell tool ".ar" command.
 #
@@ -42,11 +43,13 @@ proc compare_dbs {db1 db2} {
 proc recover_with_opts {opts} {
   set cmd ".recover $opts"
   set fd [open [list |$::CLI test.db $cmd]]
-  fconfigure $fd -encoding binary
   fconfigure $fd -translation binary
   set sql [read $fd]
   close $fd
 
+  # Remove the ".dbconfig defensive off" line
+  set sql [string map {".dbconfig defensive off" ""} $sql]
+
   forcedelete test.db2
   sqlite3 db2 test.db2
   execsql $sql db2
diff --git a/test/regexp2.test b/test/regexp2.test
index 3e1da9f230..55ce59d05a 100644
--- a/test/regexp2.test
+++ b/test/regexp2.test
@@ -141,4 +141,19 @@ do_execsql_test 4.16 {SELECT '   ' REGEXP '[^a-z]{2}'} {1}
 do_execsql_test 4.17 {SELECT 'abc' REGEXP '\W{1,1}'} {0}
 do_execsql_test 4.18 {SELECT 'abc' REGEXP '\W{1}'} {0}
 
+do_execsql_test 5.0 {
+  SELECT 'abc' REGEXP 'a{1,999}bc';
+} 1
+do_catchsql_test 5.1 {
+  SELECT 'abc' REGEXP 'a{1,25000}bc';
+} {1 {REGEXP pattern too big}}
+do_execsql_test 5.2 {
+  SELECT 'abc' REGEXP 'a{999}bc';
+} 0
+do_catchsql_test 5.3 {
+  SELECT 'abc' REGEXP 'a{25000}bc';
+} {1 {REGEXP pattern too big}}
+
+
+
 finish_test
diff --git a/test/releasetest_data.tcl b/test/releasetest_data.tcl
deleted file mode 100644
index 2f4e71e8ce..0000000000
--- a/test/releasetest_data.tcl
+++ /dev/null
@@ -1,827 +0,0 @@
-# 2019 August 01
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file implements a program that produces scripts (either shell scripts
-# or batch files) to implement a particular test that is part of the SQLite
-# release testing procedure. For example, to run veryquick.test with a 
-# specified set of -D compiler switches.
-#
-# A "configuration" is a set of options passed to [./configure] and [make]
-# to build the SQLite library in a particular fashion. A "platform" is a
-# list of tests; most platforms are named after the hardware/OS platform
-# that the tests will be run on as part of the release procedure. Each 
-# "test" is a combination of a configuration and a makefile target (e.g.
-# "fulltest"). The program may be invoked as follows:
-#
-set USAGE {
-$argv0 script ?-msvc? CONFIGURATION TARGET
-    Given a configuration and make target, return a bash (or, if -msvc
-    is specified, batch) script to execute the test. The first argument
-    passed to the script must be a directory containing SQLite source code.
-
-$argv0 configurations
-    List available configurations.
-
-$argv0 platforms
-    List available platforms.
-
-$argv0 tests ?-nodebug? PLATFORM
-    List tests in a specified platform. If the -nodebug switch is 
-    specified, synthetic debug/ndebug configurations are omitted. Each
-    test is a combination of a configuration and a makefile target.
-}
-
-# Omit comments (text between # and \n) in a long multi-line string.
-#
-proc strip_comments {in} {
-  regsub -all {#[^\n]*\n} $in {} out
-  return $out
-}
-
-array set ::Configs [strip_comments {
-  "Default" {
-    -O2
-    --disable-amalgamation --disable-shared
-    --enable-session
-    -DSQLITE_ENABLE_RBU
-  }
-  "Sanitize" {
-    CC=clang -fsanitize=address,undefined
-    -DSQLITE_ENABLE_STAT4
-    -DCONFIG_SLOWDOWN_FACTOR=5.0
-    --enable-debug
-    --enable-all
-  }
-  "Stdcall" {
-    -DUSE_STDCALL=1
-    -O2
-  }
-  "Have-Not" {
-    # The "Have-Not" configuration sets all possible -UHAVE_feature options
-    # in order to verify that the code works even on platforms that lack
-    # these support services.
-    -DHAVE_FDATASYNC=0
-    -DHAVE_GMTIME_R=0
-    -DHAVE_ISNAN=0
-    -DHAVE_LOCALTIME_R=0
-    -DHAVE_LOCALTIME_S=0
-    -DHAVE_MALLOC_USABLE_SIZE=0
-    -DHAVE_STRCHRNUL=0
-    -DHAVE_USLEEP=0
-    -DHAVE_UTIME=0
-  }
-  "Unlock-Notify" {
-    -O2
-    -DSQLITE_ENABLE_UNLOCK_NOTIFY
-    -DSQLITE_THREADSAFE
-    -DSQLITE_TCL_DEFAULT_FULLMUTEX=1
-  }
-  "User-Auth" {
-    -O2
-    -DSQLITE_USER_AUTHENTICATION=1
-  }
-  "Secure-Delete" {
-    -O2
-    -DSQLITE_SECURE_DELETE=1
-    -DSQLITE_SOUNDEX=1
-  }
-  "Update-Delete-Limit" {
-    -O2
-    -DSQLITE_DEFAULT_FILE_FORMAT=4
-    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
-    -DSQLITE_ENABLE_STMT_SCANSTATUS
-    -DSQLITE_LIKE_DOESNT_MATCH_BLOBS
-    -DSQLITE_ENABLE_CURSOR_HINTS
-  }
-  "Check-Symbols" {
-    -DSQLITE_MEMDEBUG=1
-    -DSQLITE_ENABLE_FTS3_PARENTHESIS=1
-    -DSQLITE_ENABLE_FTS3=1
-    -DSQLITE_ENABLE_RTREE=1
-    -DSQLITE_ENABLE_MEMSYS5=1
-    -DSQLITE_ENABLE_MEMSYS3=1
-    -DSQLITE_ENABLE_COLUMN_METADATA=1
-    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
-    -DSQLITE_SECURE_DELETE=1
-    -DSQLITE_SOUNDEX=1
-    -DSQLITE_ENABLE_ATOMIC_WRITE=1
-    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
-    -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
-    -DSQLITE_ENABLE_STAT4
-    -DSQLITE_ENABLE_STMT_SCANSTATUS
-    --enable-fts5 --enable-session
-  }
-  "Debug-One" {
-    --disable-shared
-    -O2 -funsigned-char
-    -DSQLITE_DEBUG=1
-    -DSQLITE_MEMDEBUG=1
-    -DSQLITE_MUTEX_NOOP=1
-    -DSQLITE_TCL_DEFAULT_FULLMUTEX=1
-    -DSQLITE_ENABLE_FTS3=1
-    -DSQLITE_ENABLE_RTREE=1
-    -DSQLITE_ENABLE_MEMSYS5=1
-    -DSQLITE_ENABLE_COLUMN_METADATA=1
-    -DSQLITE_ENABLE_STAT4
-    -DSQLITE_ENABLE_HIDDEN_COLUMNS
-    -DSQLITE_MAX_ATTACHED=125
-    -DSQLITE_MUTATION_TEST
-    --enable-fts5
-  }
-  "Debug-Two" {
-    -DSQLITE_DEFAULT_MEMSTATUS=0
-    -DSQLITE_MAX_EXPR_DEPTH=0
-    --enable-debug
-  }
-  "Fast-One" {
-    -O6
-    -DSQLITE_ENABLE_FTS4=1
-    -DSQLITE_ENABLE_RTREE=1
-    -DSQLITE_ENABLE_STAT4
-    -DSQLITE_ENABLE_RBU
-    -DSQLITE_MAX_ATTACHED=125
-    -DSQLITE_MAX_MMAP_SIZE=12884901888
-    -DSQLITE_ENABLE_SORTER_MMAP=1
-    -DLONGDOUBLE_TYPE=double
-    --enable-session
-  }
-  "Device-One" {
-    -O2
-    -DSQLITE_DEBUG=1
-    -DSQLITE_DEFAULT_AUTOVACUUM=1
-    -DSQLITE_DEFAULT_CACHE_SIZE=64
-    -DSQLITE_DEFAULT_PAGE_SIZE=1024
-    -DSQLITE_DEFAULT_TEMP_CACHE_SIZE=32
-    -DSQLITE_DISABLE_LFS=1
-    -DSQLITE_ENABLE_ATOMIC_WRITE=1
-    -DSQLITE_ENABLE_IOTRACE=1
-    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
-    -DSQLITE_MAX_PAGE_SIZE=4096
-    -DSQLITE_OMIT_LOAD_EXTENSION=1
-    -DSQLITE_OMIT_PROGRESS_CALLBACK=1
-    -DSQLITE_OMIT_VIRTUALTABLE=1
-    -DSQLITE_ENABLE_HIDDEN_COLUMNS
-    -DSQLITE_TEMP_STORE=3
-  }
-  "Device-Two" {
-    -DSQLITE_4_BYTE_ALIGNED_MALLOC=1
-    -DSQLITE_DEFAULT_AUTOVACUUM=1
-    -DSQLITE_DEFAULT_CACHE_SIZE=1000
-    -DSQLITE_DEFAULT_LOCKING_MODE=0
-    -DSQLITE_DEFAULT_PAGE_SIZE=1024
-    -DSQLITE_DEFAULT_TEMP_CACHE_SIZE=1000
-    -DSQLITE_DISABLE_LFS=1
-    -DSQLITE_ENABLE_FTS3=1
-    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
-    -DSQLITE_ENABLE_RTREE=1
-    -DSQLITE_MAX_COMPOUND_SELECT=50
-    -DSQLITE_MAX_PAGE_SIZE=32768
-    -DSQLITE_OMIT_TRACE=1
-    -DSQLITE_TEMP_STORE=3
-    -DSQLITE_THREADSAFE=2
-    --enable-fts5 --enable-session
-  }
-  "Locking-Style" {
-    -O2
-    -DSQLITE_ENABLE_LOCKING_STYLE=1
-  }
-  "Apple" {
-    -Os
-    -DHAVE_GMTIME_R=1
-    -DHAVE_ISNAN=1
-    -DHAVE_LOCALTIME_R=1
-    -DHAVE_PREAD=1
-    -DHAVE_PWRITE=1
-    -DHAVE_UTIME=1
-    -DSQLITE_DEFAULT_CACHE_SIZE=1000
-    -DSQLITE_DEFAULT_CKPTFULLFSYNC=1
-    -DSQLITE_DEFAULT_MEMSTATUS=1
-    -DSQLITE_DEFAULT_PAGE_SIZE=1024
-    -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS=1
-    -DSQLITE_ENABLE_API_ARMOR=1
-    -DSQLITE_ENABLE_AUTO_PROFILE=1
-    -DSQLITE_ENABLE_FLOCKTIMEOUT=1
-    -DSQLITE_ENABLE_FTS3=1
-    -DSQLITE_ENABLE_FTS3_PARENTHESIS=1
-    -DSQLITE_ENABLE_FTS3_TOKENIZER=1
-    -DSQLITE_ENABLE_PERSIST_WAL=1
-    -DSQLITE_ENABLE_PURGEABLE_PCACHE=1
-    -DSQLITE_ENABLE_RTREE=1
-    -DSQLITE_ENABLE_SNAPSHOT=1
-    # -DSQLITE_ENABLE_SQLLOG=1
-    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
-    -DSQLITE_MAX_LENGTH=2147483645
-    -DSQLITE_MAX_VARIABLE_NUMBER=500000
-    # -DSQLITE_MEMDEBUG=1
-    -DSQLITE_NO_SYNC=1
-    -DSQLITE_OMIT_AUTORESET=1
-    -DSQLITE_OMIT_LOAD_EXTENSION=1
-    -DSQLITE_PREFER_PROXY_LOCKING=1
-    -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
-    -DSQLITE_THREADSAFE=2
-    -DSQLITE_USE_URI=1
-    -DSQLITE_WRITE_WALFRAME_PREBUFFERED=1
-    -DUSE_GUARDED_FD=1
-    -DUSE_PREAD=1
-    --enable-fts5
-  }
-  "Extra-Robustness" {
-    -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
-    -DSQLITE_MAX_ATTACHED=62
-  }
-  "Devkit" {
-    -DSQLITE_DEFAULT_FILE_FORMAT=4
-    -DSQLITE_MAX_ATTACHED=30
-    -DSQLITE_ENABLE_COLUMN_METADATA
-    -DSQLITE_ENABLE_FTS4
-    -DSQLITE_ENABLE_FTS5
-    -DSQLITE_ENABLE_FTS4_PARENTHESIS
-    -DSQLITE_DISABLE_FTS4_DEFERRED
-    -DSQLITE_ENABLE_RTREE
-    --enable-fts5
-  }
-  "No-lookaside" {
-    -DSQLITE_TEST_REALLOC_STRESS=1
-    -DSQLITE_OMIT_LOOKASIDE=1
-  }
-  "Valgrind" {
-    -DSQLITE_ENABLE_STAT4
-    -DSQLITE_ENABLE_FTS4
-    -DSQLITE_ENABLE_RTREE
-    -DSQLITE_ENABLE_HIDDEN_COLUMNS
-    -DCONFIG_SLOWDOWN_FACTOR=8.0
-  }
-
-  "Windows-Memdebug" {
-    MEMDEBUG=1
-    DEBUG=3
-  }
-  "Windows-Win32Heap" {
-    WIN32HEAP=1
-    DEBUG=4
-  }
-
-  # The next group of configurations are used only by the
-  # Failure-Detection platform.  They are all the same, but we need
-  # different names for them all so that they results appear in separate
-  # subdirectories.
-  #
-  Fail0     {-O0}
-  Fail2     {-O0}
-  Fail3     {-O0}
-  Fail4     {-O0}
-  FuzzFail1 {-O0}
-  FuzzFail2 {-O0}
-}]
-if {$tcl_platform(os)=="Darwin"} {
-  lappend Configs(Apple) -DSQLITE_ENABLE_LOCKING_STYLE=1
-}
-
-array set ::Platforms [strip_comments {
-  Linux-x86_64 {
-    "Check-Symbols*"          "" checksymbols
-    "Fast-One"                QUICKTEST_INCLUDE=rbu.test "fuzztest test"
-    "Debug-One"               "" "mptest test"
-    "Debug-Two"               "" test
-    "Have-Not"                "" test
-    "Secure-Delete"           "" test
-    "Unlock-Notify"           QUICKTEST_INCLUDE=notify2.test test
-    "User-Auth"               "" tcltest
-    "Update-Delete-Limit"     "" test
-    "Extra-Robustness"        "" test
-    "Device-Two"              "" "threadtest test"
-    "No-lookaside"            "" test
-    "Devkit"                  "" test
-    "Apple"                   "" test
-    "Sanitize*"               "" test
-    "Device-One"              "" alltest
-    "Default"                 "" "threadtest fuzztest alltest"
-    "Valgrind*"               "" valgrindtest
-  }
-  Linux-i686 {
-    "Devkit"                  "" test
-    "Have-Not"                "" test
-    "Unlock-Notify"           QUICKTEST_INCLUDE=notify2.test test
-    "Device-One"              "" test
-    "Device-Two"              "" test
-    "Default"                 "" "threadtest fuzztest alltest"
-  }
-  Darwin-i386 {
-    "Locking-Style"           "" "mptest test"
-    "Have-Not"                "" test
-    "Apple"                   "" "threadtest fuzztest alltest"
-  }
-  Darwin-x86_64 {
-    "Locking-Style"           "" "mptest test"
-    "Have-Not"                "" test
-    "Apple"                   "" "threadtest fuzztest alltest"
-  }
-  Darwin-arm64 {
-    "Locking-Style"           "" "mptest test"
-    "Have-Not"                "" test
-    "Apple"                   "" "threadtest fuzztest alltest"
-  }
-  "Windows NT-intel" {
-    "Stdcall"                 "" test
-    "Have-Not"                "" test
-    "Windows-Memdebug*"       "" test
-    "Windows-Win32Heap*"      "" test
-    "Default"                 "" "mptest fulltestonly"
-  }
-  "Windows NT-amd64" {
-    "Stdcall"                 "" test
-    "Have-Not"                "" test
-    "Windows-Memdebug*"       "" test
-    "Windows-Win32Heap*"      "" test
-    "Default"                 "" "mptest fulltestonly"
-  }
-
-  # The Failure-Detection platform runs various tests that deliberately
-  # fail.  This is used as a test of this script to verify that this script
-  # correctly identifies failures.
-  #
-  Failure-Detection {
-    Fail0*     "TEST_FAILURE=0" test
-    Sanitize*  "TEST_FAILURE=1" test
-    Fail2*     "TEST_FAILURE=2" valgrindtest
-    Fail3*     "TEST_FAILURE=3" valgrindtest
-    Fail4*     "TEST_FAILURE=4" test
-    FuzzFail1* "TEST_FAILURE=5" test
-    FuzzFail2* "TEST_FAILURE=5" valgrindtest
-  }
-}]
-
-#--------------------------------------------------------------------------
-#--------------------------------------------------------------------------
-#--------------------------------------------------------------------------
-# End of configuration section.
-#--------------------------------------------------------------------------
-#--------------------------------------------------------------------------
-#--------------------------------------------------------------------------
-
-# Configuration verification: Check that each entry in the list of configs
-# specified for each platforms exists.
-#
-foreach {key value} [array get ::Platforms] {
-  foreach {v vars t} $value {
-    if {[string range $v end end]=="*"} {
-      set v [string range $v 0 end-1]
-    }
-    if {0==[info exists ::Configs($v)]} {
-      puts stderr "No such configuration: \"$v\""
-      exit -1
-    }
-  }
-}
-
-proc usage {} {
-  global argv0
-  puts stderr [subst $::USAGE]
-  exit 1
-}
-
-proc is_prefix {p str min} {
-  set n [string length $p]
-  if {$n<$min} { return 0 }
-  if {[string range $str 0 [expr $n-1]]!=$p} { return 0 }
-  return 1
-}
-
-proc main_configurations {} {
-  foreach k [lsort [array names ::Configs]] {
-    puts $k
-  }
-}
-
-proc main_platforms {} {
-  foreach k [lsort [array names ::Platforms]] {
-    puts "\"$k\""
-  }
-}
-
-proc main_script {args} {
-  set bMsvc 0
-  set nArg [llength $args]
-  if {$nArg==3} {
-    if {![is_prefix [lindex $args 0] -msvc 2]} usage
-    set bMsvc 1
-  } elseif {$nArg<2 || $nArg>3} {
-    usage
-  }
-  set config [lindex $args end-1]
-  set target [lindex $args end]
-
-  set opts       [list]                         ;# OPTS value
-  set cflags     [expr {$bMsvc ? "-Zi" : "-g"}] ;# CFLAGS value
-  set makeOpts   [list]                         ;# Extra args for [make]
-  set configOpts [list]                         ;# Extra args for [configure]
-
-  if {$::tcl_platform(platform)=="windows" || $bMsvc} {
-    lappend opts -DSQLITE_OS_WIN=1
-  } else {
-    lappend opts -DSQLITE_OS_UNIX=1
-  }
-
-  # Figure out if this is a synthetic ndebug or debug configuration.
-  #
-  set bRemoveDebug 0
-  if {[string match *-ndebug $config]} {
-    set bRemoveDebug 1
-    set config [string range $config 0 end-7]
-  }
-  if {[string match *-debug $config]} {
-    lappend opts -DSQLITE_DEBUG
-    lappend opts -DSQLITE_EXTRA_IFNULLROW
-    set config [string range $config 0 end-6]
-  }
-  regexp {^(.*)-[0-9]+} $config -> config
-
-  # Ensure that the named configuration exists.
-  #
-  if {![info exists ::Configs($config)]} {
-    puts stderr "No such config: $config"
-    exit 1
-  }
-
-  # Loop through the parameters of the nominated configuration, updating
-  # $opts, $cflags, $makeOpts and $configOpts along the way. Rules are as
-  # follows:
-  #
-  #   1. If the parameter begins with a "*", discard it.
-  #
-  #   2. If $bRemoveDebug is set and the parameter is -DSQLITE_DEBUG or
-  #      -DSQLITE_DEBUG=1, discard it
-  #
-  #   3. If the parameter begins with "-D", add it to $opts.
-  #
-  #   4. If the parameter begins with "--" add it to $configOpts. Unless
-  #      this command is preparing a script for MSVC - then add an 
-  #      equivalent to $makeOpts or $opts.
-  #
-  #   5. If the parameter begins with "-" add it to $cflags. If in MSVC
-  #      mode and the parameter is an -O<integer> option, instead add
-  #      an OPTIMIZATIONS=<integer> switch to $makeOpts.
-  #
-  #   6. If none of the above apply, add the parameter to $makeOpts
-  #
-  foreach param $::Configs($config) {
-    if {[string range $param 0 0]=="*"} continue
-
-    if {$bRemoveDebug} {
-      if {$param=="-DSQLITE_DEBUG" || $param=="-DSQLITE_DEBUG=1"
-       || $param=="-DSQLITE_MEMDEBUG" || $param=="-DSQLITE_MEMDEBUG=1"
-       || $param=="--enable-debug"
-      } {
-        continue
-      }
-    }
-
-    if {[string range $param 0 1]=="-D"} {
-      lappend opts $param
-      continue
-    }
-
-    if {[string range $param 0 1]=="--"} {
-      if {$bMsvc} {
-        switch -- $param {
-          --disable-amalgamation {
-            lappend makeOpts USE_AMALGAMATION=0
-          }
-          --disable-shared {
-            lappend makeOpts USE_CRT_DLL=0 DYNAMIC_SHELL=0
-          }
-          --enable-fts5 {
-            lappend opts -DSQLITE_ENABLE_FTS5
-          } 
-          --enable-shared {
-            lappend makeOpts USE_CRT_DLL=1 DYNAMIC_SHELL=1
-          }
-          --enable-session {
-            lappend opts -DSQLITE_ENABLE_PREUPDATE_HOOK
-            lappend opts -DSQLITE_ENABLE_SESSION
-          }
-          default {
-            error "Cannot translate $param for MSVC"
-          }
-        }
-      } else {
-        lappend configOpts $param
-      }
-
-      continue
-    }
-
-    if {[string range $param 0 0]=="-"} {
-      if {$bMsvc && [regexp -- {^-O(\d+)$} $param -> level]} {
-        lappend makeOpts OPTIMIZATIONS=$level
-      } else {
-        lappend cflags $param
-      }
-      continue
-    }
-
-    lappend makeOpts $param
-  }
-
-  # Some configurations specify -DHAVE_USLEEP=0. For all others, add
-  # -DHAVE_USLEEP=1.
-  #
-  if {[lsearch $opts "-DHAVE_USLEEP=0"]<0} {
-    lappend opts -DHAVE_USLEEP=1
-  }
-
-  if {$bMsvc==0} {
-    puts {set -e}
-    puts {}
-    puts {if [ "$#" -ne 1 ] ; then}
-    puts {  echo "Usage: $0 <sqlite-src-dir>" }
-    puts {  exit -1 }
-    puts {fi }
-    puts {SRCDIR=$1}
-    puts {}
-    puts "TCL=\"[::tcl::pkgconfig get libdir,install]\""
-
-    puts "\$SRCDIR/configure --with-tcl=\$TCL $configOpts"
-    puts {}
-    puts {OPTS="      -DSQLITE_NO_SYNC=1"}
-    foreach o $opts { 
-      puts "OPTS=\"\$OPTS $o\"" 
-    }
-    puts {}
-    puts "CFLAGS=\"$cflags\""
-    puts {}
-    puts "make $target \"CFLAGS=\$CFLAGS\" \"OPTS=\$OPTS\" $makeOpts"
-  } else {
-
-    puts {set SRCDIR=%1}
-    set makecmd    "nmake /f %SRCDIR%\\Makefile.msc TOP=%SRCDIR% $target "
-    append makecmd "\"CFLAGS=$cflags\" \"OPTS=$opts\" $makeOpts"
-
-    puts "set TMP=%CD%"
-    puts $makecmd
-  }
-}
-
-proc main_trscript {args} {
-  set bMsvc 0
-  set nArg [llength $args]
-  if {$nArg==3} {
-    if {![is_prefix [lindex $args 0] -msvc 2]} usage
-    set bMsvc 1
-  } elseif {$nArg<2 || $nArg>3} {
-    usage
-  }
-  set config [lindex $args end-1]
-  set srcdir [lindex $args end]
-
-  set opts       [list]                         ;# OPTS value
-  set cflags     [expr {$bMsvc ? "-Zi" : "-g"}] ;# CFLAGS value
-  set makeOpts   [list]                         ;# Extra args for [make]
-  set configOpts [list]                         ;# Extra args for [configure]
-
-  if {$::tcl_platform(platform)=="windows" || $bMsvc} {
-    lappend opts -DSQLITE_OS_WIN=1
-  } else {
-    lappend opts -DSQLITE_OS_UNIX=1
-  }
-
-  # Figure out if this is a synthetic ndebug or debug configuration.
-  #
-  set bRemoveDebug 0
-  if {[string match *-ndebug $config]} {
-    set bRemoveDebug 1
-    set config [string range $config 0 end-7]
-  }
-  if {[string match *-debug $config]} {
-    lappend opts -DSQLITE_DEBUG
-    lappend opts -DSQLITE_EXTRA_IFNULLROW
-    set config [string range $config 0 end-6]
-  }
-  regexp {^(.*)-[0-9]+} $config -> config
-
-  # Ensure that the named configuration exists.
-  #
-  if {![info exists ::Configs($config)]} {
-    puts stderr "No such config: $config"
-    exit 1
-  }
-
-  # Loop through the parameters of the nominated configuration, updating
-  # $opts, $cflags, $makeOpts and $configOpts along the way. Rules are as
-  # follows:
-  #
-  #   1. If the parameter begins with a "*", discard it.
-  #
-  #   2. If $bRemoveDebug is set and the parameter is -DSQLITE_DEBUG or
-  #      -DSQLITE_DEBUG=1, discard it
-  #
-  #   3. If the parameter begins with "-D", add it to $opts.
-  #
-  #   4. If the parameter begins with "--" add it to $configOpts. Unless
-  #      this command is preparing a script for MSVC - then add an 
-  #      equivalent to $makeOpts or $opts.
-  #
-  #   5. If the parameter begins with "-" add it to $cflags. If in MSVC
-  #      mode and the parameter is an -O<integer> option, instead add
-  #      an OPTIMIZATIONS=<integer> switch to $makeOpts.
-  #
-  #   6. If none of the above apply, add the parameter to $makeOpts
-  #
-  foreach param $::Configs($config) {
-    if {[string range $param 0 0]=="*"} continue
-
-    if {$bRemoveDebug} {
-      if {$param=="-DSQLITE_DEBUG" || $param=="-DSQLITE_DEBUG=1"
-       || $param=="-DSQLITE_MEMDEBUG" || $param=="-DSQLITE_MEMDEBUG=1"
-       || $param=="--enable-debug"
-      } {
-        continue
-      }
-    }
-
-    if {[string range $param 0 1]=="-D"} {
-      lappend opts $param
-      continue
-    }
-
-    if {[string range $param 0 1]=="--"} {
-      if {$bMsvc} {
-        switch -- $param {
-          --disable-amalgamation {
-            lappend makeOpts USE_AMALGAMATION=0
-          }
-          --disable-shared {
-            lappend makeOpts USE_CRT_DLL=0 DYNAMIC_SHELL=0
-          }
-          --enable-fts5 {
-            lappend opts -DSQLITE_ENABLE_FTS5
-          } 
-          --enable-shared {
-            lappend makeOpts USE_CRT_DLL=1 DYNAMIC_SHELL=1
-          }
-          --enable-session {
-            lappend opts -DSQLITE_ENABLE_PREUPDATE_HOOK
-            lappend opts -DSQLITE_ENABLE_SESSION
-          }
-          default {
-            error "Cannot translate $param for MSVC"
-          }
-        }
-      } else {
-        lappend configOpts $param
-      }
-
-      continue
-    }
-
-    if {[string range $param 0 0]=="-"} {
-      if {$bMsvc && [regexp -- {^-O(\d+)$} $param -> level]} {
-        lappend makeOpts OPTIMIZATIONS=$level
-      } else {
-        lappend cflags $param
-      }
-      continue
-    }
-
-    lappend makeOpts $param
-  }
-
-  # Some configurations specify -DHAVE_USLEEP=0. For all others, add
-  # -DHAVE_USLEEP=1.
-  #
-  if {[lsearch $opts "-DHAVE_USLEEP=0"]<0} {
-    lappend opts -DHAVE_USLEEP=1
-  }
-
-  if {$bMsvc==0} {
-    puts {set -e}
-    puts {}
-    puts {if [ "$#" -ne 1 ] ; then}
-    puts {  echo "Usage: $0 <target>" }
-    puts {  exit -1 }
-    puts {fi }
-    puts "SRCDIR=\"$srcdir\""
-    puts {}
-    puts "TCL=\"[::tcl::pkgconfig get libdir,install]\""
-
-    puts {if [ ! -f Makefile ] ; then}
-    puts "  \$SRCDIR/configure --with-tcl=\$TCL $configOpts"
-    puts {fi}
-    puts {}
-    puts {OPTS="      -DSQLITE_NO_SYNC=1"}
-    foreach o $opts { 
-      puts "OPTS=\"\$OPTS $o\"" 
-    }
-    puts {}
-    puts "CFLAGS=\"$cflags\""
-    puts {}
-    puts "make \$1 \"CFLAGS=\$CFLAGS\" \"OPTS=\$OPTS\" $makeOpts"
-  } else {
-
-    set srcdir [file nativename [file normalize $srcdir]]
-    # set srcdir [string map [list "\\" "\\\\"] $srcdir]
-
-    puts {set TARGET=%1}
-    set makecmd    "nmake /f $srcdir\\Makefile.msc TOP=\"$srcdir\" %TARGET% "
-    append makecmd "\"CFLAGS=$cflags\" \"OPTS=$opts\" $makeOpts"
-
-    puts "set TMP=%CD%"
-    puts $makecmd
-  }
-}
-
-proc main_tests {args} {
-  set bNodebug 0
-  set nArg [llength $args]
-  if {$nArg==2} {
-    if {[is_prefix [lindex $args 0] -nodebug 2]} {
-      set bNodebug 1
-    } elseif {[is_prefix [lindex $args 0] -debug 2]} {
-      set bNodebug 0
-    } else usage
-  } elseif {$nArg==0 || $nArg>2} {
-    usage
-  }
-  set p [lindex $args end]
-  if {![info exists ::Platforms($p)]} {
-    puts stderr "No such platform: $p"
-    exit 1
-  }
-
-  set lTest [list]
-
-  foreach {config vars target} $::Platforms($p) {
-    if {[string range $config end end]=="*"} {
-      set config [string range $config 0 end-1]
-    } elseif {$bNodebug==0} {
-      set dtarget test
-      if {[lsearch $target fuzztest]<0 && [lsearch $target test]<0} {
-        set dtarget tcltest
-      }
-      if {$vars!=""} { set dtarget "$vars $dtarget" }
-
-      if {[string first SQLITE_DEBUG $::Configs($config)]>=0
-       || [string first --enable-debug $::Configs($config)]>=0
-      } {
-        lappend lTest "$config-ndebug \"$dtarget\""
-      } else {
-        lappend lTest "$config-debug \"$dtarget\""
-      }
-    }
-
-    if {[llength $target]==1 && ([string match "*TEST_FAILURE*" $vars] || (
-        [lsearch $target "valgrindtest"]<0
-     && [lsearch $target "alltest"]<0
-     && [lsearch $target "fulltestonly"]<0
-     && ![string match Sanitize* $config]
-    ))} {
-      if {$vars!=""} { set target "$vars $target" }
-      lappend lTest "$config \"$target\""
-    } else {
-      set idir -1
-      foreach t $target {
-        if {$t=="valgrindtest" || $t=="alltest" || $t=="fulltestonly"
-         || [string match Sanitize* $config]
-        } {
-          if {$vars!=""} { set t "$vars $t" }
-          for {set ii 1} {$ii<=4} {incr ii} {
-            lappend lTest "$config-[incr idir] \"TCLTEST_PART=$ii/4 $t\""
-          }
-        } else {
-          if {$vars!=""} { set t "$vars $t" }
-          lappend lTest "$config-[incr idir] \"$t\""
-        }
-      }
-    }
-  }
-
-  foreach l $lTest {
-    puts $l
-  }
-
-}
-
-if {[llength $argv]==0} { usage }
-set cmd [lindex $argv 0]
-set n [expr [llength $argv]-1]
-if {[string match ${cmd}* configurations] && $n==0} {
-  main_configurations 
-} elseif {[string match ${cmd}* script]} {
-  main_script {*}[lrange $argv 1 end]
-} elseif {[string match ${cmd}* trscript]} {
-  main_trscript {*}[lrange $argv 1 end]
-} elseif {[string match ${cmd}* platforms] && $n==0} {
-  main_platforms
-} elseif {[string match ${cmd}* tests]} {
-  main_tests {*}[lrange $argv 1 end]
-} else {
-  usage
-}
diff --git a/test/reservebytes.test b/test/reservebytes.test
new file mode 100644
index 0000000000..51b76c7232
--- /dev/null
+++ b/test/reservebytes.test
@@ -0,0 +1,51 @@
+# 2025 August 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix reservebytes
+
+
+
+reset_db
+file_control_reservebytes db 0
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  CREATE INDEX i1 ON t1(b, c);
+  WITH s(i) AS (
+    VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t1 SELECT NULL, i, hex(randomblob(500)) FROM s;
+}
+
+sqlite3 db2 test.db
+if {[permutation]=="prepare"} { db2 cache size 0 }
+
+do_execsql_test -db db2 1.1 { PRAGMA integrity_check } {ok}
+
+file_control_reservebytes db 8
+do_test 1.2.1 { hexio_read test.db 20 1 } {00}
+do_execsql_test -db db2 1.2.2 { PRAGMA integrity_check } {ok}
+
+do_execsql_test 1.3.2 { VACUUM }
+do_execsql_test -db db2 1.3.4 { PRAGMA integrity_check } {ok}
+do_test 1.3.5 { hexio_read test.db 20 1 } {08}
+
+file_control_reservebytes db 16
+do_test 1.4.1 { hexio_read test.db 20 1 } {08}
+do_execsql_test 1.4.2 { VACUUM }
+do_execsql_test -db db2 1.4.3 { PRAGMA integrity_check } {ok}
+do_test 1.4.4 { hexio_read test.db 20 1 } {10}
+
+finish_test
diff --git a/test/returning1.test b/test/returning1.test
index 6c098dc256..9ab646a3b7 100644
--- a/test/returning1.test
+++ b/test/returning1.test
@@ -212,17 +212,38 @@ do_execsql_test 10.2 {
   END;
 }
 
-do_catchsql_test 10.3a {
-  INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid;
-} {1 {no such column: new.rowid}}
-
-do_catchsql_test 10.3b {
-  UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid;
-} {1 {no such column: new.rowid}}
-
-do_execsql_test 10.4 {
-  SELECT * FROM log;
-} {}
+ifcapable !allow_rowid_in_view {
+  do_catchsql_test 10.3a {
+    INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid;
+  } {1 {no such column: new.rowid}}
+  
+  do_catchsql_test 10.3b {
+    UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid;
+  } {1 {no such column: new.rowid}}
+  
+  do_execsql_test 10.4 {
+    SELECT * FROM log;
+  } {}
+} else {
+  # Note: The values returned by the RETURNING clauses of the following
+  # two statements are the rowid columns of views. These values are not
+  # well defined, so the INSERT returns -1, and the UPDATE returns NULL.
+  # These match the values used for new.rowid expressions, but not much 
+  # else.
+  do_catchsql_test 10.3a {
+    INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid;
+  } {0 -1}
+  
+  do_catchsql_test 10.3b {
+    UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid;
+  } {0 {{} {} {}}}
+  
+  do_execsql_test 10.4 {
+    SELECT * FROM log;
+  } {
+    insert -1 1234 5678 update {} z y update {} z y update {} z y
+  }
+}
 
 # 2021-04-27 dbsqlfuzz 78b9400770ef8cc7d9427dfba26f4fcf46ea7dc2
 # Returning clauses on TEMP tables with triggers.
@@ -439,4 +460,138 @@ do_catchsql_test 19.1 {
   END;
 } {0 {}}
 
+# 2024-04-24
+# https://sqlite.org/forum/forumpost/2c83569ce8945d39
+#
+# If the RETURNING clause includes subqueries that reference the
+# table being modified, make sure that the subqueries are identified
+# as correlated so that the results are recomputed after each step
+# instead of being computed once and reused.
+#
+reset_db
+db null N
+do_execsql_test 20.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT);
+  INSERT INTO t1 VALUES(1,10),(2,20),(3,30),(4,40),(6,60),(8,80);
+  BEGIN;
+  DELETE FROM t1 WHERE a<>3
+    RETURNING a,
+              (SELECT min(a) FROM t1),
+              (SELECT max(a) FROM t1),
+              (SELECT round(avg(a),2) FROM t1);
+  ROLLBACK;
+} {
+  1 2 8 4.6
+  2 3 8 5.25
+  4 3 8 5.67
+  6 3 8 5.5
+  8 3 3 3.0
+}
+do_execsql_test 20.2 {
+  BEGIN;
+  DELETE FROM t1
+    RETURNING a,
+              (SELECT min(a) FROM t1),
+              (SELECT max(a) FROM t1),
+              (SELECT round(avg(a),2) FROM t1);
+  ROLLBACK;
+} {
+  1 2 8 4.6
+  2 3 8 5.25
+  3 4 8 6.0
+  4 6 8 7.0
+  6 8 8 8.0
+  8 N N N
+}
+do_execsql_test 20.3 {
+  BEGIN;
+  DELETE FROM t1
+    RETURNING a,
+              (SELECT min(t2.a)+t1.a*100 FROM t1 AS t2),
+              (SELECT max(t2.a)+t1.a*100 FROM t1 AS t2),
+              (SELECT round(avg(t2.a),2)+t1.a*100 FROM t1 AS t2);
+  ROLLBACK;
+} {
+  1 102 108 104.6
+  2 203 208 205.25
+  3 304 308 306.0
+  4 406 408 407.0
+  6 608 608 608.0
+  8 N N N
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 21.0 {
+  PRAGMA writable_schema=ON;
+  INSERT INTO sqlite_schema DEFAULT VALUES RETURNING sqlite_schema.name;
+} {{}}
+
+do_execsql_test 21.1 {
+  INSERT INTO sqlite_temp_schema DEFAULT VALUES RETURNING sqlite_temp_schema.name;
+} {{}}
+
+#-------------------------------------------------------------------------
+reset_db 
+do_execsql_test 22.0 {
+  PRAGMA writable_schema=ON;
+  CREATE TABLE xyz (a);
+}
+do_catchsql_test 22.1 {
+  INSERT INTO sqlite_temp_schema DEFAULT VALUES 
+    RETURNING
+    (SELECT * FROM xyz AS sqlite_master WHERE a=sqlite_master.name);
+} {1 {no such column: sqlite_master.name}}
+
+
+#-------------------------------------------------------------------------
+reset_db 
+do_execsql_test 23.0 {
+  PRAGMA recursive_triggers = 1;
+  CREATE TABLE t1(x, y);
+  CREATE TRIGGER t1insert AFTER INSERT ON t1 WHEN new.x<5 BEGIN
+    INSERT INTO t1 VALUES(new.x+1, new.y);
+  END;
+}
+
+do_execsql_test 23.1 {
+  INSERT INTO t1 VALUES(1, 'one') RETURNING *;
+} {1 one}
+
+do_execsql_test 23.2 {
+  SELECT * FROM t1
+} {1 one 2 one 3 one 4 one 5 one}
+
+#-------------------------------------------------------------------------
+reset_db 
+ifcapable fts5 {
+
+  do_execsql_test 24.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(c);
+    CREATE TABLE t1(x);
+    INSERT INTO t1 VALUES('x');
+  }
+  
+  db close
+  
+  sqlite3 db test.db
+  sqlite3 db2 test.db
+  
+  do_execsql_test 24.1 {
+    SELECT * FROM t1
+  } {x}
+  
+  do_execsql_test -db db2 24.2 {
+    CREATE TABLE t2(y);
+    INSERT INTO t2 VALUES('y');
+  } {}
+
+  db2 close
+  
+  do_execsql_test 24.3 {
+    INSERT INTO ft VALUES('hello world') RETURNING *
+  } {{hello world}}
+}
+
+
 finish_test
diff --git a/test/returningfault.test b/test/returningfault.test
index 8bf6fbfe06..b0177e6dc8 100644
--- a/test/returningfault.test
+++ b/test/returningfault.test
@@ -14,13 +14,15 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 
+set ::testprefix returningfault
+
 
 do_execsql_test 1.0 {
   CREATE TABLE t1 (b);
 } {}
 faultsim_save_and_close
 
-do_faultsim_test pagerfault-1 -faults oom-t* -prep {
+do_faultsim_test 1 -faults oom-t* -prep {
   faultsim_restore_and_reopen
 } -body {
   execsql { 
@@ -32,5 +34,52 @@ do_faultsim_test pagerfault-1 -faults oom-t* -prep {
   faultsim_test_result {1 {sub-select returns 5 columns - expected 1}}
 }
 
+ifcapable vtab {
+  reset_db
+  do_execsql_test 2.0 {
+    CREATE TABLE t1(x);
+  }
+ 
+  proc eponymous_cmd {method args} {
+    switch -- $method {
+      xConnect {
+        db eval { SELECT * FROM sqlite_schema }
+        return "CREATE TABLE t1 (a, b)"
+      }
+  
+      xBestIndex {
+        return "idxnum 555"
+      }
+  
+      xFilter {
+        return [list sql {SELECT 123, 'A', 'B'}]
+      }
+  
+      xUpdate {
+        return 123
+      }
+  
+    }
+  
+    return {}
+  }
+
+  faultsim_save_and_close
+
+  do_faultsim_test 2 -faults oom* -prep {
+    faultsim_restore_and_reopen
+    register_tcl_module db eponymous_cmd
+    db eval { SELECT * FROM t1 }
+    sqlite3 db2 test.db
+    db2 eval { CREATE TABLE t2(y) }
+    db2 close
+  } -body {
+    db eval {
+      INSERT INTO tcl VALUES('hello', 'world') RETURNING *
+    }
+  } -test {
+    faultsim_test_result {0 {hello world}} {1 {vtable constructor failed: tcl}}
+  }
+}
 
 finish_test
diff --git a/test/rollback.test b/test/rollback.test
index 423bf20fce..f32ce523e5 100644
--- a/test/rollback.test
+++ b/test/rollback.test
@@ -111,7 +111,7 @@ if {$tcl_platform(platform) == "unix"
   ]
   set iOffset [expr (([file size testA.db-journal] + 511)/512)*512]
   set fd [open testA.db-journal a+]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   seek $fd $iOffset
   puts -nonewline $fd $zAppend
 
diff --git a/test/round1.test b/test/round1.test
index 2244a399ab..2857593be8 100644
--- a/test/round1.test
+++ b/test/round1.test
@@ -34,8 +34,8 @@ for {set iTest 1} {$iTest<=50000} {incr iTest} {
    set r2 $x1.$x4
    set ans [string trimright $r2 0]
    if {[string match *. $ans]} {set ans ${ans}0}
-   do_test $iTest/$n/${r}5=>$ans {
-     set x [db one "SELECT round(${r}5,$n)"]
+   do_test $iTest/$n/${r}6=>$ans {
+     set x [db one "SELECT round(${r}6,$n)"]
    } $ans
 }
 
diff --git a/test/rowid.test b/test/rowid.test
index 4327004d31..84f0e4d300 100644
--- a/test/rowid.test
+++ b/test/rowid.test
@@ -803,18 +803,31 @@ do_execsql_test 16.0 {
   INSERT INTO t3(rowid, z) VALUES(3, 3);
 }
 
-do_execsql_test 16.1 { SELECT rowid FROM t1, t2; } {1}
-do_execsql_test 16.2 { SELECT rowid FROM t1, v1; } {1}
-do_execsql_test 16.3 { SELECT rowid FROM t3, v1; } {3}
-do_execsql_test 16.4 { SELECT rowid FROM t3, (SELECT 123); } {3}
-
-do_execsql_test 16.5 { SELECT rowid FROM t2, t1; } {1}
-do_execsql_test 16.6 { SELECT rowid FROM v1, t1; } {1}
-do_execsql_test 16.7 { SELECT rowid FROM v1, t3; } {3}
-do_execsql_test 16.8 { SELECT rowid FROM (SELECT 123), t3; } {3}
-
-do_catchsql_test 16.5 { SELECT rowid FROM t1, t3; } {1 {no such column: rowid}}
-
+ifcapable allow_rowid_in_view {
+  set nosuch "1 {ambiguous column name: rowid}"
+  do_execsql_test 16.1 { SELECT rowid FROM t1, t2; } {1}
+  do_catchsql_test 16.2 { SELECT rowid FROM t1, v1; } $nosuch
+  do_catchsql_test 16.3 { SELECT rowid FROM t3, v1; } $nosuch
+  do_catchsql_test 16.4 { SELECT rowid FROM t3, (SELECT 123); } $nosuch
+
+  do_execsql_test 16.5 { SELECT rowid FROM t2, t1; } {1}
+  do_catchsql_test 16.6 { SELECT rowid FROM v1, t1; } $nosuch
+  do_catchsql_test 16.7 { SELECT rowid FROM v1, t3; } $nosuch
+  do_execsql_test 16.8 { SELECT rowid FROM (SELECT 123), t3; } {3}
+} else {
+  do_execsql_test 16.1 { SELECT rowid FROM t1, t2; } {1}
+  do_execsql_test 16.2 { SELECT rowid FROM t1, v1; } {1}
+  do_execsql_test 16.3 { SELECT rowid FROM t3, v1; } {3}
+  do_execsql_test 16.4 { SELECT rowid FROM t3, (SELECT 123); } {3}
+
+  do_execsql_test 16.5 { SELECT rowid FROM t2, t1; } {1}
+  do_execsql_test 16.6 { SELECT rowid FROM v1, t1; } {1}
+  do_execsql_test 16.7 { SELECT rowid FROM v1, t3; } {3}
+  do_execsql_test 16.8 { SELECT rowid FROM (SELECT 123), t3; } {3}
+}
 
+do_catchsql_test 16.9 { 
+  SELECT rowid FROM t1, t3; 
+} {1 {ambiguous column name: rowid}}
 
 finish_test
diff --git a/test/rowvalue.test b/test/rowvalue.test
index 59b44d9386..b6286302d1 100644
--- a/test/rowvalue.test
+++ b/test/rowvalue.test
@@ -255,7 +255,7 @@ do_catchsql_test 11.8 {
   SELECT * FROM t11 WHERE (a,a) IS NOT 1;
 } {1 {row value misused}}
 
-# 2016-10-27: https://www.sqlite.org/src/tktview/fef4bb4bd9185ec8f
+# 2016-10-27: https://sqlite.org/src/tktview/fef4bb4bd9185ec8f
 # Incorrect result from a LEFT JOIN with a row-value constraint
 #
 do_execsql_test 12.1 {
@@ -349,7 +349,7 @@ do_catchsql_test 15.5 {
 #-------------------------------------------------------------------------
 # Row-values used in UPDATE statements within TRIGGERs
 #
-# Ticket https://www.sqlite.org/src/info/8c9458e703666e1a
+# Ticket https://sqlite.org/src/info/8c9458e703666e1a
 #
 do_execsql_test 16.1 {
   CREATE TABLE t16a(a,b,c);
@@ -449,7 +449,7 @@ do_execsql_test 18.6 {
 } {1 1 1 1 2 1}
 
  
-# 2018-02-13 Ticket https://www.sqlite.org/src/tktview/f484b65f3d6230593c3
+# 2018-02-13 Ticket https://sqlite.org/src/tktview/f484b65f3d6230593c3
 # Incorrect result from a row-value comparison in the WHERE clause.
 #
 do_execsql_test 19.1 {
@@ -558,7 +558,7 @@ do_catchsql_test 20.1 {
   SELECT 1 WHERE (2,(2,0)) IS (2,(2,0));
 } {0 1}
 
-# 2018-11-03: Ticket https://www.sqlite.org/src/info/1a84668dcfdebaf1
+# 2018-11-03: Ticket https://sqlite.org/src/info/1a84668dcfdebaf1
 # Assertion fault when doing row-value operations on a primary key
 # containing duplicate columns.
 #
@@ -782,4 +782,77 @@ do_execsql_test 33.3 {
   SELECT * FROM t1 WHERE (a,b) BETWEEN (2,99) AND (4,0);
 } {3 100}
 
+# 2025-04-15 https://sqlite.org/forum/forumpost/b9647a113b465950
+# Incorrect result when the schema includes a table with a UNIQUE
+# constraint and one of the columns in the UNIQUE constraint is the
+# INTEGER PRIMARY KEY, and the columns that UNIQUE constraint are
+# used in a rowvalue-IN operator constraint.
+#
+# 2025-07-07 Discovered that the original fix was incomplete and
+# new tests added.  See tag-20250707-01 in the code.
+#
+reset_db
+do_execsql_test 34.1 {
+  CREATE TABLE items (
+    Id INTEGER  /* rowid alias */,
+    Item INTEGER  /* any type */,
+    Test TEXT  /* TEXT or BLOB */,
+    Filler,  /* any type */
+    PRIMARY KEY(Id),
+    UNIQUE(Item, Id)
+  );
+  INSERT INTO items (Id, Item)
+    VALUES (1, 2), (2, 2), (3, 3), (4, 5);
+  UPDATE items SET test='ok'
+    WHERE (Id, Item) IN (SELECT Id, Item FROM items);
+  SELECT Id, Item, test FROM items ORDER BY id;
+} {1 2 ok 2 2 ok 3 3 ok 4 5 ok}
+db null NULL
+do_execsql_test 34.2 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d);
+  CREATE INDEX idx ON t1(b,a);
+  INSERT INTO t1(a,b) VALUES (1, 22);
+  SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1);
+} {1 22 NULL NULL}
+do_execsql_test 34.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a, b, c, d);
+  CREATE INDEX idx ON t1(b,a,a);
+  INSERT INTO t1(a,b) VALUES (1, 22);
+  SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1);
+} {1 22 NULL NULL}
+do_execsql_test 34.4 {
+  DROP TABLE t1;
+  CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT);
+  CREATE INDEX t1a ON t1(a,id);  -- index includes PRIMARY KEY
+  CREATE TABLE t2(id INTEGER PRIMARY KEY);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100)
+    INSERT INTO t1(id,a) SELECT n, 777 FROM c;
+  INSERT INTO t2 SELECT id FROM t1;
+  SELECT *
+    FROM t1 JOIN t2 USING(id)
+   WHERE t1.a=777 AND t2.id>999
+   ORDER BY t1.id;
+} {}
+do_execsql_test 34.5 {
+  EXPLAIN QUERY PLAN
+  SELECT *
+    FROM t1 JOIN t2 USING(id)
+   WHERE t1.a=777 AND t2.id>999
+   ORDER BY t1.id;
+} {/SEARCH t1 USING COVERING INDEX t1a .a=. AND id>../}
+
+# 2025-08-04 forum post eab63506cf
+#
+do_execsql_test 35.0 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(c1 TEXT, c2 INTEGER, PRIMARY KEY(c1, c2));
+  INSERT INTO t1(c1, c2) VALUES ('a', 1);
+  SELECT ('a', 1) IN (SELECT c1, c2 from t1);
+} 1
+do_execsql_test 35.1 {
+  SELECT (1, 'a') IN (SELECT c2, c1 from t1);
+} 1
+
+
 finish_test
diff --git a/test/rowvalue3.test b/test/rowvalue3.test
index 80ebeb7617..7d07976948 100644
--- a/test/rowvalue3.test
+++ b/test/rowvalue3.test
@@ -203,7 +203,7 @@ foreach {tn idx} {
 #-------------------------------------------------------------------------
 
 # 2016-11-17.  Query flattening in a vector SELECT on the RHS of an IN
-# operator.  Ticket https://www.sqlite.org/src/info/da7841375186386c
+# operator.  Ticket https://sqlite.org/src/info/da7841375186386c
 #
 do_execsql_test 5.0 {
   DROP TABLE IF EXISTS t1;
diff --git a/test/rowvalue4.test b/test/rowvalue4.test
index 784859cb19..1ef5fc2920 100644
--- a/test/rowvalue4.test
+++ b/test/rowvalue4.test
@@ -236,9 +236,11 @@ do_eqp_test 5.1 {
   QUERY PLAN
   |--SEARCH d2 USING INDEX d2ab (a=? AND b=?)
   |--LIST SUBQUERY xxxxxx
-  |  `--SCAN d1
+  |  |--SCAN d1
+  |  `--CREATE BLOOM FILTER
   `--LIST SUBQUERY xxxxxx
-     `--SCAN d1
+     |--SCAN d1
+     `--CREATE BLOOM FILTER
 }
 
 do_execsql_test 6.0 {
diff --git a/test/rowvalue7.test b/test/rowvalue7.test
index 03591afaf4..2823667e73 100644
--- a/test/rowvalue7.test
+++ b/test/rowvalue7.test
@@ -56,7 +56,7 @@ do_catchsql_test 2.2 {
 } {1 {3 columns assigned 2 values}}
 
 # 2019-08-26
-# ticket https://www.sqlite.org/src/info/78acc9d40f0786e8
+# ticket https://sqlite.org/src/info/78acc9d40f0786e8
 #
 do_catchsql_test 3.0 {
   DROP TABLE IF EXISTS t1;
diff --git a/test/rowvalue9.test b/test/rowvalue9.test
index aee5e7ea4f..baa13f4f94 100644
--- a/test/rowvalue9.test
+++ b/test/rowvalue9.test
@@ -350,5 +350,3 @@ do_eqp_test 9.5e {
 
 
 finish_test
-
-
diff --git a/test/rowvalueA.test b/test/rowvalueA.test
index dc5a7a014b..8760c2c396 100644
--- a/test/rowvalueA.test
+++ b/test/rowvalueA.test
@@ -74,4 +74,3 @@ do_catchsql_test 2.3 {
 } {1 {row value misused}}
 
 finish_test
-
diff --git a/test/savepoint6.test b/test/savepoint6.test
index b1d0d46f5c..6b41ef2da9 100644
--- a/test/savepoint6.test
+++ b/test/savepoint6.test
@@ -15,6 +15,10 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
 proc sql {zSql} {
+  if {0 && $::debug_op} {
+    puts stderr "$zSql ;"
+    flush stderr
+  }
   uplevel db eval [list $zSql]
   #puts stderr "$zSql ;"
 }
@@ -67,11 +71,13 @@ proc x_to_y {x} {
 #   delete_rows XVALUES
 #
 proc savepoint {zName} {
+  if {$::debug_op} { puts stderr "savepoint $zName" ; flush stderr }
   catch { sql "SAVEPOINT $zName" }
   lappend ::lSavepoint [list $zName [array get ::aEntry]]
 }
 
 proc rollback {zName} {
+  if {$::debug_op} { puts stderr "rollback $zName" ; flush stderr }
   catch { sql "ROLLBACK TO $zName" }
   for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
     set zSavepoint [lindex $::lSavepoint $i 0]
@@ -89,6 +95,7 @@ proc rollback {zName} {
 }
 
 proc release {zName} {
+  if {$::debug_op} { puts stderr "release $zName" ; flush stderr }
   catch { sql "RELEASE $zName" }
   for {set i [expr {[llength $::lSavepoint]-1}]} {$i>=0} {incr i -1} {
     set zSavepoint [lindex $::lSavepoint $i 0]
@@ -104,6 +111,7 @@ proc release {zName} {
 }
 
 proc insert_rows {lX} {
+  if {$::debug_op} { puts stderr "insert_rows $lX" ; flush stderr }
   foreach x $lX {
     set y [x_to_y $x]
 
@@ -116,6 +124,7 @@ proc insert_rows {lX} {
 }
 
 proc delete_rows {lX} {
+  if {$::debug_op} { puts stderr "delete_rows $lX" ; flush stderr }
   foreach x $lX {
     # Update database [db]
     sql "DELETE FROM t1 WHERE x = $x"
@@ -164,6 +173,11 @@ proc random_integers {nRes nRange} {
 } 
 #-------------------------------------------------------------------------
 
+set ::debug_op 0
+proc debug_ops {} {
+  set ::debug_op 1
+}
+
 proc database_op {} {
   set i [expr int(rand()*2)] 
   if {$i==0} {
@@ -185,9 +199,6 @@ proc savepoint_op {} {
   set C [lindex $cmds [expr int(rand()*6)]]
   set N [lindex $names [expr int(rand()*5)]]
 
-  #puts stderr "   $C $N ;  "
-  #flush stderr
-
   $C $N
   return ok
 }
diff --git a/test/savepoint7.test b/test/savepoint7.test
index 59c3cd6cdc..2652cc3972 100644
--- a/test/savepoint7.test
+++ b/test/savepoint7.test
@@ -95,7 +95,7 @@ do_test savepoint7-2.2 {
   list $rc $msg [db eval {SELECT * FROM t2}]
 } {1 {abort due to ROLLBACK} {}}
 
-# Ticket: https://www.sqlite.org/src/tktview/7f7f8026eda387d544b
+# Ticket: https://sqlite.org/src/tktview/7f7f8026eda387d544b
 # Segfault in the in-memory journal logic triggered by a tricky
 # combination of SAVEPOINT operations.
 #
diff --git a/test/scanstatus2.test b/test/scanstatus2.test
index fb62af2103..c94db88f23 100644
--- a/test/scanstatus2.test
+++ b/test/scanstatus2.test
@@ -55,11 +55,12 @@ proc get_cycles {stmt} {
   dict get $r nCycle
 }
 
-proc foreach_scan {varname stmt body} {
+proc foreach_scan {varname stmt body {debug 0}} {
   upvar $varname var
-
   for {set ii 0} {1} {incr ii} {
-    set r [sqlite3_stmt_scanstatus -flags complex $stmt $ii]
+    set f "complex"
+    if {$debug} { set f "complex debug" }
+    set r [sqlite3_stmt_scanstatus -flags $f $stmt $ii]
     if {[llength $r]==0} break
     array set var $r
     uplevel $body
@@ -102,6 +103,15 @@ proc puts_graph {sql} {
   puts [string trim [get_graph $stmt]]
 }
 
+proc puts_debug_info {sql} {
+  db eval $sql
+  set stmt [db version -last-stmt-ptr]
+  foreach_scan X $stmt {
+    puts -nonewline "$X(debug_explain) $X(zExplain): "
+    puts -nonewline "loop=$X(debug_loop) visit=$X(debug_visit) "
+    puts            "csr=$X(debug_csr) range=$X(debug_range)"
+  } 1
+}
 
 do_zexplain_test 0 1.1 {
   SELECT (SELECT a FROM t1 WHERE b=x) FROM t2 WHERE y=2
@@ -135,8 +145,8 @@ do_graph_test 1.4 {
 QUERY (nCycle=nnn)
 --MATERIALIZE v2 (nCycle=nnn)
 ----SCAN t2 (nCycle=nnn)
---SCAN v2 (nCycle=nnn)
 --SCAN t1 (nCycle=nnn)
+--SCAN v2 (nCycle=nnn)
 --USE TEMP B-TREE FOR ORDER BY (nCycle=nnn)
 }
 
@@ -235,7 +245,7 @@ do_graph_test 4.5 {
 QUERY (nCycle=nnn)
 --CO-ROUTINE v1
 ----SCAN rt2 (nCycle=nnn)
-----USE TEMP B-TREE FOR GROUP BY
+----USE TEMP B-TREE FOR GROUP BY (nCycle=nnn)
 --SCAN rt1 (nCycle=nnn)
 --CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn)
 --BLOOM FILTER ON v1 (x1=?)
@@ -255,7 +265,7 @@ ifcapable trace {
   }
   
   proc trace {stmt sql} {
-    array set A [sqlite3_stmt_scanstatus -flags complex [format %x $stmt] 0]
+    array set A [sqlite3_stmt_scanstatus -flags complex [format %llx $stmt] 0]
     lappend ::trace_explain $A(zExplain)
   }
   db trace_v2 trace 
@@ -270,6 +280,66 @@ ifcapable trace {
   } {{SCAN t1} {SCAN t1} {SCAN t1}}
 }
 
-finish_test
+#-------------------------------------------------------------------------
+reset_db
+sqlite3_db_config db STMT_SCANSTATUS 1
 
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 'one');
+  INSERT INTO t1 VALUES(2, 'two');
+  INSERT INTO t1 VALUES(3, 'three');
+  INSERT INTO t1 VALUES(4, 'four');
+  INSERT INTO t1 VALUES(5, 'five');
+  INSERT INTO t1 VALUES(6, 'six');
+  INSERT INTO t1 VALUES(7, 'seven');
+  INSERT INTO t1 VALUES(8, 'eight');
+}
 
+do_graph_test 6.1 {
+  SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1
+} {
+QUERY (nCycle=nnn)
+--SCAN t1 (nCycle=nnn)
+--USE TEMP B-TREE FOR GROUP BY (nCycle=nnn)
+}
+
+set sql {
+  WITH xy(x, y) AS ( SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1)
+  SELECT * FROM xy WHERE x=1
+}
+do_graph_test 6.2 $sql {
+QUERY (nCycle=nnn)
+--CO-ROUTINE xy
+----SCAN t1 (nCycle=nnn)
+----USE TEMP B-TREE FOR GROUP BY (nCycle=nnn)
+--SCAN xy (nCycle=nnn)
+}
+
+do_graph_test 6.3 {
+  WITH xy(x, y) AS ( SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1)
+  SELECT * FROM xy, xy AS xy2
+} {
+QUERY (nCycle=nnn)
+--MATERIALIZE xy (nCycle=nnn)
+----SCAN t1 (nCycle=nnn)
+----USE TEMP B-TREE FOR GROUP BY (nCycle=nnn)
+--SCAN xy (nCycle=nnn)
+--SCAN xy2 (nCycle=nnn)
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+# Check that an OOB parameter (45) does not cause asan or valgrind errors.
+#
+do_test 7.0 {
+  db eval {SELECT * FROM sqlite_schema}
+  set stmt [db version -last-stmt-ptr]
+  sqlite3_stmt_scanstatus -flags complex $stmt 1000000
+} {}
+
+#explain_i { SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1 }
+#puts_debug_info { SELECT (a % 2), group_concat(b) FROM t1 GROUP BY 1 }
+
+finish_test
diff --git a/test/select3.test b/test/select3.test
index 4c9d71b4f5..ab16ab9fd8 100644
--- a/test/select3.test
+++ b/test/select3.test
@@ -268,7 +268,7 @@ do_test select3-8.2 {
   }
 } {real}
 
-# 2019-05-09 ticket https://www.sqlite.org/src/tktview/6c1d3febc00b22d457c7
+# 2019-05-09 ticket https://sqlite.org/src/tktview/6c1d3febc00b22d457c7
 #
 unset -nocomplain x
 foreach {id x} {
@@ -434,4 +434,3 @@ do_execsql_test 12.8 {
 }
 
 finish_test
-
diff --git a/test/select4.test b/test/select4.test
index d49708ece8..890897f2a9 100644
--- a/test/select4.test
+++ b/test/select4.test
@@ -915,7 +915,7 @@ do_execsql_test select4-14.17 {
   VALUES(1),(2),(3),(4) UNION ALL SELECT 5 LIMIT 3;
 } {1 2 3}
 
-# Ticket https://www.sqlite.org/src/info/d06a25c84454a372
+# Ticket https://sqlite.org/src/info/d06a25c84454a372
 # Incorrect answer due to two co-routines using the same registers and expecting
 # those register values to be preserved across a Yield.
 #
@@ -968,7 +968,7 @@ do_execsql_test select4-16.3 {
   ORDER BY t3.a;
 } {95 96 97 98 99}
 
-# Ticket https://www.sqlite.org/src/tktview/f7f8c97e975978d45  on 2016-04-25
+# Ticket https://sqlite.org/src/tktview/f7f8c97e975978d45  on 2016-04-25
 #
 # The where push-down optimization from 2015-06-02 is suppose to disable
 # on aggregate subqueries.  But if the subquery is a compound where the
diff --git a/test/select6.test b/test/select6.test
index 612afefa6f..301ec11c36 100644
--- a/test/select6.test
+++ b/test/select6.test
@@ -611,7 +611,7 @@ do_execsql_test 11.100 {
     FROM ( SELECT count(*) AS cnt FROM t1 );
 } {{}}
 
-# 2019-05-29 ticket https://www.sqlite.org/src/info/c41afac34f15781f
+# 2019-05-29 ticket https://sqlite.org/src/info/c41afac34f15781f
 # A LIMIT clause in a subquery is incorrectly applied to a subquery.
 #
 do_execsql_test 12.100 {
@@ -628,4 +628,43 @@ do_execsql_test 12.100 {
     SELECT * from t2);
 } {1 3}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 13.100 {
+
+  CREATE  TABLE  t1(y INT);
+  INSERT INTO t1 (y) VALUES (1);
+
+  CREATE  TABLE  t2(x INTEGER);
+  INSERT INTO t2 VALUES(0);
+
+  CREATE  TABLE  empty1(z);
+}
+
+do_execsql_test 13.110 {
+  SELECT t1.y 
+    FROM ( SELECT 'AAA' ) 
+    INNER JOIN (
+      SELECT 1 AS abc FROM (
+        SELECT 1 FROM t2 LEFT JOIN empty1
+      )
+    ) AS sub0 ON sub0.abc
+    , t1
+    RIGHT JOIN (SELECT 'BBB' FROM ( SELECT 'CCC' ))
+} {1}
+
+do_execsql_test 13.120 {
+  SELECT t1.y 
+    FROM ( SELECT 'AAA' ) 
+    INNER JOIN (
+      SELECT 1 AS abc FROM (
+        SELECT 1 FROM t2 LEFT JOIN empty1
+      )
+    ) AS sub0 ON sub0.abc
+    , t1
+    RIGHT JOIN (SELECT 'BBB' FROM ( SELECT 'CCC' ))
+    WHERE t1.y
+} {1}
+
+
 finish_test
diff --git a/test/select7.test b/test/select7.test
index d705ebfaf4..0c4051006a 100644
--- a/test/select7.test
+++ b/test/select7.test
@@ -155,6 +155,38 @@ if {[clang_sanitize_address]==0} {
   }
 }
 
+# https://issues.chromium.org/issues/358174302
+# Need to support an unlimited number of terms in a VALUES clause, even
+# if some of those terms contain double-quoted string literals.
+#
+do_execsql_test select7-6.5 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a,b,c);
+}
+sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 10
+sqlite3_db_config db SQLITE_DBCONFIG_DQS_DML 0
+do_catchsql_test select7-6.6 {
+  INSERT INTO t1 VALUES
+    (NULL,0,""),  (X'',0.0,0.0),  (X'',X'',""),  (0.0,0.0,""),  (NULL,NULL,0.0),
+    (0,"",0),  (0.0,X'',0),  ("",X'',0.0),  (0.0,X'',NULL),  (0,NULL,""),
+    (0,"",NULL),  (0.0,NULL,X''),  ("",X'',NULL),  (NULL,0,""),
+    (0,NULL,0),  (X'',X'',0.0);
+} {1 {no such column: "" - should this be a string literal in single-quotes?}}
+do_execsql_test select7-6.7 {
+  SELECT count(*) FROM t1;
+} {0}
+sqlite3_db_config db SQLITE_DBCONFIG_DQS_DML 1
+do_catchsql_test select7-6.8 {
+  INSERT INTO t1 VALUES
+    (NULL,0,""),  (X'',0.0,0.0),  (X'',X'',""),  (0.0,0.0,""),  (NULL,NULL,0.0),
+    (0,"",0),  (0.0,X'',0),  ("",X'',0.0),  (0.0,X'',NULL),  (0,NULL,""),
+    (0,"",NULL),  (0.0,NULL,X''),  ("",X'',NULL),  (NULL,0,""),
+    (0,NULL,0),  (X'',X'',0.0);
+} {0 {}}
+do_execsql_test select7-6.9 {
+  SELECT count(*) FROM t1;
+} {16}
+
 # This block of tests verifies that bug aa92c76cd4 is fixed.
 #
 do_test select7-7.1 {
diff --git a/test/selectA.test b/test/selectA.test
index 7d72bb3fa5..91b1548488 100644
--- a/test/selectA.test
+++ b/test/selectA.test
@@ -1340,10 +1340,10 @@ do_eqp_test 4.1.2 {
   `--MERGE (UNION ALL)
      |--LEFT
      |  |--SCAN t5 USING INDEX i2
-     |  `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+     |  `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY
      `--RIGHT
         |--SCAN t4 USING INDEX i1
-        `--USE TEMP B-TREE FOR RIGHT PART OF ORDER BY
+        `--USE TEMP B-TREE FOR LAST TERM OF ORDER BY
 }
 
 do_execsql_test 4.1.3 {
diff --git a/test/selectH.test b/test/selectH.test
index a97679bda2..41f0999fe3 100644
--- a/test/selectH.test
+++ b/test/selectH.test
@@ -113,6 +113,33 @@ do_execsql_test 4.1 {
   CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT);
   SELECT 1 FROM (SELECT DISTINCT name COLLATE rtrim FROM sqlite_schema
                  UNION ALL SELECT a FROM t1);
-} 1
+} {1 1}
+
+do_execsql_test 4.2 {
+  SELECT DISTINCT name COLLATE rtrim FROM sqlite_schema 
+    UNION ALL 
+  SELECT a FROM t1
+} {v1 t1}
+
+#-------------------------------------------------------------------------
+# forum post https://sqlite.org/forum/forumpost/b83c7b2168
+#
+reset_db
+do_execsql_test 5.0 {
+  CREATE TABLE t1 (val1);
+  INSERT INTO t1 VALUES(4);
+  INSERT INTO t1 VALUES(5);
+  CREATE TABLE t2 (val2);
+}
+do_execsql_test 5.1 {
+  SELECT DISTINCT val1 FROM t1 UNION ALL SELECT val2 FROM t2;
+} {
+  4 5
+}
+do_execsql_test 5.2 {
+  SELECT count(1234) FROM (
+    SELECT DISTINCT val1 FROM t1 UNION ALL SELECT val2 FROM t2
+  )
+} {2}
 
 finish_test
diff --git a/test/sessionfuzz.c b/test/sessionfuzz.c
index f2e4cd5a68..c389a5e996 100644
--- a/test/sessionfuzz.c
+++ b/test/sessionfuzz.c
@@ -60,20 +60,13 @@
 
 #define SQLITE_DEBUG 1
 #define SQLITE_THREADSAFE 0
+#undef SQLITE_OMIT_LOAD_EXTENSION
 #define SQLITE_OMIT_LOAD_EXTENSION 0
 #define SQLITE_ENABLE_SESSION 1
 #define SQLITE_ENABLE_PREUPDATE_HOOK 1
 #define SQLITE_ENABLE_DESERIALIZE 1
 #include "sqlite3.c"
 
-/* Create a test database.  This will be an in-memory database */
-static const char zInitSql[] = 
-  "CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d);\n"
-  "CREATE TABLE t2(e TEXT PRIMARY KEY NOT NULL,f,g);\n"
-  "CREATE TABLE t3(w REAL PRIMARY KEY NOT NULL,x,y);\n"
-  "CREATE TABLE t4(z PRIMARY KEY) WITHOUT ROWID;\n"
-;
-
 /* Code to populate the database */
 static const char zFillSql[] = 
   "INSERT INTO t1(a,b,c,d) VALUES\n"
diff --git a/test/shared.test b/test/shared.test
index a0cd0a6696..42292ab40f 100644
--- a/test/shared.test
+++ b/test/shared.test
@@ -161,7 +161,7 @@ do_test shared-$av.1.8 {
 
 do_test shared-$av.2.1 {
   # Open connection db3 to the database.
-  if {$::tcl_platform(platform)=="unix"} {
+  if {$::tcl_platform(platform) eq "unix"} {
     sqlite3 db3 "file:test.db?cache=private" -uri 1
   } else {
     sqlite3 db3 TEST.DB
@@ -312,7 +312,7 @@ do_test shared-$av.4.1.3 {
 } {2}
 
 # Sanity check: Create a table in ./test.db via handle db, and test that handle
-# db2 can "see" the new table immediately. A handle using a seperate pager
+# db2 can "see" the new table immediately. A handle using a separate pager
 # cache would have to reload the database schema before this were possible.
 #
 do_test shared-$av.4.2.1 {
@@ -797,7 +797,7 @@ do_test shared-$av.10.2 {
 do_test shared-$av.10.3 {
   # An external connection should be able to read the database, but not
   # prepare a write operation.
-  if {$::tcl_platform(platform)=="unix"} {
+  if {$::tcl_platform(platform) eq "unix"} {
     sqlite3 db3 "file:test.db?cache=private" -uri 1
   } else {
     sqlite3 db3 TEST.DB
diff --git a/test/shared3.test b/test/shared3.test
index e313069990..2e32398cac 100644
--- a/test/shared3.test
+++ b/test/shared3.test
@@ -70,7 +70,7 @@ do_test shared3-2.5 {
 # test case shared3-2.3 above). The goal of the following tests is to
 # ensure that the cache-size really is 10 pages.
 #
-#if {$::tcl_platform(platform)=="unix"} {
+#if {$::tcl_platform(platform) eq "unix"} {
 #  set alternative_name ./test.db
 #} else {
 #  set alternative_name TEST.DB
diff --git a/test/shared6.test b/test/shared6.test
index 499cbb0eb5..18f9f537b6 100644
--- a/test/shared6.test
+++ b/test/shared6.test
@@ -139,7 +139,7 @@ do_test shared6-1.X {
 # that connect to the same file using different VFS implementations do
 # not share a cache.
 #
-if {$::tcl_platform(platform) eq "unix"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
   do_test shared6-2.1 {
     sqlite3 db1 test.db -vfs unix
     sqlite3 db2 test.db -vfs unix
diff --git a/test/shell1.test b/test/shell1.test
index d017379107..abf214a907 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -11,6 +11,7 @@
 #
 # The focus of this file is testing the CLI shell tool.
 #
+# TESTRUNNER: shell
 #
 
 # Test plan:
@@ -295,7 +296,9 @@ do_test shell1-3.2.4 {
   catchcmd "test.db" ".bail OFF BAD"
 } {1 {Usage: .bail on|off}}
 
-ifcapable vtab {
+# This test will not work on winrt, as winrt has no concept of the absolute
+# paths that the test expects in the result. It uses relative paths only.
+ifcapable vtab&&!winrt {
 # .databases             List names and files of attached databases
 do_test shell1-3.3.1 {
   catchcmd "-csv test.db" ".databases"
@@ -449,7 +452,7 @@ do_test shell1-3.12.3 {
 #                          tcl      TCL list elements
 do_test shell1-3.13.1 {
   catchcmd "test.db" ".mode"
-} {0 {current output mode: list}}
+} {0 {current output mode: list --escape ascii}}
 do_test shell1-3.13.2 {
   catchcmd "test.db" ".mode FOO"
 } {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}}
@@ -496,20 +499,28 @@ do_test shell1-3.14.3 {
 
 # .output FILENAME       Send output to FILENAME
 do_test shell1-3.15.1 {
-  catchcmd "test.db" ".output"
-} {0 {}}
+  catchcmd "test.db" ".output
+.print x"
+} {0 x}
 do_test shell1-3.15.2 {
-  catchcmd "test.db" ".output FOO"
-} {0 {}}
+  catchcmd "test.db" ".output FOO
+.print x
+.output
+SELECT readfile('FOO');"
+} {0 {x
+}}
 do_test shell1-3.15.3 {
   # too many arguments
   catchcmd "test.db" ".output FOO BAD"
 } {1 {ERROR: extra parameter: "BAD".  Usage:
 .output ?FILE?           Send output to FILE or stdout if FILE is omitted
    If FILE begins with '|' then open it as a pipe.
+   If FILE is 'off' then output is disabled.
    Options:
      --bom                 Prefix output with a UTF8 byte-order mark
      -e                    Send output to the system text editor
+     --plain               Use text/plain for -w option
+     -w                    Send output to a web browser
      -x                    Send output as CSV to a spreadsheet
 child process exited abnormally}}
 
@@ -523,9 +534,12 @@ do_test shell1-3.16.2 {
 } {1 {ERROR: extra parameter: "BAD".  Usage:
 .output ?FILE?           Send output to FILE or stdout if FILE is omitted
    If FILE begins with '|' then open it as a pipe.
+   If FILE is 'off' then output is disabled.
    Options:
      --bom                 Prefix output with a UTF8 byte-order mark
      -e                    Send output to the system text editor
+     --plain               Use text/plain for -w option
+     -w                    Send output to a web browser
      -x                    Send output as CSV to a spreadsheet
 child process exited abnormally}}
 
@@ -734,9 +748,12 @@ do_test shell1-3.26.6 {
 do_test shell1-3.27.1 {
   catchcmd "test.db" ".timer"
 } {1 {Usage: .timer on|off}}
-do_test shell1-3.27.2 {
-  catchcmd "test.db" ".timer ON"
-} {0 {}}
+ifcapable !winrt {
+  # No timer support on winrt.
+  do_test shell1-3.27.2 {
+    catchcmd "test.db" ".timer ON"
+  } {0 {}}
+}
 do_test shell1-3.27.3 {
   catchcmd "test.db" ".timer OFF"
 } {0 {}}
@@ -928,6 +945,71 @@ do_test shell1-4.1.6 {
 
 }
 
+# DELETE content of sqlite_sequence prior to repopulating,
+# but only if the sqlite_sequence table is non-empty.
+# Forum: 2024-10-13T17:10:01z and 2025-10-29T19:38:43z
+#
+do_test shell1-4.1.7 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
+    INSERT INTO t1 VALUES(1,2),(20,21),(15,16);
+  }
+  catchcmd test2.db {.dump}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
+INSERT INTO t1 VALUES(1,2);
+INSERT INTO t1 VALUES(15,16);
+INSERT INTO t1 VALUES(20,21);
+PRAGMA writable_schema=ON;
+CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);
+DELETE FROM sqlite_sequence;
+INSERT INTO sqlite_sequence VALUES('t1',20);
+PRAGMA writable_schema=OFF;
+COMMIT;}}
+do_test shell1-4.1.8 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
+    INSERT INTO t1 VALUES(1,2),(20,21),(15,16);
+    CREATE TABLE t2(x,y);
+    INSERT INTO t2 VALUES(99,88);
+    DROP TABLE t1;
+  }
+  catchcmd test2.db {.dump}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE t2(x,y);
+INSERT INTO t2 VALUES(99,88);
+COMMIT;}}
+do_test shell1-4.1.9 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b);
+    INSERT INTO t1 VALUES(1,2),(20,21),(15,16);
+    CREATE TABLE t2(x,y);
+    INSERT INTO t2 VALUES(99,88);
+    INSERT INTO sqlite_sequence VALUES('extra',999);
+    DROP TABLE t1;
+  }
+  catchcmd test2.db {.dump}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE t2(x,y);
+INSERT INTO t2 VALUES(99,88);
+PRAGMA writable_schema=ON;
+CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);
+DELETE FROM sqlite_sequence;
+INSERT INTO sqlite_sequence VALUES('extra',999);
+PRAGMA writable_schema=OFF;
+COMMIT;}}
 
 # Test the output of ".mode insert"
 #
@@ -1049,17 +1131,20 @@ do_test shell1-5.0 {
     #       return character (and on Windows, the end-of-file character)
     #       cannot be used here.
     #
-    if {$i==0x0D || ($tcl_platform(platform)=="windows" && $i==0x1A)} {
+    if {$i==0x0D || ($tcl_platform(platform) eq "windows" && $i==0x1A)} {
       continue
     }
     # Tcl 8.7 maps 0x80 through 0x9f into valid UTF8.  So skip those tests.
-    if {$i>=0x80 && $i<=0x9f} continue
-    if {$i>=0xE0 && $tcl_platform(os)=="OpenBSD"}  continue
-    if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os)=="Linux"}  continue
+    if {$i>=0x80} {
+      if {$i<=0x9F || $tcl_version>=9.0} continue
+      if {$tcl_platform(platform) eq "windows"} continue
+    }
+    if {$i>=0xE0 && $tcl_platform(os) eq "OpenBSD"}  continue
+    if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os) eq "Linux"}  continue
     set hex [format %02X $i]
     set char [subst \\x$hex]; set oldChar $char
     set escapes [list]
-    if {$tcl_platform(platform)=="windows"} {
+    if {$tcl_platform(platform) eq "windows"} {
       #
       # NOTE: On Windows, we need to escape all the whitespace characters,
       #       the alarm (\a) character, and those with special meaning to
@@ -1067,7 +1152,7 @@ do_test shell1-5.0 {
       #
       set escapes [list \
           \a \\a \b \\b \t \\t \n \\n \v \\v \f \\f \r \\r \
-          " " "\" \"" \" \\\" ' \"'\" \\ \\\\]
+          " " "\" \"" \" \\\" \\ \\\\]
     } else {
       #
       # NOTE: On Unix, we need to escape most of the whitespace characters
@@ -1081,10 +1166,10 @@ do_test shell1-5.0 {
       #
       set escapes [list \
           \t \\t \n \\n \v \\v \f \\f \
-          " " "\" \"" \" \\\" ' \"'\" \\ \\\\]
+          " " "\" \"" \" \\\" \\ \\\\]
     }
     set char [string map $escapes $char]
-    set x [catchcmdex test.db ".print $char\n"]
+    set x [catchcmdex test.db ".print \"$char\"\n"]
     set code [lindex $x 0]
     set res [lindex $x 1]
     if {$code ne "0"} {
@@ -1165,21 +1250,21 @@ do_test shell1-7.1.2 {
 } {0 {CREATE TABLE Z (x TEXT PRIMARY KEY);
 CREATE TABLE _ (x TEXT PRIMARY KEY);}}
 do_test shell1-7.1.3 {
-  catchcmd "test.db" ".schema \\\\_"
+  catchcmd "test.db" ".schema \"\\\\_\""
 } {0 {CREATE TABLE _ (x TEXT PRIMARY KEY);}}
 do_test shell1-7.1.4 {
   catchcmd "test.db" ".schema __"
 } {0 {CREATE TABLE YY (x TEXT PRIMARY KEY);
 CREATE TABLE __ (x TEXT PRIMARY KEY);}}
 do_test shell1-7.1.5 {
-  catchcmd "test.db" ".schema \\\\_\\\\_"
+  catchcmd "test.db" ".schema \"\\\\_\\\\_\""
 } {0 {CREATE TABLE __ (x TEXT PRIMARY KEY);}}
 do_test shell1-7.1.6 {
   catchcmd "test.db" ".schema ___"
 } {0 {CREATE TABLE WWW (x TEXT PRIMARY KEY);
 CREATE TABLE ___ (x TEXT PRIMARY KEY);}}
 do_test shell1-7.1.7 {
-  catchcmd "test.db" ".schema \\\\_\\\\_\\\\_"
+  catchcmd "test.db" ".schema \"\\\\_\\\\_\\\\_\""
 } {0 {CREATE TABLE ___ (x TEXT PRIMARY KEY);}}
 
 }
@@ -1210,7 +1295,7 @@ do_test shell1-8.1 {
       FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n);
   }
 } {0 47.49000000000000198951966012828052043914794921875}
-do_test shell1-8.2 {
+do_test_with_ansi_output shell1-8.2 {
   catchcmd :memory: {
 .mode box
 SELECT ieee754(47.49) AS x;
@@ -1220,7 +1305,7 @@ SELECT ieee754(47.49) AS x;
 ├───────────────────────────────┤
 │ ieee754(6683623321994527,-47) │
 └───────────────────────────────┘}}
-do_test shell1-8.3 {
+do_test_with_ansi_output shell1-8.3 {
   catchcmd ":memory: --box" {
     select ieee754(6683623321994527,-47) as x;
   }
@@ -1236,7 +1321,7 @@ do_test shell1-8.4 {
 +------------------+-----+
 | 6683623321994527 | -47 |
 +------------------+-----+}}
-do_test shell1-8.5 {
+do_test_with_ansi_output shell1-8.5 {
   catchcmd ":memory: --box" {
 create table t(a text, b int);
 insert into t values ('too long for one line', 1), ('shorter', NULL);
@@ -1284,4 +1369,22 @@ select base64(base64(cast('digity-doo' as blob))),
 }
 } {0 digity-doo|digity-doo}
 
+#----------------------------------------------------------------------------
+# Test cases shell1-11.*: 
+#
+do_test shell1-11.1 {
+  catchcmd :memory: {
+.mode list
+.header off
+select base64(zeroblob(2000000000));
+}
+} {/1.*too big.*/}
+do_test shell1-11.2 {
+  catchcmd :memory: {
+.mode list
+.header off
+select base85(zeroblob(2000000000));
+}
+} {/1.*too big.*/}
+
 finish_test
diff --git a/test/shell2.test b/test/shell2.test
index 3fad4bd665..5f700a9a1d 100644
--- a/test/shell2.test
+++ b/test/shell2.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # The focus of this file is testing the CLI shell tool.
 #
@@ -216,37 +217,40 @@ do_test shell2-1.4.9 {
 done
 2}}
 
+ifcapable vtab {
 # Verify that generate_series stays sane near 64-bit range boundaries.
 # See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280
 do_test shell2-1.4.10 {
  set res [catchcmd :memory: [string trim {
  SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1);
  SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1);
- SELECT avg(rowid),min(value),max(value) FROM generate_series(
+ SELECT avg(value),min(value),max(value) FROM generate_series(
   -9223372036854775808,9223372036854775807,1085102592571150095);
  SELECT * FROM generate_series(-9223372036854775808,9223372036854775807,
   9223372036854775807);
- SELECT value,rowid FROM generate_series(-4611686018427387904,
+ SELECT value FROM generate_series(-4611686018427387904,
   4611686018427387904, 4611686018427387904) ORDER BY value DESC;
  SELECT * FROM generate_series(0,-2,-1);
  SELECT * FROM generate_series(0,-2);
  SELECT * FROM generate_series(0,2) LIMIT 3;}]]
 } {0 {9223372036854775807
 9223372036854775807
-9.5|-9223372036854775808|9223372036854775807
+-0.5|-9223372036854775808|9223372036854775807
 -9223372036854775808
 -1
 9223372036854775806
-4611686018427387904|3
-0|2
--4611686018427387904|1
+4611686018427387904
+0
+-4611686018427387904
 0
 -1
 -2
 0
 1
 2}}
+}  ;# ifcapable vtab
 
+ifcapable vtab {
 # Bug discovered while messing around, .import hangs with
 # bit 7 set in column separator.
 do_test shell2-1.4.11 {
@@ -261,6 +265,108 @@ do_test shell2-1.4.11 {
 .import dummy.csv t
  SELECT count(*) FROM t;}]]
 } {0 1}
+} ;# ifcapable vtab
+
+# Bug from forum post 7cbe081746dd3803
+# Keywords as column names were producing an error message.
+do_test shell2-1.4.12 {
+  set res [catchcmd :memory: [string trim {
+ CREATE TABLE "group"("order" text);
+ INSERT INTO "group" VALUES ('ABC');
+.sha3sum}]]
+} {0 ca08bc02b7e95c7df431a3a4b1cc0f8d8743914793473f55b5558e03}
+
+#-------------------------------------------------------------------------
+
+foreach {tn hexdump expect} {
+  0 {
+| size 8192 pagesize 4096 filename my.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03   ................
+|     96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00   ................
+|   4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f   ................
+|   4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43   ....../tablett.C
+|   4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29   REATE TABLE t(x)
+| page 2 offset 4096
+|      0: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02   ................
+|   4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f   ..world....hello
+| end my.db
+  }
+  {0 {}}
+
+  1 {
+| size 2147483647 pagesize 4096 filename my.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03   ................
+|     96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00   ................
+|   4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f   ................
+|   4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43   ....../tablett.C
+|   4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29   REATE TABLE t(x)
+| page 2 offset 4096
+|      0: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02   ................
+|   4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f   ..world....hello
+| end my.db
+  }
+  {1 {Error: out of memory}}
+
+  2 {
+| size 8192 pagesize 4096 filename my.db
+| page 1 offset 2147483647
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03   ................
+|     96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00   ................
+|   4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f   ................
+|   4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43   ....../tablett.C
+|   4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29   REATE TABLE t(x)
+| page 2 offset 4096
+|      0: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02   ................
+|   4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f   ..world....hello
+| end my.db
+  }
+  {0 {}}
+
+ 3 {
+| size 8192 pagesize 4096 filename my.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03   ................
+|     96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00   ................
+|   4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f   ................
+|   4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43   ....../tablett.C
+|   4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29   REATE TABLE t(x)
+| page 2 offset 4096
+|      2147483647: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02   ................
+|   4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f   ..world....hello
+| end my.db
+}
+  {0 {}}
+
+} {
+  set fd [open dump.txt w]
+  puts $fd [string trim $hexdump]
+  close $fd
+  do_test shell2-2.$tn.1 {
+    set rc [ catchcmd "" ".open --hexdb dump.txt"]
+  } $expect
+}
 
 
 finish_test
diff --git a/test/shell3.test b/test/shell3.test
index f8d69946e7..6bb49f5c3d 100644
--- a/test/shell3.test
+++ b/test/shell3.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # The focus of this file is testing the CLI shell tool.
 #
@@ -35,7 +36,7 @@ sqlite3 db test.db
 # different.  This causes problems for the tests below.  To avoid
 # issues, these tests are disabled for windows.
 #
-if {$::tcl_platform(platform)=="windows"} {
+if {$::tcl_platform(platform) eq "windows"} {
   finish_test
   return
 }
@@ -135,4 +136,13 @@ a["b"]
 .*/ x
 'y}} 1}
 
+do_test shell3-3.2 {
+  catchcmd "" {
+.open xyz.db
+SELECT ;
+  }
+} {1 {Parse error near line 3: near ";": syntax error
+  SELECT ;
+         ^--- error here}}
+
 finish_test
diff --git a/test/shell4.test b/test/shell4.test
index 193dc8b69b..3ced0702e4 100644
--- a/test/shell4.test
+++ b/test/shell4.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # The focus of this file is testing the CLI shell tool.
 # These tests are specific to the .stats command.
@@ -117,7 +118,7 @@ do_test shell4-2.2 {
 } {0 {}}
 do_test shell4-2.3 {
   catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n"
-} {/^0 {PRAGMA.*}$/}
+} {/^0 {SELECT.*}$/}
 do_test shell4-2.4 {
   catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"
 } {0 {CREATE TABLE t1(x);
@@ -139,7 +140,7 @@ do_test shell4-3.1 {
   close $fd
   exec $::CLI_ONLY :memory: --interactive ".read t1.txt"
 } {squirrel}
-do_test shell4-3.2 {
+do_test_with_ansi_output shell4-3.2 {
   set fd [open t1.txt wb]
   puts $fd "SELECT 'pound: \302\243';"
   close $fd
@@ -151,6 +152,6 @@ do_test shell4-4.1 {
   puts $fd ".read t1.txt"
   close $fd
   catchcmd ":memory:" ".read t1.txt"
-} {1 {Input nesting limit (25) reached at line 1. Check recursion.}}
+} {1 {t1.txt: Input nesting limit (25) reached at line 1. Check recursion.}}
 
 finish_test
diff --git a/test/shell5.test b/test/shell5.test
index 39018a0ce9..70a2298bcb 100644
--- a/test/shell5.test
+++ b/test/shell5.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # The focus of this file is testing the CLI shell tool.
 # These tests are specific to the .import command.
@@ -83,6 +84,14 @@ do_test shell5-1.4.1 {
 .import FOO t1}]
 } {1 {Error: cannot open "FOO"}}
 
+# the remainder of these test cases require virtual tables.
+#
+ifcapable !vtab {
+  puts "Skipping subsequent tests due to SQLITE_OMIT_VIRTUALTABLE"
+  finish_test
+  return
+}
+
 # empty import file
 do_test shell5-1.4.2 {
   forcedelete shell5.csv
@@ -409,7 +418,7 @@ CREATE TABLE t4(a, b);
 #
 do_test shell5-3.1 {
   set fd [open shell5.csv w]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   puts -nonewline $fd "\"test 1\"\x1F,test 2\r\n\x1E"
   puts -nonewline $fd "test 3\x1Ftest 4\n"
   close $fd
@@ -544,7 +553,7 @@ Columns renamed during .import shell5.csv due to duplicates:
 # Tests for preserving utf-8 that is not also ASCII.
 #
 
-do_test shell5-6.1 {
+do_test_with_ansi_output shell5-6.1 {
   set out [open shell5.csv w]
   fconfigure $out -translation lf
   puts $out {あい,うえお}
@@ -557,7 +566,7 @@ SELECT * FROM t1;}
 } {0 {   あい = 1
 うえお = 2}}
 
-do_test shell5-6.2 {
+do_test_with_ansi_output shell5-6.2 {
   set out [open shell5.csv w]
   fconfigure $out -translation lf
   puts $out {1,2}
@@ -570,4 +579,30 @@ SELECT * FROM t1;}
 } {0 {    1 = あい
     2 = うえお}}
 
+# 2024-03-11 https://sqlite.org/forum/forumpost/ca014d7358
+# Import into a table that contains computed columns.
+#
+do_test shell5-7.1 {
+  set out [open shell5.csv w]
+  fconfigure $out -translation lf
+  puts $out {aaa|bbb}
+  close $out
+  forcedelete test.db
+  catchcmd :memory: {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b));
+.import shell5.csv t1
+SELECT * FROM t1;}
+} {0 aaa|bbb|aaabbb}
+
+#-------------------------------------------------------------------------
+
+do_test shell5-8.1 {
+
+  set out [open shell5.csv w]
+  fconfigure $out -translation lf
+  puts $out x
+  close $out
+
+  catchcmd :memory: {.import --csv shell5.csv '""""""""""""""""""""""""""""""""""""""""""""""'}
+} {0 {}}
+
 finish_test
diff --git a/test/shell6.test b/test/shell6.test
index 49b4cc3344..4841d6c01a 100644
--- a/test/shell6.test
+++ b/test/shell6.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # Test the shell tool ".lint fkey-indexes" command.
 #
diff --git a/test/shell7.test b/test/shell7.test
index 898018d775..460789e544 100644
--- a/test/shell7.test
+++ b/test/shell7.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # Test the readfile() function built into the shell tool. Specifically,
 # that it does not truncate the blob read at the first embedded 0x00
@@ -32,7 +33,6 @@ do_execsql_test 1.0 {
 foreach {tn l x} [db eval { SELECT tn, length(x) AS l, x FROM f1 }] {
   forcedelete shell7_test.bin
   set fd [open shell7_test.bin w]
-  fconfigure $fd -encoding binary
   fconfigure $fd -translation binary
   puts -nonewline $fd $x
   close $fd
diff --git a/test/shell8.test b/test/shell8.test
index bee6039232..e555396365 100644
--- a/test/shell8.test
+++ b/test/shell8.test
@@ -8,6 +8,7 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
+# TESTRUNNER: shell
 #
 # Test the shell tool ".ar" command.
 #
@@ -54,7 +55,7 @@ proc dir_to_list {dirname {n -1}} {
   set res [list]
   foreach f [glob -nocomplain $dirname/*] {
     set mtime [file mtime $f]
-    if {$::tcl_platform(platform)!="windows"} {
+    if {$::tcl_platform(platform) ne "windows"} {
       set perm [file attributes $f -perm]
     } else {
       set perm 0
@@ -164,7 +165,7 @@ foreach {tn tcl} {
   #
   # Because it is slow, only do this for $tn==1.
   if {$tn==1} {
-    do_test 1.$tn.1 {
+    do_test 1.$tn.4 {
       catchcmd test_ar.db $c1
       file delete -force ar1
       after 2000
@@ -193,4 +194,29 @@ do_test 2.1.1 {
   regsub -all {ar4} [dir_content ar4] ar2
 } {ar2/file1 ar2/file2 ar2/junk1}
 
+# Test symbolic links.
+#
+if {$tcl_platform(platform)=="unix"} {
+  populate_dir ar2 {
+    file1 "1234" 
+    file2 "3456"
+  }
+  file link ar2/link1 file1
+
+  forcedelete shell8.db
+  forcedelete link1
+
+  do_test 3.1 {
+    catchcmd shell8.db {.ar -C ar2 -c file2 link1 }
+  } {0 {}}
+
+  do_test 3.2 {
+    catchcmd shell8.db {.ar -x} 
+  } {0 {}}
+
+  do_test 3.3 {
+    catchcmd shell8.db {.ar -x} 
+  } {0 {}}
+}
+
 finish_test
diff --git a/test/shell9.test b/test/shell9.test
new file mode 100644
index 0000000000..869cb075da
--- /dev/null
+++ b/test/shell9.test
@@ -0,0 +1,149 @@
+# 2024 Jan 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# TESTRUNNER: shell
+#
+# The focus of this file is testing the CLI shell tool. Specifically, 
+# testing that it is possible to run a ".dump" script that creates
+# virtual tables without explicitly disabling defensive mode.
+#
+# And, that it can process a ".dump" script that contains strings
+# delimited using double-quotes in the schema (DQS_DDL setting).
+#
+
+# Test plan:
+#
+#   shell1-1.*: Basic command line option handling.
+#   shell1-2.*: Basic "dot" command token parsing.
+#   shell1-3.*: Basic test that "dot" command can be called.
+#   shell1-{4-8}.*: Test various "dot" commands's functionality.
+#   shell1-9.*: Basic test that "dot" commands and SQL intermix ok.
+#
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set CLI [test_cli_invocation]
+
+set ::testprefix shell9
+
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+#----------------------------------------------------------------------------
+# Test cases shell9-1.* verify that scripts output by .dump may be parsed
+# by the shell tool without explicitly disabling DEFENSIVE mode, unless
+# the shell is in safe mode.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
+  INSERT INTO t1 VALUES('one', 'two', 'three');
+}
+db close
+
+# Create .dump file in "testdump.txt".
+#
+set out [open testdump.txt w]
+puts $out [lindex [catchcmd test.db .dump] 1]
+close $out
+
+# Check testdump.txt can be processed if the initial db is empty.
+#
+do_test 1.1.1 {
+  forcedelete test.db
+  catchcmd test.db ".read testdump.txt"
+} {0 {}}
+sqlite3 db test.db
+do_execsql_test 1.1.2 {
+  SELECT * FROM t1;
+} {one two three}
+
+# Check testdump.txt cannot be processed if the initial db is not empty.
+#
+reset_db
+do_execsql_test 1.2.1 {
+  CREATE TABLE t4(hello);
+}
+db close
+do_test 1.2.2 {
+  catchcmd test.db ".read testdump.txt"
+} {1 {Parse error near line 5: table sqlite_master may not be modified}}
+
+# Check testdump.txt cannot be processed if the db is in safe mode
+#
+do_test 1.3.1 {
+  forcedelete test.db
+  catchsafecmd test.db ".read testdump.txt"
+} {1 {line 1: cannot run .read in safe mode}}
+do_test 1.3.2 {
+  set fd [open testdump.txt]
+  set script [read $fd]
+  close $fd
+  forcedelete test.db
+  catchsafecmd test.db $script
+} {1 {Parse error near line 5: table sqlite_master may not be modified}}
+do_test 1.3.3 {
+  # Quick check that the above would have worked but for safe mode.
+  forcedelete test.db
+  catchcmd test.db $script
+} {0 {}}
+
+#----------------------------------------------------------------------------
+# Test cases shell9-2.* verify that a warning is printed at the top of
+# .dump scripts that contain virtual tables.
+#
+proc contains_warning {text} {
+  return [string match "*WARNING: Script requires that*" $text]
+}
+
+reset_db
+do_execsql_test 2.0.1 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y);
+  INSERT INTO t1 VALUES('one');
+  INSERT INTO t2 VALUES('two');
+}
+do_test 2.0.2 {
+  contains_warning [catchcmd test.db .dump]
+} 0
+
+do_execsql_test 2.1.1 {
+  CREATE virtual TABLE r1 USING fts5(x);
+}
+do_test 2.1.2 {
+  contains_warning [catchcmd test.db .dump]
+} 1
+
+do_test 2.2.1 {
+  contains_warning [catchcmd test.db ".dump t1"]
+} 0
+do_test 2.2.2 {
+  contains_warning [catchcmd test.db ".dump r1"]
+} 1
+
+#-------------------------------------------------------------------------
+reset_db
+sqlite3_db_config db DQS_DDL 1
+do_execsql_test 3.1.0 {
+  CREATE TABLE t4(hello, check( hello IS NOT "xyz") );
+}
+db close
+
+# Create .dump file in "testdump.txt".
+#
+set out [open testdump.txt w]
+puts $out [lindex [catchcmd test.db .dump] 1]
+close $out
+do_test 3.1.1 {
+  forcedelete test.db
+  catchcmd test.db ".read testdump.txt"
+} {0 {}}
+
+finish_test
diff --git a/test/shellA.test b/test/shellA.test
new file mode 100644
index 0000000000..f3959d4283
--- /dev/null
+++ b/test/shellA.test
@@ -0,0 +1,257 @@
+# 2025-02-24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# TESTRUNNER: shell
+#
+# Test cases for the command-line shell - focusing on .mode and
+# especially control-character escaping and the --escape option.
+#
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set CLI [test_cli_invocation]
+
+do_execsql_test shellA-1.0 {
+  CREATE TABLE t1(a INT, x TEXT);
+  INSERT INTO t1 VALUES
+   (1, 'line with '' single quote'),
+   (2, concat(char(0x1b),'[31mVT-100 codes',char(0x1b),'[0m')),
+   (3, NULL),
+   (4, 1234),
+   (5, 568.25),
+   (6, unistr('new\u000aline')),
+   (7, unistr('carriage\u000dreturn')),
+   (8, 'last line');
+} {}
+
+# Initial verification that the database created correctly
+# and that our calls to the CLI are working.
+#
+do_test_with_ansi_output shellA-1.2 {
+  exec {*}$CLI test.db {.mode box --escape symbol} {SELECT * FROM t1;}
+} {
+┌───┬──────────────────────────┐
+│ a │            x             │
+├───┼──────────────────────────┤
+│ 1 │ line with ' single quote │
+├───┼──────────────────────────┤
+│ 2 │ ␛[31mVT-100 codes␛[0m    │
+├───┼──────────────────────────┤
+│ 3 │                          │
+├───┼──────────────────────────┤
+│ 4 │ 1234                     │
+├───┼──────────────────────────┤
+│ 5 │ 568.25                   │
+├───┼──────────────────────────┤
+│ 6 │ new                      │
+│   │ line                     │
+├───┼──────────────────────────┤
+│ 7 │ carriage␍return          │
+├───┼──────────────────────────┤
+│ 8 │ last line                │
+└───┴──────────────────────────┘
+}
+
+# ".mode list"
+#
+do_test shellA-1.3 {
+  exec {*}$CLI test.db {SELECT x FROM t1 WHERE a=2;}
+} {
+^[[31mVT-100 codes^[[0m
+}
+do_test_with_ansi_output shellA-1.4 {
+  exec {*}$CLI test.db --escape symbol {SELECT x FROM t1 WHERE a=2;}
+} {
+␛[31mVT-100 codes␛[0m
+}
+do_test shellA-1.5 {
+  exec {*}$CLI test.db --escape ascii {SELECT x FROM t1 WHERE a=2;}
+} {
+^[[31mVT-100 codes^[[0m
+}
+do_test_with_ansi_output shellA-1.6 {
+  exec {*}$CLI test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;}
+} {
+␛[31mVT-100 codes␛[0m
+}
+do_test shellA-1.7 {
+  exec {*}$CLI test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;}
+} {
+^[[31mVT-100 codes^[[0m
+}
+do_test shellA-1.8 {
+  file delete -force out.txt
+  exec {*}$CLI test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \
+     >out.txt
+  set fd [open out.txt rb]
+  set res [read $fd]
+  close $fd
+  string trim $res
+} "carriage\rreturn"
+do_test shellA-1.9 {
+  set rc [catch {
+     exec {*}$CLI test.db {.mode test --escape xyz}
+  } msg]
+  lappend rc $msg
+} {1 {unknown control character escape mode "xyz" - choices: ascii symbol off}}
+do_test shellA-1.10 {
+  set rc [catch {
+     exec {*}$CLI --escape abc test.db .q
+  } msg]
+  lappend rc $msg
+} {1 {unknown control character escape mode "abc" - choices: ascii symbol off}}
+
+# ".mode quote"
+#
+do_test shellA-2.1 {
+ exec {*}$CLI test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+1,'line with '' single quote'
+2,unistr('\u001b[31mVT-100 codes\u001b[0m')
+6,'new
+line'
+7,unistr('carriage\u000dreturn')
+8,'last line'
+}
+do_test shellA-2.2 {
+  exec {*}$CLI test.db --quote {.mode}
+} {current output mode: quote --escape ascii}
+do_test shellA-2.3 {
+  exec {*}$CLI test.db --quote --escape SYMBOL {.mode}
+} {current output mode: quote --escape symbol}
+do_test shellA-2.4 {
+  exec {*}$CLI test.db --quote --escape OFF {.mode}
+} {current output mode: quote --escape off}
+
+
+# ".mode line"
+#
+do_test_with_ansi_output shellA-3.1 {
+ exec {*}$CLI test.db --line --escape symbol \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+    a = 1
+    x = line with ' single quote
+
+    a = 2
+    x = ␛[31mVT-100 codes␛[0m
+
+    a = 6
+    x = new
+line
+
+    a = 7
+    x = carriage␍return
+
+    a = 8
+    x = last line
+}
+do_test shellA-3.2 {
+ exec {*}$CLI test.db --line --escape ascii \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+    a = 1
+    x = line with ' single quote
+
+    a = 2
+    x = ^[[31mVT-100 codes^[[0m
+
+    a = 6
+    x = new
+line
+
+    a = 7
+    x = carriage^Mreturn
+
+    a = 8
+    x = last line
+}
+
+# ".mode box"
+#
+do_test_with_ansi_output shellA-4.1 {
+ exec {*}$CLI test.db --box --escape ascii \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+┌───┬──────────────────────────┐
+│ a │            x             │
+├───┼──────────────────────────┤
+│ 1 │ line with ' single quote │
+├───┼──────────────────────────┤
+│ 2 │ ^[[31mVT-100 codes^[[0m  │
+├───┼──────────────────────────┤
+│ 6 │ new                      │
+│   │ line                     │
+├───┼──────────────────────────┤
+│ 7 │ carriage^Mreturn         │
+├───┼──────────────────────────┤
+│ 8 │ last line                │
+└───┴──────────────────────────┘
+}
+do_test_with_ansi_output shellA-4.2 {
+ exec {*}$CLI test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+┌───┬───────────────────────────────────────────┐
+│ a │                     x                     │
+├───┼───────────────────────────────────────────┤
+│ 1 │ 'line with '' single quote'               │
+├───┼───────────────────────────────────────────┤
+│ 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') │
+├───┼───────────────────────────────────────────┤
+│ 6 │ 'new                                      │
+│   │ line'                                     │
+├───┼───────────────────────────────────────────┤
+│ 7 │ unistr('carriage\u000dreturn')            │
+├───┼───────────────────────────────────────────┤
+│ 8 │ 'last line'                               │
+└───┴───────────────────────────────────────────┘
+}
+
+# ".mode insert"
+#
+do_test shellA-5.1 {
+ exec {*}$CLI test.db {.mode insert t1 --escape ascii} \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+INSERT INTO t1 VALUES(1,'line with '' single quote');
+INSERT INTO t1 VALUES(2,unistr('\u001b[31mVT-100 codes\u001b[0m'));
+INSERT INTO t1 VALUES(6,unistr('new\u000aline'));
+INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn'));
+INSERT INTO t1 VALUES(8,'last line');
+}
+do_test shellA-5.2 {
+ exec {*}$CLI test.db {.mode insert t1 --escape symbol} \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)}
+} {
+INSERT INTO t1 VALUES(1,'line with '' single quote');
+INSERT INTO t1 VALUES(2,unistr('\u001b[31mVT-100 codes\u001b[0m'));
+INSERT INTO t1 VALUES(6,unistr('new\u000aline'));
+INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn'));
+INSERT INTO t1 VALUES(8,'last line');
+}
+do_test shellA-5.3 {
+  file delete -force out.txt
+  exec {*}$CLI test.db {.mode insert t1 --escape off} \
+    {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} >out.txt
+  set fd [open out.txt rb]
+  set res [read $fd]
+  close $fd
+  string trim [string map [list \r\n \n] $res]
+} "
+INSERT INTO t1 VALUES(1,'line with '' single quote');
+INSERT INTO t1 VALUES(2,'\033\13331mVT-100 codes\033\1330m');
+INSERT INTO t1 VALUES(6,'new
+line');
+INSERT INTO t1 VALUES(7,'carriage\rreturn');
+INSERT INTO t1 VALUES(8,'last line');
+"
+
+finish_test
diff --git a/test/shmlock.test b/test/shmlock.test
index 89b29fd7ac..fce0cf8f5e 100644
--- a/test/shmlock.test
+++ b/test/shmlock.test
@@ -114,7 +114,7 @@ sqlite3 db0 test.db
 sqlite3 db1 test.db
 do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2}
 do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2}
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(os) eq "Windows NT"} {
   set isWindows 1
 } else {
   set isWindows 0
diff --git a/test/skipscan1.test b/test/skipscan1.test
index bd5b83d34c..379de1956c 100644
--- a/test/skipscan1.test
+++ b/test/skipscan1.test
@@ -337,12 +337,12 @@ do_execsql_test skipscan1-9.2 {
 } {/USING INDEX t9a_ab .ANY.a. AND b=./}
 
 
-optimization_control db skip-scan 0
+optimization_control db skip-scan off
 do_execsql_test skipscan1-9.3 {
   EXPLAIN QUERY PLAN
   SELECT  * FROM t9a WHERE b IN (SELECT x FROM t9b WHERE y!=5);
 } {/{SCAN t9a}/}
-optimization_control db skip-scan 1
+optimization_control db all on
 
 do_execsql_test skipscan1-2.1 {
   CREATE TABLE t6(a TEXT, b INT, c INT, d INT);
diff --git a/test/skipscan6.test b/test/skipscan6.test
index 4f592bc0e0..a97f440eef 100644
--- a/test/skipscan6.test
+++ b/test/skipscan6.test
@@ -167,10 +167,10 @@ do_execsql_test 3.0 {
   INSERT INTO t2 SELECT * FROM t3;
 
   ANALYZE;
-  SELECT * FROM sqlite_stat1;
+  SELECT * FROM sqlite_stat1 ORDER BY +idx;
 } {
-  t2 t2_ba   {100 20 1 1}
   t2 t2_a    {100 1} 
+  t2 t2_ba   {100 20 1 1}
   t3 t3_a    {100 1} 
   t3 t3_ba   {100 20 1 1}
 }
diff --git a/test/snapshot3.test b/test/snapshot3.test
index 4dca202b1c..6d57b1d0c4 100644
--- a/test/snapshot3.test
+++ b/test/snapshot3.test
@@ -96,4 +96,46 @@ do_test 1.8 {
   list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg
 } {1 SQLITE_ERROR_SNAPSHOT}
 
+db3 close
+db2 close
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+} {wal}
+
+sqlite3 db2 test.db
+sqlite3 db3 test.db
+do_execsql_test -db db2 2.0.1 {
+  SELECT * FROM t1
+} {1 2 3 4}
+do_execsql_test -db db3 2.0.2 {
+  SELECT * FROM t1
+} {1 2 3 4}
+
+do_execsql_test -db db2 2.2 {
+  PRAGMA wal_checkpoint;
+} {0 4 4}
+
+do_test 2.1 {
+  db eval { BEGIN }
+  set snap [sqlite3_snapshot_get db main]
+  set {} {}
+} {}
+
+do_execsql_test -db db2 2.3 {
+  INSERT INTO t1 VALUES(5, 6);
+} {}
+
+do_test 2.2 {
+  execsql { BEGIN } db3
+  sqlite3_snapshot_open db3 main $snap
+} {}
+
+sqlite3_snapshot_free $snap
+
 finish_test
diff --git a/test/snapshot_fault.test b/test/snapshot_fault.test
index 2adb793650..10c5094594 100644
--- a/test/snapshot_fault.test
+++ b/test/snapshot_fault.test
@@ -23,6 +23,7 @@ set testprefix snapshot_fault
 # checkpointing the db.
 #
 do_faultsim_test 1.0 -prep {
+  catch { db2 close }
   faultsim_delete_and_reopen
   sqlite3 db2 test.db
   db2 eval { 
diff --git a/test/snapshot_up.test b/test/snapshot_up.test
index de8e5afab4..5f389e7be5 100644
--- a/test/snapshot_up.test
+++ b/test/snapshot_up.test
@@ -27,6 +27,8 @@ if {[permutation]=="inmemory_journal"} {
   return
 }
 
+db timeout 1000
+
 do_execsql_test 1.0 {
   CREATE TABLE t1(a, b, c);
   PRAGMA journal_mode = wal;
diff --git a/test/sort2.test b/test/sort2.test
index f686654d53..9fe4b4ccf3 100644
--- a/test/sort2.test
+++ b/test/sort2.test
@@ -69,7 +69,7 @@ foreach {tn script} {
   # Because it uses so much data, this test can take 12-13 seconds even on
   # a modern workstation. So it is omitted from "veryquick" and other
   # permutations.test tests.
-  if {[isquick]==0} {
+  if {[isquick]==0 && [clang_sanitize_address]==0} {
     do_execsql_test $tn.3 {
       PRAGMA cache_size = 5;
       WITH r(x,y) AS (
diff --git a/test/sort4.test b/test/sort4.test
index 84125885ab..17aa28baec 100644
--- a/test/sort4.test
+++ b/test/sort4.test
@@ -26,20 +26,22 @@ sqlite3_initialize
 sqlite3 db test.db
 
 
-# Configure the sorter to use 3 background threads.
-#
-# EVIDENCE-OF: R-19249-32353 SQLITE_LIMIT_WORKER_THREADS The maximum
-# number of auxiliary worker threads that a single prepared statement
-# may start.
-#
-do_test sort4-init001 {
-  db eval {PRAGMA threads=5}
-  sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS -1
-} {5}
-do_test sort4-init002 {
-  sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS 3
-  db eval {PRAGMA threads}
-} {3}
+if {![string match *MAX_WORKER_THREADS=0* [db eval {PRAGMA compile_options}]]} {
+  # Configure the sorter to use 3 background threads.
+  #
+  # EVIDENCE-OF: R-19249-32353 SQLITE_LIMIT_WORKER_THREADS The maximum
+  # number of auxiliary worker threads that a single prepared statement
+  # may start.
+  #
+  do_test sort4-init001 {
+    db eval {PRAGMA threads=5}
+    sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS -1
+  } {5}
+  do_test sort4-init002 {
+    sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS 3
+    db eval {PRAGMA threads}
+  } {3}
+}
 
 
 # Minimum number of seconds to run for. If the value is 0, each test
diff --git a/test/speedtest.md b/test/speedtest.md
new file mode 100644
index 0000000000..135e562aed
--- /dev/null
+++ b/test/speedtest.md
@@ -0,0 +1,53 @@
+# Performance And Size Measurements
+
+This document shows a procedure for making performance and size
+comparisons between two versions of the SQLite Amalgamation "sqlite3.c".
+You will need:
+
+  *  fossil
+  *  valgrind
+  *  tclsh
+  *  A script or program named "open" that brings up *.txt files in an
+     editor for viewing.  (Macs provide this by default.  You'll need to
+     come up with your own on Linux and Windows.)
+  *  An SQLite source tree
+
+The procedure described in this document is not the only way to make
+performance and size measurements.  Use this as a guide and make
+adjustments as needed.
+
+## Establish the baseline measurement
+
+  *  Begin at the root the SQLite source tree
+  *  <b>mkdir -p ../speed</b> <br>
+      &uarr;  Speed measurement output files will go into this directory.
+     You can actually put those files wherever you want.  This is just a
+     suggestion.  It might be good to keep these files outside of the
+     source tree so that "fossil clean" does not delete them.
+  *  Obtain the baseline SQLite amalgamation.  For the purpose of this
+     technical note, assume the baseline SQLite sources are in files
+     "../baseline/sqlite3.c" and "../baseline/sqlite3.h".
+  *  <b>test/speedtest.tcl ../baseline/sqlite3.c ../speed/baseline.txt</b> <br>
+     &uarr; The performance measure will be written into ../speed/baseline.txt
+     and that file will be brought up in an editor for easy viewing. <br>
+     &uarr; The "sqlite3.h" will be taken from the directory that contains
+     the "sqlite3.c" amalgamation file.
+
+## Comparing the current checkout against the baseline
+
+  *  <b>make sqlite3.c</b>
+  *  <b>test/speedtest.tcl sqlite3.c ../speed/test.txt ../speed/baseline.txt</b> <br>
+     &uarr; Test results written into ../speed/test.txt and then
+     "fossil xdiff" is run to compare ../speed/baseline.txt against
+     the new test results.
+
+## When to do this
+
+Performance and size checks should be done prior to trunk check-ins.
+Sometimes a seemingly innocuous change can have large performance
+impacts.  A large impact does not mean that the change cannot continue,
+but it is important to be aware of the impact.
+
+## Additional hints
+
+Use the --help option to test/speedtest.tcl to see other available options.
diff --git a/test/speedtest.tcl b/test/speedtest.tcl
new file mode 100755
index 0000000000..9cb81c0fcd
--- /dev/null
+++ b/test/speedtest.tcl
@@ -0,0 +1,313 @@
+#!/bin/sh
+# the next line restarts using tclsh \
+exec tclsh "$0" ${1+"$@"}
+#
+# This program runs performance testing on sqlite3.c.  Usage:
+set usage {USAGE:
+
+  speedtest.tcl  sqlite3.c  x1.txt  trunk.txt  -Os -DSQLITE_ENABLE_STAT4
+                     |         |        |       `-----------------------'
+    File to test ----'         |        |                |
+                               |        |                `- options
+       Output filename --------'        |
+                                        `--- optional prior output to diff
+
+Do a cache-grind performance analysis of the sqlite3.c file named and
+write the results into the output file.  The ".txt" is appended to the
+output file (and diff-file) name if it is not already present.  If the
+diff-file is specified then show a diff from the diff-file to the new
+output.
+
+Other options include:
+   CC=...                Specify an alternative C compiler.  Default is "gcc".
+   -D...                 -D and -O options are passed through to the C compiler.
+   --dryrun              Show what would happen but don't do anything.
+   --help                Show this help screen.
+   --lean                "Lean" mode.
+   --lookaside N SZ      Lookahead uses N slots of SZ bytes each.
+   --osmalloc            Use the OS native malloc() instead of MEMSYS5
+   --pagesize N          Use N as the page size.
+   --quiet | -q          "Quite".  Put results in file but don't pop up editor
+   --size N              Change the test size.  100 means 100%.  Default: 5.
+   --testset TEST        Specify the specific testset to use.  The default
+                         is "mix1".  Other options include: "main", "json",
+                         "cte", "orm", "fp", "rtree".
+}
+set srcfile {}
+set outfile {}
+set difffile {}
+set cflags {-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_THREADSAFE=0}
+set cc gcc
+set testset mix1
+set dryrun 0
+set quiet 0
+set osmalloc 0
+set speedtestflags {--shrink-memory --reprepare --stats}
+lappend speedtestflags --journal wal --size 5
+
+for {set i 0} {$i<[llength $argv]} {incr i} {
+  set arg [lindex $argv $i]
+  if {[string index $arg 0]=="-"} {
+    switch -- $arg {
+      -pagesize -
+      --pagesize {
+        lappend speedtestflags --pagesize
+        incr i
+        lappend speedtestflags [lindex $argv $i]
+      }
+      -lookaside -
+      --lookaside {
+        lappend speedtestflags --lookaside
+        incr i
+        lappend speedtestflags [lindex $argv $i]
+        incr i
+        lappend speedtestflags [lindex $argv $i]
+      }
+      -lean -
+      --lean {
+        lappend cflags \
+           -DSQLITE_DEFAULT_MEMSTATUS=0 \
+           -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 \
+           -DSQLITE_LIKE_DOESNT_MATCH_BLOBS=1 \
+           -DSQLITE_MAX_EXPR_DEPTH=1 \
+           -DSQLITE_OMIT_DECLTYPE \
+           -DSQLITE_OMIT_DEPRECATED \
+           -DSQLITE_OMIT_PROGRESS_CALLBACK \
+           -DSQLITE_OMIT_SHARED_CACHE \
+           -DSQLITE_USE_ALLOCA
+      }
+      -testset -
+      --testset {
+        incr i
+        set testset [lindex $argv $i]
+      }
+      -size -
+      --size {
+        incr i
+        set newsize [lindex $argv $i]
+        if {$newsize<1} {set newsize 1}
+        set speedtestflags \
+          [regsub {.-size \d+} $speedtestflags "-size $newsize"]
+      }
+      -n -
+      -dryrun -
+      --dryrun {
+        set dryrun 1
+      }
+      -osmalloc -
+      --osmalloc {
+        set osmalloc 1
+      }
+      -? -
+      -help -
+      --help {
+        puts $usage
+        exit 0
+      }
+      -q -
+      -quiet -
+      --quiet {
+        set quiet 1
+      }
+      default {
+        lappend cflags $arg
+      }
+    }
+    continue
+  }
+  if {[string match CC=* $arg]} {
+    set cc [string range $arg 3 end]
+    continue
+  }
+  if {[string match *.c $arg]} {
+    if {$srcfile!=""} {
+      puts stderr "multiple source files: $srcfile $arg"
+      exit 1
+    }
+    set srcfile $arg
+    continue
+  }
+  if {[lsearch {main cte rtree orm fp json parsenumber mix1} $arg]>=0} {
+    set testset $arg
+    continue
+  }
+  if {$outfile==""} {
+    set outfile $arg
+    continue
+  }
+  if {$difffile==""} {
+    set difffile $arg
+    continue
+  }
+  puts stderr "unknown option: \"$arg\".  Use --help for more info."
+  exit 1
+}
+if {[lsearch -glob $cflags -O*]<0} {
+  lappend cflags -Os
+}
+if {!$osmalloc} {
+  append speedtestflags { --heap 40000000 64}
+}
+if {!$osmalloc && [lsearch -glob $cflags {-DSQLITE_ENABLE_MEMSYS*}]<0} {
+  lappend cflags -DSQLITE_ENABLE_MEMSYS5
+}
+if {[lsearch -glob $cflags {-DSQLITE_ENABLE_RTREE*}]<0} {
+  lappend cflags -DSQLITE_ENABLE_RTREE
+}
+if {$srcfile==""} {
+  puts stderr "no sqlite3.c source file specified"
+  exit 1
+}
+if {![file readable $srcfile]} {
+  puts stderr "source file \"$srcfile\" does not exist"
+  exit 1
+}
+if {$outfile==""} {
+  puts stderr "no output file specified"
+  exit 1
+}
+if {![string match *.* [file tail $outfile]]} {
+  append outfile .txt
+}
+if {$difffile!=""} {
+  if {![file exists $difffile]} {
+    if {[file exists $difffile.txt]} {
+      append difffile .txt
+    } else {
+      puts stderr "No such file: \"$difffile\""
+      exit 1
+    }
+  }
+}
+
+set cccmd [list $cc -g]
+lappend cccmd -I[file dir $srcfile]
+lappend cccmd {*}[lsort $cflags]
+lappend cccmd [file dir $argv0]/speedtest1.c
+lappend cccmd $srcfile
+lappend cccmd -o speedtest1
+puts $cccmd
+if {!$dryrun} {
+  exec {*}$cccmd
+}
+lappend speedtestflags --testset $testset
+set stcmd [list valgrind --tool=cachegrind ./speedtest1 {*}$speedtestflags]
+lappend stcmd speedtest1.db
+lappend stcmd >valgrind-out.txt 2>valgrind-err.txt
+puts $stcmd
+if {!$dryrun} {
+   foreach file {speedtest1.db speedtest1.db-journal speedtest1.db-wal
+                 speedtest1.db-shm} {
+     if {[file exists $file]} {file delete $file}
+   }
+   exec {*}$stcmd
+}
+
+set maxmtime 0
+set cgfile {}
+foreach cgout [glob -nocomplain cachegrind.out.*] {
+  if {[file mtime $cgout]>$maxmtime} {
+    set cgfile $cgout
+    set maxmtime [file mtime $cgfile]
+  }
+}
+if {$cgfile==""} {
+  puts "no cachegrind output"
+  exit 1
+}
+
+############# Process the cachegrind.out.# file ##########################
+set fd [open $outfile wb]
+set in [open "|cg_annotate --show=Ir --auto=yes --context=40 $cgfile" r]
+set dest !
+set out(!) {}
+set linenum 0
+set cntlines 0      ;# true to remember cycle counts on each line
+set seenSqlite3 0   ;# true if we have seen the sqlite3.c file
+while {![eof $in]} {
+  set line [string map {\t {        }} [gets $in]]
+  if {[regexp {^-- Auto-annotated source: (.*)} $line all name]} {
+    set dest $name
+    if {[string match */sqlite3.c $dest]} {
+      set cntlines 1
+      set seenSqlite3 1
+    } else {
+      set cntlines 0
+    }
+  } elseif {[regexp {^-- line (\d+) ------} $line all ln]} {
+    set line [lreplace $line 2 2 {#}]
+    set linenum [expr {$ln-1}]
+  } elseif {[regexp {^The following files chosen for } $line]} {
+    set dest !
+  }
+  append out($dest) $line\n
+  if {$cntlines} {
+    incr linenum
+    if {[regexp {^ *([0-9,]+) } $line all x]} {
+      set x [string map {, {}} $x]
+      set cycles($linenum) $x
+    }
+  }
+}
+foreach x [lsort [array names out]] {
+  puts $fd $out($x)
+}
+# If the sqlite3.c file has been seen, then output a summary of the
+# cycle counts for each file that went into making up sqlite3.c
+#
+if {$seenSqlite3} {
+  close $in
+  set in [open sqlite3.c]
+  set linenum 0
+  set fn sqlite3.c
+  set pattern1 {^/\*+ Begin file ([^ ]+) \*}
+  set pattern2 {^/\*+ Continuing where we left off in ([^ ]+) \*}
+  while {![eof $in]} {
+    set line [gets $in]
+    incr linenum
+    if {[regexp $pattern1 $line all newfn]} {
+      set fn $newfn
+    } elseif {[regexp $pattern2 $line all newfn]} {
+      set fn $newfn
+    } elseif {[info exists cycles($linenum)]} {
+      incr fcycles($fn) $cycles($linenum)
+    }
+  }
+  close $in
+  puts $fd \
+     {**********************************************************************}
+  set lx {}
+  set sum 0
+  foreach {fn cnt} [array get fcycles] {
+    lappend lx [list $cnt $fn]
+    incr sum $cnt
+  }
+  puts $fd [format {%20s %14d  %8.3f%%} TOTAL $sum 100]
+  foreach entry [lsort -index 0 -integer -decreasing $lx] {
+    foreach {cnt fn} $entry break
+    puts $fd [format {%20s %14d  %8.3f%%} $fn $cnt [expr {$cnt*100.0/$sum}]]
+  }
+}
+puts $fd "Executable size:"
+close $fd
+exec size speedtest1 >>$outfile
+#
+#  Processed cachegrind output should now be in the $outfile
+#############################################################################
+
+if {$quiet} {
+  # Skip this last part of popping up a GUI viewer
+} elseif {$difffile!=""} {
+  set fossilcmd {fossil xdiff --tk -c 20}
+  lappend fossilcmd $difffile
+  lappend fossilcmd $outfile
+  lappend fossilcmd &
+  puts $fossilcmd
+  if {!$dryrun} {
+    exec {*}$fossilcmd
+  }
+} else {
+  if {!$dryrun} {
+    exec open $outfile
+  }
+}
diff --git a/test/speedtest1.c b/test/speedtest1.c
index 6aff89b376..a127f62e9f 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -1,6 +1,28 @@
 /*
 ** A program for performance testing.
 **
+** To build this program against an historical version of SQLite for comparison
+** testing:
+**
+**    Unix:
+**
+**        ./configure --all
+**        make clean speedtest1
+**        mv speedtest1 speedtest1-current
+**        cp $HISTORICAL_SQLITE3_C_H .
+**        touch sqlite3.c sqlite3.h .target_source
+**        make speedtest1
+**        mv speedtest1 speedtest1-baseline
+**
+**    Windows:
+**
+**        nmake /f Makefile.msc clean speedtest1.exe
+**        mv speedtest1.exe speedtest1-current.exe
+**        cp $HISTORICAL_SQLITE_C_H .
+**        touch sqlite3.c sqlite3.h .target_source
+**        nmake /f Makefile.msc speedtest1.exe
+**        mv speedtest1.exe speedtest1-baseline.exe
+**
 ** The available command-line options are described below:
 */
 static const char zHelp[] =
@@ -13,6 +35,7 @@ static const char zHelp[] =
   "  --exclusive         Enable locking_mode=EXCLUSIVE\n"
   "  --explain           Like --sqlonly but with added EXPLAIN keywords\n"
   "  --fullfsync         Enable fullfsync=TRUE\n"
+  "  --hard-heap-limit N The hard limit on the maximum heap size\n"
   "  --heap SZ MIN       Memory allocator uses SZ bytes & min allocation MIN\n"
   "  --incrvacuum        Enable incremenatal vacuum mode\n"
   "  --journal M         Set the journal_mode to M\n"
@@ -38,11 +61,14 @@ static const char zHelp[] =
   "  --sqlonly           No-op.  Only show the SQL that would have been run.\n"
   "  --shrink-memory     Invoke sqlite3_db_release_memory() frequently.\n"
   "  --size N            Relative test size.  Default=100\n"
+  "  --soft-heap-limit N The soft limit on the maximum heap size\n"
   "  --strict            Use STRICT table where appropriate\n"
   "  --stats             Show statistics at the end\n"
   "  --stmtscanstatus    Activate SQLITE_DBCONFIG_STMT_SCANSTATUS\n"
   "  --temp N            N from 0 to 9.  0: no temp table. 9: all temp tables\n"
-  "  --testset T         Run test-set T (main, cte, rtree, orm, fp, debug)\n"
+  "  --testset T         Run test-set T (main, cte, rtree, orm, fp, json,\n"
+  "                      star, app, debug).  Can be a comma-separated list\n"
+  "                      of values, with /SCALE suffixes or macro \"mix1\"\n"
   "  --trace             Turn on SQL tracing\n"
   "  --threads N         Use up to N threads for sorting\n"
   "  --utf16be           Set text encoding to UTF-16BE\n"
@@ -88,6 +114,8 @@ struct HashContext {
 /* All global state is held in this structure */
 static struct Global {
   sqlite3 *db;               /* The open database connection */
+  const char *zDbName;       /* Name of the database file */
+  const char *zVfs;          /* --vfs NAME */
   sqlite3_stmt *pStmt;       /* Current SQL statement */
   sqlite3_int64 iStart;      /* Start-time for the current test */
   sqlite3_int64 iTotal;      /* Total time */
@@ -99,6 +127,7 @@ static struct Global {
   int bMemShrink;            /* Call sqlite3_db_release_memory() often */
   int eTemp;                 /* 0: no TEMP.  9: always TEMP. */
   int szTest;                /* Scale factor for test iterations */
+  int szBase;                /* Base size prior to testset scaling */
   int nRepeat;               /* Repeat selects this many times */
   int doCheckpoint;          /* Run PRAGMA wal_checkpoint after each trans */
   int nReserve;              /* Reserve bytes */
@@ -130,6 +159,12 @@ static void fatal_error(const char *zMsg, ...){
   va_start(ap, zMsg);
   vfprintf(stderr, zMsg, ap);
   va_end(ap);
+#ifdef SQLITE_SPEEDTEST1_WASM
+  /* Emscripten complains when exit() is called and anything is left
+     in the I/O buffers. */
+  fflush(stdout);
+  fflush(stderr);
+#endif
   exit(1);
 }
 
@@ -515,6 +550,7 @@ char *speedtest1_once(const char *zFormat, ...){
   char *zSql;
   sqlite3_stmt *pStmt;
   char *zResult = 0;
+  int rc;
   va_start(ap, zFormat);
   zSql = sqlite3_vmprintf(zFormat, ap);
   va_end(ap);
@@ -534,6 +570,11 @@ char *speedtest1_once(const char *zFormat, ...){
       const char *z = (const char*)sqlite3_column_text(pStmt, 0);
       if( z ) zResult = sqlite3_mprintf("%s", z);
     }
+    rc = sqlite3_reset(pStmt);
+    if( rc!=SQLITE_OK ){
+      fatal_error("%s\nError code %d: %s\n",
+                  sqlite3_sql(pStmt), rc, sqlite3_errmsg(g.db));
+    }
     sqlite3_finalize(pStmt);
   }
   sqlite3_free(zSql);
@@ -563,7 +604,7 @@ void speedtest1_prepare(const char *zFormat, ...){
 
 /* Run an SQL statement previously prepared */
 void speedtest1_run(void){
-  int i, n, len;
+  int i, n, len, rc;
   if( g.bSqlOnly ) return;
   assert( g.pStmt );
   g.nResult = 0;
@@ -622,12 +663,20 @@ void speedtest1_run(void){
   if( g.bReprepare ){
     sqlite3_stmt *pNew;
     sqlite3_prepare_v2(g.db, sqlite3_sql(g.pStmt), -1, &pNew, 0);
-    sqlite3_finalize(g.pStmt);
+    rc = sqlite3_finalize(g.pStmt);
+    if( rc!=SQLITE_OK ){
+      fatal_error("%s\nError code %d: %s\n",
+                  sqlite3_sql(pNew), rc, sqlite3_errmsg(g.db));
+    }
     g.pStmt = pNew;
   }else
 #endif
   {
-    sqlite3_reset(g.pStmt);
+    rc = sqlite3_reset(g.pStmt);
+    if( rc!=SQLITE_OK ){
+      fatal_error("%s\nError code %d: %s\n",
+                  sqlite3_sql(g.pStmt), rc, sqlite3_errmsg(g.db));
+    }
   }
   speedtest1_shrink_memory();
 }
@@ -1432,6 +1481,561 @@ void testset_fp(void){
   speedtest1_end_test();
 }
 
+/*
+** A testset for star-schema queries.
+*/
+void testset_star(void){
+  int n;
+  int i;
+  n = g.szTest*50;
+  speedtest1_begin_test(100, "Create a fact table with %d entries", n);
+  speedtest1_exec(
+    "CREATE TABLE facttab("
+     " attr01 INT,"
+     " attr02 INT,"
+     " attr03 INT,"
+     " data01 TEXT,"
+     " attr04 INT,"
+     " attr05 INT,"
+     " attr06 INT,"
+     " attr07 INT,"
+     " attr08 INT,"
+     " factid INTEGER PRIMARY KEY,"
+     " data02 TEXT"
+    ");"
+  );
+  speedtest1_exec(
+    "WITH RECURSIVE counter(nnn) AS"
+       "(VALUES(1) UNION ALL SELECT nnn+1 FROM counter WHERE nnn<%d)"
+    "INSERT INTO facttab(attr01,attr02,attr03,attr04,attr05,"
+                        "attr06,attr07,attr08,data01,data02)"
+    "SELECT random()%%12, random()%%13, random()%%14, random()%%15,"
+           "random()%%16, random()%%17, random()%%18, random()%%19,"
+           "concat('data-',nnn), format('%%x',random()) FROM counter;",
+    n
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(110, "Create indexes on all attributes columns");
+  for(i=1; i<=8; i++){
+    speedtest1_exec(
+      "CREATE INDEX fact_attr%02d ON facttab(attr%02d)", i, i
+    );
+  }
+  speedtest1_end_test();
+
+  speedtest1_begin_test(120, "Create dimension tables");
+  for(i=1; i<=8; i++){
+    speedtest1_exec(
+      "CREATE TABLE dimension%02d("
+        "beta%02d INT, "
+        "content%02d TEXT, "
+        "rate%02d REAL)",
+      i, i, i, i
+    );
+    speedtest1_exec(
+      "WITH RECURSIVE ctr(nn) AS"
+      " (VALUES(1) UNION ALL SELECT nn+1 FROM ctr WHERE nn<%d)"
+      " INSERT INTO dimension%02d"
+      "   SELECT nn%%(%d), concat('content-%02d-',nn),"
+               " (random()%%10000)*0.125 FROM ctr;",
+      4*(i+1), i, 2*(i+1), i
+    );
+    if( i&2 ){
+      speedtest1_exec(
+         "CREATE INDEX dim%02d ON dimension%02d(beta%02d);",
+         i, i, i
+      );
+    }else{
+      speedtest1_exec(
+         "CREATE INDEX dim%02d ON dimension%02d(beta%02d,content%02d);",
+         i, i, i, i
+      );
+    }
+  }
+  speedtest1_end_test();
+
+  speedtest1_begin_test(130, "Star query over the entire fact table");
+  speedtest1_exec(
+    "SELECT count(*), max(content04), min(content03), sum(rate04), avg(rate05)"
+    " FROM facttab, dimension01, dimension02, dimension03, dimension04,"
+                  " dimension05, dimension06, dimension07, dimension08"
+    " WHERE attr01=beta01"
+      " AND attr02=beta02"
+      " AND attr03=beta03"
+      " AND attr04=beta04"
+      " AND attr05=beta05"
+      " AND attr06=beta06"
+      " AND attr07=beta07"
+      " AND attr08=beta08"
+    ";"
+   );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(130, "Star query with LEFT JOINs");
+  speedtest1_exec(
+    "SELECT count(*), max(content04), min(content03), sum(rate04), avg(rate05)"
+    " FROM facttab LEFT JOIN dimension01 ON attr01=beta01"
+                 " LEFT JOIN dimension02 ON attr02=beta02"
+                 " JOIN dimension03 ON attr03=beta03"
+                 " JOIN dimension04 ON attr04=beta04"
+                 " JOIN dimension05 ON attr05=beta05"
+                 " LEFT JOIN dimension06 ON attr06=beta06"
+                 " JOIN dimension07 ON attr07=beta07"
+                 " JOIN dimension08 ON attr08=beta08"
+    " WHERE facttab.data01 LIKE 'data-9%%'"
+    ";"
+   );
+  speedtest1_end_test();
+}
+
+/*
+** Tests that simulate an application opening and closing an SQLite database
+** frequently.  Fossil is used as the model.  The focus here is on rapidly
+** parsing the database schema and rapidly generating prepared statements,
+** in other words, rapid start-up of Fossil-like applications.
+**
+** The same database has no data, so the performance of sqlite3_step() is
+** not significant to this testset.
+*/
+static void testset_app(void){
+  int i, n;
+  speedtest1_begin_test(100, "Generate a Fossil-like database schema");
+  speedtest1_exec(
+    "BEGIN;"
+    "CREATE TABLE blob(\n"
+    "  rid INTEGER PRIMARY KEY,\n"
+    "  rcvid INTEGER,\n"
+    "  size INTEGER,\n"
+    "  uuid TEXT UNIQUE NOT NULL,\n"
+    "  content BLOB,\n"
+    "  CHECK( length(uuid)>=40 AND rid>0 )\n"
+    ");\n"
+    "CREATE TABLE delta(\n"
+    "  rid INTEGER PRIMARY KEY,\n"
+    "  srcid INTEGER NOT NULL REFERENCES blob\n"
+    ");\n"
+    "CREATE TABLE rcvfrom(\n"
+    "  rcvid INTEGER PRIMARY KEY,\n"
+    "  uid INTEGER REFERENCES user,\n"
+    "  mtime DATETIME,\n"
+    "  nonce TEXT UNIQUE,\n"
+    "  ipaddr TEXT\n"
+    ");\n"
+    "CREATE TABLE private(rid INTEGER PRIMARY KEY);\n"
+    "CREATE TABLE accesslog(\n"
+    "  uname TEXT,\n"
+    "  ipaddr TEXT,\n"
+    "  success BOOLEAN,\n"
+    "  mtime TIMESTAMP\n"
+    ");\n"
+    "CREATE TABLE user(\n"
+    "  uid INTEGER PRIMARY KEY,\n"
+    "  login TEXT UNIQUE,\n"
+    "  pw TEXT,\n"
+    "  cap TEXT,\n"
+    "  cookie TEXT,\n"
+    "  ipaddr TEXT,\n"
+    "  cexpire DATETIME,\n"
+    "  info TEXT,\n"
+    "  mtime DATE,\n"
+    "  photo BLOB\n"
+    ", jx TEXT DEFAULT '{}');\n"
+    "CREATE TABLE reportfmt(\n"
+    "   rn INTEGER PRIMARY KEY,\n"
+    "   owner TEXT,\n"
+    "   title TEXT UNIQUE,\n"
+    "   mtime INTEGER,\n"
+    "   cols TEXT,\n"
+    "   sqlcode TEXT\n"
+    ", jx TEXT DEFAULT '{}');\n"
+    "CREATE TABLE config(\n"
+    "  name TEXT PRIMARY KEY NOT NULL,\n"
+    "  value CLOB, mtime INTEGER,\n"
+    "  CHECK( typeof(name)='text' AND length(name)>=1 )\n"
+    ") WITHOUT ROWID;\n"
+    "CREATE TABLE shun(uuid PRIMARY KEY, mtime INTEGER, scom TEXT)\n"
+    "  WITHOUT ROWID;\n"
+    "CREATE TABLE concealed(\n"
+    "  hash TEXT PRIMARY KEY,\n"
+    "  content TEXT\n"
+    ", mtime INTEGER) WITHOUT ROWID;\n"
+    "CREATE TABLE admin_log(\n"
+    " id INTEGER PRIMARY KEY,\n"
+    " time INTEGER, -- Seconds since 1970\n"
+    " page TEXT,    -- path of page\n"
+    " who TEXT,     -- User who made the change\n"
+    "  what TEXT     -- What changed\n"
+    ");\n"
+    "CREATE TABLE unversioned(\n"
+    "  name TEXT PRIMARY KEY,\n"
+    "  rcvid INTEGER,\n"
+    "  mtime DATETIME,\n"
+    "  hash TEXT,\n"
+    "  sz INTEGER,\n"
+    "  encoding INT,\n"
+    "  content BLOB\n"
+    ") WITHOUT ROWID;\n"
+    "CREATE TABLE subscriber(\n"
+    "  subscriberId INTEGER PRIMARY KEY,\n"
+    "  subscriberCode BLOB DEFAULT (randomblob(32)) UNIQUE,\n"
+    "  semail TEXT UNIQUE COLLATE nocase,\n"
+    "  suname TEXT,\n"
+    "  sverified BOOLEAN DEFAULT true,\n"
+    "  sdonotcall BOOLEAN,\n"
+    "  sdigest BOOLEAN,\n"
+    "  ssub TEXT,\n"
+    "  sctime INTDATE,\n"
+    "  mtime INTDATE,\n"
+    "  smip TEXT\n"
+    ", lastContact INT);\n"
+    "CREATE TABLE pending_alert(\n"
+    "  eventid TEXT PRIMARY KEY,\n"
+    "  sentSep BOOLEAN DEFAULT false,\n"
+    "  sentDigest BOOLEAN DEFAULT false\n"
+    ", sentMod BOOLEAN DEFAULT false) WITHOUT ROWID;\n"
+    "CREATE TABLE filename(\n"
+    "  fnid INTEGER PRIMARY KEY,\n"
+    "  name TEXT UNIQUE\n"
+    ") STRICT;\n"
+    "CREATE TABLE mlink(\n"
+    "  mid INTEGER,\n"
+    "  fid INTEGER,\n"
+    "  pmid INTEGER,\n"
+    "  pid INTEGER,\n"
+    "  fnid INTEGER REFERENCES filename,\n"
+    "  pfnid INTEGER,\n"
+    "  mperm INTEGER,\n"
+    "  isaux INT DEFAULT 0\n"
+    ") STRICT;\n"
+    "CREATE TABLE plink(\n"
+    "  pid INTEGER REFERENCES blob,\n"
+    "  cid INTEGER REFERENCES blob,\n"
+    "  isprim INT,\n"
+    "  mtime REAL,\n"
+    "  baseid INTEGER REFERENCES blob,\n"
+    "  UNIQUE(pid, cid)\n"
+    ") STRICT;\n"
+    "CREATE TABLE leaf(rid INTEGER PRIMARY KEY);\n"
+    "CREATE TABLE event(\n"
+    "  type TEXT,\n"
+    "  mtime REAL,\n"
+    "  objid INTEGER PRIMARY KEY,\n"
+    "  tagid INTEGER,\n"
+    "  uid INTEGER REFERENCES user,\n"
+    "  bgcolor TEXT,\n"
+    "  euser TEXT,\n"
+    "  user TEXT,\n"
+    "  ecomment TEXT,\n"
+    "  comment TEXT,\n"
+    "  brief TEXT,\n"
+    "  omtime REAL\n"
+    ") STRICT;\n"
+    "CREATE TABLE phantom(\n"
+    "  rid INTEGER PRIMARY KEY\n"
+    ");\n"
+    "CREATE TABLE orphan(\n"
+    "  rid INTEGER PRIMARY KEY,\n"
+    "  baseline INTEGER\n"
+    ") STRICT;\n"
+    "CREATE TABLE unclustered(\n"
+    "  rid INTEGER PRIMARY KEY\n"
+    ");\n"
+    "CREATE TABLE unsent(\n"
+    "  rid INTEGER PRIMARY KEY\n"
+    ");\n"
+    "CREATE TABLE tag(\n"
+    "  tagid INTEGER PRIMARY KEY,\n"
+    "  tagname TEXT UNIQUE\n"
+    ") STRICT;\n"
+    "CREATE TABLE tagxref(\n"
+    "  tagid INTEGER REFERENCES tag,\n"
+    "  tagtype INTEGER,\n"
+    "  srcid INTEGER REFERENCES blob,\n"
+    "  origid INTEGER REFERENCES blob,\n"
+    "  value TEXT,\n"
+    "  mtime REAL,\n"
+    "  rid INTEGER REFERENCES blob,\n"
+    "  UNIQUE(rid, tagid)\n"
+    ") STRICT;\n"
+    "CREATE TABLE backlink(\n"
+    "  target TEXT,\n"
+    "  srctype INT,\n"
+    "  srcid INT,\n"
+    "  mtime REAL,\n"
+    "  UNIQUE(target, srctype, srcid)\n"
+    ") STRICT;\n"
+    "CREATE TABLE attachment(\n"
+    "  attachid INTEGER PRIMARY KEY,\n"
+    "  isLatest INT DEFAULT 0,\n"
+    "  mtime REAL,\n"
+    "  src TEXT,\n"
+    "  target TEXT,\n"
+    "  filename TEXT,\n"
+    "  comment TEXT,\n"
+    "  user TEXT\n"
+    ") STRICT;\n"
+    "CREATE TABLE cherrypick(\n"
+    "  parentid INT,\n"
+    "  childid INT,\n"
+    "  isExclude INT DEFAULT false,\n"
+    "  PRIMARY KEY(parentid, childid)\n"
+    ") WITHOUT ROWID, STRICT;\n"
+    "CREATE TABLE vcache(\n"
+    "  vid INTEGER,         -- check-in ID\n"
+    "  fname TEXT,          -- filename\n"
+    "  rid INTEGER,         -- artifact ID\n"
+    "  PRIMARY KEY(vid,fname)\n"
+    ") WITHOUT ROWID;\n"
+    "CREATE TABLE synclog(\n"
+    "  sfrom TEXT,\n"
+    "  sto TEXT,\n"
+    "  stime INT NOT NULL,\n"
+    "  stype TEXT,\n"
+    "  PRIMARY KEY(sfrom,sto)\n"
+    ") WITHOUT ROWID;\n"
+    "CREATE TABLE chat(\n"
+    "  msgid INTEGER PRIMARY KEY AUTOINCREMENT,\n"
+    "  mtime JULIANDAY,\n"
+    "  lmtime TEXT,\n"
+    "  xfrom TEXT,\n"
+    "  xmsg  TEXT,\n"
+    "  fname TEXT,\n"
+    "  fmime TEXT,\n"
+    "  mdel INT,\n"
+    "  file  BLOB\n"
+    ");\n"
+    "CREATE TABLE ftsdocs(\n"
+    "  rowid INTEGER PRIMARY KEY,\n"
+    "  type CHAR(1),\n"
+    "  rid INTEGER,\n"
+    "  name TEXT,\n"
+    "  idxed BOOLEAN,\n"
+    "  label TEXT,\n"
+    "  url TEXT,\n"
+    "  mtime DATE,\n"
+    "  bx TEXT,\n"
+    "  UNIQUE(type,rid)\n"
+    ");\n"
+    "CREATE TABLE ticket(\n"
+    "  -- Do not change any column that begins with tkt_\n"
+    "  tkt_id INTEGER PRIMARY KEY,\n"
+    "  tkt_uuid TEXT UNIQUE,\n"
+    "  tkt_mtime DATE,\n"
+    "  tkt_ctime DATE,\n"
+    "  -- Add as many fields as required below this line\n"
+    "  type TEXT,\n"
+    "  status TEXT,\n"
+    "  subsystem TEXT,\n"
+    "  priority TEXT,\n"
+    "  severity TEXT,\n"
+    "  foundin TEXT,\n"
+    "  private_contact TEXT,\n"
+    "  resolution TEXT,\n"
+    "  title TEXT,\n"
+    "  comment TEXT\n"
+    ");\n"
+    "CREATE TABLE ticketchng(\n"
+    "  -- Do not change any column that begins with tkt_\n"
+    "  tkt_id INTEGER REFERENCES ticket,\n"
+    "  tkt_rid INTEGER REFERENCES blob,\n"
+    "  tkt_mtime DATE,\n"
+    "  tkt_user TEXT,\n"
+    "  -- Add as many fields as required below this line\n"
+    "  login TEXT,\n"
+    "  username TEXT,\n"
+    "  mimetype TEXT,\n"
+    "  icomment TEXT\n"
+    ");\n"
+    "CREATE TABLE forumpost(\n"
+    "  fpid INTEGER PRIMARY KEY,\n"
+    "  froot INT,\n"
+    "  fprev INT,\n"
+    "  firt INT,\n"
+    "  fmtime REAL\n"
+    ");\n"
+    "CREATE INDEX delta_i1 ON delta(srcid);\n"
+    "CREATE INDEX blob_rcvid ON blob(rcvid);\n"
+    "CREATE INDEX subscriberUname\n"
+    "  ON subscriber(suname) WHERE suname IS NOT NULL;\n"
+    "CREATE INDEX mlink_i1 ON mlink(mid);\n"
+    "CREATE INDEX mlink_i2 ON mlink(fnid);\n"
+    "CREATE INDEX mlink_i3 ON mlink(fid);\n"
+    "CREATE INDEX mlink_i4 ON mlink(pid);\n"
+    "CREATE INDEX plink_i2 ON plink(cid,pid);\n"
+    "CREATE INDEX event_i1 ON event(mtime);\n"
+    "CREATE INDEX orphan_baseline ON orphan(baseline);\n"
+    "CREATE INDEX tagxref_i1 ON tagxref(tagid, mtime);\n"
+    "CREATE INDEX backlink_src ON backlink(srcid, srctype);\n"
+    "CREATE INDEX attachment_idx1 ON attachment(target, filename, mtime);\n"
+    "CREATE INDEX attachment_idx2 ON attachment(src);\n"
+    "CREATE INDEX cherrypick_cid ON cherrypick(childid);\n"
+    "CREATE INDEX ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;\n"
+    "CREATE INDEX ftsdocName ON ftsdocs(name) WHERE type='w';\n"
+    "CREATE INDEX ticketchng_idx1 ON ticketchng(tkt_id, tkt_mtime);\n"
+    "CREATE INDEX forumthread ON forumpost(froot,fmtime);\n"
+    "CREATE VIEW artifact(rid,rcvid,size,atype,srcid,hash,content) AS\n"
+    "  SELECT blob.rid,rcvid,size,1,srcid,uuid,content\n"
+    "    FROM blob LEFT JOIN delta ON (blob.rid=delta.rid);\n"
+    "CREATE VIEW ftscontent AS\n"
+    "  SELECT rowid, type, rid, name, idxed, label, url, mtime,\n"
+    "         title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'\n"
+    "    FROM ftsdocs;\n"
+  );
+  if( sqlite3_compileoption_used("ENABLE_FTS5") ){
+    speedtest1_exec(
+      "CREATE VIRTUAL TABLE ftsidx\n"
+      "  USING fts5(content=\"ftscontent\", title, body);\n"
+      "CREATE VIRTUAL TABLE chatfts1 USING fts5(\n"
+      "  xmsg, content=chat, content_rowid=msgid,tokenize=porter);\n"
+    );
+  }else{
+    speedtest1_exec(
+      "CREATE TABLE ftsidx_data(id INTEGER PRIMARY KEY, block BLOB);\n"
+      "CREATE TABLE ftsidx_idx(segid, term, pgno, PRIMARY KEY(segid, term))\n"
+      "  WITHOUT ROWID;\n"
+      "CREATE TABLE ftsidx_docsize(id INTEGER PRIMARY KEY, sz BLOB);\n"
+      "CREATE TABLE ftsidx_config(k PRIMARY KEY, v) WITHOUT ROWID;\n"
+      "CREATE TABLE chatfts1_data(id INTEGER PRIMARY KEY, block BLOB);\n"
+      "CREATE TABLE chatfts1_idx(segid, term, pgno, PRIMARY KEY(segid, term))\n"
+      "  WITHOUT ROWID;\n"
+      "CREATE TABLE chatfts1_docsize(id INTEGER PRIMARY KEY, sz BLOB);\n"
+      "CREATE TABLE chatfts1_config(k PRIMARY KEY, v) WITHOUT ROWID;\n"
+    );
+  }
+  speedtest1_exec(
+    "ANALYZE sqlite_schema;\n"
+    "INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES\n"
+    "  ('ftsidx_config','ftsidx_config','1 1'),\n"
+    "  ('ftsidx_idx','ftsidx_idx','4215 401 1'),\n"
+    "  ('user','sqlite_autoindex_user_1','25 1'),\n"
+    "  ('phantom',NULL,'26'),\n"
+    "  ('reportfmt','sqlite_autoindex_reportfmt_1','9 1'),\n"
+    "  ('rcvfrom','sqlite_autoindex_rcvfrom_1','18445 401'),\n"
+    "  ('private',NULL,'99'),\n"
+    "  ('mlink','mlink_i4','116678 401'),\n"
+    "  ('mlink','mlink_i3','121212 2'),\n"
+    "  ('mlink','mlink_i2','106372 401'),\n"
+    "  ('mlink','mlink_i1','99298 5'),\n"
+    "  ('ftsidx_data',NULL,'3795'),\n"
+    "  ('leaf',NULL,'1559'),\n"
+    "  ('delta','delta_i1','66340 1'),\n"
+    "  ('unversioned','unversioned','3 1'),\n"
+    "  ('pending_alert','pending_alert','3 1'),\n"
+    "  ('cherrypick','cherrypick_cid','680 2'),\n"
+    "  ('cherrypick','cherrypick','628 1 1'),\n"
+    "  ('config','config','128 1'),\n"
+    "  ('ftsidx_docsize',NULL,'33848'),\n"
+    "  ('event','event_i1','36096 1'),\n"
+    "  ('plink','plink_i2','38236 1 1'),\n"
+    "  ('plink','sqlite_autoindex_plink_1','38357 1 1'),\n"
+    "  ('shun','shun','10 1'),\n"
+    "  ('concealed','concealed','110 1'),\n"
+    "  ('vcache','vcache','1888 401 1'),\n"
+    "  ('ftsdocs','ftsdocName','19 1'),\n"
+    "  ('ftsdocs','ftsdocIdxed','168 84 1 1'),\n"
+    "  ('ftsdocs','sqlite_autoindex_ftsdocs_1','37312 401 1'),\n"
+    "  ('subscriber','subscriberUname','5 1'),\n"
+    "  ('subscriber','sqlite_autoindex_subscriber_2','37 1'),\n"
+    "  ('subscriber','sqlite_autoindex_subscriber_1','37 1'),\n"
+    "  ('tag','sqlite_autoindex_tag_1','2990 1'),\n"
+    "  ('filename','sqlite_autoindex_filename_1','3168 1'),\n"
+    "  ('chat',NULL,'56124'),\n"
+    "  ('tagxref','tagxref_i1','40992 401 2'),\n"
+    "  ('tagxref','sqlite_autoindex_tagxref_1','79233 3 1'),\n"
+    "  ('attachment','attachment_idx2','11 1'),\n"
+    "  ('attachment','attachment_idx1','11 2 2 1'),\n"
+    "  ('blob','blob_rcvid','128240 201'),\n"
+    "  ('blob','sqlite_autoindex_blob_1','126480 1'),\n"
+    "  ('synclog','synclog','12 3 1'),\n"
+    "  ('backlink','backlink_src','2160 2 2'),\n"
+    "  ('backlink','sqlite_autoindex_backlink_1','2340 2 2 1'),\n"
+    "  ('accesslog',NULL,'38'),\n"
+    "  ('chatfts1_config','chatfts1_config','1 1'),\n"
+    "  ('chatfts1_idx','chatfts1_idx','688 230 1'),\n"
+    "  ('ticket','sqlite_autoindex_ticket_1','794 1'),\n"
+    "  ('ticketchng','ticketchng_idx1','2089 3 1'),\n"
+    "  ('forumpost','forumthread','4 4 1'),\n"
+    "  ('unclustered',NULL,'12');\n"
+    "COMMIT;"
+  );
+  speedtest1_end_test();
+
+  n = g.szTest*3;
+  speedtest1_begin_test(110, "Open and use the database %d times", n);
+  for(i=0; i<n; i++){
+    sqlite3 *dbMain = g.db;
+    sqlite3 *dbAux = 0;
+    if( g.zDbName && g.zDbName[0] ){
+      if( sqlite3_open_v2(g.zDbName, &dbAux, SQLITE_OPEN_READWRITE, g.zVfs) ){
+        fatal_error("Cannot open database file: %s\n", g.zDbName);
+      }
+      g.db = dbAux;
+    }
+    speedtest1_exec(
+      "SELECT name FROM pragma_table_list /*scan*/"
+      " WHERE schema='repository' AND type IN ('table','virtual')"
+      " AND name NOT IN ('admin_log', 'blob','delta','rcvfrom','user','alias',"
+                        "'config','shun','private','reportfmt',"
+                        "'concealed','accesslog','modreq',"
+                        "'purgeevent','purgeitem','unversioned',"
+                        "'subscriber','pending_alert','chat')"
+      " AND name NOT GLOB 'sqlite_*'"
+      " AND name NOT GLOB 'fx_*';"
+      "SELECT 1 FROM pragma_table_xinfo('ticket') WHERE name = 'mimetype';"
+    );
+    speedtest1_exec(
+      "SELECT"
+      " name,"
+      " value,"
+      " unixepoch()/86400-value,"
+      " date(value*86400,'unixepoch')"
+      " FROM config"
+      " WHERE name in ('email-renew-warning','email-renew-cutoff');"
+      "SELECT count(*) FROM pending_alert WHERE NOT sentDigest;"
+    );
+    speedtest1_exec(
+      "WITH priors(rid,who) AS ("
+      "  SELECT firt, coalesce(euser,user)"
+      "    FROM forumpost LEFT JOIN event ON fpid=objid"
+      "   WHERE fpid=12345"
+      "  UNION ALL"
+      "  SELECT firt, coalesce(euser,user)"
+      "    FROM priors, forumpost LEFT JOIN event ON fpid=objid"
+      "   WHERE fpid=rid"
+      ")"
+      "SELECT ','||group_concat(DISTINCT 'u'||who)||"
+             "','||group_concat(rid) FROM priors;"
+    );
+    speedtest1_exec(
+      "CREATE TEMP TABLE IF NOT EXISTS ok(rid INTEGER PRIMARY KEY);\n"
+    );
+    speedtest1_exec(
+      "WITH RECURSIVE\n"
+      "  parent(pid,cid,isCP) AS (\n"
+      "    SELECT plink.pid, plink.cid, 0 AS xisCP FROM plink\n"
+      "    UNION ALL\n"
+      "    SELECT parentid, childid, 1 FROM cherrypick WHERE NOT isExclude\n"
+      "  ),\n"
+      "  ancestor(rid, mtime, isCP) AS (\n"
+      "    SELECT 123, mtime, 0 FROM event WHERE objid=$object\n"
+      "    UNION\n"
+      "    SELECT parent.pid, event.mtime, parent.isCP\n"
+      "      FROM ancestor, parent, event\n"
+      "     WHERE parent.cid=ancestor.rid\n"
+      "       AND event.objid=parent.pid\n"
+      "       AND NOT ancestor.isCP\n"
+      "       AND (event.mtime>=$date OR parent.pid=$pid)\n"
+      "     ORDER BY mtime DESC LIMIT 10\n"
+      "  )\n"
+      "  INSERT OR IGNORE INTO ok SELECT rid FROM ancestor;"
+    );
+    sqlite3_close(dbAux);
+    g.db = dbMain;
+  }
+  speedtest1_end_test();
+}
+
 #ifdef SQLITE_ENABLE_RTREE
 /* Generate two numbers between 1 and mx.  The first number is less than
 ** the second.  Usually the numbers are near each other but can sometimes
@@ -1545,7 +2149,7 @@ void testset_rtree(int p1, int p2){
     }
     speedtest1_end_test();
   }
-  
+
   n = g.szTest*200;
   speedtest1_begin_test(120, "%d one-dimensional overlap slice queries", n);
   speedtest1_prepare("SELECT count(*) FROM rt1 WHERE y1>=?1 AND y0<=?2");
@@ -1574,7 +2178,6 @@ void testset_rtree(int p1, int p2){
     }
     speedtest1_end_test();
   }
-  
 
   n = g.szTest*200;
   speedtest1_begin_test(125, "%d custom geometry callback queries", n);
@@ -2149,6 +2752,164 @@ void testset_debug1(void){
   }
 }
 
+/*
+** Performance tests for JSON.
+*/
+void testset_json(void){
+  unsigned int r = 0x12345678;
+  sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, r, g.db);
+  speedtest1_begin_test(100, "table J1 is %d rows of JSONB",
+                        g.szTest*5);
+  speedtest1_exec(
+     "CREATE TABLE j1(x JSONB);\n"
+     "WITH RECURSIVE\n"
+     "  jval(n,j) AS (\n"
+     "    VALUES(0,'{}'),(1,'[]'),(2,'true'),(3,'false'),(4,'null'),\n"
+     "          (5,'{x:1,y:2}'),(6,'0.0'),(7,'3.14159'),(8,'-99.9'),\n"
+     "          (9,'[1,2,\"\\n\\u2192\\\"\\u2190\",4]')\n"
+     "  ),\n"
+     "  c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<26*26-1),\n"
+     "  array1(y) AS MATERIALIZED (\n"
+     "    SELECT jsonb_group_array(\n"
+     "      jsonb_object('x',x,\n"
+     "                  'y',jsonb(coalesce(j,random()%%10000)),\n"
+     "                  'z',hex(randomblob(50)))\n"
+     "    )\n"
+     "    FROM c LEFT JOIN jval ON (x%%20)=n\n"
+     "  ),\n"
+     "  object1(z) AS MATERIALIZED (\n"
+     "    SELECT jsonb_group_object(char(0x61+x%%26,0x61+(x/26)%%26),\n"
+     "                      jsonb( coalesce(j,random()%%10000)))\n"
+     "      FROM c LEFT JOIN jval ON (x%%20)=n\n"
+     "  ),\n"
+     "  c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<%d)\n"
+     "INSERT INTO j1(x)\n"
+     "  SELECT jsonb_object('a',n,'b',n+10000,'c',jsonb(y),'d',jsonb(z),\n"
+     "                     'e',n+20000,'f',n+30000)\n"
+     "    FROM array1, object1, c2;",
+     g.szTest*5
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(110, "table J2 is %d rows from J1 converted to text", g.szTest);
+  speedtest1_exec(
+     "CREATE TABLE j2(x JSON TEXT);\n"
+     "INSERT INTO j2(x) SELECT json(x) FROM j1 LIMIT %d", g.szTest
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(120, "create indexes on JSON expressions on J1");
+  speedtest1_exec(
+    "BEGIN;\n"
+    "CREATE INDEX j1x1 ON j1(x->>'a');\n"
+    "CREATE INDEX j1x2 ON j1(x->>'b');\n"
+    "CREATE INDEX j1x3 ON j1(x->>'f');\n"
+    "COMMIT;\n"
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(130, "create indexes on JSON expressions on J2");
+  speedtest1_exec(
+    "BEGIN;\n"
+    "CREATE INDEX j2x1 ON j2(x->>'a');\n"
+    "CREATE INDEX j2x2 ON j2(x->>'b');\n"
+    "CREATE INDEX j2x3 ON j2(x->>'f');\n"
+    "COMMIT;\n"
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(140, "queries against J1");
+  speedtest1_exec(
+    "WITH c(n) AS (VALUES(0) UNION ALL SELECT n+1 FROM c WHERE n<7)\n"
+    "  SELECT sum(x->>format('$.c[%%d].x',n)) FROM c, j1;\n"
+
+    "WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<5)\n"
+    "  SELECT sum(x->>format('$.\"c\"[#-%%d].y',n)) FROM c, j1;\n"
+
+    "SELECT sum(x->>'$.d.ez' + x->>'$.d.\"xz\"' + x->>'a' + x->>'$.c[10].y') FROM j1;\n"
+
+    "SELECT x->>'$.d.tz[2]', x->'$.d.tz' FROM j1;\n"
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(141, "queries involving json_type()");
+  speedtest1_exec(
+    "WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20)\n"
+    "  SELECT json_type(x,format('$.c[#-%%d].y',n)), count(*)\n"
+    "    FROM c, j1\n"
+    "   WHERE j1.rowid=1\n"
+    "   GROUP BY 1 ORDER BY 2;"
+  );
+  speedtest1_end_test();
+
+
+  speedtest1_begin_test(150, "json_insert()/set()/remove() on every row of J1");
+  speedtest1_exec(
+    "BEGIN;\n"
+    "UPDATE j1 SET x=jsonb_insert(x,'$.g',(x->>'f')+1,'$.h',3.14159,'$.i','hello',\n"
+    "                               '$.j',json('{x:99}'),'$.k','{y:98}');\n"
+    "UPDATE j1 SET x=jsonb_set(x,'$.e',(x->>'f')-1);\n"
+    "UPDATE j1 SET x=jsonb_remove(x,'$.d');\n"
+    "COMMIT;\n"
+  );
+  speedtest1_end_test();
+
+  speedtest1_begin_test(160, "json_insert()/set()/remove() on every row of J2");
+  speedtest1_exec(
+    "BEGIN;\n"
+    "UPDATE j2 SET x=json_insert(x,'$.g',(x->>'f')+1);\n"
+    "UPDATE j2 SET x=json_set(x,'$.e',(x->>'f')-1);\n"
+    "UPDATE j2 SET x=json_remove(x,'$.d');\n"
+    "COMMIT;\n"
+  );
+  speedtest1_end_test();
+
+}
+
+/*
+** This testset focuses on the speed of parsing numeric literals (integers
+** and real numbers). This was added to test the impact of allowing "_"
+** characters to appear in numeric SQL literals to make them easier to read. 
+** For example, "SELECT 1_000_000;" instead of "SELECT 1000000;".
+*/
+void testset_parsenumber(void){
+  const char *zSql1 = "SELECT 1, 12, 123, 1234, 12345, 123456";
+  const char *zSql2 = "SELECT 8227256643844975616, 7932208612563860480, "
+                      "2010730661871032832, 9138463067404021760, "
+                      "2557616153664746496, 2557616153664746496";
+  const char *zSql3 = "SELECT 1.0, 1.2, 1.23, 123.4, 1.2345, 1.23456";
+  const char *zSql4 = "SELECT 8.227256643844975616, 7.932208612563860480, "
+                      "2.010730661871032832, 9.138463067404021760, "
+                      "2.557616153664746496, 2.557616153664746496";
+
+  const int NROW = 100*g.szTest;
+  int ii;
+
+  speedtest1_begin_test(100, "parsing %d small integers", NROW);
+  for(ii=0; ii<NROW; ii++){
+    sqlite3_exec(g.db, zSql1, 0, 0, 0);
+  }
+  speedtest1_end_test();
+
+  speedtest1_begin_test(110, "parsing %d large integers", NROW);
+  for(ii=0; ii<NROW; ii++){
+    sqlite3_exec(g.db, zSql2, 0, 0, 0);
+  }
+  speedtest1_end_test();
+
+  speedtest1_begin_test(200, "parsing %d small reals", NROW);
+  for(ii=0; ii<NROW; ii++){
+    sqlite3_exec(g.db, zSql3, 0, 0, 0);
+  }
+  speedtest1_end_test();
+
+  speedtest1_begin_test(210, "parsing %d large reals", NROW);
+  for(ii=0; ii<NROW; ii++){
+    sqlite3_exec(g.db, zSql4, 0, 0, 0);
+  }
+  speedtest1_end_test();
+}
+
 #ifdef __linux__
 #include <sys/types.h>
 #include <unistd.h>
@@ -2209,6 +2970,8 @@ int main(int argc, char **argv){
   int doIncrvac = 0;            /* True for --incrvacuum */
   const char *zJMode = 0;       /* Journal mode */
   const char *zKey = 0;         /* Encryption key */
+  int nHardHeapLmt = 0;         /* The hard heap limit */
+  int nSoftHeapLmt = 0;         /* The soft heap limit */
   int nLook = -1, szLook = 0;   /* --lookaside configuration */
   int noSync = 0;               /* True for --nosync */
   int pageSize = 0;             /* Desired page size.  0 means default */
@@ -2220,11 +2983,9 @@ int main(int argc, char **argv){
   int memDb = 0;                /* --memdb.  Use an in-memory database */
   int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE
     ;                           /* SQLITE_OPEN_xxx flags. */
-  char *zTSet = "main";         /* Which --testset torun */
-  const char * zVfs = 0;        /* --vfs NAME */
+  char *zTSet = "mix1";         /* Which --testset torun */
   int doTrace = 0;              /* True for --trace */
   const char *zEncoding = 0;    /* --utf16be or --utf16le */
-  const char *zDbName = 0;      /* Name of the test database */
 
   void *pHeap = 0;              /* Allocated heap space */
   void *pLook = 0;              /* Allocated lookaside space */
@@ -2233,6 +2994,16 @@ int main(int argc, char **argv){
   int i;                        /* Loop counter */
   int rc;                       /* API return code */
 
+  /* "mix1" is a macro testset: */
+  static char zMix1Tests[] =
+    "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10,star"
+#if !defined(SQLITE_SPEEDTEST1_WASM)
+    ",app"
+    /* This test misbehaves in WASM builds: sqlite3_open_v2() is
+       failing to find the db file for reasons not yet understood. */
+#endif
+    ;
+
 #ifdef SQLITE_SPEEDTEST1_WASM
   /* Resetting all state is important for the WASM build, which may
   ** call main() multiple times. */
@@ -2251,10 +3022,13 @@ int main(int argc, char **argv){
          sqlite3_libversion(), sqlite3_sourceid());
 
   /* Process command-line arguments */
+  g.zDbName = 0;
+  g.zVfs = 0;
   g.zWR = "";
   g.zNN = "";
   g.zPK = "UNIQUE";
   g.szTest = 100;
+  g.szBase = 100;
   g.nRepeat = 1;
   for(i=1; i<argc; i++){
     const char *z = argv[i];
@@ -2276,6 +3050,10 @@ int main(int argc, char **argv){
       }else if( strcmp(z,"explain")==0 ){
         g.bSqlOnly = 1;
         g.bExplain = 1;
+      }else if( strcmp(z,"hard-heap-limit")==0 ){
+        ARGC_VALUE_CHECK(1);
+        nHardHeapLmt = integerValue(argv[i+1]);
+        i += 1;
       }else if( strcmp(z,"heap")==0 ){
         ARGC_VALUE_CHECK(2);
         nHeap = integerValue(argv[i+1]);
@@ -2364,7 +3142,11 @@ int main(int argc, char **argv){
         g.bMemShrink = 1;
       }else if( strcmp(z,"size")==0 ){
         ARGC_VALUE_CHECK(1);
-        g.szTest = integerValue(argv[++i]);
+        g.szTest = g.szBase = integerValue(argv[++i]);
+      }else if( strcmp(z,"soft-heap-limit")==0 ){
+        ARGC_VALUE_CHECK(1);
+        nSoftHeapLmt = integerValue(argv[i+1]);
+        i += 1;
       }else if( strcmp(z,"stats")==0 ){
         showStats = 1;
       }else if( strcmp(z,"temp")==0 ){
@@ -2393,7 +3175,7 @@ int main(int argc, char **argv){
 #endif
       }else if( strcmp(z,"vfs")==0 ){
         ARGC_VALUE_CHECK(1);
-        zVfs = argv[++i];
+        g.zVfs = argv[++i];
       }else if( strcmp(z,"reserve")==0 ){
         ARGC_VALUE_CHECK(1);
         g.nReserve = atoi(argv[++i]);
@@ -2423,8 +3205,8 @@ int main(int argc, char **argv){
         fatal_error("unknown option: %s\nUse \"%s -?\" for help\n",
                     argv[i], argv[0]);
       }
-    }else if( zDbName==0 ){
-      zDbName = argv[i];
+    }else if( g.zDbName==0 ){
+      g.zDbName = argv[i];
     }else{
       fatal_error("surplus argument: %s\nUse \"%s -?\" for help\n",
                   argv[i], argv[0]);
@@ -2453,8 +3235,8 @@ int main(int argc, char **argv){
 #endif
   sqlite3_initialize();
 
-  if( zDbName!=0 ){
-    sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs);
+  if( g.zDbName!=0 ){
+    sqlite3_vfs *pVfs = sqlite3_vfs_find(g.zVfs);
     /* For some VFSes, e.g. opfs, unlink() is not sufficient. Use the
     ** selected (or default) VFS's xDelete method to delete the
     ** database. This is specifically important for the "opfs" VFS
@@ -2462,15 +3244,15 @@ int main(int argc, char **argv){
     ** can be cleaned up properly. For historical compatibility, we'll
     ** also simply unlink(). */
     if( pVfs!=0 ){
-      pVfs->xDelete(pVfs, zDbName, 1);
+      pVfs->xDelete(pVfs, g.zDbName, 1);
     }
-    unlink(zDbName);
+    unlink(g.zDbName);
   }
 
   /* Open the database and the input file */
-  if( sqlite3_open_v2(memDb ? ":memory:" : zDbName, &g.db,
-                      openFlags, zVfs) ){
-    fatal_error("Cannot open database file: %s\n", zDbName);
+  if( sqlite3_open_v2(memDb ? ":memory:" : g.zDbName, &g.db,
+                      openFlags, g.zVfs) ){
+    fatal_error("Cannot open database file: %s\n", g.zDbName);
   }
 #if SQLITE_VERSION_NUMBER>=3006001
   if( nLook>0 && szLook>0 ){
@@ -2526,10 +3308,21 @@ int main(int argc, char **argv){
   if( zJMode ){
     speedtest1_exec("PRAGMA journal_mode=%s", zJMode);
   }
+  if( nHardHeapLmt>0 ){
+    speedtest1_exec("PRAGMA hard_heap_limit=%d", nHardHeapLmt);
+  }
+  if( nSoftHeapLmt>0 ){
+    speedtest1_exec("PRAGMA soft_heap_limit=%d", nSoftHeapLmt);
+  }
+  if( zJMode ){
+    speedtest1_exec("PRAGMA journal_mode=%s", zJMode);
+  }
 
   if( g.bExplain ) printf(".explain\n.echo on\n");
+  if( strcmp(zTSet,"mix1")==0 ) zTSet = zMix1Tests;
   do{
     char *zThisTest = zTSet;
+    char *zSep;
     char *zComma = strchr(zThisTest,',');
     if( zComma ){
       *zComma = 0;
@@ -2537,7 +3330,20 @@ int main(int argc, char **argv){
     }else{
       zTSet = "";
     }
-    if( g.iTotal>0 || zComma!=0 ){
+    zSep = strchr(zThisTest, '/');
+    if( zSep ){
+      int kk;
+      for(kk=1; zSep[kk] && ISDIGIT(zSep[kk]); kk++){}
+      if( kk==1 || zSep[kk]!=0 ){
+        fatal_error("bad modifier on testset name: \"%s\"", zThisTest);
+      }
+      g.szTest = g.szBase*integerValue(zSep+1)/100;
+      if( g.szTest<=0 ) g.szTest = 1;
+      zSep[0] = 0;
+    }else{
+      g.szTest = g.szBase;
+    }
+    if( g.iTotal>0 || zComma==0 ){
       printf("       Begin testset \"%s\"\n", zThisTest);
     }
     if( strcmp(zThisTest,"main")==0 ){
@@ -2548,10 +3354,18 @@ int main(int argc, char **argv){
       testset_orm();
     }else if( strcmp(zThisTest,"cte")==0 ){
       testset_cte();
+    }else if( strcmp(zThisTest,"star")==0 ){
+      testset_star();
+    }else if( strcmp(zThisTest,"app")==0 ){
+      testset_app();
     }else if( strcmp(zThisTest,"fp")==0 ){
       testset_fp();
+    }else if( strcmp(zThisTest,"json")==0 ){
+      testset_json();
     }else if( strcmp(zThisTest,"trigger")==0 ){
       testset_trigger();
+    }else if( strcmp(zThisTest,"parsenumber")==0 ){
+      testset_parsenumber();
     }else if( strcmp(zThisTest,"rtree")==0 ){
 #ifdef SQLITE_ENABLE_RTREE
       testset_rtree(6, 147);
diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test
index db7241df0d..656b1c4588 100644
--- a/test/sqlcipher-compatibility.test
+++ b/test/sqlcipher-compatibility.test
@@ -69,6 +69,7 @@ do_test unencrypted-attach {
     SELECT count(*) FROM t1;
   } db2 
 } {ok 1000}
+db close
 db2 close
 file delete -force test.db
 file delete -force test2.db
@@ -103,6 +104,7 @@ do_test unencrypted-attach-raw-key {
     SELECT count(*) FROM t1;
   } db2 
 } {ok 1000}
+db close
 db2 close
 file delete -force test.db
 file delete -force test2.db
@@ -1052,36 +1054,44 @@ file delete -force test.db
 do_test migrate-1.1.8-database-to-current-format {
   file copy -force $sampleDir/sqlcipher-1.1.8-testkey.db test.db
   sqlite_orig db test.db
-  execsql {
+  set rc {}
+
+  lappend rc [execsql {
     PRAGMA key = 'testkey';
     PRAGMA cipher_migrate;
-  }
+    SELECT count(*) FROM sqlite_schema;
+  }]
   db close
     
   sqlite_orig db test.db
-  execsql {
+  lappend rc [execsql {
     PRAGMA key = 'testkey';
     SELECT count(*) FROM sqlite_schema;
-  }
-} {ok 1}
+    PRAGMA journal_mode;
+  }]
+} {{ok 0 1} {ok 1 delete}}
 db close
 file delete -force test.db 
 
 do_test migrate-2-0-le-database-to-current-format {
   file copy -force $sampleDir/sqlcipher-2.0-le-testkey.db test.db
   sqlite_orig db test.db
-  execsql {
+  set rc {}
+
+  lappend rc [execsql {
     PRAGMA key = 'testkey';
     PRAGMA cipher_migrate;
-  }
+    SELECT count(*) FROM sqlite_schema;
+  }]
   db close
     
   sqlite_orig db test.db
-  execsql {
+  lappend rc [execsql {
     PRAGMA key = 'testkey';
     SELECT count(*) FROM sqlite_schema;
-  }
-} {ok 1}
+    PRAGMA journal_mode;
+  }]
+} {{ok 0 1} {ok 1 delete}}
 db close
 file delete -force test.db 
 
@@ -1165,9 +1175,8 @@ do_test migrate-failure {
 db close
 file delete -force test.db 
 
-# leave database is a readable state after a 
-# a failed migration 
-do_test migrate-failure-readable {
+# if a migration failes the database should be in a permanent error state
+do_test migrate-failure-not-readable {
   file copy -force $sampleDir/sqlcipher-3.0-testkey.db test.db
   sqlite_orig db test.db
 
@@ -1188,7 +1197,22 @@ do_test migrate-failure-readable {
     PRAGMA cipher_test_off = fail_migrate;
     PRAGMA cipher_test;
   }]
-} {{ok 1} {1 {SQL logic error}} 0}
+} {{ok 1} {1 {out of memory}} 0}
+db close
+file delete -force test.db 
+
+# if cipher_migrate is called on a current-version databse
+# is should do nothing and just report OK
+do_test migrate-current-format-noop {
+  file copy -force $sampleDir/sqlcipher-4.0-testkey.db test.db
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA key = 'testkey';
+    PRAGMA cipher_migrate;
+    SELECT count(*) FROM sqlite_schema;
+  }
+} {ok 0 1}
 db close
 file delete -force test.db 
 
@@ -1233,6 +1257,7 @@ do_test key-multiple-databases-with-different-keys-using-pragma {
         attach database 'test.db' as test key 'foobar';
         select * from t1;
         select * from test.t1;
+        detach database test;
     }
 } {ok foo bar baz qux}
 db close
@@ -1269,6 +1294,7 @@ if_built_with_libtomcrypt verify-random-data-alters-file-content {
     lappend rc [cmpFilesChunked test.db test2.db]
     lappend rc [cmpFilesChunked test2.db test3.db]
 } {0 1}
+db close
 file delete -force test.db
 file delete -force test2.db
 file delete -force test3.db
@@ -1465,6 +1491,29 @@ do_test default-compat-open-4.0-database {
   } 
 } {ok ok 78536}
 
+# create a database using a full keyspec consising of
+# 64 characters for the encryption key, 64 for the hmac key
+# and 32 for the salt
+do_test test-full-keyspec {
+    sqlite_orig db test.db
+    execsql {
+        PRAGMA key = "x'0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'";
+        CREATE TABLE t1(a,b);
+        INSERT INTO t1(a,b) VALUES (1,2);
+    }
+    db close
+
+    sqlite_orig db test.db
+    execsql {
+        PRAGMA key = "x'0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'";
+        SELECT count(*) FROM t1;
+        PRAGMA cipher_salt;
+    }
+
+} {ok 1 00000000000000000000000000000000}
+db close
+file delete -force test.db
+
 sqlite3_test_control_pending_byte $old_pending_byte
 
 finish_test
diff --git a/test/sqlcipher-core.test b/test/sqlcipher-core.test
index 5aed9fb66c..22179feb73 100644
--- a/test/sqlcipher-core.test
+++ b/test/sqlcipher-core.test
@@ -74,6 +74,20 @@ do_test will-open-with-correct-derived-key {
 db close
 file delete -force test.db
 
+# run test with sqlite3_key function directly
+setup test.db "'testkey'"
+do_test will-open-with-sqlite3key {
+
+  sqlite_orig db test.db
+  sqlite3_key db "testkey"
+  execsql {
+    SELECT name FROM sqlite_schema WHERE type='table';
+    SELECT * from t1;
+  }
+} {t1 test1 test2}
+db close
+file delete -force test.db
+
 # set an encryption key (non-hex) and create
 # temp tables, verify you can read from
 # sqlite_temp_master
@@ -498,7 +512,7 @@ file delete -force test.db
 #    key and page size
 # 4. verify that the table is readable
 #    and the data just inserted is visible
-do_test custom-pagesize {
+do_test custom-pagesize-pragma-cipher-page-size {
   sqlite_orig db test.db
 
   execsql {
@@ -528,11 +542,55 @@ do_test custom-pagesize {
 
 } {ok 1000}
 db close
+file delete -force test.db
+
+# run the same logic as previous test but use
+# pragma page_size instead
+do_test custom-pagesize-pragma-pagesize {
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA key = 'testkey';
+    PRAGMA page_size = 8192;
+    CREATE table t1(a,b);
+    BEGIN;
+  }
+
+  for {set i 1} {$i<=1000} {incr i} {
+    set r [expr {int(rand()*500000)}]
+    execsql "INSERT INTO t1 VALUES($i,'value $r');"
+  }
+
+  execsql {
+    COMMIT;
+  }
+
+  db close
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA key = 'testkey';
+    PRAGMA page_size = 8192;
+    SELECT count(*) FROM t1;
+  }
+
+} {ok 1000}
+db close
+file delete -force test.db
 
 # open the database with the default page size
 ## and verfiy that it is not readable 
 do_test custom-pagesize-must-match {
   sqlite_orig db test.db
+  execsql {
+    PRAGMA key = 'testkey';
+    PRAGMA cipher_page_size = 8192;
+    CREATE table t1(a,b);
+  }
+
+  db close
+  sqlite_orig db test.db
+
   catchsql {
     PRAGMA key = 'testkey';
     SELECT name FROM sqlite_schema WHERE type='table';
@@ -622,6 +680,28 @@ do_test multiple-key-calls-safe {
 db close
 file delete -force test.db
 
+# open a databse and use the valid key. Then
+# use pragma cipher_compatability to adjust settings that
+# would normally trigger key derivation again. 
+# the new settings should be ignored
+
+setup test.db "'testkey'"
+do_test setting-changes-after-key-calls-safe {
+  sqlite_orig db test.db
+  execsql {
+    PRAGMA key = 'testkey';
+    PRAGMA cache_size = 0;
+    SELECT name FROM sqlite_schema WHERE type='table';
+    INSERT INTO t1(a,b) VALUES (2,zeroblob(8192));
+    PRAGMA cipher_compatibility=3;
+    INSERT INTO t1(a,b) VALUES (3,zeroblob(8192));
+    SELECT name FROM sqlite_schema WHERE type='table';
+    SELECT count(*) FROM t1;
+  }
+} {ok t1 t1 3}
+db close
+file delete -force test.db
+
 # 1. create a database with a custom hmac kdf iteration count, 
 # 2. create table and insert operations should work
 # 3. close database, open it again with the same
@@ -755,24 +835,6 @@ do_test cipher-options-before-keys {
 db close
 file delete -force test.db
 
-# verify memory security behavior
-# initially should report OFF
-# then enable, check that it is ON
-# try to turn if off, but verify that it
-# can't be unset.
-do_test verify-memory-security {
-    sqlite_orig db test.db
-    execsql {
-        PRAGMA cipher_memory_security;
-        PRAGMA cipher_memory_security = ON;
-        PRAGMA cipher_memory_security;
-        PRAGMA cipher_memory_security = OFF;
-        PRAGMA cipher_memory_security;
-    }
-} {0 1 1}
-db close
-file delete -force test.db
-
 # create two new database files, write to each
 # and verify that they have different (i.e. random)
 # salt values
@@ -789,10 +851,10 @@ do_test test-random-salt {
     CREATE TABLE t1(a,b);
     INSERT INTO t1(a,b) VALUES (1,2);
   } db2
-  db close
-  db2 close
   string equal [hexio_read test.db 0 16] [hexio_read test2.db 0 16]
 } {0}
+db close
+db2 close
 file delete -force test.db
 file delete -force test2.db
 
@@ -895,8 +957,6 @@ file delete -force test.db
 # close database
 # open normally providing key via pragma verify
 # correct key works
-sqlite3_shutdown
-sqlite3_config_uri 1
 do_test uri-key {
   sqlite_orig db file:test.db?a=a&key=testkey&c=c
 
@@ -934,8 +994,85 @@ do_test uri-key-2 {
 } {1 {file is not a database}}
 db close
 file delete -force test.db
-sqlite3_shutdown
-sqlite3_config_uri 0
+
+# test cipher_status reports encrypted for database
+do_test cipher-status-encrypted {
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA key = 'test';
+    PRAGMA cipher_status;
+  }
+
+} {ok 1}
+db close
+file delete -force test.db
+
+# test cipher_status reports not-encrypted for plaintext database
+do_test cipher-status-plaintext {
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA cipher_status;
+  }
+
+} {0}
+db close
+file delete -force test.db
+
+# test cipher_status reports encrypted for attached database
+# when main database is plaintext and attached database is
+# keyed
+do_test cipher-status-attached-encrypted {
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA cipher_status;
+    ATTACH DATABASE 'test2.db' AS test2 KEY 'test';
+    PRAGMA test2.cipher_status;
+    DETACH DATABASE test2;
+  }
+
+} {0 1}
+db close
+file delete -force test.db
+file delete -force test2.db
+
+# test cipher_status reports plaintext for attached database
+# when main database is keyed and attached database is not
+do_test cipher-status-attached-plaintext {
+  sqlite_orig db test.db
+
+  execsql {
+    PRAGMA KEY = 'test';
+    PRAGMA cipher_status;
+    ATTACH DATABASE 'test2.db' AS test2 KEY '';
+    PRAGMA test2.cipher_status;
+    DETACH DATABASE test2;
+  }
+
+} {ok 1 0}
+db close
+file delete -force test.db
+file delete -force test2.db
+
+# test cipher_status reports unencrypted if key derivation
+# and operation fails
+setup test.db "'testkey'"
+do_test cipher-status-badkey {
+  sqlite_orig db test.db
+
+  catchsql {
+    PRAGMA key = 'test';
+    SELECT count(*) FROM sqlite_master;
+  }
+  execsql {
+    PRAGMA cipher_status;
+  }
+
+} {0}
+db close
+file delete -force test.db
 
 finish_test
 
diff --git a/test/sqlcipher-integrity.test b/test/sqlcipher-integrity.test
index 5e13eff987..dbf903fcd0 100644
--- a/test/sqlcipher-integrity.test
+++ b/test/sqlcipher-integrity.test
@@ -310,7 +310,7 @@ do_test version-4-integrity-check-invalid-last-page {
     PRAGMA key = 'testkey';
     PRAGMA cipher_integrity_check;
   } 
-} {ok {page 240 has an invalid size of 2 bytes}}
+} {ok {page 240 has an invalid size of 2 bytes (expected 4096 bytes)}}
 db close
 file delete -force test.db
 
@@ -339,6 +339,7 @@ do_test integrity-check-plaintext-header {
     PRAGMA cipher_integrity_check;
   }]
 } {{} 1 {{HMAC verification failed for page 1} {HMAC verification failed for page 2}}}
+db close
 file delete -force test.db
 
 # test that changing the key in the middle of database operations does
diff --git a/test/sqlcipher-plaintext-header.test b/test/sqlcipher-plaintext-header.test
index 81e1a82312..023d414549 100644
--- a/test/sqlcipher-plaintext-header.test
+++ b/test/sqlcipher-plaintext-header.test
@@ -60,6 +60,7 @@ do_test test-pragma-salt-get {
   set header [string tolower [hexio_read test.db 0 16]]
   string equal $header $salt
 } {1}
+db close
 file delete -force test.db
 
 # explicitly set the salt of a new database 
@@ -84,6 +85,7 @@ do_test test-pragma-salt-set {
   "]
 
 } {01010101010101010101010101010101 {ok 1 01010101010101010101010101010101}}
+db close
 file delete -force test.db
 
 
diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test
index 2578277295..2ad946bd0d 100644
--- a/test/sqlcipher-pragmas.test
+++ b/test/sqlcipher-pragmas.test
@@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version {
     execsql {
         PRAGMA cipher_version;
     }
-} {{4.5.5 community}}
+} {{4.13.0 community}}
 db close
 file delete -force test.db
 
@@ -202,6 +202,35 @@ do_test verify-pragma-cipher-page-size-changed {
 db close
 file delete -force test.db
 
+# verify that a call to pragma page_size
+# will report change via both page_size and cipher_page_size
+# when there is an attached codec
+do_test verify-pragma-page-size-encrypted {
+    sqlite_orig db test.db
+    execsql {
+        PRAGMA key = 'test';
+        PRAGMA page_size = 8192;
+        PRAGMA page_size;
+        PRAGMA cipher_page_size;
+    }
+} {ok 8192 8192}
+db close
+file delete -force test.db
+
+# verify that a call to pragma page_size
+# will not report a change to cipher_page_size for an
+# unencrypted database
+do_test verify-pragma-page-size-plaintext {
+    sqlite_orig db test.db
+    execsql {
+        PRAGMA page_size = 8192;
+        PRAGMA page_size;
+        PRAGMA cipher_page_size;
+    }
+} {8192}
+db close
+file delete -force test.db
+
 # verify setting cipher_store_pass before key
 # does not cause segfault
 do_test verify-cipher-store-pass-before-key-does-not-segfault {
@@ -439,6 +468,32 @@ do_test verify-cipher_default_settings_default {
 db close
 file delete -force test.db
 
+do_test verify-cipher_log_source {
+    sqlite_orig db :memory:
+    execsql {
+        PRAGMA cipher_log_source; -- default should be ANY
+        PRAGMA cipher_log_source = NONE; --reset to NONE
+        PRAGMA cipher_log_source = PROVIDER; -- add PROVIDER to log source
+        PRAGMA cipher_log_source = CORE; -- add CORE to log source
+        PRAGMA cipher_log_source = MEMORY; -- add MEMORY to log source
+        PRAGMA cipher_log_source = NOTASOURCE; -- stay the same
+        PRAGMA cipher_log_source = ANY; -- reset to ANY
+    }
+} {ANY NONE PROVIDER {CORE PROVIDER} {CORE MEMORY PROVIDER} {CORE MEMORY PROVIDER} ANY}
+db close
 
+do_test verify-cipher_log_level {
+    sqlite_orig db :memory:
+    execsql {
+        PRAGMA cipher_log_level; -- default should be WARN
+        PRAGMA cipher_log_level = TRACE;
+        PRAGMA cipher_log_level = DEBUG;
+        PRAGMA cipher_log_level = INFO;
+        PRAGMA cipher_log_level = WARN;
+        PRAGMA cipher_log_level = ERROR;
+        PRAGMA cipher_log_level = NOTALEVEL; -- an unknown level should set back to none
+    }
+} {WARN TRACE DEBUG INFO WARN ERROR NONE}
+db close
 
 finish_test
diff --git a/test/sqlcipher-zmemory.test b/test/sqlcipher-zmemory.test
new file mode 100644
index 0000000000..272fccecfe
--- /dev/null
+++ b/test/sqlcipher-zmemory.test
@@ -0,0 +1,61 @@
+
+# SQLCipher
+# codec.test developed by Stephen Lombardo (Zetetic LLC)
+# sjlombardo at zetetic dot net
+# http://zetetic.net
+#
+# Copyright (c) 2018, ZETETIC LLC
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of the ZETETIC LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing code cipher features.
+#
+# NOTE: tester.tcl has overridden the definition of sqlite3 to
+# automatically pass in a key value. Thus tests in this file
+# should explicitly close and open db with sqlite_orig in order
+# to bypass default key assignment.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/sqlcipher.tcl
+
+# verify memory security behavior
+# initially should report OFF
+# then enable, check that it is ON
+# try to turn if off, but verify that it
+# can't be unset.
+do_test verify-memory-security {
+    sqlite_orig db test.db
+    execsql {
+        PRAGMA cipher_memory_security;
+        PRAGMA cipher_memory_security = ON;
+        PRAGMA cipher_memory_security;
+        PRAGMA cipher_memory_security = OFF;
+        PRAGMA cipher_memory_security;
+    }
+} {0 1 1}
+db close
+file delete -force test.db
+
+finish_test
+
diff --git a/test/sqlcipher.tcl b/test/sqlcipher.tcl
index 4129dca2ff..4752316c69 100644
--- a/test/sqlcipher.tcl
+++ b/test/sqlcipher.tcl
@@ -34,13 +34,13 @@
 # to bypass default key assignment.
 
 
-file delete -force test.db
+set testdir [file dirname $argv0]
+set sampleDir [file normalize [file dirname [file dirname $argv0]]]/sqlcipher-resources
+
+catch {file delete -force test.db}
 file delete -force test2.db
 file delete -force test3.db
 file delete -force test4.db
-                          
-set testdir [file dirname $argv0]
-set sampleDir [file normalize [file dirname [file dirname $argv0]]]
 
 # If the library is not compiled with has_codec support then
 # skip all tests in this file.
diff --git a/test/sqlcipher.test b/test/sqlcipher.test
index 845bc23f24..7d008f8d91 100644
--- a/test/sqlcipher.test
+++ b/test/sqlcipher.test
@@ -41,6 +41,12 @@ source $testdir/permutations.test
 
 set pretests ""
 
+sqlite_orig db :memory:
+execsql {
+  PRAGMA cipher_log = 'sqlcipher-test.log';
+}
+db close
+
 test_suite "sqlcipher" -prefix "" -description {
   Runs SQLCipher tests
 } -files [
@@ -52,7 +58,8 @@ test_suite "sqlcipher" -prefix "" -description {
            sqlcipher-pragmas.test \
            sqlcipher-integrity.test \
            sqlcipher-codecerror.test \
-           sqlcipher-backup.test
+           sqlcipher-backup.test \
+           sqlcipher-zmemory.test
 ]
 run_test_suite sqlcipher
 finish_test
diff --git a/test/sqldiff1.test b/test/sqldiff1.test
index 4ea5efbbfa..5a3c11cb9c 100644
--- a/test/sqldiff1.test
+++ b/test/sqldiff1.test
@@ -63,4 +63,26 @@ CREATE TABLE t3(a,b,c);
 INSERT INTO t3(rowid,a,b,c) VALUES(1,111,222,333);
 DROP TABLE t4;}
 
+db close
+forcedelete test.db test2.db
+sqlite3 db test.db
+
+do_test sqldiff-2.0 {
+  db eval {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY);
+  }
+  db close
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  }
+  db close
+  set line "exec $PROG test.db test2.db"
+  unset -nocomplain ::MSG
+  catch {eval $line} ::MSG
+} {0}
+do_test sqldiff-2.1 {
+  set ::MSG
+} {ALTER TABLE t1 ADD COLUMN b;}
+
 finish_test
diff --git a/test/sqllimits1.test b/test/sqllimits1.test
index f16208f234..efe656db6a 100644
--- a/test/sqllimits1.test
+++ b/test/sqllimits1.test
@@ -75,6 +75,13 @@ do_test sqllimits1-1.23 {
   sqlite3_limit db SQLITE_LIMIT_TOOBIG 123
 } {-1}
 
+# Minimum value for SQLITE_LIMIT_LENGTH is 30
+#
+do_test sqllimits1-1.30 {
+  set prior [sqlite3_limit db SQLITE_LIMIT_LENGTH 1]
+  sqlite3_limit db SQLITE_LIMIT_LENGTH $prior
+} 30
+
 
 # Decrease all limits by half.  Verify that the new limits take.
 #
@@ -707,6 +714,7 @@ if {$SQLITE_MAX_EXPR_DEPTH==0} {
     }]
   } "1 {Expression tree is too large (maximum depth $::SQLITE_MAX_EXPR_DEPTH)}"
   
+if 0 {  
   # Attempting to beat the expression depth limit using nested SELECT
   # queries causes a parser stack overflow. 
   do_test sqllimits1-9.2 {
@@ -718,7 +726,6 @@ if {$SQLITE_MAX_EXPR_DEPTH==0} {
     catchsql [subst { $expr }]
   } "1 {parser stack overflow}"
   
-if 0 {  
   do_test sqllimits1-9.3 {
     execsql {
       PRAGMA max_page_count = 1000000;  -- 1 GB
@@ -922,7 +929,7 @@ do_catchsql_test sqllimits1-18.1 {
 do_catchsql_test sqllimits1-18.2 {
   INSERT INTO b1 VALUES(1), (2), (3), (4), (5), (6), (7), (8), (9), (10)
     UNION VALUES(11);
-} {1 {too many terms in compound SELECT}}
+} {0 {}}
 
 #-------------------------------------------------------------------------
 #
diff --git a/test/starschema1.test b/test/starschema1.test
new file mode 100644
index 0000000000..bb7d8aa79b
--- /dev/null
+++ b/test/starschema1.test
@@ -0,0 +1,584 @@
+# 2024-05-28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# Test cases for the ability of the query planner to cope with
+# star-schema queries.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix starschema1
+
+do_execsql_test 1.1 {
+CREATE TABLE t1(
+  a01 INT, a02 INT, a03 INT, a04 INT, a05 INT, a06 INT, a07 INT, a08 INT,
+  a09 INT, a10 INT, a11 INT, a12 INT, a13 INT, a14 INT, a15 INT, a16 INT,
+  a17 INT, a18 INT, a19 INT, a20 INT, a21 INT, a22 INT, a23 INT, a24 INT,
+  a25 INT, a26 INT, a27 INT, a28 INT, a29 INT, a30 INT, a31 INT, a32 INT,
+  a33 INT, a34 INT, a35 INT, a36 INT, a37 INT, a38 INT, a39 INT, a40 INT,
+  a41 INT, a42 INT, a43 INT, a44 INT, a45 INT, a46 INT, a47 INT, a48 INT,
+  a49 INT, a50 INT, a51 INT, a52 INT, a53 INT, a54 INT, a55 INT, a56 INT,
+  a57 INT, a58 INT, a59 INT, a60 INT, a61 INT, a62 INT, a63 INT, d TEXT);
+CREATE TABLE x01(b01 INT, c01 TEXT);
+CREATE TABLE x02(b02 INT, c02 TEXT);
+CREATE TABLE x03(b03 INT, c03 TEXT);
+CREATE TABLE x04(b04 INT, c04 TEXT);
+CREATE TABLE x05(b05 INT, c05 TEXT);
+CREATE TABLE x06(b06 INT, c06 TEXT);
+CREATE TABLE x07(b07 INT, c07 TEXT);
+CREATE TABLE x08(b08 INT, c08 TEXT);
+CREATE TABLE x09(b09 INT, c09 TEXT);
+CREATE TABLE x10(b10 INT, c10 TEXT);
+CREATE TABLE x11(b11 INT, c11 TEXT);
+CREATE TABLE x12(b12 INT, c12 TEXT);
+CREATE TABLE x13(b13 INT, c13 TEXT);
+CREATE TABLE x14(b14 INT, c14 TEXT);
+CREATE TABLE x15(b15 INT, c15 TEXT);
+CREATE TABLE x16(b16 INT, c16 TEXT);
+CREATE TABLE x17(b17 INT, c17 TEXT);
+CREATE TABLE x18(b18 INT, c18 TEXT);
+CREATE TABLE x19(b19 INT, c19 TEXT);
+CREATE TABLE x20(b20 INT, c20 TEXT);
+CREATE TABLE x21(b21 INT, c21 TEXT);
+CREATE TABLE x22(b22 INT, c22 TEXT);
+CREATE TABLE x23(b23 INT, c23 TEXT);
+CREATE TABLE x24(b24 INT, c24 TEXT);
+CREATE TABLE x25(b25 INT, c25 TEXT);
+CREATE TABLE x26(b26 INT, c26 TEXT);
+CREATE TABLE x27(b27 INT, c27 TEXT);
+CREATE TABLE x28(b28 INT, c28 TEXT);
+CREATE TABLE x29(b29 INT, c29 TEXT);
+CREATE TABLE x30(b30 INT, c30 TEXT);
+CREATE TABLE x31(b31 INT, c31 TEXT);
+CREATE TABLE x32(b32 INT, c32 TEXT);
+CREATE TABLE x33(b33 INT, c33 TEXT);
+CREATE TABLE x34(b34 INT, c34 TEXT);
+CREATE TABLE x35(b35 INT, c35 TEXT);
+CREATE TABLE x36(b36 INT, c36 TEXT);
+CREATE TABLE x37(b37 INT, c37 TEXT);
+CREATE TABLE x38(b38 INT, c38 TEXT);
+CREATE TABLE x39(b39 INT, c39 TEXT);
+CREATE TABLE x40(b40 INT, c40 TEXT);
+CREATE TABLE x41(b41 INT, c41 TEXT);
+CREATE TABLE x42(b42 INT, c42 TEXT);
+CREATE TABLE x43(b43 INT, c43 TEXT);
+CREATE TABLE x44(b44 INT, c44 TEXT);
+CREATE TABLE x45(b45 INT, c45 TEXT);
+CREATE TABLE x46(b46 INT, c46 TEXT);
+CREATE TABLE x47(b47 INT, c47 TEXT);
+CREATE TABLE x48(b48 INT, c48 TEXT);
+CREATE TABLE x49(b49 INT, c49 TEXT);
+CREATE TABLE x50(b50 INT, c50 TEXT);
+CREATE TABLE x51(b51 INT, c51 TEXT);
+CREATE TABLE x52(b52 INT, c52 TEXT);
+CREATE TABLE x53(b53 INT, c53 TEXT);
+CREATE TABLE x54(b54 INT, c54 TEXT);
+CREATE TABLE x55(b55 INT, c55 TEXT);
+CREATE TABLE x56(b56 INT, c56 TEXT);
+CREATE TABLE x57(b57 INT, c57 TEXT);
+CREATE TABLE x58(b58 INT, c58 TEXT);
+CREATE TABLE x59(b59 INT, c59 TEXT);
+CREATE TABLE x60(b60 INT, c60 TEXT);
+CREATE TABLE x61(b61 INT, c61 TEXT);
+CREATE TABLE x62(b62 INT, c62 TEXT);
+CREATE TABLE x63(b63 INT, c63 TEXT);
+/****  Uncomment to generate actual data ************************************
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<172800)
+  INSERT INTO t1
+    SELECT stmtrand()%12, stmtrand()%13, stmtrand()%14, stmtrand()%15,
+           stmtrand()%16, stmtrand()%17, stmtrand()%18, stmtrand()%19,
+           stmtrand()%20, stmtrand()%21, stmtrand()%22, stmtrand()%23,
+           stmtrand()%24, stmtrand()%25, stmtrand()%26, stmtrand()%27,
+           stmtrand()%28, stmtrand()%29, stmtrand()%30, stmtrand()%31,
+           stmtrand()%32, stmtrand()%33, stmtrand()%34, stmtrand()%35,
+           stmtrand()%36, stmtrand()%37, stmtrand()%38, stmtrand()%39,
+           stmtrand()%40, stmtrand()%41, stmtrand()%42, stmtrand()%43,
+           stmtrand()%28, stmtrand()%29, stmtrand()%30, stmtrand()%31,
+           stmtrand()%32, stmtrand()%33, stmtrand()%34, stmtrand()%35,
+           stmtrand()%36, stmtrand()%37, stmtrand()%38, stmtrand()%39,
+           stmtrand()%40, stmtrand()%41, stmtrand()%42, stmtrand()%43,
+           stmtrand()%28, stmtrand()%29, stmtrand()%30, stmtrand()%31,
+           stmtrand()%32, stmtrand()%33, stmtrand()%34, stmtrand()%35,
+           stmtrand()%36, stmtrand()%37, stmtrand()%38, stmtrand()%39,
+           stmtrand()%40, stmtrand()%41, stmtrand()%42, stmtrand() FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8)
+  INSERT INTO x01 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12)
+  INSERT INTO x02 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16)
+  INSERT INTO x03 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20)
+  INSERT INTO x04 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24)
+  INSERT INTO x05 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32)
+  INSERT INTO x06 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36)
+  INSERT INTO x07 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40)
+  INSERT INTO x08 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44)
+  INSERT INTO x09 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48)
+  INSERT INTO x10 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52)
+  INSERT INTO x11 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56)
+  INSERT INTO x12 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60)
+  INSERT INTO x13 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64)
+  INSERT INTO x14 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72)
+  INSERT INTO x15 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<80)
+  INSERT INTO x16 SELECT n%40, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8)
+  INSERT INTO x17 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12)
+  INSERT INTO x18 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16)
+  INSERT INTO x19 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20)
+  INSERT INTO x20 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24)
+  INSERT INTO x21 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32)
+  INSERT INTO x22 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36)
+  INSERT INTO x23 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40)
+  INSERT INTO x24 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44)
+  INSERT INTO x25 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48)
+  INSERT INTO x26 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52)
+  INSERT INTO x27 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56)
+  INSERT INTO x28 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60)
+  INSERT INTO x29 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64)
+  INSERT INTO x30 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72)
+  INSERT INTO x31 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<80)
+  INSERT INTO x32 SELECT n%40, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8)
+  INSERT INTO x33 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12)
+  INSERT INTO x34 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16)
+  INSERT INTO x35 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20)
+  INSERT INTO x36 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24)
+  INSERT INTO x37 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32)
+  INSERT INTO x38 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36)
+  INSERT INTO x39 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40)
+  INSERT INTO x40 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44)
+  INSERT INTO x41 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48)
+  INSERT INTO x42 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52)
+  INSERT INTO x43 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56)
+  INSERT INTO x44 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60)
+  INSERT INTO x45 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64)
+  INSERT INTO x46 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72)
+  INSERT INTO x47 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<80)
+  INSERT INTO x48 SELECT n%40, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<8)
+  INSERT INTO x49 SELECT n%4, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<12)
+  INSERT INTO x50 SELECT n%6, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<16)
+  INSERT INTO x51 SELECT n%8, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<20)
+  INSERT INTO x52 SELECT n%10, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<24)
+  INSERT INTO x53 SELECT n%12, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<32)
+  INSERT INTO x54 SELECT n%16, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<36)
+  INSERT INTO x55 SELECT n%18, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<40)
+  INSERT INTO x56 SELECT n%20, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<44)
+  INSERT INTO x57 SELECT n%22, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<48)
+  INSERT INTO x58 SELECT n%24, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<52)
+  INSERT INTO x59 SELECT n%26, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<56)
+  INSERT INTO x60 SELECT n%28, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<60)
+  INSERT INTO x61 SELECT n%30, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<64)
+  INSERT INTO x62 SELECT n%32, format('%d-or-0x%04x',n,n) FROM c;
+WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<72)
+  INSERT INTO x63 SELECT n%36, format('%d-or-0x%04x',n,n) FROM c;
+****************************************************************************/
+CREATE INDEX t1a01 ON t1(a01);
+CREATE INDEX t1a02 ON t1(a02);
+CREATE INDEX t1a03 ON t1(a03);
+CREATE INDEX t1a04 ON t1(a04);
+CREATE INDEX t1a05 ON t1(a05);
+CREATE INDEX t1a06 ON t1(a06);
+CREATE INDEX t1a07 ON t1(a07);
+CREATE INDEX t1a08 ON t1(a08);
+CREATE INDEX t1a09 ON t1(a09);
+CREATE INDEX t1a10 ON t1(a10);
+CREATE INDEX t1a11 ON t1(a11);
+CREATE INDEX t1a12 ON t1(a12);
+CREATE INDEX t1a13 ON t1(a13);
+CREATE INDEX t1a14 ON t1(a14);
+CREATE INDEX t1a15 ON t1(a15);
+CREATE INDEX t1a16 ON t1(a16);
+CREATE INDEX t1a17 ON t1(a17);
+CREATE INDEX t1a18 ON t1(a18);
+CREATE INDEX t1a19 ON t1(a19);
+CREATE INDEX t1a20 ON t1(a20);
+CREATE INDEX t1a21 ON t1(a21);
+CREATE INDEX t1a22 ON t1(a22);
+CREATE INDEX t1a23 ON t1(a23);
+CREATE INDEX t1a24 ON t1(a24);
+CREATE INDEX t1a25 ON t1(a25);
+CREATE INDEX t1a26 ON t1(a26);
+CREATE INDEX t1a27 ON t1(a27);
+CREATE INDEX t1a28 ON t1(a28);
+CREATE INDEX t1a29 ON t1(a29);
+CREATE INDEX t1a30 ON t1(a30);
+CREATE INDEX t1a31 ON t1(a31);
+CREATE INDEX t1a32 ON t1(a32);
+CREATE INDEX t1a33 ON t1(a33);
+CREATE INDEX t1a34 ON t1(a34);
+CREATE INDEX t1a35 ON t1(a35);
+CREATE INDEX t1a36 ON t1(a36);
+CREATE INDEX t1a37 ON t1(a37);
+CREATE INDEX t1a38 ON t1(a38);
+CREATE INDEX t1a39 ON t1(a39);
+CREATE INDEX t1a40 ON t1(a40);
+CREATE INDEX t1a41 ON t1(a41);
+CREATE INDEX t1a42 ON t1(a42);
+CREATE INDEX t1a43 ON t1(a43);
+CREATE INDEX t1a44 ON t1(a44);
+CREATE INDEX t1a45 ON t1(a45);
+CREATE INDEX t1a46 ON t1(a46);
+CREATE INDEX t1a47 ON t1(a47);
+CREATE INDEX t1a48 ON t1(a48);
+CREATE INDEX t1a49 ON t1(a49);
+CREATE INDEX t1a50 ON t1(a50);
+CREATE INDEX t1a51 ON t1(a51);
+CREATE INDEX t1a52 ON t1(a52);
+CREATE INDEX t1a53 ON t1(a53);
+CREATE INDEX t1a54 ON t1(a54);
+CREATE INDEX t1a55 ON t1(a55);
+CREATE INDEX t1a56 ON t1(a56);
+CREATE INDEX t1a57 ON t1(a57);
+CREATE INDEX t1a58 ON t1(a58);
+CREATE INDEX t1a59 ON t1(a59);
+CREATE INDEX t1a60 ON t1(a60);
+CREATE INDEX t1a61 ON t1(a61);
+CREATE INDEX t1a62 ON t1(a62);
+CREATE INDEX t1a63 ON t1(a63);
+CREATE INDEX x01x ON x01(b01);
+CREATE INDEX x02x ON x02(b02);
+CREATE INDEX x03x ON x03(b03);
+CREATE INDEX x04x ON x04(b04);
+CREATE INDEX x05x ON x05(b05);
+CREATE INDEX x06x ON x06(b06);
+CREATE INDEX x07x ON x07(b07);
+CREATE INDEX x08x ON x08(b08);
+CREATE INDEX x09x ON x09(b09);
+CREATE INDEX x10x ON x10(b10);
+CREATE INDEX x11x ON x11(b11);
+CREATE INDEX x12x ON x12(b12);
+CREATE INDEX x13x ON x13(b13);
+CREATE INDEX x14x ON x14(b14);
+CREATE INDEX x15x ON x15(b15);
+CREATE INDEX x16x ON x16(b16);
+CREATE INDEX x17x ON x17(b17);
+CREATE INDEX x18x ON x18(b18);
+CREATE INDEX x19x ON x19(b19);
+CREATE INDEX x20x ON x20(b20);
+CREATE INDEX x21x ON x21(b21);
+CREATE INDEX x22x ON x22(b22);
+CREATE INDEX x23x ON x23(b23);
+CREATE INDEX x24x ON x24(b24);
+CREATE INDEX x25x ON x25(b25);
+CREATE INDEX x26x ON x26(b26);
+CREATE INDEX x27x ON x27(b27);
+CREATE INDEX x28x ON x28(b28);
+CREATE INDEX x29x ON x29(b29);
+CREATE INDEX x30x ON x30(b30);
+CREATE INDEX x31x ON x31(b31);
+CREATE INDEX x32x ON x32(b32);
+CREATE INDEX x33x ON x33(b33);
+CREATE INDEX x34x ON x34(b34);
+CREATE INDEX x35x ON x35(b35);
+CREATE INDEX x36x ON x36(b36);
+CREATE INDEX x37x ON x37(b37);
+CREATE INDEX x38x ON x38(b38);
+CREATE INDEX x39x ON x39(b39);
+CREATE INDEX x40x ON x40(b40);
+CREATE INDEX x41x ON x41(b41);
+CREATE INDEX x42x ON x42(b42);
+CREATE INDEX x43x ON x43(b43);
+CREATE INDEX x44x ON x44(b44);
+CREATE INDEX x45x ON x45(b45);
+CREATE INDEX x46x ON x46(b46);
+CREATE INDEX x47x ON x47(b47);
+CREATE INDEX x48x ON x48(b48);
+CREATE INDEX x49x ON x49(b49);
+CREATE INDEX x50x ON x50(b50);
+CREATE INDEX x51x ON x51(b51);
+CREATE INDEX x52x ON x52(b52);
+CREATE INDEX x53x ON x53(b53);
+CREATE INDEX x54x ON x54(b54);
+CREATE INDEX x55x ON x55(b55);
+CREATE INDEX x56x ON x56(b56);
+CREATE INDEX x57x ON x57(b57);
+CREATE INDEX x58x ON x58(b58);
+CREATE INDEX x59x ON x59(b59);
+CREATE INDEX x60x ON x60(b60);
+CREATE INDEX x61x ON x61(b61);
+CREATE INDEX x62x ON x62(b62);
+CREATE INDEX x63x ON x63(b63);
+ANALYZE sqlite_schema;
+INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES
+  ('t1','t1a01','172800 14400'),
+  ('t1','t1a02','172800 13293'),
+  ('t1','t1a03','172800 12343'),
+  ('t1','t1a04','172800 11520'),
+  ('t1','t1a05','172800 10800'),
+  ('t1','t1a06','172800 10165'),
+  ('t1','t1a07','172800 9600'),
+  ('t1','t1a08','172800 9095'),
+  ('t1','t1a09','172800 8640'),
+  ('t1','t1a10','172800 8229'),
+  ('t1','t1a11','172800 7855'),
+  ('t1','t1a12','172800 7514'),
+  ('t1','t1a13','172800 7200'),
+  ('t1','t1a14','172800 6912'),
+  ('t1','t1a15','172800 6647'),
+  ('t1','t1a16','172800 6400'),
+  ('t1','t1a17','172800 6172'),
+  ('t1','t1a18','172800 5959'),
+  ('t1','t1a19','172800 5760'),
+  ('t1','t1a20','172800 5575'),
+  ('t1','t1a21','172800 5400'),
+  ('t1','t1a22','172800 5237'),
+  ('t1','t1a23','172800 5083'),
+  ('t1','t1a24','172800 4938'),
+  ('t1','t1a25','172800 4800'),
+  ('t1','t1a26','172800 4671'),
+  ('t1','t1a27','172800 4548'),
+  ('t1','t1a28','172800 4431'),
+  ('t1','t1a29','172800 4320'),
+  ('t1','t1a30','172800 4215'),
+  ('t1','t1a31','172800 4115'),
+  ('t1','t1a32','172800 4019'),
+  ('t1','t1a33','172800 6172'),
+  ('t1','t1a34','172800 5959'),
+  ('t1','t1a35','172800 5760'),
+  ('t1','t1a36','172800 5575'),
+  ('t1','t1a37','172800 5400'),
+  ('t1','t1a38','172800 5237'),
+  ('t1','t1a39','172800 5083'),
+  ('t1','t1a40','172800 4938'),
+  ('t1','t1a41','172800 4800'),
+  ('t1','t1a42','172800 4671'),
+  ('t1','t1a43','172800 4548'),
+  ('t1','t1a44','172800 4431'),
+  ('t1','t1a45','172800 4320'),
+  ('t1','t1a46','172800 4215'),
+  ('t1','t1a47','172800 4115'),
+  ('t1','t1a48','172800 4019'),
+  ('t1','t1a49','172800 6172'),
+  ('t1','t1a50','172800 5959'),
+  ('t1','t1a51','172800 5760'),
+  ('t1','t1a52','172800 5575'),
+  ('t1','t1a53','172800 5400'),
+  ('t1','t1a54','172800 5237'),
+  ('t1','t1a55','172800 5083'),
+  ('t1','t1a56','172800 4938'),
+  ('t1','t1a57','172800 4800'),
+  ('t1','t1a58','172800 4671'),
+  ('t1','t1a59','172800 4548'),
+  ('t1','t1a60','172800 4431'),
+  ('t1','t1a61','172800 4320'),
+  ('t1','t1a62','172800 4215'),
+  ('t1','t1a63','172800 4115'),
+  ('x01','x01x','80 2'),
+  ('x02','x02x','120 2'),
+  ('x03','x03x','160 2'),
+  ('x04','x04x','20 2'),
+  ('x05','x05x','24 2'),
+  ('x06','x06x','32 2'),
+  ('x07','x07x','36 2'),
+  ('x08','x08x','40 2'),
+  ('x09','x09x','44 2'),
+  ('x10','x10x','48 2'),
+  ('x11','x11x','52 2'),
+  ('x12','x12x','56 2'),
+  ('x13','x13x','60 2'),
+  ('x14','x14x','64 2'),
+  ('x15','x15x','72 2'),
+  ('x16','x16x','80 2'),
+  ('x17','x17x','80 2'),
+  ('x18','x18x','120 2'),
+  ('x19','x19x','160 2'),
+  ('x20','x20x','20 2'),
+  ('x21','x21x','24 2'),
+  ('x22','x22x','32 2'),
+  ('x23','x23x','36 2'),
+  ('x24','x24x','40 2'),
+  ('x25','x25x','44 2'),
+  ('x26','x26x','48 2'),
+  ('x27','x27x','52 2'),
+  ('x28','x28x','56 2'),
+  ('x29','x29x','60 2'),
+  ('x30','x30x','64 2'),
+  ('x31','x31x','72 2'),
+  ('x32','x32x','80 2'),
+  ('x33','x33x','80 2'),
+  ('x34','x34x','120 2'),
+  ('x35','x35x','160 2'),
+  ('x36','x36x','20 2'),
+  ('x37','x37x','24 2'),
+  ('x38','x38x','32 2'),
+  ('x39','x39x','36 2'),
+  ('x40','x40x','40 2'),
+  ('x41','x41x','44 2'),
+  ('x42','x42x','48 2'),
+  ('x43','x43x','52 2'),
+  ('x44','x44x','56 2'),
+  ('x45','x45x','60 2'),
+  ('x46','x46x','64 2'),
+  ('x47','x47x','72 2'),
+  ('x48','x48x','80 2'),
+  ('x49','x49x','80 2'),
+  ('x50','x50x','120 2'),
+  ('x51','x51x','160 2'),
+  ('x52','x52x','20 2'),
+  ('x53','x53x','24 2'),
+  ('x54','x54x','32 2'),
+  ('x55','x55x','36 2'),
+  ('x56','x56x','40 2'),
+  ('x57','x57x','44 2'),
+  ('x58','x58x','48 2'),
+  ('x59','x59x','52 2'),
+  ('x60','x60x','56 2'),
+  ('x61','x61x','60 2'),
+  ('x62','x62x','64 2'),
+  ('x63','x63x','72 2');
+ANALYZE sqlite_schema;
+}
+do_execsql_test 1.2 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03
+    FROM t1, x01, x02, x03
+   WHERE a01=b01 AND a02=b02 AND a03=b03;
+} {/SCAN t1.*SEARCH.*SEARCH.*SEARCH/}  
+do_execsql_test 1.3 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04
+    FROM t1, x01, x02, x03, x04
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04;
+} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH /}  
+do_execsql_test 1.4 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04, c05
+    FROM t1, x01, x02, x03, x04, x05
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05;
+} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH/}  
+do_execsql_test 1.5 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04, c05, c06
+    FROM t1, x01, x02, x03, x04, x05, x06
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05
+     AND a06=b06;
+} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH/}  
+do_execsql_test 1.6 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04, c05, c06, c07
+    FROM t1, x01, x02, x03, x04, x05, x06, x07
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05
+     AND a06=b06 AND a07=b07;
+} {/SCAN .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH .*SEARCH/}  
+do_execsql_test 1.7 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04, c05, c06, c07, c08
+    FROM t1, x01, x02, x03, x04, x05, x06, x07, x08
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05
+     AND a06=b06 AND a07=b07 AND a08=b08;
+} {~/SCAN.*SCAN/}
+do_execsql_test 1.8 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04, c05, c06, c07, c08,
+         c09, c10, c11, c12, c13, c14, c15, c16,
+         c17, c18, c19, c20, c21, c22, c23, c24,
+         c25, c26, c27, c28, c29, c30, c31, c32
+    FROM t1, x01, x02, x03, x04, x05, x06, x07, x08,
+             x09, x10, x11, x12, x13, x14, x15, x16,
+             x17, x18, x19, x20, x21, x22, x23, x24,
+             x25, x26, x27, x28, x29, x30, x31, x32
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 AND a06=b06
+     AND a07=b07 AND a08=b08 AND a09=b09 AND a10=b10 AND a11=b11 AND a12=b12
+     AND a13=b13 AND a14=b14 AND a15=b15 AND a16=b16 AND a17=b17 AND a18=b18
+     AND a19=b19 AND a20=b20 AND a21=b21 AND a22=b22 AND a23=b23 AND a24=b24
+     AND a25=b25 AND a26=b26 AND a27=b27 AND a28=b28 AND a29=b29 AND a30=b30
+     AND a31=b31 AND a32=b32;
+} {~/SCAN.*SCAN/}
+do_execsql_test 1.9 {
+  EXPLAIN QUERY PLAN
+  SELECT c01, c02, c03, c04, c05, c06, c07, c08,
+         c09, c10, c11, c12, c13, c14, c15, c16,
+         c17, c18, c19, c20, c21, c22, c23, c24,
+         c25, c26, c27, c28, c29, c30, c31, c32,
+         c33, c34, c35, c36, c37, c38, c39, c40,
+         c41, c42, c43, c44, c45, c46, c47, c48,
+         c49, c50, c51, c52, c53, c54, c55, c56,
+         c57, c58, c59, c60, c61, c62, c63
+    FROM t1, x01, x02, x03, x04, x05, x06, x07, x08,
+             x09, x10, x11, x12, x13, x14, x15, x16,
+             x17, x18, x19, x20, x21, x22, x23, x24,
+             x25, x26, x27, x28, x29, x30, x31, x32,
+             x33, x34, x35, x36, x37, x38, x39, x40,
+             x41, x42, x43, x44, x45, x46, x47, x48,
+             x49, x50, x51, x52, x53, x54, x55, x56,
+             x57, x58, x59, x60, x61, x62, x63
+   WHERE a01=b01 AND a02=b02 AND a03=b03 AND a04=b04 AND a05=b05 AND a06=b06
+     AND a07=b07 AND a08=b08 AND a09=b09 AND a10=b10 AND a11=b11 AND a12=b12
+     AND a13=b13 AND a14=b14 AND a15=b15 AND a16=b16 AND a17=b17 AND a18=b18
+     AND a19=b19 AND a20=b20 AND a21=b21 AND a22=b22 AND a23=b23 AND a24=b24
+     AND a25=b25 AND a26=b26 AND a27=b27 AND a28=b28 AND a29=b29 AND a30=b30
+     AND a31=b31 AND a32=b32 AND a33=b33 AND a34=b34 AND a35=b35 AND a36=b36
+     AND a37=b37 AND a38=b38 AND a39=b39 AND a40=b40 AND a41=b41 AND a42=b42
+     AND a43=b43 AND a44=b44 AND a45=b45 AND a46=b46 AND a47=b47 AND a48=b48
+     AND a49=b49 AND a50=b50 AND a51=b51 AND a52=b52 AND a53=b53 AND a54=b54
+     AND a55=b55 AND a56=b56 AND a57=b57 AND a58=b58 AND a59=b59 AND a60=b60
+     AND a61=b61 AND a62=b62 AND a63=b63;
+} {~/SCAN.*SCAN/}
+
+
+
+finish_test
diff --git a/test/statfault.test b/test/statfault.test
index b5980d417d..19e0a67874 100644
--- a/test/statfault.test
+++ b/test/statfault.test
@@ -52,4 +52,3 @@ do_faultsim_test 2 -faults * -prep {
 }
 
 finish_test
-
diff --git a/test/stmtrand.test b/test/stmtrand.test
new file mode 100644
index 0000000000..e2f6935b85
--- /dev/null
+++ b/test/stmtrand.test
@@ -0,0 +1,48 @@
+# 2024-05-24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Verify that the stmtrand() extension function works.
+#
+# Stmtrand() is a pseudo-random number generator designed for testing.
+# it has the property that it returns the same sequence of pseudo-random
+# numbers for each SQL statement invocation.  This makes it suitable for
+# testing because the results are reproducible.
+#
+# The optional argument is the seed for the PRNG.  The seed is only used
+# on the first invocation.  If the argument is omitted, a seed of 0 is
+# used.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix stmtrand
+
+load_static_extension db stmtrand
+
+do_execsql_test 1.1 {
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30)
+  SELECT stmtrand()%10 FROM c
+} {4 1 8 3 6 1 4 9 2 1 2 1 0 1 8 1 4 7 6 1 8 7 0 3 0 9 4 5 9 9}
+do_execsql_test 1.2 {
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30)
+  SELECT stmtrand(0)%10 FROM c
+} {4 1 8 3 6 1 4 9 2 1 2 1 0 1 8 1 4 7 6 1 8 7 0 3 0 9 4 5 9 9}
+do_execsql_test 1.3 {
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30)
+  SELECT stmtrand(1)%10 FROM c
+} {3 8 9 4 7 2 3 8 5 2 7 0 3 0 7 2 5 6 1 8 5 6 7 8 7 2 9 6 8 0}
+do_execsql_test 1.4 {
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<30)
+  SELECT stmtrand(x)%10 FROM c
+} {3 8 9 4 7 2 3 8 5 2 7 0 3 0 7 2 5 6 1 8 5 6 7 8 7 2 9 6 8 0}
+
+
+finish_test
diff --git a/test/strict1.test b/test/strict1.test
index fc6438843f..c4c086bdb9 100644
--- a/test/strict1.test
+++ b/test/strict1.test
@@ -162,4 +162,101 @@ do_execsql_test strict1-8.2 {
   SELECT *, '|' FROM t1;
 } {/5.0 5.0 4.6116\d*e\+18 4.6116\d+e\+18 |/}
 
+# 2025-06-18 https://sqlite.org/forum/forumpost/6caf195248a849e4
+#
+# Enforce STRICT table type constraints on STORED generated columns
+#
+do_execsql_test strict1-9.1 {
+  CREATE TABLE strict (
+    k INTEGER PRIMARY KEY,
+    c1 REAL AS(if(k=11,1.5, k=12,2, k=13,'x', k=14,x'34', 0.0))   STORED,
+    c2 INT  AS(if(k=21,1.5, k=22,2, k=23,'x', k=24,x'34', 0))     STORED,
+    c3 TEXT AS(if(k=31,1.5, k=32,2, k=33,'x', k=34,x'34', 'x'))   STORED,
+    c4 BLOB AS(if(k=41,1.5, k=42,2, k=43,'x', k=44,x'34', x'00')) STORED,
+    c5 ANY  AS(if(k=51,1.5, k=52,2, k=53,'x', k=54,x'34', 0))     STORED
+  ) STRICT;
+  INSERT INTO strict(k) VALUES(11);
+  INSERT INTO strict(k) VALUES(12);
+  INSERT INTO strict(k) VALUES(22);
+  INSERT INTO strict(k) VALUES(31);
+  INSERT INTO strict(k) VALUES(32);
+  INSERT INTO strict(k) VALUES(33);
+  INSERT INTO strict(k) VALUES(44);
+  PRAGMA integrity_check;
+} {ok}
+do_catchsql_test strict1-9.2.13 {
+  INSERT INTO strict(k) VALUES(13);
+} {1 {cannot store TEXT value in REAL column strict.c1}}
+do_catchsql_test strict1-9.2.14 {
+  INSERT INTO strict(k) VALUES(14);
+} {1 {cannot store BLOB value in REAL column strict.c1}}
+do_catchsql_test strict1-9.2.21 {
+  INSERT INTO strict(k) VALUES(21);
+} {1 {cannot store REAL value in INT column strict.c2}}
+do_catchsql_test strict1-9.2.23 {
+  INSERT INTO strict(k) VALUES(23);
+} {1 {cannot store TEXT value in INT column strict.c2}}
+do_catchsql_test strict1-9.2.24 {
+  INSERT INTO strict(k) VALUES(24);
+} {1 {cannot store BLOB value in INT column strict.c2}}
+do_catchsql_test strict1-9.2.34 {
+  INSERT INTO strict(k) VALUES(34);
+} {1 {cannot store BLOB value in TEXT column strict.c3}}
+do_catchsql_test strict1-9.2.41 {
+  INSERT INTO strict(k) VALUES(41);
+} {1 {cannot store REAL value in BLOB column strict.c4}}
+do_catchsql_test strict1-9.2.42 {
+  INSERT INTO strict(k) VALUES(42);
+} {1 {cannot store INT value in BLOB column strict.c4}}
+do_catchsql_test strict1-9.2.43 {
+  INSERT INTO strict(k) VALUES(43);
+} {1 {cannot store TEXT value in BLOB column strict.c4}}
+
+do_execsql_test strict1-9.3 {
+  DROP TABLE strict;
+  CREATE TABLE strict (
+    k INTEGER PRIMARY KEY,
+    c1 REAL AS(if(k=11,1.5, k=12,2, k=13,'x', k=14,x'34', 0.0))   VIRTUAL,
+    c2 INT  AS(if(k=21,1.5, k=22,2, k=23,'x', k=24,x'34', 0))     VIRTUAL,
+    c3 TEXT AS(if(k=31,1.5, k=32,2, k=33,'x', k=34,x'34', 'x'))   VIRTUAL,
+    c4 BLOB AS(if(k=41,1.5, k=42,2, k=43,'x', k=44,x'34', x'00')) VIRTUAL,
+    c5 ANY  AS(if(k=51,1.5, k=52,2, k=53,'x', k=54,x'34', 0))     VIRTUAL
+  ) STRICT;
+  INSERT INTO strict(k) VALUES(11);
+  INSERT INTO strict(k) VALUES(12);
+  INSERT INTO strict(k) VALUES(22);
+  INSERT INTO strict(k) VALUES(31);
+  INSERT INTO strict(k) VALUES(32);
+  INSERT INTO strict(k) VALUES(33);
+  INSERT INTO strict(k) VALUES(44);
+  PRAGMA integrity_check;
+} {ok}
+do_catchsql_test strict1-9.4.13 {
+  INSERT INTO strict(k) VALUES(13);
+} {1 {cannot store TEXT value in REAL column strict.c1}}
+do_catchsql_test strict1-9.4.14 {
+  INSERT INTO strict(k) VALUES(14);
+} {1 {cannot store BLOB value in REAL column strict.c1}}
+do_catchsql_test strict1-9.4.21 {
+  INSERT INTO strict(k) VALUES(21);
+} {1 {cannot store REAL value in INT column strict.c2}}
+do_catchsql_test strict1-9.4.23 {
+  INSERT INTO strict(k) VALUES(23);
+} {1 {cannot store TEXT value in INT column strict.c2}}
+do_catchsql_test strict1-9.4.24 {
+  INSERT INTO strict(k) VALUES(24);
+} {1 {cannot store BLOB value in INT column strict.c2}}
+do_catchsql_test strict1-9.4.34 {
+  INSERT INTO strict(k) VALUES(34);
+} {1 {cannot store BLOB value in TEXT column strict.c3}}
+do_catchsql_test strict1-9.4.41 {
+  INSERT INTO strict(k) VALUES(41);
+} {1 {cannot store REAL value in BLOB column strict.c4}}
+do_catchsql_test strict1-9.4.42 {
+  INSERT INTO strict(k) VALUES(42);
+} {1 {cannot store INT value in BLOB column strict.c4}}
+do_catchsql_test strict1-9.4.43 {
+  INSERT INTO strict(k) VALUES(43);
+} {1 {cannot store TEXT value in BLOB column strict.c4}}
+
 finish_test
diff --git a/test/subquery.test b/test/subquery.test
index a048f9ed4e..898c8f7560 100644
--- a/test/subquery.test
+++ b/test/subquery.test
@@ -11,8 +11,6 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing correlated subqueries
 #
-# $Id: subquery.test,v 1.17 2009/01/09 01:12:28 drh Exp $
-#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -613,4 +611,142 @@ do_execsql_test subquery-9.4 {
   SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 2) FROM t1;
 } {{} {} {} {}}
 
+# 2023-09-15
+# Query planner performance regression reported by private email 
+# on 2023-09-14, caused by VIEWSCAN optimization of check-in 609fbb94b8f01d67
+# from 2022-09-01.
+#
+reset_db
+do_execsql_test subquery-10.1 {
+  CREATE TABLE t1(aa TEXT, bb INT, cc TEXT);
+  CREATE INDEX x11 on t1(bb);
+  CREATE INDEX x12 on t1(aa);
+  CREATE TABLE t2(aa TEXT, xx INT);
+  ANALYZE sqlite_master;
+  INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x11', '156789 28');
+  INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES('t1', 'x12', '156789 1');
+  ANALYZE sqlite_master;
+}
+do_eqp_test subquery-10.2 {
+  WITH v1(aa,cc,bb) AS (SELECT aa, cc, bb FROM t1 WHERE bb=12345),
+       v2(aa,mx)    AS (SELECT aa, max(xx) FROM t2 GROUP BY aa)
+  SELECT * FROM v1 JOIN v2 ON v1.aa=v2.aa;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE v2
+  |  |--SCAN t2
+  |  `--USE TEMP B-TREE FOR GROUP BY
+  |--SEARCH t1 USING INDEX x11 (bb=?)
+  `--SEARCH v2 USING AUTOMATIC COVERING INDEX (aa=?)
+}
+# ^^^^^^^^^^^^^
+# Prior to the fix the incorrect (slow) plan caused by the
+# VIEWSCAN optimization was:
+#
+# QUERY PLAN
+# |--CO-ROUTINE v2
+# |  |--SCAN t2
+# |  `--USE TEMP B-TREE FOR GROUP BY
+# |--SCAN v2
+# `--SEARCH t1 USING INDEX x12 (aa=?)
+#
+#---------------------------------------------------------------------------
+# Follow-up on 2025-04-14.  Performance issue found while working
+# on Fossil (Fossil check-in 2025-04-13T19:54).
+#
+do_execsql_test 10.3 {
+  CREATE TABLE blob(
+    rid INTEGER PRIMARY KEY,
+    size INT,
+    uuid TEXT
+  );
+  CREATE TABLE delta(
+    rid INTEGER PRIMARY KEY,
+    srcid INT
+  );
+  CREATE INDEX delta_i1 ON delta(srcid);
+}
+do_eqp_test subquery-10.4 {
+  WITH RECURSIVE deltasof(rid) AS (
+    SELECT rid FROM delta WHERE srcid=125020
+    UNION
+    SELECT delta.rid FROM deltasof, delta
+     WHERE delta.srcid=deltasof.rid)
+  SELECT deltasof.rid, blob.uuid FROM deltasof, blob
+   WHERE blob.rid=deltasof.rid;
+} {
+  QUERY PLAN
+  |--CO-ROUTINE deltasof
+  |  |--SETUP
+  |  |  `--SEARCH delta USING COVERING INDEX delta_i1 (srcid=?)
+  |  `--RECURSIVE STEP
+  |     |--SCAN deltasof
+  |     `--SEARCH delta USING COVERING INDEX delta_i1 (srcid=?)
+  |--SCAN deltasof
+  `--SEARCH blob USING INTEGER PRIMARY KEY (rowid=?)
+}
+# ^^^^^^^^^^^^^^^^^^
+# deltasof should be the outer loop and blob the inner loop
+# Prior to the fix, SQLite was doing it the other way around.
+
+#-----------------------------------------------------------------------------
+# 2024-04-25 Column affinities for columns of compound subqueries
+#
+reset_db
+sqlite3_test_control SQLITE_TESTCTRL_INTERNAL_FUNCTIONS db
+do_execsql_test subquery-11.1 {
+  CREATE TABLE t1(ix INT, rx REAL, bx BLOB, tx TEXT, ax);
+  INSERT INTO t1 VALUES(1,1.0,x'31','x',NULL);
+  WITH c(a) AS (SELECT 'y' UNION SELECT tx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT tx FROM t1 UNION SELECT 'y') SELECT affinity(a) FROM c;
+} {text text text text}
+do_execsql_test subquery-11.2 {
+  WITH c(a) AS (SELECT 2 UNION SELECT tx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT tx FROM t1 UNION SELECT 2) SELECT affinity(a) FROM c;
+} {blob blob blob blob}
+do_execsql_test subquery-11.3 {
+  WITH c(a) AS (SELECT 2.0 UNION SELECT tx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT tx FROM t1 UNION SELECT 2.0) SELECT affinity(a) FROM c;
+} {blob blob blob blob}
+do_execsql_test subquery-11.4 {
+  WITH c(a) AS (SELECT null UNION SELECT tx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT tx FROM t1 UNION SELECT null) SELECT affinity(a) FROM c;
+} {text text text text}
+do_execsql_test subquery-11.5 {
+  WITH c(a) AS (SELECT x'32' UNION SELECT tx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT tx FROM t1 UNION SELECT x'32') SELECT affinity(a) FROM c;
+} {text text text text}
+do_execsql_test subquery-11.6 {
+  WITH c(a) AS (SELECT 3 UNION SELECT ix FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT ix FROM t1 UNION SELECT 3) SELECT affinity(a) FROM c;
+} {integer integer integer integer}
+do_execsql_test subquery-11.7 {
+  WITH c(a) AS (SELECT 3.0 UNION SELECT ix FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT ix FROM t1 UNION SELECT 3.0) SELECT affinity(a) FROM c;
+} {integer integer integer integer}
+do_execsql_test subquery-11.8 {
+  WITH c(a) AS (SELECT '3' UNION SELECT ix FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT ix FROM t1 UNION SELECT '3') SELECT affinity(a) FROM c;
+} {blob blob blob blob}
+do_execsql_test subquery-11.10 {
+  WITH c(a) AS (SELECT x'32' UNION SELECT ix FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT ix FROM t1 UNION SELECT x'32') SELECT affinity(a) FROM c;
+} {integer integer integer integer}
+do_execsql_test subquery-11.11 {
+  WITH c(a) AS (SELECT 4 UNION SELECT rx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT rx FROM t1 UNION SELECT 4) SELECT affinity(a) FROM c;
+} {real real real real}
+do_execsql_test subquery-11.12 {
+  WITH c(a) AS (SELECT '4' UNION SELECT rx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT rx FROM t1 UNION SELECT '4') SELECT affinity(a) FROM c;
+} {blob blob blob blob}
+do_execsql_test subquery-11.13 {
+  WITH c(a) AS (SELECT null UNION SELECT rx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT rx FROM t1 UNION SELECT null) SELECT affinity(a) FROM c;
+} {real real real real}
+do_execsql_test subquery-11.14 {
+  WITH c(a) AS (SELECT x'b4' UNION SELECT rx FROM t1) SELECT affinity(a) FROM c;
+  WITH c(a) AS (SELECT rx FROM t1 UNION SELECT x'b4') SELECT affinity(a) FROM c;
+} {real real real real}
+
 finish_test
diff --git a/test/subquery2.test b/test/subquery2.test
index 0c1bdc6697..99a1e903f8 100644
--- a/test/subquery2.test
+++ b/test/subquery2.test
@@ -104,7 +104,7 @@ do_execsql_test 2.2 {
 } {2 3   3 6   4 10}
 
 ############################################################################
-# Ticket http://www.sqlite.org/src/info/d11a6e908f (2014-09-20)
+# Ticket http://sqlite.org/src/info/d11a6e908f (2014-09-20)
 # Query planner fault on three-way nested join with compound inner SELECT 
 #
 do_execsql_test 3.0 {
@@ -215,4 +215,72 @@ do_execsql_test 5.1 {
   SELECT ( SELECT y FROM t2 WHERE x = y ORDER BY y, z) FROM t1;
 } {ALFKI ANATR}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1234);
+}
+
+do_execsql_test 6.1 {
+  SELECT DISTINCT 'string' FROM t1 LIMIT 1 OFFSET 5;
+}
+
+do_execsql_test 6.2 {
+  SELECT (
+      SELECT 'string' FROM t1 LIMIT 1 OFFSET 5
+  );
+} {{}}
+
+do_execsql_test 6.3 {
+  SELECT (
+      SELECT DISTINCT 'string' FROM t1 LIMIT 1 OFFSET 5
+  );
+} {{}}
+
+do_execsql_test 6.4 {
+  SELECT (
+      SELECT DISTINCT 'string' FROM t1 ORDER BY 1 LIMIT 1 OFFSET 5
+  );
+} {{}}
+
+do_execsql_test 6.5 {
+  SELECT (
+      SELECT 'string' FROM t1 ORDER BY 1 LIMIT 1 OFFSET 5
+  );
+} {{}}
+
+do_execsql_test 6.6 {
+  SELECT (SELECT DISTINCT x, x FROM t1 LIMIT 1 OFFSET 5)==(1234, 1234)
+} {{}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE TABLE t1(x);
+  CREATE INDEX i1 ON t1(x);
+  INSERT INTO t1 VALUES(1234);
+}
+
+do_execsql_test 7.1 {
+  SELECT (
+    SELECT DISTINCT 'string' FROM t1 ORDER BY x LIMIT 1 OFFSET 5
+  );
+} {{}}
+
+do_execsql_test 7.2 {
+  DROP INDEX i1;
+  CREATE UNIQUE INDEX i1 ON t1(x);
+}
+
+do_execsql_test 7.3 {
+  SELECT (
+    SELECT DISTINCT x FROM t1 ORDER BY 1 LIMIT 1 OFFSET 5
+  );
+} {{}}
+
+do_execsql_test 7.4 {
+  SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 0);
+} {1234}
+
 finish_test
diff --git a/test/subtype1.test b/test/subtype1.test
index 4d8e68e6af..6d0df4fa6b 100644
--- a/test/subtype1.test
+++ b/test/subtype1.test
@@ -73,5 +73,15 @@ do_execsql_test subtype1-320 {
    WHERE json_quote(y)='"1"';
 } {1 {"1"}}
 
+# 2024-11-26
+# subtype survives if()
+#
+do_execsql_test subtype1-400 {
+  WITH t400(id,j) AS (VALUES
+    (1,'{a:{x:1,y:2},b:{x:3,y:4}}'),
+    (2,'not json')
+  )
+  SELECT id, subtype(if(json_valid(j,6),j->'a')) FROM t400;
+} {1 74 2 0}
 
 finish_test
diff --git a/test/superlock.test b/test/superlock.test
index 704b0677a1..10e7caa298 100644
--- a/test/superlock.test
+++ b/test/superlock.test
@@ -166,7 +166,7 @@ do_multiclient_test tn {
 proc read_content {file} {
   if {[file exists $file]==0} {return ""}
   set fd [open $file]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   set content [read $fd]
   close $fd
   return $content
@@ -174,7 +174,7 @@ proc read_content {file} {
 
 proc write_content {file content} {
   set fd [open $file w+]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   puts -nonewline $fd $content
   close $fd
 }
diff --git a/test/symlink.test b/test/symlink.test
index 685cae5a41..fc78a04720 100644
--- a/test/symlink.test
+++ b/test/symlink.test
@@ -17,7 +17,7 @@ source $testdir/tester.tcl
 set testprefix symlink
 
 # This only runs on unix.
-if {$::tcl_platform(platform)!="unix"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   finish_test
   return
 }
diff --git a/test/symlink2.test b/test/symlink2.test
index 4123092deb..7305f6dd38 100644
--- a/test/symlink2.test
+++ b/test/symlink2.test
@@ -17,7 +17,7 @@ source $testdir/tester.tcl
 set testprefix symlink2
 
 # This only runs on Windows.
-if {$::tcl_platform(platform)!="windows"} {
+if {$::tcl_platform(platform) ne "windows"} {
   finish_test
   return
 }
@@ -57,6 +57,7 @@ do_execsql_test 1.0 {
   INSERT INTO t1 VALUES(1,9999);
 }
 
+forcedelete link.db
 do_test 2.0 {
   createWin32Symlink link.db test.db
 } {}
@@ -87,12 +88,13 @@ do_test 3.4 {
   db3 close
 } {}
 
+# The -nofollow option does not work on Windows
 do_test 3.5 {
   list [catch {
     sqlite3 db4 link.db -nofollow true
     execsql { SELECT x, y FROM t1; } db4
   } res] $res
-} {1 {unable to open database file}}
+} {0 {1 9999}}
 
 catch {db4 close}
 
diff --git a/test/sync.test b/test/sync.test
index 023425e6b1..b24800d100 100644
--- a/test/sync.test
+++ b/test/sync.test
@@ -34,7 +34,7 @@ if {[atomic_batch_write test.db]} {
 set sqlite_sync_count 0
 proc cond_incr_sync_count {adj} {
   global sqlite_sync_count
-  if {$::tcl_platform(platform) == "windows"} {
+  if {$::tcl_platform(os) eq "Windows NT"} {
     incr sqlite_sync_count $adj
   } else {
     ifcapable !dirsync {
diff --git a/test/sync2.test b/test/sync2.test
index 89e66c8455..ce8132c714 100644
--- a/test/sync2.test
+++ b/test/sync2.test
@@ -26,7 +26,7 @@ ifcapable !pager_pragmas||!attach||!dirsync {
   finish_test
   return
 }
-if {$::tcl_platform(platform)!="unix" 
+if {$::tcl_platform(os) eq "Windows NT"
   || [permutation] == "journaltest"
   || [permutation] == "inmemory_journal"
   || [atomic_batch_write test.db] 
diff --git a/test/syscall.test b/test/syscall.test
index 19313a5e67..fe4a4177fe 100644
--- a/test/syscall.test
+++ b/test/syscall.test
@@ -211,7 +211,7 @@ forcedelete test.db test.db2
 
 proc create_db_file {nByte} {
   set fd [open test.db w]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   puts -nonewline $fd [string range "xSQLite" 1 $nByte]
   close $fd
 }
diff --git a/test/tabfunc01.test b/test/tabfunc01.test
index 8e90c549fa..9a2017c46a 100644
--- a/test/tabfunc01.test
+++ b/test/tabfunc01.test
@@ -22,7 +22,6 @@ ifcapable !vtab {
   return
 }
 load_static_extension db series
-load_static_extension db carray
 load_static_extension db remember
 
 do_execsql_test tabfunc01-1.1 {
@@ -61,10 +60,10 @@ do_execsql_test tabfunc01-1.8 {
 } {30 25 20 15 10 5 0}
 do_execsql_test tabfunc01-1.9 {
   SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC;
-} {7 30 6 25 5 20 4 15 3 10 2 5 1 0}
+} {30 30 25 25 20 20 15 15 10 10 5 5 0 0}
 do_execsql_test tabfunc01-1.10 {
   SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC;
-} {7 30 6 25 5 20 4 15 3 10 2 5 1 0}
+} {30 30 25 25 20 20 15 15 10 10 5 5 0 0}
 
 do_execsql_test tabfunc01-1.20 {
   CREATE VIEW v1(a,b) AS VALUES(1,2),(3,4);
@@ -121,26 +120,26 @@ do_eqp_test tabfunc01-3.10 {
   SELECT value FROM generate_series(1,10) ORDER BY value;
 } {
   QUERY PLAN
-  `--SCAN generate_series VIRTUAL TABLE INDEX 19:
+  `--SCAN generate_series VIRTUAL TABLE INDEX 0x13:
 }
 do_eqp_test tabfunc01-3.11 {
   SELECT value FROM generate_series(1,10) ORDER BY +value;
 } {
   QUERY PLAN
-  |--SCAN generate_series VIRTUAL TABLE INDEX 3:
+  |--SCAN generate_series VIRTUAL TABLE INDEX 0x3:
   `--USE TEMP B-TREE FOR ORDER BY
 }
 do_eqp_test tabfunc01-3.12 {
   SELECT value FROM generate_series(1,10) ORDER BY value, stop;
 } {
   QUERY PLAN
-  `--SCAN generate_series VIRTUAL TABLE INDEX 19:
+  `--SCAN generate_series VIRTUAL TABLE INDEX 0x13:
 }
 do_eqp_test tabfunc01-3.13 {
   SELECT value FROM generate_series(1,10) ORDER BY stop, value;
 } {
   QUERY PLAN
-  |--SCAN generate_series VIRTUAL TABLE INDEX 3:
+  |--SCAN generate_series VIRTUAL TABLE INDEX 0x3:
   `--USE TEMP B-TREE FOR ORDER BY
 }
 
@@ -156,9 +155,9 @@ do_eqp_test tabfunc01-3.20 {
   QUERY PLAN
   `--MERGE (UNION ALL)
      |--LEFT
-     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
      `--RIGHT
-        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
 }
   
 
@@ -181,6 +180,76 @@ do_execsql_test tabfunc01-4.4 {
   SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10;
 } {1 3 5}
 
+# 2025-03-13 forum post bf2dc8e909983511
+#
+do_execsql_test tabfunc01-5.1 {
+  SELECT value
+    FROM generate_series(60,73,6)
+   WHERE value=66;
+} 66
+do_execsql_test tabfunc01-5.2 {
+  SELECT value
+    FROM generate_series(73,60,-6)
+   WHERE value=67;
+} 67
+
+# 2025-03-22 forum post 0d5d63257e3ff4f6
+#
+do_execsql_test tabfunc01-6.1 {
+  SELECT value FROM generate_series(1,10) WHERE value<5.5;
+} {1 2 3 4 5}
+do_execsql_test tabfunc01-6.2 {
+  SELECT value FROM generate_series(1,10) WHERE value<5.0;
+} {1 2 3 4}
+do_execsql_test tabfunc01-6.3 {
+  SELECT value FROM generate_series(1,10) WHERE value<=5.5;
+} {1 2 3 4 5}
+do_execsql_test tabfunc01-6.4 {
+  SELECT value FROM generate_series(1,10) WHERE value<=5.0;
+} {1 2 3 4 5}
+do_execsql_test tabfunc01-6.5 {
+  SELECT value FROM generate_series(1,10) WHERE value>5.5;
+} {6 7 8 9 10}
+do_execsql_test tabfunc01-6.6 {
+  SELECT value FROM generate_series(1,10) WHERE value>5.0;
+} {6 7 8 9 10}
+do_execsql_test tabfunc01-6.7 {
+  SELECT value FROM generate_series(1,10) WHERE value>=5.5;
+} {6 7 8 9 10}
+do_execsql_test tabfunc01-6.8 {
+  SELECT value FROM generate_series(1,10) WHERE value>=5.0;
+} {5 6 7 8 9 10}
+do_execsql_test tabfunc01-6.9 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value<5.5;
+} {5 4 3 2 1}
+do_execsql_test tabfunc01-6.10 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value<5.0;
+} {4 3 2 1}
+do_execsql_test tabfunc01-6.11 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value<=5.5;
+} {5 4 3 2 1}
+do_execsql_test tabfunc01-6.12 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value<=5.0;
+} {5 4 3 2 1}
+do_execsql_test tabfunc01-6.13 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value>5.5;
+} {10 9 8 7 6}
+do_execsql_test tabfunc01-6.14 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value>5.0;
+} {10 9 8 7 6}
+do_execsql_test tabfunc01-6.15 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value>=5.5;
+} {10 9 8 7 6}
+do_execsql_test tabfunc01-6.16 {
+  SELECT value FROM generate_series(10,1,-1) WHERE value>=5.0;
+} {10 9 8 7 6 5}
+do_execsql_test tabfunc01-6.17 {
+  SELECT value FROM generate_series(1,10) WHERE value==5.5;
+} {}
+do_execsql_test tabfunc01-6.18 {
+  SELECT value FROM generate_series(1,10) WHERE value==5.0;
+} {5}
+
 # The next series of tests is verifying that virtual table are able
 # to optimize the IN operator, even on terms that are not marked "omit".
 # When the generate_series virtual table is compiled for the testfixture,
@@ -208,6 +277,7 @@ do_execsql_test tabfunc01-600 {
 
 do_test tabfunc01-700 {
   set PTR1 [intarray_addr 5 7 13 17 23]
+  sqlite3_create_function db
   db eval {
     SELECT b FROM t600, carray(inttoptr($PTR1),5) WHERE a=value;
   }
@@ -269,7 +339,7 @@ do_test tabfunc01-750 {
   }
 } {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |}
 
-# ticket https://www.sqlite.org/src/info/2ae0c599b735d59e
+# ticket https://sqlite.org/src/info/2ae0c599b735d59e
 # Verification of testtag-20230227a
 do_test tabfunc01-751 {
   db eval {
@@ -322,6 +392,283 @@ ifcapable altertable {
   } {1 {table pragma_compile_options may not be altered}}
 }
 
+#-----------------------------------------------------------------------------
+# 2024-04-26  LIMIT and OFFSET passed into virtual tables
+# https://sqlite.org/forum/forumpost/c243b8f856
+#
+do_execsql_test tabfunc01-900 {
+  SELECT * FROM (
+    SELECT * FROM generate_series(1,10)
+    UNION ALL
+    SELECT * FROM generate_series(101,104)
+  ) LIMIT 10 OFFSET 5;
+} {6 7 8 9 10 101 102 103 104}
+do_execsql_test tabfunc01-910 {
+  SELECT * FROM (
+    SELECT * FROM generate_series(1,10)
+    UNION ALL
+    SELECT * FROM generate_series(101,104)
+  ) LIMIT -1 OFFSET 5;
+} {6 7 8 9 10 101 102 103 104}
+do_execsql_test tabfunc01-920 {
+  SELECT * FROM (
+    SELECT * FROM generate_series(1,10)
+    UNION ALL
+    SELECT * FROM generate_series(101,104)
+  ) LIMIT -1 OFFSET 0;
+} {1 2 3 4 5 6 7 8 9 10 101 102 103 104}
+
+# Also:  https://sqlite.org/forum/forumpost/ce1a06c2c8
+#
+do_execsql_test tabfunc01-930 {
+  SELECT * FROM generate_series(9223372036854775807,
+                                -9223372036854775808,
+                                -9223372036854775808);
+} {9223372036854775807 -1}
+do_execsql_test tabfunc01-931 {
+  SELECT * FROM generate_series(9223372036854775807,
+                                -9223372036854775808,
+                                -9223372036854775808)
+   LIMIT 100 OFFSET 1;
+} {-1}
+do_execsql_test tabfunc01-932 {
+  SELECT * FROM generate_series(9223372036854775807,
+                                -9223372036854775808,
+                                -9223372036854775808)
+   LIMIT 100 OFFSET 2;
+} {}
+do_execsql_test tabfunc01-933 {
+  SELECT * FROM generate_series(9223372036854775807,
+                                -9223372036854775808,
+                                -9223372036854775808)
+   LIMIT 100 OFFSET 9223372036854775807;
+} {}
+do_execsql_test tabfunc01-940 {
+  SELECT * FROM generate_series(1,11,2);
+} {1 3 5 7 9 11}
+do_execsql_test tabfunc01-941 {
+  SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 4;
+} {9 11}
+do_execsql_test tabfunc01-942 {
+  SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 5;
+} {11}
+do_execsql_test tabfunc01-943 {
+  SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 6;
+} {}
+do_execsql_test tabfunc01-944 {
+  SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 7;
+} {}
+do_execsql_test tabfunc01-945 {
+  SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 9223372036854775807;
+} {}
+do_execsql_test tabfunc01-950 {
+  SELECT * FROM generate_series(1,11,1) LIMIT 100 OFFSET 10;
+} {11}
+do_execsql_test tabfunc01-951 {
+  SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 9223372036854775807;
+} {}
+do_execsql_test tabfunc01-952 {
+  SELECT * FROM generate_series(11,1,-1) LIMIT 100 OFFSET 10;
+} {1}
+do_execsql_test tabfunc01-953 {
+  SELECT * FROM generate_series(11,1,-1) LIMIT 100 OFFSET 9223372036854775807;
+} {}
+do_execsql_test tabfunc01-954 {
+  SELECT * FROM generate_series(1,11,1) LIMIT 3 OFFSET 3;
+} {4 5 6}
+
+
+#-------------------------------------------------------------------------
+# Forum post https://sqlite.org/forum/forumpost/e7c3ae1215
+#
+foreach {tn where res} {
+  1000 "where value = 2" 2
+  1010 "where value in (2)" 2
+  1020 "where value in (select 2)" 2
+  1030 "where value = 2 OR value = 4" {2 4}
+  1040 "where value in (2, 4)" {2 4}
+} {
+  do_execsql_test $tn "
+    SELECT value FROM generate_series(1, 5) $where
+  " $res
+}
+
+do_execsql_test 1100 {
+  select 1 as c_0
+    from        
+    generate_series(1, 1) as ref_3
+    where (ref_3.value) in (select 1);
+} {1}
+
+# 2025-03-18 /forumpost/1e17219c88
+# The generate_series() table-valued function is modified so that its
+# rowid is always its value. That way it can be used on the RHS of a
+# RIGHT JOIN.
+#
+do_execsql_test 1200 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(value INT);
+  INSERT INTO t1 VALUES (1),(2),(3);
+  SELECT t1.value, t2.value
+    FROM t1 RIGHT JOIN generate_series(1,3,1) AS t2 USING(value);
+} {1 1 2 2 3 3}
+
+#--------------------------------------------------------------------------
+# New test cases for generate_series associated with the refactor on branch
+# serices-refactor circa 2025-09-27.
+#
+do_execsql_test 1300 {
+  SELECT value
+    FROM generate_series(-9223372036854775808, 9223372036854775807, 2)
+   WHERE value BETWEEN 1 AND 5;
+} {2 4}
+do_execsql_test 1301 {
+  SELECT value
+    FROM generate_series(-9223372036854775808, 9223372036854775807, 2)
+   WHERE value BETWEEN 0 AND 6;
+} {0 2 4 6}
+do_execsql_test 1302 {
+  SELECT value
+    FROM generate_series(-9223372036854775808, 9223372036854775807, 2)
+   WHERE value BETWEEN 0.5 AND 6.25;
+} {2 4 6}
+do_execsql_test 1303 {
+  SELECT value
+    FROM generate_series(-9223372036854775808, 9223372036854775807, 2)
+   WHERE value BETWEEN 9223372036854775803 AND 9223372036854775807
+} {9223372036854775804 9223372036854775806}
+do_execsql_test 1304 {
+  SELECT value
+    FROM generate_series(-9223372036854775808, 9223372036854775807, 2)
+   WHERE value BETWEEN 9223372036854775803 AND 9223372036854775807
+   ORDER BY value DESC;
+} {9223372036854775806 9223372036854775804}
+
+do_execsql_test 1310 {
+  SELECT value FROM generate_series(0) LIMIT 5;
+} {0 1 2 3 4}
+do_execsql_test 1311 {
+  SELECT value FROM generate_series(5) LIMIT 5;
+} {5 6 7 8 9}
+do_execsql_test 1312 {
+  SELECT value FROM generate_series(0) WHERE stop=6;
+} {0 1 2 3 4 5 6}
+do_execsql_test 1313 {
+  SELECT value FROM generate_series(1,10) WHERE step IS NULL;
+} {}
+do_execsql_test 1314 {
+  SELECT value FROM generate_series(0xfffffffd);
+} {4294967293 4294967294 4294967295}
+do_execsql_test 1315 {
+  SELECT value FROM generate_series(4,3,1);
+} {}
+do_execsql_test 1316 {
+  SELECT value FROM generate_series(3,4,-1);
+} {}
+
+do_execsql_test 1320 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value=9.2234e18;
+} {}
+do_execsql_test 1321 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value=-9.2234e18;
+} {}
+do_execsql_test 1322 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value>9223372036854775807;
+} {}
+do_execsql_test 1323 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value>9223372036854775806;
+} {9223372036854775807}
+do_execsql_test 1324 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value>=9223372036854775807;
+} {9223372036854775807}
+
+do_execsql_test 1330 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value<-9223372036854775808;
+} {}
+do_execsql_test 1331 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value<=-9223372036854775808;
+} {-9223372036854775808}
+do_execsql_test 1332 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value<-9223372036854775807;
+} {-9223372036854775808}
+do_execsql_test 1333 {
+  SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807)
+   WHERE value BETWEEN 4 AND 1;
+} {}
+
+do_execsql_test 1340 {
+  SELECT value FROM generate_series(100,0,-10)
+   WHERE value BETWEEN 33 AND 66;
+} {60 50 40}
+do_execsql_test 1341 {
+  SELECT value FROM generate_series(100,0,-10)
+   WHERE value BETWEEN 33 AND 60;
+} {60 50 40}
+do_execsql_test 1342 {
+  SELECT value FROM generate_series(100,0,-10)
+   WHERE value BETWEEN 33 AND 59;
+} {50 40}
+do_execsql_test 1343 {
+  SELECT value
+    FROM generate_series(-9223372036854760000,-9223372036854775808,-10000);
+} {-9223372036854760000 -9223372036854770000}
+do_execsql_test 1344 {
+  SELECT value
+    FROM generate_series(-9223372036854760000,-9223372036854775808,-10000)
+   WHERE value < -9223372036854770001;
+} {}
+do_execsql_test 1345 {
+  SELECT value
+    FROM generate_series(9223372036854760000,9223372036854775807,10000);
+} {9223372036854760000 9223372036854770000}
+do_execsql_test 1346 {
+  SELECT value
+    FROM generate_series(9223372036854760000,9223372036854775807,10000)
+   WHERE value > 9223372036854770001;
+} {}
+
+do_execsql_test 1350 {
+  SELECT value FROM generate_series(100,0,-10)
+   WHERE value BETWEEN 33 AND 38;
+} {}
+do_execsql_test 1351 {
+  SELECT value FROM generate_series(0,100,+10)
+   WHERE value BETWEEN 33 AND 38;
+} {}
+
+do_execsql_test 1360 {
+  SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808);
+} {0 -9223372036854775808}
+do_execsql_test 1361 {
+  SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808)
+  LIMIT 1 OFFSET 0;
+} {0}
+do_execsql_test 1362 {
+  SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808)
+   ORDER BY value ASC;
+} {-9223372036854775808 0}
+do_execsql_test 1363 {
+  SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808)
+   ORDER BY value ASC LIMIT 10 OFFSET 1;
+} {0}
+do_execsql_test 1364 {
+  SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808)
+   ORDER BY value ASC LIMIT 10 OFFSET 40000000;
+} {}
+
+do_execsql_test 1370 {
+  SELECT * FROM generate_series(0,0,0);
+} {}
+
+
 
 # Free up memory allocations
 intarray_addr
diff --git a/test/table.test b/test/table.test
index 7be6b37695..46ecd7d23b 100644
--- a/test/table.test
+++ b/test/table.test
@@ -784,15 +784,15 @@ do_catchsql_test table-16.5 {
 } {1 {unknown function: count()}}
 do_catchsql_test table-16.6 {
   DROP TABLE t16;
-  CREATE TABLE t16(x DEFAULT(group_concat('x',',')));
+  CREATE TABLE t16(x DEFAULT(string_agg('x',',')));
   INSERT INTO t16(rowid) VALUES(123);
   SELECT rowid, x FROM t16;
-} {1 {unknown function: group_concat()}}
+} {1 {unknown function: string_agg()}}
 do_catchsql_test table-16.7 {
   INSERT INTO t16 DEFAULT VALUES;
-} {1 {unknown function: group_concat()}}
+} {1 {unknown function: string_agg()}}
 
-# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63]
+# Ticket [https://sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63]
 # describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement.
 # the following test verifies that the problem has been fixed.
 #
@@ -810,7 +810,7 @@ do_execsql_test table-17.1 {
 } {1 1 | 2 2 |}
 
 # 2015-06-16
-# Ticket [https://www.sqlite.org/src/tktview/873cae2b6e25b1991ce5e9b782f9cd0409b96063]
+# Ticket [https://sqlite.org/src/tktview/873cae2b6e25b1991ce5e9b782f9cd0409b96063]
 # Make sure a CREATE TABLE AS statement correctly rolls back partial changes to the
 # sqlite_master table when the SELECT on the right-hand side aborts.
 #
@@ -825,7 +825,7 @@ do_execsql_test table-18.2 {
 } {ok}
 
 # 2015-09-09
-# Ticket [https://www.sqlite.org/src/info/acd12990885d9276]
+# Ticket [https://sqlite.org/src/info/acd12990885d9276]
 # "CREATE TABLE ... AS SELECT ... FROM sqlite_master" fails because the row
 # in the sqlite_master table for the next table is initially populated
 # with a NULL instead of a record created by OP_Record.
diff --git a/test/tableapi.test b/test/tableapi.test
index 02633cdcac..7720737474 100644
--- a/test/tableapi.test
+++ b/test/tableapi.test
@@ -230,7 +230,7 @@ ifcapable schema_pragmas {
 }
 
 # do_malloc_test closes and deletes the usual db connections and files on
-# each iteration.  $::dbx is a seperate connection, and on Windows, will
+# each iteration.  $::dbx is a separate connection, and on Windows, will
 # cause the file deletion of test.db to fail, so we move the close of $::dbx
 # up to here before the do_malloc_test.
 do_test tableapi-99.0 {
diff --git a/test/tclsqlite.test b/test/tclsqlite.test
index 0758abd822..5f373ea18a 100644
--- a/test/tclsqlite.test
+++ b/test/tclsqlite.test
@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 # This file implements regression tests for TCL interface to the
-# SQLite library. 
+# SQLite library.
 #
 # Actually, all tests are based on the TCL interface, so the main
 # interface is pretty well tested.  This file contains some addition
@@ -121,7 +121,7 @@ ifcapable {complete} {
 do_test tcl-1.14 {
   set v [catch {db eval} msg]
   lappend v $msg
-} {1 {wrong # args: should be "db eval ?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?"}}
+} {1 {wrong # args: should be "db eval ?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?"}}
 do_test tcl-1.15 {
   set v [catch {db function} msg]
   lappend v $msg
@@ -359,6 +359,19 @@ do_test tcl-9.3 {
   execsql {SELECT typeof(ret_int())}
 } {integer}
 
+proc breakAsNullUdf args {
+  if {"1" eq [lindex $args 0]} {return -code break}
+}
+do_test tcl-9.4 {
+  db function banu breakAsNullUdf
+  execsql {SELECT typeof(banu()), typeof(banu(1))}
+} {text null}
+do_test tcl-9.5 {
+  db nullvalue banunull
+  db eval {SELECT banu(), banu(1)}
+} {{} banunull}
+
+
 # Recursive calls to the same user-defined function
 #
 ifcapable tclvar {
@@ -465,7 +478,7 @@ do_test tcl-10.13 {
   db eval {SELECT * FROM t4}
 } {1 2 5 6 7}
 
-# Now test that [db transaction] commands may be nested with 
+# Now test that [db transaction] commands may be nested with
 # the expected results.
 #
 do_test tcl-10.14 {
@@ -475,7 +488,7 @@ do_test tcl-10.14 {
       INSERT INTO t4 VALUES('one');
     }
 
-    catch { 
+    catch {
       db transaction {
         db eval { INSERT INTO t4 VALUES('two') }
         db transaction {
@@ -674,11 +687,11 @@ do_test tcl-15.5 {
 } {0}
 
 
-# 2017-06-26: The --withoutnulls flag to "db eval".
+# 2017-06-26: The -withoutnulls flag to "db eval".
 #
-# In the "db eval --withoutnulls SQL ARRAY" form, NULL results cause the
-# corresponding array entry to be unset.  The default behavior (without
-# the -withoutnulls flags) is for the corresponding array value to get
+# In the "db eval -withoutnulls SQL TARGET" form, NULL results cause the
+# corresponding target entry to be unset.  The default behavior (without
+# the -withoutnulls flags) is for the corresponding target value to get
 # the [db nullvalue] string.
 #
 catch {db close}
@@ -720,64 +733,64 @@ reset_db
 proc add {a b} { return [expr $a + $b] }
 proc ret {a} { return $a }
 
-db function add_i -returntype integer add 
+db function add_i -returntype integer add
 db function add_r -ret        real    add
-db function add_t -return     text    add 
-db function add_b -returntype blob    add 
-db function add_a -returntype any     add 
+db function add_t -return     text    add
+db function add_b -returntype blob    add
+db function add_a -returntype any     add
 
-db function ret_i -returntype int     ret 
+db function ret_i -returntype int     ret
 db function ret_r -returntype real    ret
-db function ret_t -returntype text    ret 
-db function ret_b -returntype blob    ret 
-db function ret_a -r          any     ret 
+db function ret_t -returntype text    ret
+db function ret_b -returntype blob    ret
+db function ret_a -r          any     ret
 
 do_execsql_test 17.0 {
   SELECT quote( add_i(2, 3) );
-  SELECT quote( add_r(2, 3) ); 
-  SELECT quote( add_t(2, 3) ); 
-  SELECT quote( add_b(2, 3) ); 
-  SELECT quote( add_a(2, 3) ); 
+  SELECT quote( add_r(2, 3) );
+  SELECT quote( add_t(2, 3) );
+  SELECT quote( add_b(2, 3) );
+  SELECT quote( add_a(2, 3) );
 } {5 5.0 '5' X'35' 5}
 
 do_execsql_test 17.1 {
   SELECT quote( add_i(2.2, 3.3) );
-  SELECT quote( add_r(2.2, 3.3) ); 
-  SELECT quote( add_t(2.2, 3.3) ); 
-  SELECT quote( add_b(2.2, 3.3) ); 
-  SELECT quote( add_a(2.2, 3.3) ); 
+  SELECT quote( add_r(2.2, 3.3) );
+  SELECT quote( add_t(2.2, 3.3) );
+  SELECT quote( add_b(2.2, 3.3) );
+  SELECT quote( add_a(2.2, 3.3) );
 } {5.5 5.5 '5.5' X'352E35' 5.5}
 
 do_execsql_test 17.2 {
   SELECT quote( ret_i(2.5) );
-  SELECT quote( ret_r(2.5) ); 
-  SELECT quote( ret_t(2.5) ); 
-  SELECT quote( ret_b(2.5) ); 
-  SELECT quote( ret_a(2.5) ); 
+  SELECT quote( ret_r(2.5) );
+  SELECT quote( ret_t(2.5) );
+  SELECT quote( ret_b(2.5) );
+  SELECT quote( ret_a(2.5) );
 } {2.5 2.5 '2.5' X'322E35' 2.5}
 
 do_execsql_test 17.3 {
   SELECT quote( ret_i('2.5') );
-  SELECT quote( ret_r('2.5') ); 
-  SELECT quote( ret_t('2.5') ); 
-  SELECT quote( ret_b('2.5') ); 
-  SELECT quote( ret_a('2.5') ); 
+  SELECT quote( ret_r('2.5') );
+  SELECT quote( ret_t('2.5') );
+  SELECT quote( ret_b('2.5') );
+  SELECT quote( ret_a('2.5') );
 } {2.5 2.5 '2.5' X'322E35' '2.5'}
 
 do_execsql_test 17.4 {
   SELECT quote( ret_i('abc') );
-  SELECT quote( ret_r('abc') ); 
-  SELECT quote( ret_t('abc') ); 
-  SELECT quote( ret_b('abc') ); 
-  SELECT quote( ret_a('abc') ); 
+  SELECT quote( ret_r('abc') );
+  SELECT quote( ret_t('abc') );
+  SELECT quote( ret_b('abc') );
+  SELECT quote( ret_a('abc') );
 } {'abc' 'abc' 'abc' X'616263' 'abc'}
 
 do_execsql_test 17.5 {
   SELECT quote( ret_i(X'616263') );
-  SELECT quote( ret_r(X'616263') ); 
-  SELECT quote( ret_t(X'616263') ); 
-  SELECT quote( ret_b(X'616263') ); 
-  SELECT quote( ret_a(X'616263') ); 
+  SELECT quote( ret_r(X'616263') );
+  SELECT quote( ret_t(X'616263') );
+  SELECT quote( ret_b(X'616263') );
+  SELECT quote( ret_a(X'616263') );
 } {'abc' 'abc' 'abc' X'616263' X'616263'}
 
 do_test 17.6.1 {
@@ -848,21 +861,70 @@ do_catchsql_test 19.911 {
 } {1 {invalid command name "bind_fallback_does_not_exist"}}
 db bind_fallback {}
 
-#-------------------------------------------------------------------------
+# 2025-05-05: the -asdict eval flag
+#
 do_test 20.0 {
+  execsql {CREATE TABLE tad(a,b)}
+  execsql {INSERT INTO tad(a,b) VALUES('aa','bb'),('AA','BB')}
+  db eval -asdict {
+    SELECT a, b FROM tad WHERE 0
+  } D {}
+  set D
+} {* {a b}}
+
+do_test 20.1 {
+  unset D
+  set i 0
+  set res {}
+  set colNames {}
+  db eval -asdict {
+    SELECT a, b FROM tad ORDER BY a
+  } D {
+    dict set D i [incr i]
+    lappend res $i [dict get $D a] [dict get $D b]
+    if {1 == $i} {
+      set colNames [dict get $D *]
+    }
+  }
+  lappend res $colNames
+  unset D
+  set res
+} {1 AA BB 2 aa bb {a b}}
+
+do_test 20.2 {
+  set res {}
+  db eval -asdict -withoutnulls {
+    SELECT n, a, b FROM (
+      SELECT 1 as n, 'aa' as a, NULL as b
+      UNION ALL
+      SELECT 2 as n, NULL as a, 'bb' as b
+    )
+    ORDER BY n
+  } D {
+    dict unset D *
+    lappend res [dict values $D]
+  }
+  unset D
+  execsql {DROP TABLE tad}
+  set res
+} {{1 aa} {2 bb}}
+
+#-------------------------------------------------------------------------
+do_test 21.0 {
   db transaction {
     db close
   }
 } {}
 
-do_test 20.1 {
+do_test 21.1 {
   sqlite3 db test.db
   set rc [catch {
     db eval {SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3} { db close }
   } msg]
   list $rc $msg
 } {1 {invalid command name "db"}}
-  
+
+
 
 proc closedb {} {
   db close
@@ -874,7 +936,7 @@ sqlite3 db test.db
 db func closedb closedb
 db func func1 func1
 
-do_test 20.2 {
+do_test 21.2 {
   set rc [catch {
     db eval {
       SELECT closedb(),func1() UNION ALL SELECT 20,30 UNION ALL SELECT 30,40
@@ -884,9 +946,10 @@ do_test 20.2 {
 } {0 {10 1 20 30 30 40}}
 
 sqlite3 db :memory:
-do_test 21.1 {
+do_test 22.1 {
   catch {db eval {SELECT 1 2 3;}} msg
   db erroroffset
 } {9}
 
+
 finish_test
diff --git a/test/tester.tcl b/test/tester.tcl
index 021830aa95..5a3da60cc5 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -310,66 +310,6 @@ proc do_delete_file {force args} {
   }
 }
 
-if {$::tcl_platform(platform) eq "windows"} {
-  proc do_remove_win32_dir {args} {
-    set nRetry [getFileRetries]     ;# Maximum number of retries.
-    set nDelay [getFileRetryDelay]  ;# Delay in ms before retrying.
-
-    foreach dirName $args {
-      # On windows, sometimes even a [remove_win32_dir] can fail just after
-      # a directory is emptied. The cause is usually "tag-alongs" - programs
-      # like anti-virus software, automatic backup tools and various explorer
-      # extensions that keep a file open a little longer than we expect,
-      # causing the delete to fail.
-      #
-      # The solution is to wait a short amount of time before retrying the
-      # removal.
-      #
-      if {$nRetry > 0} {
-        for {set i 0} {$i < $nRetry} {incr i} {
-          set rc [catch {
-            remove_win32_dir $dirName
-          } msg]
-          if {$rc == 0} break
-          if {$nDelay > 0} { after $nDelay }
-        }
-        if {$rc} { error $msg }
-      } else {
-        remove_win32_dir $dirName
-      }
-    }
-  }
-
-  proc do_delete_win32_file {args} {
-    set nRetry [getFileRetries]     ;# Maximum number of retries.
-    set nDelay [getFileRetryDelay]  ;# Delay in ms before retrying.
-
-    foreach fileName $args {
-      # On windows, sometimes even a [delete_win32_file] can fail just after
-      # a file is closed. The cause is usually "tag-alongs" - programs like
-      # anti-virus software, automatic backup tools and various explorer
-      # extensions that keep a file open a little longer than we expect,
-      # causing the delete to fail.
-      #
-      # The solution is to wait a short amount of time before retrying the
-      # delete.
-      #
-      if {$nRetry > 0} {
-        for {set i 0} {$i < $nRetry} {incr i} {
-          set rc [catch {
-            delete_win32_file $fileName
-          } msg]
-          if {$rc == 0} break
-          if {$nDelay > 0} { after $nDelay }
-        }
-        if {$rc} { error $msg }
-      } else {
-        delete_win32_file $fileName
-      }
-    }
-  }
-}
-
 proc execpresql {handle args} {
   trace remove execution $handle enter [list execpresql $handle]
   if {[info exists ::G(perm:presql)]} {
@@ -554,7 +494,7 @@ if {[info exists cmdlinearg]==0} {
   }
   unset -nocomplain a
   set testdir [file normalize $testdir]
-  set cmdlinearg(TESTFIXTURE_HOME) [pwd]
+  set cmdlinearg(TESTFIXTURE_HOME) [file dirname [info nameofexec]]
   set cmdlinearg(INFO_SCRIPT) [file normalize [info script]]
   set argv0 [file normalize $argv0]
   if {$cmdlinearg(testdir)!=""} {
@@ -847,6 +787,9 @@ proc do_test {name cmd expected} {
         }
       } else {
         set ok [expr {[string compare $result $expected]==0}]
+        if {!$ok} {
+          set ok [fpnum_compare $result $expected]
+        }
       }
       if {!$ok} {
         # if {![info exists ::testprefix] || $::testprefix eq ""} {
@@ -866,6 +809,15 @@ proc do_test {name cmd expected} {
   flush stdout
 }
 
+# Like do_test except the test is not run in a slave interpreter
+# on Windows because of issues with ANSI and UTF8 I/O on Win11.
+#
+proc do_test_with_ansi_output {name cmd expected} {
+  if {![info exists ::SLAVE] || $::tcl_platform(platform) ne "windows"} {
+    uplevel 1 [list do_test $name $cmd $expected]
+  }
+}
+
 proc dumpbytes {s} {
   set r ""
   for {set i 0} {$i < [string length $s]} {incr i} {
@@ -884,11 +836,20 @@ proc catchcmd {db {cmd ""}} {
   set rc [catch { eval $line } msg]
   list $rc $msg
 }
+proc catchsafecmd {db {cmd ""}} {
+  global CLI
+  set out [open cmds.txt w]
+  puts $out $cmd
+  close $out
+  set line "exec $CLI -safe $db < cmds.txt"
+  set rc [catch { eval $line } msg]
+  list $rc $msg
+}
 
 proc catchcmdex {db {cmd ""}} {
   global CLI
   set out [open cmds.txt w]
-  fconfigure $out -encoding binary -translation binary
+  fconfigure $out -translation binary
   puts -nonewline $out $cmd
   close $out
   set line "exec -keepnewline -- $CLI $db < cmds.txt"
@@ -896,7 +857,7 @@ proc catchcmdex {db {cmd ""}} {
   foreach chan $chans {
     catch {
       set modes($chan) [fconfigure $chan]
-      fconfigure $chan -encoding binary -translation binary -buffering none
+      fconfigure $chan -translation binary -buffering none
     }
   }
   set rc [catch { eval $line } msg]
@@ -911,7 +872,7 @@ proc catchcmdex {db {cmd ""}} {
 
 proc filepath_normalize {p} {
   # test cases should be written to assume "unix"-like file paths
-  if {$::tcl_platform(platform)!="unix"} {
+  if {$::tcl_platform(platform) ne "unix"} {
     string map [list \\ / \{/ / .db\} .db] \
         [regsub -nocase -all {[a-z]:[/\\]+} $p {/}]
   } {
@@ -1033,7 +994,7 @@ proc query_plan_graph {sql} {
   }
   set a "\n  QUERY PLAN\n"
   append a [append_graph "  " dx cx 0]
-  regsub -all { 0x[A-F0-9]+\y} $a { xxxxxx} a
+  regsub -all {SUBQUERY 0x[A-F0-9]+\y} $a {SUBQUERY xxxxxx} a
   regsub -all {(MATERIALIZE|CO-ROUTINE|SUBQUERY) \d+\y} $a {\1 xxxxxx} a
   regsub -all {\((join|subquery)-\d+\)} $a {(\1-xxxxxx)} a
   return $a
@@ -1104,6 +1065,29 @@ proc do_eqp_test {name sql res} {
   }
 }
 
+# Do both an eqp_test and an execsql_test on the same SQL.
+#
+proc do_eqp_execsql_test {name sql res1 res2} {
+  if {[regexp {^\s+QUERY PLAN\n} $res1]} {
+
+    set query_plan [query_plan_graph $sql]
+
+    if {[list {*}$query_plan]==[list {*}$res1]} {
+      uplevel [list do_test ${name}a [list set {} ok] ok]
+    } else {
+      uplevel [list \
+        do_test ${name}a [list query_plan_graph $sql] $res1
+      ]
+    }
+  } else {
+    if {[string index $res 0]!="/"} {
+      set res1 "/*$res1*/"
+    }
+    uplevel do_execsql_test ${name}a [list "EXPLAIN QUERY PLAN $sql"] [list $res1]
+  }
+  uplevel do_execsql_test ${name}b [list $sql] [list $res2]
+}
+
 
 #-------------------------------------------------------------------------
 #   Usage: do_select_tests PREFIX ?SWITCHES? TESTLIST
@@ -1304,10 +1288,15 @@ proc finalize_testing {} {
          out of $nTest tests"
   } else {
     set cpuinfo {}
-    if {[catch {exec hostname} hname]==0} {set cpuinfo [string trim $hname]}
+    if {[catch {exec hostname} hname]==0} {
+      regsub {\.local$} $hname {} hname
+      set cpuinfo [string trim $hname]
+    }
     append cpuinfo " $::tcl_platform(os)"
     append cpuinfo " [expr {$::tcl_platform(pointerSize)*8}]-bit"
-    append cpuinfo " [string map {E -e} $::tcl_platform(byteOrder)]"
+    if {[string match big* $::tcl_platform(byteOrder)]} {
+      append cpuinfo " [string map {E -e} $::tcl_platform(byteOrder)]"
+    }
     output2 "SQLite [sqlite3 -sourceid]"
     output2 "$nErr errors out of $nTest tests on $cpuinfo"
   }
@@ -1348,6 +1337,12 @@ proc finalize_testing {} {
       incr nErr
     }
   }
+  # BEGIN SQLCIPHER 
+  # prior to calculating malloc stats, call sqlite3_shutdown to invoke
+  # sqlcipher_extra_shutdown() to release private heap memory if all
+  # private allocations have been freed
+  sqlite3_shutdown
+  # END SQLCIPHER
   if {[lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1]>0 ||
               [sqlite3_memory_used]>0} {
     output2 "Unfreed memory: [sqlite3_memory_used] bytes in\
@@ -1744,7 +1739,7 @@ proc ifcapable {expr code {else ""} {elsecode ""}} {
   return -code $c $r
 }
 
-# This proc execs a seperate process that crashes midway through executing
+# This proc execs a separate process that crashes midway through executing
 # the SQL script $sql on database test.db.
 #
 # The crash occurs during a sync() of file $crashfile. When the crash
@@ -1796,6 +1791,11 @@ proc crashsql {args} {
   # cfSync(), which can be different then what TCL uses by
   # default, so here we force it to the "nativename" format.
   set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]]
+  ifcapable winrt {
+    # Except on winrt. Winrt has no way to transform a relative path into
+    # an absolute one, so it just uses the relative paths.
+    set cfile $crashfile
+  }
 
   set f [open crash.tcl w]
   puts $f "sqlite3_initialize ; sqlite3_shutdown"
@@ -1838,7 +1838,7 @@ proc crashsql {args} {
   # error message.  We map that to the expected message
   # so that we don't have to change all of the test
   # cases.
-  if {$::tcl_platform(platform)=="windows"} {
+  if {$::tcl_platform(platform) eq "windows"} {
     if {$msg=="child killed: unknown signal"} {
       set msg "child process exited abnormally"
     }
@@ -1889,7 +1889,7 @@ proc crash_on_write {args} {
   # error message.  We map that to the expected message
   # so that we don't have to change all of the test
   # cases.
-  if {$::tcl_platform(platform)=="windows"} {
+  if {$::tcl_platform(platform) eq "windows"} {
     if {$msg=="child killed: unknown signal"} {
       set msg "child process exited abnormally"
     }
@@ -2538,7 +2538,7 @@ proc test_restore_config_pagecache {} {
 }
 
 proc test_binary_name {nm} {
-  if {$::tcl_platform(platform)=="windows"} {
+  if {$::tcl_platform(platform) eq "windows"} {
     set ret "$nm.exe"
   } else {
     set ret $nm
diff --git a/test/testloadext.c b/test/testloadext.c
new file mode 100644
index 0000000000..94cd312e44
--- /dev/null
+++ b/test/testloadext.c
@@ -0,0 +1,98 @@
+/*
+** 2025-10-14
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Test the ability of run-time extension loading to use the
+** very latest interfaces.
+**
+** Compile something like this:
+**
+** Linux:  gcc -g -fPIC shared testloadext.c -o testloadext.so
+**
+** Mac:    cc -g -fPIC -dynamiclib testloadext.c -o testloadext.dylib
+**
+** Win11:  cl testloadext.c -link -dll -out:testloadext.dll
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <assert.h>
+#include <string.h>
+
+/*
+** Implementation of the set_errmsg(CODE,MSG) SQL function.
+**
+** Raise an error that has numeric code CODE and text message MSG
+** using the sqlite3_set_errmsg() API.
+*/
+static void seterrmsgfunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3 *db;
+  char *zRes;
+  int rc;
+  assert( argc==2 );
+  db = sqlite3_context_db_handle(context);
+  rc = sqlite3_set_errmsg(db, 
+     sqlite3_value_int(argv[0]),
+     sqlite3_value_text(argv[1]));
+  zRes = sqlite3_mprintf("%d %d %s",
+              rc, sqlite3_errcode(db), sqlite3_errmsg(db));
+  sqlite3_result_text64(context, zRes, strlen(zRes),
+                        SQLITE_TRANSIENT, SQLITE_UTF8);
+  sqlite3_free(zRes);
+}
+
+/*
+** Implementation of the tempbuf_spill() SQL function.
+**
+** Return the value of SQLITE_DBSTATUS_TEMPBUF_SPILL.
+*/
+static void tempbuf_spill_func(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  sqlite3 *db;
+  sqlite3_int64 iHi = 0, iCur = 0;
+  int rc;
+  int bReset;
+  assert( argc==1 );
+  bReset = sqlite3_value_int(argv[0]);
+  db = sqlite3_context_db_handle(context);
+  (void)sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL,
+                            &iCur, &iHi, bReset);
+  sqlite3_result_int64(context, iCur);
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_testloadext_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "set_errmsg", 2,
+                   SQLITE_UTF8,
+                   0, seterrmsgfunc, 0, 0);
+  if( rc ) return rc;
+  rc = sqlite3_create_function(db, "tempbuf_spill", 1,
+                   SQLITE_UTF8,
+                   0, tempbuf_spill_func, 0, 0);
+  if( rc ) return rc;
+  return SQLITE_OK;
+}
diff --git a/test/testrunner.tcl b/test/testrunner.tcl
old mode 100644
new mode 100755
index 4e21ce08ab..5150363685
--- a/test/testrunner.tcl
+++ b/test/testrunner.tcl
@@ -1,43 +1,145 @@
+#!/bin/sh
+# Script to runs tests for SQLite.  Run with option "help" for more info. \
+exec tclsh "$0" "$@"
 
 set dir [pwd]
-set testdir [file dirname $argv0]
+set testdir [file normalize [file dirname $argv0]]
 set saved $argv
 set argv [list]
 source [file join $testdir testrunner_data.tcl]
+
+# Estimated amount of work required by displaytype, relative to 'tcl'
+#
+set estwork(tcl)    1
+set estwork(fuzz)   22
+set estwork(bld)    66
+set estwork(make)   102
+
+set estworkfile [file join $testdir testrunner_estwork.tcl]
+if {[file readable $estworkfile]} {
+  source $estworkfile
+}
 source [file join $testdir permutations.test]
 set argv $saved
 cd $dir
 
+# This script requires an interpreter that supports [package require sqlite3]
+# to run. If this is not such an intepreter, see if there is a [testfixture]
+# in the current directory. If so, run the command using it. If not, 
+# recommend that the user build one.
+#
+proc find_interpreter {} {
+  global dir
+  set interpreter [file tail [info nameofexec]]
+  set rc [catch { package require sqlite3 }]
+  if {$rc} {
+    if {[file readable pkgIndex.tcl] && [catch {source pkgIndex.tcl}]==0} {
+      set rc [catch { package require sqlite3 }]
+    }
+  }
+  if {$rc} {
+    if { [string match -nocase testfixture* $interpreter]==0
+      && [file executable ./testfixture]
+    } {
+      puts "Failed to find tcl package sqlite3. Restarting with ./testfixture.."
+      set status [catch {
+        exec [trd_get_bin_name testfixture] [info script] {*}$::argv >@ stdout
+      } msg]
+      exit $status
+    }
+  }
+  if {$rc} {
+    puts "Cannot find tcl package sqlite3: Trying to build it now..."
+    if {$::tcl_platform(platform) eq "windows"} {
+      set bat [open make-tcl-extension.bat w]
+      puts $bat "nmake /f Makefile.msc tclextension"
+      close $bat
+      catch {exec -ignorestderr -- make-tcl-extension.bat}
+    } else {
+      catch {exec make tclextension}
+    }
+    if {[file readable pkgIndex.tcl] && [catch {source pkgIndex.tcl}]==0} {
+      set rc [catch { package require sqlite3 }]
+    }
+    if {$rc==0} {
+      puts "The SQLite tcl extension was successfully built and loaded."
+      puts "Run \"make tclextension-install\" to avoid having to rebuild\
+            it in the future."
+    } else {
+      puts "Unable to build the SQLite tcl extension"
+    }
+  }
+  if {$rc} {
+    puts stderr "Cannot find a working instance of the SQLite tcl extension."
+    puts stderr "Run \"make tclextension\" or \"make testfixture\" and\
+                 try again..."
+    exit 1
+  }
+}
+find_interpreter
+
+# Usually this script is run by [testfixture]. But it can also be run
+# by a regular [tclsh]. For these cases, emulate the [clock_milliseconds] 
+# command.
+if {[info commands clock_milliseconds]==""} {
+  proc clock_milliseconds {} {
+    clock milliseconds
+  }
+}
+
 #-------------------------------------------------------------------------
 # Usage:
 #
 proc usage {} {
   set a0 [file tail $::argv0]
 
-  puts stderr [string trim [subst -nocommands {
+  puts [string trim [subst -nocommands {
 Usage: 
     $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS?
     $a0 PERMUTATION FILE
+    $a0 errors ?-v|--verbose? ?-s|--summary? ?PATTERN?
+    $a0 help
+    $a0 joblist ?PATTERN?
     $a0 njob ?NJOB?
-    $a0 status
+    $a0 script ?-msvc? CONFIG
+    $a0 status ?-d SECS? ?--cls?
+    $a0 halt
+    $a0 estwork
 
   where SWITCHES are:
-    --jobs NUMBER-OF-JOBS
-    --fuzztest
-    --zipvfs ZIPVFS-SOURCE-DIR
+    --buildonly              Build test exes but do not run tests
+    --cases DISPLAYNAME      Only run test that match DISPLAYNAME
+    --config CONFIGS         Only use configs on comma-separate list CONFIGS
+    --dryrun                 Write what would have happened to testrunner.log
+    --explain                Write summary to stdout
+    --fuzzdb FILENAME        Additional external fuzzcheck database
+    --jobs NUM               Run tests using NUM separate processes
+    --omit CONFIGS           Omit configs on comma-separated list CONFIGS
+    --status                 Show the full "status" report while running
+    --stop-on-coredump       Stop running if any test segfaults
+    --stop-on-error          Stop running after any reported error
+    --zipvfs ZIPVFSDIR       ZIPVFS source directory
+
+Special values for PERMUTATION that work with plain tclsh:
+
+    list      - show all allowed PERMUTATION arguments.
+    mdevtest  - tests recommended prior to normal development check-ins.
+    release   - full release test with various builds.
+    sdevtest  - like mdevtest but using ASAN and UBSAN.
 
-Interesting values for PERMUTATION are:
+Other PERMUTATION arguments must be run using testfixture, not tclsh:
 
-    veryquick - a fast subset of the tcl test scripts. This is the default.
-    full      - all tcl test scripts.
     all       - all tcl test scripts, plus a subset of test scripts rerun
                 with various permutations.
-    release   - full release test with various builds.
+    full      - all tcl test scripts.
+    veryquick - a fast subset of the tcl test scripts. This is the default.
 
 If no PATTERN arguments are present, all tests specified by the PERMUTATION
 are run. Otherwise, each pattern is interpreted as a glob pattern. Only
 those tcl tests for which the final component of the filename matches at
-least one specified pattern are run.
+least one specified pattern are run.  The glob wildcard '*' is prepended
+to the pattern if it does not start with '^' and appended to every
+pattern that does not end with '$'.
 
 If no PATTERN arguments are present, then various fuzztest, threadtest
 and other tests are run as part of the "release" permutation. These are
@@ -47,16 +149,25 @@ If a PERMUTATION is specified and is followed by the path to a Tcl script
 instead of a list of patterns, then that single Tcl test script is run
 with the specified permutation.
 
-The --fuzztest option is ignored if the PERMUTATION is "release". Otherwise,
-if it is present, then "make -C <dir> fuzztest" is run as part of the tests,
-where <dir> is the directory containing the testfixture binary used to
-run the script.
-
 The "status" and "njob" commands are designed to be run from the same
 directory as a running testrunner.tcl script that is running tests. The
 "status" command prints a report describing the current state and progress 
-of the tests. The "njob" command may be used to query or modify the number
-of sub-processes the test script uses to run tests.
+of the tests.  Use the "-d N" option to have the status display clear the
+screen and repeat every N seconds.  The "njob" command may be used to query
+or modify the number of sub-processes the test script uses to run tests.
+
+The "script" command outputs the script used to build a configuration.
+Add the "-msvc" option for a Windows-compatible script. For a list of
+available configurations enter "$a0 script help".
+
+The "errors" commands shows the output of tests that failed in the
+most recent run.  Complete output is shown if the -v or --verbose options
+are used.  Otherwise, an attempt is made to minimize the output to show
+only the parts that contain the error messages.  The --summary option just
+shows the jobs that failed.  If PATTERN are provided, the error information
+is only provided for jobs that match PATTERN.
+
+Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md
   }]]
 
   exit 1
@@ -73,23 +184,30 @@ of sub-processes the test script uses to run tests.
 proc guess_number_of_cores {} {
   if {[catch {number_of_cores} ret]} {
     set ret 4
-  
-    if {$::tcl_platform(os)=="Darwin"} {
-      set cmd "sysctl -n hw.logicalcpu"
+    if {$::tcl_platform(platform) eq "windows"} {
+      catch { set ret $::env(NUMBER_OF_PROCESSORS) }
     } else {
-      set cmd "nproc"
-    }
-    catch {
-      set fd [open "|$cmd" r]
-      set ret [gets $fd]
-      close $fd
-      set ret [expr $ret]
+      if {$::tcl_platform(os)=="Darwin"} {
+        set cmd "sysctl -n hw.logicalcpu"
+      } else {
+        set cmd "nproc"
+      }
+      catch {
+        set fd [open "|$cmd" r]
+        set ret [gets $fd]
+        close $fd
+        set ret [expr $ret]
+      }
     }
   }
   return $ret
 }
 
 proc default_njob {} {
+  global env
+  if {[info exists env(NJOB)] && $env(NJOB)>=1} {
+    return $env(NJOB)
+  }
   set nCore [guess_number_of_cores]
   if {$nCore<=2} {
     set nHelper 1
@@ -114,49 +232,138 @@ set TRG(cmdline) $argv
 set TRG(reporttime) 2000
 set TRG(fuzztest) 0                 ;# is the fuzztest option present.
 set TRG(zipvfs) ""                  ;# -zipvfs option, if any
+set TRG(buildonly) 0                ;# True if --buildonly option 
+set TRG(config) {}                  ;# Only build the named configurations
+set TRG(omitconfig) {}              ;# Do not build these configurations
+set TRG(dryrun) 0                   ;# True if --dryrun option 
+set TRG(explain) 0                  ;# True for the --explain option
+set TRG(stopOnError) 0              ;# Stop running at first failure
+set TRG(stopOnCore) 0               ;# Stop on a core-dump
+set TRG(fullstatus) 0               ;# Full "status" report while running
+set TRG(case) {}                    ;# Only run cases matching this GLOB pattern
 
 switch -nocase -glob -- $tcl_platform(os) {
   *darwin* {
-    set TRG(platform) osx
-    set TRG(make)     make.sh
-    set TRG(makecmd)  "bash make.sh"
+    set TRG(platform)    osx
+    set TRG(make)        make.sh
+    set TRG(makecmd)     "bash make.sh"
+    set TRG(testfixture) testfixture
+    set TRG(shell)       sqlite3
+    set TRG(run)         run.sh
+    set TRG(runcmd)      "bash run.sh"
+  }
+  *linux* - MSYS_NT* - MINGW64_NT* - MINGW32_NT* {
+    set TRG(platform)    linux
+    set TRG(make)        make.sh
+    set TRG(makecmd)     "bash make.sh"
+    set TRG(testfixture) testfixture
+    set TRG(shell)       sqlite3
+    set TRG(run)         run.sh
+    set TRG(runcmd)      "bash run.sh"
   }
-  *linux* {
-    set TRG(platform) linux
-    set TRG(make)     make.sh
-    set TRG(makecmd)  "bash make.sh"
+  *openbsd* {
+    set TRG(platform)    linux
+    set TRG(make)        make.sh
+    set TRG(makecmd)     "sh make.sh"
+    set TRG(testfixture) testfixture
+    set TRG(shell)       sqlite3
+    set TRG(run)         run.sh
+    set TRG(runcmd)      "sh run.sh"
   }
   *win* {
-    set TRG(platform) win
-    set TRG(make)     make.bat
-    set TRG(makecmd)  make.bat
+    set TRG(platform)    win
+    set TRG(make)        make.bat
+    set TRG(makecmd)     "call make.bat"
+    set TRG(testfixture) testfixture.exe
+    set TRG(shell)       sqlite3.exe
+    set TRG(run)         run.bat
+    set TRG(runcmd)      "run.bat"
+    if {"unix" eq $tcl_platform(platform)} {
+      # Presumably cygwin. This block gets testrunner.tcl started on
+      # Cygwin but then downstream tests all fail, at least in part
+      # because of the discrepancies in build target names which need
+      # .exe on cygwin but not on other Unix-like platforms.
+      set TRG(platform)  cygwin
+      set TRG(make)      make.sh
+      set TRG(makecmd)   "bash make.sh"
+      set TRG(testfixture) testfixture
+      set TRG(shell)       sqlite3
+      set TRG(run)       run.sh
+      set TRG(runcmd)    "bash run.sh"
+    }
   }
   default {
+    puts "tcl_platform(os)=$::tcl_platform(os)"
     error "cannot determine platform!"
   }
-} 
+}
 #-------------------------------------------------------------------------
 
 #-------------------------------------------------------------------------
 # The database schema used by the testrunner.db database.
 #
 set TRG(schema) {
-  DROP TABLE IF EXISTS script;
+  DROP TABLE IF EXISTS jobs;
   DROP TABLE IF EXISTS config;
 
-  CREATE TABLE script(
-    build TEXT DEFAULT '',
-    config TEXT,
-    filename TEXT,                -- full path to test script
-    slow BOOLEAN,                 -- true if script is "slow"
-    state TEXT CHECK( state IN ('', 'ready', 'running', 'done', 'failed') ),
-    time INTEGER,                 -- Time in ms
-    output TEXT,                  -- full output of test script
-    priority AS ((config='make') + ((config='build')*2) + (slow*4)),
-    jobtype AS (
-      CASE WHEN config IN ('build', 'make') THEN config ELSE 'script' END
-    ),
-    PRIMARY KEY(build, config, filename)
+  /*
+  ** This table contains one row for each job that testrunner.tcl must run
+  ** before the entire test run is finished.
+  **
+  ** jobid:
+  **   Unique identifier for each job. Must be a +ve non-zero number.
+  **
+  ** displaytype:
+  **   3 or 4 letter mnemonic for the class of tests this belongs to e.g.
+  **   "fuzz", "tcl", "make" etc.
+  **
+  ** displayname:
+  **   Name/description of job. For display purposes.
+  **
+  ** build:
+  **   If the job requires a make.bat/make.sh make wrapper (i.e. to build
+  **   something), the name of the build configuration it uses. See 
+  **   testrunner_data.tcl for a list of build configs. e.g. "Win32-MemDebug".
+  **
+  ** dirname:
+  **   If the job should use a well-known directory name for its 
+  **   sub-directory instead of an anonymous "testdir[1234...]" sub-dir
+  **   that is deleted after the job is finished.
+  **
+  ** cmd:
+  **   Bash or batch script to run the job.
+  **
+  ** depid:
+  **   The jobid value of a job that this job depends on. This job may not
+  **   be run before its depid job has finished successfully.
+  **
+  ** priority:
+  **   Higher values run first. Sometimes.
+  */
+  CREATE TABLE jobs(
+    /* Fields populated when db is initialized */
+    jobid INTEGER PRIMARY KEY,          -- id to identify job
+    displaytype TEXT NOT NULL,          -- Type of test (for one line report)
+    displayname TEXT NOT NULL,          -- Human readable job name
+    build TEXT NOT NULL DEFAULT '',     -- make.sh/make.bat file request, if any
+    dirname TEXT NOT NULL DEFAULT '',   -- directory name, if required
+    cmd TEXT NOT NULL,                  -- shell command to run
+    depid INTEGER,                      -- identifier of dependency (or '')
+    priority INTEGER NOT NULL,          -- higher priority jobs may run earlier
+  
+    /* Fields updated as jobs run */
+    starttime INTEGER,                  -- Start time (milliseconds since 1970)
+    endtime INTEGER,                    -- End time
+    span INTEGER,                       -- Total run-time in milliseconds
+    estwork INTEGER,                    -- Estimated amount of work
+    estkey TEXT,                        -- Key used to compute estwork
+    state TEXT CHECK( state IN ('','ready','running','done','failed','omit','halt') ),
+    ntest INT,                          -- Number of test cases run
+    nerr INT,                           -- Number of errors reported
+    svers TEXT,                         -- Reported SQLite version
+    pltfm TEXT,                         -- Host platform reported
+    output TEXT,                        -- test output
+    cwd TEXT                            -- working directory for test
   );
 
   CREATE TABLE config(
@@ -164,8 +371,8 @@ set TRG(schema) {
     value 
   ) WITHOUT ROWID;
 
-  CREATE INDEX i1 ON script(state, jobtype);
-  CREATE INDEX i2 ON script(state, priority);
+  CREATE INDEX i1 ON jobs(state, priority);
+  CREATE INDEX i2 ON jobs(depid);
 }
 #-------------------------------------------------------------------------
 
@@ -181,10 +388,11 @@ if {[llength $argv]==2
   set script [file normalize [lindex $argv 1]]
   set ::argv [list]
 
+  set testdir [file dirname $argv0]
+  source $::testdir/tester.tcl
+
   if {$permutation=="full"} {
 
-    set testdir [file dirname $argv0]
-    source $::testdir/tester.tcl
     unset -nocomplain ::G(isquick)
     reset_db
 
@@ -227,8 +435,8 @@ if {([llength $argv]==2 || [llength $argv]==1)
   sqlite3 mydb $TRG(dbname)
   if {[llength $argv]==2} {
     set param [lindex $argv 1]
-    if {[string is integer $param]==0 || $param<1 || $param>128} {
-      puts stderr "parameter must be an integer between 1 and 128"
+    if {[string is integer $param]==0 || $param<0 || $param>128} {
+      puts stderr "parameter must be an integer between 0 and 128"
       exit 1
     }
 
@@ -242,101 +450,403 @@ if {([llength $argv]==2 || [llength $argv]==1)
 #--------------------------------------------------------------------------
 
 #--------------------------------------------------------------------------
-# Check if this is the "status" command:
+# Check if this is the "halt" command:
 #
-if {[llength $argv]==1 
- && [string compare -nocase status [lindex $argv 0]]==0 
+if {[llength $argv]==1
+ && [string compare -nocase halt [lindex $argv 0]]==0
 } {
+  sqlite3 mydb $TRG(dbname)
+  mydb eval {UPDATE jobs SET state='halt' WHERE state IN ('ready','')}
+  mydb close
+  exit
+}
+#--------------------------------------------------------------------------
 
-  proc display_job {build config filename {tm ""}} {
-    if {$config=="build"} {
-      set fname "build: $filename"
-      set config ""
-    } elseif {$config=="make"} {
-      set fname "make: $filename"
-      set config ""
-    } else {
-      set fname [file normalize $filename]
-      if {[string first $::srcdir $fname]==0} {
-        set fname [string range $fname [string length $::srcdir]+1 end]
-      }
+#--------------------------------------------------------------------------
+# Check if this is the "estwork" command:
+#
+# Generate (on standard output) a set of estwork() values based on the lastest
+# test case, that can be used to replace the test/testrunner_estwork.tcl file.
+#
+if {[llength $argv]==1
+ && [string compare -nocase estwork [lindex $argv 0]]==0
+} {
+  sqlite3 mydb $TRG(dbname)
+  set njob [mydb one {SELECT count(*) FROM jobs WHERE state='done'}]
+  if {$njob<1000} {
+    puts "Too few completed jobs to do a work estimate."
+    puts "Have $njob but not need at least 1000."
+    mydb close
+    exit 1
+  }
+  set badjobs [mydb one {SELECT count(*) FROM jobs WHERE state<>'done'}]
+  if {$badjobs} {
+    puts "Database contains $badjobs incomplete jobs."
+    mydb close
+    exit 1
+  }
+  set half [mydb one {SELECT count(*)/2 FROM jobs WHERE displaytype='tcl'}]
+  set scale [mydb one {SELECT span FROM jobs WHERE displaytype='tcl'
+                        ORDER BY span LIMIT 1 OFFSET $half}]
+  mydb eval {
+     SELECT estkey, CAST(avg(span)/$scale AS INT) AS cost
+       FROM jobs
+      GROUP BY estkey
+      HAVING cost>=2
+  } {
+    set estwork($estkey) $cost
+  }
+  set avgtcl [mydb one {SELECT avg(span) FROM jobs WHERE displaytype='tcl'}]
+  set estwork(tcl) 1
+  foreach type {bld fuzz make} {
+    set avg [mydb one {SELECT avg(span) FROM jobs WHERE displaytype=$type}]
+    if {$avg!=""} {
+      set estwork($type) [expr {int($avg/$avgtcl)}]
     }
-    set dfname [format %-33s $fname]
+  }
+  mydb close
+  puts "# Estimated relative cost of various jobs, based on the \"estkey\" field."
+  puts "# Computed by the \"test/testrunner.tcl estwork\" command."
+  puts "#"
+  foreach key [lsort [array names estwork]] {
+    puts "set [list estwork($key)] $estwork($key)"
+  }
+  exit
+}
+#--------------------------------------------------------------------------
 
-    set dbuild ""
-    set dconfig ""
-    set dparams ""
-    set dtm ""
-    if {$build!=""} { set dbuild $build }
-    if {$config!="" && $config!="full"} { set dconfig $config }
-    if {$dbuild!="" || $dconfig!=""} {
-      append dparams "("
-      if {$dbuild!=""}                 {append dparams "build=$dbuild"}
-      if {$dbuild!="" && $dconfig!=""} {append dparams " "}
-      if {$dconfig!=""}                {append dparams "config=$dconfig"}
-      append dparams ")"
-      set dparams [format %-33s $dparams]
-    }
-    if {$tm!=""} {
-      set dtm "\[${tm}ms\]"
-    }
-    puts "  $dfname $dparams $dtm"
+#--------------------------------------------------------------------------
+# Check if this is the "help" command:
+#
+if {[string compare -nocase help [lindex $argv 0]]==0} {
+  usage
+}
+#--------------------------------------------------------------------------
+
+#--------------------------------------------------------------------------
+# Check if this is the "script" command:
+#
+if {[string compare -nocase script [lindex $argv 0]]==0} {
+  if {[llength $argv]!=2 && !([llength $argv]==3&&[lindex $argv 1]=="-msvc")} {
+    usage
   }
 
-  sqlite3 mydb $TRG(dbname)
-  mydb timeout 1000
-  mydb eval BEGIN
+  set bMsvc [expr ([llength $argv]==3)]
+  set config [lindex $argv [expr [llength $argv]-1]]
 
-  set cmdline [mydb one { SELECT value FROM config WHERE name='cmdline' }]
-  set nJob [mydb one { SELECT value FROM config WHERE name='njob' }]
-  set tm [expr [clock_milliseconds] - [mydb one {
-    SELECT value FROM config WHERE name='start'
-  }]]
+  puts [trd_buildscript $config [file dirname $testdir] $bMsvc]
+  exit
+}
 
-  set total 0
-  foreach s {"" ready running done failed} { set S($s) 0 }
-  mydb eval {
-    SELECT state, count(*) AS cnt FROM script GROUP BY 1
+# Compute an elapse time string MM:SS or HH:MM:SS based on the
+# number of milliseconds in the argument.
+#
+proc elapsetime {ms} {
+  if {$ms==""} {set ms 0}
+  set s [expr {int(($ms+500.0)*0.001)}]
+  set hr [expr {$s/3600}]
+  set mn [expr {($s/60)%60}]
+  set sc [expr {$s%60}]
+  if {$hr>0} {
+    return [format %02d:%02d:%02d $hr $mn $sc]
+  } else {
+    return [format %02d:%02d $mn $sc]
+  }
+}
+
+# Helper routine for show_status
+#
+proc display_job {jobdict {tm ""}} {
+  array set job $jobdict
+  if {[string length $job(displayname)]>65} {
+    set dfname [format %.65s... $job(displayname)]
+  } else {
+    set dfname [format %-68s $job(displayname)]
+  }
+  set dtm ""
+  if {$tm!=""} {
+    set dtm [expr {$tm-$job(starttime)}]
+    set dtm [format %8s [elapsetime $dtm]]
+  } else {
+    set dtm [format %8s ""]
+  }
+  puts "  $dfname $dtm"
+}
+
+# This procedure shows the "status" page.  It uses the database
+# connect passed in as the "db" parameter.  If the "cls" parameter
+# is true, then VT100 escape codes are used to format the display.
+#
+proc show_status {db cls} {
+  global TRG
+  $db eval BEGIN
+  if {[catch {
+    set cmdline [$db one { SELECT value FROM config WHERE name='cmdline' }]
+    set nJob [$db one { SELECT value FROM config WHERE name='njob' }]
+  } msg]} {
+    if {$cls} {puts "\033\[H\033\[2J"}
+    puts "Cannot read database: $TRG(dbname)"
+    return
+  }
+  set now [clock_milliseconds]
+  set tm [$db one {
+    SELECT 
+      COALESCE((SELECT value FROM config WHERE name='end'), $now) -
+      (SELECT value FROM config WHERE name='start')
+  }]
+
+  set totalw 0
+  foreach s {"" ready running done failed omit} { set S($s) 0; set W($s) 0; }
+  set workpending 0
+  $db eval {
+    SELECT state, count(*) AS cnt, sum(estwork) AS ew FROM jobs GROUP BY 1
   } {
     incr S($state) $cnt
-    incr total $cnt
+    incr W($state) $ew
+    incr totalw $ew
   }
-  set fin [expr $S(done)+$S(failed)]
+  set nt 0
+  set ne 0
+  $db eval {
+    SELECT sum(ntest) AS nt, sum(nerr) AS ne FROM jobs HAVING nt>0
+  } break
+  set fin [expr $W(done)+$W(failed)+$W(omit)]
   if {$cmdline!=""} {set cmdline " $cmdline"}
 
-  set f ""
-  if {$S(failed)>0} {
-    set f "$S(failed) FAILED, "
+  if {$cls} {
+    # Move the cursor to the top-left corner.  Each iteration will simply
+    # overwrite.
+    puts -nonewline "\033\[H"
+    flush stdout
   }
-  puts "Command line: \[testrunner.tcl$cmdline\]"
-  puts "Jobs:         $nJob"
-  puts "Summary:      ${tm}ms, ($fin/$total) finished, ${f}$S(running) running"
+  puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]"]
+  puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$totalw jobs,\
+                         $ne errors, $nt tests"]
 
   set srcdir [file dirname [file dirname $TRG(info_script)]]
+  set line "Running: $S(running) (max: $nJob)"
+  if {$S(running)>0 && [set pct [expr {int(($fin*100.0)/$totalw)}]]>=4} {
+    set tmleft [expr {($tm/double($fin))*($totalw-$fin)}]
+    if {$tmleft<0.02*$tm} {
+      set tmleft [expr {$tm*0.02}]
+    }
+    set etc " ETC [elapsetime $tmleft]"
+    if {[string length $line]+[string length $etc]<80} {
+      append line $etc
+    }
+    # append line " $pct%"
+  }
+  puts [format %-79.79s $line]
   if {$S(running)>0} {
-    puts "Running: "
-    set now [clock_milliseconds]
-    mydb eval {
-      SELECT build, config, filename, time FROM script WHERE state='running'
-      ORDER BY time 
-    } {
-      display_job $build $config $filename [expr $now-$time]
+    $db eval {
+      SELECT * FROM jobs WHERE state='running' ORDER BY starttime 
+    } job {
+      display_job [array get job] $now
     }
   }
   if {$S(failed)>0} {
-    puts "Failures: "
-    mydb eval {
-      SELECT build, config, filename FROM script WHERE state='failed'
-      ORDER BY 3
-    } {
-      display_job $build $config $filename
+    # $toshow is the number of failures to report.  In $cls mode,
+    # status tries to limit the number of failure reported so that
+    # the status display does not overflow a 24-line terminal.  It will
+    # always show at least the most recent 4 failures, even if an overflow
+    # is needed.  No limit is imposed for a status within $cls.
+    #
+    if {$cls && $S(failed)>18-$S(running)} {
+      set toshow [expr {18-$S(running)}]
+      if {$toshow<4} {set toshow 4}
+      set shown " (must recent $toshow shown)"
+    } else {
+      set toshow $S(failed)
+      set shown ""
+    }
+    puts [format %-79s  "Failed:  $S(failed) $shown"]
+    $db eval {
+      SELECT * FROM jobs WHERE state='failed'
+       ORDER BY endtime DESC LIMIT $toshow
+    } job {
+      display_job [array get job]
+    }
+    set nOmit [$db one {SELECT count(*) FROM jobs WHERE state='omit'}]
+    if {$nOmit} {
+      puts [format %-79s "  ... $nOmit jobs omitted due to failures"]
+    }
+  }
+  if {$cls} {
+    # Clear everything else to the bottom of the screen
+    puts -nonewline "\033\[0J"
+    flush stdout
+  }
+  $db eval COMMIT
+}
+
+  
+
+#--------------------------------------------------------------------------
+# Check if this is the "status" command:
+#
+if {[llength $argv]>=1 
+ && [string compare -nocase status [lindex $argv 0]]==0 
+} {
+  set delay 0
+  set cls 0
+  for {set ii 1} {$ii<[llength $argv]} {incr ii} {
+    set a0 [lindex $argv $ii]
+    if {$a0=="-d" && $ii+1<[llength $argv]} {
+      incr ii
+      set delay [lindex $argv $ii]
+      if {![string is integer -strict $delay]} {
+        puts "Argument to -d should be an integer"
+        exit 1
+      }
+    } elseif {$a0=="-cls" || $a0=="--cls"} {
+      set cls 1
+    } else {
+      puts "unknown option: \"$a0\""
+      exit 1
     }
   }
- 
+
+  if {![file readable $TRG(dbname)]} {
+    puts "Database missing: $TRG(dbname)"
+    exit
+  }
+  sqlite3 mydb $TRG(dbname)
+  mydb timeout 2000
+
+  # Clear the whole screen initially.
+  #
+  if {$delay>0 || $cls} {puts -nonewline "\033\[2J"}
+
+  while {1} {
+    show_status mydb [expr {$delay>0 || $cls}]
+    if {$delay<=0} break
+    after [expr {$delay*1000}]
+  }
   mydb close
   exit
 }
 
+#--------------------------------------------------------------------------
+# Check if this is the "joblist" command:
+#
+if {[llength $argv]>=1 
+ && [string compare -nocase "joblist" [lindex $argv 0]]==0 
+} {
+  set pattern {}
+  for {set ii 1} {$ii<[llength $argv]} {incr ii} {
+    set a0 [lindex $argv $ii]
+    if {$pattern==""} {
+      set pattern [string trim $a0 *]
+    } else {
+      puts "unknown option: \"$a0\""
+      exit 1
+    }
+  }
+  set SQL {SELECT displaytype, displayname, state FROM jobs}
+  if {$pattern!=""} {
+    regsub -all {[^a-zA-Z0-9*.-/]} $pattern ? pattern
+    set pattern [string tolower $pattern]
+    append SQL \
+       " WHERE lower(concat(state,' ',displaytype,' ',displayname)) GLOB '*$pattern*'"
+  }
+  append SQL " ORDER BY starttime"
+
+  if {![file readable $TRG(dbname)]} {
+    puts "Database missing: $TRG(dbname)"
+    exit
+  }
+  sqlite3 mydb $TRG(dbname)
+  mydb timeout 2000
+
+  mydb eval $SQL {
+    set label UNKNOWN
+    switch -- $state {
+      ready {set label READY}
+      done {set label DONE}
+      failed {set label FAILED}
+      omit {set label OMIT}
+      running {set label RUNNING}
+    }
+    puts [format {%-7s %-5s %s} $label $displaytype $displayname]
+  }
+  mydb close
+  exit
+}
+
+# Scan the output of all jobs looking for the summary lines that
+# report the number of test cases and the number of errors.
+# Aggregate these numbers and return them.
+#
+proc aggregate_test_counts {db} {
+  set ne 0
+  set nt 0
+  $db eval {SELECT sum(nerr) AS ne, sum(ntest) as nt FROM jobs} break
+  return [list $ne $nt]
+}
+
+#--------------------------------------------------------------------------
+# Check if this is the "errors" command:
+#
+if {[llength $argv]>=1
+ && ([string compare -nocase errors [lindex $argv 0]]==0 ||
+     [string match err* [lindex $argv 0]]==1)
+} {
+  set verbose 0
+  set pattern {}
+  set summary 0
+  for {set ii 1} {$ii<[llength $argv]} {incr ii} {
+    set a0 [lindex $argv $ii]
+    if {$a0=="-v" || $a0=="--verbose" || $a0=="-verbose"} {
+      set verbose 1
+    } elseif {$a0=="-s" || $a0=="--summary" || $a0=="-summary"} {
+      set summary 1
+    } elseif {$pattern==""} {
+      set pattern *[string trim $a0 *]*
+    } else {
+      puts "unknown option: \"$a0\"".  Use --help for more info."
+      exit 1
+    }
+  }
+  set cnt 0
+  sqlite3 mydb $TRG(dbname)
+  mydb timeout 5000
+  if {$summary} {
+    set sql "SELECT displayname FROM jobs WHERE state='failed'"
+  } else {
+    set sql "SELECT displaytype, displayname, output FROM jobs \
+              WHERE state='failed'"
+  }
+  if {$pattern!=""} {
+    regsub -all {[^a-zA-Z0-9*/ ?]} $pattern . pattern
+    append sql " AND displayname GLOB '$pattern'"
+  }
+  mydb eval $sql {
+    if {$summary} {
+      puts "FAILED: $displayname"
+      continue
+    }
+    puts "**** $displayname ****"
+    if {$verbose || $displaytype!="tcl"} {
+      puts $output
+    } else {
+      foreach line [split $output \n] {
+        if {[string match {!*} $line] || [string match *failed* $line]} {
+          puts $line
+        }
+      }
+    }
+    incr cnt
+  }
+  if {$pattern==""} {
+    set summary [aggregate_test_counts mydb]
+    mydb close
+    puts "Total [lindex $summary 0] errors out of [lindex $summary 1] tests"
+  } else {
+    mydb close
+  }
+  exit
+}
+
 #-------------------------------------------------------------------------
 # Parse the command line.
 #
@@ -350,12 +860,42 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} {
       incr ii
       set TRG(nJob) [lindex $argv $ii]
       if {$isLast} { usage }
-    } elseif {($n>2 && [string match "$a*" --fuzztest]) || $a=="-f"} {
-      set TRG(fuzztest) 1
     } elseif {($n>2 && [string match "$a*" --zipvfs]) || $a=="-z"} {
       incr ii
-      set TRG(zipvfs) [lindex $argv $ii]
+      set TRG(zipvfs) [file normalize [lindex $argv $ii]]
       if {$isLast} { usage }
+    } elseif {($n>2 && [string match "$a*" --buildonly]) || $a=="-b"} {
+      set TRG(buildonly) 1
+    } elseif {($n>2 && [string match "$a*" --config]) || $a=="-c"} {
+      incr ii
+      set TRG(config) [lindex $argv $ii]
+    } elseif {($n>2 && [string match "$a*" --dryrun]) || $a=="-d"} {
+      set TRG(dryrun) 1
+    } elseif {($n>2 && [string match "$a*" --explain]) || $a=="-e"} {
+      set TRG(explain) 1
+    } elseif {$n>2 && [string match "$a*" --omit]} {
+      incr ii
+      set TRG(omitconfig) [lindex $argv $ii]
+    } elseif {$n>2 && [string match "$a*" --cases]} {
+      incr ii
+      set TRG(case) [lindex $argv $ii]
+    } elseif {$n>2 && [string match "$a*" --fuzzdb]} {
+      incr ii
+      set env(FUZZDB) [lindex $argv $ii]
+    } elseif {[string match "$a*" --stop-on-error]} {
+      set TRG(stopOnError) 1
+    } elseif {[string match "$a*" --stop-on-coredump]} {
+      set TRG(stopOnCore) 1
+    } elseif {[string match "$a*" --status]} {
+      if {$tcl_platform(platform) eq "windows"} {
+        puts stdout \
+"The --status option is not available on Windows. A suggested work-around"
+        puts stdout \
+"is to run the following command in a separate window:\n"
+        puts stdout "   [info nameofexe] $argv0 status -d 2\n"
+      } else {
+        set TRG(fullstatus) 1
+      }
     } else {
       usage
     }
@@ -365,8 +905,6 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} {
 }
 set argv [list]
 
-
-
 # This script runs individual tests - tcl scripts or [make xyz] commands -
 # in directories named "testdir$N", where $N is an integer. This variable
 # contains a list of integers indicating the directories in use.
@@ -411,8 +949,6 @@ proc dirs_allocDir {} {
   return $iRet
 }
 
-set testdir [file dirname $argv0]
-
 # Check that directory $dir exists. If it does not, create it. If 
 # it does, delete its contents.
 #
@@ -424,256 +960,618 @@ proc create_or_clear_dir {dir} {
   }
 }
 
-proc copy_dir {from to} {
-  foreach f [glob -nocomplain [file join $from *]] {
-    catch { file copy -force $f $to }
-  }
-}
-
 proc build_to_dirname {bname} {
   set fold [string tolower [string map {- _} $bname]]
   return "testrunner_build_$fold"
 }
 
 #-------------------------------------------------------------------------
-# Return a list of tests to run. Each element of the list is itself a
-# list of two elements - the name of a permuations.test configuration
-# followed by the full path to a test script. i.e.:
+
+proc r_write_db {tcl} {
+  trdb eval { BEGIN EXCLUSIVE }
+  uplevel $tcl
+  trdb eval { COMMIT }
+}
+
+# Obtain a new job to be run by worker $iJob (an integer). A job is
+# returned as a three element list:
 #
-#    {BUILD CONFIG FILENAME} {BUILD CONFIG FILENAME} ...
+#    {$build $config $file}
 #
-proc testset_patternlist {patternlist} {
-  global TRG
+proc r_get_next_job {iJob} {
+  global T
 
-  set testset [list]              ;# return value
+  if {($iJob%2)} {
+    set orderby "ORDER BY priority ASC"
+  } else {
+    set orderby "ORDER BY priority DESC"
+  }
 
-  set first [lindex $patternlist 0]
+  set ret [list]
 
-  if {$first=="release"} {
-    set platform $::TRG(platform)
+  r_write_db {
+    set query "
+      SELECT * FROM jobs AS j WHERE state='ready' $orderby LIMIT 1
+    " 
+    trdb eval $query job {
+      set tm [clock_milliseconds]
+      set T($iJob) $tm
+      set jobid $job(jobid)
 
-    set patternlist [lrange $patternlist 1 end]
-    foreach b [trd_builds $platform] {
-      foreach c [trd_configs $platform $b] {
-        testset_append testset $b $c $patternlist
+      set cwd $job(dirname) 
+      if {$cwd==""} {
+        set cwd [dirname $iJob]
       }
 
-      if {[llength $patternlist]==0 || $b=="User-Auth"} {
-        set target testfixture
-      } else {
-        set target coretestprogs
+      trdb eval {
+        UPDATE jobs 
+        SET starttime=$tm, state='running', cwd=$cwd 
+        WHERE jobid=$jobid
       }
-      lappend testset [list $b build $target]
-    }
 
-    if {[llength $patternlist]==0} {
-      foreach b [trd_builds $platform] {
-        foreach e [trd_extras $platform $b] {
-          lappend testset [list $b make $e]
-        }
-      }
+      set ret [array get job]
     }
+  }
+
+  return $ret
+}
+
+# Usage:
+#
+#   add_job OPTION ARG OPTION ARG...
+#
+# where available OPTIONS are:
+#
+#   -displaytype
+#   -displayname
+#   -build
+#   -dirname     
+#   -cmd 
+#   -depid 
+#   -priority 
+#
+# Returns the jobid value for the new job.
+# 
+proc add_job {args} {
+  global estwork
 
-    set TRG(fuzztest) 0           ;# ignore --fuzztest option in this case
+  set options {
+      -displaytype -displayname -build -dirname 
+      -cmd -depid -priority
+  }
 
-  } elseif {$first=="all"} {
+  # Set default values of options.
+  set A(-dirname) ""
+  set A(-depid)   ""
+  set A(-priority) 0
+  set A(-build)   ""
 
-    set clist [trd_all_configs]
-    set patternlist [lrange $patternlist 1 end]
-    foreach c $clist {
-      testset_append testset "" $c $patternlist
-    }
+  array set A $args
+
+  # Check all required options are present. And that no extras are present.
+  foreach o $options {
+    if {[info exists A($o)]==0} { error "missing required option $o" }
+  }
+  foreach o [array names A] {
+    if {[lsearch -exact $options $o]<0} { error "unrecognized option: $o" }
+  }
 
-  } elseif {[info exists ::testspec($first)]} {
-    set clist $first
-    testset_append testset "" $first [lrange $patternlist 1 end]
-  } elseif { [llength $patternlist]==0 } {
-    testset_append testset "" veryquick $patternlist
+  set state ""
+  if {$A(-depid)==""} { set state ready }
+  set type $A(-displaytype)
+  set displayname $A(-displayname)
+  switch $type {
+    tcl {
+      set ek [file tail [lindex $displayname end]]
+    }
+    bld {
+      set ek [lindex $displayname end]
+    }
+    fuzz {
+      set ek [lrange $displayname 1 2]
+    }
+    make {
+      set ek [lindex $displayname end]
+    }
+  }
+  if {[info exists estwork($ek)]} {
+    set ew $estwork($ek)
   } else {
-    testset_append testset "" full $patternlist
+    set ew $estwork($type)
   }
-  if {$TRG(fuzztest)} {
-    if {$TRG(platform)=="win"} { error "todo" }
-    lappend testset [list "" make fuzztest]
+
+  trdb eval {
+    INSERT INTO jobs(
+      displaytype, displayname, build, dirname, cmd, depid, priority,
+      estwork, estkey, state
+    ) VALUES (
+      $type,
+      $A(-displayname),
+      $A(-build),
+      $A(-dirname),
+      $A(-cmd),
+      $A(-depid),
+      $A(-priority),
+      $ew,
+      $ek,
+      $state
+    )
   }
 
-  set testset
+  trdb last_insert_rowid
+}
+
+# Look to see if $jobcmd matches any of the glob patterns given in
+# $patternlist.  Return true if there is a match.  Return false
+# if no match is seen.
+#
+# An empty patternlist matches everything
+#
+proc job_matches_any_pattern {patternlist jobcmd} {
+  set bMatch 0
+  if {[llength $patternlist]==0} {return 1}
+  foreach p $patternlist {
+    set p [string trim $p *]
+    if {[string index $p 0]=="^"} {
+      set p [string range $p 1 end]
+    } else {
+      set p "*$p"
+    }
+    if {[string index $p end]=="\$"} {
+      set p [string range $p 0 end-1]
+    } else {
+      set p "$p*"
+    }
+    if {[string match $p $jobcmd]} {
+      set bMatch 1
+      break
+    }
+  }
+  return $bMatch
 }
+       
 
-proc testset_append {listvar build config patternlist} {
-  upvar $listvar lvar
+# Argument $build is either an empty string, or else a list of length 3 
+# describing the job to build testfixture. In the usual form:
+#
+#    {ID DIRNAME DISPLAYNAME}
+# 
+# e.g    
+#
+#    {1 /home/user/sqlite/test/testrunner_bld_xyz All-Debug}
+# 
+proc add_tcl_jobs {build config patternlist {shelldepid ""}} {
+  global TRG
+  set ntcljob 0
 
-  catch { array unset O }
-  array set O $::testspec($config)
+  set topdir [file dirname $::testdir]
+  set testrunner_tcl [file normalize [info script]]
 
-  foreach f $O(-files) {
-    if {[llength $patternlist]>0} {
-      set bMatch 0
-      foreach p $patternlist {
-        if {[string match $p [file tail $f]]} {
-          set bMatch 1
-          break
-        }
-      }
-      if {$bMatch==0} continue
+  if {$build==""} {
+    set testfixture [info nameofexec]
+  } else {
+    set testfixture [file join [lindex $build 1] $TRG(testfixture)]
+  }
+  if {[lindex $build 2]=="Valgrind"} {
+    set setvar "export OMIT_MISUSE=1\n"
+    set testfixture "${setvar}valgrind -v --error-exitcode=1 $testfixture"
+  }
+
+  # The ::testspec array is populated by permutations.test
+  foreach f [dict get $::testspec($config) -files] {
+
+    if {![job_matches_any_pattern $patternlist "$config [file tail $f]"]} {
+      continue
     }
 
-    if {[file pathtype $f]!="absolute"} {
-      set f [file join $::testdir $f]
+    if {[file pathtype $f]!="absolute"} { set f [file join $::testdir $f] }
+    set f [file normalize $f]
+
+    set displayname [string map [list $topdir/ {}] $f]
+    if {$config=="full" || $config=="veryquick"} {
+      set cmd "$testfixture $f"
+    } else {
+      set cmd "$testfixture $testrunner_tcl $config $f"
+      set displayname "config=$config $displayname"
+    }
+    if {$build!=""} {
+      set displayname "[lindex $build 2] $displayname"
     }
-    lappend lvar [list $build $config $f]
+
+    set lProp [trd_test_script_properties $f]
+    set priority 0
+    if {[lsearch $lProp slow]>=0} { set priority 2 }
+    if {[lsearch $lProp superslow]>=0} { set priority 4 }
+
+    set depid [lindex $build 0]
+    if {$shelldepid!="" && [lsearch $lProp shell]>=0} { set depid $shelldepid }
+
+    incr ntcljob
+    add_job                            \
+        -displaytype tcl               \
+        -displayname $displayname      \
+        -cmd $cmd                      \
+        -depid $depid                  \
+        -priority $priority
+  }
+  if {$ntcljob==0 && [llength $build]>0} {
+    set bldid [lindex $build 0]
+    trdb eval {DELETE FROM jobs WHERE rowid=$bldid}
   }
 }
 
-#--------------------------------------------------------------------------
+proc add_build_job {buildname target {postcmd ""} {depid ""}} {
+  global TRG
 
+  set dirname "[string tolower [string map {- _} $buildname]]_$target"
+  set dirname "testrunner_bld_$dirname"
 
-proc r_write_db {tcl} {
-  trdb eval { BEGIN EXCLUSIVE }
-  uplevel $tcl
-  trdb eval { COMMIT }
+  set cmd "$TRG(makecmd) $target"
+  if {$postcmd!=""} {
+    append cmd "\n"
+    append cmd $postcmd
+  }
+
+  set id [add_job                                \
+    -displaytype bld                             \
+    -displayname "Build $buildname ($target)"    \
+    -dirname $dirname                            \
+    -build $buildname                            \
+    -cmd  $cmd                                   \
+    -depid $depid                                \
+    -priority 3
+  ]
+
+  list $id [file normalize $dirname] $buildname
 }
 
-# Obtain a new job to be run by worker $iJob (an integer). A job is
-# returned as a three element list:
-#
-#    {$build $config $file}
-#
-proc r_get_next_job {iJob} {
-  global T
+proc add_shell_build_job {buildname dirname depid} {
+  global TRG
 
-  if {($iJob%2)} {
-    set orderby "ORDER BY priority ASC"
+  if {$TRG(platform)=="win"} {
+    set path [string map {/ \\} "$dirname/"]
+    set copycmd "xcopy $TRG(shell) $path"
   } else {
-    set orderby "ORDER BY priority DESC"
+    set copycmd "cp $TRG(shell) $dirname/"
   }
 
-  r_write_db {
-    set f ""
-    set c ""
-    trdb eval "
-      SELECT build, config, filename 
-        FROM script 
-        WHERE state='ready' 
-        $orderby LIMIT 1
-    " {
-      set b $build
-      set c $config
-      set f $filename
-    }
-    if {$f!=""} {
-      set tm [clock_milliseconds]
-      set T($iJob) $tm
-      trdb eval { 
-        UPDATE script SET state='running', time=$tm
-        WHERE (build, config, filename) = ($b, $c, $f)
+  return [
+    add_build_job $buildname $TRG(shell) $copycmd $depid
+  ]
+}
+
+
+proc add_make_job {bld target} {
+  global TRG
+
+  if {$TRG(platform)=="win"} {
+    set path [string map {/ \\} [lindex $bld 1]]
+    set cmd "xcopy /S $path\\* ."
+  } else {
+    set cmd "cp -r [lindex $bld 1]/* ."
+  }
+  append cmd "\n$TRG(makecmd) $target"
+
+  add_job                                       \
+    -displaytype make                           \
+    -displayname "[lindex $bld 2] make $target" \
+    -cmd $cmd                                   \
+    -depid [lindex $bld 0]                      \
+    -priority 1
+}
+
+proc add_fuzztest_jobs {buildname patternlist} {
+  global env TRG
+  # puts buildname=$buildname
+
+  foreach {interpreter scripts} [trd_fuzztest_data $buildname] {
+    set bldDone 0
+    set subcmd [lrange $interpreter 1 end]
+    set interpreter [lindex $interpreter 0]
+
+    if {[string match fuzzcheck* $interpreter]
+     && [info exists env(FUZZDB)]
+     && [file readable $env(FUZZDB)]
+     && $buildname ne "Windows-Win32Heap"
+     && $buildname ne "Windows-Memdebug"
+    } {
+      set TRG(FUZZDB) $env(FUZZDB)
+      set fname [file normalize $env(FUZZDB)]
+      set N [expr {([file size $fname]+4999999)/5000000}]
+      for {set i 0} {$i<$N} {incr i} {
+        lappend scripts [list --slice $i $N $fname]
       }
     }
-  }
 
-  if {$f==""} { return "" }
-  list $b $c $f
+    foreach s $scripts {
+
+      # Fuzz data files fuzzdata1.db and fuzzdata2.db are larger than
+      # the others. So ensure that these are run as a higher priority.
+      if {[llength $s]==1} {
+        set tail [file tail $s]
+      } else {
+        set fname [lindex $s end]
+        set tail [lrange $s 0 end-1]
+        lappend tail [file tail $fname]
+      }
+      if {![job_matches_any_pattern $patternlist "$interpreter $tail"]} {
+        continue
+      }
+      if {!$bldDone} {
+        set bld [add_build_job $buildname $interpreter]
+        foreach {depid dirname displayname} $bld {}
+        set bldDone 1
+      }
+      if {[string match ?-slice* $tail]} {
+        set priority 15
+      } elseif {$tail=="fuzzdata1.db"
+          || $tail=="fuzzdata2.db"
+          || $tail=="fuzzdata8.db"} {
+        set priority 5
+      } else {
+        set priority 1
+      }
+      add_job                                                   \
+        -displaytype fuzz                                       \
+        -displayname "$buildname $interpreter $tail"            \
+        -depid $depid                                           \
+        -cmd "[file join $dirname $interpreter] $subcmd $s"     \
+        -priority $priority
+    }
+  }
 }
 
-#rename r_get_next_job r_get_next_job_r
-#proc r_get_next_job {iJob} {
-#  puts [time { set res [r_get_next_job_r $iJob] }]
-#  set res
-#}
+proc add_zipvfs_jobs {} {
+  global TRG
+  source [file join $TRG(zipvfs) test zipvfs_testrunner.tcl]
+
+  set bld [add_build_job Zipvfs $TRG(testfixture)]
+  foreach s [zipvfs_testrunner_files] {
+    set cmd "[file join [lindex $bld 1] $TRG(testfixture)] $s"
+    add_job                                  \
+        -displaytype tcl                     \
+        -displayname "Zipvfs [file tail $s]" \
+        -cmd $cmd                            \
+        -depid [lindex $bld 0]
+  }
 
-proc make_new_testset {} {
+  set ::env(SQLITE_TEST_DIR) $::testdir
+}
+
+# Used to add jobs for "mdevtest" and "sdevtest".
+#
+proc add_devtest_jobs {lBld patternlist} {
   global TRG
 
-  set tests [testset_patternlist $TRG(patternlist)]
+  foreach b $lBld {
+    set bld [add_build_job $b $TRG(testfixture)]
+    add_tcl_jobs $bld veryquick $patternlist SHELL
+    add_fuzztest_jobs $b $patternlist
+
+    if {[trdb one "SELECT EXISTS (SELECT 1 FROM jobs WHERE depid='SHELL')"]} {
+      set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]]
+      set sbldid [lindex $sbld 0]
+      trdb eval {
+        UPDATE jobs SET depid=$sbldid WHERE depid='SHELL'
+      }
+    }
+
+  }
+}
+
+# Check to ensure that the interpreter is a full-blown "testfixture"
+# build and not just a "tclsh".  If this is not the case, issue an
+# error message and exit.
+#
+proc must_be_testfixture {} {
+  if {[lsearch [info commands] sqlite3_soft_heap_limit]<0} {
+    puts "Use testfixture, not tclsh, for these arguments."
+    exit 1
+  }
+}
+
+proc add_jobs_from_cmdline {patternlist} {
+  global TRG
 
   if {$TRG(zipvfs)!=""} {
-    source [file join $TRG(zipvfs) test zipvfs_testrunner.tcl]
-    set tests [concat $tests [zipvfs_testrunner_testset]]
+    add_zipvfs_jobs
+    if {[llength $patternlist]==0} return
   }
 
-  r_write_db {
+  if {[llength $patternlist]==0} {
+    set patternlist [list veryquick]
+  }
 
-    trdb eval $TRG(schema)
-    set nJob $TRG(nJob)
-    set cmdline $TRG(cmdline)
-    set tm [clock_milliseconds]
-    trdb eval { REPLACE INTO config VALUES('njob', $nJob ); }
-    trdb eval { REPLACE INTO config VALUES('cmdline', $cmdline ); }
-    trdb eval { REPLACE INTO config VALUES('start', $tm ); }
+  set first [lindex $patternlist 0]
+  switch -- $first {
+    all {
+      must_be_testfixture
+      set patternlist [lrange $patternlist 1 end]
+      set clist [trd_all_configs]
+      foreach c $clist {
+        add_tcl_jobs "" $c $patternlist
+      }
+    }
 
-    foreach t $tests {
-      foreach {b c s} $t {}
-      set slow 0
-
-      if {$c!="make" && $c!="build"} {
-        set fd [open $s]
-        for {set ii 0} {$ii<100 && ![eof $fd]} {incr ii} {
-          set line [gets $fd]
-          if {[string match -nocase *testrunner:* $line]} {
-            regexp -nocase {.*testrunner:(.*)} $line -> properties
-            foreach p $properties {
-              if {$p=="slow"} { set slow 1 }
-              if {$p=="superslow"} { set slow 2 }
-            }
-          }
-        }
-        close $fd
+    devtest -
+    mdevtest {
+      set config_set {
+        All-O0
+        All-Debug
       }
+      add_devtest_jobs $config_set [lrange $patternlist 1 end]
+    }
 
-      if {$c=="make" && $b==""} {
-        # --fuzztest option
-        set slow 1
+    sdevtest {
+      set config_set {
+        All-Sanitize
+        All-Debug
       }
+      add_devtest_jobs $config_set [lrange $patternlist 1 end]
+    }
+
+    release {
+      set patternlist [lrange $patternlist 1 end]
+      foreach b [trd_builds $TRG(platform)] {
+        if {$TRG(config)!="" && ![regexp "\\y$b\\y" $TRG(config)]} continue
+        if {[regexp "\\y$b\\y" $TRG(omitconfig)]} continue
+        set bld [add_build_job $b $TRG(testfixture)]
+        foreach c [trd_configs $TRG(platform) $b] {
+          add_tcl_jobs $bld $c $patternlist SHELL
+        }
+
+        foreach e [trd_extras $TRG(platform) $b] {
+          if {$e=="fuzztest"} {
+            add_fuzztest_jobs $b $patternlist
+          } elseif {[job_matches_any_pattern $patternlist $e]} {
+            add_make_job $bld $e
+          }
+        }
 
-      if {$c=="veryquick"} {
-        set c ""
+        if {[trdb one "SELECT EXISTS(SELECT 1
+                                       FROM jobs WHERE depid='SHELL')"]} {
+          set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]]
+          set sbldid [lindex $sbld 0]
+          trdb eval {
+            UPDATE jobs SET depid=$sbldid WHERE depid='SHELL'
+          }
+        }
       }
+    }
 
-      set state ready
-      if {$b!="" && $c!="build"} {
-        set state ""
+    list {
+      set allperm [array names ::testspec]
+      lappend allperm all mdevtest sdevtest release list
+      puts "Allowed values for the PERMUTATION argument: [lsort $allperm]"
+      exit 0
+    }
+
+    default {
+      must_be_testfixture
+      if {[info exists ::testspec($first)]} {
+        add_tcl_jobs "" $first [lrange $patternlist 1 end]
+      } else {
+        add_tcl_jobs "" full $patternlist
       }
+    }
+  }
 
-      trdb eval { 
-        INSERT INTO script(build, config, filename, slow, state) 
-            VALUES ($b, $c, $s, $slow, $state) 
+  # If the "--case DISPLAYNAME" option appears on the command-line, mark
+  # all tests other than DISPLAYNAME as 'omit'.
+  #
+  if {[info exists TRG(case)] && $TRG(case) ne ""} {
+    set jid [trdb one {
+      SELECT jobid FROM jobs WHERE displayname GLOB $TRG(case)
+    }]
+    if {$jid eq ""} {
+      puts "ERROR: No jobs match \"$TRG(case)\"."
+      puts "The argument to --cases must GLOB match the jobs.displayname column"
+      puts "of the testrunner.db database."
+      trdb eval {UPDATE jobs SET state='omit'}
+    } else {
+      trdb eval {
+        WITH RECURSIVE keepers(jid,did) AS (
+           SELECT jobid,depid FROM jobs
+            WHERE displayname GLOB $TRG(case)
+           UNION
+           SELECT jobid,depid FROM jobs, keepers WHERE jobid=did
+        )
+        DELETE FROM jobs WHERE jobid NOT IN (SELECT jid FROM keepers);
       }
     }
   }
 }
 
-proc script_input_ready {fd iJob b c f} {
+proc make_new_testset {} {
+  global TRG
+
+  trdb eval {PRAGMA journal_mode=WAL;}
+  r_write_db {
+    trdb eval $TRG(schema)
+    set nJob $TRG(nJob)
+    set cmdline $TRG(cmdline)
+    set tm [clock_milliseconds]
+    trdb eval { REPLACE INTO config VALUES('njob', $nJob ); }
+    trdb eval { REPLACE INTO config VALUES('cmdline', $cmdline ); }
+    trdb eval { REPLACE INTO config VALUES('start', $tm ); }
+
+    add_jobs_from_cmdline $TRG(patternlist)
+
+  }
+}
+
+proc mark_job_as_finished {jobid output state endtm} {
+  set ntest 1
+  set nerr 0
+  if {$endtm>0} {
+    set re {\y(\d+) errors out of (\d+) tests( on [^\n]+\n)?}
+    if {[regexp $re $output all a b pltfm]} {
+      set nerr $a
+      set ntest $b
+    }
+    regexp {\ySQLite \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d [0-9a-fA-F]+} \
+         $output svers
+  }
+  r_write_db {
+    if {$state=="failed"} {
+      set childstate omit
+      if {$nerr<=0} {set nerr 1}
+    } else {
+      set childstate ready
+    }
+    if {[info exists pltfm]} {set pltfm [string trim $pltfm]}
+    trdb eval {
+      UPDATE jobs 
+        SET output=$output, state=$state, endtime=$endtm, span=$endtm-starttime,
+            ntest=$ntest, nerr=$nerr, svers=$svers, pltfm=$pltfm
+        WHERE jobid=$jobid;
+      UPDATE jobs SET state=$childstate WHERE depid=$jobid AND state!='halt';
+      UPDATE config SET value=value+$nerr WHERE name='nfail';
+      UPDATE config SET value=value+$ntest WHERE name='ntest';
+    }
+  }
+}
+
+proc script_input_ready {fd iJob jobid} {
   global TRG
   global O
   global T
 
   if {[eof $fd]} {
+    trdb eval { SELECT * FROM jobs WHERE jobid=$jobid } job {}
+
+    # If this job specified a directory name, then delete the run.sh/run.bat
+    # file from it before continuing. This is because the contents of this
+    # directory might be copied by some other job, and we don't want to copy
+    # the run.sh file in this case.
+    if {$job(dirname)!=""} {
+      file delete -force [file join $job(dirname) $TRG(run)]
+    }
+
     set ::done 1
     fconfigure $fd -blocking 1
     set state "done"
     set rc [catch { close $fd } msg]
     if {$rc} { 
-      puts "FAILED: $b $c $f"
+      puts [format %-79.79s "FAILED: $job(displayname) ($iJob)"]
       set state "failed" 
+      if {$TRG(stopOnError)} {
+        puts "OUTPUT: $O($iJob)"
+        exit 1
+      }
+      if {$TRG(stopOnCore) && [string first {core dumped} $O($iJob)]>0} {
+        puts "OUTPUT: $O($iJob)"
+        exit 1
+      }
     }
 
-    set tm [expr [clock_milliseconds] - $T($iJob)]
+    set tm [clock_milliseconds]
+    set jobtm [expr {$tm - $job(starttime)}]
 
-    puts $TRG(log) "### $b ### $c ### $f ${tm}ms ($state)"
+    puts $TRG(log) "### $job(displayname) ${jobtm}ms ($state)"
     puts $TRG(log) [string trim $O($iJob)]
 
-    r_write_db {
-      set output $O($iJob)
-      trdb eval {
-        UPDATE script SET output = $output, state=$state, time=$tm
-        WHERE (build, config, filename) = ($b, $c, $f)
-      }
-      if {$state=="done" && $c=="build"} {
-        trdb eval {
-          UPDATE script SET state = 'ready' WHERE (build, state)==($b, '')
-        }
-      }
-    }
+    mark_job_as_finished $jobid $O($iJob) $state $tm
 
     dirs_freeDir $iJob
     launch_some_jobs
@@ -702,141 +1600,129 @@ proc launch_another_job {iJob} {
   set testfixture [info nameofexec]
   set script $TRG(info_script)
 
-  set dir [dirname $iJob]
-  create_or_clear_dir $dir
-
   set O($iJob) ""
   
-  set job [r_get_next_job $iJob]
-  if {$job==""} { return 0 }
-
-  foreach {b c f} $job {}
+  set jobdict [r_get_next_job $iJob]
+  if {$jobdict==""} { return 0 }
+  array set job $jobdict
 
-  if {$c=="build"} {
-    set testdir [file dirname $TRG(info_script)]
-    set srcdir [file dirname $testdir]
-    set builddir [build_to_dirname $b]
-    create_or_clear_dir $builddir
+  set dir $job(dirname)
+  if {$dir==""} { set dir [dirname $iJob] }
+  create_or_clear_dir $dir
 
-    if {$b=="Zipvfs"} {
+  if {$job(build)!=""} {
+    set srcdir [file dirname $::testdir]
+    if {$job(build)=="Zipvfs"} {
       set script [zipvfs_testrunner_script]
     } else {
-      set     cmd [info nameofexec]
-      lappend cmd [file join $testdir releasetest_data.tcl]
-      lappend cmd trscript
-      if {$TRG(platform)=="win"} { lappend cmd -msvc }
-      lappend cmd $b $srcdir
-      set script [exec {*}$cmd]
+      set bWin [expr {$TRG(platform)=="win"}]
+      set script [trd_buildscript $job(build) $srcdir $bWin]
     }
-
-    set fd [open [file join $builddir $TRG(make)] w]
+    set fd [open [file join $dir $TRG(make)] w]
     puts $fd $script
     close $fd
+  }
 
-    puts "Launching build \"$b\" in directory $builddir..."
-    set target coretestprogs
-    if {$b=="User-Auth"}  { set target testfixture }
+  # Add a batch/shell file command to set the directory used for temp
+  # files to the test's working directory. Otherwise, tests that use
+  # large numbers of temp files (e.g. zipvfs), might generate temp 
+  # filename collisions.
+  if {$TRG(platform)=="win"} {
+    set set_tmp_dir "SET SQLITE_TMPDIR=[file normalize $dir]"
+  } else {
+    set set_tmp_dir "export SQLITE_TMPDIR=\"[file normalize $dir]\""
+  }
 
-    set cmd "$TRG(makecmd) $target"
-    set dir $builddir
+  if { $TRG(dryrun) } {
 
-  } elseif {$c=="make"} {
-    if {$b==""} {
-      if {$f!="fuzztest"} { error "corruption in testrunner.db!" }
-      # Special case - run [make fuzztest] 
-      set makedir [file dirname $testfixture]
-      if {$TRG(platform)=="win"} {
-        error "how?"
-      } else {
-        set cmd [list make -C $makedir fuzztest]
-      }
+    mark_job_as_finished $job(jobid) "" done 0
+    dirs_freeDir $iJob
+    if {$job(build)!=""} {
+      puts $TRG(log) "(cd $dir ; $job(cmd) )"
     } else {
-      set builddir [build_to_dirname $b]
-      copy_dir $builddir $dir
-      set cmd "$TRG(makecmd) $f"
+      puts $TRG(log) "$job(cmd)"
     }
+
   } else {
-    if {$b==""} {
-      set testfixture [info nameofexec]
-    } else {
-      set tail testfixture
-      if {$TRG(platform)=="win"} { set tail testfixture.exe }
-      set testfixture [file normalize [file join [build_to_dirname $b] $tail]]
-    }
+    set pwd [pwd]
+    cd $dir
+    set fd [open $TRG(run) w]
+    puts $fd $set_tmp_dir
+    puts $fd $job(cmd)
+    close $fd
+    set fd [open "|$TRG(runcmd) 2>@1" r]
+    cd $pwd
 
-    if {$c=="valgrind"} {
-      set testfixture "valgrind -v --error-exitcode=1 $testfixture"
-      set ::env(OMIT_MISUSE) 1
-    }
-    set cmd [concat $testfixture [list $script $c $f]]
+    fconfigure $fd -blocking false -translation binary
+    fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)]
   }
 
-  set pwd [pwd]
-  cd $dir
-  set fd [open "|$cmd 2>@1" r]
-  cd $pwd
-  set pid [pid $fd]
-
-  fconfigure $fd -blocking false
-  fileevent $fd readable [list script_input_ready $fd $iJob $b $c $f]
-  unset -nocomplain ::env(OMIT_MISUSE)
-
   return 1
 }
 
-proc one_line_report {} {
+# Show the testing progress report
+#
+proc progress_report {} {
   global TRG
 
-  set tm [expr [clock_milliseconds] - $TRG(starttime)]
-  set tm [format "%d" [expr int($tm/1000.0 + 0.5)]]
-
-  foreach s {ready running done failed} {
-    set v($s,build) 0
-    set v($s,make) 0
-    set v($s,script) 0
-  }
-
-  r_write_db {
-    trdb eval {
-      SELECT state, jobtype, count(*) AS cnt 
-      FROM script 
-      GROUP BY state, jobtype
-    } {
-      set v($state,$jobtype) $cnt
-      if {[info exists t($jobtype)]} {
-        incr t($jobtype) $cnt
-      } else {
-        set t($jobtype) $cnt
+  if {$TRG(fullstatus)} {
+    if {$::tcl_platform(platform) eq "windows"} {
+      exec [info nameofexe] $::argv0 status --cls
+    } else {
+      show_status trdb 1
+    }
+  } else {
+    set tmms [expr [clock_milliseconds] - $TRG(starttime)]
+    set tm [format "%d" [expr int($tmms/1000.0 + 0.5)]]
+  
+    set wtotal 0
+    set wdone 0
+    r_write_db {
+      trdb eval { 
+        SELECT displaytype, state, count(*) AS cnt, sum(estwork) AS ew
+        FROM jobs 
+        GROUP BY 1, 2 
+      } {
+        set v($state,$displaytype) $cnt
+        incr t($displaytype) $cnt
+        incr wtotal $ew
+        if {$state=="done" || $state=="failed" || $state=="omit"} {
+          incr wdone $ew
+        }
       }
     }
-  }
-
-  set text ""
-  foreach j [array names t] {
-    set fin [expr $v(done,$j) + $v(failed,$j)]
-    lappend text "$j ($fin/$t($j)) f=$v(failed,$j) r=$v(running,$j)"
-  }
-
-  if {[info exists TRG(reportlength)]} {
-    puts -nonewline "[string repeat " " $TRG(reportlength)]\r"
-  }
-  set report "${tm}s: [join $text { }]"
-  set TRG(reportlength) [string length $report]
-  if {[string length $report]<80} {
-    puts -nonewline "$report\r"
+  
+    set text ""
+    foreach j [lsort [array names t]] {
+      foreach k {done failed running} { incr v($k,$j) 0 }
+      set fin [expr $v(done,$j) + $v(failed,$j)]
+      lappend text "${j}($fin/$t($j))"
+      if {$v(failed,$j)>0} {
+        lappend text "f$v(failed,$j)"
+      }
+      if {$v(running,$j)>0} {
+        lappend text "r$v(running,$j)"
+      }
+    }
+    set report "[elapsetime $tmms] [join $text { }]"
+    if {$wdone>0 && [set pct [expr {int(($wdone*100.0)/$wtotal)}]]>=4} {
+      set tmleft [expr {($tmms/double($wdone))*($wtotal-$wdone)}]
+      set etc " ETC [elapsetime $tmleft]"  
+      if {[string length $report]+[string length $etc]<80} {
+        append report $etc
+      }
+      # append report " $pct%"
+    }
+    puts -nonewline [format %-79.79s $report]\r
     flush stdout
-  } else {
-    puts $report
   }
-
-  after $TRG(reporttime) one_line_report
+  after $TRG(reporttime) progress_report
 }
 
 proc launch_some_jobs {} {
   global TRG
-  r_write_db {
-    set nJob [trdb one { SELECT value FROM config WHERE name='njob' }]
-  }
+  set nJob [trdb one { SELECT value FROM config WHERE name='njob' }]
+
   while {[dirs_nHelper]<$nJob} {
     set iDir [dirs_allocDir]
     if {0==[launch_another_job $iDir]} {
@@ -854,40 +1740,111 @@ proc run_testset {} {
   set TRG(log) [open $TRG(logname) w]
 
   launch_some_jobs
-    # launch_another_job $ii
 
-  one_line_report
+  if {$TRG(fullstatus)} {puts "\033\[2J"}
+  progress_report
   while {[dirs_nHelper]>0} {
     after 500 {incr ::wakeup}
     vwait ::wakeup
   }
   close $TRG(log)
-  one_line_report
+  progress_report
+  puts ""
 
   r_write_db {
-    set nErr [trdb one {SELECT count(*) FROM script WHERE state='failed'}]
+    set tm [clock_milliseconds]
+    trdb eval { REPLACE INTO config VALUES('end', $tm ); }
+    set nErr [trdb one {SELECT count(*) FROM jobs WHERE state='failed'}]
     if {$nErr>0} {
       puts "$nErr failures:"
       trdb eval {
-        SELECT build, config, filename FROM script WHERE state='failed'
+        SELECT displayname FROM jobs WHERE state='failed'
       } {
-        puts "FAILED: $build $config $filename"
+        puts "FAILED: $displayname"
       }
     }
+    set nOmit [trdb one {SELECT count(*) FROM jobs WHERE state='omit'}]
+    if {$nOmit>0} {
+      puts "$nOmit jobs skipped due to prior failures"
+    }
   }
 
-  puts "\nTest database is $TRG(dbname)"
+  puts "Test database is $TRG(dbname)"
   puts "Test log is $TRG(logname)"
+  if {[info exists TRG(FUZZDB)]} {
+    puts "Extra fuzzcheck data taken from $TRG(FUZZDB)"
+  }
+  trdb eval {
+     SELECT sum(ntest) AS totaltest,
+            sum(nerr) AS totalerr
+       FROM jobs
+  } break
+  trdb eval {
+     SELECT max(endtime)-min(starttime) AS totaltime
+       FROM jobs WHERE endtime>0
+  } break;
+  set et [elapsetime $totaltime]
+  set pltfm {}
+  trdb eval {
+     SELECT pltfm, count(*) FROM jobs WHERE pltfm IS NOT NULL
+      ORDER BY 2 DESC LIMIT 1
+  } break
+  if {$totalerr==""} {set totalerr 0}
+  if {$totaltest==""} {set totaltest 0}
+  puts "$totalerr errors out of $totaltest tests in $et $pltfm"
+  trdb eval {
+     SELECT DISTINCT substr(svers,1,79) as v1 FROM jobs WHERE svers IS NOT NULL
+  } {puts $v1}
+
+}
+
+# Handle the --buildonly option, if it was specified.
+#
+proc handle_buildonly {} {
+  global TRG
+  if {$TRG(buildonly)} {
+    r_write_db {
+      trdb eval { DELETE FROM jobs WHERE displaytype!='bld' }
+    }
+  }
 }
 
+# Handle the --explain option.  Provide a human-readable
+# explanation of all the tests that are in the trdb database jobs
+# table.
+#
+proc explain_layer {indent depid} {
+  global TRG
+  if {$TRG(buildonly)} {
+    set showtests 0
+  } else {
+    set showtests 1
+  }
+  trdb eval {SELECT jobid, displayname, displaytype, dirname
+               FROM jobs WHERE depid=$depid ORDER BY displayname} {
+    if {$displaytype=="bld"} {
+      puts "${indent}$displayname in $dirname"
+      explain_layer "${indent}   " $jobid
+    } elseif {$showtests} {
+      puts "${indent}$displayname"
+    }
+  }
+}
+proc explain_tests {} {
+  explain_layer "" ""
+}
 
 sqlite3 trdb $TRG(dbname)
 trdb timeout $TRG(timeout)
 set tm [lindex [time { make_new_testset }] 0]
-if {$TRG(nJob)>1} {
-  puts "splitting work across $TRG(nJob) cores"
+if {$TRG(explain)} {
+  explain_tests
+} else {
+  if {$TRG(nJob)>1} {
+    puts "splitting work across $TRG(nJob) cores"
+  }
+  puts "built testset in [expr $tm/1000]ms.."
+  handle_buildonly
+  run_testset
 }
-puts "built testset in [expr $tm/1000]ms.."
-run_testset
 trdb close
-#puts [pwd]
diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl
index f74ee146ca..e74caee1d6 100644
--- a/test/testrunner_data.tcl
+++ b/test/testrunner_data.tcl
@@ -5,6 +5,7 @@ namespace eval trd {
   variable tcltest
   variable extra
   variable all_configs
+  variable build
 
 
   # Tcl tests to run for various builds.
@@ -15,13 +16,13 @@ namespace eval trd {
   set tcltest(linux.Have-Not)             veryquick
   set tcltest(linux.Secure-Delete)        veryquick
   set tcltest(linux.Unlock-Notify)        veryquick
-  set tcltest(linux.User-Auth)            veryquick
   set tcltest(linux.Update-Delete-Limit)  veryquick
   set tcltest(linux.Extra-Robustness)     veryquick
   set tcltest(linux.Device-Two)           veryquick
   set tcltest(linux.No-lookaside)         veryquick
   set tcltest(linux.Devkit)               veryquick
   set tcltest(linux.Apple)                veryquick
+  set tcltest(linux.Android)              veryquick
   set tcltest(linux.Sanitize)             veryquick
   set tcltest(linux.Device-One)           all
   set tcltest(linux.Default)              all_plus_autovacuum_crash
@@ -29,13 +30,15 @@ namespace eval trd {
 
   set tcltest(osx.Locking-Style)          veryquick
   set tcltest(osx.Have-Not)               veryquick
-  set tcltest(osx.Apple)                  all
+  set tcltest(osx.Apple)                  all_less_no_mutex_try
 
   set tcltest(win.Stdcall)                veryquick
   set tcltest(win.Have-Not)               veryquick
   set tcltest(win.Windows-Memdebug)       veryquick
   set tcltest(win.Windows-Win32Heap)      veryquick
-  set tcltest(win.Default)                full
+  set tcltest(win.Windows-Sanitize)       veryquick
+  set tcltest(win.Windows-WinRT)          veryquick
+  set tcltest(win.Default)                {full win_unc_locking}
 
   # Extra [make xyz] tests that should be run for various builds.
   #
@@ -51,6 +54,7 @@ namespace eval trd {
   set extra(linux.Device-Two)             {fuzztest sourcetest threadtest}
   set extra(linux.No-lookaside)           {fuzztest sourcetest}
   set extra(linux.Devkit)                 {fuzztest sourcetest}
+  set extra(linux.Android)                {fuzztest sourcetest}
   set extra(linux.Apple)                  {fuzztest sourcetest}
   set extra(linux.Sanitize)               {fuzztest sourcetest}
   set extra(linux.Default)                {fuzztest sourcetest threadtest}
@@ -63,18 +67,309 @@ namespace eval trd {
   set extra(win.Stdcall)                  {fuzztest sourcetest}
   set extra(win.Windows-Memdebug)         {fuzztest sourcetest}
   set extra(win.Windows-Win32Heap)        {fuzztest sourcetest}
+  set extra(win.Windows-Sanitize)         fuzztest
   set extra(win.Have-Not)                 {fuzztest sourcetest}
 
   # The following mirrors the set of test suites invoked by "all.test".
   #
   set all_configs {
-    full no_optimization memsubsys1 memsubsys2 singlethread 
-    multithread onefile utf16 exclusive persistent_journal 
+    full no_optimization memsubsys1 memsubsys2 singlethread
+    multithread onefile utf16 exclusive persistent_journal
     persistent_journal_error no_journal no_journal_error
-    autovacuum_ioerr no_mutex_try fullmutex journaltest 
-    inmemory_journal pcache0 pcache10 pcache50 pcache90 
+    autovacuum_ioerr no_mutex_try fullmutex journaltest
+    inmemory_journal pcache0 pcache10 pcache50 pcache90
     pcache100 prepare mmap
   }
+
+  #-----------------------------------------------------------------------
+  # Start of build() definitions.
+  #
+  set build(Default) {
+    -O2
+    --disable-amalgamation --disable-shared
+    --enable-session
+    -DSQLITE_ENABLE_RBU
+    -DSQLITE_ENABLE_STMT_SCANSTATUS
+  }
+
+  # These two are used by [testrunner.tcl mdevtest] (All-O0) and
+  # [testrunner.tcl sdevtest] (All-Sanitize).
+  #
+  set build(All-Debug) {
+    --with-debug --enable-all
+    -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES
+    -DSQLITE_ENABLE_NORMALIZE
+  }
+  set build(All-O0) {
+    -O0 --enable-all
+  }
+  set build(All-Sanitize) {
+    -DSQLITE_OMIT_LOOKASIDE=1
+    --enable-all -fsanitize=address,undefined -fno-sanitize-recover=undefined
+  }
+
+  set build(Sanitize) {
+    CC=clang -fsanitize=address,undefined -fno-sanitize-recover=undefined
+    -DSQLITE_ENABLE_STAT4
+    -DSQLITE_OMIT_LOOKASIDE=1
+    -DSQLITE_ENABLE_NORMALIZE
+    -DCONFIG_SLOWDOWN_FACTOR=5.0
+    -DSQLITE_ENABLE_RBU
+    --with-debug
+    --enable-all
+  }
+  set build(Stdcall) {
+    -DWITHOUT_JIMSH=1
+    -DUSE_STDCALL=1
+    -DSQLITE_USE_ONLY_WIN32=1
+    -O2
+  }
+
+  # The "Have-Not" configuration sets all possible -UHAVE_feature options
+  # in order to verify that the code works even on platforms that lack
+  # these support services.
+  set build(Have-Not) {
+    -DHAVE_FDATASYNC=0
+    -DHAVE_GMTIME_R=0
+    -DHAVE_ISNAN=0
+    -DHAVE_LOCALTIME_R=0
+    -DHAVE_LOCALTIME_S=0
+    -DHAVE_MALLOC_USABLE_SIZE=0
+    -DHAVE_STRCHRNUL=0
+    -DHAVE_USLEEP=0
+    -DHAVE_UTIME=0
+  }
+  set build(Unlock-Notify) {
+    -O2
+    -DSQLITE_ENABLE_UNLOCK_NOTIFY
+    -DSQLITE_THREADSAFE
+    -DSQLITE_TCL_DEFAULT_FULLMUTEX=1
+  }
+  set build(Secure-Delete) {
+    -O2
+    -DSQLITE_SECURE_DELETE=1
+    -DSQLITE_SOUNDEX=1
+  }
+  set build(Update-Delete-Limit) {
+    -O2
+    -DSQLITE_DEFAULT_FILE_FORMAT=4
+    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
+    -DSQLITE_ENABLE_STMT_SCANSTATUS
+    -DSQLITE_LIKE_DOESNT_MATCH_BLOBS
+    -DSQLITE_ENABLE_CURSOR_HINTS
+  }
+  set build(Check-Symbols) {
+    -DSQLITE_MEMDEBUG=1
+    -DSQLITE_ENABLE_FTS3_PARENTHESIS=1
+    -DSQLITE_ENABLE_FTS3=1
+    -DSQLITE_ENABLE_RTREE=1
+    -DSQLITE_ENABLE_MEMSYS5=1
+    -DSQLITE_ENABLE_MEMSYS3=1
+    -DSQLITE_ENABLE_COLUMN_METADATA=1
+    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
+    -DSQLITE_SECURE_DELETE=1
+    -DSQLITE_SOUNDEX=1
+    -DSQLITE_ENABLE_ATOMIC_WRITE=1
+    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
+    -DSQLITE_ENABLE_NORMALIZE
+    -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
+    -DSQLITE_ENABLE_STAT4
+    -DSQLITE_ENABLE_STMT_SCANSTATUS
+    --enable-fts5 --enable-session
+  }
+  set build(Debug-One) {
+    --disable-shared
+    -O2 -funsigned-char
+    -DSQLITE_DEBUG=1
+    -DSQLITE_MEMDEBUG=1
+    -DSQLITE_MUTEX_NOOP=1
+    -DSQLITE_TCL_DEFAULT_FULLMUTEX=1
+    -DSQLITE_ENABLE_FTS3=1
+    -DSQLITE_ENABLE_RTREE=1
+    -DSQLITE_ENABLE_MEMSYS5=1
+    -DSQLITE_ENABLE_NORMALIZE
+    -DSQLITE_ENABLE_COLUMN_METADATA=1
+    -DSQLITE_ENABLE_STAT4
+    -DSQLITE_ENABLE_HIDDEN_COLUMNS
+    -DSQLITE_MAX_ATTACHED=125
+    -DSQLITE_MUTATION_TEST
+    --enable-fts5
+  }
+  set build(Debug-Two) {
+    -DSQLITE_DEFAULT_MEMSTATUS=0
+    -DSQLITE_MAX_EXPR_DEPTH=0
+    --with-debug
+  }
+  set build(Fast-One) {
+    -O6
+    -DSQLITE_ENABLE_FTS4=1
+    -DSQLITE_ENABLE_RTREE=1
+    -DSQLITE_ENABLE_STAT4
+    -DSQLITE_ENABLE_RBU
+    -DSQLITE_MAX_ATTACHED=125
+    -DSQLITE_MAX_MMAP_SIZE=12884901888
+    -DSQLITE_ENABLE_SORTER_MMAP=1
+    --enable-session
+  }
+  set build(Device-One) {
+    -O2
+    -DSQLITE_DEBUG=1
+    -DSQLITE_DEFAULT_AUTOVACUUM=1
+    -DSQLITE_DEFAULT_CACHE_SIZE=64
+    -DSQLITE_DEFAULT_PAGE_SIZE=1024
+    -DSQLITE_DEFAULT_TEMP_CACHE_SIZE=32
+    -DSQLITE_DISABLE_LFS=1
+    -DSQLITE_ENABLE_ATOMIC_WRITE=1
+    -DSQLITE_ENABLE_IOTRACE=1
+    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
+    -DSQLITE_MAX_PAGE_SIZE=4096
+    -DSQLITE_OMIT_LOAD_EXTENSION=1
+    -DSQLITE_OMIT_PROGRESS_CALLBACK=1
+    -DSQLITE_OMIT_VIRTUALTABLE=1
+    -DSQLITE_ENABLE_HIDDEN_COLUMNS
+    -DSQLITE_TEMP_STORE=3
+  }
+  set build(Device-Two) {
+    -DSQLITE_4_BYTE_ALIGNED_MALLOC=1
+    -DSQLITE_DEFAULT_AUTOVACUUM=1
+    -DSQLITE_DEFAULT_CACHE_SIZE=1000
+    -DSQLITE_DEFAULT_LOCKING_MODE=0
+    -DSQLITE_DEFAULT_PAGE_SIZE=1024
+    -DSQLITE_DEFAULT_TEMP_CACHE_SIZE=1000
+    -DSQLITE_DISABLE_LFS=1
+    -DSQLITE_ENABLE_FTS3=1
+    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
+    -DSQLITE_ENABLE_RTREE=1
+    -DSQLITE_MAX_COMPOUND_SELECT=50
+    -DSQLITE_MAX_PAGE_SIZE=32768
+    -DSQLITE_OMIT_TRACE=1
+    -DSQLITE_TEMP_STORE=3
+    -DSQLITE_THREADSAFE=2
+    --enable-fts5 --enable-session
+  }
+  set build(Locking-Style) {
+    -O2
+    -DSQLITE_ENABLE_LOCKING_STYLE=1
+  }
+  set build(Android) {
+    -Os
+    -DHAVE_USLEEP=1
+    -DSQLITE_HAVE_ISNAN
+    -DSQLITE_POWERSAFE_OVERWRITE=1
+    -DSQLITE_DEFAULT_FILE_FORMAT=4
+    -DSQLITE_DEFAULT_AUTOVACUUM=1
+    -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1
+    -DSQLITE_ENABLE_FTS3
+    -DSQLITE_ENABLE_FTS3_BACKWARDS
+    -DSQLITE_ENABLE_FTS4
+    -DSQLITE_SECURE_DELETE
+    -DSQLITE_ENABLE_BATCH_ATOMIC_WRITE
+    -DBIONIC_IOCTL_NO_SIGNEDNESS_OVERLOAD
+    -DSQLITE_ALLOW_ROWID_IN_VIEW
+    -DSQLITE_ENABLE_BYTECODE_VTAB
+    -Wno-unused-parameter
+    -Werror
+    -DUSE_PREAD64
+    -Dfdatasync=fdatasync
+    -DHAVE_MALLOC_H=1
+    -DHAVE_MALLOC_USABLE_SIZE
+    -DSQLITE_ENABLE_DBSTAT_VTAB
+  }
+  # Compile-options used by Android but omitted from these
+  # tests:
+  #   -DNDEBUG=1
+  #   -DSQLITE_DEFAULT_LEGACY_ALTER_TABLE
+  #   -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576
+  #   -DSQLITE_DEFAULT_FILE_PERMISSIONS=0600
+  #   -DSQLITE_OMIT_BUILTIN_TEST
+  #   -DSQLITE_OMIT_LOAD_EXTENSION
+  #   -DSQLITE_OMIT_COMPILEOPTION_DIAGS
+  #
+  set build(Apple) {
+    -Os
+    -DHAVE_GMTIME_R=1
+    -DHAVE_ISNAN=1
+    -DHAVE_LOCALTIME_R=1
+    -DHAVE_PREAD=1
+    -DHAVE_PWRITE=1
+    -DHAVE_UTIME=1
+    -DSQLITE_DEFAULT_CACHE_SIZE=1000
+    -DSQLITE_DEFAULT_CKPTFULLFSYNC=1
+    -DSQLITE_DEFAULT_MEMSTATUS=1
+    -DSQLITE_DEFAULT_PAGE_SIZE=1024
+    -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS=1
+    -DSQLITE_ENABLE_API_ARMOR=1
+    -DSQLITE_ENABLE_AUTO_PROFILE=1
+    -DSQLITE_ENABLE_FLOCKTIMEOUT=1
+    -DSQLITE_ENABLE_FTS3=1
+    -DSQLITE_ENABLE_FTS3_PARENTHESIS=1
+    -DSQLITE_ENABLE_FTS3_TOKENIZER=1
+    -DSQLITE_ENABLE_NORMALIZE=1
+    -DSQLITE_ENABLE_PERSIST_WAL=1
+    -DSQLITE_ENABLE_PURGEABLE_PCACHE=1
+    -DSQLITE_ENABLE_RTREE=1
+    -DSQLITE_ENABLE_SETLK_TIMEOUT=2
+    -DSQLITE_ENABLE_SNAPSHOT=1
+    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
+    -DSQLITE_MAX_LENGTH=2147483645
+    -DSQLITE_MAX_VARIABLE_NUMBER=500000
+    -DSQLITE_NO_SYNC=1
+    -DSQLITE_OMIT_AUTORESET=1
+    -DSQLITE_OMIT_LOAD_EXTENSION=1
+    -DSQLITE_PREFER_PROXY_LOCKING=1
+    -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
+    -DSQLITE_THREADSAFE=2
+    -DSQLITE_USE_URI=1
+    -DSQLITE_WRITE_WALFRAME_PREBUFFERED=1
+    -DUSE_GUARDED_FD=1
+    -DUSE_PREAD=1
+    --enable-fts5
+  }
+  set build(Extra-Robustness) {
+    -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
+    -DSQLITE_MAX_ATTACHED=62
+  }
+  set build(Devkit) {
+    -DSQLITE_DEFAULT_FILE_FORMAT=4
+    -DSQLITE_MAX_ATTACHED=30
+    -DSQLITE_ENABLE_COLUMN_METADATA
+    -DSQLITE_ENABLE_FTS4
+    -DSQLITE_ENABLE_FTS5
+    -DSQLITE_ENABLE_FTS4_PARENTHESIS
+    -DSQLITE_DISABLE_FTS4_DEFERRED
+    -DSQLITE_ENABLE_RTREE
+    --enable-fts5
+  }
+  set build(No-lookaside) {
+    -DSQLITE_TEST_REALLOC_STRESS=1
+    -DSQLITE_OMIT_LOOKASIDE=1
+  }
+  set build(Valgrind) {
+    -DSQLITE_ENABLE_STAT4
+    -DSQLITE_ENABLE_FTS4
+    -DSQLITE_ENABLE_RTREE
+    -DSQLITE_ENABLE_HIDDEN_COLUMNS
+    -DCONFIG_SLOWDOWN_FACTOR=8.0
+  }
+
+  set build(Windows-Memdebug) {
+    MEMDEBUG=1
+    DEBUG=3
+  }
+  set build(Windows-Win32Heap) {
+    WIN32HEAP=1
+    DEBUG=4
+    ENABLE_SETLK=1
+  }
+  set build(Windows-Sanitize) {
+    ASAN=1
+  }
+
+  set build(Windows-WinRT) {
+    FOR_WINRT=1
+    ENABLE_SETLK=1
+    -DSQLITE_TEMP_STORE=3
+  }
 }
 
 
@@ -84,6 +379,7 @@ proc trd_import {} {
     variable ::trd::tcltest
     variable ::trd::extra
     variable ::trd::all_configs
+    variable ::trd::build
   }
 }
 
@@ -106,38 +402,310 @@ proc trd_builds {platform} {
   set ret
 }
 
-proc trd_configs {platform build} {
+proc trd_configs {platform bld} {
   trd_import
 
   set clist [list]
 
-  if {[info exists tcltest($platform.$build)]} {
-    set clist $tcltest($platform.$build)
+  if {[info exists tcltest($platform.$bld)]} {
+    set clist $tcltest($platform.$bld)
     if {$clist=="all"} {
       set clist $all_configs
     } elseif {$clist=="all_plus_autovacuum_crash"} {
       set clist [concat $all_configs autovacuum_crash]
+    } elseif {$clist=="all_less_no_mutex_try"} {
+      set idx [lsearch $all_configs no_mutex_try]
+      set clist [lreplace $all_configs $idx $idx]
     }
   }
 
   set clist
 }
 
-proc trd_extras {platform build} {
+proc trd_extras {platform bld} {
   trd_import
+  if {[info exists extra($platform.$bld)]==0} { return [list] }
+  return $extra($platform.$bld)
+}
 
-  set elist [list]
-  if {[info exists extra($platform.$build)]} {
-    set elist $extra($platform.$build)
-  }
+# Usage: 
+#
+#     trd_fuzztest_data $buildname
+#
+# This returns data used by testrunner.tcl to run commands equivalent 
+# to [make fuzztest]. The returned value is a list, which should be
+# interpreted as a sequence of pairs. The first element of each pair
+# is an interpreter name. The second element is a list of files.
+# testrunner.tcl automatically creates one job to build each interpreter,
+# and one to run each of the files with it once it has been built.
+#
+# In practice, the returned value looks like this:
+#
+# {
+#   {fuzzcheck {$testdir/fuzzdata1.db $testdir/fuzzdata2.db ...}}
+#   {{sessionfuzz run} $testdir/sessionfuzz-data1.db}
+# }
+#
+# where $testdir is replaced by the full-path to the test-directory (the
+# directory containing this file). "fuzzcheck" and "sessionfuzz" have .exe
+# extensions on windows.
+#
+proc trd_fuzztest_data {buildname} {
+  set EXE ""
+  set lFuzzDb    [glob [file join $::testdir fuzzdata*.db]] 
+  set lSessionDb [glob [file join $::testdir sessionfuzz-data*.db]]
+  set sanBuilds {All-Debug Apple Have-Not Update-Delete-Limit}
 
-  set elist
+  if {$::tcl_platform(platform) eq "windows"} {
+    return [list fuzzcheck.exe $lFuzzDb]
+  } else {
+    set lRet [list [trd_get_bin_name fuzzcheck] $lFuzzDb]
+    if {[lsearch $sanBuilds $buildname]>=0} {
+      lappend lRet [trd_get_bin_name fuzzcheck-asan] $lFuzzDb 
+      if {$::tcl_platform(os) ne "OpenBSD"} {
+        lappend lRet [trd_get_bin_name fuzzcheck-ubsan] $lFuzzDb 
+      }
+    }
+    lappend lRet {sessionfuzz run} $lSessionDb
+    return $lRet
+  }
 }
 
+
 proc trd_all_configs {} {
   trd_import
   set all_configs
 }
 
+proc trimscript {text} {
+  set text [string map {"\n    " "\n"} [string trim $text]]
+}
+
+proc make_sh_script {srcdir opts cflags makeOpts configOpts} {
+
+  set tcldir [::tcl::pkgconfig get libdir,install]
+  set myopts ""
+  if {[info exists ::env(OPTS)]} {
+    append myopts "# From environment variable:\n"
+    append myopts "OPTS=$::env(OPTS)\n\n"
+  }
+  foreach o [lsort $opts] { 
+    append myopts "OPTS=\"\$OPTS $o\"\n"
+  }
+
+  return [trimscript [subst -nocommands {
+    set -e
+    if [ "\$#" -ne 1 ] ; then
+      echo "Usage: \$0 <target>"
+      exit -1
+    fi
+    
+    SRCDIR="$srcdir"
+    TCLDIR="$tcldir"
+    
+    if [ ! -f Makefile ] ; then
+      \$SRCDIR/configure --with-tcl=\$TCLDIR $configOpts 
+    fi
+    
+    $myopts
+    CFLAGS="$cflags"
+    
+    make \$1 "CFLAGS=\$CFLAGS" "OPTS=\$OPTS" $makeOpts
+  }]]
+}
+
+# Generate the text of a *.bat script.
+#
+proc make_bat_file {srcdir opts cflags makeOpts} {
+  set srcdir [file nativename [file normalize $srcdir]]
+
+  return [trimscript [subst -nocommands {
+    set TARGET=%1
+    set TMP=%CD%
+    nmake /f $srcdir\\Makefile.msc TOP="$srcdir" %TARGET% "CCOPTS=$cflags" "OPTS=$opts" $makeOpts
+  }]]
+}
+
+
+# Generate the text of a shell script.
+#
+proc make_script {cfg srcdir bMsvc} {
+  set opts       [list]                         ;# OPTS value
+  set cflags     [expr {$bMsvc ? "-Zi" : "-g"}] ;# CFLAGS value
+  set makeOpts   [list]                         ;# Extra args for [make]
+  set configOpts [list]                         ;# Extra args for [configure]
+
+  # Define either SQLITE_OS_WIN or SQLITE_OS_UNIX, as appropriate.
+  if {$::tcl_platform(os) eq "Windows NT"} {
+    lappend opts -DSQLITE_OS_WIN=1
+  } else {
+    lappend opts -DSQLITE_OS_UNIX=1
+  }
+
+  # Unless the configuration specifies -DHAVE_USLEEP=0, set -DHAVE_USLEEP=1.
+  #
+  if {[lsearch $cfg "-DHAVE_USLEEP=0"]<0} {
+    lappend cfg -DHAVE_USLEEP=1
+  }
+
+  # Loop through the parameters of the nominated configuration, updating
+  # $opts, $cflags, $makeOpts and $configOpts along the way. Rules are as
+  # follows:
+  #
+  #   1. If the parameter begins with "-D", add it to $opts.
+  #
+  #   2. If the parameter begins with "--" add it to $configOpts. Unless
+  #      this command is preparing a script for MSVC - then add an 
+  #      equivalent to $makeOpts or $opts.
+  #
+  #   3. If the parameter begins with "-" add it to $cflags. If in MSVC
+  #      mode and the parameter is an -O<integer> option, instead add
+  #      an OPTIMIZATIONS=<integer> switch to $makeOpts.
+  #
+  #   4. If none of the above apply, add the parameter to $makeOpts
+  #
+  foreach param $cfg {
+
+    if {[string range $param 0 1]=="-D"} {
+      lappend opts $param
+      continue
+    }
+
+    if {[string range $param 0 1]=="--"} {
+      if {$bMsvc==0} {
+        lappend configOpts $param
+      } else {
+
+        switch -- $param {
+          --disable-amalgamation {
+            lappend makeOpts USE_AMALGAMATION=0
+          }
+          --disable-shared {
+            lappend makeOpts USE_CRT_DLL=0 DYNAMIC_SHELL=0
+          }
+          --enable-fts5 {
+            lappend opts -DSQLITE_ENABLE_FTS5
+          }
+          --enable-shared {
+            lappend makeOpts USE_CRT_DLL=1 DYNAMIC_SHELL=1
+          }
+          --enable-session {
+            lappend opts -DSQLITE_ENABLE_PREUPDATE_HOOK
+            lappend opts -DSQLITE_ENABLE_SESSION
+          }
+          --enable-all {
+          }
+          --with-debug {
+            # lappend makeOpts OPTIMIZATIONS=0
+            lappend opts -DSQLITE_DEBUG
+          }
+          default {
+            error "Cannot translate $param for MSVC"
+          }
+        }
+      }
+
+      continue
+    }
+
+    if {[string range $param 0 0]=="-"} {
+
+      if {$bMsvc} {
+        if {[regexp -- {^-O(\d+)$} $param -> level]} {
+          lappend makeOpts OPTIMIZATIONS=$level
+          continue
+        }
+        if {$param eq "-fsanitize=address,undefined"} {
+          lappend makeOpts ASAN=1
+          continue
+        }
+      }
+
+      lappend cflags $param
+      continue
+    }
+
+    lappend makeOpts $param
+  }
+
+  if {$bMsvc==0} {
+    set zRet [make_sh_script $srcdir $opts $cflags $makeOpts $configOpts]
+  } else {
+    set zRet [make_bat_file $srcdir $opts $cflags $makeOpts]
+  }
+}
+
+# Usage:
+#
+#    trd_buildscript CONFIG SRCDIR MSVC
+#
+# This command returns the full text of a script (either a shell script or
+# an ms-dos bat file) that may be used to build SQLite source code according
+# to a nominated configuration.
+#
+# Parameter CONFIG must be a configuration defined above in the ::trd::build
+# array. SRCDIR is the root directory of an SQLite source tree (the parent
+# directory of that containing this script). MSVC is a boolean - true to
+# use the MSVC compiler, false otherwise.
+#
+proc trd_buildscript {config srcdir bMsvc} {
+  trd_import
 
+  # Ensure that the named configuration exists.
+  if {![info exists build($config)]} {
+    if {$config!="help"} {
+      puts "No such build config: $config"
+    }
+    puts "Available configurations: [lsort [array names build]]"
+    flush stdout
+    exit 1
+  }
+
+  # Generate and return the script.
+  return [make_script $build($config) $srcdir $bMsvc]
+}
+
+# Usage:
+#
+#    trd_test_script_properties PATH
+#
+# The argument must be a path to a Tcl test script. This function scans the
+# first 100 lines of the script for lines that look like:
+#
+#    TESTRUNNER: <properties>
+#
+# where <properties> is a list of identifiers, each of which defines a 
+# property of the test script. Example properties are "slow" or "superslow".
+#
+proc trd_test_script_properties {path} {
+  # Use this global array as a cache:
+  global trd_test_script_properties_cache
 
+  if {![info exists trd_test_script_properties_cache($path)]} {
+    set fd [open $path]
+    set ret [list]
+    for {set line 0} {$line < 100 && ![eof $fd]} {incr line} {
+      set text [gets $fd]
+      if {[string match -nocase *testrunner:* $text]} {
+        regexp -nocase {.*testrunner:(.*)} $text -> properties
+        lappend ret {*}$properties
+      }
+    }
+    set trd_test_script_properties_cache($path) $ret
+    close $fd
+  }
+
+  set trd_test_script_properties_cache($path)
+}
+
+# Usage:
+#
+#    trd_get_bin_name executable-file-name
+#
+# If the tcl platform is "unix", return $bin, else return
+# ${bin}.exe.
+proc trd_get_bin_name {bin} {
+  global tcl_platform
+  if {"unix" eq $tcl_platform(platform)} {return $bin}
+  return $bin.exe
+}
diff --git a/test/testrunner_estwork.tcl b/test/testrunner_estwork.tcl
new file mode 100644
index 0000000000..c139394a56
--- /dev/null
+++ b/test/testrunner_estwork.tcl
@@ -0,0 +1,488 @@
+# Estimated relative cost of various jobs, based on the "estkey" field.
+# Computed by the "test/testrunner.tcl estwork" command.
+#
+set estwork((fuzzcheck)) 1291
+set estwork((fuzzcheck-asan)) 2427
+set estwork((fuzzcheck-ubsan)) 2749
+set estwork((sessionfuzz)) 1276
+set estwork((sqlite3)) 1281
+set estwork((testfixture)) 1546
+set estwork(aggerror.test) 5
+set estwork(alter.test) 20
+set estwork(alter3.test) 2
+set estwork(alterauth.test) 5
+set estwork(altercol.test) 5
+set estwork(alterdropcol.test) 17
+set estwork(alterlegacy.test) 2
+set estwork(altertab.test) 34
+set estwork(altertab2.test) 2
+set estwork(altertab3.test) 12
+set estwork(amatch1.test) 40
+set estwork(atof1.test) 146
+set estwork(atomic2.test) 2
+set estwork(auth.test) 2
+set estwork(autoindex1.test) 5
+set estwork(autoindex3.test) 5
+set estwork(autovacuum.test) 15
+set estwork(avfs.test) 16
+set estwork(avtrans.test) 285
+set estwork(backup2.test) 15
+set estwork(bestindex4.test) 12
+set estwork(bigrow.test) 5
+set estwork(bitvec.test) 94
+set estwork(bld) 99
+set estwork(blob.test) 2
+set estwork(boundary1.test) 8
+set estwork(boundary2.test) 16
+set estwork(boundary3.test) 14
+set estwork(btree01.test) 20
+set estwork(busy.test) 2
+set estwork(busy2.test) 288
+set estwork(capi3.test) 5
+set estwork(capi3b.test) 6
+set estwork(capi3c.test) 5
+set estwork(capi3d.test) 8
+set estwork(capi3e.test) 20
+set estwork(changes.test) 11
+set estwork(chunksize.test) 5
+set estwork(closure01.test) 63
+set estwork(collate5.test) 2
+set estwork(conflict.test) 8
+set estwork(conflict2.test) 4
+set estwork(conflict3.test) 8
+set estwork(corrupt2.test) 2
+set estwork(corrupt4.test) 91
+set estwork(corrupt7.test) 5
+set estwork(corrupt8.test) 3
+set estwork(corruptB.test) 4
+set estwork(corruptF.test) 6
+set estwork(count.test) 18
+set estwork(crash8.test) 14
+set estwork(createtab.test) 3
+set estwork(csv01.test) 7
+set estwork(cursorhint.test) 4
+set estwork(date.test) 17
+set estwork(date4.test) 64
+set estwork(date5.test) 3
+set estwork(dbdata.test) 77
+set estwork(dbfuzz001.test) 2
+set estwork(dbstatus.test) 4
+set estwork(decimal.test) 2
+set estwork(delete.test) 4
+set estwork(diskfull.test) 34
+set estwork(e_blobbytes.test) 11
+set estwork(e_createtable.test) 14
+set estwork(e_delete.test) 3
+set estwork(e_droptrigger.test) 6
+set estwork(e_dropview.test) 2
+set estwork(e_expr.test) 251
+set estwork(e_select.test) 5
+set estwork(e_select2.test) 5
+set estwork(e_vacuum.test) 6
+set estwork(e_walauto.test) 51
+set estwork(e_walckpt.test) 6
+set estwork(enc.test) 2
+set estwork(enc3.test) 11
+set estwork(enc4.test) 3
+set estwork(eval.test) 2
+set estwork(exists.test) 4
+set estwork(existsexpr.test) 44
+set estwork(expert1.test) 14
+set estwork(expr.test) 5
+set estwork(filectrl.test) 5
+set estwork(filefmt.test) 6
+set estwork(fkey2.test) 9
+set estwork(fpconv1.test) 40
+set estwork(fts3aa.test) 18
+set estwork(fts3ad.test) 10
+set estwork(fts3ag.test) 6
+set estwork(fts3aj.test) 8
+set estwork(fts3am.test) 5
+set estwork(fts3auto.test) 29
+set estwork(fts3b.test) 41
+set estwork(fts3c.test) 22
+set estwork(fts3conf.test) 3
+set estwork(fts3corrupt4.test) 7
+set estwork(fts3corrupt5.test) 8
+set estwork(fts3corrupt6.test) 13
+set estwork(fts3d.test) 2
+set estwork(fts3defer2.test) 2
+set estwork(fts3expr.test) 10
+set estwork(fts3expr2.test) 8
+set estwork(fts3expr3.test) 145
+set estwork(fts3f.test) 9
+set estwork(fts3first.test) 3
+set estwork(fts3matchinfo.test) 4
+set estwork(fts3misc.test) 54
+set estwork(fts3prefix.test) 4
+set estwork(fts3prefix2.test) 8
+set estwork(fts3query.test) 23
+set estwork(fts3varint.test) 2
+set estwork(fts4aa.test) 34
+set estwork(fts4content.test) 5
+set estwork(fts4incr.test) 19
+set estwork(fts4noti.test) 3
+set estwork(fts4opt.test) 129
+set estwork(fts4unicode.test) 45
+set estwork(fts5aa.test) 442
+set estwork(fts5ab.test) 54
+set estwork(fts5ad.test) 33
+set estwork(fts5ae.test) 3
+set estwork(fts5af.test) 13
+set estwork(fts5ag.test) 13
+set estwork(fts5ah.test) 265
+set estwork(fts5al.test) 2
+set estwork(fts5auto.test) 92
+set estwork(fts5connect.test) 5
+set estwork(fts5content.test) 9
+set estwork(fts5contentless.test) 23
+set estwork(fts5contentless2.test) 189
+set estwork(fts5contentless4.test) 234
+set estwork(fts5contentless5.test) 9
+set estwork(fts5delete.test) 62
+set estwork(fts5doclist.test) 11
+set estwork(fts5expr.test) 10
+set estwork(fts5full.test) 42
+set estwork(fts5hash.test) 17
+set estwork(fts5integrity.test) 84
+set estwork(fts5interrupt.test) 16
+set estwork(fts5locale.test) 5
+set estwork(fts5matchinfo.test) 11
+set estwork(fts5merge.test) 29
+set estwork(fts5merge2.test) 9
+set estwork(fts5misc.test) 8
+set estwork(fts5multiclient.test) 2
+set estwork(fts5optimize.test) 13
+set estwork(fts5optimize2.test) 737
+set estwork(fts5optimize3.test) 280
+set estwork(fts5origintext.test) 144
+set estwork(fts5origintext3.test) 8
+set estwork(fts5origintext4.test) 30
+set estwork(fts5origintext5.test) 462
+set estwork(fts5origintext6.test) 55
+set estwork(fts5porter.test) 57
+set estwork(fts5query.test) 8
+set estwork(fts5restart.test) 10
+set estwork(fts5rowid.test) 35
+set estwork(fts5secure.test) 61
+set estwork(fts5secure3.test) 796
+set estwork(fts5secure4.test) 141
+set estwork(fts5secure5.test) 29
+set estwork(fts5secure6.test) 33
+set estwork(fts5secure7.test) 680
+set estwork(fts5simple.test) 7
+set estwork(fts5simple2.test) 2
+set estwork(fts5synonym.test) 13
+set estwork(fts5synonym2.test) 384
+set estwork(fts5tok2.test) 92
+set estwork(fts5tokenizer.test) 2
+set estwork(fts5tokenizer3.test) 8
+set estwork(fts5trigram.test) 2
+set estwork(fts5unicode2.test) 28
+set estwork(fts5unicode3.test) 72
+set estwork(fts5unindexed.test) 7
+set estwork(fts5update.test) 161
+set estwork(fts5update2.test) 11
+set estwork(fts5vocab.test) 11
+set estwork(fts5vocab2.test) 15
+set estwork(func.test) 36
+set estwork(fuzz) 68
+set estwork(fuzz-oss1.test) 11
+set {estwork(fuzzcheck --slice)} 499
+set {estwork(fuzzcheck fuzzdata1.db)} 301
+set {estwork(fuzzcheck fuzzdata2.db)} 275
+set {estwork(fuzzcheck fuzzdata3.db)} 72
+set {estwork(fuzzcheck fuzzdata4.db)} 30
+set {estwork(fuzzcheck fuzzdata5.db)} 299
+set {estwork(fuzzcheck fuzzdata6.db)} 91
+set {estwork(fuzzcheck fuzzdata7.db)} 201
+set {estwork(fuzzcheck fuzzdata8.db)} 331
+set {estwork(fuzzcheck-asan --slice)} 2037
+set {estwork(fuzzcheck-asan fuzzdata1.db)} 3112
+set {estwork(fuzzcheck-asan fuzzdata2.db)} 8753
+set {estwork(fuzzcheck-asan fuzzdata3.db)} 459
+set {estwork(fuzzcheck-asan fuzzdata4.db)} 123
+set {estwork(fuzzcheck-asan fuzzdata5.db)} 1178
+set {estwork(fuzzcheck-asan fuzzdata6.db)} 1356
+set {estwork(fuzzcheck-asan fuzzdata7.db)} 938
+set {estwork(fuzzcheck-asan fuzzdata8.db)} 1451
+set {estwork(fuzzcheck-ubsan --slice)} 1729
+set {estwork(fuzzcheck-ubsan fuzzdata1.db)} 1180
+set {estwork(fuzzcheck-ubsan fuzzdata2.db)} 876
+set {estwork(fuzzcheck-ubsan fuzzdata3.db)} 306
+set {estwork(fuzzcheck-ubsan fuzzdata4.db)} 95
+set {estwork(fuzzcheck-ubsan fuzzdata5.db)} 1356
+set {estwork(fuzzcheck-ubsan fuzzdata6.db)} 333
+set {estwork(fuzzcheck-ubsan fuzzdata7.db)} 883
+set {estwork(fuzzcheck-ubsan fuzzdata8.db)} 1124
+set estwork(fuzzer1.test) 6
+set estwork(gencol1.test) 3
+set estwork(hook.test) 2
+set estwork(in4.test) 4
+set estwork(in7.test) 6
+set estwork(incrblob2.test) 2
+set estwork(incrblob3.test) 5
+set estwork(incrvacuum.test) 10
+set estwork(incrvacuum2.test) 17
+set estwork(incrvacuum3.test) 17
+set estwork(index.test) 3
+set estwork(index2.test) 8
+set estwork(index4.test) 33
+set estwork(index5.test) 99
+set estwork(index6.test) 2
+set estwork(indexexpr1.test) 2
+set estwork(insert3.test) 5
+set estwork(insert4.test) 2
+set estwork(intarray.test) 8
+set estwork(intck1.test) 34
+set estwork(intck2.test) 37
+set estwork(interrupt.test) 28
+set estwork(io.test) 3
+set estwork(join.test) 3
+set estwork(join3.test) 6
+set estwork(join5.test) 20
+set estwork(join7.test) 2
+set estwork(join8.test) 3
+set estwork(join9.test) 2
+set estwork(joinA.test) 11
+set estwork(joinB.test) 7
+set estwork(joinC.test) 6
+set estwork(joinD.test) 168
+set estwork(json103.test) 6
+set estwork(json106.test) 690
+set estwork(keyword1.test) 3
+set estwork(like2.test) 2
+set estwork(like3.test) 2
+set estwork(limit.test) 3
+set estwork(literal.test) 6
+set estwork(lock.test) 39
+set estwork(lock4.test) 2
+set estwork(lock5.test) 4
+set estwork(make) 102
+set estwork(manydb.test) 12
+set estwork(mem5.test) 2
+set estwork(memdb.test) 9
+set estwork(memdb1.test) 2
+set estwork(memjournal2.test) 164
+set estwork(memsubsys1.test) 8
+set estwork(misc1.test) 6
+set estwork(misc2.test) 6
+set estwork(misc5.test) 11
+set estwork(misc8.test) 5
+set estwork(mmap1.test) 8
+set estwork(mmap2.test) 10
+set estwork(mmapwarm.test) 2
+set estwork(multiplex.test) 30
+set estwork(multiplex2.test) 7
+set estwork(nan.test) 2
+set estwork(notify3.test) 5
+set estwork(orderby1.test) 11
+set estwork(orderby2.test) 2
+set estwork(orderby5.test) 11
+set estwork(orderby6.test) 6
+set estwork(orderby8.test) 20
+set estwork(orderbyA.test) 2
+set estwork(oserror.test) 7
+set estwork(ovfl.test) 11
+set estwork(pager1.test) 81
+set estwork(pager2.test) 129
+set estwork(pagesize.test) 3
+set estwork(percentile.test) 86
+set estwork(pragma.test) 4
+set estwork(pragma4.test) 18
+set estwork(printf.test) 21
+set estwork(printf2.test) 5
+set estwork(quota.test) 14
+set estwork(randexpr1.test) 35
+set estwork(rbu10.test) 16
+set estwork(rbu13.test) 5
+set estwork(rbuexlock.test) 5
+set estwork(rbumisc.test) 2
+set estwork(rbutemplimit.test) 2
+set estwork(readonly.test) 95
+set estwork(recover.test) 3
+set estwork(recover1.test) 7
+set estwork(recovercorrupt3.test) 5
+set estwork(recoverold.test) 7
+set estwork(recoverpgsz.test) 6
+set estwork(recoverrowid.test) 3
+set estwork(returning1.test) 6
+set estwork(rollback2.test) 2
+set estwork(round1.test) 261
+set estwork(rowhash.test) 85
+set estwork(rowid.test) 3
+set estwork(rowvalue.test) 2
+set estwork(rowvalue2.test) 38
+set estwork(rowvalue4.test) 2
+set estwork(rowvalueA.test) 2
+set estwork(rowvaluevtab.test) 2
+set estwork(rtree1.test) 4
+set estwork(rtree2.test) 758
+set estwork(rtree6.test) 6
+set estwork(rtree8.test) 15
+set estwork(rtree9.test) 15
+set estwork(rtreeA.test) 7
+set estwork(rtreeB.test) 7
+set estwork(rtreeE.test) 47
+set estwork(rtreeH.test) 24
+set estwork(rtreecheck.test) 2
+set estwork(rtreedoc.test) 8
+set estwork(rtreedoc3.test) 76
+set estwork(savepoint.test) 10
+set estwork(savepoint2.test) 57
+set estwork(schema2.test) 15
+set estwork(schema3.test) 2
+set estwork(schema5.test) 5
+set estwork(select1.test) 2
+set estwork(select2.test) 17
+set estwork(select3.test) 2
+set estwork(selectA.test) 2
+set estwork(selectB.test) 2
+set estwork(selectG.test) 30
+set estwork(session1.test) 5
+set estwork(session2.test) 33
+set estwork(session5.test) 63
+set estwork(session9.test) 3
+set estwork(sessionG.test) 60
+set estwork(sessionH.test) 18
+set estwork(sessionalter.test) 3
+set estwork(sessionat.test) 2
+set estwork(sessionblob.test) 3
+set {estwork(sessionfuzz sessionfuzz-data1.db)} 5
+set estwork(sessioninvert.test) 2
+set estwork(sessionnoop.test) 2
+set estwork(sessionnoop2.test) 8
+set estwork(sessionrebase.test) 8
+set estwork(shared.test) 7
+set estwork(sharedA.test) 48
+set estwork(shell1.test) 36
+set estwork(shell2.test) 11
+set estwork(shell3.test) 3
+set estwork(shell4.test) 2
+set estwork(shell5.test) 16
+set estwork(shell6.test) 3
+set estwork(shell8.test) 104
+set estwork(shell9.test) 3
+set estwork(shellA.test) 2
+set estwork(shmlock.test) 27
+set estwork(sidedelete.test) 10
+set estwork(skipscan1.test) 7
+set estwork(skipscan2.test) 5
+set estwork(sort.test) 38
+set estwork(sort2.test) 540
+set estwork(sort5.test) 16
+set estwork(spellfix.test) 5
+set estwork(spellfix2.test) 5
+set estwork(spellfix4.test) 11
+set estwork(starschema1.test) 2
+set estwork(strict1.test) 5
+set estwork(swarmvtab.test) 110
+set estwork(swarmvtab3.test) 14
+set estwork(syscall.test) 4
+set estwork(table.test) 62
+set estwork(tableapi.test) 8
+set estwork(tcl) 1
+set estwork(tclsqlite.test) 29
+set estwork(temptable2.test) 274
+set estwork(thread3.test) 21
+set estwork(timediff1.test) 5
+set estwork(tkt-2d1a5c67d.test) 6
+set estwork(tkt-38cb5df375.test) 2
+set estwork(tkt-4dd95f6943.test) 2
+set estwork(tkt-5e10420e8d.test) 5
+set estwork(tkt-6bfb98dfc0.test) 5
+set estwork(tkt-80e031a00f.test) 25
+set estwork(tkt-9d68c883.test) 3
+set estwork(tkt-9f2eb3abac.test) 5
+set estwork(tkt-b1d3a2e531.test) 5
+set estwork(tkt-b72787b1.test) 5
+set estwork(tkt-b75a9ca6b0.test) 8
+set estwork(tkt-d11f09d36e.test) 9
+set estwork(tkt-fc62af4523.test) 10
+set estwork(tkt1435.test) 7
+set estwork(tkt1644.test) 5
+set estwork(tkt1667.test) 20
+set estwork(tkt1873.test) 2
+set estwork(tkt2192.test) 3
+set estwork(tkt2285.test) 19
+set estwork(tkt2332.test) 2
+set estwork(tkt2409.test) 24
+set estwork(tkt3334.test) 5
+set estwork(tkt3357.test) 10
+set estwork(tkt3630.test) 2
+set estwork(tkt3832.test) 8
+set estwork(tkt3838.test) 5
+set estwork(tkt3918.test) 3
+set estwork(tkt4018.test) 48
+set estwork(tokenize.test) 9
+set estwork(tpch01.test) 6
+set estwork(trans.test) 258
+set estwork(trigger2.test) 15
+set estwork(trigger5.test) 2
+set estwork(trigger8.test) 30
+set estwork(triggerA.test) 36
+set estwork(triggerB.test) 4
+set estwork(triggerC.test) 67
+set estwork(triggerD.test) 3
+set estwork(types.test) 2
+set estwork(types2.test) 2
+set estwork(unionall2.test) 9
+set estwork(unionvtab.test) 2
+set estwork(update.test) 7
+set estwork(upsert3.test) 6
+set estwork(upsert5.test) 5
+set estwork(vacuum.test) 2
+set estwork(vacuum5.test) 3
+set estwork(vacuum6.test) 174
+set estwork(vacuummem.test) 132
+set estwork(varint.test) 8
+set estwork(view.test) 2
+set estwork(vtab1.test) 13
+set estwork(vtab6.test) 12
+set estwork(vtabC.test) 33
+set estwork(vtabD.test) 56
+set estwork(vtab_alter.test) 6
+set estwork(wal.test) 38
+set estwork(wal2.test) 7
+set estwork(wal3.test) 196
+set estwork(wal4.test) 70
+set estwork(wal5.test) 22
+set estwork(wal64k.test) 20
+set estwork(wal7.test) 3
+set estwork(wal9.test) 24
+set estwork(walbak.test) 2
+set estwork(walcksum.test) 3
+set estwork(walcrash4.test) 59
+set estwork(waloverwrite.test) 3
+set estwork(walpersist.test) 6
+set estwork(walprotocol2.test) 2
+set estwork(walro2.test) 11
+set estwork(walsetlk.test) 1610
+set estwork(walsetlk3.test) 3
+set estwork(walsetlk_recover.test) 193
+set estwork(walshared.test) 5
+set estwork(walvfs.test) 476
+set estwork(where.test) 24
+set estwork(where3.test) 4
+set estwork(where6.test) 2
+set estwork(where7.test) 20
+set estwork(where8.test) 95
+set estwork(where9.test) 16
+set estwork(whereB.test) 5
+set estwork(whereE.test) 6
+set estwork(whereN.test) 2
+set estwork(window1.test) 5
+set estwork(window2.test) 8
+set estwork(window3.test) 89
+set estwork(window4.test) 3
+set estwork(window5.test) 2
+set estwork(window8.test) 33
+set estwork(windowA.test) 5
+set estwork(windowD.test) 6
+set estwork(with1.test) 114
+set estwork(with2.test) 4
+set estwork(with5.test) 2
+set estwork(withM.test) 4
+set estwork(without_rowid1.test) 2
+set estwork(without_rowid3.test) 9
+set estwork(without_rowid4.test) 6
diff --git a/test/thread3.test b/test/thread3.test
index 25699b7655..79a75bdbb7 100644
--- a/test/thread3.test
+++ b/test/thread3.test
@@ -75,4 +75,3 @@ do_execsql_test "1.Total BUSY errors: $nTotalBusy .2" {
 } $nAttempt
 
 finish_test
-
diff --git a/test/thread_common.tcl b/test/thread_common.tcl
index 6b17082ad4..1ebc6573a9 100644
--- a/test/thread_common.tcl
+++ b/test/thread_common.tcl
@@ -95,7 +95,7 @@ proc run_thread_tests {{print_warning 0}} {
   if {[info commands sqlthread] eq ""} {
     set zProblem "SQLite build is not threadsafe"
   }
-  if {![info exists ::tcl_platform(threaded)]} {
+  if {![tcl::pkgconfig get threaded]} {
     set zProblem "Linked against a non-threadsafe Tcl build"
   }
   if {[info exists zProblem]} {
@@ -107,4 +107,3 @@ proc run_thread_tests {{print_warning 0}} {
 }
 
 return 0
-
diff --git a/test/timediff1.test b/test/timediff1.test
new file mode 100644
index 0000000000..f8176d8810
--- /dev/null
+++ b/test/timediff1.test
@@ -0,0 +1,222 @@
+# 2023-05-30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing date and time functions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Skip this whole file if date and time functions are omitted
+# at compile-time
+#
+ifcapable {!datetime} {
+  finish_test
+  return
+}
+
+proc datetest {tnum expr result} {
+  do_test timediff-$tnum [subst {
+    execsql "SELECT coalesce($expr,'NULL')"
+  }] [list $result]
+}
+set tcl_precision 15
+
+
+# February overflow on a leap year
+datetest 1.1 {datetime('2000-01-31','+1 month')} {2000-03-02 00:00:00}
+datetest 1.2 {datetime('2004-01-29','+1 month')} {2004-02-29 00:00:00}
+datetest 1.3 {datetime('2000-03-01','-1 day')}   {2000-02-29 00:00:00}
+datetest 1.4 {datetime('2000-03-31','-1 month')} {2000-03-02 00:00:00}
+datetest 1.5 {datetime('2000-03-30','-1 month')} {2000-03-01 00:00:00}
+datetest 1.6 {datetime('2000-03-29','-1 month')} {2000-02-29 00:00:00}
+datetest 1.7 {datetime('2000-03-28','-1 month')} {2000-02-28 00:00:00}
+datetest 1.8 {datetime('2000-02-29','+1 year')}  {2001-03-01 00:00:00}
+datetest 1.9 {datetime('2000-02-29','+4 years')} {2004-02-29 00:00:00}
+
+datetest 1.10 {datetime('1998-11-10','+0001-03-19 12:34:56')} \
+                                                 {2000-02-29 12:34:56}
+datetest 1.11 {datetime('2000-01-31','+0004-01-00 12:34:56')} \
+                                                 {2004-03-02 12:34:56}
+datetest 1.12 {datetime('2000-01-29','+0008-01-00 12:34:56')} \
+                                                 {2008-02-29 12:34:56}
+datetest 1.13 {datetime('2001-03-31','-0001-01-00 06:10')} \
+                                                 {2000-03-01 17:50:00}
+
+
+# February overflow on a non-leap year
+datetest 2.1 {datetime('2001-01-31','+1 month')} {2001-03-03 00:00:00}
+datetest 2.2 {datetime('2005-01-29','+1 month')} {2005-03-01 00:00:00}
+datetest 2.3 {datetime('2001-03-01','-1 day')}   {2001-02-28 00:00:00}
+datetest 2.4 {datetime('2001-03-31','-1 month')} {2001-03-03 00:00:00}
+datetest 2.5 {datetime('2001-03-30','-1 month')} {2001-03-02 00:00:00}
+datetest 2.6 {datetime('2001-03-29','-1 month')} {2001-03-01 00:00:00}
+datetest 2.7 {datetime('2001-03-28','-1 month')} {2001-02-28 00:00:00}
+
+datetest 2.10 {datetime('1999-11-10','+0001-03-19 12:34:56')} \
+                                                 {2001-03-01 12:34:56}
+datetest 2.11 {datetime('2000-01-31','+0005-01-00 12:34:56')} \
+                                                 {2005-03-03 12:34:56}
+datetest 2.12 {datetime('2000-01-29','+0009-01-00 12:34:56')} \
+                                                 {2009-03-01 12:34:56}
+datetest 2.13 {datetime('2002-03-31','-0001-01-00 06:10')} \
+                                                 {2001-03-02 17:50:00}
+
+# timediff
+datetest 3.1 {timediff('2000-03-02','2000-01-31')} {+0000-01-00 00:00:00.000}
+datetest 3.2 {timediff('2000-01-31','2000-03-02')} {-0000-01-02 00:00:00.000}
+datetest 3.3 {timediff('2000-03-02','1999-01-31')} {+0001-01-00 00:00:00.000}
+datetest 3.4 {timediff('1999-01-31','2000-03-02')} {-0001-01-02 00:00:00.000}
+
+unset -nocomplain p1
+unset -nocomplain p2
+set p1 {
+  0   {-4713-11-24 12:00:00}
+  1   {-2000-04-30 05:19:26}
+  2   {0000-01-01 12:34:56}
+  3   {1776-07-04 13:00:00}
+  4   {1969-07-20 20:17}
+  5   {2440587.5}
+  6   {2000-05-29 14:26}
+  7   {2023-05-29 18:11}
+  8   {2050-05-29 14:26}
+  9   {4796-02-29 11:23:55.46}
+}
+set p2 {
+  A   {1066-10-14}
+  B   {1900-02-28 11:00}
+  C   {1900-03-01 12:00}
+  D   {1904-02-29 11:25}
+  E   {2000-02-29 13:00}
+  E   {2000-03-01 14:00}
+  F   {2001-03-31 15:15}
+  G   {2002-04-01 16:59}
+  H   {2003-04-30 17:00}
+  I   {2004-05-01 23:59:59}
+  J   {2005-06-01}
+  K   {2006-06-30 01:23:45}
+  L   {2007-12-31 02:00}
+  M   {2008-01-01 01:59}
+  N   {3152-07-04 12:00}
+  P   {9999-12-31 23:59:59}
+}
+
+foreach {x1 d1} $p1 {
+  foreach {x2 d2} $p2 {
+    set r1 [db one {SELECT datetime($d1)}]
+    do_execsql_test timediff-4-$x1$x2 {
+      SELECT datetime($d2, timediff($d1,$d2));
+    } [list $r1]
+    set r2 [db one {SELECT datetime($d2)}]
+    do_execsql_test timediff-4-$x2$x1 {
+      SELECT datetime($d1, timediff($d2,$d1));
+    } [list $r2]
+  }
+}
+
+# Partial time-diffs as modifiers
+#
+datetest 5-1 {datetime('2000-01-01','+0001-02-03')} {2001-03-04 00:00:00}
+datetest 5-2 {datetime('2000-01-01','+0001-02-03x')} {NULL}
+datetest 5-3 {datetime('2000-01-01','+0001-11-03')} {2001-12-04 00:00:00}
+datetest 5-4 {datetime('2000-01-01','+0001-12-03')} {NULL}
+datetest 5-5 {datetime('2000-01-01','+0001-02-30')} {2001-03-31 00:00:00}
+datetest 5-6 {datetime('2000-01-01','+0001-02-31')} {NULL}
+datetest 5-7 {datetime('2000-01-01','+0001-02-03 0')} {NULL}
+datetest 5-8 {datetime('2000-01-01','+0001-02-03 01')} {NULL}
+datetest 5-9 {datetime('2000-01-01','+0001-02-03 01:')} {NULL}
+datetest 5-10 {datetime('2000-01-01','+0001-02-03 01:0')} {NULL}
+datetest 5-11 {datetime('2000-01-01','+0001-02-03 01:02')} {2001-03-04 01:02:00}
+datetest 5-12 {datetime('2000-01-01','+0001-02-03 01:02:')} {NULL}
+datetest 5-13 {datetime('2000-01-01','+0001-02-03 01:02:0')} {NULL}
+datetest 5-14 {datetime('2000-01-01','+0001-02-03 01:02:03')} \
+                                                   {2001-03-04 01:02:03}
+datetest 5-15 {datetime('2000-01-01','+0001-02-03 01:02:03.')} NULL
+datetest 5-16 {datetime('2000-01-01','+0001-02-03 01:02:03.5')} \
+                                                   {2001-03-04 01:02:03}
+datetest 5-17 {datetime('2000-01-01','+0001-02-03 01:02:03.50')} \
+                                                   {2001-03-04 01:02:03}
+datetest 5-18 {datetime('2000-01-01','+0001-02-03 01:02:03.500')} \
+                                                   {2001-03-04 01:02:03}
+datetest 5-19 {datetime('2000-01-01','+0001-02-03 01:02:03.500x')} {NULL}
+datetest 5-20 {datetime('2000-01-01','+0001-02-03 01:02:03.500 x')} {NULL}
+
+unset -nocomplain p1
+unset -nocomplain p2
+set p1 {
+  a   {2000-01-01 00:00:00}
+  b   {2000-01-31 23:59:59}
+  c   {2000-02-01 00:00:00}
+  d   {2000-02-29 23:59:59}
+  e   {2000-03-01 00:00:00}
+  f   {2000-03-31 23:59:59}
+  g   {2000-04-01 00:00:00}
+  h   {2000-04-30 23:59:59}
+  i   {2000-05-01 00:00:00}
+  j   {2000-05-31 23:59:59}
+  k   {2000-06-01 00:00:00}
+  l   {2000-06-30 23:59:59}
+  m   {2000-07-01 00:00:00}
+  n   {2000-07-31 23:59:59}
+  o   {2000-08-01 00:00:00}
+  p   {2000-08-31 23:59:59}
+  q   {2000-09-01 00:00:00}
+  r   {2000-09-30 23:59:59}
+  s   {2000-10-01 00:00:00}
+  t   {2000-10-31 23:59:59}
+  u   {2000-11-01 00:00:00}
+  v   {2000-11-30 23:59:59}
+  w   {2000-12-01 00:00:00}
+  x   {2000-12-31 23:59:59}
+}
+set p2 {
+  A   {2001-01-01 00:00:00}
+  B   {2001-01-31 23:59:59}
+  C   {2001-02-01 00:00:00}
+  D   {2001-02-28 23:59:59}
+  E   {2001-03-01 00:00:00}
+  F   {2001-03-31 23:59:59}
+  G   {2001-04-01 00:00:00}
+  H   {2001-04-30 23:59:59}
+  I   {2001-05-01 00:00:00}
+  J   {2001-05-31 23:59:59}
+  K   {2001-06-01 00:00:00}
+  L   {2001-06-30 23:59:59}
+  M   {2001-07-01 00:00:00}
+  N   {2001-07-31 23:59:59}
+  O   {2001-08-01 00:00:00}
+  P   {2001-08-31 23:59:59}
+  Q   {2001-09-01 00:00:00}
+  R   {2001-09-30 23:59:59}
+  S   {2001-10-01 00:00:00}
+  T   {2001-10-31 23:59:59}
+  U   {2001-11-01 00:00:00}
+  V   {2001-11-30 23:59:59}
+  W   {2001-12-01 00:00:00}
+  X   {2001-12-31 23:59:59}
+}
+
+foreach {x1 d1} $p1 {
+  foreach {x2 d2} $p2 {
+    set r1 [db one {SELECT datetime($d1)}]
+    do_execsql_test timediff-6-$x1$x2 {
+      SELECT datetime($d2, timediff($d1,$d2));
+    } [list $r1]
+    set r2 [db one {SELECT datetime($d2)}]
+    do_execsql_test timediff-6-$x2$x1 {
+      SELECT datetime($d1, timediff($d2,$d1));
+    } [list $r2]
+  }
+}
+
+
+
+finish_test
diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test
index 1f797686bf..5dad781692 100644
--- a/test/tkt-2d1a5c67d.test
+++ b/test/tkt-2d1a5c67d.test
@@ -102,6 +102,7 @@ do_test 3.4 {
   set blobs [list]
   for {set i 1} {$i<100} {incr i} {
     set b [db incrblob -readonly t3 b $i]
+    fconfigure $b -translation binary
     read $b
     lappend blobs $b
   }
diff --git a/test/tkt-8454a207b9.test b/test/tkt-8454a207b9.test
index 20e142057d..42b95c8f53 100644
--- a/test/tkt-8454a207b9.test
+++ b/test/tkt-8454a207b9.test
@@ -49,7 +49,7 @@ do_test tkt-8454a207b9.4 {
     ALTER TABLE t1 ADD COLUMN e DEFAULT -123.0;
     SELECT e, typeof(e) FROM t1;
   }
-} {-123 integer}
+} {-123.0 real}
 do_test tkt-8454a207b9.5 {
   db eval {
     ALTER TABLE t1 ADD COLUMN f DEFAULT -123.5;
diff --git a/test/tkt-94c04eaadb.test b/test/tkt-94c04eaadb.test
deleted file mode 100644
index 9de8aea28d..0000000000
--- a/test/tkt-94c04eaadb.test
+++ /dev/null
@@ -1,72 +0,0 @@
-# 2009 October 19
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# This file implements regression tests for SQLite library.
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-
-if {[info commands sqlite3async_initialize] eq ""} {
-  # The async logic is not built into this system
-  finish_test
-  return
-}
-
-# Create a database.
-do_test tkt-94c94-1.1 {
-  execsql { CREATE TABLE t1(a, b) }
-} {}
-
-# Grow the file to larger than 4096MB (2^32 bytes)
-db close
-if {[catch {fake_big_file 4096 [get_pwd]/test.db} msg]} {
-  puts "**** Unable to create a file larger than 4096 MB. *****"
-  finish_test
-  return
-}
-
-# Switch to async mode.
-sqlite3async_initialize "" 1
-sqlite3 db test.db
-sqlite3 db2 test.db
-
-# Read from and write to the db just past the 4096MB mark.
-#
-do_test tkt-94c94-2.1 {
-  execsql { CREATE TABLE t2(x, y) } db
-} {}
-do_test tkt-94c94-2.2 {
-  execsql { INSERT INTO t2 VALUES(1, 2) } db2
-} {}
-do_test tkt-94c94-2.3 {
-  execsql { SELECT * FROM t2 } db
-} {1 2}
-do_test tkt-94c94-2.4 {
-  sqlite3async_control halt idle
-  sqlite3async_start
-  sqlite3async_wait
-} {}
-do_test tkt-94c94-2.5 {
-  execsql { SELECT * FROM t2 } db
-} {1 2}
-do_test tkt-94c94-2.6 {
-  sqlite3async_start
-  sqlite3async_wait
-} {}
-
-db close
-db2 close
-sqlite3async_start
-sqlite3async_wait
-sqlite3async_control halt never
-sqlite3async_shutdown
-
-finish_test
diff --git a/test/tkt-bd484a090c.test b/test/tkt-bd484a090c.test
index 3d2b599958..7867c8dc97 100644
--- a/test/tkt-bd484a090c.test
+++ b/test/tkt-bd484a090c.test
@@ -30,7 +30,7 @@ do_test 2.1 {
   catchsql { SELECT datetime('now', 'localtime') }
 } {1 {local time unavailable}}
 do_test 2.2 {
-  catchsql { SELECT datetime('now', 'utc') }
+  catchsql { SELECT datetime('2000-01-01', 'utc') }
 } {1 {local time unavailable}}
 
 sqlite3_test_control SQLITE_TESTCTRL_LOCALTIME_FAULT 0
diff --git a/test/tkt-cbd054fa6b.test b/test/tkt-cbd054fa6b.test
index 86248ca21d..435d807873 100644
--- a/test/tkt-cbd054fa6b.test
+++ b/test/tkt-cbd054fa6b.test
@@ -65,7 +65,7 @@ do_test tkt-cbd05-1.2 {
 } {}
 do_test tkt-cbd05-1.3 {
   execsql { 
-    SELECT tbl,idx,group_concat(s(sample),' ') 
+    SELECT tbl,idx,string_agg(s(sample),' ') 
     FROM vvv 
     WHERE idx = 't1_x' 
     GROUP BY tbl,idx
diff --git a/test/tkt3457.test b/test/tkt3457.test
index 24b4f0eac0..17b6c72cd4 100644
--- a/test/tkt3457.test
+++ b/test/tkt3457.test
@@ -15,10 +15,10 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-if {$tcl_platform(platform) != "unix"} {
+if {[llength [info commands test_syscall]]==0} {
   finish_test
   return
-}
+} 
 if {[atomic_batch_write test.db]} {
   finish_test
   return
@@ -58,7 +58,7 @@ do_test tkt3457-1.1 {
   # start of the first journal-header has not been written by SQLite.
   # So write it now.
   set fd [open bak.db-journal a+]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
   seek $fd 0
   puts -nonewline $fd "\xd9\xd5\x05\xf9\x20\xa1\x63\xd7"
   close $fd
diff --git a/test/trace3.test b/test/trace3.test
index e9935acfb8..639aefafa6 100644
--- a/test/trace3.test
+++ b/test/trace3.test
@@ -132,14 +132,27 @@ do_test trace3-4.3 {
   list $stmt [expr {$ns >= 0 && $ns <= 9999999}]; # less than 0.010 seconds
 } {/^-?\d+ 1$/}
 do_test trace3-4.4 {
-  set ::stmtlist(record) {}
-  db trace_v2 trace_v2_record 2
-  execsql {
-    SELECT a, b FROM t1 ORDER BY a;
+  set cnt 0
+  while {1} {
+    set ::stmtlist(record) {}
+    db trace_v2 trace_v2_record 2
+    execsql {
+      SELECT a, b FROM t1 ORDER BY a;
+    }
+    set stmt [lindex [lindex $::stmtlist(record) 0] 0]
+    set ns [lindex [lindex $::stmtlist(record) 0] 1]
+    if {$ns<0 || $ns>9999999} {  #less than 0.010 seconds
+      incr cnt
+      if {$cnt>3} {
+        set res "time out of bounds.  Expected less than 99999999.  Got $ns"
+        break
+      }
+    } else {
+      set res 1
+      break
+    }
   }
-  set stmt [lindex [lindex $::stmtlist(record) 0] 0]
-  set ns [lindex [lindex $::stmtlist(record) 0] 1]
-  list $stmt [expr {$ns >= 0 && $ns <= 9999999}]; # less than 0.010 seconds
+  list $stmt $res
 } {/^-?\d+ 1$/}
 
 do_test trace3-5.1 {
@@ -329,5 +342,21 @@ do_test 12.1.2 {
   sqlite3_finalize $STMT
 } {SQLITE_OK}
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 13.0 {
+  CREATE TABLE T1(a, b);
+  INSERT INTO t1 VALUES(1, 2), (3, 4);
+}
+
+proc trace_callback {args} {}
+db trace_v2 trace_callback profile
+
+do_test 13.1 {
+  db eval { SELECT * FROM t1 } {
+    db trace_v2 "" ""
+  }
+  set {} {}
+} {}
 
 finish_test
diff --git a/test/trigger1.test b/test/trigger1.test
index 6de121fa9c..67943677f9 100644
--- a/test/trigger1.test
+++ b/test/trigger1.test
@@ -751,7 +751,7 @@ do_execsql_test trigger1-18.1 {
   SELECT * FROM t18;
 } {1 3 2}     ;# Not: 1 1001 1000
 
-# 2018-04-26 ticket [https://www.sqlite.org/src/tktview/d85fffd6ffe856092e]
+# 2018-04-26 ticket [https://sqlite.org/src/tktview/d85fffd6ffe856092e]
 # VDBE Program uses an expired value.
 #
 do_execsql_test trigger1-19.0 {
@@ -838,4 +838,17 @@ do_catchsql_test trigger1-23.1 {
   END;
 } {1 {near "#1": syntax error}}
 
+# 2024-05-08 Allow arbitrary expressions as the 2nd argument to RAISE().
+#
+do_catchsql_test trigger1-24.1 {
+  CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN
+    SELECT raise(abort,format('attempt to insert %d where is not a power of 2',new.a))
+     WHERE (new.a & (new.a-1))!=0;
+  END;
+  INSERT INTO t1 VALUES(0),(1),(2),(4),(8),(65536);
+} {0 {}}
+do_catchsql_test trigger1-24.2 {
+  INSERT INTO t1 VALUES(9876);
+} {1 {attempt to insert 9876 where is not a power of 2}}
+
 finish_test
diff --git a/test/trigger2.test b/test/trigger2.test
index 6e007e969a..70d59f3a0b 100644
--- a/test/trigger2.test
+++ b/test/trigger2.test
@@ -790,4 +790,3 @@ do_catchsql_test 11.2 {
 
 
 finish_test
-
diff --git a/test/trigger9.test b/test/trigger9.test
index 6e31d1af97..47940de577 100644
--- a/test/trigger9.test
+++ b/test/trigger9.test
@@ -242,12 +242,27 @@ do_execsql_test 4.1 {
   END;
 }
 
-do_catchsql_test 4.2 {
-  DELETE FROM v1 WHERE rowid=1;
-} {1 {no such column: rowid}}
+ifcapable !allow_rowid_in_view {
+  do_catchsql_test 4.2 {
+    DELETE FROM v1 WHERE rowid=1;
+  } {1 {no such column: rowid}}
 
-do_catchsql_test 4.3 {
-  UPDATE v1 SET a=b WHERE rowid=2;
-} {1 {no such column: rowid}}
+  do_catchsql_test 4.3 {
+    UPDATE v1 SET a=b WHERE rowid=2;
+  } {1 {no such column: rowid}}
+} else {
+  do_execsql_test 4.2a {
+    DELETE FROM log;
+  }
+  do_catchsql_test 4.2 {
+    DELETE FROM v1 WHERE rowid=1;
+  } {0 {}}
+  do_catchsql_test 4.3 {
+    UPDATE v1 SET a=b WHERE rowid=2;
+  } {0 {}}
+  do_execsql_test 4.3b {
+    SELECT * FROM log;
+  }
+}
 
 finish_test
diff --git a/test/triggerG.test b/test/triggerG.test
index 8cf20b5ec9..b078fffb2e 100644
--- a/test/triggerG.test
+++ b/test/triggerG.test
@@ -19,7 +19,7 @@ ifcapable {!trigger} {
 }
 
 # Test cases for ticket 
-# https://www.sqlite.org/src/tktview/06796225f59c057cd120f
+# https://sqlite.org/src/tktview/06796225f59c057cd120f
 #
 # The OP_Once opcode was not working correctly for recursive triggers.
 #
diff --git a/test/types3.test b/test/types3.test
index 807ae84f9d..457ee6d68a 100644
--- a/test/types3.test
+++ b/test/types3.test
@@ -12,18 +12,15 @@
 # of this file is testing the interaction of SQLite manifest types
 # with Tcl dual-representations.
 #
-# $Id: types3.test,v 1.8 2008/04/28 13:02:58 drh Exp $
-#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
 # A variable with only a string representation comes in as TEXT
 do_test types3-1.1 {
-  set V {}
-  append V x
+  set V [format %s xxxxx]
   concat [tcl_variable_type V] [execsql {SELECT typeof(:V)}]
-} {string text}
+} {text}
 
 # A variable with an integer representation comes in as INTEGER
 do_test types3-1.2 {
@@ -96,4 +93,32 @@ do_test types3-2.6 {
   tcl_variable_type V
 } {}
 
+# See https://sqlite.org/forum/forumpost/3776b48e71
+#
+# On a text-affinity comparison of two values where one of
+# the values has both MEM_Str and a numeric type like MEM_Int,
+# make sure that only the MEM_Str representation is used.
+#
+sqlite3_create_function db
+do_execsql_test types3-3.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x TEXT PRIMARY KEY);
+  INSERT INTO t1 VALUES('1');
+  SELECT * FROM t1 WHERE NOT x=upper(1);
+} {}
+do_execsql_test types3-3.2 {
+  SELECT * FROM t1 WHERE NOT x=add_text_type(1);
+} {}
+do_execsql_test types3-3.3 {
+  SELECT * FROM t1 WHERE NOT x=add_int_type('1');
+} {}
+do_execsql_test types3-3.4 {
+  DELETE FROM t1;
+  INSERT INTO t1 VALUES(1.25);
+  SELECT * FROM t1 WHERE NOT x=add_real_type('1.25');
+} {}
+do_execsql_test types3-3.5 {
+  SELECT * FROM t1 WHERE NOT x=add_text_type(1.25);
+} {}
+
 finish_test
diff --git a/test/unhex.test b/test/unhex.test
index f41e906a8f..af2cfae390 100644
--- a/test/unhex.test
+++ b/test/unhex.test
@@ -55,6 +55,8 @@ do_catchsql_test 3.1 {
 #--------------------------------------------------------------------------
 # Test the 2-argument version.
 #
+# Zap global x array set in some previous test.
+if {[array exists x]} {array unset x}
 foreach {tn hex} {
   1 "FFFF  ABCD"
   2 "FFFF ABCD"
@@ -98,5 +100,3 @@ do_execsql_test 6.4.3 { SELECT typeof(unhex('1234', NULL)) } {null}
 
 
 finish_test
-
-
diff --git a/test/unionall.test b/test/unionall.test
index 32fc76543a..9057199070 100644
--- a/test/unionall.test
+++ b/test/unionall.test
@@ -346,6 +346,13 @@ do_execsql_test 5.10 {
 do_execsql_test 5.20 {
   SELECT *, '+' FROM t1 LEFT JOIN t3 ON (a NOT IN(SELECT v FROM t1 LEFT JOIN t2 ON (a=k))=k);
 } {0 {} {} {} + 1 one {} {} + 2 two {} {} + 5 five {} {} + 3 three {} {} + 6 six {} {} +}
+ifcapable vtab {
+do_catchsql_test 5.30 {
+  SELECT * FROM (t1 NATURAL JOIN pragma_table_xinfo('t1_a') NATURAL JOIN t3) t1
+                NATURAL JOIN t2 NATURAL JOIN t3
+   WHERE rowid ISNULL>0 AND 0%y;
+} {1 {ambiguous column name: rowid}}
+}
 
 reset_db
 do_execsql_test 6.0 {
diff --git a/test/unixexcl.test b/test/unixexcl.test
index 8e9c4644d1..c24945e5e3 100644
--- a/test/unixexcl.test
+++ b/test/unixexcl.test
@@ -18,7 +18,7 @@ source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 source $testdir/malloc_common.tcl
 
-if {$::tcl_platform(platform)!="unix" || [info commands test_syscall]==""} {
+if {[llength [info commands test_syscall]]==0} {
   finish_test
   return
 } 
diff --git a/test/update.test b/test/update.test
index 47b6029bee..0a380fa030 100644
--- a/test/update.test
+++ b/test/update.test
@@ -616,7 +616,7 @@ do_test update-14.4 {
 
 } ;# ifcapable {trigger}
 
-# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29
+# Ticket [https://sqlite.org/src/tktview/43107840f1c02] on 2014-10-29
 # An assertion fault on UPDATE
 #
 ifcapable altertable {
@@ -703,7 +703,7 @@ do_execsql_test update-19.10 {
   SELECT * FROM t1;
 } {2 2}
 
-# 2019-12-29 ticket https://www.sqlite.org/src/info/314cc133e5ada126
+# 2019-12-29 ticket https://sqlite.org/src/info/314cc133e5ada126
 # REPLACE conflict resolution during an UPDATE causes a DELETE trigger 
 # to fire.  If that DELETE trigger subsequently modifies the row
 # being updated, bad things can happen.  Prevent this by prohibiting
@@ -711,8 +711,8 @@ do_execsql_test update-19.10 {
 # REPLACE conflict resolution on the UPDATE.
 #
 # See also tickets:
-#   https://www.sqlite.org/src/info/c1e19e12046d23fe 2019-10-25
-#   https://www.sqlite.org/src/info/a8a4847a2d96f5de 2019-10-16 
+#   https://sqlite.org/src/info/c1e19e12046d23fe 2019-10-25
+#   https://sqlite.org/src/info/a8a4847a2d96f5de 2019-10-16 
 #
 reset_db
 do_execsql_test update-20.10 {
@@ -764,5 +764,27 @@ do_execsql_test update-21.4 {
   SELECT * FROM t1 ORDER BY vkey, c5;
   ROLLBACK;
 } {6 -54 100 NULL}
+# Follow-up on 2023-07-31 (forum post https://sqlite.org/forum/forumpost/8ab195fd44e75ed0):
+# Only disable one-pass if the subquery is in the WHERE clause.  The SET expressions
+# do not count.
+do_execsql_test update-21.11 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT);
+  CREATE TABLE t2(d INT);
+}
+do_eqp_test update-21.12 {
+  WITH t3(x,y) AS (SELECT d, row_number()OVER() FROM t2)
+    UPDATE t1 SET b=(SELECT y FROM t3 WHERE t1.a=t3.x);
+} {
+  QUERY PLAN
+  |--SCAN t1
+  `--CORRELATED SCALAR SUBQUERY xxxxxx
+     |--CO-ROUTINE t3
+     |  |--CO-ROUTINE (subquery-xxxxxx)
+     |  |  `--SCAN t2
+     |  `--SCAN (subquery-xxxxxx)
+     |--BLOOM FILTER ON t3 (x=?)
+     `--SEARCH t3 USING AUTOMATIC COVERING INDEX (x=?)
+}
 
 finish_test
diff --git a/test/upfrom4.test b/test/upfrom4.test
index b20e9638aa..2228236280 100644
--- a/test/upfrom4.test
+++ b/test/upfrom4.test
@@ -127,4 +127,30 @@ do_execsql_test 400 {
   1000 -    -   -
 }
 
+# Forum post https://sqlite.org/forum/forumpost/36ff78b2a3
+#
+ifcapable update_delete_limit {
+  reset_db
+  do_execsql_test 500 {
+    CREATE TABLE t1(abc INT, def INT);  
+    INSERT INTO t1 VALUES(0,0);
+    INSERT INTO t1 VALUES(0,0);
+    INSERT INTO t1 VALUES(0,0);
+    CREATE TABLE dual(dummy TEXT);  
+    INSERT INTO dual(dummy) VALUES('X');
+  } {}
+  
+  do_execsql_test 510 {
+    UPDATE t1
+      SET (abc, def)=(SELECT  x, 123)
+      FROM dual LEFT JOIN (SELECT 789 AS 'x' FROM dual) AS d2
+      LIMIT 2
+  }
+  
+  do_execsql_test 520 {
+    SELECT * FROM t1
+  } {789 123 789 123 0 0}
+}
+
+
 finish_test
diff --git a/test/upsert1.test b/test/upsert1.test
index a321d6171d..49168f840b 100644
--- a/test/upsert1.test
+++ b/test/upsert1.test
@@ -129,7 +129,7 @@ do_execsql_test upsert1-610 {
 } {ok}
 
 # 2018-08-14
-# Ticket https://www.sqlite.org/src/info/908f001483982c43
+# Ticket https://sqlite.org/src/info/908f001483982c43
 # If there are multiple uniqueness contraints, the UPSERT should fire
 # if the one constraint it targets fails, regardless of whether or not
 # the other constraints pass or fail.  In other words, the UPSERT constraint
@@ -255,4 +255,41 @@ do_execsql_test upsert1-1100 {
   SELECT * FROM t1;
 } {1 22}
 
+# 2023-08-17 dbsqlfuzz 9983e2c77634a8ccf33b5c91fa9982599de5f9e9
+# Bound parameters in the ON CONFLICT clause of an UPSERT.
+#
+reset_db
+do_execsql_test upsert1-1200 {
+  CREATE TABLE t1(a INT, b INT);
+  CREATE UNIQUE INDEX t1x ON t1(b+3);
+}
+sqlite3_db_config db ENABLE_QPSG 1
+do_catchsql_test upsert1-1210 {
+  INSERT INTO t1(a,b) VALUES(1,2) ON CONFLICT(b+?1) DO NOTHING;
+} {1 {ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint}}
+
+# 2024-04-11 https://sqlite.org/forum/forumpost/284955a3cd454a15
+# Incorrect value passed into a trigger that fires as the result of
+# an upsert.
+#
+reset_db
+do_execsql_test upsert1-1300 {
+  CREATE TABLE t1(x INT, y TEXT);
+  INSERT INTO t1 VALUES
+    (11, printf('%.9000c','a')),
+    (11, printf('%.9000c','a')),
+    (33, printf('%.9000c','b')),
+    (33, printf('%.9000c','b'));
+  CREATE TABLE t2(x INT UNIQUE, y TEXT);
+  CREATE TRIGGER r1 BEFORE UPDATE ON t2 BEGIN
+    SELECT raise(ABORT,'Incorrect old.y value passed to trigger!')
+     WHERE old.y != new.y;
+    /* ^^^ This trigger will fire and cause the ABORT if the problem has
+    ** not been fixed, or if there is a regression. */
+  END;
+  INSERT INTO t2(x, y) SELECT x, y FROM t1
+   WHERE true
+   ON CONFLICT (x) DO UPDATE SET y = excluded.y;
+} {}
+
 finish_test
diff --git a/test/upsert5.test b/test/upsert5.test
index 3161abf15e..e56e71d4b9 100644
--- a/test/upsert5.test
+++ b/test/upsert5.test
@@ -408,4 +408,46 @@ do_catchsql_test 2.1 {
 
 } {1 {no such table: nosuchtable}}
 
+# 2024-03-08 https://sqlite.org/forum/forumpost/919c6579c8
+# A redundant ON CONFLICT clause in an upsert can lead to
+# index corruption.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(aa INTEGER PRIMARY KEY, bb INT);
+  INSERT INTO t1 VALUES(11,22);
+  CREATE UNIQUE INDEX t1bb ON t1(bb);
+  REPLACE INTO t1 VALUES(11,33)
+    ON CONFLICT(bb) DO UPDATE SET aa = 44
+    ON CONFLICT(bb) DO UPDATE SET aa = 44;
+  PRAGMA integrity_check;
+} {ok}
+do_execsql_test 3.1 {
+  SELECT * FROM t1 NOT INDEXED;
+} {11 33}
+do_execsql_test 3.2 {
+  SELECT * FROM t1 INDEXED BY t1bb;
+} {11 33}
+do_execsql_test 3.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(aa INTEGER PRIMARY KEY, bb INT, cc INT);
+  INSERT INTO t1 VALUES(10,21,32),(11,22,33),(12,23,34);
+  CREATE UNIQUE INDEX t1bb ON t1(bb);
+  CREATE UNIQUE INDEX t1cc ON t1(cc);
+  REPLACE INTO t1 VALUES(11,44,55)
+    ON CONFLICT(bb) DO UPDATE SET aa = 99
+    ON CONFLICT(cc) DO UPDATE SET aa = 99
+    ON CONFLICT(bb) DO UPDATE SET aa = 99;
+  PRAGMA integrity_check;
+} {ok}
+do_execsql_test 3.4 {
+  SELECT * FROM t1 NOT INDEXED ORDER BY +aa;
+} {10 21 32 11 44 55 12 23 34}
+do_execsql_test 3.5 {
+  SELECT * FROM t1 INDEXED BY t1bb ORDER BY +aa;
+} {10 21 32 11 44 55 12 23 34}
+do_execsql_test 3.6 {
+  SELECT * FROM t1 INDEXED BY t1cc ORDER BY +aa;
+} {10 21 32 11 44 55 12 23 34}
+
 finish_test
diff --git a/test/uri.test b/test/uri.test
index 2b388c4007..74da225acc 100644
--- a/test/uri.test
+++ b/test/uri.test
@@ -59,7 +59,7 @@ foreach {tn uri file} {
     if {[string first %00 $uri]>=0} continue
   }
 
-  if {$tcl_platform(platform)=="windows"} {
+  if {$tcl_platform(platform) eq "windows"} {
     #
     # NOTE: Due to limits on legal characters for file names imposed by
     #       Windows, we must skip the final two tests here (i.e. the
@@ -306,7 +306,7 @@ foreach {tn uri res} {
   6     "file://x/PWD/test.db"           {invalid uri authority: x}
 } {
 
-  if {$tcl_platform(platform)=="windows"} {
+  if {$tcl_platform(platform) eq "windows"} {
     set uri  [string map [list PWD [string range [get_pwd] 3 end]] $uri]
   } else {
     set uri  [string map [list PWD [string range [get_pwd] 1 end]] $uri]
diff --git a/test/userauth01.test b/test/userauth01.test
deleted file mode 100644
index 644937b192..0000000000
--- a/test/userauth01.test
+++ /dev/null
@@ -1,257 +0,0 @@
-# 2014-09-10
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-# 
-# This file implements tests of the SQLITE_USER_AUTHENTICATION extension.
-#
-
-set testdir [file dirname $argv0]
-source $testdir/tester.tcl
-set testprefix userauth01
-
-ifcapable !userauth {
-  finish_test
-  return
-}
-
-# Create a no-authentication-required database
-#
-do_execsql_test userauth01-1.0 {
-  CREATE TABLE t1(x);
-  INSERT INTO t1 VALUES(1),(2.5),('three'),(x'4444'),(NULL);
-  SELECT quote(x) FROM t1 ORDER BY x;
-  SELECT name FROM sqlite_master;
-} {NULL 1 2.5 'three' X'4444' t1}
-
-# Calling sqlite3_user_authenticate() on a no-authentication-required
-# database connection is a harmless no-op.  
-#
-do_test userauth01-1.1 {
-  sqlite3_user_authenticate db alice pw-4-alice
-  execsql {
-    SELECT quote(x) FROM t1 ORDER BY x;
-    SELECT name FROM sqlite_master;
-  }
-} {NULL 1 2.5 'three' X'4444' t1}
-
-# If sqlite3_user_add(D,U,P,N,A) is called on a no-authentication-required
-# database and A is false, then the call fails with an SQLITE_AUTH error.
-#
-do_test userauth01-1.2 {
-  sqlite3_user_add db bob pw-4-bob 0
-} {SQLITE_AUTH}
-do_test userauth01-1.3 {
-  execsql {
-    SELECT quote(x) FROM t1 ORDER BY x;
-    SELECT name FROM sqlite_master;
-  }
-} {NULL 1 2.5 'three' X'4444' t1}
-
-# When called on a no-authentication-required
-# database and when A is true, the sqlite3_user_add(D,U,P,N,A) routine
-# converts the database into an authentication-required database and
-# logs the database connection D in using user U with password P,N.
-#  
-do_test userauth01-1.4 {
-  sqlite3_user_add db alice pw-4-alice 1
-} {SQLITE_OK}
-do_test userauth01-1.5 {
-  execsql {
-    SELECT quote(x) FROM t1 ORDER BY x;
-    SELECT uname, isadmin FROM sqlite_user ORDER BY uname;
-    SELECT name FROM sqlite_master ORDER BY name;
-  }
-} {NULL 1 2.5 'three' X'4444' alice 1 sqlite_user t1}
-
-# The sqlite3_user_add() interface can be used (by an admin user only)
-# to create a new user.
-#
-do_test userauth01-1.6 {
-  sqlite3_user_add db bob pw-4-bob 0
-  sqlite3_user_add db cindy pw-4-cindy 0
-  sqlite3_user_add db david pw-4-david 0
-  execsql {
-    SELECT uname, isadmin FROM sqlite_user ORDER BY uname;
-  }
-} {alice 1 bob 0 cindy 0 david 0}
-
-# The sqlite_user table is inaccessible (unreadable and unwriteable) to
-# non-admin users and is read-only for admin users.  However, if the same
-#
-do_test userauth01-1.7 {
-  sqlite3 db2 test.db
-  sqlite3_user_authenticate db2 cindy pw-4-cindy
-  db2 eval {
-    SELECT quote(x) FROM t1 ORDER BY x;
-    SELECT name FROM sqlite_master ORDER BY name;
-  }
-} {NULL 1 2.5 'three' X'4444' sqlite_user t1}
-do_test userauth01-1.8 {
-  catchsql {
-    SELECT uname, isadmin FROM sqlite_user ORDER BY uname;
-  } db2
-} {1 {no such table: sqlite_user}}
-
-# Any user can change their own password.  
-#
-do_test userauth01-1.9 {
-  sqlite3_user_change db2 cindy xyzzy-cindy 0
-} {SQLITE_OK}
-do_test userauth01-1.10 {
-  sqlite3_user_authenticate db2 cindy pw-4-cindy
-} {SQLITE_AUTH}
-do_test userauth01-1.11 {
-  sqlite3_user_authenticate db2 cindy xyzzy-cindy
-} {SQLITE_OK}
-do_test userauth01-1.12 {
-  sqlite3_user_change db alice xyzzy-alice 1
-} {SQLITE_OK}
-do_test userauth01-1.13 {
-  sqlite3_user_authenticate db alice pw-4-alice
-} {SQLITE_AUTH}
-do_test userauth01-1.14 {
-  sqlite3_user_authenticate db alice xyzzy-alice
-} {SQLITE_OK}
-
-# No user may change their own admin privilege setting.
-#
-do_test userauth01-1.15 {
-  sqlite3_user_change db alice xyzzy-alice 0
-} {SQLITE_AUTH}
-do_test userauth01-1.16 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 bob 0 cindy 0 david 0}
-do_test userauth01-1.17 {
-  sqlite3_user_change db2 cindy xyzzy-cindy 1
-} {SQLITE_AUTH}
-do_test userauth01-1.18 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 bob 0 cindy 0 david 0}
-
-# The sqlite3_user_change() interface can be used to change a users
-# login credentials or admin privilege.
-#
-do_test userauth01-1.20 {
-  sqlite3_user_change db david xyzzy-david 1
-} {SQLITE_OK}
-do_test userauth01-1.21 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 bob 0 cindy 0 david 1}
-do_test userauth01-1.22 {
-  sqlite3_user_authenticate db2 david xyzzy-david
-} {SQLITE_OK}
-do_test userauth01-1.23 {
-  db2 eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 bob 0 cindy 0 david 1}
-do_test userauth01-1.24 {
-  sqlite3_user_change db david pw-4-david 0
-} {SQLITE_OK}
-do_test userauth01-1.25 {
-  sqlite3_user_authenticate db2 david pw-4-david
-} {SQLITE_OK}
-do_test userauth01-1.26 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 bob 0 cindy 0 david 0}
-do_test userauth01-1.27 {
-  catchsql {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} db2
-} {1 {no such table: sqlite_user}}
-
-# Only an admin user can change another users login
-# credentials or admin privilege setting.
-#
-do_test userauth01-1.30 {
-  sqlite3_user_change db2 bob xyzzy-bob 1
-} {SQLITE_AUTH}
-do_test userauth01-1.31 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 bob 0 cindy 0 david 0}
-
-# The sqlite3_user_delete() interface can be used (by an admin user only)
-# to delete a user.
-#
-do_test userauth01-1.40 {
-  sqlite3_user_delete db bob
-} {SQLITE_OK}
-do_test userauth01-1.41 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 cindy 0 david 0}
-do_test userauth01-1.42 {
-  sqlite3_user_delete db2 cindy
-} {SQLITE_AUTH}
-do_test userauth01-1.43 {
-  sqlite3_user_delete db2 alice
-} {SQLITE_AUTH}
-do_test userauth01-1.44 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 cindy 0 david 0}
-
-# The currently logged-in user cannot be deleted
-#
-do_test userauth01-1.50 {
-  sqlite3_user_delete db alice
-} {SQLITE_AUTH}
-do_test userauth01-1.51 {
-  db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname}
-} {alice 1 cindy 0 david 0}
-
-# When ATTACH-ing new database files to a connection, each newly attached
-# database that is an authentication-required database is checked using
-# the same username and password as supplied to the main database.  If that
-# check fails, then the ATTACH command fails with an SQLITE_AUTH error.
-#
-do_test userauth01-1.60 {
-  forcedelete test3.db
-  sqlite3 db3 test3.db
-  sqlite3_user_add db3 alice xyzzy-alice 1
-} {SQLITE_OK}
-do_test userauth01-1.61 {
-  db3 eval {
-    CREATE TABLE t3(a,b,c); INSERT INTO t3 VALUES(1,2,3);
-    SELECT * FROM t3;
-  }
-} {1 2 3}
-do_test userauth01-1.62 {
-  db eval {
-    ATTACH 'test3.db' AS aux;
-    SELECT * FROM t1, t3 ORDER BY x LIMIT 1;
-    DETACH aux;
-  }
-} {{} 1 2 3}
-do_test userauth01-1.63 {
-  sqlite3_user_change db alice pw-4-alice 1
-  sqlite3_user_authenticate db alice pw-4-alice
-  catchsql {
-    ATTACH 'test3.db' AS aux;
-  }
-} {1 {unable to open database: test3.db}}
-do_test userauth01-1.64 {
-  sqlite3_extended_errcode db
-} {SQLITE_AUTH}
-do_test userauth01-1.65 {
-  db eval {PRAGMA database_list}
-} {~/test3.db/}
-
-# The sqlite3_set_authorizer() callback is modified to take a 7th parameter
-# which is the username of the currently logged in user, or NULL for a
-# no-authentication-required database.
-#
-proc auth {args} {
-  lappend ::authargs $args
-  return SQLITE_OK
-}
-do_test authuser01-2.1 {
-  unset -nocomplain ::authargs
-  db auth auth
-  db eval {SELECT x FROM t1}
-  set ::authargs
-} {/SQLITE_SELECT {} {} {} {} alice/}  
-
-
-finish_test
diff --git a/test/vacuum-into.test b/test/vacuum-into.test
index 98692a108a..c041ebe7a7 100644
--- a/test/vacuum-into.test
+++ b/test/vacuum-into.test
@@ -26,13 +26,36 @@ ifcapable {!vacuum} {
 
 forcedelete out.db
 do_execsql_test vacuum-into-100 {
-  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  CREATE TABLE t1(
+    a INTEGER PRIMARY KEY,
+    b ANY,
+    c INT AS (b+1),                          --- See "2024-04-09" block
+    CHECK( typeof(b)!='integer' OR b>a-5 )   --- comment below
+  );
   WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
   INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c;
   CREATE INDEX t1b ON t1(b);
   DELETE FROM t1 WHERE a%2;
   SELECT count(*), sum(a), sum(length(b)) FROM t1;
 } {50 2550 30000}
+
+# Update 2024-04-09 for forum post eec177d68fe7fa2c.
+#
+# VACUUM INTO is sensitive to tables holding both generated columns
+# and CHECK constraints.  
+#
+# CHECK constraints are ignored for read-only databases in order to save
+# memory (see check-in 34ddf02d3d21151b on 2014-05-21).  But the xfer
+# optimization normally only works if CHECK constraints match between the
+# source and destination tables.  So the xfer optimization was not
+# working for VACUUM INTO when the source was a read-only database and the
+# table held CHECK constraints.  But if the table has generated columns,
+# then the xfer optimization is required or else VACUUM will raise an
+# error.
+#
+# Fix this by ignoring CHECK constraints when determining whether or not
+# the xfer optimization can run while doing VACUUM.
+
 do_execsql_test vacuum-into-110 {
   VACUUM main INTO 'out.db';
 } {}
@@ -88,11 +111,21 @@ do_catchsql_test vacuum-into-420 {
 
 # The ability to VACUUM INTO a read-only database
 db close
+if {$tcl_platform(platform) eq "windows"} {
+  file attributes test.db -readonly 1
+} else {
+  file attributes test.db -permissions 292  ;# 292 == 0444
+}
 sqlite3 db test.db -readonly 1
 forcedelete test.db2
 do_execsql_test vacuum-into-500 {
   VACUUM INTO 'test.db2';
 }
+if {$tcl_platform(platform) eq "windows"} {
+  file attributes test.db -readonly 0
+} else {
+  file attributes test.db -permissions 420   ;# 420 = 0644
+}
 sqlite3 db2 test.db2
 do_test vacuum-into-510 {
   db2 eval {SELECT name FROM sqlite_master ORDER BY 1}
@@ -185,6 +218,3 @@ tvfs delete
 
 
 finish_test
-
-
-
diff --git a/test/vacuum.test b/test/vacuum.test
index 57429c29ea..82dd00d09c 100644
--- a/test/vacuum.test
+++ b/test/vacuum.test
@@ -401,4 +401,25 @@ do_test vacuum-10.1 {
 } {}
 do_test vacuum-10.2 { execsql VACUUM } {}
 
+# Verify that VACUUM still works if ATTACH is disabled.
+#
+do_execsql_test vacuum-11.1 {
+  PRAGMA page_size=1024;
+  VACUUM;
+  PRAGMA page_size;
+} {1024}
+sqlite3_db_config db ATTACH_CREATE 0
+do_execsql_test vacuum-11.2 {
+  PRAGMA page_size=2048;
+  VACUUM;
+  PRAGMA page_size;
+} {2048}
+sqlite3_db_config db ATTACH_CREATE 1
+sqlite3_db_config db ATTACH_WRITE 0
+do_execsql_test vacuum-11.3 {
+  PRAGMA page_size=4096;
+  VACUUM;
+  PRAGMA page_size;
+} {4096}
+
 finish_test
diff --git a/test/values.test b/test/values.test
new file mode 100644
index 0000000000..c3c52ceb1e
--- /dev/null
+++ b/test/values.test
@@ -0,0 +1,715 @@
+# 2024 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix values
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, c);
+}
+
+
+explain_i {
+  INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4);
+}
+do_execsql_test 1.1.1 {
+  INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4);
+}
+do_execsql_test 1.1.2 {
+  SELECT * FROM x1;
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+}
+
+do_execsql_test 1.2.0 {
+  DELETE FROM x1
+}
+do_execsql_test 1.2.1 {
+  INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3) UNION ALL SELECT 4, 4, 4;
+  SELECT * FROM x1;
+} {1 1 1  2 2 2   3 3 3  4 4 4}
+
+sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 4
+
+do_execsql_test 1.2.2 {
+  DELETE FROM x1;
+  INSERT INTO x1 
+  VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, 5) 
+  UNION ALL SELECT 6, 6, 6;
+  SELECT * FROM x1;
+} {1 1 1  2 2 2   3 3 3  4 4 4  5 5 5  6 6 6}
+
+do_execsql_test 1.2.3 {
+  DELETE FROM x1;
+  INSERT INTO x1 
+  VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4)
+  UNION ALL SELECT 6, 6, 6;
+  SELECT * FROM x1;
+} {1 1 1  2 2 2   3 3 3  4 4 4  6 6 6}
+
+do_execsql_test 1.2.4 {
+  DELETE FROM x1;
+  INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3) UNION ALL SELECT 6, 6, 6;
+  SELECT * FROM x1;
+} {
+ 1 1 1
+ 2 2 2
+ 3 3 3
+ 6 6 6
+}
+
+set a 4
+set b 5
+set c 6
+do_execsql_test 1.2.5 {
+  DELETE FROM x1;
+  INSERT INTO x1 
+  VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), 
+        (4, 4, $a), (5, 5, $b), (6, 6, $c)
+}
+
+do_execsql_test 1.2.6 {
+  SELECT * FROM x1;
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+  5 5 5
+  6 6 6
+}
+
+#-------------------------------------------------------------------------
+# SQLITE_LIMIT_COMPOUND_SELECT set to 0.
+#
+reset_db
+
+do_execsql_test 2.0 {
+  CREATE TABLE x1(a, b, c);
+}
+
+sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 3
+
+do_catchsql_test 2.1.1 {
+  INSERT INTO x1 VALUES
+      (1, 1, 1), 
+      (2, 2, 2), 
+      (3, 3, 3), 
+      (4, 4, 4), 
+      (5, 5, 5), 
+      (6, 6, 6), 
+      (7, 7, 7), 
+      (8, 8, 8), 
+      (9, 9, 9), 
+      (10, 10, 10, 10)
+} {1 {all VALUES must have the same number of terms}}
+
+do_catchsql_test 2.1.2 {
+  INSERT INTO x1 VALUES
+      (1, 1, 1), 
+      (2, 2, 2, 2), 
+      (3, 3, 3), 
+      (4, 4, 4), 
+      (5, 5, 5), 
+      (6, 6, 6), 
+      (7, 7, 7), 
+      (8, 8, 8), 
+      (9, 9, 9), 
+      (10, 10, 10)
+} {1 {all VALUES must have the same number of terms}}
+
+sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 0
+
+do_execsql_test 2.2 {
+  INSERT INTO x1 VALUES
+      (1, 1, 1), 
+      (2, 2, 2), 
+      (3, 3, 3), 
+      (4, 4, 4), 
+      (5, 5, 5), 
+      (6, 6, 6), 
+      (7, 7, 7), 
+      (8, 8, 8), 
+      (9, 9, 9), 
+      (10, 10, 10)
+} {}
+do_execsql_test 2.3 {
+  INSERT INTO x1 VALUES
+      (1, 1, 1), 
+      (2, 2, 2), 
+      (3, 3, 3), 
+      (4, 4, 4), 
+      (5, 5, 5), 
+      (6, 6, 6), 
+      (7, 7, 7), 
+      (8, 8, 8), 
+      (9, 9, 9), 
+      (10, 10, 10)
+      UNION ALL 
+      SELECT 5, 12, 12
+      ORDER BY 1
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 3.0 {
+  CREATE TABLE y1(x, y);
+}
+
+do_execsql_test 3.1.1 {
+  DELETE FROM y1;
+  INSERT INTO y1 VALUES(1, 2), (3, 4), (row_number() OVER (), 5);
+}
+do_execsql_test 3.1.2 {
+  SELECT * FROM y1;
+} {1 2  3 4  1 5}
+do_execsql_test 3.2.1 {
+  DELETE FROM y1;
+  INSERT INTO y1 VALUES(1, 2), (3, 4), (row_number() OVER (), 6)
+    , (row_number() OVER (), 7)
+}
+do_execsql_test 3.1.2 {
+  SELECT * FROM y1;
+} {1 2  3 4  1 6  1 7}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 4.0 {
+  CREATE TABLE x1(a PRIMARY KEY, b) WITHOUT ROWID;
+}
+
+foreach {tn iLimit} {1 0    2 3} { 
+  sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT $iLimit
+
+  do_execsql_test 4.1.1 {
+    DELETE FROM x1;
+    INSERT INTO x1 VALUES
+        (1, 1),
+        (2, (SELECT * FROM  (VALUES('a'), ('b'), ('c'), ('d')) ))
+  }
+  do_execsql_test 4.1.2 {
+    SELECT * FROM x1
+  } {1 1 2 a}
+
+  do_execsql_test 4.2.1 {
+    DELETE FROM x1;
+    INSERT INTO x1 VALUES
+        (1, 1),
+        (2, 2),
+        (3, 3),
+        (4, 4),
+        (5, (SELECT * FROM  (VALUES('a'), ('b'), ('c'), ('d')) ))
+  }
+  do_execsql_test 4.2.2 {
+    SELECT * FROM x1
+  } {1 1 2 2 3 3 4 4 5 a}
+
+  do_execsql_test 4.3.1 {
+    DELETE FROM x1;
+    INSERT INTO x1 VALUES
+        (1, (SELECT * FROM  (VALUES('a'), ('b'), ('c'), ('d'), ('e')) ))
+  }
+  do_execsql_test 4.3.2 {
+    SELECT * FROM x1
+  } {1 a}
+}
+
+#------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 5.0 {
+  CREATE VIEW v1 AS VALUES(1, 2, 3), (4, 5, 6), (7, 8, 9);
+}
+do_execsql_test 5.1 {
+  SELECT * FROM v1
+} {1 2 3 4 5 6 7 8 9}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1), (2);
+}
+
+do_execsql_test 6.1 {
+  SELECT ( VALUES( x ), ( x ) ) FROM t1;
+} {1 2}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES('x'), ('y');
+}
+
+do_execsql_test 6.1 {
+  SELECT * FROM t1, (VALUES(1), (2))
+} {x 1 x 2 y 1 y 2}
+
+do_execsql_test 6.2 {
+  VALUES(CAST(44 AS REAL)),(55);
+} {44.0 55}
+
+#------------------------------------------------------------------------
+do_execsql_test 7.1 {
+  WITH x1(a, b) AS (
+    VALUES(1, 2), ('a', 'b')
+  )
+  SELECT * FROM x1 one, x1 two
+} {
+  1 2  1 2
+  1 2  a b
+  a b  1 2
+  a b  a b
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+set VVV {
+  ( VALUES('a', 'b'), ('c', 'd'), (123, NULL) )
+}
+set VVV2 {
+  ( 
+  SELECT 'a' AS column1, 'b' AS column2 
+  UNION ALL SELECT 'c', 'd' UNION ALL SELECT 123, NULL
+  )
+}
+
+do_execsql_test 8.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES('d'), (NULL), (123)
+}
+foreach {tn q res} {
+  1 "SELECT * FROM t1 LEFT JOIN VVV" {
+    d a b   d c d   d 123 {}
+    {} a b   {} c d   {} 123 {}
+    123 a b   123 c d   123 123 {}
+  }
+
+  2 "SELECT * FROM t1 LEFT JOIN VVV ON (column1=x)" {
+    d {} {}
+    {} {} {}
+    123 123 {}
+  }
+
+  3 "SELECT * FROM t1 RIGHT JOIN VVV" {
+    d a b   d c d   d 123 {}
+    {} a b   {} c d   {} 123 {}
+    123 a b   123 c d   123 123 {}
+  }
+
+  4 "SELECT * FROM t1 RIGHT JOIN VVV ON (column1=x)" {
+    123 123 {}
+    {} a b
+    {} c d
+  }
+
+  5 "SELECT * FROM t1 FULL OUTER JOIN VVV ON (column1=x)" {
+    d {} {}
+    {} {} {}
+    123 123 {}
+    {} a b
+    {} c d
+  }
+
+  6 "SELECT count(*) FROM VVV" { 3 }
+
+  7 "SELECT (SELECT column1 FROM VVV)" { a }
+
+  8 "SELECT * FROM VVV UNION ALL SELECT * FROM VVV" {
+    a b c d 123 {}
+    a b c d 123 {}
+  }
+
+  9 "SELECT * FROM VVV INTERSECT SELECT * FROM VVV" {
+    123 {} a b c d 
+  }
+
+  10 "SELECT * FROM VVV eXCEPT SELECT * FROM VVV" { }
+
+  11 "SELECT * FROM VVV eXCEPT SELECT 'a', 'b'" { 123 {} c d }
+
+} {
+  set q1 [string map [list VVV $VVV] $q]
+  set q2 [string map [list VVV $VVV2] $q]
+  set q3 "WITH VVV AS $VVV $q"
+
+  do_execsql_test 8.1.$tn.1 $q1 $res
+  do_execsql_test 8.1.$tn.2 $q2 $res
+  do_execsql_test 8.1.$tn.3 $q3 $res
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 9.1 {
+  VALUES(456), (123), (NULL) UNION ALL SELECT 122 ORDER BY 1
+} { {} 122 123 456 }
+
+do_execsql_test 9.2 {
+  VALUES (1, 2), (3, 4), (
+    ( SELECT column1 FROM ( VALUES (5, 6), (7, 8) ) ),
+    ( SELECT max(column2) FROM ( VALUES (5, 1), (7, 6) ) )
+  )
+} { 1 2 3 4 5 6 }
+
+do_execsql_test 10.1 {
+  CREATE TABLE a2(a, b, c DEFAULT 'xyz');
+}
+do_execsql_test 10.2 {
+  INSERT INTO a2(a) VALUES(3),(4);
+}
+
+#-------------------------------------------------------------------------
+reset_db
+ifcapable fts3 {
+  do_execsql_test 11.0 {
+    CREATE VIRTUAL TABLE ft USING fts3(x);
+  }
+  do_execsql_test 11.1 {
+    INSERT INTO ft VALUES('one'), ('two');
+  }
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 12.0 {
+  CREATE TABLE t1(a, b);
+}
+do_execsql_test 12.1 {
+  INSERT INTO t1 SELECT 1, 2 UNION ALL VALUES(3, 4), (5, 6);
+}
+do_execsql_test 12.2 {
+  SELECT * FROM t1
+} {1 2 3 4 5 6}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 13.0 {
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES('xyz');
+
+  SELECT (
+      VALUES( (max(substr('abc', 1, 1), x)) ),
+      (123),
+      (456)
+      )
+  FROM t1;
+} {xyz}
+
+do_catchsql_test 13.1 {
+  VALUES(300), (zeroblob(300) OVER win);
+} {1 {zeroblob() may not be used as a window function}}
+
+#--------------------------------------------------------------------------
+reset_db
+do_execsql_test 14.1 {
+  PRAGMA encoding = utf16;
+  CREATE TABLE t1(a, b);
+} {}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 14.2 {
+  INSERT INTO t1 VALUES
+    (17, 'craft'),
+    (16, 'urtlek' IN(1,2,3));
+}
+
+#--------------------------------------------------------------------------
+#
+reset_db
+do_eqp_test 15.1 {
+  VALUES(1),(2),(3),(4),(5);
+} {
+  QUERY PLAN
+  `--SCAN 5-ROW VALUES CLAUSE
+}
+do_execsql_test 15.2 {
+  CREATE TABLE t1(a,b);
+}
+do_eqp_test 15.3 {
+  INSERT INTO t1 VALUES
+    (1,2),(3,4),(7,8);
+} {
+  QUERY PLAN
+  `--SCAN 3-ROW VALUES CLAUSE
+}
+do_eqp_test 15.4 {
+  INSERT INTO t1 VALUES
+    (1,2),(3,4),(7,8),
+    (5,row_number()OVER());
+} {
+  QUERY PLAN
+  `--COMPOUND QUERY
+     |--LEFT-MOST SUBQUERY
+     |  `--SCAN 3-ROW VALUES CLAUSE
+     `--UNION ALL
+        |--CO-ROUTINE (subquery-xxxxxx)
+        |  `--SCAN CONSTANT ROW
+        `--SCAN (subquery-xxxxxx)
+}
+do_eqp_test 15.5 {
+  SELECT * FROM (VALUES(1),(2),(3),(4),(5),(6)), (VALUES('a'),('b'),('c'));
+} {
+  QUERY PLAN
+  |--SCAN 6-ROW VALUES CLAUSE
+  `--SCAN 3-ROW VALUES CLAUSE
+}
+do_execsql_test 15.6 {
+  CREATE TABLE t2(x,y);
+}
+do_eqp_test 15.7 {
+  SELECT * FROM t2 UNION ALL VALUES(1,2),(3,4),(5,6),(7,8);
+} {
+  QUERY PLAN
+  `--COMPOUND QUERY
+     |--LEFT-MOST SUBQUERY
+     |  `--SCAN t2
+     `--UNION ALL
+        `--SCAN 4-ROW VALUES CLAUSE
+}
+
+#--------------------------------------------------------------------------
+# The VALUES-as-coroutine optimization can be applied to later rows of
+# a VALUES clause even if earlier rows do not qualify.
+#
+reset_db
+do_execsql_test 16.1 {
+  CREATE TABLE t1(a,b);
+}
+do_execsql_test 16.2 {
+  BEGIN;
+  INSERT INTO t1 VALUES(1,2),(3,4),(5,6),
+     (7,row_number()OVER()),
+     (9,10), (11,12), (13,14), (15,16);
+  SELECT * FROM t1 ORDER BY a, b;
+  ROLLBACK;
+} {1 2 3 4 5 6 7 1 9 10 11 12 13 14 15 16}
+do_eqp_test 16.3 {
+  INSERT INTO t1 VALUES(1,2),(3,4),(5,6),
+     (7,row_number()OVER()),
+     (9,10), (11,12), (13,14), (15,16);
+} {
+  QUERY PLAN
+  `--COMPOUND QUERY
+     |--LEFT-MOST SUBQUERY
+     |  `--SCAN 3-ROW VALUES CLAUSE
+     |--UNION ALL
+     |  |--CO-ROUTINE (subquery-xxxxxx)
+     |  |  `--SCAN CONSTANT ROW
+     |  `--SCAN (subquery-xxxxxx)
+     `--UNION ALL
+        `--SCAN 4-ROW VALUES CLAUSE
+}
+do_execsql_test 16.4 {
+  BEGIN;
+  INSERT INTO t1 VALUES
+     (1,row_number()OVER()),
+     (2,3), (4,5), (6,7);
+  SELECT * FROM t1 ORDER BY a, b;
+  ROLLBACK;
+} {1 1 2 3 4 5 6 7}
+do_eqp_test 16.5 {
+  INSERT INTO t1 VALUES
+     (1,row_number()OVER()),
+     (2,3), (4,5), (6,7);
+} {
+  QUERY PLAN
+  `--COMPOUND QUERY
+     |--LEFT-MOST SUBQUERY
+     |  |--CO-ROUTINE (subquery-xxxxxx)
+     |  |  `--SCAN CONSTANT ROW
+     |  `--SCAN (subquery-xxxxxx)
+     `--UNION ALL
+        `--SCAN 3-ROW VALUES CLAUSE
+}
+do_execsql_test 16.6 {
+  BEGIN;
+  INSERT INTO t1 VALUES
+     (1,2),(3,4),
+     (5,row_number()OVER()),
+     (7,8),(9,10),(11,12),
+     (13,row_number()OVER()),
+     (15,16),(17,18),(19,20),(21,22);
+  SELECT * FROM t1 ORDER BY a, b;
+  ROLLBACK;
+} { 1 2 3 4 5 1 7 8 9 10 11 12 13 1 15 16 17 18 19 20 21 22}
+do_eqp_test 16.7 {
+  INSERT INTO t1 VALUES
+     (1,2),(3,4),
+     (5,row_number()OVER()),
+     (7,8),(9,10),(11,12),
+     (13,row_number()OVER()),
+     (15,16),(17,18),(19,20),(21,22);
+} {
+  QUERY PLAN
+  `--COMPOUND QUERY
+     |--LEFT-MOST SUBQUERY
+     |  `--SCAN 2-ROW VALUES CLAUSE
+     |--UNION ALL
+     |  |--CO-ROUTINE (subquery-xxxxxx)
+     |  |  `--SCAN CONSTANT ROW
+     |  `--SCAN (subquery-xxxxxx)
+     |--UNION ALL
+     |  `--SCAN 3-ROW VALUES CLAUSE
+     |--UNION ALL
+     |  |--CO-ROUTINE (subquery-xxxxxx)
+     |  |  `--SCAN CONSTANT ROW
+     |  `--SCAN (subquery-xxxxxx)
+     `--UNION ALL
+        `--SCAN 4-ROW VALUES CLAUSE
+}
+
+#--------------------------------------------------------------------------
+# 2024-03-23 dbsqlfuzz crash-c2c5e7e08b7e489d270a26d895077a03f678c33b
+#
+do_execsql_test 17.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1 AS SELECT * FROM (VALUES(1,2), (3,4 IN (1,2,3)));
+}
+
+do_execsql_test 17.2 {
+  SELECT * FROM t1
+} {1 2 3 0}
+
+# 2024-03-25 dbsqlfuzz crash-74cf7c9904360322a6c917e4934b127543d1cd51
+#
+do_catchsql_test 18.1 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY);
+  INSERT INTO t1 VALUES(RAISE(IGNORE)),(0);
+} {1 {RAISE() may only be used within a trigger-program}}
+do_catchsql_test 18.2 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z);
+  CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO t1(y) VALUES(RAISE(IGNORE)),(0);
+  END;
+  INSERT INTO t1 VALUES(1,2,3);
+  SELECT * FROM t1;
+} {0 {1 2 3}}
+do_catchsql_test 18.3.1 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z);
+  CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO t1(y) VALUES(RAISE(ABORT,'error 18.3')),(0);
+  END;
+  INSERT INTO t1 VALUES(1,2,3);
+} {1 {error 18.3}}
+do_execsql_test 18.3.2 {
+  SELECT * FROM t1;
+} {}
+do_catchsql_test 18.4.1 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z);
+  CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO t1(y) VALUES(1),(RAISE(ABORT,'error 18.4')),(0);
+  END;
+  INSERT INTO t1 VALUES(1,2,3);
+} {1 {error 18.4}}
+do_execsql_test 18.4.2 {
+  SELECT * FROM t1;
+} {}
+do_catchsql_test 18.5.1 {
+  DROP TABLE t1;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z);
+  CREATE TRIGGER r2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO t1(y) VALUES(1),
+       (CASE WHEN new.z>7 THEN RAISE(ABORT,'error 18.5') ELSE 2 END);
+  END;
+  INSERT INTO t1 VALUES(1,2,3);
+  SELECT * FROM t1;
+} {0 {1 2 3 2 1 {} 3 2 {}}}
+do_catchsql_test 18.5.2 {
+  DELETE FROM t1;
+  INSERT INTO t1 VALUES(1,2,13);
+} {1 {error 18.5}}
+do_catchsql_test 18.5.3 {
+  SELECT * FROM t1;
+} {0 {}}
+
+# 2024-04-18 dbsqlfuzz crash-bde3bf80aedf25afa56e2997a0545a314765d3f8
+# Verify that the VALUES expressions used as an argument to an outer
+# join work correctly.
+#
+reset_db
+db null NULL
+do_execsql_test 19.1 {
+  CREATE TABLE t1(a INT, b INT);
+  INSERT INTO t1 VALUES(11,22);
+  SELECT * FROM t1 LEFT JOIN (VALUES(33,44),(55,66)) AS t2 ON a=b;
+} {11 22 NULL NULL}
+do_execsql_test 19.2 {
+  SELECT * FROM (VALUES(33,44),(55,66)) AS t2 RIGHT JOIN t1 ON a=b;
+} {NULL NULL 11 22}
+do_execsql_test 19.3 {
+  SELECT *, '|' FROM t1 FULL JOIN (VALUES(33,44),(55,66)) AS t2 ON a=b
+  ORDER BY +column1
+} {11 22 NULL NULL | NULL NULL 33 44 | NULL NULL 55 66 |}
+do_execsql_test 19.4 {
+  SELECT *, '|' FROM (VALUES(33,44),(55,66)) AS t2 FULL JOIN t1 ON a=b
+  ORDER BY +column1
+} {NULL NULL 11 22 | 33 44 NULL NULL | 55 66 NULL NULL |}
+
+# 2024-04-21 dbsqlfuzz 6fd1ff3a64bef4a6c092e8d757548e95698b0df5
+# A continuation of the 2024-04-18 problem above.  We have to create
+# Pseudo-cursor that is always NULL on the viaCoroutine loop in case
+# there are OP_Columns generated against it by the sub-WHERE clause.
+#
+db null N
+do_execsql_test 19.5 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  DROP TABLE IF EXISTS t3;
+  CREATE TABLE t1(a,b);              INSERT INTO t1 VALUES(1,2);
+  CREATE TABLE t2(column1,column2);  INSERT INTO t2 VALUES(11,22),(33,44);
+  CREATE TABLE t3(d,e);              INSERT INTO t3 VALUES(3,4);
+}
+do_execsql_test 19.6 {
+  -- output verify using PG 14.2
+  SELECT *
+    FROM t1 CROSS JOIN t2 FULL JOIN t3 ON a=d
+   ORDER BY +d, +column1;
+} {1 2 11 22 N N
+   1 2 33 44 N N
+   N N  N  N 3 4}
+do_execsql_test 19.7 {
+  SELECT *
+    FROM t1 CROSS JOIN (VALUES(11,22),(33,44)) FULL JOIN t3 ON a=d
+   ORDER BY +d, +column1;
+} {1 2 11 22 N N
+   1 2 33 44 N N
+   N N  N  N 3 4}
+do_execsql_test 19.8 {
+  -- output verified using PG 14.2
+  SELECT *
+    FROM t1 CROSS JOIN t2 FULL JOIN t3 ON a=d
+   WHERE column1 IS NULL;
+} {N N  N  N 3 4}
+do_execsql_test 19.9 {
+  SELECT *
+    FROM t1 CROSS JOIN (VALUES(11,22),(33,44)) FULL JOIN t3 ON a=d
+   WHERE column1 IS NULL;
+} {N N  N  N 3 4}
+
+finish_test
diff --git a/test/valuesfault.test b/test/valuesfault.test
new file mode 100644
index 0000000000..bc5dddfb0e
--- /dev/null
+++ b/test/valuesfault.test
@@ -0,0 +1,37 @@
+# 2024 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix valuesfault
+source $testdir/malloc_common.tcl
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, c);
+}
+faultsim_save_and_close
+
+do_faultsim_test 1 -prep {
+  faultsim_restore_and_reopen
+  sqlite3_limit db SQLITE_LIMIT_COMPOUND_SELECT 2
+} -body {
+  execsql {
+    INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4);
+  }
+} -test {
+  faultsim_test_result {0 {}} 
+}
+
+
+finish_test
diff --git a/test/view.test b/test/view.test
index 90c9c99092..241742025c 100644
--- a/test/view.test
+++ b/test/view.test
@@ -801,4 +801,27 @@ ifcapable schema_pragmas {
   } { 0 a INT 0 {} 0 1 b BLOB 0 {} 0 }
 }
 
+#-----------------------------------------------------------------------
+# 2024-04-25   Trying to make type information on compound subqueries
+# more predictable and rational.
+#
+reset_db
+do_execsql_test view-31.1 {
+  CREATE TABLE x2(b TEXT);
+  CREATE TABLE x1(a TEXT);
+  INSERT INTO x1 VALUES('123');
+  -- Two queries get the same result even though the order of terms
+  -- in the CTE is reversed
+  WITH c(x) AS ( SELECT b FROM x2 UNION SELECT 123 )
+    SELECT count(*) FROM x1 WHERE a IN c; 
+  WITH c(x) AS ( SELECT 123 UNION SELECT b FROM x2 )
+    SELECT count(*) FROM x1 WHERE a IN c;
+} {0 0}
+do_execsql_test view-31.2 {
+  CREATE TABLE t3(a INTEGER, b TEXT);
+  INSERT INTO t3 VALUES(123, 123);
+  WITH s AS ( VALUES(123), (456) ) SELECT * FROM t3 WHERE b IN s;
+} {123 123}
+
+
 finish_test
diff --git a/test/vt02.c b/test/vt02.c
index ddad136fba..06fade0969 100644
--- a/test/vt02.c
+++ b/test/vt02.c
@@ -580,8 +580,11 @@ static void sqlite3BestIndexLog(
     sqlite3_finalize(pStmt);
   }
   sqlite3RunSql(db,pVTab,
-    "INSERT INTO temp.\"%w\"(bi,vn,ix) VALUES(%d,'nConstraint',%d)",
-    zLogTab, iBI, pInfo->nConstraint
+    "INSERT INTO temp.\"%w\"(bi,vn,ix) VALUES(%d,'nConstraint',%d)"
+      "RETURNING iif(bi=%d,'ok',RAISE(ABORT,'wrong trigger'))",
+    /* The RETURNING clause checks to see that the returning trigger fired
+    ** for the correct INSERT in the case of nested INSERT RETURNINGs. */
+    zLogTab, iBI, pInfo->nConstraint, iBI
   );
   for(i=0; i<pInfo->nConstraint; i++){
     sqlite3_value *pVal;
@@ -983,7 +986,9 @@ const sqlite3_module vt02Module = {
   /* xRename       */  0,
   /* xSavepoint    */  0,
   /* xRelease      */  0,
-  /* xRollbackTo   */  0
+  /* xRollbackTo   */  0,
+  /* xShadowName   */  0,
+  /* xIntegrity    */  0
 };
 
 static void vt02CoreInit(sqlite3 *db){
diff --git a/test/vt100-a.sql b/test/vt100-a.sql
new file mode 100644
index 0000000000..a0d3f46be7
--- /dev/null
+++ b/test/vt100-a.sql
@@ -0,0 +1,19 @@
+/*
+** Run this script using the "sqlite3" command-line shell
+** test test formatting of output text that contains
+** vt100 escape sequences.
+*/
+.mode box -escape off
+CREATE TEMP TABLE t1(a,b,c);
+INSERT INTO t1 VALUES
+  ('one','twotwotwo','thirty-three'),
+  (unistr('\u001b[91mRED\u001b[0m'),'fourfour','fifty-five'),
+  ('six','seven','eighty-eight');
+.print With -escape off
+SELECT * FROM t1;
+.mode box -escape ascii
+.print With -escape ascii
+SELECT * FROM t1;
+.mode box -escape symbol
+.print With -escape symbol
+SELECT * FROM t1;
diff --git a/test/vtabH.test b/test/vtabH.test
index f1a0466554..1496f49b53 100644
--- a/test/vtabH.test
+++ b/test/vtabH.test
@@ -124,14 +124,14 @@ foreach ::tclvar_set_omit {0 1} {
 
 #-------------------------------------------------------------------------
 #
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
   set drive [string range [pwd] 0 1]
   set ::env(fstreeDrive) $drive
 }
-if {$tcl_platform(platform)!="windows" || \
+reset_db
+register_fs_module db
+if {$tcl_platform(platform) ne "windows" || \
     [regexp -nocase -- {^[A-Z]:} $drive]} {
-  reset_db
-  register_fs_module db
   do_execsql_test 3.0 {
     SELECT name FROM fsdir WHERE dir = '.' AND name = 'test.db';
     SELECT name FROM fsdir WHERE dir = '.' AND name = '.'
@@ -190,10 +190,10 @@ if {$tcl_platform(platform)!="windows" || \
       lappend res "/$p"
     }
   }
-  set num_root_files [llength $root_files]
+  set num_root_files [llength $res]
   do_test 3.1 {
     sort_files [execsql {
-      SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files;
+      SELECT path FROM fstree WHERE path NOT GLOB '*$*' LIMIT $num_root_files
     }] true
   } [sort_files $res true]
 
diff --git a/test/vtabL.test b/test/vtabL.test
new file mode 100644
index 0000000000..45528edcbd
--- /dev/null
+++ b/test/vtabL.test
@@ -0,0 +1,75 @@
+# 2024-03-26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vtabL
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return $::create_table_sql
+    }
+  }
+
+  return {}
+}
+
+foreach {tn cts} {
+  1 {SELECT 123}
+  2 {SELECT 123, 456}
+  3 {INSERT INTO t1 VALUES(5, 6)}
+  4 {CREATE INDEX i1 ON t1(a)}
+  5 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END;}
+  6 {DROP TABLE nosuchtable}
+  7 {DROP TABLE x1}
+  8 {DROP TABLE t1}
+} {
+  set ::create_table_sql $cts
+  do_catchsql_test 1.$tn {
+    CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+  } {1 {declare_vtab: syntax error}}
+}
+
+foreach {tn cts} {
+  9 {CREATE TABLE xyz AS SELECT * FROM sqlite_schema}
+  10 {CREATE TABLE xyz AS SELECT 1 AS 'col'}
+} {
+  set ::create_table_sql $cts
+  do_catchsql_test 1.$tn {
+    CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+  } {1 {declare_vtab: SQL logic error}}
+}
+
+foreach {tn cts} {
+  1 {CREATE TABLE IF NOT EXISTS t1(a, b)}
+  2 {CREATE TABLE ""(a, b PRIMARY KEY) WITHOUT ROWID}
+} {
+  set ::create_table_sql $cts
+  execsql { DROP TABLE IF EXISTS x1 }
+  do_execsql_test 2.$tn.1 {
+    CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+  }
+  do_execsql_test 2.$tn.2 {
+    SELECT a, b FROM x1
+  }
+}
+
+finish_test
+
diff --git a/test/wal.test b/test/wal.test
index 234668240e..50988debe3 100644
--- a/test/wal.test
+++ b/test/wal.test
@@ -1219,7 +1219,7 @@ foreach {tn pgsz works} {
       set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2]
 
       set fd [open test.db-wal w]
-      fconfigure $fd -encoding binary -translation binary
+      fconfigure $fd -translation binary
       puts -nonewline $fd $walhdr
       puts -nonewline $fd $framehdr
       puts -nonewline $fd $framebody
diff --git a/test/wal2.test b/test/wal2.test
index ae6134d8b5..064bed0b20 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -26,7 +26,7 @@ ifcapable !wal {finish_test ; return }
 set sqlite_sync_count 0
 proc cond_incr_sync_count {adj} {
   global sqlite_sync_count
-  if {$::tcl_platform(platform) == "windows"} {
+  if {$::tcl_platform(os) eq "Windows NT"} {
     incr sqlite_sync_count $adj
   } {
     ifcapable !dirsync {
@@ -35,43 +35,6 @@ proc cond_incr_sync_count {adj} {
   }
 }
 
-proc set_tvfs_hdr {file args} {
-
-  # Set $nHdr to the number of bytes in the wal-index header:
-  set nHdr 48
-  set nInt [expr {$nHdr/4}]
-
-  if {[llength $args]>2} {
-    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
-  }
-
-  set blob [tvfs shm $file]
-  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
-
-  if {[llength $args]} {
-    set ia [lindex $args 0]
-    set ib $ia
-    if {[llength $args]==2} {
-      set ib [lindex $args 1]
-    }
-    binary scan $blob a[expr $nHdr*2]a* dummy tail
-    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
-    tvfs shm $file $blob
-  }
-
-  binary scan $blob ${fmt}${nInt} ints
-  return $ints
-}
-
-proc incr_tvfs_hdr {file idx incrval} {
-  set ints [set_tvfs_hdr $file]
-  set v [lindex $ints $idx]
-  incr v $incrval
-  lset ints $idx $v
-  set_tvfs_hdr $file $ints
-}
-
-
 #-------------------------------------------------------------------------
 # Test case wal2-1.*:
 #
@@ -1075,7 +1038,7 @@ tvfs delete
 # the new files with the same file-system permissions as the database 
 # file itself. Test this.
 #
-if {$::tcl_platform(platform) == "unix"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
   faultsim_delete_and_reopen
   # Changed on 2012-02-13: umask is deliberately ignored for -wal files.
   #set umask [exec /bin/sh -c umask]
@@ -1098,7 +1061,11 @@ if {$::tcl_platform(platform) == "unix"} {
    3 00600
    4 00755
   } {
-    set effective [format %.5o [expr $permissions & ~$umask]]
+    if {$tcl_version>=9.0} {
+      set effective [format %.5d [expr $permissions & ~$umask]]
+    } else {
+      set effective [format %.5o [expr $permissions & ~$umask]]
+    }
     do_test wal2-12.2.$tn.1 {
       file attributes test.db -permissions $permissions
       string map {o 0} [file attributes test.db -permissions]
@@ -1127,7 +1094,7 @@ if {$::tcl_platform(platform) == "unix"} {
 # database, wal or shm files cannot be opened, or can only be opened
 # read-only.
 #
-if {$::tcl_platform(platform) == "unix"} {
+if {$::tcl_platform(os) ne "Windows NT"} {
   proc perm {} {
     set L [list]
     foreach f {test.db test.db-wal test.db-shm} {
diff --git a/test/wal6.test b/test/wal6.test
index 9bbc58409d..081608cd30 100644
--- a/test/wal6.test
+++ b/test/wal6.test
@@ -46,7 +46,7 @@ foreach jmode $all_journal_modes {
 # Under Windows, you'll get an error trying to delete
 # a file this is already opened.  Close the first connection
 # so the other tests work.
-if {$tcl_platform(platform)=="windows"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   if {$jmode=="persist" || $jmode=="truncate"} {
     db close
   }
@@ -61,7 +61,7 @@ if {$tcl_platform(platform)=="windows"} {
     } db2
   } {wal 1 2 3 4}
 
-if {$tcl_platform(platform)=="windows"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   if {$jmode=="persist" || $jmode=="truncate"} {
     sqlite3 db test.db
   }
diff --git a/test/wal64k.test b/test/wal64k.test
index 8ff8e4b77c..bacb14328a 100644
--- a/test/wal64k.test
+++ b/test/wal64k.test
@@ -19,10 +19,10 @@ set testprefix wal64k
 
 ifcapable !wal {finish_test ; return }
 
-if {$tcl_platform(platform) != "unix"} {
+if {[llength [info commands test_syscall]]==0} {
   finish_test
   return
-}
+} 
 
 db close
 test_syscall pagesize 65536
diff --git a/test/wal_common.tcl b/test/wal_common.tcl
index 917ad598f6..d31131aa02 100644
--- a/test/wal_common.tcl
+++ b/test/wal_common.tcl
@@ -65,7 +65,6 @@ proc wal_set_walhdr {filename {intlist {}}} {
 
     set fd [open $filename r+]
     fconfigure $fd -translation binary
-    fconfigure $fd -encoding binary
     seek $fd 0
     puts -nonewline $fd $blob
     close $fd
@@ -73,7 +72,6 @@ proc wal_set_walhdr {filename {intlist {}}} {
 
   set fd [open $filename]
   fconfigure $fd -translation binary
-  fconfigure $fd -encoding binary
   set blob [read $fd 24]
   close $fd
 
@@ -90,4 +88,42 @@ proc wal_fix_walindex_cksum {hdrvar} {
   lset hdr 11 $c2
 }
 
+# This command assumes that $file is the name of a database file opened
+# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit
+# integers that make up the wal-index-header for the named file.
+#
+proc set_tvfs_hdr {file args} {
+
+  # Set $nHdr to the number of bytes in the wal-index header:
+  set nHdr 48
+  set nInt [expr {$nHdr/4}]
+
+  if {[llength $args]>2} {
+    error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"}
+  }
+
+  set blob [tvfs shm $file]
+  if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i}
 
+  if {[llength $args]} {
+    set ia [lindex $args 0]
+    set ib $ia
+    if {[llength $args]==2} {
+      set ib [lindex $args 1]
+    }
+    binary scan $blob a[expr $nHdr*2]a* dummy tail
+    set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail]
+    tvfs shm $file $blob
+  }
+
+  binary scan $blob ${fmt}${nInt} ints
+  return $ints
+}
+
+proc incr_tvfs_hdr {file idx incrval} {
+  set ints [set_tvfs_hdr $file]
+  set v [lindex $ints $idx]
+  incr v $incrval
+  lset ints $idx $v
+  set_tvfs_hdr $file $ints
+}
diff --git a/test/walblock.test b/test/walblock.test
index 23167a8830..86a52b3f96 100644
--- a/test/walblock.test
+++ b/test/walblock.test
@@ -17,7 +17,7 @@ source $testdir/wal_common.tcl
 
 finish_test; return;    #  Feature currently not implemented.
 ifcapable !wal {finish_test ; return }
-if {$::tcl_platform(platform)!="unix"} { finish_test ; return }
+if {$::tcl_platform(platform) ne "unix"} { finish_test ; return }
 set testprefix walblock
 
 catch { db close }
diff --git a/test/walckptnoop.test b/test/walckptnoop.test
new file mode 100644
index 0000000000..7ff8e90b8f
--- /dev/null
+++ b/test/walckptnoop.test
@@ -0,0 +1,110 @@
+# 2025 September 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the operation of the library in
+# "PRAGMA wal_checkpoint = noop" mode.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+
+set testprefix walckptnoop
+
+ifcapable !wal {finish_test ; return }
+
+set VAL 123
+
+proc myrand {} {
+  global VAL
+
+  set A 1103515245
+  set C 12345
+  set M 2147483648
+
+  set VAL [expr {($A * $VAL + $C) % $M}]
+  return $VAL
+}
+
+proc myrandomblob {n} {
+  set l [list]
+  for {set i 0} {$i < $n} {incr i} {
+    lappend l [expr [myrand] % 256]
+  }
+  binary format c* $l
+}
+
+db func myrandomblob myrandomblob
+
+
+do_execsql_test 1.0 {
+  PRAGMA page_size=1024;
+  PRAGMA auto_vacuum=NONE;
+  PRAGMA secure_delete=OFF;
+  VACUUM;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT);
+  CREATE INDEX i1 ON t1(y);
+  PRAGMA journal_mode = wal;
+
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  )
+  INSERT INTO t1 SELECT NULL, hex(myrandomblob(64)) FROM s;
+} {0 wal}
+
+do_execsql_test 1.1 {
+  PRAGMA wal_checkpoint = noop;
+} {0 298 0}
+do_execsql_test 1.2 {
+  PRAGMA wal_checkpoint = noop;
+} {0 298 0}
+do_execsql_test 1.3 {
+  PRAGMA wal_checkpoint = passive;
+} {0 298 298}
+do_execsql_test 1.4 {
+  PRAGMA wal_checkpoint = noop;
+} {0 298 298}
+
+db_save_and_close
+db_restore_and_reopen
+do_execsql_test 1.5 {
+  PRAGMA wal_checkpoint = noop;
+} {0 298 0}
+
+db close
+sqlite3 db test.db
+db eval {
+  PRAGMA auto_vacuum=NONE;
+  PRAGMA secure_delete=OFF;
+}
+do_execsql_test 1.6 {
+  PRAGMA wal_checkpoint = noop;
+} {0 0 0}
+
+do_catchsql_test 1.7 {
+  BEGIN;
+    DELETE FROM t1;
+    PRAGMA wal_checkpoint = noop;
+} {1 {database table is locked}}
+
+do_catchsql_test 1.8 {
+  COMMIT;
+  PRAGMA wal_checkpoint = noop;
+} {0 {0 5 0}}
+
+do_execsql_test 1.9 {
+  PRAGMA journal_mode = delete;
+  PRAGMA wal_checkpoint = noop;
+} {delete 0 -1 -1}
+
+finish_test
diff --git a/test/walcksum.test b/test/walcksum.test
index f3fc427115..0c9a7e55c0 100644
--- a/test/walcksum.test
+++ b/test/walcksum.test
@@ -16,13 +16,13 @@ source $testdir/lock_common.tcl
 source $testdir/wal_common.tcl
 
 ifcapable !wal {finish_test ; return }
+set testprefix walcksum
 
 # Read and return the contents of file $filename. Treat the content as
 # binary data.
 #
 proc readfile {filename} {
   set fd [open $filename]
-  fconfigure $fd -encoding binary
   fconfigure $fd -translation binary
   set data [read $fd]
   close $fd
@@ -59,7 +59,6 @@ proc log_checksum_write {filename iFrame endian} {
 
   set bin [binary format II $c1 $c2]
   set fd [open $filename r+]
-  fconfigure $fd -encoding binary
   fconfigure $fd -translation binary
   seek $fd $offset
   puts -nonewline $fd $bin
@@ -114,7 +113,6 @@ proc log_checksum_writemagic {filename endian} {
   set val [expr {0x377f0682 | ($endian == "big" ? 1 : 0)}]
   set bin [binary format I $val]
   set fd [open $filename r+]
-  fconfigure $fd -encoding binary
   fconfigure $fd -translation binary
   puts -nonewline $fd $bin
 
@@ -334,5 +332,152 @@ do_test walcksum-2.1 {
 catch { db close }
 catch { db2 close }
 
+#-------------------------------------------------------------------------
+# Test cases based on the bug reported at:
+# 
+#    <https://sqlite.org/forum/forumpost/b490f726db>
+#
+reset_db
+
+do_execsql_test 3.0 {
+  PRAGMA auto_vacuum = 0;
+  PRAGMA synchronous = NORMAL;
+  PRAGMA journal_mode = WAL;
+  PRAGMA cache_size = 1;
+
+  CREATE TABLE t1 (i INTEGER PRIMARY KEY, b BLOB, t TEXT);
+  PRAGMA wal_checkpoint;
+  INSERT INTO t1 VALUES(1, randomblob(2048), 'one');
+} {wal 0 2 2}
+
+do_execsql_test 3.1 {
+  BEGIN;
+    INSERT INTO t1 VALUES(2, randomblob(2048), 'two');
+    SAVEPOINT one;
+    INSERT INTO t1 VALUES(3, randomblob(2048), 'three');
+    INSERT INTO t1 VALUES(4, randomblob(2048), 'four');
+    INSERT INTO t1 VALUES(5, randomblob(2048), 'five');
+    INSERT INTO t1 VALUES(6, randomblob(2048), 'six');
+    INSERT INTO t1 VALUES(7, randomblob(2048), 'seven');
+
+    UPDATE t1 SET b=randomblob(2048) WHERE i=5;
+    UPDATE t1 SET b=randomblob(2048) WHERE i=6;
+    UPDATE t1 SET b=randomblob(2048) WHERE i=7;
+    ROLLBACK TO one;
+    INSERT INTO t1 VALUES(8, NULL, 'eight');
+  COMMIT;
+} {}
+
+do_execsql_test 3.2 {
+  SELECT i, t FROM t1
+} {1 one   2 two   8 eight}
+
+forcecopy test.db test2.db
+forcecopy test.db-wal test2.db-wal
+
+sqlite3 db2 test2.db
+do_test 1.3 {
+  execsql {
+    SELECT i, t FROM t1
+  } db2
+} {1 one   2 two   8 eight}
+
+catch { db2 close }
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 4.0 {
+  PRAGMA auto_vacuum = 0;
+  PRAGMA synchronous = NORMAL;
+  PRAGMA journal_mode = WAL;
+  PRAGMA cache_size = 1;
+
+  CREATE TABLE t1 (i INTEGER PRIMARY KEY, b BLOB, t TEXT);
+  PRAGMA wal_checkpoint;
+  INSERT INTO t1 VALUES(1, randomblob(2048), 'one');
+} {wal 0 2 2}
+
+do_execsql_test 4.1.1 {
+  SAVEPOINT one;
+    INSERT INTO t1 VALUES(2, randomblob(2048), 'two');
+    INSERT INTO t1 VALUES(3, randomblob(2048), 'three');
+    INSERT INTO t1 VALUES(4, randomblob(2048), 'four');
+    INSERT INTO t1 VALUES(5, randomblob(2048), 'five');
+    INSERT INTO t1 VALUES(6, randomblob(2048), 'six');
+    INSERT INTO t1 VALUES(7, randomblob(2048), 'seven');
+
+    UPDATE t1 SET b=randomblob(2048) WHERE i=5;
+    UPDATE t1 SET b=randomblob(2048) WHERE i=6;
+    UPDATE t1 SET b=randomblob(2048) WHERE i=7;
+}
+
+do_execsql_test 4.1.2 {
+    ROLLBACK TO one;
+    INSERT INTO t1 VALUES(8, NULL, 'eight');
+  RELEASE one;
+} {}
+
+do_execsql_test 4.2 {
+  SELECT i, t FROM t1
+} {1 one   8 eight}
+
+forcecopy test.db test2.db
+forcecopy test.db-wal test2.db-wal
+
+sqlite3 db2 test2.db
+do_test 4.3 {
+  execsql {
+    SELECT i, t FROM t1
+  } db2
+} {1 one   8 eight}
+
+catch { db2 close }
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 5.0 {
+  PRAGMA auto_vacuum = 0;
+  PRAGMA synchronous = NORMAL;
+  PRAGMA journal_mode = WAL;
+  PRAGMA cache_size = 1;
+
+  CREATE TABLE t1 (i INTEGER PRIMARY KEY, b BLOB, t TEXT);
+  INSERT INTO t1 VALUES(1, randomblob(2048), 'one');
+  INSERT INTO t1 VALUES(2, randomblob(2048), 'two');
+  INSERT INTO t1 VALUES(3, randomblob(2048), 'three');
+  PRAGMA wal_checkpoint;
+} {wal 0 14 14}
+
+do_execsql_test 5.1 {
+  BEGIN;
+    SELECT count(*) FROM t1;
+    SAVEPOINT one;
+    INSERT INTO t1 VALUES(4, randomblob(2048), 'four');
+    INSERT INTO t1 VALUES(5, randomblob(2048), 'five');
+    INSERT INTO t1 VALUES(6, randomblob(2048), 'six');
+    INSERT INTO t1 VALUES(7, randomblob(2048), 'seven');
+    ROLLBACK TO one;
+    INSERT INTO t1 VALUES(8, randomblob(2048), 'eight');
+    INSERT INTO t1 VALUES(9, randomblob(2048), 'nine');
+  COMMIT;
+} {3}
+
+forcecopy test.db test2.db
+forcecopy test.db-wal test2.db-wal
+
+sqlite3 db2 test2.db
+do_test 5.2 {
+  execsql {
+    SELECT i, t FROM t1
+  } db2
+} {1 one  2 two   3 three  8 eight 9 nine}
+db2 close
+
+do_execsql_test 5.3 {
+  SELECT i, t FROM t1
+} {1 one  2 two   3 three  8 eight 9 nine}
+
   
 finish_test
diff --git a/test/walcrash4.test b/test/walcrash4.test
index 80839b39e5..43292def42 100644
--- a/test/walcrash4.test
+++ b/test/walcrash4.test
@@ -38,7 +38,7 @@ faultsim_save_and_close
 
 # The error message is different on unix and windows
 #
-if {$::tcl_platform(platform)=="windows"} {
+if {$::tcl_platform(platform) eq "windows"} {
  set msg "child killed: unknown signal"
 } else {
  set msg "child process exited abnormally"
diff --git a/test/walmode.test b/test/walmode.test
index f760823c8d..1c3325acf2 100644
--- a/test/walmode.test
+++ b/test/walmode.test
@@ -47,7 +47,7 @@ do_test walmode-1.2 {
 
 if {[atomic_batch_write test.db]==0} {
   set expected_sync_count 3
-  if {$::tcl_platform(platform)!="windows"} {
+  if {$::tcl_platform(os) ne "Windows NT"} {
     ifcapable dirsync {
       incr expected_sync_count
     }
diff --git a/test/walnoshm.test b/test/walnoshm.test
index d4082178dd..f546c29b13 100644
--- a/test/walnoshm.test
+++ b/test/walnoshm.test
@@ -104,6 +104,7 @@ do_test 2.1.5 {
 } {exclusive delete a b c d e f g h}
 
 do_test 2.2.1 {
+  db2 close
   forcecopy test.db     test2.db
   forcecopy test.db-wal test2.db-wal
   sqlite3 db3 test2.db -vfs tvfsshm
diff --git a/test/walro.test b/test/walro.test
index cae52db6d3..a39b844d9d 100644
--- a/test/walro.test
+++ b/test/walro.test
@@ -19,7 +19,7 @@ set ::testprefix walro
 
 # These tests are only going to work on unix.
 #
-if {$::tcl_platform(platform) != "unix"} {
+if {$::tcl_platform(os) eq "Windows NT"} {
   finish_test
   return
 }
diff --git a/test/walseh1.test b/test/walseh1.test
new file mode 100644
index 0000000000..225b69f742
--- /dev/null
+++ b/test/walseh1.test
@@ -0,0 +1,148 @@
+# 2021 August 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix walseh1
+
+set ::seh_countdown 0 
+set ::seh_errno 10
+proc seh_faultsim_callback {iFault} {
+  if {$iFault==650} {
+    incr ::seh_countdown -1
+    if {$::seh_countdown==0} { return $::seh_errno }
+  }
+  return 0
+}
+
+proc seh_injectinstall {} {
+  sqlite3_test_control_fault_install seh_faultsim_callback
+}
+proc seh_injectuninstall {} {
+  sqlite3_test_control_fault_install
+}
+proc seh_injectstart {iFail} {
+  set ::seh_errno [expr 2+$iFail*10]
+  set ::seh_countdown $iFail
+}
+proc seh_injectstop {} {
+  set res [expr $::seh_countdown<=0]
+  set ::seh_countdown 0
+  set res
+}
+
+set FAULTSIM(seh) [list                   \
+  -injectinstall   seh_injectinstall   \
+  -injectstart     seh_injectstart     \
+  -injectstop      seh_injectstop      \
+  -injecterrlist   {{1 {disk I/O error}}} \
+  -injectuninstall seh_injectuninstall \
+]
+
+proc test_system_errno {db expect} {
+  set serrno [sqlite3_system_errno $db]
+  if {$serrno!=$expect} {
+    error "surprising system_errno. Expected $expect, got $serrno"
+  }  
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+} {wal}
+faultsim_save_and_close
+
+do_faultsim_test 1 -faults seh -prep {
+  catch { db2 close }
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM sqlite_schema } 
+  sqlite3 db2 test.db
+} -body {
+  execsql { SELECT * FROM t1 } db2
+} -test {
+  faultsim_test_result {0 {1 2 3 4}} 
+  if {$testrc} { test_system_errno db2 $::seh_errno }
+}
+catch { db2 close }
+
+faultsim_save_and_close
+
+do_faultsim_test 2 -faults seh -prep {
+  catch { db close }
+  faultsim_restore_and_reopen
+} -body {
+  execsql { SELECT * FROM t1 } 
+} -test {
+  faultsim_test_result {0 {1 2 3 4}} 
+  if {$testrc} { test_system_errno db $::seh_errno }
+}
+
+do_faultsim_test 3 -faults seh -prep {
+  catch { db close }
+  faultsim_restore_and_reopen
+} -body {
+  execsql { INSERT INTO t1 VALUES(5, 6) }
+  execsql { SELECT * FROM t1 }
+} -test {
+  faultsim_test_result {0 {1 2 3 4 5 6}} 
+  if {$testrc} { test_system_errno db $::seh_errno }
+}
+
+do_faultsim_test 4 -faults seh -prep {
+  catch { db close }
+  faultsim_restore_and_reopen
+} -body {
+  execsql { PRAGMA wal_checkpoint }
+  execsql { INSERT INTO t1 VALUES(7, 8) }
+  execsql { SELECT * FROM t1 }
+} -test {
+  faultsim_test_result {0 {1 2 3 4 7 8}} 
+  if {$testrc} { test_system_errno db $::seh_errno }
+}
+catch { db close }
+
+do_faultsim_test 5 -faults seh -prep {
+  catch { db close }
+  faultsim_restore_and_reopen
+  execsql {
+    PRAGMA cache_size = 5;
+    BEGIN;
+      WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50
+      )
+      INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM s;
+  }
+} -body {
+  execsql ROLLBACK
+} -test {
+  faultsim_test_result {0 {}}
+  if {$testrc} { test_system_errno db $::seh_errno }
+}
+catch { db close }
+
+do_faultsim_test 6 -faults seh -prep {
+  catch { db close }
+  faultsim_restore_and_reopen
+} -body {
+  execsql { PRAGMA wal_checkpoint = TRUNCATE }
+  execsql { INSERT INTO t1 VALUES(7, 8) }
+  execsql { SELECT * FROM t1 }
+} -test {
+  faultsim_test_result {0 {1 2 3 4 7 8}} 
+  if {$testrc} { test_system_errno db $::seh_errno }
+}
+catch { db close }
+
+finish_test
diff --git a/test/walsetlk.test b/test/walsetlk.test
index 1e09238226..969bcaf465 100644
--- a/test/walsetlk.test
+++ b/test/walsetlk.test
@@ -80,6 +80,19 @@ db2 close
 #-------------------------------------------------------------------------
 
 do_multiclient_test tn {
+
+  testvfs tvfs -fullshm 1
+  db close
+  sqlite3 db test.db -vfs tvfs
+  tvfs script xSleep_callback
+  tvfs filter xSleep
+
+  set ::sleep_count 0
+  proc xSleep_callback {xSleep nMs} {
+    after [expr $nMs / 1000]
+    incr ::sleep_count
+  }
+
   do_test 2.$tn.1 {
     sql1 {
       PRAGMA journal_mode = wal;
@@ -132,7 +145,6 @@ do_multiclient_test tn {
     set us [lindex [time { catch {db eval "PRAGMA wal_checkpoint=RESTART"} }] 0]
     expr $us>1000000 && $us<4000000
   } {1}
-
   do_test 2.$tn.9 {
     sql3 {
       INSERT INTO t1 VALUES(11, 12);
@@ -178,13 +190,50 @@ do_multiclient_test tn {
     set us [lindex [time { catch {db eval "PRAGMA wal_checkpoint=RESTART"} }] 0]
     expr $us>1000000 && $us<4000000
   } {1}
-  
+
+  db close
+  tvfs delete
+
+  # Set bSleep to true if it is expected that the above used xSleep() to
+  # wait for locks. bSleep is true unless SQLITE_ENABLE_SETLK_TIMEOUT is
+  # set to 1 and either:
+  #
+  #   * the OS is windows, or
+  #   * the OS is unix and the tests were run with each connection 
+  #     in a separate process.
+  #
+  set bSleep 1
+  if {$::sqlite_options(setlk_timeout)==1} {
+    if {$::tcl_platform(platform) eq "windows"} {
+      set bSleep 0
+    }
+    if {$::tcl_platform(platform) eq "unix"} {
+      set bSleep [expr $tn==2]
+    }
+  }
+
+  do_test 2.$tn.15.$bSleep {
+    expr $::sleep_count > 0
+  } $bSleep
 }
 
 #-------------------------------------------------------------------------
 reset_db
-sqlite3 db2 test.db
+
+testvfs tvfs -fullshm 1
+tvfs script xSleep_callback
+tvfs filter xSleep
+
+set ::sleep_count 0
+proc xSleep_callback {xSleep nMs} {
+  after [expr $nMs / 1000]
+  incr ::sleep_count
+  breakpoint
+}
+
+sqlite3 db2 test.db -vfs tvfs
 db2 timeout 1000
+
 do_execsql_test 3.0 {
   PRAGMA journal_mode = wal;
   CREATE TABLE x1(x, y);
@@ -192,8 +241,110 @@ do_execsql_test 3.0 {
     INSERT INTO x1 VALUES(1, 2);
 } {wal}
 
-do_test 3.1 {
+do_execsql_test -db db2 3.1a {
+  SELECT * FROM x1
+} {}
+
+do_test 3.1b {
   list [catch { db2 eval {BEGIN EXCLUSIVE} } msg] $msg
 } {1 {database is locked}}
 
+# Set bExpect to true if calls to xSleep() are expected. Such calls are
+# expected unless this is an SQLITE_ENABLE_SETLK_TIMEOUT=1 build.
+set bExpect 1
+if {$tcl_platform(platform) eq "windows" && $::sqlite_options(setlk_timeout)==1} {
+  set bExpect 0
+}
+do_test 3.2 {
+  expr {$::sleep_count > 0}
+} $bExpect
+set ::sleep_count 0
+
+do_execsql_test 3.3 {
+  COMMIT;
+}
+
+# Launch a non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+     INSERT INTO x1 VALUES(3, 4);
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+}
+
+after 500 {set ok 1}
+vwait ok
+
+db2 timeout 5000
+do_test 3.4 {
+  set t [lindex [time { db2 eval { BEGIN EXCLUSIVE } }] 0]
+  expr ($t>1000000)
+} {1}
+
+# Set bExpect to true if calls to xSleep() are expected. Such calls are
+# expected unless this is an SQLITE_ENABLE_SETLK_TIMEOUT=1 build.
+set bExpect 1
+if {$::sqlite_options(setlk_timeout)==1} {
+  set bExpect 0
+}
+do_test 3.5 {
+  expr {$::sleep_count > 0}
+} $bExpect
+
+do_execsql_test -db db2 3.6 {
+  INSERT INTO x1 VALUES(5, 6);
+  COMMIT;
+  SELECT * FROM x1;
+} {1 2 3 4 5 6}
+
+# Launch a non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+     INSERT INTO x1 VALUES(7, 8);
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+  db close
+}
+
+after 500 {set ok 1}
+vwait ok
+
+db2 timeout 0x7FFFFFFF
+do_test 3.7 {
+  set t [lindex [time { db2 eval { BEGIN EXCLUSIVE } }] 0]
+  expr ($t>1000000)
+} {1}
+
+# Set bExpect to true if calls to xSleep() are expected. Such calls are
+# expected unless this is an SQLITE_ENABLE_SETLK_TIMEOUT=1 build.
+set bExpect 1
+if {$::sqlite_options(setlk_timeout)==1} {
+  set bExpect 0
+}
+do_test 3.8 {
+  expr {$::sleep_count > 0}
+} $bExpect
+
+do_execsql_test -db db2 3.9 {
+  INSERT INTO x1 VALUES(9, 10);
+  COMMIT;
+  SELECT * FROM x1;
+} {1 2 3 4 5 6 7 8 9 10}
+
+db2 close
+tvfs delete
+
 finish_test
+
diff --git a/test/walsetlk2.test b/test/walsetlk2.test
new file mode 100644
index 0000000000..7ffd8f03de
--- /dev/null
+++ b/test/walsetlk2.test
@@ -0,0 +1,269 @@
+# 2025 Jan 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: slow
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix walsetlk2
+
+ifcapable !wal {finish_test ; return }
+ifcapable !setlk_timeout {finish_test ; return }
+
+#-------------------------------------------------------------------------
+# Check that xShmLock calls are as expected for write transactions in
+# setlk mode.
+#
+reset_db
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b, c);
+  INSERT INTO t1 VALUES(1, 2, 3);
+} {wal}
+db close
+
+testvfs tvfs
+tvfs script xShmLock_callback
+tvfs filter xShmLock
+
+set ::xshmlock [list]
+proc xShmLock_callback {method path name detail} {
+  lappend ::xshmlock $detail
+}
+
+sqlite3 db test.db -vfs tvfs
+db timeout 1000
+
+do_execsql_test 1.1 {
+  SELECT * FROM t1
+} {1 2 3}
+
+do_execsql_test 1.2 {
+  INSERT INTO t1 VALUES(4, 5, 6);
+}
+
+set ::xshmlock [list]
+do_execsql_test 1.3 {
+  INSERT INTO t1 VALUES(7, 8, 9);
+}
+
+do_test 1.4 {
+  set ::xshmlock
+} [list \
+  {0 1 lock exclusive}                             \
+  {4 1 lock exclusive} {4 1 unlock exclusive}      \
+  {4 1 lock shared}                                \
+  {0 1 unlock exclusive}                           \
+  {4 1 unlock shared}
+]
+
+do_execsql_test 1.5.1 { SELECT * FROM t1 } {1 2 3 4 5 6 7 8 9}
+set ::xshmlock [list]
+do_execsql_test 1.5.2 {
+  INSERT INTO t1 VALUES(10, 11, 12);
+}
+do_test 1.5.3 {
+  set ::xshmlock
+} [list \
+  {0 1 lock exclusive}                             \
+  {4 1 lock shared}                                \
+  {0 1 unlock exclusive}                           \
+  {4 1 unlock shared}
+]
+
+db close
+tvfs delete
+
+#-------------------------------------------------------------------------
+# Check that if sqlite3_setlk_timeout() is used, blocking locks timeout
+# but other operations do not use the retry mechanism.
+#
+reset_db
+db close
+sqlite3 db test.db -fullmutex 1
+
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2), (3, 4);
+}
+
+sqlite3_setlk_timeout db 2000
+
+# Launch a non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+      INSERT INTO t1 VALUES(5, 6);
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+  db close
+}
+
+after 500 {set ok 1}
+vwait ok
+
+do_catchsql_test 2.1 {
+  INSERT INTO t1 VALUES(7, 8);
+} {1 {database is locked}}
+
+sqlite3_busy_timeout db 2000
+
+do_catchsql_test 2.2 {
+  INSERT INTO t1 VALUES(7, 8);
+} {0 {}}
+
+do_execsql_test 2.3 {
+  SELECT * FROM t1
+} {1 2 3 4 5 6 7 8}
+
+do_execsql_test 2.4 {
+  PRAGMA journal_mode = wal;
+} {wal}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 2.5 {
+  INSERT INTO t1 VALUES(9, 10);
+}
+
+if {$::sqlite_options(setlk_timeout)==1} {
+
+sqlite3_setlk_timeout db 2000
+
+# Launch a non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+      INSERT INTO t1 VALUES(11, 12);
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+  db close
+}
+
+after 500 {set ok 1}
+vwait ok
+
+do_catchsql_test 2.6 {
+  INSERT INTO t1 VALUES(13, 14);
+} {0 {}}
+
+do_execsql_test 2.7 {
+  SELECT * FROM t1
+} {1 2 3 4 5 6 7 8 9 10 11 12 13 14}
+
+}
+
+
+#-------------------------------------------------------------------------
+# Check that if sqlite3_setlk_timeout(-1) is called, blocking locks are
+# enabled and last for a few seconds at least. Difficult to test that they
+# really do block indefinitely.
+#
+reset_db
+
+if {$::sqlite_options(setlk_timeout)==1} {
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(1, 'one'), (3, 'three');
+} {wal}
+
+sqlite3_setlk_timeout db -1
+
+# Launch a non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+      INSERT INTO t1 VALUES(5, 'five');
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+  db close
+}
+
+after 500 {set ok 1}
+vwait ok
+
+breakpoint
+do_catchsql_test 3.1 {
+  INSERT INTO t1 VALUES(7, 'seven');
+} {0 {}}
+
+# Launch another non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+      INSERT INTO t1 VALUES(9, 'nine');
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+  db close
+}
+
+after 500 {set ok 1}
+vwait ok
+
+do_catchsql_test 3.2 {
+  INSERT INTO t1 VALUES(9, 'ten');
+} {1 {UNIQUE constraint failed: t1.a}}
+
+do_execsql_test 3.3 {
+  SELECT * FROM t1
+} {1 one 3 three 5 five 7 seven 9 nine}
+
+db close
+
+# Launch another non-blocking testfixture process to write-lock the 
+# database for 2000 ms.
+testfixture_nb done {
+  sqlite3 db test.db
+  db eval {
+    BEGIN EXCLUSIVE;
+      INSERT INTO t1 VALUES(11, 'eleven');
+  }
+  after 2000
+  db eval {
+    COMMIT
+  }
+}
+
+sqlite3 db test.db
+sqlite3_setlk_timeout db -1
+do_catchsql_test 3.4 {
+  INSERT INTO t1 VALUES(13, 'thirteen');
+} {0 {}}
+
+}
+
+finish_test
+
diff --git a/test/walsetlk3.test b/test/walsetlk3.test
new file mode 100644
index 0000000000..b091b183d8
--- /dev/null
+++ b/test/walsetlk3.test
@@ -0,0 +1,135 @@
+# 2020 May 06
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: slow
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix walsetlk3
+
+ifcapable !wal {finish_test ; return }
+ifcapable !setlk_timeout {finish_test ; return }
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x, y);
+  PRAGMA journal_mode = wal;
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+} {wal}
+
+db close
+
+proc sql_block_on_close {sql} {
+  testfixture_nb done [string map [list %SQL% $sql] {
+    testvfs tvfs
+    tvfs script xWrite
+    tvfs filter xWrite
+  
+    set ::delay_done 0
+    proc xWrite {method fname args} {
+      if {[file tail $fname]=="test.db" && $::delay_done==0} {
+        after 3000
+        set ::delay_done 1
+      }
+      return 0
+    }
+  
+    sqlite3 db test.db -vfs tvfs
+    db eval {%SQL%}
+    db close
+  }]
+}
+
+# Start a second process that writes to the db, then blocks within the
+# [db close] holding an EXCLUSIVE on the db in order to checkpoint and 
+# delete the wal file. Then try to read the db.
+#
+# Without the SQLITE_SETLK_BLOCK_ON_CONNECT flag, this should fail with
+# SQLITE_BUSY.
+#
+sql_block_on_close {
+  INSERT INTO t1 VALUES(5, 6);
+  INSERT INTO t1 VALUES(7, 8);
+}
+after 500 {set ok 1}
+vwait ok
+sqlite3 db test.db
+sqlite3_setlk_timeout db 2000
+do_catchsql_test 1.1 {
+  SELECT * FROM t1
+} {1 {database is locked}}
+
+vwait ::done
+
+# But with SQLITE_SETLK_BLOCK_ON_CONNECT flag, it should succeed.
+#
+sql_block_on_close {
+  INSERT INTO t1 VALUES(9, 10);
+  INSERT INTO t1 VALUES(11, 12);
+}
+after 500 {set ok 1}
+vwait ok
+sqlite3 db test.db
+sqlite3_setlk_timeout -block db 2000
+do_catchsql_test 1.2 {
+  SELECT * FROM t1
+} {0 {1 2 3 4 5 6 7 8 9 10 11 12}}
+
+vwait ::done
+
+#-------------------------------------------------------------------------
+# Check that the SQLITE_SETLK_BLOCK_ON_CONNECT does not cause connections
+# to block when taking a SHARED lock on a rollback mode database.
+#
+reset_db
+do_execsql_test 2.1 {
+  CREATE TABLE x1(a);
+  INSERT INTO x1 VALUES(1), (2), (3);
+}
+
+proc sql_block_on_write {sql} {
+  testfixture_nb done [string map [list %SQL% $sql] {
+    sqlite3 db test.db 
+    db eval "BEGIN EXCLUSIVE"
+    db eval {%SQL%}
+    after 3000
+    db eval COMMIT
+    db close
+  }]
+}
+
+db close
+sql_block_on_write {
+  INSERT INTO x1 VALUES(4);
+}
+
+after 500 {set ok 1}
+vwait ok
+
+sqlite3 db test.db
+sqlite3_setlk_timeout -block db 2000
+
+do_catchsql_test 2.2 {
+  SELECT * FROM x1
+} {1 {database is locked}}
+
+vwait ::done
+after 500 {set ok 1}
+vwait ok
+
+do_catchsql_test 2.3 {
+  SELECT * FROM x1
+} {0 {1 2 3 4}}
+
+finish_test
+
diff --git a/test/walsetlk_recover.test b/test/walsetlk_recover.test
new file mode 100644
index 0000000000..1daece7470
--- /dev/null
+++ b/test/walsetlk_recover.test
@@ -0,0 +1,104 @@
+# 2025 May 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: slow
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix walsetlk_recover
+
+ifcapable !wal {finish_test ; return }
+# ifcapable !setlk_timeout {finish_test ; return }
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+  INSERT INTO t1 VALUES(5, 6);
+} {wal}
+
+db_save_and_close
+db_restore
+
+testfixture_nb myvar {
+
+  testvfs tvfs -fullshm 1
+  sqlite3 db test.db -vfs tvfs
+  tvfs script vfs_callback
+  tvfs filter xRead
+
+  set ::done 0
+  proc vfs_callback {method file args} {
+    if {$::done==0 && [string match *wal $file]} {
+      after 4000
+      set ::done 1
+    }
+    return "SQLITE_OK"
+  }
+
+  db eval {
+    SELECT * FROM t1
+  }
+
+  db close
+}
+
+# Give the [testfixture_nb] command time to start
+after 1000 {set xyz 1}
+vwait xyz
+
+testvfs tvfs -fullshm 1
+sqlite3 db test.db -vfs tvfs
+
+tvfs script sleep_callback
+tvfs filter xSleep
+set ::sleep_count 0
+proc sleep_callback {args} {
+  incr ::sleep_count
+}
+
+sqlite3 db test.db -vfs tvfs
+db timeout 500
+set tm [lindex [time {
+  catch {
+    db eval {SELECT * FROM t1}
+  } msg
+}] 0]
+
+do_test 1.2         { set ::msg } {database is locked}
+do_test 1.3.($::tm) { expr $::tm>400000 && $::tm<2000000 } 1
+
+vwait myvar
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1
+} {1 2 3 4 5 6}
+
+db close
+tvfs delete
+
+# All SQLite builds should pass the tests above. SQLITE_ENABLE_SETLK_TIMEOUT=1
+# builds do so without calling the VFS xSleep method.
+if {$::sqlite_options(setlk_timeout)==1} {
+  do_test 1.5.1 {
+    set ::sleep_count
+  } 0
+} else {
+  do_test 1.5.2 {
+    expr $::sleep_count>0
+  } 1
+}
+
+finish_test
+
diff --git a/test/walsetlk_snapshot.test b/test/walsetlk_snapshot.test
new file mode 100644
index 0000000000..e05ad69cce
--- /dev/null
+++ b/test/walsetlk_snapshot.test
@@ -0,0 +1,109 @@
+# 2025 May 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: slow
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix walsetlk_snapshot
+
+ifcapable !wal {finish_test ; return }
+ifcapable !snapshot {finish_test; return}
+
+db close
+testvfs tvfs -fullshm 1
+sqlite3 db test.db -vfs tvfs
+tvfs script sleep_callback
+tvfs filter xSleep
+set ::sleep_count 0
+proc sleep_callback {args} {
+  incr ::sleep_count
+}
+
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+  INSERT INTO t1 VALUES(5, 6);
+} {wal}
+
+do_test 1.1 {
+  db eval BEGIN
+  set ::snap [sqlite3_snapshot_get db main]
+  db eval {
+    INSERT INTO t1 VALUES(7, 8);
+    COMMIT;
+  }
+} {}
+
+testfixture_nb myvar {
+
+  testvfs tvfs -fullshm 1
+  sqlite3 db test.db -vfs tvfs
+  tvfs script vfs_callback
+  tvfs filter {xWrite}
+
+  set ::done 0
+  proc vfs_callback {args} {
+    if {$::done==0} {
+      after 4000
+      set ::done 1
+    }
+    return "SQLITE_OK"
+  }
+
+  db eval {
+    PRAGMA wal_checkpoint;
+  }
+
+  db close
+}
+
+# Give the [testfixture_nb] command time to start
+after 1000 {set xyz 1}
+vwait xyz
+
+db timeout 500
+set tm [lindex [time {
+  catch {
+    db eval BEGIN
+      sqlite3_snapshot_open db main $::snap
+  } msg
+}] 0]
+
+do_test 1.2 { set ::msg } {SQLITE_BUSY}
+do_test 1.3.($::tm) { expr $::tm<2000000 } 1
+
+do_execsql_test 1.4 {
+  SELECT * FROM t1
+} {1 2 3 4 5 6 7 8}
+
+sqlite3_snapshot_free $::snap
+
+vwait myvar
+
+# All SQLite builds should pass the tests above. SQLITE_ENABLE_SETLK_TIMEOUT=1
+# builds do so without calling the VFS xSleep method.
+if {$::sqlite_options(setlk_timeout)==1} {
+  do_test 1.5.1 {
+    set ::sleep_count
+  } 0
+} else {
+  do_test 1.5.2 {
+    expr $::sleep_count>0
+  } 1
+}
+
+finish_test
+
diff --git a/test/walslow.test b/test/walslow.test
index 2a52a225d8..6a0f147a06 100644
--- a/test/walslow.test
+++ b/test/walslow.test
@@ -109,7 +109,6 @@ foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} {
 
       forcecopy test.db-wal test2.db-wal
       set fd [open test2.db-wal r+]
-      fconfigure $fd -encoding binary
       fconfigure $fd -translation binary
   
       seek $fd $iOff
diff --git a/test/walthread.test b/test/walthread.test
index 8e5df9e589..1a0c35af08 100644
--- a/test/walthread.test
+++ b/test/walthread.test
@@ -19,6 +19,7 @@ source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 if {[run_thread_tests]==0} { finish_test ; return }
 ifcapable !wal             { finish_test ; return }
+set DBNAME wt-[expr {int(abs(rand()*100000000))}]-test.db
 
 set sqlite_walsummary_mmap_incr 64
 
@@ -74,7 +75,6 @@ proc lshift {lvar} {
 #   -check     SCRIPT                 Script to run after test.
 #
 proc do_thread_test {args} {
-
   set A $args
 
   set P(testname) [lshift A]
@@ -126,12 +126,12 @@ proc do_thread_test {args} {
     return
   }
 
-  puts "Running $P(testname) for $P(seconds) seconds..."
+  puts "Running $P(testname) for $P(seconds) seconds on $::DBNAME"
 
   catch { db close }
-  forcedelete test.db test.db-journal test.db-wal
+  forcedelete $::DBNAME $::DBNAME-journal $::DBNAME-wal
 
-  sqlite3 db test.db
+  sqlite3 db $::DBNAME
   eval $P(init)
   catch { db close }
 
@@ -146,7 +146,7 @@ proc do_thread_test {args} {
         set E(nthread) $count
         set E(seconds) $P(seconds)
       "
-      set program [string map [list %TEST% $prg %VARS% $vars] {
+      set program [string map [list %TEST% $prg %VARS% $vars %DB% $::DBNAME] {
 
         %VARS%
 
@@ -163,7 +163,7 @@ proc do_thread_test {args} {
 
         proc busyhandler {n} { usleep 10 ; return 0 }
 
-        sqlite3 db test.db
+        sqlite3 db %DB%
         db busy busyhandler
         db eval { SELECT randomblob($E(pid)*5) }
 
@@ -202,7 +202,7 @@ proc do_thread_test {args} {
   }
   puts $report
 
-  sqlite3 db test.db
+  sqlite3 db $::DBNAME
   set res ""
   if {[catch $P(check) msg]} { set res $msg }
   do_test $P(testname).check [list set {} $res] ""
@@ -327,16 +327,16 @@ do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init {
 # the number of write-transactions performed using a rollback journal.
 # For example, "192 w, 185 r".
 #
-if {[atomic_batch_write test.db]==0} {
+if {[atomic_batch_write $::DBNAME]==0} {
   do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init {
     execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) }
-  } -thread RB 2 {
+  } -thread RB 2 [string map [list %DB% $::DBNAME] {
 
     db close
     set nRun 0
     set nDel 0
     while {[tt_continue]} {
-      sqlite3 db test.db
+      sqlite3 db %DB%
       db busy busyhandler
       db eval { SELECT * FROM sqlite_master }
       catch { db eval { PRAGMA journal_mode = DELETE } }
@@ -345,8 +345,8 @@ if {[atomic_batch_write test.db]==0} {
         INSERT INTO t1 VALUES(NULL, randomblob(100+$E(pid)));
       }
       incr nRun 1
-      incr nDel [file exists test.db-journal]
-      if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
+      incr nDel [file exists %DB%-journal]
+      if {[file exists %DB%-journal] + [file exists %DB%-wal] != 1} {
         error "File-system looks bad..."
       }
       db eval COMMIT
@@ -357,12 +357,12 @@ if {[atomic_batch_write test.db]==0} {
     list $nRun $nDel
     set {} "[expr $nRun-$nDel] w, $nDel r"
   
-  } -thread WAL 2 {
+  }] -thread WAL 2 [string map [list %DB% $::DBNAME] {
     db close
     set nRun 0
     set nDel 0
     while {[tt_continue]} {
-      sqlite3 db test.db
+      sqlite3 db %DB%
       db busy busyhandler
       db eval { SELECT * FROM sqlite_master }
       catch { db eval { PRAGMA journal_mode = WAL } }
@@ -371,8 +371,8 @@ if {[atomic_batch_write test.db]==0} {
         INSERT INTO t1 VALUES(NULL, randomblob(110+$E(pid)));
       }
       incr nRun 1
-      incr nDel [file exists test.db-journal]
-      if {[file exists test.db-journal] + [file exists test.db-wal] != 1} {
+      incr nDel [file exists %DB%-journal]
+      if {[file exists %DB%-journal] + [file exists %DB%-wal] != 1} {
         error "File-system looks bad..."
       }
       db eval COMMIT
@@ -381,7 +381,7 @@ if {[atomic_batch_write test.db]==0} {
       db close
     }
     set {} "[expr $nRun-$nDel] w, $nDel r"
-  }
+  }]
 }
 
 do_thread_test walthread-3 -seconds $seconds(walthread-3) -init {
@@ -510,14 +510,14 @@ do_thread_test walthread-5 -seconds $seconds(walthread-5) -init {
     COMMIT;
   }
 
-  forcecopy test.db-wal bak.db-wal
-  forcecopy test.db bak.db
+  forcecopy $::DBNAME-wal $::DBNAME-bak.db-wal
+  forcecopy $::DBNAME $::DBNAME-bak.db
   db close
 
-  forcecopy bak.db-wal test.db-wal
-  forcecopy bak.db test.db
+  forcecopy $::DBNAME-bak.db-wal $::DBNAME-wal
+  forcecopy $::DBNAME-bak.db $::DBNAME
 
-  if {[file size test.db-wal] < [log_file_size [expr 64*1024] 1024]} {
+  if {[file size $::DBNAME-wal] < [log_file_size [expr 64*1024] 1024]} {
     error "Somehow failed to create a large log file"
   }
   puts "Database with large log file recovered. Now running clients..."
@@ -525,5 +525,8 @@ do_thread_test walthread-5 -seconds $seconds(walthread-5) -init {
   db eval { SELECT count(*) FROM t1 }
 }
 unset -nocomplain seconds
+catch {db close}
+forcedelete $::DBNAME $::DBNAME-journal $::DBNAME-wal $::DBNAME-shm
+forcedelete $::DBNAME-bak.db-wal $::DBNAME-bak.db
 
 finish_test
diff --git a/test/wapp.tcl b/test/wapp.tcl
deleted file mode 100644
index 53c21e892f..0000000000
--- a/test/wapp.tcl
+++ /dev/null
@@ -1,987 +0,0 @@
-# Copyright (c) 2017 D. Richard Hipp
-# 
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the Simplified BSD License (also
-# known as the "2-Clause License" or "FreeBSD License".)
-#
-# This program is distributed in the hope that it will be useful,
-# but without any warranty; without even the implied warranty of
-# merchantability or fitness for a particular purpose.
-#
-#---------------------------------------------------------------------------
-#
-# Design rules:
-#
-#   (1)  All identifiers in the global namespace begin with "wapp"
-#
-#   (2)  Indentifiers intended for internal use only begin with "wappInt"
-#
-package require Tcl 8.6
-
-# Add text to the end of the HTTP reply.  No interpretation or transformation
-# of the text is performs.  The argument should be enclosed within {...}
-#
-proc wapp {txt} {
-  global wapp
-  dict append wapp .reply $txt
-}
-
-# Add text to the page under construction.  Do no escaping on the text.
-#
-# Though "unsafe" in general, there are uses for this kind of thing.
-# For example, if you want to return the complete, unmodified content of
-# a file:
-#
-#         set fd [open content.html rb]
-#         wapp-unsafe [read $fd]
-#         close $fd
-#
-# You could do the same thing using ordinary "wapp" instead of "wapp-unsafe".
-# The difference is that wapp-safety-check will complain about the misuse
-# of "wapp", but it assumes that the person who write "wapp-unsafe" understands
-# the risks.
-#
-# Though occasionally necessary, the use of this interface should be minimized.
-#
-proc wapp-unsafe {txt} {
-  global wapp
-  dict append wapp .reply $txt
-}
-
-# Add text to the end of the reply under construction.  The following
-# substitutions are made:
-#
-#     %html(...)          Escape text for inclusion in HTML
-#     %url(...)           Escape text for use as a URL
-#     %qp(...)            Escape text for use as a URI query parameter
-#     %string(...)        Escape text for use within a JSON string
-#     %unsafe(...)        No transformations of the text
-#
-# The substitutions above terminate at the first ")" character.  If the
-# text of the TCL string in ... contains ")" characters itself, use instead:
-#
-#     %html%(...)%
-#     %url%(...)%
-#     %qp%(...)%
-#     %string%(...)%
-#     %unsafe%(...)%
-#
-# In other words, use "%(...)%" instead of "(...)" to include the TCL string
-# to substitute.
-#
-# The %unsafe substitution should be avoided whenever possible, obviously.
-# In addition to the substitutions above, the text also does backslash
-# escapes.
-#
-# The wapp-trim proc works the same as wapp-subst except that it also removes
-# whitespace from the left margin, so that the generated HTML/CSS/Javascript
-# does not appear to be indented when delivered to the client web browser.
-#
-if {$tcl_version>=8.7} {
-  proc wapp-subst {txt} {
-    global wapp
-    regsub -all -command \
-       {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt wappInt-enc txt
-    dict append wapp .reply [subst -novariables -nocommand $txt]
-  }
-  proc wapp-trim {txt} {
-    global wapp
-    regsub -all {\n\s+} [string trim $txt] \n txt
-    regsub -all -command \
-       {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt wappInt-enc txt
-    dict append wapp .reply [subst -novariables -nocommand $txt]
-  }
-  proc wappInt-enc {all mode nu1 txt} {
-    return [uplevel 2 "wappInt-enc-$mode \"$txt\""]
-  }
-} else {
-  proc wapp-subst {txt} {
-    global wapp
-    regsub -all {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt \
-           {[wappInt-enc-\1 "\3"]} txt
-    dict append wapp .reply [uplevel 1 [list subst -novariables $txt]]
-  }
-  proc wapp-trim {txt} {
-    global wapp
-    regsub -all {\n\s+} [string trim $txt] \n txt
-    regsub -all {%(html|url|qp|string|unsafe){1,1}?(|%)\((.+)\)\2} $txt \
-           {[wappInt-enc-\1 "\3"]} txt
-    dict append wapp .reply [uplevel 1 [list subst -novariables $txt]]
-  }
-}
-
-# There must be a wappInt-enc-NAME routine for each possible substitution
-# in wapp-subst.  Thus there are routines for "html", "url", "qp", and "unsafe".
-#
-#    wappInt-enc-html           Escape text so that it is safe to use in the
-#                               body of an HTML document.
-#
-#    wappInt-enc-url            Escape text so that it is safe to pass as an
-#                               argument to href= and src= attributes in HTML.
-#
-#    wappInt-enc-qp             Escape text so that it is safe to use as the
-#                               value of a query parameter in a URL or in
-#                               post data or in a cookie.
-#
-#    wappInt-enc-string         Escape ", ', \, and < for using inside of a
-#                               javascript string literal.  The < character
-#                               is escaped to prevent "</script>" from causing
-#                               problems in embedded javascript.
-#
-#    wappInt-enc-unsafe         Perform no encoding at all.  Unsafe.
-#
-proc wappInt-enc-html {txt} {
-  return [string map {& &amp; < &lt; > &gt; \" &quot; \\ &#92;} $txt]
-}
-proc wappInt-enc-unsafe {txt} {
-  return $txt
-}
-proc wappInt-enc-url {s} {
-  if {[regsub -all {[^-{}@~?=#_.:/a-zA-Z0-9]} $s {[wappInt-%HHchar {&}]} s]} {
-    set s [subst -novar -noback $s]
-  }
-  if {[regsub -all {[{}]} $s {[wappInt-%HHchar \\&]} s]} {
-    set s [subst -novar -noback $s]
-  }
-  return $s
-}
-proc wappInt-enc-qp {s} {
-  if {[regsub -all {[^-{}_.a-zA-Z0-9]} $s {[wappInt-%HHchar {&}]} s]} {
-    set s [subst -novar -noback $s]
-  }
-  if {[regsub -all {[{}]} $s {[wappInt-%HHchar \\&]} s]} {
-    set s [subst -novar -noback $s]
-  }
-  return $s
-}
-proc wappInt-enc-string {s} {
-  return [string map {\\ \\\\ \" \\\" ' \\' < \\u003c} $s]
-}
-
-# This is a helper routine for wappInt-enc-url and wappInt-enc-qp.  It returns
-# an appropriate %HH encoding for the single character c.  If c is a unicode
-# character, then this routine might return multiple bytes:  %HH%HH%HH
-#
-proc wappInt-%HHchar {c} {
-  if {$c==" "} {return +}
-  return [regsub -all .. [binary encode hex [encoding convertto utf-8 $c]] {%&}]
-}
-
-
-# Undo the www-url-encoded format.
-#
-# HT: This code stolen from ncgi.tcl
-#
-proc wappInt-decode-url {str} {
-  set str [string map [list + { } "\\" "\\\\" \[ \\\[ \] \\\]] $str]
-  regsub -all -- \
-      {%([Ee][A-Fa-f0-9])%([89ABab][A-Fa-f0-9])%([89ABab][A-Fa-f0-9])} \
-      $str {[encoding convertfrom utf-8 [binary decode hex \1\2\3]]} str
-  regsub -all -- \
-      {%([CDcd][A-Fa-f0-9])%([89ABab][A-Fa-f0-9])}                     \
-      $str {[encoding convertfrom utf-8 [binary decode hex \1\2]]} str
-  regsub -all -- {%([0-7][A-Fa-f0-9])} $str {\\u00\1} str
-  return [subst -novar $str]
-}
-
-# Reset the document back to an empty string.
-#
-proc wapp-reset {} {
-  global wapp
-  dict set wapp .reply {}
-}
-
-# Change the mime-type of the result document.
-#
-proc wapp-mimetype {x} {
-  global wapp
-  dict set wapp .mimetype $x
-}
-
-# Change the reply code.
-#
-proc wapp-reply-code {x} {
-  global wapp
-  dict set wapp .reply-code $x
-}
-
-# Set a cookie
-#
-proc wapp-set-cookie {name value} {
-  global wapp
-  dict lappend wapp .new-cookies $name $value
-}
-
-# Unset a cookie
-#
-proc wapp-clear-cookie {name} {
-  wapp-set-cookie $name {}
-}
-
-# Add extra entries to the reply header
-#
-proc wapp-reply-extra {name value} {
-  global wapp
-  dict lappend wapp .reply-extra $name $value
-}
-
-# Specifies how the web-page under construction should be cached.
-# The argument should be one of:
-#
-#    no-cache
-#    max-age=N             (for some integer number of seconds, N)
-#    private,max-age=N
-#
-proc wapp-cache-control {x} {
-  wapp-reply-extra Cache-Control $x
-}
-
-# Redirect to a different web page
-#
-proc wapp-redirect {uri} {
-  wapp-reply-code {307 Redirect}
-  wapp-reply-extra Location $uri
-}
-
-# Return the value of a wapp parameter
-#
-proc wapp-param {name {dflt {}}} {
-  global wapp
-  if {![dict exists $wapp $name]} {return $dflt}
-  return [dict get $wapp $name]
-}
-
-# Return true if a and only if the wapp parameter $name exists
-#
-proc wapp-param-exists {name} {
-  global wapp
-  return [dict exists $wapp $name]
-}
-
-# Set the value of a wapp parameter
-#
-proc wapp-set-param {name value} {
-  global wapp
-  dict set wapp $name $value
-}
-
-# Return all parameter names that match the GLOB pattern, or all
-# names if the GLOB pattern is omitted.
-#
-proc wapp-param-list {{glob {*}}} {
-  global wapp
-  return [dict keys $wapp $glob]
-}
-
-# By default, Wapp does not decode query parameters and POST parameters
-# for cross-origin requests.  This is a security restriction, designed to
-# help prevent cross-site request forgery (CSRF) attacks.
-#
-# As a consequence of this restriction, URLs for sites generated by Wapp
-# that contain query parameters will not work as URLs found in other
-# websites.  You cannot create a link from a second website into a Wapp
-# website if the link contains query planner, by default.
-#
-# Of course, it is sometimes desirable to allow query parameters on external
-# links.  For URLs for which this is safe, the application should invoke
-# wapp-allow-xorigin-params.  This procedure tells Wapp that it is safe to
-# go ahead and decode the query parameters even for cross-site requests.
-#
-# In other words, for Wapp security is the default setting.  Individual pages
-# need to actively disable the cross-site request security if those pages
-# are safe for cross-site access.
-#
-proc wapp-allow-xorigin-params {} {
-  global wapp
-  if {![dict exists $wapp .qp] && ![dict get $wapp SAME_ORIGIN]} {
-    wappInt-decode-query-params
-  }
-}
-
-# Set the content-security-policy.
-#
-# The default content-security-policy is very strict:  "default-src 'self'"
-# The default policy prohibits the use of in-line javascript or CSS.
-#
-# Provide an alternative CSP as the argument.  Or use "off" to disable
-# the CSP completely.
-#
-proc wapp-content-security-policy {val} {
-  global wapp
-  if {$val=="off"} {
-    dict unset wapp .csp
-  } else {
-    dict set wapp .csp $val
-  }
-}
-
-# Examine the bodys of all procedures in this program looking for
-# unsafe calls to various Wapp interfaces.  Return a text string
-# containing warnings. Return an empty string if all is ok.
-#
-# This routine is advisory only.  It misses some constructs that are
-# dangerous and flags others that are safe.
-#
-proc wapp-safety-check {} {
-  set res {}
-  foreach p [info procs] {
-    set ln 0
-    foreach x [split [info body $p] \n] {
-      incr ln
-      if {[regexp {^[ \t]*wapp[ \t]+([^\n]+)} $x all tail]
-       && [string index $tail 0]!="\173"
-       && [regexp {[[$]} $tail]
-      } {
-        append res "$p:$ln: unsafe \"wapp\" call: \"[string trim $x]\"\n"
-      }
-      if {[regexp {^[ \t]*wapp-(subst|trim)[ \t]+[^\173]} $x all cx]} {
-        append res "$p:$ln: unsafe \"wapp-$cx\" call: \"[string trim $x]\"\n"
-      }
-    }
-  }
-  return $res
-}
-
-# Return a string that descripts the current environment.  Applications
-# might find this useful for debugging.
-#
-proc wapp-debug-env {} {
-  global wapp
-  set out {}
-  foreach var [lsort [dict keys $wapp]] {
-    if {[string index $var 0]=="."} continue
-    append out "$var = [list [dict get $wapp $var]]\n"
-  }
-  append out "\[pwd\] = [list [pwd]]\n"
-  return $out
-}
-
-# Tracing function for each HTTP request.  This is overridden by wapp-start
-# if tracing is enabled.
-#
-proc wappInt-trace {} {}
-
-# Start up a listening socket.  Arrange to invoke wappInt-new-connection
-# for each inbound HTTP connection.
-#
-#    port            Listen on this TCP port.  0 means to select a port
-#                    that is not currently in use
-#
-#    wappmode        One of "scgi", "remote-scgi", "server", or "local".
-#
-#    fromip          If not {}, then reject all requests from IP addresses
-#                    other than $fromip
-#
-proc wappInt-start-listener {port wappmode fromip} {
-  if {[string match *scgi $wappmode]} {
-    set type SCGI
-    set server [list wappInt-new-connection \
-                wappInt-scgi-readable $wappmode $fromip]
-  } else {
-    set type HTTP
-    set server [list wappInt-new-connection \
-                wappInt-http-readable $wappmode $fromip]
-  }
-  if {$wappmode=="local" || $wappmode=="scgi"} {
-    set x [socket -server $server -myaddr 127.0.0.1 $port]
-  } else {
-    set x [socket -server $server $port]
-  }
-  set coninfo [chan configure $x -sockname]
-  set port [lindex $coninfo 2]
-  if {$wappmode=="local"} {
-    wappInt-start-browser http://127.0.0.1:$port/
-  } elseif {$fromip!=""} {
-    puts "Listening for $type requests on TCP port $port from IP $fromip"
-  } else {
-    puts "Listening for $type requests on TCP port $port"
-  }
-}
-
-# Start a web-browser and point it at $URL
-#
-proc wappInt-start-browser {url} {
-  global tcl_platform
-  if {$tcl_platform(platform)=="windows"} {
-    exec cmd /c start $url &
-  } elseif {$tcl_platform(os)=="Darwin"} {
-    exec open $url &
-  } elseif {[catch {exec xdg-open $url}]} {
-    exec firefox $url &
-  }
-}
-
-# This routine is a "socket -server" callback.  The $chan, $ip, and $port
-# arguments are added by the socket command.
-#
-# Arrange to invoke $callback when content is available on the new socket.
-# The $callback will process inbound HTTP or SCGI content.  Reject the
-# request if $fromip is not an empty string and does not match $ip.
-#
-proc wappInt-new-connection {callback wappmode fromip chan ip port} {
-  upvar #0 wappInt-$chan W
-  if {$fromip!="" && ![string match $fromip $ip]} {
-    close $chan
-    return
-  }
-  set W [dict create REMOTE_ADDR $ip REMOTE_PORT $port WAPP_MODE $wappmode \
-         .header {}]
-  fconfigure $chan -blocking 0 -translation binary
-  fileevent $chan readable [list $callback $chan]
-}
-
-# Close an input channel
-#
-proc wappInt-close-channel {chan} {
-  if {$chan=="stdout"} {
-    # This happens after completing a CGI request
-    exit 0
-  } else {
-    unset ::wappInt-$chan
-    close $chan
-  }
-}
-
-# Process new text received on an inbound HTTP request
-#
-proc wappInt-http-readable {chan} {
-  if {[catch [list wappInt-http-readable-unsafe $chan] msg]} {
-    puts stderr "$msg\n$::errorInfo"
-    wappInt-close-channel $chan
-  }
-}
-proc wappInt-http-readable-unsafe {chan} {
-  upvar #0 wappInt-$chan W wapp wapp
-  if {![dict exists $W .toread]} {
-    # If the .toread key is not set, that means we are still reading
-    # the header
-    set line [string trimright [gets $chan]]
-    set n [string length $line]
-    if {$n>0} {
-      if {[dict get $W .header]=="" || [regexp {^\s+} $line]} {
-        dict append W .header $line
-      } else {
-        dict append W .header \n$line
-      }
-      if {[string length [dict get $W .header]]>100000} {
-        error "HTTP request header too big - possible DOS attack"
-      }
-    } elseif {$n==0} {
-      # We have reached the blank line that terminates the header.
-      global argv0
-      set a0 [file normalize $argv0]
-      dict set W SCRIPT_FILENAME $a0
-      dict set W DOCUMENT_ROOT [file dir $a0]
-      if {[wappInt-parse-header $chan]} {
-        catch {close $chan}
-        return
-      }
-      set len 0
-      if {[dict exists $W CONTENT_LENGTH]} {
-        set len [dict get $W CONTENT_LENGTH]
-      }
-      if {$len>0} {
-        # Still need to read the query content
-        dict set W .toread $len
-      } else {
-        # There is no query content, so handle the request immediately
-        set wapp $W
-        wappInt-handle-request $chan 0
-      }
-    }
-  } else {
-    # If .toread is set, that means we are reading the query content.
-    # Continue reading until .toread reaches zero.
-    set got [read $chan [dict get $W .toread]]
-    dict append W CONTENT $got
-    dict set W .toread [expr {[dict get $W .toread]-[string length $got]}]
-    if {[dict get $W .toread]<=0} {
-      # Handle the request as soon as all the query content is received
-      set wapp $W
-      wappInt-handle-request $chan 0
-    }
-  }
-}
-
-# Decode the HTTP request header.
-#
-# This routine is always running inside of a [catch], so if
-# any problems arise, simply raise an error.
-#
-proc wappInt-parse-header {chan} {
-  upvar #0 wappInt-$chan W
-  set hdr [split [dict get $W .header] \n]
-  if {$hdr==""} {return 1}
-  set req [lindex $hdr 0]
-  dict set W REQUEST_METHOD [set method [lindex $req 0]]
-  if {[lsearch {GET HEAD POST} $method]<0} {
-    error "unsupported request method: \"[dict get $W REQUEST_METHOD]\""
-  }
-  set uri [lindex $req 1]
-  set split_uri [split $uri ?]
-  set uri0 [lindex $split_uri 0]
-  if {![regexp {^/[-.a-z0-9_/]*$} $uri0]} {
-    error "invalid request uri: \"$uri0\""
-  }
-  dict set W REQUEST_URI $uri0
-  dict set W PATH_INFO $uri0
-  set uri1 [lindex $split_uri 1]
-  dict set W QUERY_STRING $uri1
-  set n [llength $hdr]
-  for {set i 1} {$i<$n} {incr i} {
-    set x [lindex $hdr $i]
-    if {![regexp {^(.+): +(.*)$} $x all name value]} {
-      error "invalid header line: \"$x\""
-    }
-    set name [string toupper $name]
-    switch -- $name {
-      REFERER {set name HTTP_REFERER}
-      USER-AGENT {set name HTTP_USER_AGENT}
-      CONTENT-LENGTH {set name CONTENT_LENGTH}
-      CONTENT-TYPE {set name CONTENT_TYPE}
-      HOST {set name HTTP_HOST}
-      COOKIE {set name HTTP_COOKIE}
-      ACCEPT-ENCODING {set name HTTP_ACCEPT_ENCODING}
-      default {set name .hdr:$name}
-    }
-    dict set W $name $value
-  }
-  return 0
-}
-
-# Decode the QUERY_STRING parameters from a GET request or the
-# application/x-www-form-urlencoded CONTENT from a POST request.
-#
-# This routine sets the ".qp" element of the ::wapp dict as a signal
-# that query parameters have already been decoded.
-#
-proc wappInt-decode-query-params {} {
-  global wapp
-  dict set wapp .qp 1
-  if {[dict exists $wapp QUERY_STRING]} {
-    foreach qterm [split [dict get $wapp QUERY_STRING] &] {
-      set qsplit [split $qterm =]
-      set nm [lindex $qsplit 0]
-      if {[regexp {^[a-z][a-z0-9]*$} $nm]} {
-        dict set wapp $nm [wappInt-decode-url [lindex $qsplit 1]]
-      }
-    }
-  }
-  if {[dict exists $wapp CONTENT_TYPE] && [dict exists $wapp CONTENT]} {
-    set ctype [dict get $wapp CONTENT_TYPE]
-    if {$ctype=="application/x-www-form-urlencoded"} {
-      foreach qterm [split [string trim [dict get $wapp CONTENT]] &] {
-        set qsplit [split $qterm =]
-        set nm [lindex $qsplit 0]
-        if {[regexp {^[a-z][-a-z0-9_]*$} $nm]} {
-          dict set wapp $nm [wappInt-decode-url [lindex $qsplit 1]]
-        }
-      }
-    } elseif {[string match multipart/form-data* $ctype]} {
-      regexp {^(.*?)\r\n(.*)$} [dict get $wapp CONTENT] all divider body
-      set ndiv [string length $divider]
-      while {[string length $body]} {
-        set idx [string first $divider $body]
-        set unit [string range $body 0 [expr {$idx-3}]]
-        set body [string range $body [expr {$idx+$ndiv+2}] end]
-        if {[regexp {^Content-Disposition: form-data; (.*?)\r\n\r\n(.*)$} \
-             $unit unit hdr content]} {
-          if {[regexp {name="(.*)"; filename="(.*)"\r\nContent-Type: (.*?)$}\
-                $hdr hr name filename mimetype]} {
-            dict set wapp $name.filename \
-              [string map [list \\\" \" \\\\ \\] $filename]
-            dict set wapp $name.mimetype $mimetype
-            dict set wapp $name.content $content
-          } elseif {[regexp {name="(.*)"} $hdr hr name]} {
-            dict set wapp $name $content
-          }
-        }
-      }
-    }
-  }
-}
-
-# Invoke application-supplied methods to generate a reply to
-# a single HTTP request.
-#
-# This routine always runs within [catch], so handle exceptions by
-# invoking [error].
-#
-proc wappInt-handle-request {chan useCgi} {
-  global wapp
-  dict set wapp .reply {}
-  dict set wapp .mimetype {text/html; charset=utf-8}
-  dict set wapp .reply-code {200 Ok}
-  dict set wapp .csp {default-src 'self'}
-
-  # Set up additional CGI environment values
-  #
-  if {![dict exists $wapp HTTP_HOST]} {
-    dict set wapp BASE_URL {}
-  } elseif {[dict exists $wapp HTTPS]} {
-    dict set wapp BASE_URL https://[dict get $wapp HTTP_HOST]
-  } else {
-    dict set wapp BASE_URL http://[dict get $wapp HTTP_HOST]
-  }
-  if {![dict exists $wapp REQUEST_URI]} {
-    dict set wapp REQUEST_URI /
-  } elseif {[regsub {\?.*} [dict get $wapp REQUEST_URI] {} newR]} {
-    # Some servers (ex: nginx) append the query parameters to REQUEST_URI.
-    # These need to be stripped off
-    dict set wapp REQUEST_URI $newR
-  }
-  if {[dict exists $wapp SCRIPT_NAME]} {
-    dict append wapp BASE_URL [dict get $wapp SCRIPT_NAME]
-  } else {
-    dict set wapp SCRIPT_NAME {}
-  }
-  if {![dict exists $wapp PATH_INFO]} {
-    # If PATH_INFO is missing (ex: nginx) then construct it
-    set URI [dict get $wapp REQUEST_URI]
-    set skip [string length [dict get $wapp SCRIPT_NAME]]
-    dict set wapp PATH_INFO [string range $URI $skip end]
-  }
-  if {[regexp {^/([^/]+)(.*)$} [dict get $wapp PATH_INFO] all head tail]} {
-    dict set wapp PATH_HEAD $head
-    dict set wapp PATH_TAIL [string trimleft $tail /]
-  } else {
-    dict set wapp PATH_INFO {}
-    dict set wapp PATH_HEAD {}
-    dict set wapp PATH_TAIL {}
-  }
-  dict set wapp SELF_URL [dict get $wapp BASE_URL]/[dict get $wapp PATH_HEAD]
-
-  # Parse query parameters from the query string, the cookies, and
-  # POST data
-  #
-  if {[dict exists $wapp HTTP_COOKIE]} {
-    foreach qterm [split [dict get $wapp HTTP_COOKIE] {;}] {
-      set qsplit [split [string trim $qterm] =]
-      set nm [lindex $qsplit 0]
-      if {[regexp {^[a-z][-a-z0-9_]*$} $nm]} {
-        dict set wapp $nm [wappInt-decode-url [lindex $qsplit 1]]
-      }
-    }
-  }
-  set same_origin 0
-  if {[dict exists $wapp HTTP_REFERER]} {
-    set referer [dict get $wapp HTTP_REFERER]
-    set base [dict get $wapp BASE_URL]
-    if {$referer==$base || [string match $base/* $referer]} {
-      set same_origin 1
-    }
-  }
-  dict set wapp SAME_ORIGIN $same_origin
-  if {$same_origin} {
-    wappInt-decode-query-params
-  }
-
-  # Invoke the application-defined handler procedure for this page
-  # request.  If an error occurs while running that procedure, generate
-  # an HTTP reply that contains the error message.
-  #
-  wapp-before-dispatch-hook
-  wappInt-trace
-  set mname [dict get $wapp PATH_HEAD]
-  if {[catch {
-    if {$mname!="" && [llength [info proc wapp-page-$mname]]>0} {
-      wapp-page-$mname
-    } else {
-      wapp-default
-    }
-  } msg]} {
-    if {[wapp-param WAPP_MODE]=="local" || [wapp-param WAPP_MODE]=="server"} {
-      puts "ERROR: $::errorInfo"
-    }
-    wapp-reset
-    wapp-reply-code "500 Internal Server Error"
-    wapp-mimetype text/html
-    wapp-trim {
-      <h1>Wapp Application Error</h1>
-      <pre>%html($::errorInfo)</pre>
-    }
-    dict unset wapp .new-cookies
-  }
-
-  # Transmit the HTTP reply
-  #
-  if {$chan=="stdout"} {
-    puts $chan "Status: [dict get $wapp .reply-code]\r"
-  } else {
-    puts $chan "HTTP/1.1 [dict get $wapp .reply-code]\r"
-    puts $chan "Server: wapp\r"
-    puts $chan "Connection: close\r"
-  }
-  if {[dict exists $wapp .reply-extra]} {
-    foreach {name value} [dict get $wapp .reply-extra] {
-      puts $chan "$name: $value\r"
-    }
-  }
-  if {[dict exists $wapp .csp]} {
-    puts $chan "Content-Security-Policy: [dict get $wapp .csp]\r"
-  }
-  set mimetype [dict get $wapp .mimetype]
-  puts $chan "Content-Type: $mimetype\r"
-  if {[dict exists $wapp .new-cookies]} {
-    foreach {nm val} [dict get $wapp .new-cookies] {
-      if {[regexp {^[a-z][-a-z0-9_]*$} $nm]} {
-        if {$val==""} {
-          puts $chan "Set-Cookie: $nm=; HttpOnly; Path=/; Max-Age=1\r"
-        } else {
-          set val [wappInt-enc-url $val]
-          puts $chan "Set-Cookie: $nm=$val; HttpOnly; Path=/\r"
-        }
-      }
-    }
-  }
-  if {[string match text/* $mimetype]} {
-    set reply [encoding convertto utf-8 [dict get $wapp .reply]]
-    if {[regexp {\ygzip\y} [wapp-param HTTP_ACCEPT_ENCODING]]} {
-      catch {
-        set x [zlib gzip $reply]
-        set reply $x
-        puts $chan "Content-Encoding: gzip\r"
-      }
-    }
-  } else {
-    set reply [dict get $wapp .reply]
-  }
-  puts $chan "Content-Length: [string length $reply]\r"
-  puts $chan \r
-  puts -nonewline $chan $reply
-  flush $chan
-  wappInt-close-channel $chan
-}
-
-# This routine runs just prior to request-handler dispatch.  The
-# default implementation is a no-op, but applications can override
-# to do additional transformations or checks.
-#
-proc wapp-before-dispatch-hook {} {return}
-
-# Process a single CGI request
-#
-proc wappInt-handle-cgi-request {} {
-  global wapp env
-  foreach key {
-    CONTENT_LENGTH
-    CONTENT_TYPE
-    DOCUMENT_ROOT
-    HTTP_ACCEPT_ENCODING
-    HTTP_COOKIE
-    HTTP_HOST
-    HTTP_REFERER
-    HTTP_USER_AGENT
-    HTTPS
-    PATH_INFO
-    QUERY_STRING
-    REMOTE_ADDR
-    REQUEST_METHOD
-    REQUEST_URI
-    REMOTE_USER
-    SCRIPT_FILENAME
-    SCRIPT_NAME
-    SERVER_NAME
-    SERVER_PORT
-    SERVER_PROTOCOL
-  } {
-    if {[info exists env($key)]} {
-      dict set wapp $key $env($key)
-    }
-  }
-  set len 0
-  if {[dict exists $wapp CONTENT_LENGTH]} {
-    set len [dict get $wapp CONTENT_LENGTH]
-  }
-  if {$len>0} {
-    fconfigure stdin -translation binary
-    dict set wapp CONTENT [read stdin $len]
-  }
-  dict set wapp WAPP_MODE cgi
-  fconfigure stdout -translation binary
-  wappInt-handle-request stdout 1
-}
-
-# Process new text received on an inbound SCGI request
-#
-proc wappInt-scgi-readable {chan} {
-  if {[catch [list wappInt-scgi-readable-unsafe $chan] msg]} {
-    puts stderr "$msg\n$::errorInfo"
-    wappInt-close-channel $chan
-  }
-}
-proc wappInt-scgi-readable-unsafe {chan} {
-  upvar #0 wappInt-$chan W wapp wapp
-  if {![dict exists $W .toread]} {
-    # If the .toread key is not set, that means we are still reading
-    # the header.
-    #
-    # An SGI header is short.  This implementation assumes the entire
-    # header is available all at once.
-    #
-    dict set W .remove_addr [dict get $W REMOTE_ADDR]
-    set req [read $chan 15]
-    set n [string length $req]
-    scan $req %d:%s len hdr
-    incr len [string length "$len:,"]
-    append hdr [read $chan [expr {$len-15}]]
-    foreach {nm val} [split $hdr \000] {
-      if {$nm==","} break
-      dict set W $nm $val
-    }
-    set len 0
-    if {[dict exists $W CONTENT_LENGTH]} {
-      set len [dict get $W CONTENT_LENGTH]
-    }
-    if {$len>0} {
-      # Still need to read the query content
-      dict set W .toread $len
-    } else {
-      # There is no query content, so handle the request immediately
-      dict set W SERVER_ADDR [dict get $W .remove_addr]
-      set wapp $W
-      wappInt-handle-request $chan 0
-    }
-  } else {
-    # If .toread is set, that means we are reading the query content.
-    # Continue reading until .toread reaches zero.
-    set got [read $chan [dict get $W .toread]]
-    dict append W CONTENT $got
-    dict set W .toread [expr {[dict get $W .toread]-[string length $got]}]
-    if {[dict get $W .toread]<=0} {
-      # Handle the request as soon as all the query content is received
-      dict set W SERVER_ADDR [dict get $W .remove_addr]
-      set wapp $W
-      wappInt-handle-request $chan 0
-    }
-  }
-}
-
-# Start up the wapp framework.  Parameters are a list passed as the
-# single argument.
-#
-#    -server $PORT         Listen for HTTP requests on this TCP port $PORT
-#
-#    -local $PORT          Listen for HTTP requests on 127.0.0.1:$PORT
-#
-#    -scgi $PORT           Listen for SCGI requests on 127.0.0.1:$PORT
-#
-#    -remote-scgi $PORT    Listen for SCGI requests on TCP port $PORT
-#
-#    -cgi                  Handle a single CGI request
-#
-# With no arguments, the behavior is called "auto".  In "auto" mode,
-# if the GATEWAY_INTERFACE environment variable indicates CGI, then run
-# as CGI.  Otherwise, start an HTTP server bound to the loopback address
-# only, on an arbitrary TCP port, and automatically launch a web browser
-# on that TCP port.
-#
-# Additional options:
-#
-#    -fromip GLOB         Reject any incoming request where the remote
-#                         IP address does not match the GLOB pattern.  This
-#                         value defaults to '127.0.0.1' for -local and -scgi.
-#
-#    -nowait              Do not wait in the event loop.  Return immediately
-#                         after all event handlers are established.
-#
-#    -trace               "puts" each request URL as it is handled, for
-#                         debugging
-#
-#    -lint                Run wapp-safety-check on the application instead
-#                         of running the application itself
-#
-#    -Dvar=value          Set TCL global variable "var" to "value"
-#
-#
-proc wapp-start {arglist} {
-  global env
-  set mode auto
-  set port 0
-  set nowait 0
-  set fromip {}
-  set n [llength $arglist]
-  for {set i 0} {$i<$n} {incr i} {
-    set term [lindex $arglist $i]
-    if {[string match --* $term]} {set term [string range $term 1 end]}
-    switch -glob -- $term {
-      -server {
-        incr i;
-        set mode "server"
-        set port [lindex $arglist $i]
-      }
-      -local {
-        incr i;
-        set mode "local"
-        set fromip 127.0.0.1
-        set port [lindex $arglist $i]
-      }
-      -scgi {
-        incr i;
-        set mode "scgi"
-        set fromip 127.0.0.1
-        set port [lindex $arglist $i]
-      }
-      -remote-scgi {
-        incr i;
-        set mode "remote-scgi"
-        set port [lindex $arglist $i]
-      }
-      -cgi {
-        set mode "cgi"
-      }
-      -fromip {
-        incr i
-        set fromip [lindex $arglist $i]
-      }
-      -nowait {
-        set nowait 1
-      }
-      -trace {
-        proc wappInt-trace {} {
-          set q [wapp-param QUERY_STRING]
-          set uri [wapp-param BASE_URL][wapp-param PATH_INFO]
-          if {$q!=""} {append uri ?$q}
-          puts $uri
-        }
-      }
-      -lint {
-        set res [wapp-safety-check]
-        if {$res!=""} {
-          puts "Potential problems in this code:"
-          puts $res
-          exit 1
-        } else {
-          exit
-        }
-      }
-      -D*=* {
-        if {[regexp {^.D([^=]+)=(.*)$} $term all var val]} {
-          set ::$var $val
-        }
-      }
-      default {
-        error "unknown option: $term"
-      }
-    }
-  }
-  if {$mode=="auto"} {
-    if {[info exists env(GATEWAY_INTERFACE)]
-        && [string match CGI/1.* $env(GATEWAY_INTERFACE)]} {
-      set mode cgi
-    } else {
-      set mode local
-    }
-  }
-  if {$mode=="cgi"} {
-    wappInt-handle-cgi-request
-  } else {
-    wappInt-start-listener $port $mode $fromip
-    if {!$nowait} {
-      vwait ::forever
-    }
-  }
-}
-
-# Call this version 1.0
-package provide wapp 1.0
diff --git a/test/wapptest.tcl b/test/wapptest.tcl
deleted file mode 100755
index d37b2e48c6..0000000000
--- a/test/wapptest.tcl
+++ /dev/null
@@ -1,909 +0,0 @@
-#!/bin/sh
-# \
-exec wapptclsh "$0" ${1+"$@"}
-
-# package required wapp
-source [file join [file dirname [info script]] wapp.tcl]
-
-# Variables set by the "control" form:
-#
-#   G(platform) - User selected platform.
-#   G(cfgglob)  - Glob pattern that all configurations must match
-#   G(test)     - Set to "Normal", "Veryquick", "Smoketest" or "Build-Only".
-#   G(keep)     - Boolean. True to delete no files after each test.
-#   G(msvc)     - Boolean. True to use MSVC as the compiler.
-#   G(tcl)      - Use Tcl from this directory for builds.
-#   G(jobs)     - How many sub-processes to run simultaneously.
-#
-set G(platform) $::tcl_platform(os)-$::tcl_platform(machine)
-set G(cfgglob)  *
-set G(test)     Normal
-set G(keep)     1
-set G(msvc)     0
-set G(tcl)      [::tcl::pkgconfig get libdir,install]
-set G(jobs)     3
-set G(debug)    0
-
-set G(noui)     0
-set G(stdout)   0
-
-
-proc wapptest_init {} {
-  global G
-
-  set lSave [list platform test keep msvc tcl jobs debug noui stdout cfgglob] 
-  foreach k $lSave { set A($k) $G($k) }
-  array unset G
-  foreach k $lSave { set G($k) $A($k) }
-
-  # The root of the SQLite source tree.
-  set G(srcdir)   [file dirname [file dirname [info script]]]
-
-  set G(sqlite_version) "unknown"
-
-  # Either "config", "running" or "stopped":
-  set G(state) "config"
-
-  set G(hostname) "(unknown host)"
-  catch { set G(hostname) [exec hostname] } 
-  set G(host) $G(hostname)
-  append G(host) " $::tcl_platform(os) $::tcl_platform(osVersion)"
-  append G(host) " $::tcl_platform(machine) $::tcl_platform(byteOrder)"
-}
-
-proc wapptest_run {} {
-  global G
-  set_test_array
-  set G(state) "running"
-
-  wapptest_openlog
-
-  wapptest_output "Running the following for $G(platform). $G(jobs) jobs."
-  foreach t $G(test_array) {
-    set config [dict get $t config]
-    set target [dict get $t target]
-    wapptest_output [format "    %-25s%s" $config $target]
-  }
-  wapptest_output [string repeat * 70]
-}
-
-proc releasetest_data {args} {
-  global G
-  set rtd [file join $G(srcdir) test releasetest_data.tcl]
-  set fd [open "|[info nameofexecutable] $rtd $args" r+]
-  set ret [read $fd]
-  close $fd
-  return $ret
-}
-
-# Generate the text for the box at the top of the UI. The current SQLite
-# version, according to fossil, along with a warning if there are 
-# uncommitted changes in the checkout.
-#
-proc generate_fossil_info {} {
-  global G
-  set pwd [pwd]
-  cd $G(srcdir)
-  set rc [catch {
-    set r1 [exec fossil info]
-    set r2 [exec fossil changes]
-  }]
-  cd $pwd
-  if {$rc} return
-
-  foreach line [split $r1 "\n"] {
-    if {[regexp {^checkout: *(.*)$} $line -> co]} {
-      wapp-trim { <br> %html($co) }
-    }
-  }
-
-  if {[string trim $r2]!=""} {
-    wapp-trim { 
-      <br><span class=warning> 
-      WARNING: Uncommitted changes in checkout
-      </span>
-    }
-  }
-}
-
-# If the application is in "config" state, set the contents of the 
-# ::G(test_array) global to reflect the tests that will be run. If the
-# app is in some other state ("running" or "stopped"), this command
-# is a no-op.
-#
-proc set_test_array {} {
-  global G
-  if { $G(state)=="config" } {
-    set G(test_array) [list]
-    set debug "-debug"
-    if {$G(debug)==0} { set debug "-nodebug"}
-    foreach {config target} [releasetest_data tests $debug $G(platform)] {
-
-      # All configuration names must match $g(cfgglob), which defaults to *
-      #
-      if {![string match -nocase $G(cfgglob) $config]} continue
-
-      # If using MSVC, do not run sanitize or valgrind tests. Or the
-      # checksymbols test.
-      if {$G(msvc) && (
-          "Sanitize" == $config 
-       || "checksymbols" in $target
-       || "valgrindtest" in $target
-      )} {
-        continue
-      }
-
-      # If the test mode is not "Normal", override the target.
-      #
-      if {$target!="checksymbols" && $G(platform)!="Failure-Detection"} {
-        switch -- $G(test) {
-          Veryquick { set target quicktest }
-          Smoketest { set target smoketest }
-          Build-Only {
-            set target testfixture
-            if {$::tcl_platform(platform)=="windows"} {
-              set target testfixture.exe
-            }
-          }
-        }
-      }
-
-      lappend G(test_array) [dict create config $config target $target]
-    }
-  }
-}
-
-proc count_tests_and_errors {name logfile} {
-  global G
-
-  set fd [open $logfile rb]
-  set seen 0
-  while {![eof $fd]} {
-    set line [gets $fd]
-    if {[regexp {(\d+) errors out of (\d+) tests} $line all nerr ntest]} {
-      incr G(test.$name.nError) $nerr
-      incr G(test.$name.nTest) $ntest
-      set seen 1
-      if {$nerr>0} {
-        set G(test.$name.errmsg) $line
-      }
-    }
-    if {[regexp {runtime error: +(.*)} $line all msg]} {
-      # skip over "value is outside range" errors
-      if {[regexp {.* is outside the range of representable} $line]} {
-         # noop
-      } else {
-        incr G(test.$name.nError)
-        if {$G(test.$name.errmsg)==""} {
-          set G(test.$name.errmsg) $msg
-        }
-      }
-    }
-    if {[regexp {fatal error +(.*)} $line all msg]} {
-      incr G(test.$name.nError)
-      if {$G(test.$name.errmsg)==""} {
-        set G(test.$name.errmsg) $msg
-      }
-    }
-    if {[regexp {ERROR SUMMARY: (\d+) errors.*} $line all cnt] && $cnt>0} {
-      incr G(test.$name.nError)
-      if {$G(test.$name.errmsg)==""} {
-        set G(test.$name.errmsg) $all
-      }
-    }
-    if {[regexp {^VERSION: 3\.\d+.\d+} $line]} {
-      set v [string range $line 9 end]
-      if {$G(sqlite_version) eq "unknown"} {
-        set G(sqlite_version) $v
-      } elseif {$G(sqlite_version) ne $v} {
-        set G(test.$name.errmsg) "version conflict: {$G(sqlite_version)} vs. {$v}"
-      }
-    }
-  }
-  close $fd
-  if {$G(test) == "Build-Only"} {
-    incr G(test.$name.nTest)
-    if {$G(test.$name.nError)>0} {
-      set errmsg "Build failed"
-    }
-  } elseif {!$seen} {
-    set G(test.$name.errmsg) "Test did not complete"
-    if {[file readable core]} {
-      append G(test.$name.errmsg) " - core file exists"
-    }
-  }
-}
-
-proc wapptest_output {str} {
-  global G
-  if {$G(stdout)} { puts $str }
-  if {[info exists G(log)]} { 
-    puts $G(log) $str 
-    flush $G(log)
-  }
-}
-proc wapptest_openlog {} {
-  global G
-  set G(log) [open wapptest-out.txt w+]
-}
-proc wapptest_closelog {} {
-  global G
-  close $G(log)
-  unset G(log)
-}
-
-proc format_seconds {seconds} {
-  set min [format %.2d [expr ($seconds / 60) % 60]]
-  set  hr [format %.2d [expr $seconds / 3600]]
-  set sec [format %.2d [expr $seconds % 60]]
-  return "$hr:$min:$sec"
-}
-
-# This command is invoked once a slave process has finished running its
-# tests, successfully or otherwise. Parameter $name is the name of the 
-# test, $rc the exit code returned by the slave process.
-#
-proc slave_test_done {name rc} {
-  global G
-  set G(test.$name.done) [clock seconds]
-  set G(test.$name.nError) 0
-  set G(test.$name.nTest) 0
-  set G(test.$name.errmsg) ""
-  if {$rc} {
-    incr G(test.$name.nError)
-  }
-  if {[file exists $G(test.$name.log)]} {
-    count_tests_and_errors $name $G(test.$name.log)
-  }
-
-  # If the "keep files" checkbox is clear, delete all files except for
-  # the executables and test logs. And any core file that is present.
-  if {$G(keep)==0} {
-    set keeplist {
-      testfixture testfixture.exe
-      sqlite3 sqlite3.exe
-      test.log test-out.txt
-      core
-      wapptest_make.sh
-      wapptest_configure.sh
-      wapptest_run.tcl
-    }
-    foreach f [glob -nocomplain [file join $G(test.$name.dir) *]] {
-      set t [file tail $f]
-      if {[lsearch $keeplist $t]<0} {
-        catch { file delete -force $f }
-      }
-    }
-  }
-
-  # Format a message regarding the success or failure of hte test.
-  set t [format_seconds [expr $G(test.$name.done) - $G(test.$name.start)]]
-  set res "OK"
-  if {$G(test.$name.nError)} { set res "FAILED" }
-  set dots [string repeat . [expr 60 - [string length $name]]]
-  set msg "$name $dots $res ($t)"
-
-  wapptest_output $msg
-  if {[info exists G(test.$name.errmsg)] && $G(test.$name.errmsg)!=""} {
-    wapptest_output "    $G(test.$name.errmsg)"
-  }
-}
-
-# This is a fileevent callback invoked each time a file-descriptor that
-# connects this process to a slave process is readable.
-#
-proc slave_fileevent {name} {
-  global G
-  set fd $G(test.$name.channel)
-
-  if {[eof $fd]} {
-    fconfigure $fd -blocking 1
-    set rc [catch { close $fd }]
-    unset G(test.$name.channel)
-    slave_test_done $name $rc
-  } else {
-    set line [gets $fd]
-    if {[string trim $line] != ""} { puts "Trace   : $name - \"$line\"" }
-  }
-
-  do_some_stuff
-}
-
-# Return the contents of the "slave script" - the script run by slave 
-# processes to actually perform the test. All it does is execute the
-# test script already written to disk (wapptest_cmd.sh or wapptest_cmd.bat).
-#
-proc wapptest_slave_script {} {
-  global G
-  if {$G(msvc)==0} {
-    set dir [file join .. $G(srcdir)]
-    set res [subst -nocommands {
-      set rc [catch "exec sh wapptest_cmd.sh {$dir} >>& test.log" ]
-      exit [set rc]
-    }]
-  } else {
-    set dir [file nativename [file normalize $G(srcdir)]]
-    set dir [string map [list "\\" "\\\\"] $dir]
-    set res [subst -nocommands {
-      set rc [catch "exec wapptest_cmd.bat {$dir} >>& test.log" ]
-      exit [set rc]
-    }]
-  }
-
-  set res
-}
-
-
-# Launch a slave process to run a test.
-#
-proc slave_launch {name target dir} {
-  global G
-
-  catch { file mkdir $dir } msg
-  foreach f [glob -nocomplain [file join $dir *]] {
-    catch { file delete -force $f }
-  }
-  set G(test.$name.dir) $dir
-
-  # Write the test command to wapptest_cmd.sh|bat.
-  #
-  set ext sh
-  if {$G(msvc)} { set ext bat }
-  set fd1 [open [file join $dir wapptest_cmd.$ext] w]
-  if {$G(msvc)} {
-    puts $fd1 [releasetest_data script -msvc $name $target]
-  } else {
-    puts $fd1 [releasetest_data script $name $target]
-  }
-  close $fd1
-
-  # Write the wapptest_run.tcl script to the test directory. To run the
-  # commands in the other two files.
-  #
-  set fd3 [open [file join $dir wapptest_run.tcl] w]
-  puts $fd3 [wapptest_slave_script]
-  close $fd3
-
-  set pwd [pwd]
-  cd $dir
-  set fd [open "|[info nameofexecutable] wapptest_run.tcl" r+]
-  cd $pwd
-
-  set G(test.$name.channel) $fd
-  fconfigure $fd -blocking 0
-  fileevent $fd readable [list slave_fileevent $name]
-}
-
-proc do_some_stuff {} {
-  global G
-
-  # Count the number of running jobs. A running job has an entry named
-  # "channel" in its dictionary.
-  set nRunning 0
-  set bFinished 1
-  foreach j $G(test_array) {
-    set name [dict get $j config]
-    if { [info exists G(test.$name.channel)]} { incr nRunning   }
-    if {![info exists G(test.$name.done)]}    { set bFinished 0 }
-  }
-
-  if {$bFinished} {
-    set nError 0
-    set nTest 0
-    set nConfig 0
-    foreach j $G(test_array) {
-      set name [dict get $j config]
-      incr nError $G(test.$name.nError)
-      incr nTest $G(test.$name.nTest)
-      incr nConfig 
-    }
-    set G(result) "$nError errors from $nTest tests in $nConfig configurations."
-    wapptest_output [string repeat * 70]
-    wapptest_output $G(result)
-    catch {
-      append G(result) " SQLite version $G(sqlite_version)"
-      wapptest_output " SQLite version $G(sqlite_version)"
-    }
-    set G(state) "stopped"
-    wapptest_closelog
-    if {$G(noui)} { exit 0 }
-  } else {
-    set nLaunch [expr $G(jobs) - $nRunning]
-    foreach j $G(test_array) {
-      if {$nLaunch<=0} break
-      set name [dict get $j config]
-      if { ![info exists G(test.$name.channel)]
-        && ![info exists G(test.$name.done)]
-      } {
-
-        set target [dict get $j target]
-        set dir [string tolower [string map {" " _ "-" _} $name]]
-        set G(test.$name.start) [clock seconds]
-        set G(test.$name.log) [file join $dir test.log]
-
-        slave_launch $name $target $dir
-
-        incr nLaunch -1
-      }
-    }
-  }
-}
-
-proc generate_select_widget {label id lOpt opt} {
-  wapp-trim {
-    <label> %string($label) </label>
-    <select id=%string($id) name=%string($id)>
-  }
-  foreach o $lOpt {
-    set selected ""
-    if {$o==$opt} { set selected " selected=1" }
-    wapp-subst "<option $selected>$o</option>"
-  }
-  wapp-trim { </select> }
-}
-
-proc generate_main_page {{extra {}}} {
-  global G
-  set_test_array
-
-  set hostname $G(hostname)
-  wapp-trim {
-    <html>
-    <head>
-      <title> %html($hostname): wapptest.tcl </title>
-      <link rel="stylesheet" type="text/css" href="style.css"/>
-    </head>
-    <body>
-  }
-
-  set host $G(host)
-  wapp-trim {
-    <div class="border">%string($host)
-  }
-  generate_fossil_info
-  wapp-trim {
-    </div>
-    <div class="border" id=controls> 
-    <form action="control" method="post" name="control">
-  }
-
-  # Build the "platform" select widget. 
-  set lOpt [releasetest_data platforms]
-  generate_select_widget Platform control_platform $lOpt $G(platform)
-
-  # Build the "test" select widget. 
-  set lOpt [list Normal Veryquick Smoketest Build-Only] 
-  generate_select_widget Test control_test $lOpt $G(test)
-
-  # Build the "jobs" select widget. Options are 1 to 8.
-  generate_select_widget Jobs control_jobs {1 2 3 4 5 6 7 8 12 16} $G(jobs)
-
-  switch $G(state) {
-    config {
-      set txt "Run Tests!"
-      set id control_run
-    }
-    running {
-      set txt "STOP Tests!"
-      set id control_stop
-    }
-    stopped {
-      set txt "Reset!"
-      set id control_reset
-    }
-  }
-  wapp-trim {
-    <div class=right>
-    <input id=%string($id) name=%string($id) type=submit value="%string($txt)">
-    </input>
-    </div>
-  }
-
-  wapp-trim {
-  <br><br>
-        <label> Tcl: </label>
-        <input id="control_tcl" name="control_tcl"></input>
-        <label> Keep files: </label>
-        <input id="control_keep" name="control_keep" type=checkbox value=1>
-        </input>
-        <label> Use MSVC: </label>
-        <input id="control_msvc" name="control_msvc" type=checkbox value=1>
-        <label> Debug tests: </label>
-        <input id="control_debug" name="control_debug" type=checkbox value=1>
-        </input>
-  }
-  wapp-trim {
-     </form>
-  }
-  wapp-trim {
-     </div>
-     <div id=tests>
-  }
-  wapp-page-tests
-
-  set script "script/$G(state).js"
-  wapp-trim {
-    </div>
-      <script src=%string($script)></script>
-    </body>
-    </html>
-  }
-}
-
-proc wapp-default {} {
-  generate_main_page
-}
-
-proc wapp-page-tests {} {
-  global G
-  wapp-trim { <table class="border" width=100%> }
-  foreach t $G(test_array) {
-    set config [dict get $t config]
-    set target [dict get $t target]
-
-    set class "testwait"
-    set seconds ""
-
-    if {[info exists G(test.$config.log)]} {
-      if {[info exists G(test.$config.channel)]} {
-        set class "testrunning"
-        set seconds [expr [clock seconds] - $G(test.$config.start)]
-      } elseif {[info exists G(test.$config.done)]} {
-        if {$G(test.$config.nError)>0} {
-          set class "testfail" 
-        } else {
-          set class "testdone"
-        }
-        set seconds [expr $G(test.$config.done) - $G(test.$config.start)]
-      }
-      set seconds [format_seconds $seconds]
-    }
-
-    wapp-trim {
-      <tr class=%string($class)>
-      <td class="nowrap"> %html($config) 
-      <td class="padleft nowrap"> %html($target)
-      <td class="padleft nowrap"> %html($seconds)
-      <td class="padleft nowrap">
-    }
-    if {[info exists G(test.$config.log)]} {
-      set log $G(test.$config.log)
-      set uri "log/$log"
-      wapp-trim {
-        <a href=%url($uri)> %html($log) </a>
-      }
-    }
-    if {[info exists G(test.$config.errmsg)] && $G(test.$config.errmsg)!=""} {
-      set errmsg $G(test.$config.errmsg)
-      wapp-trim {
-        <tr class=testfail>
-        <td> <td class="padleft" colspan=3> %html($errmsg)
-      }
-    }
-  }
-
-  wapp-trim { </table> }
-
-  if {[info exists G(result)]} {
-    set res $G(result)
-    wapp-trim {
-      <div class=border id=result> %string($res) </div>
-    }
-  }
-}
-
-# URI: /control
-#
-# Whenever the form at the top of the application page is submitted, it
-# is submitted here.
-#
-proc wapp-page-control {} {
-  global G
-  if {$::G(state)=="config"} {
-    set lControls [list platform test tcl jobs keep msvc debug]
-    set G(msvc) 0
-    set G(keep) 0
-    set G(debug) 0
-  } else {
-    set lControls [list jobs]
-  }
-  foreach v $lControls {
-    if {[wapp-param-exists control_$v]} {
-      set G($v) [wapp-param control_$v]
-    }
-  }
-
-  if {[wapp-param-exists control_run]} {
-    # This is a "run test" command.
-    wapptest_run
-  }
-
-  if {[wapp-param-exists control_stop]} {
-    # A "STOP tests" command.
-    set G(state) "stopped"
-    set G(result) "Test halted by user"
-    foreach j $G(test_array) {
-      set name [dict get $j config]
-      if { [info exists G(test.$name.channel)] } {
-        close $G(test.$name.channel)
-        unset G(test.$name.channel)
-        slave_test_done $name 1
-      }
-    }
-    wapptest_closelog
-  }
-
-  if {[wapp-param-exists control_reset]} {
-    # A "reset app" command.
-    set G(state) "config"
-    wapptest_init
-  }
-
-  if {$::G(state) == "running"} {
-    do_some_stuff
-  }
-  wapp-redirect /
-}
-
-# URI: /style.css
-#
-# Return the stylesheet for the application main page.
-#
-proc wapp-page-style.css {} {
-  wapp-subst {
-
-    /* The boxes with black borders use this class */
-    .border {
-      border: 3px groove #444444;
-      padding: 1em;
-      margin-top: 1em;
-      margin-bottom: 1em;
-    }
-
-    /* Float to the right (used for the Run/Stop/Reset button) */
-    .right { float: right; }
-
-    /* Style for the large red warning at the top of the page */
-    .warning {
-      color: red;
-      font-weight: bold;
-    }
-
-    /* Styles used by cells in the test table */
-    .padleft { padding-left: 5ex; }
-    .nowrap  { white-space: nowrap; }
-
-    /* Styles for individual tests, depending on the outcome */
-    .testwait    {              }
-    .testrunning { color: blue  }
-    .testdone    { color: green }
-    .testfail    { color: red   }
-  }
-}
-
-# URI: /script/${state}.js
-#
-# The last part of this URI is always "config.js", "running.js" or 
-# "stopped.js", depending on the state of the application. It returns
-# the javascript part of the front-end for the requested state to the
-# browser.
-#
-proc wapp-page-script {} {
-  regexp {[^/]*$} [wapp-param REQUEST_URI] script
-
-  set tcl $::G(tcl)
-  set keep $::G(keep)
-  set msvc $::G(msvc)
-  set debug $::G(debug)
-  
-  wapp-subst {
-    var lElem = \["control_platform", "control_test", "control_msvc", 
-        "control_jobs", "control_debug"
-    \];
-    lElem.forEach(function(e) {
-      var elem = document.getElementById(e);
-      elem.addEventListener("change", function() { control.submit() } );
-    })
-
-    elem = document.getElementById("control_tcl");
-    elem.value = "%string($tcl)"
-
-    elem = document.getElementById("control_keep");
-    elem.checked = %string($keep);
-
-    elem = document.getElementById("control_msvc");
-    elem.checked = %string($msvc);
-
-    elem = document.getElementById("control_debug");
-    elem.checked = %string($debug);
-  }
-
-  if {$script != "config.js"} {
-    wapp-subst {
-      var lElem = \["control_platform", "control_test", 
-          "control_tcl", "control_keep", "control_msvc", 
-          "control_debug"
-      \];
-      lElem.forEach(function(e) {
-        var elem = document.getElementById(e);
-        elem.disabled = true;
-      })
-    }
-  }
-
-  if {$script == "running.js"} {
-    wapp-subst {
-      function reload_tests() {
-        fetch('tests')
-          .then( data => data.text() )
-          .then( data => {
-            document.getElementById("tests").innerHTML = data;
-          })
-          .then( data => {
-            if( document.getElementById("result") ){
-              document.location = document.location;
-            } else {
-              setTimeout(reload_tests, 1000)
-            }
-          });
-      }
-
-      setTimeout(reload_tests, 1000)
-    }
-  }
-}
-
-# URI: /env
-#
-# This is for debugging only. Serves no other purpose.
-#
-proc wapp-page-env {} {
-  wapp-allow-xorigin-params
-  wapp-trim {
-    <h1>Wapp Environment</h1>\n<pre>
-    <pre>%html([wapp-debug-env])</pre>
-  }
-}
-
-# URI: /log/dirname/test.log
-#
-# This URI reads file "dirname/test.log" from disk, wraps it in a <pre>
-# block, and returns it to the browser. Use for viewing log files.
-#
-proc wapp-page-log {} {
-  set log [string range [wapp-param REQUEST_URI] 5 end]
-  set fd [open $log]
-  set data [read $fd]
-  close $fd
-  wapp-trim {
-    <pre>
-    %html($data)
-    </pre>
-  }
-}
-
-# Print out a usage message. Then do [exit 1].
-#
-proc wapptest_usage {} {
-  puts stderr {
-This Tcl script is used to test various configurations of SQLite. By
-default it uses "wapp" to provide an interactive interface. Supported 
-command line options (all optional) are:
-
-    --platform    PLATFORM         (which tests to run)
-    --config      GLOB             (only run configurations matching GLOB)
-    --smoketest                    (run "make smoketest" only)
-    --veryquick                    (run veryquick.test only)
-    --buildonly                    (build executables, do not run tests)
-    --jobs        N                (number of concurrent jobs)
-    --tcl         DIR              (where to find tclConfig.sh)
-    --deletefiles                  (delete extra files after each test)
-    --msvc                         (Use MS Visual C)
-    --debug                        (Also run [n]debugging versions of tests)
-    --noui                         (do not use wapp)
-  }
-  exit 1
-}
-
-# Sort command line arguments into two groups: those that belong to wapp,
-# and those that belong to the application.
-set WAPPARG(-server)      1
-set WAPPARG(-local)       1
-set WAPPARG(-scgi)        1
-set WAPPARG(-remote-scgi) 1
-set WAPPARG(-fromip)      1
-set WAPPARG(-nowait)      0
-set WAPPARG(-cgi)         0
-set lWappArg [list]
-set lTestArg [list]
-for {set i 0} {$i < [llength $argv]} {incr i} {
-  set arg [lindex $argv $i]
-  if {[string range $arg 0 1]=="--"} {
-    set arg [string range $arg 1 end]
-  }
-  if {[info exists WAPPARG($arg)]} {
-    lappend lWappArg $arg
-    if {$WAPPARG($arg)} {
-      incr i
-      lappend lWappArg [lindex $argv $i]
-    }
-  } else {
-    lappend lTestArg $arg
-  }
-}
-
-wapptest_init
-for {set i 0} {$i < [llength $lTestArg]} {incr i} {
-  set opt [lindex $lTestArg $i]
-  if {[string range $opt 0 1]=="--"} {
-    set opt [string range $opt 1 end]
-  }
-  switch -- $opt {
-    -platform {
-      if {$i==[llength $lTestArg]-1} { wapptest_usage }
-      incr i
-      set arg [lindex $lTestArg $i]
-      set lPlatform [releasetest_data platforms]
-      if {[lsearch $lPlatform $arg]<0} {
-        puts stderr "No such platform: $arg. Platforms are: $lPlatform"
-        exit -1
-      }
-      set G(platform) $arg
-    }
-
-    -smoketest { set G(test) Smoketest }
-    -veryquick { set G(test) Veryquick }
-    -buildonly { set G(test) Build-Only }
-    -jobs {
-      if {$i==[llength $lTestArg]-1} { wapptest_usage }
-      incr i
-      set G(jobs) [lindex $lTestArg $i]
-    }
-
-    -tcl {
-      if {$i==[llength $lTestArg]-1} { wapptest_usage }
-      incr i
-      set G(tcl) [lindex $lTestArg $i]
-    }
-
-    -deletefiles {
-      set G(keep) 0
-    }
-
-    -msvc {
-      set G(msvc) 1
-    }
-
-    -debug {
-      set G(debug) 1
-    }
-
-    -noui {
-      set G(noui) 1
-      set G(stdout) 1
-    }
-
-    -config {
-      if {$i==[llength $lTestArg]-1} { wapptest_usage }
-      incr i
-      set G(cfgglob) [lindex $lTestArg $i]
-    }
-
-    -stdout {
-      set G(stdout) 1
-    }
-
-    default {
-      puts stderr "Unrecognized option: [lindex $lTestArg $i]"
-      wapptest_usage
-    }
-  }
-}
-
-if {$G(noui)==0} {
-  wapp-start $lWappArg
-} else {
-  wapptest_run
-  do_some_stuff
-  vwait forever
-}
diff --git a/test/where.test b/test/where.test
index 0a8cfd572b..c377006fb9 100644
--- a/test/where.test
+++ b/test/where.test
@@ -1388,7 +1388,7 @@ do_execsql_test where-19.0 {
   SELECT t191.rowid FROM t192, t191 WHERE (a=y OR b=y) AND x=?1;
 } {/.* sqlite_autoindex_t191_1 .* sqlite_autoindex_t191_2 .*/}
 
-# 2018-04-24 ticket [https://www.sqlite.org/src/info/4ba5abf65c5b0f9a]
+# 2018-04-24 ticket [https://sqlite.org/src/info/4ba5abf65c5b0f9a]
 # Index on expressions leads to an incorrect answer for a LEFT JOIN
 #
 do_execsql_test where-20.0 {
@@ -1422,7 +1422,7 @@ do_execsql_test where-21.1 {
   4 0 1
 }
 
-# 2018-11-05: ticket [https://www.sqlite.org/src/tktview/65eb38f6e46de8c75e188a]
+# 2018-11-05: ticket [https://sqlite.org/src/tktview/65eb38f6e46de8c75e188a]
 # Incorrect result in LEFT JOIN when STAT4 is enabled.
 #
 sqlite3 db :memory:
@@ -1435,7 +1435,7 @@ do_execsql_test where-22.1 {
 } {5}
 
 # 20190-02-22:  A bug introduced by checkin
-# https://www.sqlite.org/src/info/fa792714ae62fa98.
+# https://sqlite.org/src/info/fa792714ae62fa98.
 #
 do_execsql_test where-23.0 {
   DROP TABLE IF EXISTS t1;
@@ -1547,7 +1547,7 @@ do_catchsql_test where-25.5 {
     ON CONFLICT(c) DO UPDATE SET b=NULL
 } {1 {corrupt database}}
 
-# 2019-08-21 Ticket https://www.sqlite.org/src/info/d9f584e936c7a8d0
+# 2019-08-21 Ticket https://sqlite.org/src/info/d9f584e936c7a8d0
 #
 db close
 sqlite3 db :memory:
diff --git a/test/where2.test b/test/where2.test
index 7a7e9b92ed..83740bffdf 100644
--- a/test/where2.test
+++ b/test/where2.test
@@ -767,7 +767,7 @@ do_execsql_test where2-13.1 {
   SELECT * FROM t13 WHERE (1=2 AND a=3) OR a=4;
 } {4 5}
 
-# https://www.sqlite.org/src/info/5e3c886796e5512e  (2016-03-09)
+# https://sqlite.org/src/info/5e3c886796e5512e  (2016-03-09)
 # Correlated subquery on the RHS of an IN operator 
 #
 do_execsql_test where2-14.1 {
@@ -778,4 +778,20 @@ do_execsql_test where2-14.1 {
   SELECT x FROM t14a WHERE x NOT IN (SELECT x FROM t14b);
 } {}
 
+# Demonstrate that the code removed by
+# https://sqlite.org/src/info/2025-09-17T17:09:07Z is in fact
+# necessary.  Answer confirmed by PostgreSQL
+#
+do_execsql_test where2-15.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b INT);
+  INSERT INTO t1 VALUES(12,34),(56,78),(90,12);
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(x INT UNIQUE);
+  INSERT INTO t2 VALUES(78),(12);
+  SELECT a,b,quote(x),'|'
+    FROM t1 LEFT JOIN (SELECT x FROM t2) AS s1 ON t1.b=s1.x
+   ORDER BY a,EXISTS(SELECT 1 FROM t1 LEFT JOIN (SELECT x AS y FROM t2) AS s2 ON t1.b=s2.y),x;
+} {12 34 NULL | 56 78 78 | 90 12 12 |}
+
 finish_test
diff --git a/test/where3.test b/test/where3.test
index 20af995cfe..f574c0d55f 100644
--- a/test/where3.test
+++ b/test/where3.test
@@ -492,5 +492,27 @@ foreach disabled_opt {none omit-noop-join all} {
   } {123}
 }
 
+# 2023-12-23
+# https://sqlite.org/forum/forumpost/2568d1f6e6
+#
+# Index usage should be "x=? and y=?" - equality on both values.
+# Not: "x=? AND y>?" - inequality on "y"
+# 
+reset_db
+do_execsql_test where3-8.1 {
+  CREATE TABLE t1(a,b,c,d);  INSERT INTO t1 VALUES(1,2,3,4);
+  CREATE TABLE t2(x,y);      INSERT INTO t2 VALUES(3,4);
+  CREATE INDEX t2xy ON t2(x,y);
+  SELECT 1 FROM t1 JOIN t2 ON x=c AND y=d WHERE d>0;
+} 1
+do_eqp_test where3-8.2 {
+  SELECT 1 FROM t1 JOIN t2 ON x=c AND y=d WHERE d>0;
+} {
+  QUERY PLAN
+  |--SCAN t1
+  `--SEARCH t2 USING COVERING INDEX t2xy (x=? AND y=?)
+}
+
+
 
 finish_test
diff --git a/test/where7.test b/test/where7.test
index f76e8aba8b..681684b809 100644
--- a/test/where7.test
+++ b/test/where7.test
@@ -47,18 +47,33 @@ do_test where7-1.1 {
     SELECT * FROM t1;
   }
 } {1 2 3 4 2 3 4 5 3 4 6 8 4 5 10 15 5 10 100 1000}
-do_execsql_test where7-1.1.1 {
-  CREATE TABLE t(a);
-  CREATE INDEX ta ON t(a);
-  INSERT INTO t(a) VALUES(1),(2);
-  SELECT * FROM t ORDER BY a;
-  SELECT * FROM t WHERE a<2 OR a<3 ORDER BY a;
-  PRAGMA count_changes=ON;
-  DELETE FROM t WHERE a<2 OR a<3;
-  SELECT * FROM t;
-  PRAGMA count_changes=OFF;
-  DROP TABLE t;
-} {1 2 1 2 2}
+if {[permutation] != "no_optimization"} {
+  do_execsql_test where7-1.1.1 {
+    CREATE TABLE t(a);
+    CREATE INDEX ta ON t(a);
+    INSERT INTO t(a) VALUES(1),(2);
+    SELECT * FROM t ORDER BY a;
+    SELECT * FROM t WHERE a<2 OR a<3 ORDER BY a;
+    PRAGMA count_changes=ON;
+    DELETE FROM t WHERE a<2 OR a<3;
+    SELECT * FROM t;
+    PRAGMA count_changes=OFF;
+    DROP TABLE t;
+  } {1 2 1 2 2}
+} else {
+  do_execsql_test where7-1.1.1-noopt {
+    CREATE TABLE t(a);
+    CREATE INDEX ta ON t(a);
+    INSERT INTO t(a) VALUES(1),(2);
+    SELECT * FROM t ORDER BY a;
+    SELECT * FROM t WHERE a<2 OR a<3 ORDER BY a;
+    PRAGMA count_changes=ON;
+    DELETE FROM t WHERE a<2 OR a<3;
+    SELECT * FROM t;
+    PRAGMA count_changes=OFF;
+    DROP TABLE t;
+  } {1 2 1 2 3}
+}
 do_test where7-1.2 {
   count_steps {
     SELECT a FROM t1 WHERE b=3 OR c=6 ORDER BY a
diff --git a/test/whereG.test b/test/whereG.test
index 6ca363ed8b..6ee8634817 100644
--- a/test/whereG.test
+++ b/test/whereG.test
@@ -237,7 +237,7 @@ do_eqp_test 5.3.3 {
 } {SCAN t1}
 
 # 2015-06-18
-# Ticket [https://www.sqlite.org/see/tktview/472f0742a1868fb58862bc588ed70]
+# Ticket [https://sqlite.org/see/tktview/472f0742a1868fb58862bc588ed70]
 #
 do_execsql_test 6.0 {
   DROP TABLE IF EXISTS t1;
@@ -273,7 +273,7 @@ do_execsql_test 7.3 {
 } {1 3 1 4 9 3 9 4}
 
 # 2019-08-22
-# Ticket https://www.sqlite.org/src/info/7e07a3dbf5a8cd26
+# Ticket https://sqlite.org/src/info/7e07a3dbf5a8cd26
 #
 do_execsql_test 8.1 {
   DROP TABLE IF EXISTS t0;
@@ -311,6 +311,20 @@ do_execsql_test 8.9 {
 do_execsql_test 8.10 {
   SELECT * FROM t0 WHERE likelihood(t0.rowid <= '0', 0.5);
 } {}
+# Forum https://sqlite.org/forum/forumpost/45ec3d9788
+reset_db
+do_execsql_test 8.11 {
+  CREATE TABLE t1(c0 INT);
+  INSERT INTO t1(c0) VALUES (NULL);
+  CREATE INDEX i46 ON t1(CAST( (c0 IS TRUE) AS TEXT));
+  CREATE VIEW v0(c2) AS SELECT CAST( (c0 IS TRUE) AS TEXT ) FROM t1;
+}
+do_execsql_test 8.12 {
+  SELECT quote(c0), quote(c2) FROM t1, v0 WHERE  (0 < LIKELY(v0.c2));
+} {NULL '0'} 
+do_execsql_test 8.13 {
+  SELECT quote(c0), quote(c2) FROM t1, v0 WHERE  (0 < LIKELY(v0.c2)) IS TRUE;
+} {NULL '0'}
 
 # 2019-12-31: assertion fault discovered by Yongheng's fuzzer.
 # Harmless memIsValid() due to the code generators failure to
diff --git a/test/whereL.test b/test/whereL.test
index c3bdcb8f34..ffbae02b8d 100644
--- a/test/whereL.test
+++ b/test/whereL.test
@@ -49,6 +49,33 @@ do_eqp_test 120 {
   |--SEARCH t2 USING INDEX sqlite_autoindex_t2_1 (a=?)
   `--SCAN t3
 }
+do_eqp_test 121 {
+  SELECT * FROM t1, t2, t3
+   WHERE t1.a=t2.a AND t2.a=t3.j AND t3.j=abs(5)
+  ORDER BY t1.a;
+} {
+  QUERY PLAN
+  |--SEARCH t1 USING INDEX sqlite_autoindex_t1_1 (a=?)
+  |--SEARCH t2 USING INDEX sqlite_autoindex_t2_1 (a=?)
+  `--SCAN t3
+}
+
+# The sqlite3ExprIsConstant() routine does not believe that
+# the expression "coalesce(5,random())" is constant.  So the
+# optimization does not apply in this case.
+# 
+sqlite3_create_function db
+do_eqp_test 122 {
+  SELECT * FROM t1, t2, t3
+   WHERE t1.a=t2.a AND t2.a=t3.j AND t3.j=coalesce(5,random())
+  ORDER BY t1.a;
+} {
+  QUERY PLAN
+  |--SCAN t3
+  |--SEARCH t1 USING INDEX sqlite_autoindex_t1_1 (a=?)
+  |--SEARCH t2 USING INDEX sqlite_autoindex_t2_1 (a=?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
 
 # Constant propagation in the face of collating sequences:
 #
@@ -209,4 +236,62 @@ do_eqp_test 710 {
   `--SEARCH t1 USING INDEX idx (<expr>=?)
 }
 
+# 2024-03-07 https://sqlite.org/forum/forumpost/ecdfc02339
+# A refinement is needed to the enhancements tested by the prior test case
+# to avoid another problem with indexes on constant expressions.
+#
+reset_db
+db null NULL
+do_execsql_test 800 {
+  CREATE TABLE t0(c0, c1);
+  CREATE TABLE t1(c2);
+  CREATE INDEX i0 ON t1(NULL);
+  INSERT INTO t1(c2) VALUES (0.2);
+  CREATE VIEW v0(c3) AS SELECT DISTINCT c2 FROM t1;
+  SELECT * FROM v0 LEFT JOIN t0 ON c3<NULL LEFT JOIN t1 ON 1;
+} {0.2 NULL NULL 0.2}
+do_execsql_test 810 {
+  SELECT * FROM v0 LEFT JOIN t0 ON c3<NULL LEFT JOIN t1 ON 1 WHERE c2/0.1;
+} {0.2 NULL NULL 0.2}
+
+#-------------------------------------------------------------------------
+# 2025-04-10 https://sqlite.org/forum/forumpost/0109bca824
+reset_db
+
+do_execsql_test 900 {
+  SELECT * FROM (SELECT 1.0 AS abc) WHERE abc=1;
+} {1.0}
+do_execsql_test 910 {
+  SELECT * FROM (SELECT 1.0 AS abc) WHERE abc LIKE '1.0';
+} {1.0}
+do_execsql_test 920 {
+  SELECT * FROM (SELECT 1.0 AS abc) WHERE abc=1 AND abc LIKE '1.0';
+} {1.0}
+
+do_execsql_test 930 {
+  CREATE TABLE IF NOT EXISTS t0 (c0 BLOB);
+  CREATE TABLE IF NOT EXISTS t1 (c0 INTEGER);
+
+  INSERT INTO t1 VALUES ('1');
+  INSERT INTO t0 VALUES (''), (''), ('2');
+}
+
+do_execsql_test 940 {
+  SELECT *
+    FROM (SELECT 0.0 AS col_0) as subQuery
+    LEFT JOIN t0 ON ((CASE ''
+          WHEN t0.c0 THEN subQuery.col_0
+          ELSE (t0.c0) END) LIKE (((((subQuery.col_0))))))
+    LEFT JOIN t1 ON ((subQuery.col_0) == (false));
+} {0.0 {} 1 0.0 {} 1}
+
+do_execsql_test 950 {
+  SELECT *
+    FROM (SELECT 0.0 AS col_0) as subQuery
+    LEFT JOIN t0 ON ((CASE ''
+          WHEN t0.c0 THEN subQuery.col_0
+          ELSE (t0.c0) END) LIKE (((((subQuery.col_0))))))
+    LEFT JOIN t1 ON ((subQuery.col_0) == (false)) WHERE t1.c0;
+} {0.0 {} 1 0.0 {} 1}
+
 finish_test
diff --git a/test/whereN.test b/test/whereN.test
new file mode 100644
index 0000000000..b9b889fa28
--- /dev/null
+++ b/test/whereN.test
@@ -0,0 +1,103 @@
+# 2024-04-02
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Tests for the whereInterstageHeuristic() routine in the query planner.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix whereN
+
+# The following is a simplified and "sanitized" version of the original
+# real-world query that brought the problem to light.
+#
+# The issue is a slow query.  The answer is correct, but it was taking too
+# much time, because it was doing a full table scan rather than an indexed
+# lookup.
+#
+# The problem was that the query planner was overestimating the number of
+# output rows.  The estimated number of output rows is accurate if the
+# DSNAME parameter is "ds-one".  In that case, a large fraction of the rows
+# in "violation" end up being output.  The query planner correctly deduces
+# that it is faster to do a full table scan of the large "violation" table
+# to avoid the after-query sort that implements the ORDER BY clause. However,
+# if the DSNAME is "ds-two", then only a few rows (about 6) are generated,
+# and it is much much faster to do an indexed lookup of "violation" followed
+# by a sort operation to implement ORDER BY
+#
+# The problem, of course, is that the query planner has no way of knowing
+# in advance how many rows will be generated.  The query planner tries to
+# estimate a worst case, which is a large number of output rows, and it picks
+# the best plan for that case.  However, the plan choosen is very inefficient
+# when the number of output rows is small.
+#
+# The whereInterstageHeuristic() routine in the query planner attempts to
+# correct this by adjusting the query plan such that it avoids the very bad
+# query plan for a small number of rows, at the expense of a slightly less
+# efficient plan for a large number of rows.  The large number of rows case
+# is perhaps 5% slower with the revised plan, but the small number of
+# rows case is around 100 times faster.  That seems like a good tradeoff.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE datasource(dsid INT, name TEXT);
+  INSERT INTO datasource VALUES(1,'ds-one'),(2,'ds-two'),(3,'ds-three');
+  CREATE INDEX ds1 ON datasource(name, dsid);
+
+  CREATE TABLE rule(rid INT, team_id INT, dsid INT);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<9)
+    INSERT INTO rule(rid,team_id,dsid) SELECT n, 1, 1 FROM c;
+  WITH RECURSIVE c(n) AS (VALUES(10) UNION ALL SELECT n+1 FROM c WHERE n<24)
+    INSERT INTO rule(rid,team_id,dsid) SELECT n, 2, 2 FROM c;
+  CREATE INDEX rule2 ON rule(dsid, rid);
+
+  CREATE TABLE violation(vid INT, rid INT, vx BLOB);
+  /***  Uncomment to insert actual data
+  WITH src(rid, cnt) AS (VALUES(1,3586),(2,1343),(3,6505),(5,76230),
+                               (6,740),(7,287794),(8,457),(12,1),
+                               (14,1),(16,1),(17,1),(18,1),(19,1))
+    INSERT INTO violation(vid, rid, vx)
+      SELECT rid*1000000+value, rid, randomblob(15)
+        FROM src, generate_series(1,cnt);
+  ***/
+  CREATE INDEX v1 ON violation(rid, vid);
+  CREATE INDEX v2 ON violation(vid);
+  ANALYZE;
+  DELETE FROM sqlite_stat1;
+  DROP TABLE IF EXISTS sqlite_stat4;
+  INSERT INTO sqlite_stat1 VALUES
+    ('violation','v2','376661 1'),
+    ('violation','v1','376661 28974 1'),
+    ('rule','rule2','24 12 1'),
+    ('datasource','ds1','3 1 1');
+  ANALYZE sqlite_schema;
+}
+set DSNAME ds-two   ;#  Only a few rows.  Change to "ds-one" for many rows.
+do_eqp_test 1.1 {
+  SELECT count(*), length(group_concat(vx)) FROM (
+    SELECT V.*
+      FROM datasource DS, rule R, violation V
+     WHERE V.rid=R.rid
+       AND R.dsid=DS.dsid
+       AND DS.name=$DSNAME
+     ORDER BY V.vid desc
+  );
+} {
+  QUERY PLAN
+  |--CO-ROUTINE (subquery-xxxxxx)
+  |  |--SEARCH DS USING COVERING INDEX ds1 (name=?)
+  |  |--SEARCH R USING COVERING INDEX rule2 (dsid=?)
+  |  |--SEARCH V USING INDEX v1 (rid=?)
+  |  `--USE TEMP B-TREE FOR ORDER BY
+  `--SCAN (subquery-xxxxxx)
+}
+#       ^^^^---- We want to see three SEARCH terms.  No SCAN terms.
+#                The ORDER BY is implemented by a separate sorter pass.
+
+finish_test
diff --git a/test/wherelimit2.test b/test/wherelimit2.test
index 8e39127ac8..57288bf64c 100644
--- a/test/wherelimit2.test
+++ b/test/wherelimit2.test
@@ -299,5 +299,35 @@ do_test 5.5 {
   set ::log
 } {ax a bx b cx c dx d ex a}
 
+#-----------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t2(x);  
+  INSERT INTO t2(x) VALUES(1),(2),(3),(5),(8),(13);
+} {}
+
+do_execsql_test 6.1 {
+  WITH t2 AS MATERIALIZED (VALUES(5))
+  DELETE FROM t2 ORDER BY rank()OVER() LIMIT 2;
+}
+
+do_execsql_test 6.2 {
+  SELECT * FROM t2;
+} {3 5 8 13}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 7.0 {
+  CREATE TABLE t1(a INT); INSERT INTO t1(a) VALUES(0);
+} {}
+
+do_execsql_test 7.1 {
+  WITH t1(b) AS (SELECT * FROM (SELECT * FROM (VALUES(2))))
+  UPDATE t1 SET a=3 LIMIT 1;
+}
+
+do_execsql_test 7.2 {
+  SELECT * FROM t1;
+} {3}
 
 finish_test
diff --git a/test/wherelimit3.test b/test/wherelimit3.test
new file mode 100644
index 0000000000..dea3e97d86
--- /dev/null
+++ b/test/wherelimit3.test
@@ -0,0 +1,67 @@
+# 2024-06-06
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for query plans using LIMIT
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix wherelimit3
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INT, b INT);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1000)
+    INSERT INTO t1 SELECT n, n FROM c;
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  ANALYZE;
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT 5;
+} {
+  QUERY PLAN
+  |--SEARCH t1 USING INDEX t1a (a>? AND a<?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
+ifcapable stat4 {
+  do_eqp_test 1.2 {
+    SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT -1;
+  } {
+    QUERY PLAN
+    `--SCAN t1 USING INDEX t1b
+  }
+}
+
+set N [expr 5]
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N;
+} {
+  QUERY PLAN
+  |--SEARCH t1 USING INDEX t1a (a>? AND a<?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
+
+ifcapable stat4 {
+  set N [expr -1]
+  do_eqp_test 1.4 {
+    SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N;
+  } {
+    QUERY PLAN
+    `--SCAN t1 USING INDEX t1b
+  }
+}
+
+
+
+
+
+finish_test
diff --git a/test/win32heap.test b/test/win32heap.test
index 82a3f6b663..b62126b772 100644
--- a/test/win32heap.test
+++ b/test/win32heap.test
@@ -12,7 +12,7 @@
 # focus of this script is the Win32 heap implementation.
 #
 
-if {$tcl_platform(platform)!="windows"} return
+if {$tcl_platform(platform) ne "windows"} return
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
diff --git a/test/win32lock.test b/test/win32lock.test
index fbb2dd13cd..40ff5d5127 100644
--- a/test/win32lock.test
+++ b/test/win32lock.test
@@ -15,7 +15,7 @@
 # TESTRUNNER: slow
 #
 
-if {$tcl_platform(platform)!="windows"} return
+if {$tcl_platform(platform) ne "windows"} return
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
diff --git a/test/win32longpath.test b/test/win32longpath.test
index 01b4af70ac..2545c55a16 100644
--- a/test/win32longpath.test
+++ b/test/win32longpath.test
@@ -13,7 +13,7 @@
 # by the "win32-longpath" VFS.
 #
 
-if {$tcl_platform(platform)!="windows"} return
+if {$tcl_platform(platform) ne "windows"} return
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -49,19 +49,19 @@ set longPath(1) \\\\?\\$path\\[pid]
 set uriPath(1a) %5C%5C%3F%5C$path\\[pid]
 set uriPath(1b) %5C%5C%3F%5C$rawPath/[pid]
 
-make_win32_dir $longPath(1)
+file mkdir $longPath(1)
 
 set longPath(2) $longPath(1)\\[string repeat X 255]
 set uriPath(2a) $uriPath(1a)\\[string repeat X 255]
 set uriPath(2b) $uriPath(1b)/[string repeat X 255]
 
-make_win32_dir $longPath(2)
+file mkdir $longPath(2)
 
 set longPath(3) $longPath(2)\\[string repeat Y 255]
 set uriPath(3a) $uriPath(2a)\\[string repeat Y 255]
 set uriPath(3b) $uriPath(2b)/[string repeat Y 255]
 
-make_win32_dir $longPath(3)
+file mkdir $longPath(3)
 
 set fileName $longPath(3)\\test.db
 
@@ -92,7 +92,6 @@ do_test 1.4 {
 } {5 6 7 8}
 
 db3 close
-# puts "  Database exists \{[exists_win32_path $fileName]\}"
 
 sqlite3 db3 $fileName -vfs win32-longpath
 
@@ -115,9 +114,13 @@ do_test 1.6 {
 } {5 6 7 8 9 10 11 12}
 
 db3 close
-# puts "  Database exists \{[exists_win32_path $fileName]\}"
 
-foreach tn {1a 1b 1c 1d 1e 1f} {
+# winrt platforms do not handle paths with unix-style '/' directory separators.
+#
+set lUri [list 1a 1b 1c 1d 1e 1f]
+ifcapable winrt { set lUri [list 1a 1c 1e] }
+
+foreach tn $lUri {
   sqlite3 db3 $uri($tn) -vfs win32-longpath -uri 1 -translatefilename 0
 
   do_test 1.7.$tn {
@@ -129,11 +132,20 @@ foreach tn {1a 1b 1c 1d 1e 1f} {
   db3 close
 }
 
-do_delete_win32_file $fileName
-# puts "  Files remaining \{[find_win32_file $longPath(3)\\*]\}"
+# These over-length file and directory names are difficult to delete.
+# The "file delete -force" might not work, depending on the TCL build
+# being used.  So first try to delete using the windows rmdir command.
+#
+set fd [open cleanup.bat w]
+puts $fd "rmdir /q /s $longPath(1)"
+close $fd
+if {[catch {exec cleanup.bat} msg]} {
+  puts "Command \[cleanup.bat\] returns $msg"
+}
 
-do_remove_win32_dir $longPath(3)
-do_remove_win32_dir $longPath(2)
-do_remove_win32_dir $longPath(1)
+file delete -force $fileName
+file delete -force $longPath(3)
+file delete -force $longPath(2)
+file delete -force $longPath(1)
 
 finish_test
diff --git a/test/win32nolock.test b/test/win32nolock.test
index 8128860392..e6ff548b46 100644
--- a/test/win32nolock.test
+++ b/test/win32nolock.test
@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.
 #
 
-if {$tcl_platform(platform)!="windows"} return
+if {$tcl_platform(os) ne "Windows NT"} return
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
diff --git a/test/window1.test b/test/window1.test
index 783a739e3f..db9af62ac8 100644
--- a/test/window1.test
+++ b/test/window1.test
@@ -158,7 +158,7 @@ do_execsql_test 4.9 {
 do_execsql_test 4.10.1 {
   SELECT a, 
     count() OVER (ORDER BY a DESC),
-    group_concat(a, '.') OVER (ORDER BY a DESC) 
+    string_agg(a, '.') OVER (ORDER BY a DESC) 
   FROM t2 ORDER BY a DESC
 } {
   6 1 6
@@ -630,7 +630,7 @@ do_execsql_test 13.5 {
 } {}
 
 # 2018-12-06
-# https://www.sqlite.org/src/info/f09fcd17810f65f7
+# https://sqlite.org/src/info/f09fcd17810f65f7
 # Assertion fault when window functions are used.
 #
 # Root cause is the query flattener invoking sqlite3ExprDup() on
@@ -657,7 +657,7 @@ do_execsql_test 14.1 {
 } {1 2 3}
 
 # 2018-12-31
-# https://www.sqlite.org/src/info/d0866b26f83e9c55
+# https://sqlite.org/src/info/d0866b26f83e9c55
 # Window function in correlated subquery causes assertion fault 
 #
 do_catchsql_test 15.0 {
@@ -825,7 +825,7 @@ foreach {tn sql error} {
 }
 
 do_execsql_test 18.3.1 {
-  SELECT group_concat(c, '.') OVER (PARTITION BY b ORDER BY c)
+  SELECT string_agg(c, '.') OVER (PARTITION BY b ORDER BY c)
   FROM t1
 } {four four.six four.six.two five five.one five.one.three}
 
@@ -836,7 +836,7 @@ do_execsql_test 18.3.2 {
 } {four four.six four.six.two five five.one five.one.three}
 
 do_execsql_test 18.3.3 {
-  SELECT group_concat(c, '.') OVER win2
+  SELECT string_agg(c, '.') OVER win2
   FROM t1
   WINDOW win1 AS (PARTITION BY b),
          win2 AS (win1 ORDER BY c)
@@ -850,7 +850,7 @@ do_execsql_test 18.3.4 {
 } {four four.six four.six.two five five.one five.one.three}
 
 do_execsql_test 18.3.5 {
-  SELECT group_concat(c, '.') OVER win5
+  SELECT string_agg(c, '.') OVER win5
   FROM t1
   WINDOW win1 AS (PARTITION BY b),
          win2 AS (win1),
@@ -1127,7 +1127,7 @@ do_execsql_test 28.1.1 {
 }
 
 do_execsql_test 28.1.2 {
-  SELECT group_concat(b,'') OVER w1 FROM t1
+  SELECT string_agg(b,'') OVER w1 FROM t1
     WINDOW w1 AS (ORDER BY a RANGE BETWEEN 3 PRECEDING AND 1 PRECEDING)
 } {
   {} {}
@@ -1205,7 +1205,7 @@ do_execsql_test 29.2 {
 
 # 2019-07-18
 # Check-in [7ef7b23cbb1b9ace] (which was itself a fix for ticket
-# https://www.sqlite.org/src/info/1be72aab9) introduced a new problem
+# https://sqlite.org/src/info/1be72aab9) introduced a new problem
 # if the LHS of a BETWEEN operator is a WINDOW function.  The problem
 # was found by (the recently enhanced) dbsqlfuzz.
 #
@@ -1881,7 +1881,7 @@ do_catchsql_test 57.3 {
     SELECT max(y) OVER( ORDER BY (SELECT x FROM (SELECT sum(y) AS x FROM t1)))
   )
   FROM t3;
-} {1 {misuse of aggregate: sum()}}
+} {0 5}
 
 # 2020-06-06 ticket 1f6f353b684fc708
 reset_db
@@ -2363,4 +2363,65 @@ do_execsql_test 76.5 {
   SELECT (SELECT max(y)+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y GROUP BY x;
 } {100 {} 400}
 
+# 2023-05-23 https://sqlite.org/forum/forumpost/fbfe330a20
+#
+reset_db
+do_execsql_test 77.1 {
+  CREATE TABLE t1(x INT);
+  CREATE INDEX t1x ON t1(likely(x));
+  INSERT INTO t1 VALUES(1),(2),(4),(8);
+}
+do_execsql_test 77.2 {
+  SELECT max(~likely(x)) FILTER (WHERE true) FROM t1 INDEXED BY t1x GROUP BY x;
+} {-2 -3 -5 -9}
+
+# 2024-05-23 https://sqlite.org/forum/forumpost/bf8f43aa522c2299
+#
+# A bug in group_concat() when used as a window function, reported
+# just hours after the 3.46.0 release, though first appearing
+# in 3.28.0.
+#
+# When used as a window function, a group_concat() was not
+# correctly distinguishing between NULL and empty-string for
+# its return value.
+#
+do_execsql_test 78.1 {
+  SELECT quote(group_concat(x) OVER ()) FROM (SELECT '' AS x);
+} ''
+do_execsql_test 78.2 {
+  SELECT quote(group_concat(x) OVER (
+      ORDER BY y RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING
+    )) FROM (SELECT 'abc' AS x, 1 AS y);
+} NULL
+
+#-------------------------------------------------------------------------
+# Test that the following queries do not run for a very long time.
+#
+#   https://sqlite.org/forum/forumpost/b1993c858f
+#
+do_execsql_test 79.0 {
+  CREATE TABLE t0 (c0 INTEGER );
+  INSERT INTO t0 VALUES(1);
+  INSERT INTO t0 VALUES(2);
+  INSERT INTO t0 VALUES(3);
+}
+
+do_execsql_test 79.1 {
+  SELECT COUNT(*) 
+  OVER ( ROWS BETWEEN 0 FOLLOWING AND 100 FOLLOWING) 
+  FROM t0;
+} {3 2 1}
+
+do_execsql_test 79.2 {
+  SELECT COUNT(*) 
+  OVER ( ORDER BY c0 RANGE BETWEEN 0 FOLLOWING AND 10_000_000_000 FOLLOWING ) 
+  FROM t0;
+} {3 2 1}
+
+do_execsql_test 79.3 {
+  SELECT sum(c0), COUNT(*) 
+  OVER ( ORDER BY c0 RANGE BETWEEN 0 FOLLOWING AND 10_000_000_000 FOLLOWING ) 
+  FROM t0;
+} {6 1}
+
 finish_test
diff --git a/test/window3.test b/test/window3.test
index 4f759abb7e..1893b539d3 100644
--- a/test/window3.test
+++ b/test/window3.test
@@ -1181,7 +1181,7 @@ do_execsql_test 1.1.13.6 {
   {}   {}   {}   {}   {}   {}   {}   {}   {}   {}   {}}
 
 do_execsql_test 1.1.14.1 {
-  SELECT group_concat(CAST(b AS TEXT), '.') OVER (ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM t2
+  SELECT string_agg(CAST(b AS TEXT), '.') OVER (ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM t2
 } {89   89.81   89.81.96   89.81.96.59   89.81.96.59.38   89.81.96.59.38.68
   89.81.96.59.38.68.39   89.81.96.59.38.68.39.62   89.81.96.59.38.68.39.62.91
   89.81.96.59.38.68.39.62.91.46   89.81.96.59.38.68.39.62.91.46.6
@@ -1471,7 +1471,7 @@ do_execsql_test 1.1.14.2 {
   89.59.39.99.29.59.89.89.29.9.79.49.59.29.59.19.39.9.9.99.69.39}
 
 do_execsql_test 1.1.14.3 {
-  SELECT group_concat(CAST(b AS TEXT), '.') OVER ( ORDER BY b,a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) FROM t2
+  SELECT string_agg(CAST(b AS TEXT), '.') OVER ( ORDER BY b,a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) FROM t2
 } {1   1.1   1.1.2   1.1.2.2   1.1.2.2.3   1.1.2.2.3.3   1.1.2.2.3.3.4
   1.1.2.2.3.3.4.5   1.1.2.2.3.3.4.5.6   1.1.2.2.3.3.4.5.6.7
   1.1.2.2.3.3.4.5.6.7.7   1.1.2.2.3.3.4.5.6.7.7.7   1.1.2.2.3.3.4.5.6.7.7.7.8
@@ -1758,7 +1758,7 @@ do_execsql_test 1.1.14.4 {
   9.9.9.19.29.29.29.39.39.39.49.59.59.59.59.69.79.89.89.89.99.99}
 
 do_execsql_test 1.1.14.5 {
-  SELECT group_concat(CAST(b AS TEXT), '.') OVER ( ORDER BY b%10,a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) FROM t2
+  SELECT string_agg(CAST(b AS TEXT), '.') OVER ( ORDER BY b%10,a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ) FROM t2
 } {90   90.40   90.40.30   90.40.30.80   90.40.30.80.20   90.40.30.80.20.90
   90.40.30.80.20.90.60   90.40.30.80.20.90.60.70   90.40.30.80.20.90.60.70.80
   90.40.30.80.20.90.60.70.80.90   90.40.30.80.20.90.60.70.80.90.30
@@ -1960,7 +1960,7 @@ do_execsql_test 1.1.14.6 {
   83   27   17   7}
 
 do_execsql_test 1.1.14.7 {
-  SELECT group_concat(CAST(b AS TEXT), '.') OVER (win1 ORDER BY b%10 RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) 
+  SELECT string_agg(CAST(b AS TEXT), '.') OVER (win1 ORDER BY b%10 RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) 
     FROM t2
     WINDOW win1 AS (PARTITION BY b%2,a)
     ORDER BY 1
diff --git a/test/window8.tcl b/test/window8.tcl
index 69ad0ad263..fcb18249ae 100644
--- a/test/window8.tcl
+++ b/test/window8.tcl
@@ -489,6 +489,68 @@ execsql_test 9.2 {
   FROM t1
 }
 
+==========
+
+execsql_test 10.0 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER, b INTEGER);
+  INSERT INTO t1 VALUES (10, 1), 
+                        (20, -1), 
+                        (5, 2), 
+                        (15, 0), 
+                        (25, 3);
+}
+
+execsql_test 10.1 {
+  SELECT 
+    a, b, MIN(a) FILTER(WHERE b > 0) OVER win 
+    FROM t1
+    WINDOW win AS (ORDER BY a ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING);
+}
+
+execsql_test 10.2 {
+  SELECT 
+    a, b, MIN(a) FILTER(WHERE b > 0) OVER win 
+    FROM t1
+    WINDOW win AS ();
+}
+
+execsql_test 10.3 {
+  SELECT 
+    a, b, MIN(a) FILTER(WHERE b > 0) OVER win 
+    FROM t1
+    WINDOW win AS (ORDER BY a);
+}
+
+execsql_test 10.4 {
+  SELECT 
+    a, b, MIN(a) OVER win 
+    FROM t1
+    WINDOW win AS (ORDER BY a ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING);
+}
+
+==========
+
+execsql_test 11.0 {
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(a INTEGER, b INTEGER);
+  INSERT INTO t2 VALUES(1, 12);
+  INSERT INTO t2 VALUES(2, 10);
+  INSERT INTO t2 VALUES(3, 15);
+  INSERT INTO t2 VALUES(4, 22);
+  INSERT INTO t2 VALUES(5,  1);
+  INSERT INTO t2 VALUES(6,  4);
+  INSERT INTO t2 VALUES(7,  7);
+  INSERT INTO t2 VALUES(8,  6);
+  INSERT INTO t2 VALUES(9, 22);
+  INSERT INTO t2 VALUES(10, 2);
+}
+
+execsql_test 11.1 {
+  SELECT a, min(b) FILTER (WHERE a%2 != 0) OVER win
+  FROM t2
+  WINDOW win AS (ORDER BY a ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING);
+}
 
 finish_test
 
diff --git a/test/window8.test b/test/window8.test
index 0c5d39badb..b1b28f1c2f 100644
--- a/test/window8.test
+++ b/test/window8.test
@@ -6540,4 +6540,67 @@ do_execsql_test 9.2 {
   FROM t1
 } {}
 
+#==========================================================================
+
+do_execsql_test 10.0 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER, b INTEGER);
+  INSERT INTO t1 VALUES (10, 1), 
+                        (20, -1), 
+                        (5, 2), 
+                        (15, 0), 
+                        (25, 3);
+} {}
+
+do_execsql_test 10.1 {
+  SELECT 
+    a, b, MIN(a) FILTER(WHERE b > 0) OVER win 
+    FROM t1
+    WINDOW win AS (ORDER BY a ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING);
+} {5 2 5   10 1 5   15 0 10   20 -1 25   25 3 25}
+
+do_execsql_test 10.2 {
+  SELECT 
+    a, b, MIN(a) FILTER(WHERE b > 0) OVER win 
+    FROM t1
+    WINDOW win AS ();
+} {10 1 5   20 -1 5   5 2 5   15 0 5   25 3 5}
+
+do_execsql_test 10.3 {
+  SELECT 
+    a, b, MIN(a) FILTER(WHERE b > 0) OVER win 
+    FROM t1
+    WINDOW win AS (ORDER BY a);
+} {5 2 5   10 1 5   15 0 5   20 -1 5   25 3 5}
+
+do_execsql_test 10.4 {
+  SELECT 
+    a, b, MIN(a) OVER win 
+    FROM t1
+    WINDOW win AS (ORDER BY a ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING);
+} {5 2 5   10 1 5   15 0 10   20 -1 15   25 3 20}
+
+#==========================================================================
+
+do_execsql_test 11.0 {
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(a INTEGER, b INTEGER);
+  INSERT INTO t2 VALUES(1, 12);
+  INSERT INTO t2 VALUES(2, 10);
+  INSERT INTO t2 VALUES(3, 15);
+  INSERT INTO t2 VALUES(4, 22);
+  INSERT INTO t2 VALUES(5,  1);
+  INSERT INTO t2 VALUES(6,  4);
+  INSERT INTO t2 VALUES(7,  7);
+  INSERT INTO t2 VALUES(8,  6);
+  INSERT INTO t2 VALUES(9, 22);
+  INSERT INTO t2 VALUES(10, 2);
+} {}
+
+do_execsql_test 11.1 {
+  SELECT a, min(b) FILTER (WHERE a%2 != 0) OVER win
+  FROM t2
+  WINDOW win AS (ORDER BY a ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING);
+} {1 12   2 12   3 1   4 1   5 1   6 1   7 1   8 7   9 7   10 22}
+
 finish_test
diff --git a/test/window9.test b/test/window9.test
index 4b8e4fa58f..0548624f82 100644
--- a/test/window9.test
+++ b/test/window9.test
@@ -282,4 +282,62 @@ do_catchsql_test 9.1 {
   FROM t1
 } {1 {frame ending offset must be a non-negative number}}
 
+#--------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 10.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 'a');
+  INSERT INTO t1 VALUES(2, 'b');
+  INSERT INTO t1 VALUES(3, 'c');
+  INSERT INTO t1 VALUES(4, 'd');
+  INSERT INTO t1 VALUES(5, 'e');
+  INSERT INTO t1 VALUES(6, 'f');
+}
+
+do_execsql_test 10.1 {
+  SELECT a, min(b) OVER win
+  FROM t1
+  WINDOW win AS (ORDER BY a ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING)
+} {
+  1 a
+  2 a
+  3 a
+  4 b
+  5 c
+  6 d
+}
+
+do_execsql_test 10.2 {
+  SELECT a, min(b) FILTER (WHERE a%2) OVER win
+  FROM t1
+  WINDOW win AS (ORDER BY a ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING)
+} {
+  1 a
+  2 a
+  3 a
+  4 c
+  5 c
+  6 e
+}
+
+do_execsql_test 10.3 {
+  SELECT a, min(b) FILTER (WHERE (a%2)=0) OVER win
+  FROM t1
+  WINDOW win AS (ORDER BY a ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING)
+} {
+  1 b
+  2 b
+  3 b
+  4 b
+  5 d
+  6 d
+}
+
+do_catchsql_test 10.4 {
+  SELECT a, nth_value(b, 1) FILTER (WHERE (a%2)=0) OVER win
+  FROM t1
+  WINDOW win AS (ORDER BY a ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING)
+} {1 {FILTER clause may only be used with aggregate window functions}}
+
 finish_test
diff --git a/test/windowE.test b/test/windowE.test
index f20bcdaaa8..1cb67f56b0 100644
--- a/test/windowE.test
+++ b/test/windowE.test
@@ -54,5 +54,57 @@ do_execsql_test 1.3 {
   5 5,4 5,4,1 5,4,1,6 5,4,1,6,3 5,4,1,6,3,2
 }
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x);
+}
+
+sqlite3_create_aggregate db
+
+breakpoint
+do_catchsql_test 2.1 {
+  SELECT min(x) OVER w1 FROM t1
+    WINDOW w1 AS (PARTITION BY x_count(x) OVER w1);
+} {1 {x_count() may not be used as a window function}}
+
+do_catchsql_test 2.2 {
+  SELECT min(x) FILTER (WHERE x_count(x) OVER w1) OVER w1 FROM t1
+    WINDOW w1 AS (PARTITION BY x OVER w1);
+} {1 {near "OVER": syntax error}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.0 {
+  BEGIN TRANSACTION;
+    CREATE TABLE t2(c1 INT, c2 REAL);
+    INSERT INTO t2 VALUES
+    (447,0.0), (448,0.0), (449,0.0), (452,0.0), (453,0.0), (454,0.0), (455,0.0),
+    (456,0.0), (459,0.0), (460,0.0), (462,0.0), (463,0.0), (466,0.0), (467,0.0),
+    (468,0.0), (469,0.0), (470,0.0), (473,0.0), (474,0.0), (475,0.0), (476,0.0),
+    (477,0.0), (480,0.0), (481,0.0), (482,0.0), (483,0.0), (484,0.0), (487,0.0),
+    (488,0.0), (489,0.0), (490,0.0), (491,0.0), (494,0.0), (495,0.0), (496,0.0),
+    (497,0.0), (498,0.0), (501,0.0), (502,0.0), (503,0.0), (504,0.0), (505,0.0),
+    (508,0.0), (509,0.0), (510,0.0), (511,0.0), (512,0.0), (515,0.0), (516,0.0),
+    (517,0.0), (518,0.0), (519,0.0), (522,0.0), (523,0.0), (524,0.0), (525,0.0),
+    (526,0.0), (529,0.0), (530,0.0), (531,0.0), (532,0.0), (533,0.0), (536,0.0),
+    (537,1.0), (538,0.0), (539,0.0), (540,0.0), (543,0.0), (544,0.0);
+  COMMIT;
+}
+
+do_execsql_test 3.1 {
+  select c1, max(c2) over (order by c1 range 366.0 preceding) from t2;
+} {
+  447 0.0 448 0.0 449 0.0 452 0.0 453 0.0 454 0.0 455 0.0 456 0.0 459 0.0 
+  460 0.0 462 0.0 463 0.0 466 0.0 467 0.0 468 0.0 469 0.0 470 0.0 473 0.0 
+  474 0.0 475 0.0 476 0.0 477 0.0 480 0.0 481 0.0 482 0.0 483 0.0 484 0.0 
+  487 0.0 488 0.0 489 0.0 490 0.0 491 0.0 494 0.0 495 0.0 496 0.0 497 0.0 
+  498 0.0 501 0.0 502 0.0 503 0.0 504 0.0 505 0.0 508 0.0 509 0.0 510 0.0 
+  511 0.0 512 0.0 515 0.0 516 0.0 517 0.0 518 0.0 519 0.0 522 0.0 523 0.0 
+  524 0.0 525 0.0 526 0.0 529 0.0 530 0.0 531 0.0 532 0.0 533 0.0 536 0.0 
+  537 1.0 538 1.0 539 1.0 540 1.0 543 1.0 544 1.0
+}
+
+
 finish_test
 
diff --git a/test/windowpushd.test b/test/windowpushd.test
index f2e04d72fb..4bfb2c523d 100644
--- a/test/windowpushd.test
+++ b/test/windowpushd.test
@@ -97,7 +97,11 @@ do_execsql_test 2.0 {
 }
 
 foreach tn {0 1} {
-  optimization_control db push-down $tn
+  if {$tn} {
+    optimization_control db all on
+  } else {
+    optimization_control db push-down off
+  }
 
   do_execsql_test 2.$tn.1.1 {
     SELECT * FROM v1;
diff --git a/test/with1.test b/test/with1.test
index 7400a7adf3..5ddf9dce0b 100644
--- a/test/with1.test
+++ b/test/with1.test
@@ -1092,7 +1092,7 @@ do_catchsql_test 22.1 {
 } {1 {too many FROM clause terms, max: 200}}
 
 # 2019-05-22
-# ticket https://www.sqlite.org/src/tktview/ce823231949d3abf42453c8f20
+# ticket https://sqlite.org/src/tktview/ce823231949d3abf42453c8f20
 #
 sqlite3 db :memory:
 do_execsql_test 23.1 {
diff --git a/test/with2.test b/test/with2.test
index 660df52b77..68790fe860 100644
--- a/test/with2.test
+++ b/test/with2.test
@@ -156,6 +156,16 @@ do_execsql_test 1.15 {
   SELECT * FROM t4;
 } {4 5}
 
+do_execsql_test 1.15.2 {
+  WITH 
+  t4(x) AS ( 
+    VALUES(4)
+    UNION ALL 
+    SELECT x+1 FROM (SELECT * FROM main.t4) WHERE x<10
+  )
+  SELECT * FROM t4;
+} {4 5}
+
 do_catchsql_test 1.16 {
   WITH 
   t4(x) AS ( 
diff --git a/test/with3.test b/test/with3.test
index 650740dcc1..9b110debf3 100644
--- a/test/with3.test
+++ b/test/with3.test
@@ -132,8 +132,8 @@ do_eqp_test 3.2.2 {
   |  |  `--SCALAR SUBQUERY xxxxxx
   |  |     `--SCAN w2
   |  `--RECURSIVE STEP
-  |     |--SCAN c
-  |     `--SCAN w1
+  |     |--SCAN w1
+  |     `--SCAN c
   |--SCAN c
   |--SEARCH w2 USING INTEGER PRIMARY KEY (rowid=?)
   `--SEARCH w1 USING INTEGER PRIMARY KEY (rowid=?)
diff --git a/test/with6.test b/test/with6.test
index 2a4bfc646d..b95ec0b763 100644
--- a/test/with6.test
+++ b/test/with6.test
@@ -325,6 +325,12 @@ do_eqp_test 331 {
 # marked with M10d_Yes and hence prohibited from participating in the
 # query flattening optimization.
 #
+# Updated 2025-01-02.
+# https://sqlite.org/forum/forumpost/8f38fc9878a92aa9
+#
+# The same optimization that made Grunthos's query fast made
+# Jean-Noël Mayor's query slow.  Bummer.
+#
 reset_db
 db eval {
   CREATE TABLE raw(country,date,total,delta, UNIQUE(country,date));
@@ -350,6 +356,50 @@ do_eqp_test 400 {
     FROM (SELECT f.*, exp(b) - 1 AS nFin, exp(a* (-1) + b) - 1 AS nPrev
             FROM fit f JOIN init i on i.country = f.country AND f.date <= date(i.fin,'-3 days'))
    WHERE nPrev > 0 AND nFin > 0;
+} {
+  QUERY PLAN
+  |--MATERIALIZE sums
+  |  |--MATERIALIZE src
+  |  |  |--MATERIALIZE init
+  |  |  |  `--SCAN raw USING INDEX sqlite_autoindex_raw_1
+  |  |  |--SCAN i
+  |  |  |--SEARCH raw USING COVERING INDEX sqlite_autoindex_raw_1 (country=? AND date>?)
+  |  |  `--USE TEMP B-TREE FOR ORDER BY
+  |  |--SCAN src
+  |  `--SEARCH raw USING INDEX sqlite_autoindex_raw_1 (country=? AND date>? AND date<?)
+  |--SCAN sums
+  |--BLOOM FILTER ON sums (country=? AND date=?)
+  |--SEARCH sums USING AUTOMATIC COVERING INDEX (country=? AND date=?)
+  |--BLOOM FILTER ON sums (country=? AND date=?)
+  |--SEARCH sums USING AUTOMATIC COVERING INDEX (country=? AND date=?)
+  |--BLOOM FILTER ON sums (country=? AND date=?)
+  |--SEARCH sums USING AUTOMATIC COVERING INDEX (country=? AND date=?)
+  |--BLOOM FILTER ON i (country=?)
+  `--SEARCH i USING AUTOMATIC COVERING INDEX (country=?)
+}
+optimization_control db order-by-subquery off
+db cache flush
+do_eqp_test 410 {
+  with recursive
+    init(country, date, fin) AS (SELECT country, min(date), max(date) FROM raw WHERE total > 0 GROUP BY country),
+    src(country, date) AS (SELECT raw.country, raw.date
+        FROM raw JOIN init i on raw.country = i.country AND raw.date > i.date
+        ORDER BY raw.country, raw.date),
+    vals(country, date, x, y) AS (SELECT src.country, src.date, julianday(raw.date) - julianday(src.date), log(delta+1)
+        FROM src JOIN raw on raw.country = src.country AND raw.date > date(src.date,'-7 days') AND raw.date <= src.date AND delta >= 0),
+    sums(country, date, x2, x, n, xy, y) AS (SELECT country, date, sum(x*x*1.0), sum(x*1.0), sum(1.0), sum(x*y*1.0), sum(y*1.0) FROM vals GROUP BY 1, 2),
+    mult(country, date, m) AS (SELECT country, date, 1.0/(x2 * n - x * x) FROM sums),
+    inv(country, date, a,b,c,d) AS (SELECT mult.country, mult.date, n * m, -x * m, -x * m, x2 * m
+        FROM mult JOIN sums on sums.country=mult.country AND mult.date=sums.date),
+    fit(country, date, a, b) AS (SELECT inv.country, inv.date, a * xy + b * y, c * xy + d * y
+        FROM inv
+        JOIN mult on mult.country = inv.country AND mult.date = inv.date
+        JOIN sums on sums.country = mult.country AND sums.date = mult.date
+  )
+  SELECT *, nFin/nPrev - 1 AS growth, log(2)/log(nFin/nPrev) AS doubling
+    FROM (SELECT f.*, exp(b) - 1 AS nFin, exp(a* (-1) + b) - 1 AS nPrev
+            FROM fit f JOIN init i on i.country = f.country AND f.date <= date(i.fin,'-3 days'))
+   WHERE nPrev > 0 AND nFin > 0;
 } {
   QUERY PLAN
   |--MATERIALIZE sums
diff --git a/test/without_rowid1.test b/test/without_rowid1.test
index 3c33f733a1..5134e5e809 100644
--- a/test/without_rowid1.test
+++ b/test/without_rowid1.test
@@ -415,7 +415,7 @@ do_execsql_test 10.6 {
   SELECT * FROM t1;
 } {b a 3  b b 4}
 
-# 2019-04-29 ticket https://www.sqlite.org/src/info/3182d3879020ef3
+# 2019-04-29 ticket https://sqlite.org/src/info/3182d3879020ef3
 do_execsql_test 11.1 {
   CREATE TABLE t11(a TEXT PRIMARY KEY, b INT) WITHOUT ROWID;
   CREATE INDEX t11a ON t11(a COLLATE NOCASE);
@@ -424,7 +424,7 @@ do_execsql_test 11.1 {
   SELECT a FROM t11 ORDER BY a COLLATE binary;
 } {ok A a}
 
-# 2019-05-13 ticket https://www.sqlite.org/src/info/bba7b69f9849b5b
+# 2019-05-13 ticket https://sqlite.org/src/info/bba7b69f9849b5b
 do_execsql_test 12.1 {
   DROP TABLE IF EXISTS t0;
   CREATE TABLE t0 (c0 INTEGER PRIMARY KEY DESC, c1 UNIQUE DEFAULT NULL) WITHOUT ROWID;
@@ -433,7 +433,7 @@ do_execsql_test 12.1 {
   PRAGMA integrity_check;
 } {ok}
 
-# 2019-11-07 ticket https://www.sqlite.org/src/info/302027baf1374498
+# 2019-11-07 ticket https://sqlite.org/src/info/302027baf1374498
 # The xferCompatibleIndex() function confuses a PRIMARY KEY index
 # with a UNIQUE index.
 #
@@ -487,4 +487,33 @@ ifcapable altertable {
     SELECT name FROM sqlite_sequence ORDER BY +name;
   } {a c0 c2}
 }
+
+# Ensure that the number of columns in an index on a WITHOUT ROWID
+# table does not exceed SQLITE_LIMIT_COLUMN.
+#
+reset_db
+sqlite3_limit db SQLITE_LIMIT_COLUMN 8
+do_execsql_test 16.1 {
+  CREATE TABLE t1(
+    c1,c2,c3,c4,c5,c6,c7,c8,
+    PRIMARY KEY(c1,c2,c1 COLLATE NOCASE)
+  ) WITHOUT ROWID;
+} {}
+do_execsql_test 16.2 {
+  CREATE TABLE t2(
+    c1,c2,c3,c4,c5,c6,c7,c8,
+    PRIMARY KEY(c1 COLLATE nocase,c1 COLLATE rtrim,
+                c2 COLLATE nocase,c2 COLLATE rtrim,
+                c3 COLLATE nocase,c3 COLLATE rtrim,
+                c4 COLLATE nocase,c4 COLLATE rtrim)
+  ) WITHOUT ROWID;
+} {}
+do_execsql_test 16.3 {
+  CREATE TABLE t3(
+    c1,c2,c3,c4,c5,c6,c7,c8,
+    PRIMARY KEY(c1,c2),
+    UNIQUE(c3,c4,c5,c6,c7,c8,c3 COLLATE nocase)
+  ) WITHOUT ROWID;
+} {}
+
 finish_test
diff --git a/test/without_rowid7.test b/test/without_rowid7.test
index 56e9fb40b9..bf0273d705 100644
--- a/test/without_rowid7.test
+++ b/test/without_rowid7.test
@@ -55,6 +55,69 @@ do_execsql_test 2.4 {
   PRAGMA index_info(t3);
 } {}
 
+#-------------------------------------------------------------------------
+reset_db
+db collate mysort mysort
+db collate mysort2 mysort
+proc mysort {a b} { string compare $a $b }
+do_execsql_test 3.0 {
+  CREATE TABLE t1(
+      a PRIMARY KEY COLLATE mysort, b COLLATE mysort2
+  ) WITHOUT ROWID;
+  INSERT INTO t1 VALUES(1, 2);
+}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 3.1.1 {
+  SELECT * FROM t1 WHERE a=1;
+} {1 {no such collation sequence: mysort}}
+do_test 3.1.2 {
+  sqlite3_extended_errcode db
+} {SQLITE_ERROR_MISSING_COLLSEQ}
+
+db collate mysort mysort
+
+do_catchsql_test 3.2.1 {
+  CREATE UNIQUE INDEX i1 ON t1(b);
+} {1 {no such collation sequence: mysort2}}
+do_test 3.2.2 {
+  sqlite3_extended_errcode db
+} {SQLITE_ERROR_MISSING_COLLSEQ}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 3.3.1 {
+  CREATE UNIQUE INDEX i1 ON t1(1);
+} {1 {no such collation sequence: mysort}}
+do_test 3.3.2 {
+  sqlite3_extended_errcode db
+} {SQLITE_ERROR_MISSING_COLLSEQ}
+
+do_test 3.4.1 {
+  list [catch {
+    sqlite3_prepare_v3 db "CREATE UNIQUE INDEX i1 ON t1(1)" -1 0
+  } msg] $msg
+} {1 {(1) no such collation sequence: mysort}}
+do_test 3.4.2 {
+  sqlite3_extended_errcode db
+} {SQLITE_ERROR_MISSING_COLLSEQ}
+
+sqlite3_extended_result_codes db 1
+
+do_test 3.5.1 {
+  list [catch {
+    sqlite3_prepare_v3 db "CREATE UNIQUE INDEX i1 ON t1(1)" -1 0
+  } msg] $msg
+} {1 {(257) no such collation sequence: mysort}}
+do_test 3.5.2 {
+  sqlite3_extended_errcode db
+} {SQLITE_ERROR_MISSING_COLLSEQ}
 
+do_catchsql_test 3.6 {
+  SELECT * FROM t1 WHERE a=1;
+} {1 {no such collation sequence: mysort}}
 
 finish_test
diff --git a/test/writecrash.test b/test/writecrash.test
index 01bc4dbb56..aca89aafb8 100644
--- a/test/writecrash.test
+++ b/test/writecrash.test
@@ -21,7 +21,7 @@ set testprefix writecrash
 do_not_use_codec
 
 
-if {$tcl_platform(platform)=="windows"} {
+if {$tcl_platform(platform) eq "windows"} {
   finish_test
   return
 }
diff --git a/test/zipfile.test b/test/zipfile.test
index 4d09e08b1c..9bb35ea5db 100644
--- a/test/zipfile.test
+++ b/test/zipfile.test
@@ -10,7 +10,10 @@
 #***********************************************************************
 #
 
-package require Tcl 8.6
+if {$tcl_version<8.6} {
+  puts "Requires TCL 8.6 or later"
+  return
+}
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -30,7 +33,7 @@ if {[catch {load_static_extension db fileio} error]} {
 
 proc readfile {f} {
   set fd [open $f]
-  fconfigure $fd -translation binary -encoding binary
+  fconfigure $fd -translation binary
   set data [read $fd]
   close $fd
   set data
@@ -42,7 +45,7 @@ if {[catch {exec unzip} msg]==0 && \
     [regexp -line {^UnZip \d+\.\d+ .*? Info-ZIP\.} $msg]} {
   set ::UNZIP unzip
   proc fix_stat_mode {name mode} {
-    if {$::tcl_platform(platform)=="windows"} {
+    if {$::tcl_platform(platform) eq "windows"} {
       #
       # NOTE: Set or unset the write bits of the file permissions
       #       based on the read-only attribute because the Win32
@@ -277,7 +280,7 @@ do_execsql_test 1.6.2 {
   i.txt 33188 4 zxcvb 0
 }
 
-if {$::tcl_platform(platform)=="unix"} {
+if {$::tcl_platform(platform) eq "unix"} {
   set modes -rw-r--r-x
   set perms 33189
 } else {
@@ -404,7 +407,7 @@ if {[info exists ::UNZIP]} {
   do_test 2.5.1 {
     forcedelete dirname
     forcedelete dirname2
-    if {$::tcl_platform(platform)=="unix"} {
+    if {$::tcl_platform(platform) eq "unix"} {
       set null /dev/null
     } else {
       set null NUL
@@ -762,7 +765,7 @@ do_execsql_test 11.11 {
 } {b0suffix two b2suffix one}
 
 
-if {$tcl_platform(platform)!="windows"} {
+if {$tcl_platform(platform) ne "windows"} {
   do_test 12.0 {
     catch { file delete -force subdir }
     foreach {path sz} {
@@ -882,6 +885,27 @@ do_test 19.1 {
     INSERT INTO v0 DEFAULT VALUES;
   }
 } {}
+db close
 forcedelete zipfile19.zip
+sqlite3 db :memory:
+load_static_extension db zipfile
 
+#-------------------------------------------------------------------------
+do_catchsql_test 20.0 {
+  SELECT * FROM zipfile(X'504b050600000000010001004000000000a3e1110000');
+} {1 {zip archive is corrupt}}
+
+do_catchsql_test 20.1 {
+  SELECT * FROM zipfile(unhex('
+504b0304140000080000a60d3e5bd42728f602000000020000000500090068
+2e74787455540500012836db682120504b01021e03140000080000a60d3e5b
+d42728f602000000020000000500ffff0000000000000000a4810000000068
+2e74787455540500012836db68504b050600000000010001003c0000002e00
+00000000',char(0x0a,0x0d)));
+} {1 {zip archive is corrupt}}
+
+# https://sqlite.org/forum/forumpost/2025-12-30T23:57:19z
+do_catchsql_test 20.2 {
+  SELECT * FROM zipfile(unhex('504b0304140000000000000000008b9ed9d30100000001000000010000007841504b01021e03140000000000000000008b9ed9d3010000000100000001001e000000000000000000a4810000000078504b050600000000010001002f000000200000000000'));
+} {1 {zip archive is corrupt}}
 finish_test
diff --git a/test/zipfile2.test b/test/zipfile2.test
index 987e4f0cfd..8ee90d310b 100644
--- a/test/zipfile2.test
+++ b/test/zipfile2.test
@@ -10,7 +10,10 @@
 #***********************************************************************
 #
 
-package require Tcl 8.6
+if {$tcl_version<8.6} {
+  puts "Requires TCL 8.6 or later"
+  return
+}
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -240,4 +243,63 @@ do_execsql_test 6.3 {
   SELECT name FROM zip;
 } {test2}
 
+#-------------------------------------------------------------------------
+# Test a crafted corrupt file.
+#
+proc make_corrupt_file {fname} {
+  set central_dir_offset 1000
+  set lfh_offset 200
+  set nFile 60000
+  set nExtra 60000
+  set nComment 0
+  
+  # Build local file header at lfh_offset
+  set lfh ""
+  append lfh [binary format isssssiiiss 0x04034b50 20 0 0 0 0 0 0 0 1 0]
+  append lfh "A"
+  
+  # Build central directory structure (CDS) at central_dir_offset
+  set cds ""
+  append cds [binary format issssssiiisssssii 0x02014b50 0 20 0 0 0 0 0 0 0 $nFile $nExtra $nComment 0 0 0 $lfh_offset]
+  
+  # Payload following CDS: filename + extra + comment
+  set payload ""
+  append payload [string repeat "B" $nFile]
+  append payload [string repeat "C" $nExtra]
+  
+  # EOCD at end
+  set cdsize [expr {[string length $cds] + [string length $payload]}]
+  set eocd ""
+  append eocd [binary format issssiis 0x06054b50 0 0 1 1 $cdsize $central_dir_offset 0]
+  
+  # Assemble file
+  set buf [string repeat "X" $lfh_offset]
+  append buf $lfh
+  
+  set buflen [string length $buf]
+  if {$central_dir_offset > $buflen} {
+      append buf [string repeat "\x00" [expr {$central_dir_offset - $buflen}]]
+  }
+  
+  append buf $cds
+  append buf $payload
+  append buf $eocd
+  
+  # Write to file
+  set f [open $fname wb]
+  fconfigure $f -translation binary
+  puts -nonewline $f $buf
+  close $f
+}
+
+forcedelete test.zip
+make_corrupt_file test.zip
+do_execsql_test 7.0 {
+  DROP TABLE IF EXISTS t1;
+  CREATE VIRTUAL TABLE t1 USING zipfile('test.zip');
+}
+do_execsql_test 7.1 {
+  SELECT length(name) FROM t1;
+} {60000}
+
 finish_test
diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat
index 8f9a1b7b09..83d660deb0 100755
--- a/tool/build-all-msvc.bat
+++ b/tool/build-all-msvc.bat
@@ -198,7 +198,7 @@ IF NOT DEFINED ComSpec (
 
 REM
 REM NOTE: This batch file requires the VcInstallDir environment variable to be
-REM       set.  Tyipcally, this means this batch file needs to be run from an
+REM       set.  Typically, this means this batch file needs to be run from an
 REM       MSVC command prompt.
 REM
 IF NOT DEFINED VCINSTALLDIR (
diff --git a/tool/build-shell.sh b/tool/build-shell.sh
index 6a48299d73..7899080ebe 100644
--- a/tool/build-shell.sh
+++ b/tool/build-shell.sh
@@ -16,7 +16,6 @@ gcc -o sqlite3 -g -Os -I. \
    -DSQLITE_ENABLE_FTS4 \
    -DSQLITE_ENABLE_RTREE \
    -DHAVE_READLINE \
-   -DHAVE_USLEEP=1 \
    ../sqlite/src/shell.c \
-   ../sqlite/src/test_vfstrace.c \
+   ../sqlite/ext/misc/vfstrace.c \
    sqlite3.c -ldl -lreadline -lncurses
diff --git a/tool/buildtclext.tcl b/tool/buildtclext.tcl
new file mode 100644
index 0000000000..535ed37e72
--- /dev/null
+++ b/tool/buildtclext.tcl
@@ -0,0 +1,334 @@
+#!/usr/bin/tclsh
+#
+set help \
+{Run this TCL script to build and install the TCL interface library for
+SQLite.  Run the script with the specific "tclsh" for which the installation
+should occur.
+
+There must be a valid "tclsqlite3.c" file in the working directory prior
+to running this script.  Use "make tclsqlite3.c" to generate that file.
+
+Options:
+
+   --build-only         Only build the extension, don't install it
+   --cc COMPILER        Build using this compiler
+   --info               Show info on existing SQLite TCL extension installs
+   --install-only       Install an extension previously build
+   --uninstall          Uninstall the extension
+   --version-check      Check extension version against this source tree
+   --destdir DIR        Installation root (used by "make install DESTDIR=...")
+   --tclConfig.sh FILE  Use this tclConfig.sh instead of looking for one
+
+Other options are retained and passed through into the compiler.}
+
+
+set build 1
+set install 1
+set uninstall 0
+set infoonly 0
+set versioncheck 0
+set CC {}
+set OPTS {}
+set DESTDIR ""; # --destdir "$(DESTDIR)"
+set tclConfigSh ""; # --tclConfig.sh FILE
+for {set ii 0} {$ii<[llength $argv]} {incr ii} {
+  set a0 [lindex $argv $ii]
+  if {$a0=="--install-only"} {
+    set build 0
+  } elseif {$a0=="--build-only"} {
+    set install 0
+  } elseif {$a0=="--uninstall"} {
+    set build 0
+    set install 0
+    set versioncheck 0
+    set uninstall 1
+  } elseif {$a0=="--info"} {
+    set build 0
+    set install 0
+    set versioncheck 0
+    set infoonly 1
+  } elseif {$a0=="--version-check"} {
+    set build 0
+    set install 0
+    set infoonly 0
+    set versioncheck 1
+  } elseif {$a0=="--cc" && $ii+1<[llength $argv]} {
+    incr ii
+    set CC [lindex $argv $ii]
+  } elseif {$a0=="--destdir" && $ii+1<[llength $argv]} {
+    incr ii
+    set DESTDIR [lindex $argv $ii]
+  } elseif {$a0=="--tclConfig.sh" && $ii+1<[llength $argv]} {
+    incr ii
+    set tclConfigSh [lindex $argv $ii]
+  } elseif {[string match -* $a0]} {
+    append OPTS " $a0"
+  } else {
+    puts stderr "Unknown option: \"$a0\"\n"
+    puts stderr $help
+    exit 1
+  }
+}
+
+# Find the root of the SQLite source tree
+#
+set srcdir [file normalize [file dir $argv0]/..]
+
+# Get the SQLite version number into $VERSION
+#
+set fd [open $srcdir/VERSION]
+set VERSION [string trim [read $fd]]
+close $fd
+
+if {$tcl_platform(platform) eq "windows"} {
+  # We are only able to install, uninstall, and list on Windows.
+  # The build process is handled by the Makefile.msc, specifically
+  # using "nmake /f Makefile.msc pkgIndex.tcl tclsqlite3.dll"
+  #
+  if {$build} {
+    puts "Unable to build on Windows using the builttclext.tcl script."
+    puts "To build, run\n"
+    puts "   \"nmake /f Makefile.msc pkgIndex.tcl tclsqlite3.dll"
+    exit 1
+  }
+  set OUT tclsqlite3.dll
+} else {
+  # Read the tclConfig.sh file into the $tclConfig variable
+  #
+  if {"" eq $tclConfigSh} {
+    # Figure out the location of the tclConfig.sh file used by the
+    # tclsh that is executing this script.
+    #
+    if {[catch {
+      set LIBDIR [tcl::pkgconfig get libdir,install]
+    }]} {
+      puts stderr "$argv0: tclsh does not support tcl::pkgconfig."
+      exit 1
+    }
+    if {![file exists $LIBDIR]} {
+      puts stderr "$argv0: cannot find the tclConfig.sh file."
+      puts stderr "$argv0: tclsh reported library directory \"$LIBDIR\"\
+                 does not exist."
+      exit 1
+    }
+    if {![file exists $LIBDIR/tclConfig.sh]
+        || [file size $LIBDIR/tclConfig.sh]<5000} {
+      set n1 $LIBDIR/tcl$::tcl_version
+      if {[file exists $n1/tclConfig.sh]
+          && [file size $n1/tclConfig.sh]>5000} {
+        set LIBDIR $n1
+      } else {
+        puts stderr "$argv0: cannot find tclConfig.sh in either $LIBDIR or $n1"
+        exit 1
+      }
+    }
+    #puts "using $LIBDIR/tclConfig.sh"
+    set fd [open $LIBDIR/tclConfig.sh rb]
+    set tclConfig [read $fd]
+    close $fd
+  } else {
+    # User-provided tclConfig.sh
+    #
+    set fd [open $tclConfigSh rb]
+    set tclConfig [read $fd]
+    close $fd
+  }
+
+  # Extract parameter we will need from the tclConfig.sh file
+  #
+  set TCLMAJOR 8
+  regexp {TCL_MAJOR_VERSION='(\d)'} $tclConfig all TCLMAJOR
+  set SUFFIX so
+  regexp {TCL_SHLIB_SUFFIX='\.([^']+)'} $tclConfig all SUFFIX
+  if {$CC==""} {
+    set cc {}
+    regexp {TCL_CC='([^']+)'} $tclConfig all cc
+    if {$cc!=""} {
+      set CC $cc
+    }
+  }
+  if {$CC==""} {
+    set CC gcc
+  }
+  set CFLAGS -fPIC
+  regexp {TCL_SHLIB_CFLAGS='([^']+)'} $tclConfig all CFLAGS
+  set LIBS {}
+  regexp {TCL_STUB_LIB_SPEC='([^']+)'} $tclConfig all LIBS
+  set INC "-I$srcdir/src"
+  set inc {}
+  regexp {TCL_INCLUDE_SPEC='([^']+)'} $tclConfig all inc
+  if {$inc!=""} {
+    append INC " $inc"
+  }
+  set cmd {${CC} ${CFLAGS} ${LDFLAGS} -shared}
+  regexp {TCL_SHLIB_LD='([^']+)(-Wl,--out-implib.*)?'} $tclConfig all cmd
+  set LDFLAGS "$INC -DUSE_TCL_STUBS"
+  if {[string length $OPTS]>1} {
+    append LDFLAGS $OPTS
+  }
+  if {$tcl_platform(os) eq "Windows NT"} {
+    set OUT cyg
+  } else {
+    set OUT lib
+  }
+  if {$TCLMAJOR>8} {
+    set OUT ${OUT}tcl9sqlite$VERSION.$SUFFIX
+  } else {
+    set OUT ${OUT}sqlite$VERSION.$SUFFIX
+  }
+  set @ $OUT; # Workaround for https://sqlite.org/forum/forumpost/0683a49cb02f31a1
+              # in which Gentoo edits their tclConfig.sh to include an soname
+              # linker flag which includes ${@} (the target file's name).
+  set CMD [subst $cmd]
+}
+
+# Check the SQLite TCL extension that is loaded by default by this running
+# TCL interpreter to see if it has the same SQLITE_SOURCE_ID as the source
+# code in the directory holding this script.
+#
+if {$versioncheck} {
+  if {[catch {package require sqlite3} msg]} {
+    puts stderr "No SQLite TCL extension available: $msg"
+    exit 1
+  }
+  sqlite3 db :memory:
+  set extvers [db one {SELECT sqlite_source_id()}]
+  db close
+  set fd [open sqlite3.h rb]
+  set sqlite3h [read $fd]
+  close $fd
+  regexp {#define SQLITE_SOURCE_ID +"([^"]+)"} $sqlite3h all srcvers
+  set srcvers [string range $srcvers 0 78]
+  set extvers [string range $extvers 0 78]
+  if {$srcvers==$extvers} {
+    puts "source code and extension versions aligned:\n$extvers"
+    exit 0
+  }
+  puts stderr "source code and extension versions differ"
+  puts stderr "source: $srcvers\nextension: $extvers"
+  exit 1
+}
+
+# Show information about prior installs
+#
+if {$infoonly} {
+  set cnt 0
+  foreach dir $auto_path {
+    foreach subdir [glob -nocomplain -types d $dir/sqlite3*] {
+      if {[file exists $subdir/pkgIndex.tcl]} {
+        puts $subdir
+        incr cnt
+      }
+    }
+  }
+  if {$cnt==0} {
+    puts "no current installations of the SQLite TCL extension"
+  }
+  exit
+}
+
+# Uninstall the extension
+#
+if {$uninstall} {
+  set cnt 0
+  foreach dir $auto_path {
+    if {[file isdirectory $dir/sqlite$VERSION]} {
+      incr cnt
+      if {![file writable $dir] || ![file writable $dir/sqlite$VERSION]} {
+        puts "cannot uninstall $dir/sqlite$VERSION - permission denied"
+      } else {
+        puts "uninstalling $dir/sqlite$VERSION..."
+        file delete -force $dir/sqlite$VERSION
+      }
+    }
+  }
+  if {$cnt==0} {
+    puts "nothing to uninstall"
+  }
+  exit
+}
+
+if {$install} {
+  # Figure out where the extension will be installed.  Put the extension
+  # in the first writable directory on $auto_path.
+  #
+  set DEST {}
+  foreach dir $auto_path {
+    if {[string match //*:* $dir]} {
+      # We can't install to //zipfs: paths
+      continue
+    } elseif {"" ne $DESTDIR && ![file writable $DESTDIR]} {
+      # In the common case, ${DESTDIR}${dir} will not exist when we
+      # get to this point of the installation, and the "is writable?"
+      # check just below this will fail for that case.
+      #
+      # Assumption made for simplification's sake: if ${DESTDIR} is
+      # not writable, no part of the remaining path will
+      # be. ${DESTDIR} is typically used by OS package maintainers,
+      # not normal installations, and it "shouldn't" ever happen that
+      # the DESTDIR is read-only while the target ${DESTDIR}${prefix}
+      # is not, as it's typical for such installations to create
+      # ${prefix} on-demand under ${DESTDIR}.
+      break
+    }
+    set dir ${DESTDIR}$dir
+    if {[file writable $dir] || "" ne $DESTDIR} {
+      # the dir will be created later ^^^^^^^^
+      set DEST $dir
+      break
+    } elseif {[glob -nocomplain $dir/sqlite3*/pkgIndex.tcl]!=""} {
+      set conflict [lindex [glob $dir/sqlite3*/pkgIndex.tcl] 0]
+      puts "Unable to install. There is already a conflicting version"
+      puts "of the SQLite TCL Extension that cannot be overwritten at\n"
+      puts "   [file dirname $conflict]\n"
+      puts "Consider running using sudo to work around this problem."
+      exit 1
+    }
+  }
+  if {$DEST==""} {
+    puts "None of the directories on \$auto_path are writable by this process,"
+    puts "so the installation cannot take place.  Consider running using sudo"
+    puts "to work around this problem.\n"
+    puts "These are the (unwritable) \$auto_path directories:\n"
+    foreach dir $auto_path {
+      puts "  *  ${DESTDIR}$dir"
+    }
+    exit 1
+  }
+}
+
+if {$build} {
+  # Generate the pkgIndex.tcl file
+  #
+  puts "generating pkgIndex.tcl..."
+  set fd [open pkgIndex.tcl w]
+  puts $fd [subst -nocommands {# -*- tcl -*-
+# Tcl package index file, version ???
+#
+package ifneeded sqlite3 $VERSION \\
+    [list load [file join \$dir $OUT] Sqlite3]
+}]
+  close $fd
+
+  # Generate and execute the command with which to do the compilation.
+  #
+  set cmd "$CMD -DUSE_TCL_STUBS tclsqlite3.c -o $OUT $LIBS"
+  puts $cmd
+  file delete -force $OUT
+  catch {exec {*}$cmd} errmsg
+  if {$errmsg!="" && ![file exists $OUT]} {
+    puts $errmsg
+    exit 1
+  }
+}
+
+
+if {$install} {
+  # Install the extension
+  set DEST2 $DEST/sqlite$VERSION
+  file mkdir $DEST2
+  puts "installing $DEST2/pkgIndex.tcl"
+  file copy -force pkgIndex.tcl $DEST2
+  puts "installing $DEST2/$OUT"
+  file copy -force $OUT $DEST2
+}
diff --git a/tool/cktclsh.sh b/tool/cktclsh.sh
new file mode 100644
index 0000000000..1928a40998
--- /dev/null
+++ b/tool/cktclsh.sh
@@ -0,0 +1,11 @@
+# Fail with an error if the TCLSH named in $2 is not tclsh version $1 or later.
+#
+echo "set vers $1" >cktclsh$1.tcl
+echo 'if {$tcl_version<$vers} {exit 1}' >>cktclsh$1.tcl
+if ! $2 cktclsh$1.tcl
+then
+   echo "ERROR: This makefile target requires tclsh $1 or later."
+   rm cktclsh$1.tcl
+   exit 1
+fi
+rm cktclsh$1.tcl
diff --git a/tool/cp.tcl b/tool/cp.tcl
new file mode 100644
index 0000000000..80e4adf107
--- /dev/null
+++ b/tool/cp.tcl
@@ -0,0 +1,28 @@
+#/usr/bin/tclsh
+#
+# This is a TCL script that copies multiple files into a common directory.
+# The "cp" command will do this on unix, but no such command is available
+# by default on Windows, so we have to use this script.
+#
+#   tclsh cp.tcl FILE1 FILE2 ... FILEN DIR
+#
+
+# This should be as simple as
+#
+#     file copy -force -- {*}$argv
+#
+# But jimtcl doesn't support that.  So we have to do it the hard way.
+
+if {[llength $argv]<2} {
+  error "Usage: $argv0 SRC... DESTDIR"
+}
+set n [llength $argv]
+set destdir [lindex $argv [expr {$n-1}]]
+if {![file isdir $destdir]} {
+  error "$argv0: not a directory: \"$destdir\""
+}
+for {set i 0} {$i<$n-1} {incr i} {
+  set fn [file normalize [lindex $argv $i]]
+  set tail [file tail $fn]
+  file copy -force $fn [file normalize $destdir/$tail]
+}
diff --git a/tool/custom.txt b/tool/custom.txt
new file mode 100644
index 0000000000..a9fc4807e7
--- /dev/null
+++ b/tool/custom.txt
@@ -0,0 +1,1254 @@
+aa
+aaa
+abc
+abcdefg
+abd
+abf
+Abortable
+acc
+accessor
+accum
+acd
+activecpu
+Adaptor
+Additionallly
+addop
+addoptrace
+addr
+adjustements
+af
+aff
+afp
+afterward
+Agg
+agg
+agginfo
+alikes
+Alloc
+alloc
+alloca
+allocator
+allocators
+alphabetics
+alphanumerics
+alternateform
+altertab
+altform
+amalgamator
+amongst
+analyse
+antipenultimate
+antirez
+ap
+api
+appdef
+appendall
+appendchar
+appendf
+ar
+arg
+argc
+argcount
+arglist
+argn
+args
+argv
+arrayname
+ascii
+asm
+async
+atoi
+atomics
+auth
+authorizer
+autocheckpoint
+autocommit
+autoconf
+autoext
+autoextension
+autoinc
+autoincrement
+autoincremented
+autoindex
+autoinstall
+autovac
+autovacuum
+autovacuumed
+autovacuuming
+auxdata
+awk
+Ax
+backend
+backends
+backfill
+backfilled
+backfilling
+backtrace
+backtraces
+backtracing
+bb
+bba
+bcb
+bcc
+beginthreadex
+behavior
+behavioral
+behaviors
+behaviour
+benigncnt
+bg
+bigblob
+bitcount
+bitfield
+bitmask
+bitmasks
+bitset
+bitvec
+bitvecs
+bitwise
+blobwrite
+blockquote
+Bloomfilter
+bom
+boolean
+booleans
+Borland
+br
+breadthfirstsearch
+breakpoint
+bt
+btree
+btrees
+buf
+bufpt
+butoindex
+bytearray
+bytecode
+bytecodevtab
+byteorder
+cacheflush
+cachegrind
+Cachesize
+calc'ing
+callgrind
+cardinalities
+cardinality
+carray
+cb
+cd
+cdbaa
+ce
+ceil
+cellpadding
+cellspacing
+center
+chacha
+changecounter
+changeset
+characgter
+characterset
+checkpointed
+Checkpointer
+checkpointer
+checkpointers
+checkpointing
+checksummed
+checksums
+chmod
+chng
+chown
+chroot
+chsize
+chunksize
+cid
+cis
+ckalloc'd
+ckpt
+cksum
+cksumvfs
+clientdata
+closedir
+clownshoe
+cmd
+cmp
+cmpaff
+Cmpr
+cnt
+codec
+codepage
+collseq
+colname
+compileoption
+concat
+config
+confstr
+connetion
+consective
+considertion
+const
+coredump
+coroutine
+coroutines
+cov
+crashparams
+csr
+csv
+Cte
+ctime
+Ctrl
+ctrl
+ctx
+ctype
+cume
+Cx
+cx
+Cygwin
+cygwin
+dan
+darkstar
+databasename
+databse
+datasource
+datatypes
+datetime
+dbfuzz
+dbinfo
+dbname
+dbpage
+dbpagevfs
+dbs
+dbsize
+dbsqlfuzz
+dbstat
+dbtotxt
+De
+de
+deadman
+deallocate
+deallocated
+deallocates
+deallocating
+deallocation
+decltype
+decrementing
+defense
+defenses
+defn
+defragment
+defragmentation
+defragmented
+deinitialization
+Deinitialize
+deinitialize
+demovfs
+dependences
+dequote
+dequoted
+dequoting
+dereference
+dereferenced
+dereferences
+desc
+deserialization
+deserialize
+deserialized
+deserializes
+deserializing
+dest
+destructor
+destructors
+deterministically
+dev
+devsim
+devsym
+df
+Dfdatasync
+dflt
+dir
+directonly
+dirent
+dirs
+disjunct
+disjunction
+disjuncts
+diskfull
+divy
+dl
+dll
+dlopen
+dlsym
+docid
+docids
+dont
+dontcache
+dotfile
+dotlock
+doublearray
+drh
+dryrun
+dstr
+dt
+Duint
+dup
+Durian
+dword
+Dx
+dylib
+Dyn
+Ec
+ec
+ecel
+editability
+ef
+efc
+eg
+Ei
+elif
+emcc
+emscripten
+encodings
+endeavor
+endfor
+endian
+endianness
+endif
+endthreadex
+enum
+eof
+eph
+Ephem
+eq
+eqp
+equaling
+equalities
+errcode
+errmsg
+errno
+errorcode
+erroroffset
+errstr
+esc
+esign
+et
+etfs
+etilqs
+eval
+exe
+expander
+explainer
+expmask
+expr
+exprlist
+extern
+fakeheap
+fallocate
+fanout
+faultsim
+favor
+favors
+fb
+fc
+fchmod
+fchown
+fclose
+fcntl
+fd
+fdatasync
+feb
+fef
+Feijoa
+ferror
+ffff
+ffffeff
+ffffffe
+fffffff
+ffffffff
+fffffffffffffff
+fflush
+fg
+fgets
+fi
+fibonacci
+fid
+Fifo
+filecount
+filectrl
+filemapping
+filesize
+filesystem
+filesystems
+finalised
+finalizer
+findfirst
+findnext
+fixup
+fk
+fkctr
+fkey
+flattener
+fmt
+fopen
+foreach
+formatter
+fprintf
+fputs
+fread
+fred
+fred's
+freeblock
+freeblocks
+freelist
+freepage
+freespace
+frombind
+fs
+fsanitize
+fsctl
+fsdir
+fseek
+fstat
+fstree
+fsync
+fsynced
+fsyncs
+ftell
+ftruncate
+fts
+fullfsync
+fullname
+fullschema
+fullshm
+fullsync
+fullsyncs
+func
+funcs
+fuzzer
+fuzzers
+fwrite
+Fx
+gcc
+gcov
+gdb
+getcwd
+getenv
+gethostuuid
+getpagesize
+getpid
+getrusage
+getsubtype
+getter
+getters
+gibabyte
+gid
+glibc
+globbing
+gmtime
+Gosub
+Goto
+goto
+groupbyparent
+Groupid
+growable
+grp
+hdl
+hdr
+hexdb
+hexdouble
+hexio
+highwater
+hijklmno
+honor
+honored
+honoring
+hostid
+href
+html
+htsize
+hw
+Hwtime
+icecube
+ideographical
+idx
+idxaff
+idxnum
+idxstr
+idx'th
+ieee
+ifdef
+iff
+ifndef
+imm
+impl
+imposter
+incr
+incrblob
+incrementing
+indexable
+indexname
+infop
+ing
+init
+initfail
+initializer
+initializers
+initiallly
+inlined
+inlines
+inlining
+ino
+inode
+inodes
+inopertune
+installable
+intarray
+inteface
+interoperate
+interp
+interpretable
+intkey
+intptr
+intreal
+intrinsics
+invalidations
+invariants
+io
+ioerr
+iotrace
+ipk
+iplan
+isalnum
+isalpha
+isatty
+isdigit
+isempty
+isexplain
+ismemdb
+isnan
+isspace
+isxdigit
+i'th
+iversion
+jfd
+jj
+jointype
+jointypes
+journaled
+journaling
+journalled
+journalling
+journalmode
+jrnl
+jsl
+json
+jt
+julian
+julianday
+keyinfo
+keywordhash
+kibibytes
+kvstorage
+kvvfs
+lappend
+lasterrno
+lbl
+ldl
+le
+leafdata
+leftjustify
+len
+leveling
+lexeme
+lexemes
+lhs
+li
+libm
+libversion
+lifecycle
+lindex
+lineno
+Linenoise
+linenoise
+linux
+lised
+lld
+llvm
+lm
+ln
+lncurses
+loadext
+localhost
+localtime
+lockd
+lockdata
+lockingcontext
+lockless
+lockproxy
+locktype
+logmsg
+longvalue
+longwords
+lookahead
+Lookaside
+lookaside
+lookups
+losslessly
+lpthread
+lrc
+lreadline
+lru
+lseek
+lt
+lvalue
+lwr
+makefile
+makefiles
+malloc
+malloc'd
+malloced
+mallocs
+manditory
+manpage
+matchinfo
+materializations
+mathfuncs
+maxsize
+mbcs
+Mcafee
+md
+Meeus
+mem
+memcmp
+memcpy
+memdb
+memdebug
+memget
+memmove
+mempool
+memset
+memstatus
+memsys
+memvfs
+mergeable
+mergepatch
+middleware
+millisec
+mincore
+mingw
+mis
+miscoded
+mj
+mkctimec
+mkdir
+mkkeywordhash
+mkopcodec
+mkopcodeh
+mkpragmatab
+mmap
+mmapped
+mno
+modeof
+Movepage
+mprintf
+msd
+msdos
+msec
+msg
+msgs
+msize
+msteveb
+msvc
+mtime
+mult
+multibyte
+multiplex'ed
+multiplexor
+multithreaded
+multiwrite
+mutex
+mutexes
+mutexing
+mx
+mxpathname
+myprefix
+nal
+namecontext
+namespace
+natively
+nbr
+nbsp
+ncell
+ncol
+ncycle
+nd
+ndlt
+neighbors
+neq
+nestable
+newrowid
+nfs
+nlt
+nnn
+nocase
+nochange
+nochng
+nofollow
+nolock
+nomem
+nomutex
+nonblocking
+nonroot
+noop
+noshm
+notational
+notheld
+notnull
+nowrap
+nr
+ntile
+nul
+nullable
+nullif
+nullvalue
+Num
+objproc
+oc
+offsetof
+ofst
+ogham
+oid
+Ok
+ok
+ol
+onecolumn
+onepass
+onoff
+onwards
+oom
+opcodesnever
+openable
+opendir
+optimizers
+optimizible
+orconf
+orderby
+orderbylist
+os
+Oswrite
+overread
+overreads
+overrideable
+oversize
+overwriteable
+ovewrite
+ovfl
+pagecache
+pagecaches
+pagecount
+pagelist
+pageno
+pagesize
+pagetype
+Param
+params
+passwd
+patchset
+pathname
+pathnames
+pc
+pcache
+pclose
+pcx
+pgno
+pgoffset
+pgsize
+pgsz
+pid
+pluggable
+pmasz
+pn
+popen
+pos
+posix
+Postgres
+Powersafe
+powersafe
+pq
+pqr
+pqrstuvw
+pragma
+pragmaed
+pragma's
+pragmas
+pre
+pread
+preallocate
+preallocated
+precisions
+precompiled
+precomputed
+prefetch
+prefetched
+preformated
+preformatted
+prepend
+prepended
+prepending
+prepends
+prepopulate
+preprocess
+preprocessed
+preprocessing
+preprocessor
+prereq
+prereqs
+preupdate
+primarykey
+printf
+printfs
+prng
+proc
+procs
+profiler
+proleptic
+proxying
+proxys
+pseudocode
+pseudotable
+psow
+psz
+pthread
+pthreads
+ptr
+ptrmap
+ptrs
+purgeable
+putsnl
+pwrite
+pz
+qbox
+Qcircle
+quotaed
+quotefix
+radix
+randomblob
+randstr
+rarr
+rc
+rcauth
+Rcvr
+rds
+readdir
+readline
+readlock
+readonly
+Readr
+realloc
+reallocs
+realvalue
+rebalance
+rebalanced
+recoverability
+redefinable
+redux
+reenabled
+reentrant
+refcount
+refcounts
+regs
+reindexed
+reinitializer
+reinitializes
+rekey
+rekeyed
+relevancies
+relink
+relink
+relock
+renormalize
+reoptimize
+reparse
+reparse
+reparsed
+reparsing
+reportable
+reprepare
+reprepare
+reprepared
+reprepares
+representable
+repurpose
+Req'd
+requeries
+requote
+reregister
+reseek
+reservebytes
+resumable
+retarget
+retargeted
+retrys
+returntype
+rfc
+rhs
+Ri
+Rivest
+ro
+rootpage
+rowid
+rowids
+Rowkey
+rownumber
+rowset
+rsync
+runtime
+rw
+rwc
+samplelib
+sampleno
+sandboxed
+sandboxing
+savepoint
+savepoints
+scanf
+scanstats
+scanstatus
+schemas
+sectorsize
+selecttrace
+selftest
+setrlimit
+setsubtype
+settitle
+sharable
+shm
+shmlock
+sibs
+sig
+signaling
+significand
+sizehint
+Sizeof
+sizeof
+snprintf
+Solaris
+sorterref
+soundex
+sourceid
+speedtest
+sprintf
+sql
+sqlar
+sqlite
+sqliteplocks
+sqliterc
+sqllog
+sqllogglobal
+sqlthread
+sqr
+sqrt
+src
+srcck
+statfs
+stderr
+stdin
+stdout
+stmt
+stmts
+str
+strace
+strcasecmp
+strcmp
+strdup
+strerror
+strftime
+strglob
+stricmp
+stringify
+strlen
+strlike
+strncmp
+strncpy
+strnicmp
+stronly
+strstr
+struct
+structs
+subbitmap
+subcases
+subclassed
+subclauses
+subcomponents
+subdirectory
+subelement
+subexpression
+subexpressions
+subfunction
+subfunctions
+subitem
+subjournals
+sublevels
+subnode
+suboptimal
+subpages
+subprocedures
+subprog
+subprogs
+subq
+subqueries
+subquery
+Subrtn
+subsec
+subsecond
+subsequence
+substr
+substring
+substrings
+subsubstructure
+subsubterms
+Subtask
+subtasks
+subterm
+subterms
+subtransaction
+subtransactions
+subtree
+subtrees
+subtype
+subtypes
+sumint
+superclass
+superlock
+superset
+superunlock
+symlinks
+synching
+sys
+syscall
+sz
+szosfile
+tablename
+tailval
+tailvar
+tbl
+tblname
+Tcl
+tcl
+Tclcmd
+tclcmd
+tclsh
+tclsqlite
+tclvar
+td
+tempfilename
+testcase
+testctrl
+testfixture
+testtag
+testvfs
+textarray
+th
+threadid
+threadsafe
+threadsafety
+throughs
+tht
+timediff
+tkt
+tm
+tmp
+tmpdir
+tmpfs
+tnum
+Todo
+tokenize
+tokenizer
+tokenizing
+tolower
+toobig
+toupper
+treetrace
+treeview
+trimleft
+trimright
+truesize
+trys
+Tsd
+Ts'o
+tunable
+tvfs
+txn
+txt
+Typecheck
+typedef
+typedefed
+typedefs
+typename
+typenames
+typeof
+tz
+uber
+uid
+uint
+ul
+umask
+Un
+un
+unallocated
+unanalyzed
+unary
+unbuffered
+unclosed
+uncompiled
+uncomputed
+undefining
+underfull
+unexpanded
+unfinalized
+unfreed
+unhex
+unicode
+unindexed
+uninit
+uninitialize
+unintuitive
+unioned
+unissued
+unix
+unixepoch
+unlink
+unlinked
+unlinking
+unlinks
+unmap
+unmapped
+unmapping
+unoptimized
+unparsed
+unreduced
+unref
+unreferenced
+unrefs
+unregister
+unregistering
+unregisters
+unresolvable
+unsynced
+unterminated
+untracked
+untrusted
+Upfrom
+uppercasing
+upr
+Upsert
+upsert
+upto
+uptr
+uri
+userauth
+userdata
+Userid
+usleep
+utc
+Utf
+utf
+util
+uuu
+uuuuu
+uuzzzz
+va
+valgrind
+vanishingly
+vappendf
+vararg
+varargs
+varint
+varints
+varname
+vcolumn
+vdbe
+vdbeapi
+vdbe's
+vdbes
+vdbesort
+ve
+verifications
+vfs
+vfslog
+vfsname
+vfs's
+vfstrace
+vm
+vmprintf
+vmstep
+vsnprintf
+vt
+vtab
+vtabs
+Vugt
+vvv
+vvvv
+vvvvv
+vvvvvv
+vwait
+vxworks
+wal
+wasm
+wherecode
+whereexpr
+wheretrace
+whitespace
+Willmann
+withoutrowid
+workalike
+wr
+wrapup
+writeable
+writecrash
+writefile
+wsd
+ww
+wwww
+wwzzzzyy
+wxyz
+xa
+xac
+xb
+xbf
+xc
+xd
+xdg
+xe
+xf
+xfe
+xfer
+xff
+xfff
+xfffd
+xfffe
+xffffffff
+x'hhhhhh
+xinfo
+xlc
+xtype
+xxxx
+xxxxx
+xxxxxx
+xxxxxxx
+xxxxxxxx
+xyz
+xyzzy
+yy
+yyxxxxxx
+yyy
+yyyyy
+yyyyyy
+zeroblob
+Zeroblobs
+zerodata
+zeropad
+zipfile
+zipvfs
+zplan
+zulu
+zzzz
+zzzzyyyy
diff --git a/tool/emcc.sh.in b/tool/emcc.sh.in
new file mode 100644
index 0000000000..1264df5376
--- /dev/null
+++ b/tool/emcc.sh.in
@@ -0,0 +1,66 @@
+#!/bin/sh
+# ^^^^^^^ Please try to keep this script Bourne-compatible.
+########################################################################
+# WARNING: emcc.sh is generated from emcc.sh.in by the configure
+# process. Do not edit emcc.sh directly, as it may be deleted or
+# overwritten by the configure script.
+#
+# A wrapper around the emcc compiler which uses configure-time state
+# to locate the Emscripten SDK and import the SDK's environment
+# script, if needed.
+########################################################################
+# EMSDK_HOME comes from the configure --with-emsdk=/dir flag.
+# EMSDK_ENV_SH is ${thatDir}/emsdk_env.sh and is also set by the
+# configure process.
+EMSDK_HOME="@EMSDK_HOME@"
+EMSDK_ENV_SH="@EMSDK_ENV_SH@"
+emcc="@BIN_EMCC@"
+
+if [ x = "x${emcc}" ]; then
+    emcc=`which emcc 2>/dev/null`
+fi
+
+if [ x = "x${emcc}" ]; then
+  # If emcc is not found in the path, try to find it via an emsdk
+  # installation. The SDK variant is the official installation style
+  # supported by the Emscripten project, but emcc is also available
+  # via package managers on some OSes.
+  if [ x = "x${EMSDK_HOME}" ]; then
+    echo "EMSDK_HOME is not set. Pass --with-emsdk=/path/to/emsdk" \
+         "to the configure script." 1>&2
+    exit 1
+  fi
+
+  if [ x = "x${EMSDK_ENV_SH}" ]; then
+    if [ -f "${EMSDK_HOME}/emsdk_env.sh" ]; then
+      EMSDK_ENV_SH="${EMSDK_HOME}/emsdk_env.sh"
+    else
+      echo "EMSDK_ENV_SH is not set. Expecting configure script to set it." 1>&2
+      exit 2
+    fi
+  fi
+
+  if [ ! -f "${EMSDK_ENV_SH}" ]; then
+    echo "emsdk_env script not found: $EMSDK_ENV_SH" 1>&2
+    exit 3
+  fi
+
+  # $EMSDK is part of the state set by emsdk_env.sh.
+  if [ x = "x${EMSDK}" ]; then
+    EMSDK_QUIET=1
+    export EMSDK_QUIET
+    # ^^^ Squelches informational output from ${EMSDK_ENV_SH}.
+    source "${EMSDK_ENV_SH}" || {
+      rc=$?
+      echo "Error sourcing ${EMSDK_ENV_SH}"
+      exit $rc
+    }
+  fi
+  emcc=`which emcc 2>/dev/null`
+  if [ x = "x${emcc}" ]; then
+    echo "emcc not found in PATH. Normally that's set up by ${EMSDK_ENV_SH}." 1>&2
+    exit 4
+  fi
+fi
+
+exec $emcc "$@"
diff --git a/tool/fuzzershell.c b/tool/fuzzershell.c
index 9a27103597..7a7aef0290 100644
--- a/tool/fuzzershell.c
+++ b/tool/fuzzershell.c
@@ -680,6 +680,11 @@ static sqlite3_module seriesModule = {
   0,                         /* xRollback */
   0,                         /* xFindMethod */
   0,                         /* xRename */
+  0,                         /* xSavepoint */
+  0,                         /* xRelease */
+  0,                         /* xRollbackTo */
+  0,                         /* xShadowName */
+  0                          /* xIntegrity */
 };
 /* END the generate_series(START,END,STEP) implementation
 *********************************************************************************/
diff --git a/tool/genfkey.README b/tool/genfkey.README
index 57cdff87f8..5609945042 100644
--- a/tool/genfkey.README
+++ b/tool/genfkey.README
@@ -63,7 +63,7 @@ CAPABILITIES
 
 LIMITATIONS
 
-  Apart from those limitiations described above:
+  Apart from those limitations described above:
 
     * Implicit mapping to composite primary keys is not supported. If
       a parent table has a composite primary key, then any child table
@@ -108,7 +108,7 @@ USAGE
 
   If errors are found and the --ignore-errors option is passed, then
   no error messages are printed. No "CREATE TRIGGER" statements are generated
-  for foriegn-key definitions that contained errors, they are silently
+  for foreign-key definitions that contained errors, they are silently
   ignored by subsequent processing.
 
   All triggers generated by this command have names that match the pattern
@@ -129,7 +129,7 @@ USAGE
   they will be printed to stdout, but this can be configured using other
   dot-commands (i.e. ".output").
 
-  The simplest way to activate the foriegn key definitions in a database
+  The simplest way to activate the foreign key definitions in a database
   is simply to open it using the shell tool and enter the command 
   ".genfkey --exec":
 
diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh
deleted file mode 100644
index 5f2c8345be..0000000000
--- a/tool/kvtest-speed.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-#
-# A script for running speed tests using kvtest.
-#
-# The test database must be set up first.  Recommended
-# command-line:
-#
-#    ./kvtest init kvtest.db --count 100K --size 12K --variance 5K
-
-if test "$1" = ""
-then
-  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
-  exit
-fi
-NAME=$1
-shift
-OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD"
-KVARGS="--count 100K --stats"
-gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest
-
-# First run using SQL
-rm cachegrind.out.[1-9][0-9]*
-valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt
-mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME
-cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt
-
-# Second run using the sqlite3_blob object
-valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt
-mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME
-cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt
-
-# Diff the sqlite3_blob API analysis for non-trunk runs.
-if test "$NAME" != "trunk"; then
-  fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt &
-fi
diff --git a/tool/lemon.c b/tool/lemon.c
index fb81292d4d..324dda0c5f 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -59,6 +59,82 @@ static char *msort(char*,char**,int(*)(const char*,const char*));
 */
 #define lemonStrlen(X)   ((int)strlen(X))
 
+/*
+** Header on the linked list of memory allocations.
+*/
+typedef struct MemChunk MemChunk;
+struct MemChunk {
+  MemChunk *pNext;
+  size_t sz;
+  /* Actually memory follows */
+};  
+
+/*
+** Global linked list of all memory allocations.
+*/
+static MemChunk *memChunkList = 0;
+
+/*
+** Wrappers around malloc(), calloc(), realloc() and free().
+**
+** All memory allocations are kept on a doubly-linked list.  The
+** lemon_free_all() function can be called prior to exit to clean
+** up any memory leaks.
+**
+** This is not necessary.  But compilers and getting increasingly
+** fussy about memory leaks, even in command-line programs like Lemon
+** where they do not matter.  So this code is provided to hush the
+** warnings.
+*/
+static void *lemon_malloc(size_t nByte){
+  MemChunk *p;
+  if( nByte<0 ) return 0;
+  p = malloc( nByte + sizeof(MemChunk) );
+  if( p==0 ){
+    fprintf(stderr, "Out of memory.  Failed to allocate %lld bytes.\n",
+            (long long int)nByte);
+    exit(1);
+  }
+  p->pNext = memChunkList;
+  p->sz = nByte;
+  memChunkList = p;
+  return (void*)&p[1];
+}
+static void *lemon_calloc(size_t nElem, size_t sz){
+  void *p = lemon_malloc(nElem*sz);
+  memset(p, 0, nElem*sz);
+  return p;
+}
+static void lemon_free(void *pOld){
+  if( pOld ){
+    MemChunk *p = (MemChunk*)pOld;
+    p--;
+    memset(pOld, 0, p->sz);
+  }
+}
+static void *lemon_realloc(void *pOld, size_t nNew){
+  void *pNew;
+  MemChunk *p;
+  if( pOld==0 ) return lemon_malloc(nNew);
+  p = (MemChunk*)pOld;
+  p--;
+  if( p->sz>=nNew ) return pOld;
+  pNew = lemon_malloc( nNew );
+  memcpy(pNew, pOld, p->sz);
+  return pNew;
+}
+
+/* Free all outstanding memory allocations.
+** Do this right before exiting.
+*/
+static void lemon_free_all(void){
+  while( memChunkList ){
+    MemChunk *pNext = memChunkList->pNext;
+    free( memChunkList );
+    memChunkList = pNext;
+  }
+}
+
 /*
 ** Compilers are starting to complain about the use of sprintf() and strcpy(),
 ** saying they are unsafe.  So we define our own versions of those routines too.
@@ -418,6 +494,8 @@ struct lemon {
   char *filename;          /* Name of the input file */
   char *outname;           /* Name of the current output file */
   char *tokenprefix;       /* A prefix added to token names in the .h file */
+  char *reallocFunc;       /* Function to use to allocate stack space */
+  char *freeFunc;          /* Function to use to free stack space */
   int nconflict;           /* Number of parsing conflicts */
   int nactiontab;          /* Number of entries in the yy_action[] table */
   int nlookaheadtab;       /* Number of entries in yy_lookahead[] */
@@ -426,7 +504,8 @@ struct lemon {
   int printPreprocessed;   /* Show preprocessor output on stdout */
   int has_fallback;        /* True if any %fallback is seen in the grammar */
   int nolinenosflag;       /* True if #line statements should not be printed */
-  char *argv0;             /* Name of the program */
+  int argc;                /* Number of command-line arguments */
+  char **argv;             /* Command-line arguments */
 };
 
 #define MemoryCheck(X) if((X)==0){ \
@@ -494,7 +573,7 @@ static struct action *Action_new(void){
   if( actionfreelist==0 ){
     int i;
     int amt = 100;
-    actionfreelist = (struct action *)calloc(amt, sizeof(struct action));
+    actionfreelist = (struct action *)lemon_calloc(amt, sizeof(struct action));
     if( actionfreelist==0 ){
       fprintf(stderr,"Unable to allocate memory for a new parser action.");
       exit(1);
@@ -613,14 +692,14 @@ struct acttab {
 
 /* Free all memory associated with the given acttab */
 void acttab_free(acttab *p){
-  free( p->aAction );
-  free( p->aLookahead );
-  free( p );
+  lemon_free( p->aAction );
+  lemon_free( p->aLookahead );
+  lemon_free( p );
 }
 
 /* Allocate a new acttab structure */
 acttab *acttab_alloc(int nsymbol, int nterminal){
-  acttab *p = (acttab *) calloc( 1, sizeof(*p) );
+  acttab *p = (acttab *) lemon_calloc( 1, sizeof(*p) );
   if( p==0 ){
     fprintf(stderr,"Unable to allocate memory for a new acttab.");
     exit(1);
@@ -639,7 +718,7 @@ acttab *acttab_alloc(int nsymbol, int nterminal){
 void acttab_action(acttab *p, int lookahead, int action){
   if( p->nLookahead>=p->nLookaheadAlloc ){
     p->nLookaheadAlloc += 25;
-    p->aLookahead = (struct lookahead_action *) realloc( p->aLookahead,
+    p->aLookahead = (struct lookahead_action *) lemon_realloc( p->aLookahead,
                              sizeof(p->aLookahead[0])*p->nLookaheadAlloc );
     if( p->aLookahead==0 ){
       fprintf(stderr,"malloc failed\n");
@@ -689,7 +768,7 @@ int acttab_insert(acttab *p, int makeItSafe){
   if( p->nAction + n >= p->nActionAlloc ){
     int oldAlloc = p->nActionAlloc;
     p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20;
-    p->aAction = (struct lookahead_action *) realloc( p->aAction,
+    p->aAction = (struct lookahead_action *) lemon_realloc( p->aAction,
                           sizeof(p->aAction[0])*p->nActionAlloc);
     if( p->aAction==0 ){
       fprintf(stderr,"malloc failed\n");
@@ -1311,7 +1390,7 @@ static struct config **basisend = 0;     /* End of list of basis configs */
 
 /* Return a pointer to a new configuration */
 PRIVATE struct config *newconfig(void){
-  return (struct config*)calloc(1, sizeof(struct config));
+  return (struct config*)lemon_calloc(1, sizeof(struct config));
 }
 
 /* The configuration "old" is no longer used */
@@ -1516,8 +1595,10 @@ void memory_error(void){
   exit(1);
 }
 
-static int nDefine = 0;      /* Number of -D options on the command line */
-static char **azDefine = 0;  /* Name of the -D macros */
+static int nDefine = 0;        /* Number of -D options on the command line */
+static int nDefineUsed = 0;    /* Number of -D options actually used */
+static char **azDefine = 0;    /* Name of the -D macros */
+static char *bDefineUsed = 0;  /* True for every -D macro actually used */
 
 /* This routine is called with the argument to each -D command-line option.
 ** Add the macro defined to the azDefine array.
@@ -1525,13 +1606,19 @@ static char **azDefine = 0;  /* Name of the -D macros */
 static void handle_D_option(char *z){
   char **paz;
   nDefine++;
-  azDefine = (char **) realloc(azDefine, sizeof(azDefine[0])*nDefine);
+  azDefine = (char **) lemon_realloc(azDefine, sizeof(azDefine[0])*nDefine);
   if( azDefine==0 ){
     fprintf(stderr,"out of memory\n");
     exit(1);
   }
+  bDefineUsed = (char*)lemon_realloc(bDefineUsed, nDefine);
+  if( bDefineUsed==0 ){
+    fprintf(stderr,"out of memory\n");
+    exit(1);
+  }
+  bDefineUsed[nDefine-1] = 0;
   paz = &azDefine[nDefine-1];
-  *paz = (char *) malloc( lemonStrlen(z)+1 );
+  *paz = (char *) lemon_malloc( lemonStrlen(z)+1 );
   if( *paz==0 ){
     fprintf(stderr,"out of memory\n");
     exit(1);
@@ -1541,11 +1628,28 @@ static void handle_D_option(char *z){
   *z = 0;
 }
 
+/* This routine is called with the argument to each -U command-line option.
+** Omit a previously defined macro.
+*/
+static void handle_U_option(char *z){
+  int i;
+  for(i=0; i<nDefine; i++){
+    if( strcmp(azDefine[i],z)==0 ){
+      nDefine--;
+      if( i<nDefine ){
+        azDefine[i] = azDefine[nDefine];
+        bDefineUsed[i] = bDefineUsed[nDefine];
+      }
+      break;
+    }
+  }
+}
+
 /* Rember the name of the output directory 
 */
 static char *outputDir = NULL;
 static void handle_d_option(char *z){
-  outputDir = (char *) malloc( lemonStrlen(z)+1 );
+  outputDir = (char *) lemon_malloc( lemonStrlen(z)+1 );
   if( outputDir==0 ){
     fprintf(stderr,"out of memory\n");
     exit(1);
@@ -1555,7 +1659,7 @@ static void handle_d_option(char *z){
 
 static char *user_templatename = NULL;
 static void handle_T_option(char *z){
-  user_templatename = (char *) malloc( lemonStrlen(z)+1 );
+  user_templatename = (char *) lemon_malloc( lemonStrlen(z)+1 );
   if( user_templatename==0 ){
     memory_error();
   }
@@ -1622,6 +1726,15 @@ static void stats_line(const char *zLabel, int iValue){
          iValue);
 }
 
+/*
+** Comparison function used by qsort() to sort the azDefine[] array.
+*/
+static int defineCmp(const void *pA, const void *pB){
+  const char *zA = *(const char**)pA;
+  const char *zB = *(const char**)pB;
+  return strcmp(zA,zB);
+}
+
 /* The main program.  Parse the command line and do it... */
 int main(int argc, char **argv){
   static int version = 0;
@@ -1658,6 +1771,7 @@ int main(int argc, char **argv){
                     "Generate the *.sql file describing the parser tables."},
     {OPT_FLAG, "x", (char*)&version, "Print the version number."},
     {OPT_FSTR, "T", (char*)handle_T_option, "Specify a template file."},
+    {OPT_FSTR, "U", (char*)handle_U_option, "Undefine a macro."},
     {OPT_FSTR, "W", 0, "Ignored.  (Placeholder for '-W' compiler options.)"},
     {OPT_FLAG,0,0,0}
   };
@@ -1666,7 +1780,6 @@ int main(int argc, char **argv){
   struct lemon lem;
   struct rule *rp;
 
-  (void)argc;
   OptInit(argv,options,stderr);
   if( version ){
      printf("Lemon version 1.0\n");
@@ -1678,12 +1791,14 @@ int main(int argc, char **argv){
   }
   memset(&lem, 0, sizeof(lem));
   lem.errorcnt = 0;
+  qsort(azDefine, nDefine, sizeof(azDefine[0]), defineCmp);
 
   /* Initialize the machine */
   Strsafe_init();
   Symbol_init();
   State_init();
-  lem.argv0 = argv[0];
+  lem.argv = argv;
+  lem.argc = argc;
   lem.filename = OptArg(0);
   lem.basisflag = basisflag;
   lem.nolinenosflag = nolinenosflag;
@@ -1792,6 +1907,7 @@ int main(int argc, char **argv){
 
   /* return 0 on success, 1 on failure. */
   exitcode = ((lem.errorcnt > 0) || (lem.nconflict > 0)) ? 1 : 0;
+  lemon_free_all();
   exit(exitcode);
   return (exitcode);
 }
@@ -1907,10 +2023,10 @@ static char *msort(
     list = NEXT(list);
     NEXT(ep) = 0;
     for(i=0; i<LISTSIZE-1 && set[i]!=0; i++){
-      ep = merge(ep,set[i],cmp,offset);
+      ep = merge(set[i],ep,cmp,offset);
       set[i] = 0;
     }
-    set[i] = ep;
+    set[i] = merge(set[i],ep,cmp,offset);
   }
   ep = 0;
   for(i=0; i<LISTSIZE; i++) if( set[i] ) ep = merge(set[i],ep,cmp,offset);
@@ -2380,7 +2496,7 @@ static void parseonetoken(struct pstate *psp)
     case IN_RHS:
       if( x[0]=='.' ){
         struct rule *rp;
-        rp = (struct rule *)calloc( sizeof(struct rule) +
+        rp = (struct rule *)lemon_calloc( sizeof(struct rule) +
              sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1);
         if( rp==0 ){
           ErrorMsg(psp->filename,psp->tokenlineno,
@@ -2432,17 +2548,17 @@ static void parseonetoken(struct pstate *psp)
         struct symbol *msp = psp->rhs[psp->nrhs-1];
         if( msp->type!=MULTITERMINAL ){
           struct symbol *origsp = msp;
-          msp = (struct symbol *) calloc(1,sizeof(*msp));
+          msp = (struct symbol *) lemon_calloc(1,sizeof(*msp));
           memset(msp, 0, sizeof(*msp));
           msp->type = MULTITERMINAL;
           msp->nsubsym = 1;
-          msp->subsym = (struct symbol **) calloc(1,sizeof(struct symbol*));
+          msp->subsym = (struct symbol**)lemon_calloc(1,sizeof(struct symbol*));
           msp->subsym[0] = origsp;
           msp->name = origsp->name;
           psp->rhs[psp->nrhs-1] = msp;
         }
         msp->nsubsym++;
-        msp->subsym = (struct symbol **) realloc(msp->subsym,
+        msp->subsym = (struct symbol **) lemon_realloc(msp->subsym,
           sizeof(struct symbol*)*msp->nsubsym);
         msp->subsym[msp->nsubsym-1] = Symbol_new(&x[1]);
         if( ISLOWER(x[1]) || ISLOWER(msp->subsym[0]->name[0]) ){
@@ -2522,6 +2638,12 @@ static void parseonetoken(struct pstate *psp)
         }else if( strcmp(x,"default_type")==0 ){
           psp->declargslot = &(psp->gp->vartype);
           psp->insertLineMacro = 0;
+        }else if( strcmp(x,"realloc")==0 ){
+          psp->declargslot = &(psp->gp->reallocFunc);
+          psp->insertLineMacro = 0;
+        }else if( strcmp(x,"free")==0 ){
+          psp->declargslot = &(psp->gp->freeFunc);
+          psp->insertLineMacro = 0;
         }else if( strcmp(x,"stack_size")==0 ){
           psp->declargslot = &(psp->gp->stacksize);
           psp->insertLineMacro = 0;
@@ -2652,7 +2774,7 @@ static void parseonetoken(struct pstate *psp)
           nLine = lemonStrlen(zLine);
           n += nLine + lemonStrlen(psp->filename) + nBack;
         }
-        *psp->declargslot = (char *) realloc(*psp->declargslot, n);
+        *psp->declargslot = (char *) lemon_realloc(*psp->declargslot, n);
         zBuf = *psp->declargslot + nOld;
         if( addLineMacro ){
           if( nOld && zBuf[-1]!='\n' ){
@@ -2766,7 +2888,7 @@ static void parseonetoken(struct pstate *psp)
       }else if( ISUPPER(x[0]) || ((x[0]=='|' || x[0]=='/') && ISUPPER(x[1])) ){
         struct symbol *msp = psp->tkclass;
         msp->nsubsym++;
-        msp->subsym = (struct symbol **) realloc(msp->subsym,
+        msp->subsym = (struct symbol **) lemon_realloc(msp->subsym,
           sizeof(struct symbol*)*msp->nsubsym);
         if( !ISUPPER(x[0]) ) x++;
         msp->subsym[msp->nsubsym-1] = Symbol_new(x);
@@ -2856,6 +2978,10 @@ static int eval_preprocessor_boolean(char *z, int lineno){
       res = 0;
       for(j=0; j<nDefine; j++){
         if( strncmp(azDefine[j],&z[i],n)==0 && azDefine[j][n]==0 ){
+          if( !bDefineUsed[j] ){
+            bDefineUsed[j] = 1;
+            nDefineUsed++;
+          }
           res = 1;
           break;
         }
@@ -2977,10 +3103,10 @@ void Parse(struct lemon *gp)
   fseek(fp,0,2);
   filesize = ftell(fp);
   rewind(fp);
-  filebuf = (char *)malloc( filesize+1 );
+  filebuf = (char *)lemon_malloc( filesize+1 );
   if( filesize>100000000 || filebuf==0 ){
     ErrorMsg(ps.filename,0,"Input file too large.");
-    free(filebuf);
+    lemon_free(filebuf);
     gp->errorcnt++;
     fclose(fp);
     return;
@@ -2988,7 +3114,7 @@ void Parse(struct lemon *gp)
   if( fread(filebuf,1,filesize,fp)!=filesize ){
     ErrorMsg(ps.filename,0,"Can't read in all %d bytes of this file.",
       filesize);
-    free(filebuf);
+    lemon_free(filebuf);
     gp->errorcnt++;
     fclose(fp);
     return;
@@ -3100,7 +3226,7 @@ void Parse(struct lemon *gp)
     *cp = (char)c;                  /* Restore the buffer */
     cp = nextcp;
   }
-  free(filebuf);                    /* Release the buffer after parsing */
+  lemon_free(filebuf);                    /* Release the buffer after parsing */
   gp->rule = ps.firstrule;
   gp->errorcnt = ps.errorcnt;
 }
@@ -3118,7 +3244,7 @@ struct plink *Plink_new(void){
   if( plink_freelist==0 ){
     int i;
     int amt = 100;
-    plink_freelist = (struct plink *)calloc( amt, sizeof(struct plink) );
+    plink_freelist = (struct plink *)lemon_calloc( amt, sizeof(struct plink) );
     if( plink_freelist==0 ){
       fprintf(stderr,
       "Unable to allocate memory for a new follow-set propagation link.\n");
@@ -3171,9 +3297,7 @@ void Plink_delete(struct plink *plp)
 ** Procedures for generating reports and tables in the LEMON parser generator.
 */
 
-/* Generate a filename with the given suffix.  Space to hold the
-** name comes from malloc() and must be freed by the calling
-** function.
+/* Generate a filename with the given suffix.
 */
 PRIVATE char *file_makename(struct lemon *lemp, const char *suffix)
 {
@@ -3190,7 +3314,7 @@ PRIVATE char *file_makename(struct lemon *lemp, const char *suffix)
   sz += lemonStrlen(suffix);
   if( outputDir ) sz += lemonStrlen(outputDir) + 1;
   sz += 5;
-  name = (char*)malloc( sz );
+  name = (char*)lemon_malloc( sz );
   if( name==0 ){
     fprintf(stderr,"Can't allocate space for a filename.\n");
     exit(1);
@@ -3217,7 +3341,7 @@ PRIVATE FILE *file_open(
 ){
   FILE *fp;
 
-  if( lemp->outname ) free(lemp->outname);
+  if( lemp->outname ) lemon_free(lemp->outname);
   lemp->outname = file_makename(lemp, suffix);
   fp = fopen(lemp->outname,mode);
   if( fp==0 && *mode=='w' ){
@@ -3533,14 +3657,14 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask)
   if( cp ){
     c = *cp;
     *cp = 0;
-    path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 );
+    path = (char *)lemon_malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 );
     if( path ) lemon_sprintf(path,"%s/%s",argv0,name);
     *cp = c;
   }else{
     pathlist = getenv("PATH");
     if( pathlist==0 ) pathlist = ".:/bin:/usr/bin";
-    pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1 );
-    path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 );
+    pathbuf = (char *) lemon_malloc( lemonStrlen(pathlist) + 1 );
+    path = (char *)lemon_malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 );
     if( (pathbuf != 0) && (path!=0) ){
       pathbufptr = pathbuf;
       lemon_strcpy(pathbuf, pathlist);
@@ -3556,7 +3680,7 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask)
         if( access(path,modemask)==0 ) break;
       }
     }
-    free(pathbufptr);
+    lemon_free(pathbufptr);
   }
   return path;
 }
@@ -3571,7 +3695,7 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap)
   switch( ap->type ){
     case SHIFT:  act = ap->x.stp->statenum;                        break;
     case SHIFTREDUCE: {
-      /* Since a SHIFT is inherient after a prior REDUCE, convert any
+      /* Since a SHIFT is inherent after a prior REDUCE, convert any
       ** SHIFTREDUCE action with a nonterminal on the LHS into a simple
       ** REDUCE action: */
       if( ap->sp->index>=lemp->nterminal
@@ -3674,7 +3798,7 @@ PRIVATE FILE *tplt_open(struct lemon *lemp)
   }else if( access(templatename,004)==0 ){
     tpltname = templatename;
   }else{
-    toFree = tpltname = pathsearch(lemp->argv0,templatename,0);
+    toFree = tpltname = pathsearch(lemp->argv[0],templatename,0);
   }
   if( tpltname==0 ){
     fprintf(stderr,"Can't find the parser driver template file \"%s\".\n",
@@ -3687,7 +3811,7 @@ PRIVATE FILE *tplt_open(struct lemon *lemp)
     fprintf(stderr,"Can't open the template file \"%s\".\n",tpltname);
     lemp->errorcnt++;
   }
-  free(toFree);
+  lemon_free(toFree);
   return in;
 }
 
@@ -3816,7 +3940,7 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){
   }
   if( (int) (n+sizeof(zInt)*2+used) >= alloced ){
     alloced = n + sizeof(zInt)*2 + used + 200;
-    z = (char *) realloc(z,  alloced);
+    z = (char *) lemon_realloc(z,  alloced);
   }
   if( z==0 ) return empty;
   while( n-- > 0 ){
@@ -3911,10 +4035,10 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
     }
   }
   if( lhsdirect ){
-    sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum);
+    lemon_sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum);
   }else{
     rc = 1;
-    sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum);
+    lemon_sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum);
   }
 
   append_str(0,0,0,0);
@@ -4106,7 +4230,7 @@ void print_stack_union(
 
   /* Allocate and initialize types[] and allocate stddt[] */
   arraysize = lemp->nsymbol * 2;
-  types = (char**)calloc( arraysize, sizeof(char*) );
+  types = (char**)lemon_calloc( arraysize, sizeof(char*) );
   if( types==0 ){
     fprintf(stderr,"Out of memory.\n");
     exit(1);
@@ -4123,7 +4247,7 @@ void print_stack_union(
     len = lemonStrlen(sp->datatype);
     if( len>maxdtlength ) maxdtlength = len;
   }
-  stddt = (char*)malloc( maxdtlength*2 + 1 );
+  stddt = (char*)lemon_malloc( maxdtlength*2 + 1 );
   if( stddt==0 ){
     fprintf(stderr,"Out of memory.\n");
     exit(1);
@@ -4172,7 +4296,7 @@ void print_stack_union(
     }
     if( types[hash]==0 ){
       sp->dtnum = hash + 1;
-      types[hash] = (char*)malloc( lemonStrlen(stddt)+1 );
+      types[hash] = (char*)lemon_malloc( lemonStrlen(stddt)+1 );
       if( types[hash]==0 ){
         fprintf(stderr,"Out of memory.\n");
         exit(1);
@@ -4194,13 +4318,13 @@ void print_stack_union(
   for(i=0; i<arraysize; i++){
     if( types[i]==0 ) continue;
     fprintf(out,"  %s yy%d;\n",types[i],i+1); lineno++;
-    free(types[i]);
+    lemon_free(types[i]);
   }
   if( lemp->errsym && lemp->errsym->useCnt ){
     fprintf(out,"  int yy%d;\n",lemp->errsym->dtnum); lineno++;
   }
-  free(stddt);
-  free(types);
+  lemon_free(stddt);
+  lemon_free(types);
   fprintf(out,"} YYMINORTYPE;\n"); lineno++;
   *plineno = lineno;
 }
@@ -4296,7 +4420,7 @@ void ReportTable(
   struct action *ap;
   struct rule *rp;
   struct acttab *pActtab;
-  int i, j, n, sz;
+  int i, j, n, sz, mn, mx;
   int nLookAhead;
   int szActionType;     /* sizeof(YYACTIONTYPE) */
   int szCodeType;       /* sizeof(YYCODETYPE)   */
@@ -4396,7 +4520,17 @@ void ReportTable(
 
   fprintf(out, 
      "/* This file is automatically generated by Lemon from input grammar\n"
-     "** source file \"%s\". */\n", lemp->filename); lineno += 2;
+     "** source file \"%s\"", lemp->filename);  lineno++;
+  if( nDefineUsed==0 ){
+    fprintf(out, ".\n*/\n"); lineno += 2;
+  }else{
+    fprintf(out, " with these options:\n**\n"); lineno += 2;
+    for(i=0; i<nDefine; i++){
+      if( !bDefineUsed[i] ) continue;
+      fprintf(out, "**   -D%s\n", azDefine[i]); lineno++;
+    }
+    fprintf(out, "*/\n"); lineno++;
+  }
   
   /* The first %include directive begins with a C-language comment,
   ** then skip over the header comment of the template file
@@ -4419,7 +4553,7 @@ void ReportTable(
   if( mhflag ){
     char *incName = file_makename(lemp, ".h");
     fprintf(out,"#include \"%s\"\n", incName); lineno++;
-    free(incName);
+    lemon_free(incName);
   }
   tplt_xfer(lemp->name,in,out,&lineno);
 
@@ -4478,6 +4612,21 @@ void ReportTable(
     fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
     fprintf(out,"#define %sARG_STORE\n",name); lineno++;
   }
+  if( lemp->reallocFunc ){
+    fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++;
+  }else{
+    fprintf(out,"#define YYREALLOC realloc\n"); lineno++;
+  }
+  if( lemp->freeFunc ){
+    fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++;
+  }else{
+    fprintf(out,"#define YYFREE free\n"); lineno++;
+  }
+  if( lemp->reallocFunc && lemp->freeFunc ){
+    fprintf(out,"#define YYDYNSTACK 1\n"); lineno++;
+  }else{
+    fprintf(out,"#define YYDYNSTACK 0\n"); lineno++;
+  }
   if( lemp->ctx && lemp->ctx[0] ){
     i = lemonStrlen(lemp->ctx);
     while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
@@ -4511,7 +4660,7 @@ void ReportTable(
   ** table must be computed before generating the YYNSTATE macro because
   ** we need to know how many states can be eliminated.
   */
-  ax = (struct axset *) calloc(lemp->nxstate*2, sizeof(ax[0]));
+  ax = (struct axset *) lemon_calloc(lemp->nxstate*2, sizeof(ax[0]));
   if( ax==0 ){
     fprintf(stderr,"malloc failed\n");
     exit(1);
@@ -4569,7 +4718,7 @@ void ReportTable(
     }
 #endif
   }
-  free(ax);
+  lemon_free(ax);
 
   /* Mark rules that are actually used for reduce actions after all
   ** optimizations have been applied
@@ -4601,6 +4750,22 @@ void ReportTable(
   fprintf(out,"#define YY_MIN_REDUCE        %d\n", lemp->minReduce); lineno++;
   i = lemp->minReduce + lemp->nrule;
   fprintf(out,"#define YY_MAX_REDUCE        %d\n", i-1); lineno++;
+
+  /* Minimum and maximum token values that have a destructor */
+  mn = mx = 0;
+  for(i=0; i<lemp->nsymbol; i++){
+    struct symbol *sp = lemp->symbols[i];
+
+    if( sp && sp->type!=TERMINAL && sp->destructor ){
+      if( mn==0 || sp->index<mn ) mn = sp->index;
+      if( sp->index>mx ) mx = sp->index;
+    }
+  }
+  if( lemp->tokendest ) mn = 0;
+  if( lemp->vardest ) mx = lemp->nsymbol-1;
+  fprintf(out,"#define YY_MIN_DSTRCTR       %d\n", mn);  lineno++;
+  fprintf(out,"#define YY_MAX_DSTRCTR       %d\n", mx);  lineno++;    
+
   tplt_xfer(lemp->name,in,out,&lineno);
 
   /* Now output the action table and its associates:
@@ -4744,7 +4909,7 @@ void ReportTable(
   /* Generate the table of fallback tokens.
   */
   if( lemp->has_fallback ){
-    int mx = lemp->nterminal - 1;
+    mx = lemp->nterminal - 1;
     /* 2019-08-28:  Generate fallback entries for every token to avoid
     ** having to do a range check on the index */
     /* while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } */
@@ -5179,7 +5344,7 @@ void SetSize(int n)
 /* Allocate a new set */
 char *SetNew(void){
   char *s;
-  s = (char*)calloc( size, 1);
+  s = (char*)lemon_calloc( size, 1);
   if( s==0 ){
     memory_error();
   }
@@ -5189,7 +5354,7 @@ char *SetNew(void){
 /* Deallocate a set */
 void SetFree(char *s)
 {
-  free(s);
+  lemon_free(s);
 }
 
 /* Add a new element to the set.  Return TRUE if the element was added
@@ -5248,7 +5413,7 @@ const char *Strsafe(const char *y)
 
   if( y==0 ) return 0;
   z = Strsafe_find(y);
-  if( z==0 && (cpy=(char *)malloc( lemonStrlen(y)+1 ))!=0 ){
+  if( z==0 && (cpy=(char *)lemon_malloc( lemonStrlen(y)+1 ))!=0 ){
     lemon_strcpy(cpy,y);
     z = cpy;
     Strsafe_insert(z);
@@ -5284,13 +5449,13 @@ static struct s_x1 *x1a;
 /* Allocate a new associative array */
 void Strsafe_init(void){
   if( x1a ) return;
-  x1a = (struct s_x1*)malloc( sizeof(struct s_x1) );
+  x1a = (struct s_x1*)lemon_malloc( sizeof(struct s_x1) );
   if( x1a ){
     x1a->size = 1024;
     x1a->count = 0;
-    x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*));
+    x1a->tbl = (x1node*)lemon_calloc(1024, sizeof(x1node) + sizeof(x1node*));
     if( x1a->tbl==0 ){
-      free(x1a);
+      lemon_free(x1a);
       x1a = 0;
     }else{
       int i;
@@ -5325,7 +5490,7 @@ int Strsafe_insert(const char *data)
     struct s_x1 array;
     array.size = arrSize = x1a->size*2;
     array.count = x1a->count;
-    array.tbl = (x1node*)calloc(arrSize, sizeof(x1node) + sizeof(x1node*));
+    array.tbl = (x1node*)lemon_calloc(arrSize, sizeof(x1node)+sizeof(x1node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x1node**)&(array.tbl[arrSize]);
     for(i=0; i<arrSize; i++) array.ht[i] = 0;
@@ -5340,7 +5505,7 @@ int Strsafe_insert(const char *data)
       newnp->from = &(array.ht[h]);
       array.ht[h] = newnp;
     }
-    /* free(x1a->tbl); // This program was originally for 16-bit machines.
+    /* lemon_free(x1a->tbl); // This program was originally for 16-bit machines.
     ** Don't worry about freeing memory on modern platforms. */
     *x1a = array;
   }
@@ -5381,7 +5546,7 @@ struct symbol *Symbol_new(const char *x)
 
   sp = Symbol_find(x);
   if( sp==0 ){
-    sp = (struct symbol *)calloc(1, sizeof(struct symbol) );
+    sp = (struct symbol *)lemon_calloc(1, sizeof(struct symbol) );
     MemoryCheck(sp);
     sp->name = Strsafe(x);
     sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL;
@@ -5452,13 +5617,13 @@ static struct s_x2 *x2a;
 /* Allocate a new associative array */
 void Symbol_init(void){
   if( x2a ) return;
-  x2a = (struct s_x2*)malloc( sizeof(struct s_x2) );
+  x2a = (struct s_x2*)lemon_malloc( sizeof(struct s_x2) );
   if( x2a ){
     x2a->size = 128;
     x2a->count = 0;
-    x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*));
+    x2a->tbl = (x2node*)lemon_calloc(128, sizeof(x2node) + sizeof(x2node*));
     if( x2a->tbl==0 ){
-      free(x2a);
+      lemon_free(x2a);
       x2a = 0;
     }else{
       int i;
@@ -5493,7 +5658,7 @@ int Symbol_insert(struct symbol *data, const char *key)
     struct s_x2 array;
     array.size = arrSize = x2a->size*2;
     array.count = x2a->count;
-    array.tbl = (x2node*)calloc(arrSize, sizeof(x2node) + sizeof(x2node*));
+    array.tbl = (x2node*)lemon_calloc(arrSize, sizeof(x2node)+sizeof(x2node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x2node**)&(array.tbl[arrSize]);
     for(i=0; i<arrSize; i++) array.ht[i] = 0;
@@ -5509,7 +5674,7 @@ int Symbol_insert(struct symbol *data, const char *key)
       newnp->from = &(array.ht[h]);
       array.ht[h] = newnp;
     }
-    /* free(x2a->tbl); // This program was originally written for 16-bit
+    /* lemon_free(x2a->tbl); // This program was originally written for 16-bit
     ** machines.  Don't worry about freeing this trivial amount of memory
     ** on modern platforms.  Just leak it. */
     *x2a = array;
@@ -5570,7 +5735,7 @@ struct symbol **Symbol_arrayof()
   int i,arrSize;
   if( x2a==0 ) return 0;
   arrSize = x2a->count;
-  array = (struct symbol **)calloc(arrSize, sizeof(struct symbol *));
+  array = (struct symbol **)lemon_calloc(arrSize, sizeof(struct symbol *));
   if( array ){
     for(i=0; i<arrSize; i++) array[i] = x2a->tbl[i].data;
   }
@@ -5618,7 +5783,7 @@ PRIVATE unsigned statehash(struct config *a)
 struct state *State_new()
 {
   struct state *newstate;
-  newstate = (struct state *)calloc(1, sizeof(struct state) );
+  newstate = (struct state *)lemon_calloc(1, sizeof(struct state) );
   MemoryCheck(newstate);
   return newstate;
 }
@@ -5651,13 +5816,13 @@ static struct s_x3 *x3a;
 /* Allocate a new associative array */
 void State_init(void){
   if( x3a ) return;
-  x3a = (struct s_x3*)malloc( sizeof(struct s_x3) );
+  x3a = (struct s_x3*)lemon_malloc( sizeof(struct s_x3) );
   if( x3a ){
     x3a->size = 128;
     x3a->count = 0;
-    x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*));
+    x3a->tbl = (x3node*)lemon_calloc(128, sizeof(x3node) + sizeof(x3node*));
     if( x3a->tbl==0 ){
-      free(x3a);
+      lemon_free(x3a);
       x3a = 0;
     }else{
       int i;
@@ -5692,7 +5857,7 @@ int State_insert(struct state *data, struct config *key)
     struct s_x3 array;
     array.size = arrSize = x3a->size*2;
     array.count = x3a->count;
-    array.tbl = (x3node*)calloc(arrSize, sizeof(x3node) + sizeof(x3node*));
+    array.tbl = (x3node*)lemon_calloc(arrSize, sizeof(x3node)+sizeof(x3node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x3node**)&(array.tbl[arrSize]);
     for(i=0; i<arrSize; i++) array.ht[i] = 0;
@@ -5708,7 +5873,7 @@ int State_insert(struct state *data, struct config *key)
       newnp->from = &(array.ht[h]);
       array.ht[h] = newnp;
     }
-    free(x3a->tbl);
+    lemon_free(x3a->tbl);
     *x3a = array;
   }
   /* Insert the new data */
@@ -5749,7 +5914,7 @@ struct state **State_arrayof(void)
   int i,arrSize;
   if( x3a==0 ) return 0;
   arrSize = x3a->count;
-  array = (struct state **)calloc(arrSize, sizeof(struct state *));
+  array = (struct state **)lemon_calloc(arrSize, sizeof(struct state *));
   if( array ){
     for(i=0; i<arrSize; i++) array[i] = x3a->tbl[i].data;
   }
@@ -5760,7 +5925,7 @@ struct state **State_arrayof(void)
 PRIVATE unsigned confighash(struct config *a)
 {
   unsigned h=0;
-  h = h*571 + a->rp->index*37 + a->dot;
+  h = a->rp->index*37 + a->dot;
   return h;
 }
 
@@ -5791,13 +5956,13 @@ static struct s_x4 *x4a;
 /* Allocate a new associative array */
 void Configtable_init(void){
   if( x4a ) return;
-  x4a = (struct s_x4*)malloc( sizeof(struct s_x4) );
+  x4a = (struct s_x4*)lemon_malloc( sizeof(struct s_x4) );
   if( x4a ){
     x4a->size = 64;
     x4a->count = 0;
-    x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*));
+    x4a->tbl = (x4node*)lemon_calloc(64, sizeof(x4node) + sizeof(x4node*));
     if( x4a->tbl==0 ){
-      free(x4a);
+      lemon_free(x4a);
       x4a = 0;
     }else{
       int i;
@@ -5832,7 +5997,8 @@ int Configtable_insert(struct config *data)
     struct s_x4 array;
     array.size = arrSize = x4a->size*2;
     array.count = x4a->count;
-    array.tbl = (x4node*)calloc(arrSize, sizeof(x4node) + sizeof(x4node*));
+    array.tbl = (x4node*)lemon_calloc(arrSize,
+                                      sizeof(x4node) + sizeof(x4node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x4node**)&(array.tbl[arrSize]);
     for(i=0; i<arrSize; i++) array.ht[i] = 0;
@@ -5847,9 +6013,6 @@ int Configtable_insert(struct config *data)
       newnp->from = &(array.ht[h]);
       array.ht[h] = newnp;
     }
-    /* free(x4a->tbl); // This code was originall written for 16-bit machines.
-    ** on modern machines, don't worry about freeing this trival amount of
-    ** memory. */
     *x4a = array;
   }
   /* Insert the new data */
diff --git a/tool/lempar.c b/tool/lempar.c
index 8cc57897db..74314efeaa 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -13,7 +13,7 @@
 **
 ** The "lemon" program processes an LALR(1) input grammar file, then uses
 ** this template to construct a parser.  The "lemon" program inserts text
-** at each "%%" line.  Also, any "P-a-r-s-e" identifer prefix (without the
+** at each "%%" line.  Also, any "P-a-r-s-e" identifier prefix (without the
 ** interstitial "-" characters) contained in this template is changed into
 ** the value of the %name directive from the grammar.  Otherwise, the content
 ** of this template is copied straight through into the generate parser
@@ -67,6 +67,9 @@
 **    ParseARG_STORE     Code to store %extra_argument into yypParser
 **    ParseARG_FETCH     Code to extract %extra_argument from yypParser
 **    ParseCTX_*         As ParseARG_ except for %extra_context
+**    YYREALLOC          Name of the realloc() function to use
+**    YYFREE             Name of the free() function to use
+**    YYDYNSTACK         True if stack space should be extended on heap
 **    YYERRORSYMBOL      is the code number of the error symbol.  If not
 **                       defined, then do no error processing.
 **    YYNSTATE           the combined number of states.
@@ -80,6 +83,8 @@
 **    YY_NO_ACTION       The yy_action[] code for no-op
 **    YY_MIN_REDUCE      Minimum value for reduce actions
 **    YY_MAX_REDUCE      Maximum value for reduce actions
+**    YY_MIN_DSTRCTR     Minimum symbol value that has a destructor
+**    YY_MAX_DSTRCTR     Maximum symbol value that has a destructor
 */
 #ifndef INTERFACE
 # define INTERFACE 1
@@ -101,6 +106,22 @@
 # define yytestcase(X)
 #endif
 
+/* Macro to determine if stack space has the ability to grow using
+** heap memory.
+*/
+#if YYSTACKDEPTH<=0 || YYDYNSTACK
+# define YYGROWABLESTACK 1
+#else
+# define YYGROWABLESTACK 0
+#endif
+
+/* Guarantee a minimum number of initial stack slots.
+*/
+#if YYSTACKDEPTH<=0
+# undef YYSTACKDEPTH
+# define YYSTACKDEPTH 2  /* Need a minimum stack size */
+#endif
+
 
 /* Next are the tables used to determine what action to take based on the
 ** current state and lookahead token.  These tables are used to implement
@@ -212,14 +233,9 @@ struct yyParser {
 #endif
   ParseARG_SDECL                /* A place to hold %extra_argument */
   ParseCTX_SDECL                /* A place to hold %extra_context */
-#if YYSTACKDEPTH<=0
-  int yystksz;                  /* Current side of the stack */
-  yyStackEntry *yystack;        /* The parser's stack */
-  yyStackEntry yystk0;          /* First stack entry */
-#else
-  yyStackEntry yystack[YYSTACKDEPTH];  /* The parser's stack */
-  yyStackEntry *yystackEnd;            /* Last entry in the stack */
-#endif
+  yyStackEntry *yystackEnd;           /* Last entry in the stack */
+  yyStackEntry *yystack;              /* The parser stack */
+  yyStackEntry yystk0[YYSTACKDEPTH];  /* Initial stack space */
 };
 typedef struct yyParser yyParser;
 
@@ -273,37 +289,45 @@ static const char *const yyRuleName[] = {
 #endif /* NDEBUG */
 
 
-#if YYSTACKDEPTH<=0
+#if YYGROWABLESTACK
 /*
 ** Try to increase the size of the parser stack.  Return the number
 ** of errors.  Return 0 on success.
 */
 static int yyGrowStack(yyParser *p){
+  int oldSize = 1 + (int)(p->yystackEnd - p->yystack);
   int newSize;
   int idx;
   yyStackEntry *pNew;
 
-  newSize = p->yystksz*2 + 100;
-  idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
-  if( p->yystack==&p->yystk0 ){
-    pNew = malloc(newSize*sizeof(pNew[0]));
-    if( pNew ) pNew[0] = p->yystk0;
+  newSize = oldSize*2 + 100;
+  idx = (int)(p->yytos - p->yystack);
+  if( p->yystack==p->yystk0 ){
+    pNew = YYREALLOC(0, newSize*sizeof(pNew[0]));
+    if( pNew==0 ) return 1;
+    memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0]));
   }else{
-    pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+    pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]));
+    if( pNew==0 ) return 1;
   }
-  if( pNew ){
-    p->yystack = pNew;
-    p->yytos = &p->yystack[idx];
+  p->yystack = pNew;
+  p->yytos = &p->yystack[idx];
 #ifndef NDEBUG
-    if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
-              yyTracePrompt, p->yystksz, newSize);
-    }
-#endif
-    p->yystksz = newSize;
+  if( yyTraceFILE ){
+    fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
+            yyTracePrompt, oldSize, newSize);
   }
-  return pNew==0; 
+#endif
+  p->yystackEnd = &p->yystack[newSize-1];
+  return 0;
 }
+#endif /* YYGROWABLESTACK */
+
+#if !YYGROWABLESTACK
+/* For builds that do no have a growable stack, yyGrowStack always
+** returns an error.
+*/
+# define yyGrowStack(X) 1
 #endif
 
 /* Datatype of the argument to the memory allocated passed as the
@@ -323,24 +347,14 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){
 #ifdef YYTRACKMAXSTACKDEPTH
   yypParser->yyhwm = 0;
 #endif
-#if YYSTACKDEPTH<=0
-  yypParser->yytos = NULL;
-  yypParser->yystack = NULL;
-  yypParser->yystksz = 0;
-  if( yyGrowStack(yypParser) ){
-    yypParser->yystack = &yypParser->yystk0;
-    yypParser->yystksz = 1;
-  }
-#endif
+  yypParser->yystack = yypParser->yystk0;
+  yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
 #ifndef YYNOERRORRECOVERY
   yypParser->yyerrcnt = -1;
 #endif
   yypParser->yytos = yypParser->yystack;
   yypParser->yystack[0].stateno = 0;
   yypParser->yystack[0].major = 0;
-#if YYSTACKDEPTH>0
-  yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
-#endif
 }
 
 #ifndef Parse_ENGINEALWAYSONSTACK
@@ -426,9 +440,26 @@ static void yy_pop_parser_stack(yyParser *pParser){
 */
 void ParseFinalize(void *p){
   yyParser *pParser = (yyParser*)p;
-  while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
-#if YYSTACKDEPTH<=0
-  if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
+
+  /* In-lined version of calling yy_pop_parser_stack() for each
+  ** element left in the stack */
+  yyStackEntry *yytos = pParser->yytos;
+  while( yytos>pParser->yystack ){
+#ifndef NDEBUG
+    if( yyTraceFILE ){
+      fprintf(yyTraceFILE,"%sPopping %s\n",
+        yyTracePrompt,
+        yyTokenName[yytos->major]);
+    }
+#endif
+    if( yytos->major>=YY_MIN_DSTRCTR ){
+      yy_destructor(pParser, yytos->major, &yytos->minor);
+    }
+    yytos--;
+  }
+
+#if YYGROWABLESTACK
+  if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack);
 #endif
 }
 
@@ -654,25 +685,19 @@ static void yy_shift(
     assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
   }
 #endif
-#if YYSTACKDEPTH>0 
-  if( yypParser->yytos>yypParser->yystackEnd ){
-    yypParser->yytos--;
-    yyStackOverflow(yypParser);
-    return;
-  }
-#else
-  if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
+  yytos = yypParser->yytos;
+  if( yytos>yypParser->yystackEnd ){
     if( yyGrowStack(yypParser) ){
       yypParser->yytos--;
       yyStackOverflow(yypParser);
       return;
     }
+    yytos = yypParser->yytos;
+    assert( yytos <= yypParser->yystackEnd );
   }
-#endif
   if( yyNewState > YY_MAX_SHIFT ){
     yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
   }
-  yytos = yypParser->yytos;
   yytos->stateno = yyNewState;
   yytos->major = yyMajor;
   yytos->minor.yy0 = yyMinor;
@@ -911,19 +936,12 @@ void Parse(
                   (int)(yypParser->yytos - yypParser->yystack));
         }
 #endif
-#if YYSTACKDEPTH>0 
         if( yypParser->yytos>=yypParser->yystackEnd ){
-          yyStackOverflow(yypParser);
-          break;
-        }
-#else
-        if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
           if( yyGrowStack(yypParser) ){
             yyStackOverflow(yypParser);
             break;
           }
         }
-#endif
       }
       yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor ParseCTX_PARAM);
     }else if( yyact <= YY_MAX_SHIFTREDUCE ){
diff --git a/tool/loadfts.c b/tool/loadfts.c
index 0000797b88..5ed03b7bdc 100644
--- a/tool/loadfts.c
+++ b/tool/loadfts.c
@@ -145,6 +145,7 @@ static void traverse(
   for(e=readdir(d); e; e=readdir(d)){
     if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue;
     char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name);
+    if( zPath==0 ){ fprintf(stderr, "out of memory\n"); abort(); }
     if (e->d_type & DT_DIR) {
       traverse(zPath, pCtx, xCallback);
     }else{
diff --git a/tool/mkamalzip.tcl b/tool/mkamalzip.tcl
new file mode 100644
index 0000000000..92feb4122e
--- /dev/null
+++ b/tool/mkamalzip.tcl
@@ -0,0 +1,23 @@
+#!/usr/bin/tclsh
+#
+# Build a ZIP archive for the amalgamation source code found in the current
+# directory.
+#
+set VERSION-file [file dirname [file dirname [file normalize $argv0]]]/VERSION
+set fd [open ${VERSION-file} rb]
+set vers [read $fd]
+close $fd
+scan $vers %d.%d.%d major minor patch
+set numvers [format {3%02d%02d00} $minor $patch]
+set dir sqlite-amalgamation-$numvers
+file delete -force $dir
+file mkdir $dir
+set filelist {sqlite3.c sqlite3.h shell.c sqlite3ext.h}
+foreach f $filelist {
+  file copy $f $dir/$f
+}
+set cmd "zip -r $dir.zip $dir"
+puts $cmd
+file delete -force $dir.zip
+exec {*}$cmd
+file delete -force $dir
diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh
index eacd9fa515..3835799a6f 100644
--- a/tool/mkautoconfamal.sh
+++ b/tool/mkautoconfamal.sh
@@ -13,7 +13,7 @@
 #
 
 
-# Bail out of the script if any command returns a non-zero exit 
+# Bail out of the script if any command returns a non-zero exit
 # status. Or if the script tries to use an unset variable. These
 # may fail for old /bin/sh interpreters.
 #
@@ -22,16 +22,16 @@ set -u
 
 TMPSPACE=./mkpkg_tmp_dir
 VERSION=`cat $TOP/VERSION`
-HASH=`sed 's/^\(..........\).*/\1/' $TOP/manifest.uuid`
-DATETIME=`grep '^D' $TOP/manifest | sed -e 's/[^0-9]//g' -e 's/\(............\).*/\1/'`
+HASH=`cut -c1-10 $TOP/manifest.uuid`
+DATETIME=`grep '^D' $TOP/manifest | tr -c -d '[0-9]' | cut -c1-12`
 
 # If this script is given an argument of --snapshot, then generate a
-# snapshot tarball named for the current checkout SHA1 hash, rather than
+# snapshot tarball named for the current checkout SHA hash, rather than
 # the version number.
 #
 if test "$#" -ge 1 -a x$1 != x--snapshot
 then
-  # Set global variable $ARTIFACT to the "3xxyyzz" string incorporated 
+  # Set global variable $ARTIFACT to the "3xxyyzz" string incorporated
   # into artifact filenames. And $VERSION2 to the "3.x.y[.z]" form.
   xx=`echo $VERSION|sed 's/3\.\([0-9]*\)\..*/\1/'`
   yy=`echo $VERSION|sed 's/3\.[^.]*\.\([0-9]*\).*/\1/'`
@@ -46,6 +46,8 @@ fi
 
 rm -rf $TMPSPACE
 cp -R $TOP/autoconf       $TMPSPACE
+cp -R $TOP/autosetup      $TMPSPACE
+cp -p $TOP/configure      $TMPSPACE
 cp sqlite3.c              $TMPSPACE
 cp sqlite3.h              $TMPSPACE
 cp sqlite3ext.h           $TMPSPACE
@@ -55,39 +57,39 @@ cp $TOP/sqlite3.pc.in     $TMPSPACE
 cp shell.c                $TMPSPACE
 cp $TOP/src/sqlite3.rc    $TMPSPACE
 cp $TOP/tool/Replace.cs   $TMPSPACE
-
-cat $TMPSPACE/configure.ac |
-sed "s/--SQLITE-VERSION--/$VERSION/" > $TMPSPACE/tmp
-mv $TMPSPACE/tmp $TMPSPACE/configure.ac
+cp $TOP/VERSION           $TMPSPACE
+cp $TOP/main.mk           $TMPSPACE
 
 cd $TMPSPACE
-autoreconf -i
-#libtoolize
-#aclocal
-#autoconf
-#automake --add-missing
+
+#if true; then
+  # Clean up *~ files (emacs-generated backups).
+  # This bit is only for use during development of
+  # the autoconf bundle.
+#  find . -name '*~' -exec rm \{} \;
+#fi
 
 mkdir -p tea/generic
-echo "#ifdef USE_SYSTEM_SQLITE"      > tea/generic/tclsqlite3.c 
-echo "# include <sqlite3.h>"        >> tea/generic/tclsqlite3.c
-echo "#else"                        >> tea/generic/tclsqlite3.c
-echo "#include \"sqlite3.c\""       >> tea/generic/tclsqlite3.c
-echo "#endif"                       >> tea/generic/tclsqlite3.c
+cat <<EOF > tea/generic/tclsqlite3.c
+#ifdef USE_SYSTEM_SQLITE
+# include <sqlite3.h>
+#else
+# include "sqlite3.c"
+#endif
+EOF
 cat  $TOP/src/tclsqlite.c           >> tea/generic/tclsqlite3.c
 
-cat tea/configure.ac | 
-  sed "s/AC_INIT(\[sqlite\], .*)/AC_INIT([sqlite], [$VERSION])/" > tmp
-mv tmp tea/configure.ac
-
-cd tea
-autoconf
-rm -rf autom4te.cache
+# Clean up some local remnants from the tarball.
+rm -f tea/.env-* # autosetup environment overrides
+find . -type f -name '*~' -exec rm -f \{} \;  # backup files
+find . -type f -name '#*#' -exec rm -f \{} \; # emacs lock files
+find . -type f -name '*.o' -exec rm -f \{} \;
+find . -type f -name '*.so' -exec rm -f \{} \;
 
-cd ../
-./configure && make dist
-tar -xzf sqlite-$VERSION.tar.gz
+./configure && ${MAKE-make} dist
+tar xzf sqlite-$VERSION.tar.gz
 mv sqlite-$VERSION $TARBALLNAME
-tar -czf $TARBALLNAME.tar.gz $TARBALLNAME
+tar czf $TARBALLNAME.tar.gz $TARBALLNAME
 mv $TARBALLNAME.tar.gz ..
 cd ..
 ls -l $TARBALLNAME.tar.gz
diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl
index 41b09f1e81..8b4fae82c9 100755
--- a/tool/mkccode.tcl
+++ b/tool/mkccode.tcl
@@ -1,17 +1,16 @@
-#!/usr/bin/tclsh
+#!/bin/env tclsh
 #
-# Use this script to build C-language source code for a program that uses
-# tclsqlite.c together with custom TCL scripts and/or C extensions for
-# either SQLite or TCL.
+# This script is used to amalgamate C source code files into a single
+# unit.
 #
 # Usage example:
 #
-#     tclsh mktclsqliteprog.tcl demoapp.c.in >demoapp.c
+#     tclsh mkccode.tcl -DENABLE_FEATURE_XYZ demoapp.c.in >demoapp.c
 #
 # The demoapp.c.in file contains a mixture of C code, TCL script, and
-# processing directives used by mktclsqliteprog.tcl to build the final C-code
-# output file.  Most lines of demoapp.c.in are copied straight through into
-# the output.  The following control directives are recognized:
+# processing directives used by mkccode.tcl to build the final C-code
+# output file.  Most lines of demoapp.c.in are copied straight through
+# into the output.  The following control directives are recognized:
 #
 # BEGIN_STRING
 #
@@ -33,31 +32,72 @@
 #      then all of the text in the input file is converted into C-language
 #      string literals.
 #
+# IFDEF macro
+# IFNDEF macro
+# ELSE
+# ENDIF
+#
+#      The text from "IFDEF macro" down to the next ELSE or ENDIF is
+#      included only if -Dmacro appears as a command-line argument.
+#      The "IFNDEF macro" simply inverts the initial test.
+#
 # None of the control directives described above will nest.  Only the
 # top-level input file ("demoapp.c.in" in the example) is interpreted.
 # referenced files are copied verbatim.
 #
-if {[llength $argv]!=1} {
-  puts stderr "Usage: $argv0 TEMPLATE >OUTPUT"
+proc usage {} {
+  puts stderr "Usage: $::argv0 \[OPTIONS\] TEMPLATE >OUTPUT"
   exit 1
 }
-set infile [lindex $argv 0]
+set infile {}
+foreach ax $argv {
+  if {[string match -D* $ax]} {
+    if {[string match *=* $ax]} {
+      regexp -- {-D([^=]+)=(.*)} $ax all name value
+      set DEF($name) $value
+    } else {
+      set DEF([string range $ax 2 end]) 1
+    }
+    continue
+  }
+  if {[string match -* $ax]} {
+    puts stderr "$::argv0:  Unknown option \"$ax\""
+    usage
+  }
+  if {$infile!=""} {
+    puts stderr "$::argv0:  Surplus argument: \"$ax\""
+    usage
+  }
+  set infile $ax
+}
 set ROOT [file normalize [file dir $argv0]/..]
 set HOME [file normalize [file dir $infile]]
 set in [open $infile rb]
 puts [subst {/* DO NOT EDIT
 **
-** This file was generated by \"$argv0 $infile\".
+** This file was generated by \"$argv0 $argv\".
 ** To make changes, edit $infile then rerun the generator
 ** command.
 */}]
 set instr 0
+set omit {}
+set nomit 0
+set ln 0
 while {1} {
   set line [gets $in]
+  incr ln
   if {[eof $in]} break
   if {[regexp {^INCLUDE (.*)} $line all path]} {
-    regsub {^\$ROOT\y} $path $ROOT path
-    regsub {^\$HOME\y} $path $HOME path
+    if {$nomit>0 && [string match *1* $omit]} continue
+    if {0} {
+      # https://github.com/msteveb/jimtcl/issues/320
+      regsub {^\$ROOT\y} $path $ROOT path
+      regsub {^\$HOME\y} $path $HOME path
+    } else {
+      set path [string map "\$ROOT $ROOT" $path]
+      set path [string map "\$HOME $HOME" $path]
+      # or: set path [string map "\$HOME $HOME \$ROOT $ROOT" $path]
+    }
     set in2 [open $path rb]
     puts "/* INCLUDE $path */"
     if {$instr} {
@@ -84,10 +124,43 @@ while {1} {
     puts "/* END_STRING */"
     continue
   }
-  if {$instr} {
+  if {[regexp {^IFNDEF +([A-Za-z_0-9]+)} $line all name]} {
+    set omit $omit[info exists DEF($name)]
+    incr nomit
+    continue
+  }
+  if {[regexp {^IFDEF +([A-Za-z_0-9]+)} $line all name]} {
+    set omit $omit[expr {![info exists DEF($name)]}]
+    incr nomit
+    continue
+  }
+  if {[regexp {^ELSE} $line]} {
+    if {!$nomit} {
+      puts stderr "$infile:$ln: ELSE without a prior IFDEF"
+      exit 1
+    }
+    set omit [string range $omit 0 end-1][expr {![string index $omit end]}]
+    continue
+  }
+  if {[regexp {^ENDIF} $line]} {
+    if {!$nomit} {
+      puts stderr "$infile:$ln: ENDIF without a prior IFDEF"
+      exit 1
+    }
+    incr nomit -1
+    set omit [string range $omit 0 [expr {$nomit-1}]]
+    continue
+  }
+  if {$nomit>0 && [string match *1* $omit]} {
+    # noop
+  } elseif {$instr} {
     set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line]
     puts "\"$x\\n\""
   } else {
     puts $line
   }
 }
+if {$nomit} {
+  puts stderr "$infile:$ln: One or more unterminated IFDEFs"
+  exit 1
+}
diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl
index bcb1a54bb8..56d6b5095c 100755
--- a/tool/mkctimec.tcl
+++ b/tool/mkctimec.tcl
@@ -4,8 +4,15 @@
 #
 #   const char **azCompileOpt[]
 #
-# definition used in src/ctime.c, run this script from
-# the checkout root. It generates src/ctime.c .
+# definition used in ctime.c, run this script from
+# the checkout root. It generates ctime.c .
+#
+# Results are normally written into ctime.c.  But if an argument is
+# provided, results are written there instead.  Examples:
+#
+#    tclsh tool/mkctimec.tcl                ;# <-- ctime.c
+#
+#    tclsh tool/mkctimec.tcl /dev/tty       ;# <-- results to the terminal
 #
 
 
@@ -98,8 +105,10 @@ set boolean_defnnz_options {
 set boolean_defnil_options {
   SQLITE_32BIT_ROWID
   SQLITE_4_BYTE_ALIGNED_MALLOC
+  SQLITE_ALLOW_ROWID_IN_VIEW
   SQLITE_ALLOW_URI_AUTHORITY
   SQLITE_BUG_COMPATIBLE_20160819
+  SQLITE_BUG_COMPATIBLE_20250510
   SQLITE_CASE_SENSITIVE_LIKE
   SQLITE_CHECK_PAGES
   SQLITE_COVERAGE_TEST
@@ -125,6 +134,7 @@ set boolean_defnil_options {
   SQLITE_ENABLE_ATOMIC_WRITE
   SQLITE_ENABLE_BATCH_ATOMIC_WRITE
   SQLITE_ENABLE_BYTECODE_VTAB
+  SQLITE_ENABLE_CARRAY
   SQLITE_ENABLE_COLUMN_METADATA
   SQLITE_ENABLE_COLUMN_USED_MASK
   SQLITE_ENABLE_COSTMULT
@@ -151,13 +161,16 @@ set boolean_defnil_options {
   SQLITE_ENABLE_MULTIPLEX
   SQLITE_ENABLE_NORMALIZE
   SQLITE_ENABLE_NULL_TRIM
+  SQLITE_ENABLE_ORDERED_SET_AGGREGATES
   SQLITE_ENABLE_OFFSET_SQL_FUNC
   SQLITE_ENABLE_OVERSIZE_CELL_CHECK
+  SQLITE_ENABLE_PERCENTILE
   SQLITE_ENABLE_PREUPDATE_HOOK
   SQLITE_ENABLE_QPSG
   SQLITE_ENABLE_RBU
   SQLITE_ENABLE_RTREE
   SQLITE_ENABLE_SESSION
+  SQLITE_ENABLE_SETLK_TIMEOUT
   SQLITE_ENABLE_SNAPSHOT
   SQLITE_ENABLE_SORTER_REFERENCES
   SQLITE_ENABLE_SQLLOG
@@ -180,6 +193,7 @@ set boolean_defnil_options {
   SQLITE_IGNORE_FLOCK_LOCK_ERRORS
   SQLITE_INLINE_MEMCPY
   SQLITE_INT64_TYPE
+  SQLITE_LEGACY_JSON_VALID
   SQLITE_LIKE_DOESNT_MATCH_BLOBS
   SQLITE_LOCK_TRACE
   SQLITE_LOG_CACHE_SPILL
@@ -238,6 +252,7 @@ set boolean_defnil_options {
   SQLITE_OMIT_REINDEX
   SQLITE_OMIT_SCHEMA_PRAGMAS
   SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS
+  SQLITE_OMIT_SEH
   SQLITE_OMIT_SHARED_CACHE
   SQLITE_OMIT_SHUTDOWN_DIRECTORIES
   SQLITE_OMIT_SUBQUERY
@@ -268,11 +283,11 @@ set boolean_defnil_options {
   SQLITE_UNTESTABLE
   SQLITE_USE_ALLOCA
   SQLITE_USE_FCNTL_TRACE
-  SQLITE_USER_AUTHENTICATION
   SQLITE_USE_URI
   SQLITE_VDBE_COVERAGE
   SQLITE_WIN32_MALLOC
   SQLITE_ZERO_MALLOC
+  SQLITE_HAS_CODEC
 }
 
 # All compile time options for which the assigned value is other than boolean
@@ -307,7 +322,9 @@ set value_options {
   SQLITE_ENABLE_8_3_NAMES
   SQLITE_ENABLE_CEROD
   SQLITE_ENABLE_LOCKING_STYLE
+  SQLITE_EXTRA_AUTOEXT
   SQLITE_EXTRA_INIT
+  SQLITE_EXTRA_INIT_MUTEXED
   SQLITE_EXTRA_SHUTDOWN
   SQLITE_FTS3_MAX_EXPR_DEPTH
   SQLITE_INTEGRITY_CHECK_ERROR_MAX
@@ -426,12 +443,18 @@ foreach v $value2_options {
 }]
 }
 
-set ctime_c "src/ctime.c"
+if {$argc>0} {
+  set destfile [lindex $argv 0]
+} else {
+  set destfile ctime.c
+  puts "Overwriting $destfile..."
+}
 
-if {[catch {set cfd [open $ctime_c w]}]!=0} {
-  puts stderr "File '$ctime_c' unwritable."
+if {[catch {set cfd [open $destfile w]}]!=0} {
+  puts stderr "File '$destfile' unwritable."
   exit 1;
 }
+fconfigure $cfd -translation binary
 
 puts $cfd $::headWarning;
 puts $cfd $::headCode;
diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c
index 44e1d9c27c..188c0a29ac 100644
--- a/tool/mkkeywordhash.c
+++ b/tool/mkkeywordhash.c
@@ -164,6 +164,11 @@ struct Keyword {
 #else
 #  define RETURNING  0x00400000
 #endif
+#ifndef SQLITE_ENABLE_ORDERED_SET_AGGREGATES
+#  define ORDERSET   0
+#else
+#  define ORDERSET   0x00800000
+#endif
 
 
 /*
@@ -316,6 +321,7 @@ static Keyword aKeywordTable[] = {
   { "WHERE",            "TK_WHERE",        ALWAYS,           10     },
   { "WINDOW",           "TK_WINDOW",       WINDOWFUNC,       3      },
   { "WITH",             "TK_WITH",         CTE,              4      },
+  { "WITHIN",           "TK_WITHIN",       ORDERSET,         1      },
   { "WITHOUT",          "TK_WITHOUT",      ALWAYS,           1      },
 };
 
@@ -665,39 +671,38 @@ int main(int argc, char **argv){
   printf("static int keywordCode(const char *z, int n, int *pType){\n");
   printf("  int i, j;\n");
   printf("  const char *zKW;\n");
-  printf("  if( n>=2 ){\n");
-  printf("    i = ((charMap(z[0])*%d) %c", HASH_C0, HASH_CC);
+  printf("  assert( n>=2 );\n");
+  printf("  i = ((charMap(z[0])*%d) %c", HASH_C0, HASH_CC);
   printf(" (charMap(z[n-1])*%d) %c", HASH_C1, HASH_CC);
   printf(" n*%d) %% %d;\n", HASH_C2, bestSize);
-  printf("    for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){\n");
-  printf("      if( aKWLen[i]!=n ) continue;\n");
-  printf("      zKW = &zKWText[aKWOffset[i]];\n");
+  printf("  for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){\n");
+  printf("    if( aKWLen[i]!=n ) continue;\n");
+  printf("    zKW = &zKWText[aKWOffset[i]];\n");
   printf("#ifdef SQLITE_ASCII\n");
-  printf("      if( (z[0]&~0x20)!=zKW[0] ) continue;\n");
-  printf("      if( (z[1]&~0x20)!=zKW[1] ) continue;\n");
-  printf("      j = 2;\n");
-  printf("      while( j<n && (z[j]&~0x20)==zKW[j] ){ j++; }\n");
+  printf("    if( (z[0]&~0x20)!=zKW[0] ) continue;\n");
+  printf("    if( (z[1]&~0x20)!=zKW[1] ) continue;\n");
+  printf("    j = 2;\n");
+  printf("    while( j<n && (z[j]&~0x20)==zKW[j] ){ j++; }\n");
   printf("#endif\n");
   printf("#ifdef SQLITE_EBCDIC\n");
-  printf("      if( toupper(z[0])!=zKW[0] ) continue;\n");
-  printf("      if( toupper(z[1])!=zKW[1] ) continue;\n");
-  printf("      j = 2;\n");
-  printf("      while( j<n && toupper(z[j])==zKW[j] ){ j++; }\n");
+  printf("    if( toupper(z[0])!=zKW[0] ) continue;\n");
+  printf("    if( toupper(z[1])!=zKW[1] ) continue;\n");
+  printf("    j = 2;\n");
+  printf("    while( j<n && toupper(z[j])==zKW[j] ){ j++; }\n");
   printf("#endif\n");
-  printf("      if( j<n ) continue;\n");
+  printf("    if( j<n ) continue;\n");
   for(i=0; i<nKeyword; i++){
-    printf("      testcase( i==%d ); /* %s */\n",
+    printf("    testcase( i==%d ); /* %s */\n",
            i+1, aKeywordTable[i].zOrigName);
   }
-  printf("      *pType = aKWCode[i];\n");
-  printf("      break;\n");
-  printf("    }\n");
+  printf("    *pType = aKWCode[i];\n");
+  printf("    break;\n");
   printf("  }\n");
   printf("  return n;\n");
   printf("}\n");
   printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n");
   printf("  int id = TK_ID;\n");
-  printf("  keywordCode((char*)z, n, &id);\n");
+  printf("  if( n>=2 ) keywordCode((char*)z, n, &id);\n");
   printf("  return id;\n");
   printf("}\n");
   printf("#define SQLITE_N_KEYWORD %d\n", nKeyword);
diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl
index 6cb145db85..7926952554 100644
--- a/tool/mkmsvcmin.tcl
+++ b/tool/mkmsvcmin.tcl
@@ -23,7 +23,7 @@ if {$argc==0} {
 
 proc readFile { fileName } {
   set file_id [open $fileName RDONLY]
-  fconfigure $file_id -encoding binary -translation binary
+  fconfigure $file_id -translation binary
   set result [read $file_id]
   close $file_id
   return $result
@@ -31,7 +31,7 @@ proc readFile { fileName } {
 
 proc writeFile { fileName data } {
   set file_id [open $fileName {WRONLY CREAT TRUNC}]
-  fconfigure $file_id -encoding binary -translation binary
+  fconfigure $file_id -translation binary
   puts -nonewline $file_id $data
   close $file_id
   return ""
diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl
index 6fb3b75940..18fe1a2658 100644
--- a/tool/mkopcodeh.tcl
+++ b/tool/mkopcodeh.tcl
@@ -81,6 +81,7 @@ while {![eof $in]} {
     set op($name) -1
     set group($name) 0
     set jump($name) 0
+    set jump0($name) 0
     set in1($name) 0
     set in2($name) 0
     set in3($name) 0
@@ -109,6 +110,7 @@ while {![eof $in]} {
          out2  {set out2($name) 1}
          out3  {set out3($name) 1}
          ncycle {set ncycle($name) 1}
+         jump0 {set jump($name) 1; set jump0($name) 1;}
        }
     }
     if {$group($name)} {
@@ -137,6 +139,7 @@ puts "/* Automatically generated.  Do not edit */"
 puts "/* See the tool/mkopcodeh.tcl script for details */"
 foreach name {OP_Noop OP_Explain OP_Abortable} {
   set jump($name) 0
+  set jump0($name) 0
   set in1($name) 0
   set in2($name) 0
   set in3($name) 0
@@ -256,7 +259,9 @@ for {set i 0} {$i<=$max} {incr i} {
   set name $def($i)
   puts -nonewline [format {#define %-16s %3d} $name $i]
   set com {}
-  if {[info exists jump($name)] && $jump($name)} {
+  if {[info exists jump0($name)] && $jump0($name)} {
+    lappend com "jump0"
+  } elseif {[info exists jump($name)] && $jump($name)} {
     lappend com "jump"
   }
   if {[info exists sameas($i)]} {
@@ -289,6 +294,7 @@ for {set i 0} {$i<=$max} {incr i} {
     if {$out2($name)}  {incr x 16}
     if {$out3($name)}  {incr x 32}
     if {$ncycle($name)}  {incr x 64}
+    if {$jump0($name)}   {incr x 128}
   }
   set bv($i) $x
 }
@@ -304,6 +310,7 @@ puts "#define OPFLG_IN3         0x08  /* in3:   P3 is an input */"
 puts "#define OPFLG_OUT2        0x10  /* out2:  P2 is an output */"
 puts "#define OPFLG_OUT3        0x20  /* out3:  P3 is an output */"
 puts "#define OPFLG_NCYCLE      0x40  /* ncycle:Cycles count against P1 */"
+puts "#define OPFLG_JUMP0       0x80  /* jump0:  P2 might be zero */"
 puts "#define OPFLG_INITIALIZER \173\\"
 for {set i 0} {$i<=$max} {incr i} {
   if {$i%8==0} {
diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl
index dcf573bab5..b55e5eba0f 100644
--- a/tool/mkpragmatab.tcl
+++ b/tool/mkpragmatab.tcl
@@ -4,11 +4,19 @@
 #
 # To add new pragmas, first add the name and other relevant attributes
 # of the pragma to the "pragma_def" object below.  Then run this script
-# to generate the ../src/pragma.h header file that contains macros and
+# to generate the pragma.h header file that contains macros and
 # the lookup table needed for pragma name lookup in the pragma.c module.
 # Then add the extra "case PragTyp_XXXXX:" and subsequent code for the
 # new pragma in ../src/pragma.c.
 #
+# The results are normally written into the pragma.h file.  However,
+# if an alternative output file name is provided as an argument, then
+# results are written into the alternative.  For example:
+#
+#     tclsh tool/mkpragmatab.tcl            ;# <--- Results to pragma.h
+#
+#     tclsh tool/mkpragmatab.tcl /dev/tty   ;# <-- results to terminal
+#
 
 # Flag meanings:
 set flagMeaning(NeedSchema) {Force schema load before running}
@@ -432,8 +440,12 @@ set pragma_def {
 
 # Open the output file
 #
-set destfile "[file dir [file dir [file normal $argv0]]]/src/pragma.h"
-puts "Overwriting $destfile with new pragma table..."
+if {$argc>0} {
+  set destfile [lindex $argv 0]
+} else {
+  set destfile "pragma.h"
+  puts "Overwriting $destfile with new pragma table..."
+}
 set fd [open $destfile wb]
 puts $fd {/* DO NOT EDIT!
 ** This file is automatically generated by the script at
@@ -546,10 +558,13 @@ foreach f [lsort [array names allflags]] {
   set fv [expr {$fv*2}]
 }
 
-# Sort the column lists so that longer column lists occur first
+# Sort the column lists so that longer column lists occur first.
+# In the event of a tie, sort column lists lexicographically.
 #
 proc colscmp {a b} {
-  return [expr {[llength $b] - [llength $a]}]
+  set rc [expr {[llength $b] - [llength $a]}]
+  if {$rc} {return $rc}
+  return [string compare $a $b]
 }
 set cols_list [lsort -command colscmp $cols_list]
 
diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl
index 4f16a772e3..2f7a6ea256 100644
--- a/tool/mkshellc.tcl
+++ b/tool/mkshellc.tcl
@@ -12,6 +12,9 @@
 set topdir [file dir [file dir [file normal $argv0]]]
 set out stdout
 fconfigure stdout -translation binary
+if {[lindex $argv 0]!=""} {
+  set out [open [lindex $argv 0] wb]
+}
 puts $out {/* DO NOT EDIT!
 ** This file is automatically generated by the script in the canonical
 ** SQLite source tree at tool/mkshellc.tcl.  That script combines source
@@ -34,7 +37,8 @@ set in [open $topdir/src/shell.c.in]
 fconfigure $in -translation binary
 proc omit_redundant_typedefs {line} {
   global typedef_seen
-  if {[regexp {^typedef .*\y([a-zA-Z0-9_]+);} $line all typename]} {
+  if {[regexp {^typedef .* ([a-zA-Z0-9_]+);} $line all typename]} {
+    # --------------------\y jimtcl does not support \y
     if {[info exists typedef_seen($typename)]} {
       return "/* [string map {/* // */ //} $line] */"
     }
@@ -58,7 +62,7 @@ while {1} {
       if {[regexp {^# *include "sqlite} $lx]} {
         set lx "/* $lx */"
       }
-      if {[regexp {^# *include "test_windirent.h"} $lx]} {
+      if {[regexp {^# *include "windirent.h"} $lx]} {
         set lx "/* $lx */"
       }
       set lx [string map [list __declspec(dllexport) {}] $lx]
diff --git a/tool/mkspeedsql.tcl b/tool/mkspeedsql.tcl
deleted file mode 100644
index 04bafc04c1..0000000000
--- a/tool/mkspeedsql.tcl
+++ /dev/null
@@ -1,237 +0,0 @@
-# 2008 October 9
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file generates SQL text used for performance testing.
-#
-# $Id: mkspeedsql.tcl,v 1.1 2008/10/09 17:57:34 drh Exp $
-#
-
-# Set a uniform random seed
-expr srand(0)
-
-# The number_name procedure below converts its argment (an integer)
-# into a string which is the English-language name for that number.
-#
-# Example:
-#
-#     puts [number_name 123]   ->  "one hundred twenty three"
-#
-set ones {zero one two three four five six seven eight nine
-          ten eleven twelve thirteen fourteen fifteen sixteen seventeen
-          eighteen nineteen}
-set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety}
-proc number_name {n} {
-  if {$n>=1000} {
-    set txt "[number_name [expr {$n/1000}]] thousand"
-    set n [expr {$n%1000}]
-  } else {
-    set txt {}
-  }
-  if {$n>=100} {
-    append txt " [lindex $::ones [expr {$n/100}]] hundred"
-    set n [expr {$n%100}]
-  }
-  if {$n>=20} {
-    append txt " [lindex $::tens [expr {$n/10}]]"
-    set n [expr {$n%10}]
-  }
-  if {$n>0} {
-    append txt " [lindex $::ones $n]"
-  }
-  set txt [string trim $txt]
-  if {$txt==""} {set txt zero}
-  return $txt
-}
-
-# Create a database schema.
-#
-puts {
-  PRAGMA page_size=1024;
-  PRAGMA cache_size=8192;
-  PRAGMA locking_mode=EXCLUSIVE;
-  CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT);
-  CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT);
-  CREATE INDEX i2a ON t2(a);
-  CREATE INDEX i2b ON t2(b);
-  SELECT name FROM sqlite_master ORDER BY 1;
-}
-
-
-# 50000 INSERTs on an unindexed table
-#
-set t1c_list {}
-puts {BEGIN;}
-for {set i 1} {$i<=50000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  set x [number_name $r]
-  lappend t1c_list $x
-  puts "INSERT INTO t1 VALUES($i,$r,'$x');"
-}
-puts {COMMIT;}
-
-# 50000 INSERTs on an indexed table
-#
-puts {BEGIN;}
-for {set i 1} {$i<=50000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');"
-}
-puts {COMMIT;}
-
-
-# 50 SELECTs on an integer comparison.  There is no index so
-# a full table scan is required.
-#
-for {set i 0} {$i<50} {incr i} {
-  set lwr [expr {$i*100}]
-  set upr [expr {($i+10)*100}]
-  puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;"
-}
-
-# 50 SELECTs on an LIKE comparison.  There is no index so a full
-# table scan is required.
-#
-for {set i 0} {$i<50} {incr i} {
-  puts "SELECT count(*), avg(b) FROM t1 WHERE c LIKE '%[number_name $i]%';"
-}
-
-# Create indices
-#
-puts {BEGIN;}
-puts {
-  CREATE INDEX i1a ON t1(a);
-  CREATE INDEX i1b ON t1(b);
-  CREATE INDEX i1c ON t1(c);
-}
-puts {COMMIT;}
-
-# 5000 SELECTs on an integer comparison where the integer is
-# indexed.
-#
-set sql {}
-for {set i 0} {$i<5000} {incr i} {
-  set lwr [expr {$i*100}]
-  set upr [expr {($i+10)*100}]
-  puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;"
-}
-
-# 100000 random SELECTs against rowid.
-#
-for {set i 1} {$i<=100000} {incr i} {
-  set id [expr {int(rand()*50000)+1}]
-  puts "SELECT c FROM t1 WHERE rowid=$id;"
-}
-
-# 100000 random SELECTs against a unique indexed column.
-#
-for {set i 1} {$i<=100000} {incr i} {
-  set id [expr {int(rand()*50000)+1}]
-  puts "SELECT c FROM t1 WHERE a=$id;"
-}
-
-# 50000 random SELECTs against an indexed column text column
-#
-set nt1c [llength $t1c_list]
-for {set i 0} {$i<50000} {incr i} {
-  set r [expr {int(rand()*$nt1c)}]
-  set c [lindex $t1c_list $i]
-  puts "SELECT c FROM t1 WHERE c='$c';"
-}
-
-
-# Vacuum
-puts {VACUUM;}
-
-# 5000 updates of ranges where the field being compared is indexed.
-#
-puts {BEGIN;}
-for {set i 0} {$i<5000} {incr i} {
-  set lwr [expr {$i*2}]
-  set upr [expr {($i+1)*2}]
-  puts "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;"
-}
-puts {COMMIT;}
-
-# 50000 single-row updates.  An index is used to find the row quickly.
-#
-puts {BEGIN;}
-for {set i 0} {$i<50000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts "UPDATE t1 SET b=$r WHERE a=$i;"
-}
-puts {COMMIT;}
-
-# 1 big text update that touches every row in the table.
-#
-puts {
-  UPDATE t1 SET c=a;
-}
-
-# Many individual text updates.  Each row in the table is
-# touched through an index.
-#
-puts {BEGIN;}
-for {set i 1} {$i<=50000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts "UPDATE t1 SET c='[number_name $r]' WHERE a=$i;"
-}
-puts {COMMIT;}
-
-# Delete all content in a table.
-#
-puts {DELETE FROM t1;}
-
-# Copy one table into another
-#
-puts {INSERT INTO t1 SELECT * FROM t2;}
-
-# Delete all content in a table, one row at a time.
-#
-puts {DELETE FROM t1 WHERE 1;}
-
-# Refill the table yet again
-#
-puts {INSERT INTO t1 SELECT * FROM t2;}
-
-# Drop the table and recreate it without its indices.
-#
-puts {BEGIN;}
-puts {
-   DROP TABLE t1;
-   CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT);
-}
-puts {COMMIT;}
-
-# Refill the table yet again.  This copy should be faster because
-# there are no indices to deal with.
-#
-puts {INSERT INTO t1 SELECT * FROM t2;}
-
-# Select 20000 rows from the table at random.
-#
-puts {
-  SELECT rowid FROM t1 ORDER BY random() LIMIT 20000;
-}
-
-# Delete 20000 random rows from the table.
-#
-puts {
-  DELETE FROM t1 WHERE rowid IN
-    (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000);
-}
-puts {SELECT count(*) FROM t1;}
-    
-# Delete 20000 more rows at random from the table.
-#
-puts {
-  DELETE FROM t1 WHERE rowid IN
-    (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000);
-}
-puts {SELECT count(*) FROM t1;}
diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl
index 8452072564..1148b1c0d5 100644
--- a/tool/mksqlite3c-noext.tcl
+++ b/tool/mksqlite3c-noext.tcl
@@ -57,7 +57,7 @@ close $in
 #
 set out [open sqlite3.c w]
 # Force the output to use unix line endings, even on Windows.
-fconfigure $out -translation lf
+fconfigure $out -translation binary
 set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
 puts $out [subst \
 {/******************************************************************************
diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl
index 4d9fc59f26..93af239944 100644
--- a/tool/mksqlite3c.tcl
+++ b/tool/mksqlite3c.tcl
@@ -17,7 +17,7 @@
 # After the "tsrc" directory has been created and populated, run
 # this script:
 #
-#      tclsh mksqlite3c.tcl
+#      tclsh mksqlite3c.tcl [flags] [extra source files]
 #
 # The amalgamated SQLite code will be written into sqlite3.c
 #
@@ -42,6 +42,7 @@ set linemacros 0
 set useapicall 0
 set enable_recover 0
 set srcdir tsrc
+set extrasrc [list]
 
 for {set i 0} {$i<[llength $argv]} {incr i} {
   set x [lindex $argv $i]
@@ -63,11 +64,13 @@ for {set i 0} {$i<[llength $argv]} {incr i} {
   } elseif {[regexp {^-?-((help)|\?)$} $x]} {
     puts $help
     exit 0
-  } else {
+  } elseif {[regexp {^-?-} $x]} {
     error "unknown command-line option: $x"
+  } else {
+    lappend extrasrc $x
   }
 }
-set in [open $srcdir/sqlite3.h]
+set in [open $srcdir/sqlite3.h rb]
 set cnt 0
 set VERSION ?????
 while {![eof $in]} {
@@ -83,9 +86,9 @@ close $in
 #
 set fname sqlite3.c
 if {$enable_recover} { set fname sqlite3r.c }
-set out [open $fname w]
+set out [open $fname wb]
 # Force the output to use unix line endings, even on Windows.
-fconfigure $out -translation lf
+fconfigure $out -translation binary
 set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
 puts $out [subst \
 {/******************************************************************************
@@ -106,7 +109,36 @@ puts $out [subst \
 ** if you want a wrapper to interface SQLite with your choice of programming
 ** language. The code for the "sqlite3" command-line shell is also in a
 ** separate file. This file contains only code for the core SQLite library.
-*/
+**}]
+set srcroot [file dirname [file dirname [info script]]]
+if {$tcl_platform(platform) eq "windows"} {
+  set vsrcprog src-verify.exe
+} else {
+  set vsrcprog ./src-verify
+}
+if {[file executable $vsrcprog] && [file readable $srcroot/manifest]} {
+  set tmpfile tmp-[clock millisec]-[expr {int(rand()*100000000000)}].txt
+  exec $vsrcprog -x $srcroot > $tmpfile
+  set fd [open $tmpfile rb]
+  set res [string trim [split [read $fd] \n]]
+  close $fd
+  file delete -force $tmpfile
+  puts $out "** The content in this amalgamation comes from Fossil check-in"
+  puts -nonewline $out "** [string range [lindex $res 0] 0 35]"
+  if {[llength $res]==1} {
+    puts $out "."
+  } else {
+    puts $out " with changes in files:\n**"
+    foreach f [lrange $res 1 end] {
+       puts $out "**    [string trim $f]"
+    }
+  }
+} else {
+  puts $out "** The origin of the sources used to build this amalgamation"
+  puts $out "** is unknown."
+}
+puts $out [subst {*/
+#ifndef SQLITE_AMALGAMATION
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1}]
 if {$addstatic} {
@@ -122,7 +154,7 @@ if {$addstatic} {
 # 
 # then set the SQLITE_UDL_CAPABLE_PARSER flag in the amalgamation.
 #
-set in [open $srcdir/parse.c]
+set in [open $srcdir/parse.c rb]
 if {[regexp {ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT} [read $in]]} {
   puts $out "#define SQLITE_UDL_CAPABLE_PARSER 1"
 }
@@ -133,7 +165,6 @@ close $in
 # text of the file in-line.  The file only needs to be included once.
 #
 foreach hdr {
-   crypto.h
    sqlcipher.h
    btree.h
    btreeInt.h
@@ -219,7 +250,7 @@ proc copy_file {filename} {
   set tail [file tail $filename]
   section_comment "Begin file $tail"
   if {$linemacros} {puts $out "#line 1 \"$filename\""}
-  set in [open $filename r]
+  set in [open $filename rb]
   set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+(sqlite3[_a-zA-Z0-9]+)(\[|;| =)}
   set declpattern {([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3[_a-zA-Z0-9]+)(\(.*)}
   if {[file extension $filename]==".h"} {
@@ -328,6 +359,21 @@ proc copy_file {filename} {
   section_comment "End of $tail"
 }
 
+# Read the source file named $filename and write it into the
+# sqlite3.c output file. The only transformation is the trimming
+# of EOL whitespace.
+#
+proc copy_file_verbatim {filename} {
+  global out
+  set in [open $filename rb]
+  set tail [file tail $filename]
+  section_comment "Begin EXTRA_SRC file $tail"
+  while {![eof $in]} {
+    set line [string trimright [gets $in]]
+    puts $out $line
+  }
+  section_comment "End of EXTRA_SRC $tail"
+}
 
 # Process the source files.  Process files containing commonly
 # used subroutines first in order to help the compiler find
@@ -389,10 +435,7 @@ set flist {
    vdbevtab.c
    memjournal.c
 
-   crypto.c
-   crypto_impl.c
-   crypto_libtomcrypt.c
-   crypto_nss.c
+   sqlcipher.c
    crypto_openssl.c
    crypto_cc.c
 
@@ -454,22 +497,27 @@ set flist {
    sqlite3rbu.c
    dbstat.c
    dbpage.c
+   carray.c
    sqlite3session.c
    fts5.c
    stmt.c
-} 
+}
 if {$enable_recover} {
   lappend flist sqlite3recover.c dbdata.c
 }
 foreach file $flist {
   copy_file $srcdir/$file
 }
+foreach file $extrasrc {
+  copy_file_verbatim $file
+}
 
 puts $out \
 "/* Return the source-id for this library */
 SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }"
 
 puts $out \
-"/************************** End of sqlite3.c ******************************/"
+"#endif /* SQLITE_AMALGAMATION */
+/************************** End of sqlite3.c ******************************/"
 
 close $out
diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl
index bd579c28b0..c661cd7a33 100644
--- a/tool/mksqlite3h.tcl
+++ b/tool/mksqlite3h.tcl
@@ -24,18 +24,36 @@
 #   6) Adds the SQLITE_CALLBACK calling convention macro in front of all
 #      callback declarations.
 #
-# This script outputs to stdout.
+# This script outputs to stdout unless the -o FILENAME option is used.
 #
 # Example usage:
 #
-#   tclsh mksqlite3h.tcl ../sqlite >sqlite3.h
+#   tclsh mksqlite3h.tcl ../sqlite [OPTIONS]
+#                        ^^^^^^^^^
+#                        Root of source tree
+#
+# Where options are:
+#
+#   --enable-recover          Include the sqlite3recover extension
+#   -o FILENAME               Write results to FILENAME instead of stdout
+#   --useapicall              SQLITE_APICALL instead of SQLITE_CDECL
 #
 
+# Default output stream
+set out stdout
 
 # Get the source tree root directory from the command-line
 #
 set TOP [lindex $argv 0]
 
+# If the -o FILENAME option is present, use FILENAME for output.
+#
+set x [lsearch $argv -o]
+if {$x>0} {
+  incr x
+  set out [open [lindex $argv $x] wb]
+}
+
 # Enable use of SQLITE_APICALL macros at the right points?
 #
 set useapicall 0
@@ -44,6 +62,7 @@ set useapicall 0
 #
 set enable_recover 0
 
+# Process command-line arguments
 if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} {
   set useapicall 1
 }
@@ -53,18 +72,55 @@ if {[lsearch -regexp [lrange $argv 1 end] {^-+enable-recover}] != -1} {
 
 # Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file.
 #
-set in [open $TOP/VERSION]
+set in [open [file normalize $TOP/VERSION] rb]
 set zVersion [string trim [read $in]]
 close $in
 set nVersion [eval format "%d%03d%03d" [split $zVersion .]]
 
 # Get the source-id
 #
+proc file-content {fn} {
+  set fd [open $fn rb]
+  set rv [string trim [read $fd]]
+  close $fd
+  return $rv
+}
+
 set PWD [pwd]
 cd $TOP
-set zSourceId [exec $PWD/mksourceid manifest]
+set tmpfile $PWD/tmp-[clock millisec]-[expr {int(rand()*100000000000)}].txt
+set mksourceid $PWD/mksourceid
+if {![file exists $mksourceid] && [file exists ${mksourceid}.exe]} {
+  # Workaround for Windows-based Unix-like environments
+  # https://sqlite.org/forum/forumpost/41ba710dd9943453
+  set mksourceid ${mksourceid}.exe
+}
+exec $mksourceid manifest > $tmpfile
+set zSourceId [file-content $tmpfile]
+file delete -force $tmpfile
 cd $PWD
 
+# Collect SQLITE_SCM_BRANCH, SQLITE_SCM_TAGS, and SQLITE_SCM_DATETIME
+set zVerTime [lindex [lindex [split [file-content $TOP/manifest] "\n"] 1] 1]Z; # D-card
+if {![file exists $TOP/manifest.tags]} {
+  puts stderr "WARNING: building sqlite3.h without manifest.tags, which is generated by the SCM."
+  puts stderr "This means that we cannot record the tag/branch info. We will continue with "
+  puts stderr "a placeholder value. To remedy this, run the following command from a "
+  puts stderr "check-out:\n"
+  puts stderr "  fossil set manifest urt\n"
+  set zSourceId [string range $zSourceId 0 end-13]-experimental; # Keep SHA3 hash length
+  set zBranch "unknown"
+  set zTags "unknown"
+} else {
+  # Read the list of branch/tags from manifest.tags
+  set content [file-content $TOP/manifest.tags];
+  set zTags {}
+  foreach {x tag} [lassign $content - zBranch] {
+    if {$tag eq $zBranch} continue
+    lappend zTags $tag
+  }
+}
+
 # Set up patterns for recognizing API declarations.
 #
 set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)}
@@ -83,7 +139,7 @@ set declpattern5 \
     {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3rebaser_[_a-zA-Z0-9]+)(\(.*)$}
 
 # Force the output to use unix line endings, even on Windows.
-fconfigure stdout -translation lf
+fconfigure stdout -translation binary
 
 set filelist [subst {
   $TOP/src/sqlite.h.in
@@ -111,9 +167,9 @@ set cdecllist {
 # Process the source files.
 #
 foreach file $filelist {
-  set in [open $file]
+  set in [open $file rb]
   if {![regexp {sqlite\.h\.in} $file]} {
-    puts "/******** Begin file [file tail $file] *********/"
+    puts $out "/******** Begin file [file tail $file] *********/"
   }
   while {![eof $in]} {
 
@@ -127,6 +183,9 @@ foreach file $filelist {
     regsub -- --VERS--           $line $zVersion line
     regsub -- --VERSION-NUMBER-- $line $nVersion line
     regsub -- --SOURCE-ID--      $line "$zSourceId" line
+    regsub -- --SCM-BRANCH--     $line "$zBranch" line
+    regsub -- --SCM-TAGS--       $line "$zTags" line
+    regsub -- --SCM-DATETIME--   $line "$zVerTime" line
 
     if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} {
       set line "SQLITE_API $line"
@@ -156,10 +215,11 @@ foreach file $filelist {
           "(SQLITE_SYSAPI *sqlite3_syscall_ptr)"] $line]
       regsub {\(\*} $line {(SQLITE_CALLBACK *} line
     }
-    puts $line
+    puts $out $line
   }
   close $in
   if {![regexp {sqlite\.h\.in} $file]} {
-    puts "/******** End of [file tail $file] *********/"
+    puts $out "/******** End of [file tail $file] *********/"
   }
 }
+puts $out "#endif /* SQLITE3_H */"
diff --git a/tool/mksqlite3internalh.tcl b/tool/mksqlite3internalh.tcl
index 8db593fe75..e1a42ee776 100644
--- a/tool/mksqlite3internalh.tcl
+++ b/tool/mksqlite3internalh.tcl
@@ -92,7 +92,7 @@ proc section_comment {text} {
 
 # Read the source file named $filename and write it into the
 # sqlite3.c output file.  If any #include statements are seen,
-# process them approprately.
+# process them appropriately.
 #
 proc copy_file {filename} {
   global seen_hdr available_hdr out
diff --git a/tool/mksrczip.tcl b/tool/mksrczip.tcl
new file mode 100644
index 0000000000..4431c3d666
--- /dev/null
+++ b/tool/mksrczip.tcl
@@ -0,0 +1,14 @@
+#!/usr/bin/tclsh
+#
+# Build a ZIP archive for the complete, unedited source code that
+# corresponds to the current check-out.
+#
+set VERSION-file [file dirname [file dirname [file normalize $argv0]]]/VERSION
+set fd [open ${VERSION-file} rb]
+set vers [read $fd]
+close $fd
+scan $vers %d.%d.%d major minor patch
+set numvers [format {3%02d%02d00} $minor $patch]
+set cmd "fossil zip current sqlite-src-$numvers.zip --name sqlite-src-$numvers"
+puts $cmd
+exec {*}$cmd
diff --git a/tool/mktoolzip.tcl b/tool/mktoolzip.tcl
new file mode 100644
index 0000000000..041bf28cdd
--- /dev/null
+++ b/tool/mktoolzip.tcl
@@ -0,0 +1,105 @@
+#!/usr/bin/tclsh
+#
+# Run this script in order to generate a ZIP archive containing various
+# command-line tools.
+#
+# The makefile that invokes this script must first build the following
+# binaries:
+#
+#     testfixture             -- used to run this script
+#     sqlite3                 -- the SQLite CLI
+#     sqldiff                 -- Program to diff two databases
+#     sqlite3_analyzer        -- Space analyzer
+#     sqlite3_rsync           -- Remote db sync
+#
+# On Windows, add:
+#
+#     sqlite3.def
+#     sqlite3.dll
+#
+# Add the --snapshot option to generate a snapshot ZIP archive instead of
+# a release ZIP archive.
+#
+set bSnapshot 0
+for {set i 0} {$i<[llength $argv]} {incr i} {
+  set a [lindex $argv $i]
+  if {$a eq "-snapshot" || $a eq "--snapshot"} {
+    set bSnapshot 1
+    continue
+  }
+  puts stderr "unknown argument: $a"
+  exit 1
+} 
+switch $tcl_platform(os) {
+  {Windows NT} {
+    set OS win
+    set EXE .exe
+  }
+  Linux {
+    set OS linux
+    set EXE {}
+  }
+  Darwin {
+    set OS osx
+    set EXE {}
+  }
+  default {
+    set OS unknown
+    set EXE {}
+  }
+}
+switch $tcl_platform(machine) {
+  arm64 {
+    set ARCH arm64
+  }
+  x86_64 {
+    set ARCH x64
+  }
+  amd64 -
+  intel {
+    if {$tcl_platform(pointerSize)==4} {
+      set ARCH x86
+    } else {
+      set ARCH x64
+    }
+  }
+  default {
+    set ARCH unk
+  }
+}
+if {$bSnapshot} {
+  set in [open [file join [file dirname [file dirname [info script]]] manifest]]
+  set manifest [read $in]
+  close $in
+  regexp {\nD (.{16})} $manifest all date
+  regsub -all {[-:T]} $date {} v2
+} else {
+  set in [open [file join [file dirname [file dirname [info script]]] VERSION]]
+  set vers [read $in]
+  close $in
+  scan $vers %d.%d.%d v1 v2 v3
+  set v2 [format 3%02d%02d00 $v2 $v3]
+}
+
+set name sqlite-tools-$OS-$ARCH-$v2.zip
+set filelist "sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE sqlite3_rsync$EXE"
+proc make_zip_archive {name filelist} {
+  file delete -force $name
+  puts "fossil test-filezip $name $filelist"
+  if {[catch {exec fossil test-filezip $name {*}$filelist}]} {
+    puts "^--- Unable.  Trying again as:"
+    puts "zip $name $filelist"
+    file delete -force $name
+    exec zip $name {*}$filelist
+  }
+  puts "$name: [file size $name] bytes"
+}
+make_zip_archive $name $filelist
+
+# On Windows, also try to construct a DLL
+#
+if {$OS eq "win" && [file exists sqlite3.dll] && [file exists sqlite3.def]} {
+  set name sqlite-dll-win-$ARCH-$v2.zip
+  set filelist [list sqlite3.def sqlite3.dll]
+  make_zip_archive $name $filelist
+}
diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl
index c874d538fb..0663213639 100644
--- a/tool/mkvsix.tcl
+++ b/tool/mkvsix.tcl
@@ -156,7 +156,7 @@ proc readFile { fileName } {
   #       may contain binary data.
   #
   set file_id [open $fileName RDONLY]
-  fconfigure $file_id -encoding binary -translation binary
+  fconfigure $file_id -translation binary
   set result [read $file_id]
   close $file_id
   return $result
@@ -168,7 +168,7 @@ proc writeFile { fileName data } {
   #       binary data.
   #
   set file_id [open $fileName {WRONLY CREAT TRUNC}]
-  fconfigure $file_id -encoding binary -translation binary
+  fconfigure $file_id -translation binary
   puts -nonewline $file_id $data
   close $file_id
   return ""
diff --git a/tool/mkwinarm64ec.tcl b/tool/mkwinarm64ec.tcl
new file mode 100644
index 0000000000..388dd47a83
--- /dev/null
+++ b/tool/mkwinarm64ec.tcl
@@ -0,0 +1,45 @@
+# Script to build ARM64-EC binaries on Windows. 
+#
+# Run from the top-level of a check-out, on a Windows11 ARM machine that
+# has Fossil installed, using a command like this:
+#
+#    tclsh tool/mmkwinarm64ec.tcl
+#
+# Output will be a file named:
+#
+#    sqlite-win-arm64ec-3MMPP00.zip
+#
+# Where MM is the minor version number and PP is that patch level.
+#
+puts "Running nmake..."
+flush stdout
+exec nmake /f Makefile.msc "PLATFORM=ARM64EC" "BCC=cl -nologo -arm64EC" "TCC=cl -nologo -arm64EC -I." "CFLAGS=-nologo -arm64EC -I." clean sqlite3.exe sqldiff.exe sqlite3_rsync.exe sqlite3.dll >@ stdout 2>@ stderr
+
+proc check-type {file} {
+  set in [open "| link /dump /headers $file" rb]
+  set txt [read $in]
+  close $in
+  if {![string match {*8664 machine (x64) (ARM64X)*} $txt]} {
+    puts "$file is not an ARM64-EC binary"
+    puts "OUTPUT:\n$txt"
+    flush stdout
+    exit 1
+  }
+  puts "Confirmed: $file is an ARM64-EC binary"
+}
+check-type sqlite3.exe
+check-type sqldiff.exe
+check-type sqlite3_rsync.exe
+check-type sqlite3.dll
+
+set in [open VERSION rb]
+set VERSION [read $in]
+close $in
+regexp {3.(\d+).(\d+)} $VERSION all minor patch
+set filename [format sqlite-win-arm64ec-3%02d%02d00.zip $minor $patch]
+
+puts "Constructing $filename..."
+file delete -force $filename
+exec fossil test-filezip $filename sqlite3.def sqlite3.dll sqlite3.exe sqldiff.exe sqlite3_rsync.exe >@ stdout 2>@ stderr
+puts "$filename: [file size $filename] bytes"
+exec fossil test-filezip -l $filename >@ stdout 2>@ stderr
diff --git a/tool/omittest.tcl b/tool/omittest.tcl
index 8862c685f8..0452a4c6f6 100644
--- a/tool/omittest.tcl
+++ b/tool/omittest.tcl
@@ -1,370 +1,220 @@
-# Documentation for this script. This may be output to stderr
+#!/usr/bin/tclsh
+#
+# Documentation for this script. This may be output to
 # if the script is invoked incorrectly.
+#
 set ::USAGE_MESSAGE {
 This Tcl script is used to test the various compile time options 
-available for omitting code (the SQLITE_OMIT_xxx options). It
-should be invoked as follows:
-
-    <script> ?test-symbol? ?-makefile PATH-TO-MAKEFILE? ?-skip_run?
-
-The default value for ::MAKEFILE is "../Makefile.linux.gcc".
-
-If -skip_run option is given then only the compile part is attempted.
-
-This script builds the testfixture program and runs the SQLite test suite
-once with each SQLITE_OMIT_ option defined and then once with all options
-defined together. Each run is performed in a seperate directory created
-as a sub-directory of the current directory by the script. The output
-of the build is saved in <sub-directory>/build.log. The output of the
-test-suite is saved in <sub-directory>/test.log.
-
-Almost any SQLite makefile (except those generated by configure - see below)
-should work. The following properties are required:
+available for building SQLite, especially options taht omit
+features (the SQLITE_OMIT_xxx options). It should be invoked as follows:
 
-  * The makefile should support the "testfixture" target.
-  * The makefile should support the "test" target.
-  * The makefile should support the variable "OPTS" as a way to pass
-    options from the make command line to lemon and the C compiler.
+    ./configure CFLAGS=-O0
+    tclsh test/omittest.tcl
 
-More precisely, the following two invocations must be supported:
-
-  $::MAKEBIN -f $::MAKEFILE testfixture OPTS="-DSQLITE_OMIT_ALTERTABLE=1"
-  $::MAKEBIN -f $::MAKEFILE test
-
-Makefiles generated by the sqlite configure program cannot be used as
-they do not respect the OPTS variable.
 }
 
-
-# Build a testfixture executable and run quick.test using it. The first
-# parameter is the name of the directory to create and use to run the
-# test in. The second parameter is a list of OMIT symbols to define
-# when doing so. For example:
+# List of all options to be tested.
 #
-#     run_quick_test /tmp/testdir {SQLITE_OMIT_TRIGGER SQLITE_OMIT_VIEW}
-#
-#
-proc run_quick_test {dir omit_symbol_list} {
-  # Compile the value of the OPTS Makefile variable.
-  set opts ""
-  if {$::tcl_platform(platform)=="windows"} {
-    append opts "OPTS += -DSQLITE_OS_WIN=1\n"
-    set target "testfixture.exe"
-  } else {
-    append opts "OPTS += -DSQLITE_OS_UNIX=1\n"
-  }
-  foreach sym $omit_symbol_list {
-    append opts "OPTS += -D${sym}=1\n"
-  }
-
-  # Create the directory and do the build. If an error occurs return
-  # early without attempting to run the test suite.
-  file mkdir $dir
-  puts -nonewline "Building $dir..."
-  flush stdout
-  catch {
-    file copy -force ./config.h $dir
-    file copy -force ./libtool $dir
-  }
-  set fd [open $::MAKEFILE]
-  set mkfile [read $fd]
-  close $fd
-  regsub {\ninclude} $mkfile "\n$opts\ninclude" mkfile
-  set fd [open $dir/makefile w]
-  puts $fd $mkfile
-  close $fd
-  
-  set rc [catch {
-    exec $::MAKEBIN -C $dir -f makefile clean $::TARGET >& $dir/build.log
-  }]
-  if {$rc} {
-    puts "No good. See $dir/build.log."
-    return
-  } else {
-    puts "Ok"
-  }
-  
-  # Create an empty file "$dir/sqlite3". This is to trick the makefile out 
-  # of trying to build the sqlite shell. The sqlite shell won't build 
-  # with some of the OMIT options (i.e OMIT_COMPLETE).
-  set sqlite3_dummy $dir/sqlite3
-  if {$::tcl_platform(platform)=="windows"} {
-    append sqlite3_dummy ".exe"
-  }
-  if {![file exists $sqlite3_dummy]} {
-    set wr [open $sqlite3_dummy w]
-    puts $wr "dummy"
-    close $wr
-  }
-
-  if {$::SKIP_RUN} {
-    #  puts "Skip testing $dir."
-  } else {
-    # Run the test suite.
-    puts -nonewline "Testing $dir..."
-    flush stdout
-    set rc [catch {
-      exec $::MAKEBIN -C $dir -f makefile test >& $dir/test.log
-    }]
-    if {$rc} {
-      puts "No good. See $dir/test.log."
-    } else {
-      puts "Ok"
-    }
-  }
+set CompileOptionsToTest {
+  SQLITE_OMIT_ALTERTABLE
+  SQLITE_OMIT_ANALYZE
+  SQLITE_OMIT_ATTACH
+  SQLITE_OMIT_AUTHORIZATION
+  SQLITE_OMIT_AUTOINCREMENT
+  SQLITE_OMIT_AUTOINIT
+  SQLITE_OMIT_AUTOMATIC_INDEX
+  SQLITE_OMIT_AUTORESET
+  SQLITE_OMIT_AUTOVACUUM
+  SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS
+  SQLITE_OMIT_BETWEEN_OPTIMIZATION
+  SQLITE_OMIT_BLOB_LITERAL
+  SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA
+  SQLITE_OMIT_CAST
+  SQLITE_OMIT_CHECK
+  SQLITE_OMIT_COMPILEOPTION_DIAGS
+  SQLITE_OMIT_COMPLETE
+  SQLITE_OMIT_COMPOUND_SELECT
+  SQLITE_OMIT_CONFLICT_CLAUSE
+  SQLITE_OMIT_CTE
+  SQLITE_OMIT_DATETIME_FUNCS
+  SQLITE_OMIT_DECLTYPE
+  SQLITE_OMIT_DEPRECATED
+  SQLITE_OMIT_DESERIALIZE
+  SQLITE_OMIT_DISKIO
+  SQLITE_OMIT_EXPLAIN
+  SQLITE_OMIT_FLAG_PRAGMAS
+  SQLITE_OMIT_FLOATING_POINT
+  SQLITE_OMIT_FOREIGN_KEY
+  SQLITE_OMIT_GENERATED_COLUMNS
+  SQLITE_OMIT_GET_TABLE
+  SQLITE_OMIT_HEX_INTEGER
+  SQLITE_OMIT_INCRBLOB
+  SQLITE_OMIT_INTEGRITY_CHECK
+  SQLITE_OMIT_INTROSPECTION_PRAGMAS
+  SQLITE_OMIT_JSON
+  SQLITE_OMIT_LIKE_OPTIMIZATION
+  SQLITE_OMIT_LOAD_EXTENSION
+  SQLITE_OMIT_LOCALTIME
+  SQLITE_OMIT_LOOKASIDE
+  SQLITE_OMIT_MEMORYDB
+  SQLITE_OMIT_OR_OPTIMIZATION
+  SQLITE_OMIT_PAGER_PRAGMAS 
+  SQLITE_OMIT_PARSER_TRACE
+  SQLITE_OMIT_POPEN
+  SQLITE_OMIT_PRAGMA
+  SQLITE_OMIT_PROGRESS_CALLBACK
+  SQLITE_OMIT_QUICKBALANCE
+  SQLITE_OMIT_RANDOMNESS
+  SQLITE_OMIT_REINDEX
+  SQLITE_OMIT_SCHEMA_PRAGMAS
+  SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS
+  SQLITE_OMIT_SHARED_CACHE
+  SQLITE_OMIT_SHUTDOWN_DIRECTORIES
+  SQLITE_OMIT_SUBQUERY
+  SQLITE_OMIT_TCL_VARIABLE
+  SQLITE_OMIT_TEMPDB
+  SQLITE_OMIT_TEST_CONTROL
+  SQLITE_OMIT_TRACE
+  SQLITE_OMIT_TRIGGER
+  SQLITE_OMIT_TRUNCATE_OPTIMIZATION
+  SQLITE_OMIT_TWOSIZE_LOOKASIDE
+  SQLITE_OMIT_UPSERT
+  SQLITE_OMIT_UTF
+  SQLITE_OMIT_VACUUM
+  SQLITE_OMIT_VIEW
+  SQLITE_OMIT_VIRTUALTABLE
+  SQLITE_OMIT_WAL
+  SQLITE_OMIT_WINDOWFUNC
+  SQLITE_OMIT_WSD
+  SQLITE_OMIT_XFER_OPT
+  SQLITE_ALLOW_ROWID_IN_VIEW
+  SQLITE_DISABLE_DIRSYNC
+  SQLITE_DISABLE_FTS
+  SQLITE_DISABLE_INTRINSIC
+  SQLITE_DISABLE_LFS
+  SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS
+  SQLITE_DISABLE_SKIPAHEAD_DISTINCT
+  SQLITE_ENABLE_API_ARMOR
+  SQLITE_ENABLE_ATOMIC_WRITE
+  SQLITE_ENABLE_BATCH_ATOMIC_WRITE
+  SQLITE_ENABLE_BYTECODE_VTAB
+  SQLITE_ENABLE_CEROD
+  SQLITE_ENABLE_COLUMN_METADATA
+  SQLITE_ENABLE_COLUMN_USED_MASK
+  SQLITE_ENABLE_COMMENTS
+  SQLITE_ENABLE_CORRUPT_PGNO
+  SQLITE_ENABLE_COSTMULT
+  SQLITE_ENABLE_CURSOR_HINTS
+  SQLITE_ENABLE_DBPAGE_VTAB
+  SQLITE_ENABLE_DBSTAT_VTAB
+  SQLITE_ENABLE_EXPENSIVE_ASSERT
+  SQLITE_ENABLE_EXPLAIN_COMMENTS
+  SQLITE_ENABLE_FTS
+  SQLITE_ENABLE_GEOPOLY
+  SQLITE_ENABLE_HIDDEN_COLUMNS
+  SQLITE_ENABLE_ICU
+  SQLITE_ENABLE_ICU_COLLATIONS
+  SQLITE_ENABLE_INTERNAL_FUNCTIONS
+  SQLITE_ENABLE_IOTRACE
+  SQLITE_ENABLE_LOAD_EXTENSION
+  SQLITE_ENABLE_LOCKING_STYLE
+  SQLITE_ENABLE_MATH_FUNCTIONS
+  SQLITE_ENABLE_MEMORY_MANAGEMENT
+  SQLITE_ENABLE_MEMSYS
+  SQLITE_ENABLE_MODULE_COMMENTS
+  SQLITE_ENABLE_MULTIPLEX
+  SQLITE_ENABLE_MULTITHREADED_CHECKS
+  SQLITE_ENABLE_NORMALIZE
+  SQLITE_ENABLE_NULL_TRIM
+  SQLITE_ENABLE_OFFSET_SQL_FUNC
+  SQLITE_ENABLE_OVERSIZE_CELL_CHECK
+  SQLITE_ENABLE_PREUPDATE_HOOK
+  SQLITE_ENABLE_QPSG
+  SQLITE_ENABLE_RBU
+  SQLITE_ENABLE_RTREE
+  SQLITE_ENABLE_SELECTTRACE
+  SQLITE_ENABLE_SESSION
+  SQLITE_ENABLE_SETLK_TIMEOUT
+  SQLITE_ENABLE_SNAPSHOT
+  SQLITE_ENABLE_SORTER_MMAP
+  SQLITE_ENABLE_SORTER_REFERENCE
+  SQLITE_ENABLE_SORTER_REFERENCES
+  SQLITE_ENABLE_SQLLOG
+  SQLITE_ENABLE_STAT
+  SQLITE_ENABLE_STMT_SCANSTATUS
+  SQLITE_ENABLE_STMTVTAB
+  SQLITE_ENABLE_TREETRACE
+  SQLITE_ENABLE_UNKNOWN_FUNCTION
+  SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
+  SQLITE_ENABLE_UNLOCK_NOTIFY
+  SQLITE_ENABLE_UPDATE_DELETE_LIMIT
+  SQLITE_ENABLE_URI_00_ERROR
+  SQLITE_ENABLE_VFSTRACE
+  SQLITE_ENABLE_WHERETRACE
+  SQLITE_ENABLE_ZIPVFS
 }
 
-
-# This proc processes the command line options passed to this script.
-# Currently the only option supported is "-makefile", default
-# "../Makefile.linux-gcc". Set the ::MAKEFILE variable to the value of this
-# option.
+# Parse command-line options.
 #
-proc process_options {argv} {
-  set ::MAKEBIN make                        ;# Default value
-  if {$::tcl_platform(platform)=="windows"} {
-    set ::MAKEFILE ./Makefile               ;# Default value on Windows
-  } else {
-    set ::MAKEFILE ./Makefile.linux-gcc     ;# Default value
-  }
-  set ::SKIP_RUN 1                          ;# Default to attempt test
-  set ::TARGET testfixture                  ;# Default thing to build
-
-  for {set i 0} {$i < [llength $argv]} {incr i} {
-    switch -regexp -- [lindex $argv $i] {
-      -{1,2}makefile {
-        incr i
-        set ::MAKEFILE [lindex $argv $i]
-      }
-  
-      -{1,2}nmake {
-        set ::MAKEBIN nmake
-        set ::MAKEFILE ./Makefile.msc
-      }
-
-      -{1,2}target {
-        incr i
-        set ::TARGET [lindex $argv $i]
-      }
-
-      -{1,2}skip_run {
-        set ::SKIP_RUN 1
-      }
-      -{1,2}run {
-        set ::SKIP_RUN 0
-      }
-
-      -{1,2}help {
-        puts $::USAGE_MESSAGE
-        exit
-      }
-
-      -.* {
-        puts stderr "Unknown option: [lindex $argv i]"
-        puts stderr $::USAGE_MESSAGE
-        exit 1
-      }
-
-      default {
-        if {[info exists ::SYMBOL]} {
-          puts stderr [string trim $::USAGE_MESSAGE]
-          exit -1
-        }
-        set ::SYMBOL [lindex $argv $i]
-      }
+for {set i 0} {$i<[llength $argv]} {incr i} {
+  set arg [lindex $argv $i]
+  switch -- $arg {
+    -start -
+    --start {
+      incr i
+      set startat [lindex $argv $i]
     }
-    set ::MAKEFILE [file normalize $::MAKEFILE]
   }
 }
 
-# Main routine.
+# Additional options required for some settings.
 #
+set More(SQLITE_OMIT_DISKIO) {-DSQLITE_OMIT_WAL}
 
-proc main {argv} {
-  # List of SQLITE_OMIT_XXX symbols supported by SQLite.
-  set ::OMIT_SYMBOLS [list \
-    SQLITE_OMIT_ALTERTABLE \
-    SQLITE_OMIT_ANALYZE \
-    SQLITE_OMIT_ATTACH \
-    SQLITE_OMIT_AUTHORIZATION \
-    SQLITE_OMIT_AUTOINCREMENT \
-    SQLITE_OMIT_AUTOINIT \
-    SQLITE_OMIT_AUTOMATIC_INDEX \
-    SQLITE_OMIT_AUTORESET \
-    SQLITE_OMIT_AUTOVACUUM \
-    SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS \
-    SQLITE_OMIT_BETWEEN_OPTIMIZATION \
-    SQLITE_OMIT_BLOB_LITERAL \
-    SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA \
-    SQLITE_OMIT_CAST \
-    SQLITE_OMIT_CHECK \
-    SQLITE_OMIT_COMPILEOPTION_DIAGS \
-    SQLITE_OMIT_COMPLETE \
-    SQLITE_OMIT_COMPOUND_SELECT \
-    SQLITE_OMIT_CONFLICT_CLAUSE \
-    SQLITE_OMIT_CTE \
-    SQLITE_OMIT_DATETIME_FUNCS \
-    SQLITE_OMIT_DECLTYPE \
-    SQLITE_OMIT_DEPRECATED \
-    SQLITE_OMIT_DESERIALIZE \
-    SQLITE_OMIT_DISKIO \
-    SQLITE_OMIT_EXPLAIN \
-    SQLITE_OMIT_FLAG_PRAGMAS \
-    SQLITE_OMIT_FLOATING_POINT \
-    SQLITE_OMIT_FOREIGN_KEY \
-    SQLITE_OMIT_GENERATED_COLUMNS \
-    SQLITE_OMIT_GET_TABLE \
-    SQLITE_OMIT_HEX_INTEGER \
-    SQLITE_OMIT_INCRBLOB \
-    SQLITE_OMIT_INTEGRITY_CHECK \
-    SQLITE_OMIT_INTROSPECTION_PRAGMAS \
-    SQLITE_OMIT_JSON \
-    SQLITE_OMIT_LIKE_OPTIMIZATION \
-    SQLITE_OMIT_LOAD_EXTENSION \
-    SQLITE_OMIT_LOCALTIME \
-    SQLITE_OMIT_LOOKASIDE \
-    SQLITE_OMIT_MEMORYDB \
-    SQLITE_OMIT_OR_OPTIMIZATION \
-    SQLITE_OMIT_PAGER_PRAGMAS  \
-    SQLITE_OMIT_PARSER_TRACE \
-    SQLITE_OMIT_POPEN \
-    SQLITE_OMIT_PRAGMA \
-    SQLITE_OMIT_PROGRESS_CALLBACK \
-    SQLITE_OMIT_QUICKBALANCE \
-    SQLITE_OMIT_RANDOMNESS \
-    SQLITE_OMIT_REINDEX \
-    SQLITE_OMIT_SCHEMA_PRAGMAS \
-    SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS \
-    SQLITE_OMIT_SHARED_CACHE \
-    SQLITE_OMIT_SHUTDOWN_DIRECTORIES \
-    SQLITE_OMIT_SUBQUERY \
-    SQLITE_OMIT_TCL_VARIABLE \
-    SQLITE_OMIT_TEMPDB \
-    SQLITE_OMIT_TEST_CONTROL \
-    SQLITE_OMIT_TRACE \
-    SQLITE_OMIT_TRIGGER \
-    SQLITE_OMIT_TRUNCATE_OPTIMIZATION \
-    SQLITE_OMIT_TWOSIZE_LOOKASIDE \
-    SQLITE_OMIT_UPSERT \
-    SQLITE_OMIT_UTF \
-    SQLITE_OMIT_VACUUM \
-    SQLITE_OMIT_VIEW \
-    SQLITE_OMIT_VIRTUALTABLE \
-    SQLITE_OMIT_WAL \
-    SQLITE_OMIT_WINDOWFUNC \
-    SQLITE_OMIT_WSD \
-    SQLITE_OMIT_XFER_OPT \
-  ]
-
-  set ::ENABLE_SYMBOLS [list \
-    SQLITE_ALLOW_ROWID_IN_VIEW \
-    SQLITE_DISABLE_DIRSYNC \
-    SQLITE_DISABLE_FTS \
-    SQLITE_DISABLE_INTRINSIC \
-    SQLITE_DISABLE_LFS \
-    SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS \
-    SQLITE_DISABLE_SKIPAHEAD_DISTINCT \
-    SQLITE_ENABLE_API_ARMOR \
-    SQLITE_ENABLE_ATOMIC_WRITE \
-    SQLITE_ENABLE_BATCH_ATOMIC_WRITE \
-    SQLITE_ENABLE_BYTECODE_VTAB \
-    SQLITE_ENABLE_CEROD \
-    SQLITE_ENABLE_COLUMN_METADATA \
-    SQLITE_ENABLE_COLUMN_USED_MASK \
-    SQLITE_ENABLE_COMMENTS \
-    SQLITE_ENABLE_CORRUPT_PGNO \
-    SQLITE_ENABLE_COSTMULT \
-    SQLITE_ENABLE_CURSOR_HINTS \
-    SQLITE_ENABLE_DBPAGE_VTAB \
-    SQLITE_ENABLE_DBSTAT_VTAB \
-    SQLITE_ENABLE_EXPENSIVE_ASSERT \
-    SQLITE_ENABLE_EXPLAIN_COMMENTS \
-    SQLITE_ENABLE_FTS \
-    SQLITE_ENABLE_GEOPOLY \
-    SQLITE_ENABLE_HIDDEN_COLUMNS \
-    SQLITE_ENABLE_ICU \
-    SQLITE_ENABLE_ICU_COLLATIONS \
-    SQLITE_ENABLE_INTERNAL_FUNCTIONS \
-    SQLITE_ENABLE_IOTRACE \
-    SQLITE_ENABLE_LOAD_EXTENSION \
-    SQLITE_ENABLE_LOCKING_STYLE \
-    SQLITE_ENABLE_MATH_FUNCTIONS \
-    SQLITE_ENABLE_MEMORY_MANAGEMENT \
-    SQLITE_ENABLE_MEMSYS \
-    SQLITE_ENABLE_MODULE_COMMENTS \
-    SQLITE_ENABLE_MULTIPLEX \
-    SQLITE_ENABLE_MULTITHREADED_CHECKS \
-    SQLITE_ENABLE_NORMALIZE \
-    SQLITE_ENABLE_NULL_TRIM \
-    SQLITE_ENABLE_OFFSET_SQL_FUNC \
-    SQLITE_ENABLE_OVERSIZE_CELL_CHECK \
-    SQLITE_ENABLE_PREUPDATE_HOOK \
-    SQLITE_ENABLE_QPSG \
-    SQLITE_ENABLE_RBU \
-    SQLITE_ENABLE_RTREE \
-    SQLITE_ENABLE_SELECTTRACE \
-    SQLITE_ENABLE_SESSION \
-    SQLITE_ENABLE_SETLK_TIMEOUT \
-    SQLITE_ENABLE_SNAPSHOT \
-    SQLITE_ENABLE_SORTER_MMAP\
-    SQLITE_ENABLE_SORTER_REFERENCE \
-    SQLITE_ENABLE_SORTER_REFERENCES \
-    SQLITE_ENABLE_SQLLOG\
-    SQLITE_ENABLE_STAT \
-    SQLITE_ENABLE_STMT_SCANSTATUS \
-    SQLITE_ENABLE_STMTVTAB \
-    SQLITE_ENABLE_TREETRACE \
-    SQLITE_ENABLE_UNKNOWN_FUNCTION \
-    SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \
-    SQLITE_ENABLE_UNLOCK_NOTIFY \
-    SQLITE_ENABLE_UPDATE_DELETE_LIMIT \
-    SQLITE_ENABLE_URI_00_ERROR \
-    SQLITE_ENABLE_VFSTRACE \
-    SQLITE_ENABLE_WHERETRACE \
-    SQLITE_ENABLE_ZIPVFS \
-  ]
-
-  # Process any command line options.
-  process_options $argv
-
-  if {[info exists ::SYMBOL] } {
-    set sym $::SYMBOL
+# Compile-time options for Mac only
+#
+set MacOnly(SQLITE_ENABLE_LOCKING_STYLE) 1
 
-    if {[lsearch $::OMIT_SYMBOLS $sym]<0 && [lsearch $::ENABLE_SYMBOLS $sym]<0} {
-      puts stderr "No such symbol: $sym"
-      exit -1
+# Compile-time options that might fail, depending on what libraries
+# are installed.  Failures on these tests issue a warning, but testing
+# continues.
+#
+set FailIsOk(SQLITE_ENABLE_ICU) 1
+set FailIsOk(SQLITE_ENABLE_ICU_COLLATIONS) 1
+
+file mkdir omittest
+foreach sym $CompileOptionsToTest {
+  if {[info exists startat]} {
+    if {$startat==$sym} {
+      unset startat
+    } else {
+      continue
     }
-
-    set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]"
-    run_quick_test $dirname $sym
+  }
+  if {[info exists MacOnly($sym)] && $tcl_platform(os)!="Darwin"} {
+    continue
+  }
+  set logfile "omittest/$sym.log"
+  if {[info exists More($sym)]} {
+    append opts "OPT_FEATURE_FLAGS=-D$sym $More($sym)"
   } else {
-    # First try a test with all OMIT symbols except SQLITE_OMIT_FLOATING_POINT 
-    # and SQLITE_OMIT_PRAGMA defined. The former doesn't work (causes segfaults)
-    # and the latter is currently incompatible with the test suite (this should
-    # be fixed, but it will be a lot of work).
-    set allsyms [list]
-    foreach s $::OMIT_SYMBOLS {
-      if {$s!="SQLITE_OMIT_FLOATING_POINT" && $s!="SQLITE_OMIT_PRAGMA"} {
-        lappend allsyms $s
-      }
-    }
-    run_quick_test test_OMIT_EVERYTHING $allsyms
-  
-    # Now try one quick.test with each of the OMIT symbols defined. Included
-    # are the OMIT_FLOATING_POINT and OMIT_PRAGMA symbols, even though we
-    # know they will fail. It's good to be reminded of this from time to time.
-    foreach sym $::OMIT_SYMBOLS {
-      set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]"
-      run_quick_test $dirname $sym
-    }
-  
-    # Try the ENABLE/DISABLE symbols one at a time.  
-    # We don't do them all at once since some are conflicting.
-    foreach sym $::ENABLE_SYMBOLS {
-      set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]"
-      run_quick_test $dirname $sym
+    set opts OPT_FEATURE_FLAGS=-D$sym
+  }
+  puts            "make tidy sqlite3.o $opts"
+  if {[catch {exec make tidy sqlite3.o $opts >& $logfile}]} {
+    puts "BUILD FAILED: see $logfile for details"
+    if {[info exists FailIsOk($sym)]} {
+      set Failure($sym) 1
+    } else {
+      puts "Note:  After fixes, continue the test using:\n"
+      puts "   [info nameofexe] $argv0 --start $sym\n"
+      exit 1
     }
   }
 }
-
-main $argv
+if {[llength [array names Failure]]>0} {
+  puts "BUILD FAILED on the following:"
+  foreach sym [array names Failure] {
+     puts "  *  $sym"
+  }
+}
diff --git a/tool/pagesig.c b/tool/pagesig.c
index 540c9d7226..37cc804a53 100644
--- a/tool/pagesig.c
+++ b/tool/pagesig.c
@@ -26,7 +26,7 @@
 ** the entire block.
 **
 ** For blocks of more than 16 bytes, the signature is a hex dump of the
-** first 8 bytes followed by a 64-bit has of the entire block.
+** first 8 bytes followed by a 64-bit hash of the entire block.
 */
 static void vlogSignature(unsigned char *p, int n, char *zCksum){
   unsigned int s0 = 0, s1 = 0;
diff --git a/tool/replace.tcl b/tool/replace.tcl
index e87cb922f2..6a462d95ff 100644
--- a/tool/replace.tcl
+++ b/tool/replace.tcl
@@ -4,8 +4,8 @@
 # only lines successfully modified with a regular
 # expression.
 #
-fconfigure stdout -translation binary -encoding binary
-fconfigure stderr -translation binary -encoding binary
+fconfigure stdout -translation binary
+fconfigure stderr -translation binary
 set mode [string tolower [lindex $argv 0]]
 set from [lindex $argv 1]
 set to [lindex $argv 2]
diff --git a/tool/restore_jrnl.tcl b/tool/restore_jrnl.tcl
index 05af4f9a2a..200f9b1d29 100644
--- a/tool/restore_jrnl.tcl
+++ b/tool/restore_jrnl.tcl
@@ -114,40 +114,40 @@ proc dump_jrnl_page {jrnl_pgno} {
   set db_pgno [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset] 4]]
   set chksum [hexio_get_int [hexio_read $jrnl_name [expr $jrnl_pg_offset+4+$db_pgsz] 4]]
   set nonce [calc_nonce $jrnl_pgno]
-  puts [ format {jrnl_pg_offset: %08x (%d)  jrnl_pgno: %d  db_pgno: %d} \
-      $jrnl_pg_offset $jrnl_pg_offset \
-      $jrnl_pgno $db_pgno]
-  puts [ format {nonce: %08x chksum: %08x} \
-      $nonce $chksum]
+  puts [ format {jrnl_pg_offset: %08x (%d)  jrnl_pgno: %d  db_pgno: %d} \
+      $jrnl_pg_offset $jrnl_pg_offset \
+      $jrnl_pgno $db_pgno]
+  puts [ format {nonce: %08x chksum: %08x} \
+      $nonce $chksum]
 
   # now hex dump the data
-  # This is derived from the Tcler's WIKI
-  set fid [open $jrnl_name r]
-  fconfigure $fid -translation binary -encoding binary
-  seek $fid [expr $jrnl_pg_offset+4]
-  set data [read $fid $db_pgsz]
-  close $fid
+  # This is derived from the Tcler's WIKI
+  set fid [open $jrnl_name r]
+  fconfigure $fid -translation binary
+  seek $fid [expr $jrnl_pg_offset+4]
+  set data [read $fid $db_pgsz]
+  close $fid
   for {set addr 0} {$addr<$db_pgsz} {set addr [expr $addr+16]} {
-    # get 16 bytes of data
-    set s [string range $data $addr [expr $addr+16]]
-    
-    # Convert the data to hex and to characters.
-    binary scan $s H*@0a* hex ascii
-
-    # Replace non-printing characters in the data.
-    regsub -all -- {[^[:graph:] ]} $ascii {.} ascii
-
-    # Split the 16 bytes into two 8-byte chunks
-    regexp -- {(.{16})(.{0,16})} $hex -> hex1 hex2
-
-    # Convert the hex to pairs of hex digits
-    regsub -all -- {..} $hex1 {& } hex1
-    regsub -all -- {..} $hex2 {& } hex2
-
-    # Print the hex and ascii data
-    puts [ format {%08x %-24s %-24s %-16s} \
-        $addr $hex1 $hex2 $ascii ]
-  }
+    # get 16 bytes of data
+    set s [string range $data $addr [expr $addr+16]]
+    
+    # Convert the data to hex and to characters.
+    binary scan $s H*@0a* hex ascii
+
+    # Replace non-printing characters in the data.
+    regsub -all -- {[^[:graph:] ]} $ascii {.} ascii
+
+    # Split the 16 bytes into two 8-byte chunks
+    regexp -- {(.{16})(.{0,16})} $hex -> hex1 hex2
+
+    # Convert the hex to pairs of hex digits
+    regsub -all -- {..} $hex1 {& } hex1
+    regsub -all -- {..} $hex2 {& } hex2
+
+    # Print the hex and ascii data
+    puts [ format {%08x %-24s %-24s %-16s} \
+        $addr $hex1 $hex2 $ascii ]
+  }
 }
 
 # Setup for the tests.  Make a backup copy of the files.
@@ -230,4 +230,3 @@ do_test restore_jrnl-1.0 {
   catchsql {PRAGMA integrity_check}
 } {0 ok}
 db close
-
diff --git a/tool/run-speed-test.sh b/tool/run-speed-test.sh
deleted file mode 100644
index 0e970ea0f6..0000000000
--- a/tool/run-speed-test.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/bash
-#
-# This is a template for a script used for day-to-day size and 
-# performance monitoring of SQLite.  Typical usage:
-#
-#     sh run-speed-test.sh trunk  #  Baseline measurement of trunk
-#     sh run-speed-test.sh x1     # Measure some experimental change
-#     fossil test-diff --tk cout-trunk.txt cout-x1.txt   # View chanages
-#
-# There are multiple output files, all with a base name given by
-# the first argument:
-#
-#     summary-$BASE.txt           # Copy of standard output
-#     cout-$BASE.txt              # cachegrind output
-#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
-#
-if test "$1" = ""
-then
-  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
-  exit
-fi
-NAME=$1
-shift
-CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
-SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64"
-SIZE=5
-doExplain=0
-while test "$1" != ""; do
-  case $1 in
-    --reprepare)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --autovacuum)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --utf16be)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --stats)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --without-rowid)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --nomemstat)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --wal)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
-        ;;
-    --size)
-        shift; SIZE=$1
-        ;;
-    --explain)
-        doExplain=1
-        ;;
-    --heap)
-        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
-        shift;
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
-        ;;
-    *)
-        CC_OPTS="$CC_OPTS $1"
-        ;;
-  esac
-  shift
-done
-SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
-echo "NAME           = $NAME" | tee summary-$NAME.txt
-echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
-echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
-rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
-gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
-size sqlite3.o | tee -a summary-$NAME.txt
-if test $doExplain -eq 1; then
-  gcc -g -Os -Wall -I. $CC_OPTS \
-     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
-fi
-SRC=./speedtest1.c
-gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
-ls -l speedtest1 | tee -a summary-$NAME.txt
-valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
-    $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
-size sqlite3.o | tee -a summary-$NAME.txt
-wc sqlite3.c
-cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
-if test $doExplain -eq 1; then
-  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
-fi
diff --git a/tool/showdb.c b/tool/showdb.c
index 0e99331d7b..f0bd9737cf 100644
--- a/tool/showdb.c
+++ b/tool/showdb.c
@@ -27,7 +27,7 @@ typedef sqlite3_uint64 u64;       /* unsigned 64-bit */
 
 
 static struct GlobalData {
-  u32 pagesize;                   /* Size of a database page */
+  i64 pagesize;                   /* Size of a database page */
   int dbfd;                       /* File descriptor for reading the DB */
   u32 mxPage;                     /* Last page number */
   int perLine;                    /* HEX elements to print per line */
@@ -959,6 +959,10 @@ static void page_usage_freelist(u32 pgno){
     a = fileRead((pgno-1)*g.pagesize, g.pagesize);
     iNext = decodeInt32(a);
     n = decodeInt32(a+4);
+    if( n>(g.pagesize - 8)/4 ){
+      printf("ERROR: page %d too many freelist entries (%d)\n", pgno, n);
+      n = (g.pagesize - 8)/4;
+    }
     for(i=0; i<n; i++){
       int child = decodeInt32(a + (i*4+8));
       page_usage_msg(child, "freelist leaf, child %d of trunk page %d",
@@ -1080,17 +1084,28 @@ static void ptrmap_coverage_report(const char *zDbName){
     printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno,
            pgno+1, pgno+perPage);
     a = fileRead((pgno-1)*g.pagesize, usable);
-    for(i=0; i+5<=usable && pgno+1+i/5<=g.mxPage; i+=5){
-      const char *zType = "???";
+    for(i=0; i+5<=usable; i+=5){
+      const char *zType;
       u32 iFrom = decodeInt32(&a[i+1]);
+      const char *zExtra = pgno+1+i/5>g.mxPage ? " (off end of DB)" : "";
       switch( a[i] ){
         case 1:  zType = "b-tree root page";        break;
         case 2:  zType = "freelist page";           break;
         case 3:  zType = "first page of overflow";  break;
         case 4:  zType = "later page of overflow";  break;
         case 5:  zType = "b-tree non-root page";    break;
+        default: {
+          if( zExtra[0]==0 ){
+            printf("%5llu: invalid (0x%02x), parent=%u\n", 
+                   pgno+1+i/5, a[i], iFrom);
+          }
+          zType = 0;
+          break;
+        }
+      }
+      if( zType ){
+        printf("%5llu: %s, parent=%u%s\n", pgno+1+i/5, zType, iFrom, zExtra);
       }
-      printf("%5llu: %s, parent=%u\n", pgno+1+i/5, zType, iFrom);
     }
     sqlite3_free(a);
   }
@@ -1163,7 +1178,7 @@ int main(int argc, char **argv){
   if( g.pagesize==0 ) g.pagesize = 1024;
   sqlite3_free(zPgSz);
 
-  printf("Pagesize: %d\n", g.pagesize);
+  printf("Pagesize: %d\n", (int)g.pagesize);
   g.mxPage = (u32)((szFile+g.pagesize-1)/g.pagesize);
 
   printf("Available pages: 1..%u\n", g.mxPage);
@@ -1203,7 +1218,8 @@ int main(int argc, char **argv){
         iEnd = strtol(&zLeft[2], 0, 0);
         checkPageValidity(iEnd);
       }else if( zLeft && zLeft[0]=='b' ){
-        int ofst, nByte, hdrSize;
+        i64 ofst;
+        int nByte, hdrSize;
         unsigned char *a;
         if( iStart==1 ){
           ofst = hdrSize = 100;
diff --git a/tool/showstat4.c b/tool/showstat4.c
index b8a12ad63c..d1ff4961bb 100644
--- a/tool/showstat4.c
+++ b/tool/showstat4.c
@@ -67,6 +67,7 @@ int main(int argc, char **argv){
                         "**************\n\n");
       sqlite3_free(zIdx);
       zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0));
+      if( zIdx==0 ){ fprintf(stderr, "out of memory\n"); abort(); }
       iRow = 0;
     }
     printf("%s sample %d ------------------------------------\n", zIdx, ++iRow);
diff --git a/tool/showwal.c b/tool/showwal.c
index a8e9b53b32..7e6c0e17cd 100644
--- a/tool/showwal.c
+++ b/tool/showwal.c
@@ -543,16 +543,27 @@ int main(int argc, char **argv){
   }
   zPgSz[0] = 0;
   zPgSz[1] = 0;
-  lseek(fd, 8, SEEK_SET);
-  read(fd, zPgSz, 4);
-  pagesize = zPgSz[1]*65536 + zPgSz[2]*256 + zPgSz[3];
-  if( pagesize==0 ) pagesize = 1024;
-  printf("Pagesize: %d\n", pagesize);
   fstat(fd, &sbuf);
   if( sbuf.st_size<32 ){
-    printf("file too small to be a WAL\n");
+    printf("%s: file too small to be a WAL - only %d bytes\n",
+           argv[1], (int)sbuf.st_size);
     return 0;
   }
+  if( lseek(fd, 8, SEEK_SET)!=8 ){
+    printf("\"%s\" seems to not be a valid WAL file\n", argv[1]);
+    return 1;
+  }
+  if( read(fd, zPgSz, 4)!=4 ){
+    printf("\"%s\": cannot read the page size\n", argv[1]);
+    return 1;
+  }
+  pagesize = zPgSz[1]*65536 + zPgSz[2]*256 + zPgSz[3];
+  if( pagesize==0 ) pagesize = 1024;
+  printf("Pagesize: %d\n", pagesize);
+  if( (pagesize & (pagesize-1))!=0 || pagesize<512 || pagesize>65536 ){
+    printf("\"%s\": invalid page size.\n", argv[1]);
+    return 1;
+  }
   mxFrame = (sbuf.st_size - 32)/(pagesize + 24);
   printf("Available pages: 1..%d\n", mxFrame);
   if( argc==2 ){
diff --git a/tool/soak1.tcl b/tool/soak1.tcl
index 846f905935..e09c566b86 100644
--- a/tool/soak1.tcl
+++ b/tool/soak1.tcl
@@ -4,7 +4,7 @@
 #
 #    tclsh soak1.tcl local-makefile.mk ?target? ?scenario?
 #
-# This generates many variations on local-makefile.mk (by modifing
+# This generates many variations on local-makefile.mk (by modifying
 # the OPT = lines) and runs them will fulltest, one by one.  The
 # constructed makefiles are named "soak1.mk".
 #
diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl
index d0c5e65e38..e50415900a 100644
--- a/tool/spaceanal.tcl
+++ b/tool/spaceanal.tcl
@@ -74,6 +74,16 @@ Options:
 }
   exit 1
 }
+
+# Exit with given code, but first close db if open.
+#
+proc exit_clean {exit_code} {
+  if {0 < [llength [info commands db]]} {
+    db close
+  }
+  exit $exit_code
+}
+
 set file_to_analyze {}
 set flags(-pageinfo) 0
 set flags(-stats) 0
@@ -157,7 +167,7 @@ if {![db exists {SELECT 1 FROM pragma_compile_options
         lacks required capabilities. Recompile using the\
         -DSQLITE_ENABLE_DBSTAT_VTAB compile-time option to fix\
         this problem."
-  exit 1
+  exit_clean 1
 }
 
 db eval {SELECT count(*) FROM sqlite_schema}
@@ -168,7 +178,7 @@ if {$flags(-pageinfo)} {
   db eval {SELECT name, path, pageno FROM temp.stat ORDER BY pageno} {
     puts "$pageno $name $path"
   }
-  exit 0
+  exit_clean 0
 }
 if {$flags(-stats)} {
   db eval {CREATE VIRTUAL TABLE temp.stat USING dbstat}
@@ -198,7 +208,7 @@ if {$flags(-stats)} {
     puts "INSERT INTO stats VALUES($x);"
   }
   puts "COMMIT;"
-  exit 0
+  exit_clean 0
 }
 
 
@@ -581,6 +591,7 @@ set inuse_pgcnt   [expr wide([mem eval $sql])]
 set inuse_percent [percent $inuse_pgcnt $file_pgcnt]
 
 set free_pgcnt    [expr {$file_pgcnt-$inuse_pgcnt-$av_pgcnt}]
+if {$file_bytes>1073741824 && $free_pgcnt>0} {incr free_pgcnt -1}
 set free_percent  [percent $free_pgcnt $file_pgcnt]
 set free_pgcnt2   [db one {PRAGMA freelist_count}]
 set free_percent2 [percent $free_pgcnt2 $file_pgcnt]
@@ -595,6 +606,7 @@ set nindex [db eval {SELECT count(*) FROM sqlite_schema WHERE type='index'}]
 set sql {SELECT count(*) FROM sqlite_schema WHERE name LIKE 'sqlite_autoindex%'}
 set nautoindex [db eval $sql]
 set nmanindex [expr {$nindex-$nautoindex}]
+set nwithoutrowid [db eval {SELECT count(*) FROM pragma_table_list WHERE wr}]
 
 # set total_payload [mem eval "SELECT sum(payload) FROM space_used"]
 set user_payload [mem one {SELECT int(sum(payload)) FROM space_used
@@ -613,6 +625,7 @@ statline {Pages on the freelist (per header)} $free_pgcnt2 $free_percent2
 statline {Pages on the freelist (calculated)} $free_pgcnt $free_percent
 statline {Pages of auto-vacuum overhead} $av_pgcnt $av_percent
 statline {Number of tables in the database} $ntable
+statline {Number of WITHOUT ROWID tables} $nwithoutrowid
 statline {Number of indices} $nindex
 statline {Number of defined indices} $nmanindex
 statline {Number of implied indices} $nautoindex
@@ -671,6 +684,14 @@ if {$nindex>0} {
   subreport {All tables and indices} 1 0
 }
 subreport {All tables} {NOT is_index} 0
+if {$nwithoutrowid>0} {
+  subreport {All WITHOUT ROWID tables} {is_without_rowid} 0
+  set nrowidtab [db eval {SELECT count(*) FROM pragma_table_list
+                         WHERE type='table' AND NOT wr}]
+  if {$nrowidtab>0} {
+     subreport {ALL rowid tables} {NOT is_without_rowid AND NOT is_index} 0
+  }
+}
 if {$nindex>0} {
   subreport {All indices} {is_index} 0
 }
@@ -890,5 +911,7 @@ puts "COMMIT;"
 } err]} {
   puts "ERROR: $err"
   puts $errorInfo
-  exit 1
+  exit_clean 1
 }
+
+exit_clean 0
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
deleted file mode 100644
index 06cbd35745..0000000000
--- a/tool/speed-check.sh
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/bin/bash
-#
-# This is a template for a script used for day-to-day size and 
-# performance monitoring of SQLite.  Typical usage:
-#
-#     sh speed-check.sh trunk  #  Baseline measurement of trunk
-#     sh speed-check.sh x1     # Measure some experimental change
-#     fossil xdiff --tk cout-trunk.txt cout-x1.txt   # View chanages
-#
-# There are multiple output files, all with a base name given by
-# the first argument:
-#
-#     summary-$BASE.txt           # Copy of standard output
-#     cout-$BASE.txt              # cachegrind output
-#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
-#
-if test "$1" = ""
-then
-  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
-  exit
-fi
-NAME=$1
-shift
-#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
-CC_OPTS="-DSQLITE_ENABLE_MEMSYS5"
-CC=gcc
-SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64"
-SIZE=5
-LEAN_OPTS="-DSQLITE_THREADSAFE=0"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE"
-LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
-BASELINE="trunk"
-doExplain=0
-doCachegrind=1
-doVdbeProfile=0
-doWal=1
-doDiff=1
-while test "$1" != ""; do
-  case $1 in
-    --nodiff)
-	doDiff=0
-        ;;
-    --reprepare)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --autovacuum)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --utf16be)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --stats)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --without-rowid)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --strict)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --nomemstat)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --multithread)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --singlethread)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --serialized)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
-        ;;
-    --temp)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
-        ;;
-    --legacy)
-	doWal=0
-        CC_OPTS="$CC_OPTS -DSPEEDTEST_OMIT_HASH"
-        ;;
-    --verify)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --verify"
-        ;;
-    --wal)
-        doWal=1
-        ;;
-    --size)
-        shift; SIZE=$1
-        ;;
-    --cachesize)
-        shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1"
-        ;;
-    --stmtcache)
-        shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtcache $1"
-        ;;
-    --checkpoint)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --checkpoint"
-        ;;
-    --explain)
-        doExplain=1
-        ;;
-    --vdbeprofile)
-        rm -f vdbe_profile.out
-        CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
-        doCachegrind=0
-        doVdbeProfile=1
-        ;;
-    --lean)
-        CC_OPTS="$CC_OPTS $LEAN_OPTS"
-        ;;
-    --clang)
-        CC=clang
-        ;;
-    --icc)
-        CC=/home/drh/intel/bin/icc
-        ;;
-    --gcc7)
-        CC=gcc-7
-        ;;
-    --heap)
-        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
-        shift;
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
-        ;;
-    --lookaside)
-        shift;
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2"
-        shift;
-        ;;
-    --repeat)
-        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE"
-        shift;
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --repeat $1"
-        ;;
-    --mmap)
-        shift;
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1"
-        ;;
-    --rtree)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree"
-        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE"
-        ;;
-    --persist)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --persist"
-        ;;
-    --orm)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm"
-        ;;
-    --cte)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte"
-        ;;
-    --fp)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp"
-        ;;
-    --stmtscanstatus)
-        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus"
-        ;;
-    -*)
-        CC_OPTS="$CC_OPTS $1"
-        ;;
-    *)
-	BASELINE=$1
-        ;;
-  esac
-  shift
-done
-if test $doWal -eq 1; then
-  SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
-fi
-SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
-echo "NAME           = $NAME" | tee summary-$NAME.txt
-echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
-echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
-rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
-if test $doVdbeProfile -eq 1; then
-  rm -f vdbe_profile.out
-fi
-$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
-size sqlite3.o | tee -a summary-$NAME.txt
-if test $doExplain -eq 1; then
-  $CC -g -Os -Wall -I. $CC_OPTS \
-     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
-fi
-SRC=./speedtest1.c
-$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
-ls -l speedtest1 | tee -a summary-$NAME.txt
-if test $doCachegrind -eq 1; then
-  valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
-      $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
-else
-  ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
-fi
-size sqlite3.o | tee -a summary-$NAME.txt
-wc sqlite3.c
-if test $doCachegrind -eq 1; then
-  cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
-  echo '*****************************************************' >>cout-$NAME.txt
-  sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt
-fi
-if test $doExplain -eq 1; then
-  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
-fi
-if test $doVdbeProfile -eq 1; then
-  tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt
-  open vdbeprofile-$NAME.txt
-fi
-if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1 -a $doDiff -ne 0; then
-  fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt
-fi
diff --git a/tool/speedtest.tcl b/tool/speedtest.tcl
deleted file mode 100644
index ef39dc5461..0000000000
--- a/tool/speedtest.tcl
+++ /dev/null
@@ -1,275 +0,0 @@
-#!/usr/bin/tclsh
-#
-# Run this script using TCLSH to do a speed comparison between
-# various versions of SQLite and PostgreSQL and MySQL
-#
-
-# Run a test
-#
-set cnt 1
-proc runtest {title} {
-  global cnt
-  set sqlfile test$cnt.sql
-  puts "<h2>Test $cnt: $title</h2>"
-  incr cnt
-  set fd [open $sqlfile r]
-  set sql [string trim [read $fd [file size $sqlfile]]]
-  close $fd
-  set sx [split $sql \n]
-  set n [llength $sx]
-  if {$n>8} {
-    set sql {}
-    for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]<br>\n}
-    append sql  "<i>... [expr {$n-6}] lines omitted</i><br>\n"
-    for {set i [expr {$n-3}]} {$i<$n} {incr i} {
-      append sql [lindex $sx $i]<br>\n
-    }
-  } else {
-    regsub -all \n [string trim $sql] <br> sql
-  }
-  puts "<blockquote>"
-  puts "$sql"
-  puts "</blockquote><table border=0 cellpadding=0 cellspacing=0>"
-  set format {<tr><td>%s</td><td align="right">&nbsp;&nbsp;&nbsp;%.3f</td></tr>}
-  set delay 1000
-#  exec sync; after $delay;
-#  set t [time "exec psql drh <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format PostgreSQL: $t]
-  exec sync; after $delay;
-  set t [time "exec mysql -f drh <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format MySQL: $t]
-#  set t [time "exec ./sqlite232 s232.db <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format {SQLite 2.3.2:} $t]
-#  set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format {SQLite 2.4 (cache=100):} $t]
-  exec sync; after $delay;
-  set t [time "exec ./sqlite248 s2k.db <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format {SQLite 2.4.8:} $t]
-  exec sync; after $delay;
-  set t [time "exec ./sqlite248 sns.db <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format {SQLite 2.4.8 (nosync):} $t]
-  exec sync; after $delay;
-  set t [time "exec ./sqlite2412 s2kb.db <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format {SQLite 2.4.12:} $t]
-  exec sync; after $delay;
-  set t [time "exec ./sqlite2412 snsb.db <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format {SQLite 2.4.12 (nosync):} $t]
-#  set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format {SQLite 2.4 (test):} $t]
-  puts "</table>"
-}
-
-# Initialize the environment
-#
-expr srand(1)
-catch {exec /bin/sh -c {rm -f s*.db}}
-set fd [open clear.sql w]
-puts $fd {
-  drop table t1;
-  drop table t2;
-}
-close $fd
-catch {exec psql drh <clear.sql}
-catch {exec mysql drh <clear.sql}
-set fd [open 2kinit.sql w]
-puts $fd {
-  PRAGMA default_cache_size=2000;
-  PRAGMA default_synchronous=on;
-}
-close $fd
-exec ./sqlite248 s2k.db <2kinit.sql
-exec ./sqlite2412 s2kb.db <2kinit.sql
-set fd [open nosync-init.sql w]
-puts $fd {
-  PRAGMA default_cache_size=2000;
-  PRAGMA default_synchronous=off;
-}
-close $fd
-exec ./sqlite248 sns.db <nosync-init.sql
-exec ./sqlite2412 snsb.db <nosync-init.sql
-set ones {zero one two three four five six seven eight nine
-          ten eleven twelve thirteen fourteen fifteen sixteen seventeen
-          eighteen nineteen}
-set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety}
-proc number_name {n} {
-  if {$n>=1000} {
-    set txt "[number_name [expr {$n/1000}]] thousand"
-    set n [expr {$n%1000}]
-  } else {
-    set txt {}
-  }
-  if {$n>=100} {
-    append txt " [lindex $::ones [expr {$n/100}]] hundred"
-    set n [expr {$n%100}]
-  }
-  if {$n>=20} {
-    append txt " [lindex $::tens [expr {$n/10}]]"
-    set n [expr {$n%10}]
-  }
-  if {$n>0} {
-    append txt " [lindex $::ones $n]"
-  }
-  set txt [string trim $txt]
-  if {$txt==""} {set txt zero}
-  return $txt
-}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));"
-for {set i 1} {$i<=1000} {incr i} {
-  set r [expr {int(rand()*100000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-close $fd
-runtest {1000 INSERTs}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-puts $fd "CREATE TABLE t2(a INTEGER, b INTEGER, c VARCHAR(100));"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 INSERTs in a transaction}
-
-
-
-set fd [open test$cnt.sql w]
-for {set i 0} {$i<100} {incr i} {
-  set lwr [expr {$i*100}]
-  set upr [expr {($i+10)*100}]
-  puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;"
-}
-close $fd
-runtest {100 SELECTs without an index}
-
-
-
-set fd [open test$cnt.sql w]
-for {set i 1} {$i<=100} {incr i} {
-  puts $fd "SELECT count(*), avg(b) FROM t2 WHERE c LIKE '%[number_name $i]%';"
-}
-close $fd
-runtest {100 SELECTs on a string comparison}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd {CREATE INDEX i2a ON t2(a);}
-puts $fd {CREATE INDEX i2b ON t2(b);}
-close $fd
-runtest {Creating an index}
-
-
-
-set fd [open test$cnt.sql w]
-for {set i 0} {$i<5000} {incr i} {
-  set lwr [expr {$i*100}]
-  set upr [expr {($i+1)*100}]
-  puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;"
-}
-close $fd
-runtest {5000 SELECTs with an index}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 0} {$i<1000} {incr i} {
-  set lwr [expr {$i*10}]
-  set upr [expr {($i+1)*10}]
-  puts $fd "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {1000 UPDATEs without an index}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "UPDATE t2 SET b=$r WHERE a=$i;"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 UPDATEs with an index}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "UPDATE t2 SET c='[number_name $r]' WHERE a=$i;"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 text UPDATEs with an index}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-puts $fd "INSERT INTO t1 SELECT * FROM t2;"
-puts $fd "INSERT INTO t2 SELECT * FROM t1;"
-puts $fd "COMMIT;"
-close $fd
-runtest {INSERTs from a SELECT}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd {DELETE FROM t2 WHERE c LIKE '%fifty%';}
-close $fd
-runtest {DELETE without an index}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd {DELETE FROM t2 WHERE a>10 AND a<20000;}
-close $fd
-runtest {DELETE with an index}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd {INSERT INTO t2 SELECT * FROM t1;}
-close $fd
-runtest {A big INSERT after a big DELETE}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd {BEGIN;}
-puts $fd {DELETE FROM t1;}
-for {set i 1} {$i<=3000} {incr i} {
-  set r [expr {int(rand()*100000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd {COMMIT;}
-close $fd
-runtest {A big DELETE followed by many small INSERTs}
-
-
-
-set fd [open test$cnt.sql w]
-puts $fd {DROP TABLE t1;}
-puts $fd {DROP TABLE t2;}
-close $fd
-runtest {DROP TABLE}
diff --git a/tool/speedtest16.c b/tool/speedtest16.c
deleted file mode 100644
index 993cc19268..0000000000
--- a/tool/speedtest16.c
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
-** Performance test for SQLite.
-**
-** This program reads ASCII text from a file named on the command-line.
-** It converts each SQL statement into UTF16 and submits it to SQLite
-** for evaluation.  A new UTF16 database is created at the beginning of
-** the program.  All statements are timed using the high-resolution timer
-** built into Intel-class processors.
-**
-** To compile this program, first compile the SQLite library separately
-** will full optimizations.  For example:
-**
-**     gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c
-**
-** Then link against this program.  But to do optimize this program
-** because that defeats the hi-res timer.
-**
-**     gcc speedtest16.c sqlite3.o -ldl -I../src
-**
-** Then run this program with a single argument which is the name of
-** a file containing SQL script that you want to test:
-**
-**     ./a.out database.db test.sql
-*/
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <unistd.h>
-#include "sqlite3.h"
-
-#define ISSPACE(X)  isspace((unsigned char)(X))
-
-/* 
-** hwtime.h contains inline assembler code for implementing 
-** high-performance timing routines.
-*/
-#include "hwtime.h"
-
-/*
-** Convert a zero-terminated ASCII string into a zero-terminated
-** UTF-16le string.  Memory to hold the returned string comes 
-** from malloc() and should be freed by the caller.
-*/
-static void *asciiToUtf16le(const char *z){
-  int n = strlen(z);
-  char *z16;
-  int i, j;
-
-  z16 = malloc( n*2 + 2 );
-  for(i=j=0; i<=n; i++){
-    z16[j++] = z[i];
-    z16[j++] = 0;
-  }
-  return (void*)z16;
-}
-
-/*
-** Timers
-*/
-static sqlite_uint64 prepTime = 0;
-static sqlite_uint64 runTime = 0;
-static sqlite_uint64 finalizeTime = 0;
-
-/*
-** Prepare and run a single statement of SQL.
-*/
-static void prepareAndRun(sqlite3 *db, const char *zSql){
-  void *utf16;
-  sqlite3_stmt *pStmt;
-  const void *stmtTail;
-  sqlite_uint64 iStart, iElapse;
-  int rc;
-  
-  printf("****************************************************************\n");
-  printf("SQL statement: [%s]\n", zSql);
-  utf16 = asciiToUtf16le(zSql);
-  iStart = sqlite3Hwtime();
-  rc = sqlite3_prepare16_v2(db, utf16, -1, &pStmt, &stmtTail);
-  iElapse = sqlite3Hwtime() - iStart;
-  prepTime += iElapse;
-  printf("sqlite3_prepare16_v2() returns %d in %llu cycles\n", rc, iElapse);
-  if( rc==SQLITE_OK ){
-    int nRow = 0;
-    iStart = sqlite3Hwtime();
-    while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; }
-    iElapse = sqlite3Hwtime() - iStart;
-    runTime += iElapse;
-    printf("sqlite3_step() returns %d after %d rows in %llu cycles\n",
-           rc, nRow, iElapse);
-    iStart = sqlite3Hwtime();
-    rc = sqlite3_finalize(pStmt);
-    iElapse = sqlite3Hwtime() - iStart;
-    finalizeTime += iElapse;
-    printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse);
-  }
-  free(utf16);
-}
-
-int main(int argc, char **argv){
-  void *utf16;
-  sqlite3 *db;
-  int rc;
-  int nSql;
-  char *zSql;
-  int i, j;
-  FILE *in;
-  sqlite_uint64 iStart, iElapse;
-  sqlite_uint64 iSetup = 0;
-  int nStmt = 0;
-  int nByte = 0;
-
-  if( argc!=3 ){
-    fprintf(stderr, "Usage: %s FILENAME SQL-SCRIPT\n"
-                    "Runs SQL-SCRIPT as UTF16 against a UTF16 database\n",
-                    argv[0]);
-    exit(1);
-  }
-  in = fopen(argv[2], "r");
-  fseek(in, 0L, SEEK_END);
-  nSql = ftell(in);
-  zSql = malloc( nSql+1 );
-  fseek(in, 0L, SEEK_SET);
-  nSql = fread(zSql, 1, nSql, in);
-  zSql[nSql] = 0;
-
-  printf("SQLite version: %d\n", sqlite3_libversion_number());
-  unlink(argv[1]);
-  utf16 = asciiToUtf16le(argv[1]);
-  iStart = sqlite3Hwtime();
-  rc = sqlite3_open16(utf16, &db);
-  iElapse = sqlite3Hwtime() - iStart;
-  iSetup = iElapse;
-  printf("sqlite3_open16() returns %d in %llu cycles\n", rc, iElapse);
-  free(utf16);
-  for(i=j=0; j<nSql; j++){
-    if( zSql[j]==';' ){
-      int isComplete;
-      char c = zSql[j+1];
-      zSql[j+1] = 0;
-      isComplete = sqlite3_complete(&zSql[i]);
-      zSql[j+1] = c;
-      if( isComplete ){
-        zSql[j] = 0;
-        while( i<j && ISSPACE(zSql[i]) ){ i++; }
-        if( i<j ){
-          nStmt++;
-          nByte += j-i;
-          prepareAndRun(db, &zSql[i]);
-        }
-        zSql[j] = ';';
-        i = j+1;
-      }
-    }
-  }
-  iStart = sqlite3Hwtime();
-  sqlite3_close(db);
-  iElapse = sqlite3Hwtime() - iStart;
-  iSetup += iElapse;
-  printf("sqlite3_close() returns in %llu cycles\n", iElapse);
-  printf("\n");
-  printf("Statements run:       %15d\n", nStmt);
-  printf("Bytes of SQL text:    %15d\n", nByte);
-  printf("Total prepare time:   %15llu cycles\n", prepTime);
-  printf("Total run time:       %15llu cycles\n", runTime);
-  printf("Total finalize time:  %15llu cycles\n", finalizeTime);
-  printf("Open/Close time:      %15llu cycles\n", iSetup);
-  printf("Total Time:           %15llu cycles\n",
-      prepTime + runTime + finalizeTime + iSetup);
-  return 0;
-}
diff --git a/tool/speedtest2.tcl b/tool/speedtest2.tcl
deleted file mode 100644
index 4fd632d4c7..0000000000
--- a/tool/speedtest2.tcl
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/usr/bin/tclsh
-#
-# Run this script using TCLSH to do a speed comparison between
-# various versions of SQLite and PostgreSQL and MySQL
-#
-
-# Run a test
-#
-set cnt 1
-proc runtest {title} {
-  global cnt
-  set sqlfile test$cnt.sql
-  puts "<h2>Test $cnt: $title</h2>"
-  incr cnt
-  set fd [open $sqlfile r]
-  set sql [string trim [read $fd [file size $sqlfile]]]
-  close $fd
-  set sx [split $sql \n]
-  set n [llength $sx]
-  if {$n>8} {
-    set sql {}
-    for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]<br>\n}
-    append sql  "<i>... [expr {$n-6}] lines omitted</i><br>\n"
-    for {set i [expr {$n-3}]} {$i<$n} {incr i} {
-      append sql [lindex $sx $i]<br>\n
-    }
-  } else {
-    regsub -all \n [string trim $sql] <br> sql
-  }
-  puts "<blockquote>"
-  puts "$sql"
-  puts "</blockquote><table border=0 cellpadding=0 cellspacing=0>"
-  set format {<tr><td>%s</td><td align="right">&nbsp;&nbsp;&nbsp;%.3f</td></tr>}
-  set delay 1000
-  exec sync; after $delay;
-  set t [time "exec psql drh <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format PostgreSQL: $t]
-  exec sync; after $delay;
-  set t [time "exec mysql -f drh <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format MySQL: $t]
-#  set t [time "exec ./sqlite232 s232.db <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format {SQLite 2.3.2:} $t]
-#  set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format {SQLite 2.4 (cache=100):} $t]
-  exec sync; after $delay;
-  set t [time "exec ./sqlite240 s2k.db <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format {SQLite 2.4:} $t]
-  exec sync; after $delay;
-  set t [time "exec ./sqlite240 sns.db <$sqlfile" 1]
-  set t [expr {[lindex $t 0]/1000000.0}]
-  puts [format $format {SQLite 2.4 (nosync):} $t]
-#  set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1]
-#  set t [expr {[lindex $t 0]/1000000.0}]
-#  puts [format $format {SQLite 2.4 (test):} $t]
-  puts "</table>"
-}
-
-# Initialize the environment
-#
-expr srand(1)
-catch {exec /bin/sh -c {rm -f s*.db}}
-set fd [open clear.sql w]
-puts $fd {
-  drop table t1;
-  drop table t2;
-}
-close $fd
-catch {exec psql drh <clear.sql}
-catch {exec mysql drh <clear.sql}
-set fd [open 2kinit.sql w]
-puts $fd {
-  PRAGMA default_cache_size=2000;
-  PRAGMA default_synchronous=on;
-}
-close $fd
-exec ./sqlite240 s2k.db <2kinit.sql
-exec ./sqlite-t1 st1.db <2kinit.sql
-set fd [open nosync-init.sql w]
-puts $fd {
-  PRAGMA default_cache_size=2000;
-  PRAGMA default_synchronous=off;
-}
-close $fd
-exec ./sqlite240 sns.db <nosync-init.sql
-set ones {zero one two three four five six seven eight nine
-          ten eleven twelve thirteen fourteen fifteen sixteen seventeen
-          eighteen nineteen}
-set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety}
-proc number_name {n} {
-  if {$n>=1000} {
-    set txt "[number_name [expr {$n/1000}]] thousand"
-    set n [expr {$n%1000}]
-  } else {
-    set txt {}
-  }
-  if {$n>=100} {
-    append txt " [lindex $::ones [expr {$n/100}]] hundred"
-    set n [expr {$n%100}]
-  }
-  if {$n>=20} {
-    append txt " [lindex $::tens [expr {$n/10}]]"
-    set n [expr {$n%10}]
-  }
-  if {$n>0} {
-    append txt " [lindex $::ones $n]"
-  }
-  set txt [string trim $txt]
-  if {$txt==""} {set txt zero}
-  return $txt
-}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 INSERTs in a transaction}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "DELETE FROM t1;"
-close $fd
-runtest {DELETE everything}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 INSERTs in a transaction}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "DELETE FROM t1;"
-close $fd
-runtest {DELETE everything}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 INSERTs in a transaction}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "DELETE FROM t1;"
-close $fd
-runtest {DELETE everything}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 INSERTs in a transaction}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "DELETE FROM t1;"
-close $fd
-runtest {DELETE everything}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "BEGIN;"
-for {set i 1} {$i<=25000} {incr i} {
-  set r [expr {int(rand()*500000)}]
-  puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');"
-}
-puts $fd "COMMIT;"
-close $fd
-runtest {25000 INSERTs in a transaction}
-
-
-set fd [open test$cnt.sql w]
-puts $fd "DELETE FROM t1;"
-close $fd
-runtest {DELETE everything}
-
-
-set fd [open test$cnt.sql w]
-puts $fd {DROP TABLE t1;}
-close $fd
-runtest {DROP TABLE}
diff --git a/tool/speedtest8.c b/tool/speedtest8.c
deleted file mode 100644
index 051fc89819..0000000000
--- a/tool/speedtest8.c
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
-** Performance test for SQLite.
-**
-** This program reads ASCII text from a file named on the command-line
-** and submits that text  to SQLite for evaluation.  A new database
-** is created at the beginning of the program.  All statements are
-** timed using the high-resolution timer built into Intel-class processors.
-**
-** To compile this program, first compile the SQLite library separately
-** will full optimizations.  For example:
-**
-**     gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c
-**
-** Then link against this program.  But to do optimize this program
-** because that defeats the hi-res timer.
-**
-**     gcc speedtest8.c sqlite3.o -ldl -I../src
-**
-** Then run this program with a single argument which is the name of
-** a file containing SQL script that you want to test:
-**
-**     ./a.out test.db  test.sql
-*/
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <time.h>
-
-#if defined(_MSC_VER)
-#include <windows.h>
-#else
-#include <unistd.h>
-#include <sys/times.h>
-#include <sched.h>
-#endif
-
-#include "sqlite3.h"
-
-/* 
-** hwtime.h contains inline assembler code for implementing 
-** high-performance timing routines.
-*/
-#include "hwtime.h"
-
-/*
-** Timers
-*/
-static sqlite_uint64 prepTime = 0;
-static sqlite_uint64 runTime = 0;
-static sqlite_uint64 finalizeTime = 0;
-
-/*
-** Prepare and run a single statement of SQL.
-*/
-static void prepareAndRun(sqlite3 *db, const char *zSql, int bQuiet){
-  sqlite3_stmt *pStmt;
-  const char *stmtTail;
-  sqlite_uint64 iStart, iElapse;
-  int rc;
-  
-  if (!bQuiet){
-    printf("***************************************************************\n");
-  }
-  if (!bQuiet) printf("SQL statement: [%s]\n", zSql);
-  iStart = sqlite3Hwtime();
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail);
-  iElapse = sqlite3Hwtime() - iStart;
-  prepTime += iElapse;
-  if (!bQuiet){
-    printf("sqlite3_prepare_v2() returns %d in %llu cycles\n", rc, iElapse);
-  }
-  if( rc==SQLITE_OK ){
-    int nRow = 0;
-    iStart = sqlite3Hwtime();
-    while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; }
-    iElapse = sqlite3Hwtime() - iStart;
-    runTime += iElapse;
-    if (!bQuiet){
-      printf("sqlite3_step() returns %d after %d rows in %llu cycles\n",
-             rc, nRow, iElapse);
-    }
-    iStart = sqlite3Hwtime();
-    rc = sqlite3_finalize(pStmt);
-    iElapse = sqlite3Hwtime() - iStart;
-    finalizeTime += iElapse;
-    if (!bQuiet){
-      printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse);
-    }
-  }
-}
-
-int main(int argc, char **argv){
-  sqlite3 *db;
-  int rc;
-  int nSql;
-  char *zSql;
-  int i, j;
-  FILE *in;
-  sqlite_uint64 iStart, iElapse;
-  sqlite_uint64 iSetup = 0;
-  int nStmt = 0;
-  int nByte = 0;
-  const char *zArgv0 = argv[0];
-  int bQuiet = 0;
-#if !defined(_MSC_VER)
-  struct tms tmsStart, tmsEnd;
-  clock_t clkStart, clkEnd;
-#endif
-
-#ifdef HAVE_OSINST
-  extern sqlite3_vfs *sqlite3_instvfs_binarylog(char *, char *, char *);
-  extern void sqlite3_instvfs_destroy(sqlite3_vfs *);
-  sqlite3_vfs *pVfs = 0;
-#endif
-
-  while (argc>3)
-  {
-#ifdef HAVE_OSINST
-    if( argc>4 && (strcmp(argv[1], "-log")==0) ){
-     pVfs = sqlite3_instvfs_binarylog("oslog", 0, argv[2]);
-     sqlite3_vfs_register(pVfs, 1);
-     argv += 2;
-     argc -= 2;
-     continue;
-    }
-#endif
-
-    /*
-    ** Increasing the priority slightly above normal can help with
-    ** repeatability of testing.  Note that with Cygwin, -5 equates
-    ** to "High", +5 equates to "Low", and anything in between
-    ** equates to "Normal".
-    */
-    if( argc>4 && (strcmp(argv[1], "-priority")==0) ){
-#if defined(_MSC_VER)
-      int new_priority = atoi(argv[2]);
-      if(!SetPriorityClass(GetCurrentProcess(), 
-        (new_priority<=-5) ? HIGH_PRIORITY_CLASS : 
-        (new_priority<=0)  ? ABOVE_NORMAL_PRIORITY_CLASS : 
-        (new_priority==0)  ? NORMAL_PRIORITY_CLASS : 
-        (new_priority<5)   ? BELOW_NORMAL_PRIORITY_CLASS : 
-        IDLE_PRIORITY_CLASS)){
-        printf ("error setting priority\n"); 
-        exit(2); 
-      }
-#else
-      struct sched_param myParam;
-      sched_getparam(0, &myParam);
-      printf ("Current process priority is %d.\n", (int)myParam.sched_priority); 
-      myParam.sched_priority = atoi(argv[2]);
-      printf ("Setting process priority to %d.\n", (int)myParam.sched_priority); 
-      if (sched_setparam (0, &myParam) != 0){
-        printf ("error setting priority\n"); 
-        exit(2); 
-      }
-#endif
-      argv += 2;
-      argc -= 2;
-      continue;
-    }
-
-    if( argc>3 && strcmp(argv[1], "-quiet")==0 ){
-     bQuiet = -1;
-     argv++;
-     argc--;
-     continue;
-    }
-
-    break;
-  }
-
-  if( argc!=3 ){
-   fprintf(stderr, "Usage: %s [options] FILENAME SQL-SCRIPT\n"
-              "Runs SQL-SCRIPT against a UTF8 database\n"
-              "\toptions:\n"
-#ifdef HAVE_OSINST
-              "\t-log <log>\n"
-#endif
-              "\t-priority <value> : set priority of task\n"
-              "\t-quiet : only display summary results\n",
-              zArgv0);
-   exit(1);
-  }
-
-  in = fopen(argv[2], "r");
-  fseek(in, 0L, SEEK_END);
-  nSql = ftell(in);
-  zSql = malloc( nSql+1 );
-  fseek(in, 0L, SEEK_SET);
-  nSql = fread(zSql, 1, nSql, in);
-  zSql[nSql] = 0;
-
-  printf("SQLite version: %d\n", sqlite3_libversion_number());
-  unlink(argv[1]);
-#if !defined(_MSC_VER)
-  clkStart = times(&tmsStart);
-#endif
-  iStart = sqlite3Hwtime();
-  rc = sqlite3_open(argv[1], &db);
-  iElapse = sqlite3Hwtime() - iStart;
-  iSetup = iElapse;
-  if (!bQuiet) printf("sqlite3_open() returns %d in %llu cycles\n", rc, iElapse);
-  for(i=j=0; j<nSql; j++){
-    if( zSql[j]==';' ){
-      int isComplete;
-      char c = zSql[j+1];
-      zSql[j+1] = 0;
-      isComplete = sqlite3_complete(&zSql[i]);
-      zSql[j+1] = c;
-      if( isComplete ){
-        zSql[j] = 0;
-        while( i<j && isspace(zSql[i]) ){ i++; }
-        if( i<j ){
-          int n = j - i;
-          if( n>=6 && memcmp(&zSql[i], ".crash",6)==0 ) exit(1);
-          nStmt++;
-          nByte += n;
-          prepareAndRun(db, &zSql[i], bQuiet);
-        }
-        zSql[j] = ';';
-        i = j+1;
-      }
-    }
-  }
-  iStart = sqlite3Hwtime();
-  sqlite3_close(db);
-  iElapse = sqlite3Hwtime() - iStart;
-#if !defined(_MSC_VER)
-  clkEnd = times(&tmsEnd);
-#endif
-  iSetup += iElapse;
-  if (!bQuiet) printf("sqlite3_close() returns in %llu cycles\n", iElapse);
-
-  printf("\n");
-  printf("Statements run:        %15d stmts\n", nStmt);
-  printf("Bytes of SQL text:     %15d bytes\n", nByte);
-  printf("Total prepare time:    %15llu cycles\n", prepTime);
-  printf("Total run time:        %15llu cycles\n", runTime);
-  printf("Total finalize time:   %15llu cycles\n", finalizeTime);
-  printf("Open/Close time:       %15llu cycles\n", iSetup);
-  printf("Total time:            %15llu cycles\n",
-      prepTime + runTime + finalizeTime + iSetup);
-
-#if !defined(_MSC_VER)
-  printf("\n");
-  printf("Total user CPU time:   %15.3g secs\n", (tmsEnd.tms_utime - tmsStart.tms_utime)/(double)CLOCKS_PER_SEC );
-  printf("Total system CPU time: %15.3g secs\n", (tmsEnd.tms_stime - tmsStart.tms_stime)/(double)CLOCKS_PER_SEC );
-  printf("Total real time:       %15.3g secs\n", (clkEnd -clkStart)/(double)CLOCKS_PER_SEC );
-#endif
-
-#ifdef HAVE_OSINST
-  if( pVfs ){
-    sqlite3_instvfs_destroy(pVfs);
-    printf("vfs log written to %s\n", argv[0]);
-  }
-#endif
-
-  return 0;
-}
diff --git a/tool/speedtest8inst1.c b/tool/speedtest8inst1.c
deleted file mode 100644
index ceaeca0f16..0000000000
--- a/tool/speedtest8inst1.c
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
-** Performance test for SQLite.
-**
-** This program reads ASCII text from a file named on the command-line
-** and submits that text  to SQLite for evaluation.  A new database
-** is created at the beginning of the program.  All statements are
-** timed using the high-resolution timer built into Intel-class processors.
-**
-** To compile this program, first compile the SQLite library separately
-** will full optimizations.  For example:
-**
-**     gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c
-**
-** Then link against this program.  But to do optimize this program
-** because that defeats the hi-res timer.
-**
-**     gcc speedtest8.c sqlite3.o -ldl -I../src
-**
-** Then run this program with a single argument which is the name of
-** a file containing SQL script that you want to test:
-**
-**     ./a.out test.db  test.sql
-*/
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <unistd.h>
-#include <stdarg.h>
-#include "sqlite3.h"
-
-#define ISSPACE(X)  isspace((unsigned char)(X))
-
-#include "test_osinst.c"
-
-/*
-** Prepare and run a single statement of SQL.
-*/
-static void prepareAndRun(sqlite3_vfs *pInstVfs, sqlite3 *db, const char *zSql){
-  sqlite3_stmt *pStmt;
-  const char *stmtTail;
-  int rc;
-  char zMessage[1024];
-  zMessage[1023] = '\0';
-
-  sqlite3_uint64 iTime;
-
-  sqlite3_snprintf(1023, zMessage, "sqlite3_prepare_v2: %s", zSql);
-  sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage);
-
-  iTime = sqlite3Hwtime();
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail);
-  iTime = sqlite3Hwtime() - iTime;
-  sqlite3_instvfs_binarylog_call(pInstVfs,BINARYLOG_PREPARE_V2,iTime,rc,zSql);
-
-  if( rc==SQLITE_OK ){
-    int nRow = 0;
-
-    sqlite3_snprintf(1023, zMessage, "sqlite3_step loop: %s", zSql);
-    sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage);
-    iTime = sqlite3Hwtime();
-    while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; }
-    iTime = sqlite3Hwtime() - iTime;
-    sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_STEP, iTime, rc, zSql);
-
-    sqlite3_snprintf(1023, zMessage, "sqlite3_finalize: %s", zSql);
-    sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage);
-    iTime = sqlite3Hwtime();
-    rc = sqlite3_finalize(pStmt);
-    iTime = sqlite3Hwtime() - iTime;
-    sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_FINALIZE, iTime, rc, zSql);
-  }
-}
-
-static int stringcompare(const char *zLeft, const char *zRight){
-  int ii;
-  for(ii=0; zLeft[ii] && zRight[ii]; ii++){
-    if( zLeft[ii]!=zRight[ii] ) return 0;
-  }
-  return( zLeft[ii]==zRight[ii] );
-}
-
-static char *readScriptFile(const char *zFile, int *pnScript){
-  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
-  sqlite3_file *p;
-  int rc;
-  sqlite3_int64 nByte;
-  char *zData = 0;
-  int flags = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_DB;
-
-  p = (sqlite3_file *)malloc(pVfs->szOsFile);
-  rc = pVfs->xOpen(pVfs, zFile, p, flags, &flags);
-  if( rc!=SQLITE_OK ){
-    goto error_out;
-  }
-
-  rc = p->pMethods->xFileSize(p, &nByte);
-  if( rc!=SQLITE_OK ){
-    goto close_out;
-  }
-
-  zData = (char *)malloc(nByte+1);
-  rc = p->pMethods->xRead(p, zData, nByte, 0);
-  if( rc!=SQLITE_OK ){
-    goto close_out;
-  }
-  zData[nByte] = '\0';
-
-  p->pMethods->xClose(p);
-  free(p);
-  *pnScript = nByte;
-  return zData;
-
-close_out:
-  p->pMethods->xClose(p);
-
-error_out:
-  free(p);
-  free(zData);
-  return 0;
-}
-
-int main(int argc, char **argv){
-
-  const char zUsageMsg[] = 
-    "Usage: %s options...\n"
-    "  where available options are:\n"
-    "\n"
-    "    -db      DATABASE-FILE  (database file to operate on)\n"
-    "    -script  SCRIPT-FILE    (script file to read sql from)\n"
-    "    -log     LOG-FILE       (log file to create)\n"
-    "    -logdata                (log all data to log file)\n"
-    "\n"
-    "  Options -db, -script and -log are compulsory\n"
-    "\n"
-  ;
-
-  const char *zDb = 0;
-  const char *zScript = 0;
-  const char *zLog = 0;
-  int logdata = 0;
-
-  int ii;
-  int i, j;
-  int rc;
-
-  sqlite3_vfs *pInstVfs;                 /* Instrumentation VFS */
-
-  char *zSql = 0;
-  int nSql;
-
-  sqlite3 *db;
-
-  for(ii=1; ii<argc; ii++){
-    if( stringcompare("-db", argv[ii]) && (ii+1)<argc ){
-      zDb = argv[++ii];
-    }
-
-    else if( stringcompare("-script", argv[ii]) && (ii+1)<argc ){
-      zScript = argv[++ii];
-    }
-
-    else if( stringcompare("-log", argv[ii]) && (ii+1)<argc ){
-      zLog = argv[++ii];
-    }
-
-    else if( stringcompare("-logdata", argv[ii]) ){
-      logdata = 1;
-    }
-
-    else {
-      goto usage;
-    }
-  }
-  if( !zDb || !zScript || !zLog ) goto usage;
-
-  zSql = readScriptFile(zScript, &nSql);
-  if( !zSql ){
-    fprintf(stderr, "Failed to read script file\n");
-    return -1;
-  }
-
-  pInstVfs = sqlite3_instvfs_binarylog("logging", 0, zLog, logdata);
-
-  rc = sqlite3_open_v2(
-     zDb, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, "logging"
-  );
-  if( rc!=SQLITE_OK ){
-    fprintf(stderr, "Failed to open db: %s\n", sqlite3_errmsg(db));
-    return -2;
-  }
-
-  for(i=j=0; j<nSql; j++){
-    if( zSql[j]==';' ){
-      int isComplete;
-      char c = zSql[j+1];
-      zSql[j+1] = 0;
-      isComplete = sqlite3_complete(&zSql[i]);
-      zSql[j+1] = c;
-      if( isComplete ){
-        zSql[j] = 0;
-        while( i<j && ISSPACE(zSql[i]) ){ i++; }
-        if( i<j ){
-          prepareAndRun(pInstVfs, db, &zSql[i]);
-        }
-        zSql[j] = ';';
-        i = j+1;
-      }
-    }
-  }
-  
-  sqlite3_instvfs_destroy(pInstVfs);
-  return 0;
-  
-usage:
-  fprintf(stderr, zUsageMsg, argv[0]);
-  return -3;
-}
diff --git a/tool/spellsift.tcl b/tool/spellsift.tcl
new file mode 100755
index 0000000000..4e67c3e264
--- /dev/null
+++ b/tool/spellsift.tcl
@@ -0,0 +1,74 @@
+#!/usr/bin/tclsh
+
+set usage {
+  Usage: spellsift.tcl <source_filenames>
+  The named .c and .h source files comment blocks are spell-checked.
+}
+
+if {[llength $argv] == 0} {
+  puts stderr $usage
+  exit 0
+}
+
+# Want a Tcl version with 3-argument close.
+package require Tcl 8.6
+
+set ::spellchk "aspell --extra-dicts ./custom.rws list"
+
+# Run text through aspell with custom dictionary, return finds.
+proc misspelled {text} {
+  set spellerr [open "|$::spellchk" r+]
+  puts $spellerr $text
+  flush $spellerr
+  close $spellerr write
+  set huhq [regsub {\s*$} [read $spellerr] {}]
+  close $spellerr read
+  return [split $huhq "\n"]
+}
+
+# Eliminate some common patterns that need not be well spelled.
+proc decruft {text} {
+  set nopp [regsub -all "\n *#\[^\n\]*\n" $text "\n\n" ]
+  set noticket [regsub -all {Ticket \[?[0-9a-f]+\]?} $nopp "" ]
+  return $noticket
+}
+
+# Sift out common variable spellings not in normal dictionaries.
+proc varsift {words} {
+  set rv [list]
+  foreach w $words {
+    set n [string length $w]
+    set cr [string range $w 1 end]
+    if {[string tolower $cr] ne $cr} continue
+    lappend rv $w;
+  }
+  return $rv
+}
+
+foreach fname $argv {
+  set ich [open $fname r]
+  set dtext [decruft [read $ich]]
+  close $ich
+  set cbounds [regexp -indices -inline -all {(/\*)|(\*/)} $dtext]
+  set ccb -1
+  set cblocks [list]
+  foreach {ap cb ce} $cbounds {
+    set cib [lindex $cb 1]
+    set cie [lindex $ce 0]
+    if {$cie != -1} {
+      if {$ccb != -1} {
+        set cce [expr $cie - 1]
+        set destar [string map [list * " "] [string range $dtext $ccb $cce]]
+        lappend cblocks $destar
+        set ccb -1
+      } else continue
+    } elseif {$cib != -1} {
+      set ccb [expr $cib + 1]
+    }
+  }
+  set oddspells [varsift [misspelled [join $cblocks "\n"]]]
+  if {[llength $oddspells] > 0} {
+    puts "!? Misspellings from $fname:"
+    puts [join [lsort -nocase -unique $oddspells] "\n"]
+  }
+}
diff --git a/tool/split-sqlite3c.tcl b/tool/split-sqlite3c.tcl
index 0308431dab..3554933cd9 100644
--- a/tool/split-sqlite3c.tcl
+++ b/tool/split-sqlite3c.tcl
@@ -15,7 +15,7 @@ set END   {^/\*+ End of %s \*+/}
 
 set in [open sqlite3.c]
 set out1 [open sqlite3-all.c w]
-fconfigure $out1 -translation lf
+fconfigure $out1 -translation binary
 
 # Copy the header from sqlite3.c into sqlite3-all.c
 #
@@ -56,7 +56,7 @@ proc write_one_file {content} {
     set label $filecnt
   }
   set out [open sqlite3-$label.c w]
-  fconfigure $out -translation lf
+  fconfigure $out -translation text
   puts -nonewline $out $content
   close $out
   puts $::out1 "#include \"sqlite3-$filecnt.c\""
diff --git a/tool/sqldiff.c b/tool/sqldiff.c
index 0a017037cf..44bf488f86 100644
--- a/tool/sqldiff.c
+++ b/tool/sqldiff.c
@@ -13,7 +13,8 @@
 ** This is a utility program that computes the differences in content
 ** between two SQLite databases.
 **
-** To compile, simply link against SQLite.
+** To compile, simply link against SQLite.  (Windows builds must also link
+** against ext/misc/sqlite3_stdio.c.)
 **
 ** See the showHelp() routine below for a brief description of how to
 ** run the utility.
@@ -25,6 +26,7 @@
 #include <string.h>
 #include <assert.h>
 #include "sqlite3.h"
+#include "sqlite3_stdio.h"
 
 /*
 ** All global variables are gathered into the "g" singleton.
@@ -46,35 +48,25 @@ struct GlobalVars {
 #define DEBUG_DIFF_SQL      0x000002
 
 /*
-** Dynamic string object
+** Clear and free an sqlite3_str object
 */
-typedef struct Str Str;
-struct Str {
-  char *z;        /* Text of the string */
-  int nAlloc;     /* Bytes allocated in z[] */
-  int nUsed;      /* Bytes actually used in z[] */
-};
-
-/*
-** Initialize a Str object
-*/
-static void strInit(Str *p){
-  p->z = 0;
-  p->nAlloc = 0;
-  p->nUsed = 0;
+static void strFree(sqlite3_str *pStr){
+  sqlite3_free(sqlite3_str_finish(pStr));
 }
-  
+
 /*
 ** Print an error resulting from faulting command-line arguments and
 ** abort the program.
 */
 static void cmdlineError(const char *zFormat, ...){
+  sqlite3_str *pOut = sqlite3_str_new(0);
   va_list ap;
-  fprintf(stderr, "%s: ", g.zArgv0);
   va_start(ap, zFormat);
-  vfprintf(stderr, zFormat, ap);
+  sqlite3_str_vappendf(pOut, zFormat, ap);
   va_end(ap);
-  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  sqlite3_fprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut));
+  strFree(pOut);
+  sqlite3_fprintf(stderr, "\"%s --help\" for more help\n", g.zArgv0);
   exit(1);
 }
 
@@ -83,49 +75,16 @@ static void cmdlineError(const char *zFormat, ...){
 ** abort the program.
 */
 static void runtimeError(const char *zFormat, ...){
+  sqlite3_str *pOut = sqlite3_str_new(0);
   va_list ap;
-  fprintf(stderr, "%s: ", g.zArgv0);
   va_start(ap, zFormat);
-  vfprintf(stderr, zFormat, ap);
+  sqlite3_str_vappendf(pOut, zFormat, ap);
   va_end(ap);
-  fprintf(stderr, "\n");
+  sqlite3_fprintf(stderr, "%s: %s\n", g.zArgv0, sqlite3_str_value(pOut));
+  strFree(pOut);
   exit(1);
 }
 
-/*
-** Free all memory held by a Str object
-*/
-static void strFree(Str *p){
-  sqlite3_free(p->z);
-  strInit(p);
-}
-
-/*
-** Add formatted text to the end of a Str object
-*/
-static void strPrintf(Str *p, const char *zFormat, ...){
-  int nNew;
-  for(;;){
-    if( p->z ){
-      va_list ap;
-      va_start(ap, zFormat);
-      sqlite3_vsnprintf(p->nAlloc-p->nUsed, p->z+p->nUsed, zFormat, ap);
-      va_end(ap);
-      nNew = (int)strlen(p->z + p->nUsed);
-    }else{
-      nNew = p->nAlloc;
-    }
-    if( p->nUsed+nNew < p->nAlloc-1 ){
-      p->nUsed += nNew;
-      break;
-    }
-    p->nAlloc = p->nAlloc*2 + 1000;
-    p->z = sqlite3_realloc(p->z, p->nAlloc);
-    if( p->z==0 ) runtimeError("out of memory");
-  }
-}
-
-
 
 /* Safely quote an SQL identifier.  Use the minimum amount of transformation
 ** necessary to allow the string to be used with %s.
@@ -246,7 +205,7 @@ static char **columnNames(
   int naz = 0;             /* Number of entries in az[] */
   sqlite3_stmt *pStmt;     /* SQL statement being run */
   char *zPkIdxName = 0;    /* Name of the PRIMARY KEY index */
-  int truePk = 0;          /* PRAGMA table_info indentifies the PK to use */
+  int truePk = 0;          /* PRAGMA table_info identifies the PK to use */
   int nPK = 0;             /* Number of PRIMARY KEY columns */
   int i, j;                /* Loop counters */
 
@@ -378,11 +337,11 @@ static void printQuoted(FILE *out, sqlite3_value *X){
       char zBuf[50];
       r1 = sqlite3_value_double(X);
       sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1);
-      fprintf(out, "%s", zBuf);
+      sqlite3_fprintf(out, "%s", zBuf);
       break;
     }
     case SQLITE_INTEGER: {
-      fprintf(out, "%lld", sqlite3_value_int64(X));
+      sqlite3_fprintf(out, "%lld", sqlite3_value_int64(X));
       break;
     }
     case SQLITE_BLOB: {
@@ -390,14 +349,14 @@ static void printQuoted(FILE *out, sqlite3_value *X){
       int nBlob = sqlite3_value_bytes(X);
       if( zBlob ){
         int i;
-        fprintf(out, "x'");
+        sqlite3_fprintf(out, "x'");
         for(i=0; i<nBlob; i++){
-          fprintf(out, "%02x", zBlob[i]);
+          sqlite3_fprintf(out, "%02x", zBlob[i]);
         }
-        fprintf(out, "'");
+        sqlite3_fprintf(out, "'");
       }else{
         /* Could be an OOM, could be a zero-byte blob */
-        fprintf(out, "X''");
+        sqlite3_fprintf(out, "X''");
       }
       break;
     }
@@ -405,38 +364,38 @@ static void printQuoted(FILE *out, sqlite3_value *X){
       const unsigned char *zArg = sqlite3_value_text(X);
 
       if( zArg==0 ){
-        fprintf(out, "NULL");
+        sqlite3_fprintf(out, "NULL");
       }else{
         int inctl = 0;
         int i, j;
-        fprintf(out, "'");
+        sqlite3_fprintf(out, "'");
         for(i=j=0; zArg[i]; i++){
           char c = zArg[i];
-          int ctl = iscntrl(c);
+          int ctl = iscntrl((unsigned char)c);
           if( ctl>inctl ){
             inctl = ctl;
-            fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c);
+            sqlite3_fprintf(out, "%.*s'||X'%02x", i-j, &zArg[j], c);
             j = i+1;
           }else if( ctl ){
-            fprintf(out, "%02x", c);
+            sqlite3_fprintf(out, "%02x", c);
             j = i+1;
           }else{
             if( inctl ){
               inctl = 0;
-              fprintf(out, "'\n||'");
+              sqlite3_fprintf(out, "'\n||'");
             }
             if( c=='\'' ){
-              fprintf(out, "%.*s'", i-j+1, &zArg[j]);
+              sqlite3_fprintf(out, "%.*s'", i-j+1, &zArg[j]);
               j = i+1;
             }
           }
         }
-        fprintf(out, "%s'", &zArg[j]);
+        sqlite3_fprintf(out, "%s'", &zArg[j]);
       }
       break;
     }
     case SQLITE_NULL: {
-      fprintf(out, "NULL");
+      sqlite3_fprintf(out, "NULL");
       break;
     }
   }
@@ -453,63 +412,62 @@ static void dump_table(const char *zTab, FILE *out){
   int i;                    /* Loop counter */
   sqlite3_stmt *pStmt;      /* SQL statement */
   const char *zSep;         /* Separator string */
-  Str ins;                  /* Beginning of the INSERT statement */
+  sqlite3_str *pIns;        /* Beginning of the INSERT statement */
 
   pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema WHERE name=%Q", zTab);
   if( SQLITE_ROW==sqlite3_step(pStmt) ){
-    fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
+    sqlite3_fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
   }
   sqlite3_finalize(pStmt);
   if( !g.bSchemaOnly ){
     az = columnNames("aux", zTab, &nPk, 0);
-    strInit(&ins);
+    pIns = sqlite3_str_new(0);
     if( az==0 ){
       pStmt = db_prepare("SELECT * FROM aux.%s", zId);
-      strPrintf(&ins,"INSERT INTO %s VALUES", zId);
+      sqlite3_str_appendf(pIns,"INSERT INTO %s VALUES", zId);
     }else{
-      Str sql;
-      strInit(&sql);
+      sqlite3_str *pSql = sqlite3_str_new(0);
       zSep =  "SELECT";
       for(i=0; az[i]; i++){
-        strPrintf(&sql, "%s %s", zSep, az[i]);
+        sqlite3_str_appendf(pSql, "%s %s", zSep, az[i]);
         zSep = ",";
       }
-      strPrintf(&sql," FROM aux.%s", zId);
+      sqlite3_str_appendf(pSql," FROM aux.%s", zId);
       zSep = " ORDER BY";
       for(i=1; i<=nPk; i++){
-        strPrintf(&sql, "%s %d", zSep, i);
+        sqlite3_str_appendf(pSql, "%s %d", zSep, i);
         zSep = ",";
       }
-      pStmt = db_prepare("%s", sql.z);
-      strFree(&sql);
-      strPrintf(&ins, "INSERT INTO %s", zId);
+      pStmt = db_prepare("%s", sqlite3_str_value(pSql));
+      strFree(pSql);
+      sqlite3_str_appendf(pIns, "INSERT INTO %s", zId);
       zSep = "(";
       for(i=0; az[i]; i++){
-        strPrintf(&ins, "%s%s", zSep, az[i]);
+        sqlite3_str_appendf(pIns, "%s%s", zSep, az[i]);
         zSep = ",";
       }
-      strPrintf(&ins,") VALUES");
+      sqlite3_str_appendf(pIns,") VALUES");
       namelistFree(az);
     }
     nCol = sqlite3_column_count(pStmt);
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
-      fprintf(out, "%s",ins.z);
+      sqlite3_fprintf(out, "%s",sqlite3_str_value(pIns));
       zSep = "(";
       for(i=0; i<nCol; i++){
-        fprintf(out, "%s",zSep);
+        sqlite3_fprintf(out, "%s",zSep);
         printQuoted(out, sqlite3_column_value(pStmt,i));
         zSep = ",";
       }
-      fprintf(out, ");\n");
+      sqlite3_fprintf(out, ");\n");
     }
     sqlite3_finalize(pStmt);
-    strFree(&ins);
+    strFree(pIns);
   } /* endif !g.bSchemaOnly */
   pStmt = db_prepare("SELECT sql FROM aux.sqlite_schema"
                      " WHERE type='index' AND tbl_name=%Q AND sql IS NOT NULL",
                      zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
-    fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
+    sqlite3_fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
   }
   sqlite3_finalize(pStmt);
   sqlite3_free(zId);
@@ -531,12 +489,12 @@ static void diff_one_table(const char *zTab, FILE *out){
   int nQ;                   /* Number of output columns in the diff query */
   int i;                    /* Loop counter */
   const char *zSep;         /* Separator string */
-  Str sql;                  /* Comparison query */
+  sqlite3_str *pSql;        /* Comparison query */
   sqlite3_stmt *pStmt;      /* Query statement to do the diff */
   const char *zLead =       /* Becomes line-comment for sqlite_schema */
     (g.bSchemaCompare)? "-- " : "";
 
-  strInit(&sql);
+  pSql = sqlite3_str_new(0);
   if( g.fDebug==DEBUG_COLUMN_NAMES ){
     /* Simply run columnNames() on all tables of the origin
     ** database and show the results.  This is used for testing
@@ -544,14 +502,14 @@ static void diff_one_table(const char *zTab, FILE *out){
     */
     az = columnNames("aux",zTab, &nPk, 0);
     if( az==0 ){
-      printf("Rowid not accessible for %s\n", zId);
+      sqlite3_fprintf(stdout, "Rowid not accessible for %s\n", zId);
     }else{
-      printf("%s:", zId);
+      sqlite3_fprintf(stdout, "%s:", zId);
       for(i=0; az[i]; i++){
-        printf(" %s", az[i]);
-        if( i+1==nPk ) printf(" *");
+        sqlite3_fprintf(stdout, " %s", az[i]);
+        if( i+1==nPk ) sqlite3_fprintf(stdout, " *");
       }
-      printf("\n");
+      sqlite3_fprintf(stdout, "\n");
     }
     goto end_diff_one_table;
   }
@@ -560,19 +518,20 @@ static void diff_one_table(const char *zTab, FILE *out){
     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
       /* Table missing from second database. */
       if( g.bSchemaCompare )
-        fprintf(out, "-- 2nd DB has no %s table\n", zTab);
+        sqlite3_fprintf(out, "-- 2nd DB has no %s table\n", zTab);
       else
-        fprintf(out, "DROP TABLE %s;\n", zId);
+        sqlite3_fprintf(out, "DROP TABLE %s;\n", zId);
     }
     goto end_diff_one_table;
   }
 
   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
     /* Table missing from source */
-    if( g.bSchemaCompare )
-      fprintf(out, "-- 1st DB has no %s table\n", zTab);
-    else
+    if( g.bSchemaCompare ){
+      sqlite3_fprintf(out, "-- 1st DB has no %s table\n", zTab);
+    }else{
       dump_table(zTab, out);
+    }
     goto end_diff_one_table;
   }
 
@@ -589,99 +548,101 @@ static void diff_one_table(const char *zTab, FILE *out){
    || az[n]
   ){
     /* Schema mismatch */
-    fprintf(out, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead, zId);
+    sqlite3_fprintf(out, "%sDROP TABLE %s; -- due to schema mismatch\n", zLead, zId);
     dump_table(zTab, out);
     goto end_diff_one_table;
   }
 
   /* Build the comparison query */
   for(n2=n; az2[n2]; n2++){
-    fprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, safeId(az2[n2]));
+    char *zNTab = safeId(az2[n2]);
+    sqlite3_fprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, zNTab);
+    sqlite3_free(zNTab);
   }
   nQ = nPk2+1+2*(n2-nPk2);
   if( n2>nPk2 ){
     zSep = "SELECT ";
     for(i=0; i<nPk; i++){
-      strPrintf(&sql, "%sB.%s", zSep, az[i]);
+      sqlite3_str_appendf(pSql, "%sB.%s", zSep, az[i]);
       zSep = ", ";
     }
-    strPrintf(&sql, ", 1%s -- changed row\n", nPk==n ? "" : ",");
+    sqlite3_str_appendf(pSql, ", 1 /* changed row */");
     while( az[i] ){
-      strPrintf(&sql, "       A.%s IS NOT B.%s, B.%s%s\n",
-                az[i], az2[i], az2[i], az2[i+1]==0 ? "" : ",");
+      sqlite3_str_appendf(pSql, ", A.%s IS NOT B.%s, B.%s",
+                az[i], az2[i], az2[i]);
       i++;
     }
     while( az2[i] ){
-      strPrintf(&sql, "       B.%s IS NOT NULL, B.%s%s\n",
-                az2[i], az2[i], az2[i+1]==0 ? "" : ",");
+      sqlite3_str_appendf(pSql, ", B.%s IS NOT NULL, B.%s",
+                az2[i], az2[i]);
       i++;
     }
-    strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
+    sqlite3_str_appendf(pSql, "\n  FROM main.%s A, aux.%s B\n", zId, zId);
     zSep = " WHERE";
     for(i=0; i<nPk; i++){
-      strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
+      sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]);
       zSep = " AND";
     }
     zSep = "\n   AND (";
     while( az[i] ){
-      strPrintf(&sql, "%sA.%s IS NOT B.%s%s\n",
+      sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s%s\n",
                 zSep, az[i], az2[i], az2[i+1]==0 ? ")" : "");
       zSep = "        OR ";
       i++;
     }
     while( az2[i] ){
-      strPrintf(&sql, "%sB.%s IS NOT NULL%s\n",
+      sqlite3_str_appendf(pSql, "%sB.%s IS NOT NULL%s\n",
                 zSep, az2[i], az2[i+1]==0 ? ")" : "");
       zSep = "        OR ";
       i++;
     }
-    strPrintf(&sql, " UNION ALL\n");
+    sqlite3_str_appendf(pSql, " UNION ALL\n");
   }
   zSep = "SELECT ";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%sA.%s", zSep, az[i]);
+    sqlite3_str_appendf(pSql, "%sA.%s", zSep, az[i]);
     zSep = ", ";
   }
-  strPrintf(&sql, ", 2%s -- deleted row\n", nPk==n ? "" : ",");
+  sqlite3_str_appendf(pSql, ", 2 /* deleted row */");
   while( az2[i] ){
-    strPrintf(&sql, "       NULL, NULL%s\n", i==n2-1 ? "" : ",");
+    sqlite3_str_appendf(pSql, ", NULL, NULL");
     i++;
   }
-  strPrintf(&sql, "  FROM main.%s A\n", zId);
-  strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
+  sqlite3_str_appendf(pSql, "\n  FROM main.%s A\n", zId);
+  sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
   zSep =          "                   WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]);
     zSep = " AND";
   }
-  strPrintf(&sql, ")\n");
+  sqlite3_str_appendf(pSql, ")\n");
   zSep = " UNION ALL\nSELECT ";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%sB.%s", zSep, az[i]);
+    sqlite3_str_appendf(pSql, "%sB.%s", zSep, az[i]);
     zSep = ", ";
   }
-  strPrintf(&sql, ", 3%s -- inserted row\n", nPk==n ? "" : ",");
+  sqlite3_str_appendf(pSql, ", 3 /* inserted row */");
   while( az2[i] ){
-    strPrintf(&sql, "       1, B.%s%s\n", az2[i], az2[i+1]==0 ? "" : ",");
+    sqlite3_str_appendf(pSql, ", 1, B.%s", az2[i]);
     i++;
   }
-  strPrintf(&sql, "  FROM aux.%s B\n", zId);
-  strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
+  sqlite3_str_appendf(pSql, "\n  FROM aux.%s B\n", zId);
+  sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
   zSep =          "                   WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]);
     zSep = " AND";
   }
-  strPrintf(&sql, ")\n ORDER BY");
+  sqlite3_str_appendf(pSql, ")\n ORDER BY");
   zSep = " ";
   for(i=1; i<=nPk; i++){
-    strPrintf(&sql, "%s%d", zSep, i);
+    sqlite3_str_appendf(pSql, "%s%d", zSep, i);
     zSep = ", ";
   }
-  strPrintf(&sql, ";\n");
+  sqlite3_str_appendf(pSql, ";\n");
 
   if( g.fDebug & DEBUG_DIFF_SQL ){ 
-    printf("SQL for %s:\n%s\n", zId, sql.z);
+    printf("SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
     goto end_diff_one_table;
   }
 
@@ -696,51 +657,51 @@ static void diff_one_table(const char *zTab, FILE *out){
     zTab, zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     char *z = safeId((const char*)sqlite3_column_text(pStmt,0));
-    fprintf(out, "DROP INDEX %s;\n", z);
+    sqlite3_fprintf(out, "DROP INDEX %s;\n", z);
     sqlite3_free(z);
   }
   sqlite3_finalize(pStmt);
 
   /* Run the query and output differences */
   if( !g.bSchemaOnly ){
-    pStmt = db_prepare("%s", sql.z);
+    pStmt = db_prepare("%s", sqlite3_str_value(pSql));
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       int iType = sqlite3_column_int(pStmt, nPk);
       if( iType==1 || iType==2 ){
         if( iType==1 ){       /* Change the content of a row */
-          fprintf(out, "%sUPDATE %s", zLead, zId);
+          sqlite3_fprintf(out, "%sUPDATE %s", zLead, zId);
           zSep = " SET";
           for(i=nPk+1; i<nQ; i+=2){
             if( sqlite3_column_int(pStmt,i)==0 ) continue;
-            fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]);
+            sqlite3_fprintf(out, "%s %s=", zSep, az2[(i+nPk-1)/2]);
             zSep = ",";
             printQuoted(out, sqlite3_column_value(pStmt,i+1));
           }
         }else{                /* Delete a row */
-          fprintf(out, "%sDELETE FROM %s", zLead, zId);
+          sqlite3_fprintf(out, "%sDELETE FROM %s", zLead, zId);
         }
         zSep = " WHERE";
         for(i=0; i<nPk; i++){
-          fprintf(out, "%s %s=", zSep, az2[i]);
+          sqlite3_fprintf(out, "%s %s=", zSep, az2[i]);
           printQuoted(out, sqlite3_column_value(pStmt,i));
           zSep = " AND";
         }
-        fprintf(out, ";\n");
+        sqlite3_fprintf(out, ";\n");
       }else{                  /* Insert a row */
-        fprintf(out, "%sINSERT INTO %s(%s", zLead, zId, az2[0]);
-        for(i=1; az2[i]; i++) fprintf(out, ",%s", az2[i]);
-        fprintf(out, ") VALUES");
+        sqlite3_fprintf(out, "%sINSERT INTO %s(%s", zLead, zId, az2[0]);
+        for(i=1; az2[i]; i++) sqlite3_fprintf(out, ",%s", az2[i]);
+        sqlite3_fprintf(out, ") VALUES");
         zSep = "(";
         for(i=0; i<nPk2; i++){
-          fprintf(out, "%s", zSep);
+          sqlite3_fprintf(out, "%s", zSep);
           zSep = ",";
           printQuoted(out, sqlite3_column_value(pStmt,i));
         }
         for(i=nPk2+2; i<nQ; i+=2){
-          fprintf(out, ",");
+          sqlite3_fprintf(out, ",");
           printQuoted(out, sqlite3_column_value(pStmt,i));
         }
-        fprintf(out, ");\n");
+        sqlite3_fprintf(out, ");\n");
       }
     }
     sqlite3_finalize(pStmt);
@@ -756,12 +717,12 @@ static void diff_one_table(const char *zTab, FILE *out){
     "                      AND sql IS NOT NULL)",
     zTab, zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
-    fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
+    sqlite3_fprintf(out, "%s;\n", sqlite3_column_text(pStmt,0));
   }
   sqlite3_finalize(pStmt);
 
 end_diff_one_table:
-  strFree(&sql);
+  strFree(pSql);
   sqlite3_free(zId);
   namelistFree(az);
   namelistFree(az2);
@@ -1169,15 +1130,15 @@ static int rbuDeltaCreate(
 **************************************************************************/
 
 static void strPrintfArray(
-  Str *pStr,                      /* String object to append to */
+  sqlite3_str *pStr,              /* String object to append to */
   const char *zSep,               /* Separator string */
   const char *zFmt,               /* Format for each entry */
   char **az, int n                /* Array of strings & its size (or -1) */
 ){
   int i;
   for(i=0; az[i] && (i<n || n<0); i++){
-    if( i!=0 ) strPrintf(pStr, "%s", zSep);
-    strPrintf(pStr, zFmt, az[i], az[i], az[i]);
+    if( i!=0 ) sqlite3_str_appendf(pStr, "%s", zSep);
+    sqlite3_str_appendf(pStr, zFmt, az[i], az[i], az[i]);
   }
 }
 
@@ -1186,74 +1147,75 @@ static void getRbudiffQuery(
   char **azCol,
   int nPK,
   int bOtaRowid,
-  Str *pSql
+  sqlite3_str *pSql
 ){
   int i;
 
   /* First the newly inserted rows: **/ 
-  strPrintf(pSql, "SELECT ");
+  sqlite3_str_appendf(pSql, "SELECT ");
   strPrintfArray(pSql, ", ", "%s", azCol, -1);
-  strPrintf(pSql, ", 0, ");       /* Set ota_control to 0 for an insert */
+  sqlite3_str_appendf(pSql, ", 0, ");  /* Set ota_control to 0 for an insert */
   strPrintfArray(pSql, ", ", "NULL", azCol, -1);
-  strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
-  strPrintf(pSql, "    SELECT 1 FROM ", zTab);
-  strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
+  sqlite3_str_appendf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
+  sqlite3_str_appendf(pSql, "    SELECT 1 FROM ", zTab);
+  sqlite3_str_appendf(pSql, " main.%Q AS o WHERE ", zTab);
   strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n) AND ");
+  sqlite3_str_appendf(pSql, "\n) AND ");
   strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Deleted rows: */
-  strPrintf(pSql, "\nUNION ALL\nSELECT ");
+  sqlite3_str_appendf(pSql, "\nUNION ALL\nSELECT ");
   strPrintfArray(pSql, ", ", "%s", azCol, nPK);
   if( azCol[nPK] ){
-    strPrintf(pSql, ", ");
+    sqlite3_str_appendf(pSql, ", ");
     strPrintfArray(pSql, ", ", "NULL", &azCol[nPK], -1);
   }
-  strPrintf(pSql, ", 1, ");       /* Set ota_control to 1 for a delete */
+  sqlite3_str_appendf(pSql, ", 1, ");   /* Set ota_control to 1 for a delete */
   strPrintfArray(pSql, ", ", "NULL", azCol, -1);
-  strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
-  strPrintf(pSql, "    SELECT 1 FROM ", zTab);
-  strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
+  sqlite3_str_appendf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
+  sqlite3_str_appendf(pSql, "    SELECT 1 FROM ", zTab);
+  sqlite3_str_appendf(pSql, " aux.%Q AS o WHERE ", zTab);
   strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n) AND ");
+  sqlite3_str_appendf(pSql, "\n) AND ");
   strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Updated rows. If all table columns are part of the primary key, there 
   ** can be no updates. In this case this part of the compound SELECT can
   ** be omitted altogether. */
   if( azCol[nPK] ){
-    strPrintf(pSql, "\nUNION ALL\nSELECT ");
+    sqlite3_str_appendf(pSql, "\nUNION ALL\nSELECT ");
     strPrintfArray(pSql, ", ", "n.%s", azCol, nPK);
-    strPrintf(pSql, ",\n");
+    sqlite3_str_appendf(pSql, ",\n");
     strPrintfArray(pSql, " ,\n", 
         "    CASE WHEN n.%s IS o.%s THEN NULL ELSE n.%s END", &azCol[nPK], -1
     );
 
     if( bOtaRowid==0 ){
-      strPrintf(pSql, ", '");
+      sqlite3_str_appendf(pSql, ", '");
       strPrintfArray(pSql, "", ".", azCol, nPK);
-      strPrintf(pSql, "' ||\n");
+      sqlite3_str_appendf(pSql, "' ||\n");
     }else{
-      strPrintf(pSql, ",\n");
+      sqlite3_str_appendf(pSql, ",\n");
     }
     strPrintfArray(pSql, " ||\n", 
         "    CASE WHEN n.%s IS o.%s THEN '.' ELSE 'x' END", &azCol[nPK], -1
     );
-    strPrintf(pSql, "\nAS ota_control, ");
+    sqlite3_str_appendf(pSql, "\nAS ota_control, ");
     strPrintfArray(pSql, ", ", "NULL", azCol, nPK);
-    strPrintf(pSql, ",\n");
+    sqlite3_str_appendf(pSql, ",\n");
     strPrintfArray(pSql, " ,\n", 
         "    CASE WHEN n.%s IS o.%s THEN NULL ELSE o.%s END", &azCol[nPK], -1
     );
 
-    strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
+    sqlite3_str_appendf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ",
+                        zTab, zTab);
     strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
-    strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
+    sqlite3_str_appendf(pSql, " AND ota_control LIKE '%%x%%'");
   }
 
   /* Now add an ORDER BY clause to sort everything by PK. */
-  strPrintf(pSql, "\nORDER BY ");
-  for(i=1; i<=nPK; i++) strPrintf(pSql, "%s%d", ((i>1)?", ":""), i);
+  sqlite3_str_appendf(pSql, "\nORDER BY ");
+  for(i=1; i<=nPK; i++) sqlite3_str_appendf(pSql, "%s%d", ((i>1)?", ":""), i);
 }
 
 static void rbudiff_one_table(const char *zTab, FILE *out){
@@ -1262,14 +1224,17 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
   char **azCol;                   /* NULL terminated array of col names */
   int i;
   int nCol;
-  Str ct = {0, 0, 0};             /* The "CREATE TABLE data_xxx" statement */
-  Str sql = {0, 0, 0};            /* Query to find differences */
-  Str insert = {0, 0, 0};         /* First part of output INSERT statement */
+  sqlite3_str *pCt;               /* The "CREATE TABLE data_xxx" statement */
+  sqlite3_str *pSql;              /* Query to find differences */
+  sqlite3_str *pInsert;           /* First part of output INSERT statement */
   sqlite3_stmt *pStmt = 0;
   int nRow = 0;                   /* Total rows in data_xxx table */
 
   /* --rbu mode must use real primary keys. */
   g.bSchemaPK = 1;
+  pCt = sqlite3_str_new(0);
+  pSql = sqlite3_str_new(0);
+  pInsert = sqlite3_str_new(0);
 
   /* Check that the schemas of the two tables match. Exit early otherwise. */
   checkSchemasMatch(zTab);
@@ -1283,40 +1248,40 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
   for(nCol=0; azCol[nCol]; nCol++);
 
   /* Build and output the CREATE TABLE statement for the data_xxx table */
-  strPrintf(&ct, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
-  if( bOtaRowid ) strPrintf(&ct, "rbu_rowid, ");
-  strPrintfArray(&ct, ", ", "%s", &azCol[bOtaRowid], -1);
-  strPrintf(&ct, ", rbu_control);");
+  sqlite3_str_appendf(pCt, "CREATE TABLE IF NOT EXISTS 'data_%q'(", zTab);
+  if( bOtaRowid ) sqlite3_str_appendf(pCt, "rbu_rowid, ");
+  strPrintfArray(pCt, ", ", "%s", &azCol[bOtaRowid], -1);
+  sqlite3_str_appendf(pCt, ", rbu_control);");
 
   /* Get the SQL for the query to retrieve data from the two databases */
-  getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, &sql);
+  getRbudiffQuery(zTab, azCol, nPK, bOtaRowid, pSql);
 
   /* Build the first part of the INSERT statement output for each row
   ** in the data_xxx table. */
-  strPrintf(&insert, "INSERT INTO 'data_%q' (", zTab);
-  if( bOtaRowid ) strPrintf(&insert, "rbu_rowid, ");
-  strPrintfArray(&insert, ", ", "%s", &azCol[bOtaRowid], -1);
-  strPrintf(&insert, ", rbu_control) VALUES(");
+  sqlite3_str_appendf(pInsert, "INSERT INTO 'data_%q' (", zTab);
+  if( bOtaRowid ) sqlite3_str_appendf(pInsert, "rbu_rowid, ");
+  strPrintfArray(pInsert, ", ", "%s", &azCol[bOtaRowid], -1);
+  sqlite3_str_appendf(pInsert, ", rbu_control) VALUES(");
 
-  pStmt = db_prepare("%s", sql.z);
+  pStmt = db_prepare("%s", sqlite3_str_value(pSql));
 
   while( sqlite3_step(pStmt)==SQLITE_ROW ){
     
     /* If this is the first row output, print out the CREATE TABLE 
-    ** statement first. And then set ct.z to NULL so that it is not 
+    ** statement first. And reset pCt so that it will not be
     ** printed again.  */
-    if( ct.z ){
-      fprintf(out, "%s\n", ct.z);
-      strFree(&ct);
+    if( sqlite3_str_length(pCt) ){
+      sqlite3_fprintf(out, "%s\n", sqlite3_str_value(pCt));
+      sqlite3_str_reset(pCt);
     }
 
     /* Output the first part of the INSERT statement */
-    fprintf(out, "%s", insert.z);
+    sqlite3_fprintf(out, "%s", sqlite3_str_value(pInsert));
     nRow++;
 
     if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
       for(i=0; i<=nCol; i++){
-        if( i>0 ) fprintf(out, ", ");
+        if( i>0 ) sqlite3_fprintf(out, ", ");
         printQuoted(out, sqlite3_column_value(pStmt, i));
       }
     }else{
@@ -1343,9 +1308,9 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
           nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta);
           if( nDelta<nFinal ){
             int j;
-            fprintf(out, "x'");
-            for(j=0; j<nDelta; j++) fprintf(out, "%02x", (u8)aDelta[j]);
-            fprintf(out, "'");
+            sqlite3_fprintf(out, "x'");
+            for(j=0; j<nDelta; j++) sqlite3_fprintf(out, "%02x", (u8)aDelta[j]);
+            sqlite3_fprintf(out, "'");
             zOtaControl[i-bOtaRowid] = 'f';
             bDone = 1;
           }
@@ -1355,27 +1320,28 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
         if( bDone==0 ){
           printQuoted(out, sqlite3_column_value(pStmt, i));
         }
-        fprintf(out, ", ");
+        sqlite3_fprintf(out, ", ");
       }
-      fprintf(out, "'%s'", zOtaControl);
+      sqlite3_fprintf(out, "'%s'", zOtaControl);
       sqlite3_free(zOtaControl);
     }
 
     /* And the closing bracket of the insert statement */
-    fprintf(out, ");\n");
+    sqlite3_fprintf(out, ");\n");
   }
 
   sqlite3_finalize(pStmt);
   if( nRow>0 ){
-    Str cnt = {0, 0, 0};
-    strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
-    fprintf(out, "%s\n", cnt.z);
-    strFree(&cnt);
+    sqlite3_str *pCnt = sqlite3_str_new(0);
+    sqlite3_str_appendf(pCnt,
+         "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
+    sqlite3_fprintf(out, "%s\n", sqlite3_str_value(pCnt));
+    strFree(pCnt);
   }
 
-  strFree(&ct);
-  strFree(&sql);
-  strFree(&insert);
+  strFree(pCt);
+  strFree(pSql);
+  strFree(pInsert);
 }
 
 /*
@@ -1397,25 +1363,25 @@ static void summarize_one_table(const char *zTab, FILE *out){
   int n2;                   /* Number of columns in aux */
   int i;                    /* Loop counter */
   const char *zSep;         /* Separator string */
-  Str sql;                  /* Comparison query */
+  sqlite3_str *pSql;        /* Comparison query */
   sqlite3_stmt *pStmt;      /* Query statement to do the diff */
   sqlite3_int64 nUpdate;    /* Number of updated rows */
   sqlite3_int64 nUnchanged; /* Number of unmodified rows */
   sqlite3_int64 nDelete;    /* Number of deleted rows */
   sqlite3_int64 nInsert;    /* Number of inserted rows */
 
-  strInit(&sql);
+  pSql = sqlite3_str_new(0);
   if( sqlite3_table_column_metadata(g.db,"aux",zTab,0,0,0,0,0,0) ){
     if( !sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
       /* Table missing from second database. */
-      fprintf(out, "%s: missing from second database\n", zTab);
+      sqlite3_fprintf(out, "%s: missing from second database\n", zTab);
     }
     goto end_summarize_one_table;
   }
 
   if( sqlite3_table_column_metadata(g.db,"main",zTab,0,0,0,0,0,0) ){
     /* Table missing from source */
-    fprintf(out, "%s: missing from first database\n", zTab);
+    sqlite3_fprintf(out, "%s: missing from first database\n", zTab);
     goto end_summarize_one_table;
   }
 
@@ -1432,57 +1398,57 @@ static void summarize_one_table(const char *zTab, FILE *out){
    || az[n]
   ){
     /* Schema mismatch */
-    fprintf(out, "%s: incompatible schema\n", zTab);
+    sqlite3_fprintf(out, "%s: incompatible schema\n", zTab);
     goto end_summarize_one_table;
   }
 
   /* Build the comparison query */
   for(n2=n; az[n2]; n2++){}
-  strPrintf(&sql, "SELECT 1, count(*)");
+  sqlite3_str_appendf(pSql, "SELECT 1, count(*)");
   if( n2==nPk2 ){
-    strPrintf(&sql, ", 0\n");
+    sqlite3_str_appendf(pSql, ", 0\n");
   }else{
     zSep = ", sum(";
     for(i=nPk; az[i]; i++){
-      strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]);
+      sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s", zSep, az[i], az[i]);
       zSep = " OR ";
     }
-    strPrintf(&sql, ")\n");
+    sqlite3_str_appendf(pSql, ")\n");
   }
-  strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
+  sqlite3_str_appendf(pSql, "  FROM main.%s A, aux.%s B\n", zId, zId);
   zSep = " WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]);
     zSep = " AND";
   }
-  strPrintf(&sql, " UNION ALL\n");
-  strPrintf(&sql, "SELECT 2, count(*), 0\n");
-  strPrintf(&sql, "  FROM main.%s A\n", zId);
-  strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId);
+  sqlite3_str_appendf(pSql, " UNION ALL\n");
+  sqlite3_str_appendf(pSql, "SELECT 2, count(*), 0\n");
+  sqlite3_str_appendf(pSql, "  FROM main.%s A\n", zId);
+  sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B ", zId);
   zSep = "WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]);
     zSep = " AND";
   }
-  strPrintf(&sql, ")\n");
-  strPrintf(&sql, " UNION ALL\n");
-  strPrintf(&sql, "SELECT 3, count(*), 0\n");
-  strPrintf(&sql, "  FROM aux.%s B\n", zId);
-  strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId);
+  sqlite3_str_appendf(pSql, ")\n");
+  sqlite3_str_appendf(pSql, " UNION ALL\n");
+  sqlite3_str_appendf(pSql, "SELECT 3, count(*), 0\n");
+  sqlite3_str_appendf(pSql, "  FROM aux.%s B\n", zId);
+  sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A ", zId);
   zSep = "WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, az[i], az[i]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s", zSep, az[i], az[i]);
     zSep = " AND";
   }
-  strPrintf(&sql, ")\n ORDER BY 1;\n");
+  sqlite3_str_appendf(pSql, ")\n ORDER BY 1;\n");
 
   if( (g.fDebug & DEBUG_DIFF_SQL)!=0 ){ 
-    printf("SQL for %s:\n%s\n", zId, sql.z);
+    sqlite3_fprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
     goto end_summarize_one_table;
   }
 
   /* Run the query and output difference summary */
-  pStmt = db_prepare("%s", sql.z);
+  pStmt = db_prepare("%s", sqlite3_str_value(pSql));
   nUpdate = 0;
   nInsert = 0;
   nDelete = 0;
@@ -1502,11 +1468,12 @@ static void summarize_one_table(const char *zTab, FILE *out){
     }
   }
   sqlite3_finalize(pStmt);
-  fprintf(out, "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
+  sqlite3_fprintf(out,
+          "%s: %lld changes, %lld inserts, %lld deletes, %lld unchanged\n",
           zTab, nUpdate, nInsert, nDelete, nUnchanged);
 
 end_summarize_one_table:
-  strFree(&sql);
+  strFree(pSql);
   sqlite3_free(zId);
   namelistFree(az);
   namelistFree(az2);
@@ -1586,13 +1553,13 @@ static void changeset_one_table(const char *zTab, FILE *out){
   int *aiFlg = 0;               /* 0 if column is not part of PK */
   int *aiPk = 0;                /* Column numbers for each PK column */
   int nPk = 0;                  /* Number of PRIMARY KEY columns */
-  Str sql;                      /* SQL for the diff query */
+  sqlite3_str *pSql;            /* SQL for the diff query */
   int i, k;                     /* Loop counters */
   const char *zSep;             /* List separator */
 
   /* Check that the schemas of the two tables match. Exit early otherwise. */
   checkSchemasMatch(zTab);
-  strInit(&sql);
+  pSql = sqlite3_str_new(0);
 
   pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
@@ -1615,71 +1582,74 @@ static void changeset_one_table(const char *zTab, FILE *out){
   sqlite3_finalize(pStmt);
   if( nPk==0 ) goto end_changeset_one_table; 
   if( nCol>nPk ){
-    strPrintf(&sql, "SELECT %d", SQLITE_UPDATE);
+    sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_UPDATE);
     for(i=0; i<nCol; i++){
       if( aiFlg[i] ){
-        strPrintf(&sql, ",\n       A.%s", azCol[i]);
+        sqlite3_str_appendf(pSql, ",\n       A.%s", azCol[i]);
       }else{
-        strPrintf(&sql, ",\n       A.%s IS NOT B.%s, A.%s, B.%s",
+        sqlite3_str_appendf(pSql, ",\n       A.%s IS NOT B.%s, A.%s, B.%s",
                   azCol[i], azCol[i], azCol[i], azCol[i]);
       }
     }
-    strPrintf(&sql,"\n  FROM main.%s A, aux.%s B\n", zId, zId);
+    sqlite3_str_appendf(pSql,"\n  FROM main.%s A, aux.%s B\n", zId, zId);
     zSep = " WHERE";
     for(i=0; i<nPk; i++){
-      strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
+      sqlite3_str_appendf(pSql, "%s A.%s=B.%s",
+                          zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
       zSep = " AND";
     }
     zSep = "\n   AND (";
     for(i=0; i<nCol; i++){
       if( aiFlg[i] ) continue;
-      strPrintf(&sql, "%sA.%s IS NOT B.%s", zSep, azCol[i], azCol[i]);
+      sqlite3_str_appendf(pSql, "%sA.%s IS NOT B.%s", zSep, azCol[i], azCol[i]);
       zSep = " OR\n        ";
     }
-    strPrintf(&sql,")\n UNION ALL\n");
+    sqlite3_str_appendf(pSql,")\n UNION ALL\n");
   }
-  strPrintf(&sql, "SELECT %d", SQLITE_DELETE);
+  sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_DELETE);
   for(i=0; i<nCol; i++){
     if( aiFlg[i] ){
-      strPrintf(&sql, ",\n       A.%s", azCol[i]);
+      sqlite3_str_appendf(pSql, ",\n       A.%s", azCol[i]);
     }else{
-      strPrintf(&sql, ",\n       1, A.%s, NULL", azCol[i]);
+      sqlite3_str_appendf(pSql, ",\n       1, A.%s, NULL", azCol[i]);
     }
   }
-  strPrintf(&sql, "\n  FROM main.%s A\n", zId);
-  strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
+  sqlite3_str_appendf(pSql, "\n  FROM main.%s A\n", zId);
+  sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM aux.%s B\n", zId);
   zSep =          "                   WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s",
+                        zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
     zSep = " AND";
   }
-  strPrintf(&sql, ")\n UNION ALL\n");
-  strPrintf(&sql, "SELECT %d", SQLITE_INSERT);
+  sqlite3_str_appendf(pSql, ")\n UNION ALL\n");
+  sqlite3_str_appendf(pSql, "SELECT %d", SQLITE_INSERT);
   for(i=0; i<nCol; i++){
     if( aiFlg[i] ){
-      strPrintf(&sql, ",\n       B.%s", azCol[i]);
+      sqlite3_str_appendf(pSql, ",\n       B.%s", azCol[i]);
     }else{
-      strPrintf(&sql, ",\n       1, NULL, B.%s", azCol[i]);
+      sqlite3_str_appendf(pSql, ",\n       1, NULL, B.%s", azCol[i]);
     }
   }
-  strPrintf(&sql, "\n  FROM aux.%s B\n", zId);
-  strPrintf(&sql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
+  sqlite3_str_appendf(pSql, "\n  FROM aux.%s B\n", zId);
+  sqlite3_str_appendf(pSql, " WHERE NOT EXISTS(SELECT 1 FROM main.%s A\n", zId);
   zSep =          "                   WHERE";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s A.%s=B.%s", zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
+    sqlite3_str_appendf(pSql, "%s A.%s=B.%s",
+                        zSep, azCol[aiPk[i]], azCol[aiPk[i]]);
     zSep = " AND";
   }
-  strPrintf(&sql, ")\n");
-  strPrintf(&sql, " ORDER BY");
+  sqlite3_str_appendf(pSql, ")\n");
+  sqlite3_str_appendf(pSql, " ORDER BY");
   zSep = " ";
   for(i=0; i<nPk; i++){
-    strPrintf(&sql, "%s %d", zSep, aiPk[i]+2);
+    sqlite3_str_appendf(pSql, "%s %d", zSep, aiPk[i]+2);
     zSep = ",";
   }
-  strPrintf(&sql, ";\n");
+  sqlite3_str_appendf(pSql, ";\n");
 
   if( g.fDebug & DEBUG_DIFF_SQL ){ 
-    printf("SQL for %s:\n%s\n", zId, sql.z);
+    sqlite3_fprintf(stdout, "SQL for %s:\n%s\n", zId, sqlite3_str_value(pSql));
     goto end_changeset_one_table;
   }
 
@@ -1689,7 +1659,7 @@ static void changeset_one_table(const char *zTab, FILE *out){
   fwrite(zTab, 1, strlen(zTab), out);
   putc(0, out);
 
-  pStmt = db_prepare("%s", sql.z);
+  pStmt = db_prepare("%s", sqlite3_str_value(pSql));
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     int iType = sqlite3_column_int(pStmt,0);
     putc(iType, out);
@@ -1756,7 +1726,7 @@ static void changeset_one_table(const char *zTab, FILE *out){
   sqlite3_free(aiPk);
   sqlite3_free(zId);
   sqlite3_free(aiFlg);
-  strFree(&sql);
+  strFree(pSql);
 }
 
 /*
@@ -1909,8 +1879,8 @@ const char *all_tables_sql(){
 ** Print sketchy documentation for this utility program
 */
 static void showHelp(void){
-  printf("Usage: %s [options] DB1 DB2\n", g.zArgv0);
-  printf(
+  sqlite3_fprintf(stdout, "Usage: %s [options] DB1 DB2\n", g.zArgv0);
+  sqlite3_fprintf(stdout,
 "Output SQL text that would transform DB1 into DB2.\n"
 "Options:\n"
 "  --changeset FILE      Write a CHANGESET into FILE\n"
@@ -1953,7 +1923,7 @@ int main(int argc, char **argv){
       if( z[0]=='-' ) z++;
       if( strcmp(z,"changeset")==0 ){
         if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
-        out = fopen(argv[++i], "wb");
+        out = sqlite3_fopen(argv[++i], "wb");
         if( out==0 ) cmdlineError("cannot open: %s", argv[i]);
         xDiff = changeset_one_table;
         neverUseTransaction = 1;
@@ -2016,7 +1986,7 @@ int main(int argc, char **argv){
   if( g.bSchemaOnly && g.bSchemaCompare ){
     cmdlineError("The --schema option is useless with --table %s .", zTab);
   }
-  rc = sqlite3_open(zDb1, &g.db);
+  rc = sqlite3_open_v2(zDb1, &g.db, SQLITE_OPEN_READONLY, 0);
   if( rc ){
     cmdlineError("cannot open database file \"%s\"", zDb1);
   }
@@ -2024,6 +1994,13 @@ int main(int argc, char **argv){
   if( rc || zErrMsg ){
     cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb1);
   }
+  {
+    sqlite3 *db2 = 0;
+    if( sqlite3_open_v2(zDb2, &db2, SQLITE_OPEN_READONLY, 0) ){
+      cmdlineError("cannot open database file \"%s\"", zDb2);
+    }
+    sqlite3_close(db2);
+  }
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
   sqlite3_enable_load_extension(g.db, 1);
   for(i=0; i<nExt; i++){
@@ -2047,9 +2024,9 @@ int main(int argc, char **argv){
   }
 
   if( neverUseTransaction ) useTransaction = 0;
-  if( useTransaction ) fprintf(out, "BEGIN TRANSACTION;\n");
+  if( useTransaction ) sqlite3_fprintf(out, "BEGIN TRANSACTION;\n");
   if( xDiff==rbudiff_one_table ){
-    fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
+    sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count"
            "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) "
            "WITHOUT ROWID;\n"
     );
@@ -2064,7 +2041,7 @@ int main(int argc, char **argv){
     }
     sqlite3_finalize(pStmt);
   }
-  if( useTransaction ) printf("COMMIT;\n");
+  if( useTransaction ) sqlite3_fprintf(stdout,"COMMIT;\n");
 
   /* TBD: Handle trigger differences */
   /* TBD: Handle view differences */
diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in
index d8b000209f..9860c0b0fc 100644
--- a/tool/sqlite3_analyzer.c.in
+++ b/tool/sqlite3_analyzer.c.in
@@ -3,6 +3,8 @@
 ** text on standard output.
 */
 #define TCLSH_INIT_PROC sqlite3_analyzer_init_proc
+IFDEF INCLUDE_SQLITE3_C
+#undef SQLITE_ENABLE_DBSTAT_VTAB
 #define SQLITE_ENABLE_DBSTAT_VTAB 1
 #undef SQLITE_THREADSAFE
 #define SQLITE_THREADSAFE 0
@@ -14,13 +16,62 @@
 #define SQLITE_DEFAULT_MEMSTATUS 0
 #define SQLITE_MAX_EXPR_DEPTH 0
 #define SQLITE_OMIT_LOAD_EXTENSION 1
-#if !defined(SQLITE_AMALGAMATION) && !defined(USE_EXTERNAL_SQLITE)
 INCLUDE sqlite3.c
-#endif
+ELSE
+#include "sqlite3.h"
+ENDIF
 INCLUDE $ROOT/src/tclsqlite.c
 
+#if defined(_WIN32)
+INCLUDE $ROOT/ext/misc/sqlite3_stdio.h
+INCLUDE $ROOT/ext/misc/sqlite3_stdio.c
+
+/* Substitute "puts" command.  Only these forms recognized:
+**
+**      puts STRING
+**      puts stderr STRING
+**      puts -nonewline STRING
+*/
+static int subst_puts(
+  void *NotUsed,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *const*objv
+){
+  FILE *pOut = stdout;
+  const char *zOut;
+  int addNewLine = 1;
+  if( objc==2 ){
+    zOut = Tcl_GetString(objv[1]);
+  }else if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "?stderr|-nonewline? STRING");
+    return TCL_ERROR;
+  }else{
+    const char *zArg = Tcl_GetString(objv[1]);
+    if( zArg==0 ) return TCL_ERROR;
+    zOut = Tcl_GetString(objv[2]);
+    if( strcmp(zArg, "stderr")==0 ){
+      pOut = stderr;
+    }else if( strcmp(zArg, "-nonewline")==0 ){
+      addNewLine = 0;
+    }else{
+      Tcl_AppendResult(interp, "bad argument: ", zArg, NULL);
+      return TCL_ERROR;
+    }
+  }
+  sqlite3_fputs(zOut, pOut);
+  if( addNewLine ) sqlite3_fputs("\n", pOut);
+  fflush(pOut);
+  return TCL_OK;
+}
+#endif /* defined(_WIN32) */
+
 const char *sqlite3_analyzer_init_proc(Tcl_Interp *interp){
+#if defined(_WIN32)
+  Tcl_CreateObjCommand(interp, "puts", subst_puts, 0, 0);
+#else  
   (void)interp;
+#endif
   return
 BEGIN_STRING
 INCLUDE $ROOT/tool/spaceanal.tcl
diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c
new file mode 100644
index 0000000000..ad9f132bb4
--- /dev/null
+++ b/tool/sqlite3_rsync.c
@@ -0,0 +1,2397 @@
+/*
+** 2024-09-10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that makes a copy of a live SQLite database
+** using a bandwidth-efficient protocol, similar to "rsync".
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdarg.h>
+#include "sqlite3.h"
+
+static const char zUsage[] =
+  "sqlite3_rsync ORIGIN REPLICA ?OPTIONS?\n"
+  "\n"
+  "One of ORIGIN or REPLICA is a pathname to a database on the local\n"
+  "machine and the other is of the form \"USER@HOST:PATH\" describing\n"
+  "a database on a remote machine.  This utility makes REPLICA into a\n"
+  "copy of ORIGIN\n"
+  "\n"
+  "OPTIONS:\n"
+  "\n"
+  "   --exe PATH      Name of the sqlite3_rsync program on the remote side\n"
+  "   --help          Show this help screen\n"
+  "   --protocol N    Use sync protocol version N.\n"
+  "   --ssh PATH      Name of the SSH program used to reach the remote side\n"
+  "   -v              Verbose.  Multiple v's for increasing output\n"
+  "   --version       Show detailed version information\n"
+  "   --wal-only      Do not sync unless both databases are in WAL mode\n"
+;
+
+typedef unsigned char u8;
+typedef sqlite3_uint64 u64;
+
+/* Context for the run */
+typedef struct SQLiteRsync SQLiteRsync;
+struct SQLiteRsync {
+  const char *zOrigin;     /* Name of the origin */
+  const char *zReplica;    /* Name of the replica */
+  const char *zErrFile;    /* Append error messages to this file */
+  const char *zDebugFile;  /* Append debugging messages to this file */
+  FILE *pOut;              /* Transmit to the other side */
+  FILE *pIn;               /* Receive from the other side */
+  FILE *pLog;              /* Duplicate output here if not NULL */
+  FILE *pDebug;            /* Write debug info here if not NULL */
+  sqlite3 *db;             /* Database connection */
+  int nErr;                /* Number of errors encountered */
+  int nWrErr;              /* Number of failed attempts to write on the pipe */
+  u8 eVerbose;             /* Bigger for more output.  0 means none. */
+  u8 bCommCheck;           /* True to debug the communication protocol */
+  u8 isRemote;             /* On the remote side of a connection */
+  u8 isReplica;            /* True if running on the replica side */
+  u8 iProtocol;            /* Protocol version number */
+  u8 wrongEncoding;        /* ATTACH failed due to wrong encoding */
+  u8 bWalOnly;             /* Require WAL mode */
+  sqlite3_uint64 nOut;     /* Bytes transmitted */
+  sqlite3_uint64 nIn;      /* Bytes received */
+  unsigned int nPage;      /* Total number of pages in the database */
+  unsigned int szPage;     /* Database page size */
+  u64 nHashSent;           /* Hashes sent (replica to origin) */
+  unsigned int nRound;     /* Number of hash batches (replica to origin) */
+  unsigned int nPageSent;  /* Page contents sent (origin to replica) */
+};
+
+/* The version number of the protocol.  Sent in the *_BEGIN message
+** to verify that both sides speak the same dialect.
+*/
+#define PROTOCOL_VERSION  2
+
+
+/* Magic numbers to identify particular messages sent over the wire.
+*/
+/**** Baseline: protocol version 1 ****/
+#define ORIGIN_BEGIN    0x41     /* Initial message */
+#define ORIGIN_END      0x42     /* Time to quit */
+#define ORIGIN_ERROR    0x43     /* Error message from the remote */
+#define ORIGIN_PAGE     0x44     /* New page data */
+#define ORIGIN_TXN      0x45     /* Transaction commit */
+#define ORIGIN_MSG      0x46     /* Informational message */
+/**** Added in protocol version 2 ****/
+#define ORIGIN_DETAIL   0x47     /* Request finer-grain hash info */
+#define ORIGIN_READY    0x48     /* Ready for next round of hash exchanges */
+
+/**** Baseline: protocol version 1 ****/
+#define REPLICA_BEGIN   0x61     /* Welcome message */
+#define REPLICA_ERROR   0x62     /* Error.  Report and quit. */
+#define REPLICA_END     0x63     /* Replica wants to stop */
+#define REPLICA_HASH    0x64     /* One or more pages hashes to report */
+#define REPLICA_READY   0x65     /* Read to receive page content */
+#define REPLICA_MSG     0x66     /* Informational message */
+/**** Added in protocol version 2 ****/
+#define REPLICA_CONFIG  0x67     /* Hash exchange configuration */
+
+/****************************************************************************
+** Beginning of the popen2() implementation copied from Fossil  *************
+****************************************************************************/
+#ifdef _WIN32
+#include <windows.h>
+#include <io.h>
+#include <fcntl.h>
+/*
+** Print a fatal error and quit.
+*/
+static void win32_fatal_error(const char *zMsg){
+  fprintf(stderr, "%s", zMsg);
+  exit(1);
+}
+extern int _open_osfhandle(intptr_t,int);
+#else
+#include <unistd.h>
+#include <signal.h>
+#include <sys/wait.h>
+#endif
+
+/*
+** The following macros are used to cast pointers to integers and
+** integers to pointers.  The way you do this varies from one compiler
+** to the next, so we have developed the following set of #if statements
+** to generate appropriate macros for a wide range of compilers.
+**
+** The correct "ANSI" way to do this is to use the intptr_t type.
+** Unfortunately, that typedef is not available on all compilers, or
+** if it is available, it requires an #include of specific headers
+** that vary from one machine to the next.
+**
+** This code is copied out of SQLite.
+*/
+#if defined(__PTRDIFF_TYPE__)  /* This case should work for GCC */
+# define INT_TO_PTR(X)  ((void*)(__PTRDIFF_TYPE__)(X))
+# define PTR_TO_INT(X)  ((int)(__PTRDIFF_TYPE__)(X))
+#elif !defined(__GNUC__)       /* Works for compilers other than LLVM */
+# define INT_TO_PTR(X)  ((void*)&((char*)0)[X])
+# define PTR_TO_INT(X)  ((int)(((char*)X)-(char*)0))
+#elif defined(HAVE_STDINT_H)   /* Use this case if we have ANSI headers */
+# define INT_TO_PTR(X)  ((void*)(intptr_t)(X))
+# define PTR_TO_INT(X)  ((int)(intptr_t)(X))
+#else                          /* Generates a warning - but it always works */
+# define INT_TO_PTR(X)  ((void*)(X))
+# define PTR_TO_INT(X)  ((int)(X))
+#endif
+
+/* Register SQL functions provided by ext/misc/sha1.c */
+extern int sqlite3_sha_init(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pApi
+);
+
+#ifdef _WIN32
+/*
+** On windows, create a child process and specify the stdin, stdout,
+** and stderr channels for that process to use.
+**
+** Return the number of errors.
+*/
+static int win32_create_child_process(
+  wchar_t *zCmd,       /* The command that the child process will run */
+  HANDLE hIn,          /* Standard input */
+  HANDLE hOut,         /* Standard output */
+  HANDLE hErr,         /* Standard error */
+  DWORD *pChildPid     /* OUT: Child process handle */
+){
+  STARTUPINFOW si;
+  PROCESS_INFORMATION pi;
+  BOOL rc;
+
+  memset(&si, 0, sizeof(si));
+  si.cb = sizeof(si);
+  si.dwFlags = STARTF_USESTDHANDLES;
+  SetHandleInformation(hIn, HANDLE_FLAG_INHERIT, TRUE);
+  si.hStdInput  = hIn;
+  SetHandleInformation(hOut, HANDLE_FLAG_INHERIT, TRUE);
+  si.hStdOutput = hOut;
+  SetHandleInformation(hErr, HANDLE_FLAG_INHERIT, TRUE);
+  si.hStdError  = hErr;
+  rc = CreateProcessW(
+     NULL,  /* Application Name */
+     zCmd,  /* Command-line */
+     NULL,  /* Process attributes */
+     NULL,  /* Thread attributes */
+     TRUE,  /* Inherit Handles */
+     0,     /* Create flags  */
+     NULL,  /* Environment */
+     NULL,  /* Current directory */
+     &si,   /* Startup Info */
+     &pi    /* Process Info */
+  );
+  if( rc ){
+    CloseHandle( pi.hProcess );
+    CloseHandle( pi.hThread );
+    *pChildPid = pi.dwProcessId;
+  }else{
+    win32_fatal_error("cannot create child process");
+  }
+  return rc!=0;
+}
+void *win32_utf8_to_unicode(const char *zUtf8){
+  int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
+  wchar_t *zUnicode = malloc( nByte*2 );
+  MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
+  return zUnicode;
+}
+#endif
+
+/*
+** Create a child process running shell command "zCmd".  *ppOut is
+** a FILE that becomes the standard input of the child process.
+** (The caller writes to *ppOut in order to send text to the child.)
+** *ppIn is stdout from the child process.  (The caller
+** reads from *ppIn in order to receive input from the child.)
+** Note that *ppIn is an unbuffered file descriptor, not a FILE.
+** The process ID of the child is written into *pChildPid.
+**
+** Return the number of errors.
+*/
+static int popen2(
+  const char *zCmd,      /* Command to run in the child process */
+  FILE **ppIn,           /* Read from child using this file descriptor */
+  FILE **ppOut,          /* Write to child using this file descriptor */
+  int *pChildPid,        /* PID of the child process */
+  int bDirect            /* 0: run zCmd as a shell cmd.  1: run directly */
+){
+#ifdef _WIN32
+  HANDLE hStdinRd, hStdinWr, hStdoutRd, hStdoutWr, hStderr;
+  SECURITY_ATTRIBUTES saAttr;
+  DWORD childPid = 0;
+  int fd;
+
+  saAttr.nLength = sizeof(saAttr);
+  saAttr.bInheritHandle = TRUE;
+  saAttr.lpSecurityDescriptor = NULL;
+  hStderr = GetStdHandle(STD_ERROR_HANDLE);
+  if( !CreatePipe(&hStdoutRd, &hStdoutWr, &saAttr, 4096) ){
+    win32_fatal_error("cannot create pipe for stdout");
+  }
+  SetHandleInformation( hStdoutRd, HANDLE_FLAG_INHERIT, FALSE);
+
+  if( !CreatePipe(&hStdinRd, &hStdinWr, &saAttr, 4096) ){
+    win32_fatal_error("cannot create pipe for stdin");
+  }
+  SetHandleInformation( hStdinWr, HANDLE_FLAG_INHERIT, FALSE);
+
+  win32_create_child_process(win32_utf8_to_unicode(zCmd),
+                             hStdinRd, hStdoutWr, hStderr,&childPid);
+  *pChildPid = childPid;
+  fd = _open_osfhandle(PTR_TO_INT(hStdoutRd), 0);
+  *ppIn = fdopen(fd, "rb");
+  fd = _open_osfhandle(PTR_TO_INT(hStdinWr), 0);
+  *ppOut = _fdopen(fd, "wb");
+  CloseHandle(hStdinRd);
+  CloseHandle(hStdoutWr);
+  return 0;
+#else
+  int pin[2], pout[2];
+  *ppIn = 0;
+  *ppOut = 0;
+  *pChildPid = 0;
+
+  if( pipe(pin)<0 ){
+    return 1;
+  }
+  if( pipe(pout)<0 ){
+    close(pin[0]);
+    close(pin[1]);
+    return 1;
+  }
+  *pChildPid = fork();
+  if( *pChildPid<0 ){
+    close(pin[0]);
+    close(pin[1]);
+    close(pout[0]);
+    close(pout[1]);
+    *pChildPid = 0;
+    return 1;
+  }
+  signal(SIGPIPE,SIG_IGN);
+  if( *pChildPid==0 ){
+    int fd;
+    /* This is the child process */
+    close(0);
+    fd = dup(pout[0]);
+    if( fd!=0 ) {
+      fprintf(stderr,"popen2() failed to open file descriptor 0");
+      exit(1);
+    }
+    close(pout[0]);
+    close(pout[1]);
+    close(1);
+    fd = dup(pin[1]);
+    if( fd!=1 ){
+      fprintf(stderr,"popen() failed to open file descriptor 1");
+      exit(1);
+    }
+    close(pin[0]);
+    close(pin[1]);
+    if( bDirect ){
+      execl(zCmd, zCmd, (char*)0);
+    }else{
+      execl("/bin/sh", "/bin/sh", "-c", zCmd, (char*)0);
+    }
+    return 1;
+  }else{
+    /* This is the parent process */
+    close(pin[1]);
+    *ppIn = fdopen(pin[0], "r");
+    close(pout[0]);
+    *ppOut = fdopen(pout[1], "w");
+    return 0;
+  }
+#endif
+}
+
+/*
+** Close the connection to a child process previously created using
+** popen2().
+*/
+static void pclose2(FILE *pIn, FILE *pOut, int childPid){
+#ifdef _WIN32
+  /* Not implemented, yet */
+  fclose(pIn);
+  fclose(pOut);
+#else
+  fclose(pIn);
+  fclose(pOut);
+  while( waitpid(0, 0, WNOHANG)>0 ) {}
+#endif
+}
+/*****************************************************************************
+** End of the popen2() implementation copied from Fossil *********************
+*****************************************************************************/
+
+/*****************************************************************************
+** Beginning of the append_escaped_arg() routine, adapted from the Fossil   **
+** subroutine nameed blob_append_escaped_arg()                              **
+*****************************************************************************/
+/*
+** ASCII (for reference):
+**    x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf
+** 0x ^`  ^a  ^b  ^c  ^d  ^e  ^f  ^g  \b  \t  \n  ()  \f  \r  ^n  ^o
+** 1x ^p  ^q  ^r  ^s  ^t  ^u  ^v  ^w  ^x  ^y  ^z  ^{  ^|  ^}  ^~  ^
+** 2x ()  !   "   #   $   %   &   '   (   )   *   +   ,   -   .   /
+** 3x 0   1   2   3   4   5   6   7   8   9   :   ;   <   =   >   ?
+** 4x @   A   B   C   D   E   F   G   H   I   J   K   L   M   N   O
+** 5x P   Q   R   S   T   U   V   W   X   Y   Z   [   \   ]   ^   _
+** 6x `   a   b   c   d   e   f   g   h   i   j   k   l   m   n   o
+** 7x p   q   r   s   t   u   v   w   x   y   z   {   |   }   ~   ^_
+*/
+
+/*
+** Meanings for bytes in a filename:
+**
+**    0      Ordinary character.  No encoding required
+**    1      Needs to be escaped
+**    2      Illegal character.  Do not allow in a filename
+**    3      First byte of a 2-byte UTF-8
+**    4      First byte of a 3-byte UTF-8
+**    5      First byte of a 4-byte UTF-8
+*/
+static const char aSafeChar[256] = {
+#ifdef _WIN32
+/* Windows
+** Prohibit:  all control characters, including tab, \r and \n.
+** Escape:    (space) " # $ % & ' ( ) * ; < > ? [ ] ^ ` { | }
+*/
+/*  x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf  */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 0x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 1x */
+     1,  0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0, /* 2x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  0,  1,  1, /* 3x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 4x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  0,  1,  1,  0, /* 5x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 6x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  0,  1, /* 7x */
+#else
+/* Unix
+** Prohibit:  all control characters, including tab, \r and \n
+** Escape:    (space) ! " # $ % & ' ( ) * ; < > ? [ \ ] ^ ` { | }
+*/
+/*  x0  x1  x2  x3  x4  x5  x6  x7  x8  x9  xa  xb  xc  xd  xe  xf  */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 0x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 1x */
+     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0, /* 2x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  0,  1,  1, /* 3x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 4x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  1,  0, /* 5x */
+     1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, /* 6x */
+     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  1,  1,  0,  1, /* 7x */
+#endif
+    /* all bytes 0x80 through 0xbf are unescaped, being secondary
+    ** bytes to UTF8 characters.  Bytes 0xc0 through 0xff are the
+    ** first byte of a UTF8 character and do get escaped */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 8x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* 9x */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* ax */
+     2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2,  2, /* bx */
+     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, /* cx */
+     3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3,  3, /* dx */
+     4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4,  4, /* ex */
+     5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5,  5  /* fx */
+};
+
+/*
+** pStr is a shell command under construction.  This routine safely
+** appends filename argument zIn.  It returns 0 on success or non-zero
+** on any error.
+**
+** The argument is escaped if it contains white space or other characters
+** that need to be escaped for the shell.  If zIn contains characters
+** that cannot be safely escaped, then throw a fatal error.
+**
+** If the isFilename argument is true, then the argument is expected
+** to be a filename.  As shell commands commonly have command-line
+** options that begin with "-" and since we do not want an attacker
+** to be able to invoke these switches using filenames that begin
+** with "-", if zIn begins with "-", prepend an additional "./"
+** (or ".\\" on Windows).
+*/
+int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){
+  int i;
+  unsigned char c;
+  int needEscape = 0;
+  int n = sqlite3_str_length(pStr);
+  char *z = sqlite3_str_value(pStr);
+
+  /* Look for illegal byte-sequences and byte-sequences that require
+  ** escaping.  No control-characters are allowed.  All spaces and
+  ** non-ASCII unicode characters and some punctuation characters require
+  ** escaping. */
+  for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
+    if( aSafeChar[c] ){
+      unsigned char x = aSafeChar[c];
+      needEscape = 1;
+      if( x==2 ){
+        /* Bad ASCII character */
+        return 1;
+      }else if( x>2 ){
+        if( (zIn[i+1]&0xc0)!=0x80
+         || (x>=4 && (zIn[i+2]&0xc0)!=0x80)
+         || (x==5 && (zIn[i+3]&0xc0)!=0x80)
+        ){
+          /* Bad UTF8 character */
+          return 1;
+        }
+        i += x-2;
+      }
+    }
+  }
+
+  /* Separate from the previous argument by a space */
+  if( n>0 && !isspace(z[n-1]) ){
+    sqlite3_str_appendchar(pStr, 1, ' ');
+  }
+
+  /* Check for characters that need quoting */
+  if( !needEscape ){
+    if( isFilename && zIn[0]=='-' ){
+      sqlite3_str_appendchar(pStr, 1, '.');
+#if defined(_WIN32)
+      sqlite3_str_appendchar(pStr, 1, '\\');
+#else
+      sqlite3_str_appendchar(pStr, 1, '/');
+#endif
+    }
+    sqlite3_str_appendall(pStr, zIn);
+  }else{
+#if defined(_WIN32)
+    /* Quoting strategy for windows:
+    ** Put the entire name inside of "...".  Any " characters within
+    ** the name get doubled.
+    */
+    sqlite3_str_appendchar(pStr, 1, '"');
+    if( isFilename && zIn[0]=='-' ){
+      sqlite3_str_appendchar(pStr, 1, '.');
+      sqlite3_str_appendchar(pStr, 1, '\\');
+    }else if( zIn[0]=='/' ){
+      sqlite3_str_appendchar(pStr, 1, '.');
+    }
+    for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
+      sqlite3_str_appendchar(pStr, 1, (char)c);
+      if( c=='"' ) sqlite3_str_appendchar(pStr, 1, '"');
+      if( c=='\\' ) sqlite3_str_appendchar(pStr, 1, '\\');
+      if( c=='%' && isFilename ) sqlite3_str_append(pStr, "%cd:~,%", 7);
+    }
+    sqlite3_str_appendchar(pStr, 1, '"');
+#else
+    /* Quoting strategy for unix:
+    ** If the name does not contain ', then surround the whole thing
+    ** with '...'.   If there is one or more ' characters within the
+    ** name, then put \ before each special character.
+    */
+    if( strchr(zIn,'\'') ){
+      if( isFilename && zIn[0]=='-' ){
+        sqlite3_str_appendchar(pStr, 1, '.');
+        sqlite3_str_appendchar(pStr, 1, '/');
+      }
+      for(i=0; (c = (unsigned char)zIn[i])!=0; i++){
+        if( aSafeChar[c] && aSafeChar[c]!=2 ){
+          sqlite3_str_appendchar(pStr, 1, '\\');
+        }
+        sqlite3_str_appendchar(pStr, 1, (char)c);
+      }
+    }else{
+      sqlite3_str_appendchar(pStr, 1, '\'');
+      if( isFilename && zIn[0]=='-' ){
+        sqlite3_str_appendchar(pStr, 1, '.');
+        sqlite3_str_appendchar(pStr, 1, '/');
+      }
+      sqlite3_str_appendall(pStr, zIn);
+      sqlite3_str_appendchar(pStr, 1, '\'');
+    }
+#endif
+  }
+  return 0;
+}
+
+/* Add an approprate PATH= argument to the SSH command under construction
+** in pStr
+**
+** About This Feature
+** ==================
+**
+** On some ssh servers (Macs in particular are guilty of this) the PATH
+** variable in the shell that runs the command that is sent to the remote
+** host contains a limited number of read-only system directories:
+**
+**      /usr/bin:/bin:/usr/sbin:/sbin
+**
+** The sqlite3_rsync executable cannot be installed into any of those
+** directories because they are locked down, and so the "sqlite3_rsync"
+** command cannot run.
+**
+** To work around this, the sqlite3_rsync command is prefixed with a PATH=
+** argument, inserted by this function, to augment the PATH with additional
+** directories in which the sqlite3_rsync executable can be installed.
+**
+** But other ssh servers are confused by this initial PATH= argument.
+** Some ssh servers have a list of programs that they are allowed to run
+** and will fail if the first argument is not on that list, and PATH=....
+** is not on that list.
+**
+** So that sqlite3_rsync can invoke itself on a remote system using ssh
+** on a variety of platforms, the following algorithm is used:
+**
+**   *  First try running the sqlite3_rsync without any PATH= argument.
+**      If that works (and it does on a majority of systems) then we are
+**      done.
+**
+**   *  If the first attempt fails, then try again after adding the
+**      PATH= prefix argument.  (This function is what adds that
+**      argument.)
+**
+** A consequence of this is that if the remote system is a Mac, the
+** "ssh" command always ends up being invoked twice.  If anybody knows a
+** way around that problem, please bring it to the attention of the
+** developers.
+*/
+void add_path_argument(sqlite3_str *pStr){
+  append_escaped_arg(pStr,
+     "PATH=$HOME/bin:/usr/local/bin:/opt/homebrew/bin"
+     ":/opt/local/bin:$PATH", 0);
+}
+
+/*****************************************************************************
+** End of the append_escaped_arg() routine, adapted from the Fossil         **
+*****************************************************************************/
+
+/*****************************************************************************
+** The Hash Engine
+**
+** This is basically SHA3, though with a 160-bit hash, and reducing the
+** number of rounds in the KeccakF1600 step function from 24 to 6.
+*/
+/*
+** Macros to determine whether the machine is big or little endian,
+** and whether or not that determination is run-time or compile-time.
+**
+** For best performance, an attempt is made to guess at the byte-order
+** using C-preprocessor macros.  If that is unsuccessful, or if
+** -DHash_BYTEORDER=0 is set, then byte-order is determined
+** at run-time.
+*/
+#ifndef Hash_BYTEORDER
+# if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
+     defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
+     defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
+     defined(__arm__)
+#   define Hash_BYTEORDER    1234
+# elif defined(sparc)    || defined(__ppc__)
+#   define Hash_BYTEORDER    4321
+# else
+#   define Hash_BYTEORDER 0
+# endif
+#endif
+
+/*
+** State structure for a Hash hash in progress
+*/
+typedef struct HashContext HashContext;
+struct HashContext {
+  union {
+    u64 s[25];                /* Keccak state. 5x5 lines of 64 bits each */
+    unsigned char x[1600];    /* ... or 1600 bytes */
+  } u;
+  unsigned nRate;        /* Bytes of input accepted per Keccak iteration */
+  unsigned nLoaded;      /* Input bytes loaded into u.x[] so far this cycle */
+  unsigned ixMask;       /* Insert next input into u.x[nLoaded^ixMask]. */
+  unsigned iSize;        /* 224, 256, 358, or 512 */
+};
+
+/*
+** A single step of the Keccak mixing function for a 1600-bit state
+*/
+static void KeccakF1600Step(HashContext *p){
+  int i;
+  u64 b0, b1, b2, b3, b4;
+  u64 c0, c1, c2, c3, c4;
+  u64 d0, d1, d2, d3, d4;
+  static const u64 RC[] = {
+    0x0000000000000001ULL,  0x0000000000008082ULL,
+    0x800000000000808aULL,  0x8000000080008000ULL,
+    0x000000000000808bULL,  0x0000000080000001ULL,
+    0x8000000080008081ULL,  0x8000000000008009ULL,
+    0x000000000000008aULL,  0x0000000000000088ULL,
+    0x0000000080008009ULL,  0x000000008000000aULL,
+    0x000000008000808bULL,  0x800000000000008bULL,
+    0x8000000000008089ULL,  0x8000000000008003ULL,
+    0x8000000000008002ULL,  0x8000000000000080ULL,
+    0x000000000000800aULL,  0x800000008000000aULL,
+    0x8000000080008081ULL,  0x8000000000008080ULL,
+    0x0000000080000001ULL,  0x8000000080008008ULL
+  };
+# define a00 (p->u.s[0])
+# define a01 (p->u.s[1])
+# define a02 (p->u.s[2])
+# define a03 (p->u.s[3])
+# define a04 (p->u.s[4])
+# define a10 (p->u.s[5])
+# define a11 (p->u.s[6])
+# define a12 (p->u.s[7])
+# define a13 (p->u.s[8])
+# define a14 (p->u.s[9])
+# define a20 (p->u.s[10])
+# define a21 (p->u.s[11])
+# define a22 (p->u.s[12])
+# define a23 (p->u.s[13])
+# define a24 (p->u.s[14])
+# define a30 (p->u.s[15])
+# define a31 (p->u.s[16])
+# define a32 (p->u.s[17])
+# define a33 (p->u.s[18])
+# define a34 (p->u.s[19])
+# define a40 (p->u.s[20])
+# define a41 (p->u.s[21])
+# define a42 (p->u.s[22])
+# define a43 (p->u.s[23])
+# define a44 (p->u.s[24])
+# define ROL64(a,x) ((a<<x)|(a>>(64-x)))
+
+  /*         v---- Number of rounds.  SHA3 has 24 here. */
+  for(i=0; i<6; i++){
+    c0 = a00^a10^a20^a30^a40;
+    c1 = a01^a11^a21^a31^a41;
+    c2 = a02^a12^a22^a32^a42;
+    c3 = a03^a13^a23^a33^a43;
+    c4 = a04^a14^a24^a34^a44;
+    d0 = c4^ROL64(c1, 1);
+    d1 = c0^ROL64(c2, 1);
+    d2 = c1^ROL64(c3, 1);
+    d3 = c2^ROL64(c4, 1);
+    d4 = c3^ROL64(c0, 1);
+
+    b0 = (a00^d0);
+    b1 = ROL64((a11^d1), 44);
+    b2 = ROL64((a22^d2), 43);
+    b3 = ROL64((a33^d3), 21);
+    b4 = ROL64((a44^d4), 14);
+    a00 =   b0 ^((~b1)&  b2 );
+    a00 ^= RC[i];
+    a11 =   b1 ^((~b2)&  b3 );
+    a22 =   b2 ^((~b3)&  b4 );
+    a33 =   b3 ^((~b4)&  b0 );
+    a44 =   b4 ^((~b0)&  b1 );
+
+    b2 = ROL64((a20^d0), 3);
+    b3 = ROL64((a31^d1), 45);
+    b4 = ROL64((a42^d2), 61);
+    b0 = ROL64((a03^d3), 28);
+    b1 = ROL64((a14^d4), 20);
+    a20 =   b0 ^((~b1)&  b2 );
+    a31 =   b1 ^((~b2)&  b3 );
+    a42 =   b2 ^((~b3)&  b4 );
+    a03 =   b3 ^((~b4)&  b0 );
+    a14 =   b4 ^((~b0)&  b1 );
+
+    b4 = ROL64((a40^d0), 18);
+    b0 = ROL64((a01^d1), 1);
+    b1 = ROL64((a12^d2), 6);
+    b2 = ROL64((a23^d3), 25);
+    b3 = ROL64((a34^d4), 8);
+    a40 =   b0 ^((~b1)&  b2 );
+    a01 =   b1 ^((~b2)&  b3 );
+    a12 =   b2 ^((~b3)&  b4 );
+    a23 =   b3 ^((~b4)&  b0 );
+    a34 =   b4 ^((~b0)&  b1 );
+
+    b1 = ROL64((a10^d0), 36);
+    b2 = ROL64((a21^d1), 10);
+    b3 = ROL64((a32^d2), 15);
+    b4 = ROL64((a43^d3), 56);
+    b0 = ROL64((a04^d4), 27);
+    a10 =   b0 ^((~b1)&  b2 );
+    a21 =   b1 ^((~b2)&  b3 );
+    a32 =   b2 ^((~b3)&  b4 );
+    a43 =   b3 ^((~b4)&  b0 );
+    a04 =   b4 ^((~b0)&  b1 );
+
+    b3 = ROL64((a30^d0), 41);
+    b4 = ROL64((a41^d1), 2);
+    b0 = ROL64((a02^d2), 62);
+    b1 = ROL64((a13^d3), 55);
+    b2 = ROL64((a24^d4), 39);
+    a30 =   b0 ^((~b1)&  b2 );
+    a41 =   b1 ^((~b2)&  b3 );
+    a02 =   b2 ^((~b3)&  b4 );
+    a13 =   b3 ^((~b4)&  b0 );
+    a24 =   b4 ^((~b0)&  b1 );
+  }
+}
+
+/*
+** Initialize a new hash.  iSize determines the size of the hash
+** in bits and should be one of 224, 256, 384, or 512.  Or iSize
+** can be zero to use the default hash size of 256 bits.
+*/
+static void HashInit(HashContext *p, int iSize){
+  memset(p, 0, sizeof(*p));
+  p->iSize = iSize;
+  if( iSize>=128 && iSize<=512 ){
+    p->nRate = (1600 - ((iSize + 31)&~31)*2)/8;
+  }else{
+    p->nRate = (1600 - 2*256)/8;
+  }
+#if Hash_BYTEORDER==1234
+  /* Known to be little-endian at compile-time. No-op */
+#elif Hash_BYTEORDER==4321
+  p->ixMask = 7;  /* Big-endian */
+#else
+  {
+    static unsigned int one = 1;
+    if( 1==*(unsigned char*)&one ){
+      /* Little endian.  No byte swapping. */
+      p->ixMask = 0;
+    }else{
+      /* Big endian.  Byte swap. */
+      p->ixMask = 7;
+    }
+  }
+#endif
+}
+
+/*
+** Make consecutive calls to the HashUpdate function to add new content
+** to the hash
+*/
+static void HashUpdate(
+  HashContext *p,
+  const unsigned char *aData,
+  unsigned int nData
+){
+  unsigned int i = 0;
+  if( aData==0 ) return;
+#if Hash_BYTEORDER==1234
+  if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){
+    for(; i+7<nData; i+=8){
+      p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i];
+      p->nLoaded += 8;
+      if( p->nLoaded>=p->nRate ){
+        KeccakF1600Step(p);
+        p->nLoaded = 0;
+      }
+    }
+  }
+#endif
+  for(; i<nData; i++){
+#if Hash_BYTEORDER==1234
+    p->u.x[p->nLoaded] ^= aData[i];
+#elif Hash_BYTEORDER==4321
+    p->u.x[p->nLoaded^0x07] ^= aData[i];
+#else
+    p->u.x[p->nLoaded^p->ixMask] ^= aData[i];
+#endif
+    p->nLoaded++;
+    if( p->nLoaded==p->nRate ){
+      KeccakF1600Step(p);
+      p->nLoaded = 0;
+    }
+  }
+}
+
+/*
+** After all content has been added, invoke HashFinal() to compute
+** the final hash.  The function returns a pointer to the binary
+** hash value.
+*/
+static unsigned char *HashFinal(HashContext *p){
+  unsigned int i;
+  if( p->nLoaded==p->nRate-1 ){
+    const unsigned char c1 = 0x86;
+    HashUpdate(p, &c1, 1);
+  }else{
+    const unsigned char c2 = 0x06;
+    const unsigned char c3 = 0x80;
+    HashUpdate(p, &c2, 1);
+    p->nLoaded = p->nRate - 1;
+    HashUpdate(p, &c3, 1);
+  }
+  for(i=0; i<p->nRate; i++){
+    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
+  }
+  return &p->u.x[p->nRate];
+}
+
+/*
+** Implementation of the hash(X) function.
+**
+** Return a 160-bit BLOB which is the hash of X.
+*/
+static void hashFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  HashContext cx;
+  int eType = sqlite3_value_type(argv[0]);
+  int nByte = sqlite3_value_bytes(argv[0]);
+  if( eType==SQLITE_NULL ) return;
+  HashInit(&cx, 160);
+  if( eType==SQLITE_BLOB ){
+    HashUpdate(&cx, sqlite3_value_blob(argv[0]), nByte);
+  }else{
+    HashUpdate(&cx, sqlite3_value_text(argv[0]), nByte);
+  }
+  sqlite3_result_blob(context, HashFinal(&cx), 160/8, SQLITE_TRANSIENT);
+}
+
+/*
+** Implementation of the agghash(X) function.
+**
+** Return a 160-bit BLOB which is the hash of the concatenation
+** of all X inputs.
+*/
+static void agghashStep(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  HashContext *pCx;
+  int eType = sqlite3_value_type(argv[0]);
+  int nByte = sqlite3_value_bytes(argv[0]);
+  if( eType==SQLITE_NULL ) return;
+  pCx = (HashContext*)sqlite3_aggregate_context(context, sizeof(*pCx));
+  if( pCx==0 ) return;
+  if( pCx->iSize==0 ) HashInit(pCx, 160);
+  if( eType==SQLITE_BLOB ){
+    HashUpdate(pCx, sqlite3_value_blob(argv[0]), nByte);
+  }else{
+    HashUpdate(pCx, sqlite3_value_text(argv[0]), nByte);
+  }
+}
+static void agghashFinal(sqlite3_context *context){
+  HashContext *pCx = (HashContext*)sqlite3_aggregate_context(context, 0);
+  if( pCx ){
+    sqlite3_result_blob(context, HashFinal(pCx), 160/8, SQLITE_TRANSIENT);
+  }
+}
+
+/* Register the hash function */
+static int hashRegister(sqlite3 *db){
+  int rc;
+  rc = sqlite3_create_function(db, "hash", 1,
+                SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
+                0, hashFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "agghash", 1,
+                SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
+                0, 0, agghashStep, agghashFinal);
+  }
+  return rc;
+}
+
+/* End of the hashing logic
+*****************************************************************************/
+
+/*
+** Return the tail of a file pathname.  The tail is the last component
+** of the path.  For example, the tail of "/a/b/c.d" is "c.d".
+*/
+const char *file_tail(const char *z){
+  const char *zTail = z;
+  if( !zTail ) return 0;
+  while( z[0] ){
+    if( z[0]=='/' ) zTail = &z[1];
+    z++;
+  }
+  return zTail;
+}
+
+/*
+** Append error message text to the error file, if an error file is
+** specified.  In any case, increment the error count.
+*/
+static void logError(SQLiteRsync *p, const char *zFormat, ...){
+  if( p->zErrFile ){
+    FILE *pErr = fopen(p->zErrFile, "a");
+    if( pErr ){
+      va_list ap;
+      va_start(ap, zFormat);
+      vfprintf(pErr, zFormat, ap);
+      va_end(ap);
+      fclose(pErr);
+    }
+  }
+  p->nErr++;
+}
+
+/*
+** Append text to the debugging mesage file, if an that file is
+** specified.
+*/
+static void debugMessage(SQLiteRsync *p, const char *zFormat, ...){
+  if( p->zDebugFile ){
+    if( p->pDebug==0 ){
+      p->pDebug = fopen(p->zDebugFile, "wb");
+    }
+    if( p->pDebug ){
+      va_list ap;
+      va_start(ap, zFormat);
+      vfprintf(p->pDebug, zFormat, ap);
+      va_end(ap);
+      fflush(p->pDebug);
+    }
+  }
+}
+
+
+/* Read a single big-endian 32-bit unsigned integer from the input
+** stream.  Return 0 on success and 1 if there are any errors.
+*/
+static int readUint32(SQLiteRsync *p, unsigned int *pU){
+  unsigned char buf[4];
+  if( fread(buf, sizeof(buf), 1, p->pIn)==1 ){
+    *pU = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3];
+    p->nIn += 4;
+    return 0;
+  }else{
+    logError(p, "failed to read a 32-bit integer\n");
+    return 1;
+  }
+}
+
+/* Write a single big-endian 32-bit unsigned integer to the output stream.
+** Return 0 on success and 1 if there are any errors.
+*/
+static int writeUint32(SQLiteRsync *p, unsigned int x){
+  unsigned char buf[4];
+  buf[3] = x & 0xff;
+  x >>= 8;
+  buf[2] = x & 0xff;
+  x >>= 8;
+  buf[1] = x & 0xff;
+  x >>= 8;
+  buf[0] = x;
+  if( p->pLog ) fwrite(buf, sizeof(buf), 1, p->pLog);
+  if( fwrite(buf, sizeof(buf), 1, p->pOut)!=1 ){
+    logError(p, "failed to write 32-bit integer 0x%x\n", x);
+    p->nWrErr++;
+    return 1;
+  }
+  p->nOut += 4;
+  return 0;
+}
+
+/* Read a single byte from the wire.
+*/
+int readByte(SQLiteRsync *p){
+  int c = fgetc(p->pIn);
+  if( c!=EOF ) p->nIn++;
+  return c;
+}
+
+/* Write a single byte into the wire.
+*/
+void writeByte(SQLiteRsync *p, int c){
+  if( p->pLog ) fputc(c, p->pLog);
+  fputc(c, p->pOut);
+  p->nOut++;
+}
+
+/* Read a power of two encoded as a single byte.
+*/
+int readPow2(SQLiteRsync *p){
+  int x = readByte(p);
+  if( x<0 || x>=32 ){
+    logError(p, "read invalid page size %d\n", x);
+    return 0;
+  }
+  return 1<<x;
+}
+
+/* Write a power-of-two value onto the wire as a single byte.
+*/
+void writePow2(SQLiteRsync *p, int c){
+  int n;
+  if( c<0 || (c&(c-1))!=0 ){
+    logError(p, "trying to read invalid page size %d\n", c);
+  }
+  for(n=0; c>1; n++){ c /= 2; }
+  writeByte(p, n);
+}
+
+/* Read an array of bytes from the wire.
+*/
+void readBytes(SQLiteRsync *p, int nByte, void *pData){
+  if( fread(pData, 1, nByte, p->pIn)==nByte ){
+    p->nIn += nByte;
+  }else{
+    logError(p, "failed to read %d bytes\n", nByte);
+  }
+}
+
+/* Write an array of bytes onto the wire.
+*/
+void writeBytes(SQLiteRsync *p, int nByte, const void *pData){
+  if( p->pLog ) fwrite(pData, 1, nByte, p->pLog);
+  if( fwrite(pData, 1, nByte, p->pOut)==nByte ){
+    p->nOut += nByte;
+  }else{
+    logError(p, "failed to write %d bytes\n", nByte);
+    p->nWrErr++;
+  }
+}
+
+/* Report an error.
+**
+** If this happens on the remote side, we send back a *_ERROR
+** message.  On the local side, the error message goes to stderr.
+*/
+static void reportError(SQLiteRsync *p, const char *zFormat, ...){
+  va_list ap;
+  char *zMsg;
+  unsigned int nMsg;
+  va_start(ap, zFormat);
+  zMsg = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0;
+  if( p->isRemote ){
+    if( p->isReplica ){
+      putc(REPLICA_ERROR, p->pOut);
+    }else{
+      putc(ORIGIN_ERROR, p->pOut);
+    }
+    writeUint32(p, nMsg);
+    writeBytes(p, nMsg, zMsg);
+    fflush(p->pOut);
+  }else{
+    fprintf(stderr, "%s\n", zMsg);
+  }
+  logError(p, "%s\n", zMsg);
+  sqlite3_free(zMsg);
+}
+
+/* Send an informational message.
+**
+** If this happens on the remote side, we send back a *_MSG 
+** message.  On the local side, the message goes to stdout.
+*/
+static void infoMsg(SQLiteRsync *p, const char *zFormat, ...){
+  va_list ap;
+  char *zMsg;
+  unsigned int nMsg;
+  va_start(ap, zFormat);
+  zMsg = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0;
+  if( p->isRemote ){
+    if( p->isReplica ){
+      putc(REPLICA_MSG, p->pOut);
+    }else{
+      putc(ORIGIN_MSG, p->pOut);
+    }
+    writeUint32(p, nMsg);
+    writeBytes(p, nMsg, zMsg);
+    fflush(p->pOut);
+  }else{
+    printf("%s\n", zMsg);
+  }
+  sqlite3_free(zMsg);
+}
+
+/* Receive and report an error message coming from the other side.
+*/
+static void readAndDisplayMessage(SQLiteRsync *p, int c){
+  unsigned int n = 0;
+  char *zMsg;
+  const char *zPrefix;
+  if( c==ORIGIN_ERROR || c==REPLICA_ERROR ){
+    zPrefix = "ERROR: ";
+  }else{
+    zPrefix = "";
+  }
+  readUint32(p, &n);
+  if( n==0 ){
+    fprintf(stderr,"ERROR: unknown (possibly out-of-memory)\n");
+  }else{
+    zMsg = sqlite3_malloc64( n+1 );
+    if( zMsg==0 ){
+      fprintf(stderr, "ERROR: out-of-memory\n");
+      return;
+    }
+    memset(zMsg, 0, n+1);
+    readBytes(p, n, zMsg);
+    fprintf(stderr,"%s%s\n", zPrefix, zMsg);
+    if( zPrefix[0] ) logError(p, "%s%s\n", zPrefix, zMsg);
+    sqlite3_free(zMsg);
+  }
+}
+
+/* Construct a new prepared statement.  Report an error and return NULL
+** if anything goes wrong.
+*/
+static sqlite3_stmt *prepareStmtVA(
+  SQLiteRsync *p,
+  char *zFormat,
+  va_list ap
+){
+  sqlite3_stmt *pStmt = 0;
+  char *zSql;
+  char *zToFree = 0;
+  int rc;
+
+  if( strchr(zFormat,'%') ){
+    zSql = sqlite3_vmprintf(zFormat, ap);
+    if( zSql==0 ){
+      reportError(p, "out-of-memory");
+      return 0;
+    }else{
+      zToFree = zSql;
+    }
+  }else{
+    zSql = zFormat;
+  }
+  rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+  if( rc || pStmt==0 ){
+    reportError(p, "unable to prepare SQL [%s]: %s", zSql,
+                sqlite3_errmsg(p->db));
+    sqlite3_finalize(pStmt);
+    pStmt = 0;
+  }
+  if( zToFree ) sqlite3_free(zToFree);
+  return pStmt;
+}
+static sqlite3_stmt *prepareStmt(
+  SQLiteRsync *p,
+  char *zFormat,
+  ...
+){
+  sqlite3_stmt *pStmt;
+  va_list ap;
+  va_start(ap, zFormat);
+  pStmt = prepareStmtVA(p, zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/* Run a single SQL statement.  Report an error if something goes
+** wrong.
+**
+** As a special case, if the statement starts with "ATTACH" (but not
+** "Attach") and if the error message is about an incorrect encoding,
+** then do not report the error, but instead set the wrongEncoding flag.
+** This is a kludgy work-around to the problem of attaching a database
+** with a non-UTF8 encoding to the empty :memory: database that is
+** opened on the replica.
+*/
+static void runSql(SQLiteRsync *p, char *zSql, ...){
+  sqlite3_stmt *pStmt;
+  va_list ap;
+
+  va_start(ap, zSql);
+  pStmt = prepareStmtVA(p, zSql, ap);
+  va_end(ap);
+  if( pStmt ){
+    int rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ) rc = sqlite3_step(pStmt);
+    if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){
+      const char *zErr = sqlite3_errmsg(p->db);
+      if( strncmp(zSql,"ATTACH ", 7)==0
+       && strstr(zErr,"must use the same text encoding")!=0
+      ){
+        p->wrongEncoding = 1;
+      }else{
+        reportError(p, "SQL statement [%s] failed: %s", zSql,
+                    sqlite3_errmsg(p->db));
+      }
+    }
+    sqlite3_finalize(pStmt);
+  }
+}
+
+/* Run an SQL statement that returns a single unsigned 32-bit integer result
+*/
+static int runSqlReturnUInt(
+  SQLiteRsync *p,
+  unsigned int *pRes,
+  char *zSql,
+  ...
+){
+  sqlite3_stmt *pStmt;
+  int res = 0;
+  va_list ap;
+
+  va_start(ap, zSql);
+  pStmt = prepareStmtVA(p, zSql, ap);
+  va_end(ap);
+  if( pStmt==0 ){
+    res = 1;
+  }else{
+    int rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ){
+      *pRes = (unsigned int)(sqlite3_column_int64(pStmt, 0)&0xffffffff);
+    }else{
+      reportError(p, "SQL statement [%s] failed: %s", zSql,
+                  sqlite3_errmsg(p->db));
+      res = 1;
+    }
+    sqlite3_finalize(pStmt);
+  }
+  return res;
+}
+
+/* Run an SQL statement that returns a single TEXT value that is no more
+** than 99 bytes in length.
+*/
+static int runSqlReturnText(
+  SQLiteRsync *p,
+  char *pRes,
+  char *zSql,
+  ...
+){
+  sqlite3_stmt *pStmt;
+  int res = 0;
+  va_list ap;
+
+  va_start(ap, zSql);
+  pStmt = prepareStmtVA(p, zSql, ap);
+  va_end(ap);
+  pRes[0] = 0;
+  if( pStmt==0 ){
+    res = 1;
+  }else{
+    int rc = sqlite3_step(pStmt);
+    if( rc==SQLITE_ROW ){
+      const unsigned char *a = sqlite3_column_text(pStmt, 0);
+      int n;
+      if( a==0 ){
+        pRes[0] = 0;
+      }else{
+        n = sqlite3_column_bytes(pStmt, 0);
+        if( n>99 ) n = 99;
+        memcpy(pRes, a, n);
+        pRes[n] = 0;
+      }
+    }else{
+      reportError(p, "SQL statement [%s] failed: %s", zSql,
+                  sqlite3_errmsg(p->db));
+      res = 1;
+    }
+    sqlite3_finalize(pStmt);
+  }
+  return res;
+}
+
+/* Close the database connection associated with p
+*/
+static void closeDb(SQLiteRsync *p){
+  if( p->db ){
+    sqlite3_close(p->db);
+    p->db = 0;
+  }
+}
+
+/*
+** Run the origin-side protocol.
+**
+** Begin by sending the ORIGIN_BEGIN message with two arguments,
+** nPage, and szPage.  Then enter a loop responding to message from
+** the replica:
+**
+**    REPLICA_BEGIN  iProtocol
+**
+**         An optional message sent by the replica in response to the
+**         prior ORIGIN_BEGIN with a counter-proposal for the protocol
+**         level.  If seen, try to reduce the protocol level to what is
+**         requested and send a new ORGIN_BEGIN.
+**
+**    REPLICA_ERROR  size  text
+**
+**         Report an error from the replica and quit
+**
+**    REPLICA_END
+**
+**         The replica is terminating.  Stop processing now.
+**
+**    REPLICA_HASH  hash
+**
+**         The argument is the 20-byte SHA1 hash for the next page or
+**         block of pages.  Hashes appear in sequential order with no gaps,
+**         unless there is an intervening REPLICA_CONFIG message.
+**
+**    REPLICA_CONFIG   pgno   cnt
+**
+**         Set counters used by REPLICA_HASH.  The next hash will start
+**         on page pgno and all subsequent hashes will cover cnt pages
+**         each.  Note that for a multi-page hash, the hash value is
+**         actually a hash of the individual page hashes.
+**
+**    REPLICA_READY
+**
+**         The replica has sent all the hashes that it intends to send.
+**         This side (the origin) can now start responding with page
+**         content for pages that do not have a matching hash or with
+**         ORIGIN_DETAIL messages with requests for more detail.
+*/
+static void originSide(SQLiteRsync *p){
+  int rc = 0;
+  int c = 0;
+  unsigned int nPage = 0;
+  unsigned int iHash = 1;               /* Pgno for next hash to receive */
+  unsigned int nHash = 1;               /* Number of pages per hash received */
+  unsigned int mxHash = 0;              /* Maximum hash value received */
+  unsigned int lockBytePage = 0;
+  unsigned int szPg = 0;
+  sqlite3_stmt *pCkHash = 0;            /* Verify hash on a single page */
+  sqlite3_stmt *pCkHashN = 0;           /* Verify a multi-page hash */
+  sqlite3_stmt *pInsHash = 0;           /* Record a bad hash */
+  char buf[200];
+
+  p->isReplica = 0;
+  if( p->bCommCheck ){
+    infoMsg(p, "origin  zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d",
+               p->zOrigin, p->zReplica, p->isRemote, p->iProtocol);
+    writeByte(p, ORIGIN_END);
+    fflush(p->pOut);
+  }else{
+    /* Open the ORIGIN database. */
+    rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0);
+    if( rc ){
+      reportError(p, "cannot open origin \"%s\": %s",
+                  p->zOrigin, sqlite3_errmsg(p->db));
+      closeDb(p);
+      return;
+    }
+    hashRegister(p->db);
+    runSql(p, "BEGIN");
+    if( p->bWalOnly ){
+      runSqlReturnText(p, buf, "PRAGMA journal_mode");
+      if( sqlite3_stricmp(buf,"wal")!=0 ){
+        reportError(p, "Origin database is not in WAL mode");
+      }
+    }
+    runSqlReturnUInt(p, &nPage, "PRAGMA page_count");
+    runSqlReturnUInt(p, &szPg, "PRAGMA page_size");
+  
+    if( p->nErr==0 ){
+      /* Send the ORIGIN_BEGIN message */
+      writeByte(p, ORIGIN_BEGIN);
+      writeByte(p, p->iProtocol);
+      writePow2(p, szPg);
+      writeUint32(p, nPage);
+      fflush(p->pOut);
+      if( p->zDebugFile ){
+        debugMessage(p, "-> ORIGIN_BEGIN %u %u %u\n", p->iProtocol,szPg,nPage);
+      }
+      p->nPage = nPage;
+      p->szPage = szPg;
+      lockBytePage = (1<<30)/szPg + 1;
+    }
+  }
+  
+  /* Respond to message from the replica */
+  while( p->nErr<=p->nWrErr && (c = readByte(p))!=EOF && c!=REPLICA_END ){
+    switch( c ){
+      case REPLICA_BEGIN: {
+        /* This message is only sent if the replica received an origin-protocol
+        ** that is larger than what it knows about.  The replica sends back
+        ** a counter-proposal of an earlier protocol which the origin can
+        ** accept by resending a new ORIGIN_BEGIN. */
+        u8 newProtocol = readByte(p);
+        if( p->zDebugFile ){
+          debugMessage(p, "<- REPLICA_BEGIN %d\n", (int)newProtocol);
+        }
+        if( newProtocol < p->iProtocol ){
+          p->iProtocol = newProtocol;
+          writeByte(p, ORIGIN_BEGIN);
+          writeByte(p, p->iProtocol);
+          writePow2(p, p->szPage);
+          writeUint32(p, p->nPage);
+          fflush(p->pOut);
+          if( p->zDebugFile ){
+            debugMessage(p, "-> ORIGIN_BEGIN %d %d %u\n", p->iProtocol,
+                         p->szPage, p->nPage);
+          }
+        }else{
+          reportError(p, "Invalid REPLICA_BEGIN reply");
+        }
+        break;
+      }
+      case REPLICA_MSG:
+      case REPLICA_ERROR: {
+        readAndDisplayMessage(p, c);
+        break;
+      }
+      case REPLICA_CONFIG: {
+        readUint32(p, &iHash);
+        readUint32(p, &nHash);
+        if( p->zDebugFile ){
+          debugMessage(p, "<- REPLICA_CONFIG %u %u\n", iHash, nHash);
+        }
+        break;
+      }
+      case REPLICA_HASH: {
+        int bMatch = 0;
+        if( pCkHash==0 ){
+          runSql(p, "CREATE TEMP TABLE badHash("
+                    " pgno INTEGER PRIMARY KEY,"
+                    " sz INT)");
+          pCkHash = prepareStmt(p,
+            "SELECT hash(data)==?3 FROM sqlite_dbpage('main')"
+            " WHERE pgno=?1"
+          );
+          if( pCkHash==0 ) break;
+          pInsHash = prepareStmt(p, "INSERT INTO badHash VALUES(?1,?2)");
+          if( pInsHash==0 ) break;
+        }
+        p->nHashSent++;
+        readBytes(p, 20, buf);
+        if( nHash>1 ){
+          if( pCkHashN==0 ){
+            pCkHashN = prepareStmt(p,
+              "WITH c(n) AS "
+              "  (VALUES(?1) UNION ALL SELECT n+1 FROM c WHERE n<?2)"
+              "SELECT agghash(hash(data))==?3"
+               " FROM c CROSS JOIN sqlite_dbpage('main') ON pgno=n"
+            );
+            if( pCkHashN==0 ) break;
+          }
+          sqlite3_bind_int64(pCkHashN, 1, iHash);
+          sqlite3_bind_int64(pCkHashN, 2, iHash + nHash - 1);
+          sqlite3_bind_blob(pCkHashN, 3, buf, 20, SQLITE_STATIC);
+          rc = sqlite3_step(pCkHashN);
+          if( rc==SQLITE_ROW ){
+            bMatch = sqlite3_column_int(pCkHashN,0);
+          }else if( rc==SQLITE_ERROR ){
+            reportError(p, "SQL statement [%s] failed: %s",
+                        sqlite3_sql(pCkHashN), sqlite3_errmsg(p->db));
+          }
+          sqlite3_reset(pCkHashN);
+        }else{
+          sqlite3_bind_int64(pCkHash, 1, iHash);
+          sqlite3_bind_blob(pCkHash, 3, buf, 20, SQLITE_STATIC);
+          rc = sqlite3_step(pCkHash);
+          if( rc==SQLITE_ERROR ){
+            reportError(p, "SQL statement [%s] failed: %s",
+                        sqlite3_sql(pCkHash), sqlite3_errmsg(p->db));
+          }else if( rc==SQLITE_ROW && sqlite3_column_int(pCkHash,0) ){
+            bMatch = 1;
+          }
+          sqlite3_reset(pCkHash);
+        }
+        if( p->zDebugFile ){
+          debugMessage(p, "<- REPLICA_HASH %u %u %s %08x...\n",
+              iHash, nHash,
+              bMatch ? "match" : "fail",
+              *(unsigned int*)buf
+          );
+        }
+        if( !bMatch ){
+          sqlite3_bind_int64(pInsHash, 1, iHash);
+          sqlite3_bind_int64(pInsHash, 2, nHash);
+          rc = sqlite3_step(pInsHash);
+          if( rc!=SQLITE_DONE ){
+            reportError(p, "SQL statement [%s] failed: %s",
+                sqlite3_sql(pInsHash), sqlite3_errmsg(p->db));
+          }
+          sqlite3_reset(pInsHash);
+        }
+        if( iHash+nHash>mxHash ) mxHash = iHash+nHash;
+        iHash += nHash;
+        break;
+      }
+      case REPLICA_READY: {
+        int nMulti = 0;
+        sqlite3_stmt *pStmt;
+        if( p->zDebugFile ){
+          debugMessage(p, "<- REPLICA_READY\n");
+        }
+        p->nRound++;
+        pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1");
+        if( pStmt==0 ) break;
+        while( sqlite3_step(pStmt)==SQLITE_ROW ){
+          unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0);
+          unsigned int cnt = (unsigned int)sqlite3_column_int64(pStmt,1);
+          writeByte(p, ORIGIN_DETAIL);
+          writeUint32(p, pgno);
+          writeUint32(p, cnt);
+          nMulti++;
+          if( p->zDebugFile ){
+            debugMessage(p, "-> ORIGIN_DETAIL %u %u\n", pgno, cnt);
+          }
+        }
+        sqlite3_finalize(pStmt);
+        if( nMulti ){
+          runSql(p, "DELETE FROM badHash WHERE sz>1");
+          writeByte(p, ORIGIN_READY);
+          if( p->zDebugFile ) debugMessage(p, "-> ORIGIN_READY\n");
+        }else{
+          sqlite3_finalize(pCkHash);
+          sqlite3_finalize(pCkHashN);
+          sqlite3_finalize(pInsHash);
+          pCkHash = 0;
+          pInsHash = 0;
+          if( mxHash<=p->nPage ){
+            runSql(p, "WITH RECURSIVE c(n) AS"
+                      " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)"
+                      " INSERT INTO badHash SELECT n, 1 FROM c",
+                      mxHash, p->nPage);
+          }
+          runSql(p, "DELETE FROM badHash WHERE pgno=%d", lockBytePage);
+          pStmt = prepareStmt(p,
+                 "SELECT pgno, data"
+                 "  FROM badHash JOIN sqlite_dbpage('main') USING(pgno)");
+          if( pStmt==0 ) break;
+          while( sqlite3_step(pStmt)==SQLITE_ROW
+              && p->nErr==0
+              && p->nWrErr==0
+          ){
+            unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0);
+            const void *pContent = sqlite3_column_blob(pStmt, 1);
+            writeByte(p, ORIGIN_PAGE);
+            writeUint32(p, pgno);
+            writeBytes(p, szPg, pContent);
+            p->nPageSent++;
+            if( p->zDebugFile ){
+              debugMessage(p, "-> ORIGIN_PAGE %u\n", pgno);
+            }
+          }
+          sqlite3_finalize(pStmt);
+          writeByte(p, ORIGIN_TXN);
+          writeUint32(p, nPage);
+          if( p->zDebugFile ){
+            debugMessage(p, "-> ORIGIN_TXN %u\n", nPage);
+          }
+          writeByte(p, ORIGIN_END);
+        }
+        fflush(p->pOut);
+        break;
+      }
+      default: {
+        reportError(p, "Unknown message 0x%02x %lld bytes into conversation",
+                       c, p->nIn);
+        break;
+      }
+    }
+  }
+
+  if( pCkHash ) sqlite3_finalize(pCkHash);
+  if( pInsHash ) sqlite3_finalize(pInsHash);
+  closeDb(p);
+}
+
+/*
+** Send a REPLICA_HASH message for each entry in the sendHash table.
+** The sendHash table looks like this:
+**
+**   CREATE TABLE sendHash(
+**      fpg INTEGER PRIMARY KEY,   -- Page number of the hash
+**      npg INT                    -- Number of pages in this hash
+**   );
+**
+** If iHash is page number for the next page that the origin will
+** be expecting, and nHash is the number of pages that the origin will
+** be expecting in the hash that follows.  Send a REPLICA_CONFIG message
+** if either of these values if not correct.
+*/
+static void sendHashMessages(
+  SQLiteRsync *p,       /* The replica-side of the sync */
+  unsigned int iHash,   /* Next page expected by origin */
+  unsigned int nHash    /* Next number of pages expected by origin */
+){
+  sqlite3_stmt *pStmt;
+  pStmt = prepareStmt(p,
+    "SELECT if(npg==1,"
+    "  (SELECT hash(data) FROM sqlite_dbpage('replica') WHERE pgno=fpg),"
+    "  (WITH RECURSIVE c(n) AS"
+    "     (SELECT fpg UNION ALL SELECT n+1 FROM c WHERE n<fpg+npg-1)"
+    "   SELECT agghash(hash(data))"
+    "     FROM c CROSS JOIN sqlite_dbpage('replica') ON pgno=n)) AS hash,"
+    "  fpg,"
+    "  npg"
+    " FROM sendHash ORDER BY fpg"
+  );
+  while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){
+    const unsigned char *a = sqlite3_column_blob(pStmt, 0);
+    unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt, 1);
+    unsigned int npg = (unsigned int)sqlite3_column_int64(pStmt, 2);
+    if( pgno!=iHash || npg!=nHash ){
+      writeByte(p, REPLICA_CONFIG);
+      writeUint32(p, pgno);
+      writeUint32(p, npg);
+      if( p->zDebugFile ){
+        debugMessage(p, "-> REPLICA_CONFIG %u %u\n", pgno, npg);
+      }
+    }
+    if( a==0 ){
+      if( p->zDebugFile ){
+        debugMessage(p, "# Oops: No hash for %u %u\n", pgno, npg);
+      }
+    }else{
+      writeByte(p, REPLICA_HASH);
+      writeBytes(p, 20, a);
+      if( p->zDebugFile ){
+        debugMessage(p, "-> REPLICA_HASH %u %u (%08x...)\n",
+        pgno, npg, *(unsigned int*)a);
+      }
+    }
+    p->nHashSent++;
+    iHash = pgno + npg;
+    nHash = npg;
+  }
+  sqlite3_finalize(pStmt);
+  runSql(p, "DELETE FROM sendHash");
+  writeByte(p, REPLICA_READY);
+  fflush(p->pOut);
+  p->nRound++;
+  if( p->zDebugFile ) debugMessage(p, "-> REPLICA_READY\n", iHash);
+}
+
+/*
+** Make entries in the sendHash table to send hashes for
+** npg (mnemonic: Number of PaGes) pages starting with fpg
+** (mnemonic: First PaGe).
+*/
+static void subdivideHashRange(
+  SQLiteRsync *p,       /* The replica-side of the sync */
+  unsigned int fpg,     /* First page of the range */
+  unsigned int npg      /* Number of pages */
+){
+  unsigned int nChunk;   /* How many pages to request per hash */
+  sqlite3_uint64 iEnd;   /* One more than the last page */
+  if( npg<=30 ){
+    nChunk = 1;
+  }else if( npg<=1000 ){
+    nChunk = 30;
+  }else{
+    nChunk = 1000;
+  }
+  iEnd = fpg;
+  iEnd += npg;
+  runSql(p,
+    "WITH RECURSIVE c(n) AS"
+    "  (VALUES(%u) UNION ALL SELECT n+%u FROM c WHERE n<%llu)"
+    "REPLACE INTO sendHash(fpg,npg)"
+    " SELECT n, min(%llu-n,%u) FROM c",
+    fpg, nChunk, iEnd-nChunk, iEnd, nChunk
+  );
+}
+
+/*
+** Run the replica-side protocol.  The protocol is passive in the sense
+** that it only response to message from the origin side.
+**
+**    ORIGIN_BEGIN  idProtocol szPage nPage
+**
+**         The origin is reporting the protocol version number, the size of
+**         each page in the origin database (sent as a single-byte power-of-2),
+**         and the number of pages in the origin database.
+**         This procedure checks compatibility, and if everything is ok,
+**         it starts sending hashes back to the origin using REPLICA_HASH
+**         and/or REPLICA_CONFIG message, followed by a single REPLICA_READY.
+**         REPLICA_CONFIG is only sent if the protocol is 2 or greater.
+**
+**    ORIGIN_ERROR  size  text
+**
+**         Report an error and quit.
+**
+**    ORIGIN_DETAIL  pgno  cnt
+**
+**         The origin reports that a multi-page hash starting at pgno and
+**         spanning cnt pages failed to match.  The origin is requesting
+**         details (more REPLICA_HASH message with a smaller cnt).  The
+**         replica must wait on ORIGIN_READY before sending its reply.
+**
+**    ORIGIN_READY
+**
+**         After sending one or more ORIGIN_DETAIL messages, the ORIGIN_READY
+**         is sent by the origin to indicate that it has finished sending
+**         requests for detail and is ready for the replicate to reply
+**         with a new round of REPLICA_CONFIG and REPLICA_HASH messages.
+**
+**    ORIGIN_PAGE  pgno  content
+**
+**         Once the origin believes it knows exactly which pages need to be
+**         updated in the replica, it starts sending those pages using these
+**         messages.  These messages will only appear immediately after
+**         REPLICA_READY.  The origin never mixes ORIGIN_DETAIL and
+**         ORIGIN_PAGE messages in the same batch.
+**
+**    ORIGIN_TXN   pgno
+**
+**         Close the update transaction.  The total database size is pgno
+**         pages.
+**
+**    ORIGIN_END
+**
+**         Expect no more transmissions from the origin.
+*/
+static void replicaSide(SQLiteRsync *p){
+  int c;
+  sqlite3_stmt *pIns = 0;
+  unsigned int szOPage = 0;
+  char eJMode = 0;               /* Journal mode prior to sync */
+  char buf[65536];
+
+  p->isReplica = 1;
+  if( p->bCommCheck ){
+    infoMsg(p, "replica zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d",
+               p->zOrigin, p->zReplica, p->isRemote, p->iProtocol);
+    writeByte(p, REPLICA_END);
+    fflush(p->pOut);
+  }
+  if( p->iProtocol<=0 ) p->iProtocol = PROTOCOL_VERSION;
+
+  /* Respond to message from the origin.  The origin will initiate the
+  ** the conversation with an ORIGIN_BEGIN message.
+  */
+  while( p->nErr<=p->nWrErr && (c = readByte(p))!=EOF && c!=ORIGIN_END ){
+    switch( c ){
+      case ORIGIN_MSG:
+      case ORIGIN_ERROR: {
+        readAndDisplayMessage(p, c);
+        break;
+      }
+      case ORIGIN_BEGIN: {
+        unsigned int nOPage = 0;
+        unsigned int nRPage = 0, szRPage = 0;
+        int rc = 0;
+        u8 iProtocol;
+
+        closeDb(p);
+        iProtocol = readByte(p);
+        szOPage = readPow2(p);
+        readUint32(p, &nOPage);
+        if( p->zDebugFile ){
+          debugMessage(p, "<- ORIGIN_BEGIN %d %d %u\n", iProtocol, szOPage,
+                       nOPage);
+        }
+        if( p->nErr ) break;
+        if( iProtocol>p->iProtocol ){
+          /* If the protocol version on the origin side is larger, send back
+          ** a REPLICA_BEGIN message with the protocol version number of the
+          ** replica side.  This gives the origin an opportunity to resend
+          ** a new ORIGIN_BEGIN with a reduced protocol version. */
+          writeByte(p, REPLICA_BEGIN);
+          writeByte(p, p->iProtocol);
+          fflush(p->pOut);
+          if( p->zDebugFile ){
+            debugMessage(p, "-> REPLICA_BEGIN %u\n", p->iProtocol);
+          }
+          break;
+        }
+        p->iProtocol = iProtocol;
+        p->nPage = nOPage;
+        p->szPage = szOPage;
+        rc = sqlite3_open(":memory:", &p->db);
+        if( rc ){
+          reportError(p, "cannot open in-memory database: %s",
+                      sqlite3_errmsg(p->db));
+          closeDb(p);
+          break;
+        }
+        sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0);
+        runSql(p, "ATTACH %Q AS 'replica'", p->zReplica);
+        if( p->wrongEncoding ){
+          p->wrongEncoding = 0;
+          runSql(p, "PRAGMA encoding=utf16le");
+          runSql(p, "ATTACH %Q AS 'replica'", p->zReplica);
+          if( p->wrongEncoding ){
+            p->wrongEncoding = 0;
+            runSql(p, "PRAGMA encoding=utf16be");
+            runSql(p, "Attach %Q AS 'replica'", p->zReplica);
+          }
+        }
+        if( p->nErr ){
+          closeDb(p);
+          break;
+        }
+        runSql(p,
+          "CREATE TABLE sendHash("
+          "  fpg INTEGER PRIMARY KEY,"   /* The page number of hash to send */
+          "  npg INT"                    /* Number of pages in this hash */
+          ")"
+        );
+        hashRegister(p->db);
+        if( runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count") ){
+          break;
+        }
+        if( nRPage==0 ){
+          runSql(p, "PRAGMA replica.page_size=%u", szOPage);
+          runSql(p, "SELECT * FROM replica.sqlite_schema");
+        }
+        runSql(p, "BEGIN IMMEDIATE");
+        runSqlReturnText(p, buf, "PRAGMA replica.journal_mode");
+        if( strcmp(buf, "wal")!=0 ){
+          if( p->bWalOnly && nRPage>0 ){
+            reportError(p, "replica is not in WAL mode");
+            break;
+          }
+          eJMode = 1;      /* Non-WAL mode prior to sync */
+        }else{
+          eJMode = 2;      /* WAL-mode prior to sync */
+        }
+        runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count");
+        runSqlReturnUInt(p, &szRPage, "PRAGMA replica.page_size");
+        if( szRPage!=szOPage ){
+          reportError(p, "page size mismatch; origin is %d bytes and "
+                         "replica is %d bytes", szOPage, szRPage);
+          break;
+        }
+        if( p->iProtocol<2 || nRPage<=100 ){
+          runSql(p,
+            "WITH RECURSIVE c(n) AS"
+              "(VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<%d)"
+            "INSERT INTO sendHash(fpg, npg) SELECT n, 1 FROM c",
+            nRPage);
+        }else{
+          runSql(p,"INSERT INTO sendHash VALUES(1,1)");
+          subdivideHashRange(p, 2, nRPage-1);
+        }
+        sendHashMessages(p, 1, 1);
+        runSql(p, "PRAGMA writable_schema=ON");
+        break;
+      }
+      case ORIGIN_DETAIL: {
+        unsigned int fpg, npg;
+        readUint32(p, &fpg);
+        readUint32(p, &npg);
+        if( p->zDebugFile ){
+          debugMessage(p, "<- ORIGIN_DETAIL %u %u\n", fpg, npg);
+        }
+        subdivideHashRange(p, fpg, npg);
+        break;
+      }
+      case ORIGIN_READY: {
+        if( p->zDebugFile ){
+          debugMessage(p, "<- ORIGIN_READY\n");
+        }
+        sendHashMessages(p, 0, 0);
+        break;
+      }
+      case ORIGIN_TXN: {
+        unsigned int nOPage = 0;
+        readUint32(p, &nOPage);
+        if( p->zDebugFile ){
+          debugMessage(p, "<- ORIGIN_TXN %u\n", nOPage);
+        }
+        if( pIns==0 ){
+          /* Nothing has changed */
+          runSql(p, "COMMIT");
+        }else if( p->nErr ){
+          runSql(p, "ROLLBACK");
+        }else{
+          if( nOPage<0xffffffff ){
+            int rc;
+            sqlite3_bind_int64(pIns, 1, nOPage+1);
+            sqlite3_bind_null(pIns, 2);
+            rc = sqlite3_step(pIns);
+            if( rc!=SQLITE_DONE ){
+              reportError(p,
+                  "SQL statement [%s] failed (pgno=%u, data=NULL): %s",
+                  sqlite3_sql(pIns), nOPage, sqlite3_errmsg(p->db));
+            }
+            sqlite3_reset(pIns);
+          }
+          p->nPage = nOPage;
+          runSql(p, "COMMIT");
+        }
+        break;
+      }
+      case ORIGIN_PAGE: {
+        unsigned int pgno = 0;
+        int rc;
+        readUint32(p, &pgno);
+        if( p->zDebugFile ){
+          debugMessage(p, "<- ORIGIN_PAGE %u\n", pgno);
+        }
+        if( p->nErr ) break;
+        if( pIns==0 ){
+          pIns = prepareStmt(p,
+            "INSERT INTO sqlite_dbpage(pgno,data,schema)VALUES(?1,?2,'replica')"
+          );
+          if( pIns==0 ) break;
+        }
+        readBytes(p, szOPage, buf);
+        if( p->nErr ) break;
+        if( pgno==1 &&  eJMode==2 && buf[18]==1 ){
+          /* Do not switch the replica out of WAL mode if it started in 
+          ** WAL mode */
+          buf[18] = buf[19] = 2;
+        }
+        p->nPageSent++;
+        sqlite3_bind_int64(pIns, 1, pgno);
+        sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC);
+        rc = sqlite3_step(pIns);
+        if( rc!=SQLITE_DONE ){
+          reportError(p, "SQL statement [%s] failed (pgno=%u): %s",
+                 sqlite3_sql(pIns), pgno, sqlite3_errmsg(p->db));
+        }
+        sqlite3_reset(pIns);
+        break;
+      }
+      default: {
+        reportError(p, "Unknown message 0x%02x %lld bytes into conversation",
+                       c, p->nIn);
+        break;
+      }
+    }
+  }
+
+  if( pIns ) sqlite3_finalize(pIns);
+  closeDb(p);
+}
+
+/*
+** The argument might be -vvv...vv with any number of "v"s.  Return
+** the number of "v"s.  Return 0 if the argument is not a -vvv...v.
+*/
+static int numVs(const char *z){
+  int n = 0;
+  if( z[0]!='-' ) return 0;
+  z++;
+  if( z[0]=='-' ) z++;
+  while( z[0]=='v' ){ n++; z++; }
+  if( z[0]==0 ) return n;
+  return 0;
+}
+
+/*
+** Get the argument to an --option.  Throw an error and die if no argument
+** is available.
+*/
+static const char *cmdline_option_value(int argc, const char * const*argv,
+                                        int i){
+  if( i==argc ){
+    fprintf(stderr,"%s: Error: missing argument to %s\n",
+            argv[0], argv[argc-1]);
+    exit(1);
+  }
+  return argv[i];
+}
+
+/*
+** Return the current time in milliseconds since the Julian epoch.
+*/
+sqlite3_int64 currentTime(void){
+  sqlite3_int64 now = 0;
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
+  if( pVfs && pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64!=0 ){
+    pVfs->xCurrentTimeInt64(pVfs, &now);
+  }
+  return now;
+}
+
+/*
+** Input string zIn might be in any of these formats:
+**
+**    (1) PATH
+**    (2) HOST:PATH
+**    (3) USER@HOST:PATH
+**
+** For format 1, return NULL.  For formats 2 and 3, return
+** a pointer to the ':' character that separates the hostname
+** from the path.
+*/
+static char *hostSeparator(const char *zIn){
+  char *zPath = strchr(zIn, ':');
+  if( zPath==0 ) return 0;
+#ifdef _WIN32
+  if( isalpha(zIn[0]) && zIn[1]==':' && (zIn[2]=='/' || zIn[2]=='\\') ){
+    return 0;
+  }
+#endif
+  while( zIn<zPath ){
+    if( zIn[0]=='/' ) return 0;
+    if( zIn[0]=='\\' ) return 0;
+    zIn++;
+  }
+  return zPath;
+}
+
+
+/*
+** Parse command-line arguments.  Dispatch subroutines to do the
+** requested work.
+**
+** Input formats:
+**
+**  (1)    sqlite3_rsync  FILENAME1  USER@HOST:FILENAME2
+**
+**  (2)    sqlite3_rsync  USER@HOST:FILENAME1  FILENAME2
+**
+**  (3)    sqlite3_rsync --origin FILENAME1
+**
+**  (4)    sqlite3_rsync --replica FILENAME2
+**
+** The user types (1) or (2).  SSH launches (3) or (4).
+**
+** If (1) is seen then popen2 is used launch (4) on the remote and
+** originSide() is called locally.
+**
+** If (2) is seen, then popen2() is used to launch (3) on the remote
+** and replicaSide() is run locally.
+**
+** If (3) is seen, call originSide() on stdin and stdout.
+**
+q** If (4) is seen, call replicaSide() on stdin and stdout.
+*/
+int main(int argc, char const * const *argv){
+  int isOrigin = 0;
+  int isReplica = 0;
+  int i;
+  SQLiteRsync ctx;
+  char *zDiv;
+  FILE *pIn = 0;
+  FILE *pOut = 0;
+  int childPid = 0;
+  const char *zSsh = "ssh";
+  const char *zExe = "sqlite3_rsync";
+  char *zCmd = 0;
+  sqlite3_int64 tmStart;
+  sqlite3_int64 tmEnd;
+  sqlite3_int64 tmElapse;
+  const char *zRemoteErrFile = 0;
+  const char *zRemoteDebugFile = 0;
+
+#define cli_opt_val cmdline_option_value(argc, argv, ++i)
+  memset(&ctx, 0, sizeof(ctx));
+  ctx.iProtocol = PROTOCOL_VERSION;
+  sqlite3_initialize();
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++;
+    if( strcmp(z,"-origin")==0 ){
+      isOrigin = 1;
+      continue;
+    }
+    if( strcmp(z,"-replica")==0 ){
+      isReplica = 1;
+      continue;
+    }
+    if( numVs(z) ){
+      ctx.eVerbose += numVs(z);
+      continue;
+    }
+    if( strcmp(z, "-protocol")==0 ){
+      ctx.iProtocol = atoi(cli_opt_val);
+      if( ctx.iProtocol<1 ){
+        ctx.iProtocol = 1;
+      }else if( ctx.iProtocol>PROTOCOL_VERSION ){
+        ctx.iProtocol = PROTOCOL_VERSION;
+      }
+      continue;
+    }
+    if( strcmp(z, "-ssh")==0 ){
+      zSsh = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "-exe")==0 ){
+      zExe = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "-wal-only")==0 ){
+      ctx.bWalOnly = 1;
+      continue;
+    }
+    if( strcmp(z, "-version")==0 ){
+      printf("%s\n", sqlite3_sourceid());
+      return 0;
+    }
+    if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0
+     || strcmp(z, "-?")==0
+    ){
+      printf("%s", zUsage);
+      return 0;
+    }
+    if( strcmp(z, "-logfile")==0 ){
+      /* DEBUG OPTION:  --logfile FILENAME
+      ** Cause all local output traffic to be duplicated in FILENAME */
+      const char *zLog = cli_opt_val;
+      if( ctx.pLog ) fclose(ctx.pLog);
+      ctx.pLog = fopen(zLog, "wb");
+      if( ctx.pLog==0 ){
+        fprintf(stderr, "cannot open \"%s\" for writing\n", argv[i]);
+        return 1;
+      }
+      continue;
+    }
+    if( strcmp(z, "-errorfile")==0 ){
+      /* DEBUG OPTION:  --errorfile FILENAME
+      ** Error messages on the local side are written into FILENAME */
+      ctx.zErrFile = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "-remote-errorfile")==0 ){
+      /* DEBUG OPTION:  --remote-errorfile FILENAME
+      ** Error messages on the remote side are written into FILENAME on
+      ** the remote side. */
+      zRemoteErrFile = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "-debugfile")==0 ){
+      /* DEBUG OPTION:  --debugfile FILENAME
+      ** Debugging messages on the local side are written into FILENAME */
+      ctx.zDebugFile = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z, "-remote-debugfile")==0 ){
+      /* DEBUG OPTION:  --remote-debugfile FILENAME
+      ** Error messages on the remote side are written into FILENAME on
+      ** the remote side. */
+      zRemoteDebugFile = cli_opt_val;
+      continue;
+    }
+    if( strcmp(z,"-commcheck")==0 ){  /* DEBUG ONLY */
+      /* Run a communication check with the remote side.  Do not attempt
+      ** to exchange any database connection */
+      ctx.bCommCheck = 1;
+      continue;
+    }
+    if( strcmp(z,"-arg-escape-check")==0 ){  /* DEBUG ONLY */
+      /* Test the append_escaped_arg() routine by using it to render a
+      ** copy of the input command-line, assuming all arguments except
+      ** this one are filenames. */
+      sqlite3_str *pStr = sqlite3_str_new(0);
+      int k;
+      for(k=0; k<argc; k++){
+        append_escaped_arg(pStr, argv[k], i!=k);
+      }
+      printf("%s\n", sqlite3_str_value(pStr));
+      return 0;
+    }
+    if( z[i]=='-' ){
+      fprintf(stderr,
+         "unknown option: \"%s\". Use --help for more detail.\n", z);
+      return 1;
+    }
+    if( ctx.zOrigin==0 ){
+      ctx.zOrigin = z;
+    }else if( ctx.zReplica==0 ){
+      ctx.zReplica = z;
+    }else{
+      fprintf(stderr, "Unknown argument: \"%s\"\n", z);
+      return 1;
+    }
+  }
+  if( ctx.zOrigin==0 ){
+    fprintf(stderr, "missing ORIGIN database filename\n");
+    return 1;
+  }
+  if( ctx.zReplica==0 ){
+    fprintf(stderr, "missing REPLICA database filename\n");
+    return 1;
+  }
+  if( isOrigin && isReplica ){
+    fprintf(stderr, "bad option combination\n");
+    return 1;
+  }
+  if( isOrigin ){
+    ctx.pIn = stdin;
+    ctx.pOut = stdout;
+    ctx.isRemote = 1;
+#ifdef _WIN32
+    _setmode(_fileno(ctx.pIn), _O_BINARY);
+    _setmode(_fileno(ctx.pOut), _O_BINARY);
+#endif
+    originSide(&ctx);
+    return 0;
+  }
+  if( isReplica ){
+    ctx.pIn = stdin;
+    ctx.pOut = stdout;
+    ctx.isRemote = 1;
+#ifdef _WIN32
+    _setmode(_fileno(ctx.pIn), _O_BINARY);
+    _setmode(_fileno(ctx.pOut), _O_BINARY);
+#endif
+    replicaSide(&ctx);
+    return 0;
+  }
+  if( ctx.zReplica==0 ){
+    fprintf(stderr, "missing REPLICA database filename\n");
+    return 1;
+  }
+  tmStart = currentTime();
+  zDiv = hostSeparator(ctx.zOrigin);
+  if( zDiv ){
+    int iRetry;
+    if( hostSeparator(ctx.zReplica)!=0 ){
+      fprintf(stderr,
+         "At least one of ORIGIN and REPLICA must be a local database\n"
+         "You provided two remote databases.\n");
+      return 1;
+    }
+    *(zDiv++) = 0;
+    /* Remote ORIGIN and local REPLICA */
+    for(iRetry=0; 1 /*exit-via-break*/; iRetry++){
+      sqlite3_str *pStr = sqlite3_str_new(0);
+      append_escaped_arg(pStr, zSsh, 1);
+      sqlite3_str_appendf(pStr, " -e none");
+      append_escaped_arg(pStr, ctx.zOrigin, 0);
+      if( iRetry ) add_path_argument(pStr);
+      append_escaped_arg(pStr, zExe, 1);
+      append_escaped_arg(pStr, "--origin", 0);
+      if( ctx.bCommCheck ){
+        append_escaped_arg(pStr, "--commcheck", 0);
+        if( ctx.eVerbose==0 ) ctx.eVerbose = 1;
+      }
+      if( zRemoteErrFile ){
+        append_escaped_arg(pStr, "--errorfile", 0);
+        append_escaped_arg(pStr, zRemoteErrFile, 1);
+      }
+      if( zRemoteDebugFile ){
+        append_escaped_arg(pStr, "--debugfile", 0);
+        append_escaped_arg(pStr, zRemoteDebugFile, 1);
+      }
+      if( ctx.bWalOnly ){
+        append_escaped_arg(pStr, "--wal-only", 0);
+      }
+      append_escaped_arg(pStr, zDiv, 1);
+      append_escaped_arg(pStr, file_tail(ctx.zReplica), 1);
+      if( ctx.eVerbose<2 && iRetry==0 ){
+        append_escaped_arg(pStr, "2>/dev/null", 0);
+      }
+      zCmd = sqlite3_str_finish(pStr);
+      if( ctx.eVerbose>=2 ) printf("%s\n", zCmd);
+      if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){
+        if( iRetry>=1 ){
+          fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd);
+          return 1;
+        }
+        if( ctx.eVerbose>=2 ){
+          printf("ssh FAILED.  Retry with a PATH= argument...\n");
+        }
+        continue;
+      }
+      replicaSide(&ctx);
+      if( ctx.nHashSent==0 && iRetry==0 ) continue;
+      break;
+    }
+  }else if( (zDiv = hostSeparator(ctx.zReplica))!=0 ){
+    /* Local ORIGIN and remote REPLICA */
+    int iRetry;
+    *(zDiv++) = 0;
+    for(iRetry=0; 1 /*exit-by-break*/; iRetry++){
+      sqlite3_str *pStr = sqlite3_str_new(0);
+      append_escaped_arg(pStr, zSsh, 1);
+      sqlite3_str_appendf(pStr, " -e none");
+      append_escaped_arg(pStr, ctx.zReplica, 0);
+      if( iRetry==1 ) add_path_argument(pStr);
+      append_escaped_arg(pStr, zExe, 1);
+      append_escaped_arg(pStr, "--replica", 0);
+      if( ctx.bCommCheck ){
+        append_escaped_arg(pStr, "--commcheck", 0);
+        if( ctx.eVerbose==0 ) ctx.eVerbose = 1;
+      }
+      if( zRemoteErrFile ){
+        append_escaped_arg(pStr, "--errorfile", 0);
+        append_escaped_arg(pStr, zRemoteErrFile, 1);
+      }
+      if( zRemoteDebugFile ){
+        append_escaped_arg(pStr, "--debugfile", 0);
+        append_escaped_arg(pStr, zRemoteDebugFile, 1);
+      }
+      if( ctx.bWalOnly ){
+        append_escaped_arg(pStr, "--wal-only", 0);
+      }
+      append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1);
+      append_escaped_arg(pStr, zDiv, 1);
+      if( ctx.eVerbose<2 && iRetry==0 ){
+        append_escaped_arg(pStr, "2>/dev/null", 0);
+      }
+      zCmd = sqlite3_str_finish(pStr);
+      if( ctx.eVerbose>=2 ) printf("%s\n", zCmd);
+      if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){
+        if( iRetry>=1 ){
+          fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd);
+          return 1;
+        }else if( ctx.eVerbose>=2 ){
+          printf("ssh FAILED.  Retry with a PATH= argument...\n");
+        }
+        continue;
+      }
+      originSide(&ctx);
+      if( ctx.nHashSent==0 && iRetry==0 ) continue;
+      break;
+    }
+  }else{
+    /* Local ORIGIN and REPLICA */
+    sqlite3_str *pStr = sqlite3_str_new(0);
+    append_escaped_arg(pStr, argv[0], 1);
+    append_escaped_arg(pStr, "--replica", 0);
+    if( ctx.bCommCheck ){
+      append_escaped_arg(pStr, "--commcheck", 0);
+    }
+    if( zRemoteErrFile ){
+      append_escaped_arg(pStr, "--errorfile", 0);
+      append_escaped_arg(pStr, zRemoteErrFile, 1);
+    }
+    if( zRemoteDebugFile ){
+      append_escaped_arg(pStr, "--debugfile", 0);
+      append_escaped_arg(pStr, zRemoteDebugFile, 1);
+    }
+    append_escaped_arg(pStr, ctx.zOrigin, 1);
+    append_escaped_arg(pStr, ctx.zReplica, 1);
+    zCmd = sqlite3_str_finish(pStr);
+    if( ctx.eVerbose>=2 ) printf("%s\n", zCmd);
+    if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){
+      fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd);
+      return 1;
+    }
+    originSide(&ctx);
+  }
+  pclose2(ctx.pIn, ctx.pOut, childPid);
+  if( ctx.pLog ) fclose(ctx.pLog);
+  tmEnd = currentTime();
+  tmElapse = tmEnd - tmStart;  /* Elapse time in milliseconds */
+  if( ctx.nErr ){
+    printf("Databases were not synced due to errors\n");
+  }
+  if( ctx.eVerbose>=1 ){
+    char *zMsg;
+    sqlite3_int64 szTotal = (sqlite3_int64)ctx.nPage*(sqlite3_int64)ctx.szPage;
+    sqlite3_int64 nIO = ctx.nOut +ctx.nIn;
+    zMsg = sqlite3_mprintf("sent %,lld bytes, received %,lld bytes",
+                           ctx.nOut, ctx.nIn);
+    printf("%s", zMsg);
+    sqlite3_free(zMsg);
+    if( tmElapse>0 ){
+      zMsg = sqlite3_mprintf(", %,.2f bytes/sec",
+                             1000.0*(double)nIO/(double)tmElapse);
+      printf("%s\n", zMsg);
+      sqlite3_free(zMsg);
+    }else{
+      printf("\n");
+    }
+    if( ctx.nErr==0 ){
+      if( nIO<=szTotal && nIO>0 ){
+        zMsg = sqlite3_mprintf("total size %,lld  speedup is %.2f",
+           szTotal, (double)szTotal/(double)nIO);
+      }else{
+        zMsg = sqlite3_mprintf("total size %,lld", szTotal);
+      }
+      printf("%s\n", zMsg);
+      sqlite3_free(zMsg);
+    }
+    if( ctx.eVerbose>=3 ){
+      printf("hashes: %lld  hash-rounds: %d"
+             "  page updates: %d  protocol: %d\n",
+             ctx.nHashSent, ctx.nRound, ctx.nPageSent, ctx.iProtocol);
+    }
+  }
+  sqlite3_free(zCmd);
+  if( pIn!=0 && pOut!=0 ){
+    pclose2(pIn, pOut, childPid);
+  }
+  sqlite3_shutdown();
+  return ctx.nErr;
+}
diff --git a/tool/src-verify.c b/tool/src-verify.c
new file mode 100644
index 0000000000..6dc9f72598
--- /dev/null
+++ b/tool/src-verify.c
@@ -0,0 +1,960 @@
+/*
+** This utility program reads the "manifest" and "manifest.uuid" files
+** in a Fossil-generated source tree (where the repository has the
+** "manifest" setting turned on - this is true for SQLite and Fossil itself)
+** and verifies that the source code files are complete and unaltered by
+** checking the SHA1 and SHA3 hashes of the source files contained in the
+** "manifest" file.
+**
+** On success it prints:  "OK $HASH" where $HASH is the SHA3-256 hash of
+** the check-in for the source tree.  If it finds any discrepencies, it
+** prints "Derived from $HASH with changes to:" followed by a list of files
+** which have been altered.
+**
+** USAGE:
+**
+**     src-verify [-x] [-v] $(ROOT)
+**
+** Where ROOT is the root of the source tree - the directory that contains
+** the "manifest" and "manifest.uuid" files.  Add the "-v" option for
+** some debugging output.  With the -x option, the output is in a format
+** that is intended to be read by a script rather by a human.  The -x output
+** format always has the SHA3 hash of the source check-in on the first line
+** and lists files that have changed on subsequent lines.
+**
+** Additional debugging options:
+**
+**     src-verify --sha1 FILE ...
+**     src-verify --sha3 FILE ...
+**
+** Compute the SHA1 or SHA3-256 hashes for all of the FILEs named
+**
+** COMPILING:
+**
+** This utility is self-contained.  It uses only the standard library.
+** There are no other dependencies.  Just compile it and run it.
+**
+** LIMITATIONS:
+**
+**   *   This utility assumes that the check-in hash uses SHA3-256.
+**       It is ok for individual file hashes to be SHA1, but the
+**       check-in itself must use a SHA3-256 hash.
+*/
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#if !defined(_WIN32)
+# include <unistd.h>
+#else
+# include <io.h>
+# ifndef R_OK
+#   define R_OK 04
+# endif
+# ifndef access
+#   define access(f,m) _access((f),(m))
+# endif
+#endif
+typedef unsigned long long int u64;
+
+/*
+** The SHA1 implementation below is adapted from:
+**
+**  $NetBSD: sha1.c,v 1.6 2009/11/06 20:31:18 joerg Exp $
+**  $OpenBSD: sha1.c,v 1.9 1997/07/23 21:12:32 kstailey Exp $
+**
+** SHA-1 in C
+** By Steve Reid <steve@edmweb.com>
+** 100% Public Domain
+*/
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+void SHA1Transform(unsigned int state[5], const unsigned char buffer[64])
+{
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy context->state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+/*
+ * SHA1Init - Initialize new context
+ */
+static void SHA1Init(SHA1Context *context){
+    /* SHA1 initialization constants */
+    context->state[0] = 0x67452301;
+    context->state[1] = 0xEFCDAB89;
+    context->state[2] = 0x98BADCFE;
+    context->state[3] = 0x10325476;
+    context->state[4] = 0xC3D2E1F0;
+    context->count[0] = context->count[1] = 0;
+}
+
+
+/*
+ * Run your data through this.
+ */
+static void SHA1Update(
+  SHA1Context *context,
+  const unsigned char *data,
+  unsigned int len
+){
+    unsigned int i, j;
+
+    j = context->count[0];
+    if ((context->count[0] += len << 3) < j)
+        context->count[1] += (len>>29)+1;
+    j = (j >> 3) & 63;
+    if ((j + len) > 63) {
+        (void)memcpy(&context->buffer[j], data, (i = 64-j));
+        SHA1Transform(context->state, context->buffer);
+        for ( ; i + 63 < len; i += 64)
+            SHA1Transform(context->state, &data[i]);
+        j = 0;
+    } else {
+        i = 0;
+    }
+    (void)memcpy(&context->buffer[j], &data[i], len - i);
+}
+
+
+/*
+ * Add padding and return the message digest.
+ */
+static void SHA1Final(unsigned char *digest, SHA1Context *context){
+    unsigned int i;
+    unsigned char finalcount[8];
+
+    for (i = 0; i < 8; i++) {
+        finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)]
+         >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+    }
+    SHA1Update(context, (const unsigned char *)"\200", 1);
+    while ((context->count[0] & 504) != 448)
+        SHA1Update(context, (const unsigned char *)"\0", 1);
+    SHA1Update(context, finalcount, 8);  /* Should cause a SHA1Transform() */
+
+    if (digest) {
+        for (i = 0; i < 20; i++)
+            digest[i] = (unsigned char)
+                ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+    }
+}
+
+
+/*
+** Macros to determine whether the machine is big or little endian,
+** and whether or not that determination is run-time or compile-time.
+**
+** For best performance, an attempt is made to guess at the byte-order
+** using C-preprocessor macros.  If that is unsuccessful, or if
+** -DSHA3_BYTEORDER=0 is set, then byte-order is determined
+** at run-time.
+*/
+#ifndef SHA3_BYTEORDER
+# if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
+     defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
+     defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
+     defined(__arm__)
+#   define SHA3_BYTEORDER    1234
+# elif defined(sparc)    || defined(__ppc__)
+#   define SHA3_BYTEORDER    4321
+# else
+#   define SHA3_BYTEORDER 0
+# endif
+#endif
+
+
+/*
+** State structure for a SHA3 hash in progress
+*/
+typedef struct SHA3Context SHA3Context;
+struct SHA3Context {
+  union {
+    u64 s[25];                /* Keccak state. 5x5 lines of 64 bits each */
+    unsigned char x[1600];    /* ... or 1600 bytes */
+  } u;
+  unsigned nRate;        /* Bytes of input accepted per Keccak iteration */
+  unsigned nLoaded;      /* Input bytes loaded into u.x[] so far this cycle */
+  unsigned ixMask;       /* Insert next input into u.x[nLoaded^ixMask]. */
+};
+
+/*
+** A single step of the Keccak mixing function for a 1600-bit state
+*/
+static void KeccakF1600Step(SHA3Context *p){
+  int i;
+  u64 B0, B1, B2, B3, B4;
+  u64 C0, C1, C2, C3, C4;
+  u64 D0, D1, D2, D3, D4;
+  static const u64 RC[] = {
+    0x0000000000000001ULL,  0x0000000000008082ULL,
+    0x800000000000808aULL,  0x8000000080008000ULL,
+    0x000000000000808bULL,  0x0000000080000001ULL,
+    0x8000000080008081ULL,  0x8000000000008009ULL,
+    0x000000000000008aULL,  0x0000000000000088ULL,
+    0x0000000080008009ULL,  0x000000008000000aULL,
+    0x000000008000808bULL,  0x800000000000008bULL,
+    0x8000000000008089ULL,  0x8000000000008003ULL,
+    0x8000000000008002ULL,  0x8000000000000080ULL,
+    0x000000000000800aULL,  0x800000008000000aULL,
+    0x8000000080008081ULL,  0x8000000000008080ULL,
+    0x0000000080000001ULL,  0x8000000080008008ULL
+  };
+# define A00 (p->u.s[0])
+# define A01 (p->u.s[1])
+# define A02 (p->u.s[2])
+# define A03 (p->u.s[3])
+# define A04 (p->u.s[4])
+# define A10 (p->u.s[5])
+# define A11 (p->u.s[6])
+# define A12 (p->u.s[7])
+# define A13 (p->u.s[8])
+# define A14 (p->u.s[9])
+# define A20 (p->u.s[10])
+# define A21 (p->u.s[11])
+# define A22 (p->u.s[12])
+# define A23 (p->u.s[13])
+# define A24 (p->u.s[14])
+# define A30 (p->u.s[15])
+# define A31 (p->u.s[16])
+# define A32 (p->u.s[17])
+# define A33 (p->u.s[18])
+# define A34 (p->u.s[19])
+# define A40 (p->u.s[20])
+# define A41 (p->u.s[21])
+# define A42 (p->u.s[22])
+# define A43 (p->u.s[23])
+# define A44 (p->u.s[24])
+# define ROL64(a,x) ((a<<x)|(a>>(64-x)))
+
+  for(i=0; i<24; i+=4){
+    C0 = A00^A10^A20^A30^A40;
+    C1 = A01^A11^A21^A31^A41;
+    C2 = A02^A12^A22^A32^A42;
+    C3 = A03^A13^A23^A33^A43;
+    C4 = A04^A14^A24^A34^A44;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A11^D1), 44);
+    B2 = ROL64((A22^D2), 43);
+    B3 = ROL64((A33^D3), 21);
+    B4 = ROL64((A44^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i];
+    A11 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A20^D0), 3);
+    B3 = ROL64((A31^D1), 45);
+    B4 = ROL64((A42^D2), 61);
+    B0 = ROL64((A03^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A20 =   B0 ^((~B1)&  B2 );
+    A31 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A40^D0), 18);
+    B0 = ROL64((A01^D1), 1);
+    B1 = ROL64((A12^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A34^D4), 8);
+    A40 =   B0 ^((~B1)&  B2 );
+    A01 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A10^D0), 36);
+    B2 = ROL64((A21^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A43^D3), 56);
+    B0 = ROL64((A04^D4), 27);
+    A10 =   B0 ^((~B1)&  B2 );
+    A21 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A30^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A02^D2), 62);
+    B1 = ROL64((A13^D3), 55);
+    B2 = ROL64((A24^D4), 39);
+    A30 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    C0 = A00^A20^A40^A10^A30;
+    C1 = A11^A31^A01^A21^A41;
+    C2 = A22^A42^A12^A32^A02;
+    C3 = A33^A03^A23^A43^A13;
+    C4 = A44^A14^A34^A04^A24;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A31^D1), 44);
+    B2 = ROL64((A12^D2), 43);
+    B3 = ROL64((A43^D3), 21);
+    B4 = ROL64((A24^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i+1];
+    A31 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A40^D0), 3);
+    B3 = ROL64((A21^D1), 45);
+    B4 = ROL64((A02^D2), 61);
+    B0 = ROL64((A33^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A40 =   B0 ^((~B1)&  B2 );
+    A21 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A30^D0), 18);
+    B0 = ROL64((A11^D1), 1);
+    B1 = ROL64((A42^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A04^D4), 8);
+    A30 =   B0 ^((~B1)&  B2 );
+    A11 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A20^D0), 36);
+    B2 = ROL64((A01^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A13^D3), 56);
+    B0 = ROL64((A44^D4), 27);
+    A20 =   B0 ^((~B1)&  B2 );
+    A01 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A10^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A22^D2), 62);
+    B1 = ROL64((A03^D3), 55);
+    B2 = ROL64((A34^D4), 39);
+    A10 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    C0 = A00^A40^A30^A20^A10;
+    C1 = A31^A21^A11^A01^A41;
+    C2 = A12^A02^A42^A32^A22;
+    C3 = A43^A33^A23^A13^A03;
+    C4 = A24^A14^A04^A44^A34;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A21^D1), 44);
+    B2 = ROL64((A42^D2), 43);
+    B3 = ROL64((A13^D3), 21);
+    B4 = ROL64((A34^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i+2];
+    A21 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A30^D0), 3);
+    B3 = ROL64((A01^D1), 45);
+    B4 = ROL64((A22^D2), 61);
+    B0 = ROL64((A43^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A30 =   B0 ^((~B1)&  B2 );
+    A01 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A10^D0), 18);
+    B0 = ROL64((A31^D1), 1);
+    B1 = ROL64((A02^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A44^D4), 8);
+    A10 =   B0 ^((~B1)&  B2 );
+    A31 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A40^D0), 36);
+    B2 = ROL64((A11^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A03^D3), 56);
+    B0 = ROL64((A24^D4), 27);
+    A40 =   B0 ^((~B1)&  B2 );
+    A11 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A20^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A12^D2), 62);
+    B1 = ROL64((A33^D3), 55);
+    B2 = ROL64((A04^D4), 39);
+    A20 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    C0 = A00^A30^A10^A40^A20;
+    C1 = A21^A01^A31^A11^A41;
+    C2 = A42^A22^A02^A32^A12;
+    C3 = A13^A43^A23^A03^A33;
+    C4 = A34^A14^A44^A24^A04;
+    D0 = C4^ROL64(C1, 1);
+    D1 = C0^ROL64(C2, 1);
+    D2 = C1^ROL64(C3, 1);
+    D3 = C2^ROL64(C4, 1);
+    D4 = C3^ROL64(C0, 1);
+
+    B0 = (A00^D0);
+    B1 = ROL64((A01^D1), 44);
+    B2 = ROL64((A02^D2), 43);
+    B3 = ROL64((A03^D3), 21);
+    B4 = ROL64((A04^D4), 14);
+    A00 =   B0 ^((~B1)&  B2 );
+    A00 ^= RC[i+3];
+    A01 =   B1 ^((~B2)&  B3 );
+    A02 =   B2 ^((~B3)&  B4 );
+    A03 =   B3 ^((~B4)&  B0 );
+    A04 =   B4 ^((~B0)&  B1 );
+
+    B2 = ROL64((A10^D0), 3);
+    B3 = ROL64((A11^D1), 45);
+    B4 = ROL64((A12^D2), 61);
+    B0 = ROL64((A13^D3), 28);
+    B1 = ROL64((A14^D4), 20);
+    A10 =   B0 ^((~B1)&  B2 );
+    A11 =   B1 ^((~B2)&  B3 );
+    A12 =   B2 ^((~B3)&  B4 );
+    A13 =   B3 ^((~B4)&  B0 );
+    A14 =   B4 ^((~B0)&  B1 );
+
+    B4 = ROL64((A20^D0), 18);
+    B0 = ROL64((A21^D1), 1);
+    B1 = ROL64((A22^D2), 6);
+    B2 = ROL64((A23^D3), 25);
+    B3 = ROL64((A24^D4), 8);
+    A20 =   B0 ^((~B1)&  B2 );
+    A21 =   B1 ^((~B2)&  B3 );
+    A22 =   B2 ^((~B3)&  B4 );
+    A23 =   B3 ^((~B4)&  B0 );
+    A24 =   B4 ^((~B0)&  B1 );
+
+    B1 = ROL64((A30^D0), 36);
+    B2 = ROL64((A31^D1), 10);
+    B3 = ROL64((A32^D2), 15);
+    B4 = ROL64((A33^D3), 56);
+    B0 = ROL64((A34^D4), 27);
+    A30 =   B0 ^((~B1)&  B2 );
+    A31 =   B1 ^((~B2)&  B3 );
+    A32 =   B2 ^((~B3)&  B4 );
+    A33 =   B3 ^((~B4)&  B0 );
+    A34 =   B4 ^((~B0)&  B1 );
+
+    B3 = ROL64((A40^D0), 41);
+    B4 = ROL64((A41^D1), 2);
+    B0 = ROL64((A42^D2), 62);
+    B1 = ROL64((A43^D3), 55);
+    B2 = ROL64((A44^D4), 39);
+    A40 =   B0 ^((~B1)&  B2 );
+    A41 =   B1 ^((~B2)&  B3 );
+    A42 =   B2 ^((~B3)&  B4 );
+    A43 =   B3 ^((~B4)&  B0 );
+    A44 =   B4 ^((~B0)&  B1 );
+  }
+}
+
+/*
+** Initialize a new hash.  iSize determines the size of the hash
+** in bits and should be one of 224, 256, 384, or 512.  Or iSize
+** can be zero to use the default hash size of 256 bits.
+*/
+static void SHA3Init(SHA3Context *p, int iSize){
+  memset(p, 0, sizeof(*p));
+  if( iSize>=128 && iSize<=512 ){
+    p->nRate = (1600 - ((iSize + 31)&~31)*2)/8;
+  }else{
+    p->nRate = (1600 - 2*256)/8;
+  }
+#if SHA3_BYTEORDER==1234
+  /* Known to be little-endian at compile-time. No-op */
+#elif SHA3_BYTEORDER==4321
+  p->ixMask = 7;  /* Big-endian */
+#else
+  {
+    static unsigned int one = 1;
+    if( 1==*(unsigned char*)&one ){
+      /* Little endian.  No byte swapping. */
+      p->ixMask = 0;
+    }else{
+      /* Big endian.  Byte swap. */
+      p->ixMask = 7;
+    }
+  }
+#endif
+}
+
+/*
+** Make consecutive calls to the SHA3Update function to add new content
+** to the hash
+*/
+static void SHA3Update(
+  SHA3Context *p,
+  const unsigned char *aData,
+  unsigned int nData
+){
+  unsigned int i = 0;
+#if SHA3_BYTEORDER==1234
+  if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){
+    for(; i+7<nData; i+=8){
+      p->u.s[p->nLoaded/8] ^= *(u64*)&aData[i];
+      p->nLoaded += 8;
+      if( p->nLoaded>=p->nRate ){
+        KeccakF1600Step(p);
+        p->nLoaded = 0;
+      }
+    }
+  }
+#endif
+  for(; i<nData; i++){
+#if SHA3_BYTEORDER==1234
+    p->u.x[p->nLoaded] ^= aData[i];
+#elif SHA3_BYTEORDER==4321
+    p->u.x[p->nLoaded^0x07] ^= aData[i];
+#else
+    p->u.x[p->nLoaded^p->ixMask] ^= aData[i];
+#endif
+    p->nLoaded++;
+    if( p->nLoaded==p->nRate ){
+      KeccakF1600Step(p);
+      p->nLoaded = 0;
+    }
+  }
+}
+
+/*
+** After all content has been added, invoke SHA3Final() to compute
+** the final hash.  The function returns a pointer to the binary
+** hash value.
+*/
+static unsigned char *SHA3Final(SHA3Context *p){
+  unsigned int i;
+  if( p->nLoaded==p->nRate-1 ){
+    const unsigned char c1 = 0x86;
+    SHA3Update(p, &c1, 1);
+  }else{
+    const unsigned char c2 = 0x06;
+    const unsigned char c3 = 0x80;
+    SHA3Update(p, &c2, 1);
+    p->nLoaded = p->nRate - 1;
+    SHA3Update(p, &c3, 1);
+  }
+  for(i=0; i<p->nRate; i++){
+    p->u.x[i+p->nRate] = p->u.x[i^p->ixMask];
+  }
+  return &p->u.x[p->nRate];
+}
+
+/*
+** Convert a digest into base-16.
+*/
+static void DigestToBase16(unsigned char *digest, char *zBuf, int nByte){
+  static const char zEncode[] = "0123456789abcdef";
+  int ix;
+
+  for(ix=0; ix<nByte; ix++){
+    *zBuf++ = zEncode[(*digest>>4)&0xf];
+    *zBuf++ = zEncode[*digest++ & 0xf];
+  }
+  *zBuf = '\0';
+}
+
+/*
+** Compute the SHA3-256 checksum of a file on disk.  Store the resulting
+** checksum in the zCksum.
+**
+** Return the number of errors.
+*/
+void sha3sum_file(const char *zFilename, char *zCksum){
+  FILE *in;
+  SHA3Context ctx;
+  char zBuf[10240];
+
+  in = fopen(zFilename,"rb");
+  if( in==0 ){
+    zCksum[0] = 0;
+    return;
+  }
+  SHA3Init(&ctx, 256);
+  for(;;){
+    size_t n;
+    n = fread(zBuf, 1, sizeof(zBuf), in);
+    if( n<=0 ) break;
+    SHA3Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
+  }
+  fclose(in);
+  DigestToBase16(SHA3Final(&ctx), zCksum, 32);
+}
+
+/*
+** Compute the SHA1 checksum of a file on disk.  Store the resulting
+** checksum in the zCksum.
+**
+** Return the number of errors.
+*/
+void sha1sum_file(const char *zFilename, char *zCksum){
+  FILE *in;
+  SHA1Context ctx;
+  unsigned char zResult[20];
+  char zBuf[10240];
+
+  in = fopen(zFilename,"rb");
+  if( in==0 ){
+    zCksum[0] = 0;
+    return;
+  }
+  SHA1Init(&ctx);
+  for(;;){
+    size_t n;
+    n = fread(zBuf, 1, sizeof(zBuf), in);
+    if( n<=0 ) break;
+    SHA1Update(&ctx, (unsigned char*)zBuf, (unsigned)n);
+  }
+  fclose(in);
+  SHA1Final(zResult, &ctx);
+  DigestToBase16(zResult, zCksum, 20);
+}
+
+/*
+** Decode a fossilized string in-place.
+*/
+void defossilize(char *z){
+  int i, j, cc;
+  char *zSlash = strchr(z, '\\');
+  if( zSlash==0 ) return;
+  i = zSlash - z;
+  for(j=i; (cc=z[i])!=0; i++){
+    if( cc=='\\' && z[i+1] ){
+      i++;
+      switch( z[i] ){
+        case 'n':  cc = '\n';  break;
+        case 's':  cc = ' ';   break;
+        case 't':  cc = '\t';  break;
+        case 'r':  cc = '\r';  break;
+        case 'v':  cc = '\v';  break;
+        case 'f':  cc = '\f';  break;
+        case '0':  cc = 0;     break;
+        case '\\': cc = '\\';  break;
+        default:   cc = z[i];  break;
+      }
+    }
+    z[j++] = cc;
+  }
+  if( z[j] ) z[j] = 0;
+}
+
+/*
+** Report that a single file is incorrect.
+*/
+static void errorMsg(int *pnErr, const char *zVers, const char *zFile){
+  if( *pnErr==0 ){
+    printf("Derived from %.25s with changes to:\n", zVers);
+  }
+  printf("    %s\n", zFile);
+  (*pnErr)++;
+}
+static void errorMsgNH(int *pnErr, const char *zVers, const char *zFile){
+  if( *pnErr==0 ){
+    printf("%s\n", zVers);
+  }
+  printf("%s\n", zFile);
+  (*pnErr)++;
+}
+
+int main(int argc, char **argv){
+  int i, j;
+  int nDir;
+  FILE *in;
+  int bDebug = 0;
+  int bNonHuman = 0;
+  int bSeenManifestErr = 0;
+  int nErr = 0;
+  SHA3Context ctx3;
+  const char *zDir = 0;
+  void (*xErr)(int*,const char*,const char*);
+  char zHash[100];
+  char zCk[100];
+  char zVers[100];
+  char zLine[40000];
+  char zFile[40000];
+  xErr = errorMsg;
+  for(i=1; i<argc; i++){
+    const char *z = argv[i];
+    if( z[0]!='-' ){
+      if( zDir!=0 ){
+        fprintf(stderr, "bad argument: %s\n", z);
+        return 1;
+      }
+      zDir = z;
+      continue;
+    }
+    if( z[1]=='-' && z[2]!=0 ) z++;
+    if( strcmp(argv[1],"-sha1")==0 ){
+      /* For testing purposes, if the first argument is --sha1, then simply
+      ** compute and print the SHA1 checksum of all subsequent arguments. */
+      for(i++; i<argc; i++){
+        sha1sum_file(argv[i], zHash);
+        printf("%s  %s\n", zHash, argv[i]);
+      }
+      return 0;
+    }
+    if( strcmp(argv[1], "-sha3")==0 ){
+      /* For testing purposes, if the first argument is --sha3, then simply
+      ** compute and print the SHA3-256 checksum of all subsequent arguments. */
+      for(i++; i<argc; i++){
+        sha3sum_file(argv[i], zHash);
+        printf("%s  %s\n", zHash, argv[i]);
+      }
+      return 0;
+    }
+    if( strcmp(z,"-v")==0 ){
+      bDebug = 1;
+      continue;
+    }
+    if( strcmp(z,"-x")==0 ){
+      bNonHuman = 1;
+      xErr = errorMsgNH;
+      continue;
+    }
+  usage:
+    fprintf(stderr, "Usage: %s DIRECTORY\n"
+                    "   or: %s --sha1 FILE ...\n"
+                    "   or: %s --sha3 FILE ...\n",
+      argv[0], argv[0], argv[0]);
+    return 1;
+  }
+  if( !zDir ){
+    goto usage;
+  }
+  if( strlen(zDir)>1000 ){
+    fprintf(stderr, "Directory argument too big: [%s]\n", zDir);
+    return 1;
+  }
+  nDir = (int)strlen(zDir);
+  if( nDir<0 ){
+    fprintf(stderr, "Directory argument too short.\n");
+    return 1;
+  }
+  memcpy(zFile, zDir, nDir);
+  if( zFile[nDir-1]!='/' ){
+    zFile[nDir++] = '/';
+  }
+  memcpy(&zFile[nDir], "manifest", 9);
+  if( bDebug ){
+    printf("manifest file: [%s]\n", zFile);
+  }
+  in = fopen(zFile, "rb");
+  if( in==0 ){
+    fprintf(stderr, "missing manifest: \"%s\"\n", zFile);
+    return 1;
+  }
+  SHA3Init(&ctx3, 256);
+  while( fgets(zLine, sizeof(zLine), in) ){
+    if( zLine[0]=='#' ) break;
+    SHA3Update(&ctx3, (unsigned char*)zLine, (int)strlen(zLine));
+  }
+  DigestToBase16(SHA3Final(&ctx3), zVers, 32);
+ 
+  rewind(in);
+  while( fgets(zLine, sizeof(zLine), in) ){
+    if( zLine[0]!='F' ) continue;
+    if( zLine[1]!=' ' ) continue;
+    for(i=2, j=nDir; zLine[i]!=0 && zLine[i]!=' '; i++, j++){
+      if( j<sizeof(zFile) ) zFile[j] = zLine[i];
+    }
+    if( j<sizeof(zFile) ) zFile[j] = 0;
+    zFile[sizeof(zFile)-1] = 0;
+    defossilize(&zFile[nDir]);
+    if( zLine[i]!=' ' ){
+      bSeenManifestErr = 1;
+      continue;
+    }
+    for(i++, j=0; zLine[i]>='0' && zLine[i]<='f'; i++, j++){
+      if( j<sizeof(zHash) ) zHash[j] = zLine[i];
+    }
+    if( j<sizeof(zHash) ) zHash[j] = 0;
+    zHash[sizeof(zHash)-1] = 0;
+    if( bDebug ){
+      printf("%s %s\n", zFile, zHash);
+    }
+    if( access(zFile, R_OK)!=0 ){
+      xErr(&nErr, zVers, &zFile[nDir]);
+      continue;
+    }
+    if( strlen(zHash)==40 ){
+      sha1sum_file(zFile, zCk);
+      if( strcmp(zHash, zCk)!=0 ){
+        xErr(&nErr, zVers, &zFile[nDir]);
+      }
+    }else if( strlen(zHash)==64 ){
+      sha3sum_file(zFile, zCk);
+      if( strcmp(zHash, zCk)!=0 ){
+        xErr(&nErr, zVers, &zFile[nDir]);
+      }
+    }else{
+      bSeenManifestErr = 1;
+      xErr(&nErr, zVers, &zFile[nDir]);
+    }
+  }
+  fclose(in);
+  in = 0;
+  if( bSeenManifestErr ) xErr(&nErr, zVers, "manifest");
+  memcpy(&zFile[nDir], "manifest.uuid", 14);
+  if( access(zFile, R_OK)!=0
+   || (in = fopen(zFile,"rb"))==0
+   || fgets(zLine, sizeof(zLine), in)==0
+   || strlen(zLine)!=65
+   || zLine[64]!='\n'
+   || memcmp(zLine, zVers, 64)!=0
+  ){
+    xErr(&nErr, zVers, &zFile[nDir]);
+  }
+  if( in ) fclose(in); 
+   
+  if( bNonHuman ){
+    if( nErr ) return 0;
+    printf("%s\n", zVers);
+  }else{
+    if( nErr ) return nErr;
+    printf("OK %.25s\n", zVers);
+  }
+  return 0;
+}
diff --git a/tool/srcck1.c b/tool/srcck1.c
index 20084ac47f..5a1158beb9 100644
--- a/tool/srcck1.c
+++ b/tool/srcck1.c
@@ -13,7 +13,7 @@
 ** The aim of this utility is to prevent recurrences of errors such
 ** as the one fixed at:
 **
-**   https://www.sqlite.org/src/info/a2952231ac7abe16
+**   https://sqlite.org/src/info/a2952231ac7abe16
 **
 ** Note that another similar error was found by this utility when it was
 ** first written.  That other error was fixed by the same check-in that
diff --git a/tool/srctree-check.tcl b/tool/srctree-check.tcl
new file mode 100644
index 0000000000..921bb0976e
--- /dev/null
+++ b/tool/srctree-check.tcl
@@ -0,0 +1,47 @@
+#!/usr/bin/tclsh
+#
+# Run this script from the top of the source tree in order to confirm that
+# various aspects of the source tree are up-to-date.  Items checked include:
+#
+#     *    Makefile.msc and autoconf/Makefile.msc agree
+#     *    VERSION agrees with autoconf/tea/configure.ac
+#
+# Other tests might be added later.  
+#
+# Error messages are printed and the process exists non-zero if problems
+# are found.  If everything is ok, no output is generated and the process
+# exits with 0.
+#
+
+# Read an entire file.
+#
+proc readfile {filename} {
+  set fd [open $filename rb]
+  set txt [read $fd]
+  close $fd
+  return $txt
+}
+
+# Find the root of the tree.
+#
+set ROOT [file dir [file dir [file normalize $argv0]]]
+
+# Name of the TCL interpreter
+#
+set TCLSH [info nameofexe]
+
+# Number of errors seen.
+#
+set NERR 0
+
+######################### autoconf/Makefile.msc ###############################
+
+set f1 [readfile $ROOT/autoconf/Makefile.msc]
+exec $TCLSH $ROOT/tool/mkmsvcmin.tcl $ROOT/Makefile.msc tmp1.txt
+set f2 [readfile tmp1.txt]
+file delete tmp1.txt
+if {$f1 != $f2} {
+  puts "ERROR: ./autoconf/Makefile.msc does not agree with ./Makefile.msc"
+  puts "...... Fix: tclsh tool/mkmsvcmin.tcl"
+  incr NERR
+}
diff --git a/tool/stripccomments.c b/tool/stripccomments.c
index 53933c0138..1d85252784 100644
--- a/tool/stripccomments.c
+++ b/tool/stripccomments.c
@@ -111,7 +111,24 @@ void do_it_all(void){
         }
         else if(slash == ch){
           /* MARKER(("state 0 ==> 1 @ %d:%d\n", line, col)); */
-          state = S_SLASH1;
+          if( '\\'==prev ){
+            /**
+               JS regexes may contain slash-asterisks, as happened at:
+
+               https://github.com/emscripten-core/emscripten/issues/23412
+
+               Such regexes will always necessarily be preceded by a
+               backslash, though.
+
+               It is hypothetically possible for a legitimate comment
+               slash-asterisk to appear immediately before a
+               backslash, but that seems like an even rarer corner
+               case than the JS regex case.
+            */
+            fputc(ch, out);
+          }else{
+            state = S_SLASH1;
+          }
           break;
         }
         fputc(ch, out);
diff --git a/tool/tclConfigShToMake.sh b/tool/tclConfigShToMake.sh
new file mode 100755
index 0000000000..89ca15ad5b
--- /dev/null
+++ b/tool/tclConfigShToMake.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# A level of indirection for use soley by main.mk to extract info
+# from tclConfig.sh if it's not provided by the configure script.
+#
+# Expects to be passed a full path to a tclConfig.sh. It sources it
+# and emits TCL code which sets some vars which are exported by
+# tclConfig.sh.
+#
+# This script expects that the caller has already validated that the
+# file exists, is not a directory, and is readable.
+#
+# If passed no filename, or an empty one, then it emits config code
+# suitable for the "config not found" case.
+if test x = "x$1"; then
+  TCL_INCLUDE_SPEC=
+  TCL_LIB_SPEC=
+  TCL_STUB_LIB_SPEC=
+  TCL_EXEC_PREFIX=
+  TCL_VERSION=
+else
+  . "$1"
+fi
+
+cat <<EOF
+TCL_INCLUDE_SPEC = $TCL_INCLUDE_SPEC
+TCL_LIB_SPEC = $TCL_LIB_SPEC
+TCL_STUB_LIB_SPEC = $TCL_STUB_LIB_SPEC
+TCL_EXEC_PREFIX = $TCL_EXEC_PREFIX
+TCL_VERSION = $TCL_VERSION
+EOF
diff --git a/tool/vdbe-compress.tcl b/tool/vdbe-compress.tcl
index 5e63c96a9a..eaafc7a956 100644
--- a/tool/vdbe-compress.tcl
+++ b/tool/vdbe-compress.tcl
@@ -69,7 +69,7 @@ set namechars {abcefghjklmnopqrstuvwxyz}
 set nnc [string length $namechars]
 while {![eof stdin]} {
   set line [gets stdin]
-  if {[regexp "^case (OP_\\w+): \173" $line all operator]} {
+  if {[regexp "^case (OP_\\w+): \\x7B" $line all operator]} {
     append afterUnion $line\n
     set vlist {}
     while {![eof stdin]} {
diff --git a/ext/wasm/version-info.c b/tool/version-info.c
similarity index 89%
rename from ext/wasm/version-info.c
rename to tool/version-info.c
index 62fcd633c8..149bad4f28 100644
--- a/ext/wasm/version-info.c
+++ b/tool/version-info.c
@@ -83,11 +83,19 @@ int main(int argc, char const * const * argv){
     printf("{\"libVersion\": \"%s\", "
            "\"libVersionNumber\": %d, "
            "\"sourceId\": \"%s\","
-           "\"downloadVersion\": %d}"/*missing newline is intentional*/,
+           "\"downloadVersion\": %d,"
+           "\"scm\":{ "
+           "\"sha3-256\": \"%s\","
+           "\"branch\": \"" SQLITE_SCM_BRANCH "\","
+           "\"tags\": \"" SQLITE_SCM_TAGS "\","
+           "\"datetime\": \"" SQLITE_SCM_DATETIME "\""
+           "}"
+           "}"/*missing newline is intentional*/,
            SQLITE_VERSION,
            SQLITE_VERSION_NUMBER,
            SQLITE_SOURCE_ID,
-           dlVersion);
+           dlVersion,
+           SQLITE_SOURCE_ID+20);
   }else{
     if(fQuote) printf("%c", '"');
     if( fVersion ){
diff --git a/tool/warnings.sh b/tool/warnings.sh
index 60d2b42100..3619ce70a3 100644
--- a/tool/warnings.sh
+++ b/tool/warnings.sh
@@ -10,7 +10,7 @@ if uname | grep -i openbsd ; then
   WARNING_ANDROID_OPTS=-Wall
 else
   # Use these for testing on Linux and Mac OSX:
-  WARNING_OPTS="-Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long"
+  WARNING_OPTS="-Wshadow -Wall -Wextra -pedantic"
   gccvers=`gcc -v 2>&1 | grep '^gcc version'`
   if test "$gccvers" '<' 'gcc version 6'
   then
@@ -22,17 +22,23 @@ fi
 
 rm -f sqlite3.c
 make sqlite3.c
-echo '********** No optimizations.  Includes FTS4/5, GEOPOLY, JSON1 ***'
-echo '**********    ' Options: $WARNING_OPTS
-gcc -c $WARNING_OPTS -std=c89 \
-      -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \
+echo '**** No optimizations.  Includes FTS4/5, GEOPOLY, and others ***'
+echo '****' $WARNING_OPTS
+gcc -c $WARNING_OPTS -std=c99 \
+      -DHAVE_STDINT_H \
+      -DSQLITE_ENABLE_FTS4 \
       -DSQLITE_ENABLE_FTS5 \
+      -DSQLITE_ENABLE_GEOPOLY \
+      -DSQLITE_ENABLE_DBSTAT_VTAB \
+      -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+      -DSQLITE_ENABLE_MATH_FUNCTIONS_fixme \
+      -DSQLITE_ENABLE_STMTVTAB \
+      -DSQLITE_ENABLE_DBPAGE_VTAB \
       sqlite3.c
 if test x`uname` = 'xLinux'; then
-echo '********** Android configuration ******************************'
-echo '**********    ' Options: $WARNING_ANDROID_OPTS
+echo '**** Android configuration ******************************'
+echo '****' $WARNING_ANDROID_OPTS
 gcc -c \
-  -DHAVE_USLEEP=1 \
   -DSQLITE_HAVE_ISNAN \
   -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576 \
   -DSQLITE_THREADSAFE=2 \
@@ -53,14 +59,17 @@ gcc -c \
   $WARNING_ANDROID_OPTS \
   -Os sqlite3.c shell.c
 fi
-echo '********** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******'
-echo '**********    ' Options: $WARNING_OPTS
-gcc -c $WARNING_OPTS -std=c89 \
-      -ansi -DSQLITE_ENABLE_STAT4 -DSQLITE_THREADSAFE=0 \
+echo '**** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******'
+echo '****' $WARNING_OPTS
+gcc -c $WARNING_OPTS -std=c99 \
+      -DSQLITE_ENABLE_STAT4 \
+      -DSQLITE_THREADSAFE=0 \
       sqlite3.c
-echo '********** Optimized -O3.  Includes FTS4/5, GEOPOLY, JSON1 ******'
-echo '**********    ' Options: $WARNING_OPTS
-gcc -O3 -c $WARNING_OPTS -std=c89 \
-      -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \
+echo '**** Optimized -O3.  Includes FTS4/5, GEOPOLY, JSON1 ******'
+echo '****' $WARNING_OPTS
+gcc -O3 -c $WARNING_OPTS -std=c99 \
+      -DHAVE_STDINT_H \
+      -DSQLITE_ENABLE_FTS4 \
       -DSQLITE_ENABLE_FTS5 \
+      -DSQLITE_ENABLE_GEOPOLY \
       sqlite3.c
diff --git a/vsixtest/App.xaml b/vsixtest/App.xaml
deleted file mode 100644
index 80889024ca..0000000000
--- a/vsixtest/App.xaml
+++ /dev/null
@@ -1,8 +0,0 @@
-<Application
-    x:Class="vsixtest.App"
-    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-    xmlns:local="using:vsixtest"
-    RequestedTheme="Light">
-
-</Application>
diff --git a/vsixtest/App.xaml.cpp b/vsixtest/App.xaml.cpp
deleted file mode 100644
index c90604a830..0000000000
--- a/vsixtest/App.xaml.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-//
-// App.xaml.cpp
-// Implementation of the App class.
-//
-
-#include "pch.h"
-#include "MainPage.xaml.h"
-
-using namespace vsixtest;
-
-using namespace Platform;
-using namespace Windows::ApplicationModel;
-using namespace Windows::ApplicationModel::Activation;
-using namespace Windows::Foundation;
-using namespace Windows::Foundation::Collections;
-using namespace Windows::UI::Xaml;
-using namespace Windows::UI::Xaml::Controls;
-using namespace Windows::UI::Xaml::Controls::Primitives;
-using namespace Windows::UI::Xaml::Data;
-using namespace Windows::UI::Xaml::Input;
-using namespace Windows::UI::Xaml::Interop;
-using namespace Windows::UI::Xaml::Media;
-using namespace Windows::UI::Xaml::Navigation;
-
-/// <summary>
-/// Initializes the singleton application object.  This is the first line of authored code
-/// executed, and as such is the logical equivalent of main() or WinMain().
-/// </summary>
-App::App()
-{
-	InitializeComponent();
-	Suspending += ref new SuspendingEventHandler(this, &App::OnSuspending);
-}
-
-/// <summary>
-/// Invoked when the application is launched normally by the end user.	Other entry points
-/// will be used such as when the application is launched to open a specific file.
-/// </summary>
-/// <param name="e">Details about the launch request and process.</param>
-void App::OnLaunched(Windows::ApplicationModel::Activation::LaunchActivatedEventArgs^ e)
-{
-
-#if _DEBUG
-		// Show graphics profiling information while debugging.
-		if (IsDebuggerPresent())
-		{
-			// Display the current frame rate counters
-			 DebugSettings->EnableFrameRateCounter = true;
-		}
-#endif
-
-	auto rootFrame = dynamic_cast<Frame^>(Window::Current->Content);
-
-	// Do not repeat app initialization when the Window already has content,
-	// just ensure that the window is active
-	if (rootFrame == nullptr)
-	{
-		// Create a Frame to act as the navigation context and associate it with
-		// a SuspensionManager key
-		rootFrame = ref new Frame();
-
-		rootFrame->NavigationFailed += ref new Windows::UI::Xaml::Navigation::NavigationFailedEventHandler(this, &App::OnNavigationFailed);
-
-		if (e->PreviousExecutionState == ApplicationExecutionState::Terminated)
-		{
-			// TODO: Restore the saved session state only when appropriate, scheduling the
-			// final launch steps after the restore is complete
-
-		}
-
-		if (rootFrame->Content == nullptr)
-		{
-			// When the navigation stack isn't restored navigate to the first page,
-			// configuring the new page by passing required information as a navigation
-			// parameter
-			rootFrame->Navigate(TypeName(MainPage::typeid), e->Arguments);
-		}
-		// Place the frame in the current Window
-		Window::Current->Content = rootFrame;
-		// Ensure the current window is active
-		Window::Current->Activate();
-	}
-	else
-	{
-		if (rootFrame->Content == nullptr)
-		{
-			// When the navigation stack isn't restored navigate to the first page,
-			// configuring the new page by passing required information as a navigation
-			// parameter
-			rootFrame->Navigate(TypeName(MainPage::typeid), e->Arguments);
-		}
-		// Ensure the current window is active
-		Window::Current->Activate();
-	}
-}
-
-/// <summary>
-/// Invoked when application execution is being suspended.	Application state is saved
-/// without knowing whether the application will be terminated or resumed with the contents
-/// of memory still intact.
-/// </summary>
-/// <param name="sender">The source of the suspend request.</param>
-/// <param name="e">Details about the suspend request.</param>
-void App::OnSuspending(Object^ sender, SuspendingEventArgs^ e)
-{
-	(void) sender;	// Unused parameter
-	(void) e;	// Unused parameter
-
-	//TODO: Save application state and stop any background activity
-}
-
-/// <summary>
-/// Invoked when Navigation to a certain page fails
-/// </summary>
-/// <param name="sender">The Frame which failed navigation</param>
-/// <param name="e">Details about the navigation failure</param>
-void App::OnNavigationFailed(Platform::Object ^sender, Windows::UI::Xaml::Navigation::NavigationFailedEventArgs ^e)
-{
-	throw ref new FailureException("Failed to load Page " + e->SourcePageType.Name);
-}
diff --git a/vsixtest/App.xaml.h b/vsixtest/App.xaml.h
deleted file mode 100644
index 5fa8837d38..0000000000
--- a/vsixtest/App.xaml.h
+++ /dev/null
@@ -1,27 +0,0 @@
-//
-// App.xaml.h
-// Declaration of the App class.
-//
-
-#pragma once
-
-#include "App.g.h"
-
-namespace vsixtest
-{
-	/// <summary>
-	/// Provides application-specific behavior to supplement the default Application class.
-	/// </summary>
-	ref class App sealed
-	{
-	protected:
-		virtual void OnLaunched(Windows::ApplicationModel::Activation::LaunchActivatedEventArgs^ e) override;
-
-	internal:
-		App();
-
-	private:
-		void OnSuspending(Platform::Object^ sender, Windows::ApplicationModel::SuspendingEventArgs^ e);
-		void OnNavigationFailed(Platform::Object ^sender, Windows::UI::Xaml::Navigation::NavigationFailedEventArgs ^e);
-	};
-}
diff --git a/vsixtest/Assets/LockScreenLogo.scale-200.png b/vsixtest/Assets/LockScreenLogo.scale-200.png
deleted file mode 100644
index 735f57adb5..0000000000
Binary files a/vsixtest/Assets/LockScreenLogo.scale-200.png and /dev/null differ
diff --git a/vsixtest/Assets/SplashScreen.scale-200.png b/vsixtest/Assets/SplashScreen.scale-200.png
deleted file mode 100644
index 023e7f1fed..0000000000
Binary files a/vsixtest/Assets/SplashScreen.scale-200.png and /dev/null differ
diff --git a/vsixtest/Assets/Square150x150Logo.scale-200.png b/vsixtest/Assets/Square150x150Logo.scale-200.png
deleted file mode 100644
index af49fec1a5..0000000000
Binary files a/vsixtest/Assets/Square150x150Logo.scale-200.png and /dev/null differ
diff --git a/vsixtest/Assets/Square44x44Logo.scale-200.png b/vsixtest/Assets/Square44x44Logo.scale-200.png
deleted file mode 100644
index ce342a2ec8..0000000000
Binary files a/vsixtest/Assets/Square44x44Logo.scale-200.png and /dev/null differ
diff --git a/vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png b/vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png
deleted file mode 100644
index f6c02ce97e..0000000000
Binary files a/vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png and /dev/null differ
diff --git a/vsixtest/Assets/StoreLogo.png b/vsixtest/Assets/StoreLogo.png
deleted file mode 100644
index 7385b56c0e..0000000000
Binary files a/vsixtest/Assets/StoreLogo.png and /dev/null differ
diff --git a/vsixtest/Assets/Wide310x150Logo.scale-200.png b/vsixtest/Assets/Wide310x150Logo.scale-200.png
deleted file mode 100644
index 288995b397..0000000000
Binary files a/vsixtest/Assets/Wide310x150Logo.scale-200.png and /dev/null differ
diff --git a/vsixtest/MainPage.xaml b/vsixtest/MainPage.xaml
deleted file mode 100644
index 7472ad8626..0000000000
--- a/vsixtest/MainPage.xaml
+++ /dev/null
@@ -1,13 +0,0 @@
-<Page
-    x:Class="vsixtest.MainPage"
-    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-    xmlns:local="using:vsixtest"
-    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
-    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
-    mc:Ignorable="d">
-
-    <Grid Background="{ThemeResource ApplicationPageBackgroundThemeBrush}">
-
-    </Grid>
-</Page>
diff --git a/vsixtest/MainPage.xaml.cpp b/vsixtest/MainPage.xaml.cpp
deleted file mode 100644
index e67dcb83b2..0000000000
--- a/vsixtest/MainPage.xaml.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-//
-// MainPage.xaml.cpp
-// Implementation of the MainPage class.
-//
-
-#include "pch.h"
-#include "MainPage.xaml.h"
-#include "sqlite3.h"
-
-using namespace vsixtest;
-
-using namespace Platform;
-using namespace Windows::Foundation;
-using namespace Windows::Foundation::Collections;
-using namespace Windows::UI::Xaml;
-using namespace Windows::UI::Xaml::Controls;
-using namespace Windows::UI::Xaml::Controls::Primitives;
-using namespace Windows::UI::Xaml::Data;
-using namespace Windows::UI::Xaml::Input;
-using namespace Windows::UI::Xaml::Media;
-using namespace Windows::UI::Xaml::Navigation;
-
-// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409
-
-MainPage::MainPage()
-{
-	InitializeComponent();
-	UseSQLite();
-}
-
-void MainPage::UseSQLite(void)
-{
-    int rc = SQLITE_OK;
-    sqlite3 *pDb = nullptr;
-
-    rc = sqlite3_open_v2("test.db", &pDb,
-	SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
-
-    if (rc != SQLITE_OK)
-	throw ref new FailureException("Failed to open database.");
-
-    rc = sqlite3_exec(pDb, "VACUUM;", nullptr, nullptr, nullptr);
-
-    if (rc != SQLITE_OK)
-	throw ref new FailureException("Failed to vacuum database.");
-
-    rc = sqlite3_close(pDb);
-
-    if (rc != SQLITE_OK)
-	throw ref new FailureException("Failed to close database.");
-
-    pDb = nullptr;
-}
diff --git a/vsixtest/MainPage.xaml.h b/vsixtest/MainPage.xaml.h
deleted file mode 100644
index ea327a3e4c..0000000000
--- a/vsixtest/MainPage.xaml.h
+++ /dev/null
@@ -1,22 +0,0 @@
-//
-// MainPage.xaml.h
-// Declaration of the MainPage class.
-//
-
-#pragma once
-
-#include "MainPage.g.h"
-
-namespace vsixtest
-{
-	/// <summary>
-	/// An empty page that can be used on its own or navigated to within a Frame.
-	/// </summary>
-	public ref class MainPage sealed
-	{
-	public:
-		MainPage();
-		void UseSQLite(void);
-
-	};
-}
diff --git a/vsixtest/Package.appxmanifest b/vsixtest/Package.appxmanifest
deleted file mode 100644
index 106b3f1e41..0000000000
--- a/vsixtest/Package.appxmanifest
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<Package
-  xmlns="http://schemas.microsoft.com/appx/manifest/foundation/windows10"
-  xmlns:mp="http://schemas.microsoft.com/appx/2014/phone/manifest"
-  xmlns:uap="http://schemas.microsoft.com/appx/manifest/uap/windows10"
-  IgnorableNamespaces="uap mp">
-
-  <Identity
-    Name="bb52b3e1-5c8a-4516-a5ff-8b9f9baadef7"
-    Publisher="CN=mistachkin"
-    Version="1.0.0.0" />
-
-  <mp:PhoneIdentity PhoneProductId="bb52b3e1-5c8a-4516-a5ff-8b9f9baadef7" PhonePublisherId="00000000-0000-0000-0000-000000000000"/>
-
-  <Properties>
-    <DisplayName>vsixtest</DisplayName>
-    <PublisherDisplayName>mistachkin</PublisherDisplayName>
-    <Logo>Assets\StoreLogo.png</Logo>
-  </Properties>
-
-  <Dependencies>
-    <TargetDeviceFamily Name="Windows.Universal" MinVersion="10.0.0.0" MaxVersionTested="10.0.0.0" />
-  </Dependencies>
-
-  <Resources>
-    <Resource Language="x-generate"/>
-  </Resources>
-
-  <Applications>
-    <Application Id="App"
-      Executable="$targetnametoken$.exe"
-      EntryPoint="vsixtest.App">
-      <uap:VisualElements
-        DisplayName="vsixtest"
-        Square150x150Logo="Assets\Square150x150Logo.png"
-        Square44x44Logo="Assets\Square44x44Logo.png"
-        Description="vsixtest"
-        BackgroundColor="transparent">
-        <uap:DefaultTile Wide310x150Logo="Assets\Wide310x150Logo.png"/>
-        <uap:SplashScreen Image="Assets\SplashScreen.png" />
-      </uap:VisualElements>
-    </Application>
-  </Applications>
-
-  <Capabilities>
-    <Capability Name="internetClient" />
-  </Capabilities>
-</Package>
\ No newline at end of file
diff --git a/vsixtest/pch.cpp b/vsixtest/pch.cpp
deleted file mode 100644
index 97b544ec11..0000000000
--- a/vsixtest/pch.cpp
+++ /dev/null
@@ -1,6 +0,0 @@
-//
-// pch.cpp
-// Include the standard header and generate the precompiled header.
-//
-
-#include "pch.h"
diff --git a/vsixtest/pch.h b/vsixtest/pch.h
deleted file mode 100644
index b793236d02..0000000000
--- a/vsixtest/pch.h
+++ /dev/null
@@ -1,11 +0,0 @@
-//
-// pch.h
-// Header for standard system include files.
-//
-
-#pragma once
-
-#include <collection.h>
-#include <ppltasks.h>
-
-#include "App.xaml.h"
diff --git a/vsixtest/vsixtest.sln b/vsixtest/vsixtest.sln
deleted file mode 100644
index 1ab6e064f9..0000000000
--- a/vsixtest/vsixtest.sln
+++ /dev/null
@@ -1,39 +0,0 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 14
-VisualStudioVersion = 14.0.24720.0
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "vsixtest", "vsixtest.vcxproj", "{60BB14A5-0871-4656-BC38-4F0958230F9A}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|ARM = Debug|ARM
-		Debug|x64 = Debug|x64
-		Debug|x86 = Debug|x86
-		Release|ARM = Release|ARM
-		Release|x64 = Release|x64
-		Release|x86 = Release|x86
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|ARM.ActiveCfg = Debug|ARM
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|ARM.Build.0 = Debug|ARM
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|ARM.Deploy.0 = Debug|ARM
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x64.ActiveCfg = Debug|x64
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x64.Build.0 = Debug|x64
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x64.Deploy.0 = Debug|x64
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x86.ActiveCfg = Debug|Win32
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x86.Build.0 = Debug|Win32
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x86.Deploy.0 = Debug|Win32
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|ARM.ActiveCfg = Release|ARM
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|ARM.Build.0 = Release|ARM
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|ARM.Deploy.0 = Release|ARM
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x64.ActiveCfg = Release|x64
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x64.Build.0 = Release|x64
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x64.Deploy.0 = Release|x64
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x86.ActiveCfg = Release|Win32
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x86.Build.0 = Release|Win32
-		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x86.Deploy.0 = Release|Win32
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-EndGlobal
diff --git a/vsixtest/vsixtest.tcl b/vsixtest/vsixtest.tcl
deleted file mode 100644
index 5dce821dc4..0000000000
--- a/vsixtest/vsixtest.tcl
+++ /dev/null
@@ -1,373 +0,0 @@
-#!/usr/bin/tclsh
-#
-# This script is used to quickly test a VSIX (Visual Studio Extension) file
-# with Visual Studio 2015 on Windows.
-#
-# PREREQUISITES
-#
-# 1. This tool is Windows only.
-#
-# 2. This tool must be executed with "elevated administrator" privileges.
-#
-# 3. Tcl 8.4 and later are supported, earlier versions have not been tested.
-#
-# 4. The "sqlite-UWP-output.vsix" file is assumed to exist in the parent
-#    directory of the directory containing this script.  The [optional] first
-#    command line argument to this script may be used to specify an alternate
-#    file.  However, currently, the file must be compatible with both Visual
-#    Studio 2015 and the Universal Windows Platform.
-#
-# 5. The "VERSION" file is assumed to exist in the parent directory of the
-#    directory containing this script.  It must contain a version number that
-#    matches the VSIX file being tested.
-#
-# 6. The temporary directory specified in the TEMP or TMP environment variables
-#    must refer to an existing directory writable by the current user.
-#
-# 7. The VS140COMNTOOLS environment variable must refer to the Visual Studio
-#    2015 common tools directory.
-#
-# USAGE
-#
-# The first argument to this script is optional.  If specified, it must be the
-# name of the VSIX file to test.
-#
-package require Tcl 8.4
-
-proc fail { {error ""} {usage false} } {
-  if {[string length $error] > 0} then {
-    puts stdout $error
-    if {!$usage} then {exit 1}
-  }
-
-  puts stdout "usage:\
-[file tail [info nameofexecutable]]\
-[file tail [info script]] \[vsixFile\]"
-
-  exit 1
-}
-
-proc isWindows {} {
-  #
-  # NOTE: Returns non-zero only when running on Windows.
-  #
-  return [expr {[info exists ::tcl_platform(platform)] && \
-      $::tcl_platform(platform) eq "windows"}]
-}
-
-proc isAdministrator {} {
-  #
-  # NOTE: Returns non-zero only when running as "elevated administrator".
-  #
-  if {[isWindows]} then {
-    if {[catch {exec -- whoami /groups} groups] == 0} then {
-      set groups [string map [list \r\n \n] $groups]
-
-      foreach group [split $groups \n] {
-        #
-        # NOTE: Match this group line against the "well-known" SID for
-        #       the "Administrators" group on Windows.
-        #
-        if {[regexp -- {\sS-1-5-32-544\s} $group]} then {
-          #
-          # NOTE: Match this group line against the attributes column
-          #       sub-value that should be present when running with
-          #       elevated administrator credentials.
-          #
-          if {[regexp -- {\sEnabled group(?:,|\s)} $group]} then {
-            return true
-          }
-        }
-      }
-    }
-  }
-
-  return false
-}
-
-proc getEnvironmentVariable { name } {
-  #
-  # NOTE: Returns the value of the specified environment variable or an empty
-  #       string for environment variables that do not exist in the current
-  #       process environment.
-  #
-  return [expr {[info exists ::env($name)] ? $::env($name) : ""}]
-}
-
-proc getTemporaryPath {} {
-  #
-  # NOTE: Returns the normalized path to the first temporary directory found
-  #       in the typical set of environment variables used for that purpose
-  #       or an empty string to signal a failure to locate such a directory.
-  #
-  set names [list]
-
-  foreach name [list TEMP TMP] {
-    lappend names [string toupper $name] [string tolower $name] \
-        [string totitle $name]
-  }
-
-  foreach name $names {
-    set value [getEnvironmentVariable $name]
-
-    if {[string length $value] > 0} then {
-      return [file normalize $value]
-    }
-  }
-
-  return ""
-}
-
-proc appendArgs { args } {
-  #
-  # NOTE: Returns all passed arguments joined together as a single string
-  #       with no intervening spaces between arguments.
-  #
-  eval append result $args
-}
-
-proc readFile { fileName } {
-  #
-  # NOTE: Reads and returns the entire contents of the specified file, which
-  #       may contain binary data.
-  #
-  set file_id [open $fileName RDONLY]
-  fconfigure $file_id -encoding binary -translation binary
-  set result [read $file_id]
-  close $file_id
-  return $result
-}
-
-proc writeFile { fileName data } {
-  #
-  # NOTE: Writes the entire contents of the specified file, which may contain
-  #       binary data.
-  #
-  set file_id [open $fileName {WRONLY CREAT TRUNC}]
-  fconfigure $file_id -encoding binary -translation binary
-  puts -nonewline $file_id $data
-  close $file_id
-  return ""
-}
-
-proc putsAndEval { command } {
-  #
-  # NOTE: Outputs a command to the standard output channel and then evaluates
-  #       it in the callers context.
-  #
-  catch {
-    puts stdout [appendArgs "Running: " [lrange $command 1 end] ...\n]
-  }
-
-  return [uplevel 1 $command]
-}
-
-proc isBadDirectory { directory } {
-  #
-  # NOTE: Returns non-zero if the directory is empty, does not exist, -OR- is
-  #       not a directory.
-  #
-  catch {
-    puts stdout [appendArgs "Checking directory \"" $directory \"...\n]
-  }
-
-  return [expr {[string length $directory] == 0 || \
-      ![file exists $directory] || ![file isdirectory $directory]}]
-}
-
-proc isBadFile { fileName } {
-  #
-  # NOTE: Returns non-zero if the file name is empty, does not exist, -OR- is
-  #       not a regular file.
-  #
-  catch {
-    puts stdout [appendArgs "Checking file \"" $fileName \"...\n]
-  }
-
-  return [expr {[string length $fileName] == 0 || \
-      ![file exists $fileName] || ![file isfile $fileName]}]
-}
-
-#
-# NOTE: This is the entry point for this script.
-#
-set script [file normalize [info script]]
-
-if {[string length $script] == 0} then {
-  fail "script file currently being evaluated is unknown" true
-}
-
-if {![isWindows]} then {
-  fail "this tool only works properly on Windows"
-}
-
-if {![isAdministrator]} then {
-  fail "this tool must run with \"elevated administrator\" privileges"
-}
-
-set path [file normalize [file dirname $script]]
-set argc [llength $argv]; if {$argc > 1} then {fail "" true}
-
-if {$argc == 1} then {
-  set vsixFileName [lindex $argv 0]
-} else {
-  set vsixFileName [file join \
-      [file dirname $path] sqlite-UWP-output.vsix]
-}
-
-###############################################################################
-
-if {[isBadFile $vsixFileName]} then {
-  fail [appendArgs \
-      "VSIX file \"" $vsixFileName "\" does not exist"]
-}
-
-set versionFileName [file join [file dirname $path] VERSION]
-
-if {[isBadFile $versionFileName]} then {
-  fail [appendArgs \
-      "Version file \"" $versionFileName "\" does not exist"]
-}
-
-set projectTemplateFileName [file join $path vsixtest.vcxproj.data]
-
-if {[isBadFile $projectTemplateFileName]} then {
-  fail [appendArgs \
-      "Project template file \"" $projectTemplateFileName \
-      "\" does not exist"]
-}
-
-set envVarName VS140COMNTOOLS
-set vsDirectory [getEnvironmentVariable $envVarName]
-
-if {[isBadDirectory $vsDirectory]} then {
-  fail [appendArgs \
-      "Visual Studio 2015 directory \"" $vsDirectory \
-      "\" from environment variable \"" $envVarName \
-      "\" does not exist"]
-}
-
-set vsixInstaller [file join \
-    [file dirname $vsDirectory] IDE VSIXInstaller.exe]
-
-if {[isBadFile $vsixInstaller]} then {
-  fail [appendArgs \
-      "Visual Studio 2015 VSIX installer \"" $vsixInstaller \
-      "\" does not exist"]
-}
-
-set envVarName ProgramFiles
-set programFiles [getEnvironmentVariable $envVarName]
-
-if {[isBadDirectory $programFiles]} then {
-  fail [appendArgs \
-      "Program Files directory \"" $programFiles \
-      "\" from environment variable \"" $envVarName \
-      "\" does not exist"]
-}
-
-set msBuild [file join $programFiles MSBuild 14.0 Bin MSBuild.exe]
-
-if {[isBadFile $msBuild]} then {
-  fail [appendArgs \
-      "MSBuild v14.0 executable file \"" $msBuild \
-      "\" does not exist"]
-}
-
-set temporaryDirectory [getTemporaryPath]
-
-if {[isBadDirectory $temporaryDirectory]} then {
-  fail [appendArgs \
-      "Temporary directory \"" $temporaryDirectory \
-      "\" does not exist"]
-}
-
-###############################################################################
-
-set installLogFileName [appendArgs \
-    [file rootname [file tail $vsixFileName]] \
-    -install- [pid] .log]
-
-set commands(1) [list exec [file nativename $vsixInstaller]]
-
-lappend commands(1) /quiet /norepair
-lappend commands(1) [appendArgs /logFile: $installLogFileName]
-lappend commands(1) [file nativename $vsixFileName]
-
-###############################################################################
-
-set buildLogFileName [appendArgs \
-    [file rootname [file tail $vsixFileName]] \
-    -build-%configuration%-%platform%- [pid] .log]
-
-set commands(2) [list exec [file nativename $msBuild]]
-
-lappend commands(2) [file nativename [file join $path vsixtest.sln]]
-lappend commands(2) /target:Rebuild
-lappend commands(2) /property:Configuration=%configuration%
-lappend commands(2) /property:Platform=%platform%
-
-lappend commands(2) [appendArgs \
-    /logger:FileLogger,Microsoft.Build.Engine\;Logfile= \
-    [file nativename [file join $temporaryDirectory \
-    $buildLogFileName]] \;Verbosity=diagnostic]
-
-###############################################################################
-
-set uninstallLogFileName [appendArgs \
-    [file rootname [file tail $vsixFileName]] \
-    -uninstall- [pid] .log]
-
-set commands(3) [list exec [file nativename $vsixInstaller]]
-
-lappend commands(3) /quiet /norepair
-lappend commands(3) [appendArgs /logFile: $uninstallLogFileName]
-lappend commands(3) [appendArgs /uninstall:SQLite.UWP.2015]
-
-###############################################################################
-
-if {1} then {
-  catch {
-    puts stdout [appendArgs \
-        "Install log: \"" [file nativename [file join \
-        $temporaryDirectory $installLogFileName]] \"\n]
-  }
-
-  catch {
-    puts stdout [appendArgs \
-        "Build logs: \"" [file nativename [file join \
-        $temporaryDirectory $buildLogFileName]] \"\n]
-  }
-
-  catch {
-    puts stdout [appendArgs \
-        "Uninstall log: \"" [file nativename [file join \
-        $temporaryDirectory $uninstallLogFileName]] \"\n]
-  }
-}
-
-###############################################################################
-
-if {1} then {
-  putsAndEval $commands(1)
-
-  set versionNumber [string trim [readFile $versionFileName]]
-  set data [readFile $projectTemplateFileName]
-  set data [string map [list %versionNumber% $versionNumber] $data]
-
-  set projectFileName [file join $path vsixtest.vcxproj]
-  writeFile $projectFileName $data
-
-  set platforms [list x86 x64 ARM]
-  set configurations [list Debug Release]
-
-  foreach platform $platforms {
-    foreach configuration $configurations {
-      putsAndEval [string map [list \
-          %platform% $platform %configuration% $configuration] \
-          $commands(2)]
-    }
-  }
-
-  putsAndEval $commands(3)
-}
diff --git a/vsixtest/vsixtest.vcxproj.data b/vsixtest/vsixtest.vcxproj.data
deleted file mode 100644
index a64584c3a2..0000000000
--- a/vsixtest/vsixtest.vcxproj.data
+++ /dev/null
@@ -1,198 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <PropertyGroup Label="Globals">
-    <ProjectGuid>{60bb14a5-0871-4656-bc38-4f0958230f9a}</ProjectGuid>
-    <RootNamespace>vsixtest</RootNamespace>
-    <DefaultLanguage>en-US</DefaultLanguage>
-    <MinimumVisualStudioVersion>14.0</MinimumVisualStudioVersion>
-    <AppContainerApplication>true</AppContainerApplication>
-    <ApplicationType>Windows Store</ApplicationType>
-    <WindowsTargetPlatformVersion>10.0.10586.0</WindowsTargetPlatformVersion>
-    <WindowsTargetPlatformMinVersion>10.0.10586.0</WindowsTargetPlatformMinVersion>
-    <ApplicationTypeRevision>10.0</ApplicationTypeRevision>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
-  <ItemGroup Label="ProjectConfigurations">
-    <ProjectConfiguration Include="Debug|ARM">
-      <Configuration>Debug</Configuration>
-      <Platform>ARM</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Debug|Win32">
-      <Configuration>Debug</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Debug|x64">
-      <Configuration>Debug</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|ARM">
-      <Configuration>Release</Configuration>
-      <Platform>ARM</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|Win32">
-      <Configuration>Release</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|x64">
-      <Configuration>Release</Configuration>
-      <Platform>x64</Platform>
-    </ProjectConfiguration>
-  </ItemGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>true</UseDebugLibraries>
-    <PlatformToolset>v140</PlatformToolset>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>true</UseDebugLibraries>
-    <PlatformToolset>v140</PlatformToolset>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>true</UseDebugLibraries>
-    <PlatformToolset>v140</PlatformToolset>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>false</UseDebugLibraries>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-    <PlatformToolset>v140</PlatformToolset>
-    <UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>false</UseDebugLibraries>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-    <PlatformToolset>v140</PlatformToolset>
-    <UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <UseDebugLibraries>false</UseDebugLibraries>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-    <PlatformToolset>v140</PlatformToolset>
-    <UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-  <ImportGroup Label="ExtensionSettings">
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props" Condition="exists('$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props')" />
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props" Condition="exists('$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props')" />
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props" Condition="exists('$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props')" />
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props" Condition="exists('$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props')" />
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props" Condition="exists('$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props')" />
-  </ImportGroup>
-  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-    <Import Project="$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props" Condition="exists('$([Microsoft.Build.Utilities.ToolLocationHelper]::GetPlatformExtensionSDKLocation(`SQLite.UWP.2015, Version=%versionNumber%`, $(TargetPlatformIdentifier), $(TargetPlatformVersion),  $(SDKReferenceDirectoryRoot), $(SDKExtensionDirectoryRoot), $(SDKReferenceRegistryRoot)))\DesignTime\CommonConfiguration\Neutral\SQLite.UWP.2015.props')" />
-  </ImportGroup>
-  <PropertyGroup Label="UserMacros" />
-  <PropertyGroup>
-    <PackageCertificateKeyFile>vsixtest_TemporaryKey.pfx</PackageCertificateKeyFile>
-  </PropertyGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
-    <ClCompile>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4453;28204</DisableSpecificWarnings>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
-    <ClCompile>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4453;28204</DisableSpecificWarnings>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <ClCompile>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4453;28204</DisableSpecificWarnings>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <ClCompile>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4453;28204</DisableSpecificWarnings>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <ClCompile>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4453;28204</DisableSpecificWarnings>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
-    <ClCompile>
-      <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions>
-      <DisableSpecificWarnings>4453;28204</DisableSpecificWarnings>
-    </ClCompile>
-  </ItemDefinitionGroup>
-  <ItemGroup>
-    <ClInclude Include="pch.h" />
-    <ClInclude Include="App.xaml.h">
-      <DependentUpon>App.xaml</DependentUpon>
-    </ClInclude>
-    <ClInclude Include="MainPage.xaml.h">
-      <DependentUpon>MainPage.xaml</DependentUpon>
-    </ClInclude>
-  </ItemGroup>
-  <ItemGroup>
-    <ApplicationDefinition Include="App.xaml">
-      <SubType>Designer</SubType>
-    </ApplicationDefinition>
-    <Page Include="MainPage.xaml">
-      <SubType>Designer</SubType>
-    </Page>
-  </ItemGroup>
-  <ItemGroup>
-    <AppxManifest Include="Package.appxmanifest">
-      <SubType>Designer</SubType>
-    </AppxManifest>
-    <None Include="vsixtest_TemporaryKey.pfx" />
-  </ItemGroup>
-  <ItemGroup>
-    <Image Include="Assets\LockScreenLogo.scale-200.png" />
-    <Image Include="Assets\SplashScreen.scale-200.png" />
-    <Image Include="Assets\Square150x150Logo.scale-200.png" />
-    <Image Include="Assets\Square44x44Logo.scale-200.png" />
-    <Image Include="Assets\Square44x44Logo.targetsize-24_altform-unplated.png" />
-    <Image Include="Assets\StoreLogo.png" />
-    <Image Include="Assets\Wide310x150Logo.scale-200.png" />
-  </ItemGroup>
-  <ItemGroup>
-    <ClCompile Include="App.xaml.cpp">
-      <DependentUpon>App.xaml</DependentUpon>
-    </ClCompile>
-    <ClCompile Include="MainPage.xaml.cpp">
-      <DependentUpon>MainPage.xaml</DependentUpon>
-    </ClCompile>
-    <ClCompile Include="pch.cpp">
-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>
-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>
-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">Create</PrecompiledHeader>
-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">Create</PrecompiledHeader>
-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader>
-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>
-    </ClCompile>
-  </ItemGroup>
-  <ItemGroup>
-    <SDKReference Include="SQLite.UWP.2015, Version=%versionNumber%" />
-  </ItemGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-  <ImportGroup Label="ExtensionTargets">
-  </ImportGroup>
-</Project>
\ No newline at end of file
diff --git a/vsixtest/vsixtest.vcxproj.filters b/vsixtest/vsixtest.vcxproj.filters
deleted file mode 100644
index ac1dfca421..0000000000
--- a/vsixtest/vsixtest.vcxproj.filters
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup>
-    <Filter Include="Common">
-      <UniqueIdentifier>60bb14a5-0871-4656-bc38-4f0958230f9a</UniqueIdentifier>
-    </Filter>
-    <Filter Include="Assets">
-      <UniqueIdentifier>e6271362-8f96-476d-907f-4da227b02435</UniqueIdentifier>
-      <Extensions>bmp;fbx;gif;jpg;jpeg;tga;tiff;tif;png</Extensions>
-    </Filter>
-  </ItemGroup>
-  <ItemGroup>
-    <ApplicationDefinition Include="App.xaml" />
-  </ItemGroup>
-  <ItemGroup>
-    <ClCompile Include="App.xaml.cpp" />
-    <ClCompile Include="MainPage.xaml.cpp" />
-    <ClCompile Include="pch.cpp" />
-  </ItemGroup>
-  <ItemGroup>
-    <ClInclude Include="pch.h" />
-    <ClInclude Include="App.xaml.h" />
-    <ClInclude Include="MainPage.xaml.h" />
-  </ItemGroup>
-  <ItemGroup>
-    <Image Include="Assets\LockScreenLogo.scale-200.png">
-      <Filter>Assets</Filter>
-    </Image>
-    <Image Include="Assets\SplashScreen.scale-200.png">
-      <Filter>Assets</Filter>
-    </Image>
-    <Image Include="Assets\Square150x150Logo.scale-200.png">
-      <Filter>Assets</Filter>
-    </Image>
-    <Image Include="Assets\Square44x44Logo.scale-200.png">
-      <Filter>Assets</Filter>
-    </Image>
-    <Image Include="Assets\Square44x44Logo.targetsize-24_altform-unplated.png">
-      <Filter>Assets</Filter>
-    </Image>
-    <Image Include="Assets\StoreLogo.png">
-      <Filter>Assets</Filter>
-    </Image>
-    <Image Include="Assets\Wide310x150Logo.scale-200.png">
-      <Filter>Assets</Filter>
-    </Image>
-  </ItemGroup>
-  <ItemGroup>
-    <AppxManifest Include="Package.appxmanifest" />
-  </ItemGroup>
-  <ItemGroup>
-    <None Include="vsixtest_TemporaryKey.pfx" />
-  </ItemGroup>
-  <ItemGroup>
-    <Page Include="MainPage.xaml" />
-  </ItemGroup>
-</Project>
\ No newline at end of file
diff --git a/vsixtest/vsixtest_TemporaryKey.pfx b/vsixtest/vsixtest_TemporaryKey.pfx
deleted file mode 100644
index e6787bcad0..0000000000
Binary files a/vsixtest/vsixtest_TemporaryKey.pfx and /dev/null differ